fluent-plugin-mongo 0.7.16 → 0.8.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: faea3bdbf1168e203df36000467cfed9b780794e
-  data.tar.gz: d487ede0674b9df9d980e8c1e390b34f295848ee
+  metadata.gz: e5a76dcf9763e4ba6cd2ad37a15b1f5b3145d31e
+  data.tar.gz: 17f1414c1edf96ae7009ad701815b2f120eb6018
 SHA512:
-  metadata.gz: 4d5b3df91a3ec58b00db0ab47fd23adbc6fc88341de31772c463c437c93ef091a72ad530282e626383c565a5352f975cee03406c3ca2b68fae230eed51318553
-  data.tar.gz: 088e23c4a719fd7081e867f8b7baf23fea93f9e326e5ce7223780b8c0e5d86b50dc3322081900dad71073df5ad5bdd20890e2dfa03b37954ace24639d9f01485
+  metadata.gz: 6e45d8d167be9f710635f207edde89c8b2962f7ee16a59f34eb406617c6839ba1021ced2ae508c8450ad36f4e2a8d9475eb53020ecc4984329615105f3a3d87b
+  data.tar.gz: 2bbeb260a265b4fd040577862d8b39b72b7d3045c5e2083f758671a80eb8da1a440f6e2c5b3dd8ec1ecfde9043796a46039a82935c32279ff87f4d7e5c7a8e2f

data/.travis.yml

@@ -10,12 +10,15 @@ gemfile:
  - Gemfile
  - Gemfile.v0.12
 
+services:
+  - mongodb
+
 before_install:
   - gem update bundler
 before_script:
   - git submodule update -i
 
-script: mongod=mongod bundle exec rake test
+script: bundle exec rake test
 
 matrix:
   allow_failures:

data/VERSION

@@ -1 +1 @@
-0.7.16
+0.8.0.rc1

data/bin/mongo-tail

@@ -8,11 +8,11 @@ require 'json'
 require 'mongo'
 
 TailConfig = {
-  :d => 'fluent',
-  :c => 'out_mongo_backup',
-  :h => 'localhost',
-  :p => 27017,
-  :n => 10
+  d: 'fluent',
+  c: 'out_mongo_backup',
+  h: 'localhost',
+  p: 27017,
+  n: 10
 }
 
 OptionParser.new { |opt|
@@ -28,51 +28,58 @@ OptionParser.new { |opt|
   opt.parse!(ARGV)
 }
 
+def get_client_options(conf)
+  client_options = {}
+  client_options[:database] = conf[:d]
+  client_options
+end
+
+def get_collection_options
+  collection_options = {}
+  collection_options[:capped] = true
+  collection_options
+end
+
 def get_capped_collection(conf)
-  db = Mongo::MongoClient.new(conf[:h], conf[:p]).db(conf[:d])
-  if db.collection_names.include?(conf[:c])
-    collection = db.collection(conf[:c])
-    if collection.capped?
-      collection
-    else
-      puts "#{conf[:c]} is not capped. mongo-tail can not tail normal collection."
-    end
+  client_options = get_client_options(conf)
+  collection_options = get_collection_options
+  client = Mongo::Client.new(["#{conf[:h]}:#{conf[:p]}"], client_options)
+  collection = client["#{conf[:c]}", collection_options]
+  if collection.capped?
+    collection
   else
-    puts "#{conf[:c]} not found: server = #{conf[:h]}:#{conf[:p]}"
+    puts "#{conf[:c]} is not capped. mongo-tail can not tail normal collection."
   end
 end
 
-def create_cursor_conf(collection, conf)
+def create_skip_number(collection, conf)
   skip = collection.count - conf[:n]
-  cursor_conf = {}
-  cursor_conf[:skip] = skip if skip > 0
-  cursor_conf
+  skip
 end
 
 def tail_n(collection, conf)
-  collection.find({}, create_cursor_conf(collection, conf)).each { |doc|
+  collection.find().skip(create_skip_number(collection, conf)).each {|doc|
     puts doc.to_json
   }
 end
 
 def tail_forever(collection, conf)
-  cursor_conf = create_cursor_conf(collection, conf)
-  cursor_conf[:tailable] = true
-
-  cursor = Mongo::Cursor.new(collection, cursor_conf)
   loop {
-    # TODO: Check more detail of cursor state if needed
-    cursor = Mongo::Cursor.new(collection, cursor_conf) unless cursor.alive?
+    option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
 
-    if doc = cursor.next_document
-      STDOUT.puts doc.to_json
-      STDOUT.flush
+    documents = collection.find(option)
+    if documents.count >= 1
+      documents.each {|doc|
+        STDOUT.puts doc.to_json
+        STDOUT.flush
+        if id = doc.delete('_id')
+          @last_id = id.to_s
+        end
+      }
     else
       sleep 1
     end
   }
-rescue
-  # ignore Mongo::OperationFailuer at CURSOR_NOT_FOUND
 end
 
 exit unless collection = get_capped_collection(TailConfig)

data/fluent-plugin-mongo.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
-  gem.add_dependency "mongo", "~> 1.9"
+  gem.add_runtime_dependency "mongo", "~> 2.2.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -4,38 +4,56 @@ module Fluent
   class MongoTailInput < Input
     Plugin.register_input('mongo_tail', self)
 
-    require 'fluent/plugin/mongo_util'
-    include MongoUtil
-
-    config_param :database, :string, :default => nil
-    config_param :collection, :string
-    config_param :host, :string, :default => 'localhost'
-    config_param :port, :integer, :default => 27017
-    config_param :wait_time, :integer, :default => 1
-    config_param :url, :string, :default => nil
-
-    config_param :tag, :string, :default => nil
-    config_param :tag_key, :string, :default => nil
-    config_param :time_key, :string, :default => nil
-    config_param :time_format, :string, :default => nil
-    config_param :object_id_keys, :array, :default => nil
-
-    # To store last ObjectID
-    config_param :id_store_file, :string, :default => nil
-    config_param :id_store_collection, :string, :default => nil
-
-    # SSL connection
-    config_param :ssl, :bool, :default => false
-
     unless method_defined?(:log)
       define_method(:log) { $log }
     end
 
+    # Define `router` method of v0.12 to support v0.10 or earlier
+    unless method_defined?(:router)
+      define_method("router") { ::Fluent::Engine }
+    end
+
+    require 'fluent/plugin/mongo_auth'
+    include MongoAuthParams
+    include MongoAuth
+    require 'fluent/plugin/logger_support'
+    include LoggerSupport
+
+    desc "MongoDB database"
+    config_param :database, :string, default: nil
+    desc "MongoDB collection"
+    config_param :collection, :string
+    desc "MongoDB host"
+    config_param :host, :string, default: 'localhost'
+    desc "MongoDB port"
+    config_param :port, :integer, default: 27017
+    desc "Tailing interval"
+    config_param :wait_time, :integer, default: 1
+    desc "MongoDB node URL"
+    config_param :url, :string, default: nil
+
+    desc "Input tag"
+    config_param :tag, :string, default: nil
+    desc "Treat key as tag"
+    config_param :tag_key, :string, default: nil
+    desc "Treat key as time"
+    config_param :time_key, :string, default: nil
+    desc "Time format"
+    config_param :time_format, :string, default: nil
+    config_param :object_id_keys, :array, default: nil
+
+    desc "To store last ObjectID"
+    config_param :id_store_file, :string, default: nil
+
+    desc "SSL connection"
+    config_param :ssl, :bool, default: false
+
     def initialize
       super
       require 'mongo'
       require 'bson'
 
+      @client_options = {}
       @connection_options = {}
     end
 
@@ -54,86 +72,72 @@ module Fluent
         raise ConfigError, "One of 'database' or 'url' must be specified"
       end
 
-      @last_id = get_last_id
+      @last_id = @id_store_file ? get_last_id : nil
       @connection_options[:ssl] = @ssl
 
-      $log.debug "Setup mongo_tail configuration: mode = #{@id_store_file || @id_store_collection ? 'persistent' : 'non-persistent'}, last_id = #{@last_id}"
+      configure_logger(@mongo_log_level)
     end
 
     def start
       super
-      open_id_storage
-      @client = get_capped_collection
+
+      @file = get_id_store_file if @id_store_file
+      @collection = get_collection
+      # Resume tailing from last inserted id.
+      # Because tailable option is obsoleted since mongo driver 2.0.
+      @last_id = get_last_inserted_id if !@id_store_file and get_last_inserted_id
       @thread = Thread.new(&method(:run))
     end
 
     def shutdown
-      save_last_id(@last_id) unless @last_id
-      close_id_storage
+      if @id_store_file
+        save_last_id
+        @file.close
+      end
 
       @stop = true
       @thread.join
-      @client.db.connection.close
+      @client.close
+
       super
     end
 
     def run
       loop {
-        cursor = Mongo::Cursor.new(@client, cursor_conf)
+        option = {}
         begin
           loop {
             return if @stop
-            
-            cursor = Mongo::Cursor.new(@client, cursor_conf) unless cursor.alive?
-            if doc = cursor.next_document
-              process_document(doc)
+
+            option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
+            documents = @collection.find(option)
+            if documents.count >= 1
+              process_documents(documents)
             else
               sleep @wait_time
             end
           }
         rescue
-          # ignore Mongo::OperationFailuer at CURSOR_NOT_FOUND
+          # ignore Exceptions
         end
       }
     end
 
     private
 
-    def get_capped_collection
-      begin
-        db = get_database
-        raise ConfigError, "'#{database_name}.#{@collection}' not found: node = #{node_string}" unless db.collection_names.include?(@collection)
-        collection = db.collection(@collection)
-        raise ConfigError, "'#{database_name}.#{@collection}' is not capped: node = #{node_string}" unless collection.capped?
-        collection
-      rescue Mongo::ConnectionFailure => e
-        log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
-        exit!
-      rescue Mongo::OperationFailure => e
-        log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
-        exit!
-      end
-    end
-    
-    def get_database
-      case
-      when @database
-        authenticate(Mongo::Connection.new(@host, @port, @connection_options).db(@database))
-      when @url
-        parser = Mongo::URIParser.new(@url)
-        parser.connection.db(parser.db_name)
-      end
+    def client
+      @client_options[:database] = @database
+      @client_options[:user] = @user if @user
+      @client_options[:password] = @password if @password
+      Mongo::Client.new(["#{node_string}"], @client_options)
     end
-    
-    def database_name
-      case
-      when @database
-        @database
-      when @url
-        Mongo::URIParser.new(@url).db_name
-      end
+
+    def get_collection
+      @client = client
+      @client = authenticate(@client)
+      @client["#{@collection}"]
     end
-    
+
     def node_string
       case
       when @database
@@ -143,87 +147,67 @@ module Fluent
       end
     end
 
-    def process_document(doc)
-      time = if @time_key
-               t = doc.delete(@time_key)
-               t.nil? ? Engine.now : t.to_i
-             else
-               Engine.now
-             end
-      tag = if @tag_key
-              t = doc.delete(@tag_key)
-              t.nil? ? 'mongo.missing_tag' : t
-            else
-              @tag
-            end
-      if @object_id_keys
-        @object_id_keys.each { |id_key|
-          doc[id_key] = doc[id_key].to_s
-        }
-      end
-
-      if id = doc.delete('_id')
-        @last_id = id.to_s
-        doc['_id_str'] = @last_id
-        save_last_id(@last_id)
-      end
-
-      # Should use MultiEventStream?
-      router.emit(tag, time, doc)
-    end
+    def process_documents(documents)
+      es = MultiEventStream.new
+      documents.each {|doc|
+        time = if @time_key
+                 t = doc.delete(@time_key)
+                 t.nil? ? Engine.now : t.to_i
+               else
+                 Engine.now
+               end
+        tag = if @tag_key
+                t = doc.delete(@tag_key)
+                t.nil? ? 'mongo.missing_tag' : t
+              else
+                @tag
+              end
+        if @object_id_keys
+          @object_id_keys.each {|id_key|
+            doc[id_key] = doc[id_key].to_s
+          }
+        end
 
-    def cursor_conf
-      conf = {}
-      conf[:tailable] = true
-      conf[:selector] = {'_id' => {'$gt' => BSON::ObjectId(@last_id)}} if @last_id
-      conf
+        if id = doc.delete('_id')
+          @last_id = id.to_s
+          doc['_id_str'] = @last_id
+          save_last_id if @id_store_file
+        end
+        es.add(time, doc)
+      }
+      router.emit_stream(tag, es)
     end
 
-    # following methods are used to read/write last_id
-    
-    def open_id_storage
-      if @id_store_file
-        @id_storage = File.open(@id_store_file, 'w')
-        @id_storege.sync
-      end
-      
-      if @id_store_collection
-        @id_storage = get_database.collection(@id_store_collection)
+    def get_last_inserted_id
+      last_inserted_id = nil
+      documents = @collection.find()
+      if documents.count >= 1
+        documents.each {|doc|
+          if id = doc.delete('_id')
+            last_inserted_id = id
+          end
+        }
       end
+      last_inserted_id
     end
-    
-    def close_id_storage
-      if @id_storage.is_a?(File)
-        @id_storage.close
-      end
+
+    def get_id_store_file
+      file = File.open(@id_store_file, 'w')
+      file.sync
+      file
     end
 
     def get_last_id
-      begin
-        if @id_store_file && File.exist?(@id_store_file)
-          return BSON::ObjectId(File.read(@id_store_file)).to_s
-        end
-      
-        if @id_store_collection
-          collection = get_database.collection(@id_store_collection)
-          count = collection.find.count
-          doc = collection.find.skip(count - 1).limit(1).first
-          return doc && doc["last_id"]
-        end
-      rescue
+      if File.exist?(@id_store_file)
+        BSON::ObjectId(File.read(@id_store_file)).to_s rescue nil
+      else
         nil
       end
     end
 
-    def save_last_id(last_id)
-      if @id_storage.is_a?(File)
-        @id_storage.pos = 0
-        @id_storage.write(last_id)
-      end
-      
-      if @id_storage.is_a?(Mongo::Collection)
-        @id_storage.insert("last_id" => last_id)
-      end
+    def save_last_id
+      @file.pos = 0
+      @file.write(@last_id)
     end
   end
 end