fluent-plugin-mongo 0.7.11 → 0.7.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db2edd199b7056e1a35d3b791ac6a393d7cba246
-  data.tar.gz: 13d1f6785aa437885ffb5f38cbfb85aa58e222ba
+  metadata.gz: 306a5a645b491a0c8edb197064336662588aab89
+  data.tar.gz: a80cc84c0e68319cd68a891678a08d4def516d22
 SHA512:
-  metadata.gz: cb7afa69474f59e1c47b173ef7d34bb0b9af922731887039b9180cec9ff5aca5d90294e02dde44b2bfdc5af83f7799e99a0e59ae364570338afa2bcf7e07167f
-  data.tar.gz: 1c09da93b1179ee4dd55a5c7cefcf718e7a04309eab2049ad91bba3aec81570d2edd97cbf30962e9b066d0d58135350e9031730fc0f19fa50eae3a33a12cd1f7
+  metadata.gz: 15dc38b80bca19d1c7d52914d46f4d7688bdcfe2e5355ed0fdbe2b835d802bba2ed8436b9ed0b4565cdb51cc4cbbf9295daedff78a9fb115fe650da37d58e5e4
+  data.tar.gz: 23047da16921b3f30a3bc16e6f1881255010952f2542b921b92608aef6ecc7f73ff85c15ff2a27ab3bf05ad04b41f6cadec006eccb3436ee58c4e4c0057c0c4f

data/.travis.yml

@@ -5,6 +5,8 @@ rvm:
  - 2.2
  - rbx
 
+before_install:
+  - gem update bundler
 before_script:
   - git submodule update -i
 

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.7.12 - 2016/02/09
+
+	* Support saving last_id to mongod instead of local file
+
+
 Release 0.7.11 - 2015/11/26
 
 	* Add secret option to related parameters

data/README.rdoc

@@ -138,9 +138,6 @@ Use _mongo_tail_ type in source.
 
       # Convert 'time'(BSON's time) to fluent time(Unix time).
       time_key time
-
-      # You can store last ObjectId to tail over server's shutdown
-      id_store_file /Users/repeatedly/devel/fluent-plugin-mongo/last_id
     </source>
 
 You can also use _url_ to specify the database to connect.
@@ -154,6 +151,24 @@ You can also use _url_ to specify the database to connect.
 
 This allows the plugin to read data from a replica set.
 
+You can save last ObjectId to tail over server's shutdown to file.
+
+    <source>
+      ...
+
+      id_store_file /Users/repeatedly/devel/fluent-plugin-mongo/last_id
+    </source>
+
+Or Mongo collection can be used to keep last ObjectID.
+
+    <source>
+      ...
+
+      id_store_collection last_id
+    </source>
+
+Make sure the collection is capped. The plugin inserts records but does not remove at all.
+
 = NOTE
 
 == Broken data as a BSON

data/VERSION

@@ -1 +1 @@
-0.7.11
+0.7.12

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -19,6 +19,7 @@ module Fluent
 
     # To store last ObjectID
     config_param :id_store_file, :string, :default => nil
+    config_param :id_store_collection, :string, :default => nil
 
     # SSL connection
     config_param :ssl, :bool, :default => false
@@ -50,24 +51,22 @@ module Fluent
         raise ConfigError, "One of 'database' or 'url' must be specified"
       end
 
-      @last_id = @id_store_file ? get_last_id : nil
+      @last_id = get_last_id
       @connection_options[:ssl] = @ssl
 
-      $log.debug "Setup mongo_tail configuration: mode = #{@id_store_file ? 'persistent' : 'non-persistent'}"
+      $log.debug "Setup mongo_tail configuration: mode = #{@id_store_file || @id_store_collection ? 'persistent' : 'non-persistent'}, last_id = #{@last_id}"
     end
 
     def start
       super
-      @file = get_id_store_file if @id_store_file
+      open_id_storage
       @client = get_capped_collection
       @thread = Thread.new(&method(:run))
     end
 
     def shutdown
-      if @id_store_file
-        save_last_id
-        @file.close
-      end
+      save_last_id(@last_id) unless @last_id
+      close_id_storage
 
       @stop = true
       @thread.join
@@ -157,7 +156,7 @@ module Fluent
       if id = doc.delete('_id')
         @last_id = id.to_s
         doc['_id_str'] = @last_id
-        save_last_id if @id_store_file
+        save_last_id(@last_id)
       end
 
       # Should use MultiEventStream?
@@ -171,25 +170,51 @@ module Fluent
       conf
     end
 
-    # following methods are used when id_store_file is true
-
-    def get_id_store_file
-      file = File.open(@id_store_file, 'w')
-      file.sync
-      file
+    # following methods are used to read/write last_id
+    
+    def open_id_storage
+      if @id_store_file
+        @id_storage = File.open(@id_store_file, 'w')
+        @id_storege.sync
+      end
+      
+      if @id_store_collection
+        @id_storage = get_database.collection(@id_store_collection)
+      end
+    end
+    
+    def close_id_storage
+      if @id_storage.is_a?(File)
+        @id_storage.close
+      end
     end
 
     def get_last_id
-      if File.exist?(@id_store_file)
-        BSON::ObjectId(File.read(@id_store_file)).to_s rescue nil
-      else
+      begin
+        if @id_store_file && File.exist?(@id_store_file)
+          return BSON::ObjectId(File.read(@id_store_file)).to_s
+        end
+      
+        if @id_store_collection
+          collection = get_database.collection(@id_store_collection)
+          count = collection.find.count
+          doc = collection.find.skip(count - 1).limit(1).first
+          return doc && doc["last_id"]
+        end
+      rescue
         nil
       end
     end
 
-    def save_last_id
-      @file.pos = 0
-      @file.write(@last_id)
+    def save_last_id(last_id)
+      if @id_storage.is_a?(File)
+        @id_storage.pos = 0
+        @id_storage.write(last_id)
+      end
+      
+      if @id_storage.is_a?(Mongo::Collection)
+        @id_storage.insert("last_id" => last_id)
+      end
     end
   end
 end

data/test/plugin/in_mongo_tail.rb

@@ -13,6 +13,7 @@ class MongoTailInputTest < Test::Unit::TestCase
     tag_key tag
     time_key time
     id_store_file /tmp/fluent_mongo_last_id
+    id_store_collection test_last_id
   ]
 
   def create_driver(conf = CONFIG)
@@ -28,6 +29,7 @@ class MongoTailInputTest < Test::Unit::TestCase
     assert_equal('tag', d.instance.tag_key)
     assert_equal('time', d.instance.time_key)
     assert_equal('/tmp/fluent_mongo_last_id', d.instance.id_store_file)
+    assert_equal('test_last_id', d.instance.id_store_collection)
   end
   
   def test_url_configration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.7.11
+  version: 0.7.12
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-26 00:00:00.000000000 Z
+date: 2016-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: MongoDB plugin for Fluentd