fluent-plugin-couch 0.8.1 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: acb484385a0c998eb5db741426d6b995c96d4cb4
-  data.tar.gz: 32e42a695872fe528b992454f09360fea9425f06
+  metadata.gz: d2265d0d8be3afe5dc1f6cd28ba5549074e3f0ae
+  data.tar.gz: 50019c927b4c3c10c3720f28a1968ee66df77631
 SHA512:
-  metadata.gz: d1aec6b4edcd69d35db70378fc0038816ccc033d79f95c69f5ca54419bee6dc12889f461518d756098dc4b921bfe5a758770d22645fd988057e23eaa13757f91
-  data.tar.gz: 3c183818960750ee2d7133cab300476aa27862d214e828f218908b45e0c7a55cc3cc21b3bea3e095d38c9f76ae7ef51ca6778ff381fa40d584726e47870b2356
+  metadata.gz: 11c2d55d39f2c4c7deaa9d40ca7fb6d80a5c2f09ce16005fae476e48fb4ba8ed903391a8bdab2f53e4f6ea7c9bb3349c2f45150e686f89c95561c6330d47e873
+  data.tar.gz: 8babbc84df2b416d688862fdde07eed04503610d702b4d61ead825c41e86cd50c6f4a6f55eed6e4748fb78331fd0025bbbaff642de1d877e2e8d6b2d265fe0e2

data/README.rdoc

@@ -31,8 +31,8 @@ Store fluent-event as CouchDB Document to CouchDB database.
       protocol https               #default:http
 
       update_docs true             #default:false
-      doc_key_field doc_id         #default:nil
-      doc_key_jsonpath $.event.key #default:nil
+      doc_key_field doc_id         #default:nil. ${tag} will be replaced with actual event's tag.
+      doc_key_jsonpath $.event.key #default:nil. ${tag} will be replaced with actual event's tag.
 
       refresh_view_index viewname  #default:nil
 

data/Rakefile

@@ -12,7 +12,7 @@ begin
     gemspec.homepage = "http://github.com/ixixi/fluent-plugin-couch"
     gemspec.has_rdoc = false
     gemspec.require_paths = ["lib"]
-    gemspec.add_dependency "fluentd", [">= 0.10.0", "< 2"]
+    gemspec.add_dependency "fluentd", [">= 0.14.0", "< 2"]
     gemspec.add_dependency "couchrest", "~> 1.1.2"
     gemspec.add_dependency "jsonpath", "~> 0.4.2"
     gemspec.test_files = Dir["test/**/*.rb"]

data/VERSION

@@ -1 +1 @@
-0.8.1
+0.9.1

data/lib/fluent/plugin/out_couch.rb

@@ -1,46 +1,55 @@
-module Fluent
-    class CouchOutput < BufferedOutput
+require "fluent/plugin/output"
+
+module Fluent::Plugin
+    class CouchOutput < Output
         attr_reader :db # for tests
 
-        include SetTagKeyMixin
+        DEFAULT_BUFFER_TYPE = "memory"
+
+        helpers :compat_parameters, :inject
+
         config_set_default :include_tag_key, false
-        
-        include SetTimeKeyMixin
         config_set_default :include_time_key, true
-        
+
         Fluent::Plugin.register_output('couch', self)
-        
+
         config_param :database, :string
-        
+
         config_param :host, :string, :default => 'localhost'
         config_param :port, :string, :default => '5984'
         config_param :protocol, :string, :default => 'http'
-        
+
         config_param :refresh_view_index , :string, :default => nil
-        
+
         config_param :user, :string, :default => nil
         config_param :password, :string, :default => nil, :secret => true
-        
+
         config_param :update_docs, :bool, :default => false
         config_param :doc_key_field, :string, :default => nil
         config_param :doc_key_jsonpath, :string, :default => nil
-        
+
+        config_section :buffer do
+            config_set_default :@type, DEFAULT_BUFFER_TYPE
+            config_set_default :chunk_keys, ['tag']
+        end
+
         def initialize
             super
-            
+
             require 'msgpack'
             require 'jsonpath'
             Encoding.default_internal = 'UTF-8'
             require 'couchrest'
             Encoding.default_internal = 'ASCII-8BIT'
         end
-        
+
         def configure(conf)
+            compat_parameters_convert(conf, :buffer, :inject)
             super
             account = "#{@user}:#{@password}@" if @user && @password
             @db = CouchRest.database!("#{@protocol}://#{account}#{@host}:#{@port}/#{@database}")
         end
-        
+
         def start
             super
             @views = []
@@ -50,25 +59,31 @@ module Fluent
                         @views.push([@refresh_view_index,view_name])
                     end
                     rescue
-                    $log.error 'design document not found!'
+                    log.error 'design document not found!'
                 end
             end
         end
-        
+
         def shutdown
             super
         end
-        
+
         def format(tag, time, record)
+            record = inject_values_to_record(tag, time, record)
             record.to_msgpack
         end
-        
+
+        def formatted_to_msgpack_binary
+            true
+        end
+
         def write(chunk)
             records = []
+            doc_key_field, doc_key_jsonpath = expand_placeholders(chunk.metadata)
             chunk.msgpack_each {|record|
-                
-                id = record[@doc_key_field]
-                id = JsonPath.new(@doc_key_jsonpath).first(record) if id.nil? && !@doc_key_jsonpath.nil?
+
+                id = record[doc_key_field]
+                id = JsonPath.new(doc_key_jsonpath).first(record) if id.nil? && !doc_key_jsonpath.nil?
                 record['_id'] = id unless id.nil?
                 records << record
             }
@@ -79,7 +94,7 @@ module Fluent
             end
             update_view_index
         end
-        
+
         def update_docs(records)
             if records.length > 0
                 records.each{|record|
@@ -89,16 +104,25 @@ module Fluent
                         rescue
                     end
                     record['_rev']=doc['_rev'] unless doc.nil?
-                    $log.debug record
-                    @db.save_doc(record) 
+                    log.debug record
+                    @db.save_doc(record)
                 }
             end
         end
-        
+
         def update_view_index()
             @views.each do |design,view|
                 @db.view("#{design}/#{view}",{"limit"=>"0"})
             end
         end
+
+        private
+
+        def expand_placeholders(metadata)
+            field = jsonpath = nil
+            field = extract_placeholders(@doc_key_field, metadata) if @doc_key_field
+            jsonpath = extract_placeholders(@doc_key_jsonpath, metadata) if @doc_key_jsonpath
+            return field, jsonpath
+        end
     end
 end

data/test/test_out_couch.rb

@@ -1,7 +1,11 @@
 require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
 require "fluent/plugin/out_couch"
 
 class CouchOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
   end
@@ -19,12 +23,24 @@ class CouchOutputTest < Test::Unit::TestCase
     update_docs true
   ]
 
+  CONFIG_UPDATE_DOC_WITH_TAG_PLACEHOLDER = %[
+    database #{DATABASE_NAME}
+    doc_key_field ${tag}
+    update_docs true
+  ]
+
   CONFIG_JSONPATH = %[
     database #{DATABASE_NAME}
     doc_key_field key
     doc_key_jsonpath $.nested.key
   ]
 
+  CONFIG_JSONPATH_WITH_TAG_PLACEHOLDER = %[
+    database #{DATABASE_NAME}
+    doc_key_field key
+    doc_key_jsonpath $.${tag}.key
+  ]
+
   CONFIG_DESIGN = %[
     database #{DATABASE_NAME}
     doc_key_field key
@@ -39,7 +55,7 @@ class CouchOutputTest < Test::Unit::TestCase
   end
 
   def create_driver(config = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::CouchOutput).configure(config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::CouchOutput).configure(config)
   end
 
   def test_configure
@@ -68,9 +84,10 @@ class CouchOutputTest < Test::Unit::TestCase
 
     def test_write
       previous_rows = @d.instance.db.all_docs["total_rows"]
-      time = Time.now.to_i
-      @d.emit({"message" => "record"}, time)
-      @d.run
+      time = event_time
+      @d.run(default_tag: "tag") do
+        @d.feed(time, {"message" => "record"})
+      end
       rows = @d.instance.db.all_docs["total_rows"]
       assert_equal(rows, previous_rows + 1)
     end
@@ -87,19 +104,46 @@ class CouchOutputTest < Test::Unit::TestCase
     end
 
     def test_write
-      time = Time.now.to_i
-      @d.emit({"key" => "record-1", "message" => "record"}, time)
-      @d.run
-      previous_rows = @d.instance.db.all_docs["total_rows"]
-      @d.emit({"key" => "record-1", "message" => "record-mod"}, time)
-      @d.run
-      rows = @d.instance.db.all_docs["total_rows"]
+      time = event_time
+      previous_rows = rows = 0
+      @d.run(default_tag: "test") do
+        @d.feed(time, {"key" => "record-1", "message" => "record"})
+        previous_rows = @d.instance.db.all_docs["total_rows"]
+        @d.feed(time, {"key" => "record-1", "message" => "record-mod"})
+        rows = @d.instance.db.all_docs["total_rows"]
+      end
       record = @d.instance.db.get("record-1")
       assert_equal(rows, previous_rows)
       assert_equal("record-mod", record["message"])
     end
   end
 
+  class WriteWithUpdateDocWithTagPlaceHolderTest < self
+    def setup
+      @d = create_driver(CONFIG_UPDATE_DOC_WITH_TAG_PLACEHOLDER)
+      prepare_db
+    end
+
+    def teardown
+      @db.delete!
+    end
+
+    def test_write
+      time = event_time
+      tag = "replace.placeholder"
+      previous_rows = rows = 0
+      @d.run(default_tag: tag) do
+        @d.feed(time, {tag => "record-placeholder", "message" => "record"})
+        previous_rows = @d.instance.db.all_docs["total_rows"]
+        @d.feed(time, {tag => "record-placeholder", "message" => "record-placeholder"})
+        rows = @d.instance.db.all_docs["total_rows"]
+      end
+      assert_equal(rows, previous_rows)
+      record = @d.instance.db.get("record-placeholder")
+      assert_equal("record-placeholder", record["message"])
+    end
+  end
+
   class WriteWithJsonpathTest < self
     def setup
       @d = create_driver(CONFIG_JSONPATH)
@@ -111,14 +155,36 @@ class CouchOutputTest < Test::Unit::TestCase
     end
 
     def test_write
-      time = Time.now.to_i
-      @d.emit({"nested" => {"key" => "record-nested", "message" => "record"}}, time)
-      @d.run
+      time = event_time
+      @d.run(default_tag: "test") do
+        @d.feed(time, {"nested" => {"key" => "record-nested", "message" => "record"}})
+      end
       record = @d.instance.db.get("record-nested")
       assert_equal({"key" => "record-nested", "message" => "record"}, record["nested"])
     end
   end
 
+  class WriteWithJsonpathWithTagPlaceholderTest < self
+    def setup
+      @d = create_driver(CONFIG_JSONPATH_WITH_TAG_PLACEHOLDER)
+      prepare_db
+    end
+
+    def teardown
+      @db.delete!
+    end
+
+    def test_write
+      time = event_time
+      tag = "couchrecord"
+      @d.run(default_tag: tag) do
+        @d.feed(time, {tag => {"key" => "record-nested", "message" => "record"}})
+      end
+      record = @d.instance.db.get("record-nested")
+      assert_equal({"key" => "record-nested", "message" => "record"}, record["couchrecord"])
+    end
+  end
+
   class WriteWithDesignTest < self
     def setup
       @d = create_driver(CONFIG_DESIGN)
@@ -148,9 +214,10 @@ class CouchOutputTest < Test::Unit::TestCase
     end
 
     def test_write
-      time = Time.now.to_i
-      @d.emit({"key" => "record-design", "message" => "record"}, time)
-      @d.run
+      time = event_time
+      @d.run(default_tag: "test") do
+        @d.feed(time, {"key" => "record-design", "message" => "record"})
+      end
       record = @d.instance.db.get("record-design")
       assert_equal("record", record["message"])
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-couch
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.1
 platform: ruby
 authors:
 - Yuri Odagiri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-07 00:00:00.000000000 Z
+date: 2017-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -19,7 +19,7 @@ dependencies:
         version: '2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
   name: couchrest
   requirement: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: CouchDB output plugin for Fluentd event collector