fluent-plugin-documentdb 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97025a6f27e146af8c1b936eaa6ca59fff93b54e
-  data.tar.gz: c2395499cc0f3b150fd3121491a3dd54aa531d2d
+  metadata.gz: 8caba6ff7692f27f36840ce51b0b0c4b5bb5b20d
+  data.tar.gz: 50b0035e0f2e372ed8f71677edc31cbe6887ec74
 SHA512:
-  metadata.gz: 2f96f26a898d09456c825c7af941c3ae43df1d6d10ba751af2529fc84f4a2b0ae026b42ae228c951e4d5c8ab280511894ed2f1da68d00df3df0472a329113e6a
-  data.tar.gz: 1514d17a07fe7d1fd80903b51bec094ad1459e56de7f50c0a82999173f2b0ce66a2d60d4415069249ab4c12289c304f667c630150e13b067c7c00affdcf63497
+  metadata.gz: c7c415043e7f3282c197d86e3e7e484e11c767f9ae31141b46e0fd27a590698b7094d4bf9a72aeb7086c320c4894cbf5c420eeb9cbd8c0b2eb772dbd5c80fb56
+  data.tar.gz: c93403b5a961cdce8a72203975b7ced2cd1fa804784da8c7b2d86939f9577ec407896ad61f2f50f6eb14602d3ce207a4c890c3447a5aad506c12f62ae05860bd

data/ChangeLog.md

@@ -0,0 +1,26 @@
+## 0.3.0
+
+* Migrate to use fluentd v0.14 API - [PR#4](https://github.com/yokawasa/fluent-plugin-documentdb/pull/4)
+* Support plugin specific log level - [PR#3](https://github.com/yokawasa/fluent-plugin-documentdb/pull/3) 
+
+
+## 0.2.1
+
+* Fixup bug on Single-Collection mode
+
+## 0.2.0
+
+* Support Partitioned Collection mode
+* No longer depend on azure-documentdb-sdk instead use very tiny documentdb client library that included in the plugin
+
+## 0.1.2
+	
+* Change gem package dependency option for azure-documentdb-sdk from add_development_dependency to add_dependency
+
+## 0.1.1
+
+* Security enhanced option: Added secret option to docdb_account_key
+
+## 0.1.0
+
+* Inital Release

data/README.md

@@ -26,6 +26,7 @@ To use Microsoft Azure DocumentDB, you must create a DocumentDB database account
   
     <match documentdb.*>
         @type documentdb
+        @log_level info
         docdb_endpoint  DOCUMENTDB_ACCOUNT_ENDPOINT
         docdb_account_key DOCUMENTDB_ACCOUNT_KEY
         docdb_database  mydb
@@ -59,6 +60,7 @@ To use Microsoft Azure DocumentDB, you must create a DocumentDB database account
  * **add\_tag\_field (optional)** - Default:true. This option allows to insert a tag field to record
  * **tag\_field\_name (optional)** - Default:tag. Tag field name to be inserted
 
+[note] @log_level is a fluentd built-in parameter (optional) that controls verbosity of logging: fatal|error|warn|info|debug|trace (See also [Logging of Fluentd](http://docs.fluentd.org/articles/logging#log-level))
 
 ## Configuration examples
 
@@ -174,6 +176,17 @@ An expected output record for sample input will be like this:
 ## TODOs
  * Support automatic data expiration with TTL (Time-to-Live ). See [Expire data in DocumentDB collections automatically with time to live](https://azure.microsoft.com/en-us/documentation/articles/documentdb-time-to-live/)
 
+
+## Change log
+* [Changelog](ChangeLog.md)
+
+## Links
+
+* http://yokawasa.github.io/fluent-plugin-documentdb
+* https://rubygems.org/gems/fluent-plugin-documentdb
+* [Collecting logs into Azure DocumentDB using fluent-plugin-documentdb](http://unofficialism.info/posts/collecting-logs-into-azure-documentdb-using-fluent-plugin-documentdb/)
+* [fluent-plugin-documentdb supports Partitioned collections](http://unofficialism.info/posts/fluent-plugin-documentdb-supports-partitioned-collections/)
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yokawasa/fluent-plugin-documentdb.

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

data/conf/fluent-sample-partioned-collection.conf

@@ -8,8 +8,9 @@
 
 <match documentdb.*>
     @type documentdb
+    @log_level info
     docdb_endpoint https://yoichikademo1.documents.azure.com:443/
-    docdb_account_key  EMwUa3EzsAtJ1qYfzwo9nQ3KudofsXNm3xLh1SLffKkUHMFl80OZRZIVu4lxdKRKxkgVAj0c2mv9BZSyMN7tdg==
+    docdb_account_key  EMwUa3EzsAtJ1qYfzwo9nQ3KxxfsXNm3xLh1SLffKkUHMFl80OZRZIVu4lxdKRKxkgVAj0c2mv9BZSyMN7tdg==
     docdb_database mydb
     docdb_collection mycollection
     auto_create_database true

data/conf/fluent-sample-single-collection.conf

@@ -8,8 +8,9 @@
 
 <match documentdb.*>
     @type documentdb
+    @log_level info
     docdb_endpoint https://yoichikademo1.documents.azure.com:443/
-    docdb_account_key  EMwUa3EzsAtJ1qYfzwo9nQ3KudofsXNm3xLh1SLffKkUHMFl80OZRZIVu4lxdKRKxkgVAj0c2mv9BZSyMN7tdg==
+    docdb_account_key  EMwUa3EzsAtJ1qYfzwo9nQ3KxxfsXNm3xLh1SLffKkUHMFl80OZRZIVu4lxdKRKxkgVAj0c2mv9BZSyMN7tdg==
     docdb_database mydb
     docdb_collection mycollection
     auto_create_database true

data/fluent-plugin-documentdb.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|gem|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.15", "< 2"]
   gem.add_dependency "rest-client"
   gem.add_development_dependency "bundler", "~> 1.11"
   gem.add_development_dependency "rake", "~> 10.0"

data/lib/fluent/plugin/out_documentdb.rb

@@ -1,22 +1,22 @@
 # -*- coding: utf-8 -*-
+require 'fluent/plugin/output'
+require 'msgpack'
+require 'time'
+require 'securerandom'
+require 'fluent/plugin/documentdb/client'
+require 'fluent/plugin/documentdb/partitioned_coll_client'
+require 'fluent/plugin/documentdb/header'
+require 'fluent/plugin/documentdb/resource'
+require 'fluent/plugin/documentdb/constants'
 
-module Fluent
+module Fluent::Plugin
 
-  require 'fluent/plugin/documentdb/constants'
+  class DocumentdbOutput < Output
+    Fluent::Plugin.register_output('documentdb', self)
 
-  class DocumentdbOutput < BufferedOutput
-    Plugin.register_output('documentdb', self)
+    helpers :compat_parameters
 
-    def initialize
-      super
-      require 'msgpack'
-      require 'time'
-      require 'securerandom'
-      require 'fluent/plugin/documentdb/client'
-      require 'fluent/plugin/documentdb/partitioned_coll_client'
-      require 'fluent/plugin/documentdb/header'
-      require 'fluent/plugin/documentdb/resource'
-    end
+    DEFAULT_BUFFER_TYPE = "memory"
 
     config_param :docdb_endpoint, :string
     config_param :docdb_account_key, :string, :secret => true
@@ -34,27 +34,34 @@ module Fluent
     config_param :add_tag_field, :bool, :default => false
     config_param :tag_field_name, :string, :default => 'tag'
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ['tag']
+    end
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer)
       super
-      raise ConfigError, 'no docdb_endpoint' if @docdb_endpoint.empty?
-      raise ConfigError, 'no docdb_account_key' if @docdb_account_key.empty?
-      raise ConfigError, 'no docdb_database' if @docdb_database.empty?
-      raise ConfigError, 'no docdb_collection' if @docdb_collection.empty?
+      raise Fluent::ConfigError, 'no docdb_endpoint' if @docdb_endpoint.empty?
+      raise Fluent::ConfigError, 'no docdb_account_key' if @docdb_account_key.empty?
+      raise Fluent::ConfigError, 'no docdb_database' if @docdb_database.empty?
+      raise Fluent::ConfigError, 'no docdb_collection' if @docdb_collection.empty?
       if @add_time_field and @time_field_name.empty?
-        raise ConfigError, 'time_field_name must be set if add_time_field is true'
+        raise Fluent::ConfigError, 'time_field_name must be set if add_time_field is true'
       end
       if @add_tag_field and @tag_field_name.empty?
-        raise ConfigError, 'tag_field_name must be set if add_tag_field is true'
+        raise Fluent::ConfigError, 'tag_field_name must be set if add_tag_field is true'
       end
       if @partitioned_collection
-        raise ConfigError, 'partition_key must be set in partitioned collection mode' if @partition_key.empty?
+        raise Fluent::ConfigError, 'partition_key must be set in partitioned collection mode' if @partition_key.empty?
         if (@auto_create_collection &&
               @offer_throughput < AzureDocumentDB::PARTITIONED_COLL_MIN_THROUGHPUT)
-          raise ConfigError, sprintf("offer_throughput must be more than and equals to %s",
-                                 AzureDocumentDB::PARTITIONED_COLL_MIN_THROUGHPUT) 
+          raise Fluent::ConfigError, sprintf("offer_throughput must be more than and equals to %s",
+                                 AzureDocumentDB::PARTITIONED_COLL_MIN_THROUGHPUT)
         end
       end
-      @timef = TimeFormatter.new(@time_format, @localtime)
+      raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
+      @timef = Fluent::TimeFormatter.new(@time_format, @localtime)
     end
 
     def start
@@ -72,7 +79,7 @@ module Fluent
         ## initial operations for database
         res = @client.find_databases_by_name(@docdb_database)
         if( res[:body]["_count"].to_i == 0 )
-          raise "No database (#{docdb_database}) exists! Enable auto_create_database or create it by useself" if !@auto_create_database 
+          raise "No database (#{docdb_database}) exists! Enable auto_create_database or create it by useself" if !@auto_create_database
           # create new database as it doesn't exists
           @client.create_database(@docdb_database)
         end
@@ -94,7 +101,7 @@ module Fluent
         @coll_resource = @client.get_collection_resource(database_resource, @docdb_collection)
 
       rescue Exception =>ex
-        $log.fatal "Error: '#{ex}'"
+        log.fatal "Error: '#{ex}'"
         exit!
       end
     end
@@ -111,10 +118,18 @@ module Fluent
       end
       if @add_tag_field
         record[@tag_field_name] = tag
-      end 
+      end
       record.to_msgpack
     end
 
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
       chunk.msgpack_each { |record|
         unique_doc_identifier = record["id"]
@@ -127,12 +142,12 @@ module Fluent
         rescue RestClient::ExceptionWithResponse => rcex
           exdict = JSON.parse(rcex.response)
           if exdict['code'] == 'Conflict'
-            $log.fatal "Duplicate Error: document #{unique_document_identifier} already exists, data=>" + record.to_json
+            log.fatal "Duplicate Error: document #{unique_doc_identifier} already exists, data=>" + record.to_json
           else
-            $log.fatal "RestClient Error: '#{rcex.response}', data=>" + record.to_json
+            log.fatal "RestClient Error: '#{rcex.response}', data=>" + record.to_json
           end
         rescue => ex
-          $log.fatal "UnknownError: '#{ex}', uniqueid=>#{unique_doc_identifier}, data=>" + record.to_json
+          log.fatal "UnknownError: '#{ex}', uniqueid=>#{unique_doc_identifier}, data=>" + record.to_json
         end
       }
     end

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
+require 'fluent/test/driver/output'
 require 'fluent/plugin/out_documentdb'
 
 class Test::Unit::TestCase

data/test/plugin/test_documentdb.rb

@@ -28,41 +28,52 @@ class DocumentdbOutputTest < Test::Unit::TestCase
   #   utc
   # ]
 
-  def create_driver(conf = CONFIG, tag='documentdb.test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::DocumentdbOutput, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::DocumentdbOutput).configure(conf)
   end
 
   def test_configure
-    #### set configurations
-    # d = create_driver %[
-    #   path test_path
-    #   compress gz
-    # ]
-    #### check configurations
-    # assert_equal 'test_path', d.instance.path
-    # assert_equal :gz, d.instance.compress
+    d = create_driver
+
+    assert_equal 'https://yoichikademo1.documents.azure.com:443', d.instance.docdb_endpoint
+    assert_equal 'EMwUa3EzsAtJ1qYfzwo9nQ3KudofsXNm3xLh1SLffKkUHMFl80OZRZIVu4lxdKRKxkgVAj0c2mv9BZSyMN7tdg==',
+                 d.instance.docdb_account_key
+    assert_equal 'mydb', d.instance.docdb_database
+    assert_equal 'mycollection', d.instance.docdb_collection
+    assert_true d.instance.auto_create_database
+    assert_true d.instance.auto_create_collection
+    assert_equal 'host', d.instance.partition_key
+    assert_equal 10100, d.instance.offer_throughput
+    assert_equal '%Y%m%d-%H:%M:%S', d.instance.time_format
+    assert_false d.instance.localtime
+    assert_true d.instance.add_time_field
+    assert_equal 'time', d.instance.time_field_name
+    assert_true d.instance.add_tag_field
+    assert_equal 'tag', d.instance.tag_field_name
+
   end
 
   def test_format
     d = create_driver
 
-    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    # d.emit({"a"=>1}, time)
-    # d.emit({"a"=>2}, time)
-
-    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
-    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.run(default_tag: 'documentdb.test') do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
 
-    # d.run
+    # assert_equal EXPECTED1, d.formatted[0]
+    # assert_equal EXPECTED2, d.formatted[1]
   end
 
   def test_write
     d = create_driver
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-    data = d.run
+    data = d.run(default_tag: 'documentdb.test') do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
     puts data
     # ### FileOutput#write returns path
     # path = d.run
@@ -70,4 +81,3 @@ class DocumentdbOutputTest < Test::Unit::TestCase
     # assert_equal expect_path, path
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-documentdb
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Yoichi Kawasaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-28 00:00:00.000000000 Z
+date: 2017-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -94,7 +94,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ChangeLog
+- ChangeLog.md
 - Gemfile
 - LICENSE
 - README.md
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.2
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Azure DocumentDB output plugin for Fluentd

data/ChangeLog

@@ -1,21 +0,0 @@
-Release 0.2.1 - 2016/08/28
-
-	* Fixup bug on Single-Collection mode
-
-Release 0.2.0 - 2016/08/17
-
-	* Support Partitioned Collection mode
-	* No longer depend on azure-documentdb-sdk instead use very tiny documentdb client library that included in the plugin
-
-Release 0.1.2 - 2016/02/20
-	
-	* Change gem package dependency option for azure-documentdb-sdk from add_development_dependency to add_dependency
-
-Release 0.1.1 - 2016/02/20
-
-	* Security enhanced option: Added secret option to docdb_account_key
-
-Release 0.1.0 - 2016/01/19
-
-	* Inital Release
-	* Add out_documentdb