fluent-plugin-kafka 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dfe217eaf0f6bd1b83a27e0ab49bbc959b8aa311
-  data.tar.gz: d45ade2033759a9c41f2ec98354f792db1302e65
+  metadata.gz: 5d2a69320d98e30694e5599b3bf38bd492ba1338
+  data.tar.gz: 5faf0f964137b61e90c55bd7f2dccd3fc8022ab6
 SHA512:
-  metadata.gz: 04e02ee2614432dc8e0a07e99ef2cb25cf86e764b4e1c37e2b2b719e0e4592d890539b74d5b4ca2def0b3dace18770c28b7eb08bb797fead911b7e6188cf0083
-  data.tar.gz: 0192de37ab4a8408a8fe22effa2a71d7bf0864f553fcc9ea139d2a9146f5b8b26f3e7702d5579bff2b2e4f12b4a66f4b12bec66f7c0c9e45459d44c2e5571eaf
+  metadata.gz: b21ee5f9e9c0e6a163aa4d82e0c0c028da102441da56868beac52d8cefa755ac08d38e8f225733b408a00f055e31bfa47766497863f812c675478bcc587ff881
+  data.tar.gz: 2e5f7d4e225822f14559cda6fb4c5d103e0517d63fdc61b3fe3356db90ab494125d606a0deeaa308eaddab809d08ff52ba974ad0b70724a8f1c7e8596ec8a41c

data/README.md

@@ -70,7 +70,6 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
       @type   kafka_group
       brokers <list of broker-host:port, separate with comma, must set>
       zookeepers <list of broker-host:port, separate with comma, must set>
-      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for brokers
       consumer_group <consumer group name, must set>
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)>
@@ -94,6 +93,8 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
 
 ### Output plugin (non-buffered)
 
+This plugin uses Poseidon producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead.
+
     <match *.**>
       @type               kafka
 
@@ -141,6 +142,8 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
 
 ### Buffered output plugin
 
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
+
     <match *.**>
       @type               kafka_buffered
 
@@ -155,20 +158,20 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
       output_data_type    (json|ltsv|msgpack|attr:<record name>|<formatter name>)
       output_include_tag  (true|false) :default => false
       output_include_time (true|false) :default => false
-      max_send_retries    (integer)    :default => 3
+      max_send_retries    (integer)    :default => 1
       required_acks       (integer)    :default => 0
-      ack_timeout_ms      (integer)    :default => 1500
-      compression_codec   (none|gzip|snappy) :default => none
+      ack_timeout         (integer)    :default => 5
+      compression_codec   (gzip|snappy) :default => none
     </match>
 
-Supports following Poseidon::Producer options.
+Supports following ruby-kafka's producer options.
 
-- max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
+- max_send_retries — default: 1 — Number of times to retry sending of messages to a leader.
 - required_acks — default: 0 — The number of acks required per request.
-- ack_timeout_ms — default: 1500 — How long the producer waits for acks.
-- compression_codec - default: none - The codec the producer uses to compress messages.
+- ack_timeout — default: 5 — How long the producer waits for acks. The unit is seconds.
+- compression_codec - default: nil - The codec the producer uses to compress messages.
 
-See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
+See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about Poseidon.
 
 This plugin supports compression codec "snappy" also.
 Install snappy module before you use snappy compression.

data/Rakefile

@@ -1,11 +1,12 @@
-#!/usr/bin/env rake
-require "bundler/gem_tasks"
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
+
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
+  test.test_files = FileList['test/**/test_*.rb']
   test.verbose = true
 end
 
-task :default => :test
+task :default => [:build]

data/fluent-plugin-kafka.gemspec

@@ -12,13 +12,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.2.0'
-  gem.add_development_dependency 'test-unit'
-  gem.add_dependency 'fluentd'
+  gem.version = '0.2.1'
+  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'yajl-ruby'
-  gem.add_dependency 'msgpack'
   gem.add_dependency 'zookeeper'
-  gem.add_dependency 'ruby-kafka', '~> 0.3.2'
+  gem.add_dependency 'ruby-kafka', '~> 0.3.9'
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -143,8 +143,12 @@ DESC
     @kafka = nil
   end
 
-  def format(tag, time, record)
-    [tag, time, record].to_msgpack
+  def emit(tag, es, chain)
+    super(tag, es, chain, tag)
+  end
+
+  def format_stream(tag, es)
+    es.to_msgpack_stream
   end
 
   def shutdown_producers
@@ -189,13 +193,13 @@ DESC
     else
       @formatter = Fluent::Plugin.new_formatter(@output_data_type)
       @formatter.configure(conf)
-      Proc.new { |tag, time, record|
-        @formatter.format(tag, time, record)
-      }
+      @formatter.method(:format)
     end
   end
 
   def write(chunk)
+    tag = chunk.key
+    def_topic = @default_topic || tag
     producer = get_producer
 
     records_by_topic = {}
@@ -203,7 +207,7 @@ DESC
     messages = 0
     messages_bytes = 0
     begin
-      chunk.msgpack_each { |tag, time, record|
+      chunk.msgpack_each { |time, record|
         if @output_include_time
           if @time_format
             record['time'] = Time.at(time).strftime(@time_format)
@@ -213,7 +217,7 @@ DESC
         end
 
         record['tag'] = tag if @output_include_tag
-        topic = record['topic'] || @default_topic || tag
+        topic = record['topic'] || def_topic
         partition_key = record['partition_key'] || @default_partition_key
 
         records_by_topic[topic] ||= 0

metadata

@@ -1,43 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-18 00:00:00.000000000 Z
+date: 2016-07-06 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: test-unit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: poseidon_cluster
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +59,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: zookeeper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,47 +73,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: msgpack
+  name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.9
 - !ruby/object:Gem::Dependency
-  name: zookeeper
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: 0.9.2
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
-  type: :runtime
+        version: 3.0.8
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 3.0.8
 description: Fluentd plugin for Apache Kafka > 0.8
 email:
 - togachiro@gmail.com
@@ -159,7 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8