fluent-plugin-kafka 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 498665d29438e6b21a3425138b42d0d69fa9bc95
-  data.tar.gz: 20c8b8bc1458035a52aa9492bb83fe4f5beee0d0
+  metadata.gz: d78b37f9ea216d97ac377d5a67e1be3e33e17daf
+  data.tar.gz: cb214827bea96f3e38c5e1c45de1aa9c2acdd47d
 SHA512:
-  metadata.gz: c5c1ab553c03c001dd80b471d9c6b30a9356750c12e464b232728e54574e4e68a9e9f6f041ea995b1f6a0c3670d96040c2219df3b02e42e4ab93e285c8a302df
-  data.tar.gz: fbfa49c138e08b3dc5a988b932e20e9a33afc993fc7baa6336fd92412f3136bea11a9b42f8d40176eb8b877387545383fe8701af6d9a28202f75f41aa593504a
+  metadata.gz: a3f1e852d3f66feb00446175c9ced7a89a85d6af850d84fa8dc2cd4a67a557616af5b91734074f1fb399704fa5a9a8b9413f93fd7d92ec39e940bcb3d8fe4972
+  data.tar.gz: b8218fb071b44874c7a941504ee048d76064a5c8a913347d171903dd559c60d12e81fc87d6916ed0f8dde5bc80887bc3fe29612c10d092bc8f59071c4b933e46

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.1.0'
+  gem.version = '0.1.1'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/in_kafka.rb

@@ -25,6 +25,10 @@ class KafkaInput < Input
   config_param :min_bytes, :integer, :default => nil
   config_param :socket_timeout_ms, :integer, :default => nil
 
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   def initialize
     super
     require 'poseidon'
@@ -88,6 +92,7 @@ class KafkaInput < Input
         @add_prefix,
         @add_suffix,
         offset_manager,
+        router,
         opt)
     }
     @topic_watchers.each {|tw|
@@ -109,7 +114,7 @@ class KafkaInput < Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, offset_manager, options={})
+    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, offset_manager, router, options={})
       @topic_entry = topic_entry
       @host = host
       @port = port
@@ -122,6 +127,7 @@ class KafkaInput < Input
       @add_suffix = add_suffix
       @options = options
       @offset_manager = offset_manager
+      @router = router
 
       @next_offset = @topic_entry.offset
       if @topic_entry.offset == -1 && offset_manager
@@ -154,7 +160,7 @@ class KafkaInput < Input
         begin
           msg_record = parse_line(msg.value)
           msg_record = decorate_offset(msg_record, msg.offset) if @add_offset_in_record
-          es.add(Time.now.to_i, msg_record)
+          es.add(Engine.now, msg_record)
         rescue
           $log.warn msg_record.to_s, :error=>$!.to_s
           $log.debug_backtrace
@@ -162,7 +168,7 @@ class KafkaInput < Input
       }
 
       unless es.empty?
-        Engine.emit_stream(tag, es)
+        @router.emit_stream(tag, es)
 
         if @offset_manager
           next_offset = @consumer.next_offset
@@ -186,18 +192,16 @@ class KafkaInput < Input
     end
 
     def parse_line(record)
-      parsed_record = {}
       case @format
       when 'json'
-        parsed_record = Yajl::Parser.parse(record)
+        Yajl::Parser.parse(record)
       when 'ltsv'
-        parsed_record = LTSV.parse(record)
+        LTSV.parse(record)
       when 'msgpack'
-        parsed_record = MessagePack.unpack(record)
+        MessagePack.unpack(record)
       when 'text'
-        parsed_record[@message_key] = record
+        {@message_key => record}
       end
-      parsed_record
     end
 
     def decorate_offset(record, offset)

data/lib/fluent/plugin/in_kafka_group.rb

@@ -1,6 +1,6 @@
 module Fluent
 
-class KafkaInput < Input
+class KafkaGroupInput < Input
   Plugin.register_input('kafka_group', self)
 
   config_param :brokers, :string
@@ -19,6 +19,10 @@ class KafkaInput < Input
   config_param :min_bytes, :integer, :default => nil
   config_param :socket_timeout_ms, :integer, :default => nil
 
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   def initialize
     super
     require 'poseidon_cluster'
@@ -68,7 +72,7 @@ class KafkaInput < Input
     @topic_watchers = @topic_list.map {|topic|
       TopicWatcher.new(topic, @broker_list, @zookeeper_list, @consumer_group,
                        interval, @format, @message_key, @add_prefix,
-                       @add_suffix, opt)
+                       @add_suffix, router, opt)
     }
     @topic_watchers.each {|tw|
       tw.attach(@loop)
@@ -90,13 +94,14 @@ class KafkaInput < Input
   class TopicWatcher < Coolio::TimerWatcher
     def initialize(topic, broker_list, zookeeper_list, consumer_group,
                    interval, format, message_key, add_prefix, add_suffix,
-                   options)
+                   router, options)
       @topic = topic
       @callback = method(:consume)
       @format = format
       @message_key = message_key
       @add_prefix = add_prefix
       @add_suffix = add_suffix
+      @router = router
 
       @consumer = Poseidon::ConsumerGroup.new(
         consumer_group,
@@ -127,7 +132,7 @@ class KafkaInput < Input
         bulk.each do |msg|
           begin
             msg_record = parse_line(msg.value)
-            es.add(Time.now.to_i, msg_record)
+            es.add(Engine.now, msg_record)
           rescue
             $log.warn msg_record.to_s, :error=>$!.to_s
             $log.debug_backtrace
@@ -136,23 +141,21 @@ class KafkaInput < Input
       end
 
       unless es.empty?
-        Engine.emit_stream(tag, es)
+        @router.emit_stream(tag, es)
       end
     end
 
     def parse_line(record)
-      parsed_record = {}
       case @format
       when 'json'
-        parsed_record = Yajl::Parser.parse(record)
+        Yajl::Parser.parse(record)
       when 'ltsv'
-        parsed_record = LTSV.parse(record)
+        LTSV.parse(record)
       when 'msgpack'
-        parsed_record = MessagePack.unpack(record)
+        MessagePack.unpack(record)
       when 'text'
-        parsed_record[@message_key] = record
+        {@message_key => record}
       end
-      parsed_record
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-04 00:00:00.000000000 Z
+date: 2015-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd