fluent-plugin-kafka-enchanced 0.5.9 → 0.5.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5191424214f2a081b8ade27615726e5f1394f7ad
-  data.tar.gz: 44d566152b37364fc44e2b44799d6cc51e2d9946
+  metadata.gz: fccd71b5e7d89f156703d465fe21780d44b333ce
+  data.tar.gz: 0cee930d56cf656c5c5c0826a8aced5ab14f92bc
 SHA512:
-  metadata.gz: 358bbdc3f4d54783ac8f58ef9f54fbf42eb06222688748a6c66248c66e2d612fe6eb720bfa47c43a868fccd3e3cd93468aa15dec7c61f00564eae6a760e8eeee
-  data.tar.gz: 7c992ff750ff15eb97e156b9839c69ed5dc1526f5570e67d74ad783685cccbd90f0d3a7c9fffae3c354e1d9b1898b2ac6a6dd270e412d034e8321f669970cf47
+  metadata.gz: f88b27ee87ce61afcffb47528a974ba4f10dc864c56d3e7ef6561d5faf0a9155f85651a1bf3602a6a4fd76792b38908cb94da9b001cd0ef653f50831c0e5c0c6
+  data.tar.gz: 76f4ada2526ef2e986ad89afdebe26a8721cd38407752abdba87f4406ecba4857515af1998d08fa42079eef80de1445dcdeadd1bc4daa009742f5c2ed5644a5f

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka-enchanced"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.9'
+  gem.version       = '0.5.10'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -70,8 +70,6 @@ DESC
     super
 
     require 'kafka'
-    require "avro_turf"
-    require "avro/builder"
 
     @kafka = nil
   end
@@ -144,14 +142,24 @@ DESC
       require 'yajl'
       Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
     elsif @output_data_type == 'avro'
-      fields = JSON.parse(value).keys.map{|key| {'name' => key, 'type' => 'string'}}.to_s.gsub('=>', ':')
-      schema_json =
-          <<-JSON
-        {"type": "record",
-          "name": "#{topic}_value",
-          "fields": #{fields}}
-      JSON
-      schema = Avro::Schema.parse(schema_json)
+      require "avro_turf"
+      require 'avro_turf/messaging'
+      require "avro/builder"
+      Proc.new do |tag, time, record|
+        record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
+        fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
+        schema_name = "#{tag.to_s.gsub('.', '_')}"
+        schema_json = {
+            "type": "record",
+            "name": schema_name,
+            "fields": fields
+        }.to_json
+        schema = Avro::Schema.parse(schema_json)
+
+        avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
+        # binding.pry
+        avro.encode(record.map{|key, value| [key, value.to_s]}.to_h, schema: schema, subject: "#{schema_name}-value")
+      end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
       Proc.new { |tag, time, record| LTSV.dump(record) }
@@ -182,6 +190,7 @@ DESC
       producer = @kafka.producer(@producer_opts)
 
       es.each do |time, record|
+        record = record.select{|key, value| key.present?}
         if @output_include_time
           if @time_format
             record['time'] = Time.at(time).strftime(@time_format)
@@ -213,3 +222,4 @@ DESC
 
 end
 
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka-enchanced
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.10
 platform: ruby
 authors:
 - Paschenko Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-16 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd