fluent-plugin-kafka-enchanced 0.5.13 → 0.5.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6de4a5a5608227bd053342b9c703d7e8e06524b5
-  data.tar.gz: 3fae1288bcba880d5ebed4b9668496ad85811fa4
+  metadata.gz: fa418e81830e8815303b31ae7d4a76974adc0d76
+  data.tar.gz: a1baed0c8eecc19256606ea5900530f243c38b8a
 SHA512:
-  metadata.gz: 6c1f4ff335fecb5755949608790f468294ac4ba51c6ee676f58c4083a5fb97cd88931487e2bd2da5be7dfcd09e59b0284e91a3359a064b1b4b6728acae4248a9
-  data.tar.gz: f72942ce11d630e31187892b081342318314bcde8695c3964b4b5bcdcb73484baa6c88026027b8e855d08ee4b027692e336fd88705932f29a387e248d8d7c5ef
+  metadata.gz: b97ac0ec71bd70ae36b6241f2373a766e05d77a39977cc118e78cd9e5d7065f90dc8b92c816709ff240b39090fa85d0133644ca2ef32288be1b21e60952b2137
+  data.tar.gz: 29e7f1355c94173ae03d19a5eeda9fb91a2a5d61005c6af3dc582852b5878899ec675af742086a9278f5c4083ebbeb2f85ffa15e156e2fab86f046b47e9fd671

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka-enchanced"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.13'
+  gem.version       = '0.5.14'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -146,9 +146,10 @@ DESC
       require 'avro_turf/messaging'
       require "avro/builder"
       Proc.new do |tag, time, record|
+        record = record.select{|key, value| key.present?}.map{|k, v| [k.tr('[]-', '_').delete('$'), v.to_s]}.to_h
         record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
         fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
-        schema_name = "#{tag.to_s.gsub('.', '_')}"
+        @topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{fields.to_s.hash.abs}"
         schema_json = {
             "type": "record",
             "name": schema_name,
@@ -157,7 +158,6 @@ DESC
         schema = Avro::Schema.parse(schema_json)
 
         avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
-        # binding.pry
         avro.encode(record.map{|key, value| [key, value.to_s]}.to_h, schema: schema, subject: "#{schema_name}-value")
       end
     elsif @output_data_type == 'ltsv'
@@ -199,12 +199,12 @@ DESC
           end
         end
         record['tag'] = tag if @output_include_tag
-        topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag.to_s.gsub('.', '_')
         partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
         partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
         message_key = (@exclude_message_key ? record.delete('message_key') : record['message_key']) || @default_message_key
 
         value = @formatter_proc.call(tag, time, record)
+        topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || @topic_name
 
         log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}.") }
         producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
@@ -223,3 +223,4 @@ DESC
 end
 
 
+

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -199,10 +199,10 @@ DESC
       require 'avro_turf/messaging'
       require "avro/builder"
       Proc.new do |tag, time, record|
-        record = record.select{|key, value| key.present?}
+        record = record.select{|key, value| key.present?}.map{|k, v| [k.tr('[]-', '_').delete('$'), v.to_s]}.to_h
         record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
         fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
-        schema_name = "#{tag.to_s.gsub('.', '_')}"
+        @topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{fields.to_s.hash.abs}"
         schema_json = {
             "type": "record",
             "name": schema_name,
@@ -299,3 +299,4 @@ DESC
   end
 end
 
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka-enchanced
 version: !ruby/object:Gem::Version
-  version: 0.5.13
+  version: 0.5.14
 platform: ruby
 authors:
 - Paschenko Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-27 00:00:00.000000000 Z
+date: 2017-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd