fluent-plugin-kafka-enchanced 0.5.30 → 0.5.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38f6200587dda4cf3b76e1c6b98d0bbdfeb40df8
-  data.tar.gz: 9c5404edd306b27b9e1f3225a6df593c1821a3de
+  metadata.gz: 126ac61c90d73c137a2617876ee242a394659b14
+  data.tar.gz: fc9180ce1429f3aa2af0a4aea3eb12e9254b58cd
 SHA512:
-  metadata.gz: 54c8c4b8a1319522bf06200a9f19b142e7f0951df5877b976071fa6a6aa272b7ddd1fe4ce75c60f7ec93a0500fad678b76ae231f394ae9390607ae8ea1248a73
-  data.tar.gz: 25ef86c883b9a1cb2de5a1321d7bf534c0ce95c78036e31caa5b51abfe958a9822fd5fb779d7d0573d670ced91bd6f35b1ee983b47d40df27dcfdd4079e7bd24
+  metadata.gz: 6d2c68b75b7a6b19bed5f1b4dc7fde749ab7355632433c9a982c2daa43e2b554aa40c29ccfef903e3071ace576680e06424b2303ae325201d807ae3b55f96a49
+  data.tar.gz: f9fe60aa46e4be239a965d58063355823c12c046353226723b9edc93ab58f366b7a16800d61f1a28244dcc4b1cd7351c61a56d9da5a2fcff35b314ee3327d5c6

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka-enchanced"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.30'
+  gem.version       = '0.5.31'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -2,6 +2,7 @@ require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
 require 'pry'
 require 'digest'
+require 'redis'
 
 class Fluent::KafkaOutput < Fluent::Output
   Fluent::Plugin.register_output('kafka', self)
@@ -146,23 +147,54 @@ DESC
       require "avro_turf"
       require 'avro_turf/messaging'
       require "avro/builder"
+      init_redis
       Proc.new do |tag, time, record|
-        record = record.select{|key, value| !key.nil? && !key.empty?}.map{|k, v| [k.tr('[]-', '_').delete('$'), ((!v.is_a?(Fixnum) && !v.is_a?(Float)) ? v.to_s.force_encoding("UTF-8") : v)]}.to_h
+        record = record.select{|key, value| !key.nil? && !key.empty?}.map do |k, v|
+          [
+              k.tr('[]-', '_').delete('$'),
+              (v.is_a?(Fixnum) || v.is_a?(Float) || v.nil? ? v : v.to_s.force_encoding("UTF-8"))
+          ]
+        end.to_h
         timestamp = Time.new
         record['enchilada_time_with_format'] = timestamp.strftime("%Y-%m-%dT%H:%M:%S.%LZ")
+        @topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{Digest::MD5.new.hexdigest(record.keys.to_s)[0..5]}"
 
-        fields = record.map{|key, value| {'name' => key, 'type' => (value.is_a?(Fixnum) ? 'int' : (value.is_a?(Float) ? 'float' : 'string'))}}
-        record['enchilada_timestamp'] = timestamp.strftime('%s%3N').to_i
-        fields << {"name" => "enchilada_timestamp", "type" => "long"}
-        @topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{Digest::MD5.new.hexdigest(fields.to_s)[0..5]}"
-        schema_json = {
-            "type": "record",
-            "name": schema_name,
-            "fields": fields
-        }.to_json
-        schema = Avro::Schema.parse(schema_json)
         avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
-        avro.encode(record, schema: schema, subject: "#{schema_name}-value")
+
+        unless (stored_schema = get_schema_from_redis_by_name(schema_name))
+          fields = record.map do |key, value|
+            {
+                'name' => key,
+                'type' => ['null', (value.is_a?(Fixnum) ? 'int' : (value.is_a?(Float) ? 'float' : 'string'))]
+            }
+          end
+          field_types = fields.map{|field| [field['name'], (field['type'] - ['null']).first]}.to_h
+          fields << {"name" => "enchilada_timestamp", "type" => "long"}
+          schema_json = {
+              "type": "record",
+              "name": schema_name,
+              "fields": fields
+          }.to_json
+          registry = avro.instance_variable_get('@registry')
+          schema = Avro::Schema.parse(schema_json)
+          schema_id = registry.register("#{schema_name}-value", schema)
+
+          stored_schema = {
+              'schema' => schema,
+              'schema_id' => schema_id,
+              'field_types' => field_types
+          }
+
+          set_schema_to_redis(schema_name, stored_schema)
+
+        end
+
+        record['enchilada_timestamp'] = timestamp.strftime('%s%3N').to_i
+        record = record.map do |key, val|
+          [key, (stored_schema['field_types'][key] != 'string' || val.nil? ? val : val.to_s)]
+        end.to_h
+
+        avro.encode(record, stored_schema['schema_id'], schema: stored_schema['schema'])
       end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
@@ -224,9 +256,29 @@ DESC
     end
   end
 
+  def init_redis
+    $redis = Redis.new
+  end
+
+  def set_schema_to_redis schema_name, schema
+    $redis.set(schema_name, schema.to_json)
+  end
+
+  def get_schema_from_redis_by_name schema_name
+    if stored_schema = $redis.get(schema_name)
+      parsed_schema = JSON.parse($redis.get(schema_name))
+      {
+          'schema_id' => parsed_schema['schema_id'],
+          'schema' => Avro::Schema.parse(parsed_schema['schema']),
+          'field_types' => parsed_schema['field_types']
+      }
+    end
+  end
+
 end
 
 
 
 
 
+

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -2,6 +2,7 @@ require 'thread'
 require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
 require 'digest'
+require 'redis'
 
 class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   Fluent::Plugin.register_output('kafka_buffered', self)
@@ -199,6 +200,7 @@ DESC
       require "avro_turf"
       require 'avro_turf/messaging'
       require "avro/builder"
+      init_redis
       Proc.new do |tag, time, record|
         record = record.select{|key, value| !key.nil? && !key.empty?}.map{|k, v| [k.tr('[]-', '_').delete('$'), ((!v.is_a?(Fixnum) && !v.is_a?(Float)) ? v.to_s.force_encoding("UTF-8") : v)]}.to_h
         timestamp = Time.new
@@ -302,6 +304,25 @@ DESC
     # Raise exception to retry sendind messages
     raise e
   end
+
+  def init_redis
+    $redis = Redis.new
+  end
+
+  def set_schema_to_redis schema_name, schema
+    $redis.set(schema_name, schema.to_json)
+  end
+
+  def get_schema_from_redis_by_name schema_name
+    if stored_schema = $redis.get(schema_name)
+      parsed_schema = JSON.parse($redis.get(schema_name))
+      {
+          'schema_id' => parsed_schema['schema_id'],
+          'schema' => Avro::Schema.parse(parsed_schema['schema']),
+          'field_types' => parsed_schema['field_types']
+      }
+    end
+  end
 end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka-enchanced
 version: !ruby/object:Gem::Version
-  version: 0.5.30
+  version: 0.5.31
 platform: ruby
 authors:
 - Paschenko Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-28 00:00:00.000000000 Z
+date: 2017-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd