fluent-plugin-kafka-enchanced 0.5.10 → 0.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fccd71b5e7d89f156703d465fe21780d44b333ce
-  data.tar.gz: 0cee930d56cf656c5c5c0826a8aced5ab14f92bc
+  metadata.gz: 2637f5d354feed7fd9a579446e40942b968541fe
+  data.tar.gz: cbe27f4f4510e3fe8da693eccddd71e6254520d3
 SHA512:
-  metadata.gz: f88b27ee87ce61afcffb47528a974ba4f10dc864c56d3e7ef6561d5faf0a9155f85651a1bf3602a6a4fd76792b38908cb94da9b001cd0ef653f50831c0e5c0c6
-  data.tar.gz: 76f4ada2526ef2e986ad89afdebe26a8721cd38407752abdba87f4406ecba4857515af1998d08fa42079eef80de1445dcdeadd1bc4daa009742f5c2ed5644a5f
+  metadata.gz: 0b72a8f89543e82dbe3048d740d11055870e6366283f16347098c074995437547746dfc5891a6f3a655a69d07bc660418489ce6ef54d0ef9e260c69bab68c6e9
+  data.tar.gz: 16c93a3a4b48a65a4ea58c40422f4e67a371f3e1d7d6309062ea585040af35716f414d9110d53fd5442c43ffdc94077e6b23cb981a2a24c0e132e86d0cd57c92

data/fluent-plugin-kafka.gemspec

@@ -5,14 +5,14 @@ Gem::Specification.new do |gem|
   gem.email         = ["zhr0n4x@gmail.com"]
   gem.description   = %q{Fluentd plugin for Apache Kafka > 0.8}
   gem.summary       = %q{Fluentd plugin for Apache Kafka > 0.8}
-  gem.homepage      = "https://github.com/fluent/fluent-plugin-kafka"
+  gem.homepage      = "https://github.com/zhron4x/fluent-plugin-kafka-enchanced"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka-enchanced"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.10'
+  gem.version       = '0.5.11'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -18,6 +18,8 @@ Set brokers via Zookeeper:
 DESC
   config_param :zookeeper_path, :string, :default => '/brokers/ids',
                :desc => "Path in path for Broker id. Default to /brokers/ids"
+  config_param :schema_registry, :string, :default => nil,
+               :desc => "Set Avro Schema Registry: <schema_registry_host>:<schema_registry_port>"
   config_param :default_topic, :string, :default => nil,
                :desc => "Output topic"
   config_param :default_message_key, :string, :default => nil
@@ -192,6 +194,25 @@ DESC
     elsif @output_data_type == 'msgpack'
       require 'msgpack'
       Proc.new { |tag, time, record| record.to_msgpack }
+    elsif @output_data_type == 'avro'
+      require "avro_turf"
+      require 'avro_turf/messaging'
+      require "avro/builder"
+      Proc.new do |tag, time, record|
+        record = record.select{|key, value| key.present?}
+        record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
+        fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
+        schema_name = "#{tag.to_s.gsub('.', '_')}"
+        schema_json = {
+            "type": "record",
+            "name": schema_name,
+            "fields": fields
+        }.to_json
+        schema = Avro::Schema.parse(schema_json)
+
+        avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
+        avro.encode(record.map{|key, value| [key, value.to_s]}.to_h, schema: schema, subject: "#{schema_name}-value")
+      end
     elsif @output_data_type =~ /^attr:(.*)$/
       @custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
       @custom_attributes.unshift('time') if @output_include_time
@@ -210,7 +231,7 @@ DESC
 
   def write(chunk)
     tag = chunk.key
-    def_topic = @default_topic || tag
+    def_topic = @default_topic || tag.to_s.gsub('.', '_')
     producer = get_producer
 
     records_by_topic = {}
@@ -277,3 +298,4 @@ DESC
     raise e
   end
 end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka-enchanced
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.11
 platform: ruby
 authors:
 - Paschenko Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-24 00:00:00.000000000 Z
+date: 2017-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -152,7 +152,7 @@ files:
 - lib/fluent/plugin/out_kafka_buffered.rb
 - test/helper.rb
 - test/plugin/test_out_kafka.rb
-homepage: https://github.com/fluent/fluent-plugin-kafka
+homepage: https://github.com/zhron4x/fluent-plugin-kafka-enchanced
 licenses: []
 metadata: {}
 post_install_message: