fluent-plugin-kafka-enchanced 0.5.5 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb34d3f0443785e60ce856eadb669dc792b57361
-  data.tar.gz: 3beac44b51f83fb27402decd2f8d7f967a650966
+  metadata.gz: 4a798dba21caaa8ac5fb099baf13b5fd08897b5c
+  data.tar.gz: cb40184d340175e0c9cc1758c2a523f0c9967f74
 SHA512:
-  metadata.gz: 52e92c7c25b9484c2d8ef863610df34d8244b94f38a527d58ae27fd37bdeea9a19288ac07bd0ff273d96c3b7d1831e2c7879baf87eac0fe0be74d2c321ac7f9d
-  data.tar.gz: 37074e95f5f24aeb4e23a9c412525a38b97528725f094a1740a0b148f5e23eecf498fcb2d9b252b51c09d8bf85cfd697b5126abbdff88f9ac6b77e77ed257bcb
+  metadata.gz: f23f912de694240a799d3f80d6c3047a267bec9dd6731a174f6507dbe5928e49afd38db9ce7214ef8a6889d90b873b98086876e352c430badca996b0dfc2afaf
+  data.tar.gz: 00d03a2a336e8361b92c8622c86be9dc959c4d041af1903104e0d914cc69e81857b6cdaae65549ddaeec8099d33a34513a9e1f181777e62854efbcc48ca10979

data/fluent-plugin-kafka.gemspec

@@ -12,12 +12,13 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka-enchanced"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.5'
+  gem.version       = '0.5.6'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
   gem.add_dependency 'avro-builder'
+  gem.add_dependency 'avro-schema-registry'
   gem.add_dependency 'avro_turf'
   gem.add_dependency 'ruby-kafka', '= 0.3.16.beta2'
   gem.add_development_dependency "rake", ">= 0.9.2"

data/lib/fluent/plugin/out_kafka.rb

@@ -1,5 +1,6 @@
 require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
+require 'pry'
 
 class Fluent::KafkaOutput < Fluent::Output
   Fluent::Plugin.register_output('kafka', self)
@@ -12,6 +13,8 @@ Note that you can choose to use either brokers or zookeeper.
 DESC
   config_param :zookeeper, :string, :default => nil,
                :desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
+  config_param :schema_registry, :string, :default => nil,
+               :desc => "Set Avro Schema Registry: <schema_registry_host>:<schema_registry_port>"
   config_param :zookeeper_path, :string, :default => '/brokers/ids',
                :desc => "Path in path for Broker id. Default to /brokers/ids"
   config_param :default_topic, :string, :default => nil,
@@ -21,7 +24,7 @@ DESC
   config_param :default_partition, :integer, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json',
-               :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
+               :desc => "Supported format: (json|avro|ltsv|msgpack|attr:<record name>|<formatter name>)"
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
   config_param :exclude_partition_key, :bool, :default => false,
@@ -68,7 +71,8 @@ DESC
 
     require 'kafka'
     require "avro_turf"
-    require "avro-builder"
+    require "avro/builder"
+    require 'avro-schema-registry'
 
     @kafka = nil
   end
@@ -140,6 +144,15 @@ DESC
     if @output_data_type == 'json'
       require 'yajl'
       Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+    elsif @output_data_type == 'avro'
+      fields = JSON.parse(value).keys.map{|key| {'name' => key, 'type' => 'string'}}.to_s.gsub('=>', ':')
+      schema_json =
+          <<-JSON
+        {"type": "record",
+          "name": "#{topic}_value",
+          "fields": #{fields}}
+      JSON
+      schema = Avro::Schema.parse(schema_json)
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
       Proc.new { |tag, time, record| LTSV.dump(record) }
@@ -200,3 +213,4 @@ DESC
   end
 
 end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka-enchanced
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.6
 platform: ruby
 authors:
 - Paschenko Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-15 00:00:00.000000000 Z
+date: 2017-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: avro-schema-registry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: avro_turf
   requirement: !ruby/object:Gem::Requirement