fluent-plugin-kafka-enchanced 0.5.9 → 0.5.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/out_kafka.rb +20 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fccd71b5e7d89f156703d465fe21780d44b333ce
|
4
|
+
data.tar.gz: 0cee930d56cf656c5c5c0826a8aced5ab14f92bc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f88b27ee87ce61afcffb47528a974ba4f10dc864c56d3e7ef6561d5faf0a9155f85651a1bf3602a6a4fd76792b38908cb94da9b001cd0ef653f50831c0e5c0c6
|
7
|
+
data.tar.gz: 76f4ada2526ef2e986ad89afdebe26a8721cd38407752abdba87f4406ecba4857515af1998d08fa42079eef80de1445dcdeadd1bc4daa009742f5c2ed5644a5f
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka-enchanced"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.5.
|
15
|
+
gem.version = '0.5.10'
|
16
16
|
gem.required_ruby_version = ">= 2.1.0"
|
17
17
|
|
18
18
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -70,8 +70,6 @@ DESC
|
|
70
70
|
super
|
71
71
|
|
72
72
|
require 'kafka'
|
73
|
-
require "avro_turf"
|
74
|
-
require "avro/builder"
|
75
73
|
|
76
74
|
@kafka = nil
|
77
75
|
end
|
@@ -144,14 +142,24 @@ DESC
|
|
144
142
|
require 'yajl'
|
145
143
|
Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
|
146
144
|
elsif @output_data_type == 'avro'
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
145
|
+
require "avro_turf"
|
146
|
+
require 'avro_turf/messaging'
|
147
|
+
require "avro/builder"
|
148
|
+
Proc.new do |tag, time, record|
|
149
|
+
record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
|
150
|
+
fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
|
151
|
+
schema_name = "#{tag.to_s.gsub('.', '_')}"
|
152
|
+
schema_json = {
|
153
|
+
"type": "record",
|
154
|
+
"name": schema_name,
|
155
|
+
"fields": fields
|
156
|
+
}.to_json
|
157
|
+
schema = Avro::Schema.parse(schema_json)
|
158
|
+
|
159
|
+
avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
|
160
|
+
# binding.pry
|
161
|
+
avro.encode(record.map{|key, value| [key, value.to_s]}.to_h, schema: schema, subject: "#{schema_name}-value")
|
162
|
+
end
|
155
163
|
elsif @output_data_type == 'ltsv'
|
156
164
|
require 'ltsv'
|
157
165
|
Proc.new { |tag, time, record| LTSV.dump(record) }
|
@@ -182,6 +190,7 @@ DESC
|
|
182
190
|
producer = @kafka.producer(@producer_opts)
|
183
191
|
|
184
192
|
es.each do |time, record|
|
193
|
+
record = record.select{|key, value| key.present?}
|
185
194
|
if @output_include_time
|
186
195
|
if @time_format
|
187
196
|
record['time'] = Time.at(time).strftime(@time_format)
|
@@ -213,3 +222,4 @@ DESC
|
|
213
222
|
|
214
223
|
end
|
215
224
|
|
225
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka-enchanced
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Paschenko Konstantin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-02-
|
11
|
+
date: 2017-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|