fluent-plugin-kafka-enchanced 0.5.13 → 0.5.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/out_kafka.rb +4 -3
- data/lib/fluent/plugin/out_kafka_buffered.rb +3 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa418e81830e8815303b31ae7d4a76974adc0d76
|
4
|
+
data.tar.gz: a1baed0c8eecc19256606ea5900530f243c38b8a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b97ac0ec71bd70ae36b6241f2373a766e05d77a39977cc118e78cd9e5d7065f90dc8b92c816709ff240b39090fa85d0133644ca2ef32288be1b21e60952b2137
|
7
|
+
data.tar.gz: 29e7f1355c94173ae03d19a5eeda9fb91a2a5d61005c6af3dc582852b5878899ec675af742086a9278f5c4083ebbeb2f85ffa15e156e2fab86f046b47e9fd671
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka-enchanced"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.5.
|
15
|
+
gem.version = '0.5.14'
|
16
16
|
gem.required_ruby_version = ">= 2.1.0"
|
17
17
|
|
18
18
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -146,9 +146,10 @@ DESC
|
|
146
146
|
require 'avro_turf/messaging'
|
147
147
|
require "avro/builder"
|
148
148
|
Proc.new do |tag, time, record|
|
149
|
+
record = record.select{|key, value| key.present?}.map{|k, v| [k.tr('[]-', '_').delete('$'), v.to_s]}.to_h
|
149
150
|
record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
|
150
151
|
fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
|
151
|
-
schema_name = "#{tag.to_s.
|
152
|
+
@topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{fields.to_s.hash.abs}"
|
152
153
|
schema_json = {
|
153
154
|
"type": "record",
|
154
155
|
"name": schema_name,
|
@@ -157,7 +158,6 @@ DESC
|
|
157
158
|
schema = Avro::Schema.parse(schema_json)
|
158
159
|
|
159
160
|
avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
|
160
|
-
# binding.pry
|
161
161
|
avro.encode(record.map{|key, value| [key, value.to_s]}.to_h, schema: schema, subject: "#{schema_name}-value")
|
162
162
|
end
|
163
163
|
elsif @output_data_type == 'ltsv'
|
@@ -199,12 +199,12 @@ DESC
|
|
199
199
|
end
|
200
200
|
end
|
201
201
|
record['tag'] = tag if @output_include_tag
|
202
|
-
topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag.to_s.gsub('.', '_')
|
203
202
|
partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
|
204
203
|
partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
|
205
204
|
message_key = (@exclude_message_key ? record.delete('message_key') : record['message_key']) || @default_message_key
|
206
205
|
|
207
206
|
value = @formatter_proc.call(tag, time, record)
|
207
|
+
topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || @topic_name
|
208
208
|
|
209
209
|
log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}.") }
|
210
210
|
producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
|
@@ -223,3 +223,4 @@ DESC
|
|
223
223
|
end
|
224
224
|
|
225
225
|
|
226
|
+
|
@@ -199,10 +199,10 @@ DESC
|
|
199
199
|
require 'avro_turf/messaging'
|
200
200
|
require "avro/builder"
|
201
201
|
Proc.new do |tag, time, record|
|
202
|
-
record = record.select{|key, value| key.present?}
|
202
|
+
record = record.select{|key, value| key.present?}.map{|k, v| [k.tr('[]-', '_').delete('$'), v.to_s]}.to_h
|
203
203
|
record['enchilada_timestamp'] = (Time.new).strftime('%s%3N')
|
204
204
|
fields = record.keys.map{|key| {'name' => key, 'type' => 'string'}}
|
205
|
-
schema_name = "#{tag.to_s.
|
205
|
+
@topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{fields.to_s.hash.abs}"
|
206
206
|
schema_json = {
|
207
207
|
"type": "record",
|
208
208
|
"name": schema_name,
|
@@ -299,3 +299,4 @@ DESC
|
|
299
299
|
end
|
300
300
|
end
|
301
301
|
|
302
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka-enchanced
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Paschenko Konstantin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-02-
|
11
|
+
date: 2017-02-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|