fluent-plugin-kafka-enchanced 0.5.31 → 0.5.32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/out_kafka.rb +1 -1
- data/lib/fluent/plugin/out_kafka_buffered.rb +42 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 33fd86ef94e5d7e9340ed79329dff51ea2f32a4a
|
4
|
+
data.tar.gz: 1caa6add35ca52ee497068a8a68f77e71f8aff76
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 775560dc0091e06d95bbf45c9658c66834e774bf0c019da2f30d0e9ac25b644883568e09dba118ddb5a58baae262f41bb7af52ba6345e4b7d512df254913548d
|
7
|
+
data.tar.gz: 7eba27507c8ee929ea05dba4c845b995717429585d5b3b785b8cd4511353899688f484b3103371b26a94604b4d342b89d1c2fcd2731ddf90dc56466426b89b3d
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka-enchanced"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.5.
|
15
|
+
gem.version = '0.5.32'
|
16
16
|
gem.required_ruby_version = ">= 2.1.0"
|
17
17
|
|
18
18
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -202,23 +202,52 @@ DESC
|
|
202
202
|
require "avro/builder"
|
203
203
|
init_redis
|
204
204
|
Proc.new do |tag, time, record|
|
205
|
-
record = record.select{|key, value| !key.nil? && !key.empty?}.map
|
205
|
+
record = record.select{|key, value| !key.nil? && !key.empty?}.map do |k, v|
|
206
|
+
[
|
207
|
+
k.tr('[]-', '_').delete('$'),
|
208
|
+
(v.is_a?(Fixnum) || v.is_a?(Float) || v.nil? ? v : v.to_s.force_encoding("UTF-8"))
|
209
|
+
]
|
210
|
+
end.to_h
|
206
211
|
timestamp = Time.new
|
207
212
|
record['enchilada_time_with_format'] = timestamp.strftime("%Y-%m-%dT%H:%M:%S.%LZ")
|
213
|
+
@topic_name = schema_name = "#{tag.to_s.tr('.$:', '_')}_#{Digest::MD5.new.hexdigest(record.keys.to_s)[0..5]}"
|
214
|
+
|
215
|
+
avro = AvroTurf::Messaging.new(registry_url: @schema_registry)
|
216
|
+
|
217
|
+
unless (stored_schema = get_schema_from_redis_by_name(schema_name))
|
218
|
+
fields = record.map do |key, value|
|
219
|
+
{
|
220
|
+
'name' => key,
|
221
|
+
'type' => ['null', (value.is_a?(Fixnum) ? 'int' : (value.is_a?(Float) ? 'float' : 'string'))]
|
222
|
+
}
|
223
|
+
end
|
224
|
+
field_types = fields.map{|field| [field['name'], (field['type'] - ['null']).first]}.to_h
|
225
|
+
fields << {"name" => "enchilada_timestamp", "type" => "long"}
|
226
|
+
schema_json = {
|
227
|
+
"type": "record",
|
228
|
+
"name": schema_name,
|
229
|
+
"fields": fields
|
230
|
+
}.to_json
|
231
|
+
registry = avro.instance_variable_get('@registry')
|
232
|
+
schema = Avro::Schema.parse(schema_json)
|
233
|
+
schema_id = registry.register("#{schema_name}-value", schema)
|
234
|
+
|
235
|
+
stored_schema = {
|
236
|
+
'schema_json' => schema_json,
|
237
|
+
'schema_id' => schema_id,
|
238
|
+
'field_types' => field_types
|
239
|
+
}
|
240
|
+
|
241
|
+
set_schema_to_redis(schema_name, stored_schema)
|
242
|
+
|
243
|
+
end
|
208
244
|
|
209
|
-
fields = record.map{|key, value| {'name' => key, 'type' => (value.is_a?(Fixnum) ? 'int' : (value.is_a?(Float) ? 'float' : 'string'))}}
|
210
245
|
record['enchilada_timestamp'] = timestamp.strftime('%s%3N').to_i
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
"type": "record",
|
215
|
-
"name": schema_name,
|
216
|
-
"fields": fields
|
217
|
-
}.to_json
|
218
|
-
schema = Avro::Schema.parse(schema_json)
|
246
|
+
record = record.map do |key, val|
|
247
|
+
[key, (stored_schema['field_types'][key] != 'string' || val.nil? ? val : val.to_s)]
|
248
|
+
end.to_h
|
219
249
|
|
220
|
-
avro
|
221
|
-
avro.encode(record, schema: schema, subject: "#{schema_name}-value")
|
250
|
+
avro.encode(record, stored_schema['schema_id'], schema: Avro::Schema.parse(stored_schema['schema_json']))
|
222
251
|
end
|
223
252
|
elsif @output_data_type =~ /^attr:(.*)$/
|
224
253
|
@custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
|
@@ -318,7 +347,7 @@ DESC
|
|
318
347
|
parsed_schema = JSON.parse($redis.get(schema_name))
|
319
348
|
{
|
320
349
|
'schema_id' => parsed_schema['schema_id'],
|
321
|
-
'
|
350
|
+
'schema_json' => Avro::Schema.parse(parsed_schema['schema_json']),
|
322
351
|
'field_types' => parsed_schema['field_types']
|
323
352
|
}
|
324
353
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka-enchanced
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.32
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Paschenko Konstantin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-03-
|
11
|
+
date: 2017-03-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|