fluent-plugin-kafka 0.0.11 → 0.0.12

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4a35c789b318f3ac5d6fa70545b170a65b3bf0b3
4
- data.tar.gz: 77216b2f697aaf4a0a5b53e237f2f5d73c05105f
3
+ metadata.gz: 69682c218b44b78ebe360adaa07d2defc25c8eb0
4
+ data.tar.gz: b0e9f47b62c7753988933ac6ab1b4221ab38ef92
5
5
  SHA512:
6
- metadata.gz: 109ac7d58c512e2b38c1b9cc847e23ad59e84da495874be2135647526bfc3b99dde08a7aae8ab3255d56d834cb71dde2c1d1f9e65933c10f760d34591b5da66b
7
- data.tar.gz: cabfb8f1f4a67e2e631f13c774b8802e95f683b3042c076c8e27cfe60669e9cb36cc29b9e6947c4a8235ddaf7b47ae3d657df8faeef2d07c678a94a2561e5e06
6
+ metadata.gz: 1187e4ed38b2d93636de7699ba75a1d1759f1dafa0f3aab8b6fa93ff1738230bf7850405809764ec8d1ccabe3d6cf7b6a63d4c730867db442e04facd6081e4f4
7
+ data.tar.gz: 2fd63a22abb00ded8024596c710393cf797965beb94e22a742800b8fd5329483430c0d3fc16458e993e4bd0851456d3956c5a90593915af9a9cce534e5e42789
data/README.md CHANGED
@@ -41,7 +41,7 @@ Or install it yourself as:
41
41
  zookeeper <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
42
42
 
43
43
  default_topic <output topic>
44
- output_data_type (json|ltsv|msgpack|attr:<record name>)
44
+ output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>)
45
45
  output_include_tag (true|false) :default => false
46
46
  output_include_time (true|false) :default => false
47
47
  max_send_retries (integer) :default => 3
@@ -77,7 +77,7 @@ Install snappy module before you use snappy compression.
77
77
  default_topic <output topic>
78
78
  flush_interval <flush interval (sec) :default => 60>
79
79
  buffer_type (file|memory)
80
- output_data_type (json|ltsv|msgpack|attr:<record name>)
80
+ output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>)
81
81
  output_include_tag (true|false) :default => false
82
82
  output_include_time (true|false) :default => false
83
83
  max_send_retries (integer) :default => 3
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.0.11'
15
+ gem.version = '0.0.12'
16
16
  gem.add_dependency 'fluentd'
17
17
  gem.add_dependency 'poseidon'
18
18
  gem.add_dependency 'ltsv'
@@ -85,6 +85,8 @@ class Fluent::KafkaOutput < Fluent::Output
85
85
  elsif @output_data_type =~ /^attr:(.*)$/
86
86
  $1.split(',').map(&:strip).reject(&:empty?)
87
87
  else
88
+ @formatter = Fluent::Plugin.new_formatter(@output_data_type)
89
+ @formatter.configure(conf)
88
90
  nil
89
91
  end
90
92
 
@@ -128,7 +130,8 @@ class Fluent::KafkaOutput < Fluent::Output
128
130
  record['tag'] = tag if @output_include_tag
129
131
  topic = record['topic'] || self.default_topic || tag
130
132
  partition = record['partition'] || self.default_partition
131
- message = Poseidon::MessageToSend.new(topic, parse_record(record))
133
+ value = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record)
134
+ message = Poseidon::MessageToSend.new(topic, value)
132
135
  @producer.send_messages([message])
133
136
  end
134
137
  rescue Exception => e
@@ -91,6 +91,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
91
91
  elsif @output_data_type =~ /^attr:(.*)$/
92
92
  $1.split(',').map(&:strip).reject(&:empty?)
93
93
  else
94
+ @formatter = Fluent::Plugin.new_formatter(@output_data_type)
95
+ @formatter.configure(conf)
94
96
  nil
95
97
  end
96
98
  end
@@ -143,7 +145,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
143
145
  records_by_topic[topic] ||= 0
144
146
  bytes_by_topic[topic] ||= 0
145
147
 
146
- record_buf = parse_record(record)
148
+ record_buf = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record)
147
149
  record_buf_bytes = record_buf.bytesize
148
150
  if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
149
151
  @producer.send_messages(messages)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.11
4
+ version: 0.0.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-04 00:00:00.000000000 Z
11
+ date: 2015-04-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd