fluent-plugin-kafka 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +12 -9
- data/Rakefile +5 -4
- data/fluent-plugin-kafka.gemspec +5 -6
- data/lib/fluent/plugin/out_kafka_buffered.rb +11 -7
- metadata +27 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5d2a69320d98e30694e5599b3bf38bd492ba1338
|
4
|
+
data.tar.gz: 5faf0f964137b61e90c55bd7f2dccd3fc8022ab6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b21ee5f9e9c0e6a163aa4d82e0c0c028da102441da56868beac52d8cefa755ac08d38e8f225733b408a00f055e31bfa47766497863f812c675478bcc587ff881
|
7
|
+
data.tar.gz: 2e5f7d4e225822f14559cda6fb4c5d103e0517d63fdc61b3fe3356db90ab494125d606a0deeaa308eaddab809d08ff52ba974ad0b70724a8f1c7e8596ec8a41c
|
data/README.md
CHANGED
@@ -70,7 +70,6 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
|
|
70
70
|
@type kafka_group
|
71
71
|
brokers <list of broker-host:port, separate with comma, must set>
|
72
72
|
zookeepers <list of broker-host:port, separate with comma, must set>
|
73
|
-
zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for brokers
|
74
73
|
consumer_group <consumer group name, must set>
|
75
74
|
topics <listening topics(separate with comma',')>
|
76
75
|
format <input text type (text|json|ltsv|msgpack)>
|
@@ -94,6 +93,8 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
|
|
94
93
|
|
95
94
|
### Output plugin (non-buffered)
|
96
95
|
|
96
|
+
This plugin uses Poseidon producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead.
|
97
|
+
|
97
98
|
<match *.**>
|
98
99
|
@type kafka
|
99
100
|
|
@@ -141,6 +142,8 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
|
|
141
142
|
|
142
143
|
### Buffered output plugin
|
143
144
|
|
145
|
+
This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
|
146
|
+
|
144
147
|
<match *.**>
|
145
148
|
@type kafka_buffered
|
146
149
|
|
@@ -155,20 +158,20 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
|
|
155
158
|
output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
156
159
|
output_include_tag (true|false) :default => false
|
157
160
|
output_include_time (true|false) :default => false
|
158
|
-
max_send_retries (integer) :default =>
|
161
|
+
max_send_retries (integer) :default => 1
|
159
162
|
required_acks (integer) :default => 0
|
160
|
-
|
161
|
-
compression_codec (
|
163
|
+
ack_timeout (integer) :default => 5
|
164
|
+
compression_codec (gzip|snappy) :default => none
|
162
165
|
</match>
|
163
166
|
|
164
|
-
Supports following
|
167
|
+
Supports following ruby-kafka's producer options.
|
165
168
|
|
166
|
-
- max_send_retries — default:
|
169
|
+
- max_send_retries — default: 1 — Number of times to retry sending of messages to a leader.
|
167
170
|
- required_acks — default: 0 — The number of acks required per request.
|
168
|
-
-
|
169
|
-
- compression_codec - default:
|
171
|
+
- ack_timeout — default: 5 — How long the producer waits for acks. The unit is seconds.
|
172
|
+
- compression_codec - default: nil - The codec the producer uses to compress messages.
|
170
173
|
|
171
|
-
See also [
|
174
|
+
See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about Poseidon.
|
172
175
|
|
173
176
|
This plugin supports compression codec "snappy" also.
|
174
177
|
Install snappy module before you use snappy compression.
|
data/Rakefile
CHANGED
@@ -1,11 +1,12 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'bundler'
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
3
|
|
4
4
|
require 'rake/testtask'
|
5
|
+
|
5
6
|
Rake::TestTask.new(:test) do |test|
|
6
7
|
test.libs << 'lib' << 'test'
|
7
|
-
test.
|
8
|
+
test.test_files = FileList['test/**/test_*.rb']
|
8
9
|
test.verbose = true
|
9
10
|
end
|
10
11
|
|
11
|
-
task :default => :
|
12
|
+
task :default => [:build]
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,13 +12,12 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.2.
|
16
|
-
gem.
|
17
|
-
gem.add_dependency 'fluentd'
|
15
|
+
gem.version = '0.2.1'
|
16
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
18
17
|
gem.add_dependency 'poseidon_cluster'
|
19
18
|
gem.add_dependency 'ltsv'
|
20
|
-
gem.add_dependency 'yajl-ruby'
|
21
|
-
gem.add_dependency 'msgpack'
|
22
19
|
gem.add_dependency 'zookeeper'
|
23
|
-
gem.add_dependency 'ruby-kafka', '~> 0.3.
|
20
|
+
gem.add_dependency 'ruby-kafka', '~> 0.3.9'
|
21
|
+
gem.add_development_dependency "rake", ">= 0.9.2"
|
22
|
+
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
24
23
|
end
|
@@ -143,8 +143,12 @@ DESC
|
|
143
143
|
@kafka = nil
|
144
144
|
end
|
145
145
|
|
146
|
-
def
|
147
|
-
|
146
|
+
def emit(tag, es, chain)
|
147
|
+
super(tag, es, chain, tag)
|
148
|
+
end
|
149
|
+
|
150
|
+
def format_stream(tag, es)
|
151
|
+
es.to_msgpack_stream
|
148
152
|
end
|
149
153
|
|
150
154
|
def shutdown_producers
|
@@ -189,13 +193,13 @@ DESC
|
|
189
193
|
else
|
190
194
|
@formatter = Fluent::Plugin.new_formatter(@output_data_type)
|
191
195
|
@formatter.configure(conf)
|
192
|
-
|
193
|
-
@formatter.format(tag, time, record)
|
194
|
-
}
|
196
|
+
@formatter.method(:format)
|
195
197
|
end
|
196
198
|
end
|
197
199
|
|
198
200
|
def write(chunk)
|
201
|
+
tag = chunk.key
|
202
|
+
def_topic = @default_topic || tag
|
199
203
|
producer = get_producer
|
200
204
|
|
201
205
|
records_by_topic = {}
|
@@ -203,7 +207,7 @@ DESC
|
|
203
207
|
messages = 0
|
204
208
|
messages_bytes = 0
|
205
209
|
begin
|
206
|
-
chunk.msgpack_each { |
|
210
|
+
chunk.msgpack_each { |time, record|
|
207
211
|
if @output_include_time
|
208
212
|
if @time_format
|
209
213
|
record['time'] = Time.at(time).strftime(@time_format)
|
@@ -213,7 +217,7 @@ DESC
|
|
213
217
|
end
|
214
218
|
|
215
219
|
record['tag'] = tag if @output_include_tag
|
216
|
-
topic = record['topic'] ||
|
220
|
+
topic = record['topic'] || def_topic
|
217
221
|
partition_key = record['partition_key'] || @default_partition_key
|
218
222
|
|
219
223
|
records_by_topic[topic] ||= 0
|
metadata
CHANGED
@@ -1,43 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06
|
11
|
+
date: 2016-07-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: test-unit
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ">="
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: fluentd
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
30
16
|
requirements:
|
31
17
|
- - ">="
|
32
18
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
19
|
+
version: 0.10.58
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
34
23
|
type: :runtime
|
35
24
|
prerelease: false
|
36
25
|
version_requirements: !ruby/object:Gem::Requirement
|
37
26
|
requirements:
|
38
27
|
- - ">="
|
39
28
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
29
|
+
version: 0.10.58
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
41
33
|
- !ruby/object:Gem::Dependency
|
42
34
|
name: poseidon_cluster
|
43
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,7 +59,7 @@ dependencies:
|
|
67
59
|
- !ruby/object:Gem::Version
|
68
60
|
version: '0'
|
69
61
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
62
|
+
name: zookeeper
|
71
63
|
requirement: !ruby/object:Gem::Requirement
|
72
64
|
requirements:
|
73
65
|
- - ">="
|
@@ -81,47 +73,47 @@ dependencies:
|
|
81
73
|
- !ruby/object:Gem::Version
|
82
74
|
version: '0'
|
83
75
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
76
|
+
name: ruby-kafka
|
85
77
|
requirement: !ruby/object:Gem::Requirement
|
86
78
|
requirements:
|
87
|
-
- - "
|
79
|
+
- - "~>"
|
88
80
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
81
|
+
version: 0.3.9
|
90
82
|
type: :runtime
|
91
83
|
prerelease: false
|
92
84
|
version_requirements: !ruby/object:Gem::Requirement
|
93
85
|
requirements:
|
94
|
-
- - "
|
86
|
+
- - "~>"
|
95
87
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
88
|
+
version: 0.3.9
|
97
89
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
90
|
+
name: rake
|
99
91
|
requirement: !ruby/object:Gem::Requirement
|
100
92
|
requirements:
|
101
93
|
- - ">="
|
102
94
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
104
|
-
type: :
|
95
|
+
version: 0.9.2
|
96
|
+
type: :development
|
105
97
|
prerelease: false
|
106
98
|
version_requirements: !ruby/object:Gem::Requirement
|
107
99
|
requirements:
|
108
100
|
- - ">="
|
109
101
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
102
|
+
version: 0.9.2
|
111
103
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
104
|
+
name: test-unit
|
113
105
|
requirement: !ruby/object:Gem::Requirement
|
114
106
|
requirements:
|
115
|
-
- - "
|
107
|
+
- - ">="
|
116
108
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
118
|
-
type: :
|
109
|
+
version: 3.0.8
|
110
|
+
type: :development
|
119
111
|
prerelease: false
|
120
112
|
version_requirements: !ruby/object:Gem::Requirement
|
121
113
|
requirements:
|
122
|
-
- - "
|
114
|
+
- - ">="
|
123
115
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
116
|
+
version: 3.0.8
|
125
117
|
description: Fluentd plugin for Apache Kafka > 0.8
|
126
118
|
email:
|
127
119
|
- togachiro@gmail.com
|
@@ -159,7 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
159
151
|
version: '0'
|
160
152
|
requirements: []
|
161
153
|
rubyforge_project:
|
162
|
-
rubygems_version: 2.
|
154
|
+
rubygems_version: 2.5.1
|
163
155
|
signing_key:
|
164
156
|
specification_version: 4
|
165
157
|
summary: Fluentd plugin for Apache Kafka > 0.8
|