fluent-plugin-kafka 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +16 -16
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/in_kafka.rb +28 -14
- data/lib/fluent/plugin/in_kafka_group.rb +26 -13
- data/lib/fluent/plugin/out_kafka.rb +22 -8
- data/lib/fluent/plugin/out_kafka_buffered.rb +30 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1359c3e9e2c824e3ad36cf21eae0eeae3aea80f4
|
4
|
+
data.tar.gz: 0593adb6cac966b2461b52cacdc9b4d7be06a112
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d23dcc2070e66fb1eea47ad58318489db7c3ecc9b0e1dbf7fac434266bdfebcd5ae8933fd91208ca71c3422f00a2b354bc71425301a0ce3b8ccc036c774d2dae
|
7
|
+
data.tar.gz: b72d588c40e65e30be3f491743898f8cc0585430191c20b5d40f35eb2e26f283092eb96d6d9350ed89f49173791ae757ed7928a6146fbc63d9ad83cbb19e5d90
|
data/README.md
CHANGED
@@ -19,10 +19,10 @@ Or install it yourself as:
|
|
19
19
|
|
20
20
|
## Usage
|
21
21
|
|
22
|
-
### Input plugin (type 'kafka')
|
22
|
+
### Input plugin (@type 'kafka')
|
23
23
|
|
24
24
|
<source>
|
25
|
-
type
|
25
|
+
@type kafka
|
26
26
|
host <broker host>
|
27
27
|
port <broker port: default=9092>
|
28
28
|
topics <listening topics(separate with comma',')>
|
@@ -46,7 +46,7 @@ Supports following Poseidon::PartitionConsumer options.
|
|
46
46
|
Supports a start of processing from the assigned offset for specific topics.
|
47
47
|
|
48
48
|
<source>
|
49
|
-
type
|
49
|
+
@type kafka
|
50
50
|
host <broker host>
|
51
51
|
port <broker port: default=9092>
|
52
52
|
format <input text type (text|json|ltsv|msgpack)>
|
@@ -64,10 +64,10 @@ Supports a start of processing from the assigned offset for specific topics.
|
|
64
64
|
|
65
65
|
See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
|
66
66
|
|
67
|
-
### Input plugin (type 'kafka_group', supports kafka group)
|
67
|
+
### Input plugin (@type 'kafka_group', supports kafka group)
|
68
68
|
|
69
69
|
<source>
|
70
|
-
type kafka_group
|
70
|
+
@type kafka_group
|
71
71
|
brokers <list of broker-host:port, separate with comma, must set>
|
72
72
|
zookeepers <list of broker-host:port, separate with comma, must set>
|
73
73
|
consumer_group <consumer group name, must set>
|
@@ -94,7 +94,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
|
|
94
94
|
### Output plugin (non-buffered)
|
95
95
|
|
96
96
|
<match *.**>
|
97
|
-
type
|
97
|
+
@type kafka
|
98
98
|
|
99
99
|
# Brokers: you can choose either brokers or zookeeper.
|
100
100
|
brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
|
@@ -141,7 +141,7 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
|
|
141
141
|
### Buffered output plugin
|
142
142
|
|
143
143
|
<match *.**>
|
144
|
-
type
|
144
|
+
@type kafka_buffered
|
145
145
|
|
146
146
|
# Brokers: you can choose either brokers or zookeeper.
|
147
147
|
brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
|
@@ -174,17 +174,17 @@ Install snappy module before you use snappy compression.
|
|
174
174
|
|
175
175
|
$ gem install snappy
|
176
176
|
|
177
|
-
|
177
|
+
#### Load balancing
|
178
178
|
|
179
|
-
|
180
|
-
|
179
|
+
Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
|
180
|
+
If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
|
181
181
|
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
182
|
+
|default_partition_key|partition_key| behavior |
|
183
|
+
|-|-|
|
184
|
+
|Not set|Not exists| All messages are sent in round-robin |
|
185
|
+
|Set| Not exists| All messages are sent to specific broker |
|
186
|
+
|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
|
187
|
+
|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
|
188
188
|
|
189
189
|
## Contributing
|
190
190
|
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.1.
|
15
|
+
gem.version = '0.1.3'
|
16
16
|
gem.add_dependency 'fluentd'
|
17
17
|
gem.add_dependency 'poseidon_cluster'
|
18
18
|
gem.add_dependency 'ltsv'
|
@@ -3,27 +3,41 @@ module Fluent
|
|
3
3
|
class KafkaInput < Input
|
4
4
|
Plugin.register_input('kafka', self)
|
5
5
|
|
6
|
-
config_param :format, :string, :default => 'json'
|
7
|
-
|
8
|
-
config_param :
|
9
|
-
|
10
|
-
config_param :
|
11
|
-
|
6
|
+
config_param :format, :string, :default => 'json',
|
7
|
+
:desc => "Supported format: (json|text|ltsv|msgpack)"
|
8
|
+
config_param :message_key, :string, :default => 'message',
|
9
|
+
:desc => "For 'text' format only."
|
10
|
+
config_param :host, :string, :default => 'localhost',
|
11
|
+
:desc => "Broker host"
|
12
|
+
config_param :port, :integer, :default => 9092,
|
13
|
+
:desc => "Broker port"
|
14
|
+
config_param :interval, :integer, :default => 1, # seconds
|
15
|
+
:desc => "Interval (Unit: seconds)"
|
16
|
+
config_param :topics, :string, :default => nil,
|
17
|
+
:desc => "Listening topics(separate with comma',')"
|
12
18
|
config_param :client_id, :string, :default => 'kafka'
|
13
|
-
config_param :partition, :integer, :default => 0
|
14
|
-
|
15
|
-
config_param :
|
16
|
-
|
19
|
+
config_param :partition, :integer, :default => 0,
|
20
|
+
:desc => "Listening partition"
|
21
|
+
config_param :offset, :integer, :default => -1,
|
22
|
+
:desc => "Listening start offset"
|
23
|
+
config_param :add_prefix, :string, :default => nil,
|
24
|
+
:desc => "Tag prefix"
|
25
|
+
config_param :add_suffix, :string, :default => nil,
|
26
|
+
:desc => "tag suffix"
|
17
27
|
config_param :add_offset_in_record, :bool, :default => false
|
18
28
|
|
19
29
|
config_param :offset_zookeeper, :string, :default => nil
|
20
30
|
config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
|
21
31
|
|
22
32
|
# poseidon PartitionConsumer options
|
23
|
-
config_param :max_bytes, :integer, :default => nil
|
24
|
-
|
25
|
-
config_param :
|
26
|
-
|
33
|
+
config_param :max_bytes, :integer, :default => nil,
|
34
|
+
:desc => "Maximum number of bytes to fetch."
|
35
|
+
config_param :max_wait_ms, :integer, :default => nil,
|
36
|
+
:desc => "How long to block until the server sends us data."
|
37
|
+
config_param :min_bytes, :integer, :default => nil,
|
38
|
+
:desc => "Smallest amount of data the server should send us."
|
39
|
+
config_param :socket_timeout_ms, :integer, :default => nil,
|
40
|
+
:desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
|
27
41
|
|
28
42
|
unless method_defined?(:router)
|
29
43
|
define_method("router") { Fluent::Engine }
|
@@ -3,21 +3,34 @@ module Fluent
|
|
3
3
|
class KafkaGroupInput < Input
|
4
4
|
Plugin.register_input('kafka_group', self)
|
5
5
|
|
6
|
-
config_param :brokers, :string
|
7
|
-
|
8
|
-
config_param :
|
9
|
-
|
10
|
-
config_param :
|
11
|
-
|
12
|
-
config_param :
|
13
|
-
|
14
|
-
config_param :
|
6
|
+
config_param :brokers, :string,
|
7
|
+
:desc => "List of broker-host:port, separate with comma, must set."
|
8
|
+
config_param :zookeepers, :string,
|
9
|
+
:desc => "List of broker-host:port, separate with comma, must set."
|
10
|
+
config_param :consumer_group, :string, :default => nil,
|
11
|
+
:desc => "Consumer group name, must set."
|
12
|
+
config_param :topics, :string,
|
13
|
+
:desc => "Listening topics(separate with comma',')."
|
14
|
+
config_param :interval, :integer, :default => 1, # seconds
|
15
|
+
:desc => "Interval (Unit: seconds)"
|
16
|
+
config_param :format, :string, :default => 'json',
|
17
|
+
:desc => "Supported format: (json|text|ltsv|msgpack)"
|
18
|
+
config_param :message_key, :string, :default => 'message',
|
19
|
+
:desc => "For 'text' format only."
|
20
|
+
config_param :add_prefix, :string, :default => nil,
|
21
|
+
:desc => "Tag prefix (Optional)"
|
22
|
+
config_param :add_suffix, :string, :default => nil,
|
23
|
+
:desc => "Tag suffix (Optional)"
|
15
24
|
|
16
25
|
# poseidon PartitionConsumer options
|
17
|
-
config_param :max_bytes, :integer, :default => nil
|
18
|
-
|
19
|
-
config_param :
|
20
|
-
|
26
|
+
config_param :max_bytes, :integer, :default => nil,
|
27
|
+
:desc => "Maximum number of bytes to fetch."
|
28
|
+
config_param :max_wait_ms, :integer, :default => nil,
|
29
|
+
:desc => "How long to block until the server sends us data."
|
30
|
+
config_param :min_bytes, :integer, :default => nil,
|
31
|
+
:desc => "Smallest amount of data the server should send us."
|
32
|
+
config_param :socket_timeout_ms, :integer, :default => nil,
|
33
|
+
:desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
|
21
34
|
|
22
35
|
unless method_defined?(:router)
|
23
36
|
define_method("router") { Fluent::Engine }
|
@@ -6,20 +6,32 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
6
6
|
require 'poseidon'
|
7
7
|
end
|
8
8
|
|
9
|
-
config_param :brokers, :string, :default => 'localhost:9092'
|
10
|
-
|
11
|
-
|
9
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
10
|
+
:desc => <<-DESC
|
11
|
+
Set brokers directly
|
12
|
+
<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
|
13
|
+
Note that you can choose to use either brokers or zookeeper.
|
14
|
+
DESC
|
15
|
+
config_param :zookeeper, :string, :default => nil,
|
16
|
+
:desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
|
17
|
+
config_param :default_topic, :string, :default => nil,
|
18
|
+
:desc => "Output topic."
|
12
19
|
config_param :default_partition_key, :string, :default => nil
|
13
20
|
config_param :client_id, :string, :default => 'kafka'
|
14
|
-
config_param :output_data_type, :string, :default => 'json'
|
21
|
+
config_param :output_data_type, :string, :default => 'json',
|
22
|
+
:desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
|
15
23
|
config_param :output_include_tag, :bool, :default => false
|
16
24
|
config_param :output_include_time, :bool, :default => false
|
17
25
|
|
18
26
|
# poseidon producer options
|
19
|
-
config_param :max_send_retries, :integer, :default => 3
|
20
|
-
|
21
|
-
config_param :
|
22
|
-
|
27
|
+
config_param :max_send_retries, :integer, :default => 3,
|
28
|
+
:desc => "Number of times to retry sending of messages to a leader."
|
29
|
+
config_param :required_acks, :integer, :default => 0,
|
30
|
+
:desc => "The number of acks required per request."
|
31
|
+
config_param :ack_timeout_ms, :integer, :default => 1500,
|
32
|
+
:desc => "How long the producer waits for acks."
|
33
|
+
config_param :compression_codec, :string, :default => 'none',
|
34
|
+
:desc => "The codec the producer uses to compress messages."
|
23
35
|
|
24
36
|
attr_accessor :output_data_type
|
25
37
|
attr_accessor :field_separator
|
@@ -38,6 +50,7 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
38
50
|
broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
|
39
51
|
@seed_brokers.push("#{broker['host']}:#{broker['port']}")
|
40
52
|
end
|
53
|
+
z.close
|
41
54
|
log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
|
42
55
|
end
|
43
56
|
begin
|
@@ -141,6 +154,7 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
141
154
|
end
|
142
155
|
rescue Exception => e
|
143
156
|
log.warn("Send exception occurred: #{e}")
|
157
|
+
@producer.close if @producer
|
144
158
|
refresh_producer()
|
145
159
|
raise e
|
146
160
|
end
|
@@ -7,21 +7,41 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
7
7
|
require 'poseidon'
|
8
8
|
end
|
9
9
|
|
10
|
-
config_param :brokers, :string, :default => 'localhost:9092'
|
11
|
-
|
12
|
-
|
10
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
11
|
+
:desc => <<-DESC
|
12
|
+
Set brokers directly:
|
13
|
+
<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
|
14
|
+
Brokers: you can choose to use either brokers or zookeeper.
|
15
|
+
DESC
|
16
|
+
config_param :zookeeper, :string, :default => nil,
|
17
|
+
:desc => <<-DESC
|
18
|
+
Set brokers via Zookeeper:
|
19
|
+
<zookeeper_host>:<zookeeper_port>
|
20
|
+
DESC
|
21
|
+
config_param :default_topic, :string, :default => nil,
|
22
|
+
:desc => "Output topic"
|
13
23
|
config_param :default_partition_key, :string, :default => nil
|
14
24
|
config_param :client_id, :string, :default => 'kafka'
|
15
|
-
config_param :output_data_type, :string, :default => 'json'
|
25
|
+
config_param :output_data_type, :string, :default => 'json',
|
26
|
+
:desc => <<-DESC
|
27
|
+
Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
28
|
+
DESC
|
16
29
|
config_param :output_include_tag, :bool, :default => false
|
17
30
|
config_param :output_include_time, :bool, :default => false
|
18
31
|
config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
|
19
32
|
|
20
33
|
# poseidon producer options
|
21
|
-
config_param :max_send_retries, :integer, :default => 3
|
22
|
-
|
23
|
-
config_param :
|
24
|
-
|
34
|
+
config_param :max_send_retries, :integer, :default => 3,
|
35
|
+
:desc => "Number of times to retry sending of messages to a leader."
|
36
|
+
config_param :required_acks, :integer, :default => 0,
|
37
|
+
:desc => "The number of acks required per request."
|
38
|
+
config_param :ack_timeout_ms, :integer, :default => 1500,
|
39
|
+
:desc => "How long the producer waits for acks."
|
40
|
+
config_param :compression_codec, :string, :default => 'none',
|
41
|
+
:desc => <<-DESC
|
42
|
+
The codec the producer uses to compress messages.
|
43
|
+
Supported codecs: (none|gzip|snappy)
|
44
|
+
DESC
|
25
45
|
|
26
46
|
attr_accessor :output_data_type
|
27
47
|
attr_accessor :field_separator
|
@@ -40,6 +60,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
40
60
|
broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
|
41
61
|
@seed_brokers.push("#{broker['host']}:#{broker['port']}")
|
42
62
|
end
|
63
|
+
z.close
|
43
64
|
log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
|
44
65
|
end
|
45
66
|
begin
|
@@ -156,6 +177,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
156
177
|
end
|
157
178
|
rescue Exception => e
|
158
179
|
log.warn "Send exception occurred: #{e}"
|
180
|
+
@producer.close if @producer
|
159
181
|
refresh_producer()
|
160
182
|
# Raise exception to retry sendind messages
|
161
183
|
raise e
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|