fluent-plugin-kafka 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -16
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/in_kafka.rb +28 -14
- data/lib/fluent/plugin/in_kafka_group.rb +26 -13
- data/lib/fluent/plugin/out_kafka.rb +22 -8
- data/lib/fluent/plugin/out_kafka_buffered.rb +30 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1359c3e9e2c824e3ad36cf21eae0eeae3aea80f4
|
4
|
+
data.tar.gz: 0593adb6cac966b2461b52cacdc9b4d7be06a112
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d23dcc2070e66fb1eea47ad58318489db7c3ecc9b0e1dbf7fac434266bdfebcd5ae8933fd91208ca71c3422f00a2b354bc71425301a0ce3b8ccc036c774d2dae
|
7
|
+
data.tar.gz: b72d588c40e65e30be3f491743898f8cc0585430191c20b5d40f35eb2e26f283092eb96d6d9350ed89f49173791ae757ed7928a6146fbc63d9ad83cbb19e5d90
|
data/README.md
CHANGED
@@ -19,10 +19,10 @@ Or install it yourself as:
|
|
19
19
|
|
20
20
|
## Usage
|
21
21
|
|
22
|
-
### Input plugin (type 'kafka')
|
22
|
+
### Input plugin (@type 'kafka')
|
23
23
|
|
24
24
|
<source>
|
25
|
-
type
|
25
|
+
@type kafka
|
26
26
|
host <broker host>
|
27
27
|
port <broker port: default=9092>
|
28
28
|
topics <listening topics(separate with comma',')>
|
@@ -46,7 +46,7 @@ Supports following Poseidon::PartitionConsumer options.
|
|
46
46
|
Supports a start of processing from the assigned offset for specific topics.
|
47
47
|
|
48
48
|
<source>
|
49
|
-
type
|
49
|
+
@type kafka
|
50
50
|
host <broker host>
|
51
51
|
port <broker port: default=9092>
|
52
52
|
format <input text type (text|json|ltsv|msgpack)>
|
@@ -64,10 +64,10 @@ Supports a start of processing from the assigned offset for specific topics.
|
|
64
64
|
|
65
65
|
See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
|
66
66
|
|
67
|
-
### Input plugin (type 'kafka_group', supports kafka group)
|
67
|
+
### Input plugin (@type 'kafka_group', supports kafka group)
|
68
68
|
|
69
69
|
<source>
|
70
|
-
type kafka_group
|
70
|
+
@type kafka_group
|
71
71
|
brokers <list of broker-host:port, separate with comma, must set>
|
72
72
|
zookeepers <list of broker-host:port, separate with comma, must set>
|
73
73
|
consumer_group <consumer group name, must set>
|
@@ -94,7 +94,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
|
|
94
94
|
### Output plugin (non-buffered)
|
95
95
|
|
96
96
|
<match *.**>
|
97
|
-
type
|
97
|
+
@type kafka
|
98
98
|
|
99
99
|
# Brokers: you can choose either brokers or zookeeper.
|
100
100
|
brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
|
@@ -141,7 +141,7 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
|
|
141
141
|
### Buffered output plugin
|
142
142
|
|
143
143
|
<match *.**>
|
144
|
-
type
|
144
|
+
@type kafka_buffered
|
145
145
|
|
146
146
|
# Brokers: you can choose either brokers or zookeeper.
|
147
147
|
brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
|
@@ -174,17 +174,17 @@ Install snappy module before you use snappy compression.
|
|
174
174
|
|
175
175
|
$ gem install snappy
|
176
176
|
|
177
|
-
|
177
|
+
#### Load balancing
|
178
178
|
|
179
|
-
|
180
|
-
|
179
|
+
Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
|
180
|
+
If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
|
181
181
|
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
182
|
+
|default_partition_key|partition_key| behavior |
|
183
|
+
|-|-|
|
184
|
+
|Not set|Not exists| All messages are sent in round-robin |
|
185
|
+
|Set| Not exists| All messages are sent to specific broker |
|
186
|
+
|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
|
187
|
+
|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
|
188
188
|
|
189
189
|
## Contributing
|
190
190
|
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.1.
|
15
|
+
gem.version = '0.1.3'
|
16
16
|
gem.add_dependency 'fluentd'
|
17
17
|
gem.add_dependency 'poseidon_cluster'
|
18
18
|
gem.add_dependency 'ltsv'
|
@@ -3,27 +3,41 @@ module Fluent
|
|
3
3
|
class KafkaInput < Input
|
4
4
|
Plugin.register_input('kafka', self)
|
5
5
|
|
6
|
-
config_param :format, :string, :default => 'json'
|
7
|
-
|
8
|
-
config_param :
|
9
|
-
|
10
|
-
config_param :
|
11
|
-
|
6
|
+
config_param :format, :string, :default => 'json',
|
7
|
+
:desc => "Supported format: (json|text|ltsv|msgpack)"
|
8
|
+
config_param :message_key, :string, :default => 'message',
|
9
|
+
:desc => "For 'text' format only."
|
10
|
+
config_param :host, :string, :default => 'localhost',
|
11
|
+
:desc => "Broker host"
|
12
|
+
config_param :port, :integer, :default => 9092,
|
13
|
+
:desc => "Broker port"
|
14
|
+
config_param :interval, :integer, :default => 1, # seconds
|
15
|
+
:desc => "Interval (Unit: seconds)"
|
16
|
+
config_param :topics, :string, :default => nil,
|
17
|
+
:desc => "Listening topics(separate with comma',')"
|
12
18
|
config_param :client_id, :string, :default => 'kafka'
|
13
|
-
config_param :partition, :integer, :default => 0
|
14
|
-
|
15
|
-
config_param :
|
16
|
-
|
19
|
+
config_param :partition, :integer, :default => 0,
|
20
|
+
:desc => "Listening partition"
|
21
|
+
config_param :offset, :integer, :default => -1,
|
22
|
+
:desc => "Listening start offset"
|
23
|
+
config_param :add_prefix, :string, :default => nil,
|
24
|
+
:desc => "Tag prefix"
|
25
|
+
config_param :add_suffix, :string, :default => nil,
|
26
|
+
:desc => "tag suffix"
|
17
27
|
config_param :add_offset_in_record, :bool, :default => false
|
18
28
|
|
19
29
|
config_param :offset_zookeeper, :string, :default => nil
|
20
30
|
config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
|
21
31
|
|
22
32
|
# poseidon PartitionConsumer options
|
23
|
-
config_param :max_bytes, :integer, :default => nil
|
24
|
-
|
25
|
-
config_param :
|
26
|
-
|
33
|
+
config_param :max_bytes, :integer, :default => nil,
|
34
|
+
:desc => "Maximum number of bytes to fetch."
|
35
|
+
config_param :max_wait_ms, :integer, :default => nil,
|
36
|
+
:desc => "How long to block until the server sends us data."
|
37
|
+
config_param :min_bytes, :integer, :default => nil,
|
38
|
+
:desc => "Smallest amount of data the server should send us."
|
39
|
+
config_param :socket_timeout_ms, :integer, :default => nil,
|
40
|
+
:desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
|
27
41
|
|
28
42
|
unless method_defined?(:router)
|
29
43
|
define_method("router") { Fluent::Engine }
|
@@ -3,21 +3,34 @@ module Fluent
|
|
3
3
|
class KafkaGroupInput < Input
|
4
4
|
Plugin.register_input('kafka_group', self)
|
5
5
|
|
6
|
-
config_param :brokers, :string
|
7
|
-
|
8
|
-
config_param :
|
9
|
-
|
10
|
-
config_param :
|
11
|
-
|
12
|
-
config_param :
|
13
|
-
|
14
|
-
config_param :
|
6
|
+
config_param :brokers, :string,
|
7
|
+
:desc => "List of broker-host:port, separate with comma, must set."
|
8
|
+
config_param :zookeepers, :string,
|
9
|
+
:desc => "List of broker-host:port, separate with comma, must set."
|
10
|
+
config_param :consumer_group, :string, :default => nil,
|
11
|
+
:desc => "Consumer group name, must set."
|
12
|
+
config_param :topics, :string,
|
13
|
+
:desc => "Listening topics(separate with comma',')."
|
14
|
+
config_param :interval, :integer, :default => 1, # seconds
|
15
|
+
:desc => "Interval (Unit: seconds)"
|
16
|
+
config_param :format, :string, :default => 'json',
|
17
|
+
:desc => "Supported format: (json|text|ltsv|msgpack)"
|
18
|
+
config_param :message_key, :string, :default => 'message',
|
19
|
+
:desc => "For 'text' format only."
|
20
|
+
config_param :add_prefix, :string, :default => nil,
|
21
|
+
:desc => "Tag prefix (Optional)"
|
22
|
+
config_param :add_suffix, :string, :default => nil,
|
23
|
+
:desc => "Tag suffix (Optional)"
|
15
24
|
|
16
25
|
# poseidon PartitionConsumer options
|
17
|
-
config_param :max_bytes, :integer, :default => nil
|
18
|
-
|
19
|
-
config_param :
|
20
|
-
|
26
|
+
config_param :max_bytes, :integer, :default => nil,
|
27
|
+
:desc => "Maximum number of bytes to fetch."
|
28
|
+
config_param :max_wait_ms, :integer, :default => nil,
|
29
|
+
:desc => "How long to block until the server sends us data."
|
30
|
+
config_param :min_bytes, :integer, :default => nil,
|
31
|
+
:desc => "Smallest amount of data the server should send us."
|
32
|
+
config_param :socket_timeout_ms, :integer, :default => nil,
|
33
|
+
:desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
|
21
34
|
|
22
35
|
unless method_defined?(:router)
|
23
36
|
define_method("router") { Fluent::Engine }
|
@@ -6,20 +6,32 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
6
6
|
require 'poseidon'
|
7
7
|
end
|
8
8
|
|
9
|
-
config_param :brokers, :string, :default => 'localhost:9092'
|
10
|
-
|
11
|
-
|
9
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
10
|
+
:desc => <<-DESC
|
11
|
+
Set brokers directly
|
12
|
+
<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
|
13
|
+
Note that you can choose to use either brokers or zookeeper.
|
14
|
+
DESC
|
15
|
+
config_param :zookeeper, :string, :default => nil,
|
16
|
+
:desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
|
17
|
+
config_param :default_topic, :string, :default => nil,
|
18
|
+
:desc => "Output topic."
|
12
19
|
config_param :default_partition_key, :string, :default => nil
|
13
20
|
config_param :client_id, :string, :default => 'kafka'
|
14
|
-
config_param :output_data_type, :string, :default => 'json'
|
21
|
+
config_param :output_data_type, :string, :default => 'json',
|
22
|
+
:desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
|
15
23
|
config_param :output_include_tag, :bool, :default => false
|
16
24
|
config_param :output_include_time, :bool, :default => false
|
17
25
|
|
18
26
|
# poseidon producer options
|
19
|
-
config_param :max_send_retries, :integer, :default => 3
|
20
|
-
|
21
|
-
config_param :
|
22
|
-
|
27
|
+
config_param :max_send_retries, :integer, :default => 3,
|
28
|
+
:desc => "Number of times to retry sending of messages to a leader."
|
29
|
+
config_param :required_acks, :integer, :default => 0,
|
30
|
+
:desc => "The number of acks required per request."
|
31
|
+
config_param :ack_timeout_ms, :integer, :default => 1500,
|
32
|
+
:desc => "How long the producer waits for acks."
|
33
|
+
config_param :compression_codec, :string, :default => 'none',
|
34
|
+
:desc => "The codec the producer uses to compress messages."
|
23
35
|
|
24
36
|
attr_accessor :output_data_type
|
25
37
|
attr_accessor :field_separator
|
@@ -38,6 +50,7 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
38
50
|
broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
|
39
51
|
@seed_brokers.push("#{broker['host']}:#{broker['port']}")
|
40
52
|
end
|
53
|
+
z.close
|
41
54
|
log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
|
42
55
|
end
|
43
56
|
begin
|
@@ -141,6 +154,7 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
141
154
|
end
|
142
155
|
rescue Exception => e
|
143
156
|
log.warn("Send exception occurred: #{e}")
|
157
|
+
@producer.close if @producer
|
144
158
|
refresh_producer()
|
145
159
|
raise e
|
146
160
|
end
|
@@ -7,21 +7,41 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
7
7
|
require 'poseidon'
|
8
8
|
end
|
9
9
|
|
10
|
-
config_param :brokers, :string, :default => 'localhost:9092'
|
11
|
-
|
12
|
-
|
10
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
11
|
+
:desc => <<-DESC
|
12
|
+
Set brokers directly:
|
13
|
+
<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
|
14
|
+
Brokers: you can choose to use either brokers or zookeeper.
|
15
|
+
DESC
|
16
|
+
config_param :zookeeper, :string, :default => nil,
|
17
|
+
:desc => <<-DESC
|
18
|
+
Set brokers via Zookeeper:
|
19
|
+
<zookeeper_host>:<zookeeper_port>
|
20
|
+
DESC
|
21
|
+
config_param :default_topic, :string, :default => nil,
|
22
|
+
:desc => "Output topic"
|
13
23
|
config_param :default_partition_key, :string, :default => nil
|
14
24
|
config_param :client_id, :string, :default => 'kafka'
|
15
|
-
config_param :output_data_type, :string, :default => 'json'
|
25
|
+
config_param :output_data_type, :string, :default => 'json',
|
26
|
+
:desc => <<-DESC
|
27
|
+
Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
28
|
+
DESC
|
16
29
|
config_param :output_include_tag, :bool, :default => false
|
17
30
|
config_param :output_include_time, :bool, :default => false
|
18
31
|
config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
|
19
32
|
|
20
33
|
# poseidon producer options
|
21
|
-
config_param :max_send_retries, :integer, :default => 3
|
22
|
-
|
23
|
-
config_param :
|
24
|
-
|
34
|
+
config_param :max_send_retries, :integer, :default => 3,
|
35
|
+
:desc => "Number of times to retry sending of messages to a leader."
|
36
|
+
config_param :required_acks, :integer, :default => 0,
|
37
|
+
:desc => "The number of acks required per request."
|
38
|
+
config_param :ack_timeout_ms, :integer, :default => 1500,
|
39
|
+
:desc => "How long the producer waits for acks."
|
40
|
+
config_param :compression_codec, :string, :default => 'none',
|
41
|
+
:desc => <<-DESC
|
42
|
+
The codec the producer uses to compress messages.
|
43
|
+
Supported codecs: (none|gzip|snappy)
|
44
|
+
DESC
|
25
45
|
|
26
46
|
attr_accessor :output_data_type
|
27
47
|
attr_accessor :field_separator
|
@@ -40,6 +60,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
40
60
|
broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
|
41
61
|
@seed_brokers.push("#{broker['host']}:#{broker['port']}")
|
42
62
|
end
|
63
|
+
z.close
|
43
64
|
log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
|
44
65
|
end
|
45
66
|
begin
|
@@ -156,6 +177,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
156
177
|
end
|
157
178
|
rescue Exception => e
|
158
179
|
log.warn "Send exception occurred: #{e}"
|
180
|
+
@producer.close if @producer
|
159
181
|
refresh_producer()
|
160
182
|
# Raise exception to retry sendind messages
|
161
183
|
raise e
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|