fluent-plugin-kafka 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -15
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/out_kafka.rb +13 -3
- data/lib/fluent/plugin/out_kafka_buffered.rb +14 -3
- metadata +17 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 12a7a6c9b46188b26eddffb85896388788d5b63b
|
4
|
+
data.tar.gz: 851a09fe0932c3ec7001d671cac9fded8a707b67
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 71fa902e08bafcc1f29719246626c0d646d24478b859592b41cd06b38fc3f56d1a72bfd37f5e95891da1397d08cba39d117160aa531763a400186870f0a022d7
|
7
|
+
data.tar.gz: 1bf5eaa93b4cf5899e9c82951abb58e80d104eb074869a9d63d532e623fe51d61e71137b5706ce0ee4eb9e5c5626f29feef250306ad9b364abbe81db17464d5c
|
data/README.md
CHANGED
@@ -70,6 +70,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
|
|
70
70
|
@type kafka_group
|
71
71
|
brokers <list of broker-host:port, separate with comma, must set>
|
72
72
|
zookeepers <list of broker-host:port, separate with comma, must set>
|
73
|
+
zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for brokers
|
73
74
|
consumer_group <consumer group name, must set>
|
74
75
|
topics <listening topics(separate with comma',')>
|
75
76
|
format <input text type (text|json|ltsv|msgpack)>
|
@@ -99,7 +100,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
|
|
99
100
|
# Brokers: you can choose either brokers or zookeeper.
|
100
101
|
brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
|
101
102
|
zookeeper <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
|
102
|
-
|
103
|
+
zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
|
103
104
|
default_topic <output topic>
|
104
105
|
default_partition_key (string) :default => nil
|
105
106
|
output_data_type (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
@@ -131,7 +132,7 @@ Messages will be sent broker in a round-robin manner as default by Poseidon, but
|
|
131
132
|
If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
|
132
133
|
|
133
134
|
|default_partition_key|partition_key| behavior |
|
134
|
-
|
135
|
+
| --- | --- | --- |
|
135
136
|
|Not set|Not exists| All messages are sent in round-robin |
|
136
137
|
|Set| Not exists| All messages are sent to specific broker |
|
137
138
|
|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
|
@@ -146,7 +147,7 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
|
|
146
147
|
# Brokers: you can choose either brokers or zookeeper.
|
147
148
|
brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
|
148
149
|
zookeeper <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
|
149
|
-
|
150
|
+
zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
|
150
151
|
default_topic <output topic>
|
151
152
|
default_partition_key (string) :default => nil
|
152
153
|
flush_interval <flush interval (sec) :default => 60>
|
@@ -174,18 +175,6 @@ Install snappy module before you use snappy compression.
|
|
174
175
|
|
175
176
|
$ gem install snappy
|
176
177
|
|
177
|
-
#### Load balancing
|
178
|
-
|
179
|
-
Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
|
180
|
-
If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
|
181
|
-
|
182
|
-
|default_partition_key|partition_key| behavior |
|
183
|
-
|-|-|
|
184
|
-
|Not set|Not exists| All messages are sent in round-robin |
|
185
|
-
|Set| Not exists| All messages are sent to specific broker |
|
186
|
-
|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
|
187
|
-
|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
|
188
|
-
|
189
178
|
## Contributing
|
190
179
|
|
191
180
|
1. Fork it
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.1.
|
15
|
+
gem.version = '0.1.4'
|
16
16
|
gem.add_dependency 'fluentd'
|
17
17
|
gem.add_dependency 'poseidon_cluster'
|
18
18
|
gem.add_dependency 'ltsv'
|
@@ -14,6 +14,8 @@ Note that you can choose to use either brokers or zookeeper.
|
|
14
14
|
DESC
|
15
15
|
config_param :zookeeper, :string, :default => nil,
|
16
16
|
:desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
|
17
|
+
config_param :zookeeper_path, :string, :default => '/brokers/ids',
|
18
|
+
:desc => "Path in path for Broker id. Default to /brokers/ids"
|
17
19
|
config_param :default_topic, :string, :default => nil,
|
18
20
|
:desc => "Output topic."
|
19
21
|
config_param :default_partition_key, :string, :default => nil
|
@@ -33,6 +35,8 @@ DESC
|
|
33
35
|
config_param :compression_codec, :string, :default => 'none',
|
34
36
|
:desc => "The codec the producer uses to compress messages."
|
35
37
|
|
38
|
+
config_param :time_format, :string, :default => nil
|
39
|
+
|
36
40
|
attr_accessor :output_data_type
|
37
41
|
attr_accessor :field_separator
|
38
42
|
|
@@ -46,8 +50,8 @@ DESC
|
|
46
50
|
if @zookeeper
|
47
51
|
@seed_brokers = []
|
48
52
|
z = Zookeeper.new(@zookeeper)
|
49
|
-
z.get_children(:path =>
|
50
|
-
broker = Yajl.load(z.get(:path => "
|
53
|
+
z.get_children(:path => @zookeeper_path)[:children].each do |id|
|
54
|
+
broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
|
51
55
|
@seed_brokers.push("#{broker['host']}:#{broker['port']}")
|
52
56
|
end
|
53
57
|
z.close
|
@@ -143,7 +147,13 @@ DESC
|
|
143
147
|
begin
|
144
148
|
chain.next
|
145
149
|
es.each do |time,record|
|
146
|
-
|
150
|
+
if @output_include_time
|
151
|
+
if @time_format
|
152
|
+
record['time'] = Time.at(time).strftime(@time_format)
|
153
|
+
else
|
154
|
+
record['time'] = time
|
155
|
+
end
|
156
|
+
end
|
147
157
|
record['tag'] = tag if @output_include_tag
|
148
158
|
topic = record['topic'] || self.default_topic || tag
|
149
159
|
partition_key = record['partition_key'] || @default_partition_key
|
@@ -18,6 +18,8 @@ DESC
|
|
18
18
|
Set brokers via Zookeeper:
|
19
19
|
<zookeeper_host>:<zookeeper_port>
|
20
20
|
DESC
|
21
|
+
config_param :zookeeper_path, :string, :default => '/brokers/ids',
|
22
|
+
:desc => "Path in path for Broker id. Default to /brokers/ids"
|
21
23
|
config_param :default_topic, :string, :default => nil,
|
22
24
|
:desc => "Output topic"
|
23
25
|
config_param :default_partition_key, :string, :default => nil
|
@@ -43,6 +45,8 @@ The codec the producer uses to compress messages.
|
|
43
45
|
Supported codecs: (none|gzip|snappy)
|
44
46
|
DESC
|
45
47
|
|
48
|
+
config_param :time_format, :string, :default => nil
|
49
|
+
|
46
50
|
attr_accessor :output_data_type
|
47
51
|
attr_accessor :field_separator
|
48
52
|
|
@@ -56,8 +60,8 @@ DESC
|
|
56
60
|
if @zookeeper
|
57
61
|
@seed_brokers = []
|
58
62
|
z = Zookeeper.new(@zookeeper)
|
59
|
-
z.get_children(:path =>
|
60
|
-
broker = Yajl.load(z.get(:path => "
|
63
|
+
z.get_children(:path => @zookeeper_path)[:children].each do |id|
|
64
|
+
broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
|
61
65
|
@seed_brokers.push("#{broker['host']}:#{broker['port']}")
|
62
66
|
end
|
63
67
|
z.close
|
@@ -146,7 +150,14 @@ DESC
|
|
146
150
|
messages_bytes = 0
|
147
151
|
begin
|
148
152
|
chunk.msgpack_each { |tag, time, record|
|
149
|
-
|
153
|
+
if @output_include_time
|
154
|
+
if @time_format
|
155
|
+
record['time'] = Time.at(time).strftime(@time_format)
|
156
|
+
else
|
157
|
+
record['time'] = time
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
150
161
|
record['tag'] = tag if @output_include_tag
|
151
162
|
topic = record['topic'] || @default_topic || tag
|
152
163
|
partition_key = record['partition_key'] || @default_partition_key
|
metadata
CHANGED
@@ -1,97 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: poseidon_cluster
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ltsv
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: yajl-ruby
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: msgpack
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: zookeeper
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
description: Fluentd plugin for Apache Kafka > 0.8
|
@@ -121,17 +121,17 @@ require_paths:
|
|
121
121
|
- lib
|
122
122
|
required_ruby_version: !ruby/object:Gem::Requirement
|
123
123
|
requirements:
|
124
|
-
- -
|
124
|
+
- - ">="
|
125
125
|
- !ruby/object:Gem::Version
|
126
126
|
version: '0'
|
127
127
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
|
-
- -
|
129
|
+
- - ">="
|
130
130
|
- !ruby/object:Gem::Version
|
131
131
|
version: '0'
|
132
132
|
requirements: []
|
133
133
|
rubyforge_project:
|
134
|
-
rubygems_version: 2.
|
134
|
+
rubygems_version: 2.4.8
|
135
135
|
signing_key:
|
136
136
|
specification_version: 4
|
137
137
|
summary: Fluentd plugin for Apache Kafka > 0.8
|