fluent-plugin-kafka 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +19 -0
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/in_kafka.rb +77 -15
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1fff4cfe054606b489a5bba65e1dd904801bd745
|
4
|
+
data.tar.gz: 8450250a2968bd4104c014f2b6ba2e801719a266
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 171f15df160c6522191f6ced594e1dfb46f6a84698bba82dcbc3ff7c2a73ea508b68eb19c407b9f673b696f62357a5fd51e6cc6cf326983c8405f68c930bd4ad
|
7
|
+
data.tar.gz: d0d95aa95b9b4542cb371812216f11bb4475c2a27868e7ff248278f8698be68cd6b70d83115201750901f202fc4fdb131cd6d37ebe29e0c1419b9125e5f90aa0
|
data/README.md
CHANGED
@@ -43,6 +43,25 @@ Supports following Poseidon::PartitionConsumer options.
|
|
43
43
|
- min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
|
44
44
|
- socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
|
45
45
|
|
46
|
+
Supports a start of processing from the assigned offset for specific topics.
|
47
|
+
|
48
|
+
<source>
|
49
|
+
type kafka
|
50
|
+
host <broker host>
|
51
|
+
port <broker port: default=9092>
|
52
|
+
format <input text type (text|json|ltsv|msgpack)>
|
53
|
+
<topic>
|
54
|
+
topic <listening topic>
|
55
|
+
partition <listening partition: default=0>
|
56
|
+
offset <listening start offset: default=-1>
|
57
|
+
</topic>
|
58
|
+
<topic>
|
59
|
+
topic <listening topic>
|
60
|
+
partition <listening partition: default=0>
|
61
|
+
offset <listening start offset: default=-1>
|
62
|
+
</topic>
|
63
|
+
</source>
|
64
|
+
|
46
65
|
See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
|
47
66
|
|
48
67
|
### Output plugin (non-buffered)
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.0.
|
15
|
+
gem.version = '0.0.17'
|
16
16
|
gem.add_dependency 'fluentd'
|
17
17
|
gem.add_dependency 'poseidon'
|
18
18
|
gem.add_dependency 'ltsv'
|
@@ -8,12 +8,13 @@ class KafkaInput < Input
|
|
8
8
|
config_param :host, :string, :default => 'localhost'
|
9
9
|
config_param :port, :integer, :default => 9092
|
10
10
|
config_param :interval, :integer, :default => 1 # seconds
|
11
|
-
config_param :topics, :string
|
11
|
+
config_param :topics, :string, :default => nil
|
12
12
|
config_param :client_id, :string, :default => 'kafka'
|
13
13
|
config_param :partition, :integer, :default => 0
|
14
14
|
config_param :offset, :integer, :default => -1
|
15
15
|
config_param :add_prefix, :string, :default => nil
|
16
16
|
config_param :add_suffix, :string, :default => nil
|
17
|
+
config_param :add_offset_in_record, :bool, :default => false
|
17
18
|
|
18
19
|
# poseidon PartitionConsumer options
|
19
20
|
config_param :max_bytes, :integer, :default => nil
|
@@ -28,9 +29,25 @@ class KafkaInput < Input
|
|
28
29
|
|
29
30
|
def configure(conf)
|
30
31
|
super
|
31
|
-
|
32
|
+
|
33
|
+
@topic_list = []
|
34
|
+
if @topics
|
35
|
+
@topic_list = @topics.split(',').map { |topic|
|
36
|
+
TopicEntry.new(topic.strip, @partition, @offset)
|
37
|
+
}
|
38
|
+
else
|
39
|
+
conf.elements.select { |element| element.name == 'topic' }.each do |element|
|
40
|
+
unless element.has_key?('topic')
|
41
|
+
raise ConfigError, "kafka: 'topic' is a require parameter in 'topic element'."
|
42
|
+
end
|
43
|
+
partition = element.has_key?('partition') ? element['partition'].to_i : 0
|
44
|
+
offset = element.has_key?('offset') ? element['offset'].to_i : -1
|
45
|
+
@topic_list.push(TopicEntry.new(element['topic'], partition, offset))
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
32
49
|
if @topic_list.empty?
|
33
|
-
raise ConfigError, "kafka: 'topics' is a require parameter"
|
50
|
+
raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
|
34
51
|
end
|
35
52
|
|
36
53
|
case @format
|
@@ -51,8 +68,19 @@ class KafkaInput < Input
|
|
51
68
|
opt[:min_bytes] = @min_bytes if @min_bytes
|
52
69
|
opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
|
53
70
|
|
54
|
-
@topic_watchers = @topic_list.map {|
|
55
|
-
TopicWatcher.new(
|
71
|
+
@topic_watchers = @topic_list.map {|topic_entry|
|
72
|
+
TopicWatcher.new(
|
73
|
+
topic_entry,
|
74
|
+
@host,
|
75
|
+
@port,
|
76
|
+
@client_id,
|
77
|
+
interval,
|
78
|
+
@format,
|
79
|
+
@message_key,
|
80
|
+
@add_offset_in_record,
|
81
|
+
@add_prefix,
|
82
|
+
@add_suffix,
|
83
|
+
opt)
|
56
84
|
}
|
57
85
|
@topic_watchers.each {|tw|
|
58
86
|
tw.attach(@loop)
|
@@ -72,21 +100,22 @@ class KafkaInput < Input
|
|
72
100
|
end
|
73
101
|
|
74
102
|
class TopicWatcher < Coolio::TimerWatcher
|
75
|
-
def initialize(
|
76
|
-
@
|
103
|
+
def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, options={})
|
104
|
+
@topic_entry = topic_entry
|
105
|
+
@add_offset_in_record = add_offset_in_record
|
77
106
|
@callback = method(:consume)
|
78
107
|
@format = format
|
79
108
|
@message_key = message_key
|
80
109
|
@add_prefix = add_prefix
|
81
110
|
@add_suffix = add_suffix
|
82
111
|
@consumer = Poseidon::PartitionConsumer.new(
|
83
|
-
client_id,
|
84
|
-
host,
|
85
|
-
port,
|
86
|
-
topic,
|
87
|
-
partition,
|
88
|
-
offset,
|
89
|
-
options
|
112
|
+
client_id, # client_id
|
113
|
+
host, # host
|
114
|
+
port, # port
|
115
|
+
topic_entry.topic, # topic
|
116
|
+
topic_entry.partition, # partition
|
117
|
+
topic_entry.offset, # offset
|
118
|
+
options # options
|
90
119
|
)
|
91
120
|
|
92
121
|
super(interval, true)
|
@@ -102,12 +131,13 @@ class KafkaInput < Input
|
|
102
131
|
|
103
132
|
def consume
|
104
133
|
es = MultiEventStream.new
|
105
|
-
tag = @topic
|
134
|
+
tag = @topic_entry.topic
|
106
135
|
tag = @add_prefix + "." + tag if @add_prefix
|
107
136
|
tag = tag + "." + @add_suffix if @add_suffix
|
108
137
|
@consumer.fetch.each { |msg|
|
109
138
|
begin
|
110
139
|
msg_record = parse_line(msg.value)
|
140
|
+
msg_record = decorate_offset(msg_record, msg.offset) if @add_offset_in_record
|
111
141
|
es.add(Time.now.to_i, msg_record)
|
112
142
|
rescue
|
113
143
|
$log.warn msg_record.to_s, :error=>$!.to_s
|
@@ -134,7 +164,39 @@ class KafkaInput < Input
|
|
134
164
|
end
|
135
165
|
parsed_record
|
136
166
|
end
|
167
|
+
|
168
|
+
def decorate_offset(record, offset)
|
169
|
+
case @format
|
170
|
+
when 'json'
|
171
|
+
add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
|
172
|
+
when 'ltsv'
|
173
|
+
record.each { |line|
|
174
|
+
add_offset_in_hash(line, @topic_entry.topic, @topic_entry.partition, offset)
|
175
|
+
}
|
176
|
+
when 'msgpack'
|
177
|
+
add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
|
178
|
+
when 'text'
|
179
|
+
add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
|
180
|
+
end
|
181
|
+
record
|
182
|
+
end
|
183
|
+
|
184
|
+
def add_offset_in_hash(hash, topic, partition, offset)
|
185
|
+
hash['kafka_topic'] = topic
|
186
|
+
hash['kafka_partition'] = partition
|
187
|
+
hash['kafka_offset'] = offset
|
188
|
+
end
|
137
189
|
end
|
190
|
+
|
191
|
+
class TopicEntry
|
192
|
+
def initialize(topic, partition, offset)
|
193
|
+
@topic = topic
|
194
|
+
@partition = partition
|
195
|
+
@offset = offset
|
196
|
+
end
|
197
|
+
attr_reader :topic, :partition, :offset
|
198
|
+
end
|
199
|
+
|
138
200
|
end
|
139
201
|
|
140
202
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|