fluent-plugin-kafka 0.0.16 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a72f6359b8397411999429eeda7366e05114e925
4
- data.tar.gz: b8dc3a2860a941b43d02f8a98562eec997bd065f
3
+ metadata.gz: 1fff4cfe054606b489a5bba65e1dd904801bd745
4
+ data.tar.gz: 8450250a2968bd4104c014f2b6ba2e801719a266
5
5
  SHA512:
6
- metadata.gz: 6af36d8001d89b0b7b181118f66b6a38f3f4a567e70d4a744e3e69496c00a84939f9272aabba9650bf02ee57fe00113b6ca3782b674a20fa108e8892f8f00e81
7
- data.tar.gz: c5a8f2ad75696645cd62a40cc0aad87f471a59119d73fbf04c2be8b0d68d195e33cfb9c2b82afa61312dc2d1160de91182a64686213fec15b37ae725dfbcd043
6
+ metadata.gz: 171f15df160c6522191f6ced594e1dfb46f6a84698bba82dcbc3ff7c2a73ea508b68eb19c407b9f673b696f62357a5fd51e6cc6cf326983c8405f68c930bd4ad
7
+ data.tar.gz: d0d95aa95b9b4542cb371812216f11bb4475c2a27868e7ff248278f8698be68cd6b70d83115201750901f202fc4fdb131cd6d37ebe29e0c1419b9125e5f90aa0
data/README.md CHANGED
@@ -43,6 +43,25 @@ Supports following Poseidon::PartitionConsumer options.
43
43
  - min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
44
44
  - socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
45
45
 
46
+ Supports a start of processing from the assigned offset for specific topics.
47
+
48
+ <source>
49
+ type kafka
50
+ host <broker host>
51
+ port <broker port: default=9092>
52
+ format <input text type (text|json|ltsv|msgpack)>
53
+ <topic>
54
+ topic <listening topic>
55
+ partition <listening partition: default=0>
56
+ offset <listening start offset: default=-1>
57
+ </topic>
58
+ <topic>
59
+ topic <listening topic>
60
+ partition <listening partition: default=0>
61
+ offset <listening start offset: default=-1>
62
+ </topic>
63
+ </source>
64
+
46
65
  See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
47
66
 
48
67
  ### Output plugin (non-buffered)
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.0.16'
15
+ gem.version = '0.0.17'
16
16
  gem.add_dependency 'fluentd'
17
17
  gem.add_dependency 'poseidon'
18
18
  gem.add_dependency 'ltsv'
@@ -8,12 +8,13 @@ class KafkaInput < Input
8
8
  config_param :host, :string, :default => 'localhost'
9
9
  config_param :port, :integer, :default => 9092
10
10
  config_param :interval, :integer, :default => 1 # seconds
11
- config_param :topics, :string
11
+ config_param :topics, :string, :default => nil
12
12
  config_param :client_id, :string, :default => 'kafka'
13
13
  config_param :partition, :integer, :default => 0
14
14
  config_param :offset, :integer, :default => -1
15
15
  config_param :add_prefix, :string, :default => nil
16
16
  config_param :add_suffix, :string, :default => nil
17
+ config_param :add_offset_in_record, :bool, :default => false
17
18
 
18
19
  # poseidon PartitionConsumer options
19
20
  config_param :max_bytes, :integer, :default => nil
@@ -28,9 +29,25 @@ class KafkaInput < Input
28
29
 
29
30
  def configure(conf)
30
31
  super
31
- @topic_list = @topics.split(',').map {|topic| topic.strip }
32
+
33
+ @topic_list = []
34
+ if @topics
35
+ @topic_list = @topics.split(',').map { |topic|
36
+ TopicEntry.new(topic.strip, @partition, @offset)
37
+ }
38
+ else
39
+ conf.elements.select { |element| element.name == 'topic' }.each do |element|
40
+ unless element.has_key?('topic')
41
+ raise ConfigError, "kafka: 'topic' is a require parameter in 'topic element'."
42
+ end
43
+ partition = element.has_key?('partition') ? element['partition'].to_i : 0
44
+ offset = element.has_key?('offset') ? element['offset'].to_i : -1
45
+ @topic_list.push(TopicEntry.new(element['topic'], partition, offset))
46
+ end
47
+ end
48
+
32
49
  if @topic_list.empty?
33
- raise ConfigError, "kafka: 'topics' is a require parameter"
50
+ raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
34
51
  end
35
52
 
36
53
  case @format
@@ -51,8 +68,19 @@ class KafkaInput < Input
51
68
  opt[:min_bytes] = @min_bytes if @min_bytes
52
69
  opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
53
70
 
54
- @topic_watchers = @topic_list.map {|topic|
55
- TopicWatcher.new(topic, @host, @port, @client_id, @partition, @offset, interval, @format, @message_key, @add_prefix, @add_suffix, opt)
71
+ @topic_watchers = @topic_list.map {|topic_entry|
72
+ TopicWatcher.new(
73
+ topic_entry,
74
+ @host,
75
+ @port,
76
+ @client_id,
77
+ interval,
78
+ @format,
79
+ @message_key,
80
+ @add_offset_in_record,
81
+ @add_prefix,
82
+ @add_suffix,
83
+ opt)
56
84
  }
57
85
  @topic_watchers.each {|tw|
58
86
  tw.attach(@loop)
@@ -72,21 +100,22 @@ class KafkaInput < Input
72
100
  end
73
101
 
74
102
  class TopicWatcher < Coolio::TimerWatcher
75
- def initialize(topic, host, port, client_id, partition, offset, interval, format, message_key, add_prefix, add_suffix, options={})
76
- @topic = topic
103
+ def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, options={})
104
+ @topic_entry = topic_entry
105
+ @add_offset_in_record = add_offset_in_record
77
106
  @callback = method(:consume)
78
107
  @format = format
79
108
  @message_key = message_key
80
109
  @add_prefix = add_prefix
81
110
  @add_suffix = add_suffix
82
111
  @consumer = Poseidon::PartitionConsumer.new(
83
- client_id, # client_id
84
- host, # host
85
- port, # port
86
- topic, # topic
87
- partition, # partition
88
- offset, # offset
89
- options # options
112
+ client_id, # client_id
113
+ host, # host
114
+ port, # port
115
+ topic_entry.topic, # topic
116
+ topic_entry.partition, # partition
117
+ topic_entry.offset, # offset
118
+ options # options
90
119
  )
91
120
 
92
121
  super(interval, true)
@@ -102,12 +131,13 @@ class KafkaInput < Input
102
131
 
103
132
  def consume
104
133
  es = MultiEventStream.new
105
- tag = @topic
134
+ tag = @topic_entry.topic
106
135
  tag = @add_prefix + "." + tag if @add_prefix
107
136
  tag = tag + "." + @add_suffix if @add_suffix
108
137
  @consumer.fetch.each { |msg|
109
138
  begin
110
139
  msg_record = parse_line(msg.value)
140
+ msg_record = decorate_offset(msg_record, msg.offset) if @add_offset_in_record
111
141
  es.add(Time.now.to_i, msg_record)
112
142
  rescue
113
143
  $log.warn msg_record.to_s, :error=>$!.to_s
@@ -134,7 +164,39 @@ class KafkaInput < Input
134
164
  end
135
165
  parsed_record
136
166
  end
167
+
168
+ def decorate_offset(record, offset)
169
+ case @format
170
+ when 'json'
171
+ add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
172
+ when 'ltsv'
173
+ record.each { |line|
174
+ add_offset_in_hash(line, @topic_entry.topic, @topic_entry.partition, offset)
175
+ }
176
+ when 'msgpack'
177
+ add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
178
+ when 'text'
179
+ add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
180
+ end
181
+ record
182
+ end
183
+
184
+ def add_offset_in_hash(hash, topic, partition, offset)
185
+ hash['kafka_topic'] = topic
186
+ hash['kafka_partition'] = partition
187
+ hash['kafka_offset'] = offset
188
+ end
137
189
  end
190
+
191
+ class TopicEntry
192
+ def initialize(topic, partition, offset)
193
+ @topic = topic
194
+ @partition = partition
195
+ @offset = offset
196
+ end
197
+ attr_reader :topic, :partition, :offset
198
+ end
199
+
138
200
  end
139
201
 
140
202
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.16
4
+ version: 0.0.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-08 00:00:00.000000000 Z
11
+ date: 2015-09-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd