fluent-plugin-kafka 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -26,7 +26,7 @@ Or install it yourself as:
26
26
  host <broker host>
27
27
  port <broker port: default=9092>
28
28
  topics <listening topics(separate with comma',')>
29
- format <input text type (text|json)>
29
+ format <input text type (text|json|ltsv)>
30
30
  add_prefix <tag prefix (Optional)>
31
31
  add_suffix <tag suffix (Optional)>
32
32
  </source>
@@ -38,6 +38,8 @@ Or install it yourself as:
38
38
  brokers <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
39
39
  default_topic <output topic>
40
40
  output_data_type (json|ltsv|attr:<record name>)
41
+ output_include_tag (true|false) :default => false
42
+ output_include_time (true|false) :default => false
41
43
  </match>
42
44
 
43
45
  ### Buffered output plugin
@@ -49,6 +51,8 @@ Or install it yourself as:
49
51
  flush_interval <flush interval (sec) :default => 60>
50
52
  buffer_type (file|memory)
51
53
  output_data_type (json|ltsv|attr:<record name>)
54
+ output_include_tag (true|false) :default => false
55
+ output_include_time (true|false) :default => false
52
56
  </match>
53
57
 
54
58
  ## Contributing
@@ -12,9 +12,9 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.0.5'
15
+ gem.version = '0.0.6'
16
16
  gem.add_dependency 'fluentd'
17
17
  gem.add_dependency 'poseidon'
18
18
  gem.add_dependency 'ltsv'
19
- gem.add_dependency 'json'
19
+ gem.add_dependency 'yajl-ruby'
20
20
  end
@@ -3,7 +3,7 @@ module Fluent
3
3
  class KafkaInput < Input
4
4
  Plugin.register_input('kafka', self)
5
5
 
6
- config_param :format, :string, :default => 'json' # (json|text)
6
+ config_param :format, :string, :default => 'json' # (json|text|ltsv)
7
7
  config_param :host, :string, :default => 'localhost'
8
8
  config_param :port, :integer, :default => 2181
9
9
  config_param :interval, :integer, :default => 1 # seconds
@@ -28,7 +28,9 @@ class KafkaInput < Input
28
28
 
29
29
  case @format
30
30
  when 'json'
31
- require 'json'
31
+ require 'yajl'
32
+ when 'ltsv'
33
+ require 'ltsv'
32
34
  end
33
35
  end
34
36
 
@@ -105,7 +107,9 @@ class KafkaInput < Input
105
107
  parsed_record = {}
106
108
  case @format
107
109
  when 'json'
108
- parsed_record = JSON.parse(record)
110
+ parsed_record = Yajl::Parser.parse(record)
111
+ when 'ltsv'
112
+ parsed_record = LTSV.parse(record)
109
113
  when 'text'
110
114
  parsed_record = record
111
115
  end
@@ -11,6 +11,8 @@ class Fluent::KafkaOutput < Fluent::Output
11
11
  config_param :default_partition, :integer, :default => 0
12
12
  config_param :client_id, :string, :default => 'kafka'
13
13
  config_param :output_data_type, :string, :default => 'json'
14
+ config_param :output_include_tag, :bool, :default => false
15
+ config_param :output_include_time, :bool, :default => false
14
16
  attr_accessor :output_data_type
15
17
  attr_accessor :field_separator
16
18
 
@@ -20,7 +22,7 @@ class Fluent::KafkaOutput < Fluent::Output
20
22
  @producers = {} # keyed by topic:partition
21
23
  case @output_data_type
22
24
  when 'json'
23
- require 'json'
25
+ require 'yajl'
24
26
  when 'ltsv'
25
27
  require 'ltsv'
26
28
  end
@@ -56,13 +58,15 @@ class Fluent::KafkaOutput < Fluent::Output
56
58
  if @custom_attributes.nil?
57
59
  case @output_data_type
58
60
  when 'json'
59
- JSON.dump(record)
61
+ Yajl::Encoder.encode(record)
60
62
  when 'ltsv'
61
63
  LTSV.dump(record)
62
64
  else
63
65
  record.to_s
64
66
  end
65
67
  else
68
+ @custom_attributes.unshift('time') if @output_include_time
69
+ @custom_attributes.unshift('tag') if @output_include_tag
66
70
  @custom_attributes.map { |attr|
67
71
  record[attr].nil? ? '' : record[attr].to_s
68
72
  }.join(@f_separator)
@@ -72,6 +76,8 @@ class Fluent::KafkaOutput < Fluent::Output
72
76
  def emit(tag, es, chain)
73
77
  chain.next
74
78
  es.each do |time,record|
79
+ record['time'] = time if @output_include_time
80
+ record['tag'] = tag if @output_include_tag
75
81
  topic = record['topic'] || self.default_topic || tag
76
82
  partition = record['partition'] || self.default_partition
77
83
  message = Poseidon::MessageToSend.new(topic, parse_record(record))
@@ -1,3 +1,4 @@
1
+ # encode: utf-8
1
2
  class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
2
3
  Fluent::Plugin.register_output('kafka_buffered', self)
3
4
 
@@ -11,6 +12,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
11
12
  config_param :default_partition, :integer, :default => 0
12
13
  config_param :client_id, :string, :default => 'kafka'
13
14
  config_param :output_data_type, :string, :default => 'json'
15
+ config_param :output_include_tag, :bool, :default => false
16
+ config_param :output_include_time, :bool, :default => false
14
17
  attr_accessor :output_data_type
15
18
  attr_accessor :field_separator
16
19
 
@@ -20,7 +23,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
20
23
  @producers = {} # keyed by topic:partition
21
24
  case @output_data_type
22
25
  when 'json'
23
- require 'json'
26
+ require 'yajl'
24
27
  when 'ltsv'
25
28
  require 'ltsv'
26
29
  end
@@ -59,13 +62,15 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
59
62
  if @custom_attributes.nil?
60
63
  case @output_data_type
61
64
  when 'json'
62
- JSON.dump(record)
65
+ Yajl::Encoder.encode(record)
63
66
  when 'ltsv'
64
67
  LTSV.dump(record)
65
68
  else
66
69
  record.to_s
67
70
  end
68
71
  else
72
+ @custom_attributes.unshift('time') if @output_include_time
73
+ @custom_attributes.unshift('tag') if @output_include_tag
69
74
  @custom_attributes.map { |attr|
70
75
  record[attr].nil? ? '' : record[attr].to_s
71
76
  }.join(@f_separator)
@@ -75,6 +80,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
75
80
  def write(chunk)
76
81
  records_by_topic = {}
77
82
  chunk.msgpack_each { |tag, time, record|
83
+ record['time'] = time if @output_include_time
84
+ record['tag'] = tag if @output_include_tag
78
85
  topic = record['topic'] || self.default_topic || tag
79
86
  partition = record['partition'] || self.default_partition
80
87
  message = Poseidon::MessageToSend.new(topic, parse_record(record))
metadata CHANGED
@@ -1,69 +1,78 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Hidemasa Togashi
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2014-02-05 00:00:00.000000000 Z
12
+ date: 2014-05-10 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: fluentd
15
16
  requirement: !ruby/object:Gem::Requirement
17
+ none: false
16
18
  requirements:
17
- - - '>='
19
+ - - ! '>='
18
20
  - !ruby/object:Gem::Version
19
21
  version: '0'
20
22
  type: :runtime
21
23
  prerelease: false
22
24
  version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
23
26
  requirements:
24
- - - '>='
27
+ - - ! '>='
25
28
  - !ruby/object:Gem::Version
26
29
  version: '0'
27
30
  - !ruby/object:Gem::Dependency
28
31
  name: poseidon
29
32
  requirement: !ruby/object:Gem::Requirement
33
+ none: false
30
34
  requirements:
31
- - - '>='
35
+ - - ! '>='
32
36
  - !ruby/object:Gem::Version
33
37
  version: '0'
34
38
  type: :runtime
35
39
  prerelease: false
36
40
  version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
37
42
  requirements:
38
- - - '>='
43
+ - - ! '>='
39
44
  - !ruby/object:Gem::Version
40
45
  version: '0'
41
46
  - !ruby/object:Gem::Dependency
42
47
  name: ltsv
43
48
  requirement: !ruby/object:Gem::Requirement
49
+ none: false
44
50
  requirements:
45
- - - '>='
51
+ - - ! '>='
46
52
  - !ruby/object:Gem::Version
47
53
  version: '0'
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
51
58
  requirements:
52
- - - '>='
59
+ - - ! '>='
53
60
  - !ruby/object:Gem::Version
54
61
  version: '0'
55
62
  - !ruby/object:Gem::Dependency
56
- name: json
63
+ name: yajl-ruby
57
64
  requirement: !ruby/object:Gem::Requirement
65
+ none: false
58
66
  requirements:
59
- - - '>='
67
+ - - ! '>='
60
68
  - !ruby/object:Gem::Version
61
69
  version: '0'
62
70
  type: :runtime
63
71
  prerelease: false
64
72
  version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
65
74
  requirements:
66
- - - '>='
75
+ - - ! '>='
67
76
  - !ruby/object:Gem::Version
68
77
  version: '0'
69
78
  description: Fluentd plugin for Apache Kafka > 0.8
@@ -85,26 +94,27 @@ files:
85
94
  - test/plugin/test_out_kafka.rb
86
95
  homepage: https://github.com/htgc/fluent-plugin-kafka
87
96
  licenses: []
88
- metadata: {}
89
97
  post_install_message:
90
98
  rdoc_options: []
91
99
  require_paths:
92
100
  - lib
93
101
  required_ruby_version: !ruby/object:Gem::Requirement
102
+ none: false
94
103
  requirements:
95
- - - '>='
104
+ - - ! '>='
96
105
  - !ruby/object:Gem::Version
97
106
  version: '0'
98
107
  required_rubygems_version: !ruby/object:Gem::Requirement
108
+ none: false
99
109
  requirements:
100
- - - '>='
110
+ - - ! '>='
101
111
  - !ruby/object:Gem::Version
102
112
  version: '0'
103
113
  requirements: []
104
114
  rubyforge_project:
105
- rubygems_version: 2.0.3
115
+ rubygems_version: 1.8.23
106
116
  signing_key:
107
- specification_version: 4
117
+ specification_version: 3
108
118
  summary: Fluentd plugin for Apache Kafka > 0.8
109
119
  test_files:
110
120
  - test/helper.rb
checksums.yaml DELETED
@@ -1,7 +0,0 @@
1
- ---
2
- SHA1:
3
- metadata.gz: e7ea6df9ab262c3d1a76fd1bf8084dcacfed5e13
4
- data.tar.gz: a4e75c5d1c30425d35a7cc57a802256dd28901fb
5
- SHA512:
6
- metadata.gz: 12b5a1579ffd62f45279983cbbbc3c13c4659a0c736b29b6f8e40c765b839ed3a0e6424be27e12917b674f6f62bff076c5879de3cfe133dc9b8b269f8aebc489
7
- data.tar.gz: 963e4236af8e79c47eecfdfd0b58f73c6a9e68f3161b1c984d48691ae025c64d3d5e57bb18f8a20ce91430d11ad4007ed6893c7e20aa31efded4ad7d9dd24192