fluent-plugin-kafka 0.0.10 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +15 -0
  3. data/fluent-plugin-kafka.gemspec +1 -1
  4. data/lib/fluent/plugin/out_kafka.rb +5 -1
  5. data/lib/fluent/plugin/out_kafka_buffered.rb +5 -2
  6. metadata +17 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 887e010ab7b1ac813f4fa0c412d8348ffc192831
4
- data.tar.gz: 52f631a447899497ac9765294f697783755ae5ab
3
+ metadata.gz: 4a35c789b318f3ac5d6fa70545b170a65b3bf0b3
4
+ data.tar.gz: 77216b2f697aaf4a0a5b53e237f2f5d73c05105f
5
5
  SHA512:
6
- metadata.gz: dcd0a646d4cbb926fa3503a00e75f0ac6188e4a7a5bd8a3c3332481c2989dd913a3d4f072d839c35236e0a9ae2728076db2426d2664ec0aee8778a86246673f6
7
- data.tar.gz: bf1f03fb1134a9e16153156332da8013450c8f5a5cbc45a343ff9a6bcb4c682a8c5858339267a588a0246edce614f51f9b8ff5babdde5f26fb3a0c3b36d75bb8
6
+ metadata.gz: 109ac7d58c512e2b38c1b9cc847e23ad59e84da495874be2135647526bfc3b99dde08a7aae8ab3255d56d834cb71dde2c1d1f9e65933c10f760d34591b5da66b
7
+ data.tar.gz: cabfb8f1f4a67e2e631f13c774b8802e95f683b3042c076c8e27cfe60669e9cb36cc29b9e6947c4a8235ddaf7b47ae3d657df8faeef2d07c678a94a2561e5e06
data/README.md CHANGED
@@ -47,6 +47,7 @@ Or install it yourself as:
47
47
  max_send_retries (integer) :default => 3
48
48
  required_acks (integer) :default => 0
49
49
  ack_timeout_ms (integer) :default => 1500
50
+ compression_codec (none|gzip|snappy) :default => none
50
51
  </match>
51
52
 
52
53
  Supports following Poseidon::Producer options.
@@ -54,9 +55,16 @@ Supports following Poseidon::Producer options.
54
55
  - max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
55
56
  - required_acks — default: 0 — The number of acks required per request.
56
57
  - ack_timeout_ms — default: 1500 — How long the producer waits for acks.
58
+ - compression_codec - default: none - The codec the producer uses to compress messages.
57
59
 
58
60
  See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
59
61
 
62
+ This plugin supports compression codec "snappy" also.
63
+ Install snappy module before you use snappy compression.
64
+
65
+ $ gem install snappy
66
+
67
+
60
68
  ### Buffered output plugin
61
69
 
62
70
  <match *.**>
@@ -75,6 +83,7 @@ See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Posei
75
83
  max_send_retries (integer) :default => 3
76
84
  required_acks (integer) :default => 0
77
85
  ack_timeout_ms (integer) :default => 1500
86
+ compression_codec (none|gzip|snappy) :default => none
78
87
  </match>
79
88
 
80
89
  Supports following Poseidon::Producer options.
@@ -82,9 +91,15 @@ Supports following Poseidon::Producer options.
82
91
  - max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
83
92
  - required_acks — default: 0 — The number of acks required per request.
84
93
  - ack_timeout_ms — default: 1500 — How long the producer waits for acks.
94
+ - compression_codec - default: none - The codec the producer uses to compress messages.
85
95
 
86
96
  See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
87
97
 
98
+ This plugin supports compression codec "snappy" also.
99
+ Install snappy module before you use snappy compression.
100
+
101
+ $ gem install snappy
102
+
88
103
 
89
104
  ## Contributing
90
105
 
data/fluent-plugin-kafka.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.0.10'
15
+ gem.version = '0.0.11'
16
16
  gem.add_dependency 'fluentd'
17
17
  gem.add_dependency 'poseidon'
18
18
  gem.add_dependency 'ltsv'
data/lib/fluent/plugin/out_kafka.rb CHANGED
@@ -19,6 +19,7 @@ class Fluent::KafkaOutput < Fluent::Output
19
19
  config_param :max_send_retries, :integer, :default => 3
20
20
  config_param :required_acks, :integer, :default => 0
21
21
  config_param :ack_timeout_ms, :integer, :default => 1500
22
+ config_param :compression_codec, :string, :default => 'none'
22
23
 
23
24
  attr_accessor :output_data_type
24
25
  attr_accessor :field_separator
@@ -37,7 +38,7 @@ class Fluent::KafkaOutput < Fluent::Output
37
38
  end
38
39
  begin
39
40
  if @seed_brokers.length > 0
40
- @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
41
+ @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
41
42
  log.info "initialized producer #{@client_id}"
42
43
  else
43
44
  log.warn "No brokers found on Zookeeper"
@@ -56,6 +57,9 @@ class Fluent::KafkaOutput < Fluent::Output
56
57
  @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
57
58
  log.info "brokers has been set directly: #{@seed_brokers}"
58
59
  end
60
+ if @compression_codec == 'snappy'
61
+ require 'snappy'
62
+ end
59
63
  case @output_data_type
60
64
  when 'json'
61
65
  require 'yajl'
data/lib/fluent/plugin/out_kafka_buffered.rb CHANGED
@@ -21,6 +21,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
21
21
  config_param :max_send_retries, :integer, :default => 3
22
22
  config_param :required_acks, :integer, :default => 0
23
23
  config_param :ack_timeout_ms, :integer, :default => 1500
24
+ config_param :compression_codec, :string, :default => 'none'
24
25
 
25
26
  attr_accessor :output_data_type
26
27
  attr_accessor :field_separator
@@ -43,7 +44,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
43
44
  end
44
45
  begin
45
46
  if @seed_brokers.length > 0
46
- @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
47
+ @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
47
48
  log.info "initialized producer #{@client_id}"
48
49
  else
49
50
  log.warn "No brokers found on Zookeeper"
@@ -62,7 +63,9 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
62
63
  @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
63
64
  log.info "brokers has been set directly: #{@seed_brokers}"
64
65
  end
65
-
66
+ if @compression_codec == 'snappy'
67
+ require 'snappy'
68
+ end
66
69
  case @output_data_type
67
70
  when 'json'
68
71
  require 'yajl'
metadata CHANGED
@@ -1,97 +1,97 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-21 00:00:00.000000000 Z
11
+ date: 2015-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: poseidon
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - '>='
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - '>='
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: ltsv
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - '>='
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - '>='
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: yajl-ruby
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ">="
59
+ - - '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ">="
66
+ - - '>='
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: msgpack
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ">="
73
+ - - '>='
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ">="
80
+ - - '>='
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: zookeeper
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ">="
87
+ - - '>='
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ">="
94
+ - - '>='
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  description: Fluentd plugin for Apache Kafka > 0.8
@@ -120,17 +120,17 @@ require_paths:
120
120
  - lib
121
121
  required_ruby_version: !ruby/object:Gem::Requirement
122
122
  requirements:
123
- - - ">="
123
+ - - '>='
124
124
  - !ruby/object:Gem::Version
125
125
  version: '0'
126
126
  required_rubygems_version: !ruby/object:Gem::Requirement
127
127
  requirements:
128
- - - ">="
128
+ - - '>='
129
129
  - !ruby/object:Gem::Version
130
130
  version: '0'
131
131
  requirements: []
132
132
  rubyforge_project:
133
- rubygems_version: 2.2.2
133
+ rubygems_version: 2.0.14
134
134
  signing_key:
135
135
  specification_version: 4
136
136
  summary: Fluentd plugin for Apache Kafka > 0.8