fluent-plugin-kafka 0.0.10 → 0.0.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +15 -0
  3. data/fluent-plugin-kafka.gemspec +1 -1
  4. data/lib/fluent/plugin/out_kafka.rb +5 -1
  5. data/lib/fluent/plugin/out_kafka_buffered.rb +5 -2
  6. metadata +17 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 887e010ab7b1ac813f4fa0c412d8348ffc192831
4
- data.tar.gz: 52f631a447899497ac9765294f697783755ae5ab
3
+ metadata.gz: 4a35c789b318f3ac5d6fa70545b170a65b3bf0b3
4
+ data.tar.gz: 77216b2f697aaf4a0a5b53e237f2f5d73c05105f
5
5
  SHA512:
6
- metadata.gz: dcd0a646d4cbb926fa3503a00e75f0ac6188e4a7a5bd8a3c3332481c2989dd913a3d4f072d839c35236e0a9ae2728076db2426d2664ec0aee8778a86246673f6
7
- data.tar.gz: bf1f03fb1134a9e16153156332da8013450c8f5a5cbc45a343ff9a6bcb4c682a8c5858339267a588a0246edce614f51f9b8ff5babdde5f26fb3a0c3b36d75bb8
6
+ metadata.gz: 109ac7d58c512e2b38c1b9cc847e23ad59e84da495874be2135647526bfc3b99dde08a7aae8ab3255d56d834cb71dde2c1d1f9e65933c10f760d34591b5da66b
7
+ data.tar.gz: cabfb8f1f4a67e2e631f13c774b8802e95f683b3042c076c8e27cfe60669e9cb36cc29b9e6947c4a8235ddaf7b47ae3d657df8faeef2d07c678a94a2561e5e06
data/README.md CHANGED
@@ -47,6 +47,7 @@ Or install it yourself as:
47
47
  max_send_retries (integer) :default => 3
48
48
  required_acks (integer) :default => 0
49
49
  ack_timeout_ms (integer) :default => 1500
50
+ compression_codec (none|gzip|snappy) :default => none
50
51
  </match>
51
52
 
52
53
  Supports following Poseidon::Producer options.
@@ -54,9 +55,16 @@ Supports following Poseidon::Producer options.
54
55
  - max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
55
56
  - required_acks — default: 0 — The number of acks required per request.
56
57
  - ack_timeout_ms — default: 1500 — How long the producer waits for acks.
58
+ - compression_codec - default: none - The codec the producer uses to compress messages.
57
59
 
58
60
  See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
59
61
 
62
+ This plugin supports compression codec "snappy" also.
63
+ Install snappy module before you use snappy compression.
64
+
65
+ $ gem install snappy
66
+
67
+
60
68
  ### Buffered output plugin
61
69
 
62
70
  <match *.**>
@@ -75,6 +83,7 @@ See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Posei
75
83
  max_send_retries (integer) :default => 3
76
84
  required_acks (integer) :default => 0
77
85
  ack_timeout_ms (integer) :default => 1500
86
+ compression_codec (none|gzip|snappy) :default => none
78
87
  </match>
79
88
 
80
89
  Supports following Poseidon::Producer options.
@@ -82,9 +91,15 @@ Supports following Poseidon::Producer options.
82
91
  - max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
83
92
  - required_acks — default: 0 — The number of acks required per request.
84
93
  - ack_timeout_ms — default: 1500 — How long the producer waits for acks.
94
+ - compression_codec - default: none - The codec the producer uses to compress messages.
85
95
 
86
96
  See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
87
97
 
98
+ This plugin supports compression codec "snappy" also.
99
+ Install snappy module before you use snappy compression.
100
+
101
+ $ gem install snappy
102
+
88
103
 
89
104
  ## Contributing
90
105
 
data/fluent-plugin-kafka.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
13
  gem.name = "fluent-plugin-kafka"
14
14
  gem.require_paths = ["lib"]
15
- gem.version = '0.0.10'
15
+ gem.version = '0.0.11'
16
16
  gem.add_dependency 'fluentd'
17
17
  gem.add_dependency 'poseidon'
18
18
  gem.add_dependency 'ltsv'
data/lib/fluent/plugin/out_kafka.rb CHANGED
@@ -19,6 +19,7 @@ class Fluent::KafkaOutput < Fluent::Output
19
19
  config_param :max_send_retries, :integer, :default => 3
20
20
  config_param :required_acks, :integer, :default => 0
21
21
  config_param :ack_timeout_ms, :integer, :default => 1500
22
+ config_param :compression_codec, :string, :default => 'none'
22
23
 
23
24
  attr_accessor :output_data_type
24
25
  attr_accessor :field_separator
@@ -37,7 +38,7 @@ class Fluent::KafkaOutput < Fluent::Output
37
38
  end
38
39
  begin
39
40
  if @seed_brokers.length > 0
40
- @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
41
+ @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
41
42
  log.info "initialized producer #{@client_id}"
42
43
  else
43
44
  log.warn "No brokers found on Zookeeper"
@@ -56,6 +57,9 @@ class Fluent::KafkaOutput < Fluent::Output
56
57
  @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
57
58
  log.info "brokers has been set directly: #{@seed_brokers}"
58
59
  end
60
+ if @compression_codec == 'snappy'
61
+ require 'snappy'
62
+ end
59
63
  case @output_data_type
60
64
  when 'json'
61
65
  require 'yajl'
data/lib/fluent/plugin/out_kafka_buffered.rb CHANGED
@@ -21,6 +21,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
21
21
  config_param :max_send_retries, :integer, :default => 3
22
22
  config_param :required_acks, :integer, :default => 0
23
23
  config_param :ack_timeout_ms, :integer, :default => 1500
24
+ config_param :compression_codec, :string, :default => 'none'
24
25
 
25
26
  attr_accessor :output_data_type
26
27
  attr_accessor :field_separator
@@ -43,7 +44,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
43
44
  end
44
45
  begin
45
46
  if @seed_brokers.length > 0
46
- @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
47
+ @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
47
48
  log.info "initialized producer #{@client_id}"
48
49
  else
49
50
  log.warn "No brokers found on Zookeeper"
@@ -62,7 +63,9 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
62
63
  @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
63
64
  log.info "brokers has been set directly: #{@seed_brokers}"
64
65
  end
65
-
66
+ if @compression_codec == 'snappy'
67
+ require 'snappy'
68
+ end
66
69
  case @output_data_type
67
70
  when 'json'
68
71
  require 'yajl'
metadata CHANGED
@@ -1,97 +1,97 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-21 00:00:00.000000000 Z
11
+ date: 2015-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: poseidon
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - '>='
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - '>='
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: ltsv
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - '>='
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - '>='
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: yajl-ruby
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ">="
59
+ - - '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ">="
66
+ - - '>='
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: msgpack
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ">="
73
+ - - '>='
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ">="
80
+ - - '>='
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: zookeeper
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ">="
87
+ - - '>='
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ">="
94
+ - - '>='
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  description: Fluentd plugin for Apache Kafka > 0.8
@@ -120,17 +120,17 @@ require_paths:
120
120
  - lib
121
121
  required_ruby_version: !ruby/object:Gem::Requirement
122
122
  requirements:
123
- - - ">="
123
+ - - '>='
124
124
  - !ruby/object:Gem::Version
125
125
  version: '0'
126
126
  required_rubygems_version: !ruby/object:Gem::Requirement
127
127
  requirements:
128
- - - ">="
128
+ - - '>='
129
129
  - !ruby/object:Gem::Version
130
130
  version: '0'
131
131
  requirements: []
132
132
  rubyforge_project:
133
- rubygems_version: 2.2.2
133
+ rubygems_version: 2.0.14
134
134
  signing_key:
135
135
  specification_version: 4
136
136
  summary: Fluentd plugin for Apache Kafka > 0.8