logstash-output-kafka 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 668a28d4675fe9e8dbf1924f1a43d25e604f7d2e
4
- data.tar.gz: f8b4ad8e31b86d726d2debf4429b6746326b0e00
3
+ metadata.gz: a89fc9e7f6f239df9d54c6f8f55c0a5599c9462a
4
+ data.tar.gz: 6f80b02adcdc337c8a0b97c7e366ec59032da19a
5
5
  SHA512:
6
- metadata.gz: ce0e79ade2227ddf5bc580ae2be13aaca3768a54bb481a4186f567678e585c9bb873e0467e9b8482774851fb420f865ac7b4a41c06a1186f3750ce734101374d
7
- data.tar.gz: e9c8aa0b08def9d8fb02aa2c6a1ce5d980e45941c4afcaf990e23c2431bb23f03313547015a3b83ef4ee8cf733803210480c99b87d8919020dc960eb553f1e04
6
+ metadata.gz: 599a1317e6771bec820e7c0fba70bb74234a6fcaa6ff53635be250227bf8ddc28cd620a6df6107ebf7ec108771b03366c6d053736b42dc698d654452c1e87d2f
7
+ data.tar.gz: c14b27da3f8fe5092c07ceb34021e4bdceb8575debc9466d2f50f9091849f1c6847fc3b3f7094ab61f3dff6d23c08437a91d9ed3ae0dd5844e7ed484e5721758
@@ -23,7 +23,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
23
23
  partitioner_class => ... # string (optional) default: "kafka.producer.DefaultPartitioner"
24
24
  request_timeout_ms => ... # number (optional) default: 10000
25
25
  producer_type => ... # string (optional), one of ["sync", "async"] default => 'sync'
26
- key_serializer_class => ... # string (optional) default: nil
26
+ key_serializer_class => ... # string (optional) default: kafka.serializer.StringEncoder
27
27
  message_send_max_retries => ... # number (optional) default: 3
28
28
  retry_backoff_ms => ... # number (optional) default: 100
29
29
  topic_metadata_refresh_interval_ms => ... # number (optional) default: 600 * 1000
@@ -33,6 +33,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
33
33
  batch_num_messages => ... # number (optional) default: 200
34
34
  send_buffer_bytes => ... # number (optional) default: 100 * 1024
35
35
  client_id => ... # string (optional) default: ""
36
+ partition_key_format => ... # string (optional) default: nil, Provides a way to specify a partition key as a string
36
37
  }
37
38
  }
38
39
 
@@ -1,6 +1,6 @@
1
1
  require 'logstash/namespace'
2
2
  require 'logstash/outputs/base'
3
- require 'logstash-output-kafka_jars'
3
+ require 'jruby-kafka'
4
4
 
5
5
  # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
6
6
  # the broker.
@@ -64,7 +64,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
64
64
  # throughput) but open the possibility of a failure of the client machine dropping unsent data.
65
65
  config :producer_type, :validate => %w( sync async ), :default => 'sync'
66
66
  # The serializer class for keys (defaults to the same as for messages if nothing is given)
67
- config :key_serializer_class, :validate => :string, :default => nil
67
+ config :key_serializer_class, :validate => :string, :default => 'kafka.serializer.StringEncoder'
68
68
  # This property will cause the producer to automatically retry a failed send request. This
69
69
  # property specifies the number of retries when such failures occur. Note that setting a
70
70
  # non-zero value here can lead to duplicates in the case of network errors that cause a message
@@ -102,11 +102,21 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
102
102
  # The client id is a user-specified string sent in each request to help trace calls. It should
103
103
  # logically identify the application making the request.
104
104
  config :client_id, :validate => :string, :default => ''
105
+ # Provides a way to specify a partition key as a string. To specify a partition key for
106
+ # Kafka, configure a format that will produce the key as a string. Defaults
107
+ # `key_serializer_class` to `kafka.serializer.StringEncoder` to match. For example, to partition
108
+ # by host:
109
+ # [source,ruby]
110
+ # output {
111
+ # kafka {
112
+ # partition_key_format => "%{host}"
113
+ # }
114
+ # }
115
+ config :partition_key_format, :validate => :string, :default => nil
105
116
 
106
117
  public
107
118
  def register
108
119
  LogStash::Logger.setup_log4j(@logger)
109
- require 'jruby-kafka'
110
120
 
111
121
  options = {
112
122
  :broker_list => @broker_list,
@@ -135,7 +145,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
135
145
 
136
146
  @codec.on_event do |event, data|
137
147
  begin
138
- @producer.send_msg(event.sprintf(@topic_id),nil,data)
148
+ @producer.send_msg(event.sprintf(@topic_id),@partition_key,data)
139
149
  rescue LogStash::ShutdownSignal
140
150
  @logger.info('Kafka producer got shutdown signal')
141
151
  rescue => e
@@ -151,7 +161,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
151
161
  finished
152
162
  return
153
163
  end
164
+ @partition_key = if @partition_key_format.nil? then nil else event.sprintf(@partition_key_format) end
154
165
  @codec.encode(event)
166
+ @partition_key = nil
155
167
  end
156
168
 
157
169
  def teardown
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-kafka'
4
- s.version = '0.1.6'
4
+ s.version = '0.1.7'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,10 +19,6 @@ Gem::Specification.new do |s|
19
19
  # Special flag to let us know this is actually a logstash plugin
20
20
  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
21
21
 
22
- # Jar dependencies
23
- s.requirements << "jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'"
24
- s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.10'"
25
-
26
22
  # Gem dependencies
27
23
  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
28
24
  s.add_runtime_dependency 'logstash-codec-plain'
@@ -39,6 +35,6 @@ Gem::Specification.new do |s|
39
35
  s.add_runtime_dependency 'ruby-maven', '3.1.1.0.8'
40
36
  s.add_runtime_dependency "maven-tools", '1.0.7'
41
37
 
42
- s.add_runtime_dependency 'jruby-kafka', ['>=0.2.1']
38
+ s.add_runtime_dependency 'jruby-kafka', ['>= 1.1.0', '< 2.0.0']
43
39
  s.add_development_dependency 'logstash-devutils'
44
40
  end
@@ -1,13 +1,12 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require 'logstash/outputs/kafka'
4
- require 'logstash-output-kafka_jars'
5
4
  require 'jruby-kafka'
6
5
  require 'json'
7
6
 
8
7
  describe "outputs/kafka" do
9
8
  let (:simple_kafka_config) {{'topic_id' => 'test'}}
10
- let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic',
9
+ let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
11
10
  '@timestamp' => LogStash::Timestamp.now}) }
12
11
 
13
12
  context 'when initializing' do
@@ -44,5 +43,15 @@ describe "outputs/kafka" do
44
43
  kafka.register
45
44
  kafka.receive(event)
46
45
  end
46
+
47
+ it 'should support Event#sprintf placeholders in partition_key_format' do
48
+ partition_field = 'host'
49
+ expect_any_instance_of(Kafka::Producer).to receive(:send_msg)
50
+ .with(simple_kafka_config['topic_id'], event[partition_field], event.to_hash.to_json)
51
+ kafka = LogStash::Outputs::Kafka.new({'topic_id' => simple_kafka_config['topic_id'],
52
+ 'partition_key_format' => "%{#{partition_field}}"})
53
+ kafka.register
54
+ kafka.receive(event)
55
+ end
47
56
  end
48
57
  end
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elasticsearch
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-02-13 00:00:00.000000000 Z
11
+ date: 2015-02-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 2.0.0
22
- name: logstash
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,104 +28,112 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-plain
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-codec-plain
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-codec-json
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
51
  - - '>='
45
52
  - !ruby/object:Gem::Version
46
53
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: logstash-codec-json
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: jar-dependencies
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
- - - '>='
65
+ - - '='
59
66
  - !ruby/object:Gem::Version
60
- version: '0'
61
- - !ruby/object:Gem::Dependency
67
+ version: 0.1.7
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
70
  - - '='
65
71
  - !ruby/object:Gem::Version
66
72
  version: 0.1.7
67
- name: jar-dependencies
68
73
  prerelease: false
69
74
  type: :runtime
75
+ - !ruby/object:Gem::Dependency
76
+ name: ruby-maven
70
77
  version_requirements: !ruby/object:Gem::Requirement
71
78
  requirements:
72
79
  - - '='
73
80
  - !ruby/object:Gem::Version
74
- version: 0.1.7
75
- - !ruby/object:Gem::Dependency
81
+ version: 3.1.1.0.8
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
78
84
  - - '='
79
85
  - !ruby/object:Gem::Version
80
86
  version: 3.1.1.0.8
81
- name: ruby-maven
82
87
  prerelease: false
83
88
  type: :runtime
89
+ - !ruby/object:Gem::Dependency
90
+ name: maven-tools
84
91
  version_requirements: !ruby/object:Gem::Requirement
85
92
  requirements:
86
93
  - - '='
87
94
  - !ruby/object:Gem::Version
88
- version: 3.1.1.0.8
89
- - !ruby/object:Gem::Dependency
95
+ version: 1.0.7
90
96
  requirement: !ruby/object:Gem::Requirement
91
97
  requirements:
92
98
  - - '='
93
99
  - !ruby/object:Gem::Version
94
100
  version: 1.0.7
95
- name: maven-tools
96
101
  prerelease: false
97
102
  type: :runtime
103
+ - !ruby/object:Gem::Dependency
104
+ name: jruby-kafka
98
105
  version_requirements: !ruby/object:Gem::Requirement
99
106
  requirements:
100
- - - '='
107
+ - - '>='
101
108
  - !ruby/object:Gem::Version
102
- version: 1.0.7
103
- - !ruby/object:Gem::Dependency
109
+ version: 1.1.0
110
+ - - <
111
+ - !ruby/object:Gem::Version
112
+ version: 2.0.0
104
113
  requirement: !ruby/object:Gem::Requirement
105
114
  requirements:
106
115
  - - '>='
107
116
  - !ruby/object:Gem::Version
108
- version: 0.2.1
109
- name: jruby-kafka
117
+ version: 1.1.0
118
+ - - <
119
+ - !ruby/object:Gem::Version
120
+ version: 2.0.0
110
121
  prerelease: false
111
122
  type: :runtime
123
+ - !ruby/object:Gem::Dependency
124
+ name: logstash-devutils
112
125
  version_requirements: !ruby/object:Gem::Requirement
113
126
  requirements:
114
127
  - - '>='
115
128
  - !ruby/object:Gem::Version
116
- version: 0.2.1
117
- - !ruby/object:Gem::Dependency
129
+ version: '0'
118
130
  requirement: !ruby/object:Gem::Requirement
119
131
  requirements:
120
132
  - - '>='
121
133
  - !ruby/object:Gem::Version
122
134
  version: '0'
123
- name: logstash-devutils
124
135
  prerelease: false
125
136
  type: :development
126
- version_requirements: !ruby/object:Gem::Requirement
127
- requirements:
128
- - - '>='
129
- - !ruby/object:Gem::Version
130
- version: '0'
131
137
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
132
138
  email: info@elasticsearch.com
133
139
  executables: []
@@ -164,9 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
164
170
  - - '>='
165
171
  - !ruby/object:Gem::Version
166
172
  version: '0'
167
- requirements:
168
- - jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
169
- - jar 'org.slf4j:slf4j-log4j12', '1.7.10'
173
+ requirements: []
170
174
  rubyforge_project:
171
175
  rubygems_version: 2.1.9
172
176
  signing_key: