logstash-output-kafka 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/DEVELOPER.md +2 -1
- data/lib/logstash/outputs/kafka.rb +16 -4
- data/logstash-output-kafka.gemspec +2 -6
- data/spec/outputs/kafka_spec.rb +11 -2
- metadata +40 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a89fc9e7f6f239df9d54c6f8f55c0a5599c9462a
|
4
|
+
data.tar.gz: 6f80b02adcdc337c8a0b97c7e366ec59032da19a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 599a1317e6771bec820e7c0fba70bb74234a6fcaa6ff53635be250227bf8ddc28cd620a6df6107ebf7ec108771b03366c6d053736b42dc698d654452c1e87d2f
|
7
|
+
data.tar.gz: c14b27da3f8fe5092c07ceb34021e4bdceb8575debc9466d2f50f9091849f1c6847fc3b3f7094ab61f3dff6d23c08437a91d9ed3ae0dd5844e7ed484e5721758
|
data/DEVELOPER.md
CHANGED
@@ -23,7 +23,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
|
|
23
23
|
partitioner_class => ... # string (optional) default: "kafka.producer.DefaultPartitioner"
|
24
24
|
request_timeout_ms => ... # number (optional) default: 10000
|
25
25
|
producer_type => ... # string (optional), one of ["sync", "async"] default => 'sync'
|
26
|
-
key_serializer_class => ... # string (optional) default:
|
26
|
+
key_serializer_class => ... # string (optional) default: kafka.serializer.StringEncoder
|
27
27
|
message_send_max_retries => ... # number (optional) default: 3
|
28
28
|
retry_backoff_ms => ... # number (optional) default: 100
|
29
29
|
topic_metadata_refresh_interval_ms => ... # number (optional) default: 600 * 1000
|
@@ -33,6 +33,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
|
|
33
33
|
batch_num_messages => ... # number (optional) default: 200
|
34
34
|
send_buffer_bytes => ... # number (optional) default: 100 * 1024
|
35
35
|
client_id => ... # string (optional) default: ""
|
36
|
+
partition_key_format => ... # string (optional) default: nil, Provides a way to specify a partition key as a string
|
36
37
|
}
|
37
38
|
}
|
38
39
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'logstash/namespace'
|
2
2
|
require 'logstash/outputs/base'
|
3
|
-
require '
|
3
|
+
require 'jruby-kafka'
|
4
4
|
|
5
5
|
# Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
|
6
6
|
# the broker.
|
@@ -64,7 +64,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
|
|
64
64
|
# throughput) but open the possibility of a failure of the client machine dropping unsent data.
|
65
65
|
config :producer_type, :validate => %w( sync async ), :default => 'sync'
|
66
66
|
# The serializer class for keys (defaults to the same as for messages if nothing is given)
|
67
|
-
config :key_serializer_class, :validate => :string, :default =>
|
67
|
+
config :key_serializer_class, :validate => :string, :default => 'kafka.serializer.StringEncoder'
|
68
68
|
# This property will cause the producer to automatically retry a failed send request. This
|
69
69
|
# property specifies the number of retries when such failures occur. Note that setting a
|
70
70
|
# non-zero value here can lead to duplicates in the case of network errors that cause a message
|
@@ -102,11 +102,21 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
|
|
102
102
|
# The client id is a user-specified string sent in each request to help trace calls. It should
|
103
103
|
# logically identify the application making the request.
|
104
104
|
config :client_id, :validate => :string, :default => ''
|
105
|
+
# Provides a way to specify a partition key as a string. To specify a partition key for
|
106
|
+
# Kafka, configure a format that will produce the key as a string. Defaults
|
107
|
+
# `key_serializer_class` to `kafka.serializer.StringEncoder` to match. For example, to partition
|
108
|
+
# by host:
|
109
|
+
# [source,ruby]
|
110
|
+
# output {
|
111
|
+
# kafka {
|
112
|
+
# partition_key_format => "%{host}"
|
113
|
+
# }
|
114
|
+
# }
|
115
|
+
config :partition_key_format, :validate => :string, :default => nil
|
105
116
|
|
106
117
|
public
|
107
118
|
def register
|
108
119
|
LogStash::Logger.setup_log4j(@logger)
|
109
|
-
require 'jruby-kafka'
|
110
120
|
|
111
121
|
options = {
|
112
122
|
:broker_list => @broker_list,
|
@@ -135,7 +145,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
|
|
135
145
|
|
136
146
|
@codec.on_event do |event, data|
|
137
147
|
begin
|
138
|
-
@producer.send_msg(event.sprintf(@topic_id),
|
148
|
+
@producer.send_msg(event.sprintf(@topic_id),@partition_key,data)
|
139
149
|
rescue LogStash::ShutdownSignal
|
140
150
|
@logger.info('Kafka producer got shutdown signal')
|
141
151
|
rescue => e
|
@@ -151,7 +161,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
|
|
151
161
|
finished
|
152
162
|
return
|
153
163
|
end
|
164
|
+
@partition_key = if @partition_key_format.nil? then nil else event.sprintf(@partition_key_format) end
|
154
165
|
@codec.encode(event)
|
166
|
+
@partition_key = nil
|
155
167
|
end
|
156
168
|
|
157
169
|
def teardown
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-kafka'
|
4
|
-
s.version = '0.1.
|
4
|
+
s.version = '0.1.7'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -19,10 +19,6 @@ Gem::Specification.new do |s|
|
|
19
19
|
# Special flag to let us know this is actually a logstash plugin
|
20
20
|
s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
|
21
21
|
|
22
|
-
# Jar dependencies
|
23
|
-
s.requirements << "jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'"
|
24
|
-
s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.10'"
|
25
|
-
|
26
22
|
# Gem dependencies
|
27
23
|
s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
|
28
24
|
s.add_runtime_dependency 'logstash-codec-plain'
|
@@ -39,6 +35,6 @@ Gem::Specification.new do |s|
|
|
39
35
|
s.add_runtime_dependency 'ruby-maven', '3.1.1.0.8'
|
40
36
|
s.add_runtime_dependency "maven-tools", '1.0.7'
|
41
37
|
|
42
|
-
s.add_runtime_dependency 'jruby-kafka', ['>=0
|
38
|
+
s.add_runtime_dependency 'jruby-kafka', ['>= 1.1.0', '< 2.0.0']
|
43
39
|
s.add_development_dependency 'logstash-devutils'
|
44
40
|
end
|
data/spec/outputs/kafka_spec.rb
CHANGED
@@ -1,13 +1,12 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require 'logstash/outputs/kafka'
|
4
|
-
require 'logstash-output-kafka_jars'
|
5
4
|
require 'jruby-kafka'
|
6
5
|
require 'json'
|
7
6
|
|
8
7
|
describe "outputs/kafka" do
|
9
8
|
let (:simple_kafka_config) {{'topic_id' => 'test'}}
|
10
|
-
let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic',
|
9
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
|
11
10
|
'@timestamp' => LogStash::Timestamp.now}) }
|
12
11
|
|
13
12
|
context 'when initializing' do
|
@@ -44,5 +43,15 @@ describe "outputs/kafka" do
|
|
44
43
|
kafka.register
|
45
44
|
kafka.receive(event)
|
46
45
|
end
|
46
|
+
|
47
|
+
it 'should support Event#sprintf placeholders in partition_key_format' do
|
48
|
+
partition_field = 'host'
|
49
|
+
expect_any_instance_of(Kafka::Producer).to receive(:send_msg)
|
50
|
+
.with(simple_kafka_config['topic_id'], event[partition_field], event.to_hash.to_json)
|
51
|
+
kafka = LogStash::Outputs::Kafka.new({'topic_id' => simple_kafka_config['topic_id'],
|
52
|
+
'partition_key_format' => "%{#{partition_field}}"})
|
53
|
+
kafka.register
|
54
|
+
kafka.receive(event)
|
55
|
+
end
|
47
56
|
end
|
48
57
|
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elasticsearch
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-02-
|
11
|
+
date: 2015-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 2.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,104 +28,112 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 2.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-plain
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-codec-plain
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-codec-json
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
51
|
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
53
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: '0'
|
53
|
-
name: logstash-codec-json
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: jar-dependencies
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
|
-
- - '
|
65
|
+
- - '='
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version:
|
61
|
-
- !ruby/object:Gem::Dependency
|
67
|
+
version: 0.1.7
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
70
|
- - '='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: 0.1.7
|
67
|
-
name: jar-dependencies
|
68
73
|
prerelease: false
|
69
74
|
type: :runtime
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: ruby-maven
|
70
77
|
version_requirements: !ruby/object:Gem::Requirement
|
71
78
|
requirements:
|
72
79
|
- - '='
|
73
80
|
- !ruby/object:Gem::Version
|
74
|
-
version:
|
75
|
-
- !ruby/object:Gem::Dependency
|
81
|
+
version: 3.1.1.0.8
|
76
82
|
requirement: !ruby/object:Gem::Requirement
|
77
83
|
requirements:
|
78
84
|
- - '='
|
79
85
|
- !ruby/object:Gem::Version
|
80
86
|
version: 3.1.1.0.8
|
81
|
-
name: ruby-maven
|
82
87
|
prerelease: false
|
83
88
|
type: :runtime
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: maven-tools
|
84
91
|
version_requirements: !ruby/object:Gem::Requirement
|
85
92
|
requirements:
|
86
93
|
- - '='
|
87
94
|
- !ruby/object:Gem::Version
|
88
|
-
version:
|
89
|
-
- !ruby/object:Gem::Dependency
|
95
|
+
version: 1.0.7
|
90
96
|
requirement: !ruby/object:Gem::Requirement
|
91
97
|
requirements:
|
92
98
|
- - '='
|
93
99
|
- !ruby/object:Gem::Version
|
94
100
|
version: 1.0.7
|
95
|
-
name: maven-tools
|
96
101
|
prerelease: false
|
97
102
|
type: :runtime
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: jruby-kafka
|
98
105
|
version_requirements: !ruby/object:Gem::Requirement
|
99
106
|
requirements:
|
100
|
-
- - '
|
107
|
+
- - '>='
|
101
108
|
- !ruby/object:Gem::Version
|
102
|
-
version: 1.0
|
103
|
-
-
|
109
|
+
version: 1.1.0
|
110
|
+
- - <
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 2.0.0
|
104
113
|
requirement: !ruby/object:Gem::Requirement
|
105
114
|
requirements:
|
106
115
|
- - '>='
|
107
116
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
109
|
-
|
117
|
+
version: 1.1.0
|
118
|
+
- - <
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: 2.0.0
|
110
121
|
prerelease: false
|
111
122
|
type: :runtime
|
123
|
+
- !ruby/object:Gem::Dependency
|
124
|
+
name: logstash-devutils
|
112
125
|
version_requirements: !ruby/object:Gem::Requirement
|
113
126
|
requirements:
|
114
127
|
- - '>='
|
115
128
|
- !ruby/object:Gem::Version
|
116
|
-
version: 0
|
117
|
-
- !ruby/object:Gem::Dependency
|
129
|
+
version: '0'
|
118
130
|
requirement: !ruby/object:Gem::Requirement
|
119
131
|
requirements:
|
120
132
|
- - '>='
|
121
133
|
- !ruby/object:Gem::Version
|
122
134
|
version: '0'
|
123
|
-
name: logstash-devutils
|
124
135
|
prerelease: false
|
125
136
|
type: :development
|
126
|
-
version_requirements: !ruby/object:Gem::Requirement
|
127
|
-
requirements:
|
128
|
-
- - '>='
|
129
|
-
- !ruby/object:Gem::Version
|
130
|
-
version: '0'
|
131
137
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
132
138
|
email: info@elasticsearch.com
|
133
139
|
executables: []
|
@@ -164,9 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
164
170
|
- - '>='
|
165
171
|
- !ruby/object:Gem::Version
|
166
172
|
version: '0'
|
167
|
-
requirements:
|
168
|
-
- jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
|
169
|
-
- jar 'org.slf4j:slf4j-log4j12', '1.7.10'
|
173
|
+
requirements: []
|
170
174
|
rubyforge_project:
|
171
175
|
rubygems_version: 2.1.9
|
172
176
|
signing_key:
|