logstash-output-kinesis 2.1.1-java → 5.0.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitattributes +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +2 -0
- data/README.md +8 -4
- data/build.gradle +1 -1
- data/lib/logstash-output-kinesis/version.rb +1 -1
- data/lib/logstash/outputs/kinesis.rb +17 -11
- data/logstash-output-kinesis.gemspec +3 -8
- data/vendor/jar-dependencies/runtime-jars/{amazon-kinesis-producer-0.12.6.jar → amazon-kinesis-producer-0.12.5.jar} +0 -0
- metadata +30 -45
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c560333a0ab8f30a9e65cbea047f1a2c5e27fabe
|
4
|
+
data.tar.gz: cc10598380a7c831621151aea4c1b5ad74cbaa2b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c8e04c30bfc2d355b4534b108629aae2b6879ae58cc6eaafc5cddc4b54d9bd9e0135fcfe2889b1c6a93bbec49953cea1c86b84a9686abd0c5e75e89c1e1fc219
|
7
|
+
data.tar.gz: 14044649e49beb980b93b91ef3d6f270dba8caa5cfd90287a2546d4308a8e48a90893e5931088b9923ffd2defb06826ba09e8c2af8becea3755e30ce5b55fb5f
|
data/.gitattributes
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
vendor/jar-dependencies/runtime-jars/*.jar filter=lfs diff=lfs merge=lfs -text
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
logstash-output-kinesis
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
jruby-1.7.
|
1
|
+
jruby-9.1.7.0
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -5,7 +5,10 @@
|
|
5
5
|
|
6
6
|
This is a plugin for [Logstash](https://github.com/elasticsearch/logstash). It will send log records to a [Kinesis stream](https://aws.amazon.com/kinesis/), using the [Kinesis Producer Library (KPL)](https://docs.aws.amazon.com/kinesis/latest/dev/developing-producers-with-kpl.html).
|
7
7
|
|
8
|
-
**This version is intended for use with Logstash
|
8
|
+
**This version is intended for use with Logstash 5.x.** For plugin versions compatible with older versions of Logstash:
|
9
|
+
|
10
|
+
* [Logstash 1.5.x](https://github.com/samcday/logstash-output-kinesis/tree/1.5)
|
11
|
+
* [Logstash 2.x](https://github.com/samcday/logstash-output-kinesis/tree/2.x)
|
9
12
|
|
10
13
|
|
11
14
|
## Configuration
|
@@ -180,10 +183,11 @@ This is being tracked in [amazon-kinesis-producer#17](https://github.com/awslabs
|
|
180
183
|
|
181
184
|
## Developing
|
182
185
|
|
183
|
-
Ensure you have JRuby 1.
|
186
|
+
Ensure you have JRuby 9.1.x installed. [rvm](https://rvm.io/) is your friend :)
|
184
187
|
|
185
188
|
```sh
|
186
|
-
|
189
|
+
rvm use --install .
|
190
|
+
gem install bundler && bundle install
|
187
191
|
bundle exec rake
|
188
192
|
```
|
189
193
|
|
@@ -203,7 +207,7 @@ Are more than welcome. Raising an issue is great, raising a PR is better, raisin
|
|
203
207
|
|
204
208
|
[travis]: https://travis-ci.org/samcday/logstash-output-kinesis
|
205
209
|
[rubygems]: https://rubygems.org/gems/logstash-output-kinesis
|
206
|
-
[kpldoc]: https://github.com/awslabs/amazon-kinesis-producer/blob/v0.
|
210
|
+
[kpldoc]: https://github.com/awslabs/amazon-kinesis-producer/blob/v0.12.5/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java#L38
|
207
211
|
|
208
212
|
[badge-travis]: https://img.shields.io/travis/samcday/logstash-output-kinesis.svg?style=flat-square
|
209
213
|
[badge-gem]: https://img.shields.io/gem/v/logstash-output-kinesis.svg?style=flat-square
|
data/build.gradle
CHANGED
@@ -8,7 +8,7 @@ require "logstash-output-kinesis_jars"
|
|
8
8
|
|
9
9
|
# Sends log events to a Kinesis stream. This output plugin uses the official Amazon KPL.
|
10
10
|
# Most of the configuration options in this plugin are simply passed on to
|
11
|
-
# link:https://github.com/awslabs/amazon-kinesis-producer/blob/v0.
|
11
|
+
# link:https://github.com/awslabs/amazon-kinesis-producer/blob/v0.12.5/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java#L38[KinesisProducerConfiguration]
|
12
12
|
class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
13
13
|
config_name "kinesis"
|
14
14
|
|
@@ -44,13 +44,17 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
|
44
44
|
config :aggregation_enabled, :validate => :boolean, :default => true
|
45
45
|
config :aggregation_max_count, :validate => :number, :default => 4294967295
|
46
46
|
config :aggregation_max_size, :validate => :number, :default => 51200
|
47
|
+
config :cloudwatch_endpoint, :validate => :string, :default => nil
|
48
|
+
config :cloudwatch_port, :validate => :number, :default => 443
|
47
49
|
config :collection_max_count, :validate => :number, :default => 500
|
48
50
|
config :collection_max_size, :validate => :number, :default => 5242880
|
49
51
|
config :connect_timeout, :validate => :number, :default => 6000
|
50
52
|
config :credentials_refresh_delay, :validate => :number, :default => 5000
|
51
|
-
config :
|
53
|
+
config :enable_core_dumps, :validate => :boolean, :default => false
|
52
54
|
config :fail_if_throttled, :validate => :boolean, :default => false
|
53
|
-
config :
|
55
|
+
config :kinesis_endpoint, :validate => :string, :default => nil
|
56
|
+
config :kinesis_port, :validate => :number, :default => 443
|
57
|
+
config :log_level, :validate => ["info", "warning", "error"], :default => "info"
|
54
58
|
config :max_connections, :validate => :number, :default => 4
|
55
59
|
config :metrics_granularity, :validate => ["global", "stream", "shard"], :default => "shard"
|
56
60
|
config :metrics_level, :validate => ["none", "summary", "detailed"], :default => "detailed"
|
@@ -58,7 +62,6 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
|
58
62
|
config :metrics_upload_delay, :validate => :number, :default => 60000
|
59
63
|
config :min_connections, :validate => :number, :default => 1
|
60
64
|
config :native_executable, :validate => :string, :default => nil
|
61
|
-
config :port, :validate => :number, :default => 443
|
62
65
|
config :rate_limit, :validate => :number, :default => 150
|
63
66
|
config :record_max_buffered_time, :validate => :number, :default => 100
|
64
67
|
config :record_ttl, :validate => :number, :default => 30000
|
@@ -85,20 +88,20 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
|
85
88
|
return unless output?(event)
|
86
89
|
|
87
90
|
if @randomized_partition_key
|
88
|
-
event
|
91
|
+
event.set("[@metadata][partition_key]", SecureRandom.uuid)
|
89
92
|
else
|
90
93
|
# Haha - gawd. If I don't put an empty string in the array, then calling .join()
|
91
94
|
# on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
|
92
95
|
partition_key_parts = [""]
|
93
96
|
|
94
97
|
@event_partition_keys.each do |partition_key_name|
|
95
|
-
if not event
|
96
|
-
partition_key_parts << event
|
98
|
+
if not event.get(partition_key_name).nil? and event.get(partition_key_name).length > 0
|
99
|
+
partition_key_parts << event.get(partition_key_name).to_s
|
97
100
|
break
|
98
101
|
end
|
99
102
|
end
|
100
103
|
|
101
|
-
event
|
104
|
+
event.set("[@metadata][partition_key]", (partition_key_parts * "-").to_s[/.+/m] || "-")
|
102
105
|
end
|
103
106
|
|
104
107
|
begin
|
@@ -123,14 +126,18 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
|
123
126
|
config.setAggregationEnabled(@aggregation_enabled)
|
124
127
|
config.setAggregationMaxCount(@aggregation_max_count)
|
125
128
|
config.setAggregationMaxSize(@aggregation_max_size)
|
129
|
+
config.setCloudwatchEndpoint(@cloudwatch_endpoint) if !@cloudwatch_endpoint.nil?
|
130
|
+
config.setCloudwatchPort(@cloudwatch_port)
|
126
131
|
config.setCollectionMaxCount(@collection_max_count)
|
127
132
|
config.setCollectionMaxSize(@collection_max_size)
|
128
133
|
config.setConnectTimeout(@connect_timeout)
|
129
134
|
config.setCredentialsProvider(credentials_provider)
|
130
135
|
config.setCredentialsRefreshDelay(@credentials_refresh_delay)
|
131
|
-
config.
|
136
|
+
config.setEnableCoreDumps(@enable_core_dumps)
|
132
137
|
config.setFailIfThrottled(@fail_if_throttled)
|
133
138
|
config.setLogLevel(@log_level)
|
139
|
+
config.setKinesisEndpoint(@kinesis_endpoint) if !@kinesis_endpoint.nil?
|
140
|
+
config.setKinesisPort(@kinesis_port)
|
134
141
|
config.setMaxConnections(@max_connections)
|
135
142
|
config.setMetricsCredentialsProvider(metrics_credentials_provider)
|
136
143
|
config.setMetricsGranularity(@metrics_granularity)
|
@@ -139,7 +146,6 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
|
139
146
|
config.setMetricsUploadDelay(@metrics_upload_delay)
|
140
147
|
config.setMinConnections(@min_connections)
|
141
148
|
config.setNativeExecutable(@native_executable) if !@native_executable.nil?
|
142
|
-
config.setKinesisPort(@port)
|
143
149
|
config.setRateLimit(@rate_limit)
|
144
150
|
config.setRecordMaxBufferedTime(@record_max_buffered_time)
|
145
151
|
config.setRecordTtl(@record_ttl)
|
@@ -189,7 +195,7 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
|
|
189
195
|
def send_record(event, payload)
|
190
196
|
begin
|
191
197
|
event_blob = ByteBuffer::wrap(payload.to_java_bytes)
|
192
|
-
@producer.addUserRecord(@stream_name, event
|
198
|
+
@producer.addUserRecord(@stream_name, event.get("[@metadata][partition_key]"), event_blob)
|
193
199
|
rescue => e
|
194
200
|
@logger.warn("Error writing event to Kinesis", :exception => e)
|
195
201
|
end
|
@@ -27,14 +27,9 @@ Gem::Specification.new do |s|
|
|
27
27
|
s.platform = "java"
|
28
28
|
|
29
29
|
# Gem dependencies
|
30
|
-
s.add_runtime_dependency "logstash-core", ">=
|
31
|
-
s.add_runtime_dependency "logstash-codec-plain", "<
|
32
|
-
s.add_runtime_dependency "logstash-codec-json", "<
|
30
|
+
s.add_runtime_dependency "logstash-core", ">= 5.0.0", "< 6.0.0"
|
31
|
+
s.add_runtime_dependency "logstash-codec-plain", "< 6.0.0"
|
32
|
+
s.add_runtime_dependency "logstash-codec-json", "< 6.0.0"
|
33
33
|
s.add_development_dependency "logstash-devutils"
|
34
34
|
s.add_development_dependency "gem-release", "~>0.7.3"
|
35
|
-
|
36
|
-
# Temporary hack because Logstash devs are crazy.
|
37
|
-
# See: https://github.com/elastic/logstash/issues/4141
|
38
|
-
# We should be able to remove this once logstash-core >2.0.0 && >1.5.5 are released
|
39
|
-
s.add_development_dependency "concurrent-ruby", "0.9.1"
|
40
35
|
end
|
Binary file
|
metadata
CHANGED
@@ -1,108 +1,92 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-kinesis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 5.0.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Sam Day
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
16
|
- - ">="
|
18
17
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
18
|
+
version: 5.0.0
|
20
19
|
- - "<"
|
21
20
|
- !ruby/object:Gem::Version
|
22
|
-
version:
|
23
|
-
|
21
|
+
version: 6.0.0
|
22
|
+
name: logstash-core
|
24
23
|
prerelease: false
|
24
|
+
type: :runtime
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 5.0.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version:
|
32
|
+
version: 6.0.0
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-codec-plain
|
35
34
|
requirement: !ruby/object:Gem::Requirement
|
36
35
|
requirements:
|
37
36
|
- - "<"
|
38
37
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
40
|
-
|
38
|
+
version: 6.0.0
|
39
|
+
name: logstash-codec-plain
|
41
40
|
prerelease: false
|
41
|
+
type: :runtime
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "<"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
46
|
+
version: 6.0.0
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
|
-
name: logstash-codec-json
|
49
48
|
requirement: !ruby/object:Gem::Requirement
|
50
49
|
requirements:
|
51
50
|
- - "<"
|
52
51
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
54
|
-
|
52
|
+
version: 6.0.0
|
53
|
+
name: logstash-codec-json
|
55
54
|
prerelease: false
|
55
|
+
type: :runtime
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "<"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version:
|
60
|
+
version: 6.0.0
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
|
-
name: logstash-devutils
|
63
62
|
requirement: !ruby/object:Gem::Requirement
|
64
63
|
requirements:
|
65
64
|
- - ">="
|
66
65
|
- !ruby/object:Gem::Version
|
67
66
|
version: '0'
|
68
|
-
|
67
|
+
name: logstash-devutils
|
69
68
|
prerelease: false
|
69
|
+
type: :development
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
|
-
name: gem-release
|
77
76
|
requirement: !ruby/object:Gem::Requirement
|
78
77
|
requirements:
|
79
78
|
- - "~>"
|
80
79
|
- !ruby/object:Gem::Version
|
81
80
|
version: 0.7.3
|
82
|
-
|
81
|
+
name: gem-release
|
83
82
|
prerelease: false
|
83
|
+
type: :development
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: 0.7.3
|
89
|
-
-
|
90
|
-
name: concurrent-ruby
|
91
|
-
requirement: !ruby/object:Gem::Requirement
|
92
|
-
requirements:
|
93
|
-
- - '='
|
94
|
-
- !ruby/object:Gem::Version
|
95
|
-
version: 0.9.1
|
96
|
-
type: :development
|
97
|
-
prerelease: false
|
98
|
-
version_requirements: !ruby/object:Gem::Requirement
|
99
|
-
requirements:
|
100
|
-
- - '='
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: 0.9.1
|
103
|
-
description: This gem is a logstash plugin required to be installed on top of the
|
104
|
-
Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis.
|
105
|
-
This gem is not a stand-alone program
|
89
|
+
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis. This gem is not a stand-alone program
|
106
90
|
email: me@samcday.com.au
|
107
91
|
executables: []
|
108
92
|
extensions: []
|
@@ -110,8 +94,10 @@ extra_rdoc_files: []
|
|
110
94
|
files:
|
111
95
|
- ".editorconfig"
|
112
96
|
- ".gemrelease"
|
97
|
+
- ".gitattributes"
|
113
98
|
- ".gitignore"
|
114
99
|
- ".rspec"
|
100
|
+
- ".ruby-gemset"
|
115
101
|
- ".ruby-version"
|
116
102
|
- ".travis.yml"
|
117
103
|
- Gemfile
|
@@ -125,7 +111,7 @@ files:
|
|
125
111
|
- logstash-output-kinesis.gemspec
|
126
112
|
- spec/outputs/kinesis_spec.rb
|
127
113
|
- spec/spec_helper.rb
|
128
|
-
- vendor/jar-dependencies/runtime-jars/amazon-kinesis-producer-0.12.
|
114
|
+
- vendor/jar-dependencies/runtime-jars/amazon-kinesis-producer-0.12.5.jar
|
129
115
|
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.11.128.jar
|
130
116
|
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.11.128.jar
|
131
117
|
- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
|
@@ -150,7 +136,7 @@ licenses:
|
|
150
136
|
metadata:
|
151
137
|
logstash_plugin: 'true'
|
152
138
|
logstash_group: output
|
153
|
-
post_install_message:
|
139
|
+
post_install_message:
|
154
140
|
rdoc_options: []
|
155
141
|
require_paths:
|
156
142
|
- lib
|
@@ -165,12 +151,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
165
151
|
- !ruby/object:Gem::Version
|
166
152
|
version: '0'
|
167
153
|
requirements: []
|
168
|
-
rubyforge_project:
|
169
|
-
rubygems_version: 2.
|
170
|
-
signing_key:
|
154
|
+
rubyforge_project:
|
155
|
+
rubygems_version: 2.6.8
|
156
|
+
signing_key:
|
171
157
|
specification_version: 4
|
172
|
-
summary: This output plugin sends records to Kinesis using the Kinesis Producer Library
|
173
|
-
(KPL)
|
158
|
+
summary: This output plugin sends records to Kinesis using the Kinesis Producer Library (KPL)
|
174
159
|
test_files:
|
175
160
|
- spec/outputs/kinesis_spec.rb
|
176
161
|
- spec/spec_helper.rb
|