logstash-logger 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +26 -1
- data/lib/logstash-logger/device.rb +2 -0
- data/lib/logstash-logger/device/kinesis.rb +80 -0
- data/lib/logstash-logger/version.rb +1 -1
- data/logstash-logger.gemspec +5 -0
- data/spec/device/kinesis_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -0
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1139c199fb7c041dad2f2e0469492a51f6812127
|
4
|
+
data.tar.gz: 5133b64e944b70de35eadab44180a6bcbb99b832
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1cdc025812374e95d3fb0e2f4322a3e4c7c79db328e79f4a8bfe50f93f3d9e637703297db3e4f5d3f0c4a3d7c84ee346a9d792e79e9fef292d36332b41ee3af
|
7
|
+
data.tar.gz: 84c4c6a90f7430215cbb36f7e52d3af8a667aaa9bb16a1ac09f3cfa648efe99dfc329209a262d619642f0419ad52c991bbe1c963f4d7097aaf5972509162078f
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.22.0
|
2
|
+
|
3
|
+
- Adds support for writing to Amazon Kinesis. [#111](https://github.com/dwbutler/logstash-logger/pull/111)
|
4
|
+
|
1
5
|
## 0.21.0
|
2
6
|
|
3
7
|
- Support for merging top level configuration in MultiLogger and
|
@@ -8,6 +12,7 @@
|
|
8
12
|
- Fixes missing require for URI [#107](https://github.com/dwbutler/logstash-logger/pull/107)
|
9
13
|
|
10
14
|
## 0.20.0
|
15
|
+
|
11
16
|
- Adds support for logging debug/error information for the connectable device's buffer. [#102](https://github.com/dwbutler/logstash-logger/pull/102)
|
12
17
|
|
13
18
|
## 0.19.2
|
data/README.md
CHANGED
@@ -8,7 +8,7 @@ writing to a file or syslog since logstash can receive the structured data direc
|
|
8
8
|
## Features
|
9
9
|
|
10
10
|
* Can write directly to a logstash listener over a UDP or TCP/SSL connection.
|
11
|
-
* Can write to a file, Redis, Kafka, a unix socket, syslog, stdout, or stderr.
|
11
|
+
* Can write to a file, Redis, Kafka, Kinesis, a unix socket, syslog, stdout, or stderr.
|
12
12
|
* Logger can take a string message, a hash, a `LogStash::Event`, an object, or a JSON string as input.
|
13
13
|
* Events are automatically populated with message, timestamp, host, and severity.
|
14
14
|
* Writes in logstash JSON format, but supports other formats as well.
|
@@ -491,6 +491,30 @@ config.logstash.backoff = 1
|
|
491
491
|
|
492
492
|
```
|
493
493
|
|
494
|
+
#### Kinesis
|
495
|
+
|
496
|
+
Add the aws-sdk gem to your Gemfile:
|
497
|
+
|
498
|
+
gem 'aws-sdk'
|
499
|
+
|
500
|
+
```ruby
|
501
|
+
# Required
|
502
|
+
config.logstash.type = :kinesis
|
503
|
+
|
504
|
+
# Optional, will default to the 'logstash' stream
|
505
|
+
config.logstash.stream = 'my-stream-name'
|
506
|
+
|
507
|
+
# Optional, will default to 'us-east-1'
|
508
|
+
config.logstash.aws_region = 'us-west-2'
|
509
|
+
|
510
|
+
# Optional, will default to the AWS_ACCESS_KEY_ID environment variable
|
511
|
+
config.logstash.aws_access_key_id = 'ASKASKHLD12341'
|
512
|
+
|
513
|
+
# Optional, will default to the AWS_SECRET_ACCESS_KEY environment variable
|
514
|
+
config.logstash.aws_secret_access_key = 'ASKASKHLD1234123412341234'
|
515
|
+
|
516
|
+
```
|
517
|
+
|
494
518
|
#### File
|
495
519
|
|
496
520
|
```ruby
|
@@ -694,6 +718,7 @@ logger = LogStashLogger.new('localhost', 5228, :tcp)
|
|
694
718
|
* [Vitaly Gorodetsky](https://github.com/vitalis)
|
695
719
|
* [Courtland Caldwell](https://github.com/caldwecr)
|
696
720
|
* [Bibek Shrestha](https://github.com/bibstha)
|
721
|
+
* [Alex Ianus](https://github.com/aianus)
|
697
722
|
|
698
723
|
## Contributing
|
699
724
|
|
@@ -12,6 +12,7 @@ module LogStashLogger
|
|
12
12
|
autoload :Unix, 'logstash-logger/device/unix'
|
13
13
|
autoload :Redis, 'logstash-logger/device/redis'
|
14
14
|
autoload :Kafka, 'logstash-logger/device/kafka'
|
15
|
+
autoload :Kinesis, 'logstash-logger/device/kinesis'
|
15
16
|
autoload :File, 'logstash-logger/device/file'
|
16
17
|
autoload :IO, 'logstash-logger/device/io'
|
17
18
|
autoload :Stdout, 'logstash-logger/device/stdout'
|
@@ -51,6 +52,7 @@ module LogStashLogger
|
|
51
52
|
when :file then File
|
52
53
|
when :redis then Redis
|
53
54
|
when :kafka then Kafka
|
55
|
+
when :kinesis then Kinesis
|
54
56
|
when :io then IO
|
55
57
|
when :stdout then Stdout
|
56
58
|
when :stderr then Stderr
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module LogStashLogger
|
4
|
+
module Device
|
5
|
+
class Kinesis < Connectable
|
6
|
+
|
7
|
+
DEFAULT_REGION = 'us-east-1'
|
8
|
+
DEFAULT_STREAM = 'logstash'
|
9
|
+
RECOVERABLE_ERROR_CODES = [
|
10
|
+
"ServiceUnavailable",
|
11
|
+
"Throttling",
|
12
|
+
"RequestExpired",
|
13
|
+
"ProvisionedThroughputExceededException"
|
14
|
+
]
|
15
|
+
|
16
|
+
attr_accessor :aws_region, :stream
|
17
|
+
|
18
|
+
def initialize(opts)
|
19
|
+
super
|
20
|
+
@access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
|
21
|
+
@secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
|
22
|
+
@aws_region = opts[:aws_region] || DEFAULT_REGION
|
23
|
+
@stream = opts[:stream] || DEFAULT_STREAM
|
24
|
+
end
|
25
|
+
|
26
|
+
def connect
|
27
|
+
@io = ::Aws::Kinesis::Client.new(
|
28
|
+
region: @aws_region,
|
29
|
+
credentials: ::Aws::Credentials.new(@access_key_id, @secret_access_key)
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
def with_connection
|
34
|
+
connect unless connected?
|
35
|
+
yield
|
36
|
+
rescue => e
|
37
|
+
log_error(e)
|
38
|
+
log_warning("giving up")
|
39
|
+
close(flush: false)
|
40
|
+
end
|
41
|
+
|
42
|
+
def write_batch(messages, group = nil)
|
43
|
+
kinesis_records = messages.map do |message|
|
44
|
+
{
|
45
|
+
data: message,
|
46
|
+
partition_key: SecureRandom.uuid
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
with_connection do
|
51
|
+
resp = @io.put_records({
|
52
|
+
records: kinesis_records,
|
53
|
+
stream_name: @stream
|
54
|
+
})
|
55
|
+
|
56
|
+
# Put any failed records back into the buffer
|
57
|
+
if resp.failed_record_count != 0
|
58
|
+
resp.records.each_with_index do |record, index|
|
59
|
+
if RECOVERABLE_ERROR_CODES.include?(record.error_code)
|
60
|
+
log_warning("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
|
61
|
+
log_warning("Retrying")
|
62
|
+
write(kinesis_records[index][:data])
|
63
|
+
elsif !record.error_code.nil? && record.error_code != ''
|
64
|
+
log_error("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def write_one(message)
|
72
|
+
write_batch([message])
|
73
|
+
end
|
74
|
+
|
75
|
+
def close!
|
76
|
+
@io = nil
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
data/logstash-logger.gemspec
CHANGED
@@ -26,6 +26,7 @@ Gem::Specification.new do |gem|
|
|
26
26
|
end
|
27
27
|
gem.add_development_dependency 'redis'
|
28
28
|
gem.add_development_dependency 'poseidon'
|
29
|
+
gem.add_development_dependency 'aws-sdk'
|
29
30
|
|
30
31
|
if RUBY_VERSION < '2' || defined?(JRUBY_VERSION)
|
31
32
|
gem.add_development_dependency 'SyslogLogger'
|
@@ -35,6 +36,10 @@ Gem::Specification.new do |gem|
|
|
35
36
|
gem.add_development_dependency 'json', '~> 1.8'
|
36
37
|
end
|
37
38
|
|
39
|
+
if RUBY_VERSION < '2.1'
|
40
|
+
gem.add_development_dependency 'nokogiri', '~> 1.6.8'
|
41
|
+
end
|
42
|
+
|
38
43
|
gem.add_development_dependency 'rspec', '>= 3'
|
39
44
|
gem.add_development_dependency 'rake'
|
40
45
|
gem.add_development_dependency 'pry'
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'logstash-logger'
|
2
|
+
|
3
|
+
describe LogStashLogger::Device::Kinesis do
|
4
|
+
include_context 'device'
|
5
|
+
|
6
|
+
let(:client) { double("Aws::Kinesis::Client") }
|
7
|
+
|
8
|
+
before(:each) do
|
9
|
+
allow(Aws::Kinesis::Client).to receive(:new) { client }
|
10
|
+
end
|
11
|
+
|
12
|
+
it "writes to a Kinesis stream" do
|
13
|
+
response = ::Aws::Kinesis::Types::PutRecordsOutput.new
|
14
|
+
response.failed_record_count = 0
|
15
|
+
response.records = []
|
16
|
+
expect(client).to receive(:put_records) { response }
|
17
|
+
kinesis_device.write "foo"
|
18
|
+
|
19
|
+
expect(kinesis_device).to be_connected
|
20
|
+
kinesis_device.close!
|
21
|
+
expect(kinesis_device).not_to be_connected
|
22
|
+
end
|
23
|
+
|
24
|
+
it "it puts records with recoverable errors back in the buffer" do
|
25
|
+
failed_record = ::Aws::Kinesis::Types::PutRecordsResultEntry.new
|
26
|
+
failed_record.error_code = "ProvisionedThroughputExceededException"
|
27
|
+
failed_record.error_message = "ProvisionedThroughputExceededException"
|
28
|
+
response = ::Aws::Kinesis::Types::PutRecordsOutput.new
|
29
|
+
response.failed_record_count = 1
|
30
|
+
response.records = [failed_record]
|
31
|
+
|
32
|
+
expect(client).to receive(:put_records) { response }
|
33
|
+
expect(kinesis_device).to receive(:write).with("foo")
|
34
|
+
|
35
|
+
kinesis_device.write_one "foo"
|
36
|
+
end
|
37
|
+
|
38
|
+
it "defaults the AWS region to us-east-1" do
|
39
|
+
expect(kinesis_device.aws_region).to eq('us-east-1')
|
40
|
+
end
|
41
|
+
|
42
|
+
it "defaults the kinesis stream to logstash" do
|
43
|
+
expect(kinesis_device.stream).to eq('logstash')
|
44
|
+
end
|
45
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -61,6 +61,7 @@ RSpec.shared_context 'device' do
|
|
61
61
|
|
62
62
|
let(:redis_device) { LogStashLogger::Device.new(type: :redis, sync: true) }
|
63
63
|
let(:kafka_device) { LogStashLogger::Device.new(type: :kafka, sync: true) }
|
64
|
+
let(:kinesis_device) { LogStashLogger::Device.new(type: :kinesis, sync: true) }
|
64
65
|
|
65
66
|
let(:outputs) { [{type: :stdout}, {type: :io, io: io}] }
|
66
67
|
let(:multi_delegator_device) { LogStashLogger::Device.new(type: :multi_delegator, outputs: outputs) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-logger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.22.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Butler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-event
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: aws-sdk
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: rspec
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -167,6 +181,7 @@ files:
|
|
167
181
|
- lib/logstash-logger/device/file.rb
|
168
182
|
- lib/logstash-logger/device/io.rb
|
169
183
|
- lib/logstash-logger/device/kafka.rb
|
184
|
+
- lib/logstash-logger/device/kinesis.rb
|
170
185
|
- lib/logstash-logger/device/multi_delegator.rb
|
171
186
|
- lib/logstash-logger/device/redis.rb
|
172
187
|
- lib/logstash-logger/device/socket.rb
|
@@ -205,6 +220,7 @@ files:
|
|
205
220
|
- spec/device/file_spec.rb
|
206
221
|
- spec/device/io_spec.rb
|
207
222
|
- spec/device/kafka_spec.rb
|
223
|
+
- spec/device/kinesis_spec.rb
|
208
224
|
- spec/device/multi_delegator_spec.rb
|
209
225
|
- spec/device/redis_spec.rb
|
210
226
|
- spec/device/socket_spec.rb
|
@@ -260,6 +276,7 @@ test_files:
|
|
260
276
|
- spec/device/file_spec.rb
|
261
277
|
- spec/device/io_spec.rb
|
262
278
|
- spec/device/kafka_spec.rb
|
279
|
+
- spec/device/kinesis_spec.rb
|
263
280
|
- spec/device/multi_delegator_spec.rb
|
264
281
|
- spec/device/redis_spec.rb
|
265
282
|
- spec/device/socket_spec.rb
|