logstash-logger 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 514e39db2da9a557f7ae7d76f0983d1d0df0ad0e
4
- data.tar.gz: 5f1b91eaf92734f575210e0b47605d18b9e9a009
3
+ metadata.gz: 1139c199fb7c041dad2f2e0469492a51f6812127
4
+ data.tar.gz: 5133b64e944b70de35eadab44180a6bcbb99b832
5
5
  SHA512:
6
- metadata.gz: 1cd1f25d9c54d7819ffc8c5a8707de72df1f46ac61bf410c6f7777eb6364aca34378e7cce99701e69eeb8febb1a15e433af2fbac39bd74d43e9dcd068bbbcdb1
7
- data.tar.gz: 19eff3e6e2b01ddf299060372dc6caeae1a9fe148f8bb04d2e8d3e10f93e16e80229a3ef1819dbb08a81eda59fd53b13cf2552ab1bd05078702b56b055cc3709
6
+ metadata.gz: b1cdc025812374e95d3fb0e2f4322a3e4c7c79db328e79f4a8bfe50f93f3d9e637703297db3e4f5d3f0c4a3d7c84ee346a9d792e79e9fef292d36332b41ee3af
7
+ data.tar.gz: 84c4c6a90f7430215cbb36f7e52d3af8a667aaa9bb16a1ac09f3cfa648efe99dfc329209a262d619642f0419ad52c991bbe1c963f4d7097aaf5972509162078f
@@ -1,3 +1,7 @@
1
+ ## 0.22.0
2
+
3
+ - Adds support for writing to Amazon Kinesis. [#111](https://github.com/dwbutler/logstash-logger/pull/111)
4
+
1
5
  ## 0.21.0
2
6
 
3
7
  - Support for merging top level configuration in MultiLogger and
@@ -8,6 +12,7 @@
8
12
  - Fixes missing require for URI [#107](https://github.com/dwbutler/logstash-logger/pull/107)
9
13
 
10
14
  ## 0.20.0
15
+
11
16
  - Adds support for logging debug/error information for the connectable device's buffer. [#102](https://github.com/dwbutler/logstash-logger/pull/102)
12
17
 
13
18
  ## 0.19.2
data/README.md CHANGED
@@ -8,7 +8,7 @@ writing to a file or syslog since logstash can receive the structured data direc
8
8
  ## Features
9
9
 
10
10
  * Can write directly to a logstash listener over a UDP or TCP/SSL connection.
11
- * Can write to a file, Redis, Kafka, a unix socket, syslog, stdout, or stderr.
11
+ * Can write to a file, Redis, Kafka, Kinesis, a unix socket, syslog, stdout, or stderr.
12
12
  * Logger can take a string message, a hash, a `LogStash::Event`, an object, or a JSON string as input.
13
13
  * Events are automatically populated with message, timestamp, host, and severity.
14
14
  * Writes in logstash JSON format, but supports other formats as well.
@@ -491,6 +491,30 @@ config.logstash.backoff = 1
491
491
 
492
492
  ```
493
493
 
494
+ #### Kinesis
495
+
496
+ Add the aws-sdk gem to your Gemfile:
497
+
498
+ gem 'aws-sdk'
499
+
500
+ ```ruby
501
+ # Required
502
+ config.logstash.type = :kinesis
503
+
504
+ # Optional, will default to the 'logstash' stream
505
+ config.logstash.stream = 'my-stream-name'
506
+
507
+ # Optional, will default to 'us-east-1'
508
+ config.logstash.aws_region = 'us-west-2'
509
+
510
+ # Optional, will default to the AWS_ACCESS_KEY_ID environment variable
511
+ config.logstash.aws_access_key_id = 'ASKASKHLD12341'
512
+
513
+ # Optional, will default to the AWS_SECRET_ACCESS_KEY environment variable
514
+ config.logstash.aws_secret_access_key = 'ASKASKHLD1234123412341234'
515
+
516
+ ```
517
+
494
518
  #### File
495
519
 
496
520
  ```ruby
@@ -694,6 +718,7 @@ logger = LogStashLogger.new('localhost', 5228, :tcp)
694
718
  * [Vitaly Gorodetsky](https://github.com/vitalis)
695
719
  * [Courtland Caldwell](https://github.com/caldwecr)
696
720
  * [Bibek Shrestha](https://github.com/bibstha)
721
+ * [Alex Ianus](https://github.com/aianus)
697
722
 
698
723
  ## Contributing
699
724
 
@@ -12,6 +12,7 @@ module LogStashLogger
12
12
  autoload :Unix, 'logstash-logger/device/unix'
13
13
  autoload :Redis, 'logstash-logger/device/redis'
14
14
  autoload :Kafka, 'logstash-logger/device/kafka'
15
+ autoload :Kinesis, 'logstash-logger/device/kinesis'
15
16
  autoload :File, 'logstash-logger/device/file'
16
17
  autoload :IO, 'logstash-logger/device/io'
17
18
  autoload :Stdout, 'logstash-logger/device/stdout'
@@ -51,6 +52,7 @@ module LogStashLogger
51
52
  when :file then File
52
53
  when :redis then Redis
53
54
  when :kafka then Kafka
55
+ when :kinesis then Kinesis
54
56
  when :io then IO
55
57
  when :stdout then Stdout
56
58
  when :stderr then Stderr
@@ -0,0 +1,80 @@
1
+ require 'aws-sdk'
2
+
3
+ module LogStashLogger
4
+ module Device
5
+ class Kinesis < Connectable
6
+
7
+ DEFAULT_REGION = 'us-east-1'
8
+ DEFAULT_STREAM = 'logstash'
9
+ RECOVERABLE_ERROR_CODES = [
10
+ "ServiceUnavailable",
11
+ "Throttling",
12
+ "RequestExpired",
13
+ "ProvisionedThroughputExceededException"
14
+ ]
15
+
16
+ attr_accessor :aws_region, :stream
17
+
18
+ def initialize(opts)
19
+ super
20
+ @access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
21
+ @secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
22
+ @aws_region = opts[:aws_region] || DEFAULT_REGION
23
+ @stream = opts[:stream] || DEFAULT_STREAM
24
+ end
25
+
26
+ def connect
27
+ @io = ::Aws::Kinesis::Client.new(
28
+ region: @aws_region,
29
+ credentials: ::Aws::Credentials.new(@access_key_id, @secret_access_key)
30
+ )
31
+ end
32
+
33
+ def with_connection
34
+ connect unless connected?
35
+ yield
36
+ rescue => e
37
+ log_error(e)
38
+ log_warning("giving up")
39
+ close(flush: false)
40
+ end
41
+
42
+ def write_batch(messages, group = nil)
43
+ kinesis_records = messages.map do |message|
44
+ {
45
+ data: message,
46
+ partition_key: SecureRandom.uuid
47
+ }
48
+ end
49
+
50
+ with_connection do
51
+ resp = @io.put_records({
52
+ records: kinesis_records,
53
+ stream_name: @stream
54
+ })
55
+
56
+ # Put any failed records back into the buffer
57
+ if resp.failed_record_count != 0
58
+ resp.records.each_with_index do |record, index|
59
+ if RECOVERABLE_ERROR_CODES.include?(record.error_code)
60
+ log_warning("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
61
+ log_warning("Retrying")
62
+ write(kinesis_records[index][:data])
63
+ elsif !record.error_code.nil? && record.error_code != ''
64
+ log_error("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ def write_one(message)
72
+ write_batch([message])
73
+ end
74
+
75
+ def close!
76
+ @io = nil
77
+ end
78
+ end
79
+ end
80
+ end
@@ -1,3 +1,3 @@
1
1
  module LogStashLogger
2
- VERSION = "0.21.0"
2
+ VERSION = "0.22.0"
3
3
  end
@@ -26,6 +26,7 @@ Gem::Specification.new do |gem|
26
26
  end
27
27
  gem.add_development_dependency 'redis'
28
28
  gem.add_development_dependency 'poseidon'
29
+ gem.add_development_dependency 'aws-sdk'
29
30
 
30
31
  if RUBY_VERSION < '2' || defined?(JRUBY_VERSION)
31
32
  gem.add_development_dependency 'SyslogLogger'
@@ -35,6 +36,10 @@ Gem::Specification.new do |gem|
35
36
  gem.add_development_dependency 'json', '~> 1.8'
36
37
  end
37
38
 
39
+ if RUBY_VERSION < '2.1'
40
+ gem.add_development_dependency 'nokogiri', '~> 1.6.8'
41
+ end
42
+
38
43
  gem.add_development_dependency 'rspec', '>= 3'
39
44
  gem.add_development_dependency 'rake'
40
45
  gem.add_development_dependency 'pry'
@@ -0,0 +1,45 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Device::Kinesis do
4
+ include_context 'device'
5
+
6
+ let(:client) { double("Aws::Kinesis::Client") }
7
+
8
+ before(:each) do
9
+ allow(Aws::Kinesis::Client).to receive(:new) { client }
10
+ end
11
+
12
+ it "writes to a Kinesis stream" do
13
+ response = ::Aws::Kinesis::Types::PutRecordsOutput.new
14
+ response.failed_record_count = 0
15
+ response.records = []
16
+ expect(client).to receive(:put_records) { response }
17
+ kinesis_device.write "foo"
18
+
19
+ expect(kinesis_device).to be_connected
20
+ kinesis_device.close!
21
+ expect(kinesis_device).not_to be_connected
22
+ end
23
+
24
+ it "it puts records with recoverable errors back in the buffer" do
25
+ failed_record = ::Aws::Kinesis::Types::PutRecordsResultEntry.new
26
+ failed_record.error_code = "ProvisionedThroughputExceededException"
27
+ failed_record.error_message = "ProvisionedThroughputExceededException"
28
+ response = ::Aws::Kinesis::Types::PutRecordsOutput.new
29
+ response.failed_record_count = 1
30
+ response.records = [failed_record]
31
+
32
+ expect(client).to receive(:put_records) { response }
33
+ expect(kinesis_device).to receive(:write).with("foo")
34
+
35
+ kinesis_device.write_one "foo"
36
+ end
37
+
38
+ it "defaults the AWS region to us-east-1" do
39
+ expect(kinesis_device.aws_region).to eq('us-east-1')
40
+ end
41
+
42
+ it "defaults the kinesis stream to logstash" do
43
+ expect(kinesis_device.stream).to eq('logstash')
44
+ end
45
+ end
@@ -61,6 +61,7 @@ RSpec.shared_context 'device' do
61
61
 
62
62
  let(:redis_device) { LogStashLogger::Device.new(type: :redis, sync: true) }
63
63
  let(:kafka_device) { LogStashLogger::Device.new(type: :kafka, sync: true) }
64
+ let(:kinesis_device) { LogStashLogger::Device.new(type: :kinesis, sync: true) }
64
65
 
65
66
  let(:outputs) { [{type: :stdout}, {type: :io, io: io}] }
66
67
  let(:multi_delegator_device) { LogStashLogger::Device.new(type: :multi_delegator, outputs: outputs) }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-logger
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.21.0
4
+ version: 0.22.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Butler
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-21 00:00:00.000000000 Z
11
+ date: 2017-01-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-event
@@ -66,6 +66,20 @@ dependencies:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: aws-sdk
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: rspec
71
85
  requirement: !ruby/object:Gem::Requirement
@@ -167,6 +181,7 @@ files:
167
181
  - lib/logstash-logger/device/file.rb
168
182
  - lib/logstash-logger/device/io.rb
169
183
  - lib/logstash-logger/device/kafka.rb
184
+ - lib/logstash-logger/device/kinesis.rb
170
185
  - lib/logstash-logger/device/multi_delegator.rb
171
186
  - lib/logstash-logger/device/redis.rb
172
187
  - lib/logstash-logger/device/socket.rb
@@ -205,6 +220,7 @@ files:
205
220
  - spec/device/file_spec.rb
206
221
  - spec/device/io_spec.rb
207
222
  - spec/device/kafka_spec.rb
223
+ - spec/device/kinesis_spec.rb
208
224
  - spec/device/multi_delegator_spec.rb
209
225
  - spec/device/redis_spec.rb
210
226
  - spec/device/socket_spec.rb
@@ -260,6 +276,7 @@ test_files:
260
276
  - spec/device/file_spec.rb
261
277
  - spec/device/io_spec.rb
262
278
  - spec/device/kafka_spec.rb
279
+ - spec/device/kinesis_spec.rb
263
280
  - spec/device/multi_delegator_spec.rb
264
281
  - spec/device/redis_spec.rb
265
282
  - spec/device/socket_spec.rb