logstash-logger 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/tests.yml +5 -3
  3. data/Appraisals +2 -4
  4. data/CHANGELOG.md +33 -0
  5. data/Gemfile +2 -1
  6. data/README.md +2 -2
  7. data/docs/buffering.md +16 -0
  8. data/docs/outputs.md +23 -0
  9. data/docs/rails.md +7 -2
  10. data/docs/ssl.md +49 -37
  11. data/docs/troubleshooting.md +22 -0
  12. data/gemfiles/rails_8.0.gemfile +3 -1
  13. data/gemfiles/rails_8.1.gemfile +2 -1
  14. data/lib/logstash-logger/buffer.rb +15 -4
  15. data/lib/logstash-logger/device/aws_stream.rb +96 -17
  16. data/lib/logstash-logger/device/connectable.rb +27 -1
  17. data/lib/logstash-logger/device/firehose.rb +6 -0
  18. data/lib/logstash-logger/device/http.rb +75 -4
  19. data/lib/logstash-logger/device/kafka.rb +27 -0
  20. data/lib/logstash-logger/device/kinesis.rb +10 -1
  21. data/lib/logstash-logger/device/tcp.rb +53 -7
  22. data/lib/logstash-logger/formatter/base.rb +44 -3
  23. data/lib/logstash-logger/formatter/cee_syslog.rb +8 -11
  24. data/lib/logstash-logger/version.rb +1 -1
  25. data/logstash-logger.gemspec +2 -6
  26. data/spec/device/aws_stream_limits_spec.rb +219 -0
  27. data/spec/device/connectable_spec.rb +51 -0
  28. data/spec/device/firehose_spec.rb +4 -1
  29. data/spec/device/http_spec.rb +166 -1
  30. data/spec/device/kafka_spec.rb +92 -0
  31. data/spec/device/kinesis_spec.rb +4 -1
  32. data/spec/device/tcp_spec.rb +94 -5
  33. data/spec/device/tcp_tls_integration_spec.rb +91 -0
  34. data/spec/formatter/base_spec.rb +42 -6
  35. data/spec/formatter/cee_syslog_spec.rb +71 -19
  36. data/spec/formatter/framing_spec.rb +58 -0
  37. data/spec/formatter/json_lines_spec.rb +18 -0
  38. data/spec/spec_helper.rb +5 -0
  39. data/spec/support/tls_helper.rb +30 -0
  40. data/spec/support/unserializable_value.rb +10 -0
  41. metadata +19 -7
  42. data/gemfiles/rails_7.2.gemfile +0 -8
@@ -0,0 +1,58 @@
1
+ require 'logstash-logger'
2
+ require_relative '../support/unserializable_value'
3
+ require 'logger'
4
+ require 'net/http'
5
+
6
+ # A record that cannot be formatted must not change how its neighbours are
7
+ # framed on the wire.
8
+ describe 'Record framing when a record cannot be formatted' do
9
+ let(:failing) { { 'message' => 'x', 'n' => UnserializableValue.new } }
10
+ let(:bad_timestamp) { %q({"message":"y","@timestamp":"not-a-time"}) }
11
+
12
+ before do
13
+ LogStashLogger.configure { |c| c.default_error_logger = ::Logger.new(IO::NULL) }
14
+ end
15
+
16
+ after { LogStashLogger.configure { |c| c.default_error_logger = ::Logger.new($stderr) } }
17
+
18
+ it 'keeps every line of a json_lines stream parseable' do
19
+ io = StringIO.new
20
+ logger = LogStashLogger.new(type: :io, io: io, sync: true)
21
+ logger.info(failing)
22
+ logger.info(bad_timestamp)
23
+ logger.info('next record')
24
+
25
+ lines = io.string.lines
26
+ expect(lines.size).to eq(3)
27
+ parsed = lines.map { |l| JSON.parse(l) }
28
+ expect(parsed[0]['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
29
+ expect(parsed[1]['message']).to eq('y')
30
+ expect(parsed[2]['message']).to eq('next record')
31
+ end
32
+
33
+ it 'keeps every record of a cee_syslog stream on its own line' do
34
+ io = StringIO.new
35
+ logger = LogStashLogger.new(type: :io, io: io, sync: true, formatter: :cee_syslog)
36
+ logger.info(failing)
37
+ logger.info(%q({"message":"z","host":{"hostname":"web1\n<13>forged"}}))
38
+ logger.info('next record')
39
+
40
+ lines = io.string.lines
41
+ expect(lines.size).to eq(3)
42
+ lines.each { |l| expect(l).to start_with("#{Socket.gethostname}:@cee:") }
43
+ end
44
+
45
+ it 'keeps an HTTP batch a valid JSON array' do
46
+ formatter = LogStashLogger::Formatter.new(:json_lines)
47
+ messages = [failing, 'next record'].map { |m| formatter.call('INFO', Time.now, nil, m) }
48
+ device = LogStashLogger::Device.new(type: :http, url: 'http://localhost', sync: true)
49
+ body = nil
50
+ allow(Net::HTTP).to receive(:post) { |_url, b, _h| body = b; Net::HTTPOK.new('1.1', '200', 'OK') }
51
+
52
+ device.write_batch(messages)
53
+
54
+ batch = JSON.parse(body)
55
+ expect(batch.size).to eq(2)
56
+ expect(batch[1]['message']).to eq('next record')
57
+ end
58
+ end
@@ -1,4 +1,5 @@
1
1
  require 'logstash-logger'
2
+ require_relative '../support/unserializable_value'
2
3
 
3
4
  describe LogStashLogger::Formatter::JsonLines do
4
5
  include_context "formatter"
@@ -34,4 +35,21 @@ describe LogStashLogger::Formatter::JsonLines do
34
35
  expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
35
36
  end
36
37
  end
38
+
39
+ context 'when the record cannot be serialized' do
40
+ let(:message) { { 'message' => 'x', 'n' => UnserializableValue.new } }
41
+
42
+ before do
43
+ allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
44
+ end
45
+
46
+ it 'emits a JSON placeholder line instead of a bare string' do
47
+ expect(formatted_message).to end_with("\n")
48
+ expect(formatted_message.count("\n")).to eq(1)
49
+ data = JSON.parse(formatted_message)
50
+ expect(data['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
51
+ expect(data['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
52
+ expect(data['error']).to eq('JSON::GeneratorError')
53
+ end
54
+ end
37
55
  end
data/spec/spec_helper.rb CHANGED
@@ -2,6 +2,11 @@ require 'pry'
2
2
  require 'securerandom'
3
3
 
4
4
  require 'simplecov'
5
+ require 'simplecov-cobertura'
6
+ SimpleCov.formatters = [
7
+ SimpleCov::Formatter::HTMLFormatter,
8
+ SimpleCov::Formatter::CoberturaFormatter
9
+ ]
5
10
  SimpleCov.start
6
11
 
7
12
  RSpec.configure do |config|
@@ -0,0 +1,30 @@
1
+ require 'openssl'
2
+ require 'tempfile'
3
+
4
+ # Builds throwaway self-signed certificates for TLS specs. Nothing is written
5
+ # outside Tempfile, and the keys never leave the test process.
6
+ module TLSHelper
7
+ def self.self_signed(common_name: 'localhost', dns: ['localhost'], ips: ['127.0.0.1'])
8
+ key = OpenSSL::PKey::RSA.new(2048)
9
+ cert = OpenSSL::X509::Certificate.new
10
+ cert.version = 2
11
+ cert.serial = rand(2**64)
12
+ cert.subject = cert.issuer = OpenSSL::X509::Name.parse("/CN=#{common_name}")
13
+ cert.public_key = key.public_key
14
+ cert.not_before = Time.now - 60
15
+ cert.not_after = Time.now + 3600
16
+ ef = OpenSSL::X509::ExtensionFactory.new(cert, cert)
17
+ cert.add_extension(ef.create_extension('basicConstraints', 'CA:TRUE', true))
18
+ sans = dns.map { |d| "DNS:#{d}" } + ips.map { |i| "IP:#{i}" }
19
+ cert.add_extension(ef.create_extension('subjectAltName', sans.join(','))) unless sans.empty?
20
+ cert.sign(key, OpenSSL::Digest.new('SHA256'))
21
+ [cert, key]
22
+ end
23
+
24
+ def self.cert_file(cert)
25
+ Tempfile.new(['logstash-logger-test', '.crt']).tap do |f|
26
+ f.write(cert.to_pem)
27
+ f.flush
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,10 @@
1
+ # A value whose JSON conversion always raises, with or without ActiveSupport
2
+ class UnserializableValue
3
+ def to_json(*)
4
+ raise JSON::GeneratorError, 'cannot serialize'
5
+ end
6
+
7
+ def as_json(*)
8
+ raise JSON::GeneratorError, 'cannot serialize'
9
+ end
10
+ end
metadata CHANGED
@@ -1,13 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-logger
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Butler
8
+ autorequire:
8
9
  bindir: bin
9
10
  cert_chain: []
10
- date: 1980-01-02 00:00:00.000000000 Z
11
+ date: 2026-10-26 00:00:00.000000000 Z
11
12
  dependencies:
12
13
  - !ruby/object:Gem::Dependency
13
14
  name: logstash-event
@@ -99,14 +100,14 @@ dependencies:
99
100
  requirements:
100
101
  - - ">="
101
102
  - !ruby/object:Gem::Version
102
- version: '0'
103
+ version: '0.4'
103
104
  type: :development
104
105
  prerelease: false
105
106
  version_requirements: !ruby/object:Gem::Requirement
106
107
  requirements:
107
108
  - - ">="
108
109
  - !ruby/object:Gem::Version
109
- version: '0'
110
+ version: '0.4'
110
111
  - !ruby/object:Gem::Dependency
111
112
  name: rspec
112
113
  requirement: !ruby/object:Gem::Requirement
@@ -271,7 +272,6 @@ files:
271
272
  - docs/ssl.md
272
273
  - docs/troubleshooting.md
273
274
  - docs/usage.md
274
- - gemfiles/rails_7.2.gemfile
275
275
  - gemfiles/rails_8.0.gemfile
276
276
  - gemfiles/rails_8.1.gemfile
277
277
  - lib/logstash-logger.rb
@@ -321,6 +321,7 @@ files:
321
321
  - samples/unix.conf
322
322
  - spec/configuration_spec.rb
323
323
  - spec/constructor_spec.rb
324
+ - spec/device/aws_stream_limits_spec.rb
324
325
  - spec/device/balancer_spec.rb
325
326
  - spec/device/connectable_spec.rb
326
327
  - spec/device/file_spec.rb
@@ -335,12 +336,14 @@ files:
335
336
  - spec/device/stderr_spec.rb
336
337
  - spec/device/stdout_spec.rb
337
338
  - spec/device/tcp_spec.rb
339
+ - spec/device/tcp_tls_integration_spec.rb
338
340
  - spec/device/udp_spec.rb
339
341
  - spec/device/unix_spec.rb
340
342
  - spec/device_spec.rb
341
343
  - spec/formatter/base_spec.rb
342
344
  - spec/formatter/cee_spec.rb
343
345
  - spec/formatter/cee_syslog_spec.rb
346
+ - spec/formatter/framing_spec.rb
344
347
  - spec/formatter/json_lines_spec.rb
345
348
  - spec/formatter/json_spec.rb
346
349
  - spec/formatter/logstash_event_spec.rb
@@ -350,12 +353,15 @@ files:
350
353
  - spec/rails_spec.rb
351
354
  - spec/silenced_logging_spec.rb
352
355
  - spec/spec_helper.rb
356
+ - spec/support/tls_helper.rb
357
+ - spec/support/unserializable_value.rb
353
358
  - spec/syslog_spec.rb
354
359
  - spec/tagged_logging_spec.rb
355
360
  homepage: http://github.com/dwbutler/logstash-logger
356
361
  licenses:
357
362
  - MIT
358
363
  metadata: {}
364
+ post_install_message:
359
365
  rdoc_options: []
360
366
  require_paths:
361
367
  - lib
@@ -363,19 +369,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
363
369
  requirements:
364
370
  - - ">="
365
371
  - !ruby/object:Gem::Version
366
- version: '3.2'
372
+ version: '3.3'
367
373
  required_rubygems_version: !ruby/object:Gem::Requirement
368
374
  requirements:
369
375
  - - ">="
370
376
  - !ruby/object:Gem::Version
371
377
  version: '0'
372
378
  requirements: []
373
- rubygems_version: 4.0.3
379
+ rubygems_version: 3.5.22
380
+ signing_key:
374
381
  specification_version: 4
375
382
  summary: LogStash Logger for ruby
376
383
  test_files:
377
384
  - spec/configuration_spec.rb
378
385
  - spec/constructor_spec.rb
386
+ - spec/device/aws_stream_limits_spec.rb
379
387
  - spec/device/balancer_spec.rb
380
388
  - spec/device/connectable_spec.rb
381
389
  - spec/device/file_spec.rb
@@ -390,12 +398,14 @@ test_files:
390
398
  - spec/device/stderr_spec.rb
391
399
  - spec/device/stdout_spec.rb
392
400
  - spec/device/tcp_spec.rb
401
+ - spec/device/tcp_tls_integration_spec.rb
393
402
  - spec/device/udp_spec.rb
394
403
  - spec/device/unix_spec.rb
395
404
  - spec/device_spec.rb
396
405
  - spec/formatter/base_spec.rb
397
406
  - spec/formatter/cee_spec.rb
398
407
  - spec/formatter/cee_syslog_spec.rb
408
+ - spec/formatter/framing_spec.rb
399
409
  - spec/formatter/json_lines_spec.rb
400
410
  - spec/formatter/json_spec.rb
401
411
  - spec/formatter/logstash_event_spec.rb
@@ -405,5 +415,7 @@ test_files:
405
415
  - spec/rails_spec.rb
406
416
  - spec/silenced_logging_spec.rb
407
417
  - spec/spec_helper.rb
418
+ - spec/support/tls_helper.rb
419
+ - spec/support/unserializable_value.rb
408
420
  - spec/syslog_spec.rb
409
421
  - spec/tagged_logging_spec.rb
@@ -1,8 +0,0 @@
1
- # This file was generated by Appraisal
2
-
3
- source "https://rubygems.org"
4
-
5
- gem "codeclimate-test-reporter", group: :test, require: nil
6
- gem "rails", "~> 7.2.0"
7
-
8
- gemspec path: "../"