logstash-logger 0.26.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/tests.yml +53 -0
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +85 -105
  5. data/Appraisals +6 -20
  6. data/CHANGELOG.md +61 -0
  7. data/Gemfile +2 -2
  8. data/README.md +12 -833
  9. data/docs/buffering.md +86 -0
  10. data/docs/customization.md +86 -0
  11. data/docs/outputs.md +65 -0
  12. data/docs/rails.md +349 -0
  13. data/docs/ssl.md +102 -0
  14. data/docs/troubleshooting.md +106 -0
  15. data/docs/usage.md +148 -0
  16. data/gemfiles/rails_8.0.gemfile +10 -0
  17. data/gemfiles/rails_8.1.gemfile +9 -0
  18. data/lib/logstash-logger/buffer.rb +15 -5
  19. data/lib/logstash-logger/configuration.rb +1 -2
  20. data/lib/logstash-logger/device/aws_stream.rb +96 -17
  21. data/lib/logstash-logger/device/base.rb +2 -2
  22. data/lib/logstash-logger/device/connectable.rb +30 -12
  23. data/lib/logstash-logger/device/file.rb +21 -4
  24. data/lib/logstash-logger/device/firehose.rb +6 -0
  25. data/lib/logstash-logger/device/http.rb +104 -0
  26. data/lib/logstash-logger/device/kafka.rb +182 -38
  27. data/lib/logstash-logger/device/kinesis.rb +10 -1
  28. data/lib/logstash-logger/device/redis.rb +8 -1
  29. data/lib/logstash-logger/device/tcp.rb +54 -12
  30. data/lib/logstash-logger/device.rb +24 -2
  31. data/lib/logstash-logger/formatter/base.rb +97 -11
  32. data/lib/logstash-logger/formatter/cee_syslog.rb +8 -11
  33. data/lib/logstash-logger/formatter/json.rb +13 -0
  34. data/lib/logstash-logger/formatter/json_lines.rb +13 -0
  35. data/lib/logstash-logger/formatter.rb +14 -6
  36. data/lib/logstash-logger/logger.rb +6 -19
  37. data/lib/logstash-logger/multi_logger.rb +2 -1
  38. data/lib/logstash-logger/railtie.rb +1 -1
  39. data/lib/logstash-logger/tagged_logging.rb +3 -1
  40. data/lib/logstash-logger/version.rb +1 -1
  41. data/logstash-logger.gemspec +8 -4
  42. data/spec/device/aws_stream_limits_spec.rb +219 -0
  43. data/spec/device/connectable_spec.rb +51 -0
  44. data/spec/device/file_spec.rb +65 -0
  45. data/spec/device/firehose_spec.rb +4 -1
  46. data/spec/device/http_spec.rb +176 -0
  47. data/spec/device/kafka_spec.rb +429 -14
  48. data/spec/device/kinesis_spec.rb +4 -1
  49. data/spec/device/tcp_spec.rb +94 -5
  50. data/spec/device/tcp_tls_integration_spec.rb +91 -0
  51. data/spec/device_spec.rb +13 -0
  52. data/spec/formatter/base_spec.rb +82 -1
  53. data/spec/formatter/cee_syslog_spec.rb +71 -19
  54. data/spec/formatter/framing_spec.rb +58 -0
  55. data/spec/formatter/json_lines_spec.rb +41 -0
  56. data/spec/formatter/json_spec.rb +49 -0
  57. data/spec/formatter_spec.rb +19 -2
  58. data/spec/logger_spec.rb +5 -5
  59. data/spec/multi_logger_spec.rb +16 -0
  60. data/spec/spec_helper.rb +7 -5
  61. data/spec/support/tls_helper.rb +30 -0
  62. data/spec/support/unserializable_value.rb +10 -0
  63. data/spec/tagged_logging_spec.rb +15 -0
  64. metadata +101 -16
  65. data/.travis.yml +0 -26
  66. data/gemfiles/rails_3.2.gemfile +0 -9
  67. data/gemfiles/rails_4.0.gemfile +0 -9
  68. data/gemfiles/rails_4.1.gemfile +0 -9
  69. data/gemfiles/rails_4.2.gemfile +0 -9
  70. data/gemfiles/rails_5.0.gemfile +0 -9
  71. data/gemfiles/rails_5.1.gemfile +0 -9
@@ -1,4 +1,5 @@
1
1
  require 'logstash-logger'
2
+ require_relative '../support/tls_helper'
2
3
 
3
4
  describe LogStashLogger::Device::TCP do
4
5
  include_context 'device'
@@ -13,6 +14,9 @@ describe LogStashLogger::Device::TCP do
13
14
  allow(OpenSSL::SSL::SSLSocket).to receive(:new) { ssl_socket }
14
15
  allow(ssl_socket).to receive(:connect)
15
16
  allow(ssl_socket).to receive(:post_connection_check)
17
+ allow(ssl_socket).to receive(:sync_close=)
18
+ allow(ssl_socket).to receive(:hostname=)
19
+ allow(tcp_socket).to receive(:close)
16
20
  allow(ssl_tcp_device).to receive(:warn)
17
21
  end
18
22
 
@@ -103,16 +107,101 @@ describe LogStashLogger::Device::TCP do
103
107
  end
104
108
 
105
109
  context 'without a provided SSL context' do
106
- it 'ssl_context returns nil' do
107
- expect(ssl_tcp_device.ssl_context).to be_nil
110
+ it 'uses a context that verifies the server certificate' do
111
+ ctx = ssl_tcp_device.ssl_context
112
+ expect(ctx).to be_a(OpenSSL::SSL::SSLContext)
113
+ expect(ctx.verify_mode).to eq(OpenSSL::SSL::VERIFY_PEER)
114
+ expect(ctx.verify_hostname).to be true
115
+ expect(ctx.cert_store).not_to be_nil
116
+ end
117
+
118
+ it 'creates the socket with that context' do
119
+ expect(OpenSSL::SSL::SSLSocket).to receive(:new).with(tcp_socket, ssl_tcp_device.ssl_context)
120
+ ssl_tcp_device.connect
121
+ end
122
+
123
+ it 'verifies the hostname after connecting' do
124
+ expect(ssl_socket).to receive(:post_connection_check).with(HOST)
125
+ ssl_tcp_device.connect
126
+ end
127
+
128
+ context 'with verify_hostname: false' do
129
+ let(:ssl_tcp_device) { LogStashLogger::Device.new(type: :tcp, port: port, ssl_enable: true, sync: true, verify_hostname: false) }
130
+
131
+ it 'still verifies the certificate chain' do
132
+ expect(ssl_tcp_device.ssl_context.verify_mode).to eq(OpenSSL::SSL::VERIFY_PEER)
133
+ end
134
+
135
+ it 'does not verify the hostname' do
136
+ expect(ssl_tcp_device.ssl_context.verify_hostname).to be false
137
+ expect(ssl_socket).not_to receive(:post_connection_check)
138
+ ssl_tcp_device.connect
139
+ end
140
+ end
141
+ end
142
+
143
+ context 'server name indication' do
144
+ it 'is sent for a DNS hostname' do
145
+ device = LogStashLogger::Device.new(type: :tcp, host: 'logstash.example.com', port: port, ssl_enable: true, sync: true)
146
+ expect(ssl_socket).to receive(:hostname=).with('logstash.example.com')
147
+ device.connect
148
+ end
149
+
150
+ it 'is not sent for an IP address' do
151
+ expect(ssl_socket).not_to receive(:hostname=)
152
+ ssl_tcp_device.connect
153
+ end
154
+ end
155
+
156
+ it 'closes the TCP socket together with the SSL socket' do
157
+ expect(ssl_socket).to receive(:sync_close=).with(true)
158
+ ssl_tcp_device.connect
159
+ end
160
+
161
+ context 'when the SSL socket cannot be created' do
162
+ before do
163
+ allow(OpenSSL::SSL::SSLSocket).to receive(:new).and_raise(OpenSSL::SSL::SSLError, 'boom')
164
+ end
165
+
166
+ it 'closes the TCP socket it opened' do
167
+ expect(tcp_socket).to receive(:close).once
168
+ expect { ssl_tcp_device.connect }.to raise_error(OpenSSL::SSL::SSLError)
108
169
  end
109
170
  end
110
171
 
111
172
  context 'only providing a certificate file' do
112
- let(:ssl_tcp_device) { LogStashLogger::Device.new(type: :tcp, port: port, ssl_enable: true, sync: true, ssl_certificate: '/path/to/cert.pem') }
173
+ let(:cert_file) { TLSHelper.cert_file(TLSHelper.self_signed.first) }
174
+ let(:ssl_tcp_device) { LogStashLogger::Device.new(type: :tcp, port: port, ssl_enable: true, sync: true, ssl_certificate: cert_file.path) }
175
+
176
+ after { cert_file.close! }
177
+
178
+ it 'trusts that certificate and verifies the server against it' do
179
+ ctx = ssl_tcp_device.ssl_context
180
+ expect(ctx.ca_file).to eq(cert_file.path)
181
+ expect(ctx.verify_mode).to eq(OpenSSL::SSL::VERIFY_PEER)
182
+ expect(ctx.verify_hostname).to be true
183
+ end
184
+
185
+ it 'enables SSL implicitly' do
186
+ device = LogStashLogger::Device.new(type: :tcp, port: port, sync: true, ssl_certificate: cert_file.path)
187
+ expect(device.use_ssl?).to be_truthy
188
+ end
189
+
190
+ it 'rejects a missing file' do
191
+ expect {
192
+ LogStashLogger::Device.new(type: :tcp, port: port, sync: true, ssl_certificate: '/nonexistent/cert.pem')
193
+ }.to raise_error(ArgumentError, /readable certificate file/)
194
+ end
113
195
 
114
- it 'implicitly uses a context with the configured certificate' do
115
- expect(ssl_tcp_device.ssl_context.cert).to eq('/path/to/cert.pem')
196
+ it 'rejects a file that is not a certificate' do
197
+ not_a_cert = Tempfile.new('not-a-cert')
198
+ not_a_cert.write('hello')
199
+ not_a_cert.flush
200
+ expect {
201
+ LogStashLogger::Device.new(type: :tcp, port: port, sync: true, ssl_certificate: not_a_cert.path)
202
+ }.to raise_error(ArgumentError, /not a PEM or DER certificate/)
203
+ ensure
204
+ not_a_cert.close!
116
205
  end
117
206
  end
118
207
  end
@@ -0,0 +1,91 @@
1
+ require 'logstash-logger'
2
+ require 'logger'
3
+ require 'socket'
4
+ require_relative '../support/tls_helper'
5
+
6
+ # Real TLS handshakes against a loopback server that presents a throwaway
7
+ # self-signed certificate. Nothing leaves 127.0.0.1.
8
+ describe LogStashLogger::Device::TCP, 'TLS verification against a real server' do
9
+ before(:all) do
10
+ @cert, @key = TLSHelper.self_signed(dns: ['localhost'], ips: [])
11
+ @cert_file = TLSHelper.cert_file(@cert)
12
+ end
13
+
14
+ after(:all) { @cert_file.close! }
15
+
16
+ let(:received) { Queue.new }
17
+ let(:server) do
18
+ tcp = TCPServer.new('127.0.0.1', 0)
19
+ ctx = OpenSSL::SSL::SSLContext.new
20
+ ctx.cert = @cert
21
+ ctx.key = @key
22
+ OpenSSL::SSL::SSLServer.new(tcp, ctx)
23
+ end
24
+ let(:port) { server.to_io.addr[1] }
25
+
26
+ before do
27
+ srv = server
28
+ q = received
29
+ @server_thread = Thread.new do
30
+ loop do
31
+ begin
32
+ client = srv.accept
33
+ q << client.gets
34
+ client.close
35
+ rescue OpenSSL::SSL::SSLError, IOError, SystemCallError
36
+ # A rejected handshake is expected in several examples
37
+ break if srv.to_io.closed?
38
+ end
39
+ end
40
+ end
41
+ end
42
+
43
+ after do
44
+ server.close rescue nil
45
+ @server_thread.kill
46
+ @server_thread.join(1)
47
+ end
48
+
49
+ def device(**opts)
50
+ LogStashLogger::Device.new(
51
+ { type: :tcp, port: port, sync: true, buffer_flush_at_exit: false,
52
+ error_logger: ::Logger.new(IO::NULL) }.merge(opts)
53
+ )
54
+ end
55
+
56
+ def deliver(dev)
57
+ dev.connect
58
+ dev.io.write(%({"message":"hello"}\n))
59
+ dev.io.flush
60
+ line = received.pop(timeout: 5)
61
+ dev.close!
62
+ line
63
+ end
64
+
65
+ it 'refuses a self-signed certificate when only ssl_enable is set' do
66
+ expect { device(host: 'localhost', ssl_enable: true).connect }
67
+ .to raise_error(OpenSSL::SSL::SSLError)
68
+ end
69
+
70
+ it 'delivers when the certificate is trusted through ssl_certificate' do
71
+ expect(deliver(device(host: 'localhost', ssl_certificate: @cert_file.path)))
72
+ .to eq(%({"message":"hello"}\n))
73
+ end
74
+
75
+ it 'refuses a certificate whose name does not match the host' do
76
+ expect { device(host: '127.0.0.1', ssl_certificate: @cert_file.path).connect }
77
+ .to raise_error(OpenSSL::SSL::SSLError)
78
+ end
79
+
80
+ it 'checks the name given in verify_hostname when connecting by IP' do
81
+ dev = device(host: '127.0.0.1', ssl_certificate: @cert_file.path, verify_hostname: 'localhost')
82
+ expect(deliver(dev)).to eq(%({"message":"hello"}\n))
83
+ end
84
+
85
+ it 'still allows an explicit VERIFY_NONE context as an opt-out' do
86
+ ctx = OpenSSL::SSL::SSLContext.new
87
+ ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
88
+ dev = device(host: '127.0.0.1', ssl_context: ctx, verify_hostname: false)
89
+ expect(deliver(dev)).to eq(%({"message":"hello"}\n))
90
+ end
91
+ end
data/spec/device_spec.rb CHANGED
@@ -38,6 +38,19 @@ describe LogStashLogger::Device do
38
38
  it { is_expected.to eq({type: 'udp', host: 'localhost', port: 5228, path: ''}) }
39
39
  end
40
40
 
41
+ context "when uri includes buffer_max_items" do
42
+ let(:uri_config) { {uri: "udp://localhost:5228?buffer_max_items=75"} }
43
+ it do
44
+ expect(parse_uri_config).to eq({
45
+ type: 'udp',
46
+ host: 'localhost',
47
+ port: 5228,
48
+ path: '',
49
+ buffer_max_items: 75,
50
+ })
51
+ end
52
+ end
53
+
41
54
  context "when uri is invalid" do
42
55
  let(:uri_config) { invalid_uri_config }
43
56
  specify { expect { parse_uri_config }.to raise_error(URI::InvalidURIError) }
@@ -23,6 +23,87 @@ describe LogStashLogger::Formatter::Base do
23
23
  expect(subject.call(severity, time, progname, message)).to be_nil
24
24
  end
25
25
  end
26
+
27
+ context "when building the event raises" do
28
+ before do
29
+ allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
30
+ allow(subject).to receive(:build_event).and_raise(ArgumentError, 'boom')
31
+ end
32
+
33
+ it "logs an exception" do
34
+ expect(subject).to receive(:log_error).and_call_original
35
+ subject.call(severity, time, progname, message)
36
+ end
37
+
38
+ it "returns a well-formed placeholder event instead of a bare string" do
39
+ event = subject.call(severity, time, progname, message)
40
+ expect(event).to be_a(LogStash::Event)
41
+ expect(event['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
42
+ expect(event['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
43
+ expect(event['error']).to eq('ArgumentError')
44
+ expect(event['severity']).to eq(severity)
45
+ expect(event['@timestamp']).to eq(time.iso8601(3))
46
+ end
47
+ end
48
+
49
+ context "when even the placeholder cannot be formatted" do
50
+ before do
51
+ allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
52
+ allow(subject).to receive(:format_event).and_raise(ArgumentError, 'boom')
53
+ end
54
+
55
+ it "returns nil, which devices drop" do
56
+ expect(subject.call(severity, time, progname, message)).to be_nil
57
+ end
58
+ end
59
+
60
+ context "when logged content carries an @timestamp that is not a time" do
61
+ let(:message) { '{"message":"hello","@timestamp":"not-a-time"}' }
62
+
63
+ it "keeps the logger time and formats the record normally" do
64
+ event = subject.call(severity, time, progname, message)
65
+ expect(event['message']).to eq('hello')
66
+ expect(event['@timestamp']).to eq(time.iso8601(3))
67
+ end
68
+ end
69
+
70
+ context "when logged content carries a valid @timestamp" do
71
+ let(:message) { '{"message":"hello","@timestamp":"2001-01-01T00:00:00Z"}' }
72
+
73
+ it "keeps it" do
74
+ event = subject.call(severity, time, progname, message)
75
+ expect(event['@timestamp']).to start_with('2001-01-01T00:00:00')
76
+ end
77
+ end
78
+ end
79
+
80
+ describe '#force_utf8_encoding' do
81
+ let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT').freeze }
82
+ let(:event) { LogStash::Event.new("message" => message) }
83
+
84
+ it 'returns the same event' do
85
+ expect(subject.send(:force_utf8_encoding, event)).to eq(event)
86
+ end
87
+
88
+ it 'forces the event message to UTF-8 encoding' do
89
+ subject.send(:force_utf8_encoding, event)
90
+ updated_event_data = event.instance_variable_get(:@data)
91
+ expect(updated_event_data['message'].encoding.name).to eq('UTF-8')
92
+ end
93
+
94
+ context 'frozen message string' do
95
+ let(:message) { "foo x\xc3x".dup.force_encoding('ASCII-8BIT').freeze }
96
+
97
+ it 'returns the same event' do
98
+ expect(subject.send(:force_utf8_encoding, event)).to eq(event)
99
+ end
100
+
101
+ it 'forces the event message to UTF-8 encoding' do
102
+ subject.send(:force_utf8_encoding, event)
103
+ updated_event_data = event.instance_variable_get(:@data)
104
+ expect(updated_event_data['message'].encoding.name).to eq('UTF-8')
105
+ end
106
+ end
26
107
  end
27
108
 
28
109
  describe "#build_event" do
@@ -81,7 +162,7 @@ describe LogStashLogger::Formatter::Base do
81
162
  end
82
163
 
83
164
  it "adds host" do
84
- expect(event['host']).to eq(hostname)
165
+ expect(event['host']['hostname']).to eq(hostname)
85
166
  end
86
167
  end
87
168
 
@@ -1,43 +1,95 @@
1
1
  require 'logstash-logger'
2
+ require_relative '../support/unserializable_value'
2
3
 
3
4
  describe LogStashLogger::Formatter::CeeSyslog do
4
5
  include_context "formatter"
5
6
 
6
- describe "#call" do
7
- let(:facility) { "facility" }
8
-
9
- before do
10
- allow(subject).to receive(:build_facility).and_return(facility)
11
- end
7
+ before { subject.error_logger = Logger.new(IO::NULL) }
12
8
 
13
- it "outputs a facility before the @cee" do
14
- expect(formatted_message).to match(/\A#{facility}:@cee:/)
9
+ describe "#call" do
10
+ it "outputs this host and the progname before the @cee" do
11
+ expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
15
12
  end
16
13
 
17
14
  it "serializes the LogStash::Event data as JSON" do
18
- json_data = formatted_message[/\A#{facility}:@cee:\s?(.*)\Z/, 1]
15
+ json_data = formatted_message[/\A[^:]*:@cee:\s?(.*)\Z/, 1]
19
16
  json_message = JSON.parse(json_data)
20
17
  expect(json_message["message"]).to eq(message)
21
18
  end
19
+
20
+ it "ends the record with exactly one newline" do
21
+ expect(formatted_message.count("\n")).to eq(1)
22
+ expect(formatted_message).to end_with("\n")
23
+ end
24
+
25
+ context "without progname" do
26
+ let(:progname) { nil }
27
+
28
+ it "only includes the hostname" do
29
+ expect(formatted_message).to start_with("#{hostname}:@cee:")
30
+ end
31
+ end
22
32
  end
23
33
 
24
- describe "#build_facility" do
25
- let(:host) { Socket.gethostname }
34
+ describe "header" do
35
+ context "when logged content sets host.hostname" do
36
+ let(:message) { %q({"message":"hi","host":{"hostname":"web1\n<13>Jan 1 00:00:00 otherhost app: forged"}}) }
37
+
38
+ it "uses this process's hostname, not the logged one" do
39
+ expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
40
+ end
41
+
42
+ it "cannot start a second record" do
43
+ expect(formatted_message.count("\n")).to eq(1)
44
+ expect(formatted_message).to end_with("\n")
45
+ end
46
+ end
47
+
48
+ context "when logged content sets host to a String" do
49
+ let(:message) { %q({"message":"hi","host":"web1"}) }
26
50
 
27
- before do
28
- formatted_message
51
+ it "formats the record normally" do
52
+ expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
53
+ expect(JSON.parse(formatted_message[/@cee:(.*)\Z/, 1])['host']).to eq('web1')
54
+ end
29
55
  end
30
56
 
31
- it "includes hostname and progname" do
32
- expect(subject.send(:build_facility, host)).to match(/\A#{host}\s#{progname}\z/)
57
+ context "when the progname contains control characters" do
58
+ let(:progname) { "app\n<13>forged" }
59
+
60
+ it "keeps the header on one line" do
61
+ expect(formatted_message.count("\n")).to eq(1)
62
+ expect(formatted_message).to start_with("#{hostname} app <13>forged:@cee:")
63
+ end
33
64
  end
34
65
 
35
- context "without progname" do
36
- let(:progname) { nil }
66
+ context "when a log call is made while another record is being built" do
67
+ let(:formatter) { subject }
68
+ let(:message) do
69
+ f = formatter
70
+ Object.new.tap do |obj|
71
+ obj.define_singleton_method(:inspect) do
72
+ f.call('INFO', Time.now, 'inner', 'nested')
73
+ 'outer message'
74
+ end
75
+ end
76
+ end
37
77
 
38
- it "only includes hostname" do
39
- expect(subject.send(:build_facility, host)).to match(/\A#{host}\z/)
78
+ it "uses the outer call's progname in the outer header" do
79
+ expect(subject.call(severity, time, 'outer', message)).to start_with("#{hostname} outer:@cee:")
40
80
  end
41
81
  end
42
82
  end
83
+
84
+ describe "when the record cannot be serialized" do
85
+ let(:message) { { 'message' => 'x', 'n' => UnserializableValue.new } }
86
+
87
+ it "emits a placeholder in the same CEE/syslog framing" do
88
+ expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
89
+ expect(formatted_message.count("\n")).to eq(1)
90
+ data = JSON.parse(formatted_message[/@cee:(.*)\Z/, 1])
91
+ expect(data['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
92
+ expect(data['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
93
+ end
94
+ end
43
95
  end
@@ -0,0 +1,58 @@
1
+ require 'logstash-logger'
2
+ require_relative '../support/unserializable_value'
3
+ require 'logger'
4
+ require 'net/http'
5
+
6
+ # A record that cannot be formatted must not change how its neighbours are
7
+ # framed on the wire.
8
+ describe 'Record framing when a record cannot be formatted' do
9
+ let(:failing) { { 'message' => 'x', 'n' => UnserializableValue.new } }
10
+ let(:bad_timestamp) { %q({"message":"y","@timestamp":"not-a-time"}) }
11
+
12
+ before do
13
+ LogStashLogger.configure { |c| c.default_error_logger = ::Logger.new(IO::NULL) }
14
+ end
15
+
16
+ after { LogStashLogger.configure { |c| c.default_error_logger = ::Logger.new($stderr) } }
17
+
18
+ it 'keeps every line of a json_lines stream parseable' do
19
+ io = StringIO.new
20
+ logger = LogStashLogger.new(type: :io, io: io, sync: true)
21
+ logger.info(failing)
22
+ logger.info(bad_timestamp)
23
+ logger.info('next record')
24
+
25
+ lines = io.string.lines
26
+ expect(lines.size).to eq(3)
27
+ parsed = lines.map { |l| JSON.parse(l) }
28
+ expect(parsed[0]['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
29
+ expect(parsed[1]['message']).to eq('y')
30
+ expect(parsed[2]['message']).to eq('next record')
31
+ end
32
+
33
+ it 'keeps every record of a cee_syslog stream on its own line' do
34
+ io = StringIO.new
35
+ logger = LogStashLogger.new(type: :io, io: io, sync: true, formatter: :cee_syslog)
36
+ logger.info(failing)
37
+ logger.info(%q({"message":"z","host":{"hostname":"web1\n<13>forged"}}))
38
+ logger.info('next record')
39
+
40
+ lines = io.string.lines
41
+ expect(lines.size).to eq(3)
42
+ lines.each { |l| expect(l).to start_with("#{Socket.gethostname}:@cee:") }
43
+ end
44
+
45
+ it 'keeps an HTTP batch a valid JSON array' do
46
+ formatter = LogStashLogger::Formatter.new(:json_lines)
47
+ messages = [failing, 'next record'].map { |m| formatter.call('INFO', Time.now, nil, m) }
48
+ device = LogStashLogger::Device.new(type: :http, url: 'http://localhost', sync: true)
49
+ body = nil
50
+ allow(Net::HTTP).to receive(:post) { |_url, b, _h| body = b; Net::HTTPOK.new('1.1', '200', 'OK') }
51
+
52
+ device.write_batch(messages)
53
+
54
+ batch = JSON.parse(body)
55
+ expect(batch.size).to eq(2)
56
+ expect(batch[1]['message']).to eq('next record')
57
+ end
58
+ end
@@ -1,4 +1,5 @@
1
1
  require 'logstash-logger'
2
+ require_relative '../support/unserializable_value'
2
3
 
3
4
  describe LogStashLogger::Formatter::JsonLines do
4
5
  include_context "formatter"
@@ -11,4 +12,44 @@ describe LogStashLogger::Formatter::JsonLines do
11
12
  it "terminates with a line break" do
12
13
  expect(formatted_message[-1]).to eq("\n")
13
14
  end
15
+
16
+ context 'encoding error is raised' do
17
+ let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT') }
18
+
19
+ before do
20
+ allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
21
+ end
22
+
23
+ it 'logs the error' do
24
+ expect(subject).to receive(:log_error)
25
+ formatted_message
26
+ end
27
+
28
+ it 'forces the message encoding to utf8' do
29
+ expect(subject).to receive(:force_utf8_encoding)
30
+ formatted_message
31
+ end
32
+
33
+ it "outputs in JSON format with message encoding updated to utf8" do
34
+ json_message = JSON.parse(formatted_message)
35
+ expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
36
+ end
37
+ end
38
+
39
+ context 'when the record cannot be serialized' do
40
+ let(:message) { { 'message' => 'x', 'n' => UnserializableValue.new } }
41
+
42
+ before do
43
+ allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
44
+ end
45
+
46
+ it 'emits a JSON placeholder line instead of a bare string' do
47
+ expect(formatted_message).to end_with("\n")
48
+ expect(formatted_message.count("\n")).to eq(1)
49
+ data = JSON.parse(formatted_message)
50
+ expect(data['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
51
+ expect(data['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
52
+ expect(data['error']).to eq('JSON::GeneratorError')
53
+ end
54
+ end
14
55
  end
@@ -7,4 +7,53 @@ describe LogStashLogger::Formatter::Json do
7
7
  json_message = JSON.parse(formatted_message)
8
8
  expect(json_message["message"]).to eq(message)
9
9
  end
10
+
11
+ context 'encoding error is raised' do
12
+ let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT') }
13
+
14
+ before do
15
+ allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
16
+ end
17
+
18
+ it 'logs the error' do
19
+ expect(subject).to receive(:log_error)
20
+ formatted_message
21
+ end
22
+
23
+ it 'forces the message encoding to utf8' do
24
+ expect(subject).to receive(:force_utf8_encoding)
25
+ formatted_message
26
+ end
27
+
28
+ it "outputs in JSON format with message encoding updated to utf8" do
29
+ json_message = JSON.parse(formatted_message)
30
+ expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
31
+ end
32
+ end
33
+
34
+ context "timestamp ordering" do
35
+ let(:message) { { message: 'test', foo: 'bar' } }
36
+
37
+ it "places @timestamp first in JSON output" do
38
+ timestamp_index = formatted_message.index('"@timestamp"')
39
+ message_index = formatted_message.index('"message"')
40
+
41
+ expect(timestamp_index).not_to be_nil
42
+ expect(message_index).not_to be_nil
43
+ expect(timestamp_index).to be < message_index
44
+ end
45
+ end
46
+
47
+ context "key ordering" do
48
+ let(:message) { { foo: 'bar', baz: 'qux' } }
49
+
50
+ it "preserves custom field order in JSON output" do
51
+ foo_index = formatted_message.index('"foo"')
52
+ baz_index = formatted_message.index('"baz"')
53
+
54
+ expect(foo_index).not_to be_nil
55
+ expect(baz_index).not_to be_nil
56
+ expect(foo_index).to be < baz_index
57
+ end
58
+ end
10
59
  end
@@ -35,6 +35,23 @@ describe LogStashLogger::Formatter do
35
35
  end
36
36
  end
37
37
 
38
+ context "formatter class that accepts customize_event keyword arg in initializer" do
39
+ let(:formatter) do
40
+ Class.new do
41
+ def initialize(customize_event: nil)
42
+ end
43
+
44
+ def call
45
+ end
46
+ end
47
+ end
48
+
49
+ it "returns a new instance of the class" do
50
+ expect(formatter).to receive(:new).with(hash_including(:customize_event)).and_call_original
51
+ expect(subject).to be_a formatter
52
+ end
53
+ end
54
+
38
55
  context "formatter instance" do
39
56
  let(:formatter) { ::Logger::Formatter.new }
40
57
 
@@ -49,7 +66,7 @@ describe LogStashLogger::Formatter do
49
66
 
50
67
  context "formatter proc" do
51
68
  let(:formatter) do
52
- proc { |severity, time, progname, msg| msg }
69
+ proc { |_severity, _time, _progname, msg| msg }
53
70
  end
54
71
 
55
72
  it "returns the same formatter proc" do
@@ -63,7 +80,7 @@ describe LogStashLogger::Formatter do
63
80
 
64
81
  context "formatter lambda" do
65
82
  let(:formatter) do
66
- ->(severity, time, progname, msg) { msg }
83
+ ->(_severity, _time, _progname, msg) { msg }
67
84
  end
68
85
 
69
86
  it "returns the same formatter lambda" do