logstash-logger 0.26.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/tests.yml +53 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +85 -105
- data/Appraisals +6 -20
- data/CHANGELOG.md +61 -0
- data/Gemfile +2 -2
- data/README.md +12 -833
- data/docs/buffering.md +86 -0
- data/docs/customization.md +86 -0
- data/docs/outputs.md +65 -0
- data/docs/rails.md +349 -0
- data/docs/ssl.md +102 -0
- data/docs/troubleshooting.md +106 -0
- data/docs/usage.md +148 -0
- data/gemfiles/rails_8.0.gemfile +10 -0
- data/gemfiles/rails_8.1.gemfile +9 -0
- data/lib/logstash-logger/buffer.rb +15 -5
- data/lib/logstash-logger/configuration.rb +1 -2
- data/lib/logstash-logger/device/aws_stream.rb +96 -17
- data/lib/logstash-logger/device/base.rb +2 -2
- data/lib/logstash-logger/device/connectable.rb +30 -12
- data/lib/logstash-logger/device/file.rb +21 -4
- data/lib/logstash-logger/device/firehose.rb +6 -0
- data/lib/logstash-logger/device/http.rb +104 -0
- data/lib/logstash-logger/device/kafka.rb +182 -38
- data/lib/logstash-logger/device/kinesis.rb +10 -1
- data/lib/logstash-logger/device/redis.rb +8 -1
- data/lib/logstash-logger/device/tcp.rb +54 -12
- data/lib/logstash-logger/device.rb +24 -2
- data/lib/logstash-logger/formatter/base.rb +97 -11
- data/lib/logstash-logger/formatter/cee_syslog.rb +8 -11
- data/lib/logstash-logger/formatter/json.rb +13 -0
- data/lib/logstash-logger/formatter/json_lines.rb +13 -0
- data/lib/logstash-logger/formatter.rb +14 -6
- data/lib/logstash-logger/logger.rb +6 -19
- data/lib/logstash-logger/multi_logger.rb +2 -1
- data/lib/logstash-logger/railtie.rb +1 -1
- data/lib/logstash-logger/tagged_logging.rb +3 -1
- data/lib/logstash-logger/version.rb +1 -1
- data/logstash-logger.gemspec +8 -4
- data/spec/device/aws_stream_limits_spec.rb +219 -0
- data/spec/device/connectable_spec.rb +51 -0
- data/spec/device/file_spec.rb +65 -0
- data/spec/device/firehose_spec.rb +4 -1
- data/spec/device/http_spec.rb +176 -0
- data/spec/device/kafka_spec.rb +429 -14
- data/spec/device/kinesis_spec.rb +4 -1
- data/spec/device/tcp_spec.rb +94 -5
- data/spec/device/tcp_tls_integration_spec.rb +91 -0
- data/spec/device_spec.rb +13 -0
- data/spec/formatter/base_spec.rb +82 -1
- data/spec/formatter/cee_syslog_spec.rb +71 -19
- data/spec/formatter/framing_spec.rb +58 -0
- data/spec/formatter/json_lines_spec.rb +41 -0
- data/spec/formatter/json_spec.rb +49 -0
- data/spec/formatter_spec.rb +19 -2
- data/spec/logger_spec.rb +5 -5
- data/spec/multi_logger_spec.rb +16 -0
- data/spec/spec_helper.rb +7 -5
- data/spec/support/tls_helper.rb +30 -0
- data/spec/support/unserializable_value.rb +10 -0
- data/spec/tagged_logging_spec.rb +15 -0
- metadata +101 -16
- data/.travis.yml +0 -26
- data/gemfiles/rails_3.2.gemfile +0 -9
- data/gemfiles/rails_4.0.gemfile +0 -9
- data/gemfiles/rails_4.1.gemfile +0 -9
- data/gemfiles/rails_4.2.gemfile +0 -9
- data/gemfiles/rails_5.0.gemfile +0 -9
- data/gemfiles/rails_5.1.gemfile +0 -9
data/spec/device/tcp_spec.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'logstash-logger'
|
|
2
|
+
require_relative '../support/tls_helper'
|
|
2
3
|
|
|
3
4
|
describe LogStashLogger::Device::TCP do
|
|
4
5
|
include_context 'device'
|
|
@@ -13,6 +14,9 @@ describe LogStashLogger::Device::TCP do
|
|
|
13
14
|
allow(OpenSSL::SSL::SSLSocket).to receive(:new) { ssl_socket }
|
|
14
15
|
allow(ssl_socket).to receive(:connect)
|
|
15
16
|
allow(ssl_socket).to receive(:post_connection_check)
|
|
17
|
+
allow(ssl_socket).to receive(:sync_close=)
|
|
18
|
+
allow(ssl_socket).to receive(:hostname=)
|
|
19
|
+
allow(tcp_socket).to receive(:close)
|
|
16
20
|
allow(ssl_tcp_device).to receive(:warn)
|
|
17
21
|
end
|
|
18
22
|
|
|
@@ -103,16 +107,101 @@ describe LogStashLogger::Device::TCP do
|
|
|
103
107
|
end
|
|
104
108
|
|
|
105
109
|
context 'without a provided SSL context' do
|
|
106
|
-
it '
|
|
107
|
-
|
|
110
|
+
it 'uses a context that verifies the server certificate' do
|
|
111
|
+
ctx = ssl_tcp_device.ssl_context
|
|
112
|
+
expect(ctx).to be_a(OpenSSL::SSL::SSLContext)
|
|
113
|
+
expect(ctx.verify_mode).to eq(OpenSSL::SSL::VERIFY_PEER)
|
|
114
|
+
expect(ctx.verify_hostname).to be true
|
|
115
|
+
expect(ctx.cert_store).not_to be_nil
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
it 'creates the socket with that context' do
|
|
119
|
+
expect(OpenSSL::SSL::SSLSocket).to receive(:new).with(tcp_socket, ssl_tcp_device.ssl_context)
|
|
120
|
+
ssl_tcp_device.connect
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
it 'verifies the hostname after connecting' do
|
|
124
|
+
expect(ssl_socket).to receive(:post_connection_check).with(HOST)
|
|
125
|
+
ssl_tcp_device.connect
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
context 'with verify_hostname: false' do
|
|
129
|
+
let(:ssl_tcp_device) { LogStashLogger::Device.new(type: :tcp, port: port, ssl_enable: true, sync: true, verify_hostname: false) }
|
|
130
|
+
|
|
131
|
+
it 'still verifies the certificate chain' do
|
|
132
|
+
expect(ssl_tcp_device.ssl_context.verify_mode).to eq(OpenSSL::SSL::VERIFY_PEER)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
it 'does not verify the hostname' do
|
|
136
|
+
expect(ssl_tcp_device.ssl_context.verify_hostname).to be false
|
|
137
|
+
expect(ssl_socket).not_to receive(:post_connection_check)
|
|
138
|
+
ssl_tcp_device.connect
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
context 'server name indication' do
|
|
144
|
+
it 'is sent for a DNS hostname' do
|
|
145
|
+
device = LogStashLogger::Device.new(type: :tcp, host: 'logstash.example.com', port: port, ssl_enable: true, sync: true)
|
|
146
|
+
expect(ssl_socket).to receive(:hostname=).with('logstash.example.com')
|
|
147
|
+
device.connect
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
it 'is not sent for an IP address' do
|
|
151
|
+
expect(ssl_socket).not_to receive(:hostname=)
|
|
152
|
+
ssl_tcp_device.connect
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
it 'closes the TCP socket together with the SSL socket' do
|
|
157
|
+
expect(ssl_socket).to receive(:sync_close=).with(true)
|
|
158
|
+
ssl_tcp_device.connect
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
context 'when the SSL socket cannot be created' do
|
|
162
|
+
before do
|
|
163
|
+
allow(OpenSSL::SSL::SSLSocket).to receive(:new).and_raise(OpenSSL::SSL::SSLError, 'boom')
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
it 'closes the TCP socket it opened' do
|
|
167
|
+
expect(tcp_socket).to receive(:close).once
|
|
168
|
+
expect { ssl_tcp_device.connect }.to raise_error(OpenSSL::SSL::SSLError)
|
|
108
169
|
end
|
|
109
170
|
end
|
|
110
171
|
|
|
111
172
|
context 'only providing a certificate file' do
|
|
112
|
-
let(:
|
|
173
|
+
let(:cert_file) { TLSHelper.cert_file(TLSHelper.self_signed.first) }
|
|
174
|
+
let(:ssl_tcp_device) { LogStashLogger::Device.new(type: :tcp, port: port, ssl_enable: true, sync: true, ssl_certificate: cert_file.path) }
|
|
175
|
+
|
|
176
|
+
after { cert_file.close! }
|
|
177
|
+
|
|
178
|
+
it 'trusts that certificate and verifies the server against it' do
|
|
179
|
+
ctx = ssl_tcp_device.ssl_context
|
|
180
|
+
expect(ctx.ca_file).to eq(cert_file.path)
|
|
181
|
+
expect(ctx.verify_mode).to eq(OpenSSL::SSL::VERIFY_PEER)
|
|
182
|
+
expect(ctx.verify_hostname).to be true
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
it 'enables SSL implicitly' do
|
|
186
|
+
device = LogStashLogger::Device.new(type: :tcp, port: port, sync: true, ssl_certificate: cert_file.path)
|
|
187
|
+
expect(device.use_ssl?).to be_truthy
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
it 'rejects a missing file' do
|
|
191
|
+
expect {
|
|
192
|
+
LogStashLogger::Device.new(type: :tcp, port: port, sync: true, ssl_certificate: '/nonexistent/cert.pem')
|
|
193
|
+
}.to raise_error(ArgumentError, /readable certificate file/)
|
|
194
|
+
end
|
|
113
195
|
|
|
114
|
-
it '
|
|
115
|
-
|
|
196
|
+
it 'rejects a file that is not a certificate' do
|
|
197
|
+
not_a_cert = Tempfile.new('not-a-cert')
|
|
198
|
+
not_a_cert.write('hello')
|
|
199
|
+
not_a_cert.flush
|
|
200
|
+
expect {
|
|
201
|
+
LogStashLogger::Device.new(type: :tcp, port: port, sync: true, ssl_certificate: not_a_cert.path)
|
|
202
|
+
}.to raise_error(ArgumentError, /not a PEM or DER certificate/)
|
|
203
|
+
ensure
|
|
204
|
+
not_a_cert.close!
|
|
116
205
|
end
|
|
117
206
|
end
|
|
118
207
|
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
require 'logstash-logger'
|
|
2
|
+
require 'logger'
|
|
3
|
+
require 'socket'
|
|
4
|
+
require_relative '../support/tls_helper'
|
|
5
|
+
|
|
6
|
+
# Real TLS handshakes against a loopback server that presents a throwaway
|
|
7
|
+
# self-signed certificate. Nothing leaves 127.0.0.1.
|
|
8
|
+
describe LogStashLogger::Device::TCP, 'TLS verification against a real server' do
|
|
9
|
+
before(:all) do
|
|
10
|
+
@cert, @key = TLSHelper.self_signed(dns: ['localhost'], ips: [])
|
|
11
|
+
@cert_file = TLSHelper.cert_file(@cert)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
after(:all) { @cert_file.close! }
|
|
15
|
+
|
|
16
|
+
let(:received) { Queue.new }
|
|
17
|
+
let(:server) do
|
|
18
|
+
tcp = TCPServer.new('127.0.0.1', 0)
|
|
19
|
+
ctx = OpenSSL::SSL::SSLContext.new
|
|
20
|
+
ctx.cert = @cert
|
|
21
|
+
ctx.key = @key
|
|
22
|
+
OpenSSL::SSL::SSLServer.new(tcp, ctx)
|
|
23
|
+
end
|
|
24
|
+
let(:port) { server.to_io.addr[1] }
|
|
25
|
+
|
|
26
|
+
before do
|
|
27
|
+
srv = server
|
|
28
|
+
q = received
|
|
29
|
+
@server_thread = Thread.new do
|
|
30
|
+
loop do
|
|
31
|
+
begin
|
|
32
|
+
client = srv.accept
|
|
33
|
+
q << client.gets
|
|
34
|
+
client.close
|
|
35
|
+
rescue OpenSSL::SSL::SSLError, IOError, SystemCallError
|
|
36
|
+
# A rejected handshake is expected in several examples
|
|
37
|
+
break if srv.to_io.closed?
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
after do
|
|
44
|
+
server.close rescue nil
|
|
45
|
+
@server_thread.kill
|
|
46
|
+
@server_thread.join(1)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def device(**opts)
|
|
50
|
+
LogStashLogger::Device.new(
|
|
51
|
+
{ type: :tcp, port: port, sync: true, buffer_flush_at_exit: false,
|
|
52
|
+
error_logger: ::Logger.new(IO::NULL) }.merge(opts)
|
|
53
|
+
)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def deliver(dev)
|
|
57
|
+
dev.connect
|
|
58
|
+
dev.io.write(%({"message":"hello"}\n))
|
|
59
|
+
dev.io.flush
|
|
60
|
+
line = received.pop(timeout: 5)
|
|
61
|
+
dev.close!
|
|
62
|
+
line
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
it 'refuses a self-signed certificate when only ssl_enable is set' do
|
|
66
|
+
expect { device(host: 'localhost', ssl_enable: true).connect }
|
|
67
|
+
.to raise_error(OpenSSL::SSL::SSLError)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
it 'delivers when the certificate is trusted through ssl_certificate' do
|
|
71
|
+
expect(deliver(device(host: 'localhost', ssl_certificate: @cert_file.path)))
|
|
72
|
+
.to eq(%({"message":"hello"}\n))
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
it 'refuses a certificate whose name does not match the host' do
|
|
76
|
+
expect { device(host: '127.0.0.1', ssl_certificate: @cert_file.path).connect }
|
|
77
|
+
.to raise_error(OpenSSL::SSL::SSLError)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
it 'checks the name given in verify_hostname when connecting by IP' do
|
|
81
|
+
dev = device(host: '127.0.0.1', ssl_certificate: @cert_file.path, verify_hostname: 'localhost')
|
|
82
|
+
expect(deliver(dev)).to eq(%({"message":"hello"}\n))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
it 'still allows an explicit VERIFY_NONE context as an opt-out' do
|
|
86
|
+
ctx = OpenSSL::SSL::SSLContext.new
|
|
87
|
+
ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
88
|
+
dev = device(host: '127.0.0.1', ssl_context: ctx, verify_hostname: false)
|
|
89
|
+
expect(deliver(dev)).to eq(%({"message":"hello"}\n))
|
|
90
|
+
end
|
|
91
|
+
end
|
data/spec/device_spec.rb
CHANGED
|
@@ -38,6 +38,19 @@ describe LogStashLogger::Device do
|
|
|
38
38
|
it { is_expected.to eq({type: 'udp', host: 'localhost', port: 5228, path: ''}) }
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
+
context "when uri includes buffer_max_items" do
|
|
42
|
+
let(:uri_config) { {uri: "udp://localhost:5228?buffer_max_items=75"} }
|
|
43
|
+
it do
|
|
44
|
+
expect(parse_uri_config).to eq({
|
|
45
|
+
type: 'udp',
|
|
46
|
+
host: 'localhost',
|
|
47
|
+
port: 5228,
|
|
48
|
+
path: '',
|
|
49
|
+
buffer_max_items: 75,
|
|
50
|
+
})
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
41
54
|
context "when uri is invalid" do
|
|
42
55
|
let(:uri_config) { invalid_uri_config }
|
|
43
56
|
specify { expect { parse_uri_config }.to raise_error(URI::InvalidURIError) }
|
data/spec/formatter/base_spec.rb
CHANGED
|
@@ -23,6 +23,87 @@ describe LogStashLogger::Formatter::Base do
|
|
|
23
23
|
expect(subject.call(severity, time, progname, message)).to be_nil
|
|
24
24
|
end
|
|
25
25
|
end
|
|
26
|
+
|
|
27
|
+
context "when building the event raises" do
|
|
28
|
+
before do
|
|
29
|
+
allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
|
|
30
|
+
allow(subject).to receive(:build_event).and_raise(ArgumentError, 'boom')
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it "logs an exception" do
|
|
34
|
+
expect(subject).to receive(:log_error).and_call_original
|
|
35
|
+
subject.call(severity, time, progname, message)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it "returns a well-formed placeholder event instead of a bare string" do
|
|
39
|
+
event = subject.call(severity, time, progname, message)
|
|
40
|
+
expect(event).to be_a(LogStash::Event)
|
|
41
|
+
expect(event['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
|
|
42
|
+
expect(event['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
|
|
43
|
+
expect(event['error']).to eq('ArgumentError')
|
|
44
|
+
expect(event['severity']).to eq(severity)
|
|
45
|
+
expect(event['@timestamp']).to eq(time.iso8601(3))
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
context "when even the placeholder cannot be formatted" do
|
|
50
|
+
before do
|
|
51
|
+
allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
|
|
52
|
+
allow(subject).to receive(:format_event).and_raise(ArgumentError, 'boom')
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
it "returns nil, which devices drop" do
|
|
56
|
+
expect(subject.call(severity, time, progname, message)).to be_nil
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
context "when logged content carries an @timestamp that is not a time" do
|
|
61
|
+
let(:message) { '{"message":"hello","@timestamp":"not-a-time"}' }
|
|
62
|
+
|
|
63
|
+
it "keeps the logger time and formats the record normally" do
|
|
64
|
+
event = subject.call(severity, time, progname, message)
|
|
65
|
+
expect(event['message']).to eq('hello')
|
|
66
|
+
expect(event['@timestamp']).to eq(time.iso8601(3))
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
context "when logged content carries a valid @timestamp" do
|
|
71
|
+
let(:message) { '{"message":"hello","@timestamp":"2001-01-01T00:00:00Z"}' }
|
|
72
|
+
|
|
73
|
+
it "keeps it" do
|
|
74
|
+
event = subject.call(severity, time, progname, message)
|
|
75
|
+
expect(event['@timestamp']).to start_with('2001-01-01T00:00:00')
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
describe '#force_utf8_encoding' do
|
|
81
|
+
let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT').freeze }
|
|
82
|
+
let(:event) { LogStash::Event.new("message" => message) }
|
|
83
|
+
|
|
84
|
+
it 'returns the same event' do
|
|
85
|
+
expect(subject.send(:force_utf8_encoding, event)).to eq(event)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
it 'forces the event message to UTF-8 encoding' do
|
|
89
|
+
subject.send(:force_utf8_encoding, event)
|
|
90
|
+
updated_event_data = event.instance_variable_get(:@data)
|
|
91
|
+
expect(updated_event_data['message'].encoding.name).to eq('UTF-8')
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
context 'frozen message string' do
|
|
95
|
+
let(:message) { "foo x\xc3x".dup.force_encoding('ASCII-8BIT').freeze }
|
|
96
|
+
|
|
97
|
+
it 'returns the same event' do
|
|
98
|
+
expect(subject.send(:force_utf8_encoding, event)).to eq(event)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
it 'forces the event message to UTF-8 encoding' do
|
|
102
|
+
subject.send(:force_utf8_encoding, event)
|
|
103
|
+
updated_event_data = event.instance_variable_get(:@data)
|
|
104
|
+
expect(updated_event_data['message'].encoding.name).to eq('UTF-8')
|
|
105
|
+
end
|
|
106
|
+
end
|
|
26
107
|
end
|
|
27
108
|
|
|
28
109
|
describe "#build_event" do
|
|
@@ -81,7 +162,7 @@ describe LogStashLogger::Formatter::Base do
|
|
|
81
162
|
end
|
|
82
163
|
|
|
83
164
|
it "adds host" do
|
|
84
|
-
expect(event['host']).to eq(hostname)
|
|
165
|
+
expect(event['host']['hostname']).to eq(hostname)
|
|
85
166
|
end
|
|
86
167
|
end
|
|
87
168
|
|
|
@@ -1,43 +1,95 @@
|
|
|
1
1
|
require 'logstash-logger'
|
|
2
|
+
require_relative '../support/unserializable_value'
|
|
2
3
|
|
|
3
4
|
describe LogStashLogger::Formatter::CeeSyslog do
|
|
4
5
|
include_context "formatter"
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
let(:facility) { "facility" }
|
|
8
|
-
|
|
9
|
-
before do
|
|
10
|
-
allow(subject).to receive(:build_facility).and_return(facility)
|
|
11
|
-
end
|
|
7
|
+
before { subject.error_logger = Logger.new(IO::NULL) }
|
|
12
8
|
|
|
13
|
-
|
|
14
|
-
|
|
9
|
+
describe "#call" do
|
|
10
|
+
it "outputs this host and the progname before the @cee" do
|
|
11
|
+
expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
|
|
15
12
|
end
|
|
16
13
|
|
|
17
14
|
it "serializes the LogStash::Event data as JSON" do
|
|
18
|
-
json_data = formatted_message[/\A
|
|
15
|
+
json_data = formatted_message[/\A[^:]*:@cee:\s?(.*)\Z/, 1]
|
|
19
16
|
json_message = JSON.parse(json_data)
|
|
20
17
|
expect(json_message["message"]).to eq(message)
|
|
21
18
|
end
|
|
19
|
+
|
|
20
|
+
it "ends the record with exactly one newline" do
|
|
21
|
+
expect(formatted_message.count("\n")).to eq(1)
|
|
22
|
+
expect(formatted_message).to end_with("\n")
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
context "without progname" do
|
|
26
|
+
let(:progname) { nil }
|
|
27
|
+
|
|
28
|
+
it "only includes the hostname" do
|
|
29
|
+
expect(formatted_message).to start_with("#{hostname}:@cee:")
|
|
30
|
+
end
|
|
31
|
+
end
|
|
22
32
|
end
|
|
23
33
|
|
|
24
|
-
describe "
|
|
25
|
-
|
|
34
|
+
describe "header" do
|
|
35
|
+
context "when logged content sets host.hostname" do
|
|
36
|
+
let(:message) { %q({"message":"hi","host":{"hostname":"web1\n<13>Jan 1 00:00:00 otherhost app: forged"}}) }
|
|
37
|
+
|
|
38
|
+
it "uses this process's hostname, not the logged one" do
|
|
39
|
+
expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
it "cannot start a second record" do
|
|
43
|
+
expect(formatted_message.count("\n")).to eq(1)
|
|
44
|
+
expect(formatted_message).to end_with("\n")
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
context "when logged content sets host to a String" do
|
|
49
|
+
let(:message) { %q({"message":"hi","host":"web1"}) }
|
|
26
50
|
|
|
27
|
-
|
|
28
|
-
|
|
51
|
+
it "formats the record normally" do
|
|
52
|
+
expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
|
|
53
|
+
expect(JSON.parse(formatted_message[/@cee:(.*)\Z/, 1])['host']).to eq('web1')
|
|
54
|
+
end
|
|
29
55
|
end
|
|
30
56
|
|
|
31
|
-
|
|
32
|
-
|
|
57
|
+
context "when the progname contains control characters" do
|
|
58
|
+
let(:progname) { "app\n<13>forged" }
|
|
59
|
+
|
|
60
|
+
it "keeps the header on one line" do
|
|
61
|
+
expect(formatted_message.count("\n")).to eq(1)
|
|
62
|
+
expect(formatted_message).to start_with("#{hostname} app <13>forged:@cee:")
|
|
63
|
+
end
|
|
33
64
|
end
|
|
34
65
|
|
|
35
|
-
context "
|
|
36
|
-
let(:
|
|
66
|
+
context "when a log call is made while another record is being built" do
|
|
67
|
+
let(:formatter) { subject }
|
|
68
|
+
let(:message) do
|
|
69
|
+
f = formatter
|
|
70
|
+
Object.new.tap do |obj|
|
|
71
|
+
obj.define_singleton_method(:inspect) do
|
|
72
|
+
f.call('INFO', Time.now, 'inner', 'nested')
|
|
73
|
+
'outer message'
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
37
77
|
|
|
38
|
-
it "
|
|
39
|
-
expect(subject.
|
|
78
|
+
it "uses the outer call's progname in the outer header" do
|
|
79
|
+
expect(subject.call(severity, time, 'outer', message)).to start_with("#{hostname} outer:@cee:")
|
|
40
80
|
end
|
|
41
81
|
end
|
|
42
82
|
end
|
|
83
|
+
|
|
84
|
+
describe "when the record cannot be serialized" do
|
|
85
|
+
let(:message) { { 'message' => 'x', 'n' => UnserializableValue.new } }
|
|
86
|
+
|
|
87
|
+
it "emits a placeholder in the same CEE/syslog framing" do
|
|
88
|
+
expect(formatted_message).to start_with("#{hostname} #{progname}:@cee:")
|
|
89
|
+
expect(formatted_message.count("\n")).to eq(1)
|
|
90
|
+
data = JSON.parse(formatted_message[/@cee:(.*)\Z/, 1])
|
|
91
|
+
expect(data['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
|
|
92
|
+
expect(data['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
|
|
93
|
+
end
|
|
94
|
+
end
|
|
43
95
|
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
require 'logstash-logger'
|
|
2
|
+
require_relative '../support/unserializable_value'
|
|
3
|
+
require 'logger'
|
|
4
|
+
require 'net/http'
|
|
5
|
+
|
|
6
|
+
# A record that cannot be formatted must not change how its neighbours are
|
|
7
|
+
# framed on the wire.
|
|
8
|
+
describe 'Record framing when a record cannot be formatted' do
|
|
9
|
+
let(:failing) { { 'message' => 'x', 'n' => UnserializableValue.new } }
|
|
10
|
+
let(:bad_timestamp) { %q({"message":"y","@timestamp":"not-a-time"}) }
|
|
11
|
+
|
|
12
|
+
before do
|
|
13
|
+
LogStashLogger.configure { |c| c.default_error_logger = ::Logger.new(IO::NULL) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
after { LogStashLogger.configure { |c| c.default_error_logger = ::Logger.new($stderr) } }
|
|
17
|
+
|
|
18
|
+
it 'keeps every line of a json_lines stream parseable' do
|
|
19
|
+
io = StringIO.new
|
|
20
|
+
logger = LogStashLogger.new(type: :io, io: io, sync: true)
|
|
21
|
+
logger.info(failing)
|
|
22
|
+
logger.info(bad_timestamp)
|
|
23
|
+
logger.info('next record')
|
|
24
|
+
|
|
25
|
+
lines = io.string.lines
|
|
26
|
+
expect(lines.size).to eq(3)
|
|
27
|
+
parsed = lines.map { |l| JSON.parse(l) }
|
|
28
|
+
expect(parsed[0]['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
|
|
29
|
+
expect(parsed[1]['message']).to eq('y')
|
|
30
|
+
expect(parsed[2]['message']).to eq('next record')
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it 'keeps every record of a cee_syslog stream on its own line' do
|
|
34
|
+
io = StringIO.new
|
|
35
|
+
logger = LogStashLogger.new(type: :io, io: io, sync: true, formatter: :cee_syslog)
|
|
36
|
+
logger.info(failing)
|
|
37
|
+
logger.info(%q({"message":"z","host":{"hostname":"web1\n<13>forged"}}))
|
|
38
|
+
logger.info('next record')
|
|
39
|
+
|
|
40
|
+
lines = io.string.lines
|
|
41
|
+
expect(lines.size).to eq(3)
|
|
42
|
+
lines.each { |l| expect(l).to start_with("#{Socket.gethostname}:@cee:") }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
it 'keeps an HTTP batch a valid JSON array' do
|
|
46
|
+
formatter = LogStashLogger::Formatter.new(:json_lines)
|
|
47
|
+
messages = [failing, 'next record'].map { |m| formatter.call('INFO', Time.now, nil, m) }
|
|
48
|
+
device = LogStashLogger::Device.new(type: :http, url: 'http://localhost', sync: true)
|
|
49
|
+
body = nil
|
|
50
|
+
allow(Net::HTTP).to receive(:post) { |_url, b, _h| body = b; Net::HTTPOK.new('1.1', '200', 'OK') }
|
|
51
|
+
|
|
52
|
+
device.write_batch(messages)
|
|
53
|
+
|
|
54
|
+
batch = JSON.parse(body)
|
|
55
|
+
expect(batch.size).to eq(2)
|
|
56
|
+
expect(batch[1]['message']).to eq('next record')
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'logstash-logger'
|
|
2
|
+
require_relative '../support/unserializable_value'
|
|
2
3
|
|
|
3
4
|
describe LogStashLogger::Formatter::JsonLines do
|
|
4
5
|
include_context "formatter"
|
|
@@ -11,4 +12,44 @@ describe LogStashLogger::Formatter::JsonLines do
|
|
|
11
12
|
it "terminates with a line break" do
|
|
12
13
|
expect(formatted_message[-1]).to eq("\n")
|
|
13
14
|
end
|
|
15
|
+
|
|
16
|
+
context 'encoding error is raised' do
|
|
17
|
+
let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT') }
|
|
18
|
+
|
|
19
|
+
before do
|
|
20
|
+
allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
it 'logs the error' do
|
|
24
|
+
expect(subject).to receive(:log_error)
|
|
25
|
+
formatted_message
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
it 'forces the message encoding to utf8' do
|
|
29
|
+
expect(subject).to receive(:force_utf8_encoding)
|
|
30
|
+
formatted_message
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it "outputs in JSON format with message encoding updated to utf8" do
|
|
34
|
+
json_message = JSON.parse(formatted_message)
|
|
35
|
+
expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
context 'when the record cannot be serialized' do
|
|
40
|
+
let(:message) { { 'message' => 'x', 'n' => UnserializableValue.new } }
|
|
41
|
+
|
|
42
|
+
before do
|
|
43
|
+
allow(subject).to receive(:error_logger).and_return(Logger.new(IO::NULL))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
it 'emits a JSON placeholder line instead of a bare string' do
|
|
47
|
+
expect(formatted_message).to end_with("\n")
|
|
48
|
+
expect(formatted_message.count("\n")).to eq(1)
|
|
49
|
+
data = JSON.parse(formatted_message)
|
|
50
|
+
expect(data['message']).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
|
|
51
|
+
expect(data['tags']).to eq([LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_TAG])
|
|
52
|
+
expect(data['error']).to eq('JSON::GeneratorError')
|
|
53
|
+
end
|
|
54
|
+
end
|
|
14
55
|
end
|
data/spec/formatter/json_spec.rb
CHANGED
|
@@ -7,4 +7,53 @@ describe LogStashLogger::Formatter::Json do
|
|
|
7
7
|
json_message = JSON.parse(formatted_message)
|
|
8
8
|
expect(json_message["message"]).to eq(message)
|
|
9
9
|
end
|
|
10
|
+
|
|
11
|
+
context 'encoding error is raised' do
|
|
12
|
+
let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT') }
|
|
13
|
+
|
|
14
|
+
before do
|
|
15
|
+
allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
it 'logs the error' do
|
|
19
|
+
expect(subject).to receive(:log_error)
|
|
20
|
+
formatted_message
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
it 'forces the message encoding to utf8' do
|
|
24
|
+
expect(subject).to receive(:force_utf8_encoding)
|
|
25
|
+
formatted_message
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
it "outputs in JSON format with message encoding updated to utf8" do
|
|
29
|
+
json_message = JSON.parse(formatted_message)
|
|
30
|
+
expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
context "timestamp ordering" do
|
|
35
|
+
let(:message) { { message: 'test', foo: 'bar' } }
|
|
36
|
+
|
|
37
|
+
it "places @timestamp first in JSON output" do
|
|
38
|
+
timestamp_index = formatted_message.index('"@timestamp"')
|
|
39
|
+
message_index = formatted_message.index('"message"')
|
|
40
|
+
|
|
41
|
+
expect(timestamp_index).not_to be_nil
|
|
42
|
+
expect(message_index).not_to be_nil
|
|
43
|
+
expect(timestamp_index).to be < message_index
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
context "key ordering" do
|
|
48
|
+
let(:message) { { foo: 'bar', baz: 'qux' } }
|
|
49
|
+
|
|
50
|
+
it "preserves custom field order in JSON output" do
|
|
51
|
+
foo_index = formatted_message.index('"foo"')
|
|
52
|
+
baz_index = formatted_message.index('"baz"')
|
|
53
|
+
|
|
54
|
+
expect(foo_index).not_to be_nil
|
|
55
|
+
expect(baz_index).not_to be_nil
|
|
56
|
+
expect(foo_index).to be < baz_index
|
|
57
|
+
end
|
|
58
|
+
end
|
|
10
59
|
end
|
data/spec/formatter_spec.rb
CHANGED
|
@@ -35,6 +35,23 @@ describe LogStashLogger::Formatter do
|
|
|
35
35
|
end
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
+
context "formatter class that accepts customize_event keyword arg in initializer" do
|
|
39
|
+
let(:formatter) do
|
|
40
|
+
Class.new do
|
|
41
|
+
def initialize(customize_event: nil)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def call
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it "returns a new instance of the class" do
|
|
50
|
+
expect(formatter).to receive(:new).with(hash_including(:customize_event)).and_call_original
|
|
51
|
+
expect(subject).to be_a formatter
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
38
55
|
context "formatter instance" do
|
|
39
56
|
let(:formatter) { ::Logger::Formatter.new }
|
|
40
57
|
|
|
@@ -49,7 +66,7 @@ describe LogStashLogger::Formatter do
|
|
|
49
66
|
|
|
50
67
|
context "formatter proc" do
|
|
51
68
|
let(:formatter) do
|
|
52
|
-
proc { |
|
|
69
|
+
proc { |_severity, _time, _progname, msg| msg }
|
|
53
70
|
end
|
|
54
71
|
|
|
55
72
|
it "returns the same formatter proc" do
|
|
@@ -63,7 +80,7 @@ describe LogStashLogger::Formatter do
|
|
|
63
80
|
|
|
64
81
|
context "formatter lambda" do
|
|
65
82
|
let(:formatter) do
|
|
66
|
-
->(
|
|
83
|
+
->(_severity, _time, _progname, msg) { msg }
|
|
67
84
|
end
|
|
68
85
|
|
|
69
86
|
it "returns the same formatter lambda" do
|