logstash-logger 0.15.2 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +22 -4
- data/Appraisals +4 -0
- data/CHANGELOG.md +13 -0
- data/README.md +48 -11
- data/gemfiles/rails_5.0.gemfile +8 -0
- data/lib/logstash-logger/configuration.rb +3 -0
- data/lib/logstash-logger/device/balancer.rb +1 -1
- data/lib/logstash-logger/device/base.rb +46 -3
- data/lib/logstash-logger/device/connectable.rb +84 -13
- data/lib/logstash-logger/device/kafka.rb +12 -33
- data/lib/logstash-logger/device/multi_delegator.rb +1 -1
- data/lib/logstash-logger/device/redis.rb +12 -23
- data/lib/logstash-logger/device/socket.rb +4 -0
- data/lib/logstash-logger/device/stderr.rb +1 -1
- data/lib/logstash-logger/device/stdout.rb +1 -1
- data/lib/logstash-logger/formatter/base.rb +4 -0
- data/lib/logstash-logger/version.rb +1 -1
- data/logstash-logger.gemspec +8 -1
- data/samples/redis.conf +1 -1
- data/spec/device/connectable_spec.rb +74 -0
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c7f6fe8282858937026c5c9747835e68bd16390f
|
4
|
+
data.tar.gz: 8de048a5dab4c3a46e1f74fd7b192cc7e82ac58f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c6e11150819e39e4ffb061160e332570a8d461974b88d6a933b6a9370c26bccde71a4f53942ddfc29b12e40f2f952df659c291a682dabd7a1a9f944430e02d37
|
7
|
+
data.tar.gz: 62077f3594cf26f2e92af9576bbd392e6059f7c141f5acc7d61e6ea850491f37eaaf642138d343c7b9d22436f8608ca3b9b3163941e9db0b7ce562dc4bae918a
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -2,16 +2,34 @@ language: ruby
|
|
2
2
|
rvm:
|
3
3
|
- 1.9.3
|
4
4
|
- 2.0.0
|
5
|
-
- 2.1
|
6
|
-
- 2.2
|
5
|
+
- 2.1.9
|
6
|
+
- 2.2.5
|
7
|
+
- 2.3.1
|
7
8
|
- jruby-19mode
|
8
|
-
-
|
9
|
+
- jruby-9.0.5.0
|
10
|
+
- rbx-3.9
|
9
11
|
gemfile:
|
10
12
|
- gemfiles/rails_3.2.gemfile
|
11
13
|
- gemfiles/rails_4.0.gemfile
|
12
14
|
- gemfiles/rails_4.1.gemfile
|
13
15
|
- gemfiles/rails_4.2.gemfile
|
16
|
+
- gemfiles/rails_5.0.gemfile
|
14
17
|
matrix:
|
18
|
+
exclude:
|
19
|
+
- rvm: 1.9.3
|
20
|
+
gemfile: gemfiles/rails_5.0.gemfile
|
21
|
+
- rvm: jruby-19mode
|
22
|
+
gemfile: gemfiles/rails_5.0.gemfile
|
23
|
+
- rvm: 2.0.0
|
24
|
+
gemfile: gemfiles/rails_5.0.gemfile
|
25
|
+
- rvm: 2.1.9
|
26
|
+
gemfile: gemfiles/rails_5.0.gemfile
|
27
|
+
- rvm: 2.2.5
|
28
|
+
gemfile: gemfiles/rails_3.2.gemfile
|
29
|
+
- rvm: 2.3.1
|
30
|
+
gemfile: gemfiles/rails_3.2.gemfile
|
15
31
|
allow_failures:
|
16
|
-
- rbx-
|
32
|
+
-rvm: rbx-3.9
|
17
33
|
sudo: false
|
34
|
+
cache: bundler
|
35
|
+
|
data/Appraisals
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
## 0.16.0
|
2
|
+
|
3
|
+
This release is focused on improving the reliability of LogStashLogger.
|
4
|
+
Connectable log devices are now less likely to block or
|
5
|
+
raise an exception, which avoids impacting the operation of the program. There was
|
6
|
+
also a fix to avoid leaking connections.
|
7
|
+
|
8
|
+
- Allow log messages to be dropped. [#81](https://github.com/dwbutler/logstash-logger/pull/81)
|
9
|
+
- Provide `max_message_size` option. [#80](https://github.com/dwbutler/logstash-logger/pull/80)
|
10
|
+
- Unify error logging. [#82](https://github.com/dwbutler/logstash-logger/pull/82)
|
11
|
+
- Drop message when there is an unrecoverable error. [#83](https://github.com/dwbutler/logstash-logger/pull/83)
|
12
|
+
- Safely close connection when reconnecting. [#84](https://github.com/dwbutler/logstash-logger/pull/84)
|
13
|
+
|
1
14
|
## 0.15.2
|
2
15
|
- Fixes Windows support. [#64](https://github.com/dwbutler/logstash-logger/issues/64)
|
3
16
|
|
data/README.md
CHANGED
@@ -162,6 +162,9 @@ input {
|
|
162
162
|
}
|
163
163
|
```
|
164
164
|
|
165
|
+
File and Redis inputs should use the `json` codec instead. For more information
|
166
|
+
read the [Logstash docs](https://www.elastic.co/guide/en/logstash/current/plugins-codecs-json_lines.html).
|
167
|
+
|
165
168
|
See the [samples](https://github.com/dwbutler/logstash-logger/tree/master/samples) directory for more configuration samples.
|
166
169
|
|
167
170
|
## SSL
|
@@ -237,16 +240,20 @@ This configuration would result in the following output.
|
|
237
240
|
|
238
241
|
## Buffering / Automatic Retries
|
239
242
|
|
240
|
-
|
243
|
+
For devices that establish a connection to a remote service, log messages are buffered internally
|
244
|
+
and automatically re-sent if there is a connection problem.
|
241
245
|
Outputs that support batch writing (Redis and Kafka) will write log messages in bulk from the
|
242
246
|
buffer. This functionality is implemented using
|
243
247
|
[Stud::Buffer](https://github.com/jordansissel/ruby-stud/blob/master/lib/stud/buffer.rb).
|
244
248
|
You can configure its behavior by passing the following options to LogStashLogger:
|
245
249
|
|
246
|
-
|
247
|
-
|
250
|
+
* :buffer_max_items - Max number of items to buffer before flushing. Defaults to 50.
|
251
|
+
* :buffer_max_interval - Max number of seconds to wait between flushes. Defaults to 5.
|
252
|
+
* :drop_messages_on_flush_error - Drop messages when there is a flush error. Defaults to false.
|
253
|
+
* :drop_messages_on_full_buffer - Drop messages when the buffer is full.
|
254
|
+
Defaults to true.
|
248
255
|
|
249
|
-
You can turn
|
256
|
+
You can turn buffering off by setting `buffer_max_items` to `1` or `sync` to `true`.
|
250
257
|
|
251
258
|
Please be aware of the following caveats to this behavior:
|
252
259
|
|
@@ -259,19 +266,29 @@ Please be aware of the following caveats to this behavior:
|
|
259
266
|
immediately. In my testing, it took Ruby about 4 seconds to notice the receiving end was down
|
260
267
|
and start raising exceptions. Since logstash listeners over TCP/UDP do not acknowledge received
|
261
268
|
messages, it's not possible to know which log messages to re-send.
|
262
|
-
* If your output source is unavailable long enough,
|
263
|
-
|
269
|
+
* If your output source is unavailable long enough, the buffer will fill up
|
270
|
+
and messages will be dropped.
|
264
271
|
* If your application suddenly terminates (for example, by SIGKILL or a power outage), the whole
|
265
272
|
buffer will be lost.
|
266
273
|
|
267
|
-
|
274
|
+
By default, messages are discarded when the buffer gets full. This can happen
|
275
|
+
if the output source is down for too long.
|
276
|
+
You can make message loss less likely by increasing `buffer_max_items`
|
268
277
|
(so that more events can be held in the buffer), and increasing `buffer_max_interval` (to wait
|
269
278
|
longer between flushes). This will increase memory pressure on your application as log messages
|
270
279
|
accumulate in the buffer, so make sure you have allocated enough memory to your process.
|
271
280
|
|
281
|
+
## Error handling
|
282
|
+
|
283
|
+
If an exception occurs while writing a message to the device, the exception is
|
284
|
+
logged using an internal logger. By default, this logs to $stderr. You can
|
285
|
+
change the error logger by setting `LogStashLogger.configuration.default_error_logger`, or by passsing
|
286
|
+
your own logger object in the `:error_logger` configuration key when
|
287
|
+
instantiating a LogStashLogger.
|
288
|
+
|
272
289
|
## Rails Integration
|
273
290
|
|
274
|
-
Verified to work with both Rails 3 and
|
291
|
+
Verified to work with both Rails 3, 4, and 5.
|
275
292
|
|
276
293
|
By default, every Rails log message will be written to logstash in `LogStash::Event` JSON format.
|
277
294
|
|
@@ -304,11 +321,20 @@ config.logstash.uri = ENV['LOGSTASH_URI']
|
|
304
321
|
# they will all share the same formatter.
|
305
322
|
config.logstash.formatter = :json_lines
|
306
323
|
|
324
|
+
# Optional, the logger to log writing errors to. Defaults to logging to $stderr
|
325
|
+
config.logstash.error_logger = Logger.new($stderr)
|
326
|
+
|
307
327
|
# Optional, max number of items to buffer before flushing. Defaults to 50
|
308
328
|
config.logstash.buffer_max_items = 50
|
309
329
|
|
310
330
|
# Optional, max number of seconds to wait between flushes. Defaults to 5
|
311
331
|
config.logstash.buffer_max_interval = 5
|
332
|
+
|
333
|
+
# Optional, drop message when a connection error occurs. Defaults to false
|
334
|
+
config.logstash.drop_messages_on_flush_error = false
|
335
|
+
|
336
|
+
# Optional, drop messages when the buffer is full. Defaults to true
|
337
|
+
config.logstash.drop_messages_on_full_buffer = true
|
312
338
|
```
|
313
339
|
|
314
340
|
#### UDP
|
@@ -510,9 +536,9 @@ end
|
|
510
536
|
|
511
537
|
Verified to work with:
|
512
538
|
|
513
|
-
* MRI Ruby 1.9.3, 2.0
|
514
|
-
* JRuby 1.7
|
515
|
-
* Rubinius
|
539
|
+
* MRI Ruby 1.9.3, 2.0, 2.1, 2.2, 2.3
|
540
|
+
* JRuby 1.7, 9.0
|
541
|
+
* Rubinius
|
516
542
|
|
517
543
|
Ruby 1.8.7 is not supported.
|
518
544
|
|
@@ -556,6 +582,16 @@ If you're using UDP output and writing to a logstash listener, you are most like
|
|
556
582
|
of the logstash listener. There is no known fix at this time. See [#43](https://github.com/dwbutler/logstash-logger/issues/43)
|
557
583
|
for more information.
|
558
584
|
|
585
|
+
### Errno::EMSGSIZE - Message too long
|
586
|
+
A known drawback of using UDP is its limit on total message size. To workaround
|
587
|
+
this issue, you will have to truncate the message by setting the max message size:
|
588
|
+
|
589
|
+
```ruby
|
590
|
+
LogStashLogger.configure do |config|
|
591
|
+
config.max_message_size = 2000
|
592
|
+
end
|
593
|
+
```
|
594
|
+
|
559
595
|
## Breaking changes
|
560
596
|
|
561
597
|
### Version 0.5+
|
@@ -594,6 +630,7 @@ logger = LogStashLogger.new('localhost', 5228, :tcp)
|
|
594
630
|
* [Nikita Vorobei](https://github.com/Nikita-V)
|
595
631
|
* [fireboy1919](https://github.com/fireboy1919)
|
596
632
|
* [Mike Gunderloy](https://github.com/ffmike)
|
633
|
+
* [Vitaly Gorodetsky](https://github.com/vitalis)
|
597
634
|
|
598
635
|
## Contributing
|
599
636
|
|
@@ -10,9 +10,12 @@ module LogStashLogger
|
|
10
10
|
|
11
11
|
class Configuration
|
12
12
|
attr_accessor :customize_event_block
|
13
|
+
attr_accessor :max_message_size
|
14
|
+
attr_accessor :default_error_logger
|
13
15
|
|
14
16
|
def initialize(*args)
|
15
17
|
@customize_event_block = nil
|
18
|
+
@default_error_logger = Logger.new($stderr)
|
16
19
|
|
17
20
|
yield self if block_given?
|
18
21
|
self
|
@@ -3,9 +3,11 @@ module LogStashLogger
|
|
3
3
|
class Base
|
4
4
|
attr_reader :io
|
5
5
|
attr_accessor :sync
|
6
|
+
attr_accessor :error_logger
|
6
7
|
|
7
8
|
def initialize(opts={})
|
8
9
|
@sync = opts[:sync]
|
10
|
+
@error_logger = opts.fetch(:error_logger, LogStashLogger.configuration.default_error_logger)
|
9
11
|
end
|
10
12
|
|
11
13
|
def to_io
|
@@ -13,20 +15,61 @@ module LogStashLogger
|
|
13
15
|
end
|
14
16
|
|
15
17
|
def write(message)
|
18
|
+
write_one(message)
|
19
|
+
end
|
20
|
+
|
21
|
+
def write_one(message)
|
16
22
|
@io.write(message)
|
23
|
+
rescue => e
|
24
|
+
if unrecoverable_error?(e)
|
25
|
+
log_error(e)
|
26
|
+
log_warning("unrecoverable error, aborting write")
|
27
|
+
else
|
28
|
+
raise
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def write_batch(messages, group = nil)
|
33
|
+
messages.each do |message|
|
34
|
+
write_one(message)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def write_batch(messages, group = nil)
|
39
|
+
messages.each do |message|
|
40
|
+
@io.write(message)
|
41
|
+
end
|
17
42
|
end
|
18
43
|
|
19
44
|
def flush
|
20
45
|
@io && @io.flush
|
21
46
|
end
|
22
47
|
|
23
|
-
def close
|
24
|
-
|
48
|
+
def close(opts = {})
|
49
|
+
close!
|
25
50
|
rescue => e
|
26
|
-
|
51
|
+
log_error(e)
|
52
|
+
end
|
53
|
+
|
54
|
+
def close!
|
55
|
+
@io && @io.close
|
27
56
|
ensure
|
28
57
|
@io = nil
|
29
58
|
end
|
59
|
+
|
60
|
+
def unrecoverable_error?(e)
|
61
|
+
e.is_a?(JSON::GeneratorError)
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def log_error(e)
|
67
|
+
error_logger.error "[#{self.class}] #{e.class} - #{e.message}"
|
68
|
+
end
|
69
|
+
|
70
|
+
def log_warning(message)
|
71
|
+
error_logger.warn("[#{self.class}] #{message}")
|
72
|
+
end
|
30
73
|
end
|
31
74
|
end
|
32
75
|
end
|
@@ -16,33 +16,69 @@ module LogStashLogger
|
|
16
16
|
warn "The :batch_timeout option is deprecated. Please use :buffer_max_interval instead"
|
17
17
|
end
|
18
18
|
|
19
|
+
@buffer_group = nil
|
19
20
|
@buffer_max_items = opts[:batch_events] || opts[:buffer_max_items]
|
20
21
|
@buffer_max_interval = opts[:batch_timeout] || opts[:buffer_max_interval]
|
22
|
+
@drop_messages_on_flush_error =
|
23
|
+
if opts.key?(:drop_messages_on_flush_error)
|
24
|
+
opts.delete(:drop_messages_on_flush_error)
|
25
|
+
else
|
26
|
+
false
|
27
|
+
end
|
21
28
|
|
22
|
-
|
29
|
+
@drop_messages_on_full_buffer =
|
30
|
+
if opts.key?(:drop_messages_on_full_buffer)
|
31
|
+
opts.delete(:drop_messages_on_full_buffer)
|
32
|
+
else
|
33
|
+
true
|
34
|
+
end
|
35
|
+
|
36
|
+
reset_buffer
|
23
37
|
end
|
24
38
|
|
25
39
|
def write(message)
|
26
|
-
buffer_receive message
|
40
|
+
buffer_receive message, @buffer_group
|
27
41
|
buffer_flush(force: true) if @sync
|
42
|
+
rescue
|
28
43
|
end
|
29
44
|
|
30
45
|
def flush(*args)
|
31
46
|
if args.empty?
|
32
47
|
buffer_flush
|
33
48
|
else
|
34
|
-
|
49
|
+
messages, group = *args
|
50
|
+
write_batch(messages, group)
|
35
51
|
end
|
52
|
+
rescue
|
53
|
+
if @drop_messages_on_flush_error
|
54
|
+
reset_buffer
|
55
|
+
else
|
56
|
+
cancel_flush
|
57
|
+
end
|
58
|
+
raise
|
36
59
|
end
|
37
60
|
|
38
|
-
def
|
39
|
-
|
61
|
+
def on_flush_error(e)
|
62
|
+
raise e
|
63
|
+
end
|
64
|
+
|
65
|
+
def on_full_buffer_receive(args)
|
66
|
+
if @drop_messages_on_full_buffer
|
67
|
+
reset_buffer
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def close(opts = {})
|
72
|
+
if opts.fetch(:flush, true)
|
73
|
+
buffer_flush(final: true)
|
74
|
+
end
|
75
|
+
|
40
76
|
super
|
41
77
|
end
|
42
78
|
|
43
79
|
def to_io
|
44
80
|
with_connection do
|
45
|
-
|
81
|
+
super
|
46
82
|
end
|
47
83
|
end
|
48
84
|
|
@@ -50,11 +86,15 @@ module LogStashLogger
|
|
50
86
|
!!@io
|
51
87
|
end
|
52
88
|
|
53
|
-
def
|
89
|
+
def write_one(message)
|
54
90
|
with_connection do
|
55
|
-
|
56
|
-
|
57
|
-
|
91
|
+
super
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def write_batch(messages, group = nil)
|
96
|
+
with_connection do
|
97
|
+
super
|
58
98
|
end
|
59
99
|
end
|
60
100
|
|
@@ -64,7 +104,7 @@ module LogStashLogger
|
|
64
104
|
end
|
65
105
|
|
66
106
|
def reconnect
|
67
|
-
|
107
|
+
close(flush: false)
|
68
108
|
connect
|
69
109
|
end
|
70
110
|
|
@@ -73,10 +113,41 @@ module LogStashLogger
|
|
73
113
|
connect unless connected?
|
74
114
|
yield
|
75
115
|
rescue => e
|
76
|
-
|
77
|
-
|
116
|
+
log_error(e)
|
117
|
+
close(flush: false)
|
78
118
|
raise
|
79
119
|
end
|
120
|
+
|
121
|
+
private
|
122
|
+
|
123
|
+
def reset_buffer
|
124
|
+
buffer_initialize max_items: @buffer_max_items, max_interval: @buffer_max_interval
|
125
|
+
@buffer_state[:timer] = Thread.new do
|
126
|
+
loop do
|
127
|
+
sleep(@buffer_config[:max_interval])
|
128
|
+
begin
|
129
|
+
buffer_flush(:force => true)
|
130
|
+
rescue
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def buffer_clear_outgoing
|
137
|
+
@buffer_state[:outgoing_items] = Hash.new { |h, k| h[k] = [] }
|
138
|
+
@buffer_state[:outgoing_count] = 0
|
139
|
+
end
|
140
|
+
|
141
|
+
def cancel_flush
|
142
|
+
@buffer_state[:flush_mutex].lock rescue false
|
143
|
+
@buffer_state[:outgoing_items].each do |group, items|
|
144
|
+
@buffer_state[:pending_items][group].concat items
|
145
|
+
end
|
146
|
+
@buffer_state[:pending_count] += @buffer_state[:outgoing_count]
|
147
|
+
buffer_clear_outgoing
|
148
|
+
ensure
|
149
|
+
@buffer_state[:flush_mutex].unlock rescue false
|
150
|
+
end
|
80
151
|
end
|
81
152
|
end
|
82
153
|
end
|
@@ -20,59 +20,38 @@ module LogStashLogger
|
|
20
20
|
@topic = opts[:path] || DEFAULT_TOPIC
|
21
21
|
@producer = opts[:producer] || DEFAULT_PRODUCER
|
22
22
|
@backoff = opts[:backoff] || DEFAULT_BACKOFF
|
23
|
+
@buffer_group = @topic
|
23
24
|
end
|
24
25
|
|
25
26
|
def connect
|
26
27
|
@io = ::Poseidon::Producer.new(@hosts, @producer)
|
27
28
|
end
|
28
29
|
|
29
|
-
def reconnect
|
30
|
-
@io.close
|
31
|
-
connect
|
32
|
-
end
|
33
|
-
|
34
30
|
def with_connection
|
35
|
-
connect unless
|
31
|
+
connect unless connected?
|
36
32
|
yield
|
37
33
|
rescue ::Poseidon::Errors::ChecksumError, Poseidon::Errors::UnableToFetchMetadata => e
|
38
|
-
|
34
|
+
log_error(e)
|
35
|
+
log_warning("reconnect/retry")
|
39
36
|
sleep backoff if backoff
|
40
37
|
reconnect
|
41
38
|
retry
|
42
39
|
rescue => e
|
43
|
-
|
44
|
-
|
40
|
+
log_error(e)
|
41
|
+
log_warning("giving up")
|
42
|
+
close(flush: false)
|
45
43
|
end
|
46
44
|
|
47
|
-
def
|
48
|
-
|
49
|
-
buffer_flush(force: true) if @sync
|
50
|
-
end
|
51
|
-
|
52
|
-
def write_batch(messages)
|
45
|
+
def write_batch(messages, topic = nil)
|
46
|
+
topic ||= @topic
|
53
47
|
with_connection do
|
54
|
-
@io.send_messages messages
|
48
|
+
@io.send_messages messages.map { |message| Poseidon::MessageToSend.new(topic, message) }
|
55
49
|
end
|
56
50
|
end
|
57
51
|
|
58
|
-
def
|
59
|
-
|
60
|
-
@io && @io.close
|
61
|
-
rescue => e
|
62
|
-
warn "#{self.class} - #{e.class} - #{e.message}"
|
63
|
-
ensure
|
64
|
-
@io = nil
|
52
|
+
def write_one(message, topic = nil)
|
53
|
+
write_batch([message], topic)
|
65
54
|
end
|
66
|
-
|
67
|
-
def flush(*args)
|
68
|
-
if args.empty?
|
69
|
-
buffer_flush
|
70
|
-
else
|
71
|
-
messages = *args.first
|
72
|
-
write_batch(messages)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
55
|
end
|
77
56
|
end
|
78
57
|
end
|
@@ -10,6 +10,7 @@ module LogStashLogger
|
|
10
10
|
def initialize(opts)
|
11
11
|
super
|
12
12
|
@list = opts.delete(:list) || DEFAULT_LIST
|
13
|
+
@buffer_group = @list
|
13
14
|
|
14
15
|
normalize_path(opts)
|
15
16
|
|
@@ -22,49 +23,37 @@ module LogStashLogger
|
|
22
23
|
|
23
24
|
def reconnect
|
24
25
|
@io.client.reconnect
|
26
|
+
rescue => e
|
27
|
+
log_error(e)
|
25
28
|
end
|
26
29
|
|
27
30
|
def with_connection
|
28
|
-
connect unless
|
31
|
+
connect unless connected?
|
29
32
|
yield
|
30
33
|
rescue ::Redis::InheritedError
|
31
34
|
reconnect
|
32
35
|
retry
|
33
36
|
rescue => e
|
34
|
-
|
35
|
-
|
37
|
+
log_error(e)
|
38
|
+
close(flush: false)
|
36
39
|
raise
|
37
40
|
end
|
38
41
|
|
39
|
-
def write(message)
|
40
|
-
buffer_receive message, @list
|
41
|
-
buffer_flush(force: true) if @sync
|
42
|
-
end
|
43
|
-
|
44
42
|
def write_batch(messages, list = nil)
|
43
|
+
list ||= @list
|
45
44
|
with_connection do
|
46
45
|
@io.rpush(list, messages)
|
47
46
|
end
|
48
47
|
end
|
49
48
|
|
50
|
-
def
|
51
|
-
|
52
|
-
@io && @io.quit
|
53
|
-
rescue => e
|
54
|
-
warn "#{self.class} - #{e.class} - #{e.message}"
|
55
|
-
ensure
|
56
|
-
@io = nil
|
49
|
+
def write_one(message, list = nil)
|
50
|
+
write_batch(message, list)
|
57
51
|
end
|
58
52
|
|
59
|
-
def
|
60
|
-
|
61
|
-
buffer_flush
|
62
|
-
else
|
63
|
-
messages, list = *args
|
64
|
-
write_batch(messages, list)
|
65
|
-
end
|
53
|
+
def close!
|
54
|
+
@io && @io.quit
|
66
55
|
end
|
67
|
-
|
56
|
+
|
68
57
|
private
|
69
58
|
|
70
59
|
def normalize_path(opts)
|
@@ -44,6 +44,10 @@ module LogStashLogger
|
|
44
44
|
if event.timestamp.is_a?(Time)
|
45
45
|
event.timestamp = event.timestamp.iso8601(3)
|
46
46
|
end
|
47
|
+
|
48
|
+
if LogStashLogger.configuration.max_message_size
|
49
|
+
event['message'.freeze] = event['message'.freeze].byteslice(0, LogStashLogger.configuration.max_message_size)
|
50
|
+
end
|
47
51
|
|
48
52
|
event
|
49
53
|
end
|
data/logstash-logger.gemspec
CHANGED
@@ -22,13 +22,20 @@ Gem::Specification.new do |gem|
|
|
22
22
|
gem.add_runtime_dependency 'stud'
|
23
23
|
|
24
24
|
gem.add_development_dependency 'rails'
|
25
|
+
if RUBY_VERSION < '2'
|
26
|
+
gem.add_development_dependency 'mime-types', '< 3'
|
27
|
+
end
|
25
28
|
gem.add_development_dependency 'redis'
|
26
29
|
gem.add_development_dependency 'poseidon'
|
27
30
|
|
28
|
-
if RUBY_VERSION < '2'
|
31
|
+
if RUBY_VERSION < '2' || defined?(JRUBY_VERSION)
|
29
32
|
gem.add_development_dependency 'SyslogLogger'
|
30
33
|
end
|
31
34
|
|
35
|
+
if RUBY_VERSION < '2'
|
36
|
+
gem.add_development_dependency 'json', '~> 1.8'
|
37
|
+
end
|
38
|
+
|
32
39
|
gem.add_development_dependency 'rspec', '>= 3'
|
33
40
|
gem.add_development_dependency 'rake'
|
34
41
|
gem.add_development_dependency 'pry'
|
data/samples/redis.conf
CHANGED
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'logstash-logger'
|
2
|
+
|
3
|
+
describe LogStashLogger::Device::Connectable do
|
4
|
+
include_context 'device'
|
5
|
+
|
6
|
+
let(:io) { double("IO") }
|
7
|
+
|
8
|
+
subject { udp_device }
|
9
|
+
|
10
|
+
describe "#reconnect" do
|
11
|
+
context "with active IO connection" do
|
12
|
+
before do
|
13
|
+
subject.instance_variable_set(:@io, io)
|
14
|
+
end
|
15
|
+
|
16
|
+
it "closes the connection" do
|
17
|
+
expect(io).to receive(:close).once
|
18
|
+
subject.reconnect
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context "with no active IO connection" do
|
23
|
+
before do
|
24
|
+
subject.instance_variable_set(:@io, nil)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "does nothing" do
|
28
|
+
expect(io).to_not receive(:close)
|
29
|
+
subject.reconnect
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
describe "#with_connection" do
|
35
|
+
context "on exception" do
|
36
|
+
before do
|
37
|
+
allow(subject).to receive(:connected?) { raise(StandardError) }
|
38
|
+
allow(subject).to receive(:warn)
|
39
|
+
end
|
40
|
+
|
41
|
+
context "with active IO connection" do
|
42
|
+
before do
|
43
|
+
subject.instance_variable_set(:@io, io)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "closes the connection" do
|
47
|
+
expect(io).to receive(:close).once
|
48
|
+
|
49
|
+
expect {
|
50
|
+
subject.with_connection do |connection|
|
51
|
+
connection
|
52
|
+
end
|
53
|
+
}.to raise_error(StandardError)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
context "with no active IO connection" do
|
58
|
+
before do
|
59
|
+
subject.instance_variable_set(:@io, nil)
|
60
|
+
end
|
61
|
+
|
62
|
+
it "does nothing" do
|
63
|
+
expect(io).to_not receive(:close)
|
64
|
+
|
65
|
+
expect {
|
66
|
+
subject.with_connection do |connection|
|
67
|
+
connection
|
68
|
+
end
|
69
|
+
}.to raise_error(StandardError)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-logger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Butler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-07-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-event
|
@@ -170,6 +170,7 @@ files:
|
|
170
170
|
- gemfiles/rails_4.0.gemfile
|
171
171
|
- gemfiles/rails_4.1.gemfile
|
172
172
|
- gemfiles/rails_4.2.gemfile
|
173
|
+
- gemfiles/rails_5.0.gemfile
|
173
174
|
- lib/logstash-logger.rb
|
174
175
|
- lib/logstash-logger/configuration.rb
|
175
176
|
- lib/logstash-logger/device.rb
|
@@ -211,6 +212,7 @@ files:
|
|
211
212
|
- samples/unix.conf
|
212
213
|
- spec/configuration_spec.rb
|
213
214
|
- spec/device/balancer_spec.rb
|
215
|
+
- spec/device/connectable_spec.rb
|
214
216
|
- spec/device/file_spec.rb
|
215
217
|
- spec/device/io_spec.rb
|
216
218
|
- spec/device/kafka_spec.rb
|
@@ -256,13 +258,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
256
258
|
version: '0'
|
257
259
|
requirements: []
|
258
260
|
rubyforge_project:
|
259
|
-
rubygems_version: 2.
|
261
|
+
rubygems_version: 2.5.1
|
260
262
|
signing_key:
|
261
263
|
specification_version: 4
|
262
264
|
summary: LogStash Logger for ruby
|
263
265
|
test_files:
|
264
266
|
- spec/configuration_spec.rb
|
265
267
|
- spec/device/balancer_spec.rb
|
268
|
+
- spec/device/connectable_spec.rb
|
266
269
|
- spec/device/file_spec.rb
|
267
270
|
- spec/device/io_spec.rb
|
268
271
|
- spec/device/kafka_spec.rb
|