logstash-codec-multiline 2.0.7 → 2.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/Gemfile +1 -2
- data/lib/logstash/codecs/auto_flush.rb +7 -61
- data/lib/logstash/codecs/identity_map_codec.rb +85 -29
- data/lib/logstash/codecs/multiline.rb +11 -7
- data/lib/logstash/codecs/retriggerable_task.rb +81 -0
- data/logstash-codec-multiline.gemspec +1 -1
- data/spec/codecs/auto_flush_spec.rb +48 -55
- data/spec/codecs/identity_map_codec_spec.rb +27 -7
- data/spec/codecs/multiline_spec.rb +1 -1
- data/spec/supports/helpers.rb +14 -1
- metadata +31 -30
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f9e0a15839b1bbabcad88ee95445b72ed589a4dd
|
4
|
+
data.tar.gz: bdf59bcab8b5ad4fc35906de033bde7ed658e52a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e0bcea055b8af6ebbd45300e2901146a2770b7937de6ca8b5f08f4a01b347cc7b876d3b675589c475b3b41f4ce65632e7edc44432a2f00b2abc5f9b7dc5d899
|
7
|
+
data.tar.gz: 6333ad636771cc1e9407f9642f242ff40f59181b4073fe55e2b3aa93b9cd675ca435961aabc7f054c4cad6be072bdefc42395198b14cafb3ba7b286474e888ab
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -1,74 +1,32 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "concurrent"
|
3
|
+
require "logstash/codecs/retriggerable_task"
|
3
4
|
|
4
5
|
module LogStash module Codecs class AutoFlush
|
5
6
|
def initialize(mc, interval)
|
6
7
|
@mc, @interval = mc, interval
|
7
8
|
@stopped = Concurrent::AtomicBoolean.new # false by default
|
9
|
+
@task = RetriggerableTask.new(@interval, self)
|
8
10
|
end
|
9
11
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
# if pending?
|
14
|
-
# @task.cancel
|
15
|
-
# create_task
|
16
|
-
# elsif finished?
|
17
|
-
# create_task
|
18
|
-
# # else the task is executing
|
19
|
-
# end
|
20
|
-
# self
|
21
|
-
# end
|
12
|
+
def timeout
|
13
|
+
@mc.auto_flush
|
14
|
+
end
|
22
15
|
|
23
16
|
def start
|
24
17
|
# can't start if pipeline is stopping
|
25
18
|
return self if stopped?
|
26
|
-
|
27
|
-
if pending? && @task.cancel
|
28
|
-
create_task
|
29
|
-
return self
|
30
|
-
end
|
31
|
-
# maybe we have a timing edge case
|
32
|
-
# where pending? was true but cancel failed
|
33
|
-
# because the task started running
|
34
|
-
if finished?
|
35
|
-
create_task
|
36
|
-
return self
|
37
|
-
end
|
38
|
-
# else the task is executing
|
39
|
-
# wait for task to complete
|
40
|
-
# flush could feasibly block on queue access
|
41
|
-
@task.value
|
42
|
-
create_task
|
19
|
+
@task.retrigger
|
43
20
|
self
|
44
21
|
end
|
45
22
|
|
46
|
-
def create_task
|
47
|
-
@task = Concurrent::ScheduledTask.execute(@interval) do
|
48
|
-
@mc.auto_flush()
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
def finished?
|
53
|
-
return true if @task.nil?
|
54
|
-
@task.fulfilled?
|
55
|
-
end
|
56
|
-
|
57
|
-
def pending?
|
58
|
-
@task && @task.pending?
|
59
|
-
end
|
60
|
-
|
61
23
|
def stopped?
|
62
24
|
@stopped.value
|
63
25
|
end
|
64
26
|
|
65
27
|
def stop
|
66
28
|
@stopped.make_true
|
67
|
-
|
68
|
-
end
|
69
|
-
|
70
|
-
def cancel
|
71
|
-
@task.cancel if pending?
|
29
|
+
@task.close
|
72
30
|
end
|
73
31
|
end
|
74
32
|
|
@@ -76,10 +34,6 @@ class AutoFlushUnset
|
|
76
34
|
def initialize(mc, interval)
|
77
35
|
end
|
78
36
|
|
79
|
-
def pending?
|
80
|
-
false
|
81
|
-
end
|
82
|
-
|
83
37
|
def stopped?
|
84
38
|
true
|
85
39
|
end
|
@@ -88,15 +42,7 @@ class AutoFlushUnset
|
|
88
42
|
self
|
89
43
|
end
|
90
44
|
|
91
|
-
def finished?
|
92
|
-
true
|
93
|
-
end
|
94
|
-
|
95
45
|
def stop
|
96
46
|
self
|
97
47
|
end
|
98
|
-
|
99
|
-
def cancel
|
100
|
-
self
|
101
|
-
end
|
102
48
|
end end end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/namespace"
|
3
3
|
require "thread_safe"
|
4
|
+
require "concurrent"
|
4
5
|
|
5
6
|
# This class is a Codec duck type
|
6
7
|
# Using Composition, it maps from a stream identity to
|
@@ -37,41 +38,51 @@ module LogStash module Codecs class IdentityMapCodec
|
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
40
|
-
class
|
41
|
-
def initialize(
|
42
|
-
@
|
43
|
-
@
|
41
|
+
class PeriodicRunner
|
42
|
+
def initialize(listener, interval, method_symbol)
|
43
|
+
@listener, @interval = listener, interval
|
44
|
+
@method_symbol = method_symbol
|
45
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
44
46
|
end
|
45
47
|
|
46
48
|
def start
|
47
49
|
return self if running?
|
48
|
-
@running
|
49
|
-
@thread = Thread.new(
|
50
|
-
|
50
|
+
@running.make_true
|
51
|
+
@thread = Thread.new() do
|
52
|
+
while running? do
|
51
53
|
sleep @interval
|
52
|
-
break if
|
53
|
-
|
54
|
+
break if !running?
|
55
|
+
@listener.send(@method_symbol)
|
54
56
|
end
|
55
57
|
end
|
56
58
|
self
|
57
59
|
end
|
58
60
|
|
59
61
|
def running?
|
60
|
-
@running
|
62
|
+
@running.value
|
61
63
|
end
|
62
64
|
|
63
65
|
def stop
|
64
66
|
return if !running?
|
65
|
-
@running
|
66
|
-
|
67
|
+
@running.make_false
|
68
|
+
if @thread.alive?
|
69
|
+
@thread.wakeup
|
70
|
+
@thread.join
|
71
|
+
end
|
72
|
+
@listener = nil
|
67
73
|
end
|
68
74
|
end
|
69
75
|
|
70
|
-
|
76
|
+
class NoopRunner
|
77
|
+
attr_reader :start, :stop
|
78
|
+
def running?() false; end
|
79
|
+
end
|
80
|
+
|
81
|
+
# A composite class to hold both the codec, the eviction_timeout and a last_used timestamp
|
71
82
|
# instances of this Value Object are stored in the mapping hash
|
72
83
|
class CodecValue
|
73
84
|
attr_reader :codec
|
74
|
-
attr_accessor :
|
85
|
+
attr_accessor :eviction_timeout, :auto_flush_timeout
|
75
86
|
|
76
87
|
def initialize(codec)
|
77
88
|
@codec = codec
|
@@ -92,7 +103,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
92
103
|
CLEANER_INTERVAL = 60 * 5 # 5 minutes
|
93
104
|
|
94
105
|
attr_reader :identity_map
|
95
|
-
attr_accessor :base_codec, :cleaner
|
106
|
+
attr_accessor :base_codec, :cleaner, :auto_flusher
|
96
107
|
|
97
108
|
def initialize(codec)
|
98
109
|
@base_codec = codec
|
@@ -100,7 +111,14 @@ module LogStash module Codecs class IdentityMapCodec
|
|
100
111
|
@identity_map = ThreadSafe::Hash.new &method(:codec_builder)
|
101
112
|
@max_identities = MAX_IDENTITIES
|
102
113
|
@evict_timeout = EVICT_TIMEOUT
|
103
|
-
|
114
|
+
cleaner_interval(CLEANER_INTERVAL)
|
115
|
+
if codec.respond_to?(:use_mapper_auto_flush) &&
|
116
|
+
(@auto_flush_interval = codec.use_mapper_auto_flush)
|
117
|
+
@auto_flusher = PeriodicRunner.new(self, 0.5, :auto_flush_mapped)
|
118
|
+
else
|
119
|
+
@auto_flusher = NoopRunner.new
|
120
|
+
end
|
121
|
+
|
104
122
|
@decode_block = lambda {|*| true }
|
105
123
|
@eviction_block = nil
|
106
124
|
end
|
@@ -123,8 +141,8 @@ module LogStash module Codecs class IdentityMapCodec
|
|
123
141
|
|
124
142
|
# used to add a non-default cleaner interval
|
125
143
|
def cleaner_interval(interval)
|
126
|
-
@cleaner.stop
|
127
|
-
@cleaner =
|
144
|
+
@cleaner.stop if @cleaner
|
145
|
+
@cleaner = PeriodicRunner.new(self, interval.to_i, :map_cleanup)
|
128
146
|
self
|
129
147
|
end
|
130
148
|
|
@@ -133,6 +151,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
133
151
|
@eviction_block = block
|
134
152
|
self
|
135
153
|
end
|
154
|
+
|
136
155
|
# end Constructional/builder methods
|
137
156
|
# ==============================================
|
138
157
|
|
@@ -182,11 +201,34 @@ module LogStash module Codecs class IdentityMapCodec
|
|
182
201
|
|
183
202
|
def close()
|
184
203
|
cleaner.stop
|
204
|
+
auto_flusher.stop
|
185
205
|
all_codecs.each(&:close)
|
186
206
|
end
|
187
207
|
# end Codec API
|
188
208
|
# ==============================================
|
189
209
|
|
210
|
+
def auto_flush_mapped
|
211
|
+
if !identity_count.zero?
|
212
|
+
nowf = Time.now.to_f
|
213
|
+
identity_map.each do |identity, compo|
|
214
|
+
next if compo.auto_flush_timeout.zero?
|
215
|
+
next unless nowf > compo.auto_flush_timeout
|
216
|
+
compo.codec.auto_flush
|
217
|
+
# at eof (tail and read) no more lines for a while or ever
|
218
|
+
# so reset compo.auto_flush_timeout
|
219
|
+
compo.auto_flush_timeout = 0
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def flush_mapped(listener)
|
225
|
+
listener_has_path = listener.respond_to?(:path)
|
226
|
+
identity_map.each do |identity, compo|
|
227
|
+
listener.path = identity if listener_has_path
|
228
|
+
compo.codec.auto_flush(listener)
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
190
232
|
def all_codecs
|
191
233
|
no_streams? ? @base_codecs : identity_map.values.map(&:codec)
|
192
234
|
end
|
@@ -204,14 +246,16 @@ module LogStash module Codecs class IdentityMapCodec
|
|
204
246
|
# been accessed in the last @evict_timeout
|
205
247
|
# period (default 1 hour)
|
206
248
|
def map_cleanup
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
249
|
+
if !identity_count.zero?
|
250
|
+
nowi = Time.now.to_i
|
251
|
+
# delete_if is atomic
|
252
|
+
# contents should not mutate during this call
|
253
|
+
identity_map.delete_if do |identity, compo|
|
254
|
+
if (flag = compo.eviction_timeout <= nowi)
|
255
|
+
evict_flush(compo.codec)
|
256
|
+
end
|
257
|
+
flag
|
213
258
|
end
|
214
|
-
flag
|
215
259
|
end
|
216
260
|
current_size_and_limit
|
217
261
|
end
|
@@ -241,7 +285,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
241
285
|
end
|
242
286
|
|
243
287
|
def eviction_timestamp_for(identity)
|
244
|
-
find_codec_value(identity).
|
288
|
+
find_codec_value(identity).eviction_timeout
|
245
289
|
end
|
246
290
|
|
247
291
|
private
|
@@ -261,13 +305,20 @@ module LogStash module Codecs class IdentityMapCodec
|
|
261
305
|
# only start the cleaner if streams are in use
|
262
306
|
# continuous calls to start are OK
|
263
307
|
cleaner.start
|
308
|
+
auto_flusher.start
|
264
309
|
compo = find_codec_value(identity)
|
265
|
-
|
310
|
+
now = Time.now
|
311
|
+
compo.eviction_timeout = eviction_timestamp(now)
|
312
|
+
compo.auto_flush_timeout = auto_flush_timestamp(now)
|
266
313
|
compo.codec
|
267
314
|
end
|
268
315
|
|
269
|
-
def
|
270
|
-
|
316
|
+
def auto_flush_timestamp(now = Time.now)
|
317
|
+
now.to_f + @auto_flush_interval.to_f
|
318
|
+
end
|
319
|
+
|
320
|
+
def eviction_timestamp(now = Time.now)
|
321
|
+
now.to_i + @evict_timeout
|
271
322
|
end
|
272
323
|
|
273
324
|
def check_map_limits
|
@@ -277,6 +328,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
277
328
|
|
278
329
|
def codec_builder(hash, k)
|
279
330
|
codec = hash.empty? ? @base_codec : @base_codec.clone
|
331
|
+
codec.use_mapper_auto_flush if using_mapped_auto_flush?
|
280
332
|
compo = CodecValue.new(codec)
|
281
333
|
hash.store(k, compo)
|
282
334
|
end
|
@@ -284,4 +336,8 @@ module LogStash module Codecs class IdentityMapCodec
|
|
284
336
|
def no_streams?
|
285
337
|
identity_map.empty?
|
286
338
|
end
|
339
|
+
|
340
|
+
def using_mapped_auto_flush?
|
341
|
+
!@auto_flush_interval.nil?
|
342
|
+
end
|
287
343
|
end end end
|
@@ -171,6 +171,12 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
|
|
171
171
|
end
|
172
172
|
end # def register
|
173
173
|
|
174
|
+
def use_mapper_auto_flush
|
175
|
+
return unless auto_flush_active?
|
176
|
+
@auto_flush_runner = AutoFlushUnset.new(nil, nil)
|
177
|
+
@auto_flush_interval = @auto_flush_interval.to_f
|
178
|
+
end
|
179
|
+
|
174
180
|
def accept(listener)
|
175
181
|
# memoize references to listener that holds upstream state
|
176
182
|
@previous_listener = @last_seen_listener || listener
|
@@ -215,9 +221,11 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
|
|
215
221
|
end
|
216
222
|
end
|
217
223
|
|
218
|
-
def auto_flush
|
224
|
+
def auto_flush(listener = @last_seen_listener)
|
225
|
+
return if listener.nil?
|
226
|
+
|
219
227
|
flush do |event|
|
220
|
-
|
228
|
+
listener.process_event(event)
|
221
229
|
end
|
222
230
|
end
|
223
231
|
|
@@ -272,11 +280,7 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
|
|
272
280
|
end # def encode
|
273
281
|
|
274
282
|
def close
|
275
|
-
|
276
|
-
#will cancel task if necessary
|
277
|
-
auto_flush_runner.stop
|
278
|
-
end
|
279
|
-
auto_flush
|
283
|
+
auto_flush_runner.stop
|
280
284
|
end
|
281
285
|
|
282
286
|
def auto_flush_active?
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module LogStash module Codecs class RetriggerableTask
|
4
|
+
SLEEP_FOR = 0.25.freeze
|
5
|
+
|
6
|
+
attr_reader :thread
|
7
|
+
|
8
|
+
def initialize(delay, listener)
|
9
|
+
@count = calculate_count(delay)
|
10
|
+
@listener = listener
|
11
|
+
@counter = Concurrent::AtomicFixnum.new(0 + @count)
|
12
|
+
@stopped = Concurrent::AtomicBoolean.new(false)
|
13
|
+
@semaphore = Concurrent::Semaphore.new(1)
|
14
|
+
end
|
15
|
+
|
16
|
+
def retrigger
|
17
|
+
return if stopped?
|
18
|
+
if executing?
|
19
|
+
@semaphore.acquire
|
20
|
+
end
|
21
|
+
|
22
|
+
if pending?
|
23
|
+
reset_counter
|
24
|
+
else
|
25
|
+
start
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def close
|
30
|
+
@stopped.make_true
|
31
|
+
end
|
32
|
+
|
33
|
+
def counter
|
34
|
+
@counter.value
|
35
|
+
end
|
36
|
+
|
37
|
+
def executing?
|
38
|
+
running? && counter < 1
|
39
|
+
end
|
40
|
+
|
41
|
+
def pending?
|
42
|
+
running? && counter > 0
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def calculate_count(value)
|
48
|
+
# in multiples of SLEEP_FOR (0.25) seconds
|
49
|
+
# if delay is 10 seconds then count is 40
|
50
|
+
# this only works when SLEEP_FOR is less than 1
|
51
|
+
return 1 if value < SLEEP_FOR
|
52
|
+
(value / SLEEP_FOR).floor
|
53
|
+
end
|
54
|
+
|
55
|
+
def reset_counter
|
56
|
+
@counter.value = 0 + @count
|
57
|
+
end
|
58
|
+
|
59
|
+
def running?
|
60
|
+
@thread && @thread.alive?
|
61
|
+
end
|
62
|
+
|
63
|
+
def start()
|
64
|
+
reset_counter
|
65
|
+
@thread = Thread.new do
|
66
|
+
while counter > 0
|
67
|
+
break if stopped?
|
68
|
+
sleep SLEEP_FOR
|
69
|
+
@counter.decrement
|
70
|
+
end
|
71
|
+
|
72
|
+
@semaphore.drain_permits
|
73
|
+
@listener.timeout if !stopped?
|
74
|
+
@semaphore.release
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def stopped?
|
79
|
+
@stopped.value
|
80
|
+
end
|
81
|
+
end end end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-multiline'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.8'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -4,109 +4,102 @@ require "logstash/codecs/multiline"
|
|
4
4
|
require_relative "../supports/helpers.rb"
|
5
5
|
|
6
6
|
describe "AutoFlush and AutoFlushUnset" do
|
7
|
+
before(:all) do
|
8
|
+
@thread_abort = Thread.abort_on_exception
|
9
|
+
Thread.abort_on_exception = true
|
10
|
+
end
|
11
|
+
|
12
|
+
after(:all) do
|
13
|
+
Thread.abort_on_exception = @thread_abort
|
14
|
+
end
|
15
|
+
|
7
16
|
let(:flushable) { Mlc::AutoFlushTracer.new }
|
8
|
-
let(:flush_wait) { 0.
|
17
|
+
let(:flush_wait) { 0.25 }
|
9
18
|
|
10
19
|
describe LogStash::Codecs::AutoFlush do
|
11
20
|
subject { described_class.new(flushable, flush_wait) }
|
12
21
|
|
13
22
|
context "when initialized" do
|
14
|
-
it "#pending? is false" do
|
15
|
-
expect(subject.pending?).to be_falsy
|
16
|
-
end
|
17
|
-
|
18
23
|
it "#stopped? is false" do
|
19
|
-
expect(subject.stopped?).to
|
20
|
-
end
|
21
|
-
|
22
|
-
it "#finished? is true" do
|
23
|
-
expect(subject.finished?).to be_truthy
|
24
|
+
expect(subject.stopped?).to be_falsey
|
24
25
|
end
|
25
26
|
end
|
26
27
|
|
27
|
-
context "when
|
28
|
-
let(:flush_wait) { 20 }
|
29
|
-
|
28
|
+
context "when starting once" do
|
30
29
|
before { subject.start }
|
31
30
|
after { subject.stop }
|
32
31
|
|
33
|
-
it "
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
it "#stopped? is false" do
|
38
|
-
expect(subject.stopped?).to be_falsy
|
39
|
-
end
|
40
|
-
|
41
|
-
it "#finished? is false" do
|
42
|
-
expect(subject.finished?).to be_falsy
|
32
|
+
it "calls auto_flush on flushable" do
|
33
|
+
sleep flush_wait + 0.5
|
34
|
+
expect(flushable.trace_for(:auto_flush)).to be_truthy
|
43
35
|
end
|
44
36
|
end
|
45
37
|
|
46
|
-
context "when
|
47
|
-
before
|
48
|
-
subject.start
|
49
|
-
sleep flush_wait + 0.1
|
50
|
-
end
|
51
|
-
|
38
|
+
context "when starting multiple times" do
|
39
|
+
before { subject.start }
|
52
40
|
after { subject.stop }
|
53
41
|
|
54
42
|
it "calls auto_flush on flushable" do
|
55
|
-
expect(flushable.trace_for(:auto_flush)).to
|
56
|
-
|
43
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
44
|
+
sleep 0.1
|
45
|
+
subject.start
|
46
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
47
|
+
sleep 0.1
|
48
|
+
subject.start
|
49
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
57
50
|
|
58
|
-
|
59
|
-
expect(
|
51
|
+
sleep flush_wait + 0.5
|
52
|
+
expect(flushable.trace_for(:auto_flush)).to be_truthy
|
60
53
|
end
|
54
|
+
end
|
61
55
|
|
62
|
-
|
63
|
-
|
56
|
+
context "when retriggering during execution" do
|
57
|
+
let(:flush_wait) { 1 }
|
58
|
+
let(:delay) { 1 }
|
59
|
+
before do
|
60
|
+
flushable.set_delay(delay)
|
61
|
+
subject.start
|
64
62
|
end
|
63
|
+
after { subject.stop }
|
65
64
|
|
66
|
-
it "
|
67
|
-
|
65
|
+
it "calls auto_flush twice on flushable" do
|
66
|
+
now = Time.now.to_f
|
67
|
+
combined_time = flush_wait + delay
|
68
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
69
|
+
sleep flush_wait + 0.25 # <---- wait for execution to start
|
70
|
+
subject.start # <---- because of the semaphore, this waits for delay = 2 seconds, then starts a new thread
|
71
|
+
sleep combined_time + 0.5 # <---- wait for more than 4 seconds = flush_wait + delay
|
72
|
+
elapsed1, elapsed2 = flushable.full_trace_for(:delay).map{ |el| (el - now).floor }
|
73
|
+
expect(elapsed1).to eq(combined_time)
|
74
|
+
expect(elapsed2).to eq(2 * combined_time)
|
75
|
+
expect(flushable.full_trace_for(:auto_flush)).to eq([true, true])
|
68
76
|
end
|
69
77
|
end
|
70
78
|
|
71
|
-
context "when
|
79
|
+
context "when stopping before timeout" do
|
72
80
|
before do
|
73
81
|
subject.start
|
82
|
+
sleep 0.1
|
74
83
|
subject.stop
|
75
84
|
end
|
76
85
|
|
77
86
|
it "does not call auto_flush on flushable" do
|
78
|
-
expect(flushable.trace_for(:auto_flush)).to
|
79
|
-
end
|
80
|
-
|
81
|
-
it "#pending? is false" do
|
82
|
-
expect(subject.pending?).to be_falsy
|
87
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
83
88
|
end
|
84
89
|
|
85
90
|
it "#stopped? is true" do
|
86
91
|
expect(subject.stopped?).to be_truthy
|
87
92
|
end
|
88
|
-
|
89
|
-
it "#finished? is false" do
|
90
|
-
expect(subject.finished?).to be_falsy
|
91
|
-
end
|
92
93
|
end
|
93
94
|
end
|
94
95
|
|
95
96
|
describe LogStash::Codecs::AutoFlushUnset do
|
96
97
|
subject { described_class.new(flushable, 2) }
|
97
98
|
|
98
|
-
it "#pending? is false" do
|
99
|
-
expect(subject.pending?).to be_falsy
|
100
|
-
end
|
101
|
-
|
102
99
|
it "#stopped? is true" do
|
103
100
|
expect(subject.stopped?).to be_truthy
|
104
101
|
end
|
105
102
|
|
106
|
-
it "#finished? is true" do
|
107
|
-
expect(subject.finished?).to be_truthy
|
108
|
-
end
|
109
|
-
|
110
103
|
it "#start returns self" do
|
111
104
|
expect(subject.start).to eq(subject)
|
112
105
|
end
|
@@ -12,6 +12,15 @@ describe LogStash::Codecs::IdentityMapCodec do
|
|
12
12
|
let(:codec1) { demuxer.codec_without_usage_update(stream1) }
|
13
13
|
let(:arg1) { "data-a" }
|
14
14
|
|
15
|
+
before(:all) do
|
16
|
+
@thread_abort = Thread.abort_on_exception
|
17
|
+
Thread.abort_on_exception = true
|
18
|
+
end
|
19
|
+
|
20
|
+
after(:all) do
|
21
|
+
Thread.abort_on_exception = @thread_abort
|
22
|
+
end
|
23
|
+
|
15
24
|
after do
|
16
25
|
codec.close
|
17
26
|
end
|
@@ -206,8 +215,7 @@ describe LogStash::Codecs::IdentityMapCodec do
|
|
206
215
|
let(:imc) { described_class.new(mlc) }
|
207
216
|
|
208
217
|
before do
|
209
|
-
|
210
|
-
listener.accept("foo")
|
218
|
+
l_0 = Mlc::LineListener.new(queue, imc, identity).tap {|o| o.accept("foo")}
|
211
219
|
end
|
212
220
|
|
213
221
|
describe "normal processing" do
|
@@ -221,11 +229,23 @@ describe LogStash::Codecs::IdentityMapCodec do
|
|
221
229
|
|
222
230
|
context "when wrapped codec has auto-flush activated" do
|
223
231
|
let(:config) { {"pattern" => "^\\s", "what" => "previous", "auto_flush_interval" => 0.2} }
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
232
|
+
before do
|
233
|
+
l_1 = Mlc::LineListener.new(queue, imc, "stream2").tap {|o| o.accept("bar")}
|
234
|
+
l_2 = Mlc::LineListener.new(queue, imc, "stream3").tap {|o| o.accept("baz")}
|
235
|
+
end
|
236
|
+
it "three events are auto flushed from three different identities/codecs" do
|
237
|
+
sleep 0.6 # wait for auto_flush - in multiples of 0.5 plus 0.1
|
238
|
+
expect(queue.size).to eq(3)
|
239
|
+
e1, e2, e3 = queue
|
240
|
+
expect(e1["path"]).to eq("stream1")
|
241
|
+
expect(e1["message"]).to eq("foo")
|
242
|
+
|
243
|
+
expect(e2["path"]).to eq("stream2")
|
244
|
+
expect(e2["message"]).to eq("bar")
|
245
|
+
|
246
|
+
expect(e3["path"]).to eq("stream3")
|
247
|
+
expect(e3["message"]).to eq("baz")
|
248
|
+
expect(imc.identity_count).to eq(3)
|
229
249
|
end
|
230
250
|
end
|
231
251
|
end
|
@@ -225,7 +225,7 @@ describe LogStash::Codecs::Multiline do
|
|
225
225
|
"de.log" => ["Hallo Welt"] }
|
226
226
|
end
|
227
227
|
let(:listener_class) { Mlc::LineListener }
|
228
|
-
let(:auto_flush_interval) {
|
228
|
+
let(:auto_flush_interval) { 2 }
|
229
229
|
|
230
230
|
let(:line_producer) do
|
231
231
|
lambda do |path|
|
data/spec/supports/helpers.rb
CHANGED
@@ -73,6 +73,10 @@ module Mlc
|
|
73
73
|
params.nil? ? false : params.last
|
74
74
|
end
|
75
75
|
|
76
|
+
def full_trace_for(symbol)
|
77
|
+
@tracer.select{|array| array[0] == symbol}.map(&:last)
|
78
|
+
end
|
79
|
+
|
76
80
|
def clear()
|
77
81
|
@tracer.clear()
|
78
82
|
end
|
@@ -91,7 +95,16 @@ module Mlc
|
|
91
95
|
end
|
92
96
|
|
93
97
|
class AutoFlushTracer < TracerBase
|
94
|
-
def auto_flush() @tracer.push [:auto_flush, true]; end
|
98
|
+
def auto_flush() simulate_execution_delay; @tracer.push [:auto_flush, true]; end
|
99
|
+
def set_delay(delay)
|
100
|
+
@delay = delay
|
101
|
+
end
|
102
|
+
|
103
|
+
def simulate_execution_delay
|
104
|
+
return if @delay.nil? || @delay.zero?
|
105
|
+
sleep @delay
|
106
|
+
@tracer.push [:delay, Time.now.to_f]
|
107
|
+
end
|
95
108
|
end
|
96
109
|
|
97
110
|
class IdentityMapCodecTracer < TracerBase
|
metadata
CHANGED
@@ -1,77 +1,77 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-multiline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-02-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
|
-
- -
|
17
|
+
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
18
19
|
version: 2.0.0
|
19
|
-
- -
|
20
|
+
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
|
-
- -
|
25
|
+
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
29
27
|
version: 2.0.0
|
30
|
-
- -
|
28
|
+
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-patterns-core
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
|
-
- -
|
42
|
+
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-patterns-core
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: jls-grok
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
|
-
- -
|
51
|
+
- - ~>
|
45
52
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
47
|
-
- !ruby/object:Gem::Dependency
|
53
|
+
version: 0.11.1
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
|
-
- -
|
56
|
+
- - ~>
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: 0.11.1
|
53
|
-
name: jls-grok
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-devutils
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
|
-
- -
|
65
|
+
- - '>='
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version: 0
|
61
|
-
- !ruby/object:Gem::Dependency
|
67
|
+
version: '0'
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
|
-
- -
|
70
|
+
- - '>='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: '0'
|
67
|
-
name: logstash-devutils
|
68
73
|
prerelease: false
|
69
74
|
type: :development
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
71
|
-
requirements:
|
72
|
-
- - ">="
|
73
|
-
- !ruby/object:Gem::Version
|
74
|
-
version: '0'
|
75
75
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
76
76
|
email: info@elastic.co
|
77
77
|
executables: []
|
@@ -87,6 +87,7 @@ files:
|
|
87
87
|
- lib/logstash/codecs/auto_flush.rb
|
88
88
|
- lib/logstash/codecs/identity_map_codec.rb
|
89
89
|
- lib/logstash/codecs/multiline.rb
|
90
|
+
- lib/logstash/codecs/retriggerable_task.rb
|
90
91
|
- logstash-codec-multiline.gemspec
|
91
92
|
- spec/codecs/auto_flush_spec.rb
|
92
93
|
- spec/codecs/identity_map_codec_spec.rb
|
@@ -104,12 +105,12 @@ require_paths:
|
|
104
105
|
- lib
|
105
106
|
required_ruby_version: !ruby/object:Gem::Requirement
|
106
107
|
requirements:
|
107
|
-
- -
|
108
|
+
- - '>='
|
108
109
|
- !ruby/object:Gem::Version
|
109
110
|
version: '0'
|
110
111
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
111
112
|
requirements:
|
112
|
-
- -
|
113
|
+
- - '>='
|
113
114
|
- !ruby/object:Gem::Version
|
114
115
|
version: '0'
|
115
116
|
requirements: []
|