logstash-codec-multiline 2.0.7 → 2.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/Gemfile +1 -2
- data/lib/logstash/codecs/auto_flush.rb +7 -61
- data/lib/logstash/codecs/identity_map_codec.rb +85 -29
- data/lib/logstash/codecs/multiline.rb +11 -7
- data/lib/logstash/codecs/retriggerable_task.rb +81 -0
- data/logstash-codec-multiline.gemspec +1 -1
- data/spec/codecs/auto_flush_spec.rb +48 -55
- data/spec/codecs/identity_map_codec_spec.rb +27 -7
- data/spec/codecs/multiline_spec.rb +1 -1
- data/spec/supports/helpers.rb +14 -1
- metadata +31 -30
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f9e0a15839b1bbabcad88ee95445b72ed589a4dd
|
4
|
+
data.tar.gz: bdf59bcab8b5ad4fc35906de033bde7ed658e52a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e0bcea055b8af6ebbd45300e2901146a2770b7937de6ca8b5f08f4a01b347cc7b876d3b675589c475b3b41f4ce65632e7edc44432a2f00b2abc5f9b7dc5d899
|
7
|
+
data.tar.gz: 6333ad636771cc1e9407f9642f242ff40f59181b4073fe55e2b3aa93b9cd675ca435961aabc7f054c4cad6be072bdefc42395198b14cafb3ba7b286474e888ab
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -1,74 +1,32 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "concurrent"
|
3
|
+
require "logstash/codecs/retriggerable_task"
|
3
4
|
|
4
5
|
module LogStash module Codecs class AutoFlush
|
5
6
|
def initialize(mc, interval)
|
6
7
|
@mc, @interval = mc, interval
|
7
8
|
@stopped = Concurrent::AtomicBoolean.new # false by default
|
9
|
+
@task = RetriggerableTask.new(@interval, self)
|
8
10
|
end
|
9
11
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
# if pending?
|
14
|
-
# @task.cancel
|
15
|
-
# create_task
|
16
|
-
# elsif finished?
|
17
|
-
# create_task
|
18
|
-
# # else the task is executing
|
19
|
-
# end
|
20
|
-
# self
|
21
|
-
# end
|
12
|
+
def timeout
|
13
|
+
@mc.auto_flush
|
14
|
+
end
|
22
15
|
|
23
16
|
def start
|
24
17
|
# can't start if pipeline is stopping
|
25
18
|
return self if stopped?
|
26
|
-
|
27
|
-
if pending? && @task.cancel
|
28
|
-
create_task
|
29
|
-
return self
|
30
|
-
end
|
31
|
-
# maybe we have a timing edge case
|
32
|
-
# where pending? was true but cancel failed
|
33
|
-
# because the task started running
|
34
|
-
if finished?
|
35
|
-
create_task
|
36
|
-
return self
|
37
|
-
end
|
38
|
-
# else the task is executing
|
39
|
-
# wait for task to complete
|
40
|
-
# flush could feasibly block on queue access
|
41
|
-
@task.value
|
42
|
-
create_task
|
19
|
+
@task.retrigger
|
43
20
|
self
|
44
21
|
end
|
45
22
|
|
46
|
-
def create_task
|
47
|
-
@task = Concurrent::ScheduledTask.execute(@interval) do
|
48
|
-
@mc.auto_flush()
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
def finished?
|
53
|
-
return true if @task.nil?
|
54
|
-
@task.fulfilled?
|
55
|
-
end
|
56
|
-
|
57
|
-
def pending?
|
58
|
-
@task && @task.pending?
|
59
|
-
end
|
60
|
-
|
61
23
|
def stopped?
|
62
24
|
@stopped.value
|
63
25
|
end
|
64
26
|
|
65
27
|
def stop
|
66
28
|
@stopped.make_true
|
67
|
-
|
68
|
-
end
|
69
|
-
|
70
|
-
def cancel
|
71
|
-
@task.cancel if pending?
|
29
|
+
@task.close
|
72
30
|
end
|
73
31
|
end
|
74
32
|
|
@@ -76,10 +34,6 @@ class AutoFlushUnset
|
|
76
34
|
def initialize(mc, interval)
|
77
35
|
end
|
78
36
|
|
79
|
-
def pending?
|
80
|
-
false
|
81
|
-
end
|
82
|
-
|
83
37
|
def stopped?
|
84
38
|
true
|
85
39
|
end
|
@@ -88,15 +42,7 @@ class AutoFlushUnset
|
|
88
42
|
self
|
89
43
|
end
|
90
44
|
|
91
|
-
def finished?
|
92
|
-
true
|
93
|
-
end
|
94
|
-
|
95
45
|
def stop
|
96
46
|
self
|
97
47
|
end
|
98
|
-
|
99
|
-
def cancel
|
100
|
-
self
|
101
|
-
end
|
102
48
|
end end end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/namespace"
|
3
3
|
require "thread_safe"
|
4
|
+
require "concurrent"
|
4
5
|
|
5
6
|
# This class is a Codec duck type
|
6
7
|
# Using Composition, it maps from a stream identity to
|
@@ -37,41 +38,51 @@ module LogStash module Codecs class IdentityMapCodec
|
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
40
|
-
class
|
41
|
-
def initialize(
|
42
|
-
@
|
43
|
-
@
|
41
|
+
class PeriodicRunner
|
42
|
+
def initialize(listener, interval, method_symbol)
|
43
|
+
@listener, @interval = listener, interval
|
44
|
+
@method_symbol = method_symbol
|
45
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
44
46
|
end
|
45
47
|
|
46
48
|
def start
|
47
49
|
return self if running?
|
48
|
-
@running
|
49
|
-
@thread = Thread.new(
|
50
|
-
|
50
|
+
@running.make_true
|
51
|
+
@thread = Thread.new() do
|
52
|
+
while running? do
|
51
53
|
sleep @interval
|
52
|
-
break if
|
53
|
-
|
54
|
+
break if !running?
|
55
|
+
@listener.send(@method_symbol)
|
54
56
|
end
|
55
57
|
end
|
56
58
|
self
|
57
59
|
end
|
58
60
|
|
59
61
|
def running?
|
60
|
-
@running
|
62
|
+
@running.value
|
61
63
|
end
|
62
64
|
|
63
65
|
def stop
|
64
66
|
return if !running?
|
65
|
-
@running
|
66
|
-
|
67
|
+
@running.make_false
|
68
|
+
if @thread.alive?
|
69
|
+
@thread.wakeup
|
70
|
+
@thread.join
|
71
|
+
end
|
72
|
+
@listener = nil
|
67
73
|
end
|
68
74
|
end
|
69
75
|
|
70
|
-
|
76
|
+
class NoopRunner
|
77
|
+
attr_reader :start, :stop
|
78
|
+
def running?() false; end
|
79
|
+
end
|
80
|
+
|
81
|
+
# A composite class to hold both the codec, the eviction_timeout and a last_used timestamp
|
71
82
|
# instances of this Value Object are stored in the mapping hash
|
72
83
|
class CodecValue
|
73
84
|
attr_reader :codec
|
74
|
-
attr_accessor :
|
85
|
+
attr_accessor :eviction_timeout, :auto_flush_timeout
|
75
86
|
|
76
87
|
def initialize(codec)
|
77
88
|
@codec = codec
|
@@ -92,7 +103,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
92
103
|
CLEANER_INTERVAL = 60 * 5 # 5 minutes
|
93
104
|
|
94
105
|
attr_reader :identity_map
|
95
|
-
attr_accessor :base_codec, :cleaner
|
106
|
+
attr_accessor :base_codec, :cleaner, :auto_flusher
|
96
107
|
|
97
108
|
def initialize(codec)
|
98
109
|
@base_codec = codec
|
@@ -100,7 +111,14 @@ module LogStash module Codecs class IdentityMapCodec
|
|
100
111
|
@identity_map = ThreadSafe::Hash.new &method(:codec_builder)
|
101
112
|
@max_identities = MAX_IDENTITIES
|
102
113
|
@evict_timeout = EVICT_TIMEOUT
|
103
|
-
|
114
|
+
cleaner_interval(CLEANER_INTERVAL)
|
115
|
+
if codec.respond_to?(:use_mapper_auto_flush) &&
|
116
|
+
(@auto_flush_interval = codec.use_mapper_auto_flush)
|
117
|
+
@auto_flusher = PeriodicRunner.new(self, 0.5, :auto_flush_mapped)
|
118
|
+
else
|
119
|
+
@auto_flusher = NoopRunner.new
|
120
|
+
end
|
121
|
+
|
104
122
|
@decode_block = lambda {|*| true }
|
105
123
|
@eviction_block = nil
|
106
124
|
end
|
@@ -123,8 +141,8 @@ module LogStash module Codecs class IdentityMapCodec
|
|
123
141
|
|
124
142
|
# used to add a non-default cleaner interval
|
125
143
|
def cleaner_interval(interval)
|
126
|
-
@cleaner.stop
|
127
|
-
@cleaner =
|
144
|
+
@cleaner.stop if @cleaner
|
145
|
+
@cleaner = PeriodicRunner.new(self, interval.to_i, :map_cleanup)
|
128
146
|
self
|
129
147
|
end
|
130
148
|
|
@@ -133,6 +151,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
133
151
|
@eviction_block = block
|
134
152
|
self
|
135
153
|
end
|
154
|
+
|
136
155
|
# end Constructional/builder methods
|
137
156
|
# ==============================================
|
138
157
|
|
@@ -182,11 +201,34 @@ module LogStash module Codecs class IdentityMapCodec
|
|
182
201
|
|
183
202
|
def close()
|
184
203
|
cleaner.stop
|
204
|
+
auto_flusher.stop
|
185
205
|
all_codecs.each(&:close)
|
186
206
|
end
|
187
207
|
# end Codec API
|
188
208
|
# ==============================================
|
189
209
|
|
210
|
+
def auto_flush_mapped
|
211
|
+
if !identity_count.zero?
|
212
|
+
nowf = Time.now.to_f
|
213
|
+
identity_map.each do |identity, compo|
|
214
|
+
next if compo.auto_flush_timeout.zero?
|
215
|
+
next unless nowf > compo.auto_flush_timeout
|
216
|
+
compo.codec.auto_flush
|
217
|
+
# at eof (tail and read) no more lines for a while or ever
|
218
|
+
# so reset compo.auto_flush_timeout
|
219
|
+
compo.auto_flush_timeout = 0
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def flush_mapped(listener)
|
225
|
+
listener_has_path = listener.respond_to?(:path)
|
226
|
+
identity_map.each do |identity, compo|
|
227
|
+
listener.path = identity if listener_has_path
|
228
|
+
compo.codec.auto_flush(listener)
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
190
232
|
def all_codecs
|
191
233
|
no_streams? ? @base_codecs : identity_map.values.map(&:codec)
|
192
234
|
end
|
@@ -204,14 +246,16 @@ module LogStash module Codecs class IdentityMapCodec
|
|
204
246
|
# been accessed in the last @evict_timeout
|
205
247
|
# period (default 1 hour)
|
206
248
|
def map_cleanup
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
249
|
+
if !identity_count.zero?
|
250
|
+
nowi = Time.now.to_i
|
251
|
+
# delete_if is atomic
|
252
|
+
# contents should not mutate during this call
|
253
|
+
identity_map.delete_if do |identity, compo|
|
254
|
+
if (flag = compo.eviction_timeout <= nowi)
|
255
|
+
evict_flush(compo.codec)
|
256
|
+
end
|
257
|
+
flag
|
213
258
|
end
|
214
|
-
flag
|
215
259
|
end
|
216
260
|
current_size_and_limit
|
217
261
|
end
|
@@ -241,7 +285,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
241
285
|
end
|
242
286
|
|
243
287
|
def eviction_timestamp_for(identity)
|
244
|
-
find_codec_value(identity).
|
288
|
+
find_codec_value(identity).eviction_timeout
|
245
289
|
end
|
246
290
|
|
247
291
|
private
|
@@ -261,13 +305,20 @@ module LogStash module Codecs class IdentityMapCodec
|
|
261
305
|
# only start the cleaner if streams are in use
|
262
306
|
# continuous calls to start are OK
|
263
307
|
cleaner.start
|
308
|
+
auto_flusher.start
|
264
309
|
compo = find_codec_value(identity)
|
265
|
-
|
310
|
+
now = Time.now
|
311
|
+
compo.eviction_timeout = eviction_timestamp(now)
|
312
|
+
compo.auto_flush_timeout = auto_flush_timestamp(now)
|
266
313
|
compo.codec
|
267
314
|
end
|
268
315
|
|
269
|
-
def
|
270
|
-
|
316
|
+
def auto_flush_timestamp(now = Time.now)
|
317
|
+
now.to_f + @auto_flush_interval.to_f
|
318
|
+
end
|
319
|
+
|
320
|
+
def eviction_timestamp(now = Time.now)
|
321
|
+
now.to_i + @evict_timeout
|
271
322
|
end
|
272
323
|
|
273
324
|
def check_map_limits
|
@@ -277,6 +328,7 @@ module LogStash module Codecs class IdentityMapCodec
|
|
277
328
|
|
278
329
|
def codec_builder(hash, k)
|
279
330
|
codec = hash.empty? ? @base_codec : @base_codec.clone
|
331
|
+
codec.use_mapper_auto_flush if using_mapped_auto_flush?
|
280
332
|
compo = CodecValue.new(codec)
|
281
333
|
hash.store(k, compo)
|
282
334
|
end
|
@@ -284,4 +336,8 @@ module LogStash module Codecs class IdentityMapCodec
|
|
284
336
|
def no_streams?
|
285
337
|
identity_map.empty?
|
286
338
|
end
|
339
|
+
|
340
|
+
def using_mapped_auto_flush?
|
341
|
+
!@auto_flush_interval.nil?
|
342
|
+
end
|
287
343
|
end end end
|
@@ -171,6 +171,12 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
|
|
171
171
|
end
|
172
172
|
end # def register
|
173
173
|
|
174
|
+
def use_mapper_auto_flush
|
175
|
+
return unless auto_flush_active?
|
176
|
+
@auto_flush_runner = AutoFlushUnset.new(nil, nil)
|
177
|
+
@auto_flush_interval = @auto_flush_interval.to_f
|
178
|
+
end
|
179
|
+
|
174
180
|
def accept(listener)
|
175
181
|
# memoize references to listener that holds upstream state
|
176
182
|
@previous_listener = @last_seen_listener || listener
|
@@ -215,9 +221,11 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
|
|
215
221
|
end
|
216
222
|
end
|
217
223
|
|
218
|
-
def auto_flush
|
224
|
+
def auto_flush(listener = @last_seen_listener)
|
225
|
+
return if listener.nil?
|
226
|
+
|
219
227
|
flush do |event|
|
220
|
-
|
228
|
+
listener.process_event(event)
|
221
229
|
end
|
222
230
|
end
|
223
231
|
|
@@ -272,11 +280,7 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
|
|
272
280
|
end # def encode
|
273
281
|
|
274
282
|
def close
|
275
|
-
|
276
|
-
#will cancel task if necessary
|
277
|
-
auto_flush_runner.stop
|
278
|
-
end
|
279
|
-
auto_flush
|
283
|
+
auto_flush_runner.stop
|
280
284
|
end
|
281
285
|
|
282
286
|
def auto_flush_active?
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module LogStash module Codecs class RetriggerableTask
|
4
|
+
SLEEP_FOR = 0.25.freeze
|
5
|
+
|
6
|
+
attr_reader :thread
|
7
|
+
|
8
|
+
def initialize(delay, listener)
|
9
|
+
@count = calculate_count(delay)
|
10
|
+
@listener = listener
|
11
|
+
@counter = Concurrent::AtomicFixnum.new(0 + @count)
|
12
|
+
@stopped = Concurrent::AtomicBoolean.new(false)
|
13
|
+
@semaphore = Concurrent::Semaphore.new(1)
|
14
|
+
end
|
15
|
+
|
16
|
+
def retrigger
|
17
|
+
return if stopped?
|
18
|
+
if executing?
|
19
|
+
@semaphore.acquire
|
20
|
+
end
|
21
|
+
|
22
|
+
if pending?
|
23
|
+
reset_counter
|
24
|
+
else
|
25
|
+
start
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def close
|
30
|
+
@stopped.make_true
|
31
|
+
end
|
32
|
+
|
33
|
+
def counter
|
34
|
+
@counter.value
|
35
|
+
end
|
36
|
+
|
37
|
+
def executing?
|
38
|
+
running? && counter < 1
|
39
|
+
end
|
40
|
+
|
41
|
+
def pending?
|
42
|
+
running? && counter > 0
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def calculate_count(value)
|
48
|
+
# in multiples of SLEEP_FOR (0.25) seconds
|
49
|
+
# if delay is 10 seconds then count is 40
|
50
|
+
# this only works when SLEEP_FOR is less than 1
|
51
|
+
return 1 if value < SLEEP_FOR
|
52
|
+
(value / SLEEP_FOR).floor
|
53
|
+
end
|
54
|
+
|
55
|
+
def reset_counter
|
56
|
+
@counter.value = 0 + @count
|
57
|
+
end
|
58
|
+
|
59
|
+
def running?
|
60
|
+
@thread && @thread.alive?
|
61
|
+
end
|
62
|
+
|
63
|
+
def start()
|
64
|
+
reset_counter
|
65
|
+
@thread = Thread.new do
|
66
|
+
while counter > 0
|
67
|
+
break if stopped?
|
68
|
+
sleep SLEEP_FOR
|
69
|
+
@counter.decrement
|
70
|
+
end
|
71
|
+
|
72
|
+
@semaphore.drain_permits
|
73
|
+
@listener.timeout if !stopped?
|
74
|
+
@semaphore.release
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def stopped?
|
79
|
+
@stopped.value
|
80
|
+
end
|
81
|
+
end end end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-multiline'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.8'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -4,109 +4,102 @@ require "logstash/codecs/multiline"
|
|
4
4
|
require_relative "../supports/helpers.rb"
|
5
5
|
|
6
6
|
describe "AutoFlush and AutoFlushUnset" do
|
7
|
+
before(:all) do
|
8
|
+
@thread_abort = Thread.abort_on_exception
|
9
|
+
Thread.abort_on_exception = true
|
10
|
+
end
|
11
|
+
|
12
|
+
after(:all) do
|
13
|
+
Thread.abort_on_exception = @thread_abort
|
14
|
+
end
|
15
|
+
|
7
16
|
let(:flushable) { Mlc::AutoFlushTracer.new }
|
8
|
-
let(:flush_wait) { 0.
|
17
|
+
let(:flush_wait) { 0.25 }
|
9
18
|
|
10
19
|
describe LogStash::Codecs::AutoFlush do
|
11
20
|
subject { described_class.new(flushable, flush_wait) }
|
12
21
|
|
13
22
|
context "when initialized" do
|
14
|
-
it "#pending? is false" do
|
15
|
-
expect(subject.pending?).to be_falsy
|
16
|
-
end
|
17
|
-
|
18
23
|
it "#stopped? is false" do
|
19
|
-
expect(subject.stopped?).to
|
20
|
-
end
|
21
|
-
|
22
|
-
it "#finished? is true" do
|
23
|
-
expect(subject.finished?).to be_truthy
|
24
|
+
expect(subject.stopped?).to be_falsey
|
24
25
|
end
|
25
26
|
end
|
26
27
|
|
27
|
-
context "when
|
28
|
-
let(:flush_wait) { 20 }
|
29
|
-
|
28
|
+
context "when starting once" do
|
30
29
|
before { subject.start }
|
31
30
|
after { subject.stop }
|
32
31
|
|
33
|
-
it "
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
it "#stopped? is false" do
|
38
|
-
expect(subject.stopped?).to be_falsy
|
39
|
-
end
|
40
|
-
|
41
|
-
it "#finished? is false" do
|
42
|
-
expect(subject.finished?).to be_falsy
|
32
|
+
it "calls auto_flush on flushable" do
|
33
|
+
sleep flush_wait + 0.5
|
34
|
+
expect(flushable.trace_for(:auto_flush)).to be_truthy
|
43
35
|
end
|
44
36
|
end
|
45
37
|
|
46
|
-
context "when
|
47
|
-
before
|
48
|
-
subject.start
|
49
|
-
sleep flush_wait + 0.1
|
50
|
-
end
|
51
|
-
|
38
|
+
context "when starting multiple times" do
|
39
|
+
before { subject.start }
|
52
40
|
after { subject.stop }
|
53
41
|
|
54
42
|
it "calls auto_flush on flushable" do
|
55
|
-
expect(flushable.trace_for(:auto_flush)).to
|
56
|
-
|
43
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
44
|
+
sleep 0.1
|
45
|
+
subject.start
|
46
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
47
|
+
sleep 0.1
|
48
|
+
subject.start
|
49
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
57
50
|
|
58
|
-
|
59
|
-
expect(
|
51
|
+
sleep flush_wait + 0.5
|
52
|
+
expect(flushable.trace_for(:auto_flush)).to be_truthy
|
60
53
|
end
|
54
|
+
end
|
61
55
|
|
62
|
-
|
63
|
-
|
56
|
+
context "when retriggering during execution" do
|
57
|
+
let(:flush_wait) { 1 }
|
58
|
+
let(:delay) { 1 }
|
59
|
+
before do
|
60
|
+
flushable.set_delay(delay)
|
61
|
+
subject.start
|
64
62
|
end
|
63
|
+
after { subject.stop }
|
65
64
|
|
66
|
-
it "
|
67
|
-
|
65
|
+
it "calls auto_flush twice on flushable" do
|
66
|
+
now = Time.now.to_f
|
67
|
+
combined_time = flush_wait + delay
|
68
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
69
|
+
sleep flush_wait + 0.25 # <---- wait for execution to start
|
70
|
+
subject.start # <---- because of the semaphore, this waits for delay = 2 seconds, then starts a new thread
|
71
|
+
sleep combined_time + 0.5 # <---- wait for more than 4 seconds = flush_wait + delay
|
72
|
+
elapsed1, elapsed2 = flushable.full_trace_for(:delay).map{ |el| (el - now).floor }
|
73
|
+
expect(elapsed1).to eq(combined_time)
|
74
|
+
expect(elapsed2).to eq(2 * combined_time)
|
75
|
+
expect(flushable.full_trace_for(:auto_flush)).to eq([true, true])
|
68
76
|
end
|
69
77
|
end
|
70
78
|
|
71
|
-
context "when
|
79
|
+
context "when stopping before timeout" do
|
72
80
|
before do
|
73
81
|
subject.start
|
82
|
+
sleep 0.1
|
74
83
|
subject.stop
|
75
84
|
end
|
76
85
|
|
77
86
|
it "does not call auto_flush on flushable" do
|
78
|
-
expect(flushable.trace_for(:auto_flush)).to
|
79
|
-
end
|
80
|
-
|
81
|
-
it "#pending? is false" do
|
82
|
-
expect(subject.pending?).to be_falsy
|
87
|
+
expect(flushable.trace_for(:auto_flush)).to be_falsey
|
83
88
|
end
|
84
89
|
|
85
90
|
it "#stopped? is true" do
|
86
91
|
expect(subject.stopped?).to be_truthy
|
87
92
|
end
|
88
|
-
|
89
|
-
it "#finished? is false" do
|
90
|
-
expect(subject.finished?).to be_falsy
|
91
|
-
end
|
92
93
|
end
|
93
94
|
end
|
94
95
|
|
95
96
|
describe LogStash::Codecs::AutoFlushUnset do
|
96
97
|
subject { described_class.new(flushable, 2) }
|
97
98
|
|
98
|
-
it "#pending? is false" do
|
99
|
-
expect(subject.pending?).to be_falsy
|
100
|
-
end
|
101
|
-
|
102
99
|
it "#stopped? is true" do
|
103
100
|
expect(subject.stopped?).to be_truthy
|
104
101
|
end
|
105
102
|
|
106
|
-
it "#finished? is true" do
|
107
|
-
expect(subject.finished?).to be_truthy
|
108
|
-
end
|
109
|
-
|
110
103
|
it "#start returns self" do
|
111
104
|
expect(subject.start).to eq(subject)
|
112
105
|
end
|
@@ -12,6 +12,15 @@ describe LogStash::Codecs::IdentityMapCodec do
|
|
12
12
|
let(:codec1) { demuxer.codec_without_usage_update(stream1) }
|
13
13
|
let(:arg1) { "data-a" }
|
14
14
|
|
15
|
+
before(:all) do
|
16
|
+
@thread_abort = Thread.abort_on_exception
|
17
|
+
Thread.abort_on_exception = true
|
18
|
+
end
|
19
|
+
|
20
|
+
after(:all) do
|
21
|
+
Thread.abort_on_exception = @thread_abort
|
22
|
+
end
|
23
|
+
|
15
24
|
after do
|
16
25
|
codec.close
|
17
26
|
end
|
@@ -206,8 +215,7 @@ describe LogStash::Codecs::IdentityMapCodec do
|
|
206
215
|
let(:imc) { described_class.new(mlc) }
|
207
216
|
|
208
217
|
before do
|
209
|
-
|
210
|
-
listener.accept("foo")
|
218
|
+
l_0 = Mlc::LineListener.new(queue, imc, identity).tap {|o| o.accept("foo")}
|
211
219
|
end
|
212
220
|
|
213
221
|
describe "normal processing" do
|
@@ -221,11 +229,23 @@ describe LogStash::Codecs::IdentityMapCodec do
|
|
221
229
|
|
222
230
|
context "when wrapped codec has auto-flush activated" do
|
223
231
|
let(:config) { {"pattern" => "^\\s", "what" => "previous", "auto_flush_interval" => 0.2} }
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
232
|
+
before do
|
233
|
+
l_1 = Mlc::LineListener.new(queue, imc, "stream2").tap {|o| o.accept("bar")}
|
234
|
+
l_2 = Mlc::LineListener.new(queue, imc, "stream3").tap {|o| o.accept("baz")}
|
235
|
+
end
|
236
|
+
it "three events are auto flushed from three different identities/codecs" do
|
237
|
+
sleep 0.6 # wait for auto_flush - in multiples of 0.5 plus 0.1
|
238
|
+
expect(queue.size).to eq(3)
|
239
|
+
e1, e2, e3 = queue
|
240
|
+
expect(e1["path"]).to eq("stream1")
|
241
|
+
expect(e1["message"]).to eq("foo")
|
242
|
+
|
243
|
+
expect(e2["path"]).to eq("stream2")
|
244
|
+
expect(e2["message"]).to eq("bar")
|
245
|
+
|
246
|
+
expect(e3["path"]).to eq("stream3")
|
247
|
+
expect(e3["message"]).to eq("baz")
|
248
|
+
expect(imc.identity_count).to eq(3)
|
229
249
|
end
|
230
250
|
end
|
231
251
|
end
|
@@ -225,7 +225,7 @@ describe LogStash::Codecs::Multiline do
|
|
225
225
|
"de.log" => ["Hallo Welt"] }
|
226
226
|
end
|
227
227
|
let(:listener_class) { Mlc::LineListener }
|
228
|
-
let(:auto_flush_interval) {
|
228
|
+
let(:auto_flush_interval) { 2 }
|
229
229
|
|
230
230
|
let(:line_producer) do
|
231
231
|
lambda do |path|
|
data/spec/supports/helpers.rb
CHANGED
@@ -73,6 +73,10 @@ module Mlc
|
|
73
73
|
params.nil? ? false : params.last
|
74
74
|
end
|
75
75
|
|
76
|
+
def full_trace_for(symbol)
|
77
|
+
@tracer.select{|array| array[0] == symbol}.map(&:last)
|
78
|
+
end
|
79
|
+
|
76
80
|
def clear()
|
77
81
|
@tracer.clear()
|
78
82
|
end
|
@@ -91,7 +95,16 @@ module Mlc
|
|
91
95
|
end
|
92
96
|
|
93
97
|
class AutoFlushTracer < TracerBase
|
94
|
-
def auto_flush() @tracer.push [:auto_flush, true]; end
|
98
|
+
def auto_flush() simulate_execution_delay; @tracer.push [:auto_flush, true]; end
|
99
|
+
def set_delay(delay)
|
100
|
+
@delay = delay
|
101
|
+
end
|
102
|
+
|
103
|
+
def simulate_execution_delay
|
104
|
+
return if @delay.nil? || @delay.zero?
|
105
|
+
sleep @delay
|
106
|
+
@tracer.push [:delay, Time.now.to_f]
|
107
|
+
end
|
95
108
|
end
|
96
109
|
|
97
110
|
class IdentityMapCodecTracer < TracerBase
|
metadata
CHANGED
@@ -1,77 +1,77 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-multiline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-02-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
|
-
- -
|
17
|
+
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
18
19
|
version: 2.0.0
|
19
|
-
- -
|
20
|
+
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
|
-
- -
|
25
|
+
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
29
27
|
version: 2.0.0
|
30
|
-
- -
|
28
|
+
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-patterns-core
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
|
-
- -
|
42
|
+
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-patterns-core
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: jls-grok
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
|
-
- -
|
51
|
+
- - ~>
|
45
52
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
47
|
-
- !ruby/object:Gem::Dependency
|
53
|
+
version: 0.11.1
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
|
-
- -
|
56
|
+
- - ~>
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: 0.11.1
|
53
|
-
name: jls-grok
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-devutils
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
|
-
- -
|
65
|
+
- - '>='
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version: 0
|
61
|
-
- !ruby/object:Gem::Dependency
|
67
|
+
version: '0'
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
|
-
- -
|
70
|
+
- - '>='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: '0'
|
67
|
-
name: logstash-devutils
|
68
73
|
prerelease: false
|
69
74
|
type: :development
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
71
|
-
requirements:
|
72
|
-
- - ">="
|
73
|
-
- !ruby/object:Gem::Version
|
74
|
-
version: '0'
|
75
75
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
76
76
|
email: info@elastic.co
|
77
77
|
executables: []
|
@@ -87,6 +87,7 @@ files:
|
|
87
87
|
- lib/logstash/codecs/auto_flush.rb
|
88
88
|
- lib/logstash/codecs/identity_map_codec.rb
|
89
89
|
- lib/logstash/codecs/multiline.rb
|
90
|
+
- lib/logstash/codecs/retriggerable_task.rb
|
90
91
|
- logstash-codec-multiline.gemspec
|
91
92
|
- spec/codecs/auto_flush_spec.rb
|
92
93
|
- spec/codecs/identity_map_codec_spec.rb
|
@@ -104,12 +105,12 @@ require_paths:
|
|
104
105
|
- lib
|
105
106
|
required_ruby_version: !ruby/object:Gem::Requirement
|
106
107
|
requirements:
|
107
|
-
- -
|
108
|
+
- - '>='
|
108
109
|
- !ruby/object:Gem::Version
|
109
110
|
version: '0'
|
110
111
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
111
112
|
requirements:
|
112
|
-
- -
|
113
|
+
- - '>='
|
113
114
|
- !ruby/object:Gem::Version
|
114
115
|
version: '0'
|
115
116
|
requirements: []
|