logstash-codec-multiline 2.0.7 → 2.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 36df6112f2d0e3f3d6b3d0481a72e5250fae5352
4
- data.tar.gz: b86e58bbb8f62680e7ab478adc0d75a3d61d6530
3
+ metadata.gz: f9e0a15839b1bbabcad88ee95445b72ed589a4dd
4
+ data.tar.gz: bdf59bcab8b5ad4fc35906de033bde7ed658e52a
5
5
  SHA512:
6
- metadata.gz: afe492266d2d77c9946ff76ad71e755f8d03a655222f92673f70d0804b99543edd82dbb3365674f1f640036eb24cb42c6f3b65d6bea290378e959de6f6356bfe
7
- data.tar.gz: e053998e3802437a06a82b5997f28c8bc4983b80c936a1d9bbbe3b81a4a99be11f6086dac0f01a73c69fb2770728ee6ef030f807714c226a7e570815d11c49d2
6
+ metadata.gz: 5e0bcea055b8af6ebbd45300e2901146a2770b7937de6ca8b5f08f4a01b347cc7b876d3b675589c475b3b41f4ce65632e7edc44432a2f00b2abc5f9b7dc5d899
7
+ data.tar.gz: 6333ad636771cc1e9407f9642f242ff40f59181b4073fe55e2b3aa93b9cd675ca435961aabc7f054c4cad6be072bdefc42395198b14cafb3ba7b286474e888ab
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 2.0.8
2
+ - Fix for issue #28, Memory leak when using cancel + execute on ScheduledTask.
3
+
1
4
  ## 2.0.7
2
5
  - Fix for issue #23, Thread deadlock on auto-flush when using multiple inputs with this codec. Use cancel + execute on ScheduledTask instead of reset.
3
6
 
data/Gemfile CHANGED
@@ -1,3 +1,2 @@
1
1
  source 'https://rubygems.org'
2
- gemspec
3
- gem "concurrent-ruby", "0.9.1"
2
+ gemspec
@@ -1,74 +1,32 @@
1
1
  # encoding: utf-8
2
2
  require "concurrent"
3
+ require "logstash/codecs/retriggerable_task"
3
4
 
4
5
  module LogStash module Codecs class AutoFlush
5
6
  def initialize(mc, interval)
6
7
  @mc, @interval = mc, interval
7
8
  @stopped = Concurrent::AtomicBoolean.new # false by default
9
+ @task = RetriggerableTask.new(@interval, self)
8
10
  end
9
11
 
10
- # def start
11
- # # can't start if pipeline is stopping
12
- # return self if stopped?
13
- # if pending?
14
- # @task.cancel
15
- # create_task
16
- # elsif finished?
17
- # create_task
18
- # # else the task is executing
19
- # end
20
- # self
21
- # end
12
+ def timeout
13
+ @mc.auto_flush
14
+ end
22
15
 
23
16
  def start
24
17
  # can't start if pipeline is stopping
25
18
  return self if stopped?
26
-
27
- if pending? && @task.cancel
28
- create_task
29
- return self
30
- end
31
- # maybe we have a timing edge case
32
- # where pending? was true but cancel failed
33
- # because the task started running
34
- if finished?
35
- create_task
36
- return self
37
- end
38
- # else the task is executing
39
- # wait for task to complete
40
- # flush could feasibly block on queue access
41
- @task.value
42
- create_task
19
+ @task.retrigger
43
20
  self
44
21
  end
45
22
 
46
- def create_task
47
- @task = Concurrent::ScheduledTask.execute(@interval) do
48
- @mc.auto_flush()
49
- end
50
- end
51
-
52
- def finished?
53
- return true if @task.nil?
54
- @task.fulfilled?
55
- end
56
-
57
- def pending?
58
- @task && @task.pending?
59
- end
60
-
61
23
  def stopped?
62
24
  @stopped.value
63
25
  end
64
26
 
65
27
  def stop
66
28
  @stopped.make_true
67
- cancel
68
- end
69
-
70
- def cancel
71
- @task.cancel if pending?
29
+ @task.close
72
30
  end
73
31
  end
74
32
 
@@ -76,10 +34,6 @@ class AutoFlushUnset
76
34
  def initialize(mc, interval)
77
35
  end
78
36
 
79
- def pending?
80
- false
81
- end
82
-
83
37
  def stopped?
84
38
  true
85
39
  end
@@ -88,15 +42,7 @@ class AutoFlushUnset
88
42
  self
89
43
  end
90
44
 
91
- def finished?
92
- true
93
- end
94
-
95
45
  def stop
96
46
  self
97
47
  end
98
-
99
- def cancel
100
- self
101
- end
102
48
  end end end
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/namespace"
3
3
  require "thread_safe"
4
+ require "concurrent"
4
5
 
5
6
  # This class is a Codec duck type
6
7
  # Using Composition, it maps from a stream identity to
@@ -37,41 +38,51 @@ module LogStash module Codecs class IdentityMapCodec
37
38
  end
38
39
  end
39
40
 
40
- class MapCleaner
41
- def initialize(imc, interval)
42
- @imc, @interval = imc, interval
43
- @running = false
41
+ class PeriodicRunner
42
+ def initialize(listener, interval, method_symbol)
43
+ @listener, @interval = listener, interval
44
+ @method_symbol = method_symbol
45
+ @running = Concurrent::AtomicBoolean.new(false)
44
46
  end
45
47
 
46
48
  def start
47
49
  return self if running?
48
- @running = true
49
- @thread = Thread.new(@imc) do |imc|
50
- loop do
50
+ @running.make_true
51
+ @thread = Thread.new() do
52
+ while running? do
51
53
  sleep @interval
52
- break if !@running
53
- imc.map_cleanup
54
+ break if !running?
55
+ @listener.send(@method_symbol)
54
56
  end
55
57
  end
56
58
  self
57
59
  end
58
60
 
59
61
  def running?
60
- @running
62
+ @running.value
61
63
  end
62
64
 
63
65
  def stop
64
66
  return if !running?
65
- @running = false
66
- @thread.wakeup if @thread.alive?
67
+ @running.make_false
68
+ if @thread.alive?
69
+ @thread.wakeup
70
+ @thread.join
71
+ end
72
+ @listener = nil
67
73
  end
68
74
  end
69
75
 
70
- # A composite class to hold both the codec and the eviction_timeout
76
+ class NoopRunner
77
+ attr_reader :start, :stop
78
+ def running?() false; end
79
+ end
80
+
81
+ # A composite class to hold both the codec, the eviction_timeout and a last_used timestamp
71
82
  # instances of this Value Object are stored in the mapping hash
72
83
  class CodecValue
73
84
  attr_reader :codec
74
- attr_accessor :timeout
85
+ attr_accessor :eviction_timeout, :auto_flush_timeout
75
86
 
76
87
  def initialize(codec)
77
88
  @codec = codec
@@ -92,7 +103,7 @@ module LogStash module Codecs class IdentityMapCodec
92
103
  CLEANER_INTERVAL = 60 * 5 # 5 minutes
93
104
 
94
105
  attr_reader :identity_map
95
- attr_accessor :base_codec, :cleaner
106
+ attr_accessor :base_codec, :cleaner, :auto_flusher
96
107
 
97
108
  def initialize(codec)
98
109
  @base_codec = codec
@@ -100,7 +111,14 @@ module LogStash module Codecs class IdentityMapCodec
100
111
  @identity_map = ThreadSafe::Hash.new &method(:codec_builder)
101
112
  @max_identities = MAX_IDENTITIES
102
113
  @evict_timeout = EVICT_TIMEOUT
103
- @cleaner = MapCleaner.new(self, CLEANER_INTERVAL)
114
+ cleaner_interval(CLEANER_INTERVAL)
115
+ if codec.respond_to?(:use_mapper_auto_flush) &&
116
+ (@auto_flush_interval = codec.use_mapper_auto_flush)
117
+ @auto_flusher = PeriodicRunner.new(self, 0.5, :auto_flush_mapped)
118
+ else
119
+ @auto_flusher = NoopRunner.new
120
+ end
121
+
104
122
  @decode_block = lambda {|*| true }
105
123
  @eviction_block = nil
106
124
  end
@@ -123,8 +141,8 @@ module LogStash module Codecs class IdentityMapCodec
123
141
 
124
142
  # used to add a non-default cleaner interval
125
143
  def cleaner_interval(interval)
126
- @cleaner.stop
127
- @cleaner = MapCleaner.new(self, interval.to_i)
144
+ @cleaner.stop if @cleaner
145
+ @cleaner = PeriodicRunner.new(self, interval.to_i, :map_cleanup)
128
146
  self
129
147
  end
130
148
 
@@ -133,6 +151,7 @@ module LogStash module Codecs class IdentityMapCodec
133
151
  @eviction_block = block
134
152
  self
135
153
  end
154
+
136
155
  # end Constructional/builder methods
137
156
  # ==============================================
138
157
 
@@ -182,11 +201,34 @@ module LogStash module Codecs class IdentityMapCodec
182
201
 
183
202
  def close()
184
203
  cleaner.stop
204
+ auto_flusher.stop
185
205
  all_codecs.each(&:close)
186
206
  end
187
207
  # end Codec API
188
208
  # ==============================================
189
209
 
210
+ def auto_flush_mapped
211
+ if !identity_count.zero?
212
+ nowf = Time.now.to_f
213
+ identity_map.each do |identity, compo|
214
+ next if compo.auto_flush_timeout.zero?
215
+ next unless nowf > compo.auto_flush_timeout
216
+ compo.codec.auto_flush
217
+ # at eof (tail and read) no more lines for a while or ever
218
+ # so reset compo.auto_flush_timeout
219
+ compo.auto_flush_timeout = 0
220
+ end
221
+ end
222
+ end
223
+
224
+ def flush_mapped(listener)
225
+ listener_has_path = listener.respond_to?(:path)
226
+ identity_map.each do |identity, compo|
227
+ listener.path = identity if listener_has_path
228
+ compo.codec.auto_flush(listener)
229
+ end
230
+ end
231
+
190
232
  def all_codecs
191
233
  no_streams? ? @base_codecs : identity_map.values.map(&:codec)
192
234
  end
@@ -204,14 +246,16 @@ module LogStash module Codecs class IdentityMapCodec
204
246
  # been accessed in the last @evict_timeout
205
247
  # period (default 1 hour)
206
248
  def map_cleanup
207
- cut_off = Time.now.to_i
208
- # delete_if is atomic
209
- # contents should not mutate during this call
210
- identity_map.delete_if do |identity, compo|
211
- if (flag = compo.timeout <= cut_off)
212
- evict_flush(compo.codec)
249
+ if !identity_count.zero?
250
+ nowi = Time.now.to_i
251
+ # delete_if is atomic
252
+ # contents should not mutate during this call
253
+ identity_map.delete_if do |identity, compo|
254
+ if (flag = compo.eviction_timeout <= nowi)
255
+ evict_flush(compo.codec)
256
+ end
257
+ flag
213
258
  end
214
- flag
215
259
  end
216
260
  current_size_and_limit
217
261
  end
@@ -241,7 +285,7 @@ module LogStash module Codecs class IdentityMapCodec
241
285
  end
242
286
 
243
287
  def eviction_timestamp_for(identity)
244
- find_codec_value(identity).timeout
288
+ find_codec_value(identity).eviction_timeout
245
289
  end
246
290
 
247
291
  private
@@ -261,13 +305,20 @@ module LogStash module Codecs class IdentityMapCodec
261
305
  # only start the cleaner if streams are in use
262
306
  # continuous calls to start are OK
263
307
  cleaner.start
308
+ auto_flusher.start
264
309
  compo = find_codec_value(identity)
265
- compo.timeout = eviction_timestamp
310
+ now = Time.now
311
+ compo.eviction_timeout = eviction_timestamp(now)
312
+ compo.auto_flush_timeout = auto_flush_timestamp(now)
266
313
  compo.codec
267
314
  end
268
315
 
269
- def eviction_timestamp
270
- Time.now.to_i + @evict_timeout
316
+ def auto_flush_timestamp(now = Time.now)
317
+ now.to_f + @auto_flush_interval.to_f
318
+ end
319
+
320
+ def eviction_timestamp(now = Time.now)
321
+ now.to_i + @evict_timeout
271
322
  end
272
323
 
273
324
  def check_map_limits
@@ -277,6 +328,7 @@ module LogStash module Codecs class IdentityMapCodec
277
328
 
278
329
  def codec_builder(hash, k)
279
330
  codec = hash.empty? ? @base_codec : @base_codec.clone
331
+ codec.use_mapper_auto_flush if using_mapped_auto_flush?
280
332
  compo = CodecValue.new(codec)
281
333
  hash.store(k, compo)
282
334
  end
@@ -284,4 +336,8 @@ module LogStash module Codecs class IdentityMapCodec
284
336
  def no_streams?
285
337
  identity_map.empty?
286
338
  end
339
+
340
+ def using_mapped_auto_flush?
341
+ !@auto_flush_interval.nil?
342
+ end
287
343
  end end end
@@ -171,6 +171,12 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
171
171
  end
172
172
  end # def register
173
173
 
174
+ def use_mapper_auto_flush
175
+ return unless auto_flush_active?
176
+ @auto_flush_runner = AutoFlushUnset.new(nil, nil)
177
+ @auto_flush_interval = @auto_flush_interval.to_f
178
+ end
179
+
174
180
  def accept(listener)
175
181
  # memoize references to listener that holds upstream state
176
182
  @previous_listener = @last_seen_listener || listener
@@ -215,9 +221,11 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
215
221
  end
216
222
  end
217
223
 
218
- def auto_flush
224
+ def auto_flush(listener = @last_seen_listener)
225
+ return if listener.nil?
226
+
219
227
  flush do |event|
220
- @last_seen_listener.process_event(event)
228
+ listener.process_event(event)
221
229
  end
222
230
  end
223
231
 
@@ -272,11 +280,7 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
272
280
  end # def encode
273
281
 
274
282
  def close
275
- if auto_flush_runner.pending?
276
- #will cancel task if necessary
277
- auto_flush_runner.stop
278
- end
279
- auto_flush
283
+ auto_flush_runner.stop
280
284
  end
281
285
 
282
286
  def auto_flush_active?
@@ -0,0 +1,81 @@
1
+ require "concurrent"
2
+
3
+ module LogStash module Codecs class RetriggerableTask
4
+ SLEEP_FOR = 0.25.freeze
5
+
6
+ attr_reader :thread
7
+
8
+ def initialize(delay, listener)
9
+ @count = calculate_count(delay)
10
+ @listener = listener
11
+ @counter = Concurrent::AtomicFixnum.new(0 + @count)
12
+ @stopped = Concurrent::AtomicBoolean.new(false)
13
+ @semaphore = Concurrent::Semaphore.new(1)
14
+ end
15
+
16
+ def retrigger
17
+ return if stopped?
18
+ if executing?
19
+ @semaphore.acquire
20
+ end
21
+
22
+ if pending?
23
+ reset_counter
24
+ else
25
+ start
26
+ end
27
+ end
28
+
29
+ def close
30
+ @stopped.make_true
31
+ end
32
+
33
+ def counter
34
+ @counter.value
35
+ end
36
+
37
+ def executing?
38
+ running? && counter < 1
39
+ end
40
+
41
+ def pending?
42
+ running? && counter > 0
43
+ end
44
+
45
+ private
46
+
47
+ def calculate_count(value)
48
+ # in multiples of SLEEP_FOR (0.25) seconds
49
+ # if delay is 10 seconds then count is 40
50
+ # this only works when SLEEP_FOR is less than 1
51
+ return 1 if value < SLEEP_FOR
52
+ (value / SLEEP_FOR).floor
53
+ end
54
+
55
+ def reset_counter
56
+ @counter.value = 0 + @count
57
+ end
58
+
59
+ def running?
60
+ @thread && @thread.alive?
61
+ end
62
+
63
+ def start()
64
+ reset_counter
65
+ @thread = Thread.new do
66
+ while counter > 0
67
+ break if stopped?
68
+ sleep SLEEP_FOR
69
+ @counter.decrement
70
+ end
71
+
72
+ @semaphore.drain_permits
73
+ @listener.timeout if !stopped?
74
+ @semaphore.release
75
+ end
76
+ end
77
+
78
+ def stopped?
79
+ @stopped.value
80
+ end
81
+ end end end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-multiline'
4
- s.version = '2.0.7'
4
+ s.version = '2.0.8'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -4,109 +4,102 @@ require "logstash/codecs/multiline"
4
4
  require_relative "../supports/helpers.rb"
5
5
 
6
6
  describe "AutoFlush and AutoFlushUnset" do
7
+ before(:all) do
8
+ @thread_abort = Thread.abort_on_exception
9
+ Thread.abort_on_exception = true
10
+ end
11
+
12
+ after(:all) do
13
+ Thread.abort_on_exception = @thread_abort
14
+ end
15
+
7
16
  let(:flushable) { Mlc::AutoFlushTracer.new }
8
- let(:flush_wait) { 0.1 }
17
+ let(:flush_wait) { 0.25 }
9
18
 
10
19
  describe LogStash::Codecs::AutoFlush do
11
20
  subject { described_class.new(flushable, flush_wait) }
12
21
 
13
22
  context "when initialized" do
14
- it "#pending? is false" do
15
- expect(subject.pending?).to be_falsy
16
- end
17
-
18
23
  it "#stopped? is false" do
19
- expect(subject.stopped?).to be_falsy
20
- end
21
-
22
- it "#finished? is true" do
23
- expect(subject.finished?).to be_truthy
24
+ expect(subject.stopped?).to be_falsey
24
25
  end
25
26
  end
26
27
 
27
- context "when started" do
28
- let(:flush_wait) { 20 }
29
-
28
+ context "when starting once" do
30
29
  before { subject.start }
31
30
  after { subject.stop }
32
31
 
33
- it "#pending? is true" do
34
- expect(subject.pending?).to be_truthy
35
- end
36
-
37
- it "#stopped? is false" do
38
- expect(subject.stopped?).to be_falsy
39
- end
40
-
41
- it "#finished? is false" do
42
- expect(subject.finished?).to be_falsy
32
+ it "calls auto_flush on flushable" do
33
+ sleep flush_wait + 0.5
34
+ expect(flushable.trace_for(:auto_flush)).to be_truthy
43
35
  end
44
36
  end
45
37
 
46
- context "when finished" do
47
- before do
48
- subject.start
49
- sleep flush_wait + 0.1
50
- end
51
-
38
+ context "when starting multiple times" do
39
+ before { subject.start }
52
40
  after { subject.stop }
53
41
 
54
42
  it "calls auto_flush on flushable" do
55
- expect(flushable.trace_for(:auto_flush)).to be_truthy
56
- end
43
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
44
+ sleep 0.1
45
+ subject.start
46
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
47
+ sleep 0.1
48
+ subject.start
49
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
57
50
 
58
- it "#pending? is false" do
59
- expect(subject.pending?).to be_falsy
51
+ sleep flush_wait + 0.5
52
+ expect(flushable.trace_for(:auto_flush)).to be_truthy
60
53
  end
54
+ end
61
55
 
62
- it "#stopped? is false" do
63
- expect(subject.stopped?).to be_falsy
56
+ context "when retriggering during execution" do
57
+ let(:flush_wait) { 1 }
58
+ let(:delay) { 1 }
59
+ before do
60
+ flushable.set_delay(delay)
61
+ subject.start
64
62
  end
63
+ after { subject.stop }
65
64
 
66
- it "#finished? is true" do
67
- expect(subject.finished?).to be_truthy
65
+ it "calls auto_flush twice on flushable" do
66
+ now = Time.now.to_f
67
+ combined_time = flush_wait + delay
68
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
69
+ sleep flush_wait + 0.25 # <---- wait for execution to start
70
+ subject.start # <---- because of the semaphore, this waits for delay = 2 seconds, then starts a new thread
71
+ sleep combined_time + 0.5 # <---- wait for more than 4 seconds = flush_wait + delay
72
+ elapsed1, elapsed2 = flushable.full_trace_for(:delay).map{ |el| (el - now).floor }
73
+ expect(elapsed1).to eq(combined_time)
74
+ expect(elapsed2).to eq(2 * combined_time)
75
+ expect(flushable.full_trace_for(:auto_flush)).to eq([true, true])
68
76
  end
69
77
  end
70
78
 
71
- context "when stopped" do
79
+ context "when stopping before timeout" do
72
80
  before do
73
81
  subject.start
82
+ sleep 0.1
74
83
  subject.stop
75
84
  end
76
85
 
77
86
  it "does not call auto_flush on flushable" do
78
- expect(flushable.trace_for(:auto_flush)).to be_falsy
79
- end
80
-
81
- it "#pending? is false" do
82
- expect(subject.pending?).to be_falsy
87
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
83
88
  end
84
89
 
85
90
  it "#stopped? is true" do
86
91
  expect(subject.stopped?).to be_truthy
87
92
  end
88
-
89
- it "#finished? is false" do
90
- expect(subject.finished?).to be_falsy
91
- end
92
93
  end
93
94
  end
94
95
 
95
96
  describe LogStash::Codecs::AutoFlushUnset do
96
97
  subject { described_class.new(flushable, 2) }
97
98
 
98
- it "#pending? is false" do
99
- expect(subject.pending?).to be_falsy
100
- end
101
-
102
99
  it "#stopped? is true" do
103
100
  expect(subject.stopped?).to be_truthy
104
101
  end
105
102
 
106
- it "#finished? is true" do
107
- expect(subject.finished?).to be_truthy
108
- end
109
-
110
103
  it "#start returns self" do
111
104
  expect(subject.start).to eq(subject)
112
105
  end
@@ -12,6 +12,15 @@ describe LogStash::Codecs::IdentityMapCodec do
12
12
  let(:codec1) { demuxer.codec_without_usage_update(stream1) }
13
13
  let(:arg1) { "data-a" }
14
14
 
15
+ before(:all) do
16
+ @thread_abort = Thread.abort_on_exception
17
+ Thread.abort_on_exception = true
18
+ end
19
+
20
+ after(:all) do
21
+ Thread.abort_on_exception = @thread_abort
22
+ end
23
+
15
24
  after do
16
25
  codec.close
17
26
  end
@@ -206,8 +215,7 @@ describe LogStash::Codecs::IdentityMapCodec do
206
215
  let(:imc) { described_class.new(mlc) }
207
216
 
208
217
  before do
209
- listener = Mlc::LineListener.new(queue, imc, identity)
210
- listener.accept("foo")
218
+ l_0 = Mlc::LineListener.new(queue, imc, identity).tap {|o| o.accept("foo")}
211
219
  end
212
220
 
213
221
  describe "normal processing" do
@@ -221,11 +229,23 @@ describe LogStash::Codecs::IdentityMapCodec do
221
229
 
222
230
  context "when wrapped codec has auto-flush activated" do
223
231
  let(:config) { {"pattern" => "^\\s", "what" => "previous", "auto_flush_interval" => 0.2} }
224
- it "one event is generated" do
225
- sleep 0.4
226
- expect(queue.size).to eq(1)
227
- expect(queue[0]["message"]).to eq("foo")
228
- expect(imc.identity_count).to eq(1)
232
+ before do
233
+ l_1 = Mlc::LineListener.new(queue, imc, "stream2").tap {|o| o.accept("bar")}
234
+ l_2 = Mlc::LineListener.new(queue, imc, "stream3").tap {|o| o.accept("baz")}
235
+ end
236
+ it "three events are auto flushed from three different identities/codecs" do
237
+ sleep 0.6 # wait for auto_flush - in multiples of 0.5 plus 0.1
238
+ expect(queue.size).to eq(3)
239
+ e1, e2, e3 = queue
240
+ expect(e1["path"]).to eq("stream1")
241
+ expect(e1["message"]).to eq("foo")
242
+
243
+ expect(e2["path"]).to eq("stream2")
244
+ expect(e2["message"]).to eq("bar")
245
+
246
+ expect(e3["path"]).to eq("stream3")
247
+ expect(e3["message"]).to eq("baz")
248
+ expect(imc.identity_count).to eq(3)
229
249
  end
230
250
  end
231
251
  end
@@ -225,7 +225,7 @@ describe LogStash::Codecs::Multiline do
225
225
  "de.log" => ["Hallo Welt"] }
226
226
  end
227
227
  let(:listener_class) { Mlc::LineListener }
228
- let(:auto_flush_interval) { 0.5 }
228
+ let(:auto_flush_interval) { 2 }
229
229
 
230
230
  let(:line_producer) do
231
231
  lambda do |path|
@@ -73,6 +73,10 @@ module Mlc
73
73
  params.nil? ? false : params.last
74
74
  end
75
75
 
76
+ def full_trace_for(symbol)
77
+ @tracer.select{|array| array[0] == symbol}.map(&:last)
78
+ end
79
+
76
80
  def clear()
77
81
  @tracer.clear()
78
82
  end
@@ -91,7 +95,16 @@ module Mlc
91
95
  end
92
96
 
93
97
  class AutoFlushTracer < TracerBase
94
- def auto_flush() @tracer.push [:auto_flush, true]; end
98
+ def auto_flush() simulate_execution_delay; @tracer.push [:auto_flush, true]; end
99
+ def set_delay(delay)
100
+ @delay = delay
101
+ end
102
+
103
+ def simulate_execution_delay
104
+ return if @delay.nil? || @delay.zero?
105
+ sleep @delay
106
+ @tracer.push [:delay, Time.now.to_f]
107
+ end
95
108
  end
96
109
 
97
110
  class IdentityMapCodecTracer < TracerBase
metadata CHANGED
@@ -1,77 +1,77 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-multiline
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.7
4
+ version: 2.0.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-27 00:00:00.000000000 Z
11
+ date: 2016-02-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
- - - ">="
17
+ - - '>='
17
18
  - !ruby/object:Gem::Version
18
19
  version: 2.0.0
19
- - - "<"
20
+ - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 3.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
- - - ">="
25
+ - - '>='
28
26
  - !ruby/object:Gem::Version
29
27
  version: 2.0.0
30
- - - "<"
28
+ - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 3.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-patterns-core
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
- - - ">="
42
+ - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-patterns-core
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: jls-grok
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
- - - ">="
51
+ - - ~>
45
52
  - !ruby/object:Gem::Version
46
- version: '0'
47
- - !ruby/object:Gem::Dependency
53
+ version: 0.11.1
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
- - - "~>"
56
+ - - ~>
51
57
  - !ruby/object:Gem::Version
52
58
  version: 0.11.1
53
- name: jls-grok
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
- - - "~>"
65
+ - - '>='
59
66
  - !ruby/object:Gem::Version
60
- version: 0.11.1
61
- - !ruby/object:Gem::Dependency
67
+ version: '0'
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
- - - ">="
70
+ - - '>='
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0'
67
- name: logstash-devutils
68
73
  prerelease: false
69
74
  type: :development
70
- version_requirements: !ruby/object:Gem::Requirement
71
- requirements:
72
- - - ">="
73
- - !ruby/object:Gem::Version
74
- version: '0'
75
75
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
76
76
  email: info@elastic.co
77
77
  executables: []
@@ -87,6 +87,7 @@ files:
87
87
  - lib/logstash/codecs/auto_flush.rb
88
88
  - lib/logstash/codecs/identity_map_codec.rb
89
89
  - lib/logstash/codecs/multiline.rb
90
+ - lib/logstash/codecs/retriggerable_task.rb
90
91
  - logstash-codec-multiline.gemspec
91
92
  - spec/codecs/auto_flush_spec.rb
92
93
  - spec/codecs/identity_map_codec_spec.rb
@@ -104,12 +105,12 @@ require_paths:
104
105
  - lib
105
106
  required_ruby_version: !ruby/object:Gem::Requirement
106
107
  requirements:
107
- - - ">="
108
+ - - '>='
108
109
  - !ruby/object:Gem::Version
109
110
  version: '0'
110
111
  required_rubygems_version: !ruby/object:Gem::Requirement
111
112
  requirements:
112
- - - ">="
113
+ - - '>='
113
114
  - !ruby/object:Gem::Version
114
115
  version: '0'
115
116
  requirements: []