logstash-codec-multiline 2.0.7 → 2.0.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 36df6112f2d0e3f3d6b3d0481a72e5250fae5352
4
- data.tar.gz: b86e58bbb8f62680e7ab478adc0d75a3d61d6530
3
+ metadata.gz: f9e0a15839b1bbabcad88ee95445b72ed589a4dd
4
+ data.tar.gz: bdf59bcab8b5ad4fc35906de033bde7ed658e52a
5
5
  SHA512:
6
- metadata.gz: afe492266d2d77c9946ff76ad71e755f8d03a655222f92673f70d0804b99543edd82dbb3365674f1f640036eb24cb42c6f3b65d6bea290378e959de6f6356bfe
7
- data.tar.gz: e053998e3802437a06a82b5997f28c8bc4983b80c936a1d9bbbe3b81a4a99be11f6086dac0f01a73c69fb2770728ee6ef030f807714c226a7e570815d11c49d2
6
+ metadata.gz: 5e0bcea055b8af6ebbd45300e2901146a2770b7937de6ca8b5f08f4a01b347cc7b876d3b675589c475b3b41f4ce65632e7edc44432a2f00b2abc5f9b7dc5d899
7
+ data.tar.gz: 6333ad636771cc1e9407f9642f242ff40f59181b4073fe55e2b3aa93b9cd675ca435961aabc7f054c4cad6be072bdefc42395198b14cafb3ba7b286474e888ab
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 2.0.8
2
+ - Fix for issue #28, Memory leak when using cancel + execute on ScheduledTask.
3
+
1
4
  ## 2.0.7
2
5
  - Fix for issue #23, Thread deadlock on auto-flush when using multiple inputs with this codec. Use cancel + execute on ScheduledTask instead of reset.
3
6
 
data/Gemfile CHANGED
@@ -1,3 +1,2 @@
1
1
  source 'https://rubygems.org'
2
- gemspec
3
- gem "concurrent-ruby", "0.9.1"
2
+ gemspec
@@ -1,74 +1,32 @@
1
1
  # encoding: utf-8
2
2
  require "concurrent"
3
+ require "logstash/codecs/retriggerable_task"
3
4
 
4
5
  module LogStash module Codecs class AutoFlush
5
6
  def initialize(mc, interval)
6
7
  @mc, @interval = mc, interval
7
8
  @stopped = Concurrent::AtomicBoolean.new # false by default
9
+ @task = RetriggerableTask.new(@interval, self)
8
10
  end
9
11
 
10
- # def start
11
- # # can't start if pipeline is stopping
12
- # return self if stopped?
13
- # if pending?
14
- # @task.cancel
15
- # create_task
16
- # elsif finished?
17
- # create_task
18
- # # else the task is executing
19
- # end
20
- # self
21
- # end
12
+ def timeout
13
+ @mc.auto_flush
14
+ end
22
15
 
23
16
  def start
24
17
  # can't start if pipeline is stopping
25
18
  return self if stopped?
26
-
27
- if pending? && @task.cancel
28
- create_task
29
- return self
30
- end
31
- # maybe we have a timing edge case
32
- # where pending? was true but cancel failed
33
- # because the task started running
34
- if finished?
35
- create_task
36
- return self
37
- end
38
- # else the task is executing
39
- # wait for task to complete
40
- # flush could feasibly block on queue access
41
- @task.value
42
- create_task
19
+ @task.retrigger
43
20
  self
44
21
  end
45
22
 
46
- def create_task
47
- @task = Concurrent::ScheduledTask.execute(@interval) do
48
- @mc.auto_flush()
49
- end
50
- end
51
-
52
- def finished?
53
- return true if @task.nil?
54
- @task.fulfilled?
55
- end
56
-
57
- def pending?
58
- @task && @task.pending?
59
- end
60
-
61
23
  def stopped?
62
24
  @stopped.value
63
25
  end
64
26
 
65
27
  def stop
66
28
  @stopped.make_true
67
- cancel
68
- end
69
-
70
- def cancel
71
- @task.cancel if pending?
29
+ @task.close
72
30
  end
73
31
  end
74
32
 
@@ -76,10 +34,6 @@ class AutoFlushUnset
76
34
  def initialize(mc, interval)
77
35
  end
78
36
 
79
- def pending?
80
- false
81
- end
82
-
83
37
  def stopped?
84
38
  true
85
39
  end
@@ -88,15 +42,7 @@ class AutoFlushUnset
88
42
  self
89
43
  end
90
44
 
91
- def finished?
92
- true
93
- end
94
-
95
45
  def stop
96
46
  self
97
47
  end
98
-
99
- def cancel
100
- self
101
- end
102
48
  end end end
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/namespace"
3
3
  require "thread_safe"
4
+ require "concurrent"
4
5
 
5
6
  # This class is a Codec duck type
6
7
  # Using Composition, it maps from a stream identity to
@@ -37,41 +38,51 @@ module LogStash module Codecs class IdentityMapCodec
37
38
  end
38
39
  end
39
40
 
40
- class MapCleaner
41
- def initialize(imc, interval)
42
- @imc, @interval = imc, interval
43
- @running = false
41
+ class PeriodicRunner
42
+ def initialize(listener, interval, method_symbol)
43
+ @listener, @interval = listener, interval
44
+ @method_symbol = method_symbol
45
+ @running = Concurrent::AtomicBoolean.new(false)
44
46
  end
45
47
 
46
48
  def start
47
49
  return self if running?
48
- @running = true
49
- @thread = Thread.new(@imc) do |imc|
50
- loop do
50
+ @running.make_true
51
+ @thread = Thread.new() do
52
+ while running? do
51
53
  sleep @interval
52
- break if !@running
53
- imc.map_cleanup
54
+ break if !running?
55
+ @listener.send(@method_symbol)
54
56
  end
55
57
  end
56
58
  self
57
59
  end
58
60
 
59
61
  def running?
60
- @running
62
+ @running.value
61
63
  end
62
64
 
63
65
  def stop
64
66
  return if !running?
65
- @running = false
66
- @thread.wakeup if @thread.alive?
67
+ @running.make_false
68
+ if @thread.alive?
69
+ @thread.wakeup
70
+ @thread.join
71
+ end
72
+ @listener = nil
67
73
  end
68
74
  end
69
75
 
70
- # A composite class to hold both the codec and the eviction_timeout
76
+ class NoopRunner
77
+ attr_reader :start, :stop
78
+ def running?() false; end
79
+ end
80
+
81
+ # A composite class to hold both the codec, the eviction_timeout and a last_used timestamp
71
82
  # instances of this Value Object are stored in the mapping hash
72
83
  class CodecValue
73
84
  attr_reader :codec
74
- attr_accessor :timeout
85
+ attr_accessor :eviction_timeout, :auto_flush_timeout
75
86
 
76
87
  def initialize(codec)
77
88
  @codec = codec
@@ -92,7 +103,7 @@ module LogStash module Codecs class IdentityMapCodec
92
103
  CLEANER_INTERVAL = 60 * 5 # 5 minutes
93
104
 
94
105
  attr_reader :identity_map
95
- attr_accessor :base_codec, :cleaner
106
+ attr_accessor :base_codec, :cleaner, :auto_flusher
96
107
 
97
108
  def initialize(codec)
98
109
  @base_codec = codec
@@ -100,7 +111,14 @@ module LogStash module Codecs class IdentityMapCodec
100
111
  @identity_map = ThreadSafe::Hash.new &method(:codec_builder)
101
112
  @max_identities = MAX_IDENTITIES
102
113
  @evict_timeout = EVICT_TIMEOUT
103
- @cleaner = MapCleaner.new(self, CLEANER_INTERVAL)
114
+ cleaner_interval(CLEANER_INTERVAL)
115
+ if codec.respond_to?(:use_mapper_auto_flush) &&
116
+ (@auto_flush_interval = codec.use_mapper_auto_flush)
117
+ @auto_flusher = PeriodicRunner.new(self, 0.5, :auto_flush_mapped)
118
+ else
119
+ @auto_flusher = NoopRunner.new
120
+ end
121
+
104
122
  @decode_block = lambda {|*| true }
105
123
  @eviction_block = nil
106
124
  end
@@ -123,8 +141,8 @@ module LogStash module Codecs class IdentityMapCodec
123
141
 
124
142
  # used to add a non-default cleaner interval
125
143
  def cleaner_interval(interval)
126
- @cleaner.stop
127
- @cleaner = MapCleaner.new(self, interval.to_i)
144
+ @cleaner.stop if @cleaner
145
+ @cleaner = PeriodicRunner.new(self, interval.to_i, :map_cleanup)
128
146
  self
129
147
  end
130
148
 
@@ -133,6 +151,7 @@ module LogStash module Codecs class IdentityMapCodec
133
151
  @eviction_block = block
134
152
  self
135
153
  end
154
+
136
155
  # end Constructional/builder methods
137
156
  # ==============================================
138
157
 
@@ -182,11 +201,34 @@ module LogStash module Codecs class IdentityMapCodec
182
201
 
183
202
  def close()
184
203
  cleaner.stop
204
+ auto_flusher.stop
185
205
  all_codecs.each(&:close)
186
206
  end
187
207
  # end Codec API
188
208
  # ==============================================
189
209
 
210
+ def auto_flush_mapped
211
+ if !identity_count.zero?
212
+ nowf = Time.now.to_f
213
+ identity_map.each do |identity, compo|
214
+ next if compo.auto_flush_timeout.zero?
215
+ next unless nowf > compo.auto_flush_timeout
216
+ compo.codec.auto_flush
217
+ # at eof (tail and read) no more lines for a while or ever
218
+ # so reset compo.auto_flush_timeout
219
+ compo.auto_flush_timeout = 0
220
+ end
221
+ end
222
+ end
223
+
224
+ def flush_mapped(listener)
225
+ listener_has_path = listener.respond_to?(:path)
226
+ identity_map.each do |identity, compo|
227
+ listener.path = identity if listener_has_path
228
+ compo.codec.auto_flush(listener)
229
+ end
230
+ end
231
+
190
232
  def all_codecs
191
233
  no_streams? ? @base_codecs : identity_map.values.map(&:codec)
192
234
  end
@@ -204,14 +246,16 @@ module LogStash module Codecs class IdentityMapCodec
204
246
  # been accessed in the last @evict_timeout
205
247
  # period (default 1 hour)
206
248
  def map_cleanup
207
- cut_off = Time.now.to_i
208
- # delete_if is atomic
209
- # contents should not mutate during this call
210
- identity_map.delete_if do |identity, compo|
211
- if (flag = compo.timeout <= cut_off)
212
- evict_flush(compo.codec)
249
+ if !identity_count.zero?
250
+ nowi = Time.now.to_i
251
+ # delete_if is atomic
252
+ # contents should not mutate during this call
253
+ identity_map.delete_if do |identity, compo|
254
+ if (flag = compo.eviction_timeout <= nowi)
255
+ evict_flush(compo.codec)
256
+ end
257
+ flag
213
258
  end
214
- flag
215
259
  end
216
260
  current_size_and_limit
217
261
  end
@@ -241,7 +285,7 @@ module LogStash module Codecs class IdentityMapCodec
241
285
  end
242
286
 
243
287
  def eviction_timestamp_for(identity)
244
- find_codec_value(identity).timeout
288
+ find_codec_value(identity).eviction_timeout
245
289
  end
246
290
 
247
291
  private
@@ -261,13 +305,20 @@ module LogStash module Codecs class IdentityMapCodec
261
305
  # only start the cleaner if streams are in use
262
306
  # continuous calls to start are OK
263
307
  cleaner.start
308
+ auto_flusher.start
264
309
  compo = find_codec_value(identity)
265
- compo.timeout = eviction_timestamp
310
+ now = Time.now
311
+ compo.eviction_timeout = eviction_timestamp(now)
312
+ compo.auto_flush_timeout = auto_flush_timestamp(now)
266
313
  compo.codec
267
314
  end
268
315
 
269
- def eviction_timestamp
270
- Time.now.to_i + @evict_timeout
316
+ def auto_flush_timestamp(now = Time.now)
317
+ now.to_f + @auto_flush_interval.to_f
318
+ end
319
+
320
+ def eviction_timestamp(now = Time.now)
321
+ now.to_i + @evict_timeout
271
322
  end
272
323
 
273
324
  def check_map_limits
@@ -277,6 +328,7 @@ module LogStash module Codecs class IdentityMapCodec
277
328
 
278
329
  def codec_builder(hash, k)
279
330
  codec = hash.empty? ? @base_codec : @base_codec.clone
331
+ codec.use_mapper_auto_flush if using_mapped_auto_flush?
280
332
  compo = CodecValue.new(codec)
281
333
  hash.store(k, compo)
282
334
  end
@@ -284,4 +336,8 @@ module LogStash module Codecs class IdentityMapCodec
284
336
  def no_streams?
285
337
  identity_map.empty?
286
338
  end
339
+
340
+ def using_mapped_auto_flush?
341
+ !@auto_flush_interval.nil?
342
+ end
287
343
  end end end
@@ -171,6 +171,12 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
171
171
  end
172
172
  end # def register
173
173
 
174
+ def use_mapper_auto_flush
175
+ return unless auto_flush_active?
176
+ @auto_flush_runner = AutoFlushUnset.new(nil, nil)
177
+ @auto_flush_interval = @auto_flush_interval.to_f
178
+ end
179
+
174
180
  def accept(listener)
175
181
  # memoize references to listener that holds upstream state
176
182
  @previous_listener = @last_seen_listener || listener
@@ -215,9 +221,11 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
215
221
  end
216
222
  end
217
223
 
218
- def auto_flush
224
+ def auto_flush(listener = @last_seen_listener)
225
+ return if listener.nil?
226
+
219
227
  flush do |event|
220
- @last_seen_listener.process_event(event)
228
+ listener.process_event(event)
221
229
  end
222
230
  end
223
231
 
@@ -272,11 +280,7 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
272
280
  end # def encode
273
281
 
274
282
  def close
275
- if auto_flush_runner.pending?
276
- #will cancel task if necessary
277
- auto_flush_runner.stop
278
- end
279
- auto_flush
283
+ auto_flush_runner.stop
280
284
  end
281
285
 
282
286
  def auto_flush_active?
@@ -0,0 +1,81 @@
1
+ require "concurrent"
2
+
3
+ module LogStash module Codecs class RetriggerableTask
4
+ SLEEP_FOR = 0.25.freeze
5
+
6
+ attr_reader :thread
7
+
8
+ def initialize(delay, listener)
9
+ @count = calculate_count(delay)
10
+ @listener = listener
11
+ @counter = Concurrent::AtomicFixnum.new(0 + @count)
12
+ @stopped = Concurrent::AtomicBoolean.new(false)
13
+ @semaphore = Concurrent::Semaphore.new(1)
14
+ end
15
+
16
+ def retrigger
17
+ return if stopped?
18
+ if executing?
19
+ @semaphore.acquire
20
+ end
21
+
22
+ if pending?
23
+ reset_counter
24
+ else
25
+ start
26
+ end
27
+ end
28
+
29
+ def close
30
+ @stopped.make_true
31
+ end
32
+
33
+ def counter
34
+ @counter.value
35
+ end
36
+
37
+ def executing?
38
+ running? && counter < 1
39
+ end
40
+
41
+ def pending?
42
+ running? && counter > 0
43
+ end
44
+
45
+ private
46
+
47
+ def calculate_count(value)
48
+ # in multiples of SLEEP_FOR (0.25) seconds
49
+ # if delay is 10 seconds then count is 40
50
+ # this only works when SLEEP_FOR is less than 1
51
+ return 1 if value < SLEEP_FOR
52
+ (value / SLEEP_FOR).floor
53
+ end
54
+
55
+ def reset_counter
56
+ @counter.value = 0 + @count
57
+ end
58
+
59
+ def running?
60
+ @thread && @thread.alive?
61
+ end
62
+
63
+ def start()
64
+ reset_counter
65
+ @thread = Thread.new do
66
+ while counter > 0
67
+ break if stopped?
68
+ sleep SLEEP_FOR
69
+ @counter.decrement
70
+ end
71
+
72
+ @semaphore.drain_permits
73
+ @listener.timeout if !stopped?
74
+ @semaphore.release
75
+ end
76
+ end
77
+
78
+ def stopped?
79
+ @stopped.value
80
+ end
81
+ end end end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-multiline'
4
- s.version = '2.0.7'
4
+ s.version = '2.0.8'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -4,109 +4,102 @@ require "logstash/codecs/multiline"
4
4
  require_relative "../supports/helpers.rb"
5
5
 
6
6
  describe "AutoFlush and AutoFlushUnset" do
7
+ before(:all) do
8
+ @thread_abort = Thread.abort_on_exception
9
+ Thread.abort_on_exception = true
10
+ end
11
+
12
+ after(:all) do
13
+ Thread.abort_on_exception = @thread_abort
14
+ end
15
+
7
16
  let(:flushable) { Mlc::AutoFlushTracer.new }
8
- let(:flush_wait) { 0.1 }
17
+ let(:flush_wait) { 0.25 }
9
18
 
10
19
  describe LogStash::Codecs::AutoFlush do
11
20
  subject { described_class.new(flushable, flush_wait) }
12
21
 
13
22
  context "when initialized" do
14
- it "#pending? is false" do
15
- expect(subject.pending?).to be_falsy
16
- end
17
-
18
23
  it "#stopped? is false" do
19
- expect(subject.stopped?).to be_falsy
20
- end
21
-
22
- it "#finished? is true" do
23
- expect(subject.finished?).to be_truthy
24
+ expect(subject.stopped?).to be_falsey
24
25
  end
25
26
  end
26
27
 
27
- context "when started" do
28
- let(:flush_wait) { 20 }
29
-
28
+ context "when starting once" do
30
29
  before { subject.start }
31
30
  after { subject.stop }
32
31
 
33
- it "#pending? is true" do
34
- expect(subject.pending?).to be_truthy
35
- end
36
-
37
- it "#stopped? is false" do
38
- expect(subject.stopped?).to be_falsy
39
- end
40
-
41
- it "#finished? is false" do
42
- expect(subject.finished?).to be_falsy
32
+ it "calls auto_flush on flushable" do
33
+ sleep flush_wait + 0.5
34
+ expect(flushable.trace_for(:auto_flush)).to be_truthy
43
35
  end
44
36
  end
45
37
 
46
- context "when finished" do
47
- before do
48
- subject.start
49
- sleep flush_wait + 0.1
50
- end
51
-
38
+ context "when starting multiple times" do
39
+ before { subject.start }
52
40
  after { subject.stop }
53
41
 
54
42
  it "calls auto_flush on flushable" do
55
- expect(flushable.trace_for(:auto_flush)).to be_truthy
56
- end
43
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
44
+ sleep 0.1
45
+ subject.start
46
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
47
+ sleep 0.1
48
+ subject.start
49
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
57
50
 
58
- it "#pending? is false" do
59
- expect(subject.pending?).to be_falsy
51
+ sleep flush_wait + 0.5
52
+ expect(flushable.trace_for(:auto_flush)).to be_truthy
60
53
  end
54
+ end
61
55
 
62
- it "#stopped? is false" do
63
- expect(subject.stopped?).to be_falsy
56
+ context "when retriggering during execution" do
57
+ let(:flush_wait) { 1 }
58
+ let(:delay) { 1 }
59
+ before do
60
+ flushable.set_delay(delay)
61
+ subject.start
64
62
  end
63
+ after { subject.stop }
65
64
 
66
- it "#finished? is true" do
67
- expect(subject.finished?).to be_truthy
65
+ it "calls auto_flush twice on flushable" do
66
+ now = Time.now.to_f
67
+ combined_time = flush_wait + delay
68
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
69
+ sleep flush_wait + 0.25 # <---- wait for execution to start
70
+ subject.start # <---- because of the semaphore, this waits for delay = 2 seconds, then starts a new thread
71
+ sleep combined_time + 0.5 # <---- wait for more than 4 seconds = flush_wait + delay
72
+ elapsed1, elapsed2 = flushable.full_trace_for(:delay).map{ |el| (el - now).floor }
73
+ expect(elapsed1).to eq(combined_time)
74
+ expect(elapsed2).to eq(2 * combined_time)
75
+ expect(flushable.full_trace_for(:auto_flush)).to eq([true, true])
68
76
  end
69
77
  end
70
78
 
71
- context "when stopped" do
79
+ context "when stopping before timeout" do
72
80
  before do
73
81
  subject.start
82
+ sleep 0.1
74
83
  subject.stop
75
84
  end
76
85
 
77
86
  it "does not call auto_flush on flushable" do
78
- expect(flushable.trace_for(:auto_flush)).to be_falsy
79
- end
80
-
81
- it "#pending? is false" do
82
- expect(subject.pending?).to be_falsy
87
+ expect(flushable.trace_for(:auto_flush)).to be_falsey
83
88
  end
84
89
 
85
90
  it "#stopped? is true" do
86
91
  expect(subject.stopped?).to be_truthy
87
92
  end
88
-
89
- it "#finished? is false" do
90
- expect(subject.finished?).to be_falsy
91
- end
92
93
  end
93
94
  end
94
95
 
95
96
  describe LogStash::Codecs::AutoFlushUnset do
96
97
  subject { described_class.new(flushable, 2) }
97
98
 
98
- it "#pending? is false" do
99
- expect(subject.pending?).to be_falsy
100
- end
101
-
102
99
  it "#stopped? is true" do
103
100
  expect(subject.stopped?).to be_truthy
104
101
  end
105
102
 
106
- it "#finished? is true" do
107
- expect(subject.finished?).to be_truthy
108
- end
109
-
110
103
  it "#start returns self" do
111
104
  expect(subject.start).to eq(subject)
112
105
  end
@@ -12,6 +12,15 @@ describe LogStash::Codecs::IdentityMapCodec do
12
12
  let(:codec1) { demuxer.codec_without_usage_update(stream1) }
13
13
  let(:arg1) { "data-a" }
14
14
 
15
+ before(:all) do
16
+ @thread_abort = Thread.abort_on_exception
17
+ Thread.abort_on_exception = true
18
+ end
19
+
20
+ after(:all) do
21
+ Thread.abort_on_exception = @thread_abort
22
+ end
23
+
15
24
  after do
16
25
  codec.close
17
26
  end
@@ -206,8 +215,7 @@ describe LogStash::Codecs::IdentityMapCodec do
206
215
  let(:imc) { described_class.new(mlc) }
207
216
 
208
217
  before do
209
- listener = Mlc::LineListener.new(queue, imc, identity)
210
- listener.accept("foo")
218
+ l_0 = Mlc::LineListener.new(queue, imc, identity).tap {|o| o.accept("foo")}
211
219
  end
212
220
 
213
221
  describe "normal processing" do
@@ -221,11 +229,23 @@ describe LogStash::Codecs::IdentityMapCodec do
221
229
 
222
230
  context "when wrapped codec has auto-flush activated" do
223
231
  let(:config) { {"pattern" => "^\\s", "what" => "previous", "auto_flush_interval" => 0.2} }
224
- it "one event is generated" do
225
- sleep 0.4
226
- expect(queue.size).to eq(1)
227
- expect(queue[0]["message"]).to eq("foo")
228
- expect(imc.identity_count).to eq(1)
232
+ before do
233
+ l_1 = Mlc::LineListener.new(queue, imc, "stream2").tap {|o| o.accept("bar")}
234
+ l_2 = Mlc::LineListener.new(queue, imc, "stream3").tap {|o| o.accept("baz")}
235
+ end
236
+ it "three events are auto flushed from three different identities/codecs" do
237
+ sleep 0.6 # wait for auto_flush - in multiples of 0.5 plus 0.1
238
+ expect(queue.size).to eq(3)
239
+ e1, e2, e3 = queue
240
+ expect(e1["path"]).to eq("stream1")
241
+ expect(e1["message"]).to eq("foo")
242
+
243
+ expect(e2["path"]).to eq("stream2")
244
+ expect(e2["message"]).to eq("bar")
245
+
246
+ expect(e3["path"]).to eq("stream3")
247
+ expect(e3["message"]).to eq("baz")
248
+ expect(imc.identity_count).to eq(3)
229
249
  end
230
250
  end
231
251
  end
@@ -225,7 +225,7 @@ describe LogStash::Codecs::Multiline do
225
225
  "de.log" => ["Hallo Welt"] }
226
226
  end
227
227
  let(:listener_class) { Mlc::LineListener }
228
- let(:auto_flush_interval) { 0.5 }
228
+ let(:auto_flush_interval) { 2 }
229
229
 
230
230
  let(:line_producer) do
231
231
  lambda do |path|
@@ -73,6 +73,10 @@ module Mlc
73
73
  params.nil? ? false : params.last
74
74
  end
75
75
 
76
+ def full_trace_for(symbol)
77
+ @tracer.select{|array| array[0] == symbol}.map(&:last)
78
+ end
79
+
76
80
  def clear()
77
81
  @tracer.clear()
78
82
  end
@@ -91,7 +95,16 @@ module Mlc
91
95
  end
92
96
 
93
97
  class AutoFlushTracer < TracerBase
94
- def auto_flush() @tracer.push [:auto_flush, true]; end
98
+ def auto_flush() simulate_execution_delay; @tracer.push [:auto_flush, true]; end
99
+ def set_delay(delay)
100
+ @delay = delay
101
+ end
102
+
103
+ def simulate_execution_delay
104
+ return if @delay.nil? || @delay.zero?
105
+ sleep @delay
106
+ @tracer.push [:delay, Time.now.to_f]
107
+ end
95
108
  end
96
109
 
97
110
  class IdentityMapCodecTracer < TracerBase
metadata CHANGED
@@ -1,77 +1,77 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-multiline
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.7
4
+ version: 2.0.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-27 00:00:00.000000000 Z
11
+ date: 2016-02-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
- - - ">="
17
+ - - '>='
17
18
  - !ruby/object:Gem::Version
18
19
  version: 2.0.0
19
- - - "<"
20
+ - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 3.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
- - - ">="
25
+ - - '>='
28
26
  - !ruby/object:Gem::Version
29
27
  version: 2.0.0
30
- - - "<"
28
+ - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 3.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-patterns-core
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
- - - ">="
42
+ - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-patterns-core
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: jls-grok
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
- - - ">="
51
+ - - ~>
45
52
  - !ruby/object:Gem::Version
46
- version: '0'
47
- - !ruby/object:Gem::Dependency
53
+ version: 0.11.1
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
- - - "~>"
56
+ - - ~>
51
57
  - !ruby/object:Gem::Version
52
58
  version: 0.11.1
53
- name: jls-grok
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
- - - "~>"
65
+ - - '>='
59
66
  - !ruby/object:Gem::Version
60
- version: 0.11.1
61
- - !ruby/object:Gem::Dependency
67
+ version: '0'
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
- - - ">="
70
+ - - '>='
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0'
67
- name: logstash-devutils
68
73
  prerelease: false
69
74
  type: :development
70
- version_requirements: !ruby/object:Gem::Requirement
71
- requirements:
72
- - - ">="
73
- - !ruby/object:Gem::Version
74
- version: '0'
75
75
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
76
76
  email: info@elastic.co
77
77
  executables: []
@@ -87,6 +87,7 @@ files:
87
87
  - lib/logstash/codecs/auto_flush.rb
88
88
  - lib/logstash/codecs/identity_map_codec.rb
89
89
  - lib/logstash/codecs/multiline.rb
90
+ - lib/logstash/codecs/retriggerable_task.rb
90
91
  - logstash-codec-multiline.gemspec
91
92
  - spec/codecs/auto_flush_spec.rb
92
93
  - spec/codecs/identity_map_codec_spec.rb
@@ -104,12 +105,12 @@ require_paths:
104
105
  - lib
105
106
  required_ruby_version: !ruby/object:Gem::Requirement
106
107
  requirements:
107
- - - ">="
108
+ - - '>='
108
109
  - !ruby/object:Gem::Version
109
110
  version: '0'
110
111
  required_rubygems_version: !ruby/object:Gem::Requirement
111
112
  requirements:
112
- - - ">="
113
+ - - '>='
113
114
  - !ruby/object:Gem::Version
114
115
  version: '0'
115
116
  requirements: []