logstash-codec-multiline 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd43c9171bdc01db5f0d381b4c95994e7b6073d4
-  data.tar.gz: 07230a73ec9e7cb9ccddb94921d540323e4d0e86
+  metadata.gz: ad1702d90e9bf8fb69ffd334989def88c34b8afa
+  data.tar.gz: dc3b95f3bac523a85107eeb5c0142f6687a09b25
 SHA512:
-  metadata.gz: 2aec937d4fdf0bbde23c38dee5544198ee8bb5c89efe146bf96b24d4e77bf0f004ba197508168e37f700bd8ebcd913a4719a766b7ce7c7c50ce526b9f9d83623
-  data.tar.gz: 3174e737bc7922cbafec8cda5f1ee85ec9c1691e6918e5e61ab20da35167b6c9969147a4ff869e52057a8385190edd2058699fbc636bf5e69f3b7ccba8a60983
+  metadata.gz: f29daf9944841236fd22a7c4dfe80a3fa7f4ba3efd3e06809cc35d8b8345b7e691aec1695413bcbc1a4f288feceb923ed4798ff1eec8f5ea710b29fbd97915f5
+  data.tar.gz: 3ba39d672418818e50deafc4f24c9d6ca9196eada5b88ca884c406eafcfdb0e4d6c9650418071623d2a229e675346ac8d0f8205f902a6b298c3f8e70e727ab2c

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.5
+ - Add auto_flush config option, with no default. If not set, no auto_flush is done.
+ - Add evict method to identity_map_codec that allows for an input, when done with an identity, to auto_flush and remove the identity from the map.
+
 ## 2.0.4
  - Add constructional method to allow an eviction specific block to be set.
 

data/lib/logstash/codecs/auto_flush.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require "concurrent"
+
+module LogStash module Codecs class AutoFlush
+  def initialize(mc, interval)
+    @mc, @interval = mc, interval
+    @stopped = Concurrent::AtomicBoolean.new # false by default
+  end
+
+  def start
+    # can't start if pipeline is stopping
+    return self if stopped?
+    if pending?
+      @task.reset
+    elsif finished?
+      @task = Concurrent::ScheduledTask.execute(@interval) do
+        @mc.auto_flush()
+      end
+    # else the task is executing
+    end
+    self
+  end
+
+  def finished?
+    return true if @task.nil?
+    @task.fulfilled?
+  end
+
+  def pending?
+    @task && @task.pending?
+  end
+
+  def stopped?
+    @stopped.value
+  end
+
+  def stop
+    @stopped.make_true
+    @task.cancel if pending?
+  end
+end
+
+class AutoFlushUnset
+  def initialize(mc, interval)
+  end
+
+  def pending?
+    false
+  end
+
+  def stopped?
+    true
+  end
+
+  def start
+    self
+  end
+
+  def finished?
+    true
+  end
+
+  def stop
+    self
+  end
+end end end

data/lib/logstash/codecs/identity_map_codec.rb

@@ -63,7 +63,7 @@ module LogStash module Codecs class IdentityMapCodec
     def stop
       return if !running?
       @running = false
-      @thread.wakeup
+      @thread.wakeup if @thread.alive?
     end
   end
 
@@ -136,6 +136,17 @@ module LogStash module Codecs class IdentityMapCodec
   # end Constructional/builder methods
   # ==============================================
 
+  # ==============================================
+  # IdentityMapCodec API
+  def evict(identity)
+    # maybe called more than once
+    if (compo = identity_map.delete(identity))
+      compo.codec.auto_flush if compo.codec.respond_to?(:auto_flush)
+    end
+  end
+  # end IdentityMapCodec API
+  # ==============================================
+
   # ==============================================
   # Codec API
   def decode(data, identity = nil, &block)
@@ -143,22 +154,29 @@ module LogStash module Codecs class IdentityMapCodec
     stream_codec(identity).decode(data, &block)
   end
 
+  def accept(listener)
+    stream_codec(listener.path).accept(listener)
+  end
+
   alias_method :<<, :decode
 
   def encode(event, identity = nil)
     stream_codec(identity).encode(event)
   end
 
-  # this method will not be called from
-  # the input or the pipeline unless
-  # we implement codec flush on shutdown
-  # problematic, because we may not have
-  # received all the multiline parts yet.
-  # but if we don't flush we will lose data
   def flush(&block)
     all_codecs.each do |codec|
       #let ruby do its default args thing
-      block.nil? ? codec.flush : codec.flush(&block)
+      if block_given?
+        codec.flush(&block)
+      else
+        if codec.respond_to?(:auto_flush)
+          codec.auto_flush
+        else
+          #try this, no guarantees
+          codec.flush
+        end
+      end
     end
   end
 
@@ -191,13 +209,24 @@ module LogStash module Codecs class IdentityMapCodec
     # contents should not mutate during this call
     identity_map.delete_if do |identity, compo|
       if (flag = compo.timeout <= cut_off)
-        compo.codec.flush(&(@eviction_block || @decode_block))
+        evict_flush(compo.codec)
       end
       flag
     end
     current_size_and_limit
   end
 
+  def evict_flush(codec)
+    if codec.respond_to?(:auto_flush)
+      codec.auto_flush
+    else
+      if (block = @eviction_block || @decode_block)
+        codec.flush(&block)
+      end
+      # all else - can't do anything
+    end
+  end
+
   def current_size_and_limit
     [identity_count, max_limit]
   end

data/lib/logstash/codecs/multiline.rb

@@ -2,6 +2,7 @@
 require "logstash/codecs/base"
 require "logstash/util/charset"
 require "logstash/timestamp"
+require "logstash/codecs/auto_flush"
 
 # The multiline codec will collapse multiline messages and merge them into a
 # single event.
@@ -76,7 +77,7 @@ require "logstash/timestamp"
 # This says that any line ending with a backslash should be combined with the
 # following line.
 #
-class LogStash::Codecs::Multiline < LogStash::Codecs::Base
+module LogStash module Codecs class Multiline < LogStash::Codecs::Base
   config_name "multiline"
 
   # The regular expression to match.
@@ -126,7 +127,13 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
   # max_lines.
   config :max_bytes, :validate => :bytes, :default => "10 MiB"
 
+  # The accumulation of multiple lines will be converted to an event when either a
+  # matching new line is seen or there has been no new data appended for this time
+  # auto_flush_interval. No default.  If unset, no auto_flush
+  config :auto_flush_interval, :validate => :number
+
   public
+
   def register
     require "grok-pure" # rubygem 'jls-grok'
     require 'logstash/patterns/core'
@@ -139,8 +146,8 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
 
     @patterns_dir = patterns_path.to_a + @patterns_dir
     @patterns_dir.each do |path|
-      if File.directory?(path)
-        path = File.join(path, "*")
+      if ::File.directory?(path)
+        path = ::File.join(path, "*")
       end
 
       Dir.glob(path).each do |file|
@@ -158,11 +165,23 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
 
     @converter = LogStash::Util::Charset.new(@charset)
     @converter.logger = @logger
+    if @auto_flush_interval
+      # will start on first decode
+      @auto_flush_runner = AutoFlush.new(self, @auto_flush_interval)
+    end
   end # def register
 
+  def accept(listener)
+    # memoize references to listener that holds upstream state
+    @previous_listener = @last_seen_listener || listener
+    @last_seen_listener = listener
+    decode(listener.data) do |event|
+      what_based_listener.process_event(event)
+    end
+  end
+
   def decode(text, &block)
     text = @converter.convert(text)
-
     text.split("\n").each do |line|
       match = @grok.match(line)
       @logger.debug("Multiline", :pattern => @pattern, :text => line,
@@ -175,23 +194,41 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
   end # def decode
 
   def buffer(text)
-    @time = LogStash::Timestamp.now if @buffer.empty?
     @buffer_bytes += text.bytesize
-    @buffer << text
+    @buffer.push(text).tap do |b|
+      # do start but preserve the return value
+      auto_flush_runner.start
+    end
   end
 
   def flush(&block)
-    if @buffer.any?
-      yield merge_events
-      reset_buffer
+    if block_given? && @buffer.any?
+      no_error = true
+      events = merge_events
+      begin
+        yield events
+      rescue ::Exception => e
+        # need to rescue everything
+        # likliest cause: backpressure or timeout by exception
+        # can't really do anything but leave the data in the buffer for next time if there is one
+        @logger.error("Multiline: flush downstream error", :exception => e)
+        no_error = false
+      end
+      reset_buffer if no_error
+    end
+  end
+
+  def auto_flush
+    flush do |event|
+      @last_seen_listener.process_event(event)
     end
   end
 
   def merge_events
     event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
     event.tag @multiline_tag if @multiline_tag && @buffer.size > 1
-    event.tag "multiline_codec_max_bytes_reached" if over_maximun_bytes?
-    event.tag "multiline_codec_max_lines_reached" if over_maximun_lines?
+    event.tag "multiline_codec_max_bytes_reached" if over_maximum_bytes?
+    event.tag "multiline_codec_max_lines_reached" if over_maximum_lines?
     event
   end
 
@@ -200,6 +237,14 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
     @buffer_bytes = 0
   end
 
+  def doing_previous?
+    @what == "previous"
+  end
+
+  def what_based_listener
+    doing_previous? ? @previous_listener : @last_seen_listener
+  end
+
   def do_next(text, matched, &block)
     buffer(text)
     flush(&block) if !matched || buffer_over_limits?
@@ -210,16 +255,16 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
     buffer(text)
   end
 
-  def over_maximun_lines?
+  def over_maximum_lines?
     @buffer.size > @max_lines
   end
 
-  def over_maximun_bytes?
+  def over_maximum_bytes?
     @buffer_bytes >= @max_bytes
   end
 
   def buffer_over_limits?
-    over_maximun_lines? || over_maximun_bytes?
+    over_maximum_lines? || over_maximum_bytes?
   end
 
   def encode(event)
@@ -227,4 +272,19 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
     @on_event.call(event, event)
   end # def encode
 
-end # class LogStash::Codecs::Multiline
+  def close
+    if auto_flush_runner.pending?
+      #will cancel task if necessary
+      auto_flush_runner.stop
+    end
+    auto_flush
+  end
+
+  def auto_flush_active?
+    !@auto_flush_interval.nil?
+  end
+
+  def auto_flush_runner
+    @auto_flush_runner || AutoFlushUnset.new(nil, nil)
+  end
+end end end # class LogStash::Codecs::Multiline

data/logstash-codec-multiline.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-multiline'
-  s.version         = '2.0.4'
+  s.version         = '2.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The multiline codec will collapse multiline messages and merge them into a single event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/auto_flush_spec.rb

@@ -0,0 +1,118 @@
+# encoding: utf-8
+require "logstash/codecs/auto_flush"
+require "logstash/codecs/multiline"
+require_relative "../supports/helpers.rb"
+
+describe "AutoFlush and AutoFlushUnset" do
+  let(:flushable)  { AutoFlushTracer.new }
+  let(:flush_wait) { 0.1 }
+
+  describe LogStash::Codecs::AutoFlush do
+    subject { described_class.new(flushable, flush_wait) }
+
+    context "when initialized" do
+      it "#pending? is false" do
+        expect(subject.pending?).to be_falsy
+      end
+
+      it "#stopped? is false" do
+        expect(subject.stopped?).to be_falsy
+      end
+
+      it "#finished? is true" do
+        expect(subject.finished?).to be_truthy
+      end
+    end
+
+    context "when started" do
+      let(:flush_wait) { 20 }
+
+      before { subject.start }
+      after  { subject.stop }
+
+      it "#pending? is true" do
+        expect(subject.pending?).to be_truthy
+      end
+
+      it "#stopped? is false" do
+        expect(subject.stopped?).to be_falsy
+      end
+
+      it "#finished? is false" do
+        expect(subject.finished?).to be_falsy
+      end
+    end
+
+    context "when finished" do
+      before do
+        subject.start
+        sleep flush_wait + 0.1
+      end
+
+      after  { subject.stop }
+
+      it "calls auto_flush on flushable" do
+        expect(flushable.trace_for(:auto_flush)).to be_truthy
+      end
+
+      it "#pending? is false" do
+        expect(subject.pending?).to be_falsy
+      end
+
+      it "#stopped? is false" do
+        expect(subject.stopped?).to be_falsy
+      end
+
+      it "#finished? is true" do
+        expect(subject.finished?).to be_truthy
+      end
+    end
+
+    context "when stopped" do
+      before do
+        subject.start
+        subject.stop
+      end
+
+      it "does not call auto_flush on flushable" do
+        expect(flushable.trace_for(:auto_flush)).to be_falsy
+      end
+
+      it "#pending? is false" do
+        expect(subject.pending?).to be_falsy
+      end
+
+      it "#stopped? is true" do
+        expect(subject.stopped?).to be_truthy
+      end
+
+      it "#finished? is false" do
+        expect(subject.finished?).to be_falsy
+      end
+    end
+  end
+
+  describe LogStash::Codecs::AutoFlushUnset do
+    subject { described_class.new(flushable, 2) }
+
+    it "#pending? is false" do
+      expect(subject.pending?).to be_falsy
+    end
+
+    it "#stopped? is true" do
+      expect(subject.stopped?).to be_truthy
+    end
+
+    it "#finished? is true" do
+      expect(subject.finished?).to be_truthy
+    end
+
+    it "#start returns self" do
+      expect(subject.start).to eq(subject)
+    end
+
+    it "#stop returns self" do
+      expect(subject.start).to eq(subject)
+    end
+  end
+end

data/spec/codecs/identity_map_codec_spec.rb

@@ -1,32 +1,8 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/identity_map_codec"
-
-class LogTracer
-  def initialize() @tracer = []; end
-  def warn(*args) @tracer.push [:warn, args]; end
-  def error(*args) @tracer.push [:error, args]; end
-
-  def trace_for(symbol)
-    params = @tracer.assoc(symbol)
-    params.nil? ? false : params.last
-  end
-end
-
-class IdentityMapCodecTracer
-  def initialize() @tracer = []; end
-  def clone() self.class.new; end
-  def decode(data) @tracer.push [:decode, data]; end
-  def encode(event) @tracer.push [:encode, event]; end
-  def flush(&block) @tracer.push [:flush, block.call]; end
-  def close() @tracer.push [:close, true]; end
-  def logger() @logger ||= LogTracer.new; end
-
-  def trace_for(symbol)
-    params = @tracer.assoc(symbol)
-    params.nil? ? false : params.last
-  end
-end
+require "logstash/codecs/multiline"
+require_relative "../supports/helpers.rb"
 
 describe LogStash::Codecs::IdentityMapCodec do
   let(:codec)   { IdentityMapCodecTracer.new }
@@ -220,4 +196,49 @@ describe LogStash::Codecs::IdentityMapCodec do
       end
     end
   end
+
+  describe "observer/listener based processing" do
+    let(:listener) { LineListener }
+    let(:queue)    { [] }
+    let(:identity) { "stream1" }
+    let(:config)   { {"pattern" => "^\\s", "what" => "previous"} }
+    let(:mlc) { MultilineRspec.new(config).tap {|c| c.register } }
+    let(:imc) { described_class.new(mlc) }
+
+    before do
+      listener = LineListener.new(queue, imc, identity)
+      listener.accept("foo")
+    end
+
+    describe "normal processing" do
+      context "when wrapped codec has auto-flush deactivated" do
+        it "no events are generated (the line is buffered)" do
+          expect(imc.identity_count).to eq(1)
+          expect(queue.size).to eq(0)
+          expect(mlc.internal_buffer[0]).to eq("foo")
+        end
+      end
+
+      context "when wrapped codec has auto-flush activated" do
+        let(:config) { {"pattern" => "^\\s", "what" => "previous", "auto_flush_interval" => 0.2} }
+        it "one event is generated" do
+          sleep 0.4
+          expect(queue.size).to eq(1)
+          expect(queue[0]["message"]).to eq("foo")
+          expect(imc.identity_count).to eq(1)
+        end
+      end
+    end
+
+    describe "evict method" do
+      context "when evicting and wrapped codec implements auto-flush" do
+        it "flushes and removes the identity" do
+          expect(imc.identity_count).to eq(1)
+          imc.evict(identity)
+          expect(queue[0]["message"]).to eq("foo")
+          expect(imc.identity_count).to eq(0)
+        end
+      end
+    end
+  end
 end

data/spec/codecs/multiline_spec.rb

@@ -3,57 +3,63 @@ require "logstash/codecs/multiline"
 require "logstash/event"
 require "insist"
 require_relative "../supports/helpers.rb"
+# above helper also defines a subclass of Multiline
+# called MultilineRspec that exposes the internal buffer
+# and a Logger Mock
 
 describe LogStash::Codecs::Multiline do
   context "#decode" do
-    it "should be able to handle multiline events with additional lines space-indented" do
-      codec = LogStash::Codecs::Multiline.new("pattern" => "^\\s", "what" => "previous")
-      lines = [ "hello world", "   second line", "another first line" ]
-      events = []
-      lines.each do |line|
-        codec.decode(line) do |event|
-          events << event
+    let(:config) { {} }
+    let(:codec) { LogStash::Codecs::Multiline.new(config).tap {|c| c.register } }
+    let(:events) { [] }
+    let(:line_producer) do
+      lambda do |lines|
+        lines.each do |line|
+          codec.decode(line) do |event|
+            events << event
+          end
         end
       end
+    end
+
+    it "should be able to handle multiline events with additional lines space-indented" do
+      config.update("pattern" => "^\\s", "what" => "previous")
+      lines = [ "hello world", "   second line", "another first line" ]
+      line_producer.call(lines)
       codec.flush { |e| events << e }
-      insist { events.size } == 2
-      insist { events[0]["message"] } == "hello world\n   second line"
-      insist { events[0]["tags"] }.include?("multiline")
-      insist { events[1]["message"] } == "another first line"
-      insist { events[1]["tags"] }.nil?
+
+      expect(events.size).to eq(2)
+      expect(events[0]["message"]).to eq "hello world\n   second line"
+      expect(events[0]["tags"]).to include("multiline")
+      expect(events[1]["message"]).to eq "another first line"
+      expect(events[1]["tags"]).to be_nil
     end
 
     it "should allow custom tag added to multiline events" do
-      codec = LogStash::Codecs::Multiline.new("pattern" => "^\\s", "what" => "previous", "multiline_tag" => "hurray" )
+      config.update("pattern" => "^\\s", "what" => "previous", "multiline_tag" => "hurray")
       lines = [ "hello world", "   second line", "another first line" ]
-      events = []
-      lines.each do |line|
-        codec.decode(line) do |event|
-          events << event
-        end
-      end
+      line_producer.call(lines)
       codec.flush { |e| events << e }
-      insist { events.size } == 2
-      insist { events[0]["tags"] }.include?("hurray")
-      insist { events[1]["tags"] }.nil?
+
+      expect(events.size).to eq 2
+      expect(events[0]["tags"]).to include("hurray")
+      expect(events[1]["tags"]).to be_nil
     end
 
     it "should handle new lines in messages" do
-      codec = LogStash::Codecs::Multiline.new("pattern" => '^\s', "what" => "previous")
-      line = "one\ntwo\n  two.2\nthree\n"
-      events = []
-      codec.decode(line) do |event|
-        events << event
+      config.update("pattern" => '\D', "what" => "previous")
+      lineio = StringIO.new("1234567890\nA234567890\nB234567890\n0987654321\n")
+      until lineio.eof
+        line = lineio.read(256) #when this is set to 36 the tests fail
+        codec.decode(line) {|evt| events.push(evt)}
       end
       codec.flush { |e| events << e }
-      insist { events.size } == 3
-      insist { events[0]["message"] } == "one"
-      insist { events[1]["message"] } == "two\n  two.2"
-      insist { events[2]["message"] } == "three"
+      expect(events[0]["message"]).to eq "1234567890\nA234567890\nB234567890"
+      expect(events[1]["message"]).to eq "0987654321"
     end
 
     it "should allow grok patterns to be used" do
-      codec = LogStash::Codecs::Multiline.new(
+      config.update(
         "pattern" => "^%{NUMBER} %{TIME}",
         "negate" => true,
         "what" => "previous"
@@ -61,56 +67,47 @@ describe LogStash::Codecs::Multiline do
 
       lines = [ "120913 12:04:33 first line", "second line", "third line" ]
 
-      events = []
-      lines.each do |line|
-        codec.decode(line) do |event|
-          events << event
-        end
-      end
+      line_producer.call(lines)
       codec.flush { |e| events << e }
 
       insist { events.size } == 1
       insist { events.first["message"] } == lines.join("\n")
     end
 
-
     context "using default UTF-8 charset" do
 
       it "should decode valid UTF-8 input" do
-        codec = LogStash::Codecs::Multiline.new("pattern" => "^\\s", "what" => "previous")
+        config.update("pattern" => "^\\s", "what" => "previous")
         lines = [ "foobar", "κόσμε" ]
-        events = []
         lines.each do |line|
-          insist { line.encoding.name } == "UTF-8"
-          insist { line.valid_encoding? } == true
-
+          expect(line.encoding.name).to eq "UTF-8"
+          expect(line.valid_encoding?).to be_truthy
           codec.decode(line) { |event| events << event }
         end
         codec.flush { |e| events << e }
-        insist { events.size } == 2
+        expect(events.size).to eq 2
 
         events.zip(lines).each do |tuple|
-          insist { tuple[0]["message"] } == tuple[1]
-          insist { tuple[0]["message"].encoding.name } == "UTF-8"
+          expect(tuple[0]["message"]).to eq tuple[1]
+          expect(tuple[0]["message"].encoding.name).to eq "UTF-8"
         end
       end
 
       it "should escape invalid sequences" do
-        codec = LogStash::Codecs::Multiline.new("pattern" => "^\\s", "what" => "previous")
+        config.update("pattern" => "^\\s", "what" => "previous")
         lines = [ "foo \xED\xB9\x81\xC3", "bar \xAD" ]
-        events = []
         lines.each do |line|
-          insist { line.encoding.name } == "UTF-8"
-          insist { line.valid_encoding? } == false
+          expect(line.encoding.name).to eq "UTF-8"
+          expect(line.valid_encoding?).to eq false
 
           codec.decode(line) { |event| events << event }
         end
         codec.flush { |e| events << e }
-        insist { events.size } == 2
+        expect(events.size).to eq 2
 
         events.zip(lines).each do |tuple|
-          insist { tuple[0]["message"] } == tuple[1].inspect[1..-2]
-          insist { tuple[0]["message"].encoding.name } == "UTF-8"
+          expect(tuple[0]["message"]).to eq tuple[1].inspect[1..-2]
+          expect(tuple[0]["message"].encoding.name).to eq "UTF-8"
         end
       end
     end
@@ -119,27 +116,26 @@ describe LogStash::Codecs::Multiline do
     context "with valid non UTF-8 source encoding" do
 
       it "should encode to UTF-8" do
-        codec = LogStash::Codecs::Multiline.new("charset" => "ISO-8859-1", "pattern" => "^\\s", "what" => "previous")
+        config.update("charset" => "ISO-8859-1", "pattern" => "^\\s", "what" => "previous")
         samples = [
           ["foobar", "foobar"],
           ["\xE0 Montr\xE9al", "à Montréal"],
         ]
 
         # lines = [ "foo \xED\xB9\x81\xC3", "bar \xAD" ]
-        events = []
         samples.map{|(a, b)| a.force_encoding("ISO-8859-1")}.each do |line|
-          insist { line.encoding.name } == "ISO-8859-1"
-          insist { line.valid_encoding? } == true
+          expect(line.encoding.name).to eq "ISO-8859-1"
+          expect(line.valid_encoding?).to eq true
 
           codec.decode(line) { |event| events << event }
         end
         codec.flush { |e| events << e }
-        insist { events.size } == 2
+        expect(events.size).to eq 2
 
         events.zip(samples.map{|(a, b)| b}).each do |tuple|
-          insist { tuple[1].encoding.name } == "UTF-8"
-          insist { tuple[0]["message"] } == tuple[1]
-          insist { tuple[0]["message"].encoding.name } == "UTF-8"
+          expect(tuple[1].encoding.name).to eq "UTF-8"
+          expect(tuple[0]["message"]).to eq tuple[1]
+          expect(tuple[0]["message"].encoding.name).to eq "UTF-8"
         end
       end
     end
@@ -147,25 +143,25 @@ describe LogStash::Codecs::Multiline do
     context "with invalid non UTF-8 source encoding" do
 
      it "should encode to UTF-8" do
-        codec = LogStash::Codecs::Multiline.new("charset" => "ASCII-8BIT", "pattern" => "^\\s", "what" => "previous")
+        config.update("charset" => "ASCII-8BIT", "pattern" => "^\\s", "what" => "previous")
         samples = [
           ["\xE0 Montr\xE9al", "� Montr�al"],
           ["\xCE\xBA\xCF\x8C\xCF\x83\xCE\xBC\xCE\xB5", "����������"],
         ]
         events = []
         samples.map{|(a, b)| a.force_encoding("ASCII-8BIT")}.each do |line|
-          insist { line.encoding.name } == "ASCII-8BIT"
-          insist { line.valid_encoding? } == true
+          expect(line.encoding.name).to eq "ASCII-8BIT"
+          expect(line.valid_encoding?).to eq true
 
           codec.decode(line) { |event| events << event }
         end
         codec.flush { |e| events << e }
-        insist { events.size } == 2
+        expect(events.size).to eq 2
 
         events.zip(samples.map{|(a, b)| b}).each do |tuple|
-          insist { tuple[1].encoding.name } == "UTF-8"
-          insist { tuple[0]["message"] } == tuple[1]
-          insist { tuple[0]["message"].encoding.name } == "UTF-8"
+          expect(tuple[1].encoding.name).to eq "UTF-8"
+          expect(tuple[0]["message"]).to eq tuple[1]
+          expect(tuple[0]["message"].encoding.name).to eq "UTF-8"
         end
       end
 
@@ -183,7 +179,7 @@ describe LogStash::Codecs::Multiline do
       let(:options) {
         {
           "pattern" => "^-",
-          "what" => "previous", 
+          "what" => "previous",
           "max_lines" => max_lines,
           "max_bytes" => "2 mb"
         }
@@ -203,7 +199,7 @@ describe LogStash::Codecs::Multiline do
       let(:options) {
         {
           "pattern" => "^-",
-          "what" => "previous", 
+          "what" => "previous",
           "max_lines" => 20000,
           "max_bytes" => max_bytes
         }
@@ -218,4 +214,114 @@ describe LogStash::Codecs::Multiline do
       end
     end
   end
+
+  describe "auto flushing" do
+    let(:config) { {} }
+    let(:codec) { MultilineRspec.new(config).tap {|c| c.register} }
+    let(:events) { [] }
+    let(:lines) do
+      { "en.log" => ["hello world", " second line", " third line"],
+        "fr.log" => ["Salut le Monde", " deuxième ligne", " troisième ligne"],
+        "de.log" => ["Hallo Welt"] }
+    end
+    let(:listener_class) { LineListener }
+    let(:auto_flush_interval) { 0.5 }
+
+    let(:line_producer) do
+      lambda do |path|
+        #create a listener that holds upstream state
+        listener = listener_class.new(events, codec, path)
+        lines[path].each do |data|
+          listener.accept(data)
+        end
+      end
+    end
+
+    context "when auto_flush_interval is not set" do
+      it "does not build any events" do
+        config.update("pattern" => "^\\s", "what" => "previous")
+        line_producer.call("en.log")
+        sleep auto_flush_interval + 0.1
+        expect(events.size).to eq(0)
+        expect(codec.buffer_size).to eq(3)
+      end
+    end
+
+    context "when the auto_flush raises an exception" do
+      let(:errmsg) { "OMG, Daleks!" }
+      let(:listener_class) { LineErrorListener }
+
+      it "does not build any events, logs an error and the buffer data remains" do
+        config.update("pattern" => "^\\s", "what" => "previous",
+          "auto_flush_interval" => auto_flush_interval)
+        codec.logger = MultilineLogTracer.new
+        line_producer.call("en.log")
+        sleep(auto_flush_interval + 0.1)
+        msg, args = codec.logger.trace_for(:error)
+        expect(msg).to eq("Multiline: flush downstream error")
+        expect(args[:exception].message).to eq(errmsg)
+        expect(events.size).to eq(0)
+        expect(codec.buffer_size).to eq(3)
+      end
+    end
+
+    def assert_produced_events(key, sleeping)
+      line_producer.call(key)
+      sleep(sleeping)
+      yield
+      expect(codec).to have_an_empty_buffer
+    end
+
+    context "mode: previous, when there are pauses between multiline file writes" do
+      it "auto-flushes events from the accumulated lines to the queue" do
+        config.update("pattern" => "^\\s", "what" => "previous",
+          "auto_flush_interval" => auto_flush_interval)
+
+        assert_produced_events("en.log", auto_flush_interval + 0.1) do
+          expect(events[0]).to match_path_and_line("en.log", lines["en.log"])
+        end
+
+        line_producer.call("fr.log")
+        #next line(s) come before auto-flush i.e. assert its buffered
+        sleep(auto_flush_interval - 0.3)
+        expect(codec.buffer_size).to eq(3)
+        expect(events.size).to eq(1)
+
+        assert_produced_events("de.log", auto_flush_interval + 0.1) do
+          # now the events are generated
+          expect(events[1]).to match_path_and_line("fr.log", lines["fr.log"])
+          expect(events[2]).to match_path_and_line("de.log", lines["de.log"])
+        end
+      end
+    end
+
+    context "mode: next, when there are pauses between multiline file writes" do
+
+      let(:lines) do
+        { "en.log" => ["hello world++", "second line++", "third line"],
+          "fr.log" => ["Salut le Monde++", "deuxième ligne++", "troisième ligne"],
+          "de.log" => ["Hallo Welt"] }
+      end
+
+      it "auto-flushes events from the accumulated lines to the queue" do
+        config.update("pattern" => "\\+\\+$", "what" => "next",
+          "auto_flush_interval" => auto_flush_interval)
+
+        assert_produced_events("en.log", auto_flush_interval + 0.1) do
+          # wait for auto_flush
+          expect(events[0]).to match_path_and_line("en.log", lines["en.log"])
+        end
+
+        assert_produced_events("de.log", auto_flush_interval - 0.3) do
+          #this file is read before auto-flush
+          expect(events[1]).to match_path_and_line("de.log", lines["de.log"])
+        end
+
+        assert_produced_events("fr.log", auto_flush_interval + 0.1) do
+          # wait for auto_flush
+          expect(events[2]).to match_path_and_line("fr.log", lines["fr.log"])
+        end
+      end
+    end
+  end
 end

data/spec/supports/helpers.rb

@@ -1,3 +1,5 @@
+
+
 def decode_events
   multiline =  LogStash::Codecs::Multiline.new(options)
 
@@ -10,3 +12,112 @@ def decode_events
   multiline.flush { |event| events << event }
   events
 end
+
+class LineListener
+  attr_reader :data, :path, :queue, :codec
+  # use attr_reader to define noop methods of Listener API
+  attr_reader :deleted, :created, :error, :eof #, :line
+
+  def initialize(queue, codec, path = '')
+    # store state from upstream
+    @queue = queue
+    @codec = codec
+    @path = path
+  end
+
+  # receives a line from some upstream source
+  # and sends it downstream
+  def accept(data)
+    @codec.accept dup_adding_state(data)
+  end
+
+  def process_event(event)
+    event["path"] = path
+    @queue << event
+  end
+
+  def add_state(data)
+    @data = data
+    self
+  end
+
+  private
+
+  # dup and add state for downstream
+  def dup_adding_state(line)
+    self.class.new(queue, codec, path).add_state(line)
+  end
+end
+
+class LineErrorListener < LineListener
+  def process_event(event)
+    raise StandardError.new("OMG, Daleks!")
+  end
+end
+
+class MultilineRspec < LogStash::Codecs::Multiline
+  def internal_buffer
+    @buffer
+  end
+  def buffer_size
+    @buffer.size
+  end
+end
+
+class TracerBase
+  def initialize() @tracer = []; end
+
+  def trace_for(symbol)
+    params = @tracer.assoc(symbol)
+    params.nil? ? false : params.last
+  end
+
+  def clear()
+    @tracer.clear()
+  end
+end
+
+class MultilineLogTracer < TracerBase
+  def warn(*args) @tracer.push [:warn, args]; end
+  def error(*args) @tracer.push [:error, args]; end
+  def debug(*args) @tracer.push [:debug, args]; end
+  def info(*args) @tracer.push [:info, args]; end
+
+  def info?() true; end
+  def debug?() true; end
+  def warn?() true; end
+  def error?() true; end
+end
+
+class AutoFlushTracer < TracerBase
+  def auto_flush() @tracer.push [:auto_flush, true]; end
+end
+
+class IdentityMapCodecTracer < TracerBase
+  def clone() self.class.new; end
+  def decode(data) @tracer.push [:decode, data]; end
+  def encode(event) @tracer.push [:encode, event]; end
+  def flush(&block) @tracer.push [:flush, block.call]; end
+  def close() @tracer.push [:close, true]; end
+  def logger() @logger ||= MultilineLogTracer.new; end
+end
+
+RSpec::Matchers.define(:have_an_empty_buffer) do
+  match do |actual|
+    actual.buffer_size.zero?
+  end
+
+  failure_message do
+    "Expecting #{actual.buffer_size} to be 0"
+  end
+end
+
+RSpec::Matchers.define(:match_path_and_line) do |path, line|
+  match do |actual|
+    actual["path"] == path && actual["message"] == line.join($/)
+  end
+
+  failure_message do
+    "Expecting #{actual['path']} to equal `#{path}` and #{actual["message"]} to equal #{line.join($/)}"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-multiline
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2015-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -84,9 +84,11 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- lib/logstash/codecs/auto_flush.rb
 - lib/logstash/codecs/identity_map_codec.rb
 - lib/logstash/codecs/multiline.rb
 - logstash-codec-multiline.gemspec
+- spec/codecs/auto_flush_spec.rb
 - spec/codecs/identity_map_codec_spec.rb
 - spec/codecs/multiline_spec.rb
 - spec/supports/helpers.rb
@@ -117,6 +119,7 @@ signing_key:
 specification_version: 4
 summary: The multiline codec will collapse multiline messages and merge them into a single event.
 test_files:
+- spec/codecs/auto_flush_spec.rb
 - spec/codecs/identity_map_codec_spec.rb
 - spec/codecs/multiline_spec.rb
 - spec/supports/helpers.rb