logstash-codec-joinlines 0.1.0

data/lib/logstash/codecs/joinlines.rb

@@ -0,0 +1,301 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/util/charset"
+require "logstash/timestamp"
+require "logstash/codecs/auto_flush"
+
+# The joinlines codec will join lines mathcing specified patterns.
+# It is based on the multiline codec, but offers the opportunity to
+# specify a list of patterns, whats and negates. The lists must be
+# of equal length. 
+#
+# IMPORTANT: If you are using a Logstash input plugin that supports multiple
+# hosts, such as the <<plugins-inputs-beats>> input plugin, you should not use
+# the joinlines codec to handle multiline events. Doing so may result in the
+# mixing of streams and corrupted event data. In this situation, you need to
+# handle multiline events before sending the event data to Logstash.
+#
+# Example usage
+# [source,ruby]
+# input {
+#   stdin {
+#     codec => joinlines {
+#       patterns => [ "^The following message", "^\s*at" ]
+#       what => [ "next", "previous" ]
+#       negate => [ false, false ]
+#     }
+#   }
+# }
+#
+# The example above will join lines starting with "The following message"
+# with the next line, and stack traces with the previous line.
+#
+module LogStash module Codecs class Joinlines < LogStash::Codecs::Base
+
+  # The codec name
+  config_name "joinlines"
+
+  # The patterns to recognize
+  config :patterns, :validate => :string, :list => true, :required => true
+
+  # The patterns to recognize
+  config :what, :validate => ["previous", "next"], :list => true, :required => true
+
+  # Negate match?
+  config :negate, :validate => :boolean, :list => true, :required => true
+
+  # Logstash ships by default with a bunch of patterns, so you don't
+  # necessarily need to define this yourself unless you are adding additional
+  # patterns.
+  #
+  # Pattern files are plain text with format:
+  # [source,ruby]
+  #     NAME PATTERN
+  #
+  # For example:
+  # [source,ruby]
+  #     NUMBER \d+
+  config :patterns_dir, :validate => :array, :default => []
+
+  # The character encoding used in this input. Examples include `UTF-8`
+  # and `cp1252`
+  #
+  # This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
+  # or in another character set other than `UTF-8`.
+  #
+  # This only affects "plain" format logs since JSON is `UTF-8` already.
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  # Tag multiline events with a given tag. This tag will only be added
+  # to events that actually have multiple lines in them.
+  config :multiline_tag, :validate => :string, :default => "joinlines"
+
+  # The accumulation of events can make logstash exit with an out of memory error
+  # if event boundaries are not correctly defined. This settings make sure to flush
+  # multiline events after reaching a number of lines, it is used in combination
+  # max_bytes.
+  config :max_lines, :validate => :number, :default => 500
+
+  # The accumulation of events can make logstash exit with an out of memory error
+  # if event boundaries are not correctly defined. This settings make sure to flush
+  # multiline events after reaching a number of bytes, it is used in combination
+  # max_lines.
+  config :max_bytes, :validate => :bytes, :default => "10 MiB"
+
+  # The accumulation of multiple lines will be converted to an event when either a
+  # matching new line is seen or there has been no new data appended for this many
+  # seconds. No default.  If unset, no auto_flush. Units: seconds
+  config :auto_flush_interval, :validate => :number
+
+  public
+  def register
+    require "grok-pure" # rubygem 'jls-grok'
+    require 'logstash/patterns/core'
+
+    @matching = ""
+
+    # Detect if we are running from a jarfile, pick the right path.
+    patterns_path = []
+    patterns_path += [LogStash::Patterns::Core.path]
+
+    @patterns_dir = patterns_path.to_a + @patterns_dir
+    @groks = []
+    @handlers = []
+
+    @patterns.zip(@what).each do |pattern,what|
+      grok = Grok.new
+
+      @patterns_dir.each do |path|
+        if ::File.directory?(path)
+          path = ::File.join(path, "*")
+        end
+
+        Dir.glob(path).each do |file|
+          @logger.debug("Grok loading patterns from file", :path => file)
+          grok.add_patterns_from_file(file)
+        end
+      end
+
+      grok.compile(pattern)
+      handler = method("do_#{what}".to_sym)
+
+      @groks.push(grok)
+      @handlers.push(handler)
+    end
+
+    @logger.trace("Registered joinlines plugin", :type => @type, :config => @config)
+    reset_buffer
+
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+
+    if @auto_flush_interval
+      # will start on first decode
+      @auto_flush_runner = AutoFlush.new(self, @auto_flush_interval)
+    end
+  end # def register
+
+  def use_mapper_auto_flush
+    return unless auto_flush_active?
+    @auto_flush_runner = AutoFlushUnset.new(nil, nil)
+    @auto_flush_interval = @auto_flush_interval.to_f
+  end
+
+  def accept(listener)
+    # memoize references to listener that holds upstream state
+    @previous_listener = @last_seen_listener || listener
+    @last_seen_listener = listener
+
+    internal_decode(listener.data) do |event,what|
+      what_based_listener(what).process_event(event)
+    end
+  end
+
+  def zip_config
+    @patterns.zip(@what, @negate, @groks, @handlers)
+  end
+
+  #private
+  def internal_decode(text, &block)
+    do_flush = false
+    text = @converter.convert(text)
+    text.split("\n").each do |line|
+      matched = false
+      zip_config.each do |pattern,what,negate,grok,handler|
+        match = grok.match(line)
+        @logger.debug("Joinlines", :pattern => pattern, :text => line,
+                      :match => (match != false), :negate => negate)
+
+        # Add negate option
+        match = (match and !negate) || (!match and negate)
+
+        if match
+          do_flush = (what == "next" and @matching != "next")
+          matched = true
+          @matching = what
+          break
+        end
+      end
+
+      if !matched
+        do_flush = (@matching != "next")
+        @matching = ""
+      end
+
+      if do_flush
+        flush do |event|
+          yield(event,@matching)
+        end
+        do_flush = false
+      end
+
+      auto_flush_runner.start
+      buffer(line)
+    end
+  end
+
+  public
+  def decode(text, &block)
+    internal_decode(text) do |event,what|
+      yield(event)
+    end
+  end # def decode
+
+  def buffer(text)
+    @buffer_bytes += text.bytesize
+    @buffer.push(text)
+  end
+
+  def flush(&block)
+    if block_given? && @buffer.any?
+      no_error = true
+      events = merge_events
+      begin
+        yield events
+      rescue ::Exception => e
+        # need to rescue everything
+        # likliest cause: backpressure or timeout by exception
+        # can't really do anything but leave the data in the buffer for next time if there is one
+        @logger.error("Joinlines: flush downstream error", :exception => e)
+        no_error = false
+      end
+      reset_buffer if no_error
+    end
+  end
+
+  def auto_flush(listener = @last_seen_listener)
+    return if listener.nil?
+
+    flush do |event|
+      listener.process_event(event)
+    end
+  end
+
+  def merge_events
+    event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
+    event.tag @multiline_tag if !@multiline_tag.empty? && @buffer.size > 1
+    event.tag "joinlines_codec_max_bytes_reached" if over_maximum_bytes?
+    event.tag "joinlines_codec_max_lines_reached" if over_maximum_lines?
+    event
+  end
+
+  def reset_buffer
+    @buffer = []
+    @buffer_bytes = 0
+  end
+
+  def doing_previous?(what)
+    what != "next"
+  end
+
+  def what_based_listener(what)
+    doing_previous?(what) ? @previous_listener : @last_seen_listener
+  end
+
+  def do_next(text, matched, &block)
+    buffer(text)
+    auto_flush_runner.start
+    flush(&block) if !matched || buffer_over_limits?
+  end
+
+  def do_previous(text, matched, &block)
+    flush(&block) if !matched || buffer_over_limits?
+    auto_flush_runner.start
+    buffer(text)
+  end
+
+  def over_maximum_lines?
+    @buffer.size > @max_lines
+  end
+
+  def over_maximum_bytes?
+    @buffer_bytes >= @max_bytes
+  end
+
+  def buffer_over_limits?
+    over_maximum_lines? || over_maximum_bytes?
+  end
+
+  def encode(event)
+    # Nothing to do.
+    @on_event.call(event, event)
+  end # def encode
+
+  def close
+    auto_flush_runner.stop
+  end
+
+  def auto_flush_active?
+    !@auto_flush_interval.nil?
+  end
+
+  def auto_flush_runner
+    @auto_flush_runner || AutoFlushUnset.new(nil, nil)
+  end
+
+  def initialize_copy(source)
+    super
+    register
+  end
+
+end end end # class LogStash::Codecs::Joinlines

data/lib/logstash/codecs/retriggerable_task.rb

@@ -0,0 +1,81 @@
+require "concurrent"
+
+module LogStash module Codecs class RetriggerableTask
+  SLEEP_FOR = 0.25.freeze
+
+  attr_reader :thread
+
+  def initialize(delay, listener)
+    @count = calculate_count(delay)
+    @listener = listener
+    @counter = Concurrent::AtomicFixnum.new(0 + @count)
+    @stopped = Concurrent::AtomicBoolean.new(false)
+    @semaphore = Concurrent::Semaphore.new(1)
+  end
+
+  def retrigger
+    return if stopped?
+    if executing?
+      @semaphore.acquire
+    end
+
+    if pending?
+      reset_counter
+    else
+      start
+    end
+  end
+
+  def close
+    @stopped.make_true
+  end
+
+  def counter
+    @counter.value
+  end
+
+  def executing?
+    running? && counter < 1
+  end
+
+  def pending?
+    running? && counter > 0
+  end
+
+  private
+
+  def calculate_count(value)
+    # in multiples of SLEEP_FOR (0.25) seconds
+    # if delay is 10 seconds then count is 40
+    # this only works when SLEEP_FOR is less than 1
+    return 1 if value < SLEEP_FOR
+    (value / SLEEP_FOR).floor
+  end
+
+  def reset_counter
+    @counter.value = 0 + @count
+  end
+
+  def running?
+    @thread && @thread.alive?
+  end
+
+  def start()
+    reset_counter
+    @thread = Thread.new do
+      while counter > 0
+        break if stopped?
+        sleep SLEEP_FOR
+        @counter.decrement
+      end
+
+      @semaphore.drain_permits
+      @listener.timeout if !stopped?
+      @semaphore.release
+    end
+  end
+
+  def stopped?
+    @stopped.value
+  end
+end end end

data/logstash-codec-joinlines.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-codec-joinlines'
+  s.version       = '0.1.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Merges multiline messages into a single event, allowing for multiple patterns.'
+  s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
+  s.homepage      = 'https://github.com/lovmoen/logstash-codec-joinlines'
+  s.authors       = ['Svein L. Ellingsen (lovmoen)']
+  s.email         = 'lovmoen@gmail.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', "~> 2.0"
+  s.add_runtime_dependency 'logstash-codec-line'
+
+  s.add_runtime_dependency 'logstash-patterns-core'
+  s.add_runtime_dependency 'jls-grok', '~> 0.11.1'
+
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/codecs/joinlines_spec.rb

@@ -0,0 +1,435 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/joinlines"
+require "logstash/event"
+require "insist"
+require_relative '../spec_helper'
+
+# above helper also defines a subclass of Joinlines
+# called JoinlinesRspec that exposes the internal buffer
+# and a Logger Mock
+
+describe LogStash::Codecs::Joinlines do
+  context "#multipatterns" do
+    let(:config) { {"patterns" => "", "what" => "next", "negate" => false} }
+    let(:codec) { LogStash::Codecs::Joinlines.new(config).tap {|c| c.register } }
+    let(:events) { [] }
+    let(:line_producer) do
+      lambda do |lines|
+        lines.each do |line|
+          codec.decode(line) do |event|
+            events << event
+          end
+        end
+      end
+    end
+    
+    it "should internally decode lines to (event, what) pairs" do
+      config.update("patterns" => ["^\\s", "next"], "what" => ["previous", "next"], "negate" => [false, false])
+      text = "hello world\n   second line\nanother first line\nnext\nowns previous\n"
+
+      events = []
+      whats = []
+      codec.internal_decode(text) do |event,what|
+        events.push(event)
+        whats.push(what)
+      end
+
+      # Must flush to get last event
+      codec.flush do |event|
+        events.push(event)
+        whats.push("final") # dummy
+      end
+
+      expect(events.size).to eq(3)
+      expect(whats.size).to eq(3)
+      expect(events[0].get("message")).to eq("hello world\n   second line")
+      expect(events[1].get("message")).to eq("another first line")
+      expect(events[2].get("message")).to eq("next\nowns previous")
+    end
+
+    it "should break between consecutive previous and next" do
+      config.update("patterns" => ["^\\s", "next"], "what" => ["previous", "next"], "negate" => [false, false])
+      lines = [ "hello world", "   second line", "next", "owns previous" ]      
+      line_producer.call(lines)
+      codec.flush { |e| events << e }
+
+      expect(events.size).to eq(2)
+      expect(events[0].get("message")).to eq "hello world\n   second line"
+      expect(events[0].get("tags")).to include("joinlines")
+      expect(events[1].get("message")).to eq "next\nowns previous"
+      expect(events[1].get("tags")).to include("joinlines")
+    end
+
+    it "should stitch together consecutive next and previous" do
+      config.update("patterns" => ["^\\s", "next"], "what" => ["previous", "next"], "negate" => [false, false])
+      lines = [ "next", "owns previous and next", "   second line", "another first" ]      
+      line_producer.call(lines)
+      codec.flush { |e| events << e }
+
+      expect(events.size).to eq(2)
+      expect(events[0].get("message")).to eq "next\nowns previous and next\n   second line"
+      expect(events[0].get("tags")).to include("joinlines")
+      expect(events[1].get("message")).to eq "another first"
+      expect(events[1].get("tags")).to be_nil
+    end
+  end
+
+  context "#decode" do
+    let(:config) { {"patterns" => "", "what" => "next", "negate" => false} }
+    let(:codec) { LogStash::Codecs::Joinlines.new(config).tap {|c| c.register } }
+    let(:events) { [] }
+    let(:line_producer) do
+      lambda do |lines|
+        lines.each do |line|
+          codec.decode(line) do |event|
+            events << event
+          end
+        end
+      end
+    end
+
+    it "should be able to handle multiline events with additional lines space-indented" do
+      config.update("patterns" => "^\\s", "what" => "previous", "negate" => false)
+      lines = [ "hello world", "   second line", "another first line" ]
+      line_producer.call(lines)
+      codec.flush { |e| events << e }
+
+      expect(events.size).to eq(2)
+      expect(events[0].get("message")).to eq "hello world\n   second line"
+      expect(events[0].get("tags")).to include("joinlines")
+      expect(events[1].get("message")).to eq "another first line"
+      expect(events[1].get("tags")).to be_nil
+    end
+
+    it "should allow custom tag added to multiline events" do
+      config.update("patterns" => "^\\s", "what" => "previous", "negate" => false, "multiline_tag" => "hurray")
+      lines = [ "hello world", "   second line", "another first line" ]
+      line_producer.call(lines)
+      codec.flush { |e| events << e }
+
+      expect(events.size).to eq 2
+      expect(events[0].get("tags")).to include("hurray")
+      expect(events[1].get("tags")).to be_nil
+    end
+
+    it "should handle new lines in messages" do
+      config.update("patterns" => '\D', "what" => "previous", "negate" => false)
+      lineio = StringIO.new("1234567890\nA234567890\nB234567890\n0987654321\n")
+      until lineio.eof
+        line = lineio.read(256) #when this is set to 36 the tests fail
+        codec.decode(line) {|evt| events.push(evt)}
+      end
+      codec.flush { |e| events << e }
+      expect(events[0].get("message")).to eq "1234567890\nA234567890\nB234567890"
+      expect(events[1].get("message")).to eq "0987654321"
+    end
+
+    it "should allow grok patterns to be used" do
+      config.update(
+        "patterns" => "^%{NUMBER} %{TIME}",
+        "negate" => true,
+        "what" => "previous"
+      )
+
+      lines = [ "120913 12:04:33 first line", "second line", "third line" ]
+
+      line_producer.call(lines)
+      codec.flush { |e| events << e }
+
+      insist { events.size } == 1
+      insist { events.first.get("message") } == lines.join("\n")
+    end
+
+    context "using default UTF-8 charset" do
+
+      it "should decode valid UTF-8 input" do
+        config.update("patterns" => "^\\s", "what" => "previous", "negate" => false)
+        lines = [ "foobar", "κόσμε" ]
+        lines.each do |line|
+          expect(line.encoding.name).to eq "UTF-8"
+          expect(line.valid_encoding?).to be_truthy
+          codec.decode(line) { |event| events << event }
+        end
+
+        codec.flush { |e| events << e }
+        expect(events.size).to eq 2
+
+        events.zip(lines).each do |tuple|
+          expect(tuple[0].get("message")).to eq tuple[1]
+          expect(tuple[0].get("message").encoding.name).to eq "UTF-8"
+        end
+      end
+
+      it "should escape invalid sequences" do
+        config.update("patterns" => "^\\s", "what" => "previous", "negate" => false)
+        lines = [ "foo \xED\xB9\x81\xC3", "bar \xAD" ]
+        lines.each do |line|
+          expect(line.encoding.name).to eq "UTF-8"
+          expect(line.valid_encoding?).to eq false
+
+          codec.decode(line) { |event| events << event }
+        end
+        codec.flush { |e| events << e }
+        expect(events.size).to eq 2
+
+        events.zip(lines).each do |tuple|
+          expect(tuple[0].get("message")).to eq tuple[1].inspect[1..-2]
+          expect(tuple[0].get("message").encoding.name).to eq "UTF-8"
+        end
+      end
+
+      it "decodes and joins multiple patterns" do
+        config.update("patterns" => [ "^\\s", "^the following" ], "what" => [ "previous", "next" ], "negate" => [ false, false] )
+        lines = [ "hello world", "   second line", "another first line", "the following message belongs to next", "I own the previous", "Another first" ]
+
+        lines.each do |line|
+          codec.decode(line) do |event|
+            events << event
+          end
+        end
+
+        codec.flush { |e| events << e }
+  
+        #expect(events.size).to eq(4)
+        expect(events[0].get("message")).to eq "hello world\n   second line"
+        expect(events[0].get("tags")).to include("joinlines")
+        expect(events[1].get("message")).to eq "another first line"
+        expect(events[1].get("tags")).to be_nil
+        expect(events[2].get("message")).to eq "the following message belongs to next\nI own the previous"
+        expect(events[2].get("tags")).to include("joinlines")
+        expect(events[3].get("message")).to eq "Another first"
+        expect(events[3].get("tags")).to be_nil
+      end
+    end
+
+
+    context "with valid non UTF-8 source encoding" do
+
+      it "should encode to UTF-8" do
+        config.update("charset" => "ISO-8859-1", "patterns" => "^\\s", "what" => "previous", "negate" => false)
+        samples = [
+          ["foobar", "foobar"],
+          ["\xE0 Montr\xE9al", "à Montréal"],
+        ]
+
+        # lines = [ "foo \xED\xB9\x81\xC3", "bar \xAD" ]
+        samples.map{|(a, b)| a.force_encoding("ISO-8859-1")}.each do |line|
+          expect(line.encoding.name).to eq "ISO-8859-1"
+          expect(line.valid_encoding?).to eq true
+
+          codec.decode(line) { |event| events << event }
+        end
+        codec.flush { |e| events << e }
+        expect(events.size).to eq 2
+
+        events.zip(samples.map{|(a, b)| b}).each do |tuple|
+          expect(tuple[1].encoding.name).to eq "UTF-8"
+          expect(tuple[0].get("message")).to eq tuple[1]
+          expect(tuple[0].get("message").encoding.name).to eq "UTF-8"
+        end
+      end
+    end
+
+    context "with invalid non UTF-8 source encoding" do
+
+     it "should encode to UTF-8" do
+        config.update("charset" => "ASCII-8BIT", "patterns" => "^\\s", "what" => "previous", "negate" => false)
+        samples = [
+          ["\xE0 Montr\xE9al", "� Montr�al"],
+          ["\xCE\xBA\xCF\x8C\xCF\x83\xCE\xBC\xCE\xB5", "����������"],
+        ]
+        events = []
+        samples.map{|(a, b)| a.force_encoding("ASCII-8BIT")}.each do |line|
+          expect(line.encoding.name).to eq "ASCII-8BIT"
+          expect(line.valid_encoding?).to eq true
+
+          codec.decode(line) { |event| events << event }
+        end
+        codec.flush { |e| events << e }
+        expect(events.size).to eq 2
+
+        events.zip(samples.map{|(a, b)| b}).each do |tuple|
+          expect(tuple[1].encoding.name).to eq "UTF-8"
+          expect(tuple[0].get("message")).to eq tuple[1]
+          expect(tuple[0].get("message").encoding.name).to eq "UTF-8"
+        end
+      end
+
+    end
+  end
+
+  context "with non closed multiline events" do
+    let(:random_number_of_events) { rand(300..1000) }
+    let(:sample_event) { "- Sample event" }
+    let(:events) { decode_events }
+    let(:unmerged_events_count) { events.collect { |event| event.get("message").split(LogStash::Codecs::Joinlines::NL).size }.inject(&:+) }
+
+    context "break on maximum_lines" do
+      let(:max_lines) { rand(10..100) }
+      let(:options) {
+        {
+          "patterns" => "^-",
+          "what" => "previous",
+          "negate" => false,
+          "max_lines" => max_lines,
+          "max_bytes" => "2 mb"
+        }
+      }
+
+      it "flushes on a maximum lines" do
+        expect(unmerged_events_count).to eq(random_number_of_events)
+      end
+
+      it "tags the event" do
+        expect(events.first.get("tags")).to include("joinlines_codec_max_lines_reached")
+      end
+    end
+
+    context "break on maximum bytes" do
+      let(:max_bytes) { rand(30..100) }
+      let(:options) {
+        {
+          "patterns" => "^-",
+          "what" => "previous",
+          "negate" => false,
+          "max_lines" => 20000,
+          "max_bytes" => max_bytes
+        }
+      }
+
+      it "flushes on a maximum bytes size" do
+        expect(unmerged_events_count).to eq(random_number_of_events)
+      end
+
+      it "tags the event" do
+        expect(events.first.get("tags")).to include("joinlines_codec_max_bytes_reached")
+      end
+    end
+  end
+
+  describe "auto flushing" do
+    let(:config) { {"patterns" => "", "what" => "next", "negate" => false} }
+    let(:events) { [] }
+    let(:lines) do
+      { "en.log" => ["hello world", " second line", " third line"],
+        "fr.log" => ["Salut le Monde", " deuxième ligne", " troisième ligne"],
+        "de.log" => ["Hallo Welt"] }
+    end
+    let(:listener_class) { Jlc::LineListener }
+    let(:auto_flush_interval) { 2 }
+
+    let(:line_producer) do
+      lambda do |path|
+        #create a listener that holds upstream state
+        listener = listener_class.new(events, codec, path)
+        lines[path].each do |data|
+          listener.accept(data)
+        end
+      end
+    end
+
+    let(:codec) do
+      Jlc::JoinlinesRspec.new(config).tap {|c| c.register}
+    end
+
+    before :each do
+      expect(LogStash::Codecs::Joinlines).to receive(:logger).and_return(Jlc::JoinlinesLogTracer.new).at_least(:once)
+    end
+
+    context "when auto_flush_interval is not set" do
+      it "does not build any events" do
+        config.update("patterns" => "^\\s", "what" => "previous", "negate" => false)
+        line_producer.call("en.log")
+        sleep auto_flush_interval + 0.1
+        expect(events.size).to eq(0)
+        expect(codec.buffer_size).to eq(3)
+      end
+    end
+
+    context "when the auto_flush raises an exception" do
+      let(:errmsg) { "OMG, Daleks!" }
+      let(:listener_class) { Jlc::LineErrorListener }
+
+      it "does not build any events, logs an error and the buffer data remains" do
+        config.update("patterns" => "^\\s", "what" => "previous", "negate" => false,
+          "auto_flush_interval" => auto_flush_interval)
+        line_producer.call("en.log")
+        sleep(auto_flush_interval + 0.2)
+        msg, args = codec.logger.trace_for(:error)
+        expect(msg).to eq("Joinlines: flush downstream error")
+        expect(args[:exception].message).to eq(errmsg)
+        expect(events.size).to eq(0)
+        expect(codec.buffer_size).to eq(3)
+      end
+    end
+
+    def assert_produced_events(key, sleeping)
+      line_producer.call(key)
+      sleep(sleeping)
+      yield
+      #expect(codec).to have_an_empty_buffer
+    end
+
+    context "mode: previous, when there are pauses between multiline file writes" do
+      it "auto-flushes events from the accumulated lines to the queue" do
+        config.update("patterns" => "^\\s", "what" => "previous", "negate" => false,
+          "auto_flush_interval" => auto_flush_interval)
+
+        assert_produced_events("en.log", auto_flush_interval + 0.1) do
+          expect(events[0]).to match_path_and_line("en.log", lines["en.log"])
+        end
+
+        line_producer.call("fr.log")
+        #next line(s) come before auto-flush i.e. assert its buffered
+        sleep(auto_flush_interval - 0.3)
+        expect(codec.buffer_size).to eq(3)
+        expect(events.size).to eq(1)
+
+        assert_produced_events("de.log", auto_flush_interval + 0.1) do
+          # now the events are generated
+          expect(events[1]).to match_path_and_line("fr.log", lines["fr.log"])
+          expect(events[2]).to match_path_and_line("de.log", lines["de.log"])
+        end
+      end
+    end
+
+    context "mode: next, when there are pauses between multiline file writes" do
+
+      let(:lines) do
+        { "en.log" => ["hello world++", "second line++", "third line"],
+          "fr.log" => ["Salut le Monde++", "deuxième ligne++", "troisième ligne"],
+          "de.log" => ["Hallo Welt"] }
+      end
+
+      it "auto-flushes events from the accumulated lines to the queue" do
+        config.update("patterns" => "\\+\\+$", "what" => "next", "negate" => false,
+          "auto_flush_interval" => auto_flush_interval)
+
+        assert_produced_events("en.log", auto_flush_interval + 0.1) do
+          # wait for auto_flush
+          expect(events[0]).to match_path_and_line("en.log", lines["en.log"])
+        end
+
+        expect(codec).to have_an_empty_buffer
+
+        assert_produced_events("de.log", auto_flush_interval - 0.3) do
+          # this file is read before auto-flush, thus last event is not flushed yet
+          # This differs from logstash-codec-multiline because of not emitting
+          # last received event even if not matched
+          expect(events.size).to eq(1)
+        end
+
+        codec.flush { |event| events << event } # flushing here releases the event
+        expect(events.size).to eq(2)
+        expect(events[1]).to match_path_and_line(nil, lines["de.log"]) # but path is not set when emitted by flush
+        expect(codec).to have_an_empty_buffer
+
+        assert_produced_events("fr.log", auto_flush_interval + 0.1) do
+          # wait for auto_flush
+          expect(events[2]).to match_path_and_line("fr.log", lines["fr.log"])
+        end
+      end
+    end
+  end
+end