logstash-codec-multiline 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92893cfcb592a522435a2dbded2f18d00ccdb9f1
-  data.tar.gz: 6f8c081d02ace21b428c25fdcfb6034ca793eac3
+  metadata.gz: 40099b6b6fa37c2e7eada7dd7bd49097f67267ba
+  data.tar.gz: d0fb10c0893b4a736bd2913c9b8b9207fc3a9536
 SHA512:
-  metadata.gz: 5770dc96a99122f8884b8e09595607554c4cfeb9abd98207177f8db8f7ec6519777dc9b9320ca01896ef1533e35f30c09f47d89863084be0c58a63caceece602
-  data.tar.gz: 8d62c67ee9c287ad77912367be4fdcd1bac28edde23ae7b500dc66e0167928d029049b00cb4437290cef52e239f5c7a10f6e2f9733ca838b3a41e5f4eef184ec
+  metadata.gz: 9847d7a7c4be62cec6527062cbf0ec663015221c0f27ec1e54238622d4d1fd96d52d28c1263be91848b83f90d4852975f760611d7eb90a377fc318dbd42f3c57
+  data.tar.gz: c097ad89ff4b7853f016c75d7e8ac4270dfe83e766a4df88ce23fc93b45ba46ccba94a4cd1a616e0fac9809b11b9359f5c48bdcc0583e626c46f9f04053d6f4a

data/lib/logstash/codecs/multiline.rb

@@ -114,10 +114,23 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
   # to events that actually have multiple lines in them.
   config :multiline_tag, :validate => :string, :default => "multiline"
 
+  # The accumulation of events can make logstash exit with an out of memory error
+  # if event boundaries are not correctly defined. This settings make sure to flush
+  # multiline events after reaching a number of lines, it is used in combination
+  # max_bytes.
+  config :max_lines, :validate => :number, :default => 500
+
+  # The accumulation of events can make logstash exit with an out of memory error
+  # if event boundaries are not correctly defined. This settings make sure to flush
+  # multiline events after reaching a number of bytes, it is used in combination
+  # max_lines.
+  config :max_bytes, :validate => :bytes, :default => "10 MiB"
+
   public
   def register
     require "grok-pure" # rubygem 'jls-grok'
     require 'logstash/patterns/core'
+
     # Detect if we are running from a jarfile, pick the right path.
     patterns_path = []
     patterns_path += [LogStash::Patterns::Core.path]
@@ -139,14 +152,14 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
     @grok.compile(@pattern)
     @logger.debug("Registered multiline plugin", :type => @type, :config => @config)
 
-    @buffer = []
+    reset_buffer
+
     @handler = method("do_#{@what}".to_sym)
 
     @converter = LogStash::Util::Charset.new(@charset)
     @converter.logger = @logger
   end # def register
 
-  public
   def decode(text, &block)
     text = @converter.convert(text)
 
@@ -163,31 +176,52 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
 
   def buffer(text)
     @time = LogStash::Timestamp.now if @buffer.empty?
+    @buffer_bytes += text.bytesize
     @buffer << text
   end
 
   def flush(&block)
-    if @buffer.any?
-      event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
-      # Tag multiline events
-      event.tag @multiline_tag if @multiline_tag && @buffer.size > 1
-
-      yield event
-      @buffer = []
+    if @buffer.any? 
+      yield merge_events
+      reset_buffer
     end
   end
 
+  def merge_events
+    event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
+    event.tag @multiline_tag if @multiline_tag && @buffer.size > 1
+    event.tag "multiline_codec_max_bytes_reached" if over_maximun_bytes?
+    event.tag "multiline_codec_max_lines_reached" if over_maximun_lines?
+    event
+  end
+
+  def reset_buffer
+    @buffer = []
+    @buffer_bytes = 0
+  end
+
   def do_next(text, matched, &block)
     buffer(text)
-    flush(&block) if !matched
+    flush(&block) if !matched || buffer_over_limits?
   end
 
   def do_previous(text, matched, &block)
-    flush(&block) if !matched
+    flush(&block) if !matched || buffer_over_limits?
     buffer(text)
   end
 
-  public
+  def over_maximun_lines?
+    @buffer.size > @max_lines 
+  end
+
+  def over_maximun_bytes?
+    @buffer_bytes >= @max_bytes
+  end
+
+  def buffer_over_limits?
+    over_maximun_lines? || over_maximun_bytes?
+  end
+
   def encode(event)
     # Nothing to do.
     @on_event.call(event, event)

data/logstash-codec-multiline.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-multiline'
-  s.version         = '0.1.6'
+  s.version         = '0.1.7'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The multiline codec will collapse multiline messages and merge them into a single event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/multiline_spec.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
-
 require "logstash/codecs/multiline"
 require "logstash/event"
 require "insist"
+require_relative "../supports/helpers.rb"
 
 describe LogStash::Codecs::Multiline do
   context "#decode" do
@@ -171,4 +171,51 @@ describe LogStash::Codecs::Multiline do
 
     end
   end
+
+  context "with non closed multiline events" do
+    let(:random_number_of_events) { rand(300..1000) }
+    let(:sample_event) { "- Sample event" }
+    let(:events) { decode_events }
+    let(:unmerged_events_count) { events.collect { |event| event["message"].split(LogStash::Codecs::Multiline::NL).size }.inject(&:+) }
+
+    context "break on maximum_lines" do
+      let(:max_lines) { rand(10..100) }
+      let(:options) {
+        {
+          "pattern" => "^-",
+          "what" => "previous", 
+          "max_lines" => max_lines,
+          "max_bytes" => "2 mb"
+        }
+      }
+
+      it "flushes on a maximum lines" do
+        expect(unmerged_events_count).to eq(random_number_of_events)
+      end
+
+      it "tags the event" do
+        expect(events.first["tags"]).to include("multiline_codec_max_lines_reached")
+      end
+    end
+
+    context "break on maximum bytes" do
+      let(:max_bytes) { rand(30..100) }
+      let(:options) {
+        {
+          "pattern" => "^-",
+          "what" => "previous", 
+          "max_lines" => 20000,
+          "max_bytes" => max_bytes
+        }
+      }
+
+      it "flushes on a maximum bytes size" do
+        expect(unmerged_events_count).to eq(random_number_of_events)
+      end
+
+      it "tags the event" do
+        expect(events.first["tags"]).to include("multiline_codec_max_bytes_reached")
+      end
+    end
+  end
 end

data/spec/supports/helpers.rb

@@ -0,0 +1,14 @@
+require "lib/logstash/codecs/multiline"
+
+def decode_events
+  multiline =  LogStash::Codecs::Multiline.new(options)
+
+  events = []
+  random_number_of_events.times do |n|
+    multiline.decode(sample_event) { |event| events << event }
+  end
+
+  # Grab the in-memory-event
+  multiline.flush { |event| events << event }
+  events
+end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-multiline
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-26 00:00:00.000000000 Z
+date: 2015-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,48 +28,50 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-patterns-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-patterns-core
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jls-grok
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.11.1
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.11.1
-  name: jls-grok
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.11.1
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elasticsearch.com
 executables: []
@@ -87,6 +87,7 @@ files:
 - lib/logstash/codecs/multiline.rb
 - logstash-codec-multiline.gemspec
 - spec/codecs/multiline_spec.rb
+- spec/supports/helpers.rb
 homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -109,9 +110,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: The multiline codec will collapse multiline messages and merge them into a single event.
 test_files:
 - spec/codecs/multiline_spec.rb
+- spec/supports/helpers.rb