logstash-filter-multiline 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebb52611156af53a3586ad87b94264defff21b4d
-  data.tar.gz: d5678da5ec5b5eb35ae1c57047bb4cb08ccb318f
+  metadata.gz: c8be8f44cc90d0771e53b9c642ccd5456c81ef7d
+  data.tar.gz: 57cb66091186f91f1b0d7df4292b8dc38e2b4386
 SHA512:
-  metadata.gz: 0c6c2c36a07e1b30119beb9e76a9271c898a5e2d2259f349aaed5e941beaef81258f6fe02af4eb17844820fdfa9eda4a20ca20afa2dd289a6907d11ba6829c14
-  data.tar.gz: e6f495b4efa8cec399b9ce6aab65fdd1edd7cd2c5ba952c703d949fa65b2899c94b6646c84c947228963891877ece5730805e182cbb72d291533c8cf5c921304
+  metadata.gz: 240769acdc6b6efce550a9d052c124015d307de079c84a689fbaf2a906e3ca5f029c52d6ce793d95e4f7ee7721a0699940112cf13891c097196973fa9b59301c
+  data.tar.gz: 07af22be599d635feb5922a6a3b4997f70b6e8ee5990aaaf4cfb6007909e36b3abbfdd3c1d77d903cb41cad6a35bc315be54402e3c2c0fabf0be55e82677cb14

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+Copyright (c) 2012-2015 Elasticsearch <http://www.elasticsearch.org>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -0,0 +1,95 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+```sh
+bundle exec rspec
+```
+
+The Logstash code required to run the tests/specs is specified in the `Gemfile` by the line similar to:
+```ruby
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
+```
+To test against another version or a local Logstash, edit the `Gemfile` to specify an alternative location, for example:
+```ruby
+gem "logstash", :github => "elasticsearch/logstash", :ref => "master"
+```
+```ruby
+gem "logstash", :path => "/your/local/logstash"
+```
+
+Then update your dependencies and run your tests:
+
+```sh
+bundle install
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `tools/Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Update Logstash dependencies
+```sh
+rake vendor:gems
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to me that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/multiline.rb

@@ -63,7 +63,12 @@ require "set"
 class LogStash::Filters::Multiline < LogStash::Filters::Base
 
   config_name "multiline"
-  milestone 3
+
+  # The field name to execute the pattern match on.
+  config :source, :validate => :string, :default => "message"
+
+  # Allow duplcate values on the source field.
+  config :allow_duplicates, :validate => :boolean, :default => true
 
   # The regular expression to match.
   config :pattern, :validate => :string, :required => true
@@ -109,8 +114,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   # Optional.
   config :periodic_flush, :validate => :boolean, :default => true
 
-
-  # Detect if we are running from a jarfile, pick the right path.
+  # Register default pattern paths
   @@patterns_path = Set.new
   @@patterns_path += [LogStash::Patterns::Core.path]
 
@@ -137,10 +141,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
 
     @patterns_dir = @@patterns_path.to_a + @patterns_dir
     @patterns_dir.each do |path|
-      if File.directory?(path)
-        path = File.join(path, "*")
-      end
-
+      path = File.join(path, "*") if File.directory?(path)
       Dir.glob(path).each do |file|
         @logger.info("Grok loading patterns from file", :path => file)
         @grok.add_patterns_from_file(file)
@@ -166,17 +167,14 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   def filter(event)
     return unless filter?(event)
 
-    match = event["message"].is_a?(Array) ? @grok.match(event["message"].first) : @grok.match(event["message"])
-    match = (match and !@negate) || (!match and @negate) # add negate option
+    match = event[@source].is_a?(Array) ? @grok.match(event[@source].first) : @grok.match(event[@source])
+    match = (match && !@negate) || (!match && @negate) # add negate option
 
-    @logger.debug? && @logger.debug("Multiline", :pattern => @pattern, :message => event["message"], :match => match, :negate => @negate)
+    @logger.debug? && @logger.debug("Multiline", :pattern => @pattern, :message => event[@source], :match => match, :negate => @negate)
 
     multiline_filter!(event, match)
 
-    unless event.cancelled?
-      collapse_event!(event)
-      filter_matched(event) if match
-    end
+    filter_matched(event) unless event.cancelled?
   end # def filter
 
   # flush any pending messages
@@ -186,25 +184,28 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   # @return [Array<LogStash::Event>] list of flushed events
   public
   def flush(options = {})
-    expired = nil
-
     # note that thread safety concerns are not necessary here because the multiline filter
-    # is not thread safe thus cannot be run in multiple folterworker threads and flushing
+    # is not thread safe thus cannot be run in multiple filterworker threads and flushing
     # is called by the same thread
 
     # select all expired events from the @pending hash into a new expired hash
     # if :final flush then select all events
-    expired = @pending.inject({}) do |r, (key, event)|
-      age = Time.now - Array(event["@timestamp"]).first.time
-      r[key] = event if (age >= @max_age) || options[:final]
-      r
+    expired = @pending.inject({}) do |result, (key, events)|
+      unless events.empty?
+        age = Time.now - events.first["@timestamp"].time
+        result[key] = events if (age >= @max_age) || options[:final]
+      end
+      result
     end
 
-    # delete expired items from @pending hash
-    expired.each{|key, event| @pending.delete(key)}
-
-    # return list of uncancelled and collapsed expired events
-    expired.map{|key, event| event.uncancel; collapse_event!(event)}
+    # return list of uncancelled expired events
+    expired.map do |key, events|
+      @pending.delete(key)
+      event = merge(events)
+      event.uncancel
+      filter_matched(event)
+      event
+    end
   end # def flush
 
   public
@@ -216,29 +217,24 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
 
   def previous_filter!(event, match)
     key = event.sprintf(@stream_identity)
-
-    pending = @pending[key]
+    pending = @pending[key] ||= []
 
     if match
+      # previous previous line is part of this event. append it to the event and cancel it
       event.tag(MULTILINE_TAG)
-      # previous previous line is part of this event.
-      # append it to the event and cancel it
-      if pending
-        pending.append(event)
-      else
-        @pending[key] = event
-      end
+      pending << event
       event.cancel
     else
-      # this line is not part of the previous event
-      # if we have a pending event, it's done, send it.
+      # this line is not part of the previous event if we have a pending event, it's done, send it.
       # put the current event into pending
-      if pending
+      unless pending.empty?
         tmp = event.to_hash
-        event.overwrite(pending)
-        @pending[key] = LogStash::Event.new(tmp)
+        event.overwrite(merge(pending))
+        pending.clear # avoid array creation
+        pending << LogStash::Event.new(tmp)
       else
-        @pending[key] = event
+        pending.clear # avoid array creation
+        pending << event
         event.cancel
       end
     end # if match
@@ -246,35 +242,66 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
 
   def next_filter!(event, match)
     key = event.sprintf(@stream_identity)
-
-    # protect @pending for race condition between the flush thread and the worker thread
-    pending = @pending[key]
+    pending = @pending[key] ||= []
 
     if match
+      # this line is part of a multiline event, the next line will be part, too, put it into pending.
       event.tag(MULTILINE_TAG)
-      # this line is part of a multiline event, the next
-      # line will be part, too, put it into pending.
-      if pending
-        pending.append(event)
-      else
-        @pending[key] = event
-      end
+      pending << event
       event.cancel
     else
-      # if we have something in pending, join it with this message
-      # and send it. otherwise, this is a new message and not part of
-      # multiline, send it.
-      if pending
-        pending.append(event)
-        event.overwrite(pending)
-        @pending.delete(key)
+      # if we have something in pending, join it with this message and send it.
+      # otherwise, this is a new message and not part of multiline, send it.
+      unless pending.empty?
+        event.overwrite(merge(pending << event))
+        pending.clear
       end
     end # if match
   end
 
-  def collapse_event!(event)
-    event["message"] = event["message"].join("\n") if event["message"].is_a?(Array)
-    event.timestamp = event.timestamp.first if event.timestamp.is_a?(Array)
-    event
+  # merge a list of events. @timestamp for the resulting merged event will be from
+  # the "oldest" (events.first). all @source fields will be deduplicated depending
+  # on @allow_duplicates and joined with \n. all other fields will be deduplicated.
+  # @param events [Array<Event>] the list of events to merge
+  # @return [Event] the resulting merged event
+  def merge(events)
+    dups_key = @allow_duplicates ? @source : nil
+
+    data = events.inject({}) do |result, event|
+      self.class.event_hash_merge!(result, event.to_hash_with_metadata, dups_key)
+    end
+
+    # merged event @timestamp is from first event in sequence
+    data["@timestamp"] = Array(data["@timestamp"]).first
+    # collapse all @source field values
+    data[@source] = Array(data[@source]).join("\n")
+    LogStash::Event.new(data)
   end
+
+  # merge two events data hash, src into dst and handle duplicate values for dups_key
+  # @param dst [Hash] the event to merge into, dst will be mutated
+  # @param src [Hash] the event to merge in dst
+  # @param dups_key [String] the field key to keep duplicate values
+  # @return [Hash] mutated dst
+  def self.event_hash_merge!(dst, src, dups_key = nil)
+    src.each do |key, svalue|
+      dst[key] = if dst.has_key?(key)
+        dvalue = dst[key]
+
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          event_hash_merge!(dvalue, svalue, dups_key)
+        else
+          v = (dups_key == key) ? Array(dvalue) + Array(svalue) : Array(dvalue) | Array(svalue)
+          # the v result is always an Array, if none of the fields were arrays and there is a
+          # single value in the array, return the value, not the array
+          dvalue.is_a?(Array) || svalue.is_a?(Array) ? v : (v.size == 1 ? v.first : v)
+        end
+      else
+        svalue
+      end
+    end
+
+    dst
+  end # def self.hash_merge
+
 end # class LogStash::Filters::Multiline

data/logstash-filter-multiline.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-multiline'
-  s.version         = '0.1.2'
+  s.version         = '0.1.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter will collapse multiline messages from a single source into one Logstash event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/multiline_spec.rb

@@ -87,6 +87,7 @@ describe LogStash::Filters::Multiline do
     end
   end
 
+
   describe "multiline add/remove tags and fields only when matched" do
     config <<-CONFIG
       filter {
@@ -108,9 +109,9 @@ describe LogStash::Filters::Multiline do
       insist { subject.size } == 2
 
       subject.each do |s|
-        insist { s["tags"].include?("nope")  } == false
-        insist { s["tags"].include?("dummy") } == true
-        insist { s.include?("dummy2") } == false
+        insist { s["tags"].include?("nope")  } == true
+        insist { s["tags"].include?("dummy") } == false
+        insist { s.include?("dummy2") } == true
       end
     end
   end
@@ -121,7 +122,6 @@ describe LogStash::Filters::Multiline do
         multiline {
           pattern => "^\s"
           what => "next"
-          add_tag => ["multi"]
         }
       }
     CONFIG
@@ -138,7 +138,6 @@ describe LogStash::Filters::Multiline do
         multiline {
           pattern => "^\s"
           what => "next"
-          add_tag => ["multi"]
         }
       }
     CONFIG
@@ -150,4 +149,100 @@ describe LogStash::Filters::Multiline do
       insist { subject[1]["message"] } == "  match2\nnomatch2"
     end
   end
+
+  describe "keep duplicates by default on message field" do
+    config <<-CONFIG
+      filter {
+        multiline {
+          pattern => "^\s"
+          what => "next"
+        }
+      }
+    CONFIG
+
+    sample ["  match1", "  match1", "nomatch1", "  1match2", "  2match2", "  1match2", "nomatch2"] do
+      expect(subject).to be_a(Array)
+      insist { subject.size } == 2
+      insist { subject[0]["message"] } == "  match1\n  match1\nnomatch1"
+      insist { subject[1]["message"] } == "  1match2\n  2match2\n  1match2\nnomatch2"
+    end
+  end
+
+  describe "remove duplicates using :allow_duplicates => false on message field" do
+    config <<-CONFIG
+      filter {
+        multiline {
+          allow_duplicates => false
+          pattern => "^\s"
+          what => "next"
+        }
+      }
+    CONFIG
+
+    sample ["  match1", "  match1", "nomatch1", "  1match2", "  2match2", "  1match2", "nomatch2"] do
+      expect(subject).to be_a(Array)
+      insist { subject.size } == 2
+      insist { subject[0]["message"] } == "  match1\nnomatch1"
+      insist { subject[1]["message"] } == "  1match2\n  2match2\nnomatch2"
+    end
+  end
+
+  describe "keep duplicates only on @source field" do
+    config <<-CONFIG
+      filter {
+        multiline {
+          source => "foo"
+          pattern => "^\s"
+          what => "next"
+        }
+      }
+    CONFIG
+
+    sample [
+      {"message" => "bar", "foo" => "  match1"},
+      {"message" => "bar", "foo" => "  match1"},
+      {"message" => "baz", "foo" => "nomatch1"},
+      {"foo" => "  1match2"},
+      {"foo" => "  2match2"},
+      {"foo" => "  1match2"},
+      {"foo" => "nomatch2"}
+    ] do
+      expect(subject).to be_a(Array)
+      insist { subject.size } == 2
+      insist { subject[0]["foo"] } == "  match1\n  match1\nnomatch1"
+      insist { subject[0]["message"] } == ["bar", "baz"]
+      insist { subject[1]["foo"] } == "  1match2\n  2match2\n  1match2\nnomatch2"
+    end
+  end
+
+  describe "fix dropped duplicated lines" do
+    # as reported in https://github.com/logstash-plugins/logstash-filter-multiline/issues/3
+
+    config <<-CONFIG
+      filter {
+        multiline {
+          pattern => "^START"
+          what => "previous"
+          negate=> true
+        }
+      }
+    CONFIG
+
+    messages = [
+      "START",
+      "<Tag1 Id=\"1\">",
+        "<Tag2>Foo</Tag2>",
+      "</Tag1>",
+      "<Tag1 Id=\"2\">",
+        "<Tag2>Foo</Tag2>",
+      "</Tag1>",
+      "START",
+    ]
+    sample messages do
+      expect(subject).to be_a(Array)
+      insist { subject.size } == 2
+      insist { subject[0]["message"] } == messages[0..-2].join("\n")
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-multiline
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-19 00:00:00.000000000 Z
+date: 2015-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash
@@ -95,6 +95,7 @@ files:
 - .gitignore
 - Gemfile
 - LICENSE
+- README.md
 - Rakefile
 - lib/logstash/filters/multiline.rb
 - logstash-filter-multiline.gemspec
@@ -121,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.4
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: This filter will collapse multiline messages from a single source into one Logstash event.