logstash-codec-multiline 3.0.11 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b4b30bf975e6dfb49b71d847c4502335311ff408a52ed68e59e09e80d6ed222
-  data.tar.gz: 38c64d94b88610ec6ab7dee1664721f9d9fc2609231b91418971af3129f4fd2d
+  metadata.gz: a4ecbf8730e688adbacb1af57c98f7c5105b5677337b140a22b6ccb41e1468ea
+  data.tar.gz: 299167720ec0457af53b7f118b6e55bd6c078f6ac1b8e4978ca68872a04cbc7c
 SHA512:
-  metadata.gz: 6e013b8f23543488f6c3381be40e622025100ed78481175f955830235f752aface0b8912202e7a88fe2f825c3e6b559de22760ee18df7a71f471ac8309677989
-  data.tar.gz: 50bfbea034b309e1f808d6454ab0065c6c260b82637a0fd6a3b55012dfc68f2f1661c6c3afd4e92c640e5e68320892a20598ca272344d6a1e28dd223c9add911
+  metadata.gz: 798b7fe3f963ade7578c655a00ba2ba7ea602dced4e463b170d8d7bf8b9e0585bedda761f9ee5bbbcb8026d99e0cbc3dc22bdfc19abe063c5d3716c2b8e0fd03
+  data.tar.gz: b4758f3f75e0ad1e470800aff9ccf353f8f4493cbcd6b30f687ef3fa74973e5baf0aba65a25450b83b3269879065ab5878141914dd3c35998e0bad4ca631b8bd

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.0
+  - Feat: ECS compatibility [#69](https://github.com/logstash-plugins/logstash-codec-multiline/pull/69)
+
 ## 3.0.11
   - Fix: avoid long thread sleeps on codec close [#67](https://github.com/logstash-plugins/logstash-codec-multiline/pull/67)
 

data/docs/index.asciidoc

@@ -102,13 +102,14 @@ following line.
 
 
 [id="plugins-{type}s-{plugin}-options"]
-==== Multiline Codec Configuration Options
+==== Multiline codec configuration options
 
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-auto_flush_interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-charset>> |<<string,string>>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-max_bytes>> |<<bytes,bytes>>|No
 | <<plugins-{type}s-{plugin}-max_lines>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-multiline_tag>> |<<string,string>>|No
@@ -144,6 +145,19 @@ or in another character set other than `UTF-8`.
 
 This only affects "plain" format logs since JSON is `UTF-8` already.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: plugin only sets the `message` field
+    ** `v1`,`v8`: Elastic Common Schema compliant behavior (`[event][original]` is also added)
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+
 [id="plugins-{type}s-{plugin}-max_bytes"]
 ===== `max_bytes` 
 

data/lib/logstash/codecs/multiline.rb

@@ -3,6 +3,8 @@ require "logstash/codecs/base"
 require "logstash/util/charset"
 require "logstash/timestamp"
 require "logstash/codecs/auto_flush"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 
 # The multiline codec will collapse multiline messages and merge them into a
 # single event.
@@ -85,6 +87,10 @@ require "logstash/codecs/auto_flush"
 # following line.
 #
 module LogStash module Codecs class Multiline < LogStash::Codecs::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
   config_name "multiline"
 
   # The regular expression to match.
@@ -141,6 +147,13 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
 
   public
 
+
+  def initialize(*params)
+    super
+
+    @original_field = ecs_select[disabled: nil, v1: '[event][original]']
+  end
+
   def register
     require "grok-pure" # rubygem 'jls-grok'
     require 'logstash/patterns/core'
@@ -197,8 +210,8 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
     text = @converter.convert(text)
     text.split("\n").each do |line|
       match = @grok.match(line)
-      @logger.debug("Multiline", :pattern => @pattern, :text => line,
-                    :match => (match != false), :negate => @negate)
+      @logger.debug? && @logger.debug("Multiline", :text => line, :pattern => @pattern,
+                                      :match => (match != false), :negate => @negate)
 
       # Add negate option
       match = (match and !@negate) || (!match and @negate)
@@ -214,9 +227,9 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
   def flush(&block)
     if block_given? && @buffer.any?
       no_error = true
-      events = merge_events
+      event = merge_events
       begin
-        yield events
+        yield event
       rescue ::Exception => e
         # need to rescue everything
         # likliest cause: backpressure or timeout by exception
@@ -237,7 +250,9 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
   end
 
   def merge_events
-    event = LogStash::Event.new(LogStash::Event::TIMESTAMP => @time, "message" => @buffer.join(NL))
+    message = @buffer.join(NL)
+    event = event_factory.new_event(LogStash::Event::TIMESTAMP => @time, "message" => message)
+    event.set @original_field, message.dup.freeze if @original_field
     event.tag @multiline_tag if !@multiline_tag.empty? && @buffer.size > 1
     event.tag "multiline_codec_max_bytes_reached" if over_maximum_bytes?
     event.tag "multiline_codec_max_lines_reached" if over_maximum_lines?

data/logstash-codec-multiline.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-multiline'
-  s.version         = '3.0.11'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Merges multiline messages into a single event"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-patterns-core'
   s.add_runtime_dependency 'jls-grok', '~> 0.11.1'
   s.add_runtime_dependency 'concurrent-ruby'

data/spec/codecs/multiline_spec.rb

@@ -7,8 +7,9 @@ require_relative "../supports/helpers.rb"
 # above helper also defines a subclass of Multiline
 # called MultilineRspec that exposes the internal buffer
 # and a Logger Mock
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
-describe LogStash::Codecs::Multiline do
+describe LogStash::Codecs::Multiline, :ecs_compatibility_support do
   context "#decode" do
     let(:config) { {} }
     let(:codec) { LogStash::Codecs::Multiline.new(config).tap {|c| c.register } }
@@ -23,17 +24,35 @@ describe LogStash::Codecs::Multiline do
       end
     end
 
-    it "should be able to handle multiline events with additional lines space-indented" do
-      config.update("pattern" => "^\\s", "what" => "previous")
-      lines = [ "hello world", "   second line", "another first line" ]
-      line_producer.call(lines)
-      codec.flush { |e| events << e }
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
+
+      let(:config) { super().merge "pattern" => "^\\s", "what" => "previous" }
+      let(:lines) { [ "hello world", "   second line", "another first line" ] }
+
+      let(:expected_first_multiline) { "hello world\n   second line" }
+
+      before do
+        line_producer.call(lines)
+        codec.flush { |e| events << e }
+      end
+
+      it "should be able to handle multiline events with additional lines space-indented" do
+        expect(events.size).to eq(2)
+        expect(events[0].get("message")).to eql expected_first_multiline
+        expect(events[0].get("tags")).to include("multiline")
+        expect(events[1].get("message")).to eql "another first line"
+        expect(events[1].get("tags")).to be_nil
+      end
+
+      it "sets event.original in ECS mode" do
+        expect(events.size).to eq(2)
+        expect(events[0].get("[event][original]")).to eql expected_first_multiline
+      end if ecs_select.active_mode != :disabled
 
-      expect(events.size).to eq(2)
-      expect(events[0].get("message")).to eq "hello world\n   second line"
-      expect(events[0].get("tags")).to include("multiline")
-      expect(events[1].get("message")).to eq "another first line"
-      expect(events[1].get("tags")).to be_nil
     end
 
     it "should allow custom tag added to multiline events" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-multiline
 version: !ruby/object:Gem::Version
-  version: 3.0.11
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -30,6 +30,34 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-event_support
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -37,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-patterns-core
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -51,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.11.1
   name: jls-grok
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -65,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -79,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,8 +121,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: insist
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -145,8 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
 summary: Merges multiline messages into a single event