logstash-filter-dissect 1.1.4 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21b60b83ce9a628a4c1879f0cffa16b3266595a5858fa5874998d3ca971718a6
-  data.tar.gz: e6b49de027860dcd0f6316ff93632aa8dd33f2bc9323258babfdbe867ea23bb3
+  metadata.gz: 0426a95966922ea538ca2a8d48bcdb6f60193ad216a15ebddb6a816919de6d8b
+  data.tar.gz: 45e44ad23f1d46f9cba7a088e5842df61ed6054e6971781ebc5f2f303241893b
 SHA512:
-  metadata.gz: ee8590d5715087e826fe63853919972602161ddeddb6055a3585aef0dc6c2a9ce5ca6c1bcdc575703fb145abf41381ec1e460b5fa2c224fea16b0325555c7c47
-  data.tar.gz: 562fc016a5fe3e6312ecc1949a53764ebff4ae0a67cdfee903ca4c0f021ecdecd90fcedb690aba3acb35edb00ebf81960236315d383f33c12ef70a5b146deaaf
+  metadata.gz: 95749fcd548bf4412dc20f08d0a7aea50e5ca6e4f75154ed570014a927d6e6e95eb85c8b60e2293440e3049fd566ed9b0552cb121f06bdd4a384a10617b31ce7
+  data.tar.gz: 1763b8ee0c59bb7b2a25003146f698f2f15cc4cc9086026eded0ae9dbe1b6e22ff6c866fe7076c781cc699979b86150382f277ae5f442927267f1d1e8e08452e

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.2.0
+  - Fix Trailing Delimiters requires a false field. A skip field is
+  automatically added when a final delimiter is detected in the dissect pattern.
+  This requires that strict delimiter finding is enforced  - meaning a "no match"
+  results if every delimiter is not found in exactly the declared order
+  [Issue #22](https://github.com/logstash-plugins/logstash-filter-dissect/issues/22)
+
 ## 1.1.4
   - Replace v1.1.3 as it packaged the v1.1.1 jar and therefore does not have the fixes below
   - Yank v1.1.3 from rubygems.org

data/VERSION

@@ -1 +1 @@
-1.1.4
+1.2.0

data/docs/index.asciidoc

@@ -34,7 +34,7 @@ The dissection is described using a set of `%{}` sections:
 %{a} - %{b} - %{c}
 ....
 
-A *field* is the text from `%` to `}` inclusive.
+A *field* is the text from `%{` to `}` inclusive.
 
 A *delimiter* is the text between a `}` and next `%{` characters.
 
@@ -240,8 +240,8 @@ For example
 filter {
   dissect {
     convert_datatype => {
-      cpu => "float"
-      code => "int"
+      "cpu" => "float"
+      "code" => "int"
     }
   }
 }

data/lib/jruby-dissect-library_jars.rb

@@ -1,4 +1,4 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.logstash.dissect', 'jruby-dissect-library', '1.1.4')
+require_jar('org.logstash.dissect', 'jruby-dissect-library', '1.2.0')

data/logstash-filter-dissect.gemspec

@@ -1,8 +1,6 @@
-DISSECT_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(DISSECT_VERSION)
-
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-dissect'
-  s.version = DISSECT_VERSION
+  s.version = '1.2.0' # version will be checked against VERSION file by `rake vendor`
   s.licenses = ['Apache License (2.0)']
   s.summary = "Extracts unstructured event data into fields using delimiters"
   s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/dissect_spec.rb

@@ -328,4 +328,25 @@ describe LogStash::Filters::Dissect do
       end
     end
   end
+
+  describe "Compatibility suite" do
+    tests = LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "/../fixtures/dissect_tests.json")))
+    tests.each do |test|
+      describe test["name"] do
+        let(:options) { { "mapping" => { "message" => test["tok"] } } }
+        subject { described_class.new(options) }
+        let(:event) { LogStash::Event.new({ "message" => test["msg"] }) }
+        before(:each) do
+          subject.register
+          subject.filter(event)
+        end
+
+        it "should dissect properly" do
+          test["expected"].each do |k, v|
+            expect(event.get(k)).to eq(v)
+          end
+        end
+      end
+    end
+end
 end

data/spec/fixtures/dissect_tests.json

@@ -0,0 +1,157 @@
+[
+  {
+    "name": "Complex stack trace",
+    "tok": "%{day}-%{month}-%{year} %{hour} %{severity} [%{thread_id}] %{origin} %{message}",
+    "msg": "18-Apr-2018 06:53:20.411 INFO [http-nio-8080-exec-1] org.apache.coyote.http11.Http11Processor.service Error parsing HTTP request header\n Note: further occurrences of HTTP header parsing errors will be logged at DEBUG level.\n java.lang.IllegalArgumentException: Invalid character found in method name. HTTP method names must be tokens\n    at org.apache.coyote.http11.Http11InputBuffer.parseRequestLine(Http11InputBuffer.java:426)\n    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:687)\n    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)\n    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:790)\n    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1459)\n    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)\n    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)\n    at java.lang.Thread.run(Thread.java:748)",
+    "expected": {
+      "day": "18",
+      "hour": "06:53:20.411",
+      "message": "Error parsing HTTP request header\n Note: further occurrences of HTTP header parsing errors will be logged at DEBUG level.\n java.lang.IllegalArgumentException: Invalid character found in method name. HTTP method names must be tokens\n    at org.apache.coyote.http11.Http11InputBuffer.parseRequestLine(Http11InputBuffer.java:426)\n    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:687)\n    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)\n    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:790)\n    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1459)\n    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)\n    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)\n    at java.lang.Thread.run(Thread.java:748)",
+      "month": "Apr",
+      "origin": "org.apache.coyote.http11.Http11Processor.service",
+      "severity": "INFO",
+      "thread_id": "http-nio-8080-exec-1",
+      "year": "2018"
+    },
+    "skip": false
+  },
+  {
+    "name": "simple dissect",
+    "tok": "%{key}",
+    "msg": "foobar",
+    "expected": {
+      "key": "foobar"
+    },
+    "skip": false
+  },
+  {
+    "name": "dissect two replacement",
+    "tok": "%{key1} %{key2}",
+    "msg": "foo bar",
+    "expected": {
+      "key1": "foo",
+      "key2": "bar"
+    },
+    "skip": false
+  },
+  {
+    "name": "one level dissect not end of string",
+    "tok": "/var/%{key}/log",
+    "msg": "/var/foobar/log",
+    "expected": {
+      "key": "foobar"
+    },
+    "skip": false
+  },
+  {
+    "name": "one level dissect",
+    "tok": "/var/%{key}",
+    "msg": "/var/foobar/log",
+    "expected": {
+      "key": "foobar/log"
+    },
+    "skip": false
+  },
+  {
+    "name": "multiple keys dissect end of string",
+    "tok": "/var/%{key}/log/%{key1}",
+    "msg": "/var/foobar/log/apache",
+    "expected": {
+      "key": "foobar",
+      "key1": "apache"
+    },
+    "skip": false
+  },
+  {
+    "name": "multiple keys not end of string",
+    "tok": "/var/%{key}/log/%{key1}.log",
+    "msg": "/var/foobar/log/apache.log",
+    "expected": {
+      "key": "foobar",
+      "key1": "apache"
+    },
+    "skip": false
+  },
+  {
+    "name": "simple ordered",
+    "tok": "%{+key/3} %{+key/1} %{+key/2}",
+    "msg": "1 2 3",
+    "expected": {
+      "key": "2 3 1"
+    },
+    "skip": false
+  },
+  {
+    "name": "simple append",
+    "tok": "%{key}-%{+key}-%{+key}",
+    "msg": "1-2-3",
+    "expected": {
+      "key": "1-2-3"
+    },
+    "skip": false
+  },
+  {
+    "name": "indirect field",
+    "tok": "%{key} %{\u0026key}",
+    "msg": "hello world",
+    "expected": {
+      "hello": "world",
+      "key": "hello"
+    },
+    "skip": false
+  },
+  {
+    "name": "skip field",
+    "tok": "%{} %{key}",
+    "msg": "hello world",
+    "expected": {
+      "key": "world"
+    },
+    "skip": false
+  },
+  {
+    "name": "named skiped field with indirect",
+    "tok": "%{?key} %{\u0026key}",
+    "msg": "hello world",
+    "expected": {
+      "hello": "world"
+    },
+    "skip": false
+  },
+  {
+    "name": "missing fields",
+    "tok": "%{name},%{addr1},%{addr2},%{addr3},%{city},%{zip}",
+    "msg": "Jane Doe,4321 Fifth Avenue,,,New York,87432",
+    "expected": {
+      "addr1": "4321 Fifth Avenue",
+      "addr2": "",
+      "addr3": "",
+      "city": "New York",
+      "name": "Jane Doe",
+      "zip": "87432"
+    },
+    "skip": false
+  },
+  {
+    "name": "ignore right padding",
+    "tok": "%{id} %{function-\u003e} %{server}",
+    "msg": "00000043 ViewReceive     machine-321",
+    "expected": {
+      "function": "ViewReceive",
+      "id": "00000043",
+      "server": "machine-321"
+    },
+    "skip": false
+  },
+  {
+    "name": "ignore left padding",
+    "tok": "%{id-\u003e} %{function} %{server}",
+    "msg": "00000043    ViewReceive machine-321",
+    "expected": {
+      "function": "ViewReceive",
+      "id": "00000043",
+      "server": "machine-321"
+    },
+    "skip": false
+  }
+]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-dissect
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-15 00:00:00.000000000 Z
+date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,8 +94,9 @@ files:
 - lib/logstash/filters/dissect.rb
 - logstash-filter-dissect.gemspec
 - spec/filters/dissect_spec.rb
+- spec/fixtures/dissect_tests.json
 - spec/spec_helper.rb
-- vendor/jars/org/logstash/dissect/jruby-dissect-library/1.1.4/jruby-dissect-library-1.1.4.jar
+- vendor/jars/org/logstash/dissect/jruby-dissect-library/1.2.0/jruby-dissect-library-1.2.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -125,4 +126,5 @@ specification_version: 4
 summary: Extracts unstructured event data into fields using delimiters
 test_files:
 - spec/filters/dissect_spec.rb
+- spec/fixtures/dissect_tests.json
 - spec/spec_helper.rb