fluent-plugin-grok-parser 2.6.2 → 2.6.3

data/test/test_grok_parser.rb

@@ -53,24 +53,46 @@ class GrokParserTest < ::Test::Unit::TestCase
                                {"mac_address" => "DEAD.BEEF.1234", "ip_address" => "127.0.0.1"})
   end
 
-  test "complex pattern" do
-    internal_test_grok_pattern("%{COMBINEDAPACHELOG}", '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
-                                str2time("28/Feb/2013:12:00:00 +0900", "%d/%b/%Y:%H:%M:%S %z"),
-                                {
-                                  "clientip"    => "127.0.0.1",
-                                  "ident"       => "192.168.0.1",
-                                  "auth"        => "-",
-                                  "verb"        => "GET",
-                                  "request"     => "/",
-                                  "httpversion" => "1.1",
-                                  "response"    => "200",
-                                  "bytes"       => "777",
-                                  "referrer"    => "\"-\"",
-                                  "agent"       => "\"Opera/12.0\""
-                                },
-                                "time_key" => "timestamp",
-                                "time_format" => "%d/%b/%Y:%H:%M:%S %z"
-                              )
+  sub_test_case "complex pattern w/ grok_pattern_series" do
+    test "legacy" do
+      internal_test_grok_pattern("%{HTTPD_COMBINEDLOG}", '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
+                                  str2time("28/Feb/2013:12:00:00 +0900", "%d/%b/%Y:%H:%M:%S %z"),
+                                  {
+                                    "clientip"    => "127.0.0.1",
+                                    "ident"       => "192.168.0.1",
+                                    "auth"        => "-",
+                                    "verb"        => "GET",
+                                    "request"     => "/",
+                                    "httpversion" => "1.1",
+                                    "response"    => "200",
+                                    "bytes"       => "777",
+                                    "referrer"    => "\"-\"",
+                                    "agent"       => "\"Opera/12.0\""
+                                  },
+                                  "time_key" => "timestamp",
+                                  "time_format" => "%d/%b/%Y:%H:%M:%S %z",
+                                  "grok_pattern_series" => "legacy"
+                                )
+    end
+
+    test "ecs-v1" do
+      internal_test_grok_pattern("%{HTTPD_COMBINEDLOG}", '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
+                                  str2time("28/Feb/2013:12:00:00 +0900", "%d/%b/%Y:%H:%M:%S %z"),
+                                  {
+                                    "apache.access.user.identity" => "192.168.0.1",
+                                    "http.request.method"         => "GET",
+                                    "http.response.body.bytes"    => 777,
+                                    "http.response.status_code"   => 200,
+                                    "http.version"                => "1.1",
+                                    "source.address"              => "127.0.0.1",
+                                    "url.original"                => "/",
+                                    "user_agent.original"         => "Opera/12.0",
+                                  },
+                                  "time_key" => "timestamp",
+                                  "time_format" => "%d/%b/%Y:%H:%M:%S %z",
+                                  "grok_pattern_series" => "ecs-v1"
+                                )
+    end
   end
 
   test "custom pattern" do
@@ -334,29 +356,57 @@ class GrokParserTest < ::Test::Unit::TestCase
   end
 
   sub_test_case "grok section" do
-    test "complex pattern" do
-      d = create_driver(%[
-        <grok>
-          pattern %{COMBINEDAPACHELOG}
-          time_key timestamp
-          time_format %d/%b/%Y:%H:%M:%S %z
-        </grok>
-      ])
-      expected_record = {
-        "clientip"    => "127.0.0.1",
-        "ident"       => "192.168.0.1",
-        "auth"        => "-",
-        "verb"        => "GET",
-        "request"     => "/",
-        "httpversion" => "1.1",
-        "response"    => "200",
-        "bytes"       => "777",
-        "referrer"    => "\"-\"",
-        "agent"       => "\"Opera/12.0\""
-      }
-      d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
-        assert_equal(expected_record, record)
-        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
+    sub_test_case "complex pattern w/ grok_pattern_series" do
+      test "legacy" do
+        d = create_driver(%[
+          grok_pattern_series legacy
+          <grok>
+            pattern %{COMBINEDAPACHELOG}
+            time_key timestamp
+            time_format %d/%b/%Y:%H:%M:%S %z
+          </grok>
+        ])
+        expected_record = {
+          "clientip"    => "127.0.0.1",
+          "ident"       => "192.168.0.1",
+          "auth"        => "-",
+          "verb"        => "GET",
+          "request"     => "/",
+          "httpversion" => "1.1",
+          "response"    => "200",
+          "bytes"       => "777",
+          "referrer"    => "\"-\"",
+          "agent"       => "\"Opera/12.0\""
+        }
+        d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
+          assert_equal(expected_record, record)
+          assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
+        end
+      end
+
+      test "ecs-v1" do
+        d = create_driver(%[
+          grok_pattern_series ecs-v1
+          <grok>
+            pattern %{HTTPD_COMBINEDLOG}
+            time_key timestamp
+            time_format %d/%b/%Y:%H:%M:%S %z
+          </grok>
+        ])
+        expected_record = {
+          "apache.access.user.identity" => "192.168.0.1",
+          "http.request.method"         => "GET",
+          "http.response.body.bytes"    => 777,
+          "http.response.status_code"   => 200,
+          "http.version"                => "1.1",
+          "source.address"              => "127.0.0.1",
+          "url.original"                => "/",
+          "user_agent.original"         => "Opera/12.0"
+        }
+        d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
+          assert_equal(expected_record, record)
+          assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
+        end
       end
     end
 

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 2.6.2
+  version: 2.6.3
 platform: ruby
 authors:
 - kiyoto
 - Kenji Okimoto
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -73,7 +72,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description:
 email:
 - kiyoto@treasure-data.com
 - okimoto@clear-code.com
@@ -81,39 +79,62 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/unit-test.yml"
+- ".gitignore"
 - ".gitmodules"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
-- appveyor.yml
 - fluent-plugin-grok-parser.gemspec
 - lib/fluent/plugin/grok.rb
 - lib/fluent/plugin/parser_grok.rb
 - lib/fluent/plugin/parser_multiline_grok.rb
-- patterns/aws
-- patterns/bacula
-- patterns/bind
-- patterns/bro
-- patterns/exim
-- patterns/firewalls
-- patterns/grok-patterns
-- patterns/haproxy
-- patterns/httpd
-- patterns/java
-- patterns/junos
-- patterns/linux-syslog
-- patterns/maven
-- patterns/mcollective
-- patterns/mcollective-patterns
-- patterns/mongodb
-- patterns/nagios
-- patterns/postgresql
-- patterns/rails
-- patterns/redis
-- patterns/ruby
-- patterns/squid
+- patterns/ecs-v1/aws
+- patterns/ecs-v1/bacula
+- patterns/ecs-v1/bind
+- patterns/ecs-v1/bro
+- patterns/ecs-v1/exim
+- patterns/ecs-v1/firewalls
+- patterns/ecs-v1/grok-patterns
+- patterns/ecs-v1/haproxy
+- patterns/ecs-v1/httpd
+- patterns/ecs-v1/java
+- patterns/ecs-v1/junos
+- patterns/ecs-v1/linux-syslog
+- patterns/ecs-v1/maven
+- patterns/ecs-v1/mcollective
+- patterns/ecs-v1/mongodb
+- patterns/ecs-v1/nagios
+- patterns/ecs-v1/postgresql
+- patterns/ecs-v1/rails
+- patterns/ecs-v1/redis
+- patterns/ecs-v1/ruby
+- patterns/ecs-v1/squid
+- patterns/ecs-v1/zeek
+- patterns/legacy/aws
+- patterns/legacy/bacula
+- patterns/legacy/bind
+- patterns/legacy/bro
+- patterns/legacy/exim
+- patterns/legacy/firewalls
+- patterns/legacy/grok-patterns
+- patterns/legacy/haproxy
+- patterns/legacy/httpd
+- patterns/legacy/java
+- patterns/legacy/junos
+- patterns/legacy/linux-syslog
+- patterns/legacy/maven
+- patterns/legacy/mcollective
+- patterns/legacy/mcollective-patterns
+- patterns/legacy/mongodb
+- patterns/legacy/nagios
+- patterns/legacy/postgresql
+- patterns/legacy/rails
+- patterns/legacy/redis
+- patterns/legacy/ruby
+- patterns/legacy/squid
 - test/fixtures/my_pattern
 - test/helper.rb
 - test/test_grok_parser.rb
@@ -123,7 +144,6 @@ homepage: https://github.com/fluent/fluent-plugin-grok-parser
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -138,8 +158,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs
 test_files:

data/.travis.yml

@@ -1,7 +0,0 @@
-sudo: false
-language: ruby
-
-rvm:
-  - 2.4
-  - 2.5.3
-  - 2.6.0

data/appveyor.yml

@@ -1,23 +0,0 @@
-version: '{build}'
-
-# init:
-#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
-
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - ruby --version
-  - gem --version
-  - ridk.cmd exec bundle install
-build: off
-test_script:
-  - bundle exec rake test
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "26-x64"
-    - ruby_version: "26"
-    - ruby_version: "25-x64"
-    - ruby_version: "25"
-    - ruby_version: "24-x64"
-    - ruby_version: "24"

data/patterns/bind

@@ -1,3 +0,0 @@
-BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
-
-BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)

data/patterns/squid

@@ -1,4 +0,0 @@
-# Pattern squid3
-# Documentation of squid3 logs formats can be found at the following link:
-# http://wiki.squid-cache.org/Features/LogFormat
-SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{POSINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/%{IPORHOST:server}\s%{NOTSPACE:content_type}