fluent-plugin-grok-parser 2.1.4 → 2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c9d0d19c4e14ddb6d20cc30fce51ad12e8fc7b35
-  data.tar.gz: 9d74e54424dde5bc2db237c9c3472ddb74119005
+SHA256:
+  metadata.gz: 6d4b24f282982ed42f113abab9b094bba14f27354f1040956329986a53257f13
+  data.tar.gz: 421fb6798ce3c8636bd2258bf8a252eef1fe01025203cb9bd3017950b8541948
 SHA512:
-  metadata.gz: 4ef02c0f2168481e7cb4dca8285f26ce350ea602b315fcec73a51cf74e96aa187a628f0693f99b7d6639474b06d81742c79b1a43780afd4e5e78eb83a748f88b
-  data.tar.gz: 23e27a34ef129cdbd290a95d6f5f37313e4f1da86295e5304f3fb94459e13e71b52e2c493940df55ebf520ce57499e4a0755e8ce70802b983d28f68e241f22ae
+  metadata.gz: dee3c7bd1c321166dffa95cf65a001e8f066e2eafe745e928eab1b0c6e191a5ba4c2f21710f5ecb61f583571a029dd0c25cf187d8da98c746e5e830877c51c64
+  data.tar.gz: 4250e3d3f0b79c36f1993e2c27bfcc0258b2a69abb2ddad7765fdbdbe879d1eae9bbafdd3a9d1111c6094718dcda6d4b137df6ada6441cba89221f461a4dff95

data/Rakefile

@@ -20,6 +20,42 @@ task "patterns:import" do
       cp(pattern, "patterns/", verbose: true)
     end
   end
+
+  # copied from "./lib/fluent/plugin/grok"
+  pattern_re =
+    /%\{    # match '%{' not prefixed with '\'
+      (?<name>     # match the pattern name
+        (?<pattern>[A-z0-9]+)
+        (?::(?<subname>[@\[\]A-z0-9_:.-]+?)
+             (?::(?<type>(?:string|bool|integer|float|int|
+                            time(?::.+)?|
+                            array(?::.)?)))?)?
+      )
+    \}/x
+
+  Dir.glob("patterns/*") do |pattern_file|
+    new_lines = ""
+    File.readlines(pattern_file).each do |line|
+      case
+      when line.strip.empty?
+        new_lines << line
+      when line.start_with?("#")
+        new_lines << line
+      else
+        name, pattern = line.split(/\s+/, 2)
+        new_pattern = pattern.gsub(pattern_re) do |m|
+          matched = $~
+          if matched[:type] == "int"
+            "%{#{matched[:pattern]}:#{matched[:subname]}:integer}"
+          else
+            m
+          end
+        end
+        new_lines << "#{name} #{new_pattern}"
+      end
+    end
+    File.write(pattern_file, new_lines)
+  end
 end
 
 task :default => [:test, :build]

data/appveyor.yml

@@ -0,0 +1,31 @@
+version: '{build}'
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - "%devkit%\\devkitvars.bat"
+  - ruby --version
+  - gem --version
+  - bundle install
+build: off
+test_script:
+  - bundle exec rake test
+
+# https://www.appveyor.com/docs/installed-software/#ruby
+environment:
+  matrix:
+    - ruby_version: "23-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "23"
+      devkit: C:\Ruby23\DevKit
+    - ruby_version: "22-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "21-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "22"
+      devkit: C:\Ruby23\DevKit
+      WIN_RAPID: true
+    - ruby_version: "21"
+      devkit: C:\Ruby23\DevKit
+      WIN_RAPID: true
+matrix:
+  allow_failures:
+    - ruby_version: "21"

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "2.1.4"
+  spec.version       = "2.1.5"
   spec.authors       = ["kiyoto", "Kenji Okimoto"]
   spec.email         = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}

data/patterns/aws

@@ -1,6 +1,6 @@
 S3_REQUEST_LINE (?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
-S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
+S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:integer}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:integer}|-) (?:%{INT:object_size:integer}|-) (?:%{INT:request_time_ms:integer}|-) (?:%{INT:turnaround_time_ms:integer}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
 
 ELB_URIPATHPARAM %{URIPATH:path}(?:%{URIPARAM:params})?
 
@@ -8,7 +8,7 @@ ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{
 
 ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
-ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}"
+ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:integer} (?:(%{IP:backendip}:?:%{INT:backendport:integer})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:integer} %{INT:backend_response:integer} %{INT:received_bytes:integer} %{INT:bytes:integer} "%{ELB_REQUEST_LINE}"
 
-CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
+CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:integer}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:integer}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:integer}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
 

data/patterns/grok-patterns

@@ -37,7 +37,7 @@ PATH (?:%{UNIXPATH}|%{WINPATH})
 UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
 TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
 WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
-URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
 URIHOST %{IPORHOST}(?::%{POSINT:port})?
 # uripath comes loosely from RFC1738, but mostly from what Firefox
 # doesn't turn into %XX

data/patterns/haproxy

@@ -31,7 +31,7 @@ HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers}
 # HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified}
 
 # parse a haproxy 'httplog' line
-HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
+HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"?
 
 HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
 

data/patterns/httpd

@@ -7,9 +7,9 @@ HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
 
 # Error logs
 HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
 HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
 
 # Deprecated
 COMMONAPACHELOG %{HTTPD_COMMONLOG}
-COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}
+COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}

data/patterns/linux-syslog

@@ -11,6 +11,6 @@ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
 # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
 SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
 SYSLOG5424SD \[%{DATA}\]+
-SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
+SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{IPORHOST:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
 
 SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

data/test/fixtures/my_pattern

@@ -0,0 +1 @@
+MY_AWESOME_PATTERN %{GREEDYDATA:message}

data/test/helper.rb

@@ -2,9 +2,14 @@ require "fluent/test"
 require "fluent/test/helpers"
 require "fluent/test/driver/input"
 require "fluent/test/driver/parser"
+require "pathname"
 
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
 
+def fixtures(name)
+  Pathname(__dir__).expand_path + "fixtures" + name
+end
+
 def unused_port
   s = TCPServer.open(0)
   port = s.addr[1]

data/test/test_grok_parser.rb

@@ -69,17 +69,9 @@ class GrokParserTest < ::Test::Unit::TestCase
   end
 
   def test_call_for_custom_pattern
-    pattern_file = File.new(File.expand_path("../my_pattern", __FILE__), "w")
-    pattern_file.write("MY_AWESOME_PATTERN %{GREEDYDATA:message}\n")
-    pattern_file.close
-    begin
-      internal_test_grok_pattern("%{MY_AWESOME_PATTERN:message}", "this is awesome",
-                                 nil, {"message" => "this is awesome"},
-                                 "custom_pattern_path" => pattern_file.path
-                                )
-    ensure
-      File.delete(pattern_file.path)
-    end
+    internal_test_grok_pattern("%{MY_AWESOME_PATTERN:message}", "this is awesome",
+                               nil, {"message" => "this is awesome"},
+                               "custom_pattern_path" => fixtures("my_pattern").to_s)
   end
 
   class OptionalType < self

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 2.1.4
+  version: 2.1.5
 platform: ruby
 authors:
 - kiyoto
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-03 00:00:00.000000000 Z
+date: 2018-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,6 +81,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- appveyor.yml
 - fluent-plugin-grok-parser.gemspec
 - lib/fluent/plugin/grok.rb
 - lib/fluent/plugin/parser_grok.rb
@@ -107,6 +108,7 @@ files:
 - patterns/redis
 - patterns/ruby
 - patterns/squid
+- test/fixtures/my_pattern
 - test/helper.rb
 - test/test_grok_parser.rb
 - test/test_grok_parser_in_tcp.rb
@@ -131,11 +133,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.7.4
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs
 test_files:
+- test/fixtures/my_pattern
 - test/helper.rb
 - test/test_grok_parser.rb
 - test/test_grok_parser_in_tcp.rb