fluent-plugin-grok-parser 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 80ab11f887489e91eff7e4f3a07b0cdce24fe7ad
-  data.tar.gz: 2bd1128bd64dc2ae91b8bd37d23e306b624c3572
+SHA256:
+  metadata.gz: abba93dea7171b49379a1c09b97e93775a71c60f07975d8be225467ef9a02f3e
+  data.tar.gz: 53cabdd496599b84fda4a5facebbd93b59be9a5426dfa3142f6b94344411431d
 SHA512:
-  metadata.gz: 373475b75f2655b6d698bba92d5c09d6a0410c9909310a56d1f74300cb8086f3116688be51d794702c0c8d056b3cbd40b394f6201c1b26e6384e8061103c02ba
-  data.tar.gz: f2ba20c1de74a02fa178d26f627188f359c6c012c80920d8f83b341455333e6d12041e63500916400bb3663d889a9953ee6f84673bdd1591aab05c9ee316068d
+  metadata.gz: 0fed7231df5bddfe56b57eaa53155afb29e400d16994e3454d80a1f7dde75ff0528fd778137a65d557d71529b1de75f501206f3dfd5220cb5b2b2e86fd44d8ad
+  data.tar.gz: 37e00405e9c28e9d4bac1fe02ea609e297ea98554d98802f8deff575d5c8104953149dbbc129b00953fb5e162390904af2cf707df76f073bae3e608594fe2052

data/Rakefile

@@ -20,6 +20,42 @@ task 'patterns:import' do
       cp(pattern, "patterns/", verbose: true)
     end
   end
+
+  # copied from "./lib/fluent/plugin/grok"
+  pattern_re =
+    /%\{    # match '%{' not prefixed with '\'
+      (?<name>     # match the pattern name
+        (?<pattern>[A-z0-9]+)
+        (?::(?<subname>[@\[\]A-z0-9_:.-]+?)
+             (?::(?<type>(?:string|bool|integer|float|int|
+                            time(?::.+)?|
+                            array(?::.)?)))?)?
+      )
+    \}/x
+
+  Dir.glob("patterns/*") do |pattern_file|
+    new_lines = ""
+    File.readlines(pattern_file).each do |line|
+      case
+      when line.strip.empty?
+        new_lines << line
+      when line.start_with?("#")
+        new_lines << line
+      else
+        name, pattern = line.split(/\s+/, 2)
+        new_pattern = pattern.gsub(pattern_re) do |m|
+          matched = $~
+          if matched[:type] == "int"
+            "%{#{matched[:pattern]}:#{matched[:subname]}:integer}"
+          else
+            m
+          end
+        end
+        new_lines << "#{name} #{new_pattern}"
+      end
+    end
+    File.write(pattern_file, new_lines)
+  end
 end
 
 task :default => [:test, :build]

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "1.0.0"
+  spec.version       = "1.0.1"
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}

data/lib/fluent/plugin/grok.rb

@@ -82,7 +82,7 @@ module Fluent
           replacement_pattern = "(?<#{m["subname"]}>#{curr_pattern})"
           type_map[m["subname"]] = m["type"] || "string"
         else
-          replacement_pattern = curr_pattern
+          replacement_pattern = "(?:#{curr_pattern})"
         end
         pattern.sub!(m[0]) do |s| replacement_pattern end
       end

data/patterns/aws

@@ -1,6 +1,6 @@
 S3_REQUEST_LINE (?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
-S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
+S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:integer}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:integer}|-) (?:%{INT:object_size:integer}|-) (?:%{INT:request_time_ms:integer}|-) (?:%{INT:turnaround_time_ms:integer}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
 
 ELB_URIPATHPARAM %{URIPATH:path}(?:%{URIPARAM:params})?
 
@@ -8,7 +8,7 @@ ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{
 
 ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
-ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}"
+ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:integer} (?:(%{IP:backendip}:?:%{INT:backendport:integer})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:integer} %{INT:backend_response:integer} %{INT:received_bytes:integer} %{INT:bytes:integer} "%{ELB_REQUEST_LINE}"
 
-CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
+CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:integer}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:integer}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:integer}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
 

data/test/test_grok_parser.rb

@@ -37,6 +37,11 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
+  def test_date
+    internal_test_grok_pattern("\\[(?<date>%{DATE} %{TIME} (?:AM|PM))\\]", "[2/16/2018 10:19:34 AM]",
+                               nil, { "date" => "2/16/2018 10:19:34 AM" })
+  end
+
   def test_call_for_grok_pattern_not_found
     assert_raise Grok::GrokPatternNotFoundError do
       internal_test_grok_pattern('%{THIS_PATTERN_DOESNT_EXIST}', 'Some stuff at somewhere', nil, {})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-05 00:00:00.000000000 Z
+date: 2018-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -135,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs