logstash-filter-varnishlog 0.1.7 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b5bad4a2635a7e8d23f99156c2bb9d7dda5ab37b
-  data.tar.gz: 3336e14b86616bf2ea870769eecc06851f26ed4f
+SHA256:
+  metadata.gz: aab6611b3c969096dc49e9e27731f8058e81bda20087fc113d4949c6c0700fda
+  data.tar.gz: bef79f350873bbe4308c46bc6531e548bc93f030c4f58ef575ef0eb9344d6e45
 SHA512:
-  metadata.gz: 92d81615cc6f94db12c8dbf769922a0b9ab6f21c7d4f7d29169ea3395c6151caf6bbfdb921c819541050da4504bcb4675f7240c9eeb175fc319c5f490d1b0baf
-  data.tar.gz: 685985a711e67a5d8e1686493731b26e964f939840110d168cebbc620c6609dc8e2b278d4ab2879df9530bbf95e67a86664254f3984c3a1cd3292e983651f694
+  metadata.gz: 958ce2506c3154fa5ca4c1773e72e7f9e84732c110c4c7731049a9eebbd5f72fef15cce140c88ddb1a429887337692b5a12e5ae6025b8783b3344808d64a579a
+  data.tar.gz: ecf5c95d35d200ef55926cb338d97751bb130eb3f311939606169340be3921674ff1ca96ee2faeacb14ae28d12d282cb84718a1fd232e24ba6b0392ce9c8e588

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.2.2
+  - Fix Header detection to match only valid Header names regex:[\w-], instead all .+ characters
+## 0.2.1
+  - Add xid and begin line
+  - regex performance
+## 0.2.0
+  - Make better RegEx Patterns
+## 0.1.9
+  - Add Raw Timestamps
+## 0.1.8
+  - Fix Acct
+  - Add Fetch Error
 ## 0.1.7
   - Refactor regex parser
 ## 0.1.6

data/lib/logstash/filters/varnishlog.rb

@@ -39,9 +39,16 @@ class LogStash::Filters::Varnishlog < LogStash::Filters::Base
   public
   def filter(event)
     items = event.get("[message]").split("\n")
-    if request=/\*+\s+<<\s+(?<type>\w+)\s+>>\s+/.match(items[0])
+    if request=/\*+\s+<<\s+(?<type>\w+)\s+>>\s+(?<xid>\d+)/.match(items[0])
       event.set("type", request['type'].downcase)
+      event.set("xid", request['xid'])
     end
+
+    ##Begin
+    if first=/-+\s+Begin\s+(?<id>\w+\s\d+\s\w+)/.match(items[1])
+      event.set("begin", first['id'])
+    end
+
     ##Remove Blacklisted items from items hash
     items = items.grepv(blacklist_sections.join("|")) if blacklist_sections.any?
     ##
@@ -49,15 +56,16 @@ class LogStash::Filters::Varnishlog < LogStash::Filters::Base
     ##timestamps
     timestamps = items.grep(/Timestamp/)
     timestamps.each do |timestamp|
-      if match = /-\s+Timestamp\s+(?<step>.*): (?<time_a>.*) (?<time_b>.*) (?<time_c>.*)/.match(timestamp)
+      if match = /-{1,2}\s+Timestamp\s+(?<step>\w{3,10}): (?<time_a>\d{10}\.\d{6}) (?<time_b>(0|\d+)\.\d{6}) (?<time_c>(0|\d+)\.\d{6})/.match(timestamp)
         event.set(normalize_fields("timestamp_" + match['step'] ), match['time_a'])
+        event.set(normalize_fields("timestamp_" + match['step'] + "_raw"), match['time_a'] + " " + match['time_b'] + " " + match['time_c'])
       end
     end
 
     ##Acct
     account = items.grep(/(Be|Req)Acct/)
     account.each do |acct|
-      if acct_match = /-\s+(Be|Req)Acct\s+(?<size_a>\d)\s+(?<size_b>\d)\s+(?<size_c>\d)\s+(?<size_d>\d)\s+(?<size_e>\d)\s+(?<size_f>\d)\s+/.match(acct)
+      if acct_match = /-\s+(Be|Req)Acct\s+(?<size_a>\d+)\s+(?<size_b>\d+)\s+(?<size_c>\d+)\s+(?<size_d>\d+)\s+(?<size_e>\d+)\s+(?<size_f>\d+)/.match(acct)
         event.set("bytes", acct_match['size_e'])
       end
     end
@@ -65,7 +73,7 @@ class LogStash::Filters::Varnishlog < LogStash::Filters::Base
     vcl_log = items.grep(/VCL_Log/)
     log_lines = []
     vcl_log.each_with_index do |log, index|
-      if match = /-\s+VCL_Log\s+(?<log_line>.*)/.match(log)
+      if match = /-\s+VCL_Log\s+(?<log_line>.+)/.match(log)
         log_lines.push(match['log_line'])
       end
       if index == log_lines.size - 1
@@ -75,30 +83,33 @@ class LogStash::Filters::Varnishlog < LogStash::Filters::Base
 
     # Requests
     ## Request headers.
-    request_headers = items.grep(/(Be)?([rR]eq|[rR]esp)Header/)
+    request_headers = items.grep(/(Bereq|Req|Beresp|Resp)Header/)
     request_headers.each do |header|
-      if match = /-+\s+(Be)?([rR]eq|[rR]esp)Header\s+(?<header_name>.*?): (?<header_value>.*)/.match(header)
+      if match = /-+\s+(Bereq|Req|Beresp|Resp)Header\s+(?<header_name>[\w-]+): (?<header_value>.*)/.match(header)
         event.set(normalize_fields(match['header_name']), match['header_value'])
       end
     end
     ## Match ReqMethod.
-    if method_match = /-+\s+(Be)?([rR]eq|[rR]esp)Method\s+(?<method>.*)/.match(items.grep(/(Be)?([rR]eq|[rR]esp)Method/)[0])
+    if method_match = /-+\s+(Bereq|Req|Beresp|Resp)Method\s+(?<method>.+)/.match(items.grep(/(Bereq|Req|Beresp|Resp)Method/)[0])
       event.set("http_method", method_match['method'])
     end
     ## Match ReqURL.
-    if url_match = /-+\s+(Be)?([rR]eq|[rR]esp)URL\s+(?<url>\/.*)/.match(items.grep(/(Be)?([rR]eq|[rR]esp)URL/)[0])
+    if url_match = /-+\s+(Bereq|Req|Beresp|Resp)URL\s+(?<url>\/.+)/.match(items.grep(/(Bereq|Req|Beresp|Resp)URL/)[0])
       event.set("url", url_match['url'])
     end
     ## Match ReqProtocol.
-    if protocol_match = /-+\s+(Be)?([rR]eq|[rR]esp)Protocol\s+(?<protocol>.*)/.match(items.grep(/(Be)?([rR]eq|[rR]esp)Protocol/)[0])
+    if protocol_match = /-+\s+(Bereq|Req|Beresp|Resp)Protocol\s+(?<protocol>.+)/.match(items.grep(/(Bereq|Req|Beresp|Resp)Protocol/)[0])
       event.set("protocol", protocol_match['protocol'])
     end
-
+    ## FetchError.
+    if error_match = /-+\s+FetchError\s+(?<error>.+)/.match(items.grep(/FetchError/)[0])
+      event.set("FetchError", error_match['error'])
+    end
     ## Match RespStatus
-    status_match = items.grep(/(Be)?([rR]eq|[rR]esp)Status/)
+    status_match = items.grep(/(Bereq|Req|Beresp|Resp)Status/)
     states = []
     status_match.each_with_index do |status, index|
-      if match = /-+\s+(Be)?([rR]eq|[rR]esp)Status\s+(?<status>.*)/.match(status)
+      if match = /-+\s+(Bereq|Req|Beresp|Resp)Status\s+(?<status>\d{3})/.match(status)
         states.push(match['status'].to_i)
       end
       if index == status_match.size - 1
@@ -106,10 +117,10 @@ class LogStash::Filters::Varnishlog < LogStash::Filters::Base
       end
     end
     ## Match RespReason
-    response_reason = items.grep(/(Be)?([rR]eq|[rR]esp)Reason/)
+    response_reason = items.grep(/(Bereq|Req|Beresp|Resp)Reason/)
     reasons = []
     response_reason.each_with_index do |reason, index|
-      if match = /-+\s+(Be)?([rR]eq|[rR]esp)Reason\s+(?<reason>.*)/.match(reason)
+      if match = /-+\s+(Bereq|Req|Beresp|Resp)Reason\s+(?<reason>.+)/.match(reason)
         reasons.push(match['reason'])
       end
       if index == response_reason.size - 1

data/logstash-filter-varnishlog.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-varnishlog'
-  s.version       = '0.1.7'
+  s.version       = '0.2.2'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'A logstash plugin reading varnishlog output'
   s.description   = 'logstash filter plugin reading varnishlog grouped by id'

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-varnishlog
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.2
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-23 00:00:00.000000000 Z
+date: 2021-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
   name: logstash-core-plugin-api
@@ -21,13 +21,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -35,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: logstash filter plugin reading varnishlog grouped by id
@@ -80,17 +80,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: A logstash plugin reading varnishlog output