logstash-patterns-core 4.2.0 → 4.3.0

data/spec/spec_helper.rb

@@ -24,19 +24,55 @@ end
 require "logstash/filters/grok"
 
 module GrokHelpers
+  module PatternModeSupport
+    @@pattern_mode = nil
+    def pattern_mode
+      @@pattern_mode
+    end
+    module_function :pattern_mode
+
+    def pattern_mode=(mode)
+      @@pattern_mode = mode
+    end
+  end
+
+  def ecs_compatibility?
+    case ecs_compatibility
+    when :disabled then false
+    when nil then nil
+    else true
+    end
+  end
+
+  def ecs_compatibility
+    case mode = PatternModeSupport.pattern_mode
+    when 'legacy' then :disabled
+    when 'ecs-v1' then :v1
+    when nil then nil
+    else fail "pattern_mode: #{mode.inspect}"
+    end
+  end
+
   def grok_match(label, message, exact_match = false)
+    grok_match_event(label, message, exact_match).to_hash
+  end
+
+  def grok_match_event(label, message, exact_match = false)
     grok  = build_grok(label, exact_match)
     event = build_event(message)
     grok.filter(event)
-    event.to_hash
+    event
+  end
+
+  def grok_exact_match(label, message)
+    grok_match(label, message, true)
   end
 
   def build_grok(label, exact_match = false)
-    if exact_match
-      grok = LogStash::Filters::Grok.new("match" => ["message", "^%{#{label}}$"])
-    else
-      grok = LogStash::Filters::Grok.new("match" => ["message", "%{#{label}}"])
-    end
+    grok_opts = { "match" => [ "message", exact_match ? "^%{#{label}}$" : "%{#{label}}" ] }
+    ecs_compat = ecs_compatibility # if not set use the plugin default
+    grok_opts["ecs_compatibility"] = ecs_compat unless ecs_compat.nil?
+    grok = LogStash::Filters::Grok.new(grok_opts)
     grok.register
     grok
   end
@@ -48,6 +84,31 @@ end
 
 RSpec.configure do |c|
   c.include GrokHelpers
+  c.include GrokHelpers::PatternModeSupport
+  c.extend GrokHelpers::PatternModeSupport
+end
+
+def describe_pattern(name, pattern_modes = [ nil ], &block)
+  pattern_modes.each do |mode|
+    RSpec.describe "#{name}#{mode ? " (#{mode})" : nil}" do
+
+      before(:each) do
+        @restore_pattern_mode = pattern_mode
+        self.pattern_mode = mode
+      end
+      after(:each) do
+        self.pattern_mode = @restore_pattern_mode
+      end
+
+      let(:pattern) { name }
+      let(:message) { raise 'let(:message) { ... } is missing' }
+      let(:event) { grok_match_event(pattern, message) }
+      let(:grok) { event.to_hash }
+      subject(:grok_result) { grok }
+
+      instance_eval(&block)
+    end
+  end
 end
 
 RSpec::Matchers.define :pass do |expected|
@@ -65,3 +126,16 @@ RSpec::Matchers.define :match do |value|
   end
 end
 
+RSpec.shared_examples_for 'top-level namespaces' do |namespaces, opts|
+  let(:internal_keys) { ['@timestamp', '@version'] }
+  let(:allowed_keys) { namespaces }
+  it "event is expected to only use namespaces: #{namespaces.inspect}" do
+    if instance_exec &(opts[:if] || -> { true })
+      event_hash = subject.to_hash
+      (event_hash.keys - (internal_keys + ['message'])).each do |top_level_key|
+        fail_msg = "found event.get('#{top_level_key}') : #{event_hash[top_level_key].inspect}"
+        expect(allowed_keys).to include(top_level_key), fail_msg
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-patterns-core
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-28 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,41 +74,70 @@ files:
 - README.md
 - lib/logstash/patterns/core.rb
 - logstash-patterns-core.gemspec
-- patterns/aws
-- patterns/bacula
-- patterns/bind
-- patterns/bro
-- patterns/exim
-- patterns/firewalls
-- patterns/grok-patterns
-- patterns/haproxy
-- patterns/httpd
-- patterns/java
-- patterns/junos
-- patterns/linux-syslog
-- patterns/maven
-- patterns/mcollective
-- patterns/mcollective-patterns
-- patterns/mongodb
-- patterns/nagios
-- patterns/postgresql
-- patterns/rails
-- patterns/redis
-- patterns/ruby
-- patterns/squid
-- spec/patterns/bro.rb
+- patterns/ecs-v1/aws
+- patterns/ecs-v1/bacula
+- patterns/ecs-v1/bind
+- patterns/ecs-v1/bro
+- patterns/ecs-v1/exim
+- patterns/ecs-v1/firewalls
+- patterns/ecs-v1/grok-patterns
+- patterns/ecs-v1/haproxy
+- patterns/ecs-v1/httpd
+- patterns/ecs-v1/java
+- patterns/ecs-v1/junos
+- patterns/ecs-v1/linux-syslog
+- patterns/ecs-v1/maven
+- patterns/ecs-v1/mcollective
+- patterns/ecs-v1/mongodb
+- patterns/ecs-v1/nagios
+- patterns/ecs-v1/postgresql
+- patterns/ecs-v1/rails
+- patterns/ecs-v1/redis
+- patterns/ecs-v1/ruby
+- patterns/ecs-v1/squid
+- patterns/ecs-v1/zeek
+- patterns/legacy/aws
+- patterns/legacy/bacula
+- patterns/legacy/bind
+- patterns/legacy/bro
+- patterns/legacy/exim
+- patterns/legacy/firewalls
+- patterns/legacy/grok-patterns
+- patterns/legacy/haproxy
+- patterns/legacy/httpd
+- patterns/legacy/java
+- patterns/legacy/junos
+- patterns/legacy/linux-syslog
+- patterns/legacy/maven
+- patterns/legacy/mcollective
+- patterns/legacy/mcollective-patterns
+- patterns/legacy/mongodb
+- patterns/legacy/nagios
+- patterns/legacy/postgresql
+- patterns/legacy/rails
+- patterns/legacy/redis
+- patterns/legacy/ruby
+- patterns/legacy/squid
+- spec/patterns/aws_spec.rb
+- spec/patterns/bacula_spec.rb
+- spec/patterns/bind_spec.rb
+- spec/patterns/bro_spec.rb
 - spec/patterns/core_spec.rb
+- spec/patterns/exim_spec.rb
 - spec/patterns/firewalls_spec.rb
 - spec/patterns/haproxy_spec.rb
 - spec/patterns/httpd_spec.rb
 - spec/patterns/java_spec.rb
+- spec/patterns/junos_spec.rb
 - spec/patterns/maven_spec.rb
+- spec/patterns/mcollective_spec.rb
 - spec/patterns/mongodb_spec.rb
 - spec/patterns/nagios_spec.rb
+- spec/patterns/netscreen_spec.rb
 - spec/patterns/rails3_spec.rb
 - spec/patterns/redis_spec.rb
-- spec/patterns/s3_spec.rb
 - spec/patterns/shorewall_spec.rb
+- spec/patterns/squid_spec.rb
 - spec/patterns/syslog_spec.rb
 - spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -137,18 +166,25 @@ signing_key:
 specification_version: 4
 summary: Patterns to be used in logstash
 test_files:
-- spec/patterns/bro.rb
+- spec/patterns/aws_spec.rb
+- spec/patterns/bacula_spec.rb
+- spec/patterns/bind_spec.rb
+- spec/patterns/bro_spec.rb
 - spec/patterns/core_spec.rb
+- spec/patterns/exim_spec.rb
 - spec/patterns/firewalls_spec.rb
 - spec/patterns/haproxy_spec.rb
 - spec/patterns/httpd_spec.rb
 - spec/patterns/java_spec.rb
+- spec/patterns/junos_spec.rb
 - spec/patterns/maven_spec.rb
+- spec/patterns/mcollective_spec.rb
 - spec/patterns/mongodb_spec.rb
 - spec/patterns/nagios_spec.rb
+- spec/patterns/netscreen_spec.rb
 - spec/patterns/rails3_spec.rb
 - spec/patterns/redis_spec.rb
-- spec/patterns/s3_spec.rb
 - spec/patterns/shorewall_spec.rb
+- spec/patterns/squid_spec.rb
 - spec/patterns/syslog_spec.rb
 - spec/spec_helper.rb

data/patterns/bind

@@ -1,3 +0,0 @@
-BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
-
-BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)

data/patterns/squid

@@ -1,4 +0,0 @@
-# Pattern squid3
-# Documentation of squid3 logs formats can be found at the following link:
-# http://wiki.squid-cache.org/Features/LogFormat
-SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{POSINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/%{IPORHOST:server}\s%{NOTSPACE:content_type}

data/spec/patterns/bro.rb

@@ -1,126 +0,0 @@
-# encoding: utf-8
-require "spec_helper"
-require "logstash/patterns/core"
-
-describe "HTTP" do
-
-  let(:value)   { "1432555199.633017	COpk6E3vkURP8QQNKl	192.168.9.35	55281	178.236.7.146	80	4	POST	www.amazon.it	/xa/dealcontent/v2/GetDeals?nocache=1432555199326	http://www.amazon.it/	Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36	223	1859	200	OK	-	-	-	(empty)	-	-	-	FrLEcY3AUPKdcYGf29	text/plain	FOJpbGzIMh9syPxH8	text/plain" }
-  let(:grok)    { grok_match(subject, value) }
-
-  it "a pattern pass the grok expression" do
-    expect(grok).to pass
-  end
-
-  it "matches a simple message" do
-    expect(subject).to match(value)
-  end
-  
-  it "generates the ts field" do
-    expect(grok).to include("ts" => "1432555199.633017")
-  end
-
-  it "generates the uid field" do
-    expect(grok).to include("uid" => "COpk6E3vkURP8QQNKl")
-  end
-
-  it "generates the orig_h field" do
-    expect(grok).to include("orig_h" => "192.168.9.35")
-  end
-
-  it "generates the orig_p field" do
-    expect(grok).to include("orig_p" => "55281")
-  end
-
-  it "generates the resp_h field" do
-    expect(grok).to include("resp_h" => "178.236.7.146")
-  end
-
-  it "generates the resp_p field" do
-    expect(grok).to include("resp_p" => "80")
-  end
-
-  it "generates the trans_depth field" do
-    expect(grok).to include("trans_depth" => "4")
-  end
-
-  it "generates the method field" do
-    expect(grok).to include("method" => "POST")
-  end
-
-  it "generates the domain field" do
-    expect(grok).to include("domain" => "www.amazon.it")
-  end
-
-  it "generates the uri field" do
-    expect(grok).to include("uri" => "/xa/dealcontent/v2/GetDeals?nocache=1432555199326")
-  end
-
-  it "generates the referrer field" do
-    expect(grok).to include("referrer" => "http://www.amazon.it/")
-  end
-
-  it "generates the user_agent field" do
-    expect(grok).to include("user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36")
-  end
-
-  it "generates the request_body_len field" do
-    expect(grok).to include("request_body_len" => "223")
-  end
-
-  it "generates the response_body_len field" do
-    expect(grok).to include("response_body_len" => "1859")
-  end
-
-  it "generates the status_code field" do
-    expect(grok).to include("status_code" => "200")
-  end
-
-  it "generates the status_msg field" do
-    expect(grok).to include("status_msg" => "OK")
-  end
-
-  it "generates the info_code field" do
-    expect(grok).to include("info_code" => "-")
-  end
-
-  it "generates the info_msg field" do
-    expect(grok).to include("info_msg" => "-")
-  end
-
-  it "generates the filename field" do
-    expect(grok).to include("filename" => "-")
-  end
-
-  it "generates the bro_tags field" do
-    expect(grok).to include("bro_tags" => "(empty)")
-  end
-
-  it "generates the username field" do
-    expect(grok).to include("username" => "-")
-  end
-
-  it "generates the password field" do
-    expect(grok).to include("password" => "-")
-  end
-
-  it "generates the proxied field" do
-    expect(grok).to include("proxied" => "-")
-  end
-
-  it "generates the orig_fuids field" do
-    expect(grok).to include("orig_fuids" => "FrLEcY3AUPKdcYGf29")
-  end
-
-  it "generates the orig_mime_types field" do
-    expect(grok).to include("orig_mime_types" => "text/plain")
-  end
-
-  it "generates the resp_fuids field" do
-    expect(grok).to include("resp_fuids" => "FOJpbGzIMh9syPxH8")
-  end
-
-  it "generates the resp_mime_types field" do
-    expect(grok).to include("resp_mime_types" => "text/plain")
-  end
-
-end

data/spec/patterns/s3_spec.rb

@@ -1,173 +0,0 @@
-# encoding: utf-8
-require "spec_helper"
-require "logstash/patterns/core"
-
-
-describe "ELB_ACCESS_LOG" do
-
-  let(:pattern) { "ELB_ACCESS_LOG" }
-
-  context "parsing an access log" do
-
-    let(:value) { "2014-02-15T23:39:43.945958Z my-test-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\"" }
-
-    subject { grok_match(pattern, value) }
-
-    it { should include("timestamp" => "2014-02-15T23:39:43.945958Z" ) }
-    it { should include("elb" => "my-test-loadbalancer" ) }
-    it { should include("clientip" => "192.168.131.39" ) }
-    it { should include("clientport" => 2817 ) }
-    it { should include("backendip" => "10.0.0.1" ) }
-    it { should include("backendport" => 80 ) }
-    it { should include("request_processing_time" => 0.000073 ) }
-    it { should include("backend_processing_time" => 0.001048 ) }
-    it { should include("response_processing_time" => 0.000057 ) }
-    it { should include("response" => 200 ) }
-    it { should include("backend_response" => 200 ) }
-    it { should include("received_bytes" => 0 ) }
-    it { should include("bytes" => 29 ) }
-    it { should include("verb" => "GET" ) }
-    it { should include("request" => "http://www.example.com:80/" ) }
-    it { should include("proto" => "http" ) }
-    it { should include("httpversion" => "1.1" ) }
-    it { should include("urihost" => "www.example.com:80" ) }
-    it { should include("path" => "/" ) }
-
-    ["tags", "params"].each do |attribute|
-      it "have #{attribute} as nil" do
-        expect(subject[attribute]).to be_nil
-      end
-    end
-  end
-
-  context "parsing a PUT request access log with missing backend info" do
-
-    let(:value) { '2015-04-10T08:11:09.865823Z us-west-1-production-media 49.150.87.133:55128 - -1 -1 -1 408 0 1294336 0 "PUT https://media.xxxyyyzzz.com:443/videos/F4_M-T4X0MM6Hvy1PFHesw HTTP/1.1"' }
-
-    subject { grok_match(pattern, value) }
-
-    it "a pattern pass the grok expression" do
-      expect(subject).to pass
-    end
-
-    ["backendip", "backendport"].each do |attribute|
-      it "have #{attribute} as nil" do
-        expect(subject[attribute]).to be_nil
-      end
-    end
-  end
-end
-
-describe "S3_ACCESS_LOG" do
-
-  let(:pattern)    { "S3_ACCESS_LOG" }
-
-  context "parsing GET.VERSIONING message" do
-
-    let(:value) { "79a5 mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a5 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" -" }
-
-    subject { grok_match(pattern, value) }
-
-    it { should include("owner" => "79a5" ) }
-    it { should include("bucket" => "mybucket" ) }
-    it { should include("timestamp" => "06/Feb/2014:00:00:38 +0000" ) }
-    it { should include("clientip" => "192.0.2.3" ) }
-    it { should include("requester" => "79a5" ) }
-    it { should include("request_id" => "3E57427F3EXAMPLE" ) }
-    it { should include("operation" => "REST.GET.VERSIONING" ) }
-    it { should include("key" => "-" ) }
-
-    it { should include("verb" => "GET" ) }
-    it { should include("request" => "/mybucket?versioning" ) }
-    it { should include("httpversion" => "1.1" ) }
-    it { should include("response" => 200 ) }
-    it { should include("bytes" => 113 ) }
-
-    it { should include("request_time_ms" => 7 ) }
-    it { should include("referrer" => "\"-\"" ) }
-    it { should include("agent" => "\"S3Console/0.4\"" ) }
-
-
-    ["tags", "error_code", "object_size", "turnaround_time_ms", "version_id"].each do |attribute|
-      it "have #{attribute} as nil" do
-        expect(subject[attribute]).to be_nil
-      end
-    end
-
-  end
-
-  context "parsing a GET.OBJECT message" do
-
-    let(:value) { "79a5 mybucket [12/May/2014:07:54:01 +0000] 10.0.1.2 - 7ACC4BE89EXAMPLE REST.GET.OBJECT foo/bar.html \"GET /foo/bar.html HTTP/1.1\" 304 - - 1718 10 - \"-\" \"Mozilla/5.0\" -" }
-
-    subject { grok_match(pattern, value) }
-
-    it { should include("owner" => "79a5" ) }
-    it { should include("bucket" => "mybucket" ) }
-    it { should include("timestamp" => "12/May/2014:07:54:01 +0000" ) }
-    it { should include("clientip" => "10.0.1.2" ) }
-    it { should include("requester" => "-" ) }
-    it { should include("request_id" => "7ACC4BE89EXAMPLE" ) }
-    it { should include("operation" => "REST.GET.OBJECT" ) }
-    it { should include("key" => "foo/bar.html" ) }
-
-    it { should include("verb" => "GET" ) }
-    it { should include("request" => "/foo/bar.html" ) }
-    it { should include("httpversion" => "1.1" ) }
-    it { should include("response" => 304 ) }
-    it { should include("object_size" => 1718 ) }
-
-    it { should include("request_time_ms" => 10 ) }
-    it { should include("referrer" => "\"-\"" ) }
-    it { should include("agent" => "\"Mozilla/5.0\"" ) }
-
-
-    ["tags", "error_code", "turnaround_time_ms", "version_id", "bytes"].each do |attribute|
-      it "have #{attribute} as nil" do
-        expect(subject[attribute]).to be_nil
-      end
-    end
-
-  end
-end
-
-describe "CLOUDFRONT_ACCESS_LOG" do
-
-  let(:pattern) { "CLOUDFRONT_ACCESS_LOG" }
-
-  context "parsing a cloudfront access log" do
-
-    let(:value) { "2016-06-10	18:41:39	IAD53	224281	192.168.1.1	GET	d27enomp470abc.cloudfront.net	/content/sample/thing.pdf	200	https://example.com/	Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36	-	-	Miss	UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==	host.example.com	https	883	0.036	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Miss" }
-
-    subject { grok_match(pattern, value) }
-
-    it { should include("timestamp" => "2016-06-10	18:41:39" ) }
-    it { should include("x_edge_location" => "IAD53" ) }
-    it { should include("sc_bytes" => 224281 ) }
-    it { should include("clientip" => "192.168.1.1" ) }
-    it { should include("cs_method" =>  "GET" ) }
-    it { should include("cs_host" => "d27enomp470abc.cloudfront.net" ) }
-    it { should include("cs_uri_stem" => "/content/sample/thing.pdf" ) }
-    it { should include("sc_status" => 200 ) }
-    it { should include("referrer" => "https://example.com/" ) }
-    it { should include("agent" => "Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36" ) }
-    it { should include("cs_uri_query" => "-" ) }
-    it { should include("cookies" => "-" ) }
-    it { should include("x_edge_result_type" => "Miss" ) }
-    it { should include("x_edge_request_id" => "UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==" ) }
-    it { should include("x_host_header" => "host.example.com" ) }
-    it { should include("cs_protocol" => "https" ) }
-    it { should include("cs_bytes" => 883 ) }
-    it { should include("time_taken" => 0.036 ) }
-    it { should include("x_forwarded_for" => "-" ) }
-    it { should include("ssl_protocol" => "TLSv1.2" ) }
-    it { should include("ssl_cipher" => "ECDHE-RSA-AES128-GCM-SHA256" ) }
-    it { should include("x_edge_response_result_type" => "Miss" ) }
-
-    ["tags", "params"].each do |attribute|
-      it "have #{attribute} as nil" do
-        expect(subject[attribute]).to be_nil
-      end
-    end
-  end
-end