logstash-patterns-core 4.2.0 → 4.3.0

data/spec/patterns/netscreen_spec.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe_pattern "NETSCREENSESSIONLOG", ['legacy', 'ecs-v1'] do
+
+  context "traffic denied (Juniper)" do
+
+    let(:message) do
+      'Jun  2 14:53:31 sample-host isg1000-A2: NetScreen device_id=0000011001000011 [Root]system-notification-00257(traffic): ' +
+          'start_time="2015-11-11 10:02:10" duration=0 policy_id=244 service=https proto=6 src zone=Untrust dst zone=Trust ' +
+          'action=Permit sent=0 rcvd=0 src=74.168.138.252 dst=72.72.72.72 src_port=1732 dst_port=443 ' +
+          'src-xlated ip=1.255.20.1 port=22041 dst-xlated ip=1.244.136.50 port=443 session_id=488451 reason=Creation'
+    end
+
+    it 'matches' do
+      if ecs_compatibility?
+        expect(subject).to include("timestamp" => "Jun  2 14:53:31")
+        expect(subject).to include("netscreen"=>{
+            "session"=>{"id"=>"488451", "start_time"=>"2015-11-11 10:02:10", "duration"=>0, "type"=>"traffic", "reason"=>"Creation"},
+            "policy_id"=>"244", "service"=>"https", "protocol_number"=>6, "device_id"=>"0000011001000011"
+        })
+        expect(subject).to include("event"=>{"code"=>"00257", "action"=>"Permit"})
+        # expect(subject).to include("network"=>{"protocol"=>"https"})
+        expect(subject).to include("source"=>{"bytes"=>0, "nat"=>{"port"=>22041, "ip"=>"1.255.20.1"}, "port"=>1732, "address"=>"74.168.138.252"})
+        expect(subject).to include("destination"=>{"bytes"=>0, "nat"=>{"port"=>443, "ip"=>"1.244.136.50"}, "port"=>443, "address"=>"72.72.72.72"})
+        expect(subject).to include("observer"=>{
+            "ingress"=>{"zone"=>"Untrust"}, "hostname"=>"sample-host", "name"=>"isg1000-A2", "product"=>"NetScreen",
+            "egress"=>{"zone"=>"Trust"}
+        })
+      else
+        expect(subject).to include("date" => "Jun  2 14:53:31")
+        expect(subject).to include(
+                   "device"=>"sample-host",
+                   "device_id"=>"0000011001000011",
+                   "start_time"=>"\"2015-11-11 10:02:10\"",
+                   "duration"=>"0",
+                   "policy_id"=>"244",
+                   "service"=>"https",
+                   "proto"=>"6",
+                   "src_zone"=>"Untrust", "dst_zone"=>"Trust",
+                   "action"=>"Permit",
+                   "sent"=>"0", "rcvd"=>"0",
+                   "src_ip"=>"74.168.138.252", "dst_ip"=>"72.72.72.72",
+                   "src_port"=>"1732", "dst_port"=>"443",
+                   "src_xlated_ip"=>"1.255.20.1", "src_xlated_port"=>"22041",
+                   "dst_xlated_ip"=>"1.244.136.50", "dst_xlated_port"=>"443",
+                   "session_id"=>"488451", "reason"=>"Creation",
+                   )
+      end
+    end
+
+  end
+
+  context "traffic denied (without port/xlated/session_id/reason suffix)" do
+
+    let(:message) do
+      'Mar 18 17:56:52 192.168.56.11 lowly_lizard: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): ' +
+          'start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 ' +
+          'src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1'
+    end
+
+    it 'matches in ECS mode' do
+      if ecs_compatibility?
+        expect(subject).to include("timestamp" => "Mar 18 17:56:52")
+        expect(subject).to include("netscreen"=>{
+            "device_id"=>"netscreen2",
+            "policy_id"=>"320001",
+            "service"=>"msrpc Endpoint Mapper(tcp)",
+            "protocol_number"=>6,
+            "session"=>{"start_time"=>"2009-03-18 16:07:06", "type"=>"traffic", "duration"=>0}
+        })
+        expect(subject).to include("source"=>{"address"=>"21.10.90.125", "bytes"=>0})
+        expect(subject).to include("destination"=>{"address"=>"23.16.1.1", "bytes"=>16384})
+      else
+        expect(grok['tags']).to include('_grokparsefailure')
+      end
+    end
+  end
+
+  context "'standard' traffic denied" do
+
+    let(:message) do
+      'Jun  2 14:53:31 fire00 aka1: NetScreen device_id=aka1  [Root]system-notification-00257(traffic): start_time="2006-06-02 14:53:30" ' +
+          'duration=0 policy_id=120 service=udp/port:17210 proto=17 src zone=Trust dst zone=DMZ action=Deny sent=0 rcvd=0 ' +
+          'src=192.168.2.2 dst=1.2.3.4 src_port=53 dst_port=17210'
+    end
+
+    it 'matches (in ECS mode)' do
+      if ecs_compatibility?
+        expect(subject).to include("event"=>{"action"=>"Deny", "code"=>"00257"})
+      else
+        expect(grok['tags']).to include('_grokparsefailure')
+        expect(subject).to_not include("date" => "Jun  2 14:53:31")
+      end
+    end
+
+    context "(with session id)" do
+
+      let(:message) do
+        super + ' session_id=0 reason=Traffic Denied'
+      end
+
+      it 'matches (in ECS mode)' do
+        if ecs_compatibility?
+          expect(subject).to include("netscreen"=>hash_including("device_id"=>"aka1", "service"=>"udp/port:17210",
+                                     "session"=>hash_including("reason"=>"Traffic Denied")))
+          expect(subject).to include("observer"=>{
+              "ingress"=>{"zone"=>"Trust"},
+              "egress"=>{"zone"=>"DMZ"}, "hostname"=>"fire00", "name"=>"aka1",
+              "product"=>"NetScreen"
+          })
+        else
+          expect(grok['tags']).to include('_grokparsefailure')
+          expect(subject).to_not include("date" => "Jun  2 14:53:31")
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/patterns/rails3_spec.rb

@@ -2,55 +2,113 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "RAILS" do
-  let(:rails3_pattern)  { "RAILS3" }
+describe_pattern "RAILS3", ['legacy', 'ecs-v1'] do
 
-  context "Parsing RAILS3 single-line log from raw log file" do
+  context "single-line log" do
 
-    let(:value) { 'Started POST "/api/v3/internal/allowed" for 127.0.0.1 at 2015-08-05 11:37:01 +0200' } 
-
-    subject     { grok_match(rails3_pattern, value) }
+    let(:message) { 'Started POST "/api/v3/internal/allowed" for 127.0.0.1 at 2015-08-05 11:37:01 +0200' }
 
     # Started
-    it { should include("verb" => "POST" ) }
-    it { should include("request" => "/api/v3/internal/allowed" ) }
+    it do
+      if ecs_compatibility?
+        should include("http" => hash_including("request" => { "method" => "POST" }))
+      else
+        should include("verb" => "POST")
+      end
+    end
+
+    it do
+      if ecs_compatibility?
+      else
+        should include("request" => "/api/v3/internal/allowed")
+      end
+    end
     # for
-    it { should include("clientip" => "127.0.0.1" ) }
+    it do
+      if ecs_compatibility?
+        should include("source" => { "address" => "127.0.0.1" })
+      else
+        should include("clientip" => "127.0.0.1")
+      end
+    end
     # at
     it { should include("timestamp" => "2015-08-05 11:37:01 +0200" ) }
   end
 
-  context "Parsing RAILS3 multi-line log from raw log file" do
+  context "multi-line log" do
 
-    let(:value) { 'Started GET "/puppet/postfix/notes?target_id=162&target_type=issue&last_fetched_at=1438695732" for 127.0.0.1 at 2015-08-05 07:40:22 +0200
+    let(:message) { 'Started GET "/puppet/postfix/notes?target_id=162&target_type=issue&last_fetched_at=1438695732" for 127.0.0.1 at 2015-08-05 07:40:22 +0200
 Processing by Projects::NotesController#index as JSON
   Parameters: {"target_id"=>"162", "target_type"=>"issue", "last_fetched_at"=>"1438695732", "namespace_id"=>"puppet", "project_id"=>"postfix"}
-Completed 200 OK in 640ms (Views: 1.7ms | ActiveRecord: 91.0ms)' } 
-    subject     { grok_match(rails3_pattern, value) }
+Completed 200 OK in 640ms (Views: 1.7ms | ActiveRecord: 91.0ms)' }
 
     # started
-    it { should include("verb" => "GET" ) }
-    it { should include("request" => "/puppet/postfix/notes?target_id=162&target_type=issue&last_fetched_at=1438695732" ) }
+    it do
+      if ecs_compatibility?
+        should include("http" => hash_including("request" => { "method" => "GET" }))
+      else
+        should include("verb" => "GET")
+      end
+    end
+
+    it do
+      if ecs_compatibility?
+        should include("url" => {"original"=>"/puppet/postfix/notes?target_id=162&target_type=issue&last_fetched_at=1438695732"})
+      else
+        should include("request" => "/puppet/postfix/notes?target_id=162&target_type=issue&last_fetched_at=1438695732" )
+      end
+    end
     # for
-    it { should include("clientip" => "127.0.0.1" ) }
+    it do
+      if ecs_compatibility?
+        should include("source" => { "address" => "127.0.0.1" })
+      else
+        should include("clientip" => "127.0.0.1")
+      end
+    end
     # at
-    it { should include("timestamp" => "2015-08-05 07:40:22 +0200" ) }
+    it { should include("timestamp" => "2015-08-05 07:40:22 +0200") }
     # Processing by
-    it { should include("controller" => "Projects::NotesController" ) }
-    it { should include("action" => "index" ) }
+    it do
+      if ecs_compatibility?
+        should include("rails" => hash_including("controller" => { "class"=>"Projects::NotesController", "action"=>"index" }))
+      else
+        should include("controller" => "Projects::NotesController")
+        should include("action" => "index")
+      end
+    end
     # as
-    it { should include("format" => "JSON" ) }
+    it do
+      if ecs_compatibility?
+        should include("rails" => hash_including("request" => hash_including("format" => 'JSON')))
+      else
+        should include("format" => "JSON" )
+      end
+    end
     # Parameters
-    it { should include("params" => '"target_id"=>"162", "target_type"=>"issue", "last_fetched_at"=>"1438695732", "namespace_id"=>"puppet", "project_id"=>"postfix"' ) }
+    it do
+      params = '"target_id"=>"162", "target_type"=>"issue", "last_fetched_at"=>"1438695732", "namespace_id"=>"puppet", "project_id"=>"postfix"'
+      if ecs_compatibility?
+        should include("rails" => hash_including("request" => hash_including("params" => params)))
+      else
+        should include("params" => params)
+      end
+    end
     # Completed
-    it { should include("response" => "200" ) }
+    it do
+      if ecs_compatibility?
+        should include("http" => hash_including("response" => { "status_code" => 200 }))
+      else
+        should include("response" => "200" )
+      end
+    end
     # in
-    it { should include("totalms" =>  "640" ) }
-    # (Views: 
-    it { should include("viewms" =>  "1.7" ) }
-    # | ActiveRecord:
-    it { should include("activerecordms" =>  "91.0" ) }
-
+    it do
+      if ecs_compatibility?
+        should include("rails" => hash_including("request" => hash_including("duration" => { "total" => 640.0, "view" => 1.7, "active_record" => 91.0 })))
+      else
+        should include("totalms" => "640", "viewms" => "1.7", "activerecordms" => "91.0")
+      end
+    end
   end
-
 end

data/spec/patterns/redis_spec.rb

@@ -2,170 +2,206 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
-describe "REDISTIMESTAMP" do
+describe_pattern 'REDISTIMESTAMP', [ 'legacy', 'ecs-v1' ] do
 
-  let(:value) { '14 Nov 07:01:22.119'}
-  let(:pattern) { "REDISTIMESTAMP" }
+  let(:message) { '14 Nov 07:01:22.119'}
 
   it "a pattern pass the grok expression" do
-    expect(grok_match(pattern, value)).to pass
+    expect(grok_match(pattern, message)).to pass
   end
 
 end
 
-describe "REDISLOG" do
+describe_pattern 'REDISLOG', [ 'legacy', 'ecs-v1' ] do
 
-  let(:value)   { "[4018] 14 Nov 07:01:22.119 * Background saving terminated with success" }
-  let(:pattern) { "REDISLOG" }
-  let(:grok)    { grok_match(pattern, value) }
+  let(:message) { "[4018] 14 Nov 07:01:22.119 * Background saving terminated with success" }
 
   it "a pattern pass the grok expression" do
     expect(grok).to pass
   end
 
   it "generates the pid field" do
-    expect(grok).to include("pid" => "4018")
+    if ecs_compatibility?
+      expect(grok).to include("process" => { 'pid' => 4018 })
+    else
+      expect(grok).to include("pid" => "4018")
+    end
   end
 
 end
 
+describe_pattern 'REDISMONLOG', [ 'legacy', 'ecs-v1' ] do
+
+  context "simple command" do
+
+    let(:message) { "1470637867.953466 [0 195.168.1.1:52500] \"info\"" }
+
+    it "a pattern pass the grok expression" do
+      expect(grok).to pass
+    end
+
+    it "generates the timestamp field" do
+      expect(grok).to include("timestamp" => "1470637867.953466")
+    end
+
+    it "generates the database field" do
+      if ecs_compatibility?
+        expect(grok).to include("redis" => hash_including('database' => { 'id' => '0' }))
+      else
+        expect(grok).to include("database" => "0")
+      end
+    end
+
+    it "generates the client field" do
+      if ecs_compatibility?
+        expect(grok).to include("client" => hash_including('ip' => '195.168.1.1'))
+      else
+        expect(grok).to include("client" => "195.168.1.1")
+      end
+    end
+
+    it "generates the port field" do
+      if ecs_compatibility?
+        expect(grok).to include("client" => hash_including('port' => 52500))
+      else
+        expect(grok).to include("port" => "52500")
+      end
+    end
+
+    it "generates the command field" do
+      if ecs_compatibility?
+        expect(grok).to include("redis" => hash_including('command' => { 'name' => 'info' }))
+      else
+        expect(grok).to include("command" => "info")
+      end
+    end
+
+  end
+
+  context "one param command" do
+
+    let(:message) { "1339518083.107412 [0 127.0.0.1:60866] \"keys\" \"*\"" }
+
+    it "a pattern pass the grok expression" do
+      expect(grok).to pass
+    end
+
+    it "generates the timestamp field" do
+      expect(grok).to include("timestamp" => "1339518083.107412")
+    end
+
+    it "generates the database field" do
+      if ecs_compatibility?
+        expect(grok).to include("redis" => hash_including('database' => { 'id' => '0' }))
+      else
+        expect(grok).to include("database" => "0")
+      end
+    end
+
+    it "generates the client field" do
+      if ecs_compatibility?
+        expect(grok).to include("client" => hash_including('ip' => '127.0.0.1'))
+      else
+        expect(grok).to include("client" => "127.0.0.1")
+      end
+    end
+
+    it "generates the port field" do
+      if ecs_compatibility?
+        expect(grok).to include("client" => hash_including('port' => 60866))
+      else
+        expect(grok).to include("port" => "60866")
+      end
+    end
+
+    it "generates the command field" do
+      if ecs_compatibility?
+        expect(grok).to include("redis" => hash_including('command' => hash_including('name' => 'keys')))
+      else
+        expect(grok).to include("command" => "keys")
+      end
+    end
+
+    it "generates the params field" do
+      if ecs_compatibility?
+        expect(grok).to include("redis" => hash_including('command' => hash_including('args' => '"*"')))
+      else
+        expect(grok).to include("params" => "\"*\"")
+      end
+    end
 
-describe "REDISMONLOG - SIMPLE COMMAND" do
-
-  let(:value)   { "1470637867.953466 [0 195.168.1.1:52500] \"info\"" }
-  let(:pattern) { "REDISMONLOG" }
-  let(:grok)    { grok_match(pattern, value) }
-
-  it "a pattern pass the grok expression" do
-    expect(grok).to pass
-  end
-
-  it "generates the timestamp field" do
-    expect(grok).to include("timestamp" => "1470637867.953466")
-  end
-
-  it "generates the database field" do
-    expect(grok).to include("database" => "0")
-  end
-
-  it "generates the client field" do
-    expect(grok).to include("client" => "195.168.1.1")
-  end
-
-  it "generates the port field" do
-    expect(grok).to include("port" => "52500")
-  end
-
-  it "generates the command field" do
-    expect(grok).to include("command" => "info")
   end
 
 end
 
-describe "REDISMONLOG - ONE PARAM COMMAND" do
-
-  let(:value)   { "1339518083.107412 [0 127.0.0.1:60866] \"keys\" \"*\"" }
-  let(:pattern) { "REDISMONLOG" }
-  let(:grok)    { grok_match(pattern, value) }
-
-  it "a pattern pass the grok expression" do
-    expect(grok).to pass
-  end
-
-  it "generates the timestamp field" do
-    expect(grok).to include("timestamp" => "1339518083.107412")
-  end
-
-  it "generates the database field" do
-    expect(grok).to include("database" => "0")
-  end
-
-  it "generates the client field" do
-    expect(grok).to include("client" => "127.0.0.1")
-  end
-
-  it "generates the port field" do
-    expect(grok).to include("port" => "60866")
-  end
+describe_pattern "REDISMONLOG" do
 
-  it "generates the command field" do
-    expect(grok).to include("command" => "keys")
-  end
+  context 'two param command' do
 
-  it "generates the params field" do
-    expect(grok).to include("params" => "\"*\"")
-  end
+    let(:message) { "1470637925.186681 [0 127.0.0.1:39404] \"rpush\" \"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"" }
 
-end
+    it "a pattern pass the grok expression" do
+      expect(grok).to pass
+    end
 
-describe "REDISMONLOG - TWO PARAM COMMAND" do
+    it "generates the timestamp field" do
+      expect(grok).to include("timestamp" => "1470637925.186681")
+    end
 
-  let(:value)   { "1470637925.186681 [0 127.0.0.1:39404] \"rpush\" \"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"" }
-  let(:pattern) { "REDISMONLOG" }
-  let(:grok)    { grok_match(pattern, value) }
+    it "generates the database field" do
+      expect(grok).to include("database" => "0")
+    end
 
-  it "a pattern pass the grok expression" do
-    expect(grok).to pass
-  end
+    it "generates the client field" do
+      expect(grok).to include("client" => "127.0.0.1")
+    end
 
-  it "generates the timestamp field" do
-    expect(grok).to include("timestamp" => "1470637925.186681")
-  end
+    it "generates the port field" do
+      expect(grok).to include("port" => "39404")
+    end
 
-  it "generates the database field" do
-    expect(grok).to include("database" => "0")
-  end
+    it "generates the command field" do
+      expect(grok).to include("command" => "rpush")
+    end
 
-  it "generates the client field" do
-    expect(grok).to include("client" => "127.0.0.1")
-  end
+    it "generates the params field" do
+      expect(grok).to include("params" => "\"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"")
+    end
 
-  it "generates the port field" do
-    expect(grok).to include("port" => "39404")
   end
 
-  it "generates the command field" do
-    expect(grok).to include("command" => "rpush")
-  end
+  context "variadic command" do
 
-  it "generates the params field" do
-    expect(grok).to include("params" => "\"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"")
-  end
+    let(:message) { "1470637875.777457 [15 195.168.1.1:52500] \"intentionally\" \"broken\" \"variadic\" \"log\" \"entry\"" }
 
-end
-
-describe "REDISMONLOG - VARIADIC COMMAND" do
-
-  let(:value)   { "1470637875.777457 [15 195.168.1.1:52500] \"intentionally\" \"broken\" \"variadic\" \"log\" \"entry\"" }
-  let(:pattern) { "REDISMONLOG" }
-  let(:grok)    { grok_match(pattern, value) }
+    it "a pattern pass the grok expression" do
+      expect(grok).to pass
+    end
 
-  it "a pattern pass the grok expression" do
-    expect(grok).to pass
-  end
+    it "generates the timestamp field" do
+      expect(grok).to include("timestamp" => "1470637875.777457")
+    end
 
-  it "generates the timestamp field" do
-    expect(grok).to include("timestamp" => "1470637875.777457")
-  end
+    it "generates the database field" do
+      expect(grok).to include("database" => "15")
+    end
 
-  it "generates the database field" do
-    expect(grok).to include("database" => "15")
-  end
+    it "generates the client field" do
+      expect(grok).to include("client" => "195.168.1.1")
+    end
 
-  it "generates the client field" do
-    expect(grok).to include("client" => "195.168.1.1")
-  end
+    it "generates the port field" do
+      expect(grok).to include("port" => "52500")
+    end
 
-  it "generates the port field" do
-    expect(grok).to include("port" => "52500")
-  end
+    it "generates the command field" do
+      expect(grok).to include("command" => "intentionally")
+    end
 
-  it "generates the command field" do
-    expect(grok).to include("command" => "intentionally")
-  end
+    it "generates the params field" do
+      expect(grok).to include("params" => "\"broken\" \"variadic\" \"log\" \"entry\"")
+    end
 
-  it "generates the params field" do
-    expect(grok).to include("params" => "\"broken\" \"variadic\" \"log\" \"entry\"")
   end
 
-end
+end