logstash-patterns-core 4.0.1 → 4.2.0

data/spec/patterns/haproxy_spec.rb

@@ -56,4 +56,21 @@ describe "HAPROXY" do
 
   end
 
+  context "Parsing HAPROXY log line that is truncated and thus not ending with a double quote or HTTP version." do
+
+    let(:value) { 'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'}
+    subject     { grok_match(haproxyhttpbase_pattern, value)}
+
+    it { should include("client_ip" => "203.0.113.54") }
+    it { should include("http_verb" => "GET") }
+    it { should include("server_name" => "instance8") }
+    it { should include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss") }
+    it { should_not have_key("http_version") }
+
+    it "generates a message field" do
+      expect(subject["message"]).to include("loadbalancer default/instance8")
+    end
+
+  end
+
 end

data/spec/patterns/httpd_spec.rb

@@ -2,23 +2,183 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
+describe "HTTPD_COMBINEDLOG" do
+
+  let(:pattern) { 'HTTPD_COMBINEDLOG' }
+  let(:grok) { grok_match(pattern, message) }
+
+  context "typical test case" do
+
+    let(:message) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
+
+    it "matches" do
+      expect(grok).to include(
+        'clientip' => '83.149.9.216',
+        'verb' => 'GET',
+        'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
+        'httpversion' => '1.1',
+        'response' => '200',
+        'bytes' => '203023',
+        'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
+        'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
+      )
+    end
+
+    it "does not capture 'null' fields" do
+      expect(grok).to include('auth' => '-', 'ident' => '-')
+    end
+
+  end
+
+  context "email address in auth field" do
+
+    let(:message) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
+
+    it "gets captured" do
+      expect(grok).to include("auth" => "username@example.com")
+    end
+
+  end
+
+  context 'sample OPTIONS line' do
+
+    let(:message) { '83.149.9.216 - a.user [11/Jan/2020:23:05:27 +0100] "OPTIONS /remote.php/ HTTP/1.1" - 7908 "-" "monitoring-client (v2.2)"' }
+
+    it 'matches' do
+      expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
+    end
+
+    it 'does not capture optional response code' do
+      expect(grok.keys).to_not include("response")
+    end
+
+  end
+
+end
+
 describe "HTTPD_ERRORLOG" do
 
-  it "matches a full httpd 2.4 message" do
-    expect(subject).to match("[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product")
+  let(:pattern) { 'HTTPD_ERRORLOG' }
+  let(:grok) { grok_match(pattern, message) }
+
+  context "matches a full httpd 2.4 message" do
+    let(:message) do
+      "[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
+    end
+    it "generates the fields" do
+
+      expect(grok).to include(
+        'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
+        'module' => 'proxy_fcgi',
+        'loglevel' => 'error',
+        'pid' => '28787',
+        'tid' => '140169587934976',
+        'proxy_errorcode' => '70008',
+        'proxy_message' => 'Partial results are valid but processing is incomplete',
+        'clientip' => '58.13.45.166',
+        'clientport' => '59307',
+        'errorcode' => 'AH01075',
+        'message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ],
+      )
+    end
   end
 
-  it "matches a httpd 2.2 log message" do
-    expect(subject).to match("[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com")
+  context "HTTPD_ERRORLOG", "matches a httpd 2.2 log message" do
+    let(:message) do
+      "[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
+    end
+    it "generates the fields" do
+      expect(grok).to include(
+        'timestamp' => 'Mon Aug 31 16:27:04 2015',
+        'loglevel' => 'error',
+        'clientip' => '10.17.42.3',
+        'message' => [ message, 'Premature end of script headers: example.com' ]
+      )
+    end
   end
 
-  it "matches a short httpd 2.4 message" do
-    expect(subject).to match("[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'")
+  context "HTTPD_ERRORLOG", "a short httpd 2.4 message" do
+    let(:value1) {
+      "[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value1)).to include(
+        'timestamp' => 'Mon Aug 31 07:15:38.664897 2015',
+        'module' => 'proxy_fcgi',
+        'loglevel' => 'error',
+        'pid' => '28786',
+        'tid' => '140169629898496',
+        'clientip' => '81.139.1.34',
+        'clientport' => '52042',
+        'errorcode' => 'AH01071',
+        'message' => [ value1, "Got error 'Primary script unknown\n'" ]
+      )
+    end
+
+    let(:value2) {
+      "[Thu Apr 27 10:39:46.719636 2017] [php7:notice] [pid 17] [client 10.255.0.3:49580] Test error log record"
+    }
+	it "generates the fields" do
+      expect(grok_match(subject, value2)).to include(
+        'timestamp' => 'Thu Apr 27 10:39:46.719636 2017',
+        'module' => 'php7',
+        'loglevel' => 'notice',
+        'pid' => '17',
+        'clientip' => '10.255.0.3',
+        'clientport' => '49580',
+        'message' => [ value2, "Test error log record" ]
+      )
+    end
   end
 
-  it "matches an httpd 2.4 restart" do
-    expect(subject).to match("[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations")
-    expect(subject).to match("[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'")
+  context "HTTPD_ERRORLOG", "a httpd 2.4 restart message" do
+    let(:value1) {
+      "[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value1)).to include(
+        'timestamp' => 'Mon Aug 31 06:29:47.406518 2015',
+        'module' => 'mpm_event',
+        'loglevel' => 'notice',
+        'pid' => '24968',
+        'tid' => '140169861986176',
+        'errorcode' => 'AH00489',
+        'message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ]
+      )
+    end
+
+    let(:value2) {
+      "[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value2)).to include(
+        'timestamp' => 'Mon Aug 31 06:29:47.406530 2015',
+        'module' => 'core',
+        'loglevel' => 'notice',
+        'pid' => '24968',
+        'tid' => '140169861986176',
+        'errorcode' => 'AH00094',
+        'message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ]
+      )
+    end
   end
 
+  context 'a debug message' do
+    let(:message) do
+      '[Fri Feb 01 22:03:08.319124 2019] [authz_core:debug] [pid 9:tid 140597881775872] mod_authz_core.c(820): [client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted'
+    end
+
+    it 'matches imperfectly (legacy)' do
+      expect(grok).to include({
+                                  "timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
+                                  "module"=>"authz_core",
+                                  "loglevel"=>"debug",
+                                  "pid"=>"9",
+                                  "tid"=>"140597881775872",
+                                  "errorcode"=>"mod_authz_core.c(820)",
+                                  "message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
+                              })
+    end
+  end
+  
 end

data/spec/patterns/java_spec.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "JAVA" do
+  describe "JAVACLASS" do
+    let(:example) { 'hudson.node_monitors.AbstractAsyncNodeMonitorDescriptor' }
+    it "matches a java class with underscores" do
+      expect(grok_match(subject, example, true)['tags']).to be_nil
+    end
+  end
+  describe "JAVAFILE" do
+    let(:example) { 'Native Method' }
+    it "matches a java file name with spaces" do
+      expect(grok_match(subject, example, true)['tags']).to be_nil
+    end
+  end
+end
+
+describe "JAVASTACKTRACEPART" do
+  let(:pattern) { 'JAVASTACKTRACEPART' }
+  let(:message) { '  at com.sample.stacktrace.StackTraceExample.aMethod(StackTraceExample.java:42)' }
+  it "matches" do
+    grok = grok_match(pattern, message, true)
+    expect(grok).to include({
+                                "message"=>"  at com.sample.stacktrace.StackTraceExample.aMethod(StackTraceExample.java:42)",
+                                "method"=>"aMethod",
+                                "class"=>"com.sample.stacktrace.StackTraceExample",
+                                "file"=>"StackTraceExample.java",
+                                "line"=>"42"
+                            })
+  end
+
+  context 'generated file' do
+    let(:message) { '  at org.jruby.RubyMethod$INVOKER$i$call.call(RubyMethod$INVOKER$i$call.gen)' }
+    it "matches" do
+      grok = grok_match(pattern, message, true)
+      expect(grok).to include({
+                                  "method"=>"call",
+                                  "class"=>"org.jruby.RubyMethod$INVOKER$i$call",
+                                  "file"=>"RubyMethod$INVOKER$i$call.gen",
+                              })
+    end
+  end
+end

data/spec/patterns/maven_spec.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "MAVEN_VERSION" do
+
+  let(:pattern) { 'MAVEN_VERSION' }
+
+  context "when maven version is simple" do
+    let(:value) { '1.1.0' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version is a bit more complex" do
+    let(:value) { '2.35.128' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version contains release" do
+    let(:value) { '1.1.0.RELEASE' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version contains shapshot" do
+    let(:value) { '1.1.0.SNAPSHOT' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version contains release" do
+    context "and the version contains a dash" do
+      let(:value) { '1.1.0-RELEASE' }
+
+      it "should match the version" do
+        expect(grok_match(pattern,value)).to pass
+      end
+    end
+  end
+
+  context "when maven version contains shapshot" do
+    context "and the version contains a dash" do
+    let(:value) { '1.1.0-SNAPSHOT' }
+
+      it "should match the version" do
+        expect(grok_match(pattern,value)).to pass
+      end
+    end
+  end
+
+end

data/spec/patterns/nagios_spec.rb

@@ -82,7 +82,7 @@ end
 
 describe "NAGIOSLOGLINE - TIMEPERIOD TRANSITION" do
 
-  let(:value)   { "[1427925600] TIMEPERIOD TRANSITION: 24X7;1;1" }
+  let(:value)   { "[1427925600] TIMEPERIOD TRANSITION: 24X7;-1;1" }
   let(:grok)    { grok_match(subject, value) }
 
   it "a pattern pass the grok expression" do
@@ -105,6 +105,10 @@ describe "NAGIOSLOGLINE - TIMEPERIOD TRANSITION" do
     expect(grok).to include("nagios_service" => "24X7")
   end
 
+  it "generates the period from/to fields" do
+    expect(grok).to include("nagios_unknown1" => "-1", "nagios_unknown2" => "1")
+  end
+
   # Regression test for but fixed in Nagios patterns #30
   it "doesn't end in a semi-colon" do
     expect(grok['message']).to_not end_with(";")

data/spec/patterns/redis_spec.rb

@@ -0,0 +1,171 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "REDISTIMESTAMP" do
+
+  let(:value) { '14 Nov 07:01:22.119'}
+  let(:pattern) { "REDISTIMESTAMP" }
+
+  it "a pattern pass the grok expression" do
+    expect(grok_match(pattern, value)).to pass
+  end
+
+end
+
+describe "REDISLOG" do
+
+  let(:value)   { "[4018] 14 Nov 07:01:22.119 * Background saving terminated with success" }
+  let(:pattern) { "REDISLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the pid field" do
+    expect(grok).to include("pid" => "4018")
+  end
+
+end
+
+
+describe "REDISMONLOG - SIMPLE COMMAND" do
+
+  let(:value)   { "1470637867.953466 [0 195.168.1.1:52500] \"info\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1470637867.953466")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "0")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "195.168.1.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "52500")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "info")
+  end
+
+end
+
+describe "REDISMONLOG - ONE PARAM COMMAND" do
+
+  let(:value)   { "1339518083.107412 [0 127.0.0.1:60866] \"keys\" \"*\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1339518083.107412")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "0")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "127.0.0.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "60866")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "keys")
+  end
+
+  it "generates the params field" do
+    expect(grok).to include("params" => "\"*\"")
+  end
+
+end
+
+describe "REDISMONLOG - TWO PARAM COMMAND" do
+
+  let(:value)   { "1470637925.186681 [0 127.0.0.1:39404] \"rpush\" \"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1470637925.186681")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "0")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "127.0.0.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "39404")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "rpush")
+  end
+
+  it "generates the params field" do
+    expect(grok).to include("params" => "\"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"")
+  end
+
+end
+
+describe "REDISMONLOG - VARIADIC COMMAND" do
+
+  let(:value)   { "1470637875.777457 [15 195.168.1.1:52500] \"intentionally\" \"broken\" \"variadic\" \"log\" \"entry\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1470637875.777457")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "15")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "195.168.1.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "52500")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "intentionally")
+  end
+
+  it "generates the params field" do
+    expect(grok).to include("params" => "\"broken\" \"variadic\" \"log\" \"entry\"")
+  end
+
+end