logstash-patterns-core 4.0.2 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6121e0138e8de43548944cc443bdf0ee2a6b3847
-  data.tar.gz: c1c8ae3d54e66de0b2a103cbdca29e6a186f8456
+  metadata.gz: 36d2a300d64eee6d5b7f294802ffe2e0bc63729e
+  data.tar.gz: f8244876e124251a382333a2b10cd35acaf2b720
 SHA512:
-  metadata.gz: 14e64a42d2b8e7c31fe076b83ebaef5ddac0b77a6c69e7c28511aafce48d2d4c3d058d9354f60df96033474c4bf70755fb67c6ed96c6ea46e4113e48915e6971
-  data.tar.gz: 3805625550fcf3bd35304bdbfcb8bb666f07debfdd9c4cb3b74332992313df4cc612baef06959ba9c10d045fcb1d2014ca8fc6b0b3c342731ba680be9f444a72
+  metadata.gz: 7aeea39d789b000f5d930ca9a7df02e4e9f7f205b16fe36c71d7f0244bd2bf891d424e5b564f466245af05e705767732ee660be8dea74a74a00fc085f1913083
+  data.tar.gz: f377108083d51399bca1ee25fafd4802c5085026e02d8caf762b89844a2b3c57f41267d93a8f4ac860ed3771a9da5a5a3a3c8893269ed502d2f11fa1c09871c1

data/CHANGELOG.md

@@ -1,22 +1,50 @@
+## 4.1.0
+  - Added SYSLOG5424LINE and test ipv4/ipv6/hostname as syslog5424_host rfc5424
+  - Accordig to rcf5424 IP address should be accepted
+  - HTTPDATE is used by patterns/aws
+  - HTTPD (formerly APACHE) deserves its own pattern and test files. See #45
+  - httpd: sync names between httpd20 and httpd24
+  - Adding maven version to the list of default Grok patterns
+  - Added Redis Monitor Log format
+  - Remove extra space in ASA-6-106015 rule
+  - fix COMMONAPACHELOG specs
+  - Added SuSEfirewall2 pattern
+  - switch USER to HTTPDUSER for "auth" field (match email addresses)
+  - bind9 pattern
+  - Pattern for squid3 native format
+  - Parse Cisco ASA-5-304001
+  - use underscores instead of hyphens in field names
+  - fix timestamp expect
+  - fix cs_protocol pattern name
+  - fix cs_protocol and cs_uri_query names
+  - added cloudfront spec test
+  - add pattern for cloudfront access log
+  - Java Patterns: JAVASTACKTRACEPART was duplicate
+
 ## 4.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
 ## 4.0.1
   - Republish all the gems under jruby.
+
 ## 4.0.0
   - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
-# 2.0.5
+
+## 2.0.5
   - Specs fixes, see https://github.com/logstash-plugins/logstash-patterns-core/pull/137
-# 2.0.4
+
+## 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.3
+
+## 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 
-# 0.4.0
+## 0.4.0
  - Added grok patterns for nagios notifications
  - Added commong exim patterns
  - Allow optional space between sysloghost and colon, fixes https://github.com/elastic/logstash/issues/2101 for Cisco ASA devises.
@@ -32,17 +60,17 @@
  - Add basic apache httpd error log format
  - Support TIMESTAMP_ISO8601 in HAProxy patterns, useful for rsyslog and other systems that can be configured to use this format. Fixes https://github.com/logstash-plugins/logstash-patterns-core/pull/80
 
-# 0.3.0
+## 0.3.0
  - Updated the AWS S3 patterns
  - Added patterns for rails 3
  - Added patterns for haproxy
  - Added patterns for bro http.log
  - Added shorewall patterns
-# 0.2.0
+## 0.2.0
  - Added patterns for S3 and ELB access logs amazon services
-# 0.1.12
+## 0.1.12
  - add some missing Cisco ASA firewall system log patterns
  - fix cisco firewall policy_id regex for policies with '-' in the name
-# 0.1.11
+## 0.1.11
  - Added Catalina and Tomcat patterns
  - Added German month names

data/logstash-patterns-core.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-patterns-core'
-  s.version         = '4.0.2'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Patterns to be used in logstash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/patterns/aws

@@ -9,3 +9,6 @@ ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{
 ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
 ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}"
+
+CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
+

data/patterns/bind

@@ -0,0 +1,3 @@
+BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
+
+BIND9 %{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \(%{GREEDYDATA:query}\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \(%{IP:dns}\)

data/patterns/firewalls

@@ -36,7 +36,7 @@ CISCOFW106006_106007_106010 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction}
 # ASA-3-106014
 CISCOFW106014 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\)
 # ASA-6-106015
-CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags}  on interface %{GREEDYDATA:interface}
+CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface}
 # ASA-1-106021
 CISCOFW106021 %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface}
 # ASA-4-106023
@@ -45,6 +45,8 @@ CISCOFW106023 %{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src
 CISCOFW106100_2_3 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
 # ASA-5-106100
 CISCOFW106100 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
+# ASA-5-304001
+CISCOFW304001 %{IP:src_ip}(\(%{DATA:src_fwuser}\))? Accessed URL %{IP:dst_ip}:%{GREEDYDATA:dst_url}
 # ASA-6-110002
 CISCOFW110002 %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}
 # ASA-6-302010
@@ -84,3 +86,6 @@ CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Cu
 # Shorewall firewall logs
 SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
 #== End Shorewall
+#== SuSE Firewall 2 ==
+SFW2 ((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\s*%{HOSTNAME}\s*kernel\S+\s*%{NAGIOSTIME}\s*SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=((\s*%{USERNAME:nf_out_interface})|(\s*))MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
+#== End SuSE ==

data/patterns/grok-patterns

@@ -2,7 +2,6 @@ USERNAME [a-zA-Z0-9._-]+
 USER %{USERNAME}
 EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
 EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
-HTTPDUSER %{EMAILADDRESS}|%{USER}
 INT (?:[+-]?(?:[0-9]+))
 BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
 NUMBER (?:%{BASE10NUM})
@@ -77,7 +76,6 @@ DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
 DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
 DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
 DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
-HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
 
 # Syslog Dates: Month Day HH:MM:SS
 SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
@@ -92,12 +90,6 @@ QS %{QUOTEDSTRING}
 
 # Log formats
 SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
-COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
-COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
-HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
-HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
-
 
 # Log Levels
 LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

data/patterns/httpd

@@ -0,0 +1,15 @@
+HTTPDUSER %{EMAILADDRESS}|%{USER}
+HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
+
+# Log formats
+HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
+
+# Error logs
+HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
+HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
+
+# Deprecated
+COMMONAPACHELOG %{HTTPD_COMMONLOG}
+COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}

data/patterns/java

@@ -9,7 +9,6 @@ JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFI
 JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
 JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
 JAVAFILE (?:[A-Za-z0-9_.-]+)
-JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
 JAVALOGMESSAGE (.*)
 # MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
 CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)

data/patterns/linux-syslog

@@ -11,6 +11,6 @@ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
 # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
 SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
 SYSLOG5424SD \[%{DATA}\]+
-SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
+SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{IPORHOST:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
 
 SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

data/patterns/maven

@@ -0,0 +1 @@
+MAVEN_VERSION (?:(\d+)\.)?(?:(\d+)\.)?(\*|\d+)(?:[.-](RELEASE|SNAPSHOT))?

data/patterns/redis

@@ -1,3 +1,3 @@
 REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME}
 REDISLOG \[%{POSINT:pid}\] %{REDISTIMESTAMP:timestamp} \* 
-
+REDISMONLOG %{NUMBER:timestamp} \[%{INT:database} %{IP:client}:%{NUMBER:port}\] "%{WORD:command}"\s?%{GREEDYDATA:params}

data/patterns/squid

@@ -0,0 +1,4 @@
+# Pattern squid3
+# Documentation of squid3 logs formats can be found at the following link:
+# http://wiki.squid-cache.org/Features/LogFormat
+SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:duration}\s%{IP:client_address}\s%{WORD:cache_result}/%{POSINT:status_code}\s%{NUMBER:bytes}\s%{WORD:request_method}\s%{NOTSPACE:url}\s(%{NOTSPACE:user}|-)\s%{WORD:hierarchy_code}/%{IPORHOST:server}\s%{NOTSPACE:content_type}

data/spec/patterns/core_spec.rb

@@ -20,16 +20,6 @@ describe "SYSLOGLINE" do
 
 end
 
-describe "COMMONAPACHELOG" do
-
-  let(:value) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36'}
-
-  it "generates the clientip field" do
-    expect(grok_match(subject, value)).to include("clientip" => "83.149.9.216")
-  end
-
-end
-
 describe "HTTP DATE parsing" do
 
   context "HTTPDATE", "when having a German month" do

data/spec/patterns/firewalls_spec.rb

@@ -19,6 +19,21 @@ describe "FIREWALLS" do
       expect(subject["message"]).to include("(Secondary) Switching to ACTIVE - Service card in other unit has failed")
     end
   end
+  
+  let(:pattern106015) { "CISCOFW106015" }
+
+  context "parsing a 106015 message" do
+
+    let(:value) { "Deny TCP (no connection) from 192.168.150.65/2278 to 64.101.128.83/80 flags RST on interface inside" }
+
+    subject     { grok_match(pattern106015, value) }
+
+    it { should include("interface" => "inside") }
+
+    it "generates a message field" do
+      expect(subject["message"]).to include("Deny TCP (no connection) from 192.168.150.65/2278 to 64.101.128.83/80 flags RST on interface inside")
+    end
+  end
 
   let(:pattern106100)    { "CISCOFW106100" }
 
@@ -50,6 +65,22 @@ describe "FIREWALLS" do
     end
   end
 
+  let(:pattern304001)    { "CISCOFW304001" }
+
+  context "parsing a 304001 message" do
+
+    let(:value) { "10.20.30.40(DOMAIN\\login) Accessed URL 10.11.12.13:http://example.org/" }
+
+    subject     { grok_match(pattern304001, value) }
+
+    it 'should break the message up into fields' do
+      expect(subject['src_ip']).to eq('10.20.30.40')
+      expect(subject['src_fwuser']).to eq('DOMAIN\\login')
+      expect(subject['dst_ip']).to eq('10.11.12.13')
+      expect(subject['dst_url']).to eq('http://example.org/')
+    end
+  end
+
   let(:pattern106023) { "CISCOFW106023" }
 
   context "parsing a 106023 message" do

data/spec/patterns/httpd_spec.rb

@@ -2,23 +2,126 @@
 require "spec_helper"
 require "logstash/patterns/core"
 
+describe "HTTPD_COMBINEDLOG" do
+
+  context "HTTPD_COMBINEDLOG", "Typical test case" do
+
+    let(:value) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
+
+    it "generates the clientip field" do
+      expect(grok_match(subject, value)).to include(
+        'clientip' => '83.149.9.216',
+        'verb' => 'GET',
+        'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
+        'httpversion' => '1.1',
+        'response' => '200',
+        'bytes' => '203023',
+        'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
+        'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
+      )
+    end
+
+  end
+
+  context "HTTPD_COMBINEDLOG", "Email address in auth field" do
+
+    let(:value) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
+
+    it "generates the clientip field" do
+      expect(grok_match(subject, value)).to include("auth" => "username@example.com")
+    end
+
+  end
+
+end
+
 describe "HTTPD_ERRORLOG" do
 
-  it "matches a full httpd 2.4 message" do
-    expect(subject).to match("[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product")
+  context "HTTPD_ERRORLOG", "matches a full httpd 2.4 message" do
+    let(:value) {
+      "[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
+    }
+    it "generates the fields" do
+
+      expect(grok_match(subject, value)).to include(
+        'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
+        'module' => 'proxy_fcgi',
+        'loglevel' => 'error',
+        'pid' => '28787',
+        'tid' => '140169587934976',
+        'proxy_errorcode' => '70008',
+        'proxy_message' => 'Partial results are valid but processing is incomplete',
+        'clientip' => '58.13.45.166',
+        'clientport' => '59307',
+        'errorcode' => 'AH01075',
+        'message' => [ value, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ],
+      )
+    end
   end
 
-  it "matches a httpd 2.2 log message" do
-    expect(subject).to match("[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com")
+  context "HTTPD_ERRORLOG", "matches a httpd 2.2 log message" do
+    let(:value) {
+      "[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value)).to include(
+        'timestamp' => 'Mon Aug 31 16:27:04 2015',
+        'loglevel' => 'error',
+        'clientip' => '10.17.42.3',
+        'message' => [ value, 'Premature end of script headers: example.com' ]
+      )
+    end
   end
 
-  it "matches a short httpd 2.4 message" do
-    expect(subject).to match("[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'")
+  context "HTTPD_ERRORLOG", "matches a short httpd 2.4 message" do
+    let(:value) {
+      "[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value)).to include(
+        'timestamp' => 'Mon Aug 31 07:15:38.664897 2015',
+        'module' => 'proxy_fcgi',
+        'loglevel' => 'error',
+        'pid' => '28786',
+        'tid' => '140169629898496',
+        'clientip' => '81.139.1.34',
+        'clientport' => '52042',
+        'errorcode' => 'AH01071',
+        'message' => [ value, "Got error 'Primary script unknown\n'" ]
+      )
+    end
   end
 
-  it "matches an httpd 2.4 restart" do
-    expect(subject).to match("[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations")
-    expect(subject).to match("[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'")
+  context "HTTPD_ERRORLOG", "matches an httpd 2.4 restart" do
+    let(:value1) {
+      "[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value1)).to include(
+        'timestamp' => 'Mon Aug 31 06:29:47.406518 2015',
+        'module' => 'mpm_event',
+        'loglevel' => 'notice',
+        'pid' => '24968',
+        'tid' => '140169861986176',
+        'errorcode' => 'AH00489',
+        'message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ]
+      )
+    end
+
+    let(:value2) {
+      "[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
+    }
+    it "generates the fields" do
+      expect(grok_match(subject, value2)).to include(
+        'timestamp' => 'Mon Aug 31 06:29:47.406530 2015',
+        'module' => 'core',
+        'loglevel' => 'notice',
+        'pid' => '24968',
+        'tid' => '140169861986176',
+        'errorcode' => 'AH00094',
+        'message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ]
+      )
+    end
   end
 
 end

data/spec/patterns/maven_spec.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "MAVEN_VERSION" do
+
+  let(:pattern) { 'MAVEN_VERSION' }
+
+  context "when maven version is simple" do
+    let(:value) { '1.1.0' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version is a bit more complex" do
+    let(:value) { '2.35.128' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version contains release" do
+    let(:value) { '1.1.0.RELEASE' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version contains shapshot" do
+    let(:value) { '1.1.0.SNAPSHOT' }
+
+    it "should match the version" do
+      expect(grok_match(pattern,value)).to pass
+    end
+  end
+
+  context "when maven version contains release" do
+    context "and the version contains a dash" do
+      let(:value) { '1.1.0-RELEASE' }
+
+      it "should match the version" do
+        expect(grok_match(pattern,value)).to pass
+      end
+    end
+  end
+
+  context "when maven version contains shapshot" do
+    context "and the version contains a dash" do
+    let(:value) { '1.1.0-SNAPSHOT' }
+
+      it "should match the version" do
+        expect(grok_match(pattern,value)).to pass
+      end
+    end
+  end
+
+end

data/spec/patterns/redis_spec.rb

@@ -0,0 +1,171 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "REDISTIMESTAMP" do
+
+  let(:value) { '14 Nov 07:01:22.119'}
+  let(:pattern) { "REDISTIMESTAMP" }
+
+  it "a pattern pass the grok expression" do
+    expect(grok_match(pattern, value)).to pass
+  end
+
+end
+
+describe "REDISLOG" do
+
+  let(:value)   { "[4018] 14 Nov 07:01:22.119 * Background saving terminated with success" }
+  let(:pattern) { "REDISLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the pid field" do
+    expect(grok).to include("pid" => "4018")
+  end
+
+end
+
+
+describe "REDISMONLOG - SIMPLE COMMAND" do
+
+  let(:value)   { "1470637867.953466 [0 195.168.1.1:52500] \"info\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1470637867.953466")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "0")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "195.168.1.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "52500")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "info")
+  end
+
+end
+
+describe "REDISMONLOG - ONE PARAM COMMAND" do
+
+  let(:value)   { "1339518083.107412 [0 127.0.0.1:60866] \"keys\" \"*\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1339518083.107412")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "0")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "127.0.0.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "60866")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "keys")
+  end
+
+  it "generates the params field" do
+    expect(grok).to include("params" => "\"*\"")
+  end
+
+end
+
+describe "REDISMONLOG - TWO PARAM COMMAND" do
+
+  let(:value)   { "1470637925.186681 [0 127.0.0.1:39404] \"rpush\" \"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1470637925.186681")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "0")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "127.0.0.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "39404")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "rpush")
+  end
+
+  it "generates the params field" do
+    expect(grok).to include("params" => "\"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"")
+  end
+
+end
+
+describe "REDISMONLOG - VARIADIC COMMAND" do
+
+  let(:value)   { "1470637875.777457 [15 195.168.1.1:52500] \"intentionally\" \"broken\" \"variadic\" \"log\" \"entry\"" }
+  let(:pattern) { "REDISMONLOG" }
+  let(:grok)    { grok_match(pattern, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "generates the timestamp field" do
+    expect(grok).to include("timestamp" => "1470637875.777457")
+  end
+
+  it "generates the database field" do
+    expect(grok).to include("database" => "15")
+  end
+
+  it "generates the client field" do
+    expect(grok).to include("client" => "195.168.1.1")
+  end
+
+  it "generates the port field" do
+    expect(grok).to include("port" => "52500")
+  end
+
+  it "generates the command field" do
+    expect(grok).to include("command" => "intentionally")
+  end
+
+  it "generates the params field" do
+    expect(grok).to include("params" => "\"broken\" \"variadic\" \"log\" \"entry\"")
+  end
+
+end

data/spec/patterns/s3_spec.rb

@@ -130,3 +130,44 @@ describe "S3_ACCESS_LOG" do
 
   end
 end
+
+describe "CLOUDFRONT_ACCESS_LOG" do
+
+  let(:pattern) { "CLOUDFRONT_ACCESS_LOG" }
+
+  context "parsing a cloudfront access log" do
+
+    let(:value) { "2016-06-10	18:41:39	IAD53	224281	192.168.1.1	GET	d27enomp470abc.cloudfront.net	/content/sample/thing.pdf	200	https://example.com/	Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36	-	-	Miss	UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==	host.example.com	https	883	0.036	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Miss" }
+
+    subject { grok_match(pattern, value) }
+
+    it { should include("timestamp" => "2016-06-10	18:41:39" ) }
+    it { should include("x_edge_location" => "IAD53" ) }
+    it { should include("sc_bytes" => 224281 ) }
+    it { should include("clientip" => "192.168.1.1" ) }
+    it { should include("cs_method" =>  "GET" ) }
+    it { should include("cs_host" => "d27enomp470abc.cloudfront.net" ) }
+    it { should include("cs_uri_stem" => "/content/sample/thing.pdf" ) }
+    it { should include("sc_status" => 200 ) }
+    it { should include("referrer" => "https://example.com/" ) }
+    it { should include("agent" => "Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520WOW64)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/51.0.2704.79%2520Safari/537.36" ) }
+    it { should include("cs_uri_query" => "-" ) }
+    it { should include("cookies" => "-" ) }
+    it { should include("x_edge_result_type" => "Miss" ) }
+    it { should include("x_edge_request_id" => "UGskZ6dUKY7b4C6Pt7wAWVsU2KO-vTRe-mR4r9H-WQMjhNvY6w1Xcg==" ) }
+    it { should include("x_host_header" => "host.example.com" ) }
+    it { should include("cs_protocol" => "https" ) }
+    it { should include("cs_bytes" => 883 ) }
+    it { should include("time_taken" => 0.036 ) }
+    it { should include("x_forwarded_for" => "-" ) }
+    it { should include("ssl_protocol" => "TLSv1.2" ) }
+    it { should include("ssl_cipher" => "ECDHE-RSA-AES128-GCM-SHA256" ) }
+    it { should include("x_edge_response_result_type" => "Miss" ) }
+
+    ["tags", "params"].each do |attribute|
+      it "have #{attribute} as nil" do
+        expect(subject[attribute]).to be_nil
+      end
+    end
+  end
+end

data/spec/patterns/syslog_spec.rb

@@ -4,6 +4,20 @@ require "logstash/patterns/core"
 
 describe "SYSLOGLINE" do
 
+  describe "SYSLOG5424BASE" do
+     it "matches host names in the syslog base pattern" do
+        expect(subject).to match("<174>1 2016-11-14T09:32:44+01:00 resolver.se named 6344 - -  info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)")
+     end
+     
+     it "matches ipv4 in the syslog base pattern" do
+        expect(subject).to match("<174>1 2016-11-14T09:49:23+01:00 10.23.16.6 named 2255 - -  info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)")
+     end
+
+     it "matches ipv6 in the syslog base pattern" do
+        expect(subject).to match("<174>1 2016-11-14T09:49:23+01:00 2000:6a0:b:315:10:23:4:13 named 2255 - -  info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)")
+     end
+   end
+
   it "matches a simple message with pid" do
     expect(subject).to match("May 11 15:17:02 meow.soy.se CRON[10973]: pam_unix(cron:session): session opened for user root by (uid=0)")
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-patterns-core
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,14 +74,17 @@ files:
 - logstash-patterns-core.gemspec
 - patterns/aws
 - patterns/bacula
+- patterns/bind
 - patterns/bro
 - patterns/exim
 - patterns/firewalls
 - patterns/grok-patterns
 - patterns/haproxy
+- patterns/httpd
 - patterns/java
 - patterns/junos
 - patterns/linux-syslog
+- patterns/maven
 - patterns/mcollective
 - patterns/mcollective-patterns
 - patterns/mongodb
@@ -90,14 +93,17 @@ files:
 - patterns/rails
 - patterns/redis
 - patterns/ruby
+- patterns/squid
 - spec/patterns/bro.rb
 - spec/patterns/core_spec.rb
 - spec/patterns/firewalls_spec.rb
 - spec/patterns/haproxy_spec.rb
 - spec/patterns/httpd_spec.rb
+- spec/patterns/maven_spec.rb
 - spec/patterns/mongodb_spec.rb
 - spec/patterns/nagios_spec.rb
 - spec/patterns/rails3_spec.rb
+- spec/patterns/redis_spec.rb
 - spec/patterns/s3_spec.rb
 - spec/patterns/shorewall_spec.rb
 - spec/patterns/syslog_spec.rb
@@ -123,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Patterns to be used in logstash
@@ -133,9 +139,11 @@ test_files:
 - spec/patterns/firewalls_spec.rb
 - spec/patterns/haproxy_spec.rb
 - spec/patterns/httpd_spec.rb
+- spec/patterns/maven_spec.rb
 - spec/patterns/mongodb_spec.rb
 - spec/patterns/nagios_spec.rb
 - spec/patterns/rails3_spec.rb
+- spec/patterns/redis_spec.rb
 - spec/patterns/s3_spec.rb
 - spec/patterns/shorewall_spec.rb
 - spec/patterns/syslog_spec.rb