logstash-patterns-core 0.1.7 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fcd360fe687a5118b3700bb1cd1de088db315ac
-  data.tar.gz: 67e9076484ffd330acdf2e5d1c698ab0fa1469b5
+  metadata.gz: fb94df5ab3f3e8ee6e7204b1c6f6b44055b40d26
+  data.tar.gz: 5f6edd7688daf729dfad7b17a40f98744f36ea48
 SHA512:
-  metadata.gz: 4b35d6dc3fc597b4bc1bc0c3f757d4952ce22119b544c04741f7ac8eb8656aa1f52c7097db5ae6aab7d10f624dac8cf43df2678f82133b0fc5cb898f10504e1d
-  data.tar.gz: 1ac9b7c195dd4e8a0017492d8f9c353014ce8ecb592b74b5ec198dc41cf31e0304d803c4bb1dc1cc372af99943ab2f2d92a96a71c4fc803622fb3da46169a614
+  metadata.gz: 08a6bd3187a03e03f277b6c17bf2a78084a68271a0958cda6f1cae0f3679e7b8873f797048be49788d8039ac2456a6556947b78316e8a0d1a41b959e5dfe9c27
+  data.tar.gz: 4d7784a60f00fa548feb0946d71abe36c2cffb7249cbbf306dbb773950735ead55b501fd074392de6e633296c1f16f59aadb3bdc68c05469a2eda3355ff65194

data/logstash-patterns-core.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-patterns-core'
-  s.version         = '0.1.7'
+  s.version         = '0.1.10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Patterns to be used in logstash"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
 
+  s.add_development_dependency 'logstash-filter-grok'
   s.add_development_dependency 'logstash-devutils'
 end
 

data/patterns/mongodb

@@ -2,3 +2,6 @@ MONGO_LOG %{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:messag
 MONGO_QUERY \{ (?<={ ).*(?= } ntoreturn:) \}
 MONGO_SLOWQUERY %{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ (?<duration>[0-9]+)ms
 MONGO_WORDDASH \b[\w-]+\b
+MONGO3_SEVERITY \w
+MONGO3_COMPONENT %{WORD}|-
+MONGO3_LOG %{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:\[%{DATA:context}\])? %{GREEDYDATA:message}

data/patterns/nagios

@@ -66,7 +66,7 @@ NAGIOS_CURRENT_SERVICE_STATE %{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %
 NAGIOS_CURRENT_HOST_STATE %{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}
 
 NAGIOS_SERVICE_NOTIFICATION %{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
-NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
+NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
 
 NAGIOS_SERVICE_ALERT %{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}
 NAGIOS_HOST_ALERT %{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}
@@ -83,7 +83,7 @@ NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:
 NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 
-NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2};
+NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}
 
 ####################
 #### External checks

data/spec/patterns/core_spec.rb

@@ -1,6 +1,31 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require 'logstash/patterns/core'
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "SYSLOGLINE" do
+
+  let(:value)   { "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" }
+  let(:grok)    { grok_match(subject, value) }
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the program field" do
+    expect(grok_match(subject, value)).to include("program" => "postfix/smtpd")
+  end
+
+end
+
+describe "COMMONAPACHELOG" do
+
+  let(:value) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36'}
+
+  it "generates the clientip field" do
+    expect(grok_match(subject, value)).to include("clientip" => "83.149.9.216")
+  end
 
-describe LogStash::Patterns::Core do
 end

data/spec/patterns/mongodb_spec.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "MONGO3_LOG" do
+
+  let(:pattern)    { "MONGO3_LOG" }
+
+  context "parsing an standard/basic message" do
+
+    let(:value) { "2014-11-03T18:28:32.450-0500 I NETWORK [initandlisten] waiting for connections on port 27017" }
+
+    subject     { grok_match(pattern, value) }
+
+    it { should include("timestamp" => "2014-11-03T18:28:32.450-0500") }
+
+    it { should include("severity" => "I") }
+
+    it { should include("component" => "NETWORK") }
+
+    it { should include("context" => "initandlisten") }
+
+    it "generates a message field" do
+      expect(subject["message"]).to include("waiting for connections on port 27017")
+    end
+  end
+
+  context "parsing a message with a missing component" do
+
+    let(:value) { "2015-02-24T18:17:47.148+0000 F -        [conn11] Got signal: 11 (Segmentation fault)." }
+
+    subject     { grok_match(pattern, value) }
+
+    it { should include("timestamp" => "2015-02-24T18:17:47.148+0000") }
+
+    it { should include("severity" => "F") }
+
+    it { should include("component" => "-") }
+
+    it { should include("context" => "conn11") }
+
+    it "generates a message field" do
+      expect(subject["message"]).to include("Got signal: 11 (Segmentation fault).")
+    end
+  end
+
+  context "parsing a message with a multiwords context" do
+
+    let(:value) { "2015-04-23T06:57:28.256+0200 I JOURNAL  [journal writer] Journal writer thread started" }
+
+    subject     { grok_match(pattern, value) }
+
+    it { should include("timestamp" => "2015-04-23T06:57:28.256+0200") }
+
+    it { should include("severity" => "I") }
+
+    it { should include("component" => "JOURNAL") }
+
+    it { should include("context" => "journal writer") }
+
+    it "generates a message field" do
+      expect(subject["message"]).to include("Journal writer thread started")
+    end
+  end
+
+  context "parsing a message without context" do
+
+    let(:value) { "2015-04-23T07:00:13.864+0200 I CONTROL  Ctrl-C signal" }
+
+    subject     { grok_match(pattern, value) }
+
+    it { should include("timestamp" => "2015-04-23T07:00:13.864+0200") }
+
+    it { should include("severity" => "I") }
+
+    it { should include("component" => "CONTROL") }
+
+    it { should_not have_key("context") }
+
+    it "generates a message field" do
+      expect(subject["message"]).to include("Ctrl-C signal")
+    end
+  end
+end

data/spec/patterns/nagios_spec.rb

@@ -0,0 +1,243 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "NAGIOSLOGLINE - CURRENT HOST STATE" do
+
+  let(:value)   { "[1427925600] CURRENT HOST STATE: nagioshost;UP;HARD;1;PING OK - Packet loss = 0%, RTA = 2.24 ms" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the nagios_epoch field" do
+    expect(grok).to include("nagios_epoch" => "1427925600")
+  end
+
+  it "generates the nagios_message field" do
+    expect(grok).to include("nagios_message" => "PING OK - Packet loss = 0%, RTA = 2.24 ms")
+  end
+
+  it "generates the nagios_hostname field" do
+    expect(grok).to include("nagios_hostname" => "nagioshost")
+  end
+
+  it "generates the nagios_state field" do
+    expect(grok).to include("nagios_state" => "UP")
+  end
+
+  it "generates the nagios_statetype field" do
+    expect(grok).to include("nagios_statetype" => "HARD")
+  end
+
+end
+
+describe "NAGIOSLOGLINE - CURRENT SERVICE STATE" do
+
+  let(:value)   { "[1427925600] CURRENT SERVICE STATE: nagioshost;nagiosservice;OK;HARD;1;nagiosmessage" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the nagios_type field" do
+    expect(grok).to include("nagios_type" => "CURRENT SERVICE STATE")
+  end
+
+  it "generates the nagios_epoch field" do
+    expect(grok).to include("nagios_epoch" => "1427925600")
+  end
+
+  it "generates the nagios_message field" do
+    expect(grok).to include("nagios_message" => "nagiosmessage")
+  end
+
+  it "generates the nagios_hostname field" do
+    expect(grok).to include("nagios_hostname" => "nagioshost")
+  end
+
+  it "generates the nagios_service field" do
+    expect(grok).to include("nagios_service" => "nagiosservice")
+  end
+
+  it "generates the nagios_state field" do
+    expect(grok).to include("nagios_state" => "OK")
+  end
+
+  it "generates the nagios_statetype field" do
+    expect(grok).to include("nagios_statetype" => "HARD")
+  end
+
+end
+
+describe "NAGIOSLOGLINE - TIMEPERIOD TRANSITION" do
+
+  let(:value)   { "[1427925600] TIMEPERIOD TRANSITION: 24X7;1;1" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the nagios_type field" do
+    expect(grok).to include("nagios_type" => "TIMEPERIOD TRANSITION")
+  end
+
+  it "generates the nagios_epoch field" do
+    expect(grok).to include("nagios_epoch" => "1427925600")
+  end
+
+  it "generates the nagios_esrvice field" do
+    expect(grok).to include("nagios_service" => "24X7")
+  end
+
+  # Regression test for but fixed in Nagios patterns #30
+  it "doesn't end in a semi-colon" do
+    expect(grok['message']).to_not end_with(";")
+  end
+
+end
+
+describe "NAGIOSLOGLINE - SERVICE ALERT" do
+
+  let(:value)   { "[1427925689] SERVICE ALERT: varnish;Varnish Backend Connections;CRITICAL;SOFT;1;Current value: 154.0, warn threshold: 10.0, crit threshold: 20.0" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the nagios_type field" do
+    expect(grok).to include("nagios_type" => "SERVICE ALERT")
+  end
+
+  it "generates the nagios_epoch field" do
+    expect(grok).to include("nagios_epoch" => "1427925689")
+  end
+
+  it "generates the nagios_hostname field" do
+    expect(grok).to include("nagios_hostname" => "varnish")
+  end
+
+  it "generates the nagios_service field" do
+    expect(grok).to include("nagios_service" => "Varnish Backend Connections")
+  end
+
+  it "generates the nagios_state field" do
+    expect(grok).to include("nagios_state" => "CRITICAL")
+  end
+
+  it "generates the nagios_statelevel field" do
+    expect(grok).to include("nagios_statelevel" => "SOFT")
+  end
+
+  it "generates the nagios_message field" do
+    expect(grok).to include("nagios_message" => "Current value: 154.0, warn threshold: 10.0, crit threshold: 20.0")
+  end
+
+end
+
+describe "NAGIOSLOGLINE - SERVICE NOTIFICATION" do
+
+  let(:value)   { "[1427950229] SERVICE NOTIFICATION: nagiosadmin;varnish;Varnish Backend Connections;CRITICAL;notify-service-by-email;Current value: 337.0, warn threshold: 10.0, crit threshold: 20.0" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the nagios_type field" do
+    expect(grok).to include("nagios_type" => "SERVICE NOTIFICATION")
+  end
+
+  it "generates the nagios_epoch field" do
+    expect(grok).to include("nagios_epoch" => "1427950229")
+  end
+
+  it "generates the nagios_notifyname field" do
+    expect(grok).to include("nagios_notifyname" => "nagiosadmin")
+  end
+
+  it "generates the nagios_hostname field" do
+    expect(grok).to include("nagios_hostname" => "varnish")
+  end
+
+  it "generates the nagios_service field" do
+    expect(grok).to include("nagios_service" => "Varnish Backend Connections")
+  end
+
+  it "generates the nagios_state field" do
+    expect(grok).to include("nagios_state" => "CRITICAL")
+  end
+
+  it "generates the nagios_contact field" do
+    expect(grok).to include("nagios_contact" => "notify-service-by-email")
+  end
+
+  it "generates the nagios_message field" do
+    expect(grok).to include("nagios_message" => "Current value: 337.0, warn threshold: 10.0, crit threshold: 20.0")
+  end
+
+end
+
+
+describe "NAGIOSLOGLINE - HOST NOTIFICATION" do
+
+  let(:value)   { "[1429878690] HOST NOTIFICATION: nagiosadmin;nagioshost;DOWN;notify-host-by-email;CRITICAL - Socket timeout after 10 seconds" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the nagios_type field" do
+    expect(grok).to include("nagios_type" => "HOST NOTIFICATION")
+  end
+
+  it "generates the nagios_epoch field" do
+    expect(grok).to include("nagios_epoch" => "1429878690")
+  end
+
+  it "generates the nagios_notifyname field" do
+    expect(grok).to include("nagios_notifyname" => "nagiosadmin")
+  end
+
+  it "generates the nagios_hostname field" do
+    expect(grok).to include("nagios_hostname" => "nagioshost")
+  end
+
+  it "generates the nagios_contact field" do
+    expect(grok).to include("nagios_contact" => "notify-host-by-email")
+  end
+
+  it "generates the nagios_message field" do
+    expect(grok).to include("nagios_message" => "CRITICAL - Socket timeout after 10 seconds")
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,50 @@
+require "logstash/devutils/rspec/spec_helper"
+require 'rspec/expectations'
+
+# running the grok code outside a logstash package means
+# LOGSTASH_HOME will not be defined, so let's set it here
+# before requiring the grok filter
+unless LogStash::Environment.const_defined?(:LOGSTASH_HOME)
+  LogStash::Environment::LOGSTASH_HOME = File.expand_path("../", __FILE__)
+end
+
+require "logstash/filters/grok"
+
+module GrokHelpers
+  def grok_match(label, message)
+    grok  = build_grok(label)
+    event = build_event(message)
+    grok.filter(event)
+    event.to_hash
+  end
+
+  def build_grok(label)
+    grok = LogStash::Filters::Grok.new("match" => ["message", "%{#{label}}"])
+    grok.register
+    grok
+  end
+
+  def build_event(message)
+    LogStash::Event.new("message" => message)
+  end
+end
+
+RSpec.configure do |c|
+  c.include GrokHelpers
+end
+
+RSpec::Matchers.define :pass do |expected|
+  match do |actual|
+    !actual.include?("tags")
+  end
+end
+
+RSpec::Matchers.define :match do |value|
+  match do |grok|
+    grok  = build_grok(grok)
+    event = build_event(value)
+    grok.filter(event)
+    !event.include?("tags")
+  end
+end
+

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-patterns-core
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.10
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,20 +28,36 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-filter-grok
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -72,6 +86,9 @@ files:
 - patterns/redis
 - patterns/ruby
 - spec/patterns/core_spec.rb
+- spec/patterns/mongodb_spec.rb
+- spec/patterns/nagios_spec.rb
+- spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -99,3 +116,6 @@ specification_version: 4
 summary: Patterns to be used in logstash
 test_files:
 - spec/patterns/core_spec.rb
+- spec/patterns/mongodb_spec.rb
+- spec/patterns/nagios_spec.rb
+- spec/spec_helper.rb