logstash-codec-cef 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 287752d7fdba67aa7b4b3b3d5e484c510f41566a
-  data.tar.gz: 02fa53c9ab7b704672f587b50bad646791c1f26b
+  metadata.gz: 01d1c0ba310055e838ee41c651075e1aa1ee7189
+  data.tar.gz: 6f05794989c0ed626c67aa95ef4d81fbbbb046d7
 SHA512:
-  metadata.gz: 1649c71394718505fdde40f51e861c818951ca68a58f242ebaf697bc95b7ebe9a1b9ce198defe95e5ee65317741dc09a0838e91e34121763f5793ca10040f867
-  data.tar.gz: fbe45a2913364ce7384aeda3ef0afb064a89c56555f0636ca377fe48a181cd68b3a6b7638045b64be5fb0d9742693a54b73e8cd4c626825b8e29ebdfcf950c18
+  metadata.gz: 90b949419c532e776a5f86ed660f99e67c343d25c2d4fd93927519a3ed200f9ed0d840593b87c252a35bbf9372f56ae66ad778838a5a99a0839d9787e1d9d15e
+  data.tar.gz: 200cb50dff68168aac3e81bc64247899c149ac76dc7281cd3454f4f1d7394a6e3b8d6257513f370bd3f444d138abe273dc2c56d547284c8fbbff412aa516defb

data/.gitignore

@@ -2,3 +2,6 @@ build
 vendor
 tools
 .VERSION.mk
+*.gem
+*.lock
+*.swp

data/CONTRIBUTORS

@@ -10,6 +10,7 @@ Contributors:
 * Nick Ethier (nickethier)
 * Pete Fritchman (fetep)
 * Pier-Hugues Pellerin (ph)
+* Karl Stoney (Stono)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -13,9 +13,18 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
-## Developing
+## Developing with Docker
+You can use a docker container with all of the requirements pre installed to save you installing the development environment on your host.
+
+### 1. Starting the container
+Simply type `docker-compose run devenv` and you'll be entered into the container.  Then you'll need to do `jruby -S bundle install` to get all the dependencies down.
+
+### 2. Running tests
+Once you've done #1 above, you can run your tests with `jruby -S bundle exec rspec`
+
+## Developing without Docker
 
 ### 1. Plugin Developement and Testing
 
@@ -83,4 +92,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/docker-compose.yml

@@ -0,0 +1,5 @@
+devenv:
+  image: hpess/devenv-jruby:master
+  entrypoint: /bin/bash
+  volumes:
+    - ./:/storage

data/lib/logstash/codecs/cef.rb

@@ -2,11 +2,6 @@ require "logstash/codecs/base"
 
 class LogStash::Codecs::CEF < LogStash::Codecs::Base
   config_name "cef"
-
-
-  # Specify if the Syslog header will be expected
-  config :syslog, :validate => :boolean, :default => false
-
   config :signature, :validate => :string, :default => "Logstash"
   config :name, :validate => :string, :default => "Logstash"
   config :sev, :validate => :number, :default => 6
@@ -20,31 +15,44 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
 
   public
   def decode(data)
-    # Need to break out the headers, then return the headers as individual fields, and the extension to be processed by a filter (ie: KV)
-    # %{SYSLOGDATE} %{HOST} CEF:Version|Device Vendor|Device Product|Device Version|SignatureID|Name|Severity|Extension
-    event = LogStash::Event.new()
-    if @syslog
-      @logger.debug("Expecting SYSLOG headers")
-      event['syslog'], data = data.split('CEF:', 2)
-      # Since we have the syslog headers, lets pull them out first and put them into their own field to be handled
-    else 
-      # We don't have syslog headers, so we just need to remove CEF:
-      data.sub! /^CEF:/, ''
-    end #if @syslog
-    # Now, break out the rest of the headers
-    event['cef_version'], event['cef_vendor'], event['cef_product'], event['cef_device_version'], event['cef_sigid'], event['cef_name'], event['cef_severity'], event['message'] =  data.scan /(?:[^\|\\]|\\.)+/
-    # Now, try to break out the Extension Dictionary
-    message=event['message']
-    if message.to_s.strip.length != 0
+    # Strip any quotations at the start and end, flex connectors seem to send this
+    if data[0] == "\""
+      data = data[1..-2]
+    end
+    event = LogStash::Event.new
+
+    # Split by the pipes
+    event['cef_version'], event['cef_vendor'], event['cef_product'], event['cef_device_version'], event['cef_sigid'], event['cef_name'], event['cef_severity'], message = data.split /(?<!\\)[\|]/
+
+    # Try and parse out the syslog header if there is one
+    if event['cef_version'].include? ' '
+      event['syslog'], unused, event['cef_version'] = event['cef_version'].rpartition(' ')
+    end
+
+    # Get rid of the CEF bit in the version
+    event['cef_version'].sub! /^CEF:/, ''
+
+    # Strip any whitespace from the message
+    if not message.nil? and message.include? '='
+      message = message.strip
+
+      # If the last KVP has no value, add an empty string, this prevents hash errors below
+      if message.end_with?("=")
+        message=message + ' '
+      end
+
+      # Now parse the key value pairs into it
+      extensions = {}
       message = message.split(/ ([\w\.]+)=/)
+      key, value = message.shift.split('=', 2)
+      extensions[key] = value
+
+      Hash[*message].each{ |k, v| extensions[k] = v }
+
+      # And save the new has as the extensions
+      event['cef_ext'] = extensions
+    end
 
-      key, value = message.shift.split('=',2)
-      @logger.debug(message)
-      kv = Hash[*message]
-      @logger.debug(kv)
-      addKey(kv,key,value)
-      event.to_hash.merge!(Hash[kv.map{ |k,v| ["cef_ext_"+k,v] }])
-    end #
     yield event
   end
 
@@ -62,19 +70,6 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     @on_event.call(header + " " + values + "\n")
   end
 
-  private
-  def addKey(kv_keys, key, value)
-    if kv_keys.has_key?(key)
-      if kv_keys[key].is_a? Array
-        kv_keys[key].push(value)
-      else
-        kv_keys[key] = [kv_keys[key], value]
-      end
-    else
-      kv_keys[key] = value
-    end
-  end # addKey
-
   private
   def get_value(name, event)
     val = event[name]

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '0.1.4'
+  s.version         = '0.1.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "CEF codec to parse CEF formated logs"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/cef_spec.rb

@@ -16,29 +16,73 @@ describe LogStash::Codecs::CEF do
   context "#decode" do
     let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
 
-    it "should parse the cef header" do
+    def validate(e) 
+      insist { e.is_a?(LogStash::Event) }
+      insist { e['cef_version'] } == "0"
+      insist { e['cef_device_version'] } == "1.0"
+      insist { e['cef_sigid'] } == "100"
+      insist { e['cef_name'] } == "trojan successfully stopped"
+      insist { e['cef_severity'] } == "10"
+    end
+
+    it "should parse the cef headers" do
       subject.decode(message) do |e|
-        insist { e.is_a?(LogStash::Event) }
-        insist { e["cef_version"] } == "0"
+        validate(e)
+        ext = e['cef_ext']
         insist { e["cef_vendor"] } == "security"
         insist { e["cef_product"] } == "threatmanager"
-        insist { e["cef_device_version"] } == "1.0"
-        insist { e["cef_sigid"] } == "100"
-        insist { e["cef_name"] } == "trojan successfully stopped"
-        insist { e["cef_severity"] } == "10"
-        insist { e["message"] } == "src=10.0.0.192 dst=12.121.122.82 spt=1232"
       end
     end
 
     it "should parse the cef body" do
       subject.decode(message) do |e|
-        insist { e["cef_ext_src"] } == "10.0.0.192"
-        insist { e["cef_ext_dst"] } == "12.121.122.82"
-        insist { e["cef_ext_spt"] } == "1232"
+        ext = e['cef_ext']
+        insist { ext['src'] } == "10.0.0.192"
+        insist { ext['dst'] } == "12.121.122.82"
+        insist { ext['spt'] } == "1232"
       end
     end
 
-    it "should handle values in the body that contain spaces"
+    let (:no_ext) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|" }
+    it "should be OK with no extension dictionary" do
+      subject.decode(no_ext) do |e|
+        validate(e)
+        insist { e["cef_ext"] } == nil
+      end 
+    end
+
+    let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    it "should be OK with missing CEF headers (multiple pipes in sequence)" do
+      subject.decode(missing_headers) do |e|
+        validate(e)
+        insist { e["cef_vendor"] } == ""
+        insist { e["cef_product"] } == ""
+      end 
+    end
+
+    let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    it "should strip leading whitespace from the message" do
+      subject.decode(leading_whitespace) do |e|
+        validate(e)
+      end 
+    end
+
+    let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
+    it "should be OK with escaped pipes in the message" do
+      subject.decode(escaped_pipes) do |e|
+        ext = e['cef_ext']
+        insist { ext['moo'] } == 'this\|has an escaped pipe'
+      end 
+    end
+
+    let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    it "Should detect headers before CEF starts" do
+      subject.decode(syslog) do |e|
+        validate(e)
+        insist { e['syslog'] } == 'Syslogdate Sysloghost'
+      end 
+    end
+
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -51,11 +51,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
+- NOTICE.TXT
 - README.md
 - Rakefile
+- docker-compose.yml
 - lib/logstash/codecs/cef.rb
 - lib/logstash/version.rb
 - logstash-codec-cef.gemspec
@@ -82,9 +85,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.7
 signing_key:
 specification_version: 4
 summary: CEF codec to parse CEF formated logs
 test_files:
 - spec/codecs/cef_spec.rb
+has_rdoc: