fluent-plugin-parser_cefalt 1.0.6

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 2.3.0
+script: bundle exec rspec
+gemfile:
+  - Gemfile

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gemspec
+gem 'coveralls', require: false
+gem 'simplecov'

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Tomoyuki Sugimura
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# fluent-plugin-parser_cefalt
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-parser_cefalt.svg)](https://badge.fury.io/rb/fluent-plugin-parser_cefalt)
+[![downloads](https://img.shields.io/gem/dt/fluent-plugin-parser_cefalt.svg)](https://rubygems.org/gems/fluent-plugin-parser_cefalt)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
+
+Fluentd Parser plugin to parse CEF - common event format - Alternative
+
+## Requirements
+
+| fluent-plugin-parser_cefalt  | fluentd | ruby |
+|---------------------------|---------|------|
+| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
+|  < 1.0.0 | >= v0.12.0 | >= 1.9 |
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```bash
+
+# for fluentd v0.14 or higher
+
+fluent-gem install fluent-plugin-parser_cefalt
+
+```
+
+## Usage
+
+```
+<source>
+  @type syslog
+  port 5514
+  bind 0.0.0.0
+  <transport tcp>
+  </transport>
+  <parse>
+    @type cefalt
+    log_format syslog
+    parse_strict_mode true
+    output_raw_field false
+  </parse>
+  tag ceflog
+</source>
+<match ceflog.**>
+  @type stdout
+</match>
+```
+
+## parameters
+
+* `log_format` (default: syslog)
+
+  input log format, currently only 'syslog' is valid
+
+* `log_utc_offset` (default: nil)
+
+  set log utc_offset if each record does not have timezone information and the timezone is not local timezone
+
+  if log_utc_offset set to nil or invalid value, then use system timezone
+
+  if a log have timezone information, log_utc_offset is ignored
+
+* `syslog_timestamp` (default: '\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}')
+
+* `syslog_timestamp_rfc5424` (default: '\d{4}[-]\d{2}[-]\d{2}[T]\d{2}[:]\d{2}[:]\d{2}(?:\.\d{1,6})?(?:[+-]\d{2}[:]\d{2}|Z)')
+
+* `cef_version` (default: 0)
+
+  CEF version, this should be 0
+
+* `parse_strict_mode` (default: true)
+
+  if the CEF extensions are the following, the value of the key cs2 should 'foo hoge=fuga'
+
+  - cs1=test cs2=foo hoge=fuga cs3=bar
+
+  if parse_strict_mode is false, this is raugh parse, so the value of the key cs2 become 'foo' and non CEF key 'hoge' shown, and the value is 'fuga'
+
+* `cef_keyfilename` (default: 'config/cef_version_0_keys.yaml')
+
+  used when parse_strict_mode is true, this is the array of the valid CEF keys
+
+* `output_raw_field` (default: false)
+
+  append {"raw":\<message itself\>} key-value even if success parsing
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+1.0.6

data/fluent-plugin-parser_cefalt.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-parser_cefalt"
+  spec.version       = File.read("VERSION").strip
+  spec.authors       = ["Tomoyuki Sugimura"]
+  spec.email         = ["tomoyuki.sugimura@gmail.com"]
+  spec.description   = %q{common event format(CEF) parser plugin for fluentd}
+  spec.summary       = %q{common event format(CEF) parser plugin, currently only 'syslog' format is permitted}
+  spec.homepage      = "https://github.com/lunardial/fluent-plugin-parser_cef"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = ">= 2.1"
+
+  spec.add_runtime_dependency "fluentd", ">= 0.14.0", "< 2"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-core"
+  spec.add_development_dependency "test-unit"
+end

data/lib/fluent/plugin/config/cef_version_0_keys.yaml

@@ -0,0 +1,221 @@
+# Implementing ArcSight Common Event Format
+# Chapter2: ArcSight Extension Dictionary
+
+# CEF Key Names For Event Producers
+cef_key_names_for_event_producers:
+  - act
+  - app
+  - c6a1
+  - c6a1Label
+  - c6a2
+  - c6a2Label
+  - c6a3
+  - c6a3Label
+  - c6a4
+  - c6a4Label
+  - cfp1
+  - cfp1Label
+  - cfp2
+  - cfp2Label
+  - cfp3
+  - cfp3Label
+  - cfp4
+  - cfp4Label
+  - cn1
+  - cn1Label
+  - cn2
+  - cn2Label
+  - cn3
+  - cn3Label
+  - cnt
+  - cs1
+  - cs1Label
+  - cs2
+  - cs2Label
+  - cs3
+  - cs3Label
+  - cs4
+  - cs4Label
+  - cs5
+  - cs5Label
+  - cs6
+  - cs6Label
+  - destinationDnsDomain
+  - destinationServiceName
+  - destinationTranslatedAddress
+  - destinationTranslatedPort
+  - deviceCustomDate1
+  - deviceCustomDate1Label
+  - deviceCustomDate2
+  - deviceCustomDate2Label
+  - deviceDirection
+  - deviceDnsDomain
+  - deviceExternalId
+  - deviceFacility
+  - deviceInboundInterface
+  - deviceNtDomain
+  - deviceOutboundInterface
+  - devicePayloadId
+  - deviceProcessName
+  - deviceTranslatedAddress
+  - dhost
+  - dmac
+  - dntdom
+  - dpid
+  - dpriv
+  - dproc
+  - dpt
+  - dst
+  - dtz
+  - duid
+  - duser
+  - dvc
+  - dvchost
+  - dvcmac
+  - dvcpid
+  - end
+  - externalId
+  - fileCreateTime
+  - fileHash
+  - fileId
+  - fileModificationTime
+  - filePath
+  - filePermission
+  - fileType
+  - flexDate1
+  - flexDate1Label
+  - flexNumber1
+  - flexNumber1Label
+  - flexNumber2
+  - flexNumber2Label
+  - flexString1
+  - flexString1Label
+  - flexString2
+  - flexString2Label
+  - fname
+  - fsize
+  - in
+  - msg
+  - oldFileCreateTime
+  - oldFileHash
+  - oldFileId
+  - oldFileModificationTime
+  - oldFileName
+  - oldFilePath
+  - oldFilePermission
+  - oldFileSize
+  - oldFileType
+  - out
+  - outcome
+  - proto
+  - reason
+  - request
+  - requestClientApplication
+  - requestContext
+  - requestCookies
+  - requestMethod
+  - rt
+  - shost
+  - smac
+  - sntdom
+  - sourceDnsDomain
+  - sourceServiceName
+  - sourceTranslatedAddress
+  - sourceTranslatedPort
+  - spid
+  - spriv
+  - sproc
+  - spt
+  - src
+  - start
+  - suid
+  - suser
+  - type
+  - slocation
+  - dlocation
+  - PanOSPacketsReceived
+  - PanOSPacketsSent
+  - PanOSThreatCategory
+  - PanOSThreatID
+  - PanOSThreatName
+  - PanOSThreatDomain
+  - PanOSSeverity
+  - PanOSReferer
+  - PanOSURLCatList
+  - PanOSLogTimeStamp
+  - PanOSVirtualSystem
+  - PanOSEventID
+  - PanOSStage
+  - PanOSAuthMethod
+  - PanOSTunnelType
+  - PanOSSourceUserName
+  - PanOSSourceRegion
+  - PanOSEndpointDeviceName
+  - PanOSPublicIPv4
+  - PanOSPublicIPv6
+  - PanOSPrivateIPv4
+  - PanOSPrivateIPv6
+  - PanOSHostID
+  - PanOSDeviceSN
+  - PanOSGlobalProtectClientVersion
+  - PanOSEndpointOSType
+  - PanOSEndpointOSVersion
+  - PanOSCountOfRepeats
+  - PanOSQuarantineReason
+  - PanOSConnectionError
+  - PanOSDescription
+  - PanOSEventStatus
+  - PanOSGPGatewayLocation
+  - PanOSLoginDuration
+  - PanOSConnectionMethod
+  - PanOSConnectionErrorID
+  - PanOSPortal
+  - PanOSSequenceNo
+  - PanOSActionFlags
+  - PanOSTimeGeneratedHighResolution
+  - PanOSGatewaySelectionType
+  - PanOSSSLResponseTime
+  - PanOSGatewayPriority
+  - PanOSAttemptedGateways
+  - PanOSGateway
+  - PanOSEndpointDeviceSN
+
+# CEF Key Names For Event Consumers
+cef_key_names_for_event_consumers:
+  - agentDnsDomain
+  - agentNtDomain
+  - agentTranslatedAddress
+  - agentTranslatedZoneExternalID
+  - agentTranslatedZoneURI
+  - agentZoneExternalID
+  - agentZoneURI
+  - agt
+  - ahost
+  - aid
+  - amac
+  - art
+  - at
+  - atz
+  - av
+  - cat
+  - customerExternalID
+  - customerURI
+  - destinationTranslatedZoneExternalID
+  - destinationTranslatedZoneURI
+  - destinationZoneExternalID
+  - destinationZoneURI
+  - deviceTranslatedZoneExternalID
+  - deviceTranslatedZoneURI
+  - deviceZoneExternalID
+  - deviceZoneURI
+  - dlat
+  - dlong
+  - eventId
+  - rawEvent
+  - slat
+  - slong
+  - sourceTranslatedZoneExternalID
+  - sourceTranslatedZoneURI
+  - sourceZoneExternalID
+  - sourceZoneURI
+

data/lib/fluent/plugin/parser_cefalt.rb

@@ -0,0 +1,222 @@
+# -*- coding: utf-8
+
+require 'fluent/log'
+require 'fluent/plugin/parser'
+require 'time'
+require 'yaml'
+
+module Fluent
+  module Plugin
+    class CommonEventFormatParser < Parser
+      Fluent::Plugin.register_parser("cefalt", self)
+
+      REGEXP_DETECT_RFC5424 = /^[1-9]\d{0,2}/
+
+      config_param :log_format, :string, :default => "syslog"
+      config_param :log_utc_offset, :string, :default => nil
+      config_param :syslog_timestamp_format, :string, :default => '\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}'
+      config_param :syslog_timestamp_format_5424, :string, :default => '\d{4}[-]\d{2}[-]\d{2}[T]\d{2}[:]\d{2}[:]\d{2}(?:\.\d{1,6})?(?:[+-]\d{2}[:]\d{2}|Z)'
+      config_param :cef_version, :integer, :default => 0
+      config_param :parse_strict_mode, :bool, :default => true
+      config_param :cef_keyfilename, :string, :default => 'config/cef_version_0_keys.yaml'
+      config_param :output_raw_field, :bool, :default => false
+
+      def configure(conf)
+        super
+        @key_value_format_regexp = /([^\s=]+)=(.*?)(?:(?=[^\s=]+=)|\z)/
+        @valid_format_regexp = create_valid_format_regexp
+        @valid_format_regexp_5424 = create_valid_format_regexp_5424
+        @utc_offset = get_utc_offset(@log_utc_offset)
+        begin
+          $log.trace(@valid_format_regexp)
+          $log.trace(@valid_format_regexp_5424)
+          if @parse_strict_mode
+            if @cef_keyfilename =~ /^\//
+              yaml_fieldinfo = YAML.load_file(@cef_keyfilename)
+            else
+              yaml_fieldinfo = YAML.load_file("#{File.dirname(File.expand_path(__FILE__))}/#{@cef_keyfilename}")
+            end
+            @keys_array = []
+            yaml_fieldinfo.each {|_key, value| @keys_array.concat(value) }
+            $log.info "running with strict mode, #{@keys_array.length} keys are valid."
+          else
+            $log.info "running without strict mode"
+          end
+        rescue => e
+          @parse_strict_mode = false
+          $log.warn "running without strict mode because of the following error"
+          $log.warn "#{e.message}"
+        end
+      end
+
+      def parse(text)
+        if text.nil? || text.empty?
+          yield nil, nil
+          return
+        end
+        log.trace(text)
+        text.force_encoding("utf-8")
+        replaced_text = text.scrub('?')
+        record = {}
+        if REGEXP_DETECT_RFC5424.match(text)
+          record_overview = @valid_format_regexp_5424.match(replaced_text)
+          log.trace "match 5424"
+        else
+          record_overview = @valid_format_regexp.match(replaced_text)
+          log.trace "match 3164"
+        end
+        if record_overview.nil?
+          yield Engine.now, { "raw" => replaced_text }
+          return
+        end
+        time = get_unixtime_with_utc_offset(record_overview["syslog_timestamp"], @utc_offset)
+        begin
+          record_overview.names.each {|key| record[key] = record_overview[key] }
+          text_cef_extension = record_overview["cef_extension"]
+          record.delete("cef_extension")
+        rescue
+          yield Engine.now, { "raw" => replaced_text }
+          return
+        end
+        unless text_cef_extension.nil?
+          record_cef_extension = parse_cef_extension(text_cef_extension)
+          record.merge!(record_cef_extension)
+        end
+        record["raw"] = replaced_text if @output_raw_field
+        yield time, record
+        return
+      end
+
+      private
+
+      def get_utc_offset(text)
+        utc_offset = nil
+        begin
+          utc_offset = Time.new.localtime(text).strftime("%:z")
+          $log.info "utc_offset: #{utc_offset}"
+        rescue => e
+          utc_offset = Time.new.localtime.strftime("%:z")
+          $log.info "#{e.message}, use localtime"
+          $log.info "utc_offset: #{utc_offset}"
+        end
+        return utc_offset
+      end
+
+      def create_valid_format_regexp()
+        case @log_format
+        when "syslog"
+          syslog_header = /
+              (?<syslog_timestamp>#{@syslog_timestamp_format})\s
+              (?<syslog_hostname>\S+)\s
+              (?<syslog_tag>\S*)\s*
+            /x
+          cef_header = /
+            CEF:(?<cef_version>#{@cef_version})\|
+            (?<cef_device_vendor>[^|]*)\|
+            (?<cef_device_product>[^|]*)\|
+            (?<cef_device_version>[^|]*)\|
+            (?<cef_device_event_class_id>[^|]*)\|
+            (?<cef_name>[^|]*)\|
+            (?<cef_severity>[^|]*)
+          /x
+          valid_format_regexp = /
+              \A
+                #{syslog_header}
+                (?:\u{feff})?
+                #{cef_header}\|
+                (?<cef_extension>.*)
+              \z
+            /x
+        else
+          raise Fluent::ConfigError, "#{@log_format} is unknown format"
+        end
+        return Regexp.new(valid_format_regexp)
+      end
+
+      def create_valid_format_regexp_5424()
+        case @log_format
+        when "syslog"
+          syslog_header = /
+            (?:[1-9])\s
+            (?<syslog_timestamp>#{@syslog_timestamp_format_5424})\s
+            (?<syslog_hostname>\S+)\s
+            (?<syslog_tag>\S+)\s
+            (?<pid>\S+)\s
+            (?<msgid>\S+)\s
+            (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])+))\s
+          /x
+          cef_header = /
+            CEF:(?<cef_version>#{@cef_version})\|
+            (?<cef_device_vendor>[^|]*)\|
+            (?<cef_device_product>[^|]*)\|
+            (?<cef_device_version>[^|]*)\|
+            (?<cef_device_event_class_id>[^|]*)\|
+            (?<cef_name>[^|]*)\|
+            (?<cef_severity>[^|]*)
+          /x
+          valid_format_regexp_5424 = /
+              \A
+                #{syslog_header}
+                #{cef_header}\|
+                (?<cef_extension>.*)
+              \z
+            /x
+        else
+          raise Fluent::ConfigError, "#{@log_format} is unknown format"
+        end
+        return Regexp.new(valid_format_regexp_5424)
+      end
+
+      def get_unixtime_with_utc_offset(timestamp, utc_offset)
+        unixtime = nil
+        begin
+          if timestamp =~ /[-+]\d{2}:?\d{2}\z/
+            unixtime = Time.parse(timestamp).to_i
+          else
+            unixtime = Time.parse("#{timestamp} #{utc_offset}").to_i
+          end
+        rescue
+          unixtime = Engine.now
+        end
+        return unixtime
+      end
+
+      def parse_cef_extension(text)
+        if @parse_strict_mode == true
+          return parse_cef_extension_with_strict_mode(text)
+        else
+          return parse_cef_extension_without_strict_mode(text)
+        end
+      end
+
+      def parse_cef_extension_with_strict_mode(text)
+        record = {}
+        begin
+          last_valid_key_name = nil
+          text.scan(@key_value_format_regexp) do |key, value|
+            if @keys_array.include?(key)
+              record[key] = value
+              record[last_valid_key_name].rstrip! unless last_valid_key_name.nil?
+              last_valid_key_name = key
+            else
+              record[last_valid_key_name].concat("#{key}=#{value}")
+            end
+          end
+        rescue
+          return {}
+        end
+        return record
+      end
+
+      def parse_cef_extension_without_strict_mode(text)
+        record = {}
+        begin
+          text.scan(@key_value_format_regexp) {|key, value| record[key] = value.rstrip }
+        rescue
+          return {}
+        end
+        return record
+      end
+    end
+  end
+end