fluent-plugin-parser-winevt_xml 0.1.1 → 0.2.3.rc1

data/README.md

@@ -1,34 +1,56 @@
-# fluent-plugin-parser-winevt_xml
-
-[![Build status](https://ci.appveyor.com/api/projects/status/eb0capv0q70u381f/branch/master?svg=true)](https://ci.appveyor.com/project/fluent/fluent-plugin-parser-winevt-xml/branch/master)
-[![Build Status](https://travis-ci.org/fluent/fluent-plugin-parser-winevt_xml.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-parser-winevt_xml)
-
-## Component
-
-### Fluentd Parser plugin for XML rendered Windows EventLogs
-
-[Fluentd](https://www.fluentd.org/) plugin to parse XML rendered Windows Event Logs.
-
-### Installation
-
-```
-gem install fluent-plugin-parser-winevt_xml
-```
-
-## Configuration
-
-```aconf
-<parse>
-  @type winevt_xml
-</parse>
-```
-
-## Copyright
-
-### Copyright
-
-Copyright(C) 2019- Hiroshi Hatake, Masahiro Nakagawa
-
-### License
-
-Apache License, Version 2.0
+# fluent-plugin-parser-winevt_xml
+
+[![Build status](https://ci.appveyor.com/api/projects/status/eb0capv0q70u381f/branch/master?svg=true)](https://ci.appveyor.com/project/fluent/fluent-plugin-parser-winevt-xml/branch/master)
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-parser-winevt_xml.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-parser-winevt_xml)
+
+## Component
+
+### Fluentd Parser plugin for XML rendered Windows EventLogs
+
+[Fluentd](https://www.fluentd.org/) plugin to parse XML rendered Windows Event Logs.
+
+### Installation
+
+```
+gem install fluent-plugin-parser-winevt_xml
+```
+
+## Configuration
+
+### parser_winevt_xml
+
+```aconf
+<parse>
+  @type winevt_xml
+  preserve_qualifiers true
+</parse>
+```
+
+#### preserve_qualifiers
+
+Preserve Qualifiers key instead of calculating actual EventID with Qualifiers. Default is `true`.
+
+### parser_winevt_sax
+
+This plugin is a bit faster than `winevt_xml`.
+
+```aconf
+<parse>
+  @type winevt_sax
+  preserve_qualifiers true
+</parse>
+```
+
+#### preserve_qualifiers
+
+Preserve Qualifiers key instead of calculating actual EventID with Qualifiers. Default is `true`.
+
+## Copyright
+
+### Copyright
+
+Copyright(C) 2019- Hiroshi Hatake, Masahiro Nakagawa
+
+### License
+
+Apache License, Version 2.0

data/Rakefile

@@ -1,10 +1,10 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-task default: :test
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task default: :test

data/appveyor.yml

@@ -1,24 +1,24 @@
-version: '{build}'
-
-# init:
-#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
-
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - ruby --version
-  - gem --version
-  - ridk.cmd exec bundle install
-build: off
-test_script:
-  - bundle exec rake test
-#  - bundle exec rake test TESTOPTS=-v
-
-branches:
-  only:
-    - master
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "24-x64"
-    - ruby_version: "24"
+version: '{build}'
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - ruby --version
+  - gem --version
+  - ridk.cmd exec bundle install
+build: off
+test_script:
+  - bundle exec rake test
+#  - bundle exec rake test TESTOPTS=-v
+
+branches:
+  only:
+    - master
+
+# https://www.appveyor.com/docs/installed-software/#ruby
+environment:
+  matrix:
+    - ruby_version: "24-x64"
+    - ruby_version: "24"

data/fluent-plugin-parser-winevt_xml.gemspec

@@ -1,25 +1,25 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-Gem::Specification.new do |spec|
-  spec.name          = "fluent-plugin-parser-winevt_xml"
-  spec.version       = "0.1.1"
-  spec.authors       = ["Hiroshi Hatake", "Masahiro Nakagawa"]
-  spec.email         = ["cosmo0920.oucc@gmail.com", "repeatedly@gmail.com"]
-  spec.summary       = %q{Fluentd Parser plugin to parse XML rendered windows event log.}
-  spec.description   = %q{Fluentd Parser plugin to parse XML rendered windows event log.}
-  spec.homepage      = "https://github.com/fluent/fluent-plugin-parser-winevt_xml"
-  spec.license       = "Apache-2.0"
-
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "test-unit", "~> 3.2.0"
-  spec.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
-  spec.add_runtime_dependency "nokogiri", "~> 1.10"
-end
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-parser-winevt_xml"
+  spec.version       = "0.2.3.rc1"
+  spec.authors       = ["Hiroshi Hatake", "Masahiro Nakagawa"]
+  spec.email         = ["cosmo0920.oucc@gmail.com", "repeatedly@gmail.com"]
+  spec.summary       = %q{Fluentd Parser plugin to parse XML rendered windows event log.}
+  spec.description   = %q{Fluentd Parser plugin to parse XML rendered windows event log.}
+  spec.homepage      = "https://github.com/fluent/fluent-plugin-parser-winevt_xml"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit", "~> 3.2.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
+  spec.add_runtime_dependency "nokogiri", [">= 1.11.pre", "< 1.12"]
+end

data/lib/fluent/plugin/parser_winevt_sax.rb

@@ -0,0 +1,27 @@
+require 'fluent/plugin/parser'
+require 'fluent/plugin/winevt_sax_document'
+require 'nokogiri'
+
+module Fluent::Plugin
+  class WinevtSAXparser < Parser
+    Fluent::Plugin.register_parser('winevt_sax', self)
+
+    config_param :preserve_qualifiers, :bool, default: true
+
+    def winevt_xml?
+      true
+    end
+
+    def preserve_qualifiers?
+      @preserve_qualifiers
+    end
+
+    def parse(text)
+      evtxml = WinevtXMLDocument.new(@preserve_qualifiers)
+      parser = Nokogiri::XML::SAX::Parser.new(evtxml)
+      parser.parse(text)
+      time = @estimate_current_event ? Fluent::EventTime.now : nil
+      yield time, evtxml.result
+    end
+  end
+end

data/lib/fluent/plugin/parser_winevt_xml.rb

@@ -1,34 +1,63 @@
-require 'fluent/plugin/parser'
-require 'nokogiri'
-
-module Fluent::Plugin
-  class WinevtXMLparser < Parser
-    Fluent::Plugin.register_parser('winevt_xml', self)
-
-    def parse(text)
-      record = {}
-      doc = Nokogiri::XML(text)
-      system_elem                     = doc/'Event'/'System'
-      record["ProviderName"]          = (system_elem/"Provider").attribute("Name").text rescue nil
-      record["ProviderGUID"]          = (system_elem/"Provider").attribute("Guid").text rescue nil
-      record["EventID"]               = (system_elem/'EventID').text rescue nil
-      record["Qualifiers"]            = (system_elem/'EventID').attribute("Qualifiers").text rescue nil
-      record["Level"]                 = (system_elem/'Level').text rescue nil
-      record["Task"]                  = (system_elem/'Task').text rescue nil
-      record["Opcode"]                = (system_elem/'Opcode').text rescue nil
-      record["Keywords"]              = (system_elem/'Keywords').text rescue nil
-      record["TimeCreated"]           = (system_elem/'TimeCreated').attribute("SystemTime").text rescue nil
-      record["EventRecordID"]         = (system_elem/'EventRecordID').text rescue nil
-      record["ActivityID"]            = (system_elem/'ActivityID').text rescue nil
-      record["RelatedActivityID"]     = (system_elem/'Correlation').attribute("ActivityID").text rescue nil
-      record["ThreadID"]              = (system_elem/'Execution').attribute("ThreadID").text rescue nil
-      record["Channel"]               = (system_elem/'Channel').text rescue nil
-      record["Computer"]              = (system_elem/"Computer").text rescue nil
-      record["UserID"]                = (system_elem/'Security').attribute("UserID").text rescue nil
-      record["Version"]               = (system_elem/'Version').text rescue nil
-      record["InsertStrings"]         = [] # These parameters are processed in winevt_c.
-      time = @estimate_current_event ? Fluent::EventTime.now : nil
-      yield time, record
-    end
-  end
-end
+require 'fluent/plugin/parser'
+require 'nokogiri'
+
+module Fluent::Plugin
+  class WinevtXMLparser < Parser
+    Fluent::Plugin.register_parser('winevt_xml', self)
+
+    config_param :preserve_qualifiers, :bool, default: true
+
+    def winevt_xml?
+      true
+    end
+
+    def preserve_qualifiers?
+      @preserve_qualifiers
+    end
+
+    def MAKELONG(low, high)
+      (low & 0xffff) | (high & 0xffff) << 16
+    end
+
+    def event_id(system_elem)
+      return (system_elem/'EventID').text rescue nil if @preserve_qualifiers
+
+      qualifiers = (system_elem/'EventID').attribute("Qualifiers").text rescue nil
+      if qualifiers
+        event_id = (system_elem/'EventID').text
+        event_id = MAKELONG(event_id.to_i, qualifiers.to_i)
+        event_id.to_s
+      else
+        (system_elem/'EventID').text rescue nil
+      end
+    end
+
+    def parse(text)
+      record = {}
+      doc = Nokogiri::XML(text)
+      system_elem                     = doc/'Event'/'System'
+      record["ProviderName"]          = (system_elem/"Provider").attribute("Name").text rescue nil
+      record["ProviderGUID"]          = (system_elem/"Provider").attribute("Guid").text rescue nil
+      if @preserve_qualifiers
+        record["Qualifiers"]            = (system_elem/'EventID').attribute("Qualifiers").text rescue nil
+      end
+      record["EventID"]               = event_id(system_elem)
+      record["Level"]                 = (system_elem/'Level').text rescue nil
+      record["Task"]                  = (system_elem/'Task').text rescue nil
+      record["Opcode"]                = (system_elem/'Opcode').text rescue nil
+      record["Keywords"]              = (system_elem/'Keywords').text rescue nil
+      record["TimeCreated"]           = (system_elem/'TimeCreated').attribute("SystemTime").text rescue nil
+      record["EventRecordID"]         = (system_elem/'EventRecordID').text rescue nil
+      record["ActivityID"]            = (system_elem/'Correlation').attribute('ActivityID').text rescue nil
+      record["RelatedActivityID"]     = (system_elem/'Correlation').attribute("RelatedActivityID").text rescue nil
+      record["ThreadID"]              = (system_elem/'Execution').attribute("ThreadID").text rescue nil
+      record["ProcessID"]             = (system_elem/'Execution').attribute("ProcessID").text rescue nil
+      record["Channel"]               = (system_elem/'Channel').text rescue nil
+      record["Computer"]              = (system_elem/"Computer").text rescue nil
+      record["UserID"]                = (system_elem/'Security').attribute("UserID").text rescue nil
+      record["Version"]               = (system_elem/'Version').text rescue nil
+      time = @estimate_current_event ? Fluent::EventTime.now : nil
+      yield time, record
+    end
+  end
+end

data/lib/fluent/plugin/winevt_sax_document.rb

@@ -0,0 +1,73 @@
+require 'nokogiri'
+
+class WinevtXMLDocument < Nokogiri::XML::SAX::Document
+  def initialize(preserve_qualifiers)
+    @stack = []
+    @result = {}
+    @preserve_qualifiers = preserve_qualifiers
+    super()
+  end
+
+  def MAKELONG(low, high)
+    (low & 0xffff) | (high & 0xffff) << 16
+  end
+
+  def event_id
+    if @result.has_key?("Qualifiers")
+      qualifiers = @result.delete("Qualifiers")
+      event_id = @result['EventID']
+      event_id = MAKELONG(event_id.to_i, qualifiers.to_i)
+      @result['EventID'] = event_id.to_s
+    else
+      @result['EventID']
+    end
+  end
+
+  def result
+    return @result if @preserve_qualifiers
+
+    if @result
+      @result['EventID'] = event_id
+    end
+    @result
+  end
+
+  def start_document
+  end
+
+  def start_element(name, attributes = [])
+    @stack << name
+
+    if name == "Provider"
+      @result["ProviderName"] = attributes[0][1] rescue nil
+      @result["ProviderGUID"] = attributes[1][1] rescue nil
+    elsif name == "EventID"
+      @result["Qualifiers"] = attributes[0][1] rescue nil
+    elsif name == "TimeCreated"
+      @result["TimeCreated"] = attributes[0][1] rescue nil
+    elsif name == "Correlation"
+      @result["ActivityID"] = attributes[0][1] rescue nil
+      @result["RelatedActivityID"] = attributes[1][1] rescue nil
+    elsif name == "Execution"
+      @result["ProcessID"] = attributes[0][1] rescue nil
+      @result["ThreadID"] = attributes[1][1] rescue nil
+    elsif name == "Security"
+      @result["UserID"] = attributes[0][1] rescue nil
+    end
+  end
+
+  def characters(string)
+    element = @stack.last
+
+    if /^EventID|Level|Task|Opcode|Keywords|EventRecordID|
+        ActivityID|Channel|Computer|Security|Version$/ === element
+      @result[element] = string
+    end
+  end
+
+  def end_element(name, attributes = [])
+  end
+
+  def end_document
+  end
+end

data/test/data/eventlog-with-qualifiers.xml

@@ -0,0 +1 @@
+<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-SPP' Guid='{E23B33B0-C8C9-472C-A5F9-F2BDFEA0F156}' EventSourceName='Software Protection Platform Service'/><EventID Qualifiers='49152'>16394</EventID><Version>0</Version><Level>4</Level><Task>0</Task><Opcode>0</Opcode><Keywords>0x80000000000000</Keywords><TimeCreated SystemTime='2020-01-16T09:57:18.013693700Z'/><EventRecordID>150731</EventRecordID><Correlation/><Execution ProcessID='0' ThreadID='0'/><Channel>Application</Channel><Computer>DESKTOP-G457RDR</Computer><Security/></System><EventData></EventData></Event>

data/test/data/eventlog.xml

@@ -1 +1 @@
-<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-A5BA-3E3B0328C30D}'/><EventID>4624</EventID><Version>2</Version><Level>0</Level><Task>12544</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2019-06-13T09:21:23.345889600Z'/><EventRecordID>80688</EventRecordID><Correlation ActivityID='{587F0743-1F71-0006-5007-7F58711FD501}'/><Execution ProcessID='912' ThreadID='24708'/><Channel>Security</Channel><Computer>Fluentd-Developing-Windows</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>Fluentd-Developing-Windows$</Data><Data Name='SubjectDomainName'>WORKGROUP</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='TargetUserSid'>S-1-5-18</Data><Data Name='TargetUserName'>SYSTEM</Data><Data Name='TargetDomainName'>NT AUTHORITY</Data><Data Name='TargetLogonId'>0x3e7</Data><Data Name='LogonType'>5</Data><Data Name='LogonProcessName'>Advapi  </Data><Data Name='AuthenticationPackageName'>Negotiate</Data><Data Name='WorkstationName'>-</Data><Data Name='LogonGuid'>{00000000-0000-0000-0000-000000000000}</Data><Data Name='TransmittedServices'>-</Data><Data Name='LmPackageName'>-</Data><Data Name='KeyLength'>0</Data><Data Name='ProcessId'>0x344</Data><Data Name='ProcessName'>C:\Windows\System32\services.exe</Data><Data Name='IpAddress'>-</Data><Data Name='IpPort'>-</Data><Data Name='ImpersonationLevel'>%%1833</Data><Data Name='RestrictedAdminMode'>-</Data><Data Name='TargetOutboundUserName'>-</Data><Data Name='TargetOutboundDomainName'>-</Data><Data Name='VirtualAccount'>%%1843</Data><Data Name='TargetLinkedLogonId'>0x0</Data><Data Name='ElevatedToken'>%%1842</Data></EventData></Event>
+<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-A5BA-3E3B0328C30D}'/><EventID>4624</EventID><Version>2</Version><Level>0</Level><Task>12544</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2019-06-13T09:21:23.345889600Z'/><EventRecordID>80688</EventRecordID><Correlation ActivityID='{587F0743-1F71-0006-5007-7F58711FD501}'/><Execution ProcessID='912' ThreadID='24708'/><Channel>Security</Channel><Computer>Fluentd-Developing-Windows</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>Fluentd-Developing-Windows$</Data><Data Name='SubjectDomainName'>WORKGROUP</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='TargetUserSid'>S-1-5-18</Data><Data Name='TargetUserName'>SYSTEM</Data><Data Name='TargetDomainName'>NT AUTHORITY</Data><Data Name='TargetLogonId'>0x3e7</Data><Data Name='LogonType'>5</Data><Data Name='LogonProcessName'>Advapi  </Data><Data Name='AuthenticationPackageName'>Negotiate</Data><Data Name='WorkstationName'>-</Data><Data Name='LogonGuid'>{00000000-0000-0000-0000-000000000000}</Data><Data Name='TransmittedServices'>-</Data><Data Name='LmPackageName'>-</Data><Data Name='KeyLength'>0</Data><Data Name='ProcessId'>0x344</Data><Data Name='ProcessName'>C:\Windows\System32\services.exe</Data><Data Name='IpAddress'>-</Data><Data Name='IpPort'>-</Data><Data Name='ImpersonationLevel'>%%1833</Data><Data Name='RestrictedAdminMode'>-</Data><Data Name='TargetOutboundUserName'>-</Data><Data Name='TargetOutboundDomainName'>-</Data><Data Name='VirtualAccount'>%%1843</Data><Data Name='TargetLinkedLogonId'>0x0</Data><Data Name='ElevatedToken'>%%1842</Data></EventData></Event>

data/test/helper.rb

@@ -1,23 +1,24 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'fluent/test'
-
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser_winevt_xml'
-
-class Test::Unit::TestCase
-end
-require 'fluent/test/helpers'
-
-include Fluent::Test::Helpers
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_winevt_xml'
+require 'fluent/plugin/parser_winevt_sax'
+
+class Test::Unit::TestCase
+end
+require 'fluent/test/helpers'
+
+include Fluent::Test::Helpers