fluent-plugin-windows-eventlog 0.2.2 → 0.3.0

data/lib/fluent/plugin/in_windows_eventlog2.rb

@@ -0,0 +1,169 @@
+require 'winevt'
+require 'fluent/plugin/input'
+require 'fluent/plugin'
+
+module Fluent::Plugin
+  class WindowsEventLog2Input < Input
+    Fluent::Plugin.register_input('windows_eventlog2', self)
+
+    helpers :timer, :storage, :parser
+
+    DEFAULT_STORAGE_TYPE = 'local'
+    KEY_MAP = {"ProviderName"      => ["ProviderName",          :string],
+               "ProviderGUID"      => ["ProviderGUID",          :string],
+               "EventID"           => ["EventID",               :string],
+               "Qualifiers"        => ["Qualifiers",            :string],
+               "Level"             => ["Level",                 :string],
+               "Task"              => ["Task",                  :string],
+               "Opcode"            => ["Opcode",                :string],
+               "Keywords"          => ["Keywords",              :string],
+               "TimeCreated"       => ["TimeCreated",           :string],
+               "EventRecordID"     => ["EventRecordID",         :string],
+               "ActivityID"        => ["ActivityID",            :string],
+               "RelatedActivityID" => ["RelatedActivityID",     :string],
+               "ThreadID"          => ["ThreadID",              :string],
+               "Channel"           => ["Channel",               :string],
+               "Computer"          => ["Computer",              :string],
+               "UserID"            => ["UserID",                :string],
+               "Version"           => ["Version",               :string],
+               "Description"       => ["Description",           :string],
+               "EventData"         => ["EventData",             :array]}
+
+    config_param :tag, :string
+    config_param :read_interval, :time, default: 2
+    config_param :channels, :array, default: ['application']
+    config_param :keys, :array, default: []
+    config_param :read_from_head, :bool, default: false
+    config_param :parse_description, :bool, default: false
+
+    config_section :storage do
+      config_set_default :usage, "bookmarks"
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, true
+    end
+
+    config_section :parse do
+      config_set_default :@type, 'winevt_xml'
+      config_set_default :estimate_current_event, false
+    end
+
+    def initalize
+      super
+      @chs = []
+      @keynames = []
+    end
+
+    def configure(conf)
+      super
+      @chs = @channels.map {|ch| ch.strip.downcase }.uniq
+      @keynames = @keys.map {|k| k.strip }.uniq
+      if @keynames.empty?
+        @keynames = KEY_MAP.keys
+      end
+      @keynames.delete('EventData') if @parse_description
+
+      @tag = tag
+      @tailing = @read_from_head ? false : true
+      @bookmarks_storage = storage_create(usage: "bookmarks")
+      @parser = parser_create
+    end
+
+    def start
+      super
+
+      @chs.each do |ch|
+        bookmarkXml = @bookmarks_storage.get(ch) || ""
+        subscribe = Winevt::EventLog::Subscribe.new
+        bookmark = Winevt::EventLog::Bookmark.new(bookmarkXml)
+        subscribe.tail = @tailing
+        subscribe.subscribe(ch, "*", bookmark)
+        timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
+          on_notify(ch, subscribe)
+        end
+      end
+    end
+
+    def escape_channel(ch)
+      ch.gsub(/[^a-zA-Z0-9]/, '_')
+    end
+
+    def on_notify(ch, subscribe)
+      es = Fluent::MultiEventStream.new
+      subscribe.each do |xml, message, string_inserts|
+        @parser.parse(xml) do |time, record|
+          # record.has_key?("EventData") for none parser checking.
+          if record.has_key?("EventData")
+            record["Description"] = message
+            record["EventData"] = string_inserts
+
+            h = {}
+            @keynames.each do |k|
+              type = KEY_MAP[k][1]
+              value = record[KEY_MAP[k][0]]
+              h[k]=case type
+                   when :string
+                     value.to_s
+                   when :array
+                     value.map {|v| v.to_s}
+                   else
+                     raise "Unknown value type: #{type}"
+                   end
+            end
+            parse_desc(h) if @parse_description
+            es.add(Fluent::Engine.now, h)
+          else
+            # for none parser
+            es.add(Fluent::Engine.now, record)
+          end
+        end
+      end
+      router.emit_stream(@tag, es)
+      @bookmarks_storage.put(ch, subscribe.bookmark)
+    end
+
+    #### These lines copied from in_windows_eventlog plugin:
+    #### https://github.com/fluent/fluent-plugin-windows-eventlog/blob/528290d896a885c7721f850943daa3a43a015f3d/lib/fluent/plugin/in_windows_eventlog.rb#L192-L232
+    GROUP_DELIMITER = "\r\n\r\n".freeze
+    RECORD_DELIMITER = "\r\n\t".freeze
+    FIELD_DELIMITER = "\t\t".freeze
+    NONE_FIELD_DELIMITER = "\t".freeze
+
+    def parse_desc(record)
+      desc = record.delete("Description".freeze)
+      return if desc.nil?
+
+      elems = desc.split(GROUP_DELIMITER)
+      record['DescriptionTitle'] = elems.shift
+      elems.each { |elem|
+        parent_key = nil
+        elem.split(RECORD_DELIMITER).each { |r|
+          key, value = if r.index(FIELD_DELIMITER)
+                         r.split(FIELD_DELIMITER)
+                       else
+                         r.split(NONE_FIELD_DELIMITER)
+                       end
+          key.chop!  # remove ':' from key
+          if value.nil?
+            parent_key = to_key(key)
+          else
+            # parsed value sometimes contain unexpected "\t". So remove it.
+            value.strip!
+            if parent_key.nil?
+              record[to_key(key)] = value
+            else
+              k = "#{parent_key}.#{to_key(key)}"
+              record[k] = value
+            end
+          end
+        }
+      }
+    end
+
+    def to_key(key)
+      key.downcase!
+      key.gsub!(' '.freeze, '_'.freeze)
+      key
+    end
+    ####
+  end
+end

data/lib/fluent/plugin/parser_winevt_xml.rb

@@ -0,0 +1,34 @@
+require 'fluent/plugin/parser'
+require 'nokogiri'
+
+module Fluent::Plugin
+  class WinevtXMLparser < Parser
+    Fluent::Plugin.register_parser('winevt_xml', self)
+
+    def parse(text)
+      record = {}
+      doc = Nokogiri::XML(text)
+      system_elem                     = doc/'Event'/'System'
+      record["ProviderName"]          = (system_elem/"Provider").attribute("Name").text rescue nil
+      record["ProviderGUID"]          = (system_elem/"Provider").attribute("Guid").text rescue nil
+      record["EventID"]               = (system_elem/'EventID').text rescue nil
+      record["Qualifiers"]            = (system_elem/'EventID').attribute("Qualifiers").text rescue nil
+      record["Level"]                 = (system_elem/'Level').text rescue nil
+      record["Task"]                  = (system_elem/'Task').text rescue nil
+      record["Opcode"]                = (system_elem/'Opcode').text rescue nil
+      record["Keywords"]              = (system_elem/'Keywords').text rescue nil
+      record["TimeCreated"]           = (system_elem/'TimeCreated').attribute("SystemTime").text rescue nil
+      record["EventRecordID"]         = (system_elem/'EventRecordID').text rescue nil
+      record["ActivityID"]            = (system_elem/'ActivityID').text rescue nil
+      record["RelatedActivityID"]     = (system_elem/'Correlation').attribute("ActivityID").text rescue nil
+      record["ThreadID"]              = (system_elem/'Execution').attribute("ThreadID").text rescue nil
+      record["Channel"]               = (system_elem/'Channel').text rescue nil
+      record["Computer"]              = (system_elem/"Computer").text rescue nil
+      record["UserID"]                = (system_elem/'Security').attribute("UserID").text rescue nil
+      record["Version"]               = (system_elem/'Version').text rescue nil
+      record["EventData"]             = [] # These parameters are processed in winevt_c.
+      time = @estimate_current_event ? Fluent::EventTime.now : nil
+      yield time, record
+    end
+  end
+end

data/test/data/eventlog.xml

@@ -0,0 +1 @@
+<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-A5BA-3E3B0328C30D}'/><EventID>4624</EventID><Version>2</Version><Level>0</Level><Task>12544</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2019-06-13T09:21:23.345889600Z'/><EventRecordID>80688</EventRecordID><Correlation ActivityID='{587F0743-1F71-0006-5007-7F58711FD501}'/><Execution ProcessID='912' ThreadID='24708'/><Channel>Security</Channel><Computer>Fluentd-Developing-Windows</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>Fluentd-Developing-Windows$</Data><Data Name='SubjectDomainName'>WORKGROUP</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='TargetUserSid'>S-1-5-18</Data><Data Name='TargetUserName'>SYSTEM</Data><Data Name='TargetDomainName'>NT AUTHORITY</Data><Data Name='TargetLogonId'>0x3e7</Data><Data Name='LogonType'>5</Data><Data Name='LogonProcessName'>Advapi  </Data><Data Name='AuthenticationPackageName'>Negotiate</Data><Data Name='WorkstationName'>-</Data><Data Name='LogonGuid'>{00000000-0000-0000-0000-000000000000}</Data><Data Name='TransmittedServices'>-</Data><Data Name='LmPackageName'>-</Data><Data Name='KeyLength'>0</Data><Data Name='ProcessId'>0x344</Data><Data Name='ProcessName'>C:\Windows\System32\services.exe</Data><Data Name='IpAddress'>-</Data><Data Name='IpPort'>-</Data><Data Name='ImpersonationLevel'>%%1833</Data><Data Name='RestrictedAdminMode'>-</Data><Data Name='TargetOutboundUserName'>-</Data><Data Name='TargetOutboundDomainName'>-</Data><Data Name='VirtualAccount'>%%1843</Data><Data Name='TargetLinkedLogonId'>0x0</Data><Data Name='ElevatedToken'>%%1842</Data></EventData></Event>

data/test/generate-windows-event.rb

@@ -1,47 +1,47 @@
-require 'win32/eventlog'
-
-class EventLog
-  def initialize
-    @logger = Win32::EventLog.new
-    @app_source = "fluent-plugins"
-  end
-
-  def info(event_id, message)
-    @logger.report_event(
-      source: @app_source,
-      event_type: Win32::EventLog::INFO_TYPE,
-      event_id: event_id,
-      data: message
-    )
-  end
-
-  def warn(event_id, message)
-    @logger.report_event(
-      source: @app_source,
-      event_type: Win32::EventLog::WARN_TYPE,
-      event_id: event_id,
-      data: message
-    )
-  end
-
-  def crit(event_id, message)
-    @logger.report_event(
-      source: @app_source,
-      event_type: Win32::EventLog::ERROR_TYPE,
-      event_id: event_id,
-      data: message
-    )
-  end
-
-end
-
-module Fluent
-  module Plugin
-    class EventService
-      def run
-        eventlog = EventLog.new()
-        eventlog.info(65500, "Hi, from fluentd-plugins!! at " + Time.now.strftime("%Y/%m/%d %H:%M:%S "))
-      end
-    end
-  end
-end
+require 'win32/eventlog'
+
+class EventLog
+  def initialize
+    @logger = Win32::EventLog.new
+    @app_source = "fluent-plugins"
+  end
+
+  def info(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::INFO_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+  def warn(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::WARN_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+  def crit(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::ERROR_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+end
+
+module Fluent
+  module Plugin
+    class EventService
+      def run
+        eventlog = EventLog.new()
+        eventlog.info(65500, "Hi, from fluentd-plugins!! at " + Time.now.strftime("%Y/%m/%d %H:%M:%S "))
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -1,32 +1,35 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
-require 'fluent/test/driver/input'
-require 'fluent/plugin/in_windows_eventlog'
-
-class Test::Unit::TestCase
-end
-require 'fluent/test/helpers'
-
-include Fluent::Test::Helpers
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/in_windows_eventlog'
+require 'fluent/plugin/in_windows_eventlog2'
+require 'fluent/plugin/parser_winevt_xml'
+
+class Test::Unit::TestCase
+end
+require 'fluent/test/helpers'
+
+include Fluent::Test::Helpers

data/test/plugin/test_in_windows_eventlog2.rb

@@ -0,0 +1,182 @@
+require 'helper'
+require 'fileutils'
+require 'generate-windows-event'
+
+class WindowsEventLog2InputTest < Test::Unit::TestCase
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                            config_element("storage", "", {
+                                             '@type' => 'local',
+                                             'persistent' => false
+                                           })
+                          ])
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::WindowsEventLog2Input).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver CONFIG
+    assert_equal 'fluent.eventlog', d.instance.tag
+    assert_equal 2, d.instance.read_interval
+    assert_equal ['application'], d.instance.channels
+    assert_false d.instance.read_from_head
+  end
+
+  def test_parse_desc
+    d = create_driver
+    desc =<<-DESC
+A user's local group membership was enumerated.\r\n\r\nSubject:\r\n\tSecurity ID:\t\tS-X-Y-XX-WWWWWW-VVVV\r\n\tAccount Name:\t\tAdministrator\r\n\tAccount Domain:\t\tDESKTOP-FLUENTTEST\r\n\tLogon ID:\t\t0x3185B1\r\n\r\nUser:\r\n\tSecurity ID:\t\tS-X-Y-XX-WWWWWW-VVVV\r\n\tAccount Name:\t\tAdministrator\r\n\tAccount Domain:\t\tDESKTOP-FLUENTTEST\r\n\r\nProcess Information:\r\n\tProcess ID:\t\t0x50b8\r\n\tProcess Name:\t\tC:\\msys64\\usr\\bin\\make.exe
+DESC
+    h = {"Description" => desc}
+    expected = {"DescriptionTitle"                 => "A user's local group membership was enumerated.",
+                "subject.security_id"              => "S-X-Y-XX-WWWWWW-VVVV",
+                "subject.account_name"             => "Administrator",
+                "subject.account_domain"           => "DESKTOP-FLUENTTEST",
+                "subject.logon_id"                 => "0x3185B1",
+                "user.security_id"                 => "S-X-Y-XX-WWWWWW-VVVV",
+                "user.account_name"                => "Administrator",
+                "user.account_domain"              => "DESKTOP-FLUENTTEST",
+                "process_information.process_id"   => "0x50b8",
+                "process_information.process_name" => "C:\\msys64\\usr\\bin\\make.exe"}
+    d.instance.parse_desc(h)
+    assert_equal(expected, h)
+  end
+
+  def test_write
+    d = create_driver
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.last
+    record = event.last
+
+    assert_equal("Application", record["Channel"])
+    assert_equal("65500", record["EventID"])
+    assert_equal("4", record["Level"])
+    assert_equal("fluent-plugins", record["ProviderName"])
+  end
+
+  CONFIG_KEYS = config_element("ROOT", "", {
+                                 "tag" => "fluent.eventlog",
+                                 "keys" => ["EventID", "Level", "Channel", "ProviderName"]
+                               }, [
+                                 config_element("storage", "", {
+                                                  '@type' => 'local',
+                                                  'persistent' => false
+                                                })
+                               ])
+  def test_write_with_keys
+    d = create_driver(CONFIG_KEYS)
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.last
+    record = event.last
+
+    expected = {"EventID"      => "65500",
+                "Level"        => "4",
+                "Channel"      => "Application",
+                "ProviderName" => "fluent-plugins"}
+
+    assert_equal(expected, record)
+  end
+
+  class PersistBookMark < self
+    TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_windows_eventlog2', 'store' )
+    CONFIG2 = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                               config_element("storage", "", {
+                                                '@type' => 'local',
+                                                '@id' => 'test-02',
+                                                'path' => File.join(TEST_PLUGIN_STORAGE_PATH,
+                                                                    'json', 'test-02.json'),
+                                                'persistent' => true,
+                                              })
+                             ])
+
+    def setup
+      FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
+      FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+      FileUtils.chmod_R(0755, File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+    end
+
+    def test_write
+      d = create_driver(CONFIG2)
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      service = Fluent::Plugin::EventService.new
+
+      d.run(expect_emits: 1) do
+        service.run
+      end
+
+      assert(d.events.length >= 1)
+      event = d.events.last
+      record = event.last
+
+      prev_id = record["EventRecordID"].to_i
+      assert_equal("Application", record["Channel"])
+      assert_equal("65500", record["EventID"])
+      assert_equal("4", record["Level"])
+      assert_equal("fluent-plugins", record["ProviderName"])
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d2 = create_driver(CONFIG2)
+      d2.run(expect_emits: 1) do
+        service.run
+      end
+
+      assert(d2.events.length == 1) # should be tailing after previous context.
+      event2 = d2.events.last
+      record2 = event2.last
+
+      curr_id = record2["EventRecordID"].to_i
+      assert(curr_id > prev_id)
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+    end
+  end
+
+  def test_write_with_none_parser
+    d = create_driver(config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                                       config_element("storage", "", {
+                                                        '@type' => 'local',
+                                                        'persistent' => false
+                                                      }),
+                                       config_element("parse", "", {
+                                                        '@type' => 'none',
+                                                      }),
+                                     ]))
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.last
+    record = event.last
+
+    assert do
+      # record should be {message: <RAW XML EventLog>}.
+      record["message"]
+    end
+  end
+end