fluent-plugin-windows-eventlog 0.2.2 → 0.4.3

data/lib/fluent/plugin/in_windows_eventlog2.rb

@@ -0,0 +1,242 @@
+require 'winevt'
+require 'fluent/plugin/input'
+require 'fluent/plugin'
+
+module Fluent::Plugin
+  class WindowsEventLog2Input < Input
+    Fluent::Plugin.register_input('windows_eventlog2', self)
+
+    helpers :timer, :storage, :parser
+
+    DEFAULT_STORAGE_TYPE = 'local'
+    KEY_MAP = {"ProviderName"      => ["ProviderName",          :string],
+               "ProviderGUID"      => ["ProviderGUID",          :string],
+               "EventID"           => ["EventID",               :string],
+               "Qualifiers"        => ["Qualifiers",            :string],
+               "Level"             => ["Level",                 :string],
+               "Task"              => ["Task",                  :string],
+               "Opcode"            => ["Opcode",                :string],
+               "Keywords"          => ["Keywords",              :string],
+               "TimeCreated"       => ["TimeCreated",           :string],
+               "EventRecordID"     => ["EventRecordID",         :string],
+               "ActivityID"        => ["ActivityID",            :string],
+               "RelatedActivityID" => ["RelatedActivityID",     :string],
+               "ProcessID"         => ["ProcessID",             :string],
+               "ThreadID"          => ["ThreadID",              :string],
+               "Channel"           => ["Channel",               :string],
+               "Computer"          => ["Computer",              :string],
+               "UserID"            => ["UserID",                :string],
+               "Version"           => ["Version",               :string],
+               "Description"       => ["Description",           :string],
+               "EventData"         => ["EventData",             :array]}
+
+    config_param :tag, :string
+    config_param :read_interval, :time, default: 2
+    config_param :channels, :array, default: ['application']
+    config_param :keys, :array, default: []
+    config_param :read_from_head, :bool, default: false
+    config_param :parse_description, :bool, default: false
+    config_param :render_as_xml, :bool, default: true
+    config_param :rate_limit, :integer, default: Winevt::EventLog::Subscribe::RATE_INFINITE
+
+    config_section :storage do
+      config_set_default :usage, "bookmarks"
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, true
+    end
+
+    config_section :parse do
+      config_set_default :@type, 'winevt_xml'
+      config_set_default :estimate_current_event, false
+    end
+
+    def initalize
+      super
+      @chs = []
+      @keynames = []
+    end
+
+    def configure(conf)
+      super
+      @chs = @channels.map {|ch| ch.strip.downcase }.uniq
+      @keynames = @keys.map {|k| k.strip }.uniq
+      if @keynames.empty?
+        @keynames = KEY_MAP.keys
+      end
+      @keynames.delete('Qualifiers') unless @render_as_xml
+      @keynames.delete('EventData') if @parse_description
+
+      @tag = tag
+      @tailing = @read_from_head ? false : true
+      @bookmarks_storage = storage_create(usage: "bookmarks")
+      @winevt_xml = false
+      if @render_as_xml
+        @parser = parser_create
+        @winevt_xml = @parser.respond_to?(:winevt_xml?) && @parser.winevt_xml?
+        class << self
+          alias_method :on_notify, :on_notify_xml
+        end
+      else
+        class << self
+          alias_method :on_notify, :on_notify_hash
+        end
+      end
+    end
+
+    def start
+      super
+
+      @chs.each do |ch|
+        bookmarkXml = @bookmarks_storage.get(ch) || ""
+        subscribe = Winevt::EventLog::Subscribe.new
+        bookmark = unless bookmarkXml.empty?
+                     Winevt::EventLog::Bookmark.new(bookmarkXml)
+                   else
+                     nil
+                   end
+        subscribe.tail = @tailing
+        begin
+          subscribe.subscribe(ch, "*", bookmark)
+        rescue Winevt::EventLog::Query::Error => e
+          raise Fluent::ConfigError, "Invalid Bookmark XML is loaded. #{e}"
+        end
+        subscribe.render_as_xml = @render_as_xml
+        subscribe.rate_limit = @rate_limit
+        timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
+          on_notify(ch, subscribe)
+        end
+      end
+    end
+
+    def escape_channel(ch)
+      ch.gsub(/[^a-zA-Z0-9]/, '_')
+    end
+
+    def on_notify(ch, subscribe)
+      # for safety.
+    end
+
+    def on_notify_xml(ch, subscribe)
+      es = Fluent::MultiEventStream.new
+      begin
+        subscribe.each do |xml, message, string_inserts|
+          @parser.parse(xml) do |time, record|
+            # record.has_key?("EventData") for none parser checking.
+            if @winevt_xml
+              record["Description"] = message
+              record["EventData"] = string_inserts
+
+              h = {}
+              @keynames.each do |k|
+                type = KEY_MAP[k][1]
+                value = record[KEY_MAP[k][0]]
+                h[k]=case type
+                     when :string
+                       value.to_s
+                     when :array
+                       value.map {|v| v.to_s}
+                     else
+                       raise "Unknown value type: #{type}"
+                     end
+              end
+              parse_desc(h) if @parse_description
+              es.add(Fluent::Engine.now, h)
+            else
+              record["Description"] = message
+              record["EventData"] = string_inserts
+              # for none parser
+              es.add(Fluent::Engine.now, record)
+            end
+          end
+        end
+      rescue Winevt::EventLog::Query::Error => e
+        log.warn "Invalid XML data", error: e
+        log.warn_backtrace
+      end
+      router.emit_stream(@tag, es)
+      @bookmarks_storage.put(ch, subscribe.bookmark)
+    end
+
+    def on_notify_hash(ch, subscribe)
+      es = Fluent::MultiEventStream.new
+      begin
+        subscribe.each do |record, message, string_inserts|
+          record["Description"] = message
+          record["EventData"] = string_inserts
+          h = {}
+          @keynames.each do |k|
+            type = KEY_MAP[k][1]
+            value = record[KEY_MAP[k][0]]
+            h[k]=case type
+                 when :string
+                   value.to_s
+                 when :array
+                   value.map {|v| v.to_s}
+                 else
+                   raise "Unknown value type: #{type}"
+                 end
+          end
+          parse_desc(h) if @parse_description
+          es.add(Fluent::Engine.now, h)
+        end
+      rescue Winevt::EventLog::Query::Error => e
+        log.warn "Invalid Hash data", error: e
+        log.warn_backtrace
+      end
+      router.emit_stream(@tag, es)
+      @bookmarks_storage.put(ch, subscribe.bookmark)
+    end
+
+    #### These lines copied from in_windows_eventlog plugin:
+    #### https://github.com/fluent/fluent-plugin-windows-eventlog/blob/528290d896a885c7721f850943daa3a43a015f3d/lib/fluent/plugin/in_windows_eventlog.rb#L192-L232
+    GROUP_DELIMITER = "\r\n\r\n".freeze
+    RECORD_DELIMITER = "\r\n\t".freeze
+    FIELD_DELIMITER = "\t\t".freeze
+    NONE_FIELD_DELIMITER = "\t".freeze
+
+    def parse_desc(record)
+      desc = record.delete("Description".freeze)
+      return if desc.nil?
+
+      elems = desc.split(GROUP_DELIMITER)
+      record['DescriptionTitle'] = elems.shift
+      previous_key = nil
+      elems.each { |elem|
+        parent_key = nil
+        elem.split(RECORD_DELIMITER).each { |r|
+          key, value = if r.index(FIELD_DELIMITER)
+                         r.split(FIELD_DELIMITER)
+                       else
+                         r.split(NONE_FIELD_DELIMITER)
+                       end
+          key.chop!  # remove ':' from key
+          if value.nil?
+            parent_key = to_key(key)
+          else
+            # parsed value sometimes contain unexpected "\t". So remove it.
+            value.strip!
+            # merge empty key values into the previous non-empty key record.
+            if key.empty?
+              record[previous_key] = [record[previous_key], value].flatten
+            elsif parent_key.nil?
+              record[to_key(key)] = value
+            else
+              k = "#{parent_key}.#{to_key(key)}"
+              record[k] = value
+            end
+          end
+          # XXX: This is for empty privileges record key.
+          # We should investigate whether an another case exists or not.
+          previous_key = to_key(key) unless key.empty?
+        }
+      }
+    end
+
+    def to_key(key)
+      key.downcase!
+      key.gsub!(' '.freeze, '_'.freeze)
+      key
+    end
+    ####
+  end
+end

data/test/generate-windows-event.rb

@@ -1,47 +1,47 @@
-require 'win32/eventlog'
-
-class EventLog
-  def initialize
-    @logger = Win32::EventLog.new
-    @app_source = "fluent-plugins"
-  end
-
-  def info(event_id, message)
-    @logger.report_event(
-      source: @app_source,
-      event_type: Win32::EventLog::INFO_TYPE,
-      event_id: event_id,
-      data: message
-    )
-  end
-
-  def warn(event_id, message)
-    @logger.report_event(
-      source: @app_source,
-      event_type: Win32::EventLog::WARN_TYPE,
-      event_id: event_id,
-      data: message
-    )
-  end
-
-  def crit(event_id, message)
-    @logger.report_event(
-      source: @app_source,
-      event_type: Win32::EventLog::ERROR_TYPE,
-      event_id: event_id,
-      data: message
-    )
-  end
-
-end
-
-module Fluent
-  module Plugin
-    class EventService
-      def run
-        eventlog = EventLog.new()
-        eventlog.info(65500, "Hi, from fluentd-plugins!! at " + Time.now.strftime("%Y/%m/%d %H:%M:%S "))
-      end
-    end
-  end
-end
+require 'win32/eventlog'
+
+class EventLog
+  def initialize
+    @logger = Win32::EventLog.new
+    @app_source = "fluent-plugins"
+  end
+
+  def info(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::INFO_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+  def warn(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::WARN_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+  def crit(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::ERROR_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+end
+
+module Fluent
+  module Plugin
+    class EventService
+      def run
+        eventlog = EventLog.new()
+        eventlog.info(65500, "Hi, from fluentd-plugins!! at " + Time.now.strftime("%Y/%m/%d %H:%M:%S "))
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -1,32 +1,33 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
-require 'fluent/test/driver/input'
-require 'fluent/plugin/in_windows_eventlog'
-
-class Test::Unit::TestCase
-end
-require 'fluent/test/helpers'
-
-include Fluent::Test::Helpers
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_windows_eventlog'
+require 'fluent/plugin/in_windows_eventlog2'
+
+class Test::Unit::TestCase
+end
+require 'fluent/test/helpers'
+
+include Fluent::Test::Helpers

data/test/plugin/test_in_windows_eventlog2.rb

@@ -0,0 +1,261 @@
+require 'helper'
+require 'fileutils'
+require 'generate-windows-event'
+
+class WindowsEventLog2InputTest < Test::Unit::TestCase
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                            config_element("storage", "", {
+                                             '@type' => 'local',
+                                             'persistent' => false
+                                           })
+                          ])
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::WindowsEventLog2Input).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver CONFIG
+    assert_equal 'fluent.eventlog', d.instance.tag
+    assert_equal 2, d.instance.read_interval
+    assert_equal ['application'], d.instance.channels
+    assert_false d.instance.read_from_head
+    assert_true d.instance.render_as_xml
+  end
+
+  def test_parse_desc
+    d = create_driver
+    desc =<<-DESC
+A user's local group membership was enumerated.\r\n\r\nSubject:\r\n\tSecurity ID:\t\tS-X-Y-XX-WWWWWW-VVVV\r\n\tAccount Name:\t\tAdministrator\r\n\tAccount Domain:\t\tDESKTOP-FLUENTTEST\r\n\tLogon ID:\t\t0x3185B1\r\n\r\nUser:\r\n\tSecurity ID:\t\tS-X-Y-XX-WWWWWW-VVVV\r\n\tAccount Name:\t\tAdministrator\r\n\tAccount Domain:\t\tDESKTOP-FLUENTTEST\r\n\r\nProcess Information:\r\n\tProcess ID:\t\t0x50b8\r\n\tProcess Name:\t\tC:\\msys64\\usr\\bin\\make.exe
+DESC
+    h = {"Description" => desc}
+    expected = {"DescriptionTitle"                 => "A user's local group membership was enumerated.",
+                "subject.security_id"              => "S-X-Y-XX-WWWWWW-VVVV",
+                "subject.account_name"             => "Administrator",
+                "subject.account_domain"           => "DESKTOP-FLUENTTEST",
+                "subject.logon_id"                 => "0x3185B1",
+                "user.security_id"                 => "S-X-Y-XX-WWWWWW-VVVV",
+                "user.account_name"                => "Administrator",
+                "user.account_domain"              => "DESKTOP-FLUENTTEST",
+                "process_information.process_id"   => "0x50b8",
+                "process_information.process_name" => "C:\\msys64\\usr\\bin\\make.exe"}
+    d.instance.parse_desc(h)
+    assert_equal(expected, h)
+  end
+
+  def test_parse_privileges_description
+    d = create_driver
+    desc = ["Special privileges assigned to new logon.\r\n\r\nSubject:\r\n\tSecurity ID:\t\tS-X-Y-ZZ\r\n\t",
+            "AccountName:\t\tSYSTEM\r\n\tAccount Domain:\t\tNT AUTHORITY\r\n\tLogon ID:\t\t0x3E7\r\n\r\n",
+            "Privileges:\t\tSeAssignPrimaryTokenPrivilege\r\n\t\t\tSeTcbPrivilege\r\n\t\t\t",
+            "SeSecurityPrivilege\r\n\t\t\tSeTakeOwnershipPrivilege\r\n\t\t\tSeLoadDriverPrivilege\r\n\t\t\t",
+            "SeBackupPrivilege\r\n\t\t\tSeRestorePrivilege\r\n\t\t\tSeDebugPrivilege\r\n\t\t\t",
+            "SeAuditPrivilege\r\n\t\t\tSeSystemEnvironmentPrivilege\r\n\t\t\tSeImpersonatePrivilege\r\n\t\t\t",
+            "SeDelegateSessionUserImpersonatePrivilege"].join("")
+
+    h = {"Description" => desc}
+    expected = {"DescriptionTitle"       => "Special privileges assigned to new logon.",
+                "subject.security_id"    => "S-X-Y-ZZ",
+                "subject.accountname"    => "SYSTEM",
+                "subject.account_domain" => "NT AUTHORITY",
+                "subject.logon_id"       => "0x3E7",
+                "privileges"             => ["SeAssignPrimaryTokenPrivilege",
+                                             "SeTcbPrivilege",
+                                             "SeSecurityPrivilege",
+                                             "SeTakeOwnershipPrivilege",
+                                             "SeLoadDriverPrivilege",
+                                             "SeBackupPrivilege",
+                                             "SeRestorePrivilege",
+                                             "SeDebugPrivilege",
+                                             "SeAuditPrivilege",
+                                             "SeSystemEnvironmentPrivilege",
+                                             "SeImpersonatePrivilege",
+                                             "SeDelegateSessionUserImpersonatePrivilege"]}
+    d.instance.parse_desc(h)
+    assert_equal(expected, h)
+  end
+
+  def test_write
+    d = create_driver
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.select {|e| e.last["EventID"] == "65500" }.last
+    record = event.last
+
+    assert_equal("Application", record["Channel"])
+    assert_equal("65500", record["EventID"])
+    assert_equal("4", record["Level"])
+    assert_equal("fluent-plugins", record["ProviderName"])
+  end
+
+  CONFIG_KEYS = config_element("ROOT", "", {
+                                 "tag" => "fluent.eventlog",
+                                 "keys" => ["EventID", "Level", "Channel", "ProviderName"]
+                               }, [
+                                 config_element("storage", "", {
+                                                  '@type' => 'local',
+                                                  'persistent' => false
+                                                })
+                               ])
+  def test_write_with_keys
+    d = create_driver(CONFIG_KEYS)
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.last
+    record = event.last
+
+    expected = {"EventID"      => "65500",
+                "Level"        => "4",
+                "Channel"      => "Application",
+                "ProviderName" => "fluent-plugins"}
+
+    assert_equal(expected, record)
+  end
+
+  class HashRendered < self
+    def test_write
+      d = create_driver(config_element("ROOT", "", {"tag" => "fluent.eventlog",
+                                                    "render_as_xml" => false}, [
+                           config_element("storage", "", {
+                                            '@type' => 'local',
+                                            'persistent' => false
+                                          })
+                           ]))
+
+      service = Fluent::Plugin::EventService.new
+
+      d.run(expect_emits: 1) do
+        service.run
+      end
+
+      assert(d.events.length >= 1)
+      event = d.events.select {|e| e.last["EventID"] == "65500" }.last
+      record = event.last
+
+      assert_false(d.instance.render_as_xml)
+      assert_equal("Application", record["Channel"])
+      assert_equal("65500", record["EventID"])
+      assert_equal("4", record["Level"])
+      assert_equal("fluent-plugins", record["ProviderName"])
+    end
+  end
+
+  class PersistBookMark < self
+    TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_windows_eventlog2', 'store' )
+    CONFIG2 = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                               config_element("storage", "", {
+                                                '@type' => 'local',
+                                                '@id' => 'test-02',
+                                                'path' => File.join(TEST_PLUGIN_STORAGE_PATH,
+                                                                    'json', 'test-02.json'),
+                                                'persistent' => true,
+                                              })
+                             ])
+
+    def setup
+      FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
+      FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+      FileUtils.chmod_R(0755, File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+    end
+
+    def test_write
+      d = create_driver(CONFIG2)
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      service = Fluent::Plugin::EventService.new
+
+      d.run(expect_emits: 1) do
+        service.run
+      end
+
+      assert(d.events.length >= 1)
+      event = d.events.select {|e| e.last["EventID"] == "65500" }.last
+      record = event.last
+
+      prev_id = record["EventRecordID"].to_i
+      assert_equal("Application", record["Channel"])
+      assert_equal("65500", record["EventID"])
+      assert_equal("4", record["Level"])
+      assert_equal("fluent-plugins", record["ProviderName"])
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d2 = create_driver(CONFIG2)
+      d2.run(expect_emits: 1) do
+        service.run
+      end
+
+      assert(d2.events.length == 1) # should be tailing after previous context.
+      event2 = d2.events.last
+      record2 = event2.last
+
+      curr_id = record2["EventRecordID"].to_i
+      assert(curr_id > prev_id)
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+    end
+
+    def test_start_with_invalid_bookmark
+      invalid_storage_contents = <<-EOS
+<BookmarkList>\r\n  <Bookmark Channel='Application' RecordId='20063' IsCurrent='true'/>\r\n
+EOS
+      d = create_driver(CONFIG2)
+      storage = d.instance.instance_variable_get(:@bookmarks_storage)
+      storage.put('application', invalid_storage_contents)
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d2 = create_driver(CONFIG2)
+      assert_raise(Fluent::ConfigError) do
+        d2.instance.start
+      end
+    end
+  end
+
+  def test_write_with_none_parser
+    d = create_driver(config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                                       config_element("storage", "", {
+                                                        '@type' => 'local',
+                                                        'persistent' => false
+                                                      }),
+                                       config_element("parse", "", {
+                                                        '@type' => 'none',
+                                                      }),
+                                     ]))
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.last
+    record = event.last
+
+    assert do
+      # record should be {message: <RAW XML EventLog>}.
+      record["message"]
+    end
+
+    assert_true(record.has_key?("Description"))
+    assert_true(record.has_key?("EventData"))
+  end
+end