fluent-plugin-windows-eventlog 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c05a4b1a785316299232e1cd4c3a5f60e7458b93b5b4d717720a6de5f079cbb
-  data.tar.gz: aad0c8d51e5b88d96f86704bb72394d6a16253edecd8b975ed6c3334f433ebf4
+  metadata.gz: 5ea9a134414a5572bf7d3c9dca355ea53d47d6a8701b6a631f0426bafa0de5c9
+  data.tar.gz: 26d9fa562585f26fdbef294aa6f64a13e7749ed11915390d1822c5367656ccb7
 SHA512:
-  metadata.gz: 185857a8f3114029de23b8a0b9b262de929e20b6ad98f783a937bb4eb48e8d594a84df33405ff7a5b348c43f0f81d601bdfea72288f4e30637e98035f92ed834
-  data.tar.gz: 1bb7650803e3852a2a14c701f9e3aeeeab932438b36702719f246c8e6b2be4b6c3ffa67a567e2c2dae1fc0cff7dd82144fcc874acb7045ea9beb5c819f6f28eb
+  metadata.gz: e292b037dc58e418aa8b74b1fa8b19db22124c927876e4fbf2e59b457042e08944e912e37104891fdd93f841b306ea8d09d2b1bbf939dad2015f18e26c021a11
+  data.tar.gz: 6d82ed5588b14f92bca8bdf42a8c9a085306ed23717e9b0f0fc6cdc994b376a00195469c51dcb9ddeb5a6a786fc7c5b96e24279037bc20bfa96f42477407851d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# Release v0.4.0 - 2019/10/10
+
+* in_windows_eventlog2: Add new `render_as_xml` parameter to switch rendering as XML or Ruby Hash object
+* in_windows_eventlog2: Support rate limit with `rate_limit` option
+* parser_winevt_xml: Separate `parser_winevt_xml` plugin to other repository and published as Fluentd parser plugin
+
 # Release v0.3.0 - 2019/07/08
 
 * Add new `in_windows_eventlog2` plugin. This plugin uses newer windows event logging API.

data/README.md

@@ -4,10 +4,10 @@
 
 ### fluentd Input plugin for the Windows Event Log
 
-[Fluentd](http://fluentd.org) plugin to read the Windows Event Log.
+[Fluentd](https://www.fluentd.org/) plugin to read the Windows Event Log.
 
 ## Installation
-    gem install fluent-plugin-windows-eventlog
+    ridk exec gem install fluent-plugin-windows-eventlog
 
 ## Configuration
 
@@ -15,7 +15,7 @@
 
 Check [in_windows_eventlog2](https://github.com/fluent/fluent-plugin-windows-eventlog#in_windows_eventlog2) first. `in_windows_eventlog` will be replaced with `in_windows_eventlog2`.
 
-#### fluentd Input plugin for the Windows Event Log using old Windows Event Logging API
+fluentd Input plugin for the Windows Event Log using old Windows Event Logging API
 
     <source>
       @type windows_eventlog
@@ -130,7 +130,7 @@ If your `description` doesn't follow this format, the parsed result is only `des
 
 ### in_windows_eventlog2
 
-#### fluentd Input plugin for the Windows Event Log using newer Windows Event Logging API
+fluentd Input plugin for the Windows Event Log using newer Windows Event Logging API. This is successor to `in_windows_eventlog`. See also [this slide](https://www.slideshare.net/cosmo0920/fluentd-meetup-2019) for the details of `in_windows_eventlog2` plugin.
 
     <source>
       @type windows_eventlog2
@@ -138,6 +138,8 @@ If your `description` doesn't follow this format, the parsed result is only `des
       channels application,system
       read_interval 2
       tag winevt.raw
+      render_as_xml false       # default is true.
+      rate_limit 200            # default is -1(Winevt::EventLog::Subscribe::RATE_INFINITE).
       <storage>
         @type local             # @type local is the default.
         persistent true         # default is true. Set to false to use in-memory storage.
@@ -153,6 +155,10 @@ If your `description` doesn't follow this format, the parsed result is only `des
 
 **NOTE:** When `Description` contains error message such as `The message resource is present but the message was not found in the message table.`, eventlog's resource file (.mui) related to error generating event is something wrong. This issue is also occurred in built-in Windows Event Viewer which is the part of Windows management tool.
 
+**NOTE:** When `render_as_xml` as `false`, the dependent winevt_c gem renders Windows EventLog as Ruby Hash object directly. This reduces bottleneck to consume EventLog. Specifying `render_as_xml` as `false` should be faster consuming than `render_as_xml` as `true` case.
+
+**NOTE:** If you encountered CPU spike due to massively huge EventLog channel, `rate_limit` parameter may help you. Currently, this paramter can handle the multiples of 10 or -1(`Winevt::EventLog::Subscribe::RATE_INFINITE`).
+
 #### parameters
 
 |name      | description |

data/fluent-plugin-winevtlog.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-windows-eventlog"
-  spec.version       = "0.3.0"
+  spec.version       = "0.4.0"
   spec.authors       = ["okahashi117", "Hiroshi Hatake", "Masahiro Nakagawa"]
   spec.email         = ["naruki_okahashi@jbat.co.jp", "cosmo0920.oucc@gmail.com", "repeatedly@gmail.com"]
   spec.summary       = %q{Fluentd Input plugin to read windows event log.}
@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit", "~> 3.2.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
   spec.add_runtime_dependency "win32-eventlog"
-  spec.add_runtime_dependency "winevt_c"
+  spec.add_runtime_dependency "winevt_c", ">= 0.6.0"
   spec.add_runtime_dependency "nokogiri", "~> 1.10"
+  spec.add_runtime_dependency "fluent-plugin-parser-winevt_xml", ">= 0.1.2"
 end

data/lib/fluent/plugin/in_windows_eventlog2.rb

@@ -35,6 +35,8 @@ module Fluent::Plugin
     config_param :keys, :array, default: []
     config_param :read_from_head, :bool, default: false
     config_param :parse_description, :bool, default: false
+    config_param :render_as_xml, :bool, default: true
+    config_param :rate_limit, :integer, default: Winevt::EventLog::Subscribe::RATE_INFINITE
 
     config_section :storage do
       config_set_default :usage, "bookmarks"
@@ -60,12 +62,24 @@ module Fluent::Plugin
       if @keynames.empty?
         @keynames = KEY_MAP.keys
       end
+      @keynames.delete('Qualifiers') unless @render_as_xml
       @keynames.delete('EventData') if @parse_description
 
       @tag = tag
       @tailing = @read_from_head ? false : true
       @bookmarks_storage = storage_create(usage: "bookmarks")
-      @parser = parser_create
+      @winevt_xml = false
+      if @render_as_xml
+        @parser = parser_create
+        @winevt_xml = @parser.respond_to?(:winevt_xml?) && @parser.winevt_xml?
+        class << self
+          alias_method :on_notify, :on_notify_xml
+        end
+      else
+        class << self
+          alias_method :on_notify, :on_notify_hash
+        end
+      end
     end
 
     def start
@@ -77,6 +91,8 @@ module Fluent::Plugin
         bookmark = Winevt::EventLog::Bookmark.new(bookmarkXml)
         subscribe.tail = @tailing
         subscribe.subscribe(ch, "*", bookmark)
+        subscribe.render_as_xml = @render_as_xml
+        subscribe.rate_limit = @rate_limit
         timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
           on_notify(ch, subscribe)
         end
@@ -88,11 +104,15 @@ module Fluent::Plugin
     end
 
     def on_notify(ch, subscribe)
+      # for safety.
+    end
+
+    def on_notify_xml(ch, subscribe)
       es = Fluent::MultiEventStream.new
       subscribe.each do |xml, message, string_inserts|
         @parser.parse(xml) do |time, record|
           # record.has_key?("EventData") for none parser checking.
-          if record.has_key?("EventData")
+          if @winevt_xml
             record["Description"] = message
             record["EventData"] = string_inserts
 
@@ -112,6 +132,8 @@ module Fluent::Plugin
             parse_desc(h) if @parse_description
             es.add(Fluent::Engine.now, h)
           else
+            record["Description"] = message
+            record["EventData"] = string_inserts
             # for none parser
             es.add(Fluent::Engine.now, record)
           end
@@ -121,6 +143,31 @@ module Fluent::Plugin
       @bookmarks_storage.put(ch, subscribe.bookmark)
     end
 
+    def on_notify_hash(ch, subscribe)
+      es = Fluent::MultiEventStream.new
+      subscribe.each do |record, message, string_inserts|
+        record["Description"] = message
+        record["EventData"] = string_inserts
+        h = {}
+        @keynames.each do |k|
+          type = KEY_MAP[k][1]
+          value = record[KEY_MAP[k][0]]
+          h[k]=case type
+               when :string
+                 value.to_s
+               when :array
+                 value.map {|v| v.to_s}
+               else
+                 raise "Unknown value type: #{type}"
+               end
+        end
+        parse_desc(h) if @parse_description
+        es.add(Fluent::Engine.now, h)
+      end
+      router.emit_stream(@tag, es)
+      @bookmarks_storage.put(ch, subscribe.bookmark)
+    end
+
     #### These lines copied from in_windows_eventlog plugin:
     #### https://github.com/fluent/fluent-plugin-windows-eventlog/blob/528290d896a885c7721f850943daa3a43a015f3d/lib/fluent/plugin/in_windows_eventlog.rb#L192-L232
     GROUP_DELIMITER = "\r\n\r\n".freeze

data/test/helper.rb

@@ -23,10 +23,8 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/test/driver/input'
-require 'fluent/test/driver/parser'
 require 'fluent/plugin/in_windows_eventlog'
 require 'fluent/plugin/in_windows_eventlog2'
-require 'fluent/plugin/parser_winevt_xml'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_windows_eventlog2.rb

@@ -25,6 +25,7 @@ class WindowsEventLog2InputTest < Test::Unit::TestCase
     assert_equal 2, d.instance.read_interval
     assert_equal ['application'], d.instance.channels
     assert_false d.instance.read_from_head
+    assert_true d.instance.render_as_xml
   end
 
   def test_parse_desc
@@ -96,6 +97,34 @@ DESC
     assert_equal(expected, record)
   end
 
+  class HashRendered < self
+    def test_write
+      d = create_driver(config_element("ROOT", "", {"tag" => "fluent.eventlog",
+                                                    "render_as_xml" => false}, [
+                           config_element("storage", "", {
+                                            '@type' => 'local',
+                                            'persistent' => false
+                                          })
+                           ]))
+
+      service = Fluent::Plugin::EventService.new
+
+      d.run(expect_emits: 1) do
+        service.run
+      end
+
+      assert(d.events.length >= 1)
+      event = d.events.last
+      record = event.last
+
+      assert_false(d.instance.render_as_xml)
+      assert_equal("Application", record["Channel"])
+      assert_equal("65500", record["EventID"])
+      assert_equal("4", record["Level"])
+      assert_equal("fluent-plugins", record["ProviderName"])
+    end
+  end
+
   class PersistBookMark < self
     TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_windows_eventlog2', 'store' )
     CONFIG2 = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
@@ -178,5 +207,8 @@ DESC
       # record should be {message: <RAW XML EventLog>}.
       record["message"]
     end
+
+    assert_true(record.has_key?("Description"))
+    assert_true(record.has_key?("EventData"))
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-windows-eventlog
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - okahashi117
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-09 00:00:00.000000000 Z
+date: 2019-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,14 +94,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -116,6 +116,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-parser-winevt_xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
 description: Fluentd Input plugin to read windows event log.
 email:
 - naruki_okahashi@jbat.co.jp
@@ -135,13 +149,10 @@ files:
 - fluent-plugin-winevtlog.gemspec
 - lib/fluent/plugin/in_windows_eventlog.rb
 - lib/fluent/plugin/in_windows_eventlog2.rb
-- lib/fluent/plugin/parser_winevt_xml.rb
-- test/data/eventlog.xml
 - test/generate-windows-event.rb
 - test/helper.rb
 - test/plugin/test_in_windows_eventlog2.rb
 - test/plugin/test_in_winevtlog.rb
-- test/plugin/test_parser_winevt_xml.rb
 homepage: https://github.com/fluent/fluent-plugin-windows-eventlog
 licenses:
 - Apache-2.0
@@ -167,9 +178,7 @@ signing_key:
 specification_version: 4
 summary: Fluentd Input plugin to read windows event log.
 test_files:
-- test/data/eventlog.xml
 - test/generate-windows-event.rb
 - test/helper.rb
 - test/plugin/test_in_windows_eventlog2.rb
 - test/plugin/test_in_winevtlog.rb
-- test/plugin/test_parser_winevt_xml.rb

data/lib/fluent/plugin/parser_winevt_xml.rb

@@ -1,34 +0,0 @@
-require 'fluent/plugin/parser'
-require 'nokogiri'
-
-module Fluent::Plugin
-  class WinevtXMLparser < Parser
-    Fluent::Plugin.register_parser('winevt_xml', self)
-
-    def parse(text)
-      record = {}
-      doc = Nokogiri::XML(text)
-      system_elem                     = doc/'Event'/'System'
-      record["ProviderName"]          = (system_elem/"Provider").attribute("Name").text rescue nil
-      record["ProviderGUID"]          = (system_elem/"Provider").attribute("Guid").text rescue nil
-      record["EventID"]               = (system_elem/'EventID').text rescue nil
-      record["Qualifiers"]            = (system_elem/'EventID').attribute("Qualifiers").text rescue nil
-      record["Level"]                 = (system_elem/'Level').text rescue nil
-      record["Task"]                  = (system_elem/'Task').text rescue nil
-      record["Opcode"]                = (system_elem/'Opcode').text rescue nil
-      record["Keywords"]              = (system_elem/'Keywords').text rescue nil
-      record["TimeCreated"]           = (system_elem/'TimeCreated').attribute("SystemTime").text rescue nil
-      record["EventRecordID"]         = (system_elem/'EventRecordID').text rescue nil
-      record["ActivityID"]            = (system_elem/'ActivityID').text rescue nil
-      record["RelatedActivityID"]     = (system_elem/'Correlation').attribute("ActivityID").text rescue nil
-      record["ThreadID"]              = (system_elem/'Execution').attribute("ThreadID").text rescue nil
-      record["Channel"]               = (system_elem/'Channel').text rescue nil
-      record["Computer"]              = (system_elem/"Computer").text rescue nil
-      record["UserID"]                = (system_elem/'Security').attribute("UserID").text rescue nil
-      record["Version"]               = (system_elem/'Version').text rescue nil
-      record["EventData"]             = [] # These parameters are processed in winevt_c.
-      time = @estimate_current_event ? Fluent::EventTime.now : nil
-      yield time, record
-    end
-  end
-end

data/test/data/eventlog.xml

@@ -1 +0,0 @@
-<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-A5BA-3E3B0328C30D}'/><EventID>4624</EventID><Version>2</Version><Level>0</Level><Task>12544</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2019-06-13T09:21:23.345889600Z'/><EventRecordID>80688</EventRecordID><Correlation ActivityID='{587F0743-1F71-0006-5007-7F58711FD501}'/><Execution ProcessID='912' ThreadID='24708'/><Channel>Security</Channel><Computer>Fluentd-Developing-Windows</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>Fluentd-Developing-Windows$</Data><Data Name='SubjectDomainName'>WORKGROUP</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='TargetUserSid'>S-1-5-18</Data><Data Name='TargetUserName'>SYSTEM</Data><Data Name='TargetDomainName'>NT AUTHORITY</Data><Data Name='TargetLogonId'>0x3e7</Data><Data Name='LogonType'>5</Data><Data Name='LogonProcessName'>Advapi  </Data><Data Name='AuthenticationPackageName'>Negotiate</Data><Data Name='WorkstationName'>-</Data><Data Name='LogonGuid'>{00000000-0000-0000-0000-000000000000}</Data><Data Name='TransmittedServices'>-</Data><Data Name='LmPackageName'>-</Data><Data Name='KeyLength'>0</Data><Data Name='ProcessId'>0x344</Data><Data Name='ProcessName'>C:\Windows\System32\services.exe</Data><Data Name='IpAddress'>-</Data><Data Name='IpPort'>-</Data><Data Name='ImpersonationLevel'>%%1833</Data><Data Name='RestrictedAdminMode'>-</Data><Data Name='TargetOutboundUserName'>-</Data><Data Name='TargetOutboundDomainName'>-</Data><Data Name='VirtualAccount'>%%1843</Data><Data Name='TargetLinkedLogonId'>0x0</Data><Data Name='ElevatedToken'>%%1842</Data></EventData></Event>

data/test/plugin/test_parser_winevt_xml.rb

@@ -1,42 +0,0 @@
-require 'helper'
-require 'generate-windows-event'
-
-class WinevtXMLparserTest < Test::Unit::TestCase
-
-  def setup
-    Fluent::Test.setup
-  end
-
-  CONFIG = %[]
-  XMLLOG = File.open(File.join(__dir__, "..", "data", "eventlog.xml") )
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::WinevtXMLparser).configure(conf)
-  end
-
-  def test_parse
-    d = create_driver
-    xml = XMLLOG
-    expected = {"ProviderName"      => "Microsoft-Windows-Security-Auditing",
-                "ProviderGUID"      => "{54849625-5478-4994-A5BA-3E3B0328C30D}",
-                "EventID"           => "4624",
-                "Qualifiers"        => nil,
-                "Level"             => "0",
-                "Task"              => "12544",
-                "Opcode"            => "0",
-                "Keywords"          => "0x8020000000000000",
-                "TimeCreated"       => "2019-06-13T09:21:23.345889600Z",
-                "EventRecordID"     => "80688",
-                "ActivityID"        => "",
-                "RelatedActivityID" => "{587F0743-1F71-0006-5007-7F58711FD501}",
-                "ThreadID"          => "24708",
-                "Channel"           => "Security",
-                "Computer"          => "Fluentd-Developing-Windows",
-                "UserID"            => nil,
-                "Version"           => "2",
-                "EventData"         => []}
-    d.instance.parse(xml) do |time, record|
-      assert_equal(expected, record)
-    end
-  end
-end