fluent-plugin-windows-eventlog 0.7.1.rc1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5bfbe6035f27762fffa9e75863aee873ac0873ebd24694ae19733c230cb9170
-  data.tar.gz: 663554158b2d149821ca54701b655392afff19132c453c0be3f534aa1a684898
+  metadata.gz: 29f3b9ced1cd226756c2c6859e50cc974ec95e42a0efdc4b0e5544d69eac9c63
+  data.tar.gz: 61875762c40f75317704cc62a476dc16d947363c7dca6bf167b8c7a23c86a095
 SHA512:
-  metadata.gz: 067ec531a1b265133b924709203d54a45c9641ac982ce0a1b876dc16d4968362c691ac4950bbdb5c38516a3a9bebd0389d46a66c1d1cc1e95715d3c76a783ee6
-  data.tar.gz: '097951b41de440a39b7ca983b93320778e38d15c28bbaa1495aff593637d4b805f4413dedd7e2855ec8b3d89b72c90fd23c354d4e7e34d075af5d0a5ebaeb326'
+  metadata.gz: 1c126f6937f433079ddf703d141ecf73d1c3e1bd82e9fb605726109c20c97ab89f3307438da9fd62c80ef80770a39a884c3800c66874ca9519e680d17c9c9ba9
+  data.tar.gz: f5e9178ed7c1b477977d61d4fe306745f2efb3203bf430a612c96e9a27f1186d9980ef7ea236b679ab05c805bca3426b70d77a41288458747e684eb20009609f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# Release v0.8.0 - 2020/09/16
+* in_windows_eventlog2: Support remoting access
+
 # Release v.0.7.1.rc1 - 2020/06/23
 * in_windows_eventlog2: Depends on nokogiri 1.11 series
 

data/README.md

@@ -144,6 +144,7 @@ fluentd Input plugin for the Windows Event Log using newer Windows Event Logging
       # preserve_qualifiers_on_hash true # default is false.
       # read_all_channels false # default is false.
       # description_locale en_US # default is nil. It means that system locale is used for obtaining description.
+      # refresh_subscription_interval 10m # default is nil. It specifies refresh interval for channel subscriptions.
       <storage>
         @type local             # @type local is the default.
         persistent true         # default is true. Set to false to use in-memory storage.
@@ -161,6 +162,10 @@ fluentd Input plugin for the Windows Event Log using newer Windows Event Logging
       # <subscribe>
       #   channles application, system
       #   read_existing_events false # read_existing_events should be applied each of subscribe directive(s)
+      #   remote_server 127.0.0.1 # Remote server ip/fqdn
+      #   remote_domain WORKGROUP # Domain name
+      #   remote_username fluentd # Remoting access account name
+      #   remote_password changeme! # Remoting access account password
       # </subscribe>
     </source>
 
@@ -190,6 +195,7 @@ fluentd Input plugin for the Windows Event Log using newer Windows Event Logging
 |`preserve_qualifiers_on_hash`      | (option) When set up it as true, this plugin preserves "Qualifiers" and "EventID" keys. When set up it as false, this plugin calculates actual "EventID" from "Qualifiers" and removing "Qualifiers". Default is `false`.|
 |`read_all_channels`| (option) Read from all channels. Default is `false`|
 |`description_locale`| (option) Specify description locale. Default is `nil`. See also: [Supported locales](https://github.com/fluent-plugins-nursery/winevt_c#multilingual-description) |
+|`refresh_subscription_interval`|(option) It specifies refresh interval for channel subscriptions. Default is `nil`.|
 |`<subscribe>`          | Setting for subscribe channels. |
 
 ##### subscribe section
@@ -198,6 +204,10 @@ fluentd Input plugin for the Windows Event Log using newer Windows Event Logging
 |:-----    |:-----       |
 |`channels`             | One or more of {'application', 'system', 'setup', 'security'}. If you want to read 'setup' or 'security' logs, you must launch fluentd with administrator privileges. |
 |`read_existing_events` | (option) Read the entries which already exist before fluentd is started. Defaults to `false`. |
+|`remote_server` | (option) Remoting access server ip address/fqdn. Defaults to `nil`. |
+|`remote_domain` | (option) Remoting access server joining domain name. Defaults to `nil`. |
+|`remote_username` | (option) Remoting access access account's username. Defaults to `nil`. |
+|`remote_password` | (option) Remoting access access account's password. Defaults to `nil`. |
 
 
 **Motivation:** subscribe directive is designed for applying `read_existing_events` each of channels which is specified in subscribe section(s).
@@ -234,6 +244,33 @@ This configuration can be handled as:
 * "Application" and "Security" channels just tailing
 * "HardwareEvent" channel read existing events before launching Fluentd
 
+###### Remoting access
+
+`<subscribe>` section supports remoting access parameters:
+
+* `remote_server`
+* `remote_domain`
+* `remote_username`
+* `remote_password`
+
+These parameters are only in `<subscribe>` directive.
+
+Note that before using this feature, remoting access users should belong to "Event Log Readers" group:
+
+```console
+> net localgroup "Event Log Readers" <domain\username> /add
+```
+
+And then, users also should set up their remote box's Firewall configuration:
+
+```console
+> netsh advfirewall firewall set rule group="Remote Event Log Management" new enable=yes
+```
+
+As a security best practices, remoting access account _should not be administrator account_.
+
+For graphical instructions, please refer to [Preconfigure a Machine to Collect Remote Windows Events | Sumo Logic](https://help.sumologic.com/03Send-Data/Sources/01Sources-for-Installed-Collectors/Remote-Windows-Event-Log-Source/Preconfigure-a-Machine-to-Collect-Remote-Windows-Events) document for example.
+
 ##### Available keys
 
 This plugin reads the following fields from Windows Event Log entries. Use the `keys` configuration option to select a subset. No other customization is allowed for now.

data/fluent-plugin-winevtlog.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-windows-eventlog"
-  spec.version       = "0.7.1.rc1"
+  spec.version       = "0.8.0"
   spec.authors       = ["okahashi117", "Hiroshi Hatake", "Masahiro Nakagawa"]
   spec.email         = ["naruki_okahashi@jbat.co.jp", "cosmo0920.oucc@gmail.com", "repeatedly@gmail.com"]
   spec.summary       = %q{Fluentd Input plugin to read windows event log.}
@@ -20,9 +20,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", "~> 3.2.0"
-  spec.add_development_dependency "nokogiri", [">= 1.11.pre", "< 1.12"]
+  spec.add_development_dependency "nokogiri", [">= 1.10", "< 1.12"]
   spec.add_development_dependency "fluent-plugin-parser-winevt_xml", ">= 0.1.2"
   spec.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
   spec.add_runtime_dependency "win32-eventlog"
-  spec.add_runtime_dependency "winevt_c", ">= 0.8.1"
+  spec.add_runtime_dependency "winevt_c", ">= 0.9.1"
 end

data/lib/fluent/plugin/in_windows_eventlog2.rb

@@ -7,6 +7,8 @@ module Fluent::Plugin
   class WindowsEventLog2Input < Input
     Fluent::Plugin.register_input('windows_eventlog2', self)
 
+    class ReconnectError < Fluent::UnrecoverableError; end
+
     helpers :timer, :storage, :parser
 
     DEFAULT_STORAGE_TYPE = 'local'
@@ -43,10 +45,15 @@ module Fluent::Plugin
     config_param :preserve_qualifiers_on_hash, :bool, default: false
     config_param :read_all_channels, :bool, default: false
     config_param :description_locale, :string, default: nil
+    config_param :refresh_subscription_interval, :time, default: nil
 
     config_section :subscribe, param_name: :subscribe_configs, required: false, multi: true do
       config_param :channels, :array
       config_param :read_existing_events, :bool, default: false
+      config_param :remote_server, :string, default: nil
+      config_param :remote_domain, :string, default: nil
+      config_param :remote_username, :string, default: nil
+      config_param :remote_password, :string, default: nil, secret: true
     end
 
     config_section :storage do
@@ -68,9 +75,12 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+      @session = nil
       @chs = []
+      @subscriptions = {}
       @all_chs = Winevt::EventLog::Channel.new
       @all_chs.force_enumerate = false
+      @timers = {}
 
       if @read_all_channels
         @all_chs.each do |ch|
@@ -81,14 +91,22 @@ module Fluent::Plugin
 
       @read_existing_events = @read_from_head || @read_existing_events
       if @channels.empty? && @subscribe_configs.empty? && !@read_all_channels
-        @chs.push(['application', @read_existing_events])
+        @chs.push(['application', @read_existing_events, nil])
       else
         @channels.map {|ch| ch.strip.downcase }.uniq.each do |uch|
-          @chs.push([uch, @read_existing_events])
+          @chs.push([uch, @read_existing_events, nil])
         end
         @subscribe_configs.each do |subscribe|
+          if subscribe.remote_server
+            @session = Winevt::EventLog::Session.new(subscribe.remote_server,
+                                                     subscribe.remote_domain,
+                                                     subscribe.remote_username,
+                                                     subscribe.remote_password)
+
+            log.debug("connect to remote box (server: #{subscribe.remote_server}) domain: #{subscribe.remote_domain} username: #{subscribe.remote_username})")
+          end
           subscribe.channels.map {|ch| ch.strip.downcase }.uniq.each do |uch|
-            @chs.push([uch, subscribe.read_existing_events])
+            @chs.push([uch, subscribe.read_existing_events, @session])
           end
         end
       end
@@ -137,12 +155,73 @@ module Fluent::Plugin
     def start
       super
 
-      @chs.each do |ch, read_existing_events|
-        subscribe_channel(ch, read_existing_events)
+      refresh_subscriptions
+      if @refresh_subscription_interval
+        timer_execute(:in_windows_eventlog_refresh_subscription_timer, @refresh_subscription_interval, &method(:refresh_subscriptions))
       end
     end
 
-    def subscribe_channel(ch, read_existing_events)
+    def shutdown
+      super
+
+      @subscriptions.keys.each do |ch|
+        subscription = @subscriptions.delete(ch)
+        if subscription
+          subscription.cancel
+          log.debug "channel (#{ch}) subscription is canceled."
+        end
+      end
+    end
+
+    def retry_on_error(channel, times: 15)
+      try = 0
+      begin
+        log.debug "Retry to subscribe for #{channel}...." if try > 1
+        try += 1
+        yield
+        log.info "Retry to subscribe for #{channel} succeeded." if try > 1
+        try = 0
+      rescue Winevt::EventLog::Subscribe::RemoteHandlerError => e
+        raise ReconnectError, "Retrying limit is exceeded." if try > times
+        log.warn "#{e.message}. Remaining retry count(s): #{times - try}"
+        sleep 2**try
+        retry
+      end
+    end
+
+    def refresh_subscriptions
+      clear_subscritpions
+
+      @chs.each do |ch, read_existing_events, session|
+        retry_on_error(ch) do
+          ch, subscribe = subscription(ch, read_existing_events, session)
+          @subscriptions[ch] = subscribe
+        end
+      end
+      subscribe_channels(@subscriptions)
+    end
+
+    def clear_subscritpions
+      @subscriptions.keys.each do |ch|
+        subscription = @subscriptions.delete(ch)
+        if subscription
+          if subscription.cancel
+            log.debug "channel (#{ch}) subscription is cancelled."
+            subscription.close
+            log.debug "channel (#{ch}) subscription handles are closed forcibly."
+          end
+        end
+      end
+      @timers.keys.each do |ch|
+        timer = @timers.delete(ch)
+        if timer
+          event_loop_detach(timer)
+          log.debug "channel (#{ch}) subscription watcher is detached."
+        end
+      end
+    end
+
+    def subscription(ch, read_existing_events, remote_session)
       bookmarkXml = @bookmarks_storage.get(ch) || ""
       bookmark = nil
       if bookmark_validator(bookmarkXml, ch)
@@ -151,7 +230,7 @@ module Fluent::Plugin
       subscribe = Winevt::EventLog::Subscribe.new
       subscribe.read_existing_events = read_existing_events
       begin
-        subscribe.subscribe(ch, "*", bookmark)
+        subscribe.subscribe(ch, "*", bookmark, remote_session)
         if !@render_as_xml && @preserve_qualifiers_on_hash
           subscribe.preserve_qualifiers = @preserve_qualifiers_on_hash
         end
@@ -161,8 +240,15 @@ module Fluent::Plugin
       subscribe.render_as_xml = @render_as_xml
       subscribe.rate_limit = @rate_limit
       subscribe.locale = @description_locale if @description_locale
-      timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
-        on_notify(ch, subscribe)
+      [ch, subscribe]
+    end
+
+    def subscribe_channels(subscriptions)
+      subscriptions.each do |ch, subscribe|
+        @timers[ch] = timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
+          on_notify(ch, subscribe)
+        end
+        log.debug "channel (#{ch}) subscription is subscribed."
       end
     end
 
@@ -222,12 +308,12 @@ module Fluent::Plugin
             end
           end
         end
+        router.emit_stream(@tag, es)
+        @bookmarks_storage.put(ch, subscribe.bookmark)
       rescue Winevt::EventLog::Query::Error => e
-        log.warn "Invalid XML data", error: e
+        log.warn "Invalid XML data on #{ch}.", error: e
         log.warn_backtrace
       end
-      router.emit_stream(@tag, es)
-      @bookmarks_storage.put(ch, subscribe.bookmark)
     end
 
     def on_notify_hash(ch, subscribe)
@@ -252,12 +338,12 @@ module Fluent::Plugin
           parse_desc(h) if @parse_description
           es.add(Fluent::Engine.now, h)
         end
+        router.emit_stream(@tag, es)
+        @bookmarks_storage.put(ch, subscribe.bookmark)
       rescue Winevt::EventLog::Query::Error => e
-        log.warn "Invalid Hash data", error: e
+        log.warn "Invalid Hash data on #{ch}.", error: e
         log.warn_backtrace
       end
-      router.emit_stream(@tag, es)
-      @bookmarks_storage.put(ch, subscribe.bookmark)
     end
 
     #### These lines copied from in_windows_eventlog plugin:

data/test/plugin/test_in_windows_eventlog2.rb

@@ -2,6 +2,17 @@ require 'helper'
 require 'fileutils'
 require 'generate-windows-event'
 
+# Monkey patch for testing
+class Winevt::EventLog::Session
+  def ==(obj)
+    self.server == obj.server &&
+      self.domain == obj.domain &&
+      self.username == obj.username &&
+      self.password == obj.password &&
+      self.flags == obj.flags
+  end
+end
+
 class WindowsEventLog2InputTest < Test::Unit::TestCase
 
   def setup
@@ -34,9 +45,21 @@ class WindowsEventLog2InputTest < Test::Unit::TestCase
     assert_equal [], d.instance.channels
     assert_false d.instance.read_existing_events
     assert_false d.instance.render_as_xml
+    assert_nil d.instance.refresh_subscription_interval
   end
 
   sub_test_case "configure" do
+    test "refresh subscription interval" do
+      d = create_driver config_element("ROOT", "", {"tag" => "fluent.eventlog",
+                                                    "refresh_subscription_interval" => "2m"}, [
+                                         config_element("storage", "", {
+                                                          '@type' => 'local',
+                                                          'persistent' => false
+                                                        })
+                                       ])
+      assert_equal 120, d.instance.refresh_subscription_interval
+    end
+
     test "subscribe directive" do
       d = create_driver config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
                                          config_element("storage", "", {
@@ -51,7 +74,36 @@ class WindowsEventLog2InputTest < Test::Unit::TestCase
                                                           'read_existing_events' => true
                                                         }),
                                        ])
-      expected = [["system", false], ["windows powershell", false], ["security", true]]
+      expected = [["system", false, nil], ["windows powershell", false, nil], ["security", true, nil]]
+      assert_equal expected, d.instance.instance_variable_get(:@chs)
+    end
+
+    test "subscribe directive with remote server session" do
+      d = create_driver config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                                         config_element("storage", "", {
+                                                          '@type' => 'local',
+                                                          'persistent' => false
+                                                        }),
+                                         config_element("subscribe", "", {
+                                                          'channels' => ['System', 'Windows PowerShell'],
+                                                          'remote_server' => '127.0.0.1',
+                                                        }),
+                                         config_element("subscribe", "", {
+                                                          'channels' => ['Security'],
+                                                          'read_existing_events' => true,
+                                                          'remote_server' => '192.168.0.1',
+                                                          'remote_username' => 'fluentd',
+                                                          'remote_password' => 'changeme!'
+                                                        }),
+                                       ])
+      localhost_session = Winevt::EventLog::Session.new("127.0.0.1")
+      remote_session = Winevt::EventLog::Session.new("192.168.0.1",
+                                                     nil,
+                                                     "fluentd",
+                                                     "changeme!")
+      expected = [["system", false, localhost_session],
+                  ["windows powershell", false, localhost_session],
+                  ["security", true, remote_session]]
       assert_equal expected, d.instance.instance_variable_get(:@chs)
     end
 
@@ -71,7 +123,7 @@ class WindowsEventLog2InputTest < Test::Unit::TestCase
                                                           'read_existing_events' => true
                                                         }),
                                        ])
-      expected = [["system", false], ["windows powershell", false], ["security", true]]
+      expected = [["system", false, nil], ["windows powershell", false, nil], ["security", true, nil]]
       assert_equal 1, d.instance.instance_variable_get(:@chs).select {|ch, flag| ch == "system"}.size
       assert_equal expected, d.instance.instance_variable_get(:@chs)
     end
@@ -93,7 +145,7 @@ class WindowsEventLog2InputTest < Test::Unit::TestCase
                                                           'read_existing_events' => true
                                                         }),
                                        ])
-      expected = [["system", false], ["windows powershell", false], ["system", true], ["windows powershell", true], ["security", true]]
+      expected = [["system", false, nil], ["windows powershell", false, nil], ["system", true, nil], ["windows powershell", true, nil], ["security", true, nil]]
       assert_equal 2, d.instance.instance_variable_get(:@chs).select {|ch, flag| ch == "system"}.size
       assert_equal expected, d.instance.instance_variable_get(:@chs)
     end
@@ -269,7 +321,7 @@ DESC
     end
 
     assert(d.events.length >= 1)
-    event = d.events.last
+    event = d.events.select {|e| e.last["EventID"] == "65500" }.last
     record = event.last
 
     expected = {"EventID"      => "65500",
@@ -280,6 +332,36 @@ DESC
     assert_equal(expected, record)
   end
 
+  REMOTING_ACCESS_CONFIG = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                                            config_element("storage", "", {
+                                                             '@type' => 'local',
+                                                             'persistent' => false
+                                                           }),
+                                            config_element("subscribe", "", {
+                                                             'channels' => ['Application'],
+                                                             'remote_server' => '127.0.0.1',
+                                                           }),
+                                          ])
+
+  def test_write_with_remoting_access
+    d = create_driver(REMOTING_ACCESS_CONFIG)
+
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
+
+    assert(d.events.length >= 1)
+    event = d.events.select {|e| e.last["EventID"] == "65500" }.last
+    record = event.last
+
+    assert_equal("Application", record["Channel"])
+    assert_equal("65500", record["EventID"])
+    assert_equal("4", record["Level"])
+    assert_equal("fluent-plugins", record["ProviderName"])
+  end
+
   class HashRendered < self
     def test_write
       d = create_driver(config_element("ROOT", "", {"tag" => "fluent.eventlog",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-windows-eventlog
 version: !ruby/object:Gem::Version
-  version: 0.7.1.rc1
+  version: 0.8.0
 platform: ruby
 authors:
 - okahashi117
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2020-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,7 +60,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.11.pre
+        version: '1.10'
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -70,7 +70,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.11.pre
+        version: '1.10'
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.12'
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 0.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 0.9.1
 description: Fluentd Input plugin to read windows event log.
 email:
 - naruki_okahashi@jbat.co.jp
@@ -177,9 +177,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6.2