fluent-plugin-windows-eventlog 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 182e98e93e8d43f333025671ae6698ccddefd580
-  data.tar.gz: fdeda6fa7866b56938cfa714a0025aae635453ef
+  metadata.gz: 44561910d111a16a85de6b5d9faee8cf052fed71
+  data.tar.gz: 2bc58308403d20000efe50f2b6f73e86a9a6585a
 SHA512:
-  metadata.gz: 7e46f28f6814ed735edb7db078a5784614f4e620457f1082737496b6ae80188168ed8dd5f3a5233e0ec66a261fb26a5519fc172f6821ab170c11d45ebd9568c1
-  data.tar.gz: dcb0c4be720f231921fb7dfa5430563c96cf31b392661086e2e328e2c3d4e42412328258cceb3b062b5ab4db7563a159874c4b9d75f5b159cbd474e7b8b6a627
+  metadata.gz: 194172daa6b8dd788a785d89e392e3698fe7ed6b8ffa0de360ddc89bf2bd9d37818d8abd43bf5c6e71434cffdd603a91126f674ab80c45455aeffe969a5101c2
+  data.tar.gz: 2b45ac6f8ce1c4d142ce1aae21104945c2bdb706db26c57eb6f9a0e2ef90ba468a7b3fd5172242817c6e43f3b1575b2167bcd5f65ebd52600f4f461f3c0780ad

data/README.md

@@ -2,65 +2,62 @@
 
 ## Component
 
-#### fluentd Input plugin for Windows Event Log
+#### fluentd Input plugin for the Windows Event Log
 
-[Fluentd](http://fluentd.org) plugin to read Windows Event Log.
-You must use fluentd 'Windows' brach to use me, and it doesn't work on Linux of course.
+[Fluentd](http://fluentd.org) plugin to read the Windows Event Log.
 
 ## Installation
     gem install fluent-plugin-windows-eventlog
 
 ## Configuration
-#### fluentd Input plugin for Windows Event Log 
+#### fluentd Input plugin for the Windows Event Log
 
     <source>
-      type windows_eventlog
+      @type windows_eventlog
+      @id windows_eventlog
       channels application,system
-      pos_file c:\temp\mypos
       read_interval 2
       tag winevt.raw
+      <storage>
+        @type local             # @type local is the default.
+        persistent true         # default is true. Set to false to use in-memory storage.
+        path ./tmp/storage.json # This is required when persistent is true.
+                                # Or, please consider using <system> section's `root_dir` parameter.
+      </storage>
     </source>
 
-
 #### parameters
 
 |name      | description |
 |:-----    |:-----       |
-|channels   | (option) 'applicaion' as default. one or combination of {application, system, setup, security}. If you want to read setup or security, administrator priv is required to launch fluentd.  |
-|pos_file  | (option, but higly recommended) a path of position file to save record numbers. |
-|read_interval   | (option) a read interval in second. 2 seconds as default.|
-|from_encoding  | (option) an input characters encoding. nil as default.|
-|encoding   | (option) an output characters encoding. nil as default.|
-
+|`channels`      | (option) 'application' as default. One or more of {'application', 'system', 'setup', 'security'}. If you want to read 'setup' or 'security' logs, you must launch fluentd with administrator privileges.|
+|`keys`          | (option) A subset of [keys](#read-keys) to read. Defaults to all keys.|
+|`read_interval` | (option) Read interval in seconds. 2 seconds as default.|
+|`from_encoding` | (option) Input character encoding. `nil` as default.|
+|`encoding`      | (option) Output character encoding. `nil` as default.|
+|`read_from_head`| (option) Start to read the entries from the oldest, not from when fluentd is started. Defaults to `false`.|
+|`<storage>`     | Setting for `storage` plugin for recording read position like `in_tail`'s `pos_file`.|
 
 #### read keys
-This plugin reads follows from Windws Event Log. No customization is allowed currently.
+This plugin reads the following fields from Windows Event Log entries. Use the `keys` configuration option to select a subset. No other customization is allowed for now.
 
 |key|
 |:-----    |
-|record_number   |
-|time_generated|
-|time_written   |
-|event_id   |
-|event_type   |
-|event_category   |
-|source_name   |
-|computer_name  |
-|user   |
-|description   |
-
-
-
-## Etc.
-'read_from_head' is not supporeted currently.You can read newer records after you start first.
-No customize to read information keys.
-
-
-
+|`record_number` |
+|`time_generated`|
+|`time_written`  |
+|`event_id`      |
+|`event_type`    |
+|`event_category`|
+|`source_name`   |
+|`computer_name` |
+|`user`          |
+|`description`   |
+|`string_inserts`|
 
 ## Copyright
-####Copyright
+#### Copyright
 Copyright(C) 2014- @okahashi117
-####License
+#### License
 Apache License, Version 2.0
 

data/appveyor.yml

@@ -0,0 +1,27 @@
+version: '{build}'
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - "%devkit%\\devkitvars.bat"
+  - ruby --version
+  - gem --version
+  - bundle install
+build: off
+test_script:
+  - bundle exec rake test
+#  - bundle exec rake test TESTOPTS=-v
+
+branches:
+  only:
+    - master
+
+# https://www.appveyor.com/docs/installed-software/#ruby
+environment:
+  matrix:
+    - ruby_version: "23-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "23"
+      devkit: C:\Ruby23\DevKit

data/fluent-plugin-winevtlog.gemspec

@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-windows-eventlog"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.1"
   spec.authors       = ["okahashi117", "Hiroshi Hatake", "Masahiro Nakagawa"]
-  spec.email         = ["naruki_okahashi@jbat.co.jp", "cosmo0920.wp@gmail.com", "repeatedly@gmail.com"]
+  spec.email         = ["naruki_okahashi@jbat.co.jp", "cosmo0920.oucc@gmail.com", "repeatedly@gmail.com"]
   spec.summary       = %q{Fluentd Input plugin to read windows event log.}
   spec.description   = %q{Fluentd Input plugin to read windwos event log.}
   spec.homepage      = "https://github.com/fluent/fluent-plugin-windows-eventlog"
@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", "~> 3.2.0"
-  spec.add_runtime_dependency "fluentd", [">= 0.14.11", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
   spec.add_runtime_dependency "win32-eventlog"
 end

data/lib/fluent/plugin/in_windows_eventlog.rb

@@ -6,28 +6,37 @@ module Fluent::Plugin
   class WindowsEventLogInput < Input
     Fluent::Plugin.register_input('windows_eventlog', self)
 
-    helpers :timer
-
-    KEY_MAP = {"record_number" => :record_number,
-                 "time_generated" => :time_generated,
-                 "time_written" => :time_written,
-                 "event_id" => :event_id,
-                 "event_type" => :event_type,
-                 "event_category" => :category,
-                 "source_name" => :source,
-                 "computer_name" => :computer,
-                 "user" => :user,
-                 "description" => :description}
+    helpers :timer, :storage
+
+    DEFAULT_STORAGE_TYPE = 'local'
+    KEY_MAP = {"record_number" => [:record_number, :string],
+                 "time_generated" => [:time_generated, :string],
+                 "time_written" => [:time_written, :string],
+                 "event_id" => [:event_id, :string],
+                 "event_type" => [:event_type, :string],
+                 "event_category" => [:category, :string],
+                 "source_name" => [:source, :string],
+                 "computer_name" => [:computer, :string],
+                 "user" => [:user, :string],
+                 "description" => [:description, :string],
+                 "string_inserts" => [:string_inserts, :array]}
 
     config_param :tag, :string
     config_param :read_interval, :time, default: 2
-    config_param :pos_file, :string, default: nil
-    config_param :channels, :array, default: ['Application']
-    config_param :keys, :string, default: []
+    config_param :pos_file, :string, default: nil,
+                 obsoleted: "This section is not used anymore. Use 'store_pos' instead."
+    config_param :channels, :array, default: ['application']
+    config_param :keys, :array, default: []
     config_param :read_from_head, :bool, default: false
     config_param :from_encoding, :string, default: nil
     config_param :encoding, :string, default: nil
 
+    config_section :storage do
+      config_set_default :usage, "positions"
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, true
+    end
+
     attr_reader :chs
 
     def initialize
@@ -55,6 +64,7 @@ module Fluent::Plugin
                           else
                             method(:no_encode_record)
                           end
+      @pos_storage = storage_create(usage: "positions")
     end
 
     def configure_encoding
@@ -92,229 +102,86 @@ module Fluent::Plugin
 
     def start
       super
-      if @pos_file
-        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY)
-        @pf_file.sync = true
-        @pf = PositionFile.parse(@pf_file)
-      end
-      start_watchers(@chs)
-    end
-
-    def shutdown
-      stop_watchers(@tails.keys, true)
-      @pf_file.close if @pf_file
-      super
-    end
-
-    def setup_wacther(ch, pe)
-      wlw = WindowsLogWatcher.new(ch, pe, &method(:receive_lines))
-      wlw.attach do |watcher|
-        wlw.timer_trigger = timer_execute(:in_winevtlog, @read_interval, &watcher.method(:on_notify))
-      end
-      wlw
-    end
-
-    def start_watchers(chs)
-      chs.each { |ch|
-        pe = nil
-        if @pf
-          pe = @pf[ch]
-          if @read_from_head && pe.read_num.zero?
-            el = Win32::EventLog.open(ch)
-            pe.update(el.oldest_record_number-1,1)
-            el.close
-          end
+      @chs.each do |ch|
+        start, num = @pos_storage.get(ch)
+        if @read_from_head || (!num || num.zero?)
+          el = Win32::EventLog.open(ch)
+          @pos_storage.put(ch, [el.oldest_record_number - 1, 1])
+          el.close
         end
-        @tails[ch] = setup_wacther(ch, pe)
-      }
-    end
-
-    def stop_watchers(chs, unwatched = false)
-      chs.each { |ch|
-        wlw = @tails.delete(ch)
-        if wlw
-          wlw.unwatched = unwatched
-          close_watcher(wlw)
+        timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
+          on_notify(ch)
         end
-      }
+      end
     end
 
-    def close_watcher(wlw)
-      wlw.close
-      # flush_buffer(wlw)
+    def escape_channel(ch)
+      ch.gsub(/[^a-zA-Z0-9]/, '_')
     end
 
-    def receive_lines(ch, lines, pe)
+    def receive_lines(ch, lines)
       return if lines.empty?
       begin
         for r in lines
           h = {"channel" => ch}
-          @keynames.each {|k| h[k]=@receive_handlers.call(r.send(KEY_MAP[k]).to_s)}
-          #h = Hash[@keynames.map {|k| [k, r.send(KEY_MAP[k]).to_s]}]
+          @keynames.each do |k|
+            type = KEY_MAP[k][1]
+            value = r.send(KEY_MAP[k][0])
+            h[k]=case type
+                 when :string
+                   @receive_handlers.call(value.to_s)
+                 when :array
+                   value.map {|v| @receive_handlers.call(v.to_s)}
+                 else
+                   raise "Unknown value type: #{type}"
+                 end
+          end
+          #h = Hash[@keynames.map {|k| [k, r.send(KEY_MAP[k][0]).to_s]}]
           router.emit(@tag, Fluent::Engine.now, h)
-          pe[1] +=1
         end
-      rescue
-        $log.error "unexpected error", error: $!.to_s
-        $log.error_backtrace
+      rescue => e
+        log.error "unexpected error", error: e
+        log.error_backtrace
       end
     end
 
+    def on_notify(ch)
+      el = Win32::EventLog.open(ch)
 
-    class WindowsLogWatcher
-      def initialize(ch, pe, &receive_lines)
-        @ch = ch
-        @pe = pe || MemoryPositionEntry.new
-        @receive_lines = receive_lines
-        @timer_trigger = nil
-      end
+      current_oldest_record_number = el.oldest_record_number
+      current_total_records = el.total_records
 
-      attr_reader   :ch
-      attr_accessor :unwatched
-      attr_accessor :pe
-      attr_accessor :timer_trigger
+      read_start, read_num = @pos_storage.get(ch)
 
-      def attach
-        yield self
-        on_notify
+      # if total_records is zero, oldest_record_number has no meaning.
+      if current_total_records == 0
+        return
       end
 
-      def detach
-        @timer_trigger.detach if @timer_trigger.attached?
+      if read_start == 0 && read_num == 0
+        @pos_storage.put(ch, [current_oldest_record_number, current_total_records])
+        return
       end
 
-      def close
-        detach
-      end
-
-      def on_notify
-        el = Win32::EventLog.open(@ch)
-        rl_sn = [el.oldest_record_number, el.total_records]
-        pe_sn = [@pe.read_start, @pe.read_num]
-        # if total_records is zero, oldest_record_number has no meaning.
-        if rl_sn[1] == 0
-          return
-        end
-
-        if pe_sn[0] == 0 && pe_sn[1] == 0
-          @pe.update(rl_sn[0], rl_sn[1])
-          return
-        end
-
-        cur_end = rl_sn[0] + rl_sn[1] -1
-        old_end = pe_sn[0] + pe_sn[1] -1
-
-        if (rl_sn[0] < pe_sn[0])
-          # may be a record number rotated.
-          cur_end += 0xFFFFFFFF
-        end
-
-        if (cur_end < old_end)
-          # something occured.
-          @pe.update(rl_sn[0], rl_sn[1])
-          return
-        end
+      current_end = current_oldest_record_number + current_total_records - 1
+      old_end = read_start + read_num - 1
 
-        read_more = false
-        begin
-          numlines = cur_end - old_end
-
-          winlogs = el.read(Win32::EventLog::SEEK_READ | Win32::EventLog::FORWARDS_READ, old_end + 1)
-          @receive_lines.call(@ch, winlogs, pe_sn)
-
-          @pe.update(pe_sn[0], pe_sn[1])
-          old_end = pe_sn[0] + pe_sn[1] -1
-        end while read_more
-        el.close
+      if current_oldest_record_number < read_start
+        # may be a record number rotated.
+        current_end += 0xFFFFFFFF
       end
-    end
 
-    class PositionFile
-      def initialize(file, map, last_pos)
-        @file = file
-        @map = map
-        @last_pos = last_pos
+      if current_end < old_end
+        # something occured.
+        @pos_storage.put(ch, [current_oldest_record_number, current_total_records])
+        return
       end
 
-      def [](ch)
-        if m = @map[ch]
-          return m
-        end
-        @file.pos = @last_pos
-        @file.write ch
-        @file.write "\t"
-        seek = @file.pos
-        @file.write "00000000\t00000000\n"
-        @last_pos = @file.pos
-        @map[ch] = FilePositionEntry.new(@file, seek)
-      end
-
-      # parsing file and rebuild mysself
-      def self.parse(file)
-        map = {}
-        file.pos = 0
-        file.each_line {|line|
-          # check and get a matched line as m
-          m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
-          ch = m[1]
-          pos = m[2].to_i(16)
-          seek = file.pos - line.bytesize + ch.bytesize + 1
-          map[ch] = FilePositionEntry.new(file, seek)
-        }
-        new(file, map, file.pos)
-      end
+      winlogs = el.read(Win32::EventLog::SEEK_READ | Win32::EventLog::FORWARDS_READ, old_end + 1)
+      receive_lines(ch, winlogs)
+      @pos_storage.put(ch, [read_start, read_num + winlogs.size])
+    ensure
+      el.close
     end
-
-    class FilePositionEntry
-      START_SIZE = 8
-      NUM_OFFSET = 9
-      NUM_SIZE   = 8
-      LN_OFFSET = 17
-      SIZE = 18
-
-      def initialize(file, seek)
-        @file = file
-        @seek = seek
-      end
-
-      def update(start, num)
-        @file.pos = @seek
-        @file.write "%08x\t%08x" % [start, num]
-      end
-
-      def read_start
-        @file.pos = @seek
-        raw = @file.read(START_SIZE)
-        raw ? raw.to_i(16) : 0
-      end
-
-      def read_num
-        @file.pos = @seek + NUM_OFFSET
-        raw = @file.read(NUM_SIZE)
-        raw ? raw.to_i(16) : 0
-      end
-    end
-
-    class MemoryPositionEntry
-      def initialize
-        @start = 0
-        @num = 0
-      end
-
-      def update(start, num)
-        @start = start
-        @num = num
-      end
-
-      def read_start
-        @start
-      end
-
-      def read_num
-        @num
-      end
-    end
-
   end
 end

data/test/generate-windows-event.rb

@@ -0,0 +1,47 @@
+require 'win32/eventlog'
+
+class EventLog
+  def initialize
+    @logger = Win32::EventLog.new
+    @app_source = "fluent-plugins"
+  end
+
+  def info(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::INFO_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+  def warn(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::WARN_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+  def crit(event_id, message)
+    @logger.report_event(
+      source: @app_source,
+      event_type: Win32::EventLog::ERROR_TYPE,
+      event_id: event_id,
+      data: message
+    )
+  end
+
+end
+
+module Fluent
+  module Plugin
+    class EventService
+      def run
+        eventlog = EventLog.new()
+        eventlog.info(65500, "Hi, from fluentd-plugins!! at " + Time.now.strftime("%Y/%m/%d %H:%M:%S "))
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -27,3 +27,6 @@ require 'fluent/plugin/in_windows_eventlog'
 
 class Test::Unit::TestCase
 end
+require 'fluent/test/helpers'
+
+include Fluent::Test::Helpers

data/test/plugin/test_in_winevtlog.rb

@@ -1,13 +1,18 @@
 require 'helper'
+require 'generate-windows-event'
 
 class WindowsEventLogInputTest < Test::Unit::TestCase
+
   def setup
     Fluent::Test.setup
   end
 
-  CONFIG = %[
-    tag fluent.eventlog
-  ]
+  CONFIG = config_element("ROOT", "", {"tag" => "fluent.eventlog"}, [
+                            config_element("storage", "", {
+                                             '@type' => 'local',
+                                             'persistent' => false
+                                           })
+                          ])
 
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::WindowsEventLogInput).configure(conf)
@@ -18,34 +23,26 @@ class WindowsEventLogInputTest < Test::Unit::TestCase
     assert_equal 'fluent.eventlog', d.instance.tag
     assert_equal 2, d.instance.read_interval
     assert_nil d.instance.pos_file
-    assert_equal ['Application'], d.instance.channels
+    assert_equal ['application'], d.instance.channels
     assert_true d.instance.keys.empty?
     assert_false d.instance.read_from_head
   end
 
-  def test_format
-    d = create_driver
-
-    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    # d.emit({"a"=>1}, time)
-    # d.emit({"a"=>2}, time)
-
-    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
-    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
-
-    # d.run
-  end
-
   def test_write
     d = create_driver
 
-    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    # d.emit({"a"=>1}, time)
-    # d.emit({"a"=>2}, time)
+    service = Fluent::Plugin::EventService.new
+
+    d.run(expect_emits: 1) do
+      service.run
+    end
 
-    # ### FileOutput#write returns path
-    # path = d.run
-    # expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
-    # assert_equal expect_path, path
+    assert(d.events.length >= 1)
+    event = d.events.last
+    record = event.last
+    assert_equal("application", record["channel"])
+    assert_equal("65500", record["event_id"])
+    assert_equal("information", record["event_type"])
+    assert_equal("fluent-plugins", record["source_name"])
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-windows-eventlog
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - okahashi117
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-23 00:00:00.000000000 Z
+date: 2017-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,7 +60,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.11
+        version: 0.14.12
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -70,7 +70,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.11
+        version: 0.14.12
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -91,7 +91,7 @@ dependencies:
 description: Fluentd Input plugin to read windwos event log.
 email:
 - naruki_okahashi@jbat.co.jp
-- cosmo0920.wp@gmail.com
+- cosmo0920.oucc@gmail.com
 - repeatedly@gmail.com
 executables: []
 extensions: []
@@ -102,8 +102,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- appveyor.yml
 - fluent-plugin-winevtlog.gemspec
 - lib/fluent/plugin/in_windows_eventlog.rb
+- test/generate-windows-event.rb
 - test/helper.rb
 - test/plugin/test_in_winevtlog.rb
 homepage: https://github.com/fluent/fluent-plugin-windows-eventlog
@@ -126,10 +128,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd Input plugin to read windows event log.
 test_files:
+- test/generate-windows-event.rb
 - test/helper.rb
 - test/plugin/test_in_winevtlog.rb