fluent-plugin-windows-eventlog 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 182e98e93e8d43f333025671ae6698ccddefd580
-  data.tar.gz: fdeda6fa7866b56938cfa714a0025aae635453ef
+  metadata.gz: e4ec43a83a8b3f54edc73e2f26ee4ec43005ba73
+  data.tar.gz: 1f225569741fe7768cf373c8d287f9ff1c11cdab
 SHA512:
-  metadata.gz: 7e46f28f6814ed735edb7db078a5784614f4e620457f1082737496b6ae80188168ed8dd5f3a5233e0ec66a261fb26a5519fc172f6821ab170c11d45ebd9568c1
-  data.tar.gz: dcb0c4be720f231921fb7dfa5430563c96cf31b392661086e2e328e2c3d4e42412328258cceb3b062b5ab4db7563a159874c4b9d75f5b159cbd474e7b8b6a627
+  metadata.gz: 343bbf9e3c173ca9707821599602fa767e3d693756c823bcafb5eeea2f87d073eefc3bae34eafaa55a15745f6875cd7c670fcd0c6ac6b096d82a9f842adec025
+  data.tar.gz: 1c30bc7385783027f6d2e3c509e3b811aede90c42cd27bc9da9b786f4de960262892a8a56c3d3b59cd42cc707af303fd499a59f7283ad41de960415a8aa41564

data/README.md

@@ -14,11 +14,18 @@ You must use fluentd 'Windows' brach to use me, and it doesn't work on Linux of
 #### fluentd Input plugin for Windows Event Log 
 
     <source>
-      type windows_eventlog
+      @type windows_eventlog
       channels application,system
       pos_file c:\temp\mypos
       read_interval 2
       tag winevt.raw
+      @id windows_eventlog
+      <storage>
+        @type local             # @type local is default.
+        persistent true         # persistent true is default.
+        path ./tmp/storage.json # This is required when persistent is true.
+                                # Or, please consider to use <system> section's root_dir parameter.
+      </storage>
     </source>
 
 

data/fluent-plugin-winevtlog.gemspec

@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-windows-eventlog"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["okahashi117", "Hiroshi Hatake", "Masahiro Nakagawa"]
-  spec.email         = ["naruki_okahashi@jbat.co.jp", "cosmo0920.wp@gmail.com", "repeatedly@gmail.com"]
+  spec.email         = ["naruki_okahashi@jbat.co.jp", "cosmo0920.oucc@gmail.com", "repeatedly@gmail.com"]
   spec.summary       = %q{Fluentd Input plugin to read windows event log.}
   spec.description   = %q{Fluentd Input plugin to read windwos event log.}
   spec.homepage      = "https://github.com/fluent/fluent-plugin-windows-eventlog"
@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", "~> 3.2.0"
-  spec.add_runtime_dependency "fluentd", [">= 0.14.11", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
   spec.add_runtime_dependency "win32-eventlog"
 end

data/lib/fluent/plugin/in_windows_eventlog.rb

@@ -6,8 +6,9 @@ module Fluent::Plugin
   class WindowsEventLogInput < Input
     Fluent::Plugin.register_input('windows_eventlog', self)
 
-    helpers :timer
+    helpers :timer, :storage
 
+    DEFAULT_STORAGE_TYPE = 'local'
     KEY_MAP = {"record_number" => :record_number,
                  "time_generated" => :time_generated,
                  "time_written" => :time_written,
@@ -21,13 +22,20 @@ module Fluent::Plugin
 
     config_param :tag, :string
     config_param :read_interval, :time, default: 2
-    config_param :pos_file, :string, default: nil
+    config_param :pos_file, :string, default: nil,
+                 obsoleted: "This section is not used anymore. Use 'store_pos' instead."
     config_param :channels, :array, default: ['Application']
     config_param :keys, :string, default: []
     config_param :read_from_head, :bool, default: false
     config_param :from_encoding, :string, default: nil
     config_param :encoding, :string, default: nil
 
+    config_section :storage do
+      config_set_default :usage, "positions"
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, true
+    end
+
     attr_reader :chs
 
     def initialize
@@ -55,6 +63,7 @@ module Fluent::Plugin
                           else
                             method(:no_encode_record)
                           end
+      @pos_storage = storage_create(usage: "positions")
     end
 
     def configure_encoding
@@ -92,59 +101,24 @@ module Fluent::Plugin
 
     def start
       super
-      if @pos_file
-        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY)
-        @pf_file.sync = true
-        @pf = PositionFile.parse(@pf_file)
-      end
-      start_watchers(@chs)
-    end
-
-    def shutdown
-      stop_watchers(@tails.keys, true)
-      @pf_file.close if @pf_file
-      super
-    end
-
-    def setup_wacther(ch, pe)
-      wlw = WindowsLogWatcher.new(ch, pe, &method(:receive_lines))
-      wlw.attach do |watcher|
-        wlw.timer_trigger = timer_execute(:in_winevtlog, @read_interval, &watcher.method(:on_notify))
-      end
-      wlw
-    end
-
-    def start_watchers(chs)
-      chs.each { |ch|
-        pe = nil
-        if @pf
-          pe = @pf[ch]
-          if @read_from_head && pe.read_num.zero?
-            el = Win32::EventLog.open(ch)
-            pe.update(el.oldest_record_number-1,1)
-            el.close
-          end
+      @chs.each do |ch|
+        start, num = @pos_storage.get(ch)
+        if @read_from_head || (!num || num.zero?)
+          el = Win32::EventLog.open(ch)
+          @pos_storage.put(ch, [el.oldest_record_number - 1, 1])
+          el.close
         end
-        @tails[ch] = setup_wacther(ch, pe)
-      }
-    end
-
-    def stop_watchers(chs, unwatched = false)
-      chs.each { |ch|
-        wlw = @tails.delete(ch)
-        if wlw
-          wlw.unwatched = unwatched
-          close_watcher(wlw)
+        timer_execute("in_windows_eventlog_#{escape_channel(ch)}".to_sym, @read_interval) do
+          on_notify(ch)
         end
-      }
+      end
     end
 
-    def close_watcher(wlw)
-      wlw.close
-      # flush_buffer(wlw)
+    def escape_channel(ch)
+      ch.gsub(/[^a-zA-Z0-9]/, '_')
     end
 
-    def receive_lines(ch, lines, pe)
+    def receive_lines(ch, lines)
       return if lines.empty?
       begin
         for r in lines
@@ -152,169 +126,50 @@ module Fluent::Plugin
           @keynames.each {|k| h[k]=@receive_handlers.call(r.send(KEY_MAP[k]).to_s)}
           #h = Hash[@keynames.map {|k| [k, r.send(KEY_MAP[k]).to_s]}]
           router.emit(@tag, Fluent::Engine.now, h)
-          pe[1] +=1
-        end
-      rescue
-        $log.error "unexpected error", error: $!.to_s
-        $log.error_backtrace
-      end
-    end
-
-
-    class WindowsLogWatcher
-      def initialize(ch, pe, &receive_lines)
-        @ch = ch
-        @pe = pe || MemoryPositionEntry.new
-        @receive_lines = receive_lines
-        @timer_trigger = nil
-      end
-
-      attr_reader   :ch
-      attr_accessor :unwatched
-      attr_accessor :pe
-      attr_accessor :timer_trigger
-
-      def attach
-        yield self
-        on_notify
-      end
-
-      def detach
-        @timer_trigger.detach if @timer_trigger.attached?
-      end
-
-      def close
-        detach
-      end
-
-      def on_notify
-        el = Win32::EventLog.open(@ch)
-        rl_sn = [el.oldest_record_number, el.total_records]
-        pe_sn = [@pe.read_start, @pe.read_num]
-        # if total_records is zero, oldest_record_number has no meaning.
-        if rl_sn[1] == 0
-          return
-        end
-
-        if pe_sn[0] == 0 && pe_sn[1] == 0
-          @pe.update(rl_sn[0], rl_sn[1])
-          return
-        end
-
-        cur_end = rl_sn[0] + rl_sn[1] -1
-        old_end = pe_sn[0] + pe_sn[1] -1
-
-        if (rl_sn[0] < pe_sn[0])
-          # may be a record number rotated.
-          cur_end += 0xFFFFFFFF
-        end
-
-        if (cur_end < old_end)
-          # something occured.
-          @pe.update(rl_sn[0], rl_sn[1])
-          return
-        end
-
-        read_more = false
-        begin
-          numlines = cur_end - old_end
-
-          winlogs = el.read(Win32::EventLog::SEEK_READ | Win32::EventLog::FORWARDS_READ, old_end + 1)
-          @receive_lines.call(@ch, winlogs, pe_sn)
-
-          @pe.update(pe_sn[0], pe_sn[1])
-          old_end = pe_sn[0] + pe_sn[1] -1
-        end while read_more
-        el.close
-      end
-    end
-
-    class PositionFile
-      def initialize(file, map, last_pos)
-        @file = file
-        @map = map
-        @last_pos = last_pos
-      end
-
-      def [](ch)
-        if m = @map[ch]
-          return m
         end
-        @file.pos = @last_pos
-        @file.write ch
-        @file.write "\t"
-        seek = @file.pos
-        @file.write "00000000\t00000000\n"
-        @last_pos = @file.pos
-        @map[ch] = FilePositionEntry.new(@file, seek)
-      end
-
-      # parsing file and rebuild mysself
-      def self.parse(file)
-        map = {}
-        file.pos = 0
-        file.each_line {|line|
-          # check and get a matched line as m
-          m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
-          ch = m[1]
-          pos = m[2].to_i(16)
-          seek = file.pos - line.bytesize + ch.bytesize + 1
-          map[ch] = FilePositionEntry.new(file, seek)
-        }
-        new(file, map, file.pos)
+      rescue => e
+        log.error "unexpected error", error: e
+        log.error_backtrace
       end
     end
 
-    class FilePositionEntry
-      START_SIZE = 8
-      NUM_OFFSET = 9
-      NUM_SIZE   = 8
-      LN_OFFSET = 17
-      SIZE = 18
+    def on_notify(ch)
+      el = Win32::EventLog.open(ch)
 
-      def initialize(file, seek)
-        @file = file
-        @seek = seek
-      end
+      current_oldest_record_number = el.oldest_record_number
+      current_total_records = el.total_records
 
-      def update(start, num)
-        @file.pos = @seek
-        @file.write "%08x\t%08x" % [start, num]
-      end
+      read_start, read_num = @pos_storage.get(ch)
 
-      def read_start
-        @file.pos = @seek
-        raw = @file.read(START_SIZE)
-        raw ? raw.to_i(16) : 0
+      # if total_records is zero, oldest_record_number has no meaning.
+      if current_total_records == 0
+        return
       end
 
-      def read_num
-        @file.pos = @seek + NUM_OFFSET
-        raw = @file.read(NUM_SIZE)
-        raw ? raw.to_i(16) : 0
+      if read_start == 0 && read_num == 0
+        @pos_storage.put(ch, [current_oldest_record_number, current_total_records])
+        return
       end
-    end
 
-    class MemoryPositionEntry
-      def initialize
-        @start = 0
-        @num = 0
-      end
+      current_end = current_oldest_record_number + current_total_records - 1
+      old_end = read_start + read_num - 1
 
-      def update(start, num)
-        @start = start
-        @num = num
+      if current_oldest_record_number < read_start
+        # may be a record number rotated.
+        current_end += 0xFFFFFFFF
       end
 
-      def read_start
-        @start
+      if current_end < old_end
+        # something occured.
+        @pos_storage.put(ch, [current_oldest_record_number, current_total_records])
+        return
       end
 
-      def read_num
-        @num
-      end
+      winlogs = el.read(Win32::EventLog::SEEK_READ | Win32::EventLog::FORWARDS_READ, old_end + 1)
+      receive_lines(ch, winlogs)
+      @pos_storage.put(ch, [read_start, read_num + winlogs.size])
+    ensure
+      el.close
     end
-
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-windows-eventlog
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - okahashi117
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-23 00:00:00.000000000 Z
+date: 2017-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,7 +60,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.11
+        version: 0.14.12
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -70,7 +70,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.11
+        version: 0.14.12
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -91,7 +91,7 @@ dependencies:
 description: Fluentd Input plugin to read windwos event log.
 email:
 - naruki_okahashi@jbat.co.jp
-- cosmo0920.wp@gmail.com
+- cosmo0920.oucc@gmail.com
 - repeatedly@gmail.com
 executables: []
 extensions: []
@@ -126,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Fluentd Input plugin to read windows event log.