logstash-input-file 4.0.5 → 4.1.0

data/lib/filewatch/watch.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module FileWatch
+  class Watch
+    include LogStash::Util::Loggable
+
+    attr_accessor :lastwarn_max_files
+    attr_reader :discoverer, :watched_files_collection
+
+    def initialize(discoverer, watched_files_collection, settings)
+      @settings = settings
+      # watch and iterate_on_state can be called from different threads.
+      @lock = Mutex.new
+      # we need to be threadsafe about the quit mutation
+      @quit = false
+      @quit_lock = Mutex.new
+      @lastwarn_max_files = 0
+      @discoverer = discoverer
+      @watched_files_collection = watched_files_collection
+    end
+
+    def add_processor(processor)
+      @processor = processor
+      @processor.add_watch(self)
+      self
+    end
+
+    def watch(path)
+      synchronized do
+        @discoverer.add_path(path)
+      end
+      # don't return whatever @discoverer.add_path returns
+      return true
+    end
+
+    def discover
+      synchronized do
+        @discoverer.discover
+      end
+      # don't return whatever @discoverer.discover returns
+      return true
+    end
+
+    def subscribe(observer, sincedb_collection)
+      @processor.initialize_handlers(sincedb_collection, observer)
+
+      glob = 0
+      interval = @settings.discover_interval
+      reset_quit
+      until quit?
+        iterate_on_state
+        break if quit?
+        glob += 1
+        if glob == interval
+          discover
+          glob = 0
+        end
+        break if quit?
+        sleep(@settings.stat_interval)
+      end
+      @watched_files_collection.close_all
+    end # def subscribe
+
+    # Read mode processor will handle watched_files in the closed, ignored, watched and active state
+    # differently from Tail mode - see the ReadMode::Processor and TailMode::Processor
+    def iterate_on_state
+      return if @watched_files_collection.empty?
+      synchronized do
+        begin
+          # creates this snapshot of watched_file values just once
+          watched_files = @watched_files_collection.values
+          @processor.process_closed(watched_files)
+          return if quit?
+          @processor.process_ignored(watched_files)
+          return if quit?
+          @processor.process_watched(watched_files)
+          return if quit?
+          @processor.process_active(watched_files)
+        ensure
+          @watched_files_collection.delete(@processor.deletable_filepaths)
+          @processor.deletable_filepaths.clear
+        end
+      end
+    end # def each
+
+    def quit
+      @quit_lock.synchronize do
+        @quit = true
+      end
+    end # def quit
+
+    def quit?
+      @quit_lock.synchronize { @quit }
+    end
+
+    private
+
+    def synchronized(&block)
+      @lock.synchronize { block.call }
+    end
+
+    def reset_quit
+      @quit_lock.synchronize { @quit = false }
+    end
+  end
+end

data/lib/filewatch/watched_file.rb

@@ -0,0 +1,226 @@
+# encoding: utf-8
+
+module FileWatch
+  class WatchedFile
+    include InodeMixin # see bootstrap.rb at `if LogStash::Environment.windows?`
+
+    attr_reader :bytes_read, :state, :file, :buffer, :recent_states
+    attr_reader :path, :filestat, :accessed_at, :modified_at, :pathname
+    attr_reader :sdb_key_v1, :last_stat_size, :listener
+    attr_accessor :last_open_warning_at
+
+    # this class represents a file that has been discovered
+    def initialize(pathname, stat, settings)
+      @settings = settings
+      @pathname = Pathname.new(pathname) # given arg pathname might be a string or a Pathname object
+      @path = @pathname.to_path
+      @bytes_read = 0
+      @last_stat_size = 0
+      # the prepare_inode method is sourced from the mixed module above
+      @sdb_key_v1 = InodeStruct.new(*prepare_inode(path, stat))
+      # initial as true means we have not associated this watched_file with a previous sincedb value yet.
+      # and we should read from the beginning if necessary
+      @initial = true
+      @recent_states = [] # keep last 8 states, managed in set_state
+      @state = :watched
+      set_stat(stat) # can change @last_stat_size
+      @listener = nil
+      @last_open_warning_at = nil
+      set_accessed_at
+    end
+
+    def set_listener(observer)
+      @listener = observer.listener_for(@path)
+    end
+
+    def unset_listener
+      @listener = nil
+    end
+
+    def has_listener?
+      !@listener.nil?
+    end
+
+    def sincedb_key
+      @sdb_key_v1
+    end
+
+    def initial_completed
+      @initial = false
+    end
+
+    def set_accessed_at
+      @accessed_at = Time.now.to_f
+    end
+
+    def initial?
+      @initial
+    end
+
+    def compressed?
+      @path.end_with?('.gz','.gzip')
+    end
+
+    def size_changed?
+      @last_stat_size != bytes_read
+    end
+
+    def all_read?
+      @last_stat_size == bytes_read
+    end
+
+    def open
+      file_add_opened(FileOpener.open(@path))
+    end
+
+    def file_add_opened(rubyfile)
+      @file = rubyfile
+      @buffer = BufferedTokenizer.new(@settings.delimiter) if @buffer.nil?
+    end
+
+    def file_close
+      return if @file.nil? || @file.closed?
+      @file.close
+      @file = nil
+    end
+
+    def file_seek(amount, whence = IO::SEEK_SET)
+      @file.sysseek(amount, whence)
+    end
+
+    def file_read(amount)
+      set_accessed_at
+      @file.sysread(amount)
+    end
+
+    def file_open?
+      !@file.nil? && !@file.closed?
+    end
+
+    def reset_buffer
+      @buffer.flush
+    end
+
+    def buffer_extract(data)
+      @buffer.extract(data)
+    end
+
+    def increment_bytes_read(delta)
+      return if delta.nil?
+      @bytes_read += delta
+    end
+
+    def update_bytes_read(total_bytes_read)
+      return if total_bytes_read.nil?
+      @bytes_read = total_bytes_read
+    end
+
+    def update_path(_path)
+      @path = _path
+    end
+
+    def update_stat(st)
+      set_stat(st)
+    end
+
+    def activate
+      set_state :active
+    end
+
+    def ignore
+      set_state :ignored
+      @bytes_read = @filestat.size
+    end
+
+    def close
+      set_state :closed
+    end
+
+    def watch
+      set_state :watched
+    end
+
+    def unwatch
+      set_state :unwatched
+    end
+
+    def active?
+      @state == :active
+    end
+
+    def ignored?
+      @state == :ignored
+    end
+
+    def closed?
+      @state == :closed
+    end
+
+    def watched?
+      @state == :watched
+    end
+
+    def unwatched?
+      @state == :unwatched
+    end
+
+    def expiry_close_enabled?
+      !@settings.close_older.nil?
+    end
+
+    def expiry_ignore_enabled?
+      !@settings.ignore_older.nil?
+    end
+
+    def shrunk?
+      @last_stat_size < @bytes_read
+    end
+
+    def grown?
+      @last_stat_size > @bytes_read
+    end
+
+    def restat
+      set_stat(pathname.stat)
+    end
+
+    def set_state(value)
+      @recent_states.shift if @recent_states.size == 8
+      @recent_states << @state
+      @state = value
+    end
+
+    def recent_state_history
+      @recent_states + Array(@state)
+    end
+
+    def file_closable?
+      file_can_close? && all_read?
+    end
+
+    def file_ignorable?
+      return false unless expiry_ignore_enabled?
+      # (Time.now - stat.mtime) <- in jruby, this does int and float
+      # conversions before the subtraction and returns a float.
+      # so use all floats upfront
+      (Time.now.to_f - @modified_at) > @settings.ignore_older
+    end
+
+    def file_can_close?
+      return false unless expiry_close_enabled?
+      (Time.now.to_f - @accessed_at) > @settings.close_older
+    end
+
+    def to_s
+      inspect
+    end
+
+    private
+
+    def set_stat(stat)
+      @modified_at = stat.mtime.to_f
+      @last_stat_size = stat.size
+      @filestat = stat
+    end
+  end
+end

data/lib/filewatch/watched_files_collection.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+module FileWatch
+  class WatchedFilesCollection
+
+    def initialize(settings)
+      @sort_by = settings.file_sort_by # "last_modified" | "path"
+      @sort_direction = settings.file_sort_direction # "asc" | "desc"
+      @sort_method = method("#{@sort_by}_#{@sort_direction}".to_sym)
+      @files = []
+      @pointers = {}
+    end
+
+    def add(watched_file)
+      @files << watched_file
+      @sort_method.call
+    end
+
+    def delete(paths)
+      Array(paths).each do |f|
+        index = @pointers.delete(f)
+        @files.delete_at(index)
+      end
+      @sort_method.call
+    end
+
+    def close_all
+      @files.each(&:file_close)
+    end
+
+    def empty?
+      @files.empty?
+    end
+
+    def keys
+      @pointers.keys
+    end
+
+    def values
+      @files
+    end
+
+    def watched_file_by_path(path)
+      index = @pointers[path]
+      return nil unless index
+      @files[index]
+    end
+
+    private
+
+    def last_modified_asc
+      @files.sort! do |left, right|
+        left.modified_at <=> right.modified_at
+      end
+      refresh_pointers
+    end
+
+    def last_modified_desc
+      @files.sort! do |left, right|
+        right.modified_at <=> left.modified_at
+      end
+      refresh_pointers
+    end
+
+    def path_asc
+      @files.sort! do |left, right|
+        left.path <=> right.path
+      end
+      refresh_pointers
+    end
+
+    def path_desc
+      @files.sort! do |left, right|
+        right.path <=> left.path
+      end
+      refresh_pointers
+    end
+
+    def refresh_pointers
+      @files.each_with_index do |watched_file, index|
+        @pointers[watched_file.path] = index
+      end
+    end
+  end
+end

data/lib/filewatch/winhelper.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+require "ffi"
+
+module Winhelper
+  extend FFI::Library
+
+  ffi_lib 'kernel32'
+  ffi_convention :stdcall
+  class FileTime < FFI::Struct
+    layout :lowDateTime, :uint,
+      :highDateTime, :uint
+  end
+
+  #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx
+  class FileInformation < FFI::Struct
+    layout :fileAttributes, :uint, #DWORD    dwFileAttributes;
+      :createTime, FileTime,  #FILETIME ftCreationTime;
+      :lastAccessTime, FileTime, #FILETIME ftLastAccessTime;
+      :lastWriteTime, FileTime, #FILETIME ftLastWriteTime;
+      :volumeSerialNumber, :uint, #DWORD    dwVolumeSerialNumber;
+      :fileSizeHigh, :uint, #DWORD    nFileSizeHigh;
+      :fileSizeLow, :uint, #DWORD    nFileSizeLow;
+      :numberOfLinks, :uint, #DWORD    nNumberOfLinks;
+      :fileIndexHigh, :uint, #DWORD    nFileIndexHigh;
+      :fileIndexLow, :uint #DWORD    nFileIndexLow;
+  end
+
+
+  #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
+  #HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_  DWORD dwDesiredAccess,_In_ DWORD dwShareMode,
+  #						_In_opt_  LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_  DWORD dwCreationDisposition,
+  #						_In_      DWORD dwFlagsAndAttributes,_In_opt_  HANDLE hTemplateFile);
+  attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
+
+  #http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx
+  #BOOL WINAPI GetFileInformationByHandle(_In_   HANDLE hFile,_Out_  LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
+  attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int
+
+  attach_function :CloseHandle, [:pointer], :int
+
+
+  def self.GetWindowsUniqueFileIdentifier(path)
+    handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil)
+    fileInfo = Winhelper::FileInformation.new
+    success = GetFileInformationByHandle(handle, fileInfo)
+    CloseHandle(handle)
+    if  success == 1
+      #args = [
+      #		fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow],
+      #		fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
+      #	]
+      #p "Information: %u %u %u %u %u %u %u " % args
+      #this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier
+      return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}"
+    else
+      #p "cannot retrieve file information, returning path"
+      return path;
+    end
+  end
+end
+
+#fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log')
+#p "FileId: " + fileId
+#p "outside function, sleeping"
+#sleep(10)