logstash-input-file 4.1.3 → 4.1.4

data/lib/filewatch/watched_file.rb

@@ -2,31 +2,163 @@
 
 module FileWatch
   class WatchedFile
-    include InodeMixin # see bootstrap.rb at `if LogStash::Environment.windows?`
+    PATH_BASED_STAT = 0
+    IO_BASED_STAT = 1
 
-    attr_reader :bytes_read, :state, :file, :buffer, :recent_states
-    attr_reader :path, :filestat, :accessed_at, :modified_at, :pathname
-    attr_reader :sdb_key_v1, :last_stat_size, :listener
+    attr_reader :bytes_read, :state, :file, :buffer, :recent_states, :bytes_unread
+    attr_reader :path, :accessed_at, :modified_at, :pathname, :filename
+    attr_reader :listener, :read_loop_count, :read_chunk_size, :stat
+    attr_reader :loop_count_type, :loop_count_mode
     attr_accessor :last_open_warning_at
 
     # this class represents a file that has been discovered
+    # path based stat is taken at discovery
     def initialize(pathname, stat, settings)
       @settings = settings
       @pathname = Pathname.new(pathname) # given arg pathname might be a string or a Pathname object
       @path = @pathname.to_path
-      @bytes_read = 0
-      @last_stat_size = 0
-      # the prepare_inode method is sourced from the mixed module above
-      @sdb_key_v1 = InodeStruct.new(*prepare_inode(path, stat))
+      @filename = @pathname.basename.to_s
+      full_state_reset(stat)
+      watch
+      set_standard_read_loop
+      set_accessed_at
+    end
+
+    def no_restat_reset
+      full_state_reset(@stat)
+    end
+
+    def full_state_reset(this_stat = nil)
+      if this_stat.nil?
+        begin
+          this_stat = PathStatClass.new(pathname)
+        rescue Errno::ENOENT
+          delay_delete
+          return
+        end
+      end
+      @bytes_read = 0 # tracks bytes read from the open file or initialized from a matched sincedb_value off disk.
+      @bytes_unread = 0 # tracks bytes not yet read from the open file. So we can warn on shrink when unread bytes are seen.
+      file_close
+      set_stat(this_stat)
+      @listener = nil
+      @last_open_warning_at = nil
       # initial as true means we have not associated this watched_file with a previous sincedb value yet.
       # and we should read from the beginning if necessary
       @initial = true
       @recent_states = [] # keep last 8 states, managed in set_state
-      @state = :watched
-      set_stat(stat) # can change @last_stat_size
+      # the prepare_inode method is sourced from the mixed module above
+      watch if active? || @state.nil?
+    end
+
+    def rotate_from(other)
+      # move all state from other to this one
+      set_standard_read_loop
+      file_close
+      @bytes_read = other.bytes_read
+      @bytes_unread = other.bytes_unread
       @listener = nil
+      @initial = false
+      @recent_states = other.recent_states
+      @accessed_at = other.accessed_at
+      if !other.delayed_delete?
+        # we don't know if a file exists at the other.path yet
+        # so no reset
+        other.full_state_reset
+      end
+      set_stat PathStatClass.new(pathname)
+      ignore
+    end
+
+    def set_stat(stat)
+      @stat = stat
+      @size = @stat.size
+      @sdb_key_v1 = @stat.inode_struct
+    end
+
+    def rotate_as_initial_file
+      # rotation, when no sincedb record exists for new inode - we have never seen this inode before.
+      rotate_as_file
+      @initial = true
+    end
+
+    def rotate_as_file(bytes_read = 0)
+      # rotation, when a sincedb record exists for new inode, but no watched file to rotate from
+      # probably caused by a deletion detected in the middle of the rename cascade
+      # RARE due to delayed_delete - there would have to be a large time span between the renames.
+      @bytes_read = bytes_read # tracks bytes read from the open file or initialized from a matched sincedb_value off disk.
+      @bytes_unread = 0 # tracks bytes not yet read from the open file. So we can warn on shrink when unread bytes are seen.
       @last_open_warning_at = nil
-      set_accessed_at
+      # initial as true means we have not associated this watched_file with a previous sincedb value yet.
+      # and we should read from the beginning if necessary
+      @initial = false
+      @recent_states = [] # keep last 8 states, managed in set_state
+      set_stat(PathStatClass.new(pathname))
+      reopen
+      watch
+    end
+
+    def stat_sincedb_key
+      @stat.inode_struct
+    end
+
+    def rotation_detected?
+      stat_sincedb_key != sincedb_key
+    end
+
+    def restat
+      @stat.restat
+      if rotation_detected?
+        # switch to new state now
+        rotation_in_progress
+      else
+        @size = @stat.size
+        update_bytes_unread
+      end
+    end
+
+    def modified_at
+      @stat.modified_at
+    end
+
+    def position_for_new_sincedb_value
+      if @initial
+        # this file was found in first discovery
+        @settings.start_new_files_at == :beginning ? 0 : last_stat_size
+      else
+        # always start at the beginning if found after first discovery
+        0
+      end
+    end
+
+    def last_stat_size
+      @stat.size
+    end
+
+    def current_size
+      @size
+    end
+
+    def shrunk?
+      @size < @bytes_read
+    end
+
+    def grown?
+      @size > @bytes_read
+    end
+
+    def size_changed?
+      # called from closed and ignored
+      # before the last stat was taken file should be fully read.
+      @size != @bytes_read
+    end
+
+    def all_read?
+      @bytes_read >= @size
+    end
+
+    def file_at_path_found_again
+      restore_previous_state
     end
 
     def set_listener(observer)
@@ -61,12 +193,11 @@ module FileWatch
       @path.end_with?('.gz','.gzip')
     end
 
-    def size_changed?
-      @last_stat_size != bytes_read
-    end
-
-    def all_read?
-      @last_stat_size == bytes_read
+    def reopen
+      if file_open?
+        file_close
+        open
+      end
     end
 
     def open
@@ -88,9 +219,9 @@ module FileWatch
       @file.sysseek(amount, whence)
     end
 
-    def file_read(amount)
+    def file_read(amount = nil)
       set_accessed_at
-      @file.sysread(amount)
+      @file.sysread(amount || @read_chunk_size)
     end
 
     def file_open?
@@ -101,6 +232,13 @@ module FileWatch
       @buffer.flush
     end
 
+    def read_extract_lines(amount)
+      data = file_read(amount)
+      result = buffer_extract(data)
+      increment_bytes_read(data.bytesize)
+      result
+    end
+
     def buffer_extract(data)
       warning, additional = "", {}
       lines = @buffer.extract(data)
@@ -112,7 +250,7 @@ module FileWatch
         additional["delimiter"] = @settings.delimiter
         additional["read_position"] = @bytes_read
         additional["bytes_read_count"] = data.bytesize
-        additional["last_known_file_size"] = @last_stat_size
+        additional["last_known_file_size"] = last_stat_size
         additional["file_path"] = @path
       end
       BufferExtractResult.new(lines, warning, additional)
@@ -121,19 +259,19 @@ module FileWatch
     def increment_bytes_read(delta)
       return if delta.nil?
       @bytes_read += delta
+      update_bytes_unread
+      @bytes_read
     end
 
     def update_bytes_read(total_bytes_read)
       return if total_bytes_read.nil?
       @bytes_read = total_bytes_read
+      update_bytes_unread
+      @bytes_read
     end
 
-    def update_path(_path)
-      @path = _path
-    end
-
-    def update_stat(st)
-      set_stat(st)
+    def rotation_in_progress
+      set_state :rotation_in_progress
     end
 
     def activate
@@ -142,7 +280,11 @@ module FileWatch
 
     def ignore
       set_state :ignored
-      @bytes_read = @filestat.size
+    end
+
+    def ignore_as_unread
+      ignore
+      @bytes_read = @size
     end
 
     def close
@@ -157,10 +299,26 @@ module FileWatch
       set_state :unwatched
     end
 
+    def delay_delete
+      set_state :delayed_delete
+    end
+
+    def restore_previous_state
+      set_state @recent_states.pop
+    end
+
+    def rotation_in_progress?
+      @state == :rotation_in_progress
+    end
+
     def active?
       @state == :active
     end
 
+    def delayed_delete?
+      @state == :delayed_delete
+    end
+
     def ignored?
       @state == :ignored
     end
@@ -185,21 +343,49 @@ module FileWatch
       !@settings.ignore_older.nil?
     end
 
-    def shrunk?
-      @last_stat_size < @bytes_read
-    end
-
-    def grown?
-      @last_stat_size > @bytes_read
+    def set_standard_read_loop
+      @read_loop_count = @settings.file_chunk_count
+      @read_chunk_size = @settings.file_chunk_size
+      # e.g. 1 * 10 bytes -> 10 or 256 * 65536 -> 1677716 or 140737488355327 * 32768 -> 4611686018427355136
+      @standard_loop_max_bytes = @read_loop_count * @read_chunk_size
+    end
+
+    def set_maximum_read_loop
+      # used to quickly fully read an open file when rotation is detected
+      @read_loop_count = FileWatch::MAX_ITERATIONS
+      @read_chunk_size = FileWatch::FILE_READ_SIZE
+      @standard_loop_max_bytes = @read_loop_count * @read_chunk_size
+    end
+
+    def loop_control_adjusted_for_stat_size
+      more = false
+      to_read = current_size - @bytes_read
+      return LoopControlResult.new(0, 0, more) if to_read < 1
+      return LoopControlResult.new(1, to_read, more) if to_read < @read_chunk_size
+      # set as if to_read is greater than or equal to max_bytes
+      # use the ones from settings and don't indicate more
+      count = @read_loop_count
+      if to_read < @standard_loop_max_bytes
+        # if the defaults are used then this branch will be taken
+        # e.g. to_read is 100 and max_bytes is 4 * 30 -> 120
+        # will overrun and trigger EOF, build less iterations
+        # will generate 3 * 30 -> 90 this time and we indicate more
+        # a 2GB file in read mode will get one loop of 64666 x 32768 (2119006656 / 32768)
+        # and a second loop with 1 x 31168
+        count = to_read / @read_chunk_size
+        more = true
+      end
+      LoopControlResult.new(count, @read_chunk_size, more)
     end
 
-    def restat
-      set_stat(pathname.stat)
+    def reset_bytes_unread
+      # called from shrink
+      @bytes_unread = 0
     end
 
     def set_state(value)
       @recent_states.shift if @recent_states.size == 8
-      @recent_states << @state
+      @recent_states << @state unless @state.nil?
       @state = value
     end
 
@@ -216,7 +402,7 @@ module FileWatch
       # (Time.now - stat.mtime) <- in jruby, this does int and float
       # conversions before the subtraction and returns a float.
       # so use all floats upfront
-      (Time.now.to_f - @modified_at) > @settings.ignore_older
+      (Time.now.to_f - modified_at) > @settings.ignore_older
     end
 
     def file_can_close?
@@ -224,16 +410,26 @@ module FileWatch
       (Time.now.to_f - @accessed_at) > @settings.close_older
     end
 
+    def details
+      detail = "@filename='#{filename}', @state='#{state}', @recent_states='#{@recent_states.inspect}', "
+      detail.concat("@bytes_read='#{@bytes_read}', @bytes_unread='#{@bytes_unread}', current_size='#{current_size}', ")
+      detail.concat("last_stat_size='#{last_stat_size}', file_open?='#{file_open?}', @initial=#{@initial}")
+      "<FileWatch::WatchedFile: #{detail}, @sincedb_key='#{sincedb_key}'>"
+    end
+
+    def inspect
+      "\"<FileWatch::WatchedFile: @filename='#{filename}', @state='#{state}', @sincedb_key='#{sincedb_key}, size=#{@size}>\""
+    end
+
     def to_s
       inspect
     end
 
     private
 
-    def set_stat(stat)
-      @modified_at = stat.mtime.to_f
-      @last_stat_size = stat.size
-      @filestat = stat
+    def update_bytes_unread
+      unread = current_size - @bytes_read
+      @bytes_unread = unread < 0 ? 0 : unread
     end
   end
 end

data/lib/filewatch/watched_files_collection.rb

@@ -6,8 +6,8 @@ module FileWatch
       @sort_by = settings.file_sort_by # "last_modified" | "path"
       @sort_direction = settings.file_sort_direction # "asc" | "desc"
       @sort_method = method("#{@sort_by}_#{@sort_direction}".to_sym)
-      @files = []
-      @pointers = {}
+      @files = Concurrent::Array.new
+      @pointers = Concurrent::Hash.new
     end
 
     def add(watched_file)

data/lib/filewatch/winhelper.rb

@@ -6,59 +6,201 @@ module Winhelper
 
   ffi_lib 'kernel32'
   ffi_convention :stdcall
+
   class FileTime < FFI::Struct
-    layout :lowDateTime, :uint,
-      :highDateTime, :uint
+    layout :lowDateTime, :uint, :highDateTime, :uint
   end
 
   #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx
   class FileInformation < FFI::Struct
     layout :fileAttributes, :uint, #DWORD    dwFileAttributes;
-      :createTime, FileTime,  #FILETIME ftCreationTime;
-      :lastAccessTime, FileTime, #FILETIME ftLastAccessTime;
-      :lastWriteTime, FileTime, #FILETIME ftLastWriteTime;
-      :volumeSerialNumber, :uint, #DWORD    dwVolumeSerialNumber;
-      :fileSizeHigh, :uint, #DWORD    nFileSizeHigh;
-      :fileSizeLow, :uint, #DWORD    nFileSizeLow;
-      :numberOfLinks, :uint, #DWORD    nNumberOfLinks;
-      :fileIndexHigh, :uint, #DWORD    nFileIndexHigh;
-      :fileIndexLow, :uint #DWORD    nFileIndexLow;
+    :createTime, FileTime, #      FILETIME ftCreationTime;
+    :lastAccessTime, FileTime, #  FILETIME ftLastAccessTime;
+    :lastWriteTime, FileTime, #   FILETIME ftLastWriteTime;
+    :volumeSerialNumber, :uint, # DWORD    dwVolumeSerialNumber;
+    :fileSizeHigh, :uint, #       DWORD    nFileSizeHigh;
+    :fileSizeLow, :uint, #        DWORD    nFileSizeLow;
+    :numberOfLinks, :uint, #      DWORD    nNumberOfLinks;
+    :fileIndexHigh, :uint, #      DWORD    nFileIndexHigh;
+    :fileIndexLow, :uint #        DWORD    nFileIndexLow;
+  end
+
+  # https://msdn.microsoft.com/en-us/library/windows/desktop/hh965605(v=vs.85).aspx
+  class FileId128 < FFI::Struct
+    layout :lowPart, :ulong_long, :highPart, :ulong_long
   end
 
+  # https://msdn.microsoft.com/en-us/library/windows/desktop/hh802691(v=vs.85).aspx
+  class FileIdInfo < FFI::Struct
+    layout :volumeSerialNumber, :ulong_long, :fileId, FileId128
+    # ULONGLONG   VolumeSerialNumber;
+    # FILE_ID_128 FileId;
+  end
+
+  FileInfoEnum = enum(
+    :FileBasicInfo,
+    :FileStandardInfo,
+    :FileNameInfo,
+    :FileRenameInfo,
+    :FileDispositionInfo,
+    :FileAllocationInfo,
+    :FileEndOfFileInfo,
+    :FileStreamInfo,
+    :FileCompressionInfo,
+    :FileAttributeTagInfo,
+    :FileIdBothDirectoryInfo,
+    :FileIdBothDirectoryRestartInfo,
+    :FileIoPriorityHintInfo,
+    :FileRemoteProtocolInfo,
+    :FileFullDirectoryInfo,
+    :FileFullDirectoryRestartInfo,
+    :FileStorageInfo,
+    :FileAlignmentInfo,
+    :FileIdInfo,
+    :FileIdExtdDirectoryInfo,
+    :FileIdExtdDirectoryRestartInfo
+  )
 
   #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
-  #HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_  DWORD dwDesiredAccess,_In_ DWORD dwShareMode,
-  #						_In_opt_  LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_  DWORD dwCreationDisposition,
-  #						_In_      DWORD dwFlagsAndAttributes,_In_opt_  HANDLE hTemplateFile);
-  attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
+  #HANDLE WINAPI CreateFile(
+  # _In_      LPCTSTR lpFileName,
+  # _In_      DWORD dwDesiredAccess,
+  # _In_      DWORD dwShareMode,
+  # _In_opt_  LPSECURITY_ATTRIBUTES lpSecurityAttributes,
+  # _In_      DWORD dwCreationDisposition,
+  # _In_      DWORD dwFlagsAndAttributes, _In_opt_  HANDLE hTemplateFile);
+  attach_function :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
+  attach_function :CreateFileW, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer
 
   #http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx
-  #BOOL WINAPI GetFileInformationByHandle(_In_   HANDLE hFile,_Out_  LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
+  #BOOL WINAPI GetFileInformationByHandle(
+  # _In_   HANDLE hFile,
+  # _Out_  LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
   attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int
 
+  #https://msdn.microsoft.com/en-us/library/windows/desktop/aa364953(v=vs.85).aspx
+  #BOOL WINAPI GetFileInformationByHandleEx(
+  # _In_  HANDLE hFile,
+  # _In_  FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+  # _Out_ LPVOID lpFileInformation,
+  # _In_  DWORD  dwBufferSize );
+  attach_function :GetFileInformationByHandleEx, [:pointer, FileInfoEnum, :pointer, :uint], :uint
+
   attach_function :CloseHandle, [:pointer], :int
 
+  #https://msdn.microsoft.com/en-us/library/windows/desktop/aa964920(v=vs.85).aspx
+  #BOOL WINAPI GetVolumeInformationByHandleW(
+  # _In_ HANDLE hFile,
+  # _Out_opt_ LPWSTR lpVolumeNameBuffer,
+  # _In_      DWORD   nVolumeNameSize,
+  # _Out_opt_ LPDWORD lpVolumeSerialNumber,
+  # _Out_opt_ LPDWORD lpMaximumComponentLength,
+  # _Out_opt_ LPDWORD lpFileSystemFlags,
+  # _Out_opt_ LPWSTR lpFileSystemNameBuffer,
+  # _In_      DWORD nFileSystemNameSize);
+  attach_function :GetVolumeInformationByHandleW, [:pointer, :pointer, :uint, :pointer, :pointer, :pointer, :pointer, :uint], :int
+
+  def self.file_system_type_from_path(path)
+    file_system_type_from_handle(open_handle_from_path(path))
+  end
+
+  def self.file_system_type_from_io(io)
+    FileWatch::FileExt.io_handle(io) do |pointer|
+      file_system_type_from_handle(pointer, false)
+    end
+  end
 
-  def self.GetWindowsUniqueFileIdentifier(path)
-    handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil)
+  def self.file_system_type_from_handle(handle, close_handle = true)
+    out = FFI::MemoryPointer.new(:char, 256, true)
+    if GetVolumeInformationByHandleW(handle, nil, 0, nil, nil, nil, out, 256) > 0
+      char_pointer_to_ruby_string(out)
+    else
+      "unknown"
+    end
+  ensure
+    CloseHandle(handle) if close_handle
+  end
+
+  def self.identifier_from_io(io)
+    FileWatch::FileExt.io_handle(io) do |pointer|
+      identifier_from_handle(pointer, false)
+    end
+  end
+
+  def self.identifier_from_path(path)
+    identifier_from_handle(open_handle_from_path(path))
+  end
+
+  def self.identifier_from_path_ex(path)
+    identifier_from_handle_ex(open_handle_from_path(path))
+  end
+
+  def self.identifier_from_io_ex(io)
+    FileWatch::FileExt.io_handle(io) do |pointer|
+      identifier_from_handle_ex(pointer, false)
+    end
+  end
+
+  def self.identifier_from_handle_ex(handle, close_handle = true)
+    fileIdInfo = Winhelper::FileIdInfo.new
+    success = GetFileInformationByHandleEx(handle, :FileIdInfo, fileIdInfo, fileIdInfo.size)
+    if success > 0
+      vsn   = fileIdInfo[:volumeSerialNumber]
+      lpfid = fileIdInfo[:fileId][:lowPart]
+      hpfid = fileIdInfo[:fileId][:highPart]
+      return "#{vsn}-#{lpfid}-#{hpfid}"
+    else
+      return 'unknown'
+    end
+  ensure
+    CloseHandle(handle) if close_handle
+  end
+
+  def self.identifier_from_handle(handle, close_handle = true)
     fileInfo = Winhelper::FileInformation.new
     success = GetFileInformationByHandle(handle, fileInfo)
-    CloseHandle(handle)
-    if  success == 1
+    if  success > 0
       #args = [
-      #		fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow],
-      #		fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
-      #	]
+      #   fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow],
+      #   fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow]
+      # ]
       #p "Information: %u %u %u %u %u %u %u " % args
       #this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier
       return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}"
     else
-      #p "cannot retrieve file information, returning path"
-      return path;
+      return 'unknown'
     end
+  ensure
+    CloseHandle(handle) if close_handle
+  end
+
+  private
+
+  def self.open_handle_from_path(path)
+    CreateFileW(in_buffer(path), 0, 7, nil, 3, 128, nil)
+  end
+
+  def self.in_buffer(string)
+    utf16le(string)
+  end
+
+  def self.char_pointer_to_ruby_string(char_pointer, length = 256)
+    bytes = char_pointer.get_array_of_uchar(0, length)
+    ignore = bytes.reverse.index{|b| b != 0} - 1
+    our_bytes = bytes[0, bytes.length - ignore]
+    our_bytes.pack("C*").force_encoding("UTF-16LE").encode("UTF-8")
+  end
+
+  def self.utf16le(string)
+    string.encode("UTF-16LE")
+  end
+
+  def self.win1252(string)
+    string.encode("Windows-1252")
   end
 end
 
+
 #fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log')
 #p "FileId: " + fileId
 #p "outside function, sleeping"