logstash-input-file 4.0.5 → 4.1.0

data/lib/filewatch/discoverer.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module FileWatch
+  class Discoverer
+    # given a path or glob will prepare for and discover files to watch
+    # if they are not excluded or ignorable
+    # they are added to the watched_files collection and
+    # associated with a sincedb entry if one can be found
+    include LogStash::Util::Loggable
+
+    def initialize(watched_files_collection, sincedb_collection, settings)
+      @watching = []
+      @exclude = []
+      @watched_files_collection = watched_files_collection
+      @sincedb_collection = sincedb_collection
+      @settings = settings
+      @settings.exclude.each { |p| @exclude << p }
+    end
+
+    def add_path(path)
+      return if @watching.member?(path)
+      @watching << path
+      discover_files(path)
+      self
+    end
+
+    def discover
+      @watching.each do |path|
+        discover_files(path)
+      end
+    end
+
+    private
+
+    def can_exclude?(watched_file, new_discovery)
+      @exclude.each do |pattern|
+        if watched_file.pathname.fnmatch?(pattern)
+          if new_discovery
+            logger.debug("Discoverer can_exclude?: #{watched_file.path}: skipping " +
+              "because it matches exclude #{pattern}")
+          end
+          watched_file.unwatch
+          return true
+        end
+      end
+      false
+    end
+
+    def discover_files(path)
+      globbed = Dir.glob(path)
+      globbed = [path] if globbed.empty?
+      logger.debug("Discoverer found files, count: #{globbed.size}")
+      globbed.each do |file|
+        logger.debug("Discoverer found file, path: #{file}")
+        pathname = Pathname.new(file)
+        next unless pathname.file?
+        next if pathname.symlink?
+        new_discovery = false
+        watched_file = @watched_files_collection.watched_file_by_path(file)
+        if watched_file.nil?
+          logger.debug("Discoverer discover_files: #{path}: new: #{file} (exclude is #{@exclude.inspect})")
+          new_discovery = true
+          watched_file = WatchedFile.new(pathname, pathname.stat, @settings)
+        end
+        # if it already unwatched or its excluded then we can skip
+        next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery)
+
+        if new_discovery
+          if watched_file.file_ignorable?
+            logger.debug("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago")
+            # on discovery ignorable watched_files are put into the ignored state and that
+            # updates the size from the internal stat
+            # so the existing contents are not read.
+            # because, normally, a newly discovered file will
+            # have a watched_file size of zero
+            # they are still added to the collection so we know they are there for the next periodic discovery
+            watched_file.ignore
+          end
+          # now add the discovered file to the watched_files collection and adjust the sincedb collections
+          @watched_files_collection.add(watched_file)
+          # initially when the sincedb collection is filled with records from the persistence file
+          # each value is not associated with a watched file
+          # a sincedb_value can be:
+          #   unassociated
+          #   associated with this watched_file
+          #   associated with a different watched_file
+          @sincedb_collection.associate(watched_file)
+        end
+        # at this point the watched file is created, is in the db but not yet opened or being processed
+      end
+    end
+  end
+end

data/lib/filewatch/helper.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+# code downloaded from Ruby on Rails 4.2.1
+# https://raw.githubusercontent.com/rails/rails/v4.2.1/activesupport/lib/active_support/core_ext/file/atomic.rb
+# change method name to avoid borking active_support and vice versa
+require 'fileutils'
+
+module FileHelper
+  extend self
+  # Write to a file atomically. Useful for situations where you don't
+  # want other processes or threads to see half-written files.
+  #
+  #   File.write_atomically('important.file') do |file|
+  #     file.write('hello')
+  #   end
+  def write_atomically(file_name)
+
+    if File.exist?(file_name)
+      # Get original file permissions
+      old_stat = File.stat(file_name)
+    else
+      # If not possible, probe which are the default permissions in the
+      # destination directory.
+      old_stat = probe_stat_in(File.dirname(file_name))
+    end
+
+    mode = old_stat ? old_stat.mode : nil
+
+    # Create temporary file with identical permissions
+    temp_file = File.new(rand_filename(file_name), "w", mode)
+    temp_file.binmode
+    return_val = yield temp_file
+    temp_file.close
+
+    # Overwrite original file with temp file
+    File.rename(temp_file.path, file_name)
+
+    # Unable to get permissions of the original file => return
+    return return_val if old_stat.nil?
+
+    # Set correct uid/gid on new file
+    File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat
+
+    return_val
+  end
+
+  def device?(file_name)
+    File.chardev?(file_name) || File.blockdev?(file_name)
+  end
+
+  # Private utility method.
+  def probe_stat_in(dir) #:nodoc:
+    basename = rand_filename(".permissions_check")
+    file_name = File.join(dir, basename)
+    FileUtils.touch(file_name)
+    File.stat(file_name)
+  rescue
+    # ...
+  ensure
+    FileUtils.rm_f(file_name) if File.exist?(file_name)
+  end
+
+  def rand_filename(prefix)
+    [ prefix, Thread.current.object_id, Process.pid, rand(1000000) ].join('.')
+  end
+end

data/lib/filewatch/observing_base.rb

@@ -0,0 +1,97 @@
+# encoding: utf-8
+
+## Interface API topology
+# ObservingBase module (this file)
+#   is a module mixin proving common constructor and external API for File Input Plugin interaction
+#   calls build_specific_processor on ObservingRead or ObservingTail
+# ObservingRead and ObservingTail
+#   provides the External API method subscribe(observer = NullObserver.new)
+#   build_specific_processor(settings) - provide a Tail or Read specific Processor.
+# TailMode::Processor or ReadMode::Processor
+#   initialize_handlers(sincedb_collection, observer) - called when the observer subscribes to changes in a Mode,
+#      builds mode specific handler instances with references to the observer
+#   process_closed(watched_files) - provide specific processing of watched_files in the closed state
+#   process_ignored(watched_files) - provide specific processing of watched_files in the ignored state
+#   process_watched(watched_files) - provide specific processing of watched_files in the watched state
+#   process_active(watched_files)  - provide specific processing of watched_files in the active state
+#      These methods can call "handler" methods that delegate to the specific Handler classes.
+# TailMode::Handlers module namespace
+#   contains the Handler classes that deals with Tail mode file lifecycle "events".
+#   The TailMode::Handlers::Base
+#      handle(watched_file) - this method calls handle_specifically defined in a subclass
+#      handle_specifically(watched_file) - this is a noop method
+#      update_existing_specifically(watched_file, sincedb_value) - this is a noop method
+#   Each handler extends the Base class to provide specific implementations of these two methods:
+#      handle_specifically(watched_file)
+#      update_existing_specifically(watched_file, sincedb_value)
+# ReadMode::Handlers module namespace
+#   contains the Handler classes that deals with Read mode file lifecycle "events".
+#   The ReadMode::Handlers::Base
+#      handle(watched_file) - this method calls handle_specifically defined in a subclass
+#      handle_specifically(watched_file) - this is a noop method
+#   Each handler extends the Base class to provide specific implementations of this method:
+#      handle_specifically(watched_file)
+
+module FileWatch
+  module ObservingBase
+    attr_reader :watch, :sincedb_collection, :settings
+
+    def initialize(opts={})
+      options = {
+        :sincedb_write_interval => 10,
+        :stat_interval => 1,
+        :discover_interval => 5,
+        :exclude => [],
+        :start_new_files_at => :end,
+        :delimiter => "\n",
+        :file_chunk_count => FIXNUM_MAX,
+        :file_sort_by => "last_modified",
+        :file_sort_direction => "asc",
+      }.merge(opts)
+      unless options.include?(:sincedb_path)
+        raise NoSinceDBPathGiven.new("No sincedb_path set in options. This should have been added in the main LogStash::Inputs::File class")
+      end
+      @settings = Settings.from_options(options)
+      build_watch_and_dependencies
+    end
+
+    def build_watch_and_dependencies
+      logger.info("START, creating Discoverer, Watch with file and sincedb collections")
+      watched_files_collection = WatchedFilesCollection.new(@settings)
+      @sincedb_collection = SincedbCollection.new(@settings)
+      @sincedb_collection.open
+      discoverer = Discoverer.new(watched_files_collection, @sincedb_collection, @settings)
+      @watch = Watch.new(discoverer, watched_files_collection, @settings)
+      @watch.add_processor build_specific_processor(@settings)
+    end
+
+    def watch_this(path)
+      @watch.watch(path)
+    end
+
+    def sincedb_write(reason=nil)
+      # can be invoked from the file input
+      @sincedb_collection.write(reason)
+    end
+
+    # quit is a sort-of finalizer,
+    # it should be called for clean up
+    # before the instance is disposed of.
+    def quit
+      logger.info("QUIT - closing all files and shutting down.")
+      @watch.quit # <-- should close all the files
+      # sincedb_write("shutting down")
+    end
+
+    # close_file(path) is to be used by external code
+    # when it knows that it is completely done with a file.
+    # Other files or folders may still be being watched.
+    # Caution, once unwatched, a file can't be watched again
+    # unless a new instance of this class begins watching again.
+    # The sysadmin should rename, move or delete the file.
+    def close_file(path)
+      @watch.unwatch(path)
+      sincedb_write
+    end
+  end
+end

data/lib/filewatch/observing_read.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require_relative "read_mode/processor"
+
+module FileWatch
+  class ObservingRead
+    include LogStash::Util::Loggable
+    include ObservingBase
+
+    def subscribe(observer)
+      # observer here is the file input
+      watch.subscribe(observer, sincedb_collection)
+      sincedb_collection.write("read mode subscribe complete - shutting down")
+    end
+
+    private
+
+    def build_specific_processor(settings)
+      ReadMode::Processor.new(settings)
+    end
+
+  end
+end

data/lib/filewatch/observing_tail.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require_relative 'tail_mode/processor'
+
+module FileWatch
+  class ObservingTail
+    include LogStash::Util::Loggable
+    include ObservingBase
+
+    def subscribe(observer)
+      # observer here is the file input
+      watch.subscribe(observer, sincedb_collection)
+      sincedb_collection.write("tail mode subscribe complete - shutting down")
+    end
+
+    private
+
+    def build_specific_processor(settings)
+      TailMode::Processor.new(settings)
+    end
+  end
+end

data/lib/filewatch/read_mode/handlers/base.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module FileWatch module ReadMode module Handlers
+  class Base
+    include LogStash::Util::Loggable
+
+    attr_reader :sincedb_collection
+
+    def initialize(sincedb_collection, observer, settings)
+      @settings = settings
+      @sincedb_collection = sincedb_collection
+      @observer = observer
+    end
+
+    def handle(watched_file)
+      logger.debug("handling: #{watched_file.path}")
+      unless watched_file.has_listener?
+        watched_file.set_listener(@observer)
+      end
+      handle_specifically(watched_file)
+    end
+
+    def handle_specifically(watched_file)
+      # some handlers don't need to define this method
+    end
+
+    private
+
+    def open_file(watched_file)
+      return true if watched_file.file_open?
+      logger.debug("opening #{watched_file.path}")
+      begin
+        watched_file.open
+      rescue
+        # don't emit this message too often. if a file that we can't
+        # read is changing a lot, we'll try to open it more often, and spam the logs.
+        now = Time.now.to_i
+        logger.warn("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
+        if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
+          logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
+          watched_file.last_open_warning_at = now
+        else
+          logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+        end
+        watched_file.watch # set it back to watch so we can try it again
+      end
+      if watched_file.file_open?
+        watched_file.listener.opened
+        true
+      else
+        false
+      end
+    end
+
+    def add_or_update_sincedb_collection(watched_file)
+      sincedb_value = @sincedb_collection.find(watched_file)
+      if sincedb_value.nil?
+        add_new_value_sincedb_collection(watched_file)
+      elsif sincedb_value.watched_file == watched_file
+        update_existing_sincedb_collection_value(watched_file, sincedb_value)
+      else
+        logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer")
+      end
+      watched_file.initial_completed
+    end
+
+    def update_existing_sincedb_collection_value(watched_file, sincedb_value)
+      logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      # sincedb_value is the source of truth
+      watched_file.update_bytes_read(sincedb_value.position)
+    end
+
+    def add_new_value_sincedb_collection(watched_file)
+      sincedb_value = SincedbValue.new(0)
+      sincedb_value.set_watched_file(watched_file)
+      logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position)
+      sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
+    end
+  end
+end end end

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+module FileWatch module ReadMode module Handlers
+  class ReadFile < Base
+    def handle_specifically(watched_file)
+      if open_file(watched_file)
+        add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+        # if the `file_chunk_count` * `file_chunk_size` is less than the file size
+        # then this method will be executed multiple times
+        # and the seek is moved to just after a line boundary as recorded in the sincedb
+        # for each run - so we reset the buffer
+        watched_file.reset_buffer
+        watched_file.file_seek(watched_file.bytes_read)
+        changed = false
+        @settings.file_chunk_count.times do
+          begin
+            lines = watched_file.buffer_extract(watched_file.file_read(@settings.file_chunk_size))
+            logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty?
+            changed = true
+            lines.each do |line|
+              watched_file.listener.accept(line)
+              sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
+            end
+          rescue EOFError
+            # flush the buffer now in case there is no final delimiter
+            line = watched_file.buffer.flush
+            watched_file.listener.accept(line) unless line.empty?
+            watched_file.listener.eof
+            watched_file.file_close
+            sincedb_collection.unset_watched_file(watched_file)
+            watched_file.listener.deleted
+            watched_file.unwatch
+            break
+          rescue Errno::EWOULDBLOCK, Errno::EINTR
+            watched_file.listener.error
+            break
+          rescue => e
+            logger.error("read_to_eof: general error reading #{watched_file.path} - error: #{e.inspect}")
+            watched_file.listener.error
+            break
+          end
+        end
+        sincedb_collection.request_disk_flush if changed
+      end
+    end
+  end
+end end end

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require 'java'
+java_import java.io.InputStream
+java_import java.io.InputStreamReader
+java_import java.io.FileInputStream
+java_import java.io.BufferedReader
+java_import java.util.zip.GZIPInputStream
+java_import java.util.zip.ZipException
+
+module FileWatch module ReadMode module Handlers
+  class ReadZipFile < Base
+    def handle_specifically(watched_file)
+      add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+      # can't really stripe read a zip file, its all or nothing.
+      watched_file.listener.opened
+      # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading)
+      # should we track lines read in the sincedb and
+      # fast forward through the lines until we reach unseen content?
+      # meaning that we can quit in the middle of a zip file
+      begin
+        file_stream = FileInputStream.new(watched_file.path)
+        gzip_stream = GZIPInputStream.new(file_stream)
+        decoder = InputStreamReader.new(gzip_stream, "UTF-8")
+        buffered = BufferedReader.new(decoder)
+        while (line = buffered.readLine(false))
+          watched_file.listener.accept(line)
+        end
+        watched_file.listener.eof
+      rescue ZipException => e
+        logger.error("Cannot decompress the gzip file at path: #{watched_file.path}")
+        watched_file.listener.error
+      else
+        sincedb_collection.store_last_read(watched_file.sincedb_key, watched_file.last_stat_size)
+        sincedb_collection.request_disk_flush
+        watched_file.listener.deleted
+        watched_file.unwatch
+      ensure
+        # rescue each close individually so all close attempts are tried
+        close_and_ignore_ioexception(buffered) unless buffered.nil?
+        close_and_ignore_ioexception(decoder) unless decoder.nil?
+        close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil?
+        close_and_ignore_ioexception(file_stream) unless file_stream.nil?
+      end
+      sincedb_collection.unset_watched_file(watched_file)
+    end
+
+    private
+
+    def close_and_ignore_ioexception(closeable)
+      begin
+        closeable.close
+      rescue Exception => e # IOException can be thrown by any of the Java classes that implement the Closable interface.
+        logger.warn("Ignoring an IOException when closing an instance of #{closeable.class.name}", "exception" => e)
+      end
+    end
+  end
+end end end