logstash-input-file 4.0.5 → 4.1.0

data/lib/logstash/inputs/delete_completed_file_handler.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+module LogStash module Inputs
+  class DeleteCompletedFileHandler
+    def handle(path)
+      Pathname.new(path).unlink rescue nil
+    end
+  end
+end end

data/lib/logstash/inputs/file.rb

@@ -6,7 +6,12 @@ require "logstash/codecs/identity_map_codec"
 require "pathname"
 require "socket" # for Socket.gethostname
 require "fileutils"
+
 require_relative "file/patch"
+require_relative "file_listener"
+require_relative "delete_completed_file_handler"
+require_relative "log_completed_file_handler"
+require "filewatch/bootstrap"
 
 # Stream events from files, normally by tailing them in a manner
 # similar to `tail -0F` but optionally reading them from the
@@ -26,8 +31,8 @@ require_relative "file/patch"
 #
 # ==== Reading from remote network volumes
 #
-# The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These 
-# remote filesystems typically have behaviors that are very different from local filesystems and 
+# The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These
+# remote filesystems typically have behaviors that are very different from local filesystems and
 # are therefore unlikely to work correctly when used with the file input.
 #
 # ==== Tracking of current position in watched files
@@ -77,8 +82,8 @@ require_relative "file/patch"
 # to the rotation and its reopening under the new name (an interval
 # determined by the `stat_interval` and `discover_interval` options)
 # will not get picked up.
-
-class LogStash::Inputs::File < LogStash::Inputs::Base
+module LogStash module Inputs
+class File < LogStash::Inputs::Base
   config_name "file"
 
   # The path(s) to the file(s) to use as an input.
@@ -144,8 +149,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
 
   # The file input closes any files that were last read the specified
   # timespan in seconds ago.
-  # This has different implications depending on if a file is being tailed or
-  # read. If tailing, and there is a large time gap in incoming data the file
+  # If tailing, and there is a large time gap in incoming data the file
   # can be closed (allowing other files to be opened) but will be queued for
   # reopening when new data is detected. If reading, the file will be closed
   # after closed_older seconds from when the last bytes were read.
@@ -160,10 +164,66 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # The default of 4095 is set in filewatch.
   config :max_open_files, :validate => :number
 
+  # What mode do you want the file input to operate in.
+  # Tail a few files or read many content-complete files
+  # The default is tail
+  # If "read" is specified then the following other settings are ignored
+  #   `start_position` (files are always read from the beginning)
+  #   `delimiter` (files are assumed to use \n or \r (or both) as line endings)
+  #   `close_older` (files are automatically 'closed' when EOF is reached)
+  # If "read" is specified then the following settings are heeded
+  #   `ignore_older` (older files are not processed)
+  # "read" mode now supports gzip file processing
+  config :mode, :validate => [ "tail", "read"], :default => "tail"
+
+  # When in 'read' mode, what action should be carried out when a file is done with.
+  # If 'delete' is specified then the file will be deleted.
+  # If 'log' is specified then the full path of the file is logged to the file specified
+  # in the `file_completed_log_path` setting.
+  config :file_completed_action, :validate => ["delete", "log", "log_and_delete"], :default => "delete"
+
+  # Which file should the completely read file paths be appended to.
+  # Only specify this path to a file when `file_completed_action` is 'log' or 'log_and_delete'.
+  # IMPORTANT: this file is appended to only - it could become very large. You are responsible for file rotation.
+  config :file_completed_log_path, :validate => :string
+
+  # The sincedb entry now has a last active timestamp associated with it.
+  # If no changes are detected in tracked files in the last N days their sincedb
+  # tracking record will expire and not be persisted.
+  # This option protects against the well known inode recycling problem. (add reference)
+  config :sincedb_clean_after, :validate => :number, :default => 14 # days
+
+  # File content is read off disk in blocks or chunks, then using whatever the set delimiter
+  # is, lines are extracted from the chunk. Specify the size in bytes of each chunk.
+  # See `file_chunk_count` to see why and when to change this from the default.
+  # The default set internally is 32768 (32KB)
+  config :file_chunk_size, :validate => :number, :default => FileWatch::FILE_READ_SIZE
+
+  # When combined with the `file_chunk_size`, this option sets how many chunks
+  # are read from each file before moving to the next active file.
+  # e.g. a `chunk_count` of 32 with the default `file_chunk_size` will process
+  # 1MB from each active file. See the option `max_open_files` for more info.
+  # The default set internally is very large, 4611686018427387903. By default
+  # the file is read to the end before moving to the next active file.
+  config :file_chunk_count, :validate => :number, :default => FileWatch::FIXNUM_MAX
+
+  # Which attribute of a discovered file should be used to sort the discovered files.
+  # Files can be sort by modified date or full path alphabetic.
+  # The default is `last_modified`
+  # Previously the processing order of the discovered files was OS dependent.
+  config :file_sort_by, :validate => ["last_modified", "path"], :default => "last_modified"
+
+  # Choose between ascending and descending order when also choosing between
+  # `last_modified` and `path` file_sort_by options.
+  # If ingesting the newest data first is important then opt for last_modified + desc
+  # If ingesting the oldest data first is important then opt for last_modified + asc
+  # If you use a special naming convention for the file full paths then
+  # perhaps path + asc will help to achieve the goal of controlling the order of file ingestion
+  config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc"
+
   public
   def register
     require "addressable/uri"
-    require "filewatch/tail"
     require "digest/md5"
     @logger.trace("Registering file input", :path => @path)
     @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
@@ -171,7 +231,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     # won't in older versions of Logstash, then we need to set it to nil.
     settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil
 
-    @tail_config = {
+    @filewatch_config = {
       :exclude => @exclude,
       :stat_interval => @stat_interval,
       :discover_interval => @discover_interval,
@@ -179,9 +239,16 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
       :delimiter => @delimiter,
       :ignore_older => @ignore_older,
       :close_older => @close_older,
-      :max_open_files => @max_open_files
+      :max_open_files => @max_open_files,
+      :sincedb_clean_after => @sincedb_clean_after,
+      :file_chunk_count => @file_chunk_count,
+      :file_chunk_size => @file_chunk_size,
+      :file_sort_by => @file_sort_by,
+      :file_sort_direction => @file_sort_direction,
     }
 
+    @completed_file_handlers = []
+
     @path.each do |path|
       if Pathname.new(path).relative?
         raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}")
@@ -189,132 +256,84 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     end
 
     if @sincedb_path.nil?
-      if settings
-        datapath = File.join(settings.get_value("path.data"), "plugins", "inputs", "file")
-        # Ensure that the filepath exists before writing, since it's deeply nested.
-        FileUtils::mkdir_p datapath
-        @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(",")))
+      base_sincedb_path = build_sincedb_base_from_settings(settings) || build_sincedb_base_from_env
+      @sincedb_path = build_random_sincedb_filename(base_sincedb_path)
+      @logger.info('No sincedb_path set, generating one based on the "path" setting', :sincedb_path => @sincedb_path.to_s, :path => @path)
+    else
+      @sincedb_path = Pathname.new(@sincedb_path)
+      if @sincedb_path.directory?
+        raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
       end
     end
 
-    # This section is going to be deprecated eventually, as path.data will be
-    # the default, not an environment variable (SINCEDB_DIR or HOME)
-    if @sincedb_path.nil? # If it is _still_ nil...
-      if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil?
-        @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \
-                      "to keep track of the files I'm watching. Either set " \
-                      "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \
-                      "in your Logstash config for the file input with " \
-                      "path '#{@path.inspect}'")
-        raise # TODO(sissel): HOW DO I FAIL PROPERLY YO
-      end
-
-      #pick SINCEDB_DIR if available, otherwise use HOME
-      sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"]
-
-      # Join by ',' to make it easy for folks to know their own sincedb
-      # generated path (vs, say, inspecting the @path array)
-      @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(",")))
-
-      # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility)
-      old_sincedb = File.join(sincedb_dir, ".sincedb")
-      if File.exists?(old_sincedb)
-        @logger.debug("Renaming old ~/.sincedb to new one", :old => old_sincedb,
-                     :new => @sincedb_path)
-        File.rename(old_sincedb, @sincedb_path)
+    @filewatch_config[:sincedb_path] = @sincedb_path
+
+    @filewatch_config[:start_new_files_at] = @start_position.to_sym
+
+    if @file_completed_action.include?('log')
+      if @file_completed_log_path.nil?
+        raise ArgumentError.new('The "file_completed_log_path" setting must be provided when the "file_completed_action" is set to "log" or "log_and_delete"')
+      else
+        @file_completed_log_path = Pathname.new(@file_completed_log_path)
+        unless @file_completed_log_path.exist?
+          begin
+            FileUtils.touch(@file_completed_log_path)
+          rescue
+            raise ArgumentError.new("The \"file_completed_log_path\" file can't be created: #{@file_completed_log_path}")
+          end
+        end
       end
-
-      @logger.info("No sincedb_path set, generating one based on the file path",
-                   :sincedb_path => @sincedb_path, :path => @path)
-    end
-
-    if File.directory?(@sincedb_path)
-      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
     end
 
-    @tail_config[:sincedb_path] = @sincedb_path
-
-    if @start_position == "beginning"
-      @tail_config[:start_new_files_at] = :beginning
+    if tail_mode?
+      @watcher_class = FileWatch::ObservingTail
+    else
+      @watcher_class = FileWatch::ObservingRead
+      if @file_completed_action.include?('log')
+        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
+      end
+      if @file_completed_action.include?('delete')
+        @completed_file_handlers << DeleteCompletedFileHandler.new
+      end
     end
-
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
   end # def register
 
-  class ListenerTail
-    # use attr_reader to define noop methods
-    attr_reader :input, :path, :data
-    attr_reader :deleted, :created, :error, :eof
-
-    # construct with upstream state
-    def initialize(path, input)
-      @path, @input = path, input
-    end
-
-    def timed_out
-      input.codec.evict(path)
-    end
-
-    def accept(data)
-      # and push transient data filled dup listener downstream
-      input.log_line_received(path, data)
-      input.codec.accept(dup_adding_state(data))
-    end
-
-    def process_event(event)
-      event.set("[@metadata][path]", path)
-      event.set("path", path) if !event.include?("path")
-      input.post_process_this(event)
-    end
-
-    def add_state(data)
-      @data = data
-      self
-    end
-
-    private
-
-    # duplicate and add state for downstream
-    def dup_adding_state(line)
-      self.class.new(path, input).add_state(line)
-    end
-  end
-
-  class FlushableListener < ListenerTail
-    attr_writer :path
-  end
-
   def listener_for(path)
     # path is the identity
-    ListenerTail.new(path, self)
+    FileListener.new(path, self)
   end
 
-  def begin_tailing
+  def start_processing
     # if the pipeline restarts this input,
     # make sure previous files are closed
     stop
-    # use observer listener api
-    @tail = FileWatch::Tail.new_observing(@tail_config)
-    @tail.logger = @logger
-    @path.each { |path| @tail.tail(path) }
+    @watcher = @watcher_class.new(@filewatch_config)
+    @path.each { |path| @watcher.watch_this(path) }
   end
 
   def run(queue)
-    begin_tailing
+    start_processing
     @queue = queue
-    @tail.subscribe(self)
+    @watcher.subscribe(self) # halts here until quit is called
     exit_flush
   end # def run
 
   def post_process_this(event)
     event.set("[@metadata][host]", @host)
-    event.set("host", @host) if !event.include?("host")
+    event.set("host", @host) unless event.include?("host")
     decorate(event)
     @queue << event
   end
 
+  def handle_deletable_path(path)
+    return if tail_mode?
+    return if @completed_file_handlers.empty?
+    @completed_file_handlers.each { |handler| handler.handle(path) }
+  end
+
   def log_line_received(path, line)
-    return if !@logger.debug?
+    return unless @logger.debug?
     @logger.debug("Received line", :path => path, :text => line)
   end
 
@@ -322,14 +341,50 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     # in filewatch >= 0.6.7, quit will closes and forget all files
     # but it will write their last read positions to since_db
     # beforehand
-    if @tail
+    if @watcher
       @codec.close
-      @tail.quit
+      @watcher.quit
     end
   end
 
   private
 
+  def build_sincedb_base_from_settings(settings)
+    logstash_data_path = settings.get_value("path.data")
+    Pathname.new(logstash_data_path).join("plugins", "inputs", "file").tap do |path|
+      # Ensure that the filepath exists before writing, since it's deeply nested.
+      path.mkpath
+    end
+  end
+
+  def build_sincedb_base_from_env
+    # This section is going to be deprecated eventually, as path.data will be
+    # the default, not an environment variable (SINCEDB_DIR or LOGSTASH_HOME)
+    if ENV["SINCEDB_DIR"].nil? && ENV["LOGSTASH_HOME"].nil?
+      @logger.error("No SINCEDB_DIR or LOGSTASH_HOME environment variable set, I don't know where " \
+                    "to keep track of the files I'm watching. Either set " \
+                    "LOGSTASH_HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \
+                    "in your Logstash config for the file input with " \
+                    "path '#{@path.inspect}'")
+      raise ArgumentError.new('The "sincedb_path" setting was not given and the environment variables "SINCEDB_DIR" or "LOGSTASH_HOME" are not set so we cannot build a file path for the sincedb')
+    end
+    Pathname.new(ENV["SINCEDB_DIR"] || ENV["LOGSTASH_HOME"])
+  end
+
+  def build_random_sincedb_filename(pathname)
+    # Join by ',' to make it easy for folks to know their own sincedb
+    # generated path (vs, say, inspecting the @path array)
+    pathname.join(".sincedb_" + Digest::MD5.hexdigest(@path.join(",")))
+  end
+
+  def tail_mode?
+    @mode == "tail"
+  end
+
+  def read_mode?
+    !tail_mode?
+  end
+
   def exit_flush
     listener = FlushableListener.new("none", self)
     if @codec.identity_count.zero?
@@ -345,4 +400,4 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
       @codec.flush_mapped(listener)
     end
   end
-end # class LogStash::Inputs::File
+end end end# class LogStash::Inputs::File

data/lib/logstash/inputs/file_listener.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+module LogStash module Inputs
+  # As and when a new WatchedFile is processed FileWatch asks for an instance of this class for the
+  # file path of that WatchedFile. All subsequent callbacks are sent via this listener instance.
+  # The file is essentially a stream and the path is the identity of that stream.
+  class FileListener
+    attr_reader :input, :path, :data
+    # construct with link back to the input plugin instance.
+    def initialize(path, input)
+      @path, @input = path, input
+      @data = nil
+    end
+
+    def opened
+    end
+
+    def eof
+    end
+
+    def error
+    end
+
+    def timed_out
+      input.codec.evict(path)
+    end
+
+    def deleted
+      input.codec.evict(path)
+      input.handle_deletable_path(path)
+    end
+
+    def accept(data)
+      # and push transient data filled dup listener downstream
+      input.log_line_received(path, data)
+      input.codec.accept(dup_adding_state(data))
+    end
+
+    def process_event(event)
+      event.set("[@metadata][path]", path)
+      event.set("path", path) unless event.include?("path")
+      input.post_process_this(event)
+    end
+
+    def add_state(data)
+      @data = data
+      self
+    end
+
+    private
+
+    # duplicate and add state for downstream
+    def dup_adding_state(line)
+      self.class.new(path, input).add_state(line)
+    end
+  end
+
+  class FlushableListener < FileListener
+    attr_writer :path
+  end
+end end

data/lib/logstash/inputs/log_completed_file_handler.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+module LogStash module Inputs
+  class LogCompletedFileHandler
+    def initialize(log_completed_file_path)
+      @log_completed_file_path = Pathname.new(log_completed_file_path)
+    end
+
+    def handle(path)
+      @log_completed_file_path.open("a") { |fd| fd.puts(path) }
+    end
+  end
+end end