logstash-input-file_progress 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2412e35c82a969c7c76ff488f1f703c33b266219
-  data.tar.gz: 69285a889b89e8272172d7896a4038d7cdea3d62
+  metadata.gz: 5a98b141a82a617952182b9043cba8dc852c32b0
+  data.tar.gz: fda790b78cd799b3e3414f98fd3995e9a571937f
 SHA512:
-  metadata.gz: 5963e270c4920e6fa0f87883b3ebdbd16ce320ec893a4ff39cdcb66777880941d94ffdda452124f5a1f8cdf876865fd7a0bd0b6dcb086e3e3352b980a236ae22
-  data.tar.gz: d2076be3d2a3778e8e852e9865a8261f3e230968dfae0c1d02d9385b483a96322cb0fd7b0d65dac25cafef0c6558a31548c1f6b1b2af9679e63ebf33ea5bf37e
+  metadata.gz: 66a14a04273111a463ada9617f2729b3546e983fcf83f7ed3f520dcfc0ab1ef42afea12dac5271370e7d585f5ecf2d240fe38dbf81ee3c82cfe10371267cce84
+  data.tar.gz: 6d6a255e26ff38daca119bab8f094b57d8d8bb57d2bd6df75034653b4ccabac486701ef10a42fbeb4d18a9f6ea05ac340fa3609610509a6c17d7c9855c4f4845

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.1.0
+ - Changed dependency to new logstash-core-plugin-api
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/lib/logstash/inputs/file_progress.rb

@@ -1,97 +1,198 @@
 # encoding: utf-8
-
-require "logstash/inputs/base"
 require "logstash/namespace"
+require "logstash/inputs/base"
+require "logstash/codecs/identity_map_codec"
+require "logstash/codecs/plain"
 
 require "pathname"
 require "socket" # for Socket.gethostname
 
-# Stream events from files.
+# Stream events from files, normally by tailing them in a manner
+# similar to `tail -0F` but optionally reading them from the
+# beginning.
+#
+# By default, each event is assumed to be one line. If you would like
+# to join multiple log lines into one event, you'll want to use the
+# multiline codec or filter.
+#
+# The plugin aims to track changing files and emit new content as it's
+# appended to each file. It's not well-suited for reading a file from
+# beginning to end and storing all of it in a single event (not even
+# with the multiline codec or filter).
+#
+# ==== Tracking of current position in watched files
+#
+# The plugin keeps track of the current position in each file by
+# recording it in a separate file named sincedb. This makes it
+# possible to stop and restart Logstash and have it pick up where it
+# left off without missing the lines that were added to the file while
+# Logstash was stopped.
+#
+# By default, the sincedb file is placed in the home directory of the
+# user running Logstash with a filename based on the filename patterns
+# being watched (i.e. the `path` option). Thus, changing the filename
+# patterns will result in a new sincedb file being used and any
+# existing current position state will be lost. If you change your
+# patterns with any frequency it might make sense to explicitly choose
+# a sincedb path with the `sincedb_path` option.
 #
-# By default, each event is assumed to be one line. If you
-# want to join lines, you'll want to use the multiline filter.
+# A different `sincedb_path` must be used for each input. Using the same
+# path will cause issues. The read checkpoints for each input must be
+# stored in a different path so the information does not override.
 #
-# Files are followed in a manner similar to "tail -0F". File rotation
-# is detected and handled by this input.
+# Sincedb files are text files with four columns:
 #
-# In addition to 'normal' file input we add events with sincedb-data to the pipe.
-# This can be later used to serve progress date with e.g. faye-output
+# . The inode number (or equivalent).
+# . The major device number of the file system (or equivalent).
+# . The minor device number of the file system (or equivalent).
+# . The current byte offset within the file.
+#
+# On non-Windows systems you can obtain the inode number of a file
+# with e.g. `ls -li`.
+#
+# ==== File rotation
+#
+# File rotation is detected and handled by this input, regardless of
+# whether the file is rotated via a rename or a copy operation. To
+# support programs that write to the rotated file for some time after
+# the rotation has taken place, include both the original filename and
+# the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in
+# the filename patterns to watch (the `path` option). Note that the
+# rotated filename will be treated as a new file so if
+# `start_position` is set to 'beginning' the rotated file will be
+# reprocessed.
+#
+# With the default value of `start_position` ('end') any messages
+# written to the end of the file between the last read operation prior
+# to the rotation and its reopening under the new name (an interval
+# determined by the `stat_interval` and `discover_interval` options)
+# will not get picked up.
+
+class LogStash::Codecs::Base
+  # TODO - move this to core
+  if !method_defined?(:accept)
+    def accept(listener)
+      decode(listener.data) do |event|
+        listener.process_event(event)
+      end
+    end
+  end
+  if !method_defined?(:auto_flush)
+    def auto_flush(*)
+    end
+  end
+end
+
 class LogStash::Inputs::FileProgress < LogStash::Inputs::Base
   config_name "file_progress"
 
-  # TODO(sissel): This should switch to use the 'line' codec by default
-  # once file following
-  default :codec, "line"
-
-  # The path to the file to use as an input.
-  # You can use globs here, such as `/var/log/*.log`
+  # The path(s) to the file(s) to use as an input.
+  # You can use filename patterns here, such as `/var/log/*.log`.
+  # If you use a pattern like `/var/log/**/*.log`, a recursive search
+  # of `/var/log` will be done for all `*.log` files.
   # Paths must be absolute and cannot be relative.
+  #
+  # You may also configure multiple paths. See an example
+  # on the <<array,Logstash configuration page>>.
   config :path, :validate => :array, :required => true
 
-  # Exclusions (matched against the filename, not full path). Globs
-  # are valid here, too. For example, if you have
-  #
+  # Exclusions (matched against the filename, not full path). Filename
+  # patterns are valid here, too. For example, if you have
+  # [source,ruby]
   #     path => "/var/log/*"
   #
-  # you might want to exclude gzipped files:
-  #
+  # You might want to exclude gzipped files:
+  # [source,ruby]
   #     exclude => "*.gz"
   config :exclude, :validate => :array
 
-  # How often we stat files to see if they have been modified. Increasing
-  # this interval will decrease the number of system calls we make, but
-  # increase the time to detect new log lines.
+  # How often (in seconds) we stat files to see if they have been modified.
+  # Increasing this interval will decrease the number of system calls we make,
+  # but increase the time to detect new log lines.
   config :stat_interval, :validate => :number, :default => 1
 
-  # How often we expand globs to discover new files to watch.
+  # How often (in seconds) we expand the filename patterns in the
+  # `path` option to discover new files to watch.
   config :discover_interval, :validate => :number, :default => 15
 
-  # Where to write the since database (keeps track of the current
-  # position of monitored log files). The default will write
-  # sincedb files to some path matching "$HOME/.sincedb*"
-  config :sincedb_path, :validate => :string, :required => true
+  # Path of the sincedb database file (keeps track of the current
+  # position of monitored log files) that will be written to disk.
+  # The default will write sincedb files to some path matching `$HOME/.sincedb*`
+  # NOTE: it must be a file path and not a directory path
+  config :sincedb_path, :validate => :string
 
-  # How often to write a since database with the current position of
+  # How often (in seconds) to write a since database with the current position of
   # monitored log files.
   config :sincedb_write_interval, :validate => :number, :default => 15
 
-  # Choose where logstash starts initially reading files - at the beginning or
+  # TODO edit description
+  # Choose where Logstash starts initially reading files: at the beginning or
   # at the end. The default behavior treats files like live streams and thus
   # starts at the end. If you have old data you want to import, set this
-  # to 'beginning'
+  # to 'beginning'.
   #
-  # This option only modifieds "first contact" situations where a file is new
-  # and not seen before. If a file has already been seen before, this option
-  # has no effect.
+  # This option only modifies "first contact" situations where a file
+  # is new and not seen before, i.e. files that don't have a current
+  # position recorded in a sincedb file read by Logstash. If a file
+  # has already been seen before, this option has no effect and the
+  # position recorded in the sincedb file will be used.
   config :start_position, :validate => [ "beginning", "end"], :default => "beginning"
 
-  # Should the progressdb events be send to the pipeline
-  config :progressdb, :validate => :boolean, :default => false
+  # set the new line delimiter, defaults to "\n"
+  config :delimiter, :validate => :string, :default => "\n"
+
+  # When the file input discovers a file that was last modified
+  # before the specified timespan in seconds, the file is ignored.
+  # After it's discovery, if an ignored file is modified it is no
+  # longer ignored and any new data is read. The default is 1 year.
+  config :ignore_older, :validate => :number, :default => 24 * 60 * 60 * 365
 
-  # Should the processdb entry be deleted after file-deletion
-  config :progressdb_del, :validate => :boolean, :default => false
+  # The file input closes any files that were last read the specified
+  # timespan in seconds ago.
+  # This has different implications depending on if a file is being tailed or
+  # read. If tailing, and there is a large time gap in incoming data the file
+  # can be closed (allowing other files to be opened) but will be queued for
+  # reopening when new data is detected. If reading, the file will be closed
+  # after closed_older seconds from when the last bytes were read.
+  # The default is 1 hour
+  config :close_older, :validate => :number, :default => 1 * 60 * 60
 
-  # Close the file when end is reached
+  # What is the maximum number of file_handles that this input consumes
+  # at any one time. Use close_older to close some files if you need to
+  # process more files than this number. This should not be set to the
+  # maximum the OS can do because file handles are needed for other
+  # LS plugins and OS processes.
+  # The default of 4095 is set in filewatch.
+  config :max_open_files, :validate => :number
+
+  # Close the file when end of file is reached
   # This make sense when reading a file once from the beginning and want to e.g.
   # proceed renaming or deleting the parent folder
-  config :eof_close, :validate => :boolean, :default => false
+  config :eof_close, :validate => :boolean, :default => true
+
+  # How often (in milliseconds) to add progress-info to metadata with the current position of
+  # monitored log files.
+  config :progress_write_interval, :validate => :number, :default => 200
 
   public
   def register
     require "addressable/uri"
-    require "filewatch/ext/filetail"
+    require "filewatch/ext/tail"
     require "digest/md5"
     @logger.info("Registering file input", :path => @path)
+    @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
 
     @tail_config = {
       :exclude => @exclude,
       :stat_interval => @stat_interval,
       :discover_interval => @discover_interval,
       :sincedb_write_interval => @sincedb_write_interval,
-      :logger => @logger,
-      :progressdb => @progressdb,
-      :progressdb_del => @progressdb_del,
-      :eof_close => @eof_close,
+      :delimiter => @delimiter,
+      :ignore_older => @ignore_older,
+      :close_older => @close_older,
+      :max_open_files => @max_open_files,
+      :eof_close => true,
+      :progress_write_interval => @progress_write_interval
     }
 
     @path.each do |path|
@@ -100,52 +201,153 @@ class LogStash::Inputs::FileProgress < LogStash::Inputs::Base
       end
     end
 
+    if @sincedb_path.nil?
+      if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil?
+        @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \
+                      "to keep track of the files I'm watching. Either set " \
+                      "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \
+                      "in your Logstash config for the file input with " \
+                      "path '#{@path.inspect}'")
+        raise # TODO(sissel): HOW DO I FAIL PROPERLY YO
+      end
+
+      #pick SINCEDB_DIR if available, otherwise use HOME
+      sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"]
+
+      # Join by ',' to make it easy for folks to know their own sincedb
+      # generated path (vs, say, inspecting the @path array)
+      @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(",")))
+
+      # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility)
+      old_sincedb = File.join(sincedb_dir, ".sincedb")
+      if File.exists?(old_sincedb)
+        @logger.info("Renaming old ~/.sincedb to new one", :old => old_sincedb,
+                     :new => @sincedb_path)
+        File.rename(old_sincedb, @sincedb_path)
+      end
+
+      @logger.info("No sincedb_path set, generating one based on the file path",
+                   :sincedb_path => @sincedb_path, :path => @path)
+    end
+
+    if File.directory?(@sincedb_path)
+      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
+    end
+
     @tail_config[:sincedb_path] = @sincedb_path
 
     if @start_position == "beginning"
       @tail_config[:start_new_files_at] = :beginning
     end
 
-    @codec_plain = LogStash::Codecs::Plain.new
+    @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
   end # def register
 
-  public
-  def run(queue)
-    @tail = FileWatch::Ext::FileTail.new(@tail_config)
+  class ListenerTail
+    # use attr_reader to define noop methods
+    attr_reader :input, :path, :data, :size, :pos
+    attr_reader :deleted, :created, :error, :eof
+
+    # construct with upstream state
+    def initialize(path, input)
+      @path, @input = path, input
+    end
+
+    def timed_out
+      input.codec.evict(path)
+    end
+
+    def accept(data, size, pos)
+      # and push transient data filled dup listener downstream
+      input.log_line_received(path, data)
+      input.codec.accept(dup_adding_state(data, size, pos))
+    end
+
+    def process_event(event)
+      event["[@metadata][size]"] = size unless size.nil?
+      event["[@metadata][pos]"] = pos unless pos.nil?
+      event["[@metadata][path]"] = path
+      event["path"] = path if !event.include?("path")
+      input.post_process_this(event)
+    end
+
+    def add_state(data, size, pos)
+      @data = data
+      @size = size
+      @pos = pos
+      self
+    end
+
+    private
+
+    # duplicate and add state for downstream
+    def dup_adding_state(line, size, pos)
+      self.class.new(path, input).add_state(line, size, pos)
+    end
+  end
+
+  class FlushableListener < ListenerTail
+    attr_writer :path
+  end
+
+  def listener_for(path)
+    # path is the identity
+    ListenerTail.new(path, self)
+  end
+
+  def begin_tailing
+    # if the pipeline restarts this input,
+    # make sure previous files are closed
+    stop
+    # use observer listener api
+    @tail = FileWatch::Ext::Tail.new_observing_progress(@tail_config)
     @tail.logger = @logger
     @path.each { |path| @tail.tail(path) }
-    hostname = Socket.gethostname
+  end
 
-    @tail.subscribe do |path, data, type|
-      @logger.debug("Received line", :path => path, :data => data) if logger.debug?
+  def run(queue)
+    begin_tailing
+    @queue = queue
+    @tail.subscribe(self)
+    exit_flush
+  end # def run
 
-      if type == :log
-        @codec.decode(data) do |event|
+  def post_process_this(event)
+    event["host"] = @host if !event.include?("host")
+    decorate(event)
+    @queue << event
+  end
 
-          decorate(event)
-          event["host"] = hostname
-          event["path"] = path
+  def log_line_received(path, line)
+    return if !@logger.debug?
+    @logger.debug("Received line", :path => path, :text => line)
+  end
 
-          queue << event
-        end
-      elsif type == :progressdb
-        @codec_plain.decode(data) do |event|
-
-          decorate(event)
-          event["host"] = hostname
-          event["path"] = path
-          event["type"] = "progressdb";
-
-          queue << event
-        end        
-      end # if
-    end # subscribe
-    finished
-  end # def run
+  def stop
+    # in filewatch >= 0.6.7, quit will closes and forget all files
+    # but it will write their last read positions to since_db
+    # beforehand
+    if @tail
+      @codec.close
+      @tail.quit
+    end
+  end
 
-  public
-  def teardown
-    @tail.sincedb_write
-    @tail.quit
-  end # def teardown
-end # class LogStash::Inputs::TriggeredPackage
+  private
+
+  def exit_flush
+    listener = FlushableListener.new("none", self)
+    if @codec.identity_count.zero?
+      # using the base codec without identity/path info
+      @codec.base_codec.flush do |event|
+        begin
+          listener.process_event(event)
+        rescue => e
+          @logger.error("File Input: flush on exit downstream error", :exception => e)
+        end
+      end
+    else
+      @codec.flush_mapped(listener)
+    end
+  end
+end # class LogStash::Inputs::File

data/logstash-input-file_progress.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file_progress'
-  s.version         = '2.0.0'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files with progress-events."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,13 +20,16 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.20", "<= 2.99"
 
-  s.add_runtime_dependency 'logstash-codec-line'
+  s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
-  s.add_runtime_dependency "filewatch-ext", ["~> 0.3.0"]
-
+  s.add_runtime_dependency "filewatch-ext", ["~> 1.1.0"]
+  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.7']
 
+  s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'rspec-sequencing'
 end
 

metadata

@@ -1,42 +1,42 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file_progress
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Signify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-11 00:00:00.000000000 Z
+date: 2016-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+        version: '1.20'
+    - - <=
       - !ruby/object:Gem::Version
-        version: 3.0.0
-  name: logstash-core
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+        version: '1.20'
+    - - <=
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-line
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -63,7 +63,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 1.1.0
   name: filewatch-ext
   prerelease: false
   type: :runtime
@@ -71,7 +71,35 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.7
+  name: logstash-codec-multiline
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.7
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.19
+  name: stud
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.19
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -86,6 +114,34 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-sequencing
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: dietmar@signifydata.com
 executables: []