logstash-input-file 4.1.2 → 4.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e860fa4a6695373c1ea4f70465392f3a95b290aefc499b0e2b5732dd99f5a0c6
-  data.tar.gz: 840ac241383e9a0777867da86a71ef3045c9201b0e26074526a8ace5033ee20b
+  metadata.gz: 44f11a07375e6cf964220179c9cea4259a9bb6829dbbdda85e05fef737a9e214
+  data.tar.gz: 9005ada3317a3d947bce138f9bc014647f79ad871db7cfd373642f51f94ca8bc
 SHA512:
-  metadata.gz: fc311c3ecfe954c669d585cfb6791dfe901170b223e080540e4ecfbdec6a010df76ee3102dfbb732abab34b5cf3422c95b02452d1dddea125b7991be07f4a020
-  data.tar.gz: 97990bd44ba64927d16d70c2f7325d578d0b30798860f15a614c8aa6f73cf6ec9ec8ac311b80cabe40ae2ab5ef07372a816675ed4daa3128cdd97e49591828bd
+  metadata.gz: 772fedf54c74b08d660f2a6aca1dd43f9ab04cb9290ef67d409e26419931bcf0fa26de027c075d561f35f40d55ac8b43c376c7b7c88f1f7982f1eb243361c7a3
+  data.tar.gz: dc5dc4aa91b870368967f92fc4129faa4e2b470e0bd942dda965711fab90def3658e9efd8f9158f3f98193a5aedcf23322bf65c7f5a28a4c32b84d6aac370118

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.1.3
+  - Fixed `read` mode of regular files sincedb write is requested in each read loop
+    iteration rather than waiting for the end-of-file to be reached. Note: for gz files,
+    the sincedb entry can only be updated at the end of the file as it is not possible
+    to seek into a compressed file and begin reading from that position.
+    [#196](https://github.com/logstash-plugins/logstash-input-file/pull/196)
+  - Added support for String Durations in some settings e.g. `stat_interval => "750 ms"`
+    [#194](https://github.com/logstash-plugins/logstash-input-file/pull/194)
+
 ## 4.1.2
   - Fix `require winhelper` error in WINDOWS.
     [Issue #184](https://github.com/logstash-plugins/logstash-input-file/issues/184)

data/docs/index.asciidoc

@@ -146,10 +146,15 @@ will not get picked up.
 
 This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
+[NOTE]
+Duration settings can be specified in text form e.g. "250 ms", this string will be converted into
+decimal seconds. There are quite a few supported natural and abbreviated durations,
+see <<string_duration,string duration>> for the details.
+
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
-| <<plugins-{type}s-{plugin}-close_older>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-close_older>> |<<number,number>> or <<string_duration,string duration>>|No
 | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-discover_interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-exclude>> |<<array,array>>|No
@@ -159,15 +164,15 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-file_completed_log_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-file_sort_by>> |<<string,string>>, one of `["last_modified", "path"]`|No
 | <<plugins-{type}s-{plugin}-file_sort_direction>> |<<string,string>>, one of `["asc", "desc"]`|No
-| <<plugins-{type}s-{plugin}-ignore_older>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ignore_older>> |<<number,number>> or <<string_duration,string duration>>|No
 | <<plugins-{type}s-{plugin}-max_open_files>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-mode>> |<<string,string>>, one of `["tail", "read"]`|No
 | <<plugins-{type}s-{plugin}-path>> |<<array,array>>|Yes
-| <<plugins-{type}s-{plugin}-sincedb_clean_after>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sincedb_clean_after>> |<<number,number>> or <<string_duration,string duration>>|No
 | <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-sincedb_write_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sincedb_write_interval>> |<<number,number>> or <<string_duration,string duration>>|No
 | <<plugins-{type}s-{plugin}-start_position>> |<<string,string>>, one of `["beginning", "end"]`|No
-| <<plugins-{type}s-{plugin}-stat_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-stat_interval>> |<<number,number>> or <<string_duration,string duration>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -178,18 +183,18 @@ input plugins.
 [id="plugins-{type}s-{plugin}-close_older"]
 ===== `close_older`
 
-  * Value type is <<number,number>>
-  * Default value is `3600`
+  * Value type is <<number,number>> or <<string_duration,string duration>>
+  * Default value is `"1 hour"`
 
 The file input closes any files that were last read the specified
-timespan in seconds ago.
+duration (seconds if a number is specified) ago.
 This has different implications depending on if a file is being tailed or
 read. If tailing, and there is a large time gap in incoming data the file
 can be closed (allowing other files to be opened) but will be queued for
 reopening when new data is detected. If reading, the file will be closed
 after closed_older seconds from when the last bytes were read.
 This setting is retained for backward compatibility if you upgrade the
-plugin to 5.0.0+, are reading not tailing and do not switch to using Read mode.
+plugin to 4.1.0+, are reading not tailing and do not switch to using Read mode.
 
 [id="plugins-{type}s-{plugin}-delimiter"]
 ===== `delimiter`
@@ -206,8 +211,10 @@ this setting is not used, instead the standard Windows or Unix line endings are
   * Value type is <<number,number>>
   * Default value is `15`
 
-How often (in seconds) we expand the filename patterns in the
-`path` option to discover new files to watch.
+How often we expand the filename patterns in the `path` option to discover new files to watch.
+This value is a multiple to `stat_interval`, e.g. if `stat_interval` is "500 ms" then new files
+files could be discovered every 15 X 500 milliseconds - 7.5 seconds.
+In practice, this will be the best case because the time taken to read new content needs to be factored in.
 
 [id="plugins-{type}s-{plugin}-exclude"]
 ===== `exclude`
@@ -294,11 +301,11 @@ If you use special naming conventions for the file full paths then perhaps
 [id="plugins-{type}s-{plugin}-ignore_older"]
 ===== `ignore_older`
 
-  * Value type is <<number,number>>
+  * Value type is <<number,number>> or <<string_duration,string duration>>
   * There is no default value for this setting.
 
 When the file input discovers a file that was last modified
-before the specified timespan in seconds, the file is ignored.
+before the specified duration (seconds if a number is specified), the file is ignored.
 After it's discovery, if an ignored file is modified it is no
 longer ignored and any new data is read. By default, this option is
 disabled. Note this unit is in seconds.
@@ -354,9 +361,9 @@ on the {logstash-ref}/configuration-file-structure.html#array[Logstash configura
 [id="plugins-{type}s-{plugin}-sincedb_clean_after"]
 ===== `sincedb_clean_after`
 
-  * Value type is <<number,number>>
-  * The default value for this setting is 14.
-  * This unit is in *days* and can be decimal e.g. 0.5 is 12 hours.
+  * Value type is <<number,number>> or <<string_duration,string duration>>
+  * The default value for this setting is "2 weeks".
+  * If a number is specified then it is interpreted as *days* and can be decimal e.g. 0.5 is 12 hours.
 
 The sincedb record now has a last active timestamp associated with it.
 If no changes are detected in a tracked file in the last N days its sincedb
@@ -378,8 +385,8 @@ NOTE: it must be a file path and not a directory path
 [id="plugins-{type}s-{plugin}-sincedb_write_interval"]
 ===== `sincedb_write_interval`
 
-  * Value type is <<number,number>>
-  * Default value is `15`
+  * Value type is <<number,number>> or <<string_duration,string duration>>
+  * Default value is `"15 seconds"`
 
 How often (in seconds) to write a since database with the current position of
 monitored log files.
@@ -404,15 +411,56 @@ position recorded in the sincedb file will be used.
 [id="plugins-{type}s-{plugin}-stat_interval"]
 ===== `stat_interval`
 
-  * Value type is <<number,number>>
-  * Default value is `1`
+  * Value type is <<number,number>> or <<string_duration,string duration>>
+  * Default value is `"1 second"`
 
 How often (in seconds) we stat files to see if they have been modified.
 Increasing this interval will decrease the number of system calls we make,
 but increase the time to detect new log lines.
+[NOTE]
+Discovering new files and checking whether they have grown/or shrunk occurs in a loop.
+This loop will sleep for `stat_interval` seconds before looping again. However, if files
+have grown, the new content is read and lines are enqueued.
+Reading and enqueuing across all grown files can take time, especially if
+the pipeline is congested. So the overall loop time is a combination of the
+`stat_interval` and the file read time.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
 :default_codec!:
 
+[id="string_duration"]
+// Move this to the includes when we make string durations available generally.
+==== String Durations
+
+Format is `number` `string` and the space between these is optional.
+So "45s" and "45 s" are both valid.
+[TIP]
+Use the most suitable duration, for example, "3 days" rather than "72 hours".
+
+===== Weeks
+Supported values: `w` `week` `weeks`, e.g. "2 w", "1 week", "4 weeks".
+
+===== Days
+Supported values: `d` `day` `days`, e.g. "2 d", "1 day", "2.5 days".
+
+===== Hours
+Supported values: `h` `hour` `hours`, e.g. "4 h", "1 hour", "0.5 hours".
+
+===== Minutes
+Supported values: `m` `min` `minute` `minutes`, e.g. "45 m", "35 min", "1 minute", "6 minutes".
+
+===== Seconds
+Supported values: `s` `sec` `second` `seconds`, e.g. "45 s", "15 sec", "1 second", "2.5 seconds".
+
+===== Milliseconds
+Supported values: `ms` `msec` `msecs`, e.g. "500 ms", "750 msec", "50 msecs
+[NOTE]
+`milli` `millis` and `milliseconds` are not supported
+
+===== Microseconds
+Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs"
+[NOTE]
+`micro` `micros` and `microseconds` are not supported
+

data/lib/filewatch/observing_read.rb

@@ -18,6 +18,5 @@ module FileWatch
     def build_specific_processor(settings)
       ReadMode::Processor.new(settings)
     end
-
   end
 end

data/lib/filewatch/read_mode/handlers/base.rb

@@ -7,12 +7,17 @@ module FileWatch module ReadMode module Handlers
 
     attr_reader :sincedb_collection
 
-    def initialize(sincedb_collection, observer, settings)
+    def initialize(processor, sincedb_collection, observer, settings)
       @settings = settings
+      @processor = processor
       @sincedb_collection = sincedb_collection
       @observer = observer
     end
 
+    def quit?
+      @processor.watch.quit?
+    end
+
     def handle(watched_file)
       logger.debug("handling: #{watched_file.path}")
       unless watched_file.has_listener?

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -5,13 +5,12 @@ module FileWatch module ReadMode module Handlers
     def handle_specifically(watched_file)
       if open_file(watched_file)
         add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
-        changed = false
         @settings.file_chunk_count.times do
+          break if quit?
           begin
             data = watched_file.file_read(@settings.file_chunk_size)
             result = watched_file.buffer_extract(data) # expect BufferExtractResult
             logger.info(result.warning, result.additional) unless result.warning.empty?
-            changed = true
             result.lines.each do |line|
               watched_file.listener.accept(line)
               # sincedb position is independent from the watched_file bytes_read
@@ -20,6 +19,7 @@ module FileWatch module ReadMode module Handlers
             # instead of tracking the bytes_read line by line we need to track by the data read size.
             # because we initially seek to the bytes_read not the sincedb position
             watched_file.increment_bytes_read(data.bytesize)
+            sincedb_collection.request_disk_flush
           rescue EOFError
             # flush the buffer now in case there is no final delimiter
             line = watched_file.buffer.flush
@@ -40,7 +40,6 @@ module FileWatch module ReadMode module Handlers
             break
           end
         end
-        sincedb_collection.request_disk_flush if changed
       end
     end
   end

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -13,10 +13,6 @@ module FileWatch module ReadMode module Handlers
       add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
       # can't really stripe read a zip file, its all or nothing.
       watched_file.listener.opened
-      # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading)
-      # should we track lines read in the sincedb and
-      # fast forward through the lines until we reach unseen content?
-      # meaning that we can quit in the middle of a zip file
       begin
         file_stream = FileInputStream.new(watched_file.path)
         gzip_stream = GZIPInputStream.new(file_stream)
@@ -24,14 +20,19 @@ module FileWatch module ReadMode module Handlers
         buffered = BufferedReader.new(decoder)
         while (line = buffered.readLine(false))
           watched_file.listener.accept(line)
+          # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry
+          # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading)
+          # should we track lines read in the sincedb and
+          # fast forward through the lines until we reach unseen content?
+          # meaning that we can quit in the middle of a zip file
         end
         watched_file.listener.eof
       rescue ZipException => e
         logger.error("Cannot decompress the gzip file at path: #{watched_file.path}")
         watched_file.listener.error
       else
-        sincedb_collection.store_last_read(watched_file.sincedb_key, watched_file.last_stat_size)
-        sincedb_collection.request_disk_flush
+        watched_file.update_bytes_read(watched_file.last_stat_size)
+        sincedb_collection.unset_watched_file(watched_file)
         watched_file.listener.deleted
         watched_file.unwatch
       ensure

data/lib/filewatch/read_mode/processor.rb

@@ -25,8 +25,10 @@ module FileWatch module ReadMode
     end
 
     def initialize_handlers(sincedb_collection, observer)
-      @read_file = Handlers::ReadFile.new(sincedb_collection, observer, @settings)
-      @read_zip_file = Handlers::ReadZipFile.new(sincedb_collection, observer, @settings)
+      # we deviate from the tail mode handler initialization here
+      # by adding a reference to self so we can read the quit flag during a (depth first) read loop
+      @read_file = Handlers::ReadFile.new(self, sincedb_collection, observer, @settings)
+      @read_zip_file = Handlers::ReadZipFile.new(self, sincedb_collection, observer, @settings)
     end
 
     def read_file(watched_file)

data/lib/filewatch/sincedb_collection.rb

@@ -20,14 +20,21 @@ module FileWatch
       @write_method = LogStash::Environment.windows? || @path.chardev? || @path.blockdev? ? method(:non_atomic_write) : method(:atomic_write)
       @full_path = @path.to_path
       FileUtils.touch(@full_path)
+      @write_requested = false
+    end
+
+    def write_requested?
+      @write_requested
     end
 
     def request_disk_flush
-      now = Time.now.to_i
-      delta = now - @sincedb_last_write
-      if delta >= @settings.sincedb_write_interval
-        logger.debug("writing sincedb (delta since last write = #{delta})")
-        sincedb_write(now)
+      @write_requested = true
+      flush_at_interval
+    end
+
+    def write_if_requested
+      if write_requested?
+        flush_at_interval
       end
     end
 
@@ -51,7 +58,6 @@ module FileWatch
         #No existing sincedb to load
         logger.debug("open: error: #{path}: #{e.inspect}")
       end
-
     end
 
     def associate(watched_file)
@@ -130,10 +136,6 @@ module FileWatch
       @sincedb[key].update_position(0)
     end
 
-    def store_last_read(key, last_read)
-      @sincedb[key].update_position(last_read)
-    end
-
     def increment(key, amount)
       @sincedb[key].increment_position(amount)
     end
@@ -167,6 +169,15 @@ module FileWatch
 
     private
 
+    def flush_at_interval
+      now = Time.now.to_i
+      delta = now - @sincedb_last_write
+      if delta >= @settings.sincedb_write_interval
+        logger.debug("writing sincedb (delta since last write = #{delta})")
+        sincedb_write(now)
+      end
+    end
+
     def handle_association(sincedb_value, watched_file)
       watched_file.update_bytes_read(sincedb_value.position)
       sincedb_value.set_watched_file(watched_file)
@@ -193,6 +204,7 @@ module FileWatch
           logger.debug("sincedb_write: cleaned", "key" => "'#{key}'")
         end
         @sincedb_last_write = time
+        @write_requested = false
       rescue Errno::EACCES
         # no file handles free perhaps
         # maybe it will work next time

data/lib/filewatch/watch.rb

@@ -51,6 +51,7 @@ module FileWatch
       until quit?
         iterate_on_state
         break if quit?
+        sincedb_collection.write_if_requested
         glob += 1
         if glob == interval
           discover

data/lib/logstash/inputs/file.rb

@@ -11,6 +11,7 @@ require_relative "file/patch"
 require_relative "file_listener"
 require_relative "delete_completed_file_handler"
 require_relative "log_completed_file_handler"
+require_relative "friendly_durations"
 require "filewatch/bootstrap"
 
 # Stream events from files, normally by tailing them in a manner
@@ -109,7 +110,7 @@ class File < LogStash::Inputs::Base
   # How often (in seconds) we stat files to see if they have been modified.
   # Increasing this interval will decrease the number of system calls we make,
   # but increase the time to detect new log lines.
-  config :stat_interval, :validate => :number, :default => 1
+  config :stat_interval, :validate => [FriendlyDurations, "seconds"], :default => 1
 
   # How often (in seconds) we expand the filename patterns in the
   # `path` option to discover new files to watch.
@@ -123,7 +124,7 @@ class File < LogStash::Inputs::Base
 
   # How often (in seconds) to write a since database with the current position of
   # monitored log files.
-  config :sincedb_write_interval, :validate => :number, :default => 15
+  config :sincedb_write_interval, :validate => [FriendlyDurations, "seconds"], :default => 15
 
   # Choose where Logstash starts initially reading files: at the beginning or
   # at the end. The default behavior treats files like live streams and thus
@@ -145,7 +146,7 @@ class File < LogStash::Inputs::Base
   # After its discovery, if an ignored file is modified it is no
   # longer ignored and any new data is read. By default, this option is
   # disabled. Note this unit is in seconds.
-  config :ignore_older, :validate => :number
+  config :ignore_older, :validate => [FriendlyDurations, "seconds"]
 
   # The file input closes any files that were last read the specified
   # timespan in seconds ago.
@@ -154,7 +155,7 @@ class File < LogStash::Inputs::Base
   # reopening when new data is detected. If reading, the file will be closed
   # after closed_older seconds from when the last bytes were read.
   # The default is 1 hour
-  config :close_older, :validate => :number, :default => 1 * 60 * 60
+  config :close_older, :validate => [FriendlyDurations, "seconds"], :default => "1 hour"
 
   # What is the maximum number of file_handles that this input consumes
   # at any one time. Use close_older to close some files if you need to
@@ -191,7 +192,7 @@ class File < LogStash::Inputs::Base
   # If no changes are detected in tracked files in the last N days their sincedb
   # tracking record will expire and not be persisted.
   # This option protects against the well known inode recycling problem. (add reference)
-  config :sincedb_clean_after, :validate => :number, :default => 14 # days
+  config :sincedb_clean_after, :validate => [FriendlyDurations, "days"], :default => "14 days" # days
 
   # File content is read off disk in blocks or chunks, then using whatever the set delimiter
   # is, lines are extracted from the chunk. Specify the size in bytes of each chunk.
@@ -222,6 +223,20 @@ class File < LogStash::Inputs::Base
   config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc"
 
   public
+
+  class << self
+    alias_method :old_validate_value, :validate_value
+
+    def validate_value(value, validator)
+      if validator.is_a?(Array) && validator.size == 2 && validator.first.respond_to?(:call)
+        callable, units = *validator
+        # returns a ValidatedStruct having a `to_a` method suitable to return to the config mixin caller
+        return callable.call(value, units).to_a
+      end
+      old_validate_value(value, validator)
+    end
+  end
+
   def register
     require "addressable/uri"
     require "digest/md5"
@@ -316,6 +331,7 @@ class File < LogStash::Inputs::Base
     start_processing
     @queue = queue
     @watcher.subscribe(self) # halts here until quit is called
+    # last action of the subscribe call is to write the sincedb
     exit_flush
   end # def run
 
@@ -338,9 +354,6 @@ class File < LogStash::Inputs::Base
   end
 
   def stop
-    # in filewatch >= 0.6.7, quit will closes and forget all files
-    # but it will write their last read positions to since_db
-    # beforehand
     if @watcher
       @codec.close
       @watcher.quit

data/lib/logstash/inputs/friendly_durations.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+module LogStash module Inputs
+  module FriendlyDurations
+    NUMBERS_RE = /^(?<number>\d+(\.\d+)?)\s?(?<units>s((ec)?(ond)?)(s)?|m((in)?(ute)?)(s)?|h(our)?(s)?|d(ay)?(s)?|w(eek)?(s)?|us(ec)?(s)?|ms(ec)?(s)?)?$/
+    HOURS = 3600
+    DAYS = 24 * HOURS
+    MEGA = 10**6
+    KILO = 10**3
+
+    ValidatedStruct = Struct.new(:value, :error_message) do
+      def to_a
+        error_message.nil? ? [true, value] : [false, error_message]
+      end
+    end
+
+    def self.call(value, unit = "sec")
+      # coerce into seconds
+      val_string = value.to_s.strip
+      matched = NUMBERS_RE.match(val_string)
+      if matched.nil?
+        failed_message = "Value '#{val_string}' is not a valid duration string e.g. 200 usec, 250ms, 60 sec, 18h, 21.5d, 1 day, 2w, 6 weeks"
+        return ValidatedStruct.new(nil, failed_message)
+      end
+      multiplier = matched[:units] || unit
+      numeric = matched[:number].to_f
+      case multiplier
+      when "m","min","mins","minute","minutes"
+        ValidatedStruct.new(numeric * 60, nil)
+      when "h","hour","hours"
+        ValidatedStruct.new(numeric * HOURS, nil)
+      when "d","day","days"
+        ValidatedStruct.new(numeric * DAYS, nil)
+      when "w","week","weeks"
+        ValidatedStruct.new(numeric * 7 * DAYS, nil)
+      when "ms","msec","msecs"
+        ValidatedStruct.new(numeric / KILO, nil)
+      when "us","usec","usecs"
+        ValidatedStruct.new(numeric / MEGA, nil)
+      else
+        ValidatedStruct.new(numeric, nil)
+      end
+    end
+  end
+end end

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.2'
+  s.version         = '4.1.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filewatch/read_mode_handlers_read_file_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require_relative 'spec_helper'
+
+module FileWatch
+  describe ReadMode::Handlers::ReadFile do
+    let(:settings) do
+      Settings.from_options(
+        :sincedb_write_interval => 0,
+        :sincedb_path => File::NULL
+      )
+    end
+    let(:sdb_collection) { SincedbCollection.new(settings) }
+    let(:directory) { Pathname.new(FIXTURE_DIR) }
+    let(:pathname) { directory.join('uncompressed.log') }
+    let(:watched_file) { WatchedFile.new(pathname, pathname.stat, settings) }
+    let(:processor) { ReadMode::Processor.new(settings).add_watch(watch) }
+    let(:file) { DummyFileReader.new(settings.file_chunk_size, 2) }
+
+    context "simulate reading a 64KB file with a default chunk size of 32KB and a zero sincedb write interval" do
+      let(:watch) { double("watch", :quit? => false) }
+      it "calls 'sincedb_write' exactly 2 times" do
+        allow(FileOpener).to receive(:open).with(watched_file.path).and_return(file)
+        expect(sdb_collection).to receive(:sincedb_write).exactly(2).times
+        watched_file.activate
+        processor.initialize_handlers(sdb_collection, TestObserver.new)
+        processor.read_file(watched_file)
+      end
+    end
+
+    context "simulate reading a 64KB file with a default chunk size of 32KB and a zero sincedb write interval" do
+      let(:watch) { double("watch", :quit? => true) }
+      it "calls 'sincedb_write' exactly 0 times as shutdown is in progress" do
+        expect(sdb_collection).to receive(:sincedb_write).exactly(0).times
+        watched_file.activate
+        processor.initialize_handlers(sdb_collection, TestObserver.new)
+        processor.read_file(watched_file)
+      end
+    end
+  end
+end

data/spec/filewatch/spec_helper.rb

@@ -28,6 +28,32 @@ require 'filewatch/bootstrap'
 
 module FileWatch
 
+  class DummyFileReader
+    def initialize(read_size, iterations)
+      @read_size = read_size
+      @iterations = iterations
+      @closed = false
+      @accumulated = 0
+    end
+    def file_seek(*)
+    end
+    def close()
+      @closed = true
+    end
+    def closed?
+      @closed
+    end
+    def sysread(amount)
+      @accumulated += amount
+      if @accumulated > @read_size * @iterations
+        raise EOFError.new
+      end
+      string = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcde\n"
+      multiplier = amount / string.length
+      string * multiplier
+    end
+  end
+
   FIXTURE_DIR = File.join('spec', 'fixtures')
 
   def self.make_file_older(path, seconds)

data/spec/filewatch/tailing_spec.rb

@@ -76,7 +76,7 @@ module FileWatch
 
       context "when close_older is set" do
         let(:wait_before_quit) { 0.8 }
-        let(:opts) { super.merge(:close_older => 0.2, :max_active => 1, :stat_interval => 0.1) }
+        let(:opts) { super.merge(:close_older => 0.15, :max_active => 1, :stat_interval => 0.1) }
         it "opens both files" do
           actions.activate
           tailing.watch_this(watch_dir)

data/spec/inputs/friendly_durations_spec.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+require "helpers/spec_helper"
+require "logstash/inputs/friendly_durations"
+
+describe "FriendlyDurations module function call" do
+  context "unacceptable strings" do
+    it "gives an error message for 'foobar'" do
+      result = LogStash::Inputs::FriendlyDurations.call("foobar","sec")
+      expect(result.error_message).to start_with("Value 'foobar' is not a valid duration string e.g. 200 usec")
+    end
+    it "gives an error message for '5 5 days'" do
+      result = LogStash::Inputs::FriendlyDurations.call("5 5 days","sec")
+      expect(result.error_message).to start_with("Value '5 5 days' is not a valid duration string e.g. 200 usec")
+    end
+  end
+
+  context "when a unit is not specified, a unit override will affect the result" do
+    it "coerces 14 to 1209600.0s as days" do
+      result = LogStash::Inputs::FriendlyDurations.call(14,"d")
+      expect(result.error_message).to eq(nil)
+      expect(result.value).to eq(1209600.0)
+    end
+    it "coerces '30' to 1800.0s as minutes" do
+      result = LogStash::Inputs::FriendlyDurations.call("30","minutes")
+      expect(result.to_a).to eq([true, 1800.0])
+    end
+  end
+
+  context "acceptable strings" do
+    [
+      ["10",                10.0],
+      ["10.5 s",            10.5],
+      ["10.75 secs",        10.75],
+      ["11 second",         11.0],
+      ["10 seconds",        10.0],
+      ["500 ms",             0.5],
+      ["750.9 msec",         0.7509],
+      ["750.9 msecs",        0.7509],
+      ["750.9 us",           0.0007509],
+      ["750.9 usec",         0.0007509],
+      ["750.9 usecs",        0.0007509],
+      ["1.5m",              90.0],
+      ["2.5 m",            150.0],
+      ["1.25 min",          75.0],
+      ["1 minute",          60.0],
+      ["2.5 minutes",      150.0],
+      ["2h",              7200.0],
+      ["2 h",             7200.0],
+      ["1 hour",          3600.0],
+      ["1hour",           3600.0],
+      ["3 hours",        10800.0],
+      ["0.5d",           43200.0],
+      ["1day",           86400.0],
+      ["1 day",          86400.0],
+      ["2days",         172800.0],
+      ["14 days",      1209600.0],
+      ["1w",            604800.0],
+      ["1 w",           604800.0],
+      ["1 week",        604800.0],
+      ["2weeks",       1209600.0],
+      ["2 weeks",      1209600.0],
+      ["1.5 weeks",     907200.0],
+    ].each do |input, coerced|
+      it "coerces #{input.inspect.rjust(16)} to #{coerced.inspect}" do
+        result = LogStash::Inputs::FriendlyDurations.call(input,"sec")
+        expect(result.to_a).to eq([true, coerced])
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.2
+  version: 4.1.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2018-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -204,9 +204,11 @@ files:
 - lib/logstash/inputs/file.rb
 - lib/logstash/inputs/file/patch.rb
 - lib/logstash/inputs/file_listener.rb
+- lib/logstash/inputs/friendly_durations.rb
 - lib/logstash/inputs/log_completed_file_handler.rb
 - logstash-input-file.gemspec
 - spec/filewatch/buftok_spec.rb
+- spec/filewatch/read_mode_handlers_read_file_spec.rb
 - spec/filewatch/reading_spec.rb
 - spec/filewatch/sincedb_record_serializer_spec.rb
 - spec/filewatch/spec_helper.rb
@@ -222,6 +224,7 @@ files:
 - spec/helpers/spec_helper.rb
 - spec/inputs/file_read_spec.rb
 - spec/inputs/file_tail_spec.rb
+- spec/inputs/friendly_durations_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache-2.0
@@ -250,6 +253,7 @@ specification_version: 4
 summary: Streams events from files
 test_files:
 - spec/filewatch/buftok_spec.rb
+- spec/filewatch/read_mode_handlers_read_file_spec.rb
 - spec/filewatch/reading_spec.rb
 - spec/filewatch/sincedb_record_serializer_spec.rb
 - spec/filewatch/spec_helper.rb
@@ -265,3 +269,4 @@ test_files:
 - spec/helpers/spec_helper.rb
 - spec/inputs/file_read_spec.rb
 - spec/inputs/file_tail_spec.rb
+- spec/inputs/friendly_durations_spec.rb