logstash-input-file 4.2.0 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6cdc2495fe9eb7a2c84aaaac6ea922a6034a9ad1bd8223f021ed9bd77fedae74
-  data.tar.gz: 4e6ba1c9fc044ff6cd93cbc92c16d114415357877ea75ad2fce375514517f0d7
+  metadata.gz: '0389c6d97f2957a0701da71d3cbcc7865ed51745e2eb49bb0a6327097865a921'
+  data.tar.gz: 7bec1b1e810008641a3286406c7402e82dcb62d6132b5f39cfa035c2ec168829
 SHA512:
-  metadata.gz: a790d7f2a79551d8e8a96ca8cee597c1baba4dd9eabff1f9cc806432072a58826c6066922431f020105d7d86a53b22a2a0136e3c132d1b4d532f844a4a815f2e
-  data.tar.gz: 0d4528eab5b7c56b9b6b62b7f0e7124c8bd47745ac46fb915210a7647021fd3fd0f97f4ef031d14fdd6ec03b28b697dd255c9ec9676d888cbf1107d987609c92
+  metadata.gz: 3f83c7fcac2202d6380f7e54ecf231530ea09adf8e7019cef5d5935a2c024d85161b4749e7d1f355f8b38b59d26d57d95035c89abe0cb6b555edf0391b81475d
+  data.tar.gz: 0d939d35f02541ca57ac6a1735d6c008ba985ed977842f78a9ad95011e3146d8c2e3a939020354532d53b7975c5a572ffa9702f9ce82a36a4af235c50b6efaca

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 4.3.0
+  - Add ECS Compatibility Mode [#291](https://github.com/logstash-plugins/logstash-input-file/pull/291)
+
+## 4.2.4
+  - Fix: sincedb_write issue on Windows machines [#283](https://github.com/logstash-plugins/logstash-input-file/pull/283)
+
+## 4.2.3
+  - Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280)
+
+## 4.2.2
+  - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
+
+## 4.2.1
+  - Fix: skip sincedb eviction if read mode completion deletes file during flush [#273](https://github.com/logstash-plugins/logstash-input-file/pull/273)
+  
 ## 4.2.0
   - Fix: watched files performance with huge filesets [#268](https://github.com/logstash-plugins/logstash-input-file/pull/268) 
   - Updated logging to include full traces in debug (and trace) levels

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 Travis Build
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-file.svg)](https://travis-ci.org/logstash-plugins/logstash-input-file)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-file.svg)](https://travis-ci.com/logstash-plugins/logstash-input-file)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -78,6 +78,21 @@ Read mode also allows for an action to take place after processing the file comp
 In the past attempts to simulate a Read mode while still assuming infinite streams
 was not ideal and a dedicated Read mode is an improvement.
 
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+This plugin adds metadata about event's source, and can be configured to do so
+in an {ecs-ref}[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
+This metadata is added after the event has been decoded by the appropriate codec,
+and will never overwrite existing values.
+
+|========
+| ECS Disabled | ECS v1 | Description
+
+| `host` | `[host][name]` | The name of the {ls} host that processed the event
+| `path` | `[log][file][path]` | The full path to the log file from which the event originates
+|========
+
 ==== Tracking of current position in watched files
 
 The plugin keeps track of the current position in each file by
@@ -168,6 +183,7 @@ see <<plugins-{type}s-{plugin}-string_duration,string_duration>> for the details
 | <<plugins-{type}s-{plugin}-close_older>> |<<number,number>> or <<plugins-{type}s-{plugin}-string_duration,string_duration>>|No
 | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-discover_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-exit_after_read>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-file_chunk_count>> |<<number,number>>|No
@@ -242,6 +258,20 @@ This value is a multiple to `stat_interval`, e.g. if `stat_interval` is "500 ms"
 files could be discovered every 15 X 500 milliseconds - 7.5 seconds.
 In practice, this will be the best case because the time taken to read new content needs to be factored in.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: sets non-ECS metadata on event (such as top-level `host`, `path`)
+** `v1`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`)
+* Default value depends on which version of Logstash is running:
+** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+
 [id="plugins-{type}s-{plugin}-exclude"]
 ===== `exclude`
 

data/lib/filewatch/observing_base.rb

@@ -83,15 +83,5 @@ module FileWatch
       # sincedb_write("shutting down")
     end
 
-    # close_file(path) is to be used by external code
-    # when it knows that it is completely done with a file.
-    # Other files or folders may still be being watched.
-    # Caution, once unwatched, a file can't be watched again
-    # unless a new instance of this class begins watching again.
-    # The sysadmin should rename, move or delete the file.
-    def close_file(path)
-      @watch.unwatch(path)
-      sincedb_write
-    end
   end
 end

data/lib/filewatch/read_mode/handlers/base.rb

@@ -19,7 +19,7 @@ module FileWatch module ReadMode module Handlers
     end
 
     def handle(watched_file)
-      logger.trace("handling: #{watched_file.path}")
+      logger.trace? && logger.trace("handling:", :path => watched_file.path)
       unless watched_file.has_listener?
         watched_file.set_listener(@observer)
       end
@@ -41,14 +41,14 @@ module FileWatch module ReadMode module Handlers
         # don't emit this message too often. if a file that we can't
         # read is changing a lot, we'll try to open it more often, and spam the logs.
         now = Time.now.to_i
-        logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
+        logger.trace? && logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
         if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
           backtrace = e.backtrace
           backtrace = backtrace.take(3) if backtrace && !logger.debug?
           logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace)
           watched_file.last_open_warning_at = now
         else
-          logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
+          logger.trace? && logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
         end
         watched_file.watch # set it back to watch so we can try it again
       end
@@ -67,8 +67,7 @@ module FileWatch module ReadMode module Handlers
       elsif sincedb_value.watched_file == watched_file
         update_existing_sincedb_collection_value(watched_file, sincedb_value)
       else
-        msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
-        logger.trace(msg)
+        logger.trace? && logger.trace("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
         existing_watched_file = sincedb_value.watched_file
         if existing_watched_file.nil?
           sincedb_value.set_watched_file(watched_file)
@@ -77,7 +76,7 @@ module FileWatch module ReadMode module Handlers
           watched_file.update_bytes_read(sincedb_value.position)
         else
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
+          logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
           watched_file.rotate_from(existing_watched_file)
         end
 
@@ -86,7 +85,7 @@ module FileWatch module ReadMode module Handlers
     end
 
     def update_existing_sincedb_collection_value(watched_file, sincedb_value)
-      logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      logger.trace? && logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
       # sincedb_value is the source of truth
       watched_file.update_bytes_read(sincedb_value.position)
     end
@@ -94,7 +93,7 @@ module FileWatch module ReadMode module Handlers
     def add_new_value_sincedb_collection(watched_file)
       sincedb_value = SincedbValue.new(0)
       sincedb_value.set_watched_file(watched_file)
-      logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
+      logger.trace? && logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
       sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
     end
   end

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -19,8 +19,10 @@ module FileWatch module ReadMode module Handlers
           watched_file.listener.eof
           watched_file.file_close
           key = watched_file.sincedb_key
-          sincedb_collection.reading_completed(key)
-          sincedb_collection.clear_watched_file(key)
+          if sincedb_collection.get(key)
+            sincedb_collection.reading_completed(key)
+            sincedb_collection.clear_watched_file(key)
+          end
           watched_file.listener.deleted
           # NOTE: on top of un-watching we should also remove from the watched files collection
           # if the file is getting deleted (on completion), that part currently resides in

data/lib/filewatch/settings.rb

@@ -6,7 +6,7 @@ module FileWatch
     attr_reader :max_active, :max_warn_msg, :lastwarn_max_files
     attr_reader :sincedb_write_interval, :stat_interval, :discover_interval
     attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size
-    attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration
+    attr_reader :sincedb_path, :sincedb_expiry_duration
     attr_reader :file_sort_by, :file_sort_direction
     attr_reader :exit_after_read
     attr_reader :check_archive_validity
@@ -41,7 +41,6 @@ module FileWatch
       @file_chunk_size = @opts[:file_chunk_size]
       @close_older = @opts[:close_older]
       @ignore_older = @opts[:ignore_older]
-      @sincedb_write_interval = @opts[:sincedb_write_interval]
       @stat_interval = @opts[:stat_interval]
       @discover_interval = @opts[:discover_interval]
       @exclude = Array(@opts[:exclude])

data/lib/filewatch/sincedb_collection.rb

@@ -47,16 +47,16 @@ module FileWatch
       @time_sdb_opened = Time.now.to_f
       begin
         path.open do |file|
-          logger.trace("open: reading from #{path}")
+          logger.debug("open: reading from #{path}")
           @serializer.deserialize(file) do |key, value|
-            logger.trace("open: importing ... '#{key}' => '#{value}'")
+            logger.trace? && logger.trace("open: importing #{key.inspect} => #{value.inspect}")
             set_key_value(key, value)
           end
         end
         logger.trace("open: count of keys read: #{@sincedb.keys.size}")
       rescue => e
         #No existing sincedb to load
-        logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message)
+        logger.debug("open: error opening #{path}", :exception => e.class, :message => e.message)
       end
     end
 
@@ -68,35 +68,32 @@ module FileWatch
         # and due to the window handling of many files
         # this file may not be opened in this session.
         # a new value will be added when the file is opened
-        logger.trace("associate: unmatched")
+        logger.trace("associate: unmatched", :filename => watched_file.filename)
         return true
       end
       logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
                                     :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
-      if sincedb_value.watched_file.nil?
-        # not associated
+      if sincedb_value.watched_file.nil? # not associated
         if sincedb_value.path_in_sincedb.nil?
           handle_association(sincedb_value, watched_file)
-          logger.trace("associate: inode matched but no path in sincedb")
+          logger.trace? && logger.trace("associate: inode matched but no path in sincedb", :filename => watched_file.filename)
           return true
         end
         if sincedb_value.path_in_sincedb == watched_file.path
-          # the path on disk is the same as discovered path
-          # and the inode is the same.
+          # the path on disk is the same as discovered path and the inode is the same.
           handle_association(sincedb_value, watched_file)
-          logger.trace("associate: inode and path matched")
+          logger.trace? && logger.trace("associate: inode and path matched", :filename => watched_file.filename)
           return true
         end
-        # the path on disk is different from discovered unassociated path
-        # but they have the same key (inode)
+        # the path on disk is different from discovered unassociated path but they have the same key (inode)
         # treat as a new file, a new value will be added when the file is opened
         sincedb_value.clear_watched_file
         delete(watched_file.sincedb_key)
-        logger.trace("associate: matched but allocated to another")
+        logger.trace? && logger.trace("associate: matched but allocated to another", :filename => watched_file.filename)
         return true
       end
       if sincedb_value.watched_file.equal?(watched_file) # pointer equals
-        logger.trace("associate: already associated")
+        logger.trace? && logger.trace("associate: already associated", :filename => watched_file.filename)
         return true
       end
       # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode)
@@ -107,7 +104,7 @@ module FileWatch
       #   after the original is deleted
       # are not yet in the delete phase, let this play out
       existing_watched_file = sincedb_value.watched_file
-      logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename",
+      logger.trace? && logger.trace("associate: found sincedb_value has a watched_file - this is a rename",
                                     :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
       watched_file.rotation_in_progress
       true
@@ -180,63 +177,63 @@ module FileWatch
       get(key).watched_file.nil?
     end
 
-    private
-
     def flush_at_interval
-      now = Time.now.to_i
-      delta = now - @sincedb_last_write
+      now = Time.now
+      delta = now.to_i - @sincedb_last_write
       if delta >= @settings.sincedb_write_interval
         logger.debug("writing sincedb (delta since last write = #{delta})")
         sincedb_write(now)
       end
     end
 
+    private
+
     def handle_association(sincedb_value, watched_file)
       watched_file.update_bytes_read(sincedb_value.position)
       sincedb_value.set_watched_file(watched_file)
       watched_file.initial_completed
       if watched_file.all_read?
         watched_file.ignore
-        logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
+        logger.trace? && logger.trace("handle_association fully read, ignoring", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       end
     end
 
     def set_key_value(key, value)
       if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
-        logger.trace("open: setting #{key.inspect} to #{value.inspect}")
         set(key, value)
       else
-        logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
+        logger.debug("set_key_value: record has expired, skipping: #{key.inspect} => #{value.inspect}")
       end
     end
 
-    def sincedb_write(time = Time.now.to_i)
-      logger.trace("sincedb_write: to: #{path}")
+    def sincedb_write(time = Time.now)
+      logger.trace? && logger.trace("sincedb_write: #{path} (time = #{time})")
       begin
-        @write_method.call
-        @serializer.expired_keys.each do |key|
+        expired_keys = @write_method.call(time)
+        expired_keys.each do |key|
           @sincedb[key].unset_watched_file
-          delete(key) # delete
+          delete(key)
           logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
         end
-        @sincedb_last_write = time
+        @sincedb_last_write = time.to_i
         @write_requested = false
-      rescue Errno::EACCES
-        # no file handles free perhaps
-        # maybe it will work next time
-        logger.trace("sincedb_write: error: #{path}: #{$!}")
+      rescue Errno::EACCES => e
+        # no file handles free perhaps - maybe it will work next time
+        logger.debug("sincedb_write: #{path} error:", :exception => e.class, :message => e.message)
       end
     end
 
-    def atomic_write
+    # @return expired keys
+    def atomic_write(time)
       FileHelper.write_atomically(@full_path) do |io|
-        @serializer.serialize(@sincedb, io)
+        @serializer.serialize(@sincedb, io, time.to_f)
       end
     end
 
-    def non_atomic_write
-      IO.open(IO.sysopen(@full_path, "w+")) do |io|
-        @serializer.serialize(@sincedb, io)
+    # @return expired keys
+    def non_atomic_write(time)
+      File.open(@full_path, "w+") do |io|
+        @serializer.serialize(@sincedb, io, time.to_f)
       end
     end
   end

data/lib/filewatch/sincedb_record_serializer.rb

@@ -3,30 +3,25 @@
 module FileWatch
   class SincedbRecordSerializer
 
-    attr_reader :expired_keys
-
     def self.days_to_seconds(days)
       (24 * 3600) * days.to_f
     end
 
     def initialize(sincedb_value_expiry)
       @sincedb_value_expiry = sincedb_value_expiry
-      @expired_keys = []
-    end
-
-    def update_sincedb_value_expiry_from_days(days)
-      @sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days)
     end
 
+    # @return Array expired keys (ones that were not written to the file)
     def serialize(db, io, as_of = Time.now.to_f)
-      @expired_keys.clear
+      expired_keys = []
       db.each do |key, value|
         if as_of > value.last_changed_at_expires(@sincedb_value_expiry)
-          @expired_keys << key
+          expired_keys << key
           next
         end
         io.write(serialize_record(key, value))
       end
+      expired_keys
     end
 
     def deserialize(io)
@@ -36,8 +31,7 @@ module FileWatch
     end
 
     def serialize_record(k, v)
-      # effectively InodeStruct#to_s SincedbValue#to_s
-      "#{k} #{v}\n"
+      "#{k} #{v}\n" # effectively InodeStruct#to_s SincedbValue#to_s
     end
 
     def deserialize_record(record)

data/lib/filewatch/tail_mode/handlers/base.rb

@@ -18,7 +18,7 @@ module FileWatch module TailMode module Handlers
     end
 
     def handle(watched_file)
-      logger.trace("handling: #{watched_file.filename}")
+      logger.trace? && logger.trace("handling:", :path => watched_file.path)
       unless watched_file.has_listener?
         watched_file.set_listener(@observer)
       end
@@ -37,7 +37,7 @@ module FileWatch module TailMode module Handlers
 
     def controlled_read(watched_file, loop_control)
       changed = false
-      logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
+      logger.trace? && logger.trace(__method__.to_s, :iterations => loop_control.count, :amount => loop_control.size, :filename => watched_file.filename)
       # from a real config (has 102 file inputs)
       # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
       # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
@@ -48,7 +48,7 @@ module FileWatch module TailMode module Handlers
       loop_control.count.times do
         break if quit?
         begin
-          logger.debug("read_to_eof: get chunk")
+          logger.debug? && logger.debug("#{__method__} get chunk")
           result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult
           logger.trace(result.warning, result.additional) unless result.warning.empty?
           changed = true
@@ -57,40 +57,42 @@ module FileWatch module TailMode module Handlers
             # sincedb position is now independent from the watched_file bytes_read
             sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
           end
-        rescue EOFError
+        rescue EOFError => e
           # it only makes sense to signal EOF in "read" mode not "tail"
+          logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
           loop_control.flag_read_error
           break
-        rescue Errno::EWOULDBLOCK, Errno::EINTR
+        rescue Errno::EWOULDBLOCK, Errno::EINTR => e
+          logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
           watched_file.listener.error
           loop_control.flag_read_error
           break
         rescue => e
-          logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4))
+          logger.error("#{__method__} general error reading", exception_details(watched_file.path, e))
           watched_file.listener.error
           loop_control.flag_read_error
           break
         end
       end
-      logger.debug("read_to_eof: exit due to quit") if quit?
+      logger.debug("#{__method__} stopped loop due quit") if quit?
       sincedb_collection.request_disk_flush if changed
     end
 
     def open_file(watched_file)
       return true if watched_file.file_open?
-      logger.trace("opening #{watched_file.filename}")
+      logger.trace? && logger.trace("open_file", :filename => watched_file.filename)
       begin
         watched_file.open
-      rescue
+      rescue => e
         # don't emit this message too often. if a file that we can't
         # read is changing a lot, we'll try to open it more often, and spam the logs.
         now = Time.now.to_i
-        logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
+        logger.trace? && logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
         if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
-          logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
+          logger.warn("failed to open file", exception_details(watched_file.path, e))
           watched_file.last_open_warning_at = now
         else
-          logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
+          logger.debug("open_file suppressed warning `failed to open file`", exception_details(watched_file.path, e, false))
         end
         watched_file.watch # set it back to watch so we can try it again
       else
@@ -108,26 +110,22 @@ module FileWatch module TailMode module Handlers
         update_existing_sincedb_collection_value(watched_file, sincedb_value)
         watched_file.initial_completed
       else
-        msg = "add_or_update_sincedb_collection: found sincedb record"
-        logger.trace(msg,
-          "sincedb key" => watched_file.sincedb_key,
-          "sincedb value" => sincedb_value
-        )
+        logger.trace? && logger.trace("add_or_update_sincedb_collection: found sincedb record",
+                                      :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
         # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery.
         # we must handle it here, by transferring state and have the sincedb value track this watched file
         # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to
         # and pickup the sincedb_value from before.
-        msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
-        logger.trace(msg)
+        logger.debug("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
         existing_watched_file = sincedb_value.watched_file
         if existing_watched_file.nil?
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching as new file")
+          logger.trace? && logger.trace("add_or_update_sincedb_collection: switching as new file")
           watched_file.rotate_as_file
           watched_file.update_bytes_read(sincedb_value.position)
         else
           sincedb_value.set_watched_file(watched_file)
-          logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
+          logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from:", :watched_file => watched_file.details)
           watched_file.rotate_from(existing_watched_file)
         end
       end
@@ -135,13 +133,15 @@ module FileWatch module TailMode module Handlers
     end
 
     def update_existing_sincedb_collection_value(watched_file, sincedb_value)
-      logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
+      logger.trace? && logger.trace("update_existing_sincedb_collection_value", :position => sincedb_value.position,
+                                    :filename => watched_file.filename, :last_stat_size => watched_file.last_stat_size)
       update_existing_specifically(watched_file, sincedb_value)
     end
 
     def add_new_value_sincedb_collection(watched_file)
       sincedb_value = get_new_value_specifically(watched_file)
-      logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details)
+      logger.trace? && logger.trace("add_new_value_sincedb_collection", :position => sincedb_value.position,
+                                    :watched_file => watched_file.details)
       sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
       sincedb_value
     end
@@ -153,5 +153,14 @@ module FileWatch module TailMode module Handlers
       watched_file.update_bytes_read(position)
       value
     end
+
+    private
+
+    def exception_details(path, e, trace = true)
+      details = { :path => path, :exception => e.class, :message => e.message }
+      details[:backtrace] = e.backtrace if trace && logger.debug?
+      details
+    end
+
   end
 end end end

data/lib/filewatch/tail_mode/handlers/delete.rb

@@ -7,7 +7,7 @@ module FileWatch module TailMode module Handlers
       # TODO consider trying to find the renamed file - it will have the same inode.
       # Needs a rotate scheme rename hint from user e.g. "<name>-YYYY-MM-DD-N.<ext>" or "<name>.<ext>.N"
       # send the found content to the same listener (stream identity)
-      logger.trace("delete", :path => watched_file.path, :watched_file => watched_file.details)
+      logger.trace? && logger.trace(__method__.to_s, :path => watched_file.path, :watched_file => watched_file.details)
       if watched_file.bytes_unread > 0
         logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread)
       end

data/lib/filewatch/tail_mode/handlers/shrink.rb

@@ -14,11 +14,10 @@ module FileWatch module TailMode module Handlers
     end
 
     def update_existing_specifically(watched_file, sincedb_value)
-      # we have a match but size is smaller
-      # set all to zero
+      # we have a match but size is smaller - set all to zero
       watched_file.reset_bytes_unread
       sincedb_value.update_position(0)
-      logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+      logger.trace? && logger.trace("update_existing_specifically: was truncated seeking to beginning", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
     end
   end
 end end end

data/lib/filewatch/tail_mode/handlers/unignore.rb

@@ -13,9 +13,9 @@ module FileWatch module TailMode module Handlers
       # for file initially ignored their bytes_read was set to stat.size
       # use this value not the `start_new_files_at` for the position
       # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect)
-      SincedbValue.new(watched_file.bytes_read).tap do |val|
-        val.set_watched_file(watched_file)
-        logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val)
+      SincedbValue.new(watched_file.bytes_read).tap do |sincedb_value|
+        sincedb_value.set_watched_file(watched_file)
+        logger.trace? && logger.trace("get_new_value_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       end
     end
 
@@ -26,7 +26,7 @@ module FileWatch module TailMode module Handlers
       # we will handle grow or shrink
       # for now we seek to where we were before the file got ignored (grow)
       # or to the start (shrink)
-      logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
+      logger.trace? && logger.trace("update_existing_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
       position = 0
       if watched_file.shrunk?
         watched_file.update_bytes_read(0)

data/lib/filewatch/watch.rb

@@ -51,7 +51,10 @@ module FileWatch
           glob = 0
         end
         break if quit?
+        # NOTE: maybe the plugin should validate stat_interval <= sincedb_write_interval <= sincedb_clean_after
         sleep(@settings.stat_interval)
+        # we need to check potential expired keys (sincedb_clean_after) periodically
+        sincedb_collection.flush_at_interval
       end
       sincedb_collection.write_if_requested # does nothing if no requests to write were lodged.
       @watched_files_collection.close_all

data/lib/logstash/inputs/file.rb

@@ -2,6 +2,7 @@
 require "logstash/namespace"
 require "logstash/inputs/base"
 require "logstash/codecs/identity_map_codec"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
 
 require "pathname"
 require "socket" # for Socket.gethostname
@@ -88,6 +89,8 @@ module LogStash module Inputs
 class File < LogStash::Inputs::Base
   config_name "file"
 
+  include PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
+
   # The path(s) to the file(s) to use as an input.
   # You can use filename patterns here, such as `/var/log/*.log`.
   # If you use a pattern like `/var/log/**/*.log`, a recursive search
@@ -325,6 +328,9 @@ class File < LogStash::Inputs::Base
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
     @completely_stopped = Concurrent::AtomicBoolean.new
     @queue = Concurrent::AtomicReference.new
+
+    @source_host_field = ecs_select[disabled: 'host', v1:'[host][name]']
+    @source_path_field = ecs_select[disabled: 'path', v1:'[log][file][path]']
   end # def register
 
   def completely_stopped?
@@ -369,7 +375,11 @@ class File < LogStash::Inputs::Base
 
   def post_process_this(event)
     event.set("[@metadata][host]", @host)
-    event.set("host", @host) unless event.include?("host")
+    attempt_set(event, @source_host_field, @host)
+
+    source_path = event.get('[@metadata][path]') and
+      attempt_set(event, @source_path_field, source_path)
+
     decorate(event)
     @queue.get << event
   end
@@ -377,12 +387,12 @@ class File < LogStash::Inputs::Base
   def handle_deletable_path(path)
     return if tail_mode?
     return if @completed_file_handlers.empty?
+    @logger.debug? && @logger.debug(__method__.to_s, :path => path)
     @completed_file_handlers.each { |handler| handler.handle(path) }
   end
 
   def log_line_received(path, line)
-    return unless @logger.debug?
-    @logger.debug("Received line", :path => path, :text => line)
+    @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
   end
 
   def stop
@@ -407,6 +417,17 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # Attempt to set an event's field to the provided value
+  # without overwriting an existing value or producing an error
+  def attempt_set(event, field_reference, value)
+    return false if event.include?(field_reference)
+
+    event.set(field_reference, value)
+  rescue => e
+    logger.trace("failed to set #{field_reference} to `#{value}`", :exception => e.message)
+    false
+  end
+
   def build_sincedb_base_from_env
     # This section is going to be deprecated eventually, as path.data will be
     # the default, not an environment variable (SINCEDB_DIR or LOGSTASH_HOME)

data/lib/logstash/inputs/file_listener.rb

@@ -7,9 +7,9 @@ module LogStash module Inputs
   class FileListener
     attr_reader :input, :path, :data
     # construct with link back to the input plugin instance.
-    def initialize(path, input)
+    def initialize(path, input, data = nil)
       @path, @input = path, input
-      @data = nil
+      @data = data
     end
 
     def opened
@@ -36,26 +36,14 @@ module LogStash module Inputs
     def accept(data)
       # and push transient data filled dup listener downstream
       input.log_line_received(path, data)
-      input.codec.accept(dup_adding_state(data))
+      input.codec.accept(self.class.new(path, input, data))
     end
 
     def process_event(event)
       event.set("[@metadata][path]", path)
-      event.set("path", path) unless event.include?("path")
       input.post_process_this(event)
     end
 
-    def add_state(data)
-      @data = data
-      self
-    end
-
-    private
-
-    # duplicate and add state for downstream
-    def dup_adding_state(line)
-      self.class.new(path, input).add_state(line)
-    end
   end
 
   class FlushableListener < FileListener

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.2.0'
+  s.version         = '4.3.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'

data/spec/filewatch/reading_spec.rb

@@ -91,7 +91,7 @@ module FileWatch
     context "when watching a directory with files using striped reading" do
       let(:file_path2) { ::File.join(directory, "2.log") }
       # use a chunk size that does not align with the line boundaries
-      let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")}
+      let(:opts) { super().merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")}
       let(:lines) { [] }
       let(:observer) { TestObserver.new(lines) }
       let(:listener2) { observer.listener_for(file_path2) }
@@ -121,7 +121,7 @@ module FileWatch
     end
 
     context "when a non default delimiter is specified and it is not in the content" do
-      let(:opts) { super.merge(:delimiter => "\nø") }
+      let(:opts) { super().merge(:delimiter => "\nø") }
       let(:actions) do
         RSpec::Sequencing.run("create file") do
           File.open(file_path, "wb") { |file|  file.write("line1\nline2") }
@@ -154,7 +154,7 @@ module FileWatch
       let(:file_path2) { ::File.join(directory, "2.log") }
       let(:file_path3) { ::File.join(directory, "3.log") }
 
-      let(:opts) { super.merge(:file_sort_by => "last_modified") }
+      let(:opts) { super().merge(:file_sort_by => "last_modified") }
       let(:lines) { [] }
       let(:observer) { TestObserver.new(lines) }
 
@@ -195,7 +195,7 @@ module FileWatch
     end
 
     context "when watching a directory with files using exit_after_read" do
-      let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) }
+      let(:opts) { super().merge(:exit_after_read => true, :max_open_files => 2) }
       let(:file_path3) { ::File.join(directory, "3.log") }
       let(:file_path4) { ::File.join(directory, "4.log") }
       let(:file_path5) { ::File.join(directory, "5.log") }

data/spec/filewatch/rotate_spec.rb

@@ -219,7 +219,7 @@ module FileWatch
     end
 
     context "create + rename rotation: when a new logfile is renamed to a path we have seen before but not all content from the previous the file is read" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :file_chunk_size => line1.bytesize.succ,
           :file_chunk_count => 1
         ) }
@@ -296,7 +296,7 @@ module FileWatch
     end
 
     context "copy + truncate rotation: when a logfile is copied to a new path and truncated before the open file is fully read" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :file_chunk_size => line1.bytesize.succ,
           :file_chunk_count => 1
         ) }
@@ -370,7 +370,7 @@ module FileWatch
     end
 
     context "? rotation: when an active file is renamed inside the glob and the reading lags behind" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :file_chunk_size => line1.bytesize.succ,
           :file_chunk_count => 2
         ) }
@@ -409,7 +409,7 @@ module FileWatch
     end
 
     context "? rotation: when a not active file is rotated outside the glob before the file is read" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :close_older => 3600,
           :max_open_files => 1,
           :file_sort_by => "path"

data/spec/filewatch/sincedb_record_serializer_spec.rb

@@ -9,7 +9,9 @@ module FileWatch
     let(:io) { StringIO.new }
     let(:db) { Hash.new }
 
-    subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) }
+    let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) }
+
+    subject { SincedbRecordSerializer.new(sincedb_value_expiry) }
 
     context "deserialize from IO" do
       it 'reads V1 records' do
@@ -82,8 +84,10 @@ module FileWatch
     end
 
     context "given a non default `sincedb_clean_after`" do
+
+      let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) }
+
       it "does not write expired db entries to an IO object" do
-        subject.update_sincedb_value_expiry_from_days(2)
         one_day_ago = Time.now.to_f - (1.0*24*3600)
         three_days_ago = one_day_ago - (2.0*24*3600)
         db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)

data/spec/filewatch/tailing_spec.rb

@@ -77,7 +77,7 @@ module FileWatch
 
       context "when close_older is set" do
         let(:wait_before_quit) { 0.8 }
-        let(:opts) { super.merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) }
+        let(:opts) { super().merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) }
         let(:suffix) { "B" }
         it "opens both files" do
           actions.activate_quietly
@@ -278,7 +278,7 @@ module FileWatch
 
     context "when watching a directory with files and a file is renamed to match glob", :unix => true do
       let(:suffix) { "H" }
-      let(:opts) { super.merge(:close_older => 0) }
+      let(:opts) { super().merge(:close_older => 0) }
       let(:listener2) { observer.listener_for(file_path2) }
       let(:actions) do
         RSpec::Sequencing
@@ -346,7 +346,7 @@ module FileWatch
     end
 
     context "when close older expiry is enabled" do
-      let(:opts) { super.merge(:close_older => 1) }
+      let(:opts) { super().merge(:close_older => 1) }
       let(:suffix) { "J" }
       let(:actions) do
         RSpec::Sequencing.run("create file") do
@@ -370,7 +370,7 @@ module FileWatch
     end
 
     context "when close older expiry is enabled and after timeout the file is appended-to" do
-      let(:opts) { super.merge(:close_older => 0.5) }
+      let(:opts) { super().merge(:close_older => 0.5) }
       let(:suffix) { "K" }
       let(:actions) do
         RSpec::Sequencing
@@ -406,7 +406,7 @@ module FileWatch
     end
 
     context "when ignore older expiry is enabled and all files are already expired" do
-      let(:opts) { super.merge(:ignore_older => 1) }
+      let(:opts) { super().merge(:ignore_older => 1) }
       let(:suffix) { "L" }
       let(:actions) do
         RSpec::Sequencing
@@ -430,7 +430,7 @@ module FileWatch
 
     context "when a file is renamed before it gets activated", :unix => true do
       let(:max) { 1 }
-      let(:opts) { super.merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) }
+      let(:opts) { super().merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) }
       let(:suffix) { "M" }
       let(:start_new_files_at) { :beginning } # we are creating files and sincedb record before hand
       let(:actions) do
@@ -469,7 +469,7 @@ module FileWatch
     end
 
     context "when ignore_older is less than close_older and all files are not expired" do
-      let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.1) }
+      let(:opts) { super().merge(:ignore_older => 1, :close_older => 1.1) }
       let(:suffix) { "N" }
       let(:start_new_files_at) { :beginning }
       let(:actions) do
@@ -497,7 +497,7 @@ module FileWatch
     end
 
     context "when ignore_older is less than close_older and all files are expired" do
-      let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) }
+      let(:opts) { super().merge(:ignore_older => 10, :close_older => 1) }
       let(:suffix) { "P" }
       let(:actions) do
         RSpec::Sequencing
@@ -522,7 +522,7 @@ module FileWatch
     end
 
     context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do
-      let(:opts) { super.merge(:ignore_older => 20, :close_older => 0.5) }
+      let(:opts) { super().merge(:ignore_older => 20, :close_older => 0.5) }
       let(:suffix) { "Q" }
       let(:actions) do
         RSpec::Sequencing
@@ -551,7 +551,7 @@ module FileWatch
     end
 
     context "when a non default delimiter is specified and it is not in the content" do
-      let(:opts) { super.merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") }
+      let(:opts) { super().merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") }
       let(:suffix) { "R" }
       let(:actions) do
         RSpec::Sequencing

data/spec/inputs/file_read_spec.rb

@@ -267,7 +267,7 @@ describe LogStash::Inputs::File do
   describe 'delete on complete' do
 
     let(:options) do
-      super.merge({ 'file_completed_action' => "delete", 'exit_after_read' => false })
+      super().merge({ 'file_completed_action' => "delete", 'exit_after_read' => false })
     end
 
     let(:sample_file) { File.join(temp_directory, "sample.log") }
@@ -301,25 +301,69 @@ describe LogStash::Inputs::File do
       watched_files = plugin.watcher.watch.watched_files_collection
       expect( watched_files ).to be_empty
     end
+  end
 
-    private
+  describe 'sincedb cleanup' do
 
-    def wait_for_start_processing(run_thread, timeout: 1.0)
-      begin
-        Timeout.timeout(timeout) do
-          sleep(0.01) while run_thread.status != 'sleep'
-          sleep(timeout) unless plugin.queue
-        end
-      rescue Timeout::Error
-        raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
-      else
-        raise "plugin did not start processing" unless plugin.queue
+    let(:options) do
+      super().merge(
+          'sincedb_path' => sincedb_path,
+          'sincedb_clean_after' => '1.0 seconds',
+          'sincedb_write_interval' => 0.25,
+          'stat_interval' => 0.1,
+      )
+    end
+
+    let(:sincedb_path) { "#{temp_directory}/.sincedb" }
+
+    let(:sample_file) { File.join(temp_directory, "sample.txt") }
+
+    before do
+      plugin.register
+      @run_thread = Thread.new(plugin) do |plugin|
+        Thread.current.abort_on_exception = true
+        plugin.run queue
       end
+
+      File.open(sample_file, 'w') { |fd| fd.write("line1\nline2\n") }
+
+      wait_for_start_processing(@run_thread)
     end
 
-    def wait_for_file_removal(path, timeout: 3 * interval)
-      wait(timeout).for { File.exist?(path) }.to be_falsey
+    after { plugin.stop }
+
+    it 'cleans up sincedb entry' do
+      wait_for_file_removal(sample_file) # watched discovery
+
+      sincedb_content = File.read(sincedb_path).strip
+      expect( sincedb_content ).to_not be_empty
+
+      Stud.try(3.times) do
+        sleep(1.5) # > sincedb_clean_after
+
+        sincedb_content = File.read(sincedb_path).strip
+        expect( sincedb_content ).to be_empty
+      end
     end
 
   end
+
+  private
+
+  def wait_for_start_processing(run_thread, timeout: 1.0)
+    begin
+      Timeout.timeout(timeout) do
+        sleep(0.01) while run_thread.status != 'sleep'
+        sleep(timeout) unless plugin.queue
+      end
+    rescue Timeout::Error
+      raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
+    else
+      raise "plugin did not start processing" unless plugin.queue
+    end
+  end
+
+  def wait_for_file_removal(path, timeout: 3 * interval)
+    wait(timeout).for { File.exist?(path) }.to be_falsey
+  end
 end

data/spec/inputs/file_tail_spec.rb

@@ -3,7 +3,9 @@
 require "helpers/spec_helper"
 require "logstash/devutils/rspec/shared_examples"
 require "logstash/inputs/file"
+require "logstash/plugin_mixins/ecs_compatibility_support/spec_helper"
 
+require "json"
 require "tempfile"
 require "stud/temporary"
 require "logstash/codecs/multiline"
@@ -99,41 +101,59 @@ describe LogStash::Inputs::File do
       end
     end
 
-    context "when path and host fields exist" do
-      let(:name) { "C" }
-      it "should not overwrite them" do
-        conf = <<-CONFIG
-          input {
-            file {
-              type => "blah"
-              path => "#{path_path}"
-              start_position => "beginning"
-              sincedb_path => "#{sincedb_path}"
-              delimiter => "#{TEST_FILE_DELIMITER}"
-              codec => "json"
-            }
-          }
-        CONFIG
 
-        File.open(tmpfile_path, "w") do |fd|
-          fd.puts('{"path": "my_path", "host": "my_host"}')
-          fd.puts('{"my_field": "my_val"}')
-          fd.fsync
+    context "when path and host fields exist", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
         end
 
-        events = input(conf) do |pipeline, queue|
-          2.times.collect { queue.pop }
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
+        let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] }
+
+        let(:event_with_existing) do
+          LogStash::Event.new.tap do |e|
+            e.set(file_path_target_field, 'my_path')
+            e.set(source_host_target_field, 'my_host')
+          end.to_hash
         end
 
-        existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
+        let(:name) { "C" }
+        it "should not overwrite them" do
+          conf = <<-CONFIG
+            input {
+              file {
+                type => "blah"
+                path => "#{path_path}"
+                start_position => "beginning"
+                sincedb_path => "#{sincedb_path}"
+                delimiter => "#{TEST_FILE_DELIMITER}"
+                codec => "json"
+              }
+            }
+          CONFIG
 
-        expect(events[existing_path_index].get("path")).to eq "my_path"
-        expect(events[existing_path_index].get("host")).to eq "my_host"
-        expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+          File.open(tmpfile_path, "w") do |fd|
+            fd.puts(event_with_existing.to_json)
+            fd.puts('{"my_field": "my_val"}')
+            fd.fsync
+          end
 
-        expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
-        expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-        expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+          events = input(conf) do |pipeline, queue|
+            2.times.collect { queue.pop }
+          end
+
+          existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
+
+          expect(events[existing_path_index].get(file_path_target_field)).to eq "my_path"
+          expect(events[existing_path_index].get(source_host_target_field)).to eq "my_host"
+          expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+
+          expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}"
+          expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+          expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+        end
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-06 00:00:00.000000000 Z
+date: 2021-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +86,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: