logstash-input-file 4.3.0 → 4.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0389c6d97f2957a0701da71d3cbcc7865ed51745e2eb49bb0a6327097865a921'
-  data.tar.gz: 7bec1b1e810008641a3286406c7402e82dcb62d6132b5f39cfa035c2ec168829
+  metadata.gz: ff1f9254faa01ec5630d380861c674edbf1d3e4b4e3a95a86e0305513206dc94
+  data.tar.gz: 62783c5f3bd97b0736650625649a3c2264c38b8ae0397c0c48c4f92d6d4e5445
 SHA512:
-  metadata.gz: 3f83c7fcac2202d6380f7e54ecf231530ea09adf8e7019cef5d5935a2c024d85161b4749e7d1f355f8b38b59d26d57d95035c89abe0cb6b555edf0391b81475d
-  data.tar.gz: 0d939d35f02541ca57ac6a1735d6c008ba985ed977842f78a9ad95011e3146d8c2e3a939020354532d53b7975c5a572ffa9702f9ce82a36a4af235c50b6efaca
+  metadata.gz: 3327cfa95a7208246874f72bf21ecd104c626c840477bd25309ac94795d3699fc20e55847110bff41c9a09b30faea19069b5bf6e46c8f13494cee015a8c1230f
+  data.tar.gz: 38821490e5415889a4fa40c9cdb5723dcbaf660157dd2e84bf93160686f5a772a59293b379bc34b544dc6f635c5367020bb13a264da0c8e5e5517ddf79108685

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 4.4.1
+  - Fix: update to Gradle 7 [#305](https://github.com/logstash-plugins/logstash-input-file/pull/305)
+  - [DOC] Add version attributes to doc source file [#308](https://github.com/logstash-plugins/logstash-input-file/pull/308)
+  
+## 4.4.0
+  - Add support for ECS v8 [#301](https://github.com/logstash-plugins/logstash-input-file/pull/301)
+
+## 4.3.1
+  - Add extra safety to `chown` call in `atomic_write`, avoiding plugin crashes and falling back to a 
+  `non_atomic_write` in the event of failure [#295](https://github.com/logstash-plugins/logstash-input-file/pull/295)
+  - Refactor: unify event updates to happen in one place [#297](https://github.com/logstash-plugins/logstash-input-file/pull/297)
+  - Test: Actually retry tests on `RSpec::Expectations::ExpectationNotMetError` and retry instead of relying on timeout
+  [#297](https://github.com/logstash-plugins/logstash-input-file/pull/297)
+
 ## 4.3.0
   - Add ECS Compatibility Mode [#291](https://github.com/logstash-plugins/logstash-input-file/pull/291)
 

data/docs/index.asciidoc

@@ -9,6 +9,11 @@ START - GENERATED VARIABLES, DO NOT EDIT!
 :release_date: %RELEASE_DATE%
 :changelog_url: %CHANGELOG_URL%
 :include_path: ../../../../logstash/docs/include
+
+ifeval::["{versioned_docs}"=="true"]
+:branch: %BRANCH%
+:ecs_version: %ECS_VERSION%
+endif::[]
 ///////////////////////////////////////////
 END - GENERATED VARIABLES, DO NOT EDIT!
 ///////////////////////////////////////////
@@ -82,12 +87,12 @@ was not ideal and a dedicated Read mode is an improvement.
 ==== Compatibility with the Elastic Common Schema (ECS)
 
 This plugin adds metadata about event's source, and can be configured to do so
-in an {ecs-ref}[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
+in an https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
 This metadata is added after the event has been decoded by the appropriate codec,
 and will never overwrite existing values.
 
 |========
-| ECS Disabled | ECS v1 | Description
+| ECS Disabled | ECS `v1`, `v8` | Description
 
 | `host` | `[host][name]` | The name of the {ls} host that processed the event
 | `path` | `[log][file][path]` | The full path to the log file from which the event originates
@@ -264,13 +269,13 @@ In practice, this will be the best case because the time taken to read new conte
 * Value type is <<string,string>>
 * Supported values are:
 ** `disabled`: sets non-ECS metadata on event (such as top-level `host`, `path`)
-** `v1`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`)
+** `v1`,`v8`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`)
 * Default value depends on which version of Logstash is running:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.
 
 Controls this plugin's compatibility with the
-{ecs-ref}[Elastic Common Schema (ECS)].
+https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-exclude"]
 ===== `exclude`
@@ -426,7 +431,7 @@ of `/var/log` will be done for all `*.log` files.
 Paths must be absolute and cannot be relative.
 
 You may also configure multiple paths. See an example
-on the {logstash-ref}/configuration-file-structure.html#array[Logstash configuration page].
+on the https://www.elastic.co/guide/en/logstash/{branch}/configuration-file-structure.html#array[Logstash configuration page].
 
 [id="plugins-{type}s-{plugin}-sincedb_clean_after"]
 ===== `sincedb_clean_after`
@@ -439,7 +444,7 @@ The sincedb record now has a last active timestamp associated with it.
 If no changes are detected in a tracked file in the last N days its sincedb
 tracking record expires and will not be persisted.
 This option helps protect against the inode recycling problem.
-Filebeat has a {filebeat-ref}/inode-reuse-issue.html[FAQ about inode recycling].
+Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}}/inode-reuse-issue.html[FAQ about inode recycling].
 
 [id="plugins-{type}s-{plugin}-sincedb_path"]
 ===== `sincedb_path`
@@ -533,4 +538,9 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs"
 [NOTE]
 `micro` `micros` and `microseconds` are not supported
 
+ifeval::["{versioned_docs}"=="true"]
+:branch: current
+:ecs_version: current
+endif::[]
+
 :default_codec!:

data/lib/filewatch/helper.rb

@@ -30,6 +30,7 @@ module FileHelper
     temp_file.binmode
     return_val = yield temp_file
     temp_file.close
+    new_stat = File.stat(temp_file)
 
     # Overwrite original file with temp file
     File.rename(temp_file.path, file_name)
@@ -37,8 +38,10 @@ module FileHelper
     # Unable to get permissions of the original file => return
     return return_val if old_stat.nil?
 
-    # Set correct uid/gid on new file
-    File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat
+    # Set correct uid/gid on new file if ownership is different.
+    if old_stat && (old_stat.gid != new_stat.gid || old_stat.uid != new_stat.uid)
+      File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat
+    end
 
     return_val
   end

data/lib/filewatch/sincedb_collection.rb

@@ -225,13 +225,24 @@ module FileWatch
 
     # @return expired keys
     def atomic_write(time)
-      FileHelper.write_atomically(@full_path) do |io|
-        @serializer.serialize(@sincedb, io, time.to_f)
+      logger.trace? && logger.trace("non_atomic_write: ", :time => time)
+      begin
+        FileHelper.write_atomically(@full_path) do |io|
+          @serializer.serialize(@sincedb, io, time.to_f)
+        end
+      rescue Errno::EPERM, Errno::EACCES => e
+        logger.warn("sincedb_write: unable to write atomically due to permissions error, falling back to non-atomic write: #{path} error:", :exception => e.class, :message => e.message)
+        @write_method = method(:non_atomic_write)
+        non_atomic_write(time)
+      rescue => e
+        logger.warn("sincedb_write: unable to write atomically, attempting non-atomic write: #{path} error:", :exception => e.class, :message => e.message)
+        non_atomic_write(time)
       end
     end
 
     # @return expired keys
     def non_atomic_write(time)
+      logger.trace? && logger.trace("non_atomic_write: ", :time => time)
       File.open(@full_path, "w+") do |io|
         @serializer.serialize(@sincedb, io, time.to_f)
       end

data/lib/logstash/inputs/file.rb

@@ -89,7 +89,7 @@ module LogStash module Inputs
 class File < LogStash::Inputs::Base
   config_name "file"
 
-  include PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
+  include PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   # The path(s) to the file(s) to use as an input.
   # You can use filename patterns here, such as `/var/log/*.log`.
@@ -373,12 +373,11 @@ class File < LogStash::Inputs::Base
     @completely_stopped.make_true
   end # def run
 
-  def post_process_this(event)
+  def post_process_this(event, path)
+    event.set("[@metadata][path]", path)
     event.set("[@metadata][host]", @host)
     attempt_set(event, @source_host_field, @host)
-
-    source_path = event.get('[@metadata][path]') and
-      attempt_set(event, @source_path_field, source_path)
+    attempt_set(event, @source_path_field, path) if path
 
     decorate(event)
     @queue.get << event

data/lib/logstash/inputs/file_listener.rb

@@ -40,8 +40,7 @@ module LogStash module Inputs
     end
 
     def process_event(event)
-      event.set("[@metadata][path]", path)
-      input.post_process_this(event)
+      input.post_process_this(event, path)
     end
 
   end

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.3.0'
+  s.version         = '4.4.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -33,7 +33,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
-  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'

data/spec/helpers/spec_helper.rb

@@ -57,7 +57,7 @@ module FileInput
       @tracer.push [:close, true]
     end
     def clone
-      self.class.new
+      self
     end
   end
 end

data/spec/inputs/file_read_spec.rb

@@ -338,7 +338,7 @@ describe LogStash::Inputs::File do
       sincedb_content = File.read(sincedb_path).strip
       expect( sincedb_content ).to_not be_empty
 
-      Stud.try(3.times) do
+      try(3) do
         sleep(1.5) # > sincedb_clean_after
 
         sincedb_content = File.read(sincedb_path).strip
@@ -363,7 +363,10 @@ describe LogStash::Inputs::File do
     end
   end
 
-  def wait_for_file_removal(path, timeout: 3 * interval)
-    wait(timeout).for { File.exist?(path) }.to be_falsey
+  def wait_for_file_removal(path)
+    timeout = interval
+    try(5) do
+      wait(timeout).for { File.exist?(path) }.to be_falsey
+    end
   end
 end

data/spec/inputs/file_tail_spec.rb

@@ -103,7 +103,7 @@ describe LogStash::Inputs::File do
 
 
     context "when path and host fields exist", :ecs_compatibility_support do
-      ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
         before(:each) do
           allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
@@ -157,39 +157,49 @@ describe LogStash::Inputs::File do
       end
     end
 
-    context "running the input twice" do
-      let(:name) { "D" }
-      it "should read old files" do
-        conf = <<-CONFIG
-          input {
-            file {
-              type => "blah"
-              path => "#{path_path}"
-              start_position => "beginning"
-              codec => "json"
-            }
-          }
-        CONFIG
+    context "running the input twice", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
-        File.open(tmpfile_path, "w") do |fd|
-          fd.puts('{"path": "my_path", "host": "my_host"}')
-          fd.puts('{"my_field": "my_val"}')
-          fd.fsync
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
         end
-        # arbitrary old file (2 days)
-        FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
+        
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
+        let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] }
+        
+        let(:name) { "D" }
+        it "should read old files" do
+          conf = <<-CONFIG
+            input {
+              file {
+                type => "blah"
+                path => "#{path_path}"
+                start_position => "beginning"
+                codec => "json"
+              }
+            }
+          CONFIG
 
-        events = input(conf) do |pipeline, queue|
-          2.times.collect { queue.pop }
+          File.open(tmpfile_path, "w") do |fd|
+            fd.puts('{"path": "my_path", "host": "my_host"}')
+            fd.puts('{"my_field": "my_val"}')
+            fd.fsync
+          end
+          # arbitrary old file (2 days)
+          FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
+
+          events = input(conf) do |pipeline, queue|
+            2.times.collect { queue.pop }
+          end
+          existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
+          expect(events[existing_path_index].get("path")).to eq "my_path"
+          expect(events[existing_path_index].get("host")).to eq "my_host"
+          expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+
+          expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}"
+          expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+          expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
         end
-        existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
-        expect(events[existing_path_index].get("path")).to eq "my_path"
-        expect(events[existing_path_index].get("host")).to eq "my_host"
-        expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-
-        expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
-        expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-        expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
       end
     end
 
@@ -233,54 +243,62 @@ describe LogStash::Inputs::File do
       FileUtils.rm_rf(sincedb_path)
     end
 
-    context "when data exists and then more data is appended" do
-      subject { described_class.new(conf) }
+    context "when data exists and then more data is appended", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
-      before do
-        File.open(tmpfile_path, "w") do |fd|
-          fd.puts("ignore me 1")
-          fd.puts("ignore me 2")
-          fd.fsync
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
         end
-        mlconf.update("pattern" => "^\s", "what" => "previous")
-        conf.update("type" => "blah",
-              "path" => path_path,
-              "sincedb_path" => sincedb_path,
-              "stat_interval" => 0.1,
-              "codec" => mlcodec,
-              "delimiter" => TEST_FILE_DELIMITER)
-      end
 
-      it "reads the appended data only" do
-        subject.register
-        actions = RSpec::Sequencing
-          .run_after(1, "append two lines after delay") do
-            File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") }
-          end
-          .then("wait for one event") do
-            wait(0.75).for{events.size}.to eq(1)
-          end
-          .then("quit") do
-            subject.stop
-          end
-          .then("wait for flushed event") do
-            wait(0.75).for{events.size}.to eq(2)
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
+        subject { described_class.new(conf) }
+
+        before do
+          File.open(tmpfile_path, "w") do |fd|
+            fd.puts("ignore me 1")
+            fd.puts("ignore me 2")
+            fd.fsync
           end
+          mlconf.update("pattern" => "^\s", "what" => "previous")
+          conf.update("type" => "blah",
+                "path" => path_path,
+                "sincedb_path" => sincedb_path,
+                "stat_interval" => 0.1,
+                "codec" => mlcodec,
+                "delimiter" => TEST_FILE_DELIMITER)
+        end
 
-        subject.run(events)
-        actions.assert_no_errors
+        it "reads the appended data only" do
+          subject.register
+          actions = RSpec::Sequencing
+            .run_after(1, "append two lines after delay") do
+              File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") }
+            end
+            .then("wait for one event") do
+              wait(0.75).for{events.size}.to eq(1)
+            end
+            .then("quit") do
+              subject.stop
+            end
+            .then("wait for flushed event") do
+              wait(0.75).for{events.size}.to eq(2)
+            end
 
-        event1 = events[0]
-        expect(event1).not_to be_nil
-        expect(event1.get("path")).to eq tmpfile_path
-        expect(event1.get("[@metadata][path]")).to eq tmpfile_path
-        expect(event1.get("message")).to eq "hello"
-
-        event2 = events[1]
-        expect(event2).not_to be_nil
-        expect(event2.get("path")).to eq tmpfile_path
-        expect(event2.get("[@metadata][path]")).to eq tmpfile_path
-        expect(event2.get("message")).to eq "world"
+          subject.run(events)
+          actions.assert_no_errors
+
+          event1 = events[0]
+          expect(event1).not_to be_nil
+          expect(event1.get(file_path_target_field)).to eq tmpfile_path
+          expect(event1.get("[@metadata][path]")).to eq tmpfile_path
+          expect(event1.get("message")).to eq "hello"
+
+          event2 = events[1]
+          expect(event2).not_to be_nil
+          expect(event2.get(file_path_target_field)).to eq tmpfile_path
+          expect(event2.get("[@metadata][path]")).to eq tmpfile_path
+          expect(event2.get("message")).to eq "world"
+        end
       end
     end
 
@@ -311,12 +329,21 @@ describe LogStash::Inputs::File do
           .then_after(0.1, "identity is mapped") do
             wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped"
           end
-          .then("wait for auto_flush") do
-            wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't"
+          .then("wait accept") do
+            wait(0.75).for {
+              subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)
+            }.to eq([true]), "accept didn't"
           end
-          .then("quit") do
+          .then("request a stop") do
+            # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation
             subject.stop
           end
+          .then("wait for auto_flush") do
+            wait(2).for {
+              subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)
+                .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
+            }.to eq(true), "autoflush didn't"
+          end
         subject.run(events)
         actions.assert_no_errors
         expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true])
@@ -356,74 +383,50 @@ describe LogStash::Inputs::File do
       end
     end
 
-    context "when wildcard path and a multiline codec is specified" do
-      subject { described_class.new(conf) }
-      let(:suffix)       { "J" }
-      let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
-      before do
-        mlconf.update("pattern" => "^\s", "what" => "previous")
-        conf.update(
-              "type" => "blah",
-              "path" => path_path,
-              "start_position" => "beginning",
-              "sincedb_path" => sincedb_path,
-              "stat_interval" => 0.05,
-              "codec" => mlcodec,
-              "file_sort_by" => "path",
-              "delimiter" => TEST_FILE_DELIMITER)
+    context "when wildcard path and a multiline codec is specified", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
-        subject.register
-      end
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+        end
 
-      it "collects separate multiple line events from each file" do
-        subject
-        actions = RSpec::Sequencing
-          .run_after(0.1, "create files") do
-            File.open(tmpfile_path, "wb") do |fd|
-              fd.puts("line1.1-of-J")
-              fd.puts("  line1.2-of-J")
-              fd.puts("  line1.3-of-J")
-            end
-            File.open(tmpfile_path2, "wb") do |fd|
-              fd.puts("line1.1-of-K")
-              fd.puts("  line1.2-of-K")
-              fd.puts("  line1.3-of-K")
-            end
-          end
-          .then("assert both files are mapped as identities and stop") do
-            wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
-          end
-          .then("stop") do
-            subject.stop
-          end
-        subject.run(events)
-        # wait for actions to complete
-        actions.assert_no_errors
-        expect(events.size).to eq(2)
-        e1, e2 = events
-        e1_message = e1.get("message")
-        e2_message = e2.get("message")
-
-        expect(e1.get("path")).to match(/J.txt/)
-        expect(e2.get("path")).to match(/K.txt/)
-        expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER}  line1.2-of-J#{TEST_FILE_DELIMITER}  line1.3-of-J")
-        expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER}  line1.2-of-K#{TEST_FILE_DELIMITER}  line1.3-of-K")
-      end
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
+
+        subject { described_class.new(conf) }
+        let(:suffix)       { "J" }
+        let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
+        before do
+          mlconf.update("pattern" => "^\s", "what" => "previous")
+          conf.update(
+                "type" => "blah",
+                "path" => path_path,
+                "start_position" => "beginning",
+                "sincedb_path" => sincedb_path,
+                "stat_interval" => 0.05,
+                "codec" => mlcodec,
+                "file_sort_by" => "path",
+                "delimiter" => TEST_FILE_DELIMITER)
 
-      context "if auto_flush is enabled on the multiline codec" do
-        let(:mlconf) { { "auto_flush_interval" => 0.5 } }
-        let(:suffix)       { "M" }
-        it "an event is generated via auto_flush" do
+          subject.register
+        end
+
+        it "collects separate multiple line events from each file" do
+          subject
           actions = RSpec::Sequencing
             .run_after(0.1, "create files") do
               File.open(tmpfile_path, "wb") do |fd|
-                fd.puts("line1.1-of-a")
-                fd.puts("  line1.2-of-a")
-                fd.puts("  line1.3-of-a")
+                fd.puts("line1.1-of-J")
+                fd.puts("  line1.2-of-J")
+                fd.puts("  line1.3-of-J")
+              end
+              File.open(tmpfile_path2, "wb") do |fd|
+                fd.puts("line1.1-of-K")
+                fd.puts("  line1.2-of-K")
+                fd.puts("  line1.3-of-K")
               end
             end
-            .then("wait for auto_flush") do
-              wait(2).for{events.size}.to eq(1), "events size is not 1"
+            .then("assert both files are mapped as identities and stop") do
+              wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
             end
             .then("stop") do
               subject.stop
@@ -431,10 +434,43 @@ describe LogStash::Inputs::File do
           subject.run(events)
           # wait for actions to complete
           actions.assert_no_errors
-          e1 = events.first
+          expect(events.size).to eq(2)
+          e1, e2 = events
           e1_message = e1.get("message")
-          expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
-          expect(e1.get("path")).to match(/M.txt$/)
+          e2_message = e2.get("message")
+
+          expect(e1.get(file_path_target_field)).to match(/J.txt/)
+          expect(e2.get(file_path_target_field)).to match(/K.txt/)
+          expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER}  line1.2-of-J#{TEST_FILE_DELIMITER}  line1.3-of-J")
+          expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER}  line1.2-of-K#{TEST_FILE_DELIMITER}  line1.3-of-K")
+        end
+
+        context "if auto_flush is enabled on the multiline codec" do
+          let(:mlconf) { { "auto_flush_interval" => 0.5 } }
+          let(:suffix)       { "M" }
+          it "an event is generated via auto_flush" do
+            actions = RSpec::Sequencing
+              .run_after(0.1, "create files") do
+                File.open(tmpfile_path, "wb") do |fd|
+                  fd.puts("line1.1-of-a")
+                  fd.puts("  line1.2-of-a")
+                  fd.puts("  line1.3-of-a")
+                end
+              end
+              .then("wait for auto_flush") do
+                wait(2).for{events.size}.to eq(1), "events size is not 1"
+              end
+              .then("stop") do
+                subject.stop
+              end
+            subject.run(events)
+            # wait for actions to complete
+            actions.assert_no_errors
+            e1 = events.first
+            e1_message = e1.get("message")
+            expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
+            expect(e1.get(file_path_target_field)).to match(/M.txt$/)
+          end
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-30 00:00:00.000000000 Z
+date: 2022-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -91,7 +91,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
   prerelease: false
   type: :runtime
@@ -99,7 +99,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -282,8 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Streams events from files