logstash-input-file 4.4.0 → 4.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 667711732af65c8b27c1bf3e455d78748d0b0c9365cc612df7aa29cc65fbbb58
-  data.tar.gz: 3f25446f513d8cb7a5619bd7e00333bcade1dbb6ccd8aeb061b991b14e91cb1c
+  metadata.gz: ff1f9254faa01ec5630d380861c674edbf1d3e4b4e3a95a86e0305513206dc94
+  data.tar.gz: 62783c5f3bd97b0736650625649a3c2264c38b8ae0397c0c48c4f92d6d4e5445
 SHA512:
-  metadata.gz: 371c56fb328a940b11047b88ded84ef2c17427ae706999dcdb15a674109be7515c952b67aadb692400e76fae9edf0fd5bfb6984a9b23f9fe00762ec618d88b25
-  data.tar.gz: b980a4595e55b90783f72f9f36240c0ac9f4a0aea998ab27ae6e2b5024474961f100349f6a044aee29826d7c94789829b1f3f1598bd204ba1b4edb0d90295fa8
+  metadata.gz: 3327cfa95a7208246874f72bf21ecd104c626c840477bd25309ac94795d3699fc20e55847110bff41c9a09b30faea19069b5bf6e46c8f13494cee015a8c1230f
+  data.tar.gz: 38821490e5415889a4fa40c9cdb5723dcbaf660157dd2e84bf93160686f5a772a59293b379bc34b544dc6f635c5367020bb13a264da0c8e5e5517ddf79108685

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.4.1
+  - Fix: update to Gradle 7 [#305](https://github.com/logstash-plugins/logstash-input-file/pull/305)
+  - [DOC] Add version attributes to doc source file [#308](https://github.com/logstash-plugins/logstash-input-file/pull/308)
+  
 ## 4.4.0
   - Add support for ECS v8 [#301](https://github.com/logstash-plugins/logstash-input-file/pull/301)
 

data/docs/index.asciidoc

@@ -9,6 +9,11 @@ START - GENERATED VARIABLES, DO NOT EDIT!
 :release_date: %RELEASE_DATE%
 :changelog_url: %CHANGELOG_URL%
 :include_path: ../../../../logstash/docs/include
+
+ifeval::["{versioned_docs}"=="true"]
+:branch: %BRANCH%
+:ecs_version: %ECS_VERSION%
+endif::[]
 ///////////////////////////////////////////
 END - GENERATED VARIABLES, DO NOT EDIT!
 ///////////////////////////////////////////
@@ -82,7 +87,7 @@ was not ideal and a dedicated Read mode is an improvement.
 ==== Compatibility with the Elastic Common Schema (ECS)
 
 This plugin adds metadata about event's source, and can be configured to do so
-in an {ecs-ref}[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
+in an https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
 This metadata is added after the event has been decoded by the appropriate codec,
 and will never overwrite existing values.
 
@@ -270,7 +275,7 @@ In practice, this will be the best case because the time taken to read new conte
 ** Otherwise, the default value is `disabled`.
 
 Controls this plugin's compatibility with the
-{ecs-ref}[Elastic Common Schema (ECS)].
+https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-exclude"]
 ===== `exclude`
@@ -426,7 +431,7 @@ of `/var/log` will be done for all `*.log` files.
 Paths must be absolute and cannot be relative.
 
 You may also configure multiple paths. See an example
-on the {logstash-ref}/configuration-file-structure.html#array[Logstash configuration page].
+on the https://www.elastic.co/guide/en/logstash/{branch}/configuration-file-structure.html#array[Logstash configuration page].
 
 [id="plugins-{type}s-{plugin}-sincedb_clean_after"]
 ===== `sincedb_clean_after`
@@ -439,7 +444,7 @@ The sincedb record now has a last active timestamp associated with it.
 If no changes are detected in a tracked file in the last N days its sincedb
 tracking record expires and will not be persisted.
 This option helps protect against the inode recycling problem.
-Filebeat has a {filebeat-ref}/inode-reuse-issue.html[FAQ about inode recycling].
+Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}}/inode-reuse-issue.html[FAQ about inode recycling].
 
 [id="plugins-{type}s-{plugin}-sincedb_path"]
 ===== `sincedb_path`
@@ -533,4 +538,9 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs"
 [NOTE]
 `micro` `micros` and `microseconds` are not supported
 
+ifeval::["{versioned_docs}"=="true"]
+:branch: current
+:ecs_version: current
+endif::[]
+
 :default_codec!:

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.4.0'
+  s.version         = '4.4.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/helpers/spec_helper.rb

@@ -57,7 +57,7 @@ module FileInput
       @tracer.push [:close, true]
     end
     def clone
-      self.class.new
+      self
     end
   end
 end

data/spec/inputs/file_tail_spec.rb

@@ -157,39 +157,49 @@ describe LogStash::Inputs::File do
       end
     end
 
-    context "running the input twice" do
-      let(:name) { "D" }
-      it "should read old files" do
-        conf = <<-CONFIG
-          input {
-            file {
-              type => "blah"
-              path => "#{path_path}"
-              start_position => "beginning"
-              codec => "json"
-            }
-          }
-        CONFIG
+    context "running the input twice", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
-        File.open(tmpfile_path, "w") do |fd|
-          fd.puts('{"path": "my_path", "host": "my_host"}')
-          fd.puts('{"my_field": "my_val"}')
-          fd.fsync
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
         end
-        # arbitrary old file (2 days)
-        FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
+        
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
+        let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] }
+        
+        let(:name) { "D" }
+        it "should read old files" do
+          conf = <<-CONFIG
+            input {
+              file {
+                type => "blah"
+                path => "#{path_path}"
+                start_position => "beginning"
+                codec => "json"
+              }
+            }
+          CONFIG
 
-        events = input(conf) do |pipeline, queue|
-          2.times.collect { queue.pop }
+          File.open(tmpfile_path, "w") do |fd|
+            fd.puts('{"path": "my_path", "host": "my_host"}')
+            fd.puts('{"my_field": "my_val"}')
+            fd.fsync
+          end
+          # arbitrary old file (2 days)
+          FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
+
+          events = input(conf) do |pipeline, queue|
+            2.times.collect { queue.pop }
+          end
+          existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
+          expect(events[existing_path_index].get("path")).to eq "my_path"
+          expect(events[existing_path_index].get("host")).to eq "my_host"
+          expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+
+          expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}"
+          expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+          expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
         end
-        existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
-        expect(events[existing_path_index].get("path")).to eq "my_path"
-        expect(events[existing_path_index].get("host")).to eq "my_host"
-        expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-
-        expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
-        expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-        expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
       end
     end
 
@@ -233,54 +243,62 @@ describe LogStash::Inputs::File do
       FileUtils.rm_rf(sincedb_path)
     end
 
-    context "when data exists and then more data is appended" do
-      subject { described_class.new(conf) }
+    context "when data exists and then more data is appended", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
-      before do
-        File.open(tmpfile_path, "w") do |fd|
-          fd.puts("ignore me 1")
-          fd.puts("ignore me 2")
-          fd.fsync
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
         end
-        mlconf.update("pattern" => "^\s", "what" => "previous")
-        conf.update("type" => "blah",
-              "path" => path_path,
-              "sincedb_path" => sincedb_path,
-              "stat_interval" => 0.1,
-              "codec" => mlcodec,
-              "delimiter" => TEST_FILE_DELIMITER)
-      end
 
-      it "reads the appended data only" do
-        subject.register
-        actions = RSpec::Sequencing
-          .run_after(1, "append two lines after delay") do
-            File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") }
-          end
-          .then("wait for one event") do
-            wait(0.75).for{events.size}.to eq(1)
-          end
-          .then("quit") do
-            subject.stop
-          end
-          .then("wait for flushed event") do
-            wait(0.75).for{events.size}.to eq(2)
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
+        subject { described_class.new(conf) }
+
+        before do
+          File.open(tmpfile_path, "w") do |fd|
+            fd.puts("ignore me 1")
+            fd.puts("ignore me 2")
+            fd.fsync
           end
+          mlconf.update("pattern" => "^\s", "what" => "previous")
+          conf.update("type" => "blah",
+                "path" => path_path,
+                "sincedb_path" => sincedb_path,
+                "stat_interval" => 0.1,
+                "codec" => mlcodec,
+                "delimiter" => TEST_FILE_DELIMITER)
+        end
 
-        subject.run(events)
-        actions.assert_no_errors
+        it "reads the appended data only" do
+          subject.register
+          actions = RSpec::Sequencing
+            .run_after(1, "append two lines after delay") do
+              File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") }
+            end
+            .then("wait for one event") do
+              wait(0.75).for{events.size}.to eq(1)
+            end
+            .then("quit") do
+              subject.stop
+            end
+            .then("wait for flushed event") do
+              wait(0.75).for{events.size}.to eq(2)
+            end
 
-        event1 = events[0]
-        expect(event1).not_to be_nil
-        expect(event1.get("path")).to eq tmpfile_path
-        expect(event1.get("[@metadata][path]")).to eq tmpfile_path
-        expect(event1.get("message")).to eq "hello"
-
-        event2 = events[1]
-        expect(event2).not_to be_nil
-        expect(event2.get("path")).to eq tmpfile_path
-        expect(event2.get("[@metadata][path]")).to eq tmpfile_path
-        expect(event2.get("message")).to eq "world"
+          subject.run(events)
+          actions.assert_no_errors
+
+          event1 = events[0]
+          expect(event1).not_to be_nil
+          expect(event1.get(file_path_target_field)).to eq tmpfile_path
+          expect(event1.get("[@metadata][path]")).to eq tmpfile_path
+          expect(event1.get("message")).to eq "hello"
+
+          event2 = events[1]
+          expect(event2).not_to be_nil
+          expect(event2.get(file_path_target_field)).to eq tmpfile_path
+          expect(event2.get("[@metadata][path]")).to eq tmpfile_path
+          expect(event2.get("message")).to eq "world"
+        end
       end
     end
 
@@ -311,12 +329,21 @@ describe LogStash::Inputs::File do
           .then_after(0.1, "identity is mapped") do
             wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped"
           end
-          .then("wait for auto_flush") do
-            wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't"
+          .then("wait accept") do
+            wait(0.75).for {
+              subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)
+            }.to eq([true]), "accept didn't"
           end
-          .then("quit") do
+          .then("request a stop") do
+            # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation
             subject.stop
           end
+          .then("wait for auto_flush") do
+            wait(2).for {
+              subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)
+                .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
+            }.to eq(true), "autoflush didn't"
+          end
         subject.run(events)
         actions.assert_no_errors
         expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true])
@@ -356,74 +383,50 @@ describe LogStash::Inputs::File do
       end
     end
 
-    context "when wildcard path and a multiline codec is specified" do
-      subject { described_class.new(conf) }
-      let(:suffix)       { "J" }
-      let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
-      before do
-        mlconf.update("pattern" => "^\s", "what" => "previous")
-        conf.update(
-              "type" => "blah",
-              "path" => path_path,
-              "start_position" => "beginning",
-              "sincedb_path" => sincedb_path,
-              "stat_interval" => 0.05,
-              "codec" => mlcodec,
-              "file_sort_by" => "path",
-              "delimiter" => TEST_FILE_DELIMITER)
+    context "when wildcard path and a multiline codec is specified", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
 
-        subject.register
-      end
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+        end
 
-      it "collects separate multiple line events from each file" do
-        subject
-        actions = RSpec::Sequencing
-          .run_after(0.1, "create files") do
-            File.open(tmpfile_path, "wb") do |fd|
-              fd.puts("line1.1-of-J")
-              fd.puts("  line1.2-of-J")
-              fd.puts("  line1.3-of-J")
-            end
-            File.open(tmpfile_path2, "wb") do |fd|
-              fd.puts("line1.1-of-K")
-              fd.puts("  line1.2-of-K")
-              fd.puts("  line1.3-of-K")
-            end
-          end
-          .then("assert both files are mapped as identities and stop") do
-            wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
-          end
-          .then("stop") do
-            subject.stop
-          end
-        subject.run(events)
-        # wait for actions to complete
-        actions.assert_no_errors
-        expect(events.size).to eq(2)
-        e1, e2 = events
-        e1_message = e1.get("message")
-        e2_message = e2.get("message")
-
-        expect(e1.get("path")).to match(/J.txt/)
-        expect(e2.get("path")).to match(/K.txt/)
-        expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER}  line1.2-of-J#{TEST_FILE_DELIMITER}  line1.3-of-J")
-        expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER}  line1.2-of-K#{TEST_FILE_DELIMITER}  line1.3-of-K")
-      end
+        let(:file_path_target_field  ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
 
-      context "if auto_flush is enabled on the multiline codec" do
-        let(:mlconf) { { "auto_flush_interval" => 0.5 } }
-        let(:suffix)       { "M" }
-        it "an event is generated via auto_flush" do
+        subject { described_class.new(conf) }
+        let(:suffix)       { "J" }
+        let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
+        before do
+          mlconf.update("pattern" => "^\s", "what" => "previous")
+          conf.update(
+                "type" => "blah",
+                "path" => path_path,
+                "start_position" => "beginning",
+                "sincedb_path" => sincedb_path,
+                "stat_interval" => 0.05,
+                "codec" => mlcodec,
+                "file_sort_by" => "path",
+                "delimiter" => TEST_FILE_DELIMITER)
+
+          subject.register
+        end
+
+        it "collects separate multiple line events from each file" do
+          subject
           actions = RSpec::Sequencing
             .run_after(0.1, "create files") do
               File.open(tmpfile_path, "wb") do |fd|
-                fd.puts("line1.1-of-a")
-                fd.puts("  line1.2-of-a")
-                fd.puts("  line1.3-of-a")
+                fd.puts("line1.1-of-J")
+                fd.puts("  line1.2-of-J")
+                fd.puts("  line1.3-of-J")
+              end
+              File.open(tmpfile_path2, "wb") do |fd|
+                fd.puts("line1.1-of-K")
+                fd.puts("  line1.2-of-K")
+                fd.puts("  line1.3-of-K")
               end
             end
-            .then("wait for auto_flush") do
-              wait(2).for{events.size}.to eq(1), "events size is not 1"
+            .then("assert both files are mapped as identities and stop") do
+              wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
             end
             .then("stop") do
               subject.stop
@@ -431,10 +434,43 @@ describe LogStash::Inputs::File do
           subject.run(events)
           # wait for actions to complete
           actions.assert_no_errors
-          e1 = events.first
+          expect(events.size).to eq(2)
+          e1, e2 = events
           e1_message = e1.get("message")
-          expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
-          expect(e1.get("path")).to match(/M.txt$/)
+          e2_message = e2.get("message")
+
+          expect(e1.get(file_path_target_field)).to match(/J.txt/)
+          expect(e2.get(file_path_target_field)).to match(/K.txt/)
+          expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER}  line1.2-of-J#{TEST_FILE_DELIMITER}  line1.3-of-J")
+          expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER}  line1.2-of-K#{TEST_FILE_DELIMITER}  line1.3-of-K")
+        end
+
+        context "if auto_flush is enabled on the multiline codec" do
+          let(:mlconf) { { "auto_flush_interval" => 0.5 } }
+          let(:suffix)       { "M" }
+          it "an event is generated via auto_flush" do
+            actions = RSpec::Sequencing
+              .run_after(0.1, "create files") do
+                File.open(tmpfile_path, "wb") do |fd|
+                  fd.puts("line1.1-of-a")
+                  fd.puts("  line1.2-of-a")
+                  fd.puts("  line1.3-of-a")
+                end
+              end
+              .then("wait for auto_flush") do
+                wait(2).for{events.size}.to eq(1), "events size is not 1"
+              end
+              .then("stop") do
+                subject.stop
+              end
+            subject.run(events)
+            # wait for actions to complete
+            actions.assert_no_errors
+            e1 = events.first
+            e1_message = e1.get("message")
+            expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
+            expect(e1.get(file_path_target_field)).to match(/M.txt$/)
+          end
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2022-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement