logstash-input-file 4.1.15 → 4.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3c11c5972a494975b2acfa7d7e4f67e0ca4720ff9ec78c864febf764c9b030b
-  data.tar.gz: f80d8c34dd0848c07029199a6c57d39e25c343e32c499ecb4d350bfd50564119
+  metadata.gz: dfc29a2b231d1d04667f304746d22fd1beec91cabeb8f6e05772395bc45e86a5
+  data.tar.gz: 952de67a21b89420a91e787935ed0617a798a4c738632f0870d1c54c1162b110
 SHA512:
-  metadata.gz: '00392e4f07fe9fdfd732419c4b56b680f5b07aa7e97149ba7af91bca8aee128f396ef2330ec46814b7b3fbc9b52fcc58766a27d50b72bbf28c9b7475dc0aeda0'
-  data.tar.gz: 76079df610dfe177fd5c538b8ff5325d064f97826af30d441bd9abe54bfc1197f2178ba88862d6f144540a776493cce44efe130119ee8cc8e39c14492203f534
+  metadata.gz: 10b29419ea0fd69945fd0f9634d1a5c247516c14cec0b2d22e48155d4a53e71fd83551ffdd8df3dbff71e560bd5ba649deed94c549be92f582d05ec36c329ce1
+  data.tar.gz: 16b160f663671c865c72a09b991c6f58bc915b29266d21ffa9f2c5130f62ad1feb469d9cb850f7997cd5bd6a8ae261e24920892ec27cc1ae6196dd4bf93e2b9a

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.1.16
+  - Added configuration setting exit_after_read to read to EOF and terminate
+  the input [#240](https://github.com/logstash-plugins/logstash-input-file/pull/240)
+
 ## 4.1.15
   - Fixed bug in conversion of sincedb_clean_after setting [#257](https://github.com/logstash-plugins/logstash-input-file/pull/257)
 

data/docs/index.asciidoc

@@ -168,6 +168,7 @@ see <<plugins-{type}s-{plugin}-string_duration,string_duration>> for the details
 | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-discover_interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-exclude>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-exit_after_read>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-file_chunk_count>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-file_chunk_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-file_completed_action>> |<<string,string>>, one of `["delete", "log", "log_and_delete"]`|No
@@ -241,6 +242,19 @@ In Tail mode, you might want to exclude gzipped files:
 [source,ruby]
     exclude => "*.gz"
 
+[id="plugins-{type}s-{plugin}-exit_after_read"]
+===== `exit_after_read`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+This option can be used in `read` mode to enforce closing all watchers when file gets read.
+Can be used in situation when content of the file is static and won't change during execution.
+When set to `true` it also disables active discovery of the files - only files that were in 
+the directories when process was started will be read.
+It supports `sincedb` entries. When file was processed once, then modified - next run will only
+read newly added entries.
+
 [id="plugins-{type}s-{plugin}-file_chunk_count"]
 ===== `file_chunk_count`
 

data/lib/filewatch/read_mode/processor.rb

@@ -103,10 +103,21 @@ module FileWatch module ReadMode
         else
           read_file(watched_file)
         end
+        
+        if @settings.exit_after_read
+          common_detach_when_allread(watched_file)
+        end
         # handlers take care of closing and unwatching
       end
     end
 
+    def common_detach_when_allread(watched_file)
+      watched_file.unwatch
+      watched_file.listener.reading_completed
+      deletable_filepaths << watched_file.path
+      logger.trace("Whole file read: #{watched_file.path}, removing from collection")
+    end
+
     def common_deleted_reaction(watched_file, action)
       # file has gone away or we can't read it anymore.
       watched_file.unwatch

data/lib/filewatch/settings.rb

@@ -8,6 +8,7 @@ module FileWatch
     attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size
     attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration
     attr_reader :file_sort_by, :file_sort_direction
+    attr_reader :exit_after_read
 
     def self.from_options(opts)
       new.add_options(opts)
@@ -50,6 +51,7 @@ module FileWatch
       @sincedb_expiry_duration =  @opts.fetch(:sincedb_clean_after)
       @file_sort_by = @opts[:file_sort_by]
       @file_sort_direction = @opts[:file_sort_direction]
+      @exit_after_read = @opts[:exit_after_read]
       self
     end
 

data/lib/filewatch/watch.rb

@@ -43,10 +43,11 @@ module FileWatch
       reset_quit
       until quit?
         iterate_on_state
+        # Don't discover new files when files to read are known at the beginning
         break if quit?
         sincedb_collection.write_if_requested
         glob += 1
-        if glob == interval
+        if glob == interval && !@settings.exit_after_read
           discover
           glob = 0
         end
@@ -76,7 +77,10 @@ module FileWatch
     end
 
     def quit?
-      @quit.true?
+      if @settings.exit_after_read
+        @exit = @watched_files_collection.empty?
+      end
+      @quit.true? || @exit
     end
 
     private

data/lib/logstash/inputs/file.rb

@@ -222,6 +222,11 @@ class File < LogStash::Inputs::Base
   # perhaps path + asc will help to achieve the goal of controlling the order of file ingestion
   config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc"
 
+  # When in 'read' mode - this option is closing all file watchers when EOF is hit
+  # This option also disables discovery of new/changes files. It works only on files found at the beginning
+  # Sincedb still works, if you run LS once again after doing some changes - only new values will be read
+  config :exit_after_read, :validate => :boolean, :default => false
+
   public
 
   class << self
@@ -260,6 +265,7 @@ class File < LogStash::Inputs::Base
       :file_chunk_size => @file_chunk_size,
       :file_sort_by => @file_sort_by,
       :file_sort_direction => @file_sort_direction,
+      :exit_after_read => @exit_after_read,
     }
 
     @completed_file_handlers = []
@@ -280,7 +286,7 @@ class File < LogStash::Inputs::Base
         raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
       end
     end
-
+    
     @filewatch_config[:sincedb_path] = @sincedb_path
 
     @filewatch_config[:start_new_files_at] = @start_position.to_sym
@@ -301,6 +307,9 @@ class File < LogStash::Inputs::Base
     end
 
     if tail_mode?
+      if @exit_after_read
+        raise ArgumentError.new('The "exit_after_read" setting only works when the "mode" is set to "read"')
+      end
       @watcher_class = FileWatch::ObservingTail
     else
       @watcher_class = FileWatch::ObservingRead

data/lib/logstash/inputs/file_listener.rb

@@ -21,6 +21,9 @@ module LogStash module Inputs
     def error
     end
 
+    def reading_completed
+    end
+
     def timed_out
       input.codec.evict(path)
     end

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.15'
+  s.version         = '4.1.16'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', '~> 1.3'
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'rspec-sequencing'
   s.add_development_dependency "rspec-wait"

data/spec/filewatch/reading_spec.rb

@@ -147,6 +147,65 @@ module FileWatch
       end
     end
 
+    context "when watching a directory with files using exit_after_read" do
+      let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) }
+      let(:file_path3) { ::File.join(directory, "3.log") }
+      let(:file_path4) { ::File.join(directory, "4.log") }
+      let(:file_path5) { ::File.join(directory, "5.log") }
+      let(:lines) { [] }
+      let(:observer) { TestObserver.new(lines) }
+      let(:listener3) { observer.listener_for(file_path3) }
+      let(:file_path6) { ::File.join(directory, "6.log") }
+      let(:listener6) { observer.listener_for(file_path6) }
+
+      it "the file is read" do
+        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        expect(listener3.lines).to eq(["line1", "line2"])
+      end
+      it "multiple files are read" do
+        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"])
+      end
+      it "multiple files are read even if max_open_files is smaller then number of files" do
+        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
+        File.open(file_path5, "w") { |file| file.write("line5\nline6\n") }
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"])
+      end
+      it "file as marked as reading_completed" do
+        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
+      end
+      it "sincedb works correctly" do
+        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        sincedb_record_fields = File.read(sincedb_path).split(" ")
+        position_field_index = 3
+        expect(sincedb_record_fields[position_field_index]).to eq("12")
+      end
+      it "does not include new files added after start" do
+        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        File.open(file_path6, "w") { |file|  file.write("foob\nbar\n") }
+        expect(listener3.lines).to eq(["line1", "line2"])
+        expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
+        expect(listener6.calls).to eq([])
+
+      end
+    
+    end
+
     describe "reading fixtures" do
       let(:directory) { FIXTURE_DIR }
       let(:actions) do

data/spec/filewatch/spec_helper.rb

@@ -152,6 +152,10 @@ module FileWatch
       def timed_out
         @calls << :timed_out
       end
+
+      def reading_completed
+        @calls << :reading_completed
+      end
     end
 
     attr_reader :listeners

data/spec/filewatch/tailing_spec.rb

@@ -448,7 +448,7 @@ module FileWatch
             FileUtils.mv(file_path2, file_path3)
           end
           .then("wait") do
-            wait(2).for do
+            wait(4).for do
               listener1.lines.size == 32 && listener2.calls == [:delete] && listener3.calls == [:open, :accept, :timed_out]
             end.to eq(true), "listener1.lines != 32 or listener2.calls != [:delete] or listener3.calls != [:open, :accept, :timed_out]"
           end

data/spec/inputs/file_read_spec.rb

@@ -75,6 +75,40 @@ describe LogStash::Inputs::File do
       end
       expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world")
     end
+
+    it "should read whole file when exit_after_read is set to true" do
+      directory = Stud::Temporary.directory
+      tmpfile_path = ::File.join(directory, "B.log")
+      sincedb_path = ::File.join(directory, "readmode_B_sincedb.txt")
+      path_path = ::File.join(directory, "*.log")
+
+      conf = <<-CONFIG
+        input {
+          file {
+            id => "foo"
+            path => "#{path_path}"
+            sincedb_path => "#{sincedb_path}"
+            delimiter => "|"
+            mode => "read"
+            file_completed_action => "delete"
+            exit_after_read => true
+          }
+        }
+      CONFIG
+
+      File.open(tmpfile_path, "a") do |fd|
+        fd.write("exit|after|end")
+        fd.fsync
+      end
+
+      events = input(conf) do |pipeline, queue|
+        wait(0.5).for{File.exist?(tmpfile_path)}.to be_falsey
+        3.times.collect { queue.pop }
+      end
+
+      expect(events.map{|e| e.get("message")}).to contain_exactly("exit", "after", "end")
+    end
+
   end
 
   describe "reading fixtures" do

data/spec/inputs/file_tail_spec.rb

@@ -175,7 +175,7 @@ describe LogStash::Inputs::File do
     context "when sincedb_path is a directory" do
       let(:name) { "E" }
       subject { LogStash::Inputs::File.new("path" => path_path, "sincedb_path" => directory) }
-
+      
       after :each do
         FileUtils.rm_rf(sincedb_path)
       end
@@ -184,6 +184,15 @@ describe LogStash::Inputs::File do
         expect { subject.register }.to raise_error(ArgumentError)
       end
     end
+
+    context "when mode it set to tail and exit_after_read equals true" do
+        subject { LogStash::Inputs::File.new("path" => path_path, "exit_after_read" => true, "mode" => "tail") }
+
+      it "should raise exception" do
+        expect { subject.register }.to raise_error(ArgumentError)
+      end
+    end
+
   end
 
   describe "testing with new, register, run and stop" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.15
+  version: 4.1.16
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2020-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -89,17 +89,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: