logstash-input-file 4.4.2 → 4.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7d65df8a33a836b3e91bdd9c1d2edd86532dfb521091aa78324807e630e472ec
4
- data.tar.gz: 4bbca22cb03e662490c278d3f0d827488d6563c58bd73b37a4ee59f1ad84c1cd
3
+ metadata.gz: 26577b41ecc7118c3f4ec2dba011fe730e44d4a74c0f45b19875ba331922bb24
4
+ data.tar.gz: e799e9a5e149662f3e1ffa4e902bb8c789d8d5eb7f1b90c1776445571b358a1b
5
5
  SHA512:
6
- metadata.gz: a600671c2564ba9d81bed3078e9d99c82079dcd80a67143182102c268ac5396669b58a8dcfb2858c35bf698e77b06c12c546ec5f60569eb98472aa3c8086ad80
7
- data.tar.gz: b3b283d5b09cbe72198921992834fd7edadf4620123c1c2912d3179ff2247ff4b8dc5c3c0f09484af76d83253bbf2173fe165e3401f42ce37762a26e1db82764
6
+ metadata.gz: b8ce5e67051bb9d04947d3b3e72475f2929cecf475a5aa1196556fb63fcd37c44e931ebdff1e47fa21f01030bf2a5b15a6301658dbc249633b613c681b942fd9
7
+ data.tar.gz: 44dc2403a6ca1dcc5c0660404e4419d5acd1f020d376cd3be8fc7d46533335c10ba10e6c0510eb9c4afa3685b281e5a4a8cfb201ac2882c500f42b3bd54681ef
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 4.4.3
2
+ - Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed. [#307](https://github.com/logstash-plugins/logstash-input-file/pull/307)
3
+
1
4
  ## 4.4.2
2
5
  - Doc: Fix attribute by removing extra character [#310](https://github.com/logstash-plugins/logstash-input-file/pull/310)
3
6
 
@@ -2,9 +2,19 @@
2
2
 
3
3
  module FileWatch module ReadMode module Handlers
4
4
  class ReadFile < Base
5
+
6
+ # seek file to which ever is furthest: either current bytes read or sincedb position
7
+ private
8
+ def seek_to_furthest_position(watched_file)
9
+ previous_pos = sincedb_collection.find(watched_file).position
10
+ watched_file.file_seek([watched_file.bytes_read, previous_pos].max)
11
+ end
12
+
13
+ public
5
14
  def handle_specifically(watched_file)
6
15
  if open_file(watched_file)
7
16
  add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
17
+ seek_to_furthest_position(watched_file)
8
18
  loop do
9
19
  break if quit?
10
20
  loop_control = watched_file.loop_control_adjusted_for_stat_size
Binary file
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-file'
4
- s.version = '4.4.2'
4
+ s.version = '4.4.3'
5
5
  s.licenses = ['Apache-2.0']
6
6
  s.summary = "Streams events from files"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -36,5 +36,45 @@ module FileWatch
36
36
  processor.read_file(watched_file)
37
37
  end
38
38
  end
39
+
40
+ context "when restart from existing sincedb" do
41
+ let(:settings) do
42
+ Settings.from_options(
43
+ :sincedb_write_interval => 0,
44
+ :sincedb_path => File::NULL,
45
+ :file_chunk_size => 10
46
+ )
47
+ end
48
+
49
+ let(:processor) { double("fake processor") }
50
+ let(:observer) { TestObserver.new }
51
+ let(:watch) { double("watch") }
52
+
53
+ before(:each) {
54
+ allow(watch).to receive(:quit?).and_return(false)#.and_return(false).and_return(true)
55
+ allow(processor).to receive(:watch).and_return(watch)
56
+ }
57
+
58
+ it "read from where it left" do
59
+ listener = observer.listener_for(Pathname.new(pathname).to_path)
60
+ sut = ReadMode::Handlers::ReadFile.new(processor, sdb_collection, observer, settings)
61
+
62
+ # simulate a previous partial read of the file
63
+ sincedb_value = SincedbValue.new(0)
64
+ sincedb_value.set_watched_file(watched_file)
65
+ sdb_collection.set(watched_file.sincedb_key, sincedb_value)
66
+
67
+
68
+ # simulate a consumption of first line, (size + newline) bytes
69
+ sdb_collection.increment(watched_file.sincedb_key, File.readlines(pathname)[0].size + 2)
70
+
71
+ # exercise
72
+ sut.handle(watched_file)
73
+
74
+ # verify
75
+ expect(listener.lines.size).to eq(1)
76
+ expect(listener.lines[0]).to start_with("2010-03-12 23:51:21 SEA4 192.0.2.222 play 3914 OK")
77
+ end
78
+ end
39
79
  end
40
80
  end
@@ -80,6 +80,8 @@ module FileWatch
80
80
  multiplier = amount / string.length
81
81
  string * multiplier
82
82
  end
83
+ def sysseek(offset, whence)
84
+ end
83
85
  end
84
86
 
85
87
  FIXTURE_DIR = File.join('spec', 'fixtures')
@@ -31,7 +31,13 @@ module FileInput
31
31
 
32
32
  def trace_for(symbol)
33
33
  params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
34
- params.empty? ? false : params
34
+ if params.empty?
35
+ false
36
+ else
37
+ # merge all params with same key
38
+ # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
39
+ params.reduce {|b1, b2| b1 and b2}
40
+ end
35
41
  end
36
42
 
37
43
  def clear
@@ -332,7 +332,7 @@ describe LogStash::Inputs::File do
332
332
  .then("wait accept") do
333
333
  wait(0.75).for {
334
334
  subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)
335
- }.to eq([true]), "accept didn't"
335
+ }.to eq(true), "accept didn't"
336
336
  end
337
337
  .then("request a stop") do
338
338
  # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation
@@ -341,12 +341,11 @@ describe LogStash::Inputs::File do
341
341
  .then("wait for auto_flush") do
342
342
  wait(2).for {
343
343
  subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)
344
- .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
345
344
  }.to eq(true), "autoflush didn't"
346
345
  end
347
346
  subject.run(events)
348
347
  actions.assert_no_errors
349
- expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true])
348
+ expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq(true)
350
349
  end
351
350
  end
352
351
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.4.2
4
+ version: 4.4.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-05-11 00:00:00.000000000 Z
11
+ date: 2022-06-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement