logstash-input-file 4.0.5 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +25 -3
  3. data/JAR_VERSION +1 -0
  4. data/docs/index.asciidoc +195 -37
  5. data/lib/filewatch/bootstrap.rb +74 -0
  6. data/lib/filewatch/discoverer.rb +94 -0
  7. data/lib/filewatch/helper.rb +65 -0
  8. data/lib/filewatch/observing_base.rb +97 -0
  9. data/lib/filewatch/observing_read.rb +23 -0
  10. data/lib/filewatch/observing_tail.rb +22 -0
  11. data/lib/filewatch/read_mode/handlers/base.rb +81 -0
  12. data/lib/filewatch/read_mode/handlers/read_file.rb +47 -0
  13. data/lib/filewatch/read_mode/handlers/read_zip_file.rb +57 -0
  14. data/lib/filewatch/read_mode/processor.rb +117 -0
  15. data/lib/filewatch/settings.rb +67 -0
  16. data/lib/filewatch/sincedb_collection.rb +215 -0
  17. data/lib/filewatch/sincedb_record_serializer.rb +70 -0
  18. data/lib/filewatch/sincedb_value.rb +87 -0
  19. data/lib/filewatch/tail_mode/handlers/base.rb +124 -0
  20. data/lib/filewatch/tail_mode/handlers/create.rb +17 -0
  21. data/lib/filewatch/tail_mode/handlers/create_initial.rb +21 -0
  22. data/lib/filewatch/tail_mode/handlers/delete.rb +11 -0
  23. data/lib/filewatch/tail_mode/handlers/grow.rb +11 -0
  24. data/lib/filewatch/tail_mode/handlers/shrink.rb +20 -0
  25. data/lib/filewatch/tail_mode/handlers/timeout.rb +10 -0
  26. data/lib/filewatch/tail_mode/handlers/unignore.rb +37 -0
  27. data/lib/filewatch/tail_mode/processor.rb +209 -0
  28. data/lib/filewatch/watch.rb +107 -0
  29. data/lib/filewatch/watched_file.rb +226 -0
  30. data/lib/filewatch/watched_files_collection.rb +84 -0
  31. data/lib/filewatch/winhelper.rb +65 -0
  32. data/lib/jars/filewatch-1.0.0.jar +0 -0
  33. data/lib/logstash/inputs/delete_completed_file_handler.rb +9 -0
  34. data/lib/logstash/inputs/file.rb +162 -107
  35. data/lib/logstash/inputs/file_listener.rb +61 -0
  36. data/lib/logstash/inputs/log_completed_file_handler.rb +13 -0
  37. data/logstash-input-file.gemspec +5 -4
  38. data/spec/filewatch/buftok_spec.rb +24 -0
  39. data/spec/filewatch/reading_spec.rb +128 -0
  40. data/spec/filewatch/sincedb_record_serializer_spec.rb +71 -0
  41. data/spec/filewatch/spec_helper.rb +120 -0
  42. data/spec/filewatch/tailing_spec.rb +440 -0
  43. data/spec/filewatch/watched_file_spec.rb +38 -0
  44. data/spec/filewatch/watched_files_collection_spec.rb +73 -0
  45. data/spec/filewatch/winhelper_spec.rb +22 -0
  46. data/spec/fixtures/compressed.log.gz +0 -0
  47. data/spec/fixtures/compressed.log.gzip +0 -0
  48. data/spec/fixtures/invalid_utf8.gbk.log +2 -0
  49. data/spec/fixtures/no-final-newline.log +2 -0
  50. data/spec/fixtures/uncompressed.log +2 -0
  51. data/spec/{spec_helper.rb → helpers/spec_helper.rb} +14 -41
  52. data/spec/inputs/file_read_spec.rb +155 -0
  53. data/spec/inputs/{file_spec.rb → file_tail_spec.rb} +55 -52
  54. metadata +96 -28
@@ -1,8 +1,8 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-file'
4
- s.version = '4.0.5'
5
- s.licenses = ['Apache License (2.0)']
4
+ s.version = '4.1.0'
5
+ s.licenses = ['Apache-2.0']
6
6
  s.summary = "Streams events from files"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elastic"]
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
11
11
  s.require_paths = ["lib"]
12
12
 
13
13
  # Files
14
- s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
14
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "JAR_VERSION", "docs/**/*"]
15
15
 
16
16
  # Tests
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -24,11 +24,12 @@ Gem::Specification.new do |s|
24
24
 
25
25
  s.add_runtime_dependency 'logstash-codec-plain'
26
26
  s.add_runtime_dependency 'addressable'
27
- s.add_runtime_dependency 'filewatch', ['>= 0.8.1', '~> 0.8']
28
27
  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
29
28
 
30
29
  s.add_development_dependency 'stud', ['~> 0.0.19']
31
30
  s.add_development_dependency 'logstash-devutils'
32
31
  s.add_development_dependency 'logstash-codec-json'
33
32
  s.add_development_dependency 'rspec-sequencing'
33
+ s.add_development_dependency "rspec-wait"
34
+ s.add_development_dependency 'timecop'
34
35
  end
@@ -0,0 +1,24 @@
1
+ require_relative 'spec_helper'
2
+
3
+ describe FileWatch::BufferedTokenizer do
4
+
5
+ context "when using the default delimiter" do
6
+ it "splits the lines correctly" do
7
+ expect(subject.extract("hello\nworld\n")).to eq ["hello", "world"]
8
+ end
9
+
10
+ it "holds partial lines back until a token is found" do
11
+ buffer = described_class.new
12
+ expect(buffer.extract("hello\nwor")).to eq ["hello"]
13
+ expect(buffer.extract("ld\n")).to eq ["world"]
14
+ end
15
+ end
16
+
17
+ context "when passing a custom delimiter" do
18
+ subject { FileWatch::BufferedTokenizer.new("\r\n") }
19
+
20
+ it "splits the lines correctly" do
21
+ expect(subject.extract("hello\r\nworld\r\n")).to eq ["hello", "world"]
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,128 @@
1
+
2
+ require 'stud/temporary'
3
+ require_relative 'spec_helper'
4
+ require 'filewatch/observing_read'
5
+
6
+ module FileWatch
7
+ describe Watch do
8
+ before(:all) do
9
+ @thread_abort = Thread.abort_on_exception
10
+ Thread.abort_on_exception = true
11
+ end
12
+
13
+ after(:all) do
14
+ Thread.abort_on_exception = @thread_abort
15
+ end
16
+
17
+ let(:directory) { Stud::Temporary.directory }
18
+ let(:watch_dir) { ::File.join(directory, "*.log") }
19
+ let(:file_path) { ::File.join(directory, "1.log") }
20
+ let(:sincedb_path) { ::File.join(Stud::Temporary.directory, "reading.sdb") }
21
+ let(:stat_interval) { 0.1 }
22
+ let(:discover_interval) { 4 }
23
+ let(:start_new_files_at) { :end } # should be irrelevant for read mode
24
+ let(:opts) do
25
+ {
26
+ :stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
27
+ :delimiter => "\n", :discover_interval => discover_interval,
28
+ :ignore_older => 3600, :sincedb_path => sincedb_path
29
+ }
30
+ end
31
+ let(:observer) { TestObserver.new }
32
+ let(:reading) { ObservingRead.new(opts) }
33
+ let(:actions) do
34
+ RSpec::Sequencing.run_after(0.45, "quit after a short time") do
35
+ reading.quit
36
+ end
37
+ end
38
+
39
+ after do
40
+ FileUtils.rm_rf(directory) unless directory =~ /fixture/
41
+ end
42
+
43
+ context "when watching a directory with files" do
44
+ let(:directory) { Stud::Temporary.directory }
45
+ let(:watch_dir) { ::File.join(directory, "*.log") }
46
+ let(:file_path) { ::File.join(directory, "1.log") }
47
+
48
+ it "the file is read" do
49
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
50
+ actions.activate
51
+ reading.watch_this(watch_dir)
52
+ reading.subscribe(observer)
53
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
54
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
55
+ end
56
+ end
57
+
58
+ context "when watching a directory with files using striped reading" do
59
+ let(:directory) { Stud::Temporary.directory }
60
+ let(:watch_dir) { ::File.join(directory, "*.log") }
61
+ let(:file_path1) { ::File.join(directory, "1.log") }
62
+ let(:file_path2) { ::File.join(directory, "2.log") }
63
+ # use a chunk size that does not align with the line boundaries
64
+ let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1)}
65
+ let(:lines) { [] }
66
+ let(:observer) { TestObserver.new(lines) }
67
+
68
+ it "the files are read seemingly in parallel" do
69
+ File.open(file_path1, "w") { |file| file.write("string1\nstring2\n") }
70
+ File.open(file_path2, "w") { |file| file.write("stringA\nstringB\n") }
71
+ actions.activate
72
+ reading.watch_this(watch_dir)
73
+ reading.subscribe(observer)
74
+ if lines.first == "stringA"
75
+ expect(lines).to eq(%w(stringA string1 stringB string2))
76
+ else
77
+ expect(lines).to eq(%w(string1 stringA string2 stringB))
78
+ end
79
+ end
80
+ end
81
+
82
+ describe "reading fixtures" do
83
+ let(:directory) { FIXTURE_DIR }
84
+
85
+ context "for an uncompressed file" do
86
+ let(:watch_dir) { ::File.join(directory, "unc*.log") }
87
+ let(:file_path) { ::File.join(directory, 'uncompressed.log') }
88
+
89
+ it "the file is read" do
90
+ FileWatch.make_fixture_current(file_path)
91
+ actions.activate
92
+ reading.watch_this(watch_dir)
93
+ reading.subscribe(observer)
94
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
95
+ expect(observer.listener_for(file_path).lines.size).to eq(2)
96
+ end
97
+ end
98
+
99
+ context "for another uncompressed file" do
100
+ let(:watch_dir) { ::File.join(directory, "invalid*.log") }
101
+ let(:file_path) { ::File.join(directory, 'invalid_utf8.gbk.log') }
102
+
103
+ it "the file is read" do
104
+ FileWatch.make_fixture_current(file_path)
105
+ actions.activate
106
+ reading.watch_this(watch_dir)
107
+ reading.subscribe(observer)
108
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
109
+ expect(observer.listener_for(file_path).lines.size).to eq(2)
110
+ end
111
+ end
112
+
113
+ context "for a compressed file" do
114
+ let(:watch_dir) { ::File.join(directory, "compressed.*.gz") }
115
+ let(:file_path) { ::File.join(directory, 'compressed.log.gz') }
116
+
117
+ it "the file is read" do
118
+ FileWatch.make_fixture_current(file_path)
119
+ actions.activate
120
+ reading.watch_this(watch_dir)
121
+ reading.subscribe(observer)
122
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
123
+ expect(observer.listener_for(file_path).lines.size).to eq(2)
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,71 @@
1
+ # encoding: utf-8
2
+ require_relative 'spec_helper'
3
+ require 'filewatch/settings'
4
+ require 'filewatch/sincedb_record_serializer'
5
+
6
+ module FileWatch
7
+ describe SincedbRecordSerializer do
8
+ let(:opts) { Hash.new }
9
+ let(:io) { StringIO.new }
10
+ let(:db) { Hash.new }
11
+
12
+ subject { described_class.new(Settings.days_to_seconds(14)) }
13
+
14
+ context "deserialize from IO" do
15
+ it 'reads V1 records' do
16
+ io.write("5391299 1 4 12\n")
17
+ subject.deserialize(io) do |inode_struct, sincedb_value|
18
+ expect(inode_struct.inode).to eq("5391299")
19
+ expect(inode_struct.maj).to eq(1)
20
+ expect(inode_struct.min).to eq(4)
21
+ expect(sincedb_value.position).to eq(12)
22
+ end
23
+ end
24
+
25
+ it 'reads V2 records from an IO object' do
26
+ now = Time.now.to_f
27
+ io.write("5391299 1 4 12 #{now} /a/path/to/1.log\n")
28
+ subject.deserialize(io) do |inode_struct, sincedb_value|
29
+ expect(inode_struct.inode).to eq("5391299")
30
+ expect(inode_struct.maj).to eq(1)
31
+ expect(inode_struct.min).to eq(4)
32
+ expect(sincedb_value.position).to eq(12)
33
+ expect(sincedb_value.last_changed_at).to eq(now)
34
+ expect(sincedb_value.path_in_sincedb).to eq("/a/path/to/1.log")
35
+ end
36
+ end
37
+ end
38
+
39
+ context "serialize to IO" do
40
+ it "writes db entries" do
41
+ now = Time.now.to_f
42
+ inode_struct = InodeStruct.new("42424242", 2, 5)
43
+ sincedb_value = SincedbValue.new(42, now)
44
+ db[inode_struct] = sincedb_value
45
+ subject.serialize(db, io)
46
+ expect(io.string).to eq("42424242 2 5 42 #{now}\n")
47
+ end
48
+
49
+ it "does not write expired db entries to an IO object" do
50
+ twelve_days_ago = Time.now.to_f - (12.0*24*3600)
51
+ sixteen_days_ago = twelve_days_ago - (4.0*24*3600)
52
+ db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, twelve_days_ago)
53
+ db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, sixteen_days_ago)
54
+ subject.serialize(db, io)
55
+ expect(io.string).to eq("42424242 2 5 42 #{twelve_days_ago}\n")
56
+ end
57
+ end
58
+
59
+ context "given a non default `sincedb_clean_after`" do
60
+ it "does not write expired db entries to an IO object" do
61
+ subject.update_sincedb_value_expiry_from_days(2)
62
+ one_day_ago = Time.now.to_f - (1.0*24*3600)
63
+ three_days_ago = one_day_ago - (2.0*24*3600)
64
+ db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)
65
+ db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, three_days_ago)
66
+ subject.serialize(db, io)
67
+ expect(io.string).to eq("42424242 2 5 42 #{one_day_ago}\n")
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,120 @@
1
+ require "rspec_sequencing"
2
+ require 'rspec/wait'
3
+ require "logstash/devutils/rspec/spec_helper"
4
+ require "timecop"
5
+
6
+ def formatted_puts(text)
7
+ cfg = RSpec.configuration
8
+ return unless cfg.formatters.first.is_a?(
9
+ RSpec::Core::Formatters::DocumentationFormatter)
10
+ txt = cfg.format_docstrings_block.call(text)
11
+ cfg.output_stream.puts " #{txt}"
12
+ end
13
+
14
+ unless RSpec::Matchers.method_defined?(:receive_call_and_args)
15
+ RSpec::Matchers.define(:receive_call_and_args) do |m, args|
16
+ match do |actual|
17
+ actual.trace_for(m) == args
18
+ end
19
+
20
+ failure_message do
21
+ "Expecting method #{m} to receive: #{args} but got: #{actual.trace_for(m)}"
22
+ end
23
+ end
24
+ end
25
+
26
+ require 'filewatch/bootstrap'
27
+
28
+ module FileWatch
29
+
30
+ FIXTURE_DIR = File.join('spec', 'fixtures')
31
+
32
+ def self.make_file_older(path, seconds)
33
+ time = Time.now.to_f - seconds
34
+ ::File.utime(time, time, path)
35
+ end
36
+
37
+ def self.make_fixture_current(path, time = Time.now)
38
+ ::File.utime(time, time, path)
39
+ end
40
+
41
+ class TracerBase
42
+ def initialize
43
+ @tracer = []
44
+ end
45
+
46
+ def trace_for(symbol)
47
+ params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
48
+ params.empty? ? false : params
49
+ end
50
+
51
+ def clear
52
+ @tracer.clear
53
+ end
54
+ end
55
+
56
+ module NullCallable
57
+ def self.call
58
+ end
59
+ end
60
+
61
+ class TestObserver
62
+ class Listener
63
+ attr_reader :path, :lines, :calls
64
+
65
+ def initialize(path)
66
+ @path = path
67
+ @lines = []
68
+ @calls = []
69
+ end
70
+
71
+ def add_lines(lines)
72
+ @lines = lines
73
+ self
74
+ end
75
+
76
+ def accept(line)
77
+ @lines << line
78
+ @calls << :accept
79
+ end
80
+
81
+ def deleted
82
+ @calls << :delete
83
+ end
84
+
85
+ def opened
86
+ @calls << :open
87
+ end
88
+
89
+ def error
90
+ @calls << :error
91
+ end
92
+
93
+ def eof
94
+ @calls << :eof
95
+ end
96
+
97
+ def timed_out
98
+ @calls << :timed_out
99
+ end
100
+ end
101
+
102
+ attr_reader :listeners
103
+
104
+ def initialize(combined_lines = nil)
105
+ listener_proc = if combined_lines.nil?
106
+ lambda{|k| Listener.new(k) }
107
+ else
108
+ lambda{|k| Listener.new(k).add_lines(combined_lines) }
109
+ end
110
+ @listeners = Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
111
+ end
112
+
113
+ def listener_for(path)
114
+ @listeners[path]
115
+ end
116
+
117
+ def clear
118
+ @listeners.clear; end
119
+ end
120
+ end
@@ -0,0 +1,440 @@
1
+
2
+ require 'stud/temporary'
3
+ require_relative 'spec_helper'
4
+ require 'filewatch/observing_tail'
5
+
6
+ LogStash::Logging::Logger::configure_logging("WARN")
7
+ # LogStash::Logging::Logger::configure_logging("DEBUG")
8
+
9
+ module FileWatch
10
+ describe Watch do
11
+ before(:all) do
12
+ @thread_abort = Thread.abort_on_exception
13
+ Thread.abort_on_exception = true
14
+ end
15
+
16
+ after(:all) do
17
+ Thread.abort_on_exception = @thread_abort
18
+ end
19
+
20
+ let(:directory) { Stud::Temporary.directory }
21
+ let(:watch_dir) { ::File.join(directory, "*.log") }
22
+ let(:file_path) { ::File.join(directory, "1.log") }
23
+ let(:max) { 4095 }
24
+ let(:stat_interval) { 0.1 }
25
+ let(:discover_interval) { 4 }
26
+ let(:start_new_files_at) { :beginning }
27
+ let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
28
+ let(:opts) do
29
+ {
30
+ :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max,
31
+ :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path
32
+ }
33
+ end
34
+ let(:observer) { TestObserver.new }
35
+ let(:tailing) { ObservingTail.new(opts) }
36
+
37
+ after do
38
+ FileUtils.rm_rf(directory)
39
+ end
40
+
41
+ describe "max open files (set to 1)" do
42
+ let(:max) { 1 }
43
+ let(:file_path2) { File.join(directory, "2.log") }
44
+ let(:wait_before_quit) { 0.15 }
45
+ let(:stat_interval) { 0.01 }
46
+ let(:discover_interval) { 4 }
47
+ let(:actions) do
48
+ RSpec::Sequencing
49
+ .run_after(wait_before_quit, "quit after a short time") do
50
+ tailing.quit
51
+ end
52
+ end
53
+
54
+ before do
55
+ ENV["FILEWATCH_MAX_FILES_WARN_INTERVAL"] = "0"
56
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
57
+ File.open(file_path2, "wb") { |file| file.write("lineA\nlineB\n") }
58
+ end
59
+
60
+ context "when max_active is 1" do
61
+
62
+ it "without close_older set, opens only 1 file" do
63
+ actions.activate
64
+ tailing.watch_this(watch_dir)
65
+ tailing.subscribe(observer)
66
+ expect(tailing.settings.max_active).to eq(max)
67
+ file1_calls = observer.listener_for(file_path).calls
68
+ file2_calls = observer.listener_for(file_path2).calls
69
+ # file glob order is OS dependent
70
+ if file1_calls.empty?
71
+ expect(observer.listener_for(file_path2).lines).to eq(["lineA", "lineB"])
72
+ expect(file2_calls).to eq([:open, :accept, :accept])
73
+ else
74
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
75
+ expect(file1_calls).to eq([:open, :accept, :accept])
76
+ expect(file2_calls).to be_empty
77
+ end
78
+ end
79
+ end
80
+
81
+ context "when close_older is set" do
82
+ let(:wait_before_quit) { 0.8 }
83
+ let(:opts) { super.merge(:close_older => 0.2, :max_active => 1, :stat_interval => 0.1) }
84
+ it "opens both files" do
85
+ actions.activate
86
+ tailing.watch_this(watch_dir)
87
+ tailing.subscribe(observer)
88
+ expect(tailing.settings.max_active).to eq(1)
89
+ filelistener_1 = observer.listener_for(file_path)
90
+ filelistener_2 = observer.listener_for(file_path2)
91
+ expect(filelistener_2.calls).to eq([:open, :accept, :accept, :timed_out])
92
+ expect(filelistener_2.lines).to eq(["lineA", "lineB"])
93
+ expect(filelistener_1.calls).to eq([:open, :accept, :accept, :timed_out])
94
+ expect(filelistener_1.lines).to eq(["line1", "line2"])
95
+ end
96
+ end
97
+ end
98
+
99
+ context "when watching a directory with files" do
100
+ let(:start_new_files_at) { :beginning }
101
+ let(:actions) do
102
+ RSpec::Sequencing.run_after(0.45, "quit after a short time") do
103
+ tailing.quit
104
+ end
105
+ end
106
+
107
+ it "the file is read" do
108
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
109
+ actions.activate
110
+ tailing.watch_this(watch_dir)
111
+ tailing.subscribe(observer)
112
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept])
113
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
114
+ end
115
+ end
116
+
117
+ context "when watching a directory without files and one is added" do
118
+ let(:start_new_files_at) { :beginning }
119
+ before do
120
+ tailing.watch_this(watch_dir)
121
+ RSpec::Sequencing
122
+ .run_after(0.25, "create file") do
123
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
124
+ end
125
+ .then_after(0.45, "quit after a short time") do
126
+ tailing.quit
127
+ end
128
+ end
129
+
130
+ it "the file is read" do
131
+ tailing.subscribe(observer)
132
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept])
133
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
134
+ end
135
+ end
136
+
137
+ describe "given a previously discovered file" do
138
+ # these tests rely on the fact that the 'filepath' does not exist on disk
139
+ # it simulates that the user deleted the file
140
+ # so when a stat is taken on the file an error is raised
141
+ let(:quit_after) { 0.2 }
142
+ let(:stat) { double("stat", :size => 100, :ctime => Time.now, :mtime => Time.now, :ino => 234567, :dev_major => 3, :dev_minor => 2) }
143
+ let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
144
+
145
+ before do
146
+ tailing.watch.watched_files_collection.add(watched_file)
147
+ watched_file.initial_completed
148
+ end
149
+
150
+ context "when a close operation occurs" do
151
+ before { watched_file.close }
152
+ it "is removed from the watched_files_collection" do
153
+ expect(tailing.watch.watched_files_collection).not_to be_empty
154
+ RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
155
+ tailing.subscribe(observer)
156
+ expect(tailing.watch.watched_files_collection).to be_empty
157
+ expect(observer.listener_for(file_path).calls).to eq([:delete])
158
+ end
159
+ end
160
+
161
+ context "an ignore operation occurs" do
162
+ before { watched_file.ignore }
163
+ it "is removed from the watched_files_collection" do
164
+ RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
165
+ tailing.subscribe(observer)
166
+ expect(tailing.watch.watched_files_collection).to be_empty
167
+ expect(observer.listener_for(file_path).calls).to eq([:delete])
168
+ end
169
+ end
170
+
171
+ context "when subscribed and a watched file is no longer readable" do
172
+ before { watched_file.watch }
173
+ it "is removed from the watched_files_collection" do
174
+ RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
175
+ tailing.subscribe(observer)
176
+ expect(tailing.watch.watched_files_collection).to be_empty
177
+ expect(observer.listener_for(file_path).calls).to eq([:delete])
178
+ end
179
+ end
180
+
181
+ context "when subscribed and an active file is no longer readable" do
182
+ before { watched_file.activate }
183
+ it "is removed from the watched_files_collection" do
184
+ RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
185
+ tailing.subscribe(observer)
186
+ expect(tailing.watch.watched_files_collection).to be_empty
187
+ expect(observer.listener_for(file_path).calls).to eq([:delete])
188
+ end
189
+ end
190
+ end
191
+
192
+ context "when a processed file shrinks" do
193
+ let(:discover_interval) { 100 }
194
+ before do
195
+ RSpec::Sequencing
196
+ .run("create file") do
197
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\nline3\nline4\n") }
198
+ end
199
+ .then_after(0.25, "start watching after files are written") do
200
+ tailing.watch_this(watch_dir)
201
+ end
202
+ .then_after(0.25, "truncate file and write new content") do
203
+ File.truncate(file_path, 0)
204
+ File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") }
205
+ end
206
+ .then_after(0.25, "quit after a short time") do
207
+ tailing.quit
208
+ end
209
+ end
210
+
211
+ it "new changes to the shrunk file are read from the beginning" do
212
+ tailing.subscribe(observer)
213
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept, :accept, :accept])
214
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4", "lineA", "lineB"])
215
+ end
216
+ end
217
+
218
+ context "when watching a directory with files and a file is renamed to not match glob" do
219
+ let(:new_file_path) { file_path + ".old" }
220
+ before do
221
+ RSpec::Sequencing
222
+ .run("create file") do
223
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
224
+ end
225
+ .then_after(0.25, "start watching after files are written") do
226
+ tailing.watch_this(watch_dir)
227
+ end
228
+ .then_after(0.55, "rename file") do
229
+ FileUtils.mv(file_path, new_file_path)
230
+ end
231
+ .then_after(0.55, "then write to renamed file") do
232
+ File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") }
233
+ end
234
+ .then_after(0.45, "quit after a short time") do
235
+ tailing.quit
236
+ end
237
+ end
238
+
239
+ it "changes to the renamed file are not read" do
240
+ tailing.subscribe(observer)
241
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :delete])
242
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
243
+ expect(observer.listener_for(new_file_path).calls).to eq([])
244
+ expect(observer.listener_for(new_file_path).lines).to eq([])
245
+ end
246
+ end
247
+
248
+ context "when watching a directory with files and a file is renamed to match glob" do
249
+ let(:new_file_path) { file_path + "2.log" }
250
+ let(:opts) { super.merge(:close_older => 0) }
251
+ before do
252
+ RSpec::Sequencing
253
+ .run("create file") do
254
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
255
+ end
256
+ .then_after(0.15, "start watching after files are written") do
257
+ tailing.watch_this(watch_dir)
258
+ end
259
+ .then_after(0.25, "rename file") do
260
+ FileUtils.mv(file_path, new_file_path)
261
+ end
262
+ .then("then write to renamed file") do
263
+ File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") }
264
+ end
265
+ .then_after(0.55, "quit after a short time") do
266
+ tailing.quit
267
+ end
268
+ end
269
+
270
+ it "the first set of lines are not re-read" do
271
+ tailing.subscribe(observer)
272
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
273
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :delete])
274
+ expect(observer.listener_for(new_file_path).lines).to eq(["line3", "line4"])
275
+ expect(observer.listener_for(new_file_path).calls).to eq([:open, :accept, :accept, :timed_out])
276
+ end
277
+ end
278
+
279
+ context "when watching a directory with files and data is appended" do
280
+ before do
281
+ RSpec::Sequencing
282
+ .run("create file") do
283
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
284
+ end
285
+ .then_after(0.25, "start watching after file is written") do
286
+ tailing.watch_this(watch_dir)
287
+ end
288
+ .then_after(0.45, "append more lines to the file") do
289
+ File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
290
+ end
291
+ .then_after(0.45, "quit after a short time") do
292
+ tailing.quit
293
+ end
294
+ end
295
+
296
+ it "appended lines are read after an EOF" do
297
+ tailing.subscribe(observer)
298
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept])
299
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"])
300
+ end
301
+ end
302
+
303
+ context "when close older expiry is enabled" do
304
+ let(:opts) { super.merge(:close_older => 1) }
305
+ before do
306
+ RSpec::Sequencing
307
+ .run("create file") do
308
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
309
+ end
310
+ .then("start watching before file ages more than close_older") do
311
+ tailing.watch_this(watch_dir)
312
+ end
313
+ .then_after(2.1, "quit after allowing time to close the file") do
314
+ tailing.quit
315
+ end
316
+ end
317
+
318
+ it "lines are read and the file times out" do
319
+ tailing.subscribe(observer)
320
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
321
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
322
+ end
323
+ end
324
+
325
+ context "when close older expiry is enabled and after timeout the file is appended-to" do
326
+ let(:opts) { super.merge(:close_older => 1) }
327
+ before do
328
+ RSpec::Sequencing
329
+ .run("create file") do
330
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
331
+ end
332
+ .then("start watching before file ages more than close_older") do
333
+ tailing.watch_this(watch_dir)
334
+ end
335
+ .then_after(2.1, "append more lines to file after file ages more than close_older") do
336
+ File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
337
+ end
338
+ .then_after(2.1, "quit after allowing time to close the file") do
339
+ tailing.quit
340
+ end
341
+ end
342
+
343
+ it "all lines are read" do
344
+ tailing.subscribe(observer)
345
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :open, :accept, :accept, :timed_out])
346
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"])
347
+ end
348
+ end
349
+
350
+ context "when ignore older expiry is enabled and all files are already expired" do
351
+ let(:opts) { super.merge(:ignore_older => 1) }
352
+ before do
353
+ RSpec::Sequencing
354
+ .run("create file older than ignore_older and watch") do
355
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
356
+ FileWatch.make_file_older(file_path, 15)
357
+ tailing.watch_this(watch_dir)
358
+ end
359
+ .then_after(1.1, "quit") do
360
+ tailing.quit
361
+ end
362
+ end
363
+
364
+ it "no files are read" do
365
+ tailing.subscribe(observer)
366
+ expect(observer.listener_for(file_path).calls).to eq([])
367
+ expect(observer.listener_for(file_path).lines).to eq([])
368
+ end
369
+ end
370
+
371
+ context "when ignore_older is less than close_older and all files are not expired" do
372
+ let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.5) }
373
+ before do
374
+ RSpec::Sequencing
375
+ .run("create file") do
376
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
377
+ end
378
+ .then("start watching before file age reaches ignore_older") do
379
+ tailing.watch_this(watch_dir)
380
+ end
381
+ .then_after(1.75, "quit after allowing time to close the file") do
382
+ tailing.quit
383
+ end
384
+ end
385
+
386
+ it "reads lines normally" do
387
+ tailing.subscribe(observer)
388
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
389
+ expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
390
+ end
391
+ end
392
+
393
+ context "when ignore_older is less than close_older and all files are expired" do
394
+ let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) }
395
+ before do
396
+ RSpec::Sequencing
397
+ .run("create file older than ignore_older and watch") do
398
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
399
+ FileWatch.make_file_older(file_path, 15)
400
+ tailing.watch_this(watch_dir)
401
+ end
402
+ .then_after(1.5, "quit after allowing time to check the files") do
403
+ tailing.quit
404
+ end
405
+ end
406
+
407
+ it "no files are read" do
408
+ tailing.subscribe(observer)
409
+ expect(observer.listener_for(file_path).calls).to eq([])
410
+ expect(observer.listener_for(file_path).lines).to eq([])
411
+ end
412
+ end
413
+
414
+ context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do
415
+ let(:opts) { super.merge(:ignore_older => 20, :close_older => 1) }
416
+ before do
417
+ RSpec::Sequencing
418
+ .run("create file older than ignore_older and watch") do
419
+ File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
420
+ FileWatch.make_file_older(file_path, 25)
421
+ tailing.watch_this(watch_dir)
422
+ end
423
+ .then_after(0.15, "append more lines to file after file ages more than ignore_older") do
424
+ File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
425
+ end
426
+ .then_after(1.25, "quit after allowing time to close the file") do
427
+ tailing.quit
428
+ end
429
+ end
430
+
431
+ it "reads the added lines only" do
432
+ tailing.subscribe(observer)
433
+ expect(observer.listener_for(file_path).lines).to eq(["line3", "line4"])
434
+ expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
435
+ end
436
+ end
437
+ end
438
+ end
439
+
440
+