logstash-input-file 4.0.5 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -3
- data/JAR_VERSION +1 -0
- data/docs/index.asciidoc +195 -37
- data/lib/filewatch/bootstrap.rb +74 -0
- data/lib/filewatch/discoverer.rb +94 -0
- data/lib/filewatch/helper.rb +65 -0
- data/lib/filewatch/observing_base.rb +97 -0
- data/lib/filewatch/observing_read.rb +23 -0
- data/lib/filewatch/observing_tail.rb +22 -0
- data/lib/filewatch/read_mode/handlers/base.rb +81 -0
- data/lib/filewatch/read_mode/handlers/read_file.rb +47 -0
- data/lib/filewatch/read_mode/handlers/read_zip_file.rb +57 -0
- data/lib/filewatch/read_mode/processor.rb +117 -0
- data/lib/filewatch/settings.rb +67 -0
- data/lib/filewatch/sincedb_collection.rb +215 -0
- data/lib/filewatch/sincedb_record_serializer.rb +70 -0
- data/lib/filewatch/sincedb_value.rb +87 -0
- data/lib/filewatch/tail_mode/handlers/base.rb +124 -0
- data/lib/filewatch/tail_mode/handlers/create.rb +17 -0
- data/lib/filewatch/tail_mode/handlers/create_initial.rb +21 -0
- data/lib/filewatch/tail_mode/handlers/delete.rb +11 -0
- data/lib/filewatch/tail_mode/handlers/grow.rb +11 -0
- data/lib/filewatch/tail_mode/handlers/shrink.rb +20 -0
- data/lib/filewatch/tail_mode/handlers/timeout.rb +10 -0
- data/lib/filewatch/tail_mode/handlers/unignore.rb +37 -0
- data/lib/filewatch/tail_mode/processor.rb +209 -0
- data/lib/filewatch/watch.rb +107 -0
- data/lib/filewatch/watched_file.rb +226 -0
- data/lib/filewatch/watched_files_collection.rb +84 -0
- data/lib/filewatch/winhelper.rb +65 -0
- data/lib/jars/filewatch-1.0.0.jar +0 -0
- data/lib/logstash/inputs/delete_completed_file_handler.rb +9 -0
- data/lib/logstash/inputs/file.rb +162 -107
- data/lib/logstash/inputs/file_listener.rb +61 -0
- data/lib/logstash/inputs/log_completed_file_handler.rb +13 -0
- data/logstash-input-file.gemspec +5 -4
- data/spec/filewatch/buftok_spec.rb +24 -0
- data/spec/filewatch/reading_spec.rb +128 -0
- data/spec/filewatch/sincedb_record_serializer_spec.rb +71 -0
- data/spec/filewatch/spec_helper.rb +120 -0
- data/spec/filewatch/tailing_spec.rb +440 -0
- data/spec/filewatch/watched_file_spec.rb +38 -0
- data/spec/filewatch/watched_files_collection_spec.rb +73 -0
- data/spec/filewatch/winhelper_spec.rb +22 -0
- data/spec/fixtures/compressed.log.gz +0 -0
- data/spec/fixtures/compressed.log.gzip +0 -0
- data/spec/fixtures/invalid_utf8.gbk.log +2 -0
- data/spec/fixtures/no-final-newline.log +2 -0
- data/spec/fixtures/uncompressed.log +2 -0
- data/spec/{spec_helper.rb → helpers/spec_helper.rb} +14 -41
- data/spec/inputs/file_read_spec.rb +155 -0
- data/spec/inputs/{file_spec.rb → file_tail_spec.rb} +55 -52
- metadata +96 -28
data/logstash-input-file.gemspec
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-file'
|
4
|
-
s.version = '4.0
|
5
|
-
s.licenses = ['Apache
|
4
|
+
s.version = '4.1.0'
|
5
|
+
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ["Elastic"]
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.require_paths = ["lib"]
|
12
12
|
|
13
13
|
# Files
|
14
|
-
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "JAR_VERSION", "docs/**/*"]
|
15
15
|
|
16
16
|
# Tests
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
@@ -24,11 +24,12 @@ Gem::Specification.new do |s|
|
|
24
24
|
|
25
25
|
s.add_runtime_dependency 'logstash-codec-plain'
|
26
26
|
s.add_runtime_dependency 'addressable'
|
27
|
-
s.add_runtime_dependency 'filewatch', ['>= 0.8.1', '~> 0.8']
|
28
27
|
s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
|
29
28
|
|
30
29
|
s.add_development_dependency 'stud', ['~> 0.0.19']
|
31
30
|
s.add_development_dependency 'logstash-devutils'
|
32
31
|
s.add_development_dependency 'logstash-codec-json'
|
33
32
|
s.add_development_dependency 'rspec-sequencing'
|
33
|
+
s.add_development_dependency "rspec-wait"
|
34
|
+
s.add_development_dependency 'timecop'
|
34
35
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require_relative 'spec_helper'
|
2
|
+
|
3
|
+
describe FileWatch::BufferedTokenizer do
|
4
|
+
|
5
|
+
context "when using the default delimiter" do
|
6
|
+
it "splits the lines correctly" do
|
7
|
+
expect(subject.extract("hello\nworld\n")).to eq ["hello", "world"]
|
8
|
+
end
|
9
|
+
|
10
|
+
it "holds partial lines back until a token is found" do
|
11
|
+
buffer = described_class.new
|
12
|
+
expect(buffer.extract("hello\nwor")).to eq ["hello"]
|
13
|
+
expect(buffer.extract("ld\n")).to eq ["world"]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
context "when passing a custom delimiter" do
|
18
|
+
subject { FileWatch::BufferedTokenizer.new("\r\n") }
|
19
|
+
|
20
|
+
it "splits the lines correctly" do
|
21
|
+
expect(subject.extract("hello\r\nworld\r\n")).to eq ["hello", "world"]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
|
2
|
+
require 'stud/temporary'
|
3
|
+
require_relative 'spec_helper'
|
4
|
+
require 'filewatch/observing_read'
|
5
|
+
|
6
|
+
module FileWatch
|
7
|
+
describe Watch do
|
8
|
+
before(:all) do
|
9
|
+
@thread_abort = Thread.abort_on_exception
|
10
|
+
Thread.abort_on_exception = true
|
11
|
+
end
|
12
|
+
|
13
|
+
after(:all) do
|
14
|
+
Thread.abort_on_exception = @thread_abort
|
15
|
+
end
|
16
|
+
|
17
|
+
let(:directory) { Stud::Temporary.directory }
|
18
|
+
let(:watch_dir) { ::File.join(directory, "*.log") }
|
19
|
+
let(:file_path) { ::File.join(directory, "1.log") }
|
20
|
+
let(:sincedb_path) { ::File.join(Stud::Temporary.directory, "reading.sdb") }
|
21
|
+
let(:stat_interval) { 0.1 }
|
22
|
+
let(:discover_interval) { 4 }
|
23
|
+
let(:start_new_files_at) { :end } # should be irrelevant for read mode
|
24
|
+
let(:opts) do
|
25
|
+
{
|
26
|
+
:stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
|
27
|
+
:delimiter => "\n", :discover_interval => discover_interval,
|
28
|
+
:ignore_older => 3600, :sincedb_path => sincedb_path
|
29
|
+
}
|
30
|
+
end
|
31
|
+
let(:observer) { TestObserver.new }
|
32
|
+
let(:reading) { ObservingRead.new(opts) }
|
33
|
+
let(:actions) do
|
34
|
+
RSpec::Sequencing.run_after(0.45, "quit after a short time") do
|
35
|
+
reading.quit
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
after do
|
40
|
+
FileUtils.rm_rf(directory) unless directory =~ /fixture/
|
41
|
+
end
|
42
|
+
|
43
|
+
context "when watching a directory with files" do
|
44
|
+
let(:directory) { Stud::Temporary.directory }
|
45
|
+
let(:watch_dir) { ::File.join(directory, "*.log") }
|
46
|
+
let(:file_path) { ::File.join(directory, "1.log") }
|
47
|
+
|
48
|
+
it "the file is read" do
|
49
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
50
|
+
actions.activate
|
51
|
+
reading.watch_this(watch_dir)
|
52
|
+
reading.subscribe(observer)
|
53
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
|
54
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context "when watching a directory with files using striped reading" do
|
59
|
+
let(:directory) { Stud::Temporary.directory }
|
60
|
+
let(:watch_dir) { ::File.join(directory, "*.log") }
|
61
|
+
let(:file_path1) { ::File.join(directory, "1.log") }
|
62
|
+
let(:file_path2) { ::File.join(directory, "2.log") }
|
63
|
+
# use a chunk size that does not align with the line boundaries
|
64
|
+
let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1)}
|
65
|
+
let(:lines) { [] }
|
66
|
+
let(:observer) { TestObserver.new(lines) }
|
67
|
+
|
68
|
+
it "the files are read seemingly in parallel" do
|
69
|
+
File.open(file_path1, "w") { |file| file.write("string1\nstring2\n") }
|
70
|
+
File.open(file_path2, "w") { |file| file.write("stringA\nstringB\n") }
|
71
|
+
actions.activate
|
72
|
+
reading.watch_this(watch_dir)
|
73
|
+
reading.subscribe(observer)
|
74
|
+
if lines.first == "stringA"
|
75
|
+
expect(lines).to eq(%w(stringA string1 stringB string2))
|
76
|
+
else
|
77
|
+
expect(lines).to eq(%w(string1 stringA string2 stringB))
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
describe "reading fixtures" do
|
83
|
+
let(:directory) { FIXTURE_DIR }
|
84
|
+
|
85
|
+
context "for an uncompressed file" do
|
86
|
+
let(:watch_dir) { ::File.join(directory, "unc*.log") }
|
87
|
+
let(:file_path) { ::File.join(directory, 'uncompressed.log') }
|
88
|
+
|
89
|
+
it "the file is read" do
|
90
|
+
FileWatch.make_fixture_current(file_path)
|
91
|
+
actions.activate
|
92
|
+
reading.watch_this(watch_dir)
|
93
|
+
reading.subscribe(observer)
|
94
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
|
95
|
+
expect(observer.listener_for(file_path).lines.size).to eq(2)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
context "for another uncompressed file" do
|
100
|
+
let(:watch_dir) { ::File.join(directory, "invalid*.log") }
|
101
|
+
let(:file_path) { ::File.join(directory, 'invalid_utf8.gbk.log') }
|
102
|
+
|
103
|
+
it "the file is read" do
|
104
|
+
FileWatch.make_fixture_current(file_path)
|
105
|
+
actions.activate
|
106
|
+
reading.watch_this(watch_dir)
|
107
|
+
reading.subscribe(observer)
|
108
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
|
109
|
+
expect(observer.listener_for(file_path).lines.size).to eq(2)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
context "for a compressed file" do
|
114
|
+
let(:watch_dir) { ::File.join(directory, "compressed.*.gz") }
|
115
|
+
let(:file_path) { ::File.join(directory, 'compressed.log.gz') }
|
116
|
+
|
117
|
+
it "the file is read" do
|
118
|
+
FileWatch.make_fixture_current(file_path)
|
119
|
+
actions.activate
|
120
|
+
reading.watch_this(watch_dir)
|
121
|
+
reading.subscribe(observer)
|
122
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
|
123
|
+
expect(observer.listener_for(file_path).lines.size).to eq(2)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative 'spec_helper'
|
3
|
+
require 'filewatch/settings'
|
4
|
+
require 'filewatch/sincedb_record_serializer'
|
5
|
+
|
6
|
+
module FileWatch
|
7
|
+
describe SincedbRecordSerializer do
|
8
|
+
let(:opts) { Hash.new }
|
9
|
+
let(:io) { StringIO.new }
|
10
|
+
let(:db) { Hash.new }
|
11
|
+
|
12
|
+
subject { described_class.new(Settings.days_to_seconds(14)) }
|
13
|
+
|
14
|
+
context "deserialize from IO" do
|
15
|
+
it 'reads V1 records' do
|
16
|
+
io.write("5391299 1 4 12\n")
|
17
|
+
subject.deserialize(io) do |inode_struct, sincedb_value|
|
18
|
+
expect(inode_struct.inode).to eq("5391299")
|
19
|
+
expect(inode_struct.maj).to eq(1)
|
20
|
+
expect(inode_struct.min).to eq(4)
|
21
|
+
expect(sincedb_value.position).to eq(12)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'reads V2 records from an IO object' do
|
26
|
+
now = Time.now.to_f
|
27
|
+
io.write("5391299 1 4 12 #{now} /a/path/to/1.log\n")
|
28
|
+
subject.deserialize(io) do |inode_struct, sincedb_value|
|
29
|
+
expect(inode_struct.inode).to eq("5391299")
|
30
|
+
expect(inode_struct.maj).to eq(1)
|
31
|
+
expect(inode_struct.min).to eq(4)
|
32
|
+
expect(sincedb_value.position).to eq(12)
|
33
|
+
expect(sincedb_value.last_changed_at).to eq(now)
|
34
|
+
expect(sincedb_value.path_in_sincedb).to eq("/a/path/to/1.log")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context "serialize to IO" do
|
40
|
+
it "writes db entries" do
|
41
|
+
now = Time.now.to_f
|
42
|
+
inode_struct = InodeStruct.new("42424242", 2, 5)
|
43
|
+
sincedb_value = SincedbValue.new(42, now)
|
44
|
+
db[inode_struct] = sincedb_value
|
45
|
+
subject.serialize(db, io)
|
46
|
+
expect(io.string).to eq("42424242 2 5 42 #{now}\n")
|
47
|
+
end
|
48
|
+
|
49
|
+
it "does not write expired db entries to an IO object" do
|
50
|
+
twelve_days_ago = Time.now.to_f - (12.0*24*3600)
|
51
|
+
sixteen_days_ago = twelve_days_ago - (4.0*24*3600)
|
52
|
+
db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, twelve_days_ago)
|
53
|
+
db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, sixteen_days_ago)
|
54
|
+
subject.serialize(db, io)
|
55
|
+
expect(io.string).to eq("42424242 2 5 42 #{twelve_days_ago}\n")
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context "given a non default `sincedb_clean_after`" do
|
60
|
+
it "does not write expired db entries to an IO object" do
|
61
|
+
subject.update_sincedb_value_expiry_from_days(2)
|
62
|
+
one_day_ago = Time.now.to_f - (1.0*24*3600)
|
63
|
+
three_days_ago = one_day_ago - (2.0*24*3600)
|
64
|
+
db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)
|
65
|
+
db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, three_days_ago)
|
66
|
+
subject.serialize(db, io)
|
67
|
+
expect(io.string).to eq("42424242 2 5 42 #{one_day_ago}\n")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require "rspec_sequencing"
|
2
|
+
require 'rspec/wait'
|
3
|
+
require "logstash/devutils/rspec/spec_helper"
|
4
|
+
require "timecop"
|
5
|
+
|
6
|
+
def formatted_puts(text)
|
7
|
+
cfg = RSpec.configuration
|
8
|
+
return unless cfg.formatters.first.is_a?(
|
9
|
+
RSpec::Core::Formatters::DocumentationFormatter)
|
10
|
+
txt = cfg.format_docstrings_block.call(text)
|
11
|
+
cfg.output_stream.puts " #{txt}"
|
12
|
+
end
|
13
|
+
|
14
|
+
unless RSpec::Matchers.method_defined?(:receive_call_and_args)
|
15
|
+
RSpec::Matchers.define(:receive_call_and_args) do |m, args|
|
16
|
+
match do |actual|
|
17
|
+
actual.trace_for(m) == args
|
18
|
+
end
|
19
|
+
|
20
|
+
failure_message do
|
21
|
+
"Expecting method #{m} to receive: #{args} but got: #{actual.trace_for(m)}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'filewatch/bootstrap'
|
27
|
+
|
28
|
+
module FileWatch
|
29
|
+
|
30
|
+
FIXTURE_DIR = File.join('spec', 'fixtures')
|
31
|
+
|
32
|
+
def self.make_file_older(path, seconds)
|
33
|
+
time = Time.now.to_f - seconds
|
34
|
+
::File.utime(time, time, path)
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.make_fixture_current(path, time = Time.now)
|
38
|
+
::File.utime(time, time, path)
|
39
|
+
end
|
40
|
+
|
41
|
+
class TracerBase
|
42
|
+
def initialize
|
43
|
+
@tracer = []
|
44
|
+
end
|
45
|
+
|
46
|
+
def trace_for(symbol)
|
47
|
+
params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
|
48
|
+
params.empty? ? false : params
|
49
|
+
end
|
50
|
+
|
51
|
+
def clear
|
52
|
+
@tracer.clear
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
module NullCallable
|
57
|
+
def self.call
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class TestObserver
|
62
|
+
class Listener
|
63
|
+
attr_reader :path, :lines, :calls
|
64
|
+
|
65
|
+
def initialize(path)
|
66
|
+
@path = path
|
67
|
+
@lines = []
|
68
|
+
@calls = []
|
69
|
+
end
|
70
|
+
|
71
|
+
def add_lines(lines)
|
72
|
+
@lines = lines
|
73
|
+
self
|
74
|
+
end
|
75
|
+
|
76
|
+
def accept(line)
|
77
|
+
@lines << line
|
78
|
+
@calls << :accept
|
79
|
+
end
|
80
|
+
|
81
|
+
def deleted
|
82
|
+
@calls << :delete
|
83
|
+
end
|
84
|
+
|
85
|
+
def opened
|
86
|
+
@calls << :open
|
87
|
+
end
|
88
|
+
|
89
|
+
def error
|
90
|
+
@calls << :error
|
91
|
+
end
|
92
|
+
|
93
|
+
def eof
|
94
|
+
@calls << :eof
|
95
|
+
end
|
96
|
+
|
97
|
+
def timed_out
|
98
|
+
@calls << :timed_out
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
attr_reader :listeners
|
103
|
+
|
104
|
+
def initialize(combined_lines = nil)
|
105
|
+
listener_proc = if combined_lines.nil?
|
106
|
+
lambda{|k| Listener.new(k) }
|
107
|
+
else
|
108
|
+
lambda{|k| Listener.new(k).add_lines(combined_lines) }
|
109
|
+
end
|
110
|
+
@listeners = Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
|
111
|
+
end
|
112
|
+
|
113
|
+
def listener_for(path)
|
114
|
+
@listeners[path]
|
115
|
+
end
|
116
|
+
|
117
|
+
def clear
|
118
|
+
@listeners.clear; end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,440 @@
|
|
1
|
+
|
2
|
+
require 'stud/temporary'
|
3
|
+
require_relative 'spec_helper'
|
4
|
+
require 'filewatch/observing_tail'
|
5
|
+
|
6
|
+
LogStash::Logging::Logger::configure_logging("WARN")
|
7
|
+
# LogStash::Logging::Logger::configure_logging("DEBUG")
|
8
|
+
|
9
|
+
module FileWatch
|
10
|
+
describe Watch do
|
11
|
+
before(:all) do
|
12
|
+
@thread_abort = Thread.abort_on_exception
|
13
|
+
Thread.abort_on_exception = true
|
14
|
+
end
|
15
|
+
|
16
|
+
after(:all) do
|
17
|
+
Thread.abort_on_exception = @thread_abort
|
18
|
+
end
|
19
|
+
|
20
|
+
let(:directory) { Stud::Temporary.directory }
|
21
|
+
let(:watch_dir) { ::File.join(directory, "*.log") }
|
22
|
+
let(:file_path) { ::File.join(directory, "1.log") }
|
23
|
+
let(:max) { 4095 }
|
24
|
+
let(:stat_interval) { 0.1 }
|
25
|
+
let(:discover_interval) { 4 }
|
26
|
+
let(:start_new_files_at) { :beginning }
|
27
|
+
let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
|
28
|
+
let(:opts) do
|
29
|
+
{
|
30
|
+
:stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max,
|
31
|
+
:delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path
|
32
|
+
}
|
33
|
+
end
|
34
|
+
let(:observer) { TestObserver.new }
|
35
|
+
let(:tailing) { ObservingTail.new(opts) }
|
36
|
+
|
37
|
+
after do
|
38
|
+
FileUtils.rm_rf(directory)
|
39
|
+
end
|
40
|
+
|
41
|
+
describe "max open files (set to 1)" do
|
42
|
+
let(:max) { 1 }
|
43
|
+
let(:file_path2) { File.join(directory, "2.log") }
|
44
|
+
let(:wait_before_quit) { 0.15 }
|
45
|
+
let(:stat_interval) { 0.01 }
|
46
|
+
let(:discover_interval) { 4 }
|
47
|
+
let(:actions) do
|
48
|
+
RSpec::Sequencing
|
49
|
+
.run_after(wait_before_quit, "quit after a short time") do
|
50
|
+
tailing.quit
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
before do
|
55
|
+
ENV["FILEWATCH_MAX_FILES_WARN_INTERVAL"] = "0"
|
56
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
57
|
+
File.open(file_path2, "wb") { |file| file.write("lineA\nlineB\n") }
|
58
|
+
end
|
59
|
+
|
60
|
+
context "when max_active is 1" do
|
61
|
+
|
62
|
+
it "without close_older set, opens only 1 file" do
|
63
|
+
actions.activate
|
64
|
+
tailing.watch_this(watch_dir)
|
65
|
+
tailing.subscribe(observer)
|
66
|
+
expect(tailing.settings.max_active).to eq(max)
|
67
|
+
file1_calls = observer.listener_for(file_path).calls
|
68
|
+
file2_calls = observer.listener_for(file_path2).calls
|
69
|
+
# file glob order is OS dependent
|
70
|
+
if file1_calls.empty?
|
71
|
+
expect(observer.listener_for(file_path2).lines).to eq(["lineA", "lineB"])
|
72
|
+
expect(file2_calls).to eq([:open, :accept, :accept])
|
73
|
+
else
|
74
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
75
|
+
expect(file1_calls).to eq([:open, :accept, :accept])
|
76
|
+
expect(file2_calls).to be_empty
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
context "when close_older is set" do
|
82
|
+
let(:wait_before_quit) { 0.8 }
|
83
|
+
let(:opts) { super.merge(:close_older => 0.2, :max_active => 1, :stat_interval => 0.1) }
|
84
|
+
it "opens both files" do
|
85
|
+
actions.activate
|
86
|
+
tailing.watch_this(watch_dir)
|
87
|
+
tailing.subscribe(observer)
|
88
|
+
expect(tailing.settings.max_active).to eq(1)
|
89
|
+
filelistener_1 = observer.listener_for(file_path)
|
90
|
+
filelistener_2 = observer.listener_for(file_path2)
|
91
|
+
expect(filelistener_2.calls).to eq([:open, :accept, :accept, :timed_out])
|
92
|
+
expect(filelistener_2.lines).to eq(["lineA", "lineB"])
|
93
|
+
expect(filelistener_1.calls).to eq([:open, :accept, :accept, :timed_out])
|
94
|
+
expect(filelistener_1.lines).to eq(["line1", "line2"])
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
context "when watching a directory with files" do
|
100
|
+
let(:start_new_files_at) { :beginning }
|
101
|
+
let(:actions) do
|
102
|
+
RSpec::Sequencing.run_after(0.45, "quit after a short time") do
|
103
|
+
tailing.quit
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
it "the file is read" do
|
108
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
109
|
+
actions.activate
|
110
|
+
tailing.watch_this(watch_dir)
|
111
|
+
tailing.subscribe(observer)
|
112
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept])
|
113
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
context "when watching a directory without files and one is added" do
|
118
|
+
let(:start_new_files_at) { :beginning }
|
119
|
+
before do
|
120
|
+
tailing.watch_this(watch_dir)
|
121
|
+
RSpec::Sequencing
|
122
|
+
.run_after(0.25, "create file") do
|
123
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
124
|
+
end
|
125
|
+
.then_after(0.45, "quit after a short time") do
|
126
|
+
tailing.quit
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
it "the file is read" do
|
131
|
+
tailing.subscribe(observer)
|
132
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept])
|
133
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
describe "given a previously discovered file" do
|
138
|
+
# these tests rely on the fact that the 'filepath' does not exist on disk
|
139
|
+
# it simulates that the user deleted the file
|
140
|
+
# so when a stat is taken on the file an error is raised
|
141
|
+
let(:quit_after) { 0.2 }
|
142
|
+
let(:stat) { double("stat", :size => 100, :ctime => Time.now, :mtime => Time.now, :ino => 234567, :dev_major => 3, :dev_minor => 2) }
|
143
|
+
let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
|
144
|
+
|
145
|
+
before do
|
146
|
+
tailing.watch.watched_files_collection.add(watched_file)
|
147
|
+
watched_file.initial_completed
|
148
|
+
end
|
149
|
+
|
150
|
+
context "when a close operation occurs" do
|
151
|
+
before { watched_file.close }
|
152
|
+
it "is removed from the watched_files_collection" do
|
153
|
+
expect(tailing.watch.watched_files_collection).not_to be_empty
|
154
|
+
RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
|
155
|
+
tailing.subscribe(observer)
|
156
|
+
expect(tailing.watch.watched_files_collection).to be_empty
|
157
|
+
expect(observer.listener_for(file_path).calls).to eq([:delete])
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
context "an ignore operation occurs" do
|
162
|
+
before { watched_file.ignore }
|
163
|
+
it "is removed from the watched_files_collection" do
|
164
|
+
RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
|
165
|
+
tailing.subscribe(observer)
|
166
|
+
expect(tailing.watch.watched_files_collection).to be_empty
|
167
|
+
expect(observer.listener_for(file_path).calls).to eq([:delete])
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
context "when subscribed and a watched file is no longer readable" do
|
172
|
+
before { watched_file.watch }
|
173
|
+
it "is removed from the watched_files_collection" do
|
174
|
+
RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
|
175
|
+
tailing.subscribe(observer)
|
176
|
+
expect(tailing.watch.watched_files_collection).to be_empty
|
177
|
+
expect(observer.listener_for(file_path).calls).to eq([:delete])
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
context "when subscribed and an active file is no longer readable" do
|
182
|
+
before { watched_file.activate }
|
183
|
+
it "is removed from the watched_files_collection" do
|
184
|
+
RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
|
185
|
+
tailing.subscribe(observer)
|
186
|
+
expect(tailing.watch.watched_files_collection).to be_empty
|
187
|
+
expect(observer.listener_for(file_path).calls).to eq([:delete])
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
context "when a processed file shrinks" do
|
193
|
+
let(:discover_interval) { 100 }
|
194
|
+
before do
|
195
|
+
RSpec::Sequencing
|
196
|
+
.run("create file") do
|
197
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\nline3\nline4\n") }
|
198
|
+
end
|
199
|
+
.then_after(0.25, "start watching after files are written") do
|
200
|
+
tailing.watch_this(watch_dir)
|
201
|
+
end
|
202
|
+
.then_after(0.25, "truncate file and write new content") do
|
203
|
+
File.truncate(file_path, 0)
|
204
|
+
File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") }
|
205
|
+
end
|
206
|
+
.then_after(0.25, "quit after a short time") do
|
207
|
+
tailing.quit
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
it "new changes to the shrunk file are read from the beginning" do
|
212
|
+
tailing.subscribe(observer)
|
213
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept, :accept, :accept])
|
214
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4", "lineA", "lineB"])
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
context "when watching a directory with files and a file is renamed to not match glob" do
|
219
|
+
let(:new_file_path) { file_path + ".old" }
|
220
|
+
before do
|
221
|
+
RSpec::Sequencing
|
222
|
+
.run("create file") do
|
223
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
224
|
+
end
|
225
|
+
.then_after(0.25, "start watching after files are written") do
|
226
|
+
tailing.watch_this(watch_dir)
|
227
|
+
end
|
228
|
+
.then_after(0.55, "rename file") do
|
229
|
+
FileUtils.mv(file_path, new_file_path)
|
230
|
+
end
|
231
|
+
.then_after(0.55, "then write to renamed file") do
|
232
|
+
File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") }
|
233
|
+
end
|
234
|
+
.then_after(0.45, "quit after a short time") do
|
235
|
+
tailing.quit
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
it "changes to the renamed file are not read" do
|
240
|
+
tailing.subscribe(observer)
|
241
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :delete])
|
242
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
243
|
+
expect(observer.listener_for(new_file_path).calls).to eq([])
|
244
|
+
expect(observer.listener_for(new_file_path).lines).to eq([])
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
context "when watching a directory with files and a file is renamed to match glob" do
|
249
|
+
let(:new_file_path) { file_path + "2.log" }
|
250
|
+
let(:opts) { super.merge(:close_older => 0) }
|
251
|
+
before do
|
252
|
+
RSpec::Sequencing
|
253
|
+
.run("create file") do
|
254
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
255
|
+
end
|
256
|
+
.then_after(0.15, "start watching after files are written") do
|
257
|
+
tailing.watch_this(watch_dir)
|
258
|
+
end
|
259
|
+
.then_after(0.25, "rename file") do
|
260
|
+
FileUtils.mv(file_path, new_file_path)
|
261
|
+
end
|
262
|
+
.then("then write to renamed file") do
|
263
|
+
File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") }
|
264
|
+
end
|
265
|
+
.then_after(0.55, "quit after a short time") do
|
266
|
+
tailing.quit
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
it "the first set of lines are not re-read" do
|
271
|
+
tailing.subscribe(observer)
|
272
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
273
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :delete])
|
274
|
+
expect(observer.listener_for(new_file_path).lines).to eq(["line3", "line4"])
|
275
|
+
expect(observer.listener_for(new_file_path).calls).to eq([:open, :accept, :accept, :timed_out])
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
context "when watching a directory with files and data is appended" do
|
280
|
+
before do
|
281
|
+
RSpec::Sequencing
|
282
|
+
.run("create file") do
|
283
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
284
|
+
end
|
285
|
+
.then_after(0.25, "start watching after file is written") do
|
286
|
+
tailing.watch_this(watch_dir)
|
287
|
+
end
|
288
|
+
.then_after(0.45, "append more lines to the file") do
|
289
|
+
File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
|
290
|
+
end
|
291
|
+
.then_after(0.45, "quit after a short time") do
|
292
|
+
tailing.quit
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
it "appended lines are read after an EOF" do
|
297
|
+
tailing.subscribe(observer)
|
298
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept])
|
299
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"])
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
context "when close older expiry is enabled" do
|
304
|
+
let(:opts) { super.merge(:close_older => 1) }
|
305
|
+
before do
|
306
|
+
RSpec::Sequencing
|
307
|
+
.run("create file") do
|
308
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
309
|
+
end
|
310
|
+
.then("start watching before file ages more than close_older") do
|
311
|
+
tailing.watch_this(watch_dir)
|
312
|
+
end
|
313
|
+
.then_after(2.1, "quit after allowing time to close the file") do
|
314
|
+
tailing.quit
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
it "lines are read and the file times out" do
|
319
|
+
tailing.subscribe(observer)
|
320
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
|
321
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
context "when close older expiry is enabled and after timeout the file is appended-to" do
|
326
|
+
let(:opts) { super.merge(:close_older => 1) }
|
327
|
+
before do
|
328
|
+
RSpec::Sequencing
|
329
|
+
.run("create file") do
|
330
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
331
|
+
end
|
332
|
+
.then("start watching before file ages more than close_older") do
|
333
|
+
tailing.watch_this(watch_dir)
|
334
|
+
end
|
335
|
+
.then_after(2.1, "append more lines to file after file ages more than close_older") do
|
336
|
+
File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
|
337
|
+
end
|
338
|
+
.then_after(2.1, "quit after allowing time to close the file") do
|
339
|
+
tailing.quit
|
340
|
+
end
|
341
|
+
end
|
342
|
+
|
343
|
+
it "all lines are read" do
|
344
|
+
tailing.subscribe(observer)
|
345
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :open, :accept, :accept, :timed_out])
|
346
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"])
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
350
|
+
context "when ignore older expiry is enabled and all files are already expired" do
|
351
|
+
let(:opts) { super.merge(:ignore_older => 1) }
|
352
|
+
before do
|
353
|
+
RSpec::Sequencing
|
354
|
+
.run("create file older than ignore_older and watch") do
|
355
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
356
|
+
FileWatch.make_file_older(file_path, 15)
|
357
|
+
tailing.watch_this(watch_dir)
|
358
|
+
end
|
359
|
+
.then_after(1.1, "quit") do
|
360
|
+
tailing.quit
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
it "no files are read" do
|
365
|
+
tailing.subscribe(observer)
|
366
|
+
expect(observer.listener_for(file_path).calls).to eq([])
|
367
|
+
expect(observer.listener_for(file_path).lines).to eq([])
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
371
|
+
context "when ignore_older is less than close_older and all files are not expired" do
|
372
|
+
let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.5) }
|
373
|
+
before do
|
374
|
+
RSpec::Sequencing
|
375
|
+
.run("create file") do
|
376
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
377
|
+
end
|
378
|
+
.then("start watching before file age reaches ignore_older") do
|
379
|
+
tailing.watch_this(watch_dir)
|
380
|
+
end
|
381
|
+
.then_after(1.75, "quit after allowing time to close the file") do
|
382
|
+
tailing.quit
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
it "reads lines normally" do
|
387
|
+
tailing.subscribe(observer)
|
388
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
|
389
|
+
expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
context "when ignore_older is less than close_older and all files are expired" do
|
394
|
+
let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) }
|
395
|
+
before do
|
396
|
+
RSpec::Sequencing
|
397
|
+
.run("create file older than ignore_older and watch") do
|
398
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
399
|
+
FileWatch.make_file_older(file_path, 15)
|
400
|
+
tailing.watch_this(watch_dir)
|
401
|
+
end
|
402
|
+
.then_after(1.5, "quit after allowing time to check the files") do
|
403
|
+
tailing.quit
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
it "no files are read" do
|
408
|
+
tailing.subscribe(observer)
|
409
|
+
expect(observer.listener_for(file_path).calls).to eq([])
|
410
|
+
expect(observer.listener_for(file_path).lines).to eq([])
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do
|
415
|
+
let(:opts) { super.merge(:ignore_older => 20, :close_older => 1) }
|
416
|
+
before do
|
417
|
+
RSpec::Sequencing
|
418
|
+
.run("create file older than ignore_older and watch") do
|
419
|
+
File.open(file_path, "wb") { |file| file.write("line1\nline2\n") }
|
420
|
+
FileWatch.make_file_older(file_path, 25)
|
421
|
+
tailing.watch_this(watch_dir)
|
422
|
+
end
|
423
|
+
.then_after(0.15, "append more lines to file after file ages more than ignore_older") do
|
424
|
+
File.open(file_path, "ab") { |file| file.write("line3\nline4\n") }
|
425
|
+
end
|
426
|
+
.then_after(1.25, "quit after allowing time to close the file") do
|
427
|
+
tailing.quit
|
428
|
+
end
|
429
|
+
end
|
430
|
+
|
431
|
+
it "reads the added lines only" do
|
432
|
+
tailing.subscribe(observer)
|
433
|
+
expect(observer.listener_for(file_path).lines).to eq(["line3", "line4"])
|
434
|
+
expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
|
435
|
+
end
|
436
|
+
end
|
437
|
+
end
|
438
|
+
end
|
439
|
+
|
440
|
+
|