franz 1.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/Gemfile +15 -0
- data/LICENSE +13 -0
- data/Rakefile +52 -0
- data/Readme.md +50 -0
- data/VERSION +1 -0
- data/bin/franz +81 -0
- data/franz.gemspec +25 -0
- data/lib/franz.rb +10 -0
- data/lib/franz/agg.rb +158 -0
- data/lib/franz/config.rb +24 -0
- data/lib/franz/discover.rb +108 -0
- data/lib/franz/input.rb +174 -0
- data/lib/franz/logger.rb +66 -0
- data/lib/franz/metadata.rb +33 -0
- data/lib/franz/output.rb +81 -0
- data/lib/franz/sash.rb +81 -0
- data/lib/franz/tail.rb +191 -0
- data/lib/franz/tail_pool.rb +68 -0
- data/lib/franz/watch.rb +180 -0
- data/test/test_franz_agg.rb +97 -0
- data/test/test_franz_discover.rb +88 -0
- data/test/test_franz_tail.rb +132 -0
- data/test/test_franz_watch.rb +144 -0
- metadata +155 -0
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'tmpdir'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'pathname'
|
6
|
+
require 'minitest/autorun'
|
7
|
+
|
8
|
+
require 'deep_merge'
|
9
|
+
|
10
|
+
require_relative '../lib/franz'
|
11
|
+
|
12
|
+
Thread.abort_on_exception = true
|
13
|
+
|
14
|
+
class TestFranzAgg < MiniTest::Test
|
15
|
+
def setup
|
16
|
+
@tmpdir = Dir.mktmpdir
|
17
|
+
@discoveries = Queue.new
|
18
|
+
@deletions = Queue.new
|
19
|
+
@watch_events = Queue.new
|
20
|
+
@tail_events = Queue.new
|
21
|
+
@agg_events = Queue.new
|
22
|
+
@logger = Logger.new STDERR
|
23
|
+
@logger.level = Logger::WARN
|
24
|
+
end
|
25
|
+
|
26
|
+
def teardown
|
27
|
+
FileUtils.rm_rf @tmpdir
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_handles_multiline
|
31
|
+
sample = "multiline this\nand this should be included\n"
|
32
|
+
tmp = tempfile %w[ test1 .log ]
|
33
|
+
tmp.write sample
|
34
|
+
tmp.flush
|
35
|
+
tmp.close
|
36
|
+
start_agg
|
37
|
+
sleep 3
|
38
|
+
seqs = stop_agg
|
39
|
+
path = realpath tmp.path
|
40
|
+
assert seqs.include?(path)
|
41
|
+
assert_equal sample.strip, @agg_events.shift[:message]
|
42
|
+
assert seqs[path] == 1 # should be one line
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
def tempfile prefix=nil
|
47
|
+
Tempfile.new prefix, @tmpdir
|
48
|
+
end
|
49
|
+
|
50
|
+
def realpath path
|
51
|
+
Pathname.new(path).realpath.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
def start_agg opts={}
|
55
|
+
configs = [{
|
56
|
+
type: :test,
|
57
|
+
multiline: /^multiline/,
|
58
|
+
includes: [ "#{@tmpdir}/*.log", "#{realpath @tmpdir}/*.log" ],
|
59
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
60
|
+
}]
|
61
|
+
|
62
|
+
@discover = Franz::Discover.new({
|
63
|
+
discover_interval: 1,
|
64
|
+
discoveries: @discoveries,
|
65
|
+
deletions: @deletions,
|
66
|
+
logger: @logger,
|
67
|
+
configs: configs
|
68
|
+
}.deep_merge!(opts))
|
69
|
+
|
70
|
+
@watch = Franz::Watch.new({
|
71
|
+
watch_interval: 1,
|
72
|
+
watch_events: @watch_events,
|
73
|
+
discoveries: @discoveries,
|
74
|
+
deletions: @deletions,
|
75
|
+
logger: @logger
|
76
|
+
}.deep_merge!(opts))
|
77
|
+
|
78
|
+
@tail = Franz::Tail.new({
|
79
|
+
eviction_interval: 1,
|
80
|
+
watch_events: @watch_events,
|
81
|
+
tail_events: @tail_events,
|
82
|
+
logger: @logger
|
83
|
+
}.deep_merge!(opts))
|
84
|
+
|
85
|
+
@agg = Franz::Agg.new({
|
86
|
+
configs: configs,
|
87
|
+
flush_interval: 2,
|
88
|
+
tail_events: @tail_events,
|
89
|
+
agg_events: @agg_events,
|
90
|
+
logger: @logger
|
91
|
+
}.deep_merge!(opts))
|
92
|
+
end
|
93
|
+
|
94
|
+
def stop_agg
|
95
|
+
@agg.stop
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'tmpdir'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'minitest/autorun'
|
6
|
+
|
7
|
+
require 'deep_merge'
|
8
|
+
|
9
|
+
require_relative '../lib/franz'
|
10
|
+
|
11
|
+
Thread.abort_on_exception = true
|
12
|
+
|
13
|
+
class TestFranzDiscover < MiniTest::Test
|
14
|
+
def setup
|
15
|
+
@tmpdir = Dir.mktmpdir
|
16
|
+
@discoveries = Queue.new
|
17
|
+
@deletions = Queue.new
|
18
|
+
@logger = Logger.new STDERR
|
19
|
+
@logger.level = Logger::WARN
|
20
|
+
end
|
21
|
+
|
22
|
+
def teardown
|
23
|
+
FileUtils.rm_rf @tmpdir
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_discovers_existing_file
|
27
|
+
tmp = tempfile %w[ test .log ]
|
28
|
+
start_discovery known: []
|
29
|
+
sleep 0.001 # Time to discover
|
30
|
+
known = stop_discovery
|
31
|
+
assert known.include?(tmp.path)
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_discovers_new_file
|
35
|
+
start_discovery known: []
|
36
|
+
tmp = tempfile %w[ test .log ]
|
37
|
+
sleep 1 # Time to discover
|
38
|
+
known = stop_discovery
|
39
|
+
assert known.include?(tmp.path)
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_deletes_deleted_file
|
43
|
+
tmp = tempfile %w[ test .log ]
|
44
|
+
start_discovery known: []
|
45
|
+
# at this point, we know Discover has already picked up tmp
|
46
|
+
delete tmp.path
|
47
|
+
sleep 2
|
48
|
+
known = stop_discovery
|
49
|
+
assert !known.include?(tmp.path)
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_deletes_unknown_file
|
53
|
+
tmp = tempfile %w[ test .log ]
|
54
|
+
delete tmp.path
|
55
|
+
# tmp never exists as far as Discover is aware
|
56
|
+
start_discovery known: []
|
57
|
+
sleep 0.001
|
58
|
+
known = stop_discovery
|
59
|
+
assert !known.include?(tmp.path)
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
def tempfile prefix=nil
|
64
|
+
Tempfile.new prefix, @tmpdir
|
65
|
+
end
|
66
|
+
|
67
|
+
def start_discovery opts={}
|
68
|
+
@discover = Franz::Discover.new({
|
69
|
+
discover_interval: 1,
|
70
|
+
discoveries: @discoveries,
|
71
|
+
deletions: @deletions,
|
72
|
+
logger: @logger,
|
73
|
+
configs: [{
|
74
|
+
includes: [ "#{@tmpdir}/*.log" ],
|
75
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
76
|
+
}]
|
77
|
+
}.deep_merge!(opts))
|
78
|
+
end
|
79
|
+
|
80
|
+
def stop_discovery
|
81
|
+
@discover.stop
|
82
|
+
end
|
83
|
+
|
84
|
+
def delete path
|
85
|
+
FileUtils.rm_rf path
|
86
|
+
@deletions.push path
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'tmpdir'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'minitest/autorun'
|
6
|
+
|
7
|
+
require 'deep_merge'
|
8
|
+
|
9
|
+
require_relative '../lib/franz'
|
10
|
+
|
11
|
+
Thread.abort_on_exception = true
|
12
|
+
|
13
|
+
class TestFranzTail < MiniTest::Test
|
14
|
+
def setup
|
15
|
+
@tmpdir = Dir.mktmpdir
|
16
|
+
@discoveries = Queue.new
|
17
|
+
@deletions = Queue.new
|
18
|
+
@watch_events = Queue.new
|
19
|
+
@tail_events = Queue.new
|
20
|
+
@logger = Logger.new STDERR
|
21
|
+
@logger.level = Logger::WARN
|
22
|
+
end
|
23
|
+
|
24
|
+
def teardown
|
25
|
+
FileUtils.rm_rf @tmpdir
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_handles_existing_file
|
29
|
+
sample = "Hello, world!\n"
|
30
|
+
tmp = tempfile %w[ test1 .log ]
|
31
|
+
tmp.write sample
|
32
|
+
tmp.flush
|
33
|
+
tmp.close
|
34
|
+
start_tail
|
35
|
+
sleep 3
|
36
|
+
cursors = stop_tail
|
37
|
+
assert cursors.include?(tmp.path)
|
38
|
+
assert cursors[tmp.path] == sample.length
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_handles_new_file
|
42
|
+
sample = "Hello, world!\n"
|
43
|
+
start_tail
|
44
|
+
sleep 0
|
45
|
+
tmp = tempfile %w[ test2 .log ]
|
46
|
+
tmp.write sample
|
47
|
+
tmp.flush
|
48
|
+
tmp.close
|
49
|
+
sleep 3
|
50
|
+
cursors = stop_tail
|
51
|
+
assert cursors.include?(tmp.path)
|
52
|
+
assert cursors[tmp.path] == sample.length
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_handles_reading_after_eviction
|
56
|
+
sample = "Hello, world!\n"
|
57
|
+
eviction_interval = 2
|
58
|
+
start_tail eviction_interval: eviction_interval
|
59
|
+
sleep 0
|
60
|
+
tmp = tempfile %w[ test2 .log ]
|
61
|
+
tmp.write sample
|
62
|
+
tmp.flush
|
63
|
+
sleep eviction_interval / 2
|
64
|
+
tmp.write sample
|
65
|
+
tmp.flush
|
66
|
+
tmp.close
|
67
|
+
sleep eviction_interval * 2
|
68
|
+
cursors = stop_tail
|
69
|
+
assert cursors.include?(tmp.path)
|
70
|
+
assert cursors[tmp.path] == sample.length * 2
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_handles_reading_after_deletion
|
74
|
+
sample = "Hello, world!\n"
|
75
|
+
eviction_interval = 2
|
76
|
+
start_tail eviction_interval: eviction_interval
|
77
|
+
sleep 0
|
78
|
+
tmp = tempfile %w[ test2 .log ]
|
79
|
+
path = tmp.path
|
80
|
+
tmp.write sample
|
81
|
+
tmp.flush
|
82
|
+
tmp.close
|
83
|
+
sleep eviction_interval / 2
|
84
|
+
FileUtils.rm_rf path
|
85
|
+
sleep eviction_interval
|
86
|
+
File.open(path, 'w') do |f|
|
87
|
+
f.write sample
|
88
|
+
f.flush
|
89
|
+
end
|
90
|
+
sleep eviction_interval * 2
|
91
|
+
cursors = stop_tail
|
92
|
+
assert cursors.include?(tmp.path)
|
93
|
+
assert cursors[tmp.path] == sample.length
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def tempfile prefix=nil
|
98
|
+
Tempfile.new prefix, @tmpdir
|
99
|
+
end
|
100
|
+
|
101
|
+
def start_tail opts={}
|
102
|
+
@discover = Franz::Discover.new({
|
103
|
+
discover_interval: 1,
|
104
|
+
discoveries: @discoveries,
|
105
|
+
deletions: @deletions,
|
106
|
+
logger: @logger,
|
107
|
+
configs: [{
|
108
|
+
includes: [ "#{@tmpdir}/*.log" ],
|
109
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
110
|
+
}]
|
111
|
+
}.deep_merge!(opts))
|
112
|
+
|
113
|
+
@watch = Franz::Watch.new({
|
114
|
+
watch_interval: 1,
|
115
|
+
watch_events: @watch_events,
|
116
|
+
discoveries: @discoveries,
|
117
|
+
deletions: @deletions,
|
118
|
+
logger: @logger
|
119
|
+
}.deep_merge!(opts))
|
120
|
+
|
121
|
+
@tail = Franz::Tail.new({
|
122
|
+
eviction_interval: 1,
|
123
|
+
watch_events: @watch_events,
|
124
|
+
tail_events: @tail_events,
|
125
|
+
logger: @logger
|
126
|
+
}.deep_merge!(opts))
|
127
|
+
end
|
128
|
+
|
129
|
+
def stop_tail
|
130
|
+
@tail.stop
|
131
|
+
end
|
132
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'tmpdir'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'minitest/autorun'
|
6
|
+
|
7
|
+
require 'deep_merge'
|
8
|
+
|
9
|
+
require_relative '../lib/franz'
|
10
|
+
|
11
|
+
Thread.abort_on_exception = true
|
12
|
+
|
13
|
+
class TestFranzWatch < MiniTest::Test
|
14
|
+
def setup
|
15
|
+
@tmpdir = Dir.mktmpdir
|
16
|
+
@discoveries = Queue.new
|
17
|
+
@deletions = Queue.new
|
18
|
+
@queue = Queue.new
|
19
|
+
@logger = Logger.new STDERR
|
20
|
+
@logger.level = Logger::WARN
|
21
|
+
end
|
22
|
+
|
23
|
+
def teardown
|
24
|
+
FileUtils.rm_rf @tmpdir
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_handles_existing_file
|
28
|
+
tmp = tempfile %w[ test .log ]
|
29
|
+
start_watch
|
30
|
+
sleep 2
|
31
|
+
stats = stop_watch
|
32
|
+
assert stats.include?(tmp.path)
|
33
|
+
assert stats[tmp.path][:size] == 0
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_handles_existing_file_with_content
|
37
|
+
content = "Hello, world!\n"
|
38
|
+
tmp = tempfile %w[ test .log ]
|
39
|
+
tmp.write content
|
40
|
+
tmp.flush
|
41
|
+
start_watch
|
42
|
+
sleep 2
|
43
|
+
stats = stop_watch
|
44
|
+
assert stats.include?(tmp.path)
|
45
|
+
assert stats[tmp.path][:size] == content.length
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_handles_new_file
|
49
|
+
start_watch
|
50
|
+
tmp = tempfile %w[ test .log ]
|
51
|
+
sleep 3
|
52
|
+
stats = stop_watch
|
53
|
+
assert stats.include?(tmp.path)
|
54
|
+
assert stats[tmp.path][:size] == 0
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_handles_new_file_with_content
|
58
|
+
start_watch
|
59
|
+
content = "Hello, world!\n"
|
60
|
+
tmp = tempfile %w[ test .log ]
|
61
|
+
tmp.write content
|
62
|
+
tmp.flush
|
63
|
+
sleep 3
|
64
|
+
stats = stop_watch
|
65
|
+
assert stats.include?(tmp.path)
|
66
|
+
assert stats[tmp.path][:size] == content.length
|
67
|
+
end
|
68
|
+
|
69
|
+
def test_handles_file_truncated
|
70
|
+
long_content = "Hello, world!\n"
|
71
|
+
short_content = "Bye!\n"
|
72
|
+
|
73
|
+
tmp = tempfile %w[ test .log ]
|
74
|
+
tmp.write long_content
|
75
|
+
tmp.flush
|
76
|
+
|
77
|
+
start_watch
|
78
|
+
sleep 2
|
79
|
+
tmp.rewind
|
80
|
+
tmp.truncate 0
|
81
|
+
tmp.write short_content
|
82
|
+
tmp.flush
|
83
|
+
sleep 3
|
84
|
+
|
85
|
+
stats = stop_watch
|
86
|
+
assert stats.include?(tmp.path)
|
87
|
+
assert stats[tmp.path][:size] == short_content.length
|
88
|
+
end
|
89
|
+
|
90
|
+
def test_handles_file_replaced
|
91
|
+
content1 = "Hello, world!\n"
|
92
|
+
content2 = "Bye!\n"
|
93
|
+
|
94
|
+
tmp1 = tempfile %w[ test .log ]
|
95
|
+
tmp1.write content1
|
96
|
+
tmp1.flush
|
97
|
+
tmp1.close
|
98
|
+
|
99
|
+
tmp2 = tempfile %w[ exclude .log ]
|
100
|
+
tmp2.write content2
|
101
|
+
tmp2.flush
|
102
|
+
tmp2.close
|
103
|
+
|
104
|
+
start_watch
|
105
|
+
sleep 2
|
106
|
+
FileUtils.ln_sf tmp2.path, tmp1.path
|
107
|
+
sleep 3
|
108
|
+
|
109
|
+
stats = stop_watch
|
110
|
+
assert stats.include?(tmp1.path)
|
111
|
+
assert stats[tmp1.path][:size] == content2.length
|
112
|
+
end
|
113
|
+
|
114
|
+
private
|
115
|
+
def tempfile prefix=nil
|
116
|
+
Tempfile.new(prefix, @tmpdir)
|
117
|
+
end
|
118
|
+
|
119
|
+
def start_watch opts={}
|
120
|
+
@discover = Franz::Discover.new({
|
121
|
+
discover_interval: 1,
|
122
|
+
discoveries: @discoveries,
|
123
|
+
deletions: @deletions,
|
124
|
+
logger: @logger,
|
125
|
+
configs: [{
|
126
|
+
includes: [ "#{@tmpdir}/*.log" ],
|
127
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
128
|
+
}]
|
129
|
+
})
|
130
|
+
|
131
|
+
@watch = Franz::Watch.new({
|
132
|
+
watch_interval: 1,
|
133
|
+
watch_events: @queue,
|
134
|
+
discoveries: @discoveries,
|
135
|
+
deletions: @deletions,
|
136
|
+
logger: @logger,
|
137
|
+
stats: Hash.new
|
138
|
+
}.deep_merge!(opts))
|
139
|
+
end
|
140
|
+
|
141
|
+
def stop_watch
|
142
|
+
@watch.stop
|
143
|
+
end
|
144
|
+
end
|