franz 1.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,97 @@
1
+ require 'thread'
2
+ require 'tmpdir'
3
+ require 'tempfile'
4
+ require 'fileutils'
5
+ require 'pathname'
6
+ require 'minitest/autorun'
7
+
8
+ require 'deep_merge'
9
+
10
+ require_relative '../lib/franz'
11
+
12
+ Thread.abort_on_exception = true
13
+
14
+ class TestFranzAgg < MiniTest::Test
15
+ def setup
16
+ @tmpdir = Dir.mktmpdir
17
+ @discoveries = Queue.new
18
+ @deletions = Queue.new
19
+ @watch_events = Queue.new
20
+ @tail_events = Queue.new
21
+ @agg_events = Queue.new
22
+ @logger = Logger.new STDERR
23
+ @logger.level = Logger::WARN
24
+ end
25
+
26
+ def teardown
27
+ FileUtils.rm_rf @tmpdir
28
+ end
29
+
30
+ def test_handles_multiline
31
+ sample = "multiline this\nand this should be included\n"
32
+ tmp = tempfile %w[ test1 .log ]
33
+ tmp.write sample
34
+ tmp.flush
35
+ tmp.close
36
+ start_agg
37
+ sleep 3
38
+ seqs = stop_agg
39
+ path = realpath tmp.path
40
+ assert seqs.include?(path)
41
+ assert_equal sample.strip, @agg_events.shift[:message]
42
+ assert seqs[path] == 1 # should be one line
43
+ end
44
+
45
+ private
46
+ def tempfile prefix=nil
47
+ Tempfile.new prefix, @tmpdir
48
+ end
49
+
50
+ def realpath path
51
+ Pathname.new(path).realpath.to_s
52
+ end
53
+
54
+ def start_agg opts={}
55
+ configs = [{
56
+ type: :test,
57
+ multiline: /^multiline/,
58
+ includes: [ "#{@tmpdir}/*.log", "#{realpath @tmpdir}/*.log" ],
59
+ excludes: [ "#{@tmpdir}/exclude*" ]
60
+ }]
61
+
62
+ @discover = Franz::Discover.new({
63
+ discover_interval: 1,
64
+ discoveries: @discoveries,
65
+ deletions: @deletions,
66
+ logger: @logger,
67
+ configs: configs
68
+ }.deep_merge!(opts))
69
+
70
+ @watch = Franz::Watch.new({
71
+ watch_interval: 1,
72
+ watch_events: @watch_events,
73
+ discoveries: @discoveries,
74
+ deletions: @deletions,
75
+ logger: @logger
76
+ }.deep_merge!(opts))
77
+
78
+ @tail = Franz::Tail.new({
79
+ eviction_interval: 1,
80
+ watch_events: @watch_events,
81
+ tail_events: @tail_events,
82
+ logger: @logger
83
+ }.deep_merge!(opts))
84
+
85
+ @agg = Franz::Agg.new({
86
+ configs: configs,
87
+ flush_interval: 2,
88
+ tail_events: @tail_events,
89
+ agg_events: @agg_events,
90
+ logger: @logger
91
+ }.deep_merge!(opts))
92
+ end
93
+
94
+ def stop_agg
95
+ @agg.stop
96
+ end
97
+ end
@@ -0,0 +1,88 @@
1
+ require 'thread'
2
+ require 'tmpdir'
3
+ require 'tempfile'
4
+ require 'fileutils'
5
+ require 'minitest/autorun'
6
+
7
+ require 'deep_merge'
8
+
9
+ require_relative '../lib/franz'
10
+
11
+ Thread.abort_on_exception = true
12
+
13
+ class TestFranzDiscover < MiniTest::Test
14
+ def setup
15
+ @tmpdir = Dir.mktmpdir
16
+ @discoveries = Queue.new
17
+ @deletions = Queue.new
18
+ @logger = Logger.new STDERR
19
+ @logger.level = Logger::WARN
20
+ end
21
+
22
+ def teardown
23
+ FileUtils.rm_rf @tmpdir
24
+ end
25
+
26
+ def test_discovers_existing_file
27
+ tmp = tempfile %w[ test .log ]
28
+ start_discovery known: []
29
+ sleep 0.001 # Time to discover
30
+ known = stop_discovery
31
+ assert known.include?(tmp.path)
32
+ end
33
+
34
+ def test_discovers_new_file
35
+ start_discovery known: []
36
+ tmp = tempfile %w[ test .log ]
37
+ sleep 1 # Time to discover
38
+ known = stop_discovery
39
+ assert known.include?(tmp.path)
40
+ end
41
+
42
+ def test_deletes_deleted_file
43
+ tmp = tempfile %w[ test .log ]
44
+ start_discovery known: []
45
+ # at this point, we know Discover has already picked up tmp
46
+ delete tmp.path
47
+ sleep 2
48
+ known = stop_discovery
49
+ assert !known.include?(tmp.path)
50
+ end
51
+
52
+ def test_deletes_unknown_file
53
+ tmp = tempfile %w[ test .log ]
54
+ delete tmp.path
55
+ # tmp never exists as far as Discover is aware
56
+ start_discovery known: []
57
+ sleep 0.001
58
+ known = stop_discovery
59
+ assert !known.include?(tmp.path)
60
+ end
61
+
62
+ private
63
+ def tempfile prefix=nil
64
+ Tempfile.new prefix, @tmpdir
65
+ end
66
+
67
+ def start_discovery opts={}
68
+ @discover = Franz::Discover.new({
69
+ discover_interval: 1,
70
+ discoveries: @discoveries,
71
+ deletions: @deletions,
72
+ logger: @logger,
73
+ configs: [{
74
+ includes: [ "#{@tmpdir}/*.log" ],
75
+ excludes: [ "#{@tmpdir}/exclude*" ]
76
+ }]
77
+ }.deep_merge!(opts))
78
+ end
79
+
80
+ def stop_discovery
81
+ @discover.stop
82
+ end
83
+
84
+ def delete path
85
+ FileUtils.rm_rf path
86
+ @deletions.push path
87
+ end
88
+ end
@@ -0,0 +1,132 @@
1
+ require 'thread'
2
+ require 'tmpdir'
3
+ require 'tempfile'
4
+ require 'fileutils'
5
+ require 'minitest/autorun'
6
+
7
+ require 'deep_merge'
8
+
9
+ require_relative '../lib/franz'
10
+
11
+ Thread.abort_on_exception = true
12
+
13
+ class TestFranzTail < MiniTest::Test
14
+ def setup
15
+ @tmpdir = Dir.mktmpdir
16
+ @discoveries = Queue.new
17
+ @deletions = Queue.new
18
+ @watch_events = Queue.new
19
+ @tail_events = Queue.new
20
+ @logger = Logger.new STDERR
21
+ @logger.level = Logger::WARN
22
+ end
23
+
24
+ def teardown
25
+ FileUtils.rm_rf @tmpdir
26
+ end
27
+
28
+ def test_handles_existing_file
29
+ sample = "Hello, world!\n"
30
+ tmp = tempfile %w[ test1 .log ]
31
+ tmp.write sample
32
+ tmp.flush
33
+ tmp.close
34
+ start_tail
35
+ sleep 3
36
+ cursors = stop_tail
37
+ assert cursors.include?(tmp.path)
38
+ assert cursors[tmp.path] == sample.length
39
+ end
40
+
41
+ def test_handles_new_file
42
+ sample = "Hello, world!\n"
43
+ start_tail
44
+ sleep 0
45
+ tmp = tempfile %w[ test2 .log ]
46
+ tmp.write sample
47
+ tmp.flush
48
+ tmp.close
49
+ sleep 3
50
+ cursors = stop_tail
51
+ assert cursors.include?(tmp.path)
52
+ assert cursors[tmp.path] == sample.length
53
+ end
54
+
55
+ def test_handles_reading_after_eviction
56
+ sample = "Hello, world!\n"
57
+ eviction_interval = 2
58
+ start_tail eviction_interval: eviction_interval
59
+ sleep 0
60
+ tmp = tempfile %w[ test2 .log ]
61
+ tmp.write sample
62
+ tmp.flush
63
+ sleep eviction_interval / 2
64
+ tmp.write sample
65
+ tmp.flush
66
+ tmp.close
67
+ sleep eviction_interval * 2
68
+ cursors = stop_tail
69
+ assert cursors.include?(tmp.path)
70
+ assert cursors[tmp.path] == sample.length * 2
71
+ end
72
+
73
+ def test_handles_reading_after_deletion
74
+ sample = "Hello, world!\n"
75
+ eviction_interval = 2
76
+ start_tail eviction_interval: eviction_interval
77
+ sleep 0
78
+ tmp = tempfile %w[ test2 .log ]
79
+ path = tmp.path
80
+ tmp.write sample
81
+ tmp.flush
82
+ tmp.close
83
+ sleep eviction_interval / 2
84
+ FileUtils.rm_rf path
85
+ sleep eviction_interval
86
+ File.open(path, 'w') do |f|
87
+ f.write sample
88
+ f.flush
89
+ end
90
+ sleep eviction_interval * 2
91
+ cursors = stop_tail
92
+ assert cursors.include?(tmp.path)
93
+ assert cursors[tmp.path] == sample.length
94
+ end
95
+
96
+ private
97
+ def tempfile prefix=nil
98
+ Tempfile.new prefix, @tmpdir
99
+ end
100
+
101
+ def start_tail opts={}
102
+ @discover = Franz::Discover.new({
103
+ discover_interval: 1,
104
+ discoveries: @discoveries,
105
+ deletions: @deletions,
106
+ logger: @logger,
107
+ configs: [{
108
+ includes: [ "#{@tmpdir}/*.log" ],
109
+ excludes: [ "#{@tmpdir}/exclude*" ]
110
+ }]
111
+ }.deep_merge!(opts))
112
+
113
+ @watch = Franz::Watch.new({
114
+ watch_interval: 1,
115
+ watch_events: @watch_events,
116
+ discoveries: @discoveries,
117
+ deletions: @deletions,
118
+ logger: @logger
119
+ }.deep_merge!(opts))
120
+
121
+ @tail = Franz::Tail.new({
122
+ eviction_interval: 1,
123
+ watch_events: @watch_events,
124
+ tail_events: @tail_events,
125
+ logger: @logger
126
+ }.deep_merge!(opts))
127
+ end
128
+
129
+ def stop_tail
130
+ @tail.stop
131
+ end
132
+ end
@@ -0,0 +1,144 @@
1
+ require 'thread'
2
+ require 'tmpdir'
3
+ require 'tempfile'
4
+ require 'fileutils'
5
+ require 'minitest/autorun'
6
+
7
+ require 'deep_merge'
8
+
9
+ require_relative '../lib/franz'
10
+
11
+ Thread.abort_on_exception = true
12
+
13
+ class TestFranzWatch < MiniTest::Test
14
+ def setup
15
+ @tmpdir = Dir.mktmpdir
16
+ @discoveries = Queue.new
17
+ @deletions = Queue.new
18
+ @queue = Queue.new
19
+ @logger = Logger.new STDERR
20
+ @logger.level = Logger::WARN
21
+ end
22
+
23
+ def teardown
24
+ FileUtils.rm_rf @tmpdir
25
+ end
26
+
27
+ def test_handles_existing_file
28
+ tmp = tempfile %w[ test .log ]
29
+ start_watch
30
+ sleep 2
31
+ stats = stop_watch
32
+ assert stats.include?(tmp.path)
33
+ assert stats[tmp.path][:size] == 0
34
+ end
35
+
36
+ def test_handles_existing_file_with_content
37
+ content = "Hello, world!\n"
38
+ tmp = tempfile %w[ test .log ]
39
+ tmp.write content
40
+ tmp.flush
41
+ start_watch
42
+ sleep 2
43
+ stats = stop_watch
44
+ assert stats.include?(tmp.path)
45
+ assert stats[tmp.path][:size] == content.length
46
+ end
47
+
48
+ def test_handles_new_file
49
+ start_watch
50
+ tmp = tempfile %w[ test .log ]
51
+ sleep 3
52
+ stats = stop_watch
53
+ assert stats.include?(tmp.path)
54
+ assert stats[tmp.path][:size] == 0
55
+ end
56
+
57
+ def test_handles_new_file_with_content
58
+ start_watch
59
+ content = "Hello, world!\n"
60
+ tmp = tempfile %w[ test .log ]
61
+ tmp.write content
62
+ tmp.flush
63
+ sleep 3
64
+ stats = stop_watch
65
+ assert stats.include?(tmp.path)
66
+ assert stats[tmp.path][:size] == content.length
67
+ end
68
+
69
+ def test_handles_file_truncated
70
+ long_content = "Hello, world!\n"
71
+ short_content = "Bye!\n"
72
+
73
+ tmp = tempfile %w[ test .log ]
74
+ tmp.write long_content
75
+ tmp.flush
76
+
77
+ start_watch
78
+ sleep 2
79
+ tmp.rewind
80
+ tmp.truncate 0
81
+ tmp.write short_content
82
+ tmp.flush
83
+ sleep 3
84
+
85
+ stats = stop_watch
86
+ assert stats.include?(tmp.path)
87
+ assert stats[tmp.path][:size] == short_content.length
88
+ end
89
+
90
+ def test_handles_file_replaced
91
+ content1 = "Hello, world!\n"
92
+ content2 = "Bye!\n"
93
+
94
+ tmp1 = tempfile %w[ test .log ]
95
+ tmp1.write content1
96
+ tmp1.flush
97
+ tmp1.close
98
+
99
+ tmp2 = tempfile %w[ exclude .log ]
100
+ tmp2.write content2
101
+ tmp2.flush
102
+ tmp2.close
103
+
104
+ start_watch
105
+ sleep 2
106
+ FileUtils.ln_sf tmp2.path, tmp1.path
107
+ sleep 3
108
+
109
+ stats = stop_watch
110
+ assert stats.include?(tmp1.path)
111
+ assert stats[tmp1.path][:size] == content2.length
112
+ end
113
+
114
+ private
115
+ def tempfile prefix=nil
116
+ Tempfile.new(prefix, @tmpdir)
117
+ end
118
+
119
+ def start_watch opts={}
120
+ @discover = Franz::Discover.new({
121
+ discover_interval: 1,
122
+ discoveries: @discoveries,
123
+ deletions: @deletions,
124
+ logger: @logger,
125
+ configs: [{
126
+ includes: [ "#{@tmpdir}/*.log" ],
127
+ excludes: [ "#{@tmpdir}/exclude*" ]
128
+ }]
129
+ })
130
+
131
+ @watch = Franz::Watch.new({
132
+ watch_interval: 1,
133
+ watch_events: @queue,
134
+ discoveries: @discoveries,
135
+ deletions: @deletions,
136
+ logger: @logger,
137
+ stats: Hash.new
138
+ }.deep_merge!(opts))
139
+ end
140
+
141
+ def stop_watch
142
+ @watch.stop
143
+ end
144
+ end