franz 1.4.30 → 1.4.31
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/franz/agg.rb +8 -50
- data/lib/franz/discover.rb +6 -8
- data/lib/franz/input.rb +10 -12
- data/lib/franz/input_config.rb +56 -0
- data/lib/franz/tail.rb +20 -3
- data/lib/franz/watch.rb +4 -5
- data/lib/franz.rb +1 -0
- data/test/test_franz_agg.rb +8 -4
- data/test/test_franz_discover.rb +9 -5
- data/test/test_franz_tail.rb +11 -5
- data/test/test_franz_watch.rb +10 -5
- data/test/test_performance.rb +8 -4
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 65feda5ab2c2a00503e281b1ad629b84a21eace2
|
4
|
+
data.tar.gz: 29d29bf3d409225e8b2b67a546993ba0d153543a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 56306a42aa00a1afe1d1fb810e93618c54ea2f88854501e43eb67fa1e9bfcdce8b18a7df45eb7de6cf204b2202c41060846fbebb8851dacde99931f7cee94cfa
|
7
|
+
data.tar.gz: 952836ce6516ea8de4d347c444c1e8013f100600cb0f99b1ca4b5fda1e86ab38b8d788999b9d3683d87e191e0a8944ff44c9f37326e9a09be630ca1f70f8e483
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.4.
|
1
|
+
1.4.31
|
data/lib/franz/agg.rb
CHANGED
@@ -4,6 +4,7 @@ require 'socket'
|
|
4
4
|
require 'pathname'
|
5
5
|
|
6
6
|
require_relative 'sash'
|
7
|
+
require_relative 'input_config'
|
7
8
|
|
8
9
|
module Franz
|
9
10
|
|
@@ -18,14 +19,15 @@ module Franz
|
|
18
19
|
# Start a new Agg thread in the background.
|
19
20
|
#
|
20
21
|
# @param [Hash] opts options for the aggregator
|
21
|
-
# @option opts [
|
22
|
+
# @option opts [InputConfig] :input_config shared Franz configuration
|
22
23
|
# @option opts [Queue] :tail_events ([]) "input" queue from Tail
|
23
24
|
# @option opts [Queue] :agg_events ([]) "output" queue
|
24
25
|
# @option opts [Integer] :flush_interval (5) seconds between flushes
|
25
26
|
# @option opts [Hash<Path,Fixnum>] :seqs ({}) internal "seqs" state
|
26
27
|
# @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
|
27
28
|
def initialize opts={}
|
28
|
-
@
|
29
|
+
@ic = opts[:input_config] || raise('No input_config specified')
|
30
|
+
|
29
31
|
@tail_events = opts[:tail_events] || []
|
30
32
|
@agg_events = opts[:agg_events] || []
|
31
33
|
|
@@ -34,7 +36,6 @@ module Franz
|
|
34
36
|
@seqs = opts[:seqs] || Hash.new
|
35
37
|
@logger = opts[:logger] || Logger.new(STDOUT)
|
36
38
|
|
37
|
-
@types = Hash.new
|
38
39
|
@lock = Hash.new { |h,k| h[k] = Mutex.new }
|
39
40
|
@buffer = Franz::Sash.new
|
40
41
|
@stop = false
|
@@ -53,7 +54,6 @@ module Franz
|
|
53
54
|
|
54
55
|
log.debug \
|
55
56
|
event: 'agg started',
|
56
|
-
configs: @configs,
|
57
57
|
tail_events: @tail_events,
|
58
58
|
agg_events: @agg_events
|
59
59
|
end
|
@@ -76,58 +76,16 @@ module Franz
|
|
76
76
|
end
|
77
77
|
|
78
78
|
private
|
79
|
-
attr_reader :
|
79
|
+
attr_reader :tail_events, :agg_events, :flush_interval, :lock, :buffer
|
80
80
|
|
81
81
|
def log ; @logger end
|
82
82
|
|
83
|
-
def type path
|
84
|
-
begin
|
85
|
-
@types.fetch path
|
86
|
-
rescue KeyError
|
87
|
-
configs.each do |config|
|
88
|
-
type = config[:type] if config[:includes].any? { |glob|
|
89
|
-
included = File.fnmatch? glob, path
|
90
|
-
excludes = !config[:excludes].nil?
|
91
|
-
excluded = excludes && config[:excludes].any? { |exlude|
|
92
|
-
File.fnmatch? exlude, File::basename(path)
|
93
|
-
}
|
94
|
-
included && !excluded
|
95
|
-
}
|
96
|
-
unless type.nil?
|
97
|
-
@types[path] = type
|
98
|
-
return type
|
99
|
-
end
|
100
|
-
end
|
101
|
-
log.warn \
|
102
|
-
event: 'type unknown',
|
103
|
-
path: path
|
104
|
-
@types[path] = nil
|
105
|
-
return nil
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
def config path
|
110
|
-
t = type(path)
|
111
|
-
configs.select { |c| c[:type] == t }.shift
|
112
|
-
end
|
113
|
-
|
114
83
|
def seq path
|
115
84
|
seqs[path] = seqs.fetch(path, 0) + 1
|
116
85
|
end
|
117
86
|
|
118
|
-
def drop? path, message
|
119
|
-
drop = config(path)[:drop]
|
120
|
-
if drop
|
121
|
-
drop = drop.is_a?(Array) ? drop : [ drop ]
|
122
|
-
drop.each do |pattern|
|
123
|
-
return true if message =~ pattern
|
124
|
-
end
|
125
|
-
end
|
126
|
-
return false
|
127
|
-
end
|
128
|
-
|
129
87
|
def enqueue path, message
|
130
|
-
if drop? path, message
|
88
|
+
if @ic.drop? path, message
|
131
89
|
log.trace \
|
132
90
|
event: 'dropped',
|
133
91
|
path: path,
|
@@ -135,7 +93,7 @@ module Franz
|
|
135
93
|
return
|
136
94
|
end
|
137
95
|
|
138
|
-
t = type path
|
96
|
+
t = @ic.type path
|
139
97
|
if t.nil?
|
140
98
|
log.trace \
|
141
99
|
event: 'enqueue skipped',
|
@@ -158,7 +116,7 @@ module Franz
|
|
158
116
|
log.trace \
|
159
117
|
event: 'capture',
|
160
118
|
raw: event
|
161
|
-
multiline = config(event[:path])[:multiline] rescue nil
|
119
|
+
multiline = @ic.config(event[:path])[:multiline] rescue nil
|
162
120
|
if multiline.nil?
|
163
121
|
enqueue event[:path], event[:line] unless event[:line].empty?
|
164
122
|
else
|
data/lib/franz/discover.rb
CHANGED
@@ -11,25 +11,25 @@ class Franz::Discover
|
|
11
11
|
# Start a new Discover thread in the background.
|
12
12
|
#
|
13
13
|
# @param [Hash] opts options for the discovery
|
14
|
-
# @option opts [
|
14
|
+
# @option opts [InputConfig] :input_config shared Franz configuration
|
15
15
|
# @option opts [Queue] :discoveries ([]) "output" queue of discovered paths
|
16
16
|
# @option opts [Queue] :deletions ([]) "input" queue of deleted paths
|
17
17
|
# @option opts [Integer] :discover_interval (5) seconds between discover rounds
|
18
18
|
# @option opts [Array<Path>] :known ([]) internal "known" state
|
19
19
|
# @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
|
20
20
|
def initialize opts={}
|
21
|
-
@
|
21
|
+
@ic = opts[:input_config] || raise('No input_config specified')
|
22
|
+
|
22
23
|
@discoveries = opts[:discoveries] || []
|
23
24
|
@deletions = opts[:deletions] || []
|
24
25
|
|
25
26
|
@discover_interval = opts[:discover_interval] || 30
|
26
|
-
@ignore_before = opts[:ignore_before] || 0
|
27
27
|
@known = opts[:known] || []
|
28
28
|
@logger = opts[:logger] || Logger.new(STDOUT)
|
29
29
|
|
30
30
|
@known = Set.new(@known)
|
31
31
|
|
32
|
-
@configs = configs.map do |config|
|
32
|
+
@configs = @ic.configs.map do |config|
|
33
33
|
config[:includes] ||= []
|
34
34
|
config[:excludes] ||= []
|
35
35
|
config
|
@@ -61,11 +61,9 @@ class Franz::Discover
|
|
61
61
|
|
62
62
|
log.debug \
|
63
63
|
event: 'discover started',
|
64
|
-
configs: configs,
|
65
64
|
discoveries: discoveries,
|
66
65
|
deletions: deletions,
|
67
|
-
discover_interval: discover_interval
|
68
|
-
ignore_before: ignore_before
|
66
|
+
discover_interval: discover_interval
|
69
67
|
end
|
70
68
|
|
71
69
|
# Stop the Discover thread. Effectively only once.
|
@@ -85,7 +83,7 @@ class Franz::Discover
|
|
85
83
|
end
|
86
84
|
|
87
85
|
private
|
88
|
-
attr_reader :configs, :discoveries, :deletions, :discover_interval, :known
|
86
|
+
attr_reader :configs, :discoveries, :deletions, :discover_interval, :known
|
89
87
|
|
90
88
|
def log ; @logger end
|
91
89
|
|
data/lib/franz/input.rb
CHANGED
@@ -67,8 +67,6 @@ module Franz
|
|
67
67
|
checkpoint: last_checkpoint_path
|
68
68
|
end
|
69
69
|
|
70
|
-
full_state = state.nil? ? nil : state.dup
|
71
|
-
|
72
70
|
state = state || {}
|
73
71
|
known = state.keys
|
74
72
|
stats, cursors, seqs = {}, {}, {}
|
@@ -85,45 +83,45 @@ module Franz
|
|
85
83
|
watch_events = possibly_bounded_queue opts[:input][:watch_bound]
|
86
84
|
tail_events = possibly_bounded_queue opts[:input][:tail_bound]
|
87
85
|
|
86
|
+
ic = InputConfig.new opts[:input][:configs]
|
87
|
+
|
88
88
|
@disover = Franz::Discover.new \
|
89
|
+
input_config: ic,
|
89
90
|
discoveries: discoveries,
|
90
91
|
deletions: deletions,
|
91
|
-
configs: opts[:input][:configs],
|
92
92
|
discover_interval: opts[:input][:discover_interval],
|
93
93
|
ignore_before: opts[:input][:ignore_before],
|
94
94
|
logger: opts[:logger],
|
95
|
-
known: known
|
96
|
-
full_state: full_state
|
95
|
+
known: known
|
97
96
|
|
98
97
|
@watch = Franz::Watch.new \
|
98
|
+
input_config: ic,
|
99
99
|
discoveries: discoveries,
|
100
100
|
deletions: deletions,
|
101
101
|
watch_events: watch_events,
|
102
102
|
watch_interval: opts[:input][:watch_interval],
|
103
103
|
play_catchup?: opts[:input][:play_catchup?],
|
104
104
|
logger: opts[:logger],
|
105
|
-
stats: stats
|
106
|
-
full_state: full_state
|
105
|
+
stats: stats
|
107
106
|
|
108
107
|
@tail = Franz::Tail.new \
|
108
|
+
input_config: ic,
|
109
109
|
watch_events: watch_events,
|
110
110
|
tail_events: tail_events,
|
111
111
|
block_size: opts[:input][:block_size],
|
112
112
|
line_limit: opts[:input][:line_limit],
|
113
113
|
read_limit: opts[:input][:read_limit],
|
114
114
|
logger: opts[:logger],
|
115
|
-
cursors: cursors
|
116
|
-
full_state: full_state
|
115
|
+
cursors: cursors
|
117
116
|
|
118
117
|
@agg = Franz::Agg.new \
|
119
|
-
|
118
|
+
input_config: ic,
|
120
119
|
tail_events: tail_events,
|
121
120
|
agg_events: opts[:output],
|
122
121
|
flush_interval: opts[:input][:flush_interval],
|
123
122
|
buffer_limit: opts[:input][:buffer_limit],
|
124
123
|
logger: opts[:logger],
|
125
|
-
seqs: seqs
|
126
|
-
full_state: full_state
|
124
|
+
seqs: seqs
|
127
125
|
|
128
126
|
@stop = false
|
129
127
|
@t = Thread.new do
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Franz
|
2
|
+
class InputConfig
|
3
|
+
attr_reader :configs
|
4
|
+
|
5
|
+
def initialize configs
|
6
|
+
@configs = configs
|
7
|
+
@types = Hash.new
|
8
|
+
end
|
9
|
+
|
10
|
+
def config path
|
11
|
+
t = type(path)
|
12
|
+
configs.select { |c| c[:type] == t }.shift
|
13
|
+
end
|
14
|
+
|
15
|
+
def drop? path, message
|
16
|
+
begin
|
17
|
+
drop = config(path)[:drop]
|
18
|
+
rescue
|
19
|
+
return true # No config found, drop it!
|
20
|
+
end
|
21
|
+
if drop
|
22
|
+
drop = drop.is_a?(Array) ? drop : [ drop ]
|
23
|
+
drop.each do |pattern|
|
24
|
+
return true if message =~ pattern
|
25
|
+
end
|
26
|
+
end
|
27
|
+
return false
|
28
|
+
end
|
29
|
+
|
30
|
+
def type path
|
31
|
+
begin
|
32
|
+
@types.fetch path
|
33
|
+
rescue KeyError
|
34
|
+
configs.each do |config|
|
35
|
+
type = config[:type] if config[:includes].any? { |glob|
|
36
|
+
included = File.fnmatch? glob, path
|
37
|
+
excludes = !config[:excludes].nil?
|
38
|
+
excluded = excludes && config[:excludes].any? { |exlude|
|
39
|
+
File.fnmatch? exlude, File::basename(path)
|
40
|
+
}
|
41
|
+
included && !excluded
|
42
|
+
}
|
43
|
+
unless type.nil?
|
44
|
+
@types[path] = type
|
45
|
+
return type
|
46
|
+
end
|
47
|
+
end
|
48
|
+
log.warn \
|
49
|
+
event: 'type unknown',
|
50
|
+
path: path
|
51
|
+
@types[path] = nil
|
52
|
+
return nil
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
data/lib/franz/tail.rb
CHANGED
@@ -8,12 +8,19 @@ module Franz
|
|
8
8
|
# Tail receives low-level file events from a Watch and handles the actual
|
9
9
|
# reading of files, providing a stream of lines.
|
10
10
|
class Tail
|
11
|
+
ERR_NIL_READ = 1
|
12
|
+
ERR_INVALID_EVENT = 2
|
13
|
+
ERR_INCOMPLETE_READ = 3
|
14
|
+
|
11
15
|
attr_reader :cursors
|
12
16
|
|
13
17
|
# Start a new Tail thread in the background.
|
14
18
|
#
|
15
19
|
# @param opts [Hash] a complex Hash for tail configuration
|
20
|
+
# @option opts [InputConfig] :input_config shared Franz configuration
|
16
21
|
def initialize opts={}
|
22
|
+
@ic = opts[:input_config] || raise('No input_config specified')
|
23
|
+
|
17
24
|
@watch_events = opts[:watch_events] || []
|
18
25
|
@tail_events = opts[:tail_events] || []
|
19
26
|
|
@@ -67,6 +74,16 @@ module Franz
|
|
67
74
|
@cursors[path] ||= 0
|
68
75
|
spread = size - @cursors[path]
|
69
76
|
|
77
|
+
if spread <= 0
|
78
|
+
log.trace \
|
79
|
+
event: 'ignoring read',
|
80
|
+
path: path,
|
81
|
+
size: size,
|
82
|
+
cursor: @cursors[path],
|
83
|
+
spread: spread
|
84
|
+
return
|
85
|
+
end
|
86
|
+
|
70
87
|
if spread > @read_limit
|
71
88
|
log.trace \
|
72
89
|
event: 'large read',
|
@@ -112,7 +129,7 @@ module Franz
|
|
112
129
|
size: size,
|
113
130
|
cursor: @cursors[path],
|
114
131
|
spread: (size - @cursors[path])
|
115
|
-
exit
|
132
|
+
exit ERR_NIL_READ
|
116
133
|
end
|
117
134
|
|
118
135
|
@cursors[path] += size
|
@@ -125,7 +142,7 @@ module Franz
|
|
125
142
|
size: size,
|
126
143
|
cursor: @cursors[path],
|
127
144
|
spread: (size - @cursors[path])
|
128
|
-
exit
|
145
|
+
exit ERR_INCOMPLETE_READ
|
129
146
|
end
|
130
147
|
end
|
131
148
|
|
@@ -152,7 +169,7 @@ module Franz
|
|
152
169
|
close event[:path]
|
153
170
|
else
|
154
171
|
log.fatal event: 'invalid event', raw: event
|
155
|
-
exit
|
172
|
+
exit ERR_INVALID_EVENT
|
156
173
|
end
|
157
174
|
return event[:path]
|
158
175
|
end
|
data/lib/franz/watch.rb
CHANGED
@@ -12,6 +12,7 @@ module Franz
|
|
12
12
|
# Start a new Watch thread in the background.
|
13
13
|
#
|
14
14
|
# @param [Hash] opts options for the watch
|
15
|
+
# @option opts [InputConfig] :input_config shared Franz configuration
|
15
16
|
# @option opts [Queue] :discoveries ([]) "input" queue of discovered paths
|
16
17
|
# @option opts [Queue] :deletions ([]) "output" queue of deleted paths
|
17
18
|
# @option opts [Queue] :watch_events ([]) "output" queue of file events
|
@@ -19,6 +20,8 @@ module Franz
|
|
19
20
|
# @option opts [Hash<Path,State>] :stats ([]) internal "stats" state
|
20
21
|
# @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
|
21
22
|
def initialize opts={}
|
23
|
+
@ic = opts[:input_config] || raise('No input_config specified')
|
24
|
+
|
22
25
|
@discoveries = opts[:discoveries] || []
|
23
26
|
@deletions = opts[:deletions] || []
|
24
27
|
@watch_events = opts[:watch_events] || []
|
@@ -32,7 +35,7 @@ module Franz
|
|
32
35
|
if @play_catchup
|
33
36
|
log.debug event: 'play catchup'
|
34
37
|
stats.keys.each do |path|
|
35
|
-
stats[path][:size] =
|
38
|
+
stats[path][:size] = 0
|
36
39
|
end
|
37
40
|
end
|
38
41
|
|
@@ -97,10 +100,6 @@ module Franz
|
|
97
100
|
deleted = []
|
98
101
|
|
99
102
|
stats.keys.each do |path|
|
100
|
-
# Hacks for logs we've removed
|
101
|
-
next if File.basename(path) =~ /^rtpstat/
|
102
|
-
next if File.basename(path) == 'zuora.log'
|
103
|
-
|
104
103
|
old_stat = stats[path]
|
105
104
|
stat = stat_for path
|
106
105
|
stats[path] = stat
|
data/lib/franz.rb
CHANGED
@@ -2,6 +2,7 @@ require_relative 'franz/agg'
|
|
2
2
|
require_relative 'franz/config'
|
3
3
|
require_relative 'franz/discover'
|
4
4
|
require_relative 'franz/input'
|
5
|
+
require_relative 'franz/input_config'
|
5
6
|
require_relative 'franz/logger'
|
6
7
|
require_relative 'franz/metadata'
|
7
8
|
require_relative 'franz/output'
|
data/test/test_franz_agg.rb
CHANGED
@@ -83,21 +83,24 @@ private
|
|
83
83
|
end
|
84
84
|
|
85
85
|
def start_agg config, opts={}
|
86
|
-
configs = [{
|
86
|
+
@configs = [{
|
87
87
|
type: :test,
|
88
88
|
includes: [ "#{@tmpdir}/*.log", "#{realpath @tmpdir}/*.log" ],
|
89
89
|
excludes: [ "#{@tmpdir}/exclude*" ]
|
90
90
|
}.merge(config)]
|
91
91
|
|
92
|
+
@ic = Franz::InputConfig.new @configs
|
93
|
+
|
92
94
|
@discover = Franz::Discover.new({
|
95
|
+
input_config: @ic,
|
93
96
|
discover_interval: 1,
|
94
97
|
discoveries: @discoveries,
|
95
98
|
deletions: @deletions,
|
96
|
-
logger: @logger
|
97
|
-
configs: configs
|
99
|
+
logger: @logger
|
98
100
|
}.deep_merge!(opts))
|
99
101
|
|
100
102
|
@watch = Franz::Watch.new({
|
103
|
+
input_config: @ic,
|
101
104
|
watch_interval: 1,
|
102
105
|
watch_events: @watch_events,
|
103
106
|
discoveries: @discoveries,
|
@@ -106,6 +109,7 @@ private
|
|
106
109
|
}.deep_merge!(opts))
|
107
110
|
|
108
111
|
@tail = Franz::Tail.new({
|
112
|
+
input_config: @ic,
|
109
113
|
eviction_interval: 1,
|
110
114
|
watch_events: @watch_events,
|
111
115
|
tail_events: @tail_events,
|
@@ -113,7 +117,7 @@ private
|
|
113
117
|
}.deep_merge!(opts))
|
114
118
|
|
115
119
|
@agg = Franz::Agg.new({
|
116
|
-
|
120
|
+
input_config: @ic,
|
117
121
|
flush_interval: 2,
|
118
122
|
tail_events: @tail_events,
|
119
123
|
agg_events: @agg_events,
|
data/test/test_franz_discover.rb
CHANGED
@@ -65,15 +65,19 @@ private
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def start_discovery opts={}
|
68
|
+
@configs = [{
|
69
|
+
includes: [ "#{@tmpdir}/*.log" ],
|
70
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
71
|
+
}]
|
72
|
+
|
73
|
+
@ic = Franz::InputConfig.new @configs
|
74
|
+
|
68
75
|
@discover = Franz::Discover.new({
|
76
|
+
input_config: @ic,
|
69
77
|
discover_interval: 1,
|
70
78
|
discoveries: @discoveries,
|
71
79
|
deletions: @deletions,
|
72
|
-
logger: @logger
|
73
|
-
configs: [{
|
74
|
-
includes: [ "#{@tmpdir}/*.log" ],
|
75
|
-
excludes: [ "#{@tmpdir}/exclude*" ]
|
76
|
-
}]
|
80
|
+
logger: @logger
|
77
81
|
}.deep_merge!(opts))
|
78
82
|
end
|
79
83
|
|
data/test/test_franz_tail.rb
CHANGED
@@ -97,18 +97,23 @@ private
|
|
97
97
|
end
|
98
98
|
|
99
99
|
def start_tail opts={}
|
100
|
+
@configs = [{
|
101
|
+
includes: [ "#{@tmpdir}/*.log" ],
|
102
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
103
|
+
}]
|
104
|
+
|
105
|
+
@ic = Franz::InputConfig.new @configs
|
106
|
+
|
100
107
|
@discover = Franz::Discover.new({
|
108
|
+
input_config: @ic,
|
101
109
|
discover_interval: 1,
|
102
110
|
discoveries: @discoveries,
|
103
111
|
deletions: @deletions,
|
104
|
-
logger: @logger
|
105
|
-
configs: [{
|
106
|
-
includes: [ "#{@tmpdir}/*.log" ],
|
107
|
-
excludes: [ "#{@tmpdir}/exclude*" ]
|
108
|
-
}]
|
112
|
+
logger: @logger
|
109
113
|
}.deep_merge!(opts))
|
110
114
|
|
111
115
|
@watch = Franz::Watch.new({
|
116
|
+
input_config: @ic,
|
112
117
|
watch_interval: 1,
|
113
118
|
watch_events: @watch_events,
|
114
119
|
discoveries: @discoveries,
|
@@ -117,6 +122,7 @@ private
|
|
117
122
|
}.deep_merge!(opts))
|
118
123
|
|
119
124
|
@tail = Franz::Tail.new({
|
125
|
+
input_config: @ic,
|
120
126
|
eviction_interval: 1,
|
121
127
|
watch_events: @watch_events,
|
122
128
|
tail_events: @tail_events,
|
data/test/test_franz_watch.rb
CHANGED
@@ -117,18 +117,23 @@ private
|
|
117
117
|
end
|
118
118
|
|
119
119
|
def start_watch opts={}
|
120
|
+
@configs = [{
|
121
|
+
includes: [ "#{@tmpdir}/*.log" ],
|
122
|
+
excludes: [ "#{@tmpdir}/exclude*" ]
|
123
|
+
}]
|
124
|
+
|
125
|
+
@ic = Franz::InputConfig.new @configs
|
126
|
+
|
120
127
|
@discover = Franz::Discover.new({
|
128
|
+
input_config: @ic,
|
121
129
|
discover_interval: 1,
|
122
130
|
discoveries: @discoveries,
|
123
131
|
deletions: @deletions,
|
124
|
-
logger: @logger
|
125
|
-
configs: [{
|
126
|
-
includes: [ "#{@tmpdir}/*.log" ],
|
127
|
-
excludes: [ "#{@tmpdir}/exclude*" ]
|
128
|
-
}]
|
132
|
+
logger: @logger
|
129
133
|
})
|
130
134
|
|
131
135
|
@watch = Franz::Watch.new({
|
136
|
+
input_config: @ic,
|
132
137
|
watch_interval: 1,
|
133
138
|
watch_events: @queue,
|
134
139
|
discoveries: @discoveries,
|
data/test/test_performance.rb
CHANGED
@@ -82,21 +82,24 @@ private
|
|
82
82
|
end
|
83
83
|
|
84
84
|
def start_agg opts={}
|
85
|
-
configs = [{
|
85
|
+
@configs = [{
|
86
86
|
type: :test,
|
87
87
|
includes: [ "#{@tmpdir}/*.log" ],
|
88
88
|
excludes: [ "#{@tmpdir}/exclude*" ]
|
89
89
|
}]
|
90
90
|
|
91
|
+
@ic = Franz::InputConfig.new @configs
|
92
|
+
|
91
93
|
@discover = Franz::Discover.new({
|
94
|
+
input_config: @ic,
|
92
95
|
discover_interval: @discover_interval,
|
93
96
|
discoveries: @discoveries,
|
94
97
|
deletions: @deletions,
|
95
|
-
logger: @logger
|
96
|
-
configs: configs
|
98
|
+
logger: @logger
|
97
99
|
}.deep_merge!(opts))
|
98
100
|
|
99
101
|
@watch = Franz::Watch.new({
|
102
|
+
input_config: @ic,
|
100
103
|
watch_interval: @watch_interval,
|
101
104
|
watch_events: @watch_events,
|
102
105
|
discoveries: @discoveries,
|
@@ -105,6 +108,7 @@ private
|
|
105
108
|
}.deep_merge!(opts))
|
106
109
|
|
107
110
|
@tail = Franz::Tail.new({
|
111
|
+
input_config: @ic,
|
108
112
|
eviction_interval: @eviction_interval,
|
109
113
|
watch_events: @watch_events,
|
110
114
|
tail_events: @tail_events,
|
@@ -112,7 +116,7 @@ private
|
|
112
116
|
}.deep_merge!(opts))
|
113
117
|
|
114
118
|
@agg = Franz::Agg.new({
|
115
|
-
|
119
|
+
input_config: @ic,
|
116
120
|
flush_interval: @flush_interval,
|
117
121
|
tail_events: @tail_events,
|
118
122
|
agg_events: @agg_events,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: franz
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.4.
|
4
|
+
version: 1.4.31
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sean Clemmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bunny
|
@@ -100,6 +100,7 @@ files:
|
|
100
100
|
- lib/franz/config.rb
|
101
101
|
- lib/franz/discover.rb
|
102
102
|
- lib/franz/input.rb
|
103
|
+
- lib/franz/input_config.rb
|
103
104
|
- lib/franz/logger.rb
|
104
105
|
- lib/franz/metadata.rb
|
105
106
|
- lib/franz/output.rb
|
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
130
131
|
version: '0'
|
131
132
|
requirements: []
|
132
133
|
rubyforge_project:
|
133
|
-
rubygems_version: 2.
|
134
|
+
rubygems_version: 2.4.2
|
134
135
|
signing_key:
|
135
136
|
specification_version: 4
|
136
137
|
summary: Aggregate log file events and send them elsewhere
|