franz 1.3.1 → 1.4.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Readme.md +2 -2
- data/VERSION +1 -1
- data/bin/franz +35 -29
- data/franz.gemspec +1 -2
- data/lib/franz/agg.rb +71 -27
- data/lib/franz/discover.rb +26 -31
- data/lib/franz/input.rb +39 -32
- data/lib/franz/output.rb +33 -8
- data/lib/franz/sash.rb +16 -1
- data/lib/franz/tail.rb +42 -103
- data/lib/franz/watch.rb +61 -19
- data/lib/franz.rb +0 -3
- data/test/test_franz_agg.rb +34 -4
- data/test/test_franz_discover.rb +7 -7
- data/test/test_franz_tail.rb +22 -24
- data/test/test_franz_watch.rb +7 -7
- data/test/test_performance.rb +126 -0
- metadata +6 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ee78d80face7d2a796de3551493f5587300b236
|
4
|
+
data.tar.gz: 573935e9612b3c002f3430d9d19d6374e6354e71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d9eb1d74c5f3aafe0c05324a66831ae578936f7fd4aad758a792329c77286ccef990590669356ecc95364245c5319ce2cf76edcdc4ea81f8f3a675d5a126071
|
7
|
+
data.tar.gz: bf437303886139f402244f4161075341564ad36d161df59fc58f98f834bf50a59ac67a4fa55a14e42047bbb697eb835be4f24b48d68724ae3492c37b33732e6d
|
data/Readme.md
CHANGED
@@ -34,13 +34,13 @@ But it's not yet officially sanctioned. Such is life. At any rate, you don't
|
|
34
34
|
have to deal with this issue in Franz, he flushes inactive buffers after a time.
|
35
35
|
Easy-peasy, lemon-squeezy.
|
36
36
|
|
37
|
-
### File
|
37
|
+
### File Hande-ing
|
38
38
|
|
39
39
|
Now I'm not actually sure this issue affects logstash proper, but it's one you
|
40
40
|
might face if you decide to write your own, so here goes: If you're tailing a
|
41
41
|
bunch of files and you never let go of their file handles, you might very well
|
42
42
|
exhaust your ulimit after running for a while. Because Franz is designed to be
|
43
|
-
a daemon, he
|
43
|
+
a daemon, he only opens file handles when necessary.
|
44
44
|
|
45
45
|
### Sequential Identifiers
|
46
46
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.4.14
|
data/bin/franz
CHANGED
@@ -45,37 +45,43 @@ config = Franz::Config.new opts[:config]
|
|
45
45
|
logger = Franz::Logger.new opts[:debug], opts[:trace], opts[:log]
|
46
46
|
|
47
47
|
io_bound = config[:output][:bound] || 10_000
|
48
|
+
io = SizedQueue.new io_bound
|
48
49
|
|
49
|
-
#
|
50
|
-
|
50
|
+
# Now we'll connect to our output, RabbitMQ. This creates a new thread in the
|
51
|
+
# background, which will consume the events generated by our input on io
|
52
|
+
Franz::Output.new \
|
53
|
+
input: io,
|
54
|
+
output: config[:output][:rabbitmq],
|
55
|
+
logger: logger,
|
56
|
+
tags: config[:output][:tags]
|
51
57
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
58
|
+
# Franz has only one kind of input, plain text files.
|
59
|
+
Franz::Input.new \
|
60
|
+
input: config[:input],
|
61
|
+
output: io,
|
62
|
+
logger: logger,
|
63
|
+
checkpoint: config[:checkpoint],
|
64
|
+
checkpoint_interval: config[:checkpoint_interval]
|
59
65
|
|
60
|
-
|
61
|
-
|
62
|
-
input: config[:input],
|
63
|
-
output: io,
|
64
|
-
logger: logger,
|
65
|
-
checkpoint: config[:checkpoint],
|
66
|
-
checkpoint_interval: config[:checkpoint_interval]
|
66
|
+
# Ensure memory doesn't grow too large (> 1GB by default)
|
67
|
+
def mem_kb ; `ps -o rss= -p #{$$}`.strip.to_i ; end
|
67
68
|
|
68
|
-
|
69
|
-
|
70
|
-
fout.join
|
69
|
+
mem_limit = config[:memory_limit] || 1_000_000
|
70
|
+
mem_sleep = config[:memory_limit_interval] || 60
|
71
71
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
72
|
+
loop do
|
73
|
+
sleep mem_sleep
|
74
|
+
mem_used = mem_kb
|
75
|
+
if mem_used > mem_limit
|
76
|
+
logger.fatal \
|
77
|
+
event: 'killed',
|
78
|
+
reason: 'Consuming too much memory',
|
79
|
+
used: mem_used,
|
80
|
+
limit: mem_limit
|
81
|
+
exit(1)
|
82
|
+
end
|
83
|
+
logger.debug \
|
84
|
+
event: 'memcheck',
|
85
|
+
used: mem_used,
|
86
|
+
limit: mem_limit
|
87
|
+
end
|
data/franz.gemspec
CHANGED
@@ -12,11 +12,10 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.description = Franz::SUMMARY + '.'
|
13
13
|
|
14
14
|
s.add_runtime_dependency 'bunny', '~> 1'
|
15
|
-
s.add_runtime_dependency 'buftok', '~> 0'
|
16
15
|
s.add_runtime_dependency 'trollop', '~> 2'
|
17
16
|
s.add_runtime_dependency 'colorize', '~> 0'
|
18
17
|
s.add_runtime_dependency 'deep_merge', '~> 1'
|
19
|
-
s.add_runtime_dependency '
|
18
|
+
s.add_runtime_dependency 'eventmachine', '~> 1'
|
20
19
|
|
21
20
|
s.files = `git ls-files`.split("\n")
|
22
21
|
s.test_files = `git ls-files -- test/*`.split("\n")
|
data/lib/franz/agg.rb
CHANGED
@@ -29,18 +29,17 @@ module Franz
|
|
29
29
|
@tail_events = opts[:tail_events] || []
|
30
30
|
@agg_events = opts[:agg_events] || []
|
31
31
|
|
32
|
+
@buffer_limit = opts[:buffer_limit] || 50
|
32
33
|
@flush_interval = opts[:flush_interval] || 10
|
33
34
|
@seqs = opts[:seqs] || Hash.new
|
34
35
|
@logger = opts[:logger] || Logger.new(STDOUT)
|
35
36
|
|
36
37
|
@types = Hash.new
|
37
|
-
@lock = Mutex.new
|
38
|
+
@lock = Hash.new { |h,k| h[k] = Mutex.new }
|
38
39
|
@buffer = Franz::Sash.new
|
39
40
|
@stop = false
|
40
41
|
|
41
|
-
|
42
|
-
@configs, @tail_events, @agg_events
|
43
|
-
]
|
42
|
+
@num_events = 0
|
44
43
|
|
45
44
|
@t1 = Thread.new do
|
46
45
|
until @stop
|
@@ -54,7 +53,11 @@ module Franz
|
|
54
53
|
capture until @stop
|
55
54
|
end
|
56
55
|
|
57
|
-
log.debug
|
56
|
+
log.debug \
|
57
|
+
event: 'agg started',
|
58
|
+
configs: @configs,
|
59
|
+
tail_events: @tail_events,
|
60
|
+
agg_events: @agg_events
|
58
61
|
end
|
59
62
|
|
60
63
|
# Stop the Agg thread. Effectively only once.
|
@@ -65,7 +68,7 @@ module Franz
|
|
65
68
|
@stop = true
|
66
69
|
@t2.kill
|
67
70
|
@t1.join
|
68
|
-
log.debug 'stopped
|
71
|
+
log.debug event: 'agg stopped'
|
69
72
|
return state
|
70
73
|
end
|
71
74
|
|
@@ -92,45 +95,84 @@ module Franz
|
|
92
95
|
}
|
93
96
|
included && !excluded
|
94
97
|
}
|
95
|
-
|
98
|
+
unless type.nil?
|
99
|
+
@types[path] = type
|
100
|
+
return type
|
101
|
+
end
|
96
102
|
end
|
97
|
-
log.
|
103
|
+
log.warn \
|
104
|
+
event: 'type unknown',
|
105
|
+
path: path
|
106
|
+
@types[path] = nil
|
107
|
+
return nil
|
98
108
|
end
|
99
109
|
end
|
100
110
|
|
101
111
|
def config path
|
102
|
-
|
112
|
+
t = type(path)
|
113
|
+
configs.select { |c| c[:type] == t }.shift
|
103
114
|
end
|
104
115
|
|
105
116
|
def seq path
|
106
117
|
seqs[path] = seqs.fetch(path, 0) + 1
|
107
118
|
end
|
108
119
|
|
109
|
-
def
|
110
|
-
|
120
|
+
def drop? path, message
|
121
|
+
drop = config(path)[:drop]
|
122
|
+
if drop
|
123
|
+
drop = drop.is_a?(Array) ? drop : [ drop ]
|
124
|
+
drop.each do |pattern|
|
125
|
+
return true if message =~ pattern
|
126
|
+
end
|
127
|
+
end
|
128
|
+
return false
|
111
129
|
end
|
112
130
|
|
113
131
|
def enqueue path, message
|
114
|
-
|
132
|
+
if drop? path, message
|
133
|
+
log.trace \
|
134
|
+
event: 'dropped',
|
135
|
+
path: path,
|
136
|
+
message: message
|
137
|
+
return
|
138
|
+
end
|
139
|
+
|
115
140
|
t = type path
|
141
|
+
if t.nil?
|
142
|
+
log.trace \
|
143
|
+
event: 'enqueue skipped',
|
144
|
+
path: path,
|
145
|
+
message: message
|
146
|
+
return
|
147
|
+
end
|
148
|
+
|
149
|
+
log.trace \
|
150
|
+
event: 'enqueue',
|
151
|
+
path: path,
|
152
|
+
message: message
|
116
153
|
s = seq path
|
117
154
|
m = message.encode 'UTF-8', invalid: :replace, undef: :replace, replace: '?'
|
118
|
-
|
119
|
-
t.inspect, p.inspect, s.inspect, m.inspect
|
120
|
-
]
|
121
|
-
agg_events.push path: p, message: m, type: t, host: @@host, '@seq' => s
|
155
|
+
agg_events.push path: path, message: m, type: t, host: @@host, '@seq' => s
|
122
156
|
end
|
123
157
|
|
124
158
|
def capture
|
125
159
|
event = tail_events.shift
|
126
|
-
log.trace
|
127
|
-
event
|
128
|
-
|
129
|
-
multiline = config(event[:path])[:multiline]
|
160
|
+
log.trace \
|
161
|
+
event: 'capture',
|
162
|
+
raw: event
|
163
|
+
multiline = config(event[:path])[:multiline] rescue nil
|
130
164
|
if multiline.nil?
|
131
165
|
enqueue event[:path], event[:line] unless event[:line].empty?
|
132
166
|
else
|
133
|
-
lock.synchronize do
|
167
|
+
lock[event[:path]].synchronize do
|
168
|
+
size = buffer.size(event[:path])
|
169
|
+
if size > @buffer_limit
|
170
|
+
log.trace \
|
171
|
+
event: 'buffer overflow',
|
172
|
+
path: event[:path],
|
173
|
+
size: size,
|
174
|
+
limmit: @buffer_limit
|
175
|
+
end
|
134
176
|
if event[:line] =~ multiline
|
135
177
|
buffered = buffer.flush(event[:path])
|
136
178
|
lines = buffered.map { |e| e[:line] }.join("\n")
|
@@ -141,12 +183,14 @@ module Franz
|
|
141
183
|
end
|
142
184
|
end
|
143
185
|
|
144
|
-
def flush force=false
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
186
|
+
def flush force=false, started=Time.now
|
187
|
+
log.debug \
|
188
|
+
event: 'flush',
|
189
|
+
force: force,
|
190
|
+
started: started
|
191
|
+
buffer.keys.each do |path|
|
192
|
+
lock[path].synchronize do
|
193
|
+
if force || started - buffer.mtime(path) >= flush_interval
|
150
194
|
buffered = buffer.remove(path)
|
151
195
|
lines = buffered.map { |e| e[:line] }.join("\n")
|
152
196
|
enqueue path, lines unless lines.empty?
|
data/lib/franz/discover.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
+
require 'set'
|
1
2
|
require 'logger'
|
3
|
+
require 'shellwords'
|
2
4
|
|
3
5
|
|
4
6
|
# Discover performs half of file existence detection by expanding globs and
|
@@ -25,35 +27,45 @@ class Franz::Discover
|
|
25
27
|
@known = opts[:known] || []
|
26
28
|
@logger = opts[:logger] || Logger.new(STDOUT)
|
27
29
|
|
30
|
+
@known = Set.new(@known)
|
31
|
+
|
28
32
|
@configs = configs.map do |config|
|
29
33
|
config[:includes] ||= []
|
30
34
|
config[:excludes] ||= []
|
31
35
|
config
|
32
36
|
end
|
33
37
|
|
34
|
-
@stop = false
|
35
38
|
|
36
|
-
|
37
|
-
@configs, @discoveries, @deletions
|
38
|
-
]
|
39
|
+
@stop = false
|
39
40
|
|
40
41
|
@thread = Thread.new do
|
41
42
|
until @stop
|
42
43
|
until deletions.empty?
|
43
44
|
d = deletions.pop
|
44
45
|
@known.delete d
|
45
|
-
log.debug
|
46
|
+
log.debug \
|
47
|
+
event: 'discover deleted',
|
48
|
+
path: d
|
46
49
|
end
|
50
|
+
|
47
51
|
discover.each do |discovery|
|
48
52
|
discoveries.push discovery
|
49
|
-
@known.
|
50
|
-
log.debug
|
53
|
+
@known.add discovery
|
54
|
+
log.debug \
|
55
|
+
event: 'discover discovered',
|
56
|
+
path: discovery
|
51
57
|
end
|
52
58
|
sleep discover_interval
|
53
59
|
end
|
54
60
|
end
|
55
61
|
|
56
|
-
log.debug
|
62
|
+
log.debug \
|
63
|
+
event: 'discover started',
|
64
|
+
configs: configs,
|
65
|
+
discoveries: discoveries,
|
66
|
+
deletions: deletions,
|
67
|
+
discover_interval: discover_interval,
|
68
|
+
ignore_before: ignore_before
|
57
69
|
end
|
58
70
|
|
59
71
|
# Stop the Discover thread. Effectively only once.
|
@@ -63,51 +75,34 @@ class Franz::Discover
|
|
63
75
|
return state if @stop
|
64
76
|
@stop = true
|
65
77
|
@thread.kill
|
66
|
-
log.debug 'stopped
|
78
|
+
log.debug event: 'discover stopped'
|
67
79
|
return state
|
68
80
|
end
|
69
81
|
|
70
82
|
# Return the internal "known" state
|
71
83
|
def state
|
72
|
-
return @known.
|
84
|
+
return @known.to_a
|
73
85
|
end
|
74
86
|
|
75
87
|
private
|
76
|
-
attr_reader :configs, :discoveries, :deletions, :discover_interval, :known
|
88
|
+
attr_reader :configs, :discoveries, :deletions, :discover_interval, :known, :ignore_before
|
77
89
|
|
78
90
|
def log ; @logger end
|
79
91
|
|
80
92
|
def discover
|
93
|
+
log.debug event: 'discover'
|
81
94
|
discovered = []
|
82
95
|
configs.each do |config|
|
83
96
|
config[:includes].each do |glob|
|
84
|
-
|
97
|
+
Dir[glob].each do |path|
|
98
|
+
next if known.include? path
|
85
99
|
next if config[:excludes].any? { |exclude|
|
86
100
|
File.fnmatch? exclude, File::basename(path)
|
87
101
|
}
|
88
|
-
next if known.include? path
|
89
|
-
next unless File.file? path
|
90
|
-
next if File.mtime(path).to_i <= @ignore_before
|
91
102
|
discovered.push path
|
92
103
|
end
|
93
104
|
end
|
94
105
|
end
|
95
106
|
return discovered
|
96
107
|
end
|
97
|
-
|
98
|
-
def expand glob
|
99
|
-
Dir[glob]
|
100
|
-
# dir_glob = File.dirname(glob)
|
101
|
-
# file_glob = File.basename(glob)
|
102
|
-
# files = []
|
103
|
-
# Dir.glob(dir_glob).each do |dir|
|
104
|
-
# next unless File::directory?(dir)
|
105
|
-
# Dir.foreach(dir) do |fname|
|
106
|
-
# next if fname == '.' || fname == '..'
|
107
|
-
# next unless File.fnmatch?(file_glob, fname)
|
108
|
-
# files << File.join(dir, fname)
|
109
|
-
# end
|
110
|
-
# end
|
111
|
-
# files
|
112
|
-
end
|
113
108
|
end
|
data/lib/franz/input.rb
CHANGED
@@ -37,6 +37,11 @@ module Franz
|
|
37
37
|
watch_interval: nil,
|
38
38
|
eviction_interval: nil,
|
39
39
|
flush_interval: nil,
|
40
|
+
buffer_limit: nil,
|
41
|
+
line_limit: nil,
|
42
|
+
play_catchup?: nil,
|
43
|
+
skip_interval: nil,
|
44
|
+
stale_interval: nil,
|
40
45
|
configs: []
|
41
46
|
}
|
42
47
|
}.deep_merge!(opts)
|
@@ -47,8 +52,6 @@ module Franz
|
|
47
52
|
@checkpoint_path = opts[:checkpoint].sub('*', '%d')
|
48
53
|
@checkpoint_glob = opts[:checkpoint]
|
49
54
|
|
50
|
-
log.debug 'input: opts=%s' % JSON::pretty_generate(opts)
|
51
|
-
|
52
55
|
# The checkpoint contains a Marshalled Hash with a compact representation of
|
53
56
|
# stateful inputs to various Franz streaming classes (e.g. the "known" option
|
54
57
|
# to Franz::Discover). This state file is generated automatically every time
|
@@ -60,9 +63,13 @@ module Franz
|
|
60
63
|
unless last_checkpoint_path.nil?
|
61
64
|
last_checkpoint = File.read(last_checkpoint_path)
|
62
65
|
state = Marshal.load last_checkpoint
|
63
|
-
log.
|
66
|
+
log.info \
|
67
|
+
event: 'input checkpoint loaded',
|
68
|
+
checkpoint: last_checkpoint_path
|
64
69
|
end
|
65
70
|
|
71
|
+
full_state = state.nil? ? nil : state.dup
|
72
|
+
|
66
73
|
state = state || {}
|
67
74
|
known = state.keys
|
68
75
|
stats, cursors, seqs = {}, {}, {}
|
@@ -74,14 +81,11 @@ module Franz
|
|
74
81
|
stats[path] = state[path]
|
75
82
|
end
|
76
83
|
|
77
|
-
log.debug 'starting input...'
|
78
|
-
|
79
84
|
discoveries = SizedQueue.new opts[:input][:discover_bound]
|
80
85
|
deletions = SizedQueue.new opts[:input][:discover_bound]
|
81
86
|
watch_events = SizedQueue.new opts[:input][:watch_bound]
|
82
87
|
tail_events = SizedQueue.new opts[:input][:tail_bound]
|
83
88
|
|
84
|
-
log.debug 'starting discover...'
|
85
89
|
@disover = Franz::Discover.new \
|
86
90
|
discoveries: discoveries,
|
87
91
|
deletions: deletions,
|
@@ -89,44 +93,49 @@ module Franz
|
|
89
93
|
discover_interval: opts[:input][:discover_interval],
|
90
94
|
ignore_before: opts[:input][:ignore_before],
|
91
95
|
logger: opts[:logger],
|
92
|
-
known: known
|
96
|
+
known: known,
|
97
|
+
full_state: full_state
|
98
|
+
|
99
|
+
@watch = Franz::Watch.new \
|
100
|
+
discoveries: discoveries,
|
101
|
+
deletions: deletions,
|
102
|
+
watch_events: watch_events,
|
103
|
+
watch_interval: opts[:input][:watch_interval],
|
104
|
+
play_catchup?: opts[:input][:play_catchup?],
|
105
|
+
skip_interval: opts[:input][:skip_interval],
|
106
|
+
stale_interval: opts[:input][:stale_interval],
|
107
|
+
logger: opts[:logger],
|
108
|
+
stats: stats,
|
109
|
+
full_state: full_state
|
93
110
|
|
94
|
-
log.debug 'starting tail...'
|
95
111
|
@tail = Franz::Tail.new \
|
96
112
|
watch_events: watch_events,
|
97
113
|
tail_events: tail_events,
|
98
|
-
|
114
|
+
block_size: opts[:input][:block_size],
|
115
|
+
line_limit: opts[:input][:line_limit],
|
99
116
|
logger: opts[:logger],
|
100
|
-
cursors: cursors
|
117
|
+
cursors: cursors,
|
118
|
+
full_state: full_state
|
101
119
|
|
102
|
-
log.debug 'starting agg...'
|
103
120
|
@agg = Franz::Agg.new \
|
104
121
|
configs: opts[:input][:configs],
|
105
122
|
tail_events: tail_events,
|
106
123
|
agg_events: opts[:output],
|
107
124
|
flush_interval: opts[:input][:flush_interval],
|
125
|
+
buffer_limit: opts[:input][:buffer_limit],
|
108
126
|
logger: opts[:logger],
|
109
|
-
seqs: seqs
|
110
|
-
|
111
|
-
log.debug 'starting watch...'
|
112
|
-
@watch = Franz::Watch.new \
|
113
|
-
discoveries: discoveries,
|
114
|
-
deletions: deletions,
|
115
|
-
watch_events: watch_events,
|
116
|
-
watch_interval: opts[:input][:watch_interval],
|
117
|
-
logger: opts[:logger],
|
118
|
-
stats: stats
|
127
|
+
seqs: seqs,
|
128
|
+
full_state: full_state
|
119
129
|
|
120
130
|
@stop = false
|
121
131
|
@t = Thread.new do
|
122
|
-
log.debug 'starting checkpoint'
|
123
132
|
until @stop
|
124
133
|
checkpoint
|
125
134
|
sleep @checkpoint_interval
|
126
135
|
end
|
127
136
|
end
|
128
137
|
|
129
|
-
log.
|
138
|
+
log.info event: 'input started'
|
130
139
|
end
|
131
140
|
|
132
141
|
# Stop everything. Has the effect of draining all the Queues and waiting on
|
@@ -141,7 +150,7 @@ module Franz
|
|
141
150
|
@watch.stop
|
142
151
|
@tail.stop
|
143
152
|
@agg.stop
|
144
|
-
log.
|
153
|
+
log.info event: 'input stopped'
|
145
154
|
return state
|
146
155
|
end
|
147
156
|
|
@@ -162,14 +171,12 @@ module Franz
|
|
162
171
|
def checkpoint
|
163
172
|
old_checkpoints = Dir[@checkpoint_glob].sort_by { |p| File.mtime p }
|
164
173
|
path = @checkpoint_path % Time.now
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
log.warn 'Could not write checkpoint (too many open files)'
|
172
|
-
end
|
174
|
+
File.open(path, 'w') { |f| f.write Marshal.dump(state) }
|
175
|
+
old_checkpoints.pop # Keep last two checkpoints
|
176
|
+
old_checkpoints.map { |c| FileUtils.rm c }
|
177
|
+
log.info \
|
178
|
+
event: 'input checkpoint saved',
|
179
|
+
checkpoint: path
|
173
180
|
end
|
174
181
|
|
175
182
|
private
|
data/lib/franz/output.rb
CHANGED
@@ -34,17 +34,18 @@ module Franz
|
|
34
34
|
|
35
35
|
@logger = opts[:logger]
|
36
36
|
|
37
|
-
rabbit = Bunny.new opts[:output][:connection].merge
|
38
|
-
|
39
|
-
|
40
|
-
|
37
|
+
rabbit = Bunny.new opts[:output][:connection].merge({
|
38
|
+
network_recovery_interval: 10.0,
|
39
|
+
continuation_timeout: 10_000,
|
40
|
+
threaded: false
|
41
|
+
})
|
41
42
|
|
42
43
|
rabbit.start
|
43
44
|
|
44
45
|
channel = rabbit.create_channel
|
45
46
|
exchange = opts[:output][:exchange].delete(:name)
|
46
47
|
exchange = channel.exchange exchange, \
|
47
|
-
|
48
|
+
{ type: 'x-consistent-hash' }.merge(opts[:output][:exchange])
|
48
49
|
|
49
50
|
@stop = false
|
50
51
|
@foreground = opts[:foreground]
|
@@ -53,15 +54,38 @@ module Franz
|
|
53
54
|
rand = Random.new
|
54
55
|
until @stop
|
55
56
|
event = opts[:input].shift
|
56
|
-
|
57
|
-
|
57
|
+
|
58
|
+
event[:path] = event[:path].sub('/home/denimuser/seam-builds/rel', '')
|
59
|
+
event[:path] = event[:path].sub('/home/denimuser/seam-builds/live', '')
|
60
|
+
event[:path] = event[:path].sub('/home/denimuser/seam-builds/beta', '')
|
61
|
+
event[:path] = event[:path].sub('/home/denimuser/builds/rel', '')
|
62
|
+
event[:path] = event[:path].sub('/home/denimuser/builds/live', '')
|
63
|
+
event[:path] = event[:path].sub('/home/denimuser/builds/beta', '')
|
64
|
+
event[:path] = event[:path].sub('/home/denimuser/cobalt-builds/rel', '')
|
65
|
+
event[:path] = event[:path].sub('/home/denimuser/cobalt-builds/live', '')
|
66
|
+
event[:path] = event[:path].sub('/home/denimuser/cobalt-builds/beta', '')
|
67
|
+
event[:path] = event[:path].sub('/home/denimuser/rivet-builds', '')
|
68
|
+
event[:path] = event[:path].sub('/home/denimuser/denim/logs', '')
|
69
|
+
event[:path] = event[:path].sub('/home/denimuser/seam/logs', '')
|
70
|
+
event[:path] = event[:path].sub('/home/denimuser/rivet/bjn/logs', '')
|
71
|
+
event[:path] = event[:path].sub('/home/denimuser', '')
|
72
|
+
event[:path] = event[:path].sub('/var/log', '')
|
73
|
+
|
74
|
+
log.trace \
|
75
|
+
event: 'publish',
|
76
|
+
raw: event
|
77
|
+
|
58
78
|
exchange.publish \
|
59
79
|
JSON::generate(event),
|
60
|
-
routing_key: rand.rand(
|
80
|
+
routing_key: rand.rand(10_000),
|
61
81
|
persistent: false
|
62
82
|
end
|
63
83
|
end
|
64
84
|
|
85
|
+
log.debug \
|
86
|
+
event: 'output started',
|
87
|
+
foreground: @foreground
|
88
|
+
|
65
89
|
@thread.join if @foreground
|
66
90
|
end
|
67
91
|
|
@@ -77,6 +101,7 @@ module Franz
|
|
77
101
|
return if @foreground
|
78
102
|
@foreground = true
|
79
103
|
@thread.kill
|
104
|
+
log.debug event: 'output stopped'
|
80
105
|
end
|
81
106
|
|
82
107
|
private
|
data/lib/franz/sash.rb
CHANGED
@@ -21,6 +21,7 @@ module Franz
|
|
21
21
|
@mutex = Mutex.new
|
22
22
|
@mtime = Hash.new { |default, key| default[key] = nil }
|
23
23
|
@hash = Hash.new { |default, key| default[key] = [] }
|
24
|
+
@size = Hash.new { |default, key| default[key] = 0 }
|
24
25
|
end
|
25
26
|
|
26
27
|
# Grab a list of known keys.
|
@@ -37,6 +38,7 @@ module Franz
|
|
37
38
|
def insert key, value
|
38
39
|
@mutex.synchronize do
|
39
40
|
@hash[key] << value
|
41
|
+
@size[key] += 1
|
40
42
|
@mtime[key] = Time.now
|
41
43
|
end
|
42
44
|
return value
|
@@ -54,7 +56,10 @@ module Franz
|
|
54
56
|
# @param [Object] key
|
55
57
|
#
|
56
58
|
# @return [Array<Object>]
|
57
|
-
def remove key
|
59
|
+
def remove key
|
60
|
+
@size[key] -= 1
|
61
|
+
@hash.delete(key)
|
62
|
+
end
|
58
63
|
|
59
64
|
# Return the last time the key's value buffer was modified.
|
60
65
|
#
|
@@ -73,9 +78,19 @@ module Franz
|
|
73
78
|
@mutex.synchronize do
|
74
79
|
value = @hash[key]
|
75
80
|
@hash[key] = []
|
81
|
+
@size[key] = 0
|
76
82
|
@mtime[key] = Time.now
|
77
83
|
end
|
78
84
|
return value
|
79
85
|
end
|
86
|
+
|
87
|
+
# Return the size of a key's value buffer.
|
88
|
+
#
|
89
|
+
# @param [Object] key
|
90
|
+
#
|
91
|
+
# @return [Integer]
|
92
|
+
def size key
|
93
|
+
@size[key]
|
94
|
+
end
|
80
95
|
end
|
81
96
|
end
|