carnivore 0.3.16 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/carnivore.gemspec +4 -4
- data/lib/carnivore.rb +8 -1
- data/lib/carnivore/callback.rb +2 -1
- data/lib/carnivore/container.rb +4 -4
- data/lib/carnivore/logger.rb +5 -0
- data/lib/carnivore/runner.rb +24 -10
- data/lib/carnivore/source.rb +27 -11
- data/lib/carnivore/spec_helper.rb +1 -7
- data/lib/carnivore/supervisor.rb +7 -21
- data/lib/carnivore/utils/failure.rb +2 -0
- data/lib/carnivore/utils/logging.rb +2 -2
- data/lib/carnivore/utils/message_registry.rb +2 -0
- data/lib/carnivore/version.rb +1 -1
- metadata +15 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f7b86794cbfbbd6c6e4516b0050eaebdc7b3786b
|
4
|
+
data.tar.gz: 8587ef4f157a853b54916332fca60486555a6050
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b967f7dde24ea19e0ab8d1d94073afefac43cf5eb4c00cd460ae99a74dc2a5cee0b9a9b3f648737c21a020b8c25cb372116710a3f0e13539bc83c41785df50f8
|
7
|
+
data.tar.gz: 8e145e07ba5d343a0f09c56ad807871286a9de58e8595dc9c284f001ba9841f939c0d52d9e5a3c6878a22b97b0a7b8209266ad14c6c471e14ee11805cb74934d
|
data/CHANGELOG.md
CHANGED
data/carnivore.gemspec
CHANGED
@@ -10,9 +10,9 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.description = 'Message processing helper'
|
11
11
|
s.license = 'Apache 2.0'
|
12
12
|
s.require_path = 'lib'
|
13
|
-
s.
|
14
|
-
s.
|
15
|
-
s.
|
16
|
-
s.
|
13
|
+
s.add_runtime_dependency 'bogo-config'
|
14
|
+
s.add_runtime_dependency 'multi_json'
|
15
|
+
s.add_runtime_dependency 'hashie'
|
16
|
+
s.add_runtime_dependency 'zoidberg', '< 1.0'
|
17
17
|
s.files = Dir['lib/**/*'] + %w(carnivore.gemspec README.md CHANGELOG.md)
|
18
18
|
end
|
data/lib/carnivore.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
# Load supporters on demand
|
2
|
-
autoload :Celluloid, 'celluloid'
|
3
2
|
autoload :MultiJson, 'multi_json'
|
4
3
|
|
4
|
+
require 'securerandom'
|
5
|
+
require 'zoidberg'
|
5
6
|
require 'bogo-config'
|
6
7
|
require 'carnivore/runner'
|
7
8
|
require 'carnivore/version'
|
@@ -11,9 +12,15 @@ module Carnivore
|
|
11
12
|
autoload :Callback, 'carnivore/callback'
|
12
13
|
autoload :Container, 'carnivore/container'
|
13
14
|
autoload :Error, 'carnivore/errors'
|
15
|
+
autoload :Logger, 'carnivore/logger'
|
14
16
|
autoload :Message, 'carnivore/message'
|
15
17
|
autoload :Source, 'carnivore/source'
|
16
18
|
autoload :Supervisor, 'carnivore/supervisor'
|
17
19
|
autoload :Utils, 'carnivore/utils'
|
18
20
|
autoload :Version, 'carnivore/version'
|
21
|
+
|
22
|
+
def self.uuid
|
23
|
+
Zoidberg.uuid
|
24
|
+
end
|
25
|
+
|
19
26
|
end
|
data/lib/carnivore/callback.rb
CHANGED
data/lib/carnivore/container.rb
CHANGED
@@ -9,16 +9,16 @@ module Carnivore
|
|
9
9
|
|
10
10
|
class << self
|
11
11
|
|
12
|
-
# @return [
|
12
|
+
# @return [Logger]
|
13
13
|
def log
|
14
|
-
|
14
|
+
Carnivore::Utils::Logging::Logger
|
15
15
|
end
|
16
16
|
|
17
17
|
end
|
18
18
|
|
19
|
-
# @return [
|
19
|
+
# @return [Logger]
|
20
20
|
def log
|
21
|
-
|
21
|
+
Carnivore::Utils::Logging::Logger
|
22
22
|
end
|
23
23
|
|
24
24
|
end
|
data/lib/carnivore/runner.rb
CHANGED
@@ -43,7 +43,7 @@ module Carnivore
|
|
43
43
|
require 'carnivore/supervisor'
|
44
44
|
configure!(:verify)
|
45
45
|
supervisor = Carnivore::Supervisor.build!
|
46
|
-
|
46
|
+
Carnivore::Logger.info 'Initializing all registered sources.'
|
47
47
|
[].tap do |register|
|
48
48
|
Source.sources.each do |source|
|
49
49
|
register << Thread.new do
|
@@ -55,7 +55,12 @@ module Carnivore
|
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end.map(&:join)
|
58
|
-
|
58
|
+
Carnivore::Logger.info 'Source initializations complete. Running setup and establishing connections.'
|
59
|
+
Source.sources.each do |source|
|
60
|
+
supervisor[source.source_hash[:name]].run_setup
|
61
|
+
supervisor[source.source_hash[:name]].run_connect
|
62
|
+
end
|
63
|
+
Carnivore::Logger.info 'Sources setup and connected. Enabling message processing.'
|
59
64
|
Source.sources.each do |source|
|
60
65
|
if(source.source_hash.fetch(:auto_process, true))
|
61
66
|
supervisor[source.source_hash[:name]].start!
|
@@ -65,23 +70,32 @@ module Carnivore
|
|
65
70
|
# We do a sleep loop so we can periodically check on the
|
66
71
|
# supervisor and ensure it is still alive. If it has died,
|
67
72
|
# raise exception to allow cleanup and restart attempt
|
68
|
-
|
69
|
-
|
73
|
+
gc_interval = Carnivore::Config.fetch(:carnivore, :garbage_interval, 30)
|
74
|
+
gc_last = Time.now.to_i
|
75
|
+
while(supervisor.alive?)
|
76
|
+
sleep Carnivore::Config.get(:carnivore, :supervisor, :poll) || 5
|
77
|
+
if(gc_interval && (Time.now.to_i - gc_last) > gc_interval)
|
78
|
+
Carnivore::Logger.debug 'Starting interval forced garbage collection from runner'
|
79
|
+
GC.start
|
80
|
+
gc_last = Time.now.to_i
|
81
|
+
end
|
82
|
+
end
|
83
|
+
Carnivore::Logger.error 'Carnivore supervisor has died!'
|
70
84
|
raise Carnivore::Error::DeadSupervisor.new
|
71
85
|
end
|
72
86
|
rescue Carnivore::Error::DeadSupervisor
|
73
|
-
|
87
|
+
Carnivore::Logger.warn "Received dead supervisor exception. Attempting to restart."
|
74
88
|
begin
|
75
|
-
supervisor.terminate
|
89
|
+
supervisor.terminate if supervisor.alive?
|
76
90
|
rescue => e
|
77
|
-
|
91
|
+
Carnivore::Logger.debug "Exception raised during supervisor termination (restart cleanup): #{e}"
|
78
92
|
end
|
79
|
-
|
93
|
+
Carnivore::Logger.debug "Pausing restart for 10 seconds to prevent restart thrashing cycles"
|
80
94
|
sleep 10
|
81
95
|
retry
|
82
96
|
rescue Exception => e
|
83
|
-
|
84
|
-
|
97
|
+
Carnivore::Logger.warn "Exception type encountered forcing shutdown - #{e.class}: #{e}"
|
98
|
+
Carnivore::Logger.debug "Shutdown exception info: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
85
99
|
supervisor.terminate if supervisor
|
86
100
|
# Gracefully shut down
|
87
101
|
end
|
data/lib/carnivore/source.rb
CHANGED
@@ -27,7 +27,7 @@ module Carnivore
|
|
27
27
|
require Source.require_path(args[:type]) || "carnivore/source/#{args[:type]}"
|
28
28
|
klass = args[:type].to_s.split('_').map(&:capitalize).join
|
29
29
|
klass = Source.const_get(klass)
|
30
|
-
args[:args][:name] ||=
|
30
|
+
args[:args][:name] ||= Carnivore.uuid
|
31
31
|
inst = SourceContainer.new(klass, args[:args])
|
32
32
|
register(args[:args][:name], inst)
|
33
33
|
inst
|
@@ -83,7 +83,7 @@ module Carnivore
|
|
83
83
|
if(sources_registry[name])
|
84
84
|
sources_registry[name]
|
85
85
|
else
|
86
|
-
|
86
|
+
Carnivore.logger.error "Source lookup failed (name: #{name})"
|
87
87
|
abort KeyError.new("Requested named source is not registered: #{name}")
|
88
88
|
end
|
89
89
|
end
|
@@ -113,13 +113,14 @@ module Carnivore
|
|
113
113
|
|
114
114
|
end
|
115
115
|
|
116
|
-
include
|
116
|
+
include Zoidberg::SoftShell
|
117
|
+
include Zoidberg::Supervise
|
117
118
|
include Utils::Logging
|
118
119
|
# @!parse include Carnivore::Utils::Logging
|
119
120
|
include Utils::Failure
|
120
121
|
# @!parse include Carnivore::Utils::Failure
|
121
122
|
|
122
|
-
finalizer :teardown_cleanup
|
123
|
+
# finalizer :teardown_cleanup
|
123
124
|
|
124
125
|
# @return [String, Symbol] name of source
|
125
126
|
attr_reader :name
|
@@ -185,22 +186,37 @@ module Carnivore
|
|
185
186
|
init_registry
|
186
187
|
end
|
187
188
|
@processing = false
|
188
|
-
@name = args[:name] ||
|
189
|
+
@name = args[:name] || Carnivore.uuid
|
189
190
|
if(args[:callbacks])
|
190
191
|
args[:callbacks].each do |name, block|
|
191
192
|
add_callback(name, block)
|
192
193
|
end
|
193
194
|
end
|
195
|
+
info 'Source initialization is complete'
|
196
|
+
rescue => e
|
197
|
+
debug "Failed to initialize: #{self} - #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
198
|
+
raise
|
199
|
+
end
|
200
|
+
|
201
|
+
# Fully restore the source if it is restarted
|
202
|
+
def restarted!
|
203
|
+
run_setup
|
204
|
+
run_connect
|
205
|
+
start!
|
206
|
+
end
|
207
|
+
|
208
|
+
# Run the source setup action
|
209
|
+
def run_setup
|
194
210
|
execute_and_retry_forever(:setup) do
|
195
211
|
setup(args)
|
196
212
|
end
|
213
|
+
end
|
214
|
+
|
215
|
+
# Run the source connect action
|
216
|
+
def run_connect
|
197
217
|
execute_and_retry_forever(:connect) do
|
198
218
|
connect
|
199
219
|
end
|
200
|
-
info 'Source initialization is complete'
|
201
|
-
rescue => e
|
202
|
-
debug "Failed to initialize: #{self} - #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
203
|
-
raise
|
204
220
|
end
|
205
221
|
|
206
222
|
# Start source if auto_process is enabled
|
@@ -223,13 +239,13 @@ module Carnivore
|
|
223
239
|
end
|
224
240
|
|
225
241
|
# Ensure we cleanup our internal supervisor before bailing out
|
226
|
-
def
|
242
|
+
def terminate
|
227
243
|
warn 'Termination request received. Tearing down!'
|
228
244
|
if(callback_supervisor && callback_supervisor.alive?)
|
229
245
|
begin
|
230
246
|
warn "Tearing down callback supervisor! (#{callback_supervisor})"
|
231
247
|
callback_supervisor.terminate
|
232
|
-
rescue
|
248
|
+
rescue Zoidberg::DeadException
|
233
249
|
warn 'Terminated task error during callback supervisor teardown. Moving on.'
|
234
250
|
end
|
235
251
|
else
|
@@ -1,8 +1,7 @@
|
|
1
1
|
require 'carnivore'
|
2
|
-
require 'celluloid'
|
3
2
|
require 'minitest/autorun'
|
4
3
|
|
5
|
-
|
4
|
+
Zoidberg.logger.level = ENV['DEBUG'] ? 0 : 4
|
6
5
|
|
7
6
|
if(File.directory?(dir = File.join(Dir.pwd, 'test', 'specs')))
|
8
7
|
Dir.glob(File.join(dir, '*.rb')).each do |path|
|
@@ -13,11 +12,6 @@ else
|
|
13
12
|
exit -1
|
14
13
|
end
|
15
14
|
|
16
|
-
MiniTest::Spec.before do
|
17
|
-
Celluloid.shutdown
|
18
|
-
Celluloid.boot
|
19
|
-
end
|
20
|
-
|
21
15
|
# Simple waiter method to stall testing
|
22
16
|
#
|
23
17
|
# @param name [String, Symbol] fetch wait time from environment variable
|
data/lib/carnivore/supervisor.rb
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
require 'carnivore'
|
2
|
-
require 'celluloid/supervision_group'
|
3
2
|
|
4
3
|
module Carnivore
|
5
|
-
class Supervisor <
|
4
|
+
class Supervisor < Zoidberg::Supervisor
|
6
5
|
|
7
6
|
class << self
|
8
7
|
|
@@ -18,8 +17,8 @@ module Carnivore
|
|
18
17
|
#
|
19
18
|
# @return [Array<[Celluloid::Registry, Carnivore::Supervisor]>]
|
20
19
|
def create!
|
21
|
-
|
22
|
-
[registry,
|
20
|
+
s = Zoidberg::Supervisor.new
|
21
|
+
[s.registry, s]
|
23
22
|
end
|
24
23
|
|
25
24
|
# Get/set the default supervisor
|
@@ -28,9 +27,9 @@ module Carnivore
|
|
28
27
|
# @return [Carnivore::Supervisor]
|
29
28
|
def supervisor(sup=nil)
|
30
29
|
if(sup)
|
31
|
-
|
30
|
+
@supervisor = sup
|
32
31
|
end
|
33
|
-
|
32
|
+
@supervisor
|
34
33
|
end
|
35
34
|
|
36
35
|
# Get the registry of the default supervisor
|
@@ -49,11 +48,10 @@ module Carnivore
|
|
49
48
|
if(supervisor)
|
50
49
|
begin
|
51
50
|
supervisor.terminate
|
52
|
-
rescue
|
53
|
-
|
51
|
+
rescue Zoidberg::DeadException => e
|
52
|
+
Carnivore::Logger.warn "Default supervisor is already in dead state (#{e.class}: #{e})"
|
54
53
|
end
|
55
54
|
@supervisor = nil
|
56
|
-
@registry = nil
|
57
55
|
end
|
58
56
|
true
|
59
57
|
end
|
@@ -66,17 +64,5 @@ module Carnivore
|
|
66
64
|
end
|
67
65
|
|
68
66
|
end
|
69
|
-
|
70
|
-
# @return [Celluloid::Registry]
|
71
|
-
attr_reader :registry
|
72
|
-
|
73
|
-
# Fetch actor from registry
|
74
|
-
#
|
75
|
-
# @param k [String, Symbol] identifier
|
76
|
-
# @return [Celluloid::Actor, NilClass]
|
77
|
-
def [](k)
|
78
|
-
registry[k]
|
79
|
-
end
|
80
|
-
|
81
67
|
end
|
82
68
|
end
|
@@ -17,6 +17,8 @@ module Carnivore
|
|
17
17
|
result = yield
|
18
18
|
debug "Completed #{action} process"
|
19
19
|
result
|
20
|
+
rescue Zoidberg::DeadException
|
21
|
+
raise
|
20
22
|
rescue => e
|
21
23
|
error "#{action.to_s.capitalize} process encountered an error: #{e.class} - #{e}"
|
22
24
|
debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
@@ -19,10 +19,10 @@ module Carnivore
|
|
19
19
|
# @return [NilClass]
|
20
20
|
def log(*args)
|
21
21
|
if(args.empty?)
|
22
|
-
|
22
|
+
Zoidberg.logger
|
23
23
|
else
|
24
24
|
severity, string = args
|
25
|
-
|
25
|
+
Zoidberg.logger.send(severity.to_sym, "#{self}: #{string}")
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
data/lib/carnivore/version.rb
CHANGED
metadata
CHANGED
@@ -1,31 +1,31 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: carnivore
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Roberts
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: bogo-config
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
@@ -39,7 +39,7 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: hashie
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
@@ -53,19 +53,19 @@ dependencies:
|
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: zoidberg
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - "<"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
61
|
+
version: '1.0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - "<"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '0'
|
68
|
+
version: '1.0'
|
69
69
|
description: Message processing helper
|
70
70
|
email: chrisroberts.code@gmail.com
|
71
71
|
executables: []
|
@@ -80,6 +80,7 @@ files:
|
|
80
80
|
- lib/carnivore/config.rb
|
81
81
|
- lib/carnivore/container.rb
|
82
82
|
- lib/carnivore/errors.rb
|
83
|
+
- lib/carnivore/logger.rb
|
83
84
|
- lib/carnivore/message.rb
|
84
85
|
- lib/carnivore/runner.rb
|
85
86
|
- lib/carnivore/source.rb
|