larva 0.6.4 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/lib/larva.rb +1 -0
- data/lib/larva/configurator.rb +54 -0
- data/lib/larva/daemon.rb +1 -35
- data/lib/larva/version.rb +1 -1
- data/lib/larva/worker_pool.rb +9 -4
- data/template/config/propono.yml +1 -2
- data/test/configuration_test.rb +0 -1
- data/test/configurator_test.rb +83 -0
- data/test/daemon_test.rb +6 -62
- data/test/worker_pool_test.rb +6 -10
- metadata +5 -7
- data/template/upstart/larva_spawn.conf +0 -24
- data/test/sample_config/meducation-sdk.yml +0 -6
- data/test/sample_config/propono.yml +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2cc04eb147c0c63389af5c4e6504f8ec954bc0a
|
4
|
+
data.tar.gz: 7c76b2b988cab34763f1693328d900e903c51ffb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 40a291096f12284eaa82b3568ae9c5fb289dbf7911755be11df92e152e6c847806a926f81827ed921aeacc26d25fe07b605413d3e99300e553fdfb946c39d10c
|
7
|
+
data.tar.gz: f13153f42a05167c9b10235c5c7704a5f1b17d42267d8895a2bbf522b6a879fd1d9451c1f39b1c75cc99a47580585bfe109cedee8ce910285a0cd1af45c9a160
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
# 0.7.0 / 2014-02-24
|
2
|
+
* [FEATURE] Add configurator
|
3
|
+
|
4
|
+
# 0.6.5 / 2014-02-23
|
5
|
+
* [FEATURE] Add extra logging to worker pool
|
6
|
+
* [FEATURE] Remove upstart script
|
7
|
+
* [FEATURE] Use IAM roles in spawned config
|
8
|
+
* [FEATURE] Test template config, not sample config.
|
9
|
+
|
1
10
|
# 0.6.4 / 2014-02-23
|
2
11
|
* [BUGFIX] Use camelize, not classify
|
3
12
|
|
data/lib/larva.rb
CHANGED
@@ -0,0 +1,54 @@
|
|
1
|
+
module Larva
|
2
|
+
class Configurator
|
3
|
+
|
4
|
+
def self.configure(options = {})
|
5
|
+
c = new(options)
|
6
|
+
c.configure
|
7
|
+
c
|
8
|
+
end
|
9
|
+
|
10
|
+
def initialize(options = {})
|
11
|
+
@options = options
|
12
|
+
@config_dir = options.fetch(:config_dir) {raise LarvaError.new("Please provide :config_dir via options")}
|
13
|
+
@logfile = options.fetch(:logfile) {raise LarvaError.new("Please provide :logfile via options")}
|
14
|
+
@env = (options[:env] || "development").to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
def configure
|
18
|
+
Filum.setup(@logfile)
|
19
|
+
Filum.logger.info "Configuring Daemon"
|
20
|
+
|
21
|
+
if meducation_sdk_config = parse_config_file('meducation-sdk.yml')
|
22
|
+
MeducationSDK.config do |config|
|
23
|
+
config.access_id = meducation_sdk_config[:access_id]
|
24
|
+
config.secret_key = @options.fetch(:meducation_sdk_secret_key, meducation_sdk_config[:secret_key])
|
25
|
+
config.endpoint = @options.fetch(:meducation_sdk_endpoint, "http://localhost:3000/system") if @env == 'development'
|
26
|
+
config.logger = Filum.logger
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
if propono_config = parse_config_file('propono.yml')
|
31
|
+
Propono.config do |config|
|
32
|
+
config.use_iam_profile = propono_config[:use_iam_profile]
|
33
|
+
config.access_key = propono_config[:access_key]
|
34
|
+
config.secret_key = propono_config[:secret_key]
|
35
|
+
config.queue_region = propono_config[:region]
|
36
|
+
config.application_name = propono_config[:application_name]
|
37
|
+
config.queue_suffix = propono_config[:queue_suffix]
|
38
|
+
config.udp_host = "pergo.meducation.net"
|
39
|
+
config.udp_port = "9732"
|
40
|
+
config.logger = Filum.logger
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
def parse_config_file(filename)
|
47
|
+
contents = File.read("#{@config_dir}/#{filename}")
|
48
|
+
hash = YAML::load(contents)
|
49
|
+
hash.stringify_keys[@env].symbolize_keys
|
50
|
+
rescue
|
51
|
+
nil
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/larva/daemon.rb
CHANGED
@@ -13,9 +13,6 @@ module Larva
|
|
13
13
|
def initialize(processors, options = {})
|
14
14
|
@processors = processors
|
15
15
|
@options = options
|
16
|
-
@config_dir = options.fetch(:config_dir) {raise LarvaError.new("Please provide :config_dir via options")}
|
17
|
-
@logfile = options.fetch(:logfile) {raise LarvaError.new("Please provide :logfile via options")}
|
18
|
-
@env = options[:env] || "development"
|
19
16
|
end
|
20
17
|
|
21
18
|
def start
|
@@ -25,38 +22,7 @@ module Larva
|
|
25
22
|
end
|
26
23
|
|
27
24
|
def configure
|
28
|
-
|
29
|
-
Filum.logger.info "Configuring Daemon"
|
30
|
-
|
31
|
-
if meducation_sdk_config = parse_config_file('meducation-sdk.yml')
|
32
|
-
MeducationSDK.config do |config|
|
33
|
-
config.access_id = meducation_sdk_config[:access_id]
|
34
|
-
config.secret_key = @options[:meducation_sdk_secret_key] || meducation_sdk_config[:secret_key]
|
35
|
-
config.logger = Filum.logger
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
if propono_config = parse_config_file('propono.yml')
|
40
|
-
Propono.config do |config|
|
41
|
-
config.use_iam_profile = propono_config[:use_iam_profile]
|
42
|
-
config.access_key = propono_config[:access_key]
|
43
|
-
config.secret_key = propono_config[:secret_key]
|
44
|
-
config.queue_region = propono_config[:region]
|
45
|
-
config.application_name = propono_config[:application_name]
|
46
|
-
config.queue_suffix = propono_config[:queue_suffix]
|
47
|
-
config.udp_host = "pergo.meducation.net"
|
48
|
-
config.udp_port = "9732"
|
49
|
-
config.logger = Filum.logger
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def parse_config_file(filename)
|
55
|
-
contents = File.read("#{@config_dir}/#{filename}")
|
56
|
-
hash = YAML::load(contents)
|
57
|
-
hash.stringify_keys[@env].symbolize_keys
|
58
|
-
rescue
|
59
|
-
nil
|
25
|
+
Configurator.configure(@options)
|
60
26
|
end
|
61
27
|
end
|
62
28
|
end
|
data/lib/larva/version.rb
CHANGED
data/lib/larva/worker_pool.rb
CHANGED
@@ -16,11 +16,13 @@ module Larva
|
|
16
16
|
|
17
17
|
private
|
18
18
|
def start_workers
|
19
|
-
logger.info "Starting threads."
|
19
|
+
logger.info "Starting #{processors.count} threads."
|
20
20
|
@workers = processors.map do |topic, processor|
|
21
|
-
Thread.new { start_worker(topic, processor) }
|
21
|
+
worker = Thread.new { start_worker(topic, processor) }
|
22
|
+
worker[:name] = "Listener for #{topic}"
|
23
|
+
worker
|
22
24
|
end
|
23
|
-
logger.info "
|
25
|
+
logger.info "#{workers.count} threads started."
|
24
26
|
end
|
25
27
|
|
26
28
|
def start_worker(topic, processor)
|
@@ -31,7 +33,10 @@ module Larva
|
|
31
33
|
|
32
34
|
def keep_workers_alive
|
33
35
|
sleep(5) while workers.all? { |t| t.alive? }
|
34
|
-
logger.error
|
36
|
+
logger.error 'Some threads have died:'
|
37
|
+
workers.each do |worker|
|
38
|
+
logger.error "#{worker[:name]} was #{worker.alive? ? 'alive' : 'dead'}"
|
39
|
+
end
|
35
40
|
end
|
36
41
|
|
37
42
|
def logger
|
data/template/config/propono.yml
CHANGED
data/test/configuration_test.rb
CHANGED
@@ -0,0 +1,83 @@
|
|
1
|
+
require File.expand_path('../test_helper', __FILE__)
|
2
|
+
|
3
|
+
module Larva
|
4
|
+
class DaemonTest < Minitest::Test
|
5
|
+
def config_dir
|
6
|
+
File.expand_path('../../template/config', __FILE__)
|
7
|
+
end
|
8
|
+
|
9
|
+
def logfile
|
10
|
+
"log/test.log"
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_logfile_is_compulsary
|
14
|
+
assert_raises(LarvaError, "Please provide :logfile via options") do
|
15
|
+
Configurator.configure(config_dir: config_dir)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_config_dir_is_compulsary
|
20
|
+
assert_raises(LarvaError, "Please provide :config_dir via options") do
|
21
|
+
Configurator.configure(logfile: logfile)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_filum_gets_config
|
26
|
+
Configurator.configure(logfile: logfile, config_dir: config_dir)
|
27
|
+
assert_equal logfile, Filum.logger.logfile
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_meducation_sdk_gets_config_in_dev
|
31
|
+
sdk_yaml_path = File.expand_path('../../template/config/meducation-sdk.yml', __FILE__)
|
32
|
+
begin
|
33
|
+
optional_sdk_yaml_path = "#{sdk_yaml_path}.optional"
|
34
|
+
`cp #{optional_sdk_yaml_path} #{sdk_yaml_path}`
|
35
|
+
Configurator.configure(config_dir: config_dir, logfile: logfile)
|
36
|
+
assert_equal "LarvaSpawn", MeducationSDK.config.access_id
|
37
|
+
assert_equal "foobar", MeducationSDK.config.secret_key
|
38
|
+
assert_equal Filum.logger, MeducationSDK.config.logger
|
39
|
+
assert_equal "http://localhost:3000/system", MeducationSDK.config.endpoint
|
40
|
+
ensure
|
41
|
+
`rm #{sdk_yaml_path}`
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_meducation_sdk_gets_config_in_production
|
46
|
+
sdk_yaml_path = File.expand_path('../../template/config/meducation-sdk.yml', __FILE__)
|
47
|
+
begin
|
48
|
+
optional_sdk_yaml_path = "#{sdk_yaml_path}.optional"
|
49
|
+
`cp #{optional_sdk_yaml_path} #{sdk_yaml_path}`
|
50
|
+
Configurator.configure(config_dir: config_dir, logfile: logfile, env: 'production')
|
51
|
+
assert_equal "LarvaSpawn", MeducationSDK.config.access_id
|
52
|
+
assert_equal nil, MeducationSDK.config.secret_key
|
53
|
+
assert_equal Filum.logger, MeducationSDK.config.logger
|
54
|
+
assert_equal nil, MeducationSDK.config.endpoint
|
55
|
+
ensure
|
56
|
+
`rm #{sdk_yaml_path}`
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_propono_gets_config
|
61
|
+
Configurator.configure(config_dir: config_dir, logfile: logfile)
|
62
|
+
assert_equal "MY-DEV-ACCESS-KEY", Propono.config.access_key
|
63
|
+
assert_equal "MY-DEV-SECRET-KEY", Propono.config.secret_key
|
64
|
+
assert_equal "eu-west-1", Propono.config.queue_region
|
65
|
+
assert_equal "development_larva_spawn", Propono.config.application_name
|
66
|
+
assert_equal nil, Propono.config.queue_suffix
|
67
|
+
assert_equal "pergo.meducation.net", Propono.config.udp_host
|
68
|
+
assert_equal "9732", Propono.config.udp_port
|
69
|
+
assert_equal Filum.logger, Propono.config.logger
|
70
|
+
end
|
71
|
+
|
72
|
+
def test_propono_gets_config_with_env
|
73
|
+
Configurator.configure(config_dir: config_dir, logfile: logfile, env: 'production')
|
74
|
+
assert_equal true, Propono.config.use_iam_profile
|
75
|
+
assert_equal "larva_spawn", Propono.config.application_name
|
76
|
+
assert_equal nil, Propono.config.queue_suffix
|
77
|
+
assert_equal "pergo.meducation.net", Propono.config.udp_host
|
78
|
+
assert_equal "9732", Propono.config.udp_port
|
79
|
+
assert_equal Filum.logger, Propono.config.logger
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
data/test/daemon_test.rb
CHANGED
@@ -2,73 +2,17 @@ require File.expand_path('../test_helper', __FILE__)
|
|
2
2
|
|
3
3
|
module Larva
|
4
4
|
class DaemonTest < Minitest::Test
|
5
|
-
def config_dir
|
6
|
-
File.expand_path('../sample_config', __FILE__)
|
7
|
-
end
|
8
|
-
|
9
|
-
def logfile
|
10
|
-
"log/test.log"
|
11
|
-
end
|
12
|
-
|
13
5
|
def test_workerpool_is_started
|
14
6
|
processors = {foo: 'bar'}
|
15
|
-
|
7
|
+
Configurator.stubs(:configure)
|
8
|
+
WorkerPool.expects(:start).with(processors)
|
16
9
|
Daemon.start(processors, logfile: "./log/foo.log", config_dir: config_dir)
|
17
10
|
end
|
18
11
|
|
19
|
-
def
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end
|
24
|
-
|
25
|
-
def test_config_dir_is_compulsary
|
26
|
-
assert_raises(LarvaError, "Please provide :config_dir via options") do
|
27
|
-
Daemon.start({}, logfile: logfile)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def test_filum_gets_config
|
32
|
-
Daemon.start({}, logfile: logfile, config_dir: config_dir)
|
33
|
-
assert_equal logfile, Filum.logger.logfile
|
34
|
-
end
|
35
|
-
|
36
|
-
def test_meducation_sdk_gets_config
|
37
|
-
Daemon.start({}, config_dir: config_dir, logfile: logfile)
|
38
|
-
assert_equal "Daemon", MeducationSDK.config.access_id
|
39
|
-
assert_equal "foobar", MeducationSDK.config.secret_key
|
40
|
-
assert_equal Filum.logger, MeducationSDK.config.logger
|
41
|
-
end
|
42
|
-
|
43
|
-
def test_meducation_sdk_gets_config_with_env
|
44
|
-
Daemon.start({}, config_dir: config_dir, logfile: logfile, env: 'production')
|
45
|
-
assert_equal "Daemon", MeducationSDK.config.access_id
|
46
|
-
assert_equal nil, MeducationSDK.config.secret_key
|
47
|
-
assert_equal Filum.logger, MeducationSDK.config.logger
|
48
|
-
end
|
49
|
-
|
50
|
-
def test_propono_gets_config
|
51
|
-
Daemon.start({}, config_dir: config_dir, logfile: logfile)
|
52
|
-
assert_equal "BADASSDEVKEY", Propono.config.access_key
|
53
|
-
assert_equal "SCARYDEVSECRET", Propono.config.secret_key
|
54
|
-
assert_equal "eu-west-1", Propono.config.queue_region
|
55
|
-
assert_equal "development_daemon", Propono.config.application_name
|
56
|
-
assert_equal "-dev", Propono.config.queue_suffix
|
57
|
-
assert_equal "pergo.meducation.net", Propono.config.udp_host
|
58
|
-
assert_equal "9732", Propono.config.udp_port
|
59
|
-
assert_equal Filum.logger, Propono.config.logger
|
60
|
-
end
|
61
|
-
|
62
|
-
def test_propono_gets_config_with_env
|
63
|
-
Daemon.start({}, config_dir: config_dir, logfile: logfile, env: 'production')
|
64
|
-
assert_equal "BADASSPRODUCTIONKEY", Propono.config.access_key
|
65
|
-
assert_equal "SCARYPRODUCTIONSECRET", Propono.config.secret_key
|
66
|
-
assert_equal "eu-west-1", Propono.config.queue_region
|
67
|
-
assert_equal "daemon", Propono.config.application_name
|
68
|
-
assert_equal nil, Propono.config.queue_suffix
|
69
|
-
assert_equal "pergo.meducation.net", Propono.config.udp_host
|
70
|
-
assert_equal "9732", Propono.config.udp_port
|
71
|
-
assert_equal Filum.logger, Propono.config.logger
|
12
|
+
def test_configurator_is_called
|
13
|
+
options = {logfile: "./log/foo.log", config_dir: config_dir}
|
14
|
+
Configurator.expects(:configure).with(options)
|
15
|
+
Daemon.start({}, options)
|
72
16
|
end
|
73
17
|
end
|
74
18
|
end
|
data/test/worker_pool_test.rb
CHANGED
@@ -6,15 +6,10 @@ module Larva
|
|
6
6
|
WorkerPool.start({})
|
7
7
|
end
|
8
8
|
|
9
|
-
def
|
9
|
+
def test_process_logs_start_and_end_messages
|
10
10
|
Propono.config.logger.stubs(:info)
|
11
|
-
Propono.config.logger.expects(:info).with("Starting threads.")
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_process_logs_end_message
|
16
|
-
Propono.config.logger.stubs(:info)
|
17
|
-
Propono.config.logger.expects(:info).with("Threads Started.")
|
11
|
+
Propono.config.logger.expects(:info).with("Starting 0 threads.")
|
12
|
+
Propono.config.logger.expects(:info).with("0 threads started.")
|
18
13
|
WorkerPool.start({})
|
19
14
|
end
|
20
15
|
|
@@ -23,8 +18,9 @@ module Larva
|
|
23
18
|
Propono.config.logger.expects(:error).with do |error|
|
24
19
|
error.start_with?("Unexpected listener termination:")
|
25
20
|
end
|
26
|
-
Propono.config.logger.expects(:error).with('
|
27
|
-
|
21
|
+
Propono.config.logger.expects(:error).with('Listener for qux was dead')
|
22
|
+
Propono.config.logger.expects(:error).with('Some threads have died:')
|
23
|
+
WorkerPool.start({'qux' => nil})
|
28
24
|
end
|
29
25
|
|
30
26
|
def test_listen_is_called_correctly
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: larva
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- iHiD
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-02-
|
11
|
+
date: 2014-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: propono
|
@@ -128,6 +128,7 @@ files:
|
|
128
128
|
- larva.gemspec
|
129
129
|
- lib/larva.rb
|
130
130
|
- lib/larva/configuration.rb
|
131
|
+
- lib/larva/configurator.rb
|
131
132
|
- lib/larva/daemon.rb
|
132
133
|
- lib/larva/listener.rb
|
133
134
|
- lib/larva/mocker.rb
|
@@ -151,15 +152,13 @@ files:
|
|
151
152
|
- template/test/daemon_test.rb
|
152
153
|
- template/test/processors/media_file_processor_test.rb
|
153
154
|
- template/test/test_helper.rb
|
154
|
-
- template/upstart/larva_spawn.conf
|
155
155
|
- test/configuration_test.rb
|
156
|
+
- test/configurator_test.rb
|
156
157
|
- test/daemon_test.rb
|
157
158
|
- test/larva_test.rb
|
158
159
|
- test/listener_test.rb
|
159
160
|
- test/mocker_test.rb
|
160
161
|
- test/processor_test.rb
|
161
|
-
- test/sample_config/meducation-sdk.yml
|
162
|
-
- test/sample_config/propono.yml
|
163
162
|
- test/test_helper.rb
|
164
163
|
- test/worker_pool_test.rb
|
165
164
|
homepage: https://github.com/meducation/larva
|
@@ -188,13 +187,12 @@ specification_version: 4
|
|
188
187
|
summary: Some Meducation specific helper files for ur pub/sub network
|
189
188
|
test_files:
|
190
189
|
- test/configuration_test.rb
|
190
|
+
- test/configurator_test.rb
|
191
191
|
- test/daemon_test.rb
|
192
192
|
- test/larva_test.rb
|
193
193
|
- test/listener_test.rb
|
194
194
|
- test/mocker_test.rb
|
195
195
|
- test/processor_test.rb
|
196
|
-
- test/sample_config/meducation-sdk.yml
|
197
|
-
- test/sample_config/propono.yml
|
198
196
|
- test/test_helper.rb
|
199
197
|
- test/worker_pool_test.rb
|
200
198
|
has_rdoc:
|
@@ -1,24 +0,0 @@
|
|
1
|
-
description "LarvaSpawn Upstart Script"
|
2
|
-
author "Charles Care <charles@meducation.net>"
|
3
|
-
|
4
|
-
# Triggers
|
5
|
-
start on startup
|
6
|
-
start on runlevel [2345]
|
7
|
-
stop on runlevel [016]
|
8
|
-
|
9
|
-
# Restart if the service terminates abnormally
|
10
|
-
respawn
|
11
|
-
|
12
|
-
# Give up if the service requires restarting more than 10 times in a 30 second period.
|
13
|
-
respawn limit 10 30
|
14
|
-
|
15
|
-
# Run as a user
|
16
|
-
setuid larva_spawn
|
17
|
-
|
18
|
-
# cwd
|
19
|
-
chdir /opt/larva_spawn
|
20
|
-
|
21
|
-
#pre-start exec su -c /srv/apps/larva_spawn/bin/larva_spawn-setup ec2-user
|
22
|
-
|
23
|
-
# Run target command
|
24
|
-
exec /opt/larva_spawn/bin/larva_spawn
|
@@ -1,13 +0,0 @@
|
|
1
|
-
development:
|
2
|
-
access_key: BADASSDEVKEY
|
3
|
-
secret_key: SCARYDEVSECRET
|
4
|
-
region: eu-west-1
|
5
|
-
application_name: development_daemon
|
6
|
-
queue_suffix: -dev
|
7
|
-
|
8
|
-
production:
|
9
|
-
access_key: BADASSPRODUCTIONKEY
|
10
|
-
secret_key: SCARYPRODUCTIONSECRET
|
11
|
-
region: eu-west-1
|
12
|
-
application_name: daemon
|
13
|
-
queue_suffix:
|