larva 0.6.4 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7fe5c86ee62d31f112c08c749a292a4c2bfa3923
4
- data.tar.gz: 69518a580563e06e7765356cdb037d0b34f36da6
3
+ metadata.gz: e2cc04eb147c0c63389af5c4e6504f8ec954bc0a
4
+ data.tar.gz: 7c76b2b988cab34763f1693328d900e903c51ffb
5
5
  SHA512:
6
- metadata.gz: 7428d4afcce9d19ee7b58b097f6697f4978cb81e04112c6368feea68b71b86b1b87f7cef246f9f8162a518905dc645017d28571b3082234e3974071a9562437e
7
- data.tar.gz: 8605c1e6c276e0e235b6cc400197c4b4a798149aac6f07fec16e59a4cf4c81667152a470415846fcc53a6903a93729bbc84bac416eed602a16714ad3727dabb5
6
+ metadata.gz: 40a291096f12284eaa82b3568ae9c5fb289dbf7911755be11df92e152e6c847806a926f81827ed921aeacc26d25fe07b605413d3e99300e553fdfb946c39d10c
7
+ data.tar.gz: f13153f42a05167c9b10235c5c7704a5f1b17d42267d8895a2bbf522b6a879fd1d9451c1f39b1c75cc99a47580585bfe109cedee8ce910285a0cd1af45c9a160
data/CHANGELOG.md CHANGED
@@ -1,3 +1,12 @@
1
+ # 0.7.0 / 2014-02-24
2
+ * [FEATURE] Add configurator
3
+
4
+ # 0.6.5 / 2014-02-23
5
+ * [FEATURE] Add extra logging to worker pool
6
+ * [FEATURE] Remove upstart script
7
+ * [FEATURE] Use IAM roles in spawned config
8
+ * [FEATURE] Test template config, not sample config.
9
+
1
10
  # 0.6.4 / 2014-02-23
2
11
  * [BUGFIX] Use camelize, not classify
3
12
 
data/lib/larva.rb CHANGED
@@ -3,6 +3,7 @@ require 'filum'
3
3
  require 'propono'
4
4
 
5
5
  require 'larva/configuration'
6
+ require 'larva/configurator'
6
7
  require 'larva/mocker'
7
8
  require 'larva/listener'
8
9
  require 'larva/processor'
@@ -0,0 +1,54 @@
1
+ module Larva
2
+ class Configurator
3
+
4
+ def self.configure(options = {})
5
+ c = new(options)
6
+ c.configure
7
+ c
8
+ end
9
+
10
+ def initialize(options = {})
11
+ @options = options
12
+ @config_dir = options.fetch(:config_dir) {raise LarvaError.new("Please provide :config_dir via options")}
13
+ @logfile = options.fetch(:logfile) {raise LarvaError.new("Please provide :logfile via options")}
14
+ @env = (options[:env] || "development").to_s
15
+ end
16
+
17
+ def configure
18
+ Filum.setup(@logfile)
19
+ Filum.logger.info "Configuring Daemon"
20
+
21
+ if meducation_sdk_config = parse_config_file('meducation-sdk.yml')
22
+ MeducationSDK.config do |config|
23
+ config.access_id = meducation_sdk_config[:access_id]
24
+ config.secret_key = @options.fetch(:meducation_sdk_secret_key, meducation_sdk_config[:secret_key])
25
+ config.endpoint = @options.fetch(:meducation_sdk_endpoint, "http://localhost:3000/system") if @env == 'development'
26
+ config.logger = Filum.logger
27
+ end
28
+ end
29
+
30
+ if propono_config = parse_config_file('propono.yml')
31
+ Propono.config do |config|
32
+ config.use_iam_profile = propono_config[:use_iam_profile]
33
+ config.access_key = propono_config[:access_key]
34
+ config.secret_key = propono_config[:secret_key]
35
+ config.queue_region = propono_config[:region]
36
+ config.application_name = propono_config[:application_name]
37
+ config.queue_suffix = propono_config[:queue_suffix]
38
+ config.udp_host = "pergo.meducation.net"
39
+ config.udp_port = "9732"
40
+ config.logger = Filum.logger
41
+ end
42
+ end
43
+ end
44
+
45
+ private
46
+ def parse_config_file(filename)
47
+ contents = File.read("#{@config_dir}/#{filename}")
48
+ hash = YAML::load(contents)
49
+ hash.stringify_keys[@env].symbolize_keys
50
+ rescue
51
+ nil
52
+ end
53
+ end
54
+ end
data/lib/larva/daemon.rb CHANGED
@@ -13,9 +13,6 @@ module Larva
13
13
  def initialize(processors, options = {})
14
14
  @processors = processors
15
15
  @options = options
16
- @config_dir = options.fetch(:config_dir) {raise LarvaError.new("Please provide :config_dir via options")}
17
- @logfile = options.fetch(:logfile) {raise LarvaError.new("Please provide :logfile via options")}
18
- @env = options[:env] || "development"
19
16
  end
20
17
 
21
18
  def start
@@ -25,38 +22,7 @@ module Larva
25
22
  end
26
23
 
27
24
  def configure
28
- Filum.setup(@logfile)
29
- Filum.logger.info "Configuring Daemon"
30
-
31
- if meducation_sdk_config = parse_config_file('meducation-sdk.yml')
32
- MeducationSDK.config do |config|
33
- config.access_id = meducation_sdk_config[:access_id]
34
- config.secret_key = @options[:meducation_sdk_secret_key] || meducation_sdk_config[:secret_key]
35
- config.logger = Filum.logger
36
- end
37
- end
38
-
39
- if propono_config = parse_config_file('propono.yml')
40
- Propono.config do |config|
41
- config.use_iam_profile = propono_config[:use_iam_profile]
42
- config.access_key = propono_config[:access_key]
43
- config.secret_key = propono_config[:secret_key]
44
- config.queue_region = propono_config[:region]
45
- config.application_name = propono_config[:application_name]
46
- config.queue_suffix = propono_config[:queue_suffix]
47
- config.udp_host = "pergo.meducation.net"
48
- config.udp_port = "9732"
49
- config.logger = Filum.logger
50
- end
51
- end
52
- end
53
-
54
- def parse_config_file(filename)
55
- contents = File.read("#{@config_dir}/#{filename}")
56
- hash = YAML::load(contents)
57
- hash.stringify_keys[@env].symbolize_keys
58
- rescue
59
- nil
25
+ Configurator.configure(@options)
60
26
  end
61
27
  end
62
28
  end
data/lib/larva/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Larva
2
- VERSION = '0.6.4'
2
+ VERSION = '0.7.0'
3
3
  end
@@ -16,11 +16,13 @@ module Larva
16
16
 
17
17
  private
18
18
  def start_workers
19
- logger.info "Starting threads."
19
+ logger.info "Starting #{processors.count} threads."
20
20
  @workers = processors.map do |topic, processor|
21
- Thread.new { start_worker(topic, processor) }
21
+ worker = Thread.new { start_worker(topic, processor) }
22
+ worker[:name] = "Listener for #{topic}"
23
+ worker
22
24
  end
23
- logger.info "Threads Started."
25
+ logger.info "#{workers.count} threads started."
24
26
  end
25
27
 
26
28
  def start_worker(topic, processor)
@@ -31,7 +33,10 @@ module Larva
31
33
 
32
34
  def keep_workers_alive
33
35
  sleep(5) while workers.all? { |t| t.alive? }
34
- logger.error "Some threads have died"
36
+ logger.error 'Some threads have died:'
37
+ workers.each do |worker|
38
+ logger.error "#{worker[:name]} was #{worker.alive? ? 'alive' : 'dead'}"
39
+ end
35
40
  end
36
41
 
37
42
  def logger
@@ -6,8 +6,7 @@ development:
6
6
  queue_suffix:
7
7
 
8
8
  production:
9
- access_key: MY-PRODUCTION-ACCESS-KEY
10
- secret_key: MY-PRODUCTION-SECRET-KEY
9
+ use_iam_profile: true
11
10
  region: eu-west-1
12
11
  application_name: larva_spawn
13
12
  queue_suffix:
@@ -14,4 +14,3 @@ module Larva
14
14
  end
15
15
  end
16
16
  end
17
-
@@ -0,0 +1,83 @@
1
+ require File.expand_path('../test_helper', __FILE__)
2
+
3
+ module Larva
4
+ class DaemonTest < Minitest::Test
5
+ def config_dir
6
+ File.expand_path('../../template/config', __FILE__)
7
+ end
8
+
9
+ def logfile
10
+ "log/test.log"
11
+ end
12
+
13
+ def test_logfile_is_compulsary
14
+ assert_raises(LarvaError, "Please provide :logfile via options") do
15
+ Configurator.configure(config_dir: config_dir)
16
+ end
17
+ end
18
+
19
+ def test_config_dir_is_compulsary
20
+ assert_raises(LarvaError, "Please provide :config_dir via options") do
21
+ Configurator.configure(logfile: logfile)
22
+ end
23
+ end
24
+
25
+ def test_filum_gets_config
26
+ Configurator.configure(logfile: logfile, config_dir: config_dir)
27
+ assert_equal logfile, Filum.logger.logfile
28
+ end
29
+
30
+ def test_meducation_sdk_gets_config_in_dev
31
+ sdk_yaml_path = File.expand_path('../../template/config/meducation-sdk.yml', __FILE__)
32
+ begin
33
+ optional_sdk_yaml_path = "#{sdk_yaml_path}.optional"
34
+ `cp #{optional_sdk_yaml_path} #{sdk_yaml_path}`
35
+ Configurator.configure(config_dir: config_dir, logfile: logfile)
36
+ assert_equal "LarvaSpawn", MeducationSDK.config.access_id
37
+ assert_equal "foobar", MeducationSDK.config.secret_key
38
+ assert_equal Filum.logger, MeducationSDK.config.logger
39
+ assert_equal "http://localhost:3000/system", MeducationSDK.config.endpoint
40
+ ensure
41
+ `rm #{sdk_yaml_path}`
42
+ end
43
+ end
44
+
45
+ def test_meducation_sdk_gets_config_in_production
46
+ sdk_yaml_path = File.expand_path('../../template/config/meducation-sdk.yml', __FILE__)
47
+ begin
48
+ optional_sdk_yaml_path = "#{sdk_yaml_path}.optional"
49
+ `cp #{optional_sdk_yaml_path} #{sdk_yaml_path}`
50
+ Configurator.configure(config_dir: config_dir, logfile: logfile, env: 'production')
51
+ assert_equal "LarvaSpawn", MeducationSDK.config.access_id
52
+ assert_equal nil, MeducationSDK.config.secret_key
53
+ assert_equal Filum.logger, MeducationSDK.config.logger
54
+ assert_equal nil, MeducationSDK.config.endpoint
55
+ ensure
56
+ `rm #{sdk_yaml_path}`
57
+ end
58
+ end
59
+
60
+ def test_propono_gets_config
61
+ Configurator.configure(config_dir: config_dir, logfile: logfile)
62
+ assert_equal "MY-DEV-ACCESS-KEY", Propono.config.access_key
63
+ assert_equal "MY-DEV-SECRET-KEY", Propono.config.secret_key
64
+ assert_equal "eu-west-1", Propono.config.queue_region
65
+ assert_equal "development_larva_spawn", Propono.config.application_name
66
+ assert_equal nil, Propono.config.queue_suffix
67
+ assert_equal "pergo.meducation.net", Propono.config.udp_host
68
+ assert_equal "9732", Propono.config.udp_port
69
+ assert_equal Filum.logger, Propono.config.logger
70
+ end
71
+
72
+ def test_propono_gets_config_with_env
73
+ Configurator.configure(config_dir: config_dir, logfile: logfile, env: 'production')
74
+ assert_equal true, Propono.config.use_iam_profile
75
+ assert_equal "larva_spawn", Propono.config.application_name
76
+ assert_equal nil, Propono.config.queue_suffix
77
+ assert_equal "pergo.meducation.net", Propono.config.udp_host
78
+ assert_equal "9732", Propono.config.udp_port
79
+ assert_equal Filum.logger, Propono.config.logger
80
+ end
81
+ end
82
+ end
83
+
data/test/daemon_test.rb CHANGED
@@ -2,73 +2,17 @@ require File.expand_path('../test_helper', __FILE__)
2
2
 
3
3
  module Larva
4
4
  class DaemonTest < Minitest::Test
5
- def config_dir
6
- File.expand_path('../sample_config', __FILE__)
7
- end
8
-
9
- def logfile
10
- "log/test.log"
11
- end
12
-
13
5
  def test_workerpool_is_started
14
6
  processors = {foo: 'bar'}
15
- Larva::WorkerPool.expects(:start).with(processors)
7
+ Configurator.stubs(:configure)
8
+ WorkerPool.expects(:start).with(processors)
16
9
  Daemon.start(processors, logfile: "./log/foo.log", config_dir: config_dir)
17
10
  end
18
11
 
19
- def test_logfile_is_compulsary
20
- assert_raises(LarvaError, "Please provide :logfile via options") do
21
- Daemon.start({}, config_dir: config_dir)
22
- end
23
- end
24
-
25
- def test_config_dir_is_compulsary
26
- assert_raises(LarvaError, "Please provide :config_dir via options") do
27
- Daemon.start({}, logfile: logfile)
28
- end
29
- end
30
-
31
- def test_filum_gets_config
32
- Daemon.start({}, logfile: logfile, config_dir: config_dir)
33
- assert_equal logfile, Filum.logger.logfile
34
- end
35
-
36
- def test_meducation_sdk_gets_config
37
- Daemon.start({}, config_dir: config_dir, logfile: logfile)
38
- assert_equal "Daemon", MeducationSDK.config.access_id
39
- assert_equal "foobar", MeducationSDK.config.secret_key
40
- assert_equal Filum.logger, MeducationSDK.config.logger
41
- end
42
-
43
- def test_meducation_sdk_gets_config_with_env
44
- Daemon.start({}, config_dir: config_dir, logfile: logfile, env: 'production')
45
- assert_equal "Daemon", MeducationSDK.config.access_id
46
- assert_equal nil, MeducationSDK.config.secret_key
47
- assert_equal Filum.logger, MeducationSDK.config.logger
48
- end
49
-
50
- def test_propono_gets_config
51
- Daemon.start({}, config_dir: config_dir, logfile: logfile)
52
- assert_equal "BADASSDEVKEY", Propono.config.access_key
53
- assert_equal "SCARYDEVSECRET", Propono.config.secret_key
54
- assert_equal "eu-west-1", Propono.config.queue_region
55
- assert_equal "development_daemon", Propono.config.application_name
56
- assert_equal "-dev", Propono.config.queue_suffix
57
- assert_equal "pergo.meducation.net", Propono.config.udp_host
58
- assert_equal "9732", Propono.config.udp_port
59
- assert_equal Filum.logger, Propono.config.logger
60
- end
61
-
62
- def test_propono_gets_config_with_env
63
- Daemon.start({}, config_dir: config_dir, logfile: logfile, env: 'production')
64
- assert_equal "BADASSPRODUCTIONKEY", Propono.config.access_key
65
- assert_equal "SCARYPRODUCTIONSECRET", Propono.config.secret_key
66
- assert_equal "eu-west-1", Propono.config.queue_region
67
- assert_equal "daemon", Propono.config.application_name
68
- assert_equal nil, Propono.config.queue_suffix
69
- assert_equal "pergo.meducation.net", Propono.config.udp_host
70
- assert_equal "9732", Propono.config.udp_port
71
- assert_equal Filum.logger, Propono.config.logger
12
+ def test_configurator_is_called
13
+ options = {logfile: "./log/foo.log", config_dir: config_dir}
14
+ Configurator.expects(:configure).with(options)
15
+ Daemon.start({}, options)
72
16
  end
73
17
  end
74
18
  end
@@ -6,15 +6,10 @@ module Larva
6
6
  WorkerPool.start({})
7
7
  end
8
8
 
9
- def test_process_logs_start_message
9
+ def test_process_logs_start_and_end_messages
10
10
  Propono.config.logger.stubs(:info)
11
- Propono.config.logger.expects(:info).with("Starting threads.")
12
- WorkerPool.start({})
13
- end
14
-
15
- def test_process_logs_end_message
16
- Propono.config.logger.stubs(:info)
17
- Propono.config.logger.expects(:info).with("Threads Started.")
11
+ Propono.config.logger.expects(:info).with("Starting 0 threads.")
12
+ Propono.config.logger.expects(:info).with("0 threads started.")
18
13
  WorkerPool.start({})
19
14
  end
20
15
 
@@ -23,8 +18,9 @@ module Larva
23
18
  Propono.config.logger.expects(:error).with do |error|
24
19
  error.start_with?("Unexpected listener termination:")
25
20
  end
26
- Propono.config.logger.expects(:error).with('Some threads have died')
27
- WorkerPool.start({nil => nil})
21
+ Propono.config.logger.expects(:error).with('Listener for qux was dead')
22
+ Propono.config.logger.expects(:error).with('Some threads have died:')
23
+ WorkerPool.start({'qux' => nil})
28
24
  end
29
25
 
30
26
  def test_listen_is_called_correctly
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: larva
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.4
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - iHiD
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-02-23 00:00:00.000000000 Z
11
+ date: 2014-02-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: propono
@@ -128,6 +128,7 @@ files:
128
128
  - larva.gemspec
129
129
  - lib/larva.rb
130
130
  - lib/larva/configuration.rb
131
+ - lib/larva/configurator.rb
131
132
  - lib/larva/daemon.rb
132
133
  - lib/larva/listener.rb
133
134
  - lib/larva/mocker.rb
@@ -151,15 +152,13 @@ files:
151
152
  - template/test/daemon_test.rb
152
153
  - template/test/processors/media_file_processor_test.rb
153
154
  - template/test/test_helper.rb
154
- - template/upstart/larva_spawn.conf
155
155
  - test/configuration_test.rb
156
+ - test/configurator_test.rb
156
157
  - test/daemon_test.rb
157
158
  - test/larva_test.rb
158
159
  - test/listener_test.rb
159
160
  - test/mocker_test.rb
160
161
  - test/processor_test.rb
161
- - test/sample_config/meducation-sdk.yml
162
- - test/sample_config/propono.yml
163
162
  - test/test_helper.rb
164
163
  - test/worker_pool_test.rb
165
164
  homepage: https://github.com/meducation/larva
@@ -188,13 +187,12 @@ specification_version: 4
188
187
  summary: Some Meducation specific helper files for ur pub/sub network
189
188
  test_files:
190
189
  - test/configuration_test.rb
190
+ - test/configurator_test.rb
191
191
  - test/daemon_test.rb
192
192
  - test/larva_test.rb
193
193
  - test/listener_test.rb
194
194
  - test/mocker_test.rb
195
195
  - test/processor_test.rb
196
- - test/sample_config/meducation-sdk.yml
197
- - test/sample_config/propono.yml
198
196
  - test/test_helper.rb
199
197
  - test/worker_pool_test.rb
200
198
  has_rdoc:
@@ -1,24 +0,0 @@
1
- description "LarvaSpawn Upstart Script"
2
- author "Charles Care <charles@meducation.net>"
3
-
4
- # Triggers
5
- start on startup
6
- start on runlevel [2345]
7
- stop on runlevel [016]
8
-
9
- # Restart if the service terminates abnormally
10
- respawn
11
-
12
- # Give up if the service requires restarting more than 10 times in a 30 second period.
13
- respawn limit 10 30
14
-
15
- # Run as a user
16
- setuid larva_spawn
17
-
18
- # cwd
19
- chdir /opt/larva_spawn
20
-
21
- #pre-start exec su -c /srv/apps/larva_spawn/bin/larva_spawn-setup ec2-user
22
-
23
- # Run target command
24
- exec /opt/larva_spawn/bin/larva_spawn
@@ -1,6 +0,0 @@
1
- development:
2
- access_id: Daemon
3
- secret_key: 'foobar'
4
-
5
- production:
6
- access_id: Daemon
@@ -1,13 +0,0 @@
1
- development:
2
- access_key: BADASSDEVKEY
3
- secret_key: SCARYDEVSECRET
4
- region: eu-west-1
5
- application_name: development_daemon
6
- queue_suffix: -dev
7
-
8
- production:
9
- access_key: BADASSPRODUCTIONKEY
10
- secret_key: SCARYPRODUCTIONSECRET
11
- region: eu-west-1
12
- application_name: daemon
13
- queue_suffix: