larva 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7fe5c86ee62d31f112c08c749a292a4c2bfa3923
-  data.tar.gz: 69518a580563e06e7765356cdb037d0b34f36da6
+  metadata.gz: e2cc04eb147c0c63389af5c4e6504f8ec954bc0a
+  data.tar.gz: 7c76b2b988cab34763f1693328d900e903c51ffb
 SHA512:
-  metadata.gz: 7428d4afcce9d19ee7b58b097f6697f4978cb81e04112c6368feea68b71b86b1b87f7cef246f9f8162a518905dc645017d28571b3082234e3974071a9562437e
-  data.tar.gz: 8605c1e6c276e0e235b6cc400197c4b4a798149aac6f07fec16e59a4cf4c81667152a470415846fcc53a6903a93729bbc84bac416eed602a16714ad3727dabb5
+  metadata.gz: 40a291096f12284eaa82b3568ae9c5fb289dbf7911755be11df92e152e6c847806a926f81827ed921aeacc26d25fe07b605413d3e99300e553fdfb946c39d10c
+  data.tar.gz: f13153f42a05167c9b10235c5c7704a5f1b17d42267d8895a2bbf522b6a879fd1d9451c1f39b1c75cc99a47580585bfe109cedee8ce910285a0cd1af45c9a160

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 0.7.0 / 2014-02-24
+* [FEATURE] Add configurator
+
+# 0.6.5 / 2014-02-23
+* [FEATURE] Add extra logging to worker pool
+* [FEATURE] Remove upstart script
+* [FEATURE] Use IAM roles in spawned config
+* [FEATURE] Test template config, not sample config.
+
 # 0.6.4 / 2014-02-23
 * [BUGFIX] Use camelize, not classify
 

data/lib/larva.rb

@@ -3,6 +3,7 @@ require 'filum'
 require 'propono'
 
 require 'larva/configuration'
+require 'larva/configurator'
 require 'larva/mocker'
 require 'larva/listener'
 require 'larva/processor'

data/lib/larva/configurator.rb

@@ -0,0 +1,54 @@
+module Larva
+  class Configurator
+
+    def self.configure(options = {})
+      c = new(options)
+      c.configure
+      c
+    end
+
+    def initialize(options = {})
+      @options = options
+      @config_dir = options.fetch(:config_dir) {raise LarvaError.new("Please provide :config_dir via options")}
+      @logfile = options.fetch(:logfile) {raise LarvaError.new("Please provide :logfile via options")}
+      @env = (options[:env] || "development").to_s
+    end
+
+    def configure
+      Filum.setup(@logfile)
+      Filum.logger.info "Configuring Daemon"
+
+      if meducation_sdk_config = parse_config_file('meducation-sdk.yml')
+        MeducationSDK.config do |config|
+          config.access_id  = meducation_sdk_config[:access_id]
+          config.secret_key = @options.fetch(:meducation_sdk_secret_key,  meducation_sdk_config[:secret_key])
+          config.endpoint   = @options.fetch(:meducation_sdk_endpoint, "http://localhost:3000/system") if @env == 'development'
+          config.logger     = Filum.logger
+        end
+      end
+
+      if propono_config = parse_config_file('propono.yml')
+        Propono.config do |config|
+          config.use_iam_profile  = propono_config[:use_iam_profile]
+          config.access_key       = propono_config[:access_key]
+          config.secret_key       = propono_config[:secret_key]
+          config.queue_region     = propono_config[:region]
+          config.application_name = propono_config[:application_name]
+          config.queue_suffix     = propono_config[:queue_suffix]
+          config.udp_host         = "pergo.meducation.net"
+          config.udp_port         = "9732"
+          config.logger           = Filum.logger
+        end
+      end
+    end
+
+    private
+    def parse_config_file(filename)
+      contents = File.read("#{@config_dir}/#{filename}")
+      hash = YAML::load(contents)
+      hash.stringify_keys[@env].symbolize_keys
+    rescue
+      nil
+    end
+  end
+end

data/lib/larva/daemon.rb

@@ -13,9 +13,6 @@ module Larva
     def initialize(processors, options = {})
       @processors = processors
       @options = options
-      @config_dir = options.fetch(:config_dir) {raise LarvaError.new("Please provide :config_dir via options")}
-      @logfile = options.fetch(:logfile) {raise LarvaError.new("Please provide :logfile via options")}
-      @env = options[:env] || "development"
     end
 
     def start
@@ -25,38 +22,7 @@ module Larva
     end
 
     def configure
-      Filum.setup(@logfile)
-      Filum.logger.info "Configuring Daemon"
-
-      if meducation_sdk_config = parse_config_file('meducation-sdk.yml')
-        MeducationSDK.config do |config|
-          config.access_id  = meducation_sdk_config[:access_id]
-          config.secret_key = @options[:meducation_sdk_secret_key] || meducation_sdk_config[:secret_key]
-          config.logger     = Filum.logger
-        end
-      end
-
-      if propono_config = parse_config_file('propono.yml')
-        Propono.config do |config|
-          config.use_iam_profile  = propono_config[:use_iam_profile]
-          config.access_key       = propono_config[:access_key]
-          config.secret_key       = propono_config[:secret_key]
-          config.queue_region     = propono_config[:region]
-          config.application_name = propono_config[:application_name]
-          config.queue_suffix     = propono_config[:queue_suffix]
-          config.udp_host         = "pergo.meducation.net"
-          config.udp_port         = "9732"
-          config.logger           = Filum.logger
-        end
-      end
-    end
-
-    def parse_config_file(filename)
-      contents = File.read("#{@config_dir}/#{filename}")
-      hash = YAML::load(contents)
-      hash.stringify_keys[@env].symbolize_keys
-    rescue
-      nil
+      Configurator.configure(@options)
     end
   end
 end

data/lib/larva/version.rb

@@ -1,3 +1,3 @@
 module Larva
-  VERSION = '0.6.4'
+  VERSION = '0.7.0'
 end

data/lib/larva/worker_pool.rb

@@ -16,11 +16,13 @@ module Larva
 
     private
     def start_workers
-      logger.info "Starting threads."
+      logger.info "Starting #{processors.count} threads."
       @workers = processors.map do |topic, processor|
-        Thread.new { start_worker(topic, processor) }
+        worker = Thread.new { start_worker(topic, processor) }
+        worker[:name] = "Listener for #{topic}"
+        worker
       end
-      logger.info "Threads Started."
+      logger.info "#{workers.count} threads started."
     end
 
     def start_worker(topic, processor)
@@ -31,7 +33,10 @@ module Larva
 
     def keep_workers_alive
       sleep(5) while workers.all? { |t| t.alive?  }
-      logger.error "Some threads have died"
+      logger.error 'Some threads have died:'
+      workers.each do |worker|
+        logger.error "#{worker[:name]} was #{worker.alive? ? 'alive' : 'dead'}"
+      end
     end
 
     def logger

data/template/config/propono.yml

@@ -6,8 +6,7 @@ development:
   queue_suffix:
 
 production:
-  access_key: MY-PRODUCTION-ACCESS-KEY
-  secret_key: MY-PRODUCTION-SECRET-KEY
+  use_iam_profile: true
   region: eu-west-1
   application_name: larva_spawn
   queue_suffix:

data/test/configuration_test.rb

@@ -14,4 +14,3 @@ module Larva
     end
   end
 end
-

data/test/configurator_test.rb

@@ -0,0 +1,83 @@
+require File.expand_path('../test_helper', __FILE__)
+
+module Larva
+  class DaemonTest < Minitest::Test
+    def config_dir
+      File.expand_path('../../template/config', __FILE__)
+    end
+
+    def logfile
+      "log/test.log"
+    end
+
+    def test_logfile_is_compulsary
+      assert_raises(LarvaError, "Please provide :logfile via options") do
+        Configurator.configure(config_dir: config_dir)
+      end
+    end
+
+    def test_config_dir_is_compulsary
+      assert_raises(LarvaError, "Please provide :config_dir via options") do
+        Configurator.configure(logfile: logfile)
+      end
+    end
+
+    def test_filum_gets_config
+      Configurator.configure(logfile: logfile, config_dir: config_dir)
+      assert_equal logfile, Filum.logger.logfile
+    end
+
+    def test_meducation_sdk_gets_config_in_dev
+      sdk_yaml_path = File.expand_path('../../template/config/meducation-sdk.yml', __FILE__)
+      begin
+        optional_sdk_yaml_path = "#{sdk_yaml_path}.optional"
+        `cp #{optional_sdk_yaml_path} #{sdk_yaml_path}`
+        Configurator.configure(config_dir: config_dir, logfile: logfile)
+        assert_equal "LarvaSpawn", MeducationSDK.config.access_id
+        assert_equal "foobar", MeducationSDK.config.secret_key
+        assert_equal Filum.logger, MeducationSDK.config.logger
+        assert_equal "http://localhost:3000/system", MeducationSDK.config.endpoint
+      ensure
+        `rm #{sdk_yaml_path}`
+      end
+    end
+
+    def test_meducation_sdk_gets_config_in_production
+      sdk_yaml_path = File.expand_path('../../template/config/meducation-sdk.yml', __FILE__)
+      begin
+        optional_sdk_yaml_path = "#{sdk_yaml_path}.optional"
+        `cp #{optional_sdk_yaml_path} #{sdk_yaml_path}`
+        Configurator.configure(config_dir: config_dir, logfile: logfile, env: 'production')
+        assert_equal "LarvaSpawn", MeducationSDK.config.access_id
+        assert_equal nil, MeducationSDK.config.secret_key
+        assert_equal Filum.logger, MeducationSDK.config.logger
+        assert_equal nil, MeducationSDK.config.endpoint
+      ensure
+        `rm #{sdk_yaml_path}`
+      end
+    end
+
+    def test_propono_gets_config
+      Configurator.configure(config_dir: config_dir, logfile: logfile)
+      assert_equal "MY-DEV-ACCESS-KEY", Propono.config.access_key
+      assert_equal "MY-DEV-SECRET-KEY", Propono.config.secret_key
+      assert_equal "eu-west-1", Propono.config.queue_region
+      assert_equal "development_larva_spawn", Propono.config.application_name
+      assert_equal nil, Propono.config.queue_suffix
+      assert_equal "pergo.meducation.net", Propono.config.udp_host
+      assert_equal "9732", Propono.config.udp_port
+      assert_equal Filum.logger, Propono.config.logger
+    end
+
+    def test_propono_gets_config_with_env
+      Configurator.configure(config_dir: config_dir, logfile: logfile, env: 'production')
+      assert_equal true, Propono.config.use_iam_profile
+      assert_equal "larva_spawn", Propono.config.application_name
+      assert_equal nil, Propono.config.queue_suffix
+      assert_equal "pergo.meducation.net", Propono.config.udp_host
+      assert_equal "9732", Propono.config.udp_port
+      assert_equal Filum.logger, Propono.config.logger
+    end
+  end
+end
+

data/test/daemon_test.rb

@@ -2,73 +2,17 @@ require File.expand_path('../test_helper', __FILE__)
 
 module Larva
   class DaemonTest < Minitest::Test
-    def config_dir
-      File.expand_path('../sample_config', __FILE__)
-    end
-
-    def logfile
-      "log/test.log"
-    end
-
     def test_workerpool_is_started
       processors = {foo: 'bar'}
-      Larva::WorkerPool.expects(:start).with(processors)
+      Configurator.stubs(:configure)
+      WorkerPool.expects(:start).with(processors)
       Daemon.start(processors, logfile: "./log/foo.log", config_dir: config_dir)
     end
 
-    def test_logfile_is_compulsary
-      assert_raises(LarvaError, "Please provide :logfile via options") do
-        Daemon.start({}, config_dir: config_dir)
-      end
-    end
-
-    def test_config_dir_is_compulsary
-      assert_raises(LarvaError, "Please provide :config_dir via options") do
-        Daemon.start({}, logfile: logfile)
-      end
-    end
-
-    def test_filum_gets_config
-      Daemon.start({}, logfile: logfile, config_dir: config_dir)
-      assert_equal logfile, Filum.logger.logfile
-    end
-
-    def test_meducation_sdk_gets_config
-      Daemon.start({}, config_dir: config_dir, logfile: logfile)
-      assert_equal "Daemon", MeducationSDK.config.access_id
-      assert_equal "foobar", MeducationSDK.config.secret_key
-      assert_equal Filum.logger, MeducationSDK.config.logger
-    end
-
-    def test_meducation_sdk_gets_config_with_env
-      Daemon.start({}, config_dir: config_dir, logfile: logfile, env: 'production')
-      assert_equal "Daemon", MeducationSDK.config.access_id
-      assert_equal nil, MeducationSDK.config.secret_key
-      assert_equal Filum.logger, MeducationSDK.config.logger
-    end
-
-    def test_propono_gets_config
-      Daemon.start({}, config_dir: config_dir, logfile: logfile)
-      assert_equal "BADASSDEVKEY", Propono.config.access_key
-      assert_equal "SCARYDEVSECRET", Propono.config.secret_key
-      assert_equal "eu-west-1", Propono.config.queue_region
-      assert_equal "development_daemon", Propono.config.application_name
-      assert_equal "-dev", Propono.config.queue_suffix
-      assert_equal "pergo.meducation.net", Propono.config.udp_host
-      assert_equal "9732", Propono.config.udp_port
-      assert_equal Filum.logger, Propono.config.logger
-    end
-
-    def test_propono_gets_config_with_env
-      Daemon.start({}, config_dir: config_dir, logfile: logfile, env: 'production')
-      assert_equal "BADASSPRODUCTIONKEY", Propono.config.access_key
-      assert_equal "SCARYPRODUCTIONSECRET", Propono.config.secret_key
-      assert_equal "eu-west-1", Propono.config.queue_region
-      assert_equal "daemon", Propono.config.application_name
-      assert_equal nil, Propono.config.queue_suffix
-      assert_equal "pergo.meducation.net", Propono.config.udp_host
-      assert_equal "9732", Propono.config.udp_port
-      assert_equal Filum.logger, Propono.config.logger
+    def test_configurator_is_called
+      options = {logfile: "./log/foo.log", config_dir: config_dir}
+      Configurator.expects(:configure).with(options)
+      Daemon.start({}, options)
     end
   end
 end

data/test/worker_pool_test.rb

@@ -6,15 +6,10 @@ module Larva
       WorkerPool.start({})
     end
 
-    def test_process_logs_start_message
+    def test_process_logs_start_and_end_messages
       Propono.config.logger.stubs(:info)
-      Propono.config.logger.expects(:info).with("Starting threads.")
-      WorkerPool.start({})
-    end
-
-    def test_process_logs_end_message
-      Propono.config.logger.stubs(:info)
-      Propono.config.logger.expects(:info).with("Threads Started.")
+      Propono.config.logger.expects(:info).with("Starting 0 threads.")
+      Propono.config.logger.expects(:info).with("0 threads started.")
       WorkerPool.start({})
     end
 
@@ -23,8 +18,9 @@ module Larva
       Propono.config.logger.expects(:error).with do |error|
         error.start_with?("Unexpected listener termination:")
       end
-      Propono.config.logger.expects(:error).with('Some threads have died')
-      WorkerPool.start({nil => nil})
+      Propono.config.logger.expects(:error).with('Listener for qux was dead')
+      Propono.config.logger.expects(:error).with('Some threads have died:')
+      WorkerPool.start({'qux' => nil})
     end
 
     def test_listen_is_called_correctly

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: larva
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.7.0
 platform: ruby
 authors:
 - iHiD
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-23 00:00:00.000000000 Z
+date: 2014-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: propono
@@ -128,6 +128,7 @@ files:
 - larva.gemspec
 - lib/larva.rb
 - lib/larva/configuration.rb
+- lib/larva/configurator.rb
 - lib/larva/daemon.rb
 - lib/larva/listener.rb
 - lib/larva/mocker.rb
@@ -151,15 +152,13 @@ files:
 - template/test/daemon_test.rb
 - template/test/processors/media_file_processor_test.rb
 - template/test/test_helper.rb
-- template/upstart/larva_spawn.conf
 - test/configuration_test.rb
+- test/configurator_test.rb
 - test/daemon_test.rb
 - test/larva_test.rb
 - test/listener_test.rb
 - test/mocker_test.rb
 - test/processor_test.rb
-- test/sample_config/meducation-sdk.yml
-- test/sample_config/propono.yml
 - test/test_helper.rb
 - test/worker_pool_test.rb
 homepage: https://github.com/meducation/larva
@@ -188,13 +187,12 @@ specification_version: 4
 summary: Some Meducation specific helper files for ur pub/sub network
 test_files:
 - test/configuration_test.rb
+- test/configurator_test.rb
 - test/daemon_test.rb
 - test/larva_test.rb
 - test/listener_test.rb
 - test/mocker_test.rb
 - test/processor_test.rb
-- test/sample_config/meducation-sdk.yml
-- test/sample_config/propono.yml
 - test/test_helper.rb
 - test/worker_pool_test.rb
 has_rdoc: 

data/template/upstart/larva_spawn.conf

@@ -1,24 +0,0 @@
-description "LarvaSpawn Upstart Script"
-author "Charles Care <charles@meducation.net>"
-
-# Triggers
-start on startup
-start on runlevel [2345]
-stop on runlevel [016]
-
-# Restart if the service terminates abnormally
-respawn
-
-# Give up if the service requires restarting more than 10 times in a 30 second period.
-respawn limit 10 30
-
-# Run as a user
-setuid larva_spawn
-
-# cwd
-chdir /opt/larva_spawn
-
-#pre-start exec su -c /srv/apps/larva_spawn/bin/larva_spawn-setup ec2-user
-
-# Run target command
-exec /opt/larva_spawn/bin/larva_spawn

data/test/sample_config/meducation-sdk.yml

@@ -1,6 +0,0 @@
-development:
-  access_id: Daemon
-  secret_key: 'foobar'
-
-production:
-  access_id: Daemon

data/test/sample_config/propono.yml

@@ -1,13 +0,0 @@
-development:
-  access_key: BADASSDEVKEY
-  secret_key: SCARYDEVSECRET
-  region: eu-west-1
-  application_name: development_daemon
-  queue_suffix: -dev
-
-production:
-  access_key: BADASSPRODUCTIONKEY
-  secret_key: SCARYPRODUCTIONSECRET
-  region: eu-west-1
-  application_name: daemon
-  queue_suffix: