larva 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0004f08094d687c91fb380df5790dd141cae409d
-  data.tar.gz: bf30bed9cb37627e9c7ccb6d0df30375a89b4c2a
+  metadata.gz: cbf84efda642aa79d413f133e1b7f27db278bec6
+  data.tar.gz: 88f947616c14e531fee42dc00fbd419372303bdb
 SHA512:
-  metadata.gz: 42b25983e5579a66a6848bb4b509093bcf00a04eb9db8a9fe4d4f16a30488201677b1e82f608a9cef85431be9f7bdfb5b679d1e7c48a3fd85777bd1931e91714
-  data.tar.gz: 9f172bd1f6a72b9f6ef642747d52b2ce3de0ad55eba6158cc24ab65588ad1c91b983b36683095f1314ea16f6f98da143b3acf77ae5eb65a65b7fa331dd36a01d
+  metadata.gz: e1f4c8d39c1da9883535dc5b11f152e70a53c09906b6f4e29530514aa63dccd41c1d66f0ef540182d689a806d291c1a0b5b544be7d66d496281109043acdf795
+  data.tar.gz: 78624d5307e979b1bab7bf2d048cb14359d46a6147f2f484d6ec5c23b0146ac5fdf28b585ade020bd83feaca2969cfc575abe6f9ae48e0a82ceada5c6697c6dd

data/.gitignore

@@ -1,2 +1,3 @@
 Gemfile.lock
 log/
+pkg/

data/.travis.yml

@@ -0,0 +1,14 @@
+language: ruby
+cache: bundler
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - rbx-2.1.1
+script:
+  - bundle exec rake test:local
+#addons:
+#    code_climate:
+#        repo_token: 88dd239bc2e0010742a42a4e0234b4decd19b46d0e9d3408d8b1fe0f96dd8fc1
+matrix:
+  allow_failures:
+    - rvm: rbx-2.1.1

data/CHANGELOG

@@ -1,2 +1,8 @@
+# 0.3.0 / 2014-02-22
+* [FEATURE] Add worker pool.
+
+# 0.2.0 / 2014-02-22
+* [FEATURE] Add processor.
+
 # 0.1.0 / 2014-02-22
-* [FEATURE] Initial release with listeners
+* [FEATURE] Initial release with listeners.

data/README.md

@@ -18,17 +18,76 @@ And then execute:
 
 ## Usage
 
+Larva provides you with listeners, processors and a worker pool to build an application that listens and responds to Propono messages. 
+
+Here is a sample application. This forms the basis of a rake task for most Meducation daemons.
+
+```ruby
+require 'larva'
+
+# Setup Config for Filum and Propono
+
+class MyProcessor < Larva::Processor
+  def process(message)
+    if entity == "comment" && action == "created"
+      # Do something...
+    else
+      false
+    end
+  end
+end
+
+processors = {my_topic: MyProcessor}
+Larva::WorkerPool.start(processors, "queue-suffix")
+
+```
+
 ### Listeners
 
 Larva Listeners provide an easy way of listening to a Propono topic and processing the message, complete with lots of logging through Filum.
 
+```ruby
+Larva::Listener.listen(:my_topic, processor, "queue_suffix")
+```
+
+This will listen for messages on :my_topic and pass them to `processor.process`. It will log what is happening via Filum.
+
+### Processors
+
+Processors are used by listeners to handle the messages that are received. You are expected to subclass `Lavar::Processor` and implement `process`. 
+
+Processors expect you to have an `entity` and `action` fields in your messages.
+
+For example:
+
 ```ruby
 class MyProcessor
-  def self.process(message)
+  def process(message)
+    if entity == "comment" && action == "created"
+      # Do something...
+    else
+      false
+    end
   end
 end
+Larva::Listener.listen(:my_topic, MyProcessor, "")
+Propono.publish(:my_topic, {entity: "comment", action: "created", id: 8}
+```
+
+With this code `MyProcessor#process` will get called for each message, with extra logging around the call. Within the class you have access to `message`, `action`, `entity` and `id` methods.
+
+### Worker Pool
 
-Larva::Listener.listen(:my_topic, MyProcessor, "queue_suffix")
+The worker pool creates a listener for each topic, and proxies messages to the associated processors. If any processors die, the application will die.
+
+Creating a worker pool is trivial:
+
+```ruby
+processors = {
+  my_topic_1: MyProcessor1
+  my_topic_2: MyProcessor2
+}
+Larva::WorkerPool.start(processors, "queue-suffix")
 ```
 
 ### Is it any good?

data/lib/larva.rb

@@ -4,6 +4,8 @@ require 'propono'
 require 'filum'
 
 require 'larva/listener'
+require 'larva/processor'
+require 'larva/worker_pool'
 
 module Larva
 end

data/lib/larva/processor.rb

@@ -0,0 +1,24 @@
+module Larva
+  class Processor
+    def self.process(message)
+      new(message).process_with_logging
+    end
+
+    attr_accessor :message, :action, :entity, :id
+    def initialize(message)
+      @message = message
+      @action = message[:action]
+      @entity = message[:entity]
+      @id = message[:id]
+    end
+
+    def process_with_logging
+      Propono.config.logger.info "Processing message: #{message}"
+      if self.process
+        Propono.config.logger.info "Message Processed: #{message}"
+      else
+        Propono.config.logger.info "Unrecognized event type, entity: #{entity} action: #{action}."
+      end
+    end
+  end
+end

data/lib/larva/version.rb

@@ -1,3 +1,3 @@
 module Larva
-  VERSION = '0.1.0'
+  VERSION = '0.3.0'
 end

data/lib/larva/worker_pool.rb

@@ -0,0 +1,42 @@
+module Larva
+  class WorkerPool
+    def self.start(processors, queue_suffix)
+      new(processors, queue_suffix).start
+    end
+
+    attr_reader :processors, :queue_suffix, :workers
+    def initialize(processors, queue_suffix)
+      @processors = processors
+      @queue_suffix = queue_suffix
+    end
+
+    def start
+      start_workers
+      keep_workers_alive if workers.count > 0
+    end
+
+    private
+    def start_workers
+      logger.info "Starting threads."
+      @workers = processors.map do |topic, processor|
+        Thread.new { start_worker(topic, processor) }
+      end
+      logger.info "Threads Started."
+    end
+
+    def start_worker(topic, processor)
+      Larva::Listener.listen(topic, processor, queue_suffix)
+    rescue => e
+      logger.error "Unexpected listener termination: #{e} #{e.backtrace}"
+    end
+
+    def keep_workers_alive
+      sleep(1) while workers.all? { |t| t.alive?  }
+      logger.error "Some threads have died"
+    end
+
+    def logger
+      Propono.config.logger
+    end
+  end
+end

data/test/processor_test.rb

@@ -0,0 +1,54 @@
+require File.expand_path('../test_helper', __FILE__)
+
+module Larva
+  class ProcessorTest < Minitest::Test
+    def test_initialize_should_extract_action_and_entity
+      entity = "media_file"
+      action = "processed"
+      message = {entity: entity, action: action, media_file_id: "8"}
+      processor = Processor.new(message)
+      assert_equal entity, processor.entity
+      assert_equal action, processor.action
+    end
+
+    class GoodProcessor < Processor
+      def process
+        true
+      end
+    end
+
+    class BadProcessor < Processor
+      def process
+        false
+      end
+    end
+
+    def test_process_logs_message
+      message = {entity: "media_file", action: "processed", media_file_id: "8"}
+      output = "Processing message: #{message}"
+      Propono.config.logger.stubs(:info)
+      Propono.config.logger.expects(:info).with(output)
+      GoodProcessor.process(message)
+    end
+
+    def test_process_logs_success
+      message = {entity: "media_file", action: "processed", media_file_id: "8"}
+      output = "Message Processed: #{message}"
+      Propono.config.logger.stubs(:info)
+      Propono.config.logger.expects(:info).with(output)
+      GoodProcessor.process(message)
+    end
+
+    def test_process_logs_message
+      entity = "media_file"
+      action = "processed"
+      message = {entity: entity, action: action, media_file_id: "8"}
+      output = "Unrecognized event type, entity: #{entity} action: #{action}."
+
+      Propono.config.logger.stubs(:info)
+      Propono.config.logger.expects(:info).with(output)
+      BadProcessor.process(message)
+    end
+  end
+end
+

data/test/worker_pool_test.rb

@@ -0,0 +1,38 @@
+require File.expand_path('../test_helper', __FILE__)
+
+module Larva
+  class WorkerPoolTest < Minitest::Test
+    def test_should_complete_for_no_processors
+      WorkerPool.start({}, "")
+    end
+
+    def test_process_logs_start_message
+      Propono.config.logger.stubs(:info)
+      Propono.config.logger.expects(:info).with("Starting threads.")
+      WorkerPool.start({}, "")
+    end
+
+    def test_process_logs_end_message
+      Propono.config.logger.stubs(:info)
+      Propono.config.logger.expects(:info).with("Threads Started.")
+      WorkerPool.start({}, "")
+    end
+
+    def test_start_worker_logs_exception
+      Larva::Listener.expects(:listen).raises(RuntimeError)
+      Propono.config.logger.expects(:error).with do |error|
+        error.start_with?("Unexpected listener termination:")
+      end
+      Propono.config.logger.expects(:error).with('Some threads have died')
+      WorkerPool.start({nil => nil}, "")
+    end
+
+    def test_listen_is_called_correctly
+      topic_name = "Foo"
+      processor = mock
+      queue_suffix = "Bar"
+      Larva::Listener.expects(:listen).with(topic_name, processor, queue_suffix)
+      WorkerPool.start({topic_name => processor}, queue_suffix)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: larva
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - iHiD
@@ -102,6 +102,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG
 - CONTRIBUTING.md
 - Gemfile
@@ -111,10 +112,14 @@ files:
 - larva.gemspec
 - lib/larva.rb
 - lib/larva/listener.rb
+- lib/larva/processor.rb
 - lib/larva/version.rb
+- lib/larva/worker_pool.rb
 - test/larva_test.rb
 - test/listener_test.rb
+- test/processor_test.rb
 - test/test_helper.rb
+- test/worker_pool_test.rb
 homepage: https://github.com/meducation/larva
 licenses:
 - AGPL3
@@ -142,5 +147,7 @@ summary: Some Meducation specific helper files for ur pub/sub network
 test_files:
 - test/larva_test.rb
 - test/listener_test.rb
+- test/processor_test.rb
 - test/test_helper.rb
+- test/worker_pool_test.rb
 has_rdoc: