toiler 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5147089083d760568e0f9c58442bac5f37e0c313
-  data.tar.gz: 087b505c5d9643f04fce1989c7014dc3b96ddad4
+  metadata.gz: a2db74f11b3d29deb51e673e5bbb317b087da26b
+  data.tar.gz: b40560d3d2fbc5e0781154eb509988ca2bbb93c9
 SHA512:
-  metadata.gz: 64b2d544c5178bfa61a66a03620b1a0d29c71de424ebc60874b87fb2259095d6d01faec7b049fe2e23536968d0ba23c74129cb57caa5948fe06ff7ef785e43f5
-  data.tar.gz: bf2e185000f5086fd0cb97fc657eb0a3c888e0279e88cd93438634407f59cea9bce22b340fc5c2a9489fd1519c90e728e01238b53b933d2f47d7bef0d12112d7
+  metadata.gz: afaa74f96f030f0e521bdb8cb6948a4839772428c4bb427da62fac78a0c2bfa87aa72506080ae1c25c57b6b659f324c4c99b19b6acb8a70b52508eb69036d601
+  data.tar.gz: a6deb6ab20852811f794d3b4f97abd3c7d6b5ab9e0aa5f3578d01bc0f3dfee9a588fed13b8c8cf95c46f1165000830d4d9a9520bb052750d360e05b883cfa231

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -7,3 +7,6 @@ gem 'aws-sdk', '~> 2.2'
 group :development do
   gem 'rubocop'
 end
+group :test do
+  gem 'rspec'
+end

data/Gemfile.lock

@@ -11,6 +11,7 @@ GEM
     concurrent-ruby (1.0.2)
     concurrent-ruby-edge (0.2.2)
       concurrent-ruby (~> 1.0.2)
+    diff-lcs (1.2.5)
     jmespath (1.2.4)
       json_pure (>= 1.8.1)
     json_pure (1.8.3)
@@ -18,6 +19,19 @@ GEM
       ast (~> 2.2)
     powerpack (0.1.1)
     rainbow (2.1.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.2)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
     rubocop (0.40.0)
       parser (>= 2.3.1.0, < 3.0)
       powerpack (~> 0.1)
@@ -35,6 +49,7 @@ DEPENDENCIES
   aws-sdk (~> 2.2)
   concurrent-ruby (~> 1.0.0)
   concurrent-ruby-edge (~> 0.2.0)
+  rspec
   rubocop
 
 BUNDLED WITH

data/README.md

@@ -101,6 +101,7 @@ toiler --help
 
     -d, --daemon                     Daemonize process
     -r, --require [PATH|DIR]         Location of the worker
+    -q, --queue QUEUE1,QUEUE2,...    Queues to process
     -C, --config PATH                Path to YAML config file
     -R, --rails                      Load Rails
     -L, --logfile PATH               Path to writable logfile

data/lib/toiler.rb

@@ -28,6 +28,22 @@ module Toiler
     worker_class_registry.keys
   end
 
+  def active_worker_class_registry
+    active_queues = options[:active_queues]
+    if active_queues
+      active_queues.each_with_object({}) do |q, registry|
+        worker = @worker_class_registry[q]
+        if worker.nil?
+          logger.warn "No worker assigned to queue: #{q}"
+        else
+          registry[q] = worker
+        end
+      end
+    else
+      @worker_class_registry
+    end
+  end
+
   def fetcher(queue)
     fetchers["fetcher_#{queue}".to_sym]
   end
@@ -52,4 +68,12 @@ module Toiler
       batch: false
     }
   end
+
+  def register_worker(queue, worker)
+    @worker_class_registry[queue] = worker
+  end
+
+  def worker_class_registry=(val)
+    @worker_class_registry = val
+  end
 end

data/lib/toiler/actor/supervisor.rb

@@ -19,12 +19,8 @@ module Toiler
         pass
       end
 
-      def queues
-        Toiler.worker_class_registry
-      end
-
       def spawn_fetchers
-        queues.each do |queue, _klass|
+        Toiler.active_worker_class_registry.each do |queue, _klass|
           begin
             fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
                                             supervise: true, args: [queue, client]
@@ -36,7 +32,7 @@ module Toiler
       end
 
       def spawn_processors
-        queues.each do |queue, klass|
+        Toiler.active_worker_class_registry.each do |queue, klass|
           name = "processor_pool_#{queue}".to_sym
           count = klass.concurrency
           begin

data/lib/toiler/actor/utils/actor_logging.rb

@@ -16,7 +16,7 @@ module Toiler
         end
 
         def warn(msg)
-          log Logger::Severity::WARN, smsg
+          log Logger::Severity::WARN, msg
         end
 
         def fatal(msg)

data/lib/toiler/utils/argument_parser.rb

@@ -12,6 +12,10 @@ module Toiler
             opts[:daemon] = arg
           end
 
+          o.on '-q', '--queue QUEUE1,QUEUE2,...', 'Queues to process' do |arg|
+            opts[:active_queues] = arg.split(',')
+          end
+
           o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
             opts[:require] = arg
           end

data/lib/toiler/version.rb

@@ -1,4 +1,4 @@
 # Toiler Version
 module Toiler
-  VERSION = '0.4.1'.freeze
+  VERSION = '0.4.2'.freeze
 end

data/lib/toiler/worker.rb

@@ -34,7 +34,7 @@ module Toiler
     module ClassMethods
       def toiler_options(options = {})
         return class_variable_get(:@@toiler_options) if options.empty?
-        Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
+        Toiler.register_worker(options[:queue], self) if options[:queue]
         class_variable_get(:@@toiler_options).merge! options
       end
 

data/spec/models/fetcher_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+require 'toiler/actor/fetcher'
+RSpec.describe Toiler::Actor::Fetcher, type: :model do
+  let(:queue) { 'default' }
+  let(:client) { double(:aws_sqs_client) }
+
+  before do
+    allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:log).and_return(true)
+    allow_any_instance_of(Toiler::Aws::Queue).to receive(:visibility_timeout).and_return(100)
+    allow(client).to receive(:get_queue_url).with(queue_name: 'default').and_return double(:queue, queue_url: 'http://aws.fake/queue')
+  end
+
+  describe "#new" do
+    it 'completes sucessfully' do
+      fetcher = described_class.new(queue, client)
+      expect(fetcher.polling?).to eq(false)
+      expect(fetcher.executing?).to eq(false)
+      expect(fetcher.scheduled?).to eq(false)
+    end
+  end
+end

data/spec/models/processor_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+require 'toiler/actor/processor'
+RSpec.describe Toiler::Actor::Processor, type: :model do
+  let(:fetcher) { double(:fetcher) }
+
+  describe "#new" do
+    it 'initializes properly' do
+      allow(Toiler).to receive(:fetcher).and_return(fetcher)
+      expect(fetcher).to receive(:tell).with(:processor_finished)
+      processor = described_class.new('default')
+      expect(processor.executing?).to eq(false)
+    end
+  end
+end

data/spec/models/supervisor_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+require 'toiler/actor/supervisor'
+RSpec.describe Toiler::Actor::Supervisor, type: :model do
+  let(:sqs_client) { double(:client) }
+  describe "#new" do
+    it 'only spawns fetchers for active workers' do
+      class InactiveWorker
+        include Toiler::Worker
+        toiler_options queue: 'inactive_queue'
+        def perform(sqs_message, body); end
+      end
+
+      Toiler.options.merge!(active_queues: ['default'])
+      expect(::Aws::SQS::Client).to receive(:new).and_return(sqs_client)
+      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client])
+      expect(Concurrent::Actor::Utils::Pool).to receive(:spawn!).with(:processor_pool_default, 1)
+      supervisor = described_class.new
+    end
+
+    it 'warns when a queue is missing' do
+      Toiler.options.merge!(active_queues: ['missing'])
+      expect(Toiler::Utils::Logging.logger).to receive(:warn).with("No worker assigned to queue: missing")
+      Toiler.active_worker_class_registry
+    end
+  end
+end

data/spec/models/worker_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+require 'toiler/worker'
+RSpec.describe Toiler::Worker, type: :model do
+  describe "loading class" do
+    it "adds the class to the worker registry, under the queue name" do
+      class FakeWorker
+        include Toiler::Worker
+        toiler_options queue: 'test_queue'
+
+        def perform(sqs_message, body); end
+      end
+
+      expect(Toiler.worker_class_registry['test_queue']).to eq(FakeWorker)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,23 @@
+require 'bundler/setup'
+Bundler.setup
+
+require 'concurrent'
+require 'concurrent-edge'
+Concurrent.use_stdlib_logger Logger::FATAL
+
+require 'toiler'
+class TestWorker
+  include Toiler::Worker
+
+  toiler_options queue: 'default'
+
+  def perform(sqs_message, body); end
+end
+
+require 'rspec'
+RSpec.configure do |config|
+  config.before do
+    Toiler.worker_class_registry = {}
+    Toiler.register_worker('default', TestWorker)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Sebastian Schepens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-09 00:00:00.000000000 Z
+date: 2016-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitmodules"
+- ".rspec"
 - ".ruby-version"
 - Gemfile
 - Gemfile.lock
@@ -184,6 +185,11 @@ files:
 - lib/toiler/utils/logging.rb
 - lib/toiler/version.rb
 - lib/toiler/worker.rb
+- spec/models/fetcher_spec.rb
+- spec/models/processor_spec.rb
+- spec/models/supervisor_spec.rb
+- spec/models/worker_spec.rb
+- spec/spec_helper.rb
 - toiler.gemspec
 homepage: https://github.com/sschepens/toiler
 licenses:
@@ -209,4 +215,9 @@ rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Toiler is a super efficient AWS SQS thread based message processor
-test_files: []
+test_files:
+- spec/models/fetcher_spec.rb
+- spec/models/processor_spec.rb
+- spec/models/supervisor_spec.rb
+- spec/models/worker_spec.rb
+- spec/spec_helper.rb