micro_q 0.7.0 → 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5d7397239d88fe8ba5027ca54e052af6ee286e32
+  data.tar.gz: e52f3922fd037146a570208025aaf64451aa600c
+SHA512:
+  metadata.gz: 2318f4b75ccaaa0412809273013d7d8b563356b3cafd0219ed26b2bad66ccd3fd2c753fa8b36b76dade486182ccb12394c6679b2df4ea5edf734e66dcb2d7805
+  data.tar.gz: 9b989f109f14acaea6742c95050701aa81631137923a1896c0b43f88a9f088179bf8826cff482205cbff14e228a8e9aefc2c82a3ead9b7762469006275050a9b

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0-p0@micro_q

data/.travis.yml

@@ -1,7 +1,9 @@
 language: ruby
 rvm:
   - 1.9.3
+  - 1.9.2
   - rbx-19mode
+  - rbx-nightly-19mode
   - 2.0.0
 branches:
   only:
@@ -12,5 +14,11 @@ notifications:
       - brian.nort@gmail.com
 matrix:
   allow_failures:
+    - rvm: 1.9.3
+    - rvm: 1.9.2
     - rvm: rbx-19mode
-    - rvm: 2.0.0
+    - rvm: rbx-nightly-19mode
+branches:
+  only:
+    - master
+    - /^((feature)|(hotfix))\/.+/

data/README.md

@@ -1,4 +1,7 @@
-# MicroQ [![Build Status](https://travis-ci.org/bnorton/micro_q.png)](https://travis-ci.org/bnorton/micro_q)
+# MicroQ
+
+[![Build Status](https://travis-ci.org/bnorton/micro_q.png)](https://travis-ci.org/bnorton/micro_q)  
+[![Code Climate](https://codeclimate.com/github/bnorton/micro_q.png)](https://codeclimate.com/github/bnorton/micro_q)
 
 MicroQ is a per-process asynchronous background queue.
 
@@ -42,7 +45,7 @@ MyWorker.async_update(:user_id => user.id)
 
 ###Advanced
 
-Safely using an ActiveRecord instance via the Custom Loader API 
+Safely using an ActiveRecord instance via the [Custom Loader](https://github.com/bnorton/micro_q/wiki/Loaders) API 
 ```ruby
 # app/models/user.rb
 class User < Activerecord::Base

data/lib/micro_q/config.rb

@@ -17,6 +17,7 @@ module MicroQ
         'manager' => Manager::Default,
         'worker' => Worker::Standard,
         'queue' => Queue::Default,
+        'statistics' => Statistics::Default,
         'redis_pool' => { :size => 15, :timeout => 1 },
         'redis' => { :host => 'localhost', :port => 6379 }
       }

data/lib/micro_q/dsl.rb

@@ -1,28 +1,53 @@
 module MicroQ
+  ##
+  # Convenience methods for calling methods asynchronously
+  # Adds async_perform by default
+  #
+  # Usage
+  # class MyWorker
+  #   worker :update, :queue => 'non-default'
+  #
+  #   def update
+  #   end
+  # end
+  #
+  # MyWorker.async_update
+  # is the same as
+  # MyWorker.new.async(:queue => 'non-default').update
+  #
   module DSL
-    def worker(*opts)
-      self.class_eval do
-        def self.microq_options
-          @microq_options ||= { :methods => [:perform] }
+    ##
+    # For each of the methods given to the Object.worker method
+    # define the async_ prefixed version for convenience
+    #
+    def self.attach_async_methods(target, opts)
+      target.class_eval do
+        (target.microq_options[:methods] |= opts.flatten).each do |method|
+          target.define_singleton_method(:"async_#{method}") do |*args|
+            MicroQ::Proxy::Instance.new(
+              target.microq_options.dup.merge(:class => self)
+            ).send(method, *args)
+          end unless respond_to?(:"async_#{method}")
         end
       end
+    end
 
-      if Hash === opts.last
-        self.microq_options.merge!(opts.pop)
-      end
-
-      async_methods = self.microq_options[:methods] |= opts.flatten
+    module ClassMethods
+      def worker(*opts)
+        self.class_eval do
+          def self.microq_options
+            @microq_options ||= { :methods => [:perform] }
+          end
+        end
 
-      self.class_eval do
-        async_methods.each do |method|
-          async_method = :"async_#{method}"
-          define_singleton_method(async_method) do |*args|
-            MicroQ::Proxy::Instance.new(:class => self).send(method, *args)
-          end unless respond_to?(async_method)
+        if Hash === opts.last
+          self.microq_options.merge!(opts.pop)
         end
+
+        DSL.attach_async_methods(self, opts)
       end
     end
   end
 end
 
-Object.send(:extend, MicroQ::DSL)
+Object.send(:extend, MicroQ::DSL::ClassMethods)

data/lib/micro_q/manager/default.rb

@@ -17,24 +17,70 @@ module MicroQ
     class Default
       include Celluloid
 
-      attr_reader :queue, :workers
+      # Invoke this when the Queue or Worker pool dies
+      exit_handler :reinitialize
 
-      def initialize
-        @queue   = MicroQ.config.queue.new
-        @workers = MicroQ.config.worker.pool(:size => MicroQ.config.workers)
-      end
+      attr_reader :queue, :workers
 
       def start
-        count = workers.idle_size
+        count = workers.size
 
         if (messages = queue.dequeue(count)).any?
           messages.each do |message|
-            workers.perform!(message)
+            worker = workers.pop
+            @busy << worker
+
+            worker.perform!(message)
           end
         end
 
         after(2) { start }
       end
+
+      def work_done(worker)
+        @busy.delete(worker)
+        workers.push(worker)
+      end
+
+      ##
+      # Handle init/death of the Queue or the Worker pool
+      #
+      def reinitialize(*)
+        kill_all and return if self.class.shutdown?
+
+        unless @queue && @queue.alive?
+          @queue = MicroQ.config.queue.new_link
+        end
+
+        @busy ||= []
+
+        build_missing_workers
+      end
+
+      alias initialize reinitialize
+
+      # Don't shrink the pool if the config changes
+      def build_missing_workers
+        @workers ||= []
+
+        workers.reject! {|worker| !worker.alive? }
+
+        [MicroQ.config.workers - (workers.size + @busy.size), 0].max.times do
+          workers << MicroQ.config.worker.new_link(current_actor)
+        end
+      end
+
+      def kill_all
+        (@workers + @busy).each {|w| w.terminate if w.alive? }
+      end
+
+      def self.shutdown?
+        !!@shutdown
+      end
+
+      def self.shutdown!
+        @shutdown = true
+      end
     end
   end
 end

data/lib/micro_q/middleware/chain.rb

@@ -1,5 +1,10 @@
+require 'micro_q/middleware/util'
 require 'micro_q/middleware/server/retry'
 require 'micro_q/middleware/server/connection'
+require 'micro_q/middleware/server/timeout'
+require 'micro_q/middleware/server/statistics'
+
+require 'micro_q/middleware/client/statistics'
 
 module MicroQ
   module Middleware
@@ -35,12 +40,14 @@ module MicroQ
       end
 
       class Base
-        attr_reader :entries
-
         def initialize
           clear
         end
 
+        def entries
+          [*@entries, @last].compact
+        end
+
         ##
         # Add any number of entries to the middleware chain
         #
@@ -90,7 +97,9 @@ module MicroQ
 
       class Server < Base
         def initialize
+          @last    = MicroQ::Middleware::Server::Statistics
           @entries = [
+            MicroQ::Middleware::Server::Timeout,
             MicroQ::Middleware::Server::Retry,
             MicroQ::Middleware::Server::Connection
           ]
@@ -98,6 +107,10 @@ module MicroQ
       end
 
       class Client < Base
+        def initialize
+          @last = MicroQ::Middleware::Client::Statistics
+          super
+        end
       end
     end
   end

data/lib/micro_q/middleware/client/statistics.rb

@@ -0,0 +1,22 @@
+module MicroQ
+  module Middleware
+    module Client
+      class Statistics
+        include MicroQ::Middleware::Util
+
+        ENQUEUED = proc {|klass| klass ? "messages:#{klass}:enqueued" : 'messages:enqueued' }
+
+        def call(message, options)
+          stats(message)
+          yield
+        end
+
+        private
+
+        def stats(msg)
+          stats_incr(msg, ENQUEUED, "queues:#{msg['queue']}:enqueued")
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/middleware/server/retry.rb

@@ -11,21 +11,36 @@ module MicroQ
       # when:  The time at which the message will be retried again
       #
       class Retry
+        include MicroQ::Middleware::Util
+
+        RETRY = proc {|klass| klass ? "messages:#{klass}:retry" : 'messages:retry' }
+
         def call(worker, message)
           yield
         rescue Exception => e
           raise e unless message['retry']
 
-          message['retried'] ||= { 'count' => 0 }
-
-          message['retried']['count'] += 1
-          message['retried']['at']    = Time.now
-          message['retried']['when']  = (Time.now + 15).to_f
+          retried!(message)
+          stats(message)
 
           MicroQ.push(message, message['retried'])
 
           raise e
         end
+
+        private
+
+        def retried!(msg)
+          msg['retried'] ||= { 'count' => 0 }
+
+          msg['retried']['count'] += 1
+          msg['retried']['at']    = Time.now
+          msg['retried']['when']  = (Time.now + 15).to_f
+        end
+
+        def stats(msg)
+          stats_incr(msg, RETRY, "queues:#{msg['queue']}:retry")
+        end
       end
     end
   end

data/lib/micro_q/middleware/server/statistics.rb

@@ -0,0 +1,22 @@
+module MicroQ
+  module Middleware
+    module Server
+      class Statistics
+        include MicroQ::Middleware::Util
+
+        PERFORMED = proc {|klass| klass ? "messages:#{klass}:performed" : 'messages:performed' }
+
+        def call(_, message)
+          stats(message)
+          yield
+        end
+
+        private
+
+        def stats(msg)
+          stats_incr(msg, PERFORMED, "queues:#{msg['queue']}:performed")
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/middleware/server/timeout.rb

@@ -0,0 +1,19 @@
+module MicroQ
+  module Middleware
+    module Server
+      class Timeout
+        DEFAULT = 10 * 60
+
+        include ::Timeout
+
+        def call(_, message)
+          time = (time = message['timeout'].to_i) > 0 ? time : DEFAULT
+
+          timeout(time) do
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/middleware/util.rb

@@ -0,0 +1,21 @@
+module MicroQ
+  module Middleware
+    module Util
+      def stats_incr(msg, generator, *keys)
+        statistics do |stats|
+          stats.incr(
+            generator.call,
+            generator.call(msg['class']),
+            *keys.flatten
+          )
+        end
+      end
+
+      def statistics
+        MicroQ::Statistics::Default.stats do |stats|
+          yield stats
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/queue/default.rb

@@ -53,9 +53,9 @@ module MicroQ
       #
       def sync_push(item, options={})
         item, options = MicroQ::Util.stringify(item, options)
-        klass = item['class'] = item['class'].to_s
+        item['class'] = item['class'].to_s
 
-        MicroQ.middleware.client.call(klass, item, options) do
+        MicroQ.middleware.client.call(item, options) do
           if (time = options['when'])
             @later.push(
               'when' => time.to_f,
@@ -74,23 +74,10 @@ module MicroQ
       def dequeue(limit = 30)
         return [] if limit == 0
 
-        idx = 0
+        opts = { :i => 0, :limit => limit}
         [].tap do |items|
-          entries.each do |entry|
-            items << entry unless (idx += 1) > limit
-          end if entries.any?
-
-          items.each {|i| entries.delete(i) }
-
-          available = later.select {|entry| entry['when'] < Time.now.to_f }
-
-          if available.any?
-            available.each do |entry|
-              items << entry['worker'] unless (idx += 1) > limit
-            end
-
-            available.each {|a| later.delete(a) }
-          end
+          dequeue_entries!(items, opts)
+          dequeue_later!(items, opts)
         end
       end
 
@@ -107,6 +94,26 @@ module MicroQ
 
       private
 
+      def dequeue_entries!(items, options)
+        entries.each do |entry|
+          items << entry unless (options[:i] += 1) > options[:limit]
+        end if entries.any?
+
+        items.each {|i| entries.delete(i) }
+      end
+
+      def dequeue_later!(items, options)
+        available = later.select {|entry| entry['when'] < Time.now.to_f }
+
+        if available.any?
+          available.each do |entry|
+            items << entry['worker'] unless (options[:i] += 1) > options[:limit]
+          end
+
+          available.each {|a| later.delete(a) }
+        end
+      end
+
       ##
       # Parse the entries back into the queue from the filesystem
       #

data/lib/micro_q/queue/redis.rb

@@ -41,9 +41,9 @@ module MicroQ
 
       def sync_push(item, options = {})
         item, options = MicroQ::Util.stringify(item, options)
-        klass = item['class'] = item['class'].to_s
+        item['class'] = item['class'].to_s
 
-        MicroQ.middleware.client.call(klass, item, options) do
+        MicroQ.middleware.client.call(item, options) do
           json = JSON.dump(item)
 
           MicroQ.redis do |r|

data/lib/micro_q/statistics/base.rb

@@ -0,0 +1,15 @@
+module MicroQ
+  module Statistics
+    class Base
+      attr_reader :increment
+
+      def self.stats
+        yield instance
+      end
+
+      def self.instance
+        @instance ||= new
+      end
+    end
+  end
+end

data/lib/micro_q/statistics/default.rb

@@ -0,0 +1,18 @@
+module MicroQ
+  module Statistics
+    class Default < Base
+      def initialize
+        @increment = Hash.new { 0 }
+        @increment_mutex = Mutex.new
+      end
+
+      def incr(*keys)
+        @increment_mutex.synchronize do
+          keys.flatten.each do |key|
+            @increment[key.to_s] += 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/statistics/redis.rb

@@ -0,0 +1,23 @@
+module MicroQ
+  module Statistics
+    class Redis < Base
+      INCR = 'statistics:increment'
+
+      def increment
+        MicroQ.redis do |r|
+          r.hgetall(INCR)
+        end.each_with_object({}) do |(k, v), hash|
+          hash[k] = v.to_i
+        end
+      end
+
+      def incr(*keys)
+        MicroQ.redis do |r| r.pipelined {
+          keys.flatten.each do |key|
+            r.hincrby(INCR, key, 1)
+          end
+        } end
+      end
+    end
+  end
+end

data/lib/micro_q/version.rb

@@ -1,6 +1,6 @@
 module MicroQ
   MAJOR = 0
-  MINOR = 7
+  MINOR = 8
   POINT = 0
 
   VERSION = [MAJOR, MINOR, POINT].join('.')

data/lib/micro_q/worker/standard.rb

@@ -12,28 +12,38 @@ module MicroQ
     # A minimal message: (Calls the perform method with zero arguments)
     # { :class => 'MyWorker' }
     #
-    # A more complex message: (Calls the update_data with a single paramater as a list of ids)
+    # A more complex message: (Calls the update_data with a single parameter as a list of ids)
     # { :class => 'MyUpdater', 'method' => 'update_data', :args => [[2, 6,74, 198]]}
     #
     class Standard
       include Celluloid
 
-      def perform(message)
-        klass = MicroQ::Util.constantize(message['class'].to_s)
+      def initialize(manager)
+        @manager = manager
+      end
 
-        loader = message['loader'] ||= { 'method' => 'new' }
-        klass = klass.send(loader['method'], *loader['args']) if loader['method']
+      def perform(message)
+        klass = fetch_klass(message)
 
         method = message['method'] || 'perform'
         args = message['args']
 
-        value = nil
-
-        MicroQ.middleware.server.call(klass, message) do
-          value = klass.send(method, *args)
+        defer do
+          MicroQ.middleware.server.call(klass, message) do
+            klass.send(method, *args)
+          end
         end
 
-        value
+        @manager.work_done!(current_actor)
+      end
+
+      def fetch_klass(message)
+        klass = MicroQ::Util.constantize(message['class'].to_s)
+
+        loader = message['loader'] ||= { 'method' => 'new' }
+        klass = klass.send(loader['method'], *loader['args']) if loader['method']
+
+        klass
       end
     end
   end

data/lib/micro_q.rb

@@ -1,3 +1,4 @@
+require 'timeout'
 require 'celluloid'
 require 'connection_pool'
 require 'redis'
@@ -19,6 +20,10 @@ module MicroQ
     config.middleware
   end
 
+  def self.stats(&block)
+    config.statistics.stats(&block)
+  end
+
   def self.start
     manager
   end
@@ -43,11 +48,28 @@ module MicroQ
 end
 
 require 'micro_q/middleware'
-require 'micro_q/wrappers'
-require 'micro_q/methods'
-require 'micro_q/dsl'
 require 'micro_q/proxies'
+require 'micro_q/dsl'
 require 'micro_q/worker'
 require 'micro_q/queue'
 
 require 'micro_q/redis'
+
+require 'micro_q/wrappers/action_mailer'
+
+# add Class and Instance methods first then
+# override with additional extensions
+
+require 'micro_q/methods/class'
+require 'micro_q/methods/instance'
+require 'micro_q/methods/active_record'
+require 'micro_q/methods/action_mailer'
+
+require 'micro_q/statistics/base'
+require 'micro_q/statistics/default'
+require 'micro_q/statistics/redis'
+
+# There is a better way coming soon 2/18/13
+at_exit do
+  MicroQ::Manager::Default.shutdown!
+end

data/micro_q.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^spec/})
   gem.require_paths = %w(lib)
 
-  gem.add_dependency             "celluloid"
+  gem.add_dependency             "celluloid", '~> 0.12.0'
   gem.add_dependency             "redis"
   gem.add_dependency             "connection_pool"
   gem.add_development_dependency "rake"

data/spec/helpers/methods_examples.rb

@@ -1,4 +1,4 @@
-shared_examples_for "a_worker" do |method|
+shared_examples_for 'a_worker' do |method|
   describe "additions (#{method})" do
     it 'should add an async proxy' do
       subject.respond_to?(:async).should == true

data/spec/helpers/queues_examples.rb

@@ -24,14 +24,14 @@ shared_examples_for 'Queue#sync_push' do
 
   describe 'client middleware' do
     it 'should process the middleware chain' do
-      MicroQ.middleware.client.should_receive(:call) do |w, payload|
-        w.should == 'MyWorker'
-
+      MicroQ.middleware.client.should_receive(:call) do |payload, opts|
         payload['class'].should == 'MyWorker'
         payload['args'].should == [4]
+
+        opts['when'].should == 'now'
       end
 
-      subject.sync_push(item)
+      subject.sync_push(item, 'when' => 'now')
     end
   end
 end

data/spec/lib/config_spec.rb

@@ -63,5 +63,9 @@ describe MicroQ::Config do
     it 'should have the standard worker' do
       subject.worker.should == MicroQ::Worker::Standard
     end
+
+    it 'should have the default statistics' do
+      subject.statistics.should == MicroQ::Statistics::Default
+    end
   end
 end