micro_q 0.6.1

data/.gitignore

@@ -0,0 +1,13 @@
+*.gem
+*.rbc
+.idea
+.idea/*
+.bundle
+.config
+*.db
+Gemfile.lock
+doc/
+lib/bundler/man
+pkg
+tmp
+log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - rbx-19mode
+  - 2.0.0
+branches:
+  only:
+    - master
+notifications:
+  email:
+    recipients:
+      - brian.nort@gmail.com
+matrix:
+  allow_failures:
+    - rvm: rbx-19mode
+    - rvm: 2.0.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Brian Norton
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,27 @@
+# MicroQ
+
+MicroQ is a per-process asynchronous background queue.
+
+It's simple startup and intuitive interface makes it the best choice for new and lagacy apps.
+
+## Installation
+
+Add this line to your Gemfile:
+
+    gem 'micro_q'
+
+    $ bundle
+
+Or install it:
+
+    $ gem install micro_q
+
+## Usage
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/micro_q/config.rb

@@ -0,0 +1,27 @@
+module MicroQ
+  class Config
+    def initialize
+      @data = {
+        'workers' => 3,
+        'timeout' => 120,
+        'interval' => 5,
+        'middleware' => MicroQ::Middleware::Chain.new
+      }
+    end
+
+    def []=(key, value)
+      @data[key.to_s] = value
+    end
+
+    def [](key)
+      @data[key.to_s]
+    end
+
+    def method_missing(method, *args)
+      case method
+        when /(.+)=$/ then @data[$1] = args.first
+        else @data[method.to_s]
+      end
+    end
+  end
+end

data/lib/micro_q/manager/default.rb

@@ -0,0 +1,38 @@
+module MicroQ
+  module Manager
+    ##
+    # The default manager implementation.
+    # Wrapper for a Queue and a pool of Workers. At each time slice
+    # after start! was called, try to dequeue messages from the queue.
+    # Perform each message on the worker pool.
+    #
+    # The pool of workers (more info):
+    #   https://github.com/celluloid/celluloid/wiki/Pools
+    #
+    # The pool manages asynchronously assigning messages to available
+    # workers, handles exceptions by restarting the dead actors and
+    # is generally a beautiful abstraction on top of a group of linked
+    # actors/threads.
+    #
+    class Default
+      include Celluloid
+
+      attr_reader :queue, :workers
+
+      def initialize
+        @queue   = MicroQ::Queue::Default.new
+        @workers = MicroQ::Worker::Standard.pool(:size => MicroQ.config.workers)
+      end
+
+      def start
+        if (messages = queue.dequeue).any?
+          messages.each do |message|
+            workers.perform!(message)
+          end
+        end
+
+        after(5) { start }
+      end
+    end
+  end
+end

data/lib/micro_q/manager.rb

@@ -0,0 +1 @@
+require 'micro_q/manager/default'

data/lib/micro_q/methods/active_record.rb

@@ -0,0 +1,21 @@
+module MicroQ
+  module Methods
+    module ActiveRecord
+      def async
+        options = {
+          :class => self.class,
+          :loader => {
+            :method => 'find',
+            :args => [id]
+          }
+        }
+
+        MicroQ::Proxy::Instance.new(options)
+      end
+    end
+  end
+end
+
+if defined?(ActiveRecord::Base)
+  ActiveRecord::Base.send(:include, MicroQ::Methods::ActiveRecord)
+end

data/lib/micro_q/methods/class.rb

@@ -0,0 +1,13 @@
+module MicroQ
+  module Methods
+    module Class
+      extend MicroQ::Methods::SharedMethods
+
+      def async
+        MicroQ::Proxy::Class.new(:class => self, :loader => {})
+      end
+    end
+  end
+end
+
+Object.send(:extend,  MicroQ::Methods::Class)

data/lib/micro_q/methods/instance.rb

@@ -0,0 +1,13 @@
+module MicroQ
+  module Methods
+    module Instance
+      include MicroQ::Methods::SharedMethods
+
+      def async
+        MicroQ::Proxy::Instance.new(:class => self.class)
+      end
+    end
+  end
+end
+
+Object.send(:include, MicroQ::Methods::Instance)

data/lib/micro_q/methods.rb

@@ -0,0 +1,25 @@
+module MicroQ
+  module Methods
+    module SharedMethods
+      def method_missing(method, *other)
+        super unless /((.+)\_async$)/ === method
+
+        name = $2 && $2.to_sym
+
+        # Define the method and call through.
+        if name && respond_to?(name)
+          define_singleton_method method do |*args|
+            async.send(name, *args)
+          end
+
+          async.send(name, *other)
+        else
+          super
+        end
+      end
+    end
+  end
+end
+
+require 'micro_q/methods/class'
+require 'micro_q/methods/instance'

data/lib/micro_q/middleware/chain.rb

@@ -0,0 +1,102 @@
+require 'micro_q/middleware/server/retry'
+
+module MicroQ
+  module Middleware
+    ##
+    # An Array wrapper that holds the class name of middlewares to call
+    # around the execution of messages. The most basic middleware must
+    # yield to the given block to allow the message to be invoked. Not
+    # yielding causes the message to be dropped and not invoked.
+    #
+    # A minimal middleware:
+    # class MyFunMiddleware
+    #   def call(worker, message)
+    #     # Do something fun here ...
+    #     yield
+    #     # More fun goes here ...
+    #   end
+    # end
+    #
+    class Chain
+      ##
+      # Middleware chain that is run around execution of messages.
+      #   -- If halted, the message will not be invoked.
+      def server
+        @server ||= Server.new
+      end
+
+      ##
+      # Middleware chain that is run around message push.
+      #   -- If halted, the message will not enter the queue.
+      #
+      def client
+        @client ||= Client.new
+      end
+
+      class Base
+        attr_reader :entries
+
+        def initialize
+          clear
+        end
+
+        ##
+        # Add any number of entries to the middleware chain
+        #
+        def add(*items)
+          @entries.concat(items.flatten).uniq!
+        end
+
+        def add_before(before, *items)
+          remove(*items)
+          @entries.insert(@entries.index(before), *items).uniq! if items.any?
+        end
+
+        def add_after(after, *items)
+          remove(*items)
+          @entries.insert(@entries.index(after)+1, *items).uniq! if items.any?
+        end
+
+        ##
+        # Remove any number of entries from the middleware chain
+        #
+        def remove(*items)
+          @entries.tap do
+            items.flatten.each {|item| @entries.delete(item) }
+          end
+        end
+
+        def clear
+          @entries = []
+        end
+
+        ##
+        # Traverse the middleware chain by recursing until we reach the
+        # end of the chain and are able to invoke the given block. The block
+        # represents a message push (client) or a message invocation (server).
+        #
+        # It is, however very generic and can be used as middleware around anything.
+        #
+        def call(*args, &block)
+          chain = (index = -1) && -> {
+            (index += 1) == entries.length ?
+              block.call : entries.at(index).new.call(*args, &chain)
+          }
+
+          chain.call
+        end
+      end
+
+      class Server < Base
+        def initialize
+          @entries = [
+            MicroQ::Middleware::Server::Retry
+          ]
+        end
+      end
+
+      class Client < Base
+      end
+    end
+  end
+end

data/lib/micro_q/middleware/server/retry.rb

@@ -0,0 +1,32 @@
+module MicroQ
+  module Middleware
+    module Server
+      ##
+      # Capture, re-raise and potentially push a modified message
+      # back onto the queue. We add metadata about the retry into the
+      # 'retried' key to track the attempts.
+      #
+      # count: The number of retires thus far
+      # at:    When the last retry occurred
+      # when:  The time at which the message will be retried again
+      #
+      class Retry
+        def call(worker, message)
+          yield
+        rescue Exception => e
+          raise e unless message['retry']
+
+          message['retried'] ||= { 'count' => 0 }
+
+          message['retried']['count'] += 1
+          message['retried']['at']    = Time.now
+          message['retried']['when']  = (Time.now + 15).to_f
+
+          MicroQ.push(message, message['retried'])
+
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/middleware.rb

@@ -0,0 +1 @@
+require 'micro_q/middleware/chain'

data/lib/micro_q/proxies/base.rb

@@ -0,0 +1,49 @@
+module MicroQ
+  module Proxy
+    class Base
+      attr_reader :errors, :klass, :method, :args, :at
+
+      def initialize(options={})
+        @errors  = []
+        @options = options
+
+        parse_and_validate
+      end
+
+      def valid?
+        errors.empty?
+      end
+
+      def method_missing(meth, *args)
+        @method = meth.to_s
+        @args = args
+
+        defaults = [@options.merge(
+          :class => klass,
+          :method => method,
+          :args => args
+        )]
+
+        defaults << { :when => at } if at
+
+        MicroQ.push(*defaults)
+      end
+
+      def respond_to?(method)
+        super || klass.respond_to?(method)
+      end
+
+      private
+
+      def parse_and_validate
+        @at    = (at = @options.fetch(:at, nil)) && at.to_i
+        after  = @options.fetch(:after, nil)
+
+        @at    = Time.now.to_i + after if after
+        @klass = @options[:class] && MicroQ::Util.constantize(@options[:class].to_s)
+
+        (errors << 'Proxies require a valid class')  unless klass
+      end
+    end
+  end
+end

data/lib/micro_q/proxies/class.rb

@@ -0,0 +1,6 @@
+module MicroQ
+  module Proxy
+    class Class < Base
+    end
+  end
+end

data/lib/micro_q/proxies/instance.rb

@@ -0,0 +1,9 @@
+module MicroQ
+  module Proxy
+    class Instance < Base
+      def respond_to?(method)
+        super || klass.new.respond_to?(method)
+      end
+    end
+  end
+end

data/lib/micro_q/proxies.rb

@@ -0,0 +1,3 @@
+require 'micro_q/proxies/base'
+require 'micro_q/proxies/class'
+require 'micro_q/proxies/instance'

data/lib/micro_q/queue/default.rb

@@ -0,0 +1,90 @@
+module MicroQ
+  module Queue
+    ##
+    # The default queue implementation.
+    # Handles messages that should be run immediately as well as messages that
+    # should be run at some specified time in the future.
+    #
+    # Usage:
+    #
+    # item = { 'class' => 'MyWorker', 'args' => [user.id] }
+    #
+    # queue = MicroQ::Queue::Default.new
+    # queue.push(item)       # synchronous push
+    # queue.async.push(item) # asynchronous push (preferred)
+    #
+    # queue.entries
+    # #=> [{'class' => 'MyWorker', 'args' => [32]}]
+    #
+    # queue.push(item, :when => 15.minutes.from_now)
+    #
+    # queue.later
+    # [{'when' => 1359703628.38, 'worker' => {'class' => 'MyWorker', 'args' => 32}}]
+    #
+    class Default
+      include Celluloid
+
+      attr_reader :entries, :later
+
+      def initialize
+        @entries = []
+        @later   = []
+      end
+
+      ##
+      # Push a message item to the queue.
+      # Either push it to the immediate portion of the queue or store it for after when
+      # it should be run with the 'when' option.
+      #
+      # Options:
+      #   when: The time/timestamp after which to run the message.
+      #
+      def push(item, options={})
+        item, options = before_push(item, options)
+
+        if (time = options['when'])
+          @later.push(
+            'when' => time.to_f,
+            'worker' => item
+          )
+        else
+          @entries.push(item)
+        end
+      end
+
+      ##
+      # Remove and return all available messages.
+      #
+      def dequeue
+        [].tap do |items|
+          entries.each do |entry|
+            items << entry
+          end if entries.any?
+
+          items.each {|i| entries.delete(i) }
+
+          available = later.select {|entry| entry['when'] < Time.now.to_f }
+
+          if available.any?
+            available.each do |entry|
+              items << entry['worker']
+            end
+
+            available.each {|a| later.delete(a) }
+          end
+        end
+      end
+
+      private
+
+      ##
+      # Duplicate the given items and stringify the keys.
+      #
+      def before_push(args, options)
+        [MicroQ::Util.stringify_keys(args),
+         MicroQ::Util.stringify_keys(options)
+        ]
+      end
+    end
+  end
+end

data/lib/micro_q/queue.rb

@@ -0,0 +1 @@
+require 'micro_q/queue/default'

data/lib/micro_q/util.rb

@@ -0,0 +1,29 @@
+module MicroQ
+  module Util
+    ##
+    # Stolen from active_support/inflector/inflections with a rescue to nil.
+    #
+    def self.constantize(word)
+      names = word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name| # Compatible with Ruby 1.9 and above. Before 1.9 the arity of #const_defined? was 1.
+        constant = constant.const_defined?(name, false) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    rescue
+      nil
+    end
+
+    def self.stringify_keys(hash)
+      {}.tap do |result|
+        hash.keys.each do |key|
+          value = hash[key]
+
+          result[key.to_s] = value.is_a?(Hash) ? stringify_keys(value) : value
+        end
+      end
+    end
+  end
+end

data/lib/micro_q/version.rb

@@ -0,0 +1,7 @@
+module MicroQ
+  MAJOR = 0
+  MINOR = 6
+  POINT = 1
+
+  VERSION = [MAJOR, MINOR, POINT].join('.')
+end

data/lib/micro_q/worker/standard.rb

@@ -0,0 +1,40 @@
+module MicroQ
+  module Worker
+    ##
+    # The default worker implementation.
+    # this worker can call any method on an class instance and pass
+    # an arbitrary argument list. By default it calls the 'class'.constantize#'perform'
+    # method. It returns the result of the method call if possible (for debugging).
+    #
+    # The middleware chain can stop this message from executing by not yielding
+    # to the given block.
+    #
+    # A minimal message: (Calls the perform method with zero arguments)
+    # { :class => 'MyWorker' }
+    #
+    # A more complex message: (Calls the update_data with a single paramater as a list of ids)
+    # { :class => 'MyUpdater', 'method' => 'update_data', :args => [[2, 6,74, 198]]}
+    #
+    class Standard
+      include Celluloid
+
+      def perform(message)
+        klass = MicroQ::Util.constantize(message['class'].to_s)
+
+        loader = message['loader'] ||= { 'method' => 'new' }
+        klass = klass.send(loader['method'], *loader['args']) if loader['method']
+
+        method = message['method'] || 'perform'
+        args = message['args']
+
+        value = nil
+
+        MicroQ.middleware.server.call(klass, message) do
+          value = klass.send(method, *args)
+        end
+
+        value
+      end
+    end
+  end
+end

data/lib/micro_q/worker.rb

@@ -0,0 +1 @@
+require 'micro_q/worker/standard'

data/lib/micro_q.rb

@@ -0,0 +1,46 @@
+require 'celluloid'
+require 'micro_q/util'
+require 'micro_q/config'
+require 'micro_q/manager'
+
+module MicroQ
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure
+    yield config
+  end
+
+  def self.middleware
+    config.middleware
+  end
+
+  def self.start
+    manager
+  end
+
+  def self.push(*args)
+    manager.queue.async.push(*args)
+  end
+
+  private
+
+  def self.manager
+    @manager ||= begin
+      Manager::Default.new.tap do |manager|
+        manager.start!
+      end
+    end
+  end
+
+  def self.clear
+    @config = @manager = nil
+  end
+end
+
+require 'micro_q/middleware'
+require 'micro_q/methods'
+require 'micro_q/proxies'
+require 'micro_q/worker'
+require 'micro_q/queue'