howler 1.0.0

data/.gemrc

@@ -0,0 +1 @@
+gem: --no-rdoc --no-ri

data/.gitignore

@@ -0,0 +1,5 @@
+.bundle
+.idea
+**/*.DS_Store
+.DS_Store
+*.rdb

data/.rspec

@@ -0,0 +1,2 @@
+--format nested
+--color

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3-p125@howler --create

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/Gemfile.lock

@@ -0,0 +1,68 @@
+PATH
+  remote: .
+  specs:
+    howler (1.0.0)
+      celluloid
+      connection_pool
+      multi_json
+      redis
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    capybara (1.1.2)
+      mime-types (>= 1.16)
+      nokogiri (>= 1.3.3)
+      rack (>= 1.0.0)
+      rack-test (>= 0.5.4)
+      selenium-webdriver (~> 2.0)
+      xpath (~> 0.1.4)
+    celluloid (0.10.0)
+    childprocess (0.3.1)
+      ffi (~> 1.0.6)
+    connection_pool (0.9.1)
+    diff-lcs (1.1.3)
+    ffi (1.0.11)
+    mime-types (1.18)
+    multi_json (1.2.0)
+    nokogiri (1.5.2)
+    rack (1.4.1)
+    rack-protection (1.2.0)
+      rack
+    rack-test (0.6.1)
+      rack (>= 1.0)
+    rake (0.9.2.2)
+    redis (2.2.2)
+    rspec (2.9.0)
+      rspec-core (~> 2.9.0)
+      rspec-expectations (~> 2.9.0)
+      rspec-mocks (~> 2.9.0)
+    rspec-core (2.9.0)
+    rspec-expectations (2.9.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.9.0)
+    rubyzip (0.9.6.1)
+    selenium-webdriver (2.20.0)
+      childprocess (>= 0.2.5)
+      ffi (~> 1.0)
+      multi_json (~> 1.0)
+      rubyzip
+    sinatra (1.3.2)
+      rack (~> 1.3, >= 1.3.6)
+      rack-protection (~> 1.2)
+      tilt (~> 1.3, >= 1.3.3)
+    tilt (1.3.3)
+    timecop (0.3.5)
+    xpath (0.1.4)
+      nokogiri (~> 1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  capybara
+  howler!
+  rake
+  rspec
+  sinatra
+  timecop

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Brian Norton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,104 @@
+# Howler
+###An asynchronous message queue that's always chewing on something.
+--------------------
+
+#####Advantages
+- On-the-fly Configuration (via `Howler::Config`)
+- Simple message queueing interface.
+- Powerful and Fine-grained message retry logic.
+- Dashboard for managing and tracking message processing.
+- No need for an external Exception Notification Service.
+- Simple Message passing between Actors
+
+--------------------
+
+###Usage
+0. Rails 3 + Redis
+1. `gem 'howler'` in your Gemfile.
+2. `bundle install`.
+3. From the root of the Rails project run `[bundle exec] howler`.
+
+
+####Configuration
+
+Howler listens to configuration and updates accordingly.
+
+```ruby
+  # Scale Workers
+  Howler::Config[:concurrency] = 75
+```
+
+####Queueing Interface
+
+```ruby
+class User [< ActiveRecord::Base]
+  async :fetch_content
+
+  def self.fetch_content(user_id)
+    ...
+  end
+end
+
+User.async_fetch_content(user.id)
+#=> true
+```
+
+####Message Retry Handling
+- Retry a message every minute for up to 10 minutes
+
+```ruby
+  def self.fetch_content(user_id)
+    user = User.find(user_id)
+
+    unless user.fetchable?
+      raise Howler::Message::Retry(:after => 1.minute, :ttl => 10.minutes)
+    end
+
+    ... # fetch content
+  end
+```
+
+####Exception Notification
+- Notify when an external API is down
+
+```ruby
+  def self.fetch_content(user_id)
+    ...
+    begin
+      # Try to fetch /home_timeline for the user
+    rescue Twitter::Error::ServiceUnavailable => error
+      raise Howler::Message::Notify.new(erorr)
+    end
+    
+    ... # process the timeline
+  end
+```
+
+####Message Passing
+- Pass messages by setting values into the shared configuration (key, value).
+
+```ruby
+  def fetch_content(user_id)
+
+    ... # done fetching content
+    Howler::Config[user_id] = {:fetched_at => Time.now, :status => 'success'}.to_json
+
+    # Then to delete a key simply assign nil
+    Howler::Config[user_id] = nil
+  end
+```
+
+####Dashboard (In Development)
+--------------------
+- Global settings management.
+- Change the default message retry handling.
+- Increase or Decrease the number of workers.
+- Explicitly retry, delete, or reschedule messages
+- Change the log-level (seeing higher error rates, so switch to the debug level)
+
+#####Get rid of your Exception Notifier
+--------------------
+- Simply raise a `Howler::Message::Notify` exception
+- Raise with custom attributes and `Howler` will take care of the rest.
+- The Notifications tab will give you access to errors in real-time.
+

data/bin/howler

@@ -0,0 +1,3 @@
+require_relative '../lib/howler'
+
+Howler::Runner.new.run

data/howler.gemspec

@@ -0,0 +1,24 @@
+require File.expand_path('../lib/howler/support/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Brian Norton"]
+  gem.email         = ["brian.nort@gmail.com"]
+  gem.description   = gem.summary = "An Asynchronous Message Queue that's always Chewing on Something"
+  gem.homepage      = "http://github.com/bnorton/howler"
+
+  gem.executables   = ["howler"]
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- spec/*`.split("\n")
+  gem.name          = "howler"
+  gem.require_paths = ["lib"]
+  gem.version       = Howler::VERSION
+  gem.add_dependency              'redis'
+  gem.add_dependency              'celluloid'
+  gem.add_dependency              'connection_pool'
+  gem.add_dependency              'multi_json'
+  gem.add_development_dependency  'rake'
+  gem.add_development_dependency  'sinatra'
+  gem.add_development_dependency  'rspec'
+  gem.add_development_dependency  'capybara'
+  gem.add_development_dependency  'timecop'
+end

data/lib/howler.rb

@@ -0,0 +1,34 @@
+require 'redis'
+require 'connection_pool'
+
+module Howler
+  def self.next(id)
+    redis.with {|redis| redis.hincrby("next", id.to_s, 1) }.to_i
+  end
+  def self.args(args)
+    args.to_s.gsub(/^\[|\]$/, '')
+  end
+  def self.redis
+    @connection ||= ConnectionPool.new(:timeout => 1, :size => 5) { _redis }
+  end
+  private
+  def self._redis
+    @redis ||= ::Redis.connect(:url => 'redis://localhost:6379/0')
+  end
+end
+
+require_relative 'howler/support/config'
+require_relative 'howler/support/util'
+require_relative 'howler/support/logger'
+
+require_relative 'howler/message'
+require_relative 'howler/queue'
+require_relative 'howler/worker'
+require_relative 'howler/manager'
+require_relative 'howler/runner'
+
+require_relative 'howler/async'
+
+require_relative 'howler/exceptions'
+
+Object.extend Howler::Async

data/lib/howler/async.rb

@@ -0,0 +1,15 @@
+module Howler
+  module Async
+    def async(*methods)
+      methods = methods.to_a.flatten.compact.map(&:to_s)
+
+      class_eval do
+        methods.each do |method|
+          define_singleton_method :"async_#{method}" do |*args|
+            Howler::Manager.current.push(self.to_s, method.to_sym, args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/howler/config.ru

@@ -0,0 +1,4 @@
+require File.expand_path('../web', __FILE__)
+require 'multi_json'
+
+run Howler::Web

data/lib/howler/exceptions.rb

@@ -0,0 +1,5 @@
+require_relative 'exceptions/error'
+require_relative 'exceptions/retry'
+require_relative 'exceptions/failed'
+
+require_relative 'exceptions/notify'

data/lib/howler/exceptions/error.rb

@@ -0,0 +1,25 @@
+module Howler
+  class Message::Error < Exception
+    attr_accessor :message
+
+    def initialize(options = {})
+      @info = {}
+
+      options.each do |key, value|
+        @info[key.to_s] = value
+      end
+    end
+
+    def info
+      @info.merge!(
+        'backtrace' => self.backtrace[0..7]
+      ) if self.backtrace && !@info['backtrace']
+
+      @info.merge!(
+        'message' => self.message
+      ) unless @info['message']
+
+      @info
+    end
+  end
+end

data/lib/howler/exceptions/failed.rb

@@ -0,0 +1,9 @@
+module Howler
+  class Message::Failed < Message::Error
+
+    def initialize(*)
+      super
+      @message ||= "Message Failed at " + Howler::Util.now
+    end
+  end
+end

data/lib/howler/exceptions/notify.rb

@@ -0,0 +1,15 @@
+module Howler
+  class Message::Notify < Message::Error
+    attr_reader :cause, :env
+
+    def initialize(cause, options = {})
+      super(options)
+
+      @cause = cause
+      @env = {
+        'hostname' => `hostname`.chomp,
+        'ruby_version' => `ruby -v`.chomp
+      }
+    end
+  end
+end

data/lib/howler/exceptions/retry.rb

@@ -0,0 +1,12 @@
+module Howler
+  class Message::Retry < Howler::Message::Error
+    attr_accessor :at, :ttl
+
+    def initialize(options = {})
+      @at = options[:at]
+      @at ||= Time.now.utc + (options[:after] || 300)
+      @ttl = (Time.now.utc + options[:ttl]) if options[:ttl]
+      @ttl ||= 0
+    end
+  end
+end

data/lib/howler/manager.rb

@@ -0,0 +1,123 @@
+require 'celluloid'
+require 'multi_json'
+
+module Howler
+  class Manager
+    include Celluloid
+
+    trap_exit :worker_death
+
+    DEFAULT = "pending:default"
+
+    attr_reader :workers, :chewing
+
+    def self.current
+      @current ||= Howler::Manager.new
+    end
+
+    def initialize
+      @done = false
+      @logger = Howler::Logger.new
+      @options = {}
+      @workers = []
+      @chewing = []
+    end
+
+    def shutdown
+      @done = true
+      current_size = @workers.size
+      @workers = []
+      current_size
+    end
+
+    def run
+      @workers = build_workers
+
+      loop do
+        break if done?
+        scale_workers
+
+        messages, range_messages = [], []
+
+        Howler.redis.with do |redis|
+          range_messages = redis.zrange(DEFAULT, 0, @workers.size - 1) if @workers.size > 0
+          messages = redis.zrangebyscore(DEFAULT, '-inf', Time.now.to_f)
+
+          if messages.size >= @workers.size
+            messages = range_messages
+          end
+
+          redis.zremrangebyrank(DEFAULT, 0, messages.size - 1) unless messages.size == 0
+        end
+
+        @logger.log do |log|
+          log.info("Processing #{messages.size} Messages")
+
+          sleep(1) unless messages.any?
+
+          messages.each do |message|
+            message = Howler::Message.new(MultiJson.decode(message))
+            log.debug("MESG - #{message.id} #{message.klass}.new.#{message.method}(#{Howler.args(message.args)})")
+
+            worker = begin_chewing
+            worker.perform!(message, Howler::Queue::DEFAULT)
+          end
+        end
+      end
+    end
+
+    def push(klass, method, args, wait_until = Time.now)
+      queue = Howler::Queue.new(DEFAULT)
+
+      message = {
+        :id => Howler.next(:id),
+        :class => klass.to_s,
+        :method => method,
+        :args => args,
+        :created_at => Time.now.to_f
+      }
+
+      queue.push(message, wait_until)
+    end
+
+    def done?
+      @done
+    end
+
+    def done_chewing(worker)
+      worker = @chewing.delete(worker)
+      @workers.push(worker) if worker.alive?
+      nil
+    end
+
+    def worker_death(actor=nil, reason=nil)
+      @chewing.delete(actor)
+      @workers.push Howler::Worker.new_link
+    end
+
+    private
+
+    def begin_chewing
+      worker = @workers.pop
+      @chewing.push worker
+      worker
+    end
+
+    def build_workers
+      Howler::Config[:concurrency].to_i.times.collect do
+        Howler::Worker.new_link
+      end
+    end
+
+    def scale_workers
+      delta = ((@workers.size + @chewing.size) - Howler::Config[:concurrency].to_i)
+      return if delta == 0
+
+      if delta > 0
+        [@workers.size, delta].min.times { @workers.pop }
+      elsif delta < 0
+        delta.abs.times { @workers.push Howler::Worker.new }
+      end
+    end
+  end
+end