howler 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gemrc +1 -0
- data/.gitignore +5 -0
- data/.rspec +2 -0
- data/.rvmrc +1 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +68 -0
- data/LICENSE +19 -0
- data/README.md +104 -0
- data/bin/howler +3 -0
- data/howler.gemspec +24 -0
- data/lib/howler.rb +34 -0
- data/lib/howler/async.rb +15 -0
- data/lib/howler/config.ru +4 -0
- data/lib/howler/exceptions.rb +5 -0
- data/lib/howler/exceptions/error.rb +25 -0
- data/lib/howler/exceptions/failed.rb +9 -0
- data/lib/howler/exceptions/notify.rb +15 -0
- data/lib/howler/exceptions/retry.rb +12 -0
- data/lib/howler/manager.rb +123 -0
- data/lib/howler/message.rb +15 -0
- data/lib/howler/queue.rb +138 -0
- data/lib/howler/runner.rb +34 -0
- data/lib/howler/support/config.rb +33 -0
- data/lib/howler/support/logger.rb +57 -0
- data/lib/howler/support/util.rb +23 -0
- data/lib/howler/support/version.rb +3 -0
- data/lib/howler/web.rb +47 -0
- data/lib/howler/web/public/application.css +24 -0
- data/lib/howler/web/public/bootstrap.css +3990 -0
- data/lib/howler/web/public/bootstrap.min.css +689 -0
- data/lib/howler/web/public/queues.css +19 -0
- data/lib/howler/web/views/failed_messages.erb +27 -0
- data/lib/howler/web/views/html.erb +10 -0
- data/lib/howler/web/views/index.erb +11 -0
- data/lib/howler/web/views/navigation.erb +25 -0
- data/lib/howler/web/views/notification_messages.erb +24 -0
- data/lib/howler/web/views/notifications.erb +15 -0
- data/lib/howler/web/views/pending_messages.erb +24 -0
- data/lib/howler/web/views/processed_messages.erb +28 -0
- data/lib/howler/web/views/queue.erb +36 -0
- data/lib/howler/web/views/queue_table.erb +27 -0
- data/lib/howler/web/views/queues.erb +15 -0
- data/lib/howler/worker.rb +17 -0
- data/spec/models/async_spec.rb +76 -0
- data/spec/models/exceptions/failed_spec.rb +15 -0
- data/spec/models/exceptions/message_spec.rb +53 -0
- data/spec/models/exceptions/notify_spec.rb +26 -0
- data/spec/models/exceptions/retry_spec.rb +49 -0
- data/spec/models/howler_spec.rb +69 -0
- data/spec/models/manager_spec.rb +397 -0
- data/spec/models/message_spec.rb +78 -0
- data/spec/models/queue_spec.rb +539 -0
- data/spec/models/runner_spec.rb +109 -0
- data/spec/models/support/config_spec.rb +56 -0
- data/spec/models/support/logger_spec.rb +147 -0
- data/spec/models/support/util_spec.rb +44 -0
- data/spec/models/worker_spec.rb +54 -0
- data/spec/requests/web_spec.rb +220 -0
- data/spec/spec_helper.rb +93 -0
- metadata +265 -0
data/.gemrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
gem: --no-rdoc --no-ri
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use 1.9.3-p125@howler --create
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
howler (1.0.0)
|
5
|
+
celluloid
|
6
|
+
connection_pool
|
7
|
+
multi_json
|
8
|
+
redis
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: http://rubygems.org/
|
12
|
+
specs:
|
13
|
+
capybara (1.1.2)
|
14
|
+
mime-types (>= 1.16)
|
15
|
+
nokogiri (>= 1.3.3)
|
16
|
+
rack (>= 1.0.0)
|
17
|
+
rack-test (>= 0.5.4)
|
18
|
+
selenium-webdriver (~> 2.0)
|
19
|
+
xpath (~> 0.1.4)
|
20
|
+
celluloid (0.10.0)
|
21
|
+
childprocess (0.3.1)
|
22
|
+
ffi (~> 1.0.6)
|
23
|
+
connection_pool (0.9.1)
|
24
|
+
diff-lcs (1.1.3)
|
25
|
+
ffi (1.0.11)
|
26
|
+
mime-types (1.18)
|
27
|
+
multi_json (1.2.0)
|
28
|
+
nokogiri (1.5.2)
|
29
|
+
rack (1.4.1)
|
30
|
+
rack-protection (1.2.0)
|
31
|
+
rack
|
32
|
+
rack-test (0.6.1)
|
33
|
+
rack (>= 1.0)
|
34
|
+
rake (0.9.2.2)
|
35
|
+
redis (2.2.2)
|
36
|
+
rspec (2.9.0)
|
37
|
+
rspec-core (~> 2.9.0)
|
38
|
+
rspec-expectations (~> 2.9.0)
|
39
|
+
rspec-mocks (~> 2.9.0)
|
40
|
+
rspec-core (2.9.0)
|
41
|
+
rspec-expectations (2.9.1)
|
42
|
+
diff-lcs (~> 1.1.3)
|
43
|
+
rspec-mocks (2.9.0)
|
44
|
+
rubyzip (0.9.6.1)
|
45
|
+
selenium-webdriver (2.20.0)
|
46
|
+
childprocess (>= 0.2.5)
|
47
|
+
ffi (~> 1.0)
|
48
|
+
multi_json (~> 1.0)
|
49
|
+
rubyzip
|
50
|
+
sinatra (1.3.2)
|
51
|
+
rack (~> 1.3, >= 1.3.6)
|
52
|
+
rack-protection (~> 1.2)
|
53
|
+
tilt (~> 1.3, >= 1.3.3)
|
54
|
+
tilt (1.3.3)
|
55
|
+
timecop (0.3.5)
|
56
|
+
xpath (0.1.4)
|
57
|
+
nokogiri (~> 1.3)
|
58
|
+
|
59
|
+
PLATFORMS
|
60
|
+
ruby
|
61
|
+
|
62
|
+
DEPENDENCIES
|
63
|
+
capybara
|
64
|
+
howler!
|
65
|
+
rake
|
66
|
+
rspec
|
67
|
+
sinatra
|
68
|
+
timecop
|
data/LICENSE
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Copyright (C) 2012 Brian Norton
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
4
|
+
this software and associated documentation files (the "Software"), to deal in
|
5
|
+
the Software without restriction, including without limitation the rights to
|
6
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
7
|
+
of the Software, and to permit persons to whom the Software is furnished to do
|
8
|
+
so, subject to the following conditions:
|
9
|
+
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
11
|
+
copies or substantial portions of the Software.
|
12
|
+
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
19
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
# Howler
|
2
|
+
###An asynchronous message queue that's always chewing on something.
|
3
|
+
--------------------
|
4
|
+
|
5
|
+
#####Advantages
|
6
|
+
- On-the-fly Configuration (via `Howler::Config`)
|
7
|
+
- Simple message queueing interface.
|
8
|
+
- Powerful and Fine-grained message retry logic.
|
9
|
+
- Dashboard for managing and tracking message processing.
|
10
|
+
- No need for an external Exception Notification Service.
|
11
|
+
- Simple Message passing between Actors
|
12
|
+
|
13
|
+
--------------------
|
14
|
+
|
15
|
+
###Usage
|
16
|
+
0. Rails 3 + Redis
|
17
|
+
1. `gem 'howler'` in your Gemfile.
|
18
|
+
2. `bundle install`.
|
19
|
+
3. From the root of the Rails project run `[bundle exec] howler`.
|
20
|
+
|
21
|
+
|
22
|
+
####Configuration
|
23
|
+
|
24
|
+
Howler listens to configuration and updates accordingly.
|
25
|
+
|
26
|
+
```ruby
|
27
|
+
# Scale Workers
|
28
|
+
Howler::Config[:concurrency] = 75
|
29
|
+
```
|
30
|
+
|
31
|
+
####Queueing Interface
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
class User [< ActiveRecord::Base]
|
35
|
+
async :fetch_content
|
36
|
+
|
37
|
+
def self.fetch_content(user_id)
|
38
|
+
...
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
User.async_fetch_content(user.id)
|
43
|
+
#=> true
|
44
|
+
```
|
45
|
+
|
46
|
+
####Message Retry Handling
|
47
|
+
- Retry a message every minute for up to 10 minutes
|
48
|
+
|
49
|
+
```ruby
|
50
|
+
def self.fetch_content(user_id)
|
51
|
+
user = User.find(user_id)
|
52
|
+
|
53
|
+
unless user.fetchable?
|
54
|
+
raise Howler::Message::Retry(:after => 1.minute, :ttl => 10.minutes)
|
55
|
+
end
|
56
|
+
|
57
|
+
... # fetch content
|
58
|
+
end
|
59
|
+
```
|
60
|
+
|
61
|
+
####Exception Notification
|
62
|
+
- Notify when an external API is down
|
63
|
+
|
64
|
+
```ruby
|
65
|
+
def self.fetch_content(user_id)
|
66
|
+
...
|
67
|
+
begin
|
68
|
+
# Try to fetch /home_timeline for the user
|
69
|
+
rescue Twitter::Error::ServiceUnavailable => error
|
70
|
+
raise Howler::Message::Notify.new(erorr)
|
71
|
+
end
|
72
|
+
|
73
|
+
... # process the timeline
|
74
|
+
end
|
75
|
+
```
|
76
|
+
|
77
|
+
####Message Passing
|
78
|
+
- Pass messages by setting values into the shared configuration (key, value).
|
79
|
+
|
80
|
+
```ruby
|
81
|
+
def fetch_content(user_id)
|
82
|
+
|
83
|
+
... # done fetching content
|
84
|
+
Howler::Config[user_id] = {:fetched_at => Time.now, :status => 'success'}.to_json
|
85
|
+
|
86
|
+
# Then to delete a key simply assign nil
|
87
|
+
Howler::Config[user_id] = nil
|
88
|
+
end
|
89
|
+
```
|
90
|
+
|
91
|
+
####Dashboard (In Development)
|
92
|
+
--------------------
|
93
|
+
- Global settings management.
|
94
|
+
- Change the default message retry handling.
|
95
|
+
- Increase or Decrease the number of workers.
|
96
|
+
- Explicitly retry, delete, or reschedule messages
|
97
|
+
- Change the log-level (seeing higher error rates, so switch to the debug level)
|
98
|
+
|
99
|
+
#####Get rid of your Exception Notifier
|
100
|
+
--------------------
|
101
|
+
- Simply raise a `Howler::Message::Notify` exception
|
102
|
+
- Raise with custom attributes and `Howler` will take care of the rest.
|
103
|
+
- The Notifications tab will give you access to errors in real-time.
|
104
|
+
|
data/bin/howler
ADDED
data/howler.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path('../lib/howler/support/version', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.authors = ["Brian Norton"]
|
5
|
+
gem.email = ["brian.nort@gmail.com"]
|
6
|
+
gem.description = gem.summary = "An Asynchronous Message Queue that's always Chewing on Something"
|
7
|
+
gem.homepage = "http://github.com/bnorton/howler"
|
8
|
+
|
9
|
+
gem.executables = ["howler"]
|
10
|
+
gem.files = `git ls-files`.split("\n")
|
11
|
+
gem.test_files = `git ls-files -- spec/*`.split("\n")
|
12
|
+
gem.name = "howler"
|
13
|
+
gem.require_paths = ["lib"]
|
14
|
+
gem.version = Howler::VERSION
|
15
|
+
gem.add_dependency 'redis'
|
16
|
+
gem.add_dependency 'celluloid'
|
17
|
+
gem.add_dependency 'connection_pool'
|
18
|
+
gem.add_dependency 'multi_json'
|
19
|
+
gem.add_development_dependency 'rake'
|
20
|
+
gem.add_development_dependency 'sinatra'
|
21
|
+
gem.add_development_dependency 'rspec'
|
22
|
+
gem.add_development_dependency 'capybara'
|
23
|
+
gem.add_development_dependency 'timecop'
|
24
|
+
end
|
data/lib/howler.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'redis'
|
2
|
+
require 'connection_pool'
|
3
|
+
|
4
|
+
module Howler
|
5
|
+
def self.next(id)
|
6
|
+
redis.with {|redis| redis.hincrby("next", id.to_s, 1) }.to_i
|
7
|
+
end
|
8
|
+
def self.args(args)
|
9
|
+
args.to_s.gsub(/^\[|\]$/, '')
|
10
|
+
end
|
11
|
+
def self.redis
|
12
|
+
@connection ||= ConnectionPool.new(:timeout => 1, :size => 5) { _redis }
|
13
|
+
end
|
14
|
+
private
|
15
|
+
def self._redis
|
16
|
+
@redis ||= ::Redis.connect(:url => 'redis://localhost:6379/0')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
require_relative 'howler/support/config'
|
21
|
+
require_relative 'howler/support/util'
|
22
|
+
require_relative 'howler/support/logger'
|
23
|
+
|
24
|
+
require_relative 'howler/message'
|
25
|
+
require_relative 'howler/queue'
|
26
|
+
require_relative 'howler/worker'
|
27
|
+
require_relative 'howler/manager'
|
28
|
+
require_relative 'howler/runner'
|
29
|
+
|
30
|
+
require_relative 'howler/async'
|
31
|
+
|
32
|
+
require_relative 'howler/exceptions'
|
33
|
+
|
34
|
+
Object.extend Howler::Async
|
data/lib/howler/async.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
module Howler
|
2
|
+
module Async
|
3
|
+
def async(*methods)
|
4
|
+
methods = methods.to_a.flatten.compact.map(&:to_s)
|
5
|
+
|
6
|
+
class_eval do
|
7
|
+
methods.each do |method|
|
8
|
+
define_singleton_method :"async_#{method}" do |*args|
|
9
|
+
Howler::Manager.current.push(self.to_s, method.to_sym, args)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Howler
|
2
|
+
class Message::Error < Exception
|
3
|
+
attr_accessor :message
|
4
|
+
|
5
|
+
def initialize(options = {})
|
6
|
+
@info = {}
|
7
|
+
|
8
|
+
options.each do |key, value|
|
9
|
+
@info[key.to_s] = value
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def info
|
14
|
+
@info.merge!(
|
15
|
+
'backtrace' => self.backtrace[0..7]
|
16
|
+
) if self.backtrace && !@info['backtrace']
|
17
|
+
|
18
|
+
@info.merge!(
|
19
|
+
'message' => self.message
|
20
|
+
) unless @info['message']
|
21
|
+
|
22
|
+
@info
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Howler
|
2
|
+
class Message::Notify < Message::Error
|
3
|
+
attr_reader :cause, :env
|
4
|
+
|
5
|
+
def initialize(cause, options = {})
|
6
|
+
super(options)
|
7
|
+
|
8
|
+
@cause = cause
|
9
|
+
@env = {
|
10
|
+
'hostname' => `hostname`.chomp,
|
11
|
+
'ruby_version' => `ruby -v`.chomp
|
12
|
+
}
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Howler
|
2
|
+
class Message::Retry < Howler::Message::Error
|
3
|
+
attr_accessor :at, :ttl
|
4
|
+
|
5
|
+
def initialize(options = {})
|
6
|
+
@at = options[:at]
|
7
|
+
@at ||= Time.now.utc + (options[:after] || 300)
|
8
|
+
@ttl = (Time.now.utc + options[:ttl]) if options[:ttl]
|
9
|
+
@ttl ||= 0
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
require 'multi_json'
|
3
|
+
|
4
|
+
module Howler
|
5
|
+
class Manager
|
6
|
+
include Celluloid
|
7
|
+
|
8
|
+
trap_exit :worker_death
|
9
|
+
|
10
|
+
DEFAULT = "pending:default"
|
11
|
+
|
12
|
+
attr_reader :workers, :chewing
|
13
|
+
|
14
|
+
def self.current
|
15
|
+
@current ||= Howler::Manager.new
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@done = false
|
20
|
+
@logger = Howler::Logger.new
|
21
|
+
@options = {}
|
22
|
+
@workers = []
|
23
|
+
@chewing = []
|
24
|
+
end
|
25
|
+
|
26
|
+
def shutdown
|
27
|
+
@done = true
|
28
|
+
current_size = @workers.size
|
29
|
+
@workers = []
|
30
|
+
current_size
|
31
|
+
end
|
32
|
+
|
33
|
+
def run
|
34
|
+
@workers = build_workers
|
35
|
+
|
36
|
+
loop do
|
37
|
+
break if done?
|
38
|
+
scale_workers
|
39
|
+
|
40
|
+
messages, range_messages = [], []
|
41
|
+
|
42
|
+
Howler.redis.with do |redis|
|
43
|
+
range_messages = redis.zrange(DEFAULT, 0, @workers.size - 1) if @workers.size > 0
|
44
|
+
messages = redis.zrangebyscore(DEFAULT, '-inf', Time.now.to_f)
|
45
|
+
|
46
|
+
if messages.size >= @workers.size
|
47
|
+
messages = range_messages
|
48
|
+
end
|
49
|
+
|
50
|
+
redis.zremrangebyrank(DEFAULT, 0, messages.size - 1) unless messages.size == 0
|
51
|
+
end
|
52
|
+
|
53
|
+
@logger.log do |log|
|
54
|
+
log.info("Processing #{messages.size} Messages")
|
55
|
+
|
56
|
+
sleep(1) unless messages.any?
|
57
|
+
|
58
|
+
messages.each do |message|
|
59
|
+
message = Howler::Message.new(MultiJson.decode(message))
|
60
|
+
log.debug("MESG - #{message.id} #{message.klass}.new.#{message.method}(#{Howler.args(message.args)})")
|
61
|
+
|
62
|
+
worker = begin_chewing
|
63
|
+
worker.perform!(message, Howler::Queue::DEFAULT)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def push(klass, method, args, wait_until = Time.now)
|
70
|
+
queue = Howler::Queue.new(DEFAULT)
|
71
|
+
|
72
|
+
message = {
|
73
|
+
:id => Howler.next(:id),
|
74
|
+
:class => klass.to_s,
|
75
|
+
:method => method,
|
76
|
+
:args => args,
|
77
|
+
:created_at => Time.now.to_f
|
78
|
+
}
|
79
|
+
|
80
|
+
queue.push(message, wait_until)
|
81
|
+
end
|
82
|
+
|
83
|
+
def done?
|
84
|
+
@done
|
85
|
+
end
|
86
|
+
|
87
|
+
def done_chewing(worker)
|
88
|
+
worker = @chewing.delete(worker)
|
89
|
+
@workers.push(worker) if worker.alive?
|
90
|
+
nil
|
91
|
+
end
|
92
|
+
|
93
|
+
def worker_death(actor=nil, reason=nil)
|
94
|
+
@chewing.delete(actor)
|
95
|
+
@workers.push Howler::Worker.new_link
|
96
|
+
end
|
97
|
+
|
98
|
+
private
|
99
|
+
|
100
|
+
def begin_chewing
|
101
|
+
worker = @workers.pop
|
102
|
+
@chewing.push worker
|
103
|
+
worker
|
104
|
+
end
|
105
|
+
|
106
|
+
def build_workers
|
107
|
+
Howler::Config[:concurrency].to_i.times.collect do
|
108
|
+
Howler::Worker.new_link
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def scale_workers
|
113
|
+
delta = ((@workers.size + @chewing.size) - Howler::Config[:concurrency].to_i)
|
114
|
+
return if delta == 0
|
115
|
+
|
116
|
+
if delta > 0
|
117
|
+
[@workers.size, delta].min.times { @workers.pop }
|
118
|
+
elsif delta < 0
|
119
|
+
delta.abs.times { @workers.push Howler::Worker.new }
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|