howler 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/.gemrc +1 -0
  2. data/.gitignore +5 -0
  3. data/.rspec +2 -0
  4. data/.rvmrc +1 -0
  5. data/Gemfile +2 -0
  6. data/Gemfile.lock +68 -0
  7. data/LICENSE +19 -0
  8. data/README.md +104 -0
  9. data/bin/howler +3 -0
  10. data/howler.gemspec +24 -0
  11. data/lib/howler.rb +34 -0
  12. data/lib/howler/async.rb +15 -0
  13. data/lib/howler/config.ru +4 -0
  14. data/lib/howler/exceptions.rb +5 -0
  15. data/lib/howler/exceptions/error.rb +25 -0
  16. data/lib/howler/exceptions/failed.rb +9 -0
  17. data/lib/howler/exceptions/notify.rb +15 -0
  18. data/lib/howler/exceptions/retry.rb +12 -0
  19. data/lib/howler/manager.rb +123 -0
  20. data/lib/howler/message.rb +15 -0
  21. data/lib/howler/queue.rb +138 -0
  22. data/lib/howler/runner.rb +34 -0
  23. data/lib/howler/support/config.rb +33 -0
  24. data/lib/howler/support/logger.rb +57 -0
  25. data/lib/howler/support/util.rb +23 -0
  26. data/lib/howler/support/version.rb +3 -0
  27. data/lib/howler/web.rb +47 -0
  28. data/lib/howler/web/public/application.css +24 -0
  29. data/lib/howler/web/public/bootstrap.css +3990 -0
  30. data/lib/howler/web/public/bootstrap.min.css +689 -0
  31. data/lib/howler/web/public/queues.css +19 -0
  32. data/lib/howler/web/views/failed_messages.erb +27 -0
  33. data/lib/howler/web/views/html.erb +10 -0
  34. data/lib/howler/web/views/index.erb +11 -0
  35. data/lib/howler/web/views/navigation.erb +25 -0
  36. data/lib/howler/web/views/notification_messages.erb +24 -0
  37. data/lib/howler/web/views/notifications.erb +15 -0
  38. data/lib/howler/web/views/pending_messages.erb +24 -0
  39. data/lib/howler/web/views/processed_messages.erb +28 -0
  40. data/lib/howler/web/views/queue.erb +36 -0
  41. data/lib/howler/web/views/queue_table.erb +27 -0
  42. data/lib/howler/web/views/queues.erb +15 -0
  43. data/lib/howler/worker.rb +17 -0
  44. data/spec/models/async_spec.rb +76 -0
  45. data/spec/models/exceptions/failed_spec.rb +15 -0
  46. data/spec/models/exceptions/message_spec.rb +53 -0
  47. data/spec/models/exceptions/notify_spec.rb +26 -0
  48. data/spec/models/exceptions/retry_spec.rb +49 -0
  49. data/spec/models/howler_spec.rb +69 -0
  50. data/spec/models/manager_spec.rb +397 -0
  51. data/spec/models/message_spec.rb +78 -0
  52. data/spec/models/queue_spec.rb +539 -0
  53. data/spec/models/runner_spec.rb +109 -0
  54. data/spec/models/support/config_spec.rb +56 -0
  55. data/spec/models/support/logger_spec.rb +147 -0
  56. data/spec/models/support/util_spec.rb +44 -0
  57. data/spec/models/worker_spec.rb +54 -0
  58. data/spec/requests/web_spec.rb +220 -0
  59. data/spec/spec_helper.rb +93 -0
  60. metadata +265 -0
data/.gemrc ADDED
@@ -0,0 +1 @@
1
+ gem: --no-rdoc --no-ri
@@ -0,0 +1,5 @@
1
+ .bundle
2
+ .idea
3
+ **/*.DS_Store
4
+ .DS_Store
5
+ *.rdb
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format nested
2
+ --color
data/.rvmrc ADDED
@@ -0,0 +1 @@
1
+ rvm use 1.9.3-p125@howler --create
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source :rubygems
2
+ gemspec
@@ -0,0 +1,68 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ howler (1.0.0)
5
+ celluloid
6
+ connection_pool
7
+ multi_json
8
+ redis
9
+
10
+ GEM
11
+ remote: http://rubygems.org/
12
+ specs:
13
+ capybara (1.1.2)
14
+ mime-types (>= 1.16)
15
+ nokogiri (>= 1.3.3)
16
+ rack (>= 1.0.0)
17
+ rack-test (>= 0.5.4)
18
+ selenium-webdriver (~> 2.0)
19
+ xpath (~> 0.1.4)
20
+ celluloid (0.10.0)
21
+ childprocess (0.3.1)
22
+ ffi (~> 1.0.6)
23
+ connection_pool (0.9.1)
24
+ diff-lcs (1.1.3)
25
+ ffi (1.0.11)
26
+ mime-types (1.18)
27
+ multi_json (1.2.0)
28
+ nokogiri (1.5.2)
29
+ rack (1.4.1)
30
+ rack-protection (1.2.0)
31
+ rack
32
+ rack-test (0.6.1)
33
+ rack (>= 1.0)
34
+ rake (0.9.2.2)
35
+ redis (2.2.2)
36
+ rspec (2.9.0)
37
+ rspec-core (~> 2.9.0)
38
+ rspec-expectations (~> 2.9.0)
39
+ rspec-mocks (~> 2.9.0)
40
+ rspec-core (2.9.0)
41
+ rspec-expectations (2.9.1)
42
+ diff-lcs (~> 1.1.3)
43
+ rspec-mocks (2.9.0)
44
+ rubyzip (0.9.6.1)
45
+ selenium-webdriver (2.20.0)
46
+ childprocess (>= 0.2.5)
47
+ ffi (~> 1.0)
48
+ multi_json (~> 1.0)
49
+ rubyzip
50
+ sinatra (1.3.2)
51
+ rack (~> 1.3, >= 1.3.6)
52
+ rack-protection (~> 1.2)
53
+ tilt (~> 1.3, >= 1.3.3)
54
+ tilt (1.3.3)
55
+ timecop (0.3.5)
56
+ xpath (0.1.4)
57
+ nokogiri (~> 1.3)
58
+
59
+ PLATFORMS
60
+ ruby
61
+
62
+ DEPENDENCIES
63
+ capybara
64
+ howler!
65
+ rake
66
+ rspec
67
+ sinatra
68
+ timecop
data/LICENSE ADDED
@@ -0,0 +1,19 @@
1
+ Copyright (C) 2012 Brian Norton
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
4
+ this software and associated documentation files (the "Software"), to deal in
5
+ the Software without restriction, including without limitation the rights to
6
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
7
+ of the Software, and to permit persons to whom the Software is furnished to do
8
+ so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in all
11
+ copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19
+ SOFTWARE.
@@ -0,0 +1,104 @@
1
+ # Howler
2
+ ###An asynchronous message queue that's always chewing on something.
3
+ --------------------
4
+
5
+ #####Advantages
6
+ - On-the-fly Configuration (via `Howler::Config`)
7
+ - Simple message queueing interface.
8
+ - Powerful and Fine-grained message retry logic.
9
+ - Dashboard for managing and tracking message processing.
10
+ - No need for an external Exception Notification Service.
11
+ - Simple Message passing between Actors
12
+
13
+ --------------------
14
+
15
+ ###Usage
16
+ 0. Rails 3 + Redis
17
+ 1. `gem 'howler'` in your Gemfile.
18
+ 2. `bundle install`.
19
+ 3. From the root of the Rails project run `[bundle exec] howler`.
20
+
21
+
22
+ ####Configuration
23
+
24
+ Howler listens to configuration and updates accordingly.
25
+
26
+ ```ruby
27
+ # Scale Workers
28
+ Howler::Config[:concurrency] = 75
29
+ ```
30
+
31
+ ####Queueing Interface
32
+
33
+ ```ruby
34
+ class User [< ActiveRecord::Base]
35
+ async :fetch_content
36
+
37
+ def self.fetch_content(user_id)
38
+ ...
39
+ end
40
+ end
41
+
42
+ User.async_fetch_content(user.id)
43
+ #=> true
44
+ ```
45
+
46
+ ####Message Retry Handling
47
+ - Retry a message every minute for up to 10 minutes
48
+
49
+ ```ruby
50
+ def self.fetch_content(user_id)
51
+ user = User.find(user_id)
52
+
53
+ unless user.fetchable?
54
+ raise Howler::Message::Retry(:after => 1.minute, :ttl => 10.minutes)
55
+ end
56
+
57
+ ... # fetch content
58
+ end
59
+ ```
60
+
61
+ ####Exception Notification
62
+ - Notify when an external API is down
63
+
64
+ ```ruby
65
+ def self.fetch_content(user_id)
66
+ ...
67
+ begin
68
+ # Try to fetch /home_timeline for the user
69
+ rescue Twitter::Error::ServiceUnavailable => error
70
+ raise Howler::Message::Notify.new(erorr)
71
+ end
72
+
73
+ ... # process the timeline
74
+ end
75
+ ```
76
+
77
+ ####Message Passing
78
+ - Pass messages by setting values into the shared configuration (key, value).
79
+
80
+ ```ruby
81
+ def fetch_content(user_id)
82
+
83
+ ... # done fetching content
84
+ Howler::Config[user_id] = {:fetched_at => Time.now, :status => 'success'}.to_json
85
+
86
+ # Then to delete a key simply assign nil
87
+ Howler::Config[user_id] = nil
88
+ end
89
+ ```
90
+
91
+ ####Dashboard (In Development)
92
+ --------------------
93
+ - Global settings management.
94
+ - Change the default message retry handling.
95
+ - Increase or Decrease the number of workers.
96
+ - Explicitly retry, delete, or reschedule messages
97
+ - Change the log-level (seeing higher error rates, so switch to the debug level)
98
+
99
+ #####Get rid of your Exception Notifier
100
+ --------------------
101
+ - Simply raise a `Howler::Message::Notify` exception
102
+ - Raise with custom attributes and `Howler` will take care of the rest.
103
+ - The Notifications tab will give you access to errors in real-time.
104
+
@@ -0,0 +1,3 @@
1
+ require_relative '../lib/howler'
2
+
3
+ Howler::Runner.new.run
@@ -0,0 +1,24 @@
1
+ require File.expand_path('../lib/howler/support/version', __FILE__)
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.authors = ["Brian Norton"]
5
+ gem.email = ["brian.nort@gmail.com"]
6
+ gem.description = gem.summary = "An Asynchronous Message Queue that's always Chewing on Something"
7
+ gem.homepage = "http://github.com/bnorton/howler"
8
+
9
+ gem.executables = ["howler"]
10
+ gem.files = `git ls-files`.split("\n")
11
+ gem.test_files = `git ls-files -- spec/*`.split("\n")
12
+ gem.name = "howler"
13
+ gem.require_paths = ["lib"]
14
+ gem.version = Howler::VERSION
15
+ gem.add_dependency 'redis'
16
+ gem.add_dependency 'celluloid'
17
+ gem.add_dependency 'connection_pool'
18
+ gem.add_dependency 'multi_json'
19
+ gem.add_development_dependency 'rake'
20
+ gem.add_development_dependency 'sinatra'
21
+ gem.add_development_dependency 'rspec'
22
+ gem.add_development_dependency 'capybara'
23
+ gem.add_development_dependency 'timecop'
24
+ end
@@ -0,0 +1,34 @@
1
+ require 'redis'
2
+ require 'connection_pool'
3
+
4
+ module Howler
5
+ def self.next(id)
6
+ redis.with {|redis| redis.hincrby("next", id.to_s, 1) }.to_i
7
+ end
8
+ def self.args(args)
9
+ args.to_s.gsub(/^\[|\]$/, '')
10
+ end
11
+ def self.redis
12
+ @connection ||= ConnectionPool.new(:timeout => 1, :size => 5) { _redis }
13
+ end
14
+ private
15
+ def self._redis
16
+ @redis ||= ::Redis.connect(:url => 'redis://localhost:6379/0')
17
+ end
18
+ end
19
+
20
+ require_relative 'howler/support/config'
21
+ require_relative 'howler/support/util'
22
+ require_relative 'howler/support/logger'
23
+
24
+ require_relative 'howler/message'
25
+ require_relative 'howler/queue'
26
+ require_relative 'howler/worker'
27
+ require_relative 'howler/manager'
28
+ require_relative 'howler/runner'
29
+
30
+ require_relative 'howler/async'
31
+
32
+ require_relative 'howler/exceptions'
33
+
34
+ Object.extend Howler::Async
@@ -0,0 +1,15 @@
1
+ module Howler
2
+ module Async
3
+ def async(*methods)
4
+ methods = methods.to_a.flatten.compact.map(&:to_s)
5
+
6
+ class_eval do
7
+ methods.each do |method|
8
+ define_singleton_method :"async_#{method}" do |*args|
9
+ Howler::Manager.current.push(self.to_s, method.to_sym, args)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,4 @@
1
+ require File.expand_path('../web', __FILE__)
2
+ require 'multi_json'
3
+
4
+ run Howler::Web
@@ -0,0 +1,5 @@
1
+ require_relative 'exceptions/error'
2
+ require_relative 'exceptions/retry'
3
+ require_relative 'exceptions/failed'
4
+
5
+ require_relative 'exceptions/notify'
@@ -0,0 +1,25 @@
1
+ module Howler
2
+ class Message::Error < Exception
3
+ attr_accessor :message
4
+
5
+ def initialize(options = {})
6
+ @info = {}
7
+
8
+ options.each do |key, value|
9
+ @info[key.to_s] = value
10
+ end
11
+ end
12
+
13
+ def info
14
+ @info.merge!(
15
+ 'backtrace' => self.backtrace[0..7]
16
+ ) if self.backtrace && !@info['backtrace']
17
+
18
+ @info.merge!(
19
+ 'message' => self.message
20
+ ) unless @info['message']
21
+
22
+ @info
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,9 @@
1
+ module Howler
2
+ class Message::Failed < Message::Error
3
+
4
+ def initialize(*)
5
+ super
6
+ @message ||= "Message Failed at " + Howler::Util.now
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,15 @@
1
+ module Howler
2
+ class Message::Notify < Message::Error
3
+ attr_reader :cause, :env
4
+
5
+ def initialize(cause, options = {})
6
+ super(options)
7
+
8
+ @cause = cause
9
+ @env = {
10
+ 'hostname' => `hostname`.chomp,
11
+ 'ruby_version' => `ruby -v`.chomp
12
+ }
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,12 @@
1
+ module Howler
2
+ class Message::Retry < Howler::Message::Error
3
+ attr_accessor :at, :ttl
4
+
5
+ def initialize(options = {})
6
+ @at = options[:at]
7
+ @at ||= Time.now.utc + (options[:after] || 300)
8
+ @ttl = (Time.now.utc + options[:ttl]) if options[:ttl]
9
+ @ttl ||= 0
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,123 @@
1
+ require 'celluloid'
2
+ require 'multi_json'
3
+
4
+ module Howler
5
+ class Manager
6
+ include Celluloid
7
+
8
+ trap_exit :worker_death
9
+
10
+ DEFAULT = "pending:default"
11
+
12
+ attr_reader :workers, :chewing
13
+
14
+ def self.current
15
+ @current ||= Howler::Manager.new
16
+ end
17
+
18
+ def initialize
19
+ @done = false
20
+ @logger = Howler::Logger.new
21
+ @options = {}
22
+ @workers = []
23
+ @chewing = []
24
+ end
25
+
26
+ def shutdown
27
+ @done = true
28
+ current_size = @workers.size
29
+ @workers = []
30
+ current_size
31
+ end
32
+
33
+ def run
34
+ @workers = build_workers
35
+
36
+ loop do
37
+ break if done?
38
+ scale_workers
39
+
40
+ messages, range_messages = [], []
41
+
42
+ Howler.redis.with do |redis|
43
+ range_messages = redis.zrange(DEFAULT, 0, @workers.size - 1) if @workers.size > 0
44
+ messages = redis.zrangebyscore(DEFAULT, '-inf', Time.now.to_f)
45
+
46
+ if messages.size >= @workers.size
47
+ messages = range_messages
48
+ end
49
+
50
+ redis.zremrangebyrank(DEFAULT, 0, messages.size - 1) unless messages.size == 0
51
+ end
52
+
53
+ @logger.log do |log|
54
+ log.info("Processing #{messages.size} Messages")
55
+
56
+ sleep(1) unless messages.any?
57
+
58
+ messages.each do |message|
59
+ message = Howler::Message.new(MultiJson.decode(message))
60
+ log.debug("MESG - #{message.id} #{message.klass}.new.#{message.method}(#{Howler.args(message.args)})")
61
+
62
+ worker = begin_chewing
63
+ worker.perform!(message, Howler::Queue::DEFAULT)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ def push(klass, method, args, wait_until = Time.now)
70
+ queue = Howler::Queue.new(DEFAULT)
71
+
72
+ message = {
73
+ :id => Howler.next(:id),
74
+ :class => klass.to_s,
75
+ :method => method,
76
+ :args => args,
77
+ :created_at => Time.now.to_f
78
+ }
79
+
80
+ queue.push(message, wait_until)
81
+ end
82
+
83
+ def done?
84
+ @done
85
+ end
86
+
87
+ def done_chewing(worker)
88
+ worker = @chewing.delete(worker)
89
+ @workers.push(worker) if worker.alive?
90
+ nil
91
+ end
92
+
93
+ def worker_death(actor=nil, reason=nil)
94
+ @chewing.delete(actor)
95
+ @workers.push Howler::Worker.new_link
96
+ end
97
+
98
+ private
99
+
100
+ def begin_chewing
101
+ worker = @workers.pop
102
+ @chewing.push worker
103
+ worker
104
+ end
105
+
106
+ def build_workers
107
+ Howler::Config[:concurrency].to_i.times.collect do
108
+ Howler::Worker.new_link
109
+ end
110
+ end
111
+
112
+ def scale_workers
113
+ delta = ((@workers.size + @chewing.size) - Howler::Config[:concurrency].to_i)
114
+ return if delta == 0
115
+
116
+ if delta > 0
117
+ [@workers.size, delta].min.times { @workers.pop }
118
+ elsif delta < 0
119
+ delta.abs.times { @workers.push Howler::Worker.new }
120
+ end
121
+ end
122
+ end
123
+ end