action_event 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Capfile +23 -0
- data/MIT-LICENSE +20 -0
- data/README +102 -0
- data/Rakefile +23 -0
- data/action_event.gemspec +13 -0
- data/generators/event/event_generator.rb +18 -0
- data/generators/event/templates/application_event.rb +2 -0
- data/generators/event/templates/event.rb +4 -0
- data/generators/event/templates/migration.rb +22 -0
- data/generators/event/templates/poller +4 -0
- data/init.rb +1 -0
- data/lib/action_event.rb +5 -0
- data/lib/action_event/base.rb +77 -0
- data/lib/action_event/commands/poller.rb +237 -0
- data/lib/action_event/message.rb +39 -0
- data/lib/action_event/object_extensions.rb +6 -0
- data/test/action_event_test.rb +99 -0
- data/test/test_helper.rb +3 -0
- metadata +71 -0
data/Capfile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# after 'deploy:start', 'deploy:appmessaging:start'
|
2
|
+
# after 'deploy:stop', 'deploy:appmessaging:stop'
|
3
|
+
# after 'deploy:restart', 'deploy:appmessaging:restart'
|
4
|
+
# after 'deploy:spinner', 'deploy:appmessaging:start'
|
5
|
+
#
|
6
|
+
# namespace :deploy do
|
7
|
+
# namespace :appmessaging do
|
8
|
+
# desc "app messaging restart"
|
9
|
+
# task :restart, :roles => :messaging do
|
10
|
+
# sudo "/usr/sbin/monit -g appmessaging restart all"
|
11
|
+
# end
|
12
|
+
#
|
13
|
+
# desc "app messaging stop"
|
14
|
+
# task :stop, :roles => :messaging do
|
15
|
+
# sudo "/usr/sbin/monit -g appmessaging stop all"
|
16
|
+
# end
|
17
|
+
#
|
18
|
+
# desc "app messaging start"
|
19
|
+
# task :start, :roles => :messaging do
|
20
|
+
# sudo "/usr/sbin/monit -g appmessaging start all"
|
21
|
+
# end
|
22
|
+
# end
|
23
|
+
# end
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Warren Konkel
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
Preface
|
2
|
+
=======
|
3
|
+
|
4
|
+
This is a project that is currently used in production environments but isn't "production ready" unless
|
5
|
+
you're adventurous. Consider this as a "sneak peak" rather than a "first release".
|
6
|
+
|
7
|
+
Another thing to note... this has gone through many iterations (MySQL backed, memcached backed,
|
8
|
+
rabbitMQ, etc). At some point, I plan on adding back in support for all of these and allowing users
|
9
|
+
to choose which queue backend they want, but right now it's tied directly to rabbitmq.
|
10
|
+
|
11
|
+
|
12
|
+
Summary
|
13
|
+
=======
|
14
|
+
|
15
|
+
ActionEvent lets you "do something later". Many things can be taken out of the front-end flow and
|
16
|
+
moved to an asynchronous task, speeding up the overall user experience. LivingSocial has over a hundred
|
17
|
+
event pollers spread out on five machines processing millions of events a day.
|
18
|
+
|
19
|
+
ActionEvent allows you to:
|
20
|
+
|
21
|
+
* Queue an event from within a rails application to be processed asynchronously.
|
22
|
+
* Access your full rails environment from event processor.
|
23
|
+
* Process events across any number of physical machines (there is no master process).
|
24
|
+
* Start/stop processors on the fly without interrupting anything (think: cloud computing).
|
25
|
+
* Events support inheritance and a simple before_filter/skip_before_filter chain as well.
|
26
|
+
|
27
|
+
Queueing an Event
|
28
|
+
=================
|
29
|
+
|
30
|
+
You can queue up as many events you want from within a rails application using the "queue_action_event"
|
31
|
+
method available on all objects. This takes an event name and an optional hash of parameters. The name
|
32
|
+
and parameters are serialized and stored as an active record object. You can also prioritize events (default
|
33
|
+
priorities are :high, :medium and :low). Pollers will always look for events from the highest priority
|
34
|
+
first, so all :high messages will be processed before a single :medium message is processed.
|
35
|
+
|
36
|
+
Queueing examples:
|
37
|
+
|
38
|
+
queue_action_event(:some_event)
|
39
|
+
queue_action_event(:some_event, :people_ids => [1,2,3])
|
40
|
+
queue_action_event(:send_email, :queue => :high)
|
41
|
+
|
42
|
+
|
43
|
+
Pollers: Processing an Event
|
44
|
+
============================
|
45
|
+
|
46
|
+
You need to have one or more "pollers" which will process events as they come in the queue. Pollers will
|
47
|
+
continuously query the database trying to get another event to process. If it finds one, it will try to
|
48
|
+
take "ownership" of the event so that no other pollers will process the event. If this fails (meaning
|
49
|
+
another poller already took the message), it moves on. If it succeeds and takes ownership of the message,
|
50
|
+
it will process the event through the corresponding app/events/*_event.rb class.
|
51
|
+
|
52
|
+
If you have multiple pollers (likely), you can start and stop them as a daemon. Each poller needs a unique
|
53
|
+
ID so it can keep track of their PID file and stop and start gracefully. After processing every message,
|
54
|
+
a daemonized poller will check their PID file to see if it matches their current process id. If it doesn't
|
55
|
+
match, the poller will stop gracefully. This allows you to start up a new poller on updated code without
|
56
|
+
waiting for or interrupting an existing poller if it happens to be in the middle of processing a message.
|
57
|
+
|
58
|
+
Poller examples:
|
59
|
+
|
60
|
+
./script/poller
|
61
|
+
./script/poller -d start
|
62
|
+
./script/poller --daemon --id=2 start
|
63
|
+
./script/poller --daemon --id=2 --queues="high medium" start
|
64
|
+
|
65
|
+
|
66
|
+
Example
|
67
|
+
=======
|
68
|
+
|
69
|
+
$ ./script/generate event HelloWorld
|
70
|
+
exists db/migrate
|
71
|
+
create db/migrate/20090207172934_create_action_event_tables.rb
|
72
|
+
create script/poller
|
73
|
+
create app/events
|
74
|
+
create app/events/application_event.rb
|
75
|
+
exists app/events/
|
76
|
+
create app/events/hello_world_event.rb
|
77
|
+
|
78
|
+
# NOTE: running this migration is only necessary the very first time you create an action_event
|
79
|
+
$ rake db:migrate
|
80
|
+
== CreateActionEventTables: migrating ========================================
|
81
|
+
-- create_table(:action_event_messages)
|
82
|
+
-> 0.0844s
|
83
|
+
-- create_table(:action_event_statuses)
|
84
|
+
-> 0.0359s
|
85
|
+
== CreateActionEventTables: migrated (0.1212s) ===============================
|
86
|
+
|
87
|
+
$ cat > app/events/hello_world_event.rb
|
88
|
+
class HelloWorldEvent < ApplicationEvent
|
89
|
+
def process
|
90
|
+
puts "hello #{params[:name]}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
$ ./script/runner "queue_action_event('hello_world', :name => 'world')"
|
95
|
+
|
96
|
+
$ ./script/poller
|
97
|
+
[Sat Feb 07 12:41:43 -0500 2009] Processing queues: high,medium,low
|
98
|
+
[Sat Feb 07 12:44:31 -0500 2009] Processing medium:1 ({:name=>"world", :event=>"hello_world"})
|
99
|
+
hello world
|
100
|
+
[Sat Feb 07 12:44:31 -0500 2009] Finished processing medium:1 ({:name=>"world", :event=>"hello_world"})
|
101
|
+
|
102
|
+
Copyright (c) 2009 Warren Konkel, released under the MIT license
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'rake/rdoctask'
|
4
|
+
|
5
|
+
desc 'Default: run unit tests.'
|
6
|
+
task :default => :test
|
7
|
+
|
8
|
+
desc 'Test the action_event plugin.'
|
9
|
+
Rake::TestTask.new(:test) do |t|
|
10
|
+
t.libs << 'lib'
|
11
|
+
t.libs << 'test'
|
12
|
+
t.pattern = 'test/**/*_test.rb'
|
13
|
+
t.verbose = true
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Generate documentation for the action_event plugin.'
|
17
|
+
Rake::RDocTask.new(:rdoc) do |rdoc|
|
18
|
+
rdoc.rdoc_dir = 'rdoc'
|
19
|
+
rdoc.title = 'ActionEvent'
|
20
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
21
|
+
rdoc.rdoc_files.include('README')
|
22
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
23
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = "action_event"
|
3
|
+
s.version = "0.0.1"
|
4
|
+
s.date = "2009-10-06"
|
5
|
+
s.summary = "A framework for asynchronous message processing in a Rails application."
|
6
|
+
s.email = "wkonkel@gmail.com"
|
7
|
+
s.homepage = "http://github.com/wkonkel/action_event"
|
8
|
+
s.description = "A framework for asynchronous message processing in a Rails application."
|
9
|
+
s.has_rdoc = false
|
10
|
+
s.authors = ["Warren Konkel"]
|
11
|
+
s.files = Dir.glob('**/*') - Dir.glob('test/*.rb')
|
12
|
+
s.test_files = Dir.glob('test/*.rb')
|
13
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class EventGenerator < Rails::Generator::NamedBase
|
2
|
+
def manifest
|
3
|
+
record do |m|
|
4
|
+
# if there's not an app/events directory, do an initial install
|
5
|
+
unless File.exists?("#{RAILS_ROOT}/app/events")
|
6
|
+
#m.migration_template 'migration.rb', "db/migrate", { :migration_file_name => "create_action_event_tables" }
|
7
|
+
m.file 'poller', 'script/poller', :chmod => 0755
|
8
|
+
m.directory 'app/events'
|
9
|
+
m.file 'application_event.rb', 'app/events/application_event.rb'
|
10
|
+
end
|
11
|
+
|
12
|
+
# generate this event
|
13
|
+
m.class_collisions "#{class_name}Event"
|
14
|
+
m.directory File.join('app/events', class_path)
|
15
|
+
m.template 'event.rb', File.join('app/events', class_path, "#{file_name}_event.rb")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class CreateActionEventTables < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
ActionEvent::Message.connection.create_table(:action_event_messages) do |t|
|
4
|
+
t.string :event, :null => false
|
5
|
+
t.text :params
|
6
|
+
t.timestamps
|
7
|
+
end
|
8
|
+
|
9
|
+
ActionEvent::Message.connection.create_table(:action_event_statuses) do |t|
|
10
|
+
t.integer :last_processed_message_id, :null => false
|
11
|
+
t.string :table_name, :null => false
|
12
|
+
t.timestamps
|
13
|
+
end
|
14
|
+
|
15
|
+
ActionEvent::Message.connection.add_index :action_event_statuses, :table_name, :unique => true
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.down
|
19
|
+
ActionEvent::Message.connection.drop_table :action_event_messages
|
20
|
+
ActionEvent::Message.connection.drop_table :action_event_statuses
|
21
|
+
end
|
22
|
+
end
|
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'action_event'
|
data/lib/action_event.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
module ActionEvent
|
2
|
+
class Base
|
3
|
+
|
4
|
+
include ActionController::UrlWriter
|
5
|
+
|
6
|
+
attr_accessor :params
|
7
|
+
|
8
|
+
def self.view_paths
|
9
|
+
@view_paths ||= ActionView::Base.process_view_paths(Rails::Configuration.new.view_path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(params)
|
13
|
+
@params = params.clone
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.process(params={})
|
17
|
+
action_event = new(params)
|
18
|
+
filter_chain.each { |chain| action_event.send(chain) }
|
19
|
+
action_event.process
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.before_filter(method)
|
23
|
+
filter_chain.push(method.to_sym)
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.skip_before_filter(method)
|
27
|
+
filter_chain.delete(method.to_sym)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.helper(*names)
|
31
|
+
names.each do |name|
|
32
|
+
case name
|
33
|
+
when String, Symbol then helper("#{name.to_s.underscore}_helper".classify.constantize)
|
34
|
+
when Module then master_helper_module.module_eval { include name }
|
35
|
+
else raise
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
protected
|
41
|
+
|
42
|
+
def template
|
43
|
+
unless @template
|
44
|
+
@template = ActionView::Base.new(ActionEvent::Base.view_paths, {}, self)
|
45
|
+
@template.extend ApplicationHelper
|
46
|
+
@template.extend self.class.master_helper_module
|
47
|
+
@template.instance_eval { def protect_against_forgery?; false; end }
|
48
|
+
end
|
49
|
+
@template
|
50
|
+
end
|
51
|
+
|
52
|
+
def method_missing(method, *params, &block)
|
53
|
+
template.send(method, *params, &block)
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.filter_chain
|
57
|
+
unless chain = read_inheritable_attribute('filter_chain')
|
58
|
+
chain = Array.new
|
59
|
+
write_inheritable_attribute('filter_chain', chain)
|
60
|
+
end
|
61
|
+
return chain
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.controller_path
|
65
|
+
@controller_path ||= name.gsub(/Event$/, '').underscore
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.master_helper_module
|
69
|
+
unless mod = read_inheritable_attribute('master_helper_module')
|
70
|
+
mod = Module.new
|
71
|
+
write_inheritable_attribute('master_helper_module', mod)
|
72
|
+
end
|
73
|
+
return mod
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,237 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'optparse'
|
3
|
+
|
4
|
+
module ActionEvent
|
5
|
+
module Commands
|
6
|
+
class Poller
|
7
|
+
def initialize
|
8
|
+
@options = {
|
9
|
+
:id => 1,
|
10
|
+
:queues => %W(high medium low),
|
11
|
+
:command => 'start',
|
12
|
+
:environment => RAILS_ENV,
|
13
|
+
:daemon => false,
|
14
|
+
:max_load_average => 8,
|
15
|
+
:min_instances => 5,
|
16
|
+
:max_instances => 200,
|
17
|
+
:max_adjustment => 5,
|
18
|
+
:min_queue_size => 1000
|
19
|
+
}
|
20
|
+
|
21
|
+
OptionParser.new do |opts|
|
22
|
+
opts.banner = "Usage: #{$0} [options] <command>"
|
23
|
+
opts.on("-d", "--daemon", "Run as a daemon") { |v| @options[:daemon] = v }
|
24
|
+
opts.on("-i", "--id=N", Integer, "Specify ID used in PID file when running as daemon") { |v| @options[:id] = v }
|
25
|
+
opts.on("-q", "--queues='high medium low'", "Specify queue names in order") { |v| @options[:queues] = v.split(' ') }
|
26
|
+
opts.on("-e", "--environment=development", "Specify which rails environment to run in") { |v| @options[:environment] = v }
|
27
|
+
opts.separator ""
|
28
|
+
opts.separator "Cluster options:"
|
29
|
+
opts.on("-l", "--load-average=8", "Specify what load average to optimize to") { |v| @options[:max_load_average] = v }
|
30
|
+
opts.on("-m", "--min-instances=5", "Specify mimimum number of instances") { |v| @options[:min_instances] = v }
|
31
|
+
opts.on("-x", "--max-instances=200", "Specify maximum number of instances") { |v| @options[:max_instances] = v }
|
32
|
+
opts.on("-a", "--max-adjustment=5", "Specify how many the maximum amount of instances that will be adjusted") { |v| @options[:max_adjustment] = v }
|
33
|
+
opts.on("-s", "--min-queue-size=1000", "Specify how many must be in the queue to adjust instances") { |v| @options[:min_queue_size] = v }
|
34
|
+
opts.separator ""
|
35
|
+
opts.separator "Commands:"
|
36
|
+
opts.separator " start - starts up the poller"
|
37
|
+
opts.separator " stop - stops a poller currently running as a daemon"
|
38
|
+
opts.separator " status - prints the status of the queues"
|
39
|
+
opts.separator ""
|
40
|
+
opts.separator "Examples:"
|
41
|
+
opts.separator " #{$0} start (starts a poller running in the console)"
|
42
|
+
opts.separator " #{$0} -d -e production start (starts a poller running as a daemon with ID #1)"
|
43
|
+
opts.separator " #{$0} --daemon --id=5 start (starts poller with ID #5)"
|
44
|
+
opts.separator " #{$0} --daemon --id=5 stop (stops poller with ID #5)"
|
45
|
+
end.parse!
|
46
|
+
|
47
|
+
@options[:command] = ARGV.pop unless ARGV.empty?
|
48
|
+
|
49
|
+
case
|
50
|
+
when @options[:command] == 'start' && !@options[:daemon] then trap_ctrl_c and load_rails_environment and start_processing_loop
|
51
|
+
when @options[:command] == 'start' && @options[:daemon] then trap_term and start_daemon and load_rails_environment and start_processing_loop and remove_pid
|
52
|
+
when @options[:command] == 'stop' then stop_daemon
|
53
|
+
when @options[:command] == 'cluster' then load_rails_environment and refresh_cluster
|
54
|
+
when @options[:command] == 'status' then load_rails_environment and print_status
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def print_status
|
59
|
+
ActionEvent::Message.queue_status(@options[:queues]).to_a.sort { |a,b| a.first <=> b.first }.each do |table,messages_left|
|
60
|
+
log "#{table}:\t\t#{messages_left}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def load_rails_environment
|
65
|
+
ENV['ACTION_EVENT_USE_POLLER_DB'] = 'true'
|
66
|
+
ENV["RAILS_ENV"] = @options[:environment]
|
67
|
+
RAILS_ENV.replace(@options[:environment])
|
68
|
+
log "Loading #{RAILS_ENV} environment..."
|
69
|
+
require "#{RAILS_ROOT}/config/environment"
|
70
|
+
|
71
|
+
if defined?(NewRelic)
|
72
|
+
NewRelic::Control.instance.instance_eval do
|
73
|
+
@settings['app_name'] = @settings['app_name'] + ' (Poller)'
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
true
|
78
|
+
end
|
79
|
+
|
80
|
+
# returns the name of the PID file to use for daemons
|
81
|
+
def pid_filename
|
82
|
+
@pid_filename ||= File.join(RAILS_ROOT, "/log/poller.#{@options[:id]}.pid")
|
83
|
+
end
|
84
|
+
|
85
|
+
# forks from the current process and closes out everything
|
86
|
+
def start_daemon
|
87
|
+
log "Starting daemon ##{@options[:id]}..."
|
88
|
+
|
89
|
+
# some process magic
|
90
|
+
exit if fork # Parent exits, child continues.
|
91
|
+
Process.setsid # Become session leader.
|
92
|
+
exit if fork # Zap session leader.
|
93
|
+
Dir.chdir "/" # Release old working directory.
|
94
|
+
File.umask 0000 # Ensure sensible umask. Adjust as needed.
|
95
|
+
|
96
|
+
# Free file descriptors and point them somewhere sensible.
|
97
|
+
STDIN.reopen "/dev/null"
|
98
|
+
STDOUT.reopen File.join(RAILS_ROOT, "log/poller.log"), "a"
|
99
|
+
STDERR.reopen STDOUT
|
100
|
+
|
101
|
+
# don't start up until the previous poller is dead
|
102
|
+
while (previous_pid = File.read(pid_filename).to_i rescue nil) do
|
103
|
+
break unless File.exists?("/proc/#{previous_pid}")
|
104
|
+
log "Waiting for previous poller to finish..."
|
105
|
+
Process.kill('TERM', previous_pid)
|
106
|
+
sleep(5)
|
107
|
+
end
|
108
|
+
|
109
|
+
# record pid
|
110
|
+
File.open(pid_filename, 'w') { |f| f << Process.pid }
|
111
|
+
end
|
112
|
+
|
113
|
+
def trap_ctrl_c
|
114
|
+
trap("SIGINT") do
|
115
|
+
@stop_processing = true
|
116
|
+
log "Sending stop signal..."
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def trap_term
|
121
|
+
trap("SIGTERM") do
|
122
|
+
@stop_processing = true
|
123
|
+
log "Received stop signal..."
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def refresh_cluster
|
128
|
+
# gather some current stats
|
129
|
+
current_load = `uptime`.split(' ')[-3..-3][0].to_f
|
130
|
+
current_queue = ActionEvent::Message.queue_status(*@options[:queues]).to_a.map(&:last).sum
|
131
|
+
|
132
|
+
# remove stale pid files
|
133
|
+
current_pids = Dir[File.join(RAILS_ROOT, "log/poller.*.pid")]
|
134
|
+
active_pids, stale_pids = current_pids.partition { |f| (File.read("/proc/#{File.read(f).to_i}/cmdline").include?('poller') rescue false) }
|
135
|
+
stale_pids.each { |f| File.delete(f) }
|
136
|
+
|
137
|
+
# compute adjustment based on current load average and queue size
|
138
|
+
if active_pids.length > 0
|
139
|
+
current_instances = active_pids.length
|
140
|
+
needed_instances = ((current_instances*@options[:max_load_average])/current_load).floor
|
141
|
+
|
142
|
+
if needed_instances > current_instances
|
143
|
+
needed_instances = [needed_instances, current_instances + @options[:max_adjustment]].min
|
144
|
+
elsif needed_instances < current_instances && current_queue > @options[:min_queue_size]
|
145
|
+
needed_instances = [needed_instances, current_instances - @options[:max_adjustment]].max
|
146
|
+
end
|
147
|
+
else
|
148
|
+
current_instances = 0
|
149
|
+
needed_instances = @options[:min_instances]
|
150
|
+
end
|
151
|
+
|
152
|
+
needed_instances = @options[:max_instances] if needed_instances > @options[:max_instances]
|
153
|
+
needed_instances = @options[:min_instances] if needed_instances < @options[:min_instances]
|
154
|
+
|
155
|
+
|
156
|
+
# remove pids if there's too many or spawn new ones if there's not enough
|
157
|
+
if needed_instances < current_instances
|
158
|
+
active_pids.last(current_instances - needed_instances).each { |pid_file| puts "delete #{pid_file}" } #File.delete(pid_file) }
|
159
|
+
elsif needed_instances > current_instances
|
160
|
+
(needed_instances - current_instances).times do
|
161
|
+
next_id = (1..needed_instances).to_a.find { |i| !File.exists?(File.join(RAILS_ROOT, "log/poller.#{i}.pid")) }
|
162
|
+
puts "start at id #{next_id}"
|
163
|
+
# if fork
|
164
|
+
#
|
165
|
+
# end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def should_stop_processing?
|
171
|
+
@stop_processing || (@options[:daemon] && (File.read(pid_filename).to_i rescue 0) != Process.pid)
|
172
|
+
end
|
173
|
+
|
174
|
+
# finds the already running daemon and stops it...
|
175
|
+
def stop_daemon
|
176
|
+
if previous_pid = File.read(pid_filename).to_i rescue nil
|
177
|
+
log "Sending stop signal to daemon ##{@options[:id]}..."
|
178
|
+
Process.kill('TERM', previous_pid)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def remove_pid
|
183
|
+
if Process.pid == (File.read(pid_filename).to_i rescue nil)
|
184
|
+
log "Cleaning up PID file..."
|
185
|
+
FileUtils.rm(pid_filename)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
# loops until should_stop_processing? set to true... in local mode, this is never set so it will loop forever
|
190
|
+
def start_processing_loop
|
191
|
+
log "Processing queues: #{@options[:queues].join(',')}"
|
192
|
+
next_iteration or sleep(0.5) until should_stop_processing?
|
193
|
+
log "Got signal to stop... exiting."
|
194
|
+
end
|
195
|
+
|
196
|
+
# if we can get a message, process it
|
197
|
+
def next_iteration
|
198
|
+
reload_application if RAILS_ENV == 'development'
|
199
|
+
if message = ActionEvent::Message.try_to_get_next_message(@options[:queues])
|
200
|
+
begin
|
201
|
+
log_text = "#{message[:queue_name]}:#{message[:event]} (#{message[:params].inspect})"
|
202
|
+
log "Processing #{log_text}"
|
203
|
+
"#{message[:event]}_event".camelize.constantize.process(message[:params])
|
204
|
+
log "Finished processing #{log_text}"
|
205
|
+
rescue Exception => e
|
206
|
+
log "Error processing #{log_text}: #{e} #{e.backtrace.join("\n")}"
|
207
|
+
end
|
208
|
+
return true
|
209
|
+
else
|
210
|
+
# return false if we didn't get a message... makes start_processing_loop sleep(1)
|
211
|
+
return false
|
212
|
+
end
|
213
|
+
rescue Exception => e
|
214
|
+
log "Error getting next message (#{e})"
|
215
|
+
ActionEvent::Message.connection.verify! rescue log("Error verifying DB connection... sleeping 5 seconds. (#{$!})") and sleep(5)
|
216
|
+
return true
|
217
|
+
end
|
218
|
+
|
219
|
+
def reload_application
|
220
|
+
ActionController::Routing::Routes.reload
|
221
|
+
ActionController::Base.view_paths.reload! rescue nil
|
222
|
+
ActionView::Helpers::AssetTagHelper::AssetTag::Cache.clear rescue nil
|
223
|
+
|
224
|
+
ActiveRecord::Base.reset_subclasses
|
225
|
+
ActiveSupport::Dependencies.clear
|
226
|
+
ActiveRecord::Base.clear_reloadable_connections!
|
227
|
+
end
|
228
|
+
|
229
|
+
def log(message)
|
230
|
+
$stdout.puts "[#{"#{@options[:id]}:#{Process.pid} " if @options[:daemon]}#{Time.now}] #{message}"
|
231
|
+
$stdout.flush
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
ActionEvent::Commands::Poller.new
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class ActionEvent::Message
|
2
|
+
|
3
|
+
cattr_accessor :default_queue, :instance_writer => false
|
4
|
+
@@default_queue = :medium
|
5
|
+
|
6
|
+
def self.deliver(queue_name, event, params = {})
|
7
|
+
with_queue(queue_name) { |queue| queue.publish(Marshal.dump({:event => event, :params => params})) }
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.try_to_get_next_message(*queues)
|
11
|
+
queues.flatten.each do |queue_name|
|
12
|
+
if message = with_queue(queue_name) { |queue| m = queue.pop; m ? Marshal.load(m) : nil }
|
13
|
+
return { :queue_name => queue_name, :event => message[:event], :params => message[:params] }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
return nil
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.queue_status(*queues)
|
20
|
+
queues.flatten.inject({}) do |hash, queue_name|
|
21
|
+
hash[queue_name] = with_queue(queue_name) { |queue| queue.status }
|
22
|
+
hash
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
27
|
+
|
28
|
+
def self.with_queue(queue_name, &block)
|
29
|
+
queue_name = queue_name.to_s
|
30
|
+
@config ||= YAML.load(File.read(File.join(RAILS_ROOT, 'config', 'rabbitmq.yml')))[RAILS_ENV]
|
31
|
+
@queues ||= {}
|
32
|
+
yield @queues[queue_name] ||= Carrot.new(:host => @config['rabbitmq_server']).queue("#{@config['application_name']}-#{queue_name}")
|
33
|
+
rescue => e
|
34
|
+
@queues[queue_name] = nil
|
35
|
+
puts e
|
36
|
+
RAILS_DEFAULT_LOGGER.error "ERROR: #{e}"
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class ActionEventTest < ActiveSupport::TestCase
|
4
|
+
test "all objects have queue_action_event" do
|
5
|
+
assert respond_to?(:queue_action_event)
|
6
|
+
end
|
7
|
+
|
8
|
+
test "queue_action_event create a test message" do
|
9
|
+
assert_nil get_next_message
|
10
|
+
queue_action_event('test')
|
11
|
+
assert_equal get_next_message.event, 'test'
|
12
|
+
assert_nil get_next_message
|
13
|
+
end
|
14
|
+
|
15
|
+
test "params are serialized" do
|
16
|
+
params = {:a => 1, :b => [1,2,3]}
|
17
|
+
queue_action_event('test', params)
|
18
|
+
assert get_next_message.params == params
|
19
|
+
end
|
20
|
+
|
21
|
+
test "priority of queues is respected in the same day" do
|
22
|
+
queue_action_event('test_low', :queue => :low)
|
23
|
+
queue_action_event('test_medium', :queue => :medium)
|
24
|
+
queue_action_event('test_high', :queue => :high)
|
25
|
+
assert_equal get_next_message.event, 'test_high'
|
26
|
+
assert_equal get_next_message.event, 'test_medium'
|
27
|
+
assert_equal get_next_message.event, 'test_low'
|
28
|
+
end
|
29
|
+
|
30
|
+
test "yesterday is processed before today" do
|
31
|
+
override_time_for_queues(1.day) { queue_action_event('test_yesterday') }
|
32
|
+
override_time_for_queues(2.day) { queue_action_event('test_two_days_ago') }
|
33
|
+
queue_action_event('test_today')
|
34
|
+
assert_equal get_next_message.event, 'test_two_days_ago'
|
35
|
+
assert_equal get_next_message.event, 'test_yesterday'
|
36
|
+
assert_equal get_next_message.event, 'test_today'
|
37
|
+
end
|
38
|
+
|
39
|
+
test "priority is preserved across days" do
|
40
|
+
queue_action_event('test_low', :queue => :low)
|
41
|
+
override_time_for_queues(1.day) { queue_action_event('test_high', :queue => :high) }
|
42
|
+
override_time_for_queues(2.day) { queue_action_event('test_medium', :queue => :medium) }
|
43
|
+
assert_equal get_next_message.event, 'test_high'
|
44
|
+
assert_equal get_next_message.event, 'test_medium'
|
45
|
+
assert_equal get_next_message.event, 'test_low'
|
46
|
+
end
|
47
|
+
|
48
|
+
test "cleanup doesn't delete bad data" do
|
49
|
+
ActionEvent::Message.reset!
|
50
|
+
assert_equal ActionEvent::Message.send(:all_message_tables).length, 0
|
51
|
+
assert_nil get_next_message
|
52
|
+
|
53
|
+
queue_action_event('test1')
|
54
|
+
ActionEvent::Message.cleanup!
|
55
|
+
|
56
|
+
override_time_for_queues(1.day) { queue_action_event('test2') }
|
57
|
+
ActionEvent::Message.cleanup!
|
58
|
+
|
59
|
+
override_time_for_queues(2.day) { queue_action_event('test3') }
|
60
|
+
ActionEvent::Message.cleanup!
|
61
|
+
|
62
|
+
assert_equal get_next_message.event, 'test3'
|
63
|
+
ActionEvent::Message.cleanup!
|
64
|
+
|
65
|
+
assert_equal get_next_message.event, 'test2'
|
66
|
+
ActionEvent::Message.cleanup!
|
67
|
+
|
68
|
+
assert_equal get_next_message.event, 'test1'
|
69
|
+
ActionEvent::Message.cleanup!
|
70
|
+
|
71
|
+
assert_nil get_next_message
|
72
|
+
end
|
73
|
+
|
74
|
+
protected
|
75
|
+
|
76
|
+
def get_next_message
|
77
|
+
ActionEvent::Message.try_to_get_next_message([:high, :medium, :low])
|
78
|
+
end
|
79
|
+
|
80
|
+
def override_time_for_queues(time_offset, &block)
|
81
|
+
ActionEvent::Message.class_eval %(
|
82
|
+
class << self
|
83
|
+
alias :original_current_table_name_for_queue :current_table_name_for_queue
|
84
|
+
def current_table_name_for_queue(queue_name)
|
85
|
+
original_current_table_name_for_queue(queue_name, Time.now - #{time_offset.to_i})
|
86
|
+
end
|
87
|
+
end
|
88
|
+
)
|
89
|
+
|
90
|
+
yield block
|
91
|
+
|
92
|
+
ActionEvent::Message.class_eval %(
|
93
|
+
class << self
|
94
|
+
alias :current_table_name_for_queue :original_current_table_name_for_queue
|
95
|
+
end
|
96
|
+
)
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
data/test/test_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: action_event
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Warren Konkel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-10-06 00:00:00 -04:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: A framework for asynchronous message processing in a Rails application.
|
17
|
+
email: wkonkel@gmail.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- action_event.gemspec
|
26
|
+
- Capfile
|
27
|
+
- generators/event/event_generator.rb
|
28
|
+
- generators/event/templates/application_event.rb
|
29
|
+
- generators/event/templates/event.rb
|
30
|
+
- generators/event/templates/migration.rb
|
31
|
+
- generators/event/templates/poller
|
32
|
+
- init.rb
|
33
|
+
- lib/action_event/base.rb
|
34
|
+
- lib/action_event/commands/poller.rb
|
35
|
+
- lib/action_event/message.rb
|
36
|
+
- lib/action_event/object_extensions.rb
|
37
|
+
- lib/action_event.rb
|
38
|
+
- MIT-LICENSE
|
39
|
+
- Rakefile
|
40
|
+
- README
|
41
|
+
has_rdoc: true
|
42
|
+
homepage: http://github.com/wkonkel/action_event
|
43
|
+
licenses: []
|
44
|
+
|
45
|
+
post_install_message:
|
46
|
+
rdoc_options: []
|
47
|
+
|
48
|
+
require_paths:
|
49
|
+
- lib
|
50
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: "0"
|
55
|
+
version:
|
56
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: "0"
|
61
|
+
version:
|
62
|
+
requirements: []
|
63
|
+
|
64
|
+
rubyforge_project:
|
65
|
+
rubygems_version: 1.3.5
|
66
|
+
signing_key:
|
67
|
+
specification_version: 3
|
68
|
+
summary: A framework for asynchronous message processing in a Rails application.
|
69
|
+
test_files:
|
70
|
+
- test/action_event_test.rb
|
71
|
+
- test/test_helper.rb
|