did_workling 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +382 -0
- data/lib/rude_q/client.rb +11 -0
- data/lib/workling.rb +150 -0
- data/lib/workling/base.rb +59 -0
- data/lib/workling/clients/amqp_client.rb +40 -0
- data/lib/workling/clients/base.rb +54 -0
- data/lib/workling/clients/memcache_queue_client.rb +82 -0
- data/lib/workling/discovery.rb +14 -0
- data/lib/workling/remote.rb +42 -0
- data/lib/workling/remote/invokers/base.rb +124 -0
- data/lib/workling/remote/invokers/basic_poller.rb +41 -0
- data/lib/workling/remote/invokers/eventmachine_subscriber.rb +37 -0
- data/lib/workling/remote/invokers/threaded_poller.rb +149 -0
- data/lib/workling/remote/runners/backgroundjob_runner.rb +35 -0
- data/lib/workling/remote/runners/base.rb +42 -0
- data/lib/workling/remote/runners/client_runner.rb +45 -0
- data/lib/workling/remote/runners/not_remote_runner.rb +23 -0
- data/lib/workling/remote/runners/spawn_runner.rb +38 -0
- data/lib/workling/remote/runners/starling_runner.rb +13 -0
- data/lib/workling/return/store/base.rb +37 -0
- data/lib/workling/return/store/memory_return_store.rb +26 -0
- data/lib/workling/return/store/starling_return_store.rb +31 -0
- data/lib/workling/routing/base.rb +13 -0
- data/lib/workling/routing/class_and_method_routing.rb +55 -0
- data/test/class_and_method_routing_test.rb +18 -0
- data/test/clients/memory_queue_client.rb +36 -0
- data/test/discovery_test.rb +13 -0
- data/test/invoker_basic_poller_test.rb +29 -0
- data/test/invoker_eventmachine_subscription_test.rb +26 -0
- data/test/invoker_threaded_poller_test.rb +34 -0
- data/test/memcachequeue_client_test.rb +36 -0
- data/test/memory_return_store_test.rb +23 -0
- data/test/mocks/client.rb +9 -0
- data/test/mocks/logger.rb +5 -0
- data/test/mocks/spawn.rb +5 -0
- data/test/not_remote_runner_test.rb +11 -0
- data/test/remote_runner_test.rb +50 -0
- data/test/return_store_test.rb +18 -0
- data/test/runners/thread_runner.rb +22 -0
- data/test/spawn_runner_test.rb +10 -0
- data/test/starling_return_store_test.rb +29 -0
- data/test/starling_runner_test.rb +8 -0
- data/test/test_helper.rb +48 -0
- data/test/workers/analytics/invites.rb +10 -0
- data/test/workers/util.rb +15 -0
- metadata +132 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'workling/remote/invokers/base'
|
3
|
+
|
4
|
+
#
|
5
|
+
# Subscribes the workers to the correct queues.
|
6
|
+
#
|
7
|
+
module Workling
|
8
|
+
module Remote
|
9
|
+
module Invokers
|
10
|
+
class EventmachineSubscriber < Workling::Remote::Invokers::Base
|
11
|
+
|
12
|
+
def initialize(routing, client_class)
|
13
|
+
super
|
14
|
+
end
|
15
|
+
|
16
|
+
#
|
17
|
+
# Starts EM loop and sets up subscription callbacks for workers.
|
18
|
+
#
|
19
|
+
def listen
|
20
|
+
EM.run do
|
21
|
+
connect do
|
22
|
+
routes.each do |route|
|
23
|
+
@client.subscribe(route) do |args|
|
24
|
+
run(route, args)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def stop
|
32
|
+
EM.stop if EM.reactor_running?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,149 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'workling/remote/invokers/base'
|
3
|
+
|
4
|
+
#
|
5
|
+
# A threaded polling Invoker.
|
6
|
+
#
|
7
|
+
# TODO: refactor this to make use of the base class.
|
8
|
+
#
|
9
|
+
module Workling
|
10
|
+
module Remote
|
11
|
+
module Invokers
|
12
|
+
class ThreadedPoller < Workling::Remote::Invokers::Base
|
13
|
+
|
14
|
+
cattr_accessor :sleep_time, :reset_time
|
15
|
+
|
16
|
+
def initialize(routing, client_class)
|
17
|
+
super
|
18
|
+
|
19
|
+
ThreadedPoller.sleep_time = Workling.config[:sleep_time] || 2
|
20
|
+
ThreadedPoller.reset_time = Workling.config[:reset_time] || 30
|
21
|
+
|
22
|
+
@workers = ThreadGroup.new
|
23
|
+
@mutex = Mutex.new
|
24
|
+
end
|
25
|
+
|
26
|
+
def listen
|
27
|
+
# Allow concurrency for our tasks
|
28
|
+
ActiveRecord::Base.allow_concurrency = true
|
29
|
+
|
30
|
+
# Create a thread for each worker.
|
31
|
+
Workling::Discovery.discovered.each do |clazz|
|
32
|
+
logger.debug("Discovered listener #{clazz}")
|
33
|
+
@workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
|
34
|
+
end
|
35
|
+
|
36
|
+
# Wait for all workers to complete
|
37
|
+
@workers.list.each { |t| t.join }
|
38
|
+
|
39
|
+
logger.debug("Reaped listener threads. ")
|
40
|
+
|
41
|
+
# Clean up all the connections.
|
42
|
+
ActiveRecord::Base.verify_active_connections!
|
43
|
+
logger.debug("Cleaned up connection: out!")
|
44
|
+
end
|
45
|
+
|
46
|
+
# Check if all Worker threads have been started.
|
47
|
+
def started?
|
48
|
+
logger.debug("checking if started... list size is #{ worker_threads }")
|
49
|
+
Workling::Discovery.discovered.size == worker_threads
|
50
|
+
end
|
51
|
+
|
52
|
+
# number of worker threads running
|
53
|
+
def worker_threads
|
54
|
+
@workers.list.size
|
55
|
+
end
|
56
|
+
|
57
|
+
# Gracefully stop processing
|
58
|
+
def stop
|
59
|
+
logger.info("stopping threaded poller...")
|
60
|
+
sleep 1 until started? # give it a chance to start up before shutting down.
|
61
|
+
logger.info("Giving Listener Threads a chance to shut down. This may take a while... ")
|
62
|
+
@workers.list.each { |w| w[:shutdown] = true }
|
63
|
+
logger.info("Listener threads were shut down. ")
|
64
|
+
end
|
65
|
+
|
66
|
+
# Listen for one worker class
|
67
|
+
def clazz_listen(clazz)
|
68
|
+
logger.debug("Listener thread #{clazz.name} started")
|
69
|
+
|
70
|
+
# Read thread configuration if available
|
71
|
+
if Workling.config.has_key?(:listeners)
|
72
|
+
if Workling.config[:listeners].has_key?(clazz.to_s)
|
73
|
+
config = Workling.config[:listeners][clazz.to_s].symbolize_keys
|
74
|
+
thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
hread_sleep_time ||= self.class.sleep_time
|
79
|
+
|
80
|
+
# Setup connection to client (one per thread)
|
81
|
+
connection = @client_class.new
|
82
|
+
connection.connect
|
83
|
+
logger.info("** Starting client #{ connection.class } for #{clazz.name} queue")
|
84
|
+
|
85
|
+
# Start dispatching those messages
|
86
|
+
while (!Thread.current[:shutdown]) do
|
87
|
+
begin
|
88
|
+
|
89
|
+
# Thanks for this Brent!
|
90
|
+
#
|
91
|
+
# ...Just a heads up, due to how rails’ MySQL adapter handles this
|
92
|
+
# call ‘ActiveRecord::Base.connection.active?’, you’ll need
|
93
|
+
# to wrap the code that checks for a connection in in a mutex.
|
94
|
+
#
|
95
|
+
# ....I noticed this while working with a multi-core machine that
|
96
|
+
# was spawning multiple workling threads. Some of my workling
|
97
|
+
# threads would hit serious issues at this block of code without
|
98
|
+
# the mutex.
|
99
|
+
#
|
100
|
+
@mutex.synchronize do
|
101
|
+
ActiveRecord::Base.connection.verify! # Keep MySQL connection alive
|
102
|
+
unless ActiveRecord::Base.connection.active?
|
103
|
+
logger.fatal("Failed - Database not available!")
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
# Dispatch and process the messages
|
108
|
+
n = dispatch!(connection, clazz)
|
109
|
+
logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
|
110
|
+
sleep(self.class.sleep_time) unless n > 0
|
111
|
+
|
112
|
+
# If there is a memcache error, hang for a bit to give it a chance to fire up again
|
113
|
+
# and reset the connection.
|
114
|
+
rescue Workling::WorklingConnectionError
|
115
|
+
logger.warn("Listener thread #{clazz.name} failed to connect. Resetting connection.")
|
116
|
+
sleep(self.class.reset_time)
|
117
|
+
connection.reset
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
logger.debug("Listener thread #{clazz.name} ended")
|
122
|
+
end
|
123
|
+
|
124
|
+
# Dispatcher for one worker class. Will throw MemCacheError if unable to connect.
|
125
|
+
# Returns the number of worker methods called
|
126
|
+
def dispatch!(connection, clazz)
|
127
|
+
n = 0
|
128
|
+
for queue in @routing.queue_names_routing_class(clazz)
|
129
|
+
begin
|
130
|
+
result = connection.retrieve(queue)
|
131
|
+
if result
|
132
|
+
n += 1
|
133
|
+
handler = @routing[queue]
|
134
|
+
method_name = @routing.method_name(queue)
|
135
|
+
logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
|
136
|
+
handler.dispatch_to_worker_method(method_name, result)
|
137
|
+
end
|
138
|
+
rescue MemCache::MemCacheError => e
|
139
|
+
logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
|
140
|
+
raise e
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
return n
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'workling/remote/runners/base'
|
2
|
+
|
3
|
+
#
|
4
|
+
# Use Ara Howards BackgroundJob to run the work. BackgroundJob loads Rails once per requested Job.
|
5
|
+
# It persists over the database, and there is no requirement for separate processes to be started.
|
6
|
+
# Since rails has to load before each request, it takes a moment for the job to run.
|
7
|
+
#
|
8
|
+
module Workling
|
9
|
+
module Remote
|
10
|
+
module Runners
|
11
|
+
class BackgroundjobRunner < Workling::Remote::Runners::Base
|
12
|
+
cattr_accessor :routing
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
BackgroundjobRunner.routing =
|
16
|
+
Workling::Routing::ClassAndMethodRouting.new
|
17
|
+
end
|
18
|
+
|
19
|
+
# passes the job to bj by serializing the options to xml and passing them to
|
20
|
+
# ./script/bj_invoker.rb, which in turn routes the deserialized args to the
|
21
|
+
# appropriate worker.
|
22
|
+
def run(clazz, method, options = {})
|
23
|
+
stdin = @@routing.queue_for(clazz, method) +
|
24
|
+
" " +
|
25
|
+
options.to_xml(:indent => 0, :skip_instruct => true)
|
26
|
+
|
27
|
+
Bj.submit "./script/runner ./script/bj_invoker.rb",
|
28
|
+
:stdin => stdin
|
29
|
+
|
30
|
+
return nil # that means nothing!
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
#
|
2
|
+
# Base class for Workling Runners.
|
3
|
+
#
|
4
|
+
# Runners must subclass this and implement the method
|
5
|
+
#
|
6
|
+
# Workling::Remote::Runners::Base#run(clazz, method, options = {})
|
7
|
+
#
|
8
|
+
# which is responsible for pushing the requested job into the background. Depending
|
9
|
+
# on the Runner, this may require other code to dequeue the job. The actual
|
10
|
+
# invocation of the runner should be done like this:
|
11
|
+
#
|
12
|
+
# Workling.find(clazz, method).dispatch_to_worker_method(method, options)
|
13
|
+
#
|
14
|
+
# This ensures for consistent logging and handling of propagated exceptions. You can
|
15
|
+
# also call the convenience method
|
16
|
+
#
|
17
|
+
# Workling::Remote::Runners::Base#dispatch!(clazz, method, options)
|
18
|
+
#
|
19
|
+
# which invokes this for you.
|
20
|
+
#
|
21
|
+
module Workling
|
22
|
+
module Remote
|
23
|
+
module Runners
|
24
|
+
class Base
|
25
|
+
|
26
|
+
# runner uses this to connect to a job broker
|
27
|
+
cattr_accessor :client
|
28
|
+
|
29
|
+
# default logger defined in Workling::Base.logger
|
30
|
+
def logger
|
31
|
+
Workling::Base.logger
|
32
|
+
end
|
33
|
+
|
34
|
+
# find the worker instance and invoke it. Invoking the worker method like this ensures for
|
35
|
+
# consistent logging and handling of propagated exceptions.
|
36
|
+
def dispatch!(clazz, method, options)
|
37
|
+
Workling.find(clazz, method).dispatch_to_worker_method(method, options)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'workling/remote/runners/base'
|
2
|
+
require 'workling/clients/memcache_queue_client'
|
3
|
+
|
4
|
+
#
|
5
|
+
# Runs Jobs over a Client. The client should be a subclass of Workling::Client::Base.
|
6
|
+
# Set the client like this:
|
7
|
+
#
|
8
|
+
# Workling::Remote::Runners::ClientRunner.client = Workling::Clients::AmqpClient.new
|
9
|
+
#
|
10
|
+
# Jobs are dispatched by requesting them on the Client. The Runner takes care of mapping of queue names to worker code.
|
11
|
+
# this is done with Workling::ClassAndMethodRouting, but you can use your own by sublassing Workling::Routing.
|
12
|
+
# Don’t worry about any of this if you’re not dealing directly with the queues.
|
13
|
+
#
|
14
|
+
# There’s a workling-client daemon that uses the configured invoker to retrieve work and dispatching these to the
|
15
|
+
# responsible workers. If you intend to run this on a remote machine, then just check out your rails project
|
16
|
+
# there and start up the workling client like this: ruby script/workling_client run.
|
17
|
+
#
|
18
|
+
module Workling
|
19
|
+
module Remote
|
20
|
+
module Runners
|
21
|
+
class ClientRunner < Workling::Remote::Runners::Base
|
22
|
+
|
23
|
+
# Routing class. Workling::Routing::ClassAndMethodRouting.new by default.
|
24
|
+
cattr_accessor :routing
|
25
|
+
@@routing ||= Workling::Routing::ClassAndMethodRouting.new
|
26
|
+
|
27
|
+
# The workling Client class. Workling::Clients::MemcacheQueueClient.new by default.
|
28
|
+
cattr_accessor :client
|
29
|
+
@@client ||= Workling::Clients::MemcacheQueueClient.new
|
30
|
+
|
31
|
+
# enqueues the job onto the client
|
32
|
+
def run(clazz, method, options = {})
|
33
|
+
|
34
|
+
# neet to connect in here as opposed to the constructor, since the EM loop is
|
35
|
+
# not available there.
|
36
|
+
@connected ||= self.class.client.connect
|
37
|
+
|
38
|
+
self.class.client.request(@@routing.queue_for(clazz, method), options)
|
39
|
+
|
40
|
+
return nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'workling/remote/runners/base'
|
2
|
+
|
3
|
+
#
|
4
|
+
# directly dispatches to the worker method, in-process. options are first marshalled then dumped
|
5
|
+
# in order to simulate the sideeffects of a remote call.
|
6
|
+
#
|
7
|
+
module Workling
|
8
|
+
module Remote
|
9
|
+
module Runners
|
10
|
+
class NotRemoteRunner < Workling::Remote::Runners::Base
|
11
|
+
|
12
|
+
# directly dispatches to the worker method, in-process. options are first marshalled then dumped
|
13
|
+
# in order to simulate the sideeffects of a remote call.
|
14
|
+
def run(clazz, method, options = {})
|
15
|
+
options = Marshal.load(Marshal.dump(options)) # get this to behave more like the remote runners
|
16
|
+
dispatch!(clazz, method, options)
|
17
|
+
|
18
|
+
return nil # nada. niente.
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'workling/remote/runners/base'
|
2
|
+
|
3
|
+
#
|
4
|
+
# Run the job over the spawn plugin. Refer to the README for instructions on
|
5
|
+
# installing Spawn.
|
6
|
+
#
|
7
|
+
# Spawn forks the entire process once for each job. This means that the job starts
|
8
|
+
# with a very low latency, but takes up more memory for each job.
|
9
|
+
#
|
10
|
+
# It's also possible to configure Spawn to start a Thread for each job. Do this
|
11
|
+
# by setting
|
12
|
+
#
|
13
|
+
# Workling::Remote::Runners::SpawnRunner.options = { :method => :thread }
|
14
|
+
#
|
15
|
+
# Have a look at the Spawn README to find out more about the characteristics of this.
|
16
|
+
#
|
17
|
+
module Workling
|
18
|
+
module Remote
|
19
|
+
module Runners
|
20
|
+
class SpawnRunner < Workling::Remote::Runners::Base
|
21
|
+
cattr_accessor :options
|
22
|
+
|
23
|
+
# use thread for development and test modes. easier to hunt down exceptions that way.
|
24
|
+
@@options = { :method => (RAILS_ENV == "test" || RAILS_ENV == "development" ? :thread : :fork) }
|
25
|
+
include Spawn if Workling.spawn_installed?
|
26
|
+
|
27
|
+
# dispatches to Spawn, using the :fork option.
|
28
|
+
def run(clazz, method, options = {})
|
29
|
+
spawn(SpawnRunner.options) do # exceptions are trapped in here.
|
30
|
+
dispatch!(clazz, method, options)
|
31
|
+
end
|
32
|
+
|
33
|
+
return nil # that means nothing!
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
#
|
2
|
+
# Basic interface for getting and setting Data which needs to be passed between Workers and
|
3
|
+
# client code.
|
4
|
+
#
|
5
|
+
module Workling
|
6
|
+
module Return
|
7
|
+
module Store
|
8
|
+
mattr_accessor :instance
|
9
|
+
|
10
|
+
# set a value in the store with the given key. delegates to the returnstore.
|
11
|
+
def self.set(key, value)
|
12
|
+
self.instance.set(key, value)
|
13
|
+
end
|
14
|
+
|
15
|
+
# get a value from the store. this should be destructive. delegates to the returnstore.
|
16
|
+
def self.get(key)
|
17
|
+
self.instance.get(key)
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Base Class for Return Stores. Subclasses need to implement set and get.
|
22
|
+
#
|
23
|
+
class Base
|
24
|
+
|
25
|
+
# set a value in the store with the given key.
|
26
|
+
def set(key, value)
|
27
|
+
raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
|
28
|
+
end
|
29
|
+
|
30
|
+
# get a value from the store. this should be destructive.
|
31
|
+
def get(key)
|
32
|
+
raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'workling/return/store/base'
|
2
|
+
|
3
|
+
#
|
4
|
+
# Stores directly into memory. This is for tests only - not for production use. aight?
|
5
|
+
#
|
6
|
+
module Workling
|
7
|
+
module Return
|
8
|
+
module Store
|
9
|
+
class MemoryReturnStore < Base
|
10
|
+
attr_accessor :sky
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
self.sky = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def set(key, value)
|
17
|
+
self.sky[key] = value
|
18
|
+
end
|
19
|
+
|
20
|
+
def get(key)
|
21
|
+
self.sky.delete(key)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|