workling 0.4.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES.markdown +82 -0
- data/README.markdown +543 -0
- data/TODO.markdown +27 -0
- data/VERSION.yml +4 -0
- data/bin/workling_client +29 -0
- data/contrib/bj_invoker.rb +11 -0
- data/contrib/starling_status.rb +37 -0
- data/lib/extensions/cattr_accessor.rb +51 -0
- data/lib/extensions/mattr_accessor.rb +55 -0
- data/lib/workling.rb +213 -0
- data/lib/workling/base.rb +110 -0
- data/lib/workling/clients/amqp_client.rb +51 -0
- data/lib/workling/clients/amqp_exchange_client.rb +58 -0
- data/lib/workling/clients/backgroundjob_client.rb +25 -0
- data/lib/workling/clients/base.rb +89 -0
- data/lib/workling/clients/broker_base.rb +63 -0
- data/lib/workling/clients/memcache_queue_client.rb +104 -0
- data/lib/workling/clients/memory_queue_client.rb +34 -0
- data/lib/workling/clients/not_client.rb +14 -0
- data/lib/workling/clients/not_remote_client.rb +17 -0
- data/lib/workling/clients/rude_q_client.rb +47 -0
- data/lib/workling/clients/spawn_client.rb +46 -0
- data/lib/workling/clients/sqs_client.rb +163 -0
- data/lib/workling/clients/thread_client.rb +18 -0
- data/lib/workling/clients/xmpp_client.rb +110 -0
- data/lib/workling/discovery.rb +16 -0
- data/lib/workling/invokers/amqp_single_subscriber.rb +42 -0
- data/lib/workling/invokers/base.rb +124 -0
- data/lib/workling/invokers/basic_poller.rb +38 -0
- data/lib/workling/invokers/eventmachine_subscriber.rb +38 -0
- data/lib/workling/invokers/looped_subscriber.rb +34 -0
- data/lib/workling/invokers/thread_pool_poller.rb +165 -0
- data/lib/workling/invokers/threaded_poller.rb +149 -0
- data/lib/workling/remote.rb +38 -0
- data/lib/workling/return/store/base.rb +42 -0
- data/lib/workling/return/store/iterator.rb +24 -0
- data/lib/workling/return/store/memory_return_store.rb +24 -0
- data/lib/workling/return/store/starling_return_store.rb +30 -0
- data/lib/workling/routing/base.rb +13 -0
- data/lib/workling/routing/class_and_method_routing.rb +55 -0
- data/lib/workling/routing/static_routing.rb +43 -0
- data/lib/workling_daemon.rb +111 -0
- metadata +96 -0
@@ -0,0 +1,51 @@
|
|
1
|
+
#
|
2
|
+
# An Ampq client
|
3
|
+
#
|
4
|
+
module Workling
|
5
|
+
module Clients
|
6
|
+
class AmqpClient < Workling::Clients::BrokerBase
|
7
|
+
|
8
|
+
def self.load
|
9
|
+
begin
|
10
|
+
require 'mq'
|
11
|
+
rescue Exception => e
|
12
|
+
raise WorklingError.new(
|
13
|
+
"WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
|
14
|
+
"Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
|
15
|
+
)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# starts the client.
|
20
|
+
def connect
|
21
|
+
begin
|
22
|
+
connection = AMQP.start((Workling.config[:amqp_options] ||{}).symbolize_keys)
|
23
|
+
@amq = MQ.new connection
|
24
|
+
rescue
|
25
|
+
raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# no need for explicit closing. when the event loop
|
30
|
+
# terminates, the connection is closed anyway.
|
31
|
+
def close; true; end
|
32
|
+
|
33
|
+
# subscribe to a queue
|
34
|
+
def subscribe(key)
|
35
|
+
@amq.queue(queue_for(key)).subscribe do |value|
|
36
|
+
data = Marshal.load(value) rescue value
|
37
|
+
yield data
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# request and retrieve work
|
42
|
+
def retrieve(key); @amq.queue(queue_for(key)); end
|
43
|
+
def request(key, value); @amq.queue(queue_for(key)).publish(Marshal.dump(value)); end
|
44
|
+
|
45
|
+
private
|
46
|
+
def queue_for(key)
|
47
|
+
"#{Workling.config[:prefix]}#{key}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
#
|
2
|
+
# An Ampq client
|
3
|
+
#
|
4
|
+
module Workling
|
5
|
+
module Clients
|
6
|
+
class AmqpExchangeClient < Workling::Clients::BrokerBase
|
7
|
+
|
8
|
+
def self.load
|
9
|
+
begin
|
10
|
+
require 'mq'
|
11
|
+
rescue Exception => e
|
12
|
+
raise WorklingError.new(
|
13
|
+
"WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
|
14
|
+
"Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
|
15
|
+
)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# starts the client.
|
20
|
+
def connect
|
21
|
+
begin
|
22
|
+
@amq = MQ.new
|
23
|
+
rescue
|
24
|
+
raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# no need for explicit closing. when the event loop
|
29
|
+
# terminates, the connection is closed anyway.
|
30
|
+
def close; true; end
|
31
|
+
|
32
|
+
# subscribe to a queue
|
33
|
+
def subscribe(key)
|
34
|
+
@amq.queue(key).subscribe do |header, body|
|
35
|
+
value = YAML.load(body)
|
36
|
+
yield value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# request and retrieve work
|
41
|
+
def retrieve(key)
|
42
|
+
@amq.queue(key)
|
43
|
+
end
|
44
|
+
|
45
|
+
# publish message to exchange
|
46
|
+
# using the specified routing key
|
47
|
+
def request(exchange_name, value)
|
48
|
+
exchange_name = "amq.topic"
|
49
|
+
|
50
|
+
key = value.delete(:routing_key)
|
51
|
+
msg = YAML.dump(value)
|
52
|
+
exchange = @amq.topic(exchange_name)
|
53
|
+
exchange.publish(msg, :routing_key => key)
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Workling
|
2
|
+
module Clients
|
3
|
+
class BackgroundjobClient < Workling::Clients::Base
|
4
|
+
|
5
|
+
def self.installed?
|
6
|
+
Object.const_defined? "Bj"
|
7
|
+
end
|
8
|
+
|
9
|
+
# passes the job to bj by serializing the options to xml and passing them to
|
10
|
+
# ./script/bj_invoker.rb, which in turn routes the deserialized args to the
|
11
|
+
# appropriate worker.
|
12
|
+
def dispatch(clazz, method, options = {})
|
13
|
+
stdin = Workling::Remote.routing.queue_for(clazz, method) +
|
14
|
+
" " +
|
15
|
+
options.to_xml(:indent => 0, :skip_instruct => true)
|
16
|
+
|
17
|
+
Bj.submit "./script/runner ./script/bj_invoker.rb",
|
18
|
+
:stdin => stdin
|
19
|
+
|
20
|
+
return nil # that means nothing!
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
#
|
2
|
+
# Clients are responsible for dispatching jobs either to a broker (starling, rabbitmq etc) or invoke them (spawn)
|
3
|
+
#
|
4
|
+
# Clients that involve a broker should subclass Workling::Clients::BrokerBase
|
5
|
+
#
|
6
|
+
# Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
|
7
|
+
# from a specific type of job.
|
8
|
+
#
|
9
|
+
module Workling
|
10
|
+
module Clients
|
11
|
+
class Base
|
12
|
+
|
13
|
+
#
|
14
|
+
# Load the required libraries, for this client
|
15
|
+
#
|
16
|
+
def self.load
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
#
|
22
|
+
# See if the libraries required for this client are installed
|
23
|
+
#
|
24
|
+
def self.installed?
|
25
|
+
true
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
# returns the Workling::Base.logger
|
30
|
+
def logger; Workling::Base.logger; end
|
31
|
+
|
32
|
+
|
33
|
+
#
|
34
|
+
# Dispatch a job to the client. If this client uses a job broker, then
|
35
|
+
# this method should submit it, otherwise it should run the job
|
36
|
+
#
|
37
|
+
# clazz: Name of the worker class
|
38
|
+
# method: Name of the methods on the worker
|
39
|
+
# options: optional arguments for the job
|
40
|
+
#
|
41
|
+
def dispatch(clazz, method, options = {})
|
42
|
+
raise NotImplementedError.new("Implement dispatch(clazz, method, options) in your client. ")
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
#
|
47
|
+
# Requests a job on the broker.
|
48
|
+
#
|
49
|
+
# work_type:
|
50
|
+
# arguments: the argument to the worker method
|
51
|
+
#
|
52
|
+
def request(work_type, arguments)
|
53
|
+
raise WorklingError.new("This client does not involve a broker.")
|
54
|
+
end
|
55
|
+
|
56
|
+
#
|
57
|
+
# Gets job results off a job broker. Returns nil if there are no results.
|
58
|
+
#
|
59
|
+
# worker_uid: the uid returned by workling when the work was dispatched
|
60
|
+
#
|
61
|
+
def retrieve(work_uid)
|
62
|
+
raise WorklingError.new("This client does not involve a broker.")
|
63
|
+
end
|
64
|
+
|
65
|
+
#
|
66
|
+
# Subscribe to job results in a job broker.
|
67
|
+
#
|
68
|
+
# worker_type:
|
69
|
+
#
|
70
|
+
def subscribe(work_type)
|
71
|
+
raise WorklingError.new("This client does not involve a broker.")
|
72
|
+
end
|
73
|
+
|
74
|
+
#
|
75
|
+
# Opens a connection to the job broker.
|
76
|
+
#
|
77
|
+
def connect
|
78
|
+
raise WorklingError.new("This client does not involve a broker.")
|
79
|
+
end
|
80
|
+
|
81
|
+
#
|
82
|
+
# Closes the connection to the job broker.
|
83
|
+
#
|
84
|
+
def close
|
85
|
+
raise WorklingError.new("This client does not involve a broker.")
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Workling
|
2
|
+
module Clients
|
3
|
+
class BrokerBase < Base
|
4
|
+
|
5
|
+
#
|
6
|
+
# Dispatch a job to the client. If this client uses a job broker, then
|
7
|
+
# this method should submit it, otherwise it should run the job
|
8
|
+
#
|
9
|
+
# clazz: Name of the worker class
|
10
|
+
# method: Name of the methods on the worker
|
11
|
+
# options: optional arguments for the job
|
12
|
+
#
|
13
|
+
def dispatch(clazz, method, options = {})
|
14
|
+
@connected ||= connect
|
15
|
+
request(Workling::Remote.routing.queue_for(clazz, method), options)
|
16
|
+
return nil
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
#
|
21
|
+
# Requests a job on the broker.
|
22
|
+
#
|
23
|
+
# work_type:
|
24
|
+
# arguments: the argument to the worker method
|
25
|
+
#
|
26
|
+
def request(work_type, arguments)
|
27
|
+
raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
|
28
|
+
end
|
29
|
+
|
30
|
+
#
|
31
|
+
# Gets job results off a job broker. Returns nil if there are no results.
|
32
|
+
#
|
33
|
+
# worker_uid: the uid returned by workling when the work was dispatched
|
34
|
+
#
|
35
|
+
def retrieve(work_uid)
|
36
|
+
raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# Subscribe to job results in a job broker.
|
41
|
+
#
|
42
|
+
# worker_type:
|
43
|
+
#
|
44
|
+
def subscribe(work_type)
|
45
|
+
raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
|
46
|
+
end
|
47
|
+
|
48
|
+
#
|
49
|
+
# Opens a connection to the job broker.
|
50
|
+
#
|
51
|
+
def connect
|
52
|
+
raise NotImplementedError.new("Implement connect() in your client. ")
|
53
|
+
end
|
54
|
+
|
55
|
+
#
|
56
|
+
# Closes the connection to the job broker.
|
57
|
+
#
|
58
|
+
def close
|
59
|
+
raise NotImplementedError.new("Implement close() in your client. ")
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
#
|
2
|
+
# This client can be used for all Queue Servers that speak Memcached, such as Starling.
|
3
|
+
#
|
4
|
+
# Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client,
|
5
|
+
# or memcache-client, if this is not available. See the README for a discussion of the memcache
|
6
|
+
# clients.
|
7
|
+
#
|
8
|
+
# method_missing delegates all messages through to the underlying memcache connection.
|
9
|
+
#
|
10
|
+
module Workling
|
11
|
+
module Clients
|
12
|
+
class MemcacheQueueClient < Workling::Clients::BrokerBase
|
13
|
+
|
14
|
+
def self.installed?
|
15
|
+
begin
|
16
|
+
require 'starling'
|
17
|
+
rescue LoadError
|
18
|
+
end
|
19
|
+
|
20
|
+
Object.const_defined? "Starling"
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.load
|
24
|
+
begin
|
25
|
+
gem 'memcache-client'
|
26
|
+
require 'memcache'
|
27
|
+
rescue Gem::LoadError
|
28
|
+
Workling::Base.logger.info "WORKLING: couldn't find memcache-client. Install: \"gem install memcache-client\". "
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# the url with which the memcache client expects to reach starling
|
33
|
+
attr_accessor :queueserver_urls
|
34
|
+
|
35
|
+
# the memcache connection object
|
36
|
+
attr_accessor :connection
|
37
|
+
|
38
|
+
#
|
39
|
+
# the client attempts to connect to queueserver using the configuration options found in
|
40
|
+
#
|
41
|
+
# Workling.config. this can be configured in config/workling.yml.
|
42
|
+
#
|
43
|
+
# the initialization code will raise an exception if memcache-client cannot connect
|
44
|
+
# to queueserver.
|
45
|
+
#
|
46
|
+
def connect
|
47
|
+
listens_on = Workling.config[:listens_on] || "localhost:22122"
|
48
|
+
@queueserver_urls = listens_on.split(',').map { |url| url ? url.strip : url }
|
49
|
+
options = [@queueserver_urls, Workling.config[:memcache_options]].compact
|
50
|
+
self.connection = ::MemCache.new(*options)
|
51
|
+
|
52
|
+
raise_unless_connected!
|
53
|
+
end
|
54
|
+
|
55
|
+
# closes the memcache connection
|
56
|
+
def close
|
57
|
+
self.connection.flush_all
|
58
|
+
self.connection.reset
|
59
|
+
end
|
60
|
+
|
61
|
+
# implements the client job request and retrieval
|
62
|
+
def request(key, value)
|
63
|
+
set(key, value)
|
64
|
+
end
|
65
|
+
|
66
|
+
def retrieve(key)
|
67
|
+
begin
|
68
|
+
get(key)
|
69
|
+
rescue MemCache::MemCacheError => e
|
70
|
+
# failed to enqueue, raise a workling error so that it propagates upwards
|
71
|
+
raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
# make sure we can actually connect to queueserver on the given port
|
77
|
+
def raise_unless_connected!
|
78
|
+
begin
|
79
|
+
self.connection.stats
|
80
|
+
rescue
|
81
|
+
raise Workling::QueueserverNotFoundError.new
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
[:get, :set].each do |method|
|
86
|
+
class_eval <<-EOS
|
87
|
+
def #{method}(*args, &block)
|
88
|
+
self.connection.#{method}(*args, &block)
|
89
|
+
end
|
90
|
+
EOS
|
91
|
+
end
|
92
|
+
|
93
|
+
# delegates directly through to the memcache connection.
|
94
|
+
def method_missing(method, *args)
|
95
|
+
begin
|
96
|
+
self.connection.send(method, *args)
|
97
|
+
rescue MemCache::MemCacheError => e
|
98
|
+
raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Workling
|
2
|
+
module Clients
|
3
|
+
class MemoryQueueClient < Workling::Clients::BrokerBase
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@subscribers ||= {}
|
7
|
+
@queues ||= {}
|
8
|
+
end
|
9
|
+
|
10
|
+
# collects the worker blocks in a hash
|
11
|
+
def subscribe(work_type, &block)
|
12
|
+
@subscribers[work_type] = block
|
13
|
+
end
|
14
|
+
|
15
|
+
# immediately invokes the required worker block
|
16
|
+
def request(work_type, arguments)
|
17
|
+
if subscription = @subscribers[work_type]
|
18
|
+
subscription.call(arguments)
|
19
|
+
else
|
20
|
+
@queues[work_type] ||= []
|
21
|
+
@queues[work_type] << arguments
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def retrieve(work_type)
|
26
|
+
queue = @queues[work_type]
|
27
|
+
queue.pop if queue
|
28
|
+
end
|
29
|
+
|
30
|
+
def connect; true; end
|
31
|
+
def close; true; end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|