did_workling 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +382 -0
- data/lib/rude_q/client.rb +11 -0
- data/lib/workling.rb +150 -0
- data/lib/workling/base.rb +59 -0
- data/lib/workling/clients/amqp_client.rb +40 -0
- data/lib/workling/clients/base.rb +54 -0
- data/lib/workling/clients/memcache_queue_client.rb +82 -0
- data/lib/workling/discovery.rb +14 -0
- data/lib/workling/remote.rb +42 -0
- data/lib/workling/remote/invokers/base.rb +124 -0
- data/lib/workling/remote/invokers/basic_poller.rb +41 -0
- data/lib/workling/remote/invokers/eventmachine_subscriber.rb +37 -0
- data/lib/workling/remote/invokers/threaded_poller.rb +149 -0
- data/lib/workling/remote/runners/backgroundjob_runner.rb +35 -0
- data/lib/workling/remote/runners/base.rb +42 -0
- data/lib/workling/remote/runners/client_runner.rb +45 -0
- data/lib/workling/remote/runners/not_remote_runner.rb +23 -0
- data/lib/workling/remote/runners/spawn_runner.rb +38 -0
- data/lib/workling/remote/runners/starling_runner.rb +13 -0
- data/lib/workling/return/store/base.rb +37 -0
- data/lib/workling/return/store/memory_return_store.rb +26 -0
- data/lib/workling/return/store/starling_return_store.rb +31 -0
- data/lib/workling/routing/base.rb +13 -0
- data/lib/workling/routing/class_and_method_routing.rb +55 -0
- data/test/class_and_method_routing_test.rb +18 -0
- data/test/clients/memory_queue_client.rb +36 -0
- data/test/discovery_test.rb +13 -0
- data/test/invoker_basic_poller_test.rb +29 -0
- data/test/invoker_eventmachine_subscription_test.rb +26 -0
- data/test/invoker_threaded_poller_test.rb +34 -0
- data/test/memcachequeue_client_test.rb +36 -0
- data/test/memory_return_store_test.rb +23 -0
- data/test/mocks/client.rb +9 -0
- data/test/mocks/logger.rb +5 -0
- data/test/mocks/spawn.rb +5 -0
- data/test/not_remote_runner_test.rb +11 -0
- data/test/remote_runner_test.rb +50 -0
- data/test/return_store_test.rb +18 -0
- data/test/runners/thread_runner.rb +22 -0
- data/test/spawn_runner_test.rb +10 -0
- data/test/starling_return_store_test.rb +29 -0
- data/test/starling_runner_test.rb +8 -0
- data/test/test_helper.rb +48 -0
- data/test/workers/analytics/invites.rb +10 -0
- data/test/workers/util.rb +15 -0
- metadata +132 -0
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'workling/return/store/base'
|
2
|
+
require 'workling/clients/memcache_queue_client'
|
3
|
+
|
4
|
+
#
|
5
|
+
# Recommended Return Store if you are using the Starling Runner. This
|
6
|
+
# Simply sets and gets values against queues. 'key' is the name of the respective Queue.
|
7
|
+
#
|
8
|
+
module Workling
|
9
|
+
module Return
|
10
|
+
module Store
|
11
|
+
class StarlingReturnStore < Base
|
12
|
+
cattr_accessor :client
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
self.client = Workling::Clients::MemcacheQueueClient.new
|
16
|
+
self.client.connect
|
17
|
+
end
|
18
|
+
|
19
|
+
# set a value in the queue 'key'.
|
20
|
+
def set(key, value)
|
21
|
+
self.class.client.set(key, value)
|
22
|
+
end
|
23
|
+
|
24
|
+
# get a value from starling queue 'key'.
|
25
|
+
def get(key)
|
26
|
+
self.class.client.get(key)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#
|
2
|
+
# Base Class for Routing. Routing takes the worker method TestWorker#something,
|
3
|
+
# and serializes the signature in some way.
|
4
|
+
#
|
5
|
+
module Workling
|
6
|
+
module Routing
|
7
|
+
class Base < Hash
|
8
|
+
def method_name
|
9
|
+
raise Exception.new("method_name not implemented.")
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'workling/routing/base'
|
2
|
+
|
3
|
+
#
|
4
|
+
# Holds a hash of routes. Each Worker method has a corresponding hash entry after building.
|
5
|
+
#
|
6
|
+
module Workling
|
7
|
+
module Routing
|
8
|
+
class ClassAndMethodRouting < Base
|
9
|
+
|
10
|
+
# initializes and builds routing hash.
|
11
|
+
def initialize
|
12
|
+
super
|
13
|
+
|
14
|
+
build
|
15
|
+
end
|
16
|
+
|
17
|
+
# returns the worker method name, given the routing string.
|
18
|
+
def method_name(queue)
|
19
|
+
queue.split("__").last
|
20
|
+
end
|
21
|
+
|
22
|
+
# returns the routing string, given a class and method. delegating.
|
23
|
+
def queue_for(clazz, method)
|
24
|
+
ClassAndMethodRouting.queue_for(clazz, method)
|
25
|
+
end
|
26
|
+
|
27
|
+
# returns the routing string, given a class and method.
|
28
|
+
def self.queue_for(clazz, method)
|
29
|
+
"#{ clazz.to_s.tableize }/#{ method }".split("/").join("__") # Don't split with : because it messes up memcache stats
|
30
|
+
end
|
31
|
+
|
32
|
+
# returns all routed
|
33
|
+
def queue_names
|
34
|
+
self.keys
|
35
|
+
end
|
36
|
+
|
37
|
+
# dare you to remove this! go on!
|
38
|
+
def queue_names_routing_class(clazz)
|
39
|
+
self.select { |x, y| y.is_a?(clazz) }.map { |x, y| x }
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
def build
|
44
|
+
Workling::Discovery.discovered.each do |clazz|
|
45
|
+
methods = clazz.instance_methods(false)
|
46
|
+
methods.each do |method|
|
47
|
+
next if method == 'create' # Skip the create method
|
48
|
+
queue = queue_for(clazz, method)
|
49
|
+
self[queue] = clazz.new
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "class and method routing" do
|
4
|
+
specify "should create a queue called utils:echo for a Util class that subclasses worker and has the method echo" do
|
5
|
+
routing = Workling::Routing::ClassAndMethodRouting.new
|
6
|
+
routing['utils__echo'].class.to_s.should.equal "Util"
|
7
|
+
end
|
8
|
+
|
9
|
+
specify "should create a queue called analytics:invites:sent for an Analytics::Invites class that subclasses worker and has the method sent" do
|
10
|
+
routing = Workling::Routing::ClassAndMethodRouting.new
|
11
|
+
routing['analytics__invites__sent'].class.to_s.should.equal "Analytics::Invites"
|
12
|
+
end
|
13
|
+
|
14
|
+
specify "queue_names_routing_class should return all queue names associated with a class" do
|
15
|
+
routing = Workling::Routing::ClassAndMethodRouting.new
|
16
|
+
routing.queue_names_routing_class(Util).should.include 'utils__echo'
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'workling/clients/base'
|
2
|
+
|
3
|
+
module Workling
|
4
|
+
module Clients
|
5
|
+
class MemoryQueueClient < Workling::Clients::Base
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@subscribers ||= {}
|
9
|
+
@queues ||= {}
|
10
|
+
end
|
11
|
+
|
12
|
+
# collects the worker blocks in a hash
|
13
|
+
def subscribe(work_type, &block)
|
14
|
+
@subscribers[work_type] = block
|
15
|
+
end
|
16
|
+
|
17
|
+
# immediately invokes the required worker block
|
18
|
+
def request(work_type, arguments)
|
19
|
+
if subscription = @subscribers[work_type]
|
20
|
+
subscription.call(arguments)
|
21
|
+
else
|
22
|
+
@queues[work_type] ||= []
|
23
|
+
@queues[work_type] << arguments
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def retrieve(work_type)
|
28
|
+
queue = @queues[work_type]
|
29
|
+
queue.pop if queue
|
30
|
+
end
|
31
|
+
|
32
|
+
def connect; true; end
|
33
|
+
def close; true; end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "discovery" do
|
4
|
+
specify "should discover the Util workling, since it subclasses Workling::Base and is on the configured Workling load path." do
|
5
|
+
discovered = Workling::Discovery.discovered
|
6
|
+
discovered.map(&:to_s).should.include "Util"
|
7
|
+
end
|
8
|
+
|
9
|
+
specify "should not discover non-worker classes" do
|
10
|
+
discovered = Workling::Discovery.discovered
|
11
|
+
discovered.all? { |clazz| clazz.superclass == Workling::Base }.should.blaming("some discovered classes were not workers").equal true
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "the invoker 'basic poller'" do
|
4
|
+
setup do
|
5
|
+
routing = Workling::Routing::ClassAndMethodRouting.new
|
6
|
+
@client = Workling::Clients::MemoryQueueClient.new
|
7
|
+
@client.connect
|
8
|
+
@invoker = Workling::Remote::Invokers::BasicPoller.new(routing, @client.class)
|
9
|
+
end
|
10
|
+
|
11
|
+
specify "should not explode when listen is called, and stop nicely when stop is called. " do
|
12
|
+
connection = mock()
|
13
|
+
connection.expects(:active?).at_least_once.returns(true)
|
14
|
+
ActiveRecord::Base.expects(:connection).at_least_once.returns(connection)
|
15
|
+
|
16
|
+
client = mock()
|
17
|
+
client.expects(:retrieve).at_least_once.returns("hi")
|
18
|
+
client.expects(:connect).at_least_once.returns(true)
|
19
|
+
client.expects(:close).at_least_once.returns(true)
|
20
|
+
Workling::Clients::MemoryQueueClient.expects(:new).at_least_once.returns(client)
|
21
|
+
|
22
|
+
# Don't take longer than 10 seconds to shut this down.
|
23
|
+
Timeout::timeout(10) do
|
24
|
+
listener = Thread.new { @invoker.listen }
|
25
|
+
@invoker.stop
|
26
|
+
listener.join
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "the invoker 'eventmachine subscription'" do
|
4
|
+
setup do
|
5
|
+
routing = Workling::Routing::ClassAndMethodRouting.new
|
6
|
+
@client_class = Workling::Clients::MemoryQueueClient
|
7
|
+
@client = @client_class.new
|
8
|
+
@client.connect
|
9
|
+
@invoker = Workling::Remote::Invokers::EventmachineSubscriber.new(routing, @client_class)
|
10
|
+
end
|
11
|
+
|
12
|
+
specify "should invoke Util.echo with the arg 'hello' if the string 'hello' is set onto the queue utils__echo" do
|
13
|
+
|
14
|
+
# make sure all new instances point to the same client. that way, state is shared
|
15
|
+
Workling::Clients::MemoryQueueClient.expects(:new).at_least_once.returns @client
|
16
|
+
Util.any_instance.expects(:echo).once.with({ :message => "hello" })
|
17
|
+
|
18
|
+
# Don't take longer than 10 seconds to shut this down.
|
19
|
+
Timeout::timeout(10) do
|
20
|
+
listener = Thread.new { @invoker.listen }
|
21
|
+
@client.request("utils__echo", { :message => "hello" })
|
22
|
+
@invoker.stop
|
23
|
+
listener.join
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "the invoker 'threaded poller'" do
|
4
|
+
setup do
|
5
|
+
routing = Workling::Routing::ClassAndMethodRouting.new
|
6
|
+
@client = Workling::Clients::MemoryQueueClient.new
|
7
|
+
@client.connect
|
8
|
+
@invoker = Workling::Remote::Invokers::ThreadedPoller.new(routing, @client.class)
|
9
|
+
end
|
10
|
+
|
11
|
+
specify "should invoke Util.echo with the arg 'hello' if the string 'hello' is set onto the queue utils__echo" do
|
12
|
+
Util.any_instance.stubs(:echo).with("hello")
|
13
|
+
@client.request("utils__echo", "hello")
|
14
|
+
@invoker.dispatch!(@client, Util)
|
15
|
+
end
|
16
|
+
|
17
|
+
specify "should not explode when listen is called, and stop nicely when stop is called. " do
|
18
|
+
connection = mock()
|
19
|
+
connection.expects(:active?).at_least_once.returns(true)
|
20
|
+
ActiveRecord::Base.expects(:connection).at_least_once.returns(connection)
|
21
|
+
|
22
|
+
client = mock()
|
23
|
+
client.expects(:retrieve).at_least_once.returns("hi")
|
24
|
+
client.expects(:connect).at_least_once.returns(true)
|
25
|
+
Workling::Clients::MemoryQueueClient.expects(:new).at_least_once.returns(client)
|
26
|
+
|
27
|
+
# Don't take longer than 10 seconds to shut this down.
|
28
|
+
Timeout::timeout(10) do
|
29
|
+
listener = Thread.new { @invoker.listen }
|
30
|
+
@invoker.stop
|
31
|
+
listener.join
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
context "The memcachequeue client" do
|
4
|
+
specify "should be able to connect to multiple queue instances" do
|
5
|
+
Workling.send :class_variable_set, "@@config", { :listens_on => "localhost:12345, 127.0.0.1:12346", :memcache_options => { :namespace => "myapp_development" } }
|
6
|
+
client = Workling::Clients::MemcacheQueueClient.new
|
7
|
+
client.connect
|
8
|
+
|
9
|
+
client.queueserver_urls.should.equal ["localhost:12345", "127.0.0.1:12346"]
|
10
|
+
client.connection.servers.first.host.should == "localhost"
|
11
|
+
client.connection.servers.first.port.should == 12345
|
12
|
+
client.connection.servers[1].host.should == "127.0.0.1"
|
13
|
+
client.connection.servers[1].port.should == 12346
|
14
|
+
client.connection.namespace.should.equal "myapp_development"
|
15
|
+
end
|
16
|
+
|
17
|
+
specify "should load it's config as well as any given MemCache options from RAILS_ENV/config/workling.yml" do
|
18
|
+
Workling.send :class_variable_set, "@@config", { :listens_on => "localhost:12345", :memcache_options => { :namespace => "myapp_development" } }
|
19
|
+
client = Workling::Clients::MemcacheQueueClient.new
|
20
|
+
client.connect
|
21
|
+
|
22
|
+
client.queueserver_urls.should.equal ["localhost:12345"]
|
23
|
+
client.connection.servers.first.host.should == "localhost"
|
24
|
+
client.connection.servers.first.port.should == 12345
|
25
|
+
client.connection.namespace.should.equal "myapp_development"
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
specify "should load it's config correctly if no memcache options are given" do
|
30
|
+
Workling.send :class_variable_set, "@@config", { :listens_on => "localhost:12345" }
|
31
|
+
client = Workling::Clients::MemcacheQueueClient.new
|
32
|
+
client.connect
|
33
|
+
|
34
|
+
client.queueserver_urls.should.equal ["localhost:12345"]
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "the memory return store" do
|
4
|
+
specify "should be able to store a value with a key, and then retrieve that same value with the same key." do
|
5
|
+
store = Workling::Return::Store::MemoryReturnStore.new
|
6
|
+
key, value = :gender, :undecided
|
7
|
+
store.set(key, value)
|
8
|
+
store.get(key).should.equal(value)
|
9
|
+
end
|
10
|
+
|
11
|
+
specify "should delete values in the store once they have been get()tted." do
|
12
|
+
store = Workling::Return::Store::MemoryReturnStore.new
|
13
|
+
key, value = :gender, :undecided
|
14
|
+
store.set(key, value)
|
15
|
+
store.get(key)
|
16
|
+
store.get(key).should.equal nil
|
17
|
+
end
|
18
|
+
|
19
|
+
specify "should return nothing for a key that is not in the store" do
|
20
|
+
store = Workling::Return::Store::MemoryReturnStore.new
|
21
|
+
store.get(:bollocks).should.equal nil
|
22
|
+
end
|
23
|
+
end
|
data/test/mocks/spawn.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
context "The not remote runner" do
|
4
|
+
specify "should swallow exceptions raised in the workling" do
|
5
|
+
old_dispatcher = Workling::Remote.dispatcher
|
6
|
+
|
7
|
+
Workling::Remote.dispatcher = Workling::Remote::Runners::NotRemoteRunner.new
|
8
|
+
Workling::Remote.run(:util, :faulty)
|
9
|
+
Workling::Remote.dispatcher = old_dispatcher # set back to whence we came
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
context "the remote runner" do
|
4
|
+
specify "should be able to invoke a task on a worker" do
|
5
|
+
Util.any_instance.stubs(:echo).with("hello")
|
6
|
+
Workling::Remote.run(:util, :echo, "hello")
|
7
|
+
end
|
8
|
+
|
9
|
+
specify "should invoke the dispatcher set up in Workling::Remote.dispatcher" do
|
10
|
+
clazz, method, options = :util, :echo, { :message => "somebody_came@along.com" }
|
11
|
+
old_dispatcher = Workling::Remote.dispatcher
|
12
|
+
dispatcher = mock
|
13
|
+
dispatcher.expects(:run).with(clazz, method, options)
|
14
|
+
Workling::Remote.dispatcher = dispatcher
|
15
|
+
Workling::Remote.run(clazz, method, options)
|
16
|
+
Workling::Remote.dispatcher = old_dispatcher # set back to whence we came
|
17
|
+
end
|
18
|
+
|
19
|
+
specify "should, when being tested, use the default remote runner by when no runner was explicitly set. " do
|
20
|
+
Workling::Remote.dispatcher.class.should.equal Workling.default_runner.class
|
21
|
+
end
|
22
|
+
|
23
|
+
specify "should raise a Workling::WorklingNotFoundError if it is invoked with a worker key that cannot be constantized" do
|
24
|
+
should.raise Workling::WorklingNotFoundError do
|
25
|
+
Workling::Remote.run(:quatsch_mit_sosse, :fiddle_di_liddle)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
specify "should raise a Workling::WorklingNotFoundError if it is invoked with a valid worker key but the method is not defined on that worker" do
|
30
|
+
dispatcher = Workling::Remote.dispatcher
|
31
|
+
Workling::Remote.dispatcher = Workling::Remote::Runners::ThreadRunner.new # simulates a remote runner (workling in another context)
|
32
|
+
|
33
|
+
should.raise Workling::WorklingNotFoundError do
|
34
|
+
Workling::Remote.run(:util, :sau_sack)
|
35
|
+
end
|
36
|
+
|
37
|
+
Workling::Remote.dispatcher = dispatcher
|
38
|
+
end
|
39
|
+
|
40
|
+
specify "should invoke work when called in this way: YourWorkling.asynch_your_method(options)" do
|
41
|
+
Util.any_instance.expects(:echo).once
|
42
|
+
Util.asynch_echo
|
43
|
+
end
|
44
|
+
|
45
|
+
specify "should invoke work with the arguments intact when called in this way: YourWorkling.asynch_your_method(options)" do
|
46
|
+
stuffing = { :description => "toasted breadcrumbs with dill" }
|
47
|
+
Util.any_instance.expects(:stuffing).with(stuffing).once
|
48
|
+
Util.asynch_stuffing(stuffing)
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
context "The return store" do
|
4
|
+
specify "should set a value on the current store when invoked like this: Workling::Return::Store.set(:key, 'value')" do
|
5
|
+
Workling::Return::Store.set(:key, :value)
|
6
|
+
Workling::Return::Store.get(:key).should.equal :value
|
7
|
+
end
|
8
|
+
|
9
|
+
specify "should get a value on the current store when invoked like this: Workling::Return::Store.get(:key)" do
|
10
|
+
Workling::Return::Store.set(:key, :value)
|
11
|
+
Workling::Return::Store.get(:key).should.equal :value
|
12
|
+
end
|
13
|
+
|
14
|
+
specify "should set a value on the current store when invoked like this: Workling.return.set(:key, 'value')" do
|
15
|
+
Workling.return.set(:key, :value)
|
16
|
+
Workling.return.get(:key).should.equal :value
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'workling/remote/runners/base'
|
2
|
+
|
3
|
+
#
|
4
|
+
# Spawns a Thread. Used for Tests only, to simulate a remote runner more realistically.
|
5
|
+
#
|
6
|
+
module Workling
|
7
|
+
module Remote
|
8
|
+
module Runners
|
9
|
+
class ThreadRunner < Workling::Remote::Runners::Base
|
10
|
+
|
11
|
+
# spawns a thread.
|
12
|
+
def run(clazz, method, options = {})
|
13
|
+
Thread.new {
|
14
|
+
dispatch!(clazz, method, options)
|
15
|
+
}
|
16
|
+
|
17
|
+
return nil
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|