workling 0.4.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/CHANGES.markdown +82 -0
  2. data/README.markdown +543 -0
  3. data/TODO.markdown +27 -0
  4. data/VERSION.yml +4 -0
  5. data/bin/workling_client +29 -0
  6. data/contrib/bj_invoker.rb +11 -0
  7. data/contrib/starling_status.rb +37 -0
  8. data/lib/extensions/cattr_accessor.rb +51 -0
  9. data/lib/extensions/mattr_accessor.rb +55 -0
  10. data/lib/workling.rb +213 -0
  11. data/lib/workling/base.rb +110 -0
  12. data/lib/workling/clients/amqp_client.rb +51 -0
  13. data/lib/workling/clients/amqp_exchange_client.rb +58 -0
  14. data/lib/workling/clients/backgroundjob_client.rb +25 -0
  15. data/lib/workling/clients/base.rb +89 -0
  16. data/lib/workling/clients/broker_base.rb +63 -0
  17. data/lib/workling/clients/memcache_queue_client.rb +104 -0
  18. data/lib/workling/clients/memory_queue_client.rb +34 -0
  19. data/lib/workling/clients/not_client.rb +14 -0
  20. data/lib/workling/clients/not_remote_client.rb +17 -0
  21. data/lib/workling/clients/rude_q_client.rb +47 -0
  22. data/lib/workling/clients/spawn_client.rb +46 -0
  23. data/lib/workling/clients/sqs_client.rb +163 -0
  24. data/lib/workling/clients/thread_client.rb +18 -0
  25. data/lib/workling/clients/xmpp_client.rb +110 -0
  26. data/lib/workling/discovery.rb +16 -0
  27. data/lib/workling/invokers/amqp_single_subscriber.rb +42 -0
  28. data/lib/workling/invokers/base.rb +124 -0
  29. data/lib/workling/invokers/basic_poller.rb +38 -0
  30. data/lib/workling/invokers/eventmachine_subscriber.rb +38 -0
  31. data/lib/workling/invokers/looped_subscriber.rb +34 -0
  32. data/lib/workling/invokers/thread_pool_poller.rb +165 -0
  33. data/lib/workling/invokers/threaded_poller.rb +149 -0
  34. data/lib/workling/remote.rb +38 -0
  35. data/lib/workling/return/store/base.rb +42 -0
  36. data/lib/workling/return/store/iterator.rb +24 -0
  37. data/lib/workling/return/store/memory_return_store.rb +24 -0
  38. data/lib/workling/return/store/starling_return_store.rb +30 -0
  39. data/lib/workling/routing/base.rb +13 -0
  40. data/lib/workling/routing/class_and_method_routing.rb +55 -0
  41. data/lib/workling/routing/static_routing.rb +43 -0
  42. data/lib/workling_daemon.rb +111 -0
  43. metadata +96 -0
@@ -0,0 +1,51 @@
1
+ #
2
+ # An Ampq client
3
+ #
4
+ module Workling
5
+ module Clients
6
+ class AmqpClient < Workling::Clients::BrokerBase
7
+
8
+ def self.load
9
+ begin
10
+ require 'mq'
11
+ rescue Exception => e
12
+ raise WorklingError.new(
13
+ "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
14
+ "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
15
+ )
16
+ end
17
+ end
18
+
19
+ # starts the client.
20
+ def connect
21
+ begin
22
+ connection = AMQP.start((Workling.config[:amqp_options] ||{}).symbolize_keys)
23
+ @amq = MQ.new connection
24
+ rescue
25
+ raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
26
+ end
27
+ end
28
+
29
+ # no need for explicit closing. when the event loop
30
+ # terminates, the connection is closed anyway.
31
+ def close; true; end
32
+
33
+ # subscribe to a queue
34
+ def subscribe(key)
35
+ @amq.queue(queue_for(key)).subscribe do |value|
36
+ data = Marshal.load(value) rescue value
37
+ yield data
38
+ end
39
+ end
40
+
41
+ # request and retrieve work
42
+ def retrieve(key); @amq.queue(queue_for(key)); end
43
+ def request(key, value); @amq.queue(queue_for(key)).publish(Marshal.dump(value)); end
44
+
45
+ private
46
+ def queue_for(key)
47
+ "#{Workling.config[:prefix]}#{key}"
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,58 @@
1
+ #
2
+ # An Ampq client
3
+ #
4
+ module Workling
5
+ module Clients
6
+ class AmqpExchangeClient < Workling::Clients::BrokerBase
7
+
8
+ def self.load
9
+ begin
10
+ require 'mq'
11
+ rescue Exception => e
12
+ raise WorklingError.new(
13
+ "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
14
+ "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
15
+ )
16
+ end
17
+ end
18
+
19
+ # starts the client.
20
+ def connect
21
+ begin
22
+ @amq = MQ.new
23
+ rescue
24
+ raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
25
+ end
26
+ end
27
+
28
+ # no need for explicit closing. when the event loop
29
+ # terminates, the connection is closed anyway.
30
+ def close; true; end
31
+
32
+ # subscribe to a queue
33
+ def subscribe(key)
34
+ @amq.queue(key).subscribe do |header, body|
35
+ value = YAML.load(body)
36
+ yield value
37
+ end
38
+ end
39
+
40
+ # request and retrieve work
41
+ def retrieve(key)
42
+ @amq.queue(key)
43
+ end
44
+
45
+ # publish message to exchange
46
+ # using the specified routing key
47
+ def request(exchange_name, value)
48
+ exchange_name = "amq.topic"
49
+
50
+ key = value.delete(:routing_key)
51
+ msg = YAML.dump(value)
52
+ exchange = @amq.topic(exchange_name)
53
+ exchange.publish(msg, :routing_key => key)
54
+ end
55
+
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,25 @@
1
+ module Workling
2
+ module Clients
3
+ class BackgroundjobClient < Workling::Clients::Base
4
+
5
+ def self.installed?
6
+ Object.const_defined? "Bj"
7
+ end
8
+
9
+ # passes the job to bj by serializing the options to xml and passing them to
10
+ # ./script/bj_invoker.rb, which in turn routes the deserialized args to the
11
+ # appropriate worker.
12
+ def dispatch(clazz, method, options = {})
13
+ stdin = Workling::Remote.routing.queue_for(clazz, method) +
14
+ " " +
15
+ options.to_xml(:indent => 0, :skip_instruct => true)
16
+
17
+ Bj.submit "./script/runner ./script/bj_invoker.rb",
18
+ :stdin => stdin
19
+
20
+ return nil # that means nothing!
21
+ end
22
+
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,89 @@
1
+ #
2
+ # Clients are responsible for dispatching jobs either to a broker (starling, rabbitmq etc) or invoke them (spawn)
3
+ #
4
+ # Clients that involve a broker should subclass Workling::Clients::BrokerBase
5
+ #
6
+ # Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
7
+ # from a specific type of job.
8
+ #
9
+ module Workling
10
+ module Clients
11
+ class Base
12
+
13
+ #
14
+ # Load the required libraries, for this client
15
+ #
16
+ def self.load
17
+
18
+ end
19
+
20
+
21
+ #
22
+ # See if the libraries required for this client are installed
23
+ #
24
+ def self.installed?
25
+ true
26
+ end
27
+
28
+
29
+ # returns the Workling::Base.logger
30
+ def logger; Workling::Base.logger; end
31
+
32
+
33
+ #
34
+ # Dispatch a job to the client. If this client uses a job broker, then
35
+ # this method should submit it, otherwise it should run the job
36
+ #
37
+ # clazz: Name of the worker class
38
+ # method: Name of the methods on the worker
39
+ # options: optional arguments for the job
40
+ #
41
+ def dispatch(clazz, method, options = {})
42
+ raise NotImplementedError.new("Implement dispatch(clazz, method, options) in your client. ")
43
+ end
44
+
45
+
46
+ #
47
+ # Requests a job on the broker.
48
+ #
49
+ # work_type:
50
+ # arguments: the argument to the worker method
51
+ #
52
+ def request(work_type, arguments)
53
+ raise WorklingError.new("This client does not involve a broker.")
54
+ end
55
+
56
+ #
57
+ # Gets job results off a job broker. Returns nil if there are no results.
58
+ #
59
+ # worker_uid: the uid returned by workling when the work was dispatched
60
+ #
61
+ def retrieve(work_uid)
62
+ raise WorklingError.new("This client does not involve a broker.")
63
+ end
64
+
65
+ #
66
+ # Subscribe to job results in a job broker.
67
+ #
68
+ # worker_type:
69
+ #
70
+ def subscribe(work_type)
71
+ raise WorklingError.new("This client does not involve a broker.")
72
+ end
73
+
74
+ #
75
+ # Opens a connection to the job broker.
76
+ #
77
+ def connect
78
+ raise WorklingError.new("This client does not involve a broker.")
79
+ end
80
+
81
+ #
82
+ # Closes the connection to the job broker.
83
+ #
84
+ def close
85
+ raise WorklingError.new("This client does not involve a broker.")
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,63 @@
1
+ module Workling
2
+ module Clients
3
+ class BrokerBase < Base
4
+
5
+ #
6
+ # Dispatch a job to the client. If this client uses a job broker, then
7
+ # this method should submit it, otherwise it should run the job
8
+ #
9
+ # clazz: Name of the worker class
10
+ # method: Name of the methods on the worker
11
+ # options: optional arguments for the job
12
+ #
13
+ def dispatch(clazz, method, options = {})
14
+ @connected ||= connect
15
+ request(Workling::Remote.routing.queue_for(clazz, method), options)
16
+ return nil
17
+ end
18
+
19
+
20
+ #
21
+ # Requests a job on the broker.
22
+ #
23
+ # work_type:
24
+ # arguments: the argument to the worker method
25
+ #
26
+ def request(work_type, arguments)
27
+ raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
28
+ end
29
+
30
+ #
31
+ # Gets job results off a job broker. Returns nil if there are no results.
32
+ #
33
+ # worker_uid: the uid returned by workling when the work was dispatched
34
+ #
35
+ def retrieve(work_uid)
36
+ raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
37
+ end
38
+
39
+ #
40
+ # Subscribe to job results in a job broker.
41
+ #
42
+ # worker_type:
43
+ #
44
+ def subscribe(work_type)
45
+ raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
46
+ end
47
+
48
+ #
49
+ # Opens a connection to the job broker.
50
+ #
51
+ def connect
52
+ raise NotImplementedError.new("Implement connect() in your client. ")
53
+ end
54
+
55
+ #
56
+ # Closes the connection to the job broker.
57
+ #
58
+ def close
59
+ raise NotImplementedError.new("Implement close() in your client. ")
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,104 @@
1
+ #
2
+ # This client can be used for all Queue Servers that speak Memcached, such as Starling.
3
+ #
4
+ # Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client,
5
+ # or memcache-client, if this is not available. See the README for a discussion of the memcache
6
+ # clients.
7
+ #
8
+ # method_missing delegates all messages through to the underlying memcache connection.
9
+ #
10
+ module Workling
11
+ module Clients
12
+ class MemcacheQueueClient < Workling::Clients::BrokerBase
13
+
14
+ def self.installed?
15
+ begin
16
+ require 'starling'
17
+ rescue LoadError
18
+ end
19
+
20
+ Object.const_defined? "Starling"
21
+ end
22
+
23
+ def self.load
24
+ begin
25
+ gem 'memcache-client'
26
+ require 'memcache'
27
+ rescue Gem::LoadError
28
+ Workling::Base.logger.info "WORKLING: couldn't find memcache-client. Install: \"gem install memcache-client\". "
29
+ end
30
+ end
31
+
32
+ # the url with which the memcache client expects to reach starling
33
+ attr_accessor :queueserver_urls
34
+
35
+ # the memcache connection object
36
+ attr_accessor :connection
37
+
38
+ #
39
+ # the client attempts to connect to queueserver using the configuration options found in
40
+ #
41
+ # Workling.config. this can be configured in config/workling.yml.
42
+ #
43
+ # the initialization code will raise an exception if memcache-client cannot connect
44
+ # to queueserver.
45
+ #
46
+ def connect
47
+ listens_on = Workling.config[:listens_on] || "localhost:22122"
48
+ @queueserver_urls = listens_on.split(',').map { |url| url ? url.strip : url }
49
+ options = [@queueserver_urls, Workling.config[:memcache_options]].compact
50
+ self.connection = ::MemCache.new(*options)
51
+
52
+ raise_unless_connected!
53
+ end
54
+
55
+ # closes the memcache connection
56
+ def close
57
+ self.connection.flush_all
58
+ self.connection.reset
59
+ end
60
+
61
+ # implements the client job request and retrieval
62
+ def request(key, value)
63
+ set(key, value)
64
+ end
65
+
66
+ def retrieve(key)
67
+ begin
68
+ get(key)
69
+ rescue MemCache::MemCacheError => e
70
+ # failed to enqueue, raise a workling error so that it propagates upwards
71
+ raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")
72
+ end
73
+ end
74
+
75
+ private
76
+ # make sure we can actually connect to queueserver on the given port
77
+ def raise_unless_connected!
78
+ begin
79
+ self.connection.stats
80
+ rescue
81
+ raise Workling::QueueserverNotFoundError.new
82
+ end
83
+ end
84
+
85
+ [:get, :set].each do |method|
86
+ class_eval <<-EOS
87
+ def #{method}(*args, &block)
88
+ self.connection.#{method}(*args, &block)
89
+ end
90
+ EOS
91
+ end
92
+
93
+ # delegates directly through to the memcache connection.
94
+ def method_missing(method, *args)
95
+ begin
96
+ self.connection.send(method, *args)
97
+ rescue MemCache::MemCacheError => e
98
+ raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")
99
+ end
100
+ end
101
+
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,34 @@
1
+ module Workling
2
+ module Clients
3
+ class MemoryQueueClient < Workling::Clients::BrokerBase
4
+
5
+ def initialize
6
+ @subscribers ||= {}
7
+ @queues ||= {}
8
+ end
9
+
10
+ # collects the worker blocks in a hash
11
+ def subscribe(work_type, &block)
12
+ @subscribers[work_type] = block
13
+ end
14
+
15
+ # immediately invokes the required worker block
16
+ def request(work_type, arguments)
17
+ if subscription = @subscribers[work_type]
18
+ subscription.call(arguments)
19
+ else
20
+ @queues[work_type] ||= []
21
+ @queues[work_type] << arguments
22
+ end
23
+ end
24
+
25
+ def retrieve(work_type)
26
+ queue = @queues[work_type]
27
+ queue.pop if queue
28
+ end
29
+
30
+ def connect; true; end
31
+ def close; true; end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,14 @@
1
+ #
2
+ # Dont actually run the job...
3
+ #
4
+ module Workling
5
+ module Clients
6
+ class NotClient < Workling::Clients::Base
7
+
8
+ def dispatch(clazz, method, options = {})
9
+ return nil # nada. niente.
10
+ end
11
+
12
+ end
13
+ end
14
+ end