workling 0.4.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/CHANGES.markdown +82 -0
  2. data/README.markdown +543 -0
  3. data/TODO.markdown +27 -0
  4. data/VERSION.yml +4 -0
  5. data/bin/workling_client +29 -0
  6. data/contrib/bj_invoker.rb +11 -0
  7. data/contrib/starling_status.rb +37 -0
  8. data/lib/extensions/cattr_accessor.rb +51 -0
  9. data/lib/extensions/mattr_accessor.rb +55 -0
  10. data/lib/workling.rb +213 -0
  11. data/lib/workling/base.rb +110 -0
  12. data/lib/workling/clients/amqp_client.rb +51 -0
  13. data/lib/workling/clients/amqp_exchange_client.rb +58 -0
  14. data/lib/workling/clients/backgroundjob_client.rb +25 -0
  15. data/lib/workling/clients/base.rb +89 -0
  16. data/lib/workling/clients/broker_base.rb +63 -0
  17. data/lib/workling/clients/memcache_queue_client.rb +104 -0
  18. data/lib/workling/clients/memory_queue_client.rb +34 -0
  19. data/lib/workling/clients/not_client.rb +14 -0
  20. data/lib/workling/clients/not_remote_client.rb +17 -0
  21. data/lib/workling/clients/rude_q_client.rb +47 -0
  22. data/lib/workling/clients/spawn_client.rb +46 -0
  23. data/lib/workling/clients/sqs_client.rb +163 -0
  24. data/lib/workling/clients/thread_client.rb +18 -0
  25. data/lib/workling/clients/xmpp_client.rb +110 -0
  26. data/lib/workling/discovery.rb +16 -0
  27. data/lib/workling/invokers/amqp_single_subscriber.rb +42 -0
  28. data/lib/workling/invokers/base.rb +124 -0
  29. data/lib/workling/invokers/basic_poller.rb +38 -0
  30. data/lib/workling/invokers/eventmachine_subscriber.rb +38 -0
  31. data/lib/workling/invokers/looped_subscriber.rb +34 -0
  32. data/lib/workling/invokers/thread_pool_poller.rb +165 -0
  33. data/lib/workling/invokers/threaded_poller.rb +149 -0
  34. data/lib/workling/remote.rb +38 -0
  35. data/lib/workling/return/store/base.rb +42 -0
  36. data/lib/workling/return/store/iterator.rb +24 -0
  37. data/lib/workling/return/store/memory_return_store.rb +24 -0
  38. data/lib/workling/return/store/starling_return_store.rb +30 -0
  39. data/lib/workling/routing/base.rb +13 -0
  40. data/lib/workling/routing/class_and_method_routing.rb +55 -0
  41. data/lib/workling/routing/static_routing.rb +43 -0
  42. data/lib/workling_daemon.rb +111 -0
  43. metadata +96 -0
@@ -0,0 +1,124 @@
1
+ #
2
+ # Invokers are responsible for
3
+ #
4
+ # 1. grabbing work off a job broker (such as a starling or rabbitmq server).
5
+ # 2. routing (mapping) that work onto the correct worker method.
6
+ # 3. invoking the worker method, passing any arguments that came off the broker.
7
+ #
8
+ # Invokers should implement their own concurrency strategies. For example,
9
+ # The there is a ThreadedPoller which starts a thread for each Worker class.
10
+ #
11
+ # This base Invoker class defines the methods an Invoker needs to implement.
12
+ #
13
+ module Workling
14
+ module Invokers
15
+ class Base
16
+
17
+ attr_accessor :sleep_time, :reset_time
18
+
19
+ #
20
+ # call up with super in the subclass constructor.
21
+ #
22
+ def initialize(routing, client_class)
23
+ @routing = routing
24
+ @client_class = client_class
25
+ @sleep_time = Workling.config[:sleep_time] || 2
26
+ @reset_time = Workling.config[:reset_time] || 30
27
+ @@mutex ||= Mutex.new
28
+ end
29
+
30
+ #
31
+ # Starts main Invoker Loop. The invoker runs until stop() is called.
32
+ #
33
+ def listen
34
+ raise NotImplementedError.new("Implement listen() in your Invoker. ")
35
+ end
36
+
37
+ #
38
+ # Gracefully stops the Invoker. The currently executing Jobs should be allowed
39
+ # to finish.
40
+ #
41
+ def stop
42
+ raise NotImplementedError.new("Implement stop() in your Invoker. ")
43
+ end
44
+
45
+ #
46
+ # Runs the worker method, given
47
+ #
48
+ # type: the worker route
49
+ # args: the arguments to be passed into the worker method.
50
+ #
51
+ def run(type, args)
52
+ worker = @routing[type]
53
+ method = @routing.method_name(type)
54
+ worker.dispatch_to_worker_method(method, args)
55
+ end
56
+
57
+ # returns the Workling::Base.logger
58
+ def logger; Workling::Base.logger; end
59
+
60
+ protected
61
+
62
+ # handle opening and closing of client. pass code block to this method.
63
+ def connect
64
+ @client = @client_class.new
65
+ @client.connect
66
+
67
+ begin
68
+ yield
69
+ ensure
70
+ @client.close
71
+ ActiveRecord::Base.verify_active_connections! if defined?(ActiveRecord::Base)
72
+ end
73
+ end
74
+
75
+ #
76
+ # Loops through the available routes, yielding for each route.
77
+ # This continues until @shutdown is set on this instance.
78
+ #
79
+ def loop_routes
80
+ while(!@shutdown) do
81
+ ensure_activerecord_connection
82
+
83
+ routes.each do |route|
84
+ break if @shutdown
85
+ yield route
86
+ end
87
+
88
+ sleep self.sleep_time
89
+ end
90
+ end
91
+
92
+ #
93
+ # Returns the complete set of active routes
94
+ #
95
+ def routes
96
+ @active_routes ||= Workling::Discovery.discovered.map { |clazz| @routing.queue_names_routing_class(clazz) }.flatten
97
+ end
98
+
99
+ # Thanks for this Brent!
100
+ #
101
+ # ...Just a heads up, due to how rails’ MySQL adapter handles this
102
+ # call ‘ActiveRecord::Base.connection.active?’, you’ll need
103
+ # to wrap the code that checks for a connection in in a mutex.
104
+ #
105
+ # ....I noticed this while working with a multi-core machine that
106
+ # was spawning multiple workling threads. Some of my workling
107
+ # threads would hit serious issues at this block of code without
108
+ # the mutex.
109
+ #
110
+ def ensure_activerecord_connection
111
+ if defined?(ActiveRecord::Base)
112
+ @@mutex.synchronize do
113
+ unless ActiveRecord::Base.connection.active? # Keep MySQL connection alive
114
+ unless ActiveRecord::Base.connection.reconnect!
115
+ logger.fatal("Failed - Database not available!")
116
+ break
117
+ end
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,38 @@
1
+ #
2
+ # A basic polling invoker.
3
+ #
4
+ module Workling
5
+ module Invokers
6
+ class BasicPoller < Workling::Invokers::Base
7
+
8
+ #
9
+ # set up client, sleep time
10
+ #
11
+ def initialize(routing, client_class)
12
+ super
13
+ end
14
+
15
+ #
16
+ # Starts main Invoker Loop. The invoker runs until stop() is called.
17
+ #
18
+ def listen
19
+ connect do
20
+ loop_routes do |route|
21
+ if args = @client.retrieve(route)
22
+ run(route, args)
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ #
29
+ # Gracefully stops the Invoker. The currently executing Jobs should be allowed
30
+ # to finish.
31
+ #
32
+ def stop
33
+ @shutdown = true
34
+ end
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,38 @@
1
+ require 'eventmachine'
2
+
3
+ #
4
+ # Subscribes the workers to the correct queues.
5
+ #
6
+ module Workling
7
+ module Invokers
8
+ class EventmachineSubscriber < Workling::Invokers::Base
9
+
10
+ def initialize(routing, client_class)
11
+ super
12
+ end
13
+
14
+ #
15
+ # Starts EM loop and sets up subscription callbacks for workers.
16
+ #
17
+ def listen
18
+ EM.run do
19
+ connect do
20
+ routes.each do |route|
21
+ @client.subscribe(route) do |args|
22
+ begin
23
+ run(route, args)
24
+ rescue
25
+ logger.error("EventmachineSubscriber listen error on #{route}: #{$!}")
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+
33
+ def stop
34
+ EM.stop if EM.reactor_running?
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,34 @@
1
+ #
2
+ # Subscribes the workers to the correct queues.
3
+ #
4
+ module Workling
5
+ module Invokers
6
+ class LoopedSubscriber < Workling::Invokers::Base
7
+
8
+ def initialize(routing, client_class)
9
+ super
10
+ end
11
+
12
+ #
13
+ # Starts EM loop and sets up subscription callbacks for workers.
14
+ #
15
+ def listen
16
+ connect do
17
+ routes.each do |route|
18
+ @client.subscribe(route) do |args|
19
+ run(route, args)
20
+ end
21
+ end
22
+
23
+ loop do
24
+ sleep 1
25
+ end
26
+ end
27
+ end
28
+
29
+ def stop
30
+
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,165 @@
1
+ require 'thread'
2
+ require 'mutex_m'
3
+
4
+ #
5
+ # A polling invoker that executes the jobs using a thread pool.
6
+ #
7
+ # This invoker was designed for long running tasks and Rails 2.2. It is expected
8
+ # that each worker will manage it's own database connections using ActiveRecord::Base.connection_pool.
9
+ #
10
+ # This implementation isn't using most of the features provided by Invokers::Base
11
+ # because of it's assumptions about database connections
12
+ #
13
+ # Example:
14
+ # user = nil
15
+ # ActiveRecord::Base.connection_pool.with_connection do
16
+ # user = User.find(options[:id])
17
+ # end
18
+ #
19
+ # # Do long running stuff here
20
+ # user.wait_until_birthday
21
+ # user.buy_gift 'BRAWNDO!'
22
+ #
23
+ # ActiveRecord::Base.connection_pool.with_connection do
24
+ # user.save
25
+ # end
26
+ #
27
+ module Workling
28
+ module Invokers
29
+ class ThreadPoolPoller < Workling::Invokers::Base
30
+ attr_reader :sleep_time, :reset_time, :pool_capacity
31
+
32
+ def initialize(routing, client_class)
33
+ @routing = routing
34
+ @client_class = client_class
35
+
36
+ # Grab settings out of the config file
37
+ @sleep_time = (Workling.config[:sleep_time] || 2).to_f
38
+ @reset_time = (Workling.config[:reset_time] || 30).to_f
39
+
40
+ # Pool of polling threads
41
+ @pollers = []
42
+ @pollers.extend(Mutex_m)
43
+
44
+ # Pool of worker threads
45
+ @workers = []
46
+ @workers.extend(Mutex_m)
47
+
48
+ # Connection to the job queue
49
+ @pool_capacity = (Workling.config[:pool_size] || 25).to_i
50
+ end
51
+
52
+ # Start up the checking for items on the queue. Will block until stop is called and all pollers
53
+ # and workers have finished execution.
54
+ def listen
55
+ logger.info("Starting ThreadPoolPoller...")
56
+
57
+ # Determine which queues need checking
58
+ Workling::Discovery.discovered.map do |klass|
59
+ @pollers.synchronize do
60
+ # Polls the backing queue for jobs to be done
61
+ @pollers << Thread.new do
62
+ poller_thread(@routing.queue_names_routing_class(klass))
63
+ end
64
+ end
65
+ end
66
+
67
+ # Wait for the poller and all outstanding workers to finish.
68
+ #
69
+ # This is a little tricky because we're doing some synchronization on pollers... but
70
+ # the list of pollers is never modified after being setup above.
71
+ @pollers.synchronize { @pollers.dup }.each { |p| p.join }
72
+ @pollers.synchronize { @pollers.clear }
73
+ logger.info("Pollers have all finished")
74
+
75
+ @workers.synchronize { @workers.dup }.each { |w| w.join }
76
+ logger.info("Worker threads have all finished")
77
+ end
78
+
79
+ # Instructs the thread pool poller to stop checking for new jobs on the backing queue.
80
+ def stop
81
+ logger.info("Stopping thread pool invoker pollers and workers...")
82
+ @pollers.synchronize { @pollers.each { |p| p[:shutdown] = true } }
83
+ end
84
+
85
+ # Set pool_size in workling config to adjust the maximum number of threads in the pool
86
+ def workers_available?
87
+ worker_threads < @pool_capacity
88
+ end
89
+
90
+ # Number of correctly active worker threads
91
+ def worker_threads
92
+ @workers.synchronize { @workers.size }
93
+ end
94
+
95
+ # Number of currently active polling threads
96
+ def poller_threads
97
+ @pollers.synchronize { @pollers.size }
98
+ end
99
+
100
+ private
101
+
102
+ def poller_thread(queues)
103
+ # Make sure queues is an array
104
+ queues = [queues].flatten!
105
+
106
+ # Connect our client to the backing queue
107
+ client = @client_class.new
108
+ client.connect
109
+ logger.info("** Starting client #{ client.class } for #{ queues.inspect }") if logger.info?
110
+
111
+ # Poll each queue for new items
112
+ while(!Thread.current[:shutdown]) do
113
+ # Check each queue for a job posting
114
+ queues.each do |queue|
115
+ break if Thread.current[:shutdown]
116
+
117
+ begin
118
+ # Take a job off the queue and execute it in a new worker thread.
119
+ #
120
+ # Don't pop any jobs off the backing queue if the thread pool
121
+ # is full. This keeps them on the master queue so other instances
122
+ # can process them.
123
+ while(workers_available? && (options = client.retrieve(queue)))
124
+ logger.debug("#{queue} received job #{ options.inspect }") if logger.debug?
125
+
126
+ @workers.synchronize do
127
+ @workers << Thread.new do
128
+ begin
129
+ # Execute the job
130
+ run(queue, options)
131
+ rescue Exception => e
132
+ # Log the exception since there isn't much else we can do about it at this point
133
+ logger.error(e) if logger.error?
134
+ ensure
135
+ # Make sure the current thread's connection gets released
136
+ if(ActiveRecord::Base.connection_pool)
137
+ ActiveRecord::Base.connection_pool.release_connection
138
+ end
139
+
140
+ # Remove this thread from the list of active workers
141
+ @workers.synchronize do
142
+ @workers.delete(Thread.current)
143
+ end
144
+ end
145
+ end
146
+ end
147
+
148
+ # Break out of the checks early if shutdown was called
149
+ break if Thread.current[:shutdown]
150
+ end
151
+ rescue Workling::WorklingError => e
152
+ logger.error("FAILED to connect with queue #{ queue }: #{ e } }") if logger.error?
153
+ sleep(@reset_time)
154
+
155
+ # FIXME: This will _definitely_ blow up with AMQP since there is no reset call
156
+ client.reset
157
+ end
158
+ end
159
+
160
+ sleep(@sleep_time) unless Thread.current[:shutdown]
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,149 @@
1
+ #
2
+ # A threaded polling Invoker.
3
+ #
4
+ # TODO: refactor this to make use of the base class.
5
+ #
6
+ module Workling
7
+ module Invokers
8
+ class ThreadedPoller < Workling::Invokers::Base
9
+
10
+ cattr_accessor :sleep_time, :reset_time
11
+
12
+ def initialize(routing, client_class)
13
+ super
14
+
15
+ ThreadedPoller.sleep_time = Workling.config[:sleep_time] || 2
16
+ ThreadedPoller.reset_time = Workling.config[:reset_time] || 30
17
+
18
+ @workers = ThreadGroup.new
19
+ @mutex = Mutex.new
20
+ end
21
+
22
+ def listen
23
+ # Create a thread for each worker.
24
+ Workling::Discovery.discovered.each do |clazz|
25
+ logger.debug("Discovered listener #{clazz}")
26
+ @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
27
+ end
28
+
29
+ # Wait for all workers to complete
30
+ @workers.list.each { |t| t.join }
31
+
32
+ logger.debug("Reaped listener threads. ")
33
+
34
+ # Clean up all the connections.
35
+ if defined?(ActiveRecord::Base)
36
+ ActiveRecord::Base.verify_active_connections!
37
+ end
38
+
39
+ logger.debug("Cleaned up connection: out!")
40
+ end
41
+
42
+ # Check if all Worker threads have been started.
43
+ def started?
44
+ logger.debug("checking if started... list size is #{ worker_threads }")
45
+ Workling::Discovery.discovered.size == worker_threads
46
+ end
47
+
48
+ # number of worker threads running
49
+ def worker_threads
50
+ @workers.list.size
51
+ end
52
+
53
+ # Gracefully stop processing
54
+ def stop
55
+ logger.info("stopping threaded poller...")
56
+ sleep 1 until started? # give it a chance to start up before shutting down.
57
+ logger.info("Giving Listener Threads a chance to shut down. This may take a while... ")
58
+ @workers.list.each { |w| w[:shutdown] = true }
59
+ logger.info("Listener threads were shut down. ")
60
+ end
61
+
62
+ # Listen for one worker class
63
+ def clazz_listen(clazz)
64
+ logger.debug("Listener thread #{clazz.name} started")
65
+
66
+ # Read thread configuration if available
67
+ if Workling.config.has_key?(:listeners)
68
+ if Workling.config[:listeners].has_key?(clazz.to_s)
69
+ config = Workling.config[:listeners][clazz.to_s].symbolize_keys
70
+ thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
71
+ Thread.current.priority = config[:priority] if config.has_key?(:priority)
72
+ end
73
+ end
74
+
75
+ thread_sleep_time ||= self.class.sleep_time
76
+
77
+ # Setup connection to client (one per thread)
78
+ connection = @client_class.new
79
+ connection.connect
80
+ logger.info("** Starting client #{ connection.class } for #{clazz.name} queue")
81
+
82
+ # Start dispatching those messages
83
+ while (!Thread.current[:shutdown]) do
84
+ begin
85
+
86
+ # Thanks for this Brent!
87
+ #
88
+ # ...Just a heads up, due to how rails’ MySQL adapter handles this
89
+ # call ‘ActiveRecord::Base.connection.active?’, you’ll need
90
+ # to wrap the code that checks for a connection in in a mutex.
91
+ #
92
+ # ....I noticed this while working with a multi-core machine that
93
+ # was spawning multiple workling threads. Some of my workling
94
+ # threads would hit serious issues at this block of code without
95
+ # the mutex.
96
+ #
97
+ if defined?(ActiveRecord::Base)
98
+ @mutex.synchronize do
99
+ unless ActiveRecord::Base.connection.active? # Keep MySQL connection alive
100
+ unless ActiveRecord::Base.connection.reconnect!
101
+ logger.fatal("Failed - Database not available!")
102
+ break
103
+ end
104
+ end
105
+ end
106
+ end
107
+
108
+ # Dispatch and process the messages
109
+ n = dispatch!(connection, clazz)
110
+ logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
111
+ sleep(thread_sleep_time) unless n > 0
112
+
113
+ # If there is a memcache error, hang for a bit to give it a chance to fire up again
114
+ # and reset the connection.
115
+ rescue Workling::WorklingConnectionError
116
+ logger.warn("Listener thread #{clazz.name} failed to connect. Resetting connection.")
117
+ sleep(self.class.reset_time)
118
+ connection.reset
119
+ end
120
+ end
121
+
122
+ logger.debug("Listener thread #{clazz.name} ended")
123
+ end
124
+
125
+ # Dispatcher for one worker class. Will throw MemCacheError if unable to connect.
126
+ # Returns the number of worker methods called
127
+ def dispatch!(connection, clazz)
128
+ n = 0
129
+ for queue in @routing.queue_names_routing_class(clazz)
130
+ begin
131
+ result = connection.retrieve(queue)
132
+ if result
133
+ n += 1
134
+ handler = @routing[queue]
135
+ method_name = @routing.method_name(queue)
136
+ logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
137
+ handler.dispatch_to_worker_method(method_name, result)
138
+ end
139
+ rescue Workling::WorklingError => e
140
+ logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
141
+ raise e
142
+ end
143
+ end
144
+
145
+ return n
146
+ end
147
+ end
148
+ end
149
+ end