workling 0.4.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES.markdown +82 -0
- data/README.markdown +543 -0
- data/TODO.markdown +27 -0
- data/VERSION.yml +4 -0
- data/bin/workling_client +29 -0
- data/contrib/bj_invoker.rb +11 -0
- data/contrib/starling_status.rb +37 -0
- data/lib/extensions/cattr_accessor.rb +51 -0
- data/lib/extensions/mattr_accessor.rb +55 -0
- data/lib/workling.rb +213 -0
- data/lib/workling/base.rb +110 -0
- data/lib/workling/clients/amqp_client.rb +51 -0
- data/lib/workling/clients/amqp_exchange_client.rb +58 -0
- data/lib/workling/clients/backgroundjob_client.rb +25 -0
- data/lib/workling/clients/base.rb +89 -0
- data/lib/workling/clients/broker_base.rb +63 -0
- data/lib/workling/clients/memcache_queue_client.rb +104 -0
- data/lib/workling/clients/memory_queue_client.rb +34 -0
- data/lib/workling/clients/not_client.rb +14 -0
- data/lib/workling/clients/not_remote_client.rb +17 -0
- data/lib/workling/clients/rude_q_client.rb +47 -0
- data/lib/workling/clients/spawn_client.rb +46 -0
- data/lib/workling/clients/sqs_client.rb +163 -0
- data/lib/workling/clients/thread_client.rb +18 -0
- data/lib/workling/clients/xmpp_client.rb +110 -0
- data/lib/workling/discovery.rb +16 -0
- data/lib/workling/invokers/amqp_single_subscriber.rb +42 -0
- data/lib/workling/invokers/base.rb +124 -0
- data/lib/workling/invokers/basic_poller.rb +38 -0
- data/lib/workling/invokers/eventmachine_subscriber.rb +38 -0
- data/lib/workling/invokers/looped_subscriber.rb +34 -0
- data/lib/workling/invokers/thread_pool_poller.rb +165 -0
- data/lib/workling/invokers/threaded_poller.rb +149 -0
- data/lib/workling/remote.rb +38 -0
- data/lib/workling/return/store/base.rb +42 -0
- data/lib/workling/return/store/iterator.rb +24 -0
- data/lib/workling/return/store/memory_return_store.rb +24 -0
- data/lib/workling/return/store/starling_return_store.rb +30 -0
- data/lib/workling/routing/base.rb +13 -0
- data/lib/workling/routing/class_and_method_routing.rb +55 -0
- data/lib/workling/routing/static_routing.rb +43 -0
- data/lib/workling_daemon.rb +111 -0
- metadata +96 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
|
3
|
+
#
|
4
|
+
# Scoping Module for Runners.
|
5
|
+
#
|
6
|
+
module Workling
|
7
|
+
module Remote
|
8
|
+
|
9
|
+
# which object to use for routing
|
10
|
+
mattr_writer :routing
|
11
|
+
def self.routing
|
12
|
+
@@routing ||= Workling::Routing::ClassAndMethodRouting.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# which client to use for dispatching
|
16
|
+
mattr_accessor :client
|
17
|
+
def self.client
|
18
|
+
@@client ||= Workling.select_and_build_client
|
19
|
+
end
|
20
|
+
|
21
|
+
# generates a unique identifier for this particular job.
|
22
|
+
def self.generate_uid(clazz, method)
|
23
|
+
uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
|
24
|
+
"#{ clazz.to_s.tableize }/#{ method }/#{ uid }".split("/").join(":")
|
25
|
+
end
|
26
|
+
|
27
|
+
# dispatches to a workling. writes the :uid for this work into the options hash, so make
|
28
|
+
# sure you pass in a hash if you want write to a return store in your workling.
|
29
|
+
def self.run(clazz, method, options = {})
|
30
|
+
uid = Workling::Remote.generate_uid(clazz, method)
|
31
|
+
options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
|
32
|
+
Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist.
|
33
|
+
client.dispatch(clazz, method, options)
|
34
|
+
uid
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
#
|
2
|
+
# Basic interface for getting and setting Data which needs to be passed between Workers and
|
3
|
+
# client code.
|
4
|
+
#
|
5
|
+
module Workling
|
6
|
+
module Return
|
7
|
+
module Store
|
8
|
+
mattr_accessor :instance
|
9
|
+
|
10
|
+
# set a value in the store with the given key. delegates to the returnstore.
|
11
|
+
def self.set(key, value)
|
12
|
+
self.instance.set(key, value)
|
13
|
+
end
|
14
|
+
|
15
|
+
# get a value from the store. this should be destructive. delegates to the returnstore.
|
16
|
+
def self.get(key)
|
17
|
+
self.instance.get(key)
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Base Class for Return Stores. Subclasses need to implement set and get.
|
22
|
+
#
|
23
|
+
class Base
|
24
|
+
|
25
|
+
# set a value in the store with the given key.
|
26
|
+
def set(key, value)
|
27
|
+
raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
|
28
|
+
end
|
29
|
+
|
30
|
+
# get a value from the store. this should be destructive.
|
31
|
+
def get(key)
|
32
|
+
raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
|
33
|
+
end
|
34
|
+
|
35
|
+
def iterator(key)
|
36
|
+
Workling::Return::Store::Iterator.new(key)
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
#
|
2
|
+
# Iterator class for iterating over return values.
|
3
|
+
#
|
4
|
+
module Workling
|
5
|
+
module Return
|
6
|
+
module Store
|
7
|
+
class Iterator
|
8
|
+
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
def initialize(uid)
|
12
|
+
@uid = uid
|
13
|
+
end
|
14
|
+
|
15
|
+
def each
|
16
|
+
while item = Workling.return.get(@uid)
|
17
|
+
yield item
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
#
|
2
|
+
# Stores directly into memory. This is for tests only - not for production use. aight?
|
3
|
+
#
|
4
|
+
module Workling
|
5
|
+
module Return
|
6
|
+
module Store
|
7
|
+
class MemoryReturnStore < Base
|
8
|
+
attr_accessor :sky
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
self.sky = Hash.new([])
|
12
|
+
end
|
13
|
+
|
14
|
+
def set(key, value)
|
15
|
+
self.sky[key] << value
|
16
|
+
end
|
17
|
+
|
18
|
+
def get(key)
|
19
|
+
self.sky[key].shift
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#
|
2
|
+
# Recommended Return Store if you are using the Starling Runner. This
|
3
|
+
# Simply sets and gets values against queues. 'key' is the name of the respective Queue.
|
4
|
+
#
|
5
|
+
module Workling
|
6
|
+
module Return
|
7
|
+
module Store
|
8
|
+
class StarlingReturnStore < Base
|
9
|
+
|
10
|
+
cattr_accessor :client
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
self.client = Workling::Clients::MemcacheQueueClient.new
|
14
|
+
self.client.connect
|
15
|
+
end
|
16
|
+
|
17
|
+
# set a value in the queue 'key'.
|
18
|
+
def set(key, value)
|
19
|
+
self.client.set(key, value)
|
20
|
+
end
|
21
|
+
|
22
|
+
# get a value from starling queue 'key'.
|
23
|
+
def get(key)
|
24
|
+
self.client.get(key)
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#
|
2
|
+
# Base Class for Routing. Routing takes the worker method TestWorker#something,
|
3
|
+
# and serializes the signature in some way.
|
4
|
+
#
|
5
|
+
module Workling
|
6
|
+
module Routing
|
7
|
+
class Base < Hash
|
8
|
+
def method_name
|
9
|
+
raise Exception.new("method_name not implemented.")
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
#
|
2
|
+
# Holds a hash of routes. Each Worker method has a corresponding hash entry after building.
|
3
|
+
#
|
4
|
+
module Workling
|
5
|
+
module Routing
|
6
|
+
class ClassAndMethodRouting < Base
|
7
|
+
|
8
|
+
# initializes and builds routing hash.
|
9
|
+
def initialize(*args)
|
10
|
+
super
|
11
|
+
|
12
|
+
build
|
13
|
+
end
|
14
|
+
|
15
|
+
# returns the worker method name, given the routing string.
|
16
|
+
def method_name(queue)
|
17
|
+
self[queue].method_for(queue)
|
18
|
+
end
|
19
|
+
|
20
|
+
# returns the routing string, given a class and method. delegating.
|
21
|
+
def queue_for(clazz, method)
|
22
|
+
ClassAndMethodRouting.queue_for(clazz, method)
|
23
|
+
end
|
24
|
+
|
25
|
+
# returns the routing string, given a class and method.
|
26
|
+
def self.queue_for(clazz, method)
|
27
|
+
# this is lifted from the Rails constantize method
|
28
|
+
clazz = Object.module_eval("::#{clazz}") unless clazz.is_a?(Class)
|
29
|
+
clazz.queue_for method
|
30
|
+
end
|
31
|
+
|
32
|
+
# returns all routed
|
33
|
+
def queue_names
|
34
|
+
self.keys
|
35
|
+
end
|
36
|
+
|
37
|
+
# dare you to remove this! go on!
|
38
|
+
def queue_names_routing_class(clazz)
|
39
|
+
self.select { |x, y| y.is_a?(clazz) }.map { |x, y| x }
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
def build
|
44
|
+
Workling::Discovery.discovered.each do |clazz|
|
45
|
+
methods = clazz.public_instance_methods(false)
|
46
|
+
methods.each do |method|
|
47
|
+
next if method == 'create' # Skip the create method
|
48
|
+
queue = queue_for(clazz, method)
|
49
|
+
self[queue] = clazz.new
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#
|
2
|
+
# Holds a single route for a dedicated worker (if you want more worker processes, run more workers)
|
3
|
+
#
|
4
|
+
module Workling
|
5
|
+
module Routing
|
6
|
+
class StaticRouting < Base
|
7
|
+
|
8
|
+
#
|
9
|
+
# ./script/workling_client run -- <worker_class> <worker_method> <routing_key>
|
10
|
+
# ./script/workling_client run -- <worker_class> <worker_method> <routing_key> <queue_name>
|
11
|
+
#
|
12
|
+
def initialize(*args)
|
13
|
+
@worker = args[0].constantize.new
|
14
|
+
@method_name = args[1]
|
15
|
+
@routing_key = args[2]
|
16
|
+
|
17
|
+
if(args.size==4)
|
18
|
+
@queue_name = args[3]
|
19
|
+
else
|
20
|
+
@queue_name = [@worker.class.to_s.tableize, @method_name, @routing_key].join("__")
|
21
|
+
end
|
22
|
+
|
23
|
+
# so routing[x] hash access works as expected
|
24
|
+
self.default = @worker
|
25
|
+
end
|
26
|
+
|
27
|
+
# returns the worker method name, given the routing string.
|
28
|
+
def method_name(queue=nil); @method_name; end
|
29
|
+
|
30
|
+
def routing_key_for; @routing_key; end
|
31
|
+
|
32
|
+
# returns the routing string, given a class and method. delegating.
|
33
|
+
# TODO - we can check for consistency here with clazz and methods vs. current configuration of this single route
|
34
|
+
def queue_for(clazz=nil, method=nil); @queue_name; end
|
35
|
+
|
36
|
+
# returns array containing the single configured route
|
37
|
+
def queue_names; [@queue_name]; end
|
38
|
+
|
39
|
+
# TODO - not sure what this is...
|
40
|
+
def queue_names_routing_class(clazz); @worker.class; end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
class WorklingDaemon
|
4
|
+
|
5
|
+
def self.partition_options(args)
|
6
|
+
daemon = []
|
7
|
+
workling = []
|
8
|
+
split_point = args.index("--") || args.size
|
9
|
+
daemon = args[0...split_point] if split_point > 0
|
10
|
+
workling = args[(split_point+1)..-1] if split_point and split_point < args.size
|
11
|
+
[daemon, workling]
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.partition_daemons_options(args)
|
15
|
+
standard_options = %W{start stop restart run zap -t --ontop -f --force -h --help --version}
|
16
|
+
pass_through = args.select { |a| standard_options.include? a }
|
17
|
+
custom_options = args.reject { |a| standard_options.include? a }
|
18
|
+
|
19
|
+
[pass_through, custom_options]
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.parse_daemon_options(argv)
|
23
|
+
options = {}
|
24
|
+
pass_through, args = partition_daemons_options argv
|
25
|
+
opts = OptionParser.new do |opts|
|
26
|
+
opts.banner = 'Usage: myapp [options]'
|
27
|
+
opts.separator ''
|
28
|
+
opts.on('-a', '--app-name APP_NAME', String,"specify the process name") { |v| options[:app_name] = v }
|
29
|
+
opts.on('-d', '--dir DIR', String, "the directory to run in") { |v| options[:dir] = v }
|
30
|
+
opts.on('-m', '--monitor',"specify the process name") { |v| options[:monitor] = true }
|
31
|
+
opts.on('-t', '--ontop') { |k, v| pass_through << v }
|
32
|
+
end
|
33
|
+
opts.parse!(partition_options(args).first)
|
34
|
+
options.merge(:ARGV => pass_through)
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.parse_workling_options(args)
|
38
|
+
options = {}
|
39
|
+
opts = OptionParser.new do |opts|
|
40
|
+
opts.banner = 'Usage: myapp [options]'
|
41
|
+
opts.separator ''
|
42
|
+
opts.on('-n', '--no_rails', "do not load Rails") { |v| options[:no_rails] = true }
|
43
|
+
opts.on('-c', '--client CLIENT', String, "specify the client class") { |v| options[:client] = v }
|
44
|
+
opts.on('-i', '--invoker INVOKER', String, "specify the invoker class") { |v| options[:invoker] = v }
|
45
|
+
opts.on('-r', '--routing ROUTING', String, "specify the routing class") { |v| options[:routing] = v }
|
46
|
+
opts.on('-l', '--load-path LOADPATH', String, "specify the load_path for the workers") { |v| options[:load_path] = v }
|
47
|
+
opts.on('-f', '--config-path CONFIGPATH', String, "specify the path to the workling.yml file") { |v| options[:config_path] = v }
|
48
|
+
opts.on('-e', '--environment ENVIRONMENT', String, "specify the environment") { |v| options[:rails_env] = v }
|
49
|
+
opts.on('-p', '--prefix PREFIX', String, "specify the prefix for queues") { |v| options[:prefix] = v }
|
50
|
+
end
|
51
|
+
opts.parse!(partition_options(args).last)
|
52
|
+
options
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.extract_options(options)
|
56
|
+
result = {}
|
57
|
+
result[:client] = options[:client] if options[:client]
|
58
|
+
result[:routing] = options[:routing] if options[:routing]
|
59
|
+
result[:invoker] = options[:invoker] if options[:invoker]
|
60
|
+
result
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.initialize_workling(options)
|
64
|
+
Workling.load_path = options[:load_path] if options[:load_path]
|
65
|
+
Workling::Discovery.discover!
|
66
|
+
|
67
|
+
if options[:config_path]
|
68
|
+
Workling.config_path = options[:config_path]
|
69
|
+
Workling.config
|
70
|
+
else
|
71
|
+
Workling.config = extract_options options
|
72
|
+
end
|
73
|
+
|
74
|
+
Workling.select_and_build_invoker
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.boot_with(options)
|
78
|
+
if options[:no_rails]
|
79
|
+
# if rails is not booted we need to pull in the workling requires manually
|
80
|
+
require File.join(File.dirname(__FILE__), "workling")
|
81
|
+
else
|
82
|
+
ENV["RAILS_ENV"] = options[:rails_env]
|
83
|
+
puts "=> Loading Rails with #{ENV["RAILS_ENV"]} environment..."
|
84
|
+
require options[:rails_root] + '/config/environment'
|
85
|
+
|
86
|
+
ActiveRecord::Base.logger = Workling::Base.logger
|
87
|
+
ActionController::Base.logger = Workling::Base.logger
|
88
|
+
|
89
|
+
puts '** Rails loaded.'
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
def self.run(options)
|
95
|
+
boot_with options
|
96
|
+
poller = initialize_workling(options)
|
97
|
+
|
98
|
+
puts "** Starting #{ poller.class }..."
|
99
|
+
puts '** Use CTRL-C to stop.'
|
100
|
+
|
101
|
+
trap(:INT) { poller.stop; exit }
|
102
|
+
|
103
|
+
begin
|
104
|
+
poller.listen
|
105
|
+
ensure
|
106
|
+
puts '** No Worklings found.' if Workling::Discovery.discovered.empty?
|
107
|
+
puts '** Exiting'
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
end
|
metadata
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: workling
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.4.9.7
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Rany Keddo
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-02-25 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: easily do background work in rails, without commiting to a particular runner. comes with starling, bj and spawn runners.
|
17
|
+
email: nicolas@marchildon.net
|
18
|
+
executables:
|
19
|
+
- workling_client
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- CHANGES.markdown
|
26
|
+
- VERSION.yml
|
27
|
+
- README.markdown
|
28
|
+
- TODO.markdown
|
29
|
+
- lib/extensions/cattr_accessor.rb
|
30
|
+
- lib/extensions/mattr_accessor.rb
|
31
|
+
- lib/workling/base.rb
|
32
|
+
- lib/workling/clients/amqp_client.rb
|
33
|
+
- lib/workling/clients/amqp_exchange_client.rb
|
34
|
+
- lib/workling/clients/backgroundjob_client.rb
|
35
|
+
- lib/workling/clients/base.rb
|
36
|
+
- lib/workling/clients/broker_base.rb
|
37
|
+
- lib/workling/clients/memcache_queue_client.rb
|
38
|
+
- lib/workling/clients/memory_queue_client.rb
|
39
|
+
- lib/workling/clients/not_client.rb
|
40
|
+
- lib/workling/clients/not_remote_client.rb
|
41
|
+
- lib/workling/clients/rude_q_client.rb
|
42
|
+
- lib/workling/clients/spawn_client.rb
|
43
|
+
- lib/workling/clients/sqs_client.rb
|
44
|
+
- lib/workling/clients/thread_client.rb
|
45
|
+
- lib/workling/clients/xmpp_client.rb
|
46
|
+
- lib/workling/discovery.rb
|
47
|
+
- lib/workling/invokers/amqp_single_subscriber.rb
|
48
|
+
- lib/workling/invokers/base.rb
|
49
|
+
- lib/workling/invokers/basic_poller.rb
|
50
|
+
- lib/workling/invokers/eventmachine_subscriber.rb
|
51
|
+
- lib/workling/invokers/looped_subscriber.rb
|
52
|
+
- lib/workling/invokers/thread_pool_poller.rb
|
53
|
+
- lib/workling/invokers/threaded_poller.rb
|
54
|
+
- lib/workling/remote.rb
|
55
|
+
- lib/workling/return/store/base.rb
|
56
|
+
- lib/workling/return/store/iterator.rb
|
57
|
+
- lib/workling/return/store/memory_return_store.rb
|
58
|
+
- lib/workling/return/store/starling_return_store.rb
|
59
|
+
- lib/workling/routing/base.rb
|
60
|
+
- lib/workling/routing/class_and_method_routing.rb
|
61
|
+
- lib/workling/routing/static_routing.rb
|
62
|
+
- lib/workling.rb
|
63
|
+
- lib/workling_daemon.rb
|
64
|
+
- contrib/bj_invoker.rb
|
65
|
+
- contrib/starling_status.rb
|
66
|
+
has_rdoc: true
|
67
|
+
homepage: http://github.com/derfred/workling
|
68
|
+
licenses: []
|
69
|
+
|
70
|
+
post_install_message:
|
71
|
+
rdoc_options:
|
72
|
+
- --inline-source
|
73
|
+
- --charset=UTF-8
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: "0"
|
81
|
+
version:
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: "0"
|
87
|
+
version:
|
88
|
+
requirements: []
|
89
|
+
|
90
|
+
rubyforge_project:
|
91
|
+
rubygems_version: 1.3.5
|
92
|
+
signing_key:
|
93
|
+
specification_version: 2
|
94
|
+
summary: easily do background work in rails, without commiting to a particular runner. comes with starling, bj and spawn runners.
|
95
|
+
test_files: []
|
96
|
+
|