workling 0.4.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/CHANGES.markdown +82 -0
  2. data/README.markdown +543 -0
  3. data/TODO.markdown +27 -0
  4. data/VERSION.yml +4 -0
  5. data/bin/workling_client +29 -0
  6. data/contrib/bj_invoker.rb +11 -0
  7. data/contrib/starling_status.rb +37 -0
  8. data/lib/extensions/cattr_accessor.rb +51 -0
  9. data/lib/extensions/mattr_accessor.rb +55 -0
  10. data/lib/workling.rb +213 -0
  11. data/lib/workling/base.rb +110 -0
  12. data/lib/workling/clients/amqp_client.rb +51 -0
  13. data/lib/workling/clients/amqp_exchange_client.rb +58 -0
  14. data/lib/workling/clients/backgroundjob_client.rb +25 -0
  15. data/lib/workling/clients/base.rb +89 -0
  16. data/lib/workling/clients/broker_base.rb +63 -0
  17. data/lib/workling/clients/memcache_queue_client.rb +104 -0
  18. data/lib/workling/clients/memory_queue_client.rb +34 -0
  19. data/lib/workling/clients/not_client.rb +14 -0
  20. data/lib/workling/clients/not_remote_client.rb +17 -0
  21. data/lib/workling/clients/rude_q_client.rb +47 -0
  22. data/lib/workling/clients/spawn_client.rb +46 -0
  23. data/lib/workling/clients/sqs_client.rb +163 -0
  24. data/lib/workling/clients/thread_client.rb +18 -0
  25. data/lib/workling/clients/xmpp_client.rb +110 -0
  26. data/lib/workling/discovery.rb +16 -0
  27. data/lib/workling/invokers/amqp_single_subscriber.rb +42 -0
  28. data/lib/workling/invokers/base.rb +124 -0
  29. data/lib/workling/invokers/basic_poller.rb +38 -0
  30. data/lib/workling/invokers/eventmachine_subscriber.rb +38 -0
  31. data/lib/workling/invokers/looped_subscriber.rb +34 -0
  32. data/lib/workling/invokers/thread_pool_poller.rb +165 -0
  33. data/lib/workling/invokers/threaded_poller.rb +149 -0
  34. data/lib/workling/remote.rb +38 -0
  35. data/lib/workling/return/store/base.rb +42 -0
  36. data/lib/workling/return/store/iterator.rb +24 -0
  37. data/lib/workling/return/store/memory_return_store.rb +24 -0
  38. data/lib/workling/return/store/starling_return_store.rb +30 -0
  39. data/lib/workling/routing/base.rb +13 -0
  40. data/lib/workling/routing/class_and_method_routing.rb +55 -0
  41. data/lib/workling/routing/static_routing.rb +43 -0
  42. data/lib/workling_daemon.rb +111 -0
  43. metadata +96 -0
@@ -0,0 +1,38 @@
1
+ require 'digest/md5'
2
+
3
+ #
4
+ # Scoping Module for Runners.
5
+ #
6
+ module Workling
7
+ module Remote
8
+
9
+ # which object to use for routing
10
+ mattr_writer :routing
11
+ def self.routing
12
+ @@routing ||= Workling::Routing::ClassAndMethodRouting.new
13
+ end
14
+
15
+ # which client to use for dispatching
16
+ mattr_accessor :client
17
+ def self.client
18
+ @@client ||= Workling.select_and_build_client
19
+ end
20
+
21
+ # generates a unique identifier for this particular job.
22
+ def self.generate_uid(clazz, method)
23
+ uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
24
+ "#{ clazz.to_s.tableize }/#{ method }/#{ uid }".split("/").join(":")
25
+ end
26
+
27
+ # dispatches to a workling. writes the :uid for this work into the options hash, so make
28
+ # sure you pass in a hash if you want write to a return store in your workling.
29
+ def self.run(clazz, method, options = {})
30
+ uid = Workling::Remote.generate_uid(clazz, method)
31
+ options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
32
+ Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist.
33
+ client.dispatch(clazz, method, options)
34
+ uid
35
+ end
36
+
37
+ end
38
+ end
@@ -0,0 +1,42 @@
1
+ #
2
+ # Basic interface for getting and setting Data which needs to be passed between Workers and
3
+ # client code.
4
+ #
5
+ module Workling
6
+ module Return
7
+ module Store
8
+ mattr_accessor :instance
9
+
10
+ # set a value in the store with the given key. delegates to the returnstore.
11
+ def self.set(key, value)
12
+ self.instance.set(key, value)
13
+ end
14
+
15
+ # get a value from the store. this should be destructive. delegates to the returnstore.
16
+ def self.get(key)
17
+ self.instance.get(key)
18
+ end
19
+
20
+ #
21
+ # Base Class for Return Stores. Subclasses need to implement set and get.
22
+ #
23
+ class Base
24
+
25
+ # set a value in the store with the given key.
26
+ def set(key, value)
27
+ raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
28
+ end
29
+
30
+ # get a value from the store. this should be destructive.
31
+ def get(key)
32
+ raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
33
+ end
34
+
35
+ def iterator(key)
36
+ Workling::Return::Store::Iterator.new(key)
37
+ end
38
+
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,24 @@
1
+ #
2
+ # Iterator class for iterating over return values.
3
+ #
4
+ module Workling
5
+ module Return
6
+ module Store
7
+ class Iterator
8
+
9
+ include Enumerable
10
+
11
+ def initialize(uid)
12
+ @uid = uid
13
+ end
14
+
15
+ def each
16
+ while item = Workling.return.get(@uid)
17
+ yield item
18
+ end
19
+ end
20
+
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ #
2
+ # Stores directly into memory. This is for tests only - not for production use. aight?
3
+ #
4
+ module Workling
5
+ module Return
6
+ module Store
7
+ class MemoryReturnStore < Base
8
+ attr_accessor :sky
9
+
10
+ def initialize
11
+ self.sky = Hash.new([])
12
+ end
13
+
14
+ def set(key, value)
15
+ self.sky[key] << value
16
+ end
17
+
18
+ def get(key)
19
+ self.sky[key].shift
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,30 @@
1
+ #
2
+ # Recommended Return Store if you are using the Starling Runner. This
3
+ # Simply sets and gets values against queues. 'key' is the name of the respective Queue.
4
+ #
5
+ module Workling
6
+ module Return
7
+ module Store
8
+ class StarlingReturnStore < Base
9
+
10
+ cattr_accessor :client
11
+
12
+ def initialize
13
+ self.client = Workling::Clients::MemcacheQueueClient.new
14
+ self.client.connect
15
+ end
16
+
17
+ # set a value in the queue 'key'.
18
+ def set(key, value)
19
+ self.client.set(key, value)
20
+ end
21
+
22
+ # get a value from starling queue 'key'.
23
+ def get(key)
24
+ self.client.get(key)
25
+ end
26
+
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,13 @@
1
+ #
2
+ # Base Class for Routing. Routing takes the worker method TestWorker#something,
3
+ # and serializes the signature in some way.
4
+ #
5
+ module Workling
6
+ module Routing
7
+ class Base < Hash
8
+ def method_name
9
+ raise Exception.new("method_name not implemented.")
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,55 @@
1
+ #
2
+ # Holds a hash of routes. Each Worker method has a corresponding hash entry after building.
3
+ #
4
+ module Workling
5
+ module Routing
6
+ class ClassAndMethodRouting < Base
7
+
8
+ # initializes and builds routing hash.
9
+ def initialize(*args)
10
+ super
11
+
12
+ build
13
+ end
14
+
15
+ # returns the worker method name, given the routing string.
16
+ def method_name(queue)
17
+ self[queue].method_for(queue)
18
+ end
19
+
20
+ # returns the routing string, given a class and method. delegating.
21
+ def queue_for(clazz, method)
22
+ ClassAndMethodRouting.queue_for(clazz, method)
23
+ end
24
+
25
+ # returns the routing string, given a class and method.
26
+ def self.queue_for(clazz, method)
27
+ # this is lifted from the Rails constantize method
28
+ clazz = Object.module_eval("::#{clazz}") unless clazz.is_a?(Class)
29
+ clazz.queue_for method
30
+ end
31
+
32
+ # returns all routed
33
+ def queue_names
34
+ self.keys
35
+ end
36
+
37
+ # dare you to remove this! go on!
38
+ def queue_names_routing_class(clazz)
39
+ self.select { |x, y| y.is_a?(clazz) }.map { |x, y| x }
40
+ end
41
+
42
+ private
43
+ def build
44
+ Workling::Discovery.discovered.each do |clazz|
45
+ methods = clazz.public_instance_methods(false)
46
+ methods.each do |method|
47
+ next if method == 'create' # Skip the create method
48
+ queue = queue_for(clazz, method)
49
+ self[queue] = clazz.new
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,43 @@
1
+ #
2
+ # Holds a single route for a dedicated worker (if you want more worker processes, run more workers)
3
+ #
4
+ module Workling
5
+ module Routing
6
+ class StaticRouting < Base
7
+
8
+ #
9
+ # ./script/workling_client run -- <worker_class> <worker_method> <routing_key>
10
+ # ./script/workling_client run -- <worker_class> <worker_method> <routing_key> <queue_name>
11
+ #
12
+ def initialize(*args)
13
+ @worker = args[0].constantize.new
14
+ @method_name = args[1]
15
+ @routing_key = args[2]
16
+
17
+ if(args.size==4)
18
+ @queue_name = args[3]
19
+ else
20
+ @queue_name = [@worker.class.to_s.tableize, @method_name, @routing_key].join("__")
21
+ end
22
+
23
+ # so routing[x] hash access works as expected
24
+ self.default = @worker
25
+ end
26
+
27
+ # returns the worker method name, given the routing string.
28
+ def method_name(queue=nil); @method_name; end
29
+
30
+ def routing_key_for; @routing_key; end
31
+
32
+ # returns the routing string, given a class and method. delegating.
33
+ # TODO - we can check for consistency here with clazz and methods vs. current configuration of this single route
34
+ def queue_for(clazz=nil, method=nil); @queue_name; end
35
+
36
+ # returns array containing the single configured route
37
+ def queue_names; [@queue_name]; end
38
+
39
+ # TODO - not sure what this is...
40
+ def queue_names_routing_class(clazz); @worker.class; end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,111 @@
1
+ require 'optparse'
2
+
3
+ class WorklingDaemon
4
+
5
+ def self.partition_options(args)
6
+ daemon = []
7
+ workling = []
8
+ split_point = args.index("--") || args.size
9
+ daemon = args[0...split_point] if split_point > 0
10
+ workling = args[(split_point+1)..-1] if split_point and split_point < args.size
11
+ [daemon, workling]
12
+ end
13
+
14
+ def self.partition_daemons_options(args)
15
+ standard_options = %W{start stop restart run zap -t --ontop -f --force -h --help --version}
16
+ pass_through = args.select { |a| standard_options.include? a }
17
+ custom_options = args.reject { |a| standard_options.include? a }
18
+
19
+ [pass_through, custom_options]
20
+ end
21
+
22
+ def self.parse_daemon_options(argv)
23
+ options = {}
24
+ pass_through, args = partition_daemons_options argv
25
+ opts = OptionParser.new do |opts|
26
+ opts.banner = 'Usage: myapp [options]'
27
+ opts.separator ''
28
+ opts.on('-a', '--app-name APP_NAME', String,"specify the process name") { |v| options[:app_name] = v }
29
+ opts.on('-d', '--dir DIR', String, "the directory to run in") { |v| options[:dir] = v }
30
+ opts.on('-m', '--monitor',"specify the process name") { |v| options[:monitor] = true }
31
+ opts.on('-t', '--ontop') { |k, v| pass_through << v }
32
+ end
33
+ opts.parse!(partition_options(args).first)
34
+ options.merge(:ARGV => pass_through)
35
+ end
36
+
37
+ def self.parse_workling_options(args)
38
+ options = {}
39
+ opts = OptionParser.new do |opts|
40
+ opts.banner = 'Usage: myapp [options]'
41
+ opts.separator ''
42
+ opts.on('-n', '--no_rails', "do not load Rails") { |v| options[:no_rails] = true }
43
+ opts.on('-c', '--client CLIENT', String, "specify the client class") { |v| options[:client] = v }
44
+ opts.on('-i', '--invoker INVOKER', String, "specify the invoker class") { |v| options[:invoker] = v }
45
+ opts.on('-r', '--routing ROUTING', String, "specify the routing class") { |v| options[:routing] = v }
46
+ opts.on('-l', '--load-path LOADPATH', String, "specify the load_path for the workers") { |v| options[:load_path] = v }
47
+ opts.on('-f', '--config-path CONFIGPATH', String, "specify the path to the workling.yml file") { |v| options[:config_path] = v }
48
+ opts.on('-e', '--environment ENVIRONMENT', String, "specify the environment") { |v| options[:rails_env] = v }
49
+ opts.on('-p', '--prefix PREFIX', String, "specify the prefix for queues") { |v| options[:prefix] = v }
50
+ end
51
+ opts.parse!(partition_options(args).last)
52
+ options
53
+ end
54
+
55
+ def self.extract_options(options)
56
+ result = {}
57
+ result[:client] = options[:client] if options[:client]
58
+ result[:routing] = options[:routing] if options[:routing]
59
+ result[:invoker] = options[:invoker] if options[:invoker]
60
+ result
61
+ end
62
+
63
+ def self.initialize_workling(options)
64
+ Workling.load_path = options[:load_path] if options[:load_path]
65
+ Workling::Discovery.discover!
66
+
67
+ if options[:config_path]
68
+ Workling.config_path = options[:config_path]
69
+ Workling.config
70
+ else
71
+ Workling.config = extract_options options
72
+ end
73
+
74
+ Workling.select_and_build_invoker
75
+ end
76
+
77
+ def self.boot_with(options)
78
+ if options[:no_rails]
79
+ # if rails is not booted we need to pull in the workling requires manually
80
+ require File.join(File.dirname(__FILE__), "workling")
81
+ else
82
+ ENV["RAILS_ENV"] = options[:rails_env]
83
+ puts "=> Loading Rails with #{ENV["RAILS_ENV"]} environment..."
84
+ require options[:rails_root] + '/config/environment'
85
+
86
+ ActiveRecord::Base.logger = Workling::Base.logger
87
+ ActionController::Base.logger = Workling::Base.logger
88
+
89
+ puts '** Rails loaded.'
90
+ end
91
+ end
92
+
93
+
94
+ def self.run(options)
95
+ boot_with options
96
+ poller = initialize_workling(options)
97
+
98
+ puts "** Starting #{ poller.class }..."
99
+ puts '** Use CTRL-C to stop.'
100
+
101
+ trap(:INT) { poller.stop; exit }
102
+
103
+ begin
104
+ poller.listen
105
+ ensure
106
+ puts '** No Worklings found.' if Workling::Discovery.discovered.empty?
107
+ puts '** Exiting'
108
+ end
109
+ end
110
+
111
+ end
metadata ADDED
@@ -0,0 +1,96 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: workling
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.4.9.7
5
+ platform: ruby
6
+ authors:
7
+ - Rany Keddo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-02-25 00:00:00 +01:00
13
+ default_executable:
14
+ dependencies: []
15
+
16
+ description: easily do background work in rails, without commiting to a particular runner. comes with starling, bj and spawn runners.
17
+ email: nicolas@marchildon.net
18
+ executables:
19
+ - workling_client
20
+ extensions: []
21
+
22
+ extra_rdoc_files: []
23
+
24
+ files:
25
+ - CHANGES.markdown
26
+ - VERSION.yml
27
+ - README.markdown
28
+ - TODO.markdown
29
+ - lib/extensions/cattr_accessor.rb
30
+ - lib/extensions/mattr_accessor.rb
31
+ - lib/workling/base.rb
32
+ - lib/workling/clients/amqp_client.rb
33
+ - lib/workling/clients/amqp_exchange_client.rb
34
+ - lib/workling/clients/backgroundjob_client.rb
35
+ - lib/workling/clients/base.rb
36
+ - lib/workling/clients/broker_base.rb
37
+ - lib/workling/clients/memcache_queue_client.rb
38
+ - lib/workling/clients/memory_queue_client.rb
39
+ - lib/workling/clients/not_client.rb
40
+ - lib/workling/clients/not_remote_client.rb
41
+ - lib/workling/clients/rude_q_client.rb
42
+ - lib/workling/clients/spawn_client.rb
43
+ - lib/workling/clients/sqs_client.rb
44
+ - lib/workling/clients/thread_client.rb
45
+ - lib/workling/clients/xmpp_client.rb
46
+ - lib/workling/discovery.rb
47
+ - lib/workling/invokers/amqp_single_subscriber.rb
48
+ - lib/workling/invokers/base.rb
49
+ - lib/workling/invokers/basic_poller.rb
50
+ - lib/workling/invokers/eventmachine_subscriber.rb
51
+ - lib/workling/invokers/looped_subscriber.rb
52
+ - lib/workling/invokers/thread_pool_poller.rb
53
+ - lib/workling/invokers/threaded_poller.rb
54
+ - lib/workling/remote.rb
55
+ - lib/workling/return/store/base.rb
56
+ - lib/workling/return/store/iterator.rb
57
+ - lib/workling/return/store/memory_return_store.rb
58
+ - lib/workling/return/store/starling_return_store.rb
59
+ - lib/workling/routing/base.rb
60
+ - lib/workling/routing/class_and_method_routing.rb
61
+ - lib/workling/routing/static_routing.rb
62
+ - lib/workling.rb
63
+ - lib/workling_daemon.rb
64
+ - contrib/bj_invoker.rb
65
+ - contrib/starling_status.rb
66
+ has_rdoc: true
67
+ homepage: http://github.com/derfred/workling
68
+ licenses: []
69
+
70
+ post_install_message:
71
+ rdoc_options:
72
+ - --inline-source
73
+ - --charset=UTF-8
74
+ require_paths:
75
+ - lib
76
+ required_ruby_version: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: "0"
81
+ version:
82
+ required_rubygems_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: "0"
87
+ version:
88
+ requirements: []
89
+
90
+ rubyforge_project:
91
+ rubygems_version: 1.3.5
92
+ signing_key:
93
+ specification_version: 2
94
+ summary: easily do background work in rails, without commiting to a particular runner. comes with starling, bj and spawn runners.
95
+ test_files: []
96
+