cloudist 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +2 -6
- data/Gemfile.lock +29 -27
- data/VERSION +1 -1
- data/cloudist.gemspec +9 -21
- data/examples/sandwich_client_with_custom_listener.rb +19 -16
- data/lib/cloudist/application.rb +10 -10
- data/lib/cloudist/core_ext/class.rb +4 -4
- data/lib/cloudist/job.rb +18 -18
- data/lib/cloudist/listener.rb +32 -32
- data/lib/cloudist/message.rb +23 -23
- data/lib/cloudist/messaging.rb +9 -9
- data/lib/cloudist/payload.rb +22 -22
- data/lib/cloudist/payload_old.rb +33 -33
- data/lib/cloudist/publisher.rb +6 -6
- data/lib/cloudist/queues/basic_queue.rb +17 -17
- data/lib/cloudist/queues/job_queue.rb +6 -6
- data/lib/cloudist/queues/reply_queue.rb +5 -5
- data/lib/cloudist/request.rb +5 -5
- data/lib/cloudist/utils.rb +5 -5
- data/lib/cloudist/worker.rb +10 -10
- data/spec/cloudist/basic_queue_spec.rb +10 -10
- data/spec/cloudist/job_spec.rb +4 -4
- data/spec/cloudist/payload_spec_2_spec.rb +16 -16
- data/spec/cloudist/request_spec.rb +13 -13
- data/spec/cloudist/utils_spec.rb +3 -3
- data/spec/cloudist_spec.rb +11 -11
- data/spec/core_ext/string_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -2
- data/spec/support/amqp.rb +11 -11
- metadata +29 -73
data/Gemfile
CHANGED
@@ -1,22 +1,18 @@
|
|
1
1
|
source :rubygems
|
2
2
|
|
3
3
|
gem 'amqp', '~> 0.8.1'
|
4
|
-
gem "json", "~> 1.4
|
4
|
+
gem "json", "~> 1.4"
|
5
5
|
gem "i18n"
|
6
|
-
gem "activesupport", "
|
6
|
+
gem "activesupport", "> 3.0.0"
|
7
7
|
gem "hashie"
|
8
8
|
gem "uuid"
|
9
9
|
|
10
10
|
# Add dependencies to develop your gem here.
|
11
11
|
# Include everything needed to run rake, tests, features, etc.
|
12
12
|
group :development do
|
13
|
-
gem "rake", "~> 0.9.2"
|
14
13
|
gem "rspec", "~> 2.4.0"
|
15
14
|
gem "moqueue", :git => "git://github.com/ivanvanderbyl/moqueue.git"
|
16
|
-
gem "mocha"
|
17
|
-
gem "bundler", "~> 1.0.0"
|
18
15
|
gem "jeweler", "~> 1.6.4"
|
19
|
-
gem "rcov", ">= 0"
|
20
16
|
gem "reek", "~> 1.2.8"
|
21
17
|
gem "roodi", "~> 2.1.0"
|
22
18
|
end
|
data/Gemfile.lock
CHANGED
@@ -1,40 +1,45 @@
|
|
1
1
|
GIT
|
2
2
|
remote: git://github.com/ivanvanderbyl/moqueue.git
|
3
|
-
revision:
|
3
|
+
revision: 5d20cc7bceed3cf4f40c02e638f4398b9f0a2e00
|
4
4
|
specs:
|
5
|
-
moqueue (0.1
|
6
|
-
amqp
|
5
|
+
moqueue (0.2.1)
|
6
|
+
amqp (>= 0.8.0.rc14)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: http://rubygems.org/
|
10
10
|
specs:
|
11
|
-
activesupport (3.
|
12
|
-
|
13
|
-
|
11
|
+
activesupport (3.2.6)
|
12
|
+
i18n (~> 0.6)
|
13
|
+
multi_json (~> 1.0)
|
14
|
+
amq-client (0.8.7)
|
15
|
+
amq-protocol (>= 0.8.4)
|
14
16
|
eventmachine
|
15
|
-
amq-protocol (0.8.
|
16
|
-
amqp (0.8.
|
17
|
-
amq-client (~> 0.8.
|
18
|
-
amq-protocol (~> 0.8.
|
17
|
+
amq-protocol (0.8.4)
|
18
|
+
amqp (0.8.4)
|
19
|
+
amq-client (~> 0.8.7)
|
20
|
+
amq-protocol (~> 0.8.4)
|
19
21
|
eventmachine
|
20
22
|
diff-lcs (1.1.3)
|
21
23
|
eventmachine (0.12.10)
|
22
24
|
git (1.2.5)
|
23
|
-
hashie (1.
|
24
|
-
i18n (0.
|
25
|
+
hashie (1.2.0)
|
26
|
+
i18n (0.6.0)
|
25
27
|
jeweler (1.6.4)
|
26
28
|
bundler (~> 1.0)
|
27
29
|
git (>= 1.2.5)
|
28
30
|
rake
|
29
|
-
json (1.
|
30
|
-
macaddr (1.
|
31
|
-
|
31
|
+
json (1.7.3)
|
32
|
+
macaddr (1.6.1)
|
33
|
+
systemu (~> 2.5.0)
|
34
|
+
multi_json (1.3.6)
|
32
35
|
rake (0.9.2.2)
|
33
|
-
|
34
|
-
|
35
|
-
ruby2ruby (~> 1.2)
|
36
|
+
reek (1.2.12)
|
37
|
+
ripper_ruby_parser (~> 0.0.7)
|
38
|
+
ruby2ruby (~> 1.2.5)
|
36
39
|
ruby_parser (~> 2.0)
|
37
40
|
sexp_processor (~> 3.0)
|
41
|
+
ripper_ruby_parser (0.0.8)
|
42
|
+
sexp_processor (~> 3.0)
|
38
43
|
roodi (2.1.0)
|
39
44
|
ruby_parser
|
40
45
|
rspec (2.4.0)
|
@@ -48,27 +53,24 @@ GEM
|
|
48
53
|
ruby2ruby (1.2.5)
|
49
54
|
ruby_parser (~> 2.0)
|
50
55
|
sexp_processor (~> 3.0)
|
51
|
-
ruby_parser (2.
|
56
|
+
ruby_parser (2.3.1)
|
52
57
|
sexp_processor (~> 3.0)
|
53
|
-
sexp_processor (3.0
|
54
|
-
|
58
|
+
sexp_processor (3.2.0)
|
59
|
+
systemu (2.5.1)
|
60
|
+
uuid (2.3.5)
|
55
61
|
macaddr (~> 1.0)
|
56
62
|
|
57
63
|
PLATFORMS
|
58
64
|
ruby
|
59
65
|
|
60
66
|
DEPENDENCIES
|
61
|
-
activesupport (
|
67
|
+
activesupport (> 3.0.0)
|
62
68
|
amqp (~> 0.8.1)
|
63
|
-
bundler (~> 1.0.0)
|
64
69
|
hashie
|
65
70
|
i18n
|
66
71
|
jeweler (~> 1.6.4)
|
67
|
-
json (~> 1.4
|
68
|
-
mocha
|
72
|
+
json (~> 1.4)
|
69
73
|
moqueue!
|
70
|
-
rake (~> 0.9.2)
|
71
|
-
rcov
|
72
74
|
reek (~> 1.2.8)
|
73
75
|
roodi (~> 2.1.0)
|
74
76
|
rspec (~> 2.4.0)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.0
|
data/cloudist.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "cloudist"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Ivan Vanderbyl"]
|
12
|
-
s.date = "
|
12
|
+
s.date = "2012-07-12"
|
13
13
|
s.description = "Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, DaemonKit or your own custom application. Refer to github page for examples"
|
14
14
|
s.email = "ivanvanderbyl@me.com"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -78,7 +78,7 @@ Gem::Specification.new do |s|
|
|
78
78
|
s.homepage = "http://github.com/ivanvanderbyl/cloudist"
|
79
79
|
s.licenses = ["MIT"]
|
80
80
|
s.require_paths = ["lib"]
|
81
|
-
s.rubygems_version = "1.8.
|
81
|
+
s.rubygems_version = "1.8.15"
|
82
82
|
s.summary = "Super fast job queue using AMQP"
|
83
83
|
|
84
84
|
if s.respond_to? :specification_version then
|
@@ -86,51 +86,39 @@ Gem::Specification.new do |s|
|
|
86
86
|
|
87
87
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
88
88
|
s.add_runtime_dependency(%q<amqp>, ["~> 0.8.1"])
|
89
|
-
s.add_runtime_dependency(%q<json>, ["~> 1.4
|
89
|
+
s.add_runtime_dependency(%q<json>, ["~> 1.4"])
|
90
90
|
s.add_runtime_dependency(%q<i18n>, [">= 0"])
|
91
|
-
s.add_runtime_dependency(%q<activesupport>, ["
|
91
|
+
s.add_runtime_dependency(%q<activesupport>, ["> 3.0.0"])
|
92
92
|
s.add_runtime_dependency(%q<hashie>, [">= 0"])
|
93
93
|
s.add_runtime_dependency(%q<uuid>, [">= 0"])
|
94
|
-
s.add_development_dependency(%q<rake>, ["~> 0.9.2"])
|
95
94
|
s.add_development_dependency(%q<rspec>, ["~> 2.4.0"])
|
96
95
|
s.add_development_dependency(%q<moqueue>, [">= 0"])
|
97
|
-
s.add_development_dependency(%q<mocha>, [">= 0"])
|
98
|
-
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
99
96
|
s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
|
100
|
-
s.add_development_dependency(%q<rcov>, [">= 0"])
|
101
97
|
s.add_development_dependency(%q<reek>, ["~> 1.2.8"])
|
102
98
|
s.add_development_dependency(%q<roodi>, ["~> 2.1.0"])
|
103
99
|
else
|
104
100
|
s.add_dependency(%q<amqp>, ["~> 0.8.1"])
|
105
|
-
s.add_dependency(%q<json>, ["~> 1.4
|
101
|
+
s.add_dependency(%q<json>, ["~> 1.4"])
|
106
102
|
s.add_dependency(%q<i18n>, [">= 0"])
|
107
|
-
s.add_dependency(%q<activesupport>, ["
|
103
|
+
s.add_dependency(%q<activesupport>, ["> 3.0.0"])
|
108
104
|
s.add_dependency(%q<hashie>, [">= 0"])
|
109
105
|
s.add_dependency(%q<uuid>, [">= 0"])
|
110
|
-
s.add_dependency(%q<rake>, ["~> 0.9.2"])
|
111
106
|
s.add_dependency(%q<rspec>, ["~> 2.4.0"])
|
112
107
|
s.add_dependency(%q<moqueue>, [">= 0"])
|
113
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
114
|
-
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
115
108
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
116
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
117
109
|
s.add_dependency(%q<reek>, ["~> 1.2.8"])
|
118
110
|
s.add_dependency(%q<roodi>, ["~> 2.1.0"])
|
119
111
|
end
|
120
112
|
else
|
121
113
|
s.add_dependency(%q<amqp>, ["~> 0.8.1"])
|
122
|
-
s.add_dependency(%q<json>, ["~> 1.4
|
114
|
+
s.add_dependency(%q<json>, ["~> 1.4"])
|
123
115
|
s.add_dependency(%q<i18n>, [">= 0"])
|
124
|
-
s.add_dependency(%q<activesupport>, ["
|
116
|
+
s.add_dependency(%q<activesupport>, ["> 3.0.0"])
|
125
117
|
s.add_dependency(%q<hashie>, [">= 0"])
|
126
118
|
s.add_dependency(%q<uuid>, [">= 0"])
|
127
|
-
s.add_dependency(%q<rake>, ["~> 0.9.2"])
|
128
119
|
s.add_dependency(%q<rspec>, ["~> 2.4.0"])
|
129
120
|
s.add_dependency(%q<moqueue>, [">= 0"])
|
130
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
131
|
-
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
132
121
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
133
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
134
122
|
s.add_dependency(%q<reek>, ["~> 1.2.8"])
|
135
123
|
s.add_dependency(%q<roodi>, ["~> 2.1.0"])
|
136
124
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# Cloudst Example: Sandwich Client with custom listener class
|
2
|
-
#
|
2
|
+
#
|
3
3
|
# This example demonstrates dispatching a job to the worker and receiving event callbacks.
|
4
|
-
#
|
4
|
+
#
|
5
5
|
# Be sure to update the Cloudist connection settings if they differ from defaults:
|
6
6
|
# user: guest
|
7
7
|
# pass: guest
|
@@ -17,23 +17,23 @@ $total_jobs = 0
|
|
17
17
|
|
18
18
|
class SandwichListener < Cloudist::Listener
|
19
19
|
listen_to "make.sandwich"
|
20
|
-
|
20
|
+
|
21
21
|
before :find_job
|
22
|
-
|
22
|
+
|
23
23
|
def find_job
|
24
24
|
puts "--- #{payload.id}"
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
def progress(i)
|
28
28
|
puts "Progress: %1d%" % i
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
def runtime(seconds)
|
32
32
|
puts "#{id} Finished job in #{seconds} seconds"
|
33
33
|
$total_jobs -= 1
|
34
34
|
puts "--- #{$total_jobs} jobs remaining"
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
# def started
|
38
38
|
# puts "Started"
|
39
39
|
# end
|
@@ -41,16 +41,16 @@ class SandwichListener < Cloudist::Listener
|
|
41
41
|
def event(type)
|
42
42
|
puts "Event: #{type}"
|
43
43
|
end
|
44
|
-
|
44
|
+
|
45
45
|
def finished
|
46
46
|
puts "*** Finished ***"
|
47
|
-
|
47
|
+
|
48
48
|
if $total_jobs == 0
|
49
49
|
puts "Completed all jobs"
|
50
50
|
Cloudist.stop
|
51
51
|
end
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
def error(e)
|
55
55
|
puts "#{e.exception}: #{e.message} (#{e.backtrace.first})"
|
56
56
|
end
|
@@ -61,17 +61,20 @@ Cloudist.signal_trap!
|
|
61
61
|
|
62
62
|
Cloudist.start(:logging => true) {
|
63
63
|
puts AMQP.settings.inspect
|
64
|
-
|
65
|
-
|
64
|
+
|
65
|
+
if ARGV.empty?
|
66
|
+
puts "Please specify a number of workers to start as your first argument"
|
67
|
+
Cloudist.stop
|
68
|
+
else
|
66
69
|
puts "*** Please ensure you have a worker running ***"
|
67
|
-
|
70
|
+
|
68
71
|
job_count = ARGV.pop.to_i
|
69
72
|
$total_jobs = job_count
|
70
|
-
job_count.times { |i|
|
73
|
+
job_count.times { |i|
|
71
74
|
log.info("Dispatching sandwich making job...")
|
72
75
|
puts "Queued job: " + enqueue('make.sandwich', {:bread => 'white', :sandwich_number => i}).id
|
73
76
|
}
|
74
77
|
end
|
75
|
-
|
76
|
-
add_listener(SandwichListener)
|
78
|
+
|
79
|
+
add_listener(SandwichListener)
|
77
80
|
}
|
data/lib/cloudist/application.rb
CHANGED
@@ -3,7 +3,7 @@ require "singleton"
|
|
3
3
|
module Cloudist
|
4
4
|
class Application
|
5
5
|
include Singleton
|
6
|
-
|
6
|
+
|
7
7
|
class << self
|
8
8
|
def start(options = {}, &block)
|
9
9
|
options = instance.settings.update(options)
|
@@ -13,21 +13,21 @@ module Cloudist
|
|
13
13
|
instance.instance_eval(&block) if block_given?
|
14
14
|
end
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
def signal_trap!
|
18
18
|
::Signal.trap('INT') { Cloudist.stop }
|
19
19
|
::Signal.trap('TERM'){ Cloudist.stop }
|
20
20
|
end
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
def settings
|
24
24
|
@@settings ||= default_settings
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
def settings=(settings_hash)
|
28
28
|
@@settings = default_settings.update(settings_hash)
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
def default_settings
|
32
32
|
uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/')
|
33
33
|
{
|
@@ -42,19 +42,19 @@ module Cloudist
|
|
42
42
|
rescue Object => e
|
43
43
|
raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}"
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
private
|
47
|
-
|
47
|
+
|
48
48
|
def setup_reconnect_hook!
|
49
49
|
AMQP.conn.connection_status do |status|
|
50
|
-
|
50
|
+
|
51
51
|
log.debug("AMQP connection status changed: #{status}")
|
52
|
-
|
52
|
+
|
53
53
|
if status == :disconnected
|
54
54
|
AMQP.conn.reconnect(true)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
end
|
60
60
|
end
|
@@ -3,7 +3,7 @@ require 'cloudist/core_ext/module'
|
|
3
3
|
|
4
4
|
# Extracted from ActiveSupport 3.0
|
5
5
|
class Class
|
6
|
-
|
6
|
+
|
7
7
|
# Taken from http://coderrr.wordpress.com/2008/04/10/lets-stop-polluting-the-threadcurrent-hash/
|
8
8
|
def thread_local_accessor name, options = {}
|
9
9
|
m = Module.new
|
@@ -28,8 +28,8 @@ class Class
|
|
28
28
|
extend m
|
29
29
|
end
|
30
30
|
end
|
31
|
-
|
32
|
-
|
31
|
+
|
32
|
+
|
33
33
|
# Declare a class-level attribute whose value is inheritable by subclasses.
|
34
34
|
# Subclasses can change their own value and it will not impact parent class.
|
35
35
|
#
|
@@ -136,4 +136,4 @@ class Class
|
|
136
136
|
# MRI/YARV/JRuby all disallow creating new instances of a singleton class
|
137
137
|
true
|
138
138
|
end
|
139
|
-
end
|
139
|
+
end
|
data/lib/cloudist/job.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Job
|
3
3
|
attr_reader :payload, :reply_queue
|
4
|
-
|
4
|
+
|
5
5
|
def initialize(payload)
|
6
6
|
@payload = payload
|
7
|
-
|
7
|
+
|
8
8
|
if payload.reply_to
|
9
9
|
@reply_queue = ReplyQueue.new(payload.reply_to)
|
10
10
|
reply_queue.setup
|
@@ -12,67 +12,67 @@ module Cloudist
|
|
12
12
|
@reply_queue = nil
|
13
13
|
end
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
def id
|
17
17
|
payload.id
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
def data
|
21
21
|
payload.body
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
def body
|
25
25
|
data
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def log
|
29
29
|
Cloudist.log
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
def cleanup
|
33
33
|
# :noop
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
def reply(body, headers = {}, options = {})
|
37
37
|
raise ArgumentError, "Reply queue not ready" unless reply_queue
|
38
|
-
|
38
|
+
|
39
39
|
options = {
|
40
40
|
:echo => false
|
41
41
|
}.update(options)
|
42
|
-
|
42
|
+
|
43
43
|
headers = {
|
44
44
|
:message_id => payload.id,
|
45
45
|
:message_type => "reply"
|
46
46
|
}.update(headers)
|
47
|
-
|
47
|
+
|
48
48
|
reply_payload = Payload.new(body, headers)
|
49
49
|
published_headers = reply_queue.publish(reply_payload)
|
50
|
-
|
50
|
+
|
51
51
|
reply_payload
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
# Sends a progress update
|
55
55
|
# Inputs: percentage - Integer
|
56
56
|
# Optional description, this could be displayed to the user e.g. Resizing image
|
57
57
|
def progress(percentage, description = nil)
|
58
58
|
reply({:progress => percentage, :description => description}, {:message_type => 'progress'})
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
def event(event_name, event_data = {}, options = {})
|
62
62
|
event_data ||= {}
|
63
63
|
reply(event_data, {:event => event_name, :message_type => 'event'}, options)
|
64
64
|
end
|
65
|
-
|
65
|
+
|
66
66
|
def safely(&blk)
|
67
67
|
yield
|
68
68
|
rescue Exception => e
|
69
69
|
handle_error(e)
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def handle_error(e)
|
73
73
|
reply({:exception => e.class.name.to_s, :message => e.message.to_s, :backtrace => e.backtrace}, {:message_type => 'error'})
|
74
74
|
end
|
75
|
-
|
75
|
+
|
76
76
|
def method_missing(meth, *args, &blk)
|
77
77
|
if meth.to_s.ends_with?("!")
|
78
78
|
event(meth.to_s.gsub(/(!)$/, ''), args.shift)
|
@@ -80,6 +80,6 @@ module Cloudist
|
|
80
80
|
super
|
81
81
|
end
|
82
82
|
end
|
83
|
-
|
83
|
+
|
84
84
|
end
|
85
85
|
end
|