cloudist 0.4.4 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +2 -6
- data/Gemfile.lock +29 -27
- data/VERSION +1 -1
- data/cloudist.gemspec +9 -21
- data/examples/sandwich_client_with_custom_listener.rb +19 -16
- data/lib/cloudist/application.rb +10 -10
- data/lib/cloudist/core_ext/class.rb +4 -4
- data/lib/cloudist/job.rb +18 -18
- data/lib/cloudist/listener.rb +32 -32
- data/lib/cloudist/message.rb +23 -23
- data/lib/cloudist/messaging.rb +9 -9
- data/lib/cloudist/payload.rb +22 -22
- data/lib/cloudist/payload_old.rb +33 -33
- data/lib/cloudist/publisher.rb +6 -6
- data/lib/cloudist/queues/basic_queue.rb +17 -17
- data/lib/cloudist/queues/job_queue.rb +6 -6
- data/lib/cloudist/queues/reply_queue.rb +5 -5
- data/lib/cloudist/request.rb +5 -5
- data/lib/cloudist/utils.rb +5 -5
- data/lib/cloudist/worker.rb +10 -10
- data/spec/cloudist/basic_queue_spec.rb +10 -10
- data/spec/cloudist/job_spec.rb +4 -4
- data/spec/cloudist/payload_spec_2_spec.rb +16 -16
- data/spec/cloudist/request_spec.rb +13 -13
- data/spec/cloudist/utils_spec.rb +3 -3
- data/spec/cloudist_spec.rb +11 -11
- data/spec/core_ext/string_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -2
- data/spec/support/amqp.rb +11 -11
- metadata +29 -73
data/Gemfile
CHANGED
@@ -1,22 +1,18 @@
|
|
1
1
|
source :rubygems
|
2
2
|
|
3
3
|
gem 'amqp', '~> 0.8.1'
|
4
|
-
gem "json", "~> 1.4
|
4
|
+
gem "json", "~> 1.4"
|
5
5
|
gem "i18n"
|
6
|
-
gem "activesupport", "
|
6
|
+
gem "activesupport", "> 3.0.0"
|
7
7
|
gem "hashie"
|
8
8
|
gem "uuid"
|
9
9
|
|
10
10
|
# Add dependencies to develop your gem here.
|
11
11
|
# Include everything needed to run rake, tests, features, etc.
|
12
12
|
group :development do
|
13
|
-
gem "rake", "~> 0.9.2"
|
14
13
|
gem "rspec", "~> 2.4.0"
|
15
14
|
gem "moqueue", :git => "git://github.com/ivanvanderbyl/moqueue.git"
|
16
|
-
gem "mocha"
|
17
|
-
gem "bundler", "~> 1.0.0"
|
18
15
|
gem "jeweler", "~> 1.6.4"
|
19
|
-
gem "rcov", ">= 0"
|
20
16
|
gem "reek", "~> 1.2.8"
|
21
17
|
gem "roodi", "~> 2.1.0"
|
22
18
|
end
|
data/Gemfile.lock
CHANGED
@@ -1,40 +1,45 @@
|
|
1
1
|
GIT
|
2
2
|
remote: git://github.com/ivanvanderbyl/moqueue.git
|
3
|
-
revision:
|
3
|
+
revision: 5d20cc7bceed3cf4f40c02e638f4398b9f0a2e00
|
4
4
|
specs:
|
5
|
-
moqueue (0.1
|
6
|
-
amqp
|
5
|
+
moqueue (0.2.1)
|
6
|
+
amqp (>= 0.8.0.rc14)
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: http://rubygems.org/
|
10
10
|
specs:
|
11
|
-
activesupport (3.
|
12
|
-
|
13
|
-
|
11
|
+
activesupport (3.2.6)
|
12
|
+
i18n (~> 0.6)
|
13
|
+
multi_json (~> 1.0)
|
14
|
+
amq-client (0.8.7)
|
15
|
+
amq-protocol (>= 0.8.4)
|
14
16
|
eventmachine
|
15
|
-
amq-protocol (0.8.
|
16
|
-
amqp (0.8.
|
17
|
-
amq-client (~> 0.8.
|
18
|
-
amq-protocol (~> 0.8.
|
17
|
+
amq-protocol (0.8.4)
|
18
|
+
amqp (0.8.4)
|
19
|
+
amq-client (~> 0.8.7)
|
20
|
+
amq-protocol (~> 0.8.4)
|
19
21
|
eventmachine
|
20
22
|
diff-lcs (1.1.3)
|
21
23
|
eventmachine (0.12.10)
|
22
24
|
git (1.2.5)
|
23
|
-
hashie (1.
|
24
|
-
i18n (0.
|
25
|
+
hashie (1.2.0)
|
26
|
+
i18n (0.6.0)
|
25
27
|
jeweler (1.6.4)
|
26
28
|
bundler (~> 1.0)
|
27
29
|
git (>= 1.2.5)
|
28
30
|
rake
|
29
|
-
json (1.
|
30
|
-
macaddr (1.
|
31
|
-
|
31
|
+
json (1.7.3)
|
32
|
+
macaddr (1.6.1)
|
33
|
+
systemu (~> 2.5.0)
|
34
|
+
multi_json (1.3.6)
|
32
35
|
rake (0.9.2.2)
|
33
|
-
|
34
|
-
|
35
|
-
ruby2ruby (~> 1.2)
|
36
|
+
reek (1.2.12)
|
37
|
+
ripper_ruby_parser (~> 0.0.7)
|
38
|
+
ruby2ruby (~> 1.2.5)
|
36
39
|
ruby_parser (~> 2.0)
|
37
40
|
sexp_processor (~> 3.0)
|
41
|
+
ripper_ruby_parser (0.0.8)
|
42
|
+
sexp_processor (~> 3.0)
|
38
43
|
roodi (2.1.0)
|
39
44
|
ruby_parser
|
40
45
|
rspec (2.4.0)
|
@@ -48,27 +53,24 @@ GEM
|
|
48
53
|
ruby2ruby (1.2.5)
|
49
54
|
ruby_parser (~> 2.0)
|
50
55
|
sexp_processor (~> 3.0)
|
51
|
-
ruby_parser (2.
|
56
|
+
ruby_parser (2.3.1)
|
52
57
|
sexp_processor (~> 3.0)
|
53
|
-
sexp_processor (3.0
|
54
|
-
|
58
|
+
sexp_processor (3.2.0)
|
59
|
+
systemu (2.5.1)
|
60
|
+
uuid (2.3.5)
|
55
61
|
macaddr (~> 1.0)
|
56
62
|
|
57
63
|
PLATFORMS
|
58
64
|
ruby
|
59
65
|
|
60
66
|
DEPENDENCIES
|
61
|
-
activesupport (
|
67
|
+
activesupport (> 3.0.0)
|
62
68
|
amqp (~> 0.8.1)
|
63
|
-
bundler (~> 1.0.0)
|
64
69
|
hashie
|
65
70
|
i18n
|
66
71
|
jeweler (~> 1.6.4)
|
67
|
-
json (~> 1.4
|
68
|
-
mocha
|
72
|
+
json (~> 1.4)
|
69
73
|
moqueue!
|
70
|
-
rake (~> 0.9.2)
|
71
|
-
rcov
|
72
74
|
reek (~> 1.2.8)
|
73
75
|
roodi (~> 2.1.0)
|
74
76
|
rspec (~> 2.4.0)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.0
|
data/cloudist.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "cloudist"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Ivan Vanderbyl"]
|
12
|
-
s.date = "
|
12
|
+
s.date = "2012-07-12"
|
13
13
|
s.description = "Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, DaemonKit or your own custom application. Refer to github page for examples"
|
14
14
|
s.email = "ivanvanderbyl@me.com"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -78,7 +78,7 @@ Gem::Specification.new do |s|
|
|
78
78
|
s.homepage = "http://github.com/ivanvanderbyl/cloudist"
|
79
79
|
s.licenses = ["MIT"]
|
80
80
|
s.require_paths = ["lib"]
|
81
|
-
s.rubygems_version = "1.8.
|
81
|
+
s.rubygems_version = "1.8.15"
|
82
82
|
s.summary = "Super fast job queue using AMQP"
|
83
83
|
|
84
84
|
if s.respond_to? :specification_version then
|
@@ -86,51 +86,39 @@ Gem::Specification.new do |s|
|
|
86
86
|
|
87
87
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
88
88
|
s.add_runtime_dependency(%q<amqp>, ["~> 0.8.1"])
|
89
|
-
s.add_runtime_dependency(%q<json>, ["~> 1.4
|
89
|
+
s.add_runtime_dependency(%q<json>, ["~> 1.4"])
|
90
90
|
s.add_runtime_dependency(%q<i18n>, [">= 0"])
|
91
|
-
s.add_runtime_dependency(%q<activesupport>, ["
|
91
|
+
s.add_runtime_dependency(%q<activesupport>, ["> 3.0.0"])
|
92
92
|
s.add_runtime_dependency(%q<hashie>, [">= 0"])
|
93
93
|
s.add_runtime_dependency(%q<uuid>, [">= 0"])
|
94
|
-
s.add_development_dependency(%q<rake>, ["~> 0.9.2"])
|
95
94
|
s.add_development_dependency(%q<rspec>, ["~> 2.4.0"])
|
96
95
|
s.add_development_dependency(%q<moqueue>, [">= 0"])
|
97
|
-
s.add_development_dependency(%q<mocha>, [">= 0"])
|
98
|
-
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
99
96
|
s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
|
100
|
-
s.add_development_dependency(%q<rcov>, [">= 0"])
|
101
97
|
s.add_development_dependency(%q<reek>, ["~> 1.2.8"])
|
102
98
|
s.add_development_dependency(%q<roodi>, ["~> 2.1.0"])
|
103
99
|
else
|
104
100
|
s.add_dependency(%q<amqp>, ["~> 0.8.1"])
|
105
|
-
s.add_dependency(%q<json>, ["~> 1.4
|
101
|
+
s.add_dependency(%q<json>, ["~> 1.4"])
|
106
102
|
s.add_dependency(%q<i18n>, [">= 0"])
|
107
|
-
s.add_dependency(%q<activesupport>, ["
|
103
|
+
s.add_dependency(%q<activesupport>, ["> 3.0.0"])
|
108
104
|
s.add_dependency(%q<hashie>, [">= 0"])
|
109
105
|
s.add_dependency(%q<uuid>, [">= 0"])
|
110
|
-
s.add_dependency(%q<rake>, ["~> 0.9.2"])
|
111
106
|
s.add_dependency(%q<rspec>, ["~> 2.4.0"])
|
112
107
|
s.add_dependency(%q<moqueue>, [">= 0"])
|
113
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
114
|
-
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
115
108
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
116
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
117
109
|
s.add_dependency(%q<reek>, ["~> 1.2.8"])
|
118
110
|
s.add_dependency(%q<roodi>, ["~> 2.1.0"])
|
119
111
|
end
|
120
112
|
else
|
121
113
|
s.add_dependency(%q<amqp>, ["~> 0.8.1"])
|
122
|
-
s.add_dependency(%q<json>, ["~> 1.4
|
114
|
+
s.add_dependency(%q<json>, ["~> 1.4"])
|
123
115
|
s.add_dependency(%q<i18n>, [">= 0"])
|
124
|
-
s.add_dependency(%q<activesupport>, ["
|
116
|
+
s.add_dependency(%q<activesupport>, ["> 3.0.0"])
|
125
117
|
s.add_dependency(%q<hashie>, [">= 0"])
|
126
118
|
s.add_dependency(%q<uuid>, [">= 0"])
|
127
|
-
s.add_dependency(%q<rake>, ["~> 0.9.2"])
|
128
119
|
s.add_dependency(%q<rspec>, ["~> 2.4.0"])
|
129
120
|
s.add_dependency(%q<moqueue>, [">= 0"])
|
130
|
-
s.add_dependency(%q<mocha>, [">= 0"])
|
131
|
-
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
132
121
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
133
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
134
122
|
s.add_dependency(%q<reek>, ["~> 1.2.8"])
|
135
123
|
s.add_dependency(%q<roodi>, ["~> 2.1.0"])
|
136
124
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# Cloudst Example: Sandwich Client with custom listener class
|
2
|
-
#
|
2
|
+
#
|
3
3
|
# This example demonstrates dispatching a job to the worker and receiving event callbacks.
|
4
|
-
#
|
4
|
+
#
|
5
5
|
# Be sure to update the Cloudist connection settings if they differ from defaults:
|
6
6
|
# user: guest
|
7
7
|
# pass: guest
|
@@ -17,23 +17,23 @@ $total_jobs = 0
|
|
17
17
|
|
18
18
|
class SandwichListener < Cloudist::Listener
|
19
19
|
listen_to "make.sandwich"
|
20
|
-
|
20
|
+
|
21
21
|
before :find_job
|
22
|
-
|
22
|
+
|
23
23
|
def find_job
|
24
24
|
puts "--- #{payload.id}"
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
def progress(i)
|
28
28
|
puts "Progress: %1d%" % i
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
def runtime(seconds)
|
32
32
|
puts "#{id} Finished job in #{seconds} seconds"
|
33
33
|
$total_jobs -= 1
|
34
34
|
puts "--- #{$total_jobs} jobs remaining"
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
# def started
|
38
38
|
# puts "Started"
|
39
39
|
# end
|
@@ -41,16 +41,16 @@ class SandwichListener < Cloudist::Listener
|
|
41
41
|
def event(type)
|
42
42
|
puts "Event: #{type}"
|
43
43
|
end
|
44
|
-
|
44
|
+
|
45
45
|
def finished
|
46
46
|
puts "*** Finished ***"
|
47
|
-
|
47
|
+
|
48
48
|
if $total_jobs == 0
|
49
49
|
puts "Completed all jobs"
|
50
50
|
Cloudist.stop
|
51
51
|
end
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
def error(e)
|
55
55
|
puts "#{e.exception}: #{e.message} (#{e.backtrace.first})"
|
56
56
|
end
|
@@ -61,17 +61,20 @@ Cloudist.signal_trap!
|
|
61
61
|
|
62
62
|
Cloudist.start(:logging => true) {
|
63
63
|
puts AMQP.settings.inspect
|
64
|
-
|
65
|
-
|
64
|
+
|
65
|
+
if ARGV.empty?
|
66
|
+
puts "Please specify a number of workers to start as your first argument"
|
67
|
+
Cloudist.stop
|
68
|
+
else
|
66
69
|
puts "*** Please ensure you have a worker running ***"
|
67
|
-
|
70
|
+
|
68
71
|
job_count = ARGV.pop.to_i
|
69
72
|
$total_jobs = job_count
|
70
|
-
job_count.times { |i|
|
73
|
+
job_count.times { |i|
|
71
74
|
log.info("Dispatching sandwich making job...")
|
72
75
|
puts "Queued job: " + enqueue('make.sandwich', {:bread => 'white', :sandwich_number => i}).id
|
73
76
|
}
|
74
77
|
end
|
75
|
-
|
76
|
-
add_listener(SandwichListener)
|
78
|
+
|
79
|
+
add_listener(SandwichListener)
|
77
80
|
}
|
data/lib/cloudist/application.rb
CHANGED
@@ -3,7 +3,7 @@ require "singleton"
|
|
3
3
|
module Cloudist
|
4
4
|
class Application
|
5
5
|
include Singleton
|
6
|
-
|
6
|
+
|
7
7
|
class << self
|
8
8
|
def start(options = {}, &block)
|
9
9
|
options = instance.settings.update(options)
|
@@ -13,21 +13,21 @@ module Cloudist
|
|
13
13
|
instance.instance_eval(&block) if block_given?
|
14
14
|
end
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
def signal_trap!
|
18
18
|
::Signal.trap('INT') { Cloudist.stop }
|
19
19
|
::Signal.trap('TERM'){ Cloudist.stop }
|
20
20
|
end
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
def settings
|
24
24
|
@@settings ||= default_settings
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
def settings=(settings_hash)
|
28
28
|
@@settings = default_settings.update(settings_hash)
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
def default_settings
|
32
32
|
uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/')
|
33
33
|
{
|
@@ -42,19 +42,19 @@ module Cloudist
|
|
42
42
|
rescue Object => e
|
43
43
|
raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}"
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
private
|
47
|
-
|
47
|
+
|
48
48
|
def setup_reconnect_hook!
|
49
49
|
AMQP.conn.connection_status do |status|
|
50
|
-
|
50
|
+
|
51
51
|
log.debug("AMQP connection status changed: #{status}")
|
52
|
-
|
52
|
+
|
53
53
|
if status == :disconnected
|
54
54
|
AMQP.conn.reconnect(true)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
end
|
60
60
|
end
|
@@ -3,7 +3,7 @@ require 'cloudist/core_ext/module'
|
|
3
3
|
|
4
4
|
# Extracted from ActiveSupport 3.0
|
5
5
|
class Class
|
6
|
-
|
6
|
+
|
7
7
|
# Taken from http://coderrr.wordpress.com/2008/04/10/lets-stop-polluting-the-threadcurrent-hash/
|
8
8
|
def thread_local_accessor name, options = {}
|
9
9
|
m = Module.new
|
@@ -28,8 +28,8 @@ class Class
|
|
28
28
|
extend m
|
29
29
|
end
|
30
30
|
end
|
31
|
-
|
32
|
-
|
31
|
+
|
32
|
+
|
33
33
|
# Declare a class-level attribute whose value is inheritable by subclasses.
|
34
34
|
# Subclasses can change their own value and it will not impact parent class.
|
35
35
|
#
|
@@ -136,4 +136,4 @@ class Class
|
|
136
136
|
# MRI/YARV/JRuby all disallow creating new instances of a singleton class
|
137
137
|
true
|
138
138
|
end
|
139
|
-
end
|
139
|
+
end
|
data/lib/cloudist/job.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Job
|
3
3
|
attr_reader :payload, :reply_queue
|
4
|
-
|
4
|
+
|
5
5
|
def initialize(payload)
|
6
6
|
@payload = payload
|
7
|
-
|
7
|
+
|
8
8
|
if payload.reply_to
|
9
9
|
@reply_queue = ReplyQueue.new(payload.reply_to)
|
10
10
|
reply_queue.setup
|
@@ -12,67 +12,67 @@ module Cloudist
|
|
12
12
|
@reply_queue = nil
|
13
13
|
end
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
def id
|
17
17
|
payload.id
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
def data
|
21
21
|
payload.body
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
def body
|
25
25
|
data
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def log
|
29
29
|
Cloudist.log
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
def cleanup
|
33
33
|
# :noop
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
def reply(body, headers = {}, options = {})
|
37
37
|
raise ArgumentError, "Reply queue not ready" unless reply_queue
|
38
|
-
|
38
|
+
|
39
39
|
options = {
|
40
40
|
:echo => false
|
41
41
|
}.update(options)
|
42
|
-
|
42
|
+
|
43
43
|
headers = {
|
44
44
|
:message_id => payload.id,
|
45
45
|
:message_type => "reply"
|
46
46
|
}.update(headers)
|
47
|
-
|
47
|
+
|
48
48
|
reply_payload = Payload.new(body, headers)
|
49
49
|
published_headers = reply_queue.publish(reply_payload)
|
50
|
-
|
50
|
+
|
51
51
|
reply_payload
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
# Sends a progress update
|
55
55
|
# Inputs: percentage - Integer
|
56
56
|
# Optional description, this could be displayed to the user e.g. Resizing image
|
57
57
|
def progress(percentage, description = nil)
|
58
58
|
reply({:progress => percentage, :description => description}, {:message_type => 'progress'})
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
def event(event_name, event_data = {}, options = {})
|
62
62
|
event_data ||= {}
|
63
63
|
reply(event_data, {:event => event_name, :message_type => 'event'}, options)
|
64
64
|
end
|
65
|
-
|
65
|
+
|
66
66
|
def safely(&blk)
|
67
67
|
yield
|
68
68
|
rescue Exception => e
|
69
69
|
handle_error(e)
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def handle_error(e)
|
73
73
|
reply({:exception => e.class.name.to_s, :message => e.message.to_s, :backtrace => e.backtrace}, {:message_type => 'error'})
|
74
74
|
end
|
75
|
-
|
75
|
+
|
76
76
|
def method_missing(meth, *args, &blk)
|
77
77
|
if meth.to_s.ends_with?("!")
|
78
78
|
event(meth.to_s.gsub(/(!)$/, ''), args.shift)
|
@@ -80,6 +80,6 @@ module Cloudist
|
|
80
80
|
super
|
81
81
|
end
|
82
82
|
end
|
83
|
-
|
83
|
+
|
84
84
|
end
|
85
85
|
end
|