cloudist 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +2 -6
- data/Gemfile.lock +29 -27
- data/VERSION +1 -1
- data/cloudist.gemspec +9 -21
- data/examples/sandwich_client_with_custom_listener.rb +19 -16
- data/lib/cloudist/application.rb +10 -10
- data/lib/cloudist/core_ext/class.rb +4 -4
- data/lib/cloudist/job.rb +18 -18
- data/lib/cloudist/listener.rb +32 -32
- data/lib/cloudist/message.rb +23 -23
- data/lib/cloudist/messaging.rb +9 -9
- data/lib/cloudist/payload.rb +22 -22
- data/lib/cloudist/payload_old.rb +33 -33
- data/lib/cloudist/publisher.rb +6 -6
- data/lib/cloudist/queues/basic_queue.rb +17 -17
- data/lib/cloudist/queues/job_queue.rb +6 -6
- data/lib/cloudist/queues/reply_queue.rb +5 -5
- data/lib/cloudist/request.rb +5 -5
- data/lib/cloudist/utils.rb +5 -5
- data/lib/cloudist/worker.rb +10 -10
- data/spec/cloudist/basic_queue_spec.rb +10 -10
- data/spec/cloudist/job_spec.rb +4 -4
- data/spec/cloudist/payload_spec_2_spec.rb +16 -16
- data/spec/cloudist/request_spec.rb +13 -13
- data/spec/cloudist/utils_spec.rb +3 -3
- data/spec/cloudist_spec.rb +11 -11
- data/spec/core_ext/string_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -2
- data/spec/support/amqp.rb +11 -11
- metadata +29 -73
data/lib/cloudist/publisher.rb
CHANGED
@@ -1,22 +1,22 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Publisher
|
3
|
-
|
3
|
+
|
4
4
|
class << self
|
5
5
|
def enqueue(queue_name, data)
|
6
6
|
payload = Cloudist::Payload.new(data)
|
7
|
-
|
7
|
+
|
8
8
|
queue = Cloudist::JobQueue.new(queue_name)
|
9
|
-
|
9
|
+
|
10
10
|
queue.setup
|
11
|
-
|
11
|
+
|
12
12
|
# send_message = proc {
|
13
13
|
queue.publish(payload)
|
14
14
|
# }
|
15
15
|
# EM.next_tick(&send_message)
|
16
|
-
|
16
|
+
|
17
17
|
return Job.new(payload)
|
18
18
|
end
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
end
|
22
22
|
end
|
@@ -1,19 +1,19 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class UnknownReplyTo < RuntimeError; end
|
3
3
|
class ExpiredMessage < RuntimeError; end
|
4
|
-
|
4
|
+
|
5
5
|
module Queues
|
6
6
|
class BasicQueue
|
7
7
|
attr_reader :queue_name, :options
|
8
8
|
attr_reader :queue, :exchange, :channel, :prefetch
|
9
|
-
|
9
|
+
|
10
10
|
alias :q :queue
|
11
11
|
alias :ex :exchange
|
12
12
|
alias :mq :channel
|
13
|
-
|
13
|
+
|
14
14
|
def initialize(queue_name, options = {})
|
15
15
|
@prefetch ||= options.delete(:prefetch) || 1
|
16
|
-
|
16
|
+
|
17
17
|
options = {
|
18
18
|
:auto_delete => true,
|
19
19
|
:durable => false,
|
@@ -21,28 +21,28 @@ module Cloudist
|
|
21
21
|
}.update(options)
|
22
22
|
|
23
23
|
@queue_name, @options = queue_name, options
|
24
|
-
|
24
|
+
|
25
25
|
setup
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def inspect
|
29
29
|
"<#{self.class.name} queue_name=#{queue_name}>"
|
30
30
|
end
|
31
31
|
|
32
32
|
def setup
|
33
33
|
return if @setup.eql?(true)
|
34
|
-
|
34
|
+
|
35
35
|
@channel ||= AMQP::Channel.new(Cloudist.connection) do
|
36
|
-
channel.prefetch(self.prefetch, false) if self.prefetch
|
36
|
+
channel.prefetch(self.prefetch, false) if self.prefetch
|
37
37
|
end
|
38
|
-
|
38
|
+
|
39
39
|
@queue = @channel.queue(queue_name, options)
|
40
|
-
|
40
|
+
|
41
41
|
setup_exchange
|
42
|
-
|
42
|
+
|
43
43
|
@setup = true
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
def setup_exchange
|
47
47
|
@exchange = channel.direct("")
|
48
48
|
end
|
@@ -61,13 +61,13 @@ module Cloudist
|
|
61
61
|
s += " exchange=#{exchange.name}" if exchange
|
62
62
|
s
|
63
63
|
end
|
64
|
-
|
64
|
+
|
65
65
|
def subscribe(&block)
|
66
66
|
queue.subscribe(:ack => true) do |queue_header, encoded_message|
|
67
67
|
# next if Cloudist.closing?
|
68
68
|
|
69
69
|
request = Cloudist::Request.new(self, encoded_message, queue_header)
|
70
|
-
|
70
|
+
|
71
71
|
handle_request = proc {
|
72
72
|
begin
|
73
73
|
raise Cloudist::ExpiredMessage if request.expired?
|
@@ -86,11 +86,11 @@ module Cloudist
|
|
86
86
|
# log.debug("Finished Job in #{finished - request.start} seconds")
|
87
87
|
end
|
88
88
|
}
|
89
|
-
|
89
|
+
|
90
90
|
handle_ack = proc {
|
91
91
|
request.ack
|
92
92
|
}
|
93
|
-
|
93
|
+
|
94
94
|
EM.defer(handle_request, handle_ack)
|
95
95
|
end
|
96
96
|
log.info "AMQP Subscribed: #{tag}"
|
@@ -128,4 +128,4 @@ module Cloudist
|
|
128
128
|
end
|
129
129
|
end
|
130
130
|
end
|
131
|
-
end
|
131
|
+
end
|
@@ -1,26 +1,26 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class JobQueue < Cloudist::Queues::BasicQueue
|
3
|
-
|
3
|
+
|
4
4
|
def initialize(queue_name, options={})
|
5
5
|
options[:auto_delete] = false
|
6
6
|
options[:nowait] = false
|
7
|
-
|
7
|
+
|
8
8
|
@prefetch = Cloudist.worker_prefetch
|
9
9
|
puts "Prefetch: #{@prefetch}"
|
10
10
|
super(queue_name, options)
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
# def initialize(queue_name, options={})
|
14
14
|
# @prefetch = 1
|
15
15
|
# # opts[:auto_delete] = false
|
16
|
-
#
|
16
|
+
#
|
17
17
|
# super(queue_name, options)
|
18
18
|
# end
|
19
|
-
|
19
|
+
|
20
20
|
# def setup_exchange
|
21
21
|
# @exchange = channel.direct(queue_name)
|
22
22
|
# queue.bind(exchange)
|
23
23
|
# end
|
24
|
-
|
24
|
+
|
25
25
|
end
|
26
26
|
end
|
@@ -3,23 +3,23 @@ module Cloudist
|
|
3
3
|
def initialize(queue_name, options={})
|
4
4
|
options[:auto_delete] = true
|
5
5
|
options[:nowait] = false
|
6
|
-
|
6
|
+
|
7
7
|
@prefetch = Cloudist.listener_prefetch
|
8
|
-
|
8
|
+
|
9
9
|
super(queue_name, options)
|
10
10
|
end
|
11
|
-
|
11
|
+
|
12
12
|
# def subscribe(&block)
|
13
13
|
# super do |request|
|
14
14
|
# yield request if block_given?
|
15
15
|
# teardown
|
16
16
|
# end
|
17
17
|
# end
|
18
|
-
|
18
|
+
|
19
19
|
# def teardown
|
20
20
|
# queue.delete
|
21
21
|
# super
|
22
22
|
# end
|
23
|
-
|
23
|
+
|
24
24
|
end
|
25
25
|
end
|
data/lib/cloudist/request.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Request
|
3
3
|
include Cloudist::Encoding
|
4
|
-
|
4
|
+
|
5
5
|
attr_reader :queue_header, :qobj, :payload, :start, :headers, :body
|
6
6
|
|
7
7
|
def initialize(queue, encoded_body, queue_header)
|
@@ -9,13 +9,13 @@ module Cloudist
|
|
9
9
|
|
10
10
|
@body = decode(encoded_body)
|
11
11
|
@headers = parse_custom_headers(queue_header)
|
12
|
-
|
12
|
+
|
13
13
|
@payload = Cloudist::Payload.new(encoded_body, queue_header.headers.dup)
|
14
14
|
@headers = @payload.headers
|
15
15
|
|
16
16
|
@start = Time.now.utc.to_f
|
17
17
|
end
|
18
|
-
|
18
|
+
|
19
19
|
def parse_custom_headers(amqp_headers)
|
20
20
|
h = amqp_headers.headers.dup
|
21
21
|
|
@@ -26,7 +26,7 @@ module Cloudist
|
|
26
26
|
|
27
27
|
h
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
def for_message
|
31
31
|
[body.dup, queue_header.headers.dup]
|
32
32
|
end
|
@@ -42,7 +42,7 @@ module Cloudist
|
|
42
42
|
def mq
|
43
43
|
qobj.channel
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
def channel
|
47
47
|
mq
|
48
48
|
end
|
data/lib/cloudist/utils.rb
CHANGED
@@ -22,7 +22,7 @@ module Cloudist
|
|
22
22
|
def generate_name_for_instance(name)
|
23
23
|
"#{name}.#{Socket.gethostname}"
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
# DEPRECATED
|
27
27
|
def generate_reply_to(name)
|
28
28
|
"#{reply_prefix(name)}.#{generate_sym}"
|
@@ -40,15 +40,15 @@ module Cloudist
|
|
40
40
|
]
|
41
41
|
"%04x%04x%04x%04x%04x%06x%06x" % values
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
def encode_message(object)
|
45
45
|
Marshal.dump(object).to_s
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def decode_message(string)
|
49
49
|
Marshal.load(string)
|
50
50
|
end
|
51
|
-
|
51
|
+
|
52
52
|
def decode_json(string)
|
53
53
|
if defined? ActiveSupport::JSON
|
54
54
|
ActiveSupport::JSON.decode string
|
@@ -56,6 +56,6 @@ module Cloudist
|
|
56
56
|
JSON.load string
|
57
57
|
end
|
58
58
|
end
|
59
|
-
|
59
|
+
|
60
60
|
end
|
61
61
|
end
|
data/lib/cloudist/worker.rb
CHANGED
@@ -1,37 +1,37 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Worker
|
3
|
-
|
3
|
+
|
4
4
|
attr_reader :job, :queue, :payload
|
5
|
-
|
5
|
+
|
6
6
|
def initialize(job, queue)
|
7
7
|
@job, @queue, @payload = job, queue, job.payload
|
8
|
-
|
8
|
+
|
9
9
|
# Do custom initialization
|
10
10
|
self.setup if self.respond_to?(:setup)
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
def data
|
14
14
|
job.data
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
def headers
|
18
18
|
job.headers
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
def id
|
22
22
|
job.id
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def process
|
26
26
|
raise NotImplementedError, "Your worker class must subclass this method"
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
def log
|
30
30
|
Cloudist.log
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
class GenericWorker < Worker
|
36
36
|
def process(&block)
|
37
37
|
instance_eval(&block)
|
@@ -6,28 +6,28 @@ describe "Cloudist" do
|
|
6
6
|
overload_amqp
|
7
7
|
reset_broker
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
it "should create a queue and exchange" do
|
11
11
|
# MQ.stubs(:direct).with(:name).returns(true)
|
12
12
|
@mq = mock("MQ")
|
13
13
|
@exchange = mock("MQ Exchange")
|
14
14
|
@queue = mock("MQ Queue")
|
15
|
-
|
15
|
+
|
16
16
|
@queue.expects(:bind).with(@exchange)
|
17
17
|
# @mq.expects(:queue).with("make.sandwich")
|
18
|
-
|
18
|
+
|
19
19
|
bq = Cloudist::Queues::BasicQueue.new("make.sandwich")
|
20
|
-
bq.
|
21
|
-
bq.
|
22
|
-
bq.
|
23
|
-
|
20
|
+
bq.stub(:q).and_return(@queue)
|
21
|
+
bq.stub(:mq).and_return(@mq)
|
22
|
+
bq.stub(:ex).and_return(@exchange)
|
23
|
+
|
24
24
|
bq.setup
|
25
|
-
|
25
|
+
|
26
26
|
bq.q.should_not be_nil
|
27
27
|
bq.ex.should_not be_nil
|
28
28
|
bq.mq.should_not be_nil
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
end
|
data/spec/cloudist/job_spec.rb
CHANGED
@@ -4,20 +4,20 @@ describe Cloudist::Job do
|
|
4
4
|
before(:each) do
|
5
5
|
@payload = Cloudist::Payload.new({:bread => 'white'})
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
it "should be constructable with payload" do
|
9
9
|
job = Cloudist::Job.new(@payload)
|
10
10
|
job.payload.should == @payload
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
it "should be constructable with payload and return ID" do
|
14
14
|
job = Cloudist::Job.new(@payload)
|
15
15
|
job.id.should == @payload.id
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
it "should be constructable with payload and return data" do
|
19
19
|
job = Cloudist::Job.new(@payload)
|
20
20
|
job.data.should == @payload.body
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
end
|
@@ -2,22 +2,22 @@ require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
|
|
2
2
|
|
3
3
|
describe Cloudist::Payload do
|
4
4
|
include Cloudist::Encoding
|
5
|
-
|
5
|
+
|
6
6
|
it "should accept a hash for data" do
|
7
7
|
pl = Cloudist::Payload.new({:bread => 'white'})
|
8
8
|
pl.body.bread.should == "white"
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
it "should accept encoded message" do
|
12
12
|
pl = Cloudist::Payload.new(encode({:bread => 'white'}))
|
13
13
|
pl.body.bread.should == "white"
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
it "should retrieve id from headers" do
|
17
17
|
pl = Cloudist::Payload.new({:bread => 'white'}, {:message_id => "12345"})
|
18
18
|
pl.id.should == "12345"
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
it "should prepare headers" do
|
22
22
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
23
23
|
payload.body.bread.should == "white"
|
@@ -27,52 +27,52 @@ describe Cloudist::Payload do
|
|
27
27
|
payload.headers.has_key?("published_on").should be_true
|
28
28
|
payload.headers.has_key?("message_id").should be_true
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
it "should extract published_on from data" do
|
32
32
|
time = Time.now.to_f
|
33
33
|
payload = Cloudist::Payload.new({:bread => 'white', :timestamp => time})
|
34
34
|
payload.headers[:published_on].should == time
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
it "should not override timestamp if already present in headers" do
|
38
38
|
time = (Time.now.to_f - 10.0)
|
39
39
|
payload = Cloudist::Payload.new({:bread => 'white'}, {:published_on => time})
|
40
40
|
payload.headers[:published_on].should == time
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
it "should override timestamp if not present" do
|
44
44
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
45
45
|
payload.headers[:published_on].should be_within(0.1).of Time.now.to_f
|
46
46
|
payload.timestamp.should be_within(0.1).of Time.now.to_f
|
47
47
|
end
|
48
|
-
|
48
|
+
|
49
49
|
it "should parse custom headers" do
|
50
50
|
payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white'}), {:published_on => 12345, :message_id => "foo"})
|
51
51
|
payload.headers.to_hash.should == { "published_on"=>12345, "message_id"=>"foo", "ttl"=>300 }
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
it "should create a unique event hash" do
|
55
55
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
56
56
|
payload.id.size.should == 36
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
it "should not create a new message_id unless it doesn't have one" do
|
60
60
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
61
61
|
payload.id.size.should == 36
|
62
62
|
payload = Cloudist::Payload.new({:bread => 'white'}, {:message_id => 'foo'})
|
63
63
|
payload.id.should == 'foo'
|
64
64
|
end
|
65
|
-
|
65
|
+
|
66
66
|
it "should format payload for sending" do
|
67
67
|
payload = Cloudist::Payload.new({:bread => 'white'}, {:message_id => 'foo', :message_type => 'reply'})
|
68
68
|
body, popts = payload.to_a
|
69
69
|
headers = popts[:headers]
|
70
|
-
|
70
|
+
|
71
71
|
body.should == encode(Hashie::Mash.new({:bread => 'white'}))
|
72
72
|
headers[:ttl].should == "300"
|
73
73
|
headers[:message_type].should == 'reply'
|
74
74
|
end
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
end
|
75
|
+
|
76
|
+
|
77
|
+
|
78
|
+
end
|