cloudist 0.4.4 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +2 -6
- data/Gemfile.lock +29 -27
- data/VERSION +1 -1
- data/cloudist.gemspec +9 -21
- data/examples/sandwich_client_with_custom_listener.rb +19 -16
- data/lib/cloudist/application.rb +10 -10
- data/lib/cloudist/core_ext/class.rb +4 -4
- data/lib/cloudist/job.rb +18 -18
- data/lib/cloudist/listener.rb +32 -32
- data/lib/cloudist/message.rb +23 -23
- data/lib/cloudist/messaging.rb +9 -9
- data/lib/cloudist/payload.rb +22 -22
- data/lib/cloudist/payload_old.rb +33 -33
- data/lib/cloudist/publisher.rb +6 -6
- data/lib/cloudist/queues/basic_queue.rb +17 -17
- data/lib/cloudist/queues/job_queue.rb +6 -6
- data/lib/cloudist/queues/reply_queue.rb +5 -5
- data/lib/cloudist/request.rb +5 -5
- data/lib/cloudist/utils.rb +5 -5
- data/lib/cloudist/worker.rb +10 -10
- data/spec/cloudist/basic_queue_spec.rb +10 -10
- data/spec/cloudist/job_spec.rb +4 -4
- data/spec/cloudist/payload_spec_2_spec.rb +16 -16
- data/spec/cloudist/request_spec.rb +13 -13
- data/spec/cloudist/utils_spec.rb +3 -3
- data/spec/cloudist_spec.rb +11 -11
- data/spec/core_ext/string_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -2
- data/spec/support/amqp.rb +11 -11
- metadata +29 -73
data/lib/cloudist/publisher.rb
CHANGED
@@ -1,22 +1,22 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Publisher
|
3
|
-
|
3
|
+
|
4
4
|
class << self
|
5
5
|
def enqueue(queue_name, data)
|
6
6
|
payload = Cloudist::Payload.new(data)
|
7
|
-
|
7
|
+
|
8
8
|
queue = Cloudist::JobQueue.new(queue_name)
|
9
|
-
|
9
|
+
|
10
10
|
queue.setup
|
11
|
-
|
11
|
+
|
12
12
|
# send_message = proc {
|
13
13
|
queue.publish(payload)
|
14
14
|
# }
|
15
15
|
# EM.next_tick(&send_message)
|
16
|
-
|
16
|
+
|
17
17
|
return Job.new(payload)
|
18
18
|
end
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
end
|
22
22
|
end
|
@@ -1,19 +1,19 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class UnknownReplyTo < RuntimeError; end
|
3
3
|
class ExpiredMessage < RuntimeError; end
|
4
|
-
|
4
|
+
|
5
5
|
module Queues
|
6
6
|
class BasicQueue
|
7
7
|
attr_reader :queue_name, :options
|
8
8
|
attr_reader :queue, :exchange, :channel, :prefetch
|
9
|
-
|
9
|
+
|
10
10
|
alias :q :queue
|
11
11
|
alias :ex :exchange
|
12
12
|
alias :mq :channel
|
13
|
-
|
13
|
+
|
14
14
|
def initialize(queue_name, options = {})
|
15
15
|
@prefetch ||= options.delete(:prefetch) || 1
|
16
|
-
|
16
|
+
|
17
17
|
options = {
|
18
18
|
:auto_delete => true,
|
19
19
|
:durable => false,
|
@@ -21,28 +21,28 @@ module Cloudist
|
|
21
21
|
}.update(options)
|
22
22
|
|
23
23
|
@queue_name, @options = queue_name, options
|
24
|
-
|
24
|
+
|
25
25
|
setup
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def inspect
|
29
29
|
"<#{self.class.name} queue_name=#{queue_name}>"
|
30
30
|
end
|
31
31
|
|
32
32
|
def setup
|
33
33
|
return if @setup.eql?(true)
|
34
|
-
|
34
|
+
|
35
35
|
@channel ||= AMQP::Channel.new(Cloudist.connection) do
|
36
|
-
channel.prefetch(self.prefetch, false) if self.prefetch
|
36
|
+
channel.prefetch(self.prefetch, false) if self.prefetch
|
37
37
|
end
|
38
|
-
|
38
|
+
|
39
39
|
@queue = @channel.queue(queue_name, options)
|
40
|
-
|
40
|
+
|
41
41
|
setup_exchange
|
42
|
-
|
42
|
+
|
43
43
|
@setup = true
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
def setup_exchange
|
47
47
|
@exchange = channel.direct("")
|
48
48
|
end
|
@@ -61,13 +61,13 @@ module Cloudist
|
|
61
61
|
s += " exchange=#{exchange.name}" if exchange
|
62
62
|
s
|
63
63
|
end
|
64
|
-
|
64
|
+
|
65
65
|
def subscribe(&block)
|
66
66
|
queue.subscribe(:ack => true) do |queue_header, encoded_message|
|
67
67
|
# next if Cloudist.closing?
|
68
68
|
|
69
69
|
request = Cloudist::Request.new(self, encoded_message, queue_header)
|
70
|
-
|
70
|
+
|
71
71
|
handle_request = proc {
|
72
72
|
begin
|
73
73
|
raise Cloudist::ExpiredMessage if request.expired?
|
@@ -86,11 +86,11 @@ module Cloudist
|
|
86
86
|
# log.debug("Finished Job in #{finished - request.start} seconds")
|
87
87
|
end
|
88
88
|
}
|
89
|
-
|
89
|
+
|
90
90
|
handle_ack = proc {
|
91
91
|
request.ack
|
92
92
|
}
|
93
|
-
|
93
|
+
|
94
94
|
EM.defer(handle_request, handle_ack)
|
95
95
|
end
|
96
96
|
log.info "AMQP Subscribed: #{tag}"
|
@@ -128,4 +128,4 @@ module Cloudist
|
|
128
128
|
end
|
129
129
|
end
|
130
130
|
end
|
131
|
-
end
|
131
|
+
end
|
@@ -1,26 +1,26 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class JobQueue < Cloudist::Queues::BasicQueue
|
3
|
-
|
3
|
+
|
4
4
|
def initialize(queue_name, options={})
|
5
5
|
options[:auto_delete] = false
|
6
6
|
options[:nowait] = false
|
7
|
-
|
7
|
+
|
8
8
|
@prefetch = Cloudist.worker_prefetch
|
9
9
|
puts "Prefetch: #{@prefetch}"
|
10
10
|
super(queue_name, options)
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
# def initialize(queue_name, options={})
|
14
14
|
# @prefetch = 1
|
15
15
|
# # opts[:auto_delete] = false
|
16
|
-
#
|
16
|
+
#
|
17
17
|
# super(queue_name, options)
|
18
18
|
# end
|
19
|
-
|
19
|
+
|
20
20
|
# def setup_exchange
|
21
21
|
# @exchange = channel.direct(queue_name)
|
22
22
|
# queue.bind(exchange)
|
23
23
|
# end
|
24
|
-
|
24
|
+
|
25
25
|
end
|
26
26
|
end
|
@@ -3,23 +3,23 @@ module Cloudist
|
|
3
3
|
def initialize(queue_name, options={})
|
4
4
|
options[:auto_delete] = true
|
5
5
|
options[:nowait] = false
|
6
|
-
|
6
|
+
|
7
7
|
@prefetch = Cloudist.listener_prefetch
|
8
|
-
|
8
|
+
|
9
9
|
super(queue_name, options)
|
10
10
|
end
|
11
|
-
|
11
|
+
|
12
12
|
# def subscribe(&block)
|
13
13
|
# super do |request|
|
14
14
|
# yield request if block_given?
|
15
15
|
# teardown
|
16
16
|
# end
|
17
17
|
# end
|
18
|
-
|
18
|
+
|
19
19
|
# def teardown
|
20
20
|
# queue.delete
|
21
21
|
# super
|
22
22
|
# end
|
23
|
-
|
23
|
+
|
24
24
|
end
|
25
25
|
end
|
data/lib/cloudist/request.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Request
|
3
3
|
include Cloudist::Encoding
|
4
|
-
|
4
|
+
|
5
5
|
attr_reader :queue_header, :qobj, :payload, :start, :headers, :body
|
6
6
|
|
7
7
|
def initialize(queue, encoded_body, queue_header)
|
@@ -9,13 +9,13 @@ module Cloudist
|
|
9
9
|
|
10
10
|
@body = decode(encoded_body)
|
11
11
|
@headers = parse_custom_headers(queue_header)
|
12
|
-
|
12
|
+
|
13
13
|
@payload = Cloudist::Payload.new(encoded_body, queue_header.headers.dup)
|
14
14
|
@headers = @payload.headers
|
15
15
|
|
16
16
|
@start = Time.now.utc.to_f
|
17
17
|
end
|
18
|
-
|
18
|
+
|
19
19
|
def parse_custom_headers(amqp_headers)
|
20
20
|
h = amqp_headers.headers.dup
|
21
21
|
|
@@ -26,7 +26,7 @@ module Cloudist
|
|
26
26
|
|
27
27
|
h
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
def for_message
|
31
31
|
[body.dup, queue_header.headers.dup]
|
32
32
|
end
|
@@ -42,7 +42,7 @@ module Cloudist
|
|
42
42
|
def mq
|
43
43
|
qobj.channel
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
def channel
|
47
47
|
mq
|
48
48
|
end
|
data/lib/cloudist/utils.rb
CHANGED
@@ -22,7 +22,7 @@ module Cloudist
|
|
22
22
|
def generate_name_for_instance(name)
|
23
23
|
"#{name}.#{Socket.gethostname}"
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
# DEPRECATED
|
27
27
|
def generate_reply_to(name)
|
28
28
|
"#{reply_prefix(name)}.#{generate_sym}"
|
@@ -40,15 +40,15 @@ module Cloudist
|
|
40
40
|
]
|
41
41
|
"%04x%04x%04x%04x%04x%06x%06x" % values
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
def encode_message(object)
|
45
45
|
Marshal.dump(object).to_s
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def decode_message(string)
|
49
49
|
Marshal.load(string)
|
50
50
|
end
|
51
|
-
|
51
|
+
|
52
52
|
def decode_json(string)
|
53
53
|
if defined? ActiveSupport::JSON
|
54
54
|
ActiveSupport::JSON.decode string
|
@@ -56,6 +56,6 @@ module Cloudist
|
|
56
56
|
JSON.load string
|
57
57
|
end
|
58
58
|
end
|
59
|
-
|
59
|
+
|
60
60
|
end
|
61
61
|
end
|
data/lib/cloudist/worker.rb
CHANGED
@@ -1,37 +1,37 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Worker
|
3
|
-
|
3
|
+
|
4
4
|
attr_reader :job, :queue, :payload
|
5
|
-
|
5
|
+
|
6
6
|
def initialize(job, queue)
|
7
7
|
@job, @queue, @payload = job, queue, job.payload
|
8
|
-
|
8
|
+
|
9
9
|
# Do custom initialization
|
10
10
|
self.setup if self.respond_to?(:setup)
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
def data
|
14
14
|
job.data
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
def headers
|
18
18
|
job.headers
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
def id
|
22
22
|
job.id
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def process
|
26
26
|
raise NotImplementedError, "Your worker class must subclass this method"
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
def log
|
30
30
|
Cloudist.log
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
class GenericWorker < Worker
|
36
36
|
def process(&block)
|
37
37
|
instance_eval(&block)
|
@@ -6,28 +6,28 @@ describe "Cloudist" do
|
|
6
6
|
overload_amqp
|
7
7
|
reset_broker
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
it "should create a queue and exchange" do
|
11
11
|
# MQ.stubs(:direct).with(:name).returns(true)
|
12
12
|
@mq = mock("MQ")
|
13
13
|
@exchange = mock("MQ Exchange")
|
14
14
|
@queue = mock("MQ Queue")
|
15
|
-
|
15
|
+
|
16
16
|
@queue.expects(:bind).with(@exchange)
|
17
17
|
# @mq.expects(:queue).with("make.sandwich")
|
18
|
-
|
18
|
+
|
19
19
|
bq = Cloudist::Queues::BasicQueue.new("make.sandwich")
|
20
|
-
bq.
|
21
|
-
bq.
|
22
|
-
bq.
|
23
|
-
|
20
|
+
bq.stub(:q).and_return(@queue)
|
21
|
+
bq.stub(:mq).and_return(@mq)
|
22
|
+
bq.stub(:ex).and_return(@exchange)
|
23
|
+
|
24
24
|
bq.setup
|
25
|
-
|
25
|
+
|
26
26
|
bq.q.should_not be_nil
|
27
27
|
bq.ex.should_not be_nil
|
28
28
|
bq.mq.should_not be_nil
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
end
|
data/spec/cloudist/job_spec.rb
CHANGED
@@ -4,20 +4,20 @@ describe Cloudist::Job do
|
|
4
4
|
before(:each) do
|
5
5
|
@payload = Cloudist::Payload.new({:bread => 'white'})
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
it "should be constructable with payload" do
|
9
9
|
job = Cloudist::Job.new(@payload)
|
10
10
|
job.payload.should == @payload
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
it "should be constructable with payload and return ID" do
|
14
14
|
job = Cloudist::Job.new(@payload)
|
15
15
|
job.id.should == @payload.id
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
it "should be constructable with payload and return data" do
|
19
19
|
job = Cloudist::Job.new(@payload)
|
20
20
|
job.data.should == @payload.body
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
end
|
@@ -2,22 +2,22 @@ require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
|
|
2
2
|
|
3
3
|
describe Cloudist::Payload do
|
4
4
|
include Cloudist::Encoding
|
5
|
-
|
5
|
+
|
6
6
|
it "should accept a hash for data" do
|
7
7
|
pl = Cloudist::Payload.new({:bread => 'white'})
|
8
8
|
pl.body.bread.should == "white"
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
it "should accept encoded message" do
|
12
12
|
pl = Cloudist::Payload.new(encode({:bread => 'white'}))
|
13
13
|
pl.body.bread.should == "white"
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
it "should retrieve id from headers" do
|
17
17
|
pl = Cloudist::Payload.new({:bread => 'white'}, {:message_id => "12345"})
|
18
18
|
pl.id.should == "12345"
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
it "should prepare headers" do
|
22
22
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
23
23
|
payload.body.bread.should == "white"
|
@@ -27,52 +27,52 @@ describe Cloudist::Payload do
|
|
27
27
|
payload.headers.has_key?("published_on").should be_true
|
28
28
|
payload.headers.has_key?("message_id").should be_true
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
it "should extract published_on from data" do
|
32
32
|
time = Time.now.to_f
|
33
33
|
payload = Cloudist::Payload.new({:bread => 'white', :timestamp => time})
|
34
34
|
payload.headers[:published_on].should == time
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
it "should not override timestamp if already present in headers" do
|
38
38
|
time = (Time.now.to_f - 10.0)
|
39
39
|
payload = Cloudist::Payload.new({:bread => 'white'}, {:published_on => time})
|
40
40
|
payload.headers[:published_on].should == time
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
it "should override timestamp if not present" do
|
44
44
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
45
45
|
payload.headers[:published_on].should be_within(0.1).of Time.now.to_f
|
46
46
|
payload.timestamp.should be_within(0.1).of Time.now.to_f
|
47
47
|
end
|
48
|
-
|
48
|
+
|
49
49
|
it "should parse custom headers" do
|
50
50
|
payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white'}), {:published_on => 12345, :message_id => "foo"})
|
51
51
|
payload.headers.to_hash.should == { "published_on"=>12345, "message_id"=>"foo", "ttl"=>300 }
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
it "should create a unique event hash" do
|
55
55
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
56
56
|
payload.id.size.should == 36
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
it "should not create a new message_id unless it doesn't have one" do
|
60
60
|
payload = Cloudist::Payload.new({:bread => 'white'})
|
61
61
|
payload.id.size.should == 36
|
62
62
|
payload = Cloudist::Payload.new({:bread => 'white'}, {:message_id => 'foo'})
|
63
63
|
payload.id.should == 'foo'
|
64
64
|
end
|
65
|
-
|
65
|
+
|
66
66
|
it "should format payload for sending" do
|
67
67
|
payload = Cloudist::Payload.new({:bread => 'white'}, {:message_id => 'foo', :message_type => 'reply'})
|
68
68
|
body, popts = payload.to_a
|
69
69
|
headers = popts[:headers]
|
70
|
-
|
70
|
+
|
71
71
|
body.should == encode(Hashie::Mash.new({:bread => 'white'}))
|
72
72
|
headers[:ttl].should == "300"
|
73
73
|
headers[:message_type].should == 'reply'
|
74
74
|
end
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
end
|
75
|
+
|
76
|
+
|
77
|
+
|
78
|
+
end
|