smith 0.5.12 → 0.5.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/bin/smithctl +16 -19
  2. data/lib/smith.rb +25 -41
  3. data/lib/smith/agent.rb +96 -66
  4. data/lib/smith/agent_monitoring.rb +3 -4
  5. data/lib/smith/agent_process.rb +16 -9
  6. data/lib/smith/amqp_errors.rb +53 -0
  7. data/lib/smith/application/agency.rb +23 -20
  8. data/lib/smith/bootstrap.rb +3 -3
  9. data/lib/smith/command.rb +2 -2
  10. data/lib/smith/command_base.rb +4 -0
  11. data/lib/smith/commands/agency/agents.rb +19 -19
  12. data/lib/smith/commands/agency/kill.rb +6 -2
  13. data/lib/smith/commands/agency/list.rb +2 -4
  14. data/lib/smith/commands/agency/logger.rb +27 -28
  15. data/lib/smith/commands/agency/metadata.rb +1 -5
  16. data/lib/smith/commands/agency/object_count.rb +13 -11
  17. data/lib/smith/commands/agency/restart.rb +18 -9
  18. data/lib/smith/commands/agency/start.rb +34 -25
  19. data/lib/smith/commands/agency/stop.rb +58 -41
  20. data/lib/smith/commands/agency/version.rb +10 -10
  21. data/lib/smith/commands/common.rb +7 -4
  22. data/lib/smith/commands/smithctl/acl.rb +46 -37
  23. data/lib/smith/commands/smithctl/commands.rb +1 -1
  24. data/lib/smith/commands/smithctl/firehose.rb +30 -0
  25. data/lib/smith/commands/smithctl/pop.rb +39 -32
  26. data/lib/smith/commands/smithctl/push.rb +70 -51
  27. data/lib/smith/commands/smithctl/rm.rb +32 -9
  28. data/lib/smith/commands/smithctl/subscribe.rb +36 -0
  29. data/lib/smith/commands/smithctl/top.rb +1 -1
  30. data/lib/smith/exceptions.rb +2 -0
  31. data/lib/smith/messaging/acl/agency_command.proto +4 -0
  32. data/lib/smith/messaging/acl/default.rb +8 -1
  33. data/lib/smith/messaging/amqp_options.rb +2 -2
  34. data/lib/smith/messaging/message_counter.rb +21 -0
  35. data/lib/smith/messaging/payload.rb +47 -49
  36. data/lib/smith/messaging/queue.rb +50 -0
  37. data/lib/smith/messaging/queue_definition.rb +18 -0
  38. data/lib/smith/messaging/queue_factory.rb +20 -29
  39. data/lib/smith/messaging/receiver.rb +211 -173
  40. data/lib/smith/messaging/requeue.rb +91 -0
  41. data/lib/smith/messaging/sender.rb +184 -28
  42. data/lib/smith/messaging/util.rb +72 -0
  43. data/lib/smith/queue_definitions.rb +11 -0
  44. data/lib/smith/version.rb +1 -1
  45. metadata +18 -10
  46. data/lib/smith/messaging/endpoint.rb +0 -116
@@ -0,0 +1,91 @@
1
+ # -*- encoding: utf-8 -*-
2
+ module Smith
3
+ module Messaging
4
+ class Requeue
5
+
6
+ include Logger
7
+
8
+ def initialize(message, metadata, opts={})
9
+ @message = message
10
+ @metadata = metadata
11
+ @queue = opts[:queue]
12
+ @exchange = opts[:exchange]
13
+
14
+ @count = opts[:count] || 3
15
+ @delay = opts[:delay] || 5
16
+ @strategy = opts[:strategy] || :linear
17
+
18
+ @on_requeue = opts[:on_requeue] || ->(count, total_count, cumulative_delay) {
19
+ logger.info { "Requeuing (#{@strategy}) message on queue: #{@queue.name}, count: #{count} of #{total_count}." }
20
+ }
21
+
22
+ @on_requeue_limit = opts[:on_requeue_limit] || ->(message, count, total_count, cumulative_delay) {
23
+ logger.info { "Requeue limit reached: #{total_count} for queue: #{@queue.name}, cummulative delay: #{cumulative_delay}s." }
24
+ }
25
+ end
26
+
27
+ def requeue
28
+ requeue_with_strategy do
29
+ opts = @queue.opts.clone.tap do |o|
30
+ o.delete(:queue)
31
+ o.delete(:exchange)
32
+
33
+ o[:headers] = increment_requeue_count
34
+ o[:routing_key] = @queue.name
35
+ o[:type] = @metadata.type
36
+ end
37
+
38
+ logger.verbose { "Requeuing to: #{@queue.name}. [options]: #{opts}" }
39
+ logger.verbose { "Requeuing to: #{@queue.name}. [message]: #{@message}" }
40
+
41
+ @exchange.publish(ACL::Payload.new(ACL::Factory.create(@metadata.type, @message)).encode, opts)
42
+ end
43
+ end
44
+
45
+ private
46
+
47
+ def current_requeue_number
48
+ @metadata.headers['requeue'] || 0
49
+ end
50
+
51
+ def increment_requeue_count
52
+ @metadata.headers.tap do |m|
53
+ m['requeue'] = (m['requeue']) ? m['requeue'] + 1 : 1
54
+ end
55
+ end
56
+
57
+ def requeue_with_strategy(&block)
58
+ if current_requeue_number < @count
59
+ method = "#{@strategy}_strategy".to_sym
60
+ if respond_to?(method, true)
61
+ cumulative_delay = send(method, @delay)
62
+ @on_requeue.call(current_requeue_number + 1, @count, @delay * current_requeue_number)
63
+ EM.add_timer(cumulative_delay) do
64
+ block.call(cumulative_delay, current_requeue_number + 1)
65
+ end
66
+ else
67
+ raise RuntimeError, "Unknown requeue strategy. #{method}"
68
+ end
69
+ else
70
+ @on_requeue_limit.call(@message, current_requeue_number + 1, @count, @delay * current_requeue_number)
71
+ end
72
+ end
73
+
74
+ def exponential_no_initial_delay_strategy(delay)
75
+ delay * (2 ** current_requeue_number - 1)
76
+ end
77
+
78
+ def exponential_strategy(delay)
79
+ delay * (2 ** current_requeue_number)
80
+ end
81
+
82
+ def linear_strategy(delay)
83
+ delay * (current_requeue_number + 1)
84
+ end
85
+
86
+ def denormalise(name)
87
+ name.gsub(/^#{Regexp.escape(Smith.config.smith.namespace)}./, '')
88
+ end
89
+ end
90
+ end
91
+ end
@@ -1,61 +1,217 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  module Smith
3
3
  module Messaging
4
- class Sender < Endpoint
4
+
5
+ class Sender
5
6
 
6
7
  include Logger
8
+ include Util
9
+
10
+ attr_accessor :queue_name
11
+
12
+ def initialize(queue_definition, opts={}, &blk)
13
+
14
+ @queue_name, opts = get_queue_name_and_options(queue_definition, opts)
15
+
16
+ @reply_container = {}
17
+
18
+ normalised_queue_name = normalise(@queue_name)
19
+
20
+ prefetch = option_or_default(opts, :prefetch, Smith.config.agent.prefetch)
21
+
22
+ @options = AmqpOptions.new(opts)
23
+ @options.routing_key = normalised_queue_name
7
24
 
8
- attr_accessor :options
25
+ @message_counts = Hash.new(0)
9
26
 
10
- def initialize(queue_name, opts={})
11
- @auto_ack = opts.delete(:auto_ack) || true
12
- @threading = opts.delete(:threading) || false
13
- super(queue_name, AmqpOptions.new(opts))
27
+ @exchange_completion = EM::Completion.new
28
+ @queue_completion = EM::Completion.new
29
+ @channel_completion = EM::Completion.new
30
+
31
+ open_channel(:prefetch => prefetch) do |channel|
32
+ @channel_completion.succeed(channel)
33
+ channel.direct(normalised_queue_name, @options.exchange) do |exchange|
34
+
35
+ exchange.on_return do |basic_return,metadata,payload|
36
+ logger.error { "#{ACL::Payload.decode(payload.clone, metadata.type)} returned! Exchange: #{reply_code.exchange}, reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}" }
37
+ logger.error { "Properties: #{metadata.properties}" }
38
+ end
39
+
40
+ channel.queue(normalised_queue_name, @options.queue) do |queue|
41
+ queue.bind(exchange, :routing_key => normalised_queue_name)
42
+
43
+ @queue_completion.succeed(queue)
44
+ @exchange_completion.succeed(exchange)
45
+ end
46
+ end
47
+ end
48
+
49
+ blk.call(self) if blk
14
50
  end
15
51
 
16
- def publish(message, opts={}, &block)
17
- _publish(message, options.publish({:type => message.type}, opts) , &block)
52
+ # If reply queue is set the block will be called when the message
53
+ # recipient replies to the message and it is received.
54
+ #
55
+ # If a block is passed to this method but the :reply_queue option
56
+ # is not set it will be called when the message has been safely
57
+ # published.
58
+ #
59
+ # If the :reply_queue is an empty string then a random queue name
60
+ # will be generated.
61
+ def publish(payload, opts={}, &blk)
62
+ if @reply_queue_completion
63
+ @reply_queue_completion.completion do |reply_queue|
64
+ message_id = random
65
+ logger.verbose { "message_id: #{message_id}" }
66
+
67
+ #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
68
+ #### TODO if there is a timeout delete ####
69
+ #the proc from the @reply_container. ####
70
+ #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
71
+ @reply_container[message_id] = {:reply_proc => @reply_proc, :timeout => @timeout.clone.tap {|t| t.set_timeout(message_id) }}
72
+ _publish(ACL::Payload.new(payload), @options.publish(opts, {:reply_to => reply_queue.queue_name, :message_id => message_id}))
73
+ end
74
+ else
75
+ _publish(ACL::Payload.new(payload), @options.publish(opts), &blk)
76
+ end
18
77
  end
19
78
 
20
- def publish_and_receive(message, &block)
21
- message_id = random
22
- Receiver.new(message_id, :auto_delete => true).ready do |receiver|
79
+ def on_timeout(timeout, &blk)
80
+ @timeout = Timeout.new(timeout, &blk)
81
+ end
23
82
 
24
- receiver.subscribe do |r|
25
- raise "Incorrect correlation_id: #{metadata.correlation_id}" if r.metadata.correlation_id != message_id
83
+ # Set up a listener that will receive replies from the published
84
+ # messages. You must publish with intent to reply -- tee he.
85
+ #
86
+ # If you pass in a queue_name the same queue name will get used for every
87
+ # reply. This means that there are no create and teardown costs for every
88
+ # for every message. If no queue_name is given a random one will be
89
+ # assigned.
90
+ def on_reply(opts={}, &blk)
91
+ @reply_proc = blk
92
+ @timeout ||= Timeout.new(Smith.config.agency.timeout, :queue_name => @queue_name)
93
+ reply_queue_name = opts.delete(:reply_queue_name) || random
26
94
 
27
- cancel_timeout
95
+ options = {:auto_delete => false, :auto_ack => false}.merge(opts)
28
96
 
29
- block.call(r)
97
+ logger.debug { "reply queue: #{reply_queue_name}" }
30
98
 
31
- # Cancel the receive queue. Queues get left behind because the reply queue is
32
- # still listening. By cancelling the consumer it releases the queue and exchange.
33
- r.metadata.channel.consumers.each do |k,v|
34
- if k.start_with?(receiver.queue_name)
35
- logger.verbose { "Cancelling: #{k}" }
36
- v.cancel
99
+ @reply_queue_completion ||= EM::Completion.new.tap do |completion|
100
+ Receiver.new(reply_queue_name, options) do |queue|
101
+ queue.subscribe do |payload, receiver|
102
+ @reply_container.delete(receiver.correlation_id).tap do |reply|
103
+ if reply
104
+ reply[:timeout].cancel_timeout
105
+ reply[:reply_proc].call(payload, receiver)
106
+ else
107
+ logger.error { "Reply message has no correlation_id: #{reply.inspect}" }
108
+ end
37
109
  end
38
110
  end
111
+
112
+ EM.next_tick do
113
+ completion.succeed(queue)
114
+ end
39
115
  end
116
+ end
117
+ end
40
118
 
41
- # DO NOT MOVE THIS OUTSIDE THE READY BLOCK: YOU WILL LOSE MESSAGES. The reason is
42
- # that a message can be published and responded too before the receive queue is set up.
43
- _publish(message, options.publish(:reply_to => message_id, :message_id => message_id, :type => message.type))
119
+ def delete(&blk)
120
+ @queue_completion.completion do |queue|
121
+ queue.delete do
122
+ @exchange_completion.completion do |exchange|
123
+ exchange.delete do
124
+ @channel_completion.completion do |channel|
125
+ channel.close(&blk)
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end
132
+
133
+ # This gets called if there is a mismatch in the message_id & correlation_id.
134
+ def on_reply_error(&blk)
135
+ @reply_error = blk
136
+ end
137
+
138
+ def status(&blk)
139
+ @queue_completion.completion do |queue|
140
+ queue.status do |num_messages, num_consumers|
141
+ blk.call(num_messages, num_consumers)
142
+ end
143
+ end
144
+ end
145
+
146
+ def message_count(&blk)
147
+ status do |messages|
148
+ blk.call(messages)
149
+ end
150
+ end
151
+
152
+ def consumer_count(&blk)
153
+ status do |_, consumers|
154
+ blk.call(consumers)
155
+ end
156
+ end
157
+
158
+ def counter
159
+ @message_counts[@queue_name]
160
+ end
161
+
162
+ # Define a channel error handler.
163
+ def on_error(chain=false, &blk)
164
+ @channel_completion.completion do |channel|
165
+ channel.on_error(&blk)
44
166
  end
45
167
  end
46
168
 
47
169
  private
48
170
 
49
- def _publish(message, opts, &block)
50
- logger.verbose { "Publishing to: [queue]: #{denormalized_queue_name}. [options]: #{opts}" }
51
- logger.verbose { "Payload content: [queue]: #{denormalized_queue_name}, [metadata type]: #{message.type}, [message]: #{message.inspect}" }
171
+ def _publish(message, opts, &blk)
172
+ logger.verbose { "Publishing to: [queue]: #{@queue_name}. [options]: #{opts}" }
173
+ logger.verbose { "Payload content: [queue]: #{@queue_name}, [metadata type]: #{message._type}, [message]: #{message.inspect}" }
52
174
  if message.initialized?
53
175
  increment_counter
54
- exchange.publish(message.encode, opts, &block)
176
+ type = (message.respond_to?(:_type)) ? message._type : message.type
177
+ @exchange_completion.completion do |exchange|
178
+ exchange.publish(message.encode, opts.merge(:type => type), &blk)
179
+ end
55
180
  else
56
181
  raise IncompletePayload, "Message is incomplete: #{message.to_s}"
57
182
  end
58
183
  end
184
+
185
+ def increment_counter(value=1)
186
+ @message_counts[@queue_name] += value
187
+ end
188
+ end
189
+
190
+ class Timeout
191
+ def initialize(timeout, opts={}, &blk)
192
+ @timeout_proc = blk || proc { |message_id| raise ACLTimeoutError, "Message not received within the timeout period#{(message_id) ? ": #{message_id}" : ""}" }
193
+ @timeout_duration = timeout
194
+ end
195
+
196
+ def set_timeout(message_id)
197
+ @message_id = message_id
198
+ cancel_timeout
199
+ if @timeout_duration
200
+ @timeout = EventMachine::Timer.new(@timeout_duration) do
201
+ @timeout_proc.call(message_id)
202
+ end
203
+ else
204
+ raise ArgumentError, "on_timeout not set."
205
+ end
206
+ end
207
+
208
+ def timeout?
209
+ !@timeout_duration.nil?
210
+ end
211
+
212
+ def cancel_timeout
213
+ @timeout.cancel if @timeout
214
+ end
59
215
  end
60
216
  end
61
217
  end
@@ -0,0 +1,72 @@
1
+ # -*- encoding: utf-8 -*-
2
+ module Smith
3
+ module Messaging
4
+
5
+ module Util
6
+
7
+ def number_of_messages
8
+ status do |num_messages, _|
9
+ yield num_messages
10
+ end
11
+ end
12
+
13
+ def number_of_consumers
14
+ status do |_, num_consumers|
15
+ yield num_consumers
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ include AmqpErrors
22
+ include Logging
23
+
24
+ def open_channel(opts={}, &blk)
25
+ AMQP::Channel.new(Smith.connection) do |channel,ok|
26
+ logger.verbose { "Opened channel: #{"%x" % channel.object_id}" }
27
+
28
+ # Set up auto-recovery. This will ensure that amqp will
29
+ # automatically reconnet to the broker if there is an error.
30
+ channel.auto_recovery = true
31
+ logger.verbose { "Channel auto recovery set to ture" }
32
+
33
+ # Set up QOS. If you do not do this then any subscribes will get
34
+ # overwhelmed if there are too many messages.
35
+ prefetch = opts[:prefetch] || Smith.config.agent.prefetch
36
+
37
+ channel.prefetch(prefetch)
38
+ logger.verbose { "AMQP prefetch set to: #{prefetch}" }
39
+
40
+ blk.call(channel)
41
+ end
42
+ end
43
+
44
+ def normalise(name)
45
+ "#{Smith.config.smith.namespace}.#{name}"
46
+ end
47
+
48
+ def random(prefix = '', suffix = '')
49
+ "#{prefix}#{SecureRandom.hex(8)}#{suffix}"
50
+ end
51
+
52
+ # Return the queue name and options based on whether the
53
+ # queue_definition is of type QueueDefinition.
54
+ def get_queue_name_and_options(queue_definition, opts)
55
+ (queue_definition.is_a?(QueueDefinition)) ? queue_definition.to_a : [queue_definition, opts]
56
+ end
57
+
58
+ def option_or_default(options, key, default, &blk)
59
+ if options.is_a?(Hash)
60
+ if options.key?(key)
61
+ v = options.delete(key)
62
+ (blk) ? blk.call(v) : v
63
+ else
64
+ default
65
+ end
66
+ else
67
+ raise ArguementError, "Options must be a Hash."
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,11 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module Smith
4
+ module QueueDefinitions
5
+ Agency_control = QueueDefinition.new("#{Smith.hostname}.agency.control", :auto_delete => false, :durable => false, :persistent => false, :strict => true)
6
+ Agent_keepalive = QueueDefinition.new("#{Smith.hostname}.agent.keepalive", :auto_delete => false, :durable => false)
7
+ Agent_lifecycle = QueueDefinition.new("#{Smith.hostname}.agent.lifecycle", :auto_delete => false, :durable => false)
8
+
9
+ Agent_stats = QueueDefinition.new('agent.stats', :durable => false, :auto_delete => false)
10
+ end
11
+ end
data/lib/smith/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Smith
2
- VERSION = "0.5.12"
2
+ VERSION = "0.5.13.1"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: smith
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.12
4
+ version: 0.5.13.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-12-21 00:00:00.000000000 Z
12
+ date: 2013-01-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: amqp
@@ -248,20 +248,22 @@ files:
248
248
  - lib/smith/agent_config.rb
249
249
  - lib/smith/application/agency.rb
250
250
  - lib/smith/commands/agency/agents.rb
251
- - lib/smith/commands/agency/kill.rb
252
- - lib/smith/commands/agency/list.rb
253
251
  - lib/smith/commands/agency/logger.rb
254
252
  - lib/smith/commands/agency/metadata.rb
255
253
  - lib/smith/commands/agency/object_count.rb
254
+ - lib/smith/commands/agency/version.rb
255
+ - lib/smith/commands/agency/kill.rb
256
+ - lib/smith/commands/agency/list.rb
256
257
  - lib/smith/commands/agency/restart.rb
257
258
  - lib/smith/commands/agency/start.rb
258
259
  - lib/smith/commands/agency/stop.rb
259
- - lib/smith/commands/agency/version.rb
260
260
  - lib/smith/commands/smithctl/acl.rb
261
261
  - lib/smith/commands/smithctl/commands.rb
262
+ - lib/smith/commands/smithctl/firehose.rb
262
263
  - lib/smith/commands/smithctl/pop.rb
263
- - lib/smith/commands/smithctl/push.rb
264
264
  - lib/smith/commands/smithctl/rm.rb
265
+ - lib/smith/commands/smithctl/subscribe.rb
266
+ - lib/smith/commands/smithctl/push.rb
265
267
  - lib/smith/commands/smithctl/top.rb
266
268
  - lib/smith/commands/template.rb
267
269
  - lib/smith/commands/common.rb
@@ -276,13 +278,18 @@ files:
276
278
  - lib/smith/messaging/acl/default.rb
277
279
  - lib/smith/messaging/responder.rb
278
280
  - lib/smith/messaging/amqp_options.rb
279
- - lib/smith/messaging/endpoint.rb
280
- - lib/smith/messaging/payload.rb
281
+ - lib/smith/messaging/message_counter.rb
282
+ - lib/smith/messaging/queue.rb
281
283
  - lib/smith/messaging/queue_factory.rb
284
+ - lib/smith/messaging/requeue.rb
285
+ - lib/smith/messaging/payload.rb
286
+ - lib/smith/messaging/queue_definition.rb
282
287
  - lib/smith/messaging/receiver.rb
283
288
  - lib/smith/messaging/sender.rb
289
+ - lib/smith/messaging/util.rb
284
290
  - lib/smith/object_count.rb
285
- - lib/smith/version.rb
291
+ - lib/smith/amqp_errors.rb
292
+ - lib/smith/command_base.rb
286
293
  - lib/smith/acl_compiler.rb
287
294
  - lib/smith/agent.rb
288
295
  - lib/smith/agent_cache.rb
@@ -291,11 +298,12 @@ files:
291
298
  - lib/smith/bootstrap.rb
292
299
  - lib/smith/cache.rb
293
300
  - lib/smith/command.rb
294
- - lib/smith/command_base.rb
295
301
  - lib/smith/config.rb
296
302
  - lib/smith/daemon.rb
297
303
  - lib/smith/exceptions.rb
298
304
  - lib/smith/logger.rb
305
+ - lib/smith/queue_definitions.rb
306
+ - lib/smith/version.rb
299
307
  - lib/smith.rb
300
308
  homepage: http://github.com/filterfish/smith2/
301
309
  licenses: []