mb-minion 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile ADDED
@@ -0,0 +1,30 @@
1
+ require "bundler"
2
+ Bundler.setup
3
+
4
+ require "rake"
5
+ require "rake/rdoctask"
6
+ require "rspec"
7
+ require "rspec/core/rake_task"
8
+
9
+ $LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
10
+ require "mb-minion/version"
11
+
12
+ task :build do
13
+ system "gem build mb-minion.gemspec"
14
+ end
15
+
16
+ task :install => :build do
17
+ system "sudo gem install mb-minion-#{Minion::VERSION}.gem"
18
+ end
19
+
20
+ task :release => :build do
21
+ system "git tag -a #{Minion::VERSION} -m 'Tagging #{Minion::VERSION}'"
22
+ system "git push --tags"
23
+ system "gem push mb-minion-#{Minion::VERSION}.gem"
24
+ end
25
+
26
+ Rspec::Core::RakeTask.new(:spec) do |spec|
27
+ spec.pattern = "spec/**/*_spec.rb"
28
+ end
29
+
30
+ task :default => :spec
data/examples/batch.rb ADDED
@@ -0,0 +1,31 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use of batching
5
+ # messages together. This way work can be
6
+ # distributed in small chunks, but worked on
7
+ # in larger groups
8
+ #
9
+
10
+ $:.unshift File.dirname(__FILE__) + '/../lib'
11
+ require 'rubygems'
12
+ require 'mb-minion'
13
+
14
+ include Minion
15
+
16
+ error do |exception,queue,message,headers|
17
+ puts "got an error processing queue #{queue}"
18
+ puts exception.message
19
+ puts exception.backtrace
20
+ end
21
+
22
+ logger do |msg|
23
+ puts "--> #{msg}"
24
+ end
25
+
26
+
27
+ job "do.many", :batch_size => 10 do |msg|
28
+ puts "got #{msg.batch.size} messages"
29
+ end
30
+
31
+ 27.times{ Minion.enqueue 'do.many', {"something" => true} }
@@ -0,0 +1,36 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use batching
5
+ # when we want to wait until the hit an exact
6
+ # number of messages before running
7
+ #
8
+
9
+ $:.unshift File.dirname(__FILE__) + '/../lib'
10
+ require 'rubygems'
11
+ require 'mb-minion'
12
+
13
+ include Minion
14
+
15
+ error do |exception,queue,message,headers|
16
+ puts "got an error processing queue #{queue}"
17
+ puts exception.message
18
+ puts exception.backtrace
19
+ end
20
+
21
+ logger do |msg|
22
+ puts "--> #{msg}"
23
+ end
24
+
25
+
26
+ job "do.only_in_tens", :batch_size => 10, :wait => true do |msg|
27
+ puts "got #{msg.batch.size} messages"
28
+ end
29
+
30
+ # We won't run this, but just so you know, this is the
31
+ # same result as the "batch.rb" example
32
+ job "do.in_tens_or_emtpy", :batch_size => 10, :wait => false do |msg|
33
+ puts "got #{msg.batch.size} messages"
34
+ end
35
+
36
+ 27.times{ Minion.enqueue 'do.only_in_tens', {"something" => true} }
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use batching
5
+ # when we want to wait some # of seconds for an
6
+ # exact number of messages before running,
7
+ # but if we don't get that many, we'll go ahead
8
+ # anyways.
9
+ #
10
+ # This prevents some odd batch counts, but allows
11
+ # for flexibility of the queue size
12
+ #
13
+
14
+ $:.unshift File.dirname(__FILE__) + '/../lib'
15
+ require 'rubygems'
16
+ require 'mb-minion'
17
+
18
+ include Minion
19
+
20
+ error do |exception,queue,message,headers|
21
+ puts "got an error processing queue #{queue}"
22
+ puts exception.message
23
+ puts exception.backtrace
24
+ end
25
+
26
+ logger do |msg|
27
+ puts "--> #{msg}"
28
+ end
29
+
30
+ Thread.new do
31
+ puts "------------------------------------------"
32
+ puts "First, no waiting with artificial delays"
33
+ puts "------------------------------------------"
34
+ 3.times{ Minion.enqueue 'do.no_waiting', {"something" => true} }
35
+ sleep 0.2
36
+ 4.times{ Minion.enqueue 'do.no_waiting', {"something" => true} }
37
+ sleep 5
38
+ puts "------------------------------------------"
39
+ puts "Now if we give it a second, all is well"
40
+ puts "------------------------------------------"
41
+ 3.times{ Minion.enqueue 'do.in_tens_or_wait_2', {"something" => true} }
42
+ sleep 0.2
43
+ 4.times{ Minion.enqueue 'do.in_tens_or_wait_2', {"something" => true} }
44
+ end
45
+
46
+ job "do.no_waiting", :batch_size => 10 do |msg|
47
+ puts "got #{msg.batch.size} messages"
48
+ end
49
+
50
+ job "do.in_tens_or_wait_2", :batch_size => 10, :wait => 2 do |msg|
51
+ puts "got #{msg.batch.size} messages"
52
+ end
data/examples/math.rb ADDED
@@ -0,0 +1,42 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use of chaining
5
+ # callbacks to create building blocks with the
6
+ # message content being just an integer
7
+ #
8
+
9
+ $:.unshift File.dirname(__FILE__) + '/../lib'
10
+ require 'rubygems'
11
+ require 'minion'
12
+
13
+ include Minion
14
+
15
+ error do |exception,queue,message,headers|
16
+ puts "got an error processing queue #{queue}"
17
+ puts exception.message
18
+ puts exception.backtrace
19
+ end
20
+
21
+ logger do |msg|
22
+ puts "--> #{msg}"
23
+ end
24
+
25
+ job "math.incr" do |msg|
26
+ msg.content.to_i + 1
27
+ end
28
+
29
+ job "math.double" do |msg|
30
+ msg.content.to_i * 2
31
+ end
32
+
33
+ job "math.square" do |msg|
34
+ msg.content.to_i * msg.content.to_i
35
+ end
36
+
37
+ job "math.print" do |msg|
38
+ puts "NUMBER -----> #{msg.content}"
39
+ end
40
+
41
+ enqueue([ "math.incr", "math.double", "math.square", "math.incr", "math.double", "math.print" ], 3)
42
+
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use of chaining
5
+ # callbacks to create building blocks with the
6
+ # message content being just a hash
7
+ #
8
+
9
+ $:.unshift File.dirname(__FILE__) + '/../lib'
10
+ require 'rubygems'
11
+ require 'mb-minion'
12
+
13
+ include Minion
14
+
15
+ error do |exception,queue,message,headers|
16
+ puts "got an error processing queue #{queue}"
17
+ puts exception.message
18
+ puts exception.backtrace
19
+ end
20
+
21
+ job "add.bread" do |msg|
22
+ msg.content.merge("bread" => "sourdough")
23
+ end
24
+
25
+ job "add.meat" do |msg|
26
+ msg.content.merge("meat" => "turkey")
27
+ end
28
+
29
+ job "add.condiments" do |msg|
30
+ msg.content.merge("condiments" => "mayo")
31
+ end
32
+
33
+ job "eat.sandwich" do |msg|
34
+ puts "YUM! A #{msg['meat']} on #{msg['bread']} sandwich with #{msg['condiments']}"
35
+ end
36
+
37
+ enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ])
38
+
@@ -0,0 +1,44 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use batching
5
+ # and chaining of callbacks in a single example.
6
+ # Another example of doing a map-reduce operation.
7
+ #
8
+
9
+ $:.unshift File.dirname(__FILE__) + '/../lib'
10
+ require 'rubygems'
11
+ require 'mb-minion'
12
+
13
+ include Minion
14
+
15
+ error do |exception,queue,message,headers|
16
+ puts "got an error processing queue #{queue}"
17
+ puts exception.message
18
+ puts exception.backtrace
19
+ end
20
+
21
+ MAKINGS = {
22
+ 'bread' => %w[wheat rye sourdough white pumpernickle],
23
+ 'meat' => %w[turkey ham pastrami salami],
24
+ 'condiments' => %w[mayo mustard relish sourkraut]
25
+ }
26
+
27
+ job "add.bread", :batch_size => 5 do |msg|
28
+ puts "Puts making #{msg.batch.size} sandwiches"
29
+ msg.batch.map{|s| s.merge("bread" => MAKINGS['bread'].sample)}
30
+ end
31
+
32
+ job "add.meat" do |msg|
33
+ msg.map{|s| s.merge("meat" => MAKINGS['meat'].sample)}
34
+ end
35
+
36
+ job "add.condiments" do |msg|
37
+ msg.map{|s| s.merge("condiments" => MAKINGS['condiments'].sample)}
38
+ end
39
+
40
+ job "eat.sandwich" do |msg|
41
+ msg.each_with_index{|s, i| puts "SANDWICH ##{i}: A #{s['meat']} on #{s['bread']} sandwich with #{s['condiments']}"}
42
+ end
43
+
44
+ 15.times{ enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ]) }
data/examples/when.rb ADDED
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # This example illustrates the use of the :when
5
+ # parameter to allow/disallow subscription to the
6
+ # queue
7
+ #
8
+
9
+ $:.unshift File.dirname(__FILE__) + '/../lib'
10
+ require 'rubygems'
11
+ require 'mb-minion'
12
+
13
+ include Minion
14
+
15
+ error do |exception,queue,message,headers|
16
+ puts "got an error processing queue #{queue}"
17
+ puts exception.message
18
+ puts exception.backtrace
19
+ end
20
+
21
+ logger do |msg|
22
+ puts "--> #{msg}"
23
+ end
24
+
25
+ $listen = true
26
+
27
+ job "do.once", :when => lambda { $listen } do |args|
28
+ puts "Do this one action - then unsubscribe..."
29
+ $listen = false
30
+ end
31
+
32
+ enqueue("do.once",[])
33
+ enqueue("do.once",[])
34
+
@@ -0,0 +1,7 @@
1
+ class String
2
+ # Fake out "force_encoding" if we're on 1.8
3
+ #
4
+ def force_encoding enc
5
+ self
6
+ end unless method_defined? :force_encoding
7
+ end
data/lib/mb-minion.rb ADDED
@@ -0,0 +1,231 @@
1
+ # encoding: utf-8
2
+ require "amqp"
3
+ require "bunny"
4
+ require "json" unless defined? ActiveSupport::JSON
5
+ require "uri"
6
+ require "mb-minion/handler"
7
+ require "mb-minion/version"
8
+ require "mb-minion/message"
9
+ require "mb-ext/string"
10
+
11
+ module Minion
12
+ extend self
13
+
14
+ # Handle when an error gets raised.
15
+ #
16
+ # @example Handle the error.
17
+ # Minion.error(exception)
18
+ #
19
+ # @param [ Exception ] exception The error that was raised.
20
+ #
21
+ # @return [ Object ] The output og the error handler block.
22
+ def alert(exception)
23
+ raise(exception) unless error_handling
24
+ error_handling.call(exception)
25
+ end
26
+
27
+ # Gets the hash of configuration options.
28
+ #
29
+ # @example Get the configuration hash.
30
+ # Minion.config
31
+ #
32
+ # @return [ Hash ] The configuration options.
33
+ def config
34
+ uri = URI.parse(url)
35
+ {
36
+ :vhost => uri.path,
37
+ :host => uri.host,
38
+ :user => uri.user,
39
+ :port => (uri.port || 5672),
40
+ :pass => uri.password
41
+ }
42
+ rescue Object => e
43
+ raise("invalid AMQP_URL: #{uri.inspect} (#{e})")
44
+ end
45
+
46
+ # Add content to the supplied queue or queues. The hash will get converted to
47
+ # JSON and placed on the queue as the JSON string.
48
+ #
49
+ # @example Place data on a single queue.
50
+ # Minion.enqueue("queue.name", { field: "value" })
51
+ #
52
+ # @example Place data on multiple queues.
53
+ # Minion.enqueue([ "queue.first", "queue.second" ], { field: "value" })
54
+ #
55
+ # @param [ String, Array<String> ] name The name or names of the queues.
56
+ # @param [ Hash ] data The payload to send.
57
+ #
58
+ # @raise [ RuntimeError ] If the name is nil or empty.
59
+ def enqueue(queues, data = {})
60
+ raise "Cannot enqueue an empty or nil name" if queues.nil? || queues.empty?
61
+ # Wrap raw data when we receive it
62
+ data = {'content' => data} unless data.class == Hash && data['content']
63
+ if queues.respond_to? :shift
64
+ queue = queues.shift
65
+ data['callbacks'] = queues
66
+ else
67
+ queue = queues
68
+ end
69
+
70
+ # @todo: Durran: Any multi-byte character in the JSON causes a bad_payload
71
+ # error on the rabbitmq side. It seems a fix in the old amqp gem
72
+ # regressed in the new fork.
73
+ encoded = JSON.dump(data).force_encoding("ISO-8859-1")
74
+
75
+ Minion.info("Send: #{queue}:#{encoded}")
76
+ connect do |bunny|
77
+ q = bunny.queue(queue, :durable => true, :auto_delete => false)
78
+ e = bunny.exchange('') # Connect to default exchange
79
+ e.publish(encoded, :key => q.name)
80
+ end
81
+ end
82
+
83
+ # Get the message count for a specific queue
84
+ #
85
+ # @example Get the message count for queue 'minion.test'.
86
+ # Minion.message_count('minion.test')
87
+ #
88
+ # @return [ Fixnum ] the number of messages
89
+ def message_count(queue)
90
+ connect do |bunny|
91
+ return bunny.queue(queue, :durable => true, :auto_delete => false).message_count
92
+ end
93
+ end
94
+
95
+ # Define an optional method of changing the ways errors get handled.
96
+ #
97
+ # @example Define a custom error handler.
98
+ # Minion.error do |e|
99
+ # puts "I got an error - #{e.message}"
100
+ # end
101
+ #
102
+ # @param [ Proc ] block The block that will handle the error.
103
+ def error(&block)
104
+ @@error_handling = block
105
+ end
106
+
107
+ # Runs each of the handlers.
108
+ #
109
+ # @example Check all handlers.
110
+ # Minion.check_handlers
111
+ def execute_handlers
112
+ @@handlers.each { |handler| handler.execute }
113
+ end
114
+
115
+ # Log the supplied information message.
116
+ #
117
+ # @example Log the message.
118
+ # Minion.info("something happened")
119
+ #
120
+ # @return [ Object ] The output of the logging block.
121
+ def info(message)
122
+ logging.call(message)
123
+ end
124
+
125
+ # Sets up a subscriber to a queue to process jobs.
126
+ #
127
+ # @example Set up the subscriber.
128
+ # Minion.job "my.queue.name" do |attributes|
129
+ # puts "Here's the message data: #{attributes"
130
+ # end
131
+ #
132
+ # @param [ String ] queue The queue to subscribe to.
133
+ # @param [ Hash ] options Options for the subscriber.
134
+ #
135
+ # @option options [ lambda ] :when Conditionally process the job.
136
+ # @option options [ boolean ] :ack Should we automatically ack the message?
137
+ def job(queue, options = {}, &block)
138
+ Minion::Handler.new(queue, block, options).tap do |handler|
139
+ @@handlers ||= []
140
+ at_exit { Minion.run } if @@handlers.size == 0
141
+ @@handlers << handler
142
+ end
143
+ end
144
+
145
+ # Define an optional method of changing the ways logging is handled.
146
+ #
147
+ # @example Define a custom logger.
148
+ # Minion.logger do |message|
149
+ # puts "Something did something - #{message}"
150
+ # end
151
+ #
152
+ # @param [ Proc ] block The block that will handle the logging.
153
+ def logger(&block)
154
+ @@logging = block
155
+ end
156
+
157
+ # Runs the minion subscribers.
158
+ #
159
+ # @example Run the subscribers.
160
+ # Minion.run
161
+ def run
162
+ Minion.info("Starting minion")
163
+ Signal.trap("INT") { AMQP.stop { EM.stop } }
164
+ Signal.trap("TERM") { AMQP.stop { EM.stop } }
165
+
166
+ EM.run do
167
+ AMQP.start(config) do
168
+ AMQP::Channel.new.prefetch(1)
169
+ execute_handlers
170
+ end
171
+ end
172
+ end
173
+
174
+ # Get the url for the amqp server.
175
+ #
176
+ # @example Get the url.
177
+ # Minion.url
178
+ #
179
+ # @return [ String ] The url.
180
+ def url
181
+ @@url ||= (ENV["AMQP_URL"] || "amqp://guest:guest@localhost/")
182
+ end
183
+
184
+ # Set the url to the amqp server.
185
+ #
186
+ # @example Set the url.
187
+ # Minion.url = "amqp://user:password@host:port/vhost"
188
+ #
189
+ # @return [ String ] The new url.
190
+ def url=(url)
191
+ @@url = url
192
+ end
193
+
194
+ private
195
+
196
+ # Get the bunny instance which is used for the synchronous communication.
197
+ #
198
+ # @example Get the bunny.
199
+ # Minion.bunny
200
+ #
201
+ # @return [ Bunny ] The new bunny, all configured.
202
+ def connect
203
+ Bunny.new(config).tap do |bunny|
204
+ bunny.start
205
+ yield(bunny) if block_given?
206
+ bunny.stop
207
+ end
208
+ end
209
+
210
+ # Get the error handler for this class.
211
+ #
212
+ # @example Get the error handler.
213
+ # Minion.error_handling
214
+ #
215
+ # @return [ lambda, nil ] The handler or nil.
216
+ def error_handling
217
+ @@error_handling ||= nil
218
+ end
219
+
220
+ # Get the logger for this class. If nothing had been specified will default
221
+ # to a basic time/message print.
222
+ #
223
+ # @example Get the logger.
224
+ # Minion.logging
225
+ #
226
+ # @return [ lambda ] The logger.
227
+ def logging
228
+ @@logging ||= lambda { |msg| puts("#{Time.now} :minion: #{msg}") }
229
+ end
230
+ end
231
+