micro_q 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/.gitignore +13 -0
  2. data/.rspec +2 -0
  3. data/.travis.yml +16 -0
  4. data/Gemfile +2 -0
  5. data/LICENSE +22 -0
  6. data/README.md +27 -0
  7. data/Rakefile +6 -0
  8. data/lib/micro_q/config.rb +27 -0
  9. data/lib/micro_q/manager/default.rb +38 -0
  10. data/lib/micro_q/manager.rb +1 -0
  11. data/lib/micro_q/methods/active_record.rb +21 -0
  12. data/lib/micro_q/methods/class.rb +13 -0
  13. data/lib/micro_q/methods/instance.rb +13 -0
  14. data/lib/micro_q/methods.rb +25 -0
  15. data/lib/micro_q/middleware/chain.rb +102 -0
  16. data/lib/micro_q/middleware/server/retry.rb +32 -0
  17. data/lib/micro_q/middleware.rb +1 -0
  18. data/lib/micro_q/proxies/base.rb +49 -0
  19. data/lib/micro_q/proxies/class.rb +6 -0
  20. data/lib/micro_q/proxies/instance.rb +9 -0
  21. data/lib/micro_q/proxies.rb +3 -0
  22. data/lib/micro_q/queue/default.rb +90 -0
  23. data/lib/micro_q/queue.rb +1 -0
  24. data/lib/micro_q/util.rb +29 -0
  25. data/lib/micro_q/version.rb +7 -0
  26. data/lib/micro_q/worker/standard.rb +40 -0
  27. data/lib/micro_q/worker.rb +1 -0
  28. data/lib/micro_q.rb +46 -0
  29. data/micro_q.gemspec +27 -0
  30. data/spec/helpers/methods_examples.rb +45 -0
  31. data/spec/lib/config_spec.rb +47 -0
  32. data/spec/lib/manager/default_spec.rb +69 -0
  33. data/spec/lib/methods/active_record_spec.rb +67 -0
  34. data/spec/lib/methods/class_spec.rb +61 -0
  35. data/spec/lib/methods/instance_spec.rb +55 -0
  36. data/spec/lib/micro_q_spec.rb +80 -0
  37. data/spec/lib/middleware/chain_spec.rb +266 -0
  38. data/spec/lib/middleware/server/retry_spec.rb +87 -0
  39. data/spec/lib/proxies/base_spec.rb +184 -0
  40. data/spec/lib/proxies/class_spec.rb +15 -0
  41. data/spec/lib/proxies/instance_spec.rb +41 -0
  42. data/spec/lib/queue/default_spec.rb +158 -0
  43. data/spec/lib/util_spec.rb +51 -0
  44. data/spec/lib/worker/standard_spec.rb +88 -0
  45. data/spec/spec_helper.rb +59 -0
  46. metadata +219 -0
data/.gitignore ADDED
@@ -0,0 +1,13 @@
1
+ *.gem
2
+ *.rbc
3
+ .idea
4
+ .idea/*
5
+ .bundle
6
+ .config
7
+ *.db
8
+ Gemfile.lock
9
+ doc/
10
+ lib/bundler/man
11
+ pkg
12
+ tmp
13
+ log
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
data/.travis.yml ADDED
@@ -0,0 +1,16 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - rbx-19mode
5
+ - 2.0.0
6
+ branches:
7
+ only:
8
+ - master
9
+ notifications:
10
+ email:
11
+ recipients:
12
+ - brian.nort@gmail.com
13
+ matrix:
14
+ allow_failures:
15
+ - rvm: rbx-19mode
16
+ - rvm: 2.0.0
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Brian Norton
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,27 @@
1
+ # MicroQ
2
+
3
+ MicroQ is a per-process asynchronous background queue.
4
+
5
+ It's simple startup and intuitive interface makes it the best choice for new and lagacy apps.
6
+
7
+ ## Installation
8
+
9
+ Add this line to your Gemfile:
10
+
11
+ gem 'micro_q'
12
+
13
+ $ bundle
14
+
15
+ Or install it:
16
+
17
+ $ gem install micro_q
18
+
19
+ ## Usage
20
+
21
+ ## Contributing
22
+
23
+ 1. Fork it
24
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
25
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
26
+ 4. Push to the branch (`git push origin my-new-feature`)
27
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,27 @@
1
+ module MicroQ
2
+ class Config
3
+ def initialize
4
+ @data = {
5
+ 'workers' => 3,
6
+ 'timeout' => 120,
7
+ 'interval' => 5,
8
+ 'middleware' => MicroQ::Middleware::Chain.new
9
+ }
10
+ end
11
+
12
+ def []=(key, value)
13
+ @data[key.to_s] = value
14
+ end
15
+
16
+ def [](key)
17
+ @data[key.to_s]
18
+ end
19
+
20
+ def method_missing(method, *args)
21
+ case method
22
+ when /(.+)=$/ then @data[$1] = args.first
23
+ else @data[method.to_s]
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,38 @@
1
+ module MicroQ
2
+ module Manager
3
+ ##
4
+ # The default manager implementation.
5
+ # Wrapper for a Queue and a pool of Workers. At each time slice
6
+ # after start! was called, try to dequeue messages from the queue.
7
+ # Perform each message on the worker pool.
8
+ #
9
+ # The pool of workers (more info):
10
+ # https://github.com/celluloid/celluloid/wiki/Pools
11
+ #
12
+ # The pool manages asynchronously assigning messages to available
13
+ # workers, handles exceptions by restarting the dead actors and
14
+ # is generally a beautiful abstraction on top of a group of linked
15
+ # actors/threads.
16
+ #
17
+ class Default
18
+ include Celluloid
19
+
20
+ attr_reader :queue, :workers
21
+
22
+ def initialize
23
+ @queue = MicroQ::Queue::Default.new
24
+ @workers = MicroQ::Worker::Standard.pool(:size => MicroQ.config.workers)
25
+ end
26
+
27
+ def start
28
+ if (messages = queue.dequeue).any?
29
+ messages.each do |message|
30
+ workers.perform!(message)
31
+ end
32
+ end
33
+
34
+ after(5) { start }
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1 @@
1
+ require 'micro_q/manager/default'
@@ -0,0 +1,21 @@
1
+ module MicroQ
2
+ module Methods
3
+ module ActiveRecord
4
+ def async
5
+ options = {
6
+ :class => self.class,
7
+ :loader => {
8
+ :method => 'find',
9
+ :args => [id]
10
+ }
11
+ }
12
+
13
+ MicroQ::Proxy::Instance.new(options)
14
+ end
15
+ end
16
+ end
17
+ end
18
+
19
+ if defined?(ActiveRecord::Base)
20
+ ActiveRecord::Base.send(:include, MicroQ::Methods::ActiveRecord)
21
+ end
@@ -0,0 +1,13 @@
1
+ module MicroQ
2
+ module Methods
3
+ module Class
4
+ extend MicroQ::Methods::SharedMethods
5
+
6
+ def async
7
+ MicroQ::Proxy::Class.new(:class => self, :loader => {})
8
+ end
9
+ end
10
+ end
11
+ end
12
+
13
+ Object.send(:extend, MicroQ::Methods::Class)
@@ -0,0 +1,13 @@
1
+ module MicroQ
2
+ module Methods
3
+ module Instance
4
+ include MicroQ::Methods::SharedMethods
5
+
6
+ def async
7
+ MicroQ::Proxy::Instance.new(:class => self.class)
8
+ end
9
+ end
10
+ end
11
+ end
12
+
13
+ Object.send(:include, MicroQ::Methods::Instance)
@@ -0,0 +1,25 @@
1
+ module MicroQ
2
+ module Methods
3
+ module SharedMethods
4
+ def method_missing(method, *other)
5
+ super unless /((.+)\_async$)/ === method
6
+
7
+ name = $2 && $2.to_sym
8
+
9
+ # Define the method and call through.
10
+ if name && respond_to?(name)
11
+ define_singleton_method method do |*args|
12
+ async.send(name, *args)
13
+ end
14
+
15
+ async.send(name, *other)
16
+ else
17
+ super
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+
24
+ require 'micro_q/methods/class'
25
+ require 'micro_q/methods/instance'
@@ -0,0 +1,102 @@
1
+ require 'micro_q/middleware/server/retry'
2
+
3
+ module MicroQ
4
+ module Middleware
5
+ ##
6
+ # An Array wrapper that holds the class name of middlewares to call
7
+ # around the execution of messages. The most basic middleware must
8
+ # yield to the given block to allow the message to be invoked. Not
9
+ # yielding causes the message to be dropped and not invoked.
10
+ #
11
+ # A minimal middleware:
12
+ # class MyFunMiddleware
13
+ # def call(worker, message)
14
+ # # Do something fun here ...
15
+ # yield
16
+ # # More fun goes here ...
17
+ # end
18
+ # end
19
+ #
20
+ class Chain
21
+ ##
22
+ # Middleware chain that is run around execution of messages.
23
+ # -- If halted, the message will not be invoked.
24
+ def server
25
+ @server ||= Server.new
26
+ end
27
+
28
+ ##
29
+ # Middleware chain that is run around message push.
30
+ # -- If halted, the message will not enter the queue.
31
+ #
32
+ def client
33
+ @client ||= Client.new
34
+ end
35
+
36
+ class Base
37
+ attr_reader :entries
38
+
39
+ def initialize
40
+ clear
41
+ end
42
+
43
+ ##
44
+ # Add any number of entries to the middleware chain
45
+ #
46
+ def add(*items)
47
+ @entries.concat(items.flatten).uniq!
48
+ end
49
+
50
+ def add_before(before, *items)
51
+ remove(*items)
52
+ @entries.insert(@entries.index(before), *items).uniq! if items.any?
53
+ end
54
+
55
+ def add_after(after, *items)
56
+ remove(*items)
57
+ @entries.insert(@entries.index(after)+1, *items).uniq! if items.any?
58
+ end
59
+
60
+ ##
61
+ # Remove any number of entries from the middleware chain
62
+ #
63
+ def remove(*items)
64
+ @entries.tap do
65
+ items.flatten.each {|item| @entries.delete(item) }
66
+ end
67
+ end
68
+
69
+ def clear
70
+ @entries = []
71
+ end
72
+
73
+ ##
74
+ # Traverse the middleware chain by recursing until we reach the
75
+ # end of the chain and are able to invoke the given block. The block
76
+ # represents a message push (client) or a message invocation (server).
77
+ #
78
+ # It is, however very generic and can be used as middleware around anything.
79
+ #
80
+ def call(*args, &block)
81
+ chain = (index = -1) && -> {
82
+ (index += 1) == entries.length ?
83
+ block.call : entries.at(index).new.call(*args, &chain)
84
+ }
85
+
86
+ chain.call
87
+ end
88
+ end
89
+
90
+ class Server < Base
91
+ def initialize
92
+ @entries = [
93
+ MicroQ::Middleware::Server::Retry
94
+ ]
95
+ end
96
+ end
97
+
98
+ class Client < Base
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,32 @@
1
+ module MicroQ
2
+ module Middleware
3
+ module Server
4
+ ##
5
+ # Capture, re-raise and potentially push a modified message
6
+ # back onto the queue. We add metadata about the retry into the
7
+ # 'retried' key to track the attempts.
8
+ #
9
+ # count: The number of retires thus far
10
+ # at: When the last retry occurred
11
+ # when: The time at which the message will be retried again
12
+ #
13
+ class Retry
14
+ def call(worker, message)
15
+ yield
16
+ rescue Exception => e
17
+ raise e unless message['retry']
18
+
19
+ message['retried'] ||= { 'count' => 0 }
20
+
21
+ message['retried']['count'] += 1
22
+ message['retried']['at'] = Time.now
23
+ message['retried']['when'] = (Time.now + 15).to_f
24
+
25
+ MicroQ.push(message, message['retried'])
26
+
27
+ raise e
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1 @@
1
+ require 'micro_q/middleware/chain'
@@ -0,0 +1,49 @@
1
+ module MicroQ
2
+ module Proxy
3
+ class Base
4
+ attr_reader :errors, :klass, :method, :args, :at
5
+
6
+ def initialize(options={})
7
+ @errors = []
8
+ @options = options
9
+
10
+ parse_and_validate
11
+ end
12
+
13
+ def valid?
14
+ errors.empty?
15
+ end
16
+
17
+ def method_missing(meth, *args)
18
+ @method = meth.to_s
19
+ @args = args
20
+
21
+ defaults = [@options.merge(
22
+ :class => klass,
23
+ :method => method,
24
+ :args => args
25
+ )]
26
+
27
+ defaults << { :when => at } if at
28
+
29
+ MicroQ.push(*defaults)
30
+ end
31
+
32
+ def respond_to?(method)
33
+ super || klass.respond_to?(method)
34
+ end
35
+
36
+ private
37
+
38
+ def parse_and_validate
39
+ @at = (at = @options.fetch(:at, nil)) && at.to_i
40
+ after = @options.fetch(:after, nil)
41
+
42
+ @at = Time.now.to_i + after if after
43
+ @klass = @options[:class] && MicroQ::Util.constantize(@options[:class].to_s)
44
+
45
+ (errors << 'Proxies require a valid class') unless klass
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,6 @@
1
+ module MicroQ
2
+ module Proxy
3
+ class Class < Base
4
+ end
5
+ end
6
+ end
@@ -0,0 +1,9 @@
1
+ module MicroQ
2
+ module Proxy
3
+ class Instance < Base
4
+ def respond_to?(method)
5
+ super || klass.new.respond_to?(method)
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,3 @@
1
+ require 'micro_q/proxies/base'
2
+ require 'micro_q/proxies/class'
3
+ require 'micro_q/proxies/instance'
@@ -0,0 +1,90 @@
1
+ module MicroQ
2
+ module Queue
3
+ ##
4
+ # The default queue implementation.
5
+ # Handles messages that should be run immediately as well as messages that
6
+ # should be run at some specified time in the future.
7
+ #
8
+ # Usage:
9
+ #
10
+ # item = { 'class' => 'MyWorker', 'args' => [user.id] }
11
+ #
12
+ # queue = MicroQ::Queue::Default.new
13
+ # queue.push(item) # synchronous push
14
+ # queue.async.push(item) # asynchronous push (preferred)
15
+ #
16
+ # queue.entries
17
+ # #=> [{'class' => 'MyWorker', 'args' => [32]}]
18
+ #
19
+ # queue.push(item, :when => 15.minutes.from_now)
20
+ #
21
+ # queue.later
22
+ # [{'when' => 1359703628.38, 'worker' => {'class' => 'MyWorker', 'args' => 32}}]
23
+ #
24
+ class Default
25
+ include Celluloid
26
+
27
+ attr_reader :entries, :later
28
+
29
+ def initialize
30
+ @entries = []
31
+ @later = []
32
+ end
33
+
34
+ ##
35
+ # Push a message item to the queue.
36
+ # Either push it to the immediate portion of the queue or store it for after when
37
+ # it should be run with the 'when' option.
38
+ #
39
+ # Options:
40
+ # when: The time/timestamp after which to run the message.
41
+ #
42
+ def push(item, options={})
43
+ item, options = before_push(item, options)
44
+
45
+ if (time = options['when'])
46
+ @later.push(
47
+ 'when' => time.to_f,
48
+ 'worker' => item
49
+ )
50
+ else
51
+ @entries.push(item)
52
+ end
53
+ end
54
+
55
+ ##
56
+ # Remove and return all available messages.
57
+ #
58
+ def dequeue
59
+ [].tap do |items|
60
+ entries.each do |entry|
61
+ items << entry
62
+ end if entries.any?
63
+
64
+ items.each {|i| entries.delete(i) }
65
+
66
+ available = later.select {|entry| entry['when'] < Time.now.to_f }
67
+
68
+ if available.any?
69
+ available.each do |entry|
70
+ items << entry['worker']
71
+ end
72
+
73
+ available.each {|a| later.delete(a) }
74
+ end
75
+ end
76
+ end
77
+
78
+ private
79
+
80
+ ##
81
+ # Duplicate the given items and stringify the keys.
82
+ #
83
+ def before_push(args, options)
84
+ [MicroQ::Util.stringify_keys(args),
85
+ MicroQ::Util.stringify_keys(options)
86
+ ]
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1 @@
1
+ require 'micro_q/queue/default'
@@ -0,0 +1,29 @@
1
+ module MicroQ
2
+ module Util
3
+ ##
4
+ # Stolen from active_support/inflector/inflections with a rescue to nil.
5
+ #
6
+ def self.constantize(word)
7
+ names = word.split('::')
8
+ names.shift if names.empty? || names.first.empty?
9
+
10
+ constant = Object
11
+ names.each do |name| # Compatible with Ruby 1.9 and above. Before 1.9 the arity of #const_defined? was 1.
12
+ constant = constant.const_defined?(name, false) ? constant.const_get(name) : constant.const_missing(name)
13
+ end
14
+ constant
15
+ rescue
16
+ nil
17
+ end
18
+
19
+ def self.stringify_keys(hash)
20
+ {}.tap do |result|
21
+ hash.keys.each do |key|
22
+ value = hash[key]
23
+
24
+ result[key.to_s] = value.is_a?(Hash) ? stringify_keys(value) : value
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,7 @@
1
+ module MicroQ
2
+ MAJOR = 0
3
+ MINOR = 6
4
+ POINT = 1
5
+
6
+ VERSION = [MAJOR, MINOR, POINT].join('.')
7
+ end
@@ -0,0 +1,40 @@
1
+ module MicroQ
2
+ module Worker
3
+ ##
4
+ # The default worker implementation.
5
+ # this worker can call any method on an class instance and pass
6
+ # an arbitrary argument list. By default it calls the 'class'.constantize#'perform'
7
+ # method. It returns the result of the method call if possible (for debugging).
8
+ #
9
+ # The middleware chain can stop this message from executing by not yielding
10
+ # to the given block.
11
+ #
12
+ # A minimal message: (Calls the perform method with zero arguments)
13
+ # { :class => 'MyWorker' }
14
+ #
15
+ # A more complex message: (Calls the update_data with a single paramater as a list of ids)
16
+ # { :class => 'MyUpdater', 'method' => 'update_data', :args => [[2, 6,74, 198]]}
17
+ #
18
+ class Standard
19
+ include Celluloid
20
+
21
+ def perform(message)
22
+ klass = MicroQ::Util.constantize(message['class'].to_s)
23
+
24
+ loader = message['loader'] ||= { 'method' => 'new' }
25
+ klass = klass.send(loader['method'], *loader['args']) if loader['method']
26
+
27
+ method = message['method'] || 'perform'
28
+ args = message['args']
29
+
30
+ value = nil
31
+
32
+ MicroQ.middleware.server.call(klass, message) do
33
+ value = klass.send(method, *args)
34
+ end
35
+
36
+ value
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1 @@
1
+ require 'micro_q/worker/standard'
data/lib/micro_q.rb ADDED
@@ -0,0 +1,46 @@
1
+ require 'celluloid'
2
+ require 'micro_q/util'
3
+ require 'micro_q/config'
4
+ require 'micro_q/manager'
5
+
6
+ module MicroQ
7
+ def self.config
8
+ @config ||= Config.new
9
+ end
10
+
11
+ def self.configure
12
+ yield config
13
+ end
14
+
15
+ def self.middleware
16
+ config.middleware
17
+ end
18
+
19
+ def self.start
20
+ manager
21
+ end
22
+
23
+ def self.push(*args)
24
+ manager.queue.async.push(*args)
25
+ end
26
+
27
+ private
28
+
29
+ def self.manager
30
+ @manager ||= begin
31
+ Manager::Default.new.tap do |manager|
32
+ manager.start!
33
+ end
34
+ end
35
+ end
36
+
37
+ def self.clear
38
+ @config = @manager = nil
39
+ end
40
+ end
41
+
42
+ require 'micro_q/middleware'
43
+ require 'micro_q/methods'
44
+ require 'micro_q/proxies'
45
+ require 'micro_q/worker'
46
+ require 'micro_q/queue'