shoryuken 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,14 @@
1
+ module Shoryuken
2
+ module Middleware
3
+ module Server
4
+ class AutoDelete
5
+ def call(worker, queue, sqs_msg)
6
+ yield
7
+
8
+ sqs_msg.delete if worker.class.get_shoryuken_options['auto_delete']
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
14
+
@@ -0,0 +1,23 @@
1
+ module Shoryuken
2
+ module Middleware
3
+ module Server
4
+ class Logging
5
+ include Util
6
+
7
+ def call(worker, queue, sqs_msg)
8
+ Shoryuken::Logging.with_context("#{worker.class.to_s} '#{queue}' '#{sqs_msg.id}'") do
9
+ begin
10
+ started_at = Time.now
11
+ logger.info { "started at #{started_at}" }
12
+ yield
13
+ logger.info { "completed in: #{elapsed(started_at)} ms" }
14
+ rescue Exception
15
+ logger.info { "failed in: #{elapsed(started_at)} ms" }
16
+ raise
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,36 @@
1
+ require 'multi_json'
2
+
3
+ module Shoryuken
4
+ class Processor
5
+ include Celluloid
6
+ include Util
7
+
8
+ def initialize(manager)
9
+ @manager = manager
10
+ end
11
+
12
+ def process(queue, sqs_msg)
13
+ if worker_class = Shoryuken.workers[queue]
14
+ defer do
15
+ worker = worker_class.new
16
+
17
+ Shoryuken.server_middleware.invoke(worker, queue, sqs_msg) do
18
+ worker.perform(sqs_msg)
19
+ end
20
+ end
21
+ else
22
+ logger.error "Worker not found for queue '#{queue}'"
23
+ end
24
+
25
+ @manager.async.processor_done(queue, current_actor)
26
+ end
27
+
28
+ def self.default_middleware
29
+ Middleware::Chain.new do |m|
30
+ m.add Middleware::Server::Logging
31
+ m.add Middleware::Server::AutoDelete
32
+ # m.add Middleware::Server::RetryJobs
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,19 @@
1
+ module Shoryuken
2
+ module Util
3
+ def watchdog(last_words)
4
+ yield
5
+ rescue => ex
6
+ logger.error last_words
7
+ logger.error ex
8
+ logger.error ex.backtrace.join("\n")
9
+ end
10
+
11
+ def logger
12
+ Shoryuken.logger
13
+ end
14
+
15
+ def elapsed(started_at)
16
+ (Time.now - started_at) * 1000
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,3 @@
1
+ module Shoryuken
2
+ VERSION = '0.0.1'
3
+ end
@@ -0,0 +1,29 @@
1
+ module Shoryuken
2
+ module Worker
3
+ def self.included(base)
4
+ base.extend(ClassMethods)
5
+ end
6
+
7
+ module ClassMethods
8
+ def shoryuken_options(opts = {})
9
+ @shoryuken_options = get_shoryuken_options.merge(stringify_keys(Hash(opts)))
10
+ queue = @shoryuken_options['queue']
11
+ queue = queue.call if queue.respond_to? :call
12
+
13
+
14
+ Shoryuken.register_worker(queue, self)
15
+ end
16
+
17
+ def get_shoryuken_options # :nodoc:
18
+ @shoryuken_options || { 'queue' => 'default' }
19
+ end
20
+
21
+ def stringify_keys(hash) # :nodoc:
22
+ hash.keys.each do |key|
23
+ hash[key.to_s] = hash.delete(key)
24
+ end
25
+ hash
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,28 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'shoryuken/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "shoryuken"
8
+ spec.version = Shoryuken::VERSION
9
+ spec.authors = ["Pablo Cantero"]
10
+ spec.email = ["pablo@pablocantero.com"]
11
+ spec.description = spec.summary = %q{Shoryuken is a super efficient AWS SQS thread based message processor}
12
+ spec.homepage = "https://github.com/phstc/shoryuken"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = %w[shoryuken]
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_development_dependency "bundler", "~> 1.6"
21
+ spec.add_development_dependency "rake"
22
+ spec.add_development_dependency "rspec"
23
+ spec.add_development_dependency "pry-byebug"
24
+
25
+ spec.add_dependency "aws-sdk"
26
+ spec.add_dependency "celluloid"
27
+ spec.add_dependency "multi_json"
28
+ end
Binary file
@@ -0,0 +1,48 @@
1
+ require 'spec_helper'
2
+
3
+ describe Shoryuken::Middleware::Chain do
4
+ class CustomMiddleware
5
+ def initialize(name, recorder)
6
+ @name = name
7
+ @recorder = recorder
8
+ end
9
+
10
+ def call(*args)
11
+ @recorder << [@name, 'before']
12
+ yield
13
+ @recorder << [@name, 'after']
14
+ end
15
+ end
16
+
17
+ it 'supports custom middleware' do
18
+ subject.add CustomMiddleware, 1, []
19
+
20
+ expect(CustomMiddleware).to eq subject.entries.last.klass
21
+ end
22
+
23
+ it 'executes middleware' do
24
+ recorder = []
25
+ subject.add CustomMiddleware, 'Pablo', recorder
26
+
27
+ final_action = nil
28
+ subject.invoke { final_action = true }
29
+ expect(final_action).to eq true
30
+ expect(recorder).to eq [%w[Pablo before], %w[Pablo after]]
31
+ end
32
+
33
+ class NonYieldingMiddleware
34
+ def call(*args)
35
+ end
36
+ end
37
+
38
+ it 'allows middleware to abruptly stop processing rest of chain' do
39
+ recorder = []
40
+ subject.add NonYieldingMiddleware
41
+ subject.add CustomMiddleware, 'Pablo', recorder
42
+
43
+ final_action = nil
44
+ subject.invoke { final_action = true }
45
+ expect(final_action).to eq nil
46
+ expect(recorder).to eq []
47
+ end
48
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe Shoryuken::Client do
4
+ let(:sqs) { double 'SQS' }
5
+ let(:queue_collection) { double 'Queues Collection' }
6
+ let(:queue) { double 'Queue' }
7
+ let(:queue_name) { 'shoryuken' }
8
+
9
+ before do
10
+ allow(described_class).to receive(:sqs).and_return(sqs)
11
+ allow(sqs).to receive(:queues).and_return(queue_collection)
12
+ end
13
+
14
+ describe '.queues' do
15
+ it 'memoizes queues' do
16
+ expect(queue_collection).to receive(:named).once.with(queue_name).and_return(queue)
17
+
18
+ expect(Shoryuken::Client.queues(queue_name)).to eq queue
19
+ expect(Shoryuken::Client.queues(queue_name)).to eq queue
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,12 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'core_ext' do
4
+ describe String do
5
+ describe '#constantize' do
6
+ class HelloWorld; end
7
+ it 'returns a class from a string' do
8
+ expect('HelloWorld'.constantize).to eq HelloWorld
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,36 @@
1
+ require 'spec_helper'
2
+
3
+ describe Shoryuken::Fetcher do
4
+ let(:manager) { double Shoryuken::Manager }
5
+ let(:sqs_queue) { double 'sqs_queue' }
6
+ let(:queue) { 'shoryuken' }
7
+ let(:sqs_msg) { double 'SQS msg'}
8
+
9
+ subject { described_class.new(manager) }
10
+
11
+ before do
12
+ allow(manager).to receive(:async).and_return(manager)
13
+ allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
14
+ end
15
+
16
+ describe '#fetch' do
17
+ it 'calls pause_queue! when not found' do
18
+ allow(sqs_queue).to receive(:receive_message).with(limit: 1).and_return([])
19
+
20
+ expect(manager).to receive(:pause_queue!).with(queue)
21
+ expect(manager).to receive(:dispatch)
22
+
23
+ subject.fetch(queue, 1)
24
+ end
25
+
26
+ it 'assigns messages' do
27
+ allow(sqs_queue).to receive(:receive_message).with(limit: 5).and_return(sqs_msg)
28
+
29
+ expect(manager).to receive(:rebalance_queue_weight!).with(queue)
30
+ expect(manager).to receive(:assign).with(queue, sqs_msg)
31
+ expect(manager).to receive(:dispatch)
32
+
33
+ subject.fetch(queue, 5)
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,36 @@
1
+ require 'spec_helper'
2
+
3
+ describe Shoryuken::Launcher do
4
+ describe 'Consuming messages', slow: :true do
5
+ before do
6
+ subject.run
7
+
8
+ $received_messages = 0
9
+ end
10
+
11
+ after do
12
+ subject.stop
13
+ end
14
+
15
+ class ShoryukenWorker
16
+ include Shoryuken::Worker
17
+
18
+ shoryuken_options queue: 'shoryuken', auto_delete: true
19
+
20
+ def perform(sqs_msg)
21
+ $received_messages += 1
22
+ end
23
+ end
24
+
25
+ it 'consumes a message' do
26
+ Shoryuken::Client.queues('shoryuken').send_message('Yo')
27
+
28
+ 10.times do
29
+ break if $received_messages > 0
30
+ sleep 0.5
31
+ end
32
+
33
+ expect($received_messages).to eq 1
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,73 @@
1
+ require 'spec_helper'
2
+
3
+ describe Shoryuken::Manager do
4
+ describe 'Auto Scaling' do
5
+ it 'decreases weight' do
6
+ queue1 = 'shoryuken'
7
+ queue2 = 'uppercut'
8
+
9
+ Shoryuken.queues.clear
10
+ # [shoryuken, 2]
11
+ # [uppercut, 1]
12
+ Shoryuken.queues << queue1
13
+ Shoryuken.queues << queue1
14
+ Shoryuken.queues << queue2
15
+
16
+ expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
17
+
18
+ subject.pause_queue!(queue1)
19
+
20
+ expect(subject.instance_variable_get('@queues')).to eq [queue2]
21
+ end
22
+
23
+ it 'increases weight' do
24
+ queue1 = 'shoryuken'
25
+ queue2 = 'uppercut'
26
+
27
+ Shoryuken.queues.clear
28
+ # [shoryuken, 3]
29
+ # [uppercut, 1]
30
+ Shoryuken.queues << queue1
31
+ Shoryuken.queues << queue1
32
+ Shoryuken.queues << queue1
33
+ Shoryuken.queues << queue2
34
+
35
+ expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
36
+ subject.pause_queue!(queue1)
37
+ expect(subject.instance_variable_get('@queues')).to eq [queue2]
38
+
39
+ subject.rebalance_queue_weight!(queue1)
40
+ expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
41
+
42
+ subject.rebalance_queue_weight!(queue1)
43
+ expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1]
44
+
45
+ subject.rebalance_queue_weight!(queue1)
46
+ expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1, queue1]
47
+ end
48
+
49
+ it 'adds queue back' do
50
+ queue1 = 'shoryuken'
51
+ queue2 = 'uppercut'
52
+
53
+ Shoryuken.queues.clear
54
+ # [shoryuken, 2]
55
+ # [uppercut, 1]
56
+ Shoryuken.queues << queue1
57
+ Shoryuken.queues << queue1
58
+ Shoryuken.queues << queue2
59
+
60
+ Shoryuken.options[:delay] = 0.1
61
+
62
+ fetcher = double('Fetcher').as_null_object
63
+ subject.fetcher = fetcher
64
+
65
+ subject.pause_queue!(queue1)
66
+ expect(subject.instance_variable_get('@queues')).to eq [queue2]
67
+
68
+ sleep 0.5
69
+
70
+ expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,93 @@
1
+ require 'spec_helper'
2
+
3
+ describe Shoryuken::Processor do
4
+ let(:manager) { double Shoryuken::Manager }
5
+ let(:sqs_queue) { double 'Queue' }
6
+ let(:queue) { 'yo' }
7
+ let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e' }
8
+
9
+ subject { described_class.new(manager) }
10
+
11
+ before do
12
+ allow(manager).to receive(:async).and_return(manager)
13
+ allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
14
+ end
15
+
16
+ describe '#process' do
17
+ class YoWorker
18
+ include Shoryuken::Worker
19
+
20
+ shoryuken_options queue: 'yo'
21
+
22
+ def perform(sqs_msg); end
23
+ end
24
+
25
+ it 'skips when worker not found' do
26
+ queue = 'notfound'
27
+
28
+ expect(manager).to receive(:processor_done).with(queue, subject)
29
+
30
+ expect(sqs_msg).to_not receive(:delete)
31
+
32
+ subject.process(queue, sqs_msg)
33
+ end
34
+
35
+ context 'when custom middleware' do
36
+ class WorkerCalledMiddleware
37
+ def call(worker, queue, sqs_msg)
38
+ worker.called(sqs_msg, queue)
39
+ yield
40
+ end
41
+ end
42
+
43
+ before do
44
+ Shoryuken.configure_server do |config|
45
+ config.server_middleware do |chain|
46
+ chain.add WorkerCalledMiddleware
47
+ end
48
+ end
49
+ end
50
+
51
+ after do
52
+ Shoryuken.configure_server do |config|
53
+ config.server_middleware do |chain|
54
+ chain.remove WorkerCalledMiddleware
55
+ end
56
+ end
57
+ end
58
+
59
+ it 'invokes middleware' do
60
+ expect(manager).to receive(:processor_done).with(queue, subject)
61
+
62
+ expect_any_instance_of(YoWorker).to receive(:perform).with(sqs_msg)
63
+ expect_any_instance_of(YoWorker).to receive(:called).with(sqs_msg, queue)
64
+
65
+ subject.process(queue, sqs_msg)
66
+ end
67
+ end
68
+
69
+ it 'performs with auto delete' do
70
+ YoWorker.get_shoryuken_options['auto_delete'] = true
71
+
72
+ expect(manager).to receive(:processor_done).with(queue, subject)
73
+
74
+ expect_any_instance_of(YoWorker).to receive(:perform).with(sqs_msg)
75
+
76
+ expect(sqs_msg).to receive(:delete)
77
+
78
+ subject.process(queue, sqs_msg)
79
+ end
80
+
81
+ it 'performs without auto delete' do
82
+ YoWorker.get_shoryuken_options['auto_delete'] = false
83
+
84
+ expect(manager).to receive(:processor_done).with(queue, subject)
85
+
86
+ expect_any_instance_of(YoWorker).to receive(:perform).with(sqs_msg)
87
+
88
+ expect(sqs_msg).to_not receive(:delete)
89
+
90
+ subject.process(queue, sqs_msg)
91
+ end
92
+ end
93
+ end