kyu 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in kyu.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Omer Jakobinsky
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,66 @@
1
+ # Kyu
2
+
3
+ Kyu - SQS background processing for Ruby.
4
+
5
+ Unlike Rescue and Sidekiq, Kyu does not rely on Redis. It is simple, reliable,
6
+ and efficient way to handle background process in Ruby using SQS.
7
+
8
+ ## Installation
9
+
10
+ Add this line to your application's Gemfile:
11
+
12
+ gem 'kyu'
13
+
14
+ And then execute:
15
+
16
+ $ bundle
17
+
18
+ Or install it yourself as:
19
+
20
+ $ gem install kyu
21
+
22
+ ## Usage
23
+
24
+ # image_resize_worker.rb
25
+ ```ruby
26
+ require 'kyu'
27
+
28
+ class ImageResizerWorker
29
+ include Kyu::Worker
30
+
31
+ max_retries 3
32
+ threadpool_size 10
33
+
34
+ def process_message( msg )
35
+ # ... Asyncronously resize the image
36
+ end
37
+ end
38
+ ```
39
+
40
+ `kyu start -- image_resize_worker.rb image_resizing`
41
+
42
+ # image_resize_postman.rb
43
+ ```ruby
44
+ #!/usr/bin/env ruby
45
+ require 'kyu'
46
+
47
+ class ImageResizerPostman
48
+ include Kyu::Postman
49
+
50
+ queue_name 'image_resizing'
51
+ end
52
+
53
+ if __FILE__ == $PROGRAM_NAME
54
+ ImageResizerWorker.send_message( url: ARGV[0], width: ARGV[1], height: ARGV[2] )
55
+ end
56
+ ```
57
+
58
+ `./image_resize_postman.rb URL_FOR_A_LARGE_IMG 640 1136`
59
+
60
+ ## Contributing
61
+
62
+ 1. Fork it
63
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
64
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
65
+ 4. Push to the branch (`git push origin my-new-feature`)
66
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/bin/kyu ADDED
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/kyu'
4
+ require 'daemons'
5
+
6
+ root = Dir.pwd
7
+ marker = ARGV.index( '--' )
8
+ raise ArgumentError, 'Queue name cannot be nil' if marker.nil?
9
+ _, queue_name, filename = ARGV.slice!( marker..-1 )
10
+ raise ArgumentError, 'Queue name cannot be nil' if marker.nil?
11
+
12
+ Daemons.run_proc( queue_name, {} ) do
13
+ raise ArgumentError, 'Filename cannot be nil' if filename.nil?
14
+
15
+ load( File.join( root, filename ) )
16
+ infered_class = Kyu.infer_class_from_filename( filename )
17
+
18
+ infered_class.start( queue_name )
19
+ end
data/kyu.gemspec ADDED
@@ -0,0 +1,23 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'kyu/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "kyu"
8
+ spec.version = Kyu::VERSION
9
+ spec.authors = ["Omer Jakobinsky"]
10
+ spec.email = ["omer@jakobinsky.com"]
11
+ spec.description = %q{SQS background processing for Ruby}
12
+ spec.summary = %q{A simple background processing for Ruby backed by SQS}
13
+ spec.homepage = ""
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.3"
22
+ spec.add_development_dependency "rake"
23
+ end
@@ -0,0 +1,67 @@
1
+ require 'timeout'
2
+
3
+ module Kyu
4
+ class Manager
5
+ def initialize( worker_klass, queue_name, options={} )
6
+ @max_retries = options.fetch( :max_retries, 3 )
7
+ @threadpool_size = options.fetch( :threadpool_size, 20 )
8
+ @logger = options.fetch( :logger, Logger.new( '/dev/null' ) )
9
+ @error_callback = options.fetch( :error_callback, ->( err ){} )
10
+ queue_options = options.fetch( :queue_options, {} )
11
+ @worker_klass = worker_klass
12
+
13
+ sqs = AWS::SQS.new
14
+ @queue = sqs.queues.create( queue_name, queue_options )
15
+ @dl_queue = sqs.queues.create( deadletter_queue_name_for( queue_name ), queue_options )
16
+ end
17
+
18
+ def start
19
+ @logger.info( "Started listening for messages on: '#{@queue.arn}'" )
20
+ @logger.info(
21
+ "Messages that could not be processes would be imgrated to: '#{@dl_queue.arn}'"
22
+ )
23
+
24
+ EM.run do
25
+ EM.threadpool_size = @threadpool_size
26
+ stop = false
27
+
28
+ Signal.trap( 'INT' ) { EM.stop; stop = true }
29
+ Signal.trap( 'TERM' ) { EM.stop; stop = true }
30
+
31
+ poll_message( @queue.visibility_timeout ) until stop
32
+ end
33
+
34
+ @logger.info( "Stopped listening for messages on: '#{@queue.arn}'" )
35
+ end
36
+
37
+ private
38
+
39
+ def poll_message( visibility_timeout )
40
+ msg = @queue.receive_message( attributes: [:receive_count] )
41
+ return unless msg
42
+
43
+ EM.defer do
44
+ begin
45
+ @logger.info( "Started processing: '#{msg.body}'" )
46
+ Timeout::timeout( visibility_timeout ) do
47
+ @worker_klass.new.process_message( JSON.parse( msg.body ) )
48
+ end
49
+ msg.delete
50
+ @logger.info( "Finished processing: '#{msg.body}'" )
51
+ rescue => err
52
+ @logger.error( stringify_exception( err ) )
53
+ @error_callback.call( err )
54
+ if msg.receive_count > @max_retries
55
+ @logger.info( "Max number of reties exceeded for: '#{msg.body}'. Migrating the message to the dead-letter queue." )
56
+ @dl_queue.send_message( msg.body )
57
+ msg.delete
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ def deadletter_queue_name_for( queue_name )
64
+ queue_name + '_deadletter'
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,54 @@
1
+ require 'aws-sdk'
2
+ require 'logger'
3
+ require 'json'
4
+
5
+ module Kyu
6
+ module Postman
7
+ class << self
8
+ def send_message( queue_name, msg, options={} )
9
+ error_callback = options.fetch( :error_callback, ->( err ){} )
10
+ logger = options.fetch( :logger, Logger.new( '/dev/null' ) )
11
+
12
+ queue = fetch_queue( queue_name, logger, error_callback )
13
+ return unless queue
14
+ msg_json = msg.to_json
15
+ logger.info( "Sending message '#{msg_json}' to '#{queue.arn}'")
16
+ queue.send_message( msg.to_json )
17
+ end
18
+
19
+ def fetch_queue( queue_name, logger, error_callback )
20
+ AWS::SQS.new.queues.named( queue_name )
21
+ rescue AWS::SQS::Errors::NonExistentQueue => err
22
+ logger.error( Kyu.stringify_exception( err ) )
23
+ error_callback.call( err )
24
+ nil
25
+ end
26
+
27
+ def included( base )
28
+ base.extend( ClassMethods )
29
+ end
30
+ end
31
+
32
+ module ClassMethods
33
+ def queue_name( queue_name )
34
+ @queue_name = queue_name
35
+ end
36
+
37
+ def logger( logger )
38
+ @logger = logger
39
+ end
40
+
41
+ def error_callback( error_callback )
42
+ @error_callback = error_callback
43
+ end
44
+
45
+ def send_message( msg )
46
+ raise 'Queue cannot be nil or empty' if @queue_name.nil? || @queue_name.empty?
47
+ options = {}
48
+ options.merge!( logger: @logger ) unless @logger.nil?
49
+ options.merge!( error_callback: @error_callback ) unless @error_callback.nil?
50
+ Postman.send_message( @queue_name, msg, options )
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,3 @@
1
+ module Kyu
2
+ VERSION = "0.0.1"
3
+ end
data/lib/kyu/worker.rb ADDED
@@ -0,0 +1,42 @@
1
+ module Kyu
2
+ module Worker
3
+ class << self
4
+ def included( base )
5
+ base.extend( ClassMethods )
6
+ end
7
+ end
8
+
9
+ module ClassMethods
10
+ def start( queue_name )
11
+ options = {}
12
+ options.merge!( max_retries: @max_retries ) unless @max_retries.nil?
13
+ options.merge!( threadpool_size: @threadpool_size ) unless @threadpool_size.nil?
14
+ options.merge!( logger: @logger ) unless @logger.nil?
15
+ options.merge!( error_callback: @error_callback ) unless @error_callback.nil?
16
+ options.merge!( queue_options: @queue_options ) unless @queue_options.nil?
17
+
18
+ Manager.new( self, queue_name, options ).start
19
+ end
20
+
21
+ def max_retries( max_retries )
22
+ @max_retries = max_retries
23
+ end
24
+
25
+ def threadpool_size( threadpool_size )
26
+ @threadpool_size = threadpool_size
27
+ end
28
+
29
+ def logger( logger )
30
+ @logger = logger
31
+ end
32
+
33
+ def error_callback( error_callback )
34
+ @error_callback = error_callback
35
+ end
36
+
37
+ def queue_options( queue_options )
38
+ @queue_options = queue_options
39
+ end
40
+ end
41
+ end
42
+ end
data/lib/kyu.rb ADDED
@@ -0,0 +1,35 @@
1
+ require 'eventmachine'
2
+ require 'aws-sdk'
3
+ require 'logger'
4
+ require 'json'
5
+
6
+ require_relative 'kyu/version'
7
+ require_relative 'kyu/worker'
8
+ require_relative 'kyu/manager'
9
+ require_relative 'kyu/postman'
10
+
11
+ # Required for Ruby < 2.0. For more information see:
12
+ # http://ruby.awsblog.com/post/Tx16QY1CI5GVBFT/Threading-with-the-AWS-SDK-for-Ruby
13
+ if RUBY_VERSION < '2'
14
+ AWS.eager_autoload!( AWS::Core )
15
+ AWS.eager_autoload!( AWS::SQS )
16
+ end
17
+
18
+ module Kyu
19
+ def stringify_exception( exception )
20
+ backtrace = exception.backtrace.join( ' | ' )
21
+ "(#{exception.class}) #{exception.message}; <trace>#{backtrace}</trace>"
22
+ end
23
+
24
+ def self.infer_class_from_filename( filename )
25
+ class_name = camel_case( File.basename( filename, '.rb' ) )
26
+ Kernel.const_get( class_name )
27
+ rescue NameError => err
28
+ raise err
29
+ end
30
+
31
+ def self.camel_case( str )
32
+ return str if str !~ /_/ && self =~ /[A-Z]+.*/
33
+ str.split( '_' ).map { |e| e.capitalize }.join
34
+ end
35
+ end
@@ -0,0 +1,124 @@
1
+ require_relative '../lib/kyu/postman'
2
+
3
+ describe Kyu::Postman do
4
+ let( :msg ) { { body: 'EMPTY' } }
5
+ let( :queue_name ) { 'TEST_QUEUE' }
6
+
7
+ describe 'when included' do
8
+ describe '#send_message' do
9
+ describe 'allows to optionally set the logger' do
10
+ before do
11
+ class IncludedWithLogger
12
+ include Kyu::Postman
13
+ queue_name 'TEST_QUEUE'
14
+ logger Logger.new( '/dev/null' )
15
+ end
16
+ end
17
+
18
+ it 'delegates to ::send_message' do
19
+ Kyu::Postman.should_receive( :send_message ).
20
+ with( queue_name, msg, { logger: kind_of( Logger ) } )
21
+ IncludedWithLogger.send_message( msg )
22
+ end
23
+ end
24
+
25
+ describe 'allow to optionally set an error callback' do
26
+ before do
27
+ class IncludedWithErrorCallback
28
+ include Kyu::Postman
29
+ queue_name 'TEST_QUEUE'
30
+ error_callback ->( err ){}
31
+ end
32
+ end
33
+
34
+ it 'delegates to ::send_message' do
35
+ Kyu::Postman.should_receive( :send_message ).
36
+ with( queue_name, msg, { error_callback: kind_of( Proc ) } )
37
+ IncludedWithErrorCallback.send_message( msg )
38
+ end
39
+ end
40
+
41
+ describe 'when queue_name is properly set' do
42
+ before do
43
+ class IncludedWithQueueName
44
+ include Kyu::Postman
45
+ queue_name 'TEST_QUEUE'
46
+ end
47
+ end
48
+
49
+ it 'delegates to ::send_message' do
50
+ Kyu::Postman.should_receive( :send_message ).
51
+ with( queue_name, msg, {} )
52
+ IncludedWithQueueName.send_message( msg )
53
+ end
54
+ end
55
+
56
+ describe 'when queue_name is not properly set' do
57
+ before do
58
+ class IncludedWithoutQueueName
59
+ include Kyu::Postman
60
+ end
61
+ end
62
+
63
+ it 'raises exception' do
64
+ expect(
65
+ -> { IncludedWithoutQueueName.send_message( msg ) }
66
+ ).to raise_error
67
+ end
68
+ end
69
+ end
70
+ end
71
+
72
+ describe '::send_message' do
73
+ it 'sends the SQS message' do
74
+ options = {
75
+ logger: Logger.new( STDOUT ),
76
+ error_callback: ->( err ){}
77
+ }
78
+ Kyu::Postman.should_receive( :fetch_queue ).
79
+ with( queue_name, options[:logger], options[:error_callback] )
80
+ Kyu::Postman.send_message( queue_name, msg, options )
81
+ end
82
+ end
83
+
84
+ describe '::fetch_queue' do
85
+ let( :logger ) { double( Logger ) }
86
+ let( :error_callback ) { double( Proc ) }
87
+ let( :queue_double ) { double( AWS::SQS::Queue ) }
88
+ let( :queue_collection_double ) { double( AWS::SQS::QueueCollection ) }
89
+ let( :sqs_double ) { double( AWS::SQS ) }
90
+
91
+ describe 'when the SQS queue exists' do
92
+ before do
93
+ AWS::SQS.should_receive( :new ).and_return( sqs_double )
94
+ sqs_double.should_receive( :queues ).and_return( queue_collection_double )
95
+ queue_collection_double.should_receive( :named ).with( queue_name ).
96
+ and_return( queue_double )
97
+ end
98
+
99
+ it 'returns the SQS queue' do
100
+ queue = Kyu::Postman.fetch_queue( queue_name, logger, error_callback )
101
+ expect( queue ).to be( queue_double )
102
+ end
103
+ end
104
+
105
+ describe 'when the SQS does not exist' do
106
+ before do
107
+ AWS::SQS.should_receive( :new ).and_return( sqs_double )
108
+ sqs_double.should_receive( :queues ).and_return( queue_collection_double )
109
+ queue_collection_double.should_receive( :named ).with( queue_name ) {
110
+ raise AWS::SQS::Errors::NonExistentQueue
111
+ }
112
+ end
113
+
114
+ it 'logs the error' do
115
+ Kyu.should_receive( :stringify_exception )
116
+ logger.should_receive( :error )
117
+ error_callback.should_receive( :call )
118
+
119
+ queue = Kyu::Postman.fetch_queue( queue_name, logger, error_callback )
120
+ expect( queue ).to be( nil )
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,51 @@
1
+ require_relative '../lib/kyu/manager'
2
+ require_relative '../lib/kyu/worker'
3
+
4
+ describe Kyu::Worker do
5
+ describe 'when included' do
6
+ before do
7
+ class TestClass
8
+ include Kyu::Worker
9
+
10
+ max_retries 3
11
+ threadpool_size 10
12
+ logger 'LOGGER'
13
+ error_callback 'CALLBACK'
14
+ queue_options( foo: :bar )
15
+ end
16
+ end
17
+
18
+ def should_delegate_with( option )
19
+ manager_double = double( Kyu::Manager, start: nil )
20
+
21
+ Kyu::Manager.should_receive( :new ) do |klass, queue_name, options|
22
+ expect( klass ).to be( TestClass )
23
+ expect( queue_name ).to eq( 'TEST_QUEUE' )
24
+ expect( options ).to include( option )
25
+ manager_double
26
+ end
27
+
28
+ TestClass.start( 'TEST_QUEUE' )
29
+ end
30
+
31
+ it 'allows to set the max_retries' do
32
+ should_delegate_with( max_retries: 3 )
33
+ end
34
+
35
+ it 'allows to set the threadpool_size' do
36
+ should_delegate_with( threadpool_size: 10 )
37
+ end
38
+
39
+ it 'allows to set the logger' do
40
+ should_delegate_with( logger: 'LOGGER' )
41
+ end
42
+
43
+ it 'allows to set the error_callback' do
44
+ should_delegate_with( error_callback: 'CALLBACK' )
45
+ end
46
+
47
+ it 'allows to set the queue_options' do
48
+ should_delegate_with( queue_options: { foo: :bar } )
49
+ end
50
+ end
51
+ end
metadata ADDED
@@ -0,0 +1,96 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: kyu
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Omer Jakobinsky
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-12-01 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: '1.3'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '1.3'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ description: SQS background processing for Ruby
47
+ email:
48
+ - omer@jakobinsky.com
49
+ executables:
50
+ - kyu
51
+ extensions: []
52
+ extra_rdoc_files: []
53
+ files:
54
+ - .gitignore
55
+ - Gemfile
56
+ - LICENSE.txt
57
+ - README.md
58
+ - Rakefile
59
+ - bin/kyu
60
+ - kyu.gemspec
61
+ - lib/kyu.rb
62
+ - lib/kyu/manager.rb
63
+ - lib/kyu/postman.rb
64
+ - lib/kyu/version.rb
65
+ - lib/kyu/worker.rb
66
+ - spec/postman_spec.rb
67
+ - spec/worker_spec.rb
68
+ homepage: ''
69
+ licenses:
70
+ - MIT
71
+ post_install_message:
72
+ rdoc_options: []
73
+ require_paths:
74
+ - lib
75
+ required_ruby_version: !ruby/object:Gem::Requirement
76
+ none: false
77
+ requirements:
78
+ - - ! '>='
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ required_rubygems_version: !ruby/object:Gem::Requirement
82
+ none: false
83
+ requirements:
84
+ - - ! '>='
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ requirements: []
88
+ rubyforge_project:
89
+ rubygems_version: 1.8.23
90
+ signing_key:
91
+ specification_version: 3
92
+ summary: A simple background processing for Ruby backed by SQS
93
+ test_files:
94
+ - spec/postman_spec.rb
95
+ - spec/worker_spec.rb
96
+ has_rdoc: