kyu 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kyu.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Omer Jakobinsky
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,66 @@
+# Kyu
+
+Kyu - SQS background processing for Ruby.
+
+Unlike Rescue and Sidekiq, Kyu does not rely on Redis. It is simple, reliable,
+and efficient way to handle background process in Ruby using SQS.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'kyu'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kyu
+
+## Usage
+
+# image_resize_worker.rb
+```ruby
+require 'kyu'
+
+class ImageResizerWorker
+  include Kyu::Worker
+
+  max_retries 3
+  threadpool_size 10
+
+  def process_message( msg )
+    # ... Asyncronously resize the image
+  end
+end
+```
+
+`kyu start -- image_resize_worker.rb image_resizing`
+
+# image_resize_postman.rb
+```ruby
+#!/usr/bin/env ruby
+require 'kyu'
+
+class ImageResizerPostman
+  include Kyu::Postman
+
+  queue_name 'image_resizing'
+end
+
+if __FILE__ == $PROGRAM_NAME
+  ImageResizerWorker.send_message( url: ARGV[0], width: ARGV[1], height: ARGV[2] )
+end
+```
+
+`./image_resize_postman.rb URL_FOR_A_LARGE_IMG 640 1136`
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/kyu

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/kyu'
+require 'daemons'
+
+root = Dir.pwd
+marker = ARGV.index( '--' )
+raise ArgumentError, 'Queue name cannot be nil' if marker.nil?
+_, queue_name, filename = ARGV.slice!( marker..-1 )
+raise ArgumentError, 'Queue name cannot be nil' if marker.nil?
+
+Daemons.run_proc( queue_name, {} ) do
+  raise ArgumentError, 'Filename cannot be nil' if filename.nil?
+
+  load( File.join( root, filename ) )
+  infered_class = Kyu.infer_class_from_filename( filename )
+
+  infered_class.start( queue_name )
+end

data/kyu.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kyu/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kyu"
+  spec.version       = Kyu::VERSION
+  spec.authors       = ["Omer Jakobinsky"]
+  spec.email         = ["omer@jakobinsky.com"]
+  spec.description   = %q{SQS background processing for Ruby}
+  spec.summary       = %q{A simple background processing for Ruby backed by SQS}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/kyu/manager.rb

@@ -0,0 +1,67 @@
+require 'timeout'
+
+module Kyu
+  class Manager
+    def initialize( worker_klass, queue_name, options={} )
+      @max_retries = options.fetch( :max_retries, 3 )
+      @threadpool_size = options.fetch( :threadpool_size, 20 )
+      @logger = options.fetch( :logger, Logger.new( '/dev/null' ) )
+      @error_callback = options.fetch( :error_callback, ->( err ){} )
+      queue_options = options.fetch( :queue_options, {} )
+      @worker_klass = worker_klass
+
+      sqs = AWS::SQS.new
+      @queue = sqs.queues.create( queue_name, queue_options )
+      @dl_queue = sqs.queues.create( deadletter_queue_name_for( queue_name ), queue_options )
+    end
+
+    def start
+      @logger.info( "Started listening for messages on: '#{@queue.arn}'" )
+      @logger.info(
+        "Messages that could not be processes would be imgrated to: '#{@dl_queue.arn}'"
+      )
+
+      EM.run do
+        EM.threadpool_size = @threadpool_size
+        stop = false
+
+        Signal.trap( 'INT' ) { EM.stop; stop = true }
+        Signal.trap( 'TERM' ) { EM.stop; stop = true }
+
+        poll_message( @queue.visibility_timeout ) until stop
+      end
+
+      @logger.info( "Stopped listening for messages on: '#{@queue.arn}'" )
+    end
+
+    private
+
+    def poll_message( visibility_timeout )
+      msg = @queue.receive_message( attributes: [:receive_count] )
+      return unless msg
+
+      EM.defer do
+        begin
+          @logger.info( "Started processing: '#{msg.body}'" )
+          Timeout::timeout( visibility_timeout ) do
+            @worker_klass.new.process_message( JSON.parse( msg.body ) )
+          end
+          msg.delete
+          @logger.info( "Finished processing: '#{msg.body}'" )
+        rescue => err
+          @logger.error( stringify_exception( err ) )
+          @error_callback.call( err )
+          if msg.receive_count > @max_retries
+            @logger.info( "Max number of reties exceeded for: '#{msg.body}'. Migrating the message to the dead-letter queue." )
+            @dl_queue.send_message( msg.body )
+            msg.delete
+          end
+        end
+      end
+    end
+
+    def deadletter_queue_name_for( queue_name )
+      queue_name + '_deadletter'
+    end
+  end
+end

data/lib/kyu/postman.rb

@@ -0,0 +1,54 @@
+require 'aws-sdk'
+require 'logger'
+require 'json'
+
+module Kyu
+  module Postman
+    class << self
+      def send_message( queue_name, msg, options={} )
+        error_callback = options.fetch( :error_callback, ->( err ){} )
+        logger = options.fetch( :logger, Logger.new( '/dev/null' ) )
+
+        queue = fetch_queue( queue_name, logger, error_callback )
+        return unless queue
+        msg_json = msg.to_json
+        logger.info( "Sending message '#{msg_json}' to '#{queue.arn}'")
+        queue.send_message( msg.to_json )
+      end
+
+      def fetch_queue( queue_name, logger, error_callback )
+        AWS::SQS.new.queues.named( queue_name )
+      rescue AWS::SQS::Errors::NonExistentQueue => err
+        logger.error( Kyu.stringify_exception( err ) )
+        error_callback.call( err )
+        nil
+      end
+
+      def included( base )
+        base.extend( ClassMethods )
+      end
+    end
+
+    module ClassMethods
+      def queue_name( queue_name )
+        @queue_name = queue_name
+      end
+
+      def logger( logger )
+        @logger = logger
+      end
+
+      def error_callback( error_callback )
+        @error_callback = error_callback
+      end
+
+      def send_message( msg )
+        raise 'Queue cannot be nil or empty' if @queue_name.nil? || @queue_name.empty?
+        options = {}
+        options.merge!( logger: @logger ) unless @logger.nil?
+        options.merge!( error_callback: @error_callback ) unless @error_callback.nil?
+        Postman.send_message( @queue_name, msg, options )
+      end
+    end
+  end
+end

data/lib/kyu/version.rb

@@ -0,0 +1,3 @@
+module Kyu
+  VERSION = "0.0.1"
+end

data/lib/kyu/worker.rb

@@ -0,0 +1,42 @@
+module Kyu
+  module Worker
+    class << self
+      def included( base )
+        base.extend( ClassMethods )
+      end
+    end
+
+    module ClassMethods
+      def start( queue_name )
+        options = {}
+        options.merge!( max_retries: @max_retries ) unless @max_retries.nil?
+        options.merge!( threadpool_size: @threadpool_size ) unless @threadpool_size.nil?
+        options.merge!( logger: @logger ) unless @logger.nil?
+        options.merge!( error_callback: @error_callback ) unless @error_callback.nil?
+        options.merge!( queue_options: @queue_options ) unless @queue_options.nil?
+
+        Manager.new( self, queue_name, options ).start
+      end
+
+      def max_retries( max_retries )
+        @max_retries = max_retries
+      end
+
+      def threadpool_size( threadpool_size )
+        @threadpool_size = threadpool_size
+      end
+
+      def logger( logger )
+        @logger = logger
+      end
+
+      def error_callback( error_callback )
+        @error_callback = error_callback
+      end
+
+      def queue_options( queue_options )
+        @queue_options = queue_options
+      end
+    end
+  end
+end

data/lib/kyu.rb

@@ -0,0 +1,35 @@
+require 'eventmachine'
+require 'aws-sdk'
+require 'logger'
+require 'json'
+
+require_relative 'kyu/version'
+require_relative 'kyu/worker'
+require_relative 'kyu/manager'
+require_relative 'kyu/postman'
+
+# Required for Ruby < 2.0. For more information see:
+# http://ruby.awsblog.com/post/Tx16QY1CI5GVBFT/Threading-with-the-AWS-SDK-for-Ruby
+if RUBY_VERSION < '2'
+  AWS.eager_autoload!( AWS::Core )
+  AWS.eager_autoload!( AWS::SQS )
+end
+
+module Kyu
+  def stringify_exception( exception )
+    backtrace = exception.backtrace.join( ' | ' )
+    "(#{exception.class}) #{exception.message}; <trace>#{backtrace}</trace>"
+  end
+
+  def self.infer_class_from_filename( filename )
+    class_name = camel_case( File.basename( filename, '.rb' ) )
+    Kernel.const_get( class_name )
+  rescue NameError => err
+    raise err
+  end
+
+  def self.camel_case( str )
+    return str if str !~ /_/ && self =~ /[A-Z]+.*/
+    str.split( '_' ).map { |e| e.capitalize }.join
+  end
+end

data/spec/postman_spec.rb

@@ -0,0 +1,124 @@
+require_relative '../lib/kyu/postman'
+
+describe Kyu::Postman do
+  let( :msg ) { { body: 'EMPTY' } }
+  let( :queue_name ) { 'TEST_QUEUE' }
+
+  describe 'when included' do
+    describe '#send_message' do
+      describe 'allows to optionally set the logger' do
+        before do
+          class IncludedWithLogger
+            include Kyu::Postman
+            queue_name 'TEST_QUEUE'
+            logger Logger.new( '/dev/null' )
+          end
+        end
+
+        it 'delegates to ::send_message' do
+          Kyu::Postman.should_receive( :send_message ).
+            with( queue_name, msg, { logger: kind_of( Logger ) } )
+          IncludedWithLogger.send_message( msg )
+        end
+      end
+
+      describe 'allow to optionally set an error callback' do
+        before do
+          class IncludedWithErrorCallback
+            include Kyu::Postman
+            queue_name 'TEST_QUEUE'
+            error_callback ->( err ){}
+          end
+        end
+
+        it 'delegates to ::send_message' do
+          Kyu::Postman.should_receive( :send_message ).
+            with( queue_name, msg, { error_callback: kind_of( Proc ) } )
+          IncludedWithErrorCallback.send_message( msg )
+        end
+      end
+
+      describe 'when queue_name is properly set' do
+        before do
+          class IncludedWithQueueName
+            include Kyu::Postman
+            queue_name 'TEST_QUEUE'
+          end
+        end
+
+        it 'delegates to ::send_message' do
+          Kyu::Postman.should_receive( :send_message ).
+            with( queue_name, msg, {} )
+          IncludedWithQueueName.send_message( msg )
+        end
+      end
+
+      describe 'when queue_name is not properly set' do
+        before do
+          class IncludedWithoutQueueName
+            include Kyu::Postman
+          end
+        end
+
+        it 'raises exception' do
+          expect(
+            -> { IncludedWithoutQueueName.send_message( msg ) }
+          ).to raise_error
+        end
+      end
+    end
+  end
+
+  describe '::send_message' do
+    it 'sends the SQS message' do
+      options = {
+        logger: Logger.new( STDOUT ),
+        error_callback: ->( err ){}
+      }
+      Kyu::Postman.should_receive( :fetch_queue ).
+        with( queue_name, options[:logger], options[:error_callback] )
+      Kyu::Postman.send_message( queue_name, msg, options )
+    end
+  end
+
+  describe '::fetch_queue' do
+    let( :logger ) { double( Logger ) }
+    let( :error_callback ) { double( Proc ) }
+    let( :queue_double ) { double( AWS::SQS::Queue ) }
+    let( :queue_collection_double ) { double( AWS::SQS::QueueCollection ) }
+    let( :sqs_double ) { double( AWS::SQS ) }
+
+    describe 'when the SQS queue exists' do
+      before do
+        AWS::SQS.should_receive( :new ).and_return( sqs_double )
+        sqs_double.should_receive( :queues ).and_return( queue_collection_double )
+        queue_collection_double.should_receive( :named ).with( queue_name ).
+          and_return( queue_double )
+      end
+
+      it 'returns the SQS queue' do
+        queue = Kyu::Postman.fetch_queue( queue_name, logger, error_callback )
+        expect( queue ).to be( queue_double )
+      end
+    end
+
+    describe 'when the SQS does not exist' do
+      before do
+        AWS::SQS.should_receive( :new ).and_return( sqs_double )
+        sqs_double.should_receive( :queues ).and_return( queue_collection_double )
+        queue_collection_double.should_receive( :named ).with( queue_name ) {
+          raise AWS::SQS::Errors::NonExistentQueue
+        }
+      end
+
+      it 'logs the error' do
+        Kyu.should_receive( :stringify_exception )
+        logger.should_receive( :error )
+        error_callback.should_receive( :call )
+
+        queue = Kyu::Postman.fetch_queue( queue_name, logger, error_callback )
+        expect( queue ).to be( nil )
+      end
+    end
+  end
+end

data/spec/worker_spec.rb

@@ -0,0 +1,51 @@
+require_relative '../lib/kyu/manager'
+require_relative '../lib/kyu/worker'
+
+describe Kyu::Worker do
+  describe 'when included' do
+    before do
+      class TestClass
+        include Kyu::Worker
+
+        max_retries 3
+        threadpool_size 10
+        logger 'LOGGER'
+        error_callback 'CALLBACK'
+        queue_options( foo: :bar )
+      end
+    end
+
+    def should_delegate_with( option )
+      manager_double = double( Kyu::Manager, start: nil )
+
+      Kyu::Manager.should_receive( :new ) do |klass, queue_name, options|
+        expect( klass ).to be( TestClass )
+        expect( queue_name ).to eq( 'TEST_QUEUE' )
+        expect( options ).to include( option )
+        manager_double
+      end
+
+      TestClass.start( 'TEST_QUEUE' )
+    end
+
+    it 'allows to set the max_retries' do
+      should_delegate_with( max_retries: 3 )
+    end
+
+    it 'allows to set the threadpool_size' do
+      should_delegate_with( threadpool_size: 10 )
+    end
+
+    it 'allows to set the logger' do
+      should_delegate_with( logger: 'LOGGER' )
+    end
+
+    it 'allows to set the error_callback' do
+      should_delegate_with( error_callback: 'CALLBACK' )
+    end
+
+    it 'allows to set the queue_options' do
+      should_delegate_with( queue_options: { foo: :bar } )
+    end
+  end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: kyu
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Omer Jakobinsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: SQS background processing for Ruby
+email:
+- omer@jakobinsky.com
+executables:
+- kyu
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/kyu
+- kyu.gemspec
+- lib/kyu.rb
+- lib/kyu/manager.rb
+- lib/kyu/postman.rb
+- lib/kyu/version.rb
+- lib/kyu/worker.rb
+- spec/postman_spec.rb
+- spec/worker_spec.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: A simple background processing for Ruby backed by SQS
+test_files:
+- spec/postman_spec.rb
+- spec/worker_spec.rb
+has_rdoc: