FanSQS 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cb1bcbb591271928ff993e7fe68b63c3b7f8e577
+  data.tar.gz: cd7e8775fe377c7c9158be44ea7bbd61676f61c9
+SHA512:
+  metadata.gz: 396454510c18f3653fe53ebeb352cdd3574eeed301bd5ad0845f9147e9caf0ac394a320c3202001eb023ed953a899996d9830f42c8019c8999710b98f108df16
+  data.tar.gz: 8c7e247c2a1ea5912f04dd83f004f698c8c1e73123f1c6999037214562d8898d70b9aeef5066e26c360571e57eac600baaca23bafad7eb73c1fcb4ff89bddb2e

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp
+*.swo
+Session.vim
+dump.rdb

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/FanSQS.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'FanSQS/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "FanSQS"
+  spec.version       = FanSQS::VERSION
+  spec.authors       = ["Khang Pham"]
+  spec.email         = ["vkhang55@gmail.com"]
+  spec.summary       = %q{Distributed messages queuing system using AWS SQS}
+  spec.description   = %q{Distributed messages queuing system using AWS SQS}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rake"
+  spec.add_dependency "aws-sdk", "~> 1.41.0"
+
+  # For testing
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency 'rspec-mocks'
+  spec.add_development_dependency 'shoulda-matchers'
+end

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in FanSQS.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,13 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+  watch(%r{^spec/support/(.+)\.rb$})  { "spec" }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Khang Pham
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/NOTES.txt

@@ -0,0 +1,6 @@
+TO-DOs
+======
+1. Add rake task to start poller
+2. Priority queues
+3. Auto retries on error
+4. Hook up Travis

data/README.md

@@ -0,0 +1,62 @@
+# FanSQS
+
+FanSQS is a background job processing engine for Ruby using [AWS SQS](http://aws.amazon.com/sqs/) for message storage.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'FanSQS'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install FanSQS
+    
+## Rails Setup
+
+Create a **config/initializer.rb** file in your application directory and insert this line:
+
+```ruby
+AWS.config( access_key_id: '<your_access_key_id>', secret_access_key: '<your_secret_access_key>')
+```
+
+## Usage
+
+FanSQS integrates seemlessly with your Rails applications. Just *include* the module **FanSQS::Worker** in the class that encapsulates your job logic and define a class method called *perform*. The method *perform* should be able to accept any number of parameters of any type.
+
+```ruby
+class MessagePublisher
+  include FanSQS::Worker       
+  set_fan_sqs_options queue: :message_queue # use the key :queue to define the message queue name
+
+  def self.perform(arg1, arg2, ...)
+    # code to do work
+  end
+end
+```
+
+To push jobs into an SQS queue for processing, simply call:
+
+```ruby
+MessagePublisher.perform_async(arg1, arg2, ...)
+```
+
+## Features to add
+
+1. Rake task to start poller
+2. Priority queues
+3. Auto retries on error
+4. Hook up to Travis
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/FanSQS/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Test coverage for your changes. Sorry I will not merge changes without test coverage!
+5. Push to the branch (`git push origin my-new-feature`)
+6. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/FanSQS.rb

@@ -0,0 +1,26 @@
+require "FanSQS/pool"
+require "FanSQS/version"
+require "FanSQS/message_parser"
+require "FanSQS/queue_wrapper"
+require "FanSQS/queues_cache"
+require "FanSQS/poller"
+require "FanSQS/worker"
+require "aws"
+
+# Reference: [http://ruby.awsblog.com/post/Tx16QY1CI5GVBFT/Threading-with-the-AWS-SDK-for-Ruby]
+# Issue this line will give significant performance boost
+AWS.eager_autoload! AWS::SQS
+
+# $pool = Pool.new(50) # Performs max at 50 threads. Need to make this value configurable later.
+
+module FanSQS
+  extend self
+
+  ErrorQueue = :fan_sqs_queue_error
+  class << self
+    attr_accessor :pool
+  end
+  self.pool = Pool.new(100)
+end
+
+require 'FanSQS/railtie' if defined?(Rails) # include FanSQS Rake task if Rails is defined

data/lib/FanSQS/message_parser.rb

@@ -0,0 +1,12 @@
+module FanSQS
+  class MessageParser
+    class << self
+      def parse(msg)
+        json = JSON.parse(msg, symbolize_names: true)
+        return json
+      rescue JSON::ParserError # malformed JSON
+        return nil
+      end
+    end
+  end
+end

data/lib/FanSQS/poller.rb

@@ -0,0 +1,42 @@
+require 'thread'
+
+module FanSQS
+  class Poller
+    class << self
+      def start(qnames = [])
+        @queues_cache = FanSQS::QueuesCache.new(qnames)
+        loop do
+          @queues_cache.fetch.each do |queue|
+            FanSQS.pool.schedule do # Allows for multiple concurrent (non-blocking) HTTP requests to SQS
+              queue.receive_messages(limit: 10) do |message|
+                process(message.body)
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      def process(msg)
+        message = MessageParser.parse(msg)
+        Thread.new do
+          begin
+            klass = Object::const_get(message[:class])
+            klass.send(:perform, *message[:arguments])
+          rescue ArgumentError => e
+            store_exception(e, message)
+          end
+        end
+      end
+
+      def store_exception(exception, message)
+        error_message = { class: message[:class],
+                          arguments: message[:arguments],
+                          stack_trace: exception.backtrace.join("\n") }
+        queue = FanSQS::QueueWrapper.instantiate(FanSQS::ErrorQueue)
+        queue.send_message(error_message.to_json)
+      end
+    end
+  end
+end

data/lib/FanSQS/pool.rb

@@ -0,0 +1,155 @@
+# Credit to Kim Burgestrand @ http://burgestrand.se/articles/quick-and-simple-ruby-thread-pool.html
+
+# Ruby Thread Pool
+# ================
+# A thread pool is useful when you wish to do some work in a thread, but do
+# not know how much work you will be doing in advance. Spawning one thread
+# for each task is potentially expensive, as threads are not free.
+# 
+# In this case, it might be more beneficial to start a predefined set of
+# threads and then hand off work to them as it becomes available. This is
+# the pure essence of what a thread pool is: an array of threads, all just
+# waiting to do some work for you!
+#
+# Prerequisites
+# -------------
+
+# We need the [Queue](http://rdoc.info/stdlib/thread/1.9.2/Queue), as our
+# thread pool is largely dependent on it. Thanks to this, the implementation
+# becomes very simple!
+require 'thread'
+
+# Public Interface
+# ----------------
+
+# `Pool` is our thread pool class. It will allow us to do three operations:
+# 
+# - `.new(size)` creates a thread pool of a given size
+# - `#schedule(*args, &job)` schedules a new job to be executed
+# - `#shutdown` shuts down all threads (after letting them finish working, of course)
+module FanSQS
+  class Pool
+
+    # ### initialization, or `Pool.new(size)`
+    # Creating a new `Pool` involves a certain amount of work. First, however,
+    # we need to define its’ `size`. It defines how many threads we will have
+    # working internally.
+    # 
+    # Which size is best for you is hard to answer. You do not want it to be
+    # too low, as then you won’t be able to do as many things concurrently.
+    # However, if you make it too high Ruby will spend too much time switching
+    # between threads, and that will also degrade performance!
+    def initialize(size)
+      # Before we do anything else, we need to store some information about
+      # our pool. `@size` is useful later, when we want to shut our pool down,
+      # and `@jobs` is the heart of our pool that allows us to schedule work.
+      @size = size
+      @jobs = Queue.new
+
+      # #### Creating our pool of threads
+      # Once preparation is done, it’s time to create our pool of threads.
+      # Each thread store its’ index in a thread-local variable, in case we
+      # need to know which thread a job is executing in later on.
+      @pool = Array.new(@size) do |i|
+        Thread.new do
+          Thread.current[:id] = i
+
+          # We start off by defining a `catch` around our worker loop. This
+          # way we’ve provided a method for graceful shutdown of our threads.
+          # Shutting down is merely a `#schedule { throw :exit }` away!
+          catch(:exit) do
+            # The worker thread life-cycle is very simple. We continuously wait
+            # for tasks to be put into our job `Queue`. If the `Queue` is empty,
+            # we will wait until it’s not.
+            loop do
+              # Once we have a piece of work to be done, we will pull out the
+              # information we need and get to work.
+              job, args = @jobs.pop
+              job.call(*args)
+            end
+          end
+        end
+      end
+    end
+
+    # ### Work scheduling
+
+    # To schedule a piece of work to be done is to say to the `Pool` that you
+    # want something done.
+    def schedule(*args, &block)
+      # Your given task will not be run immediately; rather, it will be put
+      # into the work `Queue` and executed once a thread is ready to work.
+      @jobs << [block, args]
+    end
+
+    # ### Graceful shutdown
+
+    # If you ever wish to close down your application, I took the liberty of
+    # making it easy for you to wait for any currently executing jobs to finish
+    # before you exit.
+    def shutdown
+      # A graceful shutdown involves threads exiting cleanly themselves, and
+      # since we’ve defined a `catch`-handler around the threads’ worker loop
+      # it is simply a matter of throwing `:exit`. Thus, if we throw one `:exit`
+      # for each thread in our pool, they will all exit eventually!
+      @size.times do
+        schedule { throw :exit }
+      end
+
+      # And now one final thing: wait for our `throw :exit` jobs to be run on
+      # all our worker threads. This call will not return until all worker threads
+      # have exited.
+      @pool.map(&:join)
+    end
+  end
+end
+
+# Demonstration
+# -------------
+# Running this file will display how the thread pool works.
+# if $0 == __FILE__
+#   # - First, we create a new thread pool with a size of 10. This number is
+#   #   lower than our planned amount of work, to show that threads do not
+#   #   exit once they have finished a task.
+#   p = Pool.new(10)
+# 
+#   # - Next we simulate some workload by scheduling a large amount of work
+#   #   to be done. The actual time taken for each job is randomized. This
+#   #   is to demonstrate that even if two tasks are scheduled approximately
+#   #   at the same time, the one that takes less time to execute is likely
+#   #   to finish before the other one.
+#   20.times do |i|
+#     p.schedule do
+#       sleep rand(4) + 2
+#       puts "Job #{i} finished by thread #{Thread.current[:id]}"
+#     end
+#   end
+# 
+#   # - Finally, register an `at_exit`-hook that will wait for our thread pool
+#   #   to properly shut down before allowing our script to completely exit.
+#   at_exit { p.shutdown }
+# end
+
+# License (X11 License)
+# =====================
+#
+# Copyright (c) 2012, Kim Burgestrand <kim@burgestrand.se>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+

data/lib/FanSQS/queue_wrapper.rb

@@ -0,0 +1,37 @@
+module FanSQS
+  class QueueWrapper
+    @cache ||= {} # cache for queues
+
+    class << self
+      # Find out if the queue exists. If it does not exist, create a new one
+      # This line below has a high chance of causing confusion as it will try to use either FanSQS::Queue or AWS::SQS::Queue. Maybe use the class from AWS gem instead?
+      def instantiate(qname)
+        name = formatted_queue_name(qname)
+        exists?(name) || create_new(name)
+      end
+
+      def create_new(name)
+        AWS.sqs.queues.create(name.to_s)
+      end
+
+      def cache
+        @cache
+      end
+
+      def exists?(qname)
+        if @cache[qname]
+          return @cache[qname]
+        else
+          return @cache[qname] = AWS.sqs.queues.named(formatted_queue_name(qname)) while @cache[qname] == nil
+        end
+      rescue AWS::SQS::Errors::NonExistentQueue
+        return false
+      end
+
+      private
+      def formatted_queue_name(qname)
+        qname.to_s
+      end
+    end
+  end
+end

data/lib/FanSQS/queues_cache.rb

@@ -0,0 +1,59 @@
+# QueuesCache's purpose is for speed optimization. Using this class, we do not have to constantly ping
+# AWS for new queues. It will ping AWS only every X times.
+module FanSQS
+  class QueuesCache
+    RESET_THRESHOLD = 20000
+
+    def initialize(qnames)
+      @sqs_client ||= AWS::SQS::Client.new
+      @qnames = resolve_qnames(qnames)
+      @counter = 1
+      @queues = []
+    end
+
+    def fetch
+      if @qnames.empty?
+        fetch_all_queues
+      else
+        fetch_specific_queues
+      end
+    end
+
+    private
+    # Fetch all queues except for the queue equals to FanSQS::ErrorQueue
+    def fetch_all_queues
+      if @counter % RESET_THRESHOLD == 0
+        @counter = 1 # reset counter
+        @queue_names = @sqs_client.list_queues[:queue_urls].map { |q| q.split('/').last }.uniq
+        @queue_names.reject! { |name| name == FanSQS::ErrorQueue.to_s } # do not include the error queue
+        @queues = @queue_names.inject([]) { |queues, name| queues << QueueWrapper.instantiate(name) } # reset cache & fetch queues from AWS
+      else
+        @counter += 1
+        @queues
+      end
+    end
+
+    # Fetch only specific queues. Can also fetch queues with prefix = "xxx*"
+    def fetch_specific_queues
+      return @queues unless @queues.empty?  
+      @qnames.each do |qname|
+        if qname =~ /\*/
+          queues_with_prefix = AWS::SQS.new.queues.with_prefix(qname.split('*').first)
+          queues_with_prefix.each do |queue|
+            @queues << queue
+          end
+        else
+          @queues << AWS::SQS.new.queues.named(qname)
+        end
+      end
+    end
+
+    def resolve_qnames(qnames)
+      if qnames.is_a?(Array)
+        qnames.map(&:to_s)
+      else
+        qnames.to_s.split(',').uniq
+      end
+    end
+  end
+end

data/lib/FanSQS/railtie.rb

@@ -0,0 +1,7 @@
+module FanSQS
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path(File.dirname(__FILE__) + "/tasks/FanSQS.rake")
+    end
+  end
+end

data/lib/FanSQS/tasks/FanSQS.rake

@@ -0,0 +1,7 @@
+namespace :FanSQS do
+  desc "FanSQS constantly polls for new job from all queues in SQS and execute jobs"
+  task :start_polling => :environment do
+    puts "FanSQS polling starts ..."
+    FanSQS::Poller.start
+  end
+end

data/lib/FanSQS/version.rb

@@ -0,0 +1,3 @@
+module FanSQS
+  VERSION = "0.0.1"
+end

data/lib/FanSQS/worker.rb

@@ -0,0 +1,30 @@
+require 'thread'
+
+module FanSQS
+  module Worker
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.class_attribute :fan_sqs_options_hash
+    end
+
+    module ClassMethods
+      def perform_async(*args)
+        FanSQS.pool.schedule do # Allows for multiple concurrent (non-blocking) HTTP requests to SQS
+          queue, sent_message = nil
+          qname = fan_sqs_options_hash ? fan_sqs_options_hash[:queue] : :fan_sqs_queue
+          queue = FanSQS::QueueWrapper.instantiate(qname)
+          params = { class: self.name, arguments: args }
+          sent_message = queue.send_message(params.to_json) while sent_message.try(&:md5) == nil # retry until receives sent_message confirmation
+        end
+      end
+
+      def set_fan_sqs_options(options = {})
+        if options.empty?
+          self.fan_sqs_options_hash = { queue: :fan_sqs_queue }
+        else
+          self.fan_sqs_options_hash = options
+        end
+      end
+    end
+  end
+end

data/spec/FanSQS/message_parser_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe FanSQS::MessageParser do
+  describe "#parse" do
+    it "should return a JSON object with expected result" do
+      msg = { a: 1, b: 2}
+      json = FanSQS::MessageParser.parse(msg.to_json)
+      expect(json).to eq(msg)
+    end
+  end
+end

data/spec/FanSQS/poller_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe FanSQS::Poller do
+  describe "#start" do
+    it "is a pending example"
+  end
+
+  describe "#process" do
+    it "is a pending example"
+  end
+
+  describe "#store_exception" do
+    it "is a pending example"
+  end
+end

data/spec/FanSQS/pool_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe FanSQS::Pool do
+  describe "#initialize" do
+    it "is a pending example"
+  end
+
+  describe "#schedule" do
+    it "is a pending example"
+  end
+
+  describe "#shutdown" do
+    it "is a pending example"
+  end
+end

data/spec/FanSQS/queue_wrapper_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe FanSQS::QueueWrapper do
+  before do
+    stub_retrieving_named_queues
+  end
+
+  describe "#exists?" do
+    context "cached" do
+      it "should returns a named queue from @cache" do
+        sample_queue = mocked_queue
+        FanSQS::QueueWrapper.instance_variable_set(:@cache, {:sample_qname => sample_queue})
+        expect(FanSQS::QueueWrapper.exists?(:sample_qname)).to eq(sample_queue)
+      end
+    end
+
+    context "not cached" do
+      before do
+        FanSQS::QueueWrapper.instance_variable_set(:@cache, {}) #empty cache
+      end
+
+      it "should returns fetches the queue from AWS" do
+        expect_any_instance_of(AWS::SQS::QueueCollection).to receive(:named)
+        FanSQS::QueueWrapper.exists?(:sample_qname)
+      end
+
+      context "non-existent queue" do
+        it "should raise exception AWS::SQS::Errors::NonExistentQueue and returns false" do
+          stub_retrieving_named_queues_raise_exception
+          expect(FanSQS::QueueWrapper.exists?(:sample_qname)).to eq(false)
+        end
+      end
+    end
+  end
+
+  describe "#create_new" do
+    it "should call #create method from QueueCollection" do
+      expect_any_instance_of(AWS::SQS::QueueCollection).to receive(:create).once
+      FanSQS::QueueWrapper.create_new(:sample_queue)
+    end
+  end
+
+  describe "#instantiate" do
+    context "cached" do
+      it "should call exists?, receive a mocked queue, and not call create_new" do
+        sample_queue = mocked_queue
+        FanSQS::QueueWrapper.instance_variable_set(:@cache, {:sample_qname => sample_queue})
+        expect(FanSQS::QueueWrapper).to_not receive(:create_new)
+        FanSQS::QueueWrapper.instantiate(:sample_qname)
+      end
+
+      it "should call exists?, catch an exception, and call create_new" do
+        stub_retrieving_named_queues_raise_exception
+        FanSQS::QueueWrapper.instance_variable_set(:@cache, {})
+        expect(FanSQS::QueueWrapper).to_not receive(:create_new).once
+        FanSQS::QueueWrapper.instantiate(:sample_qname)
+      end
+    end
+  end
+
+  it "formats symbol to string qname" do
+    expect(FanSQS::QueueWrapper.send(:formatted_queue_name, :symbol)).to be_an_instance_of(String)
+  end
+end

data/spec/FanSQS/queues_cache_spec.rb

@@ -0,0 +1,104 @@
+require 'spec_helper'
+
+describe FanSQS::QueuesCache do
+  describe "#initialize" do
+    it "should call these methods and end up with these values" do
+      qnames = [ :queue_1, :queue_2 ]
+      @queues_cache = FanSQS::QueuesCache.new(qnames)
+      expect(@queues_cache.instance_variable_get(:@qnames)).to eq(qnames.map(&:to_s))
+      expect(@queues_cache.instance_variable_get(:@counter)).to eq(1)
+      expect(@queues_cache.instance_variable_get(:@queues)).to eq([])
+    end
+  end
+
+  describe "#fetch" do
+    context "@qnames.empty" do
+      it "should call fetch_all_queues" do
+        qnames = []
+        @queues_cache = FanSQS::QueuesCache.new(qnames)
+        expect(@queues_cache).to receive(:fetch_all_queues).once
+        @queues_cache.fetch
+      end
+    end
+
+    context "@qnames is not empty" do
+      it "should call fetch_specific_queues" do
+        qnames = [:queue_1, :queue_2]
+        @queues_cache = FanSQS::QueuesCache.new(qnames)
+        expect(@queues_cache).to receive(:fetch_specific_queues).once
+        @queues_cache.fetch
+      end
+    end
+  end
+
+  describe "#fetch_all_queues" do
+    context "counter % RESET_THRESHOLD != 0" do
+      before do
+        @queues_cache = FanSQS::QueuesCache.new([:queue_1, :queue_2])
+        @sample_queues = [ mocked_queue('queue_1'), mocked_queue('queue_2') ]
+        @queues_cache.instance_variable_set(:@queues, @sample_queues)
+      end
+
+      it "should return 2 queues" do
+        queues = @queues_cache.send(:fetch_all_queues)
+        expect(queues.size).to eq(2)
+        expect(queues).to eq(@sample_queues)
+      end
+    end
+
+    context "counter % RESET_THRESHOLD == 0" do
+      before do
+        stub_client_list_queues
+        stub_retrieving_named_queues
+        @queues_cache = FanSQS::QueuesCache.new([:queue_1, :queue_2, :queue_3])
+        @queues_cache.instance_variable_set(:@cache, { :queue_1 => mocked_queue('queue_1'), :queue_2 => mocked_queue('queue_2'), :queue_3 => mocked_queue('queue_3')})
+        @queues_cache.instance_variable_set(:@counter, FanSQS::QueuesCache::RESET_THRESHOLD)
+      end
+
+      it "should return 3 queues" do
+        queues = @queues_cache.send(:fetch_all_queues)
+        expect(queues.size).to eq(3)
+      end
+    end
+  end
+
+  describe "#fetch_specific_queues" do
+    context "@queues is not empty" do
+      before do
+        @queues_cache = FanSQS::QueuesCache.new([:queue_1, :queue_2, :queue_3])
+        @queues_cache.instance_variable_set(:@queues, [ mocked_queue('queue_1') ])
+      end
+
+      it "should return @queues" do
+        queues = @queues_cache.send(:fetch_specific_queues)
+        expect(queues.size).to eq(1)
+      end
+    end
+
+    context "@queues is empty" do
+      before do
+        stub_retrieving_named_queues
+        @queues_cache = FanSQS::QueuesCache.new([:queue_1, :queue_2])
+        @queues_cache.instance_variable_set(:@queues, [ ])
+      end
+
+      it "should return 2 queues" do
+        queues = @queues_cache.send(:fetch_specific_queues)
+        expect(queues.size).to eq(2)
+      end
+    end
+  end
+
+  describe "#resolve_qnames" do
+    let!(:qnames) { [ :q1, :q2 ] }
+    let!(:obj) { FanSQS::QueuesCache.new(:sample) }
+
+    it "should return the argument as it is passed in if the argument is an Array" do
+      expect(obj.send(:resolve_qnames, qnames)).to eq(qnames.map(&:to_s))
+    end
+
+    it "should return the list of queues if it is passed in as a String" do
+      expect(obj.send(:resolve_qnames, 'q1,q2,q3,q1').sort).to eq([ 'q1', 'q2', 'q3' ])
+    end
+  end
+end

data/spec/FanSQS/worker_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe FanSQS::Worker do
+  describe "#included" do
+    it "is a pending example"
+  end
+
+  describe "#perform_async" do
+    it "is a pending example"
+  end
+
+  describe "#set_fan_sqs_options" do
+    it "is a pending example"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,26 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+
+require 'FanSQS'
+
+Dir["./spec/support/**/*.rb"].each {|f| require f}  
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  config.include SQSQueueMocks
+  config.include SQSQueueStubs
+  config.include SQSReceivedMessageMocks
+  config.include SQSSentMessageMocks
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/spec/support/sqs_queue_mocks.rb

@@ -0,0 +1,13 @@
+module SQSQueueMocks
+  def mocked_queue(name = 'queue')
+    queue = double(name)
+    received_messages = mocked_received_message_collection(10)
+    allow(queue).to receive(:receive_messages).with({ limit: 10 }).and_return(received_messages)
+  end
+
+  def mocked_queue_collection(size = 3)
+    queues = []
+    size.times { |i| queues << mocked_queue("queue_#{i}") }
+    queues
+  end
+end

data/spec/support/sqs_queue_stubs.rb

@@ -0,0 +1,47 @@
+module SQSQueueStubs
+  # stubbed for 
+  #     AWS.sqs.queues.create(name.to_s)
+  def stub_queue_create
+    allow_any_instance_of(AWS::SQS::QueueCollection).to receive(:create).with(an_instance_of(String)).and_return(mocked_queue)
+  end
+
+  # stubbed for 
+  #     AWS::SQS.new.queues.named(qname)
+  #     AWS.sqs.queues.named(formatted_queue_name(qname))
+  def stub_retrieving_named_queues
+    allow_any_instance_of(AWS::SQS::QueueCollection).to receive(:named).with(an_instance_of(String)).and_return(mocked_queue)
+  end
+
+  # stubbed for 
+  #     AWS::SQS.new.queues.named(qname)
+  def stub_retrieving_named_queues_raise_exception
+    allow_any_instance_of(AWS::SQS::QueueCollection).to receive(:named).with(an_instance_of(String)).and_raise(AWS::SQS::Errors::NonExistentQueue)
+  end
+
+  # stubbed for 
+  #     AWS::SQS.new.queues.with_prefix(qname.split('*').first)
+  def stub_retrieving_queues_with_prefix
+    allow_any_instance_of(AWS::SQS::QueueCollection).to receive(:with_refix).with(any_args).and_return(mocked_queue_collection)
+  end
+
+  # stubbed for 
+  #     AWS::SQS::Client.new.list_queues[:queue_urls]
+  def stub_client_list_queues
+    queue_urls = { queue_urls: ["https://sqs.us-east-1.amazonaws.com/1/queue_1", "https://sqs.us-east-1.amazonaws.com/2/queue_2", "https://sqs.us-east-1.amazonaws.com/3/queue_3"] }
+    allow_any_instance_of(AWS::SQS::Client::V20121105).to receive(:list_queues).and_return(queue_urls)
+  end
+
+  # stub for
+  #     queue.receive_messages(limit: 10) do |message|
+  def stub_queue_receive_messages(size = 10)
+    received_messages = mocked_received_message_collection(size)
+    allow_any_instance_of(AWS::SQS::Queue).to receive(:receive_messages).with(limit: size).and_yield(receive_messages)
+  end
+
+  # stub for
+  #     params = { class: self.name, arguments: args }
+  #     queue.send_message(params.to_json)
+  def stub_queue_send_message(size = 10)
+    allow_any_instance_of(AWS::SQS::Queue).to receive(:send_message).with(an_instance_of(Hash)).and_return(mocked_sent_message)
+  end
+end

data/spec/support/sqs_received_message_mocks.rb

@@ -0,0 +1,17 @@
+module SQSReceivedMessageMocks
+  def mocked_received_message(name = 'received_message')
+    received_message = double(name)
+    allow(received_message).to receive(:body).and_return(
+      {
+        class: 'MessagePublisher',
+        arguments: [ name ]
+      }.to_json
+    )
+  end
+
+  def mocked_received_message_collection(size = 3)
+    received_messages = []
+    size.times { |i| received_messages << mocked_received_message("received_message_#{i}") }
+    received_messages
+  end
+end

data/spec/support/sqs_sent_message_mocks.rb

@@ -0,0 +1,7 @@
+module SQSSentMessageMocks
+  def mocked_sent_message(name = 'sent_message')
+    sent_message = double(name)
+    allow(sent_message).to receive(:md5).and_return('md5_hash_value')
+    sent_message
+  end
+end

metadata

@@ -0,0 +1,211 @@
+--- !ruby/object:Gem::Specification
+name: FanSQS
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Khang Pham
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.41.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.41.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda-matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Distributed messages queuing system using AWS SQS
+email:
+- vkhang55@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- FanSQS.gemspec
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- NOTES.txt
+- README.md
+- Rakefile
+- lib/FanSQS.rb
+- lib/FanSQS/message_parser.rb
+- lib/FanSQS/poller.rb
+- lib/FanSQS/pool.rb
+- lib/FanSQS/queue_wrapper.rb
+- lib/FanSQS/queues_cache.rb
+- lib/FanSQS/railtie.rb
+- lib/FanSQS/tasks/FanSQS.rake
+- lib/FanSQS/version.rb
+- lib/FanSQS/worker.rb
+- spec/FanSQS/message_parser_spec.rb
+- spec/FanSQS/poller_spec.rb
+- spec/FanSQS/pool_spec.rb
+- spec/FanSQS/queue_wrapper_spec.rb
+- spec/FanSQS/queues_cache_spec.rb
+- spec/FanSQS/worker_spec.rb
+- spec/spec_helper.rb
+- spec/support/sqs_queue_mocks.rb
+- spec/support/sqs_queue_stubs.rb
+- spec/support/sqs_received_message_mocks.rb
+- spec/support/sqs_sent_message_mocks.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Distributed messages queuing system using AWS SQS
+test_files:
+- spec/FanSQS/message_parser_spec.rb
+- spec/FanSQS/poller_spec.rb
+- spec/FanSQS/pool_spec.rb
+- spec/FanSQS/queue_wrapper_spec.rb
+- spec/FanSQS/queues_cache_spec.rb
+- spec/FanSQS/worker_spec.rb
+- spec/spec_helper.rb
+- spec/support/sqs_queue_mocks.rb
+- spec/support/sqs_queue_stubs.rb
+- spec/support/sqs_received_message_mocks.rb
+- spec/support/sqs_sent_message_mocks.rb