sqskiq 0.0.1

data/lib/sqskiq/aws.rb

@@ -0,0 +1,20 @@
+require 'aws-sdk'
+
+module Sqskiq
+  module AWS
+
+    def init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
+      sqs = ::AWS::SQS.new(:access_key_id => aws_access_key_id, :secret_access_key => aws_secret_access_key)
+      @queue = sqs.queues.named(queue_name.to_s)
+    end
+
+    def fetch_sqs_messages
+      @queue.receive_message(:limit => 10, :attributes => [:receive_count])
+    end
+
+    def delete_sqs_messages(messages)
+      @queue.batch_delete(messages)
+    end
+
+  end
+end

data/lib/sqskiq/batch_process.rb

@@ -0,0 +1,46 @@
+require 'celluloid'
+require 'celluloid/autostart'
+
+module Sqskiq
+  class BatchProcessor
+    include Celluloid
+    include Celluloid::Notifications
+
+    def initialize
+      @manager = Celluloid::Actor[:manager]
+      @processor = Celluloid::Actor[:processor]
+
+      subscribe_interrupt
+    end
+
+    def batch_process(messages)
+      p "processing #{messages.size} messages"
+
+      process_result = []
+      messages.each do |message|
+        process_result << @processor.future.process(message)
+      end
+
+      success_messages = []
+      process_result.each do |result|
+        value = result.value
+        if value[:success]
+          success_messages << value[:message]
+        end
+      end
+
+      @manager.async.batch_process_done(success_messages)
+    end
+
+    def subscribe_interrupt
+      subscribe('SIGINT', :interrupt)
+      subscribe('TERM', :interrupt)
+      subscribe('SIGTERM', :interrupt)
+    end
+
+    def interrupt(signal)
+      self.terminate
+    end
+
+  end
+end

data/lib/sqskiq/delete.rb

@@ -0,0 +1,19 @@
+require 'celluloid'
+require 'celluloid/autostart'
+require 'sqskiq/aws'
+
+module Sqskiq
+  class Deleter
+    include Celluloid
+    include Celluloid::Notifications
+    include Sqskiq::AWS
+
+    def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
+      init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
+    end
+
+    def delete(messages)
+      delete_sqs_messages(messages) if not messages.empty?
+    end
+  end
+end

data/lib/sqskiq/fetch.rb

@@ -0,0 +1,34 @@
+require 'celluloid'
+require 'celluloid/autostart'
+require 'sqskiq/aws'
+
+module Sqskiq
+  class Fetcher
+    include Celluloid
+    include Celluloid::Notifications
+    include Sqskiq::AWS
+
+    def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
+      init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
+      subscribe_interrupt
+
+      @manager = Celluloid::Actor[:manager]
+    end
+
+    def fetch
+      messages = fetch_sqs_messages
+      @manager.async.fetch_done(messages)
+    end
+
+    def subscribe_interrupt
+      subscribe('SIGINT', :interrupt)
+      subscribe('SIGTERM', :interrupt)
+      subscribe('TERM', :interrupt)
+    end
+
+    def interrupt(signal)
+      self.terminate
+    end
+
+  end
+end

data/lib/sqskiq/manager.rb

@@ -0,0 +1,50 @@
+require 'celluloid'
+require 'celluloid/autostart'
+
+
+module Sqskiq
+  class Manager
+    include Celluloid
+    include Celluloid::Notifications
+
+    def initialize
+      @shutting_down = false
+      subscribe_shutting_down
+    end
+
+    def bootstrap
+      @fetcher = Celluloid::Actor[:fetcher]
+      @batch_processor = Celluloid::Actor[:batch_processor]
+      @deleter = Celluloid::Actor[:deleter]
+      #TODO default value. Should be configurable
+      new_fetch(2)
+    end
+
+    def fetch_done(messages)
+      @batch_processor.async.batch_process(messages) unless @shutting_down
+    end
+
+    def batch_process_done(messages)
+      @deleter.async.delete(messages) if @deleter.alive?
+      new_fetch(1)
+    end
+
+    def new_fetch(num)
+      num.times { @fetcher.async.fetch unless @shutting_down }
+    end
+
+    def shutting_down(signal)
+      @shutting_down = true
+    end
+
+    def running?
+      not (@deleter.busy_size == 0 and @shutting_down)
+    end
+
+    def subscribe_shutting_down
+      subscribe('SIGINT', :shutting_down)
+      subscribe('TERM', :shutting_down)
+      subscribe('SIGTERM', :shutting_down)
+    end
+  end
+end

data/lib/sqskiq/process.rb

@@ -0,0 +1,36 @@
+require 'celluloid'
+require 'celluloid/autostart'
+
+module Sqskiq
+  class Processor
+    include Celluloid
+    include Celluloid::Notifications
+
+    def initialize(worker_class)
+      @worker_instance = worker_class.new
+
+      subscribe_interrupt
+    end
+
+    def process(message)
+      result = true
+      begin
+        @worker_instance.perform(message)
+      rescue Exception => e
+        result = false
+      end
+      { :success => result, :message => message }
+    end
+
+    def subscribe_interrupt
+      subscribe('SIGINT', :interrupt)
+      subscribe('TERM', :interrupt)
+      subscribe('SIGTERM', :interrupt)
+    end
+
+    def interrupt(signal)
+      self.terminate
+    end
+
+  end
+end

data/lib/sqskiq/worker.rb

@@ -0,0 +1,21 @@
+module Sqskiq
+  module Worker
+
+    module ClassMethods
+      def run
+        Sqskiq.bootstrap(sqskiq_options_hash, self)
+      end
+
+      def sqskiq_options(options)
+        self.sqskiq_options_hash = options
+      end
+    end
+
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.class_attribute :sqskiq_options_hash
+    end
+
+  end
+
+end

data/lib/sqskiq.rb

@@ -0,0 +1,59 @@
+require "sqskiq/manager"
+require 'sqskiq/fetch'
+require 'sqskiq/process'
+require 'sqskiq/delete'
+require 'sqskiq/worker'
+require 'sqskiq/batch_process'
+
+module Sqskiq
+
+  def self.bootstrap(options, worker_class)
+
+    params = [ @aws_access_key_id, @aws_secret_access_key, options[:queue_name] ]
+
+    Celluloid::Actor[:manager] = @manager = Manager.new
+    Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => 2, :args => params)
+    Celluloid::Actor[:processor] = @processor = Processor.pool(:size => 20, :args => worker_class)
+    Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => 2)
+    Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => 2, :args => params)
+
+    p "pid = #{Process.pid}"
+
+    trap('SIGTERM') do
+      @manager.publish('SIGTERM')
+    end
+
+    trap('TERM') do
+      @manager.publish('TERM')
+    end
+
+    trap('SIGINT') do
+      @manager.publish('SIGINT')
+    end
+
+    @manager.bootstrap
+    while @manager.running? do
+      sleep 2
+    end
+
+    @fetcher.__shutdown__
+    @batch_processor.__shutdown__
+    @processor.__shutdown__
+    @deleter.__shutdown__
+
+    @manager.terminate
+  end
+
+  def self.configure
+    yield self
+  end
+
+  def self.aws_access_key_id=(value)
+    @aws_access_key_id = value
+  end
+
+  def self.aws_secret_access_key=(value)
+    @aws_secret_access_key = value
+  end
+
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: sqskiq
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Ricardo Lazaro de Oliveira
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.9.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.9.1
+description: sqskiq
+email: ri.vanlazar@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sqskiq.rb
+- lib/sqskiq/manager.rb
+- lib/sqskiq/fetch.rb
+- lib/sqskiq/process.rb
+- lib/sqskiq/delete.rb
+- lib/sqskiq/worker.rb
+- lib/sqskiq/batch_process.rb
+- lib/sqskiq/aws.rb
+homepage: http://rubygems.org/gems/sqskiq
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: sqskiq
+test_files: []