queuel 0.0.1 → 0.1.0

data/.cane

@@ -0,0 +1,3 @@
+--no-doc
+--style-measure 120
+--abc-max 25

data/.gitignore

@@ -3,7 +3,6 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage
@@ -15,3 +14,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.rbx

data/.ruby-gemset

@@ -0,0 +1 @@
+queuel

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3

data/Gemfile.lock

@@ -0,0 +1,112 @@
+PATH
+  remote: .
+  specs:
+    queuel (0.1.0)
+      mono_logger
+      thread
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cane (2.5.2)
+      parallel
+    coderay (1.0.9)
+    diff-lcs (1.2.4)
+    ethon (0.5.12)
+      ffi (>= 1.3.0)
+      mime-types (~> 1.18)
+    ffi (1.8.1)
+    ffi (1.8.1-java)
+    formatador (0.2.4)
+    guard (1.8.0)
+      formatador (>= 0.2.4)
+      listen (>= 1.0.0)
+      lumberjack (>= 1.0.2)
+      pry (>= 0.9.10)
+      thor (>= 0.14.6)
+    guard-bundler (1.0.0)
+      bundler (~> 1.0)
+      guard (~> 1.1)
+    guard-cane (0.1.2)
+      cane
+      guard
+    guard-rspec (3.0.0)
+      guard (>= 1.8)
+      rspec (~> 2.13)
+    iron_core (0.6.2)
+      rest (>= 2.2.0)
+    iron_mq (4.0.3)
+      iron_core (>= 0.5.1)
+    json (1.7.7)
+    json (1.7.7-java)
+    listen (1.1.1)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+      rb-kqueue (>= 0.2)
+    lumberjack (1.0.3)
+    method_source (0.8.1)
+    mime-types (1.23)
+    mono_logger (1.0.1)
+    multi_json (1.7.3)
+    parallel (0.6.5)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry (0.9.12.2-java)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    rake (10.0.4)
+    rb-fchange (0.0.6)
+      ffi
+    rb-fsevent (0.9.3)
+    rb-inotify (0.9.0)
+      ffi (>= 0.5.0)
+    rb-kqueue (0.2.0)
+      ffi (>= 0.5.0)
+    rest (2.6.0)
+      rest-client (>= 0.3.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    slop (3.4.5)
+    spoon (0.0.4)
+      ffi
+    thor (0.18.1)
+    thread (0.0.8)
+    typhoeus (0.6.3)
+      ethon (~> 0.5.11)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  cane
+  guard-bundler
+  guard-cane
+  guard-rspec
+  iron_mq
+  json (~> 1.7.7)
+  queuel!
+  rake
+  rb-fchange
+  rb-fsevent
+  rb-inotify
+  rspec
+  simplecov
+  typhoeus

data/Guardfile

@@ -0,0 +1,22 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'bundler' do
+  watch('Gemfile')
+  watch(/^.+\.gemspec/)
+end
+
+guard :cane do
+  watch(%r{^lib/.+\.rb$})
+end
+
+guard 'rspec',
+  all_on_start: true,
+  keep_failed: true,
+  rvm: ["1.9.3-p392@queuel"] do
+  # would like to add "jruby-1.7.3@queuel", "rbx-head@queuel"
+  watch(%r{^spec/.+\.rb$})                  { "spec" }
+  watch(%r{^lib/queuel/base/(.+)\.rb$})     { |m| ["spec/lib/queuel/iron_mq/#{m[1]}_spec.rb", "spec/lib/queuel/null/#{m[1]}_spec.rb"] }
+  watch(%r{^lib/(.+)\.rb$})                 { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')              { "spec" }
+end

data/README.md

@@ -1,4 +1,5 @@
 # Queuel
+[![Gem Version](https://badge.fury.io/rb/queuel.png)](http://badge.fury.io/rb/queuel)
 [![Build Status](https://travis-ci.org/sportngin/queuel.png?branch=master)](https://travis-ci.org/sportngin/queuel)
 
 Queuel is a kewl, lite wrapper around Queue interfaces. Currently it implements:
@@ -41,6 +42,16 @@ Queuel.configure do
 
   # currently only [:iron_mq, :null] available
   engine :iron_mq
+
+  # For Queuel.recevier {} you can configure more than one thread to
+  # handle incoming messages
+  receiver_threads 3 # default: 1
+
+  # Logging: Default is MonoLogger, because its a non-blocking log-extension
+  # To the standard lib Logger. Any Log4r solution should work.
+  logger Logger # default: MonoLogger.new(STDOUT)
+
+  log_level MonoLogger::DEBUG # default: MonoLogger::ERROR # => 3
 end
 ```
 
@@ -49,7 +60,7 @@ end
 ### General Queue API
 
 ```ruby
-# Using default Queue
+# Using default Queue from config
 Queuel.pop
 Queuel.push "My message to you"
 Queuel.receive do |message|
@@ -63,22 +74,16 @@ Queuel.with("officials").receive do |message|
   puts "I received #{message.body}" # NOTE the message interface may change, this is currently not wrapped by the gem
 end
 
-# Timeout the receiving
-Queuel.receive poll_timeout: 60 do |message|
-  puts "I received #{message.body}" # NOTE the message interface may change, this is currently not wrapped by the gem
-end
-
-# Don't receive more than 10 nil messages
-Queuel.receive max_consecutive_fails: 10 do |message|
-  puts "I received #{message.body}" # NOTE the message interface may change, this is currently not wrapped by the gem
-end
-
 # Break on nil
 Queuel.receive break_if_nil: true do |message|
   puts "I received #{message.body}" # NOTE the message interface may change, this is currently not wrapped by the gem
 end
 ```
 
+#### Caveats of the receiver
+
+* Your block must return true in order to not replace the message to the Queue
+
 ### The message
 
 ```ruby
@@ -91,6 +96,7 @@ message.delete    # => Delete the message
 
 * Implement AMQP
 * Configureable exponential back-off on `receive`
+* Provide a Daemon
 
 ## Contributing
 

data/Rakefile

@@ -4,3 +4,10 @@ require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+
+namespace :spec do
+  desc "run just tasks flagged for perf checks"
+  RSpec::Core::RakeTask.new(:perf) do |t|
+    t.rspec_opts = ['--tag perf', '--profile']
+  end
+end

data/lib/queuel.rb

@@ -1,26 +1,38 @@
 require "queuel/version"
 require "forwardable"
-require "queuel/null/message"
+require "queuel/configurator"
+require "queuel/introspect"
+
+require "queuel/base/engine"
+require "queuel/base/queue"
+require "queuel/base/message"
+require "queuel/base/poller"
+
 require "queuel/null/engine"
-require "queuel/null/poller"
 require "queuel/null/queue"
-require "queuel/iron_mq/message"
+require "queuel/null/message"
+require "queuel/null/poller"
+
 require "queuel/iron_mq/engine"
-require "queuel/iron_mq/poller"
 require "queuel/iron_mq/queue"
+require "queuel/iron_mq/message"
+require "queuel/iron_mq/poller"
+
 require "queuel/client"
 
 module Queuel
+  extend Introspect
   class << self
     extend Forwardable
     def_delegators :client, :push, :pop, :receive, :with
-    def_delegators :config, :credentials, :default_queue
+    def_delegators :config, :credentials, :default_queue, :receiver_threads
     alias << pop
   end
 
   def self.engine
     requires
-    Object.module_eval("::#{const_name}", __FILE__, __LINE__)
+    warn_engine_selection
+    const_with_nesting engine_const_name
   end
 
   def self.configure(&block)
@@ -35,34 +47,45 @@ module Queuel
     Client.new engine, credentials
   end
 
+  def self.logger
+    config.logger.tap { |log|
+      log.level = config.log_level
+    }
+  end
+
+  private
+
+  def self.warn_engine_selection
+    @warned_null_engine ||= logger.warn(engine_config[:message])
+  end
+
+  def self.engine_config
+    engines.fetch(config.engine) { engines[:null] }
+  end
+
+  def self.configured_engine_name
+    engine_config[:const]
+  end
+
   def self.engines
     {
-      iron_mq: { require: 'iron_mq', const: "IronMq" },
-      null: { const: "Null" }
+      iron_mq: {
+        require: 'iron_mq',
+        const: "IronMq",
+        message: "Using IronMQ"
+      },
+      null: {
+        const: "Null",
+        message: "Using Null Engine, for compatability."
+      }
     }
   end
 
   def self.requires
-    require engines[config.engine][:require] if engines.fetch(config.engine, {})[:require]
-  end
-
-  def self.const_name
-    "Queuel::#{engines.fetch(config.engine, {}).fetch(:const, nil) || engines[:null][:const]}::Engine"
+    require engine_config[:require] if engine_config[:require]
   end
 
-  class Configurator
-    def self.param(*params)
-      params.each do |name|
-        attr_accessor name
-        define_method name do |*values|
-          value = values.first
-          value ? self.send("#{name}=", value) : instance_variable_get("@#{name}")
-        end
-      end
-    end
-
-    param :credentials
-    param :engine
-    param :default_queue
+  def self.engine_const_name
+    "Queuel::#{configured_engine_name}::Engine"
   end
 end

data/lib/queuel/base/engine.rb

@@ -0,0 +1,35 @@
+module Queuel
+  module Base
+    class Engine
+      extend Introspect
+      def self.inherited(klass)
+        klass.class_eval do
+          def queue_klass
+            self.class.const_with_nesting "Queue"
+          end
+        end
+      end
+
+      def initialize(credentials = {})
+        self.credentials = credentials
+        self.memoized_queues = {}
+      end
+
+      def queue(which_queue)
+        memoized_queues[which_queue.to_s] ||= queue_klass.new(client, which_queue)
+      end
+
+      private
+      attr_accessor :credentials
+      attr_accessor :memoized_queues
+
+      def client
+        @client ||= client_klass.new credentials
+      end
+
+      def client_klass
+        raise NotImplementedError, "Must define a Queue class"
+      end
+    end
+  end
+end

data/lib/queuel/base/message.rb

@@ -0,0 +1,32 @@
+module Queuel
+  module Base
+    class Message
+      def initialize(message_object)
+        self.message_object = message_object
+      end
+
+      def delete
+        raise NotImplementedError, "must define method #delete"
+      end
+
+      def empty?
+        body.to_s.empty?
+      end
+      alias blank? empty?
+
+      def present?
+        !empty?
+      end
+
+      attr_reader :id
+      attr_reader :body
+      attr_reader :queue
+
+      private
+      attr_accessor :message_object
+      attr_writer :id
+      attr_writer :body
+      attr_writer :queue
+    end
+  end
+end

data/lib/queuel/base/poller.rb

@@ -0,0 +1,161 @@
+require 'thread'
+require "thread/pool"
+require "forwardable"
+module Queuel
+  module Base
+    class Poller
+      extend Forwardable
+      def_delegators :Queuel, :logger
+
+      def initialize(queue, param_block, options = {}, workers = 1)
+        self.workers = workers
+        self.queue = queue
+        self.options = options || {}
+        self.inst_block = param_block
+        self.tries = 0
+        self.continue_looping = true
+      end
+
+      def poll
+        register_trappers
+        logger.debug("Beginning Poll...")
+        self.master = master_thread
+        log_action(:joining, :master) { master.join }
+      rescue SignalException => e
+        logger.warn "Caught (#{e}), shutting Poller down"
+        log_action(:killing, :poller, :warn) { shutdown }
+      end
+
+      protected
+      attr_accessor :tries
+      attr_accessor :workers
+      attr_accessor :inst_block
+
+      private
+      attr_accessor :master
+      attr_accessor :queue
+      attr_accessor :args
+      attr_accessor :options
+      attr_accessor :continue_looping
+
+      def register_trappers
+        trap(:SIGINT) { shutdown }
+        trap(:INT) { shutdown }
+      end
+
+      def log_action(verb, subject, level = :debug)
+        verb = verb.to_s.upcase.gsub(/\_/, " ")
+        subject = subject.to_s.upcase.gsub(/\_/, " ")
+        logger.public_send level, "#{verb} #{subject} START"
+        yield
+        logger.public_send level, "#{verb} #{subject} COMPLETE"
+      end
+
+      def shutdown
+        log_action(:shutting_down, :thread_pool) { pool.shutdown }
+        log_action(:killing, :master_thread) { master.kill }
+        log_action(:killing, :loop) { quit_looping! }
+      end
+
+      def pool
+        @pool ||= Thread.pool workers
+      end
+
+      def master_thread
+        Thread.new do
+          master_looper
+        end
+      end
+
+      def peek_options
+        {}
+      end
+
+      def peek
+        queue.peek peek_options
+      end
+
+      def queue_size # TODO optionize the peek options
+        Array(peek).size
+      end
+
+      def process_off_peek
+        mem_queue_size = queue_size
+        if mem_queue_size > 0
+          reset_tries
+          mem_queue_size.times do
+            process_on_thread
+          end
+        else
+          tried
+          quit_looping! if quit_on_empty?
+        end
+      end
+
+      def process_on_thread
+        pool.process do
+          process_message
+        end
+      end
+
+      def process_message
+        register_trappers
+        message = pop_new_message
+        message.delete if self.inst_block.call message
+      rescue => e
+        logger.warn "Received #{e} when processing #{message}"
+        logger.warn "Backtrace:"
+        logger.warn e.backtrace
+      end
+
+      def master_looper
+        loop do
+          break unless continue_looping?
+          process_off_peek
+          sleep sleep_time
+        end
+      end
+
+      def built_options
+        raise NotImplementedError
+      end
+
+      def continue_looping?
+        !!continue_looping
+      end
+
+      def break_if_nil?
+        !!options[:break_if_nil]
+      end
+      alias quit_on_empty? break_if_nil?
+
+      def quit_looping!
+        self.continue_looping = false
+      end
+
+      def pop_new_message
+        queue.pop built_options
+      end
+
+      def start_sleep_time
+        0
+      end
+
+      def increment_sleep_time
+        0.1
+      end
+
+      def sleep_time
+        tries < 30 ? ((start_sleep_time + increment_sleep_time) * tries) : 3
+      end
+
+      def reset_tries
+        self.tries = 0
+      end
+
+      def tried
+        self.tries += 1
+      end
+    end
+  end
+end