refinery 0.10.11 → 0.11.0

data/VERSION

@@ -1 +1 @@
-0.10.11
+0.11.0

data/lib/refinery.rb

@@ -48,6 +48,8 @@ module Refinery
     if require_optional_library('java', 'JRuby')
       require_optional_library('typica', 'JRuby Typica wrapper')
     end
+    
+    require_optional_library('beanstalk-client', 'Beanstalk Client')
   end
   
   # Require internal code files
@@ -74,6 +76,10 @@ module Refinery
     require 'refinery/statistics'
     require 'refinery/stats_server'
     
+    if defined?(Beanstalk)
+      require 'refinery/beanstalk_queue'
+      require 'refinery/beanstalk_queue_provider'
+    end
   end
   
   # Raised if a source file cannot be loaded

data/lib/refinery/beanstalk_queue.rb

@@ -0,0 +1,36 @@
+module Refinery #:nodoc:
+  # An interface to beanstalk using SQS-compatible methods.
+  class BeanstalkQueue
+    include Refinery::Loggable
+    attr_reader :name
+    
+    # Construct a BeanstalkQueue instance.
+    #
+    # *<tt>name</tt>: if specified then that "tube" will be used.
+    # *<tt>host</tt>: if specified then those host:port combos will be used.
+    def initialize(name=nil, hosts=nil)
+      @name = name.gsub(/_/, '-') # beanstalk does not like underscores in tube names
+      @hosts = hosts || ['localhost:11300']
+    end
+    
+    # Get the next message from the queue
+    def receive(visibility=nil)
+      beanstalk.reserve(visibility)
+    end
+    
+    # Get the approximate queue size
+    def size
+      beanstalk.stats_tube(name)['current-jobs-ready']
+    end
+    
+    # Send a message
+    def send_message(message)
+      beanstalk.put(message)
+    end
+    
+    protected
+    def beanstalk
+      @beanstalk ||= Beanstalk::Pool.new(@hosts, name)
+    end
+  end
+end

data/lib/refinery/beanstalk_queue_provider.rb

@@ -0,0 +1,18 @@
+module Refinery #:nodoc:
+  # A queue provider for beanstalk
+  class BeanstalkQueueProvider
+    include Refinery::Loggable
+    
+    attr_reader :queues
+    
+    # Initialize the queue provider
+    def initialize(hosts=nil)
+      @hosts = hosts
+      @queues = {}
+    end
+    # Get the named queue
+    def queue(name)
+      queues[name] ||= Refinery::BeanstalkQueue.new(name, @hosts)
+    end
+  end
+end

data/lib/refinery/queueable.rb

@@ -14,7 +14,8 @@ module Refinery #:nodoc:
       begin
         yield queue(name)
       rescue Exception => e
-        logger.error "An error occurred when communicating with queue #{name}: #{e}"
+        logger.error "An error occurred when communicating with queue #{name}: #{e.message}"
+        puts e.backtrace.map { |t| "\t#{t}" }.join("\n")
         sleep(30)
       end
     end
@@ -23,17 +24,21 @@ module Refinery #:nodoc:
     # Get the queue provider. Defaults to RightAws::SqsGen2 running
     # in multi-thread mode.
     def queue_provider
-      if defined?(Typica)
-        @queue_provider ||= Typica::Sqs::QueueService.new(
-          config['aws']['credentials']["access_key_id"], 
-          config['aws']['credentials']["secret_access_key"]
-        )
+      if config['queue_engine'] == 'beanstalk' && defined?(Beanstalk)
+        @queue_provider ||= Refinery::BeanstalkQueueProvider.new
       else
-        @queue_provider ||= RightAws::SqsGen2.new(
-          config['aws']['credentials']["access_key_id"], 
-          config['aws']['credentials']["secret_access_key"],
-          {:multi_thread => true}
-        )
+        if defined?(Typica)
+          @queue_provider ||= Typica::Sqs::QueueService.new(
+            config['aws']['credentials']["access_key_id"], 
+            config['aws']['credentials']["secret_access_key"]
+          )
+        else
+          @queue_provider ||= RightAws::SqsGen2.new(
+            config['aws']['credentials']["access_key_id"], 
+            config['aws']['credentials']["secret_access_key"],
+            {:multi_thread => true}
+          )
+        end
       end
     end
   end

data/refinery.gemspec

@@ -1,12 +1,15 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{refinery}
-  s.version = "0.10.11"
+  s.version = "0.11.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Anthony Eden"]
-  s.date = %q{2009-09-18}
+  s.date = %q{2009-11-11}
   s.description = %q{Process data in a distributed fashion.}
   s.email = %q{anthonyeden@gmail.com}
   s.executables = ["epub", "monitor", "pubnow", "refinery"]
@@ -29,6 +32,8 @@ Gem::Specification.new do |s|
      "bin/refinery",
      "config/config.example.yml",
      "lib/refinery.rb",
+     "lib/refinery/beanstalk_queue.rb",
+     "lib/refinery/beanstalk_queue_provider.rb",
      "lib/refinery/config.rb",
      "lib/refinery/configurable.rb",
      "lib/refinery/daemon.rb",
@@ -70,12 +75,11 @@ Gem::Specification.new do |s|
      "workers/sample.rb",
      "workers/sleep.rb"
   ]
-  s.has_rdoc = true
   s.homepage = %q{http://github.com/aeden/refinery}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{refinery}
-  s.rubygems_version = %q{1.3.1}
+  s.rubygems_version = %q{1.3.5}
   s.summary = %q{Refinery processes data in a distributed environment.}
   s.test_files = [
     "test/test_helper.rb",
@@ -97,7 +101,7 @@ Gem::Specification.new do |s|
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 2
+    s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<right_aws>, [">= 0"])
@@ -114,3 +118,4 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<addressable>, [">= 0"])
   end
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: refinery
 version: !ruby/object:Gem::Version 
-  version: 0.10.11
+  version: 0.11.0
 platform: ruby
 authors: 
 - Anthony Eden
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-18 00:00:00 -10:00
+date: 2009-11-11 00:00:00 -10:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -69,6 +69,8 @@ files:
 - bin/refinery
 - config/config.example.yml
 - lib/refinery.rb
+- lib/refinery/beanstalk_queue.rb
+- lib/refinery/beanstalk_queue_provider.rb
 - lib/refinery/config.rb
 - lib/refinery/configurable.rb
 - lib/refinery/daemon.rb
@@ -111,6 +113,8 @@ files:
 - workers/sleep.rb
 has_rdoc: true
 homepage: http://github.com/aeden/refinery
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --charset=UTF-8
@@ -131,9 +135,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: refinery
-rubygems_version: 1.3.1
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Refinery processes data in a distributed environment.
 test_files: 
 - test/test_helper.rb