myobie-merb_queue 0.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Nathan Herald
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,47 @@
+merb_queue
+==========
+
+<http://github.com/myobie/merb_queue>
+
+Description
+-----------
+
+A persistent queue used to perform repetitive actions with many merb processes across many servers if needed.
+
+Requirements
+------------
+
+*    merb-core
+*    sequel-core (if using db persistence)
+*    fiveruns-memcache-client (if using memcache persistence)
+
+Install
+-------
+
+gem install myobie-merb_queue -s http://gems.github.com
+
+License
+-------
+
+(The MIT License)
+
+Copyright (c) 2008 Nathan Herald
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,61 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+require "spec/rake/spectask"
+
+PLUGIN = "merb_queue"
+NAME = "merb_queue"
+GEM_VERSION = "0.0.1"
+AUTHOR = "Nathan Herald"
+EMAIL = "nathan@myobie.com"
+HOMEPAGE = "http://github.com/myobie/merb_queue"
+SUMMARY = "A persistent queue across multiple merb processes"
+
+spec = Gem::Specification.new do |s|
+  s.name = NAME
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.markdown", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  s.add_dependency('merb-core', '>= 0.9.3')
+  s.require_path = 'lib'
+  s.autorequire = PLUGIN
+  s.files = %w(LICENSE README.markdown Rakefile TODO) + Dir.glob("{lib,spec}/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the plugin locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{NAME}-#{VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{NAME}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end
+
+namespace :jruby do
+
+  desc "Run :package and install the resulting .gem with jruby"
+  task :install => :package do
+    sh %{#{SUDO} jruby -S gem install pkg/#{NAME}-#{Merb::VERSION}.gem --no-rdoc --no-ri}
+  end
+  
+end
+
+desc "Run the specs"
+Spec::Rake::SpecTask.new("specs") do |t|
+  t.spec_opts = ["--format", "specdoc", "--colour"]
+  t.spec_files = Dir["spec/**/*_spec.rb"].sort
+end

data/TODO

@@ -0,0 +1,5 @@
+- More specs!
+- Make it actually persistent using sqlite3
+- Use PStore for persistence
+- Use memcache for persistence
+- Use starling for persistence

data/lib/merb_queue.rb

@@ -0,0 +1,45 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'logger'
+require 'merb-core'
+
+module MerbQueueError
+  class QueueEmpty < StandardError; end
+  class WrongQueueDatatype < StandardError; end
+end
+
+require 'merb_queue/base'
+require 'merb_queue/queue'
+
+module MerbQueue
+
+  def self.[](what)
+    MerbQueue::Base[what]
+  end
+
+  def self.register(name, &block)
+    MerbQueue::Base.register(name, &block)
+  end
+
+  def self.add(name, &block)
+    MerbQueue::Base.add(name, &block)
+  end
+
+  def self.all
+    MerbQueue::Base.all
+  end
+
+  def self.logger
+    MerbQueue::Base.logger
+  end
+  
+  def self.delete(what)
+    MerbQueue::Base.delete(what)
+  end
+
+end
+
+Merb::BootLoader.after_app_loads do
+  require Merb.root / "config" / "queues.rb" if File.exists?(Merb.root / "config" / "queues.rb")
+end

data/lib/merb_queue/base.rb

@@ -0,0 +1,38 @@
+module MerbQueue
+  
+  class Base
+    @@queues = {}
+    @@logger = Logger.new(Merb.root / "log" / "queue.log", 5, 512000)
+    
+    def self.logger
+      @@logger
+    end
+    
+    def self.register(name, &block)
+      logger.info "Registering #{name}..."
+      a = MerbQueue.add(name, &block)
+      a.start
+    end
+    
+    def self.add(name, &block)
+      logger.info "Adding #{name}..."
+      a = MerbQueue::Queue.new(name, &block)
+      @@queues[name] = a
+      a
+    end
+    
+    def self.[](what)
+      @@queues[what]
+    end
+    
+    def self.all
+      @@queues
+    end
+    
+    def self.delete(what)
+      @@queues.delete(what)
+    end
+    
+  end
+  
+end

data/lib/merb_queue/queue.rb

@@ -0,0 +1,130 @@
+module MerbQueue
+  
+  class Queue
+    
+    attr_accessor :queue, :name, :running, :logger, :t, :wait_seconds, :max_attempts
+    
+    def initialize(name, &block)
+      @queue = []
+      @running = false
+      @max_attempts = 3
+      @wait_seconds = 10
+      @name = name
+      @logger = MerbQueue::Base.logger
+      @work_to_be_done = block
+    end
+    
+    def running?
+      @running
+    end
+
+    def queue
+      @queue
+    end
+
+    def length
+      queue.length
+    end
+    
+    def working?
+      length > 0
+    end
+
+    def push(something)
+      self.queue = queue.push(something)
+    end
+
+    def unshift(something)
+      self.queue = queue.unshift(something)
+    end
+
+    def shift
+      queue.shift
+    end
+
+    def first
+      queue.first
+    end
+
+    alias :get :shift
+    alias :remove! :shift
+    alias :add :push
+    alias :amount :length
+    alias :size :length
+    alias :<< :push
+    alias :>> :unshift
+
+    def start!
+      unless running?
+        logger.info "Starting!!! #{name}..."
+        
+        @t = Thread.new(name) do |my_name|
+          loop do
+            MerbQueue[my_name].next!
+            MerbQueue[my_name].working? ? sleep(0.25) : sleep(wait_seconds)
+          end
+        end
+        @t.priority = -9
+
+        @running = true
+      end
+    end#of start!
+
+    def start
+      unless running?
+        logger.info "Starting #{name}..."
+        
+        @t = Thread.new(name) do |my_name|
+          loop do
+            MerbQueue[my_name].next
+            MerbQueue[my_name].working? ? sleep(0.25) : sleep(wait_seconds)
+          end
+        end
+        @t.priority = -9
+
+        @running = true
+      end
+    end#of start
+
+    def clear!
+      @queue = [] # TODO: this might should be a bit nicer?
+    end
+    
+    def next
+      begin
+        self.next!
+      rescue MerbQueueError::QueueEmpty
+      rescue Exception => e
+        logger.error(e) # TODO: log this somewhere?
+      end
+    end#of next
+    
+    def next!
+      
+      current_work = self.get # pull an item off the top of the queue
+      max_tries = @max_attempts
+      raise MerbQueueError::QueueEmpty if current_work.nil?
+      raise MerbQueueError::WrongQueueDatatype unless current_work.is_a?(Hash)
+      current_work[:attempts] = current_work[:attempts] ? current_work[:attempts] + 1 : 1
+
+      begin
+        @work_to_be_done.call(current_work)
+      rescue Exception => e
+        if current_work[:attempts] >= max_tries
+          logger.error "#### This crap just doesn't work!"
+          logger.error current_work.to_yaml
+          # TODO: log this failure and all the errors somewhere?
+        else
+          current_work[:errors] = current_work[:errors] ? current_work[:errors] << e : current_work[:errors] = [e]
+          self << current_work # put it back so it can try again
+
+          logger.warn "There was an error, trying again."
+          logger.error e
+        end
+      end
+
+    end#of next!
+    
+  end#of Queue class
+  
+end

data/lib/merb_queue/version.rb

@@ -0,0 +1,9 @@
+module MerbQueue
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/spec/merb_queue_spec.rb

@@ -0,0 +1,97 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+# Time to add your specs!
+# http://rspec.info/
+describe MerbQueue do
+  
+  describe "adding" do
+    
+    before(:each) do
+      MerbQueue.delete(:test)
+    end
+    
+    it "should let you add" do
+      MerbQueue.should respond_to(:add)
+      MerbQueue.add(:test) { |w| true }
+      MerbQueue[:test].should_not be_nil
+    end
+    
+    it "should know how long the queue is" do
+      MerbQueue[:test].should be_nil # just to make sure the queue from before is now gone
+      
+      MerbQueue::Queue.stubs(:next).returns(true)
+      
+      MerbQueue.add(:test) { |w| true }
+      MerbQueue[:test].length.should == 0
+      
+      MerbQueue[:test] << {}
+      MerbQueue[:test].length.should == 1
+    end
+    
+    it "should report that it's not running" do
+      MerbQueue.add(:test) { |w| true }
+      MerbQueue[:test].should_not be_running
+    end
+  end
+  
+  describe "registering" do
+    
+    before(:each) do
+      MerbQueue.delete(:test)
+    end
+    
+    it "should let you register" do
+      MerbQueue::Queue.stubs(:next).returns(true)
+      
+      MerbQueue.should respond_to(:register)
+      MerbQueue.register(:test) { |w| true }
+      MerbQueue[:test].should_not be_nil
+    end
+    
+    it "should start a queue that is registered" do
+      MerbQueue::Queue.expects(:start).returns(true)
+      MerbQueue.register(:test) { |w| true }
+    end
+    
+    it "should know that it's running" do
+      MerbQueue::Queue.stubs(:next).returns(true)
+      
+      MerbQueue.register(:test) { |w| true }
+      MerbQueue[:test].should be_running
+    end
+    
+  end
+  
+  describe "looping" do
+    
+    before(:each) do
+      MerbQueue.delete(:test)
+    end
+    
+    it "should be processing the queue every so often and in order" do
+      $counter = 1
+      MerbQueue.add(:test) { |w| $counter = w[:amount] }
+      MerbQueue[:test].wait_seconds = 0
+      
+      MerbQueue[:test] << { :amount => 1 }
+      MerbQueue[:test] << { :amount => 2 }
+      MerbQueue[:test] << { :amount => 3 }
+      MerbQueue[:test] << { :amount => 4 }
+      
+      MerbQueue[:test].start
+      
+      sleep(1.25) # wait on the thread to process
+      
+      MerbQueue[:test].should_not be_working
+      $counter.should == 4
+    end
+    
+    it "should know how many items are left to go" do
+      MerbQueue.add(:test) { |w| true }
+      MerbQueue[:test] << {}
+      MerbQueue[:test] << {}
+      MerbQueue[:test].amount.should == 2
+    end
+  end
+  
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  gem 'rspec'
+  require 'spec'
+end
+
+$TESTING=true
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'rubygems'
+require 'mocha'
+require "merb-core"
+require 'merb_queue'
+
+Merb.load_dependencies :environment => "test"
+ 
+Spec::Runner.configure do |config|
+  # config.include(Merb::Test::ViewHelper)
+  config.include Merb::Test::RouteHelper
+  config.include Merb::Test::RequestHelper
+  config.include Merb::Test::ControllerHelper
+end
+ 
+Merb.load_dependencies(:environment => 'test')

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: myobie-merb_queue
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Nathan Herald
+autorequire: merb_queue
+bindir: bin
+cert_chain: []
+
+date: 2008-06-25 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: merb-core
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.3
+    version: 
+description: A persistent queue across multiple merb processes
+email: nathan@myobie.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+- LICENSE
+- TODO
+files: 
+- LICENSE
+- README.markdown
+- Rakefile
+- TODO
+- lib/merb_queue
+- lib/merb_queue/base.rb
+- lib/merb_queue/queue.rb
+- lib/merb_queue/version.rb
+- lib/merb_queue.rb
+- spec/merb_queue_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: false
+homepage: http://github.com/myobie/merb_queue
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A persistent queue across multiple merb processes
+test_files: []
+