matthewrudy-rudeq 0.1

data/README

@@ -0,0 +1,62 @@
+== Author
+  Matthew Rudy Jacobs
+ 
+== Contact
+  MatthewRudyJacobs@gmail.com
+ 
+RudeQ
+=============
+  A simple DB based queue,
+  designed for situations where a server based queue is unnecessary.
+ 
+ 
+INSTALL
+============
+This plugin requires Rails 2.* currently, and has only been tested on MySQL.
+It definitely does not work on SQLite (as it requires :limit and :order options for a SQL UPDATE command)
+
+On rails 2.1 you can install straight from github:
+  ruby script/plugin install git://github.com/matthewrudy/rudeq.git
+
+Else just check it out into your plugins directory:
+  git clone git://github.com/matthewrudy/rudeq.git vendor/plugins/rudeq
+ 
+USAGE
+============
+After you've installed it just run
+  rake queue:setup
+
+  matthew@iRudy:~/code/jbequeueing $ rake queue:setup
+  (in /Users/matthew/code/jbequeueing)
+      exists  app/models/
+      exists  spec/fixtures/
+      exists  spec/models/
+      create  app/models/rude_queue.rb
+      create  spec/fixtures/rude_queues.yml
+      create  spec/models/rude_queue_spec.rb
+      exists  db/migrate
+      create  db/migrate/029_create_rude_queues.rb
+
+  and you're done.
+  Fully tested, fully index... BOOM!
+
+  Now run migrations, start up a console, and;
+
+      RudeQueue.set(:queue_name, RandomObject)
+      RudeQueue.get(:queue_name)
+
+  And, to keep the queue running fast,
+  set up a cron job to run
+
+      rake queue:cleanup
+  
+  the cleanup will remove any queued items which have been processed longer than an hour ago.
+
+      rake queue:cleanup CLEANUP_TIME=86,400
+
+  will clear processed queue items processed longer than 86,400 seconds ago (1 day)
+
+Try Yourself!
+ 
+Copyright (c) 2008 [Matthew Rudy Jacobs Email: MatthewRudyJacobs@gmail.com],
+released under the MIT license

data/Rakefile

@@ -0,0 +1,22 @@
+require 'rake'
+require 'spec'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+desc 'Default: run the specs.'
+task :default => :spec
+
+desc 'Run specs for rude_q plugin'
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts = ['--options', "\"spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+desc 'Generate documentation for the rude_q plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'RudeQ'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/lib/rude_q.rb

@@ -0,0 +1,84 @@
+# RudeQ
+require 'digest/sha1'
+module RudeQ
+
+  def self.included(mod)
+    mod.extend(ClassMethods)
+    mod.serialize(:data)
+  end
+  
+  module ClassMethods
+    # Cleanup old processed items
+    #
+    #   RudeQueue.cleanup!
+    #   RudeQueue.cleanup!(1.week)
+    def cleanup!(expiry=1.hour)
+      self.delete_all(["processed = ? AND updated_at < ?", true, expiry.to_i.ago])
+    end
+  
+    # Add any serialize-able *data* to the queue *queue_name* (strings and symbols are treated the same)
+    #
+    #   RudeQueue.set(:sausage_queue, Sausage.new(:sauce => "yummy"))
+    #   RudeQueue.set("sausage_queue", Sausage.new(:other => true))
+    #
+    #     RudeQueue.get("sausage_queue")
+    #     -> *yummy sausage*
+    #     RudeQueue.get(:sausage_queue)
+    #     -> *other_sausage*
+    #     RudeQueue.get(:sausage_queue)
+    #     -> nil
+    def set(queue_name, data)
+      queue_name = sanitize_queue_name(queue_name)
+      self.create!(:queue_name => queue_name, :data => data)
+      return nil # in line with Starling
+    end
+
+    # Grab the first item from the queue *queue_name* (strings and symbols are treated the same)
+    # - it should always come out the same as it went in
+    # - they should always come out in the same order they went in
+    # - it will return a nil if there is no unprocessed entry in the queue
+    #
+    #  RudeQueue.get(21)
+    #  -> {:a => "hash"}
+    #  RudeQueue.get(:a_symbol)
+    #  -> 255
+    #  RudeQueue.get("a string")
+    #  -> nil
+    def get(queue_name)
+      qname = sanitize_queue_name(queue_name)
+      token = get_unique_token
+    
+      self.update_all(["token = ?", token], ["queue_name = ? AND processed = ? AND token IS NULL", qname, false], :limit => 1, :order => "id ASC")
+      queued = self.find_by_queue_name_and_token_and_processed(qname, token, false)
+      if queued
+        queued.update_attribute(:processed, true)
+        return queued.data
+      else
+        return nil # in line with Starling
+      end
+    end
+  
+    def get_unique_token # :nodoc:
+
+      digest = Digest::SHA1.new
+      digest << Time.now.to_s
+      digest << Process.pid.to_s
+      digest << Socket.gethostname
+      digest << self.token_count!.to_s # multiple requests from the same pid in the same second get different token
+    
+      return digest.hexdigest
+    end
+    
+    protected
+    
+    def token_count!
+      @token_count ||= 0
+      @token_count += 1
+      return @token_count
+    end
+    
+    def sanitize_queue_name(queue_name)
+      queue_name.to_s
+    end
+  end
+end

data/spec/database.yml

@@ -0,0 +1,4 @@
+rude_q_test:
+  adapter: mysql
+  username: rude_q
+  database: rude_q_test

data/spec/process_queue.rb

@@ -0,0 +1,3 @@
+class ProcessQueue < ActiveRecord::Base
+  include RudeQ
+end

data/spec/rude_q_spec.rb

@@ -0,0 +1,225 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe ProcessQueue do
+  before(:each) do
+    ProcessQueue.delete_all
+    create_some_noise
+  end
+  
+  def create_some_noise
+    ProcessQueue.create!(:queue_name => "doNT use this in Specs", :data => {:not => "to be messed with"})
+    ProcessQueue.create!(:queue_name => "abcde", :data => {:same_as => "the specs but already processed"}, :processed => true)
+    ProcessQueue.create!(:queue_name => "abcde", :data => {:same_as => "the specs but with token"}, :token => "    unlikely    ")
+  end
+  
+  describe "get and set" do
+    it "should work with strings" do
+      ProcessQueue.set('abcde', "Something to set")
+      ProcessQueue.get('abcde').should == "Something to set"
+    end
+    it "should work with symbols" do
+      ProcessQueue.set('abcde', :a_symbol)
+      ProcessQueue.get('abcde').should == :a_symbol
+    end
+    it "should work with arrays" do
+      array = [1, :b, "C"]
+      ProcessQueue.set('abcde', array)
+      ProcessQueue.get('abcde').should == array
+    end
+    it "should work with hashes" do
+      hash = {:symbol => "A string", "stringy" => 23, 74 => :cheese}
+      ProcessQueue.set('abcde', hash)
+      ProcessQueue.get('abcde').should == hash
+    end
+    
+    it "should :get in the same order they are :set" do
+      ProcessQueue.set('abcde', :first)
+      ProcessQueue.set('abcde', "second")
+      
+      ProcessQueue.get('abcde').should == :first
+      
+      ProcessQueue.set('abcde', 33.3333)
+      
+      ProcessQueue.get('abcde').should == "second"
+      ProcessQueue.get('abcde').should == 33.3333
+      ProcessQueue.get('abcde').should be(nil)
+    end
+    
+    it "should keep queues seperated" do
+      ProcessQueue.set('queue_1', :data_1)
+      ProcessQueue.set('queue_2', "DATA2")
+      
+      ProcessQueue.get('queue_2').should == "DATA2"
+      ProcessQueue.get('queue_2').should be(nil)
+      ProcessQueue.get('queue_1').should == :data_1
+      ProcessQueue.get('queue_1').should be(nil)
+    end
+    
+    it "should call to_s on inputs" do
+      qname = stub("fake input")
+      qname.should_receive(:to_s).exactly(:twice).and_return("fake queue name")
+      
+      ProcessQueue.set(qname, ["Data"])
+      ProcessQueue.get(qname).should == ["Data"]
+    end
+    
+    it "should work with queue name as strings or symbols" do
+      ProcessQueue.set(:bah, "something about bah")
+      ProcessQueue.get("bah").should == "something about bah"
+      
+      ProcessQueue.set("girah", {:craziness => "embodied"})
+      ProcessQueue.get(:girah).should == {:craziness => "embodied"}
+    end
+  end
+  
+  describe ".set" do
+    it "should delegate to :create!" do
+      ProcessQueue.should_receive(:create!).with(:queue_name => 'abcde', :data => :magical_planet)
+      ProcessQueue.set('abcde', :magical_planet)
+    end
+    it "should return nil" do
+      ProcessQueue.set('abcde', "something").should be(nil)
+    end
+  end
+  
+  describe ".get" do
+    it "should not return a processed item with the same token" do
+      @token = "tokEEEannn"
+      ProcessQueue.should_receive(:get_unique_token).exactly(3).times.and_return(@token)
+      @existing = ProcessQueue.create!(:queue_name => 'abcde', :data => :old_data, :token => @token, :processed => true)
+
+      ProcessQueue.get('abcde').should be(nil)
+        
+      ProcessQueue.set('abcde', :new_data)
+      ProcessQueue.get('abcde').should == :new_data
+      ProcessQueue.get('abcde').should be(nil)
+    end
+  end
+  
+  describe ".get_unique_token" do
+    it "should create a unique token" do
+      lots_of_tokens = Array.new(50) do
+        ProcessQueue.get_unique_token
+      end
+      lots_of_tokens.uniq.should == lots_of_tokens
+    end
+    
+    it "should create a unique token even if time stands still" do
+      time_now = Time.now
+      Time.should_receive(:now).at_least(50).times.and_return(time_now)
+      lots_of_tokens = Array.new(50) do
+        ProcessQueue.get_unique_token
+      end
+      lots_of_tokens.uniq.should == lots_of_tokens
+    end
+  end
+  
+  describe ".cleanup!" do
+    it "should use :delete_all" do
+      ProcessQueue.should_receive(:delete_all) # not :destroy_all
+      ProcessQueue.cleanup!
+    end
+    
+    it "should allow string inputs" do
+      ProcessQueue.cleanup!("3600")
+    end
+    
+    it "should allow integer inputs" do
+      ProcessQueue.cleanup!(3600)
+    end
+    
+    it "should not clear unprocessed items" do
+      ProcessQueue.set('abcde', :giraffe)
+      ProcessQueue.set('abcde', :monkey)
+      ProcessQueue.count.should >= 2
+      
+      ProcessQueue.cleanup!
+      
+      ProcessQueue.count.should >=2
+      ProcessQueue.get('abcde').should == :giraffe
+    end
+    
+    it "should not clear old unprocessed items" do
+      ProcessQueue.set('abcde', :giraffe)
+      giraffe = ProcessQueue.find(:first, :conditions => {:data => :giraffe})
+      
+      time_now = Time.now
+      Time.stub!(:now).and_return(time_now + 1.year)
+      
+      giraffe.updated_at.should < 2.weeks.ago
+      
+      ProcessQueue.cleanup!
+      
+      giraffe.reload
+      ProcessQueue.get('abcde').should == :giraffe
+    end
+    
+    it "should not clear processed items newer than the argument" do
+      ProcessQueue.set('abcde', :giraffe)
+      ProcessQueue.get('abcde').should == :giraffe
+      
+      giraffe = ProcessQueue.find(:first, :conditions => {:data => :giraffe})
+      
+      time_now = Time.now
+      Time.stub!(:now).and_return(time_now + 1.week - 5.minutes)
+      
+      giraffe.updated_at.should > 1.week.ago
+      giraffe.processed.should be(true)
+      
+      ProcessQueue.cleanup!(1.week)
+      
+      giraffe.reload
+    end
+    
+    it "should not clear processed items newer than one hour, by default" do
+      ProcessQueue.set('abcde', :giraffe)
+      ProcessQueue.get('abcde').should == :giraffe
+      
+      giraffe = ProcessQueue.find(:first, :conditions => {:data => :giraffe})
+      
+      time_now = Time.now
+      Time.stub!(:now).and_return(time_now + 59.minutes)
+      
+      giraffe.updated_at.should > 1.hour.ago
+      giraffe.processed.should be(true)
+      
+      ProcessQueue.cleanup!()
+      
+      giraffe.reload
+    end
+    
+    it "should clear processed items older than the argument" do
+      ProcessQueue.set('abcde', :giraffe)
+      ProcessQueue.get('abcde').should == :giraffe
+      
+      giraffe = ProcessQueue.find(:first, :conditions => {:data => :giraffe})
+      
+      time_now = Time.now
+      Time.stub!(:now).and_return(time_now + 1.week + 5.minutes)
+      
+      giraffe.updated_at.should < 1.week.ago
+      giraffe.processed.should be(true)
+      
+      ProcessQueue.cleanup!(1.week)
+      
+      lambda { giraffe.reload }.should raise_error(ActiveRecord::RecordNotFound)
+    end
+    
+    it "should clear processed items older than one hour, by default" do
+      ProcessQueue.set('abcde', :giraffe)
+      ProcessQueue.get('abcde').should == :giraffe
+      
+      giraffe = ProcessQueue.find(:first, :conditions => {:data => :giraffe})
+      
+      time_now = Time.now()
+      Time.stub!(:now).and_return(time_now + 61.minutes)
+      
+      giraffe.updated_at.should < 1.hour.ago
+      giraffe.processed.should be(true)
+      
+      ProcessQueue.cleanup!
+      
+      lambda { giraffe.reload }.should raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+end

data/spec/schema.rb

@@ -0,0 +1,12 @@
+ActiveRecord::Schema.define(:version => 1) do
+  create_table :process_queues, :force => true do |t|
+    t.string :queue_name
+    t.text :data
+    t.string :token, :default => nil
+    t.boolean :processed, :default => false, :null => false
+
+    t.timestamps
+  end
+  add_index :process_queues, :processed
+  add_index :process_queues, [:queue_name, :processed]
+end

data/spec/spec.opts

@@ -0,0 +1,6 @@
+--colour
+--format
+progress
+--loadby
+mtime
+--reverse

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'spec'
+require 'active_record'
+
+current_dir = File.dirname(__FILE__)
+require "#{current_dir}/../lib/rude_q"
+require "#{current_dir}/process_queue"
+ 
+config = YAML::load(IO.read(current_dir + '/database.yml'))
+ActiveRecord::Base.logger = Logger.new(current_dir + "/debug.log")
+ActiveRecord::Base.establish_connection(config['rude_q_test'])
+load(current_dir + "/schema.rb")

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification 
+name: matthewrudy-rudeq
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- Matthew Rudy Jacobs
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-06-21 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: A simple DB queueing library built on top of ActiveRecord, and designed for use with MySQL.
+email: MatthewRudyJacobs@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- README
+- Rakefile
+- lib/rude_q.rb
+- spec/database.yml
+- spec/process_queue.rb
+- spec/rude_q_spec.rb
+- spec/schema.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/matthewrudy/rudeq
+post_install_message: 
+rdoc_options: 
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: ActiveRecord-based DB-queue
+test_files: 
+- spec/rude_q_spec.rb