pgq 0.1

data/lib/tasks/pgq.rake

@@ -0,0 +1,64 @@
+namespace :pgq do
+
+  desc "Start worker params: QUEUES, LOGGER, WATCH_FILE"
+  task :worker => [:environment] do 
+    queues = ENV['QUEUES']
+    logger_file = ENV['LOGGER'] || Rails.root.join("log/pgq_#{queues}.log")
+    watch_file = ENV['WATCH_FILE'] || Rails.root.join("tmp/pgq_#{queues}.stop")
+    logger = Logger.new(logger_file)
+    w = Pgq::Worker.new(:logger => logger, :queues => queues, :watch_file => watch_file)
+    w.run
+  end
+
+  desc "Install PgQ to database RAILS_ENV"
+  task :install do
+    Dir["#{Rails.root}/config/pgq_#{Rails.env}*.ini"].each do |conf|
+      ENV['PGQFILE']=conf
+      Rake::Task['pgq:install_from_file'].invoke
+    end
+  end
+
+  desc "Install PgQ from file ENV['PGQFILE'] to database RAILS_ENV"
+  task :install_from_file do
+    puts "No file specified" and return if !ENV['PGQFILE'] || ENV['PGQFILE'].empty?
+
+    conf = ENV['PGQFILE']
+    
+    puts "installing pgq, running: pgqadm.py #{conf} install"
+
+    output = `which pgqadm.py && pgqadm.py #{conf} install 2>&1 || which pgqadm && pgqadm #{conf} install 2>&1`
+    puts output
+    if output =~ /pgq is installed/ || output =~ /Reading from.*?pgq.sql$/
+      puts "PgQ installed successfully"
+    else
+      raise "Something went wrong(see above)... Check that you install skytools package and create #{conf}"
+    end
+  end
+  
+  
+  namespace :ticker do
+  
+    desc "Start PgQ ticker daemon"
+    task :start do
+      conf = Rails.root.join("config/pgq_#{Rails.env}.ini")
+      output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
+      if output.empty?
+        puts "ticker daemon started"
+      else
+        puts output
+      end
+    end
+  
+    desc "Stop PgQ ticker daemon"
+    task :stop do
+      conf = Rails.root.join("config/pgq_#{Rails.env}.ini")
+      output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
+      if output.empty?
+        puts "ticker daemon stoped"
+      else
+        puts output
+      end
+    end
+  
+  end
+end

data/pgq.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+
+require File.dirname(__FILE__) + "/lib/pgq/version"
+
+Gem::Specification.new do |s|
+  s.name = %q{pgq}
+  s.version = Pgq::VERSION
+
+  s.authors = ["Makarchev Konstantin"]
+  s.autorequire = %q{init}
+  
+  s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.}
+  s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.}
+  
+  s.email = %q{kostya27@gmail.com}
+  s.homepage = %q{http://github.com/kostya/pgq}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'activesupport', ">=2.3.2"  
+  s.add_dependency 'activerecord', ">=2.3.2"
+  
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+end

data/spec/consumer_base_spec.rb

@@ -0,0 +1,149 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class PgqTata < Pgq::ConsumerBase
+end
+
+class PgqTata2 < Pgq::ConsumerBase
+  @queue_name = "huhu"
+end
+
+class PgqTata3 < Pgq::ConsumerBase
+  set_queue_name 'hehu'  
+
+  def self.next_queue_name
+    'tutturu'
+  end
+  
+  def perform(name, a, b)
+    $a = a
+    $b = b
+    10
+  end
+ 
+end
+
+describe Pgq::ConsumerBase do
+  before :each do
+    @consumer = PgqTata.new
+    @consumer2 = PgqTata2.new
+    @consumer3 = PgqTata3.new
+
+    @coder = @consumer.coder
+  end
+
+  it "queue_name" do
+    PgqTata.queue_name.should == 'tata'
+    @consumer.queue_name.should == 'tata'
+
+    PgqTata2.queue_name.should == 'huhu'
+    @consumer2.queue_name.should == 'huhu'
+
+    PgqTata3.queue_name.should == 'hehu'
+    @consumer3.queue_name.should == 'hehu'
+  end
+
+  it "default encode/decode" do
+    data = {:a => [1, 2, 3, {:b => 'c'}]}
+    s = @coder.dump(data)
+    @coder.load(s).should == data
+  end
+
+  it "enqueue all cases" do
+    PgqTata.database.should_receive(:pgq_insert_event).with('tata', 'method1', @coder.dump([1,2,3]))
+    PgqTata.enqueue(:method1, 1, 2, 3)
+
+    PgqTata2.database.should_receive(:pgq_insert_event).with('huhu', 'method1', @coder.dump([[1,2,3]]))
+    PgqTata2.enqueue(:method1, [1, 2, 3]) 
+
+    PgqTata3.database.should_receive(:pgq_insert_event).with('tutturu', 'method1', @coder.dump([]))
+    PgqTata3.enqueue(:method1) 
+  end
+
+
+  describe "consuming" do
+    before :each do
+      @data = [1, 2, 3]
+      @event = {'ev_type' => 'bla', 'ev_data' => @coder.dump(@data)}
+    end
+
+    it "perform_batch should_receive perform_events" do
+      @consumer.should_receive(:get_batch_events).and_return([@event])
+      @consumer.should_not_receive(:all_events_failed)
+      @consumer.should_receive(:perform_events) do |events|
+        events.size.should == 1
+        ev = events.first
+        ev.type.should == 'bla'
+        ev.data.should == @data
+      end
+      @consumer.should_receive(:finish_batch).with(1)
+      @consumer.perform_batch.should == 1
+    end
+
+    it "perform_batch raises" do
+      @consumer.should_receive(:get_batch_events).and_return([@event])
+      @consumer.should_receive(:perform_events).and_raise(:wow)
+      @consumer.should_receive(:all_events_failed)
+      @consumer.should_receive(:finish_batch).with(1)
+      @consumer.perform_batch.should == 1
+    end
+
+    it "perform_batch empty" do
+      @consumer.should_receive(:get_batch_events).and_return([])
+      @consumer.should_not_receive(:all_events_failed)
+      @consumer.should_not_receive(:perform_events)
+      @consumer.should_receive(:finish_batch).with(0)
+      @consumer.perform_batch.should == 0
+    end
+
+  end
+
+  describe "actions with events" do
+    before :each do
+      @data = [1, 2, 3]
+      @event = Pgq::Event.new(@consumer, {'ev_type' => 'bla', 'ev_data' => @coder.dump(@data)})
+      @events = [@event]      
+    end
+
+    it "all_events_failed" do
+      @event.should_receive(:failed!).with(an_instance_of(String))
+      @consumer.all_events_failed(@events, Exception.new('wow'))
+    end
+
+    it "perform_events" do
+      @consumer.should_receive(:perform_event).with(@event)
+      @consumer.perform_events(@events)
+    end
+
+    it "perform_event ok" do
+      @consumer.should_receive(:perform).with('bla', *@data)
+      @consumer.perform_event(@event)
+    end
+
+    it "perform_event raised" do
+      @consumer.should_receive(:perform).with('bla', *@data).and_throw(:wow)
+      @event.should_receive(:failed!).with(an_instance_of(String))
+      @consumer.perform_event(@event)
+    end
+
+  end
+
+  describe "migration" do
+    it "up" do
+      Pgq::ConsumerBase.database.should_receive(:pgq_add_queue).with('super', Pgq::ConsumerBase.consumer_name)
+      Pgq::ConsumerBase.add_queue("super")
+    end
+
+    it "down" do
+      Pgq::ConsumerBase.database.should_receive(:pgq_remove_queue).with('super', Pgq::ConsumerBase.consumer_name)
+      Pgq::ConsumerBase.remove_queue("super")
+    end
+  end
+  
+  it "should proxy consumer" do
+    PgqTata3.proxy(:ptest)
+    PgqTata3.ptest(111, 'abc').should == 10
+    $a.should == 111
+    $b.should == 'abc'
+  end
+
+end

data/spec/consumer_group_spec.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class PgqGr < Pgq::ConsumerGroup
+  def perform_group opts; end
+  def hahah; end
+  def bla; end
+end
+
+describe Pgq::ConsumerGroup do
+  before :each do
+    @consumer = PgqGr.new
+    @consumer.stub!(:get_next_batch).and_return ''
+    @consumer.stub!(:finish_batch).and_return ''
+    @coder = @consumer.coder
+  end
+
+  it "queue_name" do
+    PgqGr.queue_name.should == 'gr'
+  end
+
+  it "should call consume" do
+    @consumer.should_receive(:perform_group) do |h|
+      ev1 = h['hahah']
+      ev1.size.should == 2
+      ev1.first.data.should == '1'
+      ev1.second.data.should == '3'
+
+      ev2 = h['bla']
+      ev2.size.should == 1
+      ev2.first.data.should == '2'
+    end
+    @consumer.should_receive(:get_batch_events).and_return([{'ev_type' => 'hahah', 'ev_data' => @coder.dump('1')}, 
+      {'ev_type' => 'bla', 'ev_data' => @coder.dump('2')}, {'ev_type' => 'hahah', 'ev_data' => @coder.dump('3')}])
+    @consumer.perform_batch
+  end
+
+end

data/spec/consumer_spec.rb

@@ -0,0 +1,53 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class PgqHaha < Pgq::Consumer
+
+  def ptest2(a, b)
+    $a = a
+    $b = b
+    10
+  end
+
+end
+
+describe PgqHaha do
+  before :each do
+    @consumer = PgqHaha.new
+    @coder = @consumer.coder
+    @data = [1,2,[3,4], nil]
+
+    @consumer.stub!(:get_next_batch).and_return ''
+    @consumer.stub!(:finish_batch).and_return ''
+  end
+
+  it "queue name" do
+    @consumer.queue_name.should == 'haha'
+  end
+
+  it "missing method should insert event" do
+    PgqHaha.should_receive(:enqueue).with(:bla, 1,2,[3,4], nil)
+    PgqHaha.bla 1, 2, [3, 4], nil
+  end
+
+  it "should enqueue with add_event" do
+    PgqHaha.should_receive(:enqueue).with(:bla, 1,2,[3,4], nil)
+    PgqHaha.add_event :bla, 1, 2, [3, 4], nil
+  end
+
+  it "magick perform" do
+    @consumer.should_receive(:bla) do |*h|
+      h.should == @data
+    end
+
+    @consumer.should_receive(:get_batch_events).and_return([{'ev_type' => 'bla', 'ev_data' => @coder.dump(@data)}])
+    @consumer.perform_batch
+  end
+  
+  it "should proxy consumer" do
+    PgqHaha.proxy(:ptest2)
+    PgqHaha.ptest2(111, 'abc').should == 10
+    $a.should == 111
+    $b.should == 'abc'
+  end
+  
+end

data/spec/event_spec.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Pgq::Event do
+  before :each do
+    kl = Class.new(Pgq::ConsumerGroup) do
+    end
+    @consumer = kl.new
+    @consumer.stub!(:get_next_batch).and_return ''
+    @consumer.stub!(:finish_batch).and_return ''
+    @coder = @consumer.coder
+
+    @ev = Pgq::Event.new(@consumer, {'ev_type' => 'haha', 'ev_data' => @coder.dump('aaaaaaa'), 'ev_id' => 123})
+  end
+
+  it "parse data" do
+    @ev.type.should == 'haha'
+    @ev.data.should == 'aaaaaaa'
+    @ev.id.should == 123
+    @ev.consumer.should == @consumer
+  end
+
+  it "should failed!" do
+    @consumer.should_receive(:event_failed).with(123, an_instance_of(String))
+    @ev.failed!
+  end
+
+  it "should retry!" do
+    @consumer.should_receive(:event_retry).with(123)    
+    @ev.retry!
+  end
+ 
+end

data/spec/marshal64_coder_spec.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Pgq::Marshal64Coder do
+  before :each do
+    @coder = Pgq::Marshal64Coder
+  end
+
+  it "work" do
+    data = {:a => [1, 2, 3, {:b => 'c'}]}
+    s = @coder.dump(data)
+    @coder.load(s).should == data
+  end
+
+  it "work on large data" do
+    data = {}
+    10000.times{|i| data["#{i}".to_sym] = [i, "i_#{i}"]}
+    
+    s = @coder.dump(data)
+    @coder.load(s).should == data
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+require 'rubygems'
+require "bundler/setup"
+
+Bundler.require 
+
+class Pgq::Consumer
+    # rspec fuckup
+  def self.to_ary 
+  end
+end
+

data/spec/worker_spec.rb

@@ -0,0 +1,53 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Pgq::Worker do
+  class PgqBla < Pgq::Consumer
+  end
+
+  it "should find class not in hash" do
+    Pgq::Worker.predict_queue_class('bla').should == PgqBla
+    Pgq::Worker.predict_queue_class('bla_1').should == PgqBla
+    Pgq::Worker.predict_queue_class('bla_2').should == PgqBla
+    Pgq::Worker.predict_queue_class('bla3').should == PgqBla
+    Pgq::Worker.predict_queue_class('bla_').should == PgqBla
+    Pgq::Worker.predict_queue_class('pgq_bla_22').should == PgqBla
+
+    Pgq::Worker.predict_queue_class('blah').should == nil
+    Pgq::Worker.predict_queue_class('').should == nil
+    Pgq::Worker.predict_queue_class(nil).should == nil
+  end
+
+  def process_batch
+    processed_count = 0
+
+    @consumers.each do |consumer|
+      processed_count += consumer.perform_batch
+
+      if @watch_file && File.exists?(@watch_file)
+        logger.info "Found file #{@watch_file}, now exiting"
+        File.unlink(@watch_file)
+        return
+      end
+    end
+
+    processed_count
+  end  
+
+  it "initialize" do
+    @w = Pgq::Worker.new :queues => ['bla']
+
+    @w.consumers.size.should == 1
+    cons = @w.consumers.first
+    cons.class.should == PgqBla
+    cons.queue_name.should == 'bla'
+  end
+
+  it "process_batch" do
+    @w = Pgq::Worker.new :queues => ['bla']
+    cons = @w.consumers.first
+
+    cons.should_receive(:perform_batch).and_return(10)
+    @w.process_batch.should == 10
+  end
+
+end