pgq 0.1

data/.gitignore

@@ -0,0 +1,4 @@
+TODO
+*.gem
+lib/serv
+README2

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    pgq (0.1)
+      activerecord (>= 2.3.2)
+      activesupport (>= 2.3.2)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.2.3)
+      activesupport (= 3.2.3)
+      builder (~> 3.0.0)
+    activerecord (3.2.3)
+      activemodel (= 3.2.3)
+      activesupport (= 3.2.3)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activesupport (3.2.3)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    builder (3.0.0)
+    diff-lcs (1.1.3)
+    i18n (0.6.0)
+    multi_json (1.3.4)
+    rake (0.9.2.2)
+    rspec (2.10.0)
+      rspec-core (~> 2.10.0)
+      rspec-expectations (~> 2.10.0)
+      rspec-mocks (~> 2.10.0)
+    rspec-core (2.10.0)
+    rspec-expectations (2.10.0)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.10.1)
+    tzinfo (0.3.33)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pgq!
+  rake
+  rspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Makarchev K
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,207 @@
+Pgq Rails
+=========
+
+Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
+ 
+About PgQ
+*  http://wiki.postgresql.org/wiki/SkyTools#PgQ
+
+
+Install
+-------
+
+Install skytools:
+Ubuntu 11.10: 
+
+    # apt-get install postgresql-server postgresql-client
+    # apt-get install skytools
+
+Gemfile:
+
+```ruby
+gem 'pgq'
+```
+
+Create ticker configs:
+
+    $ rails generate pgq:config
+    edit config: config/pgq_development.ini
+
+
+Install pgq to database (if test database recreates all the time, should reinstall pgq each time):
+
+    $ rake pgq:install
+    or execute
+    $ pgqadm config/pgq_development.ini install
+
+   
+
+Run ticker daemon (ticker needs on production database, or development if we test the process of consuming):
+Daemon run once, bind to the database. (If worker not consuming check that daemon started).
+
+    $ rake pgq:ticker:start (stop)
+    or execute
+    $ pgqadm config/pgq_development.ini ticker -d
+    
+
+Last, add to config/application.rb
+
+    config.autoload_paths += %W( #{config.root}/app/models/pgq )
+    
+
+Create Pgq consumer
+-------------------
+
+    $ rails generate pgq:add my
+
+
+This creates file app/models/pgq/pgq_my.rb and migration
+
+    $ rake db:migrate
+
+
+```ruby
+class PgqMy < Pgq::Consumer
+
+  def some_method1(a, b, c)
+    logger.info "async call some_method1 with #{[a, b, c].inspect}"
+  end
+  
+  def some_method2(x)
+    logger.info "async called some_method2 with #{x.inspect}"
+  end
+  
+end
+```
+
+Insert event into queue like this:
+
+    PgqMy.some_method1(1, 2, 3)
+    
+    or
+        
+    PgqMy.add_event(:some_method2, some_x)
+            
+    or
+    
+    PgqMy.enqueue(:some_method1, 1, 2, 3)
+  
+
+Workers
+-------
+Start worker for queue:
+
+    $ rake pgq:worker QUEUES="my"
+    $ rake pgq:worker QUEUES="my,mailer,other"
+    $ rake pgq:worker QUEUES="all" RAILS_ENV=production
+    
+
+
+Also can consume manual, or write [bin_script](http://github.com/kostya/bin_script) like this:
+```ruby
+class PgqRunnerScript < BinScript
+
+  self.enable_locking = false
+
+  required :q, "queues separated by ','"
+  required :w, "watch file"
+  
+  def do!
+    worker = Pgq::Worker.new(:logger => self.logger, :queues => params(:q), 
+                             :watch_file => params(:w) || "./tmp/stop_all.txt")
+    worker.run
+  end
+                                                                     
+end
+```
+
+and run:
+
+    $ ./bin/pgq_runner.rb -q my -l ./log/pgq_my.log
+    $ ./bin/pgq_runner.rb -q all -l ./log/pgq_all.log   # this will consume all queues from config/queues_list.yml
+
+
+
+
+### Admin interface
+Admins interface is possible like Resque, but not realized yet.
+For queues info, run in console:
+
+    > Pgq::Consumer.database.pgq_get_consumer_info
+
+
+### Failed events
+When any raise happens in consumer, its produce failed event, which can be retry, or delete.
+
+Retry manual:
+
+    Pgq::Consumer.resend_failed_events(queue_name)
+
+Delete manual:
+    
+    Pgq::Consumer.clear_failed_events(queue_name)
+
+
+### Divide events between workers, for one consumer class
+create more queues: my_2, my_3
+
+```ruby
+class PgqMy < Pgq::Consumer
+
+  QUEUES = %w(my my_2 my_3).freeze
+
+  def self.next_queue_name
+    QUEUES.choice
+  end
+  
+end
+```
+And run 3 workers: for my, my_2, my_3
+
+
+
+
+### Consume group of events extracted from pgq(batch): 
+Usually batch size is ~500 events.
+
+```ruby
+class PgqMy < Pgq::ConsumerGroup
+
+  # {'type' => [events]}
+  def perform_group(events_hash)
+    raise "realize me"
+  end
+  
+end
+```
+
+
+
+### Options
+create initializer with:
+
+```ruby
+class Pgq::Consumer
+
+  # specify database with pgq queues
+  def self.database
+    SomeDatabase 
+  end
+
+  # specify coder  
+  def self.coder
+    Pgq::Marshal64Coder
+  end
+    
+end
+```
+
+### Proxy method to consumer
+Usefull in specs
+
+``` ruby
+  PgqMy.proxy(:some_method1)
+```
+ 
+When code call PgqMy.some_method1(a,b,c) this would be convert into PgqMy.new.some_method1(a,b,c)
+  

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+
+task :default => :spec
+RSpec::Core::RakeTask.new(:spec)

data/init.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/lib/pgq'

data/lib/generators/pgq/add_generator.rb

@@ -0,0 +1,40 @@
+module Pgq
+  class AddGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+    source_root File.expand_path("../add_templates", __FILE__)
+    argument :queue_name, :type => :string
+    
+    def self.next_migration_number(path)
+      Time.now.utc.strftime("%Y%m%d%H%M%S")
+    end    
+
+    def add_to_queues_list
+      filename = Rails.root + 'config/queues_list.yml'
+      if File.exists?(filename)
+        x = YAML.load_file(filename)
+        x << name
+        File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
+      else
+        x = [name]
+        File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
+      end
+    end
+    
+    def add_files
+      template "pgq_class.rb", "app/models/pgq/pgq_#{name}.rb"
+      template "spec.rb", "spec/models/pgq/pgq_#{name}_spec.rb"
+      migration_template "migration.rb", "db/migrate/create_#{name}_queue.rb"
+    end
+    
+  private
+  
+    def name 
+      queue_name.underscore
+    end
+    
+    def name_c
+      name.classify
+    end
+  end
+  
+end

data/lib/generators/pgq/add_templates/migration.rb

@@ -0,0 +1,9 @@
+class Create<%= name_c %>Queue < ActiveRecord::Migration
+  def self.up
+    Pgq::Consumer.add_queue :<%= name %>
+  end
+
+  def self.down
+    Pgq::Consumer.remove_queue :<%= name %>
+  end
+end

data/lib/generators/pgq/add_templates/pgq_class.rb

@@ -0,0 +1,10 @@
+class Pgq<%= name_c %> < Pgq::Consumer
+
+  # to insert event: Pgq<%= name_c %>.my_event(1, 2, 3)
+  
+  # async execute
+  def my_event(a, b, c)
+    logger.info "async call my_event with #{[a, b, c].inspect}"
+  end
+  
+end

data/lib/generators/pgq/add_templates/spec.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe Pgq<%= name_c %> do
+  before(:each) do
+    @consumer = Pgq<%= name_c %>.new
+  end
+  
+end

data/lib/generators/pgq/config_generator.rb

@@ -0,0 +1,18 @@
+module Pgq
+  class ConfigGenerator < Rails::Generators::Base
+    source_root File.expand_path("../config_templates", __FILE__)
+    
+    def copy_configs
+      self.env_name = 'development'
+      template "pgq.rb", "config/pgq_development.ini"
+      self.env_name = 'test'
+      template "pgq.rb", "config/pgq_test.ini"
+      self.env_name = 'production'
+      template "pgq.rb", "config/pgq_production.ini"
+    end
+    
+  private  
+    attr_accessor :env_name
+    
+  end
+end

data/lib/generators/pgq/config_templates/pgq.rb

@@ -0,0 +1,12 @@
+[pgqadm]
+job_name = pgqadm_<%= env_name %>
+db = dbname=dbname user=user password=password host=127.0.0.1 port=5432
+
+# how often to run maintenance [seconds]
+maint_delay = 600
+
+# how often to check for activity [seconds]
+loop_delay = 0.1
+
+logfile = log/%(job_name)s.log
+pidfile = tmp/%(job_name)s.pid

data/lib/pgq.rb

@@ -0,0 +1,9 @@
+require 'active_support'
+require 'active_record'
+
+module Pgq
+end
+
+Dir[File.dirname(__FILE__) + "/pgq/*.rb"].each{|f| require f }
+
+ActiveRecord::Base.extend(Pgq::Api) if defined?(ActiveRecord::Base)

data/lib/pgq/api.rb

@@ -0,0 +1,88 @@
+module Pgq::Api
+  # should mixin to class, which have connection
+
+  def pgq_create_queue(queue_name)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.create_queue(?)", queue_name]).to_i
+  end
+
+  def pgq_drop_queue(queue_name)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.drop_queue(?)", queue_name]).to_i
+  end
+
+  def pgq_register_consumer(queue_name, consumer_id)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.register_consumer(?, ?)", queue_name, consumer_id]).to_i
+  end
+
+  def pgq_unregister_consumer(queue_name, consumer_id)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.unregister_consumer(?, ?)", queue_name, consumer_id]).to_i
+  end
+
+  def pgq_add_queue(queue_name, consumer_name)
+    pgq_create_queue(queue_name.to_s)
+    pgq_register_consumer(queue_name.to_s, consumer_name.to_s)
+  end
+
+  def pgq_remove_queue(queue_name, consumer_name)
+    pgq_unregister_consumer(queue_name.to_s, consumer_name.to_s)
+    pgq_drop_queue(queue_name.to_s)
+  end
+
+  def pgq_insert_event(queue_name, ev_type, ev_data, ev_extra1 = nil, ev_extra2 = nil, ev_extra3 = nil, ev_extra4 = nil)
+    result = connection.select_value(sanitize_sql_array ["SELECT pgq.insert_event(?, ?, ?, ?, ?, ?, ?)", 
+                                                         queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4])
+    result ? result.to_i : nil
+  end
+  
+  def pgq_next_batch(queue_name, consumer_id)
+    result = connection.select_value(sanitize_sql_array ["SELECT pgq.next_batch(?, ?)", queue_name, consumer_id])
+    result ? result.to_i : nil
+  end
+  
+  def pgq_get_batch_events(batch_id)
+    connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.get_batch_events(?)", batch_id])
+  end
+  
+  def pgq_event_failed(batch_id, event_id, reason)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.event_failed(?, ?, ?)", batch_id, event_id, reason]).to_i
+  end
+  
+  def pgq_event_retry(batch_id, event_id, retry_seconds)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.event_retry(?, ?, ?)", batch_id, event_id, retry_seconds]).to_i
+  end
+  
+  def pgq_finish_batch(batch_id)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.finish_batch(?)", batch_id])
+  end
+
+  def pgq_get_queue_info(queue_name)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.get_queue_info(?)", queue_name])
+  end
+
+  def pgq_failed_event_retry(queue_name, consumer, event_id)
+    connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_retry(?, ?, ?)", queue_name, consumer, event_id])
+  end
+      
+  def pgq_failed_event_delete(queue_name, consumer, event_id)
+    connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_delete(?, ?, ?)", queue_name, consumer, event_id])
+  end
+     
+  def pgq_failed_events_count(queue_name, consumer)
+    res = connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_count(?, ?)", queue_name, consumer])
+    res ? res.to_i : nil
+  end
+  
+  def pgq_failed_event_list queue_name, consumer, limit = nil, offset = nil, order = 'desc'
+    order = (order.to_s == 'desc') ? order : 'asc'
+    connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) order by ev_id #{order}", queue_name, consumer, limit.to_i, offset.to_i])
+  end
+
+  # queue lag in seconds
+  def pgq_queue_lag(queue_name)
+    connection.select_value(sanitize_sql_array ["SELECT Max(EXTRACT(epoch FROM lag)) FROM pgq.get_consumer_info() where queue_name = ?", queue_name]).to_f
+  end
+  
+  def pgq_get_consumer_info
+    connection.select_all("SELECT * FROM pgq.get_consumer_info()")
+  end
+  
+end