pgq 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore CHANGED
@@ -1,4 +1,8 @@
1
1
  TODO
2
2
  *.gem
3
3
  lib/serv
4
- README2
4
+ README2
5
+ Gemfile.lock
6
+ pgq_test.ini
7
+ log/
8
+ tmp/
data/README.md CHANGED
@@ -1,7 +1,7 @@
1
- Pgq Rails
2
- =========
1
+ Pgq
2
+ ===
3
3
 
4
- Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
4
+ Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
5
5
 
6
6
  About PgQ
7
7
  * http://wiki.postgresql.org/wiki/SkyTools#PgQ
@@ -22,9 +22,9 @@ Gemfile:
22
22
  gem 'pgq'
23
23
  ```
24
24
 
25
- Create ticker configs:
25
+ Create ticker configs from database.yml:
26
26
 
27
- $ rails generate pgq:config
27
+ $ rake pgq:generate_configs
28
28
  edit config: config/pgq_development.ini
29
29
 
30
30
 
@@ -46,7 +46,7 @@ Daemon run once, bind to the database. (If worker not consuming check that daemo
46
46
 
47
47
  Last, add to config/application.rb
48
48
 
49
- config.autoload_paths += %W( #{config.root}/app/models/pgq )
49
+ config.autoload_paths += %W( #{config.root}/app/workers )
50
50
 
51
51
 
52
52
  Create Pgq consumer
@@ -55,7 +55,7 @@ Create Pgq consumer
55
55
  $ rails generate pgq:add my
56
56
 
57
57
 
58
- This creates file app/models/pgq/pgq_my.rb and migration
58
+ This creates file app/workers/pgq_my.rb and migration
59
59
 
60
60
  $ rake db:migrate
61
61
 
@@ -174,7 +174,7 @@ class Pgq::Consumer
174
174
 
175
175
  # specify coder
176
176
  def self.coder
177
- Pgq::Marshal64Coder
177
+ Marshal64 # class/module with self methods dump/load
178
178
  end
179
179
 
180
180
  end
@@ -188,4 +188,17 @@ Usefull in specs
188
188
  ```
189
189
 
190
190
  When code call PgqMy.some_method1(a,b,c) this would be convert into PgqMy.new.some_method1(a,b,c)
191
-
191
+
192
+
193
+ ### for Rails 2.3
194
+
195
+ Add to Rakefile:
196
+ ```
197
+ load 'tasks/pgq.rake'
198
+ ```
199
+
200
+ Copy generators from gem to lib/generators.
201
+
202
+ ```
203
+ ./script/generate pgq bla
204
+ ```
data/Rakefile CHANGED
@@ -3,7 +3,24 @@ require 'bundler/setup'
3
3
  require 'bundler'
4
4
  Bundler::GemHelper.install_tasks
5
5
 
6
+ load 'lib/tasks/pgq.rake'
7
+
6
8
  require 'rspec/core/rake_task'
7
9
 
8
10
  task :default => :spec
9
- RSpec::Core::RakeTask.new(:spec)
11
+ RSpec::Core::RakeTask.new(:spec)
12
+
13
+ desc "set_env"
14
+ task :set_env do
15
+ ENV["PGQFILE"] = File.join(File.dirname(__FILE__), %w{ spec app config pgq_test.ini })
16
+ ENV["RAILS_ROOT"] = File.join(File.dirname(__FILE__), %w{ spec app })
17
+ ENV["RAILS_ENV"] = "test"
18
+ end
19
+
20
+ desc "prepare test"
21
+ task :prepare_test => :set_env do
22
+ Rake::Task["pgq:generate_config"].invoke
23
+ Rake::Task["pgq:install"].invoke
24
+ end
25
+
26
+ task :spec => :prepare_test
@@ -1,8 +1,10 @@
1
+ # for Rails 3
2
+ if defined?(Rails) && Rails::VERSION::MAJOR >= 3
3
+
1
4
  module Pgq
2
- class AddGenerator < Rails::Generators::Base
5
+ class AddGenerator < Rails::Generators::NamedBase
3
6
  include Rails::Generators::Migration
4
- source_root File.expand_path("../add_templates", __FILE__)
5
- argument :queue_name, :type => :string
7
+ source_root File.expand_path("../templates", __FILE__)
6
8
 
7
9
  def self.next_migration_number(path)
8
10
  Time.now.utc.strftime("%Y%m%d%H%M%S")
@@ -21,20 +23,12 @@ module Pgq
21
23
  end
22
24
 
23
25
  def add_files
24
- template "pgq_class.rb", "app/models/pgq/pgq_#{name}.rb"
25
- template "spec.rb", "spec/models/pgq/pgq_#{name}_spec.rb"
26
- migration_template "migration.rb", "db/migrate/create_#{name}_queue.rb"
27
- end
28
-
29
- private
30
-
31
- def name
32
- queue_name.underscore
33
- end
34
-
35
- def name_c
36
- name.camelize
26
+ template "pgq_class.rb", "app/workers/pgq_#{file_path}.rb"
27
+ template "spec.rb", "spec/workers/pgq_#{file_path}_spec.rb"
28
+ migration_template "migration.rb", "db/migrate/create_#{file_path}_queue.rb"
37
29
  end
38
30
  end
39
31
 
40
32
  end
33
+
34
+ end
@@ -0,0 +1,34 @@
1
+ # for Rails 2
2
+ if defined?(Rails) && Rails::VERSION::MAJOR == 2
3
+
4
+ module Pgq
5
+
6
+ class PgqGenerator < Rails::Generator::NamedBase
7
+ def manifest
8
+ record do |m|
9
+ m.template "pgq_class.rb", "app/workers/pgq_#{file_path}.rb"
10
+ m.directory "spec/workers"
11
+ m.template "spec.rb", "spec/workers/pgq_#{file_path}_spec.rb"
12
+ m.template "migration.rb", "db/migrate/#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_create_#{file_path}_queue.rb"
13
+
14
+ add_to_queues_list
15
+ end
16
+ end
17
+
18
+ private
19
+ def add_to_queues_list
20
+ filename = Rails.root + 'config/queues_list.yml'
21
+ if File.exists?(filename)
22
+ x = YAML.load_file(filename)
23
+ x << name
24
+ File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
25
+ else
26
+ x = [name]
27
+ File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
28
+ end
29
+ end
30
+ end
31
+
32
+ end
33
+
34
+ end
@@ -0,0 +1,9 @@
1
+ class Create<%= class_name %>Queue < ActiveRecord::Migration
2
+ def self.up
3
+ Pgq::Consumer.add_queue :<%= file_path %>
4
+ end
5
+
6
+ def self.down
7
+ Pgq::Consumer.remove_queue :<%= file_path %>
8
+ end
9
+ end
@@ -1,6 +1,6 @@
1
- class Pgq<%= name_c %> < Pgq::Consumer
1
+ class Pgq<%= class_name %> < Pgq::Consumer
2
2
 
3
- # to insert event: Pgq<%= name_c %>.my_event(1, 2, 3)
3
+ # insert event: Pgq<%= class_name %>.my_event(1, 2, 3)
4
4
 
5
5
  # async execute
6
6
  def my_event(a, b, c)
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ describe Pgq<%= class_name %> do
4
+ before(:each) do
5
+ @consumer = Pgq<%= class_name %>.new
6
+ end
7
+
8
+ end
data/lib/pgq.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'active_support'
2
2
  require 'active_record'
3
+ require 'marshal64'
3
4
 
4
5
  module Pgq
5
6
  end
@@ -68,7 +68,7 @@ module Pgq::Api
68
68
  end
69
69
 
70
70
  def pgq_failed_event_list queue_name, consumer_name, limit = nil, offset = nil, order = 'desc'
71
- order = (order.to_s == 'desc') ? order : 'asc'
71
+ order = (order.to_s == 'desc') ? 'desc' : 'asc'
72
72
  connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) ORDER BY ev_id #{order.upcase}", queue_name, consumer_name, limit.to_i, offset.to_i])
73
73
  end
74
74
 
@@ -128,5 +128,9 @@ module Pgq::Api
128
128
 
129
129
  events.length
130
130
  end
131
-
131
+
132
+ def pgq_force_tick(queue_name)
133
+ connection.select_value(sanitize_sql_array ["SELECT pgq.force_tick(?)", queue_name]).to_i
134
+ end
135
+
132
136
  end
@@ -60,7 +60,7 @@ class Pgq::ConsumerBase
60
60
  # == coder
61
61
 
62
62
  def self.coder
63
- Pgq::Marshal64Coder
63
+ Marshal64
64
64
  end
65
65
 
66
66
  def coder
@@ -4,13 +4,13 @@ class Pgq::Event
4
4
  def initialize(consumer, event)
5
5
  @id = event['ev_id']
6
6
  @type = event['ev_type']
7
- @data = consumer.coder.load(event['ev_data']) if event['ev_data']
8
7
  @consumer = consumer
8
+ @data = @consumer.coder.load(event['ev_data']) if event['ev_data']
9
9
  end
10
10
 
11
11
  def failed!(ex)
12
12
  h = {:class => ex.class.to_s, :message => ex.message, :backtrace => ex.backtrace}
13
- @consumer.event_failed @id, consumer.coder.dump(h)
13
+ @consumer.event_failed @id, @consumer.coder.dump(h)
14
14
  end
15
15
 
16
16
  def retry!(seconds = 0)
@@ -29,8 +29,8 @@ Backtrace: #{e.backtrace.join("\n") rescue ''}
29
29
  def exception_message(e)
30
30
  <<-EXCEPTION
31
31
  Exception happend
32
- Type: #{type.inspect}
33
- Data: #{data.inspect}
32
+ Type: #{@type.inspect}
33
+ Data: #{@data.inspect}
34
34
  Error occurs: #{e.class.inspect}(#{e.message})
35
35
  Backtrace: #{e.backtrace.join("\n") rescue ''}
36
36
  EXCEPTION
@@ -1,3 +1,3 @@
1
1
  module Pgq
2
- VERSION = "0.1.2"
2
+ VERSION = "0.1.3"
3
3
  end
@@ -33,7 +33,9 @@ class Pgq::Worker
33
33
  raise "Queue not selected" if @queues.blank?
34
34
 
35
35
  if @queues == ['all'] || @queues == 'all'
36
- if defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
36
+ if h[:queues_list]
37
+ @queues = YAML.load_file(h[:queues_list])
38
+ elsif defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
37
39
  @queues = YAML.load_file(Rails.root + "config/queues_list.yml")
38
40
  else
39
41
  raise "You shoud create config/queues_list.yml for all queues"
@@ -1,10 +1,15 @@
1
+ if defined?(Rails)
2
+ ENV['RAILS_ROOT'] ||= Rails.root.to_s
3
+ ENV['RAILS_ENV'] ||= Rails.env.to_s
4
+ end
5
+
1
6
  namespace :pgq do
2
7
 
3
8
  desc "Start worker params: QUEUES, LOGGER, WATCH_FILE"
4
9
  task :worker => [:environment] do
5
10
  queues = ENV['QUEUES']
6
- logger_file = ENV['LOGGER'] || Rails.root.join("log/pgq_#{queues}.log")
7
- watch_file = ENV['WATCH_FILE'] || Rails.root.join("tmp/pgq_#{queues}.stop")
11
+ logger_file = ENV['LOGGER'] || "#{ENV['RAILS_ROOT']}/log/pgq_#{queues}.log"
12
+ watch_file = ENV['WATCH_FILE'] || "#{ENV['RAILS_ROOT']}/tmp/pgq_#{queues}.stop"
8
13
  logger = Logger.new(logger_file)
9
14
  w = Pgq::Worker.new(:logger => logger, :queues => queues, :watch_file => watch_file)
10
15
  w.run
@@ -12,7 +17,7 @@ namespace :pgq do
12
17
 
13
18
  desc "Install PgQ to database RAILS_ENV"
14
19
  task :install do
15
- Dir["#{Rails.root}/config/pgq_#{Rails.env}*.ini"].each do |conf|
20
+ Dir["#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}*.ini"].each do |conf|
16
21
  ENV['PGQFILE']=conf
17
22
  Rake::Task['pgq:install_from_file'].invoke
18
23
  end
@@ -34,13 +39,57 @@ namespace :pgq do
34
39
  raise "Something went wrong(see above)... Check that you install skytools package and create #{conf}"
35
40
  end
36
41
  end
42
+
43
+ desc "Generate Pgq config from database.yml and RAILS_ENV"
44
+ task :generate_config do
45
+ require 'yaml'
46
+ raise "RAILS_ROOT should be" unless ENV['RAILS_ROOT']
47
+ raise "RAILS_ENV should be" unless ENV['RAILS_ENV']
48
+
49
+ cfg = if ENV['DATABASE_YML']
50
+ YAML.load_file(ENV['DATABASE_YML']) if File.exists?(ENV['DATABASE_YML'])
51
+ else
52
+ file = File.join(ENV['RAILS_ROOT'], %w{config database.yml})
53
+ YAML.load_file(file) if File.exists?(file)
54
+ end
55
+
56
+ raise "Not found database.yml" unless cfg
57
+ cfg = cfg[ ENV["RAILS_ENV"] ]
37
58
 
59
+ pgq_config = <<-SQL
60
+ [pgqadm]
61
+ job_name = #{cfg['database']}_pgqadm
62
+ db = host=#{cfg['host'] || '127.0.0.1'} dbname=#{cfg['database']} user=#{cfg['username']} password=#{cfg['password']} port=#{cfg['port'] || 5432}
63
+
64
+ # how often to run maintenance [seconds]
65
+ maint_delay = 500
66
+
67
+ # how often to check for activity [seconds]
68
+ loop_delay = 0.05
69
+
70
+ logfile = log/%(job_name)s.log
71
+ pidfile = tmp/%(job_name)s.pid
72
+ SQL
73
+
74
+ output = ENV["PGQ_CONFIG"] || File.join(ENV["RAILS_ROOT"], ["config", "pgq_#{ENV["RAILS_ENV"]}.ini"])
75
+ File.open(output, 'w'){|f| f.write(pgq_config) }
76
+ puts "Config #{ENV["RAILS_ENV"]} generated to '#{output}'"
77
+ end
78
+
79
+ desc "Generate Pgq configs from database.yml for all environments"
80
+ task :generate_configs do
81
+ task = Rake::Task["pgq:generate_config"]
82
+ %w{development test production}.each do |env|
83
+ ENV['RAILS_ENV'] = env
84
+ task.execute
85
+ end
86
+ end
38
87
 
39
88
  namespace :ticker do
40
89
 
41
90
  desc "Start PgQ ticker daemon"
42
91
  task :start do
43
- conf = Rails.root.join("config/pgq_#{Rails.env}.ini")
92
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
44
93
  output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
45
94
  if output.empty?
46
95
  puts "ticker daemon started"
@@ -51,7 +100,7 @@ namespace :pgq do
51
100
 
52
101
  desc "Stop PgQ ticker daemon"
53
102
  task :stop do
54
- conf = Rails.root.join("config/pgq_#{Rails.env}.ini")
103
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
55
104
  output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
56
105
  if output.empty?
57
106
  puts "ticker daemon stoped"
@@ -9,8 +9,8 @@ Gem::Specification.new do |s|
9
9
  s.authors = ["Makarchev Konstantin"]
10
10
  s.autorequire = %q{init}
11
11
 
12
- s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.}
13
- s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.}
12
+ s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.}
13
+ s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.}
14
14
 
15
15
  s.email = %q{kostya27@gmail.com}
16
16
  s.homepage = %q{http://github.com/kostya/pgq}
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
20
20
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
21
21
  s.require_paths = ["lib"]
22
22
 
23
- s.add_dependency 'activesupport', ">=2.3.2"
24
- s.add_dependency 'activerecord', ">=2.3.2"
23
+ s.add_dependency 'activesupport', ">= 2.3.2"
24
+ s.add_dependency 'activerecord', ">= 2.3.2"
25
25
  s.add_dependency 'pg'
26
+ s.add_dependency 'marshal64'
26
27
 
27
28
  s.add_development_dependency "rspec"
28
29
  s.add_development_dependency "rake"
@@ -0,0 +1,48 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Pgq::Api do
4
+ before :each do
5
+ re_queues
6
+ end
7
+
8
+ after :each do
9
+ stop_ticker rescue nil
10
+ end
11
+
12
+ it "should insert event" do
13
+ x = ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
14
+ x.should > 0
15
+ end
16
+
17
+ it "consuming" do
18
+ ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
19
+
20
+ bid, events = api_extract_batch('bla')
21
+
22
+ events.size.should == 1
23
+ event = events.first
24
+
25
+ event['ev_extra1'].should == '5,6,7'
26
+ event['ev_type'].should == 'tp'
27
+ event['ev_id'].to_i.should >= 1
28
+ event['ev_data'].should == '1,2,3'
29
+ end
30
+
31
+ it "failed event and retry" do
32
+ ActiveRecord::Base.pgq_failed_event_count('bla', 'default').should == 0
33
+ ActiveRecord::Base.pgq_failed_event_list('bla', 'default').should == []
34
+
35
+ ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
36
+
37
+ bid, events = api_extract_batch('bla')
38
+
39
+ ActiveRecord::Base.pgq_event_failed(bid, events[0]['ev_id'], 'aaa')
40
+
41
+ ActiveRecord::Base.pgq_finish_batch(bid)
42
+
43
+ ActiveRecord::Base.pgq_force_tick('bla')
44
+ ActiveRecord::Base.pgq_failed_event_count('bla', 'default').should == 1
45
+ # ActiveRecord::Base.pgq_failed_event_list('bla', 'default').should == [...]
46
+ end
47
+
48
+ end
@@ -0,0 +1,6 @@
1
+ test:
2
+ database: gem_pgq_test
3
+ adapter: postgresql
4
+ username: kostya
5
+ password: password
6
+
@@ -0,0 +1,30 @@
1
+ require 'rubygems'
2
+ require "bundler"
3
+ Bundler.setup
4
+
5
+ $:.unshift(File.dirname(__FILE__) + '/../../lib')
6
+ require 'pgq'
7
+ require 'benchmark'
8
+
9
+ require File.join(File.dirname(__FILE__), %w{.. support spec_support})
10
+
11
+ N = 5000
12
+ puts 'start benchmark'
13
+
14
+ re_queues
15
+ tm = Benchmark.realtime{ N.times{ PgqBla.bla } }
16
+ puts "insert without arguments method missing #{tm}"
17
+
18
+ re_queues
19
+ tm = Benchmark.realtime{ N.times{ PgqBla.add_event 'bla' } }
20
+ puts "insert without arguments add_event #{tm}"
21
+
22
+ x = (0..100).to_a.map{|c| {1 => c, :b => "a#{c}", 2 => c + 2}}
23
+
24
+ re_queues
25
+ tm = Benchmark.realtime{ N.times{ PgqBla.bla x } }
26
+ puts "insert arguments method missing #{tm}"
27
+
28
+ re_queues
29
+ tm = Benchmark.realtime{ N.times{ PgqBla.add_event 'bla', x } }
30
+ puts "insert arguments add_event #{tm}"
@@ -150,4 +150,4 @@ describe Pgq::ConsumerBase do
150
150
  $b.should == 'abc'
151
151
  end
152
152
 
153
- end
153
+ end
@@ -0,0 +1,4 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Pgq::ConsumerBase do
4
+ end
@@ -1,15 +1,5 @@
1
1
  require File.dirname(__FILE__) + '/spec_helper'
2
2
 
3
- class PgqHaha < Pgq::Consumer
4
-
5
- def ptest2(a, b)
6
- $a = a
7
- $b = b
8
- 10
9
- end
10
-
11
- end
12
-
13
3
  describe PgqHaha do
14
4
  before :each do
15
5
  @consumer = PgqHaha.new
@@ -2,8 +2,7 @@ require File.dirname(__FILE__) + '/spec_helper'
2
2
 
3
3
  describe Pgq::Event do
4
4
  before :each do
5
- kl = Class.new(Pgq::ConsumerGroup) do
6
- end
5
+ kl = Class.new(Pgq::ConsumerGroup){}
7
6
  @consumer = kl.new
8
7
  @consumer.stub!(:get_next_batch).and_return ''
9
8
  @consumer.stub!(:finish_batch).and_return ''
@@ -3,9 +3,10 @@ require "bundler/setup"
3
3
 
4
4
  Bundler.require
5
5
 
6
+ require File.join(File.dirname(__FILE__), %w{support spec_support})
7
+
6
8
  class Pgq::Consumer
7
9
  # rspec fuckup
8
10
  def self.to_ary
9
11
  end
10
12
  end
11
-
@@ -0,0 +1,70 @@
1
+ require 'yaml'
2
+
3
+ file = File.join(File.dirname(__FILE__), %w{.. app config database.yml})
4
+ cfg = YAML.load_file(file)
5
+
6
+ ActiveRecord::Base.establish_connection cfg['test']
7
+
8
+ def re_queues
9
+ Pgq::Consumer.remove_queue(:bla) rescue nil
10
+ Pgq::Consumer.remove_queue(:test) rescue nil
11
+
12
+ Pgq::Consumer.add_queue :bla
13
+ Pgq::Consumer.add_queue :test
14
+ end
15
+
16
+ re_queues
17
+
18
+ class PgqBla < Pgq::Consumer
19
+ end
20
+
21
+ class PgqTest < Pgq::Consumer
22
+ end
23
+
24
+ class PgqHaha < Pgq::Consumer
25
+
26
+ def ptest2(a, b)
27
+ $a = a
28
+ $b = b
29
+ 10
30
+ end
31
+
32
+ end
33
+
34
+ def start_ticker
35
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
36
+ output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
37
+ end
38
+
39
+ def stop_ticker
40
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
41
+ output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
42
+ end
43
+
44
+ def api_extract_batch(queue_name, consumer_name = 'default')
45
+ start_ticker
46
+ bid = nil
47
+ events = []
48
+ count = 0
49
+
50
+ loop do
51
+ 100.times{ ActiveRecord::Base.pgq_force_tick(queue_name) }
52
+
53
+ bid = ActiveRecord::Base.pgq_next_batch(queue_name, consumer_name)
54
+
55
+ if bid
56
+ events = ActiveRecord::Base.pgq_get_batch_events(bid)
57
+ if events.present?
58
+ break
59
+ end
60
+ end
61
+
62
+ count += 1
63
+ raise "count > 100" if count > 100
64
+ end
65
+
66
+ return [bid, events]
67
+
68
+ ensure
69
+ stop_ticker
70
+ end
@@ -0,0 +1,14 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Pgq::Utils do
4
+ after :each do
5
+ Pgq::Consumer.remove_queue :hoho
6
+ end
7
+
8
+ it "queues list" do
9
+ Pgq::Consumer.queues_list.sort.should == %w{bla test}.sort
10
+
11
+ Pgq::Consumer.add_queue :hoho
12
+ Pgq::Consumer.queues_list.sort.should == %w{bla test hoho}.sort
13
+ end
14
+ end
@@ -1,9 +1,6 @@
1
1
  require File.dirname(__FILE__) + '/spec_helper'
2
2
 
3
3
  describe Pgq::Worker do
4
- class PgqBla < Pgq::Consumer
5
- end
6
-
7
4
  it "should find class not in hash" do
8
5
  Pgq::Worker.predict_queue_class('bla').should == PgqBla
9
6
  Pgq::Worker.predict_queue_class('bla_1').should == PgqBla
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgq
3
3
  version: !ruby/object:Gem::Version
4
- hash: 31
5
- prerelease: false
4
+ hash: 29
5
+ prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 1
9
- - 2
10
- version: 0.1.2
9
+ - 3
10
+ version: 0.1.3
11
11
  platform: ruby
12
12
  authors:
13
13
  - Makarchev Konstantin
@@ -15,13 +15,11 @@ autorequire: init
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-05-26 00:00:00 +04:00
18
+ date: 2012-09-22 00:00:00 +04:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
22
- name: activesupport
23
- prerelease: false
24
- requirement: &id001 !ruby/object:Gem::Requirement
22
+ version_requirements: &id001 !ruby/object:Gem::Requirement
25
23
  none: false
26
24
  requirements:
27
25
  - - ">="
@@ -32,12 +30,12 @@ dependencies:
32
30
  - 3
33
31
  - 2
34
32
  version: 2.3.2
33
+ prerelease: false
35
34
  type: :runtime
36
- version_requirements: *id001
35
+ requirement: *id001
36
+ name: activesupport
37
37
  - !ruby/object:Gem::Dependency
38
- name: activerecord
39
- prerelease: false
40
- requirement: &id002 !ruby/object:Gem::Requirement
38
+ version_requirements: &id002 !ruby/object:Gem::Requirement
41
39
  none: false
42
40
  requirements:
43
41
  - - ">="
@@ -48,12 +46,12 @@ dependencies:
48
46
  - 3
49
47
  - 2
50
48
  version: 2.3.2
49
+ prerelease: false
51
50
  type: :runtime
52
- version_requirements: *id002
51
+ requirement: *id002
52
+ name: activerecord
53
53
  - !ruby/object:Gem::Dependency
54
- name: pg
55
- prerelease: false
56
- requirement: &id003 !ruby/object:Gem::Requirement
54
+ version_requirements: &id003 !ruby/object:Gem::Requirement
57
55
  none: false
58
56
  requirements:
59
57
  - - ">="
@@ -62,12 +60,26 @@ dependencies:
62
60
  segments:
63
61
  - 0
64
62
  version: "0"
63
+ prerelease: false
65
64
  type: :runtime
66
- version_requirements: *id003
65
+ requirement: *id003
66
+ name: pg
67
67
  - !ruby/object:Gem::Dependency
68
- name: rspec
68
+ version_requirements: &id004 !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ hash: 3
74
+ segments:
75
+ - 0
76
+ version: "0"
69
77
  prerelease: false
70
- requirement: &id004 !ruby/object:Gem::Requirement
78
+ type: :runtime
79
+ requirement: *id004
80
+ name: marshal64
81
+ - !ruby/object:Gem::Dependency
82
+ version_requirements: &id005 !ruby/object:Gem::Requirement
71
83
  none: false
72
84
  requirements:
73
85
  - - ">="
@@ -76,12 +88,12 @@ dependencies:
76
88
  segments:
77
89
  - 0
78
90
  version: "0"
91
+ prerelease: false
79
92
  type: :development
80
- version_requirements: *id004
93
+ requirement: *id005
94
+ name: rspec
81
95
  - !ruby/object:Gem::Dependency
82
- name: rake
83
- prerelease: false
84
- requirement: &id005 !ruby/object:Gem::Requirement
96
+ version_requirements: &id006 !ruby/object:Gem::Requirement
85
97
  none: false
86
98
  requirements:
87
99
  - - ">="
@@ -90,9 +102,11 @@ dependencies:
90
102
  segments:
91
103
  - 0
92
104
  version: "0"
105
+ prerelease: false
93
106
  type: :development
94
- version_requirements: *id005
95
- description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
107
+ requirement: *id006
108
+ name: rake
109
+ description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
96
110
  email: kostya27@gmail.com
97
111
  executables: []
98
112
 
@@ -103,36 +117,38 @@ extra_rdoc_files: []
103
117
  files:
104
118
  - .gitignore
105
119
  - Gemfile
106
- - Gemfile.lock
107
120
  - LICENSE
108
121
  - README.md
109
122
  - Rakefile
110
123
  - init.rb
111
124
  - lib/generators/pgq/add_generator.rb
112
- - lib/generators/pgq/add_templates/migration.rb
113
- - lib/generators/pgq/add_templates/pgq_class.rb
114
- - lib/generators/pgq/add_templates/spec.rb
115
- - lib/generators/pgq/config_generator.rb
116
- - lib/generators/pgq/config_templates/pgq.rb
125
+ - lib/generators/pgq/pgq_generator.rb
126
+ - lib/generators/pgq/templates/migration.rb
127
+ - lib/generators/pgq/templates/pgq_class.rb
128
+ - lib/generators/pgq/templates/spec.rb
117
129
  - lib/pgq.rb
118
130
  - lib/pgq/api.rb
119
131
  - lib/pgq/consumer.rb
120
132
  - lib/pgq/consumer_base.rb
121
133
  - lib/pgq/consumer_group.rb
122
134
  - lib/pgq/event.rb
123
- - lib/pgq/marshal64_coder.rb
124
135
  - lib/pgq/railtie.rb
125
136
  - lib/pgq/utils.rb
126
137
  - lib/pgq/version.rb
127
138
  - lib/pgq/worker.rb
128
139
  - lib/tasks/pgq.rake
129
140
  - pgq.gemspec
141
+ - spec/api_spec.rb
142
+ - spec/app/config/database.yml
143
+ - spec/benchmark/insert_bench.rb
130
144
  - spec/consumer_base_spec.rb
131
145
  - spec/consumer_group_spec.rb
146
+ - spec/consumer_intergration_spec.rb
132
147
  - spec/consumer_spec.rb
133
148
  - spec/event_spec.rb
134
- - spec/marshal64_coder_spec.rb
135
149
  - spec/spec_helper.rb
150
+ - spec/support/spec_support.rb
151
+ - spec/utils_spec.rb
136
152
  - spec/worker_spec.rb
137
153
  has_rdoc: true
138
154
  homepage: http://github.com/kostya/pgq
@@ -164,9 +180,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
164
180
  requirements: []
165
181
 
166
182
  rubyforge_project:
167
- rubygems_version: 1.3.7
183
+ rubygems_version: 1.4.2
168
184
  signing_key:
169
185
  specification_version: 3
170
- summary: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
186
+ summary: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
171
187
  test_files: []
172
188
 
@@ -1,46 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- pgq (0.1.2)
5
- activerecord (>= 2.3.2)
6
- activesupport (>= 2.3.2)
7
- pg
8
-
9
- GEM
10
- remote: http://rubygems.org/
11
- specs:
12
- activemodel (3.2.3)
13
- activesupport (= 3.2.3)
14
- builder (~> 3.0.0)
15
- activerecord (3.2.3)
16
- activemodel (= 3.2.3)
17
- activesupport (= 3.2.3)
18
- arel (~> 3.0.2)
19
- tzinfo (~> 0.3.29)
20
- activesupport (3.2.3)
21
- i18n (~> 0.6)
22
- multi_json (~> 1.0)
23
- arel (3.0.2)
24
- builder (3.0.0)
25
- diff-lcs (1.1.3)
26
- i18n (0.6.0)
27
- multi_json (1.3.5)
28
- pg (0.13.2)
29
- rake (0.9.2.2)
30
- rspec (2.10.0)
31
- rspec-core (~> 2.10.0)
32
- rspec-expectations (~> 2.10.0)
33
- rspec-mocks (~> 2.10.0)
34
- rspec-core (2.10.0)
35
- rspec-expectations (2.10.0)
36
- diff-lcs (~> 1.1.3)
37
- rspec-mocks (2.10.1)
38
- tzinfo (0.3.33)
39
-
40
- PLATFORMS
41
- ruby
42
-
43
- DEPENDENCIES
44
- pgq!
45
- rake
46
- rspec
@@ -1,9 +0,0 @@
1
- class Create<%= name_c %>Queue < ActiveRecord::Migration
2
- def self.up
3
- Pgq::Consumer.add_queue :<%= name %>
4
- end
5
-
6
- def self.down
7
- Pgq::Consumer.remove_queue :<%= name %>
8
- end
9
- end
@@ -1,8 +0,0 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
2
-
3
- describe Pgq<%= name_c %> do
4
- before(:each) do
5
- @consumer = Pgq<%= name_c %>.new
6
- end
7
-
8
- end
@@ -1,18 +0,0 @@
1
- module Pgq
2
- class ConfigGenerator < Rails::Generators::Base
3
- source_root File.expand_path("../config_templates", __FILE__)
4
-
5
- def copy_configs
6
- self.env_name = 'development'
7
- template "pgq.rb", "config/pgq_development.ini"
8
- self.env_name = 'test'
9
- template "pgq.rb", "config/pgq_test.ini"
10
- self.env_name = 'production'
11
- template "pgq.rb", "config/pgq_production.ini"
12
- end
13
-
14
- private
15
- attr_accessor :env_name
16
-
17
- end
18
- end
@@ -1,12 +0,0 @@
1
- [pgqadm]
2
- job_name = pgqadm_<%= env_name %>
3
- db = dbname=dbname user=user password=password host=127.0.0.1 port=5432
4
-
5
- # how often to run maintenance [seconds]
6
- maint_delay = 600
7
-
8
- # how often to check for activity [seconds]
9
- loop_delay = 0.1
10
-
11
- logfile = log/%(job_name)s.log
12
- pidfile = tmp/%(job_name)s.pid
@@ -1,13 +0,0 @@
1
- require 'base64'
2
-
3
- module Pgq::Marshal64Coder
4
-
5
- def self.dump(s)
6
- Base64::encode64(Marshal.dump(s))
7
- end
8
-
9
- def self.load(s)
10
- Marshal.load(Base64::decode64(s))
11
- end
12
-
13
- end
@@ -1,22 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe Pgq::Marshal64Coder do
4
- before :each do
5
- @coder = Pgq::Marshal64Coder
6
- end
7
-
8
- it "work" do
9
- data = {:a => [1, 2, 3, {:b => 'c'}]}
10
- s = @coder.dump(data)
11
- @coder.load(s).should == data
12
- end
13
-
14
- it "work on large data" do
15
- data = {}
16
- 10000.times{|i| data["#{i}".to_sym] = [i, "i_#{i}"]}
17
-
18
- s = @coder.dump(data)
19
- @coder.load(s).should == data
20
- end
21
-
22
- end