pgq 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -1
- data/README.md +22 -9
- data/Rakefile +18 -1
- data/lib/generators/pgq/add_generator.rb +10 -16
- data/lib/generators/pgq/pgq_generator.rb +34 -0
- data/lib/generators/pgq/templates/migration.rb +9 -0
- data/lib/generators/pgq/{add_templates → templates}/pgq_class.rb +2 -2
- data/lib/generators/pgq/templates/spec.rb +8 -0
- data/lib/pgq.rb +1 -0
- data/lib/pgq/api.rb +6 -2
- data/lib/pgq/consumer_base.rb +1 -1
- data/lib/pgq/event.rb +4 -4
- data/lib/pgq/version.rb +1 -1
- data/lib/pgq/worker.rb +3 -1
- data/lib/tasks/pgq.rake +54 -5
- data/pgq.gemspec +5 -4
- data/spec/api_spec.rb +48 -0
- data/spec/app/config/database.yml +6 -0
- data/spec/benchmark/insert_bench.rb +30 -0
- data/spec/consumer_base_spec.rb +1 -1
- data/spec/consumer_intergration_spec.rb +4 -0
- data/spec/consumer_spec.rb +0 -10
- data/spec/event_spec.rb +1 -2
- data/spec/spec_helper.rb +2 -1
- data/spec/support/spec_support.rb +70 -0
- data/spec/utils_spec.rb +14 -0
- data/spec/worker_spec.rb +0 -3
- metadata +51 -35
- data/Gemfile.lock +0 -46
- data/lib/generators/pgq/add_templates/migration.rb +0 -9
- data/lib/generators/pgq/add_templates/spec.rb +0 -8
- data/lib/generators/pgq/config_generator.rb +0 -18
- data/lib/generators/pgq/config_templates/pgq.rb +0 -12
- data/lib/pgq/marshal64_coder.rb +0 -13
- data/spec/marshal64_coder_spec.rb +0 -22
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
Pgq
|
2
|
-
|
1
|
+
Pgq
|
2
|
+
===
|
3
3
|
|
4
|
-
Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3
|
4
|
+
Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
|
5
5
|
|
6
6
|
About PgQ
|
7
7
|
* http://wiki.postgresql.org/wiki/SkyTools#PgQ
|
@@ -22,9 +22,9 @@ Gemfile:
|
|
22
22
|
gem 'pgq'
|
23
23
|
```
|
24
24
|
|
25
|
-
Create ticker configs:
|
25
|
+
Create ticker configs from database.yml:
|
26
26
|
|
27
|
-
$
|
27
|
+
$ rake pgq:generate_configs
|
28
28
|
edit config: config/pgq_development.ini
|
29
29
|
|
30
30
|
|
@@ -46,7 +46,7 @@ Daemon run once, bind to the database. (If worker not consuming check that daemo
|
|
46
46
|
|
47
47
|
Last, add to config/application.rb
|
48
48
|
|
49
|
-
config.autoload_paths += %W( #{config.root}/app/
|
49
|
+
config.autoload_paths += %W( #{config.root}/app/workers )
|
50
50
|
|
51
51
|
|
52
52
|
Create Pgq consumer
|
@@ -55,7 +55,7 @@ Create Pgq consumer
|
|
55
55
|
$ rails generate pgq:add my
|
56
56
|
|
57
57
|
|
58
|
-
This creates file app/
|
58
|
+
This creates file app/workers/pgq_my.rb and migration
|
59
59
|
|
60
60
|
$ rake db:migrate
|
61
61
|
|
@@ -174,7 +174,7 @@ class Pgq::Consumer
|
|
174
174
|
|
175
175
|
# specify coder
|
176
176
|
def self.coder
|
177
|
-
|
177
|
+
Marshal64 # class/module with self methods dump/load
|
178
178
|
end
|
179
179
|
|
180
180
|
end
|
@@ -188,4 +188,17 @@ Usefull in specs
|
|
188
188
|
```
|
189
189
|
|
190
190
|
When code call PgqMy.some_method1(a,b,c) this would be convert into PgqMy.new.some_method1(a,b,c)
|
191
|
-
|
191
|
+
|
192
|
+
|
193
|
+
### for Rails 2.3
|
194
|
+
|
195
|
+
Add to Rakefile:
|
196
|
+
```
|
197
|
+
load 'tasks/pgq.rake'
|
198
|
+
```
|
199
|
+
|
200
|
+
Copy generators from gem to lib/generators.
|
201
|
+
|
202
|
+
```
|
203
|
+
./script/generate pgq bla
|
204
|
+
```
|
data/Rakefile
CHANGED
@@ -3,7 +3,24 @@ require 'bundler/setup'
|
|
3
3
|
require 'bundler'
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
|
+
load 'lib/tasks/pgq.rake'
|
7
|
+
|
6
8
|
require 'rspec/core/rake_task'
|
7
9
|
|
8
10
|
task :default => :spec
|
9
|
-
RSpec::Core::RakeTask.new(:spec)
|
11
|
+
RSpec::Core::RakeTask.new(:spec)
|
12
|
+
|
13
|
+
desc "set_env"
|
14
|
+
task :set_env do
|
15
|
+
ENV["PGQFILE"] = File.join(File.dirname(__FILE__), %w{ spec app config pgq_test.ini })
|
16
|
+
ENV["RAILS_ROOT"] = File.join(File.dirname(__FILE__), %w{ spec app })
|
17
|
+
ENV["RAILS_ENV"] = "test"
|
18
|
+
end
|
19
|
+
|
20
|
+
desc "prepare test"
|
21
|
+
task :prepare_test => :set_env do
|
22
|
+
Rake::Task["pgq:generate_config"].invoke
|
23
|
+
Rake::Task["pgq:install"].invoke
|
24
|
+
end
|
25
|
+
|
26
|
+
task :spec => :prepare_test
|
@@ -1,8 +1,10 @@
|
|
1
|
+
# for Rails 3
|
2
|
+
if defined?(Rails) && Rails::VERSION::MAJOR >= 3
|
3
|
+
|
1
4
|
module Pgq
|
2
|
-
class AddGenerator < Rails::Generators::
|
5
|
+
class AddGenerator < Rails::Generators::NamedBase
|
3
6
|
include Rails::Generators::Migration
|
4
|
-
source_root File.expand_path("../
|
5
|
-
argument :queue_name, :type => :string
|
7
|
+
source_root File.expand_path("../templates", __FILE__)
|
6
8
|
|
7
9
|
def self.next_migration_number(path)
|
8
10
|
Time.now.utc.strftime("%Y%m%d%H%M%S")
|
@@ -21,20 +23,12 @@ module Pgq
|
|
21
23
|
end
|
22
24
|
|
23
25
|
def add_files
|
24
|
-
template "pgq_class.rb", "app/
|
25
|
-
template "spec.rb", "spec/
|
26
|
-
migration_template "migration.rb", "db/migrate/create_#{
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
def name
|
32
|
-
queue_name.underscore
|
33
|
-
end
|
34
|
-
|
35
|
-
def name_c
|
36
|
-
name.camelize
|
26
|
+
template "pgq_class.rb", "app/workers/pgq_#{file_path}.rb"
|
27
|
+
template "spec.rb", "spec/workers/pgq_#{file_path}_spec.rb"
|
28
|
+
migration_template "migration.rb", "db/migrate/create_#{file_path}_queue.rb"
|
37
29
|
end
|
38
30
|
end
|
39
31
|
|
40
32
|
end
|
33
|
+
|
34
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# for Rails 2
|
2
|
+
if defined?(Rails) && Rails::VERSION::MAJOR == 2
|
3
|
+
|
4
|
+
module Pgq
|
5
|
+
|
6
|
+
class PgqGenerator < Rails::Generator::NamedBase
|
7
|
+
def manifest
|
8
|
+
record do |m|
|
9
|
+
m.template "pgq_class.rb", "app/workers/pgq_#{file_path}.rb"
|
10
|
+
m.directory "spec/workers"
|
11
|
+
m.template "spec.rb", "spec/workers/pgq_#{file_path}_spec.rb"
|
12
|
+
m.template "migration.rb", "db/migrate/#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_create_#{file_path}_queue.rb"
|
13
|
+
|
14
|
+
add_to_queues_list
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def add_to_queues_list
|
20
|
+
filename = Rails.root + 'config/queues_list.yml'
|
21
|
+
if File.exists?(filename)
|
22
|
+
x = YAML.load_file(filename)
|
23
|
+
x << name
|
24
|
+
File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
|
25
|
+
else
|
26
|
+
x = [name]
|
27
|
+
File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
data/lib/pgq.rb
CHANGED
data/lib/pgq/api.rb
CHANGED
@@ -68,7 +68,7 @@ module Pgq::Api
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def pgq_failed_event_list queue_name, consumer_name, limit = nil, offset = nil, order = 'desc'
|
71
|
-
order = (order.to_s == 'desc') ?
|
71
|
+
order = (order.to_s == 'desc') ? 'desc' : 'asc'
|
72
72
|
connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) ORDER BY ev_id #{order.upcase}", queue_name, consumer_name, limit.to_i, offset.to_i])
|
73
73
|
end
|
74
74
|
|
@@ -128,5 +128,9 @@ module Pgq::Api
|
|
128
128
|
|
129
129
|
events.length
|
130
130
|
end
|
131
|
-
|
131
|
+
|
132
|
+
def pgq_force_tick(queue_name)
|
133
|
+
connection.select_value(sanitize_sql_array ["SELECT pgq.force_tick(?)", queue_name]).to_i
|
134
|
+
end
|
135
|
+
|
132
136
|
end
|
data/lib/pgq/consumer_base.rb
CHANGED
data/lib/pgq/event.rb
CHANGED
@@ -4,13 +4,13 @@ class Pgq::Event
|
|
4
4
|
def initialize(consumer, event)
|
5
5
|
@id = event['ev_id']
|
6
6
|
@type = event['ev_type']
|
7
|
-
@data = consumer.coder.load(event['ev_data']) if event['ev_data']
|
8
7
|
@consumer = consumer
|
8
|
+
@data = @consumer.coder.load(event['ev_data']) if event['ev_data']
|
9
9
|
end
|
10
10
|
|
11
11
|
def failed!(ex)
|
12
12
|
h = {:class => ex.class.to_s, :message => ex.message, :backtrace => ex.backtrace}
|
13
|
-
@consumer.event_failed @id, consumer.coder.dump(h)
|
13
|
+
@consumer.event_failed @id, @consumer.coder.dump(h)
|
14
14
|
end
|
15
15
|
|
16
16
|
def retry!(seconds = 0)
|
@@ -29,8 +29,8 @@ Backtrace: #{e.backtrace.join("\n") rescue ''}
|
|
29
29
|
def exception_message(e)
|
30
30
|
<<-EXCEPTION
|
31
31
|
Exception happend
|
32
|
-
Type: #{type.inspect}
|
33
|
-
Data: #{data.inspect}
|
32
|
+
Type: #{@type.inspect}
|
33
|
+
Data: #{@data.inspect}
|
34
34
|
Error occurs: #{e.class.inspect}(#{e.message})
|
35
35
|
Backtrace: #{e.backtrace.join("\n") rescue ''}
|
36
36
|
EXCEPTION
|
data/lib/pgq/version.rb
CHANGED
data/lib/pgq/worker.rb
CHANGED
@@ -33,7 +33,9 @@ class Pgq::Worker
|
|
33
33
|
raise "Queue not selected" if @queues.blank?
|
34
34
|
|
35
35
|
if @queues == ['all'] || @queues == 'all'
|
36
|
-
if
|
36
|
+
if h[:queues_list]
|
37
|
+
@queues = YAML.load_file(h[:queues_list])
|
38
|
+
elsif defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
|
37
39
|
@queues = YAML.load_file(Rails.root + "config/queues_list.yml")
|
38
40
|
else
|
39
41
|
raise "You shoud create config/queues_list.yml for all queues"
|
data/lib/tasks/pgq.rake
CHANGED
@@ -1,10 +1,15 @@
|
|
1
|
+
if defined?(Rails)
|
2
|
+
ENV['RAILS_ROOT'] ||= Rails.root.to_s
|
3
|
+
ENV['RAILS_ENV'] ||= Rails.env.to_s
|
4
|
+
end
|
5
|
+
|
1
6
|
namespace :pgq do
|
2
7
|
|
3
8
|
desc "Start worker params: QUEUES, LOGGER, WATCH_FILE"
|
4
9
|
task :worker => [:environment] do
|
5
10
|
queues = ENV['QUEUES']
|
6
|
-
logger_file = ENV['LOGGER'] ||
|
7
|
-
watch_file = ENV['WATCH_FILE'] ||
|
11
|
+
logger_file = ENV['LOGGER'] || "#{ENV['RAILS_ROOT']}/log/pgq_#{queues}.log"
|
12
|
+
watch_file = ENV['WATCH_FILE'] || "#{ENV['RAILS_ROOT']}/tmp/pgq_#{queues}.stop"
|
8
13
|
logger = Logger.new(logger_file)
|
9
14
|
w = Pgq::Worker.new(:logger => logger, :queues => queues, :watch_file => watch_file)
|
10
15
|
w.run
|
@@ -12,7 +17,7 @@ namespace :pgq do
|
|
12
17
|
|
13
18
|
desc "Install PgQ to database RAILS_ENV"
|
14
19
|
task :install do
|
15
|
-
Dir["#{
|
20
|
+
Dir["#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}*.ini"].each do |conf|
|
16
21
|
ENV['PGQFILE']=conf
|
17
22
|
Rake::Task['pgq:install_from_file'].invoke
|
18
23
|
end
|
@@ -34,13 +39,57 @@ namespace :pgq do
|
|
34
39
|
raise "Something went wrong(see above)... Check that you install skytools package and create #{conf}"
|
35
40
|
end
|
36
41
|
end
|
42
|
+
|
43
|
+
desc "Generate Pgq config from database.yml and RAILS_ENV"
|
44
|
+
task :generate_config do
|
45
|
+
require 'yaml'
|
46
|
+
raise "RAILS_ROOT should be" unless ENV['RAILS_ROOT']
|
47
|
+
raise "RAILS_ENV should be" unless ENV['RAILS_ENV']
|
48
|
+
|
49
|
+
cfg = if ENV['DATABASE_YML']
|
50
|
+
YAML.load_file(ENV['DATABASE_YML']) if File.exists?(ENV['DATABASE_YML'])
|
51
|
+
else
|
52
|
+
file = File.join(ENV['RAILS_ROOT'], %w{config database.yml})
|
53
|
+
YAML.load_file(file) if File.exists?(file)
|
54
|
+
end
|
55
|
+
|
56
|
+
raise "Not found database.yml" unless cfg
|
57
|
+
cfg = cfg[ ENV["RAILS_ENV"] ]
|
37
58
|
|
59
|
+
pgq_config = <<-SQL
|
60
|
+
[pgqadm]
|
61
|
+
job_name = #{cfg['database']}_pgqadm
|
62
|
+
db = host=#{cfg['host'] || '127.0.0.1'} dbname=#{cfg['database']} user=#{cfg['username']} password=#{cfg['password']} port=#{cfg['port'] || 5432}
|
63
|
+
|
64
|
+
# how often to run maintenance [seconds]
|
65
|
+
maint_delay = 500
|
66
|
+
|
67
|
+
# how often to check for activity [seconds]
|
68
|
+
loop_delay = 0.05
|
69
|
+
|
70
|
+
logfile = log/%(job_name)s.log
|
71
|
+
pidfile = tmp/%(job_name)s.pid
|
72
|
+
SQL
|
73
|
+
|
74
|
+
output = ENV["PGQ_CONFIG"] || File.join(ENV["RAILS_ROOT"], ["config", "pgq_#{ENV["RAILS_ENV"]}.ini"])
|
75
|
+
File.open(output, 'w'){|f| f.write(pgq_config) }
|
76
|
+
puts "Config #{ENV["RAILS_ENV"]} generated to '#{output}'"
|
77
|
+
end
|
78
|
+
|
79
|
+
desc "Generate Pgq configs from database.yml for all environments"
|
80
|
+
task :generate_configs do
|
81
|
+
task = Rake::Task["pgq:generate_config"]
|
82
|
+
%w{development test production}.each do |env|
|
83
|
+
ENV['RAILS_ENV'] = env
|
84
|
+
task.execute
|
85
|
+
end
|
86
|
+
end
|
38
87
|
|
39
88
|
namespace :ticker do
|
40
89
|
|
41
90
|
desc "Start PgQ ticker daemon"
|
42
91
|
task :start do
|
43
|
-
conf =
|
92
|
+
conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
|
44
93
|
output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
|
45
94
|
if output.empty?
|
46
95
|
puts "ticker daemon started"
|
@@ -51,7 +100,7 @@ namespace :pgq do
|
|
51
100
|
|
52
101
|
desc "Stop PgQ ticker daemon"
|
53
102
|
task :stop do
|
54
|
-
conf =
|
103
|
+
conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
|
55
104
|
output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
|
56
105
|
if output.empty?
|
57
106
|
puts "ticker daemon stoped"
|
data/pgq.gemspec
CHANGED
@@ -9,8 +9,8 @@ Gem::Specification.new do |s|
|
|
9
9
|
s.authors = ["Makarchev Konstantin"]
|
10
10
|
s.autorequire = %q{init}
|
11
11
|
|
12
|
-
s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3
|
13
|
-
s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3
|
12
|
+
s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.}
|
13
|
+
s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.}
|
14
14
|
|
15
15
|
s.email = %q{kostya27@gmail.com}
|
16
16
|
s.homepage = %q{http://github.com/kostya/pgq}
|
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
21
21
|
s.require_paths = ["lib"]
|
22
22
|
|
23
|
-
s.add_dependency 'activesupport', ">=2.3.2"
|
24
|
-
s.add_dependency 'activerecord', ">=2.3.2"
|
23
|
+
s.add_dependency 'activesupport', ">= 2.3.2"
|
24
|
+
s.add_dependency 'activerecord', ">= 2.3.2"
|
25
25
|
s.add_dependency 'pg'
|
26
|
+
s.add_dependency 'marshal64'
|
26
27
|
|
27
28
|
s.add_development_dependency "rspec"
|
28
29
|
s.add_development_dependency "rake"
|
data/spec/api_spec.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe Pgq::Api do
|
4
|
+
before :each do
|
5
|
+
re_queues
|
6
|
+
end
|
7
|
+
|
8
|
+
after :each do
|
9
|
+
stop_ticker rescue nil
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should insert event" do
|
13
|
+
x = ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
|
14
|
+
x.should > 0
|
15
|
+
end
|
16
|
+
|
17
|
+
it "consuming" do
|
18
|
+
ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
|
19
|
+
|
20
|
+
bid, events = api_extract_batch('bla')
|
21
|
+
|
22
|
+
events.size.should == 1
|
23
|
+
event = events.first
|
24
|
+
|
25
|
+
event['ev_extra1'].should == '5,6,7'
|
26
|
+
event['ev_type'].should == 'tp'
|
27
|
+
event['ev_id'].to_i.should >= 1
|
28
|
+
event['ev_data'].should == '1,2,3'
|
29
|
+
end
|
30
|
+
|
31
|
+
it "failed event and retry" do
|
32
|
+
ActiveRecord::Base.pgq_failed_event_count('bla', 'default').should == 0
|
33
|
+
ActiveRecord::Base.pgq_failed_event_list('bla', 'default').should == []
|
34
|
+
|
35
|
+
ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
|
36
|
+
|
37
|
+
bid, events = api_extract_batch('bla')
|
38
|
+
|
39
|
+
ActiveRecord::Base.pgq_event_failed(bid, events[0]['ev_id'], 'aaa')
|
40
|
+
|
41
|
+
ActiveRecord::Base.pgq_finish_batch(bid)
|
42
|
+
|
43
|
+
ActiveRecord::Base.pgq_force_tick('bla')
|
44
|
+
ActiveRecord::Base.pgq_failed_event_count('bla', 'default').should == 1
|
45
|
+
# ActiveRecord::Base.pgq_failed_event_list('bla', 'default').should == [...]
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require "bundler"
|
3
|
+
Bundler.setup
|
4
|
+
|
5
|
+
$:.unshift(File.dirname(__FILE__) + '/../../lib')
|
6
|
+
require 'pgq'
|
7
|
+
require 'benchmark'
|
8
|
+
|
9
|
+
require File.join(File.dirname(__FILE__), %w{.. support spec_support})
|
10
|
+
|
11
|
+
N = 5000
|
12
|
+
puts 'start benchmark'
|
13
|
+
|
14
|
+
re_queues
|
15
|
+
tm = Benchmark.realtime{ N.times{ PgqBla.bla } }
|
16
|
+
puts "insert without arguments method missing #{tm}"
|
17
|
+
|
18
|
+
re_queues
|
19
|
+
tm = Benchmark.realtime{ N.times{ PgqBla.add_event 'bla' } }
|
20
|
+
puts "insert without arguments add_event #{tm}"
|
21
|
+
|
22
|
+
x = (0..100).to_a.map{|c| {1 => c, :b => "a#{c}", 2 => c + 2}}
|
23
|
+
|
24
|
+
re_queues
|
25
|
+
tm = Benchmark.realtime{ N.times{ PgqBla.bla x } }
|
26
|
+
puts "insert arguments method missing #{tm}"
|
27
|
+
|
28
|
+
re_queues
|
29
|
+
tm = Benchmark.realtime{ N.times{ PgqBla.add_event 'bla', x } }
|
30
|
+
puts "insert arguments add_event #{tm}"
|
data/spec/consumer_base_spec.rb
CHANGED
data/spec/consumer_spec.rb
CHANGED
data/spec/event_spec.rb
CHANGED
@@ -2,8 +2,7 @@ require File.dirname(__FILE__) + '/spec_helper'
|
|
2
2
|
|
3
3
|
describe Pgq::Event do
|
4
4
|
before :each do
|
5
|
-
kl = Class.new(Pgq::ConsumerGroup)
|
6
|
-
end
|
5
|
+
kl = Class.new(Pgq::ConsumerGroup){}
|
7
6
|
@consumer = kl.new
|
8
7
|
@consumer.stub!(:get_next_batch).and_return ''
|
9
8
|
@consumer.stub!(:finish_batch).and_return ''
|
data/spec/spec_helper.rb
CHANGED
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
file = File.join(File.dirname(__FILE__), %w{.. app config database.yml})
|
4
|
+
cfg = YAML.load_file(file)
|
5
|
+
|
6
|
+
ActiveRecord::Base.establish_connection cfg['test']
|
7
|
+
|
8
|
+
def re_queues
|
9
|
+
Pgq::Consumer.remove_queue(:bla) rescue nil
|
10
|
+
Pgq::Consumer.remove_queue(:test) rescue nil
|
11
|
+
|
12
|
+
Pgq::Consumer.add_queue :bla
|
13
|
+
Pgq::Consumer.add_queue :test
|
14
|
+
end
|
15
|
+
|
16
|
+
re_queues
|
17
|
+
|
18
|
+
class PgqBla < Pgq::Consumer
|
19
|
+
end
|
20
|
+
|
21
|
+
class PgqTest < Pgq::Consumer
|
22
|
+
end
|
23
|
+
|
24
|
+
class PgqHaha < Pgq::Consumer
|
25
|
+
|
26
|
+
def ptest2(a, b)
|
27
|
+
$a = a
|
28
|
+
$b = b
|
29
|
+
10
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
def start_ticker
|
35
|
+
conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
|
36
|
+
output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
|
37
|
+
end
|
38
|
+
|
39
|
+
def stop_ticker
|
40
|
+
conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
|
41
|
+
output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
|
42
|
+
end
|
43
|
+
|
44
|
+
def api_extract_batch(queue_name, consumer_name = 'default')
|
45
|
+
start_ticker
|
46
|
+
bid = nil
|
47
|
+
events = []
|
48
|
+
count = 0
|
49
|
+
|
50
|
+
loop do
|
51
|
+
100.times{ ActiveRecord::Base.pgq_force_tick(queue_name) }
|
52
|
+
|
53
|
+
bid = ActiveRecord::Base.pgq_next_batch(queue_name, consumer_name)
|
54
|
+
|
55
|
+
if bid
|
56
|
+
events = ActiveRecord::Base.pgq_get_batch_events(bid)
|
57
|
+
if events.present?
|
58
|
+
break
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
count += 1
|
63
|
+
raise "count > 100" if count > 100
|
64
|
+
end
|
65
|
+
|
66
|
+
return [bid, events]
|
67
|
+
|
68
|
+
ensure
|
69
|
+
stop_ticker
|
70
|
+
end
|
data/spec/utils_spec.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe Pgq::Utils do
|
4
|
+
after :each do
|
5
|
+
Pgq::Consumer.remove_queue :hoho
|
6
|
+
end
|
7
|
+
|
8
|
+
it "queues list" do
|
9
|
+
Pgq::Consumer.queues_list.sort.should == %w{bla test}.sort
|
10
|
+
|
11
|
+
Pgq::Consumer.add_queue :hoho
|
12
|
+
Pgq::Consumer.queues_list.sort.should == %w{bla test hoho}.sort
|
13
|
+
end
|
14
|
+
end
|
data/spec/worker_spec.rb
CHANGED
@@ -1,9 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
3
|
describe Pgq::Worker do
|
4
|
-
class PgqBla < Pgq::Consumer
|
5
|
-
end
|
6
|
-
|
7
4
|
it "should find class not in hash" do
|
8
5
|
Pgq::Worker.predict_queue_class('bla').should == PgqBla
|
9
6
|
Pgq::Worker.predict_queue_class('bla_1').should == PgqBla
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
5
|
-
prerelease:
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
9
|
+
- 3
|
10
|
+
version: 0.1.3
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Makarchev Konstantin
|
@@ -15,13 +15,11 @@ autorequire: init
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-
|
18
|
+
date: 2012-09-22 00:00:00 +04:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
22
|
+
version_requirements: &id001 !ruby/object:Gem::Requirement
|
25
23
|
none: false
|
26
24
|
requirements:
|
27
25
|
- - ">="
|
@@ -32,12 +30,12 @@ dependencies:
|
|
32
30
|
- 3
|
33
31
|
- 2
|
34
32
|
version: 2.3.2
|
33
|
+
prerelease: false
|
35
34
|
type: :runtime
|
36
|
-
|
35
|
+
requirement: *id001
|
36
|
+
name: activesupport
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
|
39
|
-
prerelease: false
|
40
|
-
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
version_requirements: &id002 !ruby/object:Gem::Requirement
|
41
39
|
none: false
|
42
40
|
requirements:
|
43
41
|
- - ">="
|
@@ -48,12 +46,12 @@ dependencies:
|
|
48
46
|
- 3
|
49
47
|
- 2
|
50
48
|
version: 2.3.2
|
49
|
+
prerelease: false
|
51
50
|
type: :runtime
|
52
|
-
|
51
|
+
requirement: *id002
|
52
|
+
name: activerecord
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
|
55
|
-
prerelease: false
|
56
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
54
|
+
version_requirements: &id003 !ruby/object:Gem::Requirement
|
57
55
|
none: false
|
58
56
|
requirements:
|
59
57
|
- - ">="
|
@@ -62,12 +60,26 @@ dependencies:
|
|
62
60
|
segments:
|
63
61
|
- 0
|
64
62
|
version: "0"
|
63
|
+
prerelease: false
|
65
64
|
type: :runtime
|
66
|
-
|
65
|
+
requirement: *id003
|
66
|
+
name: pg
|
67
67
|
- !ruby/object:Gem::Dependency
|
68
|
-
|
68
|
+
version_requirements: &id004 !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
hash: 3
|
74
|
+
segments:
|
75
|
+
- 0
|
76
|
+
version: "0"
|
69
77
|
prerelease: false
|
70
|
-
|
78
|
+
type: :runtime
|
79
|
+
requirement: *id004
|
80
|
+
name: marshal64
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
version_requirements: &id005 !ruby/object:Gem::Requirement
|
71
83
|
none: false
|
72
84
|
requirements:
|
73
85
|
- - ">="
|
@@ -76,12 +88,12 @@ dependencies:
|
|
76
88
|
segments:
|
77
89
|
- 0
|
78
90
|
version: "0"
|
91
|
+
prerelease: false
|
79
92
|
type: :development
|
80
|
-
|
93
|
+
requirement: *id005
|
94
|
+
name: rspec
|
81
95
|
- !ruby/object:Gem::Dependency
|
82
|
-
|
83
|
-
prerelease: false
|
84
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
96
|
+
version_requirements: &id006 !ruby/object:Gem::Requirement
|
85
97
|
none: false
|
86
98
|
requirements:
|
87
99
|
- - ">="
|
@@ -90,9 +102,11 @@ dependencies:
|
|
90
102
|
segments:
|
91
103
|
- 0
|
92
104
|
version: "0"
|
105
|
+
prerelease: false
|
93
106
|
type: :development
|
94
|
-
|
95
|
-
|
107
|
+
requirement: *id006
|
108
|
+
name: rake
|
109
|
+
description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
|
96
110
|
email: kostya27@gmail.com
|
97
111
|
executables: []
|
98
112
|
|
@@ -103,36 +117,38 @@ extra_rdoc_files: []
|
|
103
117
|
files:
|
104
118
|
- .gitignore
|
105
119
|
- Gemfile
|
106
|
-
- Gemfile.lock
|
107
120
|
- LICENSE
|
108
121
|
- README.md
|
109
122
|
- Rakefile
|
110
123
|
- init.rb
|
111
124
|
- lib/generators/pgq/add_generator.rb
|
112
|
-
- lib/generators/pgq/
|
113
|
-
- lib/generators/pgq/
|
114
|
-
- lib/generators/pgq/
|
115
|
-
- lib/generators/pgq/
|
116
|
-
- lib/generators/pgq/config_templates/pgq.rb
|
125
|
+
- lib/generators/pgq/pgq_generator.rb
|
126
|
+
- lib/generators/pgq/templates/migration.rb
|
127
|
+
- lib/generators/pgq/templates/pgq_class.rb
|
128
|
+
- lib/generators/pgq/templates/spec.rb
|
117
129
|
- lib/pgq.rb
|
118
130
|
- lib/pgq/api.rb
|
119
131
|
- lib/pgq/consumer.rb
|
120
132
|
- lib/pgq/consumer_base.rb
|
121
133
|
- lib/pgq/consumer_group.rb
|
122
134
|
- lib/pgq/event.rb
|
123
|
-
- lib/pgq/marshal64_coder.rb
|
124
135
|
- lib/pgq/railtie.rb
|
125
136
|
- lib/pgq/utils.rb
|
126
137
|
- lib/pgq/version.rb
|
127
138
|
- lib/pgq/worker.rb
|
128
139
|
- lib/tasks/pgq.rake
|
129
140
|
- pgq.gemspec
|
141
|
+
- spec/api_spec.rb
|
142
|
+
- spec/app/config/database.yml
|
143
|
+
- spec/benchmark/insert_bench.rb
|
130
144
|
- spec/consumer_base_spec.rb
|
131
145
|
- spec/consumer_group_spec.rb
|
146
|
+
- spec/consumer_intergration_spec.rb
|
132
147
|
- spec/consumer_spec.rb
|
133
148
|
- spec/event_spec.rb
|
134
|
-
- spec/marshal64_coder_spec.rb
|
135
149
|
- spec/spec_helper.rb
|
150
|
+
- spec/support/spec_support.rb
|
151
|
+
- spec/utils_spec.rb
|
136
152
|
- spec/worker_spec.rb
|
137
153
|
has_rdoc: true
|
138
154
|
homepage: http://github.com/kostya/pgq
|
@@ -164,9 +180,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
164
180
|
requirements: []
|
165
181
|
|
166
182
|
rubyforge_project:
|
167
|
-
rubygems_version: 1.
|
183
|
+
rubygems_version: 1.4.2
|
168
184
|
signing_key:
|
169
185
|
specification_version: 3
|
170
|
-
summary: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3
|
186
|
+
summary: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
|
171
187
|
test_files: []
|
172
188
|
|
data/Gemfile.lock
DELETED
@@ -1,46 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
pgq (0.1.2)
|
5
|
-
activerecord (>= 2.3.2)
|
6
|
-
activesupport (>= 2.3.2)
|
7
|
-
pg
|
8
|
-
|
9
|
-
GEM
|
10
|
-
remote: http://rubygems.org/
|
11
|
-
specs:
|
12
|
-
activemodel (3.2.3)
|
13
|
-
activesupport (= 3.2.3)
|
14
|
-
builder (~> 3.0.0)
|
15
|
-
activerecord (3.2.3)
|
16
|
-
activemodel (= 3.2.3)
|
17
|
-
activesupport (= 3.2.3)
|
18
|
-
arel (~> 3.0.2)
|
19
|
-
tzinfo (~> 0.3.29)
|
20
|
-
activesupport (3.2.3)
|
21
|
-
i18n (~> 0.6)
|
22
|
-
multi_json (~> 1.0)
|
23
|
-
arel (3.0.2)
|
24
|
-
builder (3.0.0)
|
25
|
-
diff-lcs (1.1.3)
|
26
|
-
i18n (0.6.0)
|
27
|
-
multi_json (1.3.5)
|
28
|
-
pg (0.13.2)
|
29
|
-
rake (0.9.2.2)
|
30
|
-
rspec (2.10.0)
|
31
|
-
rspec-core (~> 2.10.0)
|
32
|
-
rspec-expectations (~> 2.10.0)
|
33
|
-
rspec-mocks (~> 2.10.0)
|
34
|
-
rspec-core (2.10.0)
|
35
|
-
rspec-expectations (2.10.0)
|
36
|
-
diff-lcs (~> 1.1.3)
|
37
|
-
rspec-mocks (2.10.1)
|
38
|
-
tzinfo (0.3.33)
|
39
|
-
|
40
|
-
PLATFORMS
|
41
|
-
ruby
|
42
|
-
|
43
|
-
DEPENDENCIES
|
44
|
-
pgq!
|
45
|
-
rake
|
46
|
-
rspec
|
@@ -1,18 +0,0 @@
|
|
1
|
-
module Pgq
|
2
|
-
class ConfigGenerator < Rails::Generators::Base
|
3
|
-
source_root File.expand_path("../config_templates", __FILE__)
|
4
|
-
|
5
|
-
def copy_configs
|
6
|
-
self.env_name = 'development'
|
7
|
-
template "pgq.rb", "config/pgq_development.ini"
|
8
|
-
self.env_name = 'test'
|
9
|
-
template "pgq.rb", "config/pgq_test.ini"
|
10
|
-
self.env_name = 'production'
|
11
|
-
template "pgq.rb", "config/pgq_production.ini"
|
12
|
-
end
|
13
|
-
|
14
|
-
private
|
15
|
-
attr_accessor :env_name
|
16
|
-
|
17
|
-
end
|
18
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
[pgqadm]
|
2
|
-
job_name = pgqadm_<%= env_name %>
|
3
|
-
db = dbname=dbname user=user password=password host=127.0.0.1 port=5432
|
4
|
-
|
5
|
-
# how often to run maintenance [seconds]
|
6
|
-
maint_delay = 600
|
7
|
-
|
8
|
-
# how often to check for activity [seconds]
|
9
|
-
loop_delay = 0.1
|
10
|
-
|
11
|
-
logfile = log/%(job_name)s.log
|
12
|
-
pidfile = tmp/%(job_name)s.pid
|
data/lib/pgq/marshal64_coder.rb
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
-
|
3
|
-
describe Pgq::Marshal64Coder do
|
4
|
-
before :each do
|
5
|
-
@coder = Pgq::Marshal64Coder
|
6
|
-
end
|
7
|
-
|
8
|
-
it "work" do
|
9
|
-
data = {:a => [1, 2, 3, {:b => 'c'}]}
|
10
|
-
s = @coder.dump(data)
|
11
|
-
@coder.load(s).should == data
|
12
|
-
end
|
13
|
-
|
14
|
-
it "work on large data" do
|
15
|
-
data = {}
|
16
|
-
10000.times{|i| data["#{i}".to_sym] = [i, "i_#{i}"]}
|
17
|
-
|
18
|
-
s = @coder.dump(data)
|
19
|
-
@coder.load(s).should == data
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|