pgq 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore CHANGED
@@ -1,4 +1,8 @@
1
1
  TODO
2
2
  *.gem
3
3
  lib/serv
4
- README2
4
+ README2
5
+ Gemfile.lock
6
+ pgq_test.ini
7
+ log/
8
+ tmp/
data/README.md CHANGED
@@ -1,7 +1,7 @@
1
- Pgq Rails
2
- =========
1
+ Pgq
2
+ ===
3
3
 
4
- Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
4
+ Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
5
5
 
6
6
  About PgQ
7
7
  * http://wiki.postgresql.org/wiki/SkyTools#PgQ
@@ -22,9 +22,9 @@ Gemfile:
22
22
  gem 'pgq'
23
23
  ```
24
24
 
25
- Create ticker configs:
25
+ Create ticker configs from database.yml:
26
26
 
27
- $ rails generate pgq:config
27
+ $ rake pgq:generate_configs
28
28
  edit config: config/pgq_development.ini
29
29
 
30
30
 
@@ -46,7 +46,7 @@ Daemon run once, bind to the database. (If worker not consuming check that daemo
46
46
 
47
47
  Last, add to config/application.rb
48
48
 
49
- config.autoload_paths += %W( #{config.root}/app/models/pgq )
49
+ config.autoload_paths += %W( #{config.root}/app/workers )
50
50
 
51
51
 
52
52
  Create Pgq consumer
@@ -55,7 +55,7 @@ Create Pgq consumer
55
55
  $ rails generate pgq:add my
56
56
 
57
57
 
58
- This creates file app/models/pgq/pgq_my.rb and migration
58
+ This creates file app/workers/pgq_my.rb and migration
59
59
 
60
60
  $ rake db:migrate
61
61
 
@@ -174,7 +174,7 @@ class Pgq::Consumer
174
174
 
175
175
  # specify coder
176
176
  def self.coder
177
- Pgq::Marshal64Coder
177
+ Marshal64 # class/module with self methods dump/load
178
178
  end
179
179
 
180
180
  end
@@ -188,4 +188,17 @@ Usefull in specs
188
188
  ```
189
189
 
190
190
  When code call PgqMy.some_method1(a,b,c) this would be convert into PgqMy.new.some_method1(a,b,c)
191
-
191
+
192
+
193
+ ### for Rails 2.3
194
+
195
+ Add to Rakefile:
196
+ ```
197
+ load 'tasks/pgq.rake'
198
+ ```
199
+
200
+ Copy generators from gem to lib/generators.
201
+
202
+ ```
203
+ ./script/generate pgq bla
204
+ ```
data/Rakefile CHANGED
@@ -3,7 +3,24 @@ require 'bundler/setup'
3
3
  require 'bundler'
4
4
  Bundler::GemHelper.install_tasks
5
5
 
6
+ load 'lib/tasks/pgq.rake'
7
+
6
8
  require 'rspec/core/rake_task'
7
9
 
8
10
  task :default => :spec
9
- RSpec::Core::RakeTask.new(:spec)
11
+ RSpec::Core::RakeTask.new(:spec)
12
+
13
+ desc "set_env"
14
+ task :set_env do
15
+ ENV["PGQFILE"] = File.join(File.dirname(__FILE__), %w{ spec app config pgq_test.ini })
16
+ ENV["RAILS_ROOT"] = File.join(File.dirname(__FILE__), %w{ spec app })
17
+ ENV["RAILS_ENV"] = "test"
18
+ end
19
+
20
+ desc "prepare test"
21
+ task :prepare_test => :set_env do
22
+ Rake::Task["pgq:generate_config"].invoke
23
+ Rake::Task["pgq:install"].invoke
24
+ end
25
+
26
+ task :spec => :prepare_test
@@ -1,8 +1,10 @@
1
+ # for Rails 3
2
+ if defined?(Rails) && Rails::VERSION::MAJOR >= 3
3
+
1
4
  module Pgq
2
- class AddGenerator < Rails::Generators::Base
5
+ class AddGenerator < Rails::Generators::NamedBase
3
6
  include Rails::Generators::Migration
4
- source_root File.expand_path("../add_templates", __FILE__)
5
- argument :queue_name, :type => :string
7
+ source_root File.expand_path("../templates", __FILE__)
6
8
 
7
9
  def self.next_migration_number(path)
8
10
  Time.now.utc.strftime("%Y%m%d%H%M%S")
@@ -21,20 +23,12 @@ module Pgq
21
23
  end
22
24
 
23
25
  def add_files
24
- template "pgq_class.rb", "app/models/pgq/pgq_#{name}.rb"
25
- template "spec.rb", "spec/models/pgq/pgq_#{name}_spec.rb"
26
- migration_template "migration.rb", "db/migrate/create_#{name}_queue.rb"
27
- end
28
-
29
- private
30
-
31
- def name
32
- queue_name.underscore
33
- end
34
-
35
- def name_c
36
- name.camelize
26
+ template "pgq_class.rb", "app/workers/pgq_#{file_path}.rb"
27
+ template "spec.rb", "spec/workers/pgq_#{file_path}_spec.rb"
28
+ migration_template "migration.rb", "db/migrate/create_#{file_path}_queue.rb"
37
29
  end
38
30
  end
39
31
 
40
32
  end
33
+
34
+ end
@@ -0,0 +1,34 @@
1
+ # for Rails 2
2
+ if defined?(Rails) && Rails::VERSION::MAJOR == 2
3
+
4
+ module Pgq
5
+
6
+ class PgqGenerator < Rails::Generator::NamedBase
7
+ def manifest
8
+ record do |m|
9
+ m.template "pgq_class.rb", "app/workers/pgq_#{file_path}.rb"
10
+ m.directory "spec/workers"
11
+ m.template "spec.rb", "spec/workers/pgq_#{file_path}_spec.rb"
12
+ m.template "migration.rb", "db/migrate/#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_create_#{file_path}_queue.rb"
13
+
14
+ add_to_queues_list
15
+ end
16
+ end
17
+
18
+ private
19
+ def add_to_queues_list
20
+ filename = Rails.root + 'config/queues_list.yml'
21
+ if File.exists?(filename)
22
+ x = YAML.load_file(filename)
23
+ x << name
24
+ File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
25
+ else
26
+ x = [name]
27
+ File.open(filename, 'w'){|f| f.write(YAML.dump(x))}
28
+ end
29
+ end
30
+ end
31
+
32
+ end
33
+
34
+ end
@@ -0,0 +1,9 @@
1
+ class Create<%= class_name %>Queue < ActiveRecord::Migration
2
+ def self.up
3
+ Pgq::Consumer.add_queue :<%= file_path %>
4
+ end
5
+
6
+ def self.down
7
+ Pgq::Consumer.remove_queue :<%= file_path %>
8
+ end
9
+ end
@@ -1,6 +1,6 @@
1
- class Pgq<%= name_c %> < Pgq::Consumer
1
+ class Pgq<%= class_name %> < Pgq::Consumer
2
2
 
3
- # to insert event: Pgq<%= name_c %>.my_event(1, 2, 3)
3
+ # insert event: Pgq<%= class_name %>.my_event(1, 2, 3)
4
4
 
5
5
  # async execute
6
6
  def my_event(a, b, c)
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ describe Pgq<%= class_name %> do
4
+ before(:each) do
5
+ @consumer = Pgq<%= class_name %>.new
6
+ end
7
+
8
+ end
data/lib/pgq.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'active_support'
2
2
  require 'active_record'
3
+ require 'marshal64'
3
4
 
4
5
  module Pgq
5
6
  end
@@ -68,7 +68,7 @@ module Pgq::Api
68
68
  end
69
69
 
70
70
  def pgq_failed_event_list queue_name, consumer_name, limit = nil, offset = nil, order = 'desc'
71
- order = (order.to_s == 'desc') ? order : 'asc'
71
+ order = (order.to_s == 'desc') ? 'desc' : 'asc'
72
72
  connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) ORDER BY ev_id #{order.upcase}", queue_name, consumer_name, limit.to_i, offset.to_i])
73
73
  end
74
74
 
@@ -128,5 +128,9 @@ module Pgq::Api
128
128
 
129
129
  events.length
130
130
  end
131
-
131
+
132
+ def pgq_force_tick(queue_name)
133
+ connection.select_value(sanitize_sql_array ["SELECT pgq.force_tick(?)", queue_name]).to_i
134
+ end
135
+
132
136
  end
@@ -60,7 +60,7 @@ class Pgq::ConsumerBase
60
60
  # == coder
61
61
 
62
62
  def self.coder
63
- Pgq::Marshal64Coder
63
+ Marshal64
64
64
  end
65
65
 
66
66
  def coder
@@ -4,13 +4,13 @@ class Pgq::Event
4
4
  def initialize(consumer, event)
5
5
  @id = event['ev_id']
6
6
  @type = event['ev_type']
7
- @data = consumer.coder.load(event['ev_data']) if event['ev_data']
8
7
  @consumer = consumer
8
+ @data = @consumer.coder.load(event['ev_data']) if event['ev_data']
9
9
  end
10
10
 
11
11
  def failed!(ex)
12
12
  h = {:class => ex.class.to_s, :message => ex.message, :backtrace => ex.backtrace}
13
- @consumer.event_failed @id, consumer.coder.dump(h)
13
+ @consumer.event_failed @id, @consumer.coder.dump(h)
14
14
  end
15
15
 
16
16
  def retry!(seconds = 0)
@@ -29,8 +29,8 @@ Backtrace: #{e.backtrace.join("\n") rescue ''}
29
29
  def exception_message(e)
30
30
  <<-EXCEPTION
31
31
  Exception happend
32
- Type: #{type.inspect}
33
- Data: #{data.inspect}
32
+ Type: #{@type.inspect}
33
+ Data: #{@data.inspect}
34
34
  Error occurs: #{e.class.inspect}(#{e.message})
35
35
  Backtrace: #{e.backtrace.join("\n") rescue ''}
36
36
  EXCEPTION
@@ -1,3 +1,3 @@
1
1
  module Pgq
2
- VERSION = "0.1.2"
2
+ VERSION = "0.1.3"
3
3
  end
@@ -33,7 +33,9 @@ class Pgq::Worker
33
33
  raise "Queue not selected" if @queues.blank?
34
34
 
35
35
  if @queues == ['all'] || @queues == 'all'
36
- if defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
36
+ if h[:queues_list]
37
+ @queues = YAML.load_file(h[:queues_list])
38
+ elsif defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
37
39
  @queues = YAML.load_file(Rails.root + "config/queues_list.yml")
38
40
  else
39
41
  raise "You shoud create config/queues_list.yml for all queues"
@@ -1,10 +1,15 @@
1
+ if defined?(Rails)
2
+ ENV['RAILS_ROOT'] ||= Rails.root.to_s
3
+ ENV['RAILS_ENV'] ||= Rails.env.to_s
4
+ end
5
+
1
6
  namespace :pgq do
2
7
 
3
8
  desc "Start worker params: QUEUES, LOGGER, WATCH_FILE"
4
9
  task :worker => [:environment] do
5
10
  queues = ENV['QUEUES']
6
- logger_file = ENV['LOGGER'] || Rails.root.join("log/pgq_#{queues}.log")
7
- watch_file = ENV['WATCH_FILE'] || Rails.root.join("tmp/pgq_#{queues}.stop")
11
+ logger_file = ENV['LOGGER'] || "#{ENV['RAILS_ROOT']}/log/pgq_#{queues}.log"
12
+ watch_file = ENV['WATCH_FILE'] || "#{ENV['RAILS_ROOT']}/tmp/pgq_#{queues}.stop"
8
13
  logger = Logger.new(logger_file)
9
14
  w = Pgq::Worker.new(:logger => logger, :queues => queues, :watch_file => watch_file)
10
15
  w.run
@@ -12,7 +17,7 @@ namespace :pgq do
12
17
 
13
18
  desc "Install PgQ to database RAILS_ENV"
14
19
  task :install do
15
- Dir["#{Rails.root}/config/pgq_#{Rails.env}*.ini"].each do |conf|
20
+ Dir["#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}*.ini"].each do |conf|
16
21
  ENV['PGQFILE']=conf
17
22
  Rake::Task['pgq:install_from_file'].invoke
18
23
  end
@@ -34,13 +39,57 @@ namespace :pgq do
34
39
  raise "Something went wrong(see above)... Check that you install skytools package and create #{conf}"
35
40
  end
36
41
  end
42
+
43
+ desc "Generate Pgq config from database.yml and RAILS_ENV"
44
+ task :generate_config do
45
+ require 'yaml'
46
+ raise "RAILS_ROOT should be" unless ENV['RAILS_ROOT']
47
+ raise "RAILS_ENV should be" unless ENV['RAILS_ENV']
48
+
49
+ cfg = if ENV['DATABASE_YML']
50
+ YAML.load_file(ENV['DATABASE_YML']) if File.exists?(ENV['DATABASE_YML'])
51
+ else
52
+ file = File.join(ENV['RAILS_ROOT'], %w{config database.yml})
53
+ YAML.load_file(file) if File.exists?(file)
54
+ end
55
+
56
+ raise "Not found database.yml" unless cfg
57
+ cfg = cfg[ ENV["RAILS_ENV"] ]
37
58
 
59
+ pgq_config = <<-SQL
60
+ [pgqadm]
61
+ job_name = #{cfg['database']}_pgqadm
62
+ db = host=#{cfg['host'] || '127.0.0.1'} dbname=#{cfg['database']} user=#{cfg['username']} password=#{cfg['password']} port=#{cfg['port'] || 5432}
63
+
64
+ # how often to run maintenance [seconds]
65
+ maint_delay = 500
66
+
67
+ # how often to check for activity [seconds]
68
+ loop_delay = 0.05
69
+
70
+ logfile = log/%(job_name)s.log
71
+ pidfile = tmp/%(job_name)s.pid
72
+ SQL
73
+
74
+ output = ENV["PGQ_CONFIG"] || File.join(ENV["RAILS_ROOT"], ["config", "pgq_#{ENV["RAILS_ENV"]}.ini"])
75
+ File.open(output, 'w'){|f| f.write(pgq_config) }
76
+ puts "Config #{ENV["RAILS_ENV"]} generated to '#{output}'"
77
+ end
78
+
79
+ desc "Generate Pgq configs from database.yml for all environments"
80
+ task :generate_configs do
81
+ task = Rake::Task["pgq:generate_config"]
82
+ %w{development test production}.each do |env|
83
+ ENV['RAILS_ENV'] = env
84
+ task.execute
85
+ end
86
+ end
38
87
 
39
88
  namespace :ticker do
40
89
 
41
90
  desc "Start PgQ ticker daemon"
42
91
  task :start do
43
- conf = Rails.root.join("config/pgq_#{Rails.env}.ini")
92
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
44
93
  output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
45
94
  if output.empty?
46
95
  puts "ticker daemon started"
@@ -51,7 +100,7 @@ namespace :pgq do
51
100
 
52
101
  desc "Stop PgQ ticker daemon"
53
102
  task :stop do
54
- conf = Rails.root.join("config/pgq_#{Rails.env}.ini")
103
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
55
104
  output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
56
105
  if output.empty?
57
106
  puts "ticker daemon stoped"
@@ -9,8 +9,8 @@ Gem::Specification.new do |s|
9
9
  s.authors = ["Makarchev Konstantin"]
10
10
  s.autorequire = %q{init}
11
11
 
12
- s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.}
13
- s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.}
12
+ s.description = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.}
13
+ s.summary = %q{Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.}
14
14
 
15
15
  s.email = %q{kostya27@gmail.com}
16
16
  s.homepage = %q{http://github.com/kostya/pgq}
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
20
20
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
21
21
  s.require_paths = ["lib"]
22
22
 
23
- s.add_dependency 'activesupport', ">=2.3.2"
24
- s.add_dependency 'activerecord', ">=2.3.2"
23
+ s.add_dependency 'activesupport', ">= 2.3.2"
24
+ s.add_dependency 'activerecord', ">= 2.3.2"
25
25
  s.add_dependency 'pg'
26
+ s.add_dependency 'marshal64'
26
27
 
27
28
  s.add_development_dependency "rspec"
28
29
  s.add_development_dependency "rake"
@@ -0,0 +1,48 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Pgq::Api do
4
+ before :each do
5
+ re_queues
6
+ end
7
+
8
+ after :each do
9
+ stop_ticker rescue nil
10
+ end
11
+
12
+ it "should insert event" do
13
+ x = ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
14
+ x.should > 0
15
+ end
16
+
17
+ it "consuming" do
18
+ ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
19
+
20
+ bid, events = api_extract_batch('bla')
21
+
22
+ events.size.should == 1
23
+ event = events.first
24
+
25
+ event['ev_extra1'].should == '5,6,7'
26
+ event['ev_type'].should == 'tp'
27
+ event['ev_id'].to_i.should >= 1
28
+ event['ev_data'].should == '1,2,3'
29
+ end
30
+
31
+ it "failed event and retry" do
32
+ ActiveRecord::Base.pgq_failed_event_count('bla', 'default').should == 0
33
+ ActiveRecord::Base.pgq_failed_event_list('bla', 'default').should == []
34
+
35
+ ActiveRecord::Base.pgq_insert_event('bla', 'tp', '1,2,3', '5,6,7')
36
+
37
+ bid, events = api_extract_batch('bla')
38
+
39
+ ActiveRecord::Base.pgq_event_failed(bid, events[0]['ev_id'], 'aaa')
40
+
41
+ ActiveRecord::Base.pgq_finish_batch(bid)
42
+
43
+ ActiveRecord::Base.pgq_force_tick('bla')
44
+ ActiveRecord::Base.pgq_failed_event_count('bla', 'default').should == 1
45
+ # ActiveRecord::Base.pgq_failed_event_list('bla', 'default').should == [...]
46
+ end
47
+
48
+ end
@@ -0,0 +1,6 @@
1
+ test:
2
+ database: gem_pgq_test
3
+ adapter: postgresql
4
+ username: kostya
5
+ password: password
6
+
@@ -0,0 +1,30 @@
1
+ require 'rubygems'
2
+ require "bundler"
3
+ Bundler.setup
4
+
5
+ $:.unshift(File.dirname(__FILE__) + '/../../lib')
6
+ require 'pgq'
7
+ require 'benchmark'
8
+
9
+ require File.join(File.dirname(__FILE__), %w{.. support spec_support})
10
+
11
+ N = 5000
12
+ puts 'start benchmark'
13
+
14
+ re_queues
15
+ tm = Benchmark.realtime{ N.times{ PgqBla.bla } }
16
+ puts "insert without arguments method missing #{tm}"
17
+
18
+ re_queues
19
+ tm = Benchmark.realtime{ N.times{ PgqBla.add_event 'bla' } }
20
+ puts "insert without arguments add_event #{tm}"
21
+
22
+ x = (0..100).to_a.map{|c| {1 => c, :b => "a#{c}", 2 => c + 2}}
23
+
24
+ re_queues
25
+ tm = Benchmark.realtime{ N.times{ PgqBla.bla x } }
26
+ puts "insert arguments method missing #{tm}"
27
+
28
+ re_queues
29
+ tm = Benchmark.realtime{ N.times{ PgqBla.add_event 'bla', x } }
30
+ puts "insert arguments add_event #{tm}"
@@ -150,4 +150,4 @@ describe Pgq::ConsumerBase do
150
150
  $b.should == 'abc'
151
151
  end
152
152
 
153
- end
153
+ end
@@ -0,0 +1,4 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Pgq::ConsumerBase do
4
+ end
@@ -1,15 +1,5 @@
1
1
  require File.dirname(__FILE__) + '/spec_helper'
2
2
 
3
- class PgqHaha < Pgq::Consumer
4
-
5
- def ptest2(a, b)
6
- $a = a
7
- $b = b
8
- 10
9
- end
10
-
11
- end
12
-
13
3
  describe PgqHaha do
14
4
  before :each do
15
5
  @consumer = PgqHaha.new
@@ -2,8 +2,7 @@ require File.dirname(__FILE__) + '/spec_helper'
2
2
 
3
3
  describe Pgq::Event do
4
4
  before :each do
5
- kl = Class.new(Pgq::ConsumerGroup) do
6
- end
5
+ kl = Class.new(Pgq::ConsumerGroup){}
7
6
  @consumer = kl.new
8
7
  @consumer.stub!(:get_next_batch).and_return ''
9
8
  @consumer.stub!(:finish_batch).and_return ''
@@ -3,9 +3,10 @@ require "bundler/setup"
3
3
 
4
4
  Bundler.require
5
5
 
6
+ require File.join(File.dirname(__FILE__), %w{support spec_support})
7
+
6
8
  class Pgq::Consumer
7
9
  # rspec fuckup
8
10
  def self.to_ary
9
11
  end
10
12
  end
11
-
@@ -0,0 +1,70 @@
1
+ require 'yaml'
2
+
3
+ file = File.join(File.dirname(__FILE__), %w{.. app config database.yml})
4
+ cfg = YAML.load_file(file)
5
+
6
+ ActiveRecord::Base.establish_connection cfg['test']
7
+
8
+ def re_queues
9
+ Pgq::Consumer.remove_queue(:bla) rescue nil
10
+ Pgq::Consumer.remove_queue(:test) rescue nil
11
+
12
+ Pgq::Consumer.add_queue :bla
13
+ Pgq::Consumer.add_queue :test
14
+ end
15
+
16
+ re_queues
17
+
18
+ class PgqBla < Pgq::Consumer
19
+ end
20
+
21
+ class PgqTest < Pgq::Consumer
22
+ end
23
+
24
+ class PgqHaha < Pgq::Consumer
25
+
26
+ def ptest2(a, b)
27
+ $a = a
28
+ $b = b
29
+ 10
30
+ end
31
+
32
+ end
33
+
34
+ def start_ticker
35
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
36
+ output = `which pgqadm.py && pgqadm.py #{conf} -d ticker 2>&1 || which pgqadm && pgqadm #{conf} -d ticker 2>&1`
37
+ end
38
+
39
+ def stop_ticker
40
+ conf = "#{ENV['RAILS_ROOT']}/config/pgq_#{ENV['RAILS_ENV']}.ini"
41
+ output = `which pgqadm.py && pgqadm.py #{conf} -s 2>&1 || which pgqadm && pgqadm #{conf} -s 2>&1`
42
+ end
43
+
44
+ def api_extract_batch(queue_name, consumer_name = 'default')
45
+ start_ticker
46
+ bid = nil
47
+ events = []
48
+ count = 0
49
+
50
+ loop do
51
+ 100.times{ ActiveRecord::Base.pgq_force_tick(queue_name) }
52
+
53
+ bid = ActiveRecord::Base.pgq_next_batch(queue_name, consumer_name)
54
+
55
+ if bid
56
+ events = ActiveRecord::Base.pgq_get_batch_events(bid)
57
+ if events.present?
58
+ break
59
+ end
60
+ end
61
+
62
+ count += 1
63
+ raise "count > 100" if count > 100
64
+ end
65
+
66
+ return [bid, events]
67
+
68
+ ensure
69
+ stop_ticker
70
+ end
@@ -0,0 +1,14 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Pgq::Utils do
4
+ after :each do
5
+ Pgq::Consumer.remove_queue :hoho
6
+ end
7
+
8
+ it "queues list" do
9
+ Pgq::Consumer.queues_list.sort.should == %w{bla test}.sort
10
+
11
+ Pgq::Consumer.add_queue :hoho
12
+ Pgq::Consumer.queues_list.sort.should == %w{bla test hoho}.sort
13
+ end
14
+ end
@@ -1,9 +1,6 @@
1
1
  require File.dirname(__FILE__) + '/spec_helper'
2
2
 
3
3
  describe Pgq::Worker do
4
- class PgqBla < Pgq::Consumer
5
- end
6
-
7
4
  it "should find class not in hash" do
8
5
  Pgq::Worker.predict_queue_class('bla').should == PgqBla
9
6
  Pgq::Worker.predict_queue_class('bla_1').should == PgqBla
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgq
3
3
  version: !ruby/object:Gem::Version
4
- hash: 31
5
- prerelease: false
4
+ hash: 29
5
+ prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 1
9
- - 2
10
- version: 0.1.2
9
+ - 3
10
+ version: 0.1.3
11
11
  platform: ruby
12
12
  authors:
13
13
  - Makarchev Konstantin
@@ -15,13 +15,11 @@ autorequire: init
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-05-26 00:00:00 +04:00
18
+ date: 2012-09-22 00:00:00 +04:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
22
- name: activesupport
23
- prerelease: false
24
- requirement: &id001 !ruby/object:Gem::Requirement
22
+ version_requirements: &id001 !ruby/object:Gem::Requirement
25
23
  none: false
26
24
  requirements:
27
25
  - - ">="
@@ -32,12 +30,12 @@ dependencies:
32
30
  - 3
33
31
  - 2
34
32
  version: 2.3.2
33
+ prerelease: false
35
34
  type: :runtime
36
- version_requirements: *id001
35
+ requirement: *id001
36
+ name: activesupport
37
37
  - !ruby/object:Gem::Dependency
38
- name: activerecord
39
- prerelease: false
40
- requirement: &id002 !ruby/object:Gem::Requirement
38
+ version_requirements: &id002 !ruby/object:Gem::Requirement
41
39
  none: false
42
40
  requirements:
43
41
  - - ">="
@@ -48,12 +46,12 @@ dependencies:
48
46
  - 3
49
47
  - 2
50
48
  version: 2.3.2
49
+ prerelease: false
51
50
  type: :runtime
52
- version_requirements: *id002
51
+ requirement: *id002
52
+ name: activerecord
53
53
  - !ruby/object:Gem::Dependency
54
- name: pg
55
- prerelease: false
56
- requirement: &id003 !ruby/object:Gem::Requirement
54
+ version_requirements: &id003 !ruby/object:Gem::Requirement
57
55
  none: false
58
56
  requirements:
59
57
  - - ">="
@@ -62,12 +60,26 @@ dependencies:
62
60
  segments:
63
61
  - 0
64
62
  version: "0"
63
+ prerelease: false
65
64
  type: :runtime
66
- version_requirements: *id003
65
+ requirement: *id003
66
+ name: pg
67
67
  - !ruby/object:Gem::Dependency
68
- name: rspec
68
+ version_requirements: &id004 !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ hash: 3
74
+ segments:
75
+ - 0
76
+ version: "0"
69
77
  prerelease: false
70
- requirement: &id004 !ruby/object:Gem::Requirement
78
+ type: :runtime
79
+ requirement: *id004
80
+ name: marshal64
81
+ - !ruby/object:Gem::Dependency
82
+ version_requirements: &id005 !ruby/object:Gem::Requirement
71
83
  none: false
72
84
  requirements:
73
85
  - - ">="
@@ -76,12 +88,12 @@ dependencies:
76
88
  segments:
77
89
  - 0
78
90
  version: "0"
91
+ prerelease: false
79
92
  type: :development
80
- version_requirements: *id004
93
+ requirement: *id005
94
+ name: rspec
81
95
  - !ruby/object:Gem::Dependency
82
- name: rake
83
- prerelease: false
84
- requirement: &id005 !ruby/object:Gem::Requirement
96
+ version_requirements: &id006 !ruby/object:Gem::Requirement
85
97
  none: false
86
98
  requirements:
87
99
  - - ">="
@@ -90,9 +102,11 @@ dependencies:
90
102
  segments:
91
103
  - 0
92
104
  version: "0"
105
+ prerelease: false
93
106
  type: :development
94
- version_requirements: *id005
95
- description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
107
+ requirement: *id006
108
+ name: rake
109
+ description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
96
110
  email: kostya27@gmail.com
97
111
  executables: []
98
112
 
@@ -103,36 +117,38 @@ extra_rdoc_files: []
103
117
  files:
104
118
  - .gitignore
105
119
  - Gemfile
106
- - Gemfile.lock
107
120
  - LICENSE
108
121
  - README.md
109
122
  - Rakefile
110
123
  - init.rb
111
124
  - lib/generators/pgq/add_generator.rb
112
- - lib/generators/pgq/add_templates/migration.rb
113
- - lib/generators/pgq/add_templates/pgq_class.rb
114
- - lib/generators/pgq/add_templates/spec.rb
115
- - lib/generators/pgq/config_generator.rb
116
- - lib/generators/pgq/config_templates/pgq.rb
125
+ - lib/generators/pgq/pgq_generator.rb
126
+ - lib/generators/pgq/templates/migration.rb
127
+ - lib/generators/pgq/templates/pgq_class.rb
128
+ - lib/generators/pgq/templates/spec.rb
117
129
  - lib/pgq.rb
118
130
  - lib/pgq/api.rb
119
131
  - lib/pgq/consumer.rb
120
132
  - lib/pgq/consumer_base.rb
121
133
  - lib/pgq/consumer_group.rb
122
134
  - lib/pgq/event.rb
123
- - lib/pgq/marshal64_coder.rb
124
135
  - lib/pgq/railtie.rb
125
136
  - lib/pgq/utils.rb
126
137
  - lib/pgq/version.rb
127
138
  - lib/pgq/worker.rb
128
139
  - lib/tasks/pgq.rake
129
140
  - pgq.gemspec
141
+ - spec/api_spec.rb
142
+ - spec/app/config/database.yml
143
+ - spec/benchmark/insert_bench.rb
130
144
  - spec/consumer_base_spec.rb
131
145
  - spec/consumer_group_spec.rb
146
+ - spec/consumer_intergration_spec.rb
132
147
  - spec/consumer_spec.rb
133
148
  - spec/event_spec.rb
134
- - spec/marshal64_coder_spec.rb
135
149
  - spec/spec_helper.rb
150
+ - spec/support/spec_support.rb
151
+ - spec/utils_spec.rb
136
152
  - spec/worker_spec.rb
137
153
  has_rdoc: true
138
154
  homepage: http://github.com/kostya/pgq
@@ -164,9 +180,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
164
180
  requirements: []
165
181
 
166
182
  rubyforge_project:
167
- rubygems_version: 1.3.7
183
+ rubygems_version: 1.4.2
168
184
  signing_key:
169
185
  specification_version: 3
170
- summary: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
186
+ summary: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 2.3 and 3 compatible.
171
187
  test_files: []
172
188
 
@@ -1,46 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- pgq (0.1.2)
5
- activerecord (>= 2.3.2)
6
- activesupport (>= 2.3.2)
7
- pg
8
-
9
- GEM
10
- remote: http://rubygems.org/
11
- specs:
12
- activemodel (3.2.3)
13
- activesupport (= 3.2.3)
14
- builder (~> 3.0.0)
15
- activerecord (3.2.3)
16
- activemodel (= 3.2.3)
17
- activesupport (= 3.2.3)
18
- arel (~> 3.0.2)
19
- tzinfo (~> 0.3.29)
20
- activesupport (3.2.3)
21
- i18n (~> 0.6)
22
- multi_json (~> 1.0)
23
- arel (3.0.2)
24
- builder (3.0.0)
25
- diff-lcs (1.1.3)
26
- i18n (0.6.0)
27
- multi_json (1.3.5)
28
- pg (0.13.2)
29
- rake (0.9.2.2)
30
- rspec (2.10.0)
31
- rspec-core (~> 2.10.0)
32
- rspec-expectations (~> 2.10.0)
33
- rspec-mocks (~> 2.10.0)
34
- rspec-core (2.10.0)
35
- rspec-expectations (2.10.0)
36
- diff-lcs (~> 1.1.3)
37
- rspec-mocks (2.10.1)
38
- tzinfo (0.3.33)
39
-
40
- PLATFORMS
41
- ruby
42
-
43
- DEPENDENCIES
44
- pgq!
45
- rake
46
- rspec
@@ -1,9 +0,0 @@
1
- class Create<%= name_c %>Queue < ActiveRecord::Migration
2
- def self.up
3
- Pgq::Consumer.add_queue :<%= name %>
4
- end
5
-
6
- def self.down
7
- Pgq::Consumer.remove_queue :<%= name %>
8
- end
9
- end
@@ -1,8 +0,0 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
2
-
3
- describe Pgq<%= name_c %> do
4
- before(:each) do
5
- @consumer = Pgq<%= name_c %>.new
6
- end
7
-
8
- end
@@ -1,18 +0,0 @@
1
- module Pgq
2
- class ConfigGenerator < Rails::Generators::Base
3
- source_root File.expand_path("../config_templates", __FILE__)
4
-
5
- def copy_configs
6
- self.env_name = 'development'
7
- template "pgq.rb", "config/pgq_development.ini"
8
- self.env_name = 'test'
9
- template "pgq.rb", "config/pgq_test.ini"
10
- self.env_name = 'production'
11
- template "pgq.rb", "config/pgq_production.ini"
12
- end
13
-
14
- private
15
- attr_accessor :env_name
16
-
17
- end
18
- end
@@ -1,12 +0,0 @@
1
- [pgqadm]
2
- job_name = pgqadm_<%= env_name %>
3
- db = dbname=dbname user=user password=password host=127.0.0.1 port=5432
4
-
5
- # how often to run maintenance [seconds]
6
- maint_delay = 600
7
-
8
- # how often to check for activity [seconds]
9
- loop_delay = 0.1
10
-
11
- logfile = log/%(job_name)s.log
12
- pidfile = tmp/%(job_name)s.pid
@@ -1,13 +0,0 @@
1
- require 'base64'
2
-
3
- module Pgq::Marshal64Coder
4
-
5
- def self.dump(s)
6
- Base64::encode64(Marshal.dump(s))
7
- end
8
-
9
- def self.load(s)
10
- Marshal.load(Base64::decode64(s))
11
- end
12
-
13
- end
@@ -1,22 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe Pgq::Marshal64Coder do
4
- before :each do
5
- @coder = Pgq::Marshal64Coder
6
- end
7
-
8
- it "work" do
9
- data = {:a => [1, 2, 3, {:b => 'c'}]}
10
- s = @coder.dump(data)
11
- @coder.load(s).should == data
12
- end
13
-
14
- it "work on large data" do
15
- data = {}
16
- 10000.times{|i| data["#{i}".to_sym] = [i, "i_#{i}"]}
17
-
18
- s = @coder.dump(data)
19
- @coder.load(s).should == data
20
- end
21
-
22
- end