queue_classic_batches 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +66 -0
- data/Rakefile +1 -0
- data/lib/generators/queue_classic_batches/install_generator.rb +24 -0
- data/lib/generators/queue_classic_batches/templates/add_queue_classic_batches.rb +12 -0
- data/lib/queue_classic_batches.rb +15 -0
- data/lib/queue_classic_batches/batch.rb +97 -0
- data/lib/queue_classic_batches/queries.rb +50 -0
- data/lib/queue_classic_batches/queue.rb +35 -0
- data/lib/queue_classic_batches/setup.rb +26 -0
- data/lib/queue_classic_batches/version.rb +3 -0
- data/lib/queue_classic_batches/worker.rb +15 -0
- data/queue_classic_batches.gemspec +26 -0
- data/spec/batch_spec.rb +139 -0
- data/spec/helper.rb +26 -0
- data/spec/queue_spec.rb +29 -0
- data/spec/worker_spec.rb +70 -0
- data/sql/add_columns.sql +4 -0
- data/sql/create_batches.sql +20 -0
- metadata +138 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f4efc7cc0a30ae0cbe1aaef21e52f0e04f1c4e4b
|
4
|
+
data.tar.gz: e7cf187ff93fd137e2b1c34c930ad1b47305e0c1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d03b53bc8f5400d2099c9f2f16c991b8b2f76c363dd01173011a38d444766df640a610ed70768c631fd369c1832f22517cbab26a9d2e2d44190c88645abde100
|
7
|
+
data.tar.gz: 87bef6b26296dd98c915f8d77403f04bf9b7c8c7fb4b10354706c1e0af38d4bc80521fac07da274b64abb45ecf49e105151d72838730e07526dc5e142c7a74ac
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Shane Blazek
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
# queue_classic_batches
|
2
|
+
|
3
|
+
Adds support to queue_classic to enable queuing another job when a group of jobs have all completed.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'queue_classic_batches'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install queue_classic_batches
|
18
|
+
|
19
|
+
Ruby on Rails Setup
|
20
|
+
|
21
|
+
Declare dependencies in Gemfile.
|
22
|
+
|
23
|
+
source "http://rubygems.org"
|
24
|
+
gem "queue_classic_batches", "0.0.1"
|
25
|
+
|
26
|
+
Add the database tables and columns.
|
27
|
+
|
28
|
+
rails generate queue_classic_batches:install
|
29
|
+
rake db:migrate
|
30
|
+
|
31
|
+
## Usage
|
32
|
+
|
33
|
+
1. Create a batch
|
34
|
+
2. Queue jobs on the batch
|
35
|
+
3. Mark queuing complete
|
36
|
+
|
37
|
+
Here is an example:
|
38
|
+
|
39
|
+
batch = QC::Batches::Batch.create(complete_method:'MyCompleteJob.perform',
|
40
|
+
complete_args: [123, 'abc'],
|
41
|
+
complete_q_name: 'optional-queue-name')
|
42
|
+
(1..20) do |i|
|
43
|
+
batch.enqueue("MyJob.perform", i, "Job #{i}")
|
44
|
+
end
|
45
|
+
batch.queuing_complete
|
46
|
+
|
47
|
+
|
48
|
+
Make sure your worker deletes or re-queues failed jobs or else queue_classic will leave the job in the jobs table and the batch won't know it has been completed.
|
49
|
+
|
50
|
+
FailedQueue = QC::Queue.new("failed_jobs")
|
51
|
+
class MyWorker < QC::Worker
|
52
|
+
|
53
|
+
def handle_failure(job, e)
|
54
|
+
FailedQueue.enqueue(job[:method], *job[:args])
|
55
|
+
QC.delete job[:id]
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
## Contributing
|
61
|
+
|
62
|
+
1. Fork it
|
63
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
64
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
65
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
66
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'rails/generators'
|
2
|
+
require 'rails/generators/migration'
|
3
|
+
require 'active_record'
|
4
|
+
|
5
|
+
module QueueClassicBatches
|
6
|
+
class InstallGenerator < Rails::Generators::Base
|
7
|
+
include Rails::Generators::Migration
|
8
|
+
|
9
|
+
namespace "queue_classic_batches:install"
|
10
|
+
self.source_paths << File.join(File.dirname(__FILE__), 'templates')
|
11
|
+
desc 'Generates (but does not run) a migration to add batch functionality to the queue_classic table.'
|
12
|
+
|
13
|
+
def self.next_migration_number(dirname)
|
14
|
+
next_migration_number = current_migration_number(dirname) + 1
|
15
|
+
ActiveRecord::Migration.next_migration_number(next_migration_number)
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_migration_file
|
19
|
+
if self.class.migration_exists?('db/migrate', 'add_queue_classic_batches').nil?
|
20
|
+
migration_template 'add_queue_classic_batches.rb', 'db/migrate/add_queue_classic_batches.rb'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require "queue_classic_batches/version"
|
2
|
+
require "queue_classic_batches/queue"
|
3
|
+
require "queue_classic_batches/setup"
|
4
|
+
require "queue_classic_batches/queries"
|
5
|
+
require "queue_classic_batches/batch"
|
6
|
+
require "queue_classic_batches/worker"
|
7
|
+
|
8
|
+
module QC
|
9
|
+
module Batches
|
10
|
+
# Why do you want to change the table name?
|
11
|
+
# Just deal with the default OK?
|
12
|
+
# Come on. Don't do it.... Just stick with the default.
|
13
|
+
TABLE_NAME = "queue_classic_batches"
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module QC
|
2
|
+
module Batches
|
3
|
+
class Batch
|
4
|
+
attr_accessor :id
|
5
|
+
attr_accessor :queue
|
6
|
+
attr_accessor :created_at
|
7
|
+
attr_accessor :complete_q_name
|
8
|
+
attr_accessor :complete_method
|
9
|
+
attr_accessor :complete_args
|
10
|
+
|
11
|
+
def initialize(args=nil)
|
12
|
+
args.each { |k,v|
|
13
|
+
instance_variable_set("@#{k}", v) unless v.nil?
|
14
|
+
} if args.is_a?(Hash)
|
15
|
+
@queuing_complete = args[:queuing_complete] || false
|
16
|
+
end
|
17
|
+
|
18
|
+
def enqueue(method, *args)
|
19
|
+
(self.queue || QC).enqueue_batch method, self.id, args
|
20
|
+
end
|
21
|
+
|
22
|
+
def queuing_complete?
|
23
|
+
return @queuing_complete
|
24
|
+
end
|
25
|
+
|
26
|
+
def queuing_complete
|
27
|
+
@queuing_complete = true
|
28
|
+
Queries.save_queuing_complete(self.id)
|
29
|
+
Batch.complete_if_finished self.id
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.complete_if_finished(batch_id)
|
33
|
+
|
34
|
+
if Batch.finished?(batch_id)
|
35
|
+
QC.default_conn_adapter.connection.transaction do |conn|
|
36
|
+
batch = Batch.find(batch_id, lock: true)
|
37
|
+
return unless batch
|
38
|
+
batch.complete
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
def complete
|
45
|
+
return unless queuing_complete? && finished?
|
46
|
+
if complete_method
|
47
|
+
queue = complete_q_name ? Queue.new(queue) : QC
|
48
|
+
queue.enqueue complete_method, *complete_args
|
49
|
+
end
|
50
|
+
delete
|
51
|
+
|
52
|
+
time_to_complete = Integer((Time.now - created_at) * 1000)
|
53
|
+
QC.log(:'time-to-complete-batch'=>time_to_complete, :source=>id)
|
54
|
+
end
|
55
|
+
|
56
|
+
def delete
|
57
|
+
Queries.delete_batch(self.id)
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.create(attributes = nil)
|
61
|
+
attributes ||= {}
|
62
|
+
complete_method = attributes[:complete_method]
|
63
|
+
complete_args = attributes[:complete_args]
|
64
|
+
if complete_args && !complete_method; raise 'args was passed but no method' end
|
65
|
+
|
66
|
+
id = Queries.create_batch(attributes)
|
67
|
+
new_attributes = attributes.clone
|
68
|
+
new_attributes[:id] = id
|
69
|
+
|
70
|
+
Batch.new(new_attributes)
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.find(id, lock=false)
|
74
|
+
if attributes = QC::Batches::Queries.find_batch(id, lock)
|
75
|
+
return Batch.new(attributes)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def finished?
|
80
|
+
Batch.finished?(self.id)
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.finished?(id)
|
84
|
+
!QC::Batches::Queries.has_pending_jobs?(id)
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.perform_job(method, batch_id, args=nil)
|
88
|
+
receiver_str, _, message = method.rpartition('.')
|
89
|
+
receiver = eval(receiver_str)
|
90
|
+
result = receiver.send(message, *args)
|
91
|
+
|
92
|
+
return result
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'queue_classic'
|
2
|
+
|
3
|
+
module QC
|
4
|
+
module Batches
|
5
|
+
module Queries
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def create_batch(attributes)
|
9
|
+
s="INSERT INTO #{TABLE_NAME} (complete_q_name, complete_method, complete_args, queuing_complete) VALUES ($1, $2, $3, $4) returning id"
|
10
|
+
res = QC.default_conn_adapter.execute(s, attributes[:complete_q_name], attributes[:complete_method], attributes.has_key?(:complete_args) ? JSON.dump(attributes[:complete_args]) : nil, attributes[:queuing_complete])
|
11
|
+
res['id'].to_i
|
12
|
+
end
|
13
|
+
|
14
|
+
def delete_batch(id)
|
15
|
+
s="DELETE FROM #{TABLE_NAME} WHERE id=$1"
|
16
|
+
res = QC.default_conn_adapter.execute(s, id)
|
17
|
+
end
|
18
|
+
|
19
|
+
def save_queuing_complete(id)
|
20
|
+
s="UPDATE #{TABLE_NAME} SET queuing_complete = true WHERE id=$1"
|
21
|
+
res = QC.default_conn_adapter.execute(s, id)
|
22
|
+
end
|
23
|
+
|
24
|
+
def find_batch(id, lock=false)
|
25
|
+
s = "SELECT * FROM #{TABLE_NAME} WHERE id = $1"
|
26
|
+
s << " FOR UPDATE" if lock
|
27
|
+
if r = QC.default_conn_adapter.execute(s, id)
|
28
|
+
{}.tap do |batch|
|
29
|
+
batch[:id] = r['id'].to_i
|
30
|
+
batch[:complete_method] = r['complete_method']
|
31
|
+
batch[:complete_args] = JSON.parse(r['complete_args']) if r['complete_args']
|
32
|
+
batch[:complete_queue] = r['complete_queue']
|
33
|
+
batch[:queuing_complete] = r['queuing_complete'] == 't'
|
34
|
+
if r['created_at']
|
35
|
+
batch[:created_at] = Time.parse(r['created_at'])
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def has_pending_jobs?(batch_id)
|
42
|
+
s="SELECT count(id) from #{QC::TABLE_NAME} WHERE batch_id = $1 LIMIT 1"
|
43
|
+
res = QC.default_conn_adapter.execute(s, batch_id)
|
44
|
+
|
45
|
+
return res['count'].to_i != 0
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require "queue_classic"
|
2
|
+
|
3
|
+
module QC
|
4
|
+
class Queue
|
5
|
+
def enqueue_batch(method, batch_id, *args)
|
6
|
+
batch_method = 'QC::Batches::Batch.perform_job'
|
7
|
+
args = args.unshift(batch_id)
|
8
|
+
args = args.unshift(method)
|
9
|
+
QC.log_yield(:measure => 'queue.enqueue_batch') do
|
10
|
+
s="INSERT INTO #{TABLE_NAME} (q_name, method, batch_id, args) VALUES ($1, $2, $3, $4)"
|
11
|
+
res = conn_adapter.execute(s, name, batch_method, batch_id, JSON.dump(args))
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def lock
|
16
|
+
#we have to patch the entire lock method to get it to return batch_id
|
17
|
+
QC.log_yield(:measure => 'queue.lock') do
|
18
|
+
s = "SELECT * FROM lock_head($1, $2)"
|
19
|
+
if r = conn_adapter.execute(s, name, top_bound)
|
20
|
+
{}.tap do |job|
|
21
|
+
job[:id] = r["id"]
|
22
|
+
job[:method] = r["method"]
|
23
|
+
job[:batch_id] = r["batch_id"]
|
24
|
+
job[:args] = JSON.parse(r["args"])
|
25
|
+
if r["created_at"]
|
26
|
+
job[:created_at] = Time.parse(r["created_at"])
|
27
|
+
ttl = Integer((Time.now - job[:created_at]) * 1000)
|
28
|
+
QC.measure("time-to-lock=#{ttl}ms source=#{name}")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
|
3
|
+
module QC
|
4
|
+
module Batches
|
5
|
+
module Setup
|
6
|
+
Root = File.expand_path("../..", File.dirname(__FILE__))
|
7
|
+
AddColumns = File.join(Root, "/sql/add_columns.sql")
|
8
|
+
CreateBatches = File.join(Root, "/sql/create_batches.sql")
|
9
|
+
|
10
|
+
def self.add_batches(c = QC::default_conn_adapter.connection)
|
11
|
+
conn = QC::ConnAdapter.new(c)
|
12
|
+
conn.execute(File.read(AddColumns))
|
13
|
+
conn.execute(File.read(CreateBatches))
|
14
|
+
conn.disconnect if c.nil? #Don't close a conn we didn't create.
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.remove_batches(c = QC::default_conn_adapter.connection)
|
18
|
+
conn = QC::ConnAdapter.new(c)
|
19
|
+
conn.execute("DROP TABLE IF EXISTS queue_classic_batches CASCADE")
|
20
|
+
#todo: remove the batch_id column
|
21
|
+
conn.disconnect if c.nil? #Don't close a conn we didn't create.
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require "queue_classic"
|
2
|
+
module QC
|
3
|
+
class Worker
|
4
|
+
alias_method :qc_base_process, :process
|
5
|
+
|
6
|
+
def process(queue, job)
|
7
|
+
result = qc_base_process queue, job
|
8
|
+
if job[:batch_id]
|
9
|
+
#note, for errors if the worker doesn't delete the job in handle_failure, this never fires
|
10
|
+
QC::Batches::Batch.complete_if_finished job[:batch_id]
|
11
|
+
end
|
12
|
+
return result
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'queue_classic_batches/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "queue_classic_batches"
|
8
|
+
spec.version = QueueClassicBatches::VERSION
|
9
|
+
spec.authors = ["shanewho"]
|
10
|
+
spec.description = %q{Adds batch functionality to the queue_classic gem}
|
11
|
+
spec.summary = %q{Adds batch functionality to the queue_classic gem}
|
12
|
+
spec.homepage = ""
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files`.split($/)
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_runtime_dependency "queue_classic", "~> 3.0.0rc"
|
21
|
+
|
22
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
23
|
+
spec.add_development_dependency "rake"
|
24
|
+
spec.add_development_dependency "rspec", "~> 3.0.0.beta"
|
25
|
+
spec.add_development_dependency "activerecord", "~> 4"
|
26
|
+
end
|
data/spec/batch_spec.rb
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
module QC
|
4
|
+
module Batches
|
5
|
+
|
6
|
+
describe Batch do
|
7
|
+
include_context "init database"
|
8
|
+
let(:batch) { Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a']) }
|
9
|
+
|
10
|
+
context '#create' do
|
11
|
+
it 'returns an id' do
|
12
|
+
expect(batch.id).to be > 0
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'doesnt require args' do
|
16
|
+
expect(Batch.create.id).to be > 0
|
17
|
+
end
|
18
|
+
|
19
|
+
context 'with complete job' do
|
20
|
+
context 'without args' do
|
21
|
+
it 'doesnt error' do
|
22
|
+
Batch.create(complete_method: '"abcd".insert')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
context 'with args and no method' do
|
26
|
+
it 'raises an error' do
|
27
|
+
expect{Batch.create(complete_args: [1,'a'])}.to raise_error
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context '#find' do
|
34
|
+
it 'returns a batch' do
|
35
|
+
expect(Batch.find(batch.id).id).to eq(batch.id)
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'returns a batch created without args' do
|
39
|
+
id = Batch.create.id
|
40
|
+
expect(Batch.find(id).id).to eq(id)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
context '#delete' do
|
45
|
+
it 'deletes a batch' do
|
46
|
+
batch.delete
|
47
|
+
expect(Batch.find(batch.id)).to eq(nil)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context '#enqueue' do
|
52
|
+
context 'with default queue' do
|
53
|
+
it 'queues a job with batch id' do
|
54
|
+
expect_any_instance_of(Queue).to receive(:enqueue_batch).with('"abcd".insert', batch.id, [0, 'x']).once
|
55
|
+
batch.enqueue '"abcd".insert', 0, 'x'
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context 'with a specified queue' do
|
60
|
+
it 'queues a job with batch id' do
|
61
|
+
queue = Queue.new 'test queue'
|
62
|
+
batch.queue = queue
|
63
|
+
expect(queue).to receive(:enqueue_batch).with('"abcd".insert', batch.id, [0, 'x']).once
|
64
|
+
batch.enqueue '"abcd".insert', 0, 'x'
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
context '#queuing_complete' do
|
70
|
+
context 'with uncompleted jobs' do
|
71
|
+
before(:each) { batch.enqueue 'Time.now' }
|
72
|
+
|
73
|
+
it 'marks queuing complete' do
|
74
|
+
batch.queuing_complete
|
75
|
+
b = Batch.find(batch.id)
|
76
|
+
expect(batch.queuing_complete?).to eq(true)
|
77
|
+
expect(b.queuing_complete?).to eq(true)
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'doesnt queue the complete job' do
|
81
|
+
expect(QC).not_to receive(:enqueue)
|
82
|
+
batch.queuing_complete
|
83
|
+
end
|
84
|
+
|
85
|
+
it 'doesnt delete' do
|
86
|
+
expect(Batch.find(batch.id)).not_to eq(nil)
|
87
|
+
batch.queuing_complete
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
context 'with no more jobs' do
|
93
|
+
it 'queues the complete job' do
|
94
|
+
expect(QC).to receive(:enqueue)
|
95
|
+
batch.queuing_complete
|
96
|
+
end
|
97
|
+
|
98
|
+
it 'deletes the batch' do
|
99
|
+
batch.queuing_complete
|
100
|
+
expect(Batch.find(batch.id)).to eq(nil)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
context '#finished?' do
|
106
|
+
context 'with no jobs' do
|
107
|
+
it 'returns true' do
|
108
|
+
expect(Batch.finished?(batch.id)).to eq(true)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
context 'with jobs' do
|
112
|
+
it 'returns false' do
|
113
|
+
batch.enqueue 'Time.now'
|
114
|
+
expect(Batch.finished?(batch.id)).to eq(false)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
context '#perform job' do
|
120
|
+
it 'performs the job' do
|
121
|
+
expect(Batch.perform_job '"abcd".insert', batch.id, [0, 'x']).to eq('xabcd');
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
#context '#create' do
|
131
|
+
#it 'creates a batch' do
|
132
|
+
#batch = Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a'])
|
133
|
+
#batch.enqueue '"abcd".insert', 0, 'x'
|
134
|
+
#batch.enqueue '"abcd".insert', 1, 'y'
|
135
|
+
#batch.enqueue '"abcd".insert', 2, 'z'
|
136
|
+
|
137
|
+
#expect(Batch.find(batch.id).id).to eq(batch.id)
|
138
|
+
#end
|
139
|
+
#end
|
data/spec/helper.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require "queue_classic"
|
2
|
+
require "queue_classic_batches"
|
3
|
+
require "stringio"
|
4
|
+
|
5
|
+
module QC
|
6
|
+
module Batches
|
7
|
+
|
8
|
+
ENV["DATABASE_URL"] ||= "postgres://localhost/queue_classic_batches_test"
|
9
|
+
|
10
|
+
shared_context "init database" do
|
11
|
+
before(:all) do
|
12
|
+
setup_db
|
13
|
+
end
|
14
|
+
def setup_db
|
15
|
+
c = QC::ConnAdapter.new
|
16
|
+
c.execute("SET client_min_messages TO 'warning'")
|
17
|
+
QC::Setup.drop(c.connection)
|
18
|
+
QC::Setup.create(c.connection)
|
19
|
+
QC::Batches::Setup.remove_batches
|
20
|
+
QC::Batches::Setup.add_batches
|
21
|
+
c.disconnect
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
data/spec/queue_spec.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require "helper"
|
2
|
+
|
3
|
+
module QC
|
4
|
+
module Batches
|
5
|
+
describe Queue do
|
6
|
+
include_context "init database"
|
7
|
+
|
8
|
+
describe '#enqueue_batch' do
|
9
|
+
before(:each) do
|
10
|
+
QC.enqueue_batch('"abcd".insert', 'abc-123', 0, "x")
|
11
|
+
end
|
12
|
+
|
13
|
+
subject{QC.lock}
|
14
|
+
|
15
|
+
it 'adds a batch_id column' do
|
16
|
+
expect(subject[:batch_id]).to eq('abc-123');
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'saves the original method' do
|
20
|
+
expect(subject[:args]).to eq(['"abcd".insert', 'abc-123', 0, 'x']);
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'queues call to BatchJob.perform' do
|
24
|
+
expect(subject[:method]).to eq('QC::Batches::Batch.perform_job');
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/spec/worker_spec.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
require "helper"
|
2
|
+
|
3
|
+
module QC
|
4
|
+
module Batches
|
5
|
+
describe Worker do
|
6
|
+
include_context "init database"
|
7
|
+
|
8
|
+
describe '#process' do
|
9
|
+
before(:each) do
|
10
|
+
end
|
11
|
+
let(:worker){ Worker.new }
|
12
|
+
let(:queue){ Queue.new 'default' }
|
13
|
+
let(:batch) { Batch.create(queuing_complete: true, complete_method: '"abcd".insert', complete_args: [0, 'a']) }
|
14
|
+
let(:job){ {id: 1, method: 'Time.now', batch_id: batch.id} }
|
15
|
+
|
16
|
+
context 'with complete job' do
|
17
|
+
context 'with jobs complete' do
|
18
|
+
#let(:batch) { Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a']) }
|
19
|
+
|
20
|
+
context 'with queuing complete' do
|
21
|
+
|
22
|
+
it 'queues the complete job' do
|
23
|
+
expect(QC).to receive(:enqueue).with('"abcd".insert', 0, 'a').once
|
24
|
+
worker.process queue, job
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'deletes the batch' do
|
28
|
+
worker.process queue, job
|
29
|
+
expect(Batch.find(batch.id)).to eq(nil)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'but not done queuing' do
|
34
|
+
let(:batch) { Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a']) }
|
35
|
+
|
36
|
+
it 'doesnt queues the complete job' do
|
37
|
+
expect(QC).not_to receive(:enqueue)
|
38
|
+
worker.process queue, job
|
39
|
+
end
|
40
|
+
it 'doesnt delete the batch' do
|
41
|
+
worker.process queue, job
|
42
|
+
expect(Batch.find(batch.id)).not_to eq(nil)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
context 'without complete job' do
|
49
|
+
let(:batch) { Batch.create }
|
50
|
+
context 'with queuing complete' do
|
51
|
+
before(:each) { batch.queuing_complete }
|
52
|
+
it 'deletes the batch' do
|
53
|
+
worker.process queue, job
|
54
|
+
expect(Batch.find(batch.id)).to eq(nil)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'but not done queuing' do
|
59
|
+
it 'doesnt delete the batch' do
|
60
|
+
worker.process queue, job
|
61
|
+
expect(Batch.find(batch.id)).not_to eq(nil)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
data/sql/add_columns.sql
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
do $$ begin
|
2
|
+
|
3
|
+
CREATE TABLE queue_classic_batches (
|
4
|
+
id bigserial PRIMARY KEY,
|
5
|
+
complete_q_name text, --Allow null - batches without complete jobs can be useful
|
6
|
+
complete_method text,
|
7
|
+
complete_args text,
|
8
|
+
queuing_complete boolean,
|
9
|
+
created_at timestamptz default now()
|
10
|
+
);
|
11
|
+
|
12
|
+
-- If json type is available, use it for the complete_args column.
|
13
|
+
perform * from pg_type where typname = 'json';
|
14
|
+
if found then
|
15
|
+
alter table queue_classic_batches alter column complete_args type json using (complete_args::json);
|
16
|
+
end if;
|
17
|
+
|
18
|
+
end $$ language plpgsql;
|
19
|
+
|
20
|
+
--todo: CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
|
metadata
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: queue_classic_batches
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- shanewho
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-04-26 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: queue_classic
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 3.0.0rc
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 3.0.0rc
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 3.0.0.beta
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 3.0.0.beta
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: activerecord
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ~>
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '4'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ~>
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '4'
|
83
|
+
description: Adds batch functionality to the queue_classic gem
|
84
|
+
email:
|
85
|
+
executables: []
|
86
|
+
extensions: []
|
87
|
+
extra_rdoc_files: []
|
88
|
+
files:
|
89
|
+
- .gitignore
|
90
|
+
- Gemfile
|
91
|
+
- LICENSE.txt
|
92
|
+
- README.md
|
93
|
+
- Rakefile
|
94
|
+
- lib/generators/queue_classic_batches/install_generator.rb
|
95
|
+
- lib/generators/queue_classic_batches/templates/add_queue_classic_batches.rb
|
96
|
+
- lib/queue_classic_batches.rb
|
97
|
+
- lib/queue_classic_batches/batch.rb
|
98
|
+
- lib/queue_classic_batches/queries.rb
|
99
|
+
- lib/queue_classic_batches/queue.rb
|
100
|
+
- lib/queue_classic_batches/setup.rb
|
101
|
+
- lib/queue_classic_batches/version.rb
|
102
|
+
- lib/queue_classic_batches/worker.rb
|
103
|
+
- queue_classic_batches.gemspec
|
104
|
+
- spec/batch_spec.rb
|
105
|
+
- spec/helper.rb
|
106
|
+
- spec/queue_spec.rb
|
107
|
+
- spec/worker_spec.rb
|
108
|
+
- sql/add_columns.sql
|
109
|
+
- sql/create_batches.sql
|
110
|
+
homepage: ''
|
111
|
+
licenses:
|
112
|
+
- MIT
|
113
|
+
metadata: {}
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - '>='
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - '>='
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.2.2
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: Adds batch functionality to the queue_classic gem
|
134
|
+
test_files:
|
135
|
+
- spec/batch_spec.rb
|
136
|
+
- spec/helper.rb
|
137
|
+
- spec/queue_spec.rb
|
138
|
+
- spec/worker_spec.rb
|