queue_classic_batches 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: f4efc7cc0a30ae0cbe1aaef21e52f0e04f1c4e4b
4
+ data.tar.gz: e7cf187ff93fd137e2b1c34c930ad1b47305e0c1
5
+ SHA512:
6
+ metadata.gz: d03b53bc8f5400d2099c9f2f16c991b8b2f76c363dd01173011a38d444766df640a610ed70768c631fd369c1832f22517cbab26a9d2e2d44190c88645abde100
7
+ data.tar.gz: 87bef6b26296dd98c915f8d77403f04bf9b7c8c7fb4b10354706c1e0af38d4bc80521fac07da274b64abb45ecf49e105151d72838730e07526dc5e142c7a74ac
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in queue_classic_batches.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Shane Blazek
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,66 @@
1
+ # queue_classic_batches
2
+
3
+ Adds support to queue_classic to enable queuing another job when a group of jobs have all completed.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'queue_classic_batches'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install queue_classic_batches
18
+
19
+ Ruby on Rails Setup
20
+
21
+ Declare dependencies in Gemfile.
22
+
23
+ source "http://rubygems.org"
24
+ gem "queue_classic_batches", "0.0.1"
25
+
26
+ Add the database tables and columns.
27
+
28
+ rails generate queue_classic_batches:install
29
+ rake db:migrate
30
+
31
+ ## Usage
32
+
33
+ 1. Create a batch
34
+ 2. Queue jobs on the batch
35
+ 3. Mark queuing complete
36
+
37
+ Here is an example:
38
+
39
+ batch = QC::Batches::Batch.create(complete_method:'MyCompleteJob.perform',
40
+ complete_args: [123, 'abc'],
41
+ complete_q_name: 'optional-queue-name')
42
+ (1..20) do |i|
43
+ batch.enqueue("MyJob.perform", i, "Job #{i}")
44
+ end
45
+ batch.queuing_complete
46
+
47
+
48
+ Make sure your worker deletes or re-queues failed jobs or else queue_classic will leave the job in the jobs table and the batch won't know it has been completed.
49
+
50
+ FailedQueue = QC::Queue.new("failed_jobs")
51
+ class MyWorker < QC::Worker
52
+
53
+ def handle_failure(job, e)
54
+ FailedQueue.enqueue(job[:method], *job[:args])
55
+ QC.delete job[:id]
56
+ end
57
+
58
+ end
59
+
60
+ ## Contributing
61
+
62
+ 1. Fork it
63
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
64
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
65
+ 4. Push to the branch (`git push origin my-new-feature`)
66
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,24 @@
1
+ require 'rails/generators'
2
+ require 'rails/generators/migration'
3
+ require 'active_record'
4
+
5
+ module QueueClassicBatches
6
+ class InstallGenerator < Rails::Generators::Base
7
+ include Rails::Generators::Migration
8
+
9
+ namespace "queue_classic_batches:install"
10
+ self.source_paths << File.join(File.dirname(__FILE__), 'templates')
11
+ desc 'Generates (but does not run) a migration to add batch functionality to the queue_classic table.'
12
+
13
+ def self.next_migration_number(dirname)
14
+ next_migration_number = current_migration_number(dirname) + 1
15
+ ActiveRecord::Migration.next_migration_number(next_migration_number)
16
+ end
17
+
18
+ def create_migration_file
19
+ if self.class.migration_exists?('db/migrate', 'add_queue_classic_batches').nil?
20
+ migration_template 'add_queue_classic_batches.rb', 'db/migrate/add_queue_classic_batches.rb'
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,12 @@
1
+ require 'active_record'
2
+ require 'queue_classic_batches'
3
+
4
+ class AddQueueClassicBatches < ActiveRecord::Migration
5
+ def self.up
6
+ QC::Batches::Setup.add_batches
7
+ end
8
+
9
+ def self.down
10
+ QC::Batches::Setup.remove_batches
11
+ end
12
+ end
@@ -0,0 +1,15 @@
1
+ require "queue_classic_batches/version"
2
+ require "queue_classic_batches/queue"
3
+ require "queue_classic_batches/setup"
4
+ require "queue_classic_batches/queries"
5
+ require "queue_classic_batches/batch"
6
+ require "queue_classic_batches/worker"
7
+
8
+ module QC
9
+ module Batches
10
+ # Why do you want to change the table name?
11
+ # Just deal with the default OK?
12
+ # Come on. Don't do it.... Just stick with the default.
13
+ TABLE_NAME = "queue_classic_batches"
14
+ end
15
+ end
@@ -0,0 +1,97 @@
1
+ module QC
2
+ module Batches
3
+ class Batch
4
+ attr_accessor :id
5
+ attr_accessor :queue
6
+ attr_accessor :created_at
7
+ attr_accessor :complete_q_name
8
+ attr_accessor :complete_method
9
+ attr_accessor :complete_args
10
+
11
+ def initialize(args=nil)
12
+ args.each { |k,v|
13
+ instance_variable_set("@#{k}", v) unless v.nil?
14
+ } if args.is_a?(Hash)
15
+ @queuing_complete = args[:queuing_complete] || false
16
+ end
17
+
18
+ def enqueue(method, *args)
19
+ (self.queue || QC).enqueue_batch method, self.id, args
20
+ end
21
+
22
+ def queuing_complete?
23
+ return @queuing_complete
24
+ end
25
+
26
+ def queuing_complete
27
+ @queuing_complete = true
28
+ Queries.save_queuing_complete(self.id)
29
+ Batch.complete_if_finished self.id
30
+ end
31
+
32
+ def self.complete_if_finished(batch_id)
33
+
34
+ if Batch.finished?(batch_id)
35
+ QC.default_conn_adapter.connection.transaction do |conn|
36
+ batch = Batch.find(batch_id, lock: true)
37
+ return unless batch
38
+ batch.complete
39
+ end
40
+ end
41
+
42
+ end
43
+
44
+ def complete
45
+ return unless queuing_complete? && finished?
46
+ if complete_method
47
+ queue = complete_q_name ? Queue.new(queue) : QC
48
+ queue.enqueue complete_method, *complete_args
49
+ end
50
+ delete
51
+
52
+ time_to_complete = Integer((Time.now - created_at) * 1000)
53
+ QC.log(:'time-to-complete-batch'=>time_to_complete, :source=>id)
54
+ end
55
+
56
+ def delete
57
+ Queries.delete_batch(self.id)
58
+ end
59
+
60
+ def self.create(attributes = nil)
61
+ attributes ||= {}
62
+ complete_method = attributes[:complete_method]
63
+ complete_args = attributes[:complete_args]
64
+ if complete_args && !complete_method; raise 'args was passed but no method' end
65
+
66
+ id = Queries.create_batch(attributes)
67
+ new_attributes = attributes.clone
68
+ new_attributes[:id] = id
69
+
70
+ Batch.new(new_attributes)
71
+ end
72
+
73
+ def self.find(id, lock=false)
74
+ if attributes = QC::Batches::Queries.find_batch(id, lock)
75
+ return Batch.new(attributes)
76
+ end
77
+ end
78
+
79
+ def finished?
80
+ Batch.finished?(self.id)
81
+ end
82
+
83
+ def self.finished?(id)
84
+ !QC::Batches::Queries.has_pending_jobs?(id)
85
+ end
86
+
87
+ def self.perform_job(method, batch_id, args=nil)
88
+ receiver_str, _, message = method.rpartition('.')
89
+ receiver = eval(receiver_str)
90
+ result = receiver.send(message, *args)
91
+
92
+ return result
93
+ end
94
+
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,50 @@
1
+ require 'queue_classic'
2
+
3
+ module QC
4
+ module Batches
5
+ module Queries
6
+ extend self
7
+
8
+ def create_batch(attributes)
9
+ s="INSERT INTO #{TABLE_NAME} (complete_q_name, complete_method, complete_args, queuing_complete) VALUES ($1, $2, $3, $4) returning id"
10
+ res = QC.default_conn_adapter.execute(s, attributes[:complete_q_name], attributes[:complete_method], attributes.has_key?(:complete_args) ? JSON.dump(attributes[:complete_args]) : nil, attributes[:queuing_complete])
11
+ res['id'].to_i
12
+ end
13
+
14
+ def delete_batch(id)
15
+ s="DELETE FROM #{TABLE_NAME} WHERE id=$1"
16
+ res = QC.default_conn_adapter.execute(s, id)
17
+ end
18
+
19
+ def save_queuing_complete(id)
20
+ s="UPDATE #{TABLE_NAME} SET queuing_complete = true WHERE id=$1"
21
+ res = QC.default_conn_adapter.execute(s, id)
22
+ end
23
+
24
+ def find_batch(id, lock=false)
25
+ s = "SELECT * FROM #{TABLE_NAME} WHERE id = $1"
26
+ s << " FOR UPDATE" if lock
27
+ if r = QC.default_conn_adapter.execute(s, id)
28
+ {}.tap do |batch|
29
+ batch[:id] = r['id'].to_i
30
+ batch[:complete_method] = r['complete_method']
31
+ batch[:complete_args] = JSON.parse(r['complete_args']) if r['complete_args']
32
+ batch[:complete_queue] = r['complete_queue']
33
+ batch[:queuing_complete] = r['queuing_complete'] == 't'
34
+ if r['created_at']
35
+ batch[:created_at] = Time.parse(r['created_at'])
36
+ end
37
+ end
38
+ end
39
+ end
40
+
41
+ def has_pending_jobs?(batch_id)
42
+ s="SELECT count(id) from #{QC::TABLE_NAME} WHERE batch_id = $1 LIMIT 1"
43
+ res = QC.default_conn_adapter.execute(s, batch_id)
44
+
45
+ return res['count'].to_i != 0
46
+ end
47
+
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,35 @@
1
+ require "queue_classic"
2
+
3
+ module QC
4
+ class Queue
5
+ def enqueue_batch(method, batch_id, *args)
6
+ batch_method = 'QC::Batches::Batch.perform_job'
7
+ args = args.unshift(batch_id)
8
+ args = args.unshift(method)
9
+ QC.log_yield(:measure => 'queue.enqueue_batch') do
10
+ s="INSERT INTO #{TABLE_NAME} (q_name, method, batch_id, args) VALUES ($1, $2, $3, $4)"
11
+ res = conn_adapter.execute(s, name, batch_method, batch_id, JSON.dump(args))
12
+ end
13
+ end
14
+
15
+ def lock
16
+ #we have to patch the entire lock method to get it to return batch_id
17
+ QC.log_yield(:measure => 'queue.lock') do
18
+ s = "SELECT * FROM lock_head($1, $2)"
19
+ if r = conn_adapter.execute(s, name, top_bound)
20
+ {}.tap do |job|
21
+ job[:id] = r["id"]
22
+ job[:method] = r["method"]
23
+ job[:batch_id] = r["batch_id"]
24
+ job[:args] = JSON.parse(r["args"])
25
+ if r["created_at"]
26
+ job[:created_at] = Time.parse(r["created_at"])
27
+ ttl = Integer((Time.now - job[:created_at]) * 1000)
28
+ QC.measure("time-to-lock=#{ttl}ms source=#{name}")
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,26 @@
1
+ require 'active_record'
2
+
3
+ module QC
4
+ module Batches
5
+ module Setup
6
+ Root = File.expand_path("../..", File.dirname(__FILE__))
7
+ AddColumns = File.join(Root, "/sql/add_columns.sql")
8
+ CreateBatches = File.join(Root, "/sql/create_batches.sql")
9
+
10
+ def self.add_batches(c = QC::default_conn_adapter.connection)
11
+ conn = QC::ConnAdapter.new(c)
12
+ conn.execute(File.read(AddColumns))
13
+ conn.execute(File.read(CreateBatches))
14
+ conn.disconnect if c.nil? #Don't close a conn we didn't create.
15
+ end
16
+
17
+ def self.remove_batches(c = QC::default_conn_adapter.connection)
18
+ conn = QC::ConnAdapter.new(c)
19
+ conn.execute("DROP TABLE IF EXISTS queue_classic_batches CASCADE")
20
+ #todo: remove the batch_id column
21
+ conn.disconnect if c.nil? #Don't close a conn we didn't create.
22
+ end
23
+
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,3 @@
1
+ module QueueClassicBatches
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,15 @@
1
+ require "queue_classic"
2
+ module QC
3
+ class Worker
4
+ alias_method :qc_base_process, :process
5
+
6
+ def process(queue, job)
7
+ result = qc_base_process queue, job
8
+ if job[:batch_id]
9
+ #note, for errors if the worker doesn't delete the job in handle_failure, this never fires
10
+ QC::Batches::Batch.complete_if_finished job[:batch_id]
11
+ end
12
+ return result
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'queue_classic_batches/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "queue_classic_batches"
8
+ spec.version = QueueClassicBatches::VERSION
9
+ spec.authors = ["shanewho"]
10
+ spec.description = %q{Adds batch functionality to the queue_classic gem}
11
+ spec.summary = %q{Adds batch functionality to the queue_classic gem}
12
+ spec.homepage = ""
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files`.split($/)
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_runtime_dependency "queue_classic", "~> 3.0.0rc"
21
+
22
+ spec.add_development_dependency "bundler", "~> 1.3"
23
+ spec.add_development_dependency "rake"
24
+ spec.add_development_dependency "rspec", "~> 3.0.0.beta"
25
+ spec.add_development_dependency "activerecord", "~> 4"
26
+ end
@@ -0,0 +1,139 @@
1
+ require 'helper'
2
+
3
+ module QC
4
+ module Batches
5
+
6
+ describe Batch do
7
+ include_context "init database"
8
+ let(:batch) { Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a']) }
9
+
10
+ context '#create' do
11
+ it 'returns an id' do
12
+ expect(batch.id).to be > 0
13
+ end
14
+
15
+ it 'doesnt require args' do
16
+ expect(Batch.create.id).to be > 0
17
+ end
18
+
19
+ context 'with complete job' do
20
+ context 'without args' do
21
+ it 'doesnt error' do
22
+ Batch.create(complete_method: '"abcd".insert')
23
+ end
24
+ end
25
+ context 'with args and no method' do
26
+ it 'raises an error' do
27
+ expect{Batch.create(complete_args: [1,'a'])}.to raise_error
28
+ end
29
+ end
30
+ end
31
+ end
32
+
33
+ context '#find' do
34
+ it 'returns a batch' do
35
+ expect(Batch.find(batch.id).id).to eq(batch.id)
36
+ end
37
+
38
+ it 'returns a batch created without args' do
39
+ id = Batch.create.id
40
+ expect(Batch.find(id).id).to eq(id)
41
+ end
42
+ end
43
+
44
+ context '#delete' do
45
+ it 'deletes a batch' do
46
+ batch.delete
47
+ expect(Batch.find(batch.id)).to eq(nil)
48
+ end
49
+ end
50
+
51
+ context '#enqueue' do
52
+ context 'with default queue' do
53
+ it 'queues a job with batch id' do
54
+ expect_any_instance_of(Queue).to receive(:enqueue_batch).with('"abcd".insert', batch.id, [0, 'x']).once
55
+ batch.enqueue '"abcd".insert', 0, 'x'
56
+ end
57
+ end
58
+
59
+ context 'with a specified queue' do
60
+ it 'queues a job with batch id' do
61
+ queue = Queue.new 'test queue'
62
+ batch.queue = queue
63
+ expect(queue).to receive(:enqueue_batch).with('"abcd".insert', batch.id, [0, 'x']).once
64
+ batch.enqueue '"abcd".insert', 0, 'x'
65
+ end
66
+ end
67
+ end
68
+
69
+ context '#queuing_complete' do
70
+ context 'with uncompleted jobs' do
71
+ before(:each) { batch.enqueue 'Time.now' }
72
+
73
+ it 'marks queuing complete' do
74
+ batch.queuing_complete
75
+ b = Batch.find(batch.id)
76
+ expect(batch.queuing_complete?).to eq(true)
77
+ expect(b.queuing_complete?).to eq(true)
78
+ end
79
+
80
+ it 'doesnt queue the complete job' do
81
+ expect(QC).not_to receive(:enqueue)
82
+ batch.queuing_complete
83
+ end
84
+
85
+ it 'doesnt delete' do
86
+ expect(Batch.find(batch.id)).not_to eq(nil)
87
+ batch.queuing_complete
88
+ end
89
+
90
+ end
91
+
92
+ context 'with no more jobs' do
93
+ it 'queues the complete job' do
94
+ expect(QC).to receive(:enqueue)
95
+ batch.queuing_complete
96
+ end
97
+
98
+ it 'deletes the batch' do
99
+ batch.queuing_complete
100
+ expect(Batch.find(batch.id)).to eq(nil)
101
+ end
102
+ end
103
+ end
104
+
105
+ context '#finished?' do
106
+ context 'with no jobs' do
107
+ it 'returns true' do
108
+ expect(Batch.finished?(batch.id)).to eq(true)
109
+ end
110
+ end
111
+ context 'with jobs' do
112
+ it 'returns false' do
113
+ batch.enqueue 'Time.now'
114
+ expect(Batch.finished?(batch.id)).to eq(false)
115
+ end
116
+ end
117
+ end
118
+
119
+ context '#perform job' do
120
+ it 'performs the job' do
121
+ expect(Batch.perform_job '"abcd".insert', batch.id, [0, 'x']).to eq('xabcd');
122
+ end
123
+ end
124
+ end
125
+
126
+ end
127
+ end
128
+
129
+
130
+ #context '#create' do
131
+ #it 'creates a batch' do
132
+ #batch = Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a'])
133
+ #batch.enqueue '"abcd".insert', 0, 'x'
134
+ #batch.enqueue '"abcd".insert', 1, 'y'
135
+ #batch.enqueue '"abcd".insert', 2, 'z'
136
+
137
+ #expect(Batch.find(batch.id).id).to eq(batch.id)
138
+ #end
139
+ #end
data/spec/helper.rb ADDED
@@ -0,0 +1,26 @@
1
+ require "queue_classic"
2
+ require "queue_classic_batches"
3
+ require "stringio"
4
+
5
+ module QC
6
+ module Batches
7
+
8
+ ENV["DATABASE_URL"] ||= "postgres://localhost/queue_classic_batches_test"
9
+
10
+ shared_context "init database" do
11
+ before(:all) do
12
+ setup_db
13
+ end
14
+ def setup_db
15
+ c = QC::ConnAdapter.new
16
+ c.execute("SET client_min_messages TO 'warning'")
17
+ QC::Setup.drop(c.connection)
18
+ QC::Setup.create(c.connection)
19
+ QC::Batches::Setup.remove_batches
20
+ QC::Batches::Setup.add_batches
21
+ c.disconnect
22
+ end
23
+ end
24
+
25
+ end
26
+ end
@@ -0,0 +1,29 @@
1
+ require "helper"
2
+
3
+ module QC
4
+ module Batches
5
+ describe Queue do
6
+ include_context "init database"
7
+
8
+ describe '#enqueue_batch' do
9
+ before(:each) do
10
+ QC.enqueue_batch('"abcd".insert', 'abc-123', 0, "x")
11
+ end
12
+
13
+ subject{QC.lock}
14
+
15
+ it 'adds a batch_id column' do
16
+ expect(subject[:batch_id]).to eq('abc-123');
17
+ end
18
+
19
+ it 'saves the original method' do
20
+ expect(subject[:args]).to eq(['"abcd".insert', 'abc-123', 0, 'x']);
21
+ end
22
+
23
+ it 'queues call to BatchJob.perform' do
24
+ expect(subject[:method]).to eq('QC::Batches::Batch.perform_job');
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,70 @@
1
+ require "helper"
2
+
3
+ module QC
4
+ module Batches
5
+ describe Worker do
6
+ include_context "init database"
7
+
8
+ describe '#process' do
9
+ before(:each) do
10
+ end
11
+ let(:worker){ Worker.new }
12
+ let(:queue){ Queue.new 'default' }
13
+ let(:batch) { Batch.create(queuing_complete: true, complete_method: '"abcd".insert', complete_args: [0, 'a']) }
14
+ let(:job){ {id: 1, method: 'Time.now', batch_id: batch.id} }
15
+
16
+ context 'with complete job' do
17
+ context 'with jobs complete' do
18
+ #let(:batch) { Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a']) }
19
+
20
+ context 'with queuing complete' do
21
+
22
+ it 'queues the complete job' do
23
+ expect(QC).to receive(:enqueue).with('"abcd".insert', 0, 'a').once
24
+ worker.process queue, job
25
+ end
26
+
27
+ it 'deletes the batch' do
28
+ worker.process queue, job
29
+ expect(Batch.find(batch.id)).to eq(nil)
30
+ end
31
+ end
32
+
33
+ context 'but not done queuing' do
34
+ let(:batch) { Batch.create(complete_method: '"abcd".insert', complete_args: [0, 'a']) }
35
+
36
+ it 'doesnt queues the complete job' do
37
+ expect(QC).not_to receive(:enqueue)
38
+ worker.process queue, job
39
+ end
40
+ it 'doesnt delete the batch' do
41
+ worker.process queue, job
42
+ expect(Batch.find(batch.id)).not_to eq(nil)
43
+ end
44
+ end
45
+ end
46
+ end
47
+
48
+ context 'without complete job' do
49
+ let(:batch) { Batch.create }
50
+ context 'with queuing complete' do
51
+ before(:each) { batch.queuing_complete }
52
+ it 'deletes the batch' do
53
+ worker.process queue, job
54
+ expect(Batch.find(batch.id)).to eq(nil)
55
+ end
56
+ end
57
+
58
+ context 'but not done queuing' do
59
+ it 'doesnt delete the batch' do
60
+ worker.process queue, job
61
+ expect(Batch.find(batch.id)).not_to eq(nil)
62
+ end
63
+ end
64
+ end
65
+
66
+ end
67
+ end
68
+ end
69
+ end
70
+
@@ -0,0 +1,4 @@
1
+ --do $$ begin
2
+ ALTER TABLE queue_classic_jobs ADD COLUMN batch_id varchar(128);
3
+
4
+ --CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
@@ -0,0 +1,20 @@
1
+ do $$ begin
2
+
3
+ CREATE TABLE queue_classic_batches (
4
+ id bigserial PRIMARY KEY,
5
+ complete_q_name text, --Allow null - batches without complete jobs can be useful
6
+ complete_method text,
7
+ complete_args text,
8
+ queuing_complete boolean,
9
+ created_at timestamptz default now()
10
+ );
11
+
12
+ -- If json type is available, use it for the complete_args column.
13
+ perform * from pg_type where typname = 'json';
14
+ if found then
15
+ alter table queue_classic_batches alter column complete_args type json using (complete_args::json);
16
+ end if;
17
+
18
+ end $$ language plpgsql;
19
+
20
+ --todo: CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
metadata ADDED
@@ -0,0 +1,138 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: queue_classic_batches
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - shanewho
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-04-26 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: queue_classic
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: 3.0.0rc
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: 3.0.0rc
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '1.3'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: '1.3'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: 3.0.0.beta
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: 3.0.0.beta
69
+ - !ruby/object:Gem::Dependency
70
+ name: activerecord
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ~>
74
+ - !ruby/object:Gem::Version
75
+ version: '4'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ~>
81
+ - !ruby/object:Gem::Version
82
+ version: '4'
83
+ description: Adds batch functionality to the queue_classic gem
84
+ email:
85
+ executables: []
86
+ extensions: []
87
+ extra_rdoc_files: []
88
+ files:
89
+ - .gitignore
90
+ - Gemfile
91
+ - LICENSE.txt
92
+ - README.md
93
+ - Rakefile
94
+ - lib/generators/queue_classic_batches/install_generator.rb
95
+ - lib/generators/queue_classic_batches/templates/add_queue_classic_batches.rb
96
+ - lib/queue_classic_batches.rb
97
+ - lib/queue_classic_batches/batch.rb
98
+ - lib/queue_classic_batches/queries.rb
99
+ - lib/queue_classic_batches/queue.rb
100
+ - lib/queue_classic_batches/setup.rb
101
+ - lib/queue_classic_batches/version.rb
102
+ - lib/queue_classic_batches/worker.rb
103
+ - queue_classic_batches.gemspec
104
+ - spec/batch_spec.rb
105
+ - spec/helper.rb
106
+ - spec/queue_spec.rb
107
+ - spec/worker_spec.rb
108
+ - sql/add_columns.sql
109
+ - sql/create_batches.sql
110
+ homepage: ''
111
+ licenses:
112
+ - MIT
113
+ metadata: {}
114
+ post_install_message:
115
+ rdoc_options: []
116
+ require_paths:
117
+ - lib
118
+ required_ruby_version: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - '>='
121
+ - !ruby/object:Gem::Version
122
+ version: '0'
123
+ required_rubygems_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - '>='
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ requirements: []
129
+ rubyforge_project:
130
+ rubygems_version: 2.2.2
131
+ signing_key:
132
+ specification_version: 4
133
+ summary: Adds batch functionality to the queue_classic gem
134
+ test_files:
135
+ - spec/batch_spec.rb
136
+ - spec/helper.rb
137
+ - spec/queue_spec.rb
138
+ - spec/worker_spec.rb