postqueue 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +10 -10
- data/bin/postqueue +4 -0
- data/lib/postqueue/cli.rb +54 -0
- data/lib/postqueue/cli/options_parser.rb +46 -0
- data/lib/postqueue/default_queue.rb +1 -0
- data/lib/postqueue/queue.rb +15 -14
- data/lib/postqueue/queue/callback.rb +22 -11
- data/lib/postqueue/queue/logging.rb +18 -1
- data/lib/postqueue/queue/processing.rb +4 -4
- data/lib/postqueue/queue/runner.rb +24 -0
- data/lib/postqueue/version.rb +1 -1
- data/spec/postqueue/default_queue_spec.rb +2 -2
- data/spec/postqueue/idempotent_ops_spec.rb +1 -2
- data/spec/postqueue/process_errors_spec.rb +2 -2
- data/spec/postqueue/process_spec.rb +3 -3
- metadata +22 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d9f6d7f05810076cd9c15f8b47515817b2e331be
|
4
|
+
data.tar.gz: 0d0bff63426af6568730ea89e8da3ce3e5a3b015
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b84633af24db245a98380b96ed17021650945d0ffe554eaaa54c0a3adf8fbf7ecdc8b3036443044371822980f0f0129ab6bb93b89bad4dfce5308ab05d406e7
|
7
|
+
data.tar.gz: 6e9950292583f300ad5f0d03eb777c8337704986231a6b5dee71b19f4d00e4c102ff6b64e7124a66e930520b7193443a1d294d763b5993048c26ebb5d3eb41c8
|
data/README.md
CHANGED
@@ -33,15 +33,11 @@ and therefore needs at least PostgresQL >= 9.5.
|
|
33
33
|
```ruby
|
34
34
|
queue = Postqueue.new
|
35
35
|
queue.enqueue op: "product/reindex", entity_id: [12,13,14,15]
|
36
|
-
queue.
|
37
|
-
|
38
|
-
case op
|
39
|
-
when "product/reindex"
|
40
|
-
Product.index_many(Product.where(id: entity_ids))
|
41
|
-
else
|
42
|
-
raise "Unsupported op: #{op}"
|
43
|
-
end
|
36
|
+
queue.on "product/reindex" do |op, entity_ids|
|
37
|
+
Product.index_many(Product.where(id: entity_ids))
|
44
38
|
end
|
39
|
+
|
40
|
+
queue.process
|
45
41
|
```
|
46
42
|
|
47
43
|
The process call will select a number of queue items for processing. They will all have
|
@@ -88,7 +84,9 @@ When enqueueing items duplicate idempotent operations are not enqueued. Whether
|
|
88
84
|
should be considered idempotent is defined when configuring the queue:
|
89
85
|
|
90
86
|
Postqueue.new do |queue|
|
91
|
-
queue.
|
87
|
+
queue.on "idempotent", idempotent: true do ]op, entity_ids|
|
88
|
+
# .. handle queue item
|
89
|
+
end
|
92
90
|
end
|
93
91
|
|
94
92
|
### Processing a single entry
|
@@ -132,7 +130,9 @@ or an operation-specific batch_size:
|
|
132
130
|
|
133
131
|
Postqueue.new do |queue|
|
134
132
|
queue.default_batch_size = 100
|
135
|
-
queue.
|
133
|
+
queue.on "batchable", batch_size: 10 do
|
134
|
+
...
|
135
|
+
end
|
136
136
|
end
|
137
137
|
|
138
138
|
## Test mode
|
data/bin/postqueue
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
require "ostruct"
|
2
|
+
|
3
|
+
require_relative "cli/options_parser"
|
4
|
+
|
5
|
+
module Postqueue
|
6
|
+
module CLI
|
7
|
+
extend self
|
8
|
+
|
9
|
+
attr_reader :options
|
10
|
+
|
11
|
+
def run(argv)
|
12
|
+
@options = OptionsParser.parse_args(argv)
|
13
|
+
|
14
|
+
case options.sub_command
|
15
|
+
when "stats"
|
16
|
+
connect_to_database!
|
17
|
+
sql = "SELECT * FROM #{Postqueue.item_class.table_name}"
|
18
|
+
tp Postqueue.item_class.find_by_sql(sql)
|
19
|
+
when "enqueue"
|
20
|
+
connect_to_database!
|
21
|
+
count = Postqueue.enqueue op: options.op, entity_id: options.entity_ids
|
22
|
+
puts "returned #{count.inspect}"
|
23
|
+
Postqueue.logger.info "Enqueued #{count} queue items"
|
24
|
+
when "process"
|
25
|
+
connect_to_instance!
|
26
|
+
Postqueue.process batch_size: 1
|
27
|
+
when "run"
|
28
|
+
connect_to_instance!
|
29
|
+
Postqueue.run!
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def connect_to_instance!
|
34
|
+
path = "#{Dir.getwd}/config/postqueue.rb"
|
35
|
+
Postqueue.logger.info "Loading postqueue configuration from #{path}"
|
36
|
+
load path
|
37
|
+
end
|
38
|
+
|
39
|
+
def connect_to_database!
|
40
|
+
abc = active_record_config
|
41
|
+
username, host, database = abc.values_at "username", "host", "database"
|
42
|
+
Postqueue.logger.info "Connecting to postgres:#{username}@#{host}/#{database}"
|
43
|
+
|
44
|
+
ActiveRecord::Base.establish_connection(abc)
|
45
|
+
end
|
46
|
+
|
47
|
+
def active_record_config
|
48
|
+
require "yaml"
|
49
|
+
database_config = YAML.load_file "config/database.yml"
|
50
|
+
env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
51
|
+
database_config.fetch(env)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "ostruct"
|
2
|
+
|
3
|
+
module Postqueue
|
4
|
+
module CLI
|
5
|
+
class OptionsParser
|
6
|
+
def self.parse_args(argv)
|
7
|
+
new(argv).parse_args
|
8
|
+
end
|
9
|
+
|
10
|
+
attr_reader :argv
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@argv = argv
|
14
|
+
end
|
15
|
+
|
16
|
+
def parse_args
|
17
|
+
require "optparse"
|
18
|
+
options = OpenStruct.new
|
19
|
+
options.sub_command = argv.shift || usage!
|
20
|
+
|
21
|
+
case options.sub_command
|
22
|
+
when "enqueue"
|
23
|
+
options.op = next_arg!
|
24
|
+
options.entity_ids = next_arg!.split(",").map { |s| Integer(s) }
|
25
|
+
end
|
26
|
+
options
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def next_arg!
|
32
|
+
argv.shift || usage!
|
33
|
+
end
|
34
|
+
|
35
|
+
def usage!
|
36
|
+
STDERR.puts <<-USAGE
|
37
|
+
postqueue stats
|
38
|
+
postqueue enqueue op entity_id,entity_id,entity_id
|
39
|
+
postqueue run
|
40
|
+
postqueue process
|
41
|
+
USAGE
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/postqueue/queue.rb
CHANGED
@@ -8,9 +8,6 @@ module Postqueue
|
|
8
8
|
# for an operation.
|
9
9
|
attr_accessor :default_batch_size
|
10
10
|
|
11
|
-
# batch size for a given op
|
12
|
-
attr_reader :batch_sizes
|
13
|
-
|
14
11
|
# maximum number of processing attempts.
|
15
12
|
attr_reader :max_attemps
|
16
13
|
|
@@ -26,31 +23,34 @@ module Postqueue
|
|
26
23
|
@default_batch_size = 1
|
27
24
|
@max_attemps = 5
|
28
25
|
@async_processing = Postqueue.async_processing?
|
26
|
+
@idempotent_operations = {}
|
27
|
+
@batch_sizes = {}
|
28
|
+
|
29
|
+
on :missing_handler do |op, entity_ids|
|
30
|
+
raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
|
31
|
+
end
|
32
|
+
|
33
|
+
on_exception do |e, _, _|
|
34
|
+
e.send :raise
|
35
|
+
end
|
29
36
|
|
30
37
|
yield self if block
|
31
38
|
end
|
32
39
|
|
33
40
|
def batch_size(op:)
|
34
|
-
batch_sizes[op] || default_batch_size || 1
|
35
|
-
end
|
36
|
-
|
37
|
-
def idempotent_operations
|
38
|
-
@idempotent_operations ||= {}
|
41
|
+
@batch_sizes[op] || default_batch_size || 1
|
39
42
|
end
|
40
43
|
|
41
44
|
def idempotent_operation?(op)
|
42
|
-
idempotent_operations.fetch(op) { idempotent_operations.fetch("*", false) }
|
43
|
-
end
|
44
|
-
|
45
|
-
def idempotent_operation(op, flag = true)
|
46
|
-
idempotent_operations[op] = flag
|
45
|
+
@idempotent_operations.fetch(op) { @idempotent_operations.fetch("*", false) }
|
47
46
|
end
|
48
47
|
|
49
48
|
def enqueue(op:, entity_id:)
|
50
49
|
enqueued_items = item_class.enqueue op: op, entity_id: entity_id, ignore_duplicates: idempotent_operation?(op)
|
51
|
-
return unless enqueued_items > 0
|
50
|
+
return enqueued_items unless enqueued_items > 0
|
52
51
|
|
53
52
|
process_until_empty(op: op) unless async_processing?
|
53
|
+
return enqueued_items
|
54
54
|
end
|
55
55
|
end
|
56
56
|
end
|
@@ -59,3 +59,4 @@ require_relative "queue/select_and_lock"
|
|
59
59
|
require_relative "queue/processing"
|
60
60
|
require_relative "queue/callback"
|
61
61
|
require_relative "queue/logging"
|
62
|
+
require_relative "queue/runner"
|
@@ -16,9 +16,27 @@ module Postqueue
|
|
16
16
|
class Queue
|
17
17
|
Timing = Struct.new(:avg_queue_time, :max_queue_time, :total_processing_time, :processing_time)
|
18
18
|
|
19
|
-
def
|
20
|
-
|
19
|
+
def assert_valid_op!(op)
|
20
|
+
return if op == :missing_handler
|
21
|
+
return if op.is_a?(String)
|
22
|
+
|
23
|
+
raise ArgumentError, "Invalid op #{op.inspect}, must be a string"
|
24
|
+
end
|
25
|
+
|
26
|
+
def on(op, batch_size: nil, idempotent: nil, &block)
|
27
|
+
assert_valid_op! op
|
21
28
|
callbacks[op] = block
|
29
|
+
|
30
|
+
if batch_size
|
31
|
+
raise ArgumentError, "Can't set per-op batchsize for op '*'" if op == "*"
|
32
|
+
@batch_sizes[op] = batch_size
|
33
|
+
end
|
34
|
+
|
35
|
+
unless idempotent.nil?
|
36
|
+
raise ArgumentError, "Can't idempotent for default op '*'" if op == "*"
|
37
|
+
@idempotent_operations[op] = idempotent
|
38
|
+
end
|
39
|
+
|
22
40
|
self
|
23
41
|
end
|
24
42
|
|
@@ -32,10 +50,6 @@ module Postqueue
|
|
32
50
|
callbacks[op] || callbacks["*"]
|
33
51
|
end
|
34
52
|
|
35
|
-
def on_missing_handler(op:, entity_ids:)
|
36
|
-
raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
|
37
|
-
end
|
38
|
-
|
39
53
|
private
|
40
54
|
|
41
55
|
def run_callback(op:, entity_ids:)
|
@@ -47,11 +61,8 @@ module Postqueue
|
|
47
61
|
queue_time = queue_times.first
|
48
62
|
|
49
63
|
total_processing_time = Benchmark.realtime do
|
50
|
-
|
51
|
-
|
52
|
-
else
|
53
|
-
on_missing_handler(op: op, entity_ids: entity_ids)
|
54
|
-
end
|
64
|
+
callback = callback_for(op: op) || callbacks.fetch(:missing_handler)
|
65
|
+
callback.call(op, entity_ids)
|
55
66
|
end
|
56
67
|
|
57
68
|
Timing.new(queue_time.avg, queue_time.max, total_processing_time, total_processing_time / entity_ids.length)
|
@@ -11,12 +11,29 @@ module Postqueue
|
|
11
11
|
logger.info msg
|
12
12
|
end
|
13
13
|
|
14
|
-
def
|
14
|
+
def log_exception(exception, op, entity_ids)
|
15
15
|
logger.warn "processing '#{op}' for id(s) #{entity_ids.inspect}: caught #{exception}"
|
16
16
|
end
|
17
17
|
|
18
|
+
def on_exception(&block)
|
19
|
+
if block
|
20
|
+
@on_exception = block
|
21
|
+
if block.arity > -3 && block.arity != 3
|
22
|
+
raise ArgumentError, "Invalid on_exception block: must accept 3 arguments"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
@on_exception
|
27
|
+
end
|
28
|
+
|
29
|
+
public
|
30
|
+
|
18
31
|
def logger
|
19
32
|
Postqueue.logger
|
20
33
|
end
|
34
|
+
|
35
|
+
def to_s
|
36
|
+
item_class.table_name
|
37
|
+
end
|
21
38
|
end
|
22
39
|
end
|
@@ -4,7 +4,7 @@ module Postqueue
|
|
4
4
|
# Processes up to batch_size entries
|
5
5
|
#
|
6
6
|
# process batch_size: 100
|
7
|
-
def process(op: nil, batch_size:
|
7
|
+
def process(op: nil, batch_size: nil)
|
8
8
|
item_class.transaction do
|
9
9
|
process_inside_transaction(op: op, batch_size: batch_size)
|
10
10
|
end
|
@@ -15,7 +15,7 @@ module Postqueue
|
|
15
15
|
process(op: op, batch_size: 1)
|
16
16
|
end
|
17
17
|
|
18
|
-
def process_until_empty(op: nil, batch_size:
|
18
|
+
def process_until_empty(op: nil, batch_size: nil)
|
19
19
|
count = 0
|
20
20
|
loop do
|
21
21
|
processed_items = process(op: op, batch_size: batch_size)
|
@@ -49,9 +49,9 @@ module Postqueue
|
|
49
49
|
|
50
50
|
entity_ids.length
|
51
51
|
rescue => e
|
52
|
-
on_exception(e, match.op, entity_ids)
|
53
52
|
item_class.postpone items.map(&:id)
|
54
|
-
|
53
|
+
log_exception(e, match.op, entity_ids)
|
54
|
+
on_exception.call(e, match.op, entity_ids)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Postqueue
|
2
|
+
# The Postqueue processor processes items in a single Postqueue table.
|
3
|
+
class Queue
|
4
|
+
def run(&block)
|
5
|
+
@run = block if block
|
6
|
+
@run
|
7
|
+
end
|
8
|
+
|
9
|
+
def run!
|
10
|
+
if !run
|
11
|
+
run do |queue|
|
12
|
+
while true do
|
13
|
+
queue.logger.debug "#{queue}: Processing until empty"
|
14
|
+
queue.process_until_empty
|
15
|
+
queue.logger.debug "#{queue}: sleeping"
|
16
|
+
sleep 1
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
run.call(self)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/postqueue/version.rb
CHANGED
@@ -8,8 +8,8 @@ describe "default_queue" do
|
|
8
8
|
let(:processed_events) { @processed_events ||= [] }
|
9
9
|
|
10
10
|
before do
|
11
|
-
queue.
|
12
|
-
queue.
|
11
|
+
queue.on "batchable", batch_size: 10
|
12
|
+
queue.on "other-batchable", batch_size: 10
|
13
13
|
|
14
14
|
queue.on "*" do |op, entity_ids|
|
15
15
|
processed_events << [ op, entity_ids ]
|
@@ -3,8 +3,8 @@ require "spec_helper"
|
|
3
3
|
describe "error handling" do
|
4
4
|
let(:queue) do
|
5
5
|
Postqueue.new do |queue|
|
6
|
-
queue.
|
7
|
-
queue.
|
6
|
+
queue.on "batchable", batch_size: 10
|
7
|
+
queue.on "other-batchable", batch_size: 10
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
@@ -7,8 +7,8 @@ describe "processing" do
|
|
7
7
|
|
8
8
|
let(:queue) do
|
9
9
|
Postqueue.new do |queue|
|
10
|
-
queue.
|
11
|
-
queue.
|
10
|
+
queue.on "batchable", batch_size: 10
|
11
|
+
queue.on "other-batchable", batch_size: 10
|
12
12
|
|
13
13
|
queue.on "*" do |op, entity_ids|
|
14
14
|
processed_events << [ op, entity_ids ]
|
@@ -40,7 +40,7 @@ describe "processing" do
|
|
40
40
|
expect(items.map(&:entity_id)).to contain_exactly(12, 13, 14)
|
41
41
|
end
|
42
42
|
|
43
|
-
it "
|
43
|
+
it "calls the registered handler and returns the processed entries" do
|
44
44
|
queue.enqueue op: "otherop", entity_id: 112
|
45
45
|
called = false
|
46
46
|
queue.process_one(op: "otherop")
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- radiospiel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -136,15 +136,33 @@ dependencies:
|
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: table_print
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
139
153
|
description: a postgres based queue implementation
|
140
154
|
email:
|
141
155
|
- radiospiel@open-lab.org
|
142
|
-
executables:
|
156
|
+
executables:
|
157
|
+
- postqueue
|
143
158
|
extensions: []
|
144
159
|
extra_rdoc_files: []
|
145
160
|
files:
|
146
161
|
- README.md
|
162
|
+
- bin/postqueue
|
147
163
|
- lib/postqueue.rb
|
164
|
+
- lib/postqueue/cli.rb
|
165
|
+
- lib/postqueue/cli/options_parser.rb
|
148
166
|
- lib/postqueue/default_queue.rb
|
149
167
|
- lib/postqueue/item.rb
|
150
168
|
- lib/postqueue/item/enqueue.rb
|
@@ -154,6 +172,7 @@ files:
|
|
154
172
|
- lib/postqueue/queue/callback.rb
|
155
173
|
- lib/postqueue/queue/logging.rb
|
156
174
|
- lib/postqueue/queue/processing.rb
|
175
|
+
- lib/postqueue/queue/runner.rb
|
157
176
|
- lib/postqueue/queue/select_and_lock.rb
|
158
177
|
- lib/postqueue/version.rb
|
159
178
|
- lib/tracker.rb
|