postqueue 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -5
- data/lib/postqueue/cli.rb +9 -6
- data/lib/postqueue/default_queue.rb +1 -0
- data/lib/postqueue/item/enqueue.rb +1 -0
- data/lib/postqueue/logger.rb +9 -1
- data/lib/postqueue/queue/callback.rb +1 -1
- data/lib/postqueue/queue.rb +21 -6
- data/lib/postqueue/railtie.rb +17 -0
- data/lib/postqueue/version.rb +1 -1
- data/lib/postqueue.rb +4 -12
- data/spec/postqueue/enqueue_spec.rb +11 -1
- data/spec/postqueue/{syncmode_spec.rb → process_mode_spec.rb} +23 -9
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6849910b76abd215646bad9dc1f6af63d586e445
|
4
|
+
data.tar.gz: 19eca3b819e0b69db20d0b3a8a5e4eb4816a6100
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5af615c665298638e1e319f633ceacda6cedc2216f31b54fa2616031b6151ca5549e1612b22bd3a4cf581e467f3736825374b0501e05b44a77e8d80b9269d48a
|
7
|
+
data.tar.gz: 2b56b04af5a9a4a2d83d15aeb5e7aee6ffc345404c969f881e3156eeac8a8cf167d62d22bf249d47e07f9d94b1acbb130a5b3e645a1cee93932122f8ca4cc240
|
data/README.md
CHANGED
@@ -137,15 +137,21 @@ or an operation-specific batch_size:
|
|
137
137
|
|
138
138
|
## Test mode
|
139
139
|
|
140
|
-
|
141
|
-
|
140
|
+
Postqueue works usually in an async mode: queue items that are enqueued are kept in a queue,
|
141
|
+
and must be picked up later explicitely for processing (via one of the `process`, `process_one` or `process_until_empty` methods).
|
142
142
|
|
143
|
-
|
143
|
+
During unit tests it is likely preferrable to process queue items in synchronous fashion -
|
144
|
+
if you are interested in actual processing - or, at least, in a mode which validates that
|
145
|
+
the `op` value is valid (that means that a handler is registered for that op). You can
|
146
|
+
change the processing mode via
|
144
147
|
|
145
|
-
|
148
|
+
# can be :sync, :async, :verify
|
149
|
+
Postqueue.processing = :sync
|
150
|
+
|
151
|
+
You can also change the processing mnode on a queue-by-queue base:
|
146
152
|
|
147
153
|
Postqueue.new do |queue|
|
148
|
-
queue.
|
154
|
+
queue.processing = :sync
|
149
155
|
end
|
150
156
|
|
151
157
|
## Installation
|
data/lib/postqueue/cli.rb
CHANGED
@@ -21,18 +21,21 @@ module Postqueue
|
|
21
21
|
count = Postqueue.enqueue op: options.op, entity_id: options.entity_ids
|
22
22
|
Postqueue.logger.info "Enqueued #{count} queue items"
|
23
23
|
when "process"
|
24
|
-
|
24
|
+
connect_to_rails!
|
25
25
|
Postqueue.process batch_size: 1
|
26
26
|
when "run"
|
27
|
-
|
27
|
+
connect_to_rails!
|
28
28
|
Postqueue.run!
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
|
-
def
|
33
|
-
|
34
|
-
|
35
|
-
|
32
|
+
def connect_to_rails!
|
33
|
+
if File.exists?("config/environment.rb")
|
34
|
+
load "config/environment.rb"
|
35
|
+
else
|
36
|
+
logger.warn "Trying to load postqueue configuration from config/postqueue.rb"
|
37
|
+
load "config/postqueue.rb"
|
38
|
+
end
|
36
39
|
end
|
37
40
|
|
38
41
|
def connect_to_database!
|
@@ -14,5 +14,6 @@ module Postqueue
|
|
14
14
|
def_delegators :default_queue, :enqueue
|
15
15
|
def_delegators :default_queue, :item_class, :batch_sizes, :on
|
16
16
|
def_delegators :default_queue, :process, :process_one
|
17
|
+
def_delegators :default_queue, :processing
|
17
18
|
def_delegators :default_queue, :run, :run!
|
18
19
|
end
|
data/lib/postqueue/logger.rb
CHANGED
@@ -4,6 +4,14 @@ module Postqueue
|
|
4
4
|
end
|
5
5
|
|
6
6
|
def self.logger
|
7
|
-
@logger
|
7
|
+
@logger || default_logger
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.default_logger
|
11
|
+
defined?(Rails) ? Rails.logger : stdout_logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.stdout_logger
|
15
|
+
@stdout_logger ||= Logger.new(STDOUT)
|
8
16
|
end
|
9
17
|
end
|
data/lib/postqueue/queue.rb
CHANGED
@@ -11,20 +11,27 @@ module Postqueue
|
|
11
11
|
# maximum number of processing attempts.
|
12
12
|
attr_reader :max_attemps
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
VALID_PROCESSING_VALUES = [ :async, :sync, :verify ]
|
15
|
+
|
16
|
+
# sets or return the processing mode. This must be one of :async, :sync
|
17
|
+
# or :verify
|
18
|
+
def processing(processing = nil)
|
19
|
+
return @processing if processing.nil?
|
17
20
|
|
18
|
-
|
21
|
+
unless VALID_PROCESSING_VALUES.include?(processing)
|
22
|
+
raise ArgumentError, "Invalid processing value, must be one of #{VALID_PROCESSING_VALUES.inspect}"
|
23
|
+
end
|
24
|
+
@processing = processing
|
25
|
+
end
|
19
26
|
|
20
27
|
def initialize(&block)
|
21
28
|
@batch_sizes = {}
|
22
29
|
@item_class = ::Postqueue::Item
|
23
30
|
@default_batch_size = 1
|
24
31
|
@max_attemps = 5
|
25
|
-
@async_processing = Postqueue.async_processing?
|
26
32
|
@idempotent_operations = {}
|
27
33
|
@batch_sizes = {}
|
34
|
+
@processing = :async
|
28
35
|
|
29
36
|
on :missing_handler do |op, entity_ids|
|
30
37
|
raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
|
@@ -49,7 +56,15 @@ module Postqueue
|
|
49
56
|
enqueued_items = item_class.enqueue op: op, entity_id: entity_id, ignore_duplicates: idempotent_operation?(op)
|
50
57
|
return enqueued_items unless enqueued_items > 0
|
51
58
|
|
52
|
-
|
59
|
+
case processing
|
60
|
+
when :async
|
61
|
+
:nop
|
62
|
+
when :sync
|
63
|
+
process_until_empty(op: op)
|
64
|
+
when :verify
|
65
|
+
raise(MissingHandler, queue: self, op: op, entity_ids: [entity_id]) unless callback_for(op: op)
|
66
|
+
end
|
67
|
+
|
53
68
|
enqueued_items
|
54
69
|
end
|
55
70
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Postqueue
|
2
|
+
class Railtie < Rails::Railtie
|
3
|
+
# initializer "postqueue.configure_rails_initialization" do
|
4
|
+
# Postqueue.load_config_from "config/postqueue.rb"
|
5
|
+
# end
|
6
|
+
end
|
7
|
+
|
8
|
+
# def self.load_config_from(path)
|
9
|
+
# @config_file = path
|
10
|
+
# end
|
11
|
+
#
|
12
|
+
# def self.load_config
|
13
|
+
# return unless @config_file
|
14
|
+
# load @config_file
|
15
|
+
# @config_file = nil
|
16
|
+
# end
|
17
|
+
end
|
data/lib/postqueue/version.rb
CHANGED
data/lib/postqueue.rb
CHANGED
@@ -5,19 +5,11 @@ require_relative "postqueue/queue"
|
|
5
5
|
require_relative "postqueue/default_queue"
|
6
6
|
|
7
7
|
module Postqueue
|
8
|
-
|
9
|
-
|
8
|
+
class << self
|
9
|
+
def new(*args, &block)
|
10
|
+
::Postqueue::Queue.new(*args, &block)
|
11
|
+
end
|
10
12
|
end
|
11
|
-
|
12
|
-
def self.async_processing=(async_processing)
|
13
|
-
@async_processing = async_processing
|
14
|
-
end
|
15
|
-
|
16
|
-
def self.async_processing?
|
17
|
-
@async_processing
|
18
|
-
end
|
19
|
-
|
20
|
-
self.async_processing = true
|
21
13
|
end
|
22
14
|
|
23
15
|
# require_relative 'postqueue/railtie' if defined?(Rails)
|
@@ -5,7 +5,7 @@ describe "enqueuing" do
|
|
5
5
|
let(:items) { queue.item_class.all }
|
6
6
|
let(:item) { queue.item_class.first }
|
7
7
|
|
8
|
-
context "when enqueueing
|
8
|
+
context "when enqueueing individual items" do
|
9
9
|
before do
|
10
10
|
queue.enqueue op: "myop", entity_id: 12
|
11
11
|
end
|
@@ -15,6 +15,16 @@ describe "enqueuing" do
|
|
15
15
|
expect(item.entity_id).to eq(12)
|
16
16
|
end
|
17
17
|
|
18
|
+
it "enqueues arrays" do
|
19
|
+
queue.enqueue op: "myop", entity_id: [13,14,15]
|
20
|
+
expect(items.pluck(:entity_id)).to eq([12,13,14,15])
|
21
|
+
end
|
22
|
+
|
23
|
+
it "enqueues sets" do
|
24
|
+
queue.enqueue op: "myop", entity_id: Set.new([13,14,15])
|
25
|
+
expect(items.pluck(:entity_id)).to eq([12,13,14,15])
|
26
|
+
end
|
27
|
+
|
18
28
|
it "sets defaults" do
|
19
29
|
expect(item.created_at).to be > (Time.now - 1.second)
|
20
30
|
expect(item.next_run_at).to be > (Time.now - 1.second)
|
@@ -1,16 +1,8 @@
|
|
1
1
|
require "spec_helper"
|
2
2
|
|
3
|
-
describe "
|
3
|
+
describe "process mode" do
|
4
4
|
let(:callback_invocations) { @callback_invocations ||= [] }
|
5
5
|
|
6
|
-
before :all do
|
7
|
-
Postqueue.async_processing = false
|
8
|
-
end
|
9
|
-
|
10
|
-
after :all do
|
11
|
-
Postqueue.async_processing = true
|
12
|
-
end
|
13
|
-
|
14
6
|
let(:queue) do
|
15
7
|
Postqueue.new do |queue|
|
16
8
|
queue.on "op" do |op, entity_ids|
|
@@ -24,6 +16,7 @@ describe "sync_processing" do
|
|
24
16
|
|
25
17
|
context "when enqueuing in sync mode" do
|
26
18
|
before do
|
19
|
+
queue.processing :sync
|
27
20
|
queue.enqueue op: "op", entity_id: 12
|
28
21
|
end
|
29
22
|
|
@@ -35,4 +28,25 @@ describe "sync_processing" do
|
|
35
28
|
expect(items.count).to eq(0)
|
36
29
|
end
|
37
30
|
end
|
31
|
+
|
32
|
+
context "when enqueuing in test mode" do
|
33
|
+
before do
|
34
|
+
queue.processing :verify
|
35
|
+
queue.enqueue op: "op", entity_id: 12
|
36
|
+
end
|
37
|
+
|
38
|
+
it "does not process the items" do
|
39
|
+
expect(callback_invocations.length).to eq(0)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "does not remove items" do
|
43
|
+
expect(items.count).to eq(1)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "raises an error for invalid ops" do
|
47
|
+
expect {
|
48
|
+
queue.enqueue op: "invalid", entity_id: 12
|
49
|
+
}.to raise_error(Postqueue::MissingHandler)
|
50
|
+
end
|
51
|
+
end
|
38
52
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- radiospiel
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 3.
|
19
|
+
version: 3.5.0
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 3.
|
26
|
+
version: 3.5.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: pry
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -175,6 +175,7 @@ files:
|
|
175
175
|
- lib/postqueue/queue/processing.rb
|
176
176
|
- lib/postqueue/queue/runner.rb
|
177
177
|
- lib/postqueue/queue/select_and_lock.rb
|
178
|
+
- lib/postqueue/railtie.rb
|
178
179
|
- lib/postqueue/version.rb
|
179
180
|
- lib/tracker.rb
|
180
181
|
- lib/tracker/advisory_lock.rb
|
@@ -187,8 +188,8 @@ files:
|
|
187
188
|
- spec/postqueue/idempotent_ops_spec.rb
|
188
189
|
- spec/postqueue/postqueue_spec.rb
|
189
190
|
- spec/postqueue/process_errors_spec.rb
|
191
|
+
- spec/postqueue/process_mode_spec.rb
|
190
192
|
- spec/postqueue/process_spec.rb
|
191
|
-
- spec/postqueue/syncmode_spec.rb
|
192
193
|
- spec/postqueue/wildcard_spec.rb
|
193
194
|
- spec/spec_helper.rb
|
194
195
|
- spec/support/configure_active_record.rb
|