postqueue 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 22dcc3ebd8cee3a0da1beec4578662009694403c
4
- data.tar.gz: bcfc1044a1970d7055e4ecdde02c9b626d8cc3b6
3
+ metadata.gz: d9f6d7f05810076cd9c15f8b47515817b2e331be
4
+ data.tar.gz: 0d0bff63426af6568730ea89e8da3ce3e5a3b015
5
5
  SHA512:
6
- metadata.gz: 923f4b85b6861c796cb36e33f1dd0687ce9f90ac9d76fe83dcb5911af380ea24b763e7e887e53adc8db4b1410e725ded983d6e012fc78c8ef8f1d252982989b3
7
- data.tar.gz: a2343de315aa9be8387e88d13f12f7b65204fe8ecd3671c98241f31f8365fa04701f2c0a95f123e3d78ab7fba00758f72bcfb9c279e54aedcb4919653cb4291c
6
+ metadata.gz: 6b84633af24db245a98380b96ed17021650945d0ffe554eaaa54c0a3adf8fbf7ecdc8b3036443044371822980f0f0129ab6bb93b89bad4dfce5308ab05d406e7
7
+ data.tar.gz: 6e9950292583f300ad5f0d03eb777c8337704986231a6b5dee71b19f4d00e4c102ff6b64e7124a66e930520b7193443a1d294d763b5993048c26ebb5d3eb41c8
data/README.md CHANGED
@@ -33,15 +33,11 @@ and therefore needs at least PostgresQL >= 9.5.
33
33
  ```ruby
34
34
  queue = Postqueue.new
35
35
  queue.enqueue op: "product/reindex", entity_id: [12,13,14,15]
36
- queue.process do |op, entity_ids|
37
- # note: entity_ids is always an Array of ids.
38
- case op
39
- when "product/reindex"
40
- Product.index_many(Product.where(id: entity_ids))
41
- else
42
- raise "Unsupported op: #{op}"
43
- end
36
+ queue.on "product/reindex" do |op, entity_ids|
37
+ Product.index_many(Product.where(id: entity_ids))
44
38
  end
39
+
40
+ queue.process
45
41
  ```
46
42
 
47
43
  The process call will select a number of queue items for processing. They will all have
@@ -88,7 +84,9 @@ When enqueueing items duplicate idempotent operations are not enqueued. Whether
88
84
  should be considered idempotent is defined when configuring the queue:
89
85
 
90
86
  Postqueue.new do |queue|
91
- queue.idempotent_operation "idempotent"
87
+ queue.on "idempotent", idempotent: true do ]op, entity_ids|
88
+ # .. handle queue item
89
+ end
92
90
  end
93
91
 
94
92
  ### Processing a single entry
@@ -132,7 +130,9 @@ or an operation-specific batch_size:
132
130
 
133
131
  Postqueue.new do |queue|
134
132
  queue.default_batch_size = 100
135
- queue.batch_sizes["batchable"] = 10
133
+ queue.on "batchable", batch_size: 10 do
134
+ ...
135
+ end
136
136
  end
137
137
 
138
138
  ## Test mode
data/bin/postqueue ADDED
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+ require "postqueue"
3
+ require "postqueue/cli"
4
+ Postqueue::CLI.run ARGV
@@ -0,0 +1,54 @@
1
+ require "ostruct"
2
+
3
+ require_relative "cli/options_parser"
4
+
5
+ module Postqueue
6
+ module CLI
7
+ extend self
8
+
9
+ attr_reader :options
10
+
11
+ def run(argv)
12
+ @options = OptionsParser.parse_args(argv)
13
+
14
+ case options.sub_command
15
+ when "stats"
16
+ connect_to_database!
17
+ sql = "SELECT * FROM #{Postqueue.item_class.table_name}"
18
+ tp Postqueue.item_class.find_by_sql(sql)
19
+ when "enqueue"
20
+ connect_to_database!
21
+ count = Postqueue.enqueue op: options.op, entity_id: options.entity_ids
22
+ puts "returned #{count.inspect}"
23
+ Postqueue.logger.info "Enqueued #{count} queue items"
24
+ when "process"
25
+ connect_to_instance!
26
+ Postqueue.process batch_size: 1
27
+ when "run"
28
+ connect_to_instance!
29
+ Postqueue.run!
30
+ end
31
+ end
32
+
33
+ def connect_to_instance!
34
+ path = "#{Dir.getwd}/config/postqueue.rb"
35
+ Postqueue.logger.info "Loading postqueue configuration from #{path}"
36
+ load path
37
+ end
38
+
39
+ def connect_to_database!
40
+ abc = active_record_config
41
+ username, host, database = abc.values_at "username", "host", "database"
42
+ Postqueue.logger.info "Connecting to postgres:#{username}@#{host}/#{database}"
43
+
44
+ ActiveRecord::Base.establish_connection(abc)
45
+ end
46
+
47
+ def active_record_config
48
+ require "yaml"
49
+ database_config = YAML.load_file "config/database.yml"
50
+ env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
51
+ database_config.fetch(env)
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,46 @@
1
+ require "ostruct"
2
+
3
+ module Postqueue
4
+ module CLI
5
+ class OptionsParser
6
+ def self.parse_args(argv)
7
+ new(argv).parse_args
8
+ end
9
+
10
+ attr_reader :argv
11
+
12
+ def initialize(argv)
13
+ @argv = argv
14
+ end
15
+
16
+ def parse_args
17
+ require "optparse"
18
+ options = OpenStruct.new
19
+ options.sub_command = argv.shift || usage!
20
+
21
+ case options.sub_command
22
+ when "enqueue"
23
+ options.op = next_arg!
24
+ options.entity_ids = next_arg!.split(",").map { |s| Integer(s) }
25
+ end
26
+ options
27
+ end
28
+
29
+ private
30
+
31
+ def next_arg!
32
+ argv.shift || usage!
33
+ end
34
+
35
+ def usage!
36
+ STDERR.puts <<-USAGE
37
+ postqueue stats
38
+ postqueue enqueue op entity_id,entity_id,entity_id
39
+ postqueue run
40
+ postqueue process
41
+ USAGE
42
+ exit 1
43
+ end
44
+ end
45
+ end
46
+ end
@@ -14,4 +14,5 @@ module Postqueue
14
14
  def_delegators :default_queue, :enqueue
15
15
  def_delegators :default_queue, :item_class, :batch_sizes, :on
16
16
  def_delegators :default_queue, :process, :process_one
17
+ def_delegators :default_queue, :run, :run!
17
18
  end
@@ -8,9 +8,6 @@ module Postqueue
8
8
  # for an operation.
9
9
  attr_accessor :default_batch_size
10
10
 
11
- # batch size for a given op
12
- attr_reader :batch_sizes
13
-
14
11
  # maximum number of processing attempts.
15
12
  attr_reader :max_attemps
16
13
 
@@ -26,31 +23,34 @@ module Postqueue
26
23
  @default_batch_size = 1
27
24
  @max_attemps = 5
28
25
  @async_processing = Postqueue.async_processing?
26
+ @idempotent_operations = {}
27
+ @batch_sizes = {}
28
+
29
+ on :missing_handler do |op, entity_ids|
30
+ raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
31
+ end
32
+
33
+ on_exception do |e, _, _|
34
+ e.send :raise
35
+ end
29
36
 
30
37
  yield self if block
31
38
  end
32
39
 
33
40
  def batch_size(op:)
34
- batch_sizes[op] || default_batch_size || 1
35
- end
36
-
37
- def idempotent_operations
38
- @idempotent_operations ||= {}
41
+ @batch_sizes[op] || default_batch_size || 1
39
42
  end
40
43
 
41
44
  def idempotent_operation?(op)
42
- idempotent_operations.fetch(op) { idempotent_operations.fetch("*", false) }
43
- end
44
-
45
- def idempotent_operation(op, flag = true)
46
- idempotent_operations[op] = flag
45
+ @idempotent_operations.fetch(op) { @idempotent_operations.fetch("*", false) }
47
46
  end
48
47
 
49
48
  def enqueue(op:, entity_id:)
50
49
  enqueued_items = item_class.enqueue op: op, entity_id: entity_id, ignore_duplicates: idempotent_operation?(op)
51
- return unless enqueued_items > 0
50
+ return enqueued_items unless enqueued_items > 0
52
51
 
53
52
  process_until_empty(op: op) unless async_processing?
53
+ return enqueued_items
54
54
  end
55
55
  end
56
56
  end
@@ -59,3 +59,4 @@ require_relative "queue/select_and_lock"
59
59
  require_relative "queue/processing"
60
60
  require_relative "queue/callback"
61
61
  require_relative "queue/logging"
62
+ require_relative "queue/runner"
@@ -16,9 +16,27 @@ module Postqueue
16
16
  class Queue
17
17
  Timing = Struct.new(:avg_queue_time, :max_queue_time, :total_processing_time, :processing_time)
18
18
 
19
- def on(op, &block)
20
- raise ArgumentError, "Invalid op #{op.inspect}, must be a string" unless op.is_a?(String)
19
+ def assert_valid_op!(op)
20
+ return if op == :missing_handler
21
+ return if op.is_a?(String)
22
+
23
+ raise ArgumentError, "Invalid op #{op.inspect}, must be a string"
24
+ end
25
+
26
+ def on(op, batch_size: nil, idempotent: nil, &block)
27
+ assert_valid_op! op
21
28
  callbacks[op] = block
29
+
30
+ if batch_size
31
+ raise ArgumentError, "Can't set per-op batchsize for op '*'" if op == "*"
32
+ @batch_sizes[op] = batch_size
33
+ end
34
+
35
+ unless idempotent.nil?
36
+ raise ArgumentError, "Can't idempotent for default op '*'" if op == "*"
37
+ @idempotent_operations[op] = idempotent
38
+ end
39
+
22
40
  self
23
41
  end
24
42
 
@@ -32,10 +50,6 @@ module Postqueue
32
50
  callbacks[op] || callbacks["*"]
33
51
  end
34
52
 
35
- def on_missing_handler(op:, entity_ids:)
36
- raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
37
- end
38
-
39
53
  private
40
54
 
41
55
  def run_callback(op:, entity_ids:)
@@ -47,11 +61,8 @@ module Postqueue
47
61
  queue_time = queue_times.first
48
62
 
49
63
  total_processing_time = Benchmark.realtime do
50
- if callback = callback_for(op: op)
51
- callback.call(op, entity_ids)
52
- else
53
- on_missing_handler(op: op, entity_ids: entity_ids)
54
- end
64
+ callback = callback_for(op: op) || callbacks.fetch(:missing_handler)
65
+ callback.call(op, entity_ids)
55
66
  end
56
67
 
57
68
  Timing.new(queue_time.avg, queue_time.max, total_processing_time, total_processing_time / entity_ids.length)
@@ -11,12 +11,29 @@ module Postqueue
11
11
  logger.info msg
12
12
  end
13
13
 
14
- def on_exception(exception, op, entity_ids)
14
+ def log_exception(exception, op, entity_ids)
15
15
  logger.warn "processing '#{op}' for id(s) #{entity_ids.inspect}: caught #{exception}"
16
16
  end
17
17
 
18
+ def on_exception(&block)
19
+ if block
20
+ @on_exception = block
21
+ if block.arity > -3 && block.arity != 3
22
+ raise ArgumentError, "Invalid on_exception block: must accept 3 arguments"
23
+ end
24
+ end
25
+
26
+ @on_exception
27
+ end
28
+
29
+ public
30
+
18
31
  def logger
19
32
  Postqueue.logger
20
33
  end
34
+
35
+ def to_s
36
+ item_class.table_name
37
+ end
21
38
  end
22
39
  end
@@ -4,7 +4,7 @@ module Postqueue
4
4
  # Processes up to batch_size entries
5
5
  #
6
6
  # process batch_size: 100
7
- def process(op: nil, batch_size: 100)
7
+ def process(op: nil, batch_size: nil)
8
8
  item_class.transaction do
9
9
  process_inside_transaction(op: op, batch_size: batch_size)
10
10
  end
@@ -15,7 +15,7 @@ module Postqueue
15
15
  process(op: op, batch_size: 1)
16
16
  end
17
17
 
18
- def process_until_empty(op: nil, batch_size: 100)
18
+ def process_until_empty(op: nil, batch_size: nil)
19
19
  count = 0
20
20
  loop do
21
21
  processed_items = process(op: op, batch_size: batch_size)
@@ -49,9 +49,9 @@ module Postqueue
49
49
 
50
50
  entity_ids.length
51
51
  rescue => e
52
- on_exception(e, match.op, entity_ids)
53
52
  item_class.postpone items.map(&:id)
54
- raise
53
+ log_exception(e, match.op, entity_ids)
54
+ on_exception.call(e, match.op, entity_ids)
55
55
  end
56
56
  end
57
57
  end
@@ -0,0 +1,24 @@
1
+ module Postqueue
2
+ # The Postqueue processor processes items in a single Postqueue table.
3
+ class Queue
4
+ def run(&block)
5
+ @run = block if block
6
+ @run
7
+ end
8
+
9
+ def run!
10
+ if !run
11
+ run do |queue|
12
+ while true do
13
+ queue.logger.debug "#{queue}: Processing until empty"
14
+ queue.process_until_empty
15
+ queue.logger.debug "#{queue}: sleeping"
16
+ sleep 1
17
+ end
18
+ end
19
+ end
20
+
21
+ run.call(self)
22
+ end
23
+ end
24
+ end
@@ -1,3 +1,3 @@
1
1
  module Postqueue
2
- VERSION = "0.4.1"
2
+ VERSION = "0.4.2"
3
3
  end
@@ -8,8 +8,8 @@ describe "default_queue" do
8
8
  let(:processed_events) { @processed_events ||= [] }
9
9
 
10
10
  before do
11
- queue.batch_sizes["batchable"] = 10
12
- queue.batch_sizes["other-batchable"] = 10
11
+ queue.on "batchable", batch_size: 10
12
+ queue.on "other-batchable", batch_size: 10
13
13
 
14
14
  queue.on "*" do |op, entity_ids|
15
15
  processed_events << [ op, entity_ids ]
@@ -3,8 +3,7 @@ require "spec_helper"
3
3
  describe "idempotent operations" do
4
4
  let(:queue) do
5
5
  Postqueue.new do |queue|
6
- queue.batch_sizes["batchable"] = 10
7
- queue.idempotent_operation "idempotent"
6
+ queue.on "idempotent", idempotent: true
8
7
  end
9
8
  end
10
9
 
@@ -3,8 +3,8 @@ require "spec_helper"
3
3
  describe "error handling" do
4
4
  let(:queue) do
5
5
  Postqueue.new do |queue|
6
- queue.batch_sizes["batchable"] = 10
7
- queue.batch_sizes["other-batchable"] = 10
6
+ queue.on "batchable", batch_size: 10
7
+ queue.on "other-batchable", batch_size: 10
8
8
  end
9
9
  end
10
10
 
@@ -7,8 +7,8 @@ describe "processing" do
7
7
 
8
8
  let(:queue) do
9
9
  Postqueue.new do |queue|
10
- queue.batch_sizes["batchable"] = 10
11
- queue.batch_sizes["other-batchable"] = 10
10
+ queue.on "batchable", batch_size: 10
11
+ queue.on "other-batchable", batch_size: 10
12
12
 
13
13
  queue.on "*" do |op, entity_ids|
14
14
  processed_events << [ op, entity_ids ]
@@ -40,7 +40,7 @@ describe "processing" do
40
40
  expect(items.map(&:entity_id)).to contain_exactly(12, 13, 14)
41
41
  end
42
42
 
43
- it "yields a block and returns the processed entries" do
43
+ it "calls the registered handler and returns the processed entries" do
44
44
  queue.enqueue op: "otherop", entity_id: 112
45
45
  called = false
46
46
  queue.process_one(op: "otherop")
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: postqueue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - radiospiel
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-16 00:00:00.000000000 Z
11
+ date: 2016-12-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -136,15 +136,33 @@ dependencies:
136
136
  - - ">="
137
137
  - !ruby/object:Gem::Version
138
138
  version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: table_print
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
139
153
  description: a postgres based queue implementation
140
154
  email:
141
155
  - radiospiel@open-lab.org
142
- executables: []
156
+ executables:
157
+ - postqueue
143
158
  extensions: []
144
159
  extra_rdoc_files: []
145
160
  files:
146
161
  - README.md
162
+ - bin/postqueue
147
163
  - lib/postqueue.rb
164
+ - lib/postqueue/cli.rb
165
+ - lib/postqueue/cli/options_parser.rb
148
166
  - lib/postqueue/default_queue.rb
149
167
  - lib/postqueue/item.rb
150
168
  - lib/postqueue/item/enqueue.rb
@@ -154,6 +172,7 @@ files:
154
172
  - lib/postqueue/queue/callback.rb
155
173
  - lib/postqueue/queue/logging.rb
156
174
  - lib/postqueue/queue/processing.rb
175
+ - lib/postqueue/queue/runner.rb
157
176
  - lib/postqueue/queue/select_and_lock.rb
158
177
  - lib/postqueue/version.rb
159
178
  - lib/tracker.rb