postqueue 0.2.1 → 0.4.0

data/spec/postqueue/process_errors_spec.rb

@@ -1,68 +1,55 @@
 require "spec_helper"
 
-describe "::queue.process_one" do
-  let(:queue) { Postqueue::Base.new }
+describe "error handling" do
+  let(:queue) do
+    Postqueue.new do |queue|
+      queue.batch_sizes["batchable"] = 10
+      queue.batch_sizes["other-batchable"] = 10
+    end
+  end
 
-  class E < RuntimeError; end
+  let(:items) { queue.item_class.all }
+  let(:item)  { queue.item_class.first }
 
-  before do
-    queue.enqueue op: "mytype", entity_id: 12
-  end
+  class E < RuntimeError; end
 
-  context "block raises an exception" do
+  context "when handler raises an exception" do
     before do
-      expect { queue.process_one { |_op, _type, _ids| raise E } }.to raise_error(E)
-    end
-
-    it "reraises the exception" do
-      # checked in before block
+      queue.on "mytype" do raise E end
+      queue.enqueue op: "mytype", entity_id: 12
     end
 
-    it "keeps the item in the queue" do
+    it "reraises the exception, keeps the item in the queue and increments the failed_attempt count" do
+      expect { queue.process_one }.to raise_error(E)
       expect(items.map(&:entity_id)).to contain_exactly(12)
-    end
-
-    it "increments the failed_attempt count" do
       expect(items.map(&:failed_attempts)).to contain_exactly(1)
     end
   end
 
-  context "block returns false" do
+  context "when no handler can be found" do
     before do
-      @result = queue.process_one { |_op, _type, _ids| false }
-    end
-
-    it "returns false" do
-      expect(@result).to be false
-    end
-
-    it "keeps the item in the queue" do
-      expect(items.map(&:entity_id)).to contain_exactly(12)
+      queue.enqueue op: "mytype", entity_id: 12
     end
 
-    it "increments the failed_attempt count" do
-      expect(items.map(&:failed_attempts)).to contain_exactly(1)
+    it "raises a MissingHandler exception" do
+      expect { queue.process_one }.to raise_error(::Postqueue::MissingHandler)
     end
   end
 
   context "failed_attempts reached MAX_ATTEMPTS" do
     before do
-      expect(Postqueue::MAX_ATTEMPTS).to be >= 3
-      items.update_all(failed_attempts: Postqueue::MAX_ATTEMPTS)
-
-      @called_block = 0
-      @result = queue.process_one do
-        @called_block += 1
-        false
-      end
+      expect(queue.max_attemps).to be >= 3
+      queue.enqueue op: "mytype", entity_id: 12
+      items.update_all(failed_attempts: queue.max_attemps)
+      queue.process_one
     end
 
     it "does not call the block" do
-      expect(@called_block).to eq(0)
+      expect { queue.process_one }.not_to raise_error
     end
 
-    it "returns nil" do
-      expect(@result).to eq(nil)
+    it "returns 0" do
+      expect(queue.process_one).to eq(0)
     end
 
     it "does not remove the item" do
@@ -70,7 +57,7 @@ describe "::queue.process_one" do
     end
 
     it "does not increment the failed_attempts count" do
-      expect(items.first.failed_attempts).to eq(Postqueue::MAX_ATTEMPTS)
+      expect(items.first.failed_attempts).to eq(queue.max_attemps)
     end
   end
 end

data/spec/postqueue/process_spec.rb

@@ -1,14 +1,23 @@
 require "spec_helper"
 
-describe "::queue.process" do
-  class Testqueue < Postqueue::Base
-    def batch_size(op:)
-      _ = op
-      10
+describe "processing" do
+  let(:processed_events) do
+    @processed_events ||= []
+  end
+
+  let(:queue) do
+    Postqueue.new do |queue|
+      queue.batch_sizes["batchable"] = 10
+      queue.batch_sizes["other-batchable"] = 10
+
+      queue.on '*' do |op, entity_ids|
+        processed_events << [ op, entity_ids ]
+      end
     end
   end
 
-  let(:queue) { Testqueue.new }
+  let(:items) { queue.item_class.all }
+  let(:item)  { queue.item_class.first }
 
   describe "basics" do
     before do
@@ -19,7 +28,7 @@ describe "::queue.process" do
 
     it "processes the first entry" do
       r = queue.process_one
-      expect(r).to eq(["myop", [12]])
+      expect(r).to eq(1)
       expect(items.map(&:entity_id)).to contain_exactly(13, 14)
     end
 
@@ -27,53 +36,53 @@ describe "::queue.process" do
       queue.enqueue(op: "otherop", entity_id: 112)
 
       r = queue.process_one(op: "otherop")
-      expect(r).to eq(["otherop", [112]])
+      expect(r).to eq(1)
       expect(items.map(&:entity_id)).to contain_exactly(12, 13, 14)
     end
 
-    it "yields a block and returns its return value" do
+    it "yields a block and returns the processed entries" do
       queue.enqueue op: "otherop", entity_id: 112
-      r = queue.process_one(op: "otherop") do |op, ids|
-        expect(op).to eq("otherop")
-        expect(ids).to eq([112])
-        "yihaa"
-      end
+      called = false
+      queue.process_one(op: "otherop")
+      
+      op, ids = processed_events.first
+      expect(op).to eq("otherop")
+      expect(ids).to eq([112])
 
-      expect(r).to eq("yihaa")
       expect(items.map(&:entity_id)).to contain_exactly(12, 13, 14)
     end
   end
 
   context "when having entries with different entity_type and op" do
     before do
-      queue.enqueue op: "myop", entity_id: 12
-      queue.enqueue op: "myop", entity_id: 13
-      queue.enqueue op: "otherop", entity_id: 14
-      queue.enqueue op: "myop", entity_id: 15
-      queue.enqueue op: "otherop", entity_id: 16
+      queue.enqueue op: "batchable", entity_id: 12
+      queue.enqueue op: "batchable", entity_id: 13
+      queue.enqueue op: "other-batchable", entity_id: 14
+      queue.enqueue op: "batchable", entity_id: 15
+      queue.enqueue op: "other-batchable", entity_id: 16
     end
 
-    it "processes one entries" do
+    it "processes one matching entry with batch_size 1" do
       r = queue.process batch_size: 1
-      expect(r).to eq(["myop", [12]])
+      expect(r).to eq(1)
       expect(items.map(&:entity_id)).to contain_exactly(13, 14, 15, 16)
     end
 
-    it "processes two entries" do
+    it "processes two matching entries" do
       r = queue.process batch_size: 2
-      expect(r).to eq(["myop", [12, 13]])
+      expect(r).to eq(2)
       expect(items.map(&:entity_id)).to contain_exactly(14, 15, 16)
     end
 
-    it "processes only matching entries when asked for more" do
+    it "processes all matching entries" do
       r = queue.process
-      expect(r).to eq(["myop", [12, 13, 15]])
+      expect(r).to eq(3)
       expect(items.map(&:entity_id)).to contain_exactly(14, 16)
     end
 
     it "honors search conditions" do
-      r = queue.process(op: "otherop")
-      expect(r).to eq(["otherop", [14, 16]])
+      r = queue.process(op: "other-batchable")
+      expect(r).to eq(2)
       expect(items.map(&:entity_id)).to contain_exactly(12, 13, 15)
     end
   end

data/spec/postqueue/syncmode_spec.rb

@@ -0,0 +1,38 @@
+require "spec_helper"
+
+describe "sync_processing" do
+  let(:callback_invocations) { @callback_invocations ||= [] }
+
+  before :all do
+    Postqueue.async_processing = false
+  end
+
+  after :all do
+    Postqueue.async_processing = true
+  end
+
+  let(:queue) do
+    Postqueue.new do |queue|
+      queue.on "op" do |op, entity_ids|
+        callback_invocations << [ op, entity_ids ]
+      end
+    end
+  end
+
+  let(:items) { queue.item_class.all }
+  let(:item)  { queue.item_class.first }
+
+  context "when enqueuing in sync mode" do
+    before do
+      queue.enqueue op: "op", entity_id: 12
+    end
+
+    it "processed the items" do
+      expect(callback_invocations.length).to eq(1)
+    end
+
+    it "removed all items" do
+      expect(items.count).to eq(0)
+    end
+  end
+end

data/spec/postqueue/wildcard_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+describe "wildcard processing" do
+  let(:callback_invocations) { @callback_invocations ||= [] }
+
+  let(:queue) do
+    Postqueue.new do |queue|
+      queue.on "*" do |op, entity_ids|
+        callback_invocations << [ op, entity_ids ]
+      end
+    end
+  end
+
+  let(:items) { queue.item_class.all }
+  let(:item)  { queue.item_class.first }
+
+  context "when enqueuing in sync mode" do
+    before do
+      queue.enqueue op: "op", entity_id: 12
+      queue.process
+    end
+
+    it "processed the items" do
+      expect(callback_invocations.length).to eq(1)
+    end
+
+    it "removed all items" do
+      expect(items.count).to eq(0)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,6 +1,3 @@
-path = File.expand_path("../../mpx/lib", __FILE__)
-$LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
-
 ENV["RACK_ENV"] = "test"
 
 require "rspec"
@@ -14,15 +11,7 @@ end
 require "postqueue"
 require "./spec/support/configure_active_record"
 
-module Postqueue
-  def self.logger
-    @logger ||= Logger.new(File.open("log/test.log", "a"))
-  end
-end
-
-def items
-  Postqueue::Item.all
-end
+Postqueue.logger = Logger.new(File.open("log/test.log", "a"))
 
 RSpec.configure do |config|
   config.run_all_when_everything_filtered = true

data/spec/support/configure_active_record.rb

@@ -1,23 +1,17 @@
-require "active_record"
-
-$LOAD_PATH << File.dirname(__FILE__)
-
-ActiveRecord::Base.establish_connection(adapter: "postgresql",
-                                        database: "postqueue_test",
-                                        username: "postqueue",
-                                        password: "postqueue")
-
-# require_relative "schema.rb"
-# require_relative "models.rb"
+require_relative "./connect_active_record"
 
 Postqueue.unmigrate!
 Postqueue.migrate!
 
 RSpec.configure do |config|
   config.around(:each) do |example|
-    ActiveRecord::Base.connection.transaction do
+    if example.metadata[:transactions] == false
       example.run
-      raise ActiveRecord::Rollback, "Clean up"
+    else
+      ActiveRecord::Base.connection.transaction do
+        example.run
+        raise ActiveRecord::Rollback, "Clean up"
+      end
     end
   end
 end

data/spec/support/connect_active_record.rb

@@ -0,0 +1,5 @@
+require "active_record"
+ActiveRecord::Base.establish_connection(adapter: "postgresql",
+                                        database: "postqueue_test",
+                                        username: "postqueue",
+                                        password: "postqueue")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postqueue
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.4.0
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-11 00:00:00.000000000 Z
+date: 2016-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -145,19 +145,32 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/postqueue.rb
-- lib/postqueue/base.rb
-- lib/postqueue/base/callback.rb
-- lib/postqueue/base/enqueue.rb
-- lib/postqueue/base/processing.rb
-- lib/postqueue/base/select_and_lock.rb
 - lib/postqueue/item.rb
+- lib/postqueue/item/enqueue.rb
+- lib/postqueue/item/inserter.rb
+- lib/postqueue/logger.rb
+- lib/postqueue/queue.rb
+- lib/postqueue/queue/callback.rb
+- lib/postqueue/queue/logging.rb
+- lib/postqueue/queue/processing.rb
+- lib/postqueue/queue/select_and_lock.rb
 - lib/postqueue/version.rb
+- lib/tracker.rb
+- lib/tracker/advisory_lock.rb
+- lib/tracker/migration.rb
+- lib/tracker/registry.rb
+- lib/tracker/tracker.sql
+- spec/postqueue/concurrency_spec.rb
 - spec/postqueue/enqueue_spec.rb
+- spec/postqueue/idempotent_ops_spec.rb
 - spec/postqueue/postqueue_spec.rb
 - spec/postqueue/process_errors_spec.rb
 - spec/postqueue/process_spec.rb
+- spec/postqueue/syncmode_spec.rb
+- spec/postqueue/wildcard_spec.rb
 - spec/spec_helper.rb
 - spec/support/configure_active_record.rb
+- spec/support/connect_active_record.rb
 homepage: https://github.com/radiospiel/postqueue
 licenses:
 - MIT

data/lib/postqueue/base/callback.rb

@@ -1,23 +0,0 @@
-module Postqueue
-  class Base
-    def run_callback(op:, entity_ids:, &_block)
-      queue_times = item_class.find_by_sql <<-SQL
-        SELECT extract('epoch' from AVG(now() - created_at)) AS avg,
-               extract('epoch' from MAX(now() - created_at)) AS max
-        FROM #{item_class.table_name} WHERE entity_id IN (#{entity_ids.join(',')})
-      SQL
-      queue_time = queue_times.first
-
-      # run callback.
-      result = [ op, entity_ids ]
-
-      total_processing_time = Benchmark.realtime do
-        result = yield(*result) if block_given?
-      end
-
-      timing = Timing.new(queue_time.avg, queue_time.max, total_processing_time, total_processing_time / entity_ids.length)
-
-      [ result, timing ]
-    end
-  end
-end

data/lib/postqueue/base/enqueue.rb

@@ -1,31 +0,0 @@
-module Postqueue
-  class Base
-    # Enqueues an queue item. If the operation is duplicate, and an entry with
-    # the same combination of op and entity_id exists already, no new entry will
-    # be added to the queue.
-    #
-    # [TODO] An optimized code path, talking directly to PG, might be faster by a factor of 4 or so.
-    def enqueue(op:, entity_id:, duplicate: true)
-      if entity_id.is_a?(Array)
-        enqueue_many(op: op, entity_ids: entity_id, duplicate: duplicate)
-        return
-      end
-
-      if !duplicate && item_class.where(op: op, entity_id: entity_id).present?
-        return
-      end
-
-      item_class.create!(op: op, entity_id: entity_id)
-    end
-
-    private
-
-    def enqueue_many(op:, entity_ids:, duplicate:) #:nodoc:
-      item_class.transaction do
-        entity_ids.each do |entity_id|
-          enqueue(op: op, entity_id: entity_id, duplicate: duplicate)
-        end
-      end
-    end
-  end
-end

data/lib/postqueue/base/processing.rb

@@ -1,57 +0,0 @@
-module Postqueue
-  MAX_ATTEMPTS = 5
-
-  class Base
-    # Processes many entries
-    #
-    # process batch_size: 100
-    def process(op: nil, batch_size: 100, &block)
-      status, result = item_class.transaction do
-        process_inside_transaction(op: op, batch_size: batch_size, &block)
-      end
-
-      raise result if status == :err
-      result
-    end
-
-    def process_one(op: nil, &block)
-      process(op: op, batch_size: 1, &block)
-    end
-
-    private
-
-    # The actual processing. Returns [ :ok, number-of-items ] or  [ :err, exception ]
-    def process_inside_transaction(op:, batch_size:, &block)
-      batch = select_and_lock_batch(op: op, batch_size: batch_size)
-
-      match = batch.first
-      return [ :ok, nil ] unless match
-
-      entity_ids = batch.map(&:entity_id)
-      result, timing = run_callback(op: match.op, entity_ids: entity_ids, &block)
-
-      # Depending on the result either reprocess or delete all items
-      if result == false
-        postpone batch.map(&:id)
-      else
-        on_processing(match.op, entity_ids, timing)
-        item_class.where(id: batch.map(&:id)).delete_all
-      end
-
-      [ :ok, result ]
-    rescue => e
-      on_exception(e, match.op, entity_ids)
-      postpone batch.map(&:id)
-      [ :err, e ]
-    end
-
-    def postpone(ids)
-      item_class.connection.exec_query <<-SQL
-        UPDATE #{item_class.table_name}
-          SET failed_attempts = failed_attempts+1,
-              next_run_at = next_run_at + power(failed_attempts + 1, 1.5) * interval '10 second'
-          WHERE id IN (#{ids.join(',')})
-      SQL
-    end
-  end
-end

data/lib/postqueue/base.rb

@@ -1,41 +0,0 @@
-module Postqueue
-  Timing = Struct.new(:avg_queue_time, :max_queue_time, :total_processing_time, :processing_time)
-
-  class Base
-    private
-
-    def batch_size(op:)
-      _ = op
-      1
-    end
-
-    def item_class
-      Postqueue::Item
-    end
-
-    def logger
-      Postqueue.logger
-    end
-
-    def on_processing(op, entity_ids, timing)
-      msg = "processing '#{op}' for id(s) #{entity_ids.join(',')}: "
-      msg += "processing #{entity_ids.length} items took #{'%.3f msecs' % timing.total_processing_time}"
-
-      msg += ", queue_time: avg: #{'%.3f msecs' % timing.avg_queue_time}/max: #{'%.3f msecs' % timing.max_queue_time}"
-      logger.info msg
-    end
-
-    def on_exception(exception, op, entity_ids)
-      logger.warn "processing '#{op}' for id(s) #{entity_ids.inspect}: caught #{exception}"
-    end
-  end
-
-  def self.logger
-    Logger.new(STDERR)
-  end
-end
-
-require "postqueue/base/enqueue"
-require "postqueue/base/select_and_lock"
-require "postqueue/base/processing"
-require "postqueue/base/callback"