postqueue 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22dcc3ebd8cee3a0da1beec4578662009694403c
-  data.tar.gz: bcfc1044a1970d7055e4ecdde02c9b626d8cc3b6
+  metadata.gz: d9f6d7f05810076cd9c15f8b47515817b2e331be
+  data.tar.gz: 0d0bff63426af6568730ea89e8da3ce3e5a3b015
 SHA512:
-  metadata.gz: 923f4b85b6861c796cb36e33f1dd0687ce9f90ac9d76fe83dcb5911af380ea24b763e7e887e53adc8db4b1410e725ded983d6e012fc78c8ef8f1d252982989b3
-  data.tar.gz: a2343de315aa9be8387e88d13f12f7b65204fe8ecd3671c98241f31f8365fa04701f2c0a95f123e3d78ab7fba00758f72bcfb9c279e54aedcb4919653cb4291c
+  metadata.gz: 6b84633af24db245a98380b96ed17021650945d0ffe554eaaa54c0a3adf8fbf7ecdc8b3036443044371822980f0f0129ab6bb93b89bad4dfce5308ab05d406e7
+  data.tar.gz: 6e9950292583f300ad5f0d03eb777c8337704986231a6b5dee71b19f4d00e4c102ff6b64e7124a66e930520b7193443a1d294d763b5993048c26ebb5d3eb41c8

data/README.md

@@ -33,15 +33,11 @@ and therefore needs at least PostgresQL >= 9.5.
 ```ruby
 queue = Postqueue.new
 queue.enqueue op: "product/reindex", entity_id: [12,13,14,15]
-queue.process do |op, entity_ids|
-  # note: entity_ids is always an Array of ids.
-  case op
-  when "product/reindex"
-    Product.index_many(Product.where(id: entity_ids))
-  else
-    raise "Unsupported op: #{op}"
-  end
+queue.on "product/reindex" do |op, entity_ids|
+  Product.index_many(Product.where(id: entity_ids))
 end
+
+queue.process
 ```
 
 The process call will select a number of queue items for processing. They will all have 
@@ -88,7 +84,9 @@ When enqueueing items duplicate idempotent operations are not enqueued. Whether
 should be considered idempotent is defined when configuring the queue:
 
      Postqueue.new do |queue|
-       queue.idempotent_operation "idempotent"
+       queue.on "idempotent", idempotent: true do ]op, entity_ids|
+         # .. handle queue item
+       end
      end
 
 ### Processing a single entry
@@ -132,7 +130,9 @@ or an operation-specific batch_size:
 
     Postqueue.new do |queue|
       queue.default_batch_size = 100
-      queue.batch_sizes["batchable"] = 10
+      queue.on "batchable", batch_size: 10 do
+        ...
+      end
     end
 
 ## Test mode

data/bin/postqueue

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "postqueue"
+require "postqueue/cli"
+Postqueue::CLI.run ARGV

data/lib/postqueue/cli.rb

@@ -0,0 +1,54 @@
+require "ostruct"
+
+require_relative "cli/options_parser"
+
+module Postqueue
+  module CLI
+    extend self
+
+    attr_reader :options
+
+    def run(argv)
+      @options = OptionsParser.parse_args(argv)
+
+      case options.sub_command
+      when "stats"
+        connect_to_database!
+        sql = "SELECT * FROM #{Postqueue.item_class.table_name}"
+        tp Postqueue.item_class.find_by_sql(sql)
+      when "enqueue"
+        connect_to_database!
+        count = Postqueue.enqueue op: options.op, entity_id: options.entity_ids
+        puts "returned #{count.inspect}"
+        Postqueue.logger.info "Enqueued #{count} queue items"
+      when "process"
+        connect_to_instance!
+        Postqueue.process batch_size: 1
+      when "run"
+        connect_to_instance!
+        Postqueue.run!
+      end
+    end
+
+    def connect_to_instance!
+      path = "#{Dir.getwd}/config/postqueue.rb"
+      Postqueue.logger.info "Loading postqueue configuration from #{path}"
+      load path
+    end
+
+    def connect_to_database!
+      abc = active_record_config
+      username, host, database = abc.values_at "username", "host", "database"
+      Postqueue.logger.info "Connecting to postgres:#{username}@#{host}/#{database}"
+
+      ActiveRecord::Base.establish_connection(abc)
+    end
+
+    def active_record_config
+      require "yaml"
+      database_config = YAML.load_file "config/database.yml"
+      env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      database_config.fetch(env)
+    end
+  end
+end

data/lib/postqueue/cli/options_parser.rb

@@ -0,0 +1,46 @@
+require "ostruct"
+
+module Postqueue
+  module CLI
+    class OptionsParser
+      def self.parse_args(argv)
+        new(argv).parse_args
+      end
+
+      attr_reader :argv
+
+      def initialize(argv)
+        @argv = argv
+      end
+
+      def parse_args
+        require "optparse"
+        options = OpenStruct.new
+        options.sub_command = argv.shift || usage!
+
+        case options.sub_command
+        when "enqueue"
+          options.op = next_arg!
+          options.entity_ids = next_arg!.split(",").map { |s| Integer(s) }
+        end
+        options
+      end
+
+      private
+
+      def next_arg!
+        argv.shift || usage!
+      end
+
+      def usage!
+        STDERR.puts <<-USAGE
+  postqueue stats
+  postqueue enqueue op entity_id,entity_id,entity_id
+  postqueue run
+  postqueue process
+  USAGE
+        exit 1
+      end
+    end
+  end
+end

data/lib/postqueue/default_queue.rb

@@ -14,4 +14,5 @@ module Postqueue
   def_delegators :default_queue, :enqueue
   def_delegators :default_queue, :item_class, :batch_sizes, :on
   def_delegators :default_queue, :process, :process_one
+  def_delegators :default_queue, :run, :run!
 end

data/lib/postqueue/queue.rb

@@ -8,9 +8,6 @@ module Postqueue
     # for an operation.
     attr_accessor :default_batch_size
 
-    # batch size for a given op
-    attr_reader :batch_sizes
-
     # maximum number of processing attempts.
     attr_reader :max_attemps
 
@@ -26,31 +23,34 @@ module Postqueue
       @default_batch_size = 1
       @max_attemps = 5
       @async_processing = Postqueue.async_processing?
+      @idempotent_operations = {}
+      @batch_sizes = {}
+
+      on :missing_handler do |op, entity_ids|
+        raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
+      end
+
+      on_exception do |e, _, _|
+        e.send :raise
+      end
 
       yield self if block
     end
 
     def batch_size(op:)
-      batch_sizes[op] || default_batch_size || 1
-    end
-
-    def idempotent_operations
-      @idempotent_operations ||= {}
+      @batch_sizes[op] || default_batch_size || 1
     end
 
     def idempotent_operation?(op)
-      idempotent_operations.fetch(op) { idempotent_operations.fetch("*", false) }
-    end
-
-    def idempotent_operation(op, flag = true)
-      idempotent_operations[op] = flag
+      @idempotent_operations.fetch(op) { @idempotent_operations.fetch("*", false) }
     end
 
     def enqueue(op:, entity_id:)
       enqueued_items = item_class.enqueue op: op, entity_id: entity_id, ignore_duplicates: idempotent_operation?(op)
-      return unless enqueued_items > 0
+      return enqueued_items unless enqueued_items > 0
 
       process_until_empty(op: op) unless async_processing?
+      return enqueued_items
     end
   end
 end
@@ -59,3 +59,4 @@ require_relative "queue/select_and_lock"
 require_relative "queue/processing"
 require_relative "queue/callback"
 require_relative "queue/logging"
+require_relative "queue/runner"

data/lib/postqueue/queue/callback.rb

@@ -16,9 +16,27 @@ module Postqueue
   class Queue
     Timing = Struct.new(:avg_queue_time, :max_queue_time, :total_processing_time, :processing_time)
 
-    def on(op, &block)
-      raise ArgumentError, "Invalid op #{op.inspect}, must be a string" unless op.is_a?(String)
+    def assert_valid_op!(op)
+      return if op == :missing_handler
+      return if op.is_a?(String)
+
+      raise ArgumentError, "Invalid op #{op.inspect}, must be a string"
+    end
+
+    def on(op, batch_size: nil, idempotent: nil, &block)
+      assert_valid_op! op
       callbacks[op] = block
+
+      if batch_size
+        raise ArgumentError, "Can't set per-op batchsize for op '*'" if op == "*"
+        @batch_sizes[op] = batch_size
+      end
+
+      unless idempotent.nil?
+        raise ArgumentError, "Can't idempotent for default op '*'" if op == "*"
+        @idempotent_operations[op] = idempotent
+      end
+
       self
     end
 
@@ -32,10 +50,6 @@ module Postqueue
       callbacks[op] || callbacks["*"]
     end
 
-    def on_missing_handler(op:, entity_ids:)
-      raise MissingHandler, queue: self, op: op, entity_ids: entity_ids
-    end
-
     private
 
     def run_callback(op:, entity_ids:)
@@ -47,11 +61,8 @@ module Postqueue
       queue_time = queue_times.first
 
       total_processing_time = Benchmark.realtime do
-        if callback = callback_for(op: op)
-          callback.call(op, entity_ids)
-        else
-          on_missing_handler(op: op, entity_ids: entity_ids)
-        end
+        callback = callback_for(op: op) || callbacks.fetch(:missing_handler)
+        callback.call(op, entity_ids)
       end
 
       Timing.new(queue_time.avg, queue_time.max, total_processing_time, total_processing_time / entity_ids.length)

data/lib/postqueue/queue/logging.rb

@@ -11,12 +11,29 @@ module Postqueue
       logger.info msg
     end
 
-    def on_exception(exception, op, entity_ids)
+    def log_exception(exception, op, entity_ids)
       logger.warn "processing '#{op}' for id(s) #{entity_ids.inspect}: caught #{exception}"
     end
 
+    def on_exception(&block)
+      if block
+        @on_exception = block
+        if block.arity > -3 && block.arity != 3
+          raise ArgumentError, "Invalid on_exception block: must accept 3 arguments"
+        end
+      end
+
+      @on_exception
+    end
+
+    public
+
     def logger
       Postqueue.logger
     end
+
+    def to_s
+      item_class.table_name
+    end
   end
 end

data/lib/postqueue/queue/processing.rb

@@ -4,7 +4,7 @@ module Postqueue
     # Processes up to batch_size entries
     #
     # process batch_size: 100
-    def process(op: nil, batch_size: 100)
+    def process(op: nil, batch_size: nil)
       item_class.transaction do
         process_inside_transaction(op: op, batch_size: batch_size)
       end
@@ -15,7 +15,7 @@ module Postqueue
       process(op: op, batch_size: 1)
     end
 
-    def process_until_empty(op: nil, batch_size: 100)
+    def process_until_empty(op: nil, batch_size: nil)
       count = 0
       loop do
         processed_items = process(op: op, batch_size: batch_size)
@@ -49,9 +49,9 @@ module Postqueue
 
       entity_ids.length
     rescue => e
-      on_exception(e, match.op, entity_ids)
       item_class.postpone items.map(&:id)
-      raise
+      log_exception(e, match.op, entity_ids)
+      on_exception.call(e, match.op, entity_ids)
     end
   end
 end

data/lib/postqueue/queue/runner.rb

@@ -0,0 +1,24 @@
+module Postqueue
+  # The Postqueue processor processes items in a single Postqueue table.
+  class Queue
+    def run(&block)
+      @run = block if block
+      @run
+    end
+
+    def run!
+      if !run
+        run do |queue|
+          while true do
+            queue.logger.debug "#{queue}: Processing until empty"
+            queue.process_until_empty
+            queue.logger.debug "#{queue}: sleeping"
+            sleep 1
+          end
+        end
+      end
+
+      run.call(self)
+    end
+  end
+end

data/lib/postqueue/version.rb

@@ -1,3 +1,3 @@
 module Postqueue
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

data/spec/postqueue/default_queue_spec.rb

@@ -8,8 +8,8 @@ describe "default_queue" do
   let(:processed_events) { @processed_events ||= [] }
 
   before do
-    queue.batch_sizes["batchable"] = 10
-    queue.batch_sizes["other-batchable"] = 10
+    queue.on "batchable", batch_size: 10
+    queue.on "other-batchable", batch_size: 10
 
     queue.on "*" do |op, entity_ids|
       processed_events << [ op, entity_ids ]

data/spec/postqueue/idempotent_ops_spec.rb

@@ -3,8 +3,7 @@ require "spec_helper"
 describe "idempotent operations" do
   let(:queue) do
     Postqueue.new do |queue|
-      queue.batch_sizes["batchable"] = 10
-      queue.idempotent_operation "idempotent"
+      queue.on "idempotent", idempotent: true
     end
   end
 

data/spec/postqueue/process_errors_spec.rb

@@ -3,8 +3,8 @@ require "spec_helper"
 describe "error handling" do
   let(:queue) do
     Postqueue.new do |queue|
-      queue.batch_sizes["batchable"] = 10
-      queue.batch_sizes["other-batchable"] = 10
+      queue.on "batchable", batch_size: 10
+      queue.on "other-batchable", batch_size: 10
     end
   end
 

data/spec/postqueue/process_spec.rb

@@ -7,8 +7,8 @@ describe "processing" do
 
   let(:queue) do
     Postqueue.new do |queue|
-      queue.batch_sizes["batchable"] = 10
-      queue.batch_sizes["other-batchable"] = 10
+      queue.on "batchable", batch_size: 10
+      queue.on "other-batchable", batch_size: 10
 
       queue.on "*" do |op, entity_ids|
         processed_events << [ op, entity_ids ]
@@ -40,7 +40,7 @@ describe "processing" do
       expect(items.map(&:entity_id)).to contain_exactly(12, 13, 14)
     end
 
-    it "yields a block and returns the processed entries" do
+    it "calls the registered handler and returns the processed entries" do
       queue.enqueue op: "otherop", entity_id: 112
       called = false
       queue.process_one(op: "otherop")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postqueue
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - radiospiel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-16 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -136,15 +136,33 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: table_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: a postgres based queue implementation
 email:
 - radiospiel@open-lab.org
-executables: []
+executables:
+- postqueue
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
+- bin/postqueue
 - lib/postqueue.rb
+- lib/postqueue/cli.rb
+- lib/postqueue/cli/options_parser.rb
 - lib/postqueue/default_queue.rb
 - lib/postqueue/item.rb
 - lib/postqueue/item/enqueue.rb
@@ -154,6 +172,7 @@ files:
 - lib/postqueue/queue/callback.rb
 - lib/postqueue/queue/logging.rb
 - lib/postqueue/queue/processing.rb
+- lib/postqueue/queue/runner.rb
 - lib/postqueue/queue/select_and_lock.rb
 - lib/postqueue/version.rb
 - lib/tracker.rb