postqueue 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9f6d7f05810076cd9c15f8b47515817b2e331be
-  data.tar.gz: 0d0bff63426af6568730ea89e8da3ce3e5a3b015
+  metadata.gz: 2edb04093f702ef2fb42d716de3ebdc71c9a65f3
+  data.tar.gz: f2148b265f5669b0b8a81610d1b355d7e8a9036e
 SHA512:
-  metadata.gz: 6b84633af24db245a98380b96ed17021650945d0ffe554eaaa54c0a3adf8fbf7ecdc8b3036443044371822980f0f0129ab6bb93b89bad4dfce5308ab05d406e7
-  data.tar.gz: 6e9950292583f300ad5f0d03eb777c8337704986231a6b5dee71b19f4d00e4c102ff6b64e7124a66e930520b7193443a1d294d763b5993048c26ebb5d3eb41c8
+  metadata.gz: 6b2a2098f86dfa48039e57e3c61d7491c22f19cb0e016af0af0e055d561dc0ff91605e0b3b2a4ae3c5f16acda96aa4e606a8fb4a9d326e97740bca50f2e5a615
+  data.tar.gz: 9bb0c84cb618a457d5410ef7c2337035ffe15573085940011730165a2b1791b5f05f4a2bf8ca766ea879d7a6d68ebe00e179147ba3fc7ac5247647363dc813a9

data/lib/postqueue/cli/stats.rb

@@ -0,0 +1,29 @@
+require "ostruct"
+
+module Postqueue
+  module CLI
+    module Stats
+      module_function
+
+      def stats(_options)
+        require "table_print"
+        sql = <<-SQL
+        SELECT op,
+          COUNT(*) AS count,
+          MIN(now() - created_at) AS min_age,
+          MAX(now() - created_at) AS max_age,
+          AVG(now() - created_at) AS avg_age
+        FROM #{Postqueue.item_class.table_name} GROUP BY op
+        SQL
+
+        recs = Postqueue.item_class.find_by_sql(sql)
+        tp recs, :op, :count, :avg_age, :min_age, :max_age
+      end
+
+      def peek(_options)
+        require "table_print"
+        tp Postqueue.default_queue.upcoming(subselect: false).limit(100).all
+      end
+    end
+  end
+end

data/lib/postqueue/cli.rb

@@ -1,6 +1,7 @@
 require "ostruct"
 
 require_relative "cli/options_parser"
+require_relative "cli/stats"
 
 module Postqueue
   module CLI
@@ -12,14 +13,12 @@ module Postqueue
       @options = OptionsParser.parse_args(argv)
 
       case options.sub_command
-      when "stats"
+      when "stats", "peek"
         connect_to_database!
-        sql = "SELECT * FROM #{Postqueue.item_class.table_name}"
-        tp Postqueue.item_class.find_by_sql(sql)
+        Stats.send options.sub_command, options
       when "enqueue"
         connect_to_database!
         count = Postqueue.enqueue op: options.op, entity_id: options.entity_ids
-        puts "returned #{count.inspect}"
         Postqueue.logger.info "Enqueued #{count} queue items"
       when "process"
         connect_to_instance!

data/lib/postqueue/queue/callback.rb

@@ -50,8 +50,6 @@ module Postqueue
       callbacks[op] || callbacks["*"]
     end
 
-    private
-
     def run_callback(op:, entity_ids:)
       queue_times = item_class.find_by_sql <<-SQL
         SELECT extract('epoch' from AVG(now() - created_at)) AS avg,

data/lib/postqueue/queue/runner.rb

@@ -7,18 +7,19 @@ module Postqueue
     end
 
     def run!
-      if !run
-        run do |queue|
-          while true do
-            queue.logger.debug "#{queue}: Processing until empty"
-            queue.process_until_empty
-            queue.logger.debug "#{queue}: sleeping"
-            sleep 1
-          end
+      set_default_runner unless @run
+      @run.call(self)
+    end
+
+    def set_default_runner
+      run do |queue|
+        loop do
+          queue.logger.debug "#{queue}: Processing until empty"
+          queue.process_until_empty
+          queue.logger.debug "#{queue}: sleeping"
+          sleep 1
         end
       end
-
-      run.call(self)
     end
   end
 end

data/lib/postqueue/queue/select_and_lock.rb

@@ -1,14 +1,19 @@
 module Postqueue
   class Queue
+    def upcoming(relation = nil, subselect: true) #:nodoc:
+      relation = item_class.all if relation.nil?
+      relation = relation.select(:id, :entity_id, :op) if subselect
+
+      # Ordering by next_run_at and id should not strictly be necessary, but helps
+      # processing entries in the passed in order when enqueued at the same time.
+      relation.where("failed_attempts < ? AND next_run_at < ?", max_attemps, Time.now)
+              .order(:next_run_at, :id)
+    end
+
     # Select and lock up to \a limit unlocked items in the queue. Used by
     # select_and_lock_batch.
     def select_and_lock(relation, limit:)
-      # Ordering by next_run_at and id should not strictly be necessary, but helps
-      # processing entries in the passed in order when enqueued at the same time.
-      relation = relation
-                 .select(:id, :entity_id, :op)
-                 .where("failed_attempts < ? AND next_run_at < ?", max_attemps, Time.now)
-                 .order(:next_run_at, :id)
+      relation = upcoming(relation)
 
       # FOR UPDATE SKIP LOCKED selects and locks entries, but skips those that
       # are already locked - preventing this transaction from being locked.

data/lib/postqueue/queue.rb

@@ -50,7 +50,7 @@ module Postqueue
       return enqueued_items unless enqueued_items > 0
 
       process_until_empty(op: op) unless async_processing?
-      return enqueued_items
+      enqueued_items
     end
   end
 end

data/lib/postqueue/version.rb

@@ -1,3 +1,3 @@
 module Postqueue
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/spec/postqueue/concurrency_spec.rb

@@ -21,7 +21,8 @@ describe "concurrency tests" do
       log = File.open(LOG_FILE, "a")
       queue = Postqueue.new
       queue.on "*" do |_op, entity_ids|
-        sleep(0.0001); log.write "#{entity_ids.first}\n"
+        sleep(0.0001)
+        log.write "#{entity_ids.first}\n"
       end
       queue.process_until_empty
       log.close
@@ -33,9 +34,7 @@ describe "concurrency tests" do
   def run_scenario(cnt, n_threads)
     FileUtils.rm_rf LOG_FILE
 
-    queue = Postqueue.new do |queue|
-      # queue.default_batch_size = 10
-    end
+    queue = Postqueue.new
 
     benchmark "enqueuing #{cnt} ops" do
       queue.enqueue op: "myop", entity_id: (1..cnt)
@@ -67,9 +66,7 @@ describe "concurrency tests" do
   it "enqueues many entries" do
     cnt = 1000
 
-    queue = Postqueue.new do |queue|
-      # queue.default_batch_size = 10
-    end
+    queue = Postqueue.new
     benchmark "enqueuing #{cnt} ops" do
       queue.enqueue op: "myop", entity_id: (1..cnt)
     end

data/spec/postqueue/process_errors_spec.rb

@@ -15,7 +15,9 @@ describe "error handling" do
 
   context "when handler raises an exception" do
     before do
-      queue.on "mytype" do raise E end
+      queue.on "mytype" do
+        raise E
+      end
       queue.enqueue op: "mytype", entity_id: 12
     end
 

data/spec/postqueue/process_spec.rb

@@ -42,7 +42,6 @@ describe "processing" do
 
     it "calls the registered handler and returns the processed entries" do
       queue.enqueue op: "otherop", entity_id: 112
-      called = false
       queue.process_one(op: "otherop")
 
       op, ids = processed_events.first

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: postqueue
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - radiospiel
@@ -163,6 +163,7 @@ files:
 - lib/postqueue.rb
 - lib/postqueue/cli.rb
 - lib/postqueue/cli/options_parser.rb
+- lib/postqueue/cli/stats.rb
 - lib/postqueue/default_queue.rb
 - lib/postqueue/item.rb
 - lib/postqueue/item/enqueue.rb