postburner 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74d577811ded71db9296138442288dd53a0eb46718b8980564a84032c3a7b564
-  data.tar.gz: b97fb1adb55232e5c2293d88c852ab496f3519ad740b2f0d5df8d68f7d663e97
+  metadata.gz: b3f5758fde9443c7c094a85825c343aa003fd43a969323d549fe631d3c114fd8
+  data.tar.gz: aa30ba2e1ef5669bc9dd86243553dcc3f89c5dfa0a084c154afd313919df82db
 SHA512:
-  metadata.gz: 995fe1db553b3e2b698427eeb061a6e5463011ed29315dbd5280f8a1d4c46f6f1c4a15607b80c58d2ba393d896e15f38baa4581a616a758f7f57ffda8d57ac70
-  data.tar.gz: d123415e6a7a4329a1897922ab6619a032fc5c61b501fb612dcfa9eb370ec45da97b3ff0787a2baf25101fd7fadee572c620f1fb32dbb202ee76c4f4e67a224d
+  metadata.gz: 9f87f62fdca9594f365fb1641a3c0d83adc78271a0ac22e4b711e684cf02ba7fa5e9ad51c0ec2eac999ee62ac95d998bd7dccf3ef33e64856b77b40fa39329a5
+  data.tar.gz: 9e518552995537a13204210f5fdffb76a7d709a64c0064dec45a0b02150c871774e4ed0f7c82da90ac2fd3d76c313577f70420a95461d426b21fb20abaf428e5

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## v0.8.0 - 2022-11-13
+
+### Added
+- add Postburner::Tube for Beaneater job introspection.
+- add note about tube names and hyphens/dashes to README.
+
+### Fixed
+- fix Backburner::Queue methods in subclasses for queue selection, etc.
+
+## v0.7.1 - 2022-10-30
+
+### Required Changes
+- rename `enqueue` callbacks to `insert`.
+
+### Changed
+- convert to ActiveModel callbacks.
+
 ## v0.7.1 - 2022-10-30
 
 ### Changed

data/README.md

@@ -203,6 +203,24 @@ to add your own authentication or changes - or just create your own routes, cont
 [Override the views](https://guides.rubyonrails.org/engines.html#overriding-views) to make them
 prettier - or follow the suggestion above and use your own.
 
+## Known Issues
+
+1. Beaneater and/or Beanstalkd seems to transform a `tube` `name` with hyphens to
+   underscores upon instrospection of `stats`. See example. Reccommend using
+   names without hyphens / dashes.
+
+```ruby
+(main):001:0> Postburner.connection.tubes["backburner.worker.queue.cat-dog"].put("{}", delay: 1.week.to_i)
+=> {:status=>"INSERTED", :id=>"9"}
+irb(main):002:0> Postburner.connection.jobs.find(9)
+=> #<Beaneater::Job id=9 body="{}">
+irb(main):003:0> Postburner.connection.jobs.find(9).stats
+=> #<Beaneater::StatStruct id=9, tube="backburner.worker.queue.cat_dog", state="delayed", pri=65536, age=23, delay=604800, ttr=120, time_left=604776, file=0, reserves=0, timeouts=0, releases=0, buries=0, kicks=0>
+#
+# NOTE 'cat-dog' on line 1 and 'cat_dog' on line 3.
+#
+```
+
 ## Installation
 
 First [install beanstalkd](https://beanstalkd.github.io/download.html). On Debian-based systems, that goes like this:

data/app/models/postburner/job.rb

@@ -16,7 +16,11 @@ module Postburner
   #
   class Job < ApplicationRecord
     include Backburner::Queue
-    include Callbacks
+
+    define_model_callbacks :insert
+    define_model_callbacks :attempt
+    define_model_callbacks :processing
+    define_model_callbacks :processed, only: :after
 
     LOG_LEVELS = [
       :debug,
@@ -51,9 +55,7 @@ module Postburner
 
       @_insert_options = options
 
-      run_callbacks :enqueue do
-        self.save!
-      end
+      self.save!
     end
 
     def requeue!(options={})
@@ -83,7 +85,7 @@ module Postburner
     end
 
     def perform!(args={})
-      run_callbacks :attempt do
+      _run_attempt_callbacks do
         self.attempting
 
         self.update_columns(
@@ -124,7 +126,7 @@ module Postburner
 
           self.log!("START (bkid #{self.bkid})")
 
-          run_callbacks :processing do
+          _run_processing_callbacks do
             begin
               self.perform(args)
             rescue Exception => exception
@@ -140,7 +142,7 @@ module Postburner
             now = Time.zone.now
             _duration =  (now - self.processing_at) * 1000 rescue nil
 
-            run_callbacks :processed do
+            _run_processed_callbacks do
               persist_metadata!(
                 processed_at: now,
                 duration:     _duration,
@@ -269,16 +271,18 @@ module Postburner
     def insert!(options={})
       response = nil
 
-      Job.transaction do
-        response = Backburner::Worker.enqueue(
-          Postburner::Job,
-          self.id,
-          options
-        )
-
-        persist_metadata!(
-          bkid: response[:id],
-        )
+      _run_insert_callbacks do
+        Job.transaction do
+          response = Backburner::Worker.enqueue(
+            self.class,
+            self.id,
+            options
+          )
+
+          persist_metadata!(
+            bkid: response[:id],
+          )
+        end
       end
 
       self.log("QUEUED: #{response}")

data/lib/postburner/tube.rb

@@ -0,0 +1,53 @@
+module Postburner
+  class Tube
+    def initialize(tube)
+      @tube = tube
+    end
+
+    # Get all tubes as Postburner::Tube instances.
+    #
+    def self.all
+      Postburner.connection.tubes.all.map { |tube| self.new(tube) }
+    end
+
+    # Get all peeked ids across all known tubes.
+    #
+    def self.peek_ids
+      self.all.map(&:peek_ids).flatten.sort
+    end
+
+    # Get all peeked ids.
+    #
+    def peek_ids
+      [ :buried, :ready, :delayed ].map { |type| @tube.peek(type) }.
+        reject(&:nil?).map(&:id).map(&:to_i)
+    end
+
+    # Get paginated array of jobs.
+    #
+    # Attempts to do this efficiently as possible, by peeking at known
+    # ids and exiting when count has been fulfilled.
+    #
+    # Just pass the last known id to after for the next batch.
+    #
+    def jobs(count=20, limit: 1000, after: nil)
+      stats = @tube.stats
+      jobs = Array.new
+
+      min_known = (
+        peek_ids.any? ? self.peek_ids.min : self.class.peek_ids.min
+      ).to_i
+      min = after ? after + 1 : min_known
+      max = min + limit
+
+      for i in min..max
+        job = @tube.client.jobs.find(i)
+        jobs << job if job && stats[:name] == job.stats[:tube]
+        break if jobs.length >= count
+      end
+
+      return jobs
+    end
+
+  end
+end

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '0.7.1'
+  VERSION = '0.8.0'
 end

data/lib/postburner.rb

@@ -1,5 +1,6 @@
 require "postburner/version"
 require "postburner/engine"
+require "postburner/tube"
 
 module Postburner
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postburner
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Matt Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-30 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -79,7 +79,6 @@ files:
 - Rakefile
 - app/assets/config/postburner_manifest.js
 - app/assets/stylesheets/postburner/application.css
-- app/concerns/postburner/callbacks.rb
 - app/controllers/postburner/application_controller.rb
 - app/controllers/postburner/jobs_controller.rb
 - app/controllers/postburner/static_controller.rb
@@ -99,6 +98,7 @@ files:
 - lib/generators/postburner/install/templates/migrations/create_postburner_jobs.rb.erb
 - lib/postburner.rb
 - lib/postburner/engine.rb
+- lib/postburner/tube.rb
 - lib/postburner/version.rb
 - lib/tasks/postburner_tasks.rake
 homepage: https://gitlab.nearapogee.com/opensource/postburner

data/app/concerns/postburner/callbacks.rb

@@ -1,286 +0,0 @@
-# frozen_string_literal: true
-
-require "active_support/callbacks"
-require "active_support/core_ext/object/with_options"
-require "active_support/core_ext/module/attribute_accessors"
-
-module Postburner
-  # = Postburner Job Callbacks
-  #
-  # Postburner Job provides hooks during the life cycle of a job. Callbacks allow you
-  # to trigger logic during this cycle. In called order, available callbacks are:
-  #
-  # * <tt>before_enqueue</tt>
-  # * <tt>after_enqueue</tt>
-  # * <tt>before_attempt</tt>
-  # * <tt>before_processing</tt>
-  # * <tt>after_processing</tt>
-  # * <tt>after_processed</tt>
-  # * <tt>after_attempt</tt>
-  #
-  # Around callbacks are also defined:
-  # * <tt>around_enqueue</tt>
-  # * <tt>around_attempt</tt>
-  # * <tt>around_processing</tt>
-  #
-  # NOTE: Calling the same callback multiple times will overwrite previous callback definitions.
-  #
-  module Callbacks
-    extend  ActiveSupport::Concern
-    include ActiveSupport::Callbacks
-
-    included do
-      cattr_accessor :skip_after_callbacks_if_terminated,
-        instance_accessor: false,
-        default: false
-
-      with_options(skip_after_callbacks_if_terminated: skip_after_callbacks_if_terminated) do
-        define_callbacks :enqueue
-        define_callbacks :attempt
-        define_callbacks :processing
-        define_callbacks :processed
-      end
-    end
-    module ClassMethods
-      def inherited(klass)
-        klass.get_callbacks(:enqueue).
-          config[:skip_after_callbacks_if_terminated] =
-            skip_after_callbacks_if_terminated
-        klass.get_callbacks(:processing).
-          config[:skip_after_callbacks_if_terminated] =
-            skip_after_callbacks_if_terminated
-        klass.get_callbacks(:attempt).
-          config[:skip_after_callbacks_if_terminated] =
-            skip_after_callbacks_if_terminated
-        klass.get_callbacks(:processed).
-          config[:skip_after_callbacks_if_terminated] =
-            skip_after_callbacks_if_terminated
-        super
-      end
-
-      # Defines a callback that will get called right before the
-      # job starts and any metadata is persisted.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     before_attempt do |job|
-      #       $statsd.increment "attempting-video-job.try"
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def before_attempt(*filters, &blk)
-        set_callback(:attempt, :before, *filters, &blk)
-      end
-
-      # Defines a callback that will get called right after the
-      # job finishes and all metadata is persisted.
-      #
-      # This isn't called unless the attempt finishes without error,
-      # though this is subject to change in the future
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     before_attempt do |job|
-      #       $statsd.increment "attempt-complete-video-job.try"
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def after_attempt(*filters, &blk)
-        set_callback(:attempt, :after, *filters, &blk)
-      end
-
-      # Defines a callback that will get called around the attempt
-      # of the job.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     around_enqueue do |job, block|
-      #       $statsd.time "video-job.process" do
-      #         block.call
-      #       end
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def around_attempt(*filters, &blk)
-        set_callback(:attempt, :around, *filters, &blk)
-      end
-
-      # Defines a callback that will get called right before the
-      # job's perform method is executed.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     before_processing do |job|
-      #       UserMailer.notify_video_started_processing(job.arguments.first)
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def before_processing(*filters, &blk)
-        set_callback(:processing, :before, *filters, &blk)
-      end
-
-      # Defines a callback that will get called right after the
-      # job's perform method has finished.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     after_processing do |job|
-      #       UserMailer.notify_video_processed(job.arguments.first)
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def after_processing(*filters, &blk)
-        set_callback(:processing, :after, *filters, &blk)
-      end
-
-      # Defines a callback that will get called around the job's perform method.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     around_processing do |job, block|
-      #       UserMailer.notify_video_started_processing(job.arguments.first)
-      #       block.call
-      #       UserMailer.notify_video_processed(job.arguments.first)
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      # You can access the return value of the job only if the execution wasn't halted.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #     around_processing do |job, block|
-      #       value = block.call
-      #       puts value # => "Hello World!"
-      #     end
-      #
-      #     def perform
-      #       "Hello World!"
-      #     end
-      #   end
-      #
-      def around_processing(*filters, &blk)
-        set_callback(:processing, :around, *filters, &blk)
-      end
-
-      # Defines a callback that will get called right after the
-      # job's processed_at timestamp has been persisted, marking the
-      # job as successfully completed, i.e. no errors raised.
-      #
-      # This is rather similar to after_processing but all of the
-      # job's metadata has been set.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     after_processed do |job|
-      #       UserMailer.notify_video_processed(job.arguments.first)
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def after_processed(*filters, &blk)
-        set_callback(:processed, :after, *filters, &blk)
-      end
-
-      # Defines a callback that will get called right before the
-      # job is enqueued.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     before_enqueue do |job|
-      #       $statsd.increment "enqueue-video-job.try"
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def before_enqueue(*filters, &blk)
-        set_callback(:enqueue, :before, *filters, &blk)
-      end
-
-      # Defines a callback that will get called right after the
-      # job is enqueued.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     after_enqueue do |job|
-      #       $statsd.increment "enqueue-video-job.success"
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def after_enqueue(*filters, &blk)
-        set_callback(:enqueue, :after, *filters, &blk)
-      end
-
-      # Defines a callback that will get called around the enqueuing
-      # of the job.
-      #
-      #   class VideoProcessJob < Postburner::Base
-      #
-      #     around_enqueue do |job, block|
-      #       $statsd.time "video-job.process" do
-      #         block.call
-      #       end
-      #     end
-      #
-      #     def perform(video_id)
-      #       Video.find(video_id).process
-      #     end
-      #   end
-      #
-      def around_enqueue(*filters, &blk)
-        set_callback(:enqueue, :around, *filters, &blk)
-      end
-    end
-
-    private
-
-    # NOTE processed / attempt exclueded here -- intentionally
-    #
-    def halted_callback_hook(_filter, name) # :nodoc:
-      return super unless %i(enqueue processing).include?(name.to_sym)
-      callbacks = public_send("_#{name}_callbacks")
-
-      if !self.class.skip_after_callbacks_if_terminated && callbacks.any? { |c| c.kind == :after }
-        ActiveSupport::Deprecation.warn(<<~EOM)
-            In Rails 7.0, `after_enqueue`/`after_work` callbacks no longer run if `before_enqueue`/`before_work` respectively halts with `throw :abort`.
-            To enable this behavior, uncomment the `config.active_job.skip_after_callbacks_if_terminated` config
-            in the new 6.1 framework defaults initializer.
-        EOM
-      end
-
-      super
-    end
-  end
-end