postburner 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71a903596aa63a03c214e5367c603b3941b8ee4d4f002383208534c31f2c4d7a
-  data.tar.gz: a86e68e87419b3d40317b31beb76bf2ff3f0cf36915ae45e8a4f4cdd52422bf5
+  metadata.gz: 6faa3078340ee2bc8d90e931ee1db203114e32ff69b288ec69340e8b86a987cd
+  data.tar.gz: 0a2ddf78a320e9b01c26a0e3fbd3158b78d37a89bca04a5fdd8c64b70eada1a7
 SHA512:
-  metadata.gz: 76590506bc73dc5f875551021183080e0693ce242278401519a0c0c77c36b4ea0861b96e738a0f5491aa6c6029217b5c6d52b5248fd6e963e65eedfe494df721
-  data.tar.gz: 4b0cae0a80716e081a519d14b83a56dd545578ac3a45a9e59af7ca6429f610e93f2d180f3de0b57e512134190ab1e3c2f73c5ae2d0afedf6fb348468cdbff6f7
+  metadata.gz: 5253309eb7aff502623040d49e923faff7c91df6aac407ec7a4e306a8317b1363db43a4b0408eb20fbdad6ee017ad68489480ac50d7e2ac68a612a1337488457
+  data.tar.gz: 49d0253fe3ecde9cfd4de8b48a2ce95997245bea4310f7c751ccca3685fea8a42b29310a8058f222031ee8aa91f6ec61f990ec5c330157e39853c95830236cfa

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## v0.4.0 - 2021-10-12
+
+### Added
+- Change Job logging to persisted in perform.
+- Update mailer job to use #with and support serializations.
+
+## v0.3.3 - 2021-06-15
+
+### Fixed
+- re-raise exception after logging during perform.
+- log but swallow post perform data update exceptions.
+- only allow remove! if job still present.
+
+### Added
+- Job#log! and Job#log_exception! that save the data immediately.
+
+## v0.3.2 - 2021-05-07
+
+### Fixed
+- add processed_at guard w/ logs.
+
+## v0.3.1 - 2021-05-07
+
+### Fixed
+- calling warning on Rails.logger.
+
 ## v0.3.0 - 2021-05-07
 
 ### Added

data/README.md

@@ -1,29 +1,15 @@
 # Postburner
-An ActiveRecord layer on top of Backburner for inspecting and auditing the
-queue, especially for delayed jobs. It isn't meant to be fast, but safe.
+An ActiveRecord layer on top of [Backburner](https://github.com/nesquena/backburner) for inspecting and auditing the
+queue, especially for delayed jobs. It isn't meant to be outperform other queues, but be safe (and inspectable).
 
-It is meant to be complementary to Backburner. Use Backburner as the default
+It is meant to be complementary to [Backburner](https://github.com/nesquena/backburner). Use Backburner as the default
 ActiveJob processor for mailers, active storage, and the like. Use a
-`Postburner::Job` for things that you want to track.
+`Postburner::Job` for things that you want to track. See [Comparison to Backburner](#comparison-to-backburner) for more.
 
-Comes with a mountable interface that can be password protected with whatever
-authentication you use in your project.
-
-Also meant to be a replacement/upgrade for [Que](https://github.com/que-rb/que).
-If you need something faster, but backed with ACID compliance, check out Que.
-There are some additional features in Postburner such as retained jobs
-after processing, stats, per job logging, etc.
-
-Compared to plain [Backburner](https://github.com/nesquena/backburner),
-Postburner adds:
-- Database Jobs for inspection, linking, auditing, removal (and deletion)
-- Direct access to associated [Beanstalk](https://beanstalkd.github.io/) (via [beaneater](https://github.com/beanstalkd/beaneater))
-- Job Statistics (lag, attempts, logs, tracked errors)
-- Convenience methods to clear tubes, stats, and connections for Beanstalk.
-
-Otherwise, Postburner tries to be a super simple layer on `Backburner::Queue`
-and `ActiveRecord`. Every tool with either of those are avilabel in
-`Postburner::Job`s.
+Postburner meant to be a replacement/upgrade for [Que](https://github.com/que-rb/que).
+If you need something faster, check out Que - we love it! Postburner is built for a slightly different purpose: to be
+simple, safe, and inspectable. All of which Que has (or can be added), but are included by default with some architecture
+differences. See [Comparison to Que](#comparison-to-que) for more.
 
 ## Usage
 
@@ -33,51 +19,240 @@ class RunDonation < Postburner::Job
   queue_priority 0 # 0 is highest priority
   queue_max_job_retries 0 # don't retry
 
-  def process(args)
+  def perform(args)
     # do long tasks here
     # also have access to self.args
   end
 end
 
-RunDonation.create!(args: {donation_id: 123}).queue!
+# RunDonation#create! is the `ActiveRecord` method, so it returns a model,
+# you can manipulate, add columns to the table to, reference with foreign
+# keys, etc.
+job = RunDonation.create!(args: {donation_id: 123})
+# NOTE Make sure use use an after_commit or after_save_commit, etc to avoid
+#      any race conditions of Postburner trying to process before a required
+#      database mutation is commited.
+job.queue!
 => {:status=>"INSERTED", :id=>"1139"}
+
+# queue for later with `:at`
 RunDonation.create!(args: {donation_id: 123}).queue! at: Time.zone.now + 2.days
 => {:status=>"INSERTED", :id=>"1140"}
 
-# `delay` takes priority over `at`
+# queue for later with `:delay`
+#
+# `:delay` takes priority over `:at`takes priority over `:at` because the
+# beanstalkd protocol uses uses `delay`
 RunDonation.create!(args: {donation_id: 123}).queue! delay: 1.hour
 => {:status=>"INSERTED", :id=>"1141"}
 ```
 
+### [Beaneater](https://github.com/beanstalkd/beaneater) and [beanstalkd](https://raw.githubusercontent.com/beanstalkd/beanstalkd/master/doc/protocol.txt) attributes and methods
+```ruby
+# get the beanstalkd job id
+job.bkid
+=> 1104
+
+# get the beaneater job, call any beaneater methods on this object
+job.beanstalk_job
+
+# get the beanstald stats
+job.beanstalk_job.stats
+
+# kick the beanstalk job
+job.kick!
+
+# delete beankstalkd job, retain the job model
+job.delete!
+
+# delete beankstalkd job, retain the job model, but set `removed_at` on model.
+job.remove!
+
+# or simply remove the model, which will clean up the beanstalkd job in a before_destroy hook
+job.destroy # OR job.destroy!
+
+# get a cached Backburner connection and inspect it (or even use it directly)
+c = Postburner.connection
+c.beanstalk.tubes.to_a
+c.beanstalk.tubes.to_a.map{|t| c.tubes[t.name].peek(:buried)}
+c.beanstalk.tubes['ss.development.caching'].stats
+c.beanstalk.tubes['ss.development.caching'].peek(:buried).kick
+c.beanstalk.tubes['ss.development.caching'].kick(3)
+c.close
+
+# automatically close
+Postburner.connected do |connection|
+  # do stuff with connection
+end
+```
+
+Read about the [beanstalkd protocol](https://raw.githubusercontent.com/beanstalkd/beanstalkd/master/doc/protocol.txt).
+
+### Basic model fields
+```ruby
+# ActiveRecord primary key
+job.id
+
+# int id of the beankstalkd job
+job.bkid
+
+# string uuid
+job.sid
+
+# ActiveRecord STI job subtype
+job.type
+
+# jsonb arguments for use in job
+job.args
+
+# time job should run - not intended to be changed
+# TODO run_at should be readonly after create
+job.run_at
+```
+
+### Job statistics
+```ruby
+# when job was inserted into beankstalkd
+job.queued_at
+
+# last time attempted
+job.attempting_at
+
+# last time processing started
+job.processing_at
+
+# when completed
+job.processed_at
+
+# when removed, may be nil
+job.removed_at
+
+# lag in ms from run_at/queued_at to attempting_at
+job.lag
+
+# duration of processing in ms
+job.duration
+
+# number of attempts
+job.attempt_count
+
+# number of errors (length of errata)
+job.error_count
+
+# number of log entries (length of logs)
+job.log_count
+
+# array of attempting_at times
+job.attempts
+
+# array of errors
+job.errata
+
+# array of log messages
+job.logs
+```
+
+### Job logging and exceptions
+
+Optionally, you can:
+1. Add log messages to the job during processing to `logs`
+1. Add log your own exceptions to `errata`
+
+```ruby
+class RunDonation < Postburner::Job
+  queue 'critical'
+  queue_priority 0 # 0 is highest priority
+  queue_max_job_retries 0 # don't retry
+
+  def perform(args)
+    # log at task, defaults to `:info`, but `:debug`, `:warning`, `:error`
+    log "Log bad condition...", level: :error
+
+    begin
+      # danger
+    rescue Exception => e
+      log_exception e
+    end
+  end
+end
+```
+
+### Optionally, mount the engine
+
+```ruby
+mount Postburner::Engine => "/postburner"
+
+# mount only for development inspection
+mount Postburner::Engine => "/postburner" if Rails.env.development?
+```
+
+[Open the controller](https://guides.rubyonrails.org/engines.html#overriding-models-and-controllers)
+to add your own authentication or changes - or just create your own routes, controllers, and views.
+
+[Override the views](https://guides.rubyonrails.org/engines.html#overriding-views) to make them
+prettier - or follow the suggestion above and use your own.
+
 ## Installation
 
+First [install beanstalkd](https://beanstalkd.github.io/download.html). On Debian-based systems, that goes like this:
+```bash
+sudo apt-get install beanstalkd
+```
+
+Then add to your Gemfile.
 ```ruby
-# in Gemfile
 gem 'postburner'
 ```
 
+Install...
 ```bash
 $ bundle
+```
 
+After bundling, install the migration.
+```
 # install migration, possible to edit and add attributes or indexes as needed.
 $ bundle exec rails g postburner:install
 ```
 
-Add a `config/initializers/backburner.rb` with option found [here](https://github.com/nesquena/backburner#configuration).
+Inspect `XXX_create_postburner_jobs.rb`. (The template is [here](-/blob/master/lib/generators/postburner/install/templates/migrations/create_postburner_jobs.rb.erb)).The required attributes are in there, but add more if you would like to use them,
+or do it in a separate migration. By default, several indexes are added 
 
-Set `Backburner` for `ActiveJob`
+Because `Postburner` is built on `Backburner`, add a `config/initializers/backburner.rb` with option found [here](https://github.com/nesquena/backburner#configuration).
+```ruby
+Backburner.configure do |config|
+  config.beanstalk_url       = "beanstalk://127.0.0.1"
+  config.tube_namespace      = "some.app.production"
+  config.namespace_separator = "."
+  config.on_error            = lambda { |e| puts e }
+  config.max_job_retries     = 3 # default 0 retries
+  config.retry_delay         = 2 # default 5 seconds
+  config.retry_delay_proc    = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+  config.default_priority    = 65536
+  config.respond_timeout     = 120
+  config.default_worker      = Backburner::Workers::Simple
+  config.logger              = Logger.new(STDOUT)
+  config.primary_queue       = "backburner-jobs"
+  config.priority_labels     = { :custom => 50, :useless => 1000 }
+  config.reserve_timeout     = nil
+  config.job_serializer_proc = lambda { |body| JSON.dump(body) }
+  config.job_parser_proc     = lambda { |body| JSON.parse(body) }
+end
+```
+
+Finally, set `Backburner` for `ActiveJob`
 ```
 # config/application.rb
 config.active_job.queue_adapter = :backburner
 ```
 
 Postburner may later provide an adapter, but we recommend using `Postburner::Job` classes
-directyly.
+directly.
 
 Add jobs to `app/jobs/`. There currently is no generator.
+
 ```ruby
 # app/jobs/run_donation.rb
-
 class RunDonation < Postburner::Job
   queue 'critical'
   queue_priority 0 # 0 is highest priority
@@ -88,6 +263,34 @@ class RunDonation < Postburner::Job
     # also have access to self.args
   end
 end
+```
+
+### Comparison to Backburner
+
+Compared to plain [Backburner](https://github.com/nesquena/backburner),
+Postburner adds:
+- Database Jobs for inspection, linking, auditing, removal (and deletion)
+- Direct access to associated [Beanstalk](https://beanstalkd.github.io/) (via [beaneater](https://github.com/beanstalkd/beaneater))
+- Job Statistics (lag, attempts, logs, tracked errors)
+- Convenience methods to clear tubes, stats, and connections for Beanstalk.
+
+Otherwise, Postburner tries to be a super simple layer on `Backburner::Queue`
+and `ActiveRecord`. Every tool with either of those are available in
+`Postburner::Job`s.
+
+Comes with a mountable interface that can be password protected with whatever
+authentication you use in your project.
+
+### Comparison to Que
+
+Postburner meant to be a replacement/upgrade for [Que](https://github.com/que-rb/que).
+However, if you need something faster and backed with ACID compliance, check out Que.  
+
+Postburner has some additional features such as retained jobs after processing,
+stats, per job logging, etc.
+
+Postburner is meant to be simpler than `Que`. Que is incredible, but jobs should
+be simple so the logic and history can be transparent.
 
 ## Contributing
 Submit a pull request. Follow conventions of the project. Be nice.
@@ -102,7 +305,6 @@ Submit a pull request. Follow conventions of the project. Be nice.
 - Job generator
   - Build file in app/jobs
   - Inherit from Postburner::Job
-- Job generator
 - Add before/after/around hooks
 - Add destroy, and remove actions on show page
 - Clear tubes.
@@ -112,5 +314,23 @@ Submit a pull request. Follow conventions of the project. Be nice.
 - Add logging with Job.args in backburner logs
 - MAYBE - ActiveJob adapter
 
+
+### Running in Development
+
+```
+cd test/dummy
+bundle exec backburner
+```
+
+### Releasing
+
+1. `gem bump -v minor -t`
+  Where <minor> can be: major|minor|patch|pre|release or a version number
+2. Edit `CHANGELOG.md` with details from `git log --oneline`
+3. `git commit --amend`
+4. `gem release -k nac -p`
+  Where <nac> is an authorized key with push capabilities.
+
+
 ## License
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/app/models/postburner/job.rb

@@ -37,7 +37,7 @@ module Postburner
         job = self.find(id)
         job.perform!(job.args)
       rescue ActiveRecord::RecordNotFound => e
-        Rails.logger.warning <<-MSG
+        Rails.logger.warn <<-MSG
 [Postburner::Job] [#{id}] Not Found.
         MSG
       end
@@ -55,45 +55,52 @@ module Postburner
       )
 
       begin
+        if self.processed_at.present?
+          self.log! "Already Processed", level: :error
+          self.delete!
+          return
+        end
+
         if self.removed_at.present?
-          self.log "Removed", level: :error
-          update_column :logs, self.logs
+          self.log! "Removed", level: :error
           return
         end
 
         if self.run_at && self.run_at > Time.zone.now
           response = self.insert! delay: self.run_at - Time.zone.now
-          self.log "PREMATURE; RE-INSERTED: #{response}"
-          update_column :logs, self.logs
+          self.log! "PREMATURE; RE-INSERTED: #{response}"
           return
         end
 
-        self.log('START')
+        self.log!('START')
 
         self.perform(args)
 
-        self.log('DONE')
+        self.log!('DONE')
+
+        begin
+          now = Time.zone.now
+          _duration =  (now - self.processing_at) * 1000 rescue nil
+          self.update_columns(
+            processed_at: now,
+            duration:     _duration,
+            errata:       self.errata,
+            error_count:  self.errata.length,
+            logs:         self.logs,
+            log_count:    self.logs.length,
+          )
+        rescue Exception => e
+          self.log_exception!(e)
+          self.log! '[Postburner] Could not set data after processing.'
+          # TODO README doesn't retry if Postburner is to blame
+        end
 
-      rescue Exception => e
-        self.log_exception(e)
+      rescue Exception => exception
+        self.log_exception!(exception)
+        self.log! '[Postburner] Exception raised during perform prevented completion.'
+        raise exception
       end
 
-      begin
-        now = Time.zone.now
-        _duration =  (now - self.processing_at) * 1000 rescue nil
-        self.update_columns(
-          processed_at: now,
-          duration:     _duration,
-          errata:       self.errata,
-          error_count:  self.errata.length,
-          logs:         self.logs,
-          log_count:    self.logs.length,
-        )
-      rescue Exception => e
-        raise e if Rails.env.development? || Rails.env.production?
-        self.log_exception(e)
-        update_column :errata, self.errata
-      end
     end
 
     def delete!
@@ -115,9 +122,12 @@ module Postburner
     end
 
     def remove!
-      return false if self.attempts.any?
-      self.delete!
-      self.update_column(removed_at: Time.zone.now)
+      if self.beanstalk_job
+        self.delete!
+        self.update_column(:removed_at, Time.zone.now)
+      else
+        false
+      end
     end
 
     def beanstalk_job
@@ -145,6 +155,11 @@ module Postburner
       ]
     end
 
+    def log_exception!(exception)
+      self.log_exception(exception)
+      self.update_column :errata, self.errata
+    end
+
     def log(message, options={})
       options[:level] ||= :info
       options[:level] = :error unless LOG_LEVELS.member?(options[:level])
@@ -156,6 +171,11 @@ module Postburner
       ]
     end
 
+    def log!(message, options={})
+      self.log(message, options)
+      self.update_column :logs, self.logs
+    end
+
     private
 
     def insert!(options={})

data/app/models/postburner/mailer.rb

@@ -1,13 +1,13 @@
 module Postburner
   # Send a mailer, tracked.
   #
-  # Postburner::Mailer.deliver(UserMailer, :welcome, user_id: 1)
-  # Postburner::Mailer.deliver(UserMailer, :welcome, user_id: 1).queue! at: Time.zone.now + 1.day
-  # Postburner::Mailer.deliver(UserMailer, :welcome, user_id: 1).queue! delay: 5.minutes
+  # Postburner::Mailer.deliver(UserMailer, :welcome).with(user_id: 1)
+  # Postburner::Mailer.deliver(UserMailer, :welcome).with(user_id: 1).queue! at: Time.zone.now + 1.day
+  # Postburner::Mailer.deliver(UserMailer, :welcome).with(user_id: 1).queue! delay: 5.minutes
   class Mailer < Job
     #queue 'mailers'
 
-    def self.deliver!(mailer, action)
+    def self.deliver(mailer, action)
       job = self.create!(
         args: {
           mailer: mailer.to_s,
@@ -17,9 +17,32 @@ module Postburner
       job
     end
 
-    def process(args)
-      cls = args['mailer'].constantize
-      cls.send(args['action']).deliver_now
+    # Similar to ActionMailer #with - set the parameters
+    #
+    def with(params={})
+      self.args.merge!(
+        params: ActiveJob::Arguments.serialize(params)
+      )
+      self.save!
+      self
+    end
+
+    # Build the mail but don't send.
+    #
+    def assemble(args=nil)
+      _args = args || self.args
+
+      cls = _args['mailer'].constantize
+
+      mail = cls.
+        with(ActiveJob::Arguments.deserialize(_args['params']).to_h).
+        send(_args['action'])
+
+      mail
+    end
+
+    def perform(args)
+      self.assemble(args).deliver_now
     end
   end
 end

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postburner
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Matt Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-07 00:00:00.000000000 Z
+date: 2021-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -128,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Postgres backed beanstalkd queue via backburner