postburner 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +12 -0
- data/MIT-LICENSE +20 -0
- data/README.md +116 -0
- data/Rakefile +18 -0
- data/app/assets/config/postburner_manifest.js +1 -0
- data/app/assets/stylesheets/postburner/application.css +15 -0
- data/app/controllers/postburner/application_controller.rb +4 -0
- data/app/controllers/postburner/jobs_controller.rb +13 -0
- data/app/controllers/postburner/static_controller.rb +9 -0
- data/app/helpers/postburner/application_helper.rb +4 -0
- data/app/jobs/postburner/application_job.rb +4 -0
- data/app/mailers/postburner/application_mailer.rb +6 -0
- data/app/models/postburner/application_record.rb +5 -0
- data/app/models/postburner/job.rb +196 -0
- data/app/views/layouts/postburner/application.html.haml +10 -0
- data/app/views/postburner/jobs/index.html.haml +34 -0
- data/app/views/postburner/jobs/show.html.haml +13 -0
- data/config/environment.rb +0 -0
- data/config/routes.rb +6 -0
- data/lib/generators/postburner/install/USAGE +8 -0
- data/lib/generators/postburner/install/install_generator.rb +47 -0
- data/lib/generators/postburner/install/templates/migrations/create_postburner_jobs.rb.erb +46 -0
- data/lib/postburner.rb +31 -0
- data/lib/postburner/engine.rb +17 -0
- data/lib/postburner/version.rb +3 -0
- data/lib/tasks/postburner_tasks.rake +4 -0
- metadata +134 -0
    
        checksums.yaml
    ADDED
    
    | @@ -0,0 +1,7 @@ | |
| 1 | 
            +
            ---
         | 
| 2 | 
            +
            SHA256:
         | 
| 3 | 
            +
              metadata.gz: b86343f81143907f3df739be458355fd5ac083cc83c6785a8eb3c45b4f2d2e3c
         | 
| 4 | 
            +
              data.tar.gz: 2db76cad93eeb4e5fa551ca7b7a5a9734c81b1d50c64acdbba7e5f71d75cc579
         | 
| 5 | 
            +
            SHA512:
         | 
| 6 | 
            +
              metadata.gz: 167fd6de8a0e985d09baa86ef49e63c9a7d5e642732e10cc6812dcc2bc562a6e1fc95a3910691a18450dd84ff6440ec69a884cd53f8fb236e12fb032e6fe5a97
         | 
| 7 | 
            +
              data.tar.gz: 2fb90a8b7a4ad78a8b8c5163bbf9585552fc0e72522d19d0c7779b1706e084899f347932bc5e70741ad64dad83a8bc5a31dae7e07ef8028e20adc4503f778c95
         | 
    
        data/CHANGELOG.md
    ADDED
    
    
    
        data/MIT-LICENSE
    ADDED
    
    | @@ -0,0 +1,20 @@ | |
| 1 | 
            +
            Copyright 2021 Matt Smith
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            Permission is hereby granted, free of charge, to any person obtaining
         | 
| 4 | 
            +
            a copy of this software and associated documentation files (the
         | 
| 5 | 
            +
            "Software"), to deal in the Software without restriction, including
         | 
| 6 | 
            +
            without limitation the rights to use, copy, modify, merge, publish,
         | 
| 7 | 
            +
            distribute, sublicense, and/or sell copies of the Software, and to
         | 
| 8 | 
            +
            permit persons to whom the Software is furnished to do so, subject to
         | 
| 9 | 
            +
            the following conditions:
         | 
| 10 | 
            +
             | 
| 11 | 
            +
            The above copyright notice and this permission notice shall be
         | 
| 12 | 
            +
            included in all copies or substantial portions of the Software.
         | 
| 13 | 
            +
             | 
| 14 | 
            +
            THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
         | 
| 15 | 
            +
            EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
         | 
| 16 | 
            +
            MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
         | 
| 17 | 
            +
            NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
         | 
| 18 | 
            +
            LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
         | 
| 19 | 
            +
            OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
         | 
| 20 | 
            +
            WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
         | 
    
        data/README.md
    ADDED
    
    | @@ -0,0 +1,116 @@ | |
| 1 | 
            +
            # Postburner
         | 
| 2 | 
            +
            An ActiveRecord layer on top of Backburner for inspecting and auditing the
         | 
| 3 | 
            +
            queue, especially for delayed jobs. It isn't meant to be fast, but safe.
         | 
| 4 | 
            +
             | 
| 5 | 
            +
            It is meant to be complementary to Backburner. Use Backburner as the default
         | 
| 6 | 
            +
            ActiveJob processor for mailers, active storage, and the like. Use a
         | 
| 7 | 
            +
            `Postburner::Job` for things that you want to track.
         | 
| 8 | 
            +
             | 
| 9 | 
            +
            Comes with a mountable interface that can be password protected with whatever
         | 
| 10 | 
            +
            authentication you use in your project.
         | 
| 11 | 
            +
             | 
| 12 | 
            +
            Also meant to be a replacement/upgrade for [Que](https://github.com/que-rb/que).
         | 
| 13 | 
            +
            If you need something faster, but backed with ACID compliance, check out Que.
         | 
| 14 | 
            +
            There are some additional features in Postburner such as retained jobs
         | 
| 15 | 
            +
            after processing, stats, per job logging, etc.
         | 
| 16 | 
            +
             | 
| 17 | 
            +
            Compared to plain [Backburner](https://github.com/nesquena/backburner),
         | 
| 18 | 
            +
            Postburner adds:
         | 
| 19 | 
            +
            - Database Jobs for inspection, linking, auditing, removal (and deletion)
         | 
| 20 | 
            +
            - Direct access to associated [Beanstalk](https://beanstalkd.github.io/) (via [beaneater](https://github.com/beanstalkd/beaneater))
         | 
| 21 | 
            +
            - Job Statistics (lag, attempts, logs, tracked errors)
         | 
| 22 | 
            +
            - Convenience methods to clear tubes, stats, and connections for Beanstalk.
         | 
| 23 | 
            +
             | 
| 24 | 
            +
            Otherwise, Postburner tries to be a super simple layer on `Backburner::Queue`
         | 
| 25 | 
            +
            and `ActiveRecord`. Every tool with either of those are avilabel in
         | 
| 26 | 
            +
            `Postburner::Job`s.
         | 
| 27 | 
            +
             | 
| 28 | 
            +
            ## Usage
         | 
| 29 | 
            +
             | 
| 30 | 
            +
            ```ruby
         | 
| 31 | 
            +
            class RunDonation < Postburner::Job
         | 
| 32 | 
            +
              queue 'critical'
         | 
| 33 | 
            +
              queue_priority 0 # 0 is highest priority
         | 
| 34 | 
            +
              queue_max_job_retries 0 # don't retry
         | 
| 35 | 
            +
             | 
| 36 | 
            +
              def process(args)
         | 
| 37 | 
            +
                # do long tasks here
         | 
| 38 | 
            +
                # also have access to self.args
         | 
| 39 | 
            +
              end
         | 
| 40 | 
            +
            end
         | 
| 41 | 
            +
             | 
| 42 | 
            +
            RunDonation.create!(args: {donation_id: 123}).queue!
         | 
| 43 | 
            +
            => {:status=>"INSERTED", :id=>"1139"}
         | 
| 44 | 
            +
            RunDonation.create!(args: {donation_id: 123}).queue! at: Time.zone.now + 2.days
         | 
| 45 | 
            +
            => {:status=>"INSERTED", :id=>"1140"}
         | 
| 46 | 
            +
             | 
| 47 | 
            +
            # `delay` takes priority over `at`
         | 
| 48 | 
            +
            RunDonation.create!(args: {donation_id: 123}).queue! delay: 1.hour
         | 
| 49 | 
            +
            => {:status=>"INSERTED", :id=>"1141"}
         | 
| 50 | 
            +
            ```
         | 
| 51 | 
            +
             | 
| 52 | 
            +
            ## Installation
         | 
| 53 | 
            +
             | 
| 54 | 
            +
            ```ruby
         | 
| 55 | 
            +
            # in Gemfile
         | 
| 56 | 
            +
            gem 'postburner'
         | 
| 57 | 
            +
            ```
         | 
| 58 | 
            +
             | 
| 59 | 
            +
            ```bash
         | 
| 60 | 
            +
            $ bundle
         | 
| 61 | 
            +
             | 
| 62 | 
            +
            # install migration, possible to edit and add attributes or indexes as needed.
         | 
| 63 | 
            +
            $ bundle exec rails g postburner:install
         | 
| 64 | 
            +
            ```
         | 
| 65 | 
            +
             | 
| 66 | 
            +
            Add a `config/initializers/backburner.rb` with option found [here](https://github.com/nesquena/backburner#configuration).
         | 
| 67 | 
            +
             | 
| 68 | 
            +
            Set `Backburner` for `ActiveJob`
         | 
| 69 | 
            +
            ```
         | 
| 70 | 
            +
            # config/application.rb
         | 
| 71 | 
            +
            config.active_job.queue_adapter = :backburner
         | 
| 72 | 
            +
            ```
         | 
| 73 | 
            +
             | 
| 74 | 
            +
            Postburner may later provide an adapter, but we recommend using `Postburner::Job` classes
         | 
| 75 | 
            +
            directyly.
         | 
| 76 | 
            +
             | 
| 77 | 
            +
            Add jobs to `app/jobs/`. There currently is no generator.
         | 
| 78 | 
            +
            ```ruby
         | 
| 79 | 
            +
            # app/jobs/run_donation.rb
         | 
| 80 | 
            +
             | 
| 81 | 
            +
            class RunDonation < Postburner::Job
         | 
| 82 | 
            +
              queue 'critical'
         | 
| 83 | 
            +
              queue_priority 0 # 0 is highest priority
         | 
| 84 | 
            +
              queue_max_job_retries 0 # don't retry
         | 
| 85 | 
            +
             | 
| 86 | 
            +
              def process(args)
         | 
| 87 | 
            +
                # do long tasks here
         | 
| 88 | 
            +
                # also have access to self.args
         | 
| 89 | 
            +
              end
         | 
| 90 | 
            +
            end
         | 
| 91 | 
            +
             | 
| 92 | 
            +
            ## Contributing
         | 
| 93 | 
            +
            Submit a pull request. Follow conventions of the project. Be nice.
         | 
| 94 | 
            +
             | 
| 95 | 
            +
            ### V1 TODO
         | 
| 96 | 
            +
            - Basic tests
         | 
| 97 | 
            +
            - Add Authentication modules for engine mount.
         | 
| 98 | 
            +
             | 
| 99 | 
            +
            ### V1+ TODO
         | 
| 100 | 
            +
            - Install generator
         | 
| 101 | 
            +
              - Sub to backburner
         | 
| 102 | 
            +
            - Job generator
         | 
| 103 | 
            +
              - Build file in app/jobs
         | 
| 104 | 
            +
              - Inherit from Postburner::Job
         | 
| 105 | 
            +
            - Job generator
         | 
| 106 | 
            +
            - Add before/after/around hooks
         | 
| 107 | 
            +
            - Add destroy, and remove actions on show page
         | 
| 108 | 
            +
            - Clear tubes.
         | 
| 109 | 
            +
            - Document how/when to use activerecord hooks
         | 
| 110 | 
            +
            - Document how/when to use backburner hooks
         | 
| 111 | 
            +
            - Document how/when to use postburner hooks
         | 
| 112 | 
            +
            - Add logging with Job.args in backburner logs
         | 
| 113 | 
            +
            - MAYBE - ActiveJob adapter
         | 
| 114 | 
            +
             | 
| 115 | 
            +
            ## License
         | 
| 116 | 
            +
            The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
         | 
    
        data/Rakefile
    ADDED
    
    | @@ -0,0 +1,18 @@ | |
| 1 | 
            +
            require "bundler/setup"
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
         | 
| 4 | 
            +
            load "rails/tasks/engine.rake"
         | 
| 5 | 
            +
             | 
| 6 | 
            +
            load "rails/tasks/statistics.rake"
         | 
| 7 | 
            +
             | 
| 8 | 
            +
            require "bundler/gem_tasks"
         | 
| 9 | 
            +
             | 
| 10 | 
            +
            require "rake/testtask"
         | 
| 11 | 
            +
             | 
| 12 | 
            +
            Rake::TestTask.new(:test) do |t|
         | 
| 13 | 
            +
              t.libs << 'test'
         | 
| 14 | 
            +
              t.pattern = 'test/**/*_test.rb'
         | 
| 15 | 
            +
              t.verbose = false
         | 
| 16 | 
            +
            end
         | 
| 17 | 
            +
             | 
| 18 | 
            +
            task default: :test
         | 
| @@ -0,0 +1 @@ | |
| 1 | 
            +
            //= link_directory ../stylesheets/postburner .css
         | 
| @@ -0,0 +1,15 @@ | |
| 1 | 
            +
            /*
         | 
| 2 | 
            +
             * This is a manifest file that'll be compiled into application.css, which will include all the files
         | 
| 3 | 
            +
             * listed below.
         | 
| 4 | 
            +
             *
         | 
| 5 | 
            +
             * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
         | 
| 6 | 
            +
             * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
         | 
| 7 | 
            +
             *
         | 
| 8 | 
            +
             * You're free to add application-wide styles to this file and they'll appear at the bottom of the
         | 
| 9 | 
            +
             * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
         | 
| 10 | 
            +
             * files in this directory. Styles in this file should be added after the last require_* statement.
         | 
| 11 | 
            +
             * It is generally better to create a new file per style scope.
         | 
| 12 | 
            +
             *
         | 
| 13 | 
            +
             *= require_tree .
         | 
| 14 | 
            +
             *= require_self
         | 
| 15 | 
            +
             */
         | 
| @@ -0,0 +1,13 @@ | |
| 1 | 
            +
            require_dependency "postburner/application_controller"
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            module Postburner
         | 
| 4 | 
            +
              class JobsController < ApplicationController
         | 
| 5 | 
            +
                def index
         | 
| 6 | 
            +
                  @jobs = Job.order(queued_at: :desc, created_at: :desc)
         | 
| 7 | 
            +
                end
         | 
| 8 | 
            +
             | 
| 9 | 
            +
                def show
         | 
| 10 | 
            +
                  @job = Job.find(params[:id])
         | 
| 11 | 
            +
                end
         | 
| 12 | 
            +
              end
         | 
| 13 | 
            +
            end
         | 
| @@ -0,0 +1,196 @@ | |
| 1 | 
            +
            module Postburner
         | 
| 2 | 
            +
              class Job < ApplicationRecord
         | 
| 3 | 
            +
                include Backburner::Queue
         | 
| 4 | 
            +
             | 
| 5 | 
            +
                LOG_LEVELS = [
         | 
| 6 | 
            +
                  :debug,
         | 
| 7 | 
            +
                  :info,
         | 
| 8 | 
            +
                  :warning,
         | 
| 9 | 
            +
                  :error
         | 
| 10 | 
            +
                ].freeze
         | 
| 11 | 
            +
             | 
| 12 | 
            +
                before_validation :ensure_sid!
         | 
| 13 | 
            +
                before_destroy :delete!
         | 
| 14 | 
            +
             | 
| 15 | 
            +
                validates :sid, presence: {strict: true}
         | 
| 16 | 
            +
             | 
| 17 | 
            +
                def queue!(options={})
         | 
| 18 | 
            +
                  return if self.queued_at.present? && self.bkid.present?
         | 
| 19 | 
            +
             | 
| 20 | 
            +
                  case
         | 
| 21 | 
            +
                  when options[:at].present?
         | 
| 22 | 
            +
                    # this is rudimentary, add error handling
         | 
| 23 | 
            +
                    options[:delay] ||= options[:at].to_i - Time.zone.now.to_i
         | 
| 24 | 
            +
                    update_column :run_at, options[:at]
         | 
| 25 | 
            +
                    options.delete(:at)
         | 
| 26 | 
            +
                  when options[:delay].present?
         | 
| 27 | 
            +
                    update_column :run_at, Time.zone.now + options[:delay].seconds
         | 
| 28 | 
            +
                  end
         | 
| 29 | 
            +
             | 
| 30 | 
            +
                  insert!(options)
         | 
| 31 | 
            +
                end
         | 
| 32 | 
            +
             | 
| 33 | 
            +
                # tube: backburner.worker.queue.backburner-jobs
         | 
| 34 | 
            +
                #
         | 
| 35 | 
            +
                def self.perform(id, _={})
         | 
| 36 | 
            +
                  begin
         | 
| 37 | 
            +
                    job = self.find(id)
         | 
| 38 | 
            +
                    job.perform!(job.args)
         | 
| 39 | 
            +
                  rescue ActiveRecord::RecordNotFound => e
         | 
| 40 | 
            +
                    Rails.logger.warning <<-MSG
         | 
| 41 | 
            +
            [Postburner::Job] [#{id}] Not Found.
         | 
| 42 | 
            +
                    MSG
         | 
| 43 | 
            +
                  end
         | 
| 44 | 
            +
                end
         | 
| 45 | 
            +
             | 
| 46 | 
            +
                def perform!(args={})
         | 
| 47 | 
            +
                  self.attempting
         | 
| 48 | 
            +
             | 
| 49 | 
            +
                  self.update_columns(
         | 
| 50 | 
            +
                    attempting_at:  self.attempting_at,
         | 
| 51 | 
            +
                    attempts:       self.attempts,
         | 
| 52 | 
            +
                    attempt_count:  self.attempts.length,
         | 
| 53 | 
            +
                    lag:            self.lag,
         | 
| 54 | 
            +
                    processing_at:  Time.zone.now,
         | 
| 55 | 
            +
                  )
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                  begin
         | 
| 58 | 
            +
                    if self.removed_at.present?
         | 
| 59 | 
            +
                      self.log "Removed", level: :error
         | 
| 60 | 
            +
                      update_column :logs, self.logs
         | 
| 61 | 
            +
                      return
         | 
| 62 | 
            +
                    end
         | 
| 63 | 
            +
             | 
| 64 | 
            +
                    if self.run_at && self.run_at > Time.zone.now
         | 
| 65 | 
            +
                      response = self.insert! delay: self.run_at - Time.zone.now
         | 
| 66 | 
            +
                      self.log "PREMATURE; RE-INSERTED: #{response}"
         | 
| 67 | 
            +
                      update_column :logs, self.logs
         | 
| 68 | 
            +
                      return
         | 
| 69 | 
            +
                    end
         | 
| 70 | 
            +
             | 
| 71 | 
            +
                    self.log('START')
         | 
| 72 | 
            +
             | 
| 73 | 
            +
                    self.perform(args)
         | 
| 74 | 
            +
             | 
| 75 | 
            +
                    self.log('DONE')
         | 
| 76 | 
            +
             | 
| 77 | 
            +
                  rescue Exception => e
         | 
| 78 | 
            +
                    self.log_exception(e)
         | 
| 79 | 
            +
                  end
         | 
| 80 | 
            +
             | 
| 81 | 
            +
                  begin
         | 
| 82 | 
            +
                    now = Time.zone.now
         | 
| 83 | 
            +
                    _duration =  (now - self.processing_at) * 1000 rescue nil
         | 
| 84 | 
            +
                    self.update_columns(
         | 
| 85 | 
            +
                      processed_at: now,
         | 
| 86 | 
            +
                      duration:     _duration,
         | 
| 87 | 
            +
                      errata:       self.errata,
         | 
| 88 | 
            +
                      error_count:  self.errata.length,
         | 
| 89 | 
            +
                      logs:         self.logs,
         | 
| 90 | 
            +
                      log_count:    self.logs.length,
         | 
| 91 | 
            +
                    )
         | 
| 92 | 
            +
                  rescue Exception => e
         | 
| 93 | 
            +
                    raise e if Rails.env.development? || Rails.env.production?
         | 
| 94 | 
            +
                    self.log_exception(e)
         | 
| 95 | 
            +
                    update_column :errata, self.errata
         | 
| 96 | 
            +
                  end
         | 
| 97 | 
            +
                end
         | 
| 98 | 
            +
             | 
| 99 | 
            +
                def delete!
         | 
| 100 | 
            +
                  return unless self.beanstalk_job
         | 
| 101 | 
            +
                  begin
         | 
| 102 | 
            +
                    self.beanstalk_job.delete
         | 
| 103 | 
            +
                  rescue Beaneater::NotConnected => e
         | 
| 104 | 
            +
                    self.beanstalk_job!.delete
         | 
| 105 | 
            +
                  end
         | 
| 106 | 
            +
                end
         | 
| 107 | 
            +
             | 
| 108 | 
            +
                def kick!
         | 
| 109 | 
            +
                  return unless self.beanstalk_job
         | 
| 110 | 
            +
                  begin
         | 
| 111 | 
            +
                    self.beanstalk_job.kick
         | 
| 112 | 
            +
                  rescue Beaneater::NotConnected => e
         | 
| 113 | 
            +
                    self.beanstalk_job!.kick
         | 
| 114 | 
            +
                  end
         | 
| 115 | 
            +
                end
         | 
| 116 | 
            +
             | 
| 117 | 
            +
                def remove!
         | 
| 118 | 
            +
                  return false if self.attempts.any?
         | 
| 119 | 
            +
                  self.delete!
         | 
| 120 | 
            +
                  self.update_column(removed_at: Time.zone.now)
         | 
| 121 | 
            +
                end
         | 
| 122 | 
            +
             | 
| 123 | 
            +
                def beanstalk_job
         | 
| 124 | 
            +
                  return unless self.bkid.present?
         | 
| 125 | 
            +
                  return @_beanstalk_job if @_beanstalk_job
         | 
| 126 | 
            +
             | 
| 127 | 
            +
                  @_beanstalk_job = Postburner.connection.beanstalk.jobs.find(self.bkid)
         | 
| 128 | 
            +
             | 
| 129 | 
            +
                  @_beanstalk_job
         | 
| 130 | 
            +
                end
         | 
| 131 | 
            +
             | 
| 132 | 
            +
                def beanstalk_job!
         | 
| 133 | 
            +
                  @_beanstalk_job = nil
         | 
| 134 | 
            +
                  self.beanstalk_job
         | 
| 135 | 
            +
                end
         | 
| 136 | 
            +
             | 
| 137 | 
            +
                def log_exception(exception)
         | 
| 138 | 
            +
                  self.errata << [
         | 
| 139 | 
            +
                    Time.zone.now,
         | 
| 140 | 
            +
                    {
         | 
| 141 | 
            +
                      class:      exception.class,
         | 
| 142 | 
            +
                      message:    exception.message,
         | 
| 143 | 
            +
                      backtrace:  exception.backtrace,
         | 
| 144 | 
            +
                    }
         | 
| 145 | 
            +
                  ]
         | 
| 146 | 
            +
                end
         | 
| 147 | 
            +
             | 
| 148 | 
            +
                def log(message, options={})
         | 
| 149 | 
            +
                  options[:level] ||= :info
         | 
| 150 | 
            +
                  options[:level] = :error unless LOG_LEVELS.member?(options[:level])
         | 
| 151 | 
            +
             | 
| 152 | 
            +
                  self.logs << [
         | 
| 153 | 
            +
                    Time.zone.now,
         | 
| 154 | 
            +
                    options[:level],
         | 
| 155 | 
            +
                    message,
         | 
| 156 | 
            +
                  ]
         | 
| 157 | 
            +
                end
         | 
| 158 | 
            +
             | 
| 159 | 
            +
                private
         | 
| 160 | 
            +
             | 
| 161 | 
            +
                def insert!(options={})
         | 
| 162 | 
            +
                  response = Backburner::Worker.enqueue(Postburner::Job, self.id, options)
         | 
| 163 | 
            +
             | 
| 164 | 
            +
                  self.log("QUEUED: #{response}")
         | 
| 165 | 
            +
             | 
| 166 | 
            +
                  update_columns(
         | 
| 167 | 
            +
                    queued_at:  Time.zone.now,
         | 
| 168 | 
            +
                    bkid:       response[:id],
         | 
| 169 | 
            +
                    logs:       self.logs,
         | 
| 170 | 
            +
                  )
         | 
| 171 | 
            +
             | 
| 172 | 
            +
                  response
         | 
| 173 | 
            +
                end
         | 
| 174 | 
            +
             | 
| 175 | 
            +
                def attempting
         | 
| 176 | 
            +
                  now = Time.zone.now
         | 
| 177 | 
            +
                  self.attempts << now
         | 
| 178 | 
            +
                  self.attempting_at ||= now
         | 
| 179 | 
            +
                  self.lag ||= (self.attempting_at - self.intended_at) * 1000
         | 
| 180 | 
            +
                  now
         | 
| 181 | 
            +
                end
         | 
| 182 | 
            +
             | 
| 183 | 
            +
                def intended_at
         | 
| 184 | 
            +
                  self.run_at ? self.run_at : self.queued_at
         | 
| 185 | 
            +
                end
         | 
| 186 | 
            +
             | 
| 187 | 
            +
                def ensure_sid!
         | 
| 188 | 
            +
                  self.sid ||= SecureRandom.uuid
         | 
| 189 | 
            +
                end
         | 
| 190 | 
            +
             | 
| 191 | 
            +
                def message
         | 
| 192 | 
            +
                  "Job: #{self.id}"
         | 
| 193 | 
            +
                end
         | 
| 194 | 
            +
             | 
| 195 | 
            +
              end
         | 
| 196 | 
            +
            end
         | 
| @@ -0,0 +1,34 @@ | |
| 1 | 
            +
            %h1 Jobs
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            %table
         | 
| 4 | 
            +
              %thead
         | 
| 5 | 
            +
                %tr
         | 
| 6 | 
            +
                  %th ID
         | 
| 7 | 
            +
                  %th Beanstalk
         | 
| 8 | 
            +
                  %th Type
         | 
| 9 | 
            +
                  %th Arguments
         | 
| 10 | 
            +
                  %th Run
         | 
| 11 | 
            +
                  %th Attempted
         | 
| 12 | 
            +
                  %th Completed
         | 
| 13 | 
            +
                  %th Attempts
         | 
| 14 | 
            +
                  %th Errors
         | 
| 15 | 
            +
                  %th Logs
         | 
| 16 | 
            +
                  %th Lag
         | 
| 17 | 
            +
                  %th Duration
         | 
| 18 | 
            +
              %tbody
         | 
| 19 | 
            +
                - @jobs.each do |job|
         | 
| 20 | 
            +
                  %tr
         | 
| 21 | 
            +
                    %td
         | 
| 22 | 
            +
                      = link_to postburner.job_path(job) do
         | 
| 23 | 
            +
                        %span{title: job.sid}= job.id
         | 
| 24 | 
            +
                    %td= job.bkid
         | 
| 25 | 
            +
                    %td= job.type
         | 
| 26 | 
            +
                    %td= job.args.inspect
         | 
| 27 | 
            +
                    %td= job.run_at
         | 
| 28 | 
            +
                    %td= job.attempting_at
         | 
| 29 | 
            +
                    %td= job.processed_at
         | 
| 30 | 
            +
                    %td= job.attempt_count
         | 
| 31 | 
            +
                    %td= job.error_count
         | 
| 32 | 
            +
                    %td= job.log_count
         | 
| 33 | 
            +
                    %td= job.lag
         | 
| 34 | 
            +
                    %td= job.duration
         | 
| @@ -0,0 +1,13 @@ | |
| 1 | 
            +
            %h1 Job #{@job.id} / #{@job.bkid} / #{@job.sid}
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            - if @job.run_at && @job.run_at > Time.zone.now
         | 
| 4 | 
            +
              Running in
         | 
| 5 | 
            +
              = distance_of_time_in_words Time.zone.now, @job.run_at
         | 
| 6 | 
            +
              (#{@job.run_at.to_i - Time.zone.now.to_i} seconds)
         | 
| 7 | 
            +
             | 
| 8 | 
            +
            %h3 Stats
         | 
| 9 | 
            +
            - if @job.beanstalk_job
         | 
| 10 | 
            +
              %pre= JSON.pretty_generate @job.beanstalk_job.stats.as_json
         | 
| 11 | 
            +
             | 
| 12 | 
            +
            %h3 Job
         | 
| 13 | 
            +
            %pre= JSON.pretty_generate @job.as_json
         | 
| 
            File without changes
         | 
    
        data/config/routes.rb
    ADDED
    
    
| @@ -0,0 +1,47 @@ | |
| 1 | 
            +
            class Postburner::InstallGenerator < Rails::Generators::Base
         | 
| 2 | 
            +
              source_root File.expand_path('templates', __dir__)
         | 
| 3 | 
            +
              include Rails::Generators::Migration
         | 
| 4 | 
            +
             | 
| 5 | 
            +
              def install_migrations
         | 
| 6 | 
            +
                install_migration! 'create_postburner_jobs'
         | 
| 7 | 
            +
              end
         | 
| 8 | 
            +
             | 
| 9 | 
            +
              def self.next_migration_number(dirname)
         | 
| 10 | 
            +
                timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
         | 
| 11 | 
            +
                stem = timestamp[0..-3]
         | 
| 12 | 
            +
                regexp = Regexp.new("^(#{stem})(\\d\\d)_")
         | 
| 13 | 
            +
                timestamps = Dir[File.join(dirname, '*.rb')].map { |name|
         | 
| 14 | 
            +
                  match = regexp.match(File.basename(name, File.extname(name)))
         | 
| 15 | 
            +
                  match ? match[2].to_i : nil
         | 
| 16 | 
            +
                }.reject(&:nil?)
         | 
| 17 | 
            +
                _max = timestamps.max || timestamp[-2..-1].to_i
         | 
| 18 | 
            +
                _next = _max + 1
         | 
| 19 | 
            +
                raise "MISSING NEXT" if _next.blank?
         | 
| 20 | 
            +
                migration_number = "#{stem}#{_next}"
         | 
| 21 | 
            +
                if migration_number.length != 14
         | 
| 22 | 
            +
                  raise "INCORRECT LENGTH stem=#{stem} _next=#{_next} _max=#{_max}"
         | 
| 23 | 
            +
                end
         | 
| 24 | 
            +
                migration_number
         | 
| 25 | 
            +
              end
         | 
| 26 | 
            +
             | 
| 27 | 
            +
              private
         | 
| 28 | 
            +
             | 
| 29 | 
            +
              def install_migration!(filename)
         | 
| 30 | 
            +
                migrate_path = File.join(Rails.root, "db/migrate")
         | 
| 31 | 
            +
             | 
| 32 | 
            +
                if self.class.migration_exists?(migrate_path, "#{filename}")
         | 
| 33 | 
            +
                  say_status "skipped", "#{filename}.rb migration already exists"
         | 
| 34 | 
            +
                else
         | 
| 35 | 
            +
                  migration_template(
         | 
| 36 | 
            +
                    "migrations/#{filename}.rb.erb",
         | 
| 37 | 
            +
                    File.join(migrate_path, "#{filename}.rb"),
         | 
| 38 | 
            +
                    migration_version: migration_version,
         | 
| 39 | 
            +
                  )
         | 
| 40 | 
            +
                end
         | 
| 41 | 
            +
              end
         | 
| 42 | 
            +
             | 
| 43 | 
            +
              def migration_version
         | 
| 44 | 
            +
                "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
         | 
| 45 | 
            +
              end
         | 
| 46 | 
            +
             | 
| 47 | 
            +
            end
         | 
| @@ -0,0 +1,46 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
            #
         | 
| 3 | 
            +
            class CreatePostburnerJobs < ActiveRecord::Migration<%= migration_version %>
         | 
| 4 | 
            +
              def change
         | 
| 5 | 
            +
                create_table :postburner_jobs do |t|
         | 
| 6 | 
            +
                  t.bigint :bkid
         | 
| 7 | 
            +
                  t.string :sid, null: false
         | 
| 8 | 
            +
                  t.string :type
         | 
| 9 | 
            +
                  t.jsonb :args, default: {}
         | 
| 10 | 
            +
                  t.datetime :run_at
         | 
| 11 | 
            +
                  t.datetime :queued_at
         | 
| 12 | 
            +
                  t.datetime :attempting_at
         | 
| 13 | 
            +
                  t.datetime :processing_at
         | 
| 14 | 
            +
                  t.datetime :processed_at
         | 
| 15 | 
            +
                  t.datetime :removed_at
         | 
| 16 | 
            +
                  t.integer :lag
         | 
| 17 | 
            +
                  t.integer :duration
         | 
| 18 | 
            +
                  t.integer :attempt_count
         | 
| 19 | 
            +
                  t.integer :error_count
         | 
| 20 | 
            +
                  t.integer :log_count
         | 
| 21 | 
            +
                  t.datetime :attempts, array: true, default: []
         | 
| 22 | 
            +
                  t.jsonb :errata, default: []
         | 
| 23 | 
            +
                  t.jsonb :logs, default: []
         | 
| 24 | 
            +
             | 
| 25 | 
            +
                  t.timestamps
         | 
| 26 | 
            +
             | 
| 27 | 
            +
                  t.index [ :bkid ]
         | 
| 28 | 
            +
                  t.index [ :sid ], unique: true
         | 
| 29 | 
            +
                  t.index [ :type ]
         | 
| 30 | 
            +
                  t.index [ :args ], using: :gin
         | 
| 31 | 
            +
                  t.index [ :run_at ]
         | 
| 32 | 
            +
                  t.index [ :queued_at ]
         | 
| 33 | 
            +
                  t.index [ :removed_at ]
         | 
| 34 | 
            +
                  t.index [ :lag ]
         | 
| 35 | 
            +
                  t.index [ :duration ]
         | 
| 36 | 
            +
                  t.index [ :attempt_count ]
         | 
| 37 | 
            +
                  t.index [ :error_count ]
         | 
| 38 | 
            +
                  t.index [ :log_count ]
         | 
| 39 | 
            +
             | 
| 40 | 
            +
                  # Add these or more depending on what you want to search for.
         | 
| 41 | 
            +
                  #t.index [ :attempts ], using: :gin
         | 
| 42 | 
            +
                  #t.index [ :errata ], using: :gin
         | 
| 43 | 
            +
                  #t.index [ :logs ], using: :gin
         | 
| 44 | 
            +
                end
         | 
| 45 | 
            +
              end
         | 
| 46 | 
            +
            end
         | 
    
        data/lib/postburner.rb
    ADDED
    
    | @@ -0,0 +1,31 @@ | |
| 1 | 
            +
            require "postburner/version"
         | 
| 2 | 
            +
            require "postburner/engine"
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            module Postburner
         | 
| 5 | 
            +
             | 
| 6 | 
            +
              def self.connection
         | 
| 7 | 
            +
                @_connection ||= Backburner::Connection.new(
         | 
| 8 | 
            +
                  Backburner.configuration.beanstalk_url
         | 
| 9 | 
            +
                )
         | 
| 10 | 
            +
                @_connection.reconnect! unless @_connection.connected?
         | 
| 11 | 
            +
                @_connection
         | 
| 12 | 
            +
              end
         | 
| 13 | 
            +
             | 
| 14 | 
            +
              def self.connected
         | 
| 15 | 
            +
                if block_given?
         | 
| 16 | 
            +
                  begin
         | 
| 17 | 
            +
                    yield connection
         | 
| 18 | 
            +
                  ensure
         | 
| 19 | 
            +
                    connection.close if connection
         | 
| 20 | 
            +
                  end
         | 
| 21 | 
            +
                else
         | 
| 22 | 
            +
                  connection
         | 
| 23 | 
            +
                end
         | 
| 24 | 
            +
              end
         | 
| 25 | 
            +
             | 
| 26 | 
            +
              def self.remove_all!(confirm)
         | 
| 27 | 
            +
                return unless confirm == "CONFIRM"
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                # TODO
         | 
| 30 | 
            +
              end
         | 
| 31 | 
            +
            end
         | 
| @@ -0,0 +1,17 @@ | |
| 1 | 
            +
            require 'backburner'
         | 
| 2 | 
            +
            require 'haml-rails'
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            module Postburner
         | 
| 5 | 
            +
              class Engine < ::Rails::Engine
         | 
| 6 | 
            +
                isolate_namespace Postburner
         | 
| 7 | 
            +
             | 
| 8 | 
            +
                config.generators do |g|
         | 
| 9 | 
            +
                  g.template_engine :haml
         | 
| 10 | 
            +
                  g.helper false
         | 
| 11 | 
            +
                  g.fixture false
         | 
| 12 | 
            +
                  g.assets false
         | 
| 13 | 
            +
                  g.skip_routes true
         | 
| 14 | 
            +
                end
         | 
| 15 | 
            +
             | 
| 16 | 
            +
              end
         | 
| 17 | 
            +
            end
         | 
    
        metadata
    ADDED
    
    | @@ -0,0 +1,134 @@ | |
| 1 | 
            +
            --- !ruby/object:Gem::Specification
         | 
| 2 | 
            +
            name: postburner
         | 
| 3 | 
            +
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            +
              version: 0.2.2
         | 
| 5 | 
            +
            platform: ruby
         | 
| 6 | 
            +
            authors:
         | 
| 7 | 
            +
            - Matt Smith
         | 
| 8 | 
            +
            autorequire: 
         | 
| 9 | 
            +
            bindir: bin
         | 
| 10 | 
            +
            cert_chain: []
         | 
| 11 | 
            +
            date: 2021-05-05 00:00:00.000000000 Z
         | 
| 12 | 
            +
            dependencies:
         | 
| 13 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 14 | 
            +
              name: rails
         | 
| 15 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 16 | 
            +
                requirements:
         | 
| 17 | 
            +
                - - "~>"
         | 
| 18 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 19 | 
            +
                    version: 6.1.3
         | 
| 20 | 
            +
                - - ">="
         | 
| 21 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 22 | 
            +
                    version: 6.1.3.1
         | 
| 23 | 
            +
              type: :runtime
         | 
| 24 | 
            +
              prerelease: false
         | 
| 25 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 26 | 
            +
                requirements:
         | 
| 27 | 
            +
                - - "~>"
         | 
| 28 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 29 | 
            +
                    version: 6.1.3
         | 
| 30 | 
            +
                - - ">="
         | 
| 31 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 32 | 
            +
                    version: 6.1.3.1
         | 
| 33 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 34 | 
            +
              name: backburner
         | 
| 35 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 36 | 
            +
                requirements:
         | 
| 37 | 
            +
                - - ">="
         | 
| 38 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 39 | 
            +
                    version: 1.5.0
         | 
| 40 | 
            +
              type: :runtime
         | 
| 41 | 
            +
              prerelease: false
         | 
| 42 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 43 | 
            +
                requirements:
         | 
| 44 | 
            +
                - - ">="
         | 
| 45 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 46 | 
            +
                    version: 1.5.0
         | 
| 47 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 48 | 
            +
              name: pg
         | 
| 49 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 50 | 
            +
                requirements:
         | 
| 51 | 
            +
                - - ">="
         | 
| 52 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 53 | 
            +
                    version: 1.2.3
         | 
| 54 | 
            +
              type: :runtime
         | 
| 55 | 
            +
              prerelease: false
         | 
| 56 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 57 | 
            +
                requirements:
         | 
| 58 | 
            +
                - - ">="
         | 
| 59 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 60 | 
            +
                    version: 1.2.3
         | 
| 61 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 62 | 
            +
              name: haml-rails
         | 
| 63 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 64 | 
            +
                requirements:
         | 
| 65 | 
            +
                - - ">="
         | 
| 66 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 67 | 
            +
                    version: 2.0.1
         | 
| 68 | 
            +
              type: :runtime
         | 
| 69 | 
            +
              prerelease: false
         | 
| 70 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 71 | 
            +
                requirements:
         | 
| 72 | 
            +
                - - ">="
         | 
| 73 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 74 | 
            +
                    version: 2.0.1
         | 
| 75 | 
            +
            description: Queue background jobs, inspect them, and audit them afterwards.
         | 
| 76 | 
            +
            email:
         | 
| 77 | 
            +
            - matt@nearapogee.com
         | 
| 78 | 
            +
            executables: []
         | 
| 79 | 
            +
            extensions: []
         | 
| 80 | 
            +
            extra_rdoc_files: []
         | 
| 81 | 
            +
            files:
         | 
| 82 | 
            +
            - CHANGELOG.md
         | 
| 83 | 
            +
            - MIT-LICENSE
         | 
| 84 | 
            +
            - README.md
         | 
| 85 | 
            +
            - Rakefile
         | 
| 86 | 
            +
            - app/assets/config/postburner_manifest.js
         | 
| 87 | 
            +
            - app/assets/stylesheets/postburner/application.css
         | 
| 88 | 
            +
            - app/controllers/postburner/application_controller.rb
         | 
| 89 | 
            +
            - app/controllers/postburner/jobs_controller.rb
         | 
| 90 | 
            +
            - app/controllers/postburner/static_controller.rb
         | 
| 91 | 
            +
            - app/helpers/postburner/application_helper.rb
         | 
| 92 | 
            +
            - app/jobs/postburner/application_job.rb
         | 
| 93 | 
            +
            - app/mailers/postburner/application_mailer.rb
         | 
| 94 | 
            +
            - app/models/postburner/application_record.rb
         | 
| 95 | 
            +
            - app/models/postburner/job.rb
         | 
| 96 | 
            +
            - app/views/layouts/postburner/application.html.haml
         | 
| 97 | 
            +
            - app/views/postburner/jobs/index.html.haml
         | 
| 98 | 
            +
            - app/views/postburner/jobs/show.html.haml
         | 
| 99 | 
            +
            - config/environment.rb
         | 
| 100 | 
            +
            - config/routes.rb
         | 
| 101 | 
            +
            - lib/generators/postburner/install/USAGE
         | 
| 102 | 
            +
            - lib/generators/postburner/install/install_generator.rb
         | 
| 103 | 
            +
            - lib/generators/postburner/install/templates/migrations/create_postburner_jobs.rb.erb
         | 
| 104 | 
            +
            - lib/postburner.rb
         | 
| 105 | 
            +
            - lib/postburner/engine.rb
         | 
| 106 | 
            +
            - lib/postburner/version.rb
         | 
| 107 | 
            +
            - lib/tasks/postburner_tasks.rake
         | 
| 108 | 
            +
            homepage: https://gitlab.nearapogee.com/opensource/postburner
         | 
| 109 | 
            +
            licenses:
         | 
| 110 | 
            +
            - MIT
         | 
| 111 | 
            +
            metadata:
         | 
| 112 | 
            +
              homepage_uri: https://gitlab.nearapogee.com/opensource/postburner
         | 
| 113 | 
            +
              source_code_uri: https://gitlab.nearapogee.com/opensource/postburner
         | 
| 114 | 
            +
              changelog_uri: https://gitlab.nearapogee.com/opensource/postburner/-/blob/master/CHANGELOG.md
         | 
| 115 | 
            +
            post_install_message: 
         | 
| 116 | 
            +
            rdoc_options: []
         | 
| 117 | 
            +
            require_paths:
         | 
| 118 | 
            +
            - lib
         | 
| 119 | 
            +
            required_ruby_version: !ruby/object:Gem::Requirement
         | 
| 120 | 
            +
              requirements:
         | 
| 121 | 
            +
              - - ">="
         | 
| 122 | 
            +
                - !ruby/object:Gem::Version
         | 
| 123 | 
            +
                  version: '0'
         | 
| 124 | 
            +
            required_rubygems_version: !ruby/object:Gem::Requirement
         | 
| 125 | 
            +
              requirements:
         | 
| 126 | 
            +
              - - ">="
         | 
| 127 | 
            +
                - !ruby/object:Gem::Version
         | 
| 128 | 
            +
                  version: '0'
         | 
| 129 | 
            +
            requirements: []
         | 
| 130 | 
            +
            rubygems_version: 3.1.4
         | 
| 131 | 
            +
            signing_key: 
         | 
| 132 | 
            +
            specification_version: 4
         | 
| 133 | 
            +
            summary: Postgres backed beanstalkd queue via backburner
         | 
| 134 | 
            +
            test_files: []
         |