back_ops 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 543bd1222115f9fa55b46ef603aaebd4100b8cbf4e817d1d4e598d50106e2227
+  data.tar.gz: 8fc4ca44128a2d74ca4ca714ebf6dea6139fc0ee0c250f4103314013b5c7bcd2
+SHA512:
+  metadata.gz: b32d62c84c8703135789dc506243f7c5d27cec7e8987bcecf34ae9a78298ef296f0d04e27676132dca5325ef5ee1b2ac356fa413bd43f6eb77537918f66e7e89
+  data.tar.gz: 88ab67dc53c0604f95edc4f7c8e4d0473d972b7685c42bc558deaebeb714bf7d91921388b703a093ece8caff81d07311e9721a96fb9af49876e9d607f389febd

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2021 Aaron Price
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+# BackOps
+
+Back Ops is intended for background processing of jobs that require multiple tasks to be completed. It executes each task in sequence in a separate Sidekiq worker. This allows for jobs to be retryable if failures occur, but completed tasks are not retried. 
+
+Progress and error states are tracked in the database, so that that you are always aware of what was processed, and if any task fails, where the failure occured in the process, what the error message is, what the stack trace is, so you know what's happening and you can always retry the job from the failed task.
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'back_ops'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install back_ops
+```
+
+## Install migrations
+Copy the migration from the gem to your application, then run migrations.
+
+```bash
+$ rails g back_ops:install
+$ rails db:migrate
+```
+
+## Usage
+
+There are two parts to this process. First, define an operation, which accepts a set of params, and an array of actions (as the names of the classes those actions will be called). Parameters should be simple types (e.g.: Integer, String, Date/Timestamp) instead of full objects because these params will be sent serialized and sent to redis for each defined task.
+
+```ruby
+module Subscriptions
+  module Operations
+    class Fullfillment
+      def self.call(subscription)
+        BackOps::Worker.perform_async({
+          'subscription_id' => subscription.id
+        }, [
+          Subscriptions::Actions::Fulfillment::ChargeCreditCard,
+          Subscriptions::Actions::Fulfillment::SendEmailReceipt
+        ])
+      end
+    end
+  end
+end
+```
+
+Each action receives the operation object which contains the context.
+
+```ruby
+module Subscriptions
+  module Actions
+    module Fulfillment
+      class ChargeCreditCard
+        def self.call(operation)
+          subscription_id = operation.context['subscription_id']
+          subscription = Subscription.find(subscription_id)
+          # ...
+        end
+      end
+    end
+  end
+end
+```
+
+You now also have full transparency into each operation and can view it in the admin section by invoking the following code.
+
+```ruby
+params = { 'subscription_id' => subscription.id }
+operation = BackOps::Operation.includes(:actions).where("name = 'Subscriptions::Operations::Fulfillment' AND context @> ?", params.to_json).first
+```
+
+
+## Contributing
+To contribute to this project. Clone the repo and create a branch for your changes. When complete, create a pull request into the `develop` branch of this project. Ensure that positive and negative test cases cover your changes and all tests pass.
+
+Be detailed in your commit message of what changes you're making and the motivation behind the changes.
+
+## License
+
+Copyright 2021 Aaron Price
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/setup"
+
+require "bundler/gem_tasks"

data/lib/back_ops.rb

@@ -0,0 +1,10 @@
+require 'active_record'
+require 'sidekiq'
+
+require 'back_ops/action'
+require 'back_ops/operation'
+require 'back_ops/version'
+require 'back_ops/worker'
+
+module BackOps
+end

data/lib/back_ops/action.rb

@@ -0,0 +1,26 @@
+module BackOps
+  class Action < ActiveRecord::Base
+    
+    # == Constants ============================================================
+
+    # == Attributes ===========================================================
+
+    # == Extensions ===========================================================
+
+    # == Relationships ========================================================
+
+    belongs_to :operation, class_name: 'BackOps::Operation'
+
+    # == Validations ==========================================================
+
+    # == Scopes ===============================================================
+
+    # == Callbacks ============================================================
+
+    # == Class Methods ========================================================
+
+    self.table_name = 'back_ops_actions'
+
+    # == Instance Methods =====================================================
+  end
+end

data/lib/back_ops/operation.rb

@@ -0,0 +1,26 @@
+module BackOps
+  class Operation < ActiveRecord::Base
+
+    # == Constants ============================================================
+
+    # == Attributes ===========================================================
+
+    # == Extensions ===========================================================
+
+    # == Relationships ========================================================
+
+    has_many :actions, class_name: 'BackOps::Action'
+
+    # == Validations ==========================================================
+
+    # == Scopes ===============================================================
+
+    # == Callbacks ============================================================
+
+    # == Class Methods ========================================================
+
+    self.table_name = 'back_ops_operations'
+
+    # == Instance Methods =====================================================
+  end
+end

data/lib/back_ops/version.rb

@@ -0,0 +1,3 @@
+module BackOps
+  VERSION = '0.1.0'
+end

data/lib/back_ops/worker.rb

@@ -0,0 +1,102 @@
+module BackOps
+  class Worker
+
+    # == Constants ============================================================
+
+    # == Attributes ===========================================================
+
+    # == Extensions ===========================================================
+
+    include Sidekiq::Worker
+
+    # == Aliases ==============================================================
+
+    # == Class Methods ========================================================
+
+    def self.perform_async(context, actions)
+      operation = setup_operation_and_actions(context, actions)
+      super(operation.id)
+    end
+
+    def self.perform_in(interval, context, actions)
+      operation = setup_operation_and_actions(context, actions)
+      super(interval, operation.id)
+    end
+
+    def self.perform_at(interval, context, action)
+      perform_in(interval, context, action)
+    end
+
+    def self.setup_operation_and_actions(context, actions)
+      raise ArgumentError, 'Cannot process empty actions' if actions.blank?
+
+      operation = BackOps::Operation.create_or_find_by({
+        params_hash: Digest::MD5.hexdigest("#{context}|#{actions}"),
+        name: ancestors[1]
+      })
+      operation.context.merge!(context)
+      operation.save!
+      
+      actions.each_with_index do |action, index|
+        BackOps::Action.create_or_find_by({
+          operation: operation,
+          name: action,
+          order: index
+        })
+      end
+
+      operation
+    end
+
+    # == Instance Methods =====================================================
+
+    def perform(operation_id)
+      operation = BackOps::Operation.find(operation_id)
+      process(operation)
+    end
+
+    private
+
+    def process(operation)
+      action_items = BackOps::Action.where({
+        operation: operation,
+        completed_at: nil
+      }).order(order: :asc)
+
+      return true if action_items.blank?
+
+      active_item = action_items[0]
+      next_item = action_items[1]
+
+      if active_item.errored_at.present?
+        active_item.errored_at = nil
+        active_item.error_message = nil
+        active_item.stack_trace = nil
+        active_item.save!
+      end
+      
+      begin
+        active_item.name.constantize.call(operation)
+
+        active_item.completed_at = Time.zone.now
+        active_item.attempts_count += 1
+        active_item.save!
+
+        if next_item.present?
+          Sidekiq::Client.push('class' => self.class.name, 'args' => [operation.id])
+        else
+          operation.completed_at = active_item.completed_at
+          operation.save!
+        end
+      rescue => e
+        active_item.error_message = e.message
+        active_item.stack_trace = e.backtrace
+        active_item.errored_at = Time.zone.now
+        active_item.attempts_count += 1
+        active_item.save!
+
+        raise
+      end
+    end
+  end
+end

data/lib/generators/back_ops/install_generator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# require 'rails/generators'
+require 'rails/generators/active_record'
+
+module BackOps
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+    source_root File.expand_path('templates', __dir__)
+    desc 'Installs the BackOps migration file.'
+
+    def create_migration_file
+      migration_template(
+        'migration.rb',
+        'db/migrate/create_back_ops_tables.rb',
+        migration_version: migration_version,
+      )
+    end
+
+    # def self.next_migration_number(dirname)
+    #   ActiveRecord::Migration.next_migration_number(dirname)
+    # end
+
+    private
+
+    def migration_version
+      "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+    end
+  end
+end

data/lib/generators/back_ops/templates/migration.rb.tt

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :back_ops_operations do |t|
+      t.string :name
+      t.string :params_hash
+      t.jsonb :context, null: false, default: {}
+      t.timestamp :completed_at
+
+      t.timestamps
+    end
+
+    add_index :back_ops_operations, [:name, :params_hash]
+    
+    create_table :back_ops_actions do |t|
+      t.integer :operation_id, limit: 8
+      t.integer :order, null: false, default: 0
+      t.text :name
+      t.text :error_message
+      t.text :stack_trace
+      t.timestamp :errored_at
+      t.timestamp :completed_at
+      t.integer :attempts_count, null: false, default: 0
+
+      t.timestamps
+    end
+
+    add_index :back_ops_actions, :operation_id
+  end
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: back_ops
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Aaron Price
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: BackOps processes multi-step jobs using Sidekiq in a retry-from-failed
+  fashion.
+email:
+- price.aaron@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/back_ops.rb
+- lib/back_ops/action.rb
+- lib/back_ops/operation.rb
+- lib/back_ops/version.rb
+- lib/back_ops/worker.rb
+- lib/generators/back_ops/install_generator.rb
+- lib/generators/back_ops/templates/migration.rb.tt
+homepage: https://github.com/aaronprice/back_ops
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/aaronprice/back_ops
+  source_code_uri: https://github.com/aaronprice/back_ops
+  changelog_uri: https://github.com/aaronprice/back_ops/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Multi-step background job processor
+test_files: []