scatter_gather 0.1.0

data/rbi/scatter_gather.rbi

@@ -0,0 +1,135 @@
+# typed: strong
+# Scatter-Gather Pattern for ActiveJob
+# 
+# This module provides a scatter-gather pattern for coordinating job execution.
+# Jobs can wait for other jobs to complete before executing, with configurable
+# polling, retry, and timeout behavior.
+# 
+# Example workflow:
+#   # Start some scatter jobs
+#   email_parser_job = EmailParserJob.perform_later(email_id: 123)
+#   attachment_processor_job = AttachmentProcessorJob.perform_later(email_id: 123)
+#   ai_categorizer_job = AICategorizerJob.perform_later(email_id: 123)
+# 
+#   # Create a gather job that waits for all dependencies to complete
+#   NotifyCompleteJob.gather(email_parser_job, attachment_processor_job, ai_categorizer_job).perform_later
+# 
+# The gather job will:
+# - Check if all dependencies are complete
+# - If complete: enqueue the target job immediately
+# - If not complete: poll every 2 seconds (configurable), re-enqueuing itself
+# - After 10 attempts (configurable): discard with error reporting
+# 
+# Configuration options:
+#   - max_attempts: Number of polling attempts before giving up (default: 10)
+#   - poll_interval: Time between polling attempts (default: 2.seconds)
+# 
+# Example with custom configuration:
+#   TouchingJob.gather(jobs, poll_interval: 0.2.seconds, max_attempts: 4).perform_later(final_path)
+module ScatterGather
+  extend ActiveSupport::Concern
+  DEFAULT_GATHER_CONFIG = T.let({
+  max_attempts: 10,
+  poll_interval: 2.seconds
+}.freeze, T.untyped)
+  VERSION = T.let("0.1.0", T.untyped)
+
+  # sord omit - no YARD return type given, using untyped
+  # Updates the completions table with the status of this job
+  sig { returns(T.untyped) }
+  def register_completion_for_gathering; end
+
+  class Completion < ActiveRecord::Base
+    # sord omit - no YARD type given for "active_job_ids", using untyped
+    # sord omit - no YARD return type given, using untyped
+    sig { params(active_job_ids: T.untyped).returns(T.untyped) }
+    def self.collect_statuses(active_job_ids); end
+  end
+
+  # Proxy class that mimics ActiveJob behavior for gather jobs
+  class GatherJobProxy
+    # sord omit - no YARD type given for "target_class", using untyped
+    # sord omit - no YARD type given for "ids", using untyped
+    # sord omit - no YARD type given for "config", using untyped
+    sig { params(target_class: T.untyped, ids: T.untyped, config: T.untyped).void }
+    def initialize(target_class, ids, config); end
+
+    # Mimic ActiveJob's perform_later method
+    # 
+    # _@param_ `args` — Positional arguments to pass to the target job's perform method
+    # 
+    # _@param_ `kwargs` — Keyword arguments to pass to the target job's perform method
+    # 
+    # _@return_ — Enqueues the gather job
+    sig { params(args: T::Array[T.untyped], kwargs: T::Hash[T.untyped, T.untyped]).void }
+    def perform_later(*args, **kwargs); end
+  end
+
+  # Custom exception for when gather job exhausts attempts
+  class DependencyTimeoutError < StandardError
+    # sord omit - no YARD type given for "max_attempts", using untyped
+    # sord omit - no YARD type given for "dependency_status", using untyped
+    sig { params(max_attempts: T.untyped, dependency_status: T.untyped).void }
+    def initialize(max_attempts, dependency_status); end
+
+    # sord omit - no YARD type given for :dependency_status, using untyped
+    # Returns the value of attribute dependency_status.
+    sig { returns(T.untyped) }
+    attr_reader :dependency_status
+  end
+
+  # Internal job class for polling and coordinating gather operations
+  class GatherJob < ActiveJob::Base
+    include ScatterGather
+
+    # sord omit - no YARD return type given, using untyped
+    sig { returns(T.untyped) }
+    def logger; end
+
+    # sord omit - no YARD type given for "wait_for_active_job_ids:", using untyped
+    # sord omit - no YARD type given for "target_job:", using untyped
+    # sord omit - no YARD type given for "gather_config:", using untyped
+    # sord omit - no YARD type given for "remaining_attempts:", using untyped
+    # sord omit - no YARD return type given, using untyped
+    sig do
+      params(
+        wait_for_active_job_ids: T.untyped,
+        target_job: T.untyped,
+        gather_config: T.untyped,
+        remaining_attempts: T.untyped
+      ).returns(T.untyped)
+    end
+    def perform(wait_for_active_job_ids:, target_job:, gather_config:, remaining_attempts:); end
+
+    # sord omit - no YARD type given for "hash", using untyped
+    # sord omit - no YARD return type given, using untyped
+    sig { params(hash: T.untyped).returns(T.untyped) }
+    def tally_in_logger_format(hash); end
+
+    # sord omit - no YARD type given for "target_job", using untyped
+    # sord omit - no YARD return type given, using untyped
+    sig { params(target_job: T.untyped).returns(T.untyped) }
+    def perform_target_later_from_args(target_job); end
+
+    # sord omit - no YARD return type given, using untyped
+    # Updates the completions table with the status of this job
+    sig { returns(T.untyped) }
+    def register_completion_for_gathering; end
+  end
+
+  # The generator is used to install ScatterGather. It adds the migration that creates
+  # the scatter_gather_completions table.
+  # Run it with `bin/rails g scatter_gather:install` in your console.
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    # sord omit - no YARD return type given, using untyped
+    # Generates migration file that creates the scatter_gather_completions table.
+    sig { returns(T.untyped) }
+    def create_migration_file; end
+
+    # sord omit - no YARD return type given, using untyped
+    sig { returns(T.untyped) }
+    def migration_version; end
+  end
+end

data/scatter_gather.gemspec

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "lib/scatter_gather/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "scatter_gather"
+  spec.version = ScatterGather::VERSION
+  spec.authors = ["Julik Tarkhanov"]
+  spec.email = ["me@julik.nl"]
+  spec.license = "MIT"
+
+  spec.summary = "Effortless step workflows that embed nicely inside Rails"
+  spec.description = "Step workflows for Rails/ActiveRecord"
+  spec.homepage = "https://scattergather.dev"
+  spec.required_ruby_version = ">= 3.1.0"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/julik/scatter_gather"
+  spec.metadata["changelog_uri"] = "https://github.com/julik/scatter_gather/blob/main/CHANGELOG.md"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      File.basename(f).start_with?(".")
+    end
+  end
+
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activerecord", ">= 7"
+  spec.add_dependency "activejob"
+  spec.add_dependency "railties"
+
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "rails", "~> 7.0"
+  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "standard", "~> 1.50.0", "< 2.0"
+  spec.add_development_dependency "magic_frozen_string_literal"
+  spec.add_development_dependency "yard"
+  spec.add_development_dependency "redcarpet" # needed for the yard gem to enable Github Flavored Markdown
+  spec.add_development_dependency "sord"
+end

data/sig/scatter_gather.rbs

@@ -0,0 +1,116 @@
+# Scatter-Gather Pattern for ActiveJob
+# 
+# This module provides a scatter-gather pattern for coordinating job execution.
+# Jobs can wait for other jobs to complete before executing, with configurable
+# polling, retry, and timeout behavior.
+# 
+# Example workflow:
+#   # Start some scatter jobs
+#   email_parser_job = EmailParserJob.perform_later(email_id: 123)
+#   attachment_processor_job = AttachmentProcessorJob.perform_later(email_id: 123)
+#   ai_categorizer_job = AICategorizerJob.perform_later(email_id: 123)
+# 
+#   # Create a gather job that waits for all dependencies to complete
+#   NotifyCompleteJob.gather(email_parser_job, attachment_processor_job, ai_categorizer_job).perform_later
+# 
+# The gather job will:
+# - Check if all dependencies are complete
+# - If complete: enqueue the target job immediately
+# - If not complete: poll every 2 seconds (configurable), re-enqueuing itself
+# - After 10 attempts (configurable): discard with error reporting
+# 
+# Configuration options:
+#   - max_attempts: Number of polling attempts before giving up (default: 10)
+#   - poll_interval: Time between polling attempts (default: 2.seconds)
+# 
+# Example with custom configuration:
+#   TouchingJob.gather(jobs, poll_interval: 0.2.seconds, max_attempts: 4).perform_later(final_path)
+module ScatterGather
+  extend ActiveSupport::Concern
+  DEFAULT_GATHER_CONFIG: untyped
+  VERSION: untyped
+
+  # sord omit - no YARD return type given, using untyped
+  # Updates the completions table with the status of this job
+  def register_completion_for_gathering: () -> untyped
+
+  class Completion < ActiveRecord::Base
+    # sord omit - no YARD type given for "active_job_ids", using untyped
+    # sord omit - no YARD return type given, using untyped
+    def self.collect_statuses: (untyped active_job_ids) -> untyped
+  end
+
+  # Proxy class that mimics ActiveJob behavior for gather jobs
+  class GatherJobProxy
+    # sord omit - no YARD type given for "target_class", using untyped
+    # sord omit - no YARD type given for "ids", using untyped
+    # sord omit - no YARD type given for "config", using untyped
+    def initialize: (untyped target_class, untyped ids, untyped config) -> void
+
+    # Mimic ActiveJob's perform_later method
+    # 
+    # _@param_ `args` — Positional arguments to pass to the target job's perform method
+    # 
+    # _@param_ `kwargs` — Keyword arguments to pass to the target job's perform method
+    # 
+    # _@return_ — Enqueues the gather job
+    def perform_later: (*::Array[untyped] args, **::Hash[untyped, untyped] kwargs) -> void
+  end
+
+  # Custom exception for when gather job exhausts attempts
+  class DependencyTimeoutError < StandardError
+    # sord omit - no YARD type given for "max_attempts", using untyped
+    # sord omit - no YARD type given for "dependency_status", using untyped
+    def initialize: (untyped max_attempts, untyped dependency_status) -> void
+
+    # sord omit - no YARD type given for :dependency_status, using untyped
+    # Returns the value of attribute dependency_status.
+    attr_reader dependency_status: untyped
+  end
+
+  # Internal job class for polling and coordinating gather operations
+  class GatherJob < ActiveJob::Base
+    include ScatterGather
+
+    # sord omit - no YARD return type given, using untyped
+    def logger: () -> untyped
+
+    # sord omit - no YARD type given for "wait_for_active_job_ids:", using untyped
+    # sord omit - no YARD type given for "target_job:", using untyped
+    # sord omit - no YARD type given for "gather_config:", using untyped
+    # sord omit - no YARD type given for "remaining_attempts:", using untyped
+    # sord omit - no YARD return type given, using untyped
+    def perform: (
+                   wait_for_active_job_ids: untyped,
+                   target_job: untyped,
+                   gather_config: untyped,
+                   remaining_attempts: untyped
+                 ) -> untyped
+
+    # sord omit - no YARD type given for "hash", using untyped
+    # sord omit - no YARD return type given, using untyped
+    def tally_in_logger_format: (untyped hash) -> untyped
+
+    # sord omit - no YARD type given for "target_job", using untyped
+    # sord omit - no YARD return type given, using untyped
+    def perform_target_later_from_args: (untyped target_job) -> untyped
+
+    # sord omit - no YARD return type given, using untyped
+    # Updates the completions table with the status of this job
+    def register_completion_for_gathering: () -> untyped
+  end
+
+  # The generator is used to install ScatterGather. It adds the migration that creates
+  # the scatter_gather_completions table.
+  # Run it with `bin/rails g scatter_gather:install` in your console.
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    # sord omit - no YARD return type given, using untyped
+    # Generates migration file that creates the scatter_gather_completions table.
+    def create_migration_file: () -> untyped
+
+    # sord omit - no YARD return type given, using untyped
+    def migration_version: () -> untyped
+  end
+end

data/test/dummy/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative "config/application"
+
+Rails.application.load_tasks

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1 @@
+/* Application styles */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class ApplicationController < ActionController::Base
+  # Only allow modern browsers supporting webp images, web push, badges, import maps, CSS nesting, and CSS :has.
+  allow_browser versions: :modern
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module ApplicationHelper
+end

data/test/dummy/app/jobs/application_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/test/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class ApplicationMailer < ActionMailer::Base
+  default from: "from@example.com"
+  layout "mailer"
+end

data/test/dummy/app/models/application_record.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ApplicationRecord < ActiveRecord::Base
+  primary_abstract_class
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title><%= content_for(:title) || "Dummy" %></title>
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <meta name="apple-mobile-web-app-capable" content="yes">
+    <meta name="mobile-web-app-capable" content="yes">
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= yield :head %>
+
+    <%# Enable PWA manifest for installable apps (make sure to enable in config/routes.rb too!) %>
+    <%#= tag.link rel: "manifest", href: pwa_manifest_path(format: :json) %>
+
+    <link rel="icon" href="/icon.png" type="image/png">
+    <link rel="icon" href="/icon.svg" type="image/svg+xml">
+    <link rel="apple-touch-icon" href="/icon.png">
+
+    <%# Includes all stylesheet files in app/assets/stylesheets %>
+    <%= stylesheet_link_tag "application" %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/test/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/test/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/test/dummy/app/views/pwa/manifest.json.erb

@@ -0,0 +1,22 @@
+{
+  "name": "Dummy",
+  "icons": [
+    {
+      "src": "/icon.png",
+      "type": "image/png",
+      "sizes": "512x512"
+    },
+    {
+      "src": "/icon.png",
+      "type": "image/png",
+      "sizes": "512x512",
+      "purpose": "maskable"
+    }
+  ],
+  "start_url": "/",
+  "display": "standalone",
+  "scope": "/",
+  "description": "Dummy.",
+  "theme_color": "red",
+  "background_color": "red"
+}

data/test/dummy/app/views/pwa/service-worker.js

@@ -0,0 +1,26 @@
+// Add a service worker for processing Web Push notifications:
+//
+// self.addEventListener("push", async (event) => {
+//   const { title, options } = await event.data.json()
+//   event.waitUntil(self.registration.showNotification(title, options))
+// })
+//
+// self.addEventListener("notificationclick", function(event) {
+//   event.notification.close()
+//   event.waitUntil(
+//     clients.matchAll({ type: "window" }).then((clientList) => {
+//       for (let i = 0; i < clientList.length; i++) {
+//         let client = clientList[i]
+//         let clientPath = (new URL(client.url)).pathname
+//
+//         if (clientPath == event.notification.data.path && "focus" in client) {
+//           return client.focus()
+//         }
+//       }
+//
+//       if (clients.openWindow) {
+//         return clients.openWindow(event.notification.data.path)
+//       }
+//     })
+//   )
+// })

data/test/dummy/bin/dev

@@ -0,0 +1,2 @@
+#!/usr/bin/env ruby
+exec "./bin/rails", "server", *ARGV

data/test/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"

data/test/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative "../config/boot"
+require "rake"
+Rake.application.run

data/test/dummy/bin/setup

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+require "fileutils"
+
+APP_ROOT = File.expand_path("..", __dir__)
+
+def system!(*args)
+  system(*args, exception: true)
+end
+
+FileUtils.chdir APP_ROOT do
+  # This script is a way to set up or update your development environment automatically.
+  # This script is idempotent, so that you can run it at any time and get an expectable outcome.
+  # Add necessary setup steps to this file.
+
+  puts "== Installing dependencies =="
+  system("bundle check") || system!("bundle install")
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?("config/database.yml")
+  #   FileUtils.cp "config/database.yml.sample", "config/database.yml"
+  # end
+
+  puts "\n== Preparing database =="
+  system! "bin/rails db:prepare"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! "bin/rails log:clear tmp:clear"
+
+  unless ARGV.include?("--skip-server")
+    puts "\n== Starting development server =="
+    $stdout.flush # flush the output before exec(2) so that it displays
+    exec "bin/dev"
+  end
+end

data/test/dummy/config/application.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "boot"
+
+require "rails/all"
+
+# Require the gems listed in Gemfile, including any gems
+# you've limited to :test, :development, or :production.
+Bundler.require(*Rails.groups)
+
+module Dummy
+  class Application < Rails::Application
+    config.load_defaults Rails::VERSION::STRING.to_f
+
+    # Please, add to the `ignore` list any other `lib` subdirectories that do
+    # not contain `.rb` files, or that should not be reloaded or eager loaded.
+    # Common ones are `templates`, `generators`, or `middleware`, for example.
+    config.autoload_lib(ignore: %w[assets tasks])
+
+    # Configuration for the application, engines, and railties goes here.
+    #
+    # These settings can be overridden in specific environments using the files
+    # in config/environments, which are processed later.
+    #
+    # config.time_zone = "Central Time (US & Canada)"
+    # config.eager_load_paths << Rails.root.join("extras")
+  end
+end

data/test/dummy/config/boot.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Set up gems listed in the Gemfile.
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)
+
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+$LOAD_PATH.unshift File.expand_path("../../../lib", __dir__)

data/test/dummy/config/cable.yml

@@ -0,0 +1,10 @@
+development:
+  adapter: async
+
+test:
+  adapter: test
+
+production:
+  adapter: redis
+  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  channel_prefix: dummy_production

data/test/dummy/config/database.sqlite3.yml

@@ -0,0 +1,32 @@
+# SQLite. Versions 3.8.0 and up are supported.
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem "sqlite3"
+#
+default: &default
+  adapter: sqlite3
+  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  timeout: 5000
+
+development:
+  <<: *default
+  database: storage/development.sqlite3
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  <<: *default
+  database: storage/test.sqlite3
+
+
+# SQLite3 write its data on the local filesystem, as such it requires
+# persistent disks. If you are deploying to a managed service, you should
+# make sure it provides disk persistence, as many don't.
+#
+# Similarly, if you deploy your application as a Docker container, you must
+# ensure the database is located in a persisted volume.
+production:
+  <<: *default
+  # database: path/to/persistent/storage/production.sqlite3

data/test/dummy/config/database.yml

@@ -0,0 +1,32 @@
+# SQLite. Versions 3.8.0 and up are supported.
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem "sqlite3"
+#
+default: &default
+  adapter: sqlite3
+  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  timeout: 5000
+
+development:
+  <<: *default
+  database: storage/development.sqlite3
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  <<: *default
+  database: storage/test.sqlite3
+
+
+# SQLite3 write its data on the local filesystem, as such it requires
+# persistent disks. If you are deploying to a managed service, you should
+# make sure it provides disk persistence, as many don't.
+#
+# Similarly, if you deploy your application as a Docker container, you must
+# ensure the database is located in a persisted volume.
+production:
+  <<: *default
+  # database: path/to/persistent/storage/production.sqlite3

data/test/dummy/config/environment.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Load the Rails application.
+require_relative "application"
+
+# Initialize the Rails application.
+Rails.application.initialize!