scatter_gather 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 423f509998336fa071a30240b18544524dbb9ee711c1250e3daf597c10bd9993
+  data.tar.gz: 14b5c6c4084f4c9c9dc440ab4c015e85f67695b6512ba1a23dc94e477bdaca6c
+SHA512:
+  metadata.gz: df506e118824334796d6f2da4464672740bbfc376a58a29e754f7224958bf369983ca253f7e38887bbcbdab1f1159616d31790bbdefe6f0ed4f216f8b7d522dd
+  data.tar.gz: a47b8afba198ff4ab3428ed11fdbb4469f83ab45139e8f8dcf2f29261c0c144f54d38c26181eff9a74c6d694d72d01f6e505c744a1451a867543102a13cc838b

data/.cursor/rules/instructions.mdc

@@ -0,0 +1,21 @@
+---
+alwaysApply: true
+---
+
+## Linting
+
+* Run standardrb --fix-unsafely on any .rb file you create or make edits to
+
+
+## Commit message titles
+
+* Do not use feat/chore prefixes for commit message titles
+
+## Type definitions
+
+* Do not update type definitions (rbi and rbs files) they get updated automatically on every gem build
+
+## Comments
+
+* Add generous YARD comments to the public methods you add. Add type definitions for parameters and return values
+* Specify @return [void] for methods that return no value / undefined value

data/.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -0,0 +1,55 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [ main ]
+
+jobs:
+  lint:
+    name: "Lint (standardrb)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.2.2
+          bundler-cache: true
+
+      - name: Lint code for consistent style
+        run: bundle exec standardrb
+
+  test:
+    name: "Tests (SQLite)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.2.2
+          bundler-cache: true
+
+      - name: Remove existing schema.rb
+        run: rm -f test/dummy/db/schema.rb
+
+      - name: Setup database
+        env:
+          RAILS_ENV: test
+          DATABASE_URL: sqlite3:db/test.sqlite3
+        run: |
+          cd test/dummy
+          bundle exec rails db:create
+          bundle exec rails db:migrate
+          cd ../..
+
+      - name: Run tests
+        env:
+          RAILS_ENV: test
+          DATABASE_URL: sqlite3:db/test.sqlite3
+        run: bin/test

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## Unreleased
+
+## [0.1.0] - 2025-09-30
+
+- Initial release

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "puma"
+gem "sqlite3"
+
+# Start debugger with binding.b [https://github.com/ruby/debug]
+# gem "debug", ">= 1.0.0"

data/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Julik Tarkhanov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# scatter_gather
+
+A Ruby gem that provides a scatter-gather pattern for coordinating ActiveJob execution. Jobs can wait for other jobs to complete before executing, with configurable polling, retry, and timeout behavior.
+
+## Usage
+
+Start some scatter jobs and create a gather job that waits for all dependencies to complete:
+
+```ruby
+class EmailProcessorJob < ActiveJob::Base
+  include ScatterGather
+
+  def perform(email_id)
+    # Process email
+  end
+end
+
+class AttachmentProcessorJob < ActiveJob::Base
+  include ScatterGather
+
+  def perform(email_id)
+    # Process attachments
+  end
+end
+
+class AICategorizerJob < ActiveJob::Base
+  include ScatterGather
+
+  def perform(email_id)
+    # Categorize email with AI
+  end
+end
+
+class NotifyCompleteJob < ActiveJob::Base
+  include ScatterGather
+
+  def perform(email_id)
+    # Notify that all processing is complete
+  end
+end
+
+# Start the scatter jobs
+email_parser_job = EmailProcessorJob.perform_later(email_id: 123)
+attachment_processor_job = AttachmentProcessorJob.perform_later(email_id: 123)
+ai_categorizer_job = AICategorizerJob.perform_later(email_id: 123)
+
+# Create a gather job that waits for all dependencies to complete
+NotifyCompleteJob.gather(email_parser_job, attachment_processor_job, ai_categorizer_job).perform_later(email_id: 123)
+```
+
+The gather job will:
+- Check if all dependencies are complete
+- If complete: enqueue the target job immediately
+- If not complete: poll every 2 seconds (configurable), re-enqueuing itself
+- After 10 attempts (configurable): discard with error reporting
+
+### Configuration Options
+
+- `max_attempts`: Number of polling attempts before giving up (default: 10)
+- `poll_interval`: Time between polling attempts (default: 2.seconds)
+
+```ruby
+# Example with custom configuration
+TouchingJob.gather(jobs, poll_interval: 0.2.seconds, max_attempts: 4).perform_later(final_path)
+```
+
+## Installation
+
+Add the gem to the application's Gemfile, and then generate and run the migration:
+
+    $ bundle add scatter_gather
+    $ bundle install
+    $ bin/rails g scatter_gather:install
+    $ bin/rails db:migrate
+
+## Development
+
+After checking out the repo, run `bundle` to install dependencies. The development process from there on is like any other gem.
+
+## License
+
+This gem is made available under the MIT license
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/julik/scatter_gather.

data/Rakefile

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks(name: "scatter_gather")
+require "rake/testtask"
+require "standard/rake"
+require "yard"
+
+YARD::Rake::YardocTask.new(:doc)
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.warning = false # To avoid any warnings from dependencies
+end
+
+task :format do
+  `bundle exec standardrb --fix`
+  `bundle exec magic_frozen_string_literal .`
+end
+
+task :generate_typedefs do
+  `bundle exec sord rbi/scatter_gather.rbi`
+  `bundle exec sord sig/scatter_gather.rbs`
+end
+
+task default: [:test, :standard, :generate_typedefs]
+
+# When building the gem, generate typedefs beforehand,
+# so that they get included
+Rake::Task["build"].enhance(["generate_typedefs"])

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "scatter_gather"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/test

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$: << File.expand_path("../test", __dir__)
+
+require "bundler/setup"
+require "rails/plugin/test"

data/lib/generators/install_generator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module ScatterGather
+  # The generator is used to install ScatterGather. It adds the migration that creates
+  # the scatter_gather_completions table.
+  # Run it with `bin/rails g scatter_gather:install` in your console.
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    source_paths << File.join(File.dirname(__FILE__, 2))
+
+    # Generates migration file that creates the scatter_gather_completions table.
+    def create_migration_file
+      # Migration files are named "...migration_001.rb" etc. This allows them to be emitted
+      # as they get added, and the order of the migrations can be controlled using predictable sorting.
+      # Adding a new migration to the gem is then just adding a file.
+      migration_file_paths_in_order = Dir.glob(__dir__ + "/*_migration_*.rb.erb").sort
+      migration_file_paths_in_order.each do |migration_template_path|
+        untemplated_migration_filename = File.basename(migration_template_path).gsub(/\.erb$/, "")
+        migration_template(migration_template_path, File.join(db_migrate_path, untemplated_migration_filename))
+      end
+    end
+
+    private
+
+    def migration_version
+      ActiveRecord::VERSION::STRING.split(".").take(2).join(".")
+    end
+  end
+end

data/lib/generators/scatter_gather_migration_001.rb.erb

@@ -0,0 +1,16 @@
+class AddScatterGatherCompletions < ActiveRecord::Migration[<%= migration_version %>]
+  def up
+    create_table :scatter_gather_completions do |t|
+      t.string :active_job_id, null: false
+      t.string :active_job_class_name
+      t.string :status, default: "unknown"
+      t.timestamps
+    end
+    add_index :scatter_gather_completions, [:active_job_id], unique: true # For lookups
+    add_index :scatter_gather_completions, [:created_at] # For cleanup
+  end
+
+  def down
+    drop_table :scatter_gather_completions
+  end
+end

data/lib/scatter_gather/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ScatterGather
+  VERSION = "0.1.0"
+end

data/lib/scatter_gather.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require "active_support"
+require "active_record"
+require "active_job"
+require "json"
+
+# Scatter-Gather Pattern for ActiveJob
+#
+# This module provides a scatter-gather pattern for coordinating job execution.
+# Jobs can wait for other jobs to complete before executing, with configurable
+# polling, retry, and timeout behavior.
+#
+# Example workflow:
+#   # Start some scatter jobs
+#   email_parser_job = EmailParserJob.perform_later(email_id: 123)
+#   attachment_processor_job = AttachmentProcessorJob.perform_later(email_id: 123)
+#   ai_categorizer_job = AICategorizerJob.perform_later(email_id: 123)
+#
+#   # Create a gather job that waits for all dependencies to complete
+#   NotifyCompleteJob.gather(email_parser_job, attachment_processor_job, ai_categorizer_job).perform_later
+#
+# The gather job will:
+# - Check if all dependencies are complete
+# - If complete: enqueue the target job immediately
+# - If not complete: poll every 2 seconds (configurable), re-enqueuing itself
+# - After 10 attempts (configurable): discard with error reporting
+#
+# Configuration options:
+#   - max_attempts: Number of polling attempts before giving up (default: 10)
+#   - poll_interval: Time between polling attempts (default: 2.seconds)
+#
+# Example with custom configuration:
+#   TouchingJob.gather(jobs, poll_interval: 0.2.seconds, max_attempts: 4).perform_later(final_path)
+module ScatterGather
+  extend ActiveSupport::Concern
+
+  class Completion < ActiveRecord::Base
+    self.table_name = "scatter_gather_completions"
+
+    def self.collect_statuses(active_job_ids)
+      statuses = active_job_ids.map { |it| [it, :unknown] }.to_h
+      statuses_from_completions = where(active_job_id: active_job_ids)
+        .pluck(:active_job_id, :status)
+        .map do |(id, st)|
+          [id, st.to_sym]
+        end.to_h
+      statuses.merge!(statuses_from_completions)
+    end
+  end
+
+  # Default configuration for gather jobs
+  DEFAULT_GATHER_CONFIG = {
+    max_attempts: 10,
+    poll_interval: 2.seconds
+  }.freeze
+
+  # Proxy class that mimics ActiveJob behavior for gather jobs
+  class GatherJobProxy
+    def initialize(target_class, ids, config)
+      @target_class = target_class
+      @ids = ids
+      @config = config.with_indifferent_access
+    end
+
+    # Mimic ActiveJob's perform_later method
+    # @param args [Array] Positional arguments to pass to the target job's perform method
+    # @param kwargs [Hash] Keyword arguments to pass to the target job's perform method
+    # @return [void] Enqueues the gather job
+    def perform_later(*args, **kwargs)
+      job_arguments = {cn: @target_class.name, p: args, k: kwargs}
+      gather_job_params = {
+        wait_for_active_job_ids: @ids,
+        target_job: job_arguments,
+        gather_config: @config,
+        remaining_attempts: @config.fetch(:max_attempts) - 1
+      }
+      tagged = ActiveSupport::TaggedLogging.new(Rails.logger).tagged("ScatterGather")
+      tagged.info { "Enqueueing gather job waiting for #{@ids.inspect} to run a #{@target_class.name} after" }
+      GatherJob.perform_later(**gather_job_params)
+    end
+  end
+
+  # Custom exception for when gather job exhausts attempts
+  class DependencyTimeoutError < StandardError
+    attr_reader :dependency_status
+
+    def initialize(max_attempts, dependency_status)
+      @dependency_status = dependency_status
+      super(<<~MSG)
+        Gather failed after #{max_attempts} attempts. Dependencies:
+        
+        #{JSON.pretty_generate(dependency_status)}
+      MSG
+    end
+  end
+
+  # Internal job class for polling and coordinating gather operations
+  class GatherJob < ActiveJob::Base
+    include ScatterGather
+    discard_on DependencyTimeoutError
+
+    def logger = ActiveSupport::TaggedLogging.new(super).tagged("ScatterGather")
+
+    def perform(wait_for_active_job_ids:, target_job:, gather_config:, remaining_attempts:)
+      deps = ScatterGather::Completion.collect_statuses(wait_for_active_job_ids)
+      logger.info { "Gathered completions #{tally_in_logger_format(deps)}" }
+
+      all_done = deps.values.all? { |it| it == :completed }
+      if all_done
+        logger.info { "Dependencies done, enqueueing #{target_job.fetch(:cn)}" }
+        perform_target_later_from_args(target_job)
+        Completion.where(active_job_id: wait_for_active_job_ids).delete_all
+      elsif remaining_attempts < 1
+        max_attempts = gather_config.fetch(:max_attempts)
+        error = DependencyTimeoutError.new(max_attempts, deps)
+        logger.warn { "Failed to gather dependencies after #{max_attempts} attempts" }
+        Completion.where(active_job_id: wait_for_active_job_ids).delete_all
+
+        # We configure our job to discard on timeout, and discard does not report the error by default
+        Rails.error.report(error)
+        raise error
+      else
+        # Re-enqueue with delay. We could poll only for dependencies which are still remaining,
+        # but for debugging this is actually worse because for hanging stuff there will be one
+        # job that hangs in the end. Knowing which jobs were part of the batch is useful!
+        args = {
+          wait_for_active_job_ids:,
+          target_job:,
+          gather_config:,
+          remaining_attempts: remaining_attempts - 1
+        }
+        wait = gather_config.fetch(:poll_interval)
+        self.class.set(wait:).perform_later(**args)
+      end
+    end
+
+    private
+
+    def tally_in_logger_format(hash)
+      hash.values.tally.map do |k, count|
+        "#{k}=#{count}"
+      end.join(" ")
+    end
+
+    def perform_target_later_from_args(target_job)
+      # The only purpose of this is to pass all variations
+      # of `perform_later` argument shapes correctly
+      job_class = target_job.fetch(:cn).constantize
+      if target_job[:p].any? && target_job[:k] # Both
+        job_class.perform_later(*target_job[:p], **target_job[:k])
+      elsif target_job[:k] # Just kwargs
+        job_class.perform_later(**target_job[:k])
+      elsif target_job[:p] # Just posargs
+        job_class.perform_later(*target_job[:p])
+      else
+        job_class.perform_later # No args
+      end
+    end
+  end
+
+  included do
+    after_perform :register_completion_for_gathering
+    discard_on ScatterGather::DependencyTimeoutError
+
+    def self.gather(*active_jobs, **gather_config_options)
+      active_jobs = Array(active_jobs).flatten
+      config = DEFAULT_GATHER_CONFIG.merge(gather_config_options)
+
+      # Pre-insert IDs to wait for
+      t = Time.current
+      attrs = active_jobs.map do |aj|
+        {
+          active_job_id: aj.job_id,
+          active_job_class_name: aj.class.name,
+          status: "pending",
+          created_at: t,
+          updated_at: t
+        }
+      end
+      ScatterGather::Completion.insert_all(attrs)
+      ScatterGather::Completion.where("created_at < ?", 1.week.ago).delete_all
+
+      # Return a proxy object that behaves like an ActiveJob proxy
+      GatherJobProxy.new(self, active_jobs.map(&:job_id), config)
+    end
+  end
+
+  # Updates the completions table with the status of this job
+  def register_completion_for_gathering
+    n_updated = ScatterGather::Completion.where(active_job_id: job_id).update_all(status: "completed", updated_at: Time.current)
+    if n_updated > 0
+      logger.tagged("ScatterGather").info { "Registered completion of #{self.class.name} id=#{job_id} since it will be gathered" }
+    end
+  end
+end

data/lib/tasks/scatter_gather_tasks.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+namespace :scatter_gather do
+  desc "Recover all journeys hanging in the 'performing' state"
+  task :recovery do
+    ScatterGather::RecoverStuckJourneysJob.perform_now
+  end
+end