chrono_forge 0.10.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f03445b6275e345beb34505d4d59a01d8450df220e94f07bc909c8c69059ab8d
-  data.tar.gz: 9ba7aaa7364736f66778da68af4f21d7c944ac01270c7b92ca76bf09bf880738
+  metadata.gz: fd4d0c02357d2d185a6d82034d21a6e09c034c0ae04a9b2cbd5cfcb5bb29b8f0
+  data.tar.gz: 981e5d7208c8a80dacb9a13a439e28a896037980364b9a810f976daa93597d65
 SHA512:
-  metadata.gz: f761f180b4e8323721cfffc0a7c2569f30ea8e5b8e085cc52ab32aeefa64ed2a45caac13dff38c168ae497c437816201cdf5c2946a85ab987814eecd852d97c6
-  data.tar.gz: 22ca2b2ca99188b5117c06e2d9b313e726e0087ed48d3635d1064bcb82eee69b9f649a56a441a42e603c3b32ceb98c1aa788782dc01c5d2066f09dae0593900b
+  metadata.gz: 7a4ccac2097e6c808103327201f00c1e79341bbd433f1564c5a2ffc306c3740509e195825e613781d62d27844e230ce12b3a048d116a2d44ba3b61ea11f16f7c
+  data.tar.gz: e974b5cfd5aaf41045b84947993cb1052a19503bab85e2309b8b5ea13563340fac5136800c30d1a63ee34408c47262ed67922dc21166a6f2361d3742d021be18

data/CHANGELOG.md

@@ -1,4 +1,47 @@
-## [Unreleased]
+## [0.12.0] - 2026-07-09
+
+### Documentation
+
+- Document the stalled-workflow reaper and fix broken README anchors
+
+### Features
+
+- Reap workflows stranded in running by hard-killed workers
+
+## [0.11.0] - 2026-07-04
+
+### Bug Fixes
+
+- Back off BranchMergeJob polling for non-progressing children
+- Leave prepare uncommitted for review; publish commits
+- Converge merges promptly via drain-ETA cadence and never-started-count rekick ([#12](https://github.com/radioactive-labs/chrono_forge/issues/12))
+- Overlay/analyzer correctness, detail-panel XSS, and UX polish
+
+### Documentation
+
+- Refresh dashboard screenshots, badges; fix upgrade note and API reference
+- Restructure READMEs — promote branches, refine cadence note, dashboard cross-link
+- Add ActiveJob Continuations, Rails-version, and dashboard rows to comparison
+- Correct "workflow is just a Ruby method" framing
+- Add ChronoForge equivalent of the step DSL; fix step-vs-workflow framing
+
+### Features
+
+- Add bulk merge / merge_once for context ([#11](https://github.com/radioactive-labs/chrono_forge/issues/11))
+- Workflow definition graph with static DAG and live run overlay ([#13](https://github.com/radioactive-labs/chrono_forge/issues/13))
+
+### Miscellaneous Tasks
+
+- Add cliff-driven release script for both gems
+- Replace bin/release with per-gem rake release flow
+
+### Performance
+
+- Consolidate per-child commits and flatten log writes
+
+### Styling
+
+- Apply standardrb blank-line formatting to existing files
 
 ## [0.10.0] - 2026-06-27
 

data/README.md

@@ -11,6 +11,7 @@ ChronoForge handles long-running processes, manages state, and recovers from fai
 
 Workflows are **plain Ruby**. Ordinary `if`/`else`, loops, and early returns drive the flow. There's no declarative DSL to learn and no extra service to run, which makes ChronoForge a good fit for business processes whose shape depends on runtime state: conditional branches, iteration over data, and built-in periodic tasks (`durably_repeat`).
 
+> [!NOTE]
 > **In production** at **achieve by Petra**, an investment platform in the Petra Group — where it has executed over 3.6 million workflows and 32 million durable steps across scheduled payments, investment rollovers, and membership lifecycle management.
 
 ## 🧭 Why ChronoForge
@@ -23,19 +24,44 @@ step :remind_of_tasks, wait: 2.days
 step :complete_onboarding, wait: 15.days
 ```
 
-That reads cleanly for a fixed, linear sequence. But many business processes branch, loop, and react to data that only exists at runtime, and a declarative schema gets awkward there. ChronoForge takes the opposite approach: **a workflow is just a Ruby method.** Conditionals, iteration, early returns, and helper methods all work the way they normally do.
+That reads cleanly for a fixed, linear sequence. But many business processes branch, loop, and react to data that only exists at runtime, and a declarative schema gets awkward there. ChronoForge takes the opposite approach: **the workflow is plain Ruby, and each step is just a method.** Conditionals, iteration, early returns, and helper methods all work the way they normally do; you drive durable steps, waits, and retries inline with a few primitives like `durably_execute` and `wait_until`:
 
-There is a real trade-off. Because the flow is ordinary code, ChronoForge can show the steps that **have run** (a replay/history view), but not a roadmap of steps that *haven't* run yet, which a declarative engine can. For workflows whose path isn't fixed in advance, that's a trade worth making; for a simple, fixed sequence ("send email, wait 2 days, send another"), a declarative DSL may read more cleanly, and that's a fine reason to reach for one.
+```ruby
+class OnboardingWorkflow < ApplicationJob
+  prepend ChronoForge::Executor
+
+  def perform(user_id:)
+    @user_id = user_id
+
+    durably_execute :send_welcome_email
+    wait 2.days, :remind_delay
+    durably_execute :remind_of_tasks
+    wait 15.days, :onboarding_delay
+    durably_execute :complete_onboarding
+  end
+
+  private
+
+  def send_welcome_email = UserMailer.welcome(@user_id).deliver_now
+  def remind_of_tasks = UserMailer.task_reminder(@user_id).deliver_now
+  def complete_onboarding = User.find(@user_id).complete_onboarding!
+end
+```
+
+There is a trade-off, though a smaller one than it used to be. A declarative engine can show the steps a run *hasn't* reached yet; plain imperative code can't. The [definition graph](#-dashboard) closes most of that gap. It reads your `perform` with Prism (without running it) and draws the steps the run will take, with live status on each. The view is best-effort: a computed step name or a data-dependent loop collapses to one `dynamic` node. For a simple, fixed sequence ("send email, wait 2 days, send another") a declarative DSL can still read more cleanly, and that's a fine reason to use one.
 
 ### How it compares
 
-|                              | ChronoForge          | GenevaDrive        | AcidicJob       | Temporal        |
-| ---------------------------- | -------------------- | ------------------ | --------------- | --------------- |
-| Programming model            | procedural (plain Ruby) | declarative DSL | declarative DSL | procedural (via SDK) |
-| Built-in periodic tasks      | ✓ `durably_repeat`   | ✗                  | ✗               | ✓               |
-| Pending-step visibility      | ✗ (procedural)       | ✓                  | ✓               | ✗ (procedural)  |
-| Extra infrastructure         | none (DB + ActiveJob)| none               | none            | server required |
-| License                      | MIT                  | LGPL / commercial  | MIT             | MIT             |
+|                              | ChronoForge          | AJ Continuations           | GenevaDrive        | AcidicJob       | Temporal        |
+| ---------------------------- | -------------------- | -------------------------- | ------------------ | --------------- | --------------- |
+| Programming model            | procedural (plain Ruby) | procedural (`step` blocks) | declarative DSL | declarative DSL | procedural (via SDK) |
+| Built-in periodic tasks      | ✓ `durably_repeat`   | ✗                          | ✗                  | ✗               | ✓               |
+| Parallel sub-workflows       | ✓ `branch` / `spawn` | ✗                          | ✗                  | ✗               | ✓               |
+| Pending-step visibility      | ✗ (procedural)       | ✗ (procedural)             | ✓                  | ✓               | ✗ (procedural)  |
+| Web dashboard                | ✓ (free gem)         | job-level (Mission Control)| paid only          | ✗               | ✓               |
+| Extra infrastructure         | none (DB + ActiveJob)| none (built into Rails)    | none               | none            | server required |
+| Rails support                | 7.1+                 | 8.1+                       | 7.2+               | 7.1+            | any (Ruby SDK)  |
+| License                      | MIT                  | MIT                        | LGPL / commercial  | MIT             | MIT             |
 
 <sub>Comparison reflects each project's documented features as of mid-2026, to the best of our knowledge; corrections welcome via PR.</sub>
 
@@ -43,8 +69,11 @@ A few deliberate choices behind that table:
 
 - **Periodic tasks are built in.** `durably_repeat` runs a step on a schedule until a condition holds, with automatic catch-up for missed runs, so a workflow can be its own recurring job and cron-style monitor, right alongside the rest of its logic. Without built-in support, periodic behavior usually lives in a separate scheduler that you reconcile with workflow state by hand.
 - **No extra infrastructure.** ChronoForge is a gem over your existing database and ActiveJob backend. There's no separate server or daemon to operate, unlike Temporal.
+- **Large-scale fan-out is built in.** `branch` with `spawn`/`spawn_each` fans a workflow out into child workflows that run concurrently and join when their results are needed, streaming ActiveRecord relations in constant memory for large sets. Among the Ruby-native engines here, only ChronoForge offers this without a separate orchestration server (Temporal does, server-side).
 - **Recovery is built into the model.** Steps are append-only history, so a crashed step leaves the workflow `stalled`, recoverable directly with `retry_later`.
+- **A real dashboard, free.** ChronoForge's [mountable dashboard](#-dashboard) — workflow list, step-replay timeline, a per-run **definition graph** (the steps a run will take, read from `perform`, with live status on each), context inspector, retry/unlock — ships as a separate MIT gem.
 - **MIT licensed.** Permissive and dependency-policy-friendly.
+- **ActiveJob Continuations solve a narrower problem.** Rails 8.1's built-in [continuations](https://api.rubyonrails.org/classes/ActiveJob/Continuation.html) make a *single* long job survive interruptions: you wrap work in `step` blocks and track a `cursor`, and at each checkpoint the job asks the queue adapter whether it's `stopping?`, re-enqueuing to resume from the last completed step/cursor — no gem, no tables. They deliberately stop short of being a workflow engine: there's no durable waiting on time, conditions, or external events; no periodic steps; no parallel fan-out; and no persisted, queryable history. Reach for continuations to make one big job restart-safe; reach for ChronoForge when a process spans steps that wait, recur, fan out, and need recovery and visibility. They also compose — a ChronoForge workflow *is* ActiveJob work.
 
 ## 🌟 Features
 
@@ -52,6 +81,7 @@ A few deliberate choices behind that table:
 - **Durable Execution**: Automatically tracks and recovers from failures during workflow execution
 - **Periodic tasks built in**: `durably_repeat` runs a step on an interval until a condition is met, with catch-up for missed runs. Acts as a recurring task and a cron-style monitor in one
 - **Wait States**: Time-based waits and condition-based waiting (`wait_until`) that survive restarts
+- **Parallel sub-workflows**: `branch` with `spawn`/`spawn_each` fans out into concurrent child workflows and joins them (`automerge` or `merge_branches`); large sets stream in constant memory. See [Branches](#-branches-parallel-sub-workflows)
 - **State Management**: Built-in workflow state tracking with persistent context storage
 - **Concurrency Control**: Advanced locking mechanisms to prevent parallel execution of the same workflow
 - **Error Handling**: Error tracking with a unified, configurable [`RetryPolicy`](#-retry-policies) (including per-error-type policies)
@@ -60,12 +90,16 @@ A few deliberate choices behind that table:
 - **ActiveJob Integration**: Compatible with all ActiveJob backends, though database-backed processors (like Solid Queue) provide the most reliable experience for long-running workflows
 - **Retention & Cleanup**: A schedulable job to prune finished workflows and the unbounded logs that periodic tasks accumulate (see [Cleanup & Retention](#-cleanup--retention))
 
-## 🖥️ Dashboard
+## 🖥 Dashboard
 
-ChronoForge has a free, mountable dashboard for visibility and recovery: workflow list, step replay timeline, context inspector, periodic-task health, wait-state age, and retry/unlock actions. It ships as a separate gem, `chrono_forge-dashboard`, so the core stays lean.
+ChronoForge has a free, mountable dashboard for visibility and recovery: workflow list, step replay timeline, a per-run **definition graph** (the durable steps a workflow will run, statically parsed from `perform`, with live run status overlaid), context inspector, periodic-task health, wait-state age, and retry/unlock actions. It ships as a separate gem, `chrono_forge-dashboard`, so the core stays lean.
 
 [![ChronoForge dashboard](chrono_forge-dashboard/docs/screenshots/workflows.png)](chrono_forge-dashboard/README.md#screenshots)
 
+The per-run **definition graph**: the steps a workflow will run, read from `perform`, with the run's status shown on each node.
+
+[![Definition graph](chrono_forge-dashboard/docs/screenshots/definition-graph-scheduled-payment.png)](chrono_forge-dashboard/README.md#screenshots)
+
 ```ruby
 # Gemfile
 gem "chrono_forge-dashboard"
@@ -114,9 +148,9 @@ $ rails generate chrono_forge:upgrade
 $ rails db:migrate
 ```
 
-The upgrade migration is idempotent (`if_not_exists`), so it is safe to run even
-if your schema already has the index. Fresh installs get the index from the
-install migration and do **not** need to run the upgrade.
+Re-running the generator is safe — it skips any migrations you already have.
+Fresh installs get everything from `chrono_forge:install` and don't need the
+upgrade.
 
 ## 📋 Usage
 
@@ -310,7 +344,15 @@ end
 - **Idempotent**: Same operation won't be executed twice during replays
 - **Automatic Retries**: Failed executions retry per a unified `RetryPolicy` (exponential backoff with jitter; the step default caps at 30s over 3 attempts)
 - **Error Tracking**: All failures are logged with detailed error information
-- **Configurable**: Pass a `retry_policy:` per call, or set a class-wide default with the `retry_policy` DSL (see [Retry Policies](#retry-policies))
+- **Configurable**: Pass a `retry_policy:` per call, or set a class-wide default with the `retry_policy` DSL (see [Retry Policies](#-retry-policies))
+
+> **Write your side effects to be idempotent.** A step short-circuits on replay only
+> once its log row reaches `completed`. If a worker is hard-killed (SIGKILL/OOM/eviction)
+> *after* a step's side effect commits but *before* its log is marked `completed`, the
+> step re-runs when the workflow is resumed (including by the [reaper](#recovering-stranded-workflows-hard-killed-workers)).
+> Make external side effects safe to repeat — use a natural/unique key with
+> `create_or_find_by`, an upsert, or a rescue on the uniqueness violation — rather than
+> assuming exactly-once execution.
 
 #### 🔁 Retry Policies
 
@@ -590,6 +632,21 @@ def cleanup_files(scheduled_time)
 end
 ```
 
+#### 🌿 Branches
+
+When a workflow needs to fan out — process every pending order, reconcile each region — `branch` spawns child workflows that run concurrently and join when their results are needed:
+
+```ruby
+branch :reconcile, automerge: true do
+  spawn :eu, ReconcileWorkflow, region: "EU"
+  spawn_each :orders, Order.pending do |order|
+    [OrderWorkflow, { order_id: order.id }]
+  end
+end
+```
+
+`spawn_each` streams ActiveRecord relations by primary key in constant memory, so a branch can fan out over very large sets. Join inline with `automerge: true`, or open branches without it and `merge_branches` them later after doing other work. See [Branches: parallel sub-workflows](#-branches-parallel-sub-workflows) for the full model, a worked example, and the crash-recovery caveats.
+
 #### 🔄 Workflow Context
 
 ChronoForge provides a persistent context that survives job restarts. The context behaves like a Hash but with additional capabilities:
@@ -611,6 +668,12 @@ context.set(:total_amount, 99.99)
 # Set a value only if the key doesn't already exist
 context.set_once(:created_at, Time.current.iso8601)
 
+# Set several values at once (alias: set_multiple)
+context.merge(status: "processing", total_amount: 99.99, attempts: 0)
+
+# Set several values at once, but only keys that don't already exist (alias: set_multiple_once)
+context.merge_once(created_at: Time.current.iso8601, attempts: 0)
+
 # Check if a key exists
 if context.key?(:user_id)
   # Do something with the user ID
@@ -652,6 +715,113 @@ end
 
 To make an error non-retryable, leave it out of `retry_on:` (an empty `retry_on: []` retries nothing).
 
+## 🌿 Branches: parallel sub-workflows
+
+`branch` / `spawn` / `spawn_each` / `merge_branches` let a workflow fan out into
+child workflows that run concurrently, then join them when their results are
+needed.
+
+### Model
+
+- **`branch :name do … end`** opens a named branch (a durable step). Inside the
+  block, `spawn` and `spawn_each` create and immediately enqueue child workflows —
+  children start running as soon as the branch block is entered.
+- **`spawn :name, WorkflowClass, **kwargs`** — enqueues one child workflow.
+- **`spawn_each :name, source do |item| [WorkflowClass, kwargs] end`** — enqueues
+  one child per item. The block returns the class and kwargs, so one branch can
+  fan out into mixed workflow types. Sources are iterated in constant memory;
+  ActiveRecord relations are streamed by primary key — pass them **without** an
+  explicit `.order`.
+- **`automerge: true`** — joins the branch **inline at the block's close**.
+  Execution does not continue past the `branch` call until every child has
+  completed. Use it for "dispatch this group and wait right here."
+- **`merge_branches :a, :b`** (or the singular alias `merge_branch :a`) — the
+  separate join point. Open branches without `automerge`, do other work while the
+  children run, then join when you need their results. `merge_branches` blocks
+  until all named branches are complete.
+
+### Worked example
+
+```ruby
+class FulfillmentWorkflow < ApplicationJob
+  prepend ChronoForge::Executor
+
+  def perform(cycle_id:)
+    # automerge: the branch is joined inline, right where the block closes —
+    # `perform` does not continue past it until every child has completed.
+    branch :reconcile, automerge: true do
+      spawn :eu, ReconcileWorkflow, region: "EU"
+      spawn_each :orders, Order.pending do |order|
+        order.priority? ? [PriorityOrderWorkflow, { order_id: order.id }]
+                        : [OrderWorkflow, { order_id: order.id }]
+      end
+    end
+
+    # For branches you want to run concurrently and join later, omit automerge
+    # and use merge_branches:
+    branch :invoices do
+      spawn_each :unpaid, Invoice.unpaid do |inv|
+        [InvoiceWorkflow, { invoice_id: inv.id }]
+      end
+    end
+    branch :shipments do
+      spawn_each :ready, Shipment.ready do |s|
+        [ShipmentWorkflow, { shipment_id: s.id }]
+      end
+    end
+    do_other_work                        # runs while :invoices and :shipments dispatch/run
+    merge_branches :invoices, :shipments # join both here
+
+    durably_execute :finalize
+  end
+end
+```
+
+### Caveats
+
+> **Every branch must be joined.** A branch opened and never joined raises
+> `ChronoForge::Executor::UnmergedBranchError` when the workflow tries to
+> complete — fail-fast, no silently-orphaned children. Use either
+> `automerge: true` or a matching `merge_branches` call.
+
+> **The parent isn't replayed while waiting.** A lightweight
+> `ChronoForge::BranchMergeJob` polls for child completion; the parent workflow
+> only runs again once the branch is fully done. Polling cadence tracks the
+> **estimated time-to-drain** (measured from the children's completion rate), so
+> the parent is woken within ~`min_interval` of the last child finishing rather
+> than up to `max_interval` late; a branch that can only wait or is blocked on a
+> failure backs off to `max_interval` (those need a wait to elapse or operator
+> recovery, not faster polling).
+>
+> **Queue placement matters.** The poller is enqueued *after* the branch's
+> children, so on a queue those children saturate it starves behind the backlog
+> (the parent then converges up to `max_interval` late). Point it at a queue that
+> isn't saturated by the fan-out:
+>
+> ```ruby
+> ChronoForge.configure { |c| c.branch_merge_queue = :chrono_forge_pollers } # default: :default
+> ```
+
+> **`spawn_each` sources must re-enumerate deterministically across replays.**
+> ActiveRecord relations are streamed by primary key (children are keyed by
+> record id, so crash-resume is idempotent); a relation carrying an explicit
+> `.order(...)` raises. For non-AR enumerables, items are keyed by position, so
+> inserting or removing items mid-dispatch would shift keys and break idempotency.
+
+> **`spawn_each` AR sources must have stable membership.** Dispatch streams by
+> ascending primary key and resumes from the last key on crash-recovery, so a row
+> that enters the relation *below* the cursor after it has passed (e.g. a
+> `where(state: …)` scope whose rows mutate mid-dispatch) will never get a child.
+> Point `spawn_each` at a set that is fixed for the branch's lifetime — a frozen id
+> range, an append-only table, or `where(id: [...])` over a snapshot.
+
+> **`branch` blocks cannot be lexically nested within one workflow.** Opening a
+> `branch` inside another `branch` block raises `ArgumentError`; spawns belong to
+> exactly one branch. (A *spawned child workflow* may open its own branches — it
+> runs in its own executor — so cross-workflow nesting is fine.)
+
+Verified correct at 500,000 children on a single Postgres instance. A follow-up commit-consolidation change halved per-child execution time. See the [scale test](docs/fanout-scale-test.md).
+
 ## 🧪 Testing
 
 ChronoForge is designed to be easily testable using [ChaoticJob](https://github.com/fractaledmind/chaotic_job), a testing framework that makes it simple to test complex job workflows:
@@ -756,6 +926,7 @@ stateDiagram-v2
 - **Identifiers**: Has locked_at and locked_by values set
 - **Behavior**: Protected against concurrent execution
 - **Duration**: Should be brief unless performing long operations
+- **Note**: A workflow whose worker was hard-killed mid-pass stays stuck here with a stale lock. See [Recovering stranded workflows](#recovering-stranded-workflows-hard-killed-workers).
 
 #### Completed
 - **Description**: The workflow has successfully executed all steps
@@ -804,25 +975,60 @@ ChronoForge::Workflow.failed.find_each(&:retry_later)
 The class-level form (`MyWorkflow.retry_now(key)` / `retry_later(key)`) still
 works if you have the class and key rather than the record.
 
-#### Monitoring Running Workflows
+#### Recovering stranded workflows (hard-killed workers)
+
+When a worker is **hard-killed** mid-pass — SIGKILL from a deploy/rollout, an OOM kill,
+a node eviction, or a SolidQueue heartbeat prune — Ruby's `ensure` block does not run.
+The executor releases the workflow's lock and publishes the resume continuation in that
+`ensure`, so a hard kill leaves the workflow stuck in `running` with a stale lock **and**
+nothing scheduled to wake it. It is fully resumable (a resuming pass steals the stale
+lock and replays completed steps as no-ops), but nothing re-enqueues it on its own:
+`retry_now`/`retry_later` refuse a `running` workflow, and a dashboard "force unlock"
+clears the lock but enqueues no job.
 
-Long-running workflows might indicate issues:
+`ChronoForge::Workflow.reap_stalled` reconciles these. It finds every workflow in
+`running` whose lock is older than `reap_stale_after` (top-level workflows **and** branch
+children) and re-enqueues it, returning the number reaped:
 
 ```ruby
-# Find workflows running for too long
-long_running = ChronoForge::Workflow.where(state: :running)
-                                   .where('locked_at < ?', 30.minutes.ago)
+ChronoForge::Workflow.reap_stalled
+# => 3   (also logs "ChronoForge reaped 3 stalled workflow(s)")
 
-long_running.each do |workflow|
-  # Log potential issues for investigation
-  Rails.logger.warn "Workflow #{workflow.key} has been running for >30 minutes"
-  
-  # Optionally force unlock if you suspect deadlock
-  # CAUTION: Only do this if you're certain the job is stuck
-  # workflow.update!(locked_at: nil, locked_by: nil, state: :idle)
+# Override the threshold for a one-off sweep:
+ChronoForge::Workflow.reap_stalled(stale_after: 15.minutes)
+```
+
+It is **not** run automatically — schedule it from your own scheduler, the same way you
+schedule `ChronoForge::Cleanup`. For example, a SolidQueue recurring task:
+
+```yaml
+# config/recurring.yml
+reap_stalled_workflows:
+  command: "ChronoForge::Workflow.reap_stalled"
+  schedule: "every 5 minutes"
+```
+
+Re-enqueue is safe under concurrency: overlapping sweeps (or a re-enqueue landing while
+the old stale lock still shows) at worst enqueue a duplicate, which loses the
+lock-acquisition race and no-ops. Because reaping replays the interrupted pass, steps
+with external side effects must be idempotent — see the note under
+[Durable Execution](#-durable-execution).
+
+`reap_stale_after` defaults to **3× `max_duration`** (30 minutes out of the box). Both are
+configurable, and because the reap threshold derives from `max_duration` it always stays
+safely above the lock-steal threshold — raise one and the other follows:
+
+```ruby
+ChronoForge.configure do |c|
+  c.max_duration    = 10.minutes  # how long one pass may hold its lock before it's stealable
+  c.reap_stale_after = 45.minutes # optional: pin the reap threshold explicitly (else 3x max_duration)
 end
 ```
 
+> **Not covered:** a workflow parked on a branch merge whose `BranchMergeJob` poller was
+> itself hard-killed sits `idle` (not `running`), so the reaper does not sweep it. That is
+> a distinct failure mode.
+
 ## 🧹 Cleanup & Retention
 
 ChronoForge keeps every workflow and execution-log row indefinitely so that
@@ -895,98 +1101,6 @@ production:
     schedule: every day at 3am
 ```
 
-## 🌿 Branches: parallel sub-workflows
-
-`branch` / `spawn` / `spawn_each` / `merge_branches` let a workflow fan out into
-child workflows that run concurrently, then join them when their results are
-needed.
-
-### Model
-
-- **`branch :name do … end`** opens a named branch (a durable step). Inside the
-  block, `spawn` and `spawn_each` create and immediately enqueue child workflows —
-  children start running as soon as the branch block is entered.
-- **`spawn :name, WorkflowClass, **kwargs`** — enqueues one child workflow.
-- **`spawn_each :name, source do |item| [WorkflowClass, kwargs] end`** — enqueues
-  one child per item. The block returns the class and kwargs, so one branch can
-  fan out into mixed workflow types. Sources are iterated in constant memory;
-  ActiveRecord relations are streamed by primary key — pass them **without** an
-  explicit `.order`.
-- **`automerge: true`** — joins the branch **inline at the block's close**.
-  Execution does not continue past the `branch` call until every child has
-  completed. Use it for "dispatch this group and wait right here."
-- **`merge_branches :a, :b`** (or the singular alias `merge_branch :a`) — the
-  separate join point. Open branches without `automerge`, do other work while the
-  children run, then join when you need their results. `merge_branches` blocks
-  until all named branches are complete.
-
-### Worked example
-
-```ruby
-class FulfillmentWorkflow < ApplicationJob
-  prepend ChronoForge::Executor
-
-  def perform(cycle_id:)
-    # automerge: the branch is joined inline, right where the block closes —
-    # `perform` does not continue past it until every child has completed.
-    branch :reconcile, automerge: true do
-      spawn :eu, ReconcileWorkflow, region: "EU"
-      spawn_each :orders, Order.pending do |order|
-        order.priority? ? [PriorityOrderWorkflow, { order_id: order.id }]
-                        : [OrderWorkflow, { order_id: order.id }]
-      end
-    end
-
-    # For branches you want to run concurrently and join later, omit automerge
-    # and use merge_branches:
-    branch :invoices do
-      spawn_each :unpaid, Invoice.unpaid do |inv|
-        [InvoiceWorkflow, { invoice_id: inv.id }]
-      end
-    end
-    branch :shipments do
-      spawn_each :ready, Shipment.ready do |s|
-        [ShipmentWorkflow, { shipment_id: s.id }]
-      end
-    end
-    do_other_work                        # runs while :invoices and :shipments dispatch/run
-    merge_branches :invoices, :shipments # join both here
-
-    durably_execute :finalize
-  end
-end
-```
-
-### Caveats
-
-> **Every branch must be joined.** A branch opened and never joined raises
-> `ChronoForge::Executor::UnmergedBranchError` when the workflow tries to
-> complete — fail-fast, no silently-orphaned children. Use either
-> `automerge: true` or a matching `merge_branches` call.
-
-> **The parent isn't replayed while waiting.** A lightweight
-> `ChronoForge::BranchMergeJob` polls for child completion; the parent workflow
-> only runs again once the branch is fully done. Polling cadence adapts to how
-> many children remain.
-
-> **`spawn_each` sources must re-enumerate deterministically across replays.**
-> ActiveRecord relations are streamed by primary key (children are keyed by
-> record id, so crash-resume is idempotent); a relation carrying an explicit
-> `.order(...)` raises. For non-AR enumerables, items are keyed by position, so
-> inserting or removing items mid-dispatch would shift keys and break idempotency.
-
-> **`spawn_each` AR sources must have stable membership.** Dispatch streams by
-> ascending primary key and resumes from the last key on crash-recovery, so a row
-> that enters the relation *below* the cursor after it has passed (e.g. a
-> `where(state: …)` scope whose rows mutate mid-dispatch) will never get a child.
-> Point `spawn_each` at a set that is fixed for the branch's lifetime — a frozen id
-> range, an append-only table, or `where(id: [...])` over a snapshot.
-
-> **`branch` blocks cannot be lexically nested within one workflow.** Opening a
-> `branch` inside another `branch` block raises `ArgumentError`; spawns belong to
-> exactly one branch. (A *spawned child workflow* may open its own branches — it
-> runs in its own executor — so cross-workflow nesting is fine.)
-
 ## 🚀 Development
 
 After checking out the repo, run:
@@ -1029,6 +1143,17 @@ This gem is available as open source under the terms of the [MIT License](https:
 | `continue_if` | Manual continuation wait | `condition`, `name: nil` |
 | `durably_repeat` | Periodic task execution | `method`, `every:`, `till:`, `start_at: nil`, `retry_policy: nil`, `timeout: 1.hour`, `on_error: :continue` |
 
+### Branch Methods
+
+Fan a workflow out into parallel child sub-workflows (see [Branches](#-branches-parallel-sub-workflows)).
+
+| Method | Purpose | Key Parameters |
+|--------|---------|----------------|
+| `branch` | Open a named branch (takes a block) to dispatch children | `name`, `automerge: false` |
+| `spawn` | Enqueue one child workflow inside a branch | `name`, `workflow_class`, `**kwargs` |
+| `spawn_each` | Enqueue one child per item, streamed (block returns `[WorkflowClass, kwargs]`) | `name`, `source`, `of: 1000` |
+| `merge_branches` | Join named branches; blocks until all complete (alias `merge_branch`) | `*names`, `min_interval: 5.seconds`, `max_interval: 5.minutes` |
+
 ### Context Methods
 
 | Method | Purpose | Example |
@@ -1037,6 +1162,8 @@ This gem is available as open source under the terms of the [MIT License](https:
 | `context[:key]` | Get context value | `user_id = context[:user_id]` |
 | `context.set(key, value)` | Set context value (alias) | `context.set(:status, "active")` |
 | `context.set_once(key, value)` | Set only if key doesn't exist | `context.set_once(:created_at, Time.current)` |
+| `context.merge(hash)` | Set multiple values atomically (alias: `set_multiple`) | `context.merge(status: "active", count: 0)` |
+| `context.merge_once(hash)` | Set multiple values, skipping existing keys (alias: `set_multiple_once`) | `context.merge_once(created_at: Time.current, count: 0)` |
 | `context.fetch(key, default)` | Get with default value | `context.fetch(:count, 0)` |
 | `context.key?(key)` | Check if key exists | `context.key?(:user_id)` |
 

data/Rakefile

@@ -12,3 +12,7 @@ Rake::TestTask.new do |t|
   t.test_files = FileList["test/**/*_test.rb"]
   t.verbose = true
 end
+
+# Release tasks (release:core:*, release:dashboard:*). Loaded after
+# bundler/gem_tasks so it can neutralize the bare `rake release` footgun.
+Dir.glob(File.expand_path("lib/tasks/*.rake", __dir__)).sort.each { |f| load f }

data/cliff.toml

@@ -0,0 +1,62 @@
+# git-cliff configuration for the chrono_forge monorepo.
+# https://git-cliff.org/docs/configuration
+#
+# Two gems share this one config. bin/release passes per-gem --tag-pattern and
+# --include-path/--exclude-path so each CHANGELOG only reflects its own subtree
+# (core = everything except chrono_forge-dashboard/, dashboard = that subtree).
+# The version heading trims both tag prefixes (`v` and `chrono_forge-dashboard-v`).
+
+[changelog]
+header = """
+# Changelog\n
+All notable changes to this project are documented in this file.\n
+"""
+body = """
+{% if version %}\
+    ## [{{ version | trim_start_matches(pat="chrono_forge-dashboard-v") | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}\
+    ## [Unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | upper_first }}
+    {% for commit in commits %}
+        - {% if commit.breaking %}[**breaking**] {% endif %}{{ commit.message | upper_first }}\
+    {% endfor %}
+{% endfor %}\n
+"""
+trim = true
+footer = """
+<!-- generated by git-cliff -->
+"""
+
+[git]
+conventional_commits = true
+filter_unconventional = true
+split_commits = false
+commit_preprocessors = [
+  { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](https://github.com/radioactive-labs/chrono_forge/issues/${2}))" },
+]
+commit_parsers = [
+  { message = "^feat", group = "Features" },
+  { message = "^fix", group = "Bug Fixes" },
+  { message = "^doc", group = "Documentation" },
+  { message = "^perf", group = "Performance" },
+  { message = "^refactor", group = "Refactoring" },
+  { message = "^style", group = "Styling" },
+  { message = "^test", group = "Testing" },
+  { message = "^chore\\(release\\):", skip = true },
+  { message = "^chore", group = "Miscellaneous Tasks" },
+  { body = ".*security", group = "Security" },
+]
+protect_breaking_commits = false
+filter_commits = false
+tag_pattern = "^v[0-9]"
+topo_order = false
+sort_commits = "oldest"
+
+[bump]
+# Pre-1.0 semver: both gems are on 0.x. A feature bumps the minor, and a
+# breaking change ALSO bumps the minor (not the major) while we're on 0.x.
+# Fixes bump the patch. Revisit once a gem reaches 1.0.
+features_always_bump_minor = true
+breaking_always_bump_major = false