chrono_forge 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +34 -1
- data/README.md +188 -105
- data/Rakefile +4 -0
- data/cliff.toml +62 -0
- data/docs/design/per-child-commit-overhead.md +213 -0
- data/docs/fanout-scale-test.md +247 -0
- data/docs/superpowers/plans/2026-06-30-poller-rekick-and-eta-cadence.md +205 -0
- data/docs/superpowers/plans/2026-06-30-poller-rekick-and-eta-cadence.md.tasks.json +33 -0
- data/docs/superpowers/plans/2026-07-01-workflow-definition-dag.md +1373 -0
- data/docs/superpowers/plans/2026-07-01-workflow-definition-dag.md.tasks.json +68 -0
- data/docs/superpowers/specs/2026-07-01-workflow-definition-dag-design.md +203 -0
- data/lib/chrono_forge/branch_merge_job.rb +158 -21
- data/lib/chrono_forge/branch_probe.rb +44 -0
- data/lib/chrono_forge/configuration.rb +25 -0
- data/lib/chrono_forge/definition.rb +37 -0
- data/lib/chrono_forge/definition_analyzer.rb +501 -0
- data/lib/chrono_forge/executor/context.rb +23 -0
- data/lib/chrono_forge/executor/lock_strategy.rb +10 -3
- data/lib/chrono_forge/executor/methods/continue_if.rb +15 -6
- data/lib/chrono_forge/executor/methods/durably_execute.rb +15 -7
- data/lib/chrono_forge/executor/methods/durably_repeat.rb +30 -14
- data/lib/chrono_forge/executor/methods/merge_branches.rb +5 -4
- data/lib/chrono_forge/executor/methods/workflow_states.rb +35 -47
- data/lib/chrono_forge/executor.rb +34 -9
- data/lib/chrono_forge/version.rb +1 -1
- data/lib/chrono_forge.rb +8 -0
- data/lib/tasks/release.rake +212 -0
- metadata +28 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c2b3299dbce8cb34289b708af6e0d22d19d764de09978ae257cb87fd37322b5b
|
|
4
|
+
data.tar.gz: f8ca2ae0af221d262d8109b06bd4c319a679d14ea0b440404ef96997134f8b17
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c0e81af8116529cbcb1b13fc2f23c952c3622b13f615281f7f97c82ce41cda244316bbb77a99666c4709e8211747d931bdf31d7b02cad011119db8b49b72714d
|
|
7
|
+
data.tar.gz: 427469cfe3d72bd2891d156227be9a32a1879cfa1dc8f52c8c54f87bf99feeabc1b8d9dfca46fb37e58bae6cb84f560f77fca94888dcfcbd32fededfc70bd1f1
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,37 @@
|
|
|
1
|
-
## [
|
|
1
|
+
## [0.11.0] - 2026-07-04
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
- Back off BranchMergeJob polling for non-progressing children
|
|
6
|
+
- Leave prepare uncommitted for review; publish commits
|
|
7
|
+
- Converge merges promptly via drain-ETA cadence and never-started-count rekick ([#12](https://github.com/radioactive-labs/chrono_forge/issues/12))
|
|
8
|
+
- Overlay/analyzer correctness, detail-panel XSS, and UX polish
|
|
9
|
+
|
|
10
|
+
### Documentation
|
|
11
|
+
|
|
12
|
+
- Refresh dashboard screenshots, badges; fix upgrade note and API reference
|
|
13
|
+
- Restructure READMEs — promote branches, refine cadence note, dashboard cross-link
|
|
14
|
+
- Add ActiveJob Continuations, Rails-version, and dashboard rows to comparison
|
|
15
|
+
- Correct "workflow is just a Ruby method" framing
|
|
16
|
+
- Add ChronoForge equivalent of the step DSL; fix step-vs-workflow framing
|
|
17
|
+
|
|
18
|
+
### Features
|
|
19
|
+
|
|
20
|
+
- Add bulk merge / merge_once for context ([#11](https://github.com/radioactive-labs/chrono_forge/issues/11))
|
|
21
|
+
- Workflow definition graph with static DAG and live run overlay ([#13](https://github.com/radioactive-labs/chrono_forge/issues/13))
|
|
22
|
+
|
|
23
|
+
### Miscellaneous Tasks
|
|
24
|
+
|
|
25
|
+
- Add cliff-driven release script for both gems
|
|
26
|
+
- Replace bin/release with per-gem rake release flow
|
|
27
|
+
|
|
28
|
+
### Performance
|
|
29
|
+
|
|
30
|
+
- Consolidate per-child commits and flatten log writes
|
|
31
|
+
|
|
32
|
+
### Styling
|
|
33
|
+
|
|
34
|
+
- Apply standardrb blank-line formatting to existing files
|
|
2
35
|
|
|
3
36
|
## [0.10.0] - 2026-06-27
|
|
4
37
|
|
data/README.md
CHANGED
|
@@ -11,6 +11,7 @@ ChronoForge handles long-running processes, manages state, and recovers from fai
|
|
|
11
11
|
|
|
12
12
|
Workflows are **plain Ruby**. Ordinary `if`/`else`, loops, and early returns drive the flow. There's no declarative DSL to learn and no extra service to run, which makes ChronoForge a good fit for business processes whose shape depends on runtime state: conditional branches, iteration over data, and built-in periodic tasks (`durably_repeat`).
|
|
13
13
|
|
|
14
|
+
> [!NOTE]
|
|
14
15
|
> **In production** at **achieve by Petra**, an investment platform in the Petra Group — where it has executed over 3.6 million workflows and 32 million durable steps across scheduled payments, investment rollovers, and membership lifecycle management.
|
|
15
16
|
|
|
16
17
|
## 🧭 Why ChronoForge
|
|
@@ -23,19 +24,44 @@ step :remind_of_tasks, wait: 2.days
|
|
|
23
24
|
step :complete_onboarding, wait: 15.days
|
|
24
25
|
```
|
|
25
26
|
|
|
26
|
-
That reads cleanly for a fixed, linear sequence. But many business processes branch, loop, and react to data that only exists at runtime, and a declarative schema gets awkward there. ChronoForge takes the opposite approach: **
|
|
27
|
+
That reads cleanly for a fixed, linear sequence. But many business processes branch, loop, and react to data that only exists at runtime, and a declarative schema gets awkward there. ChronoForge takes the opposite approach: **the workflow is plain Ruby, and each step is just a method.** Conditionals, iteration, early returns, and helper methods all work the way they normally do; you drive durable steps, waits, and retries inline with a few primitives like `durably_execute` and `wait_until`:
|
|
27
28
|
|
|
28
|
-
|
|
29
|
+
```ruby
|
|
30
|
+
class OnboardingWorkflow < ApplicationJob
|
|
31
|
+
prepend ChronoForge::Executor
|
|
32
|
+
|
|
33
|
+
def perform(user_id:)
|
|
34
|
+
@user_id = user_id
|
|
35
|
+
|
|
36
|
+
durably_execute :send_welcome_email
|
|
37
|
+
wait 2.days, :remind_delay
|
|
38
|
+
durably_execute :remind_of_tasks
|
|
39
|
+
wait 15.days, :onboarding_delay
|
|
40
|
+
durably_execute :complete_onboarding
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def send_welcome_email = UserMailer.welcome(@user_id).deliver_now
|
|
46
|
+
def remind_of_tasks = UserMailer.task_reminder(@user_id).deliver_now
|
|
47
|
+
def complete_onboarding = User.find(@user_id).complete_onboarding!
|
|
48
|
+
end
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
There is a trade-off, though a smaller one than it used to be. A declarative engine can show the steps a run *hasn't* reached yet; plain imperative code can't. The [definition graph](#-dashboard) closes most of that gap. It reads your `perform` with Prism (without running it) and draws the steps the run will take, with live status on each. The view is best-effort: a computed step name or a data-dependent loop collapses to one `dynamic` node. For a simple, fixed sequence ("send email, wait 2 days, send another") a declarative DSL can still read more cleanly, and that's a fine reason to use one.
|
|
29
52
|
|
|
30
53
|
### How it compares
|
|
31
54
|
|
|
32
|
-
| | ChronoForge | GenevaDrive | AcidicJob | Temporal |
|
|
33
|
-
| ---------------------------- | -------------------- | ------------------ | --------------- | --------------- |
|
|
34
|
-
| Programming model | procedural (plain Ruby) | declarative DSL | declarative DSL | procedural (via SDK) |
|
|
35
|
-
| Built-in periodic tasks | ✓ `durably_repeat` | ✗ | ✗ | ✓ |
|
|
36
|
-
|
|
|
37
|
-
|
|
|
38
|
-
|
|
|
55
|
+
| | ChronoForge | AJ Continuations | GenevaDrive | AcidicJob | Temporal |
|
|
56
|
+
| ---------------------------- | -------------------- | -------------------------- | ------------------ | --------------- | --------------- |
|
|
57
|
+
| Programming model | procedural (plain Ruby) | procedural (`step` blocks) | declarative DSL | declarative DSL | procedural (via SDK) |
|
|
58
|
+
| Built-in periodic tasks | ✓ `durably_repeat` | ✗ | ✗ | ✗ | ✓ |
|
|
59
|
+
| Parallel sub-workflows | ✓ `branch` / `spawn` | ✗ | ✗ | ✗ | ✓ |
|
|
60
|
+
| Pending-step visibility | ✗ (procedural) | ✗ (procedural) | ✓ | ✓ | ✗ (procedural) |
|
|
61
|
+
| Web dashboard | ✓ (free gem) | job-level (Mission Control)| paid only | ✗ | ✓ |
|
|
62
|
+
| Extra infrastructure | none (DB + ActiveJob)| none (built into Rails) | none | none | server required |
|
|
63
|
+
| Rails support | 7.1+ | 8.1+ | 7.2+ | 7.1+ | any (Ruby SDK) |
|
|
64
|
+
| License | MIT | MIT | LGPL / commercial | MIT | MIT |
|
|
39
65
|
|
|
40
66
|
<sub>Comparison reflects each project's documented features as of mid-2026, to the best of our knowledge; corrections welcome via PR.</sub>
|
|
41
67
|
|
|
@@ -43,8 +69,11 @@ A few deliberate choices behind that table:
|
|
|
43
69
|
|
|
44
70
|
- **Periodic tasks are built in.** `durably_repeat` runs a step on a schedule until a condition holds, with automatic catch-up for missed runs, so a workflow can be its own recurring job and cron-style monitor, right alongside the rest of its logic. Without built-in support, periodic behavior usually lives in a separate scheduler that you reconcile with workflow state by hand.
|
|
45
71
|
- **No extra infrastructure.** ChronoForge is a gem over your existing database and ActiveJob backend. There's no separate server or daemon to operate, unlike Temporal.
|
|
72
|
+
- **Large-scale fan-out is built in.** `branch` with `spawn`/`spawn_each` fans a workflow out into child workflows that run concurrently and join when their results are needed, streaming ActiveRecord relations in constant memory for large sets. Among the Ruby-native engines here, only ChronoForge offers this without a separate orchestration server (Temporal does, server-side).
|
|
46
73
|
- **Recovery is built into the model.** Steps are append-only history, so a crashed step leaves the workflow `stalled`, recoverable directly with `retry_later`.
|
|
74
|
+
- **A real dashboard, free.** ChronoForge's [mountable dashboard](#-dashboard) — workflow list, step-replay timeline, a per-run **definition graph** (the steps a run will take, read from `perform`, with live status on each), context inspector, retry/unlock — ships as a separate MIT gem.
|
|
47
75
|
- **MIT licensed.** Permissive and dependency-policy-friendly.
|
|
76
|
+
- **ActiveJob Continuations solve a narrower problem.** Rails 8.1's built-in [continuations](https://api.rubyonrails.org/classes/ActiveJob/Continuation.html) make a *single* long job survive interruptions: you wrap work in `step` blocks and track a `cursor`, and at each checkpoint the job asks the queue adapter whether it's `stopping?`, re-enqueuing to resume from the last completed step/cursor — no gem, no tables. They deliberately stop short of being a workflow engine: there's no durable waiting on time, conditions, or external events; no periodic steps; no parallel fan-out; and no persisted, queryable history. Reach for continuations to make one big job restart-safe; reach for ChronoForge when a process spans steps that wait, recur, fan out, and need recovery and visibility. They also compose — a ChronoForge workflow *is* ActiveJob work.
|
|
48
77
|
|
|
49
78
|
## 🌟 Features
|
|
50
79
|
|
|
@@ -52,6 +81,7 @@ A few deliberate choices behind that table:
|
|
|
52
81
|
- **Durable Execution**: Automatically tracks and recovers from failures during workflow execution
|
|
53
82
|
- **Periodic tasks built in**: `durably_repeat` runs a step on an interval until a condition is met, with catch-up for missed runs. Acts as a recurring task and a cron-style monitor in one
|
|
54
83
|
- **Wait States**: Time-based waits and condition-based waiting (`wait_until`) that survive restarts
|
|
84
|
+
- **Parallel sub-workflows**: `branch` with `spawn`/`spawn_each` fans out into concurrent child workflows and joins them (`automerge` or `merge_branches`); large sets stream in constant memory. See [Branches](#-branches-parallel-sub-workflows)
|
|
55
85
|
- **State Management**: Built-in workflow state tracking with persistent context storage
|
|
56
86
|
- **Concurrency Control**: Advanced locking mechanisms to prevent parallel execution of the same workflow
|
|
57
87
|
- **Error Handling**: Error tracking with a unified, configurable [`RetryPolicy`](#-retry-policies) (including per-error-type policies)
|
|
@@ -62,10 +92,14 @@ A few deliberate choices behind that table:
|
|
|
62
92
|
|
|
63
93
|
## 🖥️ Dashboard
|
|
64
94
|
|
|
65
|
-
ChronoForge has a free, mountable dashboard for visibility and recovery: workflow list, step replay timeline, context inspector, periodic-task health, wait-state age, and retry/unlock actions. It ships as a separate gem, `chrono_forge-dashboard`, so the core stays lean.
|
|
95
|
+
ChronoForge has a free, mountable dashboard for visibility and recovery: workflow list, step replay timeline, a per-run **definition graph** (the durable steps a workflow will run, statically parsed from `perform`, with live run status overlaid), context inspector, periodic-task health, wait-state age, and retry/unlock actions. It ships as a separate gem, `chrono_forge-dashboard`, so the core stays lean.
|
|
66
96
|
|
|
67
97
|
[](chrono_forge-dashboard/README.md#screenshots)
|
|
68
98
|
|
|
99
|
+
The per-run **definition graph**: the steps a workflow will run, read from `perform`, with the run's status shown on each node.
|
|
100
|
+
|
|
101
|
+
[](chrono_forge-dashboard/README.md#screenshots)
|
|
102
|
+
|
|
69
103
|
```ruby
|
|
70
104
|
# Gemfile
|
|
71
105
|
gem "chrono_forge-dashboard"
|
|
@@ -114,9 +148,9 @@ $ rails generate chrono_forge:upgrade
|
|
|
114
148
|
$ rails db:migrate
|
|
115
149
|
```
|
|
116
150
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
151
|
+
Re-running the generator is safe — it skips any migrations you already have.
|
|
152
|
+
Fresh installs get everything from `chrono_forge:install` and don't need the
|
|
153
|
+
upgrade.
|
|
120
154
|
|
|
121
155
|
## 📋 Usage
|
|
122
156
|
|
|
@@ -590,6 +624,21 @@ def cleanup_files(scheduled_time)
|
|
|
590
624
|
end
|
|
591
625
|
```
|
|
592
626
|
|
|
627
|
+
#### 🌿 Branches
|
|
628
|
+
|
|
629
|
+
When a workflow needs to fan out — process every pending order, reconcile each region — `branch` spawns child workflows that run concurrently and join when their results are needed:
|
|
630
|
+
|
|
631
|
+
```ruby
|
|
632
|
+
branch :reconcile, automerge: true do
|
|
633
|
+
spawn :eu, ReconcileWorkflow, region: "EU"
|
|
634
|
+
spawn_each :orders, Order.pending do |order|
|
|
635
|
+
[OrderWorkflow, { order_id: order.id }]
|
|
636
|
+
end
|
|
637
|
+
end
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
`spawn_each` streams ActiveRecord relations by primary key in constant memory, so a branch can fan out over very large sets. Join inline with `automerge: true`, or open branches without it and `merge_branches` them later after doing other work. See [Branches: parallel sub-workflows](#-branches-parallel-sub-workflows) for the full model, a worked example, and the crash-recovery caveats.
|
|
641
|
+
|
|
593
642
|
#### 🔄 Workflow Context
|
|
594
643
|
|
|
595
644
|
ChronoForge provides a persistent context that survives job restarts. The context behaves like a Hash but with additional capabilities:
|
|
@@ -611,6 +660,12 @@ context.set(:total_amount, 99.99)
|
|
|
611
660
|
# Set a value only if the key doesn't already exist
|
|
612
661
|
context.set_once(:created_at, Time.current.iso8601)
|
|
613
662
|
|
|
663
|
+
# Set several values at once (alias: set_multiple)
|
|
664
|
+
context.merge(status: "processing", total_amount: 99.99, attempts: 0)
|
|
665
|
+
|
|
666
|
+
# Set several values at once, but only keys that don't already exist (alias: set_multiple_once)
|
|
667
|
+
context.merge_once(created_at: Time.current.iso8601, attempts: 0)
|
|
668
|
+
|
|
614
669
|
# Check if a key exists
|
|
615
670
|
if context.key?(:user_id)
|
|
616
671
|
# Do something with the user ID
|
|
@@ -652,6 +707,113 @@ end
|
|
|
652
707
|
|
|
653
708
|
To make an error non-retryable, leave it out of `retry_on:` (an empty `retry_on: []` retries nothing).
|
|
654
709
|
|
|
710
|
+
## 🌿 Branches: parallel sub-workflows
|
|
711
|
+
|
|
712
|
+
`branch` / `spawn` / `spawn_each` / `merge_branches` let a workflow fan out into
|
|
713
|
+
child workflows that run concurrently, then join them when their results are
|
|
714
|
+
needed.
|
|
715
|
+
|
|
716
|
+
### Model
|
|
717
|
+
|
|
718
|
+
- **`branch :name do … end`** opens a named branch (a durable step). Inside the
|
|
719
|
+
block, `spawn` and `spawn_each` create and immediately enqueue child workflows —
|
|
720
|
+
children start running as soon as the branch block is entered.
|
|
721
|
+
- **`spawn :name, WorkflowClass, **kwargs`** — enqueues one child workflow.
|
|
722
|
+
- **`spawn_each :name, source do |item| [WorkflowClass, kwargs] end`** — enqueues
|
|
723
|
+
one child per item. The block returns the class and kwargs, so one branch can
|
|
724
|
+
fan out into mixed workflow types. Sources are iterated in constant memory;
|
|
725
|
+
ActiveRecord relations are streamed by primary key — pass them **without** an
|
|
726
|
+
explicit `.order`.
|
|
727
|
+
- **`automerge: true`** — joins the branch **inline at the block's close**.
|
|
728
|
+
Execution does not continue past the `branch` call until every child has
|
|
729
|
+
completed. Use it for "dispatch this group and wait right here."
|
|
730
|
+
- **`merge_branches :a, :b`** (or the singular alias `merge_branch :a`) — the
|
|
731
|
+
separate join point. Open branches without `automerge`, do other work while the
|
|
732
|
+
children run, then join when you need their results. `merge_branches` blocks
|
|
733
|
+
until all named branches are complete.
|
|
734
|
+
|
|
735
|
+
### Worked example
|
|
736
|
+
|
|
737
|
+
```ruby
|
|
738
|
+
class FulfillmentWorkflow < ApplicationJob
|
|
739
|
+
prepend ChronoForge::Executor
|
|
740
|
+
|
|
741
|
+
def perform(cycle_id:)
|
|
742
|
+
# automerge: the branch is joined inline, right where the block closes —
|
|
743
|
+
# `perform` does not continue past it until every child has completed.
|
|
744
|
+
branch :reconcile, automerge: true do
|
|
745
|
+
spawn :eu, ReconcileWorkflow, region: "EU"
|
|
746
|
+
spawn_each :orders, Order.pending do |order|
|
|
747
|
+
order.priority? ? [PriorityOrderWorkflow, { order_id: order.id }]
|
|
748
|
+
: [OrderWorkflow, { order_id: order.id }]
|
|
749
|
+
end
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
# For branches you want to run concurrently and join later, omit automerge
|
|
753
|
+
# and use merge_branches:
|
|
754
|
+
branch :invoices do
|
|
755
|
+
spawn_each :unpaid, Invoice.unpaid do |inv|
|
|
756
|
+
[InvoiceWorkflow, { invoice_id: inv.id }]
|
|
757
|
+
end
|
|
758
|
+
end
|
|
759
|
+
branch :shipments do
|
|
760
|
+
spawn_each :ready, Shipment.ready do |s|
|
|
761
|
+
[ShipmentWorkflow, { shipment_id: s.id }]
|
|
762
|
+
end
|
|
763
|
+
end
|
|
764
|
+
do_other_work # runs while :invoices and :shipments dispatch/run
|
|
765
|
+
merge_branches :invoices, :shipments # join both here
|
|
766
|
+
|
|
767
|
+
durably_execute :finalize
|
|
768
|
+
end
|
|
769
|
+
end
|
|
770
|
+
```
|
|
771
|
+
|
|
772
|
+
### Caveats
|
|
773
|
+
|
|
774
|
+
> **Every branch must be joined.** A branch opened and never joined raises
|
|
775
|
+
> `ChronoForge::Executor::UnmergedBranchError` when the workflow tries to
|
|
776
|
+
> complete — fail-fast, no silently-orphaned children. Use either
|
|
777
|
+
> `automerge: true` or a matching `merge_branches` call.
|
|
778
|
+
|
|
779
|
+
> **The parent isn't replayed while waiting.** A lightweight
|
|
780
|
+
> `ChronoForge::BranchMergeJob` polls for child completion; the parent workflow
|
|
781
|
+
> only runs again once the branch is fully done. Polling cadence tracks the
|
|
782
|
+
> **estimated time-to-drain** (measured from the children's completion rate), so
|
|
783
|
+
> the parent is woken within ~`min_interval` of the last child finishing rather
|
|
784
|
+
> than up to `max_interval` late; a branch that can only wait or is blocked on a
|
|
785
|
+
> failure backs off to `max_interval` (those need a wait to elapse or operator
|
|
786
|
+
> recovery, not faster polling).
|
|
787
|
+
>
|
|
788
|
+
> **Queue placement matters.** The poller is enqueued *after* the branch's
|
|
789
|
+
> children, so on a queue those children saturate it starves behind the backlog
|
|
790
|
+
> (the parent then converges up to `max_interval` late). Point it at a queue that
|
|
791
|
+
> isn't saturated by the fan-out:
|
|
792
|
+
>
|
|
793
|
+
> ```ruby
|
|
794
|
+
> ChronoForge.configure { |c| c.branch_merge_queue = :chrono_forge_pollers } # default: :default
|
|
795
|
+
> ```
|
|
796
|
+
|
|
797
|
+
> **`spawn_each` sources must re-enumerate deterministically across replays.**
|
|
798
|
+
> ActiveRecord relations are streamed by primary key (children are keyed by
|
|
799
|
+
> record id, so crash-resume is idempotent); a relation carrying an explicit
|
|
800
|
+
> `.order(...)` raises. For non-AR enumerables, items are keyed by position, so
|
|
801
|
+
> inserting or removing items mid-dispatch would shift keys and break idempotency.
|
|
802
|
+
|
|
803
|
+
> **`spawn_each` AR sources must have stable membership.** Dispatch streams by
|
|
804
|
+
> ascending primary key and resumes from the last key on crash-recovery, so a row
|
|
805
|
+
> that enters the relation *below* the cursor after it has passed (e.g. a
|
|
806
|
+
> `where(state: …)` scope whose rows mutate mid-dispatch) will never get a child.
|
|
807
|
+
> Point `spawn_each` at a set that is fixed for the branch's lifetime — a frozen id
|
|
808
|
+
> range, an append-only table, or `where(id: [...])` over a snapshot.
|
|
809
|
+
|
|
810
|
+
> **`branch` blocks cannot be lexically nested within one workflow.** Opening a
|
|
811
|
+
> `branch` inside another `branch` block raises `ArgumentError`; spawns belong to
|
|
812
|
+
> exactly one branch. (A *spawned child workflow* may open its own branches — it
|
|
813
|
+
> runs in its own executor — so cross-workflow nesting is fine.)
|
|
814
|
+
|
|
815
|
+
Verified correct at 500,000 children on a single Postgres instance. A follow-up commit-consolidation change halved per-child execution time. See the [scale test](docs/fanout-scale-test.md).
|
|
816
|
+
|
|
655
817
|
## 🧪 Testing
|
|
656
818
|
|
|
657
819
|
ChronoForge is designed to be easily testable using [ChaoticJob](https://github.com/fractaledmind/chaotic_job), a testing framework that makes it simple to test complex job workflows:
|
|
@@ -895,98 +1057,6 @@ production:
|
|
|
895
1057
|
schedule: every day at 3am
|
|
896
1058
|
```
|
|
897
1059
|
|
|
898
|
-
## 🌿 Branches: parallel sub-workflows
|
|
899
|
-
|
|
900
|
-
`branch` / `spawn` / `spawn_each` / `merge_branches` let a workflow fan out into
|
|
901
|
-
child workflows that run concurrently, then join them when their results are
|
|
902
|
-
needed.
|
|
903
|
-
|
|
904
|
-
### Model
|
|
905
|
-
|
|
906
|
-
- **`branch :name do … end`** opens a named branch (a durable step). Inside the
|
|
907
|
-
block, `spawn` and `spawn_each` create and immediately enqueue child workflows —
|
|
908
|
-
children start running as soon as the branch block is entered.
|
|
909
|
-
- **`spawn :name, WorkflowClass, **kwargs`** — enqueues one child workflow.
|
|
910
|
-
- **`spawn_each :name, source do |item| [WorkflowClass, kwargs] end`** — enqueues
|
|
911
|
-
one child per item. The block returns the class and kwargs, so one branch can
|
|
912
|
-
fan out into mixed workflow types. Sources are iterated in constant memory;
|
|
913
|
-
ActiveRecord relations are streamed by primary key — pass them **without** an
|
|
914
|
-
explicit `.order`.
|
|
915
|
-
- **`automerge: true`** — joins the branch **inline at the block's close**.
|
|
916
|
-
Execution does not continue past the `branch` call until every child has
|
|
917
|
-
completed. Use it for "dispatch this group and wait right here."
|
|
918
|
-
- **`merge_branches :a, :b`** (or the singular alias `merge_branch :a`) — the
|
|
919
|
-
separate join point. Open branches without `automerge`, do other work while the
|
|
920
|
-
children run, then join when you need their results. `merge_branches` blocks
|
|
921
|
-
until all named branches are complete.
|
|
922
|
-
|
|
923
|
-
### Worked example
|
|
924
|
-
|
|
925
|
-
```ruby
|
|
926
|
-
class FulfillmentWorkflow < ApplicationJob
|
|
927
|
-
prepend ChronoForge::Executor
|
|
928
|
-
|
|
929
|
-
def perform(cycle_id:)
|
|
930
|
-
# automerge: the branch is joined inline, right where the block closes —
|
|
931
|
-
# `perform` does not continue past it until every child has completed.
|
|
932
|
-
branch :reconcile, automerge: true do
|
|
933
|
-
spawn :eu, ReconcileWorkflow, region: "EU"
|
|
934
|
-
spawn_each :orders, Order.pending do |order|
|
|
935
|
-
order.priority? ? [PriorityOrderWorkflow, { order_id: order.id }]
|
|
936
|
-
: [OrderWorkflow, { order_id: order.id }]
|
|
937
|
-
end
|
|
938
|
-
end
|
|
939
|
-
|
|
940
|
-
# For branches you want to run concurrently and join later, omit automerge
|
|
941
|
-
# and use merge_branches:
|
|
942
|
-
branch :invoices do
|
|
943
|
-
spawn_each :unpaid, Invoice.unpaid do |inv|
|
|
944
|
-
[InvoiceWorkflow, { invoice_id: inv.id }]
|
|
945
|
-
end
|
|
946
|
-
end
|
|
947
|
-
branch :shipments do
|
|
948
|
-
spawn_each :ready, Shipment.ready do |s|
|
|
949
|
-
[ShipmentWorkflow, { shipment_id: s.id }]
|
|
950
|
-
end
|
|
951
|
-
end
|
|
952
|
-
do_other_work # runs while :invoices and :shipments dispatch/run
|
|
953
|
-
merge_branches :invoices, :shipments # join both here
|
|
954
|
-
|
|
955
|
-
durably_execute :finalize
|
|
956
|
-
end
|
|
957
|
-
end
|
|
958
|
-
```
|
|
959
|
-
|
|
960
|
-
### Caveats
|
|
961
|
-
|
|
962
|
-
> **Every branch must be joined.** A branch opened and never joined raises
|
|
963
|
-
> `ChronoForge::Executor::UnmergedBranchError` when the workflow tries to
|
|
964
|
-
> complete — fail-fast, no silently-orphaned children. Use either
|
|
965
|
-
> `automerge: true` or a matching `merge_branches` call.
|
|
966
|
-
|
|
967
|
-
> **The parent isn't replayed while waiting.** A lightweight
|
|
968
|
-
> `ChronoForge::BranchMergeJob` polls for child completion; the parent workflow
|
|
969
|
-
> only runs again once the branch is fully done. Polling cadence adapts to how
|
|
970
|
-
> many children remain.
|
|
971
|
-
|
|
972
|
-
> **`spawn_each` sources must re-enumerate deterministically across replays.**
|
|
973
|
-
> ActiveRecord relations are streamed by primary key (children are keyed by
|
|
974
|
-
> record id, so crash-resume is idempotent); a relation carrying an explicit
|
|
975
|
-
> `.order(...)` raises. For non-AR enumerables, items are keyed by position, so
|
|
976
|
-
> inserting or removing items mid-dispatch would shift keys and break idempotency.
|
|
977
|
-
|
|
978
|
-
> **`spawn_each` AR sources must have stable membership.** Dispatch streams by
|
|
979
|
-
> ascending primary key and resumes from the last key on crash-recovery, so a row
|
|
980
|
-
> that enters the relation *below* the cursor after it has passed (e.g. a
|
|
981
|
-
> `where(state: …)` scope whose rows mutate mid-dispatch) will never get a child.
|
|
982
|
-
> Point `spawn_each` at a set that is fixed for the branch's lifetime — a frozen id
|
|
983
|
-
> range, an append-only table, or `where(id: [...])` over a snapshot.
|
|
984
|
-
|
|
985
|
-
> **`branch` blocks cannot be lexically nested within one workflow.** Opening a
|
|
986
|
-
> `branch` inside another `branch` block raises `ArgumentError`; spawns belong to
|
|
987
|
-
> exactly one branch. (A *spawned child workflow* may open its own branches — it
|
|
988
|
-
> runs in its own executor — so cross-workflow nesting is fine.)
|
|
989
|
-
|
|
990
1060
|
## 🚀 Development
|
|
991
1061
|
|
|
992
1062
|
After checking out the repo, run:
|
|
@@ -1029,6 +1099,17 @@ This gem is available as open source under the terms of the [MIT License](https:
|
|
|
1029
1099
|
| `continue_if` | Manual continuation wait | `condition`, `name: nil` |
|
|
1030
1100
|
| `durably_repeat` | Periodic task execution | `method`, `every:`, `till:`, `start_at: nil`, `retry_policy: nil`, `timeout: 1.hour`, `on_error: :continue` |
|
|
1031
1101
|
|
|
1102
|
+
### Branch Methods
|
|
1103
|
+
|
|
1104
|
+
Fan a workflow out into parallel child sub-workflows (see [Branches](#-branches-parallel-sub-workflows)).
|
|
1105
|
+
|
|
1106
|
+
| Method | Purpose | Key Parameters |
|
|
1107
|
+
|--------|---------|----------------|
|
|
1108
|
+
| `branch` | Open a named branch (takes a block) to dispatch children | `name`, `automerge: false` |
|
|
1109
|
+
| `spawn` | Enqueue one child workflow inside a branch | `name`, `workflow_class`, `**kwargs` |
|
|
1110
|
+
| `spawn_each` | Enqueue one child per item, streamed (block returns `[WorkflowClass, kwargs]`) | `name`, `source`, `of: 1000` |
|
|
1111
|
+
| `merge_branches` | Join named branches; blocks until all complete (alias `merge_branch`) | `*names`, `min_interval: 5.seconds`, `max_interval: 5.minutes` |
|
|
1112
|
+
|
|
1032
1113
|
### Context Methods
|
|
1033
1114
|
|
|
1034
1115
|
| Method | Purpose | Example |
|
|
@@ -1037,6 +1118,8 @@ This gem is available as open source under the terms of the [MIT License](https:
|
|
|
1037
1118
|
| `context[:key]` | Get context value | `user_id = context[:user_id]` |
|
|
1038
1119
|
| `context.set(key, value)` | Set context value (alias) | `context.set(:status, "active")` |
|
|
1039
1120
|
| `context.set_once(key, value)` | Set only if key doesn't exist | `context.set_once(:created_at, Time.current)` |
|
|
1121
|
+
| `context.merge(hash)` | Set multiple values atomically (alias: `set_multiple`) | `context.merge(status: "active", count: 0)` |
|
|
1122
|
+
| `context.merge_once(hash)` | Set multiple values, skipping existing keys (alias: `set_multiple_once`) | `context.merge_once(created_at: Time.current, count: 0)` |
|
|
1040
1123
|
| `context.fetch(key, default)` | Get with default value | `context.fetch(:count, 0)` |
|
|
1041
1124
|
| `context.key?(key)` | Check if key exists | `context.key?(:user_id)` |
|
|
1042
1125
|
|
data/Rakefile
CHANGED
|
@@ -12,3 +12,7 @@ Rake::TestTask.new do |t|
|
|
|
12
12
|
t.test_files = FileList["test/**/*_test.rb"]
|
|
13
13
|
t.verbose = true
|
|
14
14
|
end
|
|
15
|
+
|
|
16
|
+
# Release tasks (release:core:*, release:dashboard:*). Loaded after
|
|
17
|
+
# bundler/gem_tasks so it can neutralize the bare `rake release` footgun.
|
|
18
|
+
Dir.glob(File.expand_path("lib/tasks/*.rake", __dir__)).sort.each { |f| load f }
|
data/cliff.toml
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# git-cliff configuration for the chrono_forge monorepo.
|
|
2
|
+
# https://git-cliff.org/docs/configuration
|
|
3
|
+
#
|
|
4
|
+
# Two gems share this one config. bin/release passes per-gem --tag-pattern and
|
|
5
|
+
# --include-path/--exclude-path so each CHANGELOG only reflects its own subtree
|
|
6
|
+
# (core = everything except chrono_forge-dashboard/, dashboard = that subtree).
|
|
7
|
+
# The version heading trims both tag prefixes (`v` and `chrono_forge-dashboard-v`).
|
|
8
|
+
|
|
9
|
+
[changelog]
|
|
10
|
+
header = """
|
|
11
|
+
# Changelog\n
|
|
12
|
+
All notable changes to this project are documented in this file.\n
|
|
13
|
+
"""
|
|
14
|
+
body = """
|
|
15
|
+
{% if version %}\
|
|
16
|
+
## [{{ version | trim_start_matches(pat="chrono_forge-dashboard-v") | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
|
|
17
|
+
{% else %}\
|
|
18
|
+
## [Unreleased]
|
|
19
|
+
{% endif %}\
|
|
20
|
+
{% for group, commits in commits | group_by(attribute="group") %}
|
|
21
|
+
### {{ group | upper_first }}
|
|
22
|
+
{% for commit in commits %}
|
|
23
|
+
- {% if commit.breaking %}[**breaking**] {% endif %}{{ commit.message | upper_first }}\
|
|
24
|
+
{% endfor %}
|
|
25
|
+
{% endfor %}\n
|
|
26
|
+
"""
|
|
27
|
+
trim = true
|
|
28
|
+
footer = """
|
|
29
|
+
<!-- generated by git-cliff -->
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
[git]
|
|
33
|
+
conventional_commits = true
|
|
34
|
+
filter_unconventional = true
|
|
35
|
+
split_commits = false
|
|
36
|
+
commit_preprocessors = [
|
|
37
|
+
{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](https://github.com/radioactive-labs/chrono_forge/issues/${2}))" },
|
|
38
|
+
]
|
|
39
|
+
commit_parsers = [
|
|
40
|
+
{ message = "^feat", group = "Features" },
|
|
41
|
+
{ message = "^fix", group = "Bug Fixes" },
|
|
42
|
+
{ message = "^doc", group = "Documentation" },
|
|
43
|
+
{ message = "^perf", group = "Performance" },
|
|
44
|
+
{ message = "^refactor", group = "Refactoring" },
|
|
45
|
+
{ message = "^style", group = "Styling" },
|
|
46
|
+
{ message = "^test", group = "Testing" },
|
|
47
|
+
{ message = "^chore\\(release\\):", skip = true },
|
|
48
|
+
{ message = "^chore", group = "Miscellaneous Tasks" },
|
|
49
|
+
{ body = ".*security", group = "Security" },
|
|
50
|
+
]
|
|
51
|
+
protect_breaking_commits = false
|
|
52
|
+
filter_commits = false
|
|
53
|
+
tag_pattern = "^v[0-9]"
|
|
54
|
+
topo_order = false
|
|
55
|
+
sort_commits = "oldest"
|
|
56
|
+
|
|
57
|
+
[bump]
|
|
58
|
+
# Pre-1.0 semver: both gems are on 0.x. A feature bumps the minor, and a
|
|
59
|
+
# breaking change ALSO bumps the minor (not the major) while we're on 0.x.
|
|
60
|
+
# Fixes bump the patch. Revisit once a gem reaches 1.0.
|
|
61
|
+
features_always_bump_minor = true
|
|
62
|
+
breaking_always_bump_major = false
|