@smithers-orchestrator/time-travel 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/LICENSE +21 -0
  2. package/package.json +68 -0
  3. package/src/BranchInfo.ts +11 -0
  4. package/src/ForkParams.ts +11 -0
  5. package/src/JUMP_MAX_FRAME_NO.js +1 -0
  6. package/src/JUMP_RUN_ID_PATTERN.js +1 -0
  7. package/src/JumpResult.ts +9 -0
  8. package/src/JumpStepName.ts +10 -0
  9. package/src/JumpToFrameError.js +23 -0
  10. package/src/JumpToFrameInput.ts +36 -0
  11. package/src/NodeChange.ts +7 -0
  12. package/src/NodeSnapshot.ts +8 -0
  13. package/src/OutputChange.ts +5 -0
  14. package/src/ParsedSnapshot.ts +18 -0
  15. package/src/REWIND_RATE_LIMIT_MAX.js +1 -0
  16. package/src/REWIND_RATE_LIMIT_WINDOW_MS.js +1 -0
  17. package/src/RalphChange.ts +7 -0
  18. package/src/RalphSnapshot.ts +5 -0
  19. package/src/ReplayParams.ts +12 -0
  20. package/src/ReplayResult.ts +11 -0
  21. package/src/RetryTaskOptions.ts +10 -0
  22. package/src/RetryTaskResult.ts +5 -0
  23. package/src/RevertOptions.ts +9 -0
  24. package/src/RevertResult.ts +5 -0
  25. package/src/RewindAuditResult.ts +5 -0
  26. package/src/RewindLockHandle.ts +4 -0
  27. package/src/RunTimeline.ts +11 -0
  28. package/src/SnapshotDiff.ts +18 -0
  29. package/src/TimeTravelOptions.ts +11 -0
  30. package/src/TimeTravelResult.ts +7 -0
  31. package/src/TimelineFrame.ts +11 -0
  32. package/src/TimelineTree.ts +9 -0
  33. package/src/acquireRewindLock.js +32 -0
  34. package/src/countRecentRewindAuditRows.js +27 -0
  35. package/src/diff.js +189 -0
  36. package/src/evaluateRewindRateLimit.js +41 -0
  37. package/src/fork/_helpers.js +28 -0
  38. package/src/fork/forkRunEffect.js +147 -0
  39. package/src/fork/getBranchInfoEffect.js +26 -0
  40. package/src/fork/index.js +41 -0
  41. package/src/fork/listBranchesEffect.js +25 -0
  42. package/src/hasRewindLock.js +11 -0
  43. package/src/index.d.ts +1170 -0
  44. package/src/index.js +43 -0
  45. package/src/jumpToFrame.js +1077 -0
  46. package/src/listRewindAuditRows.js +83 -0
  47. package/src/metrics.js +4 -0
  48. package/src/recoverInProgressRewindAudits.js +72 -0
  49. package/src/replay.js +22 -0
  50. package/src/replayFromCheckpointEffect.js +59 -0
  51. package/src/replaysStarted.js +2 -0
  52. package/src/resetRewindLocksForTests.js +8 -0
  53. package/src/resolveRewindAuditClient.js +38 -0
  54. package/src/retry-task.js +215 -0
  55. package/src/revert.js +68 -0
  56. package/src/rewindAudit.js +9 -0
  57. package/src/rewindLock.js +7 -0
  58. package/src/rewindLockStore.js +8 -0
  59. package/src/rewindRateLimit.js +3 -0
  60. package/src/runForksCreated.js +2 -0
  61. package/src/schema.js +46 -0
  62. package/src/snapshot/Snapshot.ts +15 -0
  63. package/src/snapshot/SnapshotData.ts +19 -0
  64. package/src/snapshot/captureSnapshotEffect.js +70 -0
  65. package/src/snapshot/index.js +57 -0
  66. package/src/snapshot/listSnapshotsEffect.js +32 -0
  67. package/src/snapshot/loadSnapshotEffect.js +46 -0
  68. package/src/snapshot/parseSnapshot.js +31 -0
  69. package/src/snapshotDuration.js +7 -0
  70. package/src/snapshotsCaptured.js +2 -0
  71. package/src/timeline/_helpers.js +7 -0
  72. package/src/timeline/buildTimelineEffect.js +38 -0
  73. package/src/timeline/buildTimelineTreeEffect.js +30 -0
  74. package/src/timeline/formatTimelineAsJson.js +23 -0
  75. package/src/timeline/formatTimelineForTui.js +31 -0
  76. package/src/timeline/index.js +31 -0
  77. package/src/timetravel.js +247 -0
  78. package/src/types.ts +15 -0
  79. package/src/updateRewindAuditRow.js +35 -0
  80. package/src/validateJumpFrameNo.js +23 -0
  81. package/src/validateJumpRunId.js +18 -0
  82. package/src/vcs-version/VcsTag.ts +9 -0
  83. package/src/vcs-version/index.js +61 -0
  84. package/src/vcs-version/loadVcsTagEffect.js +27 -0
  85. package/src/vcs-version/rerunAtRevisionEffect.js +25 -0
  86. package/src/vcs-version/resolveWorkflowAtRevisionEffect.js +32 -0
  87. package/src/vcs-version/tagSnapshotVcsEffect.js +56 -0
  88. package/src/writeRewindAuditRow.js +46 -0
@@ -0,0 +1,83 @@
1
+ import { resolveRewindAuditClient } from "./resolveRewindAuditClient.js";
2
+
3
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
4
+ /** @typedef {import("./RewindAuditResult.ts").RewindAuditResult} RewindAuditResult */
5
+
6
+ /**
7
+ * @typedef {{
8
+ * id: number;
9
+ * runId: string;
10
+ * fromFrameNo: number;
11
+ * toFrameNo: number;
12
+ * caller: string;
13
+ * timestampMs: number;
14
+ * result: RewindAuditResult;
15
+ * durationMs: number | null;
16
+ * }} RewindAuditRow
17
+ */
18
+
19
+ /**
20
+ * @param {Array<Record<string, unknown>>} rows
21
+ * @returns {Array<RewindAuditRow>}
22
+ */
23
+ function mapRewindAuditRows(rows) {
24
+ return rows.map((row) => ({
25
+ id: Number(row.id),
26
+ runId: String(row.runId),
27
+ fromFrameNo: Number(row.fromFrameNo),
28
+ toFrameNo: Number(row.toFrameNo),
29
+ caller: String(row.caller),
30
+ timestampMs: Number(row.timestampMs),
31
+ result: /** @type {RewindAuditResult} */ (String(row.result)),
32
+ durationMs: row.durationMs == null ? null : Number(row.durationMs),
33
+ }));
34
+ }
35
+
36
+ /**
37
+ * Fetch audit rows for tests and diagnostics.
38
+ *
39
+ * @param {SmithersDb} adapter
40
+ * @param {{ runId?: string; limit?: number; }} [input]
41
+ * @returns {Promise<Array<RewindAuditRow>>}
42
+ */
43
+ export async function listRewindAuditRows(adapter, input = {}) {
44
+ const client = resolveRewindAuditClient(adapter);
45
+ const limit = Number.isInteger(input.limit) ? Math.max(1, Number(input.limit)) : 100;
46
+ if (typeof input.runId === "string") {
47
+ const rows = client
48
+ .query(
49
+ `SELECT
50
+ id,
51
+ run_id AS runId,
52
+ from_frame_no AS fromFrameNo,
53
+ to_frame_no AS toFrameNo,
54
+ caller,
55
+ timestamp_ms AS timestampMs,
56
+ result,
57
+ duration_ms AS durationMs
58
+ FROM _smithers_time_travel_audit
59
+ WHERE run_id = ?
60
+ ORDER BY id ASC
61
+ LIMIT ?`,
62
+ )
63
+ .all(input.runId, limit);
64
+ return mapRewindAuditRows(rows);
65
+ }
66
+ const rows = client
67
+ .query(
68
+ `SELECT
69
+ id,
70
+ run_id AS runId,
71
+ from_frame_no AS fromFrameNo,
72
+ to_frame_no AS toFrameNo,
73
+ caller,
74
+ timestamp_ms AS timestampMs,
75
+ result,
76
+ duration_ms AS durationMs
77
+ FROM _smithers_time_travel_audit
78
+ ORDER BY id ASC
79
+ LIMIT ?`,
80
+ )
81
+ .all(limit);
82
+ return mapRewindAuditRows(rows);
83
+ }
package/src/metrics.js ADDED
@@ -0,0 +1,4 @@
1
+ export { replaysStarted } from "./replaysStarted.js";
2
+ export { runForksCreated } from "./runForksCreated.js";
3
+ export { snapshotDuration } from "./snapshotDuration.js";
4
+ export { snapshotsCaptured } from "./snapshotsCaptured.js";
@@ -0,0 +1,72 @@
1
+ import { resolveRewindAuditClient } from "./resolveRewindAuditClient.js";
2
+
3
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
4
+
5
+ /**
6
+ * On startup, find rewind audit rows left in `in_progress` by a prior crash,
7
+ * mark them as `partial`, and flag the associated runs as `needs_attention`.
8
+ *
9
+ * @param {SmithersDb} adapter
10
+ * @param {{ nowMs?: () => number }} [options]
11
+ * @returns {Promise<{ recovered: Array<{ id: number; runId: string }> }>}
12
+ */
13
+ export async function recoverInProgressRewindAudits(adapter, options = {}) {
14
+ const nowMs = options.nowMs ?? (() => Date.now());
15
+ const client = resolveRewindAuditClient(adapter);
16
+ const rows = /** @type {Array<{ id: number; runId: string; timestampMs: number }>} */ (
17
+ client
18
+ .query(
19
+ `SELECT id, run_id AS runId, timestamp_ms AS timestampMs
20
+ FROM _smithers_time_travel_audit
21
+ WHERE result = 'in_progress'`,
22
+ )
23
+ .all()
24
+ );
25
+ if (rows.length === 0) {
26
+ return { recovered: [] };
27
+ }
28
+ const now = nowMs();
29
+ const updateStmt = client.query(
30
+ `UPDATE _smithers_time_travel_audit
31
+ SET result = 'partial',
32
+ duration_ms = COALESCE(duration_ms, ?)
33
+ WHERE id = ?`,
34
+ );
35
+ const recovered = [];
36
+ for (const row of rows) {
37
+ const duration = Math.max(0, now - Number(row.timestampMs ?? now));
38
+ updateStmt.run(duration, row.id);
39
+ try {
40
+ const payload = JSON.stringify({
41
+ code: "RewindFailed",
42
+ needsAttention: true,
43
+ message: `Rewind audit ${row.id} was in_progress at startup; marked partial.`,
44
+ timestampMs: now,
45
+ });
46
+ await adapter.updateRun(row.runId, {
47
+ status: "needs_attention",
48
+ heartbeatAtMs: null,
49
+ runtimeOwnerId: null,
50
+ errorJson: payload,
51
+ });
52
+ } catch {
53
+ try {
54
+ await adapter.updateRun(row.runId, {
55
+ status: "failed",
56
+ heartbeatAtMs: null,
57
+ runtimeOwnerId: null,
58
+ errorJson: JSON.stringify({
59
+ code: "RewindFailed",
60
+ needsAttention: true,
61
+ message: "Rewind was in_progress at startup.",
62
+ timestampMs: now,
63
+ }),
64
+ });
65
+ } catch {
66
+ // best-effort: nothing to do if the run row was deleted.
67
+ }
68
+ }
69
+ recovered.push({ id: row.id, runId: row.runId });
70
+ }
71
+ return { recovered };
72
+ }
package/src/replay.js ADDED
@@ -0,0 +1,22 @@
1
+ // @smithers-type-exports-begin
2
+ /** @typedef {import("./ReplayResult.ts").ReplayResult} ReplayResult */
3
+ // @smithers-type-exports-end
4
+
5
+ import { Effect } from "effect";
6
+ import * as BunContext from "@effect/platform-bun/BunContext";
7
+ import { replayFromCheckpoint as replayFromCheckpointEffect } from "./replayFromCheckpointEffect.js";
8
+ export { replayFromCheckpointEffect };
9
+
10
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
11
+ /** @typedef {import("./ReplayParams.ts").ReplayParams} ReplayParams */
12
+
13
+ /**
14
+ * Fork a run from a checkpoint and optionally restore the VCS working copy.
15
+ *
16
+ * @param {SmithersDb} adapter
17
+ * @param {ReplayParams} params
18
+ * @returns {Promise<ReplayResult>}
19
+ */
20
+ export function replayFromCheckpoint(adapter, params) {
21
+ return Effect.runPromise(replayFromCheckpointEffect(adapter, params).pipe(Effect.provide(BunContext.layer)));
22
+ }
@@ -0,0 +1,59 @@
1
+ import { Effect, Metric } from "effect";
2
+ import { forkRun as forkRunEffect } from "./fork/forkRunEffect.js";
3
+ import { rerunAtRevision as rerunAtRevisionEffect } from "./vcs-version/rerunAtRevisionEffect.js";
4
+ import { replaysStarted } from "./replaysStarted.js";
5
+
6
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
7
+ /** @typedef {import("./ReplayParams.ts").ReplayParams} ReplayParams */
8
+ /** @typedef {import("./ReplayResult.ts").ReplayResult} ReplayResult */
9
+
10
+ /**
11
+ * Fork a run from a checkpoint, optionally restore the VCS working copy
12
+ * to the revision that was active at the source frame, then return the
13
+ * new run metadata so the caller can resume execution.
14
+ *
15
+ * @param {SmithersDb} adapter
16
+ * @param {ReplayParams} params
17
+ */
18
+ export function replayFromCheckpoint(adapter, params) {
19
+ return Effect.gen(function* () {
20
+ const { parentRunId, frameNo, inputOverrides, resetNodes, branchLabel, restoreVcs, cwd, } = params;
21
+ // 1. Fork the run
22
+ const { runId, branch, snapshot } = yield* forkRunEffect(adapter, {
23
+ parentRunId,
24
+ frameNo,
25
+ inputOverrides,
26
+ resetNodes,
27
+ branchLabel,
28
+ forkDescription: `Replay from ${parentRunId}:${frameNo}`,
29
+ });
30
+ // 2. Optionally restore VCS state
31
+ let vcsRestored = false;
32
+ let vcsPointer = null;
33
+ let vcsError;
34
+ if (restoreVcs) {
35
+ const vcsResult = yield* rerunAtRevisionEffect(adapter, parentRunId, frameNo, { cwd });
36
+ vcsRestored = vcsResult.restored;
37
+ vcsPointer = vcsResult.vcsPointer;
38
+ vcsError = vcsResult.error;
39
+ }
40
+ yield* Metric.increment(replaysStarted);
41
+ yield* Effect.logInfo("Replay started").pipe(Effect.annotateLogs({
42
+ parentRunId,
43
+ parentFrameNo: String(frameNo),
44
+ childRunId: runId,
45
+ vcsRestored: String(vcsRestored),
46
+ }));
47
+ return {
48
+ runId,
49
+ branch,
50
+ snapshot,
51
+ vcsRestored,
52
+ vcsPointer,
53
+ vcsError,
54
+ };
55
+ }).pipe(Effect.annotateLogs({
56
+ parentRunId: params.parentRunId,
57
+ parentFrameNo: String(params.frameNo),
58
+ }), Effect.withLogSpan("time-travel:replay-from-checkpoint"));
59
+ }
@@ -0,0 +1,2 @@
1
+ import { Metric } from "effect";
2
+ export const replaysStarted = Metric.counter("smithers.replays.started");
@@ -0,0 +1,8 @@
1
+ import { rewindLockStore } from "./rewindLockStore.js";
2
+
3
+ /**
4
+ * Reset lock state for tests.
5
+ */
6
+ export function resetRewindLocksForTests() {
7
+ rewindLockStore.clear();
8
+ }
@@ -0,0 +1,38 @@
1
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
2
+
3
+ /**
4
+ * Minimal Bun SQLite client surface used by the rewind audit helpers.
5
+ * This mirrors the shape returned by `bun:sqlite`'s `Database.query()`
6
+ * and is sufficient for the raw SQL writes / reads we perform against
7
+ * the `_smithers_time_travel_audit` and `_smithers_frames` tables.
8
+ *
9
+ * @typedef {{
10
+ * query: (sql: string) => {
11
+ * run: (...args: unknown[]) => unknown;
12
+ * get: (...args: unknown[]) => Record<string, unknown> | null | undefined;
13
+ * all: (...args: unknown[]) => Array<Record<string, unknown>>;
14
+ * };
15
+ * }} RewindAuditSqliteClient
16
+ */
17
+
18
+ /**
19
+ * Resolve the Bun SQLite client from a {@link SmithersDb} instance for audit writes.
20
+ *
21
+ * @param {SmithersDb} adapter
22
+ * @returns {RewindAuditSqliteClient}
23
+ */
24
+ export function resolveRewindAuditClient(adapter) {
25
+ const db = /** @type {{ session?: { client?: unknown }; $client?: unknown } | null | undefined} */ (
26
+ /** @type {unknown} */ (adapter?.db)
27
+ );
28
+ const candidate = /** @type {unknown} */ (db?.session?.client ?? db?.$client);
29
+ if (
30
+ !candidate ||
31
+ typeof (/** @type {{ query?: unknown }} */ (candidate).query) !== "function"
32
+ ) {
33
+ throw new TypeError(
34
+ "Could not resolve a Bun SQLite client for rewind audit writes.",
35
+ );
36
+ }
37
+ return /** @type {RewindAuditSqliteClient} */ (candidate);
38
+ }
@@ -0,0 +1,215 @@
1
+ import { Effect } from "effect";
2
+ import { nowMs } from "@smithers-orchestrator/scheduler/nowMs";
3
+ /** @typedef {import("./RetryTaskOptions.ts").RetryTaskOptions} RetryTaskOptions */
4
+ /** @typedef {import("./RetryTaskResult.ts").RetryTaskResult} RetryTaskResult */
5
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
6
+
7
+ /**
8
+ * @param {string} nodeId
9
+ * @param {number} iteration
10
+ */
11
+ function buildNodeKey(nodeId, iteration) {
12
+ return `${nodeId}::${iteration}`;
13
+ }
14
+ /**
15
+ * @param {Array<{ nodeId: string; iteration: number }>} nodes
16
+ * @returns {string[]}
17
+ */
18
+ function uniqueNodeIds(nodes) {
19
+ const seen = new Set();
20
+ const result = [];
21
+ for (const node of nodes) {
22
+ if (seen.has(node.nodeId))
23
+ continue;
24
+ seen.add(node.nodeId);
25
+ result.push(node.nodeId);
26
+ }
27
+ return result;
28
+ }
29
+ /**
30
+ * @param {string | null | undefined} status
31
+ */
32
+ function isActiveRunStatus(status) {
33
+ return (status === "running" ||
34
+ status === "waiting-approval" ||
35
+ status === "waiting-event" ||
36
+ status === "waiting-timer");
37
+ }
38
+ /**
39
+ * @param {SmithersDb} adapter
40
+ * @param {Required<Pick<RetryTaskOptions, "runId" | "resetDependents">> & { targetNode: any; }} opts
41
+ */
42
+ async function resolveResetNodes(adapter, opts) {
43
+ const { runId, targetNode, resetDependents } = opts;
44
+ if (!resetDependents) {
45
+ return [targetNode];
46
+ }
47
+ const nodes = await adapter.listNodes(runId);
48
+ const attempts = await adapter.listAttemptsForRun(runId);
49
+ const attemptOrder = new Map();
50
+ for (let index = 0; index < attempts.length; index += 1) {
51
+ const attempt = attempts[index];
52
+ attemptOrder.set(buildNodeKey(attempt.nodeId, attempt.iteration ?? 0), index);
53
+ }
54
+ const targetKey = buildNodeKey(targetNode.nodeId, targetNode.iteration ?? 0);
55
+ const targetOrder = attemptOrder.get(targetKey);
56
+ const targetIteration = targetNode.iteration ?? 0;
57
+ const targetUpdatedAtMs = targetNode.updatedAtMs ?? 0;
58
+ return nodes.filter((node) => {
59
+ const nodeIteration = node.iteration ?? 0;
60
+ const nodeKey = buildNodeKey(node.nodeId, nodeIteration);
61
+ if (nodeKey === targetKey)
62
+ return true;
63
+ if (nodeIteration > targetIteration)
64
+ return true;
65
+ const nodeOrder = attemptOrder.get(nodeKey);
66
+ if (targetOrder !== undefined && nodeOrder !== undefined) {
67
+ return nodeOrder > targetOrder;
68
+ }
69
+ return (node.updatedAtMs ?? 0) > targetUpdatedAtMs;
70
+ });
71
+ }
72
+ /**
73
+ * @param {RetryTaskOptions} opts
74
+ * @param {{ runId: string; nodeId: string; iteration: number; resetNodes: string[]; success: boolean; error?: string; }} payload
75
+ */
76
+ function emitRetryFinished(opts, payload) {
77
+ opts.onProgress?.({
78
+ type: "RetryTaskFinished",
79
+ ...payload,
80
+ timestampMs: nowMs(),
81
+ });
82
+ }
83
+ /**
84
+ * @param {SmithersDb} adapter
85
+ * @param {RetryTaskOptions} opts
86
+ * @returns {Promise<RetryTaskResult>}
87
+ */
88
+ export async function retryTask(adapter, opts) {
89
+ const runId = opts.runId;
90
+ const nodeId = opts.nodeId;
91
+ const iteration = opts.iteration ?? 0;
92
+ const resetDependents = opts.resetDependents ?? true;
93
+ const force = opts.force ?? false;
94
+ const node = await adapter.getNode(runId, nodeId, iteration);
95
+ if (!node) {
96
+ const error = `Node not found: ${runId}/${nodeId}/${iteration}`;
97
+ emitRetryFinished(opts, {
98
+ runId,
99
+ nodeId,
100
+ iteration,
101
+ resetNodes: [],
102
+ success: false,
103
+ error,
104
+ });
105
+ return { success: false, resetNodes: [], error };
106
+ }
107
+ const run = await adapter.getRun(runId);
108
+ if (!run) {
109
+ const error = `Run not found: ${runId}`;
110
+ emitRetryFinished(opts, {
111
+ runId,
112
+ nodeId,
113
+ iteration,
114
+ resetNodes: [],
115
+ success: false,
116
+ error,
117
+ });
118
+ return { success: false, resetNodes: [], error };
119
+ }
120
+ if (!force && isActiveRunStatus(run.status)) {
121
+ const error = `Run is still running: ${runId}`;
122
+ emitRetryFinished(opts, {
123
+ runId,
124
+ nodeId,
125
+ iteration,
126
+ resetNodes: [],
127
+ success: false,
128
+ error,
129
+ });
130
+ return { success: false, resetNodes: [], error };
131
+ }
132
+ const resetNodes = await resolveResetNodes(adapter, {
133
+ runId,
134
+ targetNode: node,
135
+ resetDependents,
136
+ });
137
+ const resetNodeIds = uniqueNodeIds(resetNodes.map((candidate) => ({
138
+ nodeId: candidate.nodeId,
139
+ iteration: candidate.iteration ?? 0,
140
+ })));
141
+ const attemptsByNode = new Map();
142
+ for (const resetNode of resetNodes) {
143
+ const resetIteration = resetNode.iteration ?? 0;
144
+ attemptsByNode.set(buildNodeKey(resetNode.nodeId, resetIteration), await adapter.listAttempts(runId, resetNode.nodeId, resetIteration));
145
+ }
146
+ opts.onProgress?.({
147
+ type: "RetryTaskStarted",
148
+ runId,
149
+ nodeId,
150
+ iteration,
151
+ resetDependents,
152
+ resetNodes: resetNodeIds,
153
+ timestampMs: nowMs(),
154
+ });
155
+ const resetTimestampMs = nowMs();
156
+ await adapter.withTransaction("retry-task-reset", Effect.gen(function* () {
157
+ for (const resetNode of resetNodes) {
158
+ const resetIteration = resetNode.iteration ?? 0;
159
+ const attempts = attemptsByNode.get(buildNodeKey(resetNode.nodeId, resetIteration)) ??
160
+ [];
161
+ for (const attempt of attempts) {
162
+ if (attempt.state !== "failed" &&
163
+ attempt.state !== "in-progress" &&
164
+ attempt.state !== "waiting-approval" &&
165
+ attempt.state !== "waiting-event" &&
166
+ attempt.state !== "waiting-timer") {
167
+ continue;
168
+ }
169
+ const patch = { state: "cancelled" };
170
+ if (attempt.finishedAtMs == null) {
171
+ patch.finishedAtMs = resetTimestampMs;
172
+ }
173
+ yield* adapter.updateAttemptEffect(runId, resetNode.nodeId, resetIteration, attempt.attempt, patch);
174
+ }
175
+ if (resetNode.outputTable) {
176
+ yield* adapter.deleteOutputRowEffect(resetNode.outputTable, {
177
+ runId,
178
+ nodeId: resetNode.nodeId,
179
+ iteration: resetIteration,
180
+ });
181
+ }
182
+ yield* adapter.insertNodeEffect({
183
+ runId,
184
+ nodeId: resetNode.nodeId,
185
+ iteration: resetIteration,
186
+ state: "pending",
187
+ lastAttempt: resetNode.lastAttempt ?? null,
188
+ updatedAtMs: resetTimestampMs,
189
+ outputTable: resetNode.outputTable ?? "",
190
+ label: resetNode.label ?? null,
191
+ });
192
+ }
193
+ yield* adapter.updateRunEffect(runId, {
194
+ status: "running",
195
+ finishedAtMs: null,
196
+ heartbeatAtMs: null,
197
+ runtimeOwnerId: null,
198
+ cancelRequestedAtMs: null,
199
+ hijackRequestedAtMs: null,
200
+ hijackTarget: null,
201
+ errorJson: null,
202
+ });
203
+ }));
204
+ emitRetryFinished(opts, {
205
+ runId,
206
+ nodeId,
207
+ iteration,
208
+ resetNodes: resetNodeIds,
209
+ success: true,
210
+ });
211
+ return {
212
+ success: true,
213
+ resetNodes: resetNodeIds,
214
+ };
215
+ }
package/src/revert.js ADDED
@@ -0,0 +1,68 @@
1
+ import { Effect } from "effect";
2
+ import { revertToJjPointer } from "@smithers-orchestrator/vcs/jj";
3
+ import * as BunContext from "@effect/platform-bun/BunContext";
4
+ import { nowMs } from "@smithers-orchestrator/scheduler/nowMs";
5
+ /** @typedef {import("./RevertOptions.ts").RevertOptions} RevertOptions */
6
+ /** @typedef {import("./RevertResult.ts").RevertResult} RevertResult */
7
+ /** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
8
+
9
+ /**
10
+ * @param {SmithersDb} adapter
11
+ * @param {RevertOptions} opts
12
+ * @returns {Promise<RevertResult>}
13
+ */
14
+ export async function revertToAttempt(adapter, opts) {
15
+ const { runId, nodeId, iteration, attempt, onProgress } = opts;
16
+ const attemptRow = await Effect.runPromise(adapter.getAttempt(runId, nodeId, iteration, attempt));
17
+ if (!attemptRow) {
18
+ return {
19
+ success: false,
20
+ error: `Attempt not found: ${runId}/${nodeId}/${iteration}/${attempt}`,
21
+ };
22
+ }
23
+ const jjPointer = attemptRow.jjPointer;
24
+ if (!jjPointer) {
25
+ return { success: false, error: `Attempt has no jjPointer recorded` };
26
+ }
27
+ onProgress?.({
28
+ type: "RevertStarted",
29
+ runId,
30
+ nodeId,
31
+ iteration,
32
+ attempt,
33
+ jjPointer,
34
+ timestampMs: nowMs(),
35
+ });
36
+ // Revert must target the same repository/worktree where the attempt ran.
37
+ const cwd = attemptRow.jjCwd ?? undefined;
38
+ const result = await Effect.runPromise(revertToJjPointer(jjPointer, cwd).pipe(Effect.provide(BunContext.layer)));
39
+ onProgress?.({
40
+ type: "RevertFinished",
41
+ runId,
42
+ nodeId,
43
+ iteration,
44
+ attempt,
45
+ jjPointer,
46
+ success: result.success,
47
+ error: result.error,
48
+ timestampMs: nowMs(),
49
+ });
50
+ if (!result.success) {
51
+ return { success: false, error: result.error, jjPointer };
52
+ }
53
+ // Clean up DB frames recorded after the reverted attempt started.
54
+ // Find the latest frame created before the attempt's start time and
55
+ // discard everything after it so the DB matches the reverted VCS state.
56
+ const frames = await Effect.runPromise(adapter.listFrames(runId, 1_000_000));
57
+ const cutoff = attemptRow.startedAtMs;
58
+ let lastValidFrameNo = -1;
59
+ for (const f of frames) {
60
+ if (f.createdAtMs <= cutoff && f.frameNo > lastValidFrameNo) {
61
+ lastValidFrameNo = f.frameNo;
62
+ }
63
+ }
64
+ if (lastValidFrameNo >= 0) {
65
+ await Effect.runPromise(adapter.deleteFramesAfter(runId, lastValidFrameNo));
66
+ }
67
+ return { success: true, jjPointer };
68
+ }
@@ -0,0 +1,9 @@
1
+ // @smithers-type-exports-begin
2
+ /** @typedef {import("./RewindAuditResult.ts").RewindAuditResult} RewindAuditResult */
3
+ // @smithers-type-exports-end
4
+
5
+ export { writeRewindAuditRow } from "./writeRewindAuditRow.js";
6
+ export { updateRewindAuditRow } from "./updateRewindAuditRow.js";
7
+ export { countRecentRewindAuditRows } from "./countRecentRewindAuditRows.js";
8
+ export { listRewindAuditRows } from "./listRewindAuditRows.js";
9
+ export { recoverInProgressRewindAudits } from "./recoverInProgressRewindAudits.js";
@@ -0,0 +1,7 @@
1
+ // @smithers-type-exports-begin
2
+ /** @typedef {import("./RewindLockHandle.ts").RewindLockHandle} RewindLockHandle */
3
+ // @smithers-type-exports-end
4
+
5
+ export { acquireRewindLock } from "./acquireRewindLock.js";
6
+ export { hasRewindLock } from "./hasRewindLock.js";
7
+ export { resetRewindLocksForTests } from "./resetRewindLocksForTests.js";
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Shared per-run single-flight lock table.
3
+ * Used by {@link acquireRewindLock}, {@link hasRewindLock},
4
+ * and {@link resetRewindLocksForTests}.
5
+ *
6
+ * @type {Map<string, symbol>}
7
+ */
8
+ export const rewindLockStore = new Map();
@@ -0,0 +1,3 @@
1
+ export { REWIND_RATE_LIMIT_MAX } from "./REWIND_RATE_LIMIT_MAX.js";
2
+ export { REWIND_RATE_LIMIT_WINDOW_MS } from "./REWIND_RATE_LIMIT_WINDOW_MS.js";
3
+ export { evaluateRewindRateLimit } from "./evaluateRewindRateLimit.js";
@@ -0,0 +1,2 @@
1
+ import { Metric } from "effect";
2
+ export const runForksCreated = Metric.counter("smithers.forks.created");
package/src/schema.js ADDED
@@ -0,0 +1,46 @@
1
+ import { integer, sqliteTable, text, primaryKey, } from "drizzle-orm/sqlite-core";
2
+ /**
3
+ * Full state snapshot captured at each frame commit.
4
+ * PK: (run_id, frame_no)
5
+ */
6
+ export const smithersSnapshots = sqliteTable("_smithers_snapshots", {
7
+ runId: text("run_id").notNull(),
8
+ frameNo: integer("frame_no").notNull(),
9
+ nodesJson: text("nodes_json").notNull(),
10
+ outputsJson: text("outputs_json").notNull(),
11
+ ralphJson: text("ralph_json").notNull(),
12
+ inputJson: text("input_json").notNull(),
13
+ vcsPointer: text("vcs_pointer"),
14
+ workflowHash: text("workflow_hash"),
15
+ contentHash: text("content_hash").notNull(),
16
+ createdAtMs: integer("created_at_ms").notNull(),
17
+ }, (t) => ({
18
+ pk: primaryKey({ columns: [t.runId, t.frameNo] }),
19
+ }));
20
+ /**
21
+ * Parent-child fork relationships between runs.
22
+ * PK: run_id (the child run)
23
+ */
24
+ export const smithersBranches = sqliteTable("_smithers_branches", {
25
+ runId: text("run_id").primaryKey(),
26
+ parentRunId: text("parent_run_id").notNull(),
27
+ parentFrameNo: integer("parent_frame_no").notNull(),
28
+ branchLabel: text("branch_label"),
29
+ forkDescription: text("fork_description"),
30
+ createdAtMs: integer("created_at_ms").notNull(),
31
+ });
32
+ /**
33
+ * VCS revision metadata per snapshot.
34
+ * PK: (run_id, frame_no)
35
+ */
36
+ export const smithersVcsTags = sqliteTable("_smithers_vcs_tags", {
37
+ runId: text("run_id").notNull(),
38
+ frameNo: integer("frame_no").notNull(),
39
+ vcsType: text("vcs_type").notNull(),
40
+ vcsPointer: text("vcs_pointer").notNull(),
41
+ vcsRoot: text("vcs_root"),
42
+ jjOperationId: text("jj_operation_id"),
43
+ createdAtMs: integer("created_at_ms").notNull(),
44
+ }, (t) => ({
45
+ pk: primaryKey({ columns: [t.runId, t.frameNo] }),
46
+ }));
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Serialized snapshot of workflow state at a specific frame.
3
+ */
4
+ export type Snapshot = {
5
+ runId: string;
6
+ frameNo: number;
7
+ nodesJson: string;
8
+ outputsJson: string;
9
+ ralphJson: string;
10
+ inputJson: string;
11
+ vcsPointer: string | null;
12
+ workflowHash: string | null;
13
+ contentHash: string;
14
+ createdAtMs: number;
15
+ };