@smithers-orchestrator/time-travel 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/package.json +68 -0
- package/src/BranchInfo.ts +11 -0
- package/src/ForkParams.ts +11 -0
- package/src/JUMP_MAX_FRAME_NO.js +1 -0
- package/src/JUMP_RUN_ID_PATTERN.js +1 -0
- package/src/JumpResult.ts +9 -0
- package/src/JumpStepName.ts +10 -0
- package/src/JumpToFrameError.js +23 -0
- package/src/JumpToFrameInput.ts +36 -0
- package/src/NodeChange.ts +7 -0
- package/src/NodeSnapshot.ts +8 -0
- package/src/OutputChange.ts +5 -0
- package/src/ParsedSnapshot.ts +18 -0
- package/src/REWIND_RATE_LIMIT_MAX.js +1 -0
- package/src/REWIND_RATE_LIMIT_WINDOW_MS.js +1 -0
- package/src/RalphChange.ts +7 -0
- package/src/RalphSnapshot.ts +5 -0
- package/src/ReplayParams.ts +12 -0
- package/src/ReplayResult.ts +11 -0
- package/src/RetryTaskOptions.ts +10 -0
- package/src/RetryTaskResult.ts +5 -0
- package/src/RevertOptions.ts +9 -0
- package/src/RevertResult.ts +5 -0
- package/src/RewindAuditResult.ts +5 -0
- package/src/RewindLockHandle.ts +4 -0
- package/src/RunTimeline.ts +11 -0
- package/src/SnapshotDiff.ts +18 -0
- package/src/TimeTravelOptions.ts +11 -0
- package/src/TimeTravelResult.ts +7 -0
- package/src/TimelineFrame.ts +11 -0
- package/src/TimelineTree.ts +9 -0
- package/src/acquireRewindLock.js +32 -0
- package/src/countRecentRewindAuditRows.js +27 -0
- package/src/diff.js +189 -0
- package/src/evaluateRewindRateLimit.js +41 -0
- package/src/fork/_helpers.js +28 -0
- package/src/fork/forkRunEffect.js +147 -0
- package/src/fork/getBranchInfoEffect.js +26 -0
- package/src/fork/index.js +41 -0
- package/src/fork/listBranchesEffect.js +25 -0
- package/src/hasRewindLock.js +11 -0
- package/src/index.d.ts +1170 -0
- package/src/index.js +43 -0
- package/src/jumpToFrame.js +1077 -0
- package/src/listRewindAuditRows.js +83 -0
- package/src/metrics.js +4 -0
- package/src/recoverInProgressRewindAudits.js +72 -0
- package/src/replay.js +22 -0
- package/src/replayFromCheckpointEffect.js +59 -0
- package/src/replaysStarted.js +2 -0
- package/src/resetRewindLocksForTests.js +8 -0
- package/src/resolveRewindAuditClient.js +38 -0
- package/src/retry-task.js +215 -0
- package/src/revert.js +68 -0
- package/src/rewindAudit.js +9 -0
- package/src/rewindLock.js +7 -0
- package/src/rewindLockStore.js +8 -0
- package/src/rewindRateLimit.js +3 -0
- package/src/runForksCreated.js +2 -0
- package/src/schema.js +46 -0
- package/src/snapshot/Snapshot.ts +15 -0
- package/src/snapshot/SnapshotData.ts +19 -0
- package/src/snapshot/captureSnapshotEffect.js +70 -0
- package/src/snapshot/index.js +57 -0
- package/src/snapshot/listSnapshotsEffect.js +32 -0
- package/src/snapshot/loadSnapshotEffect.js +46 -0
- package/src/snapshot/parseSnapshot.js +31 -0
- package/src/snapshotDuration.js +7 -0
- package/src/snapshotsCaptured.js +2 -0
- package/src/timeline/_helpers.js +7 -0
- package/src/timeline/buildTimelineEffect.js +38 -0
- package/src/timeline/buildTimelineTreeEffect.js +30 -0
- package/src/timeline/formatTimelineAsJson.js +23 -0
- package/src/timeline/formatTimelineForTui.js +31 -0
- package/src/timeline/index.js +31 -0
- package/src/timetravel.js +247 -0
- package/src/types.ts +15 -0
- package/src/updateRewindAuditRow.js +35 -0
- package/src/validateJumpFrameNo.js +23 -0
- package/src/validateJumpRunId.js +18 -0
- package/src/vcs-version/VcsTag.ts +9 -0
- package/src/vcs-version/index.js +61 -0
- package/src/vcs-version/loadVcsTagEffect.js +27 -0
- package/src/vcs-version/rerunAtRevisionEffect.js +25 -0
- package/src/vcs-version/resolveWorkflowAtRevisionEffect.js +32 -0
- package/src/vcs-version/tagSnapshotVcsEffect.js +56 -0
- package/src/writeRewindAuditRow.js +46 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { resolveRewindAuditClient } from "./resolveRewindAuditClient.js";
|
|
2
|
+
|
|
3
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
4
|
+
/** @typedef {import("./RewindAuditResult.ts").RewindAuditResult} RewindAuditResult */
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @typedef {{
|
|
8
|
+
* id: number;
|
|
9
|
+
* runId: string;
|
|
10
|
+
* fromFrameNo: number;
|
|
11
|
+
* toFrameNo: number;
|
|
12
|
+
* caller: string;
|
|
13
|
+
* timestampMs: number;
|
|
14
|
+
* result: RewindAuditResult;
|
|
15
|
+
* durationMs: number | null;
|
|
16
|
+
* }} RewindAuditRow
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {Array<Record<string, unknown>>} rows
|
|
21
|
+
* @returns {Array<RewindAuditRow>}
|
|
22
|
+
*/
|
|
23
|
+
function mapRewindAuditRows(rows) {
|
|
24
|
+
return rows.map((row) => ({
|
|
25
|
+
id: Number(row.id),
|
|
26
|
+
runId: String(row.runId),
|
|
27
|
+
fromFrameNo: Number(row.fromFrameNo),
|
|
28
|
+
toFrameNo: Number(row.toFrameNo),
|
|
29
|
+
caller: String(row.caller),
|
|
30
|
+
timestampMs: Number(row.timestampMs),
|
|
31
|
+
result: /** @type {RewindAuditResult} */ (String(row.result)),
|
|
32
|
+
durationMs: row.durationMs == null ? null : Number(row.durationMs),
|
|
33
|
+
}));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Fetch audit rows for tests and diagnostics.
|
|
38
|
+
*
|
|
39
|
+
* @param {SmithersDb} adapter
|
|
40
|
+
* @param {{ runId?: string; limit?: number; }} [input]
|
|
41
|
+
* @returns {Promise<Array<RewindAuditRow>>}
|
|
42
|
+
*/
|
|
43
|
+
export async function listRewindAuditRows(adapter, input = {}) {
|
|
44
|
+
const client = resolveRewindAuditClient(adapter);
|
|
45
|
+
const limit = Number.isInteger(input.limit) ? Math.max(1, Number(input.limit)) : 100;
|
|
46
|
+
if (typeof input.runId === "string") {
|
|
47
|
+
const rows = client
|
|
48
|
+
.query(
|
|
49
|
+
`SELECT
|
|
50
|
+
id,
|
|
51
|
+
run_id AS runId,
|
|
52
|
+
from_frame_no AS fromFrameNo,
|
|
53
|
+
to_frame_no AS toFrameNo,
|
|
54
|
+
caller,
|
|
55
|
+
timestamp_ms AS timestampMs,
|
|
56
|
+
result,
|
|
57
|
+
duration_ms AS durationMs
|
|
58
|
+
FROM _smithers_time_travel_audit
|
|
59
|
+
WHERE run_id = ?
|
|
60
|
+
ORDER BY id ASC
|
|
61
|
+
LIMIT ?`,
|
|
62
|
+
)
|
|
63
|
+
.all(input.runId, limit);
|
|
64
|
+
return mapRewindAuditRows(rows);
|
|
65
|
+
}
|
|
66
|
+
const rows = client
|
|
67
|
+
.query(
|
|
68
|
+
`SELECT
|
|
69
|
+
id,
|
|
70
|
+
run_id AS runId,
|
|
71
|
+
from_frame_no AS fromFrameNo,
|
|
72
|
+
to_frame_no AS toFrameNo,
|
|
73
|
+
caller,
|
|
74
|
+
timestamp_ms AS timestampMs,
|
|
75
|
+
result,
|
|
76
|
+
duration_ms AS durationMs
|
|
77
|
+
FROM _smithers_time_travel_audit
|
|
78
|
+
ORDER BY id ASC
|
|
79
|
+
LIMIT ?`,
|
|
80
|
+
)
|
|
81
|
+
.all(limit);
|
|
82
|
+
return mapRewindAuditRows(rows);
|
|
83
|
+
}
|
package/src/metrics.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { resolveRewindAuditClient } from "./resolveRewindAuditClient.js";
|
|
2
|
+
|
|
3
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* On startup, find rewind audit rows left in `in_progress` by a prior crash,
|
|
7
|
+
* mark them as `partial`, and flag the associated runs as `needs_attention`.
|
|
8
|
+
*
|
|
9
|
+
* @param {SmithersDb} adapter
|
|
10
|
+
* @param {{ nowMs?: () => number }} [options]
|
|
11
|
+
* @returns {Promise<{ recovered: Array<{ id: number; runId: string }> }>}
|
|
12
|
+
*/
|
|
13
|
+
export async function recoverInProgressRewindAudits(adapter, options = {}) {
|
|
14
|
+
const nowMs = options.nowMs ?? (() => Date.now());
|
|
15
|
+
const client = resolveRewindAuditClient(adapter);
|
|
16
|
+
const rows = /** @type {Array<{ id: number; runId: string; timestampMs: number }>} */ (
|
|
17
|
+
client
|
|
18
|
+
.query(
|
|
19
|
+
`SELECT id, run_id AS runId, timestamp_ms AS timestampMs
|
|
20
|
+
FROM _smithers_time_travel_audit
|
|
21
|
+
WHERE result = 'in_progress'`,
|
|
22
|
+
)
|
|
23
|
+
.all()
|
|
24
|
+
);
|
|
25
|
+
if (rows.length === 0) {
|
|
26
|
+
return { recovered: [] };
|
|
27
|
+
}
|
|
28
|
+
const now = nowMs();
|
|
29
|
+
const updateStmt = client.query(
|
|
30
|
+
`UPDATE _smithers_time_travel_audit
|
|
31
|
+
SET result = 'partial',
|
|
32
|
+
duration_ms = COALESCE(duration_ms, ?)
|
|
33
|
+
WHERE id = ?`,
|
|
34
|
+
);
|
|
35
|
+
const recovered = [];
|
|
36
|
+
for (const row of rows) {
|
|
37
|
+
const duration = Math.max(0, now - Number(row.timestampMs ?? now));
|
|
38
|
+
updateStmt.run(duration, row.id);
|
|
39
|
+
try {
|
|
40
|
+
const payload = JSON.stringify({
|
|
41
|
+
code: "RewindFailed",
|
|
42
|
+
needsAttention: true,
|
|
43
|
+
message: `Rewind audit ${row.id} was in_progress at startup; marked partial.`,
|
|
44
|
+
timestampMs: now,
|
|
45
|
+
});
|
|
46
|
+
await adapter.updateRun(row.runId, {
|
|
47
|
+
status: "needs_attention",
|
|
48
|
+
heartbeatAtMs: null,
|
|
49
|
+
runtimeOwnerId: null,
|
|
50
|
+
errorJson: payload,
|
|
51
|
+
});
|
|
52
|
+
} catch {
|
|
53
|
+
try {
|
|
54
|
+
await adapter.updateRun(row.runId, {
|
|
55
|
+
status: "failed",
|
|
56
|
+
heartbeatAtMs: null,
|
|
57
|
+
runtimeOwnerId: null,
|
|
58
|
+
errorJson: JSON.stringify({
|
|
59
|
+
code: "RewindFailed",
|
|
60
|
+
needsAttention: true,
|
|
61
|
+
message: "Rewind was in_progress at startup.",
|
|
62
|
+
timestampMs: now,
|
|
63
|
+
}),
|
|
64
|
+
});
|
|
65
|
+
} catch {
|
|
66
|
+
// best-effort: nothing to do if the run row was deleted.
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
recovered.push({ id: row.id, runId: row.runId });
|
|
70
|
+
}
|
|
71
|
+
return { recovered };
|
|
72
|
+
}
|
package/src/replay.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// @smithers-type-exports-begin
|
|
2
|
+
/** @typedef {import("./ReplayResult.ts").ReplayResult} ReplayResult */
|
|
3
|
+
// @smithers-type-exports-end
|
|
4
|
+
|
|
5
|
+
import { Effect } from "effect";
|
|
6
|
+
import * as BunContext from "@effect/platform-bun/BunContext";
|
|
7
|
+
import { replayFromCheckpoint as replayFromCheckpointEffect } from "./replayFromCheckpointEffect.js";
|
|
8
|
+
export { replayFromCheckpointEffect };
|
|
9
|
+
|
|
10
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
11
|
+
/** @typedef {import("./ReplayParams.ts").ReplayParams} ReplayParams */
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Fork a run from a checkpoint and optionally restore the VCS working copy.
|
|
15
|
+
*
|
|
16
|
+
* @param {SmithersDb} adapter
|
|
17
|
+
* @param {ReplayParams} params
|
|
18
|
+
* @returns {Promise<ReplayResult>}
|
|
19
|
+
*/
|
|
20
|
+
export function replayFromCheckpoint(adapter, params) {
|
|
21
|
+
return Effect.runPromise(replayFromCheckpointEffect(adapter, params).pipe(Effect.provide(BunContext.layer)));
|
|
22
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Effect, Metric } from "effect";
|
|
2
|
+
import { forkRun as forkRunEffect } from "./fork/forkRunEffect.js";
|
|
3
|
+
import { rerunAtRevision as rerunAtRevisionEffect } from "./vcs-version/rerunAtRevisionEffect.js";
|
|
4
|
+
import { replaysStarted } from "./replaysStarted.js";
|
|
5
|
+
|
|
6
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
7
|
+
/** @typedef {import("./ReplayParams.ts").ReplayParams} ReplayParams */
|
|
8
|
+
/** @typedef {import("./ReplayResult.ts").ReplayResult} ReplayResult */
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Fork a run from a checkpoint, optionally restore the VCS working copy
|
|
12
|
+
* to the revision that was active at the source frame, then return the
|
|
13
|
+
* new run metadata so the caller can resume execution.
|
|
14
|
+
*
|
|
15
|
+
* @param {SmithersDb} adapter
|
|
16
|
+
* @param {ReplayParams} params
|
|
17
|
+
*/
|
|
18
|
+
export function replayFromCheckpoint(adapter, params) {
|
|
19
|
+
return Effect.gen(function* () {
|
|
20
|
+
const { parentRunId, frameNo, inputOverrides, resetNodes, branchLabel, restoreVcs, cwd, } = params;
|
|
21
|
+
// 1. Fork the run
|
|
22
|
+
const { runId, branch, snapshot } = yield* forkRunEffect(adapter, {
|
|
23
|
+
parentRunId,
|
|
24
|
+
frameNo,
|
|
25
|
+
inputOverrides,
|
|
26
|
+
resetNodes,
|
|
27
|
+
branchLabel,
|
|
28
|
+
forkDescription: `Replay from ${parentRunId}:${frameNo}`,
|
|
29
|
+
});
|
|
30
|
+
// 2. Optionally restore VCS state
|
|
31
|
+
let vcsRestored = false;
|
|
32
|
+
let vcsPointer = null;
|
|
33
|
+
let vcsError;
|
|
34
|
+
if (restoreVcs) {
|
|
35
|
+
const vcsResult = yield* rerunAtRevisionEffect(adapter, parentRunId, frameNo, { cwd });
|
|
36
|
+
vcsRestored = vcsResult.restored;
|
|
37
|
+
vcsPointer = vcsResult.vcsPointer;
|
|
38
|
+
vcsError = vcsResult.error;
|
|
39
|
+
}
|
|
40
|
+
yield* Metric.increment(replaysStarted);
|
|
41
|
+
yield* Effect.logInfo("Replay started").pipe(Effect.annotateLogs({
|
|
42
|
+
parentRunId,
|
|
43
|
+
parentFrameNo: String(frameNo),
|
|
44
|
+
childRunId: runId,
|
|
45
|
+
vcsRestored: String(vcsRestored),
|
|
46
|
+
}));
|
|
47
|
+
return {
|
|
48
|
+
runId,
|
|
49
|
+
branch,
|
|
50
|
+
snapshot,
|
|
51
|
+
vcsRestored,
|
|
52
|
+
vcsPointer,
|
|
53
|
+
vcsError,
|
|
54
|
+
};
|
|
55
|
+
}).pipe(Effect.annotateLogs({
|
|
56
|
+
parentRunId: params.parentRunId,
|
|
57
|
+
parentFrameNo: String(params.frameNo),
|
|
58
|
+
}), Effect.withLogSpan("time-travel:replay-from-checkpoint"));
|
|
59
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Minimal Bun SQLite client surface used by the rewind audit helpers.
|
|
5
|
+
* This mirrors the shape returned by `bun:sqlite`'s `Database.query()`
|
|
6
|
+
* and is sufficient for the raw SQL writes / reads we perform against
|
|
7
|
+
* the `_smithers_time_travel_audit` and `_smithers_frames` tables.
|
|
8
|
+
*
|
|
9
|
+
* @typedef {{
|
|
10
|
+
* query: (sql: string) => {
|
|
11
|
+
* run: (...args: unknown[]) => unknown;
|
|
12
|
+
* get: (...args: unknown[]) => Record<string, unknown> | null | undefined;
|
|
13
|
+
* all: (...args: unknown[]) => Array<Record<string, unknown>>;
|
|
14
|
+
* };
|
|
15
|
+
* }} RewindAuditSqliteClient
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Resolve the Bun SQLite client from a {@link SmithersDb} instance for audit writes.
|
|
20
|
+
*
|
|
21
|
+
* @param {SmithersDb} adapter
|
|
22
|
+
* @returns {RewindAuditSqliteClient}
|
|
23
|
+
*/
|
|
24
|
+
export function resolveRewindAuditClient(adapter) {
|
|
25
|
+
const db = /** @type {{ session?: { client?: unknown }; $client?: unknown } | null | undefined} */ (
|
|
26
|
+
/** @type {unknown} */ (adapter?.db)
|
|
27
|
+
);
|
|
28
|
+
const candidate = /** @type {unknown} */ (db?.session?.client ?? db?.$client);
|
|
29
|
+
if (
|
|
30
|
+
!candidate ||
|
|
31
|
+
typeof (/** @type {{ query?: unknown }} */ (candidate).query) !== "function"
|
|
32
|
+
) {
|
|
33
|
+
throw new TypeError(
|
|
34
|
+
"Could not resolve a Bun SQLite client for rewind audit writes.",
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
return /** @type {RewindAuditSqliteClient} */ (candidate);
|
|
38
|
+
}
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { nowMs } from "@smithers-orchestrator/scheduler/nowMs";
|
|
3
|
+
/** @typedef {import("./RetryTaskOptions.ts").RetryTaskOptions} RetryTaskOptions */
|
|
4
|
+
/** @typedef {import("./RetryTaskResult.ts").RetryTaskResult} RetryTaskResult */
|
|
5
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {string} nodeId
|
|
9
|
+
* @param {number} iteration
|
|
10
|
+
*/
|
|
11
|
+
function buildNodeKey(nodeId, iteration) {
|
|
12
|
+
return `${nodeId}::${iteration}`;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* @param {Array<{ nodeId: string; iteration: number }>} nodes
|
|
16
|
+
* @returns {string[]}
|
|
17
|
+
*/
|
|
18
|
+
function uniqueNodeIds(nodes) {
|
|
19
|
+
const seen = new Set();
|
|
20
|
+
const result = [];
|
|
21
|
+
for (const node of nodes) {
|
|
22
|
+
if (seen.has(node.nodeId))
|
|
23
|
+
continue;
|
|
24
|
+
seen.add(node.nodeId);
|
|
25
|
+
result.push(node.nodeId);
|
|
26
|
+
}
|
|
27
|
+
return result;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* @param {string | null | undefined} status
|
|
31
|
+
*/
|
|
32
|
+
function isActiveRunStatus(status) {
|
|
33
|
+
return (status === "running" ||
|
|
34
|
+
status === "waiting-approval" ||
|
|
35
|
+
status === "waiting-event" ||
|
|
36
|
+
status === "waiting-timer");
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* @param {SmithersDb} adapter
|
|
40
|
+
* @param {Required<Pick<RetryTaskOptions, "runId" | "resetDependents">> & { targetNode: any; }} opts
|
|
41
|
+
*/
|
|
42
|
+
async function resolveResetNodes(adapter, opts) {
|
|
43
|
+
const { runId, targetNode, resetDependents } = opts;
|
|
44
|
+
if (!resetDependents) {
|
|
45
|
+
return [targetNode];
|
|
46
|
+
}
|
|
47
|
+
const nodes = await adapter.listNodes(runId);
|
|
48
|
+
const attempts = await adapter.listAttemptsForRun(runId);
|
|
49
|
+
const attemptOrder = new Map();
|
|
50
|
+
for (let index = 0; index < attempts.length; index += 1) {
|
|
51
|
+
const attempt = attempts[index];
|
|
52
|
+
attemptOrder.set(buildNodeKey(attempt.nodeId, attempt.iteration ?? 0), index);
|
|
53
|
+
}
|
|
54
|
+
const targetKey = buildNodeKey(targetNode.nodeId, targetNode.iteration ?? 0);
|
|
55
|
+
const targetOrder = attemptOrder.get(targetKey);
|
|
56
|
+
const targetIteration = targetNode.iteration ?? 0;
|
|
57
|
+
const targetUpdatedAtMs = targetNode.updatedAtMs ?? 0;
|
|
58
|
+
return nodes.filter((node) => {
|
|
59
|
+
const nodeIteration = node.iteration ?? 0;
|
|
60
|
+
const nodeKey = buildNodeKey(node.nodeId, nodeIteration);
|
|
61
|
+
if (nodeKey === targetKey)
|
|
62
|
+
return true;
|
|
63
|
+
if (nodeIteration > targetIteration)
|
|
64
|
+
return true;
|
|
65
|
+
const nodeOrder = attemptOrder.get(nodeKey);
|
|
66
|
+
if (targetOrder !== undefined && nodeOrder !== undefined) {
|
|
67
|
+
return nodeOrder > targetOrder;
|
|
68
|
+
}
|
|
69
|
+
return (node.updatedAtMs ?? 0) > targetUpdatedAtMs;
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* @param {RetryTaskOptions} opts
|
|
74
|
+
* @param {{ runId: string; nodeId: string; iteration: number; resetNodes: string[]; success: boolean; error?: string; }} payload
|
|
75
|
+
*/
|
|
76
|
+
function emitRetryFinished(opts, payload) {
|
|
77
|
+
opts.onProgress?.({
|
|
78
|
+
type: "RetryTaskFinished",
|
|
79
|
+
...payload,
|
|
80
|
+
timestampMs: nowMs(),
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* @param {SmithersDb} adapter
|
|
85
|
+
* @param {RetryTaskOptions} opts
|
|
86
|
+
* @returns {Promise<RetryTaskResult>}
|
|
87
|
+
*/
|
|
88
|
+
export async function retryTask(adapter, opts) {
|
|
89
|
+
const runId = opts.runId;
|
|
90
|
+
const nodeId = opts.nodeId;
|
|
91
|
+
const iteration = opts.iteration ?? 0;
|
|
92
|
+
const resetDependents = opts.resetDependents ?? true;
|
|
93
|
+
const force = opts.force ?? false;
|
|
94
|
+
const node = await adapter.getNode(runId, nodeId, iteration);
|
|
95
|
+
if (!node) {
|
|
96
|
+
const error = `Node not found: ${runId}/${nodeId}/${iteration}`;
|
|
97
|
+
emitRetryFinished(opts, {
|
|
98
|
+
runId,
|
|
99
|
+
nodeId,
|
|
100
|
+
iteration,
|
|
101
|
+
resetNodes: [],
|
|
102
|
+
success: false,
|
|
103
|
+
error,
|
|
104
|
+
});
|
|
105
|
+
return { success: false, resetNodes: [], error };
|
|
106
|
+
}
|
|
107
|
+
const run = await adapter.getRun(runId);
|
|
108
|
+
if (!run) {
|
|
109
|
+
const error = `Run not found: ${runId}`;
|
|
110
|
+
emitRetryFinished(opts, {
|
|
111
|
+
runId,
|
|
112
|
+
nodeId,
|
|
113
|
+
iteration,
|
|
114
|
+
resetNodes: [],
|
|
115
|
+
success: false,
|
|
116
|
+
error,
|
|
117
|
+
});
|
|
118
|
+
return { success: false, resetNodes: [], error };
|
|
119
|
+
}
|
|
120
|
+
if (!force && isActiveRunStatus(run.status)) {
|
|
121
|
+
const error = `Run is still running: ${runId}`;
|
|
122
|
+
emitRetryFinished(opts, {
|
|
123
|
+
runId,
|
|
124
|
+
nodeId,
|
|
125
|
+
iteration,
|
|
126
|
+
resetNodes: [],
|
|
127
|
+
success: false,
|
|
128
|
+
error,
|
|
129
|
+
});
|
|
130
|
+
return { success: false, resetNodes: [], error };
|
|
131
|
+
}
|
|
132
|
+
const resetNodes = await resolveResetNodes(adapter, {
|
|
133
|
+
runId,
|
|
134
|
+
targetNode: node,
|
|
135
|
+
resetDependents,
|
|
136
|
+
});
|
|
137
|
+
const resetNodeIds = uniqueNodeIds(resetNodes.map((candidate) => ({
|
|
138
|
+
nodeId: candidate.nodeId,
|
|
139
|
+
iteration: candidate.iteration ?? 0,
|
|
140
|
+
})));
|
|
141
|
+
const attemptsByNode = new Map();
|
|
142
|
+
for (const resetNode of resetNodes) {
|
|
143
|
+
const resetIteration = resetNode.iteration ?? 0;
|
|
144
|
+
attemptsByNode.set(buildNodeKey(resetNode.nodeId, resetIteration), await adapter.listAttempts(runId, resetNode.nodeId, resetIteration));
|
|
145
|
+
}
|
|
146
|
+
opts.onProgress?.({
|
|
147
|
+
type: "RetryTaskStarted",
|
|
148
|
+
runId,
|
|
149
|
+
nodeId,
|
|
150
|
+
iteration,
|
|
151
|
+
resetDependents,
|
|
152
|
+
resetNodes: resetNodeIds,
|
|
153
|
+
timestampMs: nowMs(),
|
|
154
|
+
});
|
|
155
|
+
const resetTimestampMs = nowMs();
|
|
156
|
+
await adapter.withTransaction("retry-task-reset", Effect.gen(function* () {
|
|
157
|
+
for (const resetNode of resetNodes) {
|
|
158
|
+
const resetIteration = resetNode.iteration ?? 0;
|
|
159
|
+
const attempts = attemptsByNode.get(buildNodeKey(resetNode.nodeId, resetIteration)) ??
|
|
160
|
+
[];
|
|
161
|
+
for (const attempt of attempts) {
|
|
162
|
+
if (attempt.state !== "failed" &&
|
|
163
|
+
attempt.state !== "in-progress" &&
|
|
164
|
+
attempt.state !== "waiting-approval" &&
|
|
165
|
+
attempt.state !== "waiting-event" &&
|
|
166
|
+
attempt.state !== "waiting-timer") {
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
const patch = { state: "cancelled" };
|
|
170
|
+
if (attempt.finishedAtMs == null) {
|
|
171
|
+
patch.finishedAtMs = resetTimestampMs;
|
|
172
|
+
}
|
|
173
|
+
yield* adapter.updateAttemptEffect(runId, resetNode.nodeId, resetIteration, attempt.attempt, patch);
|
|
174
|
+
}
|
|
175
|
+
if (resetNode.outputTable) {
|
|
176
|
+
yield* adapter.deleteOutputRowEffect(resetNode.outputTable, {
|
|
177
|
+
runId,
|
|
178
|
+
nodeId: resetNode.nodeId,
|
|
179
|
+
iteration: resetIteration,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
yield* adapter.insertNodeEffect({
|
|
183
|
+
runId,
|
|
184
|
+
nodeId: resetNode.nodeId,
|
|
185
|
+
iteration: resetIteration,
|
|
186
|
+
state: "pending",
|
|
187
|
+
lastAttempt: resetNode.lastAttempt ?? null,
|
|
188
|
+
updatedAtMs: resetTimestampMs,
|
|
189
|
+
outputTable: resetNode.outputTable ?? "",
|
|
190
|
+
label: resetNode.label ?? null,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
yield* adapter.updateRunEffect(runId, {
|
|
194
|
+
status: "running",
|
|
195
|
+
finishedAtMs: null,
|
|
196
|
+
heartbeatAtMs: null,
|
|
197
|
+
runtimeOwnerId: null,
|
|
198
|
+
cancelRequestedAtMs: null,
|
|
199
|
+
hijackRequestedAtMs: null,
|
|
200
|
+
hijackTarget: null,
|
|
201
|
+
errorJson: null,
|
|
202
|
+
});
|
|
203
|
+
}));
|
|
204
|
+
emitRetryFinished(opts, {
|
|
205
|
+
runId,
|
|
206
|
+
nodeId,
|
|
207
|
+
iteration,
|
|
208
|
+
resetNodes: resetNodeIds,
|
|
209
|
+
success: true,
|
|
210
|
+
});
|
|
211
|
+
return {
|
|
212
|
+
success: true,
|
|
213
|
+
resetNodes: resetNodeIds,
|
|
214
|
+
};
|
|
215
|
+
}
|
package/src/revert.js
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { revertToJjPointer } from "@smithers-orchestrator/vcs/jj";
|
|
3
|
+
import * as BunContext from "@effect/platform-bun/BunContext";
|
|
4
|
+
import { nowMs } from "@smithers-orchestrator/scheduler/nowMs";
|
|
5
|
+
/** @typedef {import("./RevertOptions.ts").RevertOptions} RevertOptions */
|
|
6
|
+
/** @typedef {import("./RevertResult.ts").RevertResult} RevertResult */
|
|
7
|
+
/** @typedef {import("@smithers-orchestrator/db/adapter").SmithersDb} SmithersDb */
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {SmithersDb} adapter
|
|
11
|
+
* @param {RevertOptions} opts
|
|
12
|
+
* @returns {Promise<RevertResult>}
|
|
13
|
+
*/
|
|
14
|
+
export async function revertToAttempt(adapter, opts) {
|
|
15
|
+
const { runId, nodeId, iteration, attempt, onProgress } = opts;
|
|
16
|
+
const attemptRow = await Effect.runPromise(adapter.getAttempt(runId, nodeId, iteration, attempt));
|
|
17
|
+
if (!attemptRow) {
|
|
18
|
+
return {
|
|
19
|
+
success: false,
|
|
20
|
+
error: `Attempt not found: ${runId}/${nodeId}/${iteration}/${attempt}`,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
const jjPointer = attemptRow.jjPointer;
|
|
24
|
+
if (!jjPointer) {
|
|
25
|
+
return { success: false, error: `Attempt has no jjPointer recorded` };
|
|
26
|
+
}
|
|
27
|
+
onProgress?.({
|
|
28
|
+
type: "RevertStarted",
|
|
29
|
+
runId,
|
|
30
|
+
nodeId,
|
|
31
|
+
iteration,
|
|
32
|
+
attempt,
|
|
33
|
+
jjPointer,
|
|
34
|
+
timestampMs: nowMs(),
|
|
35
|
+
});
|
|
36
|
+
// Revert must target the same repository/worktree where the attempt ran.
|
|
37
|
+
const cwd = attemptRow.jjCwd ?? undefined;
|
|
38
|
+
const result = await Effect.runPromise(revertToJjPointer(jjPointer, cwd).pipe(Effect.provide(BunContext.layer)));
|
|
39
|
+
onProgress?.({
|
|
40
|
+
type: "RevertFinished",
|
|
41
|
+
runId,
|
|
42
|
+
nodeId,
|
|
43
|
+
iteration,
|
|
44
|
+
attempt,
|
|
45
|
+
jjPointer,
|
|
46
|
+
success: result.success,
|
|
47
|
+
error: result.error,
|
|
48
|
+
timestampMs: nowMs(),
|
|
49
|
+
});
|
|
50
|
+
if (!result.success) {
|
|
51
|
+
return { success: false, error: result.error, jjPointer };
|
|
52
|
+
}
|
|
53
|
+
// Clean up DB frames recorded after the reverted attempt started.
|
|
54
|
+
// Find the latest frame created before the attempt's start time and
|
|
55
|
+
// discard everything after it so the DB matches the reverted VCS state.
|
|
56
|
+
const frames = await Effect.runPromise(adapter.listFrames(runId, 1_000_000));
|
|
57
|
+
const cutoff = attemptRow.startedAtMs;
|
|
58
|
+
let lastValidFrameNo = -1;
|
|
59
|
+
for (const f of frames) {
|
|
60
|
+
if (f.createdAtMs <= cutoff && f.frameNo > lastValidFrameNo) {
|
|
61
|
+
lastValidFrameNo = f.frameNo;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (lastValidFrameNo >= 0) {
|
|
65
|
+
await Effect.runPromise(adapter.deleteFramesAfter(runId, lastValidFrameNo));
|
|
66
|
+
}
|
|
67
|
+
return { success: true, jjPointer };
|
|
68
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// @smithers-type-exports-begin
|
|
2
|
+
/** @typedef {import("./RewindAuditResult.ts").RewindAuditResult} RewindAuditResult */
|
|
3
|
+
// @smithers-type-exports-end
|
|
4
|
+
|
|
5
|
+
export { writeRewindAuditRow } from "./writeRewindAuditRow.js";
|
|
6
|
+
export { updateRewindAuditRow } from "./updateRewindAuditRow.js";
|
|
7
|
+
export { countRecentRewindAuditRows } from "./countRecentRewindAuditRows.js";
|
|
8
|
+
export { listRewindAuditRows } from "./listRewindAuditRows.js";
|
|
9
|
+
export { recoverInProgressRewindAudits } from "./recoverInProgressRewindAudits.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// @smithers-type-exports-begin
|
|
2
|
+
/** @typedef {import("./RewindLockHandle.ts").RewindLockHandle} RewindLockHandle */
|
|
3
|
+
// @smithers-type-exports-end
|
|
4
|
+
|
|
5
|
+
export { acquireRewindLock } from "./acquireRewindLock.js";
|
|
6
|
+
export { hasRewindLock } from "./hasRewindLock.js";
|
|
7
|
+
export { resetRewindLocksForTests } from "./resetRewindLocksForTests.js";
|
package/src/schema.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { integer, sqliteTable, text, primaryKey, } from "drizzle-orm/sqlite-core";
|
|
2
|
+
/**
|
|
3
|
+
* Full state snapshot captured at each frame commit.
|
|
4
|
+
* PK: (run_id, frame_no)
|
|
5
|
+
*/
|
|
6
|
+
export const smithersSnapshots = sqliteTable("_smithers_snapshots", {
|
|
7
|
+
runId: text("run_id").notNull(),
|
|
8
|
+
frameNo: integer("frame_no").notNull(),
|
|
9
|
+
nodesJson: text("nodes_json").notNull(),
|
|
10
|
+
outputsJson: text("outputs_json").notNull(),
|
|
11
|
+
ralphJson: text("ralph_json").notNull(),
|
|
12
|
+
inputJson: text("input_json").notNull(),
|
|
13
|
+
vcsPointer: text("vcs_pointer"),
|
|
14
|
+
workflowHash: text("workflow_hash"),
|
|
15
|
+
contentHash: text("content_hash").notNull(),
|
|
16
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
17
|
+
}, (t) => ({
|
|
18
|
+
pk: primaryKey({ columns: [t.runId, t.frameNo] }),
|
|
19
|
+
}));
|
|
20
|
+
/**
|
|
21
|
+
* Parent-child fork relationships between runs.
|
|
22
|
+
* PK: run_id (the child run)
|
|
23
|
+
*/
|
|
24
|
+
export const smithersBranches = sqliteTable("_smithers_branches", {
|
|
25
|
+
runId: text("run_id").primaryKey(),
|
|
26
|
+
parentRunId: text("parent_run_id").notNull(),
|
|
27
|
+
parentFrameNo: integer("parent_frame_no").notNull(),
|
|
28
|
+
branchLabel: text("branch_label"),
|
|
29
|
+
forkDescription: text("fork_description"),
|
|
30
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
31
|
+
});
|
|
32
|
+
/**
|
|
33
|
+
* VCS revision metadata per snapshot.
|
|
34
|
+
* PK: (run_id, frame_no)
|
|
35
|
+
*/
|
|
36
|
+
export const smithersVcsTags = sqliteTable("_smithers_vcs_tags", {
|
|
37
|
+
runId: text("run_id").notNull(),
|
|
38
|
+
frameNo: integer("frame_no").notNull(),
|
|
39
|
+
vcsType: text("vcs_type").notNull(),
|
|
40
|
+
vcsPointer: text("vcs_pointer").notNull(),
|
|
41
|
+
vcsRoot: text("vcs_root"),
|
|
42
|
+
jjOperationId: text("jj_operation_id"),
|
|
43
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
44
|
+
}, (t) => ({
|
|
45
|
+
pk: primaryKey({ columns: [t.runId, t.frameNo] }),
|
|
46
|
+
}));
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialized snapshot of workflow state at a specific frame.
|
|
3
|
+
*/
|
|
4
|
+
export type Snapshot = {
|
|
5
|
+
runId: string;
|
|
6
|
+
frameNo: number;
|
|
7
|
+
nodesJson: string;
|
|
8
|
+
outputsJson: string;
|
|
9
|
+
ralphJson: string;
|
|
10
|
+
inputJson: string;
|
|
11
|
+
vcsPointer: string | null;
|
|
12
|
+
workflowHash: string | null;
|
|
13
|
+
contentHash: string;
|
|
14
|
+
createdAtMs: number;
|
|
15
|
+
};
|