ai-foreman 1.7.19 → 1.7.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/dist/branch/runner.js +2 -1
- package/dist/buildAdmission.d.ts +61 -0
- package/dist/buildAdmission.js +284 -0
- package/dist/buildApproval.d.ts +2 -0
- package/dist/buildApproval.js +17 -0
- package/dist/buildLaunchGate.d.ts +1 -0
- package/dist/buildLaunchGate.js +36 -0
- package/dist/buildRuns.d.ts +18 -1
- package/dist/buildRuns.js +198 -58
- package/dist/cli/runtimeAuthPrompt.d.ts +5 -0
- package/dist/cli/runtimeAuthPrompt.js +16 -2
- package/dist/cli/start.d.ts +8 -0
- package/dist/cli/start.js +1980 -1842
- package/dist/continuity.d.ts +5 -1
- package/dist/continuity.js +46 -6
- package/dist/foreman.d.ts +2 -2
- package/dist/foreman.js +79 -33
- package/dist/humanDecision.d.ts +1 -1
- package/dist/humanDecision.js +4 -3
- package/dist/index.js +2 -1
- package/dist/processIdentity.d.ts +21 -1
- package/dist/processIdentity.js +80 -5
- package/dist/qaReview.js +59 -17
- package/dist/readinessCleanup.d.ts +19 -0
- package/dist/readinessCleanup.js +115 -0
- package/dist/readinessGate.d.ts +1 -0
- package/dist/readinessGate.js +82 -0
- package/dist/recoveryGuidance.d.ts +8 -0
- package/dist/recoveryGuidance.js +25 -0
- package/dist/runtimeReadiness.d.ts +12 -0
- package/dist/runtimeReadiness.js +194 -10
- package/dist/stateTransfer.js +35 -7
- package/dist/supervisedStart.d.ts +6 -0
- package/dist/supervisedStart.js +160 -54
- package/dist/supervisor.d.ts +2 -0
- package/dist/supervisor.js +26 -2
- package/dist/windowsProbeJob.d.ts +12 -0
- package/dist/windowsProbeJob.js +144 -0
- package/dist/workflowDb.d.ts +70 -1
- package/dist/workflowDb.js +538 -25
- package/dist/workflowReader.d.ts +7 -0
- package/dist/workflowReader.js +83 -0
- package/package.json +8 -4
package/dist/buildRuns.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
export { withBuildInvocation as withBuildRecoveryInvocation, launchDigest as launchArgumentDigest } from "./buildAdmission.js";
|
|
1
2
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { hostname } from "node:os";
|
|
4
5
|
import { join, resolve } from "node:path";
|
|
5
6
|
import { execFileSync } from "node:child_process";
|
|
@@ -12,11 +13,26 @@ import { captureCurrentWorkflowSessionIdentity, currentWorkflowIdentityKey } fro
|
|
|
12
13
|
import { isTicketsInitialized, loadTicketsConfig, resolveTicketPaths } from "./tickets/config.js";
|
|
13
14
|
import { loadTickets } from "./tickets/ticketLoader.js";
|
|
14
15
|
import { loadProjectAutonomyConfig, resolveAutonomyPolicy } from "./recoveryPolicy.js";
|
|
15
|
-
import {
|
|
16
|
+
import { classifyProcess, processStartIdentity } from "./processIdentity.js";
|
|
16
17
|
export const BUILD_RUN_VERSION = 3;
|
|
17
18
|
export const BUILD_RUN_DIRECTORY = ".foreman/runs";
|
|
18
19
|
export const BUILD_HEARTBEAT_MS = 10_000;
|
|
19
20
|
export const BUILD_LEASE_STALE_MS = 45_000;
|
|
21
|
+
const buildAuthority = Symbol("buildAuthority");
|
|
22
|
+
export function bindBuildAuthority(run, lease) {
|
|
23
|
+
return { ...run, [buildAuthority]: lease };
|
|
24
|
+
}
|
|
25
|
+
function assertBuildAuthority(workflow, run) {
|
|
26
|
+
const token = run[buildAuthority];
|
|
27
|
+
if (!token)
|
|
28
|
+
throw new Error("build mutation requires the caller's original workflow authority");
|
|
29
|
+
const held = workflow.currentLease();
|
|
30
|
+
const authorizedSupersession = run.status === "superseded" && run.supersededBy === held?.runId;
|
|
31
|
+
if (token && (!held || held.owner !== token.owner || held.generation !== token.generation || held.runId !== token.runId))
|
|
32
|
+
throw new Error("workflow lease ownership changed");
|
|
33
|
+
if (held && (!token || (held.runId !== run.runId && !authorizedSupersession)))
|
|
34
|
+
throw new Error("build mutation requires the caller's original workflow authority");
|
|
35
|
+
}
|
|
20
36
|
export function createBuildRun(input) {
|
|
21
37
|
const now = input.now ?? new Date();
|
|
22
38
|
// Complete the one-time legacy import before the new mutable build snapshot
|
|
@@ -61,45 +77,36 @@ export function createBuildRun(input) {
|
|
|
61
77
|
createdAt: stamp,
|
|
62
78
|
updatedAt: stamp,
|
|
63
79
|
};
|
|
64
|
-
const saved = saveBuildRun(input.repositoryRoot, run, now);
|
|
65
80
|
const workflow = new WorkflowDb(input.repositoryRoot);
|
|
66
81
|
try {
|
|
67
|
-
workflow.
|
|
82
|
+
workflow.ensureRun(run.runId, "build", now);
|
|
83
|
+
const authority = workflow.acquireLease(run.runId, undefined, now, BUILD_LEASE_STALE_MS);
|
|
84
|
+
return saveBuildRun(input.repositoryRoot, bindBuildAuthority(run, authority), now);
|
|
68
85
|
}
|
|
69
86
|
finally {
|
|
70
87
|
workflow.close();
|
|
71
88
|
}
|
|
72
|
-
return saved;
|
|
73
89
|
}
|
|
74
90
|
export function resumeBuildRun(projectDir, runId, patch, now = new Date()) {
|
|
75
91
|
let existing = readBuildRuns(projectDir).find((run) => run.runId === runId);
|
|
76
92
|
if (!existing)
|
|
77
93
|
throw new Error(`recoverable build run not found: ${runId}`);
|
|
78
|
-
if (
|
|
94
|
+
if (["completed", "cancelled", "superseded"].includes(existing.status))
|
|
79
95
|
throw new Error(`build run ${runId} is ${existing.status} and cannot resume`);
|
|
80
96
|
const workflow = new WorkflowDb(projectDir);
|
|
97
|
+
let authority;
|
|
81
98
|
try {
|
|
82
|
-
|
|
83
|
-
if (held && held.runId === runId && held.pid === process.pid && held.host === hostname() && held.processStart === processStartIdentity(process.pid)) {
|
|
84
|
-
workflow.heartbeatLease(held, now);
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
workflow.acquireLease(runId, undefined, now, BUILD_LEASE_STALE_MS);
|
|
88
|
-
}
|
|
99
|
+
authority = workflow.acquireLease(runId, undefined, now, BUILD_LEASE_STALE_MS);
|
|
89
100
|
if (patch.expectedRecoveryDecisionDigest) {
|
|
90
101
|
const decision = workflow.getRun(runId)?.state.recoveryDecision;
|
|
91
102
|
if (createHash("sha256").update(JSON.stringify(decision ?? null)).digest("hex") !== patch.expectedRecoveryDecisionDigest) {
|
|
92
|
-
|
|
93
|
-
if (lease?.runId === runId && lease.pid === process.pid)
|
|
94
|
-
workflow.releaseLease(lease, now);
|
|
103
|
+
workflow.releaseLease(authority, now);
|
|
95
104
|
throw new Error("Recovery decision changed before the child supervisor acquired its lease; inspect and retry");
|
|
96
105
|
}
|
|
97
106
|
}
|
|
98
107
|
existing = readBuildRuns(projectDir).find((run) => run.runId === runId);
|
|
99
|
-
if (!existing ||
|
|
100
|
-
|
|
101
|
-
if (lease?.runId === runId && lease.pid === process.pid)
|
|
102
|
-
workflow.releaseLease(lease, now);
|
|
108
|
+
if (!existing || ["completed", "cancelled", "superseded"].includes(existing.status)) {
|
|
109
|
+
workflow.releaseLease(authority, now);
|
|
103
110
|
throw new Error("Recovery run completed or disappeared before lease acquisition");
|
|
104
111
|
}
|
|
105
112
|
}
|
|
@@ -111,7 +118,7 @@ export function resumeBuildRun(projectDir, runId, patch, now = new Date()) {
|
|
|
111
118
|
? undefined
|
|
112
119
|
: patch.builderSessionRef?.sessionId ?? patch.builderSessionId ?? existing.builder?.sessionId;
|
|
113
120
|
return saveBuildRun(projectDir, {
|
|
114
|
-
...existing, status: "running", checkpoint: "recovery-resumed", completedAt: undefined,
|
|
121
|
+
...bindBuildAuthority(existing, authority), status: "running", checkpoint: "recovery-resumed", completedAt: undefined,
|
|
115
122
|
builder: patch.builder ? { settings: normalizeCapturedSettings(patch.builder), ...(builderSessionId ? { sessionId: builderSessionId } : {}) } : existing.builder,
|
|
116
123
|
qa: patch.qa ? { settings: normalizeCapturedSettings(patch.qa), sessionId: existing.qa?.sessionId } : existing.qa,
|
|
117
124
|
sessionBindings: patch.builderSessionRef ? upsertSessionBinding(existing.sessionBindings, patch.builderSessionRef) : existing.sessionBindings,
|
|
@@ -129,9 +136,22 @@ function normalizeCapturedSettings(settings) {
|
|
|
129
136
|
};
|
|
130
137
|
}
|
|
131
138
|
export function saveBuildRun(projectDir, run, now = new Date()) {
|
|
139
|
+
if (!run[buildAuthority])
|
|
140
|
+
throw new Error("build mutation requires the caller's original workflow authority");
|
|
141
|
+
if (canonicalSessionPath(projectDir) !== canonicalSessionPath(run.repository.root))
|
|
142
|
+
throw new Error("Build authority belongs to another project");
|
|
143
|
+
const original = run[buildAuthority];
|
|
144
|
+
const current = readCurrentWorkflowLease(projectDir);
|
|
145
|
+
if (!current || current.owner !== original.owner || current.generation !== original.generation || current.runId !== original.runId)
|
|
146
|
+
throw new Error("workflow lease ownership changed");
|
|
132
147
|
validateBuildRun(run);
|
|
133
148
|
const directory = join(resolve(projectDir), BUILD_RUN_DIRECTORY);
|
|
134
|
-
|
|
149
|
+
const progress = ["recoverable", "interrupted", "blocked", "failed"].includes(run.status) ? currentBranchTicketProgress(projectDir, run) : undefined;
|
|
150
|
+
if (progress)
|
|
151
|
+
run = { ...run,
|
|
152
|
+
currentTicket: progress.pendingQa.find(ticket => run.tickets.includes(ticket)) ?? (run.currentTicket && progress.remaining.includes(run.currentTicket) ? run.currentTicket : progress.remaining[0]),
|
|
153
|
+
progress: { ...run.progress, completedTickets: progress.completed, remainingTickets: progress.remaining },
|
|
154
|
+
};
|
|
135
155
|
const upgraded = upgradeBuildRun(run, projectDir, now);
|
|
136
156
|
const completedOperations = Object.keys(upgraded.receipts).sort();
|
|
137
157
|
let next = {
|
|
@@ -142,13 +162,15 @@ export function saveBuildRun(projectDir, run, now = new Date()) {
|
|
|
142
162
|
const workflow = new WorkflowDb(projectDir);
|
|
143
163
|
const target = join(directory, `${run.runId}.json`);
|
|
144
164
|
let publicationId;
|
|
165
|
+
let temporary;
|
|
145
166
|
try {
|
|
146
167
|
workflow.atomic(() => {
|
|
168
|
+
assertBuildAuthority(workflow, run);
|
|
147
169
|
const frozenPolicy = workflow.freezeAutonomyPolicy(next.runId, next.frozenPolicy, now);
|
|
148
170
|
next = { ...next, frozenPolicy, recoveryAttempts: workflow.recoveryAttempts(next.runId), pendingDecisions: workflow.pendingHumanDecisions(next.runId), supervisor: workflow.supervisorState(next.runId) ?? next.supervisor };
|
|
149
171
|
const existing = workflow.getRun(next.runId);
|
|
150
|
-
if (existing
|
|
151
|
-
throw new Error(
|
|
172
|
+
if (existing && ["completed", "cancelled", "superseded"].includes(existing.status) && workflowStatus(next.status) !== existing.status)
|
|
173
|
+
throw new Error(`cannot rewrite a ${existing.status} build run`);
|
|
152
174
|
if (!existing)
|
|
153
175
|
workflow.createRun({
|
|
154
176
|
runId: next.runId, kind: "build", checkpoint: next.checkpoint,
|
|
@@ -167,9 +189,17 @@ export function saveBuildRun(projectDir, run, now = new Date()) {
|
|
|
167
189
|
publicationId = workflow.beginPublication(next.runId, { operation: "build-run-projection", target,
|
|
168
190
|
revision: createHash("sha256").update(JSON.stringify(next)).digest("hex"), updatedAt: next.updatedAt }, {}, now).transactionId;
|
|
169
191
|
});
|
|
192
|
+
mkdirSync(directory, { recursive: true });
|
|
170
193
|
const temp = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
194
|
+
temporary = temp;
|
|
171
195
|
writeFileSync(temp, `${JSON.stringify(next, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
172
|
-
|
|
196
|
+
workflow.atomic(() => {
|
|
197
|
+
assertBuildAuthority(workflow, run);
|
|
198
|
+
if (JSON.stringify(workflow.getRun(run.runId)?.state) !== JSON.stringify(next))
|
|
199
|
+
throw new Error("Build projection changed before publication");
|
|
200
|
+
renameSync(temp, target);
|
|
201
|
+
temporary = undefined;
|
|
202
|
+
});
|
|
173
203
|
workflow.atomic(() => {
|
|
174
204
|
workflow.updatePublication(publicationId, "committed", now);
|
|
175
205
|
// Publishing the latest authoritative revision also reconciles any older
|
|
@@ -182,6 +212,8 @@ export function saveBuildRun(projectDir, run, now = new Date()) {
|
|
|
182
212
|
});
|
|
183
213
|
}
|
|
184
214
|
finally {
|
|
215
|
+
if (temporary)
|
|
216
|
+
rmSync(temporary, { force: true });
|
|
185
217
|
workflow.close();
|
|
186
218
|
}
|
|
187
219
|
return next;
|
|
@@ -243,7 +275,7 @@ export function recordBuildReceipt(projectDir, run, operationId, detail) {
|
|
|
243
275
|
return next;
|
|
244
276
|
}
|
|
245
277
|
export function heartbeatBuildRun(projectDir, run, now = new Date()) {
|
|
246
|
-
const lease =
|
|
278
|
+
const lease = run[buildAuthority];
|
|
247
279
|
if (!lease || lease.runId !== run.runId)
|
|
248
280
|
throw new Error("workflow lease ownership changed");
|
|
249
281
|
const next = heartbeatCurrentWorkflowLease(projectDir, lease, now);
|
|
@@ -251,21 +283,42 @@ export function heartbeatBuildRun(projectDir, run, now = new Date()) {
|
|
|
251
283
|
}
|
|
252
284
|
export function releaseBuildLease(projectDir, run, status = "interrupted") {
|
|
253
285
|
const saved = saveBuildRun(projectDir, { ...run, status, lease: undefined });
|
|
254
|
-
releaseWorkflowLease(projectDir, run
|
|
286
|
+
releaseWorkflowLease(projectDir, run);
|
|
255
287
|
return saved;
|
|
256
288
|
}
|
|
257
289
|
export function completeBuildRun(projectDir, run, now = new Date()) {
|
|
290
|
+
if (run.status === "completed") {
|
|
291
|
+
const committed = readBuildRuns(projectDir).find(item => item.runId === run.runId);
|
|
292
|
+
if (committed?.status === "completed" && JSON.stringify(committed) === JSON.stringify(run))
|
|
293
|
+
return committed;
|
|
294
|
+
}
|
|
258
295
|
const workflow = new WorkflowDb(projectDir);
|
|
259
296
|
try {
|
|
260
297
|
workflow.assertQaRunFinalizable(run.runId);
|
|
298
|
+
if (workflow.pendingHumanDecisions(run.runId).length)
|
|
299
|
+
throw new Error("build run has unanswered decisions");
|
|
300
|
+
if (["builder", "qa"].some(role => workflow.unresolvedRoleDispatches(run.runId, role).length))
|
|
301
|
+
throw new Error("build run has unresolved provider dispatches");
|
|
261
302
|
}
|
|
262
303
|
finally {
|
|
263
304
|
workflow.close();
|
|
264
305
|
}
|
|
265
306
|
const saved = saveBuildRun(projectDir, { ...run, status: "completed", checkpoint: "complete", lease: undefined, completedAt: now.toISOString(), progress: { ...run.progress, completedTickets: [...run.tickets], remainingTickets: [], nextAction: "None; build complete" } }, now);
|
|
266
|
-
releaseWorkflowLease(projectDir, run
|
|
307
|
+
releaseWorkflowLease(projectDir, run, now);
|
|
267
308
|
return saved;
|
|
268
309
|
}
|
|
310
|
+
/** A recovered ticket boundary is complete only when the saved run scope is done. */
|
|
311
|
+
export function finishRecoveredTicketScope(projectDir, run, checkpoint) {
|
|
312
|
+
if (run.branchMode !== "current" || !run.tickets.length || !isTicketsInitialized(projectDir))
|
|
313
|
+
throw new Error("Ticket recovery completion requires an initialized current-branch run scope");
|
|
314
|
+
const { completed, remaining } = currentBranchTicketProgress(projectDir, run);
|
|
315
|
+
if (!remaining.length)
|
|
316
|
+
return completeBuildRun(projectDir, run);
|
|
317
|
+
return releaseBuildLease(projectDir, checkpointBuildRun(projectDir, run, checkpoint, {
|
|
318
|
+
currentTicket: remaining[0],
|
|
319
|
+
progress: { ...run.progress, completedTickets: completed, remainingTickets: remaining, nextAction: `Resume ${remaining[0]}` },
|
|
320
|
+
}), "recoverable");
|
|
321
|
+
}
|
|
269
322
|
export function readBuildRuns(projectDir) {
|
|
270
323
|
const directory = join(resolve(projectDir), BUILD_RUN_DIRECTORY);
|
|
271
324
|
const workflow = new WorkflowReader(projectDir);
|
|
@@ -282,6 +335,17 @@ export function readBuildRuns(projectDir) {
|
|
|
282
335
|
const value = stored.state;
|
|
283
336
|
if (value?.runId)
|
|
284
337
|
candidates.set(value.runId, value);
|
|
338
|
+
else if (!["completed", "superseded", "cancelled"].includes(stored.status) && !candidates.has(stored.runId)) {
|
|
339
|
+
// Supervision/readiness can stop before the first full build snapshot.
|
|
340
|
+
// Keep that attempt visible without inventing a provider session or baseline.
|
|
341
|
+
candidates.set(stored.runId, {
|
|
342
|
+
version: 1, runId: stored.runId, status: stored.status === "running" ? "running" : "recoverable",
|
|
343
|
+
tickets: [], branchMode: "current", checkpoint: stored.checkpoint,
|
|
344
|
+
repository: { root: resolve(projectDir), worktree: resolve(projectDir) }, receipts: {},
|
|
345
|
+
createdAt: stored.createdAt, updatedAt: stored.updatedAt,
|
|
346
|
+
failure: { category: "preparation-incomplete", summary: "Build preparation stopped before a complete build checkpoint", at: stored.updatedAt },
|
|
347
|
+
});
|
|
348
|
+
}
|
|
285
349
|
}
|
|
286
350
|
const runs = [...candidates.values()].flatMap((projection) => {
|
|
287
351
|
try {
|
|
@@ -289,8 +353,11 @@ export function readBuildRuns(projectDir) {
|
|
|
289
353
|
const authoritative = workflow?.getRun(projection.runId)?.state;
|
|
290
354
|
const run = authoritative?.runId ? authoritative : projection;
|
|
291
355
|
let upgraded = upgradeBuildRun(run, projectDir);
|
|
292
|
-
|
|
293
|
-
|
|
356
|
+
const dbStatus = workflow.getRun(run.runId)?.status;
|
|
357
|
+
if (dbStatus === "superseded" || dbStatus === "completed")
|
|
358
|
+
upgraded = { ...upgraded, status: dbStatus };
|
|
359
|
+
if (dbStatus === "cancelled")
|
|
360
|
+
return [];
|
|
294
361
|
if (!["completed", "superseded", "cancelled"].includes(workflow?.getRun(run.runId)?.status ?? "")) {
|
|
295
362
|
for (const role of ["builder", "qa"]) {
|
|
296
363
|
const lease = workflow?.roleMutationLease(run.runId, role);
|
|
@@ -304,22 +371,41 @@ export function readBuildRuns(projectDir) {
|
|
|
304
371
|
catch {
|
|
305
372
|
return [];
|
|
306
373
|
}
|
|
307
|
-
}).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
374
|
+
}).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || a.runId.localeCompare(b.runId));
|
|
308
375
|
workflow?.close();
|
|
309
376
|
return runs;
|
|
310
377
|
}
|
|
311
378
|
export function recoverableBuildRuns(projectDir, now = new Date()) {
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
return
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
379
|
+
const reader = new WorkflowReader(projectDir);
|
|
380
|
+
const databaseLease = reader.currentLease();
|
|
381
|
+
const admission = reader.buildAdmission();
|
|
382
|
+
const unfinished = readBuildRuns(projectDir).filter(run => !["completed", "superseded"].includes(run.status));
|
|
383
|
+
const cleanupRuns = [...new Set(reader.readinessProcesses().map(row => row.run_id))].filter(id => !unfinished.some(run => run.runId === id)).map(runId => {
|
|
384
|
+
const run = reader.getRun(runId);
|
|
385
|
+
const at = run?.updatedAt ?? new Date(0).toISOString();
|
|
386
|
+
return { ...upgradeBuildRun({ version: 1, runId, status: "recoverable", tickets: [], branchMode: "current", checkpoint: "readiness-cleanup", repository: { root: resolve(projectDir), worktree: resolve(projectDir) }, receipts: {}, createdAt: run?.createdAt ?? at, updatedAt: at }, projectDir), cleanupOnly: true };
|
|
387
|
+
});
|
|
388
|
+
try {
|
|
389
|
+
return [...unfinished, ...cleanupRuns]
|
|
390
|
+
.map(run => {
|
|
391
|
+
const owners = [];
|
|
392
|
+
if (admission?.runId === run.runId)
|
|
393
|
+
owners.push(classifyProcess(admission.pid, admission.processStart, admission.host));
|
|
394
|
+
if (databaseLease?.runId === run.runId)
|
|
395
|
+
owners.push(classifyProcess(databaseLease.pid, databaseLease.processStart, databaseLease.host));
|
|
396
|
+
if (run.supervisor?.pid && ["running", "stopping"].includes(run.supervisor.status))
|
|
397
|
+
owners.push(classifyProcess(run.supervisor.pid, run.supervisor.processStart));
|
|
398
|
+
if (!owners.length && run.lease)
|
|
399
|
+
owners.push(classifyProcess(run.lease.pid, run.lease.processStart, run.lease.hostname));
|
|
400
|
+
let owner = owners.find(item => item.state === "live") ?? owners.find(item => item.state === "unknown") ?? { state: "dead", reason: "No live owner recorded" };
|
|
401
|
+
if (owner.state !== "live" && reader.pendingBuildLaunches(run.runId).some(launch => launch.state === "dispatching"))
|
|
402
|
+
owner = { state: "unknown", reason: "launch dispatch began but no child claim was confirmed; reconcile the launcher and child process evidence before retrying" };
|
|
403
|
+
return { ...inferLegacyRunTickets(projectDir, run), active: owner.state === "live", ownership: owner.state, ownershipReason: owner.reason };
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
finally {
|
|
407
|
+
reader.close();
|
|
408
|
+
}
|
|
323
409
|
}
|
|
324
410
|
/**
|
|
325
411
|
* Current-branch runs created before ticket checkpointing did not retain their
|
|
@@ -357,14 +443,8 @@ function inferLegacyRunTickets(projectDir, run) {
|
|
|
357
443
|
}
|
|
358
444
|
}
|
|
359
445
|
export function isLeaseActive(run, now = new Date()) {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const age = now.getTime() - new Date(run.lease.heartbeatAt).getTime();
|
|
363
|
-
if (age > BUILD_LEASE_STALE_MS)
|
|
364
|
-
return false;
|
|
365
|
-
if (run.lease.hostname !== hostname())
|
|
366
|
-
return true;
|
|
367
|
-
return isLiveProcessIdentity(run.lease.pid, run.lease.processStart);
|
|
446
|
+
void now;
|
|
447
|
+
return Boolean(run.lease && classifyProcess(run.lease.pid, run.lease.processStart, run.lease.hostname).state !== "dead");
|
|
368
448
|
}
|
|
369
449
|
export function buildRecoveryPreview(run) {
|
|
370
450
|
const completed = Object.keys(run.receipts).sort();
|
|
@@ -381,6 +461,59 @@ export function buildRecoveryPreview(run) {
|
|
|
381
461
|
`QA session ${buildRunSessionBinding(run, "qa") ? "QA session candidate requires validation" : "fresh QA session required"}`,
|
|
382
462
|
];
|
|
383
463
|
}
|
|
464
|
+
/** Saved context may be broader than the work authorized for this invocation. */
|
|
465
|
+
export function recoveryExecutionTickets(run, requested, frozen) {
|
|
466
|
+
const scope = frozen && run.recoveryDecision?.executionTickets !== undefined
|
|
467
|
+
? run.recoveryDecision.executionTickets
|
|
468
|
+
: requested.length ? requested : run.tickets;
|
|
469
|
+
if (scope.some(ticket => !run.tickets.includes(ticket)) || requested.some(ticket => !scope.includes(ticket)) || (run.tickets.length && !scope.length)) {
|
|
470
|
+
throw new Error("Recovery execution scope conflicts with the saved run or requested ticket");
|
|
471
|
+
}
|
|
472
|
+
return [...new Set(scope)];
|
|
473
|
+
}
|
|
474
|
+
/** Read-only reconciliation also handles crashes before the last progress checkpoint. */
|
|
475
|
+
export function currentBranchTicketProgress(projectDir, run) {
|
|
476
|
+
if (run.branchMode !== "current" || !run.tickets.length || !isTicketsInitialized(projectDir))
|
|
477
|
+
return undefined;
|
|
478
|
+
if (run.status === "completed")
|
|
479
|
+
return { completed: [...run.tickets], remaining: [], pendingQa: [] };
|
|
480
|
+
const workflow = new WorkflowReader(projectDir);
|
|
481
|
+
let pendingQa;
|
|
482
|
+
let questions;
|
|
483
|
+
try {
|
|
484
|
+
pendingQa = workflow.pendingQaTicketIds(run.runId);
|
|
485
|
+
questions = workflow.pendingHumanDecisions(run.runId).filter(item => item.interruptionId.startsWith("ticket:")).map(item => item.interruptionId.slice(7));
|
|
486
|
+
}
|
|
487
|
+
finally {
|
|
488
|
+
workflow.close();
|
|
489
|
+
}
|
|
490
|
+
let state;
|
|
491
|
+
let completed = [];
|
|
492
|
+
try {
|
|
493
|
+
const paths = resolveTicketPaths(loadTicketsConfig(projectDir), projectDir);
|
|
494
|
+
state = new Database(paths.stateDb, { readonly: true, fileMustExist: true });
|
|
495
|
+
const read = state.prepare("SELECT status FROM ticket_state WHERE ticket_id=?");
|
|
496
|
+
completed = run.tickets.filter(ticket => read.get(ticket)?.status === "done" && !pendingQa.includes(ticket) && !questions.includes(ticket));
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
// Legacy, missing or unreadable tracker evidence cannot authorize completion.
|
|
500
|
+
// Keep recovery inspectable; actual dispatch still requires a usable tracker.
|
|
501
|
+
completed = [];
|
|
502
|
+
}
|
|
503
|
+
finally {
|
|
504
|
+
state?.close();
|
|
505
|
+
}
|
|
506
|
+
return { completed, remaining: run.tickets.filter(ticket => !completed.includes(ticket)), pendingQa };
|
|
507
|
+
}
|
|
508
|
+
/** Match the frozen preview at both the resume wrapper and direct start boundary. */
|
|
509
|
+
export function currentBranchRecoveryTicket(projectDir, run) {
|
|
510
|
+
if (run.currentTicket && !run.tickets.includes(run.currentTicket))
|
|
511
|
+
throw new Error(`Recovery ticket ${run.currentTicket} is outside the saved run scope`);
|
|
512
|
+
const ticket = projectBuildRecovery(projectDir, run).ticketId;
|
|
513
|
+
if (ticket && !run.tickets.includes(ticket))
|
|
514
|
+
throw new Error(`Recovery ticket ${ticket} is outside the saved run scope`);
|
|
515
|
+
return ticket;
|
|
516
|
+
}
|
|
384
517
|
export function projectBuildRecovery(projectDir, run, now = new Date(), ticketOverride, sessionAvailability) {
|
|
385
518
|
let operationNames = [];
|
|
386
519
|
let issues = [];
|
|
@@ -397,7 +530,13 @@ export function projectBuildRecovery(projectDir, run, now = new Date(), ticketOv
|
|
|
397
530
|
}
|
|
398
531
|
}
|
|
399
532
|
const stateTicket = typeof workflowState.currentTicket === "string" ? workflowState.currentTicket : undefined;
|
|
400
|
-
const
|
|
533
|
+
const progress = currentBranchTicketProgress(projectDir, run);
|
|
534
|
+
if (progress && stateTicket && !run.tickets.includes(stateTicket))
|
|
535
|
+
throw new Error(`Recovery ticket ${stateTicket} is outside the saved run scope`);
|
|
536
|
+
const savedTicket = stateTicket ?? run.currentTicket;
|
|
537
|
+
const ticketId = ticketOverride ?? (progress
|
|
538
|
+
? progress.pendingQa.find(ticket => run.tickets.includes(ticket)) ?? (savedTicket && progress.remaining.includes(savedTicket) ? savedTicket : progress.remaining[0])
|
|
539
|
+
: savedTicket ?? run.progress.remainingTickets[0] ?? run.tickets[0]);
|
|
401
540
|
let ticketTitle;
|
|
402
541
|
if (ticketId && isTicketsInitialized(projectDir)) {
|
|
403
542
|
try {
|
|
@@ -420,15 +559,15 @@ export function projectBuildRecovery(projectDir, run, now = new Date(), ticketOv
|
|
|
420
559
|
unexpectedChanges.push({ path: recoveryWorktree, risk: "preserved worktree identity changed" });
|
|
421
560
|
if (branchChanged)
|
|
422
561
|
unexpectedChanges.push({ path: recoveryWorktree, risk: `branch changed from ${expectedBranch} to ${branchNow}` });
|
|
423
|
-
const completed = [...new Set([...run.progress.completedTickets, ...run.progress.completedOperations, ...operationNames])];
|
|
424
|
-
const remaining = run.progress.remainingTickets.length ? run.progress.remainingTickets : remainingTickets(run);
|
|
562
|
+
const completed = [...new Set([...(progress?.completed ?? run.progress.completedTickets), ...run.progress.completedOperations, ...operationNames])];
|
|
563
|
+
const remaining = progress?.remaining ?? (run.progress.remainingTickets.length ? run.progress.remainingTickets : remainingTickets(run));
|
|
425
564
|
const interruption = run.interruption?.lastError ?? run.interruption?.summary ?? run.failure?.summary ?? issues.at(-1) ?? run.status;
|
|
426
565
|
const validation = run.progress.validation
|
|
427
566
|
? `${run.progress.validation.status}${run.progress.validation.qa ? `; QA ${run.progress.validation.qa}` : ""}`
|
|
428
567
|
: run.qa?.sessionId ? "QA session checkpointed" : "no completed QA/validation checkpoint recorded";
|
|
429
568
|
const nextAction = ticketOverride
|
|
430
569
|
? `Resume ${ticketOverride} from ${run.checkpoint}`
|
|
431
|
-
: run.progress.nextAction ?? (ticketId ? `Resume ${ticketId} from ${run.checkpoint}` : `Resume from ${run.checkpoint}`);
|
|
570
|
+
: (progress ? undefined : run.progress.nextAction) ?? (ticketId ? `Resume ${ticketId} from ${run.checkpoint}` : `Resume from ${run.checkpoint}`);
|
|
432
571
|
const primary = ticketId ? `${ticketId}${ticketTitle ? `: ${ticketTitle}` : ""}` : "legacy run (ticket unavailable)";
|
|
433
572
|
const stateSessionId = typeof workflowState.sessionId === "string" ? workflowState.sessionId : undefined;
|
|
434
573
|
const sessionCandidateRef = buildRunSessionBinding(run, "builder", stateSessionId ?? run.builder?.sessionId);
|
|
@@ -524,7 +663,7 @@ function workflowStatus(status) {
|
|
|
524
663
|
}
|
|
525
664
|
function remainingTickets(run) {
|
|
526
665
|
const current = run.currentTicket ? run.tickets.indexOf(run.currentTicket) : 0;
|
|
527
|
-
return run.status === "completed" ? [] : run.tickets.slice(Math.max(0, current));
|
|
666
|
+
return run.status === "completed" ? [] : (run.branchMode === "current" ? run.tickets : run.tickets.slice(Math.max(0, current))).filter(ticket => run.branchMode !== "current" || !run.progress.completedTickets.includes(ticket));
|
|
528
667
|
}
|
|
529
668
|
function captureGitSnapshot(worktree, input) {
|
|
530
669
|
const git = (args) => {
|
|
@@ -619,11 +758,12 @@ function upgradeBuildRun(run, projectDir = run.repository.root, now = new Date()
|
|
|
619
758
|
deferredTickets: [],
|
|
620
759
|
};
|
|
621
760
|
}
|
|
622
|
-
function releaseWorkflowLease(projectDir,
|
|
761
|
+
function releaseWorkflowLease(projectDir, run, now = new Date()) {
|
|
623
762
|
const workflow = new WorkflowDb(projectDir);
|
|
624
763
|
try {
|
|
625
|
-
|
|
626
|
-
|
|
764
|
+
assertBuildAuthority(workflow, run);
|
|
765
|
+
const lease = run[buildAuthority];
|
|
766
|
+
if (lease)
|
|
627
767
|
workflow.releaseLease(lease, now);
|
|
628
768
|
}
|
|
629
769
|
finally {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { RuntimeAuthError, type AgentRuntime } from "../runtimeAuth.js";
|
|
2
|
+
import { type BuildReadinessContext } from "../runtimeReadiness.js";
|
|
2
3
|
import type { RuntimeProbeResult } from "rafi-spec";
|
|
3
4
|
export type RuntimeCommandRecoveryChoice = "retry" | "switch" | "cancel";
|
|
4
5
|
export interface RuntimeCommandRecoveryContext {
|
|
@@ -7,6 +8,7 @@ export interface RuntimeCommandRecoveryContext {
|
|
|
7
8
|
}
|
|
8
9
|
export interface RuntimeReadyForCommandOptions {
|
|
9
10
|
label: string;
|
|
11
|
+
build?: BuildReadinessContext;
|
|
10
12
|
timeoutMs?: number;
|
|
11
13
|
durable?: {
|
|
12
14
|
projectDir: string;
|
|
@@ -27,4 +29,7 @@ export interface RuntimeReadyForCommandResult {
|
|
|
27
29
|
fellBack: boolean;
|
|
28
30
|
executable: string;
|
|
29
31
|
}
|
|
32
|
+
export declare function ensureBuildRuntimeReadyForCommand(projectDir: string, runtime: AgentRuntime, opts: RuntimeReadyForCommandOptions & {
|
|
33
|
+
build: BuildReadinessContext;
|
|
34
|
+
}): Promise<RuntimeReadyForCommandResult>;
|
|
30
35
|
export declare function ensureRuntimeReadyForCommand(projectDir: string, runtime: AgentRuntime, labelOrOptions: string | RuntimeReadyForCommandOptions): Promise<RuntimeReadyForCommandResult>;
|
|
@@ -1,17 +1,27 @@
|
|
|
1
1
|
import { select, isCancel, log } from "@clack/prompts";
|
|
2
2
|
import { requireClaudeSDK } from "../adapters/claude.js";
|
|
3
3
|
import { checkRuntimeReady, RuntimeAuthError, runtimeCommandLabel, } from "../runtimeAuth.js";
|
|
4
|
-
import {
|
|
4
|
+
import { BuildOwnershipError } from "../buildAdmission.js";
|
|
5
|
+
import { RuntimeCleanupError, resolveExecutablePath } from "../runtimeReadiness.js";
|
|
5
6
|
import { otherRuntime, runtimeDisplayName } from "./runtimeSelection.js";
|
|
6
7
|
import { currentActivity } from "../activity.js";
|
|
7
8
|
import { durableHumanDecision } from "../humanDecision.js";
|
|
8
9
|
import { WorkflowDb } from "../workflowDb.js";
|
|
9
10
|
import { createHash } from "node:crypto";
|
|
11
|
+
export async function ensureBuildRuntimeReadyForCommand(projectDir, runtime, opts) {
|
|
12
|
+
if (!opts.build?.authority)
|
|
13
|
+
throw new BuildOwnershipError("stale-owner", "Build readiness requires original owning-project context");
|
|
14
|
+
return ensureRuntimeReadyForCommand(projectDir, runtime, opts);
|
|
15
|
+
}
|
|
10
16
|
export async function ensureRuntimeReadyForCommand(projectDir, runtime, labelOrOptions) {
|
|
11
17
|
const opts = typeof labelOrOptions === "string"
|
|
12
18
|
? { label: labelOrOptions }
|
|
13
19
|
: labelOrOptions;
|
|
14
|
-
const check = opts.check ?? ((projectDir, runtime) =>
|
|
20
|
+
const check = opts.check ?? (async (projectDir, runtime) => {
|
|
21
|
+
if (opts.durable && !opts.build)
|
|
22
|
+
throw new BuildOwnershipError("stale-owner", "Build readiness requires its original owning-project context");
|
|
23
|
+
return checkRuntimeReady(projectDir, runtime, { build: opts.build, onTrace: opts.onTrace, timeoutMs: opts.timeoutMs });
|
|
24
|
+
});
|
|
15
25
|
const checkClaudeSdk = opts.checkClaudeSdk ?? requireClaudeSDK;
|
|
16
26
|
const allowSwitch = opts.allowSwitch !== false;
|
|
17
27
|
const nonInteractive = !opts.choose && (Boolean(opts.yes) || !process.stdin.isTTY || !process.stdout.isTTY);
|
|
@@ -26,6 +36,8 @@ export async function ensureRuntimeReadyForCommand(projectDir, runtime, labelOrO
|
|
|
26
36
|
return { runtime, model: opts.model, fellBack: false, executable };
|
|
27
37
|
}
|
|
28
38
|
catch (err) {
|
|
39
|
+
if (err instanceof RuntimeCleanupError || err instanceof BuildOwnershipError)
|
|
40
|
+
throw err;
|
|
29
41
|
const failure = err instanceof RuntimeAuthError
|
|
30
42
|
? err
|
|
31
43
|
: new RuntimeAuthError({
|
|
@@ -64,6 +76,8 @@ export async function ensureRuntimeReadyForCommand(projectDir, runtime, labelOrO
|
|
|
64
76
|
return { runtime: fallbackRuntime, model: undefined, fellBack: true, executable };
|
|
65
77
|
}
|
|
66
78
|
catch (switchErr) {
|
|
79
|
+
if (switchErr instanceof RuntimeCleanupError || switchErr instanceof BuildOwnershipError)
|
|
80
|
+
throw switchErr;
|
|
67
81
|
const switchFailure = switchErr instanceof RuntimeAuthError
|
|
68
82
|
? switchErr
|
|
69
83
|
: new RuntimeAuthError({
|
package/dist/cli/start.d.ts
CHANGED
|
@@ -16,6 +16,14 @@ export declare function formatResumeGuidance(executable: "rafi" | "ai-foreman",
|
|
|
16
16
|
}>;
|
|
17
17
|
}>;
|
|
18
18
|
}): string[];
|
|
19
|
+
/** Save parsed CLI values as argv, never as executable shell text. */
|
|
20
|
+
export declare function savedStartArguments(command: Command, projectDir: string, opts: Record<string, unknown>): string[];
|
|
21
|
+
/** Pure validation shared by interactive start and persisted preparation. */
|
|
22
|
+
export declare function validateStartOptions(opts: Record<string, any>, requireCount: boolean): void;
|
|
23
|
+
/** Validate generated established recovery before reserving execution authority. */
|
|
24
|
+
export declare function validateRecoveryArguments(args: readonly string[], projectDir: string): void;
|
|
25
|
+
/** Validate persisted preparation argv without executing a command or provider. */
|
|
26
|
+
export declare function validatePreparationArguments(args: readonly string[], projectDir: string): string[];
|
|
19
27
|
export declare function buildStartCommand(): Command;
|
|
20
28
|
export declare function createQaNonconvergenceHandler(projectDir: string, noninteractive: boolean, planner?: AgentRoleDefaultsV1, observer?: () => RunObserver | undefined, activeRunId?: () => string): (context: QaNonconvergenceContext) => Promise<QaNonconvergenceDecision>;
|
|
21
29
|
export declare function createQaReportRecoveryHandler(projectDir: string, noninteractive: boolean, observer?: () => RunObserver | undefined): QaReportRecoveryHandler;
|