@getpipher/armory-fleet 0.10.3 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/backend/claude-session.ts +5 -0
- package/src/engine/run-registry.ts +7 -0
- package/src/engine/spawnSubagent.ts +23 -4
- package/src/index.ts +38 -5
- package/src/lifecycle/default.ts +3 -0
- package/src/lifecycle/gates/builtin.ts +14 -0
- package/src/lifecycle/gates/chain-runner.ts +37 -0
- package/src/lifecycle/gates/completeness-check.ts +36 -0
- package/src/lifecycle/gates/gate.ts +41 -0
- package/src/lifecycle/gates/registry.ts +75 -0
- package/src/lifecycle/gates/verification-before-completion.ts +50 -0
- package/src/lifecycle/gates/verify.ts +47 -0
- package/src/lifecycle/lifecycle-types.ts +7 -0
- package/src/lifecycle/prompt-template.ts +13 -1
- package/src/lifecycle/registry.ts +16 -1
- package/src/lifecycle/run-lifecycle.ts +56 -3
- package/src/panel/fleet-panel.ts +37 -2
- package/src/panel/fleet-widget.ts +20 -1
- package/src/panel/gate-line.ts +22 -0
- package/src/panel/rows.ts +3 -0
- package/src/runtime/async-runner.ts +82 -26
- package/src/runtime/reconcile.ts +27 -18
- package/src/runtime/resume.ts +21 -15
- package/src/runtime/run-journal.ts +2 -2
- package/src/runtime/run-log.ts +9 -1
- package/src/scheduling/scheduler.ts +4 -0
- package/src/tools/subagent.ts +8 -2
- package/src/worktree/worktree-service.ts +10 -0
|
@@ -5,6 +5,10 @@ import type { SpawnResult } from "../engine/spawnSubagent.ts";
|
|
|
5
5
|
import type {
|
|
6
6
|
BackendId, LifecycleDef, LifecycleMode, LifecycleStatus, PhaseRecord, CheckpointDecision,
|
|
7
7
|
} from "./lifecycle-types.ts";
|
|
8
|
+
import type { GateDef, GateCtx, GateResult, GateRegistry } from "./gates/registry.ts";
|
|
9
|
+
import type { Tier } from "../tiers/tier-registry.ts";
|
|
10
|
+
import { resolveGates } from "./gates/registry.ts";
|
|
11
|
+
import { runGateChain } from "./gates/chain-runner.ts";
|
|
8
12
|
import { renderPhasePrompt } from "./prompt-template.ts";
|
|
9
13
|
import { parseArtifacts, MAX_REVISE } from "./artifacts-parser.ts";
|
|
10
14
|
import {
|
|
@@ -36,6 +40,12 @@ export interface LifecycleRunDeps {
|
|
|
36
40
|
/** SPEC-5a (Q3=A): when present, isolated runs use worktree-diff artifact discovery
|
|
37
41
|
* instead of the prompt-baked `Artifacts:` block parser. Foreground runs leave this undefined. */
|
|
38
42
|
artifactDiscovery?: (o: { finalText: string; cwd: string; baseRef: string; terminal: boolean }) => { summary: string; paths: string[] } | { error: string };
|
|
43
|
+
/** SPEC-6-2: gate registry — when present + a phase has gates, the gate chain runs between parse-artifacts and checkpoint. */
|
|
44
|
+
gateRegistry?: GateRegistry;
|
|
45
|
+
/** SPEC-6-2: provides the gate chain's ctx extras (lifecycle cost, context tokens, tier). When absent, defaults to zeros. */
|
|
46
|
+
getGateCtxState?: (todoId: string, agentName: string) => { lifecycleCost: number; contextTokens: number; tier?: Tier };
|
|
47
|
+
/** SPEC-6-2: resolve a model's context window for the gate ctx. Optional — absent → undefined. */
|
|
48
|
+
getModelContextWindow?: (model: string) => number | undefined;
|
|
39
49
|
}
|
|
40
50
|
|
|
41
51
|
export interface LifecycleRunOpts {
|
|
@@ -63,8 +73,8 @@ export interface LifecycleRunResult {
|
|
|
63
73
|
error?: string;
|
|
64
74
|
}
|
|
65
75
|
|
|
66
|
-
/** Human (or auto) decision at a checkpoint. */
|
|
67
|
-
export type CheckpointFn = (phase: PhaseRecord) => Promise<CheckpointDecision>;
|
|
76
|
+
/** Human (or auto) decision at a checkpoint. SPEC-6-2: widened to include gate results. */
|
|
77
|
+
export type CheckpointFn = (phase: PhaseRecord, gateResults: GateResult[]) => Promise<CheckpointDecision>;
|
|
68
78
|
|
|
69
79
|
export async function runLifecycle(task: string, lifecycleName: string, opts: LifecycleRunOpts): Promise<LifecycleRunResult> {
|
|
70
80
|
const { deps } = opts;
|
|
@@ -133,6 +143,7 @@ export async function runLifecycle(task: string, lifecycleName: string, opts: Li
|
|
|
133
143
|
task, lifecycle: lifecycleName, phase: phaseDef.name,
|
|
134
144
|
prev: prev ? { name: prev.name, summary: prev.summary, paths: prev.paths } : undefined,
|
|
135
145
|
feedback,
|
|
146
|
+
challengeStep: phaseDef.challengeStep,
|
|
136
147
|
});
|
|
137
148
|
|
|
138
149
|
// e/f: spawn the phase child (links to the lifecycle todo; skips mark-done/revert — Task 8).
|
|
@@ -171,6 +182,48 @@ export async function runLifecycle(task: string, lifecycleName: string, opts: Li
|
|
|
171
182
|
// Capture this attempt's summary for the next revise iteration's feedback digest.
|
|
172
183
|
priorAttemptSummary = phaseRec.summary;
|
|
173
184
|
|
|
185
|
+
// SPEC-6-2: gate chain — runs between parse-artifacts and checkpoint.
|
|
186
|
+
// If the gate chain short-circuits (revise/abort), we handle it here BEFORE the checkpoint.
|
|
187
|
+
let gateResults: GateResult[] = [];
|
|
188
|
+
if (phaseDef.gates && phaseDef.gates.length > 0 && deps.gateRegistry) {
|
|
189
|
+
const gates = resolveGates(phaseDef.gates, deps.gateRegistry);
|
|
190
|
+
const gateCtxState = deps.getGateCtxState?.(todoId, agentName) ?? { lifecycleCost: 0, contextTokens: 0 };
|
|
191
|
+
const gateCtx: GateCtx = {
|
|
192
|
+
phaseRec, spawnRes,
|
|
193
|
+
lifecycle: { name: lifecycleName, task, todoId, backend: lifecycleBackend },
|
|
194
|
+
tier: gateCtxState.tier,
|
|
195
|
+
lifecycleCost: gateCtxState.lifecycleCost,
|
|
196
|
+
contextTokens: gateCtxState.contextTokens,
|
|
197
|
+
worktreePath: opts.worktreePath,
|
|
198
|
+
spawn: deps.spawn,
|
|
199
|
+
getModelContextWindow: deps.getModelContextWindow ?? (() => undefined),
|
|
200
|
+
};
|
|
201
|
+
const outcome = await runGateChain({ gates, ctx: gateCtx });
|
|
202
|
+
gateResults = outcome.results;
|
|
203
|
+
phaseRec.gateResults = gateResults;
|
|
204
|
+
if (outcome.shortCircuit?.action === "revise") {
|
|
205
|
+
reviseCount++;
|
|
206
|
+
lastFeedback = outcome.shortCircuit.feedback;
|
|
207
|
+
if (reviseCount > MAX_REVISE) {
|
|
208
|
+
await updateProgress(deps.todoPort, todoId, {
|
|
209
|
+
phase: phaseDef.name, done: false, last: `gate revise budget exhausted (${MAX_REVISE})`, revising: false, attempt: reviseCount,
|
|
210
|
+
}, { lifecycle: lifecycleName, task, backend: lifecycleBackend, mode: opts.mode, phases: progressPhases });
|
|
211
|
+
phaseRecords.push(phaseRec);
|
|
212
|
+
return doneResult(runId, startedAt, "failed", lifecycleName, task, lifecycleBackend, opts.mode, phaseRecords, todoId,
|
|
213
|
+
`gate revise budget exhausted (${MAX_REVISE})`);
|
|
214
|
+
}
|
|
215
|
+
await updateProgress(deps.todoPort, todoId, {
|
|
216
|
+
phase: phaseDef.name, done: false, last: `gate revise (attempt ${reviseCount}/${MAX_REVISE}): ${outcome.shortCircuit.feedback?.slice(0, 80)}`, revising: true, attempt: reviseCount,
|
|
217
|
+
}, { lifecycle: lifecycleName, task, backend: lifecycleBackend, mode: opts.mode, phases: progressPhases });
|
|
218
|
+
continue; // re-run the phase
|
|
219
|
+
}
|
|
220
|
+
if (outcome.shortCircuit?.action === "abort") {
|
|
221
|
+
await revertLifecycleTodo(deps.todoPort, todoId, `gate aborted: ${outcome.shortCircuit.reason}`);
|
|
222
|
+
phaseRecords.push(phaseRec);
|
|
223
|
+
return doneResult(runId, startedAt, "failed", lifecycleName, task, lifecycleBackend, opts.mode, phaseRecords, todoId, outcome.shortCircuit.reason);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
174
227
|
// h: update the lifecycle todo progress block.
|
|
175
228
|
await updateProgress(deps.todoPort, todoId, {
|
|
176
229
|
phase: phaseDef.name, done: phaseRec.status === "completed",
|
|
@@ -186,7 +239,7 @@ export async function runLifecycle(task: string, lifecycleName: string, opts: Li
|
|
|
186
239
|
break; // advance to next phase
|
|
187
240
|
}
|
|
188
241
|
|
|
189
|
-
const decision = await opts.onCheckpoint(phaseRec);
|
|
242
|
+
const decision = await opts.onCheckpoint(phaseRec, gateResults);
|
|
190
243
|
if (decision.action === "continue") {
|
|
191
244
|
if (forceCheckpoint) {
|
|
192
245
|
// cannot continue past a failure — treat as abort (guard against a misbehaving checkpoint fn)
|
package/src/panel/fleet-panel.ts
CHANGED
|
@@ -358,7 +358,7 @@ export class FleetPanel extends Container {
|
|
|
358
358
|
this.fullMessageEvent
|
|
359
359
|
? " esc:Back"
|
|
360
360
|
: this.infoAgent || this.selectedBackend || this.selectedLifecycle || this.selectedSchedule || this.selectedRun
|
|
361
|
-
? (this.selectedRun ? " enter:Full-message esc:Back" : " esc:Back")
|
|
361
|
+
? (this.selectedRun ? " enter:Full-message esc:Back" : this.selectedLifecycle ? " v:View-evidence g:Re-run-gate esc:Back" : " esc:Back")
|
|
362
362
|
: this.pendingCheckpoint
|
|
363
363
|
? " c:Continue v:Revise a:Abort"
|
|
364
364
|
: this.lcRevising
|
|
@@ -474,7 +474,42 @@ export class FleetPanel extends Container {
|
|
|
474
474
|
return;
|
|
475
475
|
}
|
|
476
476
|
if (this.selectedLifecycle) {
|
|
477
|
-
if (matchesKey(data, "escape")) { this.selectedLifecycle = null; this.renderShell(); }
|
|
477
|
+
if (matchesKey(data, "escape")) { this.selectedLifecycle = null; this.renderShell(); return; }
|
|
478
|
+
// SPEC-6-2: v:View-evidence — open the conversation viewer on the first agent gate's runId.
|
|
479
|
+
if (matchesKey(data, "v")) {
|
|
480
|
+
const agentGate = this.selectedLifecycle.phases
|
|
481
|
+
.flatMap((p) => p.gateResults ?? [])
|
|
482
|
+
.find((gr) => gr.runId);
|
|
483
|
+
if (agentGate?.runId && this.deps.runLog) {
|
|
484
|
+
this.selectedRun = buildRunsIndex(this.deps.runLog.dir).find((r) => r.runId === agentGate.runId) ?? null;
|
|
485
|
+
this.runTimeline = this.deps.runLog.replay(agentGate.runId);
|
|
486
|
+
this.selectedLifecycle = null;
|
|
487
|
+
this.view = "runs";
|
|
488
|
+
this.renderShell();
|
|
489
|
+
} else if (agentGate) {
|
|
490
|
+
this.onNotify(`Gate '${agentGate.gate}' evidence: ${agentGate.evidence.slice(0, 200)}`, "info");
|
|
491
|
+
} else {
|
|
492
|
+
const predGate = this.selectedLifecycle.phases
|
|
493
|
+
.flatMap((p) => p.gateResults ?? [])
|
|
494
|
+
.find((gr) => !gr.passed && gr.evidence);
|
|
495
|
+
if (predGate) {
|
|
496
|
+
this.onNotify(`Gate '${predGate.gate}' evidence: ${predGate.evidence.slice(0, 200)}`, "info");
|
|
497
|
+
} else {
|
|
498
|
+
this.onNotify("No gate evidence available for this lifecycle.", "info");
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
// SPEC-6-2: g:Re-run-gate — requires the GateCtx which the runtime holds, not the panel.
|
|
504
|
+
// Full re-run from the panel is a post-v0.11.0 enhancement (the panel doesn't have the GateCtx).
|
|
505
|
+
if (matchesKey(data, "g")) {
|
|
506
|
+
if (this.selectedLifecycle.status === "checkpoint") {
|
|
507
|
+
this.onNotify("Gate re-run from the panel is not yet supported — use the fleet tool or revise at the checkpoint to re-trigger gates.", "info");
|
|
508
|
+
} else {
|
|
509
|
+
this.onNotify("Gate re-run requires a checkpointed lifecycle (current status: " + this.selectedLifecycle.status + ").", "warning");
|
|
510
|
+
}
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
478
513
|
return;
|
|
479
514
|
}
|
|
480
515
|
if (this.selectedSchedule) {
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import type { Theme } from "@earendil-works/pi-coding-agent";
|
|
18
18
|
import type { RunRegistry } from "../engine/run-registry.ts";
|
|
19
19
|
import type { BgRunsStore } from "./bg-runs-store.ts";
|
|
20
|
+
import { reconcileRuns } from "../runtime/reconcile.ts";
|
|
20
21
|
import {
|
|
21
22
|
toWidgetRun, toWidgetRunFromBg, renderWidgetLines,
|
|
22
23
|
} from "./widget-rows.ts";
|
|
@@ -41,6 +42,10 @@ export interface FleetWidgetDeps {
|
|
|
41
42
|
clearInterval?: (id: unknown) => void;
|
|
42
43
|
/** SPEC-6-1: resolve a model's context window for the ctx% widget segment. Optional — absent → no ctx%. */
|
|
43
44
|
getModelContextWindow?: (model: string) => number | undefined;
|
|
45
|
+
/** SPEC-6-2: the session's cwd — only runs from this cwd are shown in the widget (cross-cwd filter). */
|
|
46
|
+
cwd?: string;
|
|
47
|
+
/** SPEC-6-2: RunLog for the periodic liveness probe (reconcileRuns). Optional — absent → no periodic probe. */
|
|
48
|
+
runLog?: import("../runtime/run-log.ts").RunLog;
|
|
44
49
|
}
|
|
45
50
|
|
|
46
51
|
export class FleetWidgetController {
|
|
@@ -50,6 +55,7 @@ export class FleetWidgetController {
|
|
|
50
55
|
private readonly clearIntervalFn: (id: unknown) => void;
|
|
51
56
|
private readonly unsubs: (() => void)[] = [];
|
|
52
57
|
private timerId: unknown | null = null;
|
|
58
|
+
private livenessTimerId: unknown | null = null;
|
|
53
59
|
private disposed = false;
|
|
54
60
|
|
|
55
61
|
constructor(deps: FleetWidgetDeps) {
|
|
@@ -62,11 +68,20 @@ export class FleetWidgetController {
|
|
|
62
68
|
start(): void {
|
|
63
69
|
this.unsubs.push(this.deps.runRegistry.subscribe(() => this.render()));
|
|
64
70
|
if (this.deps.bgRuns) this.unsubs.push(this.deps.bgRuns.subscribe(() => this.render()));
|
|
71
|
+
// SPEC-6-2: periodic liveness probe — reconciles dead orphans every 60s.
|
|
72
|
+
if (this.deps.runLog) {
|
|
73
|
+
this.livenessTimerId = this.setIntervalFn(() => {
|
|
74
|
+
reconcileRuns(this.deps.runLog!, { runRegistry: this.deps.runRegistry });
|
|
75
|
+
}, 60_000);
|
|
76
|
+
(this.livenessTimerId as { unref?: () => void }).unref?.();
|
|
77
|
+
}
|
|
65
78
|
this.render(); // initial — shows any runs already active on session_start (e.g. a survived bg run)
|
|
66
79
|
}
|
|
67
80
|
|
|
68
81
|
private activeRuns() {
|
|
69
|
-
const fg = this.deps.runRegistry.list()
|
|
82
|
+
const fg = this.deps.runRegistry.list()
|
|
83
|
+
.filter((r) => !this.deps.cwd || r.cwd === this.deps.cwd)
|
|
84
|
+
.map((r) => {
|
|
70
85
|
const w = toWidgetRun(r);
|
|
71
86
|
w.maxContext = this.deps.getModelContextWindow?.(r.model);
|
|
72
87
|
return w;
|
|
@@ -105,6 +120,10 @@ export class FleetWidgetController {
|
|
|
105
120
|
this.clearIntervalFn(this.timerId);
|
|
106
121
|
this.timerId = null;
|
|
107
122
|
}
|
|
123
|
+
if (this.livenessTimerId !== null) {
|
|
124
|
+
this.clearIntervalFn(this.livenessTimerId);
|
|
125
|
+
this.livenessTimerId = null;
|
|
126
|
+
}
|
|
108
127
|
}
|
|
109
128
|
|
|
110
129
|
/** Unsubscribe + clear timer + clear the widget. Idempotent. */
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { GateResult } from "../lifecycle/gates/registry.ts";
|
|
2
|
+
|
|
3
|
+
export function gateGlyph(r: GateResult): string {
|
|
4
|
+
if (r.passed) return "✅";
|
|
5
|
+
if (r.onFail === "abort") return "⛔";
|
|
6
|
+
if (r.onFail === "revise") return "↻";
|
|
7
|
+
return "⚠"; // advise
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** Pure: build the compact gate line for a Lifecycle view phase row. */
|
|
11
|
+
export function buildGateLine(results: GateResult[]): string {
|
|
12
|
+
if (results.length === 0) return "";
|
|
13
|
+
const parts = results.map((r) => `${gateGlyph(r)}${r.gate}`);
|
|
14
|
+
// If the last failing gate short-circuited, append the action.
|
|
15
|
+
const lastFail = [...results].reverse().find((r) => !r.passed);
|
|
16
|
+
let suffix = "";
|
|
17
|
+
if (lastFail) {
|
|
18
|
+
if (lastFail.onFail === "abort") suffix = " → aborted";
|
|
19
|
+
else if (lastFail.onFail === "revise") suffix = " → revising";
|
|
20
|
+
}
|
|
21
|
+
return `gates: ${parts.join(" ")}${suffix}`;
|
|
22
|
+
}
|
package/src/panel/rows.ts
CHANGED
|
@@ -92,6 +92,7 @@ export function backendInfo(b: Backend): string {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
import type { LifecycleRunRecord, LifecycleStatus } from "../lifecycle/lifecycle-types.ts";
|
|
95
|
+
import { buildGateLine } from "./gate-line.ts";
|
|
95
96
|
|
|
96
97
|
|
|
97
98
|
// SPEC-5a §11 — bg run row status (Q8=A). The fleet tab gains live status icons + phase progress
|
|
@@ -174,6 +175,8 @@ export function lifecyclePhaseTimeline(r: LifecycleRunRecord): string {
|
|
|
174
175
|
const mark = p.reviseCount > 0 ? "[~]" : p.status === "completed" ? "[x]" : "[ ]";
|
|
175
176
|
const art = p.paths.length ? ` → ${p.paths.join(", ")}` : "";
|
|
176
177
|
lines.push(` ${mark} ${p.name} ${p.status}${art}${p.paths.length ? " [Open]" : ""}`);
|
|
178
|
+
const gateLine = buildGateLine(p.gateResults ?? []);
|
|
179
|
+
if (gateLine) lines.push(` ${gateLine}`);
|
|
177
180
|
}
|
|
178
181
|
if (r.status === "checkpoint") {
|
|
179
182
|
lines.push("", "── Checkpoint ──", "[Continue] [Revise] [Abort]");
|
|
@@ -23,8 +23,8 @@ export interface FakeLifecycleResult {
|
|
|
23
23
|
|
|
24
24
|
export interface RunLifecycleOpts {
|
|
25
25
|
runId: string;
|
|
26
|
-
worktreePath
|
|
27
|
-
branch
|
|
26
|
+
worktreePath?: string;
|
|
27
|
+
branch?: string;
|
|
28
28
|
mode: "auto" | "checkpointed";
|
|
29
29
|
}
|
|
30
30
|
|
|
@@ -41,18 +41,30 @@ export interface AsyncRunnerDeps {
|
|
|
41
41
|
genRunId: () => string;
|
|
42
42
|
/** SPEC-5a: called at each run/phase transition so the host (index.ts) can update the live bgRuns map. */
|
|
43
43
|
onProgress?: (runId: string, status: import("../panel/rows.ts").BgRunStatus) => void;
|
|
44
|
+
/** SPEC-6-2: the RunRegistry so emitProgress can read the run's actual backend. */
|
|
45
|
+
runRegistry?: import("../engine/run-registry.ts").RunRegistry;
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
export interface RunBackgroundOpts {
|
|
47
49
|
deps: AsyncRunnerDeps;
|
|
48
50
|
lifecycle: string;
|
|
49
51
|
mode: "auto" | "checkpointed";
|
|
52
|
+
/** v0.11.1: edit isolation for background runs. Default "auto" (worktree when cwd is a git repo, in-place otherwise). */
|
|
53
|
+
isolation?: Isolation;
|
|
50
54
|
}
|
|
51
55
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
/** The success shape (back-compat: existing external refs to RunBackgroundHandle still typecheck). */
|
|
57
|
+
export interface RunBackgroundHandle { runId: string; status: "background"; }
|
|
58
|
+
|
|
59
|
+
/** v0.11.1: a background dispatch either starts (runId + background) or fails synchronously (error, no runId). */
|
|
60
|
+
export type RunBackgroundResult =
|
|
61
|
+
| { runId: string; status: "background" }
|
|
62
|
+
| { status: "failed"; error: string };
|
|
63
|
+
|
|
64
|
+
export type Isolation = "worktree" | "none" | "auto";
|
|
65
|
+
|
|
66
|
+
/** Per-session dedup flag for the auto-fallback in-place notify (resets on process restart = new session). */
|
|
67
|
+
let inPlaceFallbackWarned = false;
|
|
56
68
|
|
|
57
69
|
function emitProgress(deps: AsyncRunnerDeps, runId: string, partial: Partial<import("../panel/rows.ts").BgRunStatus> & { status: import("../panel/rows.ts").BgStatus; phase: string; phaseIndex: number; phaseTotal: number }): void {
|
|
58
70
|
if (!deps.onProgress) return;
|
|
@@ -60,7 +72,7 @@ function emitProgress(deps: AsyncRunnerDeps, runId: string, partial: Partial<imp
|
|
|
60
72
|
runId,
|
|
61
73
|
lifecycle: "",
|
|
62
74
|
mode: "auto",
|
|
63
|
-
backend: "pi",
|
|
75
|
+
backend: deps.runRegistry?.get(runId)?.backend ?? "pi",
|
|
64
76
|
task: "",
|
|
65
77
|
...partial,
|
|
66
78
|
});
|
|
@@ -70,54 +82,98 @@ function sh(cmd: string, cwd: string): void {
|
|
|
70
82
|
execSync(cmd, { cwd, encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] });
|
|
71
83
|
}
|
|
72
84
|
|
|
73
|
-
|
|
85
|
+
/** The git-agnostic core: pool slot → journal → runLifecycle in ctx.cwd (or a worktree, when `isolated` is set) → inbox + notify.
|
|
86
|
+
* Fire-and-forget. When `isolated` is present, journals the worktree field, commits on completion, and removes the worktree. */
|
|
87
|
+
function runBackgroundInPlace(runId: string, task: string, opts: RunBackgroundOpts, isolated?: { worktreePath: string; branch: string }): void {
|
|
74
88
|
const { deps } = opts;
|
|
75
|
-
const runId = deps.genRunId();
|
|
76
|
-
const baseRef = "HEAD";
|
|
77
|
-
|
|
78
|
-
// Fire-and-forget: the pool gates concurrency; the journal records the run.
|
|
79
89
|
void deps.pool.withSlot(async () => {
|
|
80
|
-
let wt: { path: string; branch: string } | null = null;
|
|
81
90
|
try {
|
|
82
|
-
|
|
83
|
-
|
|
91
|
+
const ev0: JournalEvent = isolated
|
|
92
|
+
? { type: "run:started", runId, task, lifecycle: opts.lifecycle, worktree: { path: isolated.worktreePath, branch: isolated.branch }, mode: opts.mode, ts: Date.now() }
|
|
93
|
+
: { type: "run:started", runId, task, lifecycle: opts.lifecycle, mode: opts.mode, ts: Date.now() };
|
|
84
94
|
deps.journal.append(runId, ev0);
|
|
85
95
|
emitProgress(deps, runId, { status: "running", phase: "", phaseIndex: 0, phaseTotal: 0, lifecycle: opts.lifecycle, mode: opts.mode, task });
|
|
86
96
|
|
|
87
|
-
const res = await deps.runLifecycle(task, opts.lifecycle, { runId, worktreePath:
|
|
97
|
+
const res = await deps.runLifecycle(task, opts.lifecycle, { runId, worktreePath: isolated?.worktreePath, branch: isolated?.branch, mode: opts.mode });
|
|
88
98
|
|
|
89
99
|
if (res.status === "completed") {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
100
|
+
if (isolated) {
|
|
101
|
+
try { sh("git add -A && git commit -m 'fleet run complete'", isolated.worktreePath); } catch { /* nothing to commit */ }
|
|
102
|
+
}
|
|
103
|
+
deps.journal.append(runId, isolated
|
|
104
|
+
? { type: "run:completed", runId, branch: isolated.branch, ts: Date.now() }
|
|
105
|
+
: { type: "run:completed", runId, ts: Date.now() });
|
|
93
106
|
const total = res.phases.length;
|
|
94
|
-
const lastIdx = total;
|
|
95
|
-
emitProgress(deps, runId, { status: "completed", phase: res.phases[total - 1]?.name ?? "finish", phaseIndex: lastIdx, phaseTotal: total, lifecycle: opts.lifecycle, mode: opts.mode, task, branch:
|
|
107
|
+
const lastIdx = total;
|
|
108
|
+
emitProgress(deps, runId, { status: "completed", phase: res.phases[total - 1]?.name ?? "finish", phaseIndex: lastIdx, phaseTotal: total, lifecycle: opts.lifecycle, mode: opts.mode, task, ...(isolated ? { branch: isolated.branch } : {}) });
|
|
96
109
|
const lastPhase = res.phases[res.phases.length - 1];
|
|
97
110
|
const result: RunResult = {
|
|
98
111
|
runId, task, status: "completed",
|
|
99
112
|
summary: lastPhase?.summary ?? "",
|
|
100
113
|
paths: res.phases.flatMap((p) => p.paths),
|
|
101
|
-
|
|
114
|
+
completedAt: Date.now(),
|
|
115
|
+
...(isolated ? { branch: isolated.branch } : {}),
|
|
102
116
|
};
|
|
103
117
|
deps.inbox.push(result);
|
|
104
118
|
deps.notify(`fleet run ${runId} completed`, "info");
|
|
105
|
-
|
|
106
|
-
deps.worktree.removeWorktree(runId);
|
|
119
|
+
if (isolated) deps.worktree.removeWorktree(runId);
|
|
107
120
|
} else {
|
|
108
121
|
deps.journal.append(runId, { type: "run:aborted", runId, reason: res.error ?? res.status, ts: Date.now() });
|
|
109
|
-
deps.worktree.remove(runId);
|
|
122
|
+
if (isolated) deps.worktree.remove(runId);
|
|
110
123
|
emitProgress(deps, runId, { status: "failed", phase: "", phaseIndex: 0, phaseTotal: res.phases.length, lifecycle: opts.lifecycle, mode: opts.mode, task });
|
|
111
124
|
deps.notify(`fleet run ${runId} ${res.status}: ${res.error ?? ""}`, "warning");
|
|
112
125
|
}
|
|
113
126
|
} catch (e) {
|
|
114
127
|
const msg = (e as Error).message;
|
|
115
128
|
deps.journal.append(runId, { type: "run:aborted", runId, reason: msg, ts: Date.now() });
|
|
116
|
-
if (
|
|
129
|
+
if (isolated) deps.worktree.remove(runId);
|
|
117
130
|
deps.notify(`fleet run ${runId} failed: ${msg}`, "error");
|
|
118
131
|
emitProgress(deps, runId, { status: "failed", phase: "", phaseIndex: 0, phaseTotal: 0, lifecycle: opts.lifecycle, mode: opts.mode, task });
|
|
119
132
|
}
|
|
120
133
|
});
|
|
134
|
+
}
|
|
121
135
|
|
|
136
|
+
/** The worktree wrapper: SYNCHRONOUS pre-flight + worktree create, then core-with-isolation. */
|
|
137
|
+
function runBackgroundIsolated(task: string, opts: RunBackgroundOpts): RunBackgroundResult {
|
|
138
|
+
const { deps } = opts;
|
|
139
|
+
if (!deps.worktree.isGitRepo()) {
|
|
140
|
+
return { status: "failed", error: "isolation: 'worktree' requires a git repo; cwd is not one — use isolation: 'none' or run in a git repo" };
|
|
141
|
+
}
|
|
142
|
+
const runId = deps.genRunId();
|
|
143
|
+
const baseRef = "HEAD";
|
|
144
|
+
let wt: { path: string; branch: string };
|
|
145
|
+
try {
|
|
146
|
+
wt = deps.worktree.create(runId, baseRef);
|
|
147
|
+
} catch (e) {
|
|
148
|
+
return { status: "failed", error: (e as Error).message };
|
|
149
|
+
}
|
|
150
|
+
runBackgroundInPlace(runId, task, opts, { worktreePath: wt.path, branch: wt.branch });
|
|
122
151
|
return { runId, status: "background" };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The auto router: isolated when cwd is a git repo, in-place + one per-session notify when not. */
|
|
155
|
+
function runBackgroundAuto(task: string, opts: RunBackgroundOpts): RunBackgroundResult {
|
|
156
|
+
const { deps } = opts;
|
|
157
|
+
if (deps.worktree.isGitRepo()) {
|
|
158
|
+
return runBackgroundIsolated(task, opts);
|
|
159
|
+
}
|
|
160
|
+
if (!inPlaceFallbackWarned) {
|
|
161
|
+
inPlaceFallbackWarned = true;
|
|
162
|
+
deps.notify("background run in-place (no worktree isolation — parallel edits may conflict)", "warning");
|
|
163
|
+
}
|
|
164
|
+
const runId = deps.genRunId();
|
|
165
|
+
runBackgroundInPlace(runId, task, opts);
|
|
166
|
+
return { runId, status: "background" };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Public dispatcher (keeps the `runBackground` name + RunBackgroundHandle success shape for back-compat). */
|
|
170
|
+
export function runBackground(task: string, opts: RunBackgroundOpts): RunBackgroundResult {
|
|
171
|
+
const isolation = opts.isolation ?? "auto";
|
|
172
|
+
if (isolation === "worktree") return runBackgroundIsolated(task, opts);
|
|
173
|
+
if (isolation === "none") {
|
|
174
|
+
const runId = opts.deps.genRunId();
|
|
175
|
+
runBackgroundInPlace(runId, task, opts);
|
|
176
|
+
return { runId, status: "background" };
|
|
177
|
+
}
|
|
178
|
+
return runBackgroundAuto(task, opts);
|
|
123
179
|
}
|
package/src/runtime/reconcile.ts
CHANGED
|
@@ -3,27 +3,36 @@
|
|
|
3
3
|
// is gone) as aborted so the Runs tab doesn't show stale "running" rows across restarts.
|
|
4
4
|
// Foreground orphans; bg/lifecycle orphans are already handled by scanResumeCandidates (SPEC-5a).
|
|
5
5
|
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
6
|
+
// SPEC-6-2: rewritten to be probe-driven — uses `probeRun` (handle/pid/age-fallback) instead of
|
|
7
|
+
// age+grace alone. This catches orphans whose process died but whose startedAt is within grace
|
|
8
|
+
// (e.g. a crash seconds after start), and avoids aborting runs whose process is alive but old
|
|
9
|
+
// (e.g. a long-running build).
|
|
10
10
|
import type { RunLog } from "./run-log.ts";
|
|
11
|
-
import type { RunRegistry } from "../engine/run-registry.ts";
|
|
11
|
+
import type { RunRegistry, RunRecord } from "../engine/run-registry.ts";
|
|
12
|
+
|
|
13
|
+
export type Liveness = "alive" | "dead";
|
|
14
|
+
|
|
15
|
+
/** SPEC-6-2: real probe first (handle / pid), age+grace fallback for cross-process pi-backend orphans. */
|
|
16
|
+
export function probeRun(rec: { status: string; session?: { isAlive?: () => boolean }; pid?: number; startedAt: number }, now: number, grace: number): Liveness {
|
|
17
|
+
// 1. in-process handle
|
|
18
|
+
if (rec.session && typeof rec.session.isAlive === "function") {
|
|
19
|
+
return rec.session.isAlive() ? "alive" : "dead";
|
|
20
|
+
}
|
|
21
|
+
// 2. pid (works cross-process — system-wide)
|
|
22
|
+
if (typeof rec.pid === "number") {
|
|
23
|
+
try { process.kill(rec.pid, 0); return "alive"; } catch { return "dead"; }
|
|
24
|
+
}
|
|
25
|
+
// 3. fallback — cross-process pi-backend orphan, no reachable probe
|
|
26
|
+
return (now - rec.startedAt > grace) ? "dead" : "alive";
|
|
27
|
+
}
|
|
12
28
|
|
|
13
29
|
export interface ReconcileOpts {
|
|
14
|
-
/** Orphans whose startedAt is older than (now - graceMs) are marked aborted. Default 60000. */
|
|
15
30
|
graceMs?: number;
|
|
16
|
-
/** Test injection. Default Date.now(). */
|
|
17
31
|
now?: number;
|
|
18
|
-
/**
|
|
19
|
-
* v0.10.2: the in-memory RunRegistry to sync alongside the durable log. When set, each orphan
|
|
20
|
-
* reconciled in the log is also transitioned to status:"aborted" in memory so the live widget
|
|
21
|
-
* clears its stale ▶ row. Optional — existing callers that pass only a RunLog are unaffected.
|
|
22
|
-
*/
|
|
23
32
|
runRegistry?: RunRegistry;
|
|
24
33
|
}
|
|
25
34
|
|
|
26
|
-
/** Returns the runIds it marked aborted.
|
|
35
|
+
/** Returns the runIds it marked aborted. Probe-driven (SPEC-6-2); idempotent. */
|
|
27
36
|
export function reconcileRuns(log: RunLog, opts: ReconcileOpts = {}): string[] {
|
|
28
37
|
const grace = opts.graceMs ?? 60_000;
|
|
29
38
|
const now = opts.now ?? Date.now();
|
|
@@ -31,14 +40,14 @@ export function reconcileRuns(log: RunLog, opts: ReconcileOpts = {}): string[] {
|
|
|
31
40
|
const aborted: string[] = [];
|
|
32
41
|
for (const meta of log.scanMeta()) {
|
|
33
42
|
if (meta.status !== "running") continue;
|
|
34
|
-
|
|
43
|
+
// The in-memory record (if present) carries the live handle/pid; the log meta carries pid for cross-process.
|
|
44
|
+
const memRec = reg?.get(meta.runId);
|
|
45
|
+
const probeRec = memRec ?? { status: meta.status, pid: (meta as { pid?: number }).pid, startedAt: meta.startedAt };
|
|
46
|
+
if (probeRun(probeRec, now, grace) !== "dead") continue;
|
|
35
47
|
log.append(meta.runId, {
|
|
36
48
|
type: "run:ended", runId: meta.runId, status: "aborted",
|
|
37
|
-
endedAt: now, resultSummary: "process-gone", tokenTotal: meta.tokenTotal,
|
|
49
|
+
endedAt: now, resultSummary: "process-gone (probe)", tokenTotal: meta.tokenTotal,
|
|
38
50
|
});
|
|
39
|
-
// v0.10.2: sync the in-memory registry so the live widget (which reads runRegistry.list(),
|
|
40
|
-
// not the RunLog) clears the orphan's stale ▶ row. No-op when the run isn't in the registry
|
|
41
|
-
// (e.g. a cross-cwd orphan from another session — out of scope for this patch).
|
|
42
51
|
reg?.update(meta.runId, { status: "aborted", endedAt: now });
|
|
43
52
|
aborted.push(meta.runId);
|
|
44
53
|
}
|
package/src/runtime/resume.ts
CHANGED
|
@@ -8,8 +8,10 @@ export interface ResumeCandidate {
|
|
|
8
8
|
runId: string;
|
|
9
9
|
task: string;
|
|
10
10
|
lifecycle: string;
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
/** Present only for isolated (worktree) runs. */
|
|
12
|
+
worktreePath?: string;
|
|
13
|
+
/** Present only for isolated (worktree) runs. */
|
|
14
|
+
branch?: string;
|
|
13
15
|
lastPhase: string | null;
|
|
14
16
|
canResume: boolean;
|
|
15
17
|
}
|
|
@@ -26,23 +28,27 @@ export function scanResumeCandidates(_projectDir: string, opts: ScanResumeOpts):
|
|
|
26
28
|
for (const runId of ids) {
|
|
27
29
|
const events = journal.replay(runId);
|
|
28
30
|
const started = events.find((e) => e.type === "run:started") as
|
|
29
|
-
| (JournalEvent & { type: "run:started" }) | undefined;
|
|
31
|
+
| (JournalEvent & { type: "run:started"; worktree?: { path: string; branch: string } }) | undefined;
|
|
30
32
|
if (!started) continue;
|
|
31
33
|
const phaseEvents = events.filter((e) => e.type === "phase:completed" || e.type === "phase:started" || e.type === "phase:failed") as Array<{ phase: string }>;
|
|
32
34
|
const lastPhase = phaseEvents.length > 0 ? phaseEvents[phaseEvents.length - 1]!.phase : null;
|
|
33
|
-
|
|
34
|
-
if (
|
|
35
|
-
|
|
35
|
+
|
|
36
|
+
if (started.worktree) {
|
|
37
|
+
// isolated run: resume iff the worktree still exists
|
|
38
|
+
const wtExists = opts.worktree.exists(runId);
|
|
39
|
+
if (!wtExists) {
|
|
40
|
+
journal.append(runId, { type: "run:aborted", runId, reason: "worktree-missing", ts: Date.now() });
|
|
41
|
+
}
|
|
42
|
+
cands.push({
|
|
43
|
+
runId, task: started.task, lifecycle: started.lifecycle,
|
|
44
|
+
worktreePath: started.worktree.path, branch: started.worktree.branch,
|
|
45
|
+
lastPhase, canResume: wtExists,
|
|
46
|
+
});
|
|
47
|
+
} else {
|
|
48
|
+
// v0.11.1: in-place interrupted run — no worktree to clean; abort (partial edits may remain in cwd).
|
|
49
|
+
journal.append(runId, { type: "run:aborted", runId, reason: "in-place interrupted (partial edits may remain in cwd)", ts: Date.now() });
|
|
50
|
+
cands.push({ runId, task: started.task, lifecycle: started.lifecycle, lastPhase, canResume: false });
|
|
36
51
|
}
|
|
37
|
-
cands.push({
|
|
38
|
-
runId,
|
|
39
|
-
task: started.task,
|
|
40
|
-
lifecycle: started.lifecycle,
|
|
41
|
-
worktreePath: started.worktree.path,
|
|
42
|
-
branch: started.worktree.branch,
|
|
43
|
-
lastPhase,
|
|
44
|
-
canResume: wtExists,
|
|
45
|
-
});
|
|
46
52
|
}
|
|
47
53
|
return cands;
|
|
48
54
|
}
|
|
@@ -4,12 +4,12 @@
|
|
|
4
4
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
|
|
7
|
-
export interface RunStartedEvent { type: "run:started"; runId: string; task: string; lifecycle: string; worktree
|
|
7
|
+
export interface RunStartedEvent { type: "run:started"; runId: string; task: string; lifecycle: string; worktree?: { path: string; branch: string }; mode: "auto" | "checkpointed"; ts: number; }
|
|
8
8
|
export interface PhaseStartedEvent { type: "phase:started"; phase: string; ts: number; }
|
|
9
9
|
export interface PhaseCompletedEvent { type: "phase:completed"; phase: string; summary: string; paths: string[]; ts: number; }
|
|
10
10
|
export interface PhaseFailedEvent { type: "phase:failed"; phase: string; error: string; ts: number; }
|
|
11
11
|
export interface CheckpointEvent { type: "checkpoint"; phase: string; decision: "continue" | "revise" | "abort"; ts: number; }
|
|
12
|
-
export interface RunCompletedEvent { type: "run:completed"; runId: string; branch
|
|
12
|
+
export interface RunCompletedEvent { type: "run:completed"; runId: string; branch?: string; ts: number; }
|
|
13
13
|
export interface RunAbortedEvent { type: "run:aborted"; runId: string; reason: string; ts: number; }
|
|
14
14
|
|
|
15
15
|
export type JournalEvent =
|
package/src/runtime/run-log.ts
CHANGED
|
@@ -11,6 +11,10 @@ export interface RunMetaEvent {
|
|
|
11
11
|
type: "run:meta"; runId: string; agent: string; model: string; task: string;
|
|
12
12
|
startedAt: number; track: boolean; todoId: string | null;
|
|
13
13
|
backendSessionId?: string; sessionKey?: string;
|
|
14
|
+
/** SPEC-6-2: claude child PID (cross-process liveness probe). */
|
|
15
|
+
pid?: number;
|
|
16
|
+
/** SPEC-6-2: the cwd this run belongs to. */
|
|
17
|
+
cwd?: string;
|
|
14
18
|
}
|
|
15
19
|
export interface MessageEvent {
|
|
16
20
|
type: "message"; role: string; text: string;
|
|
@@ -41,6 +45,10 @@ export interface RunMeta {
|
|
|
41
45
|
costTotal?: number;
|
|
42
46
|
/** SPEC-6-1: latest context-token snapshot at run end. */
|
|
43
47
|
contextTokens?: number;
|
|
48
|
+
/** SPEC-6-2: claude child PID. */
|
|
49
|
+
pid?: number;
|
|
50
|
+
/** SPEC-6-2: the cwd this run belongs to. */
|
|
51
|
+
cwd?: string;
|
|
44
52
|
}
|
|
45
53
|
|
|
46
54
|
const ARGS_LIMIT = 200;
|
|
@@ -96,7 +104,7 @@ export class RunLog {
|
|
|
96
104
|
if (!meta) {
|
|
97
105
|
meta = { runId: e.runId, agent: e.agent, model: e.model, task: e.task, startedAt: e.startedAt,
|
|
98
106
|
track: e.track, todoId: e.todoId, backendSessionId: e.backendSessionId, sessionKey: e.sessionKey,
|
|
99
|
-
status: "running", tokenTotal: 0 };
|
|
107
|
+
status: "running", tokenTotal: 0, pid: e.pid, cwd: e.cwd };
|
|
100
108
|
} else {
|
|
101
109
|
// latest binding wins
|
|
102
110
|
if (e.backendSessionId) meta.backendSessionId = e.backendSessionId;
|
|
@@ -11,6 +11,8 @@ export interface ScheduleSpec {
|
|
|
11
11
|
expression: string;
|
|
12
12
|
lifecycle?: string; // default "default"
|
|
13
13
|
auto?: boolean;
|
|
14
|
+
/** v0.11.1: edit isolation for the background run on fire. Default "auto". */
|
|
15
|
+
isolation?: "worktree" | "none" | "auto";
|
|
14
16
|
}
|
|
15
17
|
|
|
16
18
|
export interface Schedule extends ScheduleSpec {
|
|
@@ -71,6 +73,7 @@ export class Scheduler {
|
|
|
71
73
|
expression: spec.expression,
|
|
72
74
|
lifecycle: spec.lifecycle ?? "default",
|
|
73
75
|
auto: spec.auto ?? true,
|
|
76
|
+
isolation: spec.isolation,
|
|
74
77
|
paused: false,
|
|
75
78
|
};
|
|
76
79
|
this.schedules.set(id, { spec: stored, expr, timer: null });
|
|
@@ -86,6 +89,7 @@ export class Scheduler {
|
|
|
86
89
|
expression: e.spec.expression,
|
|
87
90
|
lifecycle: e.spec.lifecycle,
|
|
88
91
|
auto: e.spec.auto,
|
|
92
|
+
isolation: e.spec.isolation,
|
|
89
93
|
paused: e.spec.paused,
|
|
90
94
|
nextFire: e.spec.paused ? null : e.expr.nextFire(new Date()),
|
|
91
95
|
}));
|