@alexeiled/pi-fusion 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/agents/fusion-panelist-full.md +3 -1
- package/agents/fusion-panelist-web.md +3 -1
- package/agents/fusion-panelist.md +3 -1
- package/docs/user-guide.md +56 -12
- package/package.json +1 -1
- package/skills/fusion-review/SKILL.md +2 -1
- package/src/commands.ts +17 -0
- package/src/config.ts +27 -0
- package/src/fusion-args.ts +29 -2
- package/src/fusion-rpc.ts +29 -0
- package/src/index.ts +50 -7
- package/src/lifecycle-reconcile.ts +234 -7
- package/src/orchestrator.ts +622 -79
- package/src/panel-completion.ts +65 -17
- package/src/panel-deadlines.ts +90 -0
- package/src/panel-quorum.ts +22 -0
- package/src/report.ts +85 -3
- package/src/result-extract.ts +67 -5
- package/src/run-builder.ts +129 -9
- package/src/run-observations.ts +3 -1
- package/src/run-store.ts +385 -11
- package/src/status.ts +1 -1
- package/src/subagents-rpc.ts +10 -0
- package/src/types.ts +89 -0
package/src/orchestrator.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { deadlineSteerMessage, planPanelDeadlines, PANEL_DECISION_WAIT_MS } from "./panel-deadlines.js";
|
|
1
2
|
import { applyClaudeAliasShorthand } from "./claude-aliases.js";
|
|
2
3
|
import {
|
|
3
4
|
detectCallerOutputContract,
|
|
@@ -11,7 +12,10 @@ import {
|
|
|
11
12
|
} from "./config.js";
|
|
12
13
|
import { FusionArgsError } from "./errors.js";
|
|
13
14
|
import { parseFusionArgs } from "./fusion-args.js";
|
|
14
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
decidePanelCompletion,
|
|
17
|
+
resolveMinimumSuccessfulPanelists,
|
|
18
|
+
} from "./panel-completion.js";
|
|
15
19
|
import {
|
|
16
20
|
renderCancelledReport,
|
|
17
21
|
renderFailureReport,
|
|
@@ -20,13 +24,23 @@ import {
|
|
|
20
24
|
import {
|
|
21
25
|
extractPanelResults,
|
|
22
26
|
type ExtractPanelResultsSuccess,
|
|
27
|
+
type ExtractPanelResultsResult,
|
|
23
28
|
} from "./result-extract.js";
|
|
24
29
|
import {
|
|
25
30
|
reconcileIndexedLifecycleResult,
|
|
26
31
|
reconcilePanelResults,
|
|
32
|
+
type ReconcilePanelResultsOptions,
|
|
27
33
|
} from "./lifecycle-reconcile.js";
|
|
28
|
-
import {
|
|
29
|
-
|
|
34
|
+
import {
|
|
35
|
+
appendThinkingSuffix,
|
|
36
|
+
buildPanelSpawnParams,
|
|
37
|
+
resolveEffectiveTimeouts,
|
|
38
|
+
} from "./run-builder.js";
|
|
39
|
+
import {
|
|
40
|
+
FusionRunStore,
|
|
41
|
+
FusionRunStoreError,
|
|
42
|
+
validateFusionRunPanelSlots,
|
|
43
|
+
} from "./run-store.js";
|
|
30
44
|
import {
|
|
31
45
|
clearFusionUi,
|
|
32
46
|
extractFusionProgressCounts,
|
|
@@ -45,8 +59,10 @@ import {
|
|
|
45
59
|
resolveSynthesisMode,
|
|
46
60
|
type FailedPanelSummary,
|
|
47
61
|
type FusionProfile,
|
|
62
|
+
type FusionProfileSnapshot,
|
|
48
63
|
type FusionRun,
|
|
49
64
|
type PanelOutput,
|
|
65
|
+
type PanelDeadlineState,
|
|
50
66
|
type ParsedFusionArgs,
|
|
51
67
|
} from "./types.js";
|
|
52
68
|
import {
|
|
@@ -54,7 +70,7 @@ import {
|
|
|
54
70
|
hasStrongPanelAgreement,
|
|
55
71
|
mergeRunObservations,
|
|
56
72
|
} from "./run-observations.js";
|
|
57
|
-
import type { SubagentsTargetParams } from "./subagents-rpc.js";
|
|
73
|
+
import type { SubagentsTargetParams, SubagentsSteerParams } from "./subagents-rpc.js";
|
|
58
74
|
|
|
59
75
|
export const SUBAGENT_ASYNC_COMPLETE_EVENT = "subagent:async-complete";
|
|
60
76
|
|
|
@@ -83,6 +99,7 @@ export interface FusionRpcClientLike {
|
|
|
83
99
|
status(params?: SubagentsTargetParams): Promise<unknown>;
|
|
84
100
|
stop(params: SubagentsTargetParams): Promise<unknown>;
|
|
85
101
|
interrupt(params: SubagentsTargetParams): Promise<unknown>;
|
|
102
|
+
steer?(params: SubagentsSteerParams): Promise<unknown>;
|
|
86
103
|
}
|
|
87
104
|
|
|
88
105
|
export interface FusionMessageSink {
|
|
@@ -91,7 +108,7 @@ export interface FusionMessageSink {
|
|
|
91
108
|
content: string;
|
|
92
109
|
display: boolean;
|
|
93
110
|
details?: unknown;
|
|
94
|
-
}): void;
|
|
111
|
+
}, options?: { triggerTurn: boolean; deliverAs: "steer" }): void;
|
|
95
112
|
}
|
|
96
113
|
|
|
97
114
|
export interface FusionOrchestratorDeps {
|
|
@@ -113,6 +130,8 @@ export type FusionCommandResult =
|
|
|
113
130
|
interface RunLifecycleSnapshot {
|
|
114
131
|
statusPayload?: unknown;
|
|
115
132
|
resultPayload?: unknown;
|
|
133
|
+
/** Compact event results need stable slots; status/artifacts are ordered snapshots. */
|
|
134
|
+
resultSource?: "artifact" | "event" | "status";
|
|
116
135
|
resultIsTerminal: boolean;
|
|
117
136
|
resultArtifactPending?: boolean;
|
|
118
137
|
}
|
|
@@ -130,6 +149,7 @@ export class FusionOrchestrator {
|
|
|
130
149
|
private reconcileTimer: NodeJS.Timeout | undefined;
|
|
131
150
|
private reconciling = false;
|
|
132
151
|
private pendingCompletionPayload: unknown;
|
|
152
|
+
private readonly incompleteTerminalSince = new Map<string, number>();
|
|
133
153
|
|
|
134
154
|
constructor(deps: FusionOrchestratorDeps) {
|
|
135
155
|
this.rpc = deps.rpc;
|
|
@@ -146,6 +166,11 @@ export class FusionOrchestrator {
|
|
|
146
166
|
this.context = ctx;
|
|
147
167
|
|
|
148
168
|
const args = typeof input === "string" ? parseFusionArgs(input) : input;
|
|
169
|
+
const inputError = validateStartArgs(args);
|
|
170
|
+
if (inputError) {
|
|
171
|
+
this.notify(ctx, inputError, "error");
|
|
172
|
+
return { status: "failed", error: inputError };
|
|
173
|
+
}
|
|
149
174
|
const existing = this.runStore.getActiveRun();
|
|
150
175
|
if (existing) {
|
|
151
176
|
const message = `Fusion run ${existing.id} is already active.`;
|
|
@@ -153,8 +178,9 @@ export class FusionOrchestrator {
|
|
|
153
178
|
return { status: "conflict", activeRunId: existing.id };
|
|
154
179
|
}
|
|
155
180
|
|
|
181
|
+
let subagentsInfo: unknown;
|
|
156
182
|
try {
|
|
157
|
-
await this.rpc.ping();
|
|
183
|
+
subagentsInfo = await this.rpc.ping();
|
|
158
184
|
this.installWarning = undefined;
|
|
159
185
|
} catch (error: unknown) {
|
|
160
186
|
const message = `pi-subagents RPC is unavailable: ${errorMessage(error)}`;
|
|
@@ -188,6 +214,11 @@ export class FusionOrchestrator {
|
|
|
188
214
|
);
|
|
189
215
|
resolved = this.resolveProfile(aliased, inlineName);
|
|
190
216
|
}
|
|
217
|
+
if (resolved.profile.panelistSoftTimeoutMs !== undefined &&
|
|
218
|
+
(!this.rpc.steer || !isRecord(subagentsInfo) || !isRecord(subagentsInfo.capabilities) ||
|
|
219
|
+
subagentsInfo.capabilities.nonRecoveringSteer !== true)) {
|
|
220
|
+
throw new FusionArgsError("Soft deadlines require pi-subagents RPC with nonRecoveringSteer. Update pi-subagents and reload Pi, or omit panelistSoftTimeoutMs.");
|
|
221
|
+
}
|
|
191
222
|
this.configWarning = undefined;
|
|
192
223
|
} catch (error: unknown) {
|
|
193
224
|
const message = errorMessage(error);
|
|
@@ -198,6 +229,7 @@ export class FusionOrchestrator {
|
|
|
198
229
|
|
|
199
230
|
const outputContract =
|
|
200
231
|
args.outputContract ?? detectCallerOutputContract(args.prompt);
|
|
232
|
+
const profileSnapshot = snapshotProfile(resolved.profile);
|
|
201
233
|
let run: FusionRun;
|
|
202
234
|
try {
|
|
203
235
|
run = this.runStore.startRun({
|
|
@@ -210,10 +242,24 @@ export class FusionOrchestrator {
|
|
|
210
242
|
? { operationId: args.operationId }
|
|
211
243
|
: {}),
|
|
212
244
|
...(outputContract ? { outputContract } : {}),
|
|
245
|
+
// profileSnapshot is the sole quorum record for new runs. The
|
|
246
|
+
// run-level field remains readable only for legacy snapshots.
|
|
247
|
+
profileSnapshot,
|
|
248
|
+
...(args.timeoutOverrides
|
|
249
|
+
? { timeoutOverrides: args.timeoutOverrides }
|
|
250
|
+
: {}),
|
|
251
|
+
effectiveTimeouts: resolveEffectiveTimeouts(
|
|
252
|
+
resolved.profile,
|
|
253
|
+
args.timeoutOverrides,
|
|
254
|
+
),
|
|
213
255
|
phase: "panel",
|
|
214
256
|
});
|
|
215
257
|
} catch (error: unknown) {
|
|
216
|
-
if (!(error instanceof FusionRunStoreError))
|
|
258
|
+
if (!(error instanceof FusionRunStoreError)) {
|
|
259
|
+
const message = errorMessage(error);
|
|
260
|
+
this.notify(ctx, message, "error");
|
|
261
|
+
return { status: "failed", error: message };
|
|
262
|
+
}
|
|
217
263
|
const active = this.runStore.getActiveRun();
|
|
218
264
|
if (active) {
|
|
219
265
|
this.notify(
|
|
@@ -225,12 +271,25 @@ export class FusionOrchestrator {
|
|
|
225
271
|
}
|
|
226
272
|
return { status: "failed", error: errorMessage(error) };
|
|
227
273
|
}
|
|
228
|
-
|
|
274
|
+
// Keep runtime behavior aligned with the exact durable profile that a
|
|
275
|
+
// restart will use, rather than retaining a mutable config object.
|
|
276
|
+
this.activeProfile = profileFromSnapshot(profileSnapshot);
|
|
229
277
|
publishFusionStatus(ctx, run);
|
|
230
278
|
|
|
231
279
|
try {
|
|
280
|
+
// Persist before the side effect. Public pi-subagents RPC has no
|
|
281
|
+
// correlation-key lookup, so restore treats this intent without its ID
|
|
282
|
+
// as unsafe to replay rather than creating an orphaned duplicate.
|
|
283
|
+
this.runStore.updateRun(run.id, {
|
|
284
|
+
spawnIntent: { stage: "panel", requestedAt: Date.now() },
|
|
285
|
+
});
|
|
232
286
|
const spawnResult = await this.rpc.spawn(
|
|
233
|
-
buildPanelSpawnParams(
|
|
287
|
+
buildPanelSpawnParams(
|
|
288
|
+
resolved.profile,
|
|
289
|
+
args.prompt,
|
|
290
|
+
outputContract,
|
|
291
|
+
args.timeoutOverrides,
|
|
292
|
+
),
|
|
234
293
|
);
|
|
235
294
|
const spawnError = extractSubagentFailure(spawnResult);
|
|
236
295
|
if (spawnError) throw new FusionArgsError(spawnError);
|
|
@@ -251,10 +310,19 @@ export class FusionOrchestrator {
|
|
|
251
310
|
? { status: "cancelled", run: cancelled, report: cancelled.report }
|
|
252
311
|
: { status: "ignored" };
|
|
253
312
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
313
|
+
let updated: FusionRun;
|
|
314
|
+
try {
|
|
315
|
+
updated = this.runStore.updateRun(run.id, {
|
|
316
|
+
spawnIntent: null,
|
|
317
|
+
panelRunId,
|
|
318
|
+
...(panelAsyncDir ? { panelAsyncDir } : {}),
|
|
319
|
+
});
|
|
320
|
+
} catch (persistenceError: unknown) {
|
|
321
|
+
// The remote run is now known but its ID is not durable. It cannot be
|
|
322
|
+
// safely recovered through the public RPC, so stop it before failing.
|
|
323
|
+
await this.stopOrphanedRun(panelRunId);
|
|
324
|
+
throw persistenceError;
|
|
325
|
+
}
|
|
258
326
|
publishFusionStatus(ctx, updated);
|
|
259
327
|
this.ensureReconcileLoop();
|
|
260
328
|
this.notify(
|
|
@@ -341,7 +409,7 @@ export class FusionOrchestrator {
|
|
|
341
409
|
this.context,
|
|
342
410
|
active,
|
|
343
411
|
progress,
|
|
344
|
-
deriveFusionStatusPhase(active, statusPayload),
|
|
412
|
+
deriveFusionStatusPhase(active, statusPayload, this.activeProfile),
|
|
345
413
|
);
|
|
346
414
|
}
|
|
347
415
|
return statusPayload;
|
|
@@ -419,34 +487,72 @@ export class FusionOrchestrator {
|
|
|
419
487
|
const summary = this.runStore.restoreFromSession(ctx);
|
|
420
488
|
this.clearActiveRuntime();
|
|
421
489
|
|
|
490
|
+
const restoreError = this.runStore.getRestoreError();
|
|
491
|
+
if (restoreError) {
|
|
492
|
+
this.configWarning = restoreError;
|
|
493
|
+
this.notify(ctx, restoreError, "warning");
|
|
494
|
+
clearFusionUi(ctx);
|
|
495
|
+
return summary;
|
|
496
|
+
}
|
|
497
|
+
|
|
422
498
|
const active = this.runStore.getActiveRun();
|
|
499
|
+
if (active && hasUnresolvedSpawnIntent(active)) {
|
|
500
|
+
const message = `Fusion recovery stopped: ${active.spawnIntent!.stage} spawn may have reached pi-subagents, but its run ID was not persisted. It will not be replayed because public RPC cannot safely adopt it.`;
|
|
501
|
+
this.failActiveRun(message);
|
|
502
|
+
this.notify(ctx, message, "warning");
|
|
503
|
+
return this.runStore.getLastRunSummary();
|
|
504
|
+
}
|
|
423
505
|
if (!active) {
|
|
424
506
|
this.stopReconcileLoop();
|
|
425
507
|
clearFusionUi(ctx);
|
|
426
508
|
return summary;
|
|
427
509
|
}
|
|
428
510
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
)
|
|
441
|
-
this.activeProfile = active.inlinePanel?.length
|
|
442
|
-
? buildInlinePanelProfile(base, active.inlinePanel)
|
|
443
|
-
: base;
|
|
511
|
+
const lifecycleError = validateRestoredRunLifecycle(active);
|
|
512
|
+
if (lifecycleError) {
|
|
513
|
+
this.failActiveRun(lifecycleError);
|
|
514
|
+
this.notify(ctx, lifecycleError, "warning");
|
|
515
|
+
return this.runStore.getLastRunSummary();
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
if (active.profileSnapshot) {
|
|
519
|
+
// The snapshot was schema-validated while deserializing the run. It is
|
|
520
|
+
// the source of truth for labels, quorum, synthesis, and judge spawning;
|
|
521
|
+
// config edits made while a run is active must not rewrite that run.
|
|
522
|
+
this.activeProfile = profileFromSnapshot(active.profileSnapshot);
|
|
444
523
|
this.configWarning = undefined;
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
524
|
+
} else {
|
|
525
|
+
try {
|
|
526
|
+
const config = await this.loadConfig(ctx);
|
|
527
|
+
// Backward compatibility for sessions written before profile snapshots.
|
|
528
|
+
// An inline run's display name has no config entry, so rebuild it from
|
|
529
|
+
// its base profile plus persisted inline entries.
|
|
530
|
+
const base = this.resolveProfile(
|
|
531
|
+
config,
|
|
532
|
+
active.inlinePanel?.length
|
|
533
|
+
? active.baseProfileName
|
|
534
|
+
: active.profileName,
|
|
535
|
+
).profile;
|
|
536
|
+
this.activeProfile = active.inlinePanel?.length
|
|
537
|
+
? buildInlinePanelProfile(base, active.inlinePanel)
|
|
538
|
+
: base;
|
|
539
|
+
this.configWarning = undefined;
|
|
540
|
+
} catch (error: unknown) {
|
|
541
|
+
const message = `Could not restore legacy fusion profile "${active.profileName}": ${errorMessage(error)}`;
|
|
542
|
+
this.configWarning = message;
|
|
543
|
+
this.notify(ctx, message, "warning");
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
const panelSlotError = validateFusionRunPanelSlots(
|
|
547
|
+
active,
|
|
548
|
+
this.activeProfile?.panel.length ?? 0,
|
|
549
|
+
);
|
|
550
|
+
if (panelSlotError) {
|
|
551
|
+
this.failActiveRun(panelSlotError);
|
|
552
|
+
this.notify(ctx, panelSlotError, "warning");
|
|
553
|
+
return this.runStore.getLastRunSummary();
|
|
449
554
|
}
|
|
555
|
+
|
|
450
556
|
publishFusionStatus(ctx, active);
|
|
451
557
|
this.ensureReconcileLoop();
|
|
452
558
|
await this.reconcileActiveRun();
|
|
@@ -514,6 +620,109 @@ export class FusionOrchestrator {
|
|
|
514
620
|
return this.runStore.getActiveRun();
|
|
515
621
|
}
|
|
516
622
|
|
|
623
|
+
async resolvePanelDeadline(
|
|
624
|
+
runId: string,
|
|
625
|
+
panelist: number,
|
|
626
|
+
decision: "continue" | "finish",
|
|
627
|
+
): Promise<{ decision: string; receipt: unknown }> {
|
|
628
|
+
const active = this.runStore.getActiveRun();
|
|
629
|
+
if (!active || active.id !== runId || active.phase !== "panel" || !active.panelRunId) {
|
|
630
|
+
throw new FusionArgsError("The requested Fusion panel is no longer active.");
|
|
631
|
+
}
|
|
632
|
+
if (!Number.isInteger(panelist) || panelist < 1 || (decision !== "continue" && decision !== "finish")) {
|
|
633
|
+
throw new FusionArgsError("Expected a one-based panelist number and continue or finish.");
|
|
634
|
+
}
|
|
635
|
+
const state = active.panelDeadlines?.find((item) => item.index === panelist - 1);
|
|
636
|
+
if (!state || state.status !== "pending") throw new FusionArgsError("No pending deadline decision for that panelist.");
|
|
637
|
+
const payload = await this.rpc.status({ id: active.panelRunId });
|
|
638
|
+
const matches = findStepsArray(payload).filter((step) => isRecord(step) &&
|
|
639
|
+
step.runId === state.childRunId && (step.status ?? step.state) === "running" &&
|
|
640
|
+
(step.workflowKey ?? step.key ?? step.agent) === `panel-${panelist}`);
|
|
641
|
+
const latest = this.runStore.getActiveRun();
|
|
642
|
+
if (latest?.id !== runId || latest.phase !== "panel" ||
|
|
643
|
+
latest.panelDeadlines?.find((item) => item.index === state.index)?.status !== "pending" ||
|
|
644
|
+
isTerminalSubagentState(extractSubagentState(payload)) || matches.length !== 1 ||
|
|
645
|
+
Date.now() >= Math.min(state.requestedAt + PANEL_DECISION_WAIT_MS, state.finalizeAt)) {
|
|
646
|
+
throw new FusionArgsError("Deadline decision expired or the panelist is no longer running.");
|
|
647
|
+
}
|
|
648
|
+
const updated: PanelDeadlineState = { ...state, status: decision === "continue" ? "continued" : "finishing" };
|
|
649
|
+
this.savePanelDeadline(runId, updated);
|
|
650
|
+
const receipt = await this.steerPanelDeadline(runId, updated);
|
|
651
|
+
return { decision, receipt };
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
private savePanelDeadline(runId: string, state: PanelDeadlineState): void {
|
|
655
|
+
const active = this.runStore.getActiveRun();
|
|
656
|
+
if (active?.id !== runId || active.phase !== "panel") return;
|
|
657
|
+
this.runStore.updateRun(runId, {
|
|
658
|
+
panelDeadlines: [...(active.panelDeadlines ?? []).filter((item) => item.index !== state.index), state],
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
private async steerPanelDeadline(runId: string, state: PanelDeadlineState): Promise<unknown> {
|
|
663
|
+
try {
|
|
664
|
+
if (!this.rpc.steer) throw new Error("pi-subagents steer RPC is unavailable.");
|
|
665
|
+
return await this.rpc.steer({ id: state.childRunId, message: deadlineSteerMessage(state), mode: "auto" });
|
|
666
|
+
} catch (error: unknown) {
|
|
667
|
+
const message = `Could not send deadline guidance to panel-${state.index + 1}: ${errorMessage(error)}`;
|
|
668
|
+
const latest = this.runStore.getActiveRun();
|
|
669
|
+
if (latest?.id === runId && latest.panelDeadlines?.find((item) => item.index === state.index)?.status === state.status) {
|
|
670
|
+
this.savePanelDeadline(runId, { ...state, deliveryError: message });
|
|
671
|
+
}
|
|
672
|
+
this.notify(this.context, message, "warning");
|
|
673
|
+
throw new Error(message, { cause: error });
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
private async handlePanelDeadlines(active: FusionRun, statusPayload: unknown): Promise<void> {
|
|
678
|
+
const actions = planPanelDeadlines(this.runStore.getActiveRun() ?? active, findStepsArray(statusPayload), Date.now());
|
|
679
|
+
for (const action of actions) {
|
|
680
|
+
if (this.runStore.getActiveRun()?.id !== active.id) return;
|
|
681
|
+
this.savePanelDeadline(active.id, action.state);
|
|
682
|
+
if (action.kind === "ask") {
|
|
683
|
+
const panelist = action.state.index + 1;
|
|
684
|
+
const member = active.profileSnapshot?.panel[action.state.index];
|
|
685
|
+
const step = findStepsArray(statusPayload).find((item) => isRecord(item) && item.runId === action.state.childRunId);
|
|
686
|
+
const content = [
|
|
687
|
+
`Fusion soft deadline: ${member ? memberLabel(member) : `panel-${panelist}`}.`,
|
|
688
|
+
`Fusion run: ${active.id}. Child run: ${action.state.childRunId}.`,
|
|
689
|
+
`Last observed activity (not verified findings): ${describeStepActivity(step)?.slice(0,500) ?? "unknown"}.`,
|
|
690
|
+
`Call resolve_fusion_deadline with runId=${active.id}, panelist=${panelist}, decision=continue or finish. Ask the user if the extra work needs their decision.`,
|
|
691
|
+
`Without a decision within ${PANEL_DECISION_WAIT_MS / 1000}s, the panelist will be asked to return its current answer. One continuation is allowed, only within the existing hard deadline.`,
|
|
692
|
+
`User command: /fusion continue ${active.id} ${panelist} or /fusion finish ${active.id} ${panelist}.`,
|
|
693
|
+
].join("\n");
|
|
694
|
+
this.sendMessage?.({ customType: "fusion-deadline", content, display: true }, { triggerTurn: true, deliverAs: "steer" });
|
|
695
|
+
}
|
|
696
|
+
try {
|
|
697
|
+
await this.steerPanelDeadline(active.id, action.state);
|
|
698
|
+
} catch {
|
|
699
|
+
// The recorded delivery error is visible; never revive a child or reset its budget.
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
private reconcileTerminalPanel(
|
|
705
|
+
active: FusionRun,
|
|
706
|
+
extracted: ExtractPanelResultsSuccess,
|
|
707
|
+
statusPayload: unknown,
|
|
708
|
+
profile: FusionProfile,
|
|
709
|
+
lifecyclePayload: unknown,
|
|
710
|
+
options: ReconcilePanelResultsOptions,
|
|
711
|
+
): ExtractPanelResultsResult | undefined {
|
|
712
|
+
const since = this.incompleteTerminalSince.get(active.id);
|
|
713
|
+
const graceExpired = since !== undefined && Date.now() - since >= WORKFLOW_RESULT_ARTIFACT_GRACE_MS;
|
|
714
|
+
const result = reconcilePanelResults(extracted, statusPayload, profile, lifecyclePayload, {
|
|
715
|
+
...options,
|
|
716
|
+
...(graceExpired && options.terminalizeRunning ? { allowMissingAtDeadline: true } : {}),
|
|
717
|
+
});
|
|
718
|
+
if (!result.ok && result.error.code === "incomplete-lifecycle" && !graceExpired) {
|
|
719
|
+
if (since === undefined) this.incompleteTerminalSince.set(active.id, Date.now());
|
|
720
|
+
return undefined;
|
|
721
|
+
}
|
|
722
|
+
this.incompleteTerminalSince.delete(active.id);
|
|
723
|
+
return result;
|
|
724
|
+
}
|
|
725
|
+
|
|
517
726
|
private async reconcileActiveRun(
|
|
518
727
|
eventPayload?: unknown,
|
|
519
728
|
): Promise<FusionCommandResult> {
|
|
@@ -529,13 +738,13 @@ export class FusionOrchestrator {
|
|
|
529
738
|
this.reconciling = true;
|
|
530
739
|
try {
|
|
531
740
|
if (active.phase === "panel") {
|
|
532
|
-
return this.handleLegacyPanelComplete(active, eventPayload);
|
|
741
|
+
return await this.handleLegacyPanelComplete(active, eventPayload);
|
|
533
742
|
}
|
|
534
743
|
if (active.phase === "chain") {
|
|
535
|
-
return this.handleChainComplete(active, eventPayload);
|
|
744
|
+
return await this.handleChainComplete(active, eventPayload);
|
|
536
745
|
}
|
|
537
746
|
if (active.phase === "judge") {
|
|
538
|
-
return this.handleJudgeComplete(active, eventPayload);
|
|
747
|
+
return await this.handleJudgeComplete(active, eventPayload);
|
|
539
748
|
}
|
|
540
749
|
return { status: "ignored" };
|
|
541
750
|
} finally {
|
|
@@ -570,6 +779,7 @@ export class FusionOrchestrator {
|
|
|
570
779
|
eventPayload: payload,
|
|
571
780
|
});
|
|
572
781
|
if (snapshot.resultArtifactPending) return { status: "ignored" };
|
|
782
|
+
this.persistVerifiedPanelResults(active, profile, snapshot.statusPayload);
|
|
573
783
|
const terminalPayload =
|
|
574
784
|
snapshot.resultPayload ?? snapshot.statusPayload ?? payload;
|
|
575
785
|
if (
|
|
@@ -589,6 +799,9 @@ export class FusionOrchestrator {
|
|
|
589
799
|
const extracted = extractPanelResults(lifecyclePayload, {
|
|
590
800
|
panel: profile.panel,
|
|
591
801
|
limit: profile.panel.length,
|
|
802
|
+
...(snapshot.resultSource === "event"
|
|
803
|
+
? { requireStableSlotIdentity: true }
|
|
804
|
+
: {}),
|
|
592
805
|
});
|
|
593
806
|
if (!extracted.ok) {
|
|
594
807
|
return this.failActiveRun(
|
|
@@ -596,13 +809,21 @@ export class FusionOrchestrator {
|
|
|
596
809
|
);
|
|
597
810
|
}
|
|
598
811
|
|
|
599
|
-
const observedPanels =
|
|
812
|
+
const observedPanels = this.reconcileTerminalPanel(
|
|
813
|
+
active,
|
|
600
814
|
extracted,
|
|
601
815
|
snapshot.statusPayload,
|
|
602
816
|
profile,
|
|
603
817
|
lifecyclePayload,
|
|
604
|
-
{
|
|
818
|
+
{
|
|
819
|
+
allowedTrailingResults: 1,
|
|
820
|
+
...(isWorkflowDeadline(lifecyclePayload) ||
|
|
821
|
+
isWorkflowDeadline(snapshot.statusPayload)
|
|
822
|
+
? { terminalizeRunning: true }
|
|
823
|
+
: {}),
|
|
824
|
+
},
|
|
605
825
|
);
|
|
826
|
+
if (!observedPanels) return { status: "ignored" };
|
|
606
827
|
if (!observedPanels.ok) {
|
|
607
828
|
return this.failActiveRun(
|
|
608
829
|
`${observedPanels.error.message} (${observedPanels.error.path})`,
|
|
@@ -614,6 +835,26 @@ export class FusionOrchestrator {
|
|
|
614
835
|
observedPanels.failures,
|
|
615
836
|
);
|
|
616
837
|
|
|
838
|
+
// A legacy workflow may contain an embedded judge result even when the
|
|
839
|
+
// current persisted quorum policy would not permit synthesis. Apply the
|
|
840
|
+
// same completion decision as modern panel-only runs before accepting it.
|
|
841
|
+
const completion = decidePanelCompletion({
|
|
842
|
+
run: updated,
|
|
843
|
+
profile,
|
|
844
|
+
panelOutputs: observedPanels.outputs,
|
|
845
|
+
panelFailures: observedPanels.failures,
|
|
846
|
+
fallbackJudge: true,
|
|
847
|
+
});
|
|
848
|
+
if (completion.kind === "fail") {
|
|
849
|
+
return this.failActiveRun(completion.error, completion.report);
|
|
850
|
+
}
|
|
851
|
+
if (completion.kind === "complete") {
|
|
852
|
+
this.runStore.updateRun(updated.id, {
|
|
853
|
+
completionQuality: completionQuality(profile, observedPanels.outputs, observedPanels.failures),
|
|
854
|
+
});
|
|
855
|
+
return this.completeActiveRun(completion.report);
|
|
856
|
+
}
|
|
857
|
+
|
|
617
858
|
const judgePayload =
|
|
618
859
|
hasJudgeResult(snapshot.resultPayload, profile.panel.length)
|
|
619
860
|
? snapshot.resultPayload
|
|
@@ -641,6 +882,7 @@ export class FusionOrchestrator {
|
|
|
641
882
|
),
|
|
642
883
|
);
|
|
643
884
|
const observed = this.runStore.updateRun(updated.id, {
|
|
885
|
+
completionQuality: completionQuality(profile, observedPanels.outputs, observedPanels.failures),
|
|
644
886
|
judgeObservation,
|
|
645
887
|
});
|
|
646
888
|
const callerContract =
|
|
@@ -694,17 +936,24 @@ export class FusionOrchestrator {
|
|
|
694
936
|
});
|
|
695
937
|
if (snapshot.resultArtifactPending) return { status: "ignored" };
|
|
696
938
|
|
|
939
|
+
const partial = this.persistVerifiedPanelResults(
|
|
940
|
+
active,
|
|
941
|
+
profile,
|
|
942
|
+
snapshot.statusPayload,
|
|
943
|
+
);
|
|
697
944
|
const panelIsTerminal =
|
|
698
945
|
snapshot.resultIsTerminal ||
|
|
699
946
|
isTerminalSubagentState(extractSubagentState(snapshot.statusPayload));
|
|
947
|
+
if (!panelIsTerminal) await this.handlePanelDeadlines(active, snapshot.statusPayload);
|
|
700
948
|
if (!active.panelStopReason && !panelIsTerminal) {
|
|
701
|
-
const partial = extractPanelResults(snapshot.statusPayload, {
|
|
702
|
-
panel: profile.panel,
|
|
703
|
-
completedOnly: true,
|
|
704
|
-
});
|
|
705
949
|
if (
|
|
706
|
-
partial
|
|
707
|
-
shouldStopWhenPanelAgrees(
|
|
950
|
+
partial &&
|
|
951
|
+
shouldStopWhenPanelAgrees(
|
|
952
|
+
profile,
|
|
953
|
+
partial.outputs,
|
|
954
|
+
partial.failures,
|
|
955
|
+
active.minimumSuccessfulPanelists,
|
|
956
|
+
)
|
|
708
957
|
) {
|
|
709
958
|
return this.stopPanelAfterAgreement(
|
|
710
959
|
active,
|
|
@@ -733,6 +982,11 @@ export class FusionOrchestrator {
|
|
|
733
982
|
const workflowStoppedIndices = extractWorkflowStoppedPanelIndices(
|
|
734
983
|
lifecyclePayload,
|
|
735
984
|
);
|
|
985
|
+
if (workflowStoppedIndices.some((index) => index >= profile.panel.length)) {
|
|
986
|
+
return this.failActiveRun(
|
|
987
|
+
"Terminal subagents data referenced a panel slot outside the configured panel.",
|
|
988
|
+
);
|
|
989
|
+
}
|
|
736
990
|
const stoppedPanelIndices =
|
|
737
991
|
active.panelStoppedIndices ??
|
|
738
992
|
(workflowStoppedIndices.length > 0 ? workflowStoppedIndices : undefined);
|
|
@@ -740,6 +994,9 @@ export class FusionOrchestrator {
|
|
|
740
994
|
panel: profile.panel,
|
|
741
995
|
limit: profile.panel.length,
|
|
742
996
|
...(stoppedPanelIndices ? { stoppedPanelIndices } : {}),
|
|
997
|
+
...(snapshot.resultSource === "event"
|
|
998
|
+
? { requireStableSlotIdentity: true }
|
|
999
|
+
: {}),
|
|
743
1000
|
});
|
|
744
1001
|
if (!extracted.ok) {
|
|
745
1002
|
return this.failActiveRun(
|
|
@@ -747,22 +1004,38 @@ export class FusionOrchestrator {
|
|
|
747
1004
|
);
|
|
748
1005
|
}
|
|
749
1006
|
|
|
750
|
-
const observedPanels =
|
|
1007
|
+
const observedPanels = this.reconcileTerminalPanel(
|
|
1008
|
+
active,
|
|
751
1009
|
extracted,
|
|
752
1010
|
snapshot.statusPayload,
|
|
753
1011
|
profile,
|
|
754
1012
|
lifecyclePayload,
|
|
755
|
-
{
|
|
1013
|
+
{
|
|
1014
|
+
...(stoppedPanelIndices ? { stoppedPanelIndices } : {}),
|
|
1015
|
+
...(isWorkflowDeadline(lifecyclePayload) ||
|
|
1016
|
+
isWorkflowDeadline(snapshot.statusPayload)
|
|
1017
|
+
? { terminalizeRunning: true }
|
|
1018
|
+
: {}),
|
|
1019
|
+
},
|
|
756
1020
|
);
|
|
1021
|
+
if (!observedPanels) return { status: "ignored" };
|
|
757
1022
|
if (!observedPanels.ok) {
|
|
758
1023
|
return this.failActiveRun(
|
|
759
1024
|
`${observedPanels.error.message} (${observedPanels.error.path})`,
|
|
760
1025
|
);
|
|
761
1026
|
}
|
|
1027
|
+
const reconciledFailures = withWorkflowDeadlineFailures(
|
|
1028
|
+
observedPanels.failures,
|
|
1029
|
+
isWorkflowDeadline(lifecyclePayload) ||
|
|
1030
|
+
isWorkflowDeadline(snapshot.statusPayload)
|
|
1031
|
+
? extractSubagentFailure(snapshot.statusPayload) ??
|
|
1032
|
+
extractSubagentFailure(lifecyclePayload)
|
|
1033
|
+
: undefined,
|
|
1034
|
+
);
|
|
762
1035
|
const stored = this.storePanelResults(
|
|
763
1036
|
active.id,
|
|
764
1037
|
observedPanels.outputs,
|
|
765
|
-
|
|
1038
|
+
reconciledFailures,
|
|
766
1039
|
);
|
|
767
1040
|
const updated =
|
|
768
1041
|
workflowStoppedIndices.length > 0 && !active.panelStopReason
|
|
@@ -776,7 +1049,7 @@ export class FusionOrchestrator {
|
|
|
776
1049
|
updated,
|
|
777
1050
|
profile,
|
|
778
1051
|
observedPanels.outputs,
|
|
779
|
-
|
|
1052
|
+
reconciledFailures,
|
|
780
1053
|
{ fallbackJudge: false },
|
|
781
1054
|
);
|
|
782
1055
|
}
|
|
@@ -854,10 +1127,16 @@ export class FusionOrchestrator {
|
|
|
854
1127
|
return this.failActiveRun(decision.error, decision.report);
|
|
855
1128
|
}
|
|
856
1129
|
if (decision.kind === "complete") {
|
|
1130
|
+
this.runStore.updateRun(run.id, {
|
|
1131
|
+
completionQuality: completionQuality(profile, panelOutputs, panelFailures),
|
|
1132
|
+
});
|
|
857
1133
|
return this.completeActiveRun(decision.report);
|
|
858
1134
|
}
|
|
859
1135
|
|
|
860
1136
|
try {
|
|
1137
|
+
this.runStore.updateRun(run.id, {
|
|
1138
|
+
spawnIntent: { stage: "judge", requestedAt: Date.now() },
|
|
1139
|
+
});
|
|
861
1140
|
const spawnResult = await this.rpc.spawn(decision.params);
|
|
862
1141
|
const spawnError = extractSubagentFailure(spawnResult);
|
|
863
1142
|
if (spawnError) throw new FusionArgsError(spawnError);
|
|
@@ -870,13 +1149,23 @@ export class FusionOrchestrator {
|
|
|
870
1149
|
await this.stopOrphanedRun(judgeRunId, "judge");
|
|
871
1150
|
return { status: "ignored" };
|
|
872
1151
|
}
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
1152
|
+
let nextRun: FusionRun;
|
|
1153
|
+
try {
|
|
1154
|
+
nextRun = this.runStore.updateRun(run.id, {
|
|
1155
|
+
spawnIntent: null,
|
|
1156
|
+
phase: "judge",
|
|
1157
|
+
completionQuality: completionQuality(profile, panelOutputs, panelFailures),
|
|
1158
|
+
judgeRunId,
|
|
1159
|
+
...(judgeAsyncDir ? { judgeAsyncDir } : {}),
|
|
1160
|
+
panelOutputs: [...panelOutputs],
|
|
1161
|
+
panelFailures: [...panelFailures],
|
|
1162
|
+
});
|
|
1163
|
+
} catch (persistenceError: unknown) {
|
|
1164
|
+
// As for the panel, never leave a remotely started synthesis run
|
|
1165
|
+
// alive when recording its public ID failed.
|
|
1166
|
+
await this.stopOrphanedRun(judgeRunId, "judge");
|
|
1167
|
+
throw persistenceError;
|
|
1168
|
+
}
|
|
880
1169
|
publishFusionStatus(this.context, nextRun);
|
|
881
1170
|
this.notify(
|
|
882
1171
|
this.context,
|
|
@@ -1005,7 +1294,7 @@ export class FusionOrchestrator {
|
|
|
1005
1294
|
this.context,
|
|
1006
1295
|
input.run,
|
|
1007
1296
|
progress,
|
|
1008
|
-
deriveFusionStatusPhase(input.run, statusPayload),
|
|
1297
|
+
deriveFusionStatusPhase(input.run, statusPayload, this.activeProfile),
|
|
1009
1298
|
);
|
|
1010
1299
|
}
|
|
1011
1300
|
|
|
@@ -1015,23 +1304,20 @@ export class FusionOrchestrator {
|
|
|
1015
1304
|
const eventIsTerminal =
|
|
1016
1305
|
isTerminalSubagentState(extractSubagentState(input.eventPayload)) ||
|
|
1017
1306
|
isTerminalFusionProgress(input.eventPayload);
|
|
1018
|
-
const eventHasResults =
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
resultIsTerminal: eventIsTerminal || statusIsTerminal,
|
|
1024
|
-
};
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1307
|
+
const eventHasResults = hasExplicitResultsArray(input.eventPayload);
|
|
1308
|
+
// Completion events are compact and can truncate long inline outputs. A
|
|
1309
|
+
// result artifact is the complete terminal record, so select it before an
|
|
1310
|
+
// event and use it as the reconciliation snapshot as well. Otherwise a
|
|
1311
|
+
// partial status/event could discard verified slots or the judge end tag.
|
|
1027
1312
|
const artifactResult = readSubagentResultArtifact({
|
|
1028
1313
|
...(input.runId ? { runId: input.runId } : {}),
|
|
1029
1314
|
...(input.asyncDir ? { asyncDir: input.asyncDir } : {}),
|
|
1030
1315
|
});
|
|
1031
|
-
if (
|
|
1316
|
+
if (hasExplicitResultsArray(artifactResult)) {
|
|
1032
1317
|
return {
|
|
1033
|
-
statusPayload,
|
|
1318
|
+
statusPayload: artifactResult,
|
|
1034
1319
|
resultPayload: artifactResult,
|
|
1320
|
+
resultSource: "artifact",
|
|
1035
1321
|
resultIsTerminal: true,
|
|
1036
1322
|
};
|
|
1037
1323
|
}
|
|
@@ -1047,10 +1333,20 @@ export class FusionOrchestrator {
|
|
|
1047
1333
|
};
|
|
1048
1334
|
}
|
|
1049
1335
|
|
|
1050
|
-
if (
|
|
1336
|
+
if (eventPayloadMatches && eventHasResults) {
|
|
1337
|
+
return {
|
|
1338
|
+
statusPayload,
|
|
1339
|
+
resultPayload: input.eventPayload,
|
|
1340
|
+
resultSource: "event",
|
|
1341
|
+
resultIsTerminal: eventIsTerminal || statusIsTerminal,
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
if (hasExplicitResultsArray(statusPayload)) {
|
|
1051
1346
|
return {
|
|
1052
1347
|
statusPayload,
|
|
1053
1348
|
resultPayload: statusPayload,
|
|
1349
|
+
resultSource: "status",
|
|
1054
1350
|
resultIsTerminal: statusIsTerminal,
|
|
1055
1351
|
};
|
|
1056
1352
|
}
|
|
@@ -1074,14 +1370,64 @@ export class FusionOrchestrator {
|
|
|
1074
1370
|
return { statusPayload, resultIsTerminal: false };
|
|
1075
1371
|
}
|
|
1076
1372
|
|
|
1373
|
+
/** Persists every terminal child slot observed while the workflow continues. */
|
|
1374
|
+
private persistVerifiedPanelResults(
|
|
1375
|
+
run: FusionRun,
|
|
1376
|
+
profile: FusionProfile,
|
|
1377
|
+
statusPayload: unknown,
|
|
1378
|
+
): ExtractPanelResultsSuccess | undefined {
|
|
1379
|
+
if (statusPayload === undefined) return undefined;
|
|
1380
|
+
const partial = extractPanelResults(statusPayload, {
|
|
1381
|
+
panel: profile.panel,
|
|
1382
|
+
completedOnly: true,
|
|
1383
|
+
limit: profile.panel.length,
|
|
1384
|
+
});
|
|
1385
|
+
if (!partial.ok) return undefined;
|
|
1386
|
+
if (partial.outputs.length > 0 || partial.failures.length > 0) {
|
|
1387
|
+
this.storePanelResults(run.id, partial.outputs, partial.failures);
|
|
1388
|
+
}
|
|
1389
|
+
return partial;
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1077
1392
|
private storePanelResults(
|
|
1078
1393
|
runId: string,
|
|
1079
1394
|
panelOutputs: readonly PanelOutput[],
|
|
1080
1395
|
panelFailures: readonly FailedPanelSummary[],
|
|
1081
1396
|
): FusionRun {
|
|
1397
|
+
// Lifecycle status is append-only in intent but not guaranteed to repeat
|
|
1398
|
+
// older slots on every poll. Keep persisted verified slots until a newer
|
|
1399
|
+
// observation for that exact stable index supersedes them.
|
|
1400
|
+
const existing = this.runStore.getActiveRun();
|
|
1401
|
+
const slots = new Map<
|
|
1402
|
+
number,
|
|
1403
|
+
{ output?: PanelOutput; failure?: FailedPanelSummary }
|
|
1404
|
+
>();
|
|
1405
|
+
for (const output of existing?.panelOutputs ?? []) {
|
|
1406
|
+
slots.set(output.index, { output });
|
|
1407
|
+
}
|
|
1408
|
+
for (const failure of existing?.panelFailures ?? []) {
|
|
1409
|
+
slots.set(failure.index, { failure });
|
|
1410
|
+
}
|
|
1411
|
+
for (const output of panelOutputs) slots.set(output.index, { output });
|
|
1412
|
+
for (const failure of panelFailures) slots.set(failure.index, { failure });
|
|
1413
|
+
const mergedOutputs = Array.from(slots.values())
|
|
1414
|
+
.flatMap((slot) => (slot.output ? [slot.output] : []))
|
|
1415
|
+
.sort((left, right) => left.index - right.index);
|
|
1416
|
+
const mergedFailures = Array.from(slots.values())
|
|
1417
|
+
.flatMap((slot) => (slot.failure ? [slot.failure] : []))
|
|
1418
|
+
.sort((left, right) => left.index - right.index);
|
|
1419
|
+
// Status polling repeats complete snapshots. Persist only a material slot
|
|
1420
|
+
// change so a long-running restore does not append identical session data.
|
|
1421
|
+
if (
|
|
1422
|
+
existing?.id === runId &&
|
|
1423
|
+
sameSnapshot(existing.panelOutputs, mergedOutputs) &&
|
|
1424
|
+
sameSnapshot(existing.panelFailures, mergedFailures)
|
|
1425
|
+
) {
|
|
1426
|
+
return existing;
|
|
1427
|
+
}
|
|
1082
1428
|
return this.runStore.updateRun(runId, {
|
|
1083
|
-
panelOutputs:
|
|
1084
|
-
panelFailures:
|
|
1429
|
+
panelOutputs: mergedOutputs,
|
|
1430
|
+
panelFailures: mergedFailures,
|
|
1085
1431
|
});
|
|
1086
1432
|
}
|
|
1087
1433
|
|
|
@@ -1112,6 +1458,12 @@ export class FusionOrchestrator {
|
|
|
1112
1458
|
...(active.chainRunId ? { chainRunId: active.chainRunId } : {}),
|
|
1113
1459
|
...(active.panelRunId ? { panelRunId: active.panelRunId } : {}),
|
|
1114
1460
|
...(active.judgeRunId ? { judgeRunId: active.judgeRunId } : {}),
|
|
1461
|
+
recovery: {
|
|
1462
|
+
retryDeferred: true,
|
|
1463
|
+
failedPanelIndices: storedPanelFailures(active)
|
|
1464
|
+
.map((failure) => failure.index)
|
|
1465
|
+
.sort((left, right) => left - right),
|
|
1466
|
+
},
|
|
1115
1467
|
report,
|
|
1116
1468
|
error,
|
|
1117
1469
|
});
|
|
@@ -1149,6 +1501,7 @@ export class FusionOrchestrator {
|
|
|
1149
1501
|
}
|
|
1150
1502
|
|
|
1151
1503
|
private clearActiveRuntime(): void {
|
|
1504
|
+
this.incompleteTerminalSince.clear();
|
|
1152
1505
|
this.activeProfile = undefined;
|
|
1153
1506
|
this.stopReconcileLoop();
|
|
1154
1507
|
}
|
|
@@ -1199,6 +1552,124 @@ export class FusionOrchestrator {
|
|
|
1199
1552
|
}
|
|
1200
1553
|
}
|
|
1201
1554
|
|
|
1555
|
+
function sameSnapshot<T>(
|
|
1556
|
+
left: readonly T[] | undefined,
|
|
1557
|
+
right: readonly T[],
|
|
1558
|
+
): boolean {
|
|
1559
|
+
return JSON.stringify(left ?? []) === JSON.stringify(right);
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
/** Captures only settings that can affect post-restart reconciliation/reporting. */
|
|
1563
|
+
function snapshotProfile(profile: FusionProfile): FusionProfileSnapshot {
|
|
1564
|
+
return {
|
|
1565
|
+
panel: profile.panel.map((member) => ({ ...member })),
|
|
1566
|
+
judge: { ...profile.judge },
|
|
1567
|
+
minimumSuccessfulPanelists: resolveMinimumSuccessfulPanelists(
|
|
1568
|
+
profile.minimumSuccessfulPanelists,
|
|
1569
|
+
profile.panel.length,
|
|
1570
|
+
),
|
|
1571
|
+
...(profile.context !== undefined ? { context: profile.context } : {}),
|
|
1572
|
+
...(profile.stopWhenPanelAgrees !== undefined
|
|
1573
|
+
? { stopWhenPanelAgrees: profile.stopWhenPanelAgrees }
|
|
1574
|
+
: {}),
|
|
1575
|
+
...(profile.blindPanelLabels !== undefined
|
|
1576
|
+
? { blindPanelLabels: profile.blindPanelLabels }
|
|
1577
|
+
: {}),
|
|
1578
|
+
...(profile.judgeToolBudget !== undefined
|
|
1579
|
+
? {
|
|
1580
|
+
judgeToolBudget: {
|
|
1581
|
+
...profile.judgeToolBudget,
|
|
1582
|
+
...(Array.isArray(profile.judgeToolBudget.block)
|
|
1583
|
+
? { block: [...profile.judgeToolBudget.block] }
|
|
1584
|
+
: {}),
|
|
1585
|
+
},
|
|
1586
|
+
}
|
|
1587
|
+
: {}),
|
|
1588
|
+
...(profile.synthesis !== undefined ? { synthesis: profile.synthesis } : {}),
|
|
1589
|
+
};
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
function profileFromSnapshot(snapshot: FusionProfileSnapshot): FusionProfile {
|
|
1593
|
+
return {
|
|
1594
|
+
panel: snapshot.panel.map((member) => ({ ...member })),
|
|
1595
|
+
judge: { ...snapshot.judge },
|
|
1596
|
+
minimumSuccessfulPanelists: snapshot.minimumSuccessfulPanelists,
|
|
1597
|
+
...(snapshot.context !== undefined ? { context: snapshot.context } : {}),
|
|
1598
|
+
...(snapshot.stopWhenPanelAgrees !== undefined
|
|
1599
|
+
? { stopWhenPanelAgrees: snapshot.stopWhenPanelAgrees }
|
|
1600
|
+
: {}),
|
|
1601
|
+
...(snapshot.blindPanelLabels !== undefined
|
|
1602
|
+
? { blindPanelLabels: snapshot.blindPanelLabels }
|
|
1603
|
+
: {}),
|
|
1604
|
+
...(snapshot.judgeToolBudget !== undefined
|
|
1605
|
+
? {
|
|
1606
|
+
judgeToolBudget: {
|
|
1607
|
+
...snapshot.judgeToolBudget,
|
|
1608
|
+
...(Array.isArray(snapshot.judgeToolBudget.block)
|
|
1609
|
+
? { block: [...snapshot.judgeToolBudget.block] }
|
|
1610
|
+
: {}),
|
|
1611
|
+
},
|
|
1612
|
+
}
|
|
1613
|
+
: {}),
|
|
1614
|
+
...(snapshot.synthesis !== undefined ? { synthesis: snapshot.synthesis } : {}),
|
|
1615
|
+
};
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
function completionQuality(
|
|
1619
|
+
profile: FusionProfile,
|
|
1620
|
+
outputs: readonly PanelOutput[],
|
|
1621
|
+
failures: readonly FailedPanelSummary[],
|
|
1622
|
+
): "complete" | "partial" {
|
|
1623
|
+
return outputs.length === profile.panel.length ||
|
|
1624
|
+
(profile.stopWhenPanelAgrees === true &&
|
|
1625
|
+
failures.length > 0 &&
|
|
1626
|
+
failures.every((failure) => failure.reason === "stopped-after-agreement"))
|
|
1627
|
+
? "complete"
|
|
1628
|
+
: "partial";
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
function validateStartArgs(args: ParsedFusionArgs): string | undefined {
|
|
1632
|
+
if (typeof args.prompt !== "string" || !args.prompt.trim()) {
|
|
1633
|
+
return "Fusion prompt must not be blank.";
|
|
1634
|
+
}
|
|
1635
|
+
if (args.profile !== undefined && !args.profile.trim()) {
|
|
1636
|
+
return "Fusion profile must not be blank.";
|
|
1637
|
+
}
|
|
1638
|
+
if (
|
|
1639
|
+
args.panel !== undefined &&
|
|
1640
|
+
(args.panel.length === 0 || args.panel.some((entry) => !entry.trim()))
|
|
1641
|
+
) {
|
|
1642
|
+
return "Fusion panel must contain at least one non-blank entry.";
|
|
1643
|
+
}
|
|
1644
|
+
return undefined;
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
function hasUnresolvedSpawnIntent(run: FusionRun): boolean {
|
|
1648
|
+
if (!run.spawnIntent) return false;
|
|
1649
|
+
return run.spawnIntent.stage === "panel"
|
|
1650
|
+
? !run.panelRunId && !run.chainRunId
|
|
1651
|
+
: !run.judgeRunId;
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
/**
|
|
1655
|
+
* A restored nonterminal phase without the remote ID cannot be reconciled.
|
|
1656
|
+
* Spawn intents are handled first so their more specific no-replay failure is
|
|
1657
|
+
* preserved; all other incomplete records are terminalized rather than left
|
|
1658
|
+
* active forever.
|
|
1659
|
+
*/
|
|
1660
|
+
function validateRestoredRunLifecycle(run: FusionRun): string | undefined {
|
|
1661
|
+
if (run.phase === "judge" && !run.judgeRunId) {
|
|
1662
|
+
return "Fusion recovery stopped: judge phase has no persisted judge run ID.";
|
|
1663
|
+
}
|
|
1664
|
+
if (run.phase === "chain" && !run.chainRunId) {
|
|
1665
|
+
return "Fusion recovery stopped: chain phase has no persisted chain run ID.";
|
|
1666
|
+
}
|
|
1667
|
+
if (run.phase === "panel" && !run.panelRunId && !run.chainRunId) {
|
|
1668
|
+
return "Fusion recovery stopped: panel phase has no persisted panel run ID.";
|
|
1669
|
+
}
|
|
1670
|
+
return undefined;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1202
1673
|
export function extractSubagentRunId(payload: unknown): string | undefined {
|
|
1203
1674
|
if (!isRecord(payload)) return undefined;
|
|
1204
1675
|
const direct = firstNonBlankString(
|
|
@@ -1303,14 +1774,20 @@ function formatFusionStatusReport(input: {
|
|
|
1303
1774
|
);
|
|
1304
1775
|
lines.push(`Profile: ${input.active.profileName}`);
|
|
1305
1776
|
lines.push(`Phase: ${input.details?.phaseLabel ?? input.active.phase}`);
|
|
1777
|
+
appendEffectiveTimeouts(lines, input.active);
|
|
1778
|
+
for (const deadline of input.active.panelDeadlines ?? []) {
|
|
1779
|
+
lines.push(`Panel-${deadline.index + 1} deadline: ${deadline.status}${deadline.deliveryError ? ` (${deadline.deliveryError})` : ""}`);
|
|
1780
|
+
}
|
|
1306
1781
|
if (input.active.chainRunId)
|
|
1307
1782
|
lines.push(`Chain run: ${input.active.chainRunId}`);
|
|
1308
1783
|
else if (input.active.panelRunId)
|
|
1309
1784
|
lines.push(`Panel run: ${input.active.panelRunId}`);
|
|
1310
1785
|
if (input.active.judgeRunId) {
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1786
|
+
const synthesisLabel =
|
|
1787
|
+
input.details?.fallbackJudge?.label ??
|
|
1788
|
+
input.details?.judge?.label ??
|
|
1789
|
+
(input.active.chainRunId ? "Fallback judge" : "Judge");
|
|
1790
|
+
lines.push(`${synthesisLabel} run: ${input.active.judgeRunId}`);
|
|
1314
1791
|
}
|
|
1315
1792
|
lines.push(
|
|
1316
1793
|
`Progress: ${input.progress ? formatProgressCounts(input.progress) : "unknown"}`,
|
|
@@ -1322,6 +1799,7 @@ function formatFusionStatusReport(input: {
|
|
|
1322
1799
|
lines.push(`Prompt: ${firstLine(input.last.prompt)}`);
|
|
1323
1800
|
lines.push(`Profile: ${input.last.profileName}`);
|
|
1324
1801
|
lines.push(`Phase: ${input.last.phase}`);
|
|
1802
|
+
appendEffectiveTimeouts(lines, input.last);
|
|
1325
1803
|
if (input.last.chainRunId)
|
|
1326
1804
|
lines.push(`Chain run: ${input.last.chainRunId}`);
|
|
1327
1805
|
else if (input.last.panelRunId)
|
|
@@ -1343,6 +1821,23 @@ function formatFusionStatusReport(input: {
|
|
|
1343
1821
|
return lines.join("\n");
|
|
1344
1822
|
}
|
|
1345
1823
|
|
|
1824
|
+
function appendEffectiveTimeouts(
|
|
1825
|
+
lines: string[],
|
|
1826
|
+
run: Pick<FusionRun, "effectiveTimeouts">,
|
|
1827
|
+
): void {
|
|
1828
|
+
const timeouts = run.effectiveTimeouts;
|
|
1829
|
+
if (!timeouts) return;
|
|
1830
|
+
lines.push(
|
|
1831
|
+
`Deadlines: panelist ${timeouts.panelistTimeoutMs}ms, panel ${timeouts.panelTimeoutMs}ms (+${timeouts.panelGraceMs}ms grace), judge ${timeouts.judgeTimeoutMs}ms`,
|
|
1832
|
+
);
|
|
1833
|
+
if (timeouts.panelistSoftTimeoutMs !== undefined) {
|
|
1834
|
+
lines.push(`Soft deadline: ${timeouts.panelistSoftTimeoutMs}ms per child; decision wait ${PANEL_DECISION_WAIT_MS}ms; hard deadlines are unchanged.`);
|
|
1835
|
+
}
|
|
1836
|
+
if (timeouts.usesLegacyTimeout) {
|
|
1837
|
+
lines.push("Warning: legacy timeoutMs supplied one or more effective deadlines.");
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1346
1841
|
function appendStatusDetails(
|
|
1347
1842
|
lines: string[],
|
|
1348
1843
|
details: FusionStatusDetails | undefined,
|
|
@@ -1397,7 +1892,7 @@ function buildFusionStatusDetails(
|
|
|
1397
1892
|
): FusionStatusDetails {
|
|
1398
1893
|
const details: FusionStatusDetails = {
|
|
1399
1894
|
prompt: run.prompt,
|
|
1400
|
-
phaseLabel: deriveFusionStatusPhase(run, payload),
|
|
1895
|
+
phaseLabel: deriveFusionStatusPhase(run, payload, profile),
|
|
1401
1896
|
};
|
|
1402
1897
|
if (!profile) return details;
|
|
1403
1898
|
|
|
@@ -1410,7 +1905,7 @@ function buildFusionStatusDetails(
|
|
|
1410
1905
|
storedPanelFailures(run),
|
|
1411
1906
|
);
|
|
1412
1907
|
details.fallbackJudge = {
|
|
1413
|
-
label: run.chainRunId
|
|
1908
|
+
label: synthesisStatusLabel(profile, Boolean(run.chainRunId)),
|
|
1414
1909
|
...(judgeModel ? { model: judgeModel } : {}),
|
|
1415
1910
|
status: describeStandaloneRunStatus(payload),
|
|
1416
1911
|
};
|
|
@@ -1439,7 +1934,7 @@ function buildFusionStatusDetails(
|
|
|
1439
1934
|
const judgeActivity = describeStepActivity(steps[profile.panel.length]);
|
|
1440
1935
|
const judgeMetrics = describeStepMetrics(steps[profile.panel.length]);
|
|
1441
1936
|
details.judge = {
|
|
1442
|
-
label:
|
|
1937
|
+
label: synthesisStatusLabel(profile),
|
|
1443
1938
|
...(judgeModel ? { model: judgeModel } : {}),
|
|
1444
1939
|
status: describeChainJudgeStatus(steps[profile.panel.length], panelists),
|
|
1445
1940
|
...([judgeActivity, judgeMetrics].filter(Boolean).length > 0
|
|
@@ -1452,9 +1947,10 @@ function buildFusionStatusDetails(
|
|
|
1452
1947
|
function deriveFusionStatusPhase(
|
|
1453
1948
|
run: Pick<FusionRun, "phase" | "chainRunId">,
|
|
1454
1949
|
payload: unknown,
|
|
1950
|
+
profile?: Pick<FusionProfile, "panel" | "synthesis">,
|
|
1455
1951
|
): string {
|
|
1456
1952
|
if (run.phase === "judge") {
|
|
1457
|
-
return run.chainRunId
|
|
1953
|
+
return synthesisStatusLabel(profile, Boolean(run.chainRunId)).toLowerCase();
|
|
1458
1954
|
}
|
|
1459
1955
|
if (run.phase !== "chain") return run.phase;
|
|
1460
1956
|
|
|
@@ -1462,7 +1958,7 @@ function deriveFusionStatusPhase(
|
|
|
1462
1958
|
if (steps.length === 0) return "chain";
|
|
1463
1959
|
const judgeStatus = normalizeStatusLabel(steps.at(-1));
|
|
1464
1960
|
if (judgeStatus === "running" || judgeStatus === "completed") {
|
|
1465
|
-
return
|
|
1961
|
+
return synthesisStatusLabel(profile).toLowerCase();
|
|
1466
1962
|
}
|
|
1467
1963
|
const panelSteps = steps.slice(0, -1);
|
|
1468
1964
|
if (panelSteps.some((step) => normalizeStatusLabel(step) === "running")) {
|
|
@@ -1473,11 +1969,23 @@ function deriveFusionStatusPhase(
|
|
|
1473
1969
|
const status = normalizeStatusLabel(step);
|
|
1474
1970
|
return status === "completed" || status === "failed";
|
|
1475
1971
|
});
|
|
1476
|
-
return allPanelsFinished
|
|
1972
|
+
return allPanelsFinished
|
|
1973
|
+
? synthesisStatusLabel(profile).toLowerCase()
|
|
1974
|
+
: "panel";
|
|
1477
1975
|
}
|
|
1478
1976
|
return "panel";
|
|
1479
1977
|
}
|
|
1480
1978
|
|
|
1979
|
+
function synthesisStatusLabel(
|
|
1980
|
+
profile: Pick<FusionProfile, "panel" | "synthesis"> | undefined,
|
|
1981
|
+
fallback = false,
|
|
1982
|
+
): "Judge" | "Fallback judge" | "Composer" | "Fallback composer" {
|
|
1983
|
+
const base = profile && resolveSynthesisMode(profile) === "merge"
|
|
1984
|
+
? "Composer"
|
|
1985
|
+
: "Judge";
|
|
1986
|
+
return fallback ? `Fallback ${base.toLowerCase()}` as "Fallback judge" | "Fallback composer" : base;
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1481
1989
|
function buildCompletedPanelStatusLines(
|
|
1482
1990
|
panel: FusionProfile["panel"],
|
|
1483
1991
|
outputs: readonly PanelOutput[],
|
|
@@ -1635,6 +2143,10 @@ function findResultsArray(payload: unknown): readonly unknown[] | undefined {
|
|
|
1635
2143
|
return undefined;
|
|
1636
2144
|
}
|
|
1637
2145
|
|
|
2146
|
+
function hasExplicitResultsArray(payload: unknown): boolean {
|
|
2147
|
+
return findResultsArray(payload) !== undefined;
|
|
2148
|
+
}
|
|
2149
|
+
|
|
1638
2150
|
function hasResultsArray(payload: unknown): boolean {
|
|
1639
2151
|
return (findResultsArray(payload)?.length ?? 0) > 0;
|
|
1640
2152
|
}
|
|
@@ -1745,6 +2257,31 @@ function extractWorkflowStoppedPanelIndices(payload: unknown): number[] {
|
|
|
1745
2257
|
return [];
|
|
1746
2258
|
}
|
|
1747
2259
|
|
|
2260
|
+
function withWorkflowDeadlineFailures(
|
|
2261
|
+
failures: readonly FailedPanelSummary[],
|
|
2262
|
+
deadlineError: string | undefined,
|
|
2263
|
+
): FailedPanelSummary[] {
|
|
2264
|
+
if (!deadlineError) return [...failures];
|
|
2265
|
+
return failures.map((failure) => ({
|
|
2266
|
+
...failure,
|
|
2267
|
+
summary: failure.summary.includes(deadlineError)
|
|
2268
|
+
? failure.summary
|
|
2269
|
+
: `${deadlineError} ${failure.summary}`,
|
|
2270
|
+
reason: failure.reason ?? "timeout",
|
|
2271
|
+
}));
|
|
2272
|
+
}
|
|
2273
|
+
|
|
2274
|
+
function isWorkflowDeadline(payload: unknown): boolean {
|
|
2275
|
+
if (!isRecord(payload)) return false;
|
|
2276
|
+
if (payload.timedOut === true) return true;
|
|
2277
|
+
const error = firstNonBlankString(payload.error, payload.errorMessage);
|
|
2278
|
+
if (error && /(?:workflow.*(?:timed out|timeout)|(?:timed out|timeout).*workflow)/i.test(error)) {
|
|
2279
|
+
return true;
|
|
2280
|
+
}
|
|
2281
|
+
if (isRecord(payload.details) && isWorkflowDeadline(payload.details)) return true;
|
|
2282
|
+
return isRecord(payload.data) ? isWorkflowDeadline(payload.data) : false;
|
|
2283
|
+
}
|
|
2284
|
+
|
|
1748
2285
|
function extractSubagentFailure(payload: unknown): string | undefined {
|
|
1749
2286
|
if (!isRecord(payload)) return undefined;
|
|
1750
2287
|
const direct = firstNonBlankString(payload.error, payload.errorMessage);
|
|
@@ -1823,9 +2360,15 @@ function shouldStopWhenPanelAgrees(
|
|
|
1823
2360
|
profile: FusionProfile,
|
|
1824
2361
|
outputs: readonly PanelOutput[],
|
|
1825
2362
|
failures: readonly FailedPanelSummary[],
|
|
2363
|
+
persistedPolicy?: FusionRun["minimumSuccessfulPanelists"],
|
|
1826
2364
|
): boolean {
|
|
2365
|
+
const required = resolveMinimumSuccessfulPanelists(
|
|
2366
|
+
persistedPolicy ?? profile.minimumSuccessfulPanelists,
|
|
2367
|
+
profile.panel.length,
|
|
2368
|
+
);
|
|
1827
2369
|
return (
|
|
1828
2370
|
profile.stopWhenPanelAgrees === true &&
|
|
2371
|
+
outputs.length >= required &&
|
|
1829
2372
|
hasStrongPanelAgreement(
|
|
1830
2373
|
outputs,
|
|
1831
2374
|
outputs.length + failures.length,
|