@signalridge/pi-workflows 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +31 -1
- package/package.json +2 -2
- package/skills/workflow-authoring/SKILL.md +1 -1
- package/skills/workflow-authoring/references/capabilities.md +1 -1
- package/skills/workflow-authoring/references/capability-details.md +1 -1
- package/skills/workflow-patterns/SKILL.md +1 -1
- package/skills/workflow-review/SKILL.md +1 -1
- package/src/engine.ts +1088 -395
- package/src/index.ts +286 -53
- package/src/inspection.ts +544 -0
- package/src/journal.ts +33 -11
- package/src/orchestrator.ts +22 -4
- package/src/runtime.ts +387 -148
- package/src/status-actions.ts +143 -0
- package/src/ui-bus.ts +243 -0
- package/src/ui-command-center.ts +1032 -0
- package/src/ui-inspection.ts +206 -0
- package/src/ui-presentation.ts +196 -0
- package/src/ui.ts +43 -265
package/src/engine.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
21
21
|
import type { ManagedSpawnResponse } from "@signalridge/pi-subagents-protocol";
|
|
22
22
|
import { isManagedAgentTier } from "@signalridge/pi-subagents-protocol";
|
|
23
23
|
import { describeWorkflowError, WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
|
|
24
|
+
import { type WorkflowInspection, type WorkflowInspectionHeader, WorkflowPresentation } from "./inspection.js";
|
|
24
25
|
import {
|
|
25
26
|
applyRecoveryEvent,
|
|
26
27
|
buildResumeJournal,
|
|
@@ -174,6 +175,25 @@ interface AgentTerminalDelivery {
|
|
|
174
175
|
history?: unknown[];
|
|
175
176
|
}
|
|
176
177
|
|
|
178
|
+
interface ManagedCallFacts {
|
|
179
|
+
executionGeneration: number;
|
|
180
|
+
attemptId: string;
|
|
181
|
+
agentId: string;
|
|
182
|
+
tokenCount?: number;
|
|
183
|
+
outputFile?: string;
|
|
184
|
+
compactionCount: number;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
interface OwnedCleanup {
|
|
188
|
+
agentId: string;
|
|
189
|
+
spawnKey?: string;
|
|
190
|
+
owner: WorkflowOwner;
|
|
191
|
+
epoch: number;
|
|
192
|
+
confirmed: Promise<void>;
|
|
193
|
+
resolve: () => void;
|
|
194
|
+
operation?: Promise<void>;
|
|
195
|
+
}
|
|
196
|
+
|
|
177
197
|
interface BufferedAgentTerminal extends AgentTerminalDelivery {
|
|
178
198
|
owner: WorkflowOwner;
|
|
179
199
|
executionGeneration: number;
|
|
@@ -181,6 +201,15 @@ interface BufferedAgentTerminal extends AgentTerminalDelivery {
|
|
|
181
201
|
|
|
182
202
|
export interface ScriptRunState {
|
|
183
203
|
run: ScriptRun;
|
|
204
|
+
/** Volatile, execution-fenced presentation; never journaled. */
|
|
205
|
+
presentation: WorkflowPresentation;
|
|
206
|
+
/** Exact-attempt metadata merged into the runtime's single schema-transformed result fact. */
|
|
207
|
+
managedCallFacts: Map<string, ManagedCallFacts>;
|
|
208
|
+
/** Ownership remains busy even after a cleanup deadline fences execution. */
|
|
209
|
+
pendingCleanup: Set<Promise<void>>;
|
|
210
|
+
activeExecutionGenerations: Set<number>;
|
|
211
|
+
stopOperation?: Promise<void>;
|
|
212
|
+
removeOperation?: Promise<void>;
|
|
184
213
|
/** Frozen run parameters — resume reuses the original values, ignoring new budget/toolset overrides. */
|
|
185
214
|
frozenOptions: Pick<
|
|
186
215
|
ScriptStartOptions,
|
|
@@ -202,8 +231,18 @@ export interface ScriptRunState {
|
|
|
202
231
|
agentWaiters: Map<string, AgentLifecycleWaiter>;
|
|
203
232
|
/** Owner-validated terminal events emitted before their spawn reply/waiter. */
|
|
204
233
|
bufferedTerminals: Map<string, BufferedAgentTerminal>;
|
|
205
|
-
/**
|
|
206
|
-
pendingSpawns: Map<
|
|
234
|
+
/** Local reservations and submitted allocations retained through terminal admission. */
|
|
235
|
+
pendingSpawns: Map<
|
|
236
|
+
string,
|
|
237
|
+
{
|
|
238
|
+
spawnKey: string;
|
|
239
|
+
owner: WorkflowOwner;
|
|
240
|
+
executionGeneration: number;
|
|
241
|
+
phase: "reserved" | "submitted";
|
|
242
|
+
startedAgentId?: string;
|
|
243
|
+
agentId?: string;
|
|
244
|
+
}
|
|
245
|
+
>;
|
|
207
246
|
/** Journal retry bookkeeping. */
|
|
208
247
|
pendingJournal: Set<string>;
|
|
209
248
|
/** Whether a lifecycle suspension is in effect (branch change). */
|
|
@@ -309,6 +348,29 @@ function sameWorkflowOwner(left: WorkflowOwner, right: WorkflowOwner): boolean {
|
|
|
309
348
|
);
|
|
310
349
|
}
|
|
311
350
|
|
|
351
|
+
/** Optional owned lifecycle usage is per-attempt; legacy `tokens` remains lifetime-scoped. */
|
|
352
|
+
function managedLifecycleUsage(raw: unknown): AgentUsage | undefined {
|
|
353
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
|
|
354
|
+
const { input, output, total } = raw as Record<string, unknown>;
|
|
355
|
+
if (
|
|
356
|
+
typeof input !== "number" ||
|
|
357
|
+
typeof output !== "number" ||
|
|
358
|
+
typeof total !== "number" ||
|
|
359
|
+
!Number.isSafeInteger(input) ||
|
|
360
|
+
!Number.isSafeInteger(output) ||
|
|
361
|
+
!Number.isSafeInteger(total) ||
|
|
362
|
+
input < 0 ||
|
|
363
|
+
output < 0 ||
|
|
364
|
+
total < 0 ||
|
|
365
|
+
!Number.isSafeInteger(input + output) ||
|
|
366
|
+
total < input + output
|
|
367
|
+
)
|
|
368
|
+
return undefined;
|
|
369
|
+
// The managed total includes cache writes, but excludes cache reads. The
|
|
370
|
+
// additive lifecycle field has no cost/cache-read detail; do not invent it.
|
|
371
|
+
return { input, output, total, cost: 0, cacheRead: 0, cacheWrite: total - input - output };
|
|
372
|
+
}
|
|
373
|
+
|
|
312
374
|
export class WorkflowEngine {
|
|
313
375
|
private readonly runs = new Map<string, ScriptRunState>();
|
|
314
376
|
/** Fired when a run settles terminal; wired by the UI surface for widget refresh. */
|
|
@@ -317,6 +379,10 @@ export class WorkflowEngine {
|
|
|
317
379
|
onBackgroundExecution: ((runId: string) => void) | undefined;
|
|
318
380
|
/** Optional in-process observer for phase/task/quality runtime events. */
|
|
319
381
|
onRuntimeEvent: ((runId: string, event: WorkflowRuntimeEvent) => void) | undefined;
|
|
382
|
+
/** Additive notification after accepted presentation mutations; consumers re-read getInspection(). */
|
|
383
|
+
onInspectionChanged: ((runId: string) => void) | undefined;
|
|
384
|
+
/** Fresh saved-workflow resolver for control and automatic resumes. */
|
|
385
|
+
loadSavedWorkflow: ((name: string) => string | undefined) | undefined;
|
|
320
386
|
/**
|
|
321
387
|
* The user's current strength → Agent tier table, read fresh per execution.
|
|
322
388
|
*
|
|
@@ -352,6 +418,10 @@ export class WorkflowEngine {
|
|
|
352
418
|
private journalMutationKey: string | undefined;
|
|
353
419
|
private journalMutationSequence = 0;
|
|
354
420
|
private readonly quarantinedRunIds = new Set<string>();
|
|
421
|
+
private readonly cleanupObligations = new WeakMap<ScriptRunState, Map<string, OwnedCleanup>>();
|
|
422
|
+
/** Retired states keep their exact leases; replacement states never inherit them. */
|
|
423
|
+
private readonly retainedCleanupStates = new Set<ScriptRunState>();
|
|
424
|
+
private readonly terminalNotified = new WeakSet<ScriptRunState>();
|
|
355
425
|
|
|
356
426
|
constructor(
|
|
357
427
|
readonly events: WorkflowEventBus,
|
|
@@ -367,6 +437,15 @@ export class WorkflowEngine {
|
|
|
367
437
|
this.unsubscribeLifecycle = this.attachLifecycle();
|
|
368
438
|
}
|
|
369
439
|
|
|
440
|
+
private notifyInspection(state: ScriptRunState): void {
|
|
441
|
+
state.presentation.changed();
|
|
442
|
+
try {
|
|
443
|
+
this.onInspectionChanged?.(state.run.runId);
|
|
444
|
+
} catch {
|
|
445
|
+
// Presentation observers must never change execution or durable control.
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
370
449
|
private async awaitProtocol(): Promise<ManagedProtocolCheck> {
|
|
371
450
|
if (this.disposed) throw new WorkflowEngineDisposedError();
|
|
372
451
|
let rejectDisposed: ((reason?: unknown) => void) | undefined;
|
|
@@ -390,6 +469,40 @@ export class WorkflowEngine {
|
|
|
390
469
|
return this.disposed;
|
|
391
470
|
}
|
|
392
471
|
|
|
472
|
+
/** Pure exact-state cleanup uncertainty query for owner control surfaces. */
|
|
473
|
+
hasPendingCleanup(runId: string): boolean {
|
|
474
|
+
return (this.runs.get(runId)?.pendingCleanup.size ?? 0) > 0;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
private isCurrentState(state: ScriptRunState, generation: number, epoch: number): boolean {
|
|
478
|
+
return (
|
|
479
|
+
this.runs.get(state.run.runId) === state &&
|
|
480
|
+
state.executionGeneration === generation &&
|
|
481
|
+
this.branchEpoch === epoch &&
|
|
482
|
+
!state.lifecycleSuspended &&
|
|
483
|
+
!this.lifecyclePaused &&
|
|
484
|
+
!this.disposed
|
|
485
|
+
);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private notifyTerminal(state: ScriptRunState): void {
|
|
489
|
+
const generation = state.executionGeneration;
|
|
490
|
+
const epoch = this.branchEpoch;
|
|
491
|
+
if (
|
|
492
|
+
!this.isCurrentState(state, generation, epoch) ||
|
|
493
|
+
state.pendingCleanup.size > 0 ||
|
|
494
|
+
this.terminalNotified.has(state)
|
|
495
|
+
)
|
|
496
|
+
return;
|
|
497
|
+
this.terminalNotified.add(state);
|
|
498
|
+
this.settleWaiters(state.run.runId, state.run);
|
|
499
|
+
try {
|
|
500
|
+
this.onRunSettled?.();
|
|
501
|
+
} catch {
|
|
502
|
+
/* terminal observers are presentation-only */
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
393
506
|
/** A synchronous snapshot for cancellable navigation, not an admission lock. */
|
|
394
507
|
hasUnsettledWork(): boolean {
|
|
395
508
|
return (
|
|
@@ -397,6 +510,7 @@ export class WorkflowEngine {
|
|
|
397
510
|
this.branchQuiescePromise !== undefined ||
|
|
398
511
|
this.journalRetries.size > 0 ||
|
|
399
512
|
this.providerResumeTimers.size > 0 ||
|
|
513
|
+
this.retainedCleanupStates.size > 0 ||
|
|
400
514
|
[...this.runs.values()].some(
|
|
401
515
|
(state) =>
|
|
402
516
|
state.run.status === "running" ||
|
|
@@ -404,7 +518,11 @@ export class WorkflowEngine {
|
|
|
404
518
|
state.run.status === "stopping" ||
|
|
405
519
|
state.pendingSpawns.size > 0 ||
|
|
406
520
|
state.agentWaiters.size > 0 ||
|
|
407
|
-
state.pendingJournal.size > 0
|
|
521
|
+
state.pendingJournal.size > 0 ||
|
|
522
|
+
state.pendingCleanup.size > 0 ||
|
|
523
|
+
state.activeExecutionGenerations.size > 0 ||
|
|
524
|
+
state.stopOperation !== undefined ||
|
|
525
|
+
state.removeOperation !== undefined,
|
|
408
526
|
)
|
|
409
527
|
);
|
|
410
528
|
}
|
|
@@ -418,6 +536,8 @@ export class WorkflowEngine {
|
|
|
418
536
|
this.clearJournalRetries();
|
|
419
537
|
for (const state of this.runs.values()) {
|
|
420
538
|
state.lifecycleSuspended = true;
|
|
539
|
+
state.presentation.reset(false);
|
|
540
|
+
this.notifyInspection(state);
|
|
421
541
|
state.controller.abort(new DOMException("Workflow branch replaced", "AbortError"));
|
|
422
542
|
}
|
|
423
543
|
}
|
|
@@ -444,7 +564,8 @@ export class WorkflowEngine {
|
|
|
444
564
|
eventName !== "subagents:created" &&
|
|
445
565
|
eventName !== "subagents:started" &&
|
|
446
566
|
eventName !== "subagents:completed" &&
|
|
447
|
-
eventName !== "subagents:failed"
|
|
567
|
+
eventName !== "subagents:failed" &&
|
|
568
|
+
eventName !== "subagents:compacted"
|
|
448
569
|
)
|
|
449
570
|
return;
|
|
450
571
|
if (!raw || typeof raw !== "object") return;
|
|
@@ -497,13 +618,55 @@ export class WorkflowEngine {
|
|
|
497
618
|
if (currentAttemptId !== undefined && currentAttemptId !== `${owner.runId}/${callNodeId}/${owner.attemptId}`)
|
|
498
619
|
return;
|
|
499
620
|
|
|
621
|
+
const acceptedGeneration = state.executionGeneration;
|
|
622
|
+
const acceptedEpoch = this.branchEpoch;
|
|
623
|
+
const current = () =>
|
|
624
|
+
this.isCurrentState(state, acceptedGeneration, acceptedEpoch) && !state.controller.signal.aborted;
|
|
625
|
+
const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
|
|
626
|
+
const hasManagedUsage = terminalEvent && "managedTokens" in data;
|
|
627
|
+
const managedUsage = hasManagedUsage ? managedLifecycleUsage(data.managedTokens) : undefined;
|
|
628
|
+
// A malformed present field cannot fall back to cumulative lifetime usage.
|
|
629
|
+
// Reject before identity, presentation, buffering, or waiter mutations.
|
|
630
|
+
if (hasManagedUsage && !managedUsage) return;
|
|
631
|
+
|
|
632
|
+
if (eventName === "subagents:compacted") {
|
|
633
|
+
this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
|
|
634
|
+
if (!current()) return;
|
|
635
|
+
if (
|
|
636
|
+
typeof data.compactionCount === "number" &&
|
|
637
|
+
Number.isSafeInteger(data.compactionCount) &&
|
|
638
|
+
data.compactionCount >= 0
|
|
639
|
+
) {
|
|
640
|
+
const facts = state.managedCallFacts.get(callNodeId);
|
|
641
|
+
if (facts) facts.compactionCount = Math.max(facts.compactionCount, data.compactionCount);
|
|
642
|
+
if (state.run.attemptIds[callNodeId])
|
|
643
|
+
state.run.compactions[callNodeId] = facts?.compactionCount ?? data.compactionCount;
|
|
644
|
+
state.presentation.managed(callNodeId, { compactionCount: facts?.compactionCount ?? data.compactionCount });
|
|
645
|
+
this.notifyInspection(state);
|
|
646
|
+
}
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
500
649
|
if (updateCallTierIdentity(state, callNodeId, data.tier)) {
|
|
501
650
|
this.persistCallTierIdentity(state, callNodeId);
|
|
502
651
|
}
|
|
652
|
+
if (!current()) return;
|
|
653
|
+
this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
|
|
654
|
+
if (!current()) return;
|
|
503
655
|
// A start is the moment the host admitted this agent from its queue. It is
|
|
504
656
|
// the only signal that separates waiting for a slot from doing the work, so
|
|
505
657
|
// the call's wall clock is restarted here rather than at dispatch.
|
|
506
|
-
if (eventName === "subagents:started"
|
|
658
|
+
if (eventName === "subagents:started") {
|
|
659
|
+
if (pendingMatches && pending) pending.startedAgentId = agentId;
|
|
660
|
+
if (waiterMatches && waiter) notifyAgentRunning(waiter);
|
|
661
|
+
if (!current()) return;
|
|
662
|
+
}
|
|
663
|
+
state.presentation.managed(callNodeId, {
|
|
664
|
+
agentId,
|
|
665
|
+
status: eventName === "subagents:started" ? "running" : undefined,
|
|
666
|
+
...(isManagedAgentTier(data.tier) ? { tier: data.tier } : {}),
|
|
667
|
+
});
|
|
668
|
+
this.notifyInspection(state);
|
|
669
|
+
if (!current()) return;
|
|
507
670
|
// Created/started lifecycle facts carry managed identity before the RPC
|
|
508
671
|
// reply is necessarily available. They are observation-only here; terminal
|
|
509
672
|
// ownership is handled below once a completion/failure fact arrives.
|
|
@@ -516,13 +679,27 @@ export class WorkflowEngine {
|
|
|
516
679
|
? "stopped"
|
|
517
680
|
: "failed";
|
|
518
681
|
const tokenCount =
|
|
519
|
-
|
|
520
|
-
?
|
|
521
|
-
: data.
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
682
|
+
managedUsage !== undefined
|
|
683
|
+
? managedUsage.total
|
|
684
|
+
: typeof data.tokenCount === "number"
|
|
685
|
+
? data.tokenCount
|
|
686
|
+
: data.tokens &&
|
|
687
|
+
typeof data.tokens === "object" &&
|
|
688
|
+
typeof (data.tokens as Record<string, unknown>).total === "number"
|
|
689
|
+
? ((data.tokens as Record<string, unknown>).total as number)
|
|
690
|
+
: undefined;
|
|
691
|
+
state.presentation.managed(callNodeId, {
|
|
692
|
+
agentId,
|
|
693
|
+
status: lifecycleStatus,
|
|
694
|
+
tokenCount,
|
|
695
|
+
...(typeof data.outputFile === "string" ? { outputFile: data.outputFile } : {}),
|
|
696
|
+
compactionCount: Math.max(
|
|
697
|
+
typeof data.compactionCount === "number" ? data.compactionCount : 0,
|
|
698
|
+
state.managedCallFacts.get(callNodeId)?.compactionCount ?? 0,
|
|
699
|
+
),
|
|
700
|
+
});
|
|
701
|
+
this.notifyInspection(state);
|
|
702
|
+
if (!current()) return;
|
|
526
703
|
const delivery: AgentTerminalDelivery = {
|
|
527
704
|
response: {
|
|
528
705
|
id: agentId,
|
|
@@ -536,9 +713,11 @@ export class WorkflowEngine {
|
|
|
536
713
|
completedAt: typeof data.completedAt === "number" ? data.completedAt : Date.now(),
|
|
537
714
|
},
|
|
538
715
|
},
|
|
539
|
-
...(
|
|
540
|
-
? {}
|
|
541
|
-
:
|
|
716
|
+
...(managedUsage !== undefined
|
|
717
|
+
? { usage: managedUsage }
|
|
718
|
+
: tokenCount === undefined
|
|
719
|
+
? {}
|
|
720
|
+
: { usage: { input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 } }),
|
|
542
721
|
...(Array.isArray(data.history) ? { history: [...data.history] } : {}),
|
|
543
722
|
};
|
|
544
723
|
|
|
@@ -555,6 +734,60 @@ export class WorkflowEngine {
|
|
|
555
734
|
state.bufferedTerminals.set(spawnKey, { ...delivery, owner, executionGeneration: state.executionGeneration });
|
|
556
735
|
}
|
|
557
736
|
|
|
737
|
+
private recordManagedAdmission(
|
|
738
|
+
state: ScriptRunState,
|
|
739
|
+
nodeId: string,
|
|
740
|
+
agentId: string,
|
|
741
|
+
executionGeneration: number,
|
|
742
|
+
): void {
|
|
743
|
+
const attemptId = state.run.attemptIds[nodeId];
|
|
744
|
+
if (!attemptId) return; // Nested facts remain volatile, owned by the parent boundary.
|
|
745
|
+
const existing = state.managedCallFacts.get(nodeId);
|
|
746
|
+
if (
|
|
747
|
+
existing?.executionGeneration === executionGeneration &&
|
|
748
|
+
existing.attemptId === attemptId &&
|
|
749
|
+
existing.agentId === agentId
|
|
750
|
+
)
|
|
751
|
+
return;
|
|
752
|
+
state.managedCallFacts.set(nodeId, { executionGeneration, attemptId, agentId, compactionCount: 0 });
|
|
753
|
+
state.run.agentIds[nodeId] = agentId;
|
|
754
|
+
state.run.compactions[nodeId] = 0;
|
|
755
|
+
const event: JournalEvent = {
|
|
756
|
+
kind: "call_transition",
|
|
757
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
758
|
+
runId: state.run.runId,
|
|
759
|
+
nodeId,
|
|
760
|
+
status: "running",
|
|
761
|
+
agentId,
|
|
762
|
+
...(state.callTiers.get(nodeId)?.tier === undefined ? {} : { tier: state.callTiers.get(nodeId)?.tier }),
|
|
763
|
+
attemptId,
|
|
764
|
+
owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
|
|
765
|
+
timestamp: Date.now(),
|
|
766
|
+
};
|
|
767
|
+
this.persist(state.run.runId, `call-identity:${nodeId}`, () => this.write(event));
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
private recordManagedTerminal(
|
|
771
|
+
state: ScriptRunState,
|
|
772
|
+
nodeId: string,
|
|
773
|
+
agentId: string,
|
|
774
|
+
terminal: NonNullable<ManagedSpawnResponse["terminal"]>,
|
|
775
|
+
executionGeneration: number,
|
|
776
|
+
): void {
|
|
777
|
+
this.recordManagedAdmission(state, nodeId, agentId, executionGeneration);
|
|
778
|
+
const facts = state.managedCallFacts.get(nodeId);
|
|
779
|
+
if (
|
|
780
|
+
!facts ||
|
|
781
|
+
facts.executionGeneration !== executionGeneration ||
|
|
782
|
+
facts.attemptId !== state.run.attemptIds[nodeId] ||
|
|
783
|
+
facts.agentId !== agentId
|
|
784
|
+
)
|
|
785
|
+
return;
|
|
786
|
+
facts.tokenCount = terminal.tokenCount;
|
|
787
|
+
facts.outputFile = terminal.outputFile;
|
|
788
|
+
facts.compactionCount = Math.max(facts.compactionCount, terminal.compactionCount);
|
|
789
|
+
}
|
|
790
|
+
|
|
558
791
|
/** Persist a host-normalized tier learned before the managed reply arrives. */
|
|
559
792
|
private persistCallTierIdentity(state: ScriptRunState, nodeId: string): void {
|
|
560
793
|
const attemptId = state.run.attemptIds[nodeId];
|
|
@@ -562,19 +795,18 @@ export class WorkflowEngine {
|
|
|
562
795
|
// Nested workflow frames are in-memory only; only a top-level call has a
|
|
563
796
|
// durable attempt owner and can safely receive a journal transition.
|
|
564
797
|
if (!attemptId || identity?.tier === undefined) return;
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
);
|
|
798
|
+
const event: JournalEvent = {
|
|
799
|
+
kind: "call_transition",
|
|
800
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
801
|
+
runId: state.run.runId,
|
|
802
|
+
nodeId,
|
|
803
|
+
status: "running",
|
|
804
|
+
tier: identity.tier,
|
|
805
|
+
attemptId,
|
|
806
|
+
owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
|
|
807
|
+
timestamp: Date.now(),
|
|
808
|
+
};
|
|
809
|
+
this.persist(state.run.runId, `call-tier:${nodeId}`, () => this.write(event));
|
|
578
810
|
}
|
|
579
811
|
|
|
580
812
|
private resolveAgentWaiter(
|
|
@@ -586,9 +818,23 @@ export class WorkflowEngine {
|
|
|
586
818
|
if (waiter.settled) return;
|
|
587
819
|
waiter.settled = true;
|
|
588
820
|
state.agentWaiters.delete(agentId);
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
821
|
+
const epoch = this.branchEpoch;
|
|
822
|
+
try {
|
|
823
|
+
if (delivery.usage) waiter.onUsage?.(delivery.usage);
|
|
824
|
+
if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
|
|
825
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
826
|
+
return;
|
|
827
|
+
}
|
|
828
|
+
if (delivery.history) waiter.onHistory?.(delivery.history);
|
|
829
|
+
if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
|
|
830
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
waiter.resolve(delivery.response);
|
|
834
|
+
} catch (error) {
|
|
835
|
+
// A throwing observer cannot strand a removed waiter and its child lease.
|
|
836
|
+
waiter.reject(error);
|
|
837
|
+
}
|
|
592
838
|
}
|
|
593
839
|
|
|
594
840
|
dispose(): void {
|
|
@@ -597,12 +843,14 @@ export class WorkflowEngine {
|
|
|
597
843
|
this.disposeController.abort();
|
|
598
844
|
this.unsubscribeLifecycle();
|
|
599
845
|
this.quarantinedRunIds.clear();
|
|
600
|
-
|
|
846
|
+
const retiredStates = [...this.runs.values()];
|
|
847
|
+
for (const state of retiredStates) {
|
|
601
848
|
this.cleanupAgents(state.run.runId);
|
|
602
849
|
state.controller.abort(new DOMException("Workflow engine disposed", "AbortError"));
|
|
603
850
|
}
|
|
604
851
|
this.rejectWaiters(new WorkflowWaitAbortedError());
|
|
605
852
|
this.runs.clear();
|
|
853
|
+
for (const state of retiredStates) this.notifyInspection(state);
|
|
606
854
|
this.clearJournalRetries();
|
|
607
855
|
this.branchQuiescePromise = undefined;
|
|
608
856
|
}
|
|
@@ -621,68 +869,58 @@ export class WorkflowEngine {
|
|
|
621
869
|
|
|
622
870
|
/** Stop owned agents while Pi replaces the active session-tree branch. */
|
|
623
871
|
async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
|
|
624
|
-
if (this.disposed) return { settled: true, pending: [] };
|
|
625
872
|
if (this.branchQuiescePromise) return this.branchQuiescePromise;
|
|
626
873
|
this.admissionsClosed = true;
|
|
627
874
|
this.clearJournalRetries();
|
|
628
875
|
|
|
629
|
-
//
|
|
630
|
-
//
|
|
631
|
-
//
|
|
632
|
-
const
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
const
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
876
|
+
// Snapshot exact states/owners, not run ids or status. Terminal and retired
|
|
877
|
+
// states may still own cleanup, and observation must not donate those
|
|
878
|
+
// leases to a same-id replacement on a newly committed branch.
|
|
879
|
+
const states = [...new Set([...this.runs.values(), ...this.retainedCleanupStates])];
|
|
880
|
+
const targets = states.flatMap((state) => {
|
|
881
|
+
const owners = new Map<string, { agentId: string; owner: WorkflowOwner; generation: number }>();
|
|
882
|
+
for (const pending of state.pendingSpawns.values()) {
|
|
883
|
+
if (pending.phase !== "submitted") continue;
|
|
884
|
+
owners.set(JSON.stringify(pending.owner), {
|
|
885
|
+
agentId: pending.agentId ?? pending.startedAgentId ?? pending.spawnKey,
|
|
886
|
+
owner: { ...pending.owner },
|
|
887
|
+
generation: pending.executionGeneration,
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
for (const [agentId, waiter] of state.agentWaiters) {
|
|
891
|
+
const owner: WorkflowOwner = {
|
|
892
|
+
extension: "pi-workflows",
|
|
640
893
|
runId: state.run.runId,
|
|
641
|
-
nodeId:
|
|
642
|
-
attemptId: `attempt-${waiter
|
|
894
|
+
nodeId: `call-${waiter.callIndex}`,
|
|
895
|
+
attemptId: `attempt-${waiter.generation}`,
|
|
643
896
|
};
|
|
644
|
-
|
|
645
|
-
targets.set(state.run.runId, { agentIds, owners });
|
|
646
|
-
state.lifecycleSuspended = true;
|
|
647
|
-
state.bufferedTerminals.clear();
|
|
648
|
-
state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
const operation = (async () => {
|
|
652
|
-
const operations: Array<Promise<WorkflowQuiesceResult>> = [];
|
|
653
|
-
for (const state of this.runs.values()) {
|
|
654
|
-
const run = state.run;
|
|
655
|
-
const target = targets.get(run.runId);
|
|
656
|
-
if (isTerminalWorkflow(run.status)) {
|
|
657
|
-
this.reconcilePendingSpawns(state);
|
|
658
|
-
continue;
|
|
659
|
-
}
|
|
660
|
-
if (state.pendingSpawns.size > 0) operations.push(this.quiescePendingSpawns(state));
|
|
661
|
-
const activeAgentIds = target?.agentIds ?? [];
|
|
662
|
-
if (activeAgentIds.length === 0) continue;
|
|
663
|
-
operations.push(
|
|
664
|
-
this.client
|
|
665
|
-
.quiesceOwned?.(run.runId, activeAgentIds, BRANCH_QUIESCE_TIMEOUT_MS, target?.owners)
|
|
666
|
-
.then((result) => ({
|
|
667
|
-
settled: result.settled && result.pending.length === 0,
|
|
668
|
-
pending: [...new Set(result.pending)].slice(0, 256),
|
|
669
|
-
}))
|
|
670
|
-
.catch((error: unknown) => ({
|
|
671
|
-
settled: false,
|
|
672
|
-
pending: activeAgentIds.slice(0, 256),
|
|
673
|
-
diagnostic: error instanceof Error ? error.message : String(error),
|
|
674
|
-
})) ?? Promise.resolve({ settled: false, pending: activeAgentIds.slice(0, 256) }),
|
|
675
|
-
);
|
|
897
|
+
owners.set(JSON.stringify(owner), { agentId, owner, generation: waiter.executionGeneration });
|
|
676
898
|
}
|
|
677
|
-
const
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
};
|
|
685
|
-
})
|
|
899
|
+
for (const cleanup of this.cleanupObligations.get(state)?.values() ?? []) {
|
|
900
|
+
owners.set(JSON.stringify(cleanup.owner), {
|
|
901
|
+
agentId: cleanup.agentId,
|
|
902
|
+
owner: { ...cleanup.owner },
|
|
903
|
+
generation: state.executionGeneration,
|
|
904
|
+
});
|
|
905
|
+
}
|
|
906
|
+
return [...owners.values()].map((target) => ({ state, ...target }));
|
|
907
|
+
});
|
|
908
|
+
// Publish the handshake before RPC, abort or inspection can re-enter.
|
|
909
|
+
let acceptOperations!: (operations: Promise<void>[]) => void;
|
|
910
|
+
const operations = new Promise<Promise<void>[]>((resolve) => {
|
|
911
|
+
acceptOperations = resolve;
|
|
912
|
+
});
|
|
913
|
+
const operation = operations.then(async (cleanup) => {
|
|
914
|
+
await Promise.all(cleanup);
|
|
915
|
+
const pending = [
|
|
916
|
+
...new Set(
|
|
917
|
+
states.flatMap((state) =>
|
|
918
|
+
[...(this.cleanupObligations.get(state)?.values() ?? [])].map((cleanup) => cleanup.agentId),
|
|
919
|
+
),
|
|
920
|
+
),
|
|
921
|
+
].slice(0, 256);
|
|
922
|
+
return { settled: pending.length === 0, pending };
|
|
923
|
+
});
|
|
686
924
|
const tracked = operation.then(
|
|
687
925
|
(result) => {
|
|
688
926
|
if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
|
|
@@ -694,6 +932,34 @@ export class WorkflowEngine {
|
|
|
694
932
|
},
|
|
695
933
|
);
|
|
696
934
|
this.branchQuiescePromise = tracked;
|
|
935
|
+
// Fence every captured state before the first notifying observer. Start
|
|
936
|
+
// known-id owner checks synchronously for the bus-first shutdown handshake;
|
|
937
|
+
// their positive result is consumed by the same stable cleanup obligation,
|
|
938
|
+
// never treated as a status-only shortcut or a separate donated lease.
|
|
939
|
+
for (const state of states) state.lifecycleSuspended = true;
|
|
940
|
+
acceptOperations(
|
|
941
|
+
targets.map(({ state, agentId, owner, generation }) => {
|
|
942
|
+
const existing = [...(this.cleanupObligations.get(state)?.values() ?? [])].find((cleanup) =>
|
|
943
|
+
sameWorkflowOwner(cleanup.owner, owner),
|
|
944
|
+
);
|
|
945
|
+
let quiescence: Promise<{ settled: boolean; pending: string[] }> | undefined;
|
|
946
|
+
if (!existing?.operation && agentId !== `${owner.runId}/${owner.nodeId}/${owner.attemptId}`) {
|
|
947
|
+
try {
|
|
948
|
+
quiescence = this.client.quiesceOwned?.(owner.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner]);
|
|
949
|
+
} catch (error) {
|
|
950
|
+
quiescence = Promise.reject(error);
|
|
951
|
+
}
|
|
952
|
+
void quiescence?.catch(() => {});
|
|
953
|
+
}
|
|
954
|
+
return this.stopAndQuiesce(state, agentId, owner, generation, undefined, quiescence);
|
|
955
|
+
}),
|
|
956
|
+
);
|
|
957
|
+
for (const state of states) {
|
|
958
|
+
state.bufferedTerminals.clear();
|
|
959
|
+
state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
|
|
960
|
+
state.presentation.reset(false);
|
|
961
|
+
this.notifyInspection(state);
|
|
962
|
+
}
|
|
697
963
|
return tracked;
|
|
698
964
|
}
|
|
699
965
|
|
|
@@ -788,8 +1054,14 @@ export class WorkflowEngine {
|
|
|
788
1054
|
...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
|
|
789
1055
|
timestamp: now,
|
|
790
1056
|
});
|
|
1057
|
+
if (epoch !== this.branchEpoch || this.disposed || this.lifecyclePaused || this.admissionsClosed)
|
|
1058
|
+
throw new Error("Workflow branch changed before admission");
|
|
791
1059
|
const state: ScriptRunState = {
|
|
792
1060
|
run,
|
|
1061
|
+
presentation: new WorkflowPresentation(runId),
|
|
1062
|
+
managedCallFacts: new Map(),
|
|
1063
|
+
pendingCleanup: new Set(),
|
|
1064
|
+
activeExecutionGenerations: new Set(),
|
|
793
1065
|
frozenOptions: {
|
|
794
1066
|
...(frozenTokenBudget !== undefined ? { tokenBudget: frozenTokenBudget } : {}),
|
|
795
1067
|
...(frozenMaxAgents !== undefined ? { maxAgents: frozenMaxAgents } : {}),
|
|
@@ -810,13 +1082,19 @@ export class WorkflowEngine {
|
|
|
810
1082
|
executionGeneration: 0,
|
|
811
1083
|
};
|
|
812
1084
|
this.runs.set(runId, state);
|
|
813
|
-
|
|
814
|
-
state.
|
|
815
|
-
|
|
1085
|
+
// Accept identity before any synchronous running/root observer can stop or
|
|
1086
|
+
// replace this state. Dispatch is deferred until its promise is published.
|
|
1087
|
+
const generation = ++state.executionGeneration;
|
|
1088
|
+
state.presentation.reset();
|
|
1089
|
+
state.execution = Promise.resolve().then(() =>
|
|
1090
|
+
this.execute(state, script, startOptions, undefined, generation, epoch, protocol),
|
|
1091
|
+
);
|
|
816
1092
|
const execution = state.execution;
|
|
817
1093
|
// A run-fatal abort/stop already journals its terminal state; never let a
|
|
818
1094
|
// leftover rejection become an unhandled promise rejection.
|
|
819
|
-
void
|
|
1095
|
+
void execution.catch(() => {});
|
|
1096
|
+
this.setWorkflowStatus(run, "running");
|
|
1097
|
+
if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
|
|
820
1098
|
if (options.background) return { runId, status: run.status, background: true };
|
|
821
1099
|
try {
|
|
822
1100
|
const settled = await this.waitFor(runId, options.signal);
|
|
@@ -867,7 +1145,7 @@ export class WorkflowEngine {
|
|
|
867
1145
|
throw new Error("Workflow branch changed before admission");
|
|
868
1146
|
const state = this.runs.get(runId);
|
|
869
1147
|
if (!state) return undefined;
|
|
870
|
-
if (state.lifecycleSuspended) return undefined;
|
|
1148
|
+
if (state.lifecycleSuspended || state.removeOperation) return undefined;
|
|
871
1149
|
const run = state.run;
|
|
872
1150
|
if (run.status === "running" || run.status === "pausing" || run.status === "stopping") return undefined;
|
|
873
1151
|
const providerTimer = this.providerResumeTimers.get(runId);
|
|
@@ -918,6 +1196,7 @@ export class WorkflowEngine {
|
|
|
918
1196
|
timestamp,
|
|
919
1197
|
};
|
|
920
1198
|
this.appendRequired(recovery);
|
|
1199
|
+
if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
|
|
921
1200
|
applyRecoveryEvent(run, recovery);
|
|
922
1201
|
state.result = undefined;
|
|
923
1202
|
}
|
|
@@ -941,6 +1220,7 @@ export class WorkflowEngine {
|
|
|
941
1220
|
}),
|
|
942
1221
|
);
|
|
943
1222
|
}
|
|
1223
|
+
if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
|
|
944
1224
|
// Arguments are frozen in every schema-v4 run_created fact. Resume never accepts
|
|
945
1225
|
// replacement args and never backfills the pre-schema-v4 run_args contract.
|
|
946
1226
|
const resumeArgs = cloneWorkflowArgs(run.args);
|
|
@@ -950,8 +1230,6 @@ export class WorkflowEngine {
|
|
|
950
1230
|
for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
|
|
951
1231
|
state.generations.set(callIndex, generationFromAttemptId(attemptId));
|
|
952
1232
|
}
|
|
953
|
-
this.setWorkflowStatus(run, "running");
|
|
954
|
-
state.controller = new AbortController();
|
|
955
1233
|
// Freeze semantics: original run parameters and args win over new options.
|
|
956
1234
|
// Only signal/confirm/background/loadSavedWorkflow/mainModel/strengths are
|
|
957
1235
|
// per-resume context; input/budget/scale/toolset are frozen from the
|
|
@@ -963,6 +1241,7 @@ export class WorkflowEngine {
|
|
|
963
1241
|
// a caller that passed none is asking the engine to read the current one.
|
|
964
1242
|
strengths: options.strengths ?? this.strengths?.(),
|
|
965
1243
|
args: resumeArgs,
|
|
1244
|
+
loadSavedWorkflow: options.loadSavedWorkflow ?? this.loadSavedWorkflow,
|
|
966
1245
|
tokenBudget: frozen.tokenBudget,
|
|
967
1246
|
maxAgents: frozen.maxAgents,
|
|
968
1247
|
concurrency: frozen.concurrency,
|
|
@@ -971,18 +1250,21 @@ export class WorkflowEngine {
|
|
|
971
1250
|
toolset: frozen.toolset,
|
|
972
1251
|
excludeTools: frozen.excludeTools ? [...frozen.excludeTools] : undefined,
|
|
973
1252
|
};
|
|
1253
|
+
if (!this.isCurrentState(state, state.executionGeneration, epoch) || run.nonResumable || state.stopOperation)
|
|
1254
|
+
return undefined;
|
|
974
1255
|
state.bufferedTerminals.clear();
|
|
975
|
-
state
|
|
976
|
-
state.
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
state
|
|
982
|
-
protocol,
|
|
1256
|
+
this.terminalNotified.delete(state);
|
|
1257
|
+
state.controller = new AbortController();
|
|
1258
|
+
const generation = ++state.executionGeneration;
|
|
1259
|
+
const acceptedScript = run.script;
|
|
1260
|
+
state.presentation.reset();
|
|
1261
|
+
state.execution = Promise.resolve().then(() =>
|
|
1262
|
+
this.execute(state, acceptedScript, resumeOptions, resumeJournal, generation, epoch, protocol),
|
|
983
1263
|
);
|
|
984
1264
|
const execution = state.execution;
|
|
985
1265
|
void execution.catch(() => {});
|
|
1266
|
+
this.setWorkflowStatus(run, "running");
|
|
1267
|
+
if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
|
|
986
1268
|
if (options.background) {
|
|
987
1269
|
this.onBackgroundExecution?.(runId);
|
|
988
1270
|
return { runId, status: run.status, background: true };
|
|
@@ -1041,11 +1323,7 @@ export class WorkflowEngine {
|
|
|
1041
1323
|
await this.stop(state);
|
|
1042
1324
|
} else if (action === "rm") {
|
|
1043
1325
|
if (!isTerminalWorkflow(run.status)) throw new Error("cannot remove a running workflow");
|
|
1044
|
-
this.
|
|
1045
|
-
this.write({ kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp: Date.now() }),
|
|
1046
|
-
);
|
|
1047
|
-
this.runs.delete(runId);
|
|
1048
|
-
this.waiters.delete(runId);
|
|
1326
|
+
await this.remove(state);
|
|
1049
1327
|
return { action, run: this.summary(run) };
|
|
1050
1328
|
}
|
|
1051
1329
|
return { action, run: this.summary(run) };
|
|
@@ -1054,6 +1332,7 @@ export class WorkflowEngine {
|
|
|
1054
1332
|
async waitFor(runId: string, signal?: AbortSignal): Promise<ScriptRun> {
|
|
1055
1333
|
const state = this.runs.get(runId);
|
|
1056
1334
|
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
1335
|
+
if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size > 0) throw new WorkflowWaitAbortedError();
|
|
1057
1336
|
if (isTerminalWorkflow(state.run.status) || state.run.status === "paused" || state.run.status === "interrupted")
|
|
1058
1337
|
return state.run;
|
|
1059
1338
|
if (signal?.aborted) throw new WorkflowWaitAbortedError();
|
|
@@ -1102,6 +1381,7 @@ export class WorkflowEngine {
|
|
|
1102
1381
|
}
|
|
1103
1382
|
|
|
1104
1383
|
private settleWaiters(runId: string, run: ScriptRun): void {
|
|
1384
|
+
if (this.runs.get(runId)?.run !== run || this.hasPendingCleanup(runId)) return;
|
|
1105
1385
|
const all = this.waiters.get(runId);
|
|
1106
1386
|
if (!all) return;
|
|
1107
1387
|
for (const waiter of [...all]) {
|
|
@@ -1117,24 +1397,24 @@ export class WorkflowEngine {
|
|
|
1117
1397
|
// ── execution ──────────────────────────────────────────────────────────────
|
|
1118
1398
|
|
|
1119
1399
|
private async execute(
|
|
1120
|
-
|
|
1400
|
+
state: ScriptRunState,
|
|
1121
1401
|
script: string,
|
|
1122
1402
|
options: ScriptStartOptions,
|
|
1123
|
-
resumeJournal
|
|
1124
|
-
|
|
1403
|
+
resumeJournal: Map<string, JournalEntry> | undefined,
|
|
1404
|
+
generation: number,
|
|
1405
|
+
epoch: number,
|
|
1125
1406
|
protocol?: ManagedProtocolCheck,
|
|
1126
1407
|
): Promise<WorkflowRunResult<unknown>> {
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
1130
|
-
const generation = executionGeneration ?? state.executionGeneration;
|
|
1131
|
-
const epoch = this.branchEpoch;
|
|
1408
|
+
const runId = state.run.runId;
|
|
1409
|
+
state.activeExecutionGenerations.add(generation);
|
|
1132
1410
|
const isSameBranch = () =>
|
|
1133
1411
|
this.runs.get(runId) === state &&
|
|
1134
1412
|
state.executionGeneration === generation &&
|
|
1135
1413
|
this.branchEpoch === epoch &&
|
|
1136
1414
|
!this.disposed;
|
|
1137
1415
|
const isCurrent = () => isSameBranch() && !state.lifecycleSuspended && !this.lifecyclePaused;
|
|
1416
|
+
const isPresentationCurrent = () => isCurrent() && state.activeExecutionGenerations.has(generation);
|
|
1417
|
+
const explicitlyStopped = () => state.stopOperation !== undefined || state.run.terminalIntent === "stop";
|
|
1138
1418
|
const executionSignal =
|
|
1139
1419
|
!options.background && options.signal
|
|
1140
1420
|
? AbortSignal.any([state.controller.signal, options.signal])
|
|
@@ -1143,7 +1423,8 @@ export class WorkflowEngine {
|
|
|
1143
1423
|
const cleanupTasks = new Set<Promise<void>>();
|
|
1144
1424
|
const runner: WorkflowAgentRunner = {
|
|
1145
1425
|
run: async (prompt: string, runOptions: AgentRunOptions = {}) => {
|
|
1146
|
-
if (!isCurrent()
|
|
1426
|
+
if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
|
|
1427
|
+
throw new WorkflowError("stale workflow execution", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1147
1428
|
const dispatch = this.dispatchAgent(runId, prompt, runOptions);
|
|
1148
1429
|
const settled = dispatch.then(
|
|
1149
1430
|
() => {},
|
|
@@ -1157,6 +1438,15 @@ export class WorkflowEngine {
|
|
|
1157
1438
|
if (runOptions.signal?.aborted) trackCleanup();
|
|
1158
1439
|
try {
|
|
1159
1440
|
return await dispatch;
|
|
1441
|
+
} catch (error: unknown) {
|
|
1442
|
+
if (isCurrent()) {
|
|
1443
|
+
const frameId = runOptions.runId ?? runId;
|
|
1444
|
+
const nodeId =
|
|
1445
|
+
frameId === runId ? String(runOptions.callIndex ?? 0) : `${frameId}:${runOptions.callIndex ?? 0}`;
|
|
1446
|
+
state.presentation.managed(nodeId, { status: runOptions.signal?.aborted ? "stopped" : "failed" });
|
|
1447
|
+
this.notifyInspection(state);
|
|
1448
|
+
}
|
|
1449
|
+
throw error;
|
|
1160
1450
|
} finally {
|
|
1161
1451
|
runOptions.signal?.removeEventListener("abort", trackCleanup);
|
|
1162
1452
|
}
|
|
@@ -1165,6 +1455,8 @@ export class WorkflowEngine {
|
|
|
1165
1455
|
};
|
|
1166
1456
|
|
|
1167
1457
|
try {
|
|
1458
|
+
if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
|
|
1459
|
+
throw new WorkflowError("workflow execution aborted before dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1168
1460
|
const result = await runWorkflow(script, {
|
|
1169
1461
|
cwd: this.cwd,
|
|
1170
1462
|
args: options.args,
|
|
@@ -1189,29 +1481,74 @@ export class WorkflowEngine {
|
|
|
1189
1481
|
runId,
|
|
1190
1482
|
executionNonce: `${generation}-${randomUUID()}`,
|
|
1191
1483
|
resumeJournal,
|
|
1484
|
+
resumeWorkflowGenerations: new Map(
|
|
1485
|
+
Object.entries(state.run.workflowResultGenerations).map(([index, generation]) => [Number(index), generation]),
|
|
1486
|
+
),
|
|
1192
1487
|
toolset: options.toolset,
|
|
1193
1488
|
strengths: options.strengths,
|
|
1194
1489
|
excludeTools: options.excludeTools,
|
|
1195
|
-
confirm:
|
|
1490
|
+
confirm:
|
|
1491
|
+
options.confirm &&
|
|
1492
|
+
(async (prompt, checkpointOptions, signal) => {
|
|
1493
|
+
if (!isCurrent() || executionSignal.aborted || signal?.aborted || state.stopOperation)
|
|
1494
|
+
throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1495
|
+
const value = await options.confirm?.(prompt, checkpointOptions, signal);
|
|
1496
|
+
if (!isCurrent() || executionSignal.aborted || signal?.aborted)
|
|
1497
|
+
throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1498
|
+
return value;
|
|
1499
|
+
}),
|
|
1196
1500
|
loadSavedWorkflow: options.loadSavedWorkflow,
|
|
1501
|
+
onAgentStart: (event) => {
|
|
1502
|
+
if (!isPresentationCurrent()) return;
|
|
1503
|
+
state.presentation.agentStart(event);
|
|
1504
|
+
this.notifyInspection(state);
|
|
1505
|
+
},
|
|
1506
|
+
onAgentEnd: (event) => {
|
|
1507
|
+
if (!isPresentationCurrent()) return;
|
|
1508
|
+
const nodeId = event.id.startsWith(`${runId}:`) ? event.id.slice(runId.length + 1) : undefined;
|
|
1509
|
+
const persisted = nodeId === undefined ? undefined : state.run.callResults[nodeId];
|
|
1510
|
+
const persistedTokens =
|
|
1511
|
+
nodeId !== undefined && persisted?.attemptId === state.run.attemptIds[nodeId]
|
|
1512
|
+
? persisted?.tokenCount
|
|
1513
|
+
: undefined;
|
|
1514
|
+
// Runtime replay reports zero new spend, not the historical call's
|
|
1515
|
+
// usage. Explicit managed zero is also authoritative over estimates.
|
|
1516
|
+
const tokenCount = event.tokenUsage?.total ?? persistedTokens ?? event.tokens;
|
|
1517
|
+
state.presentation.agentEnd({ ...event, tokens: tokenCount });
|
|
1518
|
+
this.notifyInspection(state);
|
|
1519
|
+
},
|
|
1520
|
+
onLog: (message, scope) => {
|
|
1521
|
+
if (!isPresentationCurrent()) return;
|
|
1522
|
+
state.presentation.log(message, scope?.frameId);
|
|
1523
|
+
this.notifyInspection(state);
|
|
1524
|
+
},
|
|
1197
1525
|
onAgentJournal: (entry) => {
|
|
1198
1526
|
if (!isCurrent()) return;
|
|
1199
|
-
//
|
|
1200
|
-
//
|
|
1201
|
-
// workflow_result fact below owns the nested replay boundary.
|
|
1527
|
+
// Persist the accepted owner before presentation observers can replace it.
|
|
1528
|
+
// Nested frames are owned by their parent's workflow_result boundary.
|
|
1202
1529
|
if (entry.runId === undefined || entry.runId === runId) this.journalCallResult(runId, entry);
|
|
1530
|
+
if (!isCurrent()) return;
|
|
1531
|
+
state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
|
|
1532
|
+
this.notifyInspection(state);
|
|
1203
1533
|
},
|
|
1204
1534
|
onWorkflowJournal: (entry) => {
|
|
1205
1535
|
if (!isCurrent()) return;
|
|
1206
1536
|
this.journalWorkflowResult(runId, entry);
|
|
1537
|
+
if (!isCurrent()) return;
|
|
1538
|
+
state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
|
|
1539
|
+
this.notifyInspection(state);
|
|
1207
1540
|
},
|
|
1208
1541
|
onRuntimeEvent: (event) => {
|
|
1209
|
-
if (!
|
|
1542
|
+
if (!isPresentationCurrent()) return;
|
|
1543
|
+
state.presentation.runtime(event);
|
|
1544
|
+
this.notifyInspection(state);
|
|
1545
|
+
if (!isPresentationCurrent()) return;
|
|
1210
1546
|
try {
|
|
1211
1547
|
this.events.emit("pi-workflows:runtime", { runId, event: cloneRuntimeEvent(event) });
|
|
1212
1548
|
} catch {
|
|
1213
1549
|
// Cross-extension observers cannot change workflow execution.
|
|
1214
1550
|
}
|
|
1551
|
+
if (!isPresentationCurrent()) return;
|
|
1215
1552
|
try {
|
|
1216
1553
|
this.onRuntimeEvent?.(runId, cloneRuntimeEvent(event));
|
|
1217
1554
|
} catch {
|
|
@@ -1228,6 +1565,16 @@ export class WorkflowEngine {
|
|
|
1228
1565
|
// A terminal frame cannot overtake accepted journal facts. In particular,
|
|
1229
1566
|
// a delayed call_attempt must precede its result, not reset it on replay.
|
|
1230
1567
|
await this.drainJournal(runId);
|
|
1568
|
+
if (
|
|
1569
|
+
executionSignal.aborted ||
|
|
1570
|
+
!isCurrent() ||
|
|
1571
|
+
state.run.status === "stopping" ||
|
|
1572
|
+
state.stopOperation !== undefined ||
|
|
1573
|
+
state.run.terminalIntent === "stop"
|
|
1574
|
+
)
|
|
1575
|
+
throw new WorkflowError("workflow execution aborted", WorkflowErrorCode.WORKFLOW_ABORTED, {
|
|
1576
|
+
recoverable: true,
|
|
1577
|
+
});
|
|
1231
1578
|
const live = this.runs.get(runId);
|
|
1232
1579
|
const run = live?.run;
|
|
1233
1580
|
if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
|
|
@@ -1238,7 +1585,7 @@ export class WorkflowEngine {
|
|
|
1238
1585
|
} catch (error: unknown) {
|
|
1239
1586
|
await runner.waitForCleanup?.();
|
|
1240
1587
|
const run = this.runs.get(runId)?.run;
|
|
1241
|
-
if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
|
|
1588
|
+
if (run && isSameBranch() && !isTerminalWorkflow(run.status) && run.status !== "stopping") {
|
|
1242
1589
|
const message = describeWorkflowError(error);
|
|
1243
1590
|
const workflowError = error instanceof WorkflowError ? error : wrapError(error);
|
|
1244
1591
|
if (workflowError.code === WorkflowErrorCode.PROVIDER_USAGE_LIMIT) {
|
|
@@ -1261,15 +1608,22 @@ export class WorkflowEngine {
|
|
|
1261
1608
|
run.status = "interrupted";
|
|
1262
1609
|
run.updatedAt = Date.now();
|
|
1263
1610
|
} else {
|
|
1264
|
-
this.setWorkflowStatus(run, "failed", message);
|
|
1265
1611
|
this.cleanupAgents(runId);
|
|
1612
|
+
this.setWorkflowStatus(run, "failed", message);
|
|
1266
1613
|
}
|
|
1267
1614
|
}
|
|
1268
1615
|
throw error;
|
|
1269
1616
|
} finally {
|
|
1617
|
+
state.activeExecutionGenerations.delete(generation);
|
|
1270
1618
|
// Reject any waiters that never settled (agents left in flight).
|
|
1271
1619
|
const run = this.runs.get(runId)?.run;
|
|
1272
|
-
if (run && isSameBranch())
|
|
1620
|
+
if (run && isSameBranch()) {
|
|
1621
|
+
if (isCurrent()) {
|
|
1622
|
+
state.presentation.settle(run.status);
|
|
1623
|
+
this.notifyInspection(state);
|
|
1624
|
+
}
|
|
1625
|
+
if (isSameBranch() && run.status !== "stopping") this.settleWaiters(runId, run);
|
|
1626
|
+
}
|
|
1273
1627
|
}
|
|
1274
1628
|
}
|
|
1275
1629
|
|
|
@@ -1326,6 +1680,7 @@ export class WorkflowEngine {
|
|
|
1326
1680
|
const tierIdentity: CallTierIdentity = tier === undefined ? {} : { tier };
|
|
1327
1681
|
state.callTiers.set(nodeId, tierIdentity);
|
|
1328
1682
|
run.callTiers[nodeId] = tierIdentity;
|
|
1683
|
+
state.managedCallFacts.delete(nodeId);
|
|
1329
1684
|
const task: DispatchTask = {
|
|
1330
1685
|
subagent_type: type,
|
|
1331
1686
|
prompt,
|
|
@@ -1341,6 +1696,15 @@ export class WorkflowEngine {
|
|
|
1341
1696
|
// represented by its parent workflow_result replay boundary instead of
|
|
1342
1697
|
// leaking child call facts into the parent's numeric call namespace.
|
|
1343
1698
|
if (!nested) {
|
|
1699
|
+
// Match journal replay's call_attempt semantics: historical results stay
|
|
1700
|
+
// durable, but are not current facts for this accepted live rotation.
|
|
1701
|
+
run.callStatus[nodeId] = "running";
|
|
1702
|
+
run.attempts[nodeId] = generation;
|
|
1703
|
+
run.attemptIds[nodeId] = attemptId;
|
|
1704
|
+
delete run.agentIds[nodeId];
|
|
1705
|
+
delete run.callResults[nodeId];
|
|
1706
|
+
run.compactions[nodeId] = 0;
|
|
1707
|
+
run.updatedAt = Date.now();
|
|
1344
1708
|
this.persist(runId, `call-attempt:${nodeId}`, () =>
|
|
1345
1709
|
this.write({
|
|
1346
1710
|
kind: "call_attempt",
|
|
@@ -1354,10 +1718,12 @@ export class WorkflowEngine {
|
|
|
1354
1718
|
timestamp: Date.now(),
|
|
1355
1719
|
}),
|
|
1356
1720
|
);
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1721
|
+
if (
|
|
1722
|
+
!this.isCurrentState(state, executionGeneration, dispatchEpoch) ||
|
|
1723
|
+
runOptions.signal?.aborted ||
|
|
1724
|
+
state.controller.signal.aborted
|
|
1725
|
+
)
|
|
1726
|
+
throw new WorkflowError("stale workflow attempt", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1361
1727
|
}
|
|
1362
1728
|
|
|
1363
1729
|
// Abort linkage: an external signal (pause/stop/Esc) cancels the wait and
|
|
@@ -1371,241 +1737,362 @@ export class WorkflowEngine {
|
|
|
1371
1737
|
nodeId: spawnNodeId,
|
|
1372
1738
|
attemptId: spawnAttemptId,
|
|
1373
1739
|
};
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1740
|
+
const pending = {
|
|
1741
|
+
spawnKey,
|
|
1742
|
+
owner,
|
|
1743
|
+
executionGeneration,
|
|
1744
|
+
phase: "reserved" as "reserved" | "submitted",
|
|
1745
|
+
startedAgentId: undefined as string | undefined,
|
|
1746
|
+
agentId: undefined as string | undefined,
|
|
1379
1747
|
};
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
let
|
|
1385
|
-
let
|
|
1748
|
+
// Keep this exact allocation lease through every publication callback and
|
|
1749
|
+
// terminal delivery. A waiter alone is insufficient: it is removed before
|
|
1750
|
+
// onUsage/onHistory, which may synchronously pause or replace the branch.
|
|
1751
|
+
state.pendingSpawns.set(spawnKey, pending);
|
|
1752
|
+
let responseId: string | undefined;
|
|
1753
|
+
let terminalAccepted = false;
|
|
1754
|
+
let ownedCleanup: Promise<void> | undefined;
|
|
1755
|
+
const cleanupResponse = () =>
|
|
1756
|
+
(ownedCleanup ??= this.stopAndQuiesce(
|
|
1757
|
+
state,
|
|
1758
|
+
responseId ?? pending.startedAgentId ?? spawnKey,
|
|
1759
|
+
owner,
|
|
1760
|
+
executionGeneration,
|
|
1761
|
+
));
|
|
1386
1762
|
try {
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
if (externalSignal.aborted)
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1763
|
+
state.presentation.dispatch(runOptions, attemptId);
|
|
1764
|
+
this.notifyInspection(state);
|
|
1765
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1766
|
+
throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1767
|
+
const reconcilePending = () => {
|
|
1768
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1769
|
+
const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
|
|
1770
|
+
void reconciliation?.catch(() => {});
|
|
1771
|
+
};
|
|
1772
|
+
const onPendingAbort = () => reconcilePending();
|
|
1773
|
+
externalSignal.addEventListener("abort", onPendingAbort, { once: true });
|
|
1774
|
+
if (externalSignal.aborted) onPendingAbort();
|
|
1775
|
+
|
|
1776
|
+
let pendingId = "";
|
|
1777
|
+
let response: ManagedSpawnResponse | string;
|
|
1778
|
+
try {
|
|
1779
|
+
// Reservation is local until this exact boundary. Observers above can
|
|
1780
|
+
// cancel without creating uncertainty for an RPC that was never sent.
|
|
1781
|
+
pending.phase = "submitted";
|
|
1782
|
+
response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, externalSignal);
|
|
1783
|
+
} catch (error: unknown) {
|
|
1784
|
+
// Transport failure is not evidence that the host never allocated. Keep
|
|
1785
|
+
// the exact spawn owner even without a reply or an aborted signal.
|
|
1786
|
+
await cleanupResponse();
|
|
1787
|
+
throw error;
|
|
1788
|
+
} finally {
|
|
1789
|
+
externalSignal.removeEventListener("abort", onPendingAbort);
|
|
1399
1790
|
}
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
throw new WorkflowError(
|
|
1418
|
-
"workflow dispatch became stale during pause/resume",
|
|
1419
|
-
WorkflowErrorCode.WORKFLOW_ABORTED,
|
|
1420
|
-
{ recoverable: true },
|
|
1421
|
-
);
|
|
1422
|
-
}
|
|
1423
|
-
if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
|
|
1424
|
-
this.persistCallTierIdentity(state, nodeId);
|
|
1425
|
-
}
|
|
1426
|
-
// Report the tier the host actually selected back to the runtime, which
|
|
1427
|
-
// journals it: a call that named none still resolved to something, and the
|
|
1428
|
-
// run's own record of what ran should say what.
|
|
1429
|
-
const resolvedTierIdentity = state.callTiers.get(nodeId);
|
|
1430
|
-
if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
|
|
1431
|
-
if (typeof response === "string") {
|
|
1432
|
-
pendingId = response;
|
|
1433
|
-
} else if (response.terminal) {
|
|
1434
|
-
state.bufferedTerminals.delete(spawnKey);
|
|
1435
|
-
// The spawn response already carries a terminal snapshot (fast completion
|
|
1436
|
-
// or interrupted replay). The runtime journals successful calls itself
|
|
1437
|
-
// (onAgentJournal); a terminal snapshot that is NOT a success is
|
|
1438
|
-
// journaled here so the resume journal reflects the failure.
|
|
1439
|
-
const terminal = response.terminal;
|
|
1440
|
-
if (typeof terminal.tokenCount === "number")
|
|
1441
|
-
runOptions.onUsage?.({
|
|
1442
|
-
input: 0,
|
|
1443
|
-
output: terminal.tokenCount,
|
|
1444
|
-
total: terminal.tokenCount,
|
|
1445
|
-
cost: 0,
|
|
1446
|
-
cacheRead: 0,
|
|
1447
|
-
cacheWrite: 0,
|
|
1448
|
-
});
|
|
1449
|
-
const status = managedTerminalStatus(terminal);
|
|
1450
|
-
if (status !== "completed") {
|
|
1451
|
-
const result = resultFromLifecycle(
|
|
1452
|
-
status,
|
|
1453
|
-
response.id,
|
|
1454
|
-
terminal.result,
|
|
1455
|
-
terminal.error,
|
|
1456
|
-
terminal.compactionCount,
|
|
1457
|
-
terminal.tokenCount,
|
|
1458
|
-
resolvedTierIdentity?.tier ?? tier,
|
|
1791
|
+
responseId = typeof response === "string" ? response : response.id;
|
|
1792
|
+
pending.agentId = responseId;
|
|
1793
|
+
if (
|
|
1794
|
+
state.executionGeneration !== executionGeneration ||
|
|
1795
|
+
this.runs.get(runId) !== state ||
|
|
1796
|
+
this.branchEpoch !== dispatchEpoch ||
|
|
1797
|
+
state.lifecycleSuspended ||
|
|
1798
|
+
this.lifecyclePaused ||
|
|
1799
|
+
state.controller.signal.aborted ||
|
|
1800
|
+
externalSignal.aborted
|
|
1801
|
+
) {
|
|
1802
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1803
|
+
await cleanupResponse();
|
|
1804
|
+
throw new WorkflowError(
|
|
1805
|
+
"workflow dispatch became stale during pause/resume",
|
|
1806
|
+
WorkflowErrorCode.WORKFLOW_ABORTED,
|
|
1807
|
+
{ recoverable: true },
|
|
1459
1808
|
);
|
|
1460
|
-
result.attemptId = attemptId;
|
|
1461
|
-
if (!nested) this.commitCallTerminal(runId, nodeId, result);
|
|
1462
|
-
if (status === "failed" && terminal.error) {
|
|
1463
|
-
throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
|
|
1464
|
-
}
|
|
1465
1809
|
}
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
state.
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
return;
|
|
1810
|
+
if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
|
|
1811
|
+
this.persistCallTierIdentity(state, nodeId);
|
|
1812
|
+
}
|
|
1813
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1814
|
+
throw new WorkflowError("stale workflow identity", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1815
|
+
this.recordManagedAdmission(state, nodeId, responseId, executionGeneration);
|
|
1816
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1817
|
+
throw new WorkflowError("stale workflow admission", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1818
|
+
if (typeof response !== "string" && response.terminal)
|
|
1819
|
+
this.recordManagedTerminal(state, nodeId, responseId, response.terminal, executionGeneration);
|
|
1820
|
+
const bufferedTerminal = state.bufferedTerminals.get(spawnKey);
|
|
1821
|
+
const observedTerminal =
|
|
1822
|
+
bufferedTerminal?.response.id === responseId &&
|
|
1823
|
+
bufferedTerminal.executionGeneration === executionGeneration &&
|
|
1824
|
+
sameWorkflowOwner(bufferedTerminal.owner, owner)
|
|
1825
|
+
? bufferedTerminal.response.terminal
|
|
1826
|
+
: undefined;
|
|
1827
|
+
const observedRunning = pending.startedAgentId === responseId;
|
|
1828
|
+
state.presentation.managed(nodeId, {
|
|
1829
|
+
agentId: responseId,
|
|
1830
|
+
status: observedTerminal
|
|
1831
|
+
? managedTerminalStatus(observedTerminal)
|
|
1832
|
+
: observedRunning
|
|
1833
|
+
? "running"
|
|
1834
|
+
: typeof response === "string"
|
|
1835
|
+
? "pending"
|
|
1836
|
+
: response.state === "queued"
|
|
1837
|
+
? "queued"
|
|
1838
|
+
: "running",
|
|
1839
|
+
tier: state.callTiers.get(nodeId)?.tier,
|
|
1840
|
+
...(typeof response !== "string" && response.terminal
|
|
1841
|
+
? {
|
|
1842
|
+
status: managedTerminalStatus(response.terminal),
|
|
1843
|
+
tokenCount: response.terminal.tokenCount,
|
|
1844
|
+
outputFile: response.terminal.outputFile,
|
|
1845
|
+
compactionCount: Math.max(
|
|
1846
|
+
response.terminal.compactionCount,
|
|
1847
|
+
state.managedCallFacts.get(nodeId)?.compactionCount ?? 0,
|
|
1848
|
+
),
|
|
1506
1849
|
}
|
|
1507
|
-
}
|
|
1508
|
-
|
|
1509
|
-
|
|
1850
|
+
: {}),
|
|
1851
|
+
});
|
|
1852
|
+
this.notifyInspection(state);
|
|
1853
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1854
|
+
throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1855
|
+
// Report the tier the host actually selected back to the runtime, which
|
|
1856
|
+
// journals it: a call that named none still resolved to something, and the
|
|
1857
|
+
// run's own record of what ran should say what.
|
|
1858
|
+
const resolvedTierIdentity = state.callTiers.get(nodeId);
|
|
1859
|
+
if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
|
|
1860
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1861
|
+
throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1862
|
+
if (typeof response === "string") {
|
|
1863
|
+
pendingId = response;
|
|
1864
|
+
} else if (response.terminal) {
|
|
1865
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1866
|
+
// The spawn response already carries a terminal snapshot (fast completion
|
|
1867
|
+
// or interrupted replay). The runtime journals successful calls itself
|
|
1868
|
+
// (onAgentJournal); a terminal snapshot that is NOT a success is
|
|
1869
|
+
// journaled here so the resume journal reflects the failure.
|
|
1870
|
+
const terminal = response.terminal;
|
|
1871
|
+
if (typeof terminal.tokenCount === "number")
|
|
1872
|
+
runOptions.onUsage?.({
|
|
1873
|
+
input: 0,
|
|
1874
|
+
output: terminal.tokenCount,
|
|
1875
|
+
total: terminal.tokenCount,
|
|
1876
|
+
cost: 0,
|
|
1877
|
+
cacheRead: 0,
|
|
1878
|
+
cacheWrite: 0,
|
|
1879
|
+
});
|
|
1880
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1881
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1882
|
+
terminalAccepted = true;
|
|
1883
|
+
const status = managedTerminalStatus(terminal);
|
|
1884
|
+
if (status !== "completed") {
|
|
1885
|
+
const result = resultFromLifecycle(
|
|
1886
|
+
status,
|
|
1887
|
+
response.id,
|
|
1888
|
+
terminal.result,
|
|
1889
|
+
terminal.error,
|
|
1890
|
+
terminal.compactionCount,
|
|
1891
|
+
terminal.tokenCount,
|
|
1892
|
+
resolvedTierIdentity?.tier ?? tier,
|
|
1510
1893
|
);
|
|
1511
|
-
|
|
1894
|
+
result.attemptId = attemptId;
|
|
1895
|
+
if (!nested) this.commitCallTerminal(runId, nodeId, result);
|
|
1896
|
+
if (status === "failed" && terminal.error) {
|
|
1897
|
+
throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1901
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1902
|
+
return terminal.result ?? "";
|
|
1903
|
+
} else {
|
|
1904
|
+
pendingId = response.id;
|
|
1512
1905
|
}
|
|
1513
|
-
};
|
|
1514
1906
|
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1907
|
+
const waitAbort = () => {
|
|
1908
|
+
const waiter = state.agentWaiters.get(pendingId);
|
|
1909
|
+
if (waiter && !waiter.settled) {
|
|
1910
|
+
waiter.settled = true;
|
|
1911
|
+
state.agentWaiters.delete(pendingId);
|
|
1912
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1913
|
+
// Stop acknowledges cancellation, not provider/tool finally completion.
|
|
1914
|
+
void cleanupResponse().then(async () => {
|
|
1915
|
+
if (
|
|
1916
|
+
!nested &&
|
|
1917
|
+
this.runs.get(runId) === state &&
|
|
1918
|
+
state.executionGeneration === executionGeneration &&
|
|
1919
|
+
!run.nonResumable
|
|
1920
|
+
) {
|
|
1921
|
+
const result = resultFromLifecycle(
|
|
1922
|
+
"stopped",
|
|
1923
|
+
pendingId,
|
|
1924
|
+
undefined,
|
|
1925
|
+
"workflow call cancelled",
|
|
1926
|
+
run.compactions[nodeId] ?? 0,
|
|
1927
|
+
undefined,
|
|
1928
|
+
resolvedTierIdentity?.tier ?? tier,
|
|
1929
|
+
);
|
|
1930
|
+
result.attemptId = attemptId;
|
|
1931
|
+
try {
|
|
1932
|
+
await this.drainJournal(runId);
|
|
1933
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch)) {
|
|
1934
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
this.commitCallTerminal(runId, nodeId, result, true);
|
|
1938
|
+
} catch (error) {
|
|
1939
|
+
this.fenceExecution(
|
|
1940
|
+
state,
|
|
1941
|
+
executionGeneration,
|
|
1942
|
+
"Could not persist cancelled call terminal fact; run is non-resumable",
|
|
1943
|
+
);
|
|
1944
|
+
waiter.reject(error);
|
|
1945
|
+
return;
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
waiter.reject(
|
|
1949
|
+
new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }),
|
|
1950
|
+
);
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1529
1953
|
};
|
|
1530
|
-
state.agentWaiters.set(pendingId, waiter);
|
|
1531
|
-
// A spawn that was admitted straight away is already running, and its
|
|
1532
|
-
// `subagents:started` event fired before this waiter existed. Only a
|
|
1533
|
-
// reply that says "queued" still has a start to wait for.
|
|
1534
|
-
if (typeof response !== "string" && response.state !== "queued") notifyAgentRunning(waiter);
|
|
1535
|
-
externalSignal.addEventListener("abort", waitAbort, { once: true });
|
|
1536
|
-
if (externalSignal.aborted) {
|
|
1537
|
-
waitAbort();
|
|
1538
|
-
return;
|
|
1539
|
-
}
|
|
1540
1954
|
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1955
|
+
// Wait for the terminal lifecycle event, consuming a matching terminal that
|
|
1956
|
+
// arrived while the managed-spawn reply was still pending.
|
|
1957
|
+
const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
|
|
1958
|
+
const waiter: AgentLifecycleWaiter = {
|
|
1959
|
+
callIndex: nodeId,
|
|
1960
|
+
generation,
|
|
1961
|
+
executionGeneration,
|
|
1962
|
+
onUsage: runOptions.onUsage,
|
|
1963
|
+
onHistory: runOptions.onHistory,
|
|
1964
|
+
onRunning: runOptions.onRunning,
|
|
1965
|
+
startedNotified: false,
|
|
1966
|
+
settled: false,
|
|
1967
|
+
resolve,
|
|
1968
|
+
reject,
|
|
1969
|
+
};
|
|
1970
|
+
state.agentWaiters.set(pendingId, waiter);
|
|
1971
|
+
// A spawn that was admitted straight away is already running, and its
|
|
1972
|
+
// `subagents:started` event fired before this waiter existed. Only a
|
|
1973
|
+
// reply that says "queued" still has a start to wait for.
|
|
1974
|
+
if (observedRunning || (typeof response !== "string" && response.state !== "queued"))
|
|
1975
|
+
notifyAgentRunning(waiter);
|
|
1976
|
+
externalSignal.addEventListener("abort", waitAbort, { once: true });
|
|
1977
|
+
if (externalSignal.aborted) {
|
|
1978
|
+
waitAbort();
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
const buffered = state.bufferedTerminals.get(spawnKey);
|
|
1983
|
+
if (!buffered) return;
|
|
1984
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1985
|
+
if (
|
|
1986
|
+
buffered.executionGeneration !== executionGeneration ||
|
|
1987
|
+
state.executionGeneration !== executionGeneration ||
|
|
1988
|
+
state.lifecycleSuspended ||
|
|
1989
|
+
state.controller.signal.aborted ||
|
|
1990
|
+
buffered.response.id !== pendingId ||
|
|
1991
|
+
!sameWorkflowOwner(buffered.owner, owner)
|
|
1992
|
+
)
|
|
1993
|
+
return;
|
|
1994
|
+
this.resolveAgentWaiter(state, pendingId, waiter, buffered);
|
|
1995
|
+
}).finally(() => {
|
|
1996
|
+
externalSignal.removeEventListener("abort", waitAbort);
|
|
1997
|
+
});
|
|
1998
|
+
|
|
1544
1999
|
if (
|
|
1545
|
-
|
|
1546
|
-
|
|
2000
|
+
this.runs.get(runId) !== state ||
|
|
2001
|
+
this.branchEpoch !== dispatchEpoch ||
|
|
1547
2002
|
state.lifecycleSuspended ||
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
state.
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
throw new WorkflowError("subagent settled without a terminal snapshot", WorkflowErrorCode.AGENT_EXECUTION_ERROR, {
|
|
1569
|
-
recoverable: true,
|
|
2003
|
+
this.lifecyclePaused
|
|
2004
|
+
) {
|
|
2005
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
|
|
2006
|
+
}
|
|
2007
|
+
const snapshot = terminal.terminal;
|
|
2008
|
+
if (!snapshot)
|
|
2009
|
+
throw new WorkflowError(
|
|
2010
|
+
"subagent settled without a terminal snapshot",
|
|
2011
|
+
WorkflowErrorCode.AGENT_EXECUTION_ERROR,
|
|
2012
|
+
{
|
|
2013
|
+
recoverable: true,
|
|
2014
|
+
},
|
|
2015
|
+
);
|
|
2016
|
+
this.recordManagedTerminal(state, nodeId, terminal.id, snapshot, executionGeneration);
|
|
2017
|
+
state.presentation.managed(nodeId, {
|
|
2018
|
+
agentId: terminal.id,
|
|
2019
|
+
status: managedTerminalStatus(snapshot),
|
|
2020
|
+
tokenCount: snapshot.tokenCount,
|
|
2021
|
+
outputFile: snapshot.outputFile,
|
|
2022
|
+
compactionCount: Math.max(snapshot.compactionCount, state.managedCallFacts.get(nodeId)?.compactionCount ?? 0),
|
|
1570
2023
|
});
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
const
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
2024
|
+
this.notifyInspection(state);
|
|
2025
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch))
|
|
2026
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
2027
|
+
if (externalSignal.aborted) throw new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
2028
|
+
terminalAccepted = true;
|
|
2029
|
+
const status = managedTerminalStatus(snapshot);
|
|
2030
|
+
// Successful calls are journaled by the runtime's onAgentJournal (the
|
|
2031
|
+
// single writer for completed results); only failures settle here so the
|
|
2032
|
+
// resume journal reflects them.
|
|
2033
|
+
if (status !== "completed") {
|
|
2034
|
+
const result = resultFromLifecycle(
|
|
2035
|
+
status,
|
|
2036
|
+
terminal.id,
|
|
2037
|
+
snapshot.result,
|
|
2038
|
+
snapshot.error,
|
|
2039
|
+
snapshot.compactionCount,
|
|
2040
|
+
snapshot.tokenCount,
|
|
2041
|
+
state.callTiers.get(nodeId)?.tier ?? tier,
|
|
2042
|
+
);
|
|
2043
|
+
result.attemptId = attemptId;
|
|
2044
|
+
if (!nested) this.commitCallTerminal(runId, nodeId, result);
|
|
2045
|
+
if (status === "failed" && snapshot.error) {
|
|
2046
|
+
throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
|
|
2047
|
+
}
|
|
1589
2048
|
}
|
|
2049
|
+
const completedTierIdentity = state.callTiers.get(nodeId);
|
|
2050
|
+
if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
|
|
2051
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
2052
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
2053
|
+
return snapshot.result ?? "";
|
|
2054
|
+
} finally {
|
|
2055
|
+
// Publication does not donate ownership to an observer or a replacement
|
|
2056
|
+
// state. Even a terminal reply must STOP + QUIESCE its captured owner
|
|
2057
|
+
// when observation retired this dispatch.
|
|
2058
|
+
if (
|
|
2059
|
+
pending.phase === "submitted" &&
|
|
2060
|
+
(!terminalAccepted || !this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
2061
|
+
)
|
|
2062
|
+
await cleanupResponse();
|
|
2063
|
+
state.pendingSpawns.delete(spawnKey);
|
|
2064
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1590
2065
|
}
|
|
1591
|
-
const completedTierIdentity = state.callTiers.get(nodeId);
|
|
1592
|
-
if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
|
|
1593
|
-
return snapshot.result ?? "";
|
|
1594
2066
|
}
|
|
1595
2067
|
|
|
1596
2068
|
private journalCallResult(runId: string, entry: JournalEntry): void {
|
|
2069
|
+
entry = structuredClone(entry);
|
|
1597
2070
|
const state = this.runs.get(runId);
|
|
1598
2071
|
if (!state) return;
|
|
1599
2072
|
const run = state.run;
|
|
2073
|
+
const executionGeneration = state.executionGeneration;
|
|
2074
|
+
const epoch = this.branchEpoch;
|
|
1600
2075
|
const nodeId = String(entry.index);
|
|
1601
2076
|
const result = entry.result;
|
|
1602
2077
|
const callTierIdentity: CallTierIdentity =
|
|
1603
2078
|
entry.tier === undefined ? (run.callTiers[nodeId] ?? {}) : { tier: entry.tier };
|
|
2079
|
+
const managed = state.managedCallFacts.get(nodeId);
|
|
2080
|
+
const currentManaged =
|
|
2081
|
+
managed?.executionGeneration === state.executionGeneration && managed.attemptId === run.attemptIds[nodeId]
|
|
2082
|
+
? managed
|
|
2083
|
+
: undefined;
|
|
1604
2084
|
const callResult: CallResult = {
|
|
2085
|
+
...(currentManaged
|
|
2086
|
+
? {
|
|
2087
|
+
agentId: currentManaged.agentId,
|
|
2088
|
+
...(currentManaged.tokenCount === undefined ? {} : { tokenCount: currentManaged.tokenCount }),
|
|
2089
|
+
...(currentManaged.outputFile === undefined ? {} : { outputFile: currentManaged.outputFile }),
|
|
2090
|
+
}
|
|
2091
|
+
: {}),
|
|
1605
2092
|
status: "completed",
|
|
1606
2093
|
...(typeof result === "string" && result.length === 0 ? {} : { result }),
|
|
1607
2094
|
...callTierIdentity,
|
|
1608
|
-
compactionCount: 0,
|
|
2095
|
+
compactionCount: currentManaged?.compactionCount ?? 0,
|
|
1609
2096
|
updatedAt: Date.now(),
|
|
1610
2097
|
};
|
|
1611
2098
|
// Checkpoints are local calls, not managed children. Give their completed
|
|
@@ -1623,6 +2110,7 @@ export class WorkflowEngine {
|
|
|
1623
2110
|
owner: { extension: "pi-workflows", runId, nodeId, attemptId },
|
|
1624
2111
|
timestamp: Date.now(),
|
|
1625
2112
|
});
|
|
2113
|
+
if (!this.isCurrentState(state, executionGeneration, epoch)) return;
|
|
1626
2114
|
state.generations.set(nodeId, generation);
|
|
1627
2115
|
run.attemptTracking = true;
|
|
1628
2116
|
run.attempts[nodeId] = generation;
|
|
@@ -1645,16 +2133,23 @@ export class WorkflowEngine {
|
|
|
1645
2133
|
...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
|
|
1646
2134
|
timestamp: Date.now(),
|
|
1647
2135
|
};
|
|
1648
|
-
|
|
2136
|
+
const acceptedEvent = structuredClone(event);
|
|
2137
|
+
this.persist(runId, `call-result:${nodeId}`, () => this.write(acceptedEvent));
|
|
2138
|
+
if (!this.isCurrentState(state, executionGeneration, epoch)) return;
|
|
1649
2139
|
run.callResults[nodeId] = callResult;
|
|
1650
2140
|
run.callTiers[nodeId] = callResult.tier === undefined ? {} : { tier: callResult.tier };
|
|
1651
2141
|
run.callStatus[nodeId] = "completed";
|
|
2142
|
+
run.compactions[nodeId] = callResult.compactionCount;
|
|
2143
|
+
state.managedCallFacts.delete(nodeId);
|
|
1652
2144
|
run.updatedAt = Date.now();
|
|
1653
2145
|
}
|
|
1654
2146
|
|
|
1655
2147
|
private journalWorkflowResult(runId: string, entry: JournalEntry): void {
|
|
2148
|
+
entry = structuredClone(entry);
|
|
1656
2149
|
const state = this.runs.get(runId);
|
|
1657
2150
|
if (!state) return;
|
|
2151
|
+
const generation = state.executionGeneration;
|
|
2152
|
+
const epoch = this.branchEpoch;
|
|
1658
2153
|
const nodeId = String(entry.index);
|
|
1659
2154
|
const result: CallResult = {
|
|
1660
2155
|
status: "completed",
|
|
@@ -1681,6 +2176,7 @@ export class WorkflowEngine {
|
|
|
1681
2176
|
// otherwise a transient append retry would arrive after the terminal
|
|
1682
2177
|
// transition and be discarded as stale during restore.
|
|
1683
2178
|
this.appendRequired(event);
|
|
2179
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1684
2180
|
state.run.workflowResultGenerations[nodeId] = event.generation;
|
|
1685
2181
|
state.run.callResults[nodeId] = result;
|
|
1686
2182
|
state.run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
|
|
@@ -1692,6 +2188,8 @@ export class WorkflowEngine {
|
|
|
1692
2188
|
const state = this.runs.get(runId);
|
|
1693
2189
|
if (!state) return;
|
|
1694
2190
|
const run = state.run;
|
|
2191
|
+
const generation = state.executionGeneration;
|
|
2192
|
+
const epoch = this.branchEpoch;
|
|
1695
2193
|
const attemptId = result.attemptId ?? run.attemptIds[nodeId];
|
|
1696
2194
|
const event: JournalEvent = {
|
|
1697
2195
|
kind: "call_result",
|
|
@@ -1705,6 +2203,7 @@ export class WorkflowEngine {
|
|
|
1705
2203
|
};
|
|
1706
2204
|
if (required) this.appendRequired(event);
|
|
1707
2205
|
else this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
|
|
2206
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1708
2207
|
run.callResults[nodeId] = result;
|
|
1709
2208
|
run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
|
|
1710
2209
|
run.callStatus[nodeId] = result.status;
|
|
@@ -1717,36 +2216,26 @@ export class WorkflowEngine {
|
|
|
1717
2216
|
|
|
1718
2217
|
private reconcilePendingSpawns(state: ScriptRunState): void {
|
|
1719
2218
|
for (const pending of state.pendingSpawns.values()) {
|
|
2219
|
+
if (pending.phase !== "submitted" || pending.agentId) continue;
|
|
1720
2220
|
const reconciliation = this.client.reconcileManaged?.(pending.spawnKey, pending.owner);
|
|
1721
2221
|
void reconciliation?.catch(() => {});
|
|
1722
2222
|
}
|
|
1723
|
-
|
|
2223
|
+
// Reconciliation is not positive exact-owner quiescence. The allocation
|
|
2224
|
+
// lease remains with dispatch until terminal admission or owned cleanup.
|
|
1724
2225
|
state.bufferedTerminals.clear();
|
|
1725
2226
|
}
|
|
1726
2227
|
|
|
1727
|
-
private async quiescePendingSpawns(state: ScriptRunState): Promise<WorkflowQuiesceResult> {
|
|
1728
|
-
const pending = [...state.pendingSpawns.values()];
|
|
1729
|
-
if (pending.length === 0) return { settled: true, pending: [] };
|
|
1730
|
-
if (!this.client.reconcileManaged) {
|
|
1731
|
-
return { settled: false, pending: pending.map(({ spawnKey }) => spawnKey).slice(0, 256) };
|
|
1732
|
-
}
|
|
1733
|
-
const failed: string[] = [];
|
|
1734
|
-
await Promise.all(
|
|
1735
|
-
pending.map(async ({ spawnKey, owner }) => {
|
|
1736
|
-
try {
|
|
1737
|
-
const result = await this.client.reconcileManaged?.(spawnKey, owner);
|
|
1738
|
-
if (result === undefined) failed.push(spawnKey);
|
|
1739
|
-
} catch {
|
|
1740
|
-
failed.push(spawnKey);
|
|
1741
|
-
}
|
|
1742
|
-
}),
|
|
1743
|
-
);
|
|
1744
|
-
return { settled: failed.length === 0, pending: failed.slice(0, 256) };
|
|
1745
|
-
}
|
|
1746
|
-
|
|
1747
2228
|
private cleanupAgents(runId: string): void {
|
|
1748
2229
|
const state = this.runs.get(runId);
|
|
1749
2230
|
if (!state) return;
|
|
2231
|
+
// Retain captured allocations before reconciliation, abort, or branch replay
|
|
2232
|
+
// can notify observers. Dispatch's finally may not run until a later turn;
|
|
2233
|
+
// its cleanup and any waiter cleanup share this stable exact-owner lease.
|
|
2234
|
+
for (const pending of state.pendingSpawns.values()) {
|
|
2235
|
+
if (pending.phase !== "submitted") continue;
|
|
2236
|
+
const agentId = pending.agentId ?? pending.startedAgentId ?? pending.spawnKey;
|
|
2237
|
+
void this.stopAndQuiesce(state, agentId, pending.owner, pending.executionGeneration);
|
|
2238
|
+
}
|
|
1750
2239
|
this.reconcilePendingSpawns(state);
|
|
1751
2240
|
for (const [agentId, waiter] of [...state.agentWaiters]) {
|
|
1752
2241
|
if (waiter.settled) continue;
|
|
@@ -1757,14 +2246,17 @@ export class WorkflowEngine {
|
|
|
1757
2246
|
recoverable: true,
|
|
1758
2247
|
}),
|
|
1759
2248
|
);
|
|
1760
|
-
void this.
|
|
1761
|
-
|
|
2249
|
+
void this.stopAndQuiesce(
|
|
2250
|
+
state,
|
|
2251
|
+
agentId,
|
|
2252
|
+
{
|
|
1762
2253
|
extension: "pi-workflows",
|
|
1763
2254
|
runId,
|
|
1764
2255
|
nodeId: `call-${waiter.callIndex}`,
|
|
1765
2256
|
attemptId: `attempt-${waiter.generation}`,
|
|
1766
|
-
}
|
|
1767
|
-
.
|
|
2257
|
+
},
|
|
2258
|
+
state.executionGeneration,
|
|
2259
|
+
);
|
|
1768
2260
|
}
|
|
1769
2261
|
}
|
|
1770
2262
|
|
|
@@ -1774,51 +2266,108 @@ export class WorkflowEngine {
|
|
|
1774
2266
|
owner: WorkflowOwner,
|
|
1775
2267
|
executionGeneration: number,
|
|
1776
2268
|
unconfirmed?: string,
|
|
2269
|
+
quiescence?: Promise<{ settled: boolean; pending: string[] }>,
|
|
1777
2270
|
): Promise<void> {
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
2271
|
+
const obligations = this.cleanupObligations.get(state) ?? new Map<string, OwnedCleanup>();
|
|
2272
|
+
this.cleanupObligations.set(state, obligations);
|
|
2273
|
+
const allocationKey = `${owner.runId}/${owner.nodeId}/${owner.attemptId}`;
|
|
2274
|
+
const existing = [...obligations.entries()].find(([, cleanup]) => sameWorkflowOwner(cleanup.owner, owner));
|
|
2275
|
+
const key = existing?.[0] ?? JSON.stringify([owner, agentId]);
|
|
2276
|
+
let obligation = existing?.[1];
|
|
2277
|
+
if (!obligation) {
|
|
2278
|
+
let resolve!: () => void;
|
|
2279
|
+
const confirmed = new Promise<void>((done) => {
|
|
2280
|
+
resolve = done;
|
|
2281
|
+
});
|
|
2282
|
+
obligation = {
|
|
2283
|
+
agentId,
|
|
2284
|
+
...(agentId === allocationKey ? { spawnKey: allocationKey } : {}),
|
|
2285
|
+
owner: { ...owner },
|
|
2286
|
+
epoch: this.branchEpoch,
|
|
2287
|
+
confirmed,
|
|
2288
|
+
resolve,
|
|
2289
|
+
};
|
|
2290
|
+
obligations.set(key, obligation);
|
|
2291
|
+
this.retainedCleanupStates.add(state);
|
|
2292
|
+
state.pendingCleanup.add(confirmed);
|
|
2293
|
+
} else if (obligation.spawnKey && agentId !== allocationKey) {
|
|
2294
|
+
// A late reply refines the same allocation lease, not a new obligation.
|
|
2295
|
+
obligation.agentId = agentId;
|
|
2296
|
+
obligation.spawnKey = undefined;
|
|
2297
|
+
}
|
|
2298
|
+
if (obligation.operation) return obligation.operation;
|
|
2299
|
+
const accepted = obligation;
|
|
2300
|
+
const currentOwner = () => this.isCurrentState(state, state.executionGeneration, accepted.epoch);
|
|
2301
|
+
const operation = Promise.resolve().then(async () => {
|
|
2302
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
2303
|
+
let stopError: unknown;
|
|
2304
|
+
try {
|
|
2305
|
+
if (unconfirmed) throw new Error(unconfirmed);
|
|
2306
|
+
const cleanup = (async () => {
|
|
2307
|
+
// An allocation without a confirmed id can only be retried by its exact spawn owner.
|
|
2308
|
+
if (accepted.spawnKey) {
|
|
2309
|
+
if (!this.client.reconcileManaged) throw new Error("owned allocation reconciliation unavailable");
|
|
2310
|
+
const reply = await this.client.reconcileManaged(accepted.spawnKey, accepted.owner);
|
|
2311
|
+
const id = typeof reply === "string" ? reply : reply?.id;
|
|
2312
|
+
if (!id) throw new Error("cancelled spawn allocation could not be confirmed");
|
|
2313
|
+
accepted.agentId = id;
|
|
2314
|
+
accepted.spawnKey = undefined;
|
|
2315
|
+
}
|
|
1784
2316
|
try {
|
|
1785
2317
|
if (!this.client.stopOwned) throw new Error("owned stop RPC unavailable");
|
|
1786
|
-
await this.client.stopOwned(agentId, owner);
|
|
2318
|
+
await this.client.stopOwned(accepted.agentId, accepted.owner);
|
|
1787
2319
|
} catch (error: unknown) {
|
|
1788
|
-
// Reconciliation may already have stopped the record. Only exact-owner
|
|
1789
|
-
// quiescence confirms that provider/tool cleanup has actually settled.
|
|
1790
2320
|
stopError = error;
|
|
1791
2321
|
}
|
|
1792
2322
|
if (!this.client.quiesceOwned) throw new Error("owned cleanup RPC unavailable");
|
|
1793
|
-
const result = await
|
|
2323
|
+
const result = await (quiescence ??
|
|
2324
|
+
this.client.quiesceOwned(accepted.owner.runId, [accepted.agentId], BRANCH_QUIESCE_TIMEOUT_MS, [
|
|
2325
|
+
accepted.owner,
|
|
2326
|
+
]));
|
|
1794
2327
|
if (!result?.settled || result.pending.length !== 0) throw new Error("owned agents are not quiescent");
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
2328
|
+
// A bounded negative/error is not evidence that cleanup settled. Only
|
|
2329
|
+
// this positive owner check releases the stable obligation, even late.
|
|
2330
|
+
obligations.delete(key);
|
|
2331
|
+
if (obligations.size === 0) this.retainedCleanupStates.delete(state);
|
|
2332
|
+
state.pendingCleanup.delete(accepted.confirmed);
|
|
2333
|
+
accepted.resolve();
|
|
2334
|
+
if (currentOwner() && isTerminalWorkflow(state.run.status)) this.notifyTerminal(state);
|
|
2335
|
+
})();
|
|
2336
|
+
await Promise.race([
|
|
2337
|
+
cleanup,
|
|
2338
|
+
new Promise<never>((_resolve, reject) => {
|
|
2339
|
+
timer = setTimeout(() => reject(new Error("owned cleanup timed out")), BRANCH_QUIESCE_TIMEOUT_MS + 1_000);
|
|
2340
|
+
}),
|
|
2341
|
+
]);
|
|
2342
|
+
} catch (error: unknown) {
|
|
2343
|
+
if (!currentOwner() || state.executionGeneration !== executionGeneration) return;
|
|
2344
|
+
const stopDiagnostic = stopError === undefined ? "" : `; stop failed: ${String(stopError)}`;
|
|
2345
|
+
const diagnostic = `Owned cleanup unconfirmed for ${agentId}: ${error instanceof Error ? error.message : String(error)}${stopDiagnostic}; run is non-resumable`;
|
|
2346
|
+
this.fenceExecution(state, executionGeneration, diagnostic);
|
|
2347
|
+
} finally {
|
|
2348
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
2349
|
+
}
|
|
2350
|
+
});
|
|
2351
|
+
accepted.operation = operation;
|
|
2352
|
+
try {
|
|
2353
|
+
await operation;
|
|
1807
2354
|
} finally {
|
|
1808
|
-
if (
|
|
2355
|
+
if (accepted.operation === operation) accepted.operation = undefined;
|
|
1809
2356
|
}
|
|
1810
2357
|
}
|
|
1811
2358
|
|
|
1812
2359
|
private async drainExecution(state: ScriptRunState, execution?: Promise<unknown>): Promise<void> {
|
|
1813
2360
|
if (!execution) return;
|
|
1814
2361
|
const generation = state.executionGeneration;
|
|
2362
|
+
const epoch = this.branchEpoch;
|
|
1815
2363
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1816
2364
|
try {
|
|
1817
2365
|
await Promise.race([
|
|
1818
2366
|
execution.catch(() => {}),
|
|
1819
2367
|
new Promise<void>((resolve) => {
|
|
1820
2368
|
timer = setTimeout(() => {
|
|
1821
|
-
this.
|
|
2369
|
+
if (this.isCurrentState(state, generation, epoch))
|
|
2370
|
+
this.fenceExecution(state, generation, "Workflow execution cleanup timed out; run is non-resumable");
|
|
1822
2371
|
resolve();
|
|
1823
2372
|
}, EXECUTION_DRAIN_TIMEOUT_MS);
|
|
1824
2373
|
}),
|
|
@@ -1830,7 +2379,8 @@ export class WorkflowEngine {
|
|
|
1830
2379
|
|
|
1831
2380
|
private fenceExecution(state: ScriptRunState, generation: number, diagnostic: string): void {
|
|
1832
2381
|
const run = state.run;
|
|
1833
|
-
|
|
2382
|
+
const epoch = this.branchEpoch;
|
|
2383
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1834
2384
|
if (run.nonResumable && run.status === "stopped") return;
|
|
1835
2385
|
run.nonResumable = true;
|
|
1836
2386
|
const blockedNodeIds = Object.keys(run.callStatus).filter((nodeId) => run.callStatus[nodeId] !== "completed");
|
|
@@ -1853,21 +2403,119 @@ export class WorkflowEngine {
|
|
|
1853
2403
|
timestamp: Date.now(),
|
|
1854
2404
|
};
|
|
1855
2405
|
this.persist(run.runId, "cleanup-unconfirmed", () => this.write(event));
|
|
2406
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1856
2407
|
// Fence before aborting: synchronous abort handlers may re-enter cleanup.
|
|
1857
2408
|
// Exact-owner cleanup continues, but no later result may revive this run.
|
|
1858
2409
|
state.executionGeneration += 1;
|
|
2410
|
+
state.presentation.reset(false);
|
|
2411
|
+
this.notifyInspection(state);
|
|
2412
|
+
if (!this.isCurrentState(state, generation + 1, epoch)) return;
|
|
1859
2413
|
state.controller.abort();
|
|
2414
|
+
if (!this.isCurrentState(state, generation + 1, epoch)) return;
|
|
1860
2415
|
this.cleanupAgents(run.runId);
|
|
1861
|
-
|
|
2416
|
+
// Uncertainty is an error, never a terminal result delivery. Foreground
|
|
2417
|
+
// callers can unwind, while owner UI retains an explicit cleanup recheck.
|
|
2418
|
+
if (state.pendingCleanup.size > 0) {
|
|
2419
|
+
const all = this.waiters.get(run.runId);
|
|
2420
|
+
this.waiters.delete(run.runId);
|
|
2421
|
+
for (const waiter of all ?? []) {
|
|
2422
|
+
if (waiter.settled) continue;
|
|
2423
|
+
waiter.settled = true;
|
|
2424
|
+
waiter.cleanup();
|
|
2425
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
2426
|
+
}
|
|
2427
|
+
} else this.notifyTerminal(state);
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
private async remove(state: ScriptRunState): Promise<void> {
|
|
2431
|
+
if (state.removeOperation) return state.removeOperation;
|
|
2432
|
+
const isBusy = () =>
|
|
2433
|
+
state.activeExecutionGenerations.size > 0 ||
|
|
2434
|
+
state.pendingCleanup.size > 0 ||
|
|
2435
|
+
state.pendingSpawns.size > 0 ||
|
|
2436
|
+
state.agentWaiters.size > 0 ||
|
|
2437
|
+
state.stopOperation !== undefined;
|
|
2438
|
+
if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
|
|
2439
|
+
const epoch = this.branchEpoch;
|
|
2440
|
+
const operation = Promise.resolve().then(async () => {
|
|
2441
|
+
await this.drainJournal(state.run.runId);
|
|
2442
|
+
if (
|
|
2443
|
+
this.runs.get(state.run.runId) !== state ||
|
|
2444
|
+
this.branchEpoch !== epoch ||
|
|
2445
|
+
this.disposed ||
|
|
2446
|
+
this.lifecyclePaused ||
|
|
2447
|
+
state.lifecycleSuspended
|
|
2448
|
+
)
|
|
2449
|
+
throw new Error("workflow branch changed before removal");
|
|
2450
|
+
if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
|
|
2451
|
+
// No queued best-effort tombstone: memory survives failure so removal can be retried.
|
|
2452
|
+
this.appendRequired({
|
|
2453
|
+
kind: "run_removed",
|
|
2454
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2455
|
+
runId: state.run.runId,
|
|
2456
|
+
timestamp: Date.now(),
|
|
2457
|
+
});
|
|
2458
|
+
this.runs.delete(state.run.runId);
|
|
2459
|
+
this.waiters.delete(state.run.runId);
|
|
2460
|
+
this.notifyInspection(state);
|
|
2461
|
+
});
|
|
2462
|
+
state.removeOperation = operation;
|
|
2463
|
+
try {
|
|
2464
|
+
await operation;
|
|
2465
|
+
} finally {
|
|
2466
|
+
if (state.removeOperation === operation) state.removeOperation = undefined;
|
|
2467
|
+
}
|
|
1862
2468
|
}
|
|
1863
2469
|
|
|
1864
2470
|
private async stop(state: ScriptRunState): Promise<void> {
|
|
2471
|
+
if (state.stopOperation) return state.stopOperation;
|
|
2472
|
+
if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size === 0) return;
|
|
2473
|
+
const generation = state.executionGeneration;
|
|
2474
|
+
const epoch = this.branchEpoch;
|
|
2475
|
+
const operation = Promise.resolve().then(async () => {
|
|
2476
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
2477
|
+
const pending = [...(this.cleanupObligations.get(state)?.values() ?? [])];
|
|
2478
|
+
if (pending.length > 0) {
|
|
2479
|
+
await Promise.all(
|
|
2480
|
+
pending.map((cleanup) => this.stopAndQuiesce(state, cleanup.agentId, cleanup.owner, generation)),
|
|
2481
|
+
);
|
|
2482
|
+
if (!this.isCurrentState(state, generation, epoch) || state.pendingCleanup.size > 0) return;
|
|
2483
|
+
if (isTerminalWorkflow(state.run.status)) {
|
|
2484
|
+
this.notifyTerminal(state);
|
|
2485
|
+
return;
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
await this.stopExecution(state, generation, epoch);
|
|
2489
|
+
});
|
|
2490
|
+
state.stopOperation = operation;
|
|
2491
|
+
try {
|
|
2492
|
+
await operation;
|
|
2493
|
+
} finally {
|
|
2494
|
+
if (state.stopOperation === operation) state.stopOperation = undefined;
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
private async stopExecution(state: ScriptRunState, generation: number, epoch: number): Promise<void> {
|
|
1865
2499
|
const run = state.run;
|
|
1866
|
-
|
|
2500
|
+
const current = () => this.isCurrentState(state, generation, epoch);
|
|
2501
|
+
if (!current() || isTerminalWorkflow(run.status)) return;
|
|
1867
2502
|
this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
|
|
2503
|
+
if (!current()) return;
|
|
1868
2504
|
run.nonResumable = true;
|
|
2505
|
+
// Runtime abort listeners own exact cleanup. Rejecting their waiters here would
|
|
2506
|
+
// falsely settle the dispatch before its stop-and-quiesce barrier completes.
|
|
1869
2507
|
state.controller.abort();
|
|
1870
|
-
|
|
2508
|
+
if (!current()) return;
|
|
2509
|
+
await this.drainExecution(state, state.execution);
|
|
2510
|
+
if (!current() || state.pendingCleanup.size > 0) return;
|
|
2511
|
+
try {
|
|
2512
|
+
await this.drainJournal(run.runId);
|
|
2513
|
+
} catch {
|
|
2514
|
+
if (current())
|
|
2515
|
+
this.fenceExecution(state, generation, "Could not drain workflow journal during stop; run is non-resumable");
|
|
2516
|
+
return;
|
|
2517
|
+
}
|
|
2518
|
+
if (!current()) return;
|
|
1871
2519
|
const terminalResults: RecoveryTerminalResult[] = [];
|
|
1872
2520
|
for (const [nodeId, status] of Object.entries(run.callStatus)) {
|
|
1873
2521
|
if (status !== "completed") {
|
|
@@ -1887,8 +2535,6 @@ export class WorkflowEngine {
|
|
|
1887
2535
|
result,
|
|
1888
2536
|
owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId: run.attemptIds[nodeId] ?? "" },
|
|
1889
2537
|
});
|
|
1890
|
-
run.callResults[nodeId] = result;
|
|
1891
|
-
run.callStatus[nodeId] = "stopped";
|
|
1892
2538
|
}
|
|
1893
2539
|
}
|
|
1894
2540
|
const event: TerminalRecoveryEvent = {
|
|
@@ -1904,10 +2550,22 @@ export class WorkflowEngine {
|
|
|
1904
2550
|
recoveryId: deriveRecoveryId({ runId: run.runId, rotations: [], timestamp: Date.now() }),
|
|
1905
2551
|
timestamp: Date.now(),
|
|
1906
2552
|
};
|
|
1907
|
-
|
|
2553
|
+
try {
|
|
2554
|
+
this.appendRequired(event);
|
|
2555
|
+
} catch {
|
|
2556
|
+
this.fenceExecution(state, generation, "Could not persist workflow stop terminal; run is non-resumable");
|
|
2557
|
+
return;
|
|
2558
|
+
}
|
|
2559
|
+
if (!current()) return;
|
|
2560
|
+
for (const terminal of terminalResults) {
|
|
2561
|
+
run.callResults[terminal.nodeId] = terminal.result;
|
|
2562
|
+
run.callStatus[terminal.nodeId] = "stopped";
|
|
2563
|
+
}
|
|
1908
2564
|
run.status = "stopped";
|
|
1909
2565
|
run.updatedAt = Date.now();
|
|
1910
|
-
|
|
2566
|
+
state.presentation.settle(run.status);
|
|
2567
|
+
this.notifyInspection(state);
|
|
2568
|
+
if (current()) this.notifyTerminal(state);
|
|
1911
2569
|
}
|
|
1912
2570
|
|
|
1913
2571
|
// ── journal writes ─────────────────────────────────────────────────────────
|
|
@@ -2022,6 +2680,14 @@ export class WorkflowEngine {
|
|
|
2022
2680
|
* a transient failure, schedules a retry of the exact same append.
|
|
2023
2681
|
*/
|
|
2024
2682
|
private persist(runId: string, key: string, mutation: () => void): void {
|
|
2683
|
+
const acceptedState = this.runs.get(runId);
|
|
2684
|
+
const epoch = this.branchEpoch;
|
|
2685
|
+
const append = mutation;
|
|
2686
|
+
mutation = () => {
|
|
2687
|
+
if (this.disposed || this.lifecyclePaused || this.runs.get(runId) !== acceptedState || this.branchEpoch !== epoch)
|
|
2688
|
+
return;
|
|
2689
|
+
append();
|
|
2690
|
+
};
|
|
2025
2691
|
if ([...this.journalRetries.keys()].some((id) => id.startsWith(`${runId}\u0000`))) {
|
|
2026
2692
|
this.scheduleJournalRetry(runId, key, mutation);
|
|
2027
2693
|
return;
|
|
@@ -2098,6 +2764,10 @@ export class WorkflowEngine {
|
|
|
2098
2764
|
finalResult?: unknown,
|
|
2099
2765
|
): void {
|
|
2100
2766
|
if (isTerminalWorkflow(run.status) && !isTerminalWorkflow(status)) return;
|
|
2767
|
+
const state = this.runs.get(run.runId);
|
|
2768
|
+
const generation = state?.executionGeneration ?? 0;
|
|
2769
|
+
const epoch = this.branchEpoch;
|
|
2770
|
+
if (!state || state.run !== run || !this.isCurrentState(state, generation, epoch)) return;
|
|
2101
2771
|
// Validate and isolate terminal values before best-effort persistence or any
|
|
2102
2772
|
// status mutation. A bad result must still be able to transition the run to failed.
|
|
2103
2773
|
const durableFinalResult = finalResult === undefined ? undefined : cloneWorkflowResult(finalResult);
|
|
@@ -2115,16 +2785,18 @@ export class WorkflowEngine {
|
|
|
2115
2785
|
timestamp,
|
|
2116
2786
|
}),
|
|
2117
2787
|
);
|
|
2788
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
2118
2789
|
run.status = status;
|
|
2119
2790
|
if (status === "running") run.error = undefined;
|
|
2120
2791
|
if (error !== undefined) run.error = error;
|
|
2121
2792
|
if (runFinalResult !== undefined) run.finalResult = runFinalResult;
|
|
2122
2793
|
if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
|
|
2123
2794
|
run.updatedAt = timestamp;
|
|
2795
|
+
this.notifyInspection(state);
|
|
2796
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
2124
2797
|
if (isTerminalWorkflow(status)) {
|
|
2125
|
-
|
|
2126
|
-
this.
|
|
2127
|
-
this.onRunSettled?.();
|
|
2798
|
+
state.bufferedTerminals.clear();
|
|
2799
|
+
this.notifyTerminal(state);
|
|
2128
2800
|
}
|
|
2129
2801
|
}
|
|
2130
2802
|
|
|
@@ -2132,6 +2804,9 @@ export class WorkflowEngine {
|
|
|
2132
2804
|
|
|
2133
2805
|
restore(entries: readonly SessionEntryLike[], branchGeneration = 0): void {
|
|
2134
2806
|
if (this.disposed) return;
|
|
2807
|
+
const retiredStates = [...this.runs.values()];
|
|
2808
|
+
this.branchEpoch++;
|
|
2809
|
+
for (const state of retiredStates) state.lifecycleSuspended = true;
|
|
2135
2810
|
// A branch replay is a replacement, not a merge. Runs absent from the new
|
|
2136
2811
|
// branch must not remain controllable or append facts into it.
|
|
2137
2812
|
this.clearJournalRetries();
|
|
@@ -2149,6 +2824,10 @@ export class WorkflowEngine {
|
|
|
2149
2824
|
for (const run of runs.values()) {
|
|
2150
2825
|
const state: ScriptRunState = {
|
|
2151
2826
|
run,
|
|
2827
|
+
presentation: new WorkflowPresentation(run.runId, true),
|
|
2828
|
+
managedCallFacts: new Map(),
|
|
2829
|
+
pendingCleanup: new Set(),
|
|
2830
|
+
activeExecutionGenerations: new Set(),
|
|
2152
2831
|
frozenOptions: {
|
|
2153
2832
|
...(run.frozenTokenBudget !== undefined ? { tokenBudget: run.frozenTokenBudget } : {}),
|
|
2154
2833
|
...(run.frozenMaxAgents !== undefined ? { maxAgents: run.frozenMaxAgents } : {}),
|
|
@@ -2183,6 +2862,8 @@ export class WorkflowEngine {
|
|
|
2183
2862
|
run.updatedAt = Date.now();
|
|
2184
2863
|
this.runs.set(run.runId, state);
|
|
2185
2864
|
}
|
|
2865
|
+
for (const state of retiredStates) this.notifyInspection(state);
|
|
2866
|
+
for (const state of this.runs.values()) this.notifyInspection(state);
|
|
2186
2867
|
}
|
|
2187
2868
|
|
|
2188
2869
|
list(): Array<Record<string, unknown>> {
|
|
@@ -2200,6 +2881,18 @@ export class WorkflowEngine {
|
|
|
2200
2881
|
return this.runs.get(runId)?.run;
|
|
2201
2882
|
}
|
|
2202
2883
|
|
|
2884
|
+
/** Primitive-only Fleet metadata; avoids cloning full inspection arrays for every run. */
|
|
2885
|
+
getInspectionHeader(runId: string): WorkflowInspectionHeader | undefined {
|
|
2886
|
+
const state = this.runs.get(runId);
|
|
2887
|
+
return state?.presentation.header(state.executionGeneration);
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
/** Detached bounded facts for late-opening UI mounts; no event callback ownership required. */
|
|
2891
|
+
getInspection(runId: string): WorkflowInspection | undefined {
|
|
2892
|
+
const state = this.runs.get(runId);
|
|
2893
|
+
return state?.presentation.snapshot(state.run, state.executionGeneration);
|
|
2894
|
+
}
|
|
2895
|
+
|
|
2203
2896
|
summary(run: ScriptRun, detailed = false): Record<string, unknown> {
|
|
2204
2897
|
const base: Record<string, unknown> = {
|
|
2205
2898
|
runId: run.runId,
|
|
@@ -2250,5 +2943,5 @@ export class WorkflowWaitAbortedError extends Error {
|
|
|
2250
2943
|
}
|
|
2251
2944
|
}
|
|
2252
2945
|
|
|
2253
|
-
export type { ScriptRun, WorkflowOwner };
|
|
2946
|
+
export type { ScriptRun, WorkflowInspection, WorkflowInspectionHeader, WorkflowOwner };
|
|
2254
2947
|
export { createManagedSpawnClient, JOURNAL_ENTRY_TYPE, snapshotRun };
|