@signalridge/pi-workflows 1.8.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts CHANGED
@@ -21,6 +21,7 @@ import { createHash, randomUUID } from "node:crypto";
21
21
  import type { ManagedSpawnResponse } from "@signalridge/pi-subagents-protocol";
22
22
  import { isManagedAgentTier } from "@signalridge/pi-subagents-protocol";
23
23
  import { describeWorkflowError, WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
24
+ import { type WorkflowInspection, type WorkflowInspectionHeader, WorkflowPresentation } from "./inspection.js";
24
25
  import {
25
26
  applyRecoveryEvent,
26
27
  buildResumeJournal,
@@ -174,6 +175,25 @@ interface AgentTerminalDelivery {
174
175
  history?: unknown[];
175
176
  }
176
177
 
178
+ interface ManagedCallFacts {
179
+ executionGeneration: number;
180
+ attemptId: string;
181
+ agentId: string;
182
+ tokenCount?: number;
183
+ outputFile?: string;
184
+ compactionCount: number;
185
+ }
186
+
187
+ interface OwnedCleanup {
188
+ agentId: string;
189
+ spawnKey?: string;
190
+ owner: WorkflowOwner;
191
+ epoch: number;
192
+ confirmed: Promise<void>;
193
+ resolve: () => void;
194
+ operation?: Promise<void>;
195
+ }
196
+
177
197
  interface BufferedAgentTerminal extends AgentTerminalDelivery {
178
198
  owner: WorkflowOwner;
179
199
  executionGeneration: number;
@@ -181,6 +201,15 @@ interface BufferedAgentTerminal extends AgentTerminalDelivery {
181
201
 
182
202
  export interface ScriptRunState {
183
203
  run: ScriptRun;
204
+ /** Volatile, execution-fenced presentation; never journaled. */
205
+ presentation: WorkflowPresentation;
206
+ /** Exact-attempt metadata merged into the runtime's single schema-transformed result fact. */
207
+ managedCallFacts: Map<string, ManagedCallFacts>;
208
+ /** Ownership remains busy even after a cleanup deadline fences execution. */
209
+ pendingCleanup: Set<Promise<void>>;
210
+ activeExecutionGenerations: Set<number>;
211
+ stopOperation?: Promise<void>;
212
+ removeOperation?: Promise<void>;
184
213
  /** Frozen run parameters — resume reuses the original values, ignoring new budget/toolset overrides. */
185
214
  frozenOptions: Pick<
186
215
  ScriptStartOptions,
@@ -202,8 +231,18 @@ export interface ScriptRunState {
202
231
  agentWaiters: Map<string, AgentLifecycleWaiter>;
203
232
  /** Owner-validated terminal events emitted before their spawn reply/waiter. */
204
233
  bufferedTerminals: Map<string, BufferedAgentTerminal>;
205
- /** Allocations whose managed-spawn reply has not arrived yet. */
206
- pendingSpawns: Map<string, { spawnKey: string; owner: WorkflowOwner; executionGeneration: number }>;
234
+ /** Local reservations and submitted allocations retained through terminal admission. */
235
+ pendingSpawns: Map<
236
+ string,
237
+ {
238
+ spawnKey: string;
239
+ owner: WorkflowOwner;
240
+ executionGeneration: number;
241
+ phase: "reserved" | "submitted";
242
+ startedAgentId?: string;
243
+ agentId?: string;
244
+ }
245
+ >;
207
246
  /** Journal retry bookkeeping. */
208
247
  pendingJournal: Set<string>;
209
248
  /** Whether a lifecycle suspension is in effect (branch change). */
@@ -309,6 +348,29 @@ function sameWorkflowOwner(left: WorkflowOwner, right: WorkflowOwner): boolean {
309
348
  );
310
349
  }
311
350
 
351
+ /** Optional owned lifecycle usage is per-attempt; legacy `tokens` remains lifetime-scoped. */
352
+ function managedLifecycleUsage(raw: unknown): AgentUsage | undefined {
353
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
354
+ const { input, output, total } = raw as Record<string, unknown>;
355
+ if (
356
+ typeof input !== "number" ||
357
+ typeof output !== "number" ||
358
+ typeof total !== "number" ||
359
+ !Number.isSafeInteger(input) ||
360
+ !Number.isSafeInteger(output) ||
361
+ !Number.isSafeInteger(total) ||
362
+ input < 0 ||
363
+ output < 0 ||
364
+ total < 0 ||
365
+ !Number.isSafeInteger(input + output) ||
366
+ total < input + output
367
+ )
368
+ return undefined;
369
+ // The managed total includes cache writes, but excludes cache reads. The
370
+ // additive lifecycle field has no cost/cache-read detail; do not invent it.
371
+ return { input, output, total, cost: 0, cacheRead: 0, cacheWrite: total - input - output };
372
+ }
373
+
312
374
  export class WorkflowEngine {
313
375
  private readonly runs = new Map<string, ScriptRunState>();
314
376
  /** Fired when a run settles terminal; wired by the UI surface for widget refresh. */
@@ -317,6 +379,10 @@ export class WorkflowEngine {
317
379
  onBackgroundExecution: ((runId: string) => void) | undefined;
318
380
  /** Optional in-process observer for phase/task/quality runtime events. */
319
381
  onRuntimeEvent: ((runId: string, event: WorkflowRuntimeEvent) => void) | undefined;
382
+ /** Additive notification after accepted presentation mutations; consumers re-read getInspection(). */
383
+ onInspectionChanged: ((runId: string) => void) | undefined;
384
+ /** Fresh saved-workflow resolver for control and automatic resumes. */
385
+ loadSavedWorkflow: ((name: string) => string | undefined) | undefined;
320
386
  /**
321
387
  * The user's current strength → Agent tier table, read fresh per execution.
322
388
  *
@@ -352,6 +418,10 @@ export class WorkflowEngine {
352
418
  private journalMutationKey: string | undefined;
353
419
  private journalMutationSequence = 0;
354
420
  private readonly quarantinedRunIds = new Set<string>();
421
+ private readonly cleanupObligations = new WeakMap<ScriptRunState, Map<string, OwnedCleanup>>();
422
+ /** Retired states keep their exact leases; replacement states never inherit them. */
423
+ private readonly retainedCleanupStates = new Set<ScriptRunState>();
424
+ private readonly terminalNotified = new WeakSet<ScriptRunState>();
355
425
 
356
426
  constructor(
357
427
  readonly events: WorkflowEventBus,
@@ -367,6 +437,15 @@ export class WorkflowEngine {
367
437
  this.unsubscribeLifecycle = this.attachLifecycle();
368
438
  }
369
439
 
440
+ private notifyInspection(state: ScriptRunState): void {
441
+ state.presentation.changed();
442
+ try {
443
+ this.onInspectionChanged?.(state.run.runId);
444
+ } catch {
445
+ // Presentation observers must never change execution or durable control.
446
+ }
447
+ }
448
+
370
449
  private async awaitProtocol(): Promise<ManagedProtocolCheck> {
371
450
  if (this.disposed) throw new WorkflowEngineDisposedError();
372
451
  let rejectDisposed: ((reason?: unknown) => void) | undefined;
@@ -390,6 +469,40 @@ export class WorkflowEngine {
390
469
  return this.disposed;
391
470
  }
392
471
 
472
+ /** Pure exact-state cleanup uncertainty query for owner control surfaces. */
473
+ hasPendingCleanup(runId: string): boolean {
474
+ return (this.runs.get(runId)?.pendingCleanup.size ?? 0) > 0;
475
+ }
476
+
477
+ private isCurrentState(state: ScriptRunState, generation: number, epoch: number): boolean {
478
+ return (
479
+ this.runs.get(state.run.runId) === state &&
480
+ state.executionGeneration === generation &&
481
+ this.branchEpoch === epoch &&
482
+ !state.lifecycleSuspended &&
483
+ !this.lifecyclePaused &&
484
+ !this.disposed
485
+ );
486
+ }
487
+
488
+ private notifyTerminal(state: ScriptRunState): void {
489
+ const generation = state.executionGeneration;
490
+ const epoch = this.branchEpoch;
491
+ if (
492
+ !this.isCurrentState(state, generation, epoch) ||
493
+ state.pendingCleanup.size > 0 ||
494
+ this.terminalNotified.has(state)
495
+ )
496
+ return;
497
+ this.terminalNotified.add(state);
498
+ this.settleWaiters(state.run.runId, state.run);
499
+ try {
500
+ this.onRunSettled?.();
501
+ } catch {
502
+ /* terminal observers are presentation-only */
503
+ }
504
+ }
505
+
393
506
  /** A synchronous snapshot for cancellable navigation, not an admission lock. */
394
507
  hasUnsettledWork(): boolean {
395
508
  return (
@@ -397,6 +510,7 @@ export class WorkflowEngine {
397
510
  this.branchQuiescePromise !== undefined ||
398
511
  this.journalRetries.size > 0 ||
399
512
  this.providerResumeTimers.size > 0 ||
513
+ this.retainedCleanupStates.size > 0 ||
400
514
  [...this.runs.values()].some(
401
515
  (state) =>
402
516
  state.run.status === "running" ||
@@ -404,7 +518,11 @@ export class WorkflowEngine {
404
518
  state.run.status === "stopping" ||
405
519
  state.pendingSpawns.size > 0 ||
406
520
  state.agentWaiters.size > 0 ||
407
- state.pendingJournal.size > 0,
521
+ state.pendingJournal.size > 0 ||
522
+ state.pendingCleanup.size > 0 ||
523
+ state.activeExecutionGenerations.size > 0 ||
524
+ state.stopOperation !== undefined ||
525
+ state.removeOperation !== undefined,
408
526
  )
409
527
  );
410
528
  }
@@ -418,6 +536,8 @@ export class WorkflowEngine {
418
536
  this.clearJournalRetries();
419
537
  for (const state of this.runs.values()) {
420
538
  state.lifecycleSuspended = true;
539
+ state.presentation.reset(false);
540
+ this.notifyInspection(state);
421
541
  state.controller.abort(new DOMException("Workflow branch replaced", "AbortError"));
422
542
  }
423
543
  }
@@ -444,7 +564,8 @@ export class WorkflowEngine {
444
564
  eventName !== "subagents:created" &&
445
565
  eventName !== "subagents:started" &&
446
566
  eventName !== "subagents:completed" &&
447
- eventName !== "subagents:failed"
567
+ eventName !== "subagents:failed" &&
568
+ eventName !== "subagents:compacted"
448
569
  )
449
570
  return;
450
571
  if (!raw || typeof raw !== "object") return;
@@ -497,13 +618,55 @@ export class WorkflowEngine {
497
618
  if (currentAttemptId !== undefined && currentAttemptId !== `${owner.runId}/${callNodeId}/${owner.attemptId}`)
498
619
  return;
499
620
 
621
+ const acceptedGeneration = state.executionGeneration;
622
+ const acceptedEpoch = this.branchEpoch;
623
+ const current = () =>
624
+ this.isCurrentState(state, acceptedGeneration, acceptedEpoch) && !state.controller.signal.aborted;
625
+ const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
626
+ const hasManagedUsage = terminalEvent && "managedTokens" in data;
627
+ const managedUsage = hasManagedUsage ? managedLifecycleUsage(data.managedTokens) : undefined;
628
+ // A malformed present field cannot fall back to cumulative lifetime usage.
629
+ // Reject before identity, presentation, buffering, or waiter mutations.
630
+ if (hasManagedUsage && !managedUsage) return;
631
+
632
+ if (eventName === "subagents:compacted") {
633
+ this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
634
+ if (!current()) return;
635
+ if (
636
+ typeof data.compactionCount === "number" &&
637
+ Number.isSafeInteger(data.compactionCount) &&
638
+ data.compactionCount >= 0
639
+ ) {
640
+ const facts = state.managedCallFacts.get(callNodeId);
641
+ if (facts) facts.compactionCount = Math.max(facts.compactionCount, data.compactionCount);
642
+ if (state.run.attemptIds[callNodeId])
643
+ state.run.compactions[callNodeId] = facts?.compactionCount ?? data.compactionCount;
644
+ state.presentation.managed(callNodeId, { compactionCount: facts?.compactionCount ?? data.compactionCount });
645
+ this.notifyInspection(state);
646
+ }
647
+ return;
648
+ }
500
649
  if (updateCallTierIdentity(state, callNodeId, data.tier)) {
501
650
  this.persistCallTierIdentity(state, callNodeId);
502
651
  }
652
+ if (!current()) return;
653
+ this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
654
+ if (!current()) return;
503
655
  // A start is the moment the host admitted this agent from its queue. It is
504
656
  // the only signal that separates waiting for a slot from doing the work, so
505
657
  // the call's wall clock is restarted here rather than at dispatch.
506
- if (eventName === "subagents:started" && waiterMatches && waiter) notifyAgentRunning(waiter);
658
+ if (eventName === "subagents:started") {
659
+ if (pendingMatches && pending) pending.startedAgentId = agentId;
660
+ if (waiterMatches && waiter) notifyAgentRunning(waiter);
661
+ if (!current()) return;
662
+ }
663
+ state.presentation.managed(callNodeId, {
664
+ agentId,
665
+ status: eventName === "subagents:started" ? "running" : undefined,
666
+ ...(isManagedAgentTier(data.tier) ? { tier: data.tier } : {}),
667
+ });
668
+ this.notifyInspection(state);
669
+ if (!current()) return;
507
670
  // Created/started lifecycle facts carry managed identity before the RPC
508
671
  // reply is necessarily available. They are observation-only here; terminal
509
672
  // ownership is handled below once a completion/failure fact arrives.
@@ -516,13 +679,27 @@ export class WorkflowEngine {
516
679
  ? "stopped"
517
680
  : "failed";
518
681
  const tokenCount =
519
- typeof data.tokenCount === "number"
520
- ? data.tokenCount
521
- : data.tokens &&
522
- typeof data.tokens === "object" &&
523
- typeof (data.tokens as Record<string, unknown>).total === "number"
524
- ? ((data.tokens as Record<string, unknown>).total as number)
525
- : undefined;
682
+ managedUsage !== undefined
683
+ ? managedUsage.total
684
+ : typeof data.tokenCount === "number"
685
+ ? data.tokenCount
686
+ : data.tokens &&
687
+ typeof data.tokens === "object" &&
688
+ typeof (data.tokens as Record<string, unknown>).total === "number"
689
+ ? ((data.tokens as Record<string, unknown>).total as number)
690
+ : undefined;
691
+ state.presentation.managed(callNodeId, {
692
+ agentId,
693
+ status: lifecycleStatus,
694
+ tokenCount,
695
+ ...(typeof data.outputFile === "string" ? { outputFile: data.outputFile } : {}),
696
+ compactionCount: Math.max(
697
+ typeof data.compactionCount === "number" ? data.compactionCount : 0,
698
+ state.managedCallFacts.get(callNodeId)?.compactionCount ?? 0,
699
+ ),
700
+ });
701
+ this.notifyInspection(state);
702
+ if (!current()) return;
526
703
  const delivery: AgentTerminalDelivery = {
527
704
  response: {
528
705
  id: agentId,
@@ -536,9 +713,11 @@ export class WorkflowEngine {
536
713
  completedAt: typeof data.completedAt === "number" ? data.completedAt : Date.now(),
537
714
  },
538
715
  },
539
- ...(tokenCount === undefined
540
- ? {}
541
- : { usage: { input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 } }),
716
+ ...(managedUsage !== undefined
717
+ ? { usage: managedUsage }
718
+ : tokenCount === undefined
719
+ ? {}
720
+ : { usage: { input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 } }),
542
721
  ...(Array.isArray(data.history) ? { history: [...data.history] } : {}),
543
722
  };
544
723
 
@@ -555,6 +734,60 @@ export class WorkflowEngine {
555
734
  state.bufferedTerminals.set(spawnKey, { ...delivery, owner, executionGeneration: state.executionGeneration });
556
735
  }
557
736
 
737
+ private recordManagedAdmission(
738
+ state: ScriptRunState,
739
+ nodeId: string,
740
+ agentId: string,
741
+ executionGeneration: number,
742
+ ): void {
743
+ const attemptId = state.run.attemptIds[nodeId];
744
+ if (!attemptId) return; // Nested facts remain volatile, owned by the parent boundary.
745
+ const existing = state.managedCallFacts.get(nodeId);
746
+ if (
747
+ existing?.executionGeneration === executionGeneration &&
748
+ existing.attemptId === attemptId &&
749
+ existing.agentId === agentId
750
+ )
751
+ return;
752
+ state.managedCallFacts.set(nodeId, { executionGeneration, attemptId, agentId, compactionCount: 0 });
753
+ state.run.agentIds[nodeId] = agentId;
754
+ state.run.compactions[nodeId] = 0;
755
+ const event: JournalEvent = {
756
+ kind: "call_transition",
757
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
758
+ runId: state.run.runId,
759
+ nodeId,
760
+ status: "running",
761
+ agentId,
762
+ ...(state.callTiers.get(nodeId)?.tier === undefined ? {} : { tier: state.callTiers.get(nodeId)?.tier }),
763
+ attemptId,
764
+ owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
765
+ timestamp: Date.now(),
766
+ };
767
+ this.persist(state.run.runId, `call-identity:${nodeId}`, () => this.write(event));
768
+ }
769
+
770
+ private recordManagedTerminal(
771
+ state: ScriptRunState,
772
+ nodeId: string,
773
+ agentId: string,
774
+ terminal: NonNullable<ManagedSpawnResponse["terminal"]>,
775
+ executionGeneration: number,
776
+ ): void {
777
+ this.recordManagedAdmission(state, nodeId, agentId, executionGeneration);
778
+ const facts = state.managedCallFacts.get(nodeId);
779
+ if (
780
+ !facts ||
781
+ facts.executionGeneration !== executionGeneration ||
782
+ facts.attemptId !== state.run.attemptIds[nodeId] ||
783
+ facts.agentId !== agentId
784
+ )
785
+ return;
786
+ facts.tokenCount = terminal.tokenCount;
787
+ facts.outputFile = terminal.outputFile;
788
+ facts.compactionCount = Math.max(facts.compactionCount, terminal.compactionCount);
789
+ }
790
+
558
791
  /** Persist a host-normalized tier learned before the managed reply arrives. */
559
792
  private persistCallTierIdentity(state: ScriptRunState, nodeId: string): void {
560
793
  const attemptId = state.run.attemptIds[nodeId];
@@ -562,19 +795,18 @@ export class WorkflowEngine {
562
795
  // Nested workflow frames are in-memory only; only a top-level call has a
563
796
  // durable attempt owner and can safely receive a journal transition.
564
797
  if (!attemptId || identity?.tier === undefined) return;
565
- this.persist(state.run.runId, `call-tier:${nodeId}`, () =>
566
- this.write({
567
- kind: "call_transition",
568
- schemaVersion: JOURNAL_SCHEMA_VERSION,
569
- runId: state.run.runId,
570
- nodeId,
571
- status: "running",
572
- tier: identity.tier,
573
- attemptId,
574
- owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
575
- timestamp: Date.now(),
576
- }),
577
- );
798
+ const event: JournalEvent = {
799
+ kind: "call_transition",
800
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
801
+ runId: state.run.runId,
802
+ nodeId,
803
+ status: "running",
804
+ tier: identity.tier,
805
+ attemptId,
806
+ owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
807
+ timestamp: Date.now(),
808
+ };
809
+ this.persist(state.run.runId, `call-tier:${nodeId}`, () => this.write(event));
578
810
  }
579
811
 
580
812
  private resolveAgentWaiter(
@@ -586,9 +818,23 @@ export class WorkflowEngine {
586
818
  if (waiter.settled) return;
587
819
  waiter.settled = true;
588
820
  state.agentWaiters.delete(agentId);
589
- if (delivery.usage) waiter.onUsage?.(delivery.usage);
590
- if (delivery.history) waiter.onHistory?.(delivery.history);
591
- waiter.resolve(delivery.response);
821
+ const epoch = this.branchEpoch;
822
+ try {
823
+ if (delivery.usage) waiter.onUsage?.(delivery.usage);
824
+ if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
825
+ waiter.reject(new WorkflowWaitAbortedError());
826
+ return;
827
+ }
828
+ if (delivery.history) waiter.onHistory?.(delivery.history);
829
+ if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
830
+ waiter.reject(new WorkflowWaitAbortedError());
831
+ return;
832
+ }
833
+ waiter.resolve(delivery.response);
834
+ } catch (error) {
835
+ // A throwing observer cannot strand a removed waiter and its child lease.
836
+ waiter.reject(error);
837
+ }
592
838
  }
593
839
 
594
840
  dispose(): void {
@@ -597,12 +843,14 @@ export class WorkflowEngine {
597
843
  this.disposeController.abort();
598
844
  this.unsubscribeLifecycle();
599
845
  this.quarantinedRunIds.clear();
600
- for (const state of this.runs.values()) {
846
+ const retiredStates = [...this.runs.values()];
847
+ for (const state of retiredStates) {
601
848
  this.cleanupAgents(state.run.runId);
602
849
  state.controller.abort(new DOMException("Workflow engine disposed", "AbortError"));
603
850
  }
604
851
  this.rejectWaiters(new WorkflowWaitAbortedError());
605
852
  this.runs.clear();
853
+ for (const state of retiredStates) this.notifyInspection(state);
606
854
  this.clearJournalRetries();
607
855
  this.branchQuiescePromise = undefined;
608
856
  }
@@ -621,68 +869,58 @@ export class WorkflowEngine {
621
869
 
622
870
  /** Stop owned agents while Pi replaces the active session-tree branch. */
623
871
  async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
624
- if (this.disposed) return { settled: true, pending: [] };
625
872
  if (this.branchQuiescePromise) return this.branchQuiescePromise;
626
873
  this.admissionsClosed = true;
627
874
  this.clearJournalRetries();
628
875
 
629
- // Suspend and abort synchronously, before any await can let a stale runtime
630
- // dispatch another call or let a provider-limit continuation recreate an old
631
- // branch agent. `restore()` creates fresh states with suspension cleared.
632
- const targets = new Map<string, { agentIds: string[]; owners: WorkflowOwner[] }>();
633
- for (const state of this.runs.values()) {
634
- if (isTerminalWorkflow(state.run.status)) continue;
635
- const agentIds = [...state.agentWaiters.keys()];
636
- const owners = agentIds.map((agentId) => {
637
- const waiter = state.agentWaiters.get(agentId);
638
- return {
639
- extension: "pi-workflows" as const,
876
+ // Snapshot exact states/owners, not run ids or status. Terminal and retired
877
+ // states may still own cleanup, and observation must not donate those
878
+ // leases to a same-id replacement on a newly committed branch.
879
+ const states = [...new Set([...this.runs.values(), ...this.retainedCleanupStates])];
880
+ const targets = states.flatMap((state) => {
881
+ const owners = new Map<string, { agentId: string; owner: WorkflowOwner; generation: number }>();
882
+ for (const pending of state.pendingSpawns.values()) {
883
+ if (pending.phase !== "submitted") continue;
884
+ owners.set(JSON.stringify(pending.owner), {
885
+ agentId: pending.agentId ?? pending.startedAgentId ?? pending.spawnKey,
886
+ owner: { ...pending.owner },
887
+ generation: pending.executionGeneration,
888
+ });
889
+ }
890
+ for (const [agentId, waiter] of state.agentWaiters) {
891
+ const owner: WorkflowOwner = {
892
+ extension: "pi-workflows",
640
893
  runId: state.run.runId,
641
- nodeId: waiter ? `call-${waiter.callIndex}` : "",
642
- attemptId: `attempt-${waiter?.generation ?? 1}`,
894
+ nodeId: `call-${waiter.callIndex}`,
895
+ attemptId: `attempt-${waiter.generation}`,
643
896
  };
644
- });
645
- targets.set(state.run.runId, { agentIds, owners });
646
- state.lifecycleSuspended = true;
647
- state.bufferedTerminals.clear();
648
- state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
649
- }
650
-
651
- const operation = (async () => {
652
- const operations: Array<Promise<WorkflowQuiesceResult>> = [];
653
- for (const state of this.runs.values()) {
654
- const run = state.run;
655
- const target = targets.get(run.runId);
656
- if (isTerminalWorkflow(run.status)) {
657
- this.reconcilePendingSpawns(state);
658
- continue;
659
- }
660
- if (state.pendingSpawns.size > 0) operations.push(this.quiescePendingSpawns(state));
661
- const activeAgentIds = target?.agentIds ?? [];
662
- if (activeAgentIds.length === 0) continue;
663
- operations.push(
664
- this.client
665
- .quiesceOwned?.(run.runId, activeAgentIds, BRANCH_QUIESCE_TIMEOUT_MS, target?.owners)
666
- .then((result) => ({
667
- settled: result.settled && result.pending.length === 0,
668
- pending: [...new Set(result.pending)].slice(0, 256),
669
- }))
670
- .catch((error: unknown) => ({
671
- settled: false,
672
- pending: activeAgentIds.slice(0, 256),
673
- diagnostic: error instanceof Error ? error.message : String(error),
674
- })) ?? Promise.resolve({ settled: false, pending: activeAgentIds.slice(0, 256) }),
675
- );
897
+ owners.set(JSON.stringify(owner), { agentId, owner, generation: waiter.executionGeneration });
676
898
  }
677
- const results = await Promise.all(operations);
678
- const pending = [...new Set(results.flatMap((r) => r.pending))].slice(0, 256);
679
- const diagnostics = results.map((r) => r.diagnostic).filter((d): d is string => Boolean(d));
680
- return {
681
- settled: results.every((r) => r.settled) && pending.length === 0,
682
- pending,
683
- ...(diagnostics.length > 0 ? { diagnostic: diagnostics.join("; ") } : {}),
684
- };
685
- })();
899
+ for (const cleanup of this.cleanupObligations.get(state)?.values() ?? []) {
900
+ owners.set(JSON.stringify(cleanup.owner), {
901
+ agentId: cleanup.agentId,
902
+ owner: { ...cleanup.owner },
903
+ generation: state.executionGeneration,
904
+ });
905
+ }
906
+ return [...owners.values()].map((target) => ({ state, ...target }));
907
+ });
908
+ // Publish the handshake before RPC, abort or inspection can re-enter.
909
+ let acceptOperations!: (operations: Promise<void>[]) => void;
910
+ const operations = new Promise<Promise<void>[]>((resolve) => {
911
+ acceptOperations = resolve;
912
+ });
913
+ const operation = operations.then(async (cleanup) => {
914
+ await Promise.all(cleanup);
915
+ const pending = [
916
+ ...new Set(
917
+ states.flatMap((state) =>
918
+ [...(this.cleanupObligations.get(state)?.values() ?? [])].map((cleanup) => cleanup.agentId),
919
+ ),
920
+ ),
921
+ ].slice(0, 256);
922
+ return { settled: pending.length === 0, pending };
923
+ });
686
924
  const tracked = operation.then(
687
925
  (result) => {
688
926
  if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
@@ -694,6 +932,34 @@ export class WorkflowEngine {
694
932
  },
695
933
  );
696
934
  this.branchQuiescePromise = tracked;
935
+ // Fence every captured state before the first notifying observer. Start
936
+ // known-id owner checks synchronously for the bus-first shutdown handshake;
937
+ // their positive result is consumed by the same stable cleanup obligation,
938
+ // never treated as a status-only shortcut or a separate donated lease.
939
+ for (const state of states) state.lifecycleSuspended = true;
940
+ acceptOperations(
941
+ targets.map(({ state, agentId, owner, generation }) => {
942
+ const existing = [...(this.cleanupObligations.get(state)?.values() ?? [])].find((cleanup) =>
943
+ sameWorkflowOwner(cleanup.owner, owner),
944
+ );
945
+ let quiescence: Promise<{ settled: boolean; pending: string[] }> | undefined;
946
+ if (!existing?.operation && agentId !== `${owner.runId}/${owner.nodeId}/${owner.attemptId}`) {
947
+ try {
948
+ quiescence = this.client.quiesceOwned?.(owner.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner]);
949
+ } catch (error) {
950
+ quiescence = Promise.reject(error);
951
+ }
952
+ void quiescence?.catch(() => {});
953
+ }
954
+ return this.stopAndQuiesce(state, agentId, owner, generation, undefined, quiescence);
955
+ }),
956
+ );
957
+ for (const state of states) {
958
+ state.bufferedTerminals.clear();
959
+ state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
960
+ state.presentation.reset(false);
961
+ this.notifyInspection(state);
962
+ }
697
963
  return tracked;
698
964
  }
699
965
 
@@ -788,8 +1054,14 @@ export class WorkflowEngine {
788
1054
  ...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
789
1055
  timestamp: now,
790
1056
  });
1057
+ if (epoch !== this.branchEpoch || this.disposed || this.lifecyclePaused || this.admissionsClosed)
1058
+ throw new Error("Workflow branch changed before admission");
791
1059
  const state: ScriptRunState = {
792
1060
  run,
1061
+ presentation: new WorkflowPresentation(runId),
1062
+ managedCallFacts: new Map(),
1063
+ pendingCleanup: new Set(),
1064
+ activeExecutionGenerations: new Set(),
793
1065
  frozenOptions: {
794
1066
  ...(frozenTokenBudget !== undefined ? { tokenBudget: frozenTokenBudget } : {}),
795
1067
  ...(frozenMaxAgents !== undefined ? { maxAgents: frozenMaxAgents } : {}),
@@ -810,13 +1082,19 @@ export class WorkflowEngine {
810
1082
  executionGeneration: 0,
811
1083
  };
812
1084
  this.runs.set(runId, state);
813
- this.setWorkflowStatus(run, "running");
814
- state.executionGeneration += 1;
815
- state.execution = this.execute(runId, script, startOptions, undefined, state.executionGeneration, protocol);
1085
+ // Accept identity before any synchronous running/root observer can stop or
1086
+ // replace this state. Dispatch is deferred until its promise is published.
1087
+ const generation = ++state.executionGeneration;
1088
+ state.presentation.reset();
1089
+ state.execution = Promise.resolve().then(() =>
1090
+ this.execute(state, script, startOptions, undefined, generation, epoch, protocol),
1091
+ );
816
1092
  const execution = state.execution;
817
1093
  // A run-fatal abort/stop already journals its terminal state; never let a
818
1094
  // leftover rejection become an unhandled promise rejection.
819
- void state.execution.catch(() => {});
1095
+ void execution.catch(() => {});
1096
+ this.setWorkflowStatus(run, "running");
1097
+ if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
820
1098
  if (options.background) return { runId, status: run.status, background: true };
821
1099
  try {
822
1100
  const settled = await this.waitFor(runId, options.signal);
@@ -867,7 +1145,7 @@ export class WorkflowEngine {
867
1145
  throw new Error("Workflow branch changed before admission");
868
1146
  const state = this.runs.get(runId);
869
1147
  if (!state) return undefined;
870
- if (state.lifecycleSuspended) return undefined;
1148
+ if (state.lifecycleSuspended || state.removeOperation) return undefined;
871
1149
  const run = state.run;
872
1150
  if (run.status === "running" || run.status === "pausing" || run.status === "stopping") return undefined;
873
1151
  const providerTimer = this.providerResumeTimers.get(runId);
@@ -918,6 +1196,7 @@ export class WorkflowEngine {
918
1196
  timestamp,
919
1197
  };
920
1198
  this.appendRequired(recovery);
1199
+ if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
921
1200
  applyRecoveryEvent(run, recovery);
922
1201
  state.result = undefined;
923
1202
  }
@@ -941,6 +1220,7 @@ export class WorkflowEngine {
941
1220
  }),
942
1221
  );
943
1222
  }
1223
+ if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
944
1224
  // Arguments are frozen in every schema-v4 run_created fact. Resume never accepts
945
1225
  // replacement args and never backfills the pre-schema-v4 run_args contract.
946
1226
  const resumeArgs = cloneWorkflowArgs(run.args);
@@ -950,8 +1230,6 @@ export class WorkflowEngine {
950
1230
  for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
951
1231
  state.generations.set(callIndex, generationFromAttemptId(attemptId));
952
1232
  }
953
- this.setWorkflowStatus(run, "running");
954
- state.controller = new AbortController();
955
1233
  // Freeze semantics: original run parameters and args win over new options.
956
1234
  // Only signal/confirm/background/loadSavedWorkflow/mainModel/strengths are
957
1235
  // per-resume context; input/budget/scale/toolset are frozen from the
@@ -963,6 +1241,7 @@ export class WorkflowEngine {
963
1241
  // a caller that passed none is asking the engine to read the current one.
964
1242
  strengths: options.strengths ?? this.strengths?.(),
965
1243
  args: resumeArgs,
1244
+ loadSavedWorkflow: options.loadSavedWorkflow ?? this.loadSavedWorkflow,
966
1245
  tokenBudget: frozen.tokenBudget,
967
1246
  maxAgents: frozen.maxAgents,
968
1247
  concurrency: frozen.concurrency,
@@ -971,18 +1250,21 @@ export class WorkflowEngine {
971
1250
  toolset: frozen.toolset,
972
1251
  excludeTools: frozen.excludeTools ? [...frozen.excludeTools] : undefined,
973
1252
  };
1253
+ if (!this.isCurrentState(state, state.executionGeneration, epoch) || run.nonResumable || state.stopOperation)
1254
+ return undefined;
974
1255
  state.bufferedTerminals.clear();
975
- state.executionGeneration += 1;
976
- state.execution = this.execute(
977
- runId,
978
- run.script,
979
- resumeOptions,
980
- resumeJournal,
981
- state.executionGeneration,
982
- protocol,
1256
+ this.terminalNotified.delete(state);
1257
+ state.controller = new AbortController();
1258
+ const generation = ++state.executionGeneration;
1259
+ const acceptedScript = run.script;
1260
+ state.presentation.reset();
1261
+ state.execution = Promise.resolve().then(() =>
1262
+ this.execute(state, acceptedScript, resumeOptions, resumeJournal, generation, epoch, protocol),
983
1263
  );
984
1264
  const execution = state.execution;
985
1265
  void execution.catch(() => {});
1266
+ this.setWorkflowStatus(run, "running");
1267
+ if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
986
1268
  if (options.background) {
987
1269
  this.onBackgroundExecution?.(runId);
988
1270
  return { runId, status: run.status, background: true };
@@ -1041,11 +1323,7 @@ export class WorkflowEngine {
1041
1323
  await this.stop(state);
1042
1324
  } else if (action === "rm") {
1043
1325
  if (!isTerminalWorkflow(run.status)) throw new Error("cannot remove a running workflow");
1044
- this.persist(runId, "run-removed", () =>
1045
- this.write({ kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp: Date.now() }),
1046
- );
1047
- this.runs.delete(runId);
1048
- this.waiters.delete(runId);
1326
+ await this.remove(state);
1049
1327
  return { action, run: this.summary(run) };
1050
1328
  }
1051
1329
  return { action, run: this.summary(run) };
@@ -1054,6 +1332,7 @@ export class WorkflowEngine {
1054
1332
  async waitFor(runId: string, signal?: AbortSignal): Promise<ScriptRun> {
1055
1333
  const state = this.runs.get(runId);
1056
1334
  if (!state) throw new Error(`workflow run not found: ${runId}`);
1335
+ if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size > 0) throw new WorkflowWaitAbortedError();
1057
1336
  if (isTerminalWorkflow(state.run.status) || state.run.status === "paused" || state.run.status === "interrupted")
1058
1337
  return state.run;
1059
1338
  if (signal?.aborted) throw new WorkflowWaitAbortedError();
@@ -1102,6 +1381,7 @@ export class WorkflowEngine {
1102
1381
  }
1103
1382
 
1104
1383
  private settleWaiters(runId: string, run: ScriptRun): void {
1384
+ if (this.runs.get(runId)?.run !== run || this.hasPendingCleanup(runId)) return;
1105
1385
  const all = this.waiters.get(runId);
1106
1386
  if (!all) return;
1107
1387
  for (const waiter of [...all]) {
@@ -1117,24 +1397,24 @@ export class WorkflowEngine {
1117
1397
  // ── execution ──────────────────────────────────────────────────────────────
1118
1398
 
1119
1399
  private async execute(
1120
- runId: string,
1400
+ state: ScriptRunState,
1121
1401
  script: string,
1122
1402
  options: ScriptStartOptions,
1123
- resumeJournal?: Map<string, JournalEntry>,
1124
- executionGeneration?: number,
1403
+ resumeJournal: Map<string, JournalEntry> | undefined,
1404
+ generation: number,
1405
+ epoch: number,
1125
1406
  protocol?: ManagedProtocolCheck,
1126
1407
  ): Promise<WorkflowRunResult<unknown>> {
1127
- if (this.disposed) throw new WorkflowEngineDisposedError();
1128
- const state = this.runs.get(runId);
1129
- if (!state) throw new Error(`workflow run not found: ${runId}`);
1130
- const generation = executionGeneration ?? state.executionGeneration;
1131
- const epoch = this.branchEpoch;
1408
+ const runId = state.run.runId;
1409
+ state.activeExecutionGenerations.add(generation);
1132
1410
  const isSameBranch = () =>
1133
1411
  this.runs.get(runId) === state &&
1134
1412
  state.executionGeneration === generation &&
1135
1413
  this.branchEpoch === epoch &&
1136
1414
  !this.disposed;
1137
1415
  const isCurrent = () => isSameBranch() && !state.lifecycleSuspended && !this.lifecyclePaused;
1416
+ const isPresentationCurrent = () => isCurrent() && state.activeExecutionGenerations.has(generation);
1417
+ const explicitlyStopped = () => state.stopOperation !== undefined || state.run.terminalIntent === "stop";
1138
1418
  const executionSignal =
1139
1419
  !options.background && options.signal
1140
1420
  ? AbortSignal.any([state.controller.signal, options.signal])
@@ -1143,7 +1423,8 @@ export class WorkflowEngine {
1143
1423
  const cleanupTasks = new Set<Promise<void>>();
1144
1424
  const runner: WorkflowAgentRunner = {
1145
1425
  run: async (prompt: string, runOptions: AgentRunOptions = {}) => {
1146
- if (!isCurrent()) throw new WorkflowError("stale workflow execution", WorkflowErrorCode.WORKFLOW_ABORTED);
1426
+ if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
1427
+ throw new WorkflowError("stale workflow execution", WorkflowErrorCode.WORKFLOW_ABORTED);
1147
1428
  const dispatch = this.dispatchAgent(runId, prompt, runOptions);
1148
1429
  const settled = dispatch.then(
1149
1430
  () => {},
@@ -1157,6 +1438,15 @@ export class WorkflowEngine {
1157
1438
  if (runOptions.signal?.aborted) trackCleanup();
1158
1439
  try {
1159
1440
  return await dispatch;
1441
+ } catch (error: unknown) {
1442
+ if (isCurrent()) {
1443
+ const frameId = runOptions.runId ?? runId;
1444
+ const nodeId =
1445
+ frameId === runId ? String(runOptions.callIndex ?? 0) : `${frameId}:${runOptions.callIndex ?? 0}`;
1446
+ state.presentation.managed(nodeId, { status: runOptions.signal?.aborted ? "stopped" : "failed" });
1447
+ this.notifyInspection(state);
1448
+ }
1449
+ throw error;
1160
1450
  } finally {
1161
1451
  runOptions.signal?.removeEventListener("abort", trackCleanup);
1162
1452
  }
@@ -1165,6 +1455,8 @@ export class WorkflowEngine {
1165
1455
  };
1166
1456
 
1167
1457
  try {
1458
+ if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
1459
+ throw new WorkflowError("workflow execution aborted before dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1168
1460
  const result = await runWorkflow(script, {
1169
1461
  cwd: this.cwd,
1170
1462
  args: options.args,
@@ -1189,29 +1481,74 @@ export class WorkflowEngine {
1189
1481
  runId,
1190
1482
  executionNonce: `${generation}-${randomUUID()}`,
1191
1483
  resumeJournal,
1484
+ resumeWorkflowGenerations: new Map(
1485
+ Object.entries(state.run.workflowResultGenerations).map(([index, generation]) => [Number(index), generation]),
1486
+ ),
1192
1487
  toolset: options.toolset,
1193
1488
  strengths: options.strengths,
1194
1489
  excludeTools: options.excludeTools,
1195
- confirm: options.confirm,
1490
+ confirm:
1491
+ options.confirm &&
1492
+ (async (prompt, checkpointOptions, signal) => {
1493
+ if (!isCurrent() || executionSignal.aborted || signal?.aborted || state.stopOperation)
1494
+ throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
1495
+ const value = await options.confirm?.(prompt, checkpointOptions, signal);
1496
+ if (!isCurrent() || executionSignal.aborted || signal?.aborted)
1497
+ throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
1498
+ return value;
1499
+ }),
1196
1500
  loadSavedWorkflow: options.loadSavedWorkflow,
1501
+ onAgentStart: (event) => {
1502
+ if (!isPresentationCurrent()) return;
1503
+ state.presentation.agentStart(event);
1504
+ this.notifyInspection(state);
1505
+ },
1506
+ onAgentEnd: (event) => {
1507
+ if (!isPresentationCurrent()) return;
1508
+ const nodeId = event.id.startsWith(`${runId}:`) ? event.id.slice(runId.length + 1) : undefined;
1509
+ const persisted = nodeId === undefined ? undefined : state.run.callResults[nodeId];
1510
+ const persistedTokens =
1511
+ nodeId !== undefined && persisted?.attemptId === state.run.attemptIds[nodeId]
1512
+ ? persisted?.tokenCount
1513
+ : undefined;
1514
+ // Runtime replay reports zero new spend, not the historical call's
1515
+ // usage. Explicit managed zero is also authoritative over estimates.
1516
+ const tokenCount = event.tokenUsage?.total ?? persistedTokens ?? event.tokens;
1517
+ state.presentation.agentEnd({ ...event, tokens: tokenCount });
1518
+ this.notifyInspection(state);
1519
+ },
1520
+ onLog: (message, scope) => {
1521
+ if (!isPresentationCurrent()) return;
1522
+ state.presentation.log(message, scope?.frameId);
1523
+ this.notifyInspection(state);
1524
+ },
1197
1525
  onAgentJournal: (entry) => {
1198
1526
  if (!isCurrent()) return;
1199
- // Nested workflow frames keep their own lifecycle visibility, but their
1200
- // individual call facts are not top-level replay keys. The parent
1201
- // workflow_result fact below owns the nested replay boundary.
1527
+ // Persist the accepted owner before presentation observers can replace it.
1528
+ // Nested frames are owned by their parent's workflow_result boundary.
1202
1529
  if (entry.runId === undefined || entry.runId === runId) this.journalCallResult(runId, entry);
1530
+ if (!isCurrent()) return;
1531
+ state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
1532
+ this.notifyInspection(state);
1203
1533
  },
1204
1534
  onWorkflowJournal: (entry) => {
1205
1535
  if (!isCurrent()) return;
1206
1536
  this.journalWorkflowResult(runId, entry);
1537
+ if (!isCurrent()) return;
1538
+ state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
1539
+ this.notifyInspection(state);
1207
1540
  },
1208
1541
  onRuntimeEvent: (event) => {
1209
- if (!isCurrent()) return;
1542
+ if (!isPresentationCurrent()) return;
1543
+ state.presentation.runtime(event);
1544
+ this.notifyInspection(state);
1545
+ if (!isPresentationCurrent()) return;
1210
1546
  try {
1211
1547
  this.events.emit("pi-workflows:runtime", { runId, event: cloneRuntimeEvent(event) });
1212
1548
  } catch {
1213
1549
  // Cross-extension observers cannot change workflow execution.
1214
1550
  }
1551
+ if (!isPresentationCurrent()) return;
1215
1552
  try {
1216
1553
  this.onRuntimeEvent?.(runId, cloneRuntimeEvent(event));
1217
1554
  } catch {
@@ -1228,6 +1565,16 @@ export class WorkflowEngine {
1228
1565
  // A terminal frame cannot overtake accepted journal facts. In particular,
1229
1566
  // a delayed call_attempt must precede its result, not reset it on replay.
1230
1567
  await this.drainJournal(runId);
1568
+ if (
1569
+ executionSignal.aborted ||
1570
+ !isCurrent() ||
1571
+ state.run.status === "stopping" ||
1572
+ state.stopOperation !== undefined ||
1573
+ state.run.terminalIntent === "stop"
1574
+ )
1575
+ throw new WorkflowError("workflow execution aborted", WorkflowErrorCode.WORKFLOW_ABORTED, {
1576
+ recoverable: true,
1577
+ });
1231
1578
  const live = this.runs.get(runId);
1232
1579
  const run = live?.run;
1233
1580
  if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
@@ -1238,7 +1585,7 @@ export class WorkflowEngine {
1238
1585
  } catch (error: unknown) {
1239
1586
  await runner.waitForCleanup?.();
1240
1587
  const run = this.runs.get(runId)?.run;
1241
- if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
1588
+ if (run && isSameBranch() && !isTerminalWorkflow(run.status) && run.status !== "stopping") {
1242
1589
  const message = describeWorkflowError(error);
1243
1590
  const workflowError = error instanceof WorkflowError ? error : wrapError(error);
1244
1591
  if (workflowError.code === WorkflowErrorCode.PROVIDER_USAGE_LIMIT) {
@@ -1261,15 +1608,22 @@ export class WorkflowEngine {
1261
1608
  run.status = "interrupted";
1262
1609
  run.updatedAt = Date.now();
1263
1610
  } else {
1264
- this.setWorkflowStatus(run, "failed", message);
1265
1611
  this.cleanupAgents(runId);
1612
+ this.setWorkflowStatus(run, "failed", message);
1266
1613
  }
1267
1614
  }
1268
1615
  throw error;
1269
1616
  } finally {
1617
+ state.activeExecutionGenerations.delete(generation);
1270
1618
  // Reject any waiters that never settled (agents left in flight).
1271
1619
  const run = this.runs.get(runId)?.run;
1272
- if (run && isSameBranch()) this.settleWaiters(runId, run);
1620
+ if (run && isSameBranch()) {
1621
+ if (isCurrent()) {
1622
+ state.presentation.settle(run.status);
1623
+ this.notifyInspection(state);
1624
+ }
1625
+ if (isSameBranch() && run.status !== "stopping") this.settleWaiters(runId, run);
1626
+ }
1273
1627
  }
1274
1628
  }
1275
1629
 
@@ -1326,6 +1680,7 @@ export class WorkflowEngine {
1326
1680
  const tierIdentity: CallTierIdentity = tier === undefined ? {} : { tier };
1327
1681
  state.callTiers.set(nodeId, tierIdentity);
1328
1682
  run.callTiers[nodeId] = tierIdentity;
1683
+ state.managedCallFacts.delete(nodeId);
1329
1684
  const task: DispatchTask = {
1330
1685
  subagent_type: type,
1331
1686
  prompt,
@@ -1341,6 +1696,15 @@ export class WorkflowEngine {
1341
1696
  // represented by its parent workflow_result replay boundary instead of
1342
1697
  // leaking child call facts into the parent's numeric call namespace.
1343
1698
  if (!nested) {
1699
+ // Match journal replay's call_attempt semantics: historical results stay
1700
+ // durable, but are not current facts for this accepted live rotation.
1701
+ run.callStatus[nodeId] = "running";
1702
+ run.attempts[nodeId] = generation;
1703
+ run.attemptIds[nodeId] = attemptId;
1704
+ delete run.agentIds[nodeId];
1705
+ delete run.callResults[nodeId];
1706
+ run.compactions[nodeId] = 0;
1707
+ run.updatedAt = Date.now();
1344
1708
  this.persist(runId, `call-attempt:${nodeId}`, () =>
1345
1709
  this.write({
1346
1710
  kind: "call_attempt",
@@ -1354,10 +1718,12 @@ export class WorkflowEngine {
1354
1718
  timestamp: Date.now(),
1355
1719
  }),
1356
1720
  );
1357
- run.callStatus[nodeId] = "running";
1358
- run.attempts[nodeId] = generation;
1359
- run.attemptIds[nodeId] = attemptId;
1360
- run.updatedAt = Date.now();
1721
+ if (
1722
+ !this.isCurrentState(state, executionGeneration, dispatchEpoch) ||
1723
+ runOptions.signal?.aborted ||
1724
+ state.controller.signal.aborted
1725
+ )
1726
+ throw new WorkflowError("stale workflow attempt", WorkflowErrorCode.WORKFLOW_ABORTED);
1361
1727
  }
1362
1728
 
1363
1729
  // Abort linkage: an external signal (pause/stop/Esc) cancels the wait and
@@ -1371,241 +1737,362 @@ export class WorkflowEngine {
1371
1737
  nodeId: spawnNodeId,
1372
1738
  attemptId: spawnAttemptId,
1373
1739
  };
1374
- state.pendingSpawns.set(spawnKey, { spawnKey, owner, executionGeneration });
1375
- const reconcilePending = () => {
1376
- state.bufferedTerminals.delete(spawnKey);
1377
- const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
1378
- void reconciliation?.catch(() => {});
1740
+ const pending = {
1741
+ spawnKey,
1742
+ owner,
1743
+ executionGeneration,
1744
+ phase: "reserved" as "reserved" | "submitted",
1745
+ startedAgentId: undefined as string | undefined,
1746
+ agentId: undefined as string | undefined,
1379
1747
  };
1380
- const onPendingAbort = () => reconcilePending();
1381
- externalSignal.addEventListener("abort", onPendingAbort, { once: true });
1382
- if (externalSignal.aborted) onPendingAbort();
1383
-
1384
- let pendingId = "";
1385
- let response: ManagedSpawnResponse | string;
1748
+ // Keep this exact allocation lease through every publication callback and
1749
+ // terminal delivery. A waiter alone is insufficient: it is removed before
1750
+ // onUsage/onHistory, which may synchronously pause or replace the branch.
1751
+ state.pendingSpawns.set(spawnKey, pending);
1752
+ let responseId: string | undefined;
1753
+ let terminalAccepted = false;
1754
+ let ownedCleanup: Promise<void> | undefined;
1755
+ const cleanupResponse = () =>
1756
+ (ownedCleanup ??= this.stopAndQuiesce(
1757
+ state,
1758
+ responseId ?? pending.startedAgentId ?? spawnKey,
1759
+ owner,
1760
+ executionGeneration,
1761
+ ));
1386
1762
  try {
1387
- response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, externalSignal);
1388
- } catch (error: unknown) {
1389
- if (externalSignal.aborted) {
1390
- const reconciled = await this.client.reconcileManaged?.(spawnKey, owner).catch(() => undefined);
1391
- const id = typeof reconciled === "string" ? reconciled : reconciled?.id;
1392
- await this.stopAndQuiesce(
1393
- state,
1394
- id ?? spawnKey,
1395
- owner,
1396
- executionGeneration,
1397
- id ? undefined : "cancelled spawn allocation could not be confirmed",
1398
- );
1763
+ state.presentation.dispatch(runOptions, attemptId);
1764
+ this.notifyInspection(state);
1765
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1766
+ throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1767
+ const reconcilePending = () => {
1768
+ state.bufferedTerminals.delete(spawnKey);
1769
+ const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
1770
+ void reconciliation?.catch(() => {});
1771
+ };
1772
+ const onPendingAbort = () => reconcilePending();
1773
+ externalSignal.addEventListener("abort", onPendingAbort, { once: true });
1774
+ if (externalSignal.aborted) onPendingAbort();
1775
+
1776
+ let pendingId = "";
1777
+ let response: ManagedSpawnResponse | string;
1778
+ try {
1779
+ // Reservation is local until this exact boundary. Observers above can
1780
+ // cancel without creating uncertainty for an RPC that was never sent.
1781
+ pending.phase = "submitted";
1782
+ response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, externalSignal);
1783
+ } catch (error: unknown) {
1784
+ // Transport failure is not evidence that the host never allocated. Keep
1785
+ // the exact spawn owner even without a reply or an aborted signal.
1786
+ await cleanupResponse();
1787
+ throw error;
1788
+ } finally {
1789
+ externalSignal.removeEventListener("abort", onPendingAbort);
1399
1790
  }
1400
- throw error;
1401
- } finally {
1402
- externalSignal.removeEventListener("abort", onPendingAbort);
1403
- state.pendingSpawns.delete(spawnKey);
1404
- }
1405
- const responseId = typeof response === "string" ? response : response.id;
1406
- if (
1407
- state.executionGeneration !== executionGeneration ||
1408
- this.runs.get(runId) !== state ||
1409
- this.branchEpoch !== dispatchEpoch ||
1410
- state.lifecycleSuspended ||
1411
- this.lifecyclePaused ||
1412
- state.controller.signal.aborted ||
1413
- externalSignal.aborted
1414
- ) {
1415
- state.bufferedTerminals.delete(spawnKey);
1416
- await this.stopAndQuiesce(state, responseId, owner, executionGeneration);
1417
- throw new WorkflowError(
1418
- "workflow dispatch became stale during pause/resume",
1419
- WorkflowErrorCode.WORKFLOW_ABORTED,
1420
- { recoverable: true },
1421
- );
1422
- }
1423
- if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
1424
- this.persistCallTierIdentity(state, nodeId);
1425
- }
1426
- // Report the tier the host actually selected back to the runtime, which
1427
- // journals it: a call that named none still resolved to something, and the
1428
- // run's own record of what ran should say what.
1429
- const resolvedTierIdentity = state.callTiers.get(nodeId);
1430
- if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
1431
- if (typeof response === "string") {
1432
- pendingId = response;
1433
- } else if (response.terminal) {
1434
- state.bufferedTerminals.delete(spawnKey);
1435
- // The spawn response already carries a terminal snapshot (fast completion
1436
- // or interrupted replay). The runtime journals successful calls itself
1437
- // (onAgentJournal); a terminal snapshot that is NOT a success is
1438
- // journaled here so the resume journal reflects the failure.
1439
- const terminal = response.terminal;
1440
- if (typeof terminal.tokenCount === "number")
1441
- runOptions.onUsage?.({
1442
- input: 0,
1443
- output: terminal.tokenCount,
1444
- total: terminal.tokenCount,
1445
- cost: 0,
1446
- cacheRead: 0,
1447
- cacheWrite: 0,
1448
- });
1449
- const status = managedTerminalStatus(terminal);
1450
- if (status !== "completed") {
1451
- const result = resultFromLifecycle(
1452
- status,
1453
- response.id,
1454
- terminal.result,
1455
- terminal.error,
1456
- terminal.compactionCount,
1457
- terminal.tokenCount,
1458
- resolvedTierIdentity?.tier ?? tier,
1791
+ responseId = typeof response === "string" ? response : response.id;
1792
+ pending.agentId = responseId;
1793
+ if (
1794
+ state.executionGeneration !== executionGeneration ||
1795
+ this.runs.get(runId) !== state ||
1796
+ this.branchEpoch !== dispatchEpoch ||
1797
+ state.lifecycleSuspended ||
1798
+ this.lifecyclePaused ||
1799
+ state.controller.signal.aborted ||
1800
+ externalSignal.aborted
1801
+ ) {
1802
+ state.bufferedTerminals.delete(spawnKey);
1803
+ await cleanupResponse();
1804
+ throw new WorkflowError(
1805
+ "workflow dispatch became stale during pause/resume",
1806
+ WorkflowErrorCode.WORKFLOW_ABORTED,
1807
+ { recoverable: true },
1459
1808
  );
1460
- result.attemptId = attemptId;
1461
- if (!nested) this.commitCallTerminal(runId, nodeId, result);
1462
- if (status === "failed" && terminal.error) {
1463
- throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
1464
- }
1465
1809
  }
1466
- return terminal.result ?? "";
1467
- } else {
1468
- pendingId = response.id;
1469
- }
1470
-
1471
- const waitAbort = () => {
1472
- const waiter = state.agentWaiters.get(pendingId);
1473
- if (waiter && !waiter.settled) {
1474
- waiter.settled = true;
1475
- state.agentWaiters.delete(pendingId);
1476
- state.bufferedTerminals.delete(spawnKey);
1477
- // Stop acknowledges cancellation, not provider/tool finally completion.
1478
- void this.stopAndQuiesce(state, pendingId, owner, executionGeneration).then(async () => {
1479
- if (
1480
- !nested &&
1481
- this.runs.get(runId) === state &&
1482
- state.executionGeneration === executionGeneration &&
1483
- !run.nonResumable
1484
- ) {
1485
- const result = resultFromLifecycle(
1486
- "stopped",
1487
- pendingId,
1488
- undefined,
1489
- "workflow call cancelled",
1490
- run.compactions[nodeId] ?? 0,
1491
- undefined,
1492
- resolvedTierIdentity?.tier ?? tier,
1493
- );
1494
- result.attemptId = attemptId;
1495
- try {
1496
- await this.drainJournal(runId);
1497
- this.commitCallTerminal(runId, nodeId, result, true);
1498
- } catch (error) {
1499
- this.fenceExecution(
1500
- state,
1501
- executionGeneration,
1502
- "Could not persist cancelled call terminal fact; run is non-resumable",
1503
- );
1504
- waiter.reject(error);
1505
- return;
1810
+ if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
1811
+ this.persistCallTierIdentity(state, nodeId);
1812
+ }
1813
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1814
+ throw new WorkflowError("stale workflow identity", WorkflowErrorCode.WORKFLOW_ABORTED);
1815
+ this.recordManagedAdmission(state, nodeId, responseId, executionGeneration);
1816
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1817
+ throw new WorkflowError("stale workflow admission", WorkflowErrorCode.WORKFLOW_ABORTED);
1818
+ if (typeof response !== "string" && response.terminal)
1819
+ this.recordManagedTerminal(state, nodeId, responseId, response.terminal, executionGeneration);
1820
+ const bufferedTerminal = state.bufferedTerminals.get(spawnKey);
1821
+ const observedTerminal =
1822
+ bufferedTerminal?.response.id === responseId &&
1823
+ bufferedTerminal.executionGeneration === executionGeneration &&
1824
+ sameWorkflowOwner(bufferedTerminal.owner, owner)
1825
+ ? bufferedTerminal.response.terminal
1826
+ : undefined;
1827
+ const observedRunning = pending.startedAgentId === responseId;
1828
+ state.presentation.managed(nodeId, {
1829
+ agentId: responseId,
1830
+ status: observedTerminal
1831
+ ? managedTerminalStatus(observedTerminal)
1832
+ : observedRunning
1833
+ ? "running"
1834
+ : typeof response === "string"
1835
+ ? "pending"
1836
+ : response.state === "queued"
1837
+ ? "queued"
1838
+ : "running",
1839
+ tier: state.callTiers.get(nodeId)?.tier,
1840
+ ...(typeof response !== "string" && response.terminal
1841
+ ? {
1842
+ status: managedTerminalStatus(response.terminal),
1843
+ tokenCount: response.terminal.tokenCount,
1844
+ outputFile: response.terminal.outputFile,
1845
+ compactionCount: Math.max(
1846
+ response.terminal.compactionCount,
1847
+ state.managedCallFacts.get(nodeId)?.compactionCount ?? 0,
1848
+ ),
1506
1849
  }
1507
- }
1508
- waiter.reject(
1509
- new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }),
1850
+ : {}),
1851
+ });
1852
+ this.notifyInspection(state);
1853
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1854
+ throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1855
+ // Report the tier the host actually selected back to the runtime, which
1856
+ // journals it: a call that named none still resolved to something, and the
1857
+ // run's own record of what ran should say what.
1858
+ const resolvedTierIdentity = state.callTiers.get(nodeId);
1859
+ if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
1860
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1861
+ throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1862
+ if (typeof response === "string") {
1863
+ pendingId = response;
1864
+ } else if (response.terminal) {
1865
+ state.bufferedTerminals.delete(spawnKey);
1866
+ // The spawn response already carries a terminal snapshot (fast completion
1867
+ // or interrupted replay). The runtime journals successful calls itself
1868
+ // (onAgentJournal); a terminal snapshot that is NOT a success is
1869
+ // journaled here so the resume journal reflects the failure.
1870
+ const terminal = response.terminal;
1871
+ if (typeof terminal.tokenCount === "number")
1872
+ runOptions.onUsage?.({
1873
+ input: 0,
1874
+ output: terminal.tokenCount,
1875
+ total: terminal.tokenCount,
1876
+ cost: 0,
1877
+ cacheRead: 0,
1878
+ cacheWrite: 0,
1879
+ });
1880
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1881
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
1882
+ terminalAccepted = true;
1883
+ const status = managedTerminalStatus(terminal);
1884
+ if (status !== "completed") {
1885
+ const result = resultFromLifecycle(
1886
+ status,
1887
+ response.id,
1888
+ terminal.result,
1889
+ terminal.error,
1890
+ terminal.compactionCount,
1891
+ terminal.tokenCount,
1892
+ resolvedTierIdentity?.tier ?? tier,
1510
1893
  );
1511
- });
1894
+ result.attemptId = attemptId;
1895
+ if (!nested) this.commitCallTerminal(runId, nodeId, result);
1896
+ if (status === "failed" && terminal.error) {
1897
+ throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
1898
+ }
1899
+ }
1900
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1901
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
1902
+ return terminal.result ?? "";
1903
+ } else {
1904
+ pendingId = response.id;
1512
1905
  }
1513
- };
1514
1906
 
1515
- // Wait for the terminal lifecycle event, consuming a matching terminal that
1516
- // arrived while the managed-spawn reply was still pending.
1517
- const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
1518
- const waiter: AgentLifecycleWaiter = {
1519
- callIndex: nodeId,
1520
- generation,
1521
- executionGeneration,
1522
- onUsage: runOptions.onUsage,
1523
- onHistory: runOptions.onHistory,
1524
- onRunning: runOptions.onRunning,
1525
- startedNotified: false,
1526
- settled: false,
1527
- resolve,
1528
- reject,
1907
+ const waitAbort = () => {
1908
+ const waiter = state.agentWaiters.get(pendingId);
1909
+ if (waiter && !waiter.settled) {
1910
+ waiter.settled = true;
1911
+ state.agentWaiters.delete(pendingId);
1912
+ state.bufferedTerminals.delete(spawnKey);
1913
+ // Stop acknowledges cancellation, not provider/tool finally completion.
1914
+ void cleanupResponse().then(async () => {
1915
+ if (
1916
+ !nested &&
1917
+ this.runs.get(runId) === state &&
1918
+ state.executionGeneration === executionGeneration &&
1919
+ !run.nonResumable
1920
+ ) {
1921
+ const result = resultFromLifecycle(
1922
+ "stopped",
1923
+ pendingId,
1924
+ undefined,
1925
+ "workflow call cancelled",
1926
+ run.compactions[nodeId] ?? 0,
1927
+ undefined,
1928
+ resolvedTierIdentity?.tier ?? tier,
1929
+ );
1930
+ result.attemptId = attemptId;
1931
+ try {
1932
+ await this.drainJournal(runId);
1933
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch)) {
1934
+ waiter.reject(new WorkflowWaitAbortedError());
1935
+ return;
1936
+ }
1937
+ this.commitCallTerminal(runId, nodeId, result, true);
1938
+ } catch (error) {
1939
+ this.fenceExecution(
1940
+ state,
1941
+ executionGeneration,
1942
+ "Could not persist cancelled call terminal fact; run is non-resumable",
1943
+ );
1944
+ waiter.reject(error);
1945
+ return;
1946
+ }
1947
+ }
1948
+ waiter.reject(
1949
+ new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }),
1950
+ );
1951
+ });
1952
+ }
1529
1953
  };
1530
- state.agentWaiters.set(pendingId, waiter);
1531
- // A spawn that was admitted straight away is already running, and its
1532
- // `subagents:started` event fired before this waiter existed. Only a
1533
- // reply that says "queued" still has a start to wait for.
1534
- if (typeof response !== "string" && response.state !== "queued") notifyAgentRunning(waiter);
1535
- externalSignal.addEventListener("abort", waitAbort, { once: true });
1536
- if (externalSignal.aborted) {
1537
- waitAbort();
1538
- return;
1539
- }
1540
1954
 
1541
- const buffered = state.bufferedTerminals.get(spawnKey);
1542
- if (!buffered) return;
1543
- state.bufferedTerminals.delete(spawnKey);
1955
+ // Wait for the terminal lifecycle event, consuming a matching terminal that
1956
+ // arrived while the managed-spawn reply was still pending.
1957
+ const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
1958
+ const waiter: AgentLifecycleWaiter = {
1959
+ callIndex: nodeId,
1960
+ generation,
1961
+ executionGeneration,
1962
+ onUsage: runOptions.onUsage,
1963
+ onHistory: runOptions.onHistory,
1964
+ onRunning: runOptions.onRunning,
1965
+ startedNotified: false,
1966
+ settled: false,
1967
+ resolve,
1968
+ reject,
1969
+ };
1970
+ state.agentWaiters.set(pendingId, waiter);
1971
+ // A spawn that was admitted straight away is already running, and its
1972
+ // `subagents:started` event fired before this waiter existed. Only a
1973
+ // reply that says "queued" still has a start to wait for.
1974
+ if (observedRunning || (typeof response !== "string" && response.state !== "queued"))
1975
+ notifyAgentRunning(waiter);
1976
+ externalSignal.addEventListener("abort", waitAbort, { once: true });
1977
+ if (externalSignal.aborted) {
1978
+ waitAbort();
1979
+ return;
1980
+ }
1981
+
1982
+ const buffered = state.bufferedTerminals.get(spawnKey);
1983
+ if (!buffered) return;
1984
+ state.bufferedTerminals.delete(spawnKey);
1985
+ if (
1986
+ buffered.executionGeneration !== executionGeneration ||
1987
+ state.executionGeneration !== executionGeneration ||
1988
+ state.lifecycleSuspended ||
1989
+ state.controller.signal.aborted ||
1990
+ buffered.response.id !== pendingId ||
1991
+ !sameWorkflowOwner(buffered.owner, owner)
1992
+ )
1993
+ return;
1994
+ this.resolveAgentWaiter(state, pendingId, waiter, buffered);
1995
+ }).finally(() => {
1996
+ externalSignal.removeEventListener("abort", waitAbort);
1997
+ });
1998
+
1544
1999
  if (
1545
- buffered.executionGeneration !== executionGeneration ||
1546
- state.executionGeneration !== executionGeneration ||
2000
+ this.runs.get(runId) !== state ||
2001
+ this.branchEpoch !== dispatchEpoch ||
1547
2002
  state.lifecycleSuspended ||
1548
- state.controller.signal.aborted ||
1549
- buffered.response.id !== pendingId ||
1550
- !sameWorkflowOwner(buffered.owner, owner)
1551
- )
1552
- return;
1553
- this.resolveAgentWaiter(state, pendingId, waiter, buffered);
1554
- }).finally(() => {
1555
- externalSignal.removeEventListener("abort", waitAbort);
1556
- });
1557
-
1558
- if (
1559
- this.runs.get(runId) !== state ||
1560
- this.branchEpoch !== dispatchEpoch ||
1561
- state.lifecycleSuspended ||
1562
- this.lifecyclePaused
1563
- ) {
1564
- throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
1565
- }
1566
- const snapshot = terminal.terminal;
1567
- if (!snapshot)
1568
- throw new WorkflowError("subagent settled without a terminal snapshot", WorkflowErrorCode.AGENT_EXECUTION_ERROR, {
1569
- recoverable: true,
2003
+ this.lifecyclePaused
2004
+ ) {
2005
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
2006
+ }
2007
+ const snapshot = terminal.terminal;
2008
+ if (!snapshot)
2009
+ throw new WorkflowError(
2010
+ "subagent settled without a terminal snapshot",
2011
+ WorkflowErrorCode.AGENT_EXECUTION_ERROR,
2012
+ {
2013
+ recoverable: true,
2014
+ },
2015
+ );
2016
+ this.recordManagedTerminal(state, nodeId, terminal.id, snapshot, executionGeneration);
2017
+ state.presentation.managed(nodeId, {
2018
+ agentId: terminal.id,
2019
+ status: managedTerminalStatus(snapshot),
2020
+ tokenCount: snapshot.tokenCount,
2021
+ outputFile: snapshot.outputFile,
2022
+ compactionCount: Math.max(snapshot.compactionCount, state.managedCallFacts.get(nodeId)?.compactionCount ?? 0),
1570
2023
  });
1571
- const status = managedTerminalStatus(snapshot);
1572
- // Successful calls are journaled by the runtime's onAgentJournal (the
1573
- // single writer for completed results); only failures settle here so the
1574
- // resume journal reflects them.
1575
- if (status !== "completed") {
1576
- const result = resultFromLifecycle(
1577
- status,
1578
- terminal.id,
1579
- snapshot.result,
1580
- snapshot.error,
1581
- snapshot.compactionCount,
1582
- snapshot.tokenCount,
1583
- state.callTiers.get(nodeId)?.tier ?? tier,
1584
- );
1585
- result.attemptId = attemptId;
1586
- if (!nested) this.commitCallTerminal(runId, nodeId, result);
1587
- if (status === "failed" && snapshot.error) {
1588
- throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
2024
+ this.notifyInspection(state);
2025
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch))
2026
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
2027
+ if (externalSignal.aborted) throw new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED);
2028
+ terminalAccepted = true;
2029
+ const status = managedTerminalStatus(snapshot);
2030
+ // Successful calls are journaled by the runtime's onAgentJournal (the
2031
+ // single writer for completed results); only failures settle here so the
2032
+ // resume journal reflects them.
2033
+ if (status !== "completed") {
2034
+ const result = resultFromLifecycle(
2035
+ status,
2036
+ terminal.id,
2037
+ snapshot.result,
2038
+ snapshot.error,
2039
+ snapshot.compactionCount,
2040
+ snapshot.tokenCount,
2041
+ state.callTiers.get(nodeId)?.tier ?? tier,
2042
+ );
2043
+ result.attemptId = attemptId;
2044
+ if (!nested) this.commitCallTerminal(runId, nodeId, result);
2045
+ if (status === "failed" && snapshot.error) {
2046
+ throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
2047
+ }
1589
2048
  }
2049
+ const completedTierIdentity = state.callTiers.get(nodeId);
2050
+ if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
2051
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
2052
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
2053
+ return snapshot.result ?? "";
2054
+ } finally {
2055
+ // Publication does not donate ownership to an observer or a replacement
2056
+ // state. Even a terminal reply must STOP + QUIESCE its captured owner
2057
+ // when observation retired this dispatch.
2058
+ if (
2059
+ pending.phase === "submitted" &&
2060
+ (!terminalAccepted || !this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
2061
+ )
2062
+ await cleanupResponse();
2063
+ state.pendingSpawns.delete(spawnKey);
2064
+ state.bufferedTerminals.delete(spawnKey);
1590
2065
  }
1591
- const completedTierIdentity = state.callTiers.get(nodeId);
1592
- if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
1593
- return snapshot.result ?? "";
1594
2066
  }
1595
2067
 
1596
2068
  private journalCallResult(runId: string, entry: JournalEntry): void {
2069
+ entry = structuredClone(entry);
1597
2070
  const state = this.runs.get(runId);
1598
2071
  if (!state) return;
1599
2072
  const run = state.run;
2073
+ const executionGeneration = state.executionGeneration;
2074
+ const epoch = this.branchEpoch;
1600
2075
  const nodeId = String(entry.index);
1601
2076
  const result = entry.result;
1602
2077
  const callTierIdentity: CallTierIdentity =
1603
2078
  entry.tier === undefined ? (run.callTiers[nodeId] ?? {}) : { tier: entry.tier };
2079
+ const managed = state.managedCallFacts.get(nodeId);
2080
+ const currentManaged =
2081
+ managed?.executionGeneration === state.executionGeneration && managed.attemptId === run.attemptIds[nodeId]
2082
+ ? managed
2083
+ : undefined;
1604
2084
  const callResult: CallResult = {
2085
+ ...(currentManaged
2086
+ ? {
2087
+ agentId: currentManaged.agentId,
2088
+ ...(currentManaged.tokenCount === undefined ? {} : { tokenCount: currentManaged.tokenCount }),
2089
+ ...(currentManaged.outputFile === undefined ? {} : { outputFile: currentManaged.outputFile }),
2090
+ }
2091
+ : {}),
1605
2092
  status: "completed",
1606
2093
  ...(typeof result === "string" && result.length === 0 ? {} : { result }),
1607
2094
  ...callTierIdentity,
1608
- compactionCount: 0,
2095
+ compactionCount: currentManaged?.compactionCount ?? 0,
1609
2096
  updatedAt: Date.now(),
1610
2097
  };
1611
2098
  // Checkpoints are local calls, not managed children. Give their completed
@@ -1623,6 +2110,7 @@ export class WorkflowEngine {
1623
2110
  owner: { extension: "pi-workflows", runId, nodeId, attemptId },
1624
2111
  timestamp: Date.now(),
1625
2112
  });
2113
+ if (!this.isCurrentState(state, executionGeneration, epoch)) return;
1626
2114
  state.generations.set(nodeId, generation);
1627
2115
  run.attemptTracking = true;
1628
2116
  run.attempts[nodeId] = generation;
@@ -1645,16 +2133,23 @@ export class WorkflowEngine {
1645
2133
  ...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
1646
2134
  timestamp: Date.now(),
1647
2135
  };
1648
- this.persist(runId, `call-result:${nodeId}`, () => this.write(event));
2136
+ const acceptedEvent = structuredClone(event);
2137
+ this.persist(runId, `call-result:${nodeId}`, () => this.write(acceptedEvent));
2138
+ if (!this.isCurrentState(state, executionGeneration, epoch)) return;
1649
2139
  run.callResults[nodeId] = callResult;
1650
2140
  run.callTiers[nodeId] = callResult.tier === undefined ? {} : { tier: callResult.tier };
1651
2141
  run.callStatus[nodeId] = "completed";
2142
+ run.compactions[nodeId] = callResult.compactionCount;
2143
+ state.managedCallFacts.delete(nodeId);
1652
2144
  run.updatedAt = Date.now();
1653
2145
  }
1654
2146
 
1655
2147
  private journalWorkflowResult(runId: string, entry: JournalEntry): void {
2148
+ entry = structuredClone(entry);
1656
2149
  const state = this.runs.get(runId);
1657
2150
  if (!state) return;
2151
+ const generation = state.executionGeneration;
2152
+ const epoch = this.branchEpoch;
1658
2153
  const nodeId = String(entry.index);
1659
2154
  const result: CallResult = {
1660
2155
  status: "completed",
@@ -1681,6 +2176,7 @@ export class WorkflowEngine {
1681
2176
  // otherwise a transient append retry would arrive after the terminal
1682
2177
  // transition and be discarded as stale during restore.
1683
2178
  this.appendRequired(event);
2179
+ if (!this.isCurrentState(state, generation, epoch)) return;
1684
2180
  state.run.workflowResultGenerations[nodeId] = event.generation;
1685
2181
  state.run.callResults[nodeId] = result;
1686
2182
  state.run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
@@ -1692,6 +2188,8 @@ export class WorkflowEngine {
1692
2188
  const state = this.runs.get(runId);
1693
2189
  if (!state) return;
1694
2190
  const run = state.run;
2191
+ const generation = state.executionGeneration;
2192
+ const epoch = this.branchEpoch;
1695
2193
  const attemptId = result.attemptId ?? run.attemptIds[nodeId];
1696
2194
  const event: JournalEvent = {
1697
2195
  kind: "call_result",
@@ -1705,6 +2203,7 @@ export class WorkflowEngine {
1705
2203
  };
1706
2204
  if (required) this.appendRequired(event);
1707
2205
  else this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
2206
+ if (!this.isCurrentState(state, generation, epoch)) return;
1708
2207
  run.callResults[nodeId] = result;
1709
2208
  run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
1710
2209
  run.callStatus[nodeId] = result.status;
@@ -1717,36 +2216,26 @@ export class WorkflowEngine {
1717
2216
 
1718
2217
  private reconcilePendingSpawns(state: ScriptRunState): void {
1719
2218
  for (const pending of state.pendingSpawns.values()) {
2219
+ if (pending.phase !== "submitted" || pending.agentId) continue;
1720
2220
  const reconciliation = this.client.reconcileManaged?.(pending.spawnKey, pending.owner);
1721
2221
  void reconciliation?.catch(() => {});
1722
2222
  }
1723
- state.pendingSpawns.clear();
2223
+ // Reconciliation is not positive exact-owner quiescence. The allocation
2224
+ // lease remains with dispatch until terminal admission or owned cleanup.
1724
2225
  state.bufferedTerminals.clear();
1725
2226
  }
1726
2227
 
1727
- private async quiescePendingSpawns(state: ScriptRunState): Promise<WorkflowQuiesceResult> {
1728
- const pending = [...state.pendingSpawns.values()];
1729
- if (pending.length === 0) return { settled: true, pending: [] };
1730
- if (!this.client.reconcileManaged) {
1731
- return { settled: false, pending: pending.map(({ spawnKey }) => spawnKey).slice(0, 256) };
1732
- }
1733
- const failed: string[] = [];
1734
- await Promise.all(
1735
- pending.map(async ({ spawnKey, owner }) => {
1736
- try {
1737
- const result = await this.client.reconcileManaged?.(spawnKey, owner);
1738
- if (result === undefined) failed.push(spawnKey);
1739
- } catch {
1740
- failed.push(spawnKey);
1741
- }
1742
- }),
1743
- );
1744
- return { settled: failed.length === 0, pending: failed.slice(0, 256) };
1745
- }
1746
-
1747
2228
  private cleanupAgents(runId: string): void {
1748
2229
  const state = this.runs.get(runId);
1749
2230
  if (!state) return;
2231
+ // Retain captured allocations before reconciliation, abort, or branch replay
2232
+ // can notify observers. Dispatch's finally may not run until a later turn;
2233
+ // its cleanup and any waiter cleanup share this stable exact-owner lease.
2234
+ for (const pending of state.pendingSpawns.values()) {
2235
+ if (pending.phase !== "submitted") continue;
2236
+ const agentId = pending.agentId ?? pending.startedAgentId ?? pending.spawnKey;
2237
+ void this.stopAndQuiesce(state, agentId, pending.owner, pending.executionGeneration);
2238
+ }
1750
2239
  this.reconcilePendingSpawns(state);
1751
2240
  for (const [agentId, waiter] of [...state.agentWaiters]) {
1752
2241
  if (waiter.settled) continue;
@@ -1757,14 +2246,17 @@ export class WorkflowEngine {
1757
2246
  recoverable: true,
1758
2247
  }),
1759
2248
  );
1760
- void this.client
1761
- .stopOwned?.(agentId, {
2249
+ void this.stopAndQuiesce(
2250
+ state,
2251
+ agentId,
2252
+ {
1762
2253
  extension: "pi-workflows",
1763
2254
  runId,
1764
2255
  nodeId: `call-${waiter.callIndex}`,
1765
2256
  attemptId: `attempt-${waiter.generation}`,
1766
- })
1767
- .catch(() => {});
2257
+ },
2258
+ state.executionGeneration,
2259
+ );
1768
2260
  }
1769
2261
  }
1770
2262
 
@@ -1774,51 +2266,108 @@ export class WorkflowEngine {
1774
2266
  owner: WorkflowOwner,
1775
2267
  executionGeneration: number,
1776
2268
  unconfirmed?: string,
2269
+ quiescence?: Promise<{ settled: boolean; pending: string[] }>,
1777
2270
  ): Promise<void> {
1778
- let timer: ReturnType<typeof setTimeout> | undefined;
1779
- let stopError: unknown;
1780
- try {
1781
- if (unconfirmed) throw new Error(unconfirmed);
1782
- await Promise.race([
1783
- (async () => {
2271
+ const obligations = this.cleanupObligations.get(state) ?? new Map<string, OwnedCleanup>();
2272
+ this.cleanupObligations.set(state, obligations);
2273
+ const allocationKey = `${owner.runId}/${owner.nodeId}/${owner.attemptId}`;
2274
+ const existing = [...obligations.entries()].find(([, cleanup]) => sameWorkflowOwner(cleanup.owner, owner));
2275
+ const key = existing?.[0] ?? JSON.stringify([owner, agentId]);
2276
+ let obligation = existing?.[1];
2277
+ if (!obligation) {
2278
+ let resolve!: () => void;
2279
+ const confirmed = new Promise<void>((done) => {
2280
+ resolve = done;
2281
+ });
2282
+ obligation = {
2283
+ agentId,
2284
+ ...(agentId === allocationKey ? { spawnKey: allocationKey } : {}),
2285
+ owner: { ...owner },
2286
+ epoch: this.branchEpoch,
2287
+ confirmed,
2288
+ resolve,
2289
+ };
2290
+ obligations.set(key, obligation);
2291
+ this.retainedCleanupStates.add(state);
2292
+ state.pendingCleanup.add(confirmed);
2293
+ } else if (obligation.spawnKey && agentId !== allocationKey) {
2294
+ // A late reply refines the same allocation lease, not a new obligation.
2295
+ obligation.agentId = agentId;
2296
+ obligation.spawnKey = undefined;
2297
+ }
2298
+ if (obligation.operation) return obligation.operation;
2299
+ const accepted = obligation;
2300
+ const currentOwner = () => this.isCurrentState(state, state.executionGeneration, accepted.epoch);
2301
+ const operation = Promise.resolve().then(async () => {
2302
+ let timer: ReturnType<typeof setTimeout> | undefined;
2303
+ let stopError: unknown;
2304
+ try {
2305
+ if (unconfirmed) throw new Error(unconfirmed);
2306
+ const cleanup = (async () => {
2307
+ // An allocation without a confirmed id can only be retried by its exact spawn owner.
2308
+ if (accepted.spawnKey) {
2309
+ if (!this.client.reconcileManaged) throw new Error("owned allocation reconciliation unavailable");
2310
+ const reply = await this.client.reconcileManaged(accepted.spawnKey, accepted.owner);
2311
+ const id = typeof reply === "string" ? reply : reply?.id;
2312
+ if (!id) throw new Error("cancelled spawn allocation could not be confirmed");
2313
+ accepted.agentId = id;
2314
+ accepted.spawnKey = undefined;
2315
+ }
1784
2316
  try {
1785
2317
  if (!this.client.stopOwned) throw new Error("owned stop RPC unavailable");
1786
- await this.client.stopOwned(agentId, owner);
2318
+ await this.client.stopOwned(accepted.agentId, accepted.owner);
1787
2319
  } catch (error: unknown) {
1788
- // Reconciliation may already have stopped the record. Only exact-owner
1789
- // quiescence confirms that provider/tool cleanup has actually settled.
1790
2320
  stopError = error;
1791
2321
  }
1792
2322
  if (!this.client.quiesceOwned) throw new Error("owned cleanup RPC unavailable");
1793
- const result = await this.client.quiesceOwned(owner.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner]);
2323
+ const result = await (quiescence ??
2324
+ this.client.quiesceOwned(accepted.owner.runId, [accepted.agentId], BRANCH_QUIESCE_TIMEOUT_MS, [
2325
+ accepted.owner,
2326
+ ]));
1794
2327
  if (!result?.settled || result.pending.length !== 0) throw new Error("owned agents are not quiescent");
1795
- })(),
1796
- new Promise<never>((_resolve, reject) => {
1797
- timer = setTimeout(() => reject(new Error("owned cleanup timed out")), BRANCH_QUIESCE_TIMEOUT_MS + 1_000);
1798
- }),
1799
- ]);
1800
- } catch (error: unknown) {
1801
- // Never turn uncertain cleanup into a resumable interruption. Existing
1802
- // terminal recovery facts durably fence this run without a protocol change.
1803
- if (this.runs.get(owner.runId) !== state || state.executionGeneration !== executionGeneration) return;
1804
- const stopDiagnostic = stopError === undefined ? "" : `; stop failed: ${String(stopError)}`;
1805
- const diagnostic = `Owned cleanup unconfirmed for ${agentId}: ${error instanceof Error ? error.message : String(error)}${stopDiagnostic}; run is non-resumable`;
1806
- this.fenceExecution(state, executionGeneration, diagnostic);
2328
+ // A bounded negative/error is not evidence that cleanup settled. Only
2329
+ // this positive owner check releases the stable obligation, even late.
2330
+ obligations.delete(key);
2331
+ if (obligations.size === 0) this.retainedCleanupStates.delete(state);
2332
+ state.pendingCleanup.delete(accepted.confirmed);
2333
+ accepted.resolve();
2334
+ if (currentOwner() && isTerminalWorkflow(state.run.status)) this.notifyTerminal(state);
2335
+ })();
2336
+ await Promise.race([
2337
+ cleanup,
2338
+ new Promise<never>((_resolve, reject) => {
2339
+ timer = setTimeout(() => reject(new Error("owned cleanup timed out")), BRANCH_QUIESCE_TIMEOUT_MS + 1_000);
2340
+ }),
2341
+ ]);
2342
+ } catch (error: unknown) {
2343
+ if (!currentOwner() || state.executionGeneration !== executionGeneration) return;
2344
+ const stopDiagnostic = stopError === undefined ? "" : `; stop failed: ${String(stopError)}`;
2345
+ const diagnostic = `Owned cleanup unconfirmed for ${agentId}: ${error instanceof Error ? error.message : String(error)}${stopDiagnostic}; run is non-resumable`;
2346
+ this.fenceExecution(state, executionGeneration, diagnostic);
2347
+ } finally {
2348
+ if (timer !== undefined) clearTimeout(timer);
2349
+ }
2350
+ });
2351
+ accepted.operation = operation;
2352
+ try {
2353
+ await operation;
1807
2354
  } finally {
1808
- if (timer !== undefined) clearTimeout(timer);
2355
+ if (accepted.operation === operation) accepted.operation = undefined;
1809
2356
  }
1810
2357
  }
1811
2358
 
1812
2359
  private async drainExecution(state: ScriptRunState, execution?: Promise<unknown>): Promise<void> {
1813
2360
  if (!execution) return;
1814
2361
  const generation = state.executionGeneration;
2362
+ const epoch = this.branchEpoch;
1815
2363
  let timer: ReturnType<typeof setTimeout> | undefined;
1816
2364
  try {
1817
2365
  await Promise.race([
1818
2366
  execution.catch(() => {}),
1819
2367
  new Promise<void>((resolve) => {
1820
2368
  timer = setTimeout(() => {
1821
- this.fenceExecution(state, generation, "Workflow execution cleanup timed out; run is non-resumable");
2369
+ if (this.isCurrentState(state, generation, epoch))
2370
+ this.fenceExecution(state, generation, "Workflow execution cleanup timed out; run is non-resumable");
1822
2371
  resolve();
1823
2372
  }, EXECUTION_DRAIN_TIMEOUT_MS);
1824
2373
  }),
@@ -1830,7 +2379,8 @@ export class WorkflowEngine {
1830
2379
 
1831
2380
  private fenceExecution(state: ScriptRunState, generation: number, diagnostic: string): void {
1832
2381
  const run = state.run;
1833
- if (this.runs.get(run.runId) !== state || state.executionGeneration !== generation) return;
2382
+ const epoch = this.branchEpoch;
2383
+ if (!this.isCurrentState(state, generation, epoch)) return;
1834
2384
  if (run.nonResumable && run.status === "stopped") return;
1835
2385
  run.nonResumable = true;
1836
2386
  const blockedNodeIds = Object.keys(run.callStatus).filter((nodeId) => run.callStatus[nodeId] !== "completed");
@@ -1853,21 +2403,119 @@ export class WorkflowEngine {
1853
2403
  timestamp: Date.now(),
1854
2404
  };
1855
2405
  this.persist(run.runId, "cleanup-unconfirmed", () => this.write(event));
2406
+ if (!this.isCurrentState(state, generation, epoch)) return;
1856
2407
  // Fence before aborting: synchronous abort handlers may re-enter cleanup.
1857
2408
  // Exact-owner cleanup continues, but no later result may revive this run.
1858
2409
  state.executionGeneration += 1;
2410
+ state.presentation.reset(false);
2411
+ this.notifyInspection(state);
2412
+ if (!this.isCurrentState(state, generation + 1, epoch)) return;
1859
2413
  state.controller.abort();
2414
+ if (!this.isCurrentState(state, generation + 1, epoch)) return;
1860
2415
  this.cleanupAgents(run.runId);
1861
- this.settleWaiters(run.runId, run);
2416
+ // Uncertainty is an error, never a terminal result delivery. Foreground
2417
+ // callers can unwind, while owner UI retains an explicit cleanup recheck.
2418
+ if (state.pendingCleanup.size > 0) {
2419
+ const all = this.waiters.get(run.runId);
2420
+ this.waiters.delete(run.runId);
2421
+ for (const waiter of all ?? []) {
2422
+ if (waiter.settled) continue;
2423
+ waiter.settled = true;
2424
+ waiter.cleanup();
2425
+ waiter.reject(new WorkflowWaitAbortedError());
2426
+ }
2427
+ } else this.notifyTerminal(state);
2428
+ }
2429
+
2430
+ private async remove(state: ScriptRunState): Promise<void> {
2431
+ if (state.removeOperation) return state.removeOperation;
2432
+ const isBusy = () =>
2433
+ state.activeExecutionGenerations.size > 0 ||
2434
+ state.pendingCleanup.size > 0 ||
2435
+ state.pendingSpawns.size > 0 ||
2436
+ state.agentWaiters.size > 0 ||
2437
+ state.stopOperation !== undefined;
2438
+ if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
2439
+ const epoch = this.branchEpoch;
2440
+ const operation = Promise.resolve().then(async () => {
2441
+ await this.drainJournal(state.run.runId);
2442
+ if (
2443
+ this.runs.get(state.run.runId) !== state ||
2444
+ this.branchEpoch !== epoch ||
2445
+ this.disposed ||
2446
+ this.lifecyclePaused ||
2447
+ state.lifecycleSuspended
2448
+ )
2449
+ throw new Error("workflow branch changed before removal");
2450
+ if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
2451
+ // No queued best-effort tombstone: memory survives failure so removal can be retried.
2452
+ this.appendRequired({
2453
+ kind: "run_removed",
2454
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
2455
+ runId: state.run.runId,
2456
+ timestamp: Date.now(),
2457
+ });
2458
+ this.runs.delete(state.run.runId);
2459
+ this.waiters.delete(state.run.runId);
2460
+ this.notifyInspection(state);
2461
+ });
2462
+ state.removeOperation = operation;
2463
+ try {
2464
+ await operation;
2465
+ } finally {
2466
+ if (state.removeOperation === operation) state.removeOperation = undefined;
2467
+ }
1862
2468
  }
1863
2469
 
1864
2470
  private async stop(state: ScriptRunState): Promise<void> {
2471
+ if (state.stopOperation) return state.stopOperation;
2472
+ if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size === 0) return;
2473
+ const generation = state.executionGeneration;
2474
+ const epoch = this.branchEpoch;
2475
+ const operation = Promise.resolve().then(async () => {
2476
+ if (!this.isCurrentState(state, generation, epoch)) return;
2477
+ const pending = [...(this.cleanupObligations.get(state)?.values() ?? [])];
2478
+ if (pending.length > 0) {
2479
+ await Promise.all(
2480
+ pending.map((cleanup) => this.stopAndQuiesce(state, cleanup.agentId, cleanup.owner, generation)),
2481
+ );
2482
+ if (!this.isCurrentState(state, generation, epoch) || state.pendingCleanup.size > 0) return;
2483
+ if (isTerminalWorkflow(state.run.status)) {
2484
+ this.notifyTerminal(state);
2485
+ return;
2486
+ }
2487
+ }
2488
+ await this.stopExecution(state, generation, epoch);
2489
+ });
2490
+ state.stopOperation = operation;
2491
+ try {
2492
+ await operation;
2493
+ } finally {
2494
+ if (state.stopOperation === operation) state.stopOperation = undefined;
2495
+ }
2496
+ }
2497
+
2498
+ private async stopExecution(state: ScriptRunState, generation: number, epoch: number): Promise<void> {
1865
2499
  const run = state.run;
1866
- if (isTerminalWorkflow(run.status)) return;
2500
+ const current = () => this.isCurrentState(state, generation, epoch);
2501
+ if (!current() || isTerminalWorkflow(run.status)) return;
1867
2502
  this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
2503
+ if (!current()) return;
1868
2504
  run.nonResumable = true;
2505
+ // Runtime abort listeners own exact cleanup. Rejecting their waiters here would
2506
+ // falsely settle the dispatch before its stop-and-quiesce barrier completes.
1869
2507
  state.controller.abort();
1870
- this.cleanupAgents(run.runId);
2508
+ if (!current()) return;
2509
+ await this.drainExecution(state, state.execution);
2510
+ if (!current() || state.pendingCleanup.size > 0) return;
2511
+ try {
2512
+ await this.drainJournal(run.runId);
2513
+ } catch {
2514
+ if (current())
2515
+ this.fenceExecution(state, generation, "Could not drain workflow journal during stop; run is non-resumable");
2516
+ return;
2517
+ }
2518
+ if (!current()) return;
1871
2519
  const terminalResults: RecoveryTerminalResult[] = [];
1872
2520
  for (const [nodeId, status] of Object.entries(run.callStatus)) {
1873
2521
  if (status !== "completed") {
@@ -1887,8 +2535,6 @@ export class WorkflowEngine {
1887
2535
  result,
1888
2536
  owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId: run.attemptIds[nodeId] ?? "" },
1889
2537
  });
1890
- run.callResults[nodeId] = result;
1891
- run.callStatus[nodeId] = "stopped";
1892
2538
  }
1893
2539
  }
1894
2540
  const event: TerminalRecoveryEvent = {
@@ -1904,10 +2550,22 @@ export class WorkflowEngine {
1904
2550
  recoveryId: deriveRecoveryId({ runId: run.runId, rotations: [], timestamp: Date.now() }),
1905
2551
  timestamp: Date.now(),
1906
2552
  };
1907
- this.persist(run.runId, "stop-terminal", () => this.write(event));
2553
+ try {
2554
+ this.appendRequired(event);
2555
+ } catch {
2556
+ this.fenceExecution(state, generation, "Could not persist workflow stop terminal; run is non-resumable");
2557
+ return;
2558
+ }
2559
+ if (!current()) return;
2560
+ for (const terminal of terminalResults) {
2561
+ run.callResults[terminal.nodeId] = terminal.result;
2562
+ run.callStatus[terminal.nodeId] = "stopped";
2563
+ }
1908
2564
  run.status = "stopped";
1909
2565
  run.updatedAt = Date.now();
1910
- this.settleWaiters(run.runId, run);
2566
+ state.presentation.settle(run.status);
2567
+ this.notifyInspection(state);
2568
+ if (current()) this.notifyTerminal(state);
1911
2569
  }
1912
2570
 
1913
2571
  // ── journal writes ─────────────────────────────────────────────────────────
@@ -2022,6 +2680,14 @@ export class WorkflowEngine {
2022
2680
  * a transient failure, schedules a retry of the exact same append.
2023
2681
  */
2024
2682
  private persist(runId: string, key: string, mutation: () => void): void {
2683
+ const acceptedState = this.runs.get(runId);
2684
+ const epoch = this.branchEpoch;
2685
+ const append = mutation;
2686
+ mutation = () => {
2687
+ if (this.disposed || this.lifecyclePaused || this.runs.get(runId) !== acceptedState || this.branchEpoch !== epoch)
2688
+ return;
2689
+ append();
2690
+ };
2025
2691
  if ([...this.journalRetries.keys()].some((id) => id.startsWith(`${runId}\u0000`))) {
2026
2692
  this.scheduleJournalRetry(runId, key, mutation);
2027
2693
  return;
@@ -2098,6 +2764,10 @@ export class WorkflowEngine {
2098
2764
  finalResult?: unknown,
2099
2765
  ): void {
2100
2766
  if (isTerminalWorkflow(run.status) && !isTerminalWorkflow(status)) return;
2767
+ const state = this.runs.get(run.runId);
2768
+ const generation = state?.executionGeneration ?? 0;
2769
+ const epoch = this.branchEpoch;
2770
+ if (!state || state.run !== run || !this.isCurrentState(state, generation, epoch)) return;
2101
2771
  // Validate and isolate terminal values before best-effort persistence or any
2102
2772
  // status mutation. A bad result must still be able to transition the run to failed.
2103
2773
  const durableFinalResult = finalResult === undefined ? undefined : cloneWorkflowResult(finalResult);
@@ -2115,16 +2785,18 @@ export class WorkflowEngine {
2115
2785
  timestamp,
2116
2786
  }),
2117
2787
  );
2788
+ if (!this.isCurrentState(state, generation, epoch)) return;
2118
2789
  run.status = status;
2119
2790
  if (status === "running") run.error = undefined;
2120
2791
  if (error !== undefined) run.error = error;
2121
2792
  if (runFinalResult !== undefined) run.finalResult = runFinalResult;
2122
2793
  if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
2123
2794
  run.updatedAt = timestamp;
2795
+ this.notifyInspection(state);
2796
+ if (!this.isCurrentState(state, generation, epoch)) return;
2124
2797
  if (isTerminalWorkflow(status)) {
2125
- this.runs.get(run.runId)?.bufferedTerminals.clear();
2126
- this.settleWaiters(run.runId, run);
2127
- this.onRunSettled?.();
2798
+ state.bufferedTerminals.clear();
2799
+ this.notifyTerminal(state);
2128
2800
  }
2129
2801
  }
2130
2802
 
@@ -2132,6 +2804,9 @@ export class WorkflowEngine {
2132
2804
 
2133
2805
  restore(entries: readonly SessionEntryLike[], branchGeneration = 0): void {
2134
2806
  if (this.disposed) return;
2807
+ const retiredStates = [...this.runs.values()];
2808
+ this.branchEpoch++;
2809
+ for (const state of retiredStates) state.lifecycleSuspended = true;
2135
2810
  // A branch replay is a replacement, not a merge. Runs absent from the new
2136
2811
  // branch must not remain controllable or append facts into it.
2137
2812
  this.clearJournalRetries();
@@ -2149,6 +2824,10 @@ export class WorkflowEngine {
2149
2824
  for (const run of runs.values()) {
2150
2825
  const state: ScriptRunState = {
2151
2826
  run,
2827
+ presentation: new WorkflowPresentation(run.runId, true),
2828
+ managedCallFacts: new Map(),
2829
+ pendingCleanup: new Set(),
2830
+ activeExecutionGenerations: new Set(),
2152
2831
  frozenOptions: {
2153
2832
  ...(run.frozenTokenBudget !== undefined ? { tokenBudget: run.frozenTokenBudget } : {}),
2154
2833
  ...(run.frozenMaxAgents !== undefined ? { maxAgents: run.frozenMaxAgents } : {}),
@@ -2183,6 +2862,8 @@ export class WorkflowEngine {
2183
2862
  run.updatedAt = Date.now();
2184
2863
  this.runs.set(run.runId, state);
2185
2864
  }
2865
+ for (const state of retiredStates) this.notifyInspection(state);
2866
+ for (const state of this.runs.values()) this.notifyInspection(state);
2186
2867
  }
2187
2868
 
2188
2869
  list(): Array<Record<string, unknown>> {
@@ -2200,6 +2881,18 @@ export class WorkflowEngine {
2200
2881
  return this.runs.get(runId)?.run;
2201
2882
  }
2202
2883
 
2884
+ /** Primitive-only Fleet metadata; avoids cloning full inspection arrays for every run. */
2885
+ getInspectionHeader(runId: string): WorkflowInspectionHeader | undefined {
2886
+ const state = this.runs.get(runId);
2887
+ return state?.presentation.header(state.executionGeneration);
2888
+ }
2889
+
2890
+ /** Detached bounded facts for late-opening UI mounts; no event callback ownership required. */
2891
+ getInspection(runId: string): WorkflowInspection | undefined {
2892
+ const state = this.runs.get(runId);
2893
+ return state?.presentation.snapshot(state.run, state.executionGeneration);
2894
+ }
2895
+
2203
2896
  summary(run: ScriptRun, detailed = false): Record<string, unknown> {
2204
2897
  const base: Record<string, unknown> = {
2205
2898
  runId: run.runId,
@@ -2250,5 +2943,5 @@ export class WorkflowWaitAbortedError extends Error {
2250
2943
  }
2251
2944
  }
2252
2945
 
2253
- export type { ScriptRun, WorkflowOwner };
2946
+ export type { ScriptRun, WorkflowInspection, WorkflowInspectionHeader, WorkflowOwner };
2254
2947
  export { createManagedSpawnClient, JOURNAL_ENTRY_TYPE, snapshotRun };