@signalridge/pi-workflows 1.7.5 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts CHANGED
@@ -20,8 +20,10 @@
20
20
  import { createHash, randomUUID } from "node:crypto";
21
21
  import type { ManagedSpawnResponse } from "@signalridge/pi-subagents-protocol";
22
22
  import { isManagedAgentTier } from "@signalridge/pi-subagents-protocol";
23
- import { WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
23
+ import { describeWorkflowError, WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
24
+ import { type WorkflowInspection, type WorkflowInspectionHeader, WorkflowPresentation } from "./inspection.js";
24
25
  import {
26
+ applyRecoveryEvent,
25
27
  buildResumeJournal,
26
28
  type CallResult,
27
29
  type CallTierIdentity,
@@ -31,6 +33,7 @@ import {
31
33
  type JournalEvent,
32
34
  type JournalWriter,
33
35
  type RecoveryTerminalResult,
36
+ type RunRecoveryEvent,
34
37
  replayJournal,
35
38
  type ScriptRun,
36
39
  type SessionEntryLike,
@@ -172,6 +175,25 @@ interface AgentTerminalDelivery {
172
175
  history?: unknown[];
173
176
  }
174
177
 
178
+ interface ManagedCallFacts {
179
+ executionGeneration: number;
180
+ attemptId: string;
181
+ agentId: string;
182
+ tokenCount?: number;
183
+ outputFile?: string;
184
+ compactionCount: number;
185
+ }
186
+
187
+ interface OwnedCleanup {
188
+ agentId: string;
189
+ spawnKey?: string;
190
+ owner: WorkflowOwner;
191
+ epoch: number;
192
+ confirmed: Promise<void>;
193
+ resolve: () => void;
194
+ operation?: Promise<void>;
195
+ }
196
+
175
197
  interface BufferedAgentTerminal extends AgentTerminalDelivery {
176
198
  owner: WorkflowOwner;
177
199
  executionGeneration: number;
@@ -179,6 +201,15 @@ interface BufferedAgentTerminal extends AgentTerminalDelivery {
179
201
 
180
202
  export interface ScriptRunState {
181
203
  run: ScriptRun;
204
+ /** Volatile, execution-fenced presentation; never journaled. */
205
+ presentation: WorkflowPresentation;
206
+ /** Exact-attempt metadata merged into the runtime's single schema-transformed result fact. */
207
+ managedCallFacts: Map<string, ManagedCallFacts>;
208
+ /** Ownership remains busy even after a cleanup deadline fences execution. */
209
+ pendingCleanup: Set<Promise<void>>;
210
+ activeExecutionGenerations: Set<number>;
211
+ stopOperation?: Promise<void>;
212
+ removeOperation?: Promise<void>;
182
213
  /** Frozen run parameters — resume reuses the original values, ignoring new budget/toolset overrides. */
183
214
  frozenOptions: Pick<
184
215
  ScriptStartOptions,
@@ -200,8 +231,18 @@ export interface ScriptRunState {
200
231
  agentWaiters: Map<string, AgentLifecycleWaiter>;
201
232
  /** Owner-validated terminal events emitted before their spawn reply/waiter. */
202
233
  bufferedTerminals: Map<string, BufferedAgentTerminal>;
203
- /** Allocations whose managed-spawn reply has not arrived yet. */
204
- pendingSpawns: Map<string, { spawnKey: string; owner: WorkflowOwner; executionGeneration: number }>;
234
+ /** Local reservations and submitted allocations retained through terminal admission. */
235
+ pendingSpawns: Map<
236
+ string,
237
+ {
238
+ spawnKey: string;
239
+ owner: WorkflowOwner;
240
+ executionGeneration: number;
241
+ phase: "reserved" | "submitted";
242
+ startedAgentId?: string;
243
+ agentId?: string;
244
+ }
245
+ >;
205
246
  /** Journal retry bookkeeping. */
206
247
  pendingJournal: Set<string>;
207
248
  /** Whether a lifecycle suspension is in effect (branch change). */
@@ -307,12 +348,41 @@ function sameWorkflowOwner(left: WorkflowOwner, right: WorkflowOwner): boolean {
307
348
  );
308
349
  }
309
350
 
351
+ /** Optional owned lifecycle usage is per-attempt; legacy `tokens` remains lifetime-scoped. */
352
+ function managedLifecycleUsage(raw: unknown): AgentUsage | undefined {
353
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
354
+ const { input, output, total } = raw as Record<string, unknown>;
355
+ if (
356
+ typeof input !== "number" ||
357
+ typeof output !== "number" ||
358
+ typeof total !== "number" ||
359
+ !Number.isSafeInteger(input) ||
360
+ !Number.isSafeInteger(output) ||
361
+ !Number.isSafeInteger(total) ||
362
+ input < 0 ||
363
+ output < 0 ||
364
+ total < 0 ||
365
+ !Number.isSafeInteger(input + output) ||
366
+ total < input + output
367
+ )
368
+ return undefined;
369
+ // The managed total includes cache writes, but excludes cache reads. The
370
+ // additive lifecycle field has no cost/cache-read detail; do not invent it.
371
+ return { input, output, total, cost: 0, cacheRead: 0, cacheWrite: total - input - output };
372
+ }
373
+
310
374
  export class WorkflowEngine {
311
375
  private readonly runs = new Map<string, ScriptRunState>();
312
376
  /** Fired when a run settles terminal; wired by the UI surface for widget refresh. */
313
377
  onRunSettled: (() => void) | undefined;
378
+ /** Every accepted background execution, including automatic and UI resumes. */
379
+ onBackgroundExecution: ((runId: string) => void) | undefined;
314
380
  /** Optional in-process observer for phase/task/quality runtime events. */
315
381
  onRuntimeEvent: ((runId: string, event: WorkflowRuntimeEvent) => void) | undefined;
382
+ /** Additive notification after accepted presentation mutations; consumers re-read getInspection(). */
383
+ onInspectionChanged: ((runId: string) => void) | undefined;
384
+ /** Fresh saved-workflow resolver for control and automatic resumes. */
385
+ loadSavedWorkflow: ((name: string) => string | undefined) | undefined;
316
386
  /**
317
387
  * The user's current strength → Agent tier table, read fresh per execution.
318
388
  *
@@ -339,11 +409,19 @@ export class WorkflowEngine {
339
409
  private admissionsClosed = false;
340
410
  private pendingAdmissions = 0;
341
411
  private recoveryBranchGeneration = 0;
342
- private readonly journalRetries = new Map<string, { timer?: ReturnType<typeof setTimeout>; attempt: number }>();
412
+ private readonly journalRetries = new Map<
413
+ string,
414
+ { timer?: ReturnType<typeof setTimeout>; attempt: number; done: Promise<void>; resolve: () => void }
415
+ >();
343
416
  private readonly journalBlockedRuns = new Set<string>();
344
417
  private readonly providerResumeTimers = new Map<string, ReturnType<typeof setTimeout>>();
345
418
  private journalMutationKey: string | undefined;
419
+ private journalMutationSequence = 0;
346
420
  private readonly quarantinedRunIds = new Set<string>();
421
+ private readonly cleanupObligations = new WeakMap<ScriptRunState, Map<string, OwnedCleanup>>();
422
+ /** Retired states keep their exact leases; replacement states never inherit them. */
423
+ private readonly retainedCleanupStates = new Set<ScriptRunState>();
424
+ private readonly terminalNotified = new WeakSet<ScriptRunState>();
347
425
 
348
426
  constructor(
349
427
  readonly events: WorkflowEventBus,
@@ -359,6 +437,15 @@ export class WorkflowEngine {
359
437
  this.unsubscribeLifecycle = this.attachLifecycle();
360
438
  }
361
439
 
440
+ private notifyInspection(state: ScriptRunState): void {
441
+ state.presentation.changed();
442
+ try {
443
+ this.onInspectionChanged?.(state.run.runId);
444
+ } catch {
445
+ // Presentation observers must never change execution or durable control.
446
+ }
447
+ }
448
+
362
449
  private async awaitProtocol(): Promise<ManagedProtocolCheck> {
363
450
  if (this.disposed) throw new WorkflowEngineDisposedError();
364
451
  let rejectDisposed: ((reason?: unknown) => void) | undefined;
@@ -382,6 +469,40 @@ export class WorkflowEngine {
382
469
  return this.disposed;
383
470
  }
384
471
 
472
+ /** Pure exact-state cleanup uncertainty query for owner control surfaces. */
473
+ hasPendingCleanup(runId: string): boolean {
474
+ return (this.runs.get(runId)?.pendingCleanup.size ?? 0) > 0;
475
+ }
476
+
477
+ private isCurrentState(state: ScriptRunState, generation: number, epoch: number): boolean {
478
+ return (
479
+ this.runs.get(state.run.runId) === state &&
480
+ state.executionGeneration === generation &&
481
+ this.branchEpoch === epoch &&
482
+ !state.lifecycleSuspended &&
483
+ !this.lifecyclePaused &&
484
+ !this.disposed
485
+ );
486
+ }
487
+
488
+ private notifyTerminal(state: ScriptRunState): void {
489
+ const generation = state.executionGeneration;
490
+ const epoch = this.branchEpoch;
491
+ if (
492
+ !this.isCurrentState(state, generation, epoch) ||
493
+ state.pendingCleanup.size > 0 ||
494
+ this.terminalNotified.has(state)
495
+ )
496
+ return;
497
+ this.terminalNotified.add(state);
498
+ this.settleWaiters(state.run.runId, state.run);
499
+ try {
500
+ this.onRunSettled?.();
501
+ } catch {
502
+ /* terminal observers are presentation-only */
503
+ }
504
+ }
505
+
385
506
  /** A synchronous snapshot for cancellable navigation, not an admission lock. */
386
507
  hasUnsettledWork(): boolean {
387
508
  return (
@@ -389,6 +510,7 @@ export class WorkflowEngine {
389
510
  this.branchQuiescePromise !== undefined ||
390
511
  this.journalRetries.size > 0 ||
391
512
  this.providerResumeTimers.size > 0 ||
513
+ this.retainedCleanupStates.size > 0 ||
392
514
  [...this.runs.values()].some(
393
515
  (state) =>
394
516
  state.run.status === "running" ||
@@ -396,7 +518,11 @@ export class WorkflowEngine {
396
518
  state.run.status === "stopping" ||
397
519
  state.pendingSpawns.size > 0 ||
398
520
  state.agentWaiters.size > 0 ||
399
- state.pendingJournal.size > 0,
521
+ state.pendingJournal.size > 0 ||
522
+ state.pendingCleanup.size > 0 ||
523
+ state.activeExecutionGenerations.size > 0 ||
524
+ state.stopOperation !== undefined ||
525
+ state.removeOperation !== undefined,
400
526
  )
401
527
  );
402
528
  }
@@ -410,6 +536,8 @@ export class WorkflowEngine {
410
536
  this.clearJournalRetries();
411
537
  for (const state of this.runs.values()) {
412
538
  state.lifecycleSuspended = true;
539
+ state.presentation.reset(false);
540
+ this.notifyInspection(state);
413
541
  state.controller.abort(new DOMException("Workflow branch replaced", "AbortError"));
414
542
  }
415
543
  }
@@ -436,7 +564,8 @@ export class WorkflowEngine {
436
564
  eventName !== "subagents:created" &&
437
565
  eventName !== "subagents:started" &&
438
566
  eventName !== "subagents:completed" &&
439
- eventName !== "subagents:failed"
567
+ eventName !== "subagents:failed" &&
568
+ eventName !== "subagents:compacted"
440
569
  )
441
570
  return;
442
571
  if (!raw || typeof raw !== "object") return;
@@ -489,13 +618,55 @@ export class WorkflowEngine {
489
618
  if (currentAttemptId !== undefined && currentAttemptId !== `${owner.runId}/${callNodeId}/${owner.attemptId}`)
490
619
  return;
491
620
 
621
+ const acceptedGeneration = state.executionGeneration;
622
+ const acceptedEpoch = this.branchEpoch;
623
+ const current = () =>
624
+ this.isCurrentState(state, acceptedGeneration, acceptedEpoch) && !state.controller.signal.aborted;
625
+ const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
626
+ const hasManagedUsage = terminalEvent && "managedTokens" in data;
627
+ const managedUsage = hasManagedUsage ? managedLifecycleUsage(data.managedTokens) : undefined;
628
+ // A malformed present field cannot fall back to cumulative lifetime usage.
629
+ // Reject before identity, presentation, buffering, or waiter mutations.
630
+ if (hasManagedUsage && !managedUsage) return;
631
+
632
+ if (eventName === "subagents:compacted") {
633
+ this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
634
+ if (!current()) return;
635
+ if (
636
+ typeof data.compactionCount === "number" &&
637
+ Number.isSafeInteger(data.compactionCount) &&
638
+ data.compactionCount >= 0
639
+ ) {
640
+ const facts = state.managedCallFacts.get(callNodeId);
641
+ if (facts) facts.compactionCount = Math.max(facts.compactionCount, data.compactionCount);
642
+ if (state.run.attemptIds[callNodeId])
643
+ state.run.compactions[callNodeId] = facts?.compactionCount ?? data.compactionCount;
644
+ state.presentation.managed(callNodeId, { compactionCount: facts?.compactionCount ?? data.compactionCount });
645
+ this.notifyInspection(state);
646
+ }
647
+ return;
648
+ }
492
649
  if (updateCallTierIdentity(state, callNodeId, data.tier)) {
493
650
  this.persistCallTierIdentity(state, callNodeId);
494
651
  }
652
+ if (!current()) return;
653
+ this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
654
+ if (!current()) return;
495
655
  // A start is the moment the host admitted this agent from its queue. It is
496
656
  // the only signal that separates waiting for a slot from doing the work, so
497
657
  // the call's wall clock is restarted here rather than at dispatch.
498
- if (eventName === "subagents:started" && waiterMatches && waiter) notifyAgentRunning(waiter);
658
+ if (eventName === "subagents:started") {
659
+ if (pendingMatches && pending) pending.startedAgentId = agentId;
660
+ if (waiterMatches && waiter) notifyAgentRunning(waiter);
661
+ if (!current()) return;
662
+ }
663
+ state.presentation.managed(callNodeId, {
664
+ agentId,
665
+ status: eventName === "subagents:started" ? "running" : undefined,
666
+ ...(isManagedAgentTier(data.tier) ? { tier: data.tier } : {}),
667
+ });
668
+ this.notifyInspection(state);
669
+ if (!current()) return;
499
670
  // Created/started lifecycle facts carry managed identity before the RPC
500
671
  // reply is necessarily available. They are observation-only here; terminal
501
672
  // ownership is handled below once a completion/failure fact arrives.
@@ -508,13 +679,27 @@ export class WorkflowEngine {
508
679
  ? "stopped"
509
680
  : "failed";
510
681
  const tokenCount =
511
- typeof data.tokenCount === "number"
512
- ? data.tokenCount
513
- : data.tokens &&
514
- typeof data.tokens === "object" &&
515
- typeof (data.tokens as Record<string, unknown>).total === "number"
516
- ? ((data.tokens as Record<string, unknown>).total as number)
517
- : undefined;
682
+ managedUsage !== undefined
683
+ ? managedUsage.total
684
+ : typeof data.tokenCount === "number"
685
+ ? data.tokenCount
686
+ : data.tokens &&
687
+ typeof data.tokens === "object" &&
688
+ typeof (data.tokens as Record<string, unknown>).total === "number"
689
+ ? ((data.tokens as Record<string, unknown>).total as number)
690
+ : undefined;
691
+ state.presentation.managed(callNodeId, {
692
+ agentId,
693
+ status: lifecycleStatus,
694
+ tokenCount,
695
+ ...(typeof data.outputFile === "string" ? { outputFile: data.outputFile } : {}),
696
+ compactionCount: Math.max(
697
+ typeof data.compactionCount === "number" ? data.compactionCount : 0,
698
+ state.managedCallFacts.get(callNodeId)?.compactionCount ?? 0,
699
+ ),
700
+ });
701
+ this.notifyInspection(state);
702
+ if (!current()) return;
518
703
  const delivery: AgentTerminalDelivery = {
519
704
  response: {
520
705
  id: agentId,
@@ -528,9 +713,11 @@ export class WorkflowEngine {
528
713
  completedAt: typeof data.completedAt === "number" ? data.completedAt : Date.now(),
529
714
  },
530
715
  },
531
- ...(tokenCount === undefined
532
- ? {}
533
- : { usage: { input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 } }),
716
+ ...(managedUsage !== undefined
717
+ ? { usage: managedUsage }
718
+ : tokenCount === undefined
719
+ ? {}
720
+ : { usage: { input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 } }),
534
721
  ...(Array.isArray(data.history) ? { history: [...data.history] } : {}),
535
722
  };
536
723
 
@@ -547,6 +734,60 @@ export class WorkflowEngine {
547
734
  state.bufferedTerminals.set(spawnKey, { ...delivery, owner, executionGeneration: state.executionGeneration });
548
735
  }
549
736
 
737
+ private recordManagedAdmission(
738
+ state: ScriptRunState,
739
+ nodeId: string,
740
+ agentId: string,
741
+ executionGeneration: number,
742
+ ): void {
743
+ const attemptId = state.run.attemptIds[nodeId];
744
+ if (!attemptId) return; // Nested facts remain volatile, owned by the parent boundary.
745
+ const existing = state.managedCallFacts.get(nodeId);
746
+ if (
747
+ existing?.executionGeneration === executionGeneration &&
748
+ existing.attemptId === attemptId &&
749
+ existing.agentId === agentId
750
+ )
751
+ return;
752
+ state.managedCallFacts.set(nodeId, { executionGeneration, attemptId, agentId, compactionCount: 0 });
753
+ state.run.agentIds[nodeId] = agentId;
754
+ state.run.compactions[nodeId] = 0;
755
+ const event: JournalEvent = {
756
+ kind: "call_transition",
757
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
758
+ runId: state.run.runId,
759
+ nodeId,
760
+ status: "running",
761
+ agentId,
762
+ ...(state.callTiers.get(nodeId)?.tier === undefined ? {} : { tier: state.callTiers.get(nodeId)?.tier }),
763
+ attemptId,
764
+ owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
765
+ timestamp: Date.now(),
766
+ };
767
+ this.persist(state.run.runId, `call-identity:${nodeId}`, () => this.write(event));
768
+ }
769
+
770
+ private recordManagedTerminal(
771
+ state: ScriptRunState,
772
+ nodeId: string,
773
+ agentId: string,
774
+ terminal: NonNullable<ManagedSpawnResponse["terminal"]>,
775
+ executionGeneration: number,
776
+ ): void {
777
+ this.recordManagedAdmission(state, nodeId, agentId, executionGeneration);
778
+ const facts = state.managedCallFacts.get(nodeId);
779
+ if (
780
+ !facts ||
781
+ facts.executionGeneration !== executionGeneration ||
782
+ facts.attemptId !== state.run.attemptIds[nodeId] ||
783
+ facts.agentId !== agentId
784
+ )
785
+ return;
786
+ facts.tokenCount = terminal.tokenCount;
787
+ facts.outputFile = terminal.outputFile;
788
+ facts.compactionCount = Math.max(facts.compactionCount, terminal.compactionCount);
789
+ }
790
+
550
791
  /** Persist a host-normalized tier learned before the managed reply arrives. */
551
792
  private persistCallTierIdentity(state: ScriptRunState, nodeId: string): void {
552
793
  const attemptId = state.run.attemptIds[nodeId];
@@ -554,19 +795,18 @@ export class WorkflowEngine {
554
795
  // Nested workflow frames are in-memory only; only a top-level call has a
555
796
  // durable attempt owner and can safely receive a journal transition.
556
797
  if (!attemptId || identity?.tier === undefined) return;
557
- this.persist(state.run.runId, `call-tier:${nodeId}`, () =>
558
- this.write({
559
- kind: "call_transition",
560
- schemaVersion: JOURNAL_SCHEMA_VERSION,
561
- runId: state.run.runId,
562
- nodeId,
563
- status: "running",
564
- tier: identity.tier,
565
- attemptId,
566
- owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
567
- timestamp: Date.now(),
568
- }),
569
- );
798
+ const event: JournalEvent = {
799
+ kind: "call_transition",
800
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
801
+ runId: state.run.runId,
802
+ nodeId,
803
+ status: "running",
804
+ tier: identity.tier,
805
+ attemptId,
806
+ owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
807
+ timestamp: Date.now(),
808
+ };
809
+ this.persist(state.run.runId, `call-tier:${nodeId}`, () => this.write(event));
570
810
  }
571
811
 
572
812
  private resolveAgentWaiter(
@@ -578,9 +818,23 @@ export class WorkflowEngine {
578
818
  if (waiter.settled) return;
579
819
  waiter.settled = true;
580
820
  state.agentWaiters.delete(agentId);
581
- if (delivery.usage) waiter.onUsage?.(delivery.usage);
582
- if (delivery.history) waiter.onHistory?.(delivery.history);
583
- waiter.resolve(delivery.response);
821
+ const epoch = this.branchEpoch;
822
+ try {
823
+ if (delivery.usage) waiter.onUsage?.(delivery.usage);
824
+ if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
825
+ waiter.reject(new WorkflowWaitAbortedError());
826
+ return;
827
+ }
828
+ if (delivery.history) waiter.onHistory?.(delivery.history);
829
+ if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
830
+ waiter.reject(new WorkflowWaitAbortedError());
831
+ return;
832
+ }
833
+ waiter.resolve(delivery.response);
834
+ } catch (error) {
835
+ // A throwing observer cannot strand a removed waiter and its child lease.
836
+ waiter.reject(error);
837
+ }
584
838
  }
585
839
 
586
840
  dispose(): void {
@@ -589,12 +843,14 @@ export class WorkflowEngine {
589
843
  this.disposeController.abort();
590
844
  this.unsubscribeLifecycle();
591
845
  this.quarantinedRunIds.clear();
592
- for (const state of this.runs.values()) {
846
+ const retiredStates = [...this.runs.values()];
847
+ for (const state of retiredStates) {
593
848
  this.cleanupAgents(state.run.runId);
594
849
  state.controller.abort(new DOMException("Workflow engine disposed", "AbortError"));
595
850
  }
596
851
  this.rejectWaiters(new WorkflowWaitAbortedError());
597
852
  this.runs.clear();
853
+ for (const state of retiredStates) this.notifyInspection(state);
598
854
  this.clearJournalRetries();
599
855
  this.branchQuiescePromise = undefined;
600
856
  }
@@ -613,68 +869,58 @@ export class WorkflowEngine {
613
869
 
614
870
  /** Stop owned agents while Pi replaces the active session-tree branch. */
615
871
  async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
616
- if (this.disposed) return { settled: true, pending: [] };
617
872
  if (this.branchQuiescePromise) return this.branchQuiescePromise;
618
873
  this.admissionsClosed = true;
619
874
  this.clearJournalRetries();
620
875
 
621
- // Suspend and abort synchronously, before any await can let a stale runtime
622
- // dispatch another call or let a provider-limit continuation recreate an old
623
- // branch agent. `restore()` creates fresh states with suspension cleared.
624
- const targets = new Map<string, { agentIds: string[]; owners: WorkflowOwner[] }>();
625
- for (const state of this.runs.values()) {
626
- if (isTerminalWorkflow(state.run.status)) continue;
627
- const agentIds = [...state.agentWaiters.keys()];
628
- const owners = agentIds.map((agentId) => {
629
- const waiter = state.agentWaiters.get(agentId);
630
- return {
631
- extension: "pi-workflows" as const,
876
+ // Snapshot exact states/owners, not run ids or status. Terminal and retired
877
+ // states may still own cleanup, and observation must not donate those
878
+ // leases to a same-id replacement on a newly committed branch.
879
+ const states = [...new Set([...this.runs.values(), ...this.retainedCleanupStates])];
880
+ const targets = states.flatMap((state) => {
881
+ const owners = new Map<string, { agentId: string; owner: WorkflowOwner; generation: number }>();
882
+ for (const pending of state.pendingSpawns.values()) {
883
+ if (pending.phase !== "submitted") continue;
884
+ owners.set(JSON.stringify(pending.owner), {
885
+ agentId: pending.agentId ?? pending.startedAgentId ?? pending.spawnKey,
886
+ owner: { ...pending.owner },
887
+ generation: pending.executionGeneration,
888
+ });
889
+ }
890
+ for (const [agentId, waiter] of state.agentWaiters) {
891
+ const owner: WorkflowOwner = {
892
+ extension: "pi-workflows",
632
893
  runId: state.run.runId,
633
- nodeId: waiter ? `call-${waiter.callIndex}` : "",
634
- attemptId: `attempt-${waiter?.generation ?? 1}`,
894
+ nodeId: `call-${waiter.callIndex}`,
895
+ attemptId: `attempt-${waiter.generation}`,
635
896
  };
636
- });
637
- targets.set(state.run.runId, { agentIds, owners });
638
- state.lifecycleSuspended = true;
639
- state.bufferedTerminals.clear();
640
- state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
641
- }
642
-
643
- const operation = (async () => {
644
- const operations: Array<Promise<WorkflowQuiesceResult>> = [];
645
- for (const state of this.runs.values()) {
646
- const run = state.run;
647
- const target = targets.get(run.runId);
648
- if (isTerminalWorkflow(run.status)) {
649
- this.reconcilePendingSpawns(state);
650
- continue;
651
- }
652
- if (state.pendingSpawns.size > 0) operations.push(this.quiescePendingSpawns(state));
653
- const activeAgentIds = target?.agentIds ?? [];
654
- if (activeAgentIds.length === 0) continue;
655
- operations.push(
656
- this.client
657
- .quiesceOwned?.(run.runId, activeAgentIds, BRANCH_QUIESCE_TIMEOUT_MS, target?.owners)
658
- .then((result) => ({
659
- settled: result.settled && result.pending.length === 0,
660
- pending: [...new Set(result.pending)].slice(0, 256),
661
- }))
662
- .catch((error: unknown) => ({
663
- settled: false,
664
- pending: activeAgentIds.slice(0, 256),
665
- diagnostic: error instanceof Error ? error.message : String(error),
666
- })) ?? Promise.resolve({ settled: false, pending: activeAgentIds.slice(0, 256) }),
667
- );
897
+ owners.set(JSON.stringify(owner), { agentId, owner, generation: waiter.executionGeneration });
668
898
  }
669
- const results = await Promise.all(operations);
670
- const pending = [...new Set(results.flatMap((r) => r.pending))].slice(0, 256);
671
- const diagnostics = results.map((r) => r.diagnostic).filter((d): d is string => Boolean(d));
672
- return {
673
- settled: results.every((r) => r.settled) && pending.length === 0,
674
- pending,
675
- ...(diagnostics.length > 0 ? { diagnostic: diagnostics.join("; ") } : {}),
676
- };
677
- })();
899
+ for (const cleanup of this.cleanupObligations.get(state)?.values() ?? []) {
900
+ owners.set(JSON.stringify(cleanup.owner), {
901
+ agentId: cleanup.agentId,
902
+ owner: { ...cleanup.owner },
903
+ generation: state.executionGeneration,
904
+ });
905
+ }
906
+ return [...owners.values()].map((target) => ({ state, ...target }));
907
+ });
908
+ // Publish the handshake before RPC, abort or inspection can re-enter.
909
+ let acceptOperations!: (operations: Promise<void>[]) => void;
910
+ const operations = new Promise<Promise<void>[]>((resolve) => {
911
+ acceptOperations = resolve;
912
+ });
913
+ const operation = operations.then(async (cleanup) => {
914
+ await Promise.all(cleanup);
915
+ const pending = [
916
+ ...new Set(
917
+ states.flatMap((state) =>
918
+ [...(this.cleanupObligations.get(state)?.values() ?? [])].map((cleanup) => cleanup.agentId),
919
+ ),
920
+ ),
921
+ ].slice(0, 256);
922
+ return { settled: pending.length === 0, pending };
923
+ });
678
924
  const tracked = operation.then(
679
925
  (result) => {
680
926
  if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
@@ -686,6 +932,34 @@ export class WorkflowEngine {
686
932
  },
687
933
  );
688
934
  this.branchQuiescePromise = tracked;
935
+ // Fence every captured state before the first notifying observer. Start
936
+ // known-id owner checks synchronously for the bus-first shutdown handshake;
937
+ // their positive result is consumed by the same stable cleanup obligation,
938
+ // never treated as a status-only shortcut or a separate donated lease.
939
+ for (const state of states) state.lifecycleSuspended = true;
940
+ acceptOperations(
941
+ targets.map(({ state, agentId, owner, generation }) => {
942
+ const existing = [...(this.cleanupObligations.get(state)?.values() ?? [])].find((cleanup) =>
943
+ sameWorkflowOwner(cleanup.owner, owner),
944
+ );
945
+ let quiescence: Promise<{ settled: boolean; pending: string[] }> | undefined;
946
+ if (!existing?.operation && agentId !== `${owner.runId}/${owner.nodeId}/${owner.attemptId}`) {
947
+ try {
948
+ quiescence = this.client.quiesceOwned?.(owner.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner]);
949
+ } catch (error) {
950
+ quiescence = Promise.reject(error);
951
+ }
952
+ void quiescence?.catch(() => {});
953
+ }
954
+ return this.stopAndQuiesce(state, agentId, owner, generation, undefined, quiescence);
955
+ }),
956
+ );
957
+ for (const state of states) {
958
+ state.bufferedTerminals.clear();
959
+ state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
960
+ state.presentation.reset(false);
961
+ this.notifyInspection(state);
962
+ }
689
963
  return tracked;
690
964
  }
691
965
 
@@ -780,8 +1054,14 @@ export class WorkflowEngine {
780
1054
  ...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
781
1055
  timestamp: now,
782
1056
  });
1057
+ if (epoch !== this.branchEpoch || this.disposed || this.lifecyclePaused || this.admissionsClosed)
1058
+ throw new Error("Workflow branch changed before admission");
783
1059
  const state: ScriptRunState = {
784
1060
  run,
1061
+ presentation: new WorkflowPresentation(runId),
1062
+ managedCallFacts: new Map(),
1063
+ pendingCleanup: new Set(),
1064
+ activeExecutionGenerations: new Set(),
785
1065
  frozenOptions: {
786
1066
  ...(frozenTokenBudget !== undefined ? { tokenBudget: frozenTokenBudget } : {}),
787
1067
  ...(frozenMaxAgents !== undefined ? { maxAgents: frozenMaxAgents } : {}),
@@ -802,13 +1082,19 @@ export class WorkflowEngine {
802
1082
  executionGeneration: 0,
803
1083
  };
804
1084
  this.runs.set(runId, state);
805
- this.setWorkflowStatus(run, "running");
806
- state.executionGeneration += 1;
807
- state.execution = this.execute(runId, script, startOptions, undefined, state.executionGeneration, protocol);
1085
+ // Accept identity before any synchronous running/root observer can stop or
1086
+ // replace this state. Dispatch is deferred until its promise is published.
1087
+ const generation = ++state.executionGeneration;
1088
+ state.presentation.reset();
1089
+ state.execution = Promise.resolve().then(() =>
1090
+ this.execute(state, script, startOptions, undefined, generation, epoch, protocol),
1091
+ );
808
1092
  const execution = state.execution;
809
1093
  // A run-fatal abort/stop already journals its terminal state; never let a
810
1094
  // leftover rejection become an unhandled promise rejection.
811
- void state.execution.catch(() => {});
1095
+ void execution.catch(() => {});
1096
+ this.setWorkflowStatus(run, "running");
1097
+ if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
812
1098
  if (options.background) return { runId, status: run.status, background: true };
813
1099
  try {
814
1100
  const settled = await this.waitFor(runId, options.signal);
@@ -859,7 +1145,7 @@ export class WorkflowEngine {
859
1145
  throw new Error("Workflow branch changed before admission");
860
1146
  const state = this.runs.get(runId);
861
1147
  if (!state) return undefined;
862
- if (state.lifecycleSuspended) return undefined;
1148
+ if (state.lifecycleSuspended || state.removeOperation) return undefined;
863
1149
  const run = state.run;
864
1150
  if (run.status === "running" || run.status === "pausing" || run.status === "stopping") return undefined;
865
1151
  const providerTimer = this.providerResumeTimers.get(runId);
@@ -867,8 +1153,53 @@ export class WorkflowEngine {
867
1153
  clearTimeout(providerTimer);
868
1154
  this.providerResumeTimers.delete(runId);
869
1155
  }
870
- if (run.nonResumable || (isTerminalWorkflow(run.status) && run.status !== "failed" && run.status !== "stopped"))
1156
+ if (
1157
+ run.nonResumable ||
1158
+ run.terminalIntent === "stop" ||
1159
+ (isTerminalWorkflow(run.status) && run.status !== "failed" && run.status !== "stopped")
1160
+ )
871
1161
  return undefined;
1162
+ if (isTerminalWorkflow(run.status)) {
1163
+ await this.drainExecution(state, state.execution);
1164
+ for (const [nodeId, agentId] of Object.entries(run.agentIds)) {
1165
+ if (run.callStatus[nodeId] !== "running") continue; // Completed/local calls own no live child.
1166
+ const attemptId = run.attemptIds[nodeId];
1167
+ if (!attemptId) continue;
1168
+ await this.stopAndQuiesce(
1169
+ state,
1170
+ agentId,
1171
+ {
1172
+ extension: "pi-workflows",
1173
+ runId,
1174
+ nodeId: `call-${nodeId}`,
1175
+ attemptId: `attempt-${generationFromAttemptId(attemptId)}`,
1176
+ },
1177
+ state.executionGeneration,
1178
+ );
1179
+ }
1180
+ if (run.nonResumable || this.runs.get(runId) !== state || epoch !== this.branchEpoch || this.lifecyclePaused)
1181
+ return undefined;
1182
+ if (
1183
+ this.journalBlockedRuns.has(runId) ||
1184
+ [...this.journalRetries.keys()].some((key) => key.startsWith(`${runId}\u0000`))
1185
+ )
1186
+ throw new Error("Workflow journal has not settled; retry resume after persistence recovers");
1187
+ const timestamp = Date.now();
1188
+ const recovery: RunRecoveryEvent = {
1189
+ kind: "run_recovery",
1190
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
1191
+ runId,
1192
+ status: "interrupted",
1193
+ branchGeneration: this.recoveryBranchGeneration,
1194
+ rotations: [],
1195
+ recoveryId: deriveRecoveryId({ runId, rotations: [], timestamp }),
1196
+ timestamp,
1197
+ };
1198
+ this.appendRequired(recovery);
1199
+ if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
1200
+ applyRecoveryEvent(run, recovery);
1201
+ state.result = undefined;
1202
+ }
872
1203
  if (replacementScript !== undefined) {
873
1204
  const parsed = parseWorkflowScript(replacementScript);
874
1205
  const revision = (run.revision ?? 0) + 1;
@@ -889,6 +1220,7 @@ export class WorkflowEngine {
889
1220
  }),
890
1221
  );
891
1222
  }
1223
+ if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
892
1224
  // Arguments are frozen in every schema-v4 run_created fact. Resume never accepts
893
1225
  // replacement args and never backfills the pre-schema-v4 run_args contract.
894
1226
  const resumeArgs = cloneWorkflowArgs(run.args);
@@ -898,8 +1230,6 @@ export class WorkflowEngine {
898
1230
  for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
899
1231
  state.generations.set(callIndex, generationFromAttemptId(attemptId));
900
1232
  }
901
- this.setWorkflowStatus(run, "running");
902
- state.controller = new AbortController();
903
1233
  // Freeze semantics: original run parameters and args win over new options.
904
1234
  // Only signal/confirm/background/loadSavedWorkflow/mainModel/strengths are
905
1235
  // per-resume context; input/budget/scale/toolset are frozen from the
@@ -911,6 +1241,7 @@ export class WorkflowEngine {
911
1241
  // a caller that passed none is asking the engine to read the current one.
912
1242
  strengths: options.strengths ?? this.strengths?.(),
913
1243
  args: resumeArgs,
1244
+ loadSavedWorkflow: options.loadSavedWorkflow ?? this.loadSavedWorkflow,
914
1245
  tokenBudget: frozen.tokenBudget,
915
1246
  maxAgents: frozen.maxAgents,
916
1247
  concurrency: frozen.concurrency,
@@ -919,19 +1250,25 @@ export class WorkflowEngine {
919
1250
  toolset: frozen.toolset,
920
1251
  excludeTools: frozen.excludeTools ? [...frozen.excludeTools] : undefined,
921
1252
  };
1253
+ if (!this.isCurrentState(state, state.executionGeneration, epoch) || run.nonResumable || state.stopOperation)
1254
+ return undefined;
922
1255
  state.bufferedTerminals.clear();
923
- state.executionGeneration += 1;
924
- state.execution = this.execute(
925
- runId,
926
- run.script,
927
- resumeOptions,
928
- resumeJournal,
929
- state.executionGeneration,
930
- protocol,
1256
+ this.terminalNotified.delete(state);
1257
+ state.controller = new AbortController();
1258
+ const generation = ++state.executionGeneration;
1259
+ const acceptedScript = run.script;
1260
+ state.presentation.reset();
1261
+ state.execution = Promise.resolve().then(() =>
1262
+ this.execute(state, acceptedScript, resumeOptions, resumeJournal, generation, epoch, protocol),
931
1263
  );
932
1264
  const execution = state.execution;
933
1265
  void execution.catch(() => {});
934
- if (options.background) return { runId, status: run.status, background: true };
1266
+ this.setWorkflowStatus(run, "running");
1267
+ if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
1268
+ if (options.background) {
1269
+ this.onBackgroundExecution?.(runId);
1270
+ return { runId, status: run.status, background: true };
1271
+ }
935
1272
  try {
936
1273
  const settled = await this.waitFor(runId, options.signal);
937
1274
  return { runId, status: settled.status, background: false };
@@ -986,11 +1323,7 @@ export class WorkflowEngine {
986
1323
  await this.stop(state);
987
1324
  } else if (action === "rm") {
988
1325
  if (!isTerminalWorkflow(run.status)) throw new Error("cannot remove a running workflow");
989
- this.persist(runId, "run-removed", () =>
990
- this.write({ kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp: Date.now() }),
991
- );
992
- this.runs.delete(runId);
993
- this.waiters.delete(runId);
1326
+ await this.remove(state);
994
1327
  return { action, run: this.summary(run) };
995
1328
  }
996
1329
  return { action, run: this.summary(run) };
@@ -999,6 +1332,7 @@ export class WorkflowEngine {
999
1332
  async waitFor(runId: string, signal?: AbortSignal): Promise<ScriptRun> {
1000
1333
  const state = this.runs.get(runId);
1001
1334
  if (!state) throw new Error(`workflow run not found: ${runId}`);
1335
+ if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size > 0) throw new WorkflowWaitAbortedError();
1002
1336
  if (isTerminalWorkflow(state.run.status) || state.run.status === "paused" || state.run.status === "interrupted")
1003
1337
  return state.run;
1004
1338
  if (signal?.aborted) throw new WorkflowWaitAbortedError();
@@ -1047,6 +1381,7 @@ export class WorkflowEngine {
1047
1381
  }
1048
1382
 
1049
1383
  private settleWaiters(runId: string, run: ScriptRun): void {
1384
+ if (this.runs.get(runId)?.run !== run || this.hasPendingCleanup(runId)) return;
1050
1385
  const all = this.waiters.get(runId);
1051
1386
  if (!all) return;
1052
1387
  for (const waiter of [...all]) {
@@ -1062,24 +1397,24 @@ export class WorkflowEngine {
1062
1397
  // ── execution ──────────────────────────────────────────────────────────────
1063
1398
 
1064
1399
  private async execute(
1065
- runId: string,
1400
+ state: ScriptRunState,
1066
1401
  script: string,
1067
1402
  options: ScriptStartOptions,
1068
- resumeJournal?: Map<string, JournalEntry>,
1069
- executionGeneration?: number,
1403
+ resumeJournal: Map<string, JournalEntry> | undefined,
1404
+ generation: number,
1405
+ epoch: number,
1070
1406
  protocol?: ManagedProtocolCheck,
1071
1407
  ): Promise<WorkflowRunResult<unknown>> {
1072
- if (this.disposed) throw new WorkflowEngineDisposedError();
1073
- const state = this.runs.get(runId);
1074
- if (!state) throw new Error(`workflow run not found: ${runId}`);
1075
- const generation = executionGeneration ?? state.executionGeneration;
1076
- const epoch = this.branchEpoch;
1408
+ const runId = state.run.runId;
1409
+ state.activeExecutionGenerations.add(generation);
1077
1410
  const isSameBranch = () =>
1078
1411
  this.runs.get(runId) === state &&
1079
1412
  state.executionGeneration === generation &&
1080
1413
  this.branchEpoch === epoch &&
1081
1414
  !this.disposed;
1082
1415
  const isCurrent = () => isSameBranch() && !state.lifecycleSuspended && !this.lifecyclePaused;
1416
+ const isPresentationCurrent = () => isCurrent() && state.activeExecutionGenerations.has(generation);
1417
+ const explicitlyStopped = () => state.stopOperation !== undefined || state.run.terminalIntent === "stop";
1083
1418
  const executionSignal =
1084
1419
  !options.background && options.signal
1085
1420
  ? AbortSignal.any([state.controller.signal, options.signal])
@@ -1088,7 +1423,8 @@ export class WorkflowEngine {
1088
1423
  const cleanupTasks = new Set<Promise<void>>();
1089
1424
  const runner: WorkflowAgentRunner = {
1090
1425
  run: async (prompt: string, runOptions: AgentRunOptions = {}) => {
1091
- if (!isCurrent()) throw new WorkflowError("stale workflow execution", WorkflowErrorCode.WORKFLOW_ABORTED);
1426
+ if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
1427
+ throw new WorkflowError("stale workflow execution", WorkflowErrorCode.WORKFLOW_ABORTED);
1092
1428
  const dispatch = this.dispatchAgent(runId, prompt, runOptions);
1093
1429
  const settled = dispatch.then(
1094
1430
  () => {},
@@ -1102,6 +1438,15 @@ export class WorkflowEngine {
1102
1438
  if (runOptions.signal?.aborted) trackCleanup();
1103
1439
  try {
1104
1440
  return await dispatch;
1441
+ } catch (error: unknown) {
1442
+ if (isCurrent()) {
1443
+ const frameId = runOptions.runId ?? runId;
1444
+ const nodeId =
1445
+ frameId === runId ? String(runOptions.callIndex ?? 0) : `${frameId}:${runOptions.callIndex ?? 0}`;
1446
+ state.presentation.managed(nodeId, { status: runOptions.signal?.aborted ? "stopped" : "failed" });
1447
+ this.notifyInspection(state);
1448
+ }
1449
+ throw error;
1105
1450
  } finally {
1106
1451
  runOptions.signal?.removeEventListener("abort", trackCleanup);
1107
1452
  }
@@ -1110,6 +1455,8 @@ export class WorkflowEngine {
1110
1455
  };
1111
1456
 
1112
1457
  try {
1458
+ if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
1459
+ throw new WorkflowError("workflow execution aborted before dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1113
1460
  const result = await runWorkflow(script, {
1114
1461
  cwd: this.cwd,
1115
1462
  args: options.args,
@@ -1134,29 +1481,74 @@ export class WorkflowEngine {
1134
1481
  runId,
1135
1482
  executionNonce: `${generation}-${randomUUID()}`,
1136
1483
  resumeJournal,
1484
+ resumeWorkflowGenerations: new Map(
1485
+ Object.entries(state.run.workflowResultGenerations).map(([index, generation]) => [Number(index), generation]),
1486
+ ),
1137
1487
  toolset: options.toolset,
1138
1488
  strengths: options.strengths,
1139
1489
  excludeTools: options.excludeTools,
1140
- confirm: options.confirm,
1490
+ confirm:
1491
+ options.confirm &&
1492
+ (async (prompt, checkpointOptions, signal) => {
1493
+ if (!isCurrent() || executionSignal.aborted || signal?.aborted || state.stopOperation)
1494
+ throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
1495
+ const value = await options.confirm?.(prompt, checkpointOptions, signal);
1496
+ if (!isCurrent() || executionSignal.aborted || signal?.aborted)
1497
+ throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
1498
+ return value;
1499
+ }),
1141
1500
  loadSavedWorkflow: options.loadSavedWorkflow,
1501
+ onAgentStart: (event) => {
1502
+ if (!isPresentationCurrent()) return;
1503
+ state.presentation.agentStart(event);
1504
+ this.notifyInspection(state);
1505
+ },
1506
+ onAgentEnd: (event) => {
1507
+ if (!isPresentationCurrent()) return;
1508
+ const nodeId = event.id.startsWith(`${runId}:`) ? event.id.slice(runId.length + 1) : undefined;
1509
+ const persisted = nodeId === undefined ? undefined : state.run.callResults[nodeId];
1510
+ const persistedTokens =
1511
+ nodeId !== undefined && persisted?.attemptId === state.run.attemptIds[nodeId]
1512
+ ? persisted?.tokenCount
1513
+ : undefined;
1514
+ // Runtime replay reports zero new spend, not the historical call's
1515
+ // usage. Explicit managed zero is also authoritative over estimates.
1516
+ const tokenCount = event.tokenUsage?.total ?? persistedTokens ?? event.tokens;
1517
+ state.presentation.agentEnd({ ...event, tokens: tokenCount });
1518
+ this.notifyInspection(state);
1519
+ },
1520
+ onLog: (message, scope) => {
1521
+ if (!isPresentationCurrent()) return;
1522
+ state.presentation.log(message, scope?.frameId);
1523
+ this.notifyInspection(state);
1524
+ },
1142
1525
  onAgentJournal: (entry) => {
1143
1526
  if (!isCurrent()) return;
1144
- // Nested workflow frames keep their own lifecycle visibility, but their
1145
- // individual call facts are not top-level replay keys. The parent
1146
- // workflow_result fact below owns the nested replay boundary.
1527
+ // Persist the accepted owner before presentation observers can replace it.
1528
+ // Nested frames are owned by their parent's workflow_result boundary.
1147
1529
  if (entry.runId === undefined || entry.runId === runId) this.journalCallResult(runId, entry);
1530
+ if (!isCurrent()) return;
1531
+ state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
1532
+ this.notifyInspection(state);
1148
1533
  },
1149
1534
  onWorkflowJournal: (entry) => {
1150
1535
  if (!isCurrent()) return;
1151
1536
  this.journalWorkflowResult(runId, entry);
1537
+ if (!isCurrent()) return;
1538
+ state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
1539
+ this.notifyInspection(state);
1152
1540
  },
1153
1541
  onRuntimeEvent: (event) => {
1154
- if (!isCurrent()) return;
1542
+ if (!isPresentationCurrent()) return;
1543
+ state.presentation.runtime(event);
1544
+ this.notifyInspection(state);
1545
+ if (!isPresentationCurrent()) return;
1155
1546
  try {
1156
1547
  this.events.emit("pi-workflows:runtime", { runId, event: cloneRuntimeEvent(event) });
1157
1548
  } catch {
1158
1549
  // Cross-extension observers cannot change workflow execution.
1159
1550
  }
1551
+ if (!isPresentationCurrent()) return;
1160
1552
  try {
1161
1553
  this.onRuntimeEvent?.(runId, cloneRuntimeEvent(event));
1162
1554
  } catch {
@@ -1170,6 +1562,19 @@ export class WorkflowEngine {
1170
1562
  recoverable: true,
1171
1563
  });
1172
1564
  }
1565
+ // A terminal frame cannot overtake accepted journal facts. In particular,
1566
+ // a delayed call_attempt must precede its result, not reset it on replay.
1567
+ await this.drainJournal(runId);
1568
+ if (
1569
+ executionSignal.aborted ||
1570
+ !isCurrent() ||
1571
+ state.run.status === "stopping" ||
1572
+ state.stopOperation !== undefined ||
1573
+ state.run.terminalIntent === "stop"
1574
+ )
1575
+ throw new WorkflowError("workflow execution aborted", WorkflowErrorCode.WORKFLOW_ABORTED, {
1576
+ recoverable: true,
1577
+ });
1173
1578
  const live = this.runs.get(runId);
1174
1579
  const run = live?.run;
1175
1580
  if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
@@ -1180,8 +1585,8 @@ export class WorkflowEngine {
1180
1585
  } catch (error: unknown) {
1181
1586
  await runner.waitForCleanup?.();
1182
1587
  const run = this.runs.get(runId)?.run;
1183
- if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
1184
- const message = error instanceof Error ? error.message : String(error);
1588
+ if (run && isSameBranch() && !isTerminalWorkflow(run.status) && run.status !== "stopping") {
1589
+ const message = describeWorkflowError(error);
1185
1590
  const workflowError = error instanceof WorkflowError ? error : wrapError(error);
1186
1591
  if (workflowError.code === WorkflowErrorCode.PROVIDER_USAGE_LIMIT) {
1187
1592
  this.setWorkflowStatus(run, "paused", message);
@@ -1203,15 +1608,22 @@ export class WorkflowEngine {
1203
1608
  run.status = "interrupted";
1204
1609
  run.updatedAt = Date.now();
1205
1610
  } else {
1206
- this.setWorkflowStatus(run, "failed", message);
1207
1611
  this.cleanupAgents(runId);
1612
+ this.setWorkflowStatus(run, "failed", message);
1208
1613
  }
1209
1614
  }
1210
1615
  throw error;
1211
1616
  } finally {
1617
+ state.activeExecutionGenerations.delete(generation);
1212
1618
  // Reject any waiters that never settled (agents left in flight).
1213
1619
  const run = this.runs.get(runId)?.run;
1214
- if (run && isSameBranch()) this.settleWaiters(runId, run);
1620
+ if (run && isSameBranch()) {
1621
+ if (isCurrent()) {
1622
+ state.presentation.settle(run.status);
1623
+ this.notifyInspection(state);
1624
+ }
1625
+ if (isSameBranch() && run.status !== "stopping") this.settleWaiters(runId, run);
1626
+ }
1215
1627
  }
1216
1628
  }
1217
1629
 
@@ -1268,6 +1680,7 @@ export class WorkflowEngine {
1268
1680
  const tierIdentity: CallTierIdentity = tier === undefined ? {} : { tier };
1269
1681
  state.callTiers.set(nodeId, tierIdentity);
1270
1682
  run.callTiers[nodeId] = tierIdentity;
1683
+ state.managedCallFacts.delete(nodeId);
1271
1684
  const task: DispatchTask = {
1272
1685
  subagent_type: type,
1273
1686
  prompt,
@@ -1283,6 +1696,15 @@ export class WorkflowEngine {
1283
1696
  // represented by its parent workflow_result replay boundary instead of
1284
1697
  // leaking child call facts into the parent's numeric call namespace.
1285
1698
  if (!nested) {
1699
+ // Match journal replay's call_attempt semantics: historical results stay
1700
+ // durable, but are not current facts for this accepted live rotation.
1701
+ run.callStatus[nodeId] = "running";
1702
+ run.attempts[nodeId] = generation;
1703
+ run.attemptIds[nodeId] = attemptId;
1704
+ delete run.agentIds[nodeId];
1705
+ delete run.callResults[nodeId];
1706
+ run.compactions[nodeId] = 0;
1707
+ run.updatedAt = Date.now();
1286
1708
  this.persist(runId, `call-attempt:${nodeId}`, () =>
1287
1709
  this.write({
1288
1710
  kind: "call_attempt",
@@ -1296,10 +1718,12 @@ export class WorkflowEngine {
1296
1718
  timestamp: Date.now(),
1297
1719
  }),
1298
1720
  );
1299
- run.callStatus[nodeId] = "running";
1300
- run.attempts[nodeId] = generation;
1301
- run.attemptIds[nodeId] = attemptId;
1302
- run.updatedAt = Date.now();
1721
+ if (
1722
+ !this.isCurrentState(state, executionGeneration, dispatchEpoch) ||
1723
+ runOptions.signal?.aborted ||
1724
+ state.controller.signal.aborted
1725
+ )
1726
+ throw new WorkflowError("stale workflow attempt", WorkflowErrorCode.WORKFLOW_ABORTED);
1303
1727
  }
1304
1728
 
1305
1729
  // Abort linkage: an external signal (pause/stop/Esc) cancels the wait and
@@ -1313,214 +1737,386 @@ export class WorkflowEngine {
1313
1737
  nodeId: spawnNodeId,
1314
1738
  attemptId: spawnAttemptId,
1315
1739
  };
1316
- state.pendingSpawns.set(spawnKey, { spawnKey, owner, executionGeneration });
1317
- const reconcilePending = () => {
1318
- state.bufferedTerminals.delete(spawnKey);
1319
- const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
1320
- void reconciliation?.catch(() => {});
1740
+ const pending = {
1741
+ spawnKey,
1742
+ owner,
1743
+ executionGeneration,
1744
+ phase: "reserved" as "reserved" | "submitted",
1745
+ startedAgentId: undefined as string | undefined,
1746
+ agentId: undefined as string | undefined,
1321
1747
  };
1322
- const onPendingAbort = () => reconcilePending();
1323
- externalSignal.addEventListener("abort", onPendingAbort, { once: true });
1324
- if (externalSignal.aborted) onPendingAbort();
1325
-
1326
- let pendingId = "";
1327
- let response: ManagedSpawnResponse | string;
1748
+ // Keep this exact allocation lease through every publication callback and
1749
+ // terminal delivery. A waiter alone is insufficient: it is removed before
1750
+ // onUsage/onHistory, which may synchronously pause or replace the branch.
1751
+ state.pendingSpawns.set(spawnKey, pending);
1752
+ let responseId: string | undefined;
1753
+ let terminalAccepted = false;
1754
+ let ownedCleanup: Promise<void> | undefined;
1755
+ const cleanupResponse = () =>
1756
+ (ownedCleanup ??= this.stopAndQuiesce(
1757
+ state,
1758
+ responseId ?? pending.startedAgentId ?? spawnKey,
1759
+ owner,
1760
+ executionGeneration,
1761
+ ));
1328
1762
  try {
1329
- response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, externalSignal);
1330
- } catch (error: unknown) {
1331
- if (externalSignal.aborted) {
1332
- const reconciled = await this.client.reconcileManaged?.(spawnKey, owner).catch(() => undefined);
1333
- const id = typeof reconciled === "string" ? reconciled : reconciled?.id;
1334
- await this.stopAndQuiesce(
1335
- state,
1336
- id ?? spawnKey,
1337
- owner,
1338
- executionGeneration,
1339
- id ? undefined : "cancelled spawn allocation could not be confirmed",
1340
- );
1763
+ state.presentation.dispatch(runOptions, attemptId);
1764
+ this.notifyInspection(state);
1765
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1766
+ throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1767
+ const reconcilePending = () => {
1768
+ state.bufferedTerminals.delete(spawnKey);
1769
+ const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
1770
+ void reconciliation?.catch(() => {});
1771
+ };
1772
+ const onPendingAbort = () => reconcilePending();
1773
+ externalSignal.addEventListener("abort", onPendingAbort, { once: true });
1774
+ if (externalSignal.aborted) onPendingAbort();
1775
+
1776
+ let pendingId = "";
1777
+ let response: ManagedSpawnResponse | string;
1778
+ try {
1779
+ // Reservation is local until this exact boundary. Observers above can
1780
+ // cancel without creating uncertainty for an RPC that was never sent.
1781
+ pending.phase = "submitted";
1782
+ response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, externalSignal);
1783
+ } catch (error: unknown) {
1784
+ // Transport failure is not evidence that the host never allocated. Keep
1785
+ // the exact spawn owner even without a reply or an aborted signal.
1786
+ await cleanupResponse();
1787
+ throw error;
1788
+ } finally {
1789
+ externalSignal.removeEventListener("abort", onPendingAbort);
1341
1790
  }
1342
- throw error;
1343
- } finally {
1344
- externalSignal.removeEventListener("abort", onPendingAbort);
1345
- state.pendingSpawns.delete(spawnKey);
1346
- }
1347
- const responseId = typeof response === "string" ? response : response.id;
1348
- if (
1349
- state.executionGeneration !== executionGeneration ||
1350
- this.runs.get(runId) !== state ||
1351
- this.branchEpoch !== dispatchEpoch ||
1352
- state.lifecycleSuspended ||
1353
- this.lifecyclePaused ||
1354
- state.controller.signal.aborted ||
1355
- externalSignal.aborted
1356
- ) {
1357
- state.bufferedTerminals.delete(spawnKey);
1358
- await this.stopAndQuiesce(state, responseId, owner, executionGeneration);
1359
- throw new WorkflowError(
1360
- "workflow dispatch became stale during pause/resume",
1361
- WorkflowErrorCode.WORKFLOW_ABORTED,
1362
- { recoverable: true },
1363
- );
1364
- }
1365
- if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
1366
- this.persistCallTierIdentity(state, nodeId);
1367
- }
1368
- // Report the tier the host actually selected back to the runtime, which
1369
- // journals it: a call that named none still resolved to something, and the
1370
- // run's own record of what ran should say what.
1371
- const resolvedTierIdentity = state.callTiers.get(nodeId);
1372
- if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
1373
- if (typeof response === "string") {
1374
- pendingId = response;
1375
- } else if (response.terminal) {
1376
- state.bufferedTerminals.delete(spawnKey);
1377
- // The spawn response already carries a terminal snapshot (fast completion
1378
- // or interrupted replay). The runtime journals successful calls itself
1379
- // (onAgentJournal); a terminal snapshot that is NOT a success is
1380
- // journaled here so the resume journal reflects the failure.
1381
- const terminal = response.terminal;
1382
- if (typeof terminal.tokenCount === "number")
1383
- runOptions.onUsage?.({
1384
- input: 0,
1385
- output: terminal.tokenCount,
1386
- total: terminal.tokenCount,
1387
- cost: 0,
1388
- cacheRead: 0,
1389
- cacheWrite: 0,
1390
- });
1391
- const status = managedTerminalStatus(terminal);
1392
- if (status !== "completed") {
1393
- const result = resultFromLifecycle(
1394
- status,
1395
- response.id,
1396
- terminal.result,
1397
- terminal.error,
1398
- terminal.compactionCount,
1399
- terminal.tokenCount,
1400
- resolvedTierIdentity?.tier ?? tier,
1791
+ responseId = typeof response === "string" ? response : response.id;
1792
+ pending.agentId = responseId;
1793
+ if (
1794
+ state.executionGeneration !== executionGeneration ||
1795
+ this.runs.get(runId) !== state ||
1796
+ this.branchEpoch !== dispatchEpoch ||
1797
+ state.lifecycleSuspended ||
1798
+ this.lifecyclePaused ||
1799
+ state.controller.signal.aborted ||
1800
+ externalSignal.aborted
1801
+ ) {
1802
+ state.bufferedTerminals.delete(spawnKey);
1803
+ await cleanupResponse();
1804
+ throw new WorkflowError(
1805
+ "workflow dispatch became stale during pause/resume",
1806
+ WorkflowErrorCode.WORKFLOW_ABORTED,
1807
+ { recoverable: true },
1401
1808
  );
1402
- result.attemptId = attemptId;
1403
- if (!nested) this.commitCallTerminal(runId, nodeId, result);
1404
- if (status === "failed" && terminal.error) {
1405
- throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
1406
- }
1407
1809
  }
1408
- return terminal.result ?? "";
1409
- } else {
1410
- pendingId = response.id;
1411
- }
1412
-
1413
- const waitAbort = () => {
1414
- const waiter = state.agentWaiters.get(pendingId);
1415
- if (waiter && !waiter.settled) {
1416
- waiter.settled = true;
1417
- state.agentWaiters.delete(pendingId);
1810
+ if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
1811
+ this.persistCallTierIdentity(state, nodeId);
1812
+ }
1813
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1814
+ throw new WorkflowError("stale workflow identity", WorkflowErrorCode.WORKFLOW_ABORTED);
1815
+ this.recordManagedAdmission(state, nodeId, responseId, executionGeneration);
1816
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1817
+ throw new WorkflowError("stale workflow admission", WorkflowErrorCode.WORKFLOW_ABORTED);
1818
+ if (typeof response !== "string" && response.terminal)
1819
+ this.recordManagedTerminal(state, nodeId, responseId, response.terminal, executionGeneration);
1820
+ const bufferedTerminal = state.bufferedTerminals.get(spawnKey);
1821
+ const observedTerminal =
1822
+ bufferedTerminal?.response.id === responseId &&
1823
+ bufferedTerminal.executionGeneration === executionGeneration &&
1824
+ sameWorkflowOwner(bufferedTerminal.owner, owner)
1825
+ ? bufferedTerminal.response.terminal
1826
+ : undefined;
1827
+ const observedRunning = pending.startedAgentId === responseId;
1828
+ state.presentation.managed(nodeId, {
1829
+ agentId: responseId,
1830
+ status: observedTerminal
1831
+ ? managedTerminalStatus(observedTerminal)
1832
+ : observedRunning
1833
+ ? "running"
1834
+ : typeof response === "string"
1835
+ ? "pending"
1836
+ : response.state === "queued"
1837
+ ? "queued"
1838
+ : "running",
1839
+ tier: state.callTiers.get(nodeId)?.tier,
1840
+ ...(typeof response !== "string" && response.terminal
1841
+ ? {
1842
+ status: managedTerminalStatus(response.terminal),
1843
+ tokenCount: response.terminal.tokenCount,
1844
+ outputFile: response.terminal.outputFile,
1845
+ compactionCount: Math.max(
1846
+ response.terminal.compactionCount,
1847
+ state.managedCallFacts.get(nodeId)?.compactionCount ?? 0,
1848
+ ),
1849
+ }
1850
+ : {}),
1851
+ });
1852
+ this.notifyInspection(state);
1853
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1854
+ throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1855
+ // Report the tier the host actually selected back to the runtime, which
1856
+ // journals it: a call that named none still resolved to something, and the
1857
+ // run's own record of what ran should say what.
1858
+ const resolvedTierIdentity = state.callTiers.get(nodeId);
1859
+ if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
1860
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1861
+ throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
1862
+ if (typeof response === "string") {
1863
+ pendingId = response;
1864
+ } else if (response.terminal) {
1418
1865
  state.bufferedTerminals.delete(spawnKey);
1419
- // Stop acknowledges cancellation, not provider/tool finally completion.
1420
- void this.stopAndQuiesce(state, pendingId, owner, executionGeneration).then(() => {
1421
- waiter.reject(
1422
- new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }),
1866
+ // The spawn response already carries a terminal snapshot (fast completion
1867
+ // or interrupted replay). The runtime journals successful calls itself
1868
+ // (onAgentJournal); a terminal snapshot that is NOT a success is
1869
+ // journaled here so the resume journal reflects the failure.
1870
+ const terminal = response.terminal;
1871
+ if (typeof terminal.tokenCount === "number")
1872
+ runOptions.onUsage?.({
1873
+ input: 0,
1874
+ output: terminal.tokenCount,
1875
+ total: terminal.tokenCount,
1876
+ cost: 0,
1877
+ cacheRead: 0,
1878
+ cacheWrite: 0,
1879
+ });
1880
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1881
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
1882
+ terminalAccepted = true;
1883
+ const status = managedTerminalStatus(terminal);
1884
+ if (status !== "completed") {
1885
+ const result = resultFromLifecycle(
1886
+ status,
1887
+ response.id,
1888
+ terminal.result,
1889
+ terminal.error,
1890
+ terminal.compactionCount,
1891
+ terminal.tokenCount,
1892
+ resolvedTierIdentity?.tier ?? tier,
1423
1893
  );
1424
- });
1894
+ result.attemptId = attemptId;
1895
+ if (!nested) this.commitCallTerminal(runId, nodeId, result);
1896
+ if (status === "failed" && terminal.error) {
1897
+ throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
1898
+ }
1899
+ }
1900
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
1901
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
1902
+ return terminal.result ?? "";
1903
+ } else {
1904
+ pendingId = response.id;
1425
1905
  }
1426
- };
1427
1906
 
1428
- // Wait for the terminal lifecycle event, consuming a matching terminal that
1429
- // arrived while the managed-spawn reply was still pending.
1430
- const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
1431
- const waiter: AgentLifecycleWaiter = {
1432
- callIndex: nodeId,
1433
- generation,
1434
- executionGeneration,
1435
- onUsage: runOptions.onUsage,
1436
- onHistory: runOptions.onHistory,
1437
- onRunning: runOptions.onRunning,
1438
- startedNotified: false,
1439
- settled: false,
1440
- resolve,
1441
- reject,
1907
+ const waitAbort = () => {
1908
+ const waiter = state.agentWaiters.get(pendingId);
1909
+ if (waiter && !waiter.settled) {
1910
+ waiter.settled = true;
1911
+ state.agentWaiters.delete(pendingId);
1912
+ state.bufferedTerminals.delete(spawnKey);
1913
+ // Stop acknowledges cancellation, not provider/tool finally completion.
1914
+ void cleanupResponse().then(async () => {
1915
+ if (
1916
+ !nested &&
1917
+ this.runs.get(runId) === state &&
1918
+ state.executionGeneration === executionGeneration &&
1919
+ !run.nonResumable
1920
+ ) {
1921
+ const result = resultFromLifecycle(
1922
+ "stopped",
1923
+ pendingId,
1924
+ undefined,
1925
+ "workflow call cancelled",
1926
+ run.compactions[nodeId] ?? 0,
1927
+ undefined,
1928
+ resolvedTierIdentity?.tier ?? tier,
1929
+ );
1930
+ result.attemptId = attemptId;
1931
+ try {
1932
+ await this.drainJournal(runId);
1933
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch)) {
1934
+ waiter.reject(new WorkflowWaitAbortedError());
1935
+ return;
1936
+ }
1937
+ this.commitCallTerminal(runId, nodeId, result, true);
1938
+ } catch (error) {
1939
+ this.fenceExecution(
1940
+ state,
1941
+ executionGeneration,
1942
+ "Could not persist cancelled call terminal fact; run is non-resumable",
1943
+ );
1944
+ waiter.reject(error);
1945
+ return;
1946
+ }
1947
+ }
1948
+ waiter.reject(
1949
+ new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }),
1950
+ );
1951
+ });
1952
+ }
1442
1953
  };
1443
- state.agentWaiters.set(pendingId, waiter);
1444
- // A spawn that was admitted straight away is already running, and its
1445
- // `subagents:started` event fired before this waiter existed. Only a
1446
- // reply that says "queued" still has a start to wait for.
1447
- if (typeof response !== "string" && response.state !== "queued") notifyAgentRunning(waiter);
1448
- externalSignal.addEventListener("abort", waitAbort, { once: true });
1449
- if (externalSignal.aborted) {
1450
- waitAbort();
1451
- return;
1452
- }
1453
1954
 
1454
- const buffered = state.bufferedTerminals.get(spawnKey);
1455
- if (!buffered) return;
1456
- state.bufferedTerminals.delete(spawnKey);
1955
+ // Wait for the terminal lifecycle event, consuming a matching terminal that
1956
+ // arrived while the managed-spawn reply was still pending.
1957
+ const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
1958
+ const waiter: AgentLifecycleWaiter = {
1959
+ callIndex: nodeId,
1960
+ generation,
1961
+ executionGeneration,
1962
+ onUsage: runOptions.onUsage,
1963
+ onHistory: runOptions.onHistory,
1964
+ onRunning: runOptions.onRunning,
1965
+ startedNotified: false,
1966
+ settled: false,
1967
+ resolve,
1968
+ reject,
1969
+ };
1970
+ state.agentWaiters.set(pendingId, waiter);
1971
+ // A spawn that was admitted straight away is already running, and its
1972
+ // `subagents:started` event fired before this waiter existed. Only a
1973
+ // reply that says "queued" still has a start to wait for.
1974
+ if (observedRunning || (typeof response !== "string" && response.state !== "queued"))
1975
+ notifyAgentRunning(waiter);
1976
+ externalSignal.addEventListener("abort", waitAbort, { once: true });
1977
+ if (externalSignal.aborted) {
1978
+ waitAbort();
1979
+ return;
1980
+ }
1981
+
1982
+ const buffered = state.bufferedTerminals.get(spawnKey);
1983
+ if (!buffered) return;
1984
+ state.bufferedTerminals.delete(spawnKey);
1985
+ if (
1986
+ buffered.executionGeneration !== executionGeneration ||
1987
+ state.executionGeneration !== executionGeneration ||
1988
+ state.lifecycleSuspended ||
1989
+ state.controller.signal.aborted ||
1990
+ buffered.response.id !== pendingId ||
1991
+ !sameWorkflowOwner(buffered.owner, owner)
1992
+ )
1993
+ return;
1994
+ this.resolveAgentWaiter(state, pendingId, waiter, buffered);
1995
+ }).finally(() => {
1996
+ externalSignal.removeEventListener("abort", waitAbort);
1997
+ });
1998
+
1457
1999
  if (
1458
- buffered.executionGeneration !== executionGeneration ||
1459
- state.executionGeneration !== executionGeneration ||
2000
+ this.runs.get(runId) !== state ||
2001
+ this.branchEpoch !== dispatchEpoch ||
1460
2002
  state.lifecycleSuspended ||
1461
- state.controller.signal.aborted ||
1462
- buffered.response.id !== pendingId ||
1463
- !sameWorkflowOwner(buffered.owner, owner)
1464
- )
1465
- return;
1466
- this.resolveAgentWaiter(state, pendingId, waiter, buffered);
1467
- }).finally(() => {
1468
- externalSignal.removeEventListener("abort", waitAbort);
1469
- });
1470
-
1471
- if (
1472
- this.runs.get(runId) !== state ||
1473
- this.branchEpoch !== dispatchEpoch ||
1474
- state.lifecycleSuspended ||
1475
- this.lifecyclePaused
1476
- ) {
1477
- throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
1478
- }
1479
- const snapshot = terminal.terminal;
1480
- if (!snapshot)
1481
- throw new WorkflowError("subagent settled without a terminal snapshot", WorkflowErrorCode.AGENT_EXECUTION_ERROR, {
1482
- recoverable: true,
2003
+ this.lifecyclePaused
2004
+ ) {
2005
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
2006
+ }
2007
+ const snapshot = terminal.terminal;
2008
+ if (!snapshot)
2009
+ throw new WorkflowError(
2010
+ "subagent settled without a terminal snapshot",
2011
+ WorkflowErrorCode.AGENT_EXECUTION_ERROR,
2012
+ {
2013
+ recoverable: true,
2014
+ },
2015
+ );
2016
+ this.recordManagedTerminal(state, nodeId, terminal.id, snapshot, executionGeneration);
2017
+ state.presentation.managed(nodeId, {
2018
+ agentId: terminal.id,
2019
+ status: managedTerminalStatus(snapshot),
2020
+ tokenCount: snapshot.tokenCount,
2021
+ outputFile: snapshot.outputFile,
2022
+ compactionCount: Math.max(snapshot.compactionCount, state.managedCallFacts.get(nodeId)?.compactionCount ?? 0),
1483
2023
  });
1484
- const status = managedTerminalStatus(snapshot);
1485
- // Successful calls are journaled by the runtime's onAgentJournal (the
1486
- // single writer for completed results); only failures settle here so the
1487
- // resume journal reflects them.
1488
- if (status !== "completed") {
1489
- const result = resultFromLifecycle(
1490
- status,
1491
- terminal.id,
1492
- snapshot.result,
1493
- snapshot.error,
1494
- snapshot.compactionCount,
1495
- snapshot.tokenCount,
1496
- state.callTiers.get(nodeId)?.tier ?? tier,
1497
- );
1498
- result.attemptId = attemptId;
1499
- if (!nested) this.commitCallTerminal(runId, nodeId, result);
1500
- if (status === "failed" && snapshot.error) {
1501
- throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
2024
+ this.notifyInspection(state);
2025
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch))
2026
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
2027
+ if (externalSignal.aborted) throw new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED);
2028
+ terminalAccepted = true;
2029
+ const status = managedTerminalStatus(snapshot);
2030
+ // Successful calls are journaled by the runtime's onAgentJournal (the
2031
+ // single writer for completed results); only failures settle here so the
2032
+ // resume journal reflects them.
2033
+ if (status !== "completed") {
2034
+ const result = resultFromLifecycle(
2035
+ status,
2036
+ terminal.id,
2037
+ snapshot.result,
2038
+ snapshot.error,
2039
+ snapshot.compactionCount,
2040
+ snapshot.tokenCount,
2041
+ state.callTiers.get(nodeId)?.tier ?? tier,
2042
+ );
2043
+ result.attemptId = attemptId;
2044
+ if (!nested) this.commitCallTerminal(runId, nodeId, result);
2045
+ if (status === "failed" && snapshot.error) {
2046
+ throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
2047
+ }
1502
2048
  }
2049
+ const completedTierIdentity = state.callTiers.get(nodeId);
2050
+ if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
2051
+ if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
2052
+ throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
2053
+ return snapshot.result ?? "";
2054
+ } finally {
2055
+ // Publication does not donate ownership to an observer or a replacement
2056
+ // state. Even a terminal reply must STOP + QUIESCE its captured owner
2057
+ // when observation retired this dispatch.
2058
+ if (
2059
+ pending.phase === "submitted" &&
2060
+ (!terminalAccepted || !this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
2061
+ )
2062
+ await cleanupResponse();
2063
+ state.pendingSpawns.delete(spawnKey);
2064
+ state.bufferedTerminals.delete(spawnKey);
1503
2065
  }
1504
- const completedTierIdentity = state.callTiers.get(nodeId);
1505
- if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
1506
- return snapshot.result ?? "";
1507
2066
  }
1508
2067
 
1509
2068
  private journalCallResult(runId: string, entry: JournalEntry): void {
2069
+ entry = structuredClone(entry);
1510
2070
  const state = this.runs.get(runId);
1511
2071
  if (!state) return;
1512
2072
  const run = state.run;
2073
+ const executionGeneration = state.executionGeneration;
2074
+ const epoch = this.branchEpoch;
1513
2075
  const nodeId = String(entry.index);
1514
2076
  const result = entry.result;
1515
2077
  const callTierIdentity: CallTierIdentity =
1516
2078
  entry.tier === undefined ? (run.callTiers[nodeId] ?? {}) : { tier: entry.tier };
2079
+ const managed = state.managedCallFacts.get(nodeId);
2080
+ const currentManaged =
2081
+ managed?.executionGeneration === state.executionGeneration && managed.attemptId === run.attemptIds[nodeId]
2082
+ ? managed
2083
+ : undefined;
1517
2084
  const callResult: CallResult = {
2085
+ ...(currentManaged
2086
+ ? {
2087
+ agentId: currentManaged.agentId,
2088
+ ...(currentManaged.tokenCount === undefined ? {} : { tokenCount: currentManaged.tokenCount }),
2089
+ ...(currentManaged.outputFile === undefined ? {} : { outputFile: currentManaged.outputFile }),
2090
+ }
2091
+ : {}),
1518
2092
  status: "completed",
1519
2093
  ...(typeof result === "string" && result.length === 0 ? {} : { result }),
1520
2094
  ...callTierIdentity,
1521
- compactionCount: 0,
2095
+ compactionCount: currentManaged?.compactionCount ?? 0,
1522
2096
  updatedAt: Date.now(),
1523
2097
  };
2098
+ // Checkpoints are local calls, not managed children. Give their completed
2099
+ // result a journal identity without fabricating a manager record.
2100
+ if (!run.attemptIds[nodeId]) {
2101
+ const generation = (state.generations.get(nodeId) ?? 0) + 1;
2102
+ const attemptId = `${runId}/${nodeId}/attempt-${generation}`;
2103
+ this.appendRequired({
2104
+ kind: "call_attempt",
2105
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
2106
+ runId,
2107
+ nodeId,
2108
+ attemptId,
2109
+ generation,
2110
+ owner: { extension: "pi-workflows", runId, nodeId, attemptId },
2111
+ timestamp: Date.now(),
2112
+ });
2113
+ if (!this.isCurrentState(state, executionGeneration, epoch)) return;
2114
+ state.generations.set(nodeId, generation);
2115
+ run.attemptTracking = true;
2116
+ run.attempts[nodeId] = generation;
2117
+ run.attemptIds[nodeId] = attemptId;
2118
+ run.callStatus[nodeId] = "running";
2119
+ }
1524
2120
  const attemptId = run.attemptIds[nodeId];
1525
2121
  if (attemptId) callResult.attemptId = attemptId;
1526
2122
  const event: JournalEvent = {
@@ -1537,16 +2133,23 @@ export class WorkflowEngine {
1537
2133
  ...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
1538
2134
  timestamp: Date.now(),
1539
2135
  };
1540
- this.persist(runId, `call-result:${nodeId}`, () => this.write(event));
2136
+ const acceptedEvent = structuredClone(event);
2137
+ this.persist(runId, `call-result:${nodeId}`, () => this.write(acceptedEvent));
2138
+ if (!this.isCurrentState(state, executionGeneration, epoch)) return;
1541
2139
  run.callResults[nodeId] = callResult;
1542
2140
  run.callTiers[nodeId] = callResult.tier === undefined ? {} : { tier: callResult.tier };
1543
2141
  run.callStatus[nodeId] = "completed";
2142
+ run.compactions[nodeId] = callResult.compactionCount;
2143
+ state.managedCallFacts.delete(nodeId);
1544
2144
  run.updatedAt = Date.now();
1545
2145
  }
1546
2146
 
1547
2147
  private journalWorkflowResult(runId: string, entry: JournalEntry): void {
2148
+ entry = structuredClone(entry);
1548
2149
  const state = this.runs.get(runId);
1549
2150
  if (!state) return;
2151
+ const generation = state.executionGeneration;
2152
+ const epoch = this.branchEpoch;
1550
2153
  const nodeId = String(entry.index);
1551
2154
  const result: CallResult = {
1552
2155
  status: "completed",
@@ -1573,6 +2176,7 @@ export class WorkflowEngine {
1573
2176
  // otherwise a transient append retry would arrive after the terminal
1574
2177
  // transition and be discarded as stale during restore.
1575
2178
  this.appendRequired(event);
2179
+ if (!this.isCurrentState(state, generation, epoch)) return;
1576
2180
  state.run.workflowResultGenerations[nodeId] = event.generation;
1577
2181
  state.run.callResults[nodeId] = result;
1578
2182
  state.run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
@@ -1580,10 +2184,12 @@ export class WorkflowEngine {
1580
2184
  state.run.updatedAt = Date.now();
1581
2185
  }
1582
2186
 
1583
- private commitCallTerminal(runId: string, nodeId: string, result: CallResult): void {
2187
+ private commitCallTerminal(runId: string, nodeId: string, result: CallResult, required = false): void {
1584
2188
  const state = this.runs.get(runId);
1585
2189
  if (!state) return;
1586
2190
  const run = state.run;
2191
+ const generation = state.executionGeneration;
2192
+ const epoch = this.branchEpoch;
1587
2193
  const attemptId = result.attemptId ?? run.attemptIds[nodeId];
1588
2194
  const event: JournalEvent = {
1589
2195
  kind: "call_result",
@@ -1595,7 +2201,9 @@ export class WorkflowEngine {
1595
2201
  ...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
1596
2202
  timestamp: Date.now(),
1597
2203
  };
1598
- this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
2204
+ if (required) this.appendRequired(event);
2205
+ else this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
2206
+ if (!this.isCurrentState(state, generation, epoch)) return;
1599
2207
  run.callResults[nodeId] = result;
1600
2208
  run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
1601
2209
  run.callStatus[nodeId] = result.status;
@@ -1608,36 +2216,26 @@ export class WorkflowEngine {
1608
2216
 
1609
2217
  private reconcilePendingSpawns(state: ScriptRunState): void {
1610
2218
  for (const pending of state.pendingSpawns.values()) {
2219
+ if (pending.phase !== "submitted" || pending.agentId) continue;
1611
2220
  const reconciliation = this.client.reconcileManaged?.(pending.spawnKey, pending.owner);
1612
2221
  void reconciliation?.catch(() => {});
1613
2222
  }
1614
- state.pendingSpawns.clear();
2223
+ // Reconciliation is not positive exact-owner quiescence. The allocation
2224
+ // lease remains with dispatch until terminal admission or owned cleanup.
1615
2225
  state.bufferedTerminals.clear();
1616
2226
  }
1617
2227
 
1618
- private async quiescePendingSpawns(state: ScriptRunState): Promise<WorkflowQuiesceResult> {
1619
- const pending = [...state.pendingSpawns.values()];
1620
- if (pending.length === 0) return { settled: true, pending: [] };
1621
- if (!this.client.reconcileManaged) {
1622
- return { settled: false, pending: pending.map(({ spawnKey }) => spawnKey).slice(0, 256) };
1623
- }
1624
- const failed: string[] = [];
1625
- await Promise.all(
1626
- pending.map(async ({ spawnKey, owner }) => {
1627
- try {
1628
- const result = await this.client.reconcileManaged?.(spawnKey, owner);
1629
- if (result === undefined) failed.push(spawnKey);
1630
- } catch {
1631
- failed.push(spawnKey);
1632
- }
1633
- }),
1634
- );
1635
- return { settled: failed.length === 0, pending: failed.slice(0, 256) };
1636
- }
1637
-
1638
2228
  private cleanupAgents(runId: string): void {
1639
2229
  const state = this.runs.get(runId);
1640
2230
  if (!state) return;
2231
+ // Retain captured allocations before reconciliation, abort, or branch replay
2232
+ // can notify observers. Dispatch's finally may not run until a later turn;
2233
+ // its cleanup and any waiter cleanup share this stable exact-owner lease.
2234
+ for (const pending of state.pendingSpawns.values()) {
2235
+ if (pending.phase !== "submitted") continue;
2236
+ const agentId = pending.agentId ?? pending.startedAgentId ?? pending.spawnKey;
2237
+ void this.stopAndQuiesce(state, agentId, pending.owner, pending.executionGeneration);
2238
+ }
1641
2239
  this.reconcilePendingSpawns(state);
1642
2240
  for (const [agentId, waiter] of [...state.agentWaiters]) {
1643
2241
  if (waiter.settled) continue;
@@ -1648,14 +2246,17 @@ export class WorkflowEngine {
1648
2246
  recoverable: true,
1649
2247
  }),
1650
2248
  );
1651
- void this.client
1652
- .stopOwned?.(agentId, {
2249
+ void this.stopAndQuiesce(
2250
+ state,
2251
+ agentId,
2252
+ {
1653
2253
  extension: "pi-workflows",
1654
2254
  runId,
1655
2255
  nodeId: `call-${waiter.callIndex}`,
1656
2256
  attemptId: `attempt-${waiter.generation}`,
1657
- })
1658
- .catch(() => {});
2257
+ },
2258
+ state.executionGeneration,
2259
+ );
1659
2260
  }
1660
2261
  }
1661
2262
 
@@ -1665,51 +2266,108 @@ export class WorkflowEngine {
1665
2266
  owner: WorkflowOwner,
1666
2267
  executionGeneration: number,
1667
2268
  unconfirmed?: string,
2269
+ quiescence?: Promise<{ settled: boolean; pending: string[] }>,
1668
2270
  ): Promise<void> {
1669
- let timer: ReturnType<typeof setTimeout> | undefined;
1670
- let stopError: unknown;
1671
- try {
1672
- if (unconfirmed) throw new Error(unconfirmed);
1673
- await Promise.race([
1674
- (async () => {
2271
+ const obligations = this.cleanupObligations.get(state) ?? new Map<string, OwnedCleanup>();
2272
+ this.cleanupObligations.set(state, obligations);
2273
+ const allocationKey = `${owner.runId}/${owner.nodeId}/${owner.attemptId}`;
2274
+ const existing = [...obligations.entries()].find(([, cleanup]) => sameWorkflowOwner(cleanup.owner, owner));
2275
+ const key = existing?.[0] ?? JSON.stringify([owner, agentId]);
2276
+ let obligation = existing?.[1];
2277
+ if (!obligation) {
2278
+ let resolve!: () => void;
2279
+ const confirmed = new Promise<void>((done) => {
2280
+ resolve = done;
2281
+ });
2282
+ obligation = {
2283
+ agentId,
2284
+ ...(agentId === allocationKey ? { spawnKey: allocationKey } : {}),
2285
+ owner: { ...owner },
2286
+ epoch: this.branchEpoch,
2287
+ confirmed,
2288
+ resolve,
2289
+ };
2290
+ obligations.set(key, obligation);
2291
+ this.retainedCleanupStates.add(state);
2292
+ state.pendingCleanup.add(confirmed);
2293
+ } else if (obligation.spawnKey && agentId !== allocationKey) {
2294
+ // A late reply refines the same allocation lease, not a new obligation.
2295
+ obligation.agentId = agentId;
2296
+ obligation.spawnKey = undefined;
2297
+ }
2298
+ if (obligation.operation) return obligation.operation;
2299
+ const accepted = obligation;
2300
+ const currentOwner = () => this.isCurrentState(state, state.executionGeneration, accepted.epoch);
2301
+ const operation = Promise.resolve().then(async () => {
2302
+ let timer: ReturnType<typeof setTimeout> | undefined;
2303
+ let stopError: unknown;
2304
+ try {
2305
+ if (unconfirmed) throw new Error(unconfirmed);
2306
+ const cleanup = (async () => {
2307
+ // An allocation without a confirmed id can only be retried by its exact spawn owner.
2308
+ if (accepted.spawnKey) {
2309
+ if (!this.client.reconcileManaged) throw new Error("owned allocation reconciliation unavailable");
2310
+ const reply = await this.client.reconcileManaged(accepted.spawnKey, accepted.owner);
2311
+ const id = typeof reply === "string" ? reply : reply?.id;
2312
+ if (!id) throw new Error("cancelled spawn allocation could not be confirmed");
2313
+ accepted.agentId = id;
2314
+ accepted.spawnKey = undefined;
2315
+ }
1675
2316
  try {
1676
2317
  if (!this.client.stopOwned) throw new Error("owned stop RPC unavailable");
1677
- await this.client.stopOwned(agentId, owner);
2318
+ await this.client.stopOwned(accepted.agentId, accepted.owner);
1678
2319
  } catch (error: unknown) {
1679
- // Reconciliation may already have stopped the record. Only exact-owner
1680
- // quiescence confirms that provider/tool cleanup has actually settled.
1681
2320
  stopError = error;
1682
2321
  }
1683
2322
  if (!this.client.quiesceOwned) throw new Error("owned cleanup RPC unavailable");
1684
- const result = await this.client.quiesceOwned(owner.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner]);
2323
+ const result = await (quiescence ??
2324
+ this.client.quiesceOwned(accepted.owner.runId, [accepted.agentId], BRANCH_QUIESCE_TIMEOUT_MS, [
2325
+ accepted.owner,
2326
+ ]));
1685
2327
  if (!result?.settled || result.pending.length !== 0) throw new Error("owned agents are not quiescent");
1686
- })(),
1687
- new Promise<never>((_resolve, reject) => {
1688
- timer = setTimeout(() => reject(new Error("owned cleanup timed out")), BRANCH_QUIESCE_TIMEOUT_MS + 1_000);
1689
- }),
1690
- ]);
1691
- } catch (error: unknown) {
1692
- // Never turn uncertain cleanup into a resumable interruption. Existing
1693
- // terminal recovery facts durably fence this run without a protocol change.
1694
- if (this.runs.get(owner.runId) !== state || state.executionGeneration !== executionGeneration) return;
1695
- const stopDiagnostic = stopError === undefined ? "" : `; stop failed: ${String(stopError)}`;
1696
- const diagnostic = `Owned cleanup unconfirmed for ${agentId}: ${error instanceof Error ? error.message : String(error)}${stopDiagnostic}; run is non-resumable`;
1697
- this.fenceExecution(state, executionGeneration, diagnostic);
2328
+ // A bounded negative/error is not evidence that cleanup settled. Only
2329
+ // this positive owner check releases the stable obligation, even late.
2330
+ obligations.delete(key);
2331
+ if (obligations.size === 0) this.retainedCleanupStates.delete(state);
2332
+ state.pendingCleanup.delete(accepted.confirmed);
2333
+ accepted.resolve();
2334
+ if (currentOwner() && isTerminalWorkflow(state.run.status)) this.notifyTerminal(state);
2335
+ })();
2336
+ await Promise.race([
2337
+ cleanup,
2338
+ new Promise<never>((_resolve, reject) => {
2339
+ timer = setTimeout(() => reject(new Error("owned cleanup timed out")), BRANCH_QUIESCE_TIMEOUT_MS + 1_000);
2340
+ }),
2341
+ ]);
2342
+ } catch (error: unknown) {
2343
+ if (!currentOwner() || state.executionGeneration !== executionGeneration) return;
2344
+ const stopDiagnostic = stopError === undefined ? "" : `; stop failed: ${String(stopError)}`;
2345
+ const diagnostic = `Owned cleanup unconfirmed for ${agentId}: ${error instanceof Error ? error.message : String(error)}${stopDiagnostic}; run is non-resumable`;
2346
+ this.fenceExecution(state, executionGeneration, diagnostic);
2347
+ } finally {
2348
+ if (timer !== undefined) clearTimeout(timer);
2349
+ }
2350
+ });
2351
+ accepted.operation = operation;
2352
+ try {
2353
+ await operation;
1698
2354
  } finally {
1699
- if (timer !== undefined) clearTimeout(timer);
2355
+ if (accepted.operation === operation) accepted.operation = undefined;
1700
2356
  }
1701
2357
  }
1702
2358
 
1703
2359
  private async drainExecution(state: ScriptRunState, execution?: Promise<unknown>): Promise<void> {
1704
2360
  if (!execution) return;
1705
2361
  const generation = state.executionGeneration;
2362
+ const epoch = this.branchEpoch;
1706
2363
  let timer: ReturnType<typeof setTimeout> | undefined;
1707
2364
  try {
1708
2365
  await Promise.race([
1709
2366
  execution.catch(() => {}),
1710
2367
  new Promise<void>((resolve) => {
1711
2368
  timer = setTimeout(() => {
1712
- this.fenceExecution(state, generation, "Workflow execution cleanup timed out; run is non-resumable");
2369
+ if (this.isCurrentState(state, generation, epoch))
2370
+ this.fenceExecution(state, generation, "Workflow execution cleanup timed out; run is non-resumable");
1713
2371
  resolve();
1714
2372
  }, EXECUTION_DRAIN_TIMEOUT_MS);
1715
2373
  }),
@@ -1721,7 +2379,8 @@ export class WorkflowEngine {
1721
2379
 
1722
2380
  private fenceExecution(state: ScriptRunState, generation: number, diagnostic: string): void {
1723
2381
  const run = state.run;
1724
- if (this.runs.get(run.runId) !== state || state.executionGeneration !== generation) return;
2382
+ const epoch = this.branchEpoch;
2383
+ if (!this.isCurrentState(state, generation, epoch)) return;
1725
2384
  if (run.nonResumable && run.status === "stopped") return;
1726
2385
  run.nonResumable = true;
1727
2386
  const blockedNodeIds = Object.keys(run.callStatus).filter((nodeId) => run.callStatus[nodeId] !== "completed");
@@ -1744,21 +2403,119 @@ export class WorkflowEngine {
1744
2403
  timestamp: Date.now(),
1745
2404
  };
1746
2405
  this.persist(run.runId, "cleanup-unconfirmed", () => this.write(event));
2406
+ if (!this.isCurrentState(state, generation, epoch)) return;
1747
2407
  // Fence before aborting: synchronous abort handlers may re-enter cleanup.
1748
2408
  // Exact-owner cleanup continues, but no later result may revive this run.
1749
2409
  state.executionGeneration += 1;
2410
+ state.presentation.reset(false);
2411
+ this.notifyInspection(state);
2412
+ if (!this.isCurrentState(state, generation + 1, epoch)) return;
1750
2413
  state.controller.abort();
2414
+ if (!this.isCurrentState(state, generation + 1, epoch)) return;
1751
2415
  this.cleanupAgents(run.runId);
1752
- this.settleWaiters(run.runId, run);
2416
+ // Uncertainty is an error, never a terminal result delivery. Foreground
2417
+ // callers can unwind, while owner UI retains an explicit cleanup recheck.
2418
+ if (state.pendingCleanup.size > 0) {
2419
+ const all = this.waiters.get(run.runId);
2420
+ this.waiters.delete(run.runId);
2421
+ for (const waiter of all ?? []) {
2422
+ if (waiter.settled) continue;
2423
+ waiter.settled = true;
2424
+ waiter.cleanup();
2425
+ waiter.reject(new WorkflowWaitAbortedError());
2426
+ }
2427
+ } else this.notifyTerminal(state);
2428
+ }
2429
+
2430
+ private async remove(state: ScriptRunState): Promise<void> {
2431
+ if (state.removeOperation) return state.removeOperation;
2432
+ const isBusy = () =>
2433
+ state.activeExecutionGenerations.size > 0 ||
2434
+ state.pendingCleanup.size > 0 ||
2435
+ state.pendingSpawns.size > 0 ||
2436
+ state.agentWaiters.size > 0 ||
2437
+ state.stopOperation !== undefined;
2438
+ if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
2439
+ const epoch = this.branchEpoch;
2440
+ const operation = Promise.resolve().then(async () => {
2441
+ await this.drainJournal(state.run.runId);
2442
+ if (
2443
+ this.runs.get(state.run.runId) !== state ||
2444
+ this.branchEpoch !== epoch ||
2445
+ this.disposed ||
2446
+ this.lifecyclePaused ||
2447
+ state.lifecycleSuspended
2448
+ )
2449
+ throw new Error("workflow branch changed before removal");
2450
+ if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
2451
+ // No queued best-effort tombstone: memory survives failure so removal can be retried.
2452
+ this.appendRequired({
2453
+ kind: "run_removed",
2454
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
2455
+ runId: state.run.runId,
2456
+ timestamp: Date.now(),
2457
+ });
2458
+ this.runs.delete(state.run.runId);
2459
+ this.waiters.delete(state.run.runId);
2460
+ this.notifyInspection(state);
2461
+ });
2462
+ state.removeOperation = operation;
2463
+ try {
2464
+ await operation;
2465
+ } finally {
2466
+ if (state.removeOperation === operation) state.removeOperation = undefined;
2467
+ }
1753
2468
  }
1754
2469
 
1755
2470
  private async stop(state: ScriptRunState): Promise<void> {
2471
+ if (state.stopOperation) return state.stopOperation;
2472
+ if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size === 0) return;
2473
+ const generation = state.executionGeneration;
2474
+ const epoch = this.branchEpoch;
2475
+ const operation = Promise.resolve().then(async () => {
2476
+ if (!this.isCurrentState(state, generation, epoch)) return;
2477
+ const pending = [...(this.cleanupObligations.get(state)?.values() ?? [])];
2478
+ if (pending.length > 0) {
2479
+ await Promise.all(
2480
+ pending.map((cleanup) => this.stopAndQuiesce(state, cleanup.agentId, cleanup.owner, generation)),
2481
+ );
2482
+ if (!this.isCurrentState(state, generation, epoch) || state.pendingCleanup.size > 0) return;
2483
+ if (isTerminalWorkflow(state.run.status)) {
2484
+ this.notifyTerminal(state);
2485
+ return;
2486
+ }
2487
+ }
2488
+ await this.stopExecution(state, generation, epoch);
2489
+ });
2490
+ state.stopOperation = operation;
2491
+ try {
2492
+ await operation;
2493
+ } finally {
2494
+ if (state.stopOperation === operation) state.stopOperation = undefined;
2495
+ }
2496
+ }
2497
+
2498
+ private async stopExecution(state: ScriptRunState, generation: number, epoch: number): Promise<void> {
1756
2499
  const run = state.run;
1757
- if (isTerminalWorkflow(run.status)) return;
2500
+ const current = () => this.isCurrentState(state, generation, epoch);
2501
+ if (!current() || isTerminalWorkflow(run.status)) return;
1758
2502
  this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
2503
+ if (!current()) return;
1759
2504
  run.nonResumable = true;
2505
+ // Runtime abort listeners own exact cleanup. Rejecting their waiters here would
2506
+ // falsely settle the dispatch before its stop-and-quiesce barrier completes.
1760
2507
  state.controller.abort();
1761
- this.cleanupAgents(run.runId);
2508
+ if (!current()) return;
2509
+ await this.drainExecution(state, state.execution);
2510
+ if (!current() || state.pendingCleanup.size > 0) return;
2511
+ try {
2512
+ await this.drainJournal(run.runId);
2513
+ } catch {
2514
+ if (current())
2515
+ this.fenceExecution(state, generation, "Could not drain workflow journal during stop; run is non-resumable");
2516
+ return;
2517
+ }
2518
+ if (!current()) return;
1762
2519
  const terminalResults: RecoveryTerminalResult[] = [];
1763
2520
  for (const [nodeId, status] of Object.entries(run.callStatus)) {
1764
2521
  if (status !== "completed") {
@@ -1778,8 +2535,6 @@ export class WorkflowEngine {
1778
2535
  result,
1779
2536
  owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId: run.attemptIds[nodeId] ?? "" },
1780
2537
  });
1781
- run.callResults[nodeId] = result;
1782
- run.callStatus[nodeId] = "stopped";
1783
2538
  }
1784
2539
  }
1785
2540
  const event: TerminalRecoveryEvent = {
@@ -1795,10 +2550,22 @@ export class WorkflowEngine {
1795
2550
  recoveryId: deriveRecoveryId({ runId: run.runId, rotations: [], timestamp: Date.now() }),
1796
2551
  timestamp: Date.now(),
1797
2552
  };
1798
- this.persist(run.runId, "stop-terminal", () => this.write(event));
2553
+ try {
2554
+ this.appendRequired(event);
2555
+ } catch {
2556
+ this.fenceExecution(state, generation, "Could not persist workflow stop terminal; run is non-resumable");
2557
+ return;
2558
+ }
2559
+ if (!current()) return;
2560
+ for (const terminal of terminalResults) {
2561
+ run.callResults[terminal.nodeId] = terminal.result;
2562
+ run.callStatus[terminal.nodeId] = "stopped";
2563
+ }
1799
2564
  run.status = "stopped";
1800
2565
  run.updatedAt = Date.now();
1801
- this.settleWaiters(run.runId, run);
2566
+ state.presentation.settle(run.status);
2567
+ this.notifyInspection(state);
2568
+ if (current()) this.notifyTerminal(state);
1802
2569
  }
1803
2570
 
1804
2571
  // ── journal writes ─────────────────────────────────────────────────────────
@@ -1811,15 +2578,24 @@ export class WorkflowEngine {
1811
2578
  * whose appends keep failing is marked blocked (reported once, not spammed).
1812
2579
  */
1813
2580
  private scheduleJournalRetry(runId: string, key: string, mutation: () => void): void {
1814
- const retryKey = `${runId}\u0000${key}`;
1815
- if (this.journalRetries.has(retryKey)) return;
1816
- const retry = { attempt: 0, timer: undefined as ReturnType<typeof setTimeout> | undefined };
2581
+ // Every accepted fact has its own slot, including later retry generations
2582
+ // for the same call. Coalescing by node key would silently drop that fact.
2583
+ const retryKey = `${runId}\u0000${key}\u0000${++this.journalMutationSequence}`;
2584
+ const predecessors = [...this.journalRetries.entries()]
2585
+ .filter(([id]) => id.startsWith(`${runId}\u0000`))
2586
+ .map(([, pending]) => pending.done);
2587
+ let resolve!: () => void;
2588
+ const done = new Promise<void>((settled) => {
2589
+ resolve = settled;
2590
+ });
2591
+ const retry = { attempt: 0, timer: undefined as ReturnType<typeof setTimeout> | undefined, done, resolve };
1817
2592
  const attempt = (): void => {
1818
2593
  const current = this.journalRetries.get(retryKey);
1819
2594
  if (!current) return;
1820
2595
  current.timer = undefined;
1821
2596
  if (this.lifecyclePaused || !this.runs.has(runId)) {
1822
2597
  this.journalRetries.delete(retryKey);
2598
+ current.resolve();
1823
2599
  return;
1824
2600
  }
1825
2601
  const previousKey = this.journalMutationKey;
@@ -1827,9 +2603,12 @@ export class WorkflowEngine {
1827
2603
  try {
1828
2604
  mutation();
1829
2605
  this.journalRetries.delete(retryKey);
2606
+ current.resolve();
1830
2607
  } catch (error: unknown) {
1831
2608
  if (!(error instanceof JournalAppendError)) {
1832
2609
  this.journalRetries.delete(retryKey);
2610
+ current.resolve();
2611
+ this.journalBlockedRuns.add(runId);
1833
2612
  console.warn(
1834
2613
  `[pi-workflows] journal retry abandoned: ${error instanceof Error ? error.message : String(error)}`,
1835
2614
  );
@@ -1838,6 +2617,7 @@ export class WorkflowEngine {
1838
2617
  current.attempt += 1;
1839
2618
  if (current.attempt >= JOURNAL_RETRY_MAX_ATTEMPTS) {
1840
2619
  this.journalRetries.delete(retryKey);
2620
+ current.resolve();
1841
2621
  this.journalBlockedRuns.add(runId);
1842
2622
  console.warn(`[pi-workflows] journal retry exhausted for ${runId}: ${error.message}`);
1843
2623
  return;
@@ -1852,7 +2632,19 @@ export class WorkflowEngine {
1852
2632
  }
1853
2633
  };
1854
2634
  this.journalRetries.set(retryKey, retry);
1855
- queueMicrotask(attempt);
2635
+ void Promise.all(predecessors).then(attempt);
2636
+ }
2637
+
2638
+ private async drainJournal(runId: string): Promise<void> {
2639
+ let pending = [...this.journalRetries.entries()].filter(([id]) => id.startsWith(`${runId}\u0000`));
2640
+ while (pending.length > 0) {
2641
+ await Promise.all(pending.map(([, retry]) => retry.done));
2642
+ pending = [...this.journalRetries.entries()].filter(([id]) => id.startsWith(`${runId}\u0000`));
2643
+ }
2644
+ if (this.journalBlockedRuns.has(runId))
2645
+ throw new WorkflowError("Workflow journal has not settled", WorkflowErrorCode.PERSISTENCE_ERROR, {
2646
+ recoverable: true,
2647
+ });
1856
2648
  }
1857
2649
 
1858
2650
  private appendRequired(event: JournalEvent): void {
@@ -1888,6 +2680,18 @@ export class WorkflowEngine {
1888
2680
  * a transient failure, schedules a retry of the exact same append.
1889
2681
  */
1890
2682
  private persist(runId: string, key: string, mutation: () => void): void {
2683
+ const acceptedState = this.runs.get(runId);
2684
+ const epoch = this.branchEpoch;
2685
+ const append = mutation;
2686
+ mutation = () => {
2687
+ if (this.disposed || this.lifecyclePaused || this.runs.get(runId) !== acceptedState || this.branchEpoch !== epoch)
2688
+ return;
2689
+ append();
2690
+ };
2691
+ if ([...this.journalRetries.keys()].some((id) => id.startsWith(`${runId}\u0000`))) {
2692
+ this.scheduleJournalRetry(runId, key, mutation);
2693
+ return;
2694
+ }
1891
2695
  const previousKey = this.journalMutationKey;
1892
2696
  this.journalMutationKey = `${runId}:${key}`;
1893
2697
  try {
@@ -1942,8 +2746,9 @@ export class WorkflowEngine {
1942
2746
  this.providerResumeTimers.set(runId, timer);
1943
2747
  }
1944
2748
  private clearJournalRetries(): void {
1945
- for (const { timer } of this.journalRetries.values()) {
2749
+ for (const { timer, resolve } of this.journalRetries.values()) {
1946
2750
  if (timer) clearTimeout(timer);
2751
+ resolve();
1947
2752
  }
1948
2753
  this.journalRetries.clear();
1949
2754
  this.journalBlockedRuns.clear();
@@ -1959,6 +2764,10 @@ export class WorkflowEngine {
1959
2764
  finalResult?: unknown,
1960
2765
  ): void {
1961
2766
  if (isTerminalWorkflow(run.status) && !isTerminalWorkflow(status)) return;
2767
+ const state = this.runs.get(run.runId);
2768
+ const generation = state?.executionGeneration ?? 0;
2769
+ const epoch = this.branchEpoch;
2770
+ if (!state || state.run !== run || !this.isCurrentState(state, generation, epoch)) return;
1962
2771
  // Validate and isolate terminal values before best-effort persistence or any
1963
2772
  // status mutation. A bad result must still be able to transition the run to failed.
1964
2773
  const durableFinalResult = finalResult === undefined ? undefined : cloneWorkflowResult(finalResult);
@@ -1976,15 +2785,18 @@ export class WorkflowEngine {
1976
2785
  timestamp,
1977
2786
  }),
1978
2787
  );
2788
+ if (!this.isCurrentState(state, generation, epoch)) return;
1979
2789
  run.status = status;
2790
+ if (status === "running") run.error = undefined;
1980
2791
  if (error !== undefined) run.error = error;
1981
2792
  if (runFinalResult !== undefined) run.finalResult = runFinalResult;
1982
2793
  if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
1983
2794
  run.updatedAt = timestamp;
2795
+ this.notifyInspection(state);
2796
+ if (!this.isCurrentState(state, generation, epoch)) return;
1984
2797
  if (isTerminalWorkflow(status)) {
1985
- this.runs.get(run.runId)?.bufferedTerminals.clear();
1986
- this.settleWaiters(run.runId, run);
1987
- this.onRunSettled?.();
2798
+ state.bufferedTerminals.clear();
2799
+ this.notifyTerminal(state);
1988
2800
  }
1989
2801
  }
1990
2802
 
@@ -1992,6 +2804,9 @@ export class WorkflowEngine {
1992
2804
 
1993
2805
  restore(entries: readonly SessionEntryLike[], branchGeneration = 0): void {
1994
2806
  if (this.disposed) return;
2807
+ const retiredStates = [...this.runs.values()];
2808
+ this.branchEpoch++;
2809
+ for (const state of retiredStates) state.lifecycleSuspended = true;
1995
2810
  // A branch replay is a replacement, not a merge. Runs absent from the new
1996
2811
  // branch must not remain controllable or append facts into it.
1997
2812
  this.clearJournalRetries();
@@ -2009,6 +2824,10 @@ export class WorkflowEngine {
2009
2824
  for (const run of runs.values()) {
2010
2825
  const state: ScriptRunState = {
2011
2826
  run,
2827
+ presentation: new WorkflowPresentation(run.runId, true),
2828
+ managedCallFacts: new Map(),
2829
+ pendingCleanup: new Set(),
2830
+ activeExecutionGenerations: new Set(),
2012
2831
  frozenOptions: {
2013
2832
  ...(run.frozenTokenBudget !== undefined ? { tokenBudget: run.frozenTokenBudget } : {}),
2014
2833
  ...(run.frozenMaxAgents !== undefined ? { maxAgents: run.frozenMaxAgents } : {}),
@@ -2043,6 +2862,8 @@ export class WorkflowEngine {
2043
2862
  run.updatedAt = Date.now();
2044
2863
  this.runs.set(run.runId, state);
2045
2864
  }
2865
+ for (const state of retiredStates) this.notifyInspection(state);
2866
+ for (const state of this.runs.values()) this.notifyInspection(state);
2046
2867
  }
2047
2868
 
2048
2869
  list(): Array<Record<string, unknown>> {
@@ -2060,6 +2881,18 @@ export class WorkflowEngine {
2060
2881
  return this.runs.get(runId)?.run;
2061
2882
  }
2062
2883
 
2884
+ /** Primitive-only Fleet metadata; avoids cloning full inspection arrays for every run. */
2885
+ getInspectionHeader(runId: string): WorkflowInspectionHeader | undefined {
2886
+ const state = this.runs.get(runId);
2887
+ return state?.presentation.header(state.executionGeneration);
2888
+ }
2889
+
2890
+ /** Detached bounded facts for late-opening UI mounts; no event callback ownership required. */
2891
+ getInspection(runId: string): WorkflowInspection | undefined {
2892
+ const state = this.runs.get(runId);
2893
+ return state?.presentation.snapshot(state.run, state.executionGeneration);
2894
+ }
2895
+
2063
2896
  summary(run: ScriptRun, detailed = false): Record<string, unknown> {
2064
2897
  const base: Record<string, unknown> = {
2065
2898
  runId: run.runId,
@@ -2081,7 +2914,7 @@ export class WorkflowEngine {
2081
2914
  return {
2082
2915
  index: Number(nodeId),
2083
2916
  status,
2084
- ...(result?.result !== undefined ? { resultPreview: String(result.result).slice(0, 300) } : {}),
2917
+ ...(result?.result !== undefined ? { resultPreview: renderWorkflowValue(result.result).slice(0, 300) } : {}),
2085
2918
  ...(result?.error ? { error: result.error.slice(0, 300) } : {}),
2086
2919
  ...(result?.tokenCount !== undefined ? { tokenCount: result.tokenCount } : {}),
2087
2920
  ...(run.agentIds[nodeId] ? { agentId: run.agentIds[nodeId] } : {}),
@@ -2110,5 +2943,5 @@ export class WorkflowWaitAbortedError extends Error {
2110
2943
  }
2111
2944
  }
2112
2945
 
2113
- export type { ScriptRun, WorkflowOwner };
2946
+ export type { ScriptRun, WorkflowInspection, WorkflowInspectionHeader, WorkflowOwner };
2114
2947
  export { createManagedSpawnClient, JOURNAL_ENTRY_TYPE, snapshotRun };