@signalridge/pi-workflows 1.7.5 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +157 -10
- package/package.json +3 -2
- package/skills/workflow-authoring/SKILL.md +1 -1
- package/skills/workflow-authoring/references/capabilities.md +1 -1
- package/skills/workflow-authoring/references/capability-details.md +1 -1
- package/skills/workflow-patterns/SKILL.md +1 -1
- package/skills/workflow-review/SKILL.md +1 -1
- package/src/arming.ts +264 -3
- package/src/engine.ts +1212 -379
- package/src/errors.ts +26 -0
- package/src/index.ts +578 -225
- package/src/inspection.ts +544 -0
- package/src/journal.ts +38 -11
- package/src/orchestrator.ts +22 -4
- package/src/runtime.ts +427 -168
- package/src/safe-text.ts +4 -35
- package/src/ui-bus.ts +243 -0
- package/src/ui-command-center.ts +1032 -0
- package/src/ui-inspection.ts +206 -0
- package/src/ui-presentation.ts +196 -0
- package/src/ui.ts +43 -241
- package/src/vm-bootstrap.ts +218 -0
- package/src/vm-helper.ts +502 -0
package/src/engine.ts
CHANGED
|
@@ -20,8 +20,10 @@
|
|
|
20
20
|
import { createHash, randomUUID } from "node:crypto";
|
|
21
21
|
import type { ManagedSpawnResponse } from "@signalridge/pi-subagents-protocol";
|
|
22
22
|
import { isManagedAgentTier } from "@signalridge/pi-subagents-protocol";
|
|
23
|
-
import { WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
|
|
23
|
+
import { describeWorkflowError, WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
|
|
24
|
+
import { type WorkflowInspection, type WorkflowInspectionHeader, WorkflowPresentation } from "./inspection.js";
|
|
24
25
|
import {
|
|
26
|
+
applyRecoveryEvent,
|
|
25
27
|
buildResumeJournal,
|
|
26
28
|
type CallResult,
|
|
27
29
|
type CallTierIdentity,
|
|
@@ -31,6 +33,7 @@ import {
|
|
|
31
33
|
type JournalEvent,
|
|
32
34
|
type JournalWriter,
|
|
33
35
|
type RecoveryTerminalResult,
|
|
36
|
+
type RunRecoveryEvent,
|
|
34
37
|
replayJournal,
|
|
35
38
|
type ScriptRun,
|
|
36
39
|
type SessionEntryLike,
|
|
@@ -172,6 +175,25 @@ interface AgentTerminalDelivery {
|
|
|
172
175
|
history?: unknown[];
|
|
173
176
|
}
|
|
174
177
|
|
|
178
|
+
interface ManagedCallFacts {
|
|
179
|
+
executionGeneration: number;
|
|
180
|
+
attemptId: string;
|
|
181
|
+
agentId: string;
|
|
182
|
+
tokenCount?: number;
|
|
183
|
+
outputFile?: string;
|
|
184
|
+
compactionCount: number;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
interface OwnedCleanup {
|
|
188
|
+
agentId: string;
|
|
189
|
+
spawnKey?: string;
|
|
190
|
+
owner: WorkflowOwner;
|
|
191
|
+
epoch: number;
|
|
192
|
+
confirmed: Promise<void>;
|
|
193
|
+
resolve: () => void;
|
|
194
|
+
operation?: Promise<void>;
|
|
195
|
+
}
|
|
196
|
+
|
|
175
197
|
interface BufferedAgentTerminal extends AgentTerminalDelivery {
|
|
176
198
|
owner: WorkflowOwner;
|
|
177
199
|
executionGeneration: number;
|
|
@@ -179,6 +201,15 @@ interface BufferedAgentTerminal extends AgentTerminalDelivery {
|
|
|
179
201
|
|
|
180
202
|
export interface ScriptRunState {
|
|
181
203
|
run: ScriptRun;
|
|
204
|
+
/** Volatile, execution-fenced presentation; never journaled. */
|
|
205
|
+
presentation: WorkflowPresentation;
|
|
206
|
+
/** Exact-attempt metadata merged into the runtime's single schema-transformed result fact. */
|
|
207
|
+
managedCallFacts: Map<string, ManagedCallFacts>;
|
|
208
|
+
/** Ownership remains busy even after a cleanup deadline fences execution. */
|
|
209
|
+
pendingCleanup: Set<Promise<void>>;
|
|
210
|
+
activeExecutionGenerations: Set<number>;
|
|
211
|
+
stopOperation?: Promise<void>;
|
|
212
|
+
removeOperation?: Promise<void>;
|
|
182
213
|
/** Frozen run parameters — resume reuses the original values, ignoring new budget/toolset overrides. */
|
|
183
214
|
frozenOptions: Pick<
|
|
184
215
|
ScriptStartOptions,
|
|
@@ -200,8 +231,18 @@ export interface ScriptRunState {
|
|
|
200
231
|
agentWaiters: Map<string, AgentLifecycleWaiter>;
|
|
201
232
|
/** Owner-validated terminal events emitted before their spawn reply/waiter. */
|
|
202
233
|
bufferedTerminals: Map<string, BufferedAgentTerminal>;
|
|
203
|
-
/**
|
|
204
|
-
pendingSpawns: Map<
|
|
234
|
+
/** Local reservations and submitted allocations retained through terminal admission. */
|
|
235
|
+
pendingSpawns: Map<
|
|
236
|
+
string,
|
|
237
|
+
{
|
|
238
|
+
spawnKey: string;
|
|
239
|
+
owner: WorkflowOwner;
|
|
240
|
+
executionGeneration: number;
|
|
241
|
+
phase: "reserved" | "submitted";
|
|
242
|
+
startedAgentId?: string;
|
|
243
|
+
agentId?: string;
|
|
244
|
+
}
|
|
245
|
+
>;
|
|
205
246
|
/** Journal retry bookkeeping. */
|
|
206
247
|
pendingJournal: Set<string>;
|
|
207
248
|
/** Whether a lifecycle suspension is in effect (branch change). */
|
|
@@ -307,12 +348,41 @@ function sameWorkflowOwner(left: WorkflowOwner, right: WorkflowOwner): boolean {
|
|
|
307
348
|
);
|
|
308
349
|
}
|
|
309
350
|
|
|
351
|
+
/** Optional owned lifecycle usage is per-attempt; legacy `tokens` remains lifetime-scoped. */
|
|
352
|
+
function managedLifecycleUsage(raw: unknown): AgentUsage | undefined {
|
|
353
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
|
|
354
|
+
const { input, output, total } = raw as Record<string, unknown>;
|
|
355
|
+
if (
|
|
356
|
+
typeof input !== "number" ||
|
|
357
|
+
typeof output !== "number" ||
|
|
358
|
+
typeof total !== "number" ||
|
|
359
|
+
!Number.isSafeInteger(input) ||
|
|
360
|
+
!Number.isSafeInteger(output) ||
|
|
361
|
+
!Number.isSafeInteger(total) ||
|
|
362
|
+
input < 0 ||
|
|
363
|
+
output < 0 ||
|
|
364
|
+
total < 0 ||
|
|
365
|
+
!Number.isSafeInteger(input + output) ||
|
|
366
|
+
total < input + output
|
|
367
|
+
)
|
|
368
|
+
return undefined;
|
|
369
|
+
// The managed total includes cache writes, but excludes cache reads. The
|
|
370
|
+
// additive lifecycle field has no cost/cache-read detail; do not invent it.
|
|
371
|
+
return { input, output, total, cost: 0, cacheRead: 0, cacheWrite: total - input - output };
|
|
372
|
+
}
|
|
373
|
+
|
|
310
374
|
export class WorkflowEngine {
|
|
311
375
|
private readonly runs = new Map<string, ScriptRunState>();
|
|
312
376
|
/** Fired when a run settles terminal; wired by the UI surface for widget refresh. */
|
|
313
377
|
onRunSettled: (() => void) | undefined;
|
|
378
|
+
/** Every accepted background execution, including automatic and UI resumes. */
|
|
379
|
+
onBackgroundExecution: ((runId: string) => void) | undefined;
|
|
314
380
|
/** Optional in-process observer for phase/task/quality runtime events. */
|
|
315
381
|
onRuntimeEvent: ((runId: string, event: WorkflowRuntimeEvent) => void) | undefined;
|
|
382
|
+
/** Additive notification after accepted presentation mutations; consumers re-read getInspection(). */
|
|
383
|
+
onInspectionChanged: ((runId: string) => void) | undefined;
|
|
384
|
+
/** Fresh saved-workflow resolver for control and automatic resumes. */
|
|
385
|
+
loadSavedWorkflow: ((name: string) => string | undefined) | undefined;
|
|
316
386
|
/**
|
|
317
387
|
* The user's current strength → Agent tier table, read fresh per execution.
|
|
318
388
|
*
|
|
@@ -339,11 +409,19 @@ export class WorkflowEngine {
|
|
|
339
409
|
private admissionsClosed = false;
|
|
340
410
|
private pendingAdmissions = 0;
|
|
341
411
|
private recoveryBranchGeneration = 0;
|
|
342
|
-
private readonly journalRetries = new Map<
|
|
412
|
+
private readonly journalRetries = new Map<
|
|
413
|
+
string,
|
|
414
|
+
{ timer?: ReturnType<typeof setTimeout>; attempt: number; done: Promise<void>; resolve: () => void }
|
|
415
|
+
>();
|
|
343
416
|
private readonly journalBlockedRuns = new Set<string>();
|
|
344
417
|
private readonly providerResumeTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
345
418
|
private journalMutationKey: string | undefined;
|
|
419
|
+
private journalMutationSequence = 0;
|
|
346
420
|
private readonly quarantinedRunIds = new Set<string>();
|
|
421
|
+
private readonly cleanupObligations = new WeakMap<ScriptRunState, Map<string, OwnedCleanup>>();
|
|
422
|
+
/** Retired states keep their exact leases; replacement states never inherit them. */
|
|
423
|
+
private readonly retainedCleanupStates = new Set<ScriptRunState>();
|
|
424
|
+
private readonly terminalNotified = new WeakSet<ScriptRunState>();
|
|
347
425
|
|
|
348
426
|
constructor(
|
|
349
427
|
readonly events: WorkflowEventBus,
|
|
@@ -359,6 +437,15 @@ export class WorkflowEngine {
|
|
|
359
437
|
this.unsubscribeLifecycle = this.attachLifecycle();
|
|
360
438
|
}
|
|
361
439
|
|
|
440
|
+
private notifyInspection(state: ScriptRunState): void {
|
|
441
|
+
state.presentation.changed();
|
|
442
|
+
try {
|
|
443
|
+
this.onInspectionChanged?.(state.run.runId);
|
|
444
|
+
} catch {
|
|
445
|
+
// Presentation observers must never change execution or durable control.
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
362
449
|
private async awaitProtocol(): Promise<ManagedProtocolCheck> {
|
|
363
450
|
if (this.disposed) throw new WorkflowEngineDisposedError();
|
|
364
451
|
let rejectDisposed: ((reason?: unknown) => void) | undefined;
|
|
@@ -382,6 +469,40 @@ export class WorkflowEngine {
|
|
|
382
469
|
return this.disposed;
|
|
383
470
|
}
|
|
384
471
|
|
|
472
|
+
/** Pure exact-state cleanup uncertainty query for owner control surfaces. */
|
|
473
|
+
hasPendingCleanup(runId: string): boolean {
|
|
474
|
+
return (this.runs.get(runId)?.pendingCleanup.size ?? 0) > 0;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
private isCurrentState(state: ScriptRunState, generation: number, epoch: number): boolean {
|
|
478
|
+
return (
|
|
479
|
+
this.runs.get(state.run.runId) === state &&
|
|
480
|
+
state.executionGeneration === generation &&
|
|
481
|
+
this.branchEpoch === epoch &&
|
|
482
|
+
!state.lifecycleSuspended &&
|
|
483
|
+
!this.lifecyclePaused &&
|
|
484
|
+
!this.disposed
|
|
485
|
+
);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private notifyTerminal(state: ScriptRunState): void {
|
|
489
|
+
const generation = state.executionGeneration;
|
|
490
|
+
const epoch = this.branchEpoch;
|
|
491
|
+
if (
|
|
492
|
+
!this.isCurrentState(state, generation, epoch) ||
|
|
493
|
+
state.pendingCleanup.size > 0 ||
|
|
494
|
+
this.terminalNotified.has(state)
|
|
495
|
+
)
|
|
496
|
+
return;
|
|
497
|
+
this.terminalNotified.add(state);
|
|
498
|
+
this.settleWaiters(state.run.runId, state.run);
|
|
499
|
+
try {
|
|
500
|
+
this.onRunSettled?.();
|
|
501
|
+
} catch {
|
|
502
|
+
/* terminal observers are presentation-only */
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
385
506
|
/** A synchronous snapshot for cancellable navigation, not an admission lock. */
|
|
386
507
|
hasUnsettledWork(): boolean {
|
|
387
508
|
return (
|
|
@@ -389,6 +510,7 @@ export class WorkflowEngine {
|
|
|
389
510
|
this.branchQuiescePromise !== undefined ||
|
|
390
511
|
this.journalRetries.size > 0 ||
|
|
391
512
|
this.providerResumeTimers.size > 0 ||
|
|
513
|
+
this.retainedCleanupStates.size > 0 ||
|
|
392
514
|
[...this.runs.values()].some(
|
|
393
515
|
(state) =>
|
|
394
516
|
state.run.status === "running" ||
|
|
@@ -396,7 +518,11 @@ export class WorkflowEngine {
|
|
|
396
518
|
state.run.status === "stopping" ||
|
|
397
519
|
state.pendingSpawns.size > 0 ||
|
|
398
520
|
state.agentWaiters.size > 0 ||
|
|
399
|
-
state.pendingJournal.size > 0
|
|
521
|
+
state.pendingJournal.size > 0 ||
|
|
522
|
+
state.pendingCleanup.size > 0 ||
|
|
523
|
+
state.activeExecutionGenerations.size > 0 ||
|
|
524
|
+
state.stopOperation !== undefined ||
|
|
525
|
+
state.removeOperation !== undefined,
|
|
400
526
|
)
|
|
401
527
|
);
|
|
402
528
|
}
|
|
@@ -410,6 +536,8 @@ export class WorkflowEngine {
|
|
|
410
536
|
this.clearJournalRetries();
|
|
411
537
|
for (const state of this.runs.values()) {
|
|
412
538
|
state.lifecycleSuspended = true;
|
|
539
|
+
state.presentation.reset(false);
|
|
540
|
+
this.notifyInspection(state);
|
|
413
541
|
state.controller.abort(new DOMException("Workflow branch replaced", "AbortError"));
|
|
414
542
|
}
|
|
415
543
|
}
|
|
@@ -436,7 +564,8 @@ export class WorkflowEngine {
|
|
|
436
564
|
eventName !== "subagents:created" &&
|
|
437
565
|
eventName !== "subagents:started" &&
|
|
438
566
|
eventName !== "subagents:completed" &&
|
|
439
|
-
eventName !== "subagents:failed"
|
|
567
|
+
eventName !== "subagents:failed" &&
|
|
568
|
+
eventName !== "subagents:compacted"
|
|
440
569
|
)
|
|
441
570
|
return;
|
|
442
571
|
if (!raw || typeof raw !== "object") return;
|
|
@@ -489,13 +618,55 @@ export class WorkflowEngine {
|
|
|
489
618
|
if (currentAttemptId !== undefined && currentAttemptId !== `${owner.runId}/${callNodeId}/${owner.attemptId}`)
|
|
490
619
|
return;
|
|
491
620
|
|
|
621
|
+
const acceptedGeneration = state.executionGeneration;
|
|
622
|
+
const acceptedEpoch = this.branchEpoch;
|
|
623
|
+
const current = () =>
|
|
624
|
+
this.isCurrentState(state, acceptedGeneration, acceptedEpoch) && !state.controller.signal.aborted;
|
|
625
|
+
const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
|
|
626
|
+
const hasManagedUsage = terminalEvent && "managedTokens" in data;
|
|
627
|
+
const managedUsage = hasManagedUsage ? managedLifecycleUsage(data.managedTokens) : undefined;
|
|
628
|
+
// A malformed present field cannot fall back to cumulative lifetime usage.
|
|
629
|
+
// Reject before identity, presentation, buffering, or waiter mutations.
|
|
630
|
+
if (hasManagedUsage && !managedUsage) return;
|
|
631
|
+
|
|
632
|
+
if (eventName === "subagents:compacted") {
|
|
633
|
+
this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
|
|
634
|
+
if (!current()) return;
|
|
635
|
+
if (
|
|
636
|
+
typeof data.compactionCount === "number" &&
|
|
637
|
+
Number.isSafeInteger(data.compactionCount) &&
|
|
638
|
+
data.compactionCount >= 0
|
|
639
|
+
) {
|
|
640
|
+
const facts = state.managedCallFacts.get(callNodeId);
|
|
641
|
+
if (facts) facts.compactionCount = Math.max(facts.compactionCount, data.compactionCount);
|
|
642
|
+
if (state.run.attemptIds[callNodeId])
|
|
643
|
+
state.run.compactions[callNodeId] = facts?.compactionCount ?? data.compactionCount;
|
|
644
|
+
state.presentation.managed(callNodeId, { compactionCount: facts?.compactionCount ?? data.compactionCount });
|
|
645
|
+
this.notifyInspection(state);
|
|
646
|
+
}
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
492
649
|
if (updateCallTierIdentity(state, callNodeId, data.tier)) {
|
|
493
650
|
this.persistCallTierIdentity(state, callNodeId);
|
|
494
651
|
}
|
|
652
|
+
if (!current()) return;
|
|
653
|
+
this.recordManagedAdmission(state, callNodeId, agentId, state.executionGeneration);
|
|
654
|
+
if (!current()) return;
|
|
495
655
|
// A start is the moment the host admitted this agent from its queue. It is
|
|
496
656
|
// the only signal that separates waiting for a slot from doing the work, so
|
|
497
657
|
// the call's wall clock is restarted here rather than at dispatch.
|
|
498
|
-
if (eventName === "subagents:started"
|
|
658
|
+
if (eventName === "subagents:started") {
|
|
659
|
+
if (pendingMatches && pending) pending.startedAgentId = agentId;
|
|
660
|
+
if (waiterMatches && waiter) notifyAgentRunning(waiter);
|
|
661
|
+
if (!current()) return;
|
|
662
|
+
}
|
|
663
|
+
state.presentation.managed(callNodeId, {
|
|
664
|
+
agentId,
|
|
665
|
+
status: eventName === "subagents:started" ? "running" : undefined,
|
|
666
|
+
...(isManagedAgentTier(data.tier) ? { tier: data.tier } : {}),
|
|
667
|
+
});
|
|
668
|
+
this.notifyInspection(state);
|
|
669
|
+
if (!current()) return;
|
|
499
670
|
// Created/started lifecycle facts carry managed identity before the RPC
|
|
500
671
|
// reply is necessarily available. They are observation-only here; terminal
|
|
501
672
|
// ownership is handled below once a completion/failure fact arrives.
|
|
@@ -508,13 +679,27 @@ export class WorkflowEngine {
|
|
|
508
679
|
? "stopped"
|
|
509
680
|
: "failed";
|
|
510
681
|
const tokenCount =
|
|
511
|
-
|
|
512
|
-
?
|
|
513
|
-
: data.
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
682
|
+
managedUsage !== undefined
|
|
683
|
+
? managedUsage.total
|
|
684
|
+
: typeof data.tokenCount === "number"
|
|
685
|
+
? data.tokenCount
|
|
686
|
+
: data.tokens &&
|
|
687
|
+
typeof data.tokens === "object" &&
|
|
688
|
+
typeof (data.tokens as Record<string, unknown>).total === "number"
|
|
689
|
+
? ((data.tokens as Record<string, unknown>).total as number)
|
|
690
|
+
: undefined;
|
|
691
|
+
state.presentation.managed(callNodeId, {
|
|
692
|
+
agentId,
|
|
693
|
+
status: lifecycleStatus,
|
|
694
|
+
tokenCount,
|
|
695
|
+
...(typeof data.outputFile === "string" ? { outputFile: data.outputFile } : {}),
|
|
696
|
+
compactionCount: Math.max(
|
|
697
|
+
typeof data.compactionCount === "number" ? data.compactionCount : 0,
|
|
698
|
+
state.managedCallFacts.get(callNodeId)?.compactionCount ?? 0,
|
|
699
|
+
),
|
|
700
|
+
});
|
|
701
|
+
this.notifyInspection(state);
|
|
702
|
+
if (!current()) return;
|
|
518
703
|
const delivery: AgentTerminalDelivery = {
|
|
519
704
|
response: {
|
|
520
705
|
id: agentId,
|
|
@@ -528,9 +713,11 @@ export class WorkflowEngine {
|
|
|
528
713
|
completedAt: typeof data.completedAt === "number" ? data.completedAt : Date.now(),
|
|
529
714
|
},
|
|
530
715
|
},
|
|
531
|
-
...(
|
|
532
|
-
? {}
|
|
533
|
-
:
|
|
716
|
+
...(managedUsage !== undefined
|
|
717
|
+
? { usage: managedUsage }
|
|
718
|
+
: tokenCount === undefined
|
|
719
|
+
? {}
|
|
720
|
+
: { usage: { input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 } }),
|
|
534
721
|
...(Array.isArray(data.history) ? { history: [...data.history] } : {}),
|
|
535
722
|
};
|
|
536
723
|
|
|
@@ -547,6 +734,60 @@ export class WorkflowEngine {
|
|
|
547
734
|
state.bufferedTerminals.set(spawnKey, { ...delivery, owner, executionGeneration: state.executionGeneration });
|
|
548
735
|
}
|
|
549
736
|
|
|
737
|
+
private recordManagedAdmission(
|
|
738
|
+
state: ScriptRunState,
|
|
739
|
+
nodeId: string,
|
|
740
|
+
agentId: string,
|
|
741
|
+
executionGeneration: number,
|
|
742
|
+
): void {
|
|
743
|
+
const attemptId = state.run.attemptIds[nodeId];
|
|
744
|
+
if (!attemptId) return; // Nested facts remain volatile, owned by the parent boundary.
|
|
745
|
+
const existing = state.managedCallFacts.get(nodeId);
|
|
746
|
+
if (
|
|
747
|
+
existing?.executionGeneration === executionGeneration &&
|
|
748
|
+
existing.attemptId === attemptId &&
|
|
749
|
+
existing.agentId === agentId
|
|
750
|
+
)
|
|
751
|
+
return;
|
|
752
|
+
state.managedCallFacts.set(nodeId, { executionGeneration, attemptId, agentId, compactionCount: 0 });
|
|
753
|
+
state.run.agentIds[nodeId] = agentId;
|
|
754
|
+
state.run.compactions[nodeId] = 0;
|
|
755
|
+
const event: JournalEvent = {
|
|
756
|
+
kind: "call_transition",
|
|
757
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
758
|
+
runId: state.run.runId,
|
|
759
|
+
nodeId,
|
|
760
|
+
status: "running",
|
|
761
|
+
agentId,
|
|
762
|
+
...(state.callTiers.get(nodeId)?.tier === undefined ? {} : { tier: state.callTiers.get(nodeId)?.tier }),
|
|
763
|
+
attemptId,
|
|
764
|
+
owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
|
|
765
|
+
timestamp: Date.now(),
|
|
766
|
+
};
|
|
767
|
+
this.persist(state.run.runId, `call-identity:${nodeId}`, () => this.write(event));
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
private recordManagedTerminal(
|
|
771
|
+
state: ScriptRunState,
|
|
772
|
+
nodeId: string,
|
|
773
|
+
agentId: string,
|
|
774
|
+
terminal: NonNullable<ManagedSpawnResponse["terminal"]>,
|
|
775
|
+
executionGeneration: number,
|
|
776
|
+
): void {
|
|
777
|
+
this.recordManagedAdmission(state, nodeId, agentId, executionGeneration);
|
|
778
|
+
const facts = state.managedCallFacts.get(nodeId);
|
|
779
|
+
if (
|
|
780
|
+
!facts ||
|
|
781
|
+
facts.executionGeneration !== executionGeneration ||
|
|
782
|
+
facts.attemptId !== state.run.attemptIds[nodeId] ||
|
|
783
|
+
facts.agentId !== agentId
|
|
784
|
+
)
|
|
785
|
+
return;
|
|
786
|
+
facts.tokenCount = terminal.tokenCount;
|
|
787
|
+
facts.outputFile = terminal.outputFile;
|
|
788
|
+
facts.compactionCount = Math.max(facts.compactionCount, terminal.compactionCount);
|
|
789
|
+
}
|
|
790
|
+
|
|
550
791
|
/** Persist a host-normalized tier learned before the managed reply arrives. */
|
|
551
792
|
private persistCallTierIdentity(state: ScriptRunState, nodeId: string): void {
|
|
552
793
|
const attemptId = state.run.attemptIds[nodeId];
|
|
@@ -554,19 +795,18 @@ export class WorkflowEngine {
|
|
|
554
795
|
// Nested workflow frames are in-memory only; only a top-level call has a
|
|
555
796
|
// durable attempt owner and can safely receive a journal transition.
|
|
556
797
|
if (!attemptId || identity?.tier === undefined) return;
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
);
|
|
798
|
+
const event: JournalEvent = {
|
|
799
|
+
kind: "call_transition",
|
|
800
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
801
|
+
runId: state.run.runId,
|
|
802
|
+
nodeId,
|
|
803
|
+
status: "running",
|
|
804
|
+
tier: identity.tier,
|
|
805
|
+
attemptId,
|
|
806
|
+
owner: { extension: "pi-workflows", runId: state.run.runId, nodeId, attemptId },
|
|
807
|
+
timestamp: Date.now(),
|
|
808
|
+
};
|
|
809
|
+
this.persist(state.run.runId, `call-tier:${nodeId}`, () => this.write(event));
|
|
570
810
|
}
|
|
571
811
|
|
|
572
812
|
private resolveAgentWaiter(
|
|
@@ -578,9 +818,23 @@ export class WorkflowEngine {
|
|
|
578
818
|
if (waiter.settled) return;
|
|
579
819
|
waiter.settled = true;
|
|
580
820
|
state.agentWaiters.delete(agentId);
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
821
|
+
const epoch = this.branchEpoch;
|
|
822
|
+
try {
|
|
823
|
+
if (delivery.usage) waiter.onUsage?.(delivery.usage);
|
|
824
|
+
if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
|
|
825
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
826
|
+
return;
|
|
827
|
+
}
|
|
828
|
+
if (delivery.history) waiter.onHistory?.(delivery.history);
|
|
829
|
+
if (!this.isCurrentState(state, waiter.executionGeneration, epoch) || state.controller.signal.aborted) {
|
|
830
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
waiter.resolve(delivery.response);
|
|
834
|
+
} catch (error) {
|
|
835
|
+
// A throwing observer cannot strand a removed waiter and its child lease.
|
|
836
|
+
waiter.reject(error);
|
|
837
|
+
}
|
|
584
838
|
}
|
|
585
839
|
|
|
586
840
|
dispose(): void {
|
|
@@ -589,12 +843,14 @@ export class WorkflowEngine {
|
|
|
589
843
|
this.disposeController.abort();
|
|
590
844
|
this.unsubscribeLifecycle();
|
|
591
845
|
this.quarantinedRunIds.clear();
|
|
592
|
-
|
|
846
|
+
const retiredStates = [...this.runs.values()];
|
|
847
|
+
for (const state of retiredStates) {
|
|
593
848
|
this.cleanupAgents(state.run.runId);
|
|
594
849
|
state.controller.abort(new DOMException("Workflow engine disposed", "AbortError"));
|
|
595
850
|
}
|
|
596
851
|
this.rejectWaiters(new WorkflowWaitAbortedError());
|
|
597
852
|
this.runs.clear();
|
|
853
|
+
for (const state of retiredStates) this.notifyInspection(state);
|
|
598
854
|
this.clearJournalRetries();
|
|
599
855
|
this.branchQuiescePromise = undefined;
|
|
600
856
|
}
|
|
@@ -613,68 +869,58 @@ export class WorkflowEngine {
|
|
|
613
869
|
|
|
614
870
|
/** Stop owned agents while Pi replaces the active session-tree branch. */
|
|
615
871
|
async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
|
|
616
|
-
if (this.disposed) return { settled: true, pending: [] };
|
|
617
872
|
if (this.branchQuiescePromise) return this.branchQuiescePromise;
|
|
618
873
|
this.admissionsClosed = true;
|
|
619
874
|
this.clearJournalRetries();
|
|
620
875
|
|
|
621
|
-
//
|
|
622
|
-
//
|
|
623
|
-
//
|
|
624
|
-
const
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
const
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
876
|
+
// Snapshot exact states/owners, not run ids or status. Terminal and retired
|
|
877
|
+
// states may still own cleanup, and observation must not donate those
|
|
878
|
+
// leases to a same-id replacement on a newly committed branch.
|
|
879
|
+
const states = [...new Set([...this.runs.values(), ...this.retainedCleanupStates])];
|
|
880
|
+
const targets = states.flatMap((state) => {
|
|
881
|
+
const owners = new Map<string, { agentId: string; owner: WorkflowOwner; generation: number }>();
|
|
882
|
+
for (const pending of state.pendingSpawns.values()) {
|
|
883
|
+
if (pending.phase !== "submitted") continue;
|
|
884
|
+
owners.set(JSON.stringify(pending.owner), {
|
|
885
|
+
agentId: pending.agentId ?? pending.startedAgentId ?? pending.spawnKey,
|
|
886
|
+
owner: { ...pending.owner },
|
|
887
|
+
generation: pending.executionGeneration,
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
for (const [agentId, waiter] of state.agentWaiters) {
|
|
891
|
+
const owner: WorkflowOwner = {
|
|
892
|
+
extension: "pi-workflows",
|
|
632
893
|
runId: state.run.runId,
|
|
633
|
-
nodeId:
|
|
634
|
-
attemptId: `attempt-${waiter
|
|
894
|
+
nodeId: `call-${waiter.callIndex}`,
|
|
895
|
+
attemptId: `attempt-${waiter.generation}`,
|
|
635
896
|
};
|
|
636
|
-
|
|
637
|
-
targets.set(state.run.runId, { agentIds, owners });
|
|
638
|
-
state.lifecycleSuspended = true;
|
|
639
|
-
state.bufferedTerminals.clear();
|
|
640
|
-
state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
const operation = (async () => {
|
|
644
|
-
const operations: Array<Promise<WorkflowQuiesceResult>> = [];
|
|
645
|
-
for (const state of this.runs.values()) {
|
|
646
|
-
const run = state.run;
|
|
647
|
-
const target = targets.get(run.runId);
|
|
648
|
-
if (isTerminalWorkflow(run.status)) {
|
|
649
|
-
this.reconcilePendingSpawns(state);
|
|
650
|
-
continue;
|
|
651
|
-
}
|
|
652
|
-
if (state.pendingSpawns.size > 0) operations.push(this.quiescePendingSpawns(state));
|
|
653
|
-
const activeAgentIds = target?.agentIds ?? [];
|
|
654
|
-
if (activeAgentIds.length === 0) continue;
|
|
655
|
-
operations.push(
|
|
656
|
-
this.client
|
|
657
|
-
.quiesceOwned?.(run.runId, activeAgentIds, BRANCH_QUIESCE_TIMEOUT_MS, target?.owners)
|
|
658
|
-
.then((result) => ({
|
|
659
|
-
settled: result.settled && result.pending.length === 0,
|
|
660
|
-
pending: [...new Set(result.pending)].slice(0, 256),
|
|
661
|
-
}))
|
|
662
|
-
.catch((error: unknown) => ({
|
|
663
|
-
settled: false,
|
|
664
|
-
pending: activeAgentIds.slice(0, 256),
|
|
665
|
-
diagnostic: error instanceof Error ? error.message : String(error),
|
|
666
|
-
})) ?? Promise.resolve({ settled: false, pending: activeAgentIds.slice(0, 256) }),
|
|
667
|
-
);
|
|
897
|
+
owners.set(JSON.stringify(owner), { agentId, owner, generation: waiter.executionGeneration });
|
|
668
898
|
}
|
|
669
|
-
const
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
};
|
|
677
|
-
})
|
|
899
|
+
for (const cleanup of this.cleanupObligations.get(state)?.values() ?? []) {
|
|
900
|
+
owners.set(JSON.stringify(cleanup.owner), {
|
|
901
|
+
agentId: cleanup.agentId,
|
|
902
|
+
owner: { ...cleanup.owner },
|
|
903
|
+
generation: state.executionGeneration,
|
|
904
|
+
});
|
|
905
|
+
}
|
|
906
|
+
return [...owners.values()].map((target) => ({ state, ...target }));
|
|
907
|
+
});
|
|
908
|
+
// Publish the handshake before RPC, abort or inspection can re-enter.
|
|
909
|
+
let acceptOperations!: (operations: Promise<void>[]) => void;
|
|
910
|
+
const operations = new Promise<Promise<void>[]>((resolve) => {
|
|
911
|
+
acceptOperations = resolve;
|
|
912
|
+
});
|
|
913
|
+
const operation = operations.then(async (cleanup) => {
|
|
914
|
+
await Promise.all(cleanup);
|
|
915
|
+
const pending = [
|
|
916
|
+
...new Set(
|
|
917
|
+
states.flatMap((state) =>
|
|
918
|
+
[...(this.cleanupObligations.get(state)?.values() ?? [])].map((cleanup) => cleanup.agentId),
|
|
919
|
+
),
|
|
920
|
+
),
|
|
921
|
+
].slice(0, 256);
|
|
922
|
+
return { settled: pending.length === 0, pending };
|
|
923
|
+
});
|
|
678
924
|
const tracked = operation.then(
|
|
679
925
|
(result) => {
|
|
680
926
|
if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
|
|
@@ -686,6 +932,34 @@ export class WorkflowEngine {
|
|
|
686
932
|
},
|
|
687
933
|
);
|
|
688
934
|
this.branchQuiescePromise = tracked;
|
|
935
|
+
// Fence every captured state before the first notifying observer. Start
|
|
936
|
+
// known-id owner checks synchronously for the bus-first shutdown handshake;
|
|
937
|
+
// their positive result is consumed by the same stable cleanup obligation,
|
|
938
|
+
// never treated as a status-only shortcut or a separate donated lease.
|
|
939
|
+
for (const state of states) state.lifecycleSuspended = true;
|
|
940
|
+
acceptOperations(
|
|
941
|
+
targets.map(({ state, agentId, owner, generation }) => {
|
|
942
|
+
const existing = [...(this.cleanupObligations.get(state)?.values() ?? [])].find((cleanup) =>
|
|
943
|
+
sameWorkflowOwner(cleanup.owner, owner),
|
|
944
|
+
);
|
|
945
|
+
let quiescence: Promise<{ settled: boolean; pending: string[] }> | undefined;
|
|
946
|
+
if (!existing?.operation && agentId !== `${owner.runId}/${owner.nodeId}/${owner.attemptId}`) {
|
|
947
|
+
try {
|
|
948
|
+
quiescence = this.client.quiesceOwned?.(owner.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner]);
|
|
949
|
+
} catch (error) {
|
|
950
|
+
quiescence = Promise.reject(error);
|
|
951
|
+
}
|
|
952
|
+
void quiescence?.catch(() => {});
|
|
953
|
+
}
|
|
954
|
+
return this.stopAndQuiesce(state, agentId, owner, generation, undefined, quiescence);
|
|
955
|
+
}),
|
|
956
|
+
);
|
|
957
|
+
for (const state of states) {
|
|
958
|
+
state.bufferedTerminals.clear();
|
|
959
|
+
state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
|
|
960
|
+
state.presentation.reset(false);
|
|
961
|
+
this.notifyInspection(state);
|
|
962
|
+
}
|
|
689
963
|
return tracked;
|
|
690
964
|
}
|
|
691
965
|
|
|
@@ -780,8 +1054,14 @@ export class WorkflowEngine {
|
|
|
780
1054
|
...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
|
|
781
1055
|
timestamp: now,
|
|
782
1056
|
});
|
|
1057
|
+
if (epoch !== this.branchEpoch || this.disposed || this.lifecyclePaused || this.admissionsClosed)
|
|
1058
|
+
throw new Error("Workflow branch changed before admission");
|
|
783
1059
|
const state: ScriptRunState = {
|
|
784
1060
|
run,
|
|
1061
|
+
presentation: new WorkflowPresentation(runId),
|
|
1062
|
+
managedCallFacts: new Map(),
|
|
1063
|
+
pendingCleanup: new Set(),
|
|
1064
|
+
activeExecutionGenerations: new Set(),
|
|
785
1065
|
frozenOptions: {
|
|
786
1066
|
...(frozenTokenBudget !== undefined ? { tokenBudget: frozenTokenBudget } : {}),
|
|
787
1067
|
...(frozenMaxAgents !== undefined ? { maxAgents: frozenMaxAgents } : {}),
|
|
@@ -802,13 +1082,19 @@ export class WorkflowEngine {
|
|
|
802
1082
|
executionGeneration: 0,
|
|
803
1083
|
};
|
|
804
1084
|
this.runs.set(runId, state);
|
|
805
|
-
|
|
806
|
-
state.
|
|
807
|
-
|
|
1085
|
+
// Accept identity before any synchronous running/root observer can stop or
|
|
1086
|
+
// replace this state. Dispatch is deferred until its promise is published.
|
|
1087
|
+
const generation = ++state.executionGeneration;
|
|
1088
|
+
state.presentation.reset();
|
|
1089
|
+
state.execution = Promise.resolve().then(() =>
|
|
1090
|
+
this.execute(state, script, startOptions, undefined, generation, epoch, protocol),
|
|
1091
|
+
);
|
|
808
1092
|
const execution = state.execution;
|
|
809
1093
|
// A run-fatal abort/stop already journals its terminal state; never let a
|
|
810
1094
|
// leftover rejection become an unhandled promise rejection.
|
|
811
|
-
void
|
|
1095
|
+
void execution.catch(() => {});
|
|
1096
|
+
this.setWorkflowStatus(run, "running");
|
|
1097
|
+
if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
|
|
812
1098
|
if (options.background) return { runId, status: run.status, background: true };
|
|
813
1099
|
try {
|
|
814
1100
|
const settled = await this.waitFor(runId, options.signal);
|
|
@@ -859,7 +1145,7 @@ export class WorkflowEngine {
|
|
|
859
1145
|
throw new Error("Workflow branch changed before admission");
|
|
860
1146
|
const state = this.runs.get(runId);
|
|
861
1147
|
if (!state) return undefined;
|
|
862
|
-
if (state.lifecycleSuspended) return undefined;
|
|
1148
|
+
if (state.lifecycleSuspended || state.removeOperation) return undefined;
|
|
863
1149
|
const run = state.run;
|
|
864
1150
|
if (run.status === "running" || run.status === "pausing" || run.status === "stopping") return undefined;
|
|
865
1151
|
const providerTimer = this.providerResumeTimers.get(runId);
|
|
@@ -867,8 +1153,53 @@ export class WorkflowEngine {
|
|
|
867
1153
|
clearTimeout(providerTimer);
|
|
868
1154
|
this.providerResumeTimers.delete(runId);
|
|
869
1155
|
}
|
|
870
|
-
if (
|
|
1156
|
+
if (
|
|
1157
|
+
run.nonResumable ||
|
|
1158
|
+
run.terminalIntent === "stop" ||
|
|
1159
|
+
(isTerminalWorkflow(run.status) && run.status !== "failed" && run.status !== "stopped")
|
|
1160
|
+
)
|
|
871
1161
|
return undefined;
|
|
1162
|
+
if (isTerminalWorkflow(run.status)) {
|
|
1163
|
+
await this.drainExecution(state, state.execution);
|
|
1164
|
+
for (const [nodeId, agentId] of Object.entries(run.agentIds)) {
|
|
1165
|
+
if (run.callStatus[nodeId] !== "running") continue; // Completed/local calls own no live child.
|
|
1166
|
+
const attemptId = run.attemptIds[nodeId];
|
|
1167
|
+
if (!attemptId) continue;
|
|
1168
|
+
await this.stopAndQuiesce(
|
|
1169
|
+
state,
|
|
1170
|
+
agentId,
|
|
1171
|
+
{
|
|
1172
|
+
extension: "pi-workflows",
|
|
1173
|
+
runId,
|
|
1174
|
+
nodeId: `call-${nodeId}`,
|
|
1175
|
+
attemptId: `attempt-${generationFromAttemptId(attemptId)}`,
|
|
1176
|
+
},
|
|
1177
|
+
state.executionGeneration,
|
|
1178
|
+
);
|
|
1179
|
+
}
|
|
1180
|
+
if (run.nonResumable || this.runs.get(runId) !== state || epoch !== this.branchEpoch || this.lifecyclePaused)
|
|
1181
|
+
return undefined;
|
|
1182
|
+
if (
|
|
1183
|
+
this.journalBlockedRuns.has(runId) ||
|
|
1184
|
+
[...this.journalRetries.keys()].some((key) => key.startsWith(`${runId}\u0000`))
|
|
1185
|
+
)
|
|
1186
|
+
throw new Error("Workflow journal has not settled; retry resume after persistence recovers");
|
|
1187
|
+
const timestamp = Date.now();
|
|
1188
|
+
const recovery: RunRecoveryEvent = {
|
|
1189
|
+
kind: "run_recovery",
|
|
1190
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1191
|
+
runId,
|
|
1192
|
+
status: "interrupted",
|
|
1193
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
1194
|
+
rotations: [],
|
|
1195
|
+
recoveryId: deriveRecoveryId({ runId, rotations: [], timestamp }),
|
|
1196
|
+
timestamp,
|
|
1197
|
+
};
|
|
1198
|
+
this.appendRequired(recovery);
|
|
1199
|
+
if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
|
|
1200
|
+
applyRecoveryEvent(run, recovery);
|
|
1201
|
+
state.result = undefined;
|
|
1202
|
+
}
|
|
872
1203
|
if (replacementScript !== undefined) {
|
|
873
1204
|
const parsed = parseWorkflowScript(replacementScript);
|
|
874
1205
|
const revision = (run.revision ?? 0) + 1;
|
|
@@ -889,6 +1220,7 @@ export class WorkflowEngine {
|
|
|
889
1220
|
}),
|
|
890
1221
|
);
|
|
891
1222
|
}
|
|
1223
|
+
if (!this.isCurrentState(state, state.executionGeneration, epoch)) return undefined;
|
|
892
1224
|
// Arguments are frozen in every schema-v4 run_created fact. Resume never accepts
|
|
893
1225
|
// replacement args and never backfills the pre-schema-v4 run_args contract.
|
|
894
1226
|
const resumeArgs = cloneWorkflowArgs(run.args);
|
|
@@ -898,8 +1230,6 @@ export class WorkflowEngine {
|
|
|
898
1230
|
for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
|
|
899
1231
|
state.generations.set(callIndex, generationFromAttemptId(attemptId));
|
|
900
1232
|
}
|
|
901
|
-
this.setWorkflowStatus(run, "running");
|
|
902
|
-
state.controller = new AbortController();
|
|
903
1233
|
// Freeze semantics: original run parameters and args win over new options.
|
|
904
1234
|
// Only signal/confirm/background/loadSavedWorkflow/mainModel/strengths are
|
|
905
1235
|
// per-resume context; input/budget/scale/toolset are frozen from the
|
|
@@ -911,6 +1241,7 @@ export class WorkflowEngine {
|
|
|
911
1241
|
// a caller that passed none is asking the engine to read the current one.
|
|
912
1242
|
strengths: options.strengths ?? this.strengths?.(),
|
|
913
1243
|
args: resumeArgs,
|
|
1244
|
+
loadSavedWorkflow: options.loadSavedWorkflow ?? this.loadSavedWorkflow,
|
|
914
1245
|
tokenBudget: frozen.tokenBudget,
|
|
915
1246
|
maxAgents: frozen.maxAgents,
|
|
916
1247
|
concurrency: frozen.concurrency,
|
|
@@ -919,19 +1250,25 @@ export class WorkflowEngine {
|
|
|
919
1250
|
toolset: frozen.toolset,
|
|
920
1251
|
excludeTools: frozen.excludeTools ? [...frozen.excludeTools] : undefined,
|
|
921
1252
|
};
|
|
1253
|
+
if (!this.isCurrentState(state, state.executionGeneration, epoch) || run.nonResumable || state.stopOperation)
|
|
1254
|
+
return undefined;
|
|
922
1255
|
state.bufferedTerminals.clear();
|
|
923
|
-
state
|
|
924
|
-
state.
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
state
|
|
930
|
-
protocol,
|
|
1256
|
+
this.terminalNotified.delete(state);
|
|
1257
|
+
state.controller = new AbortController();
|
|
1258
|
+
const generation = ++state.executionGeneration;
|
|
1259
|
+
const acceptedScript = run.script;
|
|
1260
|
+
state.presentation.reset();
|
|
1261
|
+
state.execution = Promise.resolve().then(() =>
|
|
1262
|
+
this.execute(state, acceptedScript, resumeOptions, resumeJournal, generation, epoch, protocol),
|
|
931
1263
|
);
|
|
932
1264
|
const execution = state.execution;
|
|
933
1265
|
void execution.catch(() => {});
|
|
934
|
-
|
|
1266
|
+
this.setWorkflowStatus(run, "running");
|
|
1267
|
+
if (this.isCurrentState(state, generation, epoch)) this.notifyInspection(state);
|
|
1268
|
+
if (options.background) {
|
|
1269
|
+
this.onBackgroundExecution?.(runId);
|
|
1270
|
+
return { runId, status: run.status, background: true };
|
|
1271
|
+
}
|
|
935
1272
|
try {
|
|
936
1273
|
const settled = await this.waitFor(runId, options.signal);
|
|
937
1274
|
return { runId, status: settled.status, background: false };
|
|
@@ -986,11 +1323,7 @@ export class WorkflowEngine {
|
|
|
986
1323
|
await this.stop(state);
|
|
987
1324
|
} else if (action === "rm") {
|
|
988
1325
|
if (!isTerminalWorkflow(run.status)) throw new Error("cannot remove a running workflow");
|
|
989
|
-
this.
|
|
990
|
-
this.write({ kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp: Date.now() }),
|
|
991
|
-
);
|
|
992
|
-
this.runs.delete(runId);
|
|
993
|
-
this.waiters.delete(runId);
|
|
1326
|
+
await this.remove(state);
|
|
994
1327
|
return { action, run: this.summary(run) };
|
|
995
1328
|
}
|
|
996
1329
|
return { action, run: this.summary(run) };
|
|
@@ -999,6 +1332,7 @@ export class WorkflowEngine {
|
|
|
999
1332
|
async waitFor(runId: string, signal?: AbortSignal): Promise<ScriptRun> {
|
|
1000
1333
|
const state = this.runs.get(runId);
|
|
1001
1334
|
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
1335
|
+
if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size > 0) throw new WorkflowWaitAbortedError();
|
|
1002
1336
|
if (isTerminalWorkflow(state.run.status) || state.run.status === "paused" || state.run.status === "interrupted")
|
|
1003
1337
|
return state.run;
|
|
1004
1338
|
if (signal?.aborted) throw new WorkflowWaitAbortedError();
|
|
@@ -1047,6 +1381,7 @@ export class WorkflowEngine {
|
|
|
1047
1381
|
}
|
|
1048
1382
|
|
|
1049
1383
|
private settleWaiters(runId: string, run: ScriptRun): void {
|
|
1384
|
+
if (this.runs.get(runId)?.run !== run || this.hasPendingCleanup(runId)) return;
|
|
1050
1385
|
const all = this.waiters.get(runId);
|
|
1051
1386
|
if (!all) return;
|
|
1052
1387
|
for (const waiter of [...all]) {
|
|
@@ -1062,24 +1397,24 @@ export class WorkflowEngine {
|
|
|
1062
1397
|
// ── execution ──────────────────────────────────────────────────────────────
|
|
1063
1398
|
|
|
1064
1399
|
private async execute(
|
|
1065
|
-
|
|
1400
|
+
state: ScriptRunState,
|
|
1066
1401
|
script: string,
|
|
1067
1402
|
options: ScriptStartOptions,
|
|
1068
|
-
resumeJournal
|
|
1069
|
-
|
|
1403
|
+
resumeJournal: Map<string, JournalEntry> | undefined,
|
|
1404
|
+
generation: number,
|
|
1405
|
+
epoch: number,
|
|
1070
1406
|
protocol?: ManagedProtocolCheck,
|
|
1071
1407
|
): Promise<WorkflowRunResult<unknown>> {
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
1075
|
-
const generation = executionGeneration ?? state.executionGeneration;
|
|
1076
|
-
const epoch = this.branchEpoch;
|
|
1408
|
+
const runId = state.run.runId;
|
|
1409
|
+
state.activeExecutionGenerations.add(generation);
|
|
1077
1410
|
const isSameBranch = () =>
|
|
1078
1411
|
this.runs.get(runId) === state &&
|
|
1079
1412
|
state.executionGeneration === generation &&
|
|
1080
1413
|
this.branchEpoch === epoch &&
|
|
1081
1414
|
!this.disposed;
|
|
1082
1415
|
const isCurrent = () => isSameBranch() && !state.lifecycleSuspended && !this.lifecyclePaused;
|
|
1416
|
+
const isPresentationCurrent = () => isCurrent() && state.activeExecutionGenerations.has(generation);
|
|
1417
|
+
const explicitlyStopped = () => state.stopOperation !== undefined || state.run.terminalIntent === "stop";
|
|
1083
1418
|
const executionSignal =
|
|
1084
1419
|
!options.background && options.signal
|
|
1085
1420
|
? AbortSignal.any([state.controller.signal, options.signal])
|
|
@@ -1088,7 +1423,8 @@ export class WorkflowEngine {
|
|
|
1088
1423
|
const cleanupTasks = new Set<Promise<void>>();
|
|
1089
1424
|
const runner: WorkflowAgentRunner = {
|
|
1090
1425
|
run: async (prompt: string, runOptions: AgentRunOptions = {}) => {
|
|
1091
|
-
if (!isCurrent()
|
|
1426
|
+
if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
|
|
1427
|
+
throw new WorkflowError("stale workflow execution", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1092
1428
|
const dispatch = this.dispatchAgent(runId, prompt, runOptions);
|
|
1093
1429
|
const settled = dispatch.then(
|
|
1094
1430
|
() => {},
|
|
@@ -1102,6 +1438,15 @@ export class WorkflowEngine {
|
|
|
1102
1438
|
if (runOptions.signal?.aborted) trackCleanup();
|
|
1103
1439
|
try {
|
|
1104
1440
|
return await dispatch;
|
|
1441
|
+
} catch (error: unknown) {
|
|
1442
|
+
if (isCurrent()) {
|
|
1443
|
+
const frameId = runOptions.runId ?? runId;
|
|
1444
|
+
const nodeId =
|
|
1445
|
+
frameId === runId ? String(runOptions.callIndex ?? 0) : `${frameId}:${runOptions.callIndex ?? 0}`;
|
|
1446
|
+
state.presentation.managed(nodeId, { status: runOptions.signal?.aborted ? "stopped" : "failed" });
|
|
1447
|
+
this.notifyInspection(state);
|
|
1448
|
+
}
|
|
1449
|
+
throw error;
|
|
1105
1450
|
} finally {
|
|
1106
1451
|
runOptions.signal?.removeEventListener("abort", trackCleanup);
|
|
1107
1452
|
}
|
|
@@ -1110,6 +1455,8 @@ export class WorkflowEngine {
|
|
|
1110
1455
|
};
|
|
1111
1456
|
|
|
1112
1457
|
try {
|
|
1458
|
+
if (!isCurrent() || executionSignal.aborted || explicitlyStopped())
|
|
1459
|
+
throw new WorkflowError("workflow execution aborted before dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1113
1460
|
const result = await runWorkflow(script, {
|
|
1114
1461
|
cwd: this.cwd,
|
|
1115
1462
|
args: options.args,
|
|
@@ -1134,29 +1481,74 @@ export class WorkflowEngine {
|
|
|
1134
1481
|
runId,
|
|
1135
1482
|
executionNonce: `${generation}-${randomUUID()}`,
|
|
1136
1483
|
resumeJournal,
|
|
1484
|
+
resumeWorkflowGenerations: new Map(
|
|
1485
|
+
Object.entries(state.run.workflowResultGenerations).map(([index, generation]) => [Number(index), generation]),
|
|
1486
|
+
),
|
|
1137
1487
|
toolset: options.toolset,
|
|
1138
1488
|
strengths: options.strengths,
|
|
1139
1489
|
excludeTools: options.excludeTools,
|
|
1140
|
-
confirm:
|
|
1490
|
+
confirm:
|
|
1491
|
+
options.confirm &&
|
|
1492
|
+
(async (prompt, checkpointOptions, signal) => {
|
|
1493
|
+
if (!isCurrent() || executionSignal.aborted || signal?.aborted || state.stopOperation)
|
|
1494
|
+
throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1495
|
+
const value = await options.confirm?.(prompt, checkpointOptions, signal);
|
|
1496
|
+
if (!isCurrent() || executionSignal.aborted || signal?.aborted)
|
|
1497
|
+
throw new WorkflowError("stale workflow checkpoint", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1498
|
+
return value;
|
|
1499
|
+
}),
|
|
1141
1500
|
loadSavedWorkflow: options.loadSavedWorkflow,
|
|
1501
|
+
onAgentStart: (event) => {
|
|
1502
|
+
if (!isPresentationCurrent()) return;
|
|
1503
|
+
state.presentation.agentStart(event);
|
|
1504
|
+
this.notifyInspection(state);
|
|
1505
|
+
},
|
|
1506
|
+
onAgentEnd: (event) => {
|
|
1507
|
+
if (!isPresentationCurrent()) return;
|
|
1508
|
+
const nodeId = event.id.startsWith(`${runId}:`) ? event.id.slice(runId.length + 1) : undefined;
|
|
1509
|
+
const persisted = nodeId === undefined ? undefined : state.run.callResults[nodeId];
|
|
1510
|
+
const persistedTokens =
|
|
1511
|
+
nodeId !== undefined && persisted?.attemptId === state.run.attemptIds[nodeId]
|
|
1512
|
+
? persisted?.tokenCount
|
|
1513
|
+
: undefined;
|
|
1514
|
+
// Runtime replay reports zero new spend, not the historical call's
|
|
1515
|
+
// usage. Explicit managed zero is also authoritative over estimates.
|
|
1516
|
+
const tokenCount = event.tokenUsage?.total ?? persistedTokens ?? event.tokens;
|
|
1517
|
+
state.presentation.agentEnd({ ...event, tokens: tokenCount });
|
|
1518
|
+
this.notifyInspection(state);
|
|
1519
|
+
},
|
|
1520
|
+
onLog: (message, scope) => {
|
|
1521
|
+
if (!isPresentationCurrent()) return;
|
|
1522
|
+
state.presentation.log(message, scope?.frameId);
|
|
1523
|
+
this.notifyInspection(state);
|
|
1524
|
+
},
|
|
1142
1525
|
onAgentJournal: (entry) => {
|
|
1143
1526
|
if (!isCurrent()) return;
|
|
1144
|
-
//
|
|
1145
|
-
//
|
|
1146
|
-
// workflow_result fact below owns the nested replay boundary.
|
|
1527
|
+
// Persist the accepted owner before presentation observers can replace it.
|
|
1528
|
+
// Nested frames are owned by their parent's workflow_result boundary.
|
|
1147
1529
|
if (entry.runId === undefined || entry.runId === runId) this.journalCallResult(runId, entry);
|
|
1530
|
+
if (!isCurrent()) return;
|
|
1531
|
+
state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
|
|
1532
|
+
this.notifyInspection(state);
|
|
1148
1533
|
},
|
|
1149
1534
|
onWorkflowJournal: (entry) => {
|
|
1150
1535
|
if (!isCurrent()) return;
|
|
1151
1536
|
this.journalWorkflowResult(runId, entry);
|
|
1537
|
+
if (!isCurrent()) return;
|
|
1538
|
+
state.presentation.journalResult(entry.runId ?? runId, entry.index, entry.result);
|
|
1539
|
+
this.notifyInspection(state);
|
|
1152
1540
|
},
|
|
1153
1541
|
onRuntimeEvent: (event) => {
|
|
1154
|
-
if (!
|
|
1542
|
+
if (!isPresentationCurrent()) return;
|
|
1543
|
+
state.presentation.runtime(event);
|
|
1544
|
+
this.notifyInspection(state);
|
|
1545
|
+
if (!isPresentationCurrent()) return;
|
|
1155
1546
|
try {
|
|
1156
1547
|
this.events.emit("pi-workflows:runtime", { runId, event: cloneRuntimeEvent(event) });
|
|
1157
1548
|
} catch {
|
|
1158
1549
|
// Cross-extension observers cannot change workflow execution.
|
|
1159
1550
|
}
|
|
1551
|
+
if (!isPresentationCurrent()) return;
|
|
1160
1552
|
try {
|
|
1161
1553
|
this.onRuntimeEvent?.(runId, cloneRuntimeEvent(event));
|
|
1162
1554
|
} catch {
|
|
@@ -1170,6 +1562,19 @@ export class WorkflowEngine {
|
|
|
1170
1562
|
recoverable: true,
|
|
1171
1563
|
});
|
|
1172
1564
|
}
|
|
1565
|
+
// A terminal frame cannot overtake accepted journal facts. In particular,
|
|
1566
|
+
// a delayed call_attempt must precede its result, not reset it on replay.
|
|
1567
|
+
await this.drainJournal(runId);
|
|
1568
|
+
if (
|
|
1569
|
+
executionSignal.aborted ||
|
|
1570
|
+
!isCurrent() ||
|
|
1571
|
+
state.run.status === "stopping" ||
|
|
1572
|
+
state.stopOperation !== undefined ||
|
|
1573
|
+
state.run.terminalIntent === "stop"
|
|
1574
|
+
)
|
|
1575
|
+
throw new WorkflowError("workflow execution aborted", WorkflowErrorCode.WORKFLOW_ABORTED, {
|
|
1576
|
+
recoverable: true,
|
|
1577
|
+
});
|
|
1173
1578
|
const live = this.runs.get(runId);
|
|
1174
1579
|
const run = live?.run;
|
|
1175
1580
|
if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
|
|
@@ -1180,8 +1585,8 @@ export class WorkflowEngine {
|
|
|
1180
1585
|
} catch (error: unknown) {
|
|
1181
1586
|
await runner.waitForCleanup?.();
|
|
1182
1587
|
const run = this.runs.get(runId)?.run;
|
|
1183
|
-
if (run && isSameBranch() && !isTerminalWorkflow(run.status)) {
|
|
1184
|
-
const message =
|
|
1588
|
+
if (run && isSameBranch() && !isTerminalWorkflow(run.status) && run.status !== "stopping") {
|
|
1589
|
+
const message = describeWorkflowError(error);
|
|
1185
1590
|
const workflowError = error instanceof WorkflowError ? error : wrapError(error);
|
|
1186
1591
|
if (workflowError.code === WorkflowErrorCode.PROVIDER_USAGE_LIMIT) {
|
|
1187
1592
|
this.setWorkflowStatus(run, "paused", message);
|
|
@@ -1203,15 +1608,22 @@ export class WorkflowEngine {
|
|
|
1203
1608
|
run.status = "interrupted";
|
|
1204
1609
|
run.updatedAt = Date.now();
|
|
1205
1610
|
} else {
|
|
1206
|
-
this.setWorkflowStatus(run, "failed", message);
|
|
1207
1611
|
this.cleanupAgents(runId);
|
|
1612
|
+
this.setWorkflowStatus(run, "failed", message);
|
|
1208
1613
|
}
|
|
1209
1614
|
}
|
|
1210
1615
|
throw error;
|
|
1211
1616
|
} finally {
|
|
1617
|
+
state.activeExecutionGenerations.delete(generation);
|
|
1212
1618
|
// Reject any waiters that never settled (agents left in flight).
|
|
1213
1619
|
const run = this.runs.get(runId)?.run;
|
|
1214
|
-
if (run && isSameBranch())
|
|
1620
|
+
if (run && isSameBranch()) {
|
|
1621
|
+
if (isCurrent()) {
|
|
1622
|
+
state.presentation.settle(run.status);
|
|
1623
|
+
this.notifyInspection(state);
|
|
1624
|
+
}
|
|
1625
|
+
if (isSameBranch() && run.status !== "stopping") this.settleWaiters(runId, run);
|
|
1626
|
+
}
|
|
1215
1627
|
}
|
|
1216
1628
|
}
|
|
1217
1629
|
|
|
@@ -1268,6 +1680,7 @@ export class WorkflowEngine {
|
|
|
1268
1680
|
const tierIdentity: CallTierIdentity = tier === undefined ? {} : { tier };
|
|
1269
1681
|
state.callTiers.set(nodeId, tierIdentity);
|
|
1270
1682
|
run.callTiers[nodeId] = tierIdentity;
|
|
1683
|
+
state.managedCallFacts.delete(nodeId);
|
|
1271
1684
|
const task: DispatchTask = {
|
|
1272
1685
|
subagent_type: type,
|
|
1273
1686
|
prompt,
|
|
@@ -1283,6 +1696,15 @@ export class WorkflowEngine {
|
|
|
1283
1696
|
// represented by its parent workflow_result replay boundary instead of
|
|
1284
1697
|
// leaking child call facts into the parent's numeric call namespace.
|
|
1285
1698
|
if (!nested) {
|
|
1699
|
+
// Match journal replay's call_attempt semantics: historical results stay
|
|
1700
|
+
// durable, but are not current facts for this accepted live rotation.
|
|
1701
|
+
run.callStatus[nodeId] = "running";
|
|
1702
|
+
run.attempts[nodeId] = generation;
|
|
1703
|
+
run.attemptIds[nodeId] = attemptId;
|
|
1704
|
+
delete run.agentIds[nodeId];
|
|
1705
|
+
delete run.callResults[nodeId];
|
|
1706
|
+
run.compactions[nodeId] = 0;
|
|
1707
|
+
run.updatedAt = Date.now();
|
|
1286
1708
|
this.persist(runId, `call-attempt:${nodeId}`, () =>
|
|
1287
1709
|
this.write({
|
|
1288
1710
|
kind: "call_attempt",
|
|
@@ -1296,10 +1718,12 @@ export class WorkflowEngine {
|
|
|
1296
1718
|
timestamp: Date.now(),
|
|
1297
1719
|
}),
|
|
1298
1720
|
);
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1721
|
+
if (
|
|
1722
|
+
!this.isCurrentState(state, executionGeneration, dispatchEpoch) ||
|
|
1723
|
+
runOptions.signal?.aborted ||
|
|
1724
|
+
state.controller.signal.aborted
|
|
1725
|
+
)
|
|
1726
|
+
throw new WorkflowError("stale workflow attempt", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1303
1727
|
}
|
|
1304
1728
|
|
|
1305
1729
|
// Abort linkage: an external signal (pause/stop/Esc) cancels the wait and
|
|
@@ -1313,214 +1737,386 @@ export class WorkflowEngine {
|
|
|
1313
1737
|
nodeId: spawnNodeId,
|
|
1314
1738
|
attemptId: spawnAttemptId,
|
|
1315
1739
|
};
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1740
|
+
const pending = {
|
|
1741
|
+
spawnKey,
|
|
1742
|
+
owner,
|
|
1743
|
+
executionGeneration,
|
|
1744
|
+
phase: "reserved" as "reserved" | "submitted",
|
|
1745
|
+
startedAgentId: undefined as string | undefined,
|
|
1746
|
+
agentId: undefined as string | undefined,
|
|
1321
1747
|
};
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
let
|
|
1327
|
-
let
|
|
1748
|
+
// Keep this exact allocation lease through every publication callback and
|
|
1749
|
+
// terminal delivery. A waiter alone is insufficient: it is removed before
|
|
1750
|
+
// onUsage/onHistory, which may synchronously pause or replace the branch.
|
|
1751
|
+
state.pendingSpawns.set(spawnKey, pending);
|
|
1752
|
+
let responseId: string | undefined;
|
|
1753
|
+
let terminalAccepted = false;
|
|
1754
|
+
let ownedCleanup: Promise<void> | undefined;
|
|
1755
|
+
const cleanupResponse = () =>
|
|
1756
|
+
(ownedCleanup ??= this.stopAndQuiesce(
|
|
1757
|
+
state,
|
|
1758
|
+
responseId ?? pending.startedAgentId ?? spawnKey,
|
|
1759
|
+
owner,
|
|
1760
|
+
executionGeneration,
|
|
1761
|
+
));
|
|
1328
1762
|
try {
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
if (externalSignal.aborted)
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1763
|
+
state.presentation.dispatch(runOptions, attemptId);
|
|
1764
|
+
this.notifyInspection(state);
|
|
1765
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1766
|
+
throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1767
|
+
const reconcilePending = () => {
|
|
1768
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1769
|
+
const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
|
|
1770
|
+
void reconciliation?.catch(() => {});
|
|
1771
|
+
};
|
|
1772
|
+
const onPendingAbort = () => reconcilePending();
|
|
1773
|
+
externalSignal.addEventListener("abort", onPendingAbort, { once: true });
|
|
1774
|
+
if (externalSignal.aborted) onPendingAbort();
|
|
1775
|
+
|
|
1776
|
+
let pendingId = "";
|
|
1777
|
+
let response: ManagedSpawnResponse | string;
|
|
1778
|
+
try {
|
|
1779
|
+
// Reservation is local until this exact boundary. Observers above can
|
|
1780
|
+
// cancel without creating uncertainty for an RPC that was never sent.
|
|
1781
|
+
pending.phase = "submitted";
|
|
1782
|
+
response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, externalSignal);
|
|
1783
|
+
} catch (error: unknown) {
|
|
1784
|
+
// Transport failure is not evidence that the host never allocated. Keep
|
|
1785
|
+
// the exact spawn owner even without a reply or an aborted signal.
|
|
1786
|
+
await cleanupResponse();
|
|
1787
|
+
throw error;
|
|
1788
|
+
} finally {
|
|
1789
|
+
externalSignal.removeEventListener("abort", onPendingAbort);
|
|
1341
1790
|
}
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
throw new WorkflowError(
|
|
1360
|
-
"workflow dispatch became stale during pause/resume",
|
|
1361
|
-
WorkflowErrorCode.WORKFLOW_ABORTED,
|
|
1362
|
-
{ recoverable: true },
|
|
1363
|
-
);
|
|
1364
|
-
}
|
|
1365
|
-
if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
|
|
1366
|
-
this.persistCallTierIdentity(state, nodeId);
|
|
1367
|
-
}
|
|
1368
|
-
// Report the tier the host actually selected back to the runtime, which
|
|
1369
|
-
// journals it: a call that named none still resolved to something, and the
|
|
1370
|
-
// run's own record of what ran should say what.
|
|
1371
|
-
const resolvedTierIdentity = state.callTiers.get(nodeId);
|
|
1372
|
-
if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
|
|
1373
|
-
if (typeof response === "string") {
|
|
1374
|
-
pendingId = response;
|
|
1375
|
-
} else if (response.terminal) {
|
|
1376
|
-
state.bufferedTerminals.delete(spawnKey);
|
|
1377
|
-
// The spawn response already carries a terminal snapshot (fast completion
|
|
1378
|
-
// or interrupted replay). The runtime journals successful calls itself
|
|
1379
|
-
// (onAgentJournal); a terminal snapshot that is NOT a success is
|
|
1380
|
-
// journaled here so the resume journal reflects the failure.
|
|
1381
|
-
const terminal = response.terminal;
|
|
1382
|
-
if (typeof terminal.tokenCount === "number")
|
|
1383
|
-
runOptions.onUsage?.({
|
|
1384
|
-
input: 0,
|
|
1385
|
-
output: terminal.tokenCount,
|
|
1386
|
-
total: terminal.tokenCount,
|
|
1387
|
-
cost: 0,
|
|
1388
|
-
cacheRead: 0,
|
|
1389
|
-
cacheWrite: 0,
|
|
1390
|
-
});
|
|
1391
|
-
const status = managedTerminalStatus(terminal);
|
|
1392
|
-
if (status !== "completed") {
|
|
1393
|
-
const result = resultFromLifecycle(
|
|
1394
|
-
status,
|
|
1395
|
-
response.id,
|
|
1396
|
-
terminal.result,
|
|
1397
|
-
terminal.error,
|
|
1398
|
-
terminal.compactionCount,
|
|
1399
|
-
terminal.tokenCount,
|
|
1400
|
-
resolvedTierIdentity?.tier ?? tier,
|
|
1791
|
+
responseId = typeof response === "string" ? response : response.id;
|
|
1792
|
+
pending.agentId = responseId;
|
|
1793
|
+
if (
|
|
1794
|
+
state.executionGeneration !== executionGeneration ||
|
|
1795
|
+
this.runs.get(runId) !== state ||
|
|
1796
|
+
this.branchEpoch !== dispatchEpoch ||
|
|
1797
|
+
state.lifecycleSuspended ||
|
|
1798
|
+
this.lifecyclePaused ||
|
|
1799
|
+
state.controller.signal.aborted ||
|
|
1800
|
+
externalSignal.aborted
|
|
1801
|
+
) {
|
|
1802
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1803
|
+
await cleanupResponse();
|
|
1804
|
+
throw new WorkflowError(
|
|
1805
|
+
"workflow dispatch became stale during pause/resume",
|
|
1806
|
+
WorkflowErrorCode.WORKFLOW_ABORTED,
|
|
1807
|
+
{ recoverable: true },
|
|
1401
1808
|
);
|
|
1402
|
-
result.attemptId = attemptId;
|
|
1403
|
-
if (!nested) this.commitCallTerminal(runId, nodeId, result);
|
|
1404
|
-
if (status === "failed" && terminal.error) {
|
|
1405
|
-
throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
|
|
1406
|
-
}
|
|
1407
1809
|
}
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
state.
|
|
1810
|
+
if (typeof response !== "string" && updateCallTierIdentity(state, nodeId, response.tier)) {
|
|
1811
|
+
this.persistCallTierIdentity(state, nodeId);
|
|
1812
|
+
}
|
|
1813
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1814
|
+
throw new WorkflowError("stale workflow identity", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1815
|
+
this.recordManagedAdmission(state, nodeId, responseId, executionGeneration);
|
|
1816
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1817
|
+
throw new WorkflowError("stale workflow admission", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1818
|
+
if (typeof response !== "string" && response.terminal)
|
|
1819
|
+
this.recordManagedTerminal(state, nodeId, responseId, response.terminal, executionGeneration);
|
|
1820
|
+
const bufferedTerminal = state.bufferedTerminals.get(spawnKey);
|
|
1821
|
+
const observedTerminal =
|
|
1822
|
+
bufferedTerminal?.response.id === responseId &&
|
|
1823
|
+
bufferedTerminal.executionGeneration === executionGeneration &&
|
|
1824
|
+
sameWorkflowOwner(bufferedTerminal.owner, owner)
|
|
1825
|
+
? bufferedTerminal.response.terminal
|
|
1826
|
+
: undefined;
|
|
1827
|
+
const observedRunning = pending.startedAgentId === responseId;
|
|
1828
|
+
state.presentation.managed(nodeId, {
|
|
1829
|
+
agentId: responseId,
|
|
1830
|
+
status: observedTerminal
|
|
1831
|
+
? managedTerminalStatus(observedTerminal)
|
|
1832
|
+
: observedRunning
|
|
1833
|
+
? "running"
|
|
1834
|
+
: typeof response === "string"
|
|
1835
|
+
? "pending"
|
|
1836
|
+
: response.state === "queued"
|
|
1837
|
+
? "queued"
|
|
1838
|
+
: "running",
|
|
1839
|
+
tier: state.callTiers.get(nodeId)?.tier,
|
|
1840
|
+
...(typeof response !== "string" && response.terminal
|
|
1841
|
+
? {
|
|
1842
|
+
status: managedTerminalStatus(response.terminal),
|
|
1843
|
+
tokenCount: response.terminal.tokenCount,
|
|
1844
|
+
outputFile: response.terminal.outputFile,
|
|
1845
|
+
compactionCount: Math.max(
|
|
1846
|
+
response.terminal.compactionCount,
|
|
1847
|
+
state.managedCallFacts.get(nodeId)?.compactionCount ?? 0,
|
|
1848
|
+
),
|
|
1849
|
+
}
|
|
1850
|
+
: {}),
|
|
1851
|
+
});
|
|
1852
|
+
this.notifyInspection(state);
|
|
1853
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1854
|
+
throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1855
|
+
// Report the tier the host actually selected back to the runtime, which
|
|
1856
|
+
// journals it: a call that named none still resolved to something, and the
|
|
1857
|
+
// run's own record of what ran should say what.
|
|
1858
|
+
const resolvedTierIdentity = state.callTiers.get(nodeId);
|
|
1859
|
+
if (resolvedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(resolvedTierIdentity.tier);
|
|
1860
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1861
|
+
throw new WorkflowError("stale workflow dispatch", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1862
|
+
if (typeof response === "string") {
|
|
1863
|
+
pendingId = response;
|
|
1864
|
+
} else if (response.terminal) {
|
|
1418
1865
|
state.bufferedTerminals.delete(spawnKey);
|
|
1419
|
-
//
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1866
|
+
// The spawn response already carries a terminal snapshot (fast completion
|
|
1867
|
+
// or interrupted replay). The runtime journals successful calls itself
|
|
1868
|
+
// (onAgentJournal); a terminal snapshot that is NOT a success is
|
|
1869
|
+
// journaled here so the resume journal reflects the failure.
|
|
1870
|
+
const terminal = response.terminal;
|
|
1871
|
+
if (typeof terminal.tokenCount === "number")
|
|
1872
|
+
runOptions.onUsage?.({
|
|
1873
|
+
input: 0,
|
|
1874
|
+
output: terminal.tokenCount,
|
|
1875
|
+
total: terminal.tokenCount,
|
|
1876
|
+
cost: 0,
|
|
1877
|
+
cacheRead: 0,
|
|
1878
|
+
cacheWrite: 0,
|
|
1879
|
+
});
|
|
1880
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1881
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1882
|
+
terminalAccepted = true;
|
|
1883
|
+
const status = managedTerminalStatus(terminal);
|
|
1884
|
+
if (status !== "completed") {
|
|
1885
|
+
const result = resultFromLifecycle(
|
|
1886
|
+
status,
|
|
1887
|
+
response.id,
|
|
1888
|
+
terminal.result,
|
|
1889
|
+
terminal.error,
|
|
1890
|
+
terminal.compactionCount,
|
|
1891
|
+
terminal.tokenCount,
|
|
1892
|
+
resolvedTierIdentity?.tier ?? tier,
|
|
1423
1893
|
);
|
|
1424
|
-
|
|
1894
|
+
result.attemptId = attemptId;
|
|
1895
|
+
if (!nested) this.commitCallTerminal(runId, nodeId, result);
|
|
1896
|
+
if (status === "failed" && terminal.error) {
|
|
1897
|
+
throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
1901
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
1902
|
+
return terminal.result ?? "";
|
|
1903
|
+
} else {
|
|
1904
|
+
pendingId = response.id;
|
|
1425
1905
|
}
|
|
1426
|
-
};
|
|
1427
1906
|
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1907
|
+
const waitAbort = () => {
|
|
1908
|
+
const waiter = state.agentWaiters.get(pendingId);
|
|
1909
|
+
if (waiter && !waiter.settled) {
|
|
1910
|
+
waiter.settled = true;
|
|
1911
|
+
state.agentWaiters.delete(pendingId);
|
|
1912
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1913
|
+
// Stop acknowledges cancellation, not provider/tool finally completion.
|
|
1914
|
+
void cleanupResponse().then(async () => {
|
|
1915
|
+
if (
|
|
1916
|
+
!nested &&
|
|
1917
|
+
this.runs.get(runId) === state &&
|
|
1918
|
+
state.executionGeneration === executionGeneration &&
|
|
1919
|
+
!run.nonResumable
|
|
1920
|
+
) {
|
|
1921
|
+
const result = resultFromLifecycle(
|
|
1922
|
+
"stopped",
|
|
1923
|
+
pendingId,
|
|
1924
|
+
undefined,
|
|
1925
|
+
"workflow call cancelled",
|
|
1926
|
+
run.compactions[nodeId] ?? 0,
|
|
1927
|
+
undefined,
|
|
1928
|
+
resolvedTierIdentity?.tier ?? tier,
|
|
1929
|
+
);
|
|
1930
|
+
result.attemptId = attemptId;
|
|
1931
|
+
try {
|
|
1932
|
+
await this.drainJournal(runId);
|
|
1933
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch)) {
|
|
1934
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
this.commitCallTerminal(runId, nodeId, result, true);
|
|
1938
|
+
} catch (error) {
|
|
1939
|
+
this.fenceExecution(
|
|
1940
|
+
state,
|
|
1941
|
+
executionGeneration,
|
|
1942
|
+
"Could not persist cancelled call terminal fact; run is non-resumable",
|
|
1943
|
+
);
|
|
1944
|
+
waiter.reject(error);
|
|
1945
|
+
return;
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
waiter.reject(
|
|
1949
|
+
new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }),
|
|
1950
|
+
);
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1442
1953
|
};
|
|
1443
|
-
state.agentWaiters.set(pendingId, waiter);
|
|
1444
|
-
// A spawn that was admitted straight away is already running, and its
|
|
1445
|
-
// `subagents:started` event fired before this waiter existed. Only a
|
|
1446
|
-
// reply that says "queued" still has a start to wait for.
|
|
1447
|
-
if (typeof response !== "string" && response.state !== "queued") notifyAgentRunning(waiter);
|
|
1448
|
-
externalSignal.addEventListener("abort", waitAbort, { once: true });
|
|
1449
|
-
if (externalSignal.aborted) {
|
|
1450
|
-
waitAbort();
|
|
1451
|
-
return;
|
|
1452
|
-
}
|
|
1453
1954
|
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1955
|
+
// Wait for the terminal lifecycle event, consuming a matching terminal that
|
|
1956
|
+
// arrived while the managed-spawn reply was still pending.
|
|
1957
|
+
const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
|
|
1958
|
+
const waiter: AgentLifecycleWaiter = {
|
|
1959
|
+
callIndex: nodeId,
|
|
1960
|
+
generation,
|
|
1961
|
+
executionGeneration,
|
|
1962
|
+
onUsage: runOptions.onUsage,
|
|
1963
|
+
onHistory: runOptions.onHistory,
|
|
1964
|
+
onRunning: runOptions.onRunning,
|
|
1965
|
+
startedNotified: false,
|
|
1966
|
+
settled: false,
|
|
1967
|
+
resolve,
|
|
1968
|
+
reject,
|
|
1969
|
+
};
|
|
1970
|
+
state.agentWaiters.set(pendingId, waiter);
|
|
1971
|
+
// A spawn that was admitted straight away is already running, and its
|
|
1972
|
+
// `subagents:started` event fired before this waiter existed. Only a
|
|
1973
|
+
// reply that says "queued" still has a start to wait for.
|
|
1974
|
+
if (observedRunning || (typeof response !== "string" && response.state !== "queued"))
|
|
1975
|
+
notifyAgentRunning(waiter);
|
|
1976
|
+
externalSignal.addEventListener("abort", waitAbort, { once: true });
|
|
1977
|
+
if (externalSignal.aborted) {
|
|
1978
|
+
waitAbort();
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
const buffered = state.bufferedTerminals.get(spawnKey);
|
|
1983
|
+
if (!buffered) return;
|
|
1984
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1985
|
+
if (
|
|
1986
|
+
buffered.executionGeneration !== executionGeneration ||
|
|
1987
|
+
state.executionGeneration !== executionGeneration ||
|
|
1988
|
+
state.lifecycleSuspended ||
|
|
1989
|
+
state.controller.signal.aborted ||
|
|
1990
|
+
buffered.response.id !== pendingId ||
|
|
1991
|
+
!sameWorkflowOwner(buffered.owner, owner)
|
|
1992
|
+
)
|
|
1993
|
+
return;
|
|
1994
|
+
this.resolveAgentWaiter(state, pendingId, waiter, buffered);
|
|
1995
|
+
}).finally(() => {
|
|
1996
|
+
externalSignal.removeEventListener("abort", waitAbort);
|
|
1997
|
+
});
|
|
1998
|
+
|
|
1457
1999
|
if (
|
|
1458
|
-
|
|
1459
|
-
|
|
2000
|
+
this.runs.get(runId) !== state ||
|
|
2001
|
+
this.branchEpoch !== dispatchEpoch ||
|
|
1460
2002
|
state.lifecycleSuspended ||
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
state.
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
throw new WorkflowError("subagent settled without a terminal snapshot", WorkflowErrorCode.AGENT_EXECUTION_ERROR, {
|
|
1482
|
-
recoverable: true,
|
|
2003
|
+
this.lifecyclePaused
|
|
2004
|
+
) {
|
|
2005
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
|
|
2006
|
+
}
|
|
2007
|
+
const snapshot = terminal.terminal;
|
|
2008
|
+
if (!snapshot)
|
|
2009
|
+
throw new WorkflowError(
|
|
2010
|
+
"subagent settled without a terminal snapshot",
|
|
2011
|
+
WorkflowErrorCode.AGENT_EXECUTION_ERROR,
|
|
2012
|
+
{
|
|
2013
|
+
recoverable: true,
|
|
2014
|
+
},
|
|
2015
|
+
);
|
|
2016
|
+
this.recordManagedTerminal(state, nodeId, terminal.id, snapshot, executionGeneration);
|
|
2017
|
+
state.presentation.managed(nodeId, {
|
|
2018
|
+
agentId: terminal.id,
|
|
2019
|
+
status: managedTerminalStatus(snapshot),
|
|
2020
|
+
tokenCount: snapshot.tokenCount,
|
|
2021
|
+
outputFile: snapshot.outputFile,
|
|
2022
|
+
compactionCount: Math.max(snapshot.compactionCount, state.managedCallFacts.get(nodeId)?.compactionCount ?? 0),
|
|
1483
2023
|
});
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
const
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
2024
|
+
this.notifyInspection(state);
|
|
2025
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch))
|
|
2026
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
2027
|
+
if (externalSignal.aborted) throw new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
2028
|
+
terminalAccepted = true;
|
|
2029
|
+
const status = managedTerminalStatus(snapshot);
|
|
2030
|
+
// Successful calls are journaled by the runtime's onAgentJournal (the
|
|
2031
|
+
// single writer for completed results); only failures settle here so the
|
|
2032
|
+
// resume journal reflects them.
|
|
2033
|
+
if (status !== "completed") {
|
|
2034
|
+
const result = resultFromLifecycle(
|
|
2035
|
+
status,
|
|
2036
|
+
terminal.id,
|
|
2037
|
+
snapshot.result,
|
|
2038
|
+
snapshot.error,
|
|
2039
|
+
snapshot.compactionCount,
|
|
2040
|
+
snapshot.tokenCount,
|
|
2041
|
+
state.callTiers.get(nodeId)?.tier ?? tier,
|
|
2042
|
+
);
|
|
2043
|
+
result.attemptId = attemptId;
|
|
2044
|
+
if (!nested) this.commitCallTerminal(runId, nodeId, result);
|
|
2045
|
+
if (status === "failed" && snapshot.error) {
|
|
2046
|
+
throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
|
|
2047
|
+
}
|
|
1502
2048
|
}
|
|
2049
|
+
const completedTierIdentity = state.callTiers.get(nodeId);
|
|
2050
|
+
if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
|
|
2051
|
+
if (!this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
2052
|
+
throw new WorkflowError("stale workflow terminal", WorkflowErrorCode.WORKFLOW_ABORTED);
|
|
2053
|
+
return snapshot.result ?? "";
|
|
2054
|
+
} finally {
|
|
2055
|
+
// Publication does not donate ownership to an observer or a replacement
|
|
2056
|
+
// state. Even a terminal reply must STOP + QUIESCE its captured owner
|
|
2057
|
+
// when observation retired this dispatch.
|
|
2058
|
+
if (
|
|
2059
|
+
pending.phase === "submitted" &&
|
|
2060
|
+
(!terminalAccepted || !this.isCurrentState(state, executionGeneration, dispatchEpoch) || externalSignal.aborted)
|
|
2061
|
+
)
|
|
2062
|
+
await cleanupResponse();
|
|
2063
|
+
state.pendingSpawns.delete(spawnKey);
|
|
2064
|
+
state.bufferedTerminals.delete(spawnKey);
|
|
1503
2065
|
}
|
|
1504
|
-
const completedTierIdentity = state.callTiers.get(nodeId);
|
|
1505
|
-
if (completedTierIdentity?.tier !== undefined) runOptions.onTierResolved?.(completedTierIdentity.tier);
|
|
1506
|
-
return snapshot.result ?? "";
|
|
1507
2066
|
}
|
|
1508
2067
|
|
|
1509
2068
|
private journalCallResult(runId: string, entry: JournalEntry): void {
|
|
2069
|
+
entry = structuredClone(entry);
|
|
1510
2070
|
const state = this.runs.get(runId);
|
|
1511
2071
|
if (!state) return;
|
|
1512
2072
|
const run = state.run;
|
|
2073
|
+
const executionGeneration = state.executionGeneration;
|
|
2074
|
+
const epoch = this.branchEpoch;
|
|
1513
2075
|
const nodeId = String(entry.index);
|
|
1514
2076
|
const result = entry.result;
|
|
1515
2077
|
const callTierIdentity: CallTierIdentity =
|
|
1516
2078
|
entry.tier === undefined ? (run.callTiers[nodeId] ?? {}) : { tier: entry.tier };
|
|
2079
|
+
const managed = state.managedCallFacts.get(nodeId);
|
|
2080
|
+
const currentManaged =
|
|
2081
|
+
managed?.executionGeneration === state.executionGeneration && managed.attemptId === run.attemptIds[nodeId]
|
|
2082
|
+
? managed
|
|
2083
|
+
: undefined;
|
|
1517
2084
|
const callResult: CallResult = {
|
|
2085
|
+
...(currentManaged
|
|
2086
|
+
? {
|
|
2087
|
+
agentId: currentManaged.agentId,
|
|
2088
|
+
...(currentManaged.tokenCount === undefined ? {} : { tokenCount: currentManaged.tokenCount }),
|
|
2089
|
+
...(currentManaged.outputFile === undefined ? {} : { outputFile: currentManaged.outputFile }),
|
|
2090
|
+
}
|
|
2091
|
+
: {}),
|
|
1518
2092
|
status: "completed",
|
|
1519
2093
|
...(typeof result === "string" && result.length === 0 ? {} : { result }),
|
|
1520
2094
|
...callTierIdentity,
|
|
1521
|
-
compactionCount: 0,
|
|
2095
|
+
compactionCount: currentManaged?.compactionCount ?? 0,
|
|
1522
2096
|
updatedAt: Date.now(),
|
|
1523
2097
|
};
|
|
2098
|
+
// Checkpoints are local calls, not managed children. Give their completed
|
|
2099
|
+
// result a journal identity without fabricating a manager record.
|
|
2100
|
+
if (!run.attemptIds[nodeId]) {
|
|
2101
|
+
const generation = (state.generations.get(nodeId) ?? 0) + 1;
|
|
2102
|
+
const attemptId = `${runId}/${nodeId}/attempt-${generation}`;
|
|
2103
|
+
this.appendRequired({
|
|
2104
|
+
kind: "call_attempt",
|
|
2105
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2106
|
+
runId,
|
|
2107
|
+
nodeId,
|
|
2108
|
+
attemptId,
|
|
2109
|
+
generation,
|
|
2110
|
+
owner: { extension: "pi-workflows", runId, nodeId, attemptId },
|
|
2111
|
+
timestamp: Date.now(),
|
|
2112
|
+
});
|
|
2113
|
+
if (!this.isCurrentState(state, executionGeneration, epoch)) return;
|
|
2114
|
+
state.generations.set(nodeId, generation);
|
|
2115
|
+
run.attemptTracking = true;
|
|
2116
|
+
run.attempts[nodeId] = generation;
|
|
2117
|
+
run.attemptIds[nodeId] = attemptId;
|
|
2118
|
+
run.callStatus[nodeId] = "running";
|
|
2119
|
+
}
|
|
1524
2120
|
const attemptId = run.attemptIds[nodeId];
|
|
1525
2121
|
if (attemptId) callResult.attemptId = attemptId;
|
|
1526
2122
|
const event: JournalEvent = {
|
|
@@ -1537,16 +2133,23 @@ export class WorkflowEngine {
|
|
|
1537
2133
|
...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
|
|
1538
2134
|
timestamp: Date.now(),
|
|
1539
2135
|
};
|
|
1540
|
-
|
|
2136
|
+
const acceptedEvent = structuredClone(event);
|
|
2137
|
+
this.persist(runId, `call-result:${nodeId}`, () => this.write(acceptedEvent));
|
|
2138
|
+
if (!this.isCurrentState(state, executionGeneration, epoch)) return;
|
|
1541
2139
|
run.callResults[nodeId] = callResult;
|
|
1542
2140
|
run.callTiers[nodeId] = callResult.tier === undefined ? {} : { tier: callResult.tier };
|
|
1543
2141
|
run.callStatus[nodeId] = "completed";
|
|
2142
|
+
run.compactions[nodeId] = callResult.compactionCount;
|
|
2143
|
+
state.managedCallFacts.delete(nodeId);
|
|
1544
2144
|
run.updatedAt = Date.now();
|
|
1545
2145
|
}
|
|
1546
2146
|
|
|
1547
2147
|
private journalWorkflowResult(runId: string, entry: JournalEntry): void {
|
|
2148
|
+
entry = structuredClone(entry);
|
|
1548
2149
|
const state = this.runs.get(runId);
|
|
1549
2150
|
if (!state) return;
|
|
2151
|
+
const generation = state.executionGeneration;
|
|
2152
|
+
const epoch = this.branchEpoch;
|
|
1550
2153
|
const nodeId = String(entry.index);
|
|
1551
2154
|
const result: CallResult = {
|
|
1552
2155
|
status: "completed",
|
|
@@ -1573,6 +2176,7 @@ export class WorkflowEngine {
|
|
|
1573
2176
|
// otherwise a transient append retry would arrive after the terminal
|
|
1574
2177
|
// transition and be discarded as stale during restore.
|
|
1575
2178
|
this.appendRequired(event);
|
|
2179
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1576
2180
|
state.run.workflowResultGenerations[nodeId] = event.generation;
|
|
1577
2181
|
state.run.callResults[nodeId] = result;
|
|
1578
2182
|
state.run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
|
|
@@ -1580,10 +2184,12 @@ export class WorkflowEngine {
|
|
|
1580
2184
|
state.run.updatedAt = Date.now();
|
|
1581
2185
|
}
|
|
1582
2186
|
|
|
1583
|
-
private commitCallTerminal(runId: string, nodeId: string, result: CallResult): void {
|
|
2187
|
+
private commitCallTerminal(runId: string, nodeId: string, result: CallResult, required = false): void {
|
|
1584
2188
|
const state = this.runs.get(runId);
|
|
1585
2189
|
if (!state) return;
|
|
1586
2190
|
const run = state.run;
|
|
2191
|
+
const generation = state.executionGeneration;
|
|
2192
|
+
const epoch = this.branchEpoch;
|
|
1587
2193
|
const attemptId = result.attemptId ?? run.attemptIds[nodeId];
|
|
1588
2194
|
const event: JournalEvent = {
|
|
1589
2195
|
kind: "call_result",
|
|
@@ -1595,7 +2201,9 @@ export class WorkflowEngine {
|
|
|
1595
2201
|
...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
|
|
1596
2202
|
timestamp: Date.now(),
|
|
1597
2203
|
};
|
|
1598
|
-
|
|
2204
|
+
if (required) this.appendRequired(event);
|
|
2205
|
+
else this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
|
|
2206
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1599
2207
|
run.callResults[nodeId] = result;
|
|
1600
2208
|
run.callTiers[nodeId] = result.tier === undefined ? {} : { tier: result.tier };
|
|
1601
2209
|
run.callStatus[nodeId] = result.status;
|
|
@@ -1608,36 +2216,26 @@ export class WorkflowEngine {
|
|
|
1608
2216
|
|
|
1609
2217
|
private reconcilePendingSpawns(state: ScriptRunState): void {
|
|
1610
2218
|
for (const pending of state.pendingSpawns.values()) {
|
|
2219
|
+
if (pending.phase !== "submitted" || pending.agentId) continue;
|
|
1611
2220
|
const reconciliation = this.client.reconcileManaged?.(pending.spawnKey, pending.owner);
|
|
1612
2221
|
void reconciliation?.catch(() => {});
|
|
1613
2222
|
}
|
|
1614
|
-
|
|
2223
|
+
// Reconciliation is not positive exact-owner quiescence. The allocation
|
|
2224
|
+
// lease remains with dispatch until terminal admission or owned cleanup.
|
|
1615
2225
|
state.bufferedTerminals.clear();
|
|
1616
2226
|
}
|
|
1617
2227
|
|
|
1618
|
-
private async quiescePendingSpawns(state: ScriptRunState): Promise<WorkflowQuiesceResult> {
|
|
1619
|
-
const pending = [...state.pendingSpawns.values()];
|
|
1620
|
-
if (pending.length === 0) return { settled: true, pending: [] };
|
|
1621
|
-
if (!this.client.reconcileManaged) {
|
|
1622
|
-
return { settled: false, pending: pending.map(({ spawnKey }) => spawnKey).slice(0, 256) };
|
|
1623
|
-
}
|
|
1624
|
-
const failed: string[] = [];
|
|
1625
|
-
await Promise.all(
|
|
1626
|
-
pending.map(async ({ spawnKey, owner }) => {
|
|
1627
|
-
try {
|
|
1628
|
-
const result = await this.client.reconcileManaged?.(spawnKey, owner);
|
|
1629
|
-
if (result === undefined) failed.push(spawnKey);
|
|
1630
|
-
} catch {
|
|
1631
|
-
failed.push(spawnKey);
|
|
1632
|
-
}
|
|
1633
|
-
}),
|
|
1634
|
-
);
|
|
1635
|
-
return { settled: failed.length === 0, pending: failed.slice(0, 256) };
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
2228
|
private cleanupAgents(runId: string): void {
|
|
1639
2229
|
const state = this.runs.get(runId);
|
|
1640
2230
|
if (!state) return;
|
|
2231
|
+
// Retain captured allocations before reconciliation, abort, or branch replay
|
|
2232
|
+
// can notify observers. Dispatch's finally may not run until a later turn;
|
|
2233
|
+
// its cleanup and any waiter cleanup share this stable exact-owner lease.
|
|
2234
|
+
for (const pending of state.pendingSpawns.values()) {
|
|
2235
|
+
if (pending.phase !== "submitted") continue;
|
|
2236
|
+
const agentId = pending.agentId ?? pending.startedAgentId ?? pending.spawnKey;
|
|
2237
|
+
void this.stopAndQuiesce(state, agentId, pending.owner, pending.executionGeneration);
|
|
2238
|
+
}
|
|
1641
2239
|
this.reconcilePendingSpawns(state);
|
|
1642
2240
|
for (const [agentId, waiter] of [...state.agentWaiters]) {
|
|
1643
2241
|
if (waiter.settled) continue;
|
|
@@ -1648,14 +2246,17 @@ export class WorkflowEngine {
|
|
|
1648
2246
|
recoverable: true,
|
|
1649
2247
|
}),
|
|
1650
2248
|
);
|
|
1651
|
-
void this.
|
|
1652
|
-
|
|
2249
|
+
void this.stopAndQuiesce(
|
|
2250
|
+
state,
|
|
2251
|
+
agentId,
|
|
2252
|
+
{
|
|
1653
2253
|
extension: "pi-workflows",
|
|
1654
2254
|
runId,
|
|
1655
2255
|
nodeId: `call-${waiter.callIndex}`,
|
|
1656
2256
|
attemptId: `attempt-${waiter.generation}`,
|
|
1657
|
-
}
|
|
1658
|
-
.
|
|
2257
|
+
},
|
|
2258
|
+
state.executionGeneration,
|
|
2259
|
+
);
|
|
1659
2260
|
}
|
|
1660
2261
|
}
|
|
1661
2262
|
|
|
@@ -1665,51 +2266,108 @@ export class WorkflowEngine {
|
|
|
1665
2266
|
owner: WorkflowOwner,
|
|
1666
2267
|
executionGeneration: number,
|
|
1667
2268
|
unconfirmed?: string,
|
|
2269
|
+
quiescence?: Promise<{ settled: boolean; pending: string[] }>,
|
|
1668
2270
|
): Promise<void> {
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
2271
|
+
const obligations = this.cleanupObligations.get(state) ?? new Map<string, OwnedCleanup>();
|
|
2272
|
+
this.cleanupObligations.set(state, obligations);
|
|
2273
|
+
const allocationKey = `${owner.runId}/${owner.nodeId}/${owner.attemptId}`;
|
|
2274
|
+
const existing = [...obligations.entries()].find(([, cleanup]) => sameWorkflowOwner(cleanup.owner, owner));
|
|
2275
|
+
const key = existing?.[0] ?? JSON.stringify([owner, agentId]);
|
|
2276
|
+
let obligation = existing?.[1];
|
|
2277
|
+
if (!obligation) {
|
|
2278
|
+
let resolve!: () => void;
|
|
2279
|
+
const confirmed = new Promise<void>((done) => {
|
|
2280
|
+
resolve = done;
|
|
2281
|
+
});
|
|
2282
|
+
obligation = {
|
|
2283
|
+
agentId,
|
|
2284
|
+
...(agentId === allocationKey ? { spawnKey: allocationKey } : {}),
|
|
2285
|
+
owner: { ...owner },
|
|
2286
|
+
epoch: this.branchEpoch,
|
|
2287
|
+
confirmed,
|
|
2288
|
+
resolve,
|
|
2289
|
+
};
|
|
2290
|
+
obligations.set(key, obligation);
|
|
2291
|
+
this.retainedCleanupStates.add(state);
|
|
2292
|
+
state.pendingCleanup.add(confirmed);
|
|
2293
|
+
} else if (obligation.spawnKey && agentId !== allocationKey) {
|
|
2294
|
+
// A late reply refines the same allocation lease, not a new obligation.
|
|
2295
|
+
obligation.agentId = agentId;
|
|
2296
|
+
obligation.spawnKey = undefined;
|
|
2297
|
+
}
|
|
2298
|
+
if (obligation.operation) return obligation.operation;
|
|
2299
|
+
const accepted = obligation;
|
|
2300
|
+
const currentOwner = () => this.isCurrentState(state, state.executionGeneration, accepted.epoch);
|
|
2301
|
+
const operation = Promise.resolve().then(async () => {
|
|
2302
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
2303
|
+
let stopError: unknown;
|
|
2304
|
+
try {
|
|
2305
|
+
if (unconfirmed) throw new Error(unconfirmed);
|
|
2306
|
+
const cleanup = (async () => {
|
|
2307
|
+
// An allocation without a confirmed id can only be retried by its exact spawn owner.
|
|
2308
|
+
if (accepted.spawnKey) {
|
|
2309
|
+
if (!this.client.reconcileManaged) throw new Error("owned allocation reconciliation unavailable");
|
|
2310
|
+
const reply = await this.client.reconcileManaged(accepted.spawnKey, accepted.owner);
|
|
2311
|
+
const id = typeof reply === "string" ? reply : reply?.id;
|
|
2312
|
+
if (!id) throw new Error("cancelled spawn allocation could not be confirmed");
|
|
2313
|
+
accepted.agentId = id;
|
|
2314
|
+
accepted.spawnKey = undefined;
|
|
2315
|
+
}
|
|
1675
2316
|
try {
|
|
1676
2317
|
if (!this.client.stopOwned) throw new Error("owned stop RPC unavailable");
|
|
1677
|
-
await this.client.stopOwned(agentId, owner);
|
|
2318
|
+
await this.client.stopOwned(accepted.agentId, accepted.owner);
|
|
1678
2319
|
} catch (error: unknown) {
|
|
1679
|
-
// Reconciliation may already have stopped the record. Only exact-owner
|
|
1680
|
-
// quiescence confirms that provider/tool cleanup has actually settled.
|
|
1681
2320
|
stopError = error;
|
|
1682
2321
|
}
|
|
1683
2322
|
if (!this.client.quiesceOwned) throw new Error("owned cleanup RPC unavailable");
|
|
1684
|
-
const result = await
|
|
2323
|
+
const result = await (quiescence ??
|
|
2324
|
+
this.client.quiesceOwned(accepted.owner.runId, [accepted.agentId], BRANCH_QUIESCE_TIMEOUT_MS, [
|
|
2325
|
+
accepted.owner,
|
|
2326
|
+
]));
|
|
1685
2327
|
if (!result?.settled || result.pending.length !== 0) throw new Error("owned agents are not quiescent");
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
2328
|
+
// A bounded negative/error is not evidence that cleanup settled. Only
|
|
2329
|
+
// this positive owner check releases the stable obligation, even late.
|
|
2330
|
+
obligations.delete(key);
|
|
2331
|
+
if (obligations.size === 0) this.retainedCleanupStates.delete(state);
|
|
2332
|
+
state.pendingCleanup.delete(accepted.confirmed);
|
|
2333
|
+
accepted.resolve();
|
|
2334
|
+
if (currentOwner() && isTerminalWorkflow(state.run.status)) this.notifyTerminal(state);
|
|
2335
|
+
})();
|
|
2336
|
+
await Promise.race([
|
|
2337
|
+
cleanup,
|
|
2338
|
+
new Promise<never>((_resolve, reject) => {
|
|
2339
|
+
timer = setTimeout(() => reject(new Error("owned cleanup timed out")), BRANCH_QUIESCE_TIMEOUT_MS + 1_000);
|
|
2340
|
+
}),
|
|
2341
|
+
]);
|
|
2342
|
+
} catch (error: unknown) {
|
|
2343
|
+
if (!currentOwner() || state.executionGeneration !== executionGeneration) return;
|
|
2344
|
+
const stopDiagnostic = stopError === undefined ? "" : `; stop failed: ${String(stopError)}`;
|
|
2345
|
+
const diagnostic = `Owned cleanup unconfirmed for ${agentId}: ${error instanceof Error ? error.message : String(error)}${stopDiagnostic}; run is non-resumable`;
|
|
2346
|
+
this.fenceExecution(state, executionGeneration, diagnostic);
|
|
2347
|
+
} finally {
|
|
2348
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
2349
|
+
}
|
|
2350
|
+
});
|
|
2351
|
+
accepted.operation = operation;
|
|
2352
|
+
try {
|
|
2353
|
+
await operation;
|
|
1698
2354
|
} finally {
|
|
1699
|
-
if (
|
|
2355
|
+
if (accepted.operation === operation) accepted.operation = undefined;
|
|
1700
2356
|
}
|
|
1701
2357
|
}
|
|
1702
2358
|
|
|
1703
2359
|
private async drainExecution(state: ScriptRunState, execution?: Promise<unknown>): Promise<void> {
|
|
1704
2360
|
if (!execution) return;
|
|
1705
2361
|
const generation = state.executionGeneration;
|
|
2362
|
+
const epoch = this.branchEpoch;
|
|
1706
2363
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1707
2364
|
try {
|
|
1708
2365
|
await Promise.race([
|
|
1709
2366
|
execution.catch(() => {}),
|
|
1710
2367
|
new Promise<void>((resolve) => {
|
|
1711
2368
|
timer = setTimeout(() => {
|
|
1712
|
-
this.
|
|
2369
|
+
if (this.isCurrentState(state, generation, epoch))
|
|
2370
|
+
this.fenceExecution(state, generation, "Workflow execution cleanup timed out; run is non-resumable");
|
|
1713
2371
|
resolve();
|
|
1714
2372
|
}, EXECUTION_DRAIN_TIMEOUT_MS);
|
|
1715
2373
|
}),
|
|
@@ -1721,7 +2379,8 @@ export class WorkflowEngine {
|
|
|
1721
2379
|
|
|
1722
2380
|
private fenceExecution(state: ScriptRunState, generation: number, diagnostic: string): void {
|
|
1723
2381
|
const run = state.run;
|
|
1724
|
-
|
|
2382
|
+
const epoch = this.branchEpoch;
|
|
2383
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1725
2384
|
if (run.nonResumable && run.status === "stopped") return;
|
|
1726
2385
|
run.nonResumable = true;
|
|
1727
2386
|
const blockedNodeIds = Object.keys(run.callStatus).filter((nodeId) => run.callStatus[nodeId] !== "completed");
|
|
@@ -1744,21 +2403,119 @@ export class WorkflowEngine {
|
|
|
1744
2403
|
timestamp: Date.now(),
|
|
1745
2404
|
};
|
|
1746
2405
|
this.persist(run.runId, "cleanup-unconfirmed", () => this.write(event));
|
|
2406
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1747
2407
|
// Fence before aborting: synchronous abort handlers may re-enter cleanup.
|
|
1748
2408
|
// Exact-owner cleanup continues, but no later result may revive this run.
|
|
1749
2409
|
state.executionGeneration += 1;
|
|
2410
|
+
state.presentation.reset(false);
|
|
2411
|
+
this.notifyInspection(state);
|
|
2412
|
+
if (!this.isCurrentState(state, generation + 1, epoch)) return;
|
|
1750
2413
|
state.controller.abort();
|
|
2414
|
+
if (!this.isCurrentState(state, generation + 1, epoch)) return;
|
|
1751
2415
|
this.cleanupAgents(run.runId);
|
|
1752
|
-
|
|
2416
|
+
// Uncertainty is an error, never a terminal result delivery. Foreground
|
|
2417
|
+
// callers can unwind, while owner UI retains an explicit cleanup recheck.
|
|
2418
|
+
if (state.pendingCleanup.size > 0) {
|
|
2419
|
+
const all = this.waiters.get(run.runId);
|
|
2420
|
+
this.waiters.delete(run.runId);
|
|
2421
|
+
for (const waiter of all ?? []) {
|
|
2422
|
+
if (waiter.settled) continue;
|
|
2423
|
+
waiter.settled = true;
|
|
2424
|
+
waiter.cleanup();
|
|
2425
|
+
waiter.reject(new WorkflowWaitAbortedError());
|
|
2426
|
+
}
|
|
2427
|
+
} else this.notifyTerminal(state);
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
private async remove(state: ScriptRunState): Promise<void> {
|
|
2431
|
+
if (state.removeOperation) return state.removeOperation;
|
|
2432
|
+
const isBusy = () =>
|
|
2433
|
+
state.activeExecutionGenerations.size > 0 ||
|
|
2434
|
+
state.pendingCleanup.size > 0 ||
|
|
2435
|
+
state.pendingSpawns.size > 0 ||
|
|
2436
|
+
state.agentWaiters.size > 0 ||
|
|
2437
|
+
state.stopOperation !== undefined;
|
|
2438
|
+
if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
|
|
2439
|
+
const epoch = this.branchEpoch;
|
|
2440
|
+
const operation = Promise.resolve().then(async () => {
|
|
2441
|
+
await this.drainJournal(state.run.runId);
|
|
2442
|
+
if (
|
|
2443
|
+
this.runs.get(state.run.runId) !== state ||
|
|
2444
|
+
this.branchEpoch !== epoch ||
|
|
2445
|
+
this.disposed ||
|
|
2446
|
+
this.lifecyclePaused ||
|
|
2447
|
+
state.lifecycleSuspended
|
|
2448
|
+
)
|
|
2449
|
+
throw new Error("workflow branch changed before removal");
|
|
2450
|
+
if (isBusy()) throw new Error("cannot remove a workflow while owned cleanup is pending");
|
|
2451
|
+
// No queued best-effort tombstone: memory survives failure so removal can be retried.
|
|
2452
|
+
this.appendRequired({
|
|
2453
|
+
kind: "run_removed",
|
|
2454
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2455
|
+
runId: state.run.runId,
|
|
2456
|
+
timestamp: Date.now(),
|
|
2457
|
+
});
|
|
2458
|
+
this.runs.delete(state.run.runId);
|
|
2459
|
+
this.waiters.delete(state.run.runId);
|
|
2460
|
+
this.notifyInspection(state);
|
|
2461
|
+
});
|
|
2462
|
+
state.removeOperation = operation;
|
|
2463
|
+
try {
|
|
2464
|
+
await operation;
|
|
2465
|
+
} finally {
|
|
2466
|
+
if (state.removeOperation === operation) state.removeOperation = undefined;
|
|
2467
|
+
}
|
|
1753
2468
|
}
|
|
1754
2469
|
|
|
1755
2470
|
private async stop(state: ScriptRunState): Promise<void> {
|
|
2471
|
+
if (state.stopOperation) return state.stopOperation;
|
|
2472
|
+
if (isTerminalWorkflow(state.run.status) && state.pendingCleanup.size === 0) return;
|
|
2473
|
+
const generation = state.executionGeneration;
|
|
2474
|
+
const epoch = this.branchEpoch;
|
|
2475
|
+
const operation = Promise.resolve().then(async () => {
|
|
2476
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
2477
|
+
const pending = [...(this.cleanupObligations.get(state)?.values() ?? [])];
|
|
2478
|
+
if (pending.length > 0) {
|
|
2479
|
+
await Promise.all(
|
|
2480
|
+
pending.map((cleanup) => this.stopAndQuiesce(state, cleanup.agentId, cleanup.owner, generation)),
|
|
2481
|
+
);
|
|
2482
|
+
if (!this.isCurrentState(state, generation, epoch) || state.pendingCleanup.size > 0) return;
|
|
2483
|
+
if (isTerminalWorkflow(state.run.status)) {
|
|
2484
|
+
this.notifyTerminal(state);
|
|
2485
|
+
return;
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
await this.stopExecution(state, generation, epoch);
|
|
2489
|
+
});
|
|
2490
|
+
state.stopOperation = operation;
|
|
2491
|
+
try {
|
|
2492
|
+
await operation;
|
|
2493
|
+
} finally {
|
|
2494
|
+
if (state.stopOperation === operation) state.stopOperation = undefined;
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
private async stopExecution(state: ScriptRunState, generation: number, epoch: number): Promise<void> {
|
|
1756
2499
|
const run = state.run;
|
|
1757
|
-
|
|
2500
|
+
const current = () => this.isCurrentState(state, generation, epoch);
|
|
2501
|
+
if (!current() || isTerminalWorkflow(run.status)) return;
|
|
1758
2502
|
this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
|
|
2503
|
+
if (!current()) return;
|
|
1759
2504
|
run.nonResumable = true;
|
|
2505
|
+
// Runtime abort listeners own exact cleanup. Rejecting their waiters here would
|
|
2506
|
+
// falsely settle the dispatch before its stop-and-quiesce barrier completes.
|
|
1760
2507
|
state.controller.abort();
|
|
1761
|
-
|
|
2508
|
+
if (!current()) return;
|
|
2509
|
+
await this.drainExecution(state, state.execution);
|
|
2510
|
+
if (!current() || state.pendingCleanup.size > 0) return;
|
|
2511
|
+
try {
|
|
2512
|
+
await this.drainJournal(run.runId);
|
|
2513
|
+
} catch {
|
|
2514
|
+
if (current())
|
|
2515
|
+
this.fenceExecution(state, generation, "Could not drain workflow journal during stop; run is non-resumable");
|
|
2516
|
+
return;
|
|
2517
|
+
}
|
|
2518
|
+
if (!current()) return;
|
|
1762
2519
|
const terminalResults: RecoveryTerminalResult[] = [];
|
|
1763
2520
|
for (const [nodeId, status] of Object.entries(run.callStatus)) {
|
|
1764
2521
|
if (status !== "completed") {
|
|
@@ -1778,8 +2535,6 @@ export class WorkflowEngine {
|
|
|
1778
2535
|
result,
|
|
1779
2536
|
owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId: run.attemptIds[nodeId] ?? "" },
|
|
1780
2537
|
});
|
|
1781
|
-
run.callResults[nodeId] = result;
|
|
1782
|
-
run.callStatus[nodeId] = "stopped";
|
|
1783
2538
|
}
|
|
1784
2539
|
}
|
|
1785
2540
|
const event: TerminalRecoveryEvent = {
|
|
@@ -1795,10 +2550,22 @@ export class WorkflowEngine {
|
|
|
1795
2550
|
recoveryId: deriveRecoveryId({ runId: run.runId, rotations: [], timestamp: Date.now() }),
|
|
1796
2551
|
timestamp: Date.now(),
|
|
1797
2552
|
};
|
|
1798
|
-
|
|
2553
|
+
try {
|
|
2554
|
+
this.appendRequired(event);
|
|
2555
|
+
} catch {
|
|
2556
|
+
this.fenceExecution(state, generation, "Could not persist workflow stop terminal; run is non-resumable");
|
|
2557
|
+
return;
|
|
2558
|
+
}
|
|
2559
|
+
if (!current()) return;
|
|
2560
|
+
for (const terminal of terminalResults) {
|
|
2561
|
+
run.callResults[terminal.nodeId] = terminal.result;
|
|
2562
|
+
run.callStatus[terminal.nodeId] = "stopped";
|
|
2563
|
+
}
|
|
1799
2564
|
run.status = "stopped";
|
|
1800
2565
|
run.updatedAt = Date.now();
|
|
1801
|
-
|
|
2566
|
+
state.presentation.settle(run.status);
|
|
2567
|
+
this.notifyInspection(state);
|
|
2568
|
+
if (current()) this.notifyTerminal(state);
|
|
1802
2569
|
}
|
|
1803
2570
|
|
|
1804
2571
|
// ── journal writes ─────────────────────────────────────────────────────────
|
|
@@ -1811,15 +2578,24 @@ export class WorkflowEngine {
|
|
|
1811
2578
|
* whose appends keep failing is marked blocked (reported once, not spammed).
|
|
1812
2579
|
*/
|
|
1813
2580
|
private scheduleJournalRetry(runId: string, key: string, mutation: () => void): void {
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
const
|
|
2581
|
+
// Every accepted fact has its own slot, including later retry generations
|
|
2582
|
+
// for the same call. Coalescing by node key would silently drop that fact.
|
|
2583
|
+
const retryKey = `${runId}\u0000${key}\u0000${++this.journalMutationSequence}`;
|
|
2584
|
+
const predecessors = [...this.journalRetries.entries()]
|
|
2585
|
+
.filter(([id]) => id.startsWith(`${runId}\u0000`))
|
|
2586
|
+
.map(([, pending]) => pending.done);
|
|
2587
|
+
let resolve!: () => void;
|
|
2588
|
+
const done = new Promise<void>((settled) => {
|
|
2589
|
+
resolve = settled;
|
|
2590
|
+
});
|
|
2591
|
+
const retry = { attempt: 0, timer: undefined as ReturnType<typeof setTimeout> | undefined, done, resolve };
|
|
1817
2592
|
const attempt = (): void => {
|
|
1818
2593
|
const current = this.journalRetries.get(retryKey);
|
|
1819
2594
|
if (!current) return;
|
|
1820
2595
|
current.timer = undefined;
|
|
1821
2596
|
if (this.lifecyclePaused || !this.runs.has(runId)) {
|
|
1822
2597
|
this.journalRetries.delete(retryKey);
|
|
2598
|
+
current.resolve();
|
|
1823
2599
|
return;
|
|
1824
2600
|
}
|
|
1825
2601
|
const previousKey = this.journalMutationKey;
|
|
@@ -1827,9 +2603,12 @@ export class WorkflowEngine {
|
|
|
1827
2603
|
try {
|
|
1828
2604
|
mutation();
|
|
1829
2605
|
this.journalRetries.delete(retryKey);
|
|
2606
|
+
current.resolve();
|
|
1830
2607
|
} catch (error: unknown) {
|
|
1831
2608
|
if (!(error instanceof JournalAppendError)) {
|
|
1832
2609
|
this.journalRetries.delete(retryKey);
|
|
2610
|
+
current.resolve();
|
|
2611
|
+
this.journalBlockedRuns.add(runId);
|
|
1833
2612
|
console.warn(
|
|
1834
2613
|
`[pi-workflows] journal retry abandoned: ${error instanceof Error ? error.message : String(error)}`,
|
|
1835
2614
|
);
|
|
@@ -1838,6 +2617,7 @@ export class WorkflowEngine {
|
|
|
1838
2617
|
current.attempt += 1;
|
|
1839
2618
|
if (current.attempt >= JOURNAL_RETRY_MAX_ATTEMPTS) {
|
|
1840
2619
|
this.journalRetries.delete(retryKey);
|
|
2620
|
+
current.resolve();
|
|
1841
2621
|
this.journalBlockedRuns.add(runId);
|
|
1842
2622
|
console.warn(`[pi-workflows] journal retry exhausted for ${runId}: ${error.message}`);
|
|
1843
2623
|
return;
|
|
@@ -1852,7 +2632,19 @@ export class WorkflowEngine {
|
|
|
1852
2632
|
}
|
|
1853
2633
|
};
|
|
1854
2634
|
this.journalRetries.set(retryKey, retry);
|
|
1855
|
-
|
|
2635
|
+
void Promise.all(predecessors).then(attempt);
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
private async drainJournal(runId: string): Promise<void> {
|
|
2639
|
+
let pending = [...this.journalRetries.entries()].filter(([id]) => id.startsWith(`${runId}\u0000`));
|
|
2640
|
+
while (pending.length > 0) {
|
|
2641
|
+
await Promise.all(pending.map(([, retry]) => retry.done));
|
|
2642
|
+
pending = [...this.journalRetries.entries()].filter(([id]) => id.startsWith(`${runId}\u0000`));
|
|
2643
|
+
}
|
|
2644
|
+
if (this.journalBlockedRuns.has(runId))
|
|
2645
|
+
throw new WorkflowError("Workflow journal has not settled", WorkflowErrorCode.PERSISTENCE_ERROR, {
|
|
2646
|
+
recoverable: true,
|
|
2647
|
+
});
|
|
1856
2648
|
}
|
|
1857
2649
|
|
|
1858
2650
|
private appendRequired(event: JournalEvent): void {
|
|
@@ -1888,6 +2680,18 @@ export class WorkflowEngine {
|
|
|
1888
2680
|
* a transient failure, schedules a retry of the exact same append.
|
|
1889
2681
|
*/
|
|
1890
2682
|
private persist(runId: string, key: string, mutation: () => void): void {
|
|
2683
|
+
const acceptedState = this.runs.get(runId);
|
|
2684
|
+
const epoch = this.branchEpoch;
|
|
2685
|
+
const append = mutation;
|
|
2686
|
+
mutation = () => {
|
|
2687
|
+
if (this.disposed || this.lifecyclePaused || this.runs.get(runId) !== acceptedState || this.branchEpoch !== epoch)
|
|
2688
|
+
return;
|
|
2689
|
+
append();
|
|
2690
|
+
};
|
|
2691
|
+
if ([...this.journalRetries.keys()].some((id) => id.startsWith(`${runId}\u0000`))) {
|
|
2692
|
+
this.scheduleJournalRetry(runId, key, mutation);
|
|
2693
|
+
return;
|
|
2694
|
+
}
|
|
1891
2695
|
const previousKey = this.journalMutationKey;
|
|
1892
2696
|
this.journalMutationKey = `${runId}:${key}`;
|
|
1893
2697
|
try {
|
|
@@ -1942,8 +2746,9 @@ export class WorkflowEngine {
|
|
|
1942
2746
|
this.providerResumeTimers.set(runId, timer);
|
|
1943
2747
|
}
|
|
1944
2748
|
private clearJournalRetries(): void {
|
|
1945
|
-
for (const { timer } of this.journalRetries.values()) {
|
|
2749
|
+
for (const { timer, resolve } of this.journalRetries.values()) {
|
|
1946
2750
|
if (timer) clearTimeout(timer);
|
|
2751
|
+
resolve();
|
|
1947
2752
|
}
|
|
1948
2753
|
this.journalRetries.clear();
|
|
1949
2754
|
this.journalBlockedRuns.clear();
|
|
@@ -1959,6 +2764,10 @@ export class WorkflowEngine {
|
|
|
1959
2764
|
finalResult?: unknown,
|
|
1960
2765
|
): void {
|
|
1961
2766
|
if (isTerminalWorkflow(run.status) && !isTerminalWorkflow(status)) return;
|
|
2767
|
+
const state = this.runs.get(run.runId);
|
|
2768
|
+
const generation = state?.executionGeneration ?? 0;
|
|
2769
|
+
const epoch = this.branchEpoch;
|
|
2770
|
+
if (!state || state.run !== run || !this.isCurrentState(state, generation, epoch)) return;
|
|
1962
2771
|
// Validate and isolate terminal values before best-effort persistence or any
|
|
1963
2772
|
// status mutation. A bad result must still be able to transition the run to failed.
|
|
1964
2773
|
const durableFinalResult = finalResult === undefined ? undefined : cloneWorkflowResult(finalResult);
|
|
@@ -1976,15 +2785,18 @@ export class WorkflowEngine {
|
|
|
1976
2785
|
timestamp,
|
|
1977
2786
|
}),
|
|
1978
2787
|
);
|
|
2788
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1979
2789
|
run.status = status;
|
|
2790
|
+
if (status === "running") run.error = undefined;
|
|
1980
2791
|
if (error !== undefined) run.error = error;
|
|
1981
2792
|
if (runFinalResult !== undefined) run.finalResult = runFinalResult;
|
|
1982
2793
|
if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
|
|
1983
2794
|
run.updatedAt = timestamp;
|
|
2795
|
+
this.notifyInspection(state);
|
|
2796
|
+
if (!this.isCurrentState(state, generation, epoch)) return;
|
|
1984
2797
|
if (isTerminalWorkflow(status)) {
|
|
1985
|
-
|
|
1986
|
-
this.
|
|
1987
|
-
this.onRunSettled?.();
|
|
2798
|
+
state.bufferedTerminals.clear();
|
|
2799
|
+
this.notifyTerminal(state);
|
|
1988
2800
|
}
|
|
1989
2801
|
}
|
|
1990
2802
|
|
|
@@ -1992,6 +2804,9 @@ export class WorkflowEngine {
|
|
|
1992
2804
|
|
|
1993
2805
|
restore(entries: readonly SessionEntryLike[], branchGeneration = 0): void {
|
|
1994
2806
|
if (this.disposed) return;
|
|
2807
|
+
const retiredStates = [...this.runs.values()];
|
|
2808
|
+
this.branchEpoch++;
|
|
2809
|
+
for (const state of retiredStates) state.lifecycleSuspended = true;
|
|
1995
2810
|
// A branch replay is a replacement, not a merge. Runs absent from the new
|
|
1996
2811
|
// branch must not remain controllable or append facts into it.
|
|
1997
2812
|
this.clearJournalRetries();
|
|
@@ -2009,6 +2824,10 @@ export class WorkflowEngine {
|
|
|
2009
2824
|
for (const run of runs.values()) {
|
|
2010
2825
|
const state: ScriptRunState = {
|
|
2011
2826
|
run,
|
|
2827
|
+
presentation: new WorkflowPresentation(run.runId, true),
|
|
2828
|
+
managedCallFacts: new Map(),
|
|
2829
|
+
pendingCleanup: new Set(),
|
|
2830
|
+
activeExecutionGenerations: new Set(),
|
|
2012
2831
|
frozenOptions: {
|
|
2013
2832
|
...(run.frozenTokenBudget !== undefined ? { tokenBudget: run.frozenTokenBudget } : {}),
|
|
2014
2833
|
...(run.frozenMaxAgents !== undefined ? { maxAgents: run.frozenMaxAgents } : {}),
|
|
@@ -2043,6 +2862,8 @@ export class WorkflowEngine {
|
|
|
2043
2862
|
run.updatedAt = Date.now();
|
|
2044
2863
|
this.runs.set(run.runId, state);
|
|
2045
2864
|
}
|
|
2865
|
+
for (const state of retiredStates) this.notifyInspection(state);
|
|
2866
|
+
for (const state of this.runs.values()) this.notifyInspection(state);
|
|
2046
2867
|
}
|
|
2047
2868
|
|
|
2048
2869
|
list(): Array<Record<string, unknown>> {
|
|
@@ -2060,6 +2881,18 @@ export class WorkflowEngine {
|
|
|
2060
2881
|
return this.runs.get(runId)?.run;
|
|
2061
2882
|
}
|
|
2062
2883
|
|
|
2884
|
+
/** Primitive-only Fleet metadata; avoids cloning full inspection arrays for every run. */
|
|
2885
|
+
getInspectionHeader(runId: string): WorkflowInspectionHeader | undefined {
|
|
2886
|
+
const state = this.runs.get(runId);
|
|
2887
|
+
return state?.presentation.header(state.executionGeneration);
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
/** Detached bounded facts for late-opening UI mounts; no event callback ownership required. */
|
|
2891
|
+
getInspection(runId: string): WorkflowInspection | undefined {
|
|
2892
|
+
const state = this.runs.get(runId);
|
|
2893
|
+
return state?.presentation.snapshot(state.run, state.executionGeneration);
|
|
2894
|
+
}
|
|
2895
|
+
|
|
2063
2896
|
summary(run: ScriptRun, detailed = false): Record<string, unknown> {
|
|
2064
2897
|
const base: Record<string, unknown> = {
|
|
2065
2898
|
runId: run.runId,
|
|
@@ -2081,7 +2914,7 @@ export class WorkflowEngine {
|
|
|
2081
2914
|
return {
|
|
2082
2915
|
index: Number(nodeId),
|
|
2083
2916
|
status,
|
|
2084
|
-
...(result?.result !== undefined ? { resultPreview:
|
|
2917
|
+
...(result?.result !== undefined ? { resultPreview: renderWorkflowValue(result.result).slice(0, 300) } : {}),
|
|
2085
2918
|
...(result?.error ? { error: result.error.slice(0, 300) } : {}),
|
|
2086
2919
|
...(result?.tokenCount !== undefined ? { tokenCount: result.tokenCount } : {}),
|
|
2087
2920
|
...(run.agentIds[nodeId] ? { agentId: run.agentIds[nodeId] } : {}),
|
|
@@ -2110,5 +2943,5 @@ export class WorkflowWaitAbortedError extends Error {
|
|
|
2110
2943
|
}
|
|
2111
2944
|
}
|
|
2112
2945
|
|
|
2113
|
-
export type { ScriptRun, WorkflowOwner };
|
|
2946
|
+
export type { ScriptRun, WorkflowInspection, WorkflowInspectionHeader, WorkflowOwner };
|
|
2114
2947
|
export { createManagedSpawnClient, JOURNAL_ENTRY_TYPE, snapshotRun };
|