@opengeni/worker-bundle 0.7.4 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +2853 -834
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12358 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2052 -554
- package/src/activities/capabilities.ts +9 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +367 -92
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/parent-wake.ts +48 -38
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +1 -1
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +83 -12
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
package/src/sandbox-resume.ts
CHANGED
|
@@ -231,11 +231,24 @@ class SnapshotTimeoutError extends Error {
|
|
|
231
231
|
export async function waitForWarmSnapshot(
|
|
232
232
|
snapshot: Promise<unknown>,
|
|
233
233
|
timeoutMs: number,
|
|
234
|
+
signal?: AbortSignal,
|
|
234
235
|
): Promise<boolean> {
|
|
236
|
+
if (signal?.aborted) return false;
|
|
235
237
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
238
|
+
let cancelListener: (() => void) | undefined;
|
|
236
239
|
try {
|
|
237
240
|
await Promise.race([
|
|
238
241
|
snapshot,
|
|
242
|
+
...(signal
|
|
243
|
+
? [
|
|
244
|
+
new Promise<never>((_resolve, reject) => {
|
|
245
|
+
cancelListener = () =>
|
|
246
|
+
reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
|
|
247
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
248
|
+
if (signal.aborted) cancelListener();
|
|
249
|
+
}),
|
|
250
|
+
]
|
|
251
|
+
: []),
|
|
239
252
|
new Promise<never>((_, reject) => {
|
|
240
253
|
timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
|
|
241
254
|
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
@@ -245,6 +258,7 @@ export async function waitForWarmSnapshot(
|
|
|
245
258
|
]);
|
|
246
259
|
return true;
|
|
247
260
|
} catch (error) {
|
|
261
|
+
if (signal?.aborted) return false;
|
|
248
262
|
if (error instanceof SnapshotTimeoutError) {
|
|
249
263
|
console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
|
|
250
264
|
return false;
|
|
@@ -254,6 +268,7 @@ export async function waitForWarmSnapshot(
|
|
|
254
268
|
if (timeout) {
|
|
255
269
|
clearTimeout(timeout);
|
|
256
270
|
}
|
|
271
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
257
272
|
}
|
|
258
273
|
}
|
|
259
274
|
|
|
@@ -389,9 +404,17 @@ function preserveWorkspaceArchivesOnResumeState(
|
|
|
389
404
|
*/
|
|
390
405
|
export async function maybePersistWarmWorkspaceSnapshot(
|
|
391
406
|
services: SandboxResumeServices,
|
|
392
|
-
ids: {
|
|
407
|
+
ids: {
|
|
408
|
+
accountId: string;
|
|
409
|
+
workspaceId: string;
|
|
410
|
+
sessionId: string;
|
|
411
|
+
turnId: string;
|
|
412
|
+
attemptId: string;
|
|
413
|
+
sandboxGroupId: string;
|
|
414
|
+
},
|
|
393
415
|
session: unknown,
|
|
394
416
|
leaseEpoch: number,
|
|
417
|
+
signal?: AbortSignal,
|
|
395
418
|
): Promise<boolean> {
|
|
396
419
|
const { db, settings } = services;
|
|
397
420
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
@@ -404,6 +427,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
404
427
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
405
428
|
return false;
|
|
406
429
|
}
|
|
430
|
+
if (signal?.aborted) {
|
|
431
|
+
return false;
|
|
432
|
+
}
|
|
407
433
|
try {
|
|
408
434
|
// Cheap throttle pre-check before the (potentially slow) capture;
|
|
409
435
|
// persistWarmSnapshot re-checks atomically under the row lock, so this is
|
|
@@ -412,6 +438,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
412
438
|
if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
|
|
413
439
|
return false;
|
|
414
440
|
}
|
|
441
|
+
if (signal?.aborted) {
|
|
442
|
+
return false;
|
|
443
|
+
}
|
|
415
444
|
const sessionState =
|
|
416
445
|
lease.resumeState && typeof lease.resumeState === "object"
|
|
417
446
|
? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
|
|
@@ -437,8 +466,18 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
437
466
|
// guard this timeout exists to provide.
|
|
438
467
|
const capture = persistable.persistWorkspace();
|
|
439
468
|
capture.catch(() => undefined);
|
|
469
|
+
let cancelListener: (() => void) | undefined;
|
|
440
470
|
const bytes = await Promise.race([
|
|
441
471
|
capture,
|
|
472
|
+
...(signal
|
|
473
|
+
? [
|
|
474
|
+
new Promise<undefined>((resolve) => {
|
|
475
|
+
cancelListener = () => resolve(undefined);
|
|
476
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
477
|
+
if (signal.aborted) cancelListener();
|
|
478
|
+
}),
|
|
479
|
+
]
|
|
480
|
+
: []),
|
|
442
481
|
new Promise<undefined>((resolve) => {
|
|
443
482
|
timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
|
|
444
483
|
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
@@ -449,13 +488,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
449
488
|
if (timeout) {
|
|
450
489
|
clearTimeout(timeout);
|
|
451
490
|
}
|
|
491
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
452
492
|
});
|
|
453
493
|
if (!bytes || bytes.length === 0) {
|
|
454
494
|
return false;
|
|
455
495
|
}
|
|
496
|
+
if (signal?.aborted) {
|
|
497
|
+
return false;
|
|
498
|
+
}
|
|
456
499
|
const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
|
|
457
500
|
accountId: ids.accountId,
|
|
458
501
|
workspaceId: ids.workspaceId,
|
|
502
|
+
sessionId: ids.sessionId,
|
|
503
|
+
turnId: ids.turnId,
|
|
504
|
+
attemptId: ids.attemptId,
|
|
459
505
|
sandboxGroupId: ids.sandboxGroupId,
|
|
460
506
|
expectedEpoch: leaseEpoch,
|
|
461
507
|
workspaceArchive: Buffer.from(bytes).toString("base64"),
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
//
|
|
5
5
|
// The turn resumes its group box by id (resumeBoxForTurn) and injects it
|
|
6
6
|
// NON-OWNED into the run. With hot-swap, the injected `session` must be the
|
|
7
|
-
// STABLE routing proxy
|
|
7
|
+
// STABLE routing proxy: the SDK binds to it ONCE and calls its
|
|
8
8
|
// methods per tool call; the proxy re-reads `(active_sandbox_id, active_epoch)`
|
|
9
9
|
// per op and dispatches to the currently-active backend (the group Modal box by
|
|
10
10
|
// default, or a swap target — a sibling Modal box or a selfhosted machine).
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
import type { WorkerLifecycleState } from "./http";
|
|
3
|
+
|
|
4
|
+
export async function constructWithOwnedConnection<Connection, Result>(
|
|
5
|
+
connect: () => Promise<Connection>,
|
|
6
|
+
construct: (connection: Connection) => Promise<Result>,
|
|
7
|
+
close: (connection: Connection) => Promise<void>,
|
|
8
|
+
): Promise<Result> {
|
|
9
|
+
const connection = await connect();
|
|
10
|
+
try {
|
|
11
|
+
return await construct(connection);
|
|
12
|
+
} catch (error) {
|
|
13
|
+
await close(connection).catch(() => undefined);
|
|
14
|
+
throw error;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export type WorkerRunTarget = {
|
|
19
|
+
run(): Promise<void>;
|
|
20
|
+
shutdown(): void;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type WorkerServiceLifecycle = {
|
|
24
|
+
state(): WorkerLifecycleState;
|
|
25
|
+
run(): Promise<void>;
|
|
26
|
+
drain(reason?: string): void;
|
|
27
|
+
close(): Promise<void>;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export function createWorkerServiceLifecycle(input: {
|
|
31
|
+
role: "control" | "turn";
|
|
32
|
+
worker: WorkerRunTarget;
|
|
33
|
+
observability: Observability;
|
|
34
|
+
closeOwnedResources: () => Promise<void>;
|
|
35
|
+
onReady?: () => void;
|
|
36
|
+
}): WorkerServiceLifecycle {
|
|
37
|
+
let state: WorkerLifecycleState = "starting";
|
|
38
|
+
let runPromise: Promise<void> | undefined;
|
|
39
|
+
let resourcesClosed: Promise<void> | undefined;
|
|
40
|
+
const closeOwnedResources = () => {
|
|
41
|
+
resourcesClosed ??= input.closeOwnedResources();
|
|
42
|
+
return resourcesClosed;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const lifecycle: WorkerServiceLifecycle = {
|
|
46
|
+
state: () => state,
|
|
47
|
+
run: () => {
|
|
48
|
+
if (!runPromise && state === "draining") {
|
|
49
|
+
runPromise = (async () => {
|
|
50
|
+
state = "stopped";
|
|
51
|
+
await closeOwnedResources();
|
|
52
|
+
})();
|
|
53
|
+
return runPromise;
|
|
54
|
+
}
|
|
55
|
+
if (!runPromise && (state === "stopped" || state === "failed")) {
|
|
56
|
+
return Promise.reject(new Error(`cannot run a worker service that is ${state}`));
|
|
57
|
+
}
|
|
58
|
+
runPromise ??= (async () => {
|
|
59
|
+
if (state === "starting") {
|
|
60
|
+
state = "ready";
|
|
61
|
+
}
|
|
62
|
+
input.onReady?.();
|
|
63
|
+
try {
|
|
64
|
+
await input.worker.run();
|
|
65
|
+
state = "stopped";
|
|
66
|
+
} catch (error) {
|
|
67
|
+
state = "failed";
|
|
68
|
+
throw error;
|
|
69
|
+
} finally {
|
|
70
|
+
await closeOwnedResources();
|
|
71
|
+
}
|
|
72
|
+
})();
|
|
73
|
+
return runPromise;
|
|
74
|
+
},
|
|
75
|
+
drain: (reason = "host request") => {
|
|
76
|
+
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
state = "draining";
|
|
80
|
+
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
81
|
+
role: input.role,
|
|
82
|
+
reason,
|
|
83
|
+
});
|
|
84
|
+
try {
|
|
85
|
+
input.worker.shutdown();
|
|
86
|
+
} catch (error) {
|
|
87
|
+
input.observability.warn("worker shutdown request failed", {
|
|
88
|
+
error: error instanceof Error ? error.message : String(error),
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
close: async () => {
|
|
93
|
+
lifecycle.drain("service close");
|
|
94
|
+
if (runPromise) {
|
|
95
|
+
await runPromise.catch(() => undefined);
|
|
96
|
+
} else {
|
|
97
|
+
state = "stopped";
|
|
98
|
+
await closeOwnedResources();
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
return lifecycle;
|
|
104
|
+
}
|
|
@@ -5,10 +5,12 @@ type WorkflowControlActivities = Pick<
|
|
|
5
5
|
typeof activities,
|
|
6
6
|
| "dispatchScheduledTaskRun"
|
|
7
7
|
| "enqueueGoalRetryWake"
|
|
8
|
+
| "expireSessionHumanInput"
|
|
8
9
|
| "failSessionAttempt"
|
|
9
10
|
| "getCodexCapacityWait"
|
|
10
11
|
| "markSessionIdle"
|
|
11
12
|
| "peekSessionWork"
|
|
13
|
+
| "persistSessionAttemptQuiescence"
|
|
12
14
|
| "reconcileCodexCapacityWait"
|
|
13
15
|
| "recoverDispatch"
|
|
14
16
|
| "settleSessionInterruptions"
|
|
@@ -49,7 +51,7 @@ export function turnTaskQueue(baseTaskQueue: string): string {
|
|
|
49
51
|
return `${baseTaskQueue}-turns`;
|
|
50
52
|
}
|
|
51
53
|
|
|
52
|
-
export function turnActivityForTaskQueue(baseTaskQueue: string) {
|
|
54
|
+
export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCancellation = true) {
|
|
53
55
|
return proxyActivities<Pick<typeof activities, "runAgentTurn">>({
|
|
54
56
|
taskQueue: turnTaskQueue(baseTaskQueue),
|
|
55
57
|
// Agent segments legitimately run for days. A started turn heartbeats;
|
|
@@ -57,12 +59,16 @@ export function turnActivityForTaskQueue(baseTaskQueue: string) {
|
|
|
57
59
|
// accepts them and performs the atomic claim.
|
|
58
60
|
startToCloseTimeout: "30 days",
|
|
59
61
|
heartbeatTimeout: "2 minutes",
|
|
60
|
-
// Pause/Steer
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
//
|
|
65
|
-
|
|
62
|
+
// Pause/Steer first closes the exact attempt in Postgres, then asks
|
|
63
|
+
// Temporal to deliver cancellation. The workflow must not wait for the
|
|
64
|
+
// Temporal activity promise: a provider cleanup promise can outlive the
|
|
65
|
+
// fenced activity body, and Temporal terminalization is not proof that
|
|
66
|
+
// sandbox tools or processes are physically quiescent. The activity owns
|
|
67
|
+
// that proof and writes an exact receipt after its hard tool fence; the
|
|
68
|
+
// receipt transaction wakes the workflow to admit a replacement.
|
|
69
|
+
cancellationType: receiptGatedCancellation
|
|
70
|
+
? ActivityCancellationType.TRY_CANCEL
|
|
71
|
+
: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
|
|
66
72
|
retry: { maximumAttempts: 1 },
|
|
67
73
|
});
|
|
68
74
|
}
|
package/src/workflows/session.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ActivityFailure,
|
|
3
|
+
ApplicationFailure,
|
|
3
4
|
CancellationScope,
|
|
4
5
|
condition,
|
|
5
6
|
continueAsNew,
|
|
6
7
|
defineSignal,
|
|
7
8
|
isCancellation,
|
|
9
|
+
patched,
|
|
8
10
|
setHandler,
|
|
9
11
|
TimeoutFailure,
|
|
10
12
|
uuid4,
|
|
@@ -33,6 +35,7 @@ import {
|
|
|
33
35
|
*/
|
|
34
36
|
const TURNS_PER_RUN_BACKSTOP = 2_000;
|
|
35
37
|
const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
|
|
38
|
+
const HUMAN_INPUT_EXPIRY_STALE_RETRY_MS = 1_000;
|
|
36
39
|
|
|
37
40
|
/**
|
|
38
41
|
* The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
|
|
@@ -68,6 +71,24 @@ export function continuationHoldMs(
|
|
|
68
71
|
return Math.max(delay, 0);
|
|
69
72
|
}
|
|
70
73
|
|
|
74
|
+
/**
|
|
75
|
+
* A deferred activity result is terminal for this workflow run unless a new
|
|
76
|
+
* non-control wake committed while the activity was in flight. Keeping this
|
|
77
|
+
* predicate pure makes the compaction-convergence fence directly testable:
|
|
78
|
+
* repeated identical failures cannot synthesize more work from an unchanged
|
|
79
|
+
* durable state. Pause/Steer remain authoritative through the independent
|
|
80
|
+
* signal-version and interruption paths in `sessionWorkflow`.
|
|
81
|
+
*/
|
|
82
|
+
export function deferredResultMayContinue(entryWakeups: number, currentWakeups: number): boolean {
|
|
83
|
+
return currentWakeups !== entryWakeups;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Deterministic Temporal timer delay for a persisted structured-input deadline. */
|
|
87
|
+
export function humanInputDeadlineWaitMs(expiresAt: string, nowMs = Date.now()): number {
|
|
88
|
+
const deadline = Date.parse(expiresAt);
|
|
89
|
+
return Number.isFinite(deadline) ? Math.max(0, deadline - nowMs) : 0;
|
|
90
|
+
}
|
|
91
|
+
|
|
71
92
|
/**
|
|
72
93
|
* True when an agent-turn activity failure means "the worker hosting the
|
|
73
94
|
* turn died or vanished" rather than "the turn itself failed": the server
|
|
@@ -98,11 +119,34 @@ function workerDeathFailure(
|
|
|
98
119
|
return { timeoutType: cause.timeoutType };
|
|
99
120
|
}
|
|
100
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Classify only the exact turn-fence cancellation protocol used to arbitrate a
|
|
124
|
+
* database control commit that reaches the activity just before its Temporal
|
|
125
|
+
* signal reaches this workflow. This shape is never physical-quiescence proof:
|
|
126
|
+
* only the activity's durable post-tool-fence receipt can open replacement
|
|
127
|
+
* admission. @temporalio/worker 1.20 serializes the pre-request
|
|
128
|
+
* CancelledFailure as an ApplicationFailure with the stable `CancelledFailure`
|
|
129
|
+
* type, so accept that exact wire shape in addition to normal cancellation.
|
|
130
|
+
*/
|
|
131
|
+
export function isTurnActivityFenceCancellation(error: unknown): boolean {
|
|
132
|
+
if (isCancellation(error)) return true;
|
|
133
|
+
if (!(error instanceof ActivityFailure)) return false;
|
|
134
|
+
if (error.cause !== undefined && isCancellation(error.cause)) return true;
|
|
135
|
+
return (
|
|
136
|
+
error.activityType === "runAgentTurn" &&
|
|
137
|
+
error.cause instanceof ApplicationFailure &&
|
|
138
|
+
error.cause.type === "CancelledFailure" &&
|
|
139
|
+
error.cause.message === "TURN_ATTEMPT_FENCED"
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
101
143
|
export const userMessage = defineSignal<[string]>("userMessage");
|
|
102
144
|
export const queueChanged = defineSignal("queueChanged");
|
|
103
145
|
export const approvalDecision = defineSignal<[string]>("approvalDecision");
|
|
104
146
|
export const sessionControl = defineSignal("sessionControl");
|
|
105
147
|
export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
|
|
148
|
+
export const sessionAttemptQuiesced =
|
|
149
|
+
defineSignal<[activities.SessionAttemptQuiescenceProof]>("sessionAttemptQuiesced");
|
|
106
150
|
|
|
107
151
|
export type SessionWorkflowInput = {
|
|
108
152
|
accountId: string;
|
|
@@ -120,13 +164,19 @@ export type SessionWorkflowInput = {
|
|
|
120
164
|
};
|
|
121
165
|
|
|
122
166
|
export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
|
|
123
|
-
|
|
167
|
+
// Existing histories recorded the v1 WAIT_CANCELLATION_COMPLETED command
|
|
168
|
+
// order and workflow-side idempotent fallback. Keep that exact replay path;
|
|
169
|
+
// every new run records v2 and uses the activity-owned receipt contract.
|
|
170
|
+
const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
|
|
171
|
+
const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
|
|
124
172
|
let approvalWakeups = 0;
|
|
125
173
|
let interruptionWakeups = 0;
|
|
126
174
|
let wakeups = 0;
|
|
127
175
|
let capacityWakeups = 0;
|
|
128
176
|
let signalVersion = 0;
|
|
129
177
|
let nonControlSignalVersion = 0;
|
|
178
|
+
const pendingQuiescenceProofs = new Map<string, activities.SessionAttemptQuiescenceProof>();
|
|
179
|
+
const persistedQuiescenceProofs = new Set<string>();
|
|
130
180
|
// Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
|
|
131
181
|
// for the history-overflow guard below; bounded growth is what makes a
|
|
132
182
|
// weeks-long session survivable.
|
|
@@ -157,6 +207,38 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
157
207
|
nonControlSignalVersion += 1;
|
|
158
208
|
capacityWakeups += 1;
|
|
159
209
|
});
|
|
210
|
+
setHandler(sessionAttemptQuiesced, (proof) => {
|
|
211
|
+
// Temporal signals are untrusted transport payloads. Accept only the exact
|
|
212
|
+
// workflow/session scope; the DB control activity additionally validates
|
|
213
|
+
// the persisted attempt's account, run id, and activity id under lock.
|
|
214
|
+
if (
|
|
215
|
+
!isSessionAttemptQuiescenceProof(proof) ||
|
|
216
|
+
proof.accountId !== input.accountId ||
|
|
217
|
+
proof.workspaceId !== input.workspaceId ||
|
|
218
|
+
proof.sessionId !== input.sessionId ||
|
|
219
|
+
proof.workflowId !== workflowInfo().workflowId
|
|
220
|
+
) {
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const key = sessionAttemptQuiescenceProofKey(proof);
|
|
224
|
+
if (persistedQuiescenceProofs.has(key) || pendingQuiescenceProofs.has(key)) return;
|
|
225
|
+
pendingQuiescenceProofs.set(key, proof);
|
|
226
|
+
signalVersion += 1;
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
async function persistPendingQuiescenceProofs(): Promise<void> {
|
|
230
|
+
while (pendingQuiescenceProofs.size > 0) {
|
|
231
|
+
const entry = pendingQuiescenceProofs.entries().next().value;
|
|
232
|
+
if (!entry) return;
|
|
233
|
+
const [key, proof] = entry;
|
|
234
|
+
// This DB-only activity uses an unbounded Temporal retry policy. The
|
|
235
|
+
// workflow cannot peek, close, or continue-as-new until the exact
|
|
236
|
+
// physical proof commits its idempotent receipt and wake transaction.
|
|
237
|
+
await activity.persistSessionAttemptQuiescence(proof);
|
|
238
|
+
pendingQuiescenceProofs.delete(key);
|
|
239
|
+
persistedQuiescenceProofs.add(key);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
160
242
|
|
|
161
243
|
async function waitForCodexCapacity(
|
|
162
244
|
initial: activities.CodexCapacityWaitRef,
|
|
@@ -217,6 +299,11 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
217
299
|
// The waiter/outbox is durable in Postgres. A fresh workflow run reads
|
|
218
300
|
// it before goal continuation, reconstructs its timer, and turns any
|
|
219
301
|
// unobserved wake revision into an immediate evaluation.
|
|
302
|
+
// A quiescence proof is the one signal that is not merely a replaceable
|
|
303
|
+
// wake hint. It may have arrived while the reconciliation activity was
|
|
304
|
+
// running, so commit it before crossing this nested continue-as-new
|
|
305
|
+
// boundary.
|
|
306
|
+
await persistPendingQuiescenceProofs();
|
|
220
307
|
await continueAsNew<typeof sessionWorkflow>({
|
|
221
308
|
accountId: input.accountId,
|
|
222
309
|
workspaceId: input.workspaceId,
|
|
@@ -232,18 +319,25 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
232
319
|
}
|
|
233
320
|
|
|
234
321
|
while (true) {
|
|
322
|
+
// A proof signal can signalWithStart a fresh run or arrive at any prior
|
|
323
|
+
// close/continue-as-new boundary. Persist it before every ordinary workflow
|
|
324
|
+
// decision so no accepted physical fact can be dropped with run history.
|
|
325
|
+
await persistPendingQuiescenceProofs();
|
|
235
326
|
// History-overflow guard. The top of the loop is the only safe
|
|
236
327
|
// continueAsNew boundary: no turn is mid-flight (every path that reaches a
|
|
237
328
|
// new iteration settled or recovered its turn first). Every interruption
|
|
238
|
-
// is already durable in Postgres, so
|
|
239
|
-
// wake hints and none must be carried into the next workflow
|
|
329
|
+
// is already durable in Postgres, so ordinary Temporal signals are
|
|
330
|
+
// replaceable wake hints and none must be carried into the next workflow
|
|
331
|
+
// run. A
|
|
240
332
|
// buffered userMessage/queueChanged signal only
|
|
241
333
|
// bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
|
|
242
334
|
// was sent, so the fresh run observes it on its first durable work peek
|
|
243
335
|
// — losing the counter strands nothing. The queue living in Postgres is the
|
|
244
336
|
// safety net: continueAsNew carries only the self-contained
|
|
245
337
|
// SessionWorkflowInput (no initialEventId — the new run claims from the
|
|
246
|
-
// queue, it does not replay a seed event).
|
|
338
|
+
// queue, it does not replay a seed event). Quiescence-proof signals are the
|
|
339
|
+
// exception: persistPendingQuiescenceProofs() immediately above commits
|
|
340
|
+
// every accepted proof before this boundary.
|
|
247
341
|
//
|
|
248
342
|
// Approval signals are wakeups, never conversation truth. A genuinely
|
|
249
343
|
// accepted decision is already persisted on the turn and is rediscovered
|
|
@@ -285,6 +379,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
285
379
|
workflowId,
|
|
286
380
|
});
|
|
287
381
|
if (settlement.action === "paused") {
|
|
382
|
+
if (pendingQuiescenceProofs.size > 0) {
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
288
385
|
if (nonControlSignalVersion !== closeNonControlSignalVersion) {
|
|
289
386
|
continue;
|
|
290
387
|
}
|
|
@@ -292,6 +389,22 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
292
389
|
}
|
|
293
390
|
continue;
|
|
294
391
|
}
|
|
392
|
+
if (peek.kind === "cancellation-wait") {
|
|
393
|
+
// Logical settlement is complete, but the exact predecessor activity has
|
|
394
|
+
// not yet durably proved sandbox/tool quiescence. Wait only briefly for
|
|
395
|
+
// its transactional queueChanged wake. If provider/tool cancellation is
|
|
396
|
+
// genuinely slow, close this workflow run rather than consuming a turn
|
|
397
|
+
// slot or churning control activities; the outbox uses signalWithStart to
|
|
398
|
+
// restart this exact workflow after the receipt commits.
|
|
399
|
+
const seenSignalVersion = signalVersion;
|
|
400
|
+
const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
|
|
401
|
+
if (woke) continue;
|
|
402
|
+
// Close only against the same signal snapshot. A proof signal accepted
|
|
403
|
+
// at the timer/completion boundary must loop through the DB-only receipt
|
|
404
|
+
// activity rather than disappearing with this workflow run.
|
|
405
|
+
if (signalVersion !== seenSignalVersion || pendingQuiescenceProofs.size > 0) continue;
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
295
408
|
if (peek.kind === "capacity-wait") {
|
|
296
409
|
await waitForCodexCapacity(peek.ref);
|
|
297
410
|
continue;
|
|
@@ -300,12 +413,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
300
413
|
const seenApprovalWakeups = approvalWakeups;
|
|
301
414
|
const seenWakeups = wakeups;
|
|
302
415
|
const seenInterruptionWakeups = interruptionWakeups;
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
416
|
+
const timeoutMs =
|
|
417
|
+
peek.humanInputRequestId && peek.expiresAt
|
|
418
|
+
? humanInputDeadlineWaitMs(peek.expiresAt)
|
|
419
|
+
: undefined;
|
|
420
|
+
const wakeCondition = () =>
|
|
421
|
+
interruptionWakeups !== seenInterruptionWakeups ||
|
|
422
|
+
approvalWakeups !== seenApprovalWakeups ||
|
|
423
|
+
wakeups !== seenWakeups;
|
|
424
|
+
const woke =
|
|
425
|
+
timeoutMs === undefined
|
|
426
|
+
? await condition(wakeCondition)
|
|
427
|
+
: await condition(wakeCondition, timeoutMs);
|
|
428
|
+
if (!woke && peek.humanInputRequestId) {
|
|
429
|
+
const expiry = await activity.expireSessionHumanInput({
|
|
430
|
+
accountId: input.accountId,
|
|
431
|
+
workspaceId: input.workspaceId,
|
|
432
|
+
sessionId: input.sessionId,
|
|
433
|
+
requestId: peek.humanInputRequestId,
|
|
434
|
+
});
|
|
435
|
+
// Temporal and PostgreSQL may observe slightly different wall clocks.
|
|
436
|
+
// If the workflow timer fired first, the DB settler truthfully leaves
|
|
437
|
+
// the request pending. Bound that skew with one interruptible timer
|
|
438
|
+
// instead of spinning peek + activity at zero delay.
|
|
439
|
+
if (expiry.action === "stale") {
|
|
440
|
+
await condition(wakeCondition, HUMAN_INPUT_EXPIRY_STALE_RETRY_MS);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
309
443
|
continue;
|
|
310
444
|
}
|
|
311
445
|
if (peek.kind === "idle") {
|
|
@@ -389,7 +523,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
389
523
|
trigger,
|
|
390
524
|
}),
|
|
391
525
|
);
|
|
392
|
-
const
|
|
526
|
+
const racedOutcome:
|
|
393
527
|
| { kind: "result"; result: activities.RunAgentTurnResult }
|
|
394
528
|
| { kind: "control" }
|
|
395
529
|
| { kind: "failure"; error: unknown } = await Promise.race([
|
|
@@ -404,9 +538,55 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
404
538
|
kind: "control" as const,
|
|
405
539
|
})),
|
|
406
540
|
]);
|
|
541
|
+
// A Steer/Pause transaction fences the active attempt before its Temporal
|
|
542
|
+
// signal is necessarily handled. That fence can make the activity's typed
|
|
543
|
+
// cancellation win Promise.race by one workflow activation. Give only that
|
|
544
|
+
// confirmed cancellation shape a short deterministic arbitration window;
|
|
545
|
+
// the durable control signal is authoritative once observed. Ordinary
|
|
546
|
+
// failures and timeouts never wait here, and a cancellation with no session
|
|
547
|
+
// control still follows the failure/cancellation path below.
|
|
548
|
+
if (
|
|
549
|
+
racedOutcome.kind === "failure" &&
|
|
550
|
+
isTurnActivityFenceCancellation(racedOutcome.error) &&
|
|
551
|
+
interruptionWakeups === interruptionBaseline
|
|
552
|
+
) {
|
|
553
|
+
await condition(() => interruptionWakeups !== interruptionBaseline, "250ms");
|
|
554
|
+
}
|
|
555
|
+
const outcome =
|
|
556
|
+
interruptionWakeups !== interruptionBaseline ? ({ kind: "control" } as const) : racedOutcome;
|
|
407
557
|
|
|
408
558
|
if (outcome.kind === "control") {
|
|
409
|
-
|
|
559
|
+
if (!receiptGatedCancellation) {
|
|
560
|
+
// Replay-only v1 command order. New histories always take the v2 path
|
|
561
|
+
// below. The current runAgentTurn still owns and writes the truthful
|
|
562
|
+
// hard-fence receipt; this call remains an idempotent history command.
|
|
563
|
+
scope.cancel();
|
|
564
|
+
const settlement = await activity.settleSessionInterruptions({
|
|
565
|
+
accountId,
|
|
566
|
+
workspaceId,
|
|
567
|
+
sessionId,
|
|
568
|
+
attemptId,
|
|
569
|
+
workflowId: workflowInfo().workflowId,
|
|
570
|
+
});
|
|
571
|
+
const termination = await turn.then(
|
|
572
|
+
() => ({ kind: "completed" as const }),
|
|
573
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
574
|
+
);
|
|
575
|
+
const physicalStopConfirmed =
|
|
576
|
+
termination.kind === "completed" || isTurnActivityFenceCancellation(termination.error);
|
|
577
|
+
if (patched("session-attempt-quiescence-v1") && physicalStopConfirmed) {
|
|
578
|
+
await activity.settleSessionInterruptions({
|
|
579
|
+
accountId,
|
|
580
|
+
workspaceId,
|
|
581
|
+
sessionId,
|
|
582
|
+
attemptId,
|
|
583
|
+
workflowId: workflowInfo().workflowId,
|
|
584
|
+
phase: "attempt_quiesced",
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
if (!physicalStopConfirmed) throw termination.error;
|
|
588
|
+
return settlement.action !== "paused";
|
|
589
|
+
}
|
|
410
590
|
const settlement = await activity.settleSessionInterruptions({
|
|
411
591
|
accountId,
|
|
412
592
|
workspaceId,
|
|
@@ -414,16 +594,12 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
414
594
|
attemptId,
|
|
415
595
|
workflowId: workflowInfo().workflowId,
|
|
416
596
|
});
|
|
417
|
-
//
|
|
418
|
-
//
|
|
419
|
-
//
|
|
420
|
-
//
|
|
421
|
-
//
|
|
422
|
-
|
|
423
|
-
// the physical activity winds down. The control outcome is authoritative,
|
|
424
|
-
// so a concurrent activity completion or cancellation failure is ignored
|
|
425
|
-
// only after Temporal has durably observed that terminal activity state.
|
|
426
|
-
await turn.catch(() => undefined);
|
|
597
|
+
// The transaction above is the authority fence: every late model/tool/UI
|
|
598
|
+
// write is rejected from this point onward. Request cancellation only
|
|
599
|
+
// after it commits, then stop observing the Temporal activity promise.
|
|
600
|
+
// TRY_CANCEL allows this workflow to move to cancellation-wait without
|
|
601
|
+
// confusing Temporal completion/cancellation with local process safety.
|
|
602
|
+
scope.cancel();
|
|
427
603
|
return settlement.action !== "paused";
|
|
428
604
|
}
|
|
429
605
|
|
|
@@ -490,7 +666,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
490
666
|
}
|
|
491
667
|
|
|
492
668
|
if (outcome.result.deferredUntilWake) {
|
|
493
|
-
return
|
|
669
|
+
return deferredResultMayContinue(capacityWaitEntryBaseline.wakeups, wakeups);
|
|
494
670
|
}
|
|
495
671
|
|
|
496
672
|
if (outcome.result.status === "requires_action") return true;
|
|
@@ -512,3 +688,23 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
512
688
|
return true;
|
|
513
689
|
}
|
|
514
690
|
}
|
|
691
|
+
|
|
692
|
+
function isSessionAttemptQuiescenceProof(
|
|
693
|
+
value: unknown,
|
|
694
|
+
): value is activities.SessionAttemptQuiescenceProof {
|
|
695
|
+
if (!value || typeof value !== "object") return false;
|
|
696
|
+
const proof = value as Record<string, unknown>;
|
|
697
|
+
return [
|
|
698
|
+
"accountId",
|
|
699
|
+
"workspaceId",
|
|
700
|
+
"sessionId",
|
|
701
|
+
"attemptId",
|
|
702
|
+
"workflowId",
|
|
703
|
+
"workflowRunId",
|
|
704
|
+
"activityId",
|
|
705
|
+
].every((field) => typeof proof[field] === "string" && proof[field].length > 0);
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
function sessionAttemptQuiescenceProofKey(proof: activities.SessionAttemptQuiescenceProof): string {
|
|
709
|
+
return `${proof.attemptId}:${proof.workflowRunId}:${proof.activityId}`;
|
|
710
|
+
}
|