@opengeni/worker-bundle 0.7.4 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +2853 -834
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12358 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2052 -554
  12. package/src/activities/capabilities.ts +9 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +367 -92
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/parent-wake.ts +48 -38
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +1 -1
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +83 -12
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/sandbox-resume.ts +47 -1
  38. package/src/sandbox-routing.ts +1 -1
  39. package/src/worker-service-lifecycle.ts +104 -0
  40. package/src/workflows/activities.ts +13 -7
  41. package/src/workflows/session.ts +219 -23
  42. package/src/workflows.ts +1 -0
@@ -231,11 +231,24 @@ class SnapshotTimeoutError extends Error {
231
231
  export async function waitForWarmSnapshot(
232
232
  snapshot: Promise<unknown>,
233
233
  timeoutMs: number,
234
+ signal?: AbortSignal,
234
235
  ): Promise<boolean> {
236
+ if (signal?.aborted) return false;
235
237
  let timeout: ReturnType<typeof setTimeout> | undefined;
238
+ let cancelListener: (() => void) | undefined;
236
239
  try {
237
240
  await Promise.race([
238
241
  snapshot,
242
+ ...(signal
243
+ ? [
244
+ new Promise<never>((_resolve, reject) => {
245
+ cancelListener = () =>
246
+ reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
247
+ signal.addEventListener("abort", cancelListener, { once: true });
248
+ if (signal.aborted) cancelListener();
249
+ }),
250
+ ]
251
+ : []),
239
252
  new Promise<never>((_, reject) => {
240
253
  timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
241
254
  if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
@@ -245,6 +258,7 @@ export async function waitForWarmSnapshot(
245
258
  ]);
246
259
  return true;
247
260
  } catch (error) {
261
+ if (signal?.aborted) return false;
248
262
  if (error instanceof SnapshotTimeoutError) {
249
263
  console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
250
264
  return false;
@@ -254,6 +268,7 @@ export async function waitForWarmSnapshot(
254
268
  if (timeout) {
255
269
  clearTimeout(timeout);
256
270
  }
271
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
257
272
  }
258
273
  }
259
274
 
@@ -389,9 +404,17 @@ function preserveWorkspaceArchivesOnResumeState(
389
404
  */
390
405
  export async function maybePersistWarmWorkspaceSnapshot(
391
406
  services: SandboxResumeServices,
392
- ids: { accountId: string; workspaceId: string; sandboxGroupId: string },
407
+ ids: {
408
+ accountId: string;
409
+ workspaceId: string;
410
+ sessionId: string;
411
+ turnId: string;
412
+ attemptId: string;
413
+ sandboxGroupId: string;
414
+ },
393
415
  session: unknown,
394
416
  leaseEpoch: number,
417
+ signal?: AbortSignal,
395
418
  ): Promise<boolean> {
396
419
  const { db, settings } = services;
397
420
  const intervalMs = settings.sandboxSnapshotIntervalMs;
@@ -404,6 +427,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
404
427
  if (typeof persistable.persistWorkspace !== "function") {
405
428
  return false;
406
429
  }
430
+ if (signal?.aborted) {
431
+ return false;
432
+ }
407
433
  try {
408
434
  // Cheap throttle pre-check before the (potentially slow) capture;
409
435
  // persistWarmSnapshot re-checks atomically under the row lock, so this is
@@ -412,6 +438,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
412
438
  if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
413
439
  return false;
414
440
  }
441
+ if (signal?.aborted) {
442
+ return false;
443
+ }
415
444
  const sessionState =
416
445
  lease.resumeState && typeof lease.resumeState === "object"
417
446
  ? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
@@ -437,8 +466,18 @@ export async function maybePersistWarmWorkspaceSnapshot(
437
466
  // guard this timeout exists to provide.
438
467
  const capture = persistable.persistWorkspace();
439
468
  capture.catch(() => undefined);
469
+ let cancelListener: (() => void) | undefined;
440
470
  const bytes = await Promise.race([
441
471
  capture,
472
+ ...(signal
473
+ ? [
474
+ new Promise<undefined>((resolve) => {
475
+ cancelListener = () => resolve(undefined);
476
+ signal.addEventListener("abort", cancelListener, { once: true });
477
+ if (signal.aborted) cancelListener();
478
+ }),
479
+ ]
480
+ : []),
442
481
  new Promise<undefined>((resolve) => {
443
482
  timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
444
483
  if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
@@ -449,13 +488,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
449
488
  if (timeout) {
450
489
  clearTimeout(timeout);
451
490
  }
491
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
452
492
  });
453
493
  if (!bytes || bytes.length === 0) {
454
494
  return false;
455
495
  }
496
+ if (signal?.aborted) {
497
+ return false;
498
+ }
456
499
  const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
457
500
  accountId: ids.accountId,
458
501
  workspaceId: ids.workspaceId,
502
+ sessionId: ids.sessionId,
503
+ turnId: ids.turnId,
504
+ attemptId: ids.attemptId,
459
505
  sandboxGroupId: ids.sandboxGroupId,
460
506
  expectedEpoch: leaseEpoch,
461
507
  workspaceArchive: Buffer.from(bytes).toString("base64"),
@@ -4,7 +4,7 @@
4
4
  //
5
5
  // The turn resumes its group box by id (resumeBoxForTurn) and injects it
6
6
  // NON-OWNED into the run. With hot-swap, the injected `session` must be the
7
- // STABLE routing proxy (dossier §10.3): the SDK binds to it ONCE and calls its
7
+ // STABLE routing proxy: the SDK binds to it ONCE and calls its
8
8
  // methods per tool call; the proxy re-reads `(active_sandbox_id, active_epoch)`
9
9
  // per op and dispatches to the currently-active backend (the group Modal box by
10
10
  // default, or a swap target — a sibling Modal box or a selfhosted machine).
@@ -0,0 +1,104 @@
1
+ import type { Observability } from "@opengeni/observability";
2
+ import type { WorkerLifecycleState } from "./http";
3
+
4
+ export async function constructWithOwnedConnection<Connection, Result>(
5
+ connect: () => Promise<Connection>,
6
+ construct: (connection: Connection) => Promise<Result>,
7
+ close: (connection: Connection) => Promise<void>,
8
+ ): Promise<Result> {
9
+ const connection = await connect();
10
+ try {
11
+ return await construct(connection);
12
+ } catch (error) {
13
+ await close(connection).catch(() => undefined);
14
+ throw error;
15
+ }
16
+ }
17
+
18
+ export type WorkerRunTarget = {
19
+ run(): Promise<void>;
20
+ shutdown(): void;
21
+ };
22
+
23
+ export type WorkerServiceLifecycle = {
24
+ state(): WorkerLifecycleState;
25
+ run(): Promise<void>;
26
+ drain(reason?: string): void;
27
+ close(): Promise<void>;
28
+ };
29
+
30
+ export function createWorkerServiceLifecycle(input: {
31
+ role: "control" | "turn";
32
+ worker: WorkerRunTarget;
33
+ observability: Observability;
34
+ closeOwnedResources: () => Promise<void>;
35
+ onReady?: () => void;
36
+ }): WorkerServiceLifecycle {
37
+ let state: WorkerLifecycleState = "starting";
38
+ let runPromise: Promise<void> | undefined;
39
+ let resourcesClosed: Promise<void> | undefined;
40
+ const closeOwnedResources = () => {
41
+ resourcesClosed ??= input.closeOwnedResources();
42
+ return resourcesClosed;
43
+ };
44
+
45
+ const lifecycle: WorkerServiceLifecycle = {
46
+ state: () => state,
47
+ run: () => {
48
+ if (!runPromise && state === "draining") {
49
+ runPromise = (async () => {
50
+ state = "stopped";
51
+ await closeOwnedResources();
52
+ })();
53
+ return runPromise;
54
+ }
55
+ if (!runPromise && (state === "stopped" || state === "failed")) {
56
+ return Promise.reject(new Error(`cannot run a worker service that is ${state}`));
57
+ }
58
+ runPromise ??= (async () => {
59
+ if (state === "starting") {
60
+ state = "ready";
61
+ }
62
+ input.onReady?.();
63
+ try {
64
+ await input.worker.run();
65
+ state = "stopped";
66
+ } catch (error) {
67
+ state = "failed";
68
+ throw error;
69
+ } finally {
70
+ await closeOwnedResources();
71
+ }
72
+ })();
73
+ return runPromise;
74
+ },
75
+ drain: (reason = "host request") => {
76
+ if (state === "draining" || state === "stopped" || state === "failed") {
77
+ return;
78
+ }
79
+ state = "draining";
80
+ input.observability.info("OpenGeni worker draining (graceful shutdown)", {
81
+ role: input.role,
82
+ reason,
83
+ });
84
+ try {
85
+ input.worker.shutdown();
86
+ } catch (error) {
87
+ input.observability.warn("worker shutdown request failed", {
88
+ error: error instanceof Error ? error.message : String(error),
89
+ });
90
+ }
91
+ },
92
+ close: async () => {
93
+ lifecycle.drain("service close");
94
+ if (runPromise) {
95
+ await runPromise.catch(() => undefined);
96
+ } else {
97
+ state = "stopped";
98
+ await closeOwnedResources();
99
+ }
100
+ },
101
+ };
102
+
103
+ return lifecycle;
104
+ }
@@ -5,10 +5,12 @@ type WorkflowControlActivities = Pick<
5
5
  typeof activities,
6
6
  | "dispatchScheduledTaskRun"
7
7
  | "enqueueGoalRetryWake"
8
+ | "expireSessionHumanInput"
8
9
  | "failSessionAttempt"
9
10
  | "getCodexCapacityWait"
10
11
  | "markSessionIdle"
11
12
  | "peekSessionWork"
13
+ | "persistSessionAttemptQuiescence"
12
14
  | "reconcileCodexCapacityWait"
13
15
  | "recoverDispatch"
14
16
  | "settleSessionInterruptions"
@@ -49,7 +51,7 @@ export function turnTaskQueue(baseTaskQueue: string): string {
49
51
  return `${baseTaskQueue}-turns`;
50
52
  }
51
53
 
52
- export function turnActivityForTaskQueue(baseTaskQueue: string) {
54
+ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCancellation = true) {
53
55
  return proxyActivities<Pick<typeof activities, "runAgentTurn">>({
54
56
  taskQueue: turnTaskQueue(baseTaskQueue),
55
57
  // Agent segments legitimately run for days. A started turn heartbeats;
@@ -57,12 +59,16 @@ export function turnActivityForTaskQueue(baseTaskQueue: string) {
57
59
  // accepts them and performs the atomic claim.
58
60
  startToCloseTimeout: "30 days",
59
61
  heartbeatTimeout: "2 minutes",
60
- // Pause/Steer may complete the durable control transition immediately,
61
- // but the session workflow must remain open until the old turn activity is
62
- // physically gone. Leaving this implicit uses Temporal's try-cancel wire
63
- // default: the activity promise rejects as soon as cancellation is
64
- // requested and the workflow can close while the worker keeps streaming.
65
- cancellationType: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
62
+ // Pause/Steer first closes the exact attempt in Postgres, then asks
63
+ // Temporal to deliver cancellation. The workflow must not wait for the
64
+ // Temporal activity promise: a provider cleanup promise can outlive the
65
+ // fenced activity body, and Temporal terminalization is not proof that
66
+ // sandbox tools or processes are physically quiescent. The activity owns
67
+ // that proof and writes an exact receipt after its hard tool fence; the
68
+ // receipt transaction wakes the workflow to admit a replacement.
69
+ cancellationType: receiptGatedCancellation
70
+ ? ActivityCancellationType.TRY_CANCEL
71
+ : ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
66
72
  retry: { maximumAttempts: 1 },
67
73
  });
68
74
  }
@@ -1,10 +1,12 @@
1
1
  import {
2
2
  ActivityFailure,
3
+ ApplicationFailure,
3
4
  CancellationScope,
4
5
  condition,
5
6
  continueAsNew,
6
7
  defineSignal,
7
8
  isCancellation,
9
+ patched,
8
10
  setHandler,
9
11
  TimeoutFailure,
10
12
  uuid4,
@@ -33,6 +35,7 @@ import {
33
35
  */
34
36
  const TURNS_PER_RUN_BACKSTOP = 2_000;
35
37
  const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
38
+ const HUMAN_INPUT_EXPIRY_STALE_RETRY_MS = 1_000;
36
39
 
37
40
  /**
38
41
  * The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
@@ -68,6 +71,24 @@ export function continuationHoldMs(
68
71
  return Math.max(delay, 0);
69
72
  }
70
73
 
74
+ /**
75
+ * A deferred activity result is terminal for this workflow run unless a new
76
+ * non-control wake committed while the activity was in flight. Keeping this
77
+ * predicate pure makes the compaction-convergence fence directly testable:
78
+ * repeated identical failures cannot synthesize more work from an unchanged
79
+ * durable state. Pause/Steer remain authoritative through the independent
80
+ * signal-version and interruption paths in `sessionWorkflow`.
81
+ */
82
+ export function deferredResultMayContinue(entryWakeups: number, currentWakeups: number): boolean {
83
+ return currentWakeups !== entryWakeups;
84
+ }
85
+
86
+ /** Deterministic Temporal timer delay for a persisted structured-input deadline. */
87
+ export function humanInputDeadlineWaitMs(expiresAt: string, nowMs = Date.now()): number {
88
+ const deadline = Date.parse(expiresAt);
89
+ return Number.isFinite(deadline) ? Math.max(0, deadline - nowMs) : 0;
90
+ }
91
+
71
92
  /**
72
93
  * True when an agent-turn activity failure means "the worker hosting the
73
94
  * turn died or vanished" rather than "the turn itself failed": the server
@@ -98,11 +119,34 @@ function workerDeathFailure(
98
119
  return { timeoutType: cause.timeoutType };
99
120
  }
100
121
 
122
+ /**
123
+ * Classify only the exact turn-fence cancellation protocol used to arbitrate a
124
+ * database control commit that reaches the activity just before its Temporal
125
+ * signal reaches this workflow. This shape is never physical-quiescence proof:
126
+ * only the activity's durable post-tool-fence receipt can open replacement
127
+ * admission. @temporalio/worker 1.20 serializes the pre-request
128
+ * CancelledFailure as an ApplicationFailure with the stable `CancelledFailure`
129
+ * type, so accept that exact wire shape in addition to normal cancellation.
130
+ */
131
+ export function isTurnActivityFenceCancellation(error: unknown): boolean {
132
+ if (isCancellation(error)) return true;
133
+ if (!(error instanceof ActivityFailure)) return false;
134
+ if (error.cause !== undefined && isCancellation(error.cause)) return true;
135
+ return (
136
+ error.activityType === "runAgentTurn" &&
137
+ error.cause instanceof ApplicationFailure &&
138
+ error.cause.type === "CancelledFailure" &&
139
+ error.cause.message === "TURN_ATTEMPT_FENCED"
140
+ );
141
+ }
142
+
101
143
  export const userMessage = defineSignal<[string]>("userMessage");
102
144
  export const queueChanged = defineSignal("queueChanged");
103
145
  export const approvalDecision = defineSignal<[string]>("approvalDecision");
104
146
  export const sessionControl = defineSignal("sessionControl");
105
147
  export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
148
+ export const sessionAttemptQuiesced =
149
+ defineSignal<[activities.SessionAttemptQuiescenceProof]>("sessionAttemptQuiesced");
106
150
 
107
151
  export type SessionWorkflowInput = {
108
152
  accountId: string;
@@ -120,13 +164,19 @@ export type SessionWorkflowInput = {
120
164
  };
121
165
 
122
166
  export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
123
- const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue);
167
+ // Existing histories recorded the v1 WAIT_CANCELLATION_COMPLETED command
168
+ // order and workflow-side idempotent fallback. Keep that exact replay path;
169
+ // every new run records v2 and uses the activity-owned receipt contract.
170
+ const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
171
+ const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
124
172
  let approvalWakeups = 0;
125
173
  let interruptionWakeups = 0;
126
174
  let wakeups = 0;
127
175
  let capacityWakeups = 0;
128
176
  let signalVersion = 0;
129
177
  let nonControlSignalVersion = 0;
178
+ const pendingQuiescenceProofs = new Map<string, activities.SessionAttemptQuiescenceProof>();
179
+ const persistedQuiescenceProofs = new Set<string>();
130
180
  // Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
131
181
  // for the history-overflow guard below; bounded growth is what makes a
132
182
  // weeks-long session survivable.
@@ -157,6 +207,38 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
157
207
  nonControlSignalVersion += 1;
158
208
  capacityWakeups += 1;
159
209
  });
210
+ setHandler(sessionAttemptQuiesced, (proof) => {
211
+ // Temporal signals are untrusted transport payloads. Accept only the exact
212
+ // workflow/session scope; the DB control activity additionally validates
213
+ // the persisted attempt's account, run id, and activity id under lock.
214
+ if (
215
+ !isSessionAttemptQuiescenceProof(proof) ||
216
+ proof.accountId !== input.accountId ||
217
+ proof.workspaceId !== input.workspaceId ||
218
+ proof.sessionId !== input.sessionId ||
219
+ proof.workflowId !== workflowInfo().workflowId
220
+ ) {
221
+ return;
222
+ }
223
+ const key = sessionAttemptQuiescenceProofKey(proof);
224
+ if (persistedQuiescenceProofs.has(key) || pendingQuiescenceProofs.has(key)) return;
225
+ pendingQuiescenceProofs.set(key, proof);
226
+ signalVersion += 1;
227
+ });
228
+
229
+ async function persistPendingQuiescenceProofs(): Promise<void> {
230
+ while (pendingQuiescenceProofs.size > 0) {
231
+ const entry = pendingQuiescenceProofs.entries().next().value;
232
+ if (!entry) return;
233
+ const [key, proof] = entry;
234
+ // This DB-only activity uses an unbounded Temporal retry policy. The
235
+ // workflow cannot peek, close, or continue-as-new until the exact
236
+ // physical proof commits its idempotent receipt and wake transaction.
237
+ await activity.persistSessionAttemptQuiescence(proof);
238
+ pendingQuiescenceProofs.delete(key);
239
+ persistedQuiescenceProofs.add(key);
240
+ }
241
+ }
160
242
 
161
243
  async function waitForCodexCapacity(
162
244
  initial: activities.CodexCapacityWaitRef,
@@ -217,6 +299,11 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
217
299
  // The waiter/outbox is durable in Postgres. A fresh workflow run reads
218
300
  // it before goal continuation, reconstructs its timer, and turns any
219
301
  // unobserved wake revision into an immediate evaluation.
302
+ // A quiescence proof is the one signal that is not merely a replaceable
303
+ // wake hint. It may have arrived while the reconciliation activity was
304
+ // running, so commit it before crossing this nested continue-as-new
305
+ // boundary.
306
+ await persistPendingQuiescenceProofs();
220
307
  await continueAsNew<typeof sessionWorkflow>({
221
308
  accountId: input.accountId,
222
309
  workspaceId: input.workspaceId,
@@ -232,18 +319,25 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
232
319
  }
233
320
 
234
321
  while (true) {
322
+ // A proof signal can signalWithStart a fresh run or arrive at any prior
323
+ // close/continue-as-new boundary. Persist it before every ordinary workflow
324
+ // decision so no accepted physical fact can be dropped with run history.
325
+ await persistPendingQuiescenceProofs();
235
326
  // History-overflow guard. The top of the loop is the only safe
236
327
  // continueAsNew boundary: no turn is mid-flight (every path that reaches a
237
328
  // new iteration settled or recovered its turn first). Every interruption
238
- // is already durable in Postgres, so all Temporal signals are replaceable
239
- // wake hints and none must be carried into the next workflow run. A
329
+ // is already durable in Postgres, so ordinary Temporal signals are
330
+ // replaceable wake hints and none must be carried into the next workflow
331
+ // run. A
240
332
  // buffered userMessage/queueChanged signal only
241
333
  // bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
242
334
  // was sent, so the fresh run observes it on its first durable work peek
243
335
  // — losing the counter strands nothing. The queue living in Postgres is the
244
336
  // safety net: continueAsNew carries only the self-contained
245
337
  // SessionWorkflowInput (no initialEventId — the new run claims from the
246
- // queue, it does not replay a seed event).
338
+ // queue, it does not replay a seed event). Quiescence-proof signals are the
339
+ // exception: persistPendingQuiescenceProofs() immediately above commits
340
+ // every accepted proof before this boundary.
247
341
  //
248
342
  // Approval signals are wakeups, never conversation truth. A genuinely
249
343
  // accepted decision is already persisted on the turn and is rediscovered
@@ -285,6 +379,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
285
379
  workflowId,
286
380
  });
287
381
  if (settlement.action === "paused") {
382
+ if (pendingQuiescenceProofs.size > 0) {
383
+ continue;
384
+ }
288
385
  if (nonControlSignalVersion !== closeNonControlSignalVersion) {
289
386
  continue;
290
387
  }
@@ -292,6 +389,22 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
292
389
  }
293
390
  continue;
294
391
  }
392
+ if (peek.kind === "cancellation-wait") {
393
+ // Logical settlement is complete, but the exact predecessor activity has
394
+ // not yet durably proved sandbox/tool quiescence. Wait only briefly for
395
+ // its transactional queueChanged wake. If provider/tool cancellation is
396
+ // genuinely slow, close this workflow run rather than consuming a turn
397
+ // slot or churning control activities; the outbox uses signalWithStart to
398
+ // restart this exact workflow after the receipt commits.
399
+ const seenSignalVersion = signalVersion;
400
+ const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
401
+ if (woke) continue;
402
+ // Close only against the same signal snapshot. A proof signal accepted
403
+ // at the timer/completion boundary must loop through the DB-only receipt
404
+ // activity rather than disappearing with this workflow run.
405
+ if (signalVersion !== seenSignalVersion || pendingQuiescenceProofs.size > 0) continue;
406
+ return;
407
+ }
295
408
  if (peek.kind === "capacity-wait") {
296
409
  await waitForCodexCapacity(peek.ref);
297
410
  continue;
@@ -300,12 +413,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
300
413
  const seenApprovalWakeups = approvalWakeups;
301
414
  const seenWakeups = wakeups;
302
415
  const seenInterruptionWakeups = interruptionWakeups;
303
- await condition(
304
- () =>
305
- interruptionWakeups !== seenInterruptionWakeups ||
306
- approvalWakeups !== seenApprovalWakeups ||
307
- wakeups !== seenWakeups,
308
- );
416
+ const timeoutMs =
417
+ peek.humanInputRequestId && peek.expiresAt
418
+ ? humanInputDeadlineWaitMs(peek.expiresAt)
419
+ : undefined;
420
+ const wakeCondition = () =>
421
+ interruptionWakeups !== seenInterruptionWakeups ||
422
+ approvalWakeups !== seenApprovalWakeups ||
423
+ wakeups !== seenWakeups;
424
+ const woke =
425
+ timeoutMs === undefined
426
+ ? await condition(wakeCondition)
427
+ : await condition(wakeCondition, timeoutMs);
428
+ if (!woke && peek.humanInputRequestId) {
429
+ const expiry = await activity.expireSessionHumanInput({
430
+ accountId: input.accountId,
431
+ workspaceId: input.workspaceId,
432
+ sessionId: input.sessionId,
433
+ requestId: peek.humanInputRequestId,
434
+ });
435
+ // Temporal and PostgreSQL may observe slightly different wall clocks.
436
+ // If the workflow timer fired first, the DB settler truthfully leaves
437
+ // the request pending. Bound that skew with one interruptible timer
438
+ // instead of spinning peek + activity at zero delay.
439
+ if (expiry.action === "stale") {
440
+ await condition(wakeCondition, HUMAN_INPUT_EXPIRY_STALE_RETRY_MS);
441
+ }
442
+ }
309
443
  continue;
310
444
  }
311
445
  if (peek.kind === "idle") {
@@ -389,7 +523,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
389
523
  trigger,
390
524
  }),
391
525
  );
392
- const outcome:
526
+ const racedOutcome:
393
527
  | { kind: "result"; result: activities.RunAgentTurnResult }
394
528
  | { kind: "control" }
395
529
  | { kind: "failure"; error: unknown } = await Promise.race([
@@ -404,9 +538,55 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
404
538
  kind: "control" as const,
405
539
  })),
406
540
  ]);
541
+ // A Steer/Pause transaction fences the active attempt before its Temporal
542
+ // signal is necessarily handled. That fence can make the activity's typed
543
+ // cancellation win Promise.race by one workflow activation. Give only that
544
+ // confirmed cancellation shape a short deterministic arbitration window;
545
+ // the durable control signal is authoritative once observed. Ordinary
546
+ // failures and timeouts never wait here, and a cancellation with no session
547
+ // control still follows the failure/cancellation path below.
548
+ if (
549
+ racedOutcome.kind === "failure" &&
550
+ isTurnActivityFenceCancellation(racedOutcome.error) &&
551
+ interruptionWakeups === interruptionBaseline
552
+ ) {
553
+ await condition(() => interruptionWakeups !== interruptionBaseline, "250ms");
554
+ }
555
+ const outcome =
556
+ interruptionWakeups !== interruptionBaseline ? ({ kind: "control" } as const) : racedOutcome;
407
557
 
408
558
  if (outcome.kind === "control") {
409
- scope.cancel();
559
+ if (!receiptGatedCancellation) {
560
+ // Replay-only v1 command order. New histories always take the v2 path
561
+ // below. The current runAgentTurn still owns and writes the truthful
562
+ // hard-fence receipt; this call remains an idempotent history command.
563
+ scope.cancel();
564
+ const settlement = await activity.settleSessionInterruptions({
565
+ accountId,
566
+ workspaceId,
567
+ sessionId,
568
+ attemptId,
569
+ workflowId: workflowInfo().workflowId,
570
+ });
571
+ const termination = await turn.then(
572
+ () => ({ kind: "completed" as const }),
573
+ (error: unknown) => ({ kind: "failed" as const, error }),
574
+ );
575
+ const physicalStopConfirmed =
576
+ termination.kind === "completed" || isTurnActivityFenceCancellation(termination.error);
577
+ if (patched("session-attempt-quiescence-v1") && physicalStopConfirmed) {
578
+ await activity.settleSessionInterruptions({
579
+ accountId,
580
+ workspaceId,
581
+ sessionId,
582
+ attemptId,
583
+ workflowId: workflowInfo().workflowId,
584
+ phase: "attempt_quiesced",
585
+ });
586
+ }
587
+ if (!physicalStopConfirmed) throw termination.error;
588
+ return settlement.action !== "paused";
589
+ }
410
590
  const settlement = await activity.settleSessionInterruptions({
411
591
  accountId,
412
592
  workspaceId,
@@ -414,16 +594,12 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
414
594
  attemptId,
415
595
  workflowId: workflowInfo().workflowId,
416
596
  });
417
- // Keep the workflow alive until the cancelled activity has actually
418
- // stopped. Temporal delivers activity cancellation through a heartbeat;
419
- // returning while the activity promise is still detached lets the
420
- // workflow complete first, after which the worker can keep streaming for
421
- // the activity's entire start-to-close window. Postgres settlement above
422
- // is intentionally first: it fences every late write immediately while
423
- // the physical activity winds down. The control outcome is authoritative,
424
- // so a concurrent activity completion or cancellation failure is ignored
425
- // only after Temporal has durably observed that terminal activity state.
426
- await turn.catch(() => undefined);
597
+ // The transaction above is the authority fence: every late model/tool/UI
598
+ // write is rejected from this point onward. Request cancellation only
599
+ // after it commits, then stop observing the Temporal activity promise.
600
+ // TRY_CANCEL allows this workflow to move to cancellation-wait without
601
+ // confusing Temporal completion/cancellation with local process safety.
602
+ scope.cancel();
427
603
  return settlement.action !== "paused";
428
604
  }
429
605
 
@@ -490,7 +666,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
490
666
  }
491
667
 
492
668
  if (outcome.result.deferredUntilWake) {
493
- return wakeups !== capacityWaitEntryBaseline.wakeups;
669
+ return deferredResultMayContinue(capacityWaitEntryBaseline.wakeups, wakeups);
494
670
  }
495
671
 
496
672
  if (outcome.result.status === "requires_action") return true;
@@ -512,3 +688,23 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
512
688
  return true;
513
689
  }
514
690
  }
691
+
692
+ function isSessionAttemptQuiescenceProof(
693
+ value: unknown,
694
+ ): value is activities.SessionAttemptQuiescenceProof {
695
+ if (!value || typeof value !== "object") return false;
696
+ const proof = value as Record<string, unknown>;
697
+ return [
698
+ "accountId",
699
+ "workspaceId",
700
+ "sessionId",
701
+ "attemptId",
702
+ "workflowId",
703
+ "workflowRunId",
704
+ "activityId",
705
+ ].every((field) => typeof proof[field] === "string" && proof[field].length > 0);
706
+ }
707
+
708
+ function sessionAttemptQuiescenceProofKey(proof: activities.SessionAttemptQuiescenceProof): string {
709
+ return `${proof.attemptId}:${proof.workflowRunId}:${proof.activityId}`;
710
+ }
package/src/workflows.ts CHANGED
@@ -3,6 +3,7 @@ export {
3
3
  approvalDecision,
4
4
  codexCapacityChanged,
5
5
  sessionControl,
6
+ sessionAttemptQuiesced,
6
7
  queueChanged,
7
8
  sessionWorkflow,
8
9
  userMessage,