@opengeni/worker-bundle 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7127 -1654
  3. package/dist/index.js.map +1 -1
  4. package/package.json +19 -19
  5. package/src/activities/agent-turn.ts +4405 -1089
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
@@ -1,6 +1,22 @@
1
- import { ActivityFailure, CancellationScope, condition, continueAsNew, defineSignal, isCancellation, patched, setHandler, TimeoutFailure, workflowInfo } from "@temporalio/workflow";
1
+ import {
2
+ ActivityFailure,
3
+ CancellationScope,
4
+ condition,
5
+ continueAsNew,
6
+ defineSignal,
7
+ isCancellation,
8
+ setHandler,
9
+ TimeoutFailure,
10
+ uuid4,
11
+ workflowInfo,
12
+ } from "@temporalio/workflow";
2
13
  import type * as activities from "../activities";
3
- import { activity, workflowFailureMessage } from "./activities";
14
+ import {
15
+ activity,
16
+ goalActivity,
17
+ turnActivityForTaskQueue,
18
+ workflowFailureMessage,
19
+ } from "./activities";
4
20
 
5
21
  /**
6
22
  * Deterministic backstop for continueAsNew. A session workflow is long-lived
@@ -16,6 +32,7 @@ import { activity, workflowFailureMessage } from "./activities";
16
32
  * while keeping continueAsNew rare enough that it is not a per-turn cost.
17
33
  */
18
34
  const TURNS_PER_RUN_BACKSTOP = 2_000;
35
+ const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
19
36
 
20
37
  /**
21
38
  * The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
@@ -51,7 +68,7 @@ export function continuationHoldMs(
51
68
  * True when an agent-turn activity failure means "the worker hosting the
52
69
  * turn died or vanished" rather than "the turn itself failed": the server
53
70
  * closed the activity with a HEARTBEAT timeout (the worker was killed before
54
- * the graceful-preempt checkpoint could run — SIGKILL, OOM, node loss, or a
71
+ * the graceful recovery checkpoint could run — SIGKILL, OOM, node loss, or a
55
72
  * rollout whose grace period expired) or a SCHEDULE_TO_START timeout (no
56
73
  * worker ever picked the task up). Detection uses the SDK's typed failure
57
74
  * classes, not message-string matching: the failure converter rehydrates
@@ -61,19 +78,27 @@ export function continuationHoldMs(
61
78
  * SCHEDULE_TO_CLOSE timeouts are deliberately excluded: with the 30-day
62
79
  * startToClose they mean the turn truly overran, which stays a real failure.
63
80
  */
64
- function isWorkerDeathFailure(error: unknown): boolean {
81
+ function workerDeathFailure(
82
+ error: unknown,
83
+ ): { timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START" } | null {
65
84
  if (!(error instanceof ActivityFailure)) {
66
- return false;
85
+ return null;
67
86
  }
68
87
  const cause = error.cause;
69
- return cause instanceof TimeoutFailure
70
- && (cause.timeoutType === "HEARTBEAT" || cause.timeoutType === "SCHEDULE_TO_START");
88
+ if (
89
+ !(cause instanceof TimeoutFailure) ||
90
+ (cause.timeoutType !== "HEARTBEAT" && cause.timeoutType !== "SCHEDULE_TO_START")
91
+ ) {
92
+ return null;
93
+ }
94
+ return { timeoutType: cause.timeoutType };
71
95
  }
72
96
 
73
97
  export const userMessage = defineSignal<[string]>("userMessage");
74
98
  export const queueChanged = defineSignal("queueChanged");
75
99
  export const approvalDecision = defineSignal<[string]>("approvalDecision");
76
- export const interrupt = defineSignal<[string]>("interrupt");
100
+ export const sessionControl = defineSignal<[string]>("sessionControl");
101
+ export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
77
102
 
78
103
  export type SessionWorkflowInput = {
79
104
  accountId: string;
@@ -85,291 +110,423 @@ export type SessionWorkflowInput = {
85
110
  // the boundary without simulating thousands of turns. Never gates correctness
86
111
  // — continueAsNewSuggested is the real-world trigger.
87
112
  maxTurnsPerRun?: number;
113
+ // Test-only override for the durable capacity-wait continue-as-new
114
+ // backstop. Production uses CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP.
115
+ maxCapacityChecksPerRun?: number;
88
116
  };
89
117
 
90
118
  export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
91
- const approvalQueue: string[] = [];
92
- let interruptedEventId: string | null = null;
119
+ const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue);
120
+ let approvalWakeups = 0;
121
+ let controlEventId: string | null = null;
93
122
  let wakeups = 0;
123
+ let capacityWakeups = 0;
124
+ let signalVersion = 0;
125
+ let nonControlSignalVersion = 0;
94
126
  // Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
95
127
  // for the history-overflow guard below; bounded growth is what makes a
96
128
  // weeks-long session survivable.
97
129
  let turnsThisRun = 0;
130
+ let capacityChecksThisRun = 0;
98
131
 
99
132
  setHandler(userMessage, () => {
133
+ signalVersion += 1;
134
+ nonControlSignalVersion += 1;
100
135
  wakeups += 1;
101
136
  });
102
137
  setHandler(queueChanged, () => {
138
+ signalVersion += 1;
139
+ nonControlSignalVersion += 1;
103
140
  wakeups += 1;
104
141
  });
105
- setHandler(approvalDecision, (eventId) => {
106
- approvalQueue.push(eventId);
142
+ setHandler(approvalDecision, () => {
143
+ signalVersion += 1;
144
+ nonControlSignalVersion += 1;
145
+ approvalWakeups += 1;
146
+ });
147
+ setHandler(sessionControl, (eventId) => {
148
+ signalVersion += 1;
149
+ controlEventId = eventId;
107
150
  });
108
- setHandler(interrupt, (eventId) => {
109
- interruptedEventId = eventId;
151
+ setHandler(codexCapacityChanged, () => {
152
+ signalVersion += 1;
153
+ nonControlSignalVersion += 1;
154
+ capacityWakeups += 1;
110
155
  });
111
156
 
157
+ async function waitForCodexCapacity(
158
+ initial: activities.CodexCapacityWaitRef,
159
+ entryBaseline?: { wakeups: number; capacityWakeups: number },
160
+ ): Promise<void> {
161
+ let current = initial;
162
+ let firstEntryBaseline = entryBaseline;
163
+ for (;;) {
164
+ if (controlEventId !== null) {
165
+ return;
166
+ }
167
+ // Signals can land after the waiter commit but before runAgentTurn returns.
168
+ // Compare the first wait against pre-dispatch counters so they cannot be
169
+ // baselined away; later iterations use their normal local snapshot.
170
+ const seenWakeups = firstEntryBaseline?.wakeups ?? wakeups;
171
+ const seenCapacityWakeups = firstEntryBaseline?.capacityWakeups ?? capacityWakeups;
172
+ firstEntryBaseline = undefined;
173
+ const parsedDeadline = Date.parse(current.nextCheckAt);
174
+ const timerMs = Number.isFinite(parsedDeadline)
175
+ ? Math.max(0, parsedDeadline - Date.now())
176
+ : 0;
177
+ let cause: activities.ReconcileCodexCapacityWaitInput["cause"] = "timer";
178
+ if (wakeups !== seenWakeups) {
179
+ cause = "queue";
180
+ } else if (capacityWakeups !== seenCapacityWakeups) {
181
+ cause = "signal";
182
+ } else if (timerMs > 0) {
183
+ await condition(
184
+ () =>
185
+ controlEventId !== null ||
186
+ wakeups !== seenWakeups ||
187
+ capacityWakeups !== seenCapacityWakeups,
188
+ timerMs,
189
+ );
190
+ if (controlEventId !== null) {
191
+ return;
192
+ }
193
+ cause =
194
+ wakeups !== seenWakeups
195
+ ? "queue"
196
+ : capacityWakeups !== seenCapacityWakeups
197
+ ? "signal"
198
+ : "timer";
199
+ }
200
+ const result = await activity.reconcileCodexCapacityWait({
201
+ accountId: input.accountId,
202
+ workspaceId: input.workspaceId,
203
+ sessionId: input.sessionId,
204
+ waiterId: current.waiterId,
205
+ generation: current.generation,
206
+ cause,
207
+ });
208
+ if (result.action !== "waiting") {
209
+ return;
210
+ }
211
+ capacityChecksThisRun += 1;
212
+ const capacityCheckBackstop =
213
+ input.maxCapacityChecksPerRun ?? CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP;
214
+ if (
215
+ controlEventId === null &&
216
+ (workflowInfo().continueAsNewSuggested || capacityChecksThisRun >= capacityCheckBackstop)
217
+ ) {
218
+ // The waiter/outbox is durable in Postgres. A fresh workflow run reads
219
+ // it before goal continuation, reconstructs its timer, and turns any
220
+ // unobserved wake revision into an immediate evaluation.
221
+ await continueAsNew<typeof sessionWorkflow>({
222
+ accountId: input.accountId,
223
+ workspaceId: input.workspaceId,
224
+ sessionId: input.sessionId,
225
+ ...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
226
+ ...(input.maxCapacityChecksPerRun !== undefined
227
+ ? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
228
+ : {}),
229
+ });
230
+ }
231
+ current = result;
232
+ }
233
+ }
234
+
112
235
  while (true) {
113
236
  // History-overflow guard. The top of the loop is the only safe
114
237
  // continueAsNew boundary: no turn is mid-flight (every path that reaches a
115
- // new iteration completed or re-queued its turn first), and a pending
116
- // interrupt is unbacked in-memory state that the new run could not
238
+ // new iteration settled or recovered its turn first), and a pending
239
+ // control signal is unbacked in-memory state that the new run could not
117
240
  // reconstruct — so we refuse to continueAsNew while one is set and let the
118
241
  // loop handle it first. A buffered userMessage/queueChanged signal only
119
242
  // bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
120
- // was sent, so the fresh run re-claims it on its first claimNextQueuedTurn
243
+ // was sent, so the fresh run observes it on its first durable work peek
121
244
  // — losing the counter strands nothing. The queue living in Postgres is the
122
245
  // safety net: continueAsNew carries only the self-contained
123
246
  // SessionWorkflowInput (no initialEventId — the new run claims from the
124
- // queue, it does not replay a seed event). patched() keeps in-flight
125
- // histories (recorded before this branch existed) deterministic: they never
126
- // see the new command.
247
+ // queue, it does not replay a seed event).
127
248
  //
128
- // The approval queue is NOT a boundary condition, and is cleared here. A
129
- // genuinely-pending approval keeps the workflow blocked INSIDE runTurn (the
130
- // `await condition(...)` in the requires_action branch), so it never
131
- // reaches the top of the loop — meaning every approvalQueue entry observed
132
- // here is necessarily STALE (an approvalDecision signal for a turn that
133
- // already settled; the API guard only checks status==='requires_action', so
134
- // two decisions submitted while requires_action both land in the queue and
135
- // the surplus is left behind once the turn completes without re-blocking).
136
- // Coupling continueAsNew to `approvalQueue.length === 0` would let one such
137
- // stale entry wedge the guard forever, re-introducing the exact
138
- // history-overflow termination this branch prevents. Dropping the surplus
139
- // is safe: a real pending approval also leaves the session in
140
- // requires_action with the turn re-dispatchable from Postgres.
141
- if (patched("session-continue-as-new")) {
249
+ // Approval signals are wakeups, never conversation truth. A genuinely
250
+ // accepted decision is already persisted on the turn and is rediscovered
251
+ // by the next durable peek, including after continue-as-new. Stale or
252
+ // duplicate signals therefore cannot block this boundary or manufacture a
253
+ // second approval dispatch.
254
+ {
142
255
  const info = workflowInfo();
143
256
  const maxTurnsPerRun = input.maxTurnsPerRun ?? TURNS_PER_RUN_BACKSTOP;
144
257
  const shouldContinue = info.continueAsNewSuggested || turnsThisRun >= maxTurnsPerRun;
145
- if (shouldContinue && interruptedEventId === null) {
146
- approvalQueue.length = 0;
258
+ if (shouldContinue && controlEventId === null) {
147
259
  await continueAsNew<typeof sessionWorkflow>({
148
260
  accountId: input.accountId,
149
261
  workspaceId: input.workspaceId,
150
262
  sessionId: input.sessionId,
151
263
  ...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
264
+ ...(input.maxCapacityChecksPerRun !== undefined
265
+ ? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
266
+ : {}),
152
267
  });
153
268
  }
154
269
  }
270
+ // Capture before the final activity chain of this cycle. Temporal may
271
+ // accept a signal while an activity completion and workflow completion
272
+ // race; every terminal return below must observe that arrival and loop.
273
+ const closeSignalVersion = signalVersion;
274
+ const closeNonControlSignalVersion = nonControlSignalVersion;
155
275
  const workflowId = workflowInfo().workflowId;
156
- const turn = await activity.claimNextQueuedTurn({ workspaceId: input.workspaceId, sessionId: input.sessionId, workflowId });
157
- if (!turn) {
158
- if (interruptedEventId === null) {
159
- // With an active goal, idling out is replaced by a synthesized
160
- // continuation turn; the queue (any non-terminal turn) always wins and
161
- // the no-progress/max-continuation guards auto-pause runaway goals.
276
+ const peek = await activity.peekSessionWork({
277
+ workspaceId: input.workspaceId,
278
+ sessionId: input.sessionId,
279
+ });
280
+ if (peek.kind === "fence-settle") {
281
+ const settlement = await activity.settleSessionControl({
282
+ accountId: input.accountId,
283
+ workspaceId: input.workspaceId,
284
+ sessionId: input.sessionId,
285
+ triggerEventId: peek.controlEventId,
286
+ workflowId,
287
+ });
288
+ if (controlEventId === peek.controlEventId) controlEventId = null;
289
+ if (settlement.action === "paused") {
290
+ // The matching sessionControl signal is consumed by this exact durable
291
+ // settlement. It must not manufacture a second loop; only later work
292
+ // or a different control keeps the workflow alive.
293
+ if (nonControlSignalVersion !== closeNonControlSignalVersion || controlEventId !== null) {
294
+ continue;
295
+ }
296
+ return;
297
+ }
298
+ continue;
299
+ }
300
+ // A duplicate/repaired control signal may arrive after its durable fence
301
+ // already settled. Explicitly classify it through the same settlement
302
+ // activity and clear the in-memory signal so it cannot pin this workflow
303
+ // run forever or block continue-as-new.
304
+ if (controlEventId !== null) {
305
+ const signaledControlEventId = controlEventId;
306
+ const settlement = await activity.settleSessionControl({
307
+ accountId: input.accountId,
308
+ workspaceId: input.workspaceId,
309
+ sessionId: input.sessionId,
310
+ triggerEventId: signaledControlEventId,
311
+ workflowId,
312
+ });
313
+ if (controlEventId === signaledControlEventId) controlEventId = null;
314
+ if (settlement.action === "paused") {
315
+ if (nonControlSignalVersion !== closeNonControlSignalVersion || controlEventId !== null) {
316
+ continue;
317
+ }
318
+ return;
319
+ }
320
+ continue;
321
+ }
322
+ if (peek.kind === "capacity-wait") {
323
+ await waitForCodexCapacity(peek.ref);
324
+ continue;
325
+ }
326
+ if (peek.kind === "approval-wait") {
327
+ const seenApprovalWakeups = approvalWakeups;
328
+ const seenWakeups = wakeups;
329
+ await condition(
330
+ () =>
331
+ controlEventId !== null ||
332
+ approvalWakeups !== seenApprovalWakeups ||
333
+ wakeups !== seenWakeups,
334
+ );
335
+ continue;
336
+ }
337
+ if (peek.kind === "idle") {
338
+ if (controlEventId === null) {
162
339
  let continuation: activities.MaybeContinueGoalResult = { action: "none" };
163
340
  try {
164
- continuation = await activity.maybeContinueGoal({
341
+ continuation = await goalActivity.maybeContinueGoal({
165
342
  accountId: input.accountId,
166
343
  workspaceId: input.workspaceId,
167
344
  sessionId: input.sessionId,
168
345
  workflowId,
169
346
  });
170
347
  } catch (error) {
171
- if (isCancellation(error)) {
172
- throw error;
173
- }
174
- // A failing goal check must not kill the session (activity retry is
175
- // 1). Fall through to the idle path: the goal stays active in the
176
- // database and the next wake retries, which means the workflow CAN
177
- // complete normally with an active goal on this error path.
178
- }
179
- if (continuation.action === "continue" || continuation.action === "queue") {
180
- // "continue": a goal continuation turn was just enqueued; "queue":
181
- // queued work appeared concurrently. Claim it on the next loop pass.
182
- continue;
348
+ if (isCancellation(error)) throw error;
349
+ await activity.enqueueGoalRetryWake({
350
+ accountId: input.accountId,
351
+ workspaceId: input.workspaceId,
352
+ sessionId: input.sessionId,
353
+ workflowId,
354
+ });
183
355
  }
356
+ if (continuation.action === "continue" || continuation.action === "queue") continue;
184
357
  }
185
358
  const seenWakeups = wakeups;
186
- const woke = await condition(() => interruptedEventId !== null || wakeups !== seenWakeups, "5s");
187
- if (interruptedEventId) {
188
- const idleInterruptEventId = interruptedEventId;
189
- interruptedEventId = null;
190
- // An idle interrupt is an explicit stop: pause an active goal before
191
- // shutting down so a later wake does not auto-continue it. The
192
- // activity inspects the trigger and skips the pause for steer-tagged
193
- // interrupts (steering redirects work, it does not stop it).
194
- await activity.pauseGoalForInterrupt({ workspaceId: input.workspaceId, sessionId: input.sessionId, triggerEventId: idleInterruptEventId });
195
- await activity.markSessionIdle({ workspaceId: input.workspaceId, sessionId: input.sessionId });
196
- return;
197
- }
198
- if (woke) {
359
+ const seenApprovalWakeups = approvalWakeups;
360
+ const woke = await condition(
361
+ () =>
362
+ controlEventId !== null ||
363
+ wakeups !== seenWakeups ||
364
+ approvalWakeups !== seenApprovalWakeups,
365
+ "5s",
366
+ );
367
+ if (woke) continue;
368
+ const finalPeek = await activity.peekSessionWork({
369
+ workspaceId: input.workspaceId,
370
+ sessionId: input.sessionId,
371
+ });
372
+ if (finalPeek.kind !== "idle") continue;
373
+ await activity.markSessionIdle({
374
+ workspaceId: input.workspaceId,
375
+ sessionId: input.sessionId,
376
+ });
377
+ if (signalVersion !== closeSignalVersion) {
199
378
  continue;
200
379
  }
201
- const finalTurn = await activity.claimNextQueuedTurn({ workspaceId: input.workspaceId, sessionId: input.sessionId, workflowId });
202
- if (!finalTurn) {
203
- await activity.markSessionIdle({ workspaceId: input.workspaceId, sessionId: input.sessionId });
204
- // A queueChanged/userMessage signal can land between the final claim and
205
- // completion; Temporal blocks completion while a signal is buffered, so
206
- // re-checking here guarantees the queued turn is picked up instead of
207
- // stranding it (the signaler skips its start-child fallback on success).
208
- if (interruptedEventId !== null || wakeups !== seenWakeups) {
209
- continue;
210
- }
211
- return;
212
- }
213
- turnsThisRun += 1;
214
- if (!await runTurn(input.accountId, input.workspaceId, input.sessionId, finalTurn.id, finalTurn.triggerEventId)) {
215
- return;
216
- }
217
- continue;
380
+ return;
218
381
  }
219
382
  turnsThisRun += 1;
220
- if (!await runTurn(input.accountId, input.workspaceId, input.sessionId, turn.id, turn.triggerEventId)) {
383
+ const trigger =
384
+ peek.kind === "approval-pending"
385
+ ? ({ kind: "approval", triggerEventId: peek.triggerEventId } as const)
386
+ : ({ kind: "next" } as const);
387
+ if (!(await runTurn(input.accountId, input.workspaceId, input.sessionId, trigger))) {
388
+ if (signalVersion !== closeSignalVersion) continue;
221
389
  return;
222
390
  }
223
391
  }
224
392
 
225
- async function runTurn(accountId: string, workspaceId: string, sessionId: string, turnId: string, triggerEventId: string): Promise<boolean> {
226
- if (interruptedEventId) {
227
- await activity.interruptActiveTurn({
393
+ async function runTurn(
394
+ accountId: string,
395
+ workspaceId: string,
396
+ sessionId: string,
397
+ trigger: activities.RunAgentTurnInput["trigger"],
398
+ ): Promise<boolean> {
399
+ if (controlEventId) {
400
+ const settlement = await activity.settleSessionControl({
228
401
  accountId,
229
402
  workspaceId,
230
403
  sessionId,
231
- triggerEventId: interruptedEventId,
404
+ triggerEventId: controlEventId,
232
405
  workflowId: workflowInfo().workflowId,
233
406
  });
234
- interruptedEventId = null;
235
- return true;
407
+ controlEventId = null;
408
+ return settlement.action !== "paused";
236
409
  }
237
410
 
411
+ const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
412
+ const attemptId = uuid4();
413
+
238
414
  const scope = new CancellationScope();
239
415
  const workflowId = workflowInfo().workflowId;
240
- // P1.2 STATELESS-LEASE GATE. The stateless resume-by-id model (lease acquire
241
- // + non-owned injection + release) lives ENTIRELY in the runAgentSegment
242
- // activity, gated by the sandboxOwnershipEnabled config flag — the workflow's
243
- // command path is intentionally unchanged: the turn still dispatches on the
244
- // EXISTING GLOBAL queue (the module-level `activity` proxy, NO taskQueue
245
- // override, NO per-session worker, NO per-session queue), worker-death still
246
- // re-dispatches via the `worker-death-redispatch` patch below (any worker
247
- // re-resumes the box by id), and there is NO ownerHeartbeat / openStreamRequest
248
- // / closeStreamRequest signal. This patched() marker records the
249
- // stateless-lease cutover point in history so multi-day IN-FLIGHT workflow
250
- // histories replay deterministically on the original lease-less path, and any
251
- // future workflow-side lease change can branch on it without breaking replay.
252
- const statelessLease = patched("sandbox-stateless-lease");
253
- void statelessLease; // command path unchanged today (see comment above)
254
- // Deliberately schedules the LEGACY activity name: in-flight session
255
- // workflows (multi-day by design) recorded this name in their histories,
256
- // and scheduling a different activity type at the same position is a
257
- // non-deterministic change on replay. Migrate to runAgentTurn with
258
- // patched() once pre-rename sessions have drained.
259
- const turn: Promise<activities.RunAgentTurnResult> = scope.run(() => activity.runAgentSegment({
260
- accountId,
261
- workspaceId,
262
- sessionId,
263
- triggerEventId,
264
- workflowId,
265
- turnId,
266
- }));
267
- const outcome: { kind: "result"; result: activities.RunAgentTurnResult } | { kind: "interrupt" } | { kind: "failure"; error: unknown } = await Promise.race([
416
+ // The stateless resume-by-id model (lease acquire + non-owned injection +
417
+ // release) lives entirely in the runAgentTurn activity.
418
+ const turn: Promise<activities.RunAgentTurnResult> = scope.run(() =>
419
+ turnActivity.runAgentTurn({
420
+ accountId,
421
+ workspaceId,
422
+ sessionId,
423
+ workflowId,
424
+ attemptId,
425
+ trigger,
426
+ }),
427
+ );
428
+ const outcome:
429
+ | { kind: "result"; result: activities.RunAgentTurnResult }
430
+ | { kind: "control" }
431
+ | { kind: "failure"; error: unknown } = await Promise.race([
268
432
  turn.then(
269
433
  (result: activities.RunAgentTurnResult) => ({ kind: "result" as const, result }),
270
434
  (error: unknown) => ({ kind: "failure" as const, error }),
271
435
  ),
272
- condition(() => interruptedEventId !== null).then(() => ({ kind: "interrupt" as const })),
436
+ condition(() => controlEventId !== null).then(() => ({ kind: "control" as const })),
273
437
  ]);
274
438
 
275
- if (outcome.kind === "interrupt") {
439
+ if (outcome.kind === "control") {
276
440
  scope.cancel();
277
- await activity.interruptActiveTurn({
441
+ const settlement = await activity.settleSessionControl({
278
442
  accountId,
279
443
  workspaceId,
280
444
  sessionId,
281
- triggerEventId: interruptedEventId!,
445
+ triggerEventId: controlEventId!,
282
446
  workflowId: workflowInfo().workflowId,
283
447
  });
284
- interruptedEventId = null;
285
- return true;
448
+ controlEventId = null;
449
+ return settlement.action !== "paused";
286
450
  }
287
451
 
288
452
  if (outcome.kind === "failure") {
453
+ // A capacity settlement may have committed just before the activity
454
+ // transport/worker failed. Recover that durable boundary before generic
455
+ // failSession can overwrite the capacity-idle session.
456
+ {
457
+ const capacityWait = await activity.getCodexCapacityWait({ workspaceId, sessionId });
458
+ if (capacityWait) {
459
+ await waitForCodexCapacity(capacityWait, capacityWaitEntryBaseline);
460
+ return true;
461
+ }
462
+ }
289
463
  // An ungraceful worker death never reaches the activity's graceful
290
- // preemption path — it surfaces here as a heartbeat-timeout failure.
464
+ // recovery path — it surfaces here as a heartbeat-timeout failure.
291
465
  // Conversation truth was still dual-written during the turn, so the
292
- // turn is re-queued (resume notice when it persisted progress, original
293
- // trigger otherwise) and the loop re-claims it on a healthy worker —
466
+ // same turn is marked recovering and the loop re-claims it on a healthy worker —
294
467
  // bounded by a per-turn redispatch counter persisted on the turn row.
295
- // patched() keeps replays of histories recorded before this branch
296
- // existed on their original failSession path.
297
- if (isWorkerDeathFailure(outcome.error) && patched("worker-death-redispatch")) {
298
- const requeue = await activity.requeueTurnAfterWorkerDeath({
468
+ const workerDeath = workerDeathFailure(outcome.error);
469
+ if (workerDeath) {
470
+ const recovery = await activity.recoverDispatch({
299
471
  accountId,
300
472
  workspaceId,
301
473
  sessionId,
302
- triggerEventId,
303
- workflowId,
304
- turnId,
474
+ attemptId,
475
+ timeoutType: workerDeath.timeoutType,
305
476
  });
306
- if (requeue.action !== "exceeded") {
307
- // "requeued": the next claim re-dispatches the turn. "stale": the
477
+ if (recovery.action !== "exceeded") {
478
+ // "recovering": the next claim creates a new attempt for this same
479
+ // current inference. "stale": the
308
480
  // timed-out attempt actually settled the turn (a zombie finished
309
481
  // after the server gave up on its heartbeats); nothing to redo.
310
482
  return true;
311
483
  }
312
- await activity.failSession({
313
- accountId,
314
- workspaceId,
315
- sessionId,
316
- triggerEventId,
317
- workflowId,
318
- error: `Worker died ${requeue.redispatches + 1} times while running this turn (heartbeat timeout); giving up after ${requeue.redispatches} re-dispatches.`,
319
- });
484
+ // The worker-death activity atomically committed failed turn/session
485
+ // truth when the bounded redispatch ceiling was exceeded.
320
486
  return false;
321
487
  }
322
- await activity.failSession({
488
+ await activity.failSessionAttempt({
323
489
  accountId,
324
490
  workspaceId,
325
491
  sessionId,
326
- triggerEventId,
327
- workflowId,
492
+ attemptId,
328
493
  error: workflowFailureMessage(outcome.error),
329
494
  });
330
495
  return false;
331
496
  }
332
497
 
498
+ if (outcome.result.status === "unclaimed") {
499
+ return true;
500
+ }
501
+
333
502
  if (outcome.result.status === "failed" || outcome.result.status === "cancelled") {
334
503
  return outcome.result.status === "cancelled";
335
504
  }
336
505
 
337
- if (outcome.result.status === "preempted") {
338
- // The hosting worker shut down gracefully mid-turn: the activity
339
- // checkpointed conversation truth and put the turn back on the queue
340
- // before completing. Loop again so the next claim re-dispatches it on a
341
- // healthy worker (a pending interrupt is honored first by the loop).
506
+ if (outcome.result.status === "recovering") {
342
507
  return true;
343
508
  }
344
509
 
345
- if (outcome.result.status === "requires_action") {
346
- await condition(() => interruptedEventId !== null || approvalQueue.length > 0);
347
- if (interruptedEventId) {
348
- await activity.interruptActiveTurn({
349
- accountId,
350
- workspaceId,
351
- sessionId,
352
- triggerEventId: interruptedEventId,
353
- workflowId,
354
- });
355
- interruptedEventId = null;
356
- return true;
357
- }
358
- const approvalEventId = approvalQueue.shift();
359
- if (approvalEventId) {
360
- return await runTurn(accountId, workspaceId, sessionId, turnId, approvalEventId);
361
- }
510
+ if (outcome.result.capacityWait) {
511
+ await waitForCodexCapacity(outcome.result.capacityWait, capacityWaitEntryBaseline);
512
+ return true;
362
513
  }
363
514
 
515
+ if (outcome.result.deferredUntilWake) {
516
+ return wakeups !== capacityWaitEntryBaseline.wakeups;
517
+ }
518
+
519
+ if (outcome.result.status === "requires_action") return true;
520
+
364
521
  const holdMs = continuationHoldMs(outcome.result, ROTATION_IDLE_FLOOR_MS);
365
522
  if (holdMs > 0) {
366
523
  // Provider backpressure / rotation all-capped idle: hold the loop so an active
367
524
  // goal's continuation does not immediately re-enter the same rate-limit window.
368
525
  // A rotation all-capped idle is a MANDATORY hold (idleUntilReset) — a 0/elapsed
369
- // delay can never skip it (invariant 4: NO THRASH). An interrupt or user signal
526
+ // delay can never skip it (invariant 4: NO THRASH). A control or user signal
370
527
  // ends the wait early and is handled by the main loop.
371
528
  const seenWakeups = wakeups;
372
- await condition(() => interruptedEventId !== null || wakeups !== seenWakeups, holdMs);
529
+ await condition(() => controlEventId !== null || wakeups !== seenWakeups, holdMs);
373
530
  }
374
531
  return true;
375
532
  }