@opengeni/worker-bundle 0.7.1 → 0.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/index.d.ts +184 -68
  2. package/dist/index.js +7099 -1643
  3. package/dist/index.js.map +1 -1
  4. package/package.json +18 -18
  5. package/src/activities/agent-turn.ts +4332 -1068
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +14 -2
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +216 -42
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +165 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +106 -36
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflows/activities.ts +65 -8
  39. package/src/workflows/file-upload-reaper.ts +12 -0
  40. package/src/workflows/rig-verification.ts +28 -0
  41. package/src/workflows/scheduled-tasks.ts +12 -26
  42. package/src/workflows/session.ts +330 -192
  43. package/src/workflows/workflow-wake.ts +13 -0
  44. package/src/workflows.ts +9 -7
@@ -1,6 +1,22 @@
1
- import { ActivityFailure, CancellationScope, condition, continueAsNew, defineSignal, isCancellation, patched, setHandler, TimeoutFailure, workflowInfo } from "@temporalio/workflow";
1
+ import {
2
+ ActivityFailure,
3
+ CancellationScope,
4
+ condition,
5
+ continueAsNew,
6
+ defineSignal,
7
+ isCancellation,
8
+ setHandler,
9
+ TimeoutFailure,
10
+ uuid4,
11
+ workflowInfo,
12
+ } from "@temporalio/workflow";
2
13
  import type * as activities from "../activities";
3
- import { activity, workflowFailureMessage } from "./activities";
14
+ import {
15
+ activity,
16
+ goalActivity,
17
+ turnActivityForTaskQueue,
18
+ workflowFailureMessage,
19
+ } from "./activities";
4
20
 
5
21
  /**
6
22
  * Deterministic backstop for continueAsNew. A session workflow is long-lived
@@ -16,6 +32,7 @@ import { activity, workflowFailureMessage } from "./activities";
16
32
  * while keeping continueAsNew rare enough that it is not a per-turn cost.
17
33
  */
18
34
  const TURNS_PER_RUN_BACKSTOP = 2_000;
35
+ const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
19
36
 
20
37
  /**
21
38
  * The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
@@ -34,10 +51,14 @@ const ROTATION_IDLE_FLOOR_MS = 60_000; // 60s
34
51
  * Pure + exported so the boundedness contract is unit-testable without a workflow env.
35
52
  */
36
53
  export function continuationHoldMs(
37
- result: { status: string; continueDelayMs?: number; idleUntilReset?: boolean },
54
+ result: {
55
+ status: string;
56
+ continueDelayMs?: number;
57
+ idleUntilReset?: boolean;
58
+ },
38
59
  floorMs: number,
39
60
  ): number {
40
- if (result.status !== "idle") {
61
+ if (result.status !== "idle" && result.status !== "recovering") {
41
62
  return 0;
42
63
  }
43
64
  const delay = result.continueDelayMs ?? 0;
@@ -51,7 +72,7 @@ export function continuationHoldMs(
51
72
  * True when an agent-turn activity failure means "the worker hosting the
52
73
  * turn died or vanished" rather than "the turn itself failed": the server
53
74
  * closed the activity with a HEARTBEAT timeout (the worker was killed before
54
- * the graceful-preempt checkpoint could run — SIGKILL, OOM, node loss, or a
75
+ * the graceful recovery checkpoint could run — SIGKILL, OOM, node loss, or a
55
76
  * rollout whose grace period expired) or a SCHEDULE_TO_START timeout (no
56
77
  * worker ever picked the task up). Detection uses the SDK's typed failure
57
78
  * classes, not message-string matching: the failure converter rehydrates
@@ -61,19 +82,27 @@ export function continuationHoldMs(
61
82
  * SCHEDULE_TO_CLOSE timeouts are deliberately excluded: with the 30-day
62
83
  * startToClose they mean the turn truly overran, which stays a real failure.
63
84
  */
64
- function isWorkerDeathFailure(error: unknown): boolean {
85
+ function workerDeathFailure(
86
+ error: unknown,
87
+ ): { timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START" } | null {
65
88
  if (!(error instanceof ActivityFailure)) {
66
- return false;
89
+ return null;
67
90
  }
68
91
  const cause = error.cause;
69
- return cause instanceof TimeoutFailure
70
- && (cause.timeoutType === "HEARTBEAT" || cause.timeoutType === "SCHEDULE_TO_START");
92
+ if (
93
+ !(cause instanceof TimeoutFailure) ||
94
+ (cause.timeoutType !== "HEARTBEAT" && cause.timeoutType !== "SCHEDULE_TO_START")
95
+ ) {
96
+ return null;
97
+ }
98
+ return { timeoutType: cause.timeoutType };
71
99
  }
72
100
 
73
101
  export const userMessage = defineSignal<[string]>("userMessage");
74
102
  export const queueChanged = defineSignal("queueChanged");
75
103
  export const approvalDecision = defineSignal<[string]>("approvalDecision");
76
- export const interrupt = defineSignal<[string]>("interrupt");
104
+ export const sessionControl = defineSignal("sessionControl");
105
+ export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
77
106
 
78
107
  export type SessionWorkflowInput = {
79
108
  accountId: string;
@@ -85,291 +114,400 @@ export type SessionWorkflowInput = {
85
114
  // the boundary without simulating thousands of turns. Never gates correctness
86
115
  // — continueAsNewSuggested is the real-world trigger.
87
116
  maxTurnsPerRun?: number;
117
+ // Test-only override for the durable capacity-wait continue-as-new
118
+ // backstop. Production uses CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP.
119
+ maxCapacityChecksPerRun?: number;
88
120
  };
89
121
 
90
122
  export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
91
- const approvalQueue: string[] = [];
92
- let interruptedEventId: string | null = null;
123
+ const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue);
124
+ let approvalWakeups = 0;
125
+ let interruptionWakeups = 0;
93
126
  let wakeups = 0;
127
+ let capacityWakeups = 0;
128
+ let signalVersion = 0;
129
+ let nonControlSignalVersion = 0;
94
130
  // Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
95
131
  // for the history-overflow guard below; bounded growth is what makes a
96
132
  // weeks-long session survivable.
97
133
  let turnsThisRun = 0;
134
+ let capacityChecksThisRun = 0;
98
135
 
99
136
  setHandler(userMessage, () => {
137
+ signalVersion += 1;
138
+ nonControlSignalVersion += 1;
100
139
  wakeups += 1;
101
140
  });
102
141
  setHandler(queueChanged, () => {
142
+ signalVersion += 1;
143
+ nonControlSignalVersion += 1;
103
144
  wakeups += 1;
104
145
  });
105
- setHandler(approvalDecision, (eventId) => {
106
- approvalQueue.push(eventId);
146
+ setHandler(approvalDecision, () => {
147
+ signalVersion += 1;
148
+ nonControlSignalVersion += 1;
149
+ approvalWakeups += 1;
150
+ });
151
+ setHandler(sessionControl, () => {
152
+ signalVersion += 1;
153
+ interruptionWakeups += 1;
107
154
  });
108
- setHandler(interrupt, (eventId) => {
109
- interruptedEventId = eventId;
155
+ setHandler(codexCapacityChanged, () => {
156
+ signalVersion += 1;
157
+ nonControlSignalVersion += 1;
158
+ capacityWakeups += 1;
110
159
  });
111
160
 
161
+ async function waitForCodexCapacity(
162
+ initial: activities.CodexCapacityWaitRef,
163
+ entryBaseline?: { wakeups: number; capacityWakeups: number },
164
+ ): Promise<void> {
165
+ let current = initial;
166
+ let firstEntryBaseline = entryBaseline;
167
+ for (;;) {
168
+ // Signals can land after the waiter commit but before runAgentTurn returns.
169
+ // Compare the first wait against pre-dispatch counters so they cannot be
170
+ // baselined away; later iterations use their normal local snapshot.
171
+ const seenWakeups = firstEntryBaseline?.wakeups ?? wakeups;
172
+ const seenCapacityWakeups = firstEntryBaseline?.capacityWakeups ?? capacityWakeups;
173
+ const seenInterruptionWakeups = interruptionWakeups;
174
+ firstEntryBaseline = undefined;
175
+ const parsedDeadline = Date.parse(current.nextCheckAt);
176
+ const timerMs = Number.isFinite(parsedDeadline)
177
+ ? Math.max(0, parsedDeadline - Date.now())
178
+ : 0;
179
+ let cause: activities.ReconcileCodexCapacityWaitInput["cause"] = "timer";
180
+ if (wakeups !== seenWakeups) {
181
+ cause = "queue";
182
+ } else if (capacityWakeups !== seenCapacityWakeups) {
183
+ cause = "signal";
184
+ } else if (timerMs > 0) {
185
+ await condition(
186
+ () =>
187
+ interruptionWakeups !== seenInterruptionWakeups ||
188
+ wakeups !== seenWakeups ||
189
+ capacityWakeups !== seenCapacityWakeups,
190
+ timerMs,
191
+ );
192
+ if (interruptionWakeups !== seenInterruptionWakeups) {
193
+ return;
194
+ }
195
+ cause =
196
+ wakeups !== seenWakeups
197
+ ? "queue"
198
+ : capacityWakeups !== seenCapacityWakeups
199
+ ? "signal"
200
+ : "timer";
201
+ }
202
+ const result = await activity.reconcileCodexCapacityWait({
203
+ accountId: input.accountId,
204
+ workspaceId: input.workspaceId,
205
+ sessionId: input.sessionId,
206
+ waiterId: current.waiterId,
207
+ generation: current.generation,
208
+ cause,
209
+ });
210
+ if (result.action !== "waiting") {
211
+ return;
212
+ }
213
+ capacityChecksThisRun += 1;
214
+ const capacityCheckBackstop =
215
+ input.maxCapacityChecksPerRun ?? CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP;
216
+ if (workflowInfo().continueAsNewSuggested || capacityChecksThisRun >= capacityCheckBackstop) {
217
+ // The waiter/outbox is durable in Postgres. A fresh workflow run reads
218
+ // it before goal continuation, reconstructs its timer, and turns any
219
+ // unobserved wake revision into an immediate evaluation.
220
+ await continueAsNew<typeof sessionWorkflow>({
221
+ accountId: input.accountId,
222
+ workspaceId: input.workspaceId,
223
+ sessionId: input.sessionId,
224
+ ...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
225
+ ...(input.maxCapacityChecksPerRun !== undefined
226
+ ? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
227
+ : {}),
228
+ });
229
+ }
230
+ current = result;
231
+ }
232
+ }
233
+
112
234
  while (true) {
113
235
  // History-overflow guard. The top of the loop is the only safe
114
236
  // continueAsNew boundary: no turn is mid-flight (every path that reaches a
115
- // new iteration completed or re-queued its turn first), and a pending
116
- // interrupt is unbacked in-memory state that the new run could not
117
- // reconstruct — so we refuse to continueAsNew while one is set and let the
118
- // loop handle it first. A buffered userMessage/queueChanged signal only
237
+ // new iteration settled or recovered its turn first). Every interruption
238
+ // is already durable in Postgres, so all Temporal signals are replaceable
239
+ // wake hints and none must be carried into the next workflow run. A
240
+ // buffered userMessage/queueChanged signal only
119
241
  // bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
120
- // was sent, so the fresh run re-claims it on its first claimNextQueuedTurn
242
+ // was sent, so the fresh run observes it on its first durable work peek
121
243
  // — losing the counter strands nothing. The queue living in Postgres is the
122
244
  // safety net: continueAsNew carries only the self-contained
123
245
  // SessionWorkflowInput (no initialEventId — the new run claims from the
124
- // queue, it does not replay a seed event). patched() keeps in-flight
125
- // histories (recorded before this branch existed) deterministic: they never
126
- // see the new command.
246
+ // queue, it does not replay a seed event).
127
247
  //
128
- // The approval queue is NOT a boundary condition, and is cleared here. A
129
- // genuinely-pending approval keeps the workflow blocked INSIDE runTurn (the
130
- // `await condition(...)` in the requires_action branch), so it never
131
- // reaches the top of the loop — meaning every approvalQueue entry observed
132
- // here is necessarily STALE (an approvalDecision signal for a turn that
133
- // already settled; the API guard only checks status==='requires_action', so
134
- // two decisions submitted while requires_action both land in the queue and
135
- // the surplus is left behind once the turn completes without re-blocking).
136
- // Coupling continueAsNew to `approvalQueue.length === 0` would let one such
137
- // stale entry wedge the guard forever, re-introducing the exact
138
- // history-overflow termination this branch prevents. Dropping the surplus
139
- // is safe: a real pending approval also leaves the session in
140
- // requires_action with the turn re-dispatchable from Postgres.
141
- if (patched("session-continue-as-new")) {
248
+ // Approval signals are wakeups, never conversation truth. A genuinely
249
+ // accepted decision is already persisted on the turn and is rediscovered
250
+ // by the next durable peek, including after continue-as-new. Stale or
251
+ // duplicate signals therefore cannot block this boundary or manufacture a
252
+ // second approval dispatch.
253
+ {
142
254
  const info = workflowInfo();
143
255
  const maxTurnsPerRun = input.maxTurnsPerRun ?? TURNS_PER_RUN_BACKSTOP;
144
256
  const shouldContinue = info.continueAsNewSuggested || turnsThisRun >= maxTurnsPerRun;
145
- if (shouldContinue && interruptedEventId === null) {
146
- approvalQueue.length = 0;
257
+ if (shouldContinue) {
147
258
  await continueAsNew<typeof sessionWorkflow>({
148
259
  accountId: input.accountId,
149
260
  workspaceId: input.workspaceId,
150
261
  sessionId: input.sessionId,
151
262
  ...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
263
+ ...(input.maxCapacityChecksPerRun !== undefined
264
+ ? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
265
+ : {}),
152
266
  });
153
267
  }
154
268
  }
269
+ // Capture before the final activity chain of this cycle. Temporal may
270
+ // accept a signal while an activity completion and workflow completion
271
+ // race; every terminal return below must observe that arrival and loop.
272
+ const closeSignalVersion = signalVersion;
273
+ const closeNonControlSignalVersion = nonControlSignalVersion;
155
274
  const workflowId = workflowInfo().workflowId;
156
- const turn = await activity.claimNextQueuedTurn({ workspaceId: input.workspaceId, sessionId: input.sessionId, workflowId });
157
- if (!turn) {
158
- if (interruptedEventId === null) {
159
- // With an active goal, idling out is replaced by a synthesized
160
- // continuation turn; the queue (any non-terminal turn) always wins and
161
- // the no-progress/max-continuation guards auto-pause runaway goals.
162
- let continuation: activities.MaybeContinueGoalResult = { action: "none" };
163
- try {
164
- continuation = await activity.maybeContinueGoal({
165
- accountId: input.accountId,
166
- workspaceId: input.workspaceId,
167
- sessionId: input.sessionId,
168
- workflowId,
169
- });
170
- } catch (error) {
171
- if (isCancellation(error)) {
172
- throw error;
173
- }
174
- // A failing goal check must not kill the session (activity retry is
175
- // 1). Fall through to the idle path: the goal stays active in the
176
- // database and the next wake retries, which means the workflow CAN
177
- // complete normally with an active goal on this error path.
178
- }
179
- if (continuation.action === "continue" || continuation.action === "queue") {
180
- // "continue": a goal continuation turn was just enqueued; "queue":
181
- // queued work appeared concurrently. Claim it on the next loop pass.
275
+ const peek = await activity.peekSessionWork({
276
+ workspaceId: input.workspaceId,
277
+ sessionId: input.sessionId,
278
+ });
279
+ if (peek.kind === "interruption-pending") {
280
+ const settlement = await activity.settleSessionInterruptions({
281
+ accountId: input.accountId,
282
+ workspaceId: input.workspaceId,
283
+ sessionId: input.sessionId,
284
+ attemptId: peek.attemptId,
285
+ workflowId,
286
+ });
287
+ if (settlement.action === "paused") {
288
+ if (nonControlSignalVersion !== closeNonControlSignalVersion) {
182
289
  continue;
183
290
  }
291
+ return;
184
292
  }
293
+ continue;
294
+ }
295
+ if (peek.kind === "capacity-wait") {
296
+ await waitForCodexCapacity(peek.ref);
297
+ continue;
298
+ }
299
+ if (peek.kind === "approval-wait") {
300
+ const seenApprovalWakeups = approvalWakeups;
185
301
  const seenWakeups = wakeups;
186
- const woke = await condition(() => interruptedEventId !== null || wakeups !== seenWakeups, "5s");
187
- if (interruptedEventId) {
188
- const idleInterruptEventId = interruptedEventId;
189
- interruptedEventId = null;
190
- // An idle interrupt is an explicit stop: pause an active goal before
191
- // shutting down so a later wake does not auto-continue it. The
192
- // activity inspects the trigger and skips the pause for steer-tagged
193
- // interrupts (steering redirects work, it does not stop it).
194
- await activity.pauseGoalForInterrupt({ workspaceId: input.workspaceId, sessionId: input.sessionId, triggerEventId: idleInterruptEventId });
195
- await activity.markSessionIdle({ workspaceId: input.workspaceId, sessionId: input.sessionId });
196
- return;
302
+ const seenInterruptionWakeups = interruptionWakeups;
303
+ await condition(
304
+ () =>
305
+ interruptionWakeups !== seenInterruptionWakeups ||
306
+ approvalWakeups !== seenApprovalWakeups ||
307
+ wakeups !== seenWakeups,
308
+ );
309
+ continue;
310
+ }
311
+ if (peek.kind === "idle") {
312
+ let continuation: activities.MaybeContinueGoalResult = { action: "none" };
313
+ try {
314
+ continuation = await goalActivity.maybeContinueGoal({
315
+ accountId: input.accountId,
316
+ workspaceId: input.workspaceId,
317
+ sessionId: input.sessionId,
318
+ workflowId,
319
+ });
320
+ } catch (error) {
321
+ if (isCancellation(error)) throw error;
322
+ await activity.enqueueGoalRetryWake({
323
+ accountId: input.accountId,
324
+ workspaceId: input.workspaceId,
325
+ sessionId: input.sessionId,
326
+ workflowId,
327
+ });
197
328
  }
198
- if (woke) {
329
+ if (continuation.action === "continue" || continuation.action === "queue") continue;
330
+ const seenWakeups = wakeups;
331
+ const seenApprovalWakeups = approvalWakeups;
332
+ const seenInterruptionWakeups = interruptionWakeups;
333
+ const woke = await condition(
334
+ () =>
335
+ interruptionWakeups !== seenInterruptionWakeups ||
336
+ wakeups !== seenWakeups ||
337
+ approvalWakeups !== seenApprovalWakeups,
338
+ "5s",
339
+ );
340
+ if (woke) continue;
341
+ const finalPeek = await activity.peekSessionWork({
342
+ workspaceId: input.workspaceId,
343
+ sessionId: input.sessionId,
344
+ });
345
+ if (finalPeek.kind !== "idle") continue;
346
+ await activity.markSessionIdle({
347
+ workspaceId: input.workspaceId,
348
+ sessionId: input.sessionId,
349
+ });
350
+ if (signalVersion !== closeSignalVersion) {
199
351
  continue;
200
352
  }
201
- const finalTurn = await activity.claimNextQueuedTurn({ workspaceId: input.workspaceId, sessionId: input.sessionId, workflowId });
202
- if (!finalTurn) {
203
- await activity.markSessionIdle({ workspaceId: input.workspaceId, sessionId: input.sessionId });
204
- // A queueChanged/userMessage signal can land between the final claim and
205
- // completion; Temporal blocks completion while a signal is buffered, so
206
- // re-checking here guarantees the queued turn is picked up instead of
207
- // stranding it (the signaler skips its start-child fallback on success).
208
- if (interruptedEventId !== null || wakeups !== seenWakeups) {
209
- continue;
210
- }
211
- return;
212
- }
213
- turnsThisRun += 1;
214
- if (!await runTurn(input.accountId, input.workspaceId, input.sessionId, finalTurn.id, finalTurn.triggerEventId)) {
215
- return;
216
- }
217
- continue;
353
+ return;
218
354
  }
219
355
  turnsThisRun += 1;
220
- if (!await runTurn(input.accountId, input.workspaceId, input.sessionId, turn.id, turn.triggerEventId)) {
356
+ const trigger =
357
+ peek.kind === "approval-pending"
358
+ ? ({ kind: "approval", triggerEventId: peek.triggerEventId } as const)
359
+ : ({ kind: "next" } as const);
360
+ if (!(await runTurn(input.accountId, input.workspaceId, input.sessionId, trigger))) {
361
+ if (signalVersion !== closeSignalVersion) continue;
221
362
  return;
222
363
  }
223
364
  }
224
365
 
225
- async function runTurn(accountId: string, workspaceId: string, sessionId: string, turnId: string, triggerEventId: string): Promise<boolean> {
226
- if (interruptedEventId) {
227
- await activity.interruptActiveTurn({
366
+ async function runTurn(
367
+ accountId: string,
368
+ workspaceId: string,
369
+ sessionId: string,
370
+ trigger: activities.RunAgentTurnInput["trigger"],
371
+ ): Promise<boolean> {
372
+ const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
373
+ const attemptId = uuid4();
374
+ const interruptionBaseline = interruptionWakeups;
375
+
376
+ const scope = new CancellationScope();
377
+ const workflowExecution = workflowInfo();
378
+ const workflowId = workflowExecution.workflowId;
379
+ // The stateless resume-by-id model (lease acquire + non-owned injection +
380
+ // release) lives entirely in the runAgentTurn activity.
381
+ const turn: Promise<activities.RunAgentTurnResult> = scope.run(() =>
382
+ turnActivity.runAgentTurn({
228
383
  accountId,
229
384
  workspaceId,
230
385
  sessionId,
231
- triggerEventId: interruptedEventId,
232
- workflowId: workflowInfo().workflowId,
233
- });
234
- interruptedEventId = null;
235
- return true;
236
- }
237
-
238
- const scope = new CancellationScope();
239
- const workflowId = workflowInfo().workflowId;
240
- // P1.2 STATELESS-LEASE GATE. The stateless resume-by-id model (lease acquire
241
- // + non-owned injection + release) lives ENTIRELY in the runAgentSegment
242
- // activity, gated by the sandboxOwnershipEnabled config flag — the workflow's
243
- // command path is intentionally unchanged: the turn still dispatches on the
244
- // EXISTING GLOBAL queue (the module-level `activity` proxy, NO taskQueue
245
- // override, NO per-session worker, NO per-session queue), worker-death still
246
- // re-dispatches via the `worker-death-redispatch` patch below (any worker
247
- // re-resumes the box by id), and there is NO ownerHeartbeat / openStreamRequest
248
- // / closeStreamRequest signal. This patched() marker records the
249
- // stateless-lease cutover point in history so multi-day IN-FLIGHT workflow
250
- // histories replay deterministically on the original lease-less path, and any
251
- // future workflow-side lease change can branch on it without breaking replay.
252
- const statelessLease = patched("sandbox-stateless-lease");
253
- void statelessLease; // command path unchanged today (see comment above)
254
- // Deliberately schedules the LEGACY activity name: in-flight session
255
- // workflows (multi-day by design) recorded this name in their histories,
256
- // and scheduling a different activity type at the same position is a
257
- // non-deterministic change on replay. Migrate to runAgentTurn with
258
- // patched() once pre-rename sessions have drained.
259
- const turn: Promise<activities.RunAgentTurnResult> = scope.run(() => activity.runAgentSegment({
260
- accountId,
261
- workspaceId,
262
- sessionId,
263
- triggerEventId,
264
- workflowId,
265
- turnId,
266
- }));
267
- const outcome: { kind: "result"; result: activities.RunAgentTurnResult } | { kind: "interrupt" } | { kind: "failure"; error: unknown } = await Promise.race([
386
+ workflowId,
387
+ workflowRunId: workflowExecution.runId,
388
+ attemptId,
389
+ trigger,
390
+ }),
391
+ );
392
+ const outcome:
393
+ | { kind: "result"; result: activities.RunAgentTurnResult }
394
+ | { kind: "control" }
395
+ | { kind: "failure"; error: unknown } = await Promise.race([
268
396
  turn.then(
269
- (result: activities.RunAgentTurnResult) => ({ kind: "result" as const, result }),
397
+ (result: activities.RunAgentTurnResult) => ({
398
+ kind: "result" as const,
399
+ result,
400
+ }),
270
401
  (error: unknown) => ({ kind: "failure" as const, error }),
271
402
  ),
272
- condition(() => interruptedEventId !== null).then(() => ({ kind: "interrupt" as const })),
403
+ condition(() => interruptionWakeups !== interruptionBaseline).then(() => ({
404
+ kind: "control" as const,
405
+ })),
273
406
  ]);
274
407
 
275
- if (outcome.kind === "interrupt") {
408
+ if (outcome.kind === "control") {
276
409
  scope.cancel();
277
- await activity.interruptActiveTurn({
410
+ const settlement = await activity.settleSessionInterruptions({
278
411
  accountId,
279
412
  workspaceId,
280
413
  sessionId,
281
- triggerEventId: interruptedEventId!,
414
+ attemptId,
282
415
  workflowId: workflowInfo().workflowId,
283
416
  });
284
- interruptedEventId = null;
285
- return true;
417
+ // Keep the workflow alive until the cancelled activity has actually
418
+ // stopped. Temporal delivers activity cancellation through a heartbeat;
419
+ // returning while the activity promise is still detached lets the
420
+ // workflow complete first, after which the worker can keep streaming for
421
+ // the activity's entire start-to-close window. Postgres settlement above
422
+ // is intentionally first: it fences every late write immediately while
423
+ // the physical activity winds down. The control outcome is authoritative,
424
+ // so a concurrent activity completion or cancellation failure is ignored
425
+ // only after Temporal has durably observed that terminal activity state.
426
+ await turn.catch(() => undefined);
427
+ return settlement.action !== "paused";
286
428
  }
287
429
 
288
430
  if (outcome.kind === "failure") {
431
+ // A capacity settlement may have committed just before the activity
432
+ // transport/worker failed. Recover that durable boundary before generic
433
+ // failSession can overwrite the capacity-idle session.
434
+ {
435
+ const capacityWait = await activity.getCodexCapacityWait({
436
+ workspaceId,
437
+ sessionId,
438
+ });
439
+ if (capacityWait) {
440
+ await waitForCodexCapacity(capacityWait, capacityWaitEntryBaseline);
441
+ return true;
442
+ }
443
+ }
289
444
  // An ungraceful worker death never reaches the activity's graceful
290
- // preemption path — it surfaces here as a heartbeat-timeout failure.
445
+ // recovery path — it surfaces here as a heartbeat-timeout failure.
291
446
  // Conversation truth was still dual-written during the turn, so the
292
- // turn is re-queued (resume notice when it persisted progress, original
293
- // trigger otherwise) and the loop re-claims it on a healthy worker —
447
+ // same turn is marked recovering and the loop re-claims it on a healthy worker —
294
448
  // bounded by a per-turn redispatch counter persisted on the turn row.
295
- // patched() keeps replays of histories recorded before this branch
296
- // existed on their original failSession path.
297
- if (isWorkerDeathFailure(outcome.error) && patched("worker-death-redispatch")) {
298
- const requeue = await activity.requeueTurnAfterWorkerDeath({
449
+ const workerDeath = workerDeathFailure(outcome.error);
450
+ if (workerDeath) {
451
+ const recovery = await activity.recoverDispatch({
299
452
  accountId,
300
453
  workspaceId,
301
454
  sessionId,
302
- triggerEventId,
303
- workflowId,
304
- turnId,
455
+ attemptId,
456
+ timeoutType: workerDeath.timeoutType,
305
457
  });
306
- if (requeue.action !== "exceeded") {
307
- // "requeued": the next claim re-dispatches the turn. "stale": the
458
+ if (recovery.action !== "exceeded") {
459
+ // "recovering": the next claim creates a new attempt for this same
460
+ // current inference. "stale": the
308
461
  // timed-out attempt actually settled the turn (a zombie finished
309
462
  // after the server gave up on its heartbeats); nothing to redo.
310
463
  return true;
311
464
  }
312
- await activity.failSession({
313
- accountId,
314
- workspaceId,
315
- sessionId,
316
- triggerEventId,
317
- workflowId,
318
- error: `Worker died ${requeue.redispatches + 1} times while running this turn (heartbeat timeout); giving up after ${requeue.redispatches} re-dispatches.`,
319
- });
465
+ // The worker-death activity atomically committed failed turn/session
466
+ // truth when the bounded redispatch ceiling was exceeded.
320
467
  return false;
321
468
  }
322
- await activity.failSession({
469
+ await activity.failSessionAttempt({
323
470
  accountId,
324
471
  workspaceId,
325
472
  sessionId,
326
- triggerEventId,
327
- workflowId,
473
+ attemptId,
328
474
  error: workflowFailureMessage(outcome.error),
329
475
  });
330
476
  return false;
331
477
  }
332
478
 
479
+ if (outcome.result.status === "unclaimed") {
480
+ return true;
481
+ }
482
+
333
483
  if (outcome.result.status === "failed" || outcome.result.status === "cancelled") {
334
484
  return outcome.result.status === "cancelled";
335
485
  }
336
486
 
337
- if (outcome.result.status === "preempted") {
338
- // The hosting worker shut down gracefully mid-turn: the activity
339
- // checkpointed conversation truth and put the turn back on the queue
340
- // before completing. Loop again so the next claim re-dispatches it on a
341
- // healthy worker (a pending interrupt is honored first by the loop).
487
+ if (outcome.result.capacityWait) {
488
+ await waitForCodexCapacity(outcome.result.capacityWait, capacityWaitEntryBaseline);
342
489
  return true;
343
490
  }
344
491
 
345
- if (outcome.result.status === "requires_action") {
346
- await condition(() => interruptedEventId !== null || approvalQueue.length > 0);
347
- if (interruptedEventId) {
348
- await activity.interruptActiveTurn({
349
- accountId,
350
- workspaceId,
351
- sessionId,
352
- triggerEventId: interruptedEventId,
353
- workflowId,
354
- });
355
- interruptedEventId = null;
356
- return true;
357
- }
358
- const approvalEventId = approvalQueue.shift();
359
- if (approvalEventId) {
360
- return await runTurn(accountId, workspaceId, sessionId, turnId, approvalEventId);
361
- }
492
+ if (outcome.result.deferredUntilWake) {
493
+ return wakeups !== capacityWaitEntryBaseline.wakeups;
362
494
  }
363
495
 
496
+ if (outcome.result.status === "requires_action") return true;
497
+
364
498
  const holdMs = continuationHoldMs(outcome.result, ROTATION_IDLE_FLOOR_MS);
365
499
  if (holdMs > 0) {
366
- // Provider backpressure / rotation all-capped idle: hold the loop so an active
367
- // goal's continuation does not immediately re-enter the same rate-limit window.
500
+ // Provider recovery / rotation all-capped idle: hold the loop so the same
501
+ // turn or an active goal does not immediately re-enter the same rate-limit window.
368
502
  // A rotation all-capped idle is a MANDATORY hold (idleUntilReset) — a 0/elapsed
369
- // delay can never skip it (invariant 4: NO THRASH). An interrupt or user signal
503
+ // delay can never skip it (invariant 4: NO THRASH). A control or user signal
370
504
  // ends the wait early and is handled by the main loop.
371
505
  const seenWakeups = wakeups;
372
- await condition(() => interruptedEventId !== null || wakeups !== seenWakeups, holdMs);
506
+ const seenInterruptionWakeups = interruptionWakeups;
507
+ await condition(
508
+ () => interruptionWakeups !== seenInterruptionWakeups || wakeups !== seenWakeups,
509
+ holdMs,
510
+ );
373
511
  }
374
512
  return true;
375
513
  }