@opengeni/worker-bundle 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +184 -68
- package/dist/index.js +7099 -1643
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4332 -1068
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +14 -2
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +216 -42
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +165 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +106 -36
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflows/activities.ts +65 -8
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -192
- package/src/workflows/workflow-wake.ts +13 -0
- package/src/workflows.ts +9 -7
package/src/workflows/session.ts
CHANGED
|
@@ -1,6 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ActivityFailure,
|
|
3
|
+
CancellationScope,
|
|
4
|
+
condition,
|
|
5
|
+
continueAsNew,
|
|
6
|
+
defineSignal,
|
|
7
|
+
isCancellation,
|
|
8
|
+
setHandler,
|
|
9
|
+
TimeoutFailure,
|
|
10
|
+
uuid4,
|
|
11
|
+
workflowInfo,
|
|
12
|
+
} from "@temporalio/workflow";
|
|
2
13
|
import type * as activities from "../activities";
|
|
3
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
activity,
|
|
16
|
+
goalActivity,
|
|
17
|
+
turnActivityForTaskQueue,
|
|
18
|
+
workflowFailureMessage,
|
|
19
|
+
} from "./activities";
|
|
4
20
|
|
|
5
21
|
/**
|
|
6
22
|
* Deterministic backstop for continueAsNew. A session workflow is long-lived
|
|
@@ -16,6 +32,7 @@ import { activity, workflowFailureMessage } from "./activities";
|
|
|
16
32
|
* while keeping continueAsNew rare enough that it is not a per-turn cost.
|
|
17
33
|
*/
|
|
18
34
|
const TURNS_PER_RUN_BACKSTOP = 2_000;
|
|
35
|
+
const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
|
|
19
36
|
|
|
20
37
|
/**
|
|
21
38
|
* The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
|
|
@@ -34,10 +51,14 @@ const ROTATION_IDLE_FLOOR_MS = 60_000; // 60s
|
|
|
34
51
|
* Pure + exported so the boundedness contract is unit-testable without a workflow env.
|
|
35
52
|
*/
|
|
36
53
|
export function continuationHoldMs(
|
|
37
|
-
result: {
|
|
54
|
+
result: {
|
|
55
|
+
status: string;
|
|
56
|
+
continueDelayMs?: number;
|
|
57
|
+
idleUntilReset?: boolean;
|
|
58
|
+
},
|
|
38
59
|
floorMs: number,
|
|
39
60
|
): number {
|
|
40
|
-
if (result.status !== "idle") {
|
|
61
|
+
if (result.status !== "idle" && result.status !== "recovering") {
|
|
41
62
|
return 0;
|
|
42
63
|
}
|
|
43
64
|
const delay = result.continueDelayMs ?? 0;
|
|
@@ -51,7 +72,7 @@ export function continuationHoldMs(
|
|
|
51
72
|
* True when an agent-turn activity failure means "the worker hosting the
|
|
52
73
|
* turn died or vanished" rather than "the turn itself failed": the server
|
|
53
74
|
* closed the activity with a HEARTBEAT timeout (the worker was killed before
|
|
54
|
-
* the graceful
|
|
75
|
+
* the graceful recovery checkpoint could run — SIGKILL, OOM, node loss, or a
|
|
55
76
|
* rollout whose grace period expired) or a SCHEDULE_TO_START timeout (no
|
|
56
77
|
* worker ever picked the task up). Detection uses the SDK's typed failure
|
|
57
78
|
* classes, not message-string matching: the failure converter rehydrates
|
|
@@ -61,19 +82,27 @@ export function continuationHoldMs(
|
|
|
61
82
|
* SCHEDULE_TO_CLOSE timeouts are deliberately excluded: with the 30-day
|
|
62
83
|
* startToClose they mean the turn truly overran, which stays a real failure.
|
|
63
84
|
*/
|
|
64
|
-
function
|
|
85
|
+
function workerDeathFailure(
|
|
86
|
+
error: unknown,
|
|
87
|
+
): { timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START" } | null {
|
|
65
88
|
if (!(error instanceof ActivityFailure)) {
|
|
66
|
-
return
|
|
89
|
+
return null;
|
|
67
90
|
}
|
|
68
91
|
const cause = error.cause;
|
|
69
|
-
|
|
70
|
-
|
|
92
|
+
if (
|
|
93
|
+
!(cause instanceof TimeoutFailure) ||
|
|
94
|
+
(cause.timeoutType !== "HEARTBEAT" && cause.timeoutType !== "SCHEDULE_TO_START")
|
|
95
|
+
) {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
return { timeoutType: cause.timeoutType };
|
|
71
99
|
}
|
|
72
100
|
|
|
73
101
|
export const userMessage = defineSignal<[string]>("userMessage");
|
|
74
102
|
export const queueChanged = defineSignal("queueChanged");
|
|
75
103
|
export const approvalDecision = defineSignal<[string]>("approvalDecision");
|
|
76
|
-
export const
|
|
104
|
+
export const sessionControl = defineSignal("sessionControl");
|
|
105
|
+
export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
|
|
77
106
|
|
|
78
107
|
export type SessionWorkflowInput = {
|
|
79
108
|
accountId: string;
|
|
@@ -85,291 +114,400 @@ export type SessionWorkflowInput = {
|
|
|
85
114
|
// the boundary without simulating thousands of turns. Never gates correctness
|
|
86
115
|
// — continueAsNewSuggested is the real-world trigger.
|
|
87
116
|
maxTurnsPerRun?: number;
|
|
117
|
+
// Test-only override for the durable capacity-wait continue-as-new
|
|
118
|
+
// backstop. Production uses CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP.
|
|
119
|
+
maxCapacityChecksPerRun?: number;
|
|
88
120
|
};
|
|
89
121
|
|
|
90
122
|
export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
|
|
91
|
-
const
|
|
92
|
-
let
|
|
123
|
+
const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue);
|
|
124
|
+
let approvalWakeups = 0;
|
|
125
|
+
let interruptionWakeups = 0;
|
|
93
126
|
let wakeups = 0;
|
|
127
|
+
let capacityWakeups = 0;
|
|
128
|
+
let signalVersion = 0;
|
|
129
|
+
let nonControlSignalVersion = 0;
|
|
94
130
|
// Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
|
|
95
131
|
// for the history-overflow guard below; bounded growth is what makes a
|
|
96
132
|
// weeks-long session survivable.
|
|
97
133
|
let turnsThisRun = 0;
|
|
134
|
+
let capacityChecksThisRun = 0;
|
|
98
135
|
|
|
99
136
|
setHandler(userMessage, () => {
|
|
137
|
+
signalVersion += 1;
|
|
138
|
+
nonControlSignalVersion += 1;
|
|
100
139
|
wakeups += 1;
|
|
101
140
|
});
|
|
102
141
|
setHandler(queueChanged, () => {
|
|
142
|
+
signalVersion += 1;
|
|
143
|
+
nonControlSignalVersion += 1;
|
|
103
144
|
wakeups += 1;
|
|
104
145
|
});
|
|
105
|
-
setHandler(approvalDecision, (
|
|
106
|
-
|
|
146
|
+
setHandler(approvalDecision, () => {
|
|
147
|
+
signalVersion += 1;
|
|
148
|
+
nonControlSignalVersion += 1;
|
|
149
|
+
approvalWakeups += 1;
|
|
150
|
+
});
|
|
151
|
+
setHandler(sessionControl, () => {
|
|
152
|
+
signalVersion += 1;
|
|
153
|
+
interruptionWakeups += 1;
|
|
107
154
|
});
|
|
108
|
-
setHandler(
|
|
109
|
-
|
|
155
|
+
setHandler(codexCapacityChanged, () => {
|
|
156
|
+
signalVersion += 1;
|
|
157
|
+
nonControlSignalVersion += 1;
|
|
158
|
+
capacityWakeups += 1;
|
|
110
159
|
});
|
|
111
160
|
|
|
161
|
+
async function waitForCodexCapacity(
|
|
162
|
+
initial: activities.CodexCapacityWaitRef,
|
|
163
|
+
entryBaseline?: { wakeups: number; capacityWakeups: number },
|
|
164
|
+
): Promise<void> {
|
|
165
|
+
let current = initial;
|
|
166
|
+
let firstEntryBaseline = entryBaseline;
|
|
167
|
+
for (;;) {
|
|
168
|
+
// Signals can land after the waiter commit but before runAgentTurn returns.
|
|
169
|
+
// Compare the first wait against pre-dispatch counters so they cannot be
|
|
170
|
+
// baselined away; later iterations use their normal local snapshot.
|
|
171
|
+
const seenWakeups = firstEntryBaseline?.wakeups ?? wakeups;
|
|
172
|
+
const seenCapacityWakeups = firstEntryBaseline?.capacityWakeups ?? capacityWakeups;
|
|
173
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
174
|
+
firstEntryBaseline = undefined;
|
|
175
|
+
const parsedDeadline = Date.parse(current.nextCheckAt);
|
|
176
|
+
const timerMs = Number.isFinite(parsedDeadline)
|
|
177
|
+
? Math.max(0, parsedDeadline - Date.now())
|
|
178
|
+
: 0;
|
|
179
|
+
let cause: activities.ReconcileCodexCapacityWaitInput["cause"] = "timer";
|
|
180
|
+
if (wakeups !== seenWakeups) {
|
|
181
|
+
cause = "queue";
|
|
182
|
+
} else if (capacityWakeups !== seenCapacityWakeups) {
|
|
183
|
+
cause = "signal";
|
|
184
|
+
} else if (timerMs > 0) {
|
|
185
|
+
await condition(
|
|
186
|
+
() =>
|
|
187
|
+
interruptionWakeups !== seenInterruptionWakeups ||
|
|
188
|
+
wakeups !== seenWakeups ||
|
|
189
|
+
capacityWakeups !== seenCapacityWakeups,
|
|
190
|
+
timerMs,
|
|
191
|
+
);
|
|
192
|
+
if (interruptionWakeups !== seenInterruptionWakeups) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
cause =
|
|
196
|
+
wakeups !== seenWakeups
|
|
197
|
+
? "queue"
|
|
198
|
+
: capacityWakeups !== seenCapacityWakeups
|
|
199
|
+
? "signal"
|
|
200
|
+
: "timer";
|
|
201
|
+
}
|
|
202
|
+
const result = await activity.reconcileCodexCapacityWait({
|
|
203
|
+
accountId: input.accountId,
|
|
204
|
+
workspaceId: input.workspaceId,
|
|
205
|
+
sessionId: input.sessionId,
|
|
206
|
+
waiterId: current.waiterId,
|
|
207
|
+
generation: current.generation,
|
|
208
|
+
cause,
|
|
209
|
+
});
|
|
210
|
+
if (result.action !== "waiting") {
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
capacityChecksThisRun += 1;
|
|
214
|
+
const capacityCheckBackstop =
|
|
215
|
+
input.maxCapacityChecksPerRun ?? CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP;
|
|
216
|
+
if (workflowInfo().continueAsNewSuggested || capacityChecksThisRun >= capacityCheckBackstop) {
|
|
217
|
+
// The waiter/outbox is durable in Postgres. A fresh workflow run reads
|
|
218
|
+
// it before goal continuation, reconstructs its timer, and turns any
|
|
219
|
+
// unobserved wake revision into an immediate evaluation.
|
|
220
|
+
await continueAsNew<typeof sessionWorkflow>({
|
|
221
|
+
accountId: input.accountId,
|
|
222
|
+
workspaceId: input.workspaceId,
|
|
223
|
+
sessionId: input.sessionId,
|
|
224
|
+
...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
|
|
225
|
+
...(input.maxCapacityChecksPerRun !== undefined
|
|
226
|
+
? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
|
|
227
|
+
: {}),
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
current = result;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
112
234
|
while (true) {
|
|
113
235
|
// History-overflow guard. The top of the loop is the only safe
|
|
114
236
|
// continueAsNew boundary: no turn is mid-flight (every path that reaches a
|
|
115
|
-
// new iteration
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
237
|
+
// new iteration settled or recovered its turn first). Every interruption
|
|
238
|
+
// is already durable in Postgres, so all Temporal signals are replaceable
|
|
239
|
+
// wake hints and none must be carried into the next workflow run. A
|
|
240
|
+
// buffered userMessage/queueChanged signal only
|
|
119
241
|
// bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
|
|
120
|
-
// was sent, so the fresh run
|
|
242
|
+
// was sent, so the fresh run observes it on its first durable work peek
|
|
121
243
|
// — losing the counter strands nothing. The queue living in Postgres is the
|
|
122
244
|
// safety net: continueAsNew carries only the self-contained
|
|
123
245
|
// SessionWorkflowInput (no initialEventId — the new run claims from the
|
|
124
|
-
// queue, it does not replay a seed event).
|
|
125
|
-
// histories (recorded before this branch existed) deterministic: they never
|
|
126
|
-
// see the new command.
|
|
246
|
+
// queue, it does not replay a seed event).
|
|
127
247
|
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
|
|
134
|
-
// two decisions submitted while requires_action both land in the queue and
|
|
135
|
-
// the surplus is left behind once the turn completes without re-blocking).
|
|
136
|
-
// Coupling continueAsNew to `approvalQueue.length === 0` would let one such
|
|
137
|
-
// stale entry wedge the guard forever, re-introducing the exact
|
|
138
|
-
// history-overflow termination this branch prevents. Dropping the surplus
|
|
139
|
-
// is safe: a real pending approval also leaves the session in
|
|
140
|
-
// requires_action with the turn re-dispatchable from Postgres.
|
|
141
|
-
if (patched("session-continue-as-new")) {
|
|
248
|
+
// Approval signals are wakeups, never conversation truth. A genuinely
|
|
249
|
+
// accepted decision is already persisted on the turn and is rediscovered
|
|
250
|
+
// by the next durable peek, including after continue-as-new. Stale or
|
|
251
|
+
// duplicate signals therefore cannot block this boundary or manufacture a
|
|
252
|
+
// second approval dispatch.
|
|
253
|
+
{
|
|
142
254
|
const info = workflowInfo();
|
|
143
255
|
const maxTurnsPerRun = input.maxTurnsPerRun ?? TURNS_PER_RUN_BACKSTOP;
|
|
144
256
|
const shouldContinue = info.continueAsNewSuggested || turnsThisRun >= maxTurnsPerRun;
|
|
145
|
-
if (shouldContinue
|
|
146
|
-
approvalQueue.length = 0;
|
|
257
|
+
if (shouldContinue) {
|
|
147
258
|
await continueAsNew<typeof sessionWorkflow>({
|
|
148
259
|
accountId: input.accountId,
|
|
149
260
|
workspaceId: input.workspaceId,
|
|
150
261
|
sessionId: input.sessionId,
|
|
151
262
|
...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
|
|
263
|
+
...(input.maxCapacityChecksPerRun !== undefined
|
|
264
|
+
? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
|
|
265
|
+
: {}),
|
|
152
266
|
});
|
|
153
267
|
}
|
|
154
268
|
}
|
|
269
|
+
// Capture before the final activity chain of this cycle. Temporal may
|
|
270
|
+
// accept a signal while an activity completion and workflow completion
|
|
271
|
+
// race; every terminal return below must observe that arrival and loop.
|
|
272
|
+
const closeSignalVersion = signalVersion;
|
|
273
|
+
const closeNonControlSignalVersion = nonControlSignalVersion;
|
|
155
274
|
const workflowId = workflowInfo().workflowId;
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
} catch (error) {
|
|
171
|
-
if (isCancellation(error)) {
|
|
172
|
-
throw error;
|
|
173
|
-
}
|
|
174
|
-
// A failing goal check must not kill the session (activity retry is
|
|
175
|
-
// 1). Fall through to the idle path: the goal stays active in the
|
|
176
|
-
// database and the next wake retries, which means the workflow CAN
|
|
177
|
-
// complete normally with an active goal on this error path.
|
|
178
|
-
}
|
|
179
|
-
if (continuation.action === "continue" || continuation.action === "queue") {
|
|
180
|
-
// "continue": a goal continuation turn was just enqueued; "queue":
|
|
181
|
-
// queued work appeared concurrently. Claim it on the next loop pass.
|
|
275
|
+
const peek = await activity.peekSessionWork({
|
|
276
|
+
workspaceId: input.workspaceId,
|
|
277
|
+
sessionId: input.sessionId,
|
|
278
|
+
});
|
|
279
|
+
if (peek.kind === "interruption-pending") {
|
|
280
|
+
const settlement = await activity.settleSessionInterruptions({
|
|
281
|
+
accountId: input.accountId,
|
|
282
|
+
workspaceId: input.workspaceId,
|
|
283
|
+
sessionId: input.sessionId,
|
|
284
|
+
attemptId: peek.attemptId,
|
|
285
|
+
workflowId,
|
|
286
|
+
});
|
|
287
|
+
if (settlement.action === "paused") {
|
|
288
|
+
if (nonControlSignalVersion !== closeNonControlSignalVersion) {
|
|
182
289
|
continue;
|
|
183
290
|
}
|
|
291
|
+
return;
|
|
184
292
|
}
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
if (peek.kind === "capacity-wait") {
|
|
296
|
+
await waitForCodexCapacity(peek.ref);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
if (peek.kind === "approval-wait") {
|
|
300
|
+
const seenApprovalWakeups = approvalWakeups;
|
|
185
301
|
const seenWakeups = wakeups;
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
302
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
303
|
+
await condition(
|
|
304
|
+
() =>
|
|
305
|
+
interruptionWakeups !== seenInterruptionWakeups ||
|
|
306
|
+
approvalWakeups !== seenApprovalWakeups ||
|
|
307
|
+
wakeups !== seenWakeups,
|
|
308
|
+
);
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
if (peek.kind === "idle") {
|
|
312
|
+
let continuation: activities.MaybeContinueGoalResult = { action: "none" };
|
|
313
|
+
try {
|
|
314
|
+
continuation = await goalActivity.maybeContinueGoal({
|
|
315
|
+
accountId: input.accountId,
|
|
316
|
+
workspaceId: input.workspaceId,
|
|
317
|
+
sessionId: input.sessionId,
|
|
318
|
+
workflowId,
|
|
319
|
+
});
|
|
320
|
+
} catch (error) {
|
|
321
|
+
if (isCancellation(error)) throw error;
|
|
322
|
+
await activity.enqueueGoalRetryWake({
|
|
323
|
+
accountId: input.accountId,
|
|
324
|
+
workspaceId: input.workspaceId,
|
|
325
|
+
sessionId: input.sessionId,
|
|
326
|
+
workflowId,
|
|
327
|
+
});
|
|
197
328
|
}
|
|
198
|
-
if (
|
|
329
|
+
if (continuation.action === "continue" || continuation.action === "queue") continue;
|
|
330
|
+
const seenWakeups = wakeups;
|
|
331
|
+
const seenApprovalWakeups = approvalWakeups;
|
|
332
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
333
|
+
const woke = await condition(
|
|
334
|
+
() =>
|
|
335
|
+
interruptionWakeups !== seenInterruptionWakeups ||
|
|
336
|
+
wakeups !== seenWakeups ||
|
|
337
|
+
approvalWakeups !== seenApprovalWakeups,
|
|
338
|
+
"5s",
|
|
339
|
+
);
|
|
340
|
+
if (woke) continue;
|
|
341
|
+
const finalPeek = await activity.peekSessionWork({
|
|
342
|
+
workspaceId: input.workspaceId,
|
|
343
|
+
sessionId: input.sessionId,
|
|
344
|
+
});
|
|
345
|
+
if (finalPeek.kind !== "idle") continue;
|
|
346
|
+
await activity.markSessionIdle({
|
|
347
|
+
workspaceId: input.workspaceId,
|
|
348
|
+
sessionId: input.sessionId,
|
|
349
|
+
});
|
|
350
|
+
if (signalVersion !== closeSignalVersion) {
|
|
199
351
|
continue;
|
|
200
352
|
}
|
|
201
|
-
|
|
202
|
-
if (!finalTurn) {
|
|
203
|
-
await activity.markSessionIdle({ workspaceId: input.workspaceId, sessionId: input.sessionId });
|
|
204
|
-
// A queueChanged/userMessage signal can land between the final claim and
|
|
205
|
-
// completion; Temporal blocks completion while a signal is buffered, so
|
|
206
|
-
// re-checking here guarantees the queued turn is picked up instead of
|
|
207
|
-
// stranding it (the signaler skips its start-child fallback on success).
|
|
208
|
-
if (interruptedEventId !== null || wakeups !== seenWakeups) {
|
|
209
|
-
continue;
|
|
210
|
-
}
|
|
211
|
-
return;
|
|
212
|
-
}
|
|
213
|
-
turnsThisRun += 1;
|
|
214
|
-
if (!await runTurn(input.accountId, input.workspaceId, input.sessionId, finalTurn.id, finalTurn.triggerEventId)) {
|
|
215
|
-
return;
|
|
216
|
-
}
|
|
217
|
-
continue;
|
|
353
|
+
return;
|
|
218
354
|
}
|
|
219
355
|
turnsThisRun += 1;
|
|
220
|
-
|
|
356
|
+
const trigger =
|
|
357
|
+
peek.kind === "approval-pending"
|
|
358
|
+
? ({ kind: "approval", triggerEventId: peek.triggerEventId } as const)
|
|
359
|
+
: ({ kind: "next" } as const);
|
|
360
|
+
if (!(await runTurn(input.accountId, input.workspaceId, input.sessionId, trigger))) {
|
|
361
|
+
if (signalVersion !== closeSignalVersion) continue;
|
|
221
362
|
return;
|
|
222
363
|
}
|
|
223
364
|
}
|
|
224
365
|
|
|
225
|
-
async function runTurn(
|
|
226
|
-
|
|
227
|
-
|
|
366
|
+
async function runTurn(
|
|
367
|
+
accountId: string,
|
|
368
|
+
workspaceId: string,
|
|
369
|
+
sessionId: string,
|
|
370
|
+
trigger: activities.RunAgentTurnInput["trigger"],
|
|
371
|
+
): Promise<boolean> {
|
|
372
|
+
const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
|
|
373
|
+
const attemptId = uuid4();
|
|
374
|
+
const interruptionBaseline = interruptionWakeups;
|
|
375
|
+
|
|
376
|
+
const scope = new CancellationScope();
|
|
377
|
+
const workflowExecution = workflowInfo();
|
|
378
|
+
const workflowId = workflowExecution.workflowId;
|
|
379
|
+
// The stateless resume-by-id model (lease acquire + non-owned injection +
|
|
380
|
+
// release) lives entirely in the runAgentTurn activity.
|
|
381
|
+
const turn: Promise<activities.RunAgentTurnResult> = scope.run(() =>
|
|
382
|
+
turnActivity.runAgentTurn({
|
|
228
383
|
accountId,
|
|
229
384
|
workspaceId,
|
|
230
385
|
sessionId,
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
// + non-owned injection + release) lives ENTIRELY in the runAgentSegment
|
|
242
|
-
// activity, gated by the sandboxOwnershipEnabled config flag — the workflow's
|
|
243
|
-
// command path is intentionally unchanged: the turn still dispatches on the
|
|
244
|
-
// EXISTING GLOBAL queue (the module-level `activity` proxy, NO taskQueue
|
|
245
|
-
// override, NO per-session worker, NO per-session queue), worker-death still
|
|
246
|
-
// re-dispatches via the `worker-death-redispatch` patch below (any worker
|
|
247
|
-
// re-resumes the box by id), and there is NO ownerHeartbeat / openStreamRequest
|
|
248
|
-
// / closeStreamRequest signal. This patched() marker records the
|
|
249
|
-
// stateless-lease cutover point in history so multi-day IN-FLIGHT workflow
|
|
250
|
-
// histories replay deterministically on the original lease-less path, and any
|
|
251
|
-
// future workflow-side lease change can branch on it without breaking replay.
|
|
252
|
-
const statelessLease = patched("sandbox-stateless-lease");
|
|
253
|
-
void statelessLease; // command path unchanged today (see comment above)
|
|
254
|
-
// Deliberately schedules the LEGACY activity name: in-flight session
|
|
255
|
-
// workflows (multi-day by design) recorded this name in their histories,
|
|
256
|
-
// and scheduling a different activity type at the same position is a
|
|
257
|
-
// non-deterministic change on replay. Migrate to runAgentTurn with
|
|
258
|
-
// patched() once pre-rename sessions have drained.
|
|
259
|
-
const turn: Promise<activities.RunAgentTurnResult> = scope.run(() => activity.runAgentSegment({
|
|
260
|
-
accountId,
|
|
261
|
-
workspaceId,
|
|
262
|
-
sessionId,
|
|
263
|
-
triggerEventId,
|
|
264
|
-
workflowId,
|
|
265
|
-
turnId,
|
|
266
|
-
}));
|
|
267
|
-
const outcome: { kind: "result"; result: activities.RunAgentTurnResult } | { kind: "interrupt" } | { kind: "failure"; error: unknown } = await Promise.race([
|
|
386
|
+
workflowId,
|
|
387
|
+
workflowRunId: workflowExecution.runId,
|
|
388
|
+
attemptId,
|
|
389
|
+
trigger,
|
|
390
|
+
}),
|
|
391
|
+
);
|
|
392
|
+
const outcome:
|
|
393
|
+
| { kind: "result"; result: activities.RunAgentTurnResult }
|
|
394
|
+
| { kind: "control" }
|
|
395
|
+
| { kind: "failure"; error: unknown } = await Promise.race([
|
|
268
396
|
turn.then(
|
|
269
|
-
(result: activities.RunAgentTurnResult) => ({
|
|
397
|
+
(result: activities.RunAgentTurnResult) => ({
|
|
398
|
+
kind: "result" as const,
|
|
399
|
+
result,
|
|
400
|
+
}),
|
|
270
401
|
(error: unknown) => ({ kind: "failure" as const, error }),
|
|
271
402
|
),
|
|
272
|
-
condition(() =>
|
|
403
|
+
condition(() => interruptionWakeups !== interruptionBaseline).then(() => ({
|
|
404
|
+
kind: "control" as const,
|
|
405
|
+
})),
|
|
273
406
|
]);
|
|
274
407
|
|
|
275
|
-
if (outcome.kind === "
|
|
408
|
+
if (outcome.kind === "control") {
|
|
276
409
|
scope.cancel();
|
|
277
|
-
await activity.
|
|
410
|
+
const settlement = await activity.settleSessionInterruptions({
|
|
278
411
|
accountId,
|
|
279
412
|
workspaceId,
|
|
280
413
|
sessionId,
|
|
281
|
-
|
|
414
|
+
attemptId,
|
|
282
415
|
workflowId: workflowInfo().workflowId,
|
|
283
416
|
});
|
|
284
|
-
|
|
285
|
-
|
|
417
|
+
// Keep the workflow alive until the cancelled activity has actually
|
|
418
|
+
// stopped. Temporal delivers activity cancellation through a heartbeat;
|
|
419
|
+
// returning while the activity promise is still detached lets the
|
|
420
|
+
// workflow complete first, after which the worker can keep streaming for
|
|
421
|
+
// the activity's entire start-to-close window. Postgres settlement above
|
|
422
|
+
// is intentionally first: it fences every late write immediately while
|
|
423
|
+
// the physical activity winds down. The control outcome is authoritative,
|
|
424
|
+
// so a concurrent activity completion or cancellation failure is ignored
|
|
425
|
+
// only after Temporal has durably observed that terminal activity state.
|
|
426
|
+
await turn.catch(() => undefined);
|
|
427
|
+
return settlement.action !== "paused";
|
|
286
428
|
}
|
|
287
429
|
|
|
288
430
|
if (outcome.kind === "failure") {
|
|
431
|
+
// A capacity settlement may have committed just before the activity
|
|
432
|
+
// transport/worker failed. Recover that durable boundary before generic
|
|
433
|
+
// failSession can overwrite the capacity-idle session.
|
|
434
|
+
{
|
|
435
|
+
const capacityWait = await activity.getCodexCapacityWait({
|
|
436
|
+
workspaceId,
|
|
437
|
+
sessionId,
|
|
438
|
+
});
|
|
439
|
+
if (capacityWait) {
|
|
440
|
+
await waitForCodexCapacity(capacityWait, capacityWaitEntryBaseline);
|
|
441
|
+
return true;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
289
444
|
// An ungraceful worker death never reaches the activity's graceful
|
|
290
|
-
//
|
|
445
|
+
// recovery path — it surfaces here as a heartbeat-timeout failure.
|
|
291
446
|
// Conversation truth was still dual-written during the turn, so the
|
|
292
|
-
// turn is re-
|
|
293
|
-
// trigger otherwise) and the loop re-claims it on a healthy worker —
|
|
447
|
+
// same turn is marked recovering and the loop re-claims it on a healthy worker —
|
|
294
448
|
// bounded by a per-turn redispatch counter persisted on the turn row.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const requeue = await activity.requeueTurnAfterWorkerDeath({
|
|
449
|
+
const workerDeath = workerDeathFailure(outcome.error);
|
|
450
|
+
if (workerDeath) {
|
|
451
|
+
const recovery = await activity.recoverDispatch({
|
|
299
452
|
accountId,
|
|
300
453
|
workspaceId,
|
|
301
454
|
sessionId,
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
turnId,
|
|
455
|
+
attemptId,
|
|
456
|
+
timeoutType: workerDeath.timeoutType,
|
|
305
457
|
});
|
|
306
|
-
if (
|
|
307
|
-
// "
|
|
458
|
+
if (recovery.action !== "exceeded") {
|
|
459
|
+
// "recovering": the next claim creates a new attempt for this same
|
|
460
|
+
// current inference. "stale": the
|
|
308
461
|
// timed-out attempt actually settled the turn (a zombie finished
|
|
309
462
|
// after the server gave up on its heartbeats); nothing to redo.
|
|
310
463
|
return true;
|
|
311
464
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
workspaceId,
|
|
315
|
-
sessionId,
|
|
316
|
-
triggerEventId,
|
|
317
|
-
workflowId,
|
|
318
|
-
error: `Worker died ${requeue.redispatches + 1} times while running this turn (heartbeat timeout); giving up after ${requeue.redispatches} re-dispatches.`,
|
|
319
|
-
});
|
|
465
|
+
// The worker-death activity atomically committed failed turn/session
|
|
466
|
+
// truth when the bounded redispatch ceiling was exceeded.
|
|
320
467
|
return false;
|
|
321
468
|
}
|
|
322
|
-
await activity.
|
|
469
|
+
await activity.failSessionAttempt({
|
|
323
470
|
accountId,
|
|
324
471
|
workspaceId,
|
|
325
472
|
sessionId,
|
|
326
|
-
|
|
327
|
-
workflowId,
|
|
473
|
+
attemptId,
|
|
328
474
|
error: workflowFailureMessage(outcome.error),
|
|
329
475
|
});
|
|
330
476
|
return false;
|
|
331
477
|
}
|
|
332
478
|
|
|
479
|
+
if (outcome.result.status === "unclaimed") {
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
|
|
333
483
|
if (outcome.result.status === "failed" || outcome.result.status === "cancelled") {
|
|
334
484
|
return outcome.result.status === "cancelled";
|
|
335
485
|
}
|
|
336
486
|
|
|
337
|
-
if (outcome.result.
|
|
338
|
-
|
|
339
|
-
// checkpointed conversation truth and put the turn back on the queue
|
|
340
|
-
// before completing. Loop again so the next claim re-dispatches it on a
|
|
341
|
-
// healthy worker (a pending interrupt is honored first by the loop).
|
|
487
|
+
if (outcome.result.capacityWait) {
|
|
488
|
+
await waitForCodexCapacity(outcome.result.capacityWait, capacityWaitEntryBaseline);
|
|
342
489
|
return true;
|
|
343
490
|
}
|
|
344
491
|
|
|
345
|
-
if (outcome.result.
|
|
346
|
-
|
|
347
|
-
if (interruptedEventId) {
|
|
348
|
-
await activity.interruptActiveTurn({
|
|
349
|
-
accountId,
|
|
350
|
-
workspaceId,
|
|
351
|
-
sessionId,
|
|
352
|
-
triggerEventId: interruptedEventId,
|
|
353
|
-
workflowId,
|
|
354
|
-
});
|
|
355
|
-
interruptedEventId = null;
|
|
356
|
-
return true;
|
|
357
|
-
}
|
|
358
|
-
const approvalEventId = approvalQueue.shift();
|
|
359
|
-
if (approvalEventId) {
|
|
360
|
-
return await runTurn(accountId, workspaceId, sessionId, turnId, approvalEventId);
|
|
361
|
-
}
|
|
492
|
+
if (outcome.result.deferredUntilWake) {
|
|
493
|
+
return wakeups !== capacityWaitEntryBaseline.wakeups;
|
|
362
494
|
}
|
|
363
495
|
|
|
496
|
+
if (outcome.result.status === "requires_action") return true;
|
|
497
|
+
|
|
364
498
|
const holdMs = continuationHoldMs(outcome.result, ROTATION_IDLE_FLOOR_MS);
|
|
365
499
|
if (holdMs > 0) {
|
|
366
|
-
// Provider
|
|
367
|
-
// goal
|
|
500
|
+
// Provider recovery / rotation all-capped idle: hold the loop so the same
|
|
501
|
+
// turn or an active goal does not immediately re-enter the same rate-limit window.
|
|
368
502
|
// A rotation all-capped idle is a MANDATORY hold (idleUntilReset) — a 0/elapsed
|
|
369
|
-
// delay can never skip it (invariant 4: NO THRASH).
|
|
503
|
+
// delay can never skip it (invariant 4: NO THRASH). A control or user signal
|
|
370
504
|
// ends the wait early and is handled by the main loop.
|
|
371
505
|
const seenWakeups = wakeups;
|
|
372
|
-
|
|
506
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
507
|
+
await condition(
|
|
508
|
+
() => interruptionWakeups !== seenInterruptionWakeups || wakeups !== seenWakeups,
|
|
509
|
+
holdMs,
|
|
510
|
+
);
|
|
373
511
|
}
|
|
374
512
|
return true;
|
|
375
513
|
}
|