@opengeni/worker-bundle 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7113 -1652
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4337 -1052
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
package/src/workflows/session.ts
CHANGED
|
@@ -1,6 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ActivityFailure,
|
|
3
|
+
CancellationScope,
|
|
4
|
+
condition,
|
|
5
|
+
continueAsNew,
|
|
6
|
+
defineSignal,
|
|
7
|
+
isCancellation,
|
|
8
|
+
setHandler,
|
|
9
|
+
TimeoutFailure,
|
|
10
|
+
uuid4,
|
|
11
|
+
workflowInfo,
|
|
12
|
+
} from "@temporalio/workflow";
|
|
2
13
|
import type * as activities from "../activities";
|
|
3
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
activity,
|
|
16
|
+
goalActivity,
|
|
17
|
+
turnActivityForTaskQueue,
|
|
18
|
+
workflowFailureMessage,
|
|
19
|
+
} from "./activities";
|
|
4
20
|
|
|
5
21
|
/**
|
|
6
22
|
* Deterministic backstop for continueAsNew. A session workflow is long-lived
|
|
@@ -16,6 +32,7 @@ import { activity, workflowFailureMessage } from "./activities";
|
|
|
16
32
|
* while keeping continueAsNew rare enough that it is not a per-turn cost.
|
|
17
33
|
*/
|
|
18
34
|
const TURNS_PER_RUN_BACKSTOP = 2_000;
|
|
35
|
+
const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
|
|
19
36
|
|
|
20
37
|
/**
|
|
21
38
|
* The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
|
|
@@ -51,7 +68,7 @@ export function continuationHoldMs(
|
|
|
51
68
|
* True when an agent-turn activity failure means "the worker hosting the
|
|
52
69
|
* turn died or vanished" rather than "the turn itself failed": the server
|
|
53
70
|
* closed the activity with a HEARTBEAT timeout (the worker was killed before
|
|
54
|
-
* the graceful
|
|
71
|
+
* the graceful recovery checkpoint could run — SIGKILL, OOM, node loss, or a
|
|
55
72
|
* rollout whose grace period expired) or a SCHEDULE_TO_START timeout (no
|
|
56
73
|
* worker ever picked the task up). Detection uses the SDK's typed failure
|
|
57
74
|
* classes, not message-string matching: the failure converter rehydrates
|
|
@@ -61,19 +78,27 @@ export function continuationHoldMs(
|
|
|
61
78
|
* SCHEDULE_TO_CLOSE timeouts are deliberately excluded: with the 30-day
|
|
62
79
|
* startToClose they mean the turn truly overran, which stays a real failure.
|
|
63
80
|
*/
|
|
64
|
-
function
|
|
81
|
+
function workerDeathFailure(
|
|
82
|
+
error: unknown,
|
|
83
|
+
): { timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START" } | null {
|
|
65
84
|
if (!(error instanceof ActivityFailure)) {
|
|
66
|
-
return
|
|
85
|
+
return null;
|
|
67
86
|
}
|
|
68
87
|
const cause = error.cause;
|
|
69
|
-
|
|
70
|
-
|
|
88
|
+
if (
|
|
89
|
+
!(cause instanceof TimeoutFailure) ||
|
|
90
|
+
(cause.timeoutType !== "HEARTBEAT" && cause.timeoutType !== "SCHEDULE_TO_START")
|
|
91
|
+
) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
return { timeoutType: cause.timeoutType };
|
|
71
95
|
}
|
|
72
96
|
|
|
73
97
|
export const userMessage = defineSignal<[string]>("userMessage");
|
|
74
98
|
export const queueChanged = defineSignal("queueChanged");
|
|
75
99
|
export const approvalDecision = defineSignal<[string]>("approvalDecision");
|
|
76
|
-
export const
|
|
100
|
+
export const sessionControl = defineSignal<[string]>("sessionControl");
|
|
101
|
+
export const codexCapacityChanged = defineSignal<[number]>("codexCapacityChanged");
|
|
77
102
|
|
|
78
103
|
export type SessionWorkflowInput = {
|
|
79
104
|
accountId: string;
|
|
@@ -85,291 +110,423 @@ export type SessionWorkflowInput = {
|
|
|
85
110
|
// the boundary without simulating thousands of turns. Never gates correctness
|
|
86
111
|
// — continueAsNewSuggested is the real-world trigger.
|
|
87
112
|
maxTurnsPerRun?: number;
|
|
113
|
+
// Test-only override for the durable capacity-wait continue-as-new
|
|
114
|
+
// backstop. Production uses CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP.
|
|
115
|
+
maxCapacityChecksPerRun?: number;
|
|
88
116
|
};
|
|
89
117
|
|
|
90
118
|
export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void> {
|
|
91
|
-
const
|
|
92
|
-
let
|
|
119
|
+
const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue);
|
|
120
|
+
let approvalWakeups = 0;
|
|
121
|
+
let controlEventId: string | null = null;
|
|
93
122
|
let wakeups = 0;
|
|
123
|
+
let capacityWakeups = 0;
|
|
124
|
+
let signalVersion = 0;
|
|
125
|
+
let nonControlSignalVersion = 0;
|
|
94
126
|
// Turns dispatched on THIS run (reset to 0 by continueAsNew). The backstop
|
|
95
127
|
// for the history-overflow guard below; bounded growth is what makes a
|
|
96
128
|
// weeks-long session survivable.
|
|
97
129
|
let turnsThisRun = 0;
|
|
130
|
+
let capacityChecksThisRun = 0;
|
|
98
131
|
|
|
99
132
|
setHandler(userMessage, () => {
|
|
133
|
+
signalVersion += 1;
|
|
134
|
+
nonControlSignalVersion += 1;
|
|
100
135
|
wakeups += 1;
|
|
101
136
|
});
|
|
102
137
|
setHandler(queueChanged, () => {
|
|
138
|
+
signalVersion += 1;
|
|
139
|
+
nonControlSignalVersion += 1;
|
|
103
140
|
wakeups += 1;
|
|
104
141
|
});
|
|
105
|
-
setHandler(approvalDecision, (
|
|
106
|
-
|
|
142
|
+
setHandler(approvalDecision, () => {
|
|
143
|
+
signalVersion += 1;
|
|
144
|
+
nonControlSignalVersion += 1;
|
|
145
|
+
approvalWakeups += 1;
|
|
146
|
+
});
|
|
147
|
+
setHandler(sessionControl, (eventId) => {
|
|
148
|
+
signalVersion += 1;
|
|
149
|
+
controlEventId = eventId;
|
|
107
150
|
});
|
|
108
|
-
setHandler(
|
|
109
|
-
|
|
151
|
+
setHandler(codexCapacityChanged, () => {
|
|
152
|
+
signalVersion += 1;
|
|
153
|
+
nonControlSignalVersion += 1;
|
|
154
|
+
capacityWakeups += 1;
|
|
110
155
|
});
|
|
111
156
|
|
|
157
|
+
async function waitForCodexCapacity(
|
|
158
|
+
initial: activities.CodexCapacityWaitRef,
|
|
159
|
+
entryBaseline?: { wakeups: number; capacityWakeups: number },
|
|
160
|
+
): Promise<void> {
|
|
161
|
+
let current = initial;
|
|
162
|
+
let firstEntryBaseline = entryBaseline;
|
|
163
|
+
for (;;) {
|
|
164
|
+
if (controlEventId !== null) {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
// Signals can land after the waiter commit but before runAgentTurn returns.
|
|
168
|
+
// Compare the first wait against pre-dispatch counters so they cannot be
|
|
169
|
+
// baselined away; later iterations use their normal local snapshot.
|
|
170
|
+
const seenWakeups = firstEntryBaseline?.wakeups ?? wakeups;
|
|
171
|
+
const seenCapacityWakeups = firstEntryBaseline?.capacityWakeups ?? capacityWakeups;
|
|
172
|
+
firstEntryBaseline = undefined;
|
|
173
|
+
const parsedDeadline = Date.parse(current.nextCheckAt);
|
|
174
|
+
const timerMs = Number.isFinite(parsedDeadline)
|
|
175
|
+
? Math.max(0, parsedDeadline - Date.now())
|
|
176
|
+
: 0;
|
|
177
|
+
let cause: activities.ReconcileCodexCapacityWaitInput["cause"] = "timer";
|
|
178
|
+
if (wakeups !== seenWakeups) {
|
|
179
|
+
cause = "queue";
|
|
180
|
+
} else if (capacityWakeups !== seenCapacityWakeups) {
|
|
181
|
+
cause = "signal";
|
|
182
|
+
} else if (timerMs > 0) {
|
|
183
|
+
await condition(
|
|
184
|
+
() =>
|
|
185
|
+
controlEventId !== null ||
|
|
186
|
+
wakeups !== seenWakeups ||
|
|
187
|
+
capacityWakeups !== seenCapacityWakeups,
|
|
188
|
+
timerMs,
|
|
189
|
+
);
|
|
190
|
+
if (controlEventId !== null) {
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
cause =
|
|
194
|
+
wakeups !== seenWakeups
|
|
195
|
+
? "queue"
|
|
196
|
+
: capacityWakeups !== seenCapacityWakeups
|
|
197
|
+
? "signal"
|
|
198
|
+
: "timer";
|
|
199
|
+
}
|
|
200
|
+
const result = await activity.reconcileCodexCapacityWait({
|
|
201
|
+
accountId: input.accountId,
|
|
202
|
+
workspaceId: input.workspaceId,
|
|
203
|
+
sessionId: input.sessionId,
|
|
204
|
+
waiterId: current.waiterId,
|
|
205
|
+
generation: current.generation,
|
|
206
|
+
cause,
|
|
207
|
+
});
|
|
208
|
+
if (result.action !== "waiting") {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
capacityChecksThisRun += 1;
|
|
212
|
+
const capacityCheckBackstop =
|
|
213
|
+
input.maxCapacityChecksPerRun ?? CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP;
|
|
214
|
+
if (
|
|
215
|
+
controlEventId === null &&
|
|
216
|
+
(workflowInfo().continueAsNewSuggested || capacityChecksThisRun >= capacityCheckBackstop)
|
|
217
|
+
) {
|
|
218
|
+
// The waiter/outbox is durable in Postgres. A fresh workflow run reads
|
|
219
|
+
// it before goal continuation, reconstructs its timer, and turns any
|
|
220
|
+
// unobserved wake revision into an immediate evaluation.
|
|
221
|
+
await continueAsNew<typeof sessionWorkflow>({
|
|
222
|
+
accountId: input.accountId,
|
|
223
|
+
workspaceId: input.workspaceId,
|
|
224
|
+
sessionId: input.sessionId,
|
|
225
|
+
...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
|
|
226
|
+
...(input.maxCapacityChecksPerRun !== undefined
|
|
227
|
+
? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
|
|
228
|
+
: {}),
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
current = result;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
112
235
|
while (true) {
|
|
113
236
|
// History-overflow guard. The top of the loop is the only safe
|
|
114
237
|
// continueAsNew boundary: no turn is mid-flight (every path that reaches a
|
|
115
|
-
// new iteration
|
|
116
|
-
//
|
|
238
|
+
// new iteration settled or recovered its turn first), and a pending
|
|
239
|
+
// control signal is unbacked in-memory state that the new run could not
|
|
117
240
|
// reconstruct — so we refuse to continueAsNew while one is set and let the
|
|
118
241
|
// loop handle it first. A buffered userMessage/queueChanged signal only
|
|
119
242
|
// bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
|
|
120
|
-
// was sent, so the fresh run
|
|
243
|
+
// was sent, so the fresh run observes it on its first durable work peek
|
|
121
244
|
// — losing the counter strands nothing. The queue living in Postgres is the
|
|
122
245
|
// safety net: continueAsNew carries only the self-contained
|
|
123
246
|
// SessionWorkflowInput (no initialEventId — the new run claims from the
|
|
124
|
-
// queue, it does not replay a seed event).
|
|
125
|
-
// histories (recorded before this branch existed) deterministic: they never
|
|
126
|
-
// see the new command.
|
|
247
|
+
// queue, it does not replay a seed event).
|
|
127
248
|
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
|
|
134
|
-
// two decisions submitted while requires_action both land in the queue and
|
|
135
|
-
// the surplus is left behind once the turn completes without re-blocking).
|
|
136
|
-
// Coupling continueAsNew to `approvalQueue.length === 0` would let one such
|
|
137
|
-
// stale entry wedge the guard forever, re-introducing the exact
|
|
138
|
-
// history-overflow termination this branch prevents. Dropping the surplus
|
|
139
|
-
// is safe: a real pending approval also leaves the session in
|
|
140
|
-
// requires_action with the turn re-dispatchable from Postgres.
|
|
141
|
-
if (patched("session-continue-as-new")) {
|
|
249
|
+
// Approval signals are wakeups, never conversation truth. A genuinely
|
|
250
|
+
// accepted decision is already persisted on the turn and is rediscovered
|
|
251
|
+
// by the next durable peek, including after continue-as-new. Stale or
|
|
252
|
+
// duplicate signals therefore cannot block this boundary or manufacture a
|
|
253
|
+
// second approval dispatch.
|
|
254
|
+
{
|
|
142
255
|
const info = workflowInfo();
|
|
143
256
|
const maxTurnsPerRun = input.maxTurnsPerRun ?? TURNS_PER_RUN_BACKSTOP;
|
|
144
257
|
const shouldContinue = info.continueAsNewSuggested || turnsThisRun >= maxTurnsPerRun;
|
|
145
|
-
if (shouldContinue &&
|
|
146
|
-
approvalQueue.length = 0;
|
|
258
|
+
if (shouldContinue && controlEventId === null) {
|
|
147
259
|
await continueAsNew<typeof sessionWorkflow>({
|
|
148
260
|
accountId: input.accountId,
|
|
149
261
|
workspaceId: input.workspaceId,
|
|
150
262
|
sessionId: input.sessionId,
|
|
151
263
|
...(input.maxTurnsPerRun !== undefined ? { maxTurnsPerRun: input.maxTurnsPerRun } : {}),
|
|
264
|
+
...(input.maxCapacityChecksPerRun !== undefined
|
|
265
|
+
? { maxCapacityChecksPerRun: input.maxCapacityChecksPerRun }
|
|
266
|
+
: {}),
|
|
152
267
|
});
|
|
153
268
|
}
|
|
154
269
|
}
|
|
270
|
+
// Capture before the final activity chain of this cycle. Temporal may
|
|
271
|
+
// accept a signal while an activity completion and workflow completion
|
|
272
|
+
// race; every terminal return below must observe that arrival and loop.
|
|
273
|
+
const closeSignalVersion = signalVersion;
|
|
274
|
+
const closeNonControlSignalVersion = nonControlSignalVersion;
|
|
155
275
|
const workflowId = workflowInfo().workflowId;
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
276
|
+
const peek = await activity.peekSessionWork({
|
|
277
|
+
workspaceId: input.workspaceId,
|
|
278
|
+
sessionId: input.sessionId,
|
|
279
|
+
});
|
|
280
|
+
if (peek.kind === "fence-settle") {
|
|
281
|
+
const settlement = await activity.settleSessionControl({
|
|
282
|
+
accountId: input.accountId,
|
|
283
|
+
workspaceId: input.workspaceId,
|
|
284
|
+
sessionId: input.sessionId,
|
|
285
|
+
triggerEventId: peek.controlEventId,
|
|
286
|
+
workflowId,
|
|
287
|
+
});
|
|
288
|
+
if (controlEventId === peek.controlEventId) controlEventId = null;
|
|
289
|
+
if (settlement.action === "paused") {
|
|
290
|
+
// The matching sessionControl signal is consumed by this exact durable
|
|
291
|
+
// settlement. It must not manufacture a second loop; only later work
|
|
292
|
+
// or a different control keeps the workflow alive.
|
|
293
|
+
if (nonControlSignalVersion !== closeNonControlSignalVersion || controlEventId !== null) {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
// A duplicate/repaired control signal may arrive after its durable fence
|
|
301
|
+
// already settled. Explicitly classify it through the same settlement
|
|
302
|
+
// activity and clear the in-memory signal so it cannot pin this workflow
|
|
303
|
+
// run forever or block continue-as-new.
|
|
304
|
+
if (controlEventId !== null) {
|
|
305
|
+
const signaledControlEventId = controlEventId;
|
|
306
|
+
const settlement = await activity.settleSessionControl({
|
|
307
|
+
accountId: input.accountId,
|
|
308
|
+
workspaceId: input.workspaceId,
|
|
309
|
+
sessionId: input.sessionId,
|
|
310
|
+
triggerEventId: signaledControlEventId,
|
|
311
|
+
workflowId,
|
|
312
|
+
});
|
|
313
|
+
if (controlEventId === signaledControlEventId) controlEventId = null;
|
|
314
|
+
if (settlement.action === "paused") {
|
|
315
|
+
if (nonControlSignalVersion !== closeNonControlSignalVersion || controlEventId !== null) {
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
if (peek.kind === "capacity-wait") {
|
|
323
|
+
await waitForCodexCapacity(peek.ref);
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
if (peek.kind === "approval-wait") {
|
|
327
|
+
const seenApprovalWakeups = approvalWakeups;
|
|
328
|
+
const seenWakeups = wakeups;
|
|
329
|
+
await condition(
|
|
330
|
+
() =>
|
|
331
|
+
controlEventId !== null ||
|
|
332
|
+
approvalWakeups !== seenApprovalWakeups ||
|
|
333
|
+
wakeups !== seenWakeups,
|
|
334
|
+
);
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
if (peek.kind === "idle") {
|
|
338
|
+
if (controlEventId === null) {
|
|
162
339
|
let continuation: activities.MaybeContinueGoalResult = { action: "none" };
|
|
163
340
|
try {
|
|
164
|
-
continuation = await
|
|
341
|
+
continuation = await goalActivity.maybeContinueGoal({
|
|
165
342
|
accountId: input.accountId,
|
|
166
343
|
workspaceId: input.workspaceId,
|
|
167
344
|
sessionId: input.sessionId,
|
|
168
345
|
workflowId,
|
|
169
346
|
});
|
|
170
347
|
} catch (error) {
|
|
171
|
-
if (isCancellation(error))
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
}
|
|
179
|
-
if (continuation.action === "continue" || continuation.action === "queue") {
|
|
180
|
-
// "continue": a goal continuation turn was just enqueued; "queue":
|
|
181
|
-
// queued work appeared concurrently. Claim it on the next loop pass.
|
|
182
|
-
continue;
|
|
348
|
+
if (isCancellation(error)) throw error;
|
|
349
|
+
await activity.enqueueGoalRetryWake({
|
|
350
|
+
accountId: input.accountId,
|
|
351
|
+
workspaceId: input.workspaceId,
|
|
352
|
+
sessionId: input.sessionId,
|
|
353
|
+
workflowId,
|
|
354
|
+
});
|
|
183
355
|
}
|
|
356
|
+
if (continuation.action === "continue" || continuation.action === "queue") continue;
|
|
184
357
|
}
|
|
185
358
|
const seenWakeups = wakeups;
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
359
|
+
const seenApprovalWakeups = approvalWakeups;
|
|
360
|
+
const woke = await condition(
|
|
361
|
+
() =>
|
|
362
|
+
controlEventId !== null ||
|
|
363
|
+
wakeups !== seenWakeups ||
|
|
364
|
+
approvalWakeups !== seenApprovalWakeups,
|
|
365
|
+
"5s",
|
|
366
|
+
);
|
|
367
|
+
if (woke) continue;
|
|
368
|
+
const finalPeek = await activity.peekSessionWork({
|
|
369
|
+
workspaceId: input.workspaceId,
|
|
370
|
+
sessionId: input.sessionId,
|
|
371
|
+
});
|
|
372
|
+
if (finalPeek.kind !== "idle") continue;
|
|
373
|
+
await activity.markSessionIdle({
|
|
374
|
+
workspaceId: input.workspaceId,
|
|
375
|
+
sessionId: input.sessionId,
|
|
376
|
+
});
|
|
377
|
+
if (signalVersion !== closeSignalVersion) {
|
|
199
378
|
continue;
|
|
200
379
|
}
|
|
201
|
-
|
|
202
|
-
if (!finalTurn) {
|
|
203
|
-
await activity.markSessionIdle({ workspaceId: input.workspaceId, sessionId: input.sessionId });
|
|
204
|
-
// A queueChanged/userMessage signal can land between the final claim and
|
|
205
|
-
// completion; Temporal blocks completion while a signal is buffered, so
|
|
206
|
-
// re-checking here guarantees the queued turn is picked up instead of
|
|
207
|
-
// stranding it (the signaler skips its start-child fallback on success).
|
|
208
|
-
if (interruptedEventId !== null || wakeups !== seenWakeups) {
|
|
209
|
-
continue;
|
|
210
|
-
}
|
|
211
|
-
return;
|
|
212
|
-
}
|
|
213
|
-
turnsThisRun += 1;
|
|
214
|
-
if (!await runTurn(input.accountId, input.workspaceId, input.sessionId, finalTurn.id, finalTurn.triggerEventId)) {
|
|
215
|
-
return;
|
|
216
|
-
}
|
|
217
|
-
continue;
|
|
380
|
+
return;
|
|
218
381
|
}
|
|
219
382
|
turnsThisRun += 1;
|
|
220
|
-
|
|
383
|
+
const trigger =
|
|
384
|
+
peek.kind === "approval-pending"
|
|
385
|
+
? ({ kind: "approval", triggerEventId: peek.triggerEventId } as const)
|
|
386
|
+
: ({ kind: "next" } as const);
|
|
387
|
+
if (!(await runTurn(input.accountId, input.workspaceId, input.sessionId, trigger))) {
|
|
388
|
+
if (signalVersion !== closeSignalVersion) continue;
|
|
221
389
|
return;
|
|
222
390
|
}
|
|
223
391
|
}
|
|
224
392
|
|
|
225
|
-
async function runTurn(
|
|
226
|
-
|
|
227
|
-
|
|
393
|
+
async function runTurn(
|
|
394
|
+
accountId: string,
|
|
395
|
+
workspaceId: string,
|
|
396
|
+
sessionId: string,
|
|
397
|
+
trigger: activities.RunAgentTurnInput["trigger"],
|
|
398
|
+
): Promise<boolean> {
|
|
399
|
+
if (controlEventId) {
|
|
400
|
+
const settlement = await activity.settleSessionControl({
|
|
228
401
|
accountId,
|
|
229
402
|
workspaceId,
|
|
230
403
|
sessionId,
|
|
231
|
-
triggerEventId:
|
|
404
|
+
triggerEventId: controlEventId,
|
|
232
405
|
workflowId: workflowInfo().workflowId,
|
|
233
406
|
});
|
|
234
|
-
|
|
235
|
-
return
|
|
407
|
+
controlEventId = null;
|
|
408
|
+
return settlement.action !== "paused";
|
|
236
409
|
}
|
|
237
410
|
|
|
411
|
+
const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
|
|
412
|
+
const attemptId = uuid4();
|
|
413
|
+
|
|
238
414
|
const scope = new CancellationScope();
|
|
239
415
|
const workflowId = workflowInfo().workflowId;
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
const
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
// and scheduling a different activity type at the same position is a
|
|
257
|
-
// non-deterministic change on replay. Migrate to runAgentTurn with
|
|
258
|
-
// patched() once pre-rename sessions have drained.
|
|
259
|
-
const turn: Promise<activities.RunAgentTurnResult> = scope.run(() => activity.runAgentSegment({
|
|
260
|
-
accountId,
|
|
261
|
-
workspaceId,
|
|
262
|
-
sessionId,
|
|
263
|
-
triggerEventId,
|
|
264
|
-
workflowId,
|
|
265
|
-
turnId,
|
|
266
|
-
}));
|
|
267
|
-
const outcome: { kind: "result"; result: activities.RunAgentTurnResult } | { kind: "interrupt" } | { kind: "failure"; error: unknown } = await Promise.race([
|
|
416
|
+
// The stateless resume-by-id model (lease acquire + non-owned injection +
|
|
417
|
+
// release) lives entirely in the runAgentTurn activity.
|
|
418
|
+
const turn: Promise<activities.RunAgentTurnResult> = scope.run(() =>
|
|
419
|
+
turnActivity.runAgentTurn({
|
|
420
|
+
accountId,
|
|
421
|
+
workspaceId,
|
|
422
|
+
sessionId,
|
|
423
|
+
workflowId,
|
|
424
|
+
attemptId,
|
|
425
|
+
trigger,
|
|
426
|
+
}),
|
|
427
|
+
);
|
|
428
|
+
const outcome:
|
|
429
|
+
| { kind: "result"; result: activities.RunAgentTurnResult }
|
|
430
|
+
| { kind: "control" }
|
|
431
|
+
| { kind: "failure"; error: unknown } = await Promise.race([
|
|
268
432
|
turn.then(
|
|
269
433
|
(result: activities.RunAgentTurnResult) => ({ kind: "result" as const, result }),
|
|
270
434
|
(error: unknown) => ({ kind: "failure" as const, error }),
|
|
271
435
|
),
|
|
272
|
-
condition(() =>
|
|
436
|
+
condition(() => controlEventId !== null).then(() => ({ kind: "control" as const })),
|
|
273
437
|
]);
|
|
274
438
|
|
|
275
|
-
if (outcome.kind === "
|
|
439
|
+
if (outcome.kind === "control") {
|
|
276
440
|
scope.cancel();
|
|
277
|
-
await activity.
|
|
441
|
+
const settlement = await activity.settleSessionControl({
|
|
278
442
|
accountId,
|
|
279
443
|
workspaceId,
|
|
280
444
|
sessionId,
|
|
281
|
-
triggerEventId:
|
|
445
|
+
triggerEventId: controlEventId!,
|
|
282
446
|
workflowId: workflowInfo().workflowId,
|
|
283
447
|
});
|
|
284
|
-
|
|
285
|
-
return
|
|
448
|
+
controlEventId = null;
|
|
449
|
+
return settlement.action !== "paused";
|
|
286
450
|
}
|
|
287
451
|
|
|
288
452
|
if (outcome.kind === "failure") {
|
|
453
|
+
// A capacity settlement may have committed just before the activity
|
|
454
|
+
// transport/worker failed. Recover that durable boundary before generic
|
|
455
|
+
// failSession can overwrite the capacity-idle session.
|
|
456
|
+
{
|
|
457
|
+
const capacityWait = await activity.getCodexCapacityWait({ workspaceId, sessionId });
|
|
458
|
+
if (capacityWait) {
|
|
459
|
+
await waitForCodexCapacity(capacityWait, capacityWaitEntryBaseline);
|
|
460
|
+
return true;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
289
463
|
// An ungraceful worker death never reaches the activity's graceful
|
|
290
|
-
//
|
|
464
|
+
// recovery path — it surfaces here as a heartbeat-timeout failure.
|
|
291
465
|
// Conversation truth was still dual-written during the turn, so the
|
|
292
|
-
// turn is re-
|
|
293
|
-
// trigger otherwise) and the loop re-claims it on a healthy worker —
|
|
466
|
+
// same turn is marked recovering and the loop re-claims it on a healthy worker —
|
|
294
467
|
// bounded by a per-turn redispatch counter persisted on the turn row.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const requeue = await activity.requeueTurnAfterWorkerDeath({
|
|
468
|
+
const workerDeath = workerDeathFailure(outcome.error);
|
|
469
|
+
if (workerDeath) {
|
|
470
|
+
const recovery = await activity.recoverDispatch({
|
|
299
471
|
accountId,
|
|
300
472
|
workspaceId,
|
|
301
473
|
sessionId,
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
turnId,
|
|
474
|
+
attemptId,
|
|
475
|
+
timeoutType: workerDeath.timeoutType,
|
|
305
476
|
});
|
|
306
|
-
if (
|
|
307
|
-
// "
|
|
477
|
+
if (recovery.action !== "exceeded") {
|
|
478
|
+
// "recovering": the next claim creates a new attempt for this same
|
|
479
|
+
// current inference. "stale": the
|
|
308
480
|
// timed-out attempt actually settled the turn (a zombie finished
|
|
309
481
|
// after the server gave up on its heartbeats); nothing to redo.
|
|
310
482
|
return true;
|
|
311
483
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
workspaceId,
|
|
315
|
-
sessionId,
|
|
316
|
-
triggerEventId,
|
|
317
|
-
workflowId,
|
|
318
|
-
error: `Worker died ${requeue.redispatches + 1} times while running this turn (heartbeat timeout); giving up after ${requeue.redispatches} re-dispatches.`,
|
|
319
|
-
});
|
|
484
|
+
// The worker-death activity atomically committed failed turn/session
|
|
485
|
+
// truth when the bounded redispatch ceiling was exceeded.
|
|
320
486
|
return false;
|
|
321
487
|
}
|
|
322
|
-
await activity.
|
|
488
|
+
await activity.failSessionAttempt({
|
|
323
489
|
accountId,
|
|
324
490
|
workspaceId,
|
|
325
491
|
sessionId,
|
|
326
|
-
|
|
327
|
-
workflowId,
|
|
492
|
+
attemptId,
|
|
328
493
|
error: workflowFailureMessage(outcome.error),
|
|
329
494
|
});
|
|
330
495
|
return false;
|
|
331
496
|
}
|
|
332
497
|
|
|
498
|
+
if (outcome.result.status === "unclaimed") {
|
|
499
|
+
return true;
|
|
500
|
+
}
|
|
501
|
+
|
|
333
502
|
if (outcome.result.status === "failed" || outcome.result.status === "cancelled") {
|
|
334
503
|
return outcome.result.status === "cancelled";
|
|
335
504
|
}
|
|
336
505
|
|
|
337
|
-
if (outcome.result.status === "
|
|
338
|
-
// The hosting worker shut down gracefully mid-turn: the activity
|
|
339
|
-
// checkpointed conversation truth and put the turn back on the queue
|
|
340
|
-
// before completing. Loop again so the next claim re-dispatches it on a
|
|
341
|
-
// healthy worker (a pending interrupt is honored first by the loop).
|
|
506
|
+
if (outcome.result.status === "recovering") {
|
|
342
507
|
return true;
|
|
343
508
|
}
|
|
344
509
|
|
|
345
|
-
if (outcome.result.
|
|
346
|
-
await
|
|
347
|
-
|
|
348
|
-
await activity.interruptActiveTurn({
|
|
349
|
-
accountId,
|
|
350
|
-
workspaceId,
|
|
351
|
-
sessionId,
|
|
352
|
-
triggerEventId: interruptedEventId,
|
|
353
|
-
workflowId,
|
|
354
|
-
});
|
|
355
|
-
interruptedEventId = null;
|
|
356
|
-
return true;
|
|
357
|
-
}
|
|
358
|
-
const approvalEventId = approvalQueue.shift();
|
|
359
|
-
if (approvalEventId) {
|
|
360
|
-
return await runTurn(accountId, workspaceId, sessionId, turnId, approvalEventId);
|
|
361
|
-
}
|
|
510
|
+
if (outcome.result.capacityWait) {
|
|
511
|
+
await waitForCodexCapacity(outcome.result.capacityWait, capacityWaitEntryBaseline);
|
|
512
|
+
return true;
|
|
362
513
|
}
|
|
363
514
|
|
|
515
|
+
if (outcome.result.deferredUntilWake) {
|
|
516
|
+
return wakeups !== capacityWaitEntryBaseline.wakeups;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
if (outcome.result.status === "requires_action") return true;
|
|
520
|
+
|
|
364
521
|
const holdMs = continuationHoldMs(outcome.result, ROTATION_IDLE_FLOOR_MS);
|
|
365
522
|
if (holdMs > 0) {
|
|
366
523
|
// Provider backpressure / rotation all-capped idle: hold the loop so an active
|
|
367
524
|
// goal's continuation does not immediately re-enter the same rate-limit window.
|
|
368
525
|
// A rotation all-capped idle is a MANDATORY hold (idleUntilReset) — a 0/elapsed
|
|
369
|
-
// delay can never skip it (invariant 4: NO THRASH).
|
|
526
|
+
// delay can never skip it (invariant 4: NO THRASH). A control or user signal
|
|
370
527
|
// ends the wait early and is handled by the main loop.
|
|
371
528
|
const seenWakeups = wakeups;
|
|
372
|
-
await condition(() =>
|
|
529
|
+
await condition(() => controlEventId !== null || wakeups !== seenWakeups, holdMs);
|
|
373
530
|
}
|
|
374
531
|
return true;
|
|
375
532
|
}
|