@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
import {
|
|
2
|
+
appendSessionEventsWithLockedSessionUpdate,
|
|
3
|
+
createScheduledTaskRun,
|
|
4
|
+
createSession,
|
|
5
|
+
createSessionGoal,
|
|
6
|
+
enqueueSessionTurn,
|
|
7
|
+
getBillingBalance,
|
|
8
|
+
getWorkspaceEnvironment,
|
|
9
|
+
isCodexBilledTurn,
|
|
10
|
+
recordUsageEvent,
|
|
11
|
+
requireScheduledTask,
|
|
12
|
+
requireSession,
|
|
13
|
+
setTemporalWorkflowId,
|
|
14
|
+
sumUsageQuantity,
|
|
15
|
+
updateScheduledTask,
|
|
16
|
+
updateScheduledTaskRun,
|
|
17
|
+
upsertSessionGoal,
|
|
18
|
+
} from "@opengeni/db";
|
|
19
|
+
import { appendAndPublishEvents } from "@opengeni/events";
|
|
20
|
+
import { configuredStaticUsageLimits, type Settings } from "@opengeni/config";
|
|
21
|
+
import {
|
|
22
|
+
assertReusableSessionRevivable,
|
|
23
|
+
mergeResourceRefs,
|
|
24
|
+
mergeToolRefs,
|
|
25
|
+
scheduledUserMessagePayload,
|
|
26
|
+
workflowIdForSession,
|
|
27
|
+
} from "./common";
|
|
28
|
+
import { withFirstPartyTools } from "./goals";
|
|
29
|
+
import type {
|
|
30
|
+
ActivityServices,
|
|
31
|
+
DispatchScheduledTaskRunInput,
|
|
32
|
+
DispatchScheduledTaskRunResult,
|
|
33
|
+
} from "./types";
|
|
34
|
+
|
|
35
|
+
export function createScheduledTaskActivities(services: () => Promise<ActivityServices>) {
|
|
36
|
+
return {
|
|
37
|
+
dispatchScheduledTaskRun: async (input: DispatchScheduledTaskRunInput): Promise<DispatchScheduledTaskRunResult> => {
|
|
38
|
+
const { settings, db, bus } = await services();
|
|
39
|
+
const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
|
|
40
|
+
// The scheduled task's model can be codex/<slug>; resolve it here so the
|
|
41
|
+
// admission gate can skip the credit/cost gates for a codex-billed run
|
|
42
|
+
// (paid by the user's ChatGPT/Codex plan). This file uses BASE settings (no
|
|
43
|
+
// codex overlay), so the predicate does its own active-credential read.
|
|
44
|
+
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
45
|
+
const isCodexRun = await isCodexBilledTurn({ db, settings, workspaceId: task.workspaceId, model });
|
|
46
|
+
await ensureScheduledRunAllowed(settings, db, task.accountId, task.workspaceId, input.agentRunUsageIdempotencyKey ? 0 : 1, isCodexRun);
|
|
47
|
+
const run = await createScheduledTaskRun(db, {
|
|
48
|
+
workspaceId: task.workspaceId,
|
|
49
|
+
taskId: task.id,
|
|
50
|
+
triggerType: input.triggerType,
|
|
51
|
+
scheduledAt: null,
|
|
52
|
+
});
|
|
53
|
+
await recordUsageEvent(db, {
|
|
54
|
+
accountId: task.accountId,
|
|
55
|
+
workspaceId: task.workspaceId,
|
|
56
|
+
eventType: "scheduled_task.fired",
|
|
57
|
+
quantity: 1,
|
|
58
|
+
unit: "run",
|
|
59
|
+
sourceResourceType: "scheduled_task_run",
|
|
60
|
+
sourceResourceId: run.id,
|
|
61
|
+
idempotencyKey: `usage:scheduled_task.fired:${run.id}`,
|
|
62
|
+
});
|
|
63
|
+
let result: DispatchScheduledTaskRunResult;
|
|
64
|
+
try {
|
|
65
|
+
const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
66
|
+
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
67
|
+
const goalSpec = task.agentConfig.goal ?? null;
|
|
68
|
+
// Every dispatch carries the first-party MCP server (set_session_title,
|
|
69
|
+
// goal tools, and the permission-gated tools), matching the API path.
|
|
70
|
+
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
71
|
+
if (task.runMode === "new_session_per_run" || !task.reusableSessionId) {
|
|
72
|
+
// The FK on scheduled_tasks.environment_id is ON DELETE RESTRICT, so
|
|
73
|
+
// an attached environment must still exist here; fail closed if not.
|
|
74
|
+
const environment = task.environmentId
|
|
75
|
+
? await getWorkspaceEnvironment(db, task.workspaceId, task.environmentId)
|
|
76
|
+
: null;
|
|
77
|
+
if (task.environmentId && !environment) {
|
|
78
|
+
throw new Error(`workspace environment not found: ${task.environmentId}`);
|
|
79
|
+
}
|
|
80
|
+
const session = await createSession(db, {
|
|
81
|
+
accountId: task.accountId,
|
|
82
|
+
workspaceId: task.workspaceId,
|
|
83
|
+
initialMessage: task.agentConfig.prompt,
|
|
84
|
+
resources: task.agentConfig.resources,
|
|
85
|
+
tools: taskTools,
|
|
86
|
+
metadata: {
|
|
87
|
+
...task.agentConfig.metadata,
|
|
88
|
+
model,
|
|
89
|
+
reasoningEffort,
|
|
90
|
+
scheduledTaskId: task.id,
|
|
91
|
+
scheduledTaskRunId: run.id,
|
|
92
|
+
},
|
|
93
|
+
model,
|
|
94
|
+
sandboxBackend,
|
|
95
|
+
environmentId: task.environmentId ?? null,
|
|
96
|
+
});
|
|
97
|
+
const goal = goalSpec
|
|
98
|
+
? await createSessionGoal(db, {
|
|
99
|
+
accountId: task.accountId,
|
|
100
|
+
workspaceId: task.workspaceId,
|
|
101
|
+
sessionId: session.id,
|
|
102
|
+
text: goalSpec.text,
|
|
103
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
104
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
105
|
+
createdBy: "scheduled_task",
|
|
106
|
+
})
|
|
107
|
+
: null;
|
|
108
|
+
const workflowId = workflowIdForSession(session.id);
|
|
109
|
+
await setTemporalWorkflowId(db, task.workspaceId, session.id, workflowId);
|
|
110
|
+
if (task.runMode === "reusable_session") {
|
|
111
|
+
await updateScheduledTask(db, task.workspaceId, task.id, { reusableSessionId: session.id });
|
|
112
|
+
}
|
|
113
|
+
const events = await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
|
|
114
|
+
{
|
|
115
|
+
type: "session.created",
|
|
116
|
+
payload: {
|
|
117
|
+
status: "queued",
|
|
118
|
+
scheduledTaskId: task.id,
|
|
119
|
+
scheduledTaskRunId: run.id,
|
|
120
|
+
// Names/ids only; never values.
|
|
121
|
+
...(environment ? { environmentId: environment.id, environmentName: environment.name } : {}),
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
...(goal ? [{
|
|
125
|
+
type: "goal.set" as const,
|
|
126
|
+
payload: {
|
|
127
|
+
goalId: goal.id,
|
|
128
|
+
text: goal.text,
|
|
129
|
+
...(goal.successCriteria ? { successCriteria: goal.successCriteria } : {}),
|
|
130
|
+
version: goal.version,
|
|
131
|
+
actor: "scheduled_task",
|
|
132
|
+
replaced: false,
|
|
133
|
+
},
|
|
134
|
+
}] : []),
|
|
135
|
+
{
|
|
136
|
+
type: "user.message",
|
|
137
|
+
payload: scheduledUserMessagePayload(task.agentConfig.prompt, task.agentConfig.resources, taskTools, task.id, run.id),
|
|
138
|
+
},
|
|
139
|
+
{ type: "session.status.changed", payload: { status: "queued" } },
|
|
140
|
+
]);
|
|
141
|
+
const trigger = events.find((event) => event.type === "user.message");
|
|
142
|
+
if (!trigger) {
|
|
143
|
+
throw new Error("failed to append scheduled task trigger event");
|
|
144
|
+
}
|
|
145
|
+
const turn = await enqueueSessionTurn(db, {
|
|
146
|
+
accountId: task.accountId,
|
|
147
|
+
workspaceId: task.workspaceId,
|
|
148
|
+
sessionId: session.id,
|
|
149
|
+
triggerEventId: trigger.id,
|
|
150
|
+
temporalWorkflowId: workflowId,
|
|
151
|
+
source: "scheduled_task",
|
|
152
|
+
prompt: task.agentConfig.prompt,
|
|
153
|
+
resources: task.agentConfig.resources,
|
|
154
|
+
tools: taskTools,
|
|
155
|
+
model,
|
|
156
|
+
reasoningEffort,
|
|
157
|
+
sandboxBackend,
|
|
158
|
+
metadata: {
|
|
159
|
+
scheduledTaskId: task.id,
|
|
160
|
+
scheduledTaskRunId: run.id,
|
|
161
|
+
},
|
|
162
|
+
});
|
|
163
|
+
await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [{
|
|
164
|
+
type: "turn.queued",
|
|
165
|
+
turnId: turn.id,
|
|
166
|
+
payload: { turnId: turn.id, triggerEventId: trigger.id, source: turn.source },
|
|
167
|
+
}]);
|
|
168
|
+
await updateScheduledTaskRun(db, task.workspaceId, run.id, {
|
|
169
|
+
status: "dispatched",
|
|
170
|
+
sessionId: session.id,
|
|
171
|
+
triggerEventId: trigger.id,
|
|
172
|
+
});
|
|
173
|
+
result = {
|
|
174
|
+
action: "start",
|
|
175
|
+
accountId: task.accountId,
|
|
176
|
+
workspaceId: task.workspaceId,
|
|
177
|
+
sessionId: session.id,
|
|
178
|
+
triggerEventId: trigger.id,
|
|
179
|
+
workflowId,
|
|
180
|
+
};
|
|
181
|
+
} else {
|
|
182
|
+
const session = await requireSession(db, task.workspaceId, task.reusableSessionId);
|
|
183
|
+
// A user-cancelled (terminal) reusable session must not be revived and
|
|
184
|
+
// re-billed on the next fire. Early check avoids the pre-lock goal
|
|
185
|
+
// upsert side-effect; the locked-callback check below is the
|
|
186
|
+
// authoritative atomic guard. Mirrors apps/api/src/domain/sessions.ts.
|
|
187
|
+
assertReusableSessionRevivable(session.status);
|
|
188
|
+
// Defensive backstop for the API-level 409: a reusable session keeps
|
|
189
|
+
// its creation-time attachment, so a diverged task attachment must
|
|
190
|
+
// fail the run instead of silently running with the wrong secrets.
|
|
191
|
+
if ((session.environmentId ?? null) !== (task.environmentId ?? null)) {
|
|
192
|
+
throw new Error("scheduled task environment attachment does not match its reusable session");
|
|
193
|
+
}
|
|
194
|
+
// A recurring "maintain X" task re-establishes its objective on every
|
|
195
|
+
// fire: replace the goal text, reactivate it, and reset the counters.
|
|
196
|
+
const reusableGoal = goalSpec
|
|
197
|
+
? await upsertSessionGoal(db, {
|
|
198
|
+
accountId: task.accountId,
|
|
199
|
+
workspaceId: task.workspaceId,
|
|
200
|
+
sessionId: session.id,
|
|
201
|
+
text: goalSpec.text,
|
|
202
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
203
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
204
|
+
createdBy: "scheduled_task",
|
|
205
|
+
})
|
|
206
|
+
: null;
|
|
207
|
+
const events = await appendSessionEventsWithLockedSessionUpdate(db, task.workspaceId, session.id, (locked) => {
|
|
208
|
+
// Authoritative guard under the FOR UPDATE row lock: re-check the
|
|
209
|
+
// current status to close the TOCTOU between the early read and the
|
|
210
|
+
// lock (a cancel could land in between). Throwing aborts the whole
|
|
211
|
+
// append+update transaction, so no user.message is persisted.
|
|
212
|
+
assertReusableSessionRevivable(locked.status);
|
|
213
|
+
return {
|
|
214
|
+
events: [
|
|
215
|
+
...(reusableGoal ? [{
|
|
216
|
+
type: "goal.set" as const,
|
|
217
|
+
payload: {
|
|
218
|
+
goalId: reusableGoal.goal.id,
|
|
219
|
+
text: reusableGoal.goal.text,
|
|
220
|
+
...(reusableGoal.goal.successCriteria ? { successCriteria: reusableGoal.goal.successCriteria } : {}),
|
|
221
|
+
version: reusableGoal.goal.version,
|
|
222
|
+
actor: "scheduled_task",
|
|
223
|
+
replaced: reusableGoal.replaced,
|
|
224
|
+
},
|
|
225
|
+
}] : []),
|
|
226
|
+
{
|
|
227
|
+
type: "user.message",
|
|
228
|
+
payload: scheduledUserMessagePayload(task.agentConfig.prompt, task.agentConfig.resources, taskTools, task.id, run.id),
|
|
229
|
+
},
|
|
230
|
+
],
|
|
231
|
+
update: {
|
|
232
|
+
resources: mergeResourceRefs(locked.resources, task.agentConfig.resources),
|
|
233
|
+
tools: mergeToolRefs(locked.tools, taskTools),
|
|
234
|
+
},
|
|
235
|
+
};
|
|
236
|
+
});
|
|
237
|
+
await bus.publish(task.workspaceId, session.id, events);
|
|
238
|
+
const trigger = events.find((event) => event.type === "user.message");
|
|
239
|
+
if (!trigger) {
|
|
240
|
+
throw new Error("failed to append scheduled task trigger event");
|
|
241
|
+
}
|
|
242
|
+
const turn = await enqueueSessionTurn(db, {
|
|
243
|
+
accountId: task.accountId,
|
|
244
|
+
workspaceId: task.workspaceId,
|
|
245
|
+
sessionId: session.id,
|
|
246
|
+
triggerEventId: trigger.id,
|
|
247
|
+
temporalWorkflowId: workflowIdForSession(session.id),
|
|
248
|
+
source: "scheduled_task",
|
|
249
|
+
prompt: task.agentConfig.prompt,
|
|
250
|
+
resources: task.agentConfig.resources,
|
|
251
|
+
tools: taskTools,
|
|
252
|
+
model,
|
|
253
|
+
reasoningEffort,
|
|
254
|
+
sandboxBackend,
|
|
255
|
+
metadata: {
|
|
256
|
+
scheduledTaskId: task.id,
|
|
257
|
+
scheduledTaskRunId: run.id,
|
|
258
|
+
},
|
|
259
|
+
});
|
|
260
|
+
await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [{
|
|
261
|
+
type: "turn.queued",
|
|
262
|
+
turnId: turn.id,
|
|
263
|
+
payload: { turnId: turn.id, triggerEventId: trigger.id, source: turn.source },
|
|
264
|
+
}]);
|
|
265
|
+
await updateScheduledTaskRun(db, task.workspaceId, run.id, {
|
|
266
|
+
status: "dispatched",
|
|
267
|
+
sessionId: session.id,
|
|
268
|
+
triggerEventId: trigger.id,
|
|
269
|
+
});
|
|
270
|
+
result = {
|
|
271
|
+
action: "signal",
|
|
272
|
+
accountId: task.accountId,
|
|
273
|
+
workspaceId: task.workspaceId,
|
|
274
|
+
sessionId: session.id,
|
|
275
|
+
triggerEventId: trigger.id,
|
|
276
|
+
workflowId: workflowIdForSession(session.id),
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
} catch (error) {
|
|
280
|
+
await updateScheduledTaskRun(db, task.workspaceId, run.id, {
|
|
281
|
+
status: "failed",
|
|
282
|
+
error: error instanceof Error ? error.message : String(error),
|
|
283
|
+
}).catch(() => undefined);
|
|
284
|
+
throw error;
|
|
285
|
+
}
|
|
286
|
+
await recordUsageEvent(db, {
|
|
287
|
+
accountId: task.accountId,
|
|
288
|
+
workspaceId: task.workspaceId,
|
|
289
|
+
eventType: "agent_run.created",
|
|
290
|
+
quantity: 1,
|
|
291
|
+
unit: "run",
|
|
292
|
+
sourceResourceType: "scheduled_task_run",
|
|
293
|
+
sourceResourceId: run.id,
|
|
294
|
+
idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`,
|
|
295
|
+
});
|
|
296
|
+
return result;
|
|
297
|
+
},
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async function ensureScheduledRunAllowed(
|
|
302
|
+
settings: Settings,
|
|
303
|
+
db: ActivityServices["db"],
|
|
304
|
+
accountId: string,
|
|
305
|
+
workspaceId: string,
|
|
306
|
+
requestedAgentRuns: number,
|
|
307
|
+
isCodexRun: boolean,
|
|
308
|
+
): Promise<void> {
|
|
309
|
+
// Codex-billed runs are paid by the user's ChatGPT/Codex plan: skip the
|
|
310
|
+
// credit-balance gate and the monthly model-cost cap. The agent-run COUNT cap
|
|
311
|
+
// below is a volume quota (not a credit/cost gate) and is intentionally kept.
|
|
312
|
+
if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
313
|
+
const balance = await getBillingBalance(db, accountId);
|
|
314
|
+
if (balance.balanceMicros <= 0) {
|
|
315
|
+
throw new Error("insufficient OpenGeni credits");
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
319
|
+
const limits = configuredStaticUsageLimits(settings);
|
|
320
|
+
if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
|
|
321
|
+
const used = await sumUsageQuantity(db, {
|
|
322
|
+
accountId,
|
|
323
|
+
eventType: "model.cost",
|
|
324
|
+
since: startOfUtcMonth(),
|
|
325
|
+
});
|
|
326
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
327
|
+
throw new Error(`monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
331
|
+
const used = await sumUsageQuantity(db, {
|
|
332
|
+
workspaceId,
|
|
333
|
+
eventType: "agent_run.created",
|
|
334
|
+
since: startOfUtcMonth(),
|
|
335
|
+
});
|
|
336
|
+
if (used + requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
337
|
+
throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function startOfUtcMonth(): Date {
|
|
344
|
+
const now = new Date();
|
|
345
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
346
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import {
|
|
2
|
+
claimNextQueuedTurn as claimNextQueuedTurnDb,
|
|
3
|
+
countTurnSessionHistoryItems,
|
|
4
|
+
finishTurn,
|
|
5
|
+
getSessionEvent,
|
|
6
|
+
getSessionTurn,
|
|
7
|
+
incrementTurnWorkerDeathRedispatches,
|
|
8
|
+
requeuePreemptedTurn,
|
|
9
|
+
requireSession,
|
|
10
|
+
setSessionStatus,
|
|
11
|
+
} from "@opengeni/db";
|
|
12
|
+
import { appendAndPublishEvents } from "@opengeni/events";
|
|
13
|
+
import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
|
|
14
|
+
import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
|
|
15
|
+
import { notifyParentOfChildTerminal } from "./parent-wake";
|
|
16
|
+
import type {
|
|
17
|
+
ActivityServices,
|
|
18
|
+
ClaimNextQueuedTurnInput,
|
|
19
|
+
MarkSessionIdleInput,
|
|
20
|
+
RequeueTurnAfterWorkerDeathInput,
|
|
21
|
+
RequeueTurnAfterWorkerDeathResult,
|
|
22
|
+
RunAgentTurnInput,
|
|
23
|
+
} from "./types";
|
|
24
|
+
|
|
25
|
+
// Crash-loop guard for worker-death re-dispatch: a turn that takes a worker
|
|
26
|
+
// down this many times in a row is assumed to be the cause, not the victim,
|
|
27
|
+
// and the session fails for real on the next death. A plain constant by
|
|
28
|
+
// design — this is a pathology bound, not a run-length limit.
|
|
29
|
+
export const WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
30
|
+
|
|
31
|
+
export function createSessionStateActivities(services: () => Promise<ActivityServices>) {
|
|
32
|
+
async function failSession(input: RunAgentTurnInput & { error?: string }): Promise<void> {
|
|
33
|
+
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
34
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
35
|
+
// Already terminal: runAgentTurn settled this turn as failed (status failed,
|
|
36
|
+
// activeTurnId cleared, turn finished) and already woke any parent, then the
|
|
37
|
+
// workflow still routed here because the activity's finally threw after the
|
|
38
|
+
// failed return. Re-failing would append a second turn.failed/status.changed
|
|
39
|
+
// and a second parent wake (a different lastSequence dodges the idle dedupe).
|
|
40
|
+
// The session is already where this activity would put it, so stop.
|
|
41
|
+
if (session.status === "failed") {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
45
|
+
const turnId = session.activeTurnId ?? null;
|
|
46
|
+
await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
47
|
+
{
|
|
48
|
+
type: "turn.failed",
|
|
49
|
+
turnId,
|
|
50
|
+
payload: {
|
|
51
|
+
triggerEventId: input.triggerEventId,
|
|
52
|
+
trigger: trigger?.payload ?? null,
|
|
53
|
+
error: input.error ?? "Agent activity failed before it could report a terminal state.",
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
type: "session.status.changed",
|
|
58
|
+
turnId,
|
|
59
|
+
payload: { status: "failed" },
|
|
60
|
+
},
|
|
61
|
+
]);
|
|
62
|
+
if (turnId) {
|
|
63
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
64
|
+
}
|
|
65
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
66
|
+
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
|
|
70
|
+
const { db, bus } = await services();
|
|
71
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
72
|
+
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
73
|
+
// Pause an active goal before the early return below: an interrupt can
|
|
74
|
+
// land after the turn already cleared activeTurnId, and skipping the pause
|
|
75
|
+
// there would let the loop auto-continue the goal the user just stopped.
|
|
76
|
+
// Steer interrupts are the exception: steering cancels the running turn
|
|
77
|
+
// only to deliver the steered message next — redirection, not a stop —
|
|
78
|
+
// so the goal loop stays active.
|
|
79
|
+
if (!isSteerInterrupt(trigger)) {
|
|
80
|
+
await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
|
|
81
|
+
}
|
|
82
|
+
if (!session.activeTurnId) {
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
86
|
+
{
|
|
87
|
+
turnId: session.activeTurnId,
|
|
88
|
+
type: "turn.cancelled",
|
|
89
|
+
payload: { triggerEventId: input.triggerEventId, reason: trigger?.payload ?? null },
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
turnId: session.activeTurnId,
|
|
93
|
+
type: "session.status.changed",
|
|
94
|
+
payload: { status: "queued" },
|
|
95
|
+
},
|
|
96
|
+
]);
|
|
97
|
+
await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
98
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Put a turn whose hosting worker died WITHOUT the graceful-preempt
|
|
103
|
+
* checkpoint (heartbeat-timeout activity failure: SIGKILL, OOM, node loss)
|
|
104
|
+
* back on the session queue, so the workflow re-dispatches it instead of
|
|
105
|
+
* failing the session. The degraded-resume contract: conversation truth is
|
|
106
|
+
* dual-written after every model response during the turn, so re-running
|
|
107
|
+
* from the trigger event plus stored items loses at most the work since the
|
|
108
|
+
* last reconcile. When the dead attempt persisted items for this turn, the
|
|
109
|
+
* rerun enters through a synthesized `turn.preempted` resume notice (its
|
|
110
|
+
* partial progress is already conversation truth; replaying the original
|
|
111
|
+
* trigger would duplicate input the model has seen). When nothing was
|
|
112
|
+
* persisted — checkpoint absent — the original trigger replays cleanly.
|
|
113
|
+
* Approval reruns always replay their original trigger: the decision is
|
|
114
|
+
* applied through the RunState resume path and a swapped trigger could
|
|
115
|
+
* drop it. Re-dispatches are bounded per turn by
|
|
116
|
+
* WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
|
|
117
|
+
*/
|
|
118
|
+
async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
|
|
119
|
+
const { settings, db, bus } = await services();
|
|
120
|
+
const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
|
|
121
|
+
if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
|
|
122
|
+
// The timed-out attempt was a zombie that actually settled the turn
|
|
123
|
+
// (completed/failed/cancelled it) after the server gave up on its
|
|
124
|
+
// heartbeats. Whatever it recorded is the truth; nothing to redo.
|
|
125
|
+
return { action: "stale" };
|
|
126
|
+
}
|
|
127
|
+
const redispatches = await incrementTurnWorkerDeathRedispatches(db, input.workspaceId, input.turnId);
|
|
128
|
+
if (redispatches > WORKER_DEATH_MAX_REDISPATCHES) {
|
|
129
|
+
return { action: "exceeded", redispatches: redispatches - 1 };
|
|
130
|
+
}
|
|
131
|
+
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
132
|
+
const approvalRerun = trigger?.type === "user.approvalDecision";
|
|
133
|
+
// Legacy run-state mode has no crash checkpoint (the dying worker never
|
|
134
|
+
// captured the blob), so it always replays the original trigger against
|
|
135
|
+
// the previous RunState snapshot — the documented degraded resume.
|
|
136
|
+
const resumeWithNotice = !approvalRerun
|
|
137
|
+
&& settings.sessionHistorySource === "items"
|
|
138
|
+
&& await countTurnSessionHistoryItems(db, input.workspaceId, input.turnId) > 0;
|
|
139
|
+
const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
140
|
+
{
|
|
141
|
+
turnId: turn.id,
|
|
142
|
+
type: "turn.preempted",
|
|
143
|
+
payload: {
|
|
144
|
+
triggerEventId: input.triggerEventId,
|
|
145
|
+
reason: "worker_death",
|
|
146
|
+
redispatches,
|
|
147
|
+
resumeWithNotice,
|
|
148
|
+
...(resumeWithNotice ? { text: WORKER_DEATH_RESUME_TEXT } : {}),
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
turnId: turn.id,
|
|
153
|
+
type: "session.status.changed",
|
|
154
|
+
payload: { status: "queued" },
|
|
155
|
+
},
|
|
156
|
+
]);
|
|
157
|
+
try {
|
|
158
|
+
await requeuePreemptedTurn(db, input.workspaceId, turn.id, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
159
|
+
} catch (requeueError) {
|
|
160
|
+
// The zombie attempt can settle the turn between the status check above
|
|
161
|
+
// and this requeue (it keeps executing until it notices the timeout).
|
|
162
|
+
// A settled turn means its recorded outcome is the truth: report stale
|
|
163
|
+
// so the workflow continues instead of failing the session over a lost
|
|
164
|
+
// race. Anything else is a real persistence failure — rethrow.
|
|
165
|
+
const current = await getSessionTurn(db, input.workspaceId, input.turnId);
|
|
166
|
+
if (current && current.status !== "running" && current.status !== "requires_action") {
|
|
167
|
+
return { action: "stale" };
|
|
168
|
+
}
|
|
169
|
+
throw requeueError;
|
|
170
|
+
}
|
|
171
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
172
|
+
return { action: "requeued", redispatches };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
|
|
176
|
+
const { db } = await services();
|
|
177
|
+
return await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
181
|
+
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
182
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
183
|
+
if (session.status === "queued" || session.status === "running") {
|
|
184
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
185
|
+
}
|
|
186
|
+
// The workflow reaches markSessionIdle exactly when it has decided to stop
|
|
187
|
+
// for now (no queued turn, no goal continuation): the terminal-for-now
|
|
188
|
+
// point for a spawned worker, whatever the cause (goal completed, agent or
|
|
189
|
+
// system paused goal, goalless work finished, idle-interrupt stop). Wake
|
|
190
|
+
// the parent here, deduped per idle episode so the manager is nudged once.
|
|
191
|
+
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
failSession,
|
|
196
|
+
interruptActiveTurn,
|
|
197
|
+
requeueTurnAfterWorkerDeath,
|
|
198
|
+
claimNextQueuedTurn,
|
|
199
|
+
markSessionIdle,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { SessionEventType } from "@opengeni/contracts";
|
|
2
|
+
import type { AppendEventInput } from "@opengeni/db";
|
|
3
|
+
import { Context } from "@temporalio/activity";
|
|
4
|
+
|
|
5
|
+
export function createRuntimeBatcher(flushEvents: (events: AppendEventInput[]) => Promise<void>) {
|
|
6
|
+
let pending: AppendEventInput[] = [];
|
|
7
|
+
let lastFlush = Date.now();
|
|
8
|
+
const structural = new Set<SessionEventType>([
|
|
9
|
+
"agent.message.delta",
|
|
10
|
+
"agent.reasoning.delta",
|
|
11
|
+
"sandbox.command.output.delta",
|
|
12
|
+
// Channel-A PTY bytes flush promptly (P4.4) — a terminal stream is useless
|
|
13
|
+
// batched 33ms behind; the structural set bypasses the batcher's coalesce.
|
|
14
|
+
"terminal.pty.output.delta",
|
|
15
|
+
"agent.toolCall.created",
|
|
16
|
+
"agent.toolCall.output",
|
|
17
|
+
"agent.message.completed",
|
|
18
|
+
"session.requiresAction",
|
|
19
|
+
"turn.completed",
|
|
20
|
+
"turn.failed",
|
|
21
|
+
"turn.cancelled",
|
|
22
|
+
]);
|
|
23
|
+
return {
|
|
24
|
+
push: async (event: { type: SessionEventType; payload: unknown }) => {
|
|
25
|
+
pending.push({ type: event.type, payload: event.payload });
|
|
26
|
+
const elapsed = Date.now() - lastFlush;
|
|
27
|
+
if (pending.length >= 50 || elapsed >= 33 || structural.has(event.type)) {
|
|
28
|
+
await flush();
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
flush,
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
async function flush() {
|
|
35
|
+
if (pending.length === 0) {
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const events = pending;
|
|
39
|
+
pending = [];
|
|
40
|
+
lastFlush = Date.now();
|
|
41
|
+
await flushEvents(events);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function currentActivityContext(): Context | null {
|
|
46
|
+
try {
|
|
47
|
+
return Context.current();
|
|
48
|
+
} catch {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function startActivityHeartbeat(context: Context | null, details: Record<string, unknown>): ReturnType<typeof setInterval> | null {
|
|
54
|
+
if (!context) {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
const timer = setInterval(() => {
|
|
58
|
+
context.heartbeat({ ...details, at: new Date().toISOString() });
|
|
59
|
+
}, 10_000);
|
|
60
|
+
if ("unref" in timer && typeof timer.unref === "function") {
|
|
61
|
+
timer.unref();
|
|
62
|
+
}
|
|
63
|
+
return timer;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export async function nextStreamEvent<T>(iterator: AsyncIterator<T>, context: Context | null): Promise<IteratorResult<T>> {
|
|
67
|
+
if (!context) {
|
|
68
|
+
return await iterator.next();
|
|
69
|
+
}
|
|
70
|
+
return await Promise.race([
|
|
71
|
+
iterator.next(),
|
|
72
|
+
context.cancelled,
|
|
73
|
+
]);
|
|
74
|
+
}
|