@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +10 -2
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +3 -4
- package/dist/activities/agent-turn/codex.d.ts +8 -7
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +10 -1
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
- package/dist/activities/agent-turn/turn-context.d.ts +3 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +2 -1
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/goals.d.ts +11 -0
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/activities/types.d.ts +8 -0
- package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
- package/dist/activities-control-SGU2WEVB.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
- package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
- package/dist/chunk-S2QVRVJW.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
- package/dist/chunk-TJIPCHMP.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +69 -6
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +29 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/dist/workflow-bundle.js +26 -3
- package/dist/workflows/activities.d.ts +5 -0
- package/dist/workflows/automations.d.ts +5 -0
- package/package.json +18 -18
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +28 -6
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +53 -72
- package/src/activities/agent-turn/codex.ts +21 -12
- package/src/activities/agent-turn/compaction-prep.ts +52 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +88 -42
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +20 -14
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +106 -14
- package/src/activities/agent-turn/tool-environment.ts +72 -14
- package/src/activities/agent-turn/tool-policy.ts +7 -6
- package/src/activities/agent-turn/turn-context.ts +12 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +72 -2
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +117 -35
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +248 -30
- package/src/activities/session-state.ts +8 -1
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +6 -0
- package/src/activities.ts +1 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +8 -4
- package/src/http.ts +3 -0
- package/src/index.ts +38 -4
- package/src/observability-metrics.ts +202 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/src/workflows/activities.ts +15 -0
- package/src/workflows/automations.ts +21 -2
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
scheduledUserMessagePayload,
|
|
33
33
|
withFirstPartyTools,
|
|
34
34
|
workflowIdForSession
|
|
35
|
-
} from "./chunk-
|
|
35
|
+
} from "./chunk-S2QVRVJW.js";
|
|
36
36
|
import {
|
|
37
37
|
inspectOpenSandboxKubernetesInventory,
|
|
38
38
|
recordCreditBalanceGauges,
|
|
@@ -50,8 +50,9 @@ import {
|
|
|
50
50
|
recordSandboxOrphansTerminated,
|
|
51
51
|
recordSandboxRotationBacklogGauges,
|
|
52
52
|
recordTurnsQueuedGauge,
|
|
53
|
+
recordWorkerDeathRecoveryMetrics,
|
|
53
54
|
runtimeMetricsHooksForObservability
|
|
54
|
-
} from "./chunk-
|
|
55
|
+
} from "./chunk-TJIPCHMP.js";
|
|
55
56
|
|
|
56
57
|
// src/activities/browser-state-artifact-reaper.ts
|
|
57
58
|
import {
|
|
@@ -138,12 +139,18 @@ function cleanupErrorCategory(error) {
|
|
|
138
139
|
}
|
|
139
140
|
|
|
140
141
|
// src/activities/automations.ts
|
|
141
|
-
import {
|
|
142
|
+
import {
|
|
143
|
+
resolveTurnExecutionPolicyV1,
|
|
144
|
+
WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
|
|
145
|
+
WORKSPACE_OPENROUTER_MODEL_ID_PREFIX
|
|
146
|
+
} from "@opengeni/config";
|
|
142
147
|
import {
|
|
143
148
|
assertWorkspaceModelPolicyAllows,
|
|
144
149
|
canonicalConfiguredModel,
|
|
145
150
|
createAndStartSessionWithOutcome,
|
|
146
|
-
recordWorkspaceUsage
|
|
151
|
+
recordWorkspaceUsage,
|
|
152
|
+
resolveCatalogSettings,
|
|
153
|
+
resolveWorkspaceCatalogSettings
|
|
147
154
|
} from "@opengeni/core";
|
|
148
155
|
import {
|
|
149
156
|
AutomationAuthorityRevokedError,
|
|
@@ -154,14 +161,20 @@ import {
|
|
|
154
161
|
|
|
155
162
|
// src/activities/agent-run-admission.ts
|
|
156
163
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
164
|
+
import { modelFundingForAdmission } from "@opengeni/core";
|
|
157
165
|
import { getBillingBalance, isCodexBilledTurn, sumUsageQuantity } from "@opengeni/db";
|
|
158
166
|
async function agentRunAdmissionDenial(services, input) {
|
|
159
|
-
const
|
|
167
|
+
const codexBilled = await isCodexBilledTurn({
|
|
160
168
|
db: services.db,
|
|
161
169
|
settings: services.settings,
|
|
162
170
|
workspaceId: input.workspaceId,
|
|
163
171
|
model: input.model
|
|
164
172
|
});
|
|
173
|
+
const externallyBilled = modelFundingForAdmission(
|
|
174
|
+
services.settings,
|
|
175
|
+
input.model,
|
|
176
|
+
codexBilled
|
|
177
|
+
).fundedWithoutCredits;
|
|
165
178
|
if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
|
|
166
179
|
if (services.entitlements) {
|
|
167
180
|
const decision = await services.entitlements.admitRun({
|
|
@@ -217,8 +230,19 @@ function createAutomationActivities(services, options = {}) {
|
|
|
217
230
|
const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
|
|
218
231
|
const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
|
|
219
232
|
return {
|
|
233
|
+
settleAutomationRunFailure: async (input) => {
|
|
234
|
+
const service = await services();
|
|
235
|
+
await settle(service.db, {
|
|
236
|
+
workspaceId: input.workspaceId,
|
|
237
|
+
runId: input.runId,
|
|
238
|
+
status: "failed",
|
|
239
|
+
errorCode: "dispatch_failed"
|
|
240
|
+
});
|
|
241
|
+
},
|
|
220
242
|
dispatchAutomationRun: async (input) => {
|
|
221
243
|
const service = await services();
|
|
244
|
+
const catalogSourceSettings = service.catalogSourceSettings ?? service.settings;
|
|
245
|
+
const deploymentCatalogSettings = (await resolveCatalogSettings(service.db, catalogSourceSettings)).settings;
|
|
222
246
|
const run = await claim(service.db, input);
|
|
223
247
|
if (!run) return { action: "not_found" };
|
|
224
248
|
if (run.status === "dispatched") {
|
|
@@ -226,61 +250,87 @@ function createAutomationActivities(services, options = {}) {
|
|
|
226
250
|
return { action: "already_dispatched", sessionId: run.sessionId };
|
|
227
251
|
}
|
|
228
252
|
if (run.status === "skipped") {
|
|
229
|
-
return {
|
|
253
|
+
return {
|
|
254
|
+
action: "skipped",
|
|
255
|
+
reason: run.errorCode ?? "authority_revoked"
|
|
256
|
+
};
|
|
230
257
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
await settle(service.db, {
|
|
234
|
-
workspaceId: input.workspaceId,
|
|
235
|
-
runId: run.id,
|
|
236
|
-
status: "skipped",
|
|
237
|
-
errorCode: "accepted_execution_mismatch"
|
|
238
|
-
});
|
|
239
|
-
return { action: "skipped", reason: "accepted_execution_mismatch" };
|
|
258
|
+
if (run.status === "failed") {
|
|
259
|
+
return { action: "failed", reason: run.errorCode ?? "dispatch_failed" };
|
|
240
260
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
261
|
+
try {
|
|
262
|
+
const accepted = run.acceptedExecution;
|
|
263
|
+
if (accepted.accountId !== input.accountId || accepted.workspaceId !== input.workspaceId || accepted.sourceId !== run.sourceId || accepted.triggerId !== run.triggerId || accepted.triggerRevision !== run.triggerRevision || accepted.eventId !== run.eventId) {
|
|
264
|
+
await settle(service.db, {
|
|
265
|
+
workspaceId: input.workspaceId,
|
|
266
|
+
runId: run.id,
|
|
267
|
+
status: "skipped",
|
|
268
|
+
errorCode: "accepted_execution_mismatch"
|
|
269
|
+
});
|
|
270
|
+
return { action: "skipped", reason: "accepted_execution_mismatch" };
|
|
271
|
+
}
|
|
272
|
+
const template = accepted.sessionTemplate;
|
|
273
|
+
const requestedModel = template.model ?? deploymentCatalogSettings.openaiModel;
|
|
274
|
+
const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
|
|
275
|
+
accountId: input.accountId,
|
|
256
276
|
workspaceId: input.workspaceId,
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
277
|
+
retainedProductModelId: requestedModel
|
|
278
|
+
})).settings : deploymentCatalogSettings;
|
|
279
|
+
const catalogService = { ...service, settings: catalogSettings };
|
|
280
|
+
let model;
|
|
281
|
+
try {
|
|
282
|
+
const configuredModel = canonicalConfiguredModel(catalogSettings, requestedModel);
|
|
283
|
+
if (!configuredModel) throw new Error("automation has no configured model");
|
|
284
|
+
await assertModelPolicy(service.db, catalogSettings, input.workspaceId, configuredModel);
|
|
285
|
+
model = configuredModel;
|
|
286
|
+
} catch (error) {
|
|
287
|
+
if (!isUnprocessableEntity(error)) throw error;
|
|
288
|
+
await settle(service.db, {
|
|
289
|
+
workspaceId: input.workspaceId,
|
|
290
|
+
runId: run.id,
|
|
291
|
+
status: "failed",
|
|
292
|
+
errorCode: "dispatch_failed"
|
|
293
|
+
});
|
|
294
|
+
return { action: "failed", reason: "dispatch_failed" };
|
|
295
|
+
}
|
|
296
|
+
const denial = await admit(catalogService, {
|
|
297
|
+
accountId: input.accountId,
|
|
267
298
|
workspaceId: input.workspaceId,
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
299
|
+
model,
|
|
300
|
+
requestedAgentRuns: 1
|
|
301
|
+
});
|
|
302
|
+
if (denial) {
|
|
303
|
+
await settle(service.db, {
|
|
304
|
+
workspaceId: input.workspaceId,
|
|
305
|
+
runId: run.id,
|
|
306
|
+
status: "failed",
|
|
307
|
+
errorCode: denial
|
|
308
|
+
});
|
|
309
|
+
return { action: "failed", reason: denial };
|
|
310
|
+
}
|
|
311
|
+
const reasoningEffort = template.reasoningEffort ?? catalogSettings.openaiReasoningEffort;
|
|
312
|
+
const sandboxBackend = template.sandboxBackend ?? catalogSettings.sandboxBackend;
|
|
313
|
+
if (sandboxBackend === "selfhosted") {
|
|
314
|
+
await settle(service.db, {
|
|
315
|
+
workspaceId: input.workspaceId,
|
|
316
|
+
runId: run.id,
|
|
317
|
+
status: "skipped",
|
|
318
|
+
errorCode: "interactive_compute_not_allowed"
|
|
319
|
+
});
|
|
320
|
+
return {
|
|
321
|
+
action: "skipped",
|
|
322
|
+
reason: "interactive_compute_not_allowed"
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV1(catalogSettings, {
|
|
326
|
+
modelId: model,
|
|
327
|
+
requestedModelId: template.model,
|
|
328
|
+
modelSource: template.model ? "explicit" : "deployment",
|
|
329
|
+
reasoningEffort,
|
|
330
|
+
reasoningSource: template.reasoningEffort ? "explicit" : "deployment",
|
|
331
|
+
latencyMode: "standard",
|
|
332
|
+
latencyModeSource: "deployment"
|
|
271
333
|
});
|
|
272
|
-
return { action: "skipped", reason: "interactive_compute_not_allowed" };
|
|
273
|
-
}
|
|
274
|
-
const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
|
|
275
|
-
modelId: model,
|
|
276
|
-
requestedModelId: template.model,
|
|
277
|
-
modelSource: template.model ? "explicit" : "deployment",
|
|
278
|
-
reasoningEffort,
|
|
279
|
-
reasoningSource: template.reasoningEffort ? "explicit" : "deployment",
|
|
280
|
-
latencyMode: "standard",
|
|
281
|
-
latencyModeSource: "deployment"
|
|
282
|
-
});
|
|
283
|
-
try {
|
|
284
334
|
const created = await createSession2({
|
|
285
335
|
db: service.db,
|
|
286
336
|
bus: service.bus,
|
|
@@ -328,11 +378,12 @@ function createAutomationActivities(services, options = {}) {
|
|
|
328
378
|
policyRole: template.policyRole,
|
|
329
379
|
firstPartyMcpPermissions: template.firstPartyMcpPermissions,
|
|
330
380
|
firstPartyMcpTools: template.firstPartyMcpTools,
|
|
381
|
+
retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX),
|
|
331
382
|
createIdempotencyKey: `automation-run:${run.id}`,
|
|
332
383
|
subjectId: accepted.serviceSubjectId
|
|
333
384
|
});
|
|
334
385
|
await recordUsage(
|
|
335
|
-
{ db: service.db, settings:
|
|
386
|
+
{ db: service.db, settings: catalogSettings },
|
|
336
387
|
{
|
|
337
388
|
accountId: input.accountId,
|
|
338
389
|
workspaceId: input.workspaceId,
|
|
@@ -366,17 +417,14 @@ function createAutomationActivities(services, options = {}) {
|
|
|
366
417
|
});
|
|
367
418
|
return { action: "skipped", reason: "authority_revoked" };
|
|
368
419
|
}
|
|
369
|
-
await settle(service.db, {
|
|
370
|
-
workspaceId: input.workspaceId,
|
|
371
|
-
runId: run.id,
|
|
372
|
-
status: "failed",
|
|
373
|
-
errorCode: "dispatch_failed"
|
|
374
|
-
});
|
|
375
420
|
throw error;
|
|
376
421
|
}
|
|
377
422
|
}
|
|
378
423
|
};
|
|
379
424
|
}
|
|
425
|
+
function isUnprocessableEntity(error) {
|
|
426
|
+
return error instanceof Error && "status" in error && error.status === 422;
|
|
427
|
+
}
|
|
380
428
|
|
|
381
429
|
// src/activities/documents.ts
|
|
382
430
|
import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
|
|
@@ -6082,7 +6130,7 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
6082
6130
|
}
|
|
6083
6131
|
}
|
|
6084
6132
|
try {
|
|
6085
|
-
await settleRetainedProcess(db, {
|
|
6133
|
+
const settlement = await settleRetainedProcess(db, {
|
|
6086
6134
|
accountId: process.accountId,
|
|
6087
6135
|
workspaceId: process.workspaceId,
|
|
6088
6136
|
sessionId: process.sessionId,
|
|
@@ -6094,14 +6142,17 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
6094
6142
|
reason: proof.reason,
|
|
6095
6143
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
6096
6144
|
});
|
|
6145
|
+
if (settlement.process.state === "active") {
|
|
6146
|
+
throw new Error("Retained-process reconciliation returned an active durable process");
|
|
6147
|
+
}
|
|
6097
6148
|
await settleSessionBackgroundCommandForRetainedProcess(db, {
|
|
6098
6149
|
accountId: process.accountId,
|
|
6099
6150
|
workspaceId: process.workspaceId,
|
|
6100
6151
|
sessionId: process.sessionId,
|
|
6101
6152
|
retainedProcessId: process.id,
|
|
6102
|
-
outcome:
|
|
6103
|
-
exitCode:
|
|
6104
|
-
reason: proof.reason
|
|
6153
|
+
outcome: settlement.process.state,
|
|
6154
|
+
exitCode: settlement.process.exitCode,
|
|
6155
|
+
reason: settlement.process.settlementReason ?? proof.reason
|
|
6105
6156
|
});
|
|
6106
6157
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
6107
6158
|
} catch (error) {
|
|
@@ -7225,14 +7276,18 @@ import {
|
|
|
7225
7276
|
ScheduledTaskRunAcceptedExecution,
|
|
7226
7277
|
normalizeAutomaticSessionTitle,
|
|
7227
7278
|
scheduledOccurrencePayloadUtf8Bytes,
|
|
7228
|
-
stableJson
|
|
7279
|
+
stableJson,
|
|
7280
|
+
TurnExecutionPolicyV1
|
|
7229
7281
|
} from "@opengeni/contracts";
|
|
7230
7282
|
import {
|
|
7231
7283
|
defaultSessionMcpServerIds,
|
|
7232
7284
|
openGeniSlackBotMetadata,
|
|
7233
7285
|
requireOpenGeniSlackBotConnection,
|
|
7286
|
+
resolveWorkspaceCatalogSettings as resolveWorkspaceCatalogSettings2,
|
|
7234
7287
|
resolveSessionToolPolicy,
|
|
7235
|
-
|
|
7288
|
+
workspaceCustomModelReference,
|
|
7289
|
+
scheduledSlackBotConnectionId,
|
|
7290
|
+
swapActiveSandbox
|
|
7236
7291
|
} from "@opengeni/core";
|
|
7237
7292
|
import {
|
|
7238
7293
|
addSessionSystemUpdateWithSourceMutation,
|
|
@@ -7240,6 +7295,7 @@ import {
|
|
|
7240
7295
|
createScheduledTaskRun,
|
|
7241
7296
|
createSessionWithIdempotencyKeyResult,
|
|
7242
7297
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
7298
|
+
failScheduledGeneratedSessionRoute,
|
|
7243
7299
|
getScheduledTask as getScheduledTask2,
|
|
7244
7300
|
getScheduledTaskRunAcceptedExecution,
|
|
7245
7301
|
getScheduledTaskRunByProducerKey,
|
|
@@ -7250,6 +7306,8 @@ import {
|
|
|
7250
7306
|
getScheduledTaskPersonalResourceAuthoritySubject,
|
|
7251
7307
|
getScheduledTaskRevisionAuthority,
|
|
7252
7308
|
getScheduledTaskXaiProviderAccountAuthoritySnapshot,
|
|
7309
|
+
getEnrollment,
|
|
7310
|
+
getSandbox,
|
|
7253
7311
|
getRig as getRig2,
|
|
7254
7312
|
getScheduledScopedRigVersionMetadata,
|
|
7255
7313
|
getNestedAgentDepthDeploymentPolicy,
|
|
@@ -7257,6 +7315,8 @@ import {
|
|
|
7257
7315
|
getVariableSet,
|
|
7258
7316
|
isCodexBilledModel,
|
|
7259
7317
|
initializeSessionStartAtomically,
|
|
7318
|
+
lockActiveWorkspaceGatewayCustomModelForAdmission,
|
|
7319
|
+
lockActiveWorkspaceOpenRouterCustomModelForAdmission,
|
|
7260
7320
|
materializeScheduledTaskReusableSessionFromRun,
|
|
7261
7321
|
listEnabledMcpCapabilityServerIds,
|
|
7262
7322
|
listInstalledApiIntegrationServerIdsForDelegations,
|
|
@@ -7275,8 +7335,9 @@ import {
|
|
|
7275
7335
|
withSessionActivityRlsContext
|
|
7276
7336
|
} from "@opengeni/db";
|
|
7277
7337
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
7278
|
-
import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
|
|
7338
|
+
import { resolveFirstPartyMcpToolPolicy, resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12 } from "@opengeni/config";
|
|
7279
7339
|
import { Context as Context2 } from "@temporalio/activity";
|
|
7340
|
+
import { createHash as createHash5 } from "crypto";
|
|
7280
7341
|
|
|
7281
7342
|
// src/scheduled-alert-occurrence.ts
|
|
7282
7343
|
import { createHash as createHash4 } from "crypto";
|
|
@@ -7412,6 +7473,10 @@ function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
|
|
|
7412
7473
|
workflowId: info.workflowExecution.workflowId
|
|
7413
7474
|
})}`;
|
|
7414
7475
|
}
|
|
7476
|
+
function scheduledTaskGeneratedSessionCreateIdempotencyKey(producerKey) {
|
|
7477
|
+
const digest = createHash5("sha256").update("scheduled-task-run\0", "utf8").update(producerKey, "utf8").digest("hex");
|
|
7478
|
+
return `scheduled-task-run:${digest}`;
|
|
7479
|
+
}
|
|
7415
7480
|
function scheduledTaskSessionTitle(taskName) {
|
|
7416
7481
|
return normalizeAutomaticSessionTitle(taskName) ?? "Scheduled run";
|
|
7417
7482
|
}
|
|
@@ -7432,11 +7497,20 @@ async function stampScheduledSessionTitle(db, input) {
|
|
|
7432
7497
|
function createScheduledTaskActivities(services) {
|
|
7433
7498
|
return {
|
|
7434
7499
|
dispatchScheduledTaskRun: async (input) => {
|
|
7435
|
-
const
|
|
7436
|
-
const { settings, db, bus, wakeSessionWorkflow } = service;
|
|
7500
|
+
const baseService = await services();
|
|
7437
7501
|
if (input.triggerType !== "scheduled" && !input.initiator) {
|
|
7438
7502
|
return { action: "blocked", reason: "malformed_manual_trigger" };
|
|
7439
7503
|
}
|
|
7504
|
+
const { db, bus, wakeSessionWorkflow } = baseService;
|
|
7505
|
+
const settingsForTask = async (task2, retainedProductModelId) => (await resolveWorkspaceCatalogSettings2(
|
|
7506
|
+
db,
|
|
7507
|
+
baseService.catalogSourceSettings ?? baseService.settings,
|
|
7508
|
+
{
|
|
7509
|
+
accountId: task2.accountId,
|
|
7510
|
+
workspaceId: task2.workspaceId,
|
|
7511
|
+
...retainedProductModelId !== void 0 ? { retainedProductModelId } : {}
|
|
7512
|
+
}
|
|
7513
|
+
)).settings;
|
|
7440
7514
|
const stableProducerKey = scheduledTaskRunProducerKey(input);
|
|
7441
7515
|
const priorRun = await getScheduledTaskRunByProducerKey(db, {
|
|
7442
7516
|
workspaceId: input.workspaceId,
|
|
@@ -7466,11 +7540,16 @@ function createScheduledTaskActivities(services) {
|
|
|
7466
7540
|
});
|
|
7467
7541
|
}
|
|
7468
7542
|
if (priorRun.status === "queued") {
|
|
7543
|
+
const acceptedPolicy = acceptedExecution.turnExecutionPolicy ? TurnExecutionPolicyV1.parse(acceptedExecution.turnExecutionPolicy) : null;
|
|
7544
|
+
const settings2 = await settingsForTask(
|
|
7545
|
+
acceptedExecution.task,
|
|
7546
|
+
acceptedPolicy?.productModelId ?? acceptedExecution.targetSessionExecution?.model ?? acceptedExecution.resolvedModel
|
|
7547
|
+
);
|
|
7469
7548
|
try {
|
|
7470
7549
|
return await recoverBoundScheduledTaskDispatch({
|
|
7471
7550
|
db,
|
|
7472
7551
|
bus,
|
|
7473
|
-
settings,
|
|
7552
|
+
settings: settings2,
|
|
7474
7553
|
wakeSessionWorkflow,
|
|
7475
7554
|
run: priorRun,
|
|
7476
7555
|
acceptedExecution,
|
|
@@ -7621,12 +7700,44 @@ function createScheduledTaskActivities(services) {
|
|
|
7621
7700
|
throw new Error("scheduled authority classes have different causal humans");
|
|
7622
7701
|
}
|
|
7623
7702
|
const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
|
|
7703
|
+
const acceptedTargetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.runMode === "reusable_session" ? task.reusableSessionId : null;
|
|
7704
|
+
const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
|
|
7705
|
+
db,
|
|
7706
|
+
task.workspaceId,
|
|
7707
|
+
acceptedTargetSessionId,
|
|
7708
|
+
taskAuthoritySubjectId
|
|
7709
|
+
) : null;
|
|
7710
|
+
const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
|
|
7624
7711
|
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
7625
7712
|
const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
7626
|
-
|
|
7713
|
+
let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
7714
|
+
let sandboxOs = "linux";
|
|
7627
7715
|
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
7628
7716
|
const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
|
|
7629
7717
|
const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
7718
|
+
if (generatedTarget && task.agentConfig.machineTarget) {
|
|
7719
|
+
const access = {
|
|
7720
|
+
accountId: task.accountId,
|
|
7721
|
+
workspaceId: task.workspaceId,
|
|
7722
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
7723
|
+
};
|
|
7724
|
+
const machine = await getSandbox(
|
|
7725
|
+
db,
|
|
7726
|
+
access,
|
|
7727
|
+
task.agentConfig.machineTarget.targetSandboxId
|
|
7728
|
+
);
|
|
7729
|
+
if (!machine || machine.kind !== "selfhosted" || !machine.enrollmentId || machine.scope === "user") {
|
|
7730
|
+
throw new Error("scheduled Connected Machine target is unavailable");
|
|
7731
|
+
}
|
|
7732
|
+
const enrollment = await getEnrollment(db, access, machine.enrollmentId);
|
|
7733
|
+
if (!enrollment || enrollment.status !== "active") {
|
|
7734
|
+
throw new Error("scheduled Connected Machine enrollment is unavailable");
|
|
7735
|
+
}
|
|
7736
|
+
sandboxBackend = "selfhosted";
|
|
7737
|
+
sandboxOs = enrollment.os;
|
|
7738
|
+
} else if (generatedTarget && sandboxBackend === "selfhosted") {
|
|
7739
|
+
throw new Error("self-hosted scheduled task has no Connected Machine target");
|
|
7740
|
+
}
|
|
7630
7741
|
const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
|
|
7631
7742
|
const workspace = await requireWorkspace(db, task.workspaceId);
|
|
7632
7743
|
const deployment = await getNestedAgentDepthDeploymentPolicy(db);
|
|
@@ -7646,13 +7757,6 @@ function createScheduledTaskActivities(services) {
|
|
|
7646
7757
|
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7647
7758
|
};
|
|
7648
7759
|
})() : null;
|
|
7649
|
-
const acceptedTargetSessionId = generatedTarget ? null : task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7650
|
-
const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
|
|
7651
|
-
db,
|
|
7652
|
-
task.workspaceId,
|
|
7653
|
-
acceptedTargetSessionId,
|
|
7654
|
-
taskAuthoritySubjectId
|
|
7655
|
-
) : null;
|
|
7656
7760
|
const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
|
|
7657
7761
|
const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
|
|
7658
7762
|
listEnabledMcpCapabilityServerIds(db, task.workspaceId),
|
|
@@ -7807,20 +7911,53 @@ function createScheduledTaskActivities(services) {
|
|
|
7807
7911
|
if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
|
|
7808
7912
|
throw new Error("scheduled personal-resource execution has no causal human");
|
|
7809
7913
|
}
|
|
7810
|
-
const admissionDenial = await agentRunAdmissionDenial(
|
|
7811
|
-
|
|
7812
|
-
|
|
7813
|
-
|
|
7814
|
-
|
|
7815
|
-
|
|
7914
|
+
const admissionDenial = await agentRunAdmissionDenial(
|
|
7915
|
+
{ ...baseService, settings },
|
|
7916
|
+
{
|
|
7917
|
+
accountId: task.accountId,
|
|
7918
|
+
workspaceId: task.workspaceId,
|
|
7919
|
+
model: targetSessionExecution?.model ?? model,
|
|
7920
|
+
requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
|
|
7921
|
+
}
|
|
7922
|
+
);
|
|
7816
7923
|
if (admissionDenial) {
|
|
7817
7924
|
return { action: "blocked", reason: admissionDenial };
|
|
7818
7925
|
}
|
|
7926
|
+
const acceptedModel = targetSessionExecution?.model ?? model;
|
|
7927
|
+
const acceptedReasoningEffort = targetSessionExecution?.reasoningEffort ?? reasoningEffort;
|
|
7928
|
+
const acceptedLatencyMode = targetSessionExecution?.latencyMode ?? "standard";
|
|
7929
|
+
const acceptedCustomModel = generatedTarget ? workspaceCustomModelReference(settings, acceptedModel) : null;
|
|
7930
|
+
const beforeFreshAgentRunCommit = acceptedCustomModel ? async (tx) => {
|
|
7931
|
+
const active = acceptedCustomModel.providerKind === "openrouter" ? await lockActiveWorkspaceOpenRouterCustomModelForAdmission(tx, {
|
|
7932
|
+
accountId: task.accountId,
|
|
7933
|
+
workspaceId: task.workspaceId,
|
|
7934
|
+
upstreamModelId: acceptedCustomModel.upstreamModelId
|
|
7935
|
+
}) : await lockActiveWorkspaceGatewayCustomModelForAdmission(tx, {
|
|
7936
|
+
accountId: task.accountId,
|
|
7937
|
+
workspaceId: task.workspaceId,
|
|
7938
|
+
upstreamModelId: acceptedCustomModel.upstreamModelId
|
|
7939
|
+
});
|
|
7940
|
+
if (!active) {
|
|
7941
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
7942
|
+
"scheduled_model_unavailable",
|
|
7943
|
+
`model is not available: ${acceptedModel}`
|
|
7944
|
+
);
|
|
7945
|
+
}
|
|
7946
|
+
} : void 0;
|
|
7947
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV12(settings, {
|
|
7948
|
+
modelId: acceptedModel,
|
|
7949
|
+
requestedModelId: generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
|
|
7950
|
+
modelSource: generatedTarget ? task.agentConfig.model ? "explicit" : "deployment" : "session",
|
|
7951
|
+
reasoningEffort: acceptedReasoningEffort,
|
|
7952
|
+
reasoningSource: generatedTarget ? task.agentConfig.reasoningEffort ? "explicit" : "deployment" : "session",
|
|
7953
|
+
latencyMode: acceptedLatencyMode,
|
|
7954
|
+
latencyModeSource: generatedTarget ? "deployment" : "session"
|
|
7955
|
+
});
|
|
7819
7956
|
const deferredEvents = [];
|
|
7820
7957
|
const executeDispatch = async (dispatchDb, deferPublications) => {
|
|
7821
7958
|
const admittedRunId = crypto.randomUUID();
|
|
7822
7959
|
const generatedSessionBinding = generatedTarget ? {
|
|
7823
|
-
createIdempotencyKey: alertResponderSessionCreateIdempotencyKey ?? (task.runMode === "reusable_session" ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}` :
|
|
7960
|
+
createIdempotencyKey: alertResponderSessionCreateIdempotencyKey ?? (task.runMode === "reusable_session" ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}` : scheduledTaskGeneratedSessionCreateIdempotencyKey(stableProducerKey)),
|
|
7824
7961
|
effectiveMaxNestedAgentDepth: generatedSessionDepthPolicy.effectiveMaxNestedAgentDepth,
|
|
7825
7962
|
nestedAgentDepthPolicySource: generatedSessionDepthPolicy.nestedAgentDepthPolicySource,
|
|
7826
7963
|
codexCompactionMode: generatedSessionDepthPolicy.codexCompactionMode
|
|
@@ -7831,8 +7968,9 @@ function createScheduledTaskActivities(services) {
|
|
|
7831
7968
|
resolvedModel: model,
|
|
7832
7969
|
resolvedReasoningEffort: reasoningEffort,
|
|
7833
7970
|
resolvedLatencyMode: "standard",
|
|
7971
|
+
turnExecutionPolicy,
|
|
7834
7972
|
resolvedSandboxBackend: sandboxBackend,
|
|
7835
|
-
resolvedSandboxOs:
|
|
7973
|
+
resolvedSandboxOs: sandboxOs,
|
|
7836
7974
|
resolvedTools: taskTools,
|
|
7837
7975
|
resolvedFirstPartyMcpTools: firstPartyMcpTools,
|
|
7838
7976
|
resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
@@ -7885,7 +8023,8 @@ function createScheduledTaskActivities(services) {
|
|
|
7885
8023
|
triggerType: input.triggerType,
|
|
7886
8024
|
producerKey: stableProducerKey,
|
|
7887
8025
|
scheduledAt: null,
|
|
7888
|
-
acceptedExecutionSnapshot: acceptedExecution
|
|
8026
|
+
acceptedExecutionSnapshot: acceptedExecution,
|
|
8027
|
+
...beforeFreshAgentRunCommit ? { beforeFreshAgentRunCommit } : {}
|
|
7889
8028
|
});
|
|
7890
8029
|
await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
|
|
7891
8030
|
if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
|
|
@@ -8038,7 +8177,7 @@ function createScheduledTaskActivities(services) {
|
|
|
8038
8177
|
reasoningEffort,
|
|
8039
8178
|
latencyMode: "standard",
|
|
8040
8179
|
sandboxBackend,
|
|
8041
|
-
sandboxOs
|
|
8180
|
+
sandboxOs,
|
|
8042
8181
|
variableSetId: task.variableSetId ?? null,
|
|
8043
8182
|
rigId: frozenRigId,
|
|
8044
8183
|
rigVersionId: frozenRigVersionId,
|
|
@@ -8138,6 +8277,16 @@ function createScheduledTaskActivities(services) {
|
|
|
8138
8277
|
);
|
|
8139
8278
|
}
|
|
8140
8279
|
}
|
|
8280
|
+
session = await seedScheduledGeneratedSessionRoute({
|
|
8281
|
+
db: dispatchDb,
|
|
8282
|
+
bus,
|
|
8283
|
+
settings,
|
|
8284
|
+
task,
|
|
8285
|
+
runId: run.id,
|
|
8286
|
+
session,
|
|
8287
|
+
deferPublications,
|
|
8288
|
+
deferredEvents
|
|
8289
|
+
});
|
|
8141
8290
|
let workflowId;
|
|
8142
8291
|
if (alertOccurrence) {
|
|
8143
8292
|
const started = await initializeSessionStartAtomically(dispatchDb, {
|
|
@@ -8520,6 +8669,10 @@ function createScheduledTaskActivities(services) {
|
|
|
8520
8669
|
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8521
8670
|
return { action: "blocked", reason: error.reason };
|
|
8522
8671
|
}
|
|
8672
|
+
const terminalError = scheduledRunRecoveryTerminalError(error);
|
|
8673
|
+
if (terminalError?.code === "scheduled_model_unavailable") {
|
|
8674
|
+
return scheduledRunTerminalResult(terminalError.code);
|
|
8675
|
+
}
|
|
8523
8676
|
throw error;
|
|
8524
8677
|
}
|
|
8525
8678
|
if (dispatchOutcome.kind === "replay") {
|
|
@@ -8531,7 +8684,6 @@ function createScheduledTaskActivities(services) {
|
|
|
8531
8684
|
acceptedExecution: dispatchOutcome.acceptedExecution
|
|
8532
8685
|
});
|
|
8533
8686
|
}
|
|
8534
|
-
if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
|
|
8535
8687
|
for (const deferred of deferredEvents) {
|
|
8536
8688
|
await publishDurableSessionEvents(
|
|
8537
8689
|
bus,
|
|
@@ -8540,6 +8692,7 @@ function createScheduledTaskActivities(services) {
|
|
|
8540
8692
|
deferred.events
|
|
8541
8693
|
);
|
|
8542
8694
|
}
|
|
8695
|
+
if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
|
|
8543
8696
|
const result = dispatchOutcome.result;
|
|
8544
8697
|
if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
|
|
8545
8698
|
await wakeSessionWorkflow({
|
|
@@ -8714,6 +8867,59 @@ async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
|
|
|
8714
8867
|
idempotencyKey: `usage:scheduled_task.fired:${run.id}`
|
|
8715
8868
|
});
|
|
8716
8869
|
}
|
|
8870
|
+
async function seedScheduledGeneratedSessionRoute(input) {
|
|
8871
|
+
const target = input.task.agentConfig.machineTarget;
|
|
8872
|
+
if (!target) return input.session;
|
|
8873
|
+
const seeded = await swapActiveSandbox(
|
|
8874
|
+
{
|
|
8875
|
+
db: input.db,
|
|
8876
|
+
settings: input.settings,
|
|
8877
|
+
bus: input.bus
|
|
8878
|
+
},
|
|
8879
|
+
{
|
|
8880
|
+
accountId: input.task.accountId,
|
|
8881
|
+
workspaceId: input.task.workspaceId,
|
|
8882
|
+
sessionId: input.session.id,
|
|
8883
|
+
sessionBackend: input.session.sandboxBackend,
|
|
8884
|
+
sessionGroupId: input.session.sandboxGroupId
|
|
8885
|
+
},
|
|
8886
|
+
target.targetSandboxId,
|
|
8887
|
+
target.workingDir ?? null
|
|
8888
|
+
);
|
|
8889
|
+
if (seeded.swapped) {
|
|
8890
|
+
return await requireSession(input.db, input.task.workspaceId, input.session.id);
|
|
8891
|
+
}
|
|
8892
|
+
const failed = await failScheduledGeneratedSessionRoute(input.db, {
|
|
8893
|
+
accountId: input.task.accountId,
|
|
8894
|
+
workspaceId: input.task.workspaceId,
|
|
8895
|
+
taskId: input.task.id,
|
|
8896
|
+
runId: input.runId,
|
|
8897
|
+
sessionId: input.session.id,
|
|
8898
|
+
error: "scheduled_machine_unavailable"
|
|
8899
|
+
});
|
|
8900
|
+
if (failed.action === "advanced") {
|
|
8901
|
+
throw new Error(
|
|
8902
|
+
`scheduled run advanced to ${failed.status} while its Connected Machine route was being established`
|
|
8903
|
+
);
|
|
8904
|
+
}
|
|
8905
|
+
if (failed.events.length > 0) {
|
|
8906
|
+
if (input.deferPublications) {
|
|
8907
|
+
input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
|
|
8908
|
+
} else {
|
|
8909
|
+
await publishDurableSessionEvents(
|
|
8910
|
+
input.bus,
|
|
8911
|
+
input.task.workspaceId,
|
|
8912
|
+
input.session.id,
|
|
8913
|
+
failed.events
|
|
8914
|
+
);
|
|
8915
|
+
}
|
|
8916
|
+
}
|
|
8917
|
+
const error = failed.action === "terminal" ? failed.error ?? "scheduled_run_terminal" : failed.error;
|
|
8918
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8919
|
+
error,
|
|
8920
|
+
seeded.reason ?? "scheduled Connected Machine target is unavailable"
|
|
8921
|
+
);
|
|
8922
|
+
}
|
|
8717
8923
|
async function recoverBoundScheduledTaskDispatch(input) {
|
|
8718
8924
|
const { task } = input.acceptedExecution;
|
|
8719
8925
|
if (input.run.status !== "queued") {
|
|
@@ -8866,6 +9072,18 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
8866
9072
|
sessionId: session.id
|
|
8867
9073
|
});
|
|
8868
9074
|
}
|
|
9075
|
+
if (generatedSession) {
|
|
9076
|
+
session = await seedScheduledGeneratedSessionRoute({
|
|
9077
|
+
db: input.db,
|
|
9078
|
+
bus: input.bus,
|
|
9079
|
+
settings: input.settings,
|
|
9080
|
+
task,
|
|
9081
|
+
runId: input.run.id,
|
|
9082
|
+
session,
|
|
9083
|
+
deferPublications: false,
|
|
9084
|
+
deferredEvents: []
|
|
9085
|
+
});
|
|
9086
|
+
}
|
|
8869
9087
|
if (session.status === "cancelled") {
|
|
8870
9088
|
await markScheduledTaskRunSkippedIfQueued(input.db, {
|
|
8871
9089
|
workspaceId: task.workspaceId,
|
|
@@ -8901,7 +9119,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
|
|
|
8901
9119
|
label: "OpenGeni scheduler",
|
|
8902
9120
|
scheduledTaskId: task.id,
|
|
8903
9121
|
scheduledTaskRunId: canonicalGeneratedRunId
|
|
8904
|
-
}) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
|
|
9122
|
+
}) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
|
|
8905
9123
|
throw new ScheduledRunTerminalAuthorityError(
|
|
8906
9124
|
"scheduled_generated_session_changed",
|
|
8907
9125
|
"scheduled generated session differs from accepted execution"
|
|
@@ -9145,7 +9363,7 @@ function knowledgeSourceSyncEffectivelyPaused(task) {
|
|
|
9145
9363
|
}
|
|
9146
9364
|
|
|
9147
9365
|
// src/activities/site-auth-maintenance.ts
|
|
9148
|
-
import { resolveTurnExecutionPolicyV1 as
|
|
9366
|
+
import { resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV13 } from "@opengeni/config";
|
|
9149
9367
|
import {
|
|
9150
9368
|
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS as DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2,
|
|
9151
9369
|
SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY,
|
|
@@ -9155,7 +9373,8 @@ import {
|
|
|
9155
9373
|
assertWorkspaceModelPolicyAllows as assertWorkspaceModelPolicyAllows2,
|
|
9156
9374
|
canonicalConfiguredModel as canonicalConfiguredModel2,
|
|
9157
9375
|
createAndStartSessionWithOutcome as createAndStartSessionWithOutcome2,
|
|
9158
|
-
recordWorkspaceUsage as recordWorkspaceUsage2
|
|
9376
|
+
recordWorkspaceUsage as recordWorkspaceUsage2,
|
|
9377
|
+
resolveCatalogSettings as resolveCatalogSettings2
|
|
9159
9378
|
} from "@opengeni/core";
|
|
9160
9379
|
import {
|
|
9161
9380
|
claimSiteAuthMaintenance,
|
|
@@ -9231,23 +9450,20 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
|
|
|
9231
9450
|
return { maintainSiteAuthConnections };
|
|
9232
9451
|
}
|
|
9233
9452
|
async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
9234
|
-
const
|
|
9453
|
+
const catalogSettings = (await resolveCatalogSettings2(service.db, service.catalogSourceSettings ?? service.settings)).settings;
|
|
9454
|
+
const catalogService = { ...service, settings: catalogSettings };
|
|
9455
|
+
const model = canonicalConfiguredModel2(catalogSettings, catalogSettings.openaiModel);
|
|
9235
9456
|
if (!model) throw new Error("site auth maintenance has no configured model");
|
|
9236
|
-
await dependencies.assertModelPolicy(
|
|
9237
|
-
|
|
9238
|
-
service.settings,
|
|
9239
|
-
maintenance.workspaceId,
|
|
9240
|
-
model
|
|
9241
|
-
);
|
|
9242
|
-
const denial = await dependencies.admit(service, {
|
|
9457
|
+
await dependencies.assertModelPolicy(service.db, catalogSettings, maintenance.workspaceId, model);
|
|
9458
|
+
const denial = await dependencies.admit(catalogService, {
|
|
9243
9459
|
accountId: maintenance.accountId,
|
|
9244
9460
|
workspaceId: maintenance.workspaceId,
|
|
9245
9461
|
model,
|
|
9246
9462
|
requestedAgentRuns: 1
|
|
9247
9463
|
});
|
|
9248
9464
|
if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
|
|
9249
|
-
const reasoningEffort =
|
|
9250
|
-
const turnExecutionPolicy =
|
|
9465
|
+
const reasoningEffort = catalogSettings.openaiReasoningEffort;
|
|
9466
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV13(catalogSettings, {
|
|
9251
9467
|
modelId: model,
|
|
9252
9468
|
requestedModelId: null,
|
|
9253
9469
|
modelSource: "deployment",
|
|
@@ -9284,7 +9500,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
9284
9500
|
model,
|
|
9285
9501
|
reasoningEffort,
|
|
9286
9502
|
turnExecutionPolicy,
|
|
9287
|
-
sandboxBackend:
|
|
9503
|
+
sandboxBackend: catalogSettings.sandboxBackend,
|
|
9288
9504
|
metadata: {
|
|
9289
9505
|
role: "site_auth_maintenance",
|
|
9290
9506
|
[SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY]: maintenance.siteAuthConnectionId,
|
|
@@ -9308,7 +9524,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
9308
9524
|
});
|
|
9309
9525
|
try {
|
|
9310
9526
|
await dependencies.recordUsage(
|
|
9311
|
-
{ db: service.db, settings:
|
|
9527
|
+
{ db: service.db, settings: catalogSettings },
|
|
9312
9528
|
{
|
|
9313
9529
|
accountId: maintenance.accountId,
|
|
9314
9530
|
workspaceId: maintenance.workspaceId,
|
|
@@ -9402,6 +9618,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9402
9618
|
const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
|
|
9403
9619
|
const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
|
|
9404
9620
|
const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
|
|
9621
|
+
const recordWorkerDeathRecoveryMetricsFn = overrides.recordWorkerDeathRecoveryMetrics ?? recordWorkerDeathRecoveryMetrics;
|
|
9405
9622
|
async function failSessionAttempt(input) {
|
|
9406
9623
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
9407
9624
|
const session = await requireSessionFn(db, input.workspaceId, input.sessionId);
|
|
@@ -9624,6 +9841,10 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9624
9841
|
if (result.action === "stale" || result.action === "unclaimed") {
|
|
9625
9842
|
return { action: result.action };
|
|
9626
9843
|
}
|
|
9844
|
+
recordWorkerDeathRecoveryMetricsFn(observability, {
|
|
9845
|
+
outcome: result.action === "exceeded" ? "exhausted" : "recovering",
|
|
9846
|
+
timeoutType: input.timeoutType
|
|
9847
|
+
});
|
|
9627
9848
|
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
9628
9849
|
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
9629
9850
|
if (result.action === "exceeded") {
|
|
@@ -9825,4 +10046,4 @@ export {
|
|
|
9825
10046
|
createControlActivities,
|
|
9826
10047
|
createControlActivitiesFromServices
|
|
9827
10048
|
};
|
|
9828
|
-
//# sourceMappingURL=activities-control-
|
|
10049
|
+
//# sourceMappingURL=activities-control-SGU2WEVB.js.map
|