@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +10 -2
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +3 -4
- package/dist/activities/agent-turn/codex.d.ts +8 -7
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +10 -1
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
- package/dist/activities/agent-turn/turn-context.d.ts +3 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +2 -1
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/goals.d.ts +11 -0
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/activities/types.d.ts +8 -0
- package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
- package/dist/activities-control-SGU2WEVB.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
- package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
- package/dist/chunk-S2QVRVJW.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
- package/dist/chunk-TJIPCHMP.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +69 -6
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +29 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/dist/workflow-bundle.js +26 -3
- package/dist/workflows/activities.d.ts +5 -0
- package/dist/workflows/automations.d.ts +5 -0
- package/package.json +18 -18
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +28 -6
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +53 -72
- package/src/activities/agent-turn/codex.ts +21 -12
- package/src/activities/agent-turn/compaction-prep.ts +52 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +88 -42
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +20 -14
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +106 -14
- package/src/activities/agent-turn/tool-environment.ts +72 -14
- package/src/activities/agent-turn/tool-policy.ts +7 -6
- package/src/activities/agent-turn/turn-context.ts +12 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +72 -2
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +117 -35
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +248 -30
- package/src/activities/session-state.ts +8 -1
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +6 -0
- package/src/activities.ts +1 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +8 -4
- package/src/http.ts +3 -0
- package/src/index.ts +38 -4
- package/src/observability-metrics.ts +202 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/src/workflows/activities.ts +15 -0
- package/src/workflows/automations.ts +21 -2
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
package/dist/index.js
CHANGED
|
@@ -12,10 +12,12 @@ import {
|
|
|
12
12
|
turnWorkerConcurrencyLogFields
|
|
13
13
|
} from "./chunk-GRFQTTSQ.js";
|
|
14
14
|
import {
|
|
15
|
+
initializeWorkerOutcomeMetrics,
|
|
15
16
|
normalizeTurnTaskQueueStats,
|
|
16
17
|
observabilityEventLogger,
|
|
18
|
+
startSessionRecoveryMonitor,
|
|
17
19
|
startTurnCapacityMonitor
|
|
18
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-TJIPCHMP.js";
|
|
19
21
|
|
|
20
22
|
// src/index.ts
|
|
21
23
|
import {
|
|
@@ -28,6 +30,7 @@ import {
|
|
|
28
30
|
} from "@opengeni/config";
|
|
29
31
|
import {
|
|
30
32
|
assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
|
|
33
|
+
countSessionRecoveryBacklog,
|
|
31
34
|
createDb,
|
|
32
35
|
markSessionWorkflowWakeDelivered
|
|
33
36
|
} from "@opengeni/db";
|
|
@@ -87,6 +90,14 @@ var automationActivity = proxyActivities(
|
|
|
87
90
|
}
|
|
88
91
|
}
|
|
89
92
|
);
|
|
93
|
+
var automationFailureActivity = proxyActivities({
|
|
94
|
+
startToCloseTimeout: "2 minutes",
|
|
95
|
+
retry: {
|
|
96
|
+
initialInterval: "1 second",
|
|
97
|
+
backoffCoefficient: 2,
|
|
98
|
+
maximumInterval: "30 seconds"
|
|
99
|
+
}
|
|
100
|
+
});
|
|
90
101
|
var goalActivity = proxyActivities({
|
|
91
102
|
startToCloseTimeout: "30 seconds",
|
|
92
103
|
retry: {
|
|
@@ -120,6 +131,7 @@ import {
|
|
|
120
131
|
dbSql
|
|
121
132
|
} from "@opengeni/db";
|
|
122
133
|
import { requireSessionEventDurableFanoutCapability } from "@opengeni/events";
|
|
134
|
+
import { githubAppBotIdentityWarnings } from "@opengeni/github";
|
|
123
135
|
function createWorkerHttpHandler(input) {
|
|
124
136
|
const { settings, observability, checks, lifecycle } = input;
|
|
125
137
|
const timeoutMs = input.timeoutMs ?? 2e3;
|
|
@@ -131,12 +143,14 @@ function createWorkerHttpHandler(input) {
|
|
|
131
143
|
if (url.pathname === "/healthz") {
|
|
132
144
|
const state = lifecycle?.state();
|
|
133
145
|
const ok = state !== "failed" && state !== "stopped";
|
|
146
|
+
const warnings = githubAppBotIdentityWarnings(settings);
|
|
134
147
|
return Response.json(
|
|
135
148
|
{
|
|
136
149
|
service: settings.serviceName,
|
|
137
150
|
environment: settings.environment,
|
|
138
151
|
deploymentRevision: settings.deploymentRevision,
|
|
139
152
|
...lifecycle ? { role: lifecycle.role, state } : {},
|
|
153
|
+
...warnings.length > 0 ? { warnings } : {},
|
|
140
154
|
ok
|
|
141
155
|
},
|
|
142
156
|
{ status: ok ? 200 : 503 }
|
|
@@ -244,6 +258,7 @@ async function withTimeout(promise, timeoutMs) {
|
|
|
244
258
|
|
|
245
259
|
// src/index.ts
|
|
246
260
|
import {
|
|
261
|
+
resolveCatalogSettings,
|
|
247
262
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
248
263
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
249
264
|
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
@@ -720,6 +735,29 @@ function safeWarn(observability, message, attributes) {
|
|
|
720
735
|
}
|
|
721
736
|
}
|
|
722
737
|
|
|
738
|
+
// src/unhandled-rejection-boundary.ts
|
|
739
|
+
function isDetachedRuntimeMcpLifecycleRejection(reason) {
|
|
740
|
+
try {
|
|
741
|
+
if (!reason || typeof reason !== "object") return false;
|
|
742
|
+
const candidate = reason;
|
|
743
|
+
return candidate.name === "McpLifecycleError" && (candidate.code === "mcp_connect_failed" || candidate.code === "mcp_close_failed") && candidate.origin === "runtime" && typeof candidate.serverId === "string" && candidate.serverId.length > 0;
|
|
744
|
+
} catch {
|
|
745
|
+
return false;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
function reportIsolatedUnhandledRejection(classification) {
|
|
749
|
+
console.error("[worker] isolated detached unhandled rejection", { classification });
|
|
750
|
+
}
|
|
751
|
+
function installWorkerUnhandledRejectionBoundary(runtimeProcess = process, report = reportIsolatedUnhandledRejection) {
|
|
752
|
+
const onUnhandledRejection = (reason) => {
|
|
753
|
+
report(
|
|
754
|
+
isDetachedRuntimeMcpLifecycleRejection(reason) ? "detached_runtime_mcp_lifecycle" : "detached_unknown"
|
|
755
|
+
);
|
|
756
|
+
};
|
|
757
|
+
runtimeProcess.on("unhandledRejection", onUnhandledRejection);
|
|
758
|
+
return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
|
|
759
|
+
}
|
|
760
|
+
|
|
723
761
|
// src/index.ts
|
|
724
762
|
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
725
763
|
var FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
@@ -740,7 +778,7 @@ function temporalWorkflowExecutionNotFound(error) {
|
|
|
740
778
|
return isGrpcServiceError(error) && error.code === 5;
|
|
741
779
|
}
|
|
742
780
|
async function createDefaultWorkerActivities(role, dependencies = {}) {
|
|
743
|
-
return role === "control" ? (await import("./activities-control-
|
|
781
|
+
return role === "control" ? (await import("./activities-control-SGU2WEVB.js")).createControlActivities(dependencies) : (await import("./activities-turn-7EAQXW6Q.js")).createTurnActivities(dependencies);
|
|
744
782
|
}
|
|
745
783
|
function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
|
|
746
784
|
const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
@@ -767,6 +805,7 @@ async function createOpenGeniWorker(options) {
|
|
|
767
805
|
assertSandboxReaperActivityTimeout(settings);
|
|
768
806
|
}
|
|
769
807
|
const observability = options.activityDependencies?.observability ?? createObservability(settings, { component: `worker-${options.role}` });
|
|
808
|
+
initializeWorkerOutcomeMetrics(observability);
|
|
770
809
|
if (options.role === "turn" && options.workflowBundle) {
|
|
771
810
|
throw new Error("workflowBundle is valid only for the control worker role");
|
|
772
811
|
}
|
|
@@ -1189,10 +1228,21 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1189
1228
|
let signaler;
|
|
1190
1229
|
let workerBundle;
|
|
1191
1230
|
let turnCapacityMonitor;
|
|
1231
|
+
let sessionRecoveryMonitor;
|
|
1192
1232
|
const schedules = [];
|
|
1193
1233
|
let httpServer;
|
|
1194
1234
|
let memoryPressureGuard;
|
|
1195
1235
|
try {
|
|
1236
|
+
const resolvedCatalog = await retryStartupDependency(
|
|
1237
|
+
"model catalog",
|
|
1238
|
+
() => resolveCatalogSettings(options.activityDependencies.db, settings),
|
|
1239
|
+
{ ...retryOptions, onRetry }
|
|
1240
|
+
);
|
|
1241
|
+
observability.info("OpenGeni model catalog resolved", {
|
|
1242
|
+
role: options.role,
|
|
1243
|
+
catalogSource: resolvedCatalog.source,
|
|
1244
|
+
catalogVersion: resolvedCatalog.version
|
|
1245
|
+
});
|
|
1196
1246
|
const needsSignaler = options.role === "turn" || !options.activityDependencies.wakeSessionWorkflow || !options.activityDependencies.signalSessionAttemptQuiesced || !options.activityDependencies.inspectSessionAttemptActivity || !options.activityDependencies.signalCodexCapacityWorkflow || !options.activityDependencies.startSandboxReaperWorkflow || !options.activityDependencies.startVideoGenerationWorkflow;
|
|
1197
1247
|
if (needsSignaler) {
|
|
1198
1248
|
signaler = await retryStartupDependency(
|
|
@@ -1235,6 +1285,11 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1235
1285
|
observability,
|
|
1236
1286
|
read: signaler.getTurnTaskQueueStats
|
|
1237
1287
|
});
|
|
1288
|
+
} else {
|
|
1289
|
+
sessionRecoveryMonitor = startSessionRecoveryMonitor({
|
|
1290
|
+
observability,
|
|
1291
|
+
read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db)
|
|
1292
|
+
});
|
|
1238
1293
|
}
|
|
1239
1294
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
1240
1295
|
schedules.push(
|
|
@@ -1267,14 +1322,18 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1267
1322
|
);
|
|
1268
1323
|
}
|
|
1269
1324
|
if (options.http !== false) {
|
|
1325
|
+
const databaseReady = dbReadyCheck(
|
|
1326
|
+
options.http?.readinessDb ?? options.activityDependencies.db,
|
|
1327
|
+
options.databasePosture
|
|
1328
|
+
);
|
|
1270
1329
|
httpServer = startWorkerHttpServer({
|
|
1271
1330
|
settings,
|
|
1272
1331
|
observability,
|
|
1273
1332
|
checks: {
|
|
1274
|
-
db:
|
|
1275
|
-
|
|
1276
|
-
options.
|
|
1277
|
-
|
|
1333
|
+
db: async () => {
|
|
1334
|
+
await databaseReady();
|
|
1335
|
+
await resolveCatalogSettings(options.activityDependencies.db, settings);
|
|
1336
|
+
},
|
|
1278
1337
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
1279
1338
|
temporal: temporalReadyCheck(workerBundle.connection)
|
|
1280
1339
|
},
|
|
@@ -1286,6 +1345,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1286
1345
|
httpServer?.stop(true);
|
|
1287
1346
|
await Promise.allSettled([
|
|
1288
1347
|
turnCapacityMonitor?.close(),
|
|
1348
|
+
sessionRecoveryMonitor?.close(),
|
|
1289
1349
|
workerBundle?.connection.close(),
|
|
1290
1350
|
signaler?.close(),
|
|
1291
1351
|
...schedules.map((schedule) => schedule.close())
|
|
@@ -1306,6 +1366,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1306
1366
|
httpServer?.stop(true);
|
|
1307
1367
|
await Promise.allSettled([
|
|
1308
1368
|
turnCapacityMonitor?.close(),
|
|
1369
|
+
sessionRecoveryMonitor?.close(),
|
|
1309
1370
|
activeWorkerBundle.connection.close(),
|
|
1310
1371
|
activeSignaler?.close(),
|
|
1311
1372
|
...schedules.map((schedule) => schedule.close())
|
|
@@ -1380,6 +1441,7 @@ async function startWorker() {
|
|
|
1380
1441
|
}
|
|
1381
1442
|
const settings = getSettings();
|
|
1382
1443
|
const observability = createObservability(settings, { component: `worker-${role}` });
|
|
1444
|
+
const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
|
|
1383
1445
|
const retryOptions = startupRetryOptions(settings);
|
|
1384
1446
|
const onRetry = (event) => logStartupDependencyRetry(observability, event);
|
|
1385
1447
|
const searchPath = dbSearchPath(settings);
|
|
@@ -1428,6 +1490,7 @@ async function startWorker() {
|
|
|
1428
1490
|
});
|
|
1429
1491
|
} finally {
|
|
1430
1492
|
await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
|
|
1493
|
+
disposeUnhandledRejectionBoundary();
|
|
1431
1494
|
}
|
|
1432
1495
|
}
|
|
1433
1496
|
if (import.meta.main) {
|