@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +10 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  3. package/dist/activities/agent-turn/claim.d.ts +3 -4
  4. package/dist/activities/agent-turn/codex.d.ts +8 -7
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  6. package/dist/activities/agent-turn/errors.d.ts +13 -0
  7. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +10 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  11. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  13. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  16. package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +3 -8
  18. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  19. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  20. package/dist/activities/automations.d.ts +1 -0
  21. package/dist/activities/capabilities.d.ts +2 -1
  22. package/dist/activities/codex-rotation.d.ts +2 -0
  23. package/dist/activities/goals.d.ts +11 -0
  24. package/dist/activities/image-generation-references.d.ts +2 -1
  25. package/dist/activities/scheduled-tasks.d.ts +1 -0
  26. package/dist/activities/session-state.d.ts +2 -1
  27. package/dist/activities/types.d.ts +8 -0
  28. package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
  29. package/dist/activities-control-SGU2WEVB.js.map +1 -0
  30. package/dist/activities-control.d.ts +2 -0
  31. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
  32. package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
  33. package/dist/activities.d.ts +2 -0
  34. package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
  35. package/dist/chunk-S2QVRVJW.js.map +1 -0
  36. package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
  37. package/dist/chunk-TJIPCHMP.js.map +1 -0
  38. package/dist/index.d.ts +1 -0
  39. package/dist/index.js +69 -6
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +29 -0
  42. package/dist/personal-github-git-credentials.d.ts +1 -0
  43. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  44. package/dist/workflow-bundle.js +26 -3
  45. package/dist/workflows/activities.d.ts +5 -0
  46. package/dist/workflows/automations.d.ts +5 -0
  47. package/package.json +18 -18
  48. package/src/activities/agent-run-admission.ts +7 -1
  49. package/src/activities/agent-turn/admission.ts +28 -6
  50. package/src/activities/agent-turn/agent-build.ts +15 -31
  51. package/src/activities/agent-turn/claim.ts +53 -72
  52. package/src/activities/agent-turn/codex.ts +21 -12
  53. package/src/activities/agent-turn/compaction-prep.ts +52 -1
  54. package/src/activities/agent-turn/errors.ts +78 -0
  55. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  56. package/src/activities/agent-turn/finalization.ts +24 -21
  57. package/src/activities/agent-turn/history.ts +5 -1
  58. package/src/activities/agent-turn/list-models.ts +101 -0
  59. package/src/activities/agent-turn/model-usage.ts +88 -42
  60. package/src/activities/agent-turn/quiescence.ts +86 -0
  61. package/src/activities/agent-turn/run-credentials.ts +12 -1
  62. package/src/activities/agent-turn/run.ts +20 -14
  63. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  64. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  65. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  66. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  67. package/src/activities/agent-turn/stream-attempt.ts +106 -14
  68. package/src/activities/agent-turn/tool-environment.ts +72 -14
  69. package/src/activities/agent-turn/tool-policy.ts +7 -6
  70. package/src/activities/agent-turn/turn-context.ts +12 -14
  71. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  72. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  73. package/src/activities/automations.ts +122 -67
  74. package/src/activities/capabilities.ts +72 -2
  75. package/src/activities/codex-capacity.ts +13 -5
  76. package/src/activities/codex-rotation.ts +17 -0
  77. package/src/activities/goals.ts +117 -35
  78. package/src/activities/image-generation-references.ts +9 -9
  79. package/src/activities/retained-screenshots.ts +104 -21
  80. package/src/activities/sandbox-lease.ts +7 -4
  81. package/src/activities/scheduled-tasks.ts +248 -30
  82. package/src/activities/session-state.ts +8 -1
  83. package/src/activities/site-auth-maintenance.ts +12 -12
  84. package/src/activities/types.ts +6 -0
  85. package/src/activities.ts +1 -0
  86. package/src/activity-services.ts +4 -1
  87. package/src/github-rest-mcp.ts +8 -4
  88. package/src/http.ts +3 -0
  89. package/src/index.ts +38 -4
  90. package/src/observability-metrics.ts +202 -0
  91. package/src/personal-github-git-credentials.ts +45 -4
  92. package/src/sandbox-resume.ts +1 -1
  93. package/src/sandbox-routing.ts +7 -4
  94. package/src/unhandled-rejection-boundary.ts +73 -0
  95. package/src/workflows/activities.ts +15 -0
  96. package/src/workflows/automations.ts +21 -2
  97. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  98. package/dist/activities/recording.d.ts +0 -90
  99. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  100. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  101. package/dist/chunk-NAVSAZ56.js.map +0 -1
  102. package/dist/chunk-O4HDX5SC.js.map +0 -1
  103. package/src/activities/recording.ts +0 -279
package/dist/index.js CHANGED
@@ -12,10 +12,12 @@ import {
12
12
  turnWorkerConcurrencyLogFields
13
13
  } from "./chunk-GRFQTTSQ.js";
14
14
  import {
15
+ initializeWorkerOutcomeMetrics,
15
16
  normalizeTurnTaskQueueStats,
16
17
  observabilityEventLogger,
18
+ startSessionRecoveryMonitor,
17
19
  startTurnCapacityMonitor
18
- } from "./chunk-NAVSAZ56.js";
20
+ } from "./chunk-TJIPCHMP.js";
19
21
 
20
22
  // src/index.ts
21
23
  import {
@@ -28,6 +30,7 @@ import {
28
30
  } from "@opengeni/config";
29
31
  import {
30
32
  assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
33
+ countSessionRecoveryBacklog,
31
34
  createDb,
32
35
  markSessionWorkflowWakeDelivered
33
36
  } from "@opengeni/db";
@@ -87,6 +90,14 @@ var automationActivity = proxyActivities(
87
90
  }
88
91
  }
89
92
  );
93
+ var automationFailureActivity = proxyActivities({
94
+ startToCloseTimeout: "2 minutes",
95
+ retry: {
96
+ initialInterval: "1 second",
97
+ backoffCoefficient: 2,
98
+ maximumInterval: "30 seconds"
99
+ }
100
+ });
90
101
  var goalActivity = proxyActivities({
91
102
  startToCloseTimeout: "30 seconds",
92
103
  retry: {
@@ -120,6 +131,7 @@ import {
120
131
  dbSql
121
132
  } from "@opengeni/db";
122
133
  import { requireSessionEventDurableFanoutCapability } from "@opengeni/events";
134
+ import { githubAppBotIdentityWarnings } from "@opengeni/github";
123
135
  function createWorkerHttpHandler(input) {
124
136
  const { settings, observability, checks, lifecycle } = input;
125
137
  const timeoutMs = input.timeoutMs ?? 2e3;
@@ -131,12 +143,14 @@ function createWorkerHttpHandler(input) {
131
143
  if (url.pathname === "/healthz") {
132
144
  const state = lifecycle?.state();
133
145
  const ok = state !== "failed" && state !== "stopped";
146
+ const warnings = githubAppBotIdentityWarnings(settings);
134
147
  return Response.json(
135
148
  {
136
149
  service: settings.serviceName,
137
150
  environment: settings.environment,
138
151
  deploymentRevision: settings.deploymentRevision,
139
152
  ...lifecycle ? { role: lifecycle.role, state } : {},
153
+ ...warnings.length > 0 ? { warnings } : {},
140
154
  ok
141
155
  },
142
156
  { status: ok ? 200 : 503 }
@@ -244,6 +258,7 @@ async function withTimeout(promise, timeoutMs) {
244
258
 
245
259
  // src/index.ts
246
260
  import {
261
+ resolveCatalogSettings,
247
262
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
248
263
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
249
264
  SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
@@ -720,6 +735,29 @@ function safeWarn(observability, message, attributes) {
720
735
  }
721
736
  }
722
737
 
738
+ // src/unhandled-rejection-boundary.ts
739
+ function isDetachedRuntimeMcpLifecycleRejection(reason) {
740
+ try {
741
+ if (!reason || typeof reason !== "object") return false;
742
+ const candidate = reason;
743
+ return candidate.name === "McpLifecycleError" && (candidate.code === "mcp_connect_failed" || candidate.code === "mcp_close_failed") && candidate.origin === "runtime" && typeof candidate.serverId === "string" && candidate.serverId.length > 0;
744
+ } catch {
745
+ return false;
746
+ }
747
+ }
748
+ function reportIsolatedUnhandledRejection(classification) {
749
+ console.error("[worker] isolated detached unhandled rejection", { classification });
750
+ }
751
+ function installWorkerUnhandledRejectionBoundary(runtimeProcess = process, report = reportIsolatedUnhandledRejection) {
752
+ const onUnhandledRejection = (reason) => {
753
+ report(
754
+ isDetachedRuntimeMcpLifecycleRejection(reason) ? "detached_runtime_mcp_lifecycle" : "detached_unknown"
755
+ );
756
+ };
757
+ runtimeProcess.on("unhandledRejection", onUnhandledRejection);
758
+ return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
759
+ }
760
+
723
761
  // src/index.ts
724
762
  var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
725
763
  var FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
@@ -740,7 +778,7 @@ function temporalWorkflowExecutionNotFound(error) {
740
778
  return isGrpcServiceError(error) && error.code === 5;
741
779
  }
742
780
  async function createDefaultWorkerActivities(role, dependencies = {}) {
743
- return role === "control" ? (await import("./activities-control-VZIIV7RX.js")).createControlActivities(dependencies) : (await import("./activities-turn-5RGQDBFV.js")).createTurnActivities(dependencies);
781
+ return role === "control" ? (await import("./activities-control-SGU2WEVB.js")).createControlActivities(dependencies) : (await import("./activities-turn-7EAQXW6Q.js")).createTurnActivities(dependencies);
744
782
  }
745
783
  function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
746
784
  const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
@@ -767,6 +805,7 @@ async function createOpenGeniWorker(options) {
767
805
  assertSandboxReaperActivityTimeout(settings);
768
806
  }
769
807
  const observability = options.activityDependencies?.observability ?? createObservability(settings, { component: `worker-${options.role}` });
808
+ initializeWorkerOutcomeMetrics(observability);
770
809
  if (options.role === "turn" && options.workflowBundle) {
771
810
  throw new Error("workflowBundle is valid only for the control worker role");
772
811
  }
@@ -1189,10 +1228,21 @@ async function createOpenGeniWorkerService(options) {
1189
1228
  let signaler;
1190
1229
  let workerBundle;
1191
1230
  let turnCapacityMonitor;
1231
+ let sessionRecoveryMonitor;
1192
1232
  const schedules = [];
1193
1233
  let httpServer;
1194
1234
  let memoryPressureGuard;
1195
1235
  try {
1236
+ const resolvedCatalog = await retryStartupDependency(
1237
+ "model catalog",
1238
+ () => resolveCatalogSettings(options.activityDependencies.db, settings),
1239
+ { ...retryOptions, onRetry }
1240
+ );
1241
+ observability.info("OpenGeni model catalog resolved", {
1242
+ role: options.role,
1243
+ catalogSource: resolvedCatalog.source,
1244
+ catalogVersion: resolvedCatalog.version
1245
+ });
1196
1246
  const needsSignaler = options.role === "turn" || !options.activityDependencies.wakeSessionWorkflow || !options.activityDependencies.signalSessionAttemptQuiesced || !options.activityDependencies.inspectSessionAttemptActivity || !options.activityDependencies.signalCodexCapacityWorkflow || !options.activityDependencies.startSandboxReaperWorkflow || !options.activityDependencies.startVideoGenerationWorkflow;
1197
1247
  if (needsSignaler) {
1198
1248
  signaler = await retryStartupDependency(
@@ -1235,6 +1285,11 @@ async function createOpenGeniWorkerService(options) {
1235
1285
  observability,
1236
1286
  read: signaler.getTurnTaskQueueStats
1237
1287
  });
1288
+ } else {
1289
+ sessionRecoveryMonitor = startSessionRecoveryMonitor({
1290
+ observability,
1291
+ read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db)
1292
+ });
1238
1293
  }
1239
1294
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
1240
1295
  schedules.push(
@@ -1267,14 +1322,18 @@ async function createOpenGeniWorkerService(options) {
1267
1322
  );
1268
1323
  }
1269
1324
  if (options.http !== false) {
1325
+ const databaseReady = dbReadyCheck(
1326
+ options.http?.readinessDb ?? options.activityDependencies.db,
1327
+ options.databasePosture
1328
+ );
1270
1329
  httpServer = startWorkerHttpServer({
1271
1330
  settings,
1272
1331
  observability,
1273
1332
  checks: {
1274
- db: dbReadyCheck(
1275
- options.http?.readinessDb ?? options.activityDependencies.db,
1276
- options.databasePosture
1277
- ),
1333
+ db: async () => {
1334
+ await databaseReady();
1335
+ await resolveCatalogSettings(options.activityDependencies.db, settings);
1336
+ },
1278
1337
  nats: natsReadyCheck(options.activityDependencies.bus),
1279
1338
  temporal: temporalReadyCheck(workerBundle.connection)
1280
1339
  },
@@ -1286,6 +1345,7 @@ async function createOpenGeniWorkerService(options) {
1286
1345
  httpServer?.stop(true);
1287
1346
  await Promise.allSettled([
1288
1347
  turnCapacityMonitor?.close(),
1348
+ sessionRecoveryMonitor?.close(),
1289
1349
  workerBundle?.connection.close(),
1290
1350
  signaler?.close(),
1291
1351
  ...schedules.map((schedule) => schedule.close())
@@ -1306,6 +1366,7 @@ async function createOpenGeniWorkerService(options) {
1306
1366
  httpServer?.stop(true);
1307
1367
  await Promise.allSettled([
1308
1368
  turnCapacityMonitor?.close(),
1369
+ sessionRecoveryMonitor?.close(),
1309
1370
  activeWorkerBundle.connection.close(),
1310
1371
  activeSignaler?.close(),
1311
1372
  ...schedules.map((schedule) => schedule.close())
@@ -1380,6 +1441,7 @@ async function startWorker() {
1380
1441
  }
1381
1442
  const settings = getSettings();
1382
1443
  const observability = createObservability(settings, { component: `worker-${role}` });
1444
+ const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
1383
1445
  const retryOptions = startupRetryOptions(settings);
1384
1446
  const onRetry = (event) => logStartupDependencyRetry(observability, event);
1385
1447
  const searchPath = dbSearchPath(settings);
@@ -1428,6 +1490,7 @@ async function startWorker() {
1428
1490
  });
1429
1491
  } finally {
1430
1492
  await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
1493
+ disposeUnhandledRejectionBoundary();
1431
1494
  }
1432
1495
  }
1433
1496
  if (import.meta.main) {