@opengeni/worker-bundle 0.22.7-canary.0 → 0.23.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  2. package/dist/activities/agent-turn/claim.d.ts +2 -4
  3. package/dist/activities/agent-turn/codex.d.ts +6 -5
  4. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  5. package/dist/activities/agent-turn/errors.d.ts +13 -0
  6. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  7. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  8. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  9. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  10. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  11. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -8
  14. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  15. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  16. package/dist/activities/codex-rotation.d.ts +2 -0
  17. package/dist/activities/session-state.d.ts +2 -1
  18. package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
  19. package/dist/activities-control-52W5IDWM.js.map +1 -0
  20. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
  21. package/dist/activities-turn-A2G2KLSY.js.map +1 -0
  22. package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
  23. package/dist/chunk-GFNXVZVQ.js.map +1 -0
  24. package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
  25. package/dist/chunk-IPVMCAW5.js.map +1 -0
  26. package/dist/index.js +42 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/observability-metrics.d.ts +28 -0
  29. package/dist/personal-github-git-credentials.d.ts +1 -0
  30. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  31. package/package.json +18 -18
  32. package/src/activities/agent-turn/agent-build.ts +15 -31
  33. package/src/activities/agent-turn/claim.ts +29 -68
  34. package/src/activities/agent-turn/codex.ts +19 -10
  35. package/src/activities/agent-turn/compaction-prep.ts +50 -1
  36. package/src/activities/agent-turn/errors.ts +78 -0
  37. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  38. package/src/activities/agent-turn/finalization.ts +24 -21
  39. package/src/activities/agent-turn/history.ts +5 -1
  40. package/src/activities/agent-turn/quiescence.ts +86 -0
  41. package/src/activities/agent-turn/run-credentials.ts +12 -1
  42. package/src/activities/agent-turn/run.ts +14 -11
  43. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  44. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  45. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  46. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  47. package/src/activities/agent-turn/stream-attempt.ts +98 -14
  48. package/src/activities/agent-turn/tool-environment.ts +10 -13
  49. package/src/activities/agent-turn/tool-policy.ts +3 -55
  50. package/src/activities/agent-turn/turn-context.ts +8 -14
  51. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  52. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  53. package/src/activities/codex-capacity.ts +13 -5
  54. package/src/activities/codex-rotation.ts +17 -0
  55. package/src/activities/goals.ts +10 -4
  56. package/src/activities/retained-screenshots.ts +104 -21
  57. package/src/activities/sandbox-lease.ts +7 -4
  58. package/src/activities/scheduled-tasks.ts +128 -4
  59. package/src/activities/session-state.ts +8 -1
  60. package/src/github-rest-mcp.ts +6 -3
  61. package/src/http.ts +3 -0
  62. package/src/index.ts +19 -0
  63. package/src/observability-metrics.ts +200 -0
  64. package/src/personal-github-git-credentials.ts +45 -4
  65. package/src/sandbox-resume.ts +1 -1
  66. package/src/sandbox-routing.ts +7 -4
  67. package/src/unhandled-rejection-boundary.ts +73 -0
  68. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  69. package/dist/activities/recording.d.ts +0 -90
  70. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  71. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  72. package/dist/chunk-NAVSAZ56.js.map +0 -1
  73. package/dist/chunk-O4HDX5SC.js.map +0 -1
  74. package/src/activities/recording.ts +0 -279
@@ -150,16 +150,21 @@ function imageBytesFromSdkToolOutput(
150
150
  for (const entry of output) {
151
151
  if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
152
152
  const record = entry as Record<string, unknown>;
153
- if (record.type !== "input_image") continue;
154
- const decoded = imageBytesFromSdkImageSource(
155
- record.image ?? record.image_url ?? record.imageUrl,
156
- );
153
+ const decoded =
154
+ record.type === "input_image"
155
+ ? imageBytesFromSdkImageSource(record.image ?? record.image_url ?? record.imageUrl)
156
+ : record.type === "image"
157
+ ? decodeInlineMcpImage(record.data, record.mimeType)
158
+ : null;
157
159
  if (decoded) return decoded;
158
160
  }
159
161
  return null;
160
162
  }
161
163
  if (!output || typeof output !== "object") return null;
162
164
  const outputRecord = output as Record<string, unknown>;
165
+ if (Array.isArray(outputRecord.content)) {
166
+ return imageBytesFromSdkToolOutput(outputRecord.content);
167
+ }
163
168
  if (
164
169
  outputRecord.type !== "image" ||
165
170
  !outputRecord.image ||
@@ -202,16 +207,44 @@ export function toolOutputContainsInlineImage(output: unknown): boolean {
202
207
  if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
203
208
  const record = entry as Record<string, unknown>;
204
209
  return (
205
- record.type === "input_image" &&
206
- sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)
210
+ (record.type === "input_image" &&
211
+ sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)) ||
212
+ (record.type === "image" && decodeInlineMcpImage(record.data, record.mimeType) !== null)
207
213
  );
208
214
  });
209
215
  }
210
216
  if (!output || typeof output !== "object") return false;
211
217
  const record = output as Record<string, unknown>;
218
+ if (Array.isArray(record.content)) return toolOutputContainsInlineImage(record.content);
212
219
  return record.type === "image" && sdkImageSourceContainsInlineImage(record.image);
213
220
  }
214
221
 
222
+ function decodeInlineMcpImage(
223
+ data: unknown,
224
+ mimeType: unknown,
225
+ ): { bytes: Uint8Array; mediaType: string } | null {
226
+ if (
227
+ typeof data !== "string" ||
228
+ typeof mimeType !== "string" ||
229
+ !["image/png", "image/jpeg", "image/webp"].includes(mimeType)
230
+ ) {
231
+ return null;
232
+ }
233
+ const maxEncodedLength = Math.ceil(COMPUTER_SCREENSHOT_MAX_BYTES / 3) * 4;
234
+ if (
235
+ data.length === 0 ||
236
+ data.length > maxEncodedLength ||
237
+ data.length % 4 !== 0 ||
238
+ !/^[A-Za-z0-9+/]+={0,2}$/u.test(data)
239
+ ) {
240
+ return null;
241
+ }
242
+ const bytes = Uint8Array.from(Buffer.from(data, "base64"));
243
+ return bytes.byteLength > 0 && bytes.byteLength <= COMPUTER_SCREENSHOT_MAX_BYTES
244
+ ? { bytes, mediaType: mimeType }
245
+ : null;
246
+ }
247
+
215
248
  function decodeInlineImageDataUrl(value: unknown): { bytes: Uint8Array; mediaType: string } | null {
216
249
  if (typeof value !== "string") return null;
217
250
  const match = /^data:(image\/(?:png|jpeg|webp));base64,/u.exec(value);
@@ -680,18 +713,43 @@ export function compactRetainedScreenshotHistory(
680
713
  output: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
681
714
  };
682
715
  }
716
+ if (
717
+ item.output &&
718
+ typeof item.output === "object" &&
719
+ !Array.isArray(item.output) &&
720
+ Array.isArray((item.output as Record<string, unknown>).content)
721
+ ) {
722
+ const result = item.output as Record<string, unknown>;
723
+ const content = compactInlineImageContent(result.content as unknown[], receipt);
724
+ return content.changed ? { ...item, output: { ...result, content: content.entries } } : item;
725
+ }
683
726
  if (!Array.isArray(item.output)) return item;
684
- let changed = false;
685
- const output = item.output.map((entry) => {
686
- if (!isInlineImageContent(entry)) return entry;
687
- changed = true;
727
+ const output = compactInlineImageContent(item.output, receipt);
728
+ return output.changed ? { ...item, output: output.entries } : item;
729
+ });
730
+ }
731
+
732
+ function compactInlineImageContent(
733
+ entries: unknown[],
734
+ receipt: RetainedArtifactMetadata,
735
+ ): { entries: unknown[]; changed: boolean } {
736
+ let changed = false;
737
+ const compacted = entries.map((entry) => {
738
+ if (!isInlineImageContent(entry)) return entry;
739
+ changed = true;
740
+ const record = entry as Record<string, unknown>;
741
+ if (record.type === "image") {
688
742
  return {
689
- ...(entry as Record<string, unknown>),
743
+ type: "input_image",
690
744
  image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
691
745
  };
692
- });
693
- return changed ? { ...item, output } : item;
746
+ }
747
+ return {
748
+ ...record,
749
+ image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
750
+ };
694
751
  });
752
+ return { entries: compacted, changed };
695
753
  }
696
754
 
697
755
  /** Collect durable screenshot receipts before their provider-only expansion. */
@@ -707,8 +765,16 @@ export function collectRetainedScreenshotReceipts(
707
765
  target.set(callId, direct);
708
766
  continue;
709
767
  }
710
- if (!Array.isArray(item.output)) continue;
711
- for (const entry of item.output) {
768
+ const outputEntries = Array.isArray(item.output)
769
+ ? item.output
770
+ : item.output &&
771
+ typeof item.output === "object" &&
772
+ !Array.isArray(item.output) &&
773
+ Array.isArray((item.output as Record<string, unknown>).content)
774
+ ? ((item.output as Record<string, unknown>).content as unknown[])
775
+ : null;
776
+ if (!outputEntries) continue;
777
+ for (const entry of outputEntries) {
712
778
  const receipt = retainedReceiptFromImageContent(entry);
713
779
  if (receipt) {
714
780
  target.set(callId, receipt);
@@ -872,18 +938,34 @@ async function materializeRetainedScreenshotHistoryWithCache(
872
938
  });
873
939
  continue;
874
940
  }
875
- if (!Array.isArray(item.output)) {
941
+ const resultOutput =
942
+ item.output && typeof item.output === "object" && !Array.isArray(item.output)
943
+ ? (item.output as Record<string, unknown>)
944
+ : null;
945
+ const outputEntries = Array.isArray(item.output)
946
+ ? item.output
947
+ : resultOutput && Array.isArray(resultOutput.content)
948
+ ? resultOutput.content
949
+ : null;
950
+ if (!outputEntries) {
876
951
  materialized.push(item);
877
952
  continue;
878
953
  }
879
954
  let changed = false;
880
955
  const output: unknown[] = [];
881
- for (const entry of item.output) {
956
+ for (const entry of outputEntries) {
882
957
  const next = await materializeEntry(entry);
883
958
  changed ||= next !== entry;
884
959
  output.push(next);
885
960
  }
886
- materialized.push(changed ? { ...item, output } : item);
961
+ materialized.push(
962
+ changed
963
+ ? {
964
+ ...item,
965
+ output: resultOutput ? { ...resultOutput, content: output } : output,
966
+ }
967
+ : item,
968
+ );
887
969
  }
888
970
  return materialized;
889
971
  }
@@ -1045,9 +1127,10 @@ function isInlineImageContent(entry: unknown): boolean {
1045
1127
  if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
1046
1128
  const record = entry as Record<string, unknown>;
1047
1129
  return (
1048
- record.type === "input_image" &&
1049
- typeof record.image === "string" &&
1050
- record.image.startsWith("data:image/")
1130
+ (record.type === "input_image" &&
1131
+ typeof record.image === "string" &&
1132
+ record.image.startsWith("data:image/")) ||
1133
+ (record.type === "image" && decodeInlineMcpImage(record.data, record.mimeType) !== null)
1051
1134
  );
1052
1135
  }
1053
1136
 
@@ -1429,7 +1429,7 @@ async function reconcileTerminalRetainedProcesses(
1429
1429
  }
1430
1430
 
1431
1431
  try {
1432
- await settleRetainedProcess(db, {
1432
+ const settlement = await settleRetainedProcess(db, {
1433
1433
  accountId: process.accountId,
1434
1434
  workspaceId: process.workspaceId,
1435
1435
  sessionId: process.sessionId,
@@ -1441,14 +1441,17 @@ async function reconcileTerminalRetainedProcesses(
1441
1441
  reason: proof.reason,
1442
1442
  idleGraceMs: settings.sandboxIdleGraceMs,
1443
1443
  });
1444
+ if (settlement.process.state === "active") {
1445
+ throw new Error("Retained-process reconciliation returned an active durable process");
1446
+ }
1444
1447
  await settleSessionBackgroundCommandForRetainedProcess(db, {
1445
1448
  accountId: process.accountId,
1446
1449
  workspaceId: process.workspaceId,
1447
1450
  sessionId: process.sessionId,
1448
1451
  retainedProcessId: process.id,
1449
- outcome: proof.outcome,
1450
- exitCode: proof.exitCode,
1451
- reason: proof.reason,
1452
+ outcome: settlement.process.state,
1453
+ exitCode: settlement.process.exitCode,
1454
+ reason: settlement.process.settlementReason ?? proof.reason,
1452
1455
  });
1453
1456
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
1454
1457
  } catch (error) {
@@ -19,6 +19,7 @@ import {
19
19
  requireOpenGeniSlackBotConnection,
20
20
  resolveSessionToolPolicy,
21
21
  scheduledSlackBotConnectionId,
22
+ swapActiveSandbox,
22
23
  } from "@opengeni/core";
23
24
  import {
24
25
  appendSessionEvents,
@@ -28,6 +29,7 @@ import {
28
29
  createSession,
29
30
  createSessionWithIdempotencyKeyResult,
30
31
  enqueueSessionWorkflowWakeIfRunnable,
32
+ failScheduledGeneratedSessionRoute,
31
33
  getScheduledTask,
32
34
  getScheduledTaskRunAcceptedExecution,
33
35
  getScheduledTaskRunByProducerKey,
@@ -38,6 +40,8 @@ import {
38
40
  getScheduledTaskPersonalResourceAuthoritySubject,
39
41
  getScheduledTaskRevisionAuthority,
40
42
  getScheduledTaskXaiProviderAccountAuthoritySnapshot,
43
+ getEnrollment,
44
+ getSandbox,
41
45
  getRig,
42
46
  getScheduledScopedRigVersionMetadata,
43
47
  getNestedAgentDepthDeploymentPolicy,
@@ -442,12 +446,41 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
442
446
  taskConnectionAuthoritySubjectId;
443
447
  const model = task.agentConfig.model ?? settings.openaiModel;
444
448
  const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
445
- const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
449
+ let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
450
+ let sandboxOs: "linux" | "macos" | "windows" = "linux";
446
451
  const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
447
452
  const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
448
453
  const generatedTarget =
449
454
  task.runMode === "new_session_per_run" ||
450
455
  (task.runMode === "reusable_session" && task.reusableSessionId === null);
456
+ if (generatedTarget && task.agentConfig.machineTarget) {
457
+ const access = {
458
+ accountId: task.accountId,
459
+ workspaceId: task.workspaceId,
460
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId,
461
+ };
462
+ const machine = await getSandbox(
463
+ db,
464
+ access,
465
+ task.agentConfig.machineTarget.targetSandboxId,
466
+ );
467
+ if (
468
+ !machine ||
469
+ machine.kind !== "selfhosted" ||
470
+ !machine.enrollmentId ||
471
+ machine.scope === "user"
472
+ ) {
473
+ throw new Error("scheduled Connected Machine target is unavailable");
474
+ }
475
+ const enrollment = await getEnrollment(db, access, machine.enrollmentId);
476
+ if (!enrollment || enrollment.status !== "active") {
477
+ throw new Error("scheduled Connected Machine enrollment is unavailable");
478
+ }
479
+ sandboxBackend = "selfhosted";
480
+ sandboxOs = enrollment.os;
481
+ } else if (generatedTarget && sandboxBackend === "selfhosted") {
482
+ throw new Error("self-hosted scheduled task has no Connected Machine target");
483
+ }
451
484
  const generatedSessionDepthPolicy = generatedTarget
452
485
  ? await (async () => {
453
486
  const workspace = await requireWorkspace(db, task.workspaceId);
@@ -730,7 +763,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
730
763
  resolvedReasoningEffort: reasoningEffort,
731
764
  resolvedLatencyMode: "standard",
732
765
  resolvedSandboxBackend: sandboxBackend,
733
- resolvedSandboxOs: "linux",
766
+ resolvedSandboxOs: sandboxOs,
734
767
  resolvedTools: taskTools,
735
768
  resolvedFirstPartyMcpTools: firstPartyMcpTools,
736
769
  resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
@@ -972,7 +1005,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
972
1005
  reasoningEffort,
973
1006
  latencyMode: "standard",
974
1007
  sandboxBackend,
975
- sandboxOs: "linux",
1008
+ sandboxOs,
976
1009
  variableSetId: task.variableSetId ?? null,
977
1010
  rigId: frozenRigId,
978
1011
  rigVersionId: frozenRigVersionId,
@@ -1079,6 +1112,16 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1079
1112
  );
1080
1113
  }
1081
1114
  }
1115
+ session = await seedScheduledGeneratedSessionRoute({
1116
+ db: dispatchDb,
1117
+ bus,
1118
+ settings,
1119
+ task,
1120
+ runId: run.id,
1121
+ session,
1122
+ deferPublications,
1123
+ deferredEvents,
1124
+ });
1082
1125
  let workflowId: string;
1083
1126
  if (alertOccurrence) {
1084
1127
  const started = await initializeSessionStartAtomically(dispatchDb, {
@@ -1539,7 +1582,6 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1539
1582
  acceptedExecution: dispatchOutcome.acceptedExecution,
1540
1583
  });
1541
1584
  }
1542
- if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
1543
1585
  for (const deferred of deferredEvents) {
1544
1586
  await publishDurableSessionEvents(
1545
1587
  bus,
@@ -1548,6 +1590,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
1548
1590
  deferred.events,
1549
1591
  );
1550
1592
  }
1593
+ if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
1551
1594
  const result = dispatchOutcome.result;
1552
1595
  if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
1553
1596
  await wakeSessionWorkflow({
@@ -1773,6 +1816,74 @@ async function recordScheduledTaskFiredUsage(
1773
1816
  });
1774
1817
  }
1775
1818
 
1819
+ async function seedScheduledGeneratedSessionRoute(input: {
1820
+ db: Database;
1821
+ bus: ControlActivityServices["bus"];
1822
+ settings: ControlActivityServices["settings"];
1823
+ task: ScheduledTask;
1824
+ runId: string;
1825
+ session: Awaited<ReturnType<typeof createSession>>;
1826
+ deferPublications: boolean;
1827
+ deferredEvents: Array<{
1828
+ sessionId: string;
1829
+ events: Awaited<ReturnType<typeof appendSessionEvents>>;
1830
+ }>;
1831
+ }): Promise<Awaited<ReturnType<typeof createSession>>> {
1832
+ const target = input.task.agentConfig.machineTarget;
1833
+ if (!target) return input.session;
1834
+ const seeded = await swapActiveSandbox(
1835
+ {
1836
+ db: input.db,
1837
+ settings: input.settings,
1838
+ bus: input.bus,
1839
+ },
1840
+ {
1841
+ accountId: input.task.accountId,
1842
+ workspaceId: input.task.workspaceId,
1843
+ sessionId: input.session.id,
1844
+ sessionBackend: input.session.sandboxBackend,
1845
+ sessionGroupId: input.session.sandboxGroupId,
1846
+ },
1847
+ target.targetSandboxId,
1848
+ target.workingDir ?? null,
1849
+ );
1850
+ if (seeded.swapped) {
1851
+ return await requireSession(input.db, input.task.workspaceId, input.session.id);
1852
+ }
1853
+
1854
+ const failed = await failScheduledGeneratedSessionRoute(input.db, {
1855
+ accountId: input.task.accountId,
1856
+ workspaceId: input.task.workspaceId,
1857
+ taskId: input.task.id,
1858
+ runId: input.runId,
1859
+ sessionId: input.session.id,
1860
+ error: "scheduled_machine_unavailable",
1861
+ });
1862
+ if (failed.action === "advanced") {
1863
+ throw new Error(
1864
+ `scheduled run advanced to ${failed.status} while its Connected Machine route was being established`,
1865
+ );
1866
+ }
1867
+ if (failed.events.length > 0) {
1868
+ if (input.deferPublications) {
1869
+ input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
1870
+ } else {
1871
+ await publishDurableSessionEvents(
1872
+ input.bus,
1873
+ input.task.workspaceId,
1874
+ input.session.id,
1875
+ failed.events,
1876
+ );
1877
+ }
1878
+ }
1879
+ const error =
1880
+ failed.action === "terminal" ? (failed.error ?? "scheduled_run_terminal") : failed.error;
1881
+ throw new ScheduledRunTerminalAuthorityError(
1882
+ error,
1883
+ seeded.reason ?? "scheduled Connected Machine target is unavailable",
1884
+ );
1885
+ }
1886
+
1776
1887
  async function recoverBoundScheduledTaskDispatch(input: {
1777
1888
  db: Database;
1778
1889
  bus: ControlActivityServices["bus"];
@@ -1952,6 +2063,18 @@ async function recoverBoundScheduledTaskDispatch(input: {
1952
2063
  sessionId: session.id,
1953
2064
  });
1954
2065
  }
2066
+ if (generatedSession) {
2067
+ session = await seedScheduledGeneratedSessionRoute({
2068
+ db: input.db,
2069
+ bus: input.bus,
2070
+ settings: input.settings,
2071
+ task,
2072
+ runId: input.run.id,
2073
+ session,
2074
+ deferPublications: false,
2075
+ deferredEvents: [],
2076
+ });
2077
+ }
1955
2078
  if (session.status === "cancelled") {
1956
2079
  await markScheduledTaskRunSkippedIfQueued(input.db, {
1957
2080
  workspaceId: task.workspaceId,
@@ -2024,6 +2147,7 @@ async function recoverBoundScheduledTaskDispatch(input: {
2024
2147
  session.latencyMode !== input.acceptedExecution.resolvedLatencyMode ||
2025
2148
  session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend ||
2026
2149
  session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs ||
2150
+ session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) ||
2027
2151
  stableJson(session.resources) !== stableJson(task.agentConfig.resources) ||
2028
2152
  stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) ||
2029
2153
  stableJson(session.firstPartyMcpTools) !==
@@ -19,7 +19,7 @@ import {
19
19
  } from "@opengeni/db";
20
20
  import { publishDurableSessionEvents } from "@opengeni/events";
21
21
  import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake";
22
- import { recordTurnsQueuedGauge } from "../observability-metrics";
22
+ import { recordTurnsQueuedGauge, recordWorkerDeathRecoveryMetrics } from "../observability-metrics";
23
23
  import {
24
24
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
25
25
  providerRecoveryCountFromMetadata,
@@ -66,6 +66,7 @@ export type SessionStateActivityOverrides = Partial<{
66
66
  deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
67
67
  notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
68
68
  recordTurnsQueuedGauge: typeof recordTurnsQueuedGauge;
69
+ recordWorkerDeathRecoveryMetrics: typeof recordWorkerDeathRecoveryMetrics;
69
70
  }>;
70
71
 
71
72
  // Crash-loop guard for worker-death re-dispatch: a turn that takes a worker
@@ -112,6 +113,8 @@ export function createSessionStateActivities(
112
113
  overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
113
114
  const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
114
115
  const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
116
+ const recordWorkerDeathRecoveryMetricsFn =
117
+ overrides.recordWorkerDeathRecoveryMetrics ?? recordWorkerDeathRecoveryMetrics;
115
118
 
116
119
  async function failSessionAttempt(
117
120
  input: FailSessionAttemptInput,
@@ -397,6 +400,10 @@ export function createSessionStateActivities(
397
400
  if (result.action === "stale" || result.action === "unclaimed") {
398
401
  return { action: result.action };
399
402
  }
403
+ recordWorkerDeathRecoveryMetricsFn(observability, {
404
+ outcome: result.action === "exceeded" ? "exhausted" : "recovering",
405
+ timeoutType: input.timeoutType,
406
+ });
400
407
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
401
408
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
402
409
  if (result.action === "exceeded") {
@@ -258,9 +258,12 @@ function appRepositoryAuthority(resources: ResourceRef[]): GitHubRestRepository[
258
258
  );
259
259
  const repositoryId = positiveSafeInteger(resource.githubRepositoryId ?? resource.repositoryId);
260
260
  const fullName = canonicalGitHubFullName(resource.uri);
261
- if (!installationId || !repositoryId || !fullName) {
261
+ if (!installationId || !repositoryId) {
262
262
  throw new GitHubRestAuthorityError("GitHub App repository authority is incomplete");
263
263
  }
264
+ if (!fullName) {
265
+ throw new GitHubRestAuthorityError("GitHub App repository URL is invalid");
266
+ }
264
267
  return [
265
268
  {
266
269
  repositoryId: String(repositoryId),
@@ -645,7 +648,8 @@ function githubConnectorBindings(
645
648
  function canonicalGitHubFullName(uri: string): string | null {
646
649
  try {
647
650
  const url = new URL(uri);
648
- const path = url.pathname.replace(/^\/+|\/+$/gu, "");
651
+ const clonePath = url.pathname.replace(/^\/+|\/+$/gu, "");
652
+ const path = clonePath.endsWith(".git") ? clonePath.slice(0, -4) : clonePath;
649
653
  if (
650
654
  url.protocol !== "https:" ||
651
655
  url.hostname !== "github.com" ||
@@ -654,7 +658,6 @@ function canonicalGitHubFullName(uri: string): string | null {
654
658
  url.password ||
655
659
  url.search ||
656
660
  url.hash ||
657
- path.endsWith(".git") ||
658
661
  !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(path)
659
662
  ) {
660
663
  return null;
package/src/http.ts CHANGED
@@ -6,6 +6,7 @@ import {
6
6
  type RuntimeDatabasePostureOptions,
7
7
  } from "@opengeni/db";
8
8
  import { requireSessionEventDurableFanoutCapability, type EventBus } from "@opengeni/events";
9
+ import { githubAppBotIdentityWarnings } from "@opengeni/github";
9
10
  import type { Observability } from "@opengeni/observability";
10
11
 
11
12
  export type ReadinessCheckName = "db" | "nats" | "temporal";
@@ -45,12 +46,14 @@ export function createWorkerHttpHandler(
45
46
  if (url.pathname === "/healthz") {
46
47
  const state = lifecycle?.state();
47
48
  const ok = state !== "failed" && state !== "stopped";
49
+ const warnings = githubAppBotIdentityWarnings(settings);
48
50
  return Response.json(
49
51
  {
50
52
  service: settings.serviceName,
51
53
  environment: settings.environment,
52
54
  deploymentRevision: settings.deploymentRevision,
53
55
  ...(lifecycle ? { role: lifecycle.role, state } : {}),
56
+ ...(warnings.length > 0 ? { warnings } : {}),
54
57
  ok,
55
58
  },
56
59
  { status: ok ? 200 : 503 },
package/src/index.ts CHANGED
@@ -9,6 +9,7 @@ import {
9
9
  } from "@opengeni/config";
10
10
  import {
11
11
  assertRuntimeDatabasePosture,
12
+ countSessionRecoveryBacklog,
12
13
  createDb,
13
14
  markSessionWorkflowWakeDelivered,
14
15
  type Database,
@@ -55,8 +56,10 @@ import {
55
56
  type WorkerLifecycleState,
56
57
  } from "./http";
57
58
  import {
59
+ initializeWorkerOutcomeMetrics,
58
60
  normalizeTurnTaskQueueStats,
59
61
  observabilityEventLogger,
62
+ startSessionRecoveryMonitor,
60
63
  startTurnCapacityMonitor,
61
64
  type TurnTaskQueueStats,
62
65
  } from "./observability-metrics";
@@ -86,6 +89,7 @@ import {
86
89
  type TurnWorkerMemoryPressureGuard,
87
90
  } from "./memory-pressure-guard";
88
91
  import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
92
+ import { installWorkerUnhandledRejectionBoundary } from "./unhandled-rejection-boundary";
89
93
  import {
90
94
  SANDBOX_REAPER_V2_WORKFLOW_ID,
91
95
  sandboxLifecycleTaskQueue,
@@ -219,6 +223,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
219
223
  const observability =
220
224
  options.activityDependencies?.observability ??
221
225
  createObservability(settings, { component: `worker-${options.role}` });
226
+ initializeWorkerOutcomeMetrics(observability);
222
227
  if (options.role === "turn" && options.workflowBundle) {
223
228
  throw new Error("workflowBundle is valid only for the control worker role");
224
229
  }
@@ -793,6 +798,7 @@ export async function createOpenGeniWorkerService(
793
798
  let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
794
799
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
795
800
  let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
801
+ let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
796
802
  const schedules: Array<{ close: () => Promise<void> }> = [];
797
803
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
798
804
  let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
@@ -866,6 +872,11 @@ export async function createOpenGeniWorkerService(
866
872
  observability,
867
873
  read: signaler.getTurnTaskQueueStats,
868
874
  });
875
+ } else {
876
+ sessionRecoveryMonitor = startSessionRecoveryMonitor({
877
+ observability,
878
+ read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
879
+ });
869
880
  }
870
881
 
871
882
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
@@ -919,6 +930,7 @@ export async function createOpenGeniWorkerService(
919
930
  httpServer?.stop(true);
920
931
  await Promise.allSettled([
921
932
  turnCapacityMonitor?.close(),
933
+ sessionRecoveryMonitor?.close(),
922
934
  workerBundle?.connection.close(),
923
935
  signaler?.close(),
924
936
  ...schedules.map((schedule) => schedule.close()),
@@ -941,6 +953,7 @@ export async function createOpenGeniWorkerService(
941
953
  httpServer?.stop(true);
942
954
  await Promise.allSettled([
943
955
  turnCapacityMonitor?.close(),
956
+ sessionRecoveryMonitor?.close(),
944
957
  activeWorkerBundle.connection.close(),
945
958
  activeSignaler?.close(),
946
959
  ...schedules.map((schedule) => schedule.close()),
@@ -1032,6 +1045,11 @@ export async function startWorker() {
1032
1045
  }
1033
1046
  const settings = getSettings();
1034
1047
  const observability = createObservability(settings, { component: `worker-${role}` });
1048
+ // The Agents SDK also listens for unhandled rejections and exits the process.
1049
+ // Keep detached SDK/transport promises observational here: durable turn
1050
+ // activity boundaries own failures, while process exit causes cross-session
1051
+ // lease loss for every turn assigned to this shared worker pod.
1052
+ const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
1035
1053
  const retryOptions = startupRetryOptions(settings);
1036
1054
  const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
1037
1055
  logStartupDependencyRetry(observability, event);
@@ -1090,6 +1108,7 @@ export async function startWorker() {
1090
1108
  });
1091
1109
  } finally {
1092
1110
  await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
1111
+ disposeUnhandledRejectionBoundary();
1093
1112
  }
1094
1113
  }
1095
1114