@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +10 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  3. package/dist/activities/agent-turn/claim.d.ts +3 -4
  4. package/dist/activities/agent-turn/codex.d.ts +8 -7
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  6. package/dist/activities/agent-turn/errors.d.ts +13 -0
  7. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +10 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  11. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  13. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  16. package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +3 -8
  18. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  19. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  20. package/dist/activities/automations.d.ts +1 -0
  21. package/dist/activities/capabilities.d.ts +2 -1
  22. package/dist/activities/codex-rotation.d.ts +2 -0
  23. package/dist/activities/goals.d.ts +11 -0
  24. package/dist/activities/image-generation-references.d.ts +2 -1
  25. package/dist/activities/scheduled-tasks.d.ts +1 -0
  26. package/dist/activities/session-state.d.ts +2 -1
  27. package/dist/activities/types.d.ts +8 -0
  28. package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
  29. package/dist/activities-control-SGU2WEVB.js.map +1 -0
  30. package/dist/activities-control.d.ts +2 -0
  31. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
  32. package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
  33. package/dist/activities.d.ts +2 -0
  34. package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
  35. package/dist/chunk-S2QVRVJW.js.map +1 -0
  36. package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
  37. package/dist/chunk-TJIPCHMP.js.map +1 -0
  38. package/dist/index.d.ts +1 -0
  39. package/dist/index.js +69 -6
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +29 -0
  42. package/dist/personal-github-git-credentials.d.ts +1 -0
  43. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  44. package/dist/workflow-bundle.js +26 -3
  45. package/dist/workflows/activities.d.ts +5 -0
  46. package/dist/workflows/automations.d.ts +5 -0
  47. package/package.json +18 -18
  48. package/src/activities/agent-run-admission.ts +7 -1
  49. package/src/activities/agent-turn/admission.ts +28 -6
  50. package/src/activities/agent-turn/agent-build.ts +15 -31
  51. package/src/activities/agent-turn/claim.ts +53 -72
  52. package/src/activities/agent-turn/codex.ts +21 -12
  53. package/src/activities/agent-turn/compaction-prep.ts +52 -1
  54. package/src/activities/agent-turn/errors.ts +78 -0
  55. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  56. package/src/activities/agent-turn/finalization.ts +24 -21
  57. package/src/activities/agent-turn/history.ts +5 -1
  58. package/src/activities/agent-turn/list-models.ts +101 -0
  59. package/src/activities/agent-turn/model-usage.ts +88 -42
  60. package/src/activities/agent-turn/quiescence.ts +86 -0
  61. package/src/activities/agent-turn/run-credentials.ts +12 -1
  62. package/src/activities/agent-turn/run.ts +20 -14
  63. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  64. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  65. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  66. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  67. package/src/activities/agent-turn/stream-attempt.ts +106 -14
  68. package/src/activities/agent-turn/tool-environment.ts +72 -14
  69. package/src/activities/agent-turn/tool-policy.ts +7 -6
  70. package/src/activities/agent-turn/turn-context.ts +12 -14
  71. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  72. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  73. package/src/activities/automations.ts +122 -67
  74. package/src/activities/capabilities.ts +72 -2
  75. package/src/activities/codex-capacity.ts +13 -5
  76. package/src/activities/codex-rotation.ts +17 -0
  77. package/src/activities/goals.ts +117 -35
  78. package/src/activities/image-generation-references.ts +9 -9
  79. package/src/activities/retained-screenshots.ts +104 -21
  80. package/src/activities/sandbox-lease.ts +7 -4
  81. package/src/activities/scheduled-tasks.ts +248 -30
  82. package/src/activities/session-state.ts +8 -1
  83. package/src/activities/site-auth-maintenance.ts +12 -12
  84. package/src/activities/types.ts +6 -0
  85. package/src/activities.ts +1 -0
  86. package/src/activity-services.ts +4 -1
  87. package/src/github-rest-mcp.ts +8 -4
  88. package/src/http.ts +3 -0
  89. package/src/index.ts +38 -4
  90. package/src/observability-metrics.ts +202 -0
  91. package/src/personal-github-git-credentials.ts +45 -4
  92. package/src/sandbox-resume.ts +1 -1
  93. package/src/sandbox-routing.ts +7 -4
  94. package/src/unhandled-rejection-boundary.ts +73 -0
  95. package/src/workflows/activities.ts +15 -0
  96. package/src/workflows/automations.ts +21 -2
  97. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  98. package/dist/activities/recording.d.ts +0 -90
  99. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  100. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  101. package/dist/chunk-NAVSAZ56.js.map +0 -1
  102. package/dist/chunk-O4HDX5SC.js.map +0 -1
  103. package/src/activities/recording.ts +0 -279
@@ -200,20 +200,20 @@ async function referenceBytes(
200
200
  );
201
201
  }
202
202
  return {
203
- bytes: await readStoredFile(input.objectStorage, file),
203
+ bytes: await readStoredImageReferenceFile(input.objectStorage, file),
204
204
  file,
205
205
  };
206
206
  }
207
207
 
208
- async function readStoredFile(objectStorage: ObjectStorage, file: FileAsset): Promise<Uint8Array> {
209
- const bytes = await objectStorage.getFileRange(file, {
210
- start: 0,
211
- end: file.sizeBytes - 1,
212
- });
213
- if (!bytes) {
208
+ export async function readStoredImageReferenceFile(
209
+ objectStorage: Pick<ObjectStorage, "getFileBytes">,
210
+ file: FileAsset,
211
+ ): Promise<Uint8Array> {
212
+ const bytes = await objectStorage.getFileBytes(file);
213
+ if (bytes.byteLength !== file.sizeBytes) {
214
214
  throw new ImageGenerationReferenceError(
215
- "reference_unavailable",
216
- "The durable image reference bytes are unavailable.",
215
+ "reference_integrity_mismatch",
216
+ "The durable image reference bytes do not match their file metadata.",
217
217
  );
218
218
  }
219
219
  return bytes;
@@ -150,16 +150,21 @@ function imageBytesFromSdkToolOutput(
150
150
  for (const entry of output) {
151
151
  if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
152
152
  const record = entry as Record<string, unknown>;
153
- if (record.type !== "input_image") continue;
154
- const decoded = imageBytesFromSdkImageSource(
155
- record.image ?? record.image_url ?? record.imageUrl,
156
- );
153
+ const decoded =
154
+ record.type === "input_image"
155
+ ? imageBytesFromSdkImageSource(record.image ?? record.image_url ?? record.imageUrl)
156
+ : record.type === "image"
157
+ ? decodeInlineMcpImage(record.data, record.mimeType)
158
+ : null;
157
159
  if (decoded) return decoded;
158
160
  }
159
161
  return null;
160
162
  }
161
163
  if (!output || typeof output !== "object") return null;
162
164
  const outputRecord = output as Record<string, unknown>;
165
+ if (Array.isArray(outputRecord.content)) {
166
+ return imageBytesFromSdkToolOutput(outputRecord.content);
167
+ }
163
168
  if (
164
169
  outputRecord.type !== "image" ||
165
170
  !outputRecord.image ||
@@ -202,16 +207,44 @@ export function toolOutputContainsInlineImage(output: unknown): boolean {
202
207
  if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
203
208
  const record = entry as Record<string, unknown>;
204
209
  return (
205
- record.type === "input_image" &&
206
- sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)
210
+ (record.type === "input_image" &&
211
+ sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)) ||
212
+ (record.type === "image" && decodeInlineMcpImage(record.data, record.mimeType) !== null)
207
213
  );
208
214
  });
209
215
  }
210
216
  if (!output || typeof output !== "object") return false;
211
217
  const record = output as Record<string, unknown>;
218
+ if (Array.isArray(record.content)) return toolOutputContainsInlineImage(record.content);
212
219
  return record.type === "image" && sdkImageSourceContainsInlineImage(record.image);
213
220
  }
214
221
 
222
+ function decodeInlineMcpImage(
223
+ data: unknown,
224
+ mimeType: unknown,
225
+ ): { bytes: Uint8Array; mediaType: string } | null {
226
+ if (
227
+ typeof data !== "string" ||
228
+ typeof mimeType !== "string" ||
229
+ !["image/png", "image/jpeg", "image/webp"].includes(mimeType)
230
+ ) {
231
+ return null;
232
+ }
233
+ const maxEncodedLength = Math.ceil(COMPUTER_SCREENSHOT_MAX_BYTES / 3) * 4;
234
+ if (
235
+ data.length === 0 ||
236
+ data.length > maxEncodedLength ||
237
+ data.length % 4 !== 0 ||
238
+ !/^[A-Za-z0-9+/]+={0,2}$/u.test(data)
239
+ ) {
240
+ return null;
241
+ }
242
+ const bytes = Uint8Array.from(Buffer.from(data, "base64"));
243
+ return bytes.byteLength > 0 && bytes.byteLength <= COMPUTER_SCREENSHOT_MAX_BYTES
244
+ ? { bytes, mediaType: mimeType }
245
+ : null;
246
+ }
247
+
215
248
  function decodeInlineImageDataUrl(value: unknown): { bytes: Uint8Array; mediaType: string } | null {
216
249
  if (typeof value !== "string") return null;
217
250
  const match = /^data:(image\/(?:png|jpeg|webp));base64,/u.exec(value);
@@ -680,18 +713,43 @@ export function compactRetainedScreenshotHistory(
680
713
  output: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
681
714
  };
682
715
  }
716
+ if (
717
+ item.output &&
718
+ typeof item.output === "object" &&
719
+ !Array.isArray(item.output) &&
720
+ Array.isArray((item.output as Record<string, unknown>).content)
721
+ ) {
722
+ const result = item.output as Record<string, unknown>;
723
+ const content = compactInlineImageContent(result.content as unknown[], receipt);
724
+ return content.changed ? { ...item, output: { ...result, content: content.entries } } : item;
725
+ }
683
726
  if (!Array.isArray(item.output)) return item;
684
- let changed = false;
685
- const output = item.output.map((entry) => {
686
- if (!isInlineImageContent(entry)) return entry;
687
- changed = true;
727
+ const output = compactInlineImageContent(item.output, receipt);
728
+ return output.changed ? { ...item, output: output.entries } : item;
729
+ });
730
+ }
731
+
732
+ function compactInlineImageContent(
733
+ entries: unknown[],
734
+ receipt: RetainedArtifactMetadata,
735
+ ): { entries: unknown[]; changed: boolean } {
736
+ let changed = false;
737
+ const compacted = entries.map((entry) => {
738
+ if (!isInlineImageContent(entry)) return entry;
739
+ changed = true;
740
+ const record = entry as Record<string, unknown>;
741
+ if (record.type === "image") {
688
742
  return {
689
- ...(entry as Record<string, unknown>),
743
+ type: "input_image",
690
744
  image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
691
745
  };
692
- });
693
- return changed ? { ...item, output } : item;
746
+ }
747
+ return {
748
+ ...record,
749
+ image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
750
+ };
694
751
  });
752
+ return { entries: compacted, changed };
695
753
  }
696
754
 
697
755
  /** Collect durable screenshot receipts before their provider-only expansion. */
@@ -707,8 +765,16 @@ export function collectRetainedScreenshotReceipts(
707
765
  target.set(callId, direct);
708
766
  continue;
709
767
  }
710
- if (!Array.isArray(item.output)) continue;
711
- for (const entry of item.output) {
768
+ const outputEntries = Array.isArray(item.output)
769
+ ? item.output
770
+ : item.output &&
771
+ typeof item.output === "object" &&
772
+ !Array.isArray(item.output) &&
773
+ Array.isArray((item.output as Record<string, unknown>).content)
774
+ ? ((item.output as Record<string, unknown>).content as unknown[])
775
+ : null;
776
+ if (!outputEntries) continue;
777
+ for (const entry of outputEntries) {
712
778
  const receipt = retainedReceiptFromImageContent(entry);
713
779
  if (receipt) {
714
780
  target.set(callId, receipt);
@@ -872,18 +938,34 @@ async function materializeRetainedScreenshotHistoryWithCache(
872
938
  });
873
939
  continue;
874
940
  }
875
- if (!Array.isArray(item.output)) {
941
+ const resultOutput =
942
+ item.output && typeof item.output === "object" && !Array.isArray(item.output)
943
+ ? (item.output as Record<string, unknown>)
944
+ : null;
945
+ const outputEntries = Array.isArray(item.output)
946
+ ? item.output
947
+ : resultOutput && Array.isArray(resultOutput.content)
948
+ ? resultOutput.content
949
+ : null;
950
+ if (!outputEntries) {
876
951
  materialized.push(item);
877
952
  continue;
878
953
  }
879
954
  let changed = false;
880
955
  const output: unknown[] = [];
881
- for (const entry of item.output) {
956
+ for (const entry of outputEntries) {
882
957
  const next = await materializeEntry(entry);
883
958
  changed ||= next !== entry;
884
959
  output.push(next);
885
960
  }
886
- materialized.push(changed ? { ...item, output } : item);
961
+ materialized.push(
962
+ changed
963
+ ? {
964
+ ...item,
965
+ output: resultOutput ? { ...resultOutput, content: output } : output,
966
+ }
967
+ : item,
968
+ );
887
969
  }
888
970
  return materialized;
889
971
  }
@@ -1045,9 +1127,10 @@ function isInlineImageContent(entry: unknown): boolean {
1045
1127
  if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
1046
1128
  const record = entry as Record<string, unknown>;
1047
1129
  return (
1048
- record.type === "input_image" &&
1049
- typeof record.image === "string" &&
1050
- record.image.startsWith("data:image/")
1130
+ (record.type === "input_image" &&
1131
+ typeof record.image === "string" &&
1132
+ record.image.startsWith("data:image/")) ||
1133
+ (record.type === "image" && decodeInlineMcpImage(record.data, record.mimeType) !== null)
1051
1134
  );
1052
1135
  }
1053
1136
 
@@ -1429,7 +1429,7 @@ async function reconcileTerminalRetainedProcesses(
1429
1429
  }
1430
1430
 
1431
1431
  try {
1432
- await settleRetainedProcess(db, {
1432
+ const settlement = await settleRetainedProcess(db, {
1433
1433
  accountId: process.accountId,
1434
1434
  workspaceId: process.workspaceId,
1435
1435
  sessionId: process.sessionId,
@@ -1441,14 +1441,17 @@ async function reconcileTerminalRetainedProcesses(
1441
1441
  reason: proof.reason,
1442
1442
  idleGraceMs: settings.sandboxIdleGraceMs,
1443
1443
  });
1444
+ if (settlement.process.state === "active") {
1445
+ throw new Error("Retained-process reconciliation returned an active durable process");
1446
+ }
1444
1447
  await settleSessionBackgroundCommandForRetainedProcess(db, {
1445
1448
  accountId: process.accountId,
1446
1449
  workspaceId: process.workspaceId,
1447
1450
  sessionId: process.sessionId,
1448
1451
  retainedProcessId: process.id,
1449
- outcome: proof.outcome,
1450
- exitCode: proof.exitCode,
1451
- reason: proof.reason,
1452
+ outcome: settlement.process.state,
1453
+ exitCode: settlement.process.exitCode,
1454
+ reason: settlement.process.settlementReason ?? proof.reason,
1452
1455
  });
1453
1456
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
1454
1457
  } catch (error) {