@opengeni/worker-bundle 0.22.8 → 0.23.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +2 -4
- package/dist/activities/agent-turn/codex.d.ts +6 -5
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
- package/dist/activities/agent-turn/turn-context.d.ts +1 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
- package/dist/activities-control-52W5IDWM.js.map +1 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
- package/dist/activities-turn-A2G2KLSY.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
- package/dist/chunk-GFNXVZVQ.js.map +1 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
- package/dist/chunk-IPVMCAW5.js.map +1 -0
- package/dist/index.js +42 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +28 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/package.json +14 -14
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +29 -68
- package/src/activities/agent-turn/codex.ts +19 -10
- package/src/activities/agent-turn/compaction-prep.ts +50 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +14 -11
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +98 -14
- package/src/activities/agent-turn/tool-environment.ts +10 -13
- package/src/activities/agent-turn/tool-policy.ts +3 -55
- package/src/activities/agent-turn/turn-context.ts +8 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +10 -4
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +128 -4
- package/src/activities/session-state.ts +8 -1
- package/src/github-rest-mcp.ts +6 -3
- package/src/http.ts +3 -0
- package/src/index.ts +19 -0
- package/src/observability-metrics.ts +200 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -150,16 +150,21 @@ function imageBytesFromSdkToolOutput(
|
|
|
150
150
|
for (const entry of output) {
|
|
151
151
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
|
|
152
152
|
const record = entry as Record<string, unknown>;
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
153
|
+
const decoded =
|
|
154
|
+
record.type === "input_image"
|
|
155
|
+
? imageBytesFromSdkImageSource(record.image ?? record.image_url ?? record.imageUrl)
|
|
156
|
+
: record.type === "image"
|
|
157
|
+
? decodeInlineMcpImage(record.data, record.mimeType)
|
|
158
|
+
: null;
|
|
157
159
|
if (decoded) return decoded;
|
|
158
160
|
}
|
|
159
161
|
return null;
|
|
160
162
|
}
|
|
161
163
|
if (!output || typeof output !== "object") return null;
|
|
162
164
|
const outputRecord = output as Record<string, unknown>;
|
|
165
|
+
if (Array.isArray(outputRecord.content)) {
|
|
166
|
+
return imageBytesFromSdkToolOutput(outputRecord.content);
|
|
167
|
+
}
|
|
163
168
|
if (
|
|
164
169
|
outputRecord.type !== "image" ||
|
|
165
170
|
!outputRecord.image ||
|
|
@@ -202,16 +207,44 @@ export function toolOutputContainsInlineImage(output: unknown): boolean {
|
|
|
202
207
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
203
208
|
const record = entry as Record<string, unknown>;
|
|
204
209
|
return (
|
|
205
|
-
record.type === "input_image" &&
|
|
206
|
-
|
|
210
|
+
(record.type === "input_image" &&
|
|
211
|
+
sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)) ||
|
|
212
|
+
(record.type === "image" && decodeInlineMcpImage(record.data, record.mimeType) !== null)
|
|
207
213
|
);
|
|
208
214
|
});
|
|
209
215
|
}
|
|
210
216
|
if (!output || typeof output !== "object") return false;
|
|
211
217
|
const record = output as Record<string, unknown>;
|
|
218
|
+
if (Array.isArray(record.content)) return toolOutputContainsInlineImage(record.content);
|
|
212
219
|
return record.type === "image" && sdkImageSourceContainsInlineImage(record.image);
|
|
213
220
|
}
|
|
214
221
|
|
|
222
|
+
function decodeInlineMcpImage(
|
|
223
|
+
data: unknown,
|
|
224
|
+
mimeType: unknown,
|
|
225
|
+
): { bytes: Uint8Array; mediaType: string } | null {
|
|
226
|
+
if (
|
|
227
|
+
typeof data !== "string" ||
|
|
228
|
+
typeof mimeType !== "string" ||
|
|
229
|
+
!["image/png", "image/jpeg", "image/webp"].includes(mimeType)
|
|
230
|
+
) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
const maxEncodedLength = Math.ceil(COMPUTER_SCREENSHOT_MAX_BYTES / 3) * 4;
|
|
234
|
+
if (
|
|
235
|
+
data.length === 0 ||
|
|
236
|
+
data.length > maxEncodedLength ||
|
|
237
|
+
data.length % 4 !== 0 ||
|
|
238
|
+
!/^[A-Za-z0-9+/]+={0,2}$/u.test(data)
|
|
239
|
+
) {
|
|
240
|
+
return null;
|
|
241
|
+
}
|
|
242
|
+
const bytes = Uint8Array.from(Buffer.from(data, "base64"));
|
|
243
|
+
return bytes.byteLength > 0 && bytes.byteLength <= COMPUTER_SCREENSHOT_MAX_BYTES
|
|
244
|
+
? { bytes, mediaType: mimeType }
|
|
245
|
+
: null;
|
|
246
|
+
}
|
|
247
|
+
|
|
215
248
|
function decodeInlineImageDataUrl(value: unknown): { bytes: Uint8Array; mediaType: string } | null {
|
|
216
249
|
if (typeof value !== "string") return null;
|
|
217
250
|
const match = /^data:(image\/(?:png|jpeg|webp));base64,/u.exec(value);
|
|
@@ -680,18 +713,43 @@ export function compactRetainedScreenshotHistory(
|
|
|
680
713
|
output: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
|
|
681
714
|
};
|
|
682
715
|
}
|
|
716
|
+
if (
|
|
717
|
+
item.output &&
|
|
718
|
+
typeof item.output === "object" &&
|
|
719
|
+
!Array.isArray(item.output) &&
|
|
720
|
+
Array.isArray((item.output as Record<string, unknown>).content)
|
|
721
|
+
) {
|
|
722
|
+
const result = item.output as Record<string, unknown>;
|
|
723
|
+
const content = compactInlineImageContent(result.content as unknown[], receipt);
|
|
724
|
+
return content.changed ? { ...item, output: { ...result, content: content.entries } } : item;
|
|
725
|
+
}
|
|
683
726
|
if (!Array.isArray(item.output)) return item;
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
727
|
+
const output = compactInlineImageContent(item.output, receipt);
|
|
728
|
+
return output.changed ? { ...item, output: output.entries } : item;
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
function compactInlineImageContent(
|
|
733
|
+
entries: unknown[],
|
|
734
|
+
receipt: RetainedArtifactMetadata,
|
|
735
|
+
): { entries: unknown[]; changed: boolean } {
|
|
736
|
+
let changed = false;
|
|
737
|
+
const compacted = entries.map((entry) => {
|
|
738
|
+
if (!isInlineImageContent(entry)) return entry;
|
|
739
|
+
changed = true;
|
|
740
|
+
const record = entry as Record<string, unknown>;
|
|
741
|
+
if (record.type === "image") {
|
|
688
742
|
return {
|
|
689
|
-
|
|
743
|
+
type: "input_image",
|
|
690
744
|
image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
|
|
691
745
|
};
|
|
692
|
-
}
|
|
693
|
-
return
|
|
746
|
+
}
|
|
747
|
+
return {
|
|
748
|
+
...record,
|
|
749
|
+
image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
|
|
750
|
+
};
|
|
694
751
|
});
|
|
752
|
+
return { entries: compacted, changed };
|
|
695
753
|
}
|
|
696
754
|
|
|
697
755
|
/** Collect durable screenshot receipts before their provider-only expansion. */
|
|
@@ -707,8 +765,16 @@ export function collectRetainedScreenshotReceipts(
|
|
|
707
765
|
target.set(callId, direct);
|
|
708
766
|
continue;
|
|
709
767
|
}
|
|
710
|
-
|
|
711
|
-
|
|
768
|
+
const outputEntries = Array.isArray(item.output)
|
|
769
|
+
? item.output
|
|
770
|
+
: item.output &&
|
|
771
|
+
typeof item.output === "object" &&
|
|
772
|
+
!Array.isArray(item.output) &&
|
|
773
|
+
Array.isArray((item.output as Record<string, unknown>).content)
|
|
774
|
+
? ((item.output as Record<string, unknown>).content as unknown[])
|
|
775
|
+
: null;
|
|
776
|
+
if (!outputEntries) continue;
|
|
777
|
+
for (const entry of outputEntries) {
|
|
712
778
|
const receipt = retainedReceiptFromImageContent(entry);
|
|
713
779
|
if (receipt) {
|
|
714
780
|
target.set(callId, receipt);
|
|
@@ -872,18 +938,34 @@ async function materializeRetainedScreenshotHistoryWithCache(
|
|
|
872
938
|
});
|
|
873
939
|
continue;
|
|
874
940
|
}
|
|
875
|
-
|
|
941
|
+
const resultOutput =
|
|
942
|
+
item.output && typeof item.output === "object" && !Array.isArray(item.output)
|
|
943
|
+
? (item.output as Record<string, unknown>)
|
|
944
|
+
: null;
|
|
945
|
+
const outputEntries = Array.isArray(item.output)
|
|
946
|
+
? item.output
|
|
947
|
+
: resultOutput && Array.isArray(resultOutput.content)
|
|
948
|
+
? resultOutput.content
|
|
949
|
+
: null;
|
|
950
|
+
if (!outputEntries) {
|
|
876
951
|
materialized.push(item);
|
|
877
952
|
continue;
|
|
878
953
|
}
|
|
879
954
|
let changed = false;
|
|
880
955
|
const output: unknown[] = [];
|
|
881
|
-
for (const entry of
|
|
956
|
+
for (const entry of outputEntries) {
|
|
882
957
|
const next = await materializeEntry(entry);
|
|
883
958
|
changed ||= next !== entry;
|
|
884
959
|
output.push(next);
|
|
885
960
|
}
|
|
886
|
-
materialized.push(
|
|
961
|
+
materialized.push(
|
|
962
|
+
changed
|
|
963
|
+
? {
|
|
964
|
+
...item,
|
|
965
|
+
output: resultOutput ? { ...resultOutput, content: output } : output,
|
|
966
|
+
}
|
|
967
|
+
: item,
|
|
968
|
+
);
|
|
887
969
|
}
|
|
888
970
|
return materialized;
|
|
889
971
|
}
|
|
@@ -1045,9 +1127,10 @@ function isInlineImageContent(entry: unknown): boolean {
|
|
|
1045
1127
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
1046
1128
|
const record = entry as Record<string, unknown>;
|
|
1047
1129
|
return (
|
|
1048
|
-
record.type === "input_image" &&
|
|
1049
|
-
|
|
1050
|
-
|
|
1130
|
+
(record.type === "input_image" &&
|
|
1131
|
+
typeof record.image === "string" &&
|
|
1132
|
+
record.image.startsWith("data:image/")) ||
|
|
1133
|
+
(record.type === "image" && decodeInlineMcpImage(record.data, record.mimeType) !== null)
|
|
1051
1134
|
);
|
|
1052
1135
|
}
|
|
1053
1136
|
|
|
@@ -1429,7 +1429,7 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1429
1429
|
}
|
|
1430
1430
|
|
|
1431
1431
|
try {
|
|
1432
|
-
await settleRetainedProcess(db, {
|
|
1432
|
+
const settlement = await settleRetainedProcess(db, {
|
|
1433
1433
|
accountId: process.accountId,
|
|
1434
1434
|
workspaceId: process.workspaceId,
|
|
1435
1435
|
sessionId: process.sessionId,
|
|
@@ -1441,14 +1441,17 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1441
1441
|
reason: proof.reason,
|
|
1442
1442
|
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1443
1443
|
});
|
|
1444
|
+
if (settlement.process.state === "active") {
|
|
1445
|
+
throw new Error("Retained-process reconciliation returned an active durable process");
|
|
1446
|
+
}
|
|
1444
1447
|
await settleSessionBackgroundCommandForRetainedProcess(db, {
|
|
1445
1448
|
accountId: process.accountId,
|
|
1446
1449
|
workspaceId: process.workspaceId,
|
|
1447
1450
|
sessionId: process.sessionId,
|
|
1448
1451
|
retainedProcessId: process.id,
|
|
1449
|
-
outcome:
|
|
1450
|
-
exitCode:
|
|
1451
|
-
reason: proof.reason,
|
|
1452
|
+
outcome: settlement.process.state,
|
|
1453
|
+
exitCode: settlement.process.exitCode,
|
|
1454
|
+
reason: settlement.process.settlementReason ?? proof.reason,
|
|
1452
1455
|
});
|
|
1453
1456
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
1454
1457
|
} catch (error) {
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
requireOpenGeniSlackBotConnection,
|
|
20
20
|
resolveSessionToolPolicy,
|
|
21
21
|
scheduledSlackBotConnectionId,
|
|
22
|
+
swapActiveSandbox,
|
|
22
23
|
} from "@opengeni/core";
|
|
23
24
|
import {
|
|
24
25
|
appendSessionEvents,
|
|
@@ -28,6 +29,7 @@ import {
|
|
|
28
29
|
createSession,
|
|
29
30
|
createSessionWithIdempotencyKeyResult,
|
|
30
31
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
32
|
+
failScheduledGeneratedSessionRoute,
|
|
31
33
|
getScheduledTask,
|
|
32
34
|
getScheduledTaskRunAcceptedExecution,
|
|
33
35
|
getScheduledTaskRunByProducerKey,
|
|
@@ -38,6 +40,8 @@ import {
|
|
|
38
40
|
getScheduledTaskPersonalResourceAuthoritySubject,
|
|
39
41
|
getScheduledTaskRevisionAuthority,
|
|
40
42
|
getScheduledTaskXaiProviderAccountAuthoritySnapshot,
|
|
43
|
+
getEnrollment,
|
|
44
|
+
getSandbox,
|
|
41
45
|
getRig,
|
|
42
46
|
getScheduledScopedRigVersionMetadata,
|
|
43
47
|
getNestedAgentDepthDeploymentPolicy,
|
|
@@ -442,12 +446,41 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
442
446
|
taskConnectionAuthoritySubjectId;
|
|
443
447
|
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
444
448
|
const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
445
|
-
|
|
449
|
+
let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
450
|
+
let sandboxOs: "linux" | "macos" | "windows" = "linux";
|
|
446
451
|
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
447
452
|
const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
|
|
448
453
|
const generatedTarget =
|
|
449
454
|
task.runMode === "new_session_per_run" ||
|
|
450
455
|
(task.runMode === "reusable_session" && task.reusableSessionId === null);
|
|
456
|
+
if (generatedTarget && task.agentConfig.machineTarget) {
|
|
457
|
+
const access = {
|
|
458
|
+
accountId: task.accountId,
|
|
459
|
+
workspaceId: task.workspaceId,
|
|
460
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId,
|
|
461
|
+
};
|
|
462
|
+
const machine = await getSandbox(
|
|
463
|
+
db,
|
|
464
|
+
access,
|
|
465
|
+
task.agentConfig.machineTarget.targetSandboxId,
|
|
466
|
+
);
|
|
467
|
+
if (
|
|
468
|
+
!machine ||
|
|
469
|
+
machine.kind !== "selfhosted" ||
|
|
470
|
+
!machine.enrollmentId ||
|
|
471
|
+
machine.scope === "user"
|
|
472
|
+
) {
|
|
473
|
+
throw new Error("scheduled Connected Machine target is unavailable");
|
|
474
|
+
}
|
|
475
|
+
const enrollment = await getEnrollment(db, access, machine.enrollmentId);
|
|
476
|
+
if (!enrollment || enrollment.status !== "active") {
|
|
477
|
+
throw new Error("scheduled Connected Machine enrollment is unavailable");
|
|
478
|
+
}
|
|
479
|
+
sandboxBackend = "selfhosted";
|
|
480
|
+
sandboxOs = enrollment.os;
|
|
481
|
+
} else if (generatedTarget && sandboxBackend === "selfhosted") {
|
|
482
|
+
throw new Error("self-hosted scheduled task has no Connected Machine target");
|
|
483
|
+
}
|
|
451
484
|
const generatedSessionDepthPolicy = generatedTarget
|
|
452
485
|
? await (async () => {
|
|
453
486
|
const workspace = await requireWorkspace(db, task.workspaceId);
|
|
@@ -730,7 +763,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
730
763
|
resolvedReasoningEffort: reasoningEffort,
|
|
731
764
|
resolvedLatencyMode: "standard",
|
|
732
765
|
resolvedSandboxBackend: sandboxBackend,
|
|
733
|
-
resolvedSandboxOs:
|
|
766
|
+
resolvedSandboxOs: sandboxOs,
|
|
734
767
|
resolvedTools: taskTools,
|
|
735
768
|
resolvedFirstPartyMcpTools: firstPartyMcpTools,
|
|
736
769
|
resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
@@ -972,7 +1005,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
972
1005
|
reasoningEffort,
|
|
973
1006
|
latencyMode: "standard",
|
|
974
1007
|
sandboxBackend,
|
|
975
|
-
sandboxOs
|
|
1008
|
+
sandboxOs,
|
|
976
1009
|
variableSetId: task.variableSetId ?? null,
|
|
977
1010
|
rigId: frozenRigId,
|
|
978
1011
|
rigVersionId: frozenRigVersionId,
|
|
@@ -1079,6 +1112,16 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
1079
1112
|
);
|
|
1080
1113
|
}
|
|
1081
1114
|
}
|
|
1115
|
+
session = await seedScheduledGeneratedSessionRoute({
|
|
1116
|
+
db: dispatchDb,
|
|
1117
|
+
bus,
|
|
1118
|
+
settings,
|
|
1119
|
+
task,
|
|
1120
|
+
runId: run.id,
|
|
1121
|
+
session,
|
|
1122
|
+
deferPublications,
|
|
1123
|
+
deferredEvents,
|
|
1124
|
+
});
|
|
1082
1125
|
let workflowId: string;
|
|
1083
1126
|
if (alertOccurrence) {
|
|
1084
1127
|
const started = await initializeSessionStartAtomically(dispatchDb, {
|
|
@@ -1539,7 +1582,6 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
1539
1582
|
acceptedExecution: dispatchOutcome.acceptedExecution,
|
|
1540
1583
|
});
|
|
1541
1584
|
}
|
|
1542
|
-
if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
|
|
1543
1585
|
for (const deferred of deferredEvents) {
|
|
1544
1586
|
await publishDurableSessionEvents(
|
|
1545
1587
|
bus,
|
|
@@ -1548,6 +1590,7 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
|
|
|
1548
1590
|
deferred.events,
|
|
1549
1591
|
);
|
|
1550
1592
|
}
|
|
1593
|
+
if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
|
|
1551
1594
|
const result = dispatchOutcome.result;
|
|
1552
1595
|
if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
|
|
1553
1596
|
await wakeSessionWorkflow({
|
|
@@ -1773,6 +1816,74 @@ async function recordScheduledTaskFiredUsage(
|
|
|
1773
1816
|
});
|
|
1774
1817
|
}
|
|
1775
1818
|
|
|
1819
|
+
async function seedScheduledGeneratedSessionRoute(input: {
|
|
1820
|
+
db: Database;
|
|
1821
|
+
bus: ControlActivityServices["bus"];
|
|
1822
|
+
settings: ControlActivityServices["settings"];
|
|
1823
|
+
task: ScheduledTask;
|
|
1824
|
+
runId: string;
|
|
1825
|
+
session: Awaited<ReturnType<typeof createSession>>;
|
|
1826
|
+
deferPublications: boolean;
|
|
1827
|
+
deferredEvents: Array<{
|
|
1828
|
+
sessionId: string;
|
|
1829
|
+
events: Awaited<ReturnType<typeof appendSessionEvents>>;
|
|
1830
|
+
}>;
|
|
1831
|
+
}): Promise<Awaited<ReturnType<typeof createSession>>> {
|
|
1832
|
+
const target = input.task.agentConfig.machineTarget;
|
|
1833
|
+
if (!target) return input.session;
|
|
1834
|
+
const seeded = await swapActiveSandbox(
|
|
1835
|
+
{
|
|
1836
|
+
db: input.db,
|
|
1837
|
+
settings: input.settings,
|
|
1838
|
+
bus: input.bus,
|
|
1839
|
+
},
|
|
1840
|
+
{
|
|
1841
|
+
accountId: input.task.accountId,
|
|
1842
|
+
workspaceId: input.task.workspaceId,
|
|
1843
|
+
sessionId: input.session.id,
|
|
1844
|
+
sessionBackend: input.session.sandboxBackend,
|
|
1845
|
+
sessionGroupId: input.session.sandboxGroupId,
|
|
1846
|
+
},
|
|
1847
|
+
target.targetSandboxId,
|
|
1848
|
+
target.workingDir ?? null,
|
|
1849
|
+
);
|
|
1850
|
+
if (seeded.swapped) {
|
|
1851
|
+
return await requireSession(input.db, input.task.workspaceId, input.session.id);
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
const failed = await failScheduledGeneratedSessionRoute(input.db, {
|
|
1855
|
+
accountId: input.task.accountId,
|
|
1856
|
+
workspaceId: input.task.workspaceId,
|
|
1857
|
+
taskId: input.task.id,
|
|
1858
|
+
runId: input.runId,
|
|
1859
|
+
sessionId: input.session.id,
|
|
1860
|
+
error: "scheduled_machine_unavailable",
|
|
1861
|
+
});
|
|
1862
|
+
if (failed.action === "advanced") {
|
|
1863
|
+
throw new Error(
|
|
1864
|
+
`scheduled run advanced to ${failed.status} while its Connected Machine route was being established`,
|
|
1865
|
+
);
|
|
1866
|
+
}
|
|
1867
|
+
if (failed.events.length > 0) {
|
|
1868
|
+
if (input.deferPublications) {
|
|
1869
|
+
input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
|
|
1870
|
+
} else {
|
|
1871
|
+
await publishDurableSessionEvents(
|
|
1872
|
+
input.bus,
|
|
1873
|
+
input.task.workspaceId,
|
|
1874
|
+
input.session.id,
|
|
1875
|
+
failed.events,
|
|
1876
|
+
);
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
const error =
|
|
1880
|
+
failed.action === "terminal" ? (failed.error ?? "scheduled_run_terminal") : failed.error;
|
|
1881
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
1882
|
+
error,
|
|
1883
|
+
seeded.reason ?? "scheduled Connected Machine target is unavailable",
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1776
1887
|
async function recoverBoundScheduledTaskDispatch(input: {
|
|
1777
1888
|
db: Database;
|
|
1778
1889
|
bus: ControlActivityServices["bus"];
|
|
@@ -1952,6 +2063,18 @@ async function recoverBoundScheduledTaskDispatch(input: {
|
|
|
1952
2063
|
sessionId: session.id,
|
|
1953
2064
|
});
|
|
1954
2065
|
}
|
|
2066
|
+
if (generatedSession) {
|
|
2067
|
+
session = await seedScheduledGeneratedSessionRoute({
|
|
2068
|
+
db: input.db,
|
|
2069
|
+
bus: input.bus,
|
|
2070
|
+
settings: input.settings,
|
|
2071
|
+
task,
|
|
2072
|
+
runId: input.run.id,
|
|
2073
|
+
session,
|
|
2074
|
+
deferPublications: false,
|
|
2075
|
+
deferredEvents: [],
|
|
2076
|
+
});
|
|
2077
|
+
}
|
|
1955
2078
|
if (session.status === "cancelled") {
|
|
1956
2079
|
await markScheduledTaskRunSkippedIfQueued(input.db, {
|
|
1957
2080
|
workspaceId: task.workspaceId,
|
|
@@ -2024,6 +2147,7 @@ async function recoverBoundScheduledTaskDispatch(input: {
|
|
|
2024
2147
|
session.latencyMode !== input.acceptedExecution.resolvedLatencyMode ||
|
|
2025
2148
|
session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend ||
|
|
2026
2149
|
session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs ||
|
|
2150
|
+
session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) ||
|
|
2027
2151
|
stableJson(session.resources) !== stableJson(task.agentConfig.resources) ||
|
|
2028
2152
|
stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) ||
|
|
2029
2153
|
stableJson(session.firstPartyMcpTools) !==
|
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
} from "@opengeni/db";
|
|
20
20
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
21
21
|
import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake";
|
|
22
|
-
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
22
|
+
import { recordTurnsQueuedGauge, recordWorkerDeathRecoveryMetrics } from "../observability-metrics";
|
|
23
23
|
import {
|
|
24
24
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
25
25
|
providerRecoveryCountFromMetadata,
|
|
@@ -66,6 +66,7 @@ export type SessionStateActivityOverrides = Partial<{
|
|
|
66
66
|
deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
|
|
67
67
|
notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
|
|
68
68
|
recordTurnsQueuedGauge: typeof recordTurnsQueuedGauge;
|
|
69
|
+
recordWorkerDeathRecoveryMetrics: typeof recordWorkerDeathRecoveryMetrics;
|
|
69
70
|
}>;
|
|
70
71
|
|
|
71
72
|
// Crash-loop guard for worker-death re-dispatch: a turn that takes a worker
|
|
@@ -112,6 +113,8 @@ export function createSessionStateActivities(
|
|
|
112
113
|
overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
|
|
113
114
|
const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
|
|
114
115
|
const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
|
|
116
|
+
const recordWorkerDeathRecoveryMetricsFn =
|
|
117
|
+
overrides.recordWorkerDeathRecoveryMetrics ?? recordWorkerDeathRecoveryMetrics;
|
|
115
118
|
|
|
116
119
|
async function failSessionAttempt(
|
|
117
120
|
input: FailSessionAttemptInput,
|
|
@@ -397,6 +400,10 @@ export function createSessionStateActivities(
|
|
|
397
400
|
if (result.action === "stale" || result.action === "unclaimed") {
|
|
398
401
|
return { action: result.action };
|
|
399
402
|
}
|
|
403
|
+
recordWorkerDeathRecoveryMetricsFn(observability, {
|
|
404
|
+
outcome: result.action === "exceeded" ? "exhausted" : "recovering",
|
|
405
|
+
timeoutType: input.timeoutType,
|
|
406
|
+
});
|
|
400
407
|
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
401
408
|
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
402
409
|
if (result.action === "exceeded") {
|
package/src/github-rest-mcp.ts
CHANGED
|
@@ -258,9 +258,12 @@ function appRepositoryAuthority(resources: ResourceRef[]): GitHubRestRepository[
|
|
|
258
258
|
);
|
|
259
259
|
const repositoryId = positiveSafeInteger(resource.githubRepositoryId ?? resource.repositoryId);
|
|
260
260
|
const fullName = canonicalGitHubFullName(resource.uri);
|
|
261
|
-
if (!installationId || !repositoryId
|
|
261
|
+
if (!installationId || !repositoryId) {
|
|
262
262
|
throw new GitHubRestAuthorityError("GitHub App repository authority is incomplete");
|
|
263
263
|
}
|
|
264
|
+
if (!fullName) {
|
|
265
|
+
throw new GitHubRestAuthorityError("GitHub App repository URL is invalid");
|
|
266
|
+
}
|
|
264
267
|
return [
|
|
265
268
|
{
|
|
266
269
|
repositoryId: String(repositoryId),
|
|
@@ -645,7 +648,8 @@ function githubConnectorBindings(
|
|
|
645
648
|
function canonicalGitHubFullName(uri: string): string | null {
|
|
646
649
|
try {
|
|
647
650
|
const url = new URL(uri);
|
|
648
|
-
const
|
|
651
|
+
const clonePath = url.pathname.replace(/^\/+|\/+$/gu, "");
|
|
652
|
+
const path = clonePath.endsWith(".git") ? clonePath.slice(0, -4) : clonePath;
|
|
649
653
|
if (
|
|
650
654
|
url.protocol !== "https:" ||
|
|
651
655
|
url.hostname !== "github.com" ||
|
|
@@ -654,7 +658,6 @@ function canonicalGitHubFullName(uri: string): string | null {
|
|
|
654
658
|
url.password ||
|
|
655
659
|
url.search ||
|
|
656
660
|
url.hash ||
|
|
657
|
-
path.endsWith(".git") ||
|
|
658
661
|
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(path)
|
|
659
662
|
) {
|
|
660
663
|
return null;
|
package/src/http.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
type RuntimeDatabasePostureOptions,
|
|
7
7
|
} from "@opengeni/db";
|
|
8
8
|
import { requireSessionEventDurableFanoutCapability, type EventBus } from "@opengeni/events";
|
|
9
|
+
import { githubAppBotIdentityWarnings } from "@opengeni/github";
|
|
9
10
|
import type { Observability } from "@opengeni/observability";
|
|
10
11
|
|
|
11
12
|
export type ReadinessCheckName = "db" | "nats" | "temporal";
|
|
@@ -45,12 +46,14 @@ export function createWorkerHttpHandler(
|
|
|
45
46
|
if (url.pathname === "/healthz") {
|
|
46
47
|
const state = lifecycle?.state();
|
|
47
48
|
const ok = state !== "failed" && state !== "stopped";
|
|
49
|
+
const warnings = githubAppBotIdentityWarnings(settings);
|
|
48
50
|
return Response.json(
|
|
49
51
|
{
|
|
50
52
|
service: settings.serviceName,
|
|
51
53
|
environment: settings.environment,
|
|
52
54
|
deploymentRevision: settings.deploymentRevision,
|
|
53
55
|
...(lifecycle ? { role: lifecycle.role, state } : {}),
|
|
56
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
54
57
|
ok,
|
|
55
58
|
},
|
|
56
59
|
{ status: ok ? 200 : 503 },
|
package/src/index.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from "@opengeni/config";
|
|
10
10
|
import {
|
|
11
11
|
assertRuntimeDatabasePosture,
|
|
12
|
+
countSessionRecoveryBacklog,
|
|
12
13
|
createDb,
|
|
13
14
|
markSessionWorkflowWakeDelivered,
|
|
14
15
|
type Database,
|
|
@@ -55,8 +56,10 @@ import {
|
|
|
55
56
|
type WorkerLifecycleState,
|
|
56
57
|
} from "./http";
|
|
57
58
|
import {
|
|
59
|
+
initializeWorkerOutcomeMetrics,
|
|
58
60
|
normalizeTurnTaskQueueStats,
|
|
59
61
|
observabilityEventLogger,
|
|
62
|
+
startSessionRecoveryMonitor,
|
|
60
63
|
startTurnCapacityMonitor,
|
|
61
64
|
type TurnTaskQueueStats,
|
|
62
65
|
} from "./observability-metrics";
|
|
@@ -86,6 +89,7 @@ import {
|
|
|
86
89
|
type TurnWorkerMemoryPressureGuard,
|
|
87
90
|
} from "./memory-pressure-guard";
|
|
88
91
|
import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
|
|
92
|
+
import { installWorkerUnhandledRejectionBoundary } from "./unhandled-rejection-boundary";
|
|
89
93
|
import {
|
|
90
94
|
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
91
95
|
sandboxLifecycleTaskQueue,
|
|
@@ -219,6 +223,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
219
223
|
const observability =
|
|
220
224
|
options.activityDependencies?.observability ??
|
|
221
225
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
226
|
+
initializeWorkerOutcomeMetrics(observability);
|
|
222
227
|
if (options.role === "turn" && options.workflowBundle) {
|
|
223
228
|
throw new Error("workflowBundle is valid only for the control worker role");
|
|
224
229
|
}
|
|
@@ -793,6 +798,7 @@ export async function createOpenGeniWorkerService(
|
|
|
793
798
|
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
794
799
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
795
800
|
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
801
|
+
let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
|
|
796
802
|
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
797
803
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
798
804
|
let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
|
|
@@ -866,6 +872,11 @@ export async function createOpenGeniWorkerService(
|
|
|
866
872
|
observability,
|
|
867
873
|
read: signaler.getTurnTaskQueueStats,
|
|
868
874
|
});
|
|
875
|
+
} else {
|
|
876
|
+
sessionRecoveryMonitor = startSessionRecoveryMonitor({
|
|
877
|
+
observability,
|
|
878
|
+
read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
|
|
879
|
+
});
|
|
869
880
|
}
|
|
870
881
|
|
|
871
882
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
@@ -919,6 +930,7 @@ export async function createOpenGeniWorkerService(
|
|
|
919
930
|
httpServer?.stop(true);
|
|
920
931
|
await Promise.allSettled([
|
|
921
932
|
turnCapacityMonitor?.close(),
|
|
933
|
+
sessionRecoveryMonitor?.close(),
|
|
922
934
|
workerBundle?.connection.close(),
|
|
923
935
|
signaler?.close(),
|
|
924
936
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -941,6 +953,7 @@ export async function createOpenGeniWorkerService(
|
|
|
941
953
|
httpServer?.stop(true);
|
|
942
954
|
await Promise.allSettled([
|
|
943
955
|
turnCapacityMonitor?.close(),
|
|
956
|
+
sessionRecoveryMonitor?.close(),
|
|
944
957
|
activeWorkerBundle.connection.close(),
|
|
945
958
|
activeSignaler?.close(),
|
|
946
959
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -1032,6 +1045,11 @@ export async function startWorker() {
|
|
|
1032
1045
|
}
|
|
1033
1046
|
const settings = getSettings();
|
|
1034
1047
|
const observability = createObservability(settings, { component: `worker-${role}` });
|
|
1048
|
+
// The Agents SDK also listens for unhandled rejections and exits the process.
|
|
1049
|
+
// Keep detached SDK/transport promises observational here: durable turn
|
|
1050
|
+
// activity boundaries own failures, while process exit causes cross-session
|
|
1051
|
+
// lease loss for every turn assigned to this shared worker pod.
|
|
1052
|
+
const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
|
|
1035
1053
|
const retryOptions = startupRetryOptions(settings);
|
|
1036
1054
|
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
|
|
1037
1055
|
logStartupDependencyRetry(observability, event);
|
|
@@ -1090,6 +1108,7 @@ export async function startWorker() {
|
|
|
1090
1108
|
});
|
|
1091
1109
|
} finally {
|
|
1092
1110
|
await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
|
|
1111
|
+
disposeUnhandledRejectionBoundary();
|
|
1093
1112
|
}
|
|
1094
1113
|
}
|
|
1095
1114
|
|