@opengeni/worker-bundle 0.16.21 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +61 -14
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/environment.d.ts +1 -0
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +93 -0
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +2 -0
- package/dist/activities/toolspace-token-renewal.d.ts +1 -1
- package/dist/activities/workspace-capture.d.ts +1 -0
- package/dist/activities.d.ts +3 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/host-export.js +1 -1
- package/dist/index.js +1317 -390
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +6 -0
- package/dist/sandbox-routing.d.ts +11 -6
- package/dist/workflow-bundle.js +2 -1
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +482 -360
- package/src/activities/codex-capacity.ts +10 -1
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +9 -3
- package/src/activities/environment.ts +13 -24
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +892 -0
- package/src/activities/rig-verification.ts +13 -34
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +18 -4
- package/src/activities/scheduled-tasks.ts +19 -3
- package/src/activities/toolspace-token-renewal.ts +9 -2
- package/src/activities/workspace-capture.ts +33 -3
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/observability-metrics.ts +3 -3
- package/src/sandbox-resume.ts +45 -12
- package/src/sandbox-routing.ts +30 -15
- package/src/worker-service-lifecycle.ts +8 -4
- package/src/workflows/file-upload-reaper.ts +4 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
|
@@ -16,9 +16,6 @@ import {
|
|
|
16
16
|
markWarmLeaseInstanceLost,
|
|
17
17
|
recordWarmingSandboxCreated,
|
|
18
18
|
releaseLeaseHolder,
|
|
19
|
-
sanitizeEventPayload,
|
|
20
|
-
sanitizeEventString,
|
|
21
|
-
sanitizeMemoryText,
|
|
22
19
|
touchLeaseHolder,
|
|
23
20
|
updateRigChangeStatus,
|
|
24
21
|
type Database,
|
|
@@ -49,7 +46,6 @@ type CommandResult = {
|
|
|
49
46
|
output: string;
|
|
50
47
|
};
|
|
51
48
|
|
|
52
|
-
const OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
53
49
|
const RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 60_000;
|
|
54
50
|
const RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 60_000;
|
|
55
51
|
const RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 10_000;
|
|
@@ -245,18 +241,6 @@ function cleanupWaitMs(lifecycle: RigVerificationActivityLifecycle | undefined):
|
|
|
245
241
|
);
|
|
246
242
|
}
|
|
247
243
|
|
|
248
|
-
function tail(value: string, limit = OUTPUT_TAIL_LIMIT): string {
|
|
249
|
-
return value.length > limit ? value.slice(-limit) : value;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
function scrubVerificationOutput(value: string): string {
|
|
253
|
-
return sanitizeMemoryText(sanitizeEventString(value)).text;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
function scrubVerificationPayload<T>(value: T): T {
|
|
257
|
-
return sanitizeEventPayload(value);
|
|
258
|
-
}
|
|
259
|
-
|
|
260
244
|
function systemGrant(rig: Rig): AccessGrant {
|
|
261
245
|
return {
|
|
262
246
|
accountId: rig.accountId,
|
|
@@ -723,7 +707,7 @@ async function runCommand(
|
|
|
723
707
|
const result = await commandRunner(session, args);
|
|
724
708
|
return {
|
|
725
709
|
exitCode: sandboxCommandExitCode(result),
|
|
726
|
-
output:
|
|
710
|
+
output: sandboxCommandOutput(result),
|
|
727
711
|
};
|
|
728
712
|
}
|
|
729
713
|
|
|
@@ -891,7 +875,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
891
875
|
verification.passed = false;
|
|
892
876
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
893
877
|
status: "rejected",
|
|
894
|
-
verification
|
|
878
|
+
verification,
|
|
895
879
|
});
|
|
896
880
|
await recordRigAuditEvent(db, {
|
|
897
881
|
grant,
|
|
@@ -918,7 +902,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
918
902
|
);
|
|
919
903
|
checkResults.push({
|
|
920
904
|
name: check.name,
|
|
921
|
-
command:
|
|
905
|
+
command: check.command,
|
|
922
906
|
...result,
|
|
923
907
|
});
|
|
924
908
|
}
|
|
@@ -935,7 +919,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
935
919
|
// (a second run could reject a change whose first verification passed).
|
|
936
920
|
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
937
921
|
status: "verifying",
|
|
938
|
-
verification
|
|
922
|
+
verification,
|
|
939
923
|
});
|
|
940
924
|
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
941
925
|
...change,
|
|
@@ -951,7 +935,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
951
935
|
}
|
|
952
936
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
953
937
|
status: classified.status,
|
|
954
|
-
verification
|
|
938
|
+
verification,
|
|
955
939
|
});
|
|
956
940
|
await recordRigAuditEvent(db, {
|
|
957
941
|
grant,
|
|
@@ -973,12 +957,10 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
973
957
|
} catch (error) {
|
|
974
958
|
verification.finishedAt = new Date().toISOString();
|
|
975
959
|
verification.passed = false;
|
|
976
|
-
verification.error =
|
|
977
|
-
error instanceof Error ? error.message : String(error),
|
|
978
|
-
);
|
|
960
|
+
verification.error = error instanceof Error ? error.message : String(error);
|
|
979
961
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
980
962
|
status: "failed",
|
|
981
|
-
verification
|
|
963
|
+
verification,
|
|
982
964
|
});
|
|
983
965
|
await recordRigAuditEvent(db, {
|
|
984
966
|
grant,
|
|
@@ -1043,7 +1025,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1043
1025
|
for (const check of version.checks) {
|
|
1044
1026
|
checkResults.push({
|
|
1045
1027
|
name: check.name,
|
|
1046
|
-
command:
|
|
1028
|
+
command: check.command,
|
|
1047
1029
|
...(await runCommand(
|
|
1048
1030
|
established.session as TurnSandboxCommandSession,
|
|
1049
1031
|
check.command,
|
|
@@ -1057,13 +1039,13 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1057
1039
|
grant,
|
|
1058
1040
|
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
1059
1041
|
rigId: rig.id,
|
|
1060
|
-
metadata:
|
|
1042
|
+
metadata: {
|
|
1061
1043
|
versionId: version.id,
|
|
1062
1044
|
startedAt,
|
|
1063
1045
|
finishedAt: new Date().toISOString(),
|
|
1064
1046
|
passed,
|
|
1065
1047
|
checkResults,
|
|
1066
|
-
}
|
|
1048
|
+
},
|
|
1067
1049
|
});
|
|
1068
1050
|
return { versionId: version.id, passed, checkResults };
|
|
1069
1051
|
},
|
|
@@ -1073,21 +1055,18 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1073
1055
|
// rig.verification.failed so activeVersionHealth reflects the failed
|
|
1074
1056
|
// re-run instead of staying stale, symmetric to verifyRigChange. Then
|
|
1075
1057
|
// rethrow so the Temporal activity still surfaces the failure.
|
|
1076
|
-
const detail =
|
|
1077
|
-
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
1078
|
-
4096,
|
|
1079
|
-
);
|
|
1058
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1080
1059
|
await recordRigAuditEvent(db, {
|
|
1081
1060
|
grant,
|
|
1082
1061
|
action: "rig.verification.failed",
|
|
1083
1062
|
rigId: rig.id,
|
|
1084
|
-
metadata:
|
|
1063
|
+
metadata: {
|
|
1085
1064
|
versionId: version.id,
|
|
1086
1065
|
startedAt,
|
|
1087
1066
|
finishedAt: new Date().toISOString(),
|
|
1088
1067
|
passed: false,
|
|
1089
1068
|
error: detail,
|
|
1090
|
-
}
|
|
1069
|
+
},
|
|
1091
1070
|
});
|
|
1092
1071
|
throw error;
|
|
1093
1072
|
}
|
|
@@ -22,7 +22,10 @@ export type RunCredentialRenewalOptions = {
|
|
|
22
22
|
schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
|
|
23
23
|
clearSchedule?: (timer: RenewalTimer) => void;
|
|
24
24
|
onSuccess?: (result: { nextDelayMs: number; authNeeded: boolean }) => void;
|
|
25
|
-
onFailure?: (failure: {
|
|
25
|
+
onFailure?: (failure: {
|
|
26
|
+
retryDelayMs: number;
|
|
27
|
+
errorClass: "RunCredentialRenewalOperationError";
|
|
28
|
+
}) => void;
|
|
26
29
|
};
|
|
27
30
|
|
|
28
31
|
export function nextRunCredentialRenewalDelay(expiresAt: Date | null, nowMs = Date.now()): number {
|
|
@@ -42,11 +45,15 @@ export function nextRunCredentialRenewalDelay(expiresAt: Date | null, nowMs = Da
|
|
|
42
45
|
export function startRunCredentialRenewalLoop(
|
|
43
46
|
options: RunCredentialRenewalOptions,
|
|
44
47
|
): RunCredentialRenewalController {
|
|
45
|
-
return startExpiringMaterialRenewalLoop<
|
|
48
|
+
return startExpiringMaterialRenewalLoop<
|
|
49
|
+
NormalizedRunCredentialMaterial | null,
|
|
50
|
+
"RunCredentialRenewalOperationError"
|
|
51
|
+
>({
|
|
46
52
|
initialExpiresAt: options.initialExpiresAt,
|
|
47
53
|
resolve: options.resolve,
|
|
48
54
|
write: options.write,
|
|
49
55
|
expiresAt: (material) => material?.expiresAt ?? null,
|
|
56
|
+
publicErrorClass: "RunCredentialRenewalOperationError",
|
|
50
57
|
policy: {
|
|
51
58
|
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
52
59
|
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
@@ -80,6 +80,8 @@ export type TurnInputOptions = {
|
|
|
80
80
|
unavailableSandboxFilesNote?: string;
|
|
81
81
|
runCredentialsNote?: string;
|
|
82
82
|
providerApi: HistoryProviderApi;
|
|
83
|
+
materializeModelHistory?: ModelHistoryAttachmentProjector;
|
|
84
|
+
materializeSerializedRunState?: (serialized: string) => Promise<string>;
|
|
83
85
|
projectModelHistory?: ModelHistoryAttachmentProjector;
|
|
84
86
|
};
|
|
85
87
|
|
|
@@ -419,6 +421,7 @@ export async function turnInput(
|
|
|
419
421
|
joinInternalContext(internalContext, attachmentContext),
|
|
420
422
|
fileAttachments.map((attachment) => attachment.resource),
|
|
421
423
|
options.providerApi,
|
|
424
|
+
options.materializeModelHistory,
|
|
422
425
|
options.projectModelHistory,
|
|
423
426
|
);
|
|
424
427
|
}
|
|
@@ -435,6 +438,7 @@ export async function turnInput(
|
|
|
435
438
|
internalContext,
|
|
436
439
|
[],
|
|
437
440
|
options.providerApi,
|
|
441
|
+
options.materializeModelHistory,
|
|
438
442
|
options.projectModelHistory,
|
|
439
443
|
);
|
|
440
444
|
}
|
|
@@ -450,9 +454,12 @@ export async function turnInput(
|
|
|
450
454
|
if (!state) {
|
|
451
455
|
throw new Error("No saved run state is available for approval decision");
|
|
452
456
|
}
|
|
457
|
+
const serializedRunState = resumeRunState(state);
|
|
453
458
|
const prepared = await runtime.prepareInput(agent, {
|
|
454
459
|
kind: "approval",
|
|
455
|
-
serializedRunState:
|
|
460
|
+
serializedRunState: options.materializeSerializedRunState
|
|
461
|
+
? await options.materializeSerializedRunState(serializedRunState)
|
|
462
|
+
: serializedRunState,
|
|
456
463
|
approvalId: String(payload.approvalId ?? ""),
|
|
457
464
|
decision: payload.decision === "approve" ? "approve" : "reject",
|
|
458
465
|
...(typeof payload.message === "string" ? { message: payload.message } : {}),
|
|
@@ -478,9 +485,12 @@ export async function turnInput(
|
|
|
478
485
|
if (!resume) {
|
|
479
486
|
throw new Error("Human-input response does not resolve to a durable request");
|
|
480
487
|
}
|
|
488
|
+
const serializedRunState = resumeRunState(state);
|
|
481
489
|
const prepared = await runtime.prepareInput(agent, {
|
|
482
490
|
kind: "human_input",
|
|
483
|
-
serializedRunState:
|
|
491
|
+
serializedRunState: options.materializeSerializedRunState
|
|
492
|
+
? await options.materializeSerializedRunState(serializedRunState)
|
|
493
|
+
: serializedRunState,
|
|
484
494
|
toolCallId: resume.toolCallId,
|
|
485
495
|
});
|
|
486
496
|
return {
|
|
@@ -511,6 +521,7 @@ async function messageInput(
|
|
|
511
521
|
internalContext: string | undefined,
|
|
512
522
|
currentAttachmentRefs: FileResourceRef[] = [],
|
|
513
523
|
providerApi: HistoryProviderApi = "responses",
|
|
524
|
+
materializeModelHistory?: ModelHistoryAttachmentProjector,
|
|
514
525
|
projectModelHistory?: ModelHistoryAttachmentProjector,
|
|
515
526
|
): Promise<PreparedTurnInput> {
|
|
516
527
|
const stored = await getActiveSessionHistoryItems(db, trigger.workspaceId, trigger.sessionId);
|
|
@@ -518,9 +529,12 @@ async function messageInput(
|
|
|
518
529
|
const canonicalView = projectRejectedProviderArtifacts(stored);
|
|
519
530
|
const providerView = projectHistoryForProvider(canonicalView, providerApi);
|
|
520
531
|
const referencedHistory = withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs);
|
|
521
|
-
const
|
|
522
|
-
? await
|
|
532
|
+
const materializedHistory = materializeModelHistory
|
|
533
|
+
? await materializeModelHistory(referencedHistory)
|
|
523
534
|
: referencedHistory;
|
|
535
|
+
const historyItems = projectModelHistory
|
|
536
|
+
? await projectModelHistory(materializedHistory)
|
|
537
|
+
: materializedHistory;
|
|
524
538
|
const prepared = await runtime.prepareInput(agent, {
|
|
525
539
|
kind: "message",
|
|
526
540
|
...(text ? { text } : {}),
|
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
isCodexBilledTurn,
|
|
24
24
|
markScheduledTaskRunFailedIfQueued,
|
|
25
25
|
recordUsageEvent,
|
|
26
|
+
requireScheduledTaskTargetInTransaction,
|
|
26
27
|
requireSession,
|
|
27
28
|
setTemporalWorkflowId,
|
|
28
29
|
settleScheduledTaskRunInTransaction,
|
|
@@ -183,7 +184,15 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
183
184
|
// Every dispatch carries the first-party MCP server; runtime visibility
|
|
184
185
|
// still follows the session's exact selection and authorization.
|
|
185
186
|
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
186
|
-
|
|
187
|
+
const existingSessionId =
|
|
188
|
+
task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
189
|
+
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
190
|
+
throw new Error("scheduled task target session is unavailable");
|
|
191
|
+
}
|
|
192
|
+
if (
|
|
193
|
+
task.runMode === "new_session_per_run" ||
|
|
194
|
+
(task.runMode === "reusable_session" && !existingSessionId)
|
|
195
|
+
) {
|
|
187
196
|
// The FK on scheduled_tasks.variable_set_id is ON DELETE RESTRICT, so
|
|
188
197
|
// an attached variableSet must still exist here; fail closed if not.
|
|
189
198
|
const variableSet = task.variableSetId
|
|
@@ -369,7 +378,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
369
378
|
workflowWakeRevision: scheduledUpdate.workflowWakeRevision,
|
|
370
379
|
};
|
|
371
380
|
} else {
|
|
372
|
-
const session = await requireSession(db, task.workspaceId,
|
|
381
|
+
const session = await requireSession(db, task.workspaceId, existingSessionId!);
|
|
373
382
|
// A user-cancelled (terminal) reusable session must not be revived and
|
|
374
383
|
// re-billed on the next fire. Early check avoids the pre-lock goal
|
|
375
384
|
// upsert side-effect; the locked-callback check below is the
|
|
@@ -394,7 +403,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
394
403
|
// A recurring "maintain X" task re-establishes its objective on every
|
|
395
404
|
// fire: replace the goal text, reactivate it, and reset the counters.
|
|
396
405
|
const reusableGoal =
|
|
397
|
-
goalSpec && run.status === "queued"
|
|
406
|
+
task.runMode === "reusable_session" && goalSpec && run.status === "queued"
|
|
398
407
|
? await upsertSessionGoal(db, {
|
|
399
408
|
accountId: task.accountId,
|
|
400
409
|
workspaceId: task.workspaceId,
|
|
@@ -455,6 +464,13 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
455
464
|
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
456
465
|
},
|
|
457
466
|
async (tx, wakeEventId) => {
|
|
467
|
+
if (task.runMode === "existing_session") {
|
|
468
|
+
await requireScheduledTaskTargetInTransaction(tx, {
|
|
469
|
+
workspaceId: task.workspaceId,
|
|
470
|
+
taskId: task.id,
|
|
471
|
+
targetSessionId: session.id,
|
|
472
|
+
});
|
|
473
|
+
}
|
|
458
474
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
459
475
|
await settleScheduledTaskRunInTransaction(tx, {
|
|
460
476
|
workspaceId: task.workspaceId,
|
|
@@ -19,7 +19,10 @@ export type ToolspaceTokenRenewalOptions = {
|
|
|
19
19
|
schedule?: (callback: () => void, delayMs: number) => unknown;
|
|
20
20
|
clearSchedule?: (timer: unknown) => void;
|
|
21
21
|
onSuccess?: (result: { nextDelayMs: number }) => void;
|
|
22
|
-
onFailure?: (failure: {
|
|
22
|
+
onFailure?: (failure: {
|
|
23
|
+
retryDelayMs: number;
|
|
24
|
+
errorClass: "ToolspaceTokenRenewalOperationError";
|
|
25
|
+
}) => void;
|
|
23
26
|
};
|
|
24
27
|
|
|
25
28
|
class ToolspaceTokenRenewalError extends Error {
|
|
@@ -29,7 +32,10 @@ class ToolspaceTokenRenewalError extends Error {
|
|
|
29
32
|
export function startToolspaceTokenRenewalLoop(
|
|
30
33
|
options: ToolspaceTokenRenewalOptions,
|
|
31
34
|
): ToolspaceTokenRenewalController {
|
|
32
|
-
return startExpiringMaterialRenewalLoop<
|
|
35
|
+
return startExpiringMaterialRenewalLoop<
|
|
36
|
+
MintedSandboxToolspaceToken,
|
|
37
|
+
"ToolspaceTokenRenewalOperationError"
|
|
38
|
+
>({
|
|
33
39
|
initialExpiresAt: options.initialExpiresAt,
|
|
34
40
|
resolve: async () => {
|
|
35
41
|
const minted = await options.mint();
|
|
@@ -40,6 +46,7 @@ export function startToolspaceTokenRenewalLoop(
|
|
|
40
46
|
},
|
|
41
47
|
write: options.write,
|
|
42
48
|
expiresAt: (material) => material.expiresAt,
|
|
49
|
+
publicErrorClass: "ToolspaceTokenRenewalOperationError",
|
|
43
50
|
policy: {
|
|
44
51
|
defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
|
|
45
52
|
expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
@@ -221,6 +221,7 @@ type CaptureRepositoryReadResult =
|
|
|
221
221
|
complete: true;
|
|
222
222
|
status: Awaited<ReturnType<SandboxChannelAService["gitStatus"]>>;
|
|
223
223
|
diff: Awaited<ReturnType<SandboxChannelAService["gitDiff"]>>;
|
|
224
|
+
branchDiff?: Awaited<ReturnType<SandboxChannelAService["gitDiff"]>> | undefined;
|
|
224
225
|
}
|
|
225
226
|
| { complete: false; degradedReason: "repository_read_unavailable" };
|
|
226
227
|
|
|
@@ -304,7 +305,7 @@ export async function readCaptureRepository(
|
|
|
304
305
|
return { complete: false, degradedReason: "repository_read_unavailable" };
|
|
305
306
|
}
|
|
306
307
|
try {
|
|
307
|
-
const
|
|
308
|
+
const workingRequest = svc.gitDiff({
|
|
308
309
|
path: root,
|
|
309
310
|
staged: false,
|
|
310
311
|
includeUntracked: true,
|
|
@@ -313,7 +314,29 @@ export async function readCaptureRepository(
|
|
|
313
314
|
contextLines: 3,
|
|
314
315
|
maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES,
|
|
315
316
|
});
|
|
316
|
-
|
|
317
|
+
// Branch capture is additive. A repository without origin/HEAD must not
|
|
318
|
+
// degrade the authoritative working-tree capture.
|
|
319
|
+
const branchRequest = Promise.resolve()
|
|
320
|
+
.then(() =>
|
|
321
|
+
svc.gitDiff({
|
|
322
|
+
path: root,
|
|
323
|
+
staged: false,
|
|
324
|
+
includeUntracked: true,
|
|
325
|
+
fromRef: "origin/HEAD",
|
|
326
|
+
pathspec: [],
|
|
327
|
+
contextLines: 3,
|
|
328
|
+
maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES,
|
|
329
|
+
}),
|
|
330
|
+
)
|
|
331
|
+
.then((value) => value)
|
|
332
|
+
.catch(() => null);
|
|
333
|
+
const [diff, branchDiff] = await Promise.all([workingRequest, branchRequest]);
|
|
334
|
+
return {
|
|
335
|
+
complete: true,
|
|
336
|
+
status,
|
|
337
|
+
diff,
|
|
338
|
+
...(branchDiff ? { branchDiff } : {}),
|
|
339
|
+
};
|
|
317
340
|
} catch (error) {
|
|
318
341
|
const boxExiting = classifyCaptureEntryError(error);
|
|
319
342
|
if (boxExiting) throw boxExiting;
|
|
@@ -503,7 +526,7 @@ async function runCapture(
|
|
|
503
526
|
);
|
|
504
527
|
return;
|
|
505
528
|
}
|
|
506
|
-
const { status, diff } = repository;
|
|
529
|
+
const { status, diff, branchDiff } = repository;
|
|
507
530
|
// Drop residue churn from the review diff too. On a desktop box the seed's
|
|
508
531
|
// `git add -A` in $HOME commits ~/.config/xfce4/* into HEAD, so `git diff HEAD`
|
|
509
532
|
// lists the continuously-rewritten desktop dotfiles as changed — noise the
|
|
@@ -523,6 +546,13 @@ async function runCapture(
|
|
|
523
546
|
behind: status.behind,
|
|
524
547
|
status: statusFiles,
|
|
525
548
|
diff: diffFiles,
|
|
549
|
+
...(branchDiff
|
|
550
|
+
? {
|
|
551
|
+
branchDiff: branchDiff.files.filter(
|
|
552
|
+
(f) => !isUnderResidueDir(joinRepoPath(root, f.path)),
|
|
553
|
+
),
|
|
554
|
+
}
|
|
555
|
+
: {}),
|
|
526
556
|
});
|
|
527
557
|
for (const f of diffFiles) {
|
|
528
558
|
additions += f.additions;
|
package/src/activities.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
|
|
|
15
15
|
import { createSessionStateActivities } from "./activities/session-state";
|
|
16
16
|
import { createWorkflowWakeActivities } from "./activities/workflow-wake";
|
|
17
17
|
import { createRigVerificationActivities } from "./activities/rig-verification";
|
|
18
|
+
import { createRetainedScreenshotMaintenanceActivities } from "./activities/retained-screenshot-reaper";
|
|
18
19
|
import type { ActivityDependencies, ActivityServices } from "./activities/types";
|
|
19
20
|
import {
|
|
20
21
|
observabilityEventLogger,
|
|
@@ -136,6 +137,7 @@ function controlActivities(services: () => Promise<ActivityServices>) {
|
|
|
136
137
|
...createCodexCapacityActivities(services),
|
|
137
138
|
...createRigVerificationActivities(services),
|
|
138
139
|
...createFileUploadReaperActivities(services),
|
|
140
|
+
...createRetainedScreenshotMaintenanceActivities(services),
|
|
139
141
|
...createWorkflowWakeActivities(services),
|
|
140
142
|
// P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
|
|
141
143
|
// *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
|
|
@@ -148,13 +150,18 @@ export function createControlActivities(dependencies: ActivityDependencies = {})
|
|
|
148
150
|
}
|
|
149
151
|
|
|
150
152
|
export function createTurnActivities(dependencies: ActivityDependencies = {}) {
|
|
151
|
-
return {
|
|
153
|
+
return {
|
|
154
|
+
runAgentTurn: createRunAgentTurnActivity(createActivityServices(dependencies)),
|
|
155
|
+
};
|
|
152
156
|
}
|
|
153
157
|
|
|
154
158
|
/** Direct activity harness for tests; production workers always choose one role. */
|
|
155
159
|
export function createActivityTestHarness(dependencies: ActivityDependencies = {}) {
|
|
156
160
|
const services = createActivityServices(dependencies);
|
|
157
|
-
return {
|
|
161
|
+
return {
|
|
162
|
+
runAgentTurn: createRunAgentTurnActivity(services),
|
|
163
|
+
...controlActivities(services),
|
|
164
|
+
};
|
|
158
165
|
}
|
|
159
166
|
|
|
160
167
|
const defaultControlActivities = createControlActivities();
|
|
@@ -180,6 +187,7 @@ export const getCodexCapacityWait = defaultControlActivities.getCodexCapacityWai
|
|
|
180
187
|
export const reconcileCodexCapacityWait = defaultControlActivities.reconcileCodexCapacityWait;
|
|
181
188
|
export const reapSandboxLeases = defaultControlActivities.reapSandboxLeases;
|
|
182
189
|
export const reapExpiredFileUploads = defaultControlActivities.reapExpiredFileUploads;
|
|
190
|
+
export const maintainRetainedScreenshots = defaultControlActivities.maintainRetainedScreenshots;
|
|
183
191
|
export const dispatchSessionWorkflowWakes = defaultControlActivities.dispatchSessionWorkflowWakes;
|
|
184
192
|
export const verifyRigChange = defaultControlActivities.verifyRigChange;
|
|
185
193
|
export const verifyRigVersion = defaultControlActivities.verifyRigVersion;
|
package/src/host-export-pump.ts
CHANGED
|
@@ -166,7 +166,7 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
|
|
|
166
166
|
maxBytes: batchMaxBytes,
|
|
167
167
|
};
|
|
168
168
|
const settleFailure = async (error: unknown): Promise<HostExportDrainResult> => {
|
|
169
|
-
const message =
|
|
169
|
+
const message = exactHostExportErrorMessage(error);
|
|
170
170
|
let failures = 0;
|
|
171
171
|
try {
|
|
172
172
|
failures = await failHostExportBatch(options.db, {
|
|
@@ -179,17 +179,15 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
|
|
|
179
179
|
} catch (settlementError) {
|
|
180
180
|
options.observability?.warn("host export failure settlement was stale", {
|
|
181
181
|
kind,
|
|
182
|
-
|
|
183
|
-
error: safeErrorMessage(settlementError),
|
|
182
|
+
...hostExportPublicErrorFields(settlementError, "host_export_failure_settlement_stale"),
|
|
184
183
|
});
|
|
185
184
|
}
|
|
186
185
|
const blocked = failures >= maxFailures;
|
|
187
186
|
options.observability?.warn("host export batch failed", {
|
|
188
187
|
kind,
|
|
189
|
-
consumerId,
|
|
190
188
|
failures,
|
|
191
189
|
blocked,
|
|
192
|
-
error
|
|
190
|
+
...hostExportPublicErrorFields(error, "host_export_batch_delivery_failed"),
|
|
193
191
|
});
|
|
194
192
|
const result: HostExportDrainResult = {
|
|
195
193
|
kind,
|
|
@@ -222,8 +220,7 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
|
|
|
222
220
|
// a duplicate settlement attempt for the now-stale lease.
|
|
223
221
|
options.observability?.error("host export retention prune failed", {
|
|
224
222
|
kind,
|
|
225
|
-
|
|
226
|
-
error: safeErrorMessage(error),
|
|
223
|
+
...hostExportPublicErrorFields(error, "host_export_retention_prune_failed"),
|
|
227
224
|
});
|
|
228
225
|
}
|
|
229
226
|
const result: HostExportDrainResult = {
|
|
@@ -344,7 +341,7 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
|
|
|
344
341
|
}
|
|
345
342
|
} catch (error) {
|
|
346
343
|
options.observability?.error("host export pump iteration failed", {
|
|
347
|
-
error
|
|
344
|
+
...hostExportPublicErrorFields(error, "host_export_pump_iteration_failed"),
|
|
348
345
|
});
|
|
349
346
|
}
|
|
350
347
|
if (running) await waitForPoll();
|
|
@@ -410,7 +407,27 @@ function validateInstanceId(value: string): string {
|
|
|
410
407
|
return value;
|
|
411
408
|
}
|
|
412
409
|
|
|
413
|
-
function
|
|
414
|
-
|
|
415
|
-
|
|
410
|
+
function exactHostExportErrorMessage(error: unknown): string {
|
|
411
|
+
return error instanceof Error ? error.message : String(error);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
type HostExportPublicFailureCode =
|
|
415
|
+
| "host_export_failure_settlement_stale"
|
|
416
|
+
| "host_export_batch_delivery_failed"
|
|
417
|
+
| "host_export_retention_prune_failed"
|
|
418
|
+
| "host_export_pump_iteration_failed";
|
|
419
|
+
|
|
420
|
+
function hostExportPublicErrorFields(
|
|
421
|
+
_error: unknown,
|
|
422
|
+
errorCode: HostExportPublicFailureCode,
|
|
423
|
+
): {
|
|
424
|
+
errorClass: "HostExportOperationError";
|
|
425
|
+
errorCode: HostExportPublicFailureCode;
|
|
426
|
+
origin: "host-export";
|
|
427
|
+
} {
|
|
428
|
+
return {
|
|
429
|
+
errorClass: "HostExportOperationError",
|
|
430
|
+
errorCode,
|
|
431
|
+
origin: "host-export",
|
|
432
|
+
};
|
|
416
433
|
}
|
package/src/http.ts
CHANGED
|
@@ -72,7 +72,7 @@ export function createWorkerHttpHandler(
|
|
|
72
72
|
checks: Object.fromEntries(
|
|
73
73
|
(Object.keys(checks) as ReadinessCheckName[]).map((name) => [
|
|
74
74
|
name,
|
|
75
|
-
{ ok: false, error:
|
|
75
|
+
{ ok: false, error: "worker_not_ready" },
|
|
76
76
|
]),
|
|
77
77
|
) as ReadinessResult["checks"],
|
|
78
78
|
}
|
|
@@ -101,11 +101,8 @@ export async function runReadinessChecks(
|
|
|
101
101
|
try {
|
|
102
102
|
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
103
103
|
return [name, { ok: true }] as const;
|
|
104
|
-
} catch
|
|
105
|
-
return [
|
|
106
|
-
name,
|
|
107
|
-
{ ok: false, error: error instanceof Error ? error.message : String(error) },
|
|
108
|
-
] as const;
|
|
104
|
+
} catch {
|
|
105
|
+
return [name, { ok: false, error: "dependency_unavailable" }] as const;
|
|
109
106
|
}
|
|
110
107
|
},
|
|
111
108
|
),
|
|
@@ -37,11 +37,11 @@ function eventAttributes(attributes: Record<string, unknown> | undefined): Attri
|
|
|
37
37
|
if (!attributes) {
|
|
38
38
|
return undefined;
|
|
39
39
|
}
|
|
40
|
-
const
|
|
40
|
+
const projected: Attributes = {};
|
|
41
41
|
for (const [key, value] of Object.entries(attributes)) {
|
|
42
|
-
|
|
42
|
+
projected[key] = eventAttributeValue(value);
|
|
43
43
|
}
|
|
44
|
-
return
|
|
44
|
+
return projected;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
function eventAttributeValue(value: unknown): AttributeValue {
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -21,7 +21,6 @@
|
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
23
|
import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
|
|
24
|
-
import { redactSensitiveText } from "@opengeni/contracts";
|
|
25
24
|
import { randomUUID } from "node:crypto";
|
|
26
25
|
import {
|
|
27
26
|
acquireLease,
|
|
@@ -236,15 +235,35 @@ class SnapshotTimeoutError extends Error {
|
|
|
236
235
|
}
|
|
237
236
|
}
|
|
238
237
|
|
|
239
|
-
function safeSnapshotError(error: unknown): {
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
238
|
+
export function safeSnapshotError(error: unknown): {
|
|
239
|
+
errorClass: "SnapshotOperationError";
|
|
240
|
+
errorCode: "snapshot_operation_failed";
|
|
241
|
+
status?: number;
|
|
242
|
+
origin: "sandbox-resume";
|
|
243
|
+
} {
|
|
244
|
+
const fields: {
|
|
245
|
+
errorClass: "SnapshotOperationError";
|
|
246
|
+
errorCode: "snapshot_operation_failed";
|
|
247
|
+
status?: number;
|
|
248
|
+
origin: "sandbox-resume";
|
|
249
|
+
} = {
|
|
250
|
+
errorClass: "SnapshotOperationError",
|
|
251
|
+
errorCode: "snapshot_operation_failed",
|
|
252
|
+
origin: "sandbox-resume",
|
|
247
253
|
};
|
|
254
|
+
try {
|
|
255
|
+
const rawStatus =
|
|
256
|
+
error && typeof error === "object"
|
|
257
|
+
? ((error as { status?: unknown; statusCode?: unknown }).status ??
|
|
258
|
+
(error as { statusCode?: unknown }).statusCode)
|
|
259
|
+
: undefined;
|
|
260
|
+
const status = Number(rawStatus);
|
|
261
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
262
|
+
} catch {
|
|
263
|
+
// Public diagnostics are best-effort and must never replace the exact
|
|
264
|
+
// internal snapshot failure.
|
|
265
|
+
}
|
|
266
|
+
return fields;
|
|
248
267
|
}
|
|
249
268
|
|
|
250
269
|
export async function waitForWarmSnapshot(
|
|
@@ -501,11 +520,25 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
501
520
|
}
|
|
502
521
|
const persistable = session as {
|
|
503
522
|
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
504
|
-
state?: {
|
|
523
|
+
state?: {
|
|
524
|
+
workspacePersistence?: unknown;
|
|
525
|
+
providerState?: { workspacePersistence?: unknown };
|
|
526
|
+
};
|
|
505
527
|
};
|
|
506
528
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
507
529
|
return false;
|
|
508
530
|
}
|
|
531
|
+
// Modal filesystem snapshots terminate the source sandbox. They are safe at
|
|
532
|
+
// controlled drain/rotation (force=true), but a periodic or turn-end warm
|
|
533
|
+
// checkpoint would kill the box while its lease still points at it. The next
|
|
534
|
+
// dock read or turn would then route to a dead provider instance. Let the
|
|
535
|
+
// drain reaper capture this backend instead.
|
|
536
|
+
const workspacePersistence =
|
|
537
|
+
persistable.state?.workspacePersistence ??
|
|
538
|
+
persistable.state?.providerState?.workspacePersistence;
|
|
539
|
+
if (!force && workspacePersistence === "snapshot_filesystem") {
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
509
542
|
if (signal?.aborted) {
|
|
510
543
|
return false;
|
|
511
544
|
}
|
|
@@ -524,8 +557,8 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
524
557
|
}
|
|
525
558
|
const instanceId = lease.instanceId;
|
|
526
559
|
const nativeModalPersistence =
|
|
527
|
-
|
|
528
|
-
|
|
560
|
+
workspacePersistence === "snapshot_filesystem" ||
|
|
561
|
+
workspacePersistence === "snapshot_directory";
|
|
529
562
|
const checkpointBinding = nativeModalPersistence
|
|
530
563
|
? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
|
|
531
564
|
: null;
|