@opengeni/worker-bundle 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +970 -325
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/scheduled-tasks.ts +53 -1
- package/src/activities/types.ts +6 -6
- package/src/concurrency.ts +58 -6
- package/src/http.ts +14 -2
- package/src/index.ts +36 -12
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
environmentsEncryptionKeyBytes,
|
|
3
|
-
parseModelProvidersJson,
|
|
4
|
-
type RegistryProvider,
|
|
5
3
|
type Settings,
|
|
4
|
+
withCodexCatalogProvider,
|
|
6
5
|
} from "@opengeni/config";
|
|
7
6
|
import { settingsWithMcpCapabilityServers } from "@opengeni/core";
|
|
8
7
|
import {
|
|
@@ -10,14 +9,6 @@ import {
|
|
|
10
9
|
CODEX_APPS_MCP_SERVER_NAME,
|
|
11
10
|
CODEX_APPS_MCP_URL,
|
|
12
11
|
CODEX_APPS_STARTUP_TIMEOUT_MS,
|
|
13
|
-
CODEX_FALLBACK_MODEL_SLUGS,
|
|
14
|
-
CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
|
|
15
|
-
CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
|
|
16
|
-
CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
|
|
17
|
-
CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
|
|
18
|
-
CODEX_MODEL_ID_PREFIX,
|
|
19
|
-
CODEX_PROVIDER_BASE_URL,
|
|
20
|
-
CODEX_PROVIDER_ID,
|
|
21
12
|
} from "@opengeni/codex";
|
|
22
13
|
import {
|
|
23
14
|
listEnabledMcpCapabilityServers,
|
|
@@ -43,6 +34,9 @@ export async function settingsWithSessionMcpServersForRun(
|
|
|
43
34
|
sessionId: string,
|
|
44
35
|
attemptId: string,
|
|
45
36
|
settings: Settings,
|
|
37
|
+
options?: {
|
|
38
|
+
onResolvedServers?: (servers: readonly SessionMcpServerForRun[]) => void;
|
|
39
|
+
},
|
|
46
40
|
): Promise<Settings> {
|
|
47
41
|
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
48
42
|
if (!encryptionKey) {
|
|
@@ -56,10 +50,18 @@ export async function settingsWithSessionMcpServersForRun(
|
|
|
56
50
|
);
|
|
57
51
|
}
|
|
58
52
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
53
|
+
const servers = await listSessionMcpServersForRun(
|
|
54
|
+
db,
|
|
55
|
+
workspaceId,
|
|
56
|
+
sessionId,
|
|
57
|
+
attemptId,
|
|
58
|
+
encryptionKey ?? null,
|
|
62
59
|
);
|
|
60
|
+
// Keep credential provenance coupled to the exact decrypted rows that are
|
|
61
|
+
// overlaid into settings. A session projection read earlier in the turn can
|
|
62
|
+
// be stale after a concurrent mcpCredentialUpdates renewal.
|
|
63
|
+
options?.onResolvedServers?.(servers);
|
|
64
|
+
return settingsWithSessionMcpServers(settings, servers);
|
|
63
65
|
}
|
|
64
66
|
|
|
65
67
|
export function settingsWithSessionMcpServers(
|
|
@@ -115,6 +117,9 @@ export async function settingsWithCodexCredential(
|
|
|
115
117
|
return settings; // disabled / not connected / needs_relogin / error -> leave settings unchanged
|
|
116
118
|
}
|
|
117
119
|
const withProvider = withCodexProvider(settings);
|
|
120
|
+
if (!settings.codexConnectedAppsEnabled) {
|
|
121
|
+
return withProvider;
|
|
122
|
+
}
|
|
118
123
|
// Additive: append the synthetic codex_apps connectors MCP server for ANY
|
|
119
124
|
// active credential. Connector access is gated SERVER-SIDE per ChatGPT account
|
|
120
125
|
// (via chatgpt-account-id), NOT by token scopes — confirmed live: a `pro` token
|
|
@@ -162,28 +167,5 @@ export function withCodexAppsMcpServer(settings: Settings): Settings {
|
|
|
162
167
|
|
|
163
168
|
/** Pure: append the synthetic codex-subscription provider, idempotently. */
|
|
164
169
|
export function withCodexProvider(settings: Settings): Settings {
|
|
165
|
-
|
|
166
|
-
if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
|
|
167
|
-
return settings; // already injected
|
|
168
|
-
}
|
|
169
|
-
const codexProvider: RegistryProvider = {
|
|
170
|
-
kind: "codex-subscription",
|
|
171
|
-
id: CODEX_PROVIDER_ID,
|
|
172
|
-
label: "Codex (ChatGPT subscription)",
|
|
173
|
-
api: "responses",
|
|
174
|
-
baseUrl: CODEX_PROVIDER_BASE_URL,
|
|
175
|
-
models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
|
|
176
|
-
id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
|
|
177
|
-
upstreamModelId: slug,
|
|
178
|
-
label: slug,
|
|
179
|
-
reasoningEffort: true,
|
|
180
|
-
// These three values are the live Codex CLI catalog policy, kept distinct
|
|
181
|
-
// so proactive compaction and the effective input guard match Codex core.
|
|
182
|
-
contextWindowTokens: CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
|
|
183
|
-
effectiveContextWindowTokens: CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
|
|
184
|
-
autoCompactTokenLimit: CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
|
|
185
|
-
toolOutputTruncationTokens: CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
|
|
186
|
-
})),
|
|
187
|
-
};
|
|
188
|
-
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
170
|
+
return withCodexCatalogProvider(settings);
|
|
189
171
|
}
|
|
@@ -200,7 +200,7 @@ export function createCodexCapacityActivities(services: () => Promise<ActivitySe
|
|
|
200
200
|
}
|
|
201
201
|
}
|
|
202
202
|
if (result.action === "resumed") {
|
|
203
|
-
return { action: "resumed"
|
|
203
|
+
return { action: "resumed" };
|
|
204
204
|
}
|
|
205
205
|
if (result.action === "waiting") {
|
|
206
206
|
return {
|
|
@@ -1,56 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
* values (for example via `env`) into session events. Deep-walks payloads and
|
|
11
|
-
* replaces every exact occurrence of each secret value with
|
|
12
|
-
* `[redacted:<NAME>]`. Values shorter than six characters are skipped to avoid
|
|
13
|
-
* a false-positive flood; secrets are applied longest-value-first so
|
|
14
|
-
* overlapping values redact deterministically.
|
|
15
|
-
*/
|
|
16
|
-
export function createSecretRedactor(secrets: SecretForRedaction[]): (payload: unknown) => unknown {
|
|
17
|
-
const usable = secrets
|
|
18
|
-
.filter((secret) => secret.value.length >= MIN_REDACTABLE_VALUE_LENGTH)
|
|
19
|
-
.sort((a, b) => b.value.length - a.value.length);
|
|
20
|
-
if (usable.length === 0) {
|
|
21
|
-
return identityRedactor;
|
|
22
|
-
}
|
|
23
|
-
const redact = (payload: unknown): unknown => {
|
|
24
|
-
if (typeof payload === "string") {
|
|
25
|
-
let out = payload;
|
|
26
|
-
for (const secret of usable) {
|
|
27
|
-
if (out.includes(secret.value)) {
|
|
28
|
-
out = out.split(secret.value).join(`[redacted:${secret.name}]`);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return out;
|
|
32
|
-
}
|
|
33
|
-
if (Array.isArray(payload)) {
|
|
34
|
-
return payload.map(redact);
|
|
35
|
-
}
|
|
36
|
-
if (isPlainObject(payload)) {
|
|
37
|
-
return Object.fromEntries(
|
|
38
|
-
Object.entries(payload).map(([key, value]) => [key, redact(value)]),
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
return payload;
|
|
42
|
-
};
|
|
43
|
-
return redact;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export function identityRedactor(payload: unknown): unknown {
|
|
47
|
-
return payload;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
51
|
-
if (!value || typeof value !== "object") {
|
|
52
|
-
return false;
|
|
53
|
-
}
|
|
54
|
-
const proto = Object.getPrototypeOf(value);
|
|
55
|
-
return proto === Object.prototype || proto === null;
|
|
56
|
-
}
|
|
1
|
+
export {
|
|
2
|
+
createSecretRedactor,
|
|
3
|
+
identityRedactor,
|
|
4
|
+
isCredentialHeaderName,
|
|
5
|
+
redactSensitiveData,
|
|
6
|
+
redactSensitiveText,
|
|
7
|
+
redactSerializedJson,
|
|
8
|
+
type SecretForRedaction,
|
|
9
|
+
} from "@opengeni/contracts";
|
|
@@ -34,21 +34,32 @@ import {
|
|
|
34
34
|
accrueWarmSeconds,
|
|
35
35
|
confirmDrainCold,
|
|
36
36
|
appendSessionEventToSandboxGroup,
|
|
37
|
+
claimTerminalRetainedProcesses,
|
|
38
|
+
countActiveRetainedProcessesByOwnerState,
|
|
39
|
+
countExpiredDrainingSandboxLeases,
|
|
37
40
|
countQueuedTurns,
|
|
38
41
|
countSandboxLeasesByLiveness,
|
|
42
|
+
deferRetainedProcessReconciliation,
|
|
39
43
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
40
44
|
getBillingBalance,
|
|
41
45
|
listCreditBalancesByAccount,
|
|
42
46
|
listLiveModalSandboxLeaseAttributions,
|
|
43
47
|
listMeterableWarmLeases,
|
|
44
48
|
persistDrainSnapshot,
|
|
49
|
+
recordRetainedProcessReconciliationProof,
|
|
45
50
|
readWorkspaceArchiveCapturePreflight,
|
|
46
51
|
readLease,
|
|
47
52
|
reapExpiredSessionListSnapshots,
|
|
48
53
|
reapStaleLeaseHoldersGlobal,
|
|
54
|
+
retainedProcessReconciliationProof,
|
|
55
|
+
retainedProcessSettlementIdentity,
|
|
49
56
|
rlsContextForWorkspace,
|
|
57
|
+
settleRetainedProcess,
|
|
50
58
|
type MeterableWarmLease,
|
|
51
59
|
type ReapDrainable,
|
|
60
|
+
type RetainedProcessProviderProof,
|
|
61
|
+
type SandboxRetainedProcess,
|
|
62
|
+
type LeaseSnapshot,
|
|
52
63
|
} from "@opengeni/db";
|
|
53
64
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
54
65
|
import {
|
|
@@ -62,7 +73,9 @@ import {
|
|
|
62
73
|
createSandboxClientForBackend,
|
|
63
74
|
deletePriorPersistedSnapshot,
|
|
64
75
|
deserializeSandboxSessionStateEnvelope,
|
|
76
|
+
isExecSessionLostBanner,
|
|
65
77
|
isProviderSandboxNotFoundError,
|
|
78
|
+
parseExecBannerExitCode,
|
|
66
79
|
sweepModalOrphanSandboxes,
|
|
67
80
|
terminateModalSandboxById,
|
|
68
81
|
type ModalOrphanSweepTermination,
|
|
@@ -73,10 +86,15 @@ import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
|
73
86
|
import {
|
|
74
87
|
recordCreditBalanceGauges,
|
|
75
88
|
recordCreditMicros,
|
|
89
|
+
recordExpiredDrainingSandboxLeaseGauges,
|
|
90
|
+
recordRetainedProcessInventoryGauges,
|
|
91
|
+
recordRetainedProcessReconciliation,
|
|
92
|
+
type RetainedProcessReconciliationOutcome,
|
|
76
93
|
recordSandboxLeaseGauges,
|
|
77
94
|
recordSandboxOrphansTerminated,
|
|
78
95
|
recordTurnsQueuedGauge,
|
|
79
96
|
} from "../observability-metrics";
|
|
97
|
+
import { providerIdentityFromResumeState } from "../sandbox-routing";
|
|
80
98
|
|
|
81
99
|
export type ReapSandboxLeasesResult = {
|
|
82
100
|
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
@@ -179,8 +197,35 @@ export type SandboxLeaseActivityOptions = {
|
|
|
179
197
|
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
180
198
|
* Modal list+tag comparison when Modal is configured). */
|
|
181
199
|
sweepModalOrphans?: SweepModalOrphansFn;
|
|
200
|
+
/** Override only the read-only provider process probe. The canonical DB
|
|
201
|
+
* settlement remains real in tests and production. */
|
|
202
|
+
probeRetainedProcess?: RetainedProcessProbeFn;
|
|
182
203
|
};
|
|
183
204
|
|
|
205
|
+
export type RetainedProcessProbeResult =
|
|
206
|
+
| { status: "proved"; proof: RetainedProcessProviderProof }
|
|
207
|
+
| {
|
|
208
|
+
status: "deferred";
|
|
209
|
+
reason:
|
|
210
|
+
| "identity_mismatch"
|
|
211
|
+
| "resume_state_missing"
|
|
212
|
+
| "backend_unsupported"
|
|
213
|
+
| "provider_running"
|
|
214
|
+
| "provider_unknown"
|
|
215
|
+
| "provider_timeout"
|
|
216
|
+
| "provider_error";
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
export type RetainedProcessProbeFn = (
|
|
220
|
+
settings: ActivityServices["settings"],
|
|
221
|
+
lease: LeaseSnapshot,
|
|
222
|
+
process: SandboxRetainedProcess,
|
|
223
|
+
) => Promise<RetainedProcessProbeResult>;
|
|
224
|
+
|
|
225
|
+
export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
226
|
+
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
227
|
+
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
228
|
+
|
|
184
229
|
export function createSandboxLeaseActivities(
|
|
185
230
|
services: () => Promise<ActivityServices>,
|
|
186
231
|
options: SandboxLeaseActivityOptions = {},
|
|
@@ -188,6 +233,7 @@ export function createSandboxLeaseActivities(
|
|
|
188
233
|
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
189
234
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
190
235
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
236
|
+
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
191
237
|
/**
|
|
192
238
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
193
239
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -248,6 +294,12 @@ export function createSandboxLeaseActivities(
|
|
|
248
294
|
observability,
|
|
249
295
|
);
|
|
250
296
|
|
|
297
|
+
// (0c) Terminal-owner retained-process reconciliation. Owner state selects
|
|
298
|
+
// only a bounded inspection batch; exact provider exit/loss proof is
|
|
299
|
+
// checkpointed before canonical settlement. Ambiguous, running, timed-out,
|
|
300
|
+
// unsupported, and transient provider states preserve every durable row.
|
|
301
|
+
await reconcileTerminalRetainedProcesses(db, settings, observability, probeRetainedProcess);
|
|
302
|
+
|
|
251
303
|
// (1) The DB-only cross-workspace sweep. Returns the drainable rows.
|
|
252
304
|
const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
|
|
253
305
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
@@ -336,6 +388,345 @@ export function createSandboxLeaseActivities(
|
|
|
336
388
|
return { reapSandboxLeases };
|
|
337
389
|
}
|
|
338
390
|
|
|
391
|
+
async function reconcileTerminalRetainedProcesses(
|
|
392
|
+
db: ActivityServices["db"],
|
|
393
|
+
settings: ActivityServices["settings"],
|
|
394
|
+
observability: ActivityServices["observability"],
|
|
395
|
+
probe: RetainedProcessProbeFn,
|
|
396
|
+
): Promise<void> {
|
|
397
|
+
const claimId = crypto.randomUUID();
|
|
398
|
+
let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
|
|
399
|
+
try {
|
|
400
|
+
claims = await claimTerminalRetainedProcesses(db, {
|
|
401
|
+
claimId,
|
|
402
|
+
limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
|
|
403
|
+
claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,
|
|
404
|
+
});
|
|
405
|
+
} catch (error) {
|
|
406
|
+
recordRetainedProcessReconciliation(observability, "claim_failed");
|
|
407
|
+
observability.warn("sandbox reaper: retained-process claim failed", {
|
|
408
|
+
error: error instanceof Error ? error.message : String(error),
|
|
409
|
+
});
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
for (const claim of claims) {
|
|
414
|
+
let process = claim.process;
|
|
415
|
+
const expected = retainedProcessSettlementIdentity(process);
|
|
416
|
+
let proof = retainedProcessReconciliationProof(process);
|
|
417
|
+
if (!proof) {
|
|
418
|
+
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
419
|
+
() => null,
|
|
420
|
+
);
|
|
421
|
+
if (
|
|
422
|
+
!lease ||
|
|
423
|
+
lease.id !== process.leaseId ||
|
|
424
|
+
lease.leaseEpoch !== process.leaseEpoch ||
|
|
425
|
+
lease.backend !== process.providerBackend ||
|
|
426
|
+
lease.instanceId !== process.providerInstanceId ||
|
|
427
|
+
process.routeTargetId !== null
|
|
428
|
+
) {
|
|
429
|
+
await deferRetainedProcessClaim(
|
|
430
|
+
db,
|
|
431
|
+
settings,
|
|
432
|
+
observability,
|
|
433
|
+
process,
|
|
434
|
+
expected,
|
|
435
|
+
claim.claimId,
|
|
436
|
+
"identity_mismatch",
|
|
437
|
+
);
|
|
438
|
+
continue;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
let observation: RetainedProcessProbeResult;
|
|
442
|
+
try {
|
|
443
|
+
observation = await probe(settings, lease, process);
|
|
444
|
+
} catch (error) {
|
|
445
|
+
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
446
|
+
processId: process.id,
|
|
447
|
+
providerBackend: process.providerBackend,
|
|
448
|
+
error: error instanceof Error ? error.message : String(error),
|
|
449
|
+
});
|
|
450
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
451
|
+
}
|
|
452
|
+
if (observation.status === "deferred") {
|
|
453
|
+
await deferRetainedProcessClaim(
|
|
454
|
+
db,
|
|
455
|
+
settings,
|
|
456
|
+
observability,
|
|
457
|
+
process,
|
|
458
|
+
expected,
|
|
459
|
+
claim.claimId,
|
|
460
|
+
observation.reason,
|
|
461
|
+
);
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
try {
|
|
465
|
+
process = await recordRetainedProcessReconciliationProof(db, {
|
|
466
|
+
accountId: process.accountId,
|
|
467
|
+
workspaceId: process.workspaceId,
|
|
468
|
+
sessionId: process.sessionId,
|
|
469
|
+
processId: process.id,
|
|
470
|
+
expected,
|
|
471
|
+
claimId: claim.claimId,
|
|
472
|
+
proof: observation.proof,
|
|
473
|
+
});
|
|
474
|
+
proof = observation.proof;
|
|
475
|
+
recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
|
|
476
|
+
} catch (error) {
|
|
477
|
+
recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
|
|
478
|
+
observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
|
|
479
|
+
processId: process.id,
|
|
480
|
+
error: error instanceof Error ? error.message : String(error),
|
|
481
|
+
});
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
try {
|
|
487
|
+
await settleRetainedProcess(db, {
|
|
488
|
+
accountId: process.accountId,
|
|
489
|
+
workspaceId: process.workspaceId,
|
|
490
|
+
sessionId: process.sessionId,
|
|
491
|
+
processId: process.id,
|
|
492
|
+
expected,
|
|
493
|
+
reconciliationClaimId: claim.claimId,
|
|
494
|
+
outcome: proof.outcome,
|
|
495
|
+
exitCode: proof.exitCode,
|
|
496
|
+
reason: proof.reason,
|
|
497
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
498
|
+
});
|
|
499
|
+
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
500
|
+
} catch (error) {
|
|
501
|
+
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
502
|
+
observability.warn("sandbox reaper: retained-process settlement failed", {
|
|
503
|
+
processId: process.id,
|
|
504
|
+
error: error instanceof Error ? error.message : String(error),
|
|
505
|
+
});
|
|
506
|
+
await deferRetainedProcessClaim(
|
|
507
|
+
db,
|
|
508
|
+
settings,
|
|
509
|
+
observability,
|
|
510
|
+
process,
|
|
511
|
+
expected,
|
|
512
|
+
claim.claimId,
|
|
513
|
+
"settlement_failed",
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
async function deferRetainedProcessClaim(
|
|
520
|
+
db: ActivityServices["db"],
|
|
521
|
+
settings: ActivityServices["settings"],
|
|
522
|
+
observability: ActivityServices["observability"],
|
|
523
|
+
process: SandboxRetainedProcess,
|
|
524
|
+
expected: ReturnType<typeof retainedProcessSettlementIdentity>,
|
|
525
|
+
claimId: string,
|
|
526
|
+
outcome: RetainedProcessReconciliationOutcome,
|
|
527
|
+
): Promise<void> {
|
|
528
|
+
const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
529
|
+
const retryAfterMs = Math.min(
|
|
530
|
+
5 * 60_000,
|
|
531
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
|
|
532
|
+
);
|
|
533
|
+
try {
|
|
534
|
+
await deferRetainedProcessReconciliation(db, {
|
|
535
|
+
accountId: process.accountId,
|
|
536
|
+
workspaceId: process.workspaceId,
|
|
537
|
+
sessionId: process.sessionId,
|
|
538
|
+
processId: process.id,
|
|
539
|
+
expected,
|
|
540
|
+
claimId,
|
|
541
|
+
outcome,
|
|
542
|
+
retryAfterMs,
|
|
543
|
+
});
|
|
544
|
+
recordRetainedProcessReconciliation(observability, outcome);
|
|
545
|
+
} catch (error) {
|
|
546
|
+
recordRetainedProcessReconciliation(observability, "defer_failed");
|
|
547
|
+
observability.warn("sandbox reaper: retained-process defer failed", {
|
|
548
|
+
processId: process.id,
|
|
549
|
+
error: error instanceof Error ? error.message : String(error),
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
type RetainedProcessProbeClient = {
|
|
555
|
+
backendId: string;
|
|
556
|
+
resume?: (state: unknown) => Promise<unknown>;
|
|
557
|
+
deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
|
|
558
|
+
};
|
|
559
|
+
|
|
560
|
+
type RetainedProcessProbeSession = {
|
|
561
|
+
writeStdin?: (args: {
|
|
562
|
+
sessionId: number;
|
|
563
|
+
chars: string;
|
|
564
|
+
yieldTimeMs: number;
|
|
565
|
+
maxOutputTokens: number;
|
|
566
|
+
}) => Promise<unknown>;
|
|
567
|
+
};
|
|
568
|
+
|
|
569
|
+
const RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
570
|
+
|
|
571
|
+
async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {
|
|
572
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
573
|
+
try {
|
|
574
|
+
return await Promise.race([
|
|
575
|
+
promise,
|
|
576
|
+
new Promise<never>((_resolve, reject) => {
|
|
577
|
+
timer = setTimeout(
|
|
578
|
+
() => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
|
|
579
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
580
|
+
);
|
|
581
|
+
timer.unref?.();
|
|
582
|
+
}),
|
|
583
|
+
]);
|
|
584
|
+
} finally {
|
|
585
|
+
if (timer) clearTimeout(timer);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
export async function probeRetainedProcessAtProvider(
|
|
590
|
+
settings: ActivityServices["settings"],
|
|
591
|
+
lease: LeaseSnapshot,
|
|
592
|
+
process: SandboxRetainedProcess,
|
|
593
|
+
): Promise<RetainedProcessProbeResult> {
|
|
594
|
+
if (
|
|
595
|
+
lease.id !== process.leaseId ||
|
|
596
|
+
lease.sandboxGroupId !== process.sandboxGroupId ||
|
|
597
|
+
lease.leaseEpoch !== process.leaseEpoch ||
|
|
598
|
+
lease.backend !== process.providerBackend ||
|
|
599
|
+
lease.instanceId !== process.providerInstanceId ||
|
|
600
|
+
lease.resumeBackendId !== process.providerBackend ||
|
|
601
|
+
process.routeTargetId !== null
|
|
602
|
+
) {
|
|
603
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
604
|
+
}
|
|
605
|
+
if (!lease.resumeState) {
|
|
606
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
607
|
+
}
|
|
608
|
+
const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
|
|
609
|
+
if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
|
|
610
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
611
|
+
}
|
|
612
|
+
if (providerIdentityFromResumeState(lease.resumeState) !== process.providerInstanceId) {
|
|
613
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
let client: RetainedProcessProbeClient | undefined;
|
|
617
|
+
try {
|
|
618
|
+
client = createSandboxClientForBackend(process.providerBackend as never, settings) as
|
|
619
|
+
| RetainedProcessProbeClient
|
|
620
|
+
| undefined;
|
|
621
|
+
} catch {
|
|
622
|
+
return { status: "deferred", reason: "provider_error" };
|
|
623
|
+
}
|
|
624
|
+
if (!client || client.backendId !== process.providerBackend || !client.resume) {
|
|
625
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
626
|
+
}
|
|
627
|
+
const envelopeSessionState =
|
|
628
|
+
(lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
|
|
629
|
+
let resumedState: unknown;
|
|
630
|
+
try {
|
|
631
|
+
resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
632
|
+
client as never,
|
|
633
|
+
envelopeSessionState,
|
|
634
|
+
);
|
|
635
|
+
} catch {
|
|
636
|
+
return { status: "deferred", reason: "provider_error" };
|
|
637
|
+
}
|
|
638
|
+
if (resumedState === undefined) {
|
|
639
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
let session: RetainedProcessProbeSession;
|
|
643
|
+
try {
|
|
644
|
+
session = (await withRetainedProcessProbeTimeout(
|
|
645
|
+
client.resume(resumedState),
|
|
646
|
+
)) as RetainedProcessProbeSession;
|
|
647
|
+
} catch (error) {
|
|
648
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
649
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
650
|
+
}
|
|
651
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
652
|
+
return {
|
|
653
|
+
status: "proved",
|
|
654
|
+
proof: {
|
|
655
|
+
outcome: "lost",
|
|
656
|
+
exitCode: null,
|
|
657
|
+
reason: "provider_instance_not_found",
|
|
658
|
+
},
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
return { status: "deferred", reason: "provider_error" };
|
|
662
|
+
}
|
|
663
|
+
if (typeof session.writeStdin !== "function") {
|
|
664
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
let result: unknown;
|
|
668
|
+
try {
|
|
669
|
+
result = await withRetainedProcessProbeTimeout(
|
|
670
|
+
session.writeStdin({
|
|
671
|
+
sessionId: process.providerSessionId,
|
|
672
|
+
chars: "",
|
|
673
|
+
yieldTimeMs: 1_000,
|
|
674
|
+
maxOutputTokens: 2_000,
|
|
675
|
+
}),
|
|
676
|
+
);
|
|
677
|
+
} catch (error) {
|
|
678
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
679
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
680
|
+
}
|
|
681
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
682
|
+
return {
|
|
683
|
+
status: "proved",
|
|
684
|
+
proof: {
|
|
685
|
+
outcome: "lost",
|
|
686
|
+
exitCode: null,
|
|
687
|
+
reason: "provider_instance_not_found",
|
|
688
|
+
},
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
return { status: "deferred", reason: "provider_error" };
|
|
692
|
+
}
|
|
693
|
+
return classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** Classify only exact SDK control banners. Arbitrary output, malformed text,
|
|
697
|
+
* and a running marker are observations to retry; none is physical exit proof. */
|
|
698
|
+
export function classifyRetainedProcessPollResult(
|
|
699
|
+
result: unknown,
|
|
700
|
+
providerSessionId: number,
|
|
701
|
+
): RetainedProcessProbeResult {
|
|
702
|
+
if (typeof result !== "string") {
|
|
703
|
+
return { status: "deferred", reason: "provider_unknown" };
|
|
704
|
+
}
|
|
705
|
+
if (isExecSessionLostBanner(result, providerSessionId)) {
|
|
706
|
+
return {
|
|
707
|
+
status: "proved",
|
|
708
|
+
proof: {
|
|
709
|
+
outcome: "lost",
|
|
710
|
+
exitCode: null,
|
|
711
|
+
reason: "provider_session_lost_banner",
|
|
712
|
+
},
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
const exitCode = parseExecBannerExitCode(result);
|
|
716
|
+
if (exitCode !== null) {
|
|
717
|
+
return {
|
|
718
|
+
status: "proved",
|
|
719
|
+
proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" },
|
|
720
|
+
};
|
|
721
|
+
}
|
|
722
|
+
return {
|
|
723
|
+
status: "deferred",
|
|
724
|
+
reason: result.includes("Process running with session ID")
|
|
725
|
+
? "provider_running"
|
|
726
|
+
: "provider_unknown",
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
339
730
|
/**
|
|
340
731
|
* The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM
|
|
341
732
|
* viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those
|
|
@@ -405,6 +796,26 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
405
796
|
error: error instanceof Error ? error.message : String(error),
|
|
406
797
|
});
|
|
407
798
|
}
|
|
799
|
+
try {
|
|
800
|
+
recordRetainedProcessInventoryGauges(
|
|
801
|
+
observability,
|
|
802
|
+
await countActiveRetainedProcessesByOwnerState(db),
|
|
803
|
+
);
|
|
804
|
+
} catch (error) {
|
|
805
|
+
observability.warn("sandbox reaper: retained-process gauge refresh failed", {
|
|
806
|
+
error: error instanceof Error ? error.message : String(error),
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
try {
|
|
810
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
811
|
+
observability,
|
|
812
|
+
await countExpiredDrainingSandboxLeases(db),
|
|
813
|
+
);
|
|
814
|
+
} catch (error) {
|
|
815
|
+
observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
|
|
816
|
+
error: error instanceof Error ? error.message : String(error),
|
|
817
|
+
});
|
|
818
|
+
}
|
|
408
819
|
try {
|
|
409
820
|
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
410
821
|
} catch (error) {
|