@opengeni/worker-bundle 0.11.1 → 0.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +892 -317
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/types.ts +6 -6
- package/src/http.ts +14 -2
- package/src/index.ts +24 -2
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
environmentsEncryptionKeyBytes,
|
|
3
|
-
parseModelProvidersJson,
|
|
4
|
-
type RegistryProvider,
|
|
5
3
|
type Settings,
|
|
4
|
+
withCodexCatalogProvider,
|
|
6
5
|
} from "@opengeni/config";
|
|
7
6
|
import { settingsWithMcpCapabilityServers } from "@opengeni/core";
|
|
8
7
|
import {
|
|
@@ -10,14 +9,6 @@ import {
|
|
|
10
9
|
CODEX_APPS_MCP_SERVER_NAME,
|
|
11
10
|
CODEX_APPS_MCP_URL,
|
|
12
11
|
CODEX_APPS_STARTUP_TIMEOUT_MS,
|
|
13
|
-
CODEX_FALLBACK_MODEL_SLUGS,
|
|
14
|
-
CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
|
|
15
|
-
CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
|
|
16
|
-
CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
|
|
17
|
-
CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
|
|
18
|
-
CODEX_MODEL_ID_PREFIX,
|
|
19
|
-
CODEX_PROVIDER_BASE_URL,
|
|
20
|
-
CODEX_PROVIDER_ID,
|
|
21
12
|
} from "@opengeni/codex";
|
|
22
13
|
import {
|
|
23
14
|
listEnabledMcpCapabilityServers,
|
|
@@ -43,6 +34,9 @@ export async function settingsWithSessionMcpServersForRun(
|
|
|
43
34
|
sessionId: string,
|
|
44
35
|
attemptId: string,
|
|
45
36
|
settings: Settings,
|
|
37
|
+
options?: {
|
|
38
|
+
onResolvedServers?: (servers: readonly SessionMcpServerForRun[]) => void;
|
|
39
|
+
},
|
|
46
40
|
): Promise<Settings> {
|
|
47
41
|
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
48
42
|
if (!encryptionKey) {
|
|
@@ -56,10 +50,18 @@ export async function settingsWithSessionMcpServersForRun(
|
|
|
56
50
|
);
|
|
57
51
|
}
|
|
58
52
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
53
|
+
const servers = await listSessionMcpServersForRun(
|
|
54
|
+
db,
|
|
55
|
+
workspaceId,
|
|
56
|
+
sessionId,
|
|
57
|
+
attemptId,
|
|
58
|
+
encryptionKey ?? null,
|
|
62
59
|
);
|
|
60
|
+
// Keep credential provenance coupled to the exact decrypted rows that are
|
|
61
|
+
// overlaid into settings. A session projection read earlier in the turn can
|
|
62
|
+
// be stale after a concurrent mcpCredentialUpdates renewal.
|
|
63
|
+
options?.onResolvedServers?.(servers);
|
|
64
|
+
return settingsWithSessionMcpServers(settings, servers);
|
|
63
65
|
}
|
|
64
66
|
|
|
65
67
|
export function settingsWithSessionMcpServers(
|
|
@@ -115,6 +117,9 @@ export async function settingsWithCodexCredential(
|
|
|
115
117
|
return settings; // disabled / not connected / needs_relogin / error -> leave settings unchanged
|
|
116
118
|
}
|
|
117
119
|
const withProvider = withCodexProvider(settings);
|
|
120
|
+
if (!settings.codexConnectedAppsEnabled) {
|
|
121
|
+
return withProvider;
|
|
122
|
+
}
|
|
118
123
|
// Additive: append the synthetic codex_apps connectors MCP server for ANY
|
|
119
124
|
// active credential. Connector access is gated SERVER-SIDE per ChatGPT account
|
|
120
125
|
// (via chatgpt-account-id), NOT by token scopes — confirmed live: a `pro` token
|
|
@@ -162,28 +167,5 @@ export function withCodexAppsMcpServer(settings: Settings): Settings {
|
|
|
162
167
|
|
|
163
168
|
/** Pure: append the synthetic codex-subscription provider, idempotently. */
|
|
164
169
|
export function withCodexProvider(settings: Settings): Settings {
|
|
165
|
-
|
|
166
|
-
if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
|
|
167
|
-
return settings; // already injected
|
|
168
|
-
}
|
|
169
|
-
const codexProvider: RegistryProvider = {
|
|
170
|
-
kind: "codex-subscription",
|
|
171
|
-
id: CODEX_PROVIDER_ID,
|
|
172
|
-
label: "Codex (ChatGPT subscription)",
|
|
173
|
-
api: "responses",
|
|
174
|
-
baseUrl: CODEX_PROVIDER_BASE_URL,
|
|
175
|
-
models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
|
|
176
|
-
id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
|
|
177
|
-
upstreamModelId: slug,
|
|
178
|
-
label: slug,
|
|
179
|
-
reasoningEffort: true,
|
|
180
|
-
// These three values are the live Codex CLI catalog policy, kept distinct
|
|
181
|
-
// so proactive compaction and the effective input guard match Codex core.
|
|
182
|
-
contextWindowTokens: CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
|
|
183
|
-
effectiveContextWindowTokens: CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
|
|
184
|
-
autoCompactTokenLimit: CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
|
|
185
|
-
toolOutputTruncationTokens: CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
|
|
186
|
-
})),
|
|
187
|
-
};
|
|
188
|
-
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
170
|
+
return withCodexCatalogProvider(settings);
|
|
189
171
|
}
|
|
@@ -200,7 +200,7 @@ export function createCodexCapacityActivities(services: () => Promise<ActivitySe
|
|
|
200
200
|
}
|
|
201
201
|
}
|
|
202
202
|
if (result.action === "resumed") {
|
|
203
|
-
return { action: "resumed"
|
|
203
|
+
return { action: "resumed" };
|
|
204
204
|
}
|
|
205
205
|
if (result.action === "waiting") {
|
|
206
206
|
return {
|
|
@@ -1,56 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
* values (for example via `env`) into session events. Deep-walks payloads and
|
|
11
|
-
* replaces every exact occurrence of each secret value with
|
|
12
|
-
* `[redacted:<NAME>]`. Values shorter than six characters are skipped to avoid
|
|
13
|
-
* a false-positive flood; secrets are applied longest-value-first so
|
|
14
|
-
* overlapping values redact deterministically.
|
|
15
|
-
*/
|
|
16
|
-
export function createSecretRedactor(secrets: SecretForRedaction[]): (payload: unknown) => unknown {
|
|
17
|
-
const usable = secrets
|
|
18
|
-
.filter((secret) => secret.value.length >= MIN_REDACTABLE_VALUE_LENGTH)
|
|
19
|
-
.sort((a, b) => b.value.length - a.value.length);
|
|
20
|
-
if (usable.length === 0) {
|
|
21
|
-
return identityRedactor;
|
|
22
|
-
}
|
|
23
|
-
const redact = (payload: unknown): unknown => {
|
|
24
|
-
if (typeof payload === "string") {
|
|
25
|
-
let out = payload;
|
|
26
|
-
for (const secret of usable) {
|
|
27
|
-
if (out.includes(secret.value)) {
|
|
28
|
-
out = out.split(secret.value).join(`[redacted:${secret.name}]`);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return out;
|
|
32
|
-
}
|
|
33
|
-
if (Array.isArray(payload)) {
|
|
34
|
-
return payload.map(redact);
|
|
35
|
-
}
|
|
36
|
-
if (isPlainObject(payload)) {
|
|
37
|
-
return Object.fromEntries(
|
|
38
|
-
Object.entries(payload).map(([key, value]) => [key, redact(value)]),
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
return payload;
|
|
42
|
-
};
|
|
43
|
-
return redact;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export function identityRedactor(payload: unknown): unknown {
|
|
47
|
-
return payload;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
51
|
-
if (!value || typeof value !== "object") {
|
|
52
|
-
return false;
|
|
53
|
-
}
|
|
54
|
-
const proto = Object.getPrototypeOf(value);
|
|
55
|
-
return proto === Object.prototype || proto === null;
|
|
56
|
-
}
|
|
1
|
+
export {
|
|
2
|
+
createSecretRedactor,
|
|
3
|
+
identityRedactor,
|
|
4
|
+
isCredentialHeaderName,
|
|
5
|
+
redactSensitiveData,
|
|
6
|
+
redactSensitiveText,
|
|
7
|
+
redactSerializedJson,
|
|
8
|
+
type SecretForRedaction,
|
|
9
|
+
} from "@opengeni/contracts";
|
|
@@ -34,21 +34,32 @@ import {
|
|
|
34
34
|
accrueWarmSeconds,
|
|
35
35
|
confirmDrainCold,
|
|
36
36
|
appendSessionEventToSandboxGroup,
|
|
37
|
+
claimTerminalRetainedProcesses,
|
|
38
|
+
countActiveRetainedProcessesByOwnerState,
|
|
39
|
+
countExpiredDrainingSandboxLeases,
|
|
37
40
|
countQueuedTurns,
|
|
38
41
|
countSandboxLeasesByLiveness,
|
|
42
|
+
deferRetainedProcessReconciliation,
|
|
39
43
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
40
44
|
getBillingBalance,
|
|
41
45
|
listCreditBalancesByAccount,
|
|
42
46
|
listLiveModalSandboxLeaseAttributions,
|
|
43
47
|
listMeterableWarmLeases,
|
|
44
48
|
persistDrainSnapshot,
|
|
49
|
+
recordRetainedProcessReconciliationProof,
|
|
45
50
|
readWorkspaceArchiveCapturePreflight,
|
|
46
51
|
readLease,
|
|
47
52
|
reapExpiredSessionListSnapshots,
|
|
48
53
|
reapStaleLeaseHoldersGlobal,
|
|
54
|
+
retainedProcessReconciliationProof,
|
|
55
|
+
retainedProcessSettlementIdentity,
|
|
49
56
|
rlsContextForWorkspace,
|
|
57
|
+
settleRetainedProcess,
|
|
50
58
|
type MeterableWarmLease,
|
|
51
59
|
type ReapDrainable,
|
|
60
|
+
type RetainedProcessProviderProof,
|
|
61
|
+
type SandboxRetainedProcess,
|
|
62
|
+
type LeaseSnapshot,
|
|
52
63
|
} from "@opengeni/db";
|
|
53
64
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
54
65
|
import {
|
|
@@ -62,7 +73,9 @@ import {
|
|
|
62
73
|
createSandboxClientForBackend,
|
|
63
74
|
deletePriorPersistedSnapshot,
|
|
64
75
|
deserializeSandboxSessionStateEnvelope,
|
|
76
|
+
isExecSessionLostBanner,
|
|
65
77
|
isProviderSandboxNotFoundError,
|
|
78
|
+
parseExecBannerExitCode,
|
|
66
79
|
sweepModalOrphanSandboxes,
|
|
67
80
|
terminateModalSandboxById,
|
|
68
81
|
type ModalOrphanSweepTermination,
|
|
@@ -73,10 +86,15 @@ import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
|
73
86
|
import {
|
|
74
87
|
recordCreditBalanceGauges,
|
|
75
88
|
recordCreditMicros,
|
|
89
|
+
recordExpiredDrainingSandboxLeaseGauges,
|
|
90
|
+
recordRetainedProcessInventoryGauges,
|
|
91
|
+
recordRetainedProcessReconciliation,
|
|
92
|
+
type RetainedProcessReconciliationOutcome,
|
|
76
93
|
recordSandboxLeaseGauges,
|
|
77
94
|
recordSandboxOrphansTerminated,
|
|
78
95
|
recordTurnsQueuedGauge,
|
|
79
96
|
} from "../observability-metrics";
|
|
97
|
+
import { providerIdentityFromResumeState } from "../sandbox-routing";
|
|
80
98
|
|
|
81
99
|
export type ReapSandboxLeasesResult = {
|
|
82
100
|
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
@@ -179,8 +197,35 @@ export type SandboxLeaseActivityOptions = {
|
|
|
179
197
|
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
180
198
|
* Modal list+tag comparison when Modal is configured). */
|
|
181
199
|
sweepModalOrphans?: SweepModalOrphansFn;
|
|
200
|
+
/** Override only the read-only provider process probe. The canonical DB
|
|
201
|
+
* settlement remains real in tests and production. */
|
|
202
|
+
probeRetainedProcess?: RetainedProcessProbeFn;
|
|
182
203
|
};
|
|
183
204
|
|
|
205
|
+
export type RetainedProcessProbeResult =
|
|
206
|
+
| { status: "proved"; proof: RetainedProcessProviderProof }
|
|
207
|
+
| {
|
|
208
|
+
status: "deferred";
|
|
209
|
+
reason:
|
|
210
|
+
| "identity_mismatch"
|
|
211
|
+
| "resume_state_missing"
|
|
212
|
+
| "backend_unsupported"
|
|
213
|
+
| "provider_running"
|
|
214
|
+
| "provider_unknown"
|
|
215
|
+
| "provider_timeout"
|
|
216
|
+
| "provider_error";
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
export type RetainedProcessProbeFn = (
|
|
220
|
+
settings: ActivityServices["settings"],
|
|
221
|
+
lease: LeaseSnapshot,
|
|
222
|
+
process: SandboxRetainedProcess,
|
|
223
|
+
) => Promise<RetainedProcessProbeResult>;
|
|
224
|
+
|
|
225
|
+
export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
226
|
+
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
227
|
+
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
228
|
+
|
|
184
229
|
export function createSandboxLeaseActivities(
|
|
185
230
|
services: () => Promise<ActivityServices>,
|
|
186
231
|
options: SandboxLeaseActivityOptions = {},
|
|
@@ -188,6 +233,7 @@ export function createSandboxLeaseActivities(
|
|
|
188
233
|
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
189
234
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
190
235
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
236
|
+
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
191
237
|
/**
|
|
192
238
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
193
239
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -248,6 +294,12 @@ export function createSandboxLeaseActivities(
|
|
|
248
294
|
observability,
|
|
249
295
|
);
|
|
250
296
|
|
|
297
|
+
// (0c) Terminal-owner retained-process reconciliation. Owner state selects
|
|
298
|
+
// only a bounded inspection batch; exact provider exit/loss proof is
|
|
299
|
+
// checkpointed before canonical settlement. Ambiguous, running, timed-out,
|
|
300
|
+
// unsupported, and transient provider states preserve every durable row.
|
|
301
|
+
await reconcileTerminalRetainedProcesses(db, settings, observability, probeRetainedProcess);
|
|
302
|
+
|
|
251
303
|
// (1) The DB-only cross-workspace sweep. Returns the drainable rows.
|
|
252
304
|
const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
|
|
253
305
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
@@ -336,6 +388,345 @@ export function createSandboxLeaseActivities(
|
|
|
336
388
|
return { reapSandboxLeases };
|
|
337
389
|
}
|
|
338
390
|
|
|
391
|
+
async function reconcileTerminalRetainedProcesses(
|
|
392
|
+
db: ActivityServices["db"],
|
|
393
|
+
settings: ActivityServices["settings"],
|
|
394
|
+
observability: ActivityServices["observability"],
|
|
395
|
+
probe: RetainedProcessProbeFn,
|
|
396
|
+
): Promise<void> {
|
|
397
|
+
const claimId = crypto.randomUUID();
|
|
398
|
+
let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
|
|
399
|
+
try {
|
|
400
|
+
claims = await claimTerminalRetainedProcesses(db, {
|
|
401
|
+
claimId,
|
|
402
|
+
limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
|
|
403
|
+
claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,
|
|
404
|
+
});
|
|
405
|
+
} catch (error) {
|
|
406
|
+
recordRetainedProcessReconciliation(observability, "claim_failed");
|
|
407
|
+
observability.warn("sandbox reaper: retained-process claim failed", {
|
|
408
|
+
error: error instanceof Error ? error.message : String(error),
|
|
409
|
+
});
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
for (const claim of claims) {
|
|
414
|
+
let process = claim.process;
|
|
415
|
+
const expected = retainedProcessSettlementIdentity(process);
|
|
416
|
+
let proof = retainedProcessReconciliationProof(process);
|
|
417
|
+
if (!proof) {
|
|
418
|
+
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
419
|
+
() => null,
|
|
420
|
+
);
|
|
421
|
+
if (
|
|
422
|
+
!lease ||
|
|
423
|
+
lease.id !== process.leaseId ||
|
|
424
|
+
lease.leaseEpoch !== process.leaseEpoch ||
|
|
425
|
+
lease.backend !== process.providerBackend ||
|
|
426
|
+
lease.instanceId !== process.providerInstanceId ||
|
|
427
|
+
process.routeTargetId !== null
|
|
428
|
+
) {
|
|
429
|
+
await deferRetainedProcessClaim(
|
|
430
|
+
db,
|
|
431
|
+
settings,
|
|
432
|
+
observability,
|
|
433
|
+
process,
|
|
434
|
+
expected,
|
|
435
|
+
claim.claimId,
|
|
436
|
+
"identity_mismatch",
|
|
437
|
+
);
|
|
438
|
+
continue;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
let observation: RetainedProcessProbeResult;
|
|
442
|
+
try {
|
|
443
|
+
observation = await probe(settings, lease, process);
|
|
444
|
+
} catch (error) {
|
|
445
|
+
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
446
|
+
processId: process.id,
|
|
447
|
+
providerBackend: process.providerBackend,
|
|
448
|
+
error: error instanceof Error ? error.message : String(error),
|
|
449
|
+
});
|
|
450
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
451
|
+
}
|
|
452
|
+
if (observation.status === "deferred") {
|
|
453
|
+
await deferRetainedProcessClaim(
|
|
454
|
+
db,
|
|
455
|
+
settings,
|
|
456
|
+
observability,
|
|
457
|
+
process,
|
|
458
|
+
expected,
|
|
459
|
+
claim.claimId,
|
|
460
|
+
observation.reason,
|
|
461
|
+
);
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
try {
|
|
465
|
+
process = await recordRetainedProcessReconciliationProof(db, {
|
|
466
|
+
accountId: process.accountId,
|
|
467
|
+
workspaceId: process.workspaceId,
|
|
468
|
+
sessionId: process.sessionId,
|
|
469
|
+
processId: process.id,
|
|
470
|
+
expected,
|
|
471
|
+
claimId: claim.claimId,
|
|
472
|
+
proof: observation.proof,
|
|
473
|
+
});
|
|
474
|
+
proof = observation.proof;
|
|
475
|
+
recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
|
|
476
|
+
} catch (error) {
|
|
477
|
+
recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
|
|
478
|
+
observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
|
|
479
|
+
processId: process.id,
|
|
480
|
+
error: error instanceof Error ? error.message : String(error),
|
|
481
|
+
});
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
try {
|
|
487
|
+
await settleRetainedProcess(db, {
|
|
488
|
+
accountId: process.accountId,
|
|
489
|
+
workspaceId: process.workspaceId,
|
|
490
|
+
sessionId: process.sessionId,
|
|
491
|
+
processId: process.id,
|
|
492
|
+
expected,
|
|
493
|
+
reconciliationClaimId: claim.claimId,
|
|
494
|
+
outcome: proof.outcome,
|
|
495
|
+
exitCode: proof.exitCode,
|
|
496
|
+
reason: proof.reason,
|
|
497
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
498
|
+
});
|
|
499
|
+
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
500
|
+
} catch (error) {
|
|
501
|
+
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
502
|
+
observability.warn("sandbox reaper: retained-process settlement failed", {
|
|
503
|
+
processId: process.id,
|
|
504
|
+
error: error instanceof Error ? error.message : String(error),
|
|
505
|
+
});
|
|
506
|
+
await deferRetainedProcessClaim(
|
|
507
|
+
db,
|
|
508
|
+
settings,
|
|
509
|
+
observability,
|
|
510
|
+
process,
|
|
511
|
+
expected,
|
|
512
|
+
claim.claimId,
|
|
513
|
+
"settlement_failed",
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
async function deferRetainedProcessClaim(
|
|
520
|
+
db: ActivityServices["db"],
|
|
521
|
+
settings: ActivityServices["settings"],
|
|
522
|
+
observability: ActivityServices["observability"],
|
|
523
|
+
process: SandboxRetainedProcess,
|
|
524
|
+
expected: ReturnType<typeof retainedProcessSettlementIdentity>,
|
|
525
|
+
claimId: string,
|
|
526
|
+
outcome: RetainedProcessReconciliationOutcome,
|
|
527
|
+
): Promise<void> {
|
|
528
|
+
const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
529
|
+
const retryAfterMs = Math.min(
|
|
530
|
+
5 * 60_000,
|
|
531
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
|
|
532
|
+
);
|
|
533
|
+
try {
|
|
534
|
+
await deferRetainedProcessReconciliation(db, {
|
|
535
|
+
accountId: process.accountId,
|
|
536
|
+
workspaceId: process.workspaceId,
|
|
537
|
+
sessionId: process.sessionId,
|
|
538
|
+
processId: process.id,
|
|
539
|
+
expected,
|
|
540
|
+
claimId,
|
|
541
|
+
outcome,
|
|
542
|
+
retryAfterMs,
|
|
543
|
+
});
|
|
544
|
+
recordRetainedProcessReconciliation(observability, outcome);
|
|
545
|
+
} catch (error) {
|
|
546
|
+
recordRetainedProcessReconciliation(observability, "defer_failed");
|
|
547
|
+
observability.warn("sandbox reaper: retained-process defer failed", {
|
|
548
|
+
processId: process.id,
|
|
549
|
+
error: error instanceof Error ? error.message : String(error),
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
type RetainedProcessProbeClient = {
|
|
555
|
+
backendId: string;
|
|
556
|
+
resume?: (state: unknown) => Promise<unknown>;
|
|
557
|
+
deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
|
|
558
|
+
};
|
|
559
|
+
|
|
560
|
+
type RetainedProcessProbeSession = {
|
|
561
|
+
writeStdin?: (args: {
|
|
562
|
+
sessionId: number;
|
|
563
|
+
chars: string;
|
|
564
|
+
yieldTimeMs: number;
|
|
565
|
+
maxOutputTokens: number;
|
|
566
|
+
}) => Promise<unknown>;
|
|
567
|
+
};
|
|
568
|
+
|
|
569
|
+
const RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
570
|
+
|
|
571
|
+
async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {
|
|
572
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
573
|
+
try {
|
|
574
|
+
return await Promise.race([
|
|
575
|
+
promise,
|
|
576
|
+
new Promise<never>((_resolve, reject) => {
|
|
577
|
+
timer = setTimeout(
|
|
578
|
+
() => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
|
|
579
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
580
|
+
);
|
|
581
|
+
timer.unref?.();
|
|
582
|
+
}),
|
|
583
|
+
]);
|
|
584
|
+
} finally {
|
|
585
|
+
if (timer) clearTimeout(timer);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
export async function probeRetainedProcessAtProvider(
|
|
590
|
+
settings: ActivityServices["settings"],
|
|
591
|
+
lease: LeaseSnapshot,
|
|
592
|
+
process: SandboxRetainedProcess,
|
|
593
|
+
): Promise<RetainedProcessProbeResult> {
|
|
594
|
+
if (
|
|
595
|
+
lease.id !== process.leaseId ||
|
|
596
|
+
lease.sandboxGroupId !== process.sandboxGroupId ||
|
|
597
|
+
lease.leaseEpoch !== process.leaseEpoch ||
|
|
598
|
+
lease.backend !== process.providerBackend ||
|
|
599
|
+
lease.instanceId !== process.providerInstanceId ||
|
|
600
|
+
lease.resumeBackendId !== process.providerBackend ||
|
|
601
|
+
process.routeTargetId !== null
|
|
602
|
+
) {
|
|
603
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
604
|
+
}
|
|
605
|
+
if (!lease.resumeState) {
|
|
606
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
607
|
+
}
|
|
608
|
+
const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
|
|
609
|
+
if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
|
|
610
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
611
|
+
}
|
|
612
|
+
if (providerIdentityFromResumeState(lease.resumeState) !== process.providerInstanceId) {
|
|
613
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
let client: RetainedProcessProbeClient | undefined;
|
|
617
|
+
try {
|
|
618
|
+
client = createSandboxClientForBackend(process.providerBackend as never, settings) as
|
|
619
|
+
| RetainedProcessProbeClient
|
|
620
|
+
| undefined;
|
|
621
|
+
} catch {
|
|
622
|
+
return { status: "deferred", reason: "provider_error" };
|
|
623
|
+
}
|
|
624
|
+
if (!client || client.backendId !== process.providerBackend || !client.resume) {
|
|
625
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
626
|
+
}
|
|
627
|
+
const envelopeSessionState =
|
|
628
|
+
(lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
|
|
629
|
+
let resumedState: unknown;
|
|
630
|
+
try {
|
|
631
|
+
resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
632
|
+
client as never,
|
|
633
|
+
envelopeSessionState,
|
|
634
|
+
);
|
|
635
|
+
} catch {
|
|
636
|
+
return { status: "deferred", reason: "provider_error" };
|
|
637
|
+
}
|
|
638
|
+
if (resumedState === undefined) {
|
|
639
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
let session: RetainedProcessProbeSession;
|
|
643
|
+
try {
|
|
644
|
+
session = (await withRetainedProcessProbeTimeout(
|
|
645
|
+
client.resume(resumedState),
|
|
646
|
+
)) as RetainedProcessProbeSession;
|
|
647
|
+
} catch (error) {
|
|
648
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
649
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
650
|
+
}
|
|
651
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
652
|
+
return {
|
|
653
|
+
status: "proved",
|
|
654
|
+
proof: {
|
|
655
|
+
outcome: "lost",
|
|
656
|
+
exitCode: null,
|
|
657
|
+
reason: "provider_instance_not_found",
|
|
658
|
+
},
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
return { status: "deferred", reason: "provider_error" };
|
|
662
|
+
}
|
|
663
|
+
if (typeof session.writeStdin !== "function") {
|
|
664
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
let result: unknown;
|
|
668
|
+
try {
|
|
669
|
+
result = await withRetainedProcessProbeTimeout(
|
|
670
|
+
session.writeStdin({
|
|
671
|
+
sessionId: process.providerSessionId,
|
|
672
|
+
chars: "",
|
|
673
|
+
yieldTimeMs: 1_000,
|
|
674
|
+
maxOutputTokens: 2_000,
|
|
675
|
+
}),
|
|
676
|
+
);
|
|
677
|
+
} catch (error) {
|
|
678
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
679
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
680
|
+
}
|
|
681
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
682
|
+
return {
|
|
683
|
+
status: "proved",
|
|
684
|
+
proof: {
|
|
685
|
+
outcome: "lost",
|
|
686
|
+
exitCode: null,
|
|
687
|
+
reason: "provider_instance_not_found",
|
|
688
|
+
},
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
return { status: "deferred", reason: "provider_error" };
|
|
692
|
+
}
|
|
693
|
+
return classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** Classify only exact SDK control banners. Arbitrary output, malformed text,
|
|
697
|
+
* and a running marker are observations to retry; none is physical exit proof. */
|
|
698
|
+
export function classifyRetainedProcessPollResult(
|
|
699
|
+
result: unknown,
|
|
700
|
+
providerSessionId: number,
|
|
701
|
+
): RetainedProcessProbeResult {
|
|
702
|
+
if (typeof result !== "string") {
|
|
703
|
+
return { status: "deferred", reason: "provider_unknown" };
|
|
704
|
+
}
|
|
705
|
+
if (isExecSessionLostBanner(result, providerSessionId)) {
|
|
706
|
+
return {
|
|
707
|
+
status: "proved",
|
|
708
|
+
proof: {
|
|
709
|
+
outcome: "lost",
|
|
710
|
+
exitCode: null,
|
|
711
|
+
reason: "provider_session_lost_banner",
|
|
712
|
+
},
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
const exitCode = parseExecBannerExitCode(result);
|
|
716
|
+
if (exitCode !== null) {
|
|
717
|
+
return {
|
|
718
|
+
status: "proved",
|
|
719
|
+
proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" },
|
|
720
|
+
};
|
|
721
|
+
}
|
|
722
|
+
return {
|
|
723
|
+
status: "deferred",
|
|
724
|
+
reason: result.includes("Process running with session ID")
|
|
725
|
+
? "provider_running"
|
|
726
|
+
: "provider_unknown",
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
339
730
|
/**
|
|
340
731
|
* The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM
|
|
341
732
|
* viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those
|
|
@@ -405,6 +796,26 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
405
796
|
error: error instanceof Error ? error.message : String(error),
|
|
406
797
|
});
|
|
407
798
|
}
|
|
799
|
+
try {
|
|
800
|
+
recordRetainedProcessInventoryGauges(
|
|
801
|
+
observability,
|
|
802
|
+
await countActiveRetainedProcessesByOwnerState(db),
|
|
803
|
+
);
|
|
804
|
+
} catch (error) {
|
|
805
|
+
observability.warn("sandbox reaper: retained-process gauge refresh failed", {
|
|
806
|
+
error: error instanceof Error ? error.message : String(error),
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
try {
|
|
810
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
811
|
+
observability,
|
|
812
|
+
await countExpiredDrainingSandboxLeases(db),
|
|
813
|
+
);
|
|
814
|
+
} catch (error) {
|
|
815
|
+
observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
|
|
816
|
+
error: error instanceof Error ? error.message : String(error),
|
|
817
|
+
});
|
|
818
|
+
}
|
|
408
819
|
try {
|
|
409
820
|
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
410
821
|
} catch (error) {
|
package/src/activities/types.ts
CHANGED
|
@@ -120,8 +120,7 @@ export type ReconcileCodexCapacityWaitInput = {
|
|
|
120
120
|
|
|
121
121
|
export type ReconcileCodexCapacityWaitResult =
|
|
122
122
|
| ({ action: "waiting" } & CodexCapacityWaitRef)
|
|
123
|
-
| { action: "resumed"
|
|
124
|
-
| { action: "superseded" | "stale" };
|
|
123
|
+
| { action: "resumed" | "paused" | "superseded" | "stale" };
|
|
125
124
|
|
|
126
125
|
export type ActivityDependencies = Partial<ActivityServices>;
|
|
127
126
|
|
|
@@ -241,7 +240,7 @@ export type IndexDocumentInput = {
|
|
|
241
240
|
type ClaimedRunAgentTurnResult = {
|
|
242
241
|
// "recovering": this attempt ended after durably preserving the same current
|
|
243
242
|
// inference for a new attempt. Recovery is not prompt queue work.
|
|
244
|
-
status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering";
|
|
243
|
+
status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering" | "waiting_capacity";
|
|
245
244
|
turnId: string;
|
|
246
245
|
attemptId: string;
|
|
247
246
|
// Provider backpressure pacing: when set on an idle or recovering result, the
|
|
@@ -253,9 +252,10 @@ type ClaimedRunAgentTurnResult = {
|
|
|
253
252
|
// "continue now" (invariant 4: NO THRASH). Distinct from a normal continueDelayMs:0
|
|
254
253
|
// which legitimately means "a rotation candidate is ready, re-dispatch immediately".
|
|
255
254
|
idleUntilReset?: boolean;
|
|
256
|
-
// Durable native zero-pool wait
|
|
257
|
-
// persisted in Postgres and
|
|
258
|
-
// The workflow must not call
|
|
255
|
+
// Durable native zero-pool wait for this same nonterminal logical turn.
|
|
256
|
+
// Unlike continueDelayMs, this reference is persisted in Postgres and
|
|
257
|
+
// reconstructed after workflow/worker restart. The workflow must not call
|
|
258
|
+
// maybeContinueGoal or manufacture queue/input work while it is active.
|
|
259
259
|
capacityWait?: CodexCapacityWaitRef;
|
|
260
260
|
// This execution reached a durable terminal-for-now boundary (for example,
|
|
261
261
|
// maintenance could not run or same-turn context recovery failed). End this
|