@opengeni/worker-bundle 0.11.1 → 0.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +892 -317
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/types.ts +6 -6
- package/src/http.ts +14 -2
- package/src/index.ts +24 -2
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
package/src/http.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
assertRuntimeDatabasePosture,
|
|
4
|
+
dbSql,
|
|
5
|
+
type Database,
|
|
6
|
+
type RuntimeDatabasePostureOptions,
|
|
7
|
+
} from "@opengeni/db";
|
|
3
8
|
import type { EventBus } from "@opengeni/events";
|
|
4
9
|
import type { Observability } from "@opengeni/observability";
|
|
5
10
|
|
|
@@ -112,8 +117,15 @@ export async function runReadinessChecks(
|
|
|
112
117
|
};
|
|
113
118
|
}
|
|
114
119
|
|
|
115
|
-
export function dbReadyCheck(
|
|
120
|
+
export function dbReadyCheck(
|
|
121
|
+
db: Database,
|
|
122
|
+
posture?: RuntimeDatabasePostureOptions,
|
|
123
|
+
): () => Promise<void> {
|
|
116
124
|
return async () => {
|
|
125
|
+
if (posture) {
|
|
126
|
+
await assertRuntimeDatabasePosture(db, posture);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
117
129
|
await db.execute(dbSql`select 1`);
|
|
118
130
|
};
|
|
119
131
|
}
|
package/src/index.ts
CHANGED
|
@@ -7,7 +7,13 @@ import {
|
|
|
7
7
|
temporalConnectionOptions,
|
|
8
8
|
type Settings,
|
|
9
9
|
} from "@opengeni/config";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
assertRuntimeDatabasePosture,
|
|
12
|
+
createDb,
|
|
13
|
+
markSessionWorkflowWakeDelivered,
|
|
14
|
+
type Database,
|
|
15
|
+
type RuntimeDatabasePostureOptions,
|
|
16
|
+
} from "@opengeni/db";
|
|
11
17
|
import { createNatsEventBus, type EventBus } from "@opengeni/events";
|
|
12
18
|
import {
|
|
13
19
|
createObservability,
|
|
@@ -471,6 +477,11 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
471
477
|
|
|
472
478
|
export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
|
|
473
479
|
activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
|
|
480
|
+
/**
|
|
481
|
+
* Exact catalog posture required by the standalone runtime. Embedded hosts
|
|
482
|
+
* may omit this when they own an equivalent database isolation contract.
|
|
483
|
+
*/
|
|
484
|
+
databasePosture?: RuntimeDatabasePostureOptions;
|
|
474
485
|
/**
|
|
475
486
|
* `role-default` registers OpenGeni's internal maintenance schedules on a
|
|
476
487
|
* control worker only. These are engine maintenance schedules, not a host's
|
|
@@ -588,7 +599,7 @@ export async function createOpenGeniWorkerService(
|
|
|
588
599
|
settings,
|
|
589
600
|
observability,
|
|
590
601
|
checks: {
|
|
591
|
-
db: dbReadyCheck(options.activityDependencies.db),
|
|
602
|
+
db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
|
|
592
603
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
593
604
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
594
605
|
},
|
|
@@ -693,9 +704,19 @@ export async function startWorker() {
|
|
|
693
704
|
...(searchPath ? { searchPath } : {}),
|
|
694
705
|
rlsStrategy: settings.rlsStrategy,
|
|
695
706
|
});
|
|
707
|
+
const databasePosture = {
|
|
708
|
+
rlsStrategy: settings.rlsStrategy,
|
|
709
|
+
expectedRole: settings.runtimeDatabaseRole,
|
|
710
|
+
targetSchema: settings.dbSchema.trim() || "public",
|
|
711
|
+
} as const;
|
|
696
712
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
697
713
|
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
698
714
|
try {
|
|
715
|
+
await retryStartupDependency(
|
|
716
|
+
"PostgreSQL runtime posture",
|
|
717
|
+
() => assertRuntimeDatabasePosture(dbClient.db, databasePosture),
|
|
718
|
+
{ ...retryOptions, onRetry },
|
|
719
|
+
);
|
|
699
720
|
bus = await retryStartupDependency(
|
|
700
721
|
"NATS",
|
|
701
722
|
() =>
|
|
@@ -711,6 +732,7 @@ export async function startWorker() {
|
|
|
711
732
|
await runOpenGeniWorker({
|
|
712
733
|
role,
|
|
713
734
|
settings,
|
|
735
|
+
databasePosture,
|
|
714
736
|
activityDependencies: {
|
|
715
737
|
observability,
|
|
716
738
|
db: dbClient.db,
|
|
@@ -19,6 +19,7 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
|
19
19
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
20
20
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
21
21
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
22
|
+
const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
|
|
22
23
|
|
|
23
24
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
24
25
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -383,6 +384,142 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
|
|
|
383
384
|
});
|
|
384
385
|
}
|
|
385
386
|
|
|
387
|
+
const RETAINED_PROCESS_OWNER_STATES = [
|
|
388
|
+
"direct",
|
|
389
|
+
"queued",
|
|
390
|
+
"running",
|
|
391
|
+
"requires_action",
|
|
392
|
+
"recovering",
|
|
393
|
+
"waiting_capacity",
|
|
394
|
+
"completed",
|
|
395
|
+
"failed",
|
|
396
|
+
"cancelled",
|
|
397
|
+
"superseded",
|
|
398
|
+
"withdrawn_for_edit",
|
|
399
|
+
"missing",
|
|
400
|
+
"unknown",
|
|
401
|
+
] as const;
|
|
402
|
+
|
|
403
|
+
const RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_STATES);
|
|
404
|
+
|
|
405
|
+
export function recordRetainedProcessInventoryGauges(
|
|
406
|
+
observability: Observability,
|
|
407
|
+
counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,
|
|
408
|
+
): void {
|
|
409
|
+
const normalized = new Map<string, { active: number; terminal: number }>();
|
|
410
|
+
for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
|
|
411
|
+
normalized.set(ownerState, { active: 0, terminal: 0 });
|
|
412
|
+
}
|
|
413
|
+
for (const count of counts) {
|
|
414
|
+
const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState)
|
|
415
|
+
? count.ownerState
|
|
416
|
+
: "unknown";
|
|
417
|
+
const current = normalized.get(ownerState)!;
|
|
418
|
+
current.active += Math.max(0, count.activeCount);
|
|
419
|
+
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
let terminalTotal = 0;
|
|
423
|
+
for (const [ownerState, count] of normalized) {
|
|
424
|
+
terminalTotal += count.terminal;
|
|
425
|
+
observability.setGauge({
|
|
426
|
+
name: "opengeni_retained_processes_active",
|
|
427
|
+
help: "Current active retained provider processes by durable owner state.",
|
|
428
|
+
labels: { owner_state: ownerState },
|
|
429
|
+
value: count.active,
|
|
430
|
+
});
|
|
431
|
+
observability.setGauge({
|
|
432
|
+
name: "opengeni_retained_processes_terminal_owner_backlog",
|
|
433
|
+
help: "Current active retained processes whose exact owner attempt is terminal.",
|
|
434
|
+
labels: { owner_state: ownerState },
|
|
435
|
+
value: count.terminal,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
const previous = retainedProcessTerminalBacklogTotals.get(observability);
|
|
439
|
+
if (previous !== undefined && terminalTotal > previous) {
|
|
440
|
+
observability.incrementCounter({
|
|
441
|
+
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
442
|
+
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
443
|
+
amount: terminalTotal - previous,
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const EXPIRED_DRAINING_BACKENDS = [
|
|
450
|
+
"none",
|
|
451
|
+
"local",
|
|
452
|
+
"docker",
|
|
453
|
+
"modal",
|
|
454
|
+
"e2b",
|
|
455
|
+
"daytona",
|
|
456
|
+
"blaxel",
|
|
457
|
+
"runloop",
|
|
458
|
+
"selfhosted",
|
|
459
|
+
"unknown",
|
|
460
|
+
] as const;
|
|
461
|
+
const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
|
|
462
|
+
const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
|
|
463
|
+
|
|
464
|
+
export function recordExpiredDrainingSandboxLeaseGauges(
|
|
465
|
+
observability: Observability,
|
|
466
|
+
counts: Array<{ backend: string; ageBucket: string; count: number }>,
|
|
467
|
+
): void {
|
|
468
|
+
const normalized = new Map<string, number>();
|
|
469
|
+
for (const backend of EXPIRED_DRAINING_BACKENDS) {
|
|
470
|
+
for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
|
|
471
|
+
normalized.set(`${backend}:${ageBucket}`, 0);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
for (const count of counts) {
|
|
475
|
+
const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
|
|
476
|
+
if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket as never)) continue;
|
|
477
|
+
const key = `${backend}:${count.ageBucket}`;
|
|
478
|
+
normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
|
|
479
|
+
}
|
|
480
|
+
for (const [key, value] of normalized) {
|
|
481
|
+
const [backend, ageBucket] = key.split(":") as [string, string];
|
|
482
|
+
observability.setGauge({
|
|
483
|
+
name: "opengeni_sandbox_leases_expired_draining",
|
|
484
|
+
help: "Current expired draining sandbox leases by backend and fixed age bucket.",
|
|
485
|
+
labels: { backend, age_bucket: ageBucket },
|
|
486
|
+
value,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
|
|
492
|
+
"claim_failed",
|
|
493
|
+
"proof_exited",
|
|
494
|
+
"proof_lost",
|
|
495
|
+
"proof_checkpoint_failed",
|
|
496
|
+
"settled_exited",
|
|
497
|
+
"settled_lost",
|
|
498
|
+
"settlement_failed",
|
|
499
|
+
"identity_mismatch",
|
|
500
|
+
"resume_state_missing",
|
|
501
|
+
"backend_unsupported",
|
|
502
|
+
"provider_running",
|
|
503
|
+
"provider_unknown",
|
|
504
|
+
"provider_timeout",
|
|
505
|
+
"provider_error",
|
|
506
|
+
"defer_failed",
|
|
507
|
+
] as const;
|
|
508
|
+
|
|
509
|
+
export type RetainedProcessReconciliationOutcome =
|
|
510
|
+
(typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];
|
|
511
|
+
|
|
512
|
+
export function recordRetainedProcessReconciliation(
|
|
513
|
+
observability: Observability,
|
|
514
|
+
outcome: RetainedProcessReconciliationOutcome,
|
|
515
|
+
): void {
|
|
516
|
+
observability.incrementCounter({
|
|
517
|
+
name: "opengeni_retained_process_reconciliation_total",
|
|
518
|
+
help: "Bounded retained-process reconciliation observations by fixed app outcome.",
|
|
519
|
+
labels: { outcome },
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
|
|
386
523
|
export function recordCreditMicros(
|
|
387
524
|
observability: Observability | undefined,
|
|
388
525
|
kind: CreditMicrosKind,
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
23
|
import type { Settings } from "@opengeni/config";
|
|
24
|
+
import { redactSensitiveText } from "@opengeni/contracts";
|
|
24
25
|
import {
|
|
25
26
|
acquireLease,
|
|
26
27
|
beginSandboxRematerialization,
|
|
@@ -75,6 +76,13 @@ export function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLe
|
|
|
75
76
|
return `turn-attempt:${normalized}`;
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
export function isRetryableDegradedRestore(restore: {
|
|
80
|
+
status: string;
|
|
81
|
+
retryable?: boolean;
|
|
82
|
+
}): boolean {
|
|
83
|
+
return restore.status === "degraded" && restore.retryable === true;
|
|
84
|
+
}
|
|
85
|
+
|
|
78
86
|
/** The minimal services surface resumeBoxForTurn needs. A subset of
|
|
79
87
|
* ActivityServices so a test (and the API later) can pass a lean bag. */
|
|
80
88
|
export type SandboxResumeServices = {
|
|
@@ -207,6 +215,17 @@ class SnapshotTimeoutError extends Error {
|
|
|
207
215
|
}
|
|
208
216
|
}
|
|
209
217
|
|
|
218
|
+
function safeSnapshotError(error: unknown): { name: string; message: string } {
|
|
219
|
+
const rawName = error instanceof Error ? error.name : "Error";
|
|
220
|
+
return {
|
|
221
|
+
name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
|
|
222
|
+
message: redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(
|
|
223
|
+
0,
|
|
224
|
+
2_048,
|
|
225
|
+
),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
210
229
|
export async function waitForWarmSnapshot(
|
|
211
230
|
snapshot: Promise<unknown>,
|
|
212
231
|
timeoutMs: number,
|
|
@@ -239,7 +258,10 @@ export async function waitForWarmSnapshot(
|
|
|
239
258
|
} catch (error) {
|
|
240
259
|
if (signal?.aborted) return false;
|
|
241
260
|
if (error instanceof SnapshotTimeoutError) {
|
|
242
|
-
console.error(
|
|
261
|
+
console.error(
|
|
262
|
+
"mid-session workspace snapshot wait timed out (turn unaffected)",
|
|
263
|
+
safeSnapshotError(error),
|
|
264
|
+
);
|
|
243
265
|
return false;
|
|
244
266
|
}
|
|
245
267
|
return true;
|
|
@@ -521,7 +543,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
521
543
|
} catch (error) {
|
|
522
544
|
// Protection, not a dependency: a failed snapshot must never fail (or slow
|
|
523
545
|
// down retrying) the turn. The next heartbeat/turn-end tick retries.
|
|
524
|
-
console.error(
|
|
546
|
+
console.error(
|
|
547
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
548
|
+
safeSnapshotError(error),
|
|
549
|
+
);
|
|
525
550
|
return false;
|
|
526
551
|
}
|
|
527
552
|
}
|
|
@@ -963,118 +988,6 @@ export async function resumeBoxForTurn(
|
|
|
963
988
|
}
|
|
964
989
|
}
|
|
965
990
|
|
|
966
|
-
/**
|
|
967
|
-
* Stage D machine-primary: acquire the group lease for a turn whose ACTIVE sandbox
|
|
968
|
-
* is a connected machine (selfhosted) WITHOUT establishing a provider box. The
|
|
969
|
-
* "box" is the user's physical machine reached over NATS; there is NOTHING to
|
|
970
|
-
* spawn, snapshot, expose, or serialize — so we take the lease (the SAME group-keyed
|
|
971
|
-
* refcount/idle/epoch bookkeeping resumeBoxForTurn uses) but skip every box step.
|
|
972
|
-
*
|
|
973
|
-
* The lease is keyed backend "selfhosted": the reaper's terminateProviderBox
|
|
974
|
-
* short-circuits a selfhosted lease (it is the user's machine — NEVER provider-stop),
|
|
975
|
-
* draining it to cold with no stop; and the reaper's warm-meter rate for "selfhosted"
|
|
976
|
-
* is 0 (no cloud seconds billed for a box that does not exist). Returns the lease
|
|
977
|
-
* epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
|
|
978
|
-
* identical to the cloud path.
|
|
979
|
-
*
|
|
980
|
-
* holderId is the globally unique durable turn-attempt id. It must not be a
|
|
981
|
-
* Temporal activity id, because activity ids are only workflow-local and
|
|
982
|
-
* collide when sibling sessions share one sandbox group.
|
|
983
|
-
*/
|
|
984
|
-
export async function acquireSelfhostedLeaseForTurn(
|
|
985
|
-
services: SandboxResumeServices,
|
|
986
|
-
ids: {
|
|
987
|
-
accountId: string;
|
|
988
|
-
workspaceId: string;
|
|
989
|
-
sandboxGroupId: string;
|
|
990
|
-
sessionId: string;
|
|
991
|
-
},
|
|
992
|
-
kind: "turn",
|
|
993
|
-
holderId: TurnSandboxLeaseHolderId,
|
|
994
|
-
): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
|
|
995
|
-
const { db, settings } = services;
|
|
996
|
-
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
997
|
-
|
|
998
|
-
let released = false;
|
|
999
|
-
const release = async (): Promise<void> => {
|
|
1000
|
-
if (released) {
|
|
1001
|
-
return;
|
|
1002
|
-
}
|
|
1003
|
-
released = true;
|
|
1004
|
-
await releaseLeaseHolder(db, {
|
|
1005
|
-
accountId: ids.accountId,
|
|
1006
|
-
workspaceId: ids.workspaceId,
|
|
1007
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1008
|
-
kind,
|
|
1009
|
-
holderId,
|
|
1010
|
-
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1011
|
-
});
|
|
1012
|
-
};
|
|
1013
|
-
|
|
1014
|
-
const acquired = await acquireLease(db, {
|
|
1015
|
-
accountId: ids.accountId,
|
|
1016
|
-
workspaceId: ids.workspaceId,
|
|
1017
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1018
|
-
kind,
|
|
1019
|
-
holderId,
|
|
1020
|
-
subjectId: ids.sessionId,
|
|
1021
|
-
backend: "selfhosted",
|
|
1022
|
-
os: "linux",
|
|
1023
|
-
leaseTtlMs,
|
|
1024
|
-
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
|
|
1025
|
-
});
|
|
1026
|
-
|
|
1027
|
-
// FENCED: a newer epoch re-established the group concurrently. Release our
|
|
1028
|
-
// just-registered holder + surface the supersession (the outer turn catch
|
|
1029
|
-
// requeues, mirroring resumeBoxForTurn).
|
|
1030
|
-
if (acquired.role === "blocked") {
|
|
1031
|
-
await release();
|
|
1032
|
-
throw new SandboxLeaseRecoveryBlockedError(
|
|
1033
|
-
ids.sandboxGroupId,
|
|
1034
|
-
acquired.lease.leaseEpoch,
|
|
1035
|
-
acquired.code,
|
|
1036
|
-
acquired.lease.recovery,
|
|
1037
|
-
);
|
|
1038
|
-
}
|
|
1039
|
-
if (acquired.role === "fenced") {
|
|
1040
|
-
await release();
|
|
1041
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
// SPAWNER: we won the cold->warming CAS. There is NO box to establish — commit
|
|
1045
|
-
// warm immediately so the lease does not linger in 'warming' (no box id, no
|
|
1046
|
-
// resume_state; selfhosted is re-addressed by enrollment via the active pointer,
|
|
1047
|
-
// never resumed from an envelope). resume_backend_id "selfhosted" keeps the
|
|
1048
|
-
// reaper's never-provider-stop short-circuit correct.
|
|
1049
|
-
if (acquired.role === "spawner") {
|
|
1050
|
-
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1051
|
-
const committed = await commitWarmingToWarm(db, {
|
|
1052
|
-
accountId: ids.accountId,
|
|
1053
|
-
workspaceId: ids.workspaceId,
|
|
1054
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1055
|
-
expectedEpoch,
|
|
1056
|
-
// No provider box id; stamp a clearly non-box marker for diagnostics.
|
|
1057
|
-
instanceId: `selfhosted:${ids.sessionId}`,
|
|
1058
|
-
resumeBackendId: "selfhosted",
|
|
1059
|
-
resumeState: null,
|
|
1060
|
-
leaseTtlMs,
|
|
1061
|
-
});
|
|
1062
|
-
if (!committed.committed || !committed.lease) {
|
|
1063
|
-
// A reaper reset our warming row, or a sibling re-established + bumped the
|
|
1064
|
-
// epoch. Release our holder; surface supersession (NEVER touch any box).
|
|
1065
|
-
await release();
|
|
1066
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1067
|
-
}
|
|
1068
|
-
return { leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
// ATTACHED / REARMED: the lease is live. There is no box to resume — just carry
|
|
1072
|
-
// the lease epoch (same-session turns are serialized, so a concurrent 'warming'
|
|
1073
|
-
// spawner on this group is not expected; the machine session is independent of the
|
|
1074
|
-
// lease state regardless).
|
|
1075
|
-
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
1076
|
-
}
|
|
1077
|
-
|
|
1078
991
|
/**
|
|
1079
992
|
* Poll a warming lease until the spawner commits warm. If the warming row is
|
|
1080
993
|
* reset to cold (the spawner died and the reaper reset it), surface supersession
|
|
@@ -1104,6 +1017,12 @@ async function waitForWarm(
|
|
|
1104
1017
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1105
1018
|
}
|
|
1106
1019
|
if (lease.liveness === "cold") {
|
|
1020
|
+
if (isRetryableDegradedRestore(lease.recovery.restore)) {
|
|
1021
|
+
// The elected spawner hit a transient archive verification failure.
|
|
1022
|
+
// Re-enter admission so one caller can become the next fenced spawner;
|
|
1023
|
+
// the archive remains authoritative and must not become terminal.
|
|
1024
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1025
|
+
}
|
|
1107
1026
|
if (
|
|
1108
1027
|
lease.recovery.restore.status === "degraded" ||
|
|
1109
1028
|
lease.recovery.restore.status === "unrecoverable"
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -18,7 +18,9 @@ import type { Settings } from "@opengeni/config";
|
|
|
18
18
|
import {
|
|
19
19
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
20
|
advanceWorkspaceGeneration,
|
|
21
|
+
getRetainedProcess,
|
|
21
22
|
retainWorkspaceMutationProcess,
|
|
23
|
+
retainedProcessSettlementIdentity,
|
|
22
24
|
settleRetainedProcess,
|
|
23
25
|
verifyRetainedProcessMutationSettlement,
|
|
24
26
|
verifyWorkspaceMutationSettlement,
|
|
@@ -26,6 +28,7 @@ import {
|
|
|
26
28
|
markWarmLeaseInstanceLost,
|
|
27
29
|
readLease,
|
|
28
30
|
readActiveSandbox,
|
|
31
|
+
SandboxRetainedProcessPromotionFencedError,
|
|
29
32
|
type Database,
|
|
30
33
|
type SandboxWorkspaceMutationAdmission,
|
|
31
34
|
} from "@opengeni/db";
|
|
@@ -47,6 +50,7 @@ import {
|
|
|
47
50
|
type RoutableBackendSession,
|
|
48
51
|
type RoutableSandbox,
|
|
49
52
|
type ResolvedActiveBackend,
|
|
53
|
+
type RoutingMutationSettlementResult,
|
|
50
54
|
type RoutingRetainedProcess,
|
|
51
55
|
type RoutingRetainedProcessTerminalProof,
|
|
52
56
|
type SelfhostedOpObserver,
|
|
@@ -175,7 +179,10 @@ function selfhostedResolverTimeouts(settings: Settings): {
|
|
|
175
179
|
selfhostedExecTimeoutMs: number;
|
|
176
180
|
} {
|
|
177
181
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
178
|
-
return {
|
|
182
|
+
return {
|
|
183
|
+
selfhostedTimeoutMs: timeoutMs,
|
|
184
|
+
selfhostedExecTimeoutMs: execTimeoutMs,
|
|
185
|
+
};
|
|
179
186
|
}
|
|
180
187
|
|
|
181
188
|
type HomeRouteLeaseIdentity = {
|
|
@@ -232,7 +239,9 @@ function homeRouteRecoveryError(
|
|
|
232
239
|
);
|
|
233
240
|
}
|
|
234
241
|
|
|
235
|
-
function providerIdentityFromResumeState(
|
|
242
|
+
export function providerIdentityFromResumeState(
|
|
243
|
+
resumeState: Record<string, unknown>,
|
|
244
|
+
): string | null {
|
|
236
245
|
const sessionState =
|
|
237
246
|
resumeState.sessionState && typeof resumeState.sessionState === "object"
|
|
238
247
|
? (resumeState.sessionState as Record<string, unknown>)
|
|
@@ -288,7 +297,10 @@ async function resolveCurrentHomeBackend(
|
|
|
288
297
|
// A route epoch can advance without a provider replacement. Reuse the exact
|
|
289
298
|
// established handle only when its identity still equals the durable one.
|
|
290
299
|
if (lease.instanceId === established.instanceId) {
|
|
291
|
-
services.onHomeSandboxRebound?.({
|
|
300
|
+
services.onHomeSandboxRebound?.({
|
|
301
|
+
established,
|
|
302
|
+
leaseEpoch: lease.leaseEpoch,
|
|
303
|
+
});
|
|
292
304
|
return {
|
|
293
305
|
session: established.session as RoutableBackendSession,
|
|
294
306
|
sandboxId: null,
|
|
@@ -357,7 +369,10 @@ async function resolveCurrentHomeBackend(
|
|
|
357
369
|
if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
|
|
358
370
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
359
371
|
}
|
|
360
|
-
services.onHomeSandboxRebound?.({
|
|
372
|
+
services.onHomeSandboxRebound?.({
|
|
373
|
+
established: rebound,
|
|
374
|
+
leaseEpoch: lease.leaseEpoch,
|
|
375
|
+
});
|
|
361
376
|
// Keep the resolver's mutable home reference on the latest verified handle so
|
|
362
377
|
// a later route epoch does not resume the same replacement again. This is only
|
|
363
378
|
// a worker-side handle update; the SDK-facing RoutingSandboxSession remains the
|
|
@@ -423,7 +438,7 @@ function afterPersistableHomeMutation(
|
|
|
423
438
|
outcome: "resolved" | "rejected";
|
|
424
439
|
result?: unknown;
|
|
425
440
|
retainedProcess?: RoutingRetainedProcess;
|
|
426
|
-
}) => Promise<void>)
|
|
441
|
+
}) => Promise<void | RoutingMutationSettlementResult>)
|
|
427
442
|
| undefined {
|
|
428
443
|
const fence = ids.workspaceMutationFence;
|
|
429
444
|
if (!home || !fence) return undefined;
|
|
@@ -442,29 +457,58 @@ function afterPersistableHomeMutation(
|
|
|
442
457
|
}
|
|
443
458
|
const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
|
|
444
459
|
if (outcome === "resolved" && retainedProcess) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
460
|
+
try {
|
|
461
|
+
await retainWorkspaceMutationProcess(services.db, {
|
|
462
|
+
accountId: fence.accountId,
|
|
463
|
+
workspaceId: ids.workspaceId,
|
|
464
|
+
sessionId: ids.sessionId,
|
|
465
|
+
processId: retainedProcess.id,
|
|
466
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
467
|
+
admissionId: exactAdmission.id,
|
|
468
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
469
|
+
operation: op,
|
|
470
|
+
owner: {
|
|
471
|
+
kind: "turn",
|
|
472
|
+
turnId: fence.turnId,
|
|
473
|
+
executionGeneration: fence.executionGeneration,
|
|
474
|
+
attemptId: fence.attemptId,
|
|
475
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
476
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
477
|
+
expectedEpoch: backend.leaseEpoch,
|
|
478
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
479
|
+
routeKind: exactAdmission.routeKind,
|
|
480
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
481
|
+
routeEpoch: exactAdmission.routeEpoch,
|
|
482
|
+
},
|
|
483
|
+
});
|
|
484
|
+
} catch (error) {
|
|
485
|
+
if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
|
|
486
|
+
const durable = error.process;
|
|
487
|
+
if (
|
|
488
|
+
durable.id !== retainedProcess.id ||
|
|
489
|
+
durable.providerSessionId !== retainedProcess.providerSessionId ||
|
|
490
|
+
durable.sessionId !== ids.sessionId ||
|
|
491
|
+
durable.sandboxGroupId !== home.sandboxGroupId ||
|
|
492
|
+
durable.parentAdmissionId !== exactAdmission.id ||
|
|
493
|
+
durable.leaseEpoch !== backend.leaseEpoch ||
|
|
494
|
+
durable.providerInstanceId !== backend.providerInstanceId ||
|
|
495
|
+
durable.routeKind !== exactAdmission.routeKind ||
|
|
496
|
+
durable.routeTargetId !== exactAdmission.routeTargetId ||
|
|
497
|
+
durable.routeEpoch !== exactAdmission.routeEpoch ||
|
|
498
|
+
durable.state !== "active"
|
|
499
|
+
) {
|
|
500
|
+
throw new Error("Durable retained-process promotion returned a mismatched identity", {
|
|
501
|
+
cause: error,
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
return {
|
|
505
|
+
status: "retained_process_durable_output_rejected",
|
|
506
|
+
retainedProcess: {
|
|
507
|
+
id: durable.id,
|
|
508
|
+
providerSessionId: durable.providerSessionId,
|
|
509
|
+
},
|
|
510
|
+
};
|
|
511
|
+
}
|
|
468
512
|
return;
|
|
469
513
|
}
|
|
470
514
|
await verifyWorkspaceMutationSettlement(services.db, {
|
|
@@ -556,12 +600,30 @@ function settleRetainedProcessForTurn(
|
|
|
556
600
|
| undefined {
|
|
557
601
|
const fence = ids.workspaceMutationFence;
|
|
558
602
|
if (!fence) return undefined;
|
|
559
|
-
return async ({ process, proof }) => {
|
|
603
|
+
return async ({ backend, process, proof }) => {
|
|
604
|
+
const durable = await getRetainedProcess(services.db, {
|
|
605
|
+
workspaceId: ids.workspaceId,
|
|
606
|
+
sessionId: ids.sessionId,
|
|
607
|
+
processId: process.id,
|
|
608
|
+
});
|
|
609
|
+
if (
|
|
610
|
+
!durable ||
|
|
611
|
+
durable.providerSessionId !== process.providerSessionId ||
|
|
612
|
+
durable.providerBackend !== backend.kind ||
|
|
613
|
+
durable.providerInstanceId !== backend.providerInstanceId ||
|
|
614
|
+
durable.leaseEpoch !== backend.leaseEpoch ||
|
|
615
|
+
durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
|
|
616
|
+
durable.routeTargetId !== backend.sandboxId ||
|
|
617
|
+
durable.routeEpoch !== backend.activeEpoch
|
|
618
|
+
) {
|
|
619
|
+
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
620
|
+
}
|
|
560
621
|
await settleRetainedProcess(services.db, {
|
|
561
622
|
accountId: fence.accountId,
|
|
562
623
|
workspaceId: ids.workspaceId,
|
|
563
624
|
sessionId: ids.sessionId,
|
|
564
625
|
processId: process.id,
|
|
626
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
565
627
|
outcome: proof.outcome,
|
|
566
628
|
exitCode: proof.exitCode,
|
|
567
629
|
reason: proof.reason,
|
|
@@ -762,7 +824,10 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
762
824
|
backendId: string;
|
|
763
825
|
agentDefaultManifest: unknown;
|
|
764
826
|
provisioner: {
|
|
765
|
-
get(): Promise<{
|
|
827
|
+
get(): Promise<{
|
|
828
|
+
established: EstablishedSandboxSession;
|
|
829
|
+
leaseEpoch?: number;
|
|
830
|
+
}>;
|
|
766
831
|
};
|
|
767
832
|
homeLeaseIdentity?: {
|
|
768
833
|
accountId: string;
|