@opengeni/worker-bundle 0.10.8 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1484 -481
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +5 -4
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +484 -169
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +130 -83
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
package/src/sandbox-routing.ts
CHANGED
|
@@ -15,24 +15,46 @@
|
|
|
15
15
|
// request/reply connection.
|
|
16
16
|
|
|
17
17
|
import type { Settings } from "@opengeni/config";
|
|
18
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
|
+
advanceWorkspaceGeneration,
|
|
21
|
+
retainWorkspaceMutationProcess,
|
|
22
|
+
settleRetainedProcess,
|
|
23
|
+
verifyRetainedProcessMutationSettlement,
|
|
24
|
+
verifyWorkspaceMutationSettlement,
|
|
25
|
+
getSandbox,
|
|
26
|
+
markWarmLeaseInstanceLost,
|
|
27
|
+
readLease,
|
|
28
|
+
readActiveSandbox,
|
|
29
|
+
type Database,
|
|
30
|
+
type SandboxWorkspaceMutationAdmission,
|
|
31
|
+
} from "@opengeni/db";
|
|
19
32
|
import type { EventBus } from "@opengeni/events";
|
|
20
33
|
import {
|
|
21
34
|
buildSelfhostedBackendSession,
|
|
35
|
+
establishSandboxSessionFromEnvelope,
|
|
36
|
+
isProviderSandboxGoneDuringRoutedOperation,
|
|
37
|
+
isProviderSandboxNotFoundError,
|
|
22
38
|
makeActiveBackendResolver,
|
|
23
39
|
NatsControlRpc,
|
|
24
40
|
NatsOpStreamTransport,
|
|
41
|
+
RoutingBackendRecoveryRequiredError,
|
|
25
42
|
RoutingSandboxSession,
|
|
43
|
+
verifySandboxExecReadiness,
|
|
26
44
|
type ControlRpc,
|
|
27
45
|
type EstablishedSandboxSession,
|
|
28
46
|
type NatsRequestConnection,
|
|
29
47
|
type RoutableBackendSession,
|
|
30
48
|
type RoutableSandbox,
|
|
49
|
+
type ResolvedActiveBackend,
|
|
50
|
+
type RoutingRetainedProcess,
|
|
51
|
+
type RoutingRetainedProcessTerminalProof,
|
|
31
52
|
type SelfhostedOpObserver,
|
|
32
53
|
type SelfhostedRelayConfig,
|
|
33
54
|
type OpStreamJournal,
|
|
34
55
|
type SelfhostedOpStreamDeps,
|
|
35
56
|
} from "@opengeni/runtime";
|
|
57
|
+
import { sandboxLeaseHolderIdForAttempt } from "./sandbox-resume";
|
|
36
58
|
|
|
37
59
|
export type RoutingWiringServices = {
|
|
38
60
|
db: Database;
|
|
@@ -49,11 +71,38 @@ export type RoutingWiringServices = {
|
|
|
49
71
|
* the runtime defaults (generation "1", no persistence) — tests / non-turn
|
|
50
72
|
* callers. Only consulted when op-stream is actually enabled for the turn. */
|
|
51
73
|
opJournal?: OpStreamJournal;
|
|
74
|
+
/** Durable lifecycle notification emitted only by the observer that wins the
|
|
75
|
+
* exact warm-instance loss CAS. */
|
|
76
|
+
onHomeSandboxLost?: (input: {
|
|
77
|
+
sandboxGroupId: string;
|
|
78
|
+
instanceId: string;
|
|
79
|
+
leaseEpoch: number;
|
|
80
|
+
}) => Promise<void>;
|
|
81
|
+
/** Called when a route repair replaces the worker's original home handle. */
|
|
82
|
+
onHomeSandboxRebound?: (input: {
|
|
83
|
+
established: EstablishedSandboxSession;
|
|
84
|
+
leaseEpoch: number;
|
|
85
|
+
}) => void;
|
|
52
86
|
};
|
|
53
87
|
|
|
54
88
|
export type RoutingWiringIds = {
|
|
55
89
|
workspaceId: string;
|
|
56
90
|
sessionId: string;
|
|
91
|
+
/** Canonical turn-attempt fence used to admit persistable-home mutations.
|
|
92
|
+
* Omit outside a running worker attempt; mutation hooks then remain disabled. */
|
|
93
|
+
workspaceMutationFence?: {
|
|
94
|
+
accountId: string;
|
|
95
|
+
turnId: string;
|
|
96
|
+
executionGeneration: number;
|
|
97
|
+
attemptId: string;
|
|
98
|
+
};
|
|
99
|
+
homeLease?: {
|
|
100
|
+
accountId: string;
|
|
101
|
+
sandboxGroupId: string;
|
|
102
|
+
leaseEpoch: number;
|
|
103
|
+
instanceId: string;
|
|
104
|
+
backend: string;
|
|
105
|
+
};
|
|
57
106
|
/**
|
|
58
107
|
* The run's declared sandbox environment — the SAME object the turn passes to
|
|
59
108
|
* `runtime.buildAgent`'s `sandboxEnvironment` and to `resumeBoxForTurn` (so the
|
|
@@ -129,6 +178,398 @@ function selfhostedResolverTimeouts(settings: Settings): {
|
|
|
129
178
|
return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
|
|
130
179
|
}
|
|
131
180
|
|
|
181
|
+
type HomeRouteLeaseIdentity = {
|
|
182
|
+
accountId: string;
|
|
183
|
+
sandboxGroupId: string;
|
|
184
|
+
backend: string;
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
type HomeRouteResolutionIds = HomeRouteLeaseIdentity & {
|
|
188
|
+
workspaceId: string;
|
|
189
|
+
sessionId: string;
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
type HomeRouteRecovery = "pending" | "degraded" | "unrecoverable" | "superseded";
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Translate the durable lease/recovery state into the typed disposition the
|
|
196
|
+
* routing proxy exposes when the home cannot be safely resumed. A route repair
|
|
197
|
+
* must never fall back to the old in-memory provider handle when the lease is
|
|
198
|
+
* warming, restoring, unverifiable, or otherwise inconsistent.
|
|
199
|
+
*/
|
|
200
|
+
function homeRouteRecoveryDisposition(
|
|
201
|
+
lease: Awaited<ReturnType<typeof readLease>>,
|
|
202
|
+
): HomeRouteRecovery {
|
|
203
|
+
if (!lease) return "unrecoverable";
|
|
204
|
+
if (lease.liveness === "warming") return "pending";
|
|
205
|
+
if (lease.liveness === "draining") return "superseded";
|
|
206
|
+
if (
|
|
207
|
+
lease.recovery.restore.status === "pending" ||
|
|
208
|
+
lease.recovery.restore.status === "restoring" ||
|
|
209
|
+
lease.recovery.restore.status === "verifying"
|
|
210
|
+
) {
|
|
211
|
+
return "pending";
|
|
212
|
+
}
|
|
213
|
+
if (
|
|
214
|
+
lease.recovery.restore.status === "degraded" ||
|
|
215
|
+
lease.recovery.workspace.status === "degraded" ||
|
|
216
|
+
lease.recovery.archive.status === "unverified" ||
|
|
217
|
+
lease.recovery.archive.status === "invalid"
|
|
218
|
+
) {
|
|
219
|
+
return "degraded";
|
|
220
|
+
}
|
|
221
|
+
return "unrecoverable";
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function homeRouteRecoveryError(
|
|
225
|
+
lease: Awaited<ReturnType<typeof readLease>>,
|
|
226
|
+
fallbackEpoch: number,
|
|
227
|
+
): RoutingBackendRecoveryRequiredError {
|
|
228
|
+
return new RoutingBackendRecoveryRequiredError(
|
|
229
|
+
"resolve_home_backend",
|
|
230
|
+
lease?.leaseEpoch ?? fallbackEpoch,
|
|
231
|
+
homeRouteRecoveryDisposition(lease),
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function providerIdentityFromResumeState(resumeState: Record<string, unknown>): string | null {
|
|
236
|
+
const sessionState =
|
|
237
|
+
resumeState.sessionState && typeof resumeState.sessionState === "object"
|
|
238
|
+
? (resumeState.sessionState as Record<string, unknown>)
|
|
239
|
+
: null;
|
|
240
|
+
const providerState =
|
|
241
|
+
sessionState?.providerState && typeof sessionState.providerState === "object"
|
|
242
|
+
? (sessionState.providerState as Record<string, unknown>)
|
|
243
|
+
: null;
|
|
244
|
+
for (const field of [
|
|
245
|
+
"sandboxId",
|
|
246
|
+
"instanceId",
|
|
247
|
+
"id",
|
|
248
|
+
"hostId",
|
|
249
|
+
"containerId",
|
|
250
|
+
"workspaceRootPath",
|
|
251
|
+
"agentId",
|
|
252
|
+
]) {
|
|
253
|
+
const value = providerState?.[field];
|
|
254
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
255
|
+
}
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Resolve the current durable home identity after the route epoch changes.
|
|
261
|
+
*
|
|
262
|
+
* Same-target attach/repair can replace the provider instance while a worker
|
|
263
|
+
* turn still holds the stable routing proxy. The proxy must therefore resume
|
|
264
|
+
* the lease's CURRENT identity, not reuse the object captured at turn start.
|
|
265
|
+
* This seam is deliberately resume-only: lease election/rematerialization is
|
|
266
|
+
* owned by the API/lease state machine, and a routed tool call must never create
|
|
267
|
+
* a rival provider or replay an ambiguous mutation.
|
|
268
|
+
*/
|
|
269
|
+
async function resolveCurrentHomeBackend(
|
|
270
|
+
services: RoutingWiringServices,
|
|
271
|
+
ids: HomeRouteResolutionIds,
|
|
272
|
+
established: EstablishedSandboxSession,
|
|
273
|
+
): Promise<ResolvedActiveBackend> {
|
|
274
|
+
const lease = await readLease(services.db, ids.workspaceId, ids.sandboxGroupId);
|
|
275
|
+
const fallbackEpoch = lease?.leaseEpoch ?? 0;
|
|
276
|
+
if (
|
|
277
|
+
!lease ||
|
|
278
|
+
lease.liveness !== "warm" ||
|
|
279
|
+
lease.recovery.provider.status !== "exists" ||
|
|
280
|
+
lease.instanceId === null ||
|
|
281
|
+
lease.recovery.provider.instanceId !== lease.instanceId ||
|
|
282
|
+
lease.recovery.workspace.status !== "ready" ||
|
|
283
|
+
(lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready")
|
|
284
|
+
) {
|
|
285
|
+
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// A route epoch can advance without a provider replacement. Reuse the exact
|
|
289
|
+
// established handle only when its identity still equals the durable one.
|
|
290
|
+
if (lease.instanceId === established.instanceId) {
|
|
291
|
+
services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
|
|
292
|
+
return {
|
|
293
|
+
session: established.session as RoutableBackendSession,
|
|
294
|
+
sandboxId: null,
|
|
295
|
+
kind: established.backendId,
|
|
296
|
+
leaseEpoch: lease.leaseEpoch,
|
|
297
|
+
providerInstanceId: lease.instanceId,
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const resumeState = lease.resumeState;
|
|
302
|
+
const resumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
303
|
+
if (
|
|
304
|
+
!resumeState ||
|
|
305
|
+
resumeBackend !== ids.backend ||
|
|
306
|
+
providerIdentityFromResumeState(resumeState) !== lease.instanceId
|
|
307
|
+
) {
|
|
308
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
let rebound: EstablishedSandboxSession;
|
|
312
|
+
try {
|
|
313
|
+
rebound = await establishSandboxSessionFromEnvelope(services.settings, resumeState, {
|
|
314
|
+
sessionId: ids.sessionId,
|
|
315
|
+
recovery: "resume-only",
|
|
316
|
+
backendOverride: resumeBackend as never,
|
|
317
|
+
});
|
|
318
|
+
// A resumed handle is not enough evidence for the route to use it. Keep the
|
|
319
|
+
// same bounded readiness gate used before publishing a Modal lease warm.
|
|
320
|
+
await verifySandboxExecReadiness(rebound);
|
|
321
|
+
} catch (error) {
|
|
322
|
+
if (!isProviderSandboxNotFoundError(resumeBackend, error)) {
|
|
323
|
+
// The durable identity is present, but a transient resume/provider error
|
|
324
|
+
// did not prove the box gone. Keep the lease untouched and make the next
|
|
325
|
+
// independent route operation retry the resume; never use the old handle.
|
|
326
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
327
|
+
"resolve_home_backend",
|
|
328
|
+
lease.leaseEpoch,
|
|
329
|
+
"pending",
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
const marked = await markWarmLeaseInstanceLost(services.db, {
|
|
333
|
+
accountId: ids.accountId,
|
|
334
|
+
workspaceId: ids.workspaceId,
|
|
335
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
336
|
+
expectedEpoch: lease.leaseEpoch,
|
|
337
|
+
expectedInstanceId: lease.instanceId,
|
|
338
|
+
diagnostic: "provider_not_found_during_home_route_rebind",
|
|
339
|
+
});
|
|
340
|
+
if (marked.status === "marked") {
|
|
341
|
+
await services.onHomeSandboxLost?.({
|
|
342
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
343
|
+
instanceId: lease.instanceId,
|
|
344
|
+
leaseEpoch: marked.lease.leaseEpoch,
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
348
|
+
"resolve_home_backend",
|
|
349
|
+
marked.lease?.leaseEpoch ?? lease.leaseEpoch,
|
|
350
|
+
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease),
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// The provider identity must agree with the exact durable lease row before
|
|
355
|
+
// any caller can publish or route through this rebound handle. A mismatch is
|
|
356
|
+
// unverifiable local state, not permission to try the old handle.
|
|
357
|
+
if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
|
|
358
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
359
|
+
}
|
|
360
|
+
services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
|
|
361
|
+
// Keep the resolver's mutable home reference on the latest verified handle so
|
|
362
|
+
// a later route epoch does not resume the same replacement again. This is only
|
|
363
|
+
// a worker-side handle update; the SDK-facing RoutingSandboxSession remains the
|
|
364
|
+
// stable object returned below and is never replaced.
|
|
365
|
+
Object.assign(established, rebound);
|
|
366
|
+
return {
|
|
367
|
+
session: rebound.session as RoutableBackendSession,
|
|
368
|
+
sandboxId: null,
|
|
369
|
+
kind: rebound.backendId,
|
|
370
|
+
leaseEpoch: lease.leaseEpoch,
|
|
371
|
+
providerInstanceId: lease.instanceId,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function beforePersistableHomeMutation(
|
|
376
|
+
services: RoutingWiringServices,
|
|
377
|
+
ids: RoutingWiringIds,
|
|
378
|
+
home: { accountId: string; sandboxGroupId: string } | undefined,
|
|
379
|
+
):
|
|
380
|
+
| ((input: {
|
|
381
|
+
op: string;
|
|
382
|
+
backend: ResolvedActiveBackend;
|
|
383
|
+
}) => Promise<SandboxWorkspaceMutationAdmission | null>)
|
|
384
|
+
| undefined {
|
|
385
|
+
const fence = ids.workspaceMutationFence;
|
|
386
|
+
if (!home || !fence) return undefined;
|
|
387
|
+
return async ({ op, backend }) => {
|
|
388
|
+
// A connected-machine target is intentionally non-persistable: its writes
|
|
389
|
+
// do not dirty the session home's archive. Exact home identity metadata is
|
|
390
|
+
// present only on a verified durable home route.
|
|
391
|
+
if (
|
|
392
|
+
backend.sandboxId !== null ||
|
|
393
|
+
backend.leaseEpoch === undefined ||
|
|
394
|
+
backend.providerInstanceId === undefined
|
|
395
|
+
) {
|
|
396
|
+
return null;
|
|
397
|
+
}
|
|
398
|
+
return await advanceWorkspaceGeneration(services.db, {
|
|
399
|
+
accountId: fence.accountId,
|
|
400
|
+
workspaceId: ids.workspaceId,
|
|
401
|
+
sessionId: ids.sessionId,
|
|
402
|
+
turnId: fence.turnId,
|
|
403
|
+
executionGeneration: fence.executionGeneration,
|
|
404
|
+
attemptId: fence.attemptId,
|
|
405
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
406
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
407
|
+
expectedEpoch: backend.leaseEpoch,
|
|
408
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
409
|
+
operation: op,
|
|
410
|
+
});
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function afterPersistableHomeMutation(
|
|
415
|
+
services: RoutingWiringServices,
|
|
416
|
+
ids: RoutingWiringIds,
|
|
417
|
+
home: { accountId: string; sandboxGroupId: string } | undefined,
|
|
418
|
+
):
|
|
419
|
+
| ((input: {
|
|
420
|
+
op: string;
|
|
421
|
+
backend: ResolvedActiveBackend;
|
|
422
|
+
admission: unknown;
|
|
423
|
+
outcome: "resolved" | "rejected";
|
|
424
|
+
result?: unknown;
|
|
425
|
+
retainedProcess?: RoutingRetainedProcess;
|
|
426
|
+
}) => Promise<void>)
|
|
427
|
+
| undefined {
|
|
428
|
+
const fence = ids.workspaceMutationFence;
|
|
429
|
+
if (!home || !fence) return undefined;
|
|
430
|
+
return async ({ op, backend, admission, outcome, retainedProcess }) => {
|
|
431
|
+
if (
|
|
432
|
+
backend.sandboxId !== null ||
|
|
433
|
+
backend.leaseEpoch === undefined ||
|
|
434
|
+
backend.providerInstanceId === undefined ||
|
|
435
|
+
!admission ||
|
|
436
|
+
typeof admission !== "object" ||
|
|
437
|
+
typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
|
|
438
|
+
typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
|
|
439
|
+
"number"
|
|
440
|
+
) {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
|
|
444
|
+
if (outcome === "resolved" && retainedProcess) {
|
|
445
|
+
await retainWorkspaceMutationProcess(services.db, {
|
|
446
|
+
accountId: fence.accountId,
|
|
447
|
+
workspaceId: ids.workspaceId,
|
|
448
|
+
sessionId: ids.sessionId,
|
|
449
|
+
processId: retainedProcess.id,
|
|
450
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
451
|
+
admissionId: exactAdmission.id,
|
|
452
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
453
|
+
operation: op,
|
|
454
|
+
owner: {
|
|
455
|
+
kind: "turn",
|
|
456
|
+
turnId: fence.turnId,
|
|
457
|
+
executionGeneration: fence.executionGeneration,
|
|
458
|
+
attemptId: fence.attemptId,
|
|
459
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
460
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
461
|
+
expectedEpoch: backend.leaseEpoch,
|
|
462
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
463
|
+
routeKind: exactAdmission.routeKind,
|
|
464
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
465
|
+
routeEpoch: exactAdmission.routeEpoch,
|
|
466
|
+
},
|
|
467
|
+
});
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
await verifyWorkspaceMutationSettlement(services.db, {
|
|
471
|
+
accountId: fence.accountId,
|
|
472
|
+
workspaceId: ids.workspaceId,
|
|
473
|
+
sessionId: ids.sessionId,
|
|
474
|
+
turnId: fence.turnId,
|
|
475
|
+
executionGeneration: fence.executionGeneration,
|
|
476
|
+
attemptId: fence.attemptId,
|
|
477
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
478
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
479
|
+
expectedEpoch: backend.leaseEpoch,
|
|
480
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
481
|
+
admission: exactAdmission,
|
|
482
|
+
operation: op,
|
|
483
|
+
outcome,
|
|
484
|
+
});
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function beforeRetainedProcessMutation(
|
|
489
|
+
services: RoutingWiringServices,
|
|
490
|
+
ids: RoutingWiringIds,
|
|
491
|
+
):
|
|
492
|
+
| ((input: {
|
|
493
|
+
op: string;
|
|
494
|
+
backend: ResolvedActiveBackend;
|
|
495
|
+
process: RoutingRetainedProcess;
|
|
496
|
+
}) => Promise<SandboxWorkspaceMutationAdmission>)
|
|
497
|
+
| undefined {
|
|
498
|
+
const fence = ids.workspaceMutationFence;
|
|
499
|
+
if (!fence) return undefined;
|
|
500
|
+
return async ({ op, process }) =>
|
|
501
|
+
await advanceWorkspaceGenerationForRetainedProcess(services.db, {
|
|
502
|
+
accountId: fence.accountId,
|
|
503
|
+
workspaceId: ids.workspaceId,
|
|
504
|
+
sessionId: ids.sessionId,
|
|
505
|
+
processId: process.id,
|
|
506
|
+
operation: op,
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function afterRetainedProcessMutation(
|
|
511
|
+
services: RoutingWiringServices,
|
|
512
|
+
ids: RoutingWiringIds,
|
|
513
|
+
):
|
|
514
|
+
| ((input: {
|
|
515
|
+
op: string;
|
|
516
|
+
backend: ResolvedActiveBackend;
|
|
517
|
+
process: RoutingRetainedProcess;
|
|
518
|
+
admission: unknown;
|
|
519
|
+
outcome: "resolved" | "rejected";
|
|
520
|
+
result?: unknown;
|
|
521
|
+
}) => Promise<void>)
|
|
522
|
+
| undefined {
|
|
523
|
+
const fence = ids.workspaceMutationFence;
|
|
524
|
+
if (!fence) return undefined;
|
|
525
|
+
return async ({ op, process, admission, outcome }) => {
|
|
526
|
+
if (
|
|
527
|
+
!admission ||
|
|
528
|
+
typeof admission !== "object" ||
|
|
529
|
+
typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
|
|
530
|
+
typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
|
|
531
|
+
"number"
|
|
532
|
+
) {
|
|
533
|
+
throw new Error("Retained-process mutation settlement lacked its exact admission");
|
|
534
|
+
}
|
|
535
|
+
await verifyRetainedProcessMutationSettlement(services.db, {
|
|
536
|
+
accountId: fence.accountId,
|
|
537
|
+
workspaceId: ids.workspaceId,
|
|
538
|
+
sessionId: ids.sessionId,
|
|
539
|
+
processId: process.id,
|
|
540
|
+
admission: admission as SandboxWorkspaceMutationAdmission,
|
|
541
|
+
operation: op,
|
|
542
|
+
outcome,
|
|
543
|
+
});
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function settleRetainedProcessForTurn(
|
|
548
|
+
services: RoutingWiringServices,
|
|
549
|
+
ids: RoutingWiringIds,
|
|
550
|
+
):
|
|
551
|
+
| ((input: {
|
|
552
|
+
backend: ResolvedActiveBackend;
|
|
553
|
+
process: RoutingRetainedProcess;
|
|
554
|
+
proof: RoutingRetainedProcessTerminalProof;
|
|
555
|
+
}) => Promise<void>)
|
|
556
|
+
| undefined {
|
|
557
|
+
const fence = ids.workspaceMutationFence;
|
|
558
|
+
if (!fence) return undefined;
|
|
559
|
+
return async ({ process, proof }) => {
|
|
560
|
+
await settleRetainedProcess(services.db, {
|
|
561
|
+
accountId: fence.accountId,
|
|
562
|
+
workspaceId: ids.workspaceId,
|
|
563
|
+
sessionId: ids.sessionId,
|
|
564
|
+
processId: process.id,
|
|
565
|
+
outcome: proof.outcome,
|
|
566
|
+
exitCode: proof.exitCode,
|
|
567
|
+
reason: proof.reason,
|
|
568
|
+
idleGraceMs: services.settings.sandboxIdleGraceMs,
|
|
569
|
+
});
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
|
|
132
573
|
/** Build the selfhosted `ControlRpc` over the events bus's request/reply
|
|
133
574
|
* connection. A null bus / unconfigured NATS yields a NatsControlRpc whose
|
|
134
575
|
* connection factory returns null → agent_offline on every op (never a throw). */
|
|
@@ -160,6 +601,11 @@ export function wrapTurnBoxWithRouting(
|
|
|
160
601
|
established: EstablishedSandboxSession,
|
|
161
602
|
): EstablishedSandboxSession {
|
|
162
603
|
const { db, settings, bus, onOp } = services;
|
|
604
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
|
|
605
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
|
|
606
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
607
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
608
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
163
609
|
const resolver = makeActiveBackendResolver({
|
|
164
610
|
workspaceId: ids.workspaceId,
|
|
165
611
|
defaultBackend: established.session as RoutableBackendSession,
|
|
@@ -210,6 +656,22 @@ export function wrapTurnBoxWithRouting(
|
|
|
210
656
|
// the pinned machine — passing defaultIsHome:false makes the null branch throw typed
|
|
211
657
|
// `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
|
|
212
658
|
...(ids.defaultIsHome !== undefined ? { defaultIsHome: ids.defaultIsHome } : {}),
|
|
659
|
+
...(ids.homeLease
|
|
660
|
+
? {
|
|
661
|
+
resolveDefaultBackend: () =>
|
|
662
|
+
resolveCurrentHomeBackend(
|
|
663
|
+
services,
|
|
664
|
+
{
|
|
665
|
+
workspaceId: ids.workspaceId,
|
|
666
|
+
sessionId: ids.sessionId,
|
|
667
|
+
accountId: ids.homeLease!.accountId,
|
|
668
|
+
sandboxGroupId: ids.homeLease!.sandboxGroupId,
|
|
669
|
+
backend: ids.homeLease!.backend,
|
|
670
|
+
},
|
|
671
|
+
established,
|
|
672
|
+
),
|
|
673
|
+
}
|
|
674
|
+
: {}),
|
|
213
675
|
});
|
|
214
676
|
|
|
215
677
|
const proxy = new RoutingSandboxSession({
|
|
@@ -223,12 +685,70 @@ export function wrapTurnBoxWithRouting(
|
|
|
223
685
|
session: established.session as RoutableBackendSession,
|
|
224
686
|
sandboxId: null,
|
|
225
687
|
kind: established.backendId,
|
|
688
|
+
...(ids.homeLease
|
|
689
|
+
? {
|
|
690
|
+
leaseEpoch: ids.homeLease.leaseEpoch,
|
|
691
|
+
providerInstanceId: ids.homeLease.instanceId,
|
|
692
|
+
}
|
|
693
|
+
: {}),
|
|
226
694
|
},
|
|
227
695
|
readPointer: async () => {
|
|
696
|
+
if (!routingEnabled(settings)) {
|
|
697
|
+
return { activeSandboxId: null, activeEpoch: 0 };
|
|
698
|
+
}
|
|
228
699
|
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
229
700
|
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
230
701
|
},
|
|
231
702
|
resolveActiveBackend: resolver,
|
|
703
|
+
...(beforeMutation ? { beforeMutation } : {}),
|
|
704
|
+
...(afterMutation ? { afterMutation } : {}),
|
|
705
|
+
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
706
|
+
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
707
|
+
...(settleProcess ? { settleProcess } : {}),
|
|
708
|
+
...(ids.homeLease
|
|
709
|
+
? {
|
|
710
|
+
onDefaultBackendError: async ({
|
|
711
|
+
error,
|
|
712
|
+
backend,
|
|
713
|
+
}: {
|
|
714
|
+
error: unknown;
|
|
715
|
+
backend: ResolvedActiveBackend;
|
|
716
|
+
}) => {
|
|
717
|
+
const home = ids.homeLease!;
|
|
718
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
719
|
+
const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
|
|
720
|
+
const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
|
|
721
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
722
|
+
accountId: home.accountId,
|
|
723
|
+
workspaceId: ids.workspaceId,
|
|
724
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
725
|
+
expectedEpoch,
|
|
726
|
+
expectedInstanceId,
|
|
727
|
+
diagnostic: "provider_not_found_during_routed_operation",
|
|
728
|
+
});
|
|
729
|
+
if (marked.status === "marked") {
|
|
730
|
+
await services.onHomeSandboxLost?.({
|
|
731
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
732
|
+
instanceId: expectedInstanceId,
|
|
733
|
+
leaseEpoch: marked.lease.leaseEpoch,
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
const lease = marked.lease;
|
|
737
|
+
const restore = lease?.recovery.restore.status;
|
|
738
|
+
return {
|
|
739
|
+
leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
|
|
740
|
+
recovery:
|
|
741
|
+
marked.status === "stale"
|
|
742
|
+
? ("superseded" as const)
|
|
743
|
+
: restore === "pending"
|
|
744
|
+
? ("pending" as const)
|
|
745
|
+
: restore === "degraded"
|
|
746
|
+
? ("degraded" as const)
|
|
747
|
+
: ("unrecoverable" as const),
|
|
748
|
+
};
|
|
749
|
+
},
|
|
750
|
+
}
|
|
751
|
+
: {}),
|
|
232
752
|
});
|
|
233
753
|
|
|
234
754
|
return { ...established, session: proxy };
|
|
@@ -241,10 +761,22 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
241
761
|
client: EstablishedSandboxSession["client"];
|
|
242
762
|
backendId: string;
|
|
243
763
|
agentDefaultManifest: unknown;
|
|
244
|
-
provisioner: {
|
|
764
|
+
provisioner: {
|
|
765
|
+
get(): Promise<{ established: EstablishedSandboxSession; leaseEpoch?: number }>;
|
|
766
|
+
};
|
|
767
|
+
homeLeaseIdentity?: {
|
|
768
|
+
accountId: string;
|
|
769
|
+
sandboxGroupId: string;
|
|
770
|
+
backend: string;
|
|
771
|
+
};
|
|
245
772
|
},
|
|
246
773
|
): EstablishedSandboxSession {
|
|
247
774
|
const { db, settings, bus, onOp } = services;
|
|
775
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
776
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
777
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
778
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
779
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
248
780
|
const syntheticSession: RoutableBackendSession = {
|
|
249
781
|
state: { manifest: args.agentDefaultManifest },
|
|
250
782
|
};
|
|
@@ -287,6 +819,19 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
287
819
|
resolveActiveBackend: async (pointer) => {
|
|
288
820
|
if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
|
|
289
821
|
const provisioned = await args.provisioner.get();
|
|
822
|
+
if (args.homeLeaseIdentity && provisioned.leaseEpoch !== undefined) {
|
|
823
|
+
return resolveCurrentHomeBackend(
|
|
824
|
+
services,
|
|
825
|
+
{
|
|
826
|
+
workspaceId: ids.workspaceId,
|
|
827
|
+
sessionId: ids.sessionId,
|
|
828
|
+
accountId: args.homeLeaseIdentity.accountId,
|
|
829
|
+
sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
|
|
830
|
+
backend: args.homeLeaseIdentity.backend,
|
|
831
|
+
},
|
|
832
|
+
provisioned.established,
|
|
833
|
+
);
|
|
834
|
+
}
|
|
290
835
|
return {
|
|
291
836
|
session: provisioned.established.session as RoutableBackendSession,
|
|
292
837
|
sandboxId: null,
|
|
@@ -295,6 +840,56 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
295
840
|
}
|
|
296
841
|
return routedResolver(pointer);
|
|
297
842
|
},
|
|
843
|
+
...(beforeMutation ? { beforeMutation } : {}),
|
|
844
|
+
...(afterMutation ? { afterMutation } : {}),
|
|
845
|
+
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
846
|
+
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
847
|
+
...(settleProcess ? { settleProcess } : {}),
|
|
848
|
+
...(args.homeLeaseIdentity
|
|
849
|
+
? {
|
|
850
|
+
onDefaultBackendError: async ({
|
|
851
|
+
error,
|
|
852
|
+
backend,
|
|
853
|
+
}: {
|
|
854
|
+
error: unknown;
|
|
855
|
+
backend: ResolvedActiveBackend;
|
|
856
|
+
}) => {
|
|
857
|
+
const home = args.homeLeaseIdentity!;
|
|
858
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
859
|
+
if (backend.leaseEpoch === undefined || backend.providerInstanceId === undefined) {
|
|
860
|
+
return null;
|
|
861
|
+
}
|
|
862
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
863
|
+
accountId: home.accountId,
|
|
864
|
+
workspaceId: ids.workspaceId,
|
|
865
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
866
|
+
expectedEpoch: backend.leaseEpoch,
|
|
867
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
868
|
+
diagnostic: "provider_not_found_during_routed_operation",
|
|
869
|
+
});
|
|
870
|
+
if (marked.status === "marked") {
|
|
871
|
+
await services.onHomeSandboxLost?.({
|
|
872
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
873
|
+
instanceId: backend.providerInstanceId,
|
|
874
|
+
leaseEpoch: marked.lease.leaseEpoch,
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
const lease = marked.lease;
|
|
878
|
+
const restore = lease?.recovery.restore.status;
|
|
879
|
+
return {
|
|
880
|
+
leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
|
|
881
|
+
recovery:
|
|
882
|
+
marked.status === "stale"
|
|
883
|
+
? ("superseded" as const)
|
|
884
|
+
: restore === "pending"
|
|
885
|
+
? ("pending" as const)
|
|
886
|
+
: restore === "degraded"
|
|
887
|
+
? ("degraded" as const)
|
|
888
|
+
: ("unrecoverable" as const),
|
|
889
|
+
};
|
|
890
|
+
},
|
|
891
|
+
}
|
|
892
|
+
: {}),
|
|
298
893
|
});
|
|
299
894
|
|
|
300
895
|
return {
|
|
@@ -34,9 +34,10 @@ export const activity = proxyActivities<WorkflowControlActivities>({
|
|
|
34
34
|
},
|
|
35
35
|
});
|
|
36
36
|
|
|
37
|
-
/** Goal continuation
|
|
38
|
-
* a short retry window, then records an
|
|
39
|
-
* of relying on an unrelated
|
|
37
|
+
/** Goal continuation evaluates a durable Postgres obligation at an idle
|
|
38
|
+
* boundary. A transient failure gets a short retry window, then records an
|
|
39
|
+
* explicit delayed outbox wake instead of relying on an unrelated mutation or
|
|
40
|
+
* keeping workflow history alive with polling. */
|
|
40
41
|
export const goalActivity = proxyActivities<Pick<typeof activities, "maybeContinueGoal">>({
|
|
41
42
|
startToCloseTimeout: "30 seconds",
|
|
42
43
|
retry: {
|