@opengeni/worker-bundle 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +970 -325
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/scheduled-tasks.ts +53 -1
- package/src/activities/types.ts +6 -6
- package/src/concurrency.ts +58 -6
- package/src/http.ts +14 -2
- package/src/index.ts +36 -12
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
package/src/sandbox-routing.ts
CHANGED
|
@@ -18,7 +18,9 @@ import type { Settings } from "@opengeni/config";
|
|
|
18
18
|
import {
|
|
19
19
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
20
|
advanceWorkspaceGeneration,
|
|
21
|
+
getRetainedProcess,
|
|
21
22
|
retainWorkspaceMutationProcess,
|
|
23
|
+
retainedProcessSettlementIdentity,
|
|
22
24
|
settleRetainedProcess,
|
|
23
25
|
verifyRetainedProcessMutationSettlement,
|
|
24
26
|
verifyWorkspaceMutationSettlement,
|
|
@@ -26,6 +28,7 @@ import {
|
|
|
26
28
|
markWarmLeaseInstanceLost,
|
|
27
29
|
readLease,
|
|
28
30
|
readActiveSandbox,
|
|
31
|
+
SandboxRetainedProcessPromotionFencedError,
|
|
29
32
|
type Database,
|
|
30
33
|
type SandboxWorkspaceMutationAdmission,
|
|
31
34
|
} from "@opengeni/db";
|
|
@@ -47,6 +50,7 @@ import {
|
|
|
47
50
|
type RoutableBackendSession,
|
|
48
51
|
type RoutableSandbox,
|
|
49
52
|
type ResolvedActiveBackend,
|
|
53
|
+
type RoutingMutationSettlementResult,
|
|
50
54
|
type RoutingRetainedProcess,
|
|
51
55
|
type RoutingRetainedProcessTerminalProof,
|
|
52
56
|
type SelfhostedOpObserver,
|
|
@@ -175,7 +179,10 @@ function selfhostedResolverTimeouts(settings: Settings): {
|
|
|
175
179
|
selfhostedExecTimeoutMs: number;
|
|
176
180
|
} {
|
|
177
181
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
178
|
-
return {
|
|
182
|
+
return {
|
|
183
|
+
selfhostedTimeoutMs: timeoutMs,
|
|
184
|
+
selfhostedExecTimeoutMs: execTimeoutMs,
|
|
185
|
+
};
|
|
179
186
|
}
|
|
180
187
|
|
|
181
188
|
type HomeRouteLeaseIdentity = {
|
|
@@ -232,7 +239,9 @@ function homeRouteRecoveryError(
|
|
|
232
239
|
);
|
|
233
240
|
}
|
|
234
241
|
|
|
235
|
-
function providerIdentityFromResumeState(
|
|
242
|
+
export function providerIdentityFromResumeState(
|
|
243
|
+
resumeState: Record<string, unknown>,
|
|
244
|
+
): string | null {
|
|
236
245
|
const sessionState =
|
|
237
246
|
resumeState.sessionState && typeof resumeState.sessionState === "object"
|
|
238
247
|
? (resumeState.sessionState as Record<string, unknown>)
|
|
@@ -288,7 +297,10 @@ async function resolveCurrentHomeBackend(
|
|
|
288
297
|
// A route epoch can advance without a provider replacement. Reuse the exact
|
|
289
298
|
// established handle only when its identity still equals the durable one.
|
|
290
299
|
if (lease.instanceId === established.instanceId) {
|
|
291
|
-
services.onHomeSandboxRebound?.({
|
|
300
|
+
services.onHomeSandboxRebound?.({
|
|
301
|
+
established,
|
|
302
|
+
leaseEpoch: lease.leaseEpoch,
|
|
303
|
+
});
|
|
292
304
|
return {
|
|
293
305
|
session: established.session as RoutableBackendSession,
|
|
294
306
|
sandboxId: null,
|
|
@@ -357,7 +369,10 @@ async function resolveCurrentHomeBackend(
|
|
|
357
369
|
if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
|
|
358
370
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
359
371
|
}
|
|
360
|
-
services.onHomeSandboxRebound?.({
|
|
372
|
+
services.onHomeSandboxRebound?.({
|
|
373
|
+
established: rebound,
|
|
374
|
+
leaseEpoch: lease.leaseEpoch,
|
|
375
|
+
});
|
|
361
376
|
// Keep the resolver's mutable home reference on the latest verified handle so
|
|
362
377
|
// a later route epoch does not resume the same replacement again. This is only
|
|
363
378
|
// a worker-side handle update; the SDK-facing RoutingSandboxSession remains the
|
|
@@ -423,7 +438,7 @@ function afterPersistableHomeMutation(
|
|
|
423
438
|
outcome: "resolved" | "rejected";
|
|
424
439
|
result?: unknown;
|
|
425
440
|
retainedProcess?: RoutingRetainedProcess;
|
|
426
|
-
}) => Promise<void>)
|
|
441
|
+
}) => Promise<void | RoutingMutationSettlementResult>)
|
|
427
442
|
| undefined {
|
|
428
443
|
const fence = ids.workspaceMutationFence;
|
|
429
444
|
if (!home || !fence) return undefined;
|
|
@@ -442,29 +457,58 @@ function afterPersistableHomeMutation(
|
|
|
442
457
|
}
|
|
443
458
|
const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
|
|
444
459
|
if (outcome === "resolved" && retainedProcess) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
460
|
+
try {
|
|
461
|
+
await retainWorkspaceMutationProcess(services.db, {
|
|
462
|
+
accountId: fence.accountId,
|
|
463
|
+
workspaceId: ids.workspaceId,
|
|
464
|
+
sessionId: ids.sessionId,
|
|
465
|
+
processId: retainedProcess.id,
|
|
466
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
467
|
+
admissionId: exactAdmission.id,
|
|
468
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
469
|
+
operation: op,
|
|
470
|
+
owner: {
|
|
471
|
+
kind: "turn",
|
|
472
|
+
turnId: fence.turnId,
|
|
473
|
+
executionGeneration: fence.executionGeneration,
|
|
474
|
+
attemptId: fence.attemptId,
|
|
475
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
476
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
477
|
+
expectedEpoch: backend.leaseEpoch,
|
|
478
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
479
|
+
routeKind: exactAdmission.routeKind,
|
|
480
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
481
|
+
routeEpoch: exactAdmission.routeEpoch,
|
|
482
|
+
},
|
|
483
|
+
});
|
|
484
|
+
} catch (error) {
|
|
485
|
+
if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
|
|
486
|
+
const durable = error.process;
|
|
487
|
+
if (
|
|
488
|
+
durable.id !== retainedProcess.id ||
|
|
489
|
+
durable.providerSessionId !== retainedProcess.providerSessionId ||
|
|
490
|
+
durable.sessionId !== ids.sessionId ||
|
|
491
|
+
durable.sandboxGroupId !== home.sandboxGroupId ||
|
|
492
|
+
durable.parentAdmissionId !== exactAdmission.id ||
|
|
493
|
+
durable.leaseEpoch !== backend.leaseEpoch ||
|
|
494
|
+
durable.providerInstanceId !== backend.providerInstanceId ||
|
|
495
|
+
durable.routeKind !== exactAdmission.routeKind ||
|
|
496
|
+
durable.routeTargetId !== exactAdmission.routeTargetId ||
|
|
497
|
+
durable.routeEpoch !== exactAdmission.routeEpoch ||
|
|
498
|
+
durable.state !== "active"
|
|
499
|
+
) {
|
|
500
|
+
throw new Error("Durable retained-process promotion returned a mismatched identity", {
|
|
501
|
+
cause: error,
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
return {
|
|
505
|
+
status: "retained_process_durable_output_rejected",
|
|
506
|
+
retainedProcess: {
|
|
507
|
+
id: durable.id,
|
|
508
|
+
providerSessionId: durable.providerSessionId,
|
|
509
|
+
},
|
|
510
|
+
};
|
|
511
|
+
}
|
|
468
512
|
return;
|
|
469
513
|
}
|
|
470
514
|
await verifyWorkspaceMutationSettlement(services.db, {
|
|
@@ -556,12 +600,30 @@ function settleRetainedProcessForTurn(
|
|
|
556
600
|
| undefined {
|
|
557
601
|
const fence = ids.workspaceMutationFence;
|
|
558
602
|
if (!fence) return undefined;
|
|
559
|
-
return async ({ process, proof }) => {
|
|
603
|
+
return async ({ backend, process, proof }) => {
|
|
604
|
+
const durable = await getRetainedProcess(services.db, {
|
|
605
|
+
workspaceId: ids.workspaceId,
|
|
606
|
+
sessionId: ids.sessionId,
|
|
607
|
+
processId: process.id,
|
|
608
|
+
});
|
|
609
|
+
if (
|
|
610
|
+
!durable ||
|
|
611
|
+
durable.providerSessionId !== process.providerSessionId ||
|
|
612
|
+
durable.providerBackend !== backend.kind ||
|
|
613
|
+
durable.providerInstanceId !== backend.providerInstanceId ||
|
|
614
|
+
durable.leaseEpoch !== backend.leaseEpoch ||
|
|
615
|
+
durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
|
|
616
|
+
durable.routeTargetId !== backend.sandboxId ||
|
|
617
|
+
durable.routeEpoch !== backend.activeEpoch
|
|
618
|
+
) {
|
|
619
|
+
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
620
|
+
}
|
|
560
621
|
await settleRetainedProcess(services.db, {
|
|
561
622
|
accountId: fence.accountId,
|
|
562
623
|
workspaceId: ids.workspaceId,
|
|
563
624
|
sessionId: ids.sessionId,
|
|
564
625
|
processId: process.id,
|
|
626
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
565
627
|
outcome: proof.outcome,
|
|
566
628
|
exitCode: proof.exitCode,
|
|
567
629
|
reason: proof.reason,
|
|
@@ -762,7 +824,10 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
762
824
|
backendId: string;
|
|
763
825
|
agentDefaultManifest: unknown;
|
|
764
826
|
provisioner: {
|
|
765
|
-
get(): Promise<{
|
|
827
|
+
get(): Promise<{
|
|
828
|
+
established: EstablishedSandboxSession;
|
|
829
|
+
leaseEpoch?: number;
|
|
830
|
+
}>;
|
|
766
831
|
};
|
|
767
832
|
homeLeaseIdentity?: {
|
|
768
833
|
accountId: string;
|
package/src/workflows/session.ts
CHANGED
|
@@ -296,9 +296,11 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
296
296
|
const capacityCheckBackstop =
|
|
297
297
|
input.maxCapacityChecksPerRun ?? CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP;
|
|
298
298
|
if (workflowInfo().continueAsNewSuggested || capacityChecksThisRun >= capacityCheckBackstop) {
|
|
299
|
-
// The waiter
|
|
300
|
-
//
|
|
301
|
-
//
|
|
299
|
+
// The waiter and exact nonterminal turn are durable in Postgres, and
|
|
300
|
+
// the old attempt is already closed. A fresh workflow run reads the
|
|
301
|
+
// waiter before goal continuation, reconstructs its timer, and turns
|
|
302
|
+
// any unobserved wake revision into an immediate evaluation. This is
|
|
303
|
+
// an ownerless-attempt boundary, not a settled-turn boundary.
|
|
302
304
|
// A quiescence proof is the one signal that is not merely a replaceable
|
|
303
305
|
// wake hint. It may have arrived while the reconciliation activity was
|
|
304
306
|
// running, so commit it before crossing this nested continue-as-new
|
|
@@ -324,11 +326,12 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
324
326
|
// decision so no accepted physical fact can be dropped with run history.
|
|
325
327
|
await persistPendingQuiescenceProofs();
|
|
326
328
|
// History-overflow guard. The top of the loop is the only safe
|
|
327
|
-
// continueAsNew boundary: no
|
|
328
|
-
//
|
|
329
|
-
//
|
|
330
|
-
//
|
|
331
|
-
//
|
|
329
|
+
// continueAsNew boundary: no activity attempt owns a turn. A same logical
|
|
330
|
+
// turn may still be nonterminal (`recovering` or `waiting_capacity`), but
|
|
331
|
+
// its dispatch attempt has closed and all recovery/wait truth is durable
|
|
332
|
+
// in Postgres. Every interruption is already durable there too, so ordinary
|
|
333
|
+
// Temporal signals are replaceable wake hints and none must be carried into
|
|
334
|
+
// the next workflow run. A
|
|
332
335
|
// buffered userMessage/queueChanged signal only
|
|
333
336
|
// bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
|
|
334
337
|
// was sent, so the fresh run observes it on its first durable work peek
|
|
@@ -604,9 +607,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
604
607
|
}
|
|
605
608
|
|
|
606
609
|
if (outcome.kind === "failure") {
|
|
607
|
-
// A capacity
|
|
608
|
-
// transport/worker failed. Recover that durable boundary
|
|
609
|
-
// failSession can overwrite the
|
|
610
|
+
// A capacity wait may have committed just before the activity
|
|
611
|
+
// transport/worker failed. Recover that durable same-turn boundary
|
|
612
|
+
// before generic failSession can overwrite the nonterminal turn.
|
|
610
613
|
{
|
|
611
614
|
const capacityWait = await activity.getCodexCapacityWait({
|
|
612
615
|
workspaceId,
|