@opengeni/worker-bundle 0.12.10 → 0.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2729 -1208
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +873 -131
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
package/src/sandbox-routing.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// target lookup), and the selfhosted ControlRpc built over the events bus's NATS
|
|
15
15
|
// request/reply connection.
|
|
16
16
|
|
|
17
|
-
import type
|
|
17
|
+
import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
|
|
18
18
|
import {
|
|
19
19
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
20
|
advanceWorkspaceGeneration,
|
|
@@ -43,6 +43,7 @@ import {
|
|
|
43
43
|
NatsOpStreamTransport,
|
|
44
44
|
RoutingBackendRecoveryRequiredError,
|
|
45
45
|
RoutingSandboxSession,
|
|
46
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
46
47
|
verifySandboxExecReadiness,
|
|
47
48
|
type ControlRpc,
|
|
48
49
|
type EstablishedSandboxSession,
|
|
@@ -60,6 +61,13 @@ import {
|
|
|
60
61
|
} from "@opengeni/runtime";
|
|
61
62
|
import { sandboxLeaseHolderIdForAttempt } from "./sandbox-resume";
|
|
62
63
|
|
|
64
|
+
type PersistableMutationAdmission = {
|
|
65
|
+
admission: SandboxWorkspaceMutationAdmission;
|
|
66
|
+
providerBinding: Awaited<
|
|
67
|
+
ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
|
|
68
|
+
> | null;
|
|
69
|
+
};
|
|
70
|
+
|
|
63
71
|
export type RoutingWiringServices = {
|
|
64
72
|
db: Database;
|
|
65
73
|
settings: Settings;
|
|
@@ -390,12 +398,12 @@ async function resolveCurrentHomeBackend(
|
|
|
390
398
|
function beforePersistableHomeMutation(
|
|
391
399
|
services: RoutingWiringServices,
|
|
392
400
|
ids: RoutingWiringIds,
|
|
393
|
-
home: { accountId: string; sandboxGroupId: string } | undefined,
|
|
401
|
+
home: { accountId: string; sandboxGroupId: string; backend: string } | undefined,
|
|
394
402
|
):
|
|
395
403
|
| ((input: {
|
|
396
404
|
op: string;
|
|
397
405
|
backend: ResolvedActiveBackend;
|
|
398
|
-
}) => Promise<
|
|
406
|
+
}) => Promise<PersistableMutationAdmission | null>)
|
|
399
407
|
| undefined {
|
|
400
408
|
const fence = ids.workspaceMutationFence;
|
|
401
409
|
if (!home || !fence) return undefined;
|
|
@@ -410,7 +418,11 @@ function beforePersistableHomeMutation(
|
|
|
410
418
|
) {
|
|
411
419
|
return null;
|
|
412
420
|
}
|
|
413
|
-
|
|
421
|
+
const providerBinding =
|
|
422
|
+
home.backend === "modal"
|
|
423
|
+
? await resolveModalCheckpointProviderBindingForSession(services.settings, backend.session)
|
|
424
|
+
: null;
|
|
425
|
+
const admission = await advanceWorkspaceGeneration(services.db, {
|
|
414
426
|
accountId: fence.accountId,
|
|
415
427
|
workspaceId: ids.workspaceId,
|
|
416
428
|
sessionId: ids.sessionId,
|
|
@@ -422,14 +434,16 @@ function beforePersistableHomeMutation(
|
|
|
422
434
|
expectedEpoch: backend.leaseEpoch,
|
|
423
435
|
expectedInstanceId: backend.providerInstanceId,
|
|
424
436
|
operation: op,
|
|
437
|
+
captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
|
|
425
438
|
});
|
|
439
|
+
return { admission, providerBinding };
|
|
426
440
|
};
|
|
427
441
|
}
|
|
428
442
|
|
|
429
443
|
function afterPersistableHomeMutation(
|
|
430
444
|
services: RoutingWiringServices,
|
|
431
445
|
ids: RoutingWiringIds,
|
|
432
|
-
home: { accountId: string; sandboxGroupId: string } | undefined,
|
|
446
|
+
home: { accountId: string; sandboxGroupId: string; backend: string } | undefined,
|
|
433
447
|
):
|
|
434
448
|
| ((input: {
|
|
435
449
|
op: string;
|
|
@@ -448,14 +462,20 @@ function afterPersistableHomeMutation(
|
|
|
448
462
|
backend.leaseEpoch === undefined ||
|
|
449
463
|
backend.providerInstanceId === undefined ||
|
|
450
464
|
!admission ||
|
|
451
|
-
typeof admission !== "object"
|
|
452
|
-
typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
|
|
453
|
-
typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
|
|
454
|
-
"number"
|
|
465
|
+
typeof admission !== "object"
|
|
455
466
|
) {
|
|
456
467
|
return;
|
|
457
468
|
}
|
|
458
|
-
const
|
|
469
|
+
const boundAdmission = admission as Partial<PersistableMutationAdmission>;
|
|
470
|
+
const exactAdmission = boundAdmission.admission;
|
|
471
|
+
if (
|
|
472
|
+
!exactAdmission ||
|
|
473
|
+
typeof exactAdmission.id !== "string" ||
|
|
474
|
+
typeof exactAdmission.workspaceGeneration !== "number" ||
|
|
475
|
+
!("providerBinding" in boundAdmission)
|
|
476
|
+
) {
|
|
477
|
+
throw new Error("Persistable home mutation settlement lacked its bound admission");
|
|
478
|
+
}
|
|
459
479
|
if (outcome === "resolved" && retainedProcess) {
|
|
460
480
|
try {
|
|
461
481
|
await retainWorkspaceMutationProcess(services.db, {
|
|
@@ -467,6 +487,7 @@ function afterPersistableHomeMutation(
|
|
|
467
487
|
admissionId: exactAdmission.id,
|
|
468
488
|
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
469
489
|
operation: op,
|
|
490
|
+
providerBinding: boundAdmission.providerBinding ?? null,
|
|
470
491
|
owner: {
|
|
471
492
|
kind: "turn",
|
|
472
493
|
turnId: fence.turnId,
|
|
@@ -548,6 +569,7 @@ function beforeRetainedProcessMutation(
|
|
|
548
569
|
sessionId: ids.sessionId,
|
|
549
570
|
processId: process.id,
|
|
550
571
|
operation: op,
|
|
572
|
+
captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
|
|
551
573
|
});
|
|
552
574
|
}
|
|
553
575
|
|
package/src/workflows/session.ts
CHANGED
|
@@ -13,6 +13,11 @@ import {
|
|
|
13
13
|
workflowInfo,
|
|
14
14
|
} from "@temporalio/workflow";
|
|
15
15
|
import type * as activities from "../activities";
|
|
16
|
+
import {
|
|
17
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
18
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
19
|
+
type EscapedMcpTimeoutRecoveryDetail,
|
|
20
|
+
} from "../activities/types";
|
|
16
21
|
import {
|
|
17
22
|
activity,
|
|
18
23
|
goalActivity,
|
|
@@ -36,6 +41,7 @@ import {
|
|
|
36
41
|
const TURNS_PER_RUN_BACKSTOP = 2_000;
|
|
37
42
|
const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
|
|
38
43
|
const HUMAN_INPUT_EXPIRY_STALE_RETRY_MS = 1_000;
|
|
44
|
+
const ESCAPED_MCP_TIMEOUT_RECOVERY_DELAY_MS = 60_000;
|
|
39
45
|
|
|
40
46
|
/**
|
|
41
47
|
* The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
|
|
@@ -119,6 +125,46 @@ function workerDeathFailure(
|
|
|
119
125
|
return { timeoutType: cause.timeoutType };
|
|
120
126
|
}
|
|
121
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Parse only the explicit activity wire contract emitted when a recovered
|
|
130
|
+
* turn's MCP setup timeout could not finish its DB checkpoint. Numeric -32001
|
|
131
|
+
* and generic MCP messages remain deliberately insufficient: the activity
|
|
132
|
+
* writes this non-retryable ApplicationFailure only before any model request,
|
|
133
|
+
* with immutable turn identity in payload-converted details.
|
|
134
|
+
*/
|
|
135
|
+
export function escapedMcpTimeoutRecoveryDetail(
|
|
136
|
+
error: unknown,
|
|
137
|
+
): EscapedMcpTimeoutRecoveryDetail | null {
|
|
138
|
+
if (!(error instanceof ActivityFailure) || error.activityType !== "runAgentTurn") {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
const cause = error.cause;
|
|
142
|
+
if (
|
|
143
|
+
!(cause instanceof ApplicationFailure) ||
|
|
144
|
+
cause.type !== ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE ||
|
|
145
|
+
cause.message !== ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE
|
|
146
|
+
) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
const detail = cause.details?.[0] as Partial<EscapedMcpTimeoutRecoveryDetail> | undefined;
|
|
150
|
+
if (
|
|
151
|
+
!detail ||
|
|
152
|
+
typeof detail.turnId !== "string" ||
|
|
153
|
+
detail.turnId.length === 0 ||
|
|
154
|
+
typeof detail.triggerEventId !== "string" ||
|
|
155
|
+
detail.triggerEventId.length === 0 ||
|
|
156
|
+
!Number.isSafeInteger(detail.executionGeneration) ||
|
|
157
|
+
(detail.executionGeneration ?? 0) <= 1
|
|
158
|
+
) {
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
turnId: detail.turnId,
|
|
163
|
+
triggerEventId: detail.triggerEventId,
|
|
164
|
+
executionGeneration: detail.executionGeneration!,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
122
168
|
/**
|
|
123
169
|
* Classify only the exact turn-fence cancellation protocol used to arbitrate a
|
|
124
170
|
* database control commit that reaches the activity just before its Temporal
|
|
@@ -631,6 +677,30 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
631
677
|
return true;
|
|
632
678
|
}
|
|
633
679
|
}
|
|
680
|
+
// The turn activity already classified a precise MCP request timeout,
|
|
681
|
+
// but its DB checkpoint/fanout path itself failed before any model
|
|
682
|
+
// request. Finish that exact generation-2+ recovery through the bounded
|
|
683
|
+
// control-activity lane instead of terminalizing the turn and arming a
|
|
684
|
+
// fresh goal continuation from unchanged history.
|
|
685
|
+
const escapedMcpTimeout = escapedMcpTimeoutRecoveryDetail(outcome.error);
|
|
686
|
+
if (escapedMcpTimeout) {
|
|
687
|
+
const recovery = await activity.recoverEscapedMcpTimeout({
|
|
688
|
+
accountId,
|
|
689
|
+
workspaceId,
|
|
690
|
+
sessionId,
|
|
691
|
+
attemptId,
|
|
692
|
+
...escapedMcpTimeout,
|
|
693
|
+
});
|
|
694
|
+
if (recovery.action !== "ineligible") {
|
|
695
|
+
const seenWakeups = wakeups;
|
|
696
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
697
|
+
await condition(
|
|
698
|
+
() => interruptionWakeups !== seenInterruptionWakeups || wakeups !== seenWakeups,
|
|
699
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_DELAY_MS,
|
|
700
|
+
);
|
|
701
|
+
return true;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
634
704
|
// An ungraceful worker death never reaches the activity's graceful
|
|
635
705
|
// recovery path — it surfaces here as a heartbeat-timeout failure.
|
|
636
706
|
// Conversation truth was still dual-written during the turn, so the
|