@opengeni/worker-bundle 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +970 -325
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/scheduled-tasks.ts +53 -1
- package/src/activities/types.ts +6 -6
- package/src/concurrency.ts +58 -6
- package/src/http.ts +14 -2
- package/src/index.ts +36 -12
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
import {
|
|
2
|
+
OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
3
|
+
OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
4
|
+
OPENGENI_SLACK_BOT_SESSION_METADATA_KEY,
|
|
5
|
+
} from "@opengeni/contracts";
|
|
6
|
+
import {
|
|
7
|
+
openGeniSlackBotMetadata,
|
|
8
|
+
requireOpenGeniSlackBotConnection,
|
|
9
|
+
scheduledSlackBotConnectionId,
|
|
10
|
+
} from "@opengeni/core";
|
|
1
11
|
import {
|
|
2
12
|
appendSessionEventsWithLockedSessionUpdate,
|
|
3
13
|
addSessionSystemUpdateWithSourceMutation,
|
|
@@ -41,6 +51,23 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
41
51
|
): Promise<DispatchScheduledTaskRunResult> => {
|
|
42
52
|
const { settings, db, bus, wakeSessionWorkflow } = await services();
|
|
43
53
|
const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
|
|
54
|
+
// A scheduled bot selection was authorized when the task was written,
|
|
55
|
+
// but connection status and tenant/role binding are mutable. Revalidate
|
|
56
|
+
// before any session/model cost and never fall back to a personal Slack
|
|
57
|
+
// OAuth grant when the selected bot has been revoked or changed.
|
|
58
|
+
const slackBotConnection = task.agentConfig.slackBotConnectionId
|
|
59
|
+
? await requireOpenGeniSlackBotConnection(
|
|
60
|
+
db,
|
|
61
|
+
task.workspaceId,
|
|
62
|
+
task.agentConfig.slackBotConnectionId,
|
|
63
|
+
)
|
|
64
|
+
: null;
|
|
65
|
+
const slackBotMetadata = slackBotConnection
|
|
66
|
+
? openGeniSlackBotMetadata(slackBotConnection.metadata)
|
|
67
|
+
: null;
|
|
68
|
+
if (slackBotConnection && !slackBotMetadata) {
|
|
69
|
+
throw new Error("OpenGeni Slack bot connection metadata is invalid");
|
|
70
|
+
}
|
|
44
71
|
// The scheduled task's model can be codex/<slug>; resolve it here so the
|
|
45
72
|
// admission gate can skip the credit/cost gates for a codex-billed run
|
|
46
73
|
// (paid by the user's ChatGPT/Codex plan). This file uses BASE settings (no
|
|
@@ -160,6 +187,8 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
160
187
|
frozenRigVersionId = rig.activeVersion.id;
|
|
161
188
|
}
|
|
162
189
|
let session: Awaited<ReturnType<typeof createSession>>;
|
|
190
|
+
const taskMetadata = { ...task.agentConfig.metadata };
|
|
191
|
+
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
163
192
|
try {
|
|
164
193
|
session = await createSession(db, {
|
|
165
194
|
accountId: task.accountId,
|
|
@@ -168,11 +197,16 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
168
197
|
resources: task.agentConfig.resources,
|
|
169
198
|
tools: taskTools,
|
|
170
199
|
metadata: {
|
|
171
|
-
...
|
|
200
|
+
...taskMetadata,
|
|
172
201
|
model,
|
|
173
202
|
reasoningEffort,
|
|
174
203
|
scheduledTaskId: task.id,
|
|
175
204
|
scheduledTaskRunId: run.id,
|
|
205
|
+
...(slackBotConnection
|
|
206
|
+
? {
|
|
207
|
+
[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id,
|
|
208
|
+
}
|
|
209
|
+
: {}),
|
|
176
210
|
},
|
|
177
211
|
createdBy: {
|
|
178
212
|
kind: "service",
|
|
@@ -231,6 +265,16 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
231
265
|
...(variableSet
|
|
232
266
|
? { variableSetId: variableSet.id, variableSetName: variableSet.name }
|
|
233
267
|
: {}),
|
|
268
|
+
...(slackBotConnection && slackBotMetadata
|
|
269
|
+
? {
|
|
270
|
+
slackBotConnection: {
|
|
271
|
+
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
272
|
+
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
273
|
+
connectionId: slackBotConnection.id,
|
|
274
|
+
slackTeamId: slackBotMetadata.slackTeamId,
|
|
275
|
+
},
|
|
276
|
+
}
|
|
277
|
+
: {}),
|
|
234
278
|
},
|
|
235
279
|
},
|
|
236
280
|
...(goal
|
|
@@ -316,6 +360,14 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
316
360
|
"scheduled task variableSet attachment does not match its reusable session",
|
|
317
361
|
);
|
|
318
362
|
}
|
|
363
|
+
if (
|
|
364
|
+
scheduledSlackBotConnectionId(session.metadata) !==
|
|
365
|
+
(task.agentConfig.slackBotConnectionId ?? null)
|
|
366
|
+
) {
|
|
367
|
+
throw new Error(
|
|
368
|
+
"scheduled task OpenGeni Slack bot binding does not match its reusable session",
|
|
369
|
+
);
|
|
370
|
+
}
|
|
319
371
|
// A recurring "maintain X" task re-establishes its objective on every
|
|
320
372
|
// fire: replace the goal text, reactivate it, and reset the counters.
|
|
321
373
|
const reusableGoal =
|
package/src/activities/types.ts
CHANGED
|
@@ -120,8 +120,7 @@ export type ReconcileCodexCapacityWaitInput = {
|
|
|
120
120
|
|
|
121
121
|
export type ReconcileCodexCapacityWaitResult =
|
|
122
122
|
| ({ action: "waiting" } & CodexCapacityWaitRef)
|
|
123
|
-
| { action: "resumed"
|
|
124
|
-
| { action: "superseded" | "stale" };
|
|
123
|
+
| { action: "resumed" | "paused" | "superseded" | "stale" };
|
|
125
124
|
|
|
126
125
|
export type ActivityDependencies = Partial<ActivityServices>;
|
|
127
126
|
|
|
@@ -241,7 +240,7 @@ export type IndexDocumentInput = {
|
|
|
241
240
|
type ClaimedRunAgentTurnResult = {
|
|
242
241
|
// "recovering": this attempt ended after durably preserving the same current
|
|
243
242
|
// inference for a new attempt. Recovery is not prompt queue work.
|
|
244
|
-
status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering";
|
|
243
|
+
status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering" | "waiting_capacity";
|
|
245
244
|
turnId: string;
|
|
246
245
|
attemptId: string;
|
|
247
246
|
// Provider backpressure pacing: when set on an idle or recovering result, the
|
|
@@ -253,9 +252,10 @@ type ClaimedRunAgentTurnResult = {
|
|
|
253
252
|
// "continue now" (invariant 4: NO THRASH). Distinct from a normal continueDelayMs:0
|
|
254
253
|
// which legitimately means "a rotation candidate is ready, re-dispatch immediately".
|
|
255
254
|
idleUntilReset?: boolean;
|
|
256
|
-
// Durable native zero-pool wait
|
|
257
|
-
// persisted in Postgres and
|
|
258
|
-
// The workflow must not call
|
|
255
|
+
// Durable native zero-pool wait for this same nonterminal logical turn.
|
|
256
|
+
// Unlike continueDelayMs, this reference is persisted in Postgres and
|
|
257
|
+
// reconstructed after workflow/worker restart. The workflow must not call
|
|
258
|
+
// maybeContinueGoal or manufacture queue/input work while it is active.
|
|
259
259
|
capacityWait?: CodexCapacityWaitRef;
|
|
260
260
|
// This execution reached a durable terminal-for-now boundary (for example,
|
|
261
261
|
// maintenance could not run or same-turn context recovery failed). End this
|
package/src/concurrency.ts
CHANGED
|
@@ -1,11 +1,63 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import type { WorkerOptions } from "@temporalio/worker";
|
|
3
|
+
|
|
4
|
+
export type TurnWorkerConcurrencySettings = Pick<
|
|
5
|
+
Settings,
|
|
6
|
+
| "turnWorkerConcurrencyMode"
|
|
7
|
+
| "turnWorkerMaxConcurrentTurns"
|
|
8
|
+
| "turnWorkerTargetCpuUsage"
|
|
9
|
+
| "turnWorkerTargetMemoryUsage"
|
|
10
|
+
>;
|
|
11
|
+
|
|
12
|
+
export type TurnWorkerConcurrencyOptions = Pick<
|
|
13
|
+
WorkerOptions,
|
|
14
|
+
"maxConcurrentActivityTaskExecutions" | "tuner"
|
|
15
|
+
>;
|
|
16
|
+
|
|
1
17
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* production and the real Temporal integration harness exercise one topology.
|
|
18
|
+
* Ordinary multi-worker deployments keep a fixed per-process ceiling so HPA or
|
|
19
|
+
* an operator owns horizontal capacity. The single-machine profile selects the
|
|
20
|
+
* resource tuner instead: Temporal admits activity slots while whole-system CPU
|
|
21
|
+
* and memory remain below the configured targets, then stops polling more work.
|
|
7
22
|
*/
|
|
8
|
-
export
|
|
23
|
+
export function turnWorkerConcurrencyOptions(
|
|
24
|
+
settings: TurnWorkerConcurrencySettings,
|
|
25
|
+
): TurnWorkerConcurrencyOptions {
|
|
26
|
+
if (settings.turnWorkerConcurrencyMode === "fixed") {
|
|
27
|
+
return {
|
|
28
|
+
maxConcurrentActivityTaskExecutions: settings.turnWorkerMaxConcurrentTurns,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return {
|
|
33
|
+
tuner: {
|
|
34
|
+
tunerOptions: {
|
|
35
|
+
targetCpuUsage: settings.turnWorkerTargetCpuUsage,
|
|
36
|
+
targetMemoryUsage: settings.turnWorkerTargetMemoryUsage,
|
|
37
|
+
},
|
|
38
|
+
activityTaskSlotOptions: {
|
|
39
|
+
minimumSlots: 1,
|
|
40
|
+
maximumSlots: settings.turnWorkerMaxConcurrentTurns,
|
|
41
|
+
rampThrottle: "50ms",
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySettings): {
|
|
48
|
+
concurrencyMode: "fixed" | "resource-based";
|
|
49
|
+
maxConcurrentTurns: number;
|
|
50
|
+
targetCpuUsage: number | null;
|
|
51
|
+
targetMemoryUsage: number | null;
|
|
52
|
+
} {
|
|
53
|
+
const resourceBased = settings.turnWorkerConcurrencyMode === "resource-based";
|
|
54
|
+
return {
|
|
55
|
+
concurrencyMode: settings.turnWorkerConcurrencyMode,
|
|
56
|
+
maxConcurrentTurns: settings.turnWorkerMaxConcurrentTurns,
|
|
57
|
+
targetCpuUsage: resourceBased ? settings.turnWorkerTargetCpuUsage : null,
|
|
58
|
+
targetMemoryUsage: resourceBased ? settings.turnWorkerTargetMemoryUsage : null,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
9
61
|
|
|
10
62
|
export const CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;
|
|
11
63
|
export const CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;
|
package/src/http.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
assertRuntimeDatabasePosture,
|
|
4
|
+
dbSql,
|
|
5
|
+
type Database,
|
|
6
|
+
type RuntimeDatabasePostureOptions,
|
|
7
|
+
} from "@opengeni/db";
|
|
3
8
|
import type { EventBus } from "@opengeni/events";
|
|
4
9
|
import type { Observability } from "@opengeni/observability";
|
|
5
10
|
|
|
@@ -112,8 +117,15 @@ export async function runReadinessChecks(
|
|
|
112
117
|
};
|
|
113
118
|
}
|
|
114
119
|
|
|
115
|
-
export function dbReadyCheck(
|
|
120
|
+
export function dbReadyCheck(
|
|
121
|
+
db: Database,
|
|
122
|
+
posture?: RuntimeDatabasePostureOptions,
|
|
123
|
+
): () => Promise<void> {
|
|
116
124
|
return async () => {
|
|
125
|
+
if (posture) {
|
|
126
|
+
await assertRuntimeDatabasePosture(db, posture);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
117
129
|
await db.execute(dbSql`select 1`);
|
|
118
130
|
};
|
|
119
131
|
}
|
package/src/index.ts
CHANGED
|
@@ -7,7 +7,13 @@ import {
|
|
|
7
7
|
temporalConnectionOptions,
|
|
8
8
|
type Settings,
|
|
9
9
|
} from "@opengeni/config";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
assertRuntimeDatabasePosture,
|
|
12
|
+
createDb,
|
|
13
|
+
markSessionWorkflowWakeDelivered,
|
|
14
|
+
type Database,
|
|
15
|
+
type RuntimeDatabasePostureOptions,
|
|
16
|
+
} from "@opengeni/db";
|
|
11
17
|
import { createNatsEventBus, type EventBus } from "@opengeni/events";
|
|
12
18
|
import {
|
|
13
19
|
createObservability,
|
|
@@ -51,7 +57,8 @@ import {
|
|
|
51
57
|
import {
|
|
52
58
|
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
53
59
|
CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
54
|
-
|
|
60
|
+
turnWorkerConcurrencyLogFields,
|
|
61
|
+
turnWorkerConcurrencyOptions,
|
|
55
62
|
} from "./concurrency";
|
|
56
63
|
import {
|
|
57
64
|
constructWithOwnedConnection,
|
|
@@ -173,13 +180,10 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
173
180
|
? { workflowBundle: options.workflowBundle }
|
|
174
181
|
: resolveOpenGeniWorkflowDefinition()),
|
|
175
182
|
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
183
|
+
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
176
184
|
}
|
|
177
|
-
:
|
|
185
|
+
: turnWorkerConcurrencyOptions(settings)),
|
|
178
186
|
activities,
|
|
179
|
-
maxConcurrentActivityTaskExecutions:
|
|
180
|
-
options.role === "turn"
|
|
181
|
-
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
182
|
-
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
183
187
|
// Cancellation is delivered through an activity heartbeat. The SDK would
|
|
184
188
|
// otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
|
|
185
189
|
// making Pause/Steer take roughly a minute even though runAgentTurn emits a
|
|
@@ -471,6 +475,11 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
471
475
|
|
|
472
476
|
export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
|
|
473
477
|
activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
|
|
478
|
+
/**
|
|
479
|
+
* Exact catalog posture required by the standalone runtime. Embedded hosts
|
|
480
|
+
* may omit this when they own an equivalent database isolation contract.
|
|
481
|
+
*/
|
|
482
|
+
databasePosture?: RuntimeDatabasePostureOptions;
|
|
474
483
|
/**
|
|
475
484
|
* `role-default` registers OpenGeni's internal maintenance schedules on a
|
|
476
485
|
* control worker only. These are engine maintenance schedules, not a host's
|
|
@@ -588,7 +597,7 @@ export async function createOpenGeniWorkerService(
|
|
|
588
597
|
settings,
|
|
589
598
|
observability,
|
|
590
599
|
checks: {
|
|
591
|
-
db: dbReadyCheck(options.activityDependencies.db),
|
|
600
|
+
db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
|
|
592
601
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
593
602
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
594
603
|
},
|
|
@@ -631,10 +640,14 @@ export async function createOpenGeniWorkerService(
|
|
|
631
640
|
options.role === "control"
|
|
632
641
|
? settings.temporalTaskQueue
|
|
633
642
|
: turnTaskQueue(settings.temporalTaskQueue),
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
643
|
+
...(options.role === "turn"
|
|
644
|
+
? turnWorkerConcurrencyLogFields(settings)
|
|
645
|
+
: {
|
|
646
|
+
concurrencyMode: "fixed",
|
|
647
|
+
maxConcurrentTurns: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
648
|
+
targetCpuUsage: null,
|
|
649
|
+
targetMemoryUsage: null,
|
|
650
|
+
}),
|
|
638
651
|
maxConcurrentWorkflowTaskExecutions:
|
|
639
652
|
options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
|
|
640
653
|
httpPort: options.http === false ? null : settings.workerHttpPort,
|
|
@@ -693,9 +706,19 @@ export async function startWorker() {
|
|
|
693
706
|
...(searchPath ? { searchPath } : {}),
|
|
694
707
|
rlsStrategy: settings.rlsStrategy,
|
|
695
708
|
});
|
|
709
|
+
const databasePosture = {
|
|
710
|
+
rlsStrategy: settings.rlsStrategy,
|
|
711
|
+
expectedRole: settings.runtimeDatabaseRole,
|
|
712
|
+
targetSchema: settings.dbSchema.trim() || "public",
|
|
713
|
+
} as const;
|
|
696
714
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
697
715
|
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
698
716
|
try {
|
|
717
|
+
await retryStartupDependency(
|
|
718
|
+
"PostgreSQL runtime posture",
|
|
719
|
+
() => assertRuntimeDatabasePosture(dbClient.db, databasePosture),
|
|
720
|
+
{ ...retryOptions, onRetry },
|
|
721
|
+
);
|
|
699
722
|
bus = await retryStartupDependency(
|
|
700
723
|
"NATS",
|
|
701
724
|
() =>
|
|
@@ -711,6 +734,7 @@ export async function startWorker() {
|
|
|
711
734
|
await runOpenGeniWorker({
|
|
712
735
|
role,
|
|
713
736
|
settings,
|
|
737
|
+
databasePosture,
|
|
714
738
|
activityDependencies: {
|
|
715
739
|
observability,
|
|
716
740
|
db: dbClient.db,
|
|
@@ -19,6 +19,7 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
|
19
19
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
20
20
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
21
21
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
22
|
+
const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
|
|
22
23
|
|
|
23
24
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
24
25
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -383,6 +384,142 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
|
|
|
383
384
|
});
|
|
384
385
|
}
|
|
385
386
|
|
|
387
|
+
const RETAINED_PROCESS_OWNER_STATES = [
|
|
388
|
+
"direct",
|
|
389
|
+
"queued",
|
|
390
|
+
"running",
|
|
391
|
+
"requires_action",
|
|
392
|
+
"recovering",
|
|
393
|
+
"waiting_capacity",
|
|
394
|
+
"completed",
|
|
395
|
+
"failed",
|
|
396
|
+
"cancelled",
|
|
397
|
+
"superseded",
|
|
398
|
+
"withdrawn_for_edit",
|
|
399
|
+
"missing",
|
|
400
|
+
"unknown",
|
|
401
|
+
] as const;
|
|
402
|
+
|
|
403
|
+
const RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_STATES);
|
|
404
|
+
|
|
405
|
+
export function recordRetainedProcessInventoryGauges(
|
|
406
|
+
observability: Observability,
|
|
407
|
+
counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,
|
|
408
|
+
): void {
|
|
409
|
+
const normalized = new Map<string, { active: number; terminal: number }>();
|
|
410
|
+
for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
|
|
411
|
+
normalized.set(ownerState, { active: 0, terminal: 0 });
|
|
412
|
+
}
|
|
413
|
+
for (const count of counts) {
|
|
414
|
+
const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState)
|
|
415
|
+
? count.ownerState
|
|
416
|
+
: "unknown";
|
|
417
|
+
const current = normalized.get(ownerState)!;
|
|
418
|
+
current.active += Math.max(0, count.activeCount);
|
|
419
|
+
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
let terminalTotal = 0;
|
|
423
|
+
for (const [ownerState, count] of normalized) {
|
|
424
|
+
terminalTotal += count.terminal;
|
|
425
|
+
observability.setGauge({
|
|
426
|
+
name: "opengeni_retained_processes_active",
|
|
427
|
+
help: "Current active retained provider processes by durable owner state.",
|
|
428
|
+
labels: { owner_state: ownerState },
|
|
429
|
+
value: count.active,
|
|
430
|
+
});
|
|
431
|
+
observability.setGauge({
|
|
432
|
+
name: "opengeni_retained_processes_terminal_owner_backlog",
|
|
433
|
+
help: "Current active retained processes whose exact owner attempt is terminal.",
|
|
434
|
+
labels: { owner_state: ownerState },
|
|
435
|
+
value: count.terminal,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
const previous = retainedProcessTerminalBacklogTotals.get(observability);
|
|
439
|
+
if (previous !== undefined && terminalTotal > previous) {
|
|
440
|
+
observability.incrementCounter({
|
|
441
|
+
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
442
|
+
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
443
|
+
amount: terminalTotal - previous,
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const EXPIRED_DRAINING_BACKENDS = [
|
|
450
|
+
"none",
|
|
451
|
+
"local",
|
|
452
|
+
"docker",
|
|
453
|
+
"modal",
|
|
454
|
+
"e2b",
|
|
455
|
+
"daytona",
|
|
456
|
+
"blaxel",
|
|
457
|
+
"runloop",
|
|
458
|
+
"selfhosted",
|
|
459
|
+
"unknown",
|
|
460
|
+
] as const;
|
|
461
|
+
const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
|
|
462
|
+
const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
|
|
463
|
+
|
|
464
|
+
export function recordExpiredDrainingSandboxLeaseGauges(
|
|
465
|
+
observability: Observability,
|
|
466
|
+
counts: Array<{ backend: string; ageBucket: string; count: number }>,
|
|
467
|
+
): void {
|
|
468
|
+
const normalized = new Map<string, number>();
|
|
469
|
+
for (const backend of EXPIRED_DRAINING_BACKENDS) {
|
|
470
|
+
for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
|
|
471
|
+
normalized.set(`${backend}:${ageBucket}`, 0);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
for (const count of counts) {
|
|
475
|
+
const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
|
|
476
|
+
if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket as never)) continue;
|
|
477
|
+
const key = `${backend}:${count.ageBucket}`;
|
|
478
|
+
normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
|
|
479
|
+
}
|
|
480
|
+
for (const [key, value] of normalized) {
|
|
481
|
+
const [backend, ageBucket] = key.split(":") as [string, string];
|
|
482
|
+
observability.setGauge({
|
|
483
|
+
name: "opengeni_sandbox_leases_expired_draining",
|
|
484
|
+
help: "Current expired draining sandbox leases by backend and fixed age bucket.",
|
|
485
|
+
labels: { backend, age_bucket: ageBucket },
|
|
486
|
+
value,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
|
|
492
|
+
"claim_failed",
|
|
493
|
+
"proof_exited",
|
|
494
|
+
"proof_lost",
|
|
495
|
+
"proof_checkpoint_failed",
|
|
496
|
+
"settled_exited",
|
|
497
|
+
"settled_lost",
|
|
498
|
+
"settlement_failed",
|
|
499
|
+
"identity_mismatch",
|
|
500
|
+
"resume_state_missing",
|
|
501
|
+
"backend_unsupported",
|
|
502
|
+
"provider_running",
|
|
503
|
+
"provider_unknown",
|
|
504
|
+
"provider_timeout",
|
|
505
|
+
"provider_error",
|
|
506
|
+
"defer_failed",
|
|
507
|
+
] as const;
|
|
508
|
+
|
|
509
|
+
export type RetainedProcessReconciliationOutcome =
|
|
510
|
+
(typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];
|
|
511
|
+
|
|
512
|
+
export function recordRetainedProcessReconciliation(
|
|
513
|
+
observability: Observability,
|
|
514
|
+
outcome: RetainedProcessReconciliationOutcome,
|
|
515
|
+
): void {
|
|
516
|
+
observability.incrementCounter({
|
|
517
|
+
name: "opengeni_retained_process_reconciliation_total",
|
|
518
|
+
help: "Bounded retained-process reconciliation observations by fixed app outcome.",
|
|
519
|
+
labels: { outcome },
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
|
|
386
523
|
export function recordCreditMicros(
|
|
387
524
|
observability: Observability | undefined,
|
|
388
525
|
kind: CreditMicrosKind,
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
23
|
import type { Settings } from "@opengeni/config";
|
|
24
|
+
import { redactSensitiveText } from "@opengeni/contracts";
|
|
24
25
|
import {
|
|
25
26
|
acquireLease,
|
|
26
27
|
beginSandboxRematerialization,
|
|
@@ -75,6 +76,13 @@ export function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLe
|
|
|
75
76
|
return `turn-attempt:${normalized}`;
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
export function isRetryableDegradedRestore(restore: {
|
|
80
|
+
status: string;
|
|
81
|
+
retryable?: boolean;
|
|
82
|
+
}): boolean {
|
|
83
|
+
return restore.status === "degraded" && restore.retryable === true;
|
|
84
|
+
}
|
|
85
|
+
|
|
78
86
|
/** The minimal services surface resumeBoxForTurn needs. A subset of
|
|
79
87
|
* ActivityServices so a test (and the API later) can pass a lean bag. */
|
|
80
88
|
export type SandboxResumeServices = {
|
|
@@ -207,6 +215,17 @@ class SnapshotTimeoutError extends Error {
|
|
|
207
215
|
}
|
|
208
216
|
}
|
|
209
217
|
|
|
218
|
+
function safeSnapshotError(error: unknown): { name: string; message: string } {
|
|
219
|
+
const rawName = error instanceof Error ? error.name : "Error";
|
|
220
|
+
return {
|
|
221
|
+
name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
|
|
222
|
+
message: redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(
|
|
223
|
+
0,
|
|
224
|
+
2_048,
|
|
225
|
+
),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
210
229
|
export async function waitForWarmSnapshot(
|
|
211
230
|
snapshot: Promise<unknown>,
|
|
212
231
|
timeoutMs: number,
|
|
@@ -239,7 +258,10 @@ export async function waitForWarmSnapshot(
|
|
|
239
258
|
} catch (error) {
|
|
240
259
|
if (signal?.aborted) return false;
|
|
241
260
|
if (error instanceof SnapshotTimeoutError) {
|
|
242
|
-
console.error(
|
|
261
|
+
console.error(
|
|
262
|
+
"mid-session workspace snapshot wait timed out (turn unaffected)",
|
|
263
|
+
safeSnapshotError(error),
|
|
264
|
+
);
|
|
243
265
|
return false;
|
|
244
266
|
}
|
|
245
267
|
return true;
|
|
@@ -521,7 +543,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
521
543
|
} catch (error) {
|
|
522
544
|
// Protection, not a dependency: a failed snapshot must never fail (or slow
|
|
523
545
|
// down retrying) the turn. The next heartbeat/turn-end tick retries.
|
|
524
|
-
console.error(
|
|
546
|
+
console.error(
|
|
547
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
548
|
+
safeSnapshotError(error),
|
|
549
|
+
);
|
|
525
550
|
return false;
|
|
526
551
|
}
|
|
527
552
|
}
|
|
@@ -963,118 +988,6 @@ export async function resumeBoxForTurn(
|
|
|
963
988
|
}
|
|
964
989
|
}
|
|
965
990
|
|
|
966
|
-
/**
|
|
967
|
-
* Stage D machine-primary: acquire the group lease for a turn whose ACTIVE sandbox
|
|
968
|
-
* is a connected machine (selfhosted) WITHOUT establishing a provider box. The
|
|
969
|
-
* "box" is the user's physical machine reached over NATS; there is NOTHING to
|
|
970
|
-
* spawn, snapshot, expose, or serialize — so we take the lease (the SAME group-keyed
|
|
971
|
-
* refcount/idle/epoch bookkeeping resumeBoxForTurn uses) but skip every box step.
|
|
972
|
-
*
|
|
973
|
-
* The lease is keyed backend "selfhosted": the reaper's terminateProviderBox
|
|
974
|
-
* short-circuits a selfhosted lease (it is the user's machine — NEVER provider-stop),
|
|
975
|
-
* draining it to cold with no stop; and the reaper's warm-meter rate for "selfhosted"
|
|
976
|
-
* is 0 (no cloud seconds billed for a box that does not exist). Returns the lease
|
|
977
|
-
* epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
|
|
978
|
-
* identical to the cloud path.
|
|
979
|
-
*
|
|
980
|
-
* holderId is the globally unique durable turn-attempt id. It must not be a
|
|
981
|
-
* Temporal activity id, because activity ids are only workflow-local and
|
|
982
|
-
* collide when sibling sessions share one sandbox group.
|
|
983
|
-
*/
|
|
984
|
-
export async function acquireSelfhostedLeaseForTurn(
|
|
985
|
-
services: SandboxResumeServices,
|
|
986
|
-
ids: {
|
|
987
|
-
accountId: string;
|
|
988
|
-
workspaceId: string;
|
|
989
|
-
sandboxGroupId: string;
|
|
990
|
-
sessionId: string;
|
|
991
|
-
},
|
|
992
|
-
kind: "turn",
|
|
993
|
-
holderId: TurnSandboxLeaseHolderId,
|
|
994
|
-
): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
|
|
995
|
-
const { db, settings } = services;
|
|
996
|
-
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
997
|
-
|
|
998
|
-
let released = false;
|
|
999
|
-
const release = async (): Promise<void> => {
|
|
1000
|
-
if (released) {
|
|
1001
|
-
return;
|
|
1002
|
-
}
|
|
1003
|
-
released = true;
|
|
1004
|
-
await releaseLeaseHolder(db, {
|
|
1005
|
-
accountId: ids.accountId,
|
|
1006
|
-
workspaceId: ids.workspaceId,
|
|
1007
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1008
|
-
kind,
|
|
1009
|
-
holderId,
|
|
1010
|
-
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1011
|
-
});
|
|
1012
|
-
};
|
|
1013
|
-
|
|
1014
|
-
const acquired = await acquireLease(db, {
|
|
1015
|
-
accountId: ids.accountId,
|
|
1016
|
-
workspaceId: ids.workspaceId,
|
|
1017
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1018
|
-
kind,
|
|
1019
|
-
holderId,
|
|
1020
|
-
subjectId: ids.sessionId,
|
|
1021
|
-
backend: "selfhosted",
|
|
1022
|
-
os: "linux",
|
|
1023
|
-
leaseTtlMs,
|
|
1024
|
-
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
|
|
1025
|
-
});
|
|
1026
|
-
|
|
1027
|
-
// FENCED: a newer epoch re-established the group concurrently. Release our
|
|
1028
|
-
// just-registered holder + surface the supersession (the outer turn catch
|
|
1029
|
-
// requeues, mirroring resumeBoxForTurn).
|
|
1030
|
-
if (acquired.role === "blocked") {
|
|
1031
|
-
await release();
|
|
1032
|
-
throw new SandboxLeaseRecoveryBlockedError(
|
|
1033
|
-
ids.sandboxGroupId,
|
|
1034
|
-
acquired.lease.leaseEpoch,
|
|
1035
|
-
acquired.code,
|
|
1036
|
-
acquired.lease.recovery,
|
|
1037
|
-
);
|
|
1038
|
-
}
|
|
1039
|
-
if (acquired.role === "fenced") {
|
|
1040
|
-
await release();
|
|
1041
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
// SPAWNER: we won the cold->warming CAS. There is NO box to establish — commit
|
|
1045
|
-
// warm immediately so the lease does not linger in 'warming' (no box id, no
|
|
1046
|
-
// resume_state; selfhosted is re-addressed by enrollment via the active pointer,
|
|
1047
|
-
// never resumed from an envelope). resume_backend_id "selfhosted" keeps the
|
|
1048
|
-
// reaper's never-provider-stop short-circuit correct.
|
|
1049
|
-
if (acquired.role === "spawner") {
|
|
1050
|
-
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1051
|
-
const committed = await commitWarmingToWarm(db, {
|
|
1052
|
-
accountId: ids.accountId,
|
|
1053
|
-
workspaceId: ids.workspaceId,
|
|
1054
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1055
|
-
expectedEpoch,
|
|
1056
|
-
// No provider box id; stamp a clearly non-box marker for diagnostics.
|
|
1057
|
-
instanceId: `selfhosted:${ids.sessionId}`,
|
|
1058
|
-
resumeBackendId: "selfhosted",
|
|
1059
|
-
resumeState: null,
|
|
1060
|
-
leaseTtlMs,
|
|
1061
|
-
});
|
|
1062
|
-
if (!committed.committed || !committed.lease) {
|
|
1063
|
-
// A reaper reset our warming row, or a sibling re-established + bumped the
|
|
1064
|
-
// epoch. Release our holder; surface supersession (NEVER touch any box).
|
|
1065
|
-
await release();
|
|
1066
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1067
|
-
}
|
|
1068
|
-
return { leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
// ATTACHED / REARMED: the lease is live. There is no box to resume — just carry
|
|
1072
|
-
// the lease epoch (same-session turns are serialized, so a concurrent 'warming'
|
|
1073
|
-
// spawner on this group is not expected; the machine session is independent of the
|
|
1074
|
-
// lease state regardless).
|
|
1075
|
-
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
1076
|
-
}
|
|
1077
|
-
|
|
1078
991
|
/**
|
|
1079
992
|
* Poll a warming lease until the spawner commits warm. If the warming row is
|
|
1080
993
|
* reset to cold (the spawner died and the reaper reset it), surface supersession
|
|
@@ -1104,6 +1017,12 @@ async function waitForWarm(
|
|
|
1104
1017
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1105
1018
|
}
|
|
1106
1019
|
if (lease.liveness === "cold") {
|
|
1020
|
+
if (isRetryableDegradedRestore(lease.recovery.restore)) {
|
|
1021
|
+
// The elected spawner hit a transient archive verification failure.
|
|
1022
|
+
// Re-enter admission so one caller can become the next fenced spawner;
|
|
1023
|
+
// the archive remains authoritative and must not become terminal.
|
|
1024
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1025
|
+
}
|
|
1107
1026
|
if (
|
|
1108
1027
|
lease.recovery.restore.status === "degraded" ||
|
|
1109
1028
|
lease.recovery.restore.status === "unrecoverable"
|