@opengeni/worker-bundle 0.13.10 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +3 -3
- package/dist/index.d.ts +2 -2
- package/dist/index.js +122 -75
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +7 -1
- package/dist/sandbox-routing.d.ts +3 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +30 -7
- package/src/activities/parent-wake.ts +8 -0
- package/src/activities/sandbox-lease.ts +68 -44
- package/src/activities/scheduled-tasks.ts +9 -0
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +38 -26
- package/src/sandbox-routing.ts +5 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type SessionEventType } from "@opengeni/contracts";
|
|
2
2
|
import type { EventLogger } from "@opengeni/events";
|
|
3
3
|
import type { Observability } from "@opengeni/observability";
|
|
4
4
|
import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
|
|
@@ -66,6 +66,10 @@ export declare class TurnLifecycleMetrics {
|
|
|
66
66
|
}
|
|
67
67
|
export declare function recordTurnsQueuedGauge(observability: Observability, value: number): void;
|
|
68
68
|
export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
|
|
69
|
+
export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "rotation_backlog", "retained_processes", "expired_drains"];
|
|
70
|
+
export type SandboxInventoryProjectionDomain = (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
|
|
71
|
+
export declare function recordSandboxInventoryProjectionSuccess(observability: Observability, domain: SandboxInventoryProjectionDomain, timestampSeconds?: number): void;
|
|
72
|
+
export declare function recordSandboxInventoryProjectionFailure(observability: Observability, domain: SandboxInventoryProjectionDomain): void;
|
|
69
73
|
export declare function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void;
|
|
70
74
|
export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
|
|
71
75
|
export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
|
|
@@ -85,6 +89,8 @@ export declare function recordRetainedProcessInventoryGauges(observability: Obse
|
|
|
85
89
|
activeCount: number;
|
|
86
90
|
terminalOwnerCount: number;
|
|
87
91
|
}>): void;
|
|
92
|
+
export declare const EXPIRED_DRAINING_BACKENDS: readonly [...("blaxel" | "cloudflare" | "daytona" | "docker" | "e2b" | "local" | "modal" | "none" | "runloop" | "selfhosted" | "vercel")[], "unknown"];
|
|
93
|
+
export declare const EXPIRED_DRAINING_AGE_BUCKETS: readonly ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
|
|
88
94
|
export declare function recordExpiredDrainingSandboxLeaseGauges(observability: Observability, counts: Array<{
|
|
89
95
|
backend: string;
|
|
90
96
|
ageBucket: string;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type Settings } from "@opengeni/config";
|
|
2
2
|
import { type Database } from "@opengeni/db";
|
|
3
3
|
import type { EventBus } from "@opengeni/events";
|
|
4
|
-
import { type EstablishedSandboxSession, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
|
|
4
|
+
import { type EstablishedSandboxSession, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
|
|
5
5
|
export type RoutingWiringServices = {
|
|
6
6
|
db: Database;
|
|
7
7
|
settings: Settings;
|
|
@@ -12,6 +12,8 @@ export type RoutingWiringServices = {
|
|
|
12
12
|
/** The per-op observer wired into every selfhosted session this turn builds
|
|
13
13
|
* (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
|
|
14
14
|
onOp?: SelfhostedOpObserver;
|
|
15
|
+
/** Every physical routed provider call, across cloud and selfhosted homes. */
|
|
16
|
+
onSandboxOperation?: RoutingSandboxOperationObserver;
|
|
15
17
|
/** The op-stream durable-resume journal (the Temporal adaptation from
|
|
16
18
|
* op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
|
|
17
19
|
* the runtime defaults (generation "1", no persistence) — tests / non-turn
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.1",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -41,16 +41,16 @@
|
|
|
41
41
|
"dependencies": {
|
|
42
42
|
"@opengeni/agent-proto": "^0.3.0",
|
|
43
43
|
"@opengeni/codex": "^0.2.9",
|
|
44
|
-
"@opengeni/config": "^0.9.
|
|
45
|
-
"@opengeni/contracts": "^0.
|
|
46
|
-
"@opengeni/core": "^0.
|
|
47
|
-
"@opengeni/db": "^0.
|
|
48
|
-
"@opengeni/documents": "^0.2.
|
|
49
|
-
"@opengeni/events": "^0.3.
|
|
50
|
-
"@opengeni/github": "^0.4.
|
|
51
|
-
"@opengeni/observability": "^0.
|
|
52
|
-
"@opengeni/runtime": "^0.
|
|
53
|
-
"@opengeni/storage": "^0.2.
|
|
44
|
+
"@opengeni/config": "^0.9.3",
|
|
45
|
+
"@opengeni/contracts": "^0.30.0",
|
|
46
|
+
"@opengeni/core": "^0.17.1",
|
|
47
|
+
"@opengeni/db": "^0.21.0",
|
|
48
|
+
"@opengeni/documents": "^0.2.66",
|
|
49
|
+
"@opengeni/events": "^0.3.57",
|
|
50
|
+
"@opengeni/github": "^0.4.15",
|
|
51
|
+
"@opengeni/observability": "^0.4.2",
|
|
52
|
+
"@opengeni/runtime": "^0.16.2",
|
|
53
|
+
"@opengeni/storage": "^0.2.52",
|
|
54
54
|
"@temporalio/activity": "^1.17.0",
|
|
55
55
|
"@temporalio/client": "^1.17.0",
|
|
56
56
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
CODEX_CREDENTIAL_LEASE_TTL_MS,
|
|
28
28
|
getCodexRotationSettings,
|
|
29
29
|
getWorkspaceModelPolicy,
|
|
30
|
+
getWorkspaceGrant,
|
|
30
31
|
listCodexAccountStatuses,
|
|
31
32
|
fetchCodexUsageForAccount,
|
|
32
33
|
getSessionCodexState,
|
|
@@ -78,6 +79,7 @@ import {
|
|
|
78
79
|
type SessionTurnRecordingSettlement,
|
|
79
80
|
} from "@opengeni/db";
|
|
80
81
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
82
|
+
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
81
83
|
import {
|
|
82
84
|
sandboxStateEntryFromRunState,
|
|
83
85
|
maxTurnsExceededRunState,
|
|
@@ -200,6 +202,8 @@ import { mergeResourceRefs } from "./common";
|
|
|
200
202
|
import {
|
|
201
203
|
assertSessionAllowsProductModel,
|
|
202
204
|
defaultSessionMcpServerIds,
|
|
205
|
+
directPersonalConnectionSubjectId,
|
|
206
|
+
withFrozenPersonalConnectionDelegations,
|
|
203
207
|
resolveSessionToolPolicy,
|
|
204
208
|
} from "@opengeni/core";
|
|
205
209
|
import { maybeCompactContext, settleFailedContextCompactionLandmark } from "./context-compaction";
|
|
@@ -326,7 +330,6 @@ import {
|
|
|
326
330
|
type SessionTurn,
|
|
327
331
|
type ToolAuthNeededPayload,
|
|
328
332
|
type TurnExecutionPolicyV1,
|
|
329
|
-
type TurnInitiator,
|
|
330
333
|
} from "@opengeni/contracts";
|
|
331
334
|
import { createHash, randomUUID } from "node:crypto";
|
|
332
335
|
|
|
@@ -336,11 +339,11 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
336
339
|
// budget against the same window.
|
|
337
340
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
338
341
|
|
|
339
|
-
/**
|
|
342
|
+
/** Broad personal lookup is allowed only for a direct human/API command. */
|
|
340
343
|
export function credentialSubjectIdForTurnInitiator(
|
|
341
|
-
|
|
344
|
+
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
342
345
|
): string | undefined {
|
|
343
|
-
return
|
|
346
|
+
return directPersonalConnectionSubjectId(turn);
|
|
344
347
|
}
|
|
345
348
|
|
|
346
349
|
/**
|
|
@@ -352,7 +355,7 @@ export function credentialSubjectIdForTurnInitiator(
|
|
|
352
355
|
export function isDirectHumanTurnInitiation(
|
|
353
356
|
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
354
357
|
): boolean {
|
|
355
|
-
if (turn.source !== "user" || turn.initiator.kind !== "subject") {
|
|
358
|
+
if ((turn.source !== "user" && turn.source !== "api") || turn.initiator.kind !== "subject") {
|
|
356
359
|
return false;
|
|
357
360
|
}
|
|
358
361
|
return !["via", "viaTruncated", "provenanceError", "backfill"].some((key) =>
|
|
@@ -2297,6 +2300,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2297
2300
|
const machineOpObserver = makeMachineOpObserver(
|
|
2298
2301
|
runtimeMetricsHooksForObservability(observability),
|
|
2299
2302
|
);
|
|
2303
|
+
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
2300
2304
|
let isCodexTurn = false;
|
|
2301
2305
|
let isExternallyBilledTurn = false;
|
|
2302
2306
|
let executionGeneration = 0;
|
|
@@ -5162,6 +5166,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5162
5166
|
settings,
|
|
5163
5167
|
bus,
|
|
5164
5168
|
onOp: machineOpObserver.observer,
|
|
5169
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5165
5170
|
opJournal,
|
|
5166
5171
|
},
|
|
5167
5172
|
{
|
|
@@ -5190,6 +5195,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5190
5195
|
settings,
|
|
5191
5196
|
bus,
|
|
5192
5197
|
onOp: machineOpObserver.observer,
|
|
5198
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5193
5199
|
onHomeSandboxRebound,
|
|
5194
5200
|
},
|
|
5195
5201
|
{
|
|
@@ -5297,6 +5303,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5297
5303
|
db,
|
|
5298
5304
|
settings,
|
|
5299
5305
|
bus,
|
|
5306
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5300
5307
|
onHomeSandboxLost: publishSandboxLost,
|
|
5301
5308
|
onHomeSandboxRebound,
|
|
5302
5309
|
},
|
|
@@ -5363,8 +5370,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5363
5370
|
attemptId: input.attemptId,
|
|
5364
5371
|
turn,
|
|
5365
5372
|
});
|
|
5373
|
+
const personalConnectionDelegations = turn.personalConnectionDelegations;
|
|
5374
|
+
const delegatedMembershipChecks = new Map<string, Promise<boolean>>();
|
|
5375
|
+
const delegatedOwnerHasMembership = async (subjectId: string): Promise<boolean> => {
|
|
5376
|
+
const existing = delegatedMembershipChecks.get(subjectId);
|
|
5377
|
+
if (existing) return await existing;
|
|
5378
|
+
const check = getWorkspaceGrant(db, subjectId, input.workspaceId).then(Boolean);
|
|
5379
|
+
delegatedMembershipChecks.set(subjectId, check);
|
|
5380
|
+
return await check;
|
|
5381
|
+
};
|
|
5382
|
+
const resolveFrozenCredential = withFrozenPersonalConnectionDelegations({
|
|
5383
|
+
resolveCredential: rawResolveCredential,
|
|
5384
|
+
settings: runSettings,
|
|
5385
|
+
personalConnectionDelegations,
|
|
5386
|
+
ownerHasWorkspaceMembership: delegatedOwnerHasMembership,
|
|
5387
|
+
});
|
|
5366
5388
|
const resolveCredential: typeof rawResolveCredential = async (request) => {
|
|
5367
|
-
const result = await
|
|
5389
|
+
const result = await resolveFrozenCredential(request);
|
|
5368
5390
|
if (result.status === "ok") {
|
|
5369
5391
|
registerSecretRedactions(
|
|
5370
5392
|
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
|
|
@@ -5372,7 +5394,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5372
5394
|
}
|
|
5373
5395
|
return result;
|
|
5374
5396
|
};
|
|
5375
|
-
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn
|
|
5397
|
+
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
|
|
5376
5398
|
preparedTools = await waitForTurnOperation(
|
|
5377
5399
|
withCodex(() =>
|
|
5378
5400
|
runtime.prepareTools(runSettings, turnTools, {
|
|
@@ -5786,6 +5808,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5786
5808
|
settings,
|
|
5787
5809
|
bus,
|
|
5788
5810
|
onOp: machineOpObserver.observer,
|
|
5811
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5789
5812
|
onHomeSandboxLost: publishSandboxLost,
|
|
5790
5813
|
onHomeSandboxRebound,
|
|
5791
5814
|
},
|
|
@@ -3,6 +3,7 @@ import type { Session, SessionGoal, SessionSystemUpdatePayload } from "@opengeni
|
|
|
3
3
|
import {
|
|
4
4
|
getSession,
|
|
5
5
|
getSessionGoal,
|
|
6
|
+
getSessionParentPersonalConnectionDelegations,
|
|
6
7
|
addSessionSystemUpdateWithSourceMutation,
|
|
7
8
|
claimPendingSessionSystemUpdateOutbox,
|
|
8
9
|
claimPendingSessionWorkflowWakes,
|
|
@@ -57,6 +58,11 @@ export async function notifyParentOfChildIdle(
|
|
|
57
58
|
// guard so a nudge that slips through still cannot revive the goal.
|
|
58
59
|
const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
|
|
59
60
|
const payload = childCompletionPayload(child, goal);
|
|
61
|
+
const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
|
|
62
|
+
svc.db,
|
|
63
|
+
workspaceId,
|
|
64
|
+
childSessionId,
|
|
65
|
+
);
|
|
60
66
|
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
61
67
|
accountId: child.accountId,
|
|
62
68
|
workspaceId,
|
|
@@ -69,6 +75,7 @@ export async function notifyParentOfChildIdle(
|
|
|
69
75
|
summary: childCompletionSummary(child, goal, "idle"),
|
|
70
76
|
payload,
|
|
71
77
|
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
|
|
78
|
+
personalConnectionDelegations,
|
|
72
79
|
});
|
|
73
80
|
if (outbox.status === "delivered") {
|
|
74
81
|
return;
|
|
@@ -138,6 +145,7 @@ async function deliverParentSystemUpdateOutbox(
|
|
|
138
145
|
summary: outbox.summary,
|
|
139
146
|
payload: outbox.payload,
|
|
140
147
|
lineage: outbox.lineage,
|
|
148
|
+
personalConnectionDelegations: outbox.personalConnectionDelegations,
|
|
141
149
|
},
|
|
142
150
|
async (tx) => {
|
|
143
151
|
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
@@ -115,7 +115,10 @@ import {
|
|
|
115
115
|
recordSandboxCheckpointArtifactGauges,
|
|
116
116
|
recordSandboxCheckpointArtifactOutcome,
|
|
117
117
|
recordSandboxDeadlineRotationsRequested,
|
|
118
|
+
recordSandboxInventoryProjectionFailure,
|
|
119
|
+
recordSandboxInventoryProjectionSuccess,
|
|
118
120
|
type RetainedProcessReconciliationOutcome,
|
|
121
|
+
type SandboxInventoryProjectionDomain,
|
|
119
122
|
recordSandboxLeaseGauges,
|
|
120
123
|
recordSandboxOrphansTerminated,
|
|
121
124
|
recordSandboxRotationBacklogGauges,
|
|
@@ -286,9 +289,9 @@ export function createSandboxLeaseActivities(
|
|
|
286
289
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
287
290
|
/**
|
|
288
291
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
289
|
-
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
+
* The global Schedule always runs. A defensive mutation gate here ensures
|
|
293
|
+
* that an ownership-off deployment still refreshes read-only metrics and
|
|
294
|
+
* repairs the system-update outbox but can never terminate a box.
|
|
292
295
|
*/
|
|
293
296
|
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
294
297
|
const service = await services();
|
|
@@ -312,6 +315,11 @@ export function createSandboxLeaseActivities(
|
|
|
312
315
|
deleted: expiredSessionListSnapshots,
|
|
313
316
|
});
|
|
314
317
|
}
|
|
318
|
+
// Inventory is useful even while provider ownership is disabled, and the
|
|
319
|
+
// same globally scheduled activity remains its single projection owner.
|
|
320
|
+
// Refresh before the mutation gate so dashboards show truthful zero/history
|
|
321
|
+
// state and absence alerts do not misclassify an intentional rollout flag.
|
|
322
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
315
323
|
if (!settings.sandboxOwnershipEnabled) {
|
|
316
324
|
if (parentUpdates.claimed > 0) {
|
|
317
325
|
observability.info("system-update outbox reconciled", parentUpdates);
|
|
@@ -325,7 +333,6 @@ export function createSandboxLeaseActivities(
|
|
|
325
333
|
modalOrphansTerminated: 0,
|
|
326
334
|
};
|
|
327
335
|
}
|
|
328
|
-
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
329
336
|
|
|
330
337
|
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
331
338
|
db,
|
|
@@ -1218,54 +1225,71 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
1218
1225
|
error: error instanceof Error ? error.message : String(error),
|
|
1219
1226
|
});
|
|
1220
1227
|
}
|
|
1221
|
-
|
|
1228
|
+
await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
|
|
1222
1229
|
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
1223
|
-
}
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1230
|
+
});
|
|
1231
|
+
await refreshSandboxInventoryGauge(
|
|
1232
|
+
observability,
|
|
1233
|
+
"checkpoint_artifacts",
|
|
1234
|
+
"checkpoint-artifact",
|
|
1235
|
+
async () => {
|
|
1236
|
+
recordSandboxCheckpointArtifactGauges(
|
|
1237
|
+
observability,
|
|
1238
|
+
await countSandboxCheckpointArtifactsByState(db),
|
|
1239
|
+
);
|
|
1240
|
+
},
|
|
1241
|
+
);
|
|
1242
|
+
await refreshSandboxInventoryGauge(
|
|
1243
|
+
observability,
|
|
1244
|
+
"rotation_backlog",
|
|
1245
|
+
"rotation-backlog",
|
|
1246
|
+
async () => {
|
|
1247
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
1248
|
+
},
|
|
1249
|
+
);
|
|
1250
|
+
await refreshSandboxInventoryGauge(
|
|
1251
|
+
observability,
|
|
1252
|
+
"retained_processes",
|
|
1253
|
+
"retained-process",
|
|
1254
|
+
async () => {
|
|
1255
|
+
recordRetainedProcessInventoryGauges(
|
|
1256
|
+
observability,
|
|
1257
|
+
await countActiveRetainedProcessesByOwnerState(db),
|
|
1258
|
+
);
|
|
1259
|
+
},
|
|
1260
|
+
);
|
|
1261
|
+
await refreshSandboxInventoryGauge(
|
|
1262
|
+
observability,
|
|
1263
|
+
"expired_drains",
|
|
1264
|
+
"expired-draining",
|
|
1265
|
+
async () => {
|
|
1266
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
1267
|
+
observability,
|
|
1268
|
+
await countExpiredDrainingSandboxLeases(db),
|
|
1269
|
+
);
|
|
1270
|
+
},
|
|
1271
|
+
);
|
|
1255
1272
|
try {
|
|
1256
|
-
|
|
1257
|
-
observability,
|
|
1258
|
-
await countExpiredDrainingSandboxLeases(db),
|
|
1259
|
-
);
|
|
1273
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
1260
1274
|
} catch (error) {
|
|
1261
|
-
observability.warn("sandbox reaper:
|
|
1275
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
1262
1276
|
error: error instanceof Error ? error.message : String(error),
|
|
1263
1277
|
});
|
|
1264
1278
|
}
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
async function refreshSandboxInventoryGauge(
|
|
1282
|
+
observability: ActivityServices["observability"],
|
|
1283
|
+
domain: SandboxInventoryProjectionDomain,
|
|
1284
|
+
diagnosticName: string,
|
|
1285
|
+
refresh: () => Promise<void>,
|
|
1286
|
+
): Promise<void> {
|
|
1265
1287
|
try {
|
|
1266
|
-
|
|
1288
|
+
await refresh();
|
|
1289
|
+
recordSandboxInventoryProjectionSuccess(observability, domain);
|
|
1267
1290
|
} catch (error) {
|
|
1268
|
-
observability
|
|
1291
|
+
recordSandboxInventoryProjectionFailure(observability, domain);
|
|
1292
|
+
observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
|
|
1269
1293
|
error: error instanceof Error ? error.message : String(error),
|
|
1270
1294
|
});
|
|
1271
1295
|
}
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
18
18
|
getBillingBalance,
|
|
19
19
|
getScheduledTask,
|
|
20
|
+
getScheduledTaskPersonalConnectionDelegations,
|
|
20
21
|
getRig,
|
|
21
22
|
getVariableSet,
|
|
22
23
|
isCodexBilledTurn,
|
|
@@ -63,6 +64,11 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
63
64
|
if (!task) {
|
|
64
65
|
return { action: "deleted" };
|
|
65
66
|
}
|
|
67
|
+
const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
|
|
68
|
+
db,
|
|
69
|
+
task.workspaceId,
|
|
70
|
+
task.id,
|
|
71
|
+
);
|
|
66
72
|
// A scheduled bot selection was authorized when the task was written,
|
|
67
73
|
// but connection status and tenant/role binding are mutable. Revalidate
|
|
68
74
|
// before any session/model cost and never fall back to a personal Slack
|
|
@@ -237,6 +243,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
237
243
|
variableSetId: task.variableSetId ?? null,
|
|
238
244
|
rigId: frozenRigId,
|
|
239
245
|
rigVersionId: frozenRigVersionId,
|
|
246
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
240
247
|
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
241
248
|
// The durable agent config was privilege-checked when the task
|
|
242
249
|
// was created/updated. Preserve that explicit policy if a broader
|
|
@@ -328,6 +335,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
328
335
|
run.id,
|
|
329
336
|
),
|
|
330
337
|
lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
338
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
331
339
|
},
|
|
332
340
|
async (tx, wakeEventId) => {
|
|
333
341
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
@@ -444,6 +452,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
444
452
|
run.id,
|
|
445
453
|
),
|
|
446
454
|
lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
455
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
447
456
|
},
|
|
448
457
|
async (tx, wakeEventId) => {
|
|
449
458
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
package/src/index.ts
CHANGED
|
@@ -351,8 +351,8 @@ export async function createWorkerWorkflowSignaler(
|
|
|
351
351
|
/**
|
|
352
352
|
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
353
353
|
* driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
|
|
354
|
-
* sandbox ownership off the activity performs
|
|
355
|
-
* it never
|
|
354
|
+
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
355
|
+
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
356
356
|
*
|
|
357
357
|
* The Schedule fires sandboxReaperWorkflow on the worker's global task queue
|
|
358
358
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3
|
-
import type
|
|
3
|
+
import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
|
|
4
4
|
import type { EventLogger } from "@opengeni/events";
|
|
5
5
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
6
|
import {
|
|
@@ -19,7 +19,6 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
|
19
19
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
20
20
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
21
21
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
22
|
-
const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
|
|
23
22
|
|
|
24
23
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
25
24
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -345,6 +344,41 @@ export function recordSandboxLeaseGauges(
|
|
|
345
344
|
}
|
|
346
345
|
}
|
|
347
346
|
|
|
347
|
+
export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
|
|
348
|
+
"leases",
|
|
349
|
+
"checkpoint_artifacts",
|
|
350
|
+
"rotation_backlog",
|
|
351
|
+
"retained_processes",
|
|
352
|
+
"expired_drains",
|
|
353
|
+
] as const;
|
|
354
|
+
|
|
355
|
+
export type SandboxInventoryProjectionDomain =
|
|
356
|
+
(typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
|
|
357
|
+
|
|
358
|
+
export function recordSandboxInventoryProjectionSuccess(
|
|
359
|
+
observability: Observability,
|
|
360
|
+
domain: SandboxInventoryProjectionDomain,
|
|
361
|
+
timestampSeconds = Date.now() / 1_000,
|
|
362
|
+
): void {
|
|
363
|
+
observability.setGauge({
|
|
364
|
+
name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
|
|
365
|
+
help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
|
|
366
|
+
labels: { domain },
|
|
367
|
+
value: timestampSeconds,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
export function recordSandboxInventoryProjectionFailure(
|
|
372
|
+
observability: Observability,
|
|
373
|
+
domain: SandboxInventoryProjectionDomain,
|
|
374
|
+
): void {
|
|
375
|
+
observability.incrementCounter({
|
|
376
|
+
name: "opengeni_sandbox_inventory_refresh_failures_total",
|
|
377
|
+
help: "Failed sandbox inventory projection refreshes by domain.",
|
|
378
|
+
labels: { domain },
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
348
382
|
export function recordCreditBalanceGauges(
|
|
349
383
|
observability: Observability,
|
|
350
384
|
balances: CreditBalanceGauge[],
|
|
@@ -510,9 +544,7 @@ export function recordRetainedProcessInventoryGauges(
|
|
|
510
544
|
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
511
545
|
}
|
|
512
546
|
|
|
513
|
-
let terminalTotal = 0;
|
|
514
547
|
for (const [ownerState, count] of normalized) {
|
|
515
|
-
terminalTotal += count.terminal;
|
|
516
548
|
observability.setGauge({
|
|
517
549
|
name: "opengeni_retained_processes_active",
|
|
518
550
|
help: "Current active retained provider processes by durable owner state.",
|
|
@@ -526,31 +558,11 @@ export function recordRetainedProcessInventoryGauges(
|
|
|
526
558
|
value: count.terminal,
|
|
527
559
|
});
|
|
528
560
|
}
|
|
529
|
-
const previous = retainedProcessTerminalBacklogTotals.get(observability);
|
|
530
|
-
if (previous !== undefined && terminalTotal > previous) {
|
|
531
|
-
observability.incrementCounter({
|
|
532
|
-
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
533
|
-
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
534
|
-
amount: terminalTotal - previous,
|
|
535
|
-
});
|
|
536
|
-
}
|
|
537
|
-
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
538
561
|
}
|
|
539
562
|
|
|
540
|
-
const EXPIRED_DRAINING_BACKENDS = [
|
|
541
|
-
"none",
|
|
542
|
-
"local",
|
|
543
|
-
"docker",
|
|
544
|
-
"modal",
|
|
545
|
-
"e2b",
|
|
546
|
-
"daytona",
|
|
547
|
-
"blaxel",
|
|
548
|
-
"runloop",
|
|
549
|
-
"selfhosted",
|
|
550
|
-
"unknown",
|
|
551
|
-
] as const;
|
|
563
|
+
export const EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"] as const;
|
|
552
564
|
const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
|
|
553
|
-
const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
|
|
565
|
+
export const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
|
|
554
566
|
|
|
555
567
|
export function recordExpiredDrainingSandboxLeaseGauges(
|
|
556
568
|
observability: Observability,
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
type RoutableSandbox,
|
|
53
53
|
type ResolvedActiveBackend,
|
|
54
54
|
type RoutingMutationSettlementResult,
|
|
55
|
+
type RoutingSandboxOperationObserver,
|
|
55
56
|
type RoutingRetainedProcess,
|
|
56
57
|
type RoutingRetainedProcessTerminalProof,
|
|
57
58
|
type SelfhostedOpObserver,
|
|
@@ -78,6 +79,8 @@ export type RoutingWiringServices = {
|
|
|
78
79
|
/** The per-op observer wired into every selfhosted session this turn builds
|
|
79
80
|
* (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
|
|
80
81
|
onOp?: SelfhostedOpObserver;
|
|
82
|
+
/** Every physical routed provider call, across cloud and selfhosted homes. */
|
|
83
|
+
onSandboxOperation?: RoutingSandboxOperationObserver;
|
|
81
84
|
/** The op-stream durable-resume journal (the Temporal adaptation from
|
|
82
85
|
* op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
|
|
83
86
|
* the runtime defaults (generation "1", no persistence) — tests / non-turn
|
|
@@ -784,6 +787,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
784
787
|
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
785
788
|
},
|
|
786
789
|
resolveActiveBackend: resolver,
|
|
790
|
+
...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
|
|
787
791
|
...(beforeMutation ? { beforeMutation } : {}),
|
|
788
792
|
...(afterMutation ? { afterMutation } : {}),
|
|
789
793
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
@@ -927,6 +931,7 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
927
931
|
}
|
|
928
932
|
return routedResolver(pointer);
|
|
929
933
|
},
|
|
934
|
+
...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
|
|
930
935
|
...(beforeMutation ? { beforeMutation } : {}),
|
|
931
936
|
...(afterMutation ? { afterMutation } : {}),
|
|
932
937
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|