@opengeni/worker-bundle 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +644 -80
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +32 -4
- package/src/activities/sandbox-lease.ts +130 -3
- package/src/activities/session-state.ts +22 -4
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +197 -59
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -38,14 +38,14 @@
|
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"@opengeni/codex": "^0.2.1",
|
|
41
|
-
"@opengeni/config": "^0.2.
|
|
42
|
-
"@opengeni/contracts": "^0.
|
|
43
|
-
"@opengeni/db": "^0.
|
|
44
|
-
"@opengeni/documents": "^0.2.
|
|
45
|
-
"@opengeni/events": "^0.2.
|
|
41
|
+
"@opengeni/config": "^0.2.5",
|
|
42
|
+
"@opengeni/contracts": "^0.7.0",
|
|
43
|
+
"@opengeni/db": "^0.4.1",
|
|
44
|
+
"@opengeni/documents": "^0.2.5",
|
|
45
|
+
"@opengeni/events": "^0.2.5",
|
|
46
46
|
"@opengeni/observability": "^0.2.1",
|
|
47
|
-
"@opengeni/runtime": "^0.
|
|
48
|
-
"@opengeni/storage": "^0.2.
|
|
47
|
+
"@opengeni/runtime": "^0.3.1",
|
|
48
|
+
"@opengeni/storage": "^0.2.5",
|
|
49
49
|
"@temporalio/activity": "^1.17.0",
|
|
50
50
|
"@temporalio/client": "^1.17.0",
|
|
51
51
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -94,6 +94,7 @@ import type {
|
|
|
94
94
|
} from "./types";
|
|
95
95
|
import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
|
|
96
96
|
import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
|
|
97
|
+
import { recordCreditMicros, runtimeMetricsHooksForObservability, turnLifecycleMetricsFor, type TurnOutcome } from "../observability-metrics";
|
|
97
98
|
import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
|
|
98
99
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
99
100
|
import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
|
|
@@ -510,7 +511,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
510
511
|
"opengeni.workflow_id": input.workflowId,
|
|
511
512
|
"opengeni.trigger_event_id": input.triggerEventId,
|
|
512
513
|
});
|
|
513
|
-
let activityStatus = "unknown";
|
|
514
|
+
let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
|
|
515
|
+
let turnMetricOutcome: TurnOutcome | null = null;
|
|
514
516
|
let activityError: unknown;
|
|
515
517
|
let turnId: string | undefined;
|
|
516
518
|
let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
|
|
@@ -680,6 +682,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
680
682
|
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
681
683
|
}
|
|
682
684
|
turnId = turn.id;
|
|
685
|
+
turnLifecycleMetricsFor(observability).start(turnId);
|
|
683
686
|
// Canonical codex-billed predicate (codex/<slug> + feature enabled + active
|
|
684
687
|
// workspace credential). Computed once and threaded through every billing
|
|
685
688
|
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
@@ -836,6 +839,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
836
839
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
837
840
|
], true);
|
|
838
841
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
842
|
+
turnMetricOutcome = "failed";
|
|
839
843
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
840
844
|
activityStatus = "idle";
|
|
841
845
|
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
@@ -1100,7 +1104,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1100
1104
|
};
|
|
1101
1105
|
} else {
|
|
1102
1106
|
resolvedSandbox = await resumeBoxForTurn(
|
|
1103
|
-
{ db, settings },
|
|
1107
|
+
{ db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
|
|
1104
1108
|
{
|
|
1105
1109
|
accountId: input.accountId,
|
|
1106
1110
|
workspaceId: input.workspaceId,
|
|
@@ -1192,7 +1196,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1192
1196
|
expectedEpoch: heartbeatEpoch,
|
|
1193
1197
|
warmRateMicrosPerSecond: warmRate,
|
|
1194
1198
|
subjectId: input.sessionId,
|
|
1195
|
-
})
|
|
1199
|
+
})
|
|
1200
|
+
.then((result) => recordCreditMicros(observability, "usage", result.costMicros))
|
|
1201
|
+
.catch(() => undefined);
|
|
1196
1202
|
}, 10_000);
|
|
1197
1203
|
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
1198
1204
|
leaseHeartbeatTimer.unref();
|
|
@@ -1335,6 +1341,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1335
1341
|
: { computerToolMode: computerToolModeForTurn(null) }),
|
|
1336
1342
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
1337
1343
|
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
1344
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
1345
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
1346
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
1347
|
+
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
1338
1348
|
...(workspaceEnvironment
|
|
1339
1349
|
? {
|
|
1340
1350
|
workspaceEnvironment: {
|
|
@@ -1530,6 +1540,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1530
1540
|
dispatchId,
|
|
1531
1541
|
positionalKey: `response-${responseUsageCount}`,
|
|
1532
1542
|
}),
|
|
1543
|
+
observability,
|
|
1533
1544
|
});
|
|
1534
1545
|
await reconcileConversationTruth();
|
|
1535
1546
|
try {
|
|
@@ -1584,6 +1595,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1584
1595
|
isCodexTurn,
|
|
1585
1596
|
usage: stream.state.usage,
|
|
1586
1597
|
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1598
|
+
observability,
|
|
1587
1599
|
});
|
|
1588
1600
|
}
|
|
1589
1601
|
if (lastInputTokensObserved !== null) {
|
|
@@ -1636,6 +1648,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1636
1648
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1637
1649
|
], true);
|
|
1638
1650
|
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
1651
|
+
turnMetricOutcome = "completed";
|
|
1639
1652
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1640
1653
|
await recordUsageEvent(db, {
|
|
1641
1654
|
accountId: input.accountId,
|
|
@@ -1731,6 +1744,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1731
1744
|
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1732
1745
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1733
1746
|
activityStatus = "preempted";
|
|
1747
|
+
turnMetricOutcome = "preempted";
|
|
1734
1748
|
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
1735
1749
|
sessionId: input.sessionId,
|
|
1736
1750
|
turnId: activeTurnId,
|
|
@@ -1753,6 +1767,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1753
1767
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1754
1768
|
], true);
|
|
1755
1769
|
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1770
|
+
turnMetricOutcome = "failed";
|
|
1756
1771
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1757
1772
|
activityStatus = "idle";
|
|
1758
1773
|
activityError = attemptError;
|
|
@@ -1804,6 +1819,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1804
1819
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
1805
1820
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1806
1821
|
activityStatus = "preempted";
|
|
1822
|
+
turnMetricOutcome = "preempted";
|
|
1807
1823
|
return { status: "preempted" };
|
|
1808
1824
|
} catch (requeueError) {
|
|
1809
1825
|
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
@@ -1864,6 +1880,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1864
1880
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1865
1881
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1866
1882
|
activityStatus = "preempted";
|
|
1883
|
+
turnMetricOutcome = "preempted";
|
|
1867
1884
|
return { status: "preempted" };
|
|
1868
1885
|
} catch (preemptError) {
|
|
1869
1886
|
// A failing checkpoint/requeue must not surface as an arbitrary
|
|
@@ -1881,6 +1898,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1881
1898
|
await flushRuntimeBatcher();
|
|
1882
1899
|
if (preemptTurnId) {
|
|
1883
1900
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1901
|
+
turnMetricOutcome = "cancelled";
|
|
1884
1902
|
}
|
|
1885
1903
|
throw error;
|
|
1886
1904
|
}
|
|
@@ -1916,6 +1934,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1916
1934
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1917
1935
|
], true);
|
|
1918
1936
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1937
|
+
turnMetricOutcome = "completed";
|
|
1919
1938
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1920
1939
|
await recordUsageEvent(db, {
|
|
1921
1940
|
accountId: input.accountId,
|
|
@@ -2040,6 +2059,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2040
2059
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2041
2060
|
], true);
|
|
2042
2061
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2062
|
+
turnMetricOutcome = "failed";
|
|
2043
2063
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2044
2064
|
activityStatus = "idle";
|
|
2045
2065
|
activityError = error;
|
|
@@ -2092,6 +2112,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2092
2112
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2093
2113
|
], true);
|
|
2094
2114
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2115
|
+
turnMetricOutcome = "completed";
|
|
2095
2116
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2096
2117
|
await recordUsageEvent(db, {
|
|
2097
2118
|
accountId: input.accountId,
|
|
@@ -2142,6 +2163,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2142
2163
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2143
2164
|
], true);
|
|
2144
2165
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2166
|
+
turnMetricOutcome = "failed";
|
|
2145
2167
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2146
2168
|
activityStatus = "idle";
|
|
2147
2169
|
activityError = error;
|
|
@@ -2157,6 +2179,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2157
2179
|
{ type: "session.status.changed", payload: { status: "failed" } },
|
|
2158
2180
|
], true);
|
|
2159
2181
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2182
|
+
turnMetricOutcome = "failed";
|
|
2160
2183
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
2161
2184
|
// The common failure path ends here: runAgentTurn marks the session
|
|
2162
2185
|
// failed and returns "failed", and the session workflow then exits
|
|
@@ -2174,6 +2197,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2174
2197
|
status: activityStatus,
|
|
2175
2198
|
durationSeconds,
|
|
2176
2199
|
});
|
|
2200
|
+
if (turnId && activityStatus !== "unknown") {
|
|
2201
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
2202
|
+
}
|
|
2177
2203
|
activitySpan.end({
|
|
2178
2204
|
attributes: {
|
|
2179
2205
|
"opengeni.turn_id": turnId ?? "",
|
|
@@ -2465,6 +2491,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2465
2491
|
isCodexTurn: boolean;
|
|
2466
2492
|
usage?: ModelUsageInput | null;
|
|
2467
2493
|
sourceKey: string;
|
|
2494
|
+
observability?: ActivityServices["observability"];
|
|
2468
2495
|
}): Promise<void> {
|
|
2469
2496
|
if (!input.usage) {
|
|
2470
2497
|
return;
|
|
@@ -2525,7 +2552,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2525
2552
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
2526
2553
|
});
|
|
2527
2554
|
if (costMicros > 0) {
|
|
2528
|
-
await applyCreditDebitUpToBalance(db, {
|
|
2555
|
+
const result = await applyCreditDebitUpToBalance(db, {
|
|
2529
2556
|
accountId: input.accountId,
|
|
2530
2557
|
workspaceId: input.workspaceId,
|
|
2531
2558
|
type: "model_usage_debit",
|
|
@@ -2543,6 +2570,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2543
2570
|
totalTokens,
|
|
2544
2571
|
},
|
|
2545
2572
|
});
|
|
2573
|
+
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
2546
2574
|
}
|
|
2547
2575
|
}
|
|
2548
2576
|
|
|
@@ -33,8 +33,11 @@
|
|
|
33
33
|
import {
|
|
34
34
|
accrueWarmSeconds,
|
|
35
35
|
confirmDrainCold,
|
|
36
|
+
countQueuedTurns,
|
|
37
|
+
countSandboxLeasesByLiveness,
|
|
36
38
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
37
39
|
getBillingBalance,
|
|
40
|
+
listLiveModalSandboxLeaseAttributions,
|
|
38
41
|
listMeterableWarmLeases,
|
|
39
42
|
persistDrainSnapshot,
|
|
40
43
|
readLease,
|
|
@@ -55,8 +58,16 @@ import {
|
|
|
55
58
|
deletePriorPersistedSnapshot,
|
|
56
59
|
deserializeSandboxSessionStateEnvelope,
|
|
57
60
|
isProviderSandboxNotFoundError,
|
|
61
|
+
sweepModalOrphanSandboxes,
|
|
62
|
+
terminateModalSandboxById,
|
|
58
63
|
} from "@opengeni/runtime";
|
|
59
64
|
import type { ActivityServices } from "./types";
|
|
65
|
+
import {
|
|
66
|
+
recordCreditMicros,
|
|
67
|
+
recordSandboxLeaseGauges,
|
|
68
|
+
recordSandboxOrphansTerminated,
|
|
69
|
+
recordTurnsQueuedGauge,
|
|
70
|
+
} from "../observability-metrics";
|
|
60
71
|
|
|
61
72
|
export type ReapSandboxLeasesResult = {
|
|
62
73
|
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
@@ -73,6 +84,8 @@ export type ReapSandboxLeasesResult = {
|
|
|
73
84
|
/** Viewer-only boxes force-drained because their workspace is over a limit
|
|
74
85
|
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
75
86
|
forceDrained: number;
|
|
87
|
+
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
88
|
+
modalOrphansTerminated: number;
|
|
76
89
|
};
|
|
77
90
|
|
|
78
91
|
// The structural slice of a provider SandboxClient we need to resume-by-id and
|
|
@@ -129,10 +142,19 @@ export type TerminateBoxFn = (
|
|
|
129
142
|
persistArchive: PersistArchiveFn,
|
|
130
143
|
) => Promise<boolean>;
|
|
131
144
|
|
|
145
|
+
export type SweepModalOrphansFn = (
|
|
146
|
+
settings: ActivityServices["settings"],
|
|
147
|
+
db: ActivityServices["db"],
|
|
148
|
+
observability: ActivityServices["observability"],
|
|
149
|
+
) => Promise<number>;
|
|
150
|
+
|
|
132
151
|
export type SandboxLeaseActivityOptions = {
|
|
133
152
|
/** Override the provider terminate (tests spy this; defaults to the real
|
|
134
153
|
* resume-by-id + provider stop()). */
|
|
135
154
|
terminateBox?: TerminateBoxFn;
|
|
155
|
+
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
156
|
+
* Modal list+tag comparison when Modal is configured). */
|
|
157
|
+
sweepModalOrphans?: SweepModalOrphansFn;
|
|
136
158
|
};
|
|
137
159
|
|
|
138
160
|
export function createSandboxLeaseActivities(
|
|
@@ -140,6 +162,7 @@ export function createSandboxLeaseActivities(
|
|
|
140
162
|
options: SandboxLeaseActivityOptions = {},
|
|
141
163
|
) {
|
|
142
164
|
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
165
|
+
const sweepModalOrphans: SweepModalOrphansFn = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
143
166
|
/**
|
|
144
167
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
145
168
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -149,8 +172,9 @@ export function createSandboxLeaseActivities(
|
|
|
149
172
|
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
150
173
|
const { db, settings, observability } = await services();
|
|
151
174
|
if (!settings.sandboxOwnershipEnabled) {
|
|
152
|
-
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
175
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
153
176
|
}
|
|
177
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
154
178
|
|
|
155
179
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
156
180
|
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
@@ -172,6 +196,7 @@ export function createSandboxLeaseActivities(
|
|
|
172
196
|
|
|
173
197
|
let terminated = 0;
|
|
174
198
|
let skipped = 0;
|
|
199
|
+
let modalOrphansTerminated = 0;
|
|
175
200
|
|
|
176
201
|
// (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
|
|
177
202
|
// are isolated: one box's provider error must not abort the whole sweep (the
|
|
@@ -195,17 +220,29 @@ export function createSandboxLeaseActivities(
|
|
|
195
220
|
}
|
|
196
221
|
}
|
|
197
222
|
|
|
198
|
-
|
|
223
|
+
try {
|
|
224
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
225
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
226
|
+
} catch (error) {
|
|
227
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
228
|
+
error: error instanceof Error ? error.message : String(error),
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
233
|
+
|
|
234
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
199
235
|
observability.info("sandbox reaper swept", {
|
|
200
236
|
drainable: drainable.length,
|
|
201
237
|
terminated,
|
|
202
238
|
skipped,
|
|
203
239
|
metered: metered.accrued,
|
|
204
240
|
forceDrained,
|
|
241
|
+
modalOrphansTerminated,
|
|
205
242
|
});
|
|
206
243
|
}
|
|
207
244
|
|
|
208
|
-
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
245
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
|
|
209
246
|
}
|
|
210
247
|
|
|
211
248
|
return { reapSandboxLeases };
|
|
@@ -250,6 +287,7 @@ async function accrueWarmTick(
|
|
|
250
287
|
if (result.accrued) {
|
|
251
288
|
accrued += 1;
|
|
252
289
|
}
|
|
290
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
253
291
|
} catch (error) {
|
|
254
292
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
255
293
|
workspaceId: lease.workspaceId,
|
|
@@ -261,6 +299,26 @@ async function accrueWarmTick(
|
|
|
261
299
|
return { accrued, workspaceIds };
|
|
262
300
|
}
|
|
263
301
|
|
|
302
|
+
async function refreshQueueAndLeaseGauges(
|
|
303
|
+
db: ActivityServices["db"],
|
|
304
|
+
observability: ActivityServices["observability"],
|
|
305
|
+
): Promise<void> {
|
|
306
|
+
try {
|
|
307
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
308
|
+
} catch (error) {
|
|
309
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
310
|
+
error: error instanceof Error ? error.message : String(error),
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
try {
|
|
314
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
315
|
+
} catch (error) {
|
|
316
|
+
observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
|
|
317
|
+
error: error instanceof Error ? error.message : String(error),
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
264
322
|
/**
|
|
265
323
|
* The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
|
|
266
324
|
* live warm box, under the usage lock: if it is at 0 balance (when a billing /
|
|
@@ -310,6 +368,46 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
310
368
|
return forceDrained;
|
|
311
369
|
}
|
|
312
370
|
|
|
371
|
+
async function sweepModalOrphansForConfiguredBackend(
|
|
372
|
+
settings: ActivityServices["settings"],
|
|
373
|
+
db: ActivityServices["db"],
|
|
374
|
+
observability: ActivityServices["observability"],
|
|
375
|
+
): Promise<number> {
|
|
376
|
+
// Keep the provider-side sweep tightly scoped to deployments that have a Modal
|
|
377
|
+
// app path configured. Local/docker-only workers should not attempt Modal API
|
|
378
|
+
// calls just because modalAppName has a default.
|
|
379
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
380
|
+
return 0;
|
|
381
|
+
}
|
|
382
|
+
let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;
|
|
383
|
+
try {
|
|
384
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
385
|
+
} catch (error) {
|
|
386
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
387
|
+
error: error instanceof Error ? error.message : String(error),
|
|
388
|
+
});
|
|
389
|
+
return 0;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases);
|
|
393
|
+
for (const terminated of result.terminated) {
|
|
394
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
395
|
+
sandboxId: terminated.sandboxId,
|
|
396
|
+
reason: terminated.reason,
|
|
397
|
+
tags: JSON.stringify(terminated.tags),
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
if (result.examined > 0) {
|
|
401
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
402
|
+
examined: result.examined,
|
|
403
|
+
terminated: result.terminated.length,
|
|
404
|
+
skipped: result.skipped,
|
|
405
|
+
appName: settings.modalAppName,
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
return result.terminated.length;
|
|
409
|
+
}
|
|
410
|
+
|
|
313
411
|
/**
|
|
314
412
|
* Terminate one drainable box by id, then CAS its lease draining->cold under the
|
|
315
413
|
* epoch fence. Returns true when the lease went cold (the box is ours to stop
|
|
@@ -458,6 +556,35 @@ export async function terminateProviderBox(
|
|
|
458
556
|
return true;
|
|
459
557
|
}
|
|
460
558
|
|
|
559
|
+
// A warming-death row can have a provider instance id recorded immediately
|
|
560
|
+
// after create() but no resumable envelope yet. For Modal, instance_id is
|
|
561
|
+
// enough to terminate directly; CAS-check first so a re-arm during the sweep
|
|
562
|
+
// leaves the box running.
|
|
563
|
+
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
564
|
+
const { wrote } = await persistArchive(null);
|
|
565
|
+
if (!wrote) {
|
|
566
|
+
observability.info("sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING", {
|
|
567
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
568
|
+
backend,
|
|
569
|
+
instanceId: lease.instanceId,
|
|
570
|
+
});
|
|
571
|
+
return false;
|
|
572
|
+
}
|
|
573
|
+
try {
|
|
574
|
+
await terminateModalSandboxById(settings, lease.instanceId);
|
|
575
|
+
} catch (error) {
|
|
576
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
577
|
+
throw error;
|
|
578
|
+
}
|
|
579
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
580
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
581
|
+
backend,
|
|
582
|
+
instanceId: lease.instanceId,
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
return true;
|
|
586
|
+
}
|
|
587
|
+
|
|
461
588
|
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
462
589
|
// box was last persisted as). No envelope -> no live box to stop.
|
|
463
590
|
if (!lease.resumeState) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
claimNextQueuedTurn as claimNextQueuedTurnDb,
|
|
3
|
+
countQueuedTurns,
|
|
3
4
|
countTurnSessionHistoryItems,
|
|
4
5
|
finishTurn,
|
|
5
6
|
getSessionEvent,
|
|
@@ -13,6 +14,7 @@ import { appendAndPublishEvents } from "@opengeni/events";
|
|
|
13
14
|
import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
|
|
14
15
|
import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
|
|
15
16
|
import { notifyParentOfChildTerminal } from "./parent-wake";
|
|
17
|
+
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
16
18
|
import type {
|
|
17
19
|
ActivityServices,
|
|
18
20
|
ClaimNextQueuedTurnInput,
|
|
@@ -67,7 +69,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
|
|
70
|
-
const { db, bus } = await services();
|
|
72
|
+
const { db, bus, observability } = await services();
|
|
71
73
|
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
72
74
|
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
73
75
|
// Pause an active goal before the early return below: an interrupt can
|
|
@@ -96,6 +98,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
96
98
|
]);
|
|
97
99
|
await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
98
100
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
101
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
99
102
|
}
|
|
100
103
|
|
|
101
104
|
/**
|
|
@@ -116,7 +119,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
116
119
|
* WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
|
|
117
120
|
*/
|
|
118
121
|
async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
|
|
119
|
-
const { settings, db, bus } = await services();
|
|
122
|
+
const { settings, db, bus, observability } = await services();
|
|
120
123
|
const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
|
|
121
124
|
if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
|
|
122
125
|
// The timed-out attempt was a zombie that actually settled the turn
|
|
@@ -169,12 +172,15 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
169
172
|
throw requeueError;
|
|
170
173
|
}
|
|
171
174
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
175
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
172
176
|
return { action: "requeued", redispatches };
|
|
173
177
|
}
|
|
174
178
|
|
|
175
179
|
async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
|
|
176
|
-
const { db } = await services();
|
|
177
|
-
|
|
180
|
+
const { db, observability } = await services();
|
|
181
|
+
const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
182
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
183
|
+
return turn;
|
|
178
184
|
}
|
|
179
185
|
|
|
180
186
|
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
@@ -183,6 +189,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
183
189
|
if (session.status === "queued" || session.status === "running") {
|
|
184
190
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
185
191
|
}
|
|
192
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
186
193
|
// The workflow reaches markSessionIdle exactly when it has decided to stop
|
|
187
194
|
// for now (no queued turn, no goal continuation): the terminal-for-now
|
|
188
195
|
// point for a spawned worker, whatever the cause (goal completed, agent or
|
|
@@ -199,3 +206,14 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
199
206
|
markSessionIdle,
|
|
200
207
|
};
|
|
201
208
|
}
|
|
209
|
+
|
|
210
|
+
async function refreshQueuedTurnsGauge(
|
|
211
|
+
db: ActivityServices["db"],
|
|
212
|
+
observability: ActivityServices["observability"],
|
|
213
|
+
): Promise<void> {
|
|
214
|
+
try {
|
|
215
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
216
|
+
} catch {
|
|
217
|
+
// Best-effort telemetry; session state transitions remain authoritative.
|
|
218
|
+
}
|
|
219
|
+
}
|
package/src/activities.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
|
|
|
12
12
|
import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
|
|
13
13
|
import { createSessionStateActivities } from "./activities/session-state";
|
|
14
14
|
import type { ActivityDependencies, ActivityServices } from "./activities/types";
|
|
15
|
+
import { observabilityEventLogger, runtimeMetricsHooksForObservability } from "./observability-metrics";
|
|
15
16
|
|
|
16
17
|
export type {
|
|
17
18
|
ActivityDependencies,
|
|
@@ -33,6 +34,7 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
|
33
34
|
async function services(): Promise<ActivityServices> {
|
|
34
35
|
servicesPromise ??= (async () => {
|
|
35
36
|
const settings = dependencies.settings ?? getSettings();
|
|
37
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
36
38
|
// Step I: when not injected, build the standalone handle — searchPath
|
|
37
39
|
// undefined for standalone (public), scoped to the dedicated schema +
|
|
38
40
|
// host RLS strategy when embedded config is set. An embedded host injects
|
|
@@ -67,11 +69,14 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
|
67
69
|
(await createNatsEventBus(
|
|
68
70
|
settings.natsUrl,
|
|
69
71
|
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
72
|
+
{ logger: observabilityEventLogger(observability) },
|
|
70
73
|
)),
|
|
71
|
-
runtime: dependencies.runtime ?? createProductionAgentRuntime(
|
|
74
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime({
|
|
75
|
+
metrics: runtimeMetricsHooksForObservability(observability),
|
|
76
|
+
}),
|
|
72
77
|
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
73
78
|
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
74
|
-
observability
|
|
79
|
+
observability,
|
|
75
80
|
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
76
81
|
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
77
82
|
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import { dbSql, type Database } from "@opengeni/db";
|
|
3
|
+
import type { EventBus } from "@opengeni/events";
|
|
4
|
+
import type { Observability } from "@opengeni/observability";
|
|
5
|
+
|
|
6
|
+
export type ReadinessCheckName = "db" | "nats" | "temporal";
|
|
7
|
+
export type ReadinessChecks = Record<ReadinessCheckName, () => Promise<void> | void>;
|
|
8
|
+
|
|
9
|
+
export type ReadinessResult = {
|
|
10
|
+
ok: boolean;
|
|
11
|
+
checks: Record<ReadinessCheckName, { ok: boolean; error?: string }>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export function startWorkerHttpServer(input: {
|
|
15
|
+
settings: Settings;
|
|
16
|
+
observability: Observability;
|
|
17
|
+
checks: ReadinessChecks;
|
|
18
|
+
timeoutMs?: number;
|
|
19
|
+
}): ReturnType<typeof Bun.serve> {
|
|
20
|
+
const { settings, observability, checks } = input;
|
|
21
|
+
const timeoutMs = input.timeoutMs ?? 2_000;
|
|
22
|
+
return Bun.serve({
|
|
23
|
+
hostname: "0.0.0.0",
|
|
24
|
+
port: settings.workerHttpPort,
|
|
25
|
+
async fetch(request) {
|
|
26
|
+
const url = new URL(request.url);
|
|
27
|
+
if (request.method !== "GET") {
|
|
28
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
29
|
+
}
|
|
30
|
+
if (url.pathname === "/healthz") {
|
|
31
|
+
return Response.json({
|
|
32
|
+
service: settings.serviceName,
|
|
33
|
+
environment: settings.environment,
|
|
34
|
+
deploymentRevision: settings.deploymentRevision,
|
|
35
|
+
ok: true,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
if (url.pathname === "/metrics") {
|
|
39
|
+
return new Response(await observability.prometheusMetrics(), {
|
|
40
|
+
status: 200,
|
|
41
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
if (url.pathname === "/readyz") {
|
|
45
|
+
const result = await runReadinessChecks(checks, timeoutMs);
|
|
46
|
+
return Response.json(result, { status: result.ok ? 200 : 503 });
|
|
47
|
+
}
|
|
48
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function runReadinessChecks(checks: ReadinessChecks, timeoutMs = 2_000): Promise<ReadinessResult> {
|
|
54
|
+
const entries = await Promise.all(
|
|
55
|
+
(Object.entries(checks) as Array<[ReadinessCheckName, () => Promise<void> | void]>)
|
|
56
|
+
.map(async ([name, check]) => {
|
|
57
|
+
try {
|
|
58
|
+
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
59
|
+
return [name, { ok: true }] as const;
|
|
60
|
+
} catch (error) {
|
|
61
|
+
return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }] as const;
|
|
62
|
+
}
|
|
63
|
+
}),
|
|
64
|
+
);
|
|
65
|
+
const result = Object.fromEntries(entries) as ReadinessResult["checks"];
|
|
66
|
+
return {
|
|
67
|
+
ok: Object.values(result).every((check) => check.ok),
|
|
68
|
+
checks: result,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function dbReadyCheck(db: Database): () => Promise<void> {
|
|
73
|
+
return async () => {
|
|
74
|
+
await db.execute(dbSql`select 1`);
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function natsReadyCheck(bus: EventBus): () => void {
|
|
79
|
+
return () => {
|
|
80
|
+
if (bus.isConnected && !bus.isConnected()) {
|
|
81
|
+
throw new Error("NATS is not connected");
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
87
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
88
|
+
try {
|
|
89
|
+
return await Promise.race([
|
|
90
|
+
promise,
|
|
91
|
+
new Promise<never>((_, reject) => {
|
|
92
|
+
timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
93
|
+
}),
|
|
94
|
+
]);
|
|
95
|
+
} finally {
|
|
96
|
+
if (timer) {
|
|
97
|
+
clearTimeout(timer);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|