@opengeni/worker-bundle 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +431 -42
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +28 -4
- package/src/activities/sandbox-lease.ts +33 -0
- package/src/activities/session-state.ts +22 -4
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +21 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -38,14 +38,14 @@
|
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"@opengeni/codex": "^0.2.1",
|
|
41
|
-
"@opengeni/config": "^0.2.
|
|
42
|
-
"@opengeni/contracts": "^0.
|
|
43
|
-
"@opengeni/db": "^0.4.
|
|
44
|
-
"@opengeni/documents": "^0.2.
|
|
45
|
-
"@opengeni/events": "^0.2.
|
|
41
|
+
"@opengeni/config": "^0.2.5",
|
|
42
|
+
"@opengeni/contracts": "^0.7.0",
|
|
43
|
+
"@opengeni/db": "^0.4.1",
|
|
44
|
+
"@opengeni/documents": "^0.2.5",
|
|
45
|
+
"@opengeni/events": "^0.2.5",
|
|
46
46
|
"@opengeni/observability": "^0.2.1",
|
|
47
|
-
"@opengeni/runtime": "^0.3.
|
|
48
|
-
"@opengeni/storage": "^0.2.
|
|
47
|
+
"@opengeni/runtime": "^0.3.1",
|
|
48
|
+
"@opengeni/storage": "^0.2.5",
|
|
49
49
|
"@temporalio/activity": "^1.17.0",
|
|
50
50
|
"@temporalio/client": "^1.17.0",
|
|
51
51
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -94,6 +94,7 @@ import type {
|
|
|
94
94
|
} from "./types";
|
|
95
95
|
import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
|
|
96
96
|
import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
|
|
97
|
+
import { recordCreditMicros, runtimeMetricsHooksForObservability, turnLifecycleMetricsFor, type TurnOutcome } from "../observability-metrics";
|
|
97
98
|
import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
|
|
98
99
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
99
100
|
import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
|
|
@@ -510,7 +511,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
510
511
|
"opengeni.workflow_id": input.workflowId,
|
|
511
512
|
"opengeni.trigger_event_id": input.triggerEventId,
|
|
512
513
|
});
|
|
513
|
-
let activityStatus = "unknown";
|
|
514
|
+
let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
|
|
515
|
+
let turnMetricOutcome: TurnOutcome | null = null;
|
|
514
516
|
let activityError: unknown;
|
|
515
517
|
let turnId: string | undefined;
|
|
516
518
|
let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
|
|
@@ -680,6 +682,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
680
682
|
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
681
683
|
}
|
|
682
684
|
turnId = turn.id;
|
|
685
|
+
turnLifecycleMetricsFor(observability).start(turnId);
|
|
683
686
|
// Canonical codex-billed predicate (codex/<slug> + feature enabled + active
|
|
684
687
|
// workspace credential). Computed once and threaded through every billing
|
|
685
688
|
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
@@ -836,6 +839,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
836
839
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
837
840
|
], true);
|
|
838
841
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
842
|
+
turnMetricOutcome = "failed";
|
|
839
843
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
840
844
|
activityStatus = "idle";
|
|
841
845
|
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
@@ -1100,7 +1104,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1100
1104
|
};
|
|
1101
1105
|
} else {
|
|
1102
1106
|
resolvedSandbox = await resumeBoxForTurn(
|
|
1103
|
-
{ db, settings },
|
|
1107
|
+
{ db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
|
|
1104
1108
|
{
|
|
1105
1109
|
accountId: input.accountId,
|
|
1106
1110
|
workspaceId: input.workspaceId,
|
|
@@ -1192,7 +1196,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1192
1196
|
expectedEpoch: heartbeatEpoch,
|
|
1193
1197
|
warmRateMicrosPerSecond: warmRate,
|
|
1194
1198
|
subjectId: input.sessionId,
|
|
1195
|
-
})
|
|
1199
|
+
})
|
|
1200
|
+
.then((result) => recordCreditMicros(observability, "usage", result.costMicros))
|
|
1201
|
+
.catch(() => undefined);
|
|
1196
1202
|
}, 10_000);
|
|
1197
1203
|
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
1198
1204
|
leaseHeartbeatTimer.unref();
|
|
@@ -1534,6 +1540,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1534
1540
|
dispatchId,
|
|
1535
1541
|
positionalKey: `response-${responseUsageCount}`,
|
|
1536
1542
|
}),
|
|
1543
|
+
observability,
|
|
1537
1544
|
});
|
|
1538
1545
|
await reconcileConversationTruth();
|
|
1539
1546
|
try {
|
|
@@ -1588,6 +1595,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1588
1595
|
isCodexTurn,
|
|
1589
1596
|
usage: stream.state.usage,
|
|
1590
1597
|
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1598
|
+
observability,
|
|
1591
1599
|
});
|
|
1592
1600
|
}
|
|
1593
1601
|
if (lastInputTokensObserved !== null) {
|
|
@@ -1640,6 +1648,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1640
1648
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1641
1649
|
], true);
|
|
1642
1650
|
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
1651
|
+
turnMetricOutcome = "completed";
|
|
1643
1652
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1644
1653
|
await recordUsageEvent(db, {
|
|
1645
1654
|
accountId: input.accountId,
|
|
@@ -1735,6 +1744,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1735
1744
|
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1736
1745
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1737
1746
|
activityStatus = "preempted";
|
|
1747
|
+
turnMetricOutcome = "preempted";
|
|
1738
1748
|
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
1739
1749
|
sessionId: input.sessionId,
|
|
1740
1750
|
turnId: activeTurnId,
|
|
@@ -1757,6 +1767,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1757
1767
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1758
1768
|
], true);
|
|
1759
1769
|
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1770
|
+
turnMetricOutcome = "failed";
|
|
1760
1771
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1761
1772
|
activityStatus = "idle";
|
|
1762
1773
|
activityError = attemptError;
|
|
@@ -1808,6 +1819,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1808
1819
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
1809
1820
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1810
1821
|
activityStatus = "preempted";
|
|
1822
|
+
turnMetricOutcome = "preempted";
|
|
1811
1823
|
return { status: "preempted" };
|
|
1812
1824
|
} catch (requeueError) {
|
|
1813
1825
|
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
@@ -1868,6 +1880,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1868
1880
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1869
1881
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1870
1882
|
activityStatus = "preempted";
|
|
1883
|
+
turnMetricOutcome = "preempted";
|
|
1871
1884
|
return { status: "preempted" };
|
|
1872
1885
|
} catch (preemptError) {
|
|
1873
1886
|
// A failing checkpoint/requeue must not surface as an arbitrary
|
|
@@ -1885,6 +1898,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1885
1898
|
await flushRuntimeBatcher();
|
|
1886
1899
|
if (preemptTurnId) {
|
|
1887
1900
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1901
|
+
turnMetricOutcome = "cancelled";
|
|
1888
1902
|
}
|
|
1889
1903
|
throw error;
|
|
1890
1904
|
}
|
|
@@ -1920,6 +1934,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1920
1934
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1921
1935
|
], true);
|
|
1922
1936
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1937
|
+
turnMetricOutcome = "completed";
|
|
1923
1938
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1924
1939
|
await recordUsageEvent(db, {
|
|
1925
1940
|
accountId: input.accountId,
|
|
@@ -2044,6 +2059,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2044
2059
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2045
2060
|
], true);
|
|
2046
2061
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2062
|
+
turnMetricOutcome = "failed";
|
|
2047
2063
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2048
2064
|
activityStatus = "idle";
|
|
2049
2065
|
activityError = error;
|
|
@@ -2096,6 +2112,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2096
2112
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2097
2113
|
], true);
|
|
2098
2114
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2115
|
+
turnMetricOutcome = "completed";
|
|
2099
2116
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2100
2117
|
await recordUsageEvent(db, {
|
|
2101
2118
|
accountId: input.accountId,
|
|
@@ -2146,6 +2163,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2146
2163
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2147
2164
|
], true);
|
|
2148
2165
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2166
|
+
turnMetricOutcome = "failed";
|
|
2149
2167
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2150
2168
|
activityStatus = "idle";
|
|
2151
2169
|
activityError = error;
|
|
@@ -2161,6 +2179,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2161
2179
|
{ type: "session.status.changed", payload: { status: "failed" } },
|
|
2162
2180
|
], true);
|
|
2163
2181
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2182
|
+
turnMetricOutcome = "failed";
|
|
2164
2183
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
2165
2184
|
// The common failure path ends here: runAgentTurn marks the session
|
|
2166
2185
|
// failed and returns "failed", and the session workflow then exits
|
|
@@ -2178,6 +2197,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2178
2197
|
status: activityStatus,
|
|
2179
2198
|
durationSeconds,
|
|
2180
2199
|
});
|
|
2200
|
+
if (turnId && activityStatus !== "unknown") {
|
|
2201
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
2202
|
+
}
|
|
2181
2203
|
activitySpan.end({
|
|
2182
2204
|
attributes: {
|
|
2183
2205
|
"opengeni.turn_id": turnId ?? "",
|
|
@@ -2469,6 +2491,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2469
2491
|
isCodexTurn: boolean;
|
|
2470
2492
|
usage?: ModelUsageInput | null;
|
|
2471
2493
|
sourceKey: string;
|
|
2494
|
+
observability?: ActivityServices["observability"];
|
|
2472
2495
|
}): Promise<void> {
|
|
2473
2496
|
if (!input.usage) {
|
|
2474
2497
|
return;
|
|
@@ -2529,7 +2552,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2529
2552
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
2530
2553
|
});
|
|
2531
2554
|
if (costMicros > 0) {
|
|
2532
|
-
await applyCreditDebitUpToBalance(db, {
|
|
2555
|
+
const result = await applyCreditDebitUpToBalance(db, {
|
|
2533
2556
|
accountId: input.accountId,
|
|
2534
2557
|
workspaceId: input.workspaceId,
|
|
2535
2558
|
type: "model_usage_debit",
|
|
@@ -2547,6 +2570,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2547
2570
|
totalTokens,
|
|
2548
2571
|
},
|
|
2549
2572
|
});
|
|
2573
|
+
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
2550
2574
|
}
|
|
2551
2575
|
}
|
|
2552
2576
|
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
import {
|
|
34
34
|
accrueWarmSeconds,
|
|
35
35
|
confirmDrainCold,
|
|
36
|
+
countQueuedTurns,
|
|
37
|
+
countSandboxLeasesByLiveness,
|
|
36
38
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
37
39
|
getBillingBalance,
|
|
38
40
|
listLiveModalSandboxLeaseAttributions,
|
|
@@ -60,6 +62,12 @@ import {
|
|
|
60
62
|
terminateModalSandboxById,
|
|
61
63
|
} from "@opengeni/runtime";
|
|
62
64
|
import type { ActivityServices } from "./types";
|
|
65
|
+
import {
|
|
66
|
+
recordCreditMicros,
|
|
67
|
+
recordSandboxLeaseGauges,
|
|
68
|
+
recordSandboxOrphansTerminated,
|
|
69
|
+
recordTurnsQueuedGauge,
|
|
70
|
+
} from "../observability-metrics";
|
|
63
71
|
|
|
64
72
|
export type ReapSandboxLeasesResult = {
|
|
65
73
|
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
@@ -166,6 +174,7 @@ export function createSandboxLeaseActivities(
|
|
|
166
174
|
if (!settings.sandboxOwnershipEnabled) {
|
|
167
175
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
168
176
|
}
|
|
177
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
169
178
|
|
|
170
179
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
171
180
|
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
@@ -213,12 +222,15 @@ export function createSandboxLeaseActivities(
|
|
|
213
222
|
|
|
214
223
|
try {
|
|
215
224
|
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
225
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
216
226
|
} catch (error) {
|
|
217
227
|
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
218
228
|
error: error instanceof Error ? error.message : String(error),
|
|
219
229
|
});
|
|
220
230
|
}
|
|
221
231
|
|
|
232
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
233
|
+
|
|
222
234
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
223
235
|
observability.info("sandbox reaper swept", {
|
|
224
236
|
drainable: drainable.length,
|
|
@@ -275,6 +287,7 @@ async function accrueWarmTick(
|
|
|
275
287
|
if (result.accrued) {
|
|
276
288
|
accrued += 1;
|
|
277
289
|
}
|
|
290
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
278
291
|
} catch (error) {
|
|
279
292
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
280
293
|
workspaceId: lease.workspaceId,
|
|
@@ -286,6 +299,26 @@ async function accrueWarmTick(
|
|
|
286
299
|
return { accrued, workspaceIds };
|
|
287
300
|
}
|
|
288
301
|
|
|
302
|
+
async function refreshQueueAndLeaseGauges(
|
|
303
|
+
db: ActivityServices["db"],
|
|
304
|
+
observability: ActivityServices["observability"],
|
|
305
|
+
): Promise<void> {
|
|
306
|
+
try {
|
|
307
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
308
|
+
} catch (error) {
|
|
309
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
310
|
+
error: error instanceof Error ? error.message : String(error),
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
try {
|
|
314
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
315
|
+
} catch (error) {
|
|
316
|
+
observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
|
|
317
|
+
error: error instanceof Error ? error.message : String(error),
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
289
322
|
/**
|
|
290
323
|
* The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
|
|
291
324
|
* live warm box, under the usage lock: if it is at 0 balance (when a billing /
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
claimNextQueuedTurn as claimNextQueuedTurnDb,
|
|
3
|
+
countQueuedTurns,
|
|
3
4
|
countTurnSessionHistoryItems,
|
|
4
5
|
finishTurn,
|
|
5
6
|
getSessionEvent,
|
|
@@ -13,6 +14,7 @@ import { appendAndPublishEvents } from "@opengeni/events";
|
|
|
13
14
|
import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
|
|
14
15
|
import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
|
|
15
16
|
import { notifyParentOfChildTerminal } from "./parent-wake";
|
|
17
|
+
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
16
18
|
import type {
|
|
17
19
|
ActivityServices,
|
|
18
20
|
ClaimNextQueuedTurnInput,
|
|
@@ -67,7 +69,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
|
|
70
|
-
const { db, bus } = await services();
|
|
72
|
+
const { db, bus, observability } = await services();
|
|
71
73
|
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
72
74
|
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
73
75
|
// Pause an active goal before the early return below: an interrupt can
|
|
@@ -96,6 +98,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
96
98
|
]);
|
|
97
99
|
await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
98
100
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
101
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
99
102
|
}
|
|
100
103
|
|
|
101
104
|
/**
|
|
@@ -116,7 +119,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
116
119
|
* WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
|
|
117
120
|
*/
|
|
118
121
|
async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
|
|
119
|
-
const { settings, db, bus } = await services();
|
|
122
|
+
const { settings, db, bus, observability } = await services();
|
|
120
123
|
const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
|
|
121
124
|
if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
|
|
122
125
|
// The timed-out attempt was a zombie that actually settled the turn
|
|
@@ -169,12 +172,15 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
169
172
|
throw requeueError;
|
|
170
173
|
}
|
|
171
174
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
175
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
172
176
|
return { action: "requeued", redispatches };
|
|
173
177
|
}
|
|
174
178
|
|
|
175
179
|
async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
|
|
176
|
-
const { db } = await services();
|
|
177
|
-
|
|
180
|
+
const { db, observability } = await services();
|
|
181
|
+
const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
182
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
183
|
+
return turn;
|
|
178
184
|
}
|
|
179
185
|
|
|
180
186
|
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
@@ -183,6 +189,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
183
189
|
if (session.status === "queued" || session.status === "running") {
|
|
184
190
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
185
191
|
}
|
|
192
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
186
193
|
// The workflow reaches markSessionIdle exactly when it has decided to stop
|
|
187
194
|
// for now (no queued turn, no goal continuation): the terminal-for-now
|
|
188
195
|
// point for a spawned worker, whatever the cause (goal completed, agent or
|
|
@@ -199,3 +206,14 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
199
206
|
markSessionIdle,
|
|
200
207
|
};
|
|
201
208
|
}
|
|
209
|
+
|
|
210
|
+
async function refreshQueuedTurnsGauge(
|
|
211
|
+
db: ActivityServices["db"],
|
|
212
|
+
observability: ActivityServices["observability"],
|
|
213
|
+
): Promise<void> {
|
|
214
|
+
try {
|
|
215
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
216
|
+
} catch {
|
|
217
|
+
// Best-effort telemetry; session state transitions remain authoritative.
|
|
218
|
+
}
|
|
219
|
+
}
|
package/src/activities.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
|
|
|
12
12
|
import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
|
|
13
13
|
import { createSessionStateActivities } from "./activities/session-state";
|
|
14
14
|
import type { ActivityDependencies, ActivityServices } from "./activities/types";
|
|
15
|
+
import { observabilityEventLogger, runtimeMetricsHooksForObservability } from "./observability-metrics";
|
|
15
16
|
|
|
16
17
|
export type {
|
|
17
18
|
ActivityDependencies,
|
|
@@ -33,6 +34,7 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
|
33
34
|
async function services(): Promise<ActivityServices> {
|
|
34
35
|
servicesPromise ??= (async () => {
|
|
35
36
|
const settings = dependencies.settings ?? getSettings();
|
|
37
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
36
38
|
// Step I: when not injected, build the standalone handle — searchPath
|
|
37
39
|
// undefined for standalone (public), scoped to the dedicated schema +
|
|
38
40
|
// host RLS strategy when embedded config is set. An embedded host injects
|
|
@@ -67,11 +69,14 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
|
67
69
|
(await createNatsEventBus(
|
|
68
70
|
settings.natsUrl,
|
|
69
71
|
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
72
|
+
{ logger: observabilityEventLogger(observability) },
|
|
70
73
|
)),
|
|
71
|
-
runtime: dependencies.runtime ?? createProductionAgentRuntime(
|
|
74
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime({
|
|
75
|
+
metrics: runtimeMetricsHooksForObservability(observability),
|
|
76
|
+
}),
|
|
72
77
|
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
73
78
|
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
74
|
-
observability
|
|
79
|
+
observability,
|
|
75
80
|
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
76
81
|
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
77
82
|
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import { dbSql, type Database } from "@opengeni/db";
|
|
3
|
+
import type { EventBus } from "@opengeni/events";
|
|
4
|
+
import type { Observability } from "@opengeni/observability";
|
|
5
|
+
|
|
6
|
+
export type ReadinessCheckName = "db" | "nats" | "temporal";
|
|
7
|
+
export type ReadinessChecks = Record<ReadinessCheckName, () => Promise<void> | void>;
|
|
8
|
+
|
|
9
|
+
export type ReadinessResult = {
|
|
10
|
+
ok: boolean;
|
|
11
|
+
checks: Record<ReadinessCheckName, { ok: boolean; error?: string }>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export function startWorkerHttpServer(input: {
|
|
15
|
+
settings: Settings;
|
|
16
|
+
observability: Observability;
|
|
17
|
+
checks: ReadinessChecks;
|
|
18
|
+
timeoutMs?: number;
|
|
19
|
+
}): ReturnType<typeof Bun.serve> {
|
|
20
|
+
const { settings, observability, checks } = input;
|
|
21
|
+
const timeoutMs = input.timeoutMs ?? 2_000;
|
|
22
|
+
return Bun.serve({
|
|
23
|
+
hostname: "0.0.0.0",
|
|
24
|
+
port: settings.workerHttpPort,
|
|
25
|
+
async fetch(request) {
|
|
26
|
+
const url = new URL(request.url);
|
|
27
|
+
if (request.method !== "GET") {
|
|
28
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
29
|
+
}
|
|
30
|
+
if (url.pathname === "/healthz") {
|
|
31
|
+
return Response.json({
|
|
32
|
+
service: settings.serviceName,
|
|
33
|
+
environment: settings.environment,
|
|
34
|
+
deploymentRevision: settings.deploymentRevision,
|
|
35
|
+
ok: true,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
if (url.pathname === "/metrics") {
|
|
39
|
+
return new Response(await observability.prometheusMetrics(), {
|
|
40
|
+
status: 200,
|
|
41
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
if (url.pathname === "/readyz") {
|
|
45
|
+
const result = await runReadinessChecks(checks, timeoutMs);
|
|
46
|
+
return Response.json(result, { status: result.ok ? 200 : 503 });
|
|
47
|
+
}
|
|
48
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function runReadinessChecks(checks: ReadinessChecks, timeoutMs = 2_000): Promise<ReadinessResult> {
|
|
54
|
+
const entries = await Promise.all(
|
|
55
|
+
(Object.entries(checks) as Array<[ReadinessCheckName, () => Promise<void> | void]>)
|
|
56
|
+
.map(async ([name, check]) => {
|
|
57
|
+
try {
|
|
58
|
+
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
59
|
+
return [name, { ok: true }] as const;
|
|
60
|
+
} catch (error) {
|
|
61
|
+
return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }] as const;
|
|
62
|
+
}
|
|
63
|
+
}),
|
|
64
|
+
);
|
|
65
|
+
const result = Object.fromEntries(entries) as ReadinessResult["checks"];
|
|
66
|
+
return {
|
|
67
|
+
ok: Object.values(result).every((check) => check.ok),
|
|
68
|
+
checks: result,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function dbReadyCheck(db: Database): () => Promise<void> {
|
|
73
|
+
return async () => {
|
|
74
|
+
await db.execute(dbSql`select 1`);
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function natsReadyCheck(bus: EventBus): () => void {
|
|
79
|
+
return () => {
|
|
80
|
+
if (bus.isConnected && !bus.isConnected()) {
|
|
81
|
+
throw new Error("NATS is not connected");
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
87
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
88
|
+
try {
|
|
89
|
+
return await Promise.race([
|
|
90
|
+
promise,
|
|
91
|
+
new Promise<never>((_, reject) => {
|
|
92
|
+
timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
93
|
+
}),
|
|
94
|
+
]);
|
|
95
|
+
} finally {
|
|
96
|
+
if (timer) {
|
|
97
|
+
clearTimeout(timer);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
1
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
2
|
+
import { createDb } from "@opengeni/db";
|
|
3
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
2
4
|
import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
|
|
3
5
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
4
6
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
5
7
|
import { createActivities, type ActivityDependencies } from "./activities";
|
|
6
8
|
import type { WakeSessionWorkflowSignal } from "./activities/types";
|
|
9
|
+
import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
|
|
10
|
+
import { observabilityEventLogger } from "./observability-metrics";
|
|
7
11
|
|
|
8
12
|
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
9
13
|
// create() is idempotent across every worker in the pool: the first worker to
|
|
@@ -57,7 +61,11 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
|
|
|
57
61
|
// idled and let its run finish, so a plain signal would not start one).
|
|
58
62
|
// Separate from the worker's NativeConnection: the @temporalio/client
|
|
59
63
|
// Connection is what exposes workflow.signalWithStart.
|
|
60
|
-
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
|
|
64
|
+
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
|
|
65
|
+
wakeSessionWorkflow: WakeSessionWorkflowSignal;
|
|
66
|
+
check: () => Promise<void>;
|
|
67
|
+
close: () => Promise<void>;
|
|
68
|
+
}> {
|
|
61
69
|
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
62
70
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
63
71
|
return {
|
|
@@ -70,6 +78,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
|
|
|
70
78
|
signal: "queueChanged",
|
|
71
79
|
});
|
|
72
80
|
},
|
|
81
|
+
check: async () => {
|
|
82
|
+
await connection.workflowService.getSystemInfo({});
|
|
83
|
+
},
|
|
73
84
|
close: async () => {
|
|
74
85
|
await connection.close();
|
|
75
86
|
},
|
|
@@ -148,39 +159,76 @@ export async function registerSandboxReaperSchedule(
|
|
|
148
159
|
export async function startWorker() {
|
|
149
160
|
const settings = getSettings();
|
|
150
161
|
const observability = createObservability(settings, { component: "worker" });
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
},
|
|
158
|
-
);
|
|
159
|
-
const { worker, connection } = await createOpenGeniWorker({
|
|
160
|
-
settings,
|
|
161
|
-
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
|
|
162
|
-
});
|
|
163
|
-
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
164
|
-
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
165
|
-
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
166
|
-
// per-session worker factory.
|
|
167
|
-
const reaperSchedule = await retryStartupDependency(
|
|
168
|
-
"Temporal schedule (sandbox reaper)",
|
|
169
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
170
|
-
{
|
|
171
|
-
...startupRetryOptions(settings),
|
|
172
|
-
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
173
|
-
},
|
|
174
|
-
);
|
|
175
|
-
observability.info("OpenGeni worker listening", {
|
|
176
|
-
temporalTaskQueue: settings.temporalTaskQueue,
|
|
162
|
+
const retryOptions = startupRetryOptions(settings);
|
|
163
|
+
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
|
|
164
|
+
const searchPath = dbSearchPath(settings);
|
|
165
|
+
const dbClient = createDb(settings.databaseUrl, {
|
|
166
|
+
...(searchPath ? { searchPath } : {}),
|
|
167
|
+
rlsStrategy: settings.rlsStrategy,
|
|
177
168
|
});
|
|
169
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
170
|
+
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
171
|
+
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
172
|
+
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
173
|
+
let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
|
|
174
|
+
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
178
175
|
try {
|
|
179
|
-
await
|
|
176
|
+
bus = await retryStartupDependency(
|
|
177
|
+
"NATS",
|
|
178
|
+
() =>
|
|
179
|
+
createNatsEventBus(
|
|
180
|
+
settings.natsUrl,
|
|
181
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
182
|
+
{ logger: observabilityEventLogger(observability) },
|
|
183
|
+
),
|
|
184
|
+
{ ...retryOptions, onRetry },
|
|
185
|
+
);
|
|
186
|
+
signaler = await retryStartupDependency(
|
|
187
|
+
"Temporal client",
|
|
188
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
189
|
+
{ ...retryOptions, onRetry },
|
|
190
|
+
);
|
|
191
|
+
workerBundle = await createOpenGeniWorker({
|
|
192
|
+
settings,
|
|
193
|
+
activityDependencies: {
|
|
194
|
+
observability,
|
|
195
|
+
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
196
|
+
db: dbClient.db,
|
|
197
|
+
bus,
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
httpServer = startWorkerHttpServer({
|
|
201
|
+
settings,
|
|
202
|
+
observability,
|
|
203
|
+
checks: {
|
|
204
|
+
db: dbReadyCheck(dbClient.db),
|
|
205
|
+
nats: natsReadyCheck(bus),
|
|
206
|
+
temporal: signaler.check,
|
|
207
|
+
},
|
|
208
|
+
});
|
|
209
|
+
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
210
|
+
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
211
|
+
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
212
|
+
// per-session worker factory.
|
|
213
|
+
reaperSchedule = await retryStartupDependency(
|
|
214
|
+
"Temporal schedule (sandbox reaper)",
|
|
215
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
216
|
+
{ ...retryOptions, onRetry },
|
|
217
|
+
);
|
|
218
|
+
observability.info("OpenGeni worker listening", {
|
|
219
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
220
|
+
httpPort: settings.workerHttpPort,
|
|
221
|
+
});
|
|
222
|
+
await workerBundle.worker.run();
|
|
180
223
|
} finally {
|
|
181
|
-
|
|
182
|
-
await
|
|
183
|
-
|
|
224
|
+
httpServer?.stop(true);
|
|
225
|
+
await Promise.allSettled([
|
|
226
|
+
workerBundle?.connection.close(),
|
|
227
|
+
signaler?.close(),
|
|
228
|
+
reaperSchedule?.close(),
|
|
229
|
+
bus?.close(),
|
|
230
|
+
dbClient.close(),
|
|
231
|
+
]);
|
|
184
232
|
}
|
|
185
233
|
}
|
|
186
234
|
|