@opengeni/worker-bundle 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +431 -42
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +28 -4
- package/src/activities/sandbox-lease.ts +33 -0
- package/src/activities/session-state.ts +22 -4
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +21 -2
package/dist/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
2
|
+
import { dbSearchPath as dbSearchPath2, getSettings as getSettings2, resolveNatsControlPlaneAuth as resolveNatsControlPlaneAuth2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
3
|
+
import { createDb as createDb2 } from "@opengeni/db";
|
|
4
|
+
import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
|
|
3
5
|
import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
|
|
4
6
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
5
7
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
@@ -1240,6 +1242,15 @@ function asSandboxWarmingError(error, backend, timeoutMs) {
|
|
|
1240
1242
|
const message = error instanceof Error ? error.message : String(error);
|
|
1241
1243
|
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
|
|
1242
1244
|
}
|
|
1245
|
+
function recordSandboxWarmingTimeout(metrics, error) {
|
|
1246
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
try {
|
|
1250
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
1251
|
+
} catch {
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1243
1254
|
function workspaceArchiveFromEnvelope(envelope) {
|
|
1244
1255
|
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1245
1256
|
const archive = sessionState?.workspaceArchive;
|
|
@@ -1316,6 +1327,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1316
1327
|
sessionId: ids.sessionId,
|
|
1317
1328
|
backendOverride: ids.backend,
|
|
1318
1329
|
...ids.environment ? { environment: ids.environment } : {},
|
|
1330
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
1319
1331
|
onSandboxCreated: async (created) => {
|
|
1320
1332
|
createdEstablished = created;
|
|
1321
1333
|
const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
|
|
@@ -1381,7 +1393,9 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1381
1393
|
});
|
|
1382
1394
|
}
|
|
1383
1395
|
await release();
|
|
1384
|
-
|
|
1396
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1397
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1398
|
+
throw warmingError;
|
|
1385
1399
|
}
|
|
1386
1400
|
}
|
|
1387
1401
|
try {
|
|
@@ -1394,13 +1408,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1394
1408
|
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
1395
1409
|
sessionId: ids.sessionId,
|
|
1396
1410
|
backendOverride: ids.backend,
|
|
1397
|
-
...ids.environment ? { environment: ids.environment } : {}
|
|
1411
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1412
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
1398
1413
|
});
|
|
1399
1414
|
await ensureDisplayStack(settings, established);
|
|
1400
1415
|
return { established, leaseEpoch, release };
|
|
1401
1416
|
} catch (error) {
|
|
1402
1417
|
await release();
|
|
1403
|
-
|
|
1418
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1419
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1420
|
+
throw warmingError;
|
|
1404
1421
|
}
|
|
1405
1422
|
}
|
|
1406
1423
|
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
@@ -1627,6 +1644,201 @@ function routingEnabled(settings) {
|
|
|
1627
1644
|
return settings.sandboxSelfhostedEnabled === true;
|
|
1628
1645
|
}
|
|
1629
1646
|
|
|
1647
|
+
// src/observability-metrics.ts
|
|
1648
|
+
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
1649
|
+
function observabilityEventLogger(observability) {
|
|
1650
|
+
return {
|
|
1651
|
+
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
1652
|
+
warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
function eventAttributes(attributes) {
|
|
1656
|
+
if (!attributes) {
|
|
1657
|
+
return void 0;
|
|
1658
|
+
}
|
|
1659
|
+
const sanitized = {};
|
|
1660
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
1661
|
+
sanitized[key] = eventAttributeValue(value);
|
|
1662
|
+
}
|
|
1663
|
+
return sanitized;
|
|
1664
|
+
}
|
|
1665
|
+
function eventAttributeValue(value) {
|
|
1666
|
+
if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1667
|
+
return value;
|
|
1668
|
+
}
|
|
1669
|
+
try {
|
|
1670
|
+
return JSON.stringify(value);
|
|
1671
|
+
} catch {
|
|
1672
|
+
return String(value);
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
function runtimeMetricsHooksForObservability(observability) {
|
|
1676
|
+
return {
|
|
1677
|
+
onModelCall: ({ provider, outcome, durationSeconds }) => {
|
|
1678
|
+
observability.incrementCounter({
|
|
1679
|
+
name: "opengeni_model_calls_total",
|
|
1680
|
+
help: "Total model calls by provider and outcome.",
|
|
1681
|
+
labels: { provider, outcome }
|
|
1682
|
+
});
|
|
1683
|
+
observability.observeHistogram({
|
|
1684
|
+
name: "opengeni_model_call_duration_seconds",
|
|
1685
|
+
help: "Model call duration in seconds by provider.",
|
|
1686
|
+
labels: { provider },
|
|
1687
|
+
value: durationSeconds
|
|
1688
|
+
});
|
|
1689
|
+
},
|
|
1690
|
+
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
1691
|
+
observability.incrementCounter({
|
|
1692
|
+
name: "opengeni_sandbox_creates_total",
|
|
1693
|
+
help: "Total sandbox create attempts by backend and outcome.",
|
|
1694
|
+
labels: { backend, outcome }
|
|
1695
|
+
});
|
|
1696
|
+
observability.observeHistogram({
|
|
1697
|
+
name: "opengeni_sandbox_create_duration_seconds",
|
|
1698
|
+
help: "Sandbox create duration in seconds by backend.",
|
|
1699
|
+
labels: { backend },
|
|
1700
|
+
value: durationSeconds
|
|
1701
|
+
});
|
|
1702
|
+
},
|
|
1703
|
+
onSandboxWarmingTimeout: () => {
|
|
1704
|
+
observability.incrementCounter({
|
|
1705
|
+
name: "opengeni_sandbox_warming_timeouts_total",
|
|
1706
|
+
help: "Total sandbox warming timeouts."
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
function turnLifecycleMetricsFor(observability) {
|
|
1712
|
+
const existing = turnTrackers.get(observability);
|
|
1713
|
+
if (existing) {
|
|
1714
|
+
return existing;
|
|
1715
|
+
}
|
|
1716
|
+
const tracker = new TurnLifecycleMetrics(observability);
|
|
1717
|
+
turnTrackers.set(observability, tracker);
|
|
1718
|
+
return tracker;
|
|
1719
|
+
}
|
|
1720
|
+
var TurnLifecycleMetrics = class {
|
|
1721
|
+
constructor(observability, options = {}) {
|
|
1722
|
+
this.observability = observability;
|
|
1723
|
+
this.options = options;
|
|
1724
|
+
}
|
|
1725
|
+
observability;
|
|
1726
|
+
options;
|
|
1727
|
+
startedTurns = /* @__PURE__ */ new Map();
|
|
1728
|
+
timer = null;
|
|
1729
|
+
start(turnId) {
|
|
1730
|
+
this.startedTurns.set(turnId, this.now());
|
|
1731
|
+
this.ensureTimer();
|
|
1732
|
+
this.refreshGauges();
|
|
1733
|
+
}
|
|
1734
|
+
finish(turnId, outcome, durationSeconds) {
|
|
1735
|
+
const startedAt = this.startedTurns.get(turnId);
|
|
1736
|
+
if (startedAt !== void 0) {
|
|
1737
|
+
this.startedTurns.delete(turnId);
|
|
1738
|
+
}
|
|
1739
|
+
if (outcome) {
|
|
1740
|
+
const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
|
|
1741
|
+
this.observability.incrementCounter({
|
|
1742
|
+
name: "opengeni_turns_total",
|
|
1743
|
+
help: "Total agent turns by terminal outcome.",
|
|
1744
|
+
labels: { outcome }
|
|
1745
|
+
});
|
|
1746
|
+
this.observability.observeHistogram({
|
|
1747
|
+
name: "opengeni_turn_duration_seconds",
|
|
1748
|
+
help: "Agent turn duration in seconds by terminal outcome.",
|
|
1749
|
+
labels: { outcome },
|
|
1750
|
+
value: observedDuration
|
|
1751
|
+
});
|
|
1752
|
+
}
|
|
1753
|
+
this.refreshGauges();
|
|
1754
|
+
if (this.startedTurns.size === 0) {
|
|
1755
|
+
this.stopTimer();
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
refreshGauges() {
|
|
1759
|
+
this.observability.setGauge({
|
|
1760
|
+
name: "opengeni_turns_inflight",
|
|
1761
|
+
help: "Current number of in-flight agent turns in this worker process.",
|
|
1762
|
+
value: this.startedTurns.size
|
|
1763
|
+
});
|
|
1764
|
+
this.observability.setGauge({
|
|
1765
|
+
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
1766
|
+
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
1767
|
+
value: this.oldestInflightAgeSeconds()
|
|
1768
|
+
});
|
|
1769
|
+
}
|
|
1770
|
+
stop() {
|
|
1771
|
+
this.startedTurns.clear();
|
|
1772
|
+
this.refreshGauges();
|
|
1773
|
+
this.stopTimer();
|
|
1774
|
+
}
|
|
1775
|
+
oldestInflightAgeSeconds() {
|
|
1776
|
+
if (this.startedTurns.size === 0) {
|
|
1777
|
+
return 0;
|
|
1778
|
+
}
|
|
1779
|
+
let oldest = Number.POSITIVE_INFINITY;
|
|
1780
|
+
for (const startedAt of this.startedTurns.values()) {
|
|
1781
|
+
oldest = Math.min(oldest, startedAt);
|
|
1782
|
+
}
|
|
1783
|
+
return Math.max(0, (this.now() - oldest) / 1e3);
|
|
1784
|
+
}
|
|
1785
|
+
ensureTimer() {
|
|
1786
|
+
if (this.timer) {
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
|
|
1790
|
+
this.timer.unref?.();
|
|
1791
|
+
}
|
|
1792
|
+
stopTimer() {
|
|
1793
|
+
if (!this.timer) {
|
|
1794
|
+
return;
|
|
1795
|
+
}
|
|
1796
|
+
clearInterval(this.timer);
|
|
1797
|
+
this.timer = null;
|
|
1798
|
+
}
|
|
1799
|
+
now() {
|
|
1800
|
+
return this.options.now?.() ?? Date.now();
|
|
1801
|
+
}
|
|
1802
|
+
};
|
|
1803
|
+
function recordTurnsQueuedGauge(observability, value) {
|
|
1804
|
+
observability.setGauge({
|
|
1805
|
+
name: "opengeni_turns_queued",
|
|
1806
|
+
help: "Current number of queued session turns.",
|
|
1807
|
+
value
|
|
1808
|
+
});
|
|
1809
|
+
}
|
|
1810
|
+
function recordSandboxLeaseGauges(observability, counts) {
|
|
1811
|
+
for (const liveness of ["cold", "warming", "warm", "draining"]) {
|
|
1812
|
+
observability.setGauge({
|
|
1813
|
+
name: "opengeni_sandbox_leases",
|
|
1814
|
+
help: "Current sandbox leases by liveness state.",
|
|
1815
|
+
labels: { liveness },
|
|
1816
|
+
value: counts[liveness] ?? 0
|
|
1817
|
+
});
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
function recordSandboxOrphansTerminated(observability, count) {
|
|
1821
|
+
if (count <= 0) {
|
|
1822
|
+
return;
|
|
1823
|
+
}
|
|
1824
|
+
observability.incrementCounter({
|
|
1825
|
+
name: "opengeni_sandbox_orphans_terminated_total",
|
|
1826
|
+
help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
|
|
1827
|
+
amount: count
|
|
1828
|
+
});
|
|
1829
|
+
}
|
|
1830
|
+
function recordCreditMicros(observability, kind, amountMicros) {
|
|
1831
|
+
if (!observability || amountMicros <= 0) {
|
|
1832
|
+
return;
|
|
1833
|
+
}
|
|
1834
|
+
observability.incrementCounter({
|
|
1835
|
+
name: "opengeni_credit_micros_total",
|
|
1836
|
+
help: "Total credit micros recorded by kind.",
|
|
1837
|
+
labels: { kind },
|
|
1838
|
+
amount: amountMicros
|
|
1839
|
+
});
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1630
1842
|
// src/activities/recording.ts
|
|
1631
1843
|
import { deleteRecording, updateRecording } from "@opengeni/db";
|
|
1632
1844
|
import {
|
|
@@ -1920,6 +2132,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
1920
2132
|
"opengeni.trigger_event_id": input.triggerEventId
|
|
1921
2133
|
});
|
|
1922
2134
|
let activityStatus = "unknown";
|
|
2135
|
+
let turnMetricOutcome = null;
|
|
1923
2136
|
let activityError;
|
|
1924
2137
|
let turnId;
|
|
1925
2138
|
let heartbeatTimer;
|
|
@@ -2022,6 +2235,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2022
2235
|
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
2023
2236
|
}
|
|
2024
2237
|
turnId = turn.id;
|
|
2238
|
+
turnLifecycleMetricsFor(observability).start(turnId);
|
|
2025
2239
|
const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
|
|
2026
2240
|
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
2027
2241
|
const activityContext = currentActivityContext();
|
|
@@ -2122,6 +2336,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2122
2336
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2123
2337
|
], true);
|
|
2124
2338
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2339
|
+
turnMetricOutcome = "failed";
|
|
2125
2340
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2126
2341
|
activityStatus = "idle";
|
|
2127
2342
|
return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
|
|
@@ -2260,7 +2475,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2260
2475
|
};
|
|
2261
2476
|
} else {
|
|
2262
2477
|
resolvedSandbox = await resumeBoxForTurn(
|
|
2263
|
-
{ db, settings },
|
|
2478
|
+
{ db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
|
|
2264
2479
|
{
|
|
2265
2480
|
accountId: input.accountId,
|
|
2266
2481
|
workspaceId: input.workspaceId,
|
|
@@ -2322,7 +2537,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2322
2537
|
expectedEpoch: heartbeatEpoch,
|
|
2323
2538
|
warmRateMicrosPerSecond: warmRate,
|
|
2324
2539
|
subjectId: input.sessionId
|
|
2325
|
-
}).catch(() => void 0);
|
|
2540
|
+
}).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
|
|
2326
2541
|
}, 1e4);
|
|
2327
2542
|
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
2328
2543
|
leaseHeartbeatTimer.unref();
|
|
@@ -2567,7 +2782,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
2567
2782
|
responseId: responseUsage.responseId,
|
|
2568
2783
|
dispatchId,
|
|
2569
2784
|
positionalKey: `response-${responseUsageCount}`
|
|
2570
|
-
})
|
|
2785
|
+
}),
|
|
2786
|
+
observability
|
|
2571
2787
|
});
|
|
2572
2788
|
await reconcileConversationTruth();
|
|
2573
2789
|
try {
|
|
@@ -2613,7 +2829,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
2613
2829
|
model: turn.model,
|
|
2614
2830
|
isCodexTurn,
|
|
2615
2831
|
usage: stream.state.usage,
|
|
2616
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
|
|
2832
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
2833
|
+
observability
|
|
2617
2834
|
});
|
|
2618
2835
|
}
|
|
2619
2836
|
if (lastInputTokensObserved !== null) {
|
|
@@ -2661,6 +2878,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2661
2878
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2662
2879
|
], true);
|
|
2663
2880
|
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
2881
|
+
turnMetricOutcome = "completed";
|
|
2664
2882
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2665
2883
|
await recordUsageEvent2(db, {
|
|
2666
2884
|
accountId: input.accountId,
|
|
@@ -2741,6 +2959,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2741
2959
|
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2742
2960
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2743
2961
|
activityStatus = "preempted";
|
|
2962
|
+
turnMetricOutcome = "preempted";
|
|
2744
2963
|
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
2745
2964
|
sessionId: input.sessionId,
|
|
2746
2965
|
turnId: activeTurnId,
|
|
@@ -2763,6 +2982,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2763
2982
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2764
2983
|
], true);
|
|
2765
2984
|
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
2985
|
+
turnMetricOutcome = "failed";
|
|
2766
2986
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2767
2987
|
activityStatus = "idle";
|
|
2768
2988
|
activityError = attemptError;
|
|
@@ -2791,6 +3011,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2791
3011
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
2792
3012
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2793
3013
|
activityStatus = "preempted";
|
|
3014
|
+
turnMetricOutcome = "preempted";
|
|
2794
3015
|
return { status: "preempted" };
|
|
2795
3016
|
} catch (requeueError) {
|
|
2796
3017
|
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
@@ -2838,6 +3059,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2838
3059
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2839
3060
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2840
3061
|
activityStatus = "preempted";
|
|
3062
|
+
turnMetricOutcome = "preempted";
|
|
2841
3063
|
return { status: "preempted" };
|
|
2842
3064
|
} catch (preemptError) {
|
|
2843
3065
|
console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
|
|
@@ -2849,6 +3071,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2849
3071
|
await flushRuntimeBatcher();
|
|
2850
3072
|
if (preemptTurnId) {
|
|
2851
3073
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
|
|
3074
|
+
turnMetricOutcome = "cancelled";
|
|
2852
3075
|
}
|
|
2853
3076
|
throw error;
|
|
2854
3077
|
}
|
|
@@ -2873,6 +3096,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2873
3096
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2874
3097
|
], true);
|
|
2875
3098
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
3099
|
+
turnMetricOutcome = "completed";
|
|
2876
3100
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2877
3101
|
await recordUsageEvent2(db, {
|
|
2878
3102
|
accountId: input.accountId,
|
|
@@ -2962,6 +3186,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2962
3186
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2963
3187
|
], true);
|
|
2964
3188
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3189
|
+
turnMetricOutcome = "failed";
|
|
2965
3190
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2966
3191
|
activityStatus = "idle";
|
|
2967
3192
|
activityError = error;
|
|
@@ -2994,6 +3219,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2994
3219
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2995
3220
|
], true);
|
|
2996
3221
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
3222
|
+
turnMetricOutcome = "completed";
|
|
2997
3223
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2998
3224
|
await recordUsageEvent2(db, {
|
|
2999
3225
|
accountId: input.accountId,
|
|
@@ -3032,6 +3258,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
3032
3258
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
3033
3259
|
], true);
|
|
3034
3260
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3261
|
+
turnMetricOutcome = "failed";
|
|
3035
3262
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
3036
3263
|
activityStatus = "idle";
|
|
3037
3264
|
activityError = error;
|
|
@@ -3047,6 +3274,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
3047
3274
|
{ type: "session.status.changed", payload: { status: "failed" } }
|
|
3048
3275
|
], true);
|
|
3049
3276
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3277
|
+
turnMetricOutcome = "failed";
|
|
3050
3278
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
3051
3279
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
|
|
3052
3280
|
return { status: "failed" };
|
|
@@ -3057,6 +3285,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
3057
3285
|
status: activityStatus,
|
|
3058
3286
|
durationSeconds
|
|
3059
3287
|
});
|
|
3288
|
+
if (turnId && activityStatus !== "unknown") {
|
|
3289
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
3290
|
+
}
|
|
3060
3291
|
activitySpan.end({
|
|
3061
3292
|
attributes: {
|
|
3062
3293
|
"opengeni.turn_id": turnId ?? "",
|
|
@@ -3279,7 +3510,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
3279
3510
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
|
|
3280
3511
|
});
|
|
3281
3512
|
if (costMicros > 0) {
|
|
3282
|
-
await applyCreditDebitUpToBalance(db, {
|
|
3513
|
+
const result = await applyCreditDebitUpToBalance(db, {
|
|
3283
3514
|
accountId: input.accountId,
|
|
3284
3515
|
workspaceId: input.workspaceId,
|
|
3285
3516
|
type: "model_usage_debit",
|
|
@@ -3297,6 +3528,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
3297
3528
|
totalTokens
|
|
3298
3529
|
}
|
|
3299
3530
|
});
|
|
3531
|
+
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
3300
3532
|
}
|
|
3301
3533
|
}
|
|
3302
3534
|
function positiveInt(value) {
|
|
@@ -3417,6 +3649,8 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
|
|
|
3417
3649
|
import {
|
|
3418
3650
|
accrueWarmSeconds as accrueWarmSeconds2,
|
|
3419
3651
|
confirmDrainCold,
|
|
3652
|
+
countQueuedTurns,
|
|
3653
|
+
countSandboxLeasesByLiveness,
|
|
3420
3654
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3421
3655
|
getBillingBalance as getBillingBalance4,
|
|
3422
3656
|
listLiveModalSandboxLeaseAttributions,
|
|
@@ -3443,6 +3677,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3443
3677
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3444
3678
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3445
3679
|
}
|
|
3680
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
3446
3681
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3447
3682
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
3448
3683
|
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
@@ -3472,11 +3707,13 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3472
3707
|
}
|
|
3473
3708
|
try {
|
|
3474
3709
|
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
3710
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
3475
3711
|
} catch (error) {
|
|
3476
3712
|
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
3477
3713
|
error: error instanceof Error ? error.message : String(error)
|
|
3478
3714
|
});
|
|
3479
3715
|
}
|
|
3716
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
3480
3717
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3481
3718
|
observability.info("sandbox reaper swept", {
|
|
3482
3719
|
drainable: drainable.length,
|
|
@@ -3518,6 +3755,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3518
3755
|
if (result.accrued) {
|
|
3519
3756
|
accrued += 1;
|
|
3520
3757
|
}
|
|
3758
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3521
3759
|
} catch (error) {
|
|
3522
3760
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3523
3761
|
workspaceId: lease.workspaceId,
|
|
@@ -3528,6 +3766,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3528
3766
|
}
|
|
3529
3767
|
return { accrued, workspaceIds };
|
|
3530
3768
|
}
|
|
3769
|
+
async function refreshQueueAndLeaseGauges(db, observability) {
|
|
3770
|
+
try {
|
|
3771
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
3772
|
+
} catch (error) {
|
|
3773
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
3774
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3775
|
+
});
|
|
3776
|
+
}
|
|
3777
|
+
try {
|
|
3778
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
3779
|
+
} catch (error) {
|
|
3780
|
+
observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
|
|
3781
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3782
|
+
});
|
|
3783
|
+
}
|
|
3784
|
+
}
|
|
3531
3785
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3532
3786
|
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
3533
3787
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
@@ -4058,6 +4312,7 @@ function startOfUtcMonth4() {
|
|
|
4058
4312
|
// src/activities/session-state.ts
|
|
4059
4313
|
import {
|
|
4060
4314
|
claimNextQueuedTurn as claimNextQueuedTurnDb2,
|
|
4315
|
+
countQueuedTurns as countQueuedTurns2,
|
|
4061
4316
|
countTurnSessionHistoryItems,
|
|
4062
4317
|
finishTurn as finishTurn2,
|
|
4063
4318
|
getSessionEvent as getSessionEvent4,
|
|
@@ -4101,7 +4356,7 @@ function createSessionStateActivities(services) {
|
|
|
4101
4356
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
|
|
4102
4357
|
}
|
|
4103
4358
|
async function interruptActiveTurn2(input) {
|
|
4104
|
-
const { db, bus } = await services();
|
|
4359
|
+
const { db, bus, observability } = await services();
|
|
4105
4360
|
const session = await requireSession4(db, input.workspaceId, input.sessionId);
|
|
4106
4361
|
const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
|
|
4107
4362
|
if (!isSteerInterrupt(trigger)) {
|
|
@@ -4124,9 +4379,10 @@ function createSessionStateActivities(services) {
|
|
|
4124
4379
|
]);
|
|
4125
4380
|
await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
4126
4381
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
4382
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4127
4383
|
}
|
|
4128
4384
|
async function requeueTurnAfterWorkerDeath2(input) {
|
|
4129
|
-
const { settings, db, bus } = await services();
|
|
4385
|
+
const { settings, db, bus, observability } = await services();
|
|
4130
4386
|
const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
|
|
4131
4387
|
if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
|
|
4132
4388
|
return { action: "stale" };
|
|
@@ -4166,11 +4422,14 @@ function createSessionStateActivities(services) {
|
|
|
4166
4422
|
throw requeueError;
|
|
4167
4423
|
}
|
|
4168
4424
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
4425
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4169
4426
|
return { action: "requeued", redispatches };
|
|
4170
4427
|
}
|
|
4171
4428
|
async function claimNextQueuedTurn2(input) {
|
|
4172
|
-
const { db } = await services();
|
|
4173
|
-
|
|
4429
|
+
const { db, observability } = await services();
|
|
4430
|
+
const turn = await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
4431
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4432
|
+
return turn;
|
|
4174
4433
|
}
|
|
4175
4434
|
async function markSessionIdle2(input) {
|
|
4176
4435
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
@@ -4178,6 +4437,7 @@ function createSessionStateActivities(services) {
|
|
|
4178
4437
|
if (session.status === "queued" || session.status === "running") {
|
|
4179
4438
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
|
|
4180
4439
|
}
|
|
4440
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4181
4441
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
|
|
4182
4442
|
}
|
|
4183
4443
|
return {
|
|
@@ -4188,6 +4448,12 @@ function createSessionStateActivities(services) {
|
|
|
4188
4448
|
markSessionIdle: markSessionIdle2
|
|
4189
4449
|
};
|
|
4190
4450
|
}
|
|
4451
|
+
async function refreshQueuedTurnsGauge(db, observability) {
|
|
4452
|
+
try {
|
|
4453
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns2(db));
|
|
4454
|
+
} catch {
|
|
4455
|
+
}
|
|
4456
|
+
}
|
|
4191
4457
|
|
|
4192
4458
|
// src/activities.ts
|
|
4193
4459
|
function createActivities(dependencies = {}) {
|
|
@@ -4195,6 +4461,7 @@ function createActivities(dependencies = {}) {
|
|
|
4195
4461
|
async function services() {
|
|
4196
4462
|
servicesPromise ??= (async () => {
|
|
4197
4463
|
const settings = dependencies.settings ?? getSettings();
|
|
4464
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
4198
4465
|
const searchPath = dbSearchPath(settings);
|
|
4199
4466
|
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
4200
4467
|
...searchPath ? { searchPath } : {},
|
|
@@ -4219,12 +4486,15 @@ function createActivities(dependencies = {}) {
|
|
|
4219
4486
|
// events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
|
|
4220
4487
|
bus: dependencies.bus ?? await createNatsEventBus(
|
|
4221
4488
|
settings.natsUrl,
|
|
4222
|
-
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
|
|
4489
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
4490
|
+
{ logger: observabilityEventLogger(observability) }
|
|
4223
4491
|
),
|
|
4224
|
-
runtime: dependencies.runtime ?? createProductionAgentRuntime(
|
|
4492
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime({
|
|
4493
|
+
metrics: runtimeMetricsHooksForObservability(observability)
|
|
4494
|
+
}),
|
|
4225
4495
|
objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
|
|
4226
4496
|
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
4227
|
-
observability
|
|
4497
|
+
observability,
|
|
4228
4498
|
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
4229
4499
|
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
4230
4500
|
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
|
@@ -4273,6 +4543,86 @@ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
|
|
|
4273
4543
|
var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
|
|
4274
4544
|
var reapSandboxLeases = defaultActivities.reapSandboxLeases;
|
|
4275
4545
|
|
|
4546
|
+
// src/http.ts
|
|
4547
|
+
import { dbSql } from "@opengeni/db";
|
|
4548
|
+
function startWorkerHttpServer(input) {
|
|
4549
|
+
const { settings, observability, checks } = input;
|
|
4550
|
+
const timeoutMs = input.timeoutMs ?? 2e3;
|
|
4551
|
+
return Bun.serve({
|
|
4552
|
+
hostname: "0.0.0.0",
|
|
4553
|
+
port: settings.workerHttpPort,
|
|
4554
|
+
async fetch(request) {
|
|
4555
|
+
const url = new URL(request.url);
|
|
4556
|
+
if (request.method !== "GET") {
|
|
4557
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
4558
|
+
}
|
|
4559
|
+
if (url.pathname === "/healthz") {
|
|
4560
|
+
return Response.json({
|
|
4561
|
+
service: settings.serviceName,
|
|
4562
|
+
environment: settings.environment,
|
|
4563
|
+
deploymentRevision: settings.deploymentRevision,
|
|
4564
|
+
ok: true
|
|
4565
|
+
});
|
|
4566
|
+
}
|
|
4567
|
+
if (url.pathname === "/metrics") {
|
|
4568
|
+
return new Response(await observability.prometheusMetrics(), {
|
|
4569
|
+
status: 200,
|
|
4570
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
|
|
4571
|
+
});
|
|
4572
|
+
}
|
|
4573
|
+
if (url.pathname === "/readyz") {
|
|
4574
|
+
const result = await runReadinessChecks(checks, timeoutMs);
|
|
4575
|
+
return Response.json(result, { status: result.ok ? 200 : 503 });
|
|
4576
|
+
}
|
|
4577
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
4578
|
+
}
|
|
4579
|
+
});
|
|
4580
|
+
}
|
|
4581
|
+
async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
4582
|
+
const entries = await Promise.all(
|
|
4583
|
+
Object.entries(checks).map(async ([name, check]) => {
|
|
4584
|
+
try {
|
|
4585
|
+
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
4586
|
+
return [name, { ok: true }];
|
|
4587
|
+
} catch (error) {
|
|
4588
|
+
return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }];
|
|
4589
|
+
}
|
|
4590
|
+
})
|
|
4591
|
+
);
|
|
4592
|
+
const result = Object.fromEntries(entries);
|
|
4593
|
+
return {
|
|
4594
|
+
ok: Object.values(result).every((check) => check.ok),
|
|
4595
|
+
checks: result
|
|
4596
|
+
};
|
|
4597
|
+
}
|
|
4598
|
+
function dbReadyCheck(db) {
|
|
4599
|
+
return async () => {
|
|
4600
|
+
await db.execute(dbSql`select 1`);
|
|
4601
|
+
};
|
|
4602
|
+
}
|
|
4603
|
+
function natsReadyCheck(bus) {
|
|
4604
|
+
return () => {
|
|
4605
|
+
if (bus.isConnected && !bus.isConnected()) {
|
|
4606
|
+
throw new Error("NATS is not connected");
|
|
4607
|
+
}
|
|
4608
|
+
};
|
|
4609
|
+
}
|
|
4610
|
+
async function withTimeout(promise, timeoutMs) {
|
|
4611
|
+
let timer;
|
|
4612
|
+
try {
|
|
4613
|
+
return await Promise.race([
|
|
4614
|
+
promise,
|
|
4615
|
+
new Promise((_, reject) => {
|
|
4616
|
+
timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
4617
|
+
})
|
|
4618
|
+
]);
|
|
4619
|
+
} finally {
|
|
4620
|
+
if (timer) {
|
|
4621
|
+
clearTimeout(timer);
|
|
4622
|
+
}
|
|
4623
|
+
}
|
|
4624
|
+
}
|
|
4625
|
+
|
|
4276
4626
|
// src/index.ts
|
|
4277
4627
|
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
4278
4628
|
async function createOpenGeniWorker(options = {}) {
|
|
@@ -4313,6 +4663,9 @@ async function createWorkerWorkflowSignaler(settings) {
|
|
|
4313
4663
|
signal: "queueChanged"
|
|
4314
4664
|
});
|
|
4315
4665
|
},
|
|
4666
|
+
check: async () => {
|
|
4667
|
+
await connection.workflowService.getSystemInfo({});
|
|
4668
|
+
},
|
|
4316
4669
|
close: async () => {
|
|
4317
4670
|
await connection.close();
|
|
4318
4671
|
}
|
|
@@ -4370,35 +4723,71 @@ async function registerSandboxReaperSchedule(settings, observability) {
|
|
|
4370
4723
|
async function startWorker() {
|
|
4371
4724
|
const settings = getSettings2();
|
|
4372
4725
|
const observability = createObservability2(settings, { component: "worker" });
|
|
4373
|
-
const
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
}
|
|
4380
|
-
);
|
|
4381
|
-
const { worker, connection } = await createOpenGeniWorker({
|
|
4382
|
-
settings,
|
|
4383
|
-
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
|
|
4384
|
-
});
|
|
4385
|
-
const reaperSchedule = await retryStartupDependency(
|
|
4386
|
-
"Temporal schedule (sandbox reaper)",
|
|
4387
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
4388
|
-
{
|
|
4389
|
-
...startupRetryOptions(settings),
|
|
4390
|
-
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
4391
|
-
}
|
|
4392
|
-
);
|
|
4393
|
-
observability.info("OpenGeni worker listening", {
|
|
4394
|
-
temporalTaskQueue: settings.temporalTaskQueue
|
|
4726
|
+
const retryOptions = startupRetryOptions(settings);
|
|
4727
|
+
const onRetry = (event) => logStartupDependencyRetry(observability, event);
|
|
4728
|
+
const searchPath = dbSearchPath2(settings);
|
|
4729
|
+
const dbClient = createDb2(settings.databaseUrl, {
|
|
4730
|
+
...searchPath ? { searchPath } : {},
|
|
4731
|
+
rlsStrategy: settings.rlsStrategy
|
|
4395
4732
|
});
|
|
4733
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
|
|
4734
|
+
let bus;
|
|
4735
|
+
let signaler;
|
|
4736
|
+
let workerBundle;
|
|
4737
|
+
let reaperSchedule;
|
|
4738
|
+
let httpServer;
|
|
4396
4739
|
try {
|
|
4397
|
-
await
|
|
4740
|
+
bus = await retryStartupDependency(
|
|
4741
|
+
"NATS",
|
|
4742
|
+
() => createNatsEventBus2(
|
|
4743
|
+
settings.natsUrl,
|
|
4744
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
4745
|
+
{ logger: observabilityEventLogger(observability) }
|
|
4746
|
+
),
|
|
4747
|
+
{ ...retryOptions, onRetry }
|
|
4748
|
+
);
|
|
4749
|
+
signaler = await retryStartupDependency(
|
|
4750
|
+
"Temporal client",
|
|
4751
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
4752
|
+
{ ...retryOptions, onRetry }
|
|
4753
|
+
);
|
|
4754
|
+
workerBundle = await createOpenGeniWorker({
|
|
4755
|
+
settings,
|
|
4756
|
+
activityDependencies: {
|
|
4757
|
+
observability,
|
|
4758
|
+
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
4759
|
+
db: dbClient.db,
|
|
4760
|
+
bus
|
|
4761
|
+
}
|
|
4762
|
+
});
|
|
4763
|
+
httpServer = startWorkerHttpServer({
|
|
4764
|
+
settings,
|
|
4765
|
+
observability,
|
|
4766
|
+
checks: {
|
|
4767
|
+
db: dbReadyCheck(dbClient.db),
|
|
4768
|
+
nats: natsReadyCheck(bus),
|
|
4769
|
+
temporal: signaler.check
|
|
4770
|
+
}
|
|
4771
|
+
});
|
|
4772
|
+
reaperSchedule = await retryStartupDependency(
|
|
4773
|
+
"Temporal schedule (sandbox reaper)",
|
|
4774
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
4775
|
+
{ ...retryOptions, onRetry }
|
|
4776
|
+
);
|
|
4777
|
+
observability.info("OpenGeni worker listening", {
|
|
4778
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
4779
|
+
httpPort: settings.workerHttpPort
|
|
4780
|
+
});
|
|
4781
|
+
await workerBundle.worker.run();
|
|
4398
4782
|
} finally {
|
|
4399
|
-
|
|
4400
|
-
await
|
|
4401
|
-
|
|
4783
|
+
httpServer?.stop(true);
|
|
4784
|
+
await Promise.allSettled([
|
|
4785
|
+
workerBundle?.connection.close(),
|
|
4786
|
+
signaler?.close(),
|
|
4787
|
+
reaperSchedule?.close(),
|
|
4788
|
+
bus?.close(),
|
|
4789
|
+
dbClient.close()
|
|
4790
|
+
]);
|
|
4402
4791
|
}
|
|
4403
4792
|
}
|
|
4404
4793
|
if (import.meta.main) {
|