@opengeni/worker-bundle 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +593 -86
- package/dist/index.js.map +1 -1
- package/package.json +11 -15
- package/src/activities/agent-turn.ts +95 -12
- package/src/activities/capabilities.ts +2 -30
- package/src/activities/context-compaction.ts +96 -10
- package/src/activities/environment.ts +30 -4
- package/src/activities/sandbox-lease.ts +33 -0
- package/src/activities/session-state.ts +22 -4
- package/src/activities/streaming.ts +1 -0
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +21 -2
package/dist/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
2
|
+
import { dbSearchPath as dbSearchPath2, getSettings as getSettings2, resolveNatsControlPlaneAuth as resolveNatsControlPlaneAuth2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
3
|
+
import { createDb as createDb2 } from "@opengeni/db";
|
|
4
|
+
import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
|
|
3
5
|
import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
|
|
4
6
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
5
7
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
@@ -53,7 +55,8 @@ import {
|
|
|
53
55
|
sumUsageQuantity as sumUsageQuantity2,
|
|
54
56
|
heartbeatLeaseHolder,
|
|
55
57
|
accrueWarmSeconds,
|
|
56
|
-
SandboxLeaseSupersededError as SandboxLeaseSupersededError2
|
|
58
|
+
SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
|
|
59
|
+
buildConnectionTokenResolver
|
|
57
60
|
} from "@opengeni/db";
|
|
58
61
|
import { appendAndPublishEvents as appendAndPublishEvents2 } from "@opengeni/events";
|
|
59
62
|
import {
|
|
@@ -72,6 +75,7 @@ import { CancelledFailure } from "@temporalio/activity";
|
|
|
72
75
|
|
|
73
76
|
// src/activities/capabilities.ts
|
|
74
77
|
import { environmentsEncryptionKeyBytes, parseModelProvidersJson } from "@opengeni/config";
|
|
78
|
+
import { settingsWithMcpCapabilityServers } from "@opengeni/core";
|
|
75
79
|
import {
|
|
76
80
|
CODEX_APPS_MCP_SERVER_ID,
|
|
77
81
|
CODEX_APPS_MCP_SERVER_NAME,
|
|
@@ -83,7 +87,6 @@ import {
|
|
|
83
87
|
CODEX_PROVIDER_ID
|
|
84
88
|
} from "@opengeni/codex";
|
|
85
89
|
import {
|
|
86
|
-
decryptedCapabilityHeaders,
|
|
87
90
|
listEnabledMcpCapabilityServers,
|
|
88
91
|
listSessionMcpServerMetadata,
|
|
89
92
|
listSessionMcpServersForRun,
|
|
@@ -120,6 +123,7 @@ function settingsWithSessionMcpServers(settings, servers) {
|
|
|
120
123
|
...server.allowedTools ? { allowedTools: server.allowedTools } : {},
|
|
121
124
|
...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
|
|
122
125
|
cacheToolsList: server.cacheToolsList ?? false,
|
|
126
|
+
...server.requireApproval !== void 0 ? { requireApproval: server.requireApproval } : {},
|
|
123
127
|
headers: server.headers
|
|
124
128
|
}))
|
|
125
129
|
]
|
|
@@ -169,29 +173,6 @@ function withCodexProvider(settings) {
|
|
|
169
173
|
};
|
|
170
174
|
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
171
175
|
}
|
|
172
|
-
function settingsWithMcpCapabilityServers(settings, enabled) {
|
|
173
|
-
if (enabled.length === 0) {
|
|
174
|
-
return settings;
|
|
175
|
-
}
|
|
176
|
-
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
177
|
-
const existingIds = new Set(settings.mcpServers.map((server) => server.id));
|
|
178
|
-
const dynamicServers = enabled.filter((server) => !existingIds.has(server.id)).flatMap((server) => {
|
|
179
|
-
const headers = decryptedCapabilityHeaders(server, encryptionKey);
|
|
180
|
-
if (headers === "unavailable") {
|
|
181
|
-
return [];
|
|
182
|
-
}
|
|
183
|
-
return [{
|
|
184
|
-
id: server.id,
|
|
185
|
-
name: server.name,
|
|
186
|
-
url: server.url,
|
|
187
|
-
...server.allowedTools ? { allowedTools: server.allowedTools } : {},
|
|
188
|
-
...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
|
|
189
|
-
cacheToolsList: server.cacheToolsList ?? false,
|
|
190
|
-
...headers ? { headers } : {}
|
|
191
|
-
}];
|
|
192
|
-
});
|
|
193
|
-
return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
|
|
194
|
-
}
|
|
195
176
|
|
|
196
177
|
// src/activities/codex-rotation.ts
|
|
197
178
|
var MIN_IDLE_MS = 6e4;
|
|
@@ -366,13 +347,18 @@ import {
|
|
|
366
347
|
import {
|
|
367
348
|
SUMMARY_BUFFER_TOKENS,
|
|
368
349
|
buildCompactionPromptInput,
|
|
350
|
+
buildDeterministicFallbackCompactionHistory,
|
|
369
351
|
buildCompactionReplacementHistory,
|
|
370
352
|
decideClientCompaction,
|
|
353
|
+
estimateTokens,
|
|
371
354
|
sanitizeHistoryItemsForModel,
|
|
372
355
|
summarizeForCompaction
|
|
373
356
|
} from "@opengeni/runtime";
|
|
374
357
|
import { resolveContextCompactionMode } from "@opengeni/config";
|
|
375
|
-
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, {
|
|
358
|
+
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m, o) => summarizeForCompaction(s, m, {
|
|
359
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
360
|
+
...o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}
|
|
361
|
+
}), options = {}) {
|
|
376
362
|
if (resolveContextCompactionMode(settings) !== "client") {
|
|
377
363
|
return { compacted: false, reason: "mode_not_client" };
|
|
378
364
|
}
|
|
@@ -386,20 +372,67 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
386
372
|
lastInputTokens,
|
|
387
373
|
contextWindowTokens: settings.contextWindowTokens,
|
|
388
374
|
contextReservedOutputTokens: settings.contextReservedOutputTokens,
|
|
375
|
+
contextCompactionThresholdRatio: settings.contextCompactionThresholdRatio,
|
|
389
376
|
...options.force ? { force: true } : {}
|
|
390
377
|
});
|
|
391
378
|
if (!decision.shouldCompact) {
|
|
392
379
|
return { compacted: false, reason: decision.reason };
|
|
393
380
|
}
|
|
394
381
|
const promptInput = buildCompactionPromptInput(items);
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
382
|
+
const estimatedTokensBefore = estimateTokens(items);
|
|
383
|
+
const shrinkBaselineTokens = Math.max(estimatedTokensBefore, decision.signalTokens);
|
|
384
|
+
const requireShrink = options.requireShrink || decision.reason === "above_threshold";
|
|
385
|
+
let summarizerFailure = "summarizer returned no summary";
|
|
386
|
+
let summaryBody = null;
|
|
387
|
+
const fullAttempt = await runSummarizerAttempt(summarize, settings, promptInput, { attempt: "full" });
|
|
388
|
+
if (fullAttempt.summary) {
|
|
389
|
+
summaryBody = fullAttempt.summary;
|
|
390
|
+
} else {
|
|
391
|
+
summarizerFailure = fullAttempt.failure;
|
|
392
|
+
const hardTrimBudget = Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5));
|
|
393
|
+
const retryAttempt = await runSummarizerAttempt(summarize, settings, promptInput, {
|
|
394
|
+
attempt: "hard_trim",
|
|
395
|
+
maxTranscriptTokens: hardTrimBudget
|
|
396
|
+
});
|
|
397
|
+
if (retryAttempt.summary) {
|
|
398
|
+
summaryBody = retryAttempt.summary;
|
|
399
|
+
} else {
|
|
400
|
+
summarizerFailure = retryAttempt.failure || summarizerFailure;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
let replacementHistory = null;
|
|
404
|
+
let method = "summary";
|
|
405
|
+
if (summaryBody) {
|
|
406
|
+
const summaryReplacement = buildCompactionReplacementHistory(items, summaryBody);
|
|
407
|
+
const summaryEstimate = estimateTokens(summaryReplacement);
|
|
408
|
+
if (!requireShrink || summaryEstimate < shrinkBaselineTokens) {
|
|
409
|
+
replacementHistory = summaryReplacement;
|
|
410
|
+
} else {
|
|
411
|
+
summarizerFailure = `summary replacement did not reduce context signal (${summaryEstimate} >= ${shrinkBaselineTokens})`;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
if (!replacementHistory) {
|
|
415
|
+
method = "fallback";
|
|
416
|
+
const fallbackTargetTokens = Math.max(
|
|
417
|
+
1,
|
|
418
|
+
Math.min(
|
|
419
|
+
Math.max(1, Math.floor(shrinkBaselineTokens * 0.5)),
|
|
420
|
+
Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5))
|
|
421
|
+
)
|
|
422
|
+
);
|
|
423
|
+
replacementHistory = buildDeterministicFallbackCompactionHistory({
|
|
424
|
+
items,
|
|
425
|
+
cause: summarizerFailure,
|
|
426
|
+
targetTokens: fallbackTargetTokens
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
const estimatedTokensAfter = estimateTokens(replacementHistory);
|
|
430
|
+
if (requireShrink && estimatedTokensAfter >= shrinkBaselineTokens) {
|
|
431
|
+
return { compacted: false, reason: `compaction summarization failed: fallback did not reduce context signal (${estimatedTokensAfter} >= ${shrinkBaselineTokens})` };
|
|
398
432
|
}
|
|
399
|
-
const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
|
|
400
433
|
const summaryItem = replacementHistory.at(-1);
|
|
401
434
|
if (!summaryItem) {
|
|
402
|
-
return { compacted: false, reason:
|
|
435
|
+
return { compacted: false, reason: `compaction summarization failed: ${summarizerFailure}` };
|
|
403
436
|
}
|
|
404
437
|
const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
|
|
405
438
|
const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
|
|
@@ -422,15 +455,33 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
422
455
|
supersededFrom: nextPosition,
|
|
423
456
|
summaryPosition,
|
|
424
457
|
signalTokens: decision.signalTokens,
|
|
425
|
-
thresholdTokens: decision.thresholdTokens
|
|
458
|
+
thresholdTokens: decision.thresholdTokens,
|
|
459
|
+
method,
|
|
460
|
+
estimatedTokensBefore,
|
|
461
|
+
estimatedTokensAfter
|
|
426
462
|
};
|
|
427
463
|
}
|
|
464
|
+
async function runSummarizerAttempt(summarize, settings, promptInput, options) {
|
|
465
|
+
try {
|
|
466
|
+
const summary = await summarize(settings, promptInput, options);
|
|
467
|
+
if (summary && summary.trim().length > 0) {
|
|
468
|
+
return { summary, failure: "" };
|
|
469
|
+
}
|
|
470
|
+
return { summary: null, failure: "summarizer returned no summary" };
|
|
471
|
+
} catch (error) {
|
|
472
|
+
return { summary: null, failure: error instanceof Error ? error.message : String(error) };
|
|
473
|
+
}
|
|
474
|
+
}
|
|
428
475
|
|
|
429
476
|
// src/activities/environment.ts
|
|
430
477
|
import {
|
|
431
478
|
applyGitAuthPointerEnvironment,
|
|
479
|
+
firstPartyMcpWorkspaceUrl,
|
|
432
480
|
stableSandboxEnvironmentForRun
|
|
433
481
|
} from "@opengeni/config";
|
|
482
|
+
import {
|
|
483
|
+
signDelegatedAccessToken
|
|
484
|
+
} from "@opengeni/contracts";
|
|
434
485
|
import {
|
|
435
486
|
loadWorkspaceEnvironmentForRun as loadWorkspaceEnvironmentForRunFromDb
|
|
436
487
|
} from "@opengeni/db";
|
|
@@ -466,10 +517,24 @@ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
|
|
|
466
517
|
}
|
|
467
518
|
}
|
|
468
519
|
async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
|
|
469
|
-
const
|
|
520
|
+
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
521
|
+
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
522
|
+
let toolspaceToken;
|
|
523
|
+
if (settings.toolspaceEnabled && !options.skipToolspaceToken && settings.delegationSecret && options.scope && options.sessionId && options.runId) {
|
|
524
|
+
toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
|
|
525
|
+
accountId: options.scope.accountId,
|
|
526
|
+
workspaceId: options.scope.workspaceId,
|
|
527
|
+
subjectId: `sandbox:${options.runId}`,
|
|
528
|
+
subjectLabel: "sandbox toolspace",
|
|
529
|
+
permissions: ["toolspace:call"],
|
|
530
|
+
sessionId: options.sessionId,
|
|
531
|
+
exp: Math.floor(Date.now() / 1e3) + 60 * 60
|
|
532
|
+
});
|
|
533
|
+
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(settings, options.scope.workspaceId);
|
|
534
|
+
}
|
|
470
535
|
const selection = githubRepositorySelection(resources);
|
|
471
536
|
if (!selection || options.skipGitHubToken) {
|
|
472
|
-
return { environment };
|
|
537
|
+
return { environment, ...toolspaceToken ? { toolspaceToken } : {} };
|
|
473
538
|
}
|
|
474
539
|
let token;
|
|
475
540
|
let identity;
|
|
@@ -491,7 +556,7 @@ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironmen
|
|
|
491
556
|
identity = githubAppBotIdentity(settings);
|
|
492
557
|
}
|
|
493
558
|
applyGitAuthPointerEnvironment(environment, identity);
|
|
494
|
-
return { environment, gitToken: token };
|
|
559
|
+
return { environment, gitToken: token, ...toolspaceToken ? { toolspaceToken } : {} };
|
|
495
560
|
}
|
|
496
561
|
function githubRepositorySelection(resources) {
|
|
497
562
|
const selected = resources.flatMap((resource) => {
|
|
@@ -1109,6 +1174,7 @@ function createRuntimeBatcher(flushEvents) {
|
|
|
1109
1174
|
"agent.toolCall.created",
|
|
1110
1175
|
"agent.toolCall.output",
|
|
1111
1176
|
"agent.message.completed",
|
|
1177
|
+
"tool.auth_needed",
|
|
1112
1178
|
"session.requiresAction",
|
|
1113
1179
|
"turn.completed",
|
|
1114
1180
|
"turn.failed",
|
|
@@ -1240,6 +1306,15 @@ function asSandboxWarmingError(error, backend, timeoutMs) {
|
|
|
1240
1306
|
const message = error instanceof Error ? error.message : String(error);
|
|
1241
1307
|
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
|
|
1242
1308
|
}
|
|
1309
|
+
function recordSandboxWarmingTimeout(metrics, error) {
|
|
1310
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
try {
|
|
1314
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
1315
|
+
} catch {
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1243
1318
|
function workspaceArchiveFromEnvelope(envelope) {
|
|
1244
1319
|
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1245
1320
|
const archive = sessionState?.workspaceArchive;
|
|
@@ -1316,6 +1391,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1316
1391
|
sessionId: ids.sessionId,
|
|
1317
1392
|
backendOverride: ids.backend,
|
|
1318
1393
|
...ids.environment ? { environment: ids.environment } : {},
|
|
1394
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
1319
1395
|
onSandboxCreated: async (created) => {
|
|
1320
1396
|
createdEstablished = created;
|
|
1321
1397
|
const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
|
|
@@ -1381,7 +1457,9 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1381
1457
|
});
|
|
1382
1458
|
}
|
|
1383
1459
|
await release();
|
|
1384
|
-
|
|
1460
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1461
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1462
|
+
throw warmingError;
|
|
1385
1463
|
}
|
|
1386
1464
|
}
|
|
1387
1465
|
try {
|
|
@@ -1394,13 +1472,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1394
1472
|
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
1395
1473
|
sessionId: ids.sessionId,
|
|
1396
1474
|
backendOverride: ids.backend,
|
|
1397
|
-
...ids.environment ? { environment: ids.environment } : {}
|
|
1475
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1476
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
1398
1477
|
});
|
|
1399
1478
|
await ensureDisplayStack(settings, established);
|
|
1400
1479
|
return { established, leaseEpoch, release };
|
|
1401
1480
|
} catch (error) {
|
|
1402
1481
|
await release();
|
|
1403
|
-
|
|
1482
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1483
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1484
|
+
throw warmingError;
|
|
1404
1485
|
}
|
|
1405
1486
|
}
|
|
1406
1487
|
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
@@ -1627,6 +1708,201 @@ function routingEnabled(settings) {
|
|
|
1627
1708
|
return settings.sandboxSelfhostedEnabled === true;
|
|
1628
1709
|
}
|
|
1629
1710
|
|
|
1711
|
+
// src/observability-metrics.ts
|
|
1712
|
+
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
1713
|
+
function observabilityEventLogger(observability) {
|
|
1714
|
+
return {
|
|
1715
|
+
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
1716
|
+
warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
function eventAttributes(attributes) {
|
|
1720
|
+
if (!attributes) {
|
|
1721
|
+
return void 0;
|
|
1722
|
+
}
|
|
1723
|
+
const sanitized = {};
|
|
1724
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
1725
|
+
sanitized[key] = eventAttributeValue(value);
|
|
1726
|
+
}
|
|
1727
|
+
return sanitized;
|
|
1728
|
+
}
|
|
1729
|
+
function eventAttributeValue(value) {
|
|
1730
|
+
if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1731
|
+
return value;
|
|
1732
|
+
}
|
|
1733
|
+
try {
|
|
1734
|
+
return JSON.stringify(value);
|
|
1735
|
+
} catch {
|
|
1736
|
+
return String(value);
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
function runtimeMetricsHooksForObservability(observability) {
|
|
1740
|
+
return {
|
|
1741
|
+
onModelCall: ({ provider, outcome, durationSeconds }) => {
|
|
1742
|
+
observability.incrementCounter({
|
|
1743
|
+
name: "opengeni_model_calls_total",
|
|
1744
|
+
help: "Total model calls by provider and outcome.",
|
|
1745
|
+
labels: { provider, outcome }
|
|
1746
|
+
});
|
|
1747
|
+
observability.observeHistogram({
|
|
1748
|
+
name: "opengeni_model_call_duration_seconds",
|
|
1749
|
+
help: "Model call duration in seconds by provider.",
|
|
1750
|
+
labels: { provider },
|
|
1751
|
+
value: durationSeconds
|
|
1752
|
+
});
|
|
1753
|
+
},
|
|
1754
|
+
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
1755
|
+
observability.incrementCounter({
|
|
1756
|
+
name: "opengeni_sandbox_creates_total",
|
|
1757
|
+
help: "Total sandbox create attempts by backend and outcome.",
|
|
1758
|
+
labels: { backend, outcome }
|
|
1759
|
+
});
|
|
1760
|
+
observability.observeHistogram({
|
|
1761
|
+
name: "opengeni_sandbox_create_duration_seconds",
|
|
1762
|
+
help: "Sandbox create duration in seconds by backend.",
|
|
1763
|
+
labels: { backend },
|
|
1764
|
+
value: durationSeconds
|
|
1765
|
+
});
|
|
1766
|
+
},
|
|
1767
|
+
onSandboxWarmingTimeout: () => {
|
|
1768
|
+
observability.incrementCounter({
|
|
1769
|
+
name: "opengeni_sandbox_warming_timeouts_total",
|
|
1770
|
+
help: "Total sandbox warming timeouts."
|
|
1771
|
+
});
|
|
1772
|
+
}
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
function turnLifecycleMetricsFor(observability) {
|
|
1776
|
+
const existing = turnTrackers.get(observability);
|
|
1777
|
+
if (existing) {
|
|
1778
|
+
return existing;
|
|
1779
|
+
}
|
|
1780
|
+
const tracker = new TurnLifecycleMetrics(observability);
|
|
1781
|
+
turnTrackers.set(observability, tracker);
|
|
1782
|
+
return tracker;
|
|
1783
|
+
}
|
|
1784
|
+
var TurnLifecycleMetrics = class {
|
|
1785
|
+
constructor(observability, options = {}) {
|
|
1786
|
+
this.observability = observability;
|
|
1787
|
+
this.options = options;
|
|
1788
|
+
}
|
|
1789
|
+
observability;
|
|
1790
|
+
options;
|
|
1791
|
+
startedTurns = /* @__PURE__ */ new Map();
|
|
1792
|
+
timer = null;
|
|
1793
|
+
start(turnId) {
|
|
1794
|
+
this.startedTurns.set(turnId, this.now());
|
|
1795
|
+
this.ensureTimer();
|
|
1796
|
+
this.refreshGauges();
|
|
1797
|
+
}
|
|
1798
|
+
finish(turnId, outcome, durationSeconds) {
|
|
1799
|
+
const startedAt = this.startedTurns.get(turnId);
|
|
1800
|
+
if (startedAt !== void 0) {
|
|
1801
|
+
this.startedTurns.delete(turnId);
|
|
1802
|
+
}
|
|
1803
|
+
if (outcome) {
|
|
1804
|
+
const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
|
|
1805
|
+
this.observability.incrementCounter({
|
|
1806
|
+
name: "opengeni_turns_total",
|
|
1807
|
+
help: "Total agent turns by terminal outcome.",
|
|
1808
|
+
labels: { outcome }
|
|
1809
|
+
});
|
|
1810
|
+
this.observability.observeHistogram({
|
|
1811
|
+
name: "opengeni_turn_duration_seconds",
|
|
1812
|
+
help: "Agent turn duration in seconds by terminal outcome.",
|
|
1813
|
+
labels: { outcome },
|
|
1814
|
+
value: observedDuration
|
|
1815
|
+
});
|
|
1816
|
+
}
|
|
1817
|
+
this.refreshGauges();
|
|
1818
|
+
if (this.startedTurns.size === 0) {
|
|
1819
|
+
this.stopTimer();
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
refreshGauges() {
|
|
1823
|
+
this.observability.setGauge({
|
|
1824
|
+
name: "opengeni_turns_inflight",
|
|
1825
|
+
help: "Current number of in-flight agent turns in this worker process.",
|
|
1826
|
+
value: this.startedTurns.size
|
|
1827
|
+
});
|
|
1828
|
+
this.observability.setGauge({
|
|
1829
|
+
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
1830
|
+
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
1831
|
+
value: this.oldestInflightAgeSeconds()
|
|
1832
|
+
});
|
|
1833
|
+
}
|
|
1834
|
+
stop() {
|
|
1835
|
+
this.startedTurns.clear();
|
|
1836
|
+
this.refreshGauges();
|
|
1837
|
+
this.stopTimer();
|
|
1838
|
+
}
|
|
1839
|
+
oldestInflightAgeSeconds() {
|
|
1840
|
+
if (this.startedTurns.size === 0) {
|
|
1841
|
+
return 0;
|
|
1842
|
+
}
|
|
1843
|
+
let oldest = Number.POSITIVE_INFINITY;
|
|
1844
|
+
for (const startedAt of this.startedTurns.values()) {
|
|
1845
|
+
oldest = Math.min(oldest, startedAt);
|
|
1846
|
+
}
|
|
1847
|
+
return Math.max(0, (this.now() - oldest) / 1e3);
|
|
1848
|
+
}
|
|
1849
|
+
ensureTimer() {
|
|
1850
|
+
if (this.timer) {
|
|
1851
|
+
return;
|
|
1852
|
+
}
|
|
1853
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
|
|
1854
|
+
this.timer.unref?.();
|
|
1855
|
+
}
|
|
1856
|
+
stopTimer() {
|
|
1857
|
+
if (!this.timer) {
|
|
1858
|
+
return;
|
|
1859
|
+
}
|
|
1860
|
+
clearInterval(this.timer);
|
|
1861
|
+
this.timer = null;
|
|
1862
|
+
}
|
|
1863
|
+
now() {
|
|
1864
|
+
return this.options.now?.() ?? Date.now();
|
|
1865
|
+
}
|
|
1866
|
+
};
|
|
1867
|
+
function recordTurnsQueuedGauge(observability, value) {
|
|
1868
|
+
observability.setGauge({
|
|
1869
|
+
name: "opengeni_turns_queued",
|
|
1870
|
+
help: "Current number of queued session turns.",
|
|
1871
|
+
value
|
|
1872
|
+
});
|
|
1873
|
+
}
|
|
1874
|
+
function recordSandboxLeaseGauges(observability, counts) {
|
|
1875
|
+
for (const liveness of ["cold", "warming", "warm", "draining"]) {
|
|
1876
|
+
observability.setGauge({
|
|
1877
|
+
name: "opengeni_sandbox_leases",
|
|
1878
|
+
help: "Current sandbox leases by liveness state.",
|
|
1879
|
+
labels: { liveness },
|
|
1880
|
+
value: counts[liveness] ?? 0
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1884
|
+
function recordSandboxOrphansTerminated(observability, count) {
|
|
1885
|
+
if (count <= 0) {
|
|
1886
|
+
return;
|
|
1887
|
+
}
|
|
1888
|
+
observability.incrementCounter({
|
|
1889
|
+
name: "opengeni_sandbox_orphans_terminated_total",
|
|
1890
|
+
help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
|
|
1891
|
+
amount: count
|
|
1892
|
+
});
|
|
1893
|
+
}
|
|
1894
|
+
function recordCreditMicros(observability, kind, amountMicros) {
|
|
1895
|
+
if (!observability || amountMicros <= 0) {
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
observability.incrementCounter({
|
|
1899
|
+
name: "opengeni_credit_micros_total",
|
|
1900
|
+
help: "Total credit micros recorded by kind.",
|
|
1901
|
+
labels: { kind },
|
|
1902
|
+
amount: amountMicros
|
|
1903
|
+
});
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1630
1906
|
// src/activities/recording.ts
|
|
1631
1907
|
import { deleteRecording, updateRecording } from "@opengeni/db";
|
|
1632
1908
|
import {
|
|
@@ -1794,6 +2070,10 @@ function isWorkerShutdownCancellation(error) {
|
|
|
1794
2070
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
1795
2071
|
}
|
|
1796
2072
|
var CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE = "The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
2073
|
+
var MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS = 3;
|
|
2074
|
+
function compactionFailureReason(reason) {
|
|
2075
|
+
return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
|
|
2076
|
+
}
|
|
1797
2077
|
function classifyContextWindowOverflowError(error) {
|
|
1798
2078
|
const fields = collectErrorStrings(error);
|
|
1799
2079
|
const matched = fields.find(
|
|
@@ -1920,6 +2200,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
1920
2200
|
"opengeni.trigger_event_id": input.triggerEventId
|
|
1921
2201
|
});
|
|
1922
2202
|
let activityStatus = "unknown";
|
|
2203
|
+
let turnMetricOutcome = null;
|
|
1923
2204
|
let activityError;
|
|
1924
2205
|
let turnId;
|
|
1925
2206
|
let heartbeatTimer;
|
|
@@ -2022,6 +2303,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2022
2303
|
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
2023
2304
|
}
|
|
2024
2305
|
turnId = turn.id;
|
|
2306
|
+
turnLifecycleMetricsFor(observability).start(turnId);
|
|
2025
2307
|
const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
|
|
2026
2308
|
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
2027
2309
|
const activityContext = currentActivityContext();
|
|
@@ -2122,6 +2404,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2122
2404
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2123
2405
|
], true);
|
|
2124
2406
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2407
|
+
turnMetricOutcome = "failed";
|
|
2125
2408
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2126
2409
|
activityStatus = "idle";
|
|
2127
2410
|
return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
|
|
@@ -2199,14 +2482,21 @@ function createRunAgentTurnActivity(services) {
|
|
|
2199
2482
|
async () => activeSandboxRecord?.kind ?? null
|
|
2200
2483
|
);
|
|
2201
2484
|
const machinePrimary = activeSandboxBackend === "selfhosted" && Boolean(activeSandboxPointer?.activeSandboxId) && Boolean(activeSandboxRecord?.enrollmentId);
|
|
2202
|
-
const {
|
|
2485
|
+
const {
|
|
2486
|
+
environment: sandboxEnvironment,
|
|
2487
|
+
gitToken: sandboxGitToken,
|
|
2488
|
+
toolspaceToken: sandboxToolspaceToken
|
|
2489
|
+
} = await sandboxEnvironmentForRun(
|
|
2203
2490
|
runSettings,
|
|
2204
2491
|
turnResources,
|
|
2205
2492
|
workspaceEnvironment?.values ?? {},
|
|
2206
2493
|
{
|
|
2207
2494
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
2495
|
+
skipToolspaceToken: activeSandboxBackend === "selfhosted",
|
|
2208
2496
|
scope: connectionScope,
|
|
2209
|
-
gitCredentials: connectionCredentials?.gitCredentials
|
|
2497
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
2498
|
+
sessionId: input.sessionId,
|
|
2499
|
+
runId: turnId
|
|
2210
2500
|
}
|
|
2211
2501
|
);
|
|
2212
2502
|
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
@@ -2260,7 +2550,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2260
2550
|
};
|
|
2261
2551
|
} else {
|
|
2262
2552
|
resolvedSandbox = await resumeBoxForTurn(
|
|
2263
|
-
{ db, settings },
|
|
2553
|
+
{ db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
|
|
2264
2554
|
{
|
|
2265
2555
|
accountId: input.accountId,
|
|
2266
2556
|
workspaceId: input.workspaceId,
|
|
@@ -2322,7 +2612,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2322
2612
|
expectedEpoch: heartbeatEpoch,
|
|
2323
2613
|
warmRateMicrosPerSecond: warmRate,
|
|
2324
2614
|
subjectId: input.sessionId
|
|
2325
|
-
}).catch(() => void 0);
|
|
2615
|
+
}).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
|
|
2326
2616
|
}, 1e4);
|
|
2327
2617
|
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
2328
2618
|
leaseHeartbeatTimer.unref();
|
|
@@ -2360,12 +2650,17 @@ function createRunAgentTurnActivity(services) {
|
|
|
2360
2650
|
}
|
|
2361
2651
|
const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
|
|
2362
2652
|
throwIfWorkerShuttingDown();
|
|
2653
|
+
const resolveCredential = buildConnectionTokenResolver(db, runSettings);
|
|
2363
2654
|
preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
|
|
2364
2655
|
accountId: input.accountId,
|
|
2365
2656
|
workspaceId: input.workspaceId,
|
|
2366
2657
|
sessionId: input.sessionId,
|
|
2367
2658
|
subjectId: "worker:first-party-mcp",
|
|
2368
2659
|
subjectLabel: "OpenGeni worker",
|
|
2660
|
+
resolveCredential,
|
|
2661
|
+
onAuthNeeded: async (payload) => {
|
|
2662
|
+
await publish([{ type: "tool.auth_needed", payload }], true);
|
|
2663
|
+
},
|
|
2369
2664
|
// Manager-style sessions carry a creation-validated permission set
|
|
2370
2665
|
// for their first-party MCP token; null keeps the fixed default.
|
|
2371
2666
|
...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}
|
|
@@ -2382,6 +2677,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2382
2677
|
// seeds it to the box's token file before the repository-clone runs; it never
|
|
2383
2678
|
// touches the box/agent manifest env.
|
|
2384
2679
|
...activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
2680
|
+
...activeSandboxBackend !== "selfhosted" && sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {},
|
|
2385
2681
|
...activeSandboxBackend ? { activeSandboxBackend } : {},
|
|
2386
2682
|
fileResourceDownloads,
|
|
2387
2683
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -2437,11 +2733,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
2437
2733
|
}
|
|
2438
2734
|
} : {}
|
|
2439
2735
|
});
|
|
2440
|
-
const compactSummarizer = resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
|
|
2736
|
+
const compactSummarizer = resolvedModel ? (s, m, o) => withCodex(() => summarizeForCompaction2(s, m, {
|
|
2441
2737
|
client: resolvedModel.client,
|
|
2442
2738
|
api: resolvedModel.provider.api,
|
|
2443
2739
|
model: resolvedModel.configured.id,
|
|
2444
|
-
maxOutputTokens: SUMMARY_BUFFER_TOKENS2
|
|
2740
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
|
|
2741
|
+
...o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}
|
|
2445
2742
|
})) : void 0;
|
|
2446
2743
|
if (triggerType === "user.message" || triggerType === "goal.continuation") {
|
|
2447
2744
|
try {
|
|
@@ -2498,7 +2795,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2498
2795
|
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
2499
2796
|
session.lastInputTokens,
|
|
2500
2797
|
compactSummarizer,
|
|
2501
|
-
{ force: true }
|
|
2798
|
+
{ force: true, requireShrink: true }
|
|
2502
2799
|
);
|
|
2503
2800
|
if (outcome.compacted) {
|
|
2504
2801
|
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
@@ -2567,7 +2864,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
2567
2864
|
responseId: responseUsage.responseId,
|
|
2568
2865
|
dispatchId,
|
|
2569
2866
|
positionalKey: `response-${responseUsageCount}`
|
|
2570
|
-
})
|
|
2867
|
+
}),
|
|
2868
|
+
observability
|
|
2571
2869
|
});
|
|
2572
2870
|
await reconcileConversationTruth();
|
|
2573
2871
|
try {
|
|
@@ -2613,7 +2911,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
2613
2911
|
model: turn.model,
|
|
2614
2912
|
isCodexTurn,
|
|
2615
2913
|
usage: stream.state.usage,
|
|
2616
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
|
|
2914
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
2915
|
+
observability
|
|
2617
2916
|
});
|
|
2618
2917
|
}
|
|
2619
2918
|
if (lastInputTokensObserved !== null) {
|
|
@@ -2661,6 +2960,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2661
2960
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2662
2961
|
], true);
|
|
2663
2962
|
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
2963
|
+
turnMetricOutcome = "completed";
|
|
2664
2964
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2665
2965
|
await recordUsageEvent2(db, {
|
|
2666
2966
|
accountId: input.accountId,
|
|
@@ -2676,7 +2976,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2676
2976
|
return { status: "idle" };
|
|
2677
2977
|
};
|
|
2678
2978
|
await prepareRunAttemptInput();
|
|
2679
|
-
let
|
|
2979
|
+
let compactionRecoveryAttempts = 0;
|
|
2680
2980
|
let retriedAfterCompaction = false;
|
|
2681
2981
|
while (true) {
|
|
2682
2982
|
try {
|
|
@@ -2692,10 +2992,16 @@ function createRunAgentTurnActivity(services) {
|
|
|
2692
2992
|
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
2693
2993
|
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
2694
2994
|
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
2695
|
-
if (
|
|
2995
|
+
if (recoveryKind && compactionRecoveryAttempts >= MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS) {
|
|
2996
|
+
throw new Error(
|
|
2997
|
+
`compaction summarization failed: context compaction recovery attempt cap reached (${MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS})`,
|
|
2998
|
+
{ cause: attemptError }
|
|
2999
|
+
);
|
|
3000
|
+
}
|
|
3001
|
+
if (!recoveryKind || !publish || !turnStartedPublished) {
|
|
2696
3002
|
throw attemptError;
|
|
2697
3003
|
}
|
|
2698
|
-
|
|
3004
|
+
compactionRecoveryAttempts += 1;
|
|
2699
3005
|
await flushRuntimeBatcher();
|
|
2700
3006
|
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
2701
3007
|
const progressPersisted = modelOrToolProgressPersisted;
|
|
@@ -2710,16 +3016,46 @@ function createRunAgentTurnActivity(services) {
|
|
|
2710
3016
|
thresholdTokens: compactionNeeded?.thresholdTokens
|
|
2711
3017
|
});
|
|
2712
3018
|
let compacted = false;
|
|
3019
|
+
let compactionFailureMessage = null;
|
|
2713
3020
|
try {
|
|
2714
3021
|
const outcome = await forceContextCompaction(recoveryKind);
|
|
2715
3022
|
compacted = outcome.compacted;
|
|
3023
|
+
if (!outcome.compacted) {
|
|
3024
|
+
compactionFailureMessage = compactionFailureReason(outcome.reason);
|
|
3025
|
+
}
|
|
2716
3026
|
} catch (compactError) {
|
|
3027
|
+
compactionFailureMessage = compactionFailureReason(compactError instanceof Error ? compactError.message : String(compactError));
|
|
2717
3028
|
observability.warn("context compaction recovery compaction failed", {
|
|
2718
3029
|
sessionId: input.sessionId,
|
|
2719
3030
|
turnId: activeTurnId,
|
|
2720
|
-
error:
|
|
3031
|
+
error: compactionFailureMessage
|
|
2721
3032
|
});
|
|
2722
3033
|
}
|
|
3034
|
+
if (!compacted) {
|
|
3035
|
+
const errorMessage = compactionFailureMessage ?? "compaction summarization failed: compaction produced no replacement history";
|
|
3036
|
+
if (!progressPersisted) {
|
|
3037
|
+
throw new Error(errorMessage, { cause: attemptError });
|
|
3038
|
+
}
|
|
3039
|
+
await publish([
|
|
3040
|
+
{
|
|
3041
|
+
type: "turn.failed",
|
|
3042
|
+
payload: {
|
|
3043
|
+
error: errorMessage,
|
|
3044
|
+
code: "context_compaction_failed",
|
|
3045
|
+
retryable: false,
|
|
3046
|
+
recovery: "user_message",
|
|
3047
|
+
compacted: false
|
|
3048
|
+
}
|
|
3049
|
+
},
|
|
3050
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
3051
|
+
], true);
|
|
3052
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
3053
|
+
turnMetricOutcome = "failed";
|
|
3054
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
3055
|
+
activityStatus = "idle";
|
|
3056
|
+
activityError = attemptError;
|
|
3057
|
+
return { status: "idle" };
|
|
3058
|
+
}
|
|
2723
3059
|
if (progressPersisted) {
|
|
2724
3060
|
const triggerPayload = trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload) ? trigger.payload : {};
|
|
2725
3061
|
const compactionResumedTurn = triggerType === "turn.preempted" && (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
|
|
@@ -2741,6 +3077,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2741
3077
|
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2742
3078
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2743
3079
|
activityStatus = "preempted";
|
|
3080
|
+
turnMetricOutcome = "preempted";
|
|
2744
3081
|
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
2745
3082
|
sessionId: input.sessionId,
|
|
2746
3083
|
turnId: activeTurnId,
|
|
@@ -2763,6 +3100,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2763
3100
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2764
3101
|
], true);
|
|
2765
3102
|
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
3103
|
+
turnMetricOutcome = "failed";
|
|
2766
3104
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2767
3105
|
activityStatus = "idle";
|
|
2768
3106
|
activityError = attemptError;
|
|
@@ -2791,6 +3129,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2791
3129
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
2792
3130
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2793
3131
|
activityStatus = "preempted";
|
|
3132
|
+
turnMetricOutcome = "preempted";
|
|
2794
3133
|
return { status: "preempted" };
|
|
2795
3134
|
} catch (requeueError) {
|
|
2796
3135
|
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
@@ -2838,6 +3177,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2838
3177
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2839
3178
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2840
3179
|
activityStatus = "preempted";
|
|
3180
|
+
turnMetricOutcome = "preempted";
|
|
2841
3181
|
return { status: "preempted" };
|
|
2842
3182
|
} catch (preemptError) {
|
|
2843
3183
|
console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
|
|
@@ -2849,6 +3189,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2849
3189
|
await flushRuntimeBatcher();
|
|
2850
3190
|
if (preemptTurnId) {
|
|
2851
3191
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
|
|
3192
|
+
turnMetricOutcome = "cancelled";
|
|
2852
3193
|
}
|
|
2853
3194
|
throw error;
|
|
2854
3195
|
}
|
|
@@ -2873,6 +3214,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2873
3214
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2874
3215
|
], true);
|
|
2875
3216
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
3217
|
+
turnMetricOutcome = "completed";
|
|
2876
3218
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2877
3219
|
await recordUsageEvent2(db, {
|
|
2878
3220
|
accountId: input.accountId,
|
|
@@ -2962,6 +3304,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2962
3304
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2963
3305
|
], true);
|
|
2964
3306
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3307
|
+
turnMetricOutcome = "failed";
|
|
2965
3308
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2966
3309
|
activityStatus = "idle";
|
|
2967
3310
|
activityError = error;
|
|
@@ -2994,6 +3337,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2994
3337
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2995
3338
|
], true);
|
|
2996
3339
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
3340
|
+
turnMetricOutcome = "completed";
|
|
2997
3341
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2998
3342
|
await recordUsageEvent2(db, {
|
|
2999
3343
|
accountId: input.accountId,
|
|
@@ -3032,6 +3376,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
3032
3376
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
3033
3377
|
], true);
|
|
3034
3378
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3379
|
+
turnMetricOutcome = "failed";
|
|
3035
3380
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
3036
3381
|
activityStatus = "idle";
|
|
3037
3382
|
activityError = error;
|
|
@@ -3047,6 +3392,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
3047
3392
|
{ type: "session.status.changed", payload: { status: "failed" } }
|
|
3048
3393
|
], true);
|
|
3049
3394
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3395
|
+
turnMetricOutcome = "failed";
|
|
3050
3396
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
3051
3397
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
|
|
3052
3398
|
return { status: "failed" };
|
|
@@ -3057,6 +3403,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
3057
3403
|
status: activityStatus,
|
|
3058
3404
|
durationSeconds
|
|
3059
3405
|
});
|
|
3406
|
+
if (turnId && activityStatus !== "unknown") {
|
|
3407
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
3408
|
+
}
|
|
3060
3409
|
activitySpan.end({
|
|
3061
3410
|
attributes: {
|
|
3062
3411
|
"opengeni.turn_id": turnId ?? "",
|
|
@@ -3279,7 +3628,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
3279
3628
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
|
|
3280
3629
|
});
|
|
3281
3630
|
if (costMicros > 0) {
|
|
3282
|
-
await applyCreditDebitUpToBalance(db, {
|
|
3631
|
+
const result = await applyCreditDebitUpToBalance(db, {
|
|
3283
3632
|
accountId: input.accountId,
|
|
3284
3633
|
workspaceId: input.workspaceId,
|
|
3285
3634
|
type: "model_usage_debit",
|
|
@@ -3297,6 +3646,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
3297
3646
|
totalTokens
|
|
3298
3647
|
}
|
|
3299
3648
|
});
|
|
3649
|
+
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
3300
3650
|
}
|
|
3301
3651
|
}
|
|
3302
3652
|
function positiveInt(value) {
|
|
@@ -3417,6 +3767,8 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
|
|
|
3417
3767
|
import {
|
|
3418
3768
|
accrueWarmSeconds as accrueWarmSeconds2,
|
|
3419
3769
|
confirmDrainCold,
|
|
3770
|
+
countQueuedTurns,
|
|
3771
|
+
countSandboxLeasesByLiveness,
|
|
3420
3772
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3421
3773
|
getBillingBalance as getBillingBalance4,
|
|
3422
3774
|
listLiveModalSandboxLeaseAttributions,
|
|
@@ -3443,6 +3795,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3443
3795
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3444
3796
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3445
3797
|
}
|
|
3798
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
3446
3799
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3447
3800
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
3448
3801
|
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
@@ -3472,11 +3825,13 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3472
3825
|
}
|
|
3473
3826
|
try {
|
|
3474
3827
|
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
3828
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
3475
3829
|
} catch (error) {
|
|
3476
3830
|
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
3477
3831
|
error: error instanceof Error ? error.message : String(error)
|
|
3478
3832
|
});
|
|
3479
3833
|
}
|
|
3834
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
3480
3835
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3481
3836
|
observability.info("sandbox reaper swept", {
|
|
3482
3837
|
drainable: drainable.length,
|
|
@@ -3518,6 +3873,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3518
3873
|
if (result.accrued) {
|
|
3519
3874
|
accrued += 1;
|
|
3520
3875
|
}
|
|
3876
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3521
3877
|
} catch (error) {
|
|
3522
3878
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3523
3879
|
workspaceId: lease.workspaceId,
|
|
@@ -3528,6 +3884,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3528
3884
|
}
|
|
3529
3885
|
return { accrued, workspaceIds };
|
|
3530
3886
|
}
|
|
3887
|
+
async function refreshQueueAndLeaseGauges(db, observability) {
|
|
3888
|
+
try {
|
|
3889
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
3890
|
+
} catch (error) {
|
|
3891
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
3892
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3893
|
+
});
|
|
3894
|
+
}
|
|
3895
|
+
try {
|
|
3896
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
3897
|
+
} catch (error) {
|
|
3898
|
+
observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
|
|
3899
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3900
|
+
});
|
|
3901
|
+
}
|
|
3902
|
+
}
|
|
3531
3903
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3532
3904
|
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
3533
3905
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
@@ -4058,6 +4430,7 @@ function startOfUtcMonth4() {
|
|
|
4058
4430
|
// src/activities/session-state.ts
|
|
4059
4431
|
import {
|
|
4060
4432
|
claimNextQueuedTurn as claimNextQueuedTurnDb2,
|
|
4433
|
+
countQueuedTurns as countQueuedTurns2,
|
|
4061
4434
|
countTurnSessionHistoryItems,
|
|
4062
4435
|
finishTurn as finishTurn2,
|
|
4063
4436
|
getSessionEvent as getSessionEvent4,
|
|
@@ -4101,7 +4474,7 @@ function createSessionStateActivities(services) {
|
|
|
4101
4474
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
|
|
4102
4475
|
}
|
|
4103
4476
|
async function interruptActiveTurn2(input) {
|
|
4104
|
-
const { db, bus } = await services();
|
|
4477
|
+
const { db, bus, observability } = await services();
|
|
4105
4478
|
const session = await requireSession4(db, input.workspaceId, input.sessionId);
|
|
4106
4479
|
const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
|
|
4107
4480
|
if (!isSteerInterrupt(trigger)) {
|
|
@@ -4124,9 +4497,10 @@ function createSessionStateActivities(services) {
|
|
|
4124
4497
|
]);
|
|
4125
4498
|
await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
4126
4499
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
4500
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4127
4501
|
}
|
|
4128
4502
|
async function requeueTurnAfterWorkerDeath2(input) {
|
|
4129
|
-
const { settings, db, bus } = await services();
|
|
4503
|
+
const { settings, db, bus, observability } = await services();
|
|
4130
4504
|
const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
|
|
4131
4505
|
if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
|
|
4132
4506
|
return { action: "stale" };
|
|
@@ -4166,11 +4540,14 @@ function createSessionStateActivities(services) {
|
|
|
4166
4540
|
throw requeueError;
|
|
4167
4541
|
}
|
|
4168
4542
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
4543
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4169
4544
|
return { action: "requeued", redispatches };
|
|
4170
4545
|
}
|
|
4171
4546
|
async function claimNextQueuedTurn2(input) {
|
|
4172
|
-
const { db } = await services();
|
|
4173
|
-
|
|
4547
|
+
const { db, observability } = await services();
|
|
4548
|
+
const turn = await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
4549
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4550
|
+
return turn;
|
|
4174
4551
|
}
|
|
4175
4552
|
async function markSessionIdle2(input) {
|
|
4176
4553
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
@@ -4178,6 +4555,7 @@ function createSessionStateActivities(services) {
|
|
|
4178
4555
|
if (session.status === "queued" || session.status === "running") {
|
|
4179
4556
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
|
|
4180
4557
|
}
|
|
4558
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4181
4559
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
|
|
4182
4560
|
}
|
|
4183
4561
|
return {
|
|
@@ -4188,6 +4566,12 @@ function createSessionStateActivities(services) {
|
|
|
4188
4566
|
markSessionIdle: markSessionIdle2
|
|
4189
4567
|
};
|
|
4190
4568
|
}
|
|
4569
|
+
async function refreshQueuedTurnsGauge(db, observability) {
|
|
4570
|
+
try {
|
|
4571
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns2(db));
|
|
4572
|
+
} catch {
|
|
4573
|
+
}
|
|
4574
|
+
}
|
|
4191
4575
|
|
|
4192
4576
|
// src/activities.ts
|
|
4193
4577
|
function createActivities(dependencies = {}) {
|
|
@@ -4195,6 +4579,7 @@ function createActivities(dependencies = {}) {
|
|
|
4195
4579
|
async function services() {
|
|
4196
4580
|
servicesPromise ??= (async () => {
|
|
4197
4581
|
const settings = dependencies.settings ?? getSettings();
|
|
4582
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
4198
4583
|
const searchPath = dbSearchPath(settings);
|
|
4199
4584
|
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
4200
4585
|
...searchPath ? { searchPath } : {},
|
|
@@ -4219,12 +4604,15 @@ function createActivities(dependencies = {}) {
|
|
|
4219
4604
|
// events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
|
|
4220
4605
|
bus: dependencies.bus ?? await createNatsEventBus(
|
|
4221
4606
|
settings.natsUrl,
|
|
4222
|
-
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
|
|
4607
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
4608
|
+
{ logger: observabilityEventLogger(observability) }
|
|
4223
4609
|
),
|
|
4224
|
-
runtime: dependencies.runtime ?? createProductionAgentRuntime(
|
|
4610
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime({
|
|
4611
|
+
metrics: runtimeMetricsHooksForObservability(observability)
|
|
4612
|
+
}),
|
|
4225
4613
|
objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
|
|
4226
4614
|
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
4227
|
-
observability
|
|
4615
|
+
observability,
|
|
4228
4616
|
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
4229
4617
|
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
4230
4618
|
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
|
@@ -4273,6 +4661,86 @@ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
|
|
|
4273
4661
|
var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
|
|
4274
4662
|
var reapSandboxLeases = defaultActivities.reapSandboxLeases;
|
|
4275
4663
|
|
|
4664
|
+
// src/http.ts
|
|
4665
|
+
import { dbSql } from "@opengeni/db";
|
|
4666
|
+
function startWorkerHttpServer(input) {
|
|
4667
|
+
const { settings, observability, checks } = input;
|
|
4668
|
+
const timeoutMs = input.timeoutMs ?? 2e3;
|
|
4669
|
+
return Bun.serve({
|
|
4670
|
+
hostname: "0.0.0.0",
|
|
4671
|
+
port: settings.workerHttpPort,
|
|
4672
|
+
async fetch(request) {
|
|
4673
|
+
const url = new URL(request.url);
|
|
4674
|
+
if (request.method !== "GET") {
|
|
4675
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
4676
|
+
}
|
|
4677
|
+
if (url.pathname === "/healthz") {
|
|
4678
|
+
return Response.json({
|
|
4679
|
+
service: settings.serviceName,
|
|
4680
|
+
environment: settings.environment,
|
|
4681
|
+
deploymentRevision: settings.deploymentRevision,
|
|
4682
|
+
ok: true
|
|
4683
|
+
});
|
|
4684
|
+
}
|
|
4685
|
+
if (url.pathname === "/metrics") {
|
|
4686
|
+
return new Response(await observability.prometheusMetrics(), {
|
|
4687
|
+
status: 200,
|
|
4688
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
|
|
4689
|
+
});
|
|
4690
|
+
}
|
|
4691
|
+
if (url.pathname === "/readyz") {
|
|
4692
|
+
const result = await runReadinessChecks(checks, timeoutMs);
|
|
4693
|
+
return Response.json(result, { status: result.ok ? 200 : 503 });
|
|
4694
|
+
}
|
|
4695
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
4696
|
+
}
|
|
4697
|
+
});
|
|
4698
|
+
}
|
|
4699
|
+
async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
4700
|
+
const entries = await Promise.all(
|
|
4701
|
+
Object.entries(checks).map(async ([name, check]) => {
|
|
4702
|
+
try {
|
|
4703
|
+
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
4704
|
+
return [name, { ok: true }];
|
|
4705
|
+
} catch (error) {
|
|
4706
|
+
return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }];
|
|
4707
|
+
}
|
|
4708
|
+
})
|
|
4709
|
+
);
|
|
4710
|
+
const result = Object.fromEntries(entries);
|
|
4711
|
+
return {
|
|
4712
|
+
ok: Object.values(result).every((check) => check.ok),
|
|
4713
|
+
checks: result
|
|
4714
|
+
};
|
|
4715
|
+
}
|
|
4716
|
+
function dbReadyCheck(db) {
|
|
4717
|
+
return async () => {
|
|
4718
|
+
await db.execute(dbSql`select 1`);
|
|
4719
|
+
};
|
|
4720
|
+
}
|
|
4721
|
+
function natsReadyCheck(bus) {
|
|
4722
|
+
return () => {
|
|
4723
|
+
if (bus.isConnected && !bus.isConnected()) {
|
|
4724
|
+
throw new Error("NATS is not connected");
|
|
4725
|
+
}
|
|
4726
|
+
};
|
|
4727
|
+
}
|
|
4728
|
+
async function withTimeout(promise, timeoutMs) {
|
|
4729
|
+
let timer;
|
|
4730
|
+
try {
|
|
4731
|
+
return await Promise.race([
|
|
4732
|
+
promise,
|
|
4733
|
+
new Promise((_, reject) => {
|
|
4734
|
+
timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
4735
|
+
})
|
|
4736
|
+
]);
|
|
4737
|
+
} finally {
|
|
4738
|
+
if (timer) {
|
|
4739
|
+
clearTimeout(timer);
|
|
4740
|
+
}
|
|
4741
|
+
}
|
|
4742
|
+
}
|
|
4743
|
+
|
|
4276
4744
|
// src/index.ts
|
|
4277
4745
|
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
4278
4746
|
async function createOpenGeniWorker(options = {}) {
|
|
@@ -4313,6 +4781,9 @@ async function createWorkerWorkflowSignaler(settings) {
|
|
|
4313
4781
|
signal: "queueChanged"
|
|
4314
4782
|
});
|
|
4315
4783
|
},
|
|
4784
|
+
check: async () => {
|
|
4785
|
+
await connection.workflowService.getSystemInfo({});
|
|
4786
|
+
},
|
|
4316
4787
|
close: async () => {
|
|
4317
4788
|
await connection.close();
|
|
4318
4789
|
}
|
|
@@ -4370,35 +4841,71 @@ async function registerSandboxReaperSchedule(settings, observability) {
|
|
|
4370
4841
|
async function startWorker() {
|
|
4371
4842
|
const settings = getSettings2();
|
|
4372
4843
|
const observability = createObservability2(settings, { component: "worker" });
|
|
4373
|
-
const
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
}
|
|
4380
|
-
);
|
|
4381
|
-
const { worker, connection } = await createOpenGeniWorker({
|
|
4382
|
-
settings,
|
|
4383
|
-
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
|
|
4384
|
-
});
|
|
4385
|
-
const reaperSchedule = await retryStartupDependency(
|
|
4386
|
-
"Temporal schedule (sandbox reaper)",
|
|
4387
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
4388
|
-
{
|
|
4389
|
-
...startupRetryOptions(settings),
|
|
4390
|
-
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
4391
|
-
}
|
|
4392
|
-
);
|
|
4393
|
-
observability.info("OpenGeni worker listening", {
|
|
4394
|
-
temporalTaskQueue: settings.temporalTaskQueue
|
|
4844
|
+
const retryOptions = startupRetryOptions(settings);
|
|
4845
|
+
const onRetry = (event) => logStartupDependencyRetry(observability, event);
|
|
4846
|
+
const searchPath = dbSearchPath2(settings);
|
|
4847
|
+
const dbClient = createDb2(settings.databaseUrl, {
|
|
4848
|
+
...searchPath ? { searchPath } : {},
|
|
4849
|
+
rlsStrategy: settings.rlsStrategy
|
|
4395
4850
|
});
|
|
4851
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
|
|
4852
|
+
let bus;
|
|
4853
|
+
let signaler;
|
|
4854
|
+
let workerBundle;
|
|
4855
|
+
let reaperSchedule;
|
|
4856
|
+
let httpServer;
|
|
4396
4857
|
try {
|
|
4397
|
-
await
|
|
4858
|
+
bus = await retryStartupDependency(
|
|
4859
|
+
"NATS",
|
|
4860
|
+
() => createNatsEventBus2(
|
|
4861
|
+
settings.natsUrl,
|
|
4862
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
4863
|
+
{ logger: observabilityEventLogger(observability) }
|
|
4864
|
+
),
|
|
4865
|
+
{ ...retryOptions, onRetry }
|
|
4866
|
+
);
|
|
4867
|
+
signaler = await retryStartupDependency(
|
|
4868
|
+
"Temporal client",
|
|
4869
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
4870
|
+
{ ...retryOptions, onRetry }
|
|
4871
|
+
);
|
|
4872
|
+
workerBundle = await createOpenGeniWorker({
|
|
4873
|
+
settings,
|
|
4874
|
+
activityDependencies: {
|
|
4875
|
+
observability,
|
|
4876
|
+
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
4877
|
+
db: dbClient.db,
|
|
4878
|
+
bus
|
|
4879
|
+
}
|
|
4880
|
+
});
|
|
4881
|
+
httpServer = startWorkerHttpServer({
|
|
4882
|
+
settings,
|
|
4883
|
+
observability,
|
|
4884
|
+
checks: {
|
|
4885
|
+
db: dbReadyCheck(dbClient.db),
|
|
4886
|
+
nats: natsReadyCheck(bus),
|
|
4887
|
+
temporal: signaler.check
|
|
4888
|
+
}
|
|
4889
|
+
});
|
|
4890
|
+
reaperSchedule = await retryStartupDependency(
|
|
4891
|
+
"Temporal schedule (sandbox reaper)",
|
|
4892
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
4893
|
+
{ ...retryOptions, onRetry }
|
|
4894
|
+
);
|
|
4895
|
+
observability.info("OpenGeni worker listening", {
|
|
4896
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
4897
|
+
httpPort: settings.workerHttpPort
|
|
4898
|
+
});
|
|
4899
|
+
await workerBundle.worker.run();
|
|
4398
4900
|
} finally {
|
|
4399
|
-
|
|
4400
|
-
await
|
|
4401
|
-
|
|
4901
|
+
httpServer?.stop(true);
|
|
4902
|
+
await Promise.allSettled([
|
|
4903
|
+
workerBundle?.connection.close(),
|
|
4904
|
+
signaler?.close(),
|
|
4905
|
+
reaperSchedule?.close(),
|
|
4906
|
+
bus?.close(),
|
|
4907
|
+
dbClient.close()
|
|
4908
|
+
]);
|
|
4402
4909
|
}
|
|
4403
4910
|
}
|
|
4404
4911
|
if (import.meta.main) {
|