@opengeni/worker-bundle 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +593 -86
- package/dist/index.js.map +1 -1
- package/package.json +11 -15
- package/src/activities/agent-turn.ts +95 -12
- package/src/activities/capabilities.ts +2 -30
- package/src/activities/context-compaction.ts +96 -10
- package/src/activities/environment.ts +30 -4
- package/src/activities/sandbox-lease.ts +33 -0
- package/src/activities/session-state.ts +22 -4
- package/src/activities/streaming.ts +1 -0
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +21 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -34,27 +34,23 @@
|
|
|
34
34
|
"dev:watch": "bun --watch src/index.ts",
|
|
35
35
|
"start": "bun src/index.ts",
|
|
36
36
|
"typecheck": "tsc --noEmit",
|
|
37
|
-
"build": "tsup"
|
|
37
|
+
"build": "tsup",
|
|
38
|
+
"prepublishOnly": "bash ../../scripts/prepublish-guard"
|
|
38
39
|
},
|
|
39
40
|
"dependencies": {
|
|
40
41
|
"@opengeni/codex": "^0.2.1",
|
|
41
|
-
"@opengeni/config": "^0.
|
|
42
|
-
"@opengeni/contracts": "^0.
|
|
43
|
-
"@opengeni/
|
|
44
|
-
"@opengeni/
|
|
45
|
-
"@opengeni/
|
|
42
|
+
"@opengeni/config": "^0.3.0",
|
|
43
|
+
"@opengeni/contracts": "^0.9.0",
|
|
44
|
+
"@opengeni/core": "^0.4.3",
|
|
45
|
+
"@opengeni/db": "^0.6.0",
|
|
46
|
+
"@opengeni/documents": "^0.2.7",
|
|
47
|
+
"@opengeni/events": "^0.2.7",
|
|
46
48
|
"@opengeni/observability": "^0.2.1",
|
|
47
|
-
"@opengeni/runtime": "^0.
|
|
48
|
-
"@opengeni/storage": "^0.2.
|
|
49
|
+
"@opengeni/runtime": "^0.4.0",
|
|
50
|
+
"@opengeni/storage": "^0.2.7",
|
|
49
51
|
"@temporalio/activity": "^1.17.0",
|
|
50
52
|
"@temporalio/client": "^1.17.0",
|
|
51
53
|
"@temporalio/worker": "^1.17.0",
|
|
52
54
|
"@temporalio/workflow": "^1.17.0"
|
|
53
|
-
},
|
|
54
|
-
"devDependencies": {
|
|
55
|
-
"@opengeni/testing": "workspace:*",
|
|
56
|
-
"postgres": "^3.4.9",
|
|
57
|
-
"typescript": "^6.0.3",
|
|
58
|
-
"tsup": "^8.5.0"
|
|
59
55
|
}
|
|
60
56
|
}
|
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
heartbeatLeaseHolder,
|
|
39
39
|
accrueWarmSeconds,
|
|
40
40
|
SandboxLeaseSupersededError,
|
|
41
|
+
buildConnectionTokenResolver,
|
|
41
42
|
type AppendEventInput,
|
|
42
43
|
} from "@opengeni/db";
|
|
43
44
|
import { appendAndPublishEvents } from "@opengeni/events";
|
|
@@ -94,6 +95,7 @@ import type {
|
|
|
94
95
|
} from "./types";
|
|
95
96
|
import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
|
|
96
97
|
import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
|
|
98
|
+
import { recordCreditMicros, runtimeMetricsHooksForObservability, turnLifecycleMetricsFor, type TurnOutcome } from "../observability-metrics";
|
|
97
99
|
import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
|
|
98
100
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
99
101
|
import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
|
|
@@ -170,6 +172,13 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
170
172
|
|
|
171
173
|
export const CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE =
|
|
172
174
|
"The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
175
|
+
const MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS = 3;
|
|
176
|
+
|
|
177
|
+
function compactionFailureReason(reason: string): string {
|
|
178
|
+
return reason.startsWith("compaction summarization failed:")
|
|
179
|
+
? reason
|
|
180
|
+
: `compaction summarization failed: ${reason}`;
|
|
181
|
+
}
|
|
173
182
|
|
|
174
183
|
export function classifyContextWindowOverflowError(error: unknown): { message: string; code?: string; detail?: string } | null {
|
|
175
184
|
const fields = collectErrorStrings(error);
|
|
@@ -510,7 +519,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
510
519
|
"opengeni.workflow_id": input.workflowId,
|
|
511
520
|
"opengeni.trigger_event_id": input.triggerEventId,
|
|
512
521
|
});
|
|
513
|
-
let activityStatus = "unknown";
|
|
522
|
+
let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
|
|
523
|
+
let turnMetricOutcome: TurnOutcome | null = null;
|
|
514
524
|
let activityError: unknown;
|
|
515
525
|
let turnId: string | undefined;
|
|
516
526
|
let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
|
|
@@ -680,6 +690,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
680
690
|
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
681
691
|
}
|
|
682
692
|
turnId = turn.id;
|
|
693
|
+
turnLifecycleMetricsFor(observability).start(turnId);
|
|
683
694
|
// Canonical codex-billed predicate (codex/<slug> + feature enabled + active
|
|
684
695
|
// workspace credential). Computed once and threaded through every billing
|
|
685
696
|
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
@@ -836,6 +847,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
836
847
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
837
848
|
], true);
|
|
838
849
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
850
|
+
turnMetricOutcome = "failed";
|
|
839
851
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
840
852
|
activityStatus = "idle";
|
|
841
853
|
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
@@ -1005,14 +1017,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1005
1017
|
// minted, the host `gitCredentials` provider may supply it; unset → self-mint
|
|
1006
1018
|
// from settings. gitToken is undefined on the selfhosted skip path (the machine
|
|
1007
1019
|
// uses its own git creds).
|
|
1008
|
-
const {
|
|
1020
|
+
const {
|
|
1021
|
+
environment: sandboxEnvironment,
|
|
1022
|
+
gitToken: sandboxGitToken,
|
|
1023
|
+
toolspaceToken: sandboxToolspaceToken,
|
|
1024
|
+
} = await sandboxEnvironmentForRun(
|
|
1009
1025
|
runSettings,
|
|
1010
1026
|
turnResources,
|
|
1011
1027
|
workspaceEnvironment?.values ?? {},
|
|
1012
1028
|
{
|
|
1013
1029
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
1030
|
+
skipToolspaceToken: activeSandboxBackend === "selfhosted",
|
|
1014
1031
|
scope: connectionScope,
|
|
1015
1032
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
1033
|
+
sessionId: input.sessionId,
|
|
1034
|
+
runId: turnId,
|
|
1016
1035
|
},
|
|
1017
1036
|
);
|
|
1018
1037
|
|
|
@@ -1100,7 +1119,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1100
1119
|
};
|
|
1101
1120
|
} else {
|
|
1102
1121
|
resolvedSandbox = await resumeBoxForTurn(
|
|
1103
|
-
{ db, settings },
|
|
1122
|
+
{ db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
|
|
1104
1123
|
{
|
|
1105
1124
|
accountId: input.accountId,
|
|
1106
1125
|
workspaceId: input.workspaceId,
|
|
@@ -1192,7 +1211,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1192
1211
|
expectedEpoch: heartbeatEpoch,
|
|
1193
1212
|
warmRateMicrosPerSecond: warmRate,
|
|
1194
1213
|
subjectId: input.sessionId,
|
|
1195
|
-
})
|
|
1214
|
+
})
|
|
1215
|
+
.then((result) => recordCreditMicros(observability, "usage", result.costMicros))
|
|
1216
|
+
.catch(() => undefined);
|
|
1196
1217
|
}, 10_000);
|
|
1197
1218
|
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
1198
1219
|
leaseHeartbeatTimer.unref();
|
|
@@ -1245,12 +1266,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1245
1266
|
// (initialize + tools/list) can resolve the per-workspace bearer from
|
|
1246
1267
|
// codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
|
|
1247
1268
|
// identity on every non-codex turn, so this is a no-op for existing paths.
|
|
1269
|
+
const resolveCredential = buildConnectionTokenResolver(db, runSettings);
|
|
1248
1270
|
preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
|
|
1249
1271
|
accountId: input.accountId,
|
|
1250
1272
|
workspaceId: input.workspaceId,
|
|
1251
1273
|
sessionId: input.sessionId,
|
|
1252
1274
|
subjectId: "worker:first-party-mcp",
|
|
1253
1275
|
subjectLabel: "OpenGeni worker",
|
|
1276
|
+
resolveCredential,
|
|
1277
|
+
onAuthNeeded: async (payload) => {
|
|
1278
|
+
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
1279
|
+
},
|
|
1254
1280
|
// Manager-style sessions carry a creation-validated permission set
|
|
1255
1281
|
// for their first-party MCP token; null keeps the fixed default.
|
|
1256
1282
|
...(session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}),
|
|
@@ -1286,6 +1312,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1286
1312
|
// seeds it to the box's token file before the repository-clone runs; it never
|
|
1287
1313
|
// touches the box/agent manifest env.
|
|
1288
1314
|
...(activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {}),
|
|
1315
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {}),
|
|
1289
1316
|
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
1290
1317
|
fileResourceDownloads,
|
|
1291
1318
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -1350,18 +1377,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1350
1377
|
: {}),
|
|
1351
1378
|
});
|
|
1352
1379
|
const compactSummarizer = resolvedModel
|
|
1353
|
-
? (s: Settings, m: Array<Record<string, unknown
|
|
1380
|
+
? (s: Settings, m: Array<Record<string, unknown>>, o?: { maxTranscriptTokens?: number }) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1354
1381
|
client: resolvedModel.client,
|
|
1355
1382
|
api: resolvedModel.provider.api,
|
|
1356
1383
|
model: resolvedModel.configured.id,
|
|
1357
1384
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
1385
|
+
...(o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}),
|
|
1358
1386
|
}))
|
|
1359
1387
|
: undefined;
|
|
1360
1388
|
// Pre-turn client-side context compaction (Azure path). When the
|
|
1361
1389
|
// resolved mode is "client" and the single Codex-parity threshold is
|
|
1362
1390
|
// crossed, this summarizes the current active history and rebuilds active
|
|
1363
1391
|
// history as [user messages..., summary] BEFORE the model input is read.
|
|
1364
|
-
//
|
|
1392
|
+
// Summarizer failures fall through to hard-trim retry and deterministic
|
|
1393
|
+
// fallback; only structural no-ops leave the history unchanged.
|
|
1365
1394
|
// Skipped on approval/preempt resumes (no fresh user turn; the frozen
|
|
1366
1395
|
// RunState or in-flight tail must replay verbatim).
|
|
1367
1396
|
if (triggerType === "user.message" || triggerType === "goal.continuation") {
|
|
@@ -1457,7 +1486,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1457
1486
|
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
1458
1487
|
session.lastInputTokens,
|
|
1459
1488
|
compactSummarizer,
|
|
1460
|
-
{ force: true },
|
|
1489
|
+
{ force: true, requireShrink: true },
|
|
1461
1490
|
);
|
|
1462
1491
|
if (outcome.compacted) {
|
|
1463
1492
|
await publish!([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
@@ -1534,6 +1563,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1534
1563
|
dispatchId,
|
|
1535
1564
|
positionalKey: `response-${responseUsageCount}`,
|
|
1536
1565
|
}),
|
|
1566
|
+
observability,
|
|
1537
1567
|
});
|
|
1538
1568
|
await reconcileConversationTruth();
|
|
1539
1569
|
try {
|
|
@@ -1588,6 +1618,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1588
1618
|
isCodexTurn,
|
|
1589
1619
|
usage: stream.state.usage,
|
|
1590
1620
|
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1621
|
+
observability,
|
|
1591
1622
|
});
|
|
1592
1623
|
}
|
|
1593
1624
|
if (lastInputTokensObserved !== null) {
|
|
@@ -1640,6 +1671,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1640
1671
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1641
1672
|
], true);
|
|
1642
1673
|
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
1674
|
+
turnMetricOutcome = "completed";
|
|
1643
1675
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1644
1676
|
await recordUsageEvent(db, {
|
|
1645
1677
|
accountId: input.accountId,
|
|
@@ -1656,7 +1688,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1656
1688
|
};
|
|
1657
1689
|
|
|
1658
1690
|
await prepareRunAttemptInput();
|
|
1659
|
-
let
|
|
1691
|
+
let compactionRecoveryAttempts = 0;
|
|
1660
1692
|
let retriedAfterCompaction = false;
|
|
1661
1693
|
while (true) {
|
|
1662
1694
|
try {
|
|
@@ -1672,10 +1704,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1672
1704
|
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1673
1705
|
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1674
1706
|
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
1675
|
-
if (
|
|
1707
|
+
if (recoveryKind && compactionRecoveryAttempts >= MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS) {
|
|
1708
|
+
throw new Error(
|
|
1709
|
+
`compaction summarization failed: context compaction recovery attempt cap reached (${MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS})`,
|
|
1710
|
+
{ cause: attemptError },
|
|
1711
|
+
);
|
|
1712
|
+
}
|
|
1713
|
+
if (!recoveryKind || !publish || !turnStartedPublished) {
|
|
1676
1714
|
throw attemptError;
|
|
1677
1715
|
}
|
|
1678
|
-
|
|
1716
|
+
compactionRecoveryAttempts += 1;
|
|
1679
1717
|
await flushRuntimeBatcher();
|
|
1680
1718
|
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
1681
1719
|
const progressPersisted = modelOrToolProgressPersisted;
|
|
@@ -1690,16 +1728,46 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1690
1728
|
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1691
1729
|
});
|
|
1692
1730
|
let compacted = false;
|
|
1731
|
+
let compactionFailureMessage: string | null = null;
|
|
1693
1732
|
try {
|
|
1694
1733
|
const outcome = await forceContextCompaction(recoveryKind);
|
|
1695
1734
|
compacted = outcome.compacted;
|
|
1735
|
+
if (!outcome.compacted) {
|
|
1736
|
+
compactionFailureMessage = compactionFailureReason(outcome.reason);
|
|
1737
|
+
}
|
|
1696
1738
|
} catch (compactError) {
|
|
1739
|
+
compactionFailureMessage = compactionFailureReason(compactError instanceof Error ? compactError.message : String(compactError));
|
|
1697
1740
|
observability.warn("context compaction recovery compaction failed", {
|
|
1698
1741
|
sessionId: input.sessionId,
|
|
1699
1742
|
turnId: activeTurnId,
|
|
1700
|
-
error:
|
|
1743
|
+
error: compactionFailureMessage,
|
|
1701
1744
|
});
|
|
1702
1745
|
}
|
|
1746
|
+
if (!compacted) {
|
|
1747
|
+
const errorMessage = compactionFailureMessage ?? "compaction summarization failed: compaction produced no replacement history";
|
|
1748
|
+
if (!progressPersisted) {
|
|
1749
|
+
throw new Error(errorMessage, { cause: attemptError });
|
|
1750
|
+
}
|
|
1751
|
+
await publish!([
|
|
1752
|
+
{
|
|
1753
|
+
type: "turn.failed",
|
|
1754
|
+
payload: {
|
|
1755
|
+
error: errorMessage,
|
|
1756
|
+
code: "context_compaction_failed",
|
|
1757
|
+
retryable: false,
|
|
1758
|
+
recovery: "user_message",
|
|
1759
|
+
compacted: false,
|
|
1760
|
+
},
|
|
1761
|
+
},
|
|
1762
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1763
|
+
], true);
|
|
1764
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1765
|
+
turnMetricOutcome = "failed";
|
|
1766
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1767
|
+
activityStatus = "idle";
|
|
1768
|
+
activityError = attemptError;
|
|
1769
|
+
return { status: "idle" };
|
|
1770
|
+
}
|
|
1703
1771
|
if (progressPersisted) {
|
|
1704
1772
|
// The user's intent was live mid-turn — after a successful
|
|
1705
1773
|
// compaction the turn AUTO-CONTINUES via the same synthesized
|
|
@@ -1735,6 +1803,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1735
1803
|
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1736
1804
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1737
1805
|
activityStatus = "preempted";
|
|
1806
|
+
turnMetricOutcome = "preempted";
|
|
1738
1807
|
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
1739
1808
|
sessionId: input.sessionId,
|
|
1740
1809
|
turnId: activeTurnId,
|
|
@@ -1757,6 +1826,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1757
1826
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1758
1827
|
], true);
|
|
1759
1828
|
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1829
|
+
turnMetricOutcome = "failed";
|
|
1760
1830
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1761
1831
|
activityStatus = "idle";
|
|
1762
1832
|
activityError = attemptError;
|
|
@@ -1808,6 +1878,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1808
1878
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
1809
1879
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1810
1880
|
activityStatus = "preempted";
|
|
1881
|
+
turnMetricOutcome = "preempted";
|
|
1811
1882
|
return { status: "preempted" };
|
|
1812
1883
|
} catch (requeueError) {
|
|
1813
1884
|
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
@@ -1868,6 +1939,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1868
1939
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1869
1940
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1870
1941
|
activityStatus = "preempted";
|
|
1942
|
+
turnMetricOutcome = "preempted";
|
|
1871
1943
|
return { status: "preempted" };
|
|
1872
1944
|
} catch (preemptError) {
|
|
1873
1945
|
// A failing checkpoint/requeue must not surface as an arbitrary
|
|
@@ -1885,6 +1957,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1885
1957
|
await flushRuntimeBatcher();
|
|
1886
1958
|
if (preemptTurnId) {
|
|
1887
1959
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1960
|
+
turnMetricOutcome = "cancelled";
|
|
1888
1961
|
}
|
|
1889
1962
|
throw error;
|
|
1890
1963
|
}
|
|
@@ -1920,6 +1993,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1920
1993
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1921
1994
|
], true);
|
|
1922
1995
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1996
|
+
turnMetricOutcome = "completed";
|
|
1923
1997
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1924
1998
|
await recordUsageEvent(db, {
|
|
1925
1999
|
accountId: input.accountId,
|
|
@@ -2044,6 +2118,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2044
2118
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2045
2119
|
], true);
|
|
2046
2120
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2121
|
+
turnMetricOutcome = "failed";
|
|
2047
2122
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2048
2123
|
activityStatus = "idle";
|
|
2049
2124
|
activityError = error;
|
|
@@ -2096,6 +2171,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2096
2171
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2097
2172
|
], true);
|
|
2098
2173
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2174
|
+
turnMetricOutcome = "completed";
|
|
2099
2175
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2100
2176
|
await recordUsageEvent(db, {
|
|
2101
2177
|
accountId: input.accountId,
|
|
@@ -2146,6 +2222,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2146
2222
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
2147
2223
|
], true);
|
|
2148
2224
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2225
|
+
turnMetricOutcome = "failed";
|
|
2149
2226
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2150
2227
|
activityStatus = "idle";
|
|
2151
2228
|
activityError = error;
|
|
@@ -2161,6 +2238,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2161
2238
|
{ type: "session.status.changed", payload: { status: "failed" } },
|
|
2162
2239
|
], true);
|
|
2163
2240
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2241
|
+
turnMetricOutcome = "failed";
|
|
2164
2242
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
2165
2243
|
// The common failure path ends here: runAgentTurn marks the session
|
|
2166
2244
|
// failed and returns "failed", and the session workflow then exits
|
|
@@ -2178,6 +2256,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2178
2256
|
status: activityStatus,
|
|
2179
2257
|
durationSeconds,
|
|
2180
2258
|
});
|
|
2259
|
+
if (turnId && activityStatus !== "unknown") {
|
|
2260
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
2261
|
+
}
|
|
2181
2262
|
activitySpan.end({
|
|
2182
2263
|
attributes: {
|
|
2183
2264
|
"opengeni.turn_id": turnId ?? "",
|
|
@@ -2469,6 +2550,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2469
2550
|
isCodexTurn: boolean;
|
|
2470
2551
|
usage?: ModelUsageInput | null;
|
|
2471
2552
|
sourceKey: string;
|
|
2553
|
+
observability?: ActivityServices["observability"];
|
|
2472
2554
|
}): Promise<void> {
|
|
2473
2555
|
if (!input.usage) {
|
|
2474
2556
|
return;
|
|
@@ -2529,7 +2611,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2529
2611
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
2530
2612
|
});
|
|
2531
2613
|
if (costMicros > 0) {
|
|
2532
|
-
await applyCreditDebitUpToBalance(db, {
|
|
2614
|
+
const result = await applyCreditDebitUpToBalance(db, {
|
|
2533
2615
|
accountId: input.accountId,
|
|
2534
2616
|
workspaceId: input.workspaceId,
|
|
2535
2617
|
type: "model_usage_debit",
|
|
@@ -2547,6 +2629,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
|
|
|
2547
2629
|
totalTokens,
|
|
2548
2630
|
},
|
|
2549
2631
|
});
|
|
2632
|
+
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
2550
2633
|
}
|
|
2551
2634
|
}
|
|
2552
2635
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { environmentsEncryptionKeyBytes, parseModelProvidersJson, type RegistryProvider, type Settings } from "@opengeni/config";
|
|
2
|
+
import { settingsWithMcpCapabilityServers } from "@opengeni/core";
|
|
2
3
|
import {
|
|
3
4
|
CODEX_APPS_MCP_SERVER_ID,
|
|
4
5
|
CODEX_APPS_MCP_SERVER_NAME,
|
|
@@ -10,13 +11,11 @@ import {
|
|
|
10
11
|
CODEX_PROVIDER_ID,
|
|
11
12
|
} from "@opengeni/codex";
|
|
12
13
|
import {
|
|
13
|
-
decryptedCapabilityHeaders,
|
|
14
14
|
listEnabledMcpCapabilityServers,
|
|
15
15
|
listSessionMcpServerMetadata,
|
|
16
16
|
listSessionMcpServersForRun,
|
|
17
17
|
workspaceCodexSubscriptionActive,
|
|
18
18
|
type Database,
|
|
19
|
-
type EnabledMcpCapabilityServer,
|
|
20
19
|
type SessionMcpServerForRun,
|
|
21
20
|
} from "@opengeni/db";
|
|
22
21
|
|
|
@@ -58,6 +57,7 @@ export function settingsWithSessionMcpServers(settings: Settings, servers: Sessi
|
|
|
58
57
|
...(server.allowedTools ? { allowedTools: server.allowedTools } : {}),
|
|
59
58
|
...(server.timeoutMs ? { timeoutMs: server.timeoutMs } : {}),
|
|
60
59
|
cacheToolsList: server.cacheToolsList ?? false,
|
|
60
|
+
...(server.requireApproval !== undefined ? { requireApproval: server.requireApproval } : {}),
|
|
61
61
|
headers: server.headers,
|
|
62
62
|
})),
|
|
63
63
|
],
|
|
@@ -143,31 +143,3 @@ export function withCodexProvider(settings: Settings): Settings {
|
|
|
143
143
|
};
|
|
144
144
|
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
145
145
|
}
|
|
146
|
-
|
|
147
|
-
function settingsWithMcpCapabilityServers(settings: Settings, enabled: EnabledMcpCapabilityServer[]): Settings {
|
|
148
|
-
if (enabled.length === 0) {
|
|
149
|
-
return settings;
|
|
150
|
-
}
|
|
151
|
-
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
152
|
-
const existingIds = new Set(settings.mcpServers.map((server) => server.id));
|
|
153
|
-
const dynamicServers = enabled
|
|
154
|
-
.filter((server) => !existingIds.has(server.id))
|
|
155
|
-
.flatMap((server) => {
|
|
156
|
-
const headers = decryptedCapabilityHeaders(server, encryptionKey);
|
|
157
|
-
if (headers === "unavailable") {
|
|
158
|
-
// Without its credential headers this server can only fail auth at
|
|
159
|
-
// connect time and break agent turns; leave it out of the run.
|
|
160
|
-
return [];
|
|
161
|
-
}
|
|
162
|
-
return [{
|
|
163
|
-
id: server.id,
|
|
164
|
-
name: server.name,
|
|
165
|
-
url: server.url,
|
|
166
|
-
...(server.allowedTools ? { allowedTools: server.allowedTools } : {}),
|
|
167
|
-
...(server.timeoutMs ? { timeoutMs: server.timeoutMs } : {}),
|
|
168
|
-
cacheToolsList: server.cacheToolsList ?? false,
|
|
169
|
-
...(headers ? { headers } : {}),
|
|
170
|
-
}];
|
|
171
|
-
});
|
|
172
|
-
return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
|
|
173
|
-
}
|
|
@@ -7,8 +7,10 @@ import {
|
|
|
7
7
|
import {
|
|
8
8
|
SUMMARY_BUFFER_TOKENS,
|
|
9
9
|
buildCompactionPromptInput,
|
|
10
|
+
buildDeterministicFallbackCompactionHistory,
|
|
10
11
|
buildCompactionReplacementHistory,
|
|
11
12
|
decideClientCompaction,
|
|
13
|
+
estimateTokens,
|
|
12
14
|
sanitizeHistoryItemsForModel,
|
|
13
15
|
summarizeForCompaction,
|
|
14
16
|
type CompactionItem,
|
|
@@ -23,6 +25,9 @@ export type MaybeCompactResult =
|
|
|
23
25
|
summaryPosition: number;
|
|
24
26
|
signalTokens: number;
|
|
25
27
|
thresholdTokens: number;
|
|
28
|
+
method: "summary" | "fallback";
|
|
29
|
+
estimatedTokensBefore: number;
|
|
30
|
+
estimatedTokensAfter: number;
|
|
26
31
|
};
|
|
27
32
|
|
|
28
33
|
/**
|
|
@@ -35,16 +40,23 @@ export type MaybeCompactResult =
|
|
|
35
40
|
* supersedes the old active rows and inserts replacement active rows:
|
|
36
41
|
* all real user messages plus one summary.
|
|
37
42
|
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
43
|
+
* Required compactions are fail-closed on the summarizer: full rendered
|
|
44
|
+
* transcript, hard-trim rendered retry, then deterministic non-LLM fallback.
|
|
45
|
+
* DB errors still throw to the caller; "compacted:false" is reserved for
|
|
46
|
+
* structural no-ops or impossible shrink, not summarizer failures.
|
|
41
47
|
*
|
|
42
48
|
* There is no kept assistant/tool tail in client mode now. Assistant messages,
|
|
43
49
|
* tool calls/results, reasoning, and images stay only in inactive audit rows.
|
|
44
50
|
*/
|
|
51
|
+
export type CompactionSummarizerOptions = {
|
|
52
|
+
maxTranscriptTokens?: number;
|
|
53
|
+
attempt: "full" | "hard_trim";
|
|
54
|
+
};
|
|
55
|
+
|
|
45
56
|
export type CompactionSummarizer = (
|
|
46
57
|
settings: Settings,
|
|
47
58
|
input: CompactionItem[],
|
|
59
|
+
options?: CompactionSummarizerOptions,
|
|
48
60
|
) => Promise<string | null>;
|
|
49
61
|
|
|
50
62
|
export async function maybeCompactContext(
|
|
@@ -53,10 +65,13 @@ export async function maybeCompactContext(
|
|
|
53
65
|
scope: { accountId: string; workspaceId: string; sessionId: string; turnId?: string | null },
|
|
54
66
|
lastInputTokens: number | null,
|
|
55
67
|
// Injectable for tests; defaults to the real provider-aware model call.
|
|
56
|
-
summarize: CompactionSummarizer = (s, m) => summarizeForCompaction(s, m, {
|
|
68
|
+
summarize: CompactionSummarizer = (s, m, o) => summarizeForCompaction(s, m, {
|
|
69
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
70
|
+
...(o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}),
|
|
71
|
+
}),
|
|
57
72
|
// Operator-forced (the /compact command): bypass the budget trigger and
|
|
58
73
|
// compact now if there is anything to summarize. Structural guards still hold.
|
|
59
|
-
options: { force?: boolean } = {},
|
|
74
|
+
options: { force?: boolean; requireShrink?: boolean } = {},
|
|
60
75
|
): Promise<MaybeCompactResult> {
|
|
61
76
|
if (resolveContextCompactionMode(settings) !== "client") {
|
|
62
77
|
return { compacted: false, reason: "mode_not_client" };
|
|
@@ -73,6 +88,7 @@ export async function maybeCompactContext(
|
|
|
73
88
|
lastInputTokens,
|
|
74
89
|
contextWindowTokens: settings.contextWindowTokens,
|
|
75
90
|
contextReservedOutputTokens: settings.contextReservedOutputTokens,
|
|
91
|
+
contextCompactionThresholdRatio: settings.contextCompactionThresholdRatio,
|
|
76
92
|
...(options.force ? { force: true } : {}),
|
|
77
93
|
});
|
|
78
94
|
if (!decision.shouldCompact) {
|
|
@@ -80,15 +96,65 @@ export async function maybeCompactContext(
|
|
|
80
96
|
}
|
|
81
97
|
|
|
82
98
|
const promptInput = buildCompactionPromptInput(items);
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
99
|
+
const estimatedTokensBefore = estimateTokens(items);
|
|
100
|
+
const shrinkBaselineTokens = Math.max(estimatedTokensBefore, decision.signalTokens);
|
|
101
|
+
const requireShrink = options.requireShrink || decision.reason === "above_threshold";
|
|
102
|
+
let summarizerFailure = "summarizer returned no summary";
|
|
103
|
+
let summaryBody: string | null = null;
|
|
104
|
+
|
|
105
|
+
const fullAttempt = await runSummarizerAttempt(summarize, settings, promptInput, { attempt: "full" });
|
|
106
|
+
if (fullAttempt.summary) {
|
|
107
|
+
summaryBody = fullAttempt.summary;
|
|
108
|
+
} else {
|
|
109
|
+
summarizerFailure = fullAttempt.failure;
|
|
110
|
+
const hardTrimBudget = Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5));
|
|
111
|
+
const retryAttempt = await runSummarizerAttempt(summarize, settings, promptInput, {
|
|
112
|
+
attempt: "hard_trim",
|
|
113
|
+
maxTranscriptTokens: hardTrimBudget,
|
|
114
|
+
});
|
|
115
|
+
if (retryAttempt.summary) {
|
|
116
|
+
summaryBody = retryAttempt.summary;
|
|
117
|
+
} else {
|
|
118
|
+
summarizerFailure = retryAttempt.failure || summarizerFailure;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
let replacementHistory: CompactionItem[] | null = null;
|
|
123
|
+
let method: "summary" | "fallback" = "summary";
|
|
124
|
+
if (summaryBody) {
|
|
125
|
+
const summaryReplacement = buildCompactionReplacementHistory(items, summaryBody);
|
|
126
|
+
const summaryEstimate = estimateTokens(summaryReplacement);
|
|
127
|
+
if (!requireShrink || summaryEstimate < shrinkBaselineTokens) {
|
|
128
|
+
replacementHistory = summaryReplacement;
|
|
129
|
+
} else {
|
|
130
|
+
summarizerFailure = `summary replacement did not reduce context signal (${summaryEstimate} >= ${shrinkBaselineTokens})`;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (!replacementHistory) {
|
|
135
|
+
method = "fallback";
|
|
136
|
+
const fallbackTargetTokens = Math.max(
|
|
137
|
+
1,
|
|
138
|
+
Math.min(
|
|
139
|
+
Math.max(1, Math.floor(shrinkBaselineTokens * 0.5)),
|
|
140
|
+
Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5)),
|
|
141
|
+
),
|
|
142
|
+
);
|
|
143
|
+
replacementHistory = buildDeterministicFallbackCompactionHistory({
|
|
144
|
+
items,
|
|
145
|
+
cause: summarizerFailure,
|
|
146
|
+
targetTokens: fallbackTargetTokens,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const estimatedTokensAfter = estimateTokens(replacementHistory);
|
|
151
|
+
if (requireShrink && estimatedTokensAfter >= shrinkBaselineTokens) {
|
|
152
|
+
return { compacted: false, reason: `compaction summarization failed: fallback did not reduce context signal (${estimatedTokensAfter} >= ${shrinkBaselineTokens})` };
|
|
86
153
|
}
|
|
87
154
|
|
|
88
|
-
const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
|
|
89
155
|
const summaryItem = replacementHistory.at(-1);
|
|
90
156
|
if (!summaryItem) {
|
|
91
|
-
return { compacted: false, reason:
|
|
157
|
+
return { compacted: false, reason: `compaction summarization failed: ${summarizerFailure}` };
|
|
92
158
|
}
|
|
93
159
|
const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
|
|
94
160
|
const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
|
|
@@ -114,5 +180,25 @@ export async function maybeCompactContext(
|
|
|
114
180
|
summaryPosition,
|
|
115
181
|
signalTokens: decision.signalTokens,
|
|
116
182
|
thresholdTokens: decision.thresholdTokens,
|
|
183
|
+
method,
|
|
184
|
+
estimatedTokensBefore,
|
|
185
|
+
estimatedTokensAfter,
|
|
117
186
|
};
|
|
118
187
|
}
|
|
188
|
+
|
|
189
|
+
async function runSummarizerAttempt(
|
|
190
|
+
summarize: CompactionSummarizer,
|
|
191
|
+
settings: Settings,
|
|
192
|
+
promptInput: CompactionItem[],
|
|
193
|
+
options: CompactionSummarizerOptions,
|
|
194
|
+
): Promise<{ summary: string | null; failure: string }> {
|
|
195
|
+
try {
|
|
196
|
+
const summary = await summarize(settings, promptInput, options);
|
|
197
|
+
if (summary && summary.trim().length > 0) {
|
|
198
|
+
return { summary, failure: "" };
|
|
199
|
+
}
|
|
200
|
+
return { summary: null, failure: "summarizer returned no summary" };
|
|
201
|
+
} catch (error) {
|
|
202
|
+
return { summary: null, failure: error instanceof Error ? error.message : String(error) };
|
|
203
|
+
}
|
|
204
|
+
}
|