@opengeni/worker-bundle 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +204 -47
- package/dist/index.js.map +1 -1
- package/package.json +11 -15
- package/src/activities/agent-turn.ts +73 -8
- package/src/activities/capabilities.ts +2 -30
- package/src/activities/context-compaction.ts +96 -10
- package/src/activities/environment.ts +36 -4
- package/src/activities/sandbox-lease.ts +12 -3
- package/src/activities/streaming.ts +1 -0
- package/src/observability-metrics.ts +27 -0
package/dist/index.js
CHANGED
|
@@ -55,7 +55,8 @@ import {
|
|
|
55
55
|
sumUsageQuantity as sumUsageQuantity2,
|
|
56
56
|
heartbeatLeaseHolder,
|
|
57
57
|
accrueWarmSeconds,
|
|
58
|
-
SandboxLeaseSupersededError as SandboxLeaseSupersededError2
|
|
58
|
+
SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
|
|
59
|
+
buildConnectionTokenResolver
|
|
59
60
|
} from "@opengeni/db";
|
|
60
61
|
import { appendAndPublishEvents as appendAndPublishEvents2 } from "@opengeni/events";
|
|
61
62
|
import {
|
|
@@ -74,6 +75,7 @@ import { CancelledFailure } from "@temporalio/activity";
|
|
|
74
75
|
|
|
75
76
|
// src/activities/capabilities.ts
|
|
76
77
|
import { environmentsEncryptionKeyBytes, parseModelProvidersJson } from "@opengeni/config";
|
|
78
|
+
import { settingsWithMcpCapabilityServers } from "@opengeni/core";
|
|
77
79
|
import {
|
|
78
80
|
CODEX_APPS_MCP_SERVER_ID,
|
|
79
81
|
CODEX_APPS_MCP_SERVER_NAME,
|
|
@@ -85,7 +87,6 @@ import {
|
|
|
85
87
|
CODEX_PROVIDER_ID
|
|
86
88
|
} from "@opengeni/codex";
|
|
87
89
|
import {
|
|
88
|
-
decryptedCapabilityHeaders,
|
|
89
90
|
listEnabledMcpCapabilityServers,
|
|
90
91
|
listSessionMcpServerMetadata,
|
|
91
92
|
listSessionMcpServersForRun,
|
|
@@ -122,6 +123,7 @@ function settingsWithSessionMcpServers(settings, servers) {
|
|
|
122
123
|
...server.allowedTools ? { allowedTools: server.allowedTools } : {},
|
|
123
124
|
...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
|
|
124
125
|
cacheToolsList: server.cacheToolsList ?? false,
|
|
126
|
+
...server.requireApproval !== void 0 ? { requireApproval: server.requireApproval } : {},
|
|
125
127
|
headers: server.headers
|
|
126
128
|
}))
|
|
127
129
|
]
|
|
@@ -171,29 +173,6 @@ function withCodexProvider(settings) {
|
|
|
171
173
|
};
|
|
172
174
|
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
173
175
|
}
|
|
174
|
-
function settingsWithMcpCapabilityServers(settings, enabled) {
|
|
175
|
-
if (enabled.length === 0) {
|
|
176
|
-
return settings;
|
|
177
|
-
}
|
|
178
|
-
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
179
|
-
const existingIds = new Set(settings.mcpServers.map((server) => server.id));
|
|
180
|
-
const dynamicServers = enabled.filter((server) => !existingIds.has(server.id)).flatMap((server) => {
|
|
181
|
-
const headers = decryptedCapabilityHeaders(server, encryptionKey);
|
|
182
|
-
if (headers === "unavailable") {
|
|
183
|
-
return [];
|
|
184
|
-
}
|
|
185
|
-
return [{
|
|
186
|
-
id: server.id,
|
|
187
|
-
name: server.name,
|
|
188
|
-
url: server.url,
|
|
189
|
-
...server.allowedTools ? { allowedTools: server.allowedTools } : {},
|
|
190
|
-
...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
|
|
191
|
-
cacheToolsList: server.cacheToolsList ?? false,
|
|
192
|
-
...headers ? { headers } : {}
|
|
193
|
-
}];
|
|
194
|
-
});
|
|
195
|
-
return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
|
|
196
|
-
}
|
|
197
176
|
|
|
198
177
|
// src/activities/codex-rotation.ts
|
|
199
178
|
var MIN_IDLE_MS = 6e4;
|
|
@@ -368,13 +347,18 @@ import {
|
|
|
368
347
|
import {
|
|
369
348
|
SUMMARY_BUFFER_TOKENS,
|
|
370
349
|
buildCompactionPromptInput,
|
|
350
|
+
buildDeterministicFallbackCompactionHistory,
|
|
371
351
|
buildCompactionReplacementHistory,
|
|
372
352
|
decideClientCompaction,
|
|
353
|
+
estimateTokens,
|
|
373
354
|
sanitizeHistoryItemsForModel,
|
|
374
355
|
summarizeForCompaction
|
|
375
356
|
} from "@opengeni/runtime";
|
|
376
357
|
import { resolveContextCompactionMode } from "@opengeni/config";
|
|
377
|
-
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, {
|
|
358
|
+
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m, o) => summarizeForCompaction(s, m, {
|
|
359
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
360
|
+
...o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}
|
|
361
|
+
}), options = {}) {
|
|
378
362
|
if (resolveContextCompactionMode(settings) !== "client") {
|
|
379
363
|
return { compacted: false, reason: "mode_not_client" };
|
|
380
364
|
}
|
|
@@ -388,20 +372,67 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
388
372
|
lastInputTokens,
|
|
389
373
|
contextWindowTokens: settings.contextWindowTokens,
|
|
390
374
|
contextReservedOutputTokens: settings.contextReservedOutputTokens,
|
|
375
|
+
contextCompactionThresholdRatio: settings.contextCompactionThresholdRatio,
|
|
391
376
|
...options.force ? { force: true } : {}
|
|
392
377
|
});
|
|
393
378
|
if (!decision.shouldCompact) {
|
|
394
379
|
return { compacted: false, reason: decision.reason };
|
|
395
380
|
}
|
|
396
381
|
const promptInput = buildCompactionPromptInput(items);
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
382
|
+
const estimatedTokensBefore = estimateTokens(items);
|
|
383
|
+
const shrinkBaselineTokens = Math.max(estimatedTokensBefore, decision.signalTokens);
|
|
384
|
+
const requireShrink = options.requireShrink || decision.reason === "above_threshold";
|
|
385
|
+
let summarizerFailure = "summarizer returned no summary";
|
|
386
|
+
let summaryBody = null;
|
|
387
|
+
const fullAttempt = await runSummarizerAttempt(summarize, settings, promptInput, { attempt: "full" });
|
|
388
|
+
if (fullAttempt.summary) {
|
|
389
|
+
summaryBody = fullAttempt.summary;
|
|
390
|
+
} else {
|
|
391
|
+
summarizerFailure = fullAttempt.failure;
|
|
392
|
+
const hardTrimBudget = Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5));
|
|
393
|
+
const retryAttempt = await runSummarizerAttempt(summarize, settings, promptInput, {
|
|
394
|
+
attempt: "hard_trim",
|
|
395
|
+
maxTranscriptTokens: hardTrimBudget
|
|
396
|
+
});
|
|
397
|
+
if (retryAttempt.summary) {
|
|
398
|
+
summaryBody = retryAttempt.summary;
|
|
399
|
+
} else {
|
|
400
|
+
summarizerFailure = retryAttempt.failure || summarizerFailure;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
let replacementHistory = null;
|
|
404
|
+
let method = "summary";
|
|
405
|
+
if (summaryBody) {
|
|
406
|
+
const summaryReplacement = buildCompactionReplacementHistory(items, summaryBody);
|
|
407
|
+
const summaryEstimate = estimateTokens(summaryReplacement);
|
|
408
|
+
if (!requireShrink || summaryEstimate < shrinkBaselineTokens) {
|
|
409
|
+
replacementHistory = summaryReplacement;
|
|
410
|
+
} else {
|
|
411
|
+
summarizerFailure = `summary replacement did not reduce context signal (${summaryEstimate} >= ${shrinkBaselineTokens})`;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
if (!replacementHistory) {
|
|
415
|
+
method = "fallback";
|
|
416
|
+
const fallbackTargetTokens = Math.max(
|
|
417
|
+
1,
|
|
418
|
+
Math.min(
|
|
419
|
+
Math.max(1, Math.floor(shrinkBaselineTokens * 0.5)),
|
|
420
|
+
Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5))
|
|
421
|
+
)
|
|
422
|
+
);
|
|
423
|
+
replacementHistory = buildDeterministicFallbackCompactionHistory({
|
|
424
|
+
items,
|
|
425
|
+
cause: summarizerFailure,
|
|
426
|
+
targetTokens: fallbackTargetTokens
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
const estimatedTokensAfter = estimateTokens(replacementHistory);
|
|
430
|
+
if (requireShrink && estimatedTokensAfter >= shrinkBaselineTokens) {
|
|
431
|
+
return { compacted: false, reason: `compaction summarization failed: fallback did not reduce context signal (${estimatedTokensAfter} >= ${shrinkBaselineTokens})` };
|
|
400
432
|
}
|
|
401
|
-
const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
|
|
402
433
|
const summaryItem = replacementHistory.at(-1);
|
|
403
434
|
if (!summaryItem) {
|
|
404
|
-
return { compacted: false, reason:
|
|
435
|
+
return { compacted: false, reason: `compaction summarization failed: ${summarizerFailure}` };
|
|
405
436
|
}
|
|
406
437
|
const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
|
|
407
438
|
const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
|
|
@@ -424,15 +455,33 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
424
455
|
supersededFrom: nextPosition,
|
|
425
456
|
summaryPosition,
|
|
426
457
|
signalTokens: decision.signalTokens,
|
|
427
|
-
thresholdTokens: decision.thresholdTokens
|
|
458
|
+
thresholdTokens: decision.thresholdTokens,
|
|
459
|
+
method,
|
|
460
|
+
estimatedTokensBefore,
|
|
461
|
+
estimatedTokensAfter
|
|
428
462
|
};
|
|
429
463
|
}
|
|
464
|
+
async function runSummarizerAttempt(summarize, settings, promptInput, options) {
|
|
465
|
+
try {
|
|
466
|
+
const summary = await summarize(settings, promptInput, options);
|
|
467
|
+
if (summary && summary.trim().length > 0) {
|
|
468
|
+
return { summary, failure: "" };
|
|
469
|
+
}
|
|
470
|
+
return { summary: null, failure: "summarizer returned no summary" };
|
|
471
|
+
} catch (error) {
|
|
472
|
+
return { summary: null, failure: error instanceof Error ? error.message : String(error) };
|
|
473
|
+
}
|
|
474
|
+
}
|
|
430
475
|
|
|
431
476
|
// src/activities/environment.ts
|
|
432
477
|
import {
|
|
433
478
|
applyGitAuthPointerEnvironment,
|
|
479
|
+
firstPartyMcpWorkspaceUrl,
|
|
434
480
|
stableSandboxEnvironmentForRun
|
|
435
481
|
} from "@opengeni/config";
|
|
482
|
+
import {
|
|
483
|
+
signDelegatedAccessToken
|
|
484
|
+
} from "@opengeni/contracts";
|
|
436
485
|
import {
|
|
437
486
|
loadWorkspaceEnvironmentForRun as loadWorkspaceEnvironmentForRunFromDb
|
|
438
487
|
} from "@opengeni/db";
|
|
@@ -468,10 +517,24 @@ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
|
|
|
468
517
|
}
|
|
469
518
|
}
|
|
470
519
|
async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
|
|
471
|
-
const
|
|
520
|
+
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
521
|
+
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
522
|
+
let toolspaceToken;
|
|
523
|
+
if (settings.toolspaceEnabled && settings.delegationSecret && options.scope && options.sessionId && options.runId) {
|
|
524
|
+
toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
|
|
525
|
+
accountId: options.scope.accountId,
|
|
526
|
+
workspaceId: options.scope.workspaceId,
|
|
527
|
+
subjectId: `sandbox:${options.runId}`,
|
|
528
|
+
subjectLabel: "sandbox toolspace",
|
|
529
|
+
permissions: ["toolspace:call"],
|
|
530
|
+
sessionId: options.sessionId,
|
|
531
|
+
exp: Math.floor(Date.now() / 1e3) + 60 * 60
|
|
532
|
+
});
|
|
533
|
+
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(settings, options.scope.workspaceId);
|
|
534
|
+
}
|
|
472
535
|
const selection = githubRepositorySelection(resources);
|
|
473
536
|
if (!selection || options.skipGitHubToken) {
|
|
474
|
-
return { environment };
|
|
537
|
+
return { environment, ...toolspaceToken ? { toolspaceToken } : {} };
|
|
475
538
|
}
|
|
476
539
|
let token;
|
|
477
540
|
let identity;
|
|
@@ -493,7 +556,7 @@ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironmen
|
|
|
493
556
|
identity = githubAppBotIdentity(settings);
|
|
494
557
|
}
|
|
495
558
|
applyGitAuthPointerEnvironment(environment, identity);
|
|
496
|
-
return { environment, gitToken: token };
|
|
559
|
+
return { environment, gitToken: token, ...toolspaceToken ? { toolspaceToken } : {} };
|
|
497
560
|
}
|
|
498
561
|
function githubRepositorySelection(resources) {
|
|
499
562
|
const selected = resources.flatMap((resource) => {
|
|
@@ -1111,6 +1174,7 @@ function createRuntimeBatcher(flushEvents) {
|
|
|
1111
1174
|
"agent.toolCall.created",
|
|
1112
1175
|
"agent.toolCall.output",
|
|
1113
1176
|
"agent.message.completed",
|
|
1177
|
+
"tool.auth_needed",
|
|
1114
1178
|
"session.requiresAction",
|
|
1115
1179
|
"turn.completed",
|
|
1116
1180
|
"turn.failed",
|
|
@@ -1646,6 +1710,7 @@ function routingEnabled(settings) {
|
|
|
1646
1710
|
|
|
1647
1711
|
// src/observability-metrics.ts
|
|
1648
1712
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
1713
|
+
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
1649
1714
|
function observabilityEventLogger(observability) {
|
|
1650
1715
|
return {
|
|
1651
1716
|
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
@@ -1817,6 +1882,30 @@ function recordSandboxLeaseGauges(observability, counts) {
|
|
|
1817
1882
|
});
|
|
1818
1883
|
}
|
|
1819
1884
|
}
|
|
1885
|
+
function recordCreditBalanceGauges(observability, balances) {
|
|
1886
|
+
const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
|
|
1887
|
+
const current = /* @__PURE__ */ new Set();
|
|
1888
|
+
for (const balance of balances) {
|
|
1889
|
+
current.add(balance.accountId);
|
|
1890
|
+
observability.setGauge({
|
|
1891
|
+
name: "opengeni_credit_balance_micros",
|
|
1892
|
+
help: "Current credit balance in micros by account.",
|
|
1893
|
+
labels: { account_id: balance.accountId },
|
|
1894
|
+
value: balance.balanceMicros
|
|
1895
|
+
});
|
|
1896
|
+
}
|
|
1897
|
+
for (const accountId of previous) {
|
|
1898
|
+
if (!current.has(accountId)) {
|
|
1899
|
+
observability.setGauge({
|
|
1900
|
+
name: "opengeni_credit_balance_micros",
|
|
1901
|
+
help: "Current credit balance in micros by account.",
|
|
1902
|
+
labels: { account_id: accountId },
|
|
1903
|
+
value: 0
|
|
1904
|
+
});
|
|
1905
|
+
}
|
|
1906
|
+
}
|
|
1907
|
+
creditBalanceGaugeAccounts.set(observability, current);
|
|
1908
|
+
}
|
|
1820
1909
|
function recordSandboxOrphansTerminated(observability, count) {
|
|
1821
1910
|
if (count <= 0) {
|
|
1822
1911
|
return;
|
|
@@ -2006,6 +2095,10 @@ function isWorkerShutdownCancellation(error) {
|
|
|
2006
2095
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
2007
2096
|
}
|
|
2008
2097
|
var CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE = "The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
2098
|
+
var MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS = 3;
|
|
2099
|
+
function compactionFailureReason(reason) {
|
|
2100
|
+
return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
|
|
2101
|
+
}
|
|
2009
2102
|
function classifyContextWindowOverflowError(error) {
|
|
2010
2103
|
const fields = collectErrorStrings(error);
|
|
2011
2104
|
const matched = fields.find(
|
|
@@ -2414,14 +2507,20 @@ function createRunAgentTurnActivity(services) {
|
|
|
2414
2507
|
async () => activeSandboxRecord?.kind ?? null
|
|
2415
2508
|
);
|
|
2416
2509
|
const machinePrimary = activeSandboxBackend === "selfhosted" && Boolean(activeSandboxPointer?.activeSandboxId) && Boolean(activeSandboxRecord?.enrollmentId);
|
|
2417
|
-
const {
|
|
2510
|
+
const {
|
|
2511
|
+
environment: sandboxEnvironment,
|
|
2512
|
+
gitToken: sandboxGitToken,
|
|
2513
|
+
toolspaceToken: sandboxToolspaceToken
|
|
2514
|
+
} = await sandboxEnvironmentForRun(
|
|
2418
2515
|
runSettings,
|
|
2419
2516
|
turnResources,
|
|
2420
2517
|
workspaceEnvironment?.values ?? {},
|
|
2421
2518
|
{
|
|
2422
2519
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
2423
2520
|
scope: connectionScope,
|
|
2424
|
-
gitCredentials: connectionCredentials?.gitCredentials
|
|
2521
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
2522
|
+
sessionId: input.sessionId,
|
|
2523
|
+
runId: turnId
|
|
2425
2524
|
}
|
|
2426
2525
|
);
|
|
2427
2526
|
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
@@ -2575,12 +2674,17 @@ function createRunAgentTurnActivity(services) {
|
|
|
2575
2674
|
}
|
|
2576
2675
|
const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
|
|
2577
2676
|
throwIfWorkerShuttingDown();
|
|
2677
|
+
const resolveCredential = buildConnectionTokenResolver(db, runSettings);
|
|
2578
2678
|
preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
|
|
2579
2679
|
accountId: input.accountId,
|
|
2580
2680
|
workspaceId: input.workspaceId,
|
|
2581
2681
|
sessionId: input.sessionId,
|
|
2582
2682
|
subjectId: "worker:first-party-mcp",
|
|
2583
2683
|
subjectLabel: "OpenGeni worker",
|
|
2684
|
+
resolveCredential,
|
|
2685
|
+
onAuthNeeded: async (payload) => {
|
|
2686
|
+
await publish([{ type: "tool.auth_needed", payload }], true);
|
|
2687
|
+
},
|
|
2584
2688
|
// Manager-style sessions carry a creation-validated permission set
|
|
2585
2689
|
// for their first-party MCP token; null keeps the fixed default.
|
|
2586
2690
|
...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}
|
|
@@ -2597,6 +2701,14 @@ function createRunAgentTurnActivity(services) {
|
|
|
2597
2701
|
// seeds it to the box's token file before the repository-clone runs; it never
|
|
2598
2702
|
// touches the box/agent manifest env.
|
|
2599
2703
|
...activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
2704
|
+
// Toolspace is delivered on EVERY backend including selfhosted. The git-
|
|
2705
|
+
// token skip does NOT transfer: that token is inert on a connected
|
|
2706
|
+
// machine (it uses its own git creds), but the toolspace token is the
|
|
2707
|
+
// machine's ONLY path to programmatic tool calling and grants no more
|
|
2708
|
+
// than toolspace:call for its own session (own-session-bound, turn TTL,
|
|
2709
|
+
// budgeted, approval-tools excluded). The runtime seeds it to the box's
|
|
2710
|
+
// token file over the same exec channel, off-manifest, on every backend.
|
|
2711
|
+
...sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {},
|
|
2600
2712
|
...activeSandboxBackend ? { activeSandboxBackend } : {},
|
|
2601
2713
|
fileResourceDownloads,
|
|
2602
2714
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -2652,11 +2764,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
2652
2764
|
}
|
|
2653
2765
|
} : {}
|
|
2654
2766
|
});
|
|
2655
|
-
const compactSummarizer = resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
|
|
2767
|
+
const compactSummarizer = resolvedModel ? (s, m, o) => withCodex(() => summarizeForCompaction2(s, m, {
|
|
2656
2768
|
client: resolvedModel.client,
|
|
2657
2769
|
api: resolvedModel.provider.api,
|
|
2658
2770
|
model: resolvedModel.configured.id,
|
|
2659
|
-
maxOutputTokens: SUMMARY_BUFFER_TOKENS2
|
|
2771
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
|
|
2772
|
+
...o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}
|
|
2660
2773
|
})) : void 0;
|
|
2661
2774
|
if (triggerType === "user.message" || triggerType === "goal.continuation") {
|
|
2662
2775
|
try {
|
|
@@ -2713,7 +2826,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2713
2826
|
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
2714
2827
|
session.lastInputTokens,
|
|
2715
2828
|
compactSummarizer,
|
|
2716
|
-
{ force: true }
|
|
2829
|
+
{ force: true, requireShrink: true }
|
|
2717
2830
|
);
|
|
2718
2831
|
if (outcome.compacted) {
|
|
2719
2832
|
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
@@ -2894,7 +3007,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2894
3007
|
return { status: "idle" };
|
|
2895
3008
|
};
|
|
2896
3009
|
await prepareRunAttemptInput();
|
|
2897
|
-
let
|
|
3010
|
+
let compactionRecoveryAttempts = 0;
|
|
2898
3011
|
let retriedAfterCompaction = false;
|
|
2899
3012
|
while (true) {
|
|
2900
3013
|
try {
|
|
@@ -2910,10 +3023,16 @@ function createRunAgentTurnActivity(services) {
|
|
|
2910
3023
|
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
2911
3024
|
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
2912
3025
|
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
2913
|
-
if (
|
|
3026
|
+
if (recoveryKind && compactionRecoveryAttempts >= MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS) {
|
|
3027
|
+
throw new Error(
|
|
3028
|
+
`compaction summarization failed: context compaction recovery attempt cap reached (${MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS})`,
|
|
3029
|
+
{ cause: attemptError }
|
|
3030
|
+
);
|
|
3031
|
+
}
|
|
3032
|
+
if (!recoveryKind || !publish || !turnStartedPublished) {
|
|
2914
3033
|
throw attemptError;
|
|
2915
3034
|
}
|
|
2916
|
-
|
|
3035
|
+
compactionRecoveryAttempts += 1;
|
|
2917
3036
|
await flushRuntimeBatcher();
|
|
2918
3037
|
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
2919
3038
|
const progressPersisted = modelOrToolProgressPersisted;
|
|
@@ -2928,16 +3047,46 @@ function createRunAgentTurnActivity(services) {
|
|
|
2928
3047
|
thresholdTokens: compactionNeeded?.thresholdTokens
|
|
2929
3048
|
});
|
|
2930
3049
|
let compacted = false;
|
|
3050
|
+
let compactionFailureMessage = null;
|
|
2931
3051
|
try {
|
|
2932
3052
|
const outcome = await forceContextCompaction(recoveryKind);
|
|
2933
3053
|
compacted = outcome.compacted;
|
|
3054
|
+
if (!outcome.compacted) {
|
|
3055
|
+
compactionFailureMessage = compactionFailureReason(outcome.reason);
|
|
3056
|
+
}
|
|
2934
3057
|
} catch (compactError) {
|
|
3058
|
+
compactionFailureMessage = compactionFailureReason(compactError instanceof Error ? compactError.message : String(compactError));
|
|
2935
3059
|
observability.warn("context compaction recovery compaction failed", {
|
|
2936
3060
|
sessionId: input.sessionId,
|
|
2937
3061
|
turnId: activeTurnId,
|
|
2938
|
-
error:
|
|
3062
|
+
error: compactionFailureMessage
|
|
2939
3063
|
});
|
|
2940
3064
|
}
|
|
3065
|
+
if (!compacted) {
|
|
3066
|
+
const errorMessage = compactionFailureMessage ?? "compaction summarization failed: compaction produced no replacement history";
|
|
3067
|
+
if (!progressPersisted) {
|
|
3068
|
+
throw new Error(errorMessage, { cause: attemptError });
|
|
3069
|
+
}
|
|
3070
|
+
await publish([
|
|
3071
|
+
{
|
|
3072
|
+
type: "turn.failed",
|
|
3073
|
+
payload: {
|
|
3074
|
+
error: errorMessage,
|
|
3075
|
+
code: "context_compaction_failed",
|
|
3076
|
+
retryable: false,
|
|
3077
|
+
recovery: "user_message",
|
|
3078
|
+
compacted: false
|
|
3079
|
+
}
|
|
3080
|
+
},
|
|
3081
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
3082
|
+
], true);
|
|
3083
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
3084
|
+
turnMetricOutcome = "failed";
|
|
3085
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
3086
|
+
activityStatus = "idle";
|
|
3087
|
+
activityError = attemptError;
|
|
3088
|
+
return { status: "idle" };
|
|
3089
|
+
}
|
|
2941
3090
|
if (progressPersisted) {
|
|
2942
3091
|
const triggerPayload = trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload) ? trigger.payload : {};
|
|
2943
3092
|
const compactionResumedTurn = triggerType === "turn.preempted" && (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
|
|
@@ -3653,6 +3802,7 @@ import {
|
|
|
3653
3802
|
countSandboxLeasesByLiveness,
|
|
3654
3803
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3655
3804
|
getBillingBalance as getBillingBalance4,
|
|
3805
|
+
listCreditBalancesByAccount,
|
|
3656
3806
|
listLiveModalSandboxLeaseAttributions,
|
|
3657
3807
|
listMeterableWarmLeases,
|
|
3658
3808
|
persistDrainSnapshot,
|
|
@@ -3677,7 +3827,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3677
3827
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3678
3828
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3679
3829
|
}
|
|
3680
|
-
await
|
|
3830
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
3681
3831
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3682
3832
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
3683
3833
|
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
@@ -3713,7 +3863,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3713
3863
|
error: error instanceof Error ? error.message : String(error)
|
|
3714
3864
|
});
|
|
3715
3865
|
}
|
|
3716
|
-
await
|
|
3866
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
3717
3867
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3718
3868
|
observability.info("sandbox reaper swept", {
|
|
3719
3869
|
drainable: drainable.length,
|
|
@@ -3766,7 +3916,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3766
3916
|
}
|
|
3767
3917
|
return { accrued, workspaceIds };
|
|
3768
3918
|
}
|
|
3769
|
-
async function
|
|
3919
|
+
async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
3770
3920
|
try {
|
|
3771
3921
|
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
3772
3922
|
} catch (error) {
|
|
@@ -3781,6 +3931,13 @@ async function refreshQueueAndLeaseGauges(db, observability) {
|
|
|
3781
3931
|
error: error instanceof Error ? error.message : String(error)
|
|
3782
3932
|
});
|
|
3783
3933
|
}
|
|
3934
|
+
try {
|
|
3935
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
3936
|
+
} catch (error) {
|
|
3937
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
3938
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3939
|
+
});
|
|
3940
|
+
}
|
|
3784
3941
|
}
|
|
3785
3942
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3786
3943
|
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|