claude-code-infinite 1.0.17 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/dist/cli-main.js +6 -1
- package/dist/cli-main.js.map +1 -1
- package/dist/hooks.js +1 -1
- package/dist/hooks.js.map +1 -1
- package/dist/memtree.d.ts +47 -0
- package/dist/memtree.d.ts.map +1 -1
- package/dist/memtree.js +101 -11
- package/dist/memtree.js.map +1 -1
- package/dist/prefix-store.d.ts +59 -0
- package/dist/prefix-store.d.ts.map +1 -0
- package/dist/prefix-store.js +330 -0
- package/dist/prefix-store.js.map +1 -0
- package/dist/proxy.d.ts +7 -0
- package/dist/proxy.d.ts.map +1 -1
- package/dist/proxy.js +198 -29
- package/dist/proxy.js.map +1 -1
- package/dist/reqlog.d.ts +8 -0
- package/dist/reqlog.d.ts.map +1 -1
- package/dist/reqlog.js +11 -0
- package/dist/reqlog.js.map +1 -1
- package/dist/usage-link.d.ts +16 -0
- package/dist/usage-link.d.ts.map +1 -0
- package/dist/usage-link.js +33 -0
- package/dist/usage-link.js.map +1 -0
- package/package.json +1 -1
package/dist/proxy.js
CHANGED
|
@@ -54,6 +54,7 @@ import { cachedPromptTokenCount, checkCompressedHistory, didMemtreeCompress, ind
|
|
|
54
54
|
import { contextLimitForModel, hasEarlierNonToolUserMessage, isAwaySummaryUserMessage, isLocalBashCommandTurn, isNonToolUserMessage, isToolResultUserMessage, lastNonSystemMessage, messagesWithSystem, modelForMemtree, stripSystemReminderText, } from "./turns.js";
|
|
55
55
|
import { COMPRESSED_NOTICE, compressedTotalsText, DEGRADED_NOTICE, NOT_COMPRESSED_NOTE, recapLinkText, SseNoticeRewriter, stripNoticeBlocks, stripNoticeSystem, } from "./notices.js";
|
|
56
56
|
import { NoticeDeliveryQueue, MEMTREE_COMPACT_COMMAND, MEMTREE_HELP_COMMAND, TRAILER_LABEL, LINK_LABEL, linkLines, urlLine, terminalSupportsHyperlinks, isMemtreeViewCommand, sessionCommandArgs, parseNoticeHookInput, } from "./hooks.js";
|
|
57
|
+
import { USAGE_LABEL, USAGE_SIGN_IN_NOTE } from "./usage-link.js";
|
|
57
58
|
import { hyperlink } from "./payment-gate.js";
|
|
58
59
|
import { PaymentNoticeThrottle, paymentNoticeText } from "./payment-notice.js";
|
|
59
60
|
import { MEMTREE_LINKS_MAX_SESSIONS } from "./memtree-links.js";
|
|
@@ -70,6 +71,8 @@ const HOOK_BODY_LIMIT = 64 * 1024;
|
|
|
70
71
|
// Long enough that a real outage costs one stall, short enough that a
|
|
71
72
|
// transient blip does not disable recovery for a working session.
|
|
72
73
|
const TOOL_RECOVERY_FAILURE_COOLDOWN_MS = 60_000;
|
|
74
|
+
/** Safety cap on a long-budget compress, should a transport ignore its abort. */
|
|
75
|
+
const MAX_LONG_COMPRESS_WAIT_MS = 10 * 60_000;
|
|
73
76
|
const SKIP_REQUEST_HEADERS = new Set([
|
|
74
77
|
"host",
|
|
75
78
|
"connection",
|
|
@@ -263,6 +266,7 @@ export function startProxy(opts) {
|
|
|
263
266
|
const state = {
|
|
264
267
|
paymentNotice: new PaymentNoticeThrottle(opts.paymentNoticeIntervalMs),
|
|
265
268
|
unpaidDetail: () => opts.memtree.paymentRequiredDetail,
|
|
269
|
+
usageUrl: opts.memtree.usageDashboardUrl,
|
|
266
270
|
notices: new NoticeDeliveryQueue(),
|
|
267
271
|
mainNoticePending: false,
|
|
268
272
|
mainPromptGeneration: 0,
|
|
@@ -282,6 +286,10 @@ export function startProxy(opts) {
|
|
|
282
286
|
memorySourcePages: new Map(),
|
|
283
287
|
memtreeCallSeq: 0,
|
|
284
288
|
memtreeLinkStore: opts.memtreeLinkStore,
|
|
289
|
+
stablePrefixStore: opts.stablePrefixStore,
|
|
290
|
+
storedPrefixLookups: new Map(),
|
|
291
|
+
storageOwner: typeof opts.memtree.storageIdentity === "string"
|
|
292
|
+
? opts.memtree.storageIdentity : undefined,
|
|
285
293
|
newestTrees: opts.newestTrees,
|
|
286
294
|
memtreeLinkPlacement: opts.memtreeLinkPlacement ?? "turn",
|
|
287
295
|
compactTargets: new Map(),
|
|
@@ -945,7 +953,9 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
945
953
|
},
|
|
946
954
|
}, state.countTokensCooldowns);
|
|
947
955
|
const requestSizing = new RequestSizing(countSession, opts.countTokens === true);
|
|
948
|
-
|
|
956
|
+
if (carriesServerTools(body))
|
|
957
|
+
serverToolRecords.add(rec);
|
|
958
|
+
const sizeSlot = wholeRequestSizeSlot(state, isMainRequest, requestSessionKey, requestSizeKey, body.model);
|
|
949
959
|
const originalCandidate = requestSizing.register(rawBody, "original", sizeSlot.get());
|
|
950
960
|
const forwardCandidate = requestSizing.register(forwardBody, "original", sizeSlot.get());
|
|
951
961
|
candidateSizeOwners.set(originalCandidate, sizeSlot);
|
|
@@ -956,6 +966,9 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
956
966
|
output: bodyOutputReservation(body),
|
|
957
967
|
recountOutput: outputReservation(undefined, body.max_tokens), // full max_tokens
|
|
958
968
|
});
|
|
969
|
+
if (isMainRequest && requestSessionKey !== undefined) {
|
|
970
|
+
adoptStoredPrefix(opts, state, rec, requestSessionKey, body, messages, modelContextLimit);
|
|
971
|
+
}
|
|
959
972
|
const rides = state.memoryRoutes.has(requestRouteKey) ||
|
|
960
973
|
(isMainRequest && requestSessionKey !== undefined && state.stablePrefixes.has(requestSessionKey));
|
|
961
974
|
if (!rides) {
|
|
@@ -1020,12 +1033,14 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
1020
1033
|
// selection considers it together with the original and any replacement.
|
|
1021
1034
|
return candidate.kind === "ride" || fitsMeasured(measured, sizingLimits());
|
|
1022
1035
|
},
|
|
1023
|
-
|
|
1036
|
+
// The ride's bytes are only what failed; recovery always compresses the
|
|
1037
|
+
// ORIGINAL conversation (the one MemTree indexed), never the ride itself.
|
|
1038
|
+
recoverLate: async (_ride, rideTokens) => {
|
|
1024
1039
|
if (rec.compress || compactionMode(opts, state, requestSessionKey).mode === "off" ||
|
|
1025
1040
|
(rec.turnType.startsWith("tool") && opts.toolRouteRecovery === false))
|
|
1026
1041
|
return undefined;
|
|
1027
|
-
return recoverLateCount({ opts, state, req, rec,
|
|
1028
|
-
modelContextLimit, clientMeta, sizing: requestSizing });
|
|
1042
|
+
return recoverLateCount({ opts, state, req, rec, body, msgsForMemtree, hash, rideTokens,
|
|
1043
|
+
modelContextLimit, clientMeta, sizing: requestSizing, inputCandidate: forwardCandidate });
|
|
1029
1044
|
},
|
|
1030
1045
|
delivered: () => { state.fallbackFailures.succeeded(requestRouteKey); toolForwarding.settle(true); },
|
|
1031
1046
|
});
|
|
@@ -1263,6 +1278,7 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
1263
1278
|
...(plan.fallback ? { fallback: plan.fallback } : {}),
|
|
1264
1279
|
retryAtTokens: plan.estimateTokens +
|
|
1265
1280
|
Math.floor(plan.budgetTokens * TOOL_COMPACTION_RETRY_BUDGET_RATIO),
|
|
1281
|
+
...(plan.fallback ? {} : { wholeRefused: () => wholeRequestRefused(requestSizing, originalCandidate, sizingLimits()) }),
|
|
1266
1282
|
}).finally(() => {
|
|
1267
1283
|
recoveryAttempt.inFlight = false;
|
|
1268
1284
|
}));
|
|
@@ -1289,7 +1305,7 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
1289
1305
|
if (toolRide)
|
|
1290
1306
|
noteRideSize(toolRide.sizeHolder, rec, sendBody.length);
|
|
1291
1307
|
else {
|
|
1292
|
-
noteWholeRequestSize(state, isMainRequest, toolSessionId, requestSizeKey, rec, sendBody.length);
|
|
1308
|
+
noteWholeRequestSize(state, isMainRequest, toolSessionId, requestSizeKey, body.model, rec, sendBody.length);
|
|
1293
1309
|
}
|
|
1294
1310
|
}));
|
|
1295
1311
|
}
|
|
@@ -1528,7 +1544,7 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
1528
1544
|
const sendingWhole = () => { requestSendPolicies.get(req).failedRebuild = true; };
|
|
1529
1545
|
const noteWholeSize = () => {
|
|
1530
1546
|
if (isMainRequest)
|
|
1531
|
-
notePassthroughSize(state, sessionId, rec, forwardBody.length);
|
|
1547
|
+
notePassthroughSize(state, sessionId, body.model, rec, forwardBody.length);
|
|
1532
1548
|
};
|
|
1533
1549
|
// The complete request upload can outlive its downstream subscriber while
|
|
1534
1550
|
// MemTree compression is in flight. Track that subscriber locally without
|
|
@@ -1563,6 +1579,10 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
1563
1579
|
: edge?.kind === "off"
|
|
1564
1580
|
? { compaction: {} }
|
|
1565
1581
|
: {}),
|
|
1582
|
+
// Riding the old prefix, or sending a raw request that fits, keeps the
|
|
1583
|
+
// short budget; with neither, a timeout could only end in a refusal.
|
|
1584
|
+
longBudget: !(edge?.kind === "compress" && edge.fallback) &&
|
|
1585
|
+
await wholeRequestRefused(requestSizing, forwardCandidate, sizingLimits()),
|
|
1566
1586
|
});
|
|
1567
1587
|
}
|
|
1568
1588
|
catch {
|
|
@@ -1575,7 +1595,7 @@ async function handleMessages(req, res, opts, upstream, state) {
|
|
|
1575
1595
|
return logged(forwardRaw(req, res, rawBody, opts, upstream, state.shutdownSignal, rec)
|
|
1576
1596
|
.then((delivered) => {
|
|
1577
1597
|
if (isMainRequest)
|
|
1578
|
-
notePassthroughSize(state, sessionId, rec, rawBody.length);
|
|
1598
|
+
notePassthroughSize(state, sessionId, body.model, rec, rawBody.length);
|
|
1579
1599
|
if (delivered)
|
|
1580
1600
|
markMainPromptDelivered();
|
|
1581
1601
|
}));
|
|
@@ -1867,10 +1887,12 @@ function sessionCommandReply(state, sessionId, prompt) {
|
|
|
1867
1887
|
return [
|
|
1868
1888
|
"• /memtree-view · show this session's MemTree page link",
|
|
1869
1889
|
`• /memtree-compact [tokens | off] · compact this session on your next message (default half the budget, at least ${MEMTREE_COMPACT_MIN_TOKENS / 1000}k)`,
|
|
1870
|
-
|
|
1890
|
+
usageLines(state),
|
|
1891
|
+
].filter((line) => line !== undefined).join("\n");
|
|
1892
|
+
}
|
|
1893
|
+
if (isMemtreeViewCommand(prompt)) {
|
|
1894
|
+
return joinNoticeLines(memtreeViewLine(state, sessionId), usageLines(state)) ?? "";
|
|
1871
1895
|
}
|
|
1872
|
-
if (isMemtreeViewCommand(prompt))
|
|
1873
|
-
return memtreeViewLine(state, sessionId);
|
|
1874
1896
|
const args = sessionCommandArgs(prompt, MEMTREE_COMPACT_COMMAND);
|
|
1875
1897
|
if (args === undefined)
|
|
1876
1898
|
return undefined;
|
|
@@ -1880,6 +1902,9 @@ function sessionCommandReply(state, sessionId, prompt) {
|
|
|
1880
1902
|
state.memoryRoutes.delete(JSON.stringify([sessionId, "main"]));
|
|
1881
1903
|
state.compactNow.delete(sessionId);
|
|
1882
1904
|
state.stablePrefixes.delete(sessionId);
|
|
1905
|
+
if (state.storageOwner)
|
|
1906
|
+
state.stablePrefixStore?.delete(sessionId, state.storageOwner);
|
|
1907
|
+
state.storedPrefixLookups.set(sessionId, null);
|
|
1883
1908
|
return `${TRAILER_LABEL} compaction off: the conversation is sent whole, and MemTree compresses only when it outgrows the model's budget.`;
|
|
1884
1909
|
}
|
|
1885
1910
|
if (args === "") {
|
|
@@ -1910,7 +1935,8 @@ export const SUBAGENT_COMPACT_REPLY = "MemTree compact applies to the main sessi
|
|
|
1910
1935
|
*/
|
|
1911
1936
|
function subagentCommandReply(state, sessionId, prompt) {
|
|
1912
1937
|
if (isMemtreeViewCommand(prompt)) {
|
|
1913
|
-
|
|
1938
|
+
const view = `${memtreeViewLine(state, sessionId)}\n${SUBAGENT_VIEW_NOTE}`;
|
|
1939
|
+
return joinNoticeLines(view, usageLines(state)) ?? view;
|
|
1914
1940
|
}
|
|
1915
1941
|
if (sessionCommandArgs(prompt, MEMTREE_COMPACT_COMMAND) !== undefined) {
|
|
1916
1942
|
return `${TRAILER_LABEL} ${SUBAGENT_COMPACT_REPLY}`;
|
|
@@ -1943,6 +1969,15 @@ function memtreeViewLine(state, sessionId) {
|
|
|
1943
1969
|
? `${lines}\n${urlLine(viewUrl(page.url), "memory from ")}`
|
|
1944
1970
|
: lines;
|
|
1945
1971
|
}
|
|
1972
|
+
/**
|
|
1973
|
+
* The usage dashboard link that `/memtree` and `/memtree-view` end with,
|
|
1974
|
+
* formatted like the MemTree link; undefined when the environment has none.
|
|
1975
|
+
*/
|
|
1976
|
+
function usageLines(state) {
|
|
1977
|
+
return state.usageUrl === undefined
|
|
1978
|
+
? undefined
|
|
1979
|
+
: linkLines(USAGE_LABEL, viewUrl(state.usageUrl), USAGE_SIGN_IN_NOTE);
|
|
1980
|
+
}
|
|
1946
1981
|
/** A `/memtree-view` URL: an explicit OSC 8 link where the terminal has them. */
|
|
1947
1982
|
function viewUrl(url) {
|
|
1948
1983
|
return terminalSupportsHyperlinks() ? hyperlink(url, url) : url;
|
|
@@ -2151,6 +2186,9 @@ stable) {
|
|
|
2151
2186
|
// request carried, and the compressed bytes sent for them. Later requests
|
|
2152
2187
|
// reuse these bytes unchanged, so the prefix stays a cache read.
|
|
2153
2188
|
route.stablePrefix = storeStablePrefix(state, sessionId, originalMessages, route, stable);
|
|
2189
|
+
if (route.stablePrefix && key === JSON.stringify([sessionId, "main"])) {
|
|
2190
|
+
persistStablePrefix(state, sessionId, route.stablePrefix);
|
|
2191
|
+
}
|
|
2154
2192
|
}
|
|
2155
2193
|
setMemoryRoute(state, key, route);
|
|
2156
2194
|
state.routeDecisionsLive.set(key, new Set([decisionGeneration]));
|
|
@@ -2186,6 +2224,83 @@ function storeStablePrefix(state, sessionId, originalMessages, compressed, stabl
|
|
|
2186
2224
|
state.compactNow.delete(sessionId);
|
|
2187
2225
|
return prefix;
|
|
2188
2226
|
}
|
|
2227
|
+
/**
|
|
2228
|
+
* Persist the main lane's newly adopted prefix with its memory-source page.
|
|
2229
|
+
* Deferred one tick so the page noted right after the install is included,
|
|
2230
|
+
* and skipped when a newer prefix replaced this one meanwhile.
|
|
2231
|
+
*/
|
|
2232
|
+
function persistStablePrefix(state, sessionId, prefix) {
|
|
2233
|
+
const store = state.stablePrefixStore;
|
|
2234
|
+
const owner = state.storageOwner;
|
|
2235
|
+
if (!store || !owner)
|
|
2236
|
+
return;
|
|
2237
|
+
state.storedPrefixLookups.set(sessionId, null);
|
|
2238
|
+
setImmediate(() => {
|
|
2239
|
+
if (state.stablePrefixes.get(sessionId) !== prefix)
|
|
2240
|
+
return;
|
|
2241
|
+
const page = state.memorySourcePages.get(sessionId);
|
|
2242
|
+
store.save(sessionId, owner, {
|
|
2243
|
+
originalSystemHash: prefix.originalSystemHash,
|
|
2244
|
+
originalPrefixHashes: prefix.originalPrefixHashes,
|
|
2245
|
+
compressedMessages: prefix.compressedMessages,
|
|
2246
|
+
compressedSystem: prefix.compressedSystem,
|
|
2247
|
+
hasCompressedSystem: prefix.hasCompressedSystem,
|
|
2248
|
+
targetTokens: prefix.targetTokens,
|
|
2249
|
+
explicitTarget: prefix.explicitTarget,
|
|
2250
|
+
modelContextLimit: prefix.modelContextLimit,
|
|
2251
|
+
}, page && { url: page.url, index: page.index, compressed: page.compressed });
|
|
2252
|
+
});
|
|
2253
|
+
}
|
|
2254
|
+
/**
|
|
2255
|
+
* A fresh process's main-lane requests for a session: adopt the session's
|
|
2256
|
+
* persisted prefix when this history passes the checks the in-memory prefix
|
|
2257
|
+
* would (same session, system and covered messages; same window; a routable
|
|
2258
|
+
* body). Only an authoritative main history may discard it: one with the same
|
|
2259
|
+
* system and window, no server tools, and at least the covered length, whose
|
|
2260
|
+
* covered messages differ. Anything else (a side or helper request such as
|
|
2261
|
+
* the Haiku WebSearch call) leaves it for a later request. The disk is read
|
|
2262
|
+
* once per session; the store checks the backend/account owner.
|
|
2263
|
+
*/
|
|
2264
|
+
function adoptStoredPrefix(opts, state, rec, sessionId, body, messages, modelContextLimit) {
|
|
2265
|
+
const { stablePrefixStore: store, storageOwner: owner } = state;
|
|
2266
|
+
if (!store || !owner || state.stablePrefixes.has(sessionId))
|
|
2267
|
+
return;
|
|
2268
|
+
if (compactionMode(opts, state, sessionId).mode === "off")
|
|
2269
|
+
return;
|
|
2270
|
+
let entry = state.storedPrefixLookups.get(sessionId);
|
|
2271
|
+
if (entry === undefined) {
|
|
2272
|
+
entry = store.load(sessionId, owner) ?? null;
|
|
2273
|
+
boundedSet(state.storedPrefixLookups, sessionId, entry);
|
|
2274
|
+
}
|
|
2275
|
+
if (entry === null)
|
|
2276
|
+
return;
|
|
2277
|
+
const prefix = { sessionId, ...entry.prefix };
|
|
2278
|
+
const verdict = storedPrefixVerdict(body, messages, prefix, modelContextLimit);
|
|
2279
|
+
if (verdict === "defer")
|
|
2280
|
+
return;
|
|
2281
|
+
state.storedPrefixLookups.set(sessionId, null);
|
|
2282
|
+
if (verdict === "discard") {
|
|
2283
|
+
store.delete(sessionId, owner);
|
|
2284
|
+
rec.storedPrefix = "discarded";
|
|
2285
|
+
return;
|
|
2286
|
+
}
|
|
2287
|
+
boundedSet(state.stablePrefixes, sessionId, prefix);
|
|
2288
|
+
if (entry.source && !state.memorySourcePages.has(sessionId)) {
|
|
2289
|
+
rememberPage(state.memorySourcePages, sessionId, { sessionId, ...entry.source, seq: 0 });
|
|
2290
|
+
}
|
|
2291
|
+
rec.storedPrefix = "adopted";
|
|
2292
|
+
}
|
|
2293
|
+
/** What this request says about a persisted prefix (adoptStoredPrefix). */
|
|
2294
|
+
function storedPrefixVerdict(body, messages, prefix, modelContextLimit) {
|
|
2295
|
+
if (carriesServerTools(body) || prefix.modelContextLimit !== modelContextLimit)
|
|
2296
|
+
return "defer";
|
|
2297
|
+
const miss = prefixMismatch(body, messages, prefix, prefix.sessionId, stablePrefixMessageHash);
|
|
2298
|
+
if (miss === null)
|
|
2299
|
+
return prefixRoutedBody(body, messages, prefix) ? "adopt" : "defer";
|
|
2300
|
+
const authoritative = miss === "prefix" &&
|
|
2301
|
+
messages.length >= prefix.originalPrefixHashes.length;
|
|
2302
|
+
return authoritative ? "discard" : "defer";
|
|
2303
|
+
}
|
|
2189
2304
|
/**
|
|
2190
2305
|
* A fork of the main conversation (the away recap) sent on the main thread's
|
|
2191
2306
|
* last compressed prefix: that prefix, then the fork's own messages after it.
|
|
@@ -2367,11 +2482,31 @@ function notePrefixSize(prefix, rec, bytes) {
|
|
|
2367
2482
|
prefix.lastSize = sample;
|
|
2368
2483
|
}
|
|
2369
2484
|
/** Record the reported size of a main-thread request forwarded whole. */
|
|
2370
|
-
function notePassthroughSize(state, sessionId, rec, bytes) {
|
|
2485
|
+
function notePassthroughSize(state, sessionId, model, rec, bytes) {
|
|
2371
2486
|
const sample = reportedSample(rec, bytes);
|
|
2372
2487
|
if (sessionId === undefined || sample === undefined)
|
|
2373
2488
|
return;
|
|
2374
|
-
boundedSet(state.passthroughSizes, sessionId, sample);
|
|
2489
|
+
boundedSet(state.passthroughSizes, sessionSizeKey(sessionId, model), sample);
|
|
2490
|
+
}
|
|
2491
|
+
/** Key of a main-thread size sample (ProxyState.passthroughSizes). */
|
|
2492
|
+
function sessionSizeKey(sessionId, model) {
|
|
2493
|
+
return JSON.stringify([sessionId, typeof model === "string" ? model : null]);
|
|
2494
|
+
}
|
|
2495
|
+
/** Requests whose reported input includes server-injected tool results. */
|
|
2496
|
+
const serverToolRecords = new WeakSet();
|
|
2497
|
+
/**
|
|
2498
|
+
* Whether the request offers server tools (web search, web fetch, code
|
|
2499
|
+
* execution, tool search): their results are added to the input server-side,
|
|
2500
|
+
* so the reported size says nothing about the body's bytes.
|
|
2501
|
+
*/
|
|
2502
|
+
function carriesServerTools(body) {
|
|
2503
|
+
return Array.isArray(body.tools) && body.tools.some((tool) => typeof tool?.type === "string" && SERVER_TOOL_TYPE.test(tool.type));
|
|
2504
|
+
}
|
|
2505
|
+
const SERVER_TOOL_TYPE = /^(web_search|web_fetch|code_execution|tool_search_tool)_/;
|
|
2506
|
+
/** Whether the response reports any server tool use (server_tool_use counts). */
|
|
2507
|
+
function usedServerTools(rec) {
|
|
2508
|
+
const used = rec.usage?.server_tool_use;
|
|
2509
|
+
return !!used && Object.values(used).some((count) => count > 0);
|
|
2375
2510
|
}
|
|
2376
2511
|
/**
|
|
2377
2512
|
* The size Anthropic reported for a body of `bytes`, if it can describe one:
|
|
@@ -2381,6 +2516,8 @@ function notePassthroughSize(state, sessionId, rec, bytes) {
|
|
|
2381
2516
|
function reportedSample(rec, bytes) {
|
|
2382
2517
|
if (changedDeliveryRecords.has(rec))
|
|
2383
2518
|
return undefined;
|
|
2519
|
+
if (serverToolRecords.has(rec) || usedServerTools(rec))
|
|
2520
|
+
return undefined;
|
|
2384
2521
|
const tokens = reportedInputTokens(rec);
|
|
2385
2522
|
if (tokens === undefined)
|
|
2386
2523
|
return undefined;
|
|
@@ -2400,15 +2537,16 @@ function laneSizeKey(req, routeKey, model) {
|
|
|
2400
2537
|
]);
|
|
2401
2538
|
}
|
|
2402
2539
|
/** The session's passthrough size on the main thread, else the lane's. */
|
|
2403
|
-
function wholeRequestSizeSlot(state, isMainRequest, sessionId, sizeKey) {
|
|
2540
|
+
function wholeRequestSizeSlot(state, isMainRequest, sessionId, sizeKey, model) {
|
|
2404
2541
|
const main = isMainRequest && sessionId !== undefined;
|
|
2542
|
+
const mainKey = main ? sessionSizeKey(sessionId, model) : "";
|
|
2405
2543
|
return {
|
|
2406
|
-
get: () => (main ? state.passthroughSizes.get(
|
|
2544
|
+
get: () => (main ? state.passthroughSizes.get(mainKey) : state.laneSizes.get(sizeKey)),
|
|
2407
2545
|
set: (sample) => {
|
|
2408
2546
|
if (!plausibleSample(sample))
|
|
2409
2547
|
return;
|
|
2410
2548
|
if (main)
|
|
2411
|
-
boundedSet(state.passthroughSizes,
|
|
2549
|
+
boundedSet(state.passthroughSizes, mainKey, sample);
|
|
2412
2550
|
else
|
|
2413
2551
|
boundedSet(state.laneSizes, sizeKey, sample, MEMORY_ROUTE_MAX_LANES);
|
|
2414
2552
|
},
|
|
@@ -2420,9 +2558,9 @@ const noteRideSize = notePrefixSize;
|
|
|
2420
2558
|
* Record the reported size of a request forwarded whole: the session's
|
|
2421
2559
|
* passthrough size on the main thread, else its lane's.
|
|
2422
2560
|
*/
|
|
2423
|
-
function noteWholeRequestSize(state, isMainRequest, sessionId, sizeKey, rec, bytes) {
|
|
2561
|
+
function noteWholeRequestSize(state, isMainRequest, sessionId, sizeKey, model, rec, bytes) {
|
|
2424
2562
|
if (isMainRequest && sessionId !== undefined) {
|
|
2425
|
-
notePassthroughSize(state, sessionId, rec, bytes);
|
|
2563
|
+
notePassthroughSize(state, sessionId, model, rec, bytes);
|
|
2426
2564
|
return;
|
|
2427
2565
|
}
|
|
2428
2566
|
const sample = reportedSample(rec, bytes);
|
|
@@ -2595,7 +2733,7 @@ function planToolCompaction(args) {
|
|
|
2595
2733
|
const sample = ride
|
|
2596
2734
|
? ride.sizeHolder.lastSize
|
|
2597
2735
|
: stableLane
|
|
2598
|
-
? state.passthroughSizes.get(sessionId)
|
|
2736
|
+
? state.passthroughSizes.get(sessionSizeKey(sessionId, body.model))
|
|
2599
2737
|
: state.laneSizes.get(args.sizeKey);
|
|
2600
2738
|
const candidate = args.sizing.register(ride?.raw ?? forwardBody, ride ? "ride" : "original", sample);
|
|
2601
2739
|
if (ride)
|
|
@@ -2987,6 +3125,8 @@ async function runBlockingCompression(args) {
|
|
|
2987
3125
|
: undefined;
|
|
2988
3126
|
const messageTimes = transcriptTimesFor(opts, args.sessionId, args.agentId)?.(msgsForMemtree);
|
|
2989
3127
|
const compressStarted = Date.now();
|
|
3128
|
+
const budgetMs = args.longBudget
|
|
3129
|
+
? opts.memtree.longCompressBudgetMs : opts.memtree.compressBudgetMs;
|
|
2990
3130
|
const compressMeta = {
|
|
2991
3131
|
// Model + tools drive the server's model-based memory budget
|
|
2992
3132
|
// (e.g. 800k whole-request target for Fable / Opus 5). Omitting
|
|
@@ -3012,6 +3152,7 @@ async function runBlockingCompression(args) {
|
|
|
3012
3152
|
...(messageTimes && Object.keys(messageTimes).length ? { messageTimes } : {}),
|
|
3013
3153
|
...(args.sessionId !== undefined ? { sessionId: args.sessionId } : {}),
|
|
3014
3154
|
...(args.clientMeta ? { clientMeta: args.clientMeta } : {}),
|
|
3155
|
+
...(args.longBudget ? { timeoutMs: budgetMs } : {}),
|
|
3015
3156
|
};
|
|
3016
3157
|
// Sampled BEFORE the call, while it still describes this call: after the
|
|
3017
3158
|
// await the hash is in the cache regardless of who put it there.
|
|
@@ -3019,7 +3160,7 @@ async function runBlockingCompression(args) {
|
|
|
3019
3160
|
// compress() maps every failure to a resolved null (it never rejects).
|
|
3020
3161
|
// Keep foreground work finite even if a transport ignores its abort signal.
|
|
3021
3162
|
let deadline;
|
|
3022
|
-
const maximumWait = Math.min(30_000, Math.max(1, Number.isFinite(
|
|
3163
|
+
const maximumWait = Math.min(args.longBudget ? MAX_LONG_COMPRESS_WAIT_MS : 30_000, Math.max(1, Number.isFinite(budgetMs) ? budgetMs : 15_000));
|
|
3023
3164
|
const result = await Promise.race([
|
|
3024
3165
|
opts.memtree.compress(hash, msgsForMemtree, modelContextLimit, state.shutdownSignal, compressMeta),
|
|
3025
3166
|
new Promise((resolve) => { deadline = setTimeout(() => resolve(null), maximumWait); }),
|
|
@@ -3043,7 +3184,7 @@ async function runBlockingCompression(args) {
|
|
|
3043
3184
|
// Budget-consumed heuristic: the client maps every failure to null, so a
|
|
3044
3185
|
// null that took (roughly) the whole abort budget was almost certainly
|
|
3045
3186
|
// the AbortSignal timeout, not a fast server error.
|
|
3046
|
-
timedOut: result === null && compressMs >=
|
|
3187
|
+
timedOut: result === null && compressMs >= budgetMs,
|
|
3047
3188
|
};
|
|
3048
3189
|
return {
|
|
3049
3190
|
result,
|
|
@@ -3161,7 +3302,7 @@ async function recoverToolRouteMiss(args) {
|
|
|
3161
3302
|
capture(opts, "anthropic-request", args.originalBody);
|
|
3162
3303
|
return forwardRaw(req, res, args.originalBody, opts, upstream, state.shutdownSignal, rec).then((delivered) => {
|
|
3163
3304
|
toolForwarding.settle(delivered);
|
|
3164
|
-
noteWholeRequestSize(state, isMainRequest, sessionId, args.sizeKey, rec, args.originalBody.length);
|
|
3305
|
+
noteWholeRequestSize(state, isMainRequest, sessionId, args.sizeKey, body.model, rec, args.originalBody.length);
|
|
3165
3306
|
});
|
|
3166
3307
|
};
|
|
3167
3308
|
// Same downstream-close tracking as the followup path: compression
|
|
@@ -3190,6 +3331,7 @@ async function recoverToolRouteMiss(args) {
|
|
|
3190
3331
|
sizing: args.sizing,
|
|
3191
3332
|
inputCandidate: args.inputCandidate,
|
|
3192
3333
|
...(args.compaction ? { compaction: args.compaction } : {}),
|
|
3334
|
+
longBudget: args.wholeRefused ? await args.wholeRefused() : false,
|
|
3193
3335
|
});
|
|
3194
3336
|
}
|
|
3195
3337
|
catch {
|
|
@@ -3534,19 +3676,23 @@ async function handleCountTokens(req, res, opts, upstream, state) {
|
|
|
3534
3676
|
* picks it up without threading a parameter through each one.
|
|
3535
3677
|
*/
|
|
3536
3678
|
const recapLinkAppenders = new WeakMap();
|
|
3537
|
-
/**
|
|
3679
|
+
/**
|
|
3680
|
+
* One delivery-time retry after a ride's exact count exceeded the budget. It
|
|
3681
|
+
* compresses the request's ORIGINAL conversation at a tighter target: the
|
|
3682
|
+
* ride carries a memory prefix MemTree never indexed, so compressing it would
|
|
3683
|
+
* miss the tree, index a divergent conversation and bill for it.
|
|
3684
|
+
*/
|
|
3538
3685
|
async function recoverLateCount(args) {
|
|
3539
3686
|
try {
|
|
3540
|
-
const body =
|
|
3541
|
-
const messages = normalizeMessagesForMemtree(messagesWithSystem(body.messages, body.system));
|
|
3687
|
+
const { body, msgsForMemtree: messages } = args;
|
|
3542
3688
|
const budget = resolveBudget(args.opts, args.state, body.model, args.modelContextLimit).tokens;
|
|
3689
|
+
const target = lateRecoveryTarget(compactionTarget(compactionMode(args.opts, args.state, requestSessionId(args.req)), budget), budget, args.rideTokens);
|
|
3543
3690
|
const outcome = await runBlockingCompression({
|
|
3544
3691
|
opts: args.opts, state: args.state, body, msgsForMemtree: messages,
|
|
3545
|
-
hash:
|
|
3692
|
+
hash: args.hash, modelContextLimit: args.modelContextLimit,
|
|
3546
3693
|
rec: args.rec, sessionId: requestSessionId(args.req), agentId: agentAttributionId(args.req),
|
|
3547
|
-
clientMeta: args.clientMeta, sizing: args.sizing,
|
|
3548
|
-
|
|
3549
|
-
compaction: { target: compactionTarget(compactionMode(args.opts, args.state, requestSessionId(args.req)), budget) },
|
|
3694
|
+
clientMeta: args.clientMeta, sizing: args.sizing, inputCandidate: args.inputCandidate,
|
|
3695
|
+
compaction: { target },
|
|
3550
3696
|
});
|
|
3551
3697
|
noteMemtreeHealth(args.state, outcome);
|
|
3552
3698
|
// Like every compress path: a refusal after a lagging answer waits for
|
|
@@ -3565,6 +3711,15 @@ async function recoverLateCount(args) {
|
|
|
3565
3711
|
return undefined;
|
|
3566
3712
|
}
|
|
3567
3713
|
}
|
|
3714
|
+
/**
|
|
3715
|
+
* The ride compressed to `target` still counted `rideTokens` (over the
|
|
3716
|
+
* budget), so the recompression of the original aims proportionally lower.
|
|
3717
|
+
*/
|
|
3718
|
+
function lateRecoveryTarget(target, budget, rideTokens) {
|
|
3719
|
+
if (!(rideTokens > budget) || !(budget > 0))
|
|
3720
|
+
return target;
|
|
3721
|
+
return Math.max(SERVER_MIN_TARGET_TOKENS, Math.floor(target * budget / rideTokens));
|
|
3722
|
+
}
|
|
3568
3723
|
/** Select a measured eligible body before delivery; refusing one candidate never
|
|
3569
3724
|
* prevents considering the original or a validated prefix retained by planning. */
|
|
3570
3725
|
async function selectForwardCandidate(args) {
|
|
@@ -3609,6 +3764,17 @@ async function selectForwardCandidate(args) {
|
|
|
3609
3764
|
}
|
|
3610
3765
|
return changed ? candidate.body : intended;
|
|
3611
3766
|
}
|
|
3767
|
+
/**
|
|
3768
|
+
* Whether delivery would refuse `candidate` sent whole: it is planned over the
|
|
3769
|
+
* window, and the measurement selection would use (an exact count, else the
|
|
3770
|
+
* same estimate or bytes/4 fallback) does not fit either. Only then is there
|
|
3771
|
+
* no fallback, and only then may a compress take the long budget.
|
|
3772
|
+
*/
|
|
3773
|
+
async function wholeRequestRefused(sizing, candidate, limits) {
|
|
3774
|
+
if (fitsNative(sizing.plan(candidate).tokens, limits))
|
|
3775
|
+
return false;
|
|
3776
|
+
return !fitsMeasured(await sizing.measure(candidate, limits), limits);
|
|
3777
|
+
}
|
|
3612
3778
|
/** Write the bounded fallback refusal (route-fallback.ts); always false. */
|
|
3613
3779
|
function refuseWholeRequest(args, inputTokens, budgetTokens) {
|
|
3614
3780
|
const failure = args.state.fallbackFailures.fail(args.routeKey, { awaitingIndex: args.rec.indexBehind === true });
|
|
@@ -3647,6 +3813,9 @@ function noteSelectedCandidateSize(rec) {
|
|
|
3647
3813
|
const tokens = reportedInputTokens(rec);
|
|
3648
3814
|
if (!candidate || !owner || tokens === undefined)
|
|
3649
3815
|
return;
|
|
3816
|
+
// Server tools' injected results inflate the reported input (reportedSample).
|
|
3817
|
+
if (serverToolRecords.has(rec) || usedServerTools(rec))
|
|
3818
|
+
return;
|
|
3650
3819
|
const sample = { tokens, forwardedBytes: candidate.body.length };
|
|
3651
3820
|
if (!plausibleSample(sample))
|
|
3652
3821
|
return;
|