mixdog 0.9.41 → 0.9.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/agent-tag-reuse-smoke.mjs +124 -10
- package/scripts/anthropic-oauth-refresh-race-test.mjs +59 -0
- package/scripts/arg-guard-test.mjs +16 -0
- package/scripts/compact-pressure-test.mjs +256 -1
- package/scripts/internal-comms-bench.mjs +0 -1
- package/scripts/internal-comms-smoke.mjs +14 -32
- package/scripts/internal-tools-normalization-test.mjs +52 -0
- package/scripts/max-output-recovery-test.mjs +49 -0
- package/scripts/mcp-client-normalization-test.mjs +45 -0
- package/scripts/provider-toolcall-test.mjs +23 -0
- package/scripts/result-classification-test.mjs +75 -0
- package/scripts/smoke.mjs +1 -1
- package/scripts/submit-commandbusy-race-test.mjs +99 -7
- package/scripts/tui-transcript-perf-test.mjs +56 -1
- package/src/agents/reviewer/AGENT.md +4 -0
- package/src/runtime/agent/orchestrator/internal-tools.mjs +16 -3
- package/src/runtime/agent/orchestrator/mcp/client.mjs +17 -1
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth-credentials.mjs +49 -6
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +14 -0
- package/src/runtime/agent/orchestrator/providers/media-normalization.mjs +59 -2
- package/src/runtime/agent/orchestrator/session/agent-loop.mjs +18 -0
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +140 -81
- package/src/runtime/agent/orchestrator/session/loop/compact-policy.mjs +23 -5
- package/src/runtime/agent/orchestrator/session/loop/termination.mjs +3 -0
- package/src/runtime/agent/orchestrator/session/loop/tool-exec.mjs +12 -3
- package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs +4 -2
- package/src/runtime/agent/orchestrator/session/result-classification.mjs +4 -1
- package/src/runtime/agent/orchestrator/session/tool-batch.mjs +7 -2
- package/src/runtime/agent/orchestrator/session/tool-envelope.mjs +8 -6
- package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.mjs +15 -3
- package/src/runtime/agent/orchestrator/tools/builtin/shell-analysis.mjs +1 -1
- package/src/runtime/memory/tool-defs.mjs +4 -3
- package/src/runtime/shared/tool-surface.mjs +1 -2
- package/src/session-runtime/context-status.mjs +8 -2
- package/src/standalone/agent-tool/render.mjs +2 -0
- package/src/standalone/agent-tool.mjs +202 -22
- package/src/tui/components/StatusLine.jsx +4 -26
- package/src/tui/dist/index.mjs +46 -31
- package/src/tui/engine/session-api.mjs +3 -0
- package/src/tui/engine/session-flow.mjs +19 -8
- package/src/tui/engine/turn.mjs +24 -2
- package/src/tui/engine.mjs +0 -1
- package/src/ui/statusline-agents.mjs +0 -8
- package/src/ui/statusline.mjs +2 -4
- package/src/workflows/default/WORKFLOW.md +29 -20
- package/src/workflows/solo-review/WORKFLOW.md +47 -0
- package/src/workflows/bench/WORKFLOW.md +0 -60
|
@@ -6,6 +6,8 @@ import { randomUUID } from 'crypto';
|
|
|
6
6
|
import { smartReadTruncate } from '../tools/builtin/read-formatting.mjs';
|
|
7
7
|
import { shutdownStdioChild, killStdioChildTreeFast } from './child-tree.mjs';
|
|
8
8
|
import { readServicePort, markServiceUnreachable, isConnRefuseError } from '../../../shared/service-discovery.mjs';
|
|
9
|
+
import { makeToolEnvelope, normalizeToolEnvelope } from '../session/tool-envelope.mjs';
|
|
10
|
+
import { classifyResultKind } from '../session/result-classification.mjs';
|
|
9
11
|
// --- Types ---
|
|
10
12
|
/** Known auto-detect targets: port file path relative to tmpdir.
|
|
11
13
|
* Note: `mixdog` used to self-loopback via active-instance.json's
|
|
@@ -215,6 +217,16 @@ export async function executeMcpTool(name, args) {
|
|
|
215
217
|
throw new Error(`Tool call failed: ${firstMsg}; retry after reconnect also failed: ${retryMsg}`);
|
|
216
218
|
}
|
|
217
219
|
}
|
|
220
|
+
const normalized = normalizeToolEnvelope(normalizeMcpToolResult(result));
|
|
221
|
+
const text = capMcpOutput(normalized.result);
|
|
222
|
+
return normalized.explicitSuccess
|
|
223
|
+
? makeToolEnvelope(text, [], { explicitSuccess: true })
|
|
224
|
+
: text;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Preserve MCP failure metadata across the object→string boundary. The
|
|
228
|
+
// session loop classifies the canonical Error: prefix as toolKind:error.
|
|
229
|
+
export function normalizeMcpToolResult(result) {
|
|
218
230
|
const content = result.content;
|
|
219
231
|
let text;
|
|
220
232
|
if (Array.isArray(content)) {
|
|
@@ -224,7 +236,11 @@ export async function executeMcpTool(name, args) {
|
|
|
224
236
|
} else {
|
|
225
237
|
text = typeof content === 'string' ? content : JSON.stringify(content);
|
|
226
238
|
}
|
|
227
|
-
return
|
|
239
|
+
if (result.isError === true) return !text.startsWith('Error:') ? `Error: ${text}` : text;
|
|
240
|
+
if (result.isError === false && classifyResultKind(text) === 'error') {
|
|
241
|
+
return makeToolEnvelope(text, [], { explicitSuccess: true });
|
|
242
|
+
}
|
|
243
|
+
return text;
|
|
228
244
|
}
|
|
229
245
|
|
|
230
246
|
// MCP per-tool-call timeout. Default 2min: a hung/unresponsive MCP server
|
|
@@ -53,7 +53,10 @@ const OAUTH_SUCCESS_REDIRECT_URL = process.env.ANTHROPIC_OAUTH_SUCCESS_REDIRECT_
|
|
|
53
53
|
const OAUTH_LOGIN_TIMEOUT_MS = 5 * 60_000;
|
|
54
54
|
const OAUTH_TOKEN_TIMEOUT_MS = 30_000;
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
// Anthropic's token edge validates the Claude Code client identity. Keep this
|
|
57
|
+
// fallback aligned with the current official CLI while retaining the env
|
|
58
|
+
// override for seats where Claude Code is updated ahead of Mixdog.
|
|
59
|
+
export const DEFAULT_CLI_VERSION = '2.1.207';
|
|
57
60
|
|
|
58
61
|
export function resolveCliVersion() {
|
|
59
62
|
return process.env.MIXDOG_CLI_VERSION
|
|
@@ -210,14 +213,41 @@ export function _normalizeExpiresAt(value) {
|
|
|
210
213
|
return value < 1e12 ? value * 1000 : value;
|
|
211
214
|
}
|
|
212
215
|
|
|
213
|
-
export function _scrubTokens(text) {
|
|
214
|
-
|
|
215
|
-
.replace(/Bearer [A-Za-z0-9._\-]+/
|
|
216
|
+
export function _scrubTokens(text, secretValues = []) {
|
|
217
|
+
let scrubbed = String(text || '')
|
|
218
|
+
.replace(/Bearer [A-Za-z0-9._\-]+/gi, 'Bearer [REDACTED]')
|
|
216
219
|
.replace(/sk-ant-[A-Za-z0-9._\-]+/g, '[REDACTED]')
|
|
217
220
|
.replace(/"access[Tt]oken"\s*:\s*"[^"]+"/g, '"accessToken":"[REDACTED]"')
|
|
218
221
|
.replace(/"refresh[Tt]oken"\s*:\s*"[^"]+"/g, '"refreshToken":"[REDACTED]"')
|
|
219
222
|
.replace(/"access_token"\s*:\s*"[^"]+"/g, '"access_token":"[REDACTED]"')
|
|
220
223
|
.replace(/"refresh_token"\s*:\s*"[^"]+"/g, '"refresh_token":"[REDACTED]"');
|
|
224
|
+
// Token services sometimes echo submitted values in non-JSON diagnostics.
|
|
225
|
+
// Scrub the exact request secrets as a final guard without logging them.
|
|
226
|
+
for (const secret of secretValues) {
|
|
227
|
+
if (typeof secret === 'string' && secret) {
|
|
228
|
+
scrubbed = scrubbed.split(secret).join('[REDACTED]');
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return scrubbed;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function _tokenEndpointError(operation, status, text, secretValues = []) {
|
|
235
|
+
const safeDetail = _scrubTokens(text, secretValues)
|
|
236
|
+
.replace(/\s+/g, ' ')
|
|
237
|
+
.trim()
|
|
238
|
+
.slice(0, 500);
|
|
239
|
+
const detail = safeDetail ? `: ${safeDetail}` : '';
|
|
240
|
+
const compatibility = status === 403
|
|
241
|
+
? (
|
|
242
|
+
` Anthropic returned HTTP 403 for a request sent as claude-cli/${resolveCliVersion()}.`
|
|
243
|
+
+ ' Possible causes include OAuth client compatibility, account or scope policy,'
|
|
244
|
+
+ ' an intercepting proxy/VPN/WAF, or regional endpoint restrictions.'
|
|
245
|
+
+ ' Verify the account has Claude Code access and the required scopes;'
|
|
246
|
+
+ ' update Mixdog or set MIXDOG_CLI_VERSION to the installed official Claude Code version;'
|
|
247
|
+
+ ' then retry sign-in via /providers without an intercepting network layer if applicable.'
|
|
248
|
+
)
|
|
249
|
+
: '';
|
|
250
|
+
return new Error(`${operation} ${status}${detail}.${compatibility}`);
|
|
221
251
|
}
|
|
222
252
|
|
|
223
253
|
export function isAnthropicOAuthRefreshDisabled() {
|
|
@@ -261,7 +291,15 @@ async function _refreshOAuthCredentialsUnlocked(creds) {
|
|
|
261
291
|
try { json = text ? JSON.parse(text) : null; } catch { /* handled below */ }
|
|
262
292
|
if (!res.ok) {
|
|
263
293
|
const isInvalidGrant = text.includes('invalid_grant') || json?.error === 'invalid_grant';
|
|
264
|
-
throw Object.assign(
|
|
294
|
+
throw Object.assign(
|
|
295
|
+
_tokenEndpointError(
|
|
296
|
+
'token refresh',
|
|
297
|
+
res.status,
|
|
298
|
+
text,
|
|
299
|
+
[creds.refreshToken, creds.accessToken],
|
|
300
|
+
),
|
|
301
|
+
{ isInvalidGrant },
|
|
302
|
+
);
|
|
265
303
|
}
|
|
266
304
|
|
|
267
305
|
const accessToken = json?.access_token || json?.accessToken;
|
|
@@ -495,7 +533,12 @@ async function exchangeAuthorizationCode({ pkce, code, state, redirectUri }) {
|
|
|
495
533
|
});
|
|
496
534
|
if (!tokenRes.ok) {
|
|
497
535
|
const text = await tokenRes.text().catch(() => '');
|
|
498
|
-
throw
|
|
536
|
+
throw _tokenEndpointError(
|
|
537
|
+
'[anthropic-oauth] token exchange',
|
|
538
|
+
tokenRes.status,
|
|
539
|
+
text,
|
|
540
|
+
[cleanCode, pkce.verifier, state],
|
|
541
|
+
);
|
|
499
542
|
}
|
|
500
543
|
const json = await tokenRes.json();
|
|
501
544
|
const accessToken = json?.access_token || json?.accessToken;
|
|
@@ -111,6 +111,7 @@ export async function parseSSEStream(response, signal, abortStream, onStreamDelt
|
|
|
111
111
|
let toolCalls = [];
|
|
112
112
|
let usage = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, cacheWriteTokens: 0, raw: null };
|
|
113
113
|
let stopReason = null;
|
|
114
|
+
let stopDetails;
|
|
114
115
|
let buffer = '';
|
|
115
116
|
let idleTimedOut = false;
|
|
116
117
|
let firstMessageTimedOut = false;
|
|
@@ -546,6 +547,18 @@ export async function parseSSEStream(response, signal, abortStream, onStreamDelt
|
|
|
546
547
|
if (event.delta?.stop_reason) {
|
|
547
548
|
stopReason = event.delta.stop_reason;
|
|
548
549
|
}
|
|
550
|
+
if (event.delta && (event.delta.stop_details != null || event.delta.category != null)) {
|
|
551
|
+
const details = event.delta.stop_details;
|
|
552
|
+
stopDetails = details && typeof details === 'object' && !Array.isArray(details)
|
|
553
|
+
? {
|
|
554
|
+
...details,
|
|
555
|
+
...(event.delta.category != null ? { category: event.delta.category } : {}),
|
|
556
|
+
}
|
|
557
|
+
: {
|
|
558
|
+
...(details != null ? { value: details } : {}),
|
|
559
|
+
...(event.delta.category != null ? { category: event.delta.category } : {}),
|
|
560
|
+
};
|
|
561
|
+
}
|
|
549
562
|
if (event.usage) {
|
|
550
563
|
usage.outputTokens = event.usage.output_tokens || 0;
|
|
551
564
|
usage.raw = { ...(usage.raw || {}), ...event.usage };
|
|
@@ -615,6 +628,7 @@ export async function parseSSEStream(response, signal, abortStream, onStreamDelt
|
|
|
615
628
|
toolCalls: toolCalls.length ? toolCalls : undefined,
|
|
616
629
|
usage,
|
|
617
630
|
stopReason,
|
|
631
|
+
stopDetails,
|
|
618
632
|
hasThinkingContent,
|
|
619
633
|
contentBlockTypes: Array.from(contentBlockTypes),
|
|
620
634
|
// Ordered extended-thinking blocks (verbatim thinking text +
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
1
3
|
const DEFAULT_IMAGE_MIME = 'image/png';
|
|
2
4
|
|
|
3
5
|
function cleanMimeType(value) {
|
|
@@ -96,6 +98,9 @@ function contentParts(content) {
|
|
|
96
98
|
if (content && typeof content === 'object' && Array.isArray(content.content)) {
|
|
97
99
|
return content.content;
|
|
98
100
|
}
|
|
101
|
+
if (content && typeof content === 'object'
|
|
102
|
+
&& (imageUrlFromPart(content) || imageFileIdFromPart(content)
|
|
103
|
+
|| imageFileUriFromPart(content) || geminiInlineInfo(content))) return [content];
|
|
99
104
|
return null;
|
|
100
105
|
}
|
|
101
106
|
|
|
@@ -108,9 +113,61 @@ function jsonFallbackFromPart(block) {
|
|
|
108
113
|
}
|
|
109
114
|
|
|
110
115
|
export function contentHasImage(content) {
|
|
116
|
+
return contentImageCount(content) > 0;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function contentImageCount(content) {
|
|
120
|
+
return contentImageDescriptors(content).length;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function positiveDimension(...values) {
|
|
124
|
+
for (const value of values) {
|
|
125
|
+
const number = Number(value);
|
|
126
|
+
if (Number.isFinite(number) && number > 0) return number;
|
|
127
|
+
}
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function imageIdentity(kind, value, mimeType = '') {
|
|
132
|
+
return createHash('sha256')
|
|
133
|
+
.update(`${kind}\0${mimeType}\0`)
|
|
134
|
+
.update(String(value || ''))
|
|
135
|
+
.digest('hex');
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function imageDescriptor(part) {
|
|
139
|
+
if (!part || typeof part !== 'object') return null;
|
|
140
|
+
const info = imageInfo(part);
|
|
141
|
+
const inline = geminiInlineInfo(part);
|
|
142
|
+
const url = imageUrlFromPart(part);
|
|
143
|
+
const fileId = imageFileIdFromPart(part);
|
|
144
|
+
const fileUri = imageFileUriFromPart(part);
|
|
145
|
+
if (!info && !inline && !url && !fileId && !fileUri) return null;
|
|
146
|
+
const imageUrl = part.image_url && typeof part.image_url === 'object' ? part.image_url : null;
|
|
147
|
+
const source = part.source && typeof part.source === 'object' ? part.source : null;
|
|
148
|
+
const inlineData = part.inlineData || part.inline_data;
|
|
149
|
+
const dimensions = part.dimensions && typeof part.dimensions === 'object' ? part.dimensions : null;
|
|
150
|
+
const width = positiveDimension(part.width, dimensions?.width, imageUrl?.width, source?.width, inlineData?.width);
|
|
151
|
+
const height = positiveDimension(part.height, dimensions?.height, imageUrl?.height, source?.height, inlineData?.height);
|
|
152
|
+
const detail = String(part.detail ?? imageUrl?.detail ?? source?.detail ?? inlineData?.detail ?? 'auto').toLowerCase();
|
|
153
|
+
if (info || inline) {
|
|
154
|
+
const raw = info || inline;
|
|
155
|
+
return {
|
|
156
|
+
identity: imageIdentity('inline', raw.data, raw.mimeType),
|
|
157
|
+
width,
|
|
158
|
+
height,
|
|
159
|
+
detail,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
if (fileId) return { identity: imageIdentity('file-id', fileId), width, height, detail };
|
|
163
|
+
if (fileUri) return { identity: imageIdentity('file-uri', fileUri.fileUri, fileUri.mimeType), width, height, detail };
|
|
164
|
+
return { identity: imageIdentity('url', url), width, height, detail };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export function contentImageDescriptors(content) {
|
|
111
168
|
const parts = contentParts(content);
|
|
112
|
-
if (!parts) return
|
|
113
|
-
return parts.
|
|
169
|
+
if (!parts) return [];
|
|
170
|
+
return parts.map(imageDescriptor).filter(Boolean);
|
|
114
171
|
}
|
|
115
172
|
|
|
116
173
|
export function contentToText(content, fallback = '') {
|
|
@@ -145,6 +145,7 @@ export async function agentLoop(provider, messages, model, tools, onToolCall, cw
|
|
|
145
145
|
let lastUsage;
|
|
146
146
|
let firstTurnUsage;
|
|
147
147
|
let response;
|
|
148
|
+
let lastSendTools = tools;
|
|
148
149
|
let contextOverflowRetryUsed = false;
|
|
149
150
|
// Set when the hard iteration-cap break below fires. Consumed at the final
|
|
150
151
|
// return to tag terminationReason='iteration_cap' so a worker that exhausts
|
|
@@ -281,6 +282,7 @@ export async function agentLoop(provider, messages, model, tools, onToolCall, cw
|
|
|
281
282
|
// the loop as an explicit empty termination instead.
|
|
282
283
|
let _emptyNudgeStreak = 0;
|
|
283
284
|
const EMPTY_NUDGE_MAX = 3;
|
|
285
|
+
let _refusalRetryUsed = false;
|
|
284
286
|
let _maxOutputRecoveryCount = 0;
|
|
285
287
|
const _maxOutputContentParts = [];
|
|
286
288
|
// Claude Code parity: queued prompt/task notifications are attached after a
|
|
@@ -435,6 +437,7 @@ export async function agentLoop(provider, messages, model, tools, onToolCall, cw
|
|
|
435
437
|
const sendTools = _capFinalToolsDisabled
|
|
436
438
|
? tools
|
|
437
439
|
: (forcedFirstToolDef && toolCallsTotal === 0 ? [forcedFirstToolDef] : tools);
|
|
440
|
+
lastSendTools = sendTools;
|
|
438
441
|
// Eager-dispatch queue: when the provider streams a tool-call event,
|
|
439
442
|
// start read-only tools immediately so execution overlaps with the
|
|
440
443
|
// remaining SSE parse. Writes and unknown tools wait until send()
|
|
@@ -572,6 +575,7 @@ export async function agentLoop(provider, messages, model, tools, onToolCall, cw
|
|
|
572
575
|
// additive — callers that ignore these fields keep working.
|
|
573
576
|
deltaCachedRead: response.usage.cachedTokens || 0,
|
|
574
577
|
deltaCacheWrite: response.usage.cacheWriteTokens || 0,
|
|
578
|
+
sendTools,
|
|
575
579
|
ts: Date.now(),
|
|
576
580
|
});
|
|
577
581
|
} catch { /* best-effort — never break the loop */ }
|
|
@@ -633,6 +637,19 @@ export async function agentLoop(provider, messages, model, tools, onToolCall, cw
|
|
|
633
637
|
};
|
|
634
638
|
break;
|
|
635
639
|
}
|
|
640
|
+
if (!hasContent && stopReason === 'refusal') {
|
|
641
|
+
if (_refusalRetryUsed) {
|
|
642
|
+
process.stderr.write(`[loop] safety-classifier refusal persisted after one context-changing retry (sess=${sessionId || 'unknown'}); ending loop as refusal termination.\n`);
|
|
643
|
+
break;
|
|
644
|
+
}
|
|
645
|
+
_refusalRetryUsed = true;
|
|
646
|
+
messages.push({
|
|
647
|
+
role: 'user',
|
|
648
|
+
content: '[mixdog-runtime] The previous completion was refused by the provider safety classifier (stopReason=refusal). Do not repeat it. Complete your assigned output within policy by omitting or reframing disallowed content; if no compliant output is possible, briefly state the refusal.',
|
|
649
|
+
meta: { source: 'refusal-recovery', attempt: 1 },
|
|
650
|
+
});
|
|
651
|
+
continue;
|
|
652
|
+
}
|
|
636
653
|
if (!hasContent && !isHidden) {
|
|
637
654
|
_emptyNudgeStreak += 1;
|
|
638
655
|
if (_emptyNudgeStreak > EMPTY_NUDGE_MAX) {
|
|
@@ -751,6 +768,7 @@ export async function agentLoop(provider, messages, model, tools, onToolCall, cw
|
|
|
751
768
|
...response,
|
|
752
769
|
usage: lastUsage || response.usage,
|
|
753
770
|
lastTurnUsage: response.usage,
|
|
771
|
+
lastSendTools,
|
|
754
772
|
firstTurnUsage: firstTurnUsage || response.usage,
|
|
755
773
|
iterations,
|
|
756
774
|
toolCallsTotal,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isOffloadedToolResultText } from './tool-result-offload.mjs';
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import { isAgentOwner } from '../agent-owner.mjs';
|
|
4
|
+
import { contentImageDescriptors, contentToText } from '../providers/media-normalization.mjs';
|
|
4
5
|
|
|
5
6
|
// ---------------------------------------------------------------------------
|
|
6
7
|
// Conservative, Unicode-aware token estimator.
|
|
@@ -26,6 +27,8 @@ function readSafetyMultiplier() {
|
|
|
26
27
|
return 1.1;
|
|
27
28
|
}
|
|
28
29
|
const TOKEN_ESTIMATE_SAFETY_MULTIPLIER = readSafetyMultiplier();
|
|
30
|
+
export const IMAGE_VISUAL_TOKEN_ALLOWANCE = 4_096;
|
|
31
|
+
const IMAGE_TILE_TOKEN_ALLOWANCE = 512;
|
|
29
32
|
|
|
30
33
|
// Per-code-point token-cost weight. Tuned to overcount, not match exactly.
|
|
31
34
|
function codePointTokenWeight(cp) {
|
|
@@ -72,12 +75,59 @@ export function estimateTokens(text) {
|
|
|
72
75
|
if (s.length === 0) return 0;
|
|
73
76
|
let weighted = 0;
|
|
74
77
|
for (const ch of s) weighted += codePointTokenWeight(ch.codePointAt(0));
|
|
78
|
+
// Encoded blobs, minified JSON and generated identifiers do not get the
|
|
79
|
+
// word/whitespace merges that make prose approach chars/4. Long printable
|
|
80
|
+
// ASCII runs are commonly 0.5-0.8 tokens/byte; retain a conservative floor
|
|
81
|
+
// for those runs without penalizing ordinary spaced prose.
|
|
82
|
+
let denseAsciiFloor = 0;
|
|
83
|
+
for (const match of s.matchAll(/[\x21-\x7e]{16,}/g)) {
|
|
84
|
+
denseAsciiFloor += match[0].length * 0.75;
|
|
85
|
+
}
|
|
86
|
+
const encodedWords = s.match(/\b(?=[A-Za-z0-9]{8,}\b)(?=[A-Za-z0-9]*[A-Za-z])(?=[A-Za-z0-9]*\d)[A-Za-z0-9]+\b/g) || [];
|
|
87
|
+
if (encodedWords.length >= 3) {
|
|
88
|
+
// Encoded/generated identifiers are often wrapped at short columns or
|
|
89
|
+
// separated by spaces. Their individual runs can stay below the long-
|
|
90
|
+
// run threshold while still receiving almost no prose-style BPE merges.
|
|
91
|
+
const encodedChars = encodedWords.reduce((sum, word) => sum + word.length, 0);
|
|
92
|
+
denseAsciiFloor = Math.max(
|
|
93
|
+
denseAsciiFloor,
|
|
94
|
+
(encodedChars * 0.75) + ((s.length - encodedChars) * 0.25),
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
const lines = s.split(/\r?\n/).filter(line => line.trim());
|
|
98
|
+
const nonWhitespace = s.match(/\S/g)?.length || 0;
|
|
99
|
+
const structural = s.match(/[\[\]{}":,=<>|\\]/g)?.length || 0;
|
|
100
|
+
const jsonLikeLines = lines.filter(line => /^\s*[\[{].*[\]}],?\s*$/.test(line)).length;
|
|
101
|
+
if (lines.length >= 3 && nonWhitespace > 0
|
|
102
|
+
&& (jsonLikeLines >= Math.ceil(lines.length / 2) || structural / nonWhitespace >= 0.12)) {
|
|
103
|
+
// JSONL, compact tables and generated line protocols can consist
|
|
104
|
+
// entirely of short runs while still tokenizing like minified data.
|
|
105
|
+
denseAsciiFloor = Math.max(denseAsciiFloor, (nonWhitespace * 0.7) + ((s.length - nonWhitespace) * 0.25));
|
|
106
|
+
}
|
|
75
107
|
const asciiFloor = s.length / 4; // never below the legacy chars/4 lower bound
|
|
76
|
-
return Math.ceil(Math.max(weighted, asciiFloor) * TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
|
|
108
|
+
return Math.ceil(Math.max(weighted, asciiFloor, denseAsciiFloor) * TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function nativeBlocksEstimateText(value) {
|
|
112
|
+
const list = Array.isArray(value) ? value : [value];
|
|
113
|
+
return list.map((block) => {
|
|
114
|
+
const images = contentImageDescriptors(block);
|
|
115
|
+
if (images.length) {
|
|
116
|
+
return JSON.stringify(images.map(({ width, height, detail }) => ({
|
|
117
|
+
type: 'image', width, height, detail,
|
|
118
|
+
})));
|
|
119
|
+
}
|
|
120
|
+
try { return JSON.stringify(block); }
|
|
121
|
+
catch { return String(block ?? ''); }
|
|
122
|
+
}).join('\n');
|
|
77
123
|
}
|
|
78
124
|
export function messageEstimateText(m) {
|
|
79
125
|
if (!m || typeof m !== 'object') return '';
|
|
80
|
-
|
|
126
|
+
// Multimodal image payloads remain on the live message for provider sends,
|
|
127
|
+
// but their base64/data-url JSON is not text and must not dominate local
|
|
128
|
+
// context estimates. Use the same media-aware text projection for every
|
|
129
|
+
// estimate consumer (live gauge, compaction fallback, and summaries).
|
|
130
|
+
let text = contentToText(m.content, '');
|
|
81
131
|
if (m.role === 'assistant' && Array.isArray(m.toolCalls) && m.toolCalls.length) {
|
|
82
132
|
try { text += `\n${JSON.stringify(m.toolCalls)}`; }
|
|
83
133
|
catch { text += `\n[${m.toolCalls.length} tool calls]`; }
|
|
@@ -86,14 +136,47 @@ export function messageEstimateText(m) {
|
|
|
86
136
|
// signature / redacted data) and are re-sent on tool-continuation turns, so
|
|
87
137
|
// they consume real input tokens. Count them or trim/compact undercounts.
|
|
88
138
|
if (m.role === 'assistant' && Array.isArray(m.thinkingBlocks) && m.thinkingBlocks.length) {
|
|
89
|
-
|
|
90
|
-
|
|
139
|
+
text += `\n${nativeBlocksEstimateText(m.thinkingBlocks)}`;
|
|
140
|
+
}
|
|
141
|
+
// Some provider adapters replay their native assistant representation
|
|
142
|
+
// instead of content/toolCalls. Project it through the media normalizer so
|
|
143
|
+
// text/tool metadata and opaque reasoning are counted without base64 image
|
|
144
|
+
// bytes dominating the estimate.
|
|
145
|
+
if (m.role === 'assistant' && Array.isArray(m.assistantBlocks) && m.assistantBlocks.length) {
|
|
146
|
+
text += `\n${nativeBlocksEstimateText(m.assistantBlocks)}`;
|
|
147
|
+
}
|
|
148
|
+
if (m.role === 'assistant' && Array.isArray(m.reasoningItems) && m.reasoningItems.length) {
|
|
149
|
+
text += `\n${nativeBlocksEstimateText(m.reasoningItems)}`;
|
|
91
150
|
}
|
|
92
151
|
if (m.role === 'tool' && m.toolCallId) text += `\n${m.toolCallId}`;
|
|
93
152
|
return text;
|
|
94
153
|
}
|
|
154
|
+
function imageDescriptorAllowance(descriptor) {
|
|
155
|
+
if (descriptor.width && descriptor.height) {
|
|
156
|
+
const tiles = Math.ceil(descriptor.width / 512) * Math.ceil(descriptor.height / 512);
|
|
157
|
+
// Caller-supplied dimensions/detail are not uniformly preserved by all
|
|
158
|
+
// provider normalizers, so they may raise the allowance for a known
|
|
159
|
+
// multi-tile image but can never lower the conservative unknown-image
|
|
160
|
+
// fallback. This avoids trusting metadata the provider never sees.
|
|
161
|
+
return Math.max(IMAGE_VISUAL_TOKEN_ALLOWANCE, IMAGE_TILE_TOKEN_ALLOWANCE * (tiles + 1));
|
|
162
|
+
}
|
|
163
|
+
// Unknown-size auto/high images may be tiled by the provider. A single
|
|
164
|
+
// 1k allowance is optimistic for common screenshots and documents.
|
|
165
|
+
return IMAGE_VISUAL_TOKEN_ALLOWANCE;
|
|
166
|
+
}
|
|
167
|
+
function messageImageDescriptors(m) {
|
|
168
|
+
if (!m || typeof m !== 'object') return [];
|
|
169
|
+
return [
|
|
170
|
+
...contentImageDescriptors(m.content),
|
|
171
|
+
...(m.role === 'assistant' ? contentImageDescriptors(m.assistantBlocks) : []),
|
|
172
|
+
];
|
|
173
|
+
}
|
|
174
|
+
function messageImageAllowance(m) {
|
|
175
|
+
if (!m || typeof m !== 'object') return 0;
|
|
176
|
+
return messageImageDescriptors(m).reduce((sum, descriptor) => sum + imageDescriptorAllowance(descriptor), 0);
|
|
177
|
+
}
|
|
95
178
|
export function estimateMessageTokens(m) {
|
|
96
|
-
return estimateTokens(messageEstimateText(m)) + 4;
|
|
179
|
+
return estimateTokens(messageEstimateText(m)) + messageImageAllowance(m) + 4;
|
|
97
180
|
}
|
|
98
181
|
export function estimateMessagesTokens(messages) {
|
|
99
182
|
return messages.reduce((sum, m) => sum + estimateMessageTokens(m), 0);
|
|
@@ -112,28 +195,11 @@ const contextMessageMemo = new WeakMap();
|
|
|
112
195
|
const contextTranscriptMemo = new WeakMap();
|
|
113
196
|
|
|
114
197
|
function contextValueFingerprint(value) {
|
|
115
|
-
if (typeof value === 'string') return { value, entries: null };
|
|
116
|
-
if (Array.isArray(value)) return {
|
|
117
|
-
value,
|
|
118
|
-
entries: value.map(contextBlockFingerprint),
|
|
119
|
-
};
|
|
120
|
-
return value && typeof value === 'object'
|
|
121
|
-
? { value, entries: [contextBlockFingerprint(value)] }
|
|
122
|
-
: { value, entries: null };
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function contextBlockFingerprint(block) {
|
|
126
|
-
if (!block || typeof block !== 'object') return { value: block };
|
|
127
|
-
const fn = block.function;
|
|
128
198
|
return {
|
|
129
|
-
value
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
arguments: block.arguments,
|
|
134
|
-
input: typeof block.input === 'string' ? block.input : null,
|
|
135
|
-
function: fn,
|
|
136
|
-
functionArguments: typeof fn?.arguments === 'string' ? fn.arguments : null,
|
|
199
|
+
value,
|
|
200
|
+
snapshot: typeof value === 'string'
|
|
201
|
+
? value
|
|
202
|
+
: `${nativeBlocksEstimateText(value)}\0${JSON.stringify(contentImageDescriptors(value))}`,
|
|
137
203
|
};
|
|
138
204
|
}
|
|
139
205
|
|
|
@@ -145,6 +211,7 @@ function contextMessageFingerprint(message) {
|
|
|
145
211
|
toolCalls: contextValueFingerprint(null),
|
|
146
212
|
thinkingBlocks: contextValueFingerprint(null),
|
|
147
213
|
assistantBlocks: contextValueFingerprint(null),
|
|
214
|
+
reasoningItems: contextValueFingerprint(null),
|
|
148
215
|
toolCallId: null,
|
|
149
216
|
};
|
|
150
217
|
}
|
|
@@ -154,30 +221,13 @@ function contextMessageFingerprint(message) {
|
|
|
154
221
|
toolCalls: contextValueFingerprint(message.toolCalls),
|
|
155
222
|
thinkingBlocks: contextValueFingerprint(message.thinkingBlocks),
|
|
156
223
|
assistantBlocks: contextValueFingerprint(message.assistantBlocks),
|
|
224
|
+
reasoningItems: contextValueFingerprint(message.reasoningItems),
|
|
157
225
|
toolCallId: message?.toolCallId || null,
|
|
158
226
|
};
|
|
159
227
|
}
|
|
160
228
|
|
|
161
229
|
function sameContextValueFingerprint(a, b) {
|
|
162
|
-
|
|
163
|
-
if (a.entries === null || b.entries === null) return a.entries === b.entries;
|
|
164
|
-
if (a.entries.length !== b.entries.length) return false;
|
|
165
|
-
for (let index = 0; index < a.entries.length; index += 1) {
|
|
166
|
-
if (!sameContextBlockFingerprint(a.entries[index], b.entries[index])) return false;
|
|
167
|
-
}
|
|
168
|
-
return true;
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
function sameContextBlockFingerprint(a, b) {
|
|
172
|
-
return !!a && !!b
|
|
173
|
-
&& a.value === b.value
|
|
174
|
-
&& a.text === b.text
|
|
175
|
-
&& a.content === b.content
|
|
176
|
-
&& a.args === b.args
|
|
177
|
-
&& Object.is(a.arguments, b.arguments)
|
|
178
|
-
&& a.input === b.input
|
|
179
|
-
&& a.function === b.function
|
|
180
|
-
&& a.functionArguments === b.functionArguments;
|
|
230
|
+
return !!a && !!b && a.value === b.value && a.snapshot === b.snapshot;
|
|
181
231
|
}
|
|
182
232
|
|
|
183
233
|
function sameContextMessageFingerprint(a, b) {
|
|
@@ -186,6 +236,7 @@ function sameContextMessageFingerprint(a, b) {
|
|
|
186
236
|
&& sameContextValueFingerprint(a.toolCalls, b.toolCalls)
|
|
187
237
|
&& sameContextValueFingerprint(a.thinkingBlocks, b.thinkingBlocks)
|
|
188
238
|
&& sameContextValueFingerprint(a.assistantBlocks, b.assistantBlocks)
|
|
239
|
+
&& sameContextValueFingerprint(a.reasoningItems, b.reasoningItems)
|
|
189
240
|
&& a.toolCallId === b.toolCallId;
|
|
190
241
|
}
|
|
191
242
|
|
|
@@ -197,7 +248,7 @@ function contextMessageContribution(message) {
|
|
|
197
248
|
}
|
|
198
249
|
const role = ['system', 'user', 'assistant', 'tool'].includes(fingerprint.role) ? fingerprint.role : 'other';
|
|
199
250
|
const text = messageEstimateText(message);
|
|
200
|
-
const tokens = estimateTokens(text) + 4;
|
|
251
|
+
const tokens = estimateTokens(text) + messageImageAllowance(message) + 4;
|
|
201
252
|
const contribution = {
|
|
202
253
|
role,
|
|
203
254
|
tokens,
|
|
@@ -511,7 +562,7 @@ export function summarizeContextMessages(messages) {
|
|
|
511
562
|
if (!Array.isArray(messages)) return contextSummaryResult(emptyContextSummaryState(), 0);
|
|
512
563
|
let cached = contextTranscriptMemo.get(messages);
|
|
513
564
|
if (!cached || messages.length < cached.count) {
|
|
514
|
-
cached = { count: 0, contributions: [], state: emptyContextSummaryState() };
|
|
565
|
+
cached = { count: 0, contributions: [], state: emptyContextSummaryState(), revision: 0 };
|
|
515
566
|
contextTranscriptMemo.set(messages, cached);
|
|
516
567
|
}
|
|
517
568
|
for (let index = 0; index < messages.length; index += 1) {
|
|
@@ -521,12 +572,42 @@ export function summarizeContextMessages(messages) {
|
|
|
521
572
|
if (previous) applyContextMessageContribution(cached.state, previous, -1);
|
|
522
573
|
cached.contributions[index] = contribution;
|
|
523
574
|
applyContextMessageContribution(cached.state, contribution, 1);
|
|
575
|
+
cached.revision += 1;
|
|
524
576
|
}
|
|
525
577
|
cached.contributions.length = messages.length;
|
|
526
578
|
cached.count = messages.length;
|
|
527
579
|
return contextSummaryResult(cached.state, messages.length);
|
|
528
580
|
}
|
|
529
581
|
|
|
582
|
+
// A stable warm-cache generation for consumers that cache a derived view of
|
|
583
|
+
// the whole transcript. summarizeContextMessages() must run first so mutations
|
|
584
|
+
// to any entry, not merely the tail, advance the generation.
|
|
585
|
+
export function contextMessagesRevision(messages) {
|
|
586
|
+
if (!Array.isArray(messages)) return 0;
|
|
587
|
+
summarizeContextMessages(messages);
|
|
588
|
+
return contextTranscriptMemo.get(messages)?.revision || 0;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// Hash only estimator/provider-visible projections. In particular, images
|
|
592
|
+
// contribute their visual count but never their raw data-url/base64 bytes.
|
|
593
|
+
export function contextMessagesSignature(messages, count = messages?.length) {
|
|
594
|
+
const list = Array.isArray(messages) ? messages : [];
|
|
595
|
+
const end = Math.max(0, Math.min(list.length, Number.isInteger(count) ? count : list.length));
|
|
596
|
+
const hash = createHash('sha256');
|
|
597
|
+
for (let index = 0; index < end; index += 1) {
|
|
598
|
+
const message = list[index];
|
|
599
|
+
hash.update(JSON.stringify([
|
|
600
|
+
message?.role || '',
|
|
601
|
+
message?.toolCallId || '',
|
|
602
|
+
messageEstimateText(message),
|
|
603
|
+
messageImageAllowance(message),
|
|
604
|
+
messageImageDescriptors(message),
|
|
605
|
+
]));
|
|
606
|
+
hash.update('\0');
|
|
607
|
+
}
|
|
608
|
+
return hash.digest('hex');
|
|
609
|
+
}
|
|
610
|
+
|
|
530
611
|
// Per-request overhead the provider injects that never appears in the
|
|
531
612
|
// `messages` array: function-calling preamble + system-prompt framing the
|
|
532
613
|
// provider wraps around the request. The chars/4 message estimate misses all
|
|
@@ -535,35 +616,13 @@ const REQUEST_OVERHEAD_TOKENS = 512;
|
|
|
535
616
|
const toolSchemaTokenMemo = new WeakMap();
|
|
536
617
|
const requestReserveTokenMemo = new WeakMap();
|
|
537
618
|
|
|
538
|
-
function
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
const entry = cached.entries[index];
|
|
542
|
-
const tool = tools[index];
|
|
543
|
-
if (entry.tool !== tool
|
|
544
|
-
|| entry.name !== tool?.name
|
|
545
|
-
|| entry.description !== tool?.description
|
|
546
|
-
|| entry.inputSchema !== tool?.inputSchema
|
|
547
|
-
|| entry.input_schema !== tool?.input_schema
|
|
548
|
-
|| entry.parameters !== tool?.parameters
|
|
549
|
-
|| entry.schema !== tool?.schema) return false;
|
|
550
|
-
}
|
|
551
|
-
return true;
|
|
619
|
+
function serializeToolSchemas(tools) {
|
|
620
|
+
try { return JSON.stringify(Array.isArray(tools) ? tools : []); }
|
|
621
|
+
catch { return (Array.isArray(tools) ? tools : []).map(t => String(t?.name ?? '')).join(''); }
|
|
552
622
|
}
|
|
553
623
|
|
|
554
|
-
function
|
|
555
|
-
return
|
|
556
|
-
entries: tools.map((tool) => ({
|
|
557
|
-
tool,
|
|
558
|
-
name: tool?.name,
|
|
559
|
-
description: tool?.description,
|
|
560
|
-
inputSchema: tool?.inputSchema,
|
|
561
|
-
input_schema: tool?.input_schema,
|
|
562
|
-
parameters: tool?.parameters,
|
|
563
|
-
schema: tool?.schema,
|
|
564
|
-
})),
|
|
565
|
-
value,
|
|
566
|
-
};
|
|
624
|
+
export function toolSchemaSignature(tools) {
|
|
625
|
+
return createHash('sha256').update(serializeToolSchemas(tools)).digest('hex');
|
|
567
626
|
}
|
|
568
627
|
|
|
569
628
|
/**
|
|
@@ -576,13 +635,12 @@ function cacheToolArrayValue(tools, value) {
|
|
|
576
635
|
*/
|
|
577
636
|
export function estimateToolSchemaTokens(tools) {
|
|
578
637
|
if (!Array.isArray(tools) || tools.length === 0) return 0;
|
|
638
|
+
const signature = toolSchemaSignature(tools);
|
|
579
639
|
const cached = toolSchemaTokenMemo.get(tools);
|
|
580
|
-
if (
|
|
581
|
-
|
|
582
|
-
try { text = JSON.stringify(tools); }
|
|
583
|
-
catch { text = tools.map(t => String(t?.name ?? '')).join(''); }
|
|
640
|
+
if (cached?.signature === signature) return cached.value;
|
|
641
|
+
const text = serializeToolSchemas(tools);
|
|
584
642
|
const tokens = estimateTokens(text);
|
|
585
|
-
toolSchemaTokenMemo.set(tools,
|
|
643
|
+
toolSchemaTokenMemo.set(tools, { signature, value: tokens });
|
|
586
644
|
return tokens;
|
|
587
645
|
}
|
|
588
646
|
|
|
@@ -594,10 +652,11 @@ export function estimateToolSchemaTokens(tools) {
|
|
|
594
652
|
*/
|
|
595
653
|
export function estimateRequestReserveTokens(tools) {
|
|
596
654
|
if (!Array.isArray(tools)) return estimateToolSchemaTokens(tools) + REQUEST_OVERHEAD_TOKENS;
|
|
655
|
+
const signature = toolSchemaSignature(tools);
|
|
597
656
|
const cached = requestReserveTokenMemo.get(tools);
|
|
598
|
-
if (
|
|
657
|
+
if (cached?.signature === signature) return cached.value;
|
|
599
658
|
const reserve = estimateToolSchemaTokens(tools) + REQUEST_OVERHEAD_TOKENS;
|
|
600
|
-
requestReserveTokenMemo.set(tools,
|
|
659
|
+
requestReserveTokenMemo.set(tools, { signature, value: reserve });
|
|
601
660
|
return reserve;
|
|
602
661
|
}
|
|
603
662
|
|