dsh-context-compression-improved 0.4.0-beta.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.ja.md +68 -36
- package/CHANGELOG.ko.md +67 -35
- package/CHANGELOG.md +195 -134
- package/CHANGELOG.zh.md +64 -36
- package/README.ja.md +1 -1
- package/README.ko.md +1 -1
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/docs/installation.ja.md +2 -2
- package/docs/installation.ko.md +2 -2
- package/docs/installation.md +103 -78
- package/docs/installation.zh.md +100 -77
- package/docs/repair-log.md +54 -0
- package/package.json +1 -1
- package/packages/selector/lib/{config.js → advisor-state.js} +329 -5
- package/packages/selector/lib/client.d.ts +7 -0
- package/packages/selector/lib/client.js +33 -3
- package/packages/selector/lib/index.d.ts +7 -0
- package/packages/selector/lib/index.js +112 -3
- package/packages/selector/lib/pruner.d.ts +128 -1
- package/packages/selector/lib/pruner.js +2802 -1374
- package/packages/selector/src/client/ReviewOverlay.tsx +1 -1
- package/packages/selector/src/client/index.ts +1 -1
- package/packages/selector/src/client/preset-options.ts +2 -0
- package/packages/selector/src/index.ts +129 -49
- package/packages/selector/src/profiles.ts +48 -0
- package/packages/selector/src/pruner/content.ts +18 -5
- package/packages/selector/src/pruner/state.ts +3 -0
- package/packages/selector/src/pruner/types.ts +23 -5
- package/packages/selector/src/pruner.ts +297 -162
- package/packages/selector/src/runtime/adaptive-cost.ts +23 -12
- package/packages/selector/src/runtime/audit.ts +40 -2
- package/packages/selector/src/runtime/config.ts +88 -1
- package/packages/selector/src/runtime/measurement.ts +31 -2
- package/packages/selector/src/runtime/reducers.ts +1115 -97
- package/packages/selector/src/runtime/tokenpilot/advisor-prompt.ts +188 -0
- package/packages/selector/src/runtime/tokenpilot/advisor-state.ts +133 -0
- package/packages/selector/src/runtime/tokenpilot/advisor.ts +419 -0
- package/packages/selector/src/runtime/tokenpilot/dedup.ts +1 -1
- package/packages/selector/src/runtime/tokenpilot/estimator.ts +8 -118
- package/packages/selector/src/runtime/tokenpilot/locator.ts +1 -1
- package/packages/selector/src/runtime/tokenpilot/proposal.ts +76 -32
- package/packages/selector/src/runtime/tokenpilot/read-state.ts +23 -2
- package/packages/selector/src/runtime/tokenpilot/review-registry.ts +117 -0
- package/packages/selector/src/runtime/tokenpilot/sidechannel.ts +303 -0
- package/packages/selector/src/runtime/toolclass.ts +103 -0
- package/packages/selector/src/runtime/types.ts +37 -0
- package/packages/selector/tests/advisor-report.host.spec.ts +223 -0
- package/packages/selector/tests/public/package-contract.client.spec.ts +2 -1
- package/packages/selector/tests/review-routes-registry.host.spec.ts +142 -0
- package/packages/selector/tests/runtime/adaptive-cost.spec.ts +7 -7
- package/packages/selector/tests/runtime/advisor-invariant.spec.ts +272 -0
- package/packages/selector/tests/runtime/advisor.spec.ts +226 -0
- package/packages/selector/tests/runtime/audit.spec.ts +88 -1
- package/packages/selector/tests/runtime/char-basis.spec.ts +30 -0
- package/packages/selector/tests/runtime/code-skeleton.spec.ts +14 -3
- package/packages/selector/tests/runtime/frequency-longstrings.spec.ts +74 -0
- package/packages/selector/tests/runtime/html-reducer.spec.ts +212 -0
- package/packages/selector/tests/runtime/line-mapping.spec.ts +153 -0
- package/packages/selector/tests/runtime/prose-reducers.spec.ts +133 -0
- package/packages/selector/tests/runtime/public/public-runtime.spec.ts +198 -27
- package/packages/selector/tests/runtime/read-input-cap.spec.ts +33 -0
- package/packages/selector/tests/runtime/search-reducer.spec.ts +110 -0
- package/packages/selector/tests/runtime/sidechannel.spec.ts +241 -0
- package/packages/selector/tests/runtime/toc-and-bundled.spec.ts +159 -0
- package/packages/selector/tests/runtime/tokenpilot/profile-baseline.spec.ts +12 -0
- package/packages/selector/tests/runtime/tokenpilot/proposal.spec.ts +194 -0
- package/packages/selector/tests/runtime/tokenpilot/pruner-review.spec.ts +70 -1
- package/packages/selector/tests/runtime/tokenpilot/read-state.spec.ts +24 -0
- package/packages/selector/tests/runtime/toolclass.spec.ts +156 -0
- package/scripts/toolclass-corpus-replay.mjs +281 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { C as DEFAULT_CUSTOM_COMPRESSION_POLICY, D as COMPRESSION_PROFILES, E as deepFreeze, S as CustomCompressionPolicySchema, T as assertNever, _ as isCompressionProfile, a as registerReviewPruner, b as resolveConfig, c as ReviewQueue, d as ContextCompressionSettingsSchema, f as DEFAULTS, g as codePointLength, h as charsToTokens, i as recordScore, l as AUTO_COMPACT_THRESHOLD_LIMITS, m as charsForTokens, n as invalidateOnTaskChange, p as PRUNE_MARKER, r as recordRecertified, s as sharedReviewStore, t as getAdvisorState, u as CONTEXT_COMPRESSION_SETTINGS_NAMESPACE, v as isValidAutoCompactThresholdPercent, w as resolveCustomPolicy, x as resolvePolicy, y as parseContextCompressionSettings } from "./advisor-state.js";
|
|
2
2
|
import { a as validatePublishedTailTrim, i as tailTrimStub, n as tailTrimMessage, o as eventBySeq, r as tailTrimRef, s as sessionEvents, t as parseTailTrimRef } from "./tail-trim.js";
|
|
3
3
|
import z from "@deepseek-ai/schemastery";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
@@ -355,8 +355,135 @@ function countExactCanonicalTextFields(fields, counter, subject) {
|
|
|
355
355
|
tokens
|
|
356
356
|
});
|
|
357
357
|
}
|
|
358
|
+
/** Count the lines present in the original but absent from the replacement. */
|
|
359
|
+
function countOmittedLines(original, replacement) {
|
|
360
|
+
const omitted = original.split("\n").length - replacement.split("\n").length;
|
|
361
|
+
return omitted > 0 ? omitted : void 0;
|
|
362
|
+
}
|
|
363
|
+
/** Routed-context utilization required before capacity-pressure History may age sent history. */
|
|
364
|
+
const CAPACITY_PRESSURE_RATIO = .7;
|
|
365
|
+
//#endregion
|
|
366
|
+
//#region src/pruner/content.ts
|
|
367
|
+
function onlyTextBlock(blocks) {
|
|
368
|
+
return blocks.length === 1 && blocks[0]?.type === "text" ? blocks[0] : null;
|
|
369
|
+
}
|
|
370
|
+
function onlyTextBlocks(blocks) {
|
|
371
|
+
return blocks.every((block) => block.type === "text") ? blocks : null;
|
|
372
|
+
}
|
|
373
|
+
function countToolContent(blocks, view) {
|
|
374
|
+
const text = onlyTextBlocks(blocks);
|
|
375
|
+
if (text === null) return unavailableCount("tool result contains unsupported rich content");
|
|
376
|
+
return countExactCanonicalTextFields(text.map((block) => block.text), (candidate) => view.countCanonicalText(candidate), "tool result replacement");
|
|
377
|
+
}
|
|
378
|
+
function sameProviderMeasurementKey(left, right) {
|
|
379
|
+
return left.provider === right.provider && left.baseUrlClass === right.baseUrlClass && left.apiRoute === right.apiRoute && left.modelId === right.modelId && left.requestTemplateRevision === right.requestTemplateRevision && left.tokenizerRevision === right.tokenizerRevision && left.modality === right.modality;
|
|
380
|
+
}
|
|
381
|
+
function unavailableCount(reason) {
|
|
382
|
+
return Object.freeze({
|
|
383
|
+
kind: "unavailable",
|
|
384
|
+
reason
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
function recoveryMarker(sourceRef, label, startLine) {
|
|
388
|
+
return `\n\n[... ${label}; source=${sourceRef}; ${startLine === void 0 ? "use context_compression_retrieve if needed" : `retrieve with context_compression_retrieve({"ref":"${sourceRef}","start_line":${String(startLine)},"max_lines":80})`} ...]\n\n`;
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Measure text content in Unicode code points; non-text blocks cost zero.
|
|
392
|
+
* @param blocks - tool-result content to measure.
|
|
393
|
+
* @returns total Unicode code points across text blocks.
|
|
394
|
+
*/
|
|
395
|
+
function measureContent(blocks) {
|
|
396
|
+
let chars = 0;
|
|
397
|
+
for (const block of blocks) if (block.type === "text") chars += codePointLength(block.text);
|
|
398
|
+
return chars;
|
|
399
|
+
}
|
|
400
|
+
function pressureCost(blocks) {
|
|
401
|
+
let cost = 0;
|
|
402
|
+
for (const block of blocks) switch (block.type) {
|
|
403
|
+
case "text":
|
|
404
|
+
case "reasoning":
|
|
405
|
+
cost += codePointLength(block.text);
|
|
406
|
+
break;
|
|
407
|
+
case "tool-call":
|
|
408
|
+
cost += 256 + codePointLength(block.name) + codePointLength(block.arguments);
|
|
409
|
+
break;
|
|
410
|
+
case "tool-result":
|
|
411
|
+
cost += 256 + pressureCost(block.content);
|
|
412
|
+
break;
|
|
413
|
+
default: {
|
|
414
|
+
const serialized = JSON.stringify(block);
|
|
415
|
+
cost += Math.max(256, codePointLength(serialized));
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
return cost;
|
|
419
|
+
}
|
|
420
|
+
function nativePruneContent(blocks, thresholdChars, headChars, tailChars, marker = PRUNE_MARKER) {
|
|
421
|
+
const totalChars = measureContent(blocks);
|
|
422
|
+
if (totalChars <= thresholdChars) return null;
|
|
423
|
+
const markerChars = codePointLength(typeof marker === "function" ? marker(1) : marker);
|
|
424
|
+
const safeHead = Math.max(0, Math.min(headChars, thresholdChars - markerChars));
|
|
425
|
+
const safeTail = Math.max(0, Math.min(tailChars, thresholdChars - markerChars - safeHead));
|
|
426
|
+
const removedStart = safeHead;
|
|
427
|
+
const removedEnd = totalChars - safeTail;
|
|
428
|
+
const pruned = [];
|
|
429
|
+
let consumed = 0;
|
|
430
|
+
let markerInserted = false;
|
|
431
|
+
let newlinesBefore = 0;
|
|
432
|
+
for (const block of blocks) {
|
|
433
|
+
if (block.type !== "text") {
|
|
434
|
+
pruned.push(block);
|
|
435
|
+
newlinesBefore += 1;
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
const points = Array.from(block.text);
|
|
439
|
+
const blockStart = consumed;
|
|
440
|
+
const blockEnd = blockStart + points.length;
|
|
441
|
+
const headEnd = Math.min(points.length, Math.max(0, removedStart - blockStart));
|
|
442
|
+
const tailStart = Math.min(points.length, Math.max(0, removedEnd - blockStart));
|
|
443
|
+
const intersectsRemoved = blockStart < removedEnd && blockEnd > removedStart;
|
|
444
|
+
const headText = points.slice(0, headEnd).join("");
|
|
445
|
+
const insertion = intersectsRemoved && !markerInserted && typeof marker === "function" ? marker(1 + newlinesBefore + headText.split("\n").length - 1) : intersectsRemoved && !markerInserted ? marker : "";
|
|
446
|
+
if (insertion !== "") markerInserted = true;
|
|
447
|
+
const text = points.slice(0, headEnd).join("") + insertion + points.slice(tailStart).join("");
|
|
448
|
+
if (text !== "") pruned.push({
|
|
449
|
+
...block,
|
|
450
|
+
text
|
|
451
|
+
});
|
|
452
|
+
newlinesBefore += block.text.split("\n").length - 1 + 1;
|
|
453
|
+
consumed = blockEnd;
|
|
454
|
+
}
|
|
455
|
+
if (!markerInserted) return null;
|
|
456
|
+
const charsAfter = measureContent(pruned);
|
|
457
|
+
return charsAfter <= thresholdChars && charsAfter < totalChars ? pruned : null;
|
|
458
|
+
}
|
|
459
|
+
function summarize(entries) {
|
|
460
|
+
return {
|
|
461
|
+
pruned: entries,
|
|
462
|
+
charsRemoved: entries.reduce((sum, entry) => sum + entry.charsBefore - entry.charsAfter, 0),
|
|
463
|
+
tokensRemoved: entries.reduce((sum, entry) => sum + entry.tokensBefore - entry.tokensAfter, 0)
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
function emptyResult() {
|
|
467
|
+
return {
|
|
468
|
+
pruned: [],
|
|
469
|
+
charsRemoved: 0,
|
|
470
|
+
tokensRemoved: 0
|
|
471
|
+
};
|
|
472
|
+
}
|
|
358
473
|
//#endregion
|
|
359
474
|
//#region src/runtime/measurement.ts
|
|
475
|
+
/**
|
|
476
|
+
* Character pressure of one model-visible content array, measured on the same
|
|
477
|
+
* level `SnapshotCandidate.characterPressure` uses: a tool/result message is
|
|
478
|
+
* measured by its inner tool-result content, never by the wrapper block, so the
|
|
479
|
+
* node-level and candidate-level figures stay directly comparable.
|
|
480
|
+
* @param content - the node's model-visible content blocks.
|
|
481
|
+
* @returns character pressure in Unicode code points, plus rich-block costs.
|
|
482
|
+
*/
|
|
483
|
+
function nodeCharacterPressure(content) {
|
|
484
|
+
const only = content.length === 1 ? content[0] : void 0;
|
|
485
|
+
return pressureCost(only?.type === "tool-result" ? only.content : content);
|
|
486
|
+
}
|
|
360
487
|
const VISION_MODEL_ID = DEEPSEEK_VISION_TOKENIZER_ARTIFACT.modelIds[0];
|
|
361
488
|
/**
|
|
362
489
|
* Capture one route-bound view without calling patched Harness methods.
|
|
@@ -374,22 +501,26 @@ function measureForCompaction(ctx, session) {
|
|
|
374
501
|
const event = eventsBySeq.get(Number(node.seq));
|
|
375
502
|
if (event === void 0) return {
|
|
376
503
|
seq: node.seq,
|
|
377
|
-
count: unavailableTokenCount(`surface node ${String(node.seq)} is missing`)
|
|
504
|
+
count: unavailableTokenCount(`surface node ${String(node.seq)} is missing`),
|
|
505
|
+
characterPressure: 0
|
|
378
506
|
};
|
|
379
507
|
const message = deriveEventMessage(event);
|
|
380
508
|
if (message === null) return {
|
|
381
509
|
seq: node.seq,
|
|
382
|
-
count: unavailableTokenCount(`surface node ${String(node.seq)} is not model-visible`)
|
|
510
|
+
count: unavailableTokenCount(`surface node ${String(node.seq)} is not model-visible`),
|
|
511
|
+
characterPressure: 0
|
|
383
512
|
};
|
|
384
513
|
const count = countCanonicalContent(message.content, counter, `surface node ${String(node.seq)}`);
|
|
385
514
|
const intrinsicImageBlockEstimate = count.kind === "tokenizer-estimate" ? intrinsicImageDiagnostic(message.content, target) : void 0;
|
|
386
515
|
return {
|
|
387
516
|
seq: node.seq,
|
|
388
517
|
count,
|
|
518
|
+
characterPressure: nodeCharacterPressure(message.content),
|
|
389
519
|
...intrinsicImageBlockEstimate === void 0 ? {} : { intrinsicImageBlockEstimate }
|
|
390
520
|
};
|
|
391
521
|
});
|
|
392
522
|
const currentSurface = countSurfaceCounts(measuredNodes.map((node) => node.count), "current surface");
|
|
523
|
+
const currentSurfaceChars = measuredNodes.reduce((sum, node) => sum + node.characterPressure, 0);
|
|
393
524
|
const intrinsicImageBlockEstimateTokens = measuredNodes.reduce((sum, node) => sum + (node.intrinsicImageBlockEstimate?.paddingMinimumTokens ?? 0), 0);
|
|
394
525
|
return Object.freeze({
|
|
395
526
|
...measurement,
|
|
@@ -399,6 +530,7 @@ function measureForCompaction(ctx, session) {
|
|
|
399
530
|
},
|
|
400
531
|
measuredNodes: Object.freeze(measuredNodes),
|
|
401
532
|
currentSurface,
|
|
533
|
+
currentSurfaceChars,
|
|
402
534
|
intrinsicImageBlockEstimateTokens,
|
|
403
535
|
countCanonicalText: counter.countText
|
|
404
536
|
});
|
|
@@ -922,119 +1054,6 @@ function codePointLength$1(text) {
|
|
|
922
1054
|
for (const _point of text) count++;
|
|
923
1055
|
return count;
|
|
924
1056
|
}
|
|
925
|
-
/** Count the lines present in the original but absent from the replacement. */
|
|
926
|
-
function countOmittedLines(original, replacement) {
|
|
927
|
-
const omitted = original.split("\n").length - replacement.split("\n").length;
|
|
928
|
-
return omitted > 0 ? omitted : void 0;
|
|
929
|
-
}
|
|
930
|
-
/** Routed-context utilization required before capacity-pressure History may age sent history. */
|
|
931
|
-
const CAPACITY_PRESSURE_RATIO = .7;
|
|
932
|
-
//#endregion
|
|
933
|
-
//#region src/pruner/content.ts
|
|
934
|
-
function onlyTextBlock(blocks) {
|
|
935
|
-
return blocks.length === 1 && blocks[0]?.type === "text" ? blocks[0] : null;
|
|
936
|
-
}
|
|
937
|
-
function onlyTextBlocks(blocks) {
|
|
938
|
-
return blocks.every((block) => block.type === "text") ? blocks : null;
|
|
939
|
-
}
|
|
940
|
-
function countToolContent(blocks, view) {
|
|
941
|
-
const text = onlyTextBlocks(blocks);
|
|
942
|
-
if (text === null) return unavailableCount("tool result contains unsupported rich content");
|
|
943
|
-
return countExactCanonicalTextFields(text.map((block) => block.text), (candidate) => view.countCanonicalText(candidate), "tool result replacement");
|
|
944
|
-
}
|
|
945
|
-
function exactTokens(count) {
|
|
946
|
-
return count.kind === "exact-tokenizer" ? count.tokens : void 0;
|
|
947
|
-
}
|
|
948
|
-
function sameProviderMeasurementKey(left, right) {
|
|
949
|
-
return left.provider === right.provider && left.baseUrlClass === right.baseUrlClass && left.apiRoute === right.apiRoute && left.modelId === right.modelId && left.requestTemplateRevision === right.requestTemplateRevision && left.tokenizerRevision === right.tokenizerRevision && left.modality === right.modality;
|
|
950
|
-
}
|
|
951
|
-
function unavailableCount(reason) {
|
|
952
|
-
return Object.freeze({
|
|
953
|
-
kind: "unavailable",
|
|
954
|
-
reason
|
|
955
|
-
});
|
|
956
|
-
}
|
|
957
|
-
function recoveryMarker(sourceRef, label) {
|
|
958
|
-
return `\n\n[... ${label}; source=${sourceRef}; use context_compression_retrieve if needed ...]\n\n`;
|
|
959
|
-
}
|
|
960
|
-
/**
|
|
961
|
-
* Measure text content in Unicode code points; non-text blocks cost zero.
|
|
962
|
-
* @param blocks - tool-result content to measure.
|
|
963
|
-
* @returns total Unicode code points across text blocks.
|
|
964
|
-
*/
|
|
965
|
-
function measureContent(blocks) {
|
|
966
|
-
let chars = 0;
|
|
967
|
-
for (const block of blocks) if (block.type === "text") chars += codePointLength(block.text);
|
|
968
|
-
return chars;
|
|
969
|
-
}
|
|
970
|
-
function pressureCost(blocks) {
|
|
971
|
-
let cost = 0;
|
|
972
|
-
for (const block of blocks) switch (block.type) {
|
|
973
|
-
case "text":
|
|
974
|
-
case "reasoning":
|
|
975
|
-
cost += codePointLength(block.text);
|
|
976
|
-
break;
|
|
977
|
-
case "tool-call":
|
|
978
|
-
cost += 256 + codePointLength(block.name) + codePointLength(block.arguments);
|
|
979
|
-
break;
|
|
980
|
-
case "tool-result":
|
|
981
|
-
cost += 256 + pressureCost(block.content);
|
|
982
|
-
break;
|
|
983
|
-
default: {
|
|
984
|
-
const serialized = JSON.stringify(block);
|
|
985
|
-
cost += Math.max(256, codePointLength(serialized));
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
return cost;
|
|
989
|
-
}
|
|
990
|
-
function nativePruneContent(blocks, thresholdChars, headChars, tailChars, marker = PRUNE_MARKER) {
|
|
991
|
-
const totalChars = measureContent(blocks);
|
|
992
|
-
if (totalChars <= thresholdChars) return null;
|
|
993
|
-
const markerChars = codePointLength(marker);
|
|
994
|
-
const safeHead = Math.max(0, Math.min(headChars, thresholdChars - markerChars));
|
|
995
|
-
const safeTail = Math.max(0, Math.min(tailChars, thresholdChars - markerChars - safeHead));
|
|
996
|
-
const removedStart = safeHead;
|
|
997
|
-
const removedEnd = totalChars - safeTail;
|
|
998
|
-
const pruned = [];
|
|
999
|
-
let consumed = 0;
|
|
1000
|
-
let markerInserted = false;
|
|
1001
|
-
for (const block of blocks) {
|
|
1002
|
-
if (block.type !== "text") {
|
|
1003
|
-
pruned.push(block);
|
|
1004
|
-
continue;
|
|
1005
|
-
}
|
|
1006
|
-
const points = Array.from(block.text);
|
|
1007
|
-
const blockStart = consumed;
|
|
1008
|
-
const blockEnd = blockStart + points.length;
|
|
1009
|
-
const headEnd = Math.min(points.length, Math.max(0, removedStart - blockStart));
|
|
1010
|
-
const tailStart = Math.min(points.length, Math.max(0, removedEnd - blockStart));
|
|
1011
|
-
const insertion = blockStart < removedEnd && blockEnd > removedStart && !markerInserted ? marker : "";
|
|
1012
|
-
if (insertion !== "") markerInserted = true;
|
|
1013
|
-
const text = points.slice(0, headEnd).join("") + insertion + points.slice(tailStart).join("");
|
|
1014
|
-
if (text !== "") pruned.push({
|
|
1015
|
-
...block,
|
|
1016
|
-
text
|
|
1017
|
-
});
|
|
1018
|
-
consumed = blockEnd;
|
|
1019
|
-
}
|
|
1020
|
-
if (!markerInserted) return null;
|
|
1021
|
-
const charsAfter = measureContent(pruned);
|
|
1022
|
-
return charsAfter <= thresholdChars && charsAfter < totalChars ? pruned : null;
|
|
1023
|
-
}
|
|
1024
|
-
function summarize(entries) {
|
|
1025
|
-
return {
|
|
1026
|
-
pruned: entries,
|
|
1027
|
-
charsRemoved: entries.reduce((sum, entry) => sum + entry.charsBefore - entry.charsAfter, 0),
|
|
1028
|
-
tokensRemoved: entries.reduce((sum, entry) => sum + entry.tokensBefore - entry.tokensAfter, 0)
|
|
1029
|
-
};
|
|
1030
|
-
}
|
|
1031
|
-
function emptyResult() {
|
|
1032
|
-
return {
|
|
1033
|
-
pruned: [],
|
|
1034
|
-
charsRemoved: 0,
|
|
1035
|
-
tokensRemoved: 0
|
|
1036
|
-
};
|
|
1037
|
-
}
|
|
1038
1057
|
//#endregion
|
|
1039
1058
|
//#region src/pruner/session.ts
|
|
1040
1059
|
/** Check whether the session currently has an open (unterminated) turn. */
|
|
@@ -1177,1274 +1196,2590 @@ function buildLocatorBlock(events, shadowedRange) {
|
|
|
1177
1196
|
`- seq range: ${String(shadowedRange.start)}-${String(shadowedRange.end)}`,
|
|
1178
1197
|
...[...spillFiles].map((path) => `- spill file: ${path}`),
|
|
1179
1198
|
...[...touchedFiles].map((path) => `- file touched: ${path}`),
|
|
1180
|
-
"(Use `read <spill file>` or `context_compression_retrieve` with a `session://` source to restore exact text.)"
|
|
1199
|
+
"(Use `read <spill file>` or `context_compression_retrieve` with a `session://` source — pass start_line/max_lines to window the text — to restore exact text.)"
|
|
1181
1200
|
].join("\n"),
|
|
1182
1201
|
spillFiles: spillFiles.size,
|
|
1183
1202
|
touchedFiles: touchedFiles.size
|
|
1184
1203
|
};
|
|
1185
1204
|
}
|
|
1186
1205
|
//#endregion
|
|
1187
|
-
//#region src/runtime/
|
|
1188
|
-
/**
|
|
1189
|
-
const
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
+
//#region src/runtime/toolclass.ts
|
|
1207
|
+
/** task_11 (AD3): exact-token matching, no substring hits. */
|
|
1208
|
+
const TOKEN_SEPARATOR = /[-_/]+/;
|
|
1209
|
+
/** Multi-token names that must match as a whole word, not as a token. */
|
|
1210
|
+
const READ_NAMES = /* @__PURE__ */ new Set(["open_file"]);
|
|
1211
|
+
const READ_TOKENS = /* @__PURE__ */ new Set([
|
|
1212
|
+
"read",
|
|
1213
|
+
"cat",
|
|
1214
|
+
"view"
|
|
1215
|
+
]);
|
|
1216
|
+
const SHELL_TOKENS = /* @__PURE__ */ new Set([
|
|
1217
|
+
"shell",
|
|
1218
|
+
"bash",
|
|
1219
|
+
"pwsh",
|
|
1220
|
+
"powershell",
|
|
1221
|
+
"terminal",
|
|
1222
|
+
"exec",
|
|
1223
|
+
"command"
|
|
1224
|
+
]);
|
|
1225
|
+
const SEARCH_TOKENS = /* @__PURE__ */ new Set([
|
|
1226
|
+
"grep",
|
|
1227
|
+
"rg",
|
|
1228
|
+
"ripgrep"
|
|
1229
|
+
]);
|
|
1230
|
+
const PATH_LISTING_TOKENS = /* @__PURE__ */ new Set([
|
|
1231
|
+
"glob",
|
|
1232
|
+
"tree",
|
|
1233
|
+
"ls",
|
|
1234
|
+
"fd",
|
|
1235
|
+
"find"
|
|
1236
|
+
]);
|
|
1237
|
+
const SEARCH_COMMAND_PATTERN = /(?:^|\s)(?:rg|grep|ripgrep)\s/;
|
|
1238
|
+
const PATH_COMMAND_PATTERN = /(?:^|\s)(?:find|fd|ls|tree)\s/;
|
|
1239
|
+
/** grep-style hit line: `path:line[:column][: content]`. */
|
|
1240
|
+
const PATH_LINE_CONTENT_PATTERN = /^(.*?):(\d+)(?::\d+)?(?::|\s+-\s+)(.*)$/;
|
|
1206
1241
|
/**
|
|
1207
|
-
*
|
|
1208
|
-
*
|
|
1209
|
-
*
|
|
1242
|
+
* Classify a tool result's source.
|
|
1243
|
+
* @param name - raw tool name (case-insensitive).
|
|
1244
|
+
* @param command - extracted command argument, '' when absent.
|
|
1245
|
+
* @param text - result text; enables the C13/C14 content fallback for names
|
|
1246
|
+
* that no whitelist (and no command hit) claims.
|
|
1247
|
+
* @returns the tool source class.
|
|
1210
1248
|
*/
|
|
1211
|
-
function
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
return
|
|
1249
|
+
function classifyToolSource(name, command, text) {
|
|
1250
|
+
const lowered = name.toLowerCase();
|
|
1251
|
+
const tokens = lowered.split(TOKEN_SEPARATOR).filter((token) => token !== "");
|
|
1252
|
+
if (tokens.includes("mcp")) return "generic";
|
|
1253
|
+
const byName = classifyByName(lowered, tokens);
|
|
1254
|
+
if (byName !== "generic") return byName;
|
|
1255
|
+
const byCommand = classifyByCommand(command);
|
|
1256
|
+
if (byCommand !== "generic") return byCommand;
|
|
1257
|
+
return text === void 0 ? "generic" : classifyByContent(text);
|
|
1258
|
+
}
|
|
1259
|
+
function classifyByName(lowered, tokens) {
|
|
1260
|
+
if (READ_NAMES.has(lowered) || tokens.some((token) => READ_TOKENS.has(token))) return "read";
|
|
1261
|
+
if (tokens.some((token) => SHELL_TOKENS.has(token))) return "shell";
|
|
1262
|
+
if (tokens.some((token) => SEARCH_TOKENS.has(token))) return "search";
|
|
1263
|
+
if (tokens.some((token) => PATH_LISTING_TOKENS.has(token))) return "path-listing";
|
|
1264
|
+
return "generic";
|
|
1265
|
+
}
|
|
1266
|
+
function classifyByCommand(command) {
|
|
1267
|
+
if (command === "") return "generic";
|
|
1268
|
+
if (SEARCH_COMMAND_PATTERN.test(command)) return "search";
|
|
1269
|
+
if (PATH_COMMAND_PATTERN.test(command)) return "path-listing";
|
|
1270
|
+
return "generic";
|
|
1271
|
+
}
|
|
1272
|
+
/** A path-shaped hit locator needs a path-looking prefix, not just `:digits`. */
|
|
1273
|
+
function looksLikeFilePath(prefix) {
|
|
1274
|
+
if (prefix.includes("://")) return false;
|
|
1275
|
+
return prefix.includes("/") || prefix.includes("\\") || /\.[A-Za-z0-9]{1,8}$/.test(prefix);
|
|
1276
|
+
}
|
|
1277
|
+
/** A bare path: no whitespace, no URL scheme, with a separator or an extension. */
|
|
1278
|
+
function isPurePathLine(line) {
|
|
1279
|
+
if (/\s/.test(line) || line.includes("://")) return false;
|
|
1280
|
+
return line.includes("/") || line.includes("\\") || /\.[A-Za-z0-9]{1,8}$/.test(line);
|
|
1220
1281
|
}
|
|
1221
|
-
/** Error/warning/info line classifiers used by the omission summary. */
|
|
1222
|
-
const ERROR_LINE = /\b(error|failed|failure|fatal|exception|traceback|cannot|unable|denied)\b/i;
|
|
1223
|
-
const WARN_LINE = /\b(warn|warning|deprecated)\b/i;
|
|
1224
1282
|
/**
|
|
1225
|
-
*
|
|
1226
|
-
*
|
|
1283
|
+
* Content fallback (C13/C14) for unknown names: one path-plausible
|
|
1284
|
+
* `path:line:content` line reads as grep output; a body of ≥80% pure paths
|
|
1285
|
+
* reads as a directory listing. Both require file-path evidence so prose
|
|
1286
|
+
* (`Note: 2024 - ...`) and URLs (`https://host:8080 - ...`) stay generic.
|
|
1227
1287
|
*/
|
|
1228
|
-
function
|
|
1229
|
-
|
|
1230
|
-
let
|
|
1231
|
-
let
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
if (
|
|
1240
|
-
if (
|
|
1241
|
-
return
|
|
1288
|
+
function classifyByContent(text) {
|
|
1289
|
+
let nonEmpty = 0;
|
|
1290
|
+
let purePaths = 0;
|
|
1291
|
+
let pathLineHits = 0;
|
|
1292
|
+
for (const line of text.split("\n")) {
|
|
1293
|
+
if (line.trim() === "") continue;
|
|
1294
|
+
nonEmpty += 1;
|
|
1295
|
+
const match = PATH_LINE_CONTENT_PATTERN.exec(line);
|
|
1296
|
+
if (match !== null && looksLikeFilePath(match[1] ?? "")) pathLineHits += 1;
|
|
1297
|
+
if (isPurePathLine(line)) purePaths += 1;
|
|
1298
|
+
}
|
|
1299
|
+
if (pathLineHits >= 1) return "search";
|
|
1300
|
+
if (nonEmpty >= 4 && purePaths / nonEmpty >= .8) return "path-listing";
|
|
1301
|
+
return "generic";
|
|
1242
1302
|
}
|
|
1243
1303
|
//#endregion
|
|
1244
|
-
//#region src/runtime/
|
|
1245
|
-
/**
|
|
1246
|
-
|
|
1247
|
-
|
|
1304
|
+
//#region src/runtime/reducers.ts
|
|
1305
|
+
/** Deterministic, evidence-backed reducers for fresh tool results. */
|
|
1306
|
+
/** Ranked-first ordering: ranked ids keep their rank, the rest append in order. */
|
|
1307
|
+
function rankedFirst(items, ranking) {
|
|
1308
|
+
if (ranking === void 0 || ranking.length === 0) return items;
|
|
1309
|
+
const ranked = /* @__PURE__ */ new Map();
|
|
1310
|
+
for (const id of ranking) {
|
|
1311
|
+
const found = items.find((item) => item.id === id);
|
|
1312
|
+
if (found !== void 0 && !ranked.has(id)) ranked.set(id, found);
|
|
1313
|
+
}
|
|
1314
|
+
return [...ranked.values(), ...items.filter((item) => !ranked.has(item.id))];
|
|
1248
1315
|
}
|
|
1249
|
-
|
|
1250
|
-
|
|
1316
|
+
const ANSI_PATTERN = /\u001B(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007]*(?:\u0007|\u001B\\))/gu;
|
|
1317
|
+
const IMPORTANT_PATTERN = new RegExp([
|
|
1318
|
+
String.raw`\b(?:error|failed|failure|fatal|panic|exception|warning|warn|conflict|denied|forbidden|`,
|
|
1319
|
+
String.raw`timeout|timed out|not found|cannot|unable|invalid|exit(?:ed)?\s+(?:code|status)|traceback|`,
|
|
1320
|
+
String.raw`assert(?:ion)?|segmentation fault|oom|out of memory)\b`
|
|
1321
|
+
].join(""), "i");
|
|
1322
|
+
const STATUS_PATTERN = new RegExp([String.raw`\b(?:success|succeeded|passed|installed|added|removed|updated|built|compiled|`, String.raw`tests?\s+(?:passed|failed)|exit(?:ed)?\s+(?:code|status))\b`].join(""), "i");
|
|
1323
|
+
const PATH_LINE_PATTERN = /^(.*?):(\d+)(?::\d+)?(?::|\s+-\s+)(.*)$/;
|
|
1324
|
+
const GIT_STATUS_PATTERN = new RegExp([String.raw`^(?:On branch|Your branch|HEAD detached|Changes |Untracked |Unmerged |\s*(?:modified|deleted|`, String.raw`new file|renamed|both modified):)`].join(""), "i");
|
|
1325
|
+
const CODE_IMPORT_PATTERN = new RegExp([String.raw`^\s*(?:import\b|from\s+[\w.]+\s+import\b|use\s+\w|package\s+|#include\b|`, String.raw`using\s+[\w.]+;|require\s*\(|extern\s+crate\b)`].join(""));
|
|
1326
|
+
const CODE_STRUCTURE_PATTERN = new RegExp([
|
|
1327
|
+
String.raw`^\s*(?:@[\w.]+|export\s+|default\s+|declare\s+|abstract\s+|public\s+|private\s+|protected\s+|`,
|
|
1328
|
+
String.raw`internal\s+|static\s+|final\s+|sealed\s+|override\s+|pub(?:\([^)]*\))?\s+|async\s+|unsafe\s+)*`,
|
|
1329
|
+
String.raw`(?:function\b|class\b|interface\b|enum\b|struct\b|impl\b|trait\b|type\s+\w|fn\s|func\b|`,
|
|
1330
|
+
String.raw`def\s|module\b|namespace\b|sub\s)`
|
|
1331
|
+
].join(""));
|
|
1332
|
+
const PYTHON_STRUCTURE_PATTERN = /^\s*(?:async\s+)?def\s|^\s*class\s/;
|
|
1333
|
+
const CODE_DECORATOR_PATTERN = /^\s*@[\w.]+/;
|
|
1334
|
+
const CODE_COMMENT_PATTERN = /^\s*(?:\/\/|#|\/\*|\*)/;
|
|
1335
|
+
const MARKDOWN_HEADING_PATTERN = /^#{1,6}\s+\S/;
|
|
1336
|
+
const LIST_ITEM_PATTERN = /^\s*(?:[-*+]|\d+[.)])\s+\S/;
|
|
1337
|
+
const TABLE_ROW_PATTERN = /^\s*\|/;
|
|
1338
|
+
const FENCE_PATTERN = /^\s*(?:```|~~~)/;
|
|
1339
|
+
const UUID_PATTERN = /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g;
|
|
1340
|
+
const LONG_HEX_PATTERN = /\b[0-9a-fA-F]{64,}\b/g;
|
|
1341
|
+
const LONG_BASE64_PATTERN = /[A-Za-z0-9+/]{200,}={0,2}/g;
|
|
1342
|
+
const HTML_TAG_PATTERN = /<!DOCTYPE html|<html\b|<head\b|<div\b|<span\b|<script\b|<style\b|<body\b|<p>|<table\b|<a\s/i;
|
|
1343
|
+
const HTML_COMMENT_PATTERN = /<!--[\s\S]*?-->/g;
|
|
1344
|
+
const HTML_DROPPED_ELEMENTS = /<(script|style|noscript|svg|head)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
|
|
1345
|
+
const HTML_DATA_URI_PATTERN = /\s(?:src|href)="data:[^"]*"/gi;
|
|
1346
|
+
const HTML_TAG_PATTERN_FULL = /<([a-z][a-z0-9]*)((?:\s[^<>]*?)?)\/?>/gi;
|
|
1347
|
+
const HTML_INLINE_TAG_PATTERN = /<\/?(?:em|strong|b|i|u|s|code|small|sub|sup|span|br)\b[^<>]*>/gi;
|
|
1348
|
+
const HTML_WHITELISTED_ATTRIBUTES = /\s(?:href|src|alt|title|id)="[^"]*"/gi;
|
|
1349
|
+
const ADJACENT_REPEAT_MARKER = "[previous line repeated";
|
|
1350
|
+
/** Non-adjacent folding only pays off once a line recurs enough to beat the marker cost. */
|
|
1351
|
+
const NON_ADJACENT_FOLD_THRESHOLD = 3;
|
|
1352
|
+
/** Read-output line-number gutter added unconditionally by the host's `formatReadOutput`. */
|
|
1353
|
+
const READ_GUTTER_PATTERN = /^(\d+): ?/;
|
|
1354
|
+
/**
|
|
1355
|
+
* Block-level read-gutter detection (GF-1). The host prefixes read output with
|
|
1356
|
+
* `N: ` line numbers unconditionally and cannot be configured off. The gutter
|
|
1357
|
+
* is only recognized when the block as a whole reads like a numbered listing —
|
|
1358
|
+
* enough non-empty lines, a large majority guttered, and the numbers strictly
|
|
1359
|
+
* increasing — so prose like `12:30 pm` (one stray gutter-looking line) is
|
|
1360
|
+
* never stripped. The stripping happens on the CONTENT view only; the output
|
|
1361
|
+
* view keeps the gutter because it is the model's only inline locator into the
|
|
1362
|
+
* original file (and its measured cost, 9.16% of read bodies, never gets
|
|
1363
|
+
* retrieved anyway).
|
|
1364
|
+
*/
|
|
1365
|
+
function hasReadGutter(lines) {
|
|
1366
|
+
let nonEmpty = 0;
|
|
1367
|
+
let guttered = 0;
|
|
1368
|
+
let previousNumber = 0;
|
|
1369
|
+
for (const line of lines) {
|
|
1370
|
+
if (line.trim() === "") continue;
|
|
1371
|
+
nonEmpty += 1;
|
|
1372
|
+
const match = READ_GUTTER_PATTERN.exec(line);
|
|
1373
|
+
if (match === null) continue;
|
|
1374
|
+
const number = Number(match[1]);
|
|
1375
|
+
if (number <= previousNumber) return false;
|
|
1376
|
+
previousNumber = number;
|
|
1377
|
+
guttered += 1;
|
|
1378
|
+
}
|
|
1379
|
+
return nonEmpty >= 4 && guttered / nonEmpty >= .75;
|
|
1251
1380
|
}
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1381
|
+
/**
|
|
1382
|
+
* Replace long opaque literals with length summaries (R12). Data URIs, base64
|
|
1383
|
+
* blobs, and long hex dumps are pure noise in a compressed view; the prefix is
|
|
1384
|
+
* kept so the model can still recognize the value. Short strings are never
|
|
1385
|
+
* touched, and replacements never span lines, so the line mapping survives.
|
|
1386
|
+
*/
|
|
1387
|
+
function placeholderizeLongStrings(line) {
|
|
1388
|
+
if (!/[0-9a-zA-Z+/]{32}/.test(line) && !/\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-/.test(line)) return line;
|
|
1389
|
+
let result = line.replace(UUID_PATTERN, "[uuid]");
|
|
1390
|
+
result = result.replace(LONG_HEX_PATTERN, (match) => `[hex ${String(match.length)} chars: ${match.slice(0, 16)}…]`);
|
|
1391
|
+
result = result.replace(LONG_BASE64_PATTERN, (match) => `[base64 ${String(match.length)} chars: ${match.slice(0, 16)}…]`);
|
|
1392
|
+
return result;
|
|
1263
1393
|
}
|
|
1264
|
-
|
|
1265
|
-
|
|
1394
|
+
/**
|
|
1395
|
+
* Select a reducer from verified tool, command, and content evidence.
|
|
1396
|
+
* @param input - original result text, recovery source, and output budget.
|
|
1397
|
+
* @returns a verified candidate, or `null` when every reducer fails open.
|
|
1398
|
+
*/
|
|
1399
|
+
function reduceFreshToolResult(input, ranking) {
|
|
1400
|
+
const normalized = normalizeTerminalLines(input.text);
|
|
1401
|
+
const prepared = {
|
|
1402
|
+
...input,
|
|
1403
|
+
text: normalized.text,
|
|
1404
|
+
lines: normalized.folded,
|
|
1405
|
+
contentText: normalized.contentText
|
|
1406
|
+
};
|
|
1407
|
+
const command = extractCommand(input.argumentsText);
|
|
1408
|
+
const name = input.toolName.toLowerCase();
|
|
1409
|
+
const toolClass = classifyToolSource(input.toolName, command, normalized.contentText);
|
|
1410
|
+
const readTocFirst = toolClass === "read" && codePointLength(normalized.contentText) >= READ_TOC_MIN_CHARS;
|
|
1411
|
+
const candidates = [];
|
|
1412
|
+
if (looksLikeJson(normalized.contentText)) candidates.push(() => reduceJson(prepared));
|
|
1413
|
+
if (looksLikeMinified(normalized.contentText)) candidates.push(() => reduceBundledJs(prepared));
|
|
1414
|
+
if (toolClass === "search") candidates.push(() => reduceSearch(prepared, ranking?.files));
|
|
1415
|
+
if (isGitCommand(name, command)) candidates.push(() => reduceGit(prepared, command));
|
|
1416
|
+
if (isPackageCommand(command)) candidates.push(() => reducePatternLog(prepared, "hypa-package", packagePattern()));
|
|
1417
|
+
if (isBuildOrTestCommand(command)) candidates.push(() => reducePatternLog(prepared, "hypa-build-test", buildPattern()));
|
|
1418
|
+
if (looksLikeSourceCode(normalized.contentText)) {
|
|
1419
|
+
if (input.codeSkeleton === true) candidates.push(() => reduceCodeSkeleton(prepared));
|
|
1420
|
+
else if (readTocFirst) candidates.push(() => tocGuardedCodeSkeleton(prepared));
|
|
1421
|
+
}
|
|
1422
|
+
if (looksLikeHtml(normalized.contentText)) candidates.push(() => reduceHtml(prepared));
|
|
1423
|
+
if (looksLikeDocument(normalized.contentText)) candidates.push(() => reduceDocSkeleton(prepared, ranking?.sections));
|
|
1424
|
+
if (toolClass === "shell" || command !== "") candidates.push(() => reduceShell(prepared));
|
|
1425
|
+
candidates.push(() => reduceProseKeep(prepared));
|
|
1426
|
+
if (toolClass === "read") candidates.push(() => reduceHead(prepared, "pi-head"));
|
|
1427
|
+
candidates.push(() => reduceSalient(prepared, "generic-salience"));
|
|
1428
|
+
for (const make of candidates) {
|
|
1429
|
+
const candidate = make();
|
|
1430
|
+
if (candidate !== null && verifyReduction(input, candidate)) return candidate;
|
|
1431
|
+
}
|
|
1432
|
+
return null;
|
|
1266
1433
|
}
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1434
|
+
/**
|
|
1435
|
+
* Build a recoverable placeholder for an old tool result.
|
|
1436
|
+
* @param input - tool identity, source reference, size, status, and retained evidence.
|
|
1437
|
+
* @returns a lossy placeholder that cites the immutable source event.
|
|
1438
|
+
*/
|
|
1439
|
+
function historicalPlaceholder(input) {
|
|
1440
|
+
const anchor = input.compact ? "" : importantAnchor(input.text, 360);
|
|
1441
|
+
const normalized = normalizeTerminalLines(input.text);
|
|
1442
|
+
const lastFolded = normalized.folded.at(-1);
|
|
1443
|
+
const elidedLines = lastFolded === void 0 ? void 0 : lastFolded.originalLineEnd ?? lastFolded.originalLine;
|
|
1444
|
+
const anchorLine = input.compact ? void 0 : (normalized.folded.find((line) => IMPORTANT_PATTERN.test(line.text)) ?? void 0)?.originalLine;
|
|
1445
|
+
const retrieveHint = anchorLine === void 0 ? `retrieve: context_compression_retrieve({"ref":"${input.sourceRef}"})` : `retrieve: context_compression_retrieve({"ref":"${input.sourceRef}","start_line":${String(anchorLine)},"max_lines":${String(RETRIEVE_HINT_MAX_LINES)}})`;
|
|
1446
|
+
const lines = [
|
|
1447
|
+
"[Old tool result content cleared from active context]",
|
|
1448
|
+
`tool: ${input.toolName || "unknown"}`,
|
|
1449
|
+
`status: ${input.isError ? "error" : "completed"}`,
|
|
1450
|
+
`original_chars: ${String(input.charsBefore)}`,
|
|
1451
|
+
`source: ${input.sourceRef}`,
|
|
1452
|
+
retrieveHint
|
|
1453
|
+
];
|
|
1454
|
+
if (anchor !== "") lines.push(`retained_anchor: ${anchor}`);
|
|
1455
|
+
return {
|
|
1456
|
+
text: lines.join("\n"),
|
|
1457
|
+
reducer: input.compact ? "pair-preserving-tail-aging" : "historical-tool-result-aging",
|
|
1458
|
+
lossy: true,
|
|
1459
|
+
...elidedLines === void 0 ? {} : { elidedLines }
|
|
1460
|
+
};
|
|
1461
|
+
}
|
|
1462
|
+
/**
|
|
1463
|
+
* Validate shrinkage, budget, recovery, and error retention.
|
|
1464
|
+
* @param input - original reducer input and its safety requirements.
|
|
1465
|
+
* @param output - candidate reduced text and reducer metadata.
|
|
1466
|
+
* @returns whether the candidate is safe to land.
|
|
1467
|
+
*/
|
|
1468
|
+
function verifyReduction(input, output) {
|
|
1469
|
+
const before = codePointLength(input.text);
|
|
1470
|
+
const after = codePointLength(output.text);
|
|
1471
|
+
if (after <= 0 || after >= before || after > input.budgetChars) return false;
|
|
1472
|
+
if (output.lossy && !output.text.includes(input.sourceRef)) return false;
|
|
1473
|
+
if ((input.isError || IMPORTANT_PATTERN.test(input.text)) && !IMPORTANT_PATTERN.test(output.text) && !output.text.includes("status: error")) return false;
|
|
1474
|
+
return true;
|
|
1475
|
+
}
|
|
1476
|
+
/**
|
|
1477
|
+
* Strip ANSI, collapse carriage-return progress redraws, and fold exact repeats.
|
|
1478
|
+
* @param text - raw terminal output.
|
|
1479
|
+
* @returns normalized terminal text.
|
|
1480
|
+
*/
|
|
1481
|
+
function normalizeTerminalText(text) {
|
|
1482
|
+
return normalizeTerminalLines(text).text;
|
|
1483
|
+
}
|
|
1484
|
+
/**
|
|
1485
|
+
* Structured normalization (R9a): `retrieve` reads the original event, so any
|
|
1486
|
+
* line number a reducer prints must resolve against the ORIGINAL text, not the
|
|
1487
|
+
* normalized surface. ANSI stripping and `\r` redraw collapse never change the
|
|
1488
|
+
* line count (logical lines are 1:1 with original lines); only the adjacent
|
|
1489
|
+
* duplicate fold drops lines, so every folded entry carries the original line
|
|
1490
|
+
* (range) it was kept from.
|
|
1491
|
+
*/
|
|
1492
|
+
function normalizeTerminalLines(text) {
|
|
1493
|
+
const logical = text.replace(ANSI_PATTERN, "").split("\n").map((line) => {
|
|
1494
|
+
return placeholderizeLongStrings(line.split("\r").filter((part) => part !== "").at(-1) ?? "");
|
|
1495
|
+
});
|
|
1496
|
+
const stripGutter = hasReadGutter(logical);
|
|
1497
|
+
const folded = [];
|
|
1498
|
+
let previous;
|
|
1499
|
+
let firstText = "";
|
|
1500
|
+
let count = 0;
|
|
1501
|
+
let firstOriginal = 0;
|
|
1502
|
+
const flush = (nextOriginal) => {
|
|
1503
|
+
if (previous === void 0) return;
|
|
1504
|
+
folded.push({
|
|
1505
|
+
text: firstText,
|
|
1506
|
+
content: previous,
|
|
1507
|
+
originalLine: firstOriginal
|
|
1277
1508
|
});
|
|
1278
|
-
|
|
1279
|
-
|
|
1509
|
+
if (count > 1) {
|
|
1510
|
+
const marker = `[previous line repeated ${String(count - 1)} more times]`;
|
|
1511
|
+
folded.push({
|
|
1512
|
+
text: marker,
|
|
1513
|
+
content: marker,
|
|
1514
|
+
originalLine: firstOriginal + 1,
|
|
1515
|
+
originalLineEnd: nextOriginal - 1
|
|
1516
|
+
});
|
|
1517
|
+
}
|
|
1518
|
+
};
|
|
1519
|
+
logical.forEach((line, index) => {
|
|
1520
|
+
const originalLine = index + 1;
|
|
1521
|
+
const content = stripGutter ? line.replace(READ_GUTTER_PATTERN, "") : line;
|
|
1522
|
+
if (content === previous) {
|
|
1523
|
+
count++;
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
flush(originalLine);
|
|
1527
|
+
previous = content;
|
|
1528
|
+
firstText = line;
|
|
1529
|
+
count = 1;
|
|
1530
|
+
firstOriginal = originalLine;
|
|
1531
|
+
});
|
|
1532
|
+
flush(logical.length + 1);
|
|
1533
|
+
const result = foldNonAdjacentRepeats(folded);
|
|
1534
|
+
return {
|
|
1535
|
+
folded: result,
|
|
1536
|
+
text: result.map((line) => line.text).join("\n"),
|
|
1537
|
+
contentText: result.map((line) => line.content).join("\n")
|
|
1538
|
+
};
|
|
1280
1539
|
}
|
|
1281
1540
|
/**
|
|
1282
|
-
*
|
|
1283
|
-
*
|
|
1284
|
-
*
|
|
1541
|
+
* Fold non-adjacent exact repeats (R11). Adjacent folding runs FIRST and only
|
|
1542
|
+
* handles consecutive runs (0.03–0.32% of real duplicate content); separated
|
|
1543
|
+
* repeats reached 8.37% in large results. Each surviving occurrence — a kept
|
|
1544
|
+
* line plus its optional adjacent-repeat marker — is one unit; once a text
|
|
1545
|
+
* recurs ≥ threshold times, the first unit is kept and every later unit is
|
|
1546
|
+
* replaced by ONE counted marker citing the original-event span it covers.
|
|
1547
|
+
* A pure consecutive run forms a single unit, so this pass is a no-op on it
|
|
1548
|
+
* and can never double-fold the adjacent marker.
|
|
1285
1549
|
*/
|
|
1286
|
-
function
|
|
1287
|
-
const
|
|
1288
|
-
const
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1550
|
+
function foldNonAdjacentRepeats(folded) {
|
|
1551
|
+
const units = [];
|
|
1552
|
+
for (const entry of folded) if (entry.text.startsWith(ADJACENT_REPEAT_MARKER) && units.length > 0) units[units.length - 1].repeat = entry;
|
|
1553
|
+
else units.push({ lead: entry });
|
|
1554
|
+
const totals = /* @__PURE__ */ new Map();
|
|
1555
|
+
for (const unit of units) totals.set(unit.lead.content, (totals.get(unit.lead.content) ?? 0) + 1);
|
|
1556
|
+
if (totals.size === units.length) return [...folded];
|
|
1557
|
+
const firstOriginal = /* @__PURE__ */ new Map();
|
|
1558
|
+
const lastOriginalEnd = /* @__PURE__ */ new Map();
|
|
1559
|
+
for (const unit of units) {
|
|
1560
|
+
const content = unit.lead.content;
|
|
1561
|
+
if (totals.get(content) < NON_ADJACENT_FOLD_THRESHOLD) continue;
|
|
1562
|
+
if (!firstOriginal.has(content)) firstOriginal.set(content, unit.lead.originalLine);
|
|
1563
|
+
const end = unit.repeat?.originalLineEnd ?? unit.lead.originalLineEnd ?? unit.lead.originalLine;
|
|
1564
|
+
lastOriginalEnd.set(content, end);
|
|
1565
|
+
}
|
|
1566
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1567
|
+
const result = [];
|
|
1568
|
+
for (const unit of units) {
|
|
1569
|
+
const content = unit.lead.content;
|
|
1570
|
+
const total = totals.get(content);
|
|
1571
|
+
if (total < NON_ADJACENT_FOLD_THRESHOLD) {
|
|
1572
|
+
result.push(unit.lead);
|
|
1573
|
+
if (unit.repeat !== void 0) result.push(unit.repeat);
|
|
1574
|
+
continue;
|
|
1302
1575
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1576
|
+
if (!seen.has(content)) {
|
|
1577
|
+
seen.set(content, 1);
|
|
1578
|
+
result.push(unit.lead);
|
|
1579
|
+
if (unit.repeat !== void 0) result.push(unit.repeat);
|
|
1580
|
+
continue;
|
|
1581
|
+
}
|
|
1582
|
+
const ordinal = (seen.get(content) ?? 1) + 1;
|
|
1583
|
+
seen.set(content, ordinal);
|
|
1584
|
+
if (ordinal > 2) continue;
|
|
1585
|
+
const end = lastOriginalEnd.get(content);
|
|
1586
|
+
const marker = `[× ${String(total)} total: same as line ${String(firstOriginal.get(content))}; original lines ${String(unit.lead.originalLine)}-${String(end)}]`;
|
|
1587
|
+
result.push({
|
|
1588
|
+
text: marker,
|
|
1589
|
+
content: marker,
|
|
1590
|
+
originalLine: unit.lead.originalLine,
|
|
1591
|
+
originalLineEnd: end
|
|
1592
|
+
});
|
|
1311
1593
|
}
|
|
1594
|
+
return result;
|
|
1312
1595
|
}
|
|
1313
|
-
/**
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1596
|
+
/** Default line window a retrieve hint suggests the model paste. */
|
|
1597
|
+
const RETRIEVE_HINT_MAX_LINES = 80;
|
|
1598
|
+
/**
|
|
1599
|
+
* TOC-first (G6): read-class results at or above this size let the code
|
|
1600
|
+
* skeleton compete before head/tail truncation. The 14,000-char boundary is
|
|
1601
|
+
* the studied real-read cohort (findings §7), well above p90 of actual reads
|
|
1602
|
+
* so ordinary results keep their existing dispatch.
|
|
1603
|
+
*/
|
|
1604
|
+
const READ_TOC_MIN_CHARS = 14e3;
|
|
1605
|
+
/**
|
|
1606
|
+
* A skeleton whose output is dominated by elision markers is worse than
|
|
1607
|
+
* head/tail for the model (task_4b risk: structure-poor files degenerate into
|
|
1608
|
+
* "almost all markers") — above this marker-char share the TOC candidate fails
|
|
1609
|
+
* open to the prose reducers.
|
|
1610
|
+
*/
|
|
1611
|
+
const TOC_MARKER_RATIO_LIMIT = .5;
|
|
1612
|
+
/**
|
|
1613
|
+
* Fail-open wrapper for the TOC-first code-skeleton candidate: a skeleton that
|
|
1614
|
+
* degenerates into mostly-elision markers (minified bundles, generated files)
|
|
1615
|
+
* returns null so the prose head/tail pair takes over.
|
|
1616
|
+
*/
|
|
1617
|
+
function tocGuardedCodeSkeleton(input) {
|
|
1618
|
+
const output = reduceCodeSkeleton(input);
|
|
1619
|
+
if (output === null) return null;
|
|
1620
|
+
const total = codePointLength(output.text);
|
|
1621
|
+
return output.text.split("\n").filter((line) => line.startsWith("[...")).reduce((sum, line) => sum + codePointLength(line) + 1, 0) / total > TOC_MARKER_RATIO_LIMIT ? null : output;
|
|
1622
|
+
}
|
|
1623
|
+
/** R10a thresholds: one giant line, uniformly fat lines, or very few fat lines. */
|
|
1624
|
+
const MINIFIED_MAX_LINE_CHARS = 2e3;
|
|
1625
|
+
const MINIFIED_AVG_LINE_CHARS = 300;
|
|
1626
|
+
const MINIFIED_FEW_LINES = 40;
|
|
1627
|
+
const MINIFIED_FEW_LINES_TOTAL_CHARS = 2e4;
|
|
1628
|
+
/**
|
|
1629
|
+
* Require form evidence of a bundled/minified module (R10a): line-anchored
|
|
1630
|
+
* reducers cannot see inside a 135k-character line, and R9 line ranges on a
|
|
1631
|
+
* 53-line bundle cannot address anything smaller than the whole file.
|
|
1632
|
+
* @param text - normalized result text.
|
|
1633
|
+
* @returns whether the text reads as a bundled/minified module.
|
|
1634
|
+
*/
|
|
1635
|
+
function looksLikeMinified(text) {
|
|
1636
|
+
const lines = splitLines(text);
|
|
1637
|
+
if (lines.length === 0) return false;
|
|
1638
|
+
let total = 0;
|
|
1639
|
+
let max = 0;
|
|
1640
|
+
for (const line of lines) {
|
|
1641
|
+
const length = line.length;
|
|
1642
|
+
total += length;
|
|
1643
|
+
if (length > max) max = length;
|
|
1323
1644
|
}
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1645
|
+
if (max > MINIFIED_MAX_LINE_CHARS) return true;
|
|
1646
|
+
if (total / lines.length > MINIFIED_AVG_LINE_CHARS) return true;
|
|
1647
|
+
return lines.length < MINIFIED_FEW_LINES && total > MINIFIED_FEW_LINES_TOTAL_CHARS;
|
|
1648
|
+
}
|
|
1649
|
+
/**
|
|
1650
|
+
* Statement-level declaration patterns scanned GLOBALLY per line: a bundle's
|
|
1651
|
+
* statements are separated by `;` / `},{` / `);` inside one physical line, so
|
|
1652
|
+
* line-anchored matching is useless here. Reserved-name traces (`exports.*`,
|
|
1653
|
+
* `module.exports`) are extracted first and called out in the header because
|
|
1654
|
+
* minifiers rename local symbols.
|
|
1655
|
+
*/
|
|
1656
|
+
const BUNDLED_DECLARATION_PATTERNS = [
|
|
1657
|
+
/\bexports\.([A-Za-z_$][\w$]*)\s*=/g,
|
|
1658
|
+
/\bmodule\.exports\s*=\s*([A-Za-z_$][\w$]*)/g,
|
|
1659
|
+
/\b(?:function|class)\s+([A-Za-z_$][\w$]*)/g,
|
|
1660
|
+
/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=/g,
|
|
1661
|
+
/\b([A-Za-z_$][\w$]*)\s*:\s*function\b/g
|
|
1662
|
+
];
|
|
1663
|
+
const SOURCEMAP_DIRECTIVE = "//# sourceMappingURL=";
|
|
1664
|
+
/**
|
|
1665
|
+
* Bundled/minified JS directory (R10-B). The useful first answer is the
|
|
1666
|
+
* declaration/export directory — WHAT the bundle exposes — plus one honest
|
|
1667
|
+
* whole-span marker: the host's continuation is line-addressed, so a
|
|
1668
|
+
* line-range retrieve on a 53-line bundle hands back the whole file (R10d:
|
|
1669
|
+
* character-range retrieval is a separate, undecided extension).
|
|
1670
|
+
*/
|
|
1671
|
+
function reduceBundledJs(input) {
|
|
1672
|
+
const declarations = /* @__PURE__ */ new Map();
|
|
1673
|
+
for (const line of input.lines) for (const pattern of BUNDLED_DECLARATION_PATTERNS) {
|
|
1674
|
+
pattern.lastIndex = 0;
|
|
1675
|
+
let match = pattern.exec(line.content);
|
|
1676
|
+
while (match !== null) {
|
|
1677
|
+
const symbol = match[1];
|
|
1678
|
+
if (symbol !== void 0 && !declarations.has(symbol)) declarations.set(symbol, line.originalLine);
|
|
1679
|
+
match = pattern.exec(line.content);
|
|
1334
1680
|
}
|
|
1335
1681
|
}
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1682
|
+
if (declarations.size === 0) return null;
|
|
1683
|
+
const hasSourceMap = input.contentText.includes(SOURCEMAP_DIRECTIVE);
|
|
1684
|
+
const entries = [...declarations.entries()].sort((a, b) => a[1] - b[1]);
|
|
1685
|
+
const kept = [`[bundled/minified JS detected; ${String(entries.length)} declarations; minified symbols may be renamed — exports.*/module.exports traces are the reliable ones;${hasSourceMap ? " source map present, prefer reading the original source;" : ""} source: ${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}"}) (line-addressed: single-line bundles come back whole)]`, ...entries.slice(0, 400).map(([symbol, line]) => `${symbol} (line ${String(line)})`)];
|
|
1686
|
+
const start = input.lines[0]?.originalLine ?? 1;
|
|
1687
|
+
const end = originalEnd(input.lines, input.lines.length - 1);
|
|
1688
|
+
if (end > start) kept.push(elidedRangeMarker(start, end));
|
|
1689
|
+
const text = fitLines(kept, input.budgetChars, input.sourceRef);
|
|
1690
|
+
return text === null ? null : {
|
|
1691
|
+
text,
|
|
1692
|
+
reducer: "bundled-js-directory",
|
|
1693
|
+
lossy: true
|
|
1694
|
+
};
|
|
1695
|
+
}
|
|
1696
|
+
/**
|
|
1697
|
+
* Continuous-mask marker (R9b): cites the ORIGINAL-event line range it elides
|
|
1698
|
+
* and carries a pasteable retrieve hint starting at the first elided line.
|
|
1699
|
+
* Falls back to the compact plain marker when the hint would not fit.
|
|
1700
|
+
*/
|
|
1701
|
+
function reduceHead(input, reducer) {
|
|
1702
|
+
const marker = omissionMarker(input, reducer);
|
|
1703
|
+
const available = input.budgetChars - codePointLength(marker) - 1;
|
|
1704
|
+
if (available <= 0) return null;
|
|
1705
|
+
const head = takeWholeLinesFromHead(input.text, available);
|
|
1706
|
+
if (head === input.text || head === "") return null;
|
|
1707
|
+
const keptCount = head.split("\n").length;
|
|
1708
|
+
const firstElided = input.lines[keptCount];
|
|
1709
|
+
if (firstElided !== void 0) {
|
|
1710
|
+
const elidedEnd = originalEnd(input.lines, input.lines.length - 1);
|
|
1711
|
+
const ranged = rangeOmissionMarker(input, reducer, firstElided.originalLine, elidedEnd);
|
|
1712
|
+
if (codePointLength(head) + codePointLength(ranged) + 1 <= input.budgetChars) return {
|
|
1713
|
+
text: `${head}\n${ranged}`,
|
|
1714
|
+
reducer,
|
|
1715
|
+
lossy: true
|
|
1349
1716
|
};
|
|
1350
|
-
try {
|
|
1351
|
-
const selected = this.ctx.get("agentDefaultModel")?.currentSelection?.();
|
|
1352
|
-
const selectedProvider = selected?.provider ?? "";
|
|
1353
|
-
const selectedModel = selected?.model ?? "";
|
|
1354
|
-
if (selectedProvider.length > 0 && selectedModel.length > 0) return {
|
|
1355
|
-
provider: provider.length > 0 ? provider : selectedProvider,
|
|
1356
|
-
model: model.length > 0 ? model : selectedModel
|
|
1357
|
-
};
|
|
1358
|
-
} catch {}
|
|
1359
1717
|
}
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
}]
|
|
1382
|
-
}],
|
|
1383
|
-
system,
|
|
1384
|
-
temperature: 0,
|
|
1385
|
-
reasoningEffort: "off",
|
|
1386
|
-
maxTokens: 256,
|
|
1387
|
-
signal
|
|
1388
|
-
});
|
|
1389
|
-
for await (const chunk of stream) if ((chunk.type === "text-delta" || chunk.type === "reasoning-delta") && typeof chunk.text === "string") text += chunk.text;
|
|
1390
|
-
else if (chunk.type === "finish" && chunk.text === void 0) break;
|
|
1391
|
-
return text.trim().length > 0 ? text : void 0;
|
|
1718
|
+
return {
|
|
1719
|
+
text: `${head}\n${marker}`,
|
|
1720
|
+
reducer,
|
|
1721
|
+
lossy: true
|
|
1722
|
+
};
|
|
1723
|
+
}
|
|
1724
|
+
function reduceTail(input, reducer) {
|
|
1725
|
+
const marker = omissionMarker(input, reducer);
|
|
1726
|
+
const available = input.budgetChars - codePointLength(marker) - 1;
|
|
1727
|
+
if (available <= 0) return null;
|
|
1728
|
+
const tail = takeWholeLinesFromTail(input.text, available);
|
|
1729
|
+
if (tail === input.text || tail === "") return null;
|
|
1730
|
+
const firstKept = input.lines.length - tail.split("\n").length;
|
|
1731
|
+
if (firstKept > 0) {
|
|
1732
|
+
const elidedEnd = originalEnd(input.lines, firstKept - 1);
|
|
1733
|
+
const ranged = rangeOmissionMarker(input, reducer, input.lines[0].originalLine, elidedEnd);
|
|
1734
|
+
if (codePointLength(ranged) + codePointLength(tail) + 1 <= input.budgetChars) return {
|
|
1735
|
+
text: `${ranged}\n${tail}`,
|
|
1736
|
+
reducer,
|
|
1737
|
+
lossy: true
|
|
1738
|
+
};
|
|
1392
1739
|
}
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
messages: [{
|
|
1406
|
-
role: "system",
|
|
1407
|
-
content: system
|
|
1408
|
-
}, {
|
|
1409
|
-
role: "user",
|
|
1410
|
-
content: user
|
|
1411
|
-
}],
|
|
1412
|
-
temperature: 0,
|
|
1413
|
-
max_tokens: 256
|
|
1414
|
-
}),
|
|
1415
|
-
signal
|
|
1416
|
-
});
|
|
1417
|
-
if (!response.ok) return void 0;
|
|
1418
|
-
const text = (await response.json()).choices?.[0]?.message?.content;
|
|
1419
|
-
return typeof text === "string" && text.trim().length > 0 ? text : void 0;
|
|
1740
|
+
return {
|
|
1741
|
+
text: `${marker}\n${tail}`,
|
|
1742
|
+
reducer,
|
|
1743
|
+
lossy: true
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
1746
|
+
function reduceJson(input) {
|
|
1747
|
+
let value;
|
|
1748
|
+
try {
|
|
1749
|
+
value = JSON.parse(input.text);
|
|
1750
|
+
} catch {
|
|
1751
|
+
return null;
|
|
1420
1752
|
}
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1753
|
+
const minified = JSON.stringify(value);
|
|
1754
|
+
if (codePointLength(minified) < codePointLength(input.text) && codePointLength(minified) <= input.budgetChars) return {
|
|
1755
|
+
text: minified,
|
|
1756
|
+
reducer: "json-minify",
|
|
1757
|
+
lossy: false
|
|
1758
|
+
};
|
|
1759
|
+
const envelope = {
|
|
1760
|
+
$dsh_compression: {
|
|
1761
|
+
kind: "json-preview",
|
|
1762
|
+
source: input.sourceRef,
|
|
1763
|
+
original_chars: codePointLength(input.text)
|
|
1764
|
+
},
|
|
1765
|
+
value: shrinkJson(value, 0)
|
|
1766
|
+
};
|
|
1767
|
+
const text = JSON.stringify(envelope, null, 2);
|
|
1768
|
+
if (codePointLength(text) <= input.budgetChars) return {
|
|
1769
|
+
text,
|
|
1770
|
+
reducer: "json-structure-preview",
|
|
1771
|
+
lossy: true
|
|
1772
|
+
};
|
|
1773
|
+
return null;
|
|
1774
|
+
}
|
|
1775
|
+
function shrinkJson(value, depth) {
|
|
1776
|
+
if (depth >= 5) {
|
|
1777
|
+
if (Array.isArray(value)) return `[array length=${String(value.length)} omitted]`;
|
|
1778
|
+
if (typeof value === "object" && value !== null) return "[object omitted]";
|
|
1779
|
+
return value;
|
|
1439
1780
|
}
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
return
|
|
1781
|
+
if (Array.isArray(value)) {
|
|
1782
|
+
if (value.length <= 8) return value.map((entry) => shrinkJson(entry, depth + 1));
|
|
1783
|
+
return [
|
|
1784
|
+
...value.slice(0, 3).map((entry) => shrinkJson(entry, depth + 1)),
|
|
1785
|
+
{ $dsh_omitted_items: value.length - 5 },
|
|
1786
|
+
...value.slice(-2).map((entry) => shrinkJson(entry, depth + 1))
|
|
1787
|
+
];
|
|
1443
1788
|
}
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1789
|
+
if (typeof value !== "object" || value === null) {
|
|
1790
|
+
if (typeof value === "string" && codePointLength(value) > 800) return `${Array.from(value).slice(0, 500).join("")}…[${String(codePointLength(value) - 700)} chars omitted]…${Array.from(value).slice(-200).join("")}`;
|
|
1791
|
+
return value;
|
|
1792
|
+
}
|
|
1793
|
+
const entries = Object.entries(value);
|
|
1794
|
+
const important = entries.filter(([key]) => /error|warn|status|code|message|path|file|line|summary/i.test(key));
|
|
1795
|
+
const selected = entries.length <= 18 ? entries : [
|
|
1796
|
+
...entries.slice(0, 10),
|
|
1797
|
+
...important.filter((entry) => !entries.slice(0, 10).includes(entry)).slice(0, 6),
|
|
1798
|
+
...entries.slice(-2)
|
|
1799
|
+
];
|
|
1800
|
+
const result = {};
|
|
1801
|
+
for (const [key, entry] of selected) result[key] = shrinkJson(entry, depth + 1);
|
|
1802
|
+
if (selected.length < entries.length) result.$dsh_omitted_keys = entries.length - selected.length;
|
|
1803
|
+
return result;
|
|
1804
|
+
}
|
|
1805
|
+
/**
|
|
1806
|
+
* Two-tier search folding (R10). L1 is a LOSSLESS per-file locator —
|
|
1807
|
+
* `## <path> (<N> matches) L12,L15,…` — one line number per hit, taken from
|
|
1808
|
+
* the hit's own `path:line` prefix (falling back to the original-event line).
|
|
1809
|
+
* L2 is the content quota, water-filled round-robin so no file vanishes and
|
|
1810
|
+
* no file runs more than one row ahead of another; the budget is reserved for
|
|
1811
|
+
* L1 first. When L1 itself cannot fit, the shortfall is ANNOUNCED
|
|
1812
|
+
* (withheld file/match counts) — never silently truncated. Outputs without
|
|
1813
|
+
* any `path:line` form fail open to salience.
|
|
1814
|
+
*/
|
|
1815
|
+
function reduceSearch(input, fileRanking) {
|
|
1816
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1817
|
+
const ungrouped = [];
|
|
1818
|
+
input.lines.forEach((line) => {
|
|
1819
|
+
const match = PATH_LINE_PATTERN.exec(line.text);
|
|
1820
|
+
const row = {
|
|
1821
|
+
text: line.text,
|
|
1822
|
+
fileLine: match !== null ? Number(match[2]) : line.originalLine,
|
|
1823
|
+
important: IMPORTANT_PATTERN.test(line.text)
|
|
1824
|
+
};
|
|
1825
|
+
if (match === null) {
|
|
1826
|
+
ungrouped.push(row);
|
|
1827
|
+
return;
|
|
1451
1828
|
}
|
|
1452
|
-
|
|
1829
|
+
const path = match[1] ?? "<unknown>";
|
|
1830
|
+
const bucket = groups.get(path) ?? [];
|
|
1831
|
+
bucket.push(row);
|
|
1832
|
+
groups.set(path, bucket);
|
|
1833
|
+
});
|
|
1834
|
+
if (groups.size === 0) return reduceSalient(input, "search-salience");
|
|
1835
|
+
const totalMatches = [...groups.values()].reduce((sum, rows) => sum + rows.length, 0);
|
|
1836
|
+
const locatorFor = (path, rows) => `## ${path} (${String(rows.length)} matches) ${rows.map((row) => `L${String(row.fileLine)}`).join(",")}`;
|
|
1837
|
+
const perFile = /* @__PURE__ */ new Map();
|
|
1838
|
+
for (const entry of rankedFirst([...groups.entries()].map(([id, rows]) => ({
|
|
1839
|
+
id,
|
|
1840
|
+
rows
|
|
1841
|
+
})), fileRanking)) perFile.set(entry.id, [...entry.rows].sort((a, b) => a.important === b.important ? a.fileLine - b.fileLine : a.important ? -1 : 1));
|
|
1842
|
+
const allLocators = [...perFile.keys()].map((path) => locatorFor(path, groups.get(path)));
|
|
1843
|
+
const headerFor = (l2Rows, omitted) => `[search results compressed; ${String(groups.size)} files, ${String(totalMatches)} matches; ${String(l2Rows)} content rows shown, ${String(omitted)} matches omitted; source: ${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}"})]`;
|
|
1844
|
+
const fillL2 = (output, quotaChars) => {
|
|
1845
|
+
let used = 0;
|
|
1846
|
+
let shown = 0;
|
|
1847
|
+
let round = 0;
|
|
1848
|
+
let progress = true;
|
|
1849
|
+
while (progress && round < 512) {
|
|
1850
|
+
progress = false;
|
|
1851
|
+
for (const rows of perFile.values()) {
|
|
1852
|
+
if (round >= rows.length) continue;
|
|
1853
|
+
const row = rows[round];
|
|
1854
|
+
const cost = codePointLength(row.text) + 1;
|
|
1855
|
+
if (used + cost > quotaChars) continue;
|
|
1856
|
+
output.push(row.text);
|
|
1857
|
+
used += cost;
|
|
1858
|
+
shown += 1;
|
|
1859
|
+
progress = true;
|
|
1860
|
+
}
|
|
1861
|
+
round += 1;
|
|
1862
|
+
}
|
|
1863
|
+
for (const row of ungrouped.filter((entry) => entry.important).slice(0, 12)) {
|
|
1864
|
+
const cost = codePointLength(row.text) + 1;
|
|
1865
|
+
if (used + cost > quotaChars) break;
|
|
1866
|
+
output.push(row.text);
|
|
1867
|
+
used += cost;
|
|
1868
|
+
shown += 1;
|
|
1869
|
+
}
|
|
1870
|
+
return {
|
|
1871
|
+
shown,
|
|
1872
|
+
omitted: totalMatches - shown
|
|
1873
|
+
};
|
|
1874
|
+
};
|
|
1875
|
+
const finish = (output) => {
|
|
1876
|
+
const text = output.join("\n");
|
|
1877
|
+
return text.includes(input.sourceRef) ? {
|
|
1878
|
+
text,
|
|
1879
|
+
reducer: "search-by-file",
|
|
1880
|
+
lossy: true
|
|
1881
|
+
} : null;
|
|
1882
|
+
};
|
|
1883
|
+
const headerProbe = headerFor(0, 0);
|
|
1884
|
+
const budget = input.budgetChars - codePointLength(headerProbe) - 2;
|
|
1885
|
+
if (budget <= 0) return null;
|
|
1886
|
+
const locatorCost = allLocators.reduce((sum, line) => sum + codePointLength(line) + 1, 0);
|
|
1887
|
+
if (locatorCost > budget) {
|
|
1888
|
+
const announcementReserve = 160;
|
|
1889
|
+
const output = [];
|
|
1890
|
+
let used = 0;
|
|
1891
|
+
let withheldFiles = 0;
|
|
1892
|
+
let withheldMatches = 0;
|
|
1893
|
+
for (let index = 0; index < allLocators.length; index++) {
|
|
1894
|
+
const cost = codePointLength(allLocators[index]) + 1 + announcementReserve;
|
|
1895
|
+
if (used + cost > budget) {
|
|
1896
|
+
withheldFiles = allLocators.length - index;
|
|
1897
|
+
withheldMatches = totalMatches - [...groups.values()].slice(0, index).reduce((sum, rows) => sum + rows.length, 0);
|
|
1898
|
+
break;
|
|
1899
|
+
}
|
|
1900
|
+
output.push(allLocators[index]);
|
|
1901
|
+
used += cost - announcementReserve;
|
|
1902
|
+
}
|
|
1903
|
+
if (withheldFiles > 0) output.push(`[L1 locator partially withheld: ${String(withheldFiles)} file(s) / ${String(withheldMatches)} matches' line lists did not fit the budget; retrieve for the full hit list]`);
|
|
1904
|
+
const { shown, omitted } = fillL2(output, Math.max(0, budget - used - (withheldFiles > 0 ? announcementReserve : 0)));
|
|
1905
|
+
output.unshift(headerFor(shown, omitted + withheldMatches));
|
|
1906
|
+
return finish(output);
|
|
1453
1907
|
}
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
return text.replace(/[ \t]+\r?\n/g, "\n").replace(/(^\s+)|(\s+$)/g, "");
|
|
1908
|
+
const output = [...allLocators];
|
|
1909
|
+
const { shown, omitted } = fillL2(output, budget - locatorCost);
|
|
1910
|
+
output.unshift(headerFor(shown, omitted));
|
|
1911
|
+
return finish(output);
|
|
1459
1912
|
}
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1913
|
+
function reduceGit(input, command) {
|
|
1914
|
+
const lines = input.lines;
|
|
1915
|
+
const lower = command.toLowerCase();
|
|
1916
|
+
let keep;
|
|
1917
|
+
let reducer;
|
|
1918
|
+
if (/\bgit\s+(?:diff|show)\b/.test(lower)) {
|
|
1919
|
+
reducer = "hypa-git-diff";
|
|
1920
|
+
keep = lines.map((line) => line.text).filter((line) => /^(?:diff --git|index |--- |\+\+\+ |@@ |[+-](?![+-]))/.test(line) || IMPORTANT_PATTERN.test(line));
|
|
1921
|
+
} else if (/\bgit\s+(?:status|switch|checkout|merge|rebase|cherry-pick)\b/.test(lower)) {
|
|
1922
|
+
reducer = "hypa-git-status";
|
|
1923
|
+
keep = lines.map((line) => line.text).filter((line) => GIT_STATUS_PATTERN.test(line) || IMPORTANT_PATTERN.test(line));
|
|
1924
|
+
} else {
|
|
1925
|
+
reducer = "hypa-git-log";
|
|
1926
|
+
keep = lines.map((line) => line.text).filter((line) => /^(?:commit\s+[0-9a-f]+|Author:|Date:|[0-9a-f]{7,}\s)/i.test(line) || IMPORTANT_PATTERN.test(line));
|
|
1927
|
+
}
|
|
1928
|
+
if (keep.length === 0) return reduceSalient(input, reducer);
|
|
1929
|
+
const text = fitLines([
|
|
1930
|
+
`[git output compressed; ${scannedTotals(input, keep.length)}; source: ${input.sourceRef}; scatter-masked: retrieve with context_compression_retrieve({"ref":"${input.sourceRef}","query":"<keyword>"}) for missed rows]`,
|
|
1931
|
+
...keep,
|
|
1932
|
+
...lines.slice(-8).map((line) => line.text)
|
|
1933
|
+
], input.budgetChars, input.sourceRef);
|
|
1934
|
+
return text === null ? null : {
|
|
1935
|
+
text,
|
|
1936
|
+
reducer,
|
|
1937
|
+
lossy: true
|
|
1938
|
+
};
|
|
1463
1939
|
}
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1940
|
+
function reducePatternLog(input, reducer, pattern) {
|
|
1941
|
+
const lines = input.lines;
|
|
1942
|
+
const kept = lines.filter((line) => pattern.test(line.text) || IMPORTANT_PATTERN.test(line.text) || STATUS_PATTERN.test(line.text));
|
|
1943
|
+
const text = fitLines([
|
|
1944
|
+
`[command output compressed by ${reducer}; ${scannedTotals(input, kept.length)}; source: ${input.sourceRef}; scatter-masked: retrieve with context_compression_retrieve({"ref":"${input.sourceRef}","query":"<keyword>"}) for missed rows]`,
|
|
1945
|
+
...kept.map((line) => line.text),
|
|
1946
|
+
...lines.slice(-20).map((line) => line.text)
|
|
1947
|
+
], input.budgetChars, input.sourceRef);
|
|
1948
|
+
return text === null ? null : {
|
|
1949
|
+
text,
|
|
1950
|
+
reducer,
|
|
1951
|
+
lossy: true
|
|
1952
|
+
};
|
|
1953
|
+
}
|
|
1954
|
+
function reduceShell(input) {
|
|
1955
|
+
const lines = input.lines;
|
|
1956
|
+
const important = lines.filter((line) => IMPORTANT_PATTERN.test(line.text));
|
|
1957
|
+
if (important.length === 0) return reduceTail(input, "pi-tail");
|
|
1958
|
+
const text = fitLines([
|
|
1959
|
+
`[shell/log output compressed; ${scannedTotals(input, important.length)}; source: ${input.sourceRef}; scatter-masked: retrieve with context_compression_retrieve({"ref":"${input.sourceRef}","query":"<keyword>"}) for missed rows]`,
|
|
1960
|
+
...important.map((line) => line.text),
|
|
1961
|
+
"--- final output ---",
|
|
1962
|
+
...lines.slice(-40).map((line) => line.text)
|
|
1963
|
+
], input.budgetChars, input.sourceRef);
|
|
1964
|
+
return text === null ? null : {
|
|
1965
|
+
text,
|
|
1966
|
+
reducer: "shell-salience-tail",
|
|
1967
|
+
lossy: true
|
|
1968
|
+
};
|
|
1969
|
+
}
|
|
1970
|
+
function reduceSalient(input, reducer) {
|
|
1971
|
+
const lines = input.lines;
|
|
1972
|
+
if (lines.length < 3) return reduceHead(input, reducer);
|
|
1973
|
+
const marker = omissionMarker(input, reducer);
|
|
1974
|
+
const headBudget = Math.max(1, Math.floor((input.budgetChars - codePointLength(marker)) * .34));
|
|
1975
|
+
const tailBudget = headBudget;
|
|
1976
|
+
const head = takeWholeLinesFromHead(input.text, headBudget);
|
|
1977
|
+
const tail = takeWholeLinesFromTail(input.text, tailBudget);
|
|
1978
|
+
const salient = lines.filter((line) => IMPORTANT_PATTERN.test(line.text) || STATUS_PATTERN.test(line.text)).slice(0, 24);
|
|
1979
|
+
const keptCount = head.split("\n").length + salient.length + tail.split("\n").length;
|
|
1980
|
+
const text = fitLines([
|
|
1981
|
+
head,
|
|
1982
|
+
...salient.map((line) => line.text),
|
|
1983
|
+
`${marker} [${scannedTotals(input, keptCount)}]`,
|
|
1984
|
+
tail
|
|
1985
|
+
], input.budgetChars, input.sourceRef);
|
|
1986
|
+
return text === null ? null : {
|
|
1987
|
+
text,
|
|
1988
|
+
reducer,
|
|
1989
|
+
lossy: true
|
|
1990
|
+
};
|
|
1991
|
+
}
|
|
1992
|
+
/**
|
|
1993
|
+
* Require content evidence of a structured document: enough Markdown heading
|
|
1994
|
+
* lines among a bounded prefix. Pure form evidence — tool names, path
|
|
1995
|
+
* extensions, and commands are never read (MCP output has no predictable
|
|
1996
|
+
* identity). Real logs and build output carry no `#`-heading lines, which is
|
|
1997
|
+
* the misjudgment guard.
|
|
1998
|
+
* @param text - normalized result text.
|
|
1999
|
+
* @returns whether the text qualifies as a structured document.
|
|
2000
|
+
*/
|
|
2001
|
+
function looksLikeDocument(text) {
|
|
2002
|
+
const lines = splitLines(text);
|
|
2003
|
+
let headings = 0;
|
|
2004
|
+
for (const line of lines.slice(0, 600)) if (MARKDOWN_HEADING_PATTERN.test(line)) {
|
|
2005
|
+
headings += 1;
|
|
2006
|
+
if (headings >= 3) return true;
|
|
1470
2007
|
}
|
|
1471
|
-
return
|
|
2008
|
+
return false;
|
|
1472
2009
|
}
|
|
1473
|
-
/**
|
|
1474
|
-
function
|
|
1475
|
-
return [
|
|
1476
|
-
`[... identical to the earlier ${entry.toolName} result; first seen at ${entry.sourceRef};`,
|
|
1477
|
-
`original_chars=${String(originalChars)};`,
|
|
1478
|
-
"use context_compression_retrieve with this source if the omitted evidence is necessary.]"
|
|
1479
|
-
].join(" ");
|
|
2010
|
+
/** One R9-spec elision marker: an original-event line range plus its count. */
|
|
2011
|
+
function elidedRangeMarker(start, end) {
|
|
2012
|
+
return `[... lines ${String(start)}-${String(end)} elided (${String(end - start + 1)} lines) ...]`;
|
|
1480
2013
|
}
|
|
1481
|
-
//#endregion
|
|
1482
|
-
//#region src/runtime/tokenpilot/proposal.ts
|
|
1483
2014
|
/**
|
|
1484
|
-
*
|
|
1485
|
-
*
|
|
1486
|
-
*
|
|
1487
|
-
* host services, so every decision is unit-testable and audit-replayable.
|
|
1488
|
-
*
|
|
1489
|
-
* The cost model follows the TokenPilot paper's cache-accounting view: one
|
|
1490
|
-
* merged mutation pays a one-time tail KV-cache refill penalty of
|
|
1491
|
-
* `(1−α)·tailTokens`, and every later turn recovers the reclaimed tokens at
|
|
1492
|
-
* the cache-hit discount `α`:
|
|
1493
|
-
*
|
|
1494
|
-
* ```
|
|
1495
|
-
* R = Σ(tokensBefore − tokensAfter) // net reclaimed tokens
|
|
1496
|
-
* paybackTurns = (1−α)·tailTokens / (α·R) // one-time refill / per-turn saving
|
|
1497
|
-
* expectedSaving = α·R·max(0, Ŝ − paybackTurns) // Ŝ = estimated remaining turns
|
|
1498
|
-
* ```
|
|
1499
|
-
*
|
|
1500
|
-
* `expectedSaving` is only produced when Ŝ is known (the estimator answered
|
|
1501
|
-
* with `expectedRemainingTurns`); it is never fabricated from a guess.
|
|
1502
|
-
*/
|
|
1503
|
-
/**
|
|
1504
|
-
* Aggregate the batch-level benefit of a set of reduction candidates.
|
|
1505
|
-
*
|
|
1506
|
-
* Individual candidates whose replacement would grow the context contribute
|
|
1507
|
-
* zero recovery (they never make a batch look better than dropping them).
|
|
1508
|
-
*/
|
|
1509
|
-
function computeBenefit(candidates, input) {
|
|
1510
|
-
const { alpha, tailTokens, remainingTurns } = input;
|
|
1511
|
-
let recoveredTokens = 0;
|
|
1512
|
-
for (const candidate of candidates) recoveredTokens += Math.max(0, candidate.tokensBefore - candidate.tokensAfter);
|
|
1513
|
-
const penaltyTokens = (1 - alpha) * tailTokens;
|
|
1514
|
-
const perTurnSaving = alpha * recoveredTokens;
|
|
1515
|
-
if (perTurnSaving <= 0) return remainingTurns === void 0 ? {
|
|
1516
|
-
recoveredTokens,
|
|
1517
|
-
penaltyTokens
|
|
1518
|
-
} : {
|
|
1519
|
-
recoveredTokens,
|
|
1520
|
-
penaltyTokens,
|
|
1521
|
-
expectedSaving: -penaltyTokens
|
|
1522
|
-
};
|
|
1523
|
-
const paybackTurns = penaltyTokens / perTurnSaving;
|
|
1524
|
-
if (remainingTurns === void 0) return {
|
|
1525
|
-
recoveredTokens,
|
|
1526
|
-
penaltyTokens,
|
|
1527
|
-
paybackTurns
|
|
1528
|
-
};
|
|
1529
|
-
return {
|
|
1530
|
-
recoveredTokens,
|
|
1531
|
-
penaltyTokens,
|
|
1532
|
-
paybackTurns,
|
|
1533
|
-
expectedSaving: perTurnSaving * Math.max(0, remainingTurns - paybackTurns)
|
|
1534
|
-
};
|
|
1535
|
-
}
|
|
1536
|
-
/**
|
|
1537
|
-
* Stable proposal identity: the sha-256 of the serialized item digests, cut to
|
|
1538
|
-
* 12 hex chars. Stable across re-enqueues of the same content so a repeated
|
|
1539
|
-
* classification cannot duplicate a pending proposal.
|
|
2015
|
+
* Require content evidence of HTML: enough lines carrying real markup tags
|
|
2016
|
+
* among a bounded prefix. Angle-bracket prose (TS generics, comparisons) does
|
|
2017
|
+
* not match the tag list, which is the misjudgment guard.
|
|
1540
2018
|
*/
|
|
1541
|
-
function
|
|
1542
|
-
const
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
2019
|
+
function looksLikeHtml(text) {
|
|
2020
|
+
const lines = splitLines(text);
|
|
2021
|
+
let tags = 0;
|
|
2022
|
+
for (const line of lines.slice(0, 400)) if (HTML_TAG_PATTERN.test(line)) {
|
|
2023
|
+
tags += 1;
|
|
2024
|
+
if (tags >= 3) return true;
|
|
2025
|
+
}
|
|
2026
|
+
return false;
|
|
1546
2027
|
}
|
|
2028
|
+
const HTML_DROPPED_OPEN = /<(script|style|noscript|svg|head)\b[^>]*>/i;
|
|
2029
|
+
const HTML_BLOCK_MIN_LINES = 2;
|
|
2030
|
+
const HTML_BLOCK_MAX_LINES = 8;
|
|
2031
|
+
const HTML_BLOCK_MIN_OCCURRENCES = 3;
|
|
2032
|
+
const HTML_TABLE_TAG_PATTERN = /<table\b|<tr\b|<th\b|<\/tr\b|<\/table\b/i;
|
|
1547
2033
|
/**
|
|
1548
|
-
*
|
|
1549
|
-
*
|
|
1550
|
-
*
|
|
2034
|
+
* Deterministic repeated-block folding for slimmed HTML (R4/RK-3): contiguous
|
|
2035
|
+
* runs of 2–8 non-table lines whose digit-normalized signature recurs ≥3 times
|
|
2036
|
+
* keep their first occurrence; every later occurrence becomes ONE counted
|
|
2037
|
+
* marker. Tables never fold, and different copy never shares a signature —
|
|
2038
|
+
* only counter/number drift does.
|
|
1551
2039
|
*/
|
|
1552
|
-
function
|
|
1553
|
-
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
2040
|
+
function foldRepeatedHtmlBlocks(slim, originalLineFor) {
|
|
2041
|
+
const signatureOf = (from, length) => {
|
|
2042
|
+
let signature = `${String(length)}|`;
|
|
2043
|
+
for (let position = from; position < from + length; position++) {
|
|
2044
|
+
const text = slim[position].text;
|
|
2045
|
+
if (HTML_TABLE_TAG_PATTERN.test(text)) return null;
|
|
2046
|
+
signature += `${text.replace(/\d+/g, "#").replace(/\s+/g, " ").trim()}\n`;
|
|
2047
|
+
}
|
|
2048
|
+
return signature;
|
|
2049
|
+
};
|
|
2050
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2051
|
+
for (let length = HTML_BLOCK_MIN_LINES; length <= HTML_BLOCK_MAX_LINES; length++) for (let start = 0; start + length <= slim.length; start++) {
|
|
2052
|
+
const signature = signatureOf(start, length);
|
|
2053
|
+
if (signature === null) continue;
|
|
2054
|
+
const bucket = counts.get(signature);
|
|
2055
|
+
if (bucket === void 0) counts.set(signature, {
|
|
2056
|
+
count: 1,
|
|
2057
|
+
first: start
|
|
2058
|
+
});
|
|
2059
|
+
else bucket.count += 1;
|
|
2060
|
+
}
|
|
2061
|
+
const result = [];
|
|
2062
|
+
let position = 0;
|
|
2063
|
+
while (position < slim.length) {
|
|
2064
|
+
let foldedLength = 0;
|
|
2065
|
+
let matched;
|
|
2066
|
+
for (let length = HTML_BLOCK_MAX_LINES; length >= HTML_BLOCK_MIN_LINES; length--) {
|
|
2067
|
+
if (position + length > slim.length) continue;
|
|
2068
|
+
const signature = signatureOf(position, length);
|
|
2069
|
+
const bucket = signature === null ? void 0 : counts.get(signature);
|
|
2070
|
+
if (bucket !== void 0 && bucket.count >= HTML_BLOCK_MIN_OCCURRENCES) {
|
|
2071
|
+
foldedLength = length;
|
|
2072
|
+
matched = bucket;
|
|
2073
|
+
break;
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
if (matched === void 0) {
|
|
2077
|
+
result.push({
|
|
2078
|
+
text: slim[position].text,
|
|
2079
|
+
index: slim[position].index
|
|
2080
|
+
});
|
|
2081
|
+
position += 1;
|
|
2082
|
+
continue;
|
|
2083
|
+
}
|
|
2084
|
+
if (matched.first === position) for (let offset = 0; offset < foldedLength; offset++) result.push({
|
|
2085
|
+
text: slim[position + offset].text,
|
|
2086
|
+
index: slim[position + offset].index
|
|
2087
|
+
});
|
|
2088
|
+
else {
|
|
2089
|
+
const firstLine = originalLineFor(slim[matched.first].index);
|
|
2090
|
+
result.push({
|
|
2091
|
+
text: `[×${String(matched.count)} repeated block, first at line ${String(firstLine)}]`,
|
|
2092
|
+
index: slim[position].index,
|
|
2093
|
+
marker: true
|
|
2094
|
+
});
|
|
2095
|
+
}
|
|
2096
|
+
position += foldedLength;
|
|
2097
|
+
}
|
|
2098
|
+
return result;
|
|
1559
2099
|
}
|
|
1560
2100
|
/**
|
|
1561
|
-
*
|
|
2101
|
+
* Two-stage HTML reduction (R13). HTML previously fell into `pi-head`, which
|
|
2102
|
+
* keeps exactly the useless `<head>` metadata and drops the body.
|
|
1562
2103
|
*
|
|
1563
|
-
*
|
|
1564
|
-
*
|
|
1565
|
-
*
|
|
1566
|
-
*
|
|
1567
|
-
*
|
|
1568
|
-
*
|
|
1569
|
-
*
|
|
2104
|
+
* Stage 1 (`html-slim`) is a deterministic, line-aligned slimming pass:
|
|
2105
|
+
* comments, script/style/noscript/svg/head elements (single- or multi-line),
|
|
2106
|
+
* data URIs, non-whitelisted attributes, and inline-tag markup disappear;
|
|
2107
|
+
* every surviving line keeps its original-event position for the R9 ranges.
|
|
2108
|
+
* Stage 2 (`html-skeleton`) runs only when the slim output still exceeds the
|
|
2109
|
+
* budget: heading hierarchy, each section's first line, and table header rows
|
|
2110
|
+
* survive; the rest is elided with original-event line ranges.
|
|
1570
2111
|
*/
|
|
1571
|
-
function
|
|
1572
|
-
const
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
2112
|
+
function reduceHtml(input) {
|
|
2113
|
+
const slim = [];
|
|
2114
|
+
let dropping = null;
|
|
2115
|
+
input.lines.forEach((line, index) => {
|
|
2116
|
+
let text = line.content;
|
|
2117
|
+
if (dropping !== null) {
|
|
2118
|
+
const close = new RegExp(`</${dropping}\\s*>`, "i").exec(text);
|
|
2119
|
+
if (close === null) return;
|
|
2120
|
+
text = text.slice(close.index + close[0].length);
|
|
2121
|
+
dropping = null;
|
|
1580
2122
|
}
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
2123
|
+
text = text.replace(HTML_COMMENT_PATTERN, "");
|
|
2124
|
+
text = text.replace(HTML_DROPPED_ELEMENTS, "");
|
|
2125
|
+
const open = HTML_DROPPED_OPEN.exec(text);
|
|
2126
|
+
if (open !== null) {
|
|
2127
|
+
const close = new RegExp(`</${open[1] ?? ""}\\s*>`, "i").exec(text.slice(open.index));
|
|
2128
|
+
if (close !== null) {
|
|
2129
|
+
const end = open.index + open[0].length + close.index + close[0].length;
|
|
2130
|
+
text = text.slice(0, open.index) + text.slice(end);
|
|
2131
|
+
} else {
|
|
2132
|
+
dropping = open[1] ?? null;
|
|
2133
|
+
text = text.slice(0, open.index);
|
|
1591
2134
|
}
|
|
1592
2135
|
}
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
};
|
|
1601
|
-
review.push({
|
|
1602
|
-
id: proposalId([item.digest]),
|
|
1603
|
-
kind: item.kind,
|
|
1604
|
-
items: [item],
|
|
1605
|
-
benefit
|
|
2136
|
+
text = text.replace(HTML_DATA_URI_PATTERN, "");
|
|
2137
|
+
text = text.replace(HTML_TAG_PATTERN_FULL, (match, name, attrs) => `<${name}${attrs.match(HTML_WHITELISTED_ATTRIBUTES)?.join("") ?? ""}>`);
|
|
2138
|
+
text = text.replace(HTML_INLINE_TAG_PATTERN, "");
|
|
2139
|
+
text = text.trim();
|
|
2140
|
+
if (text !== "") slim.push({
|
|
2141
|
+
text,
|
|
2142
|
+
index
|
|
1606
2143
|
});
|
|
1607
|
-
}
|
|
1608
|
-
return
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
2144
|
+
});
|
|
2145
|
+
if (slim.length === 0) return null;
|
|
2146
|
+
const buildHeader = (reducer, firstElided) => {
|
|
2147
|
+
const startLine = firstElided === void 0 ? "" : `,"start_line":${String(firstElided.start)},"max_lines":${String(RETRIEVE_HINT_MAX_LINES)}`;
|
|
2148
|
+
return `[html compressed by ${reducer}; source: ${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}"${startLine}})]`;
|
|
1612
2149
|
};
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
return this.sessions.get(sessionId);
|
|
1621
|
-
}
|
|
1622
|
-
save(sessionId, record) {
|
|
1623
|
-
this.sessions.set(sessionId, record);
|
|
1624
|
-
}
|
|
1625
|
-
ids() {
|
|
1626
|
-
return [...this.sessions.keys()];
|
|
1627
|
-
}
|
|
1628
|
-
};
|
|
1629
|
-
var ReviewQueue = class {
|
|
1630
|
-
store;
|
|
1631
|
-
options;
|
|
1632
|
-
constructor(store, options) {
|
|
1633
|
-
this.store = store;
|
|
1634
|
-
this.options = options;
|
|
1635
|
-
}
|
|
1636
|
-
/**
|
|
1637
|
-
* Fail-open store access: a throwing seam must never break the compression
|
|
1638
|
-
* pipeline. Reads degrade to "no stored record"; writes degrade to losing
|
|
1639
|
-
* durability for that call (the store itself is expected to warn).
|
|
1640
|
-
*/
|
|
1641
|
-
safeLoad(sessionId) {
|
|
1642
|
-
try {
|
|
1643
|
-
return this.store.load(sessionId);
|
|
1644
|
-
} catch {
|
|
1645
|
-
return;
|
|
1646
|
-
}
|
|
1647
|
-
}
|
|
1648
|
-
safeSave(sessionId, record) {
|
|
1649
|
-
try {
|
|
1650
|
-
this.store.save(sessionId, record);
|
|
1651
|
-
} catch {}
|
|
1652
|
-
}
|
|
1653
|
-
sessionRecord(sessionId) {
|
|
1654
|
-
return this.safeLoad(sessionId) ?? {
|
|
1655
|
-
version: 1,
|
|
1656
|
-
proposals: []
|
|
2150
|
+
const foldedSlim = foldRepeatedHtmlBlocks(slim, (index) => input.lines[index]?.originalLine ?? 1);
|
|
2151
|
+
if (foldedSlim.reduce((sum, line) => sum + codePointLength(line.text) + 1, 0) + 160 <= input.budgetChars) {
|
|
2152
|
+
const text = fitLines([buildHeader("html-slim"), ...foldedSlim.map((line) => line.text)], input.budgetChars, input.sourceRef);
|
|
2153
|
+
if (text !== null) return {
|
|
2154
|
+
text,
|
|
2155
|
+
reducer: "html-slim",
|
|
2156
|
+
lossy: true
|
|
1657
2157
|
};
|
|
1658
2158
|
}
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
const existing = record.proposals.find((entry) => entry.id === skeleton.id);
|
|
1668
|
-
if (existing !== void 0 && existing.status !== "expired") {
|
|
1669
|
-
existing.lastTurnIndex = turnIndex;
|
|
1670
|
-
this.safeSave(sessionId, record);
|
|
1671
|
-
return false;
|
|
2159
|
+
const keep = new Array(foldedSlim.length).fill(false);
|
|
2160
|
+
let tableRows = 0;
|
|
2161
|
+
let lastHeading = -2;
|
|
2162
|
+
for (let position = 0; position < foldedSlim.length; position++) {
|
|
2163
|
+
const text = foldedSlim[position].text;
|
|
2164
|
+
if (foldedSlim[position].marker === true) {
|
|
2165
|
+
keep[position] = true;
|
|
2166
|
+
continue;
|
|
1672
2167
|
}
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
}
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
/** Live pending proposals of one session, oldest enqueue first. */
|
|
1690
|
-
listPending(sessionId) {
|
|
1691
|
-
return this.sessionRecord(sessionId).proposals.filter((entry) => entry.status === "pending").sort((left, right) => left.enqueuedTurn - right.enqueuedTurn);
|
|
1692
|
-
}
|
|
1693
|
-
/** Approved proposals waiting for the next turn-boundary batch. */
|
|
1694
|
-
listApproved(sessionId) {
|
|
1695
|
-
return this.sessionRecord(sessionId).proposals.filter((entry) => entry.status === "approved").sort((left, right) => left.enqueuedTurn - right.enqueuedTurn);
|
|
2168
|
+
if (/<h[1-6]\b/i.test(text)) {
|
|
2169
|
+
keep[position] = true;
|
|
2170
|
+
lastHeading = position;
|
|
2171
|
+
continue;
|
|
2172
|
+
}
|
|
2173
|
+
if (lastHeading === position - 1) {
|
|
2174
|
+
keep[position] = true;
|
|
2175
|
+
continue;
|
|
2176
|
+
}
|
|
2177
|
+
if (/<table\b|<tr\b|<th\b/i.test(text)) {
|
|
2178
|
+
if (tableRows < 2) keep[position] = true;
|
|
2179
|
+
tableRows += 1;
|
|
2180
|
+
continue;
|
|
2181
|
+
}
|
|
2182
|
+
if (!/<\/(tr|table)\b/i.test(text)) tableRows = 0;
|
|
2183
|
+
if (IMPORTANT_PATTERN.test(text)) keep[position] = true;
|
|
1696
2184
|
}
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
2185
|
+
const kept = [];
|
|
2186
|
+
let position = 0;
|
|
2187
|
+
let firstElided;
|
|
2188
|
+
let elidedLines = 0;
|
|
2189
|
+
while (position < foldedSlim.length) {
|
|
2190
|
+
if (keep[position]) {
|
|
2191
|
+
kept.push(foldedSlim[position].text);
|
|
2192
|
+
position += 1;
|
|
2193
|
+
continue;
|
|
2194
|
+
}
|
|
2195
|
+
const runStart = position;
|
|
2196
|
+
while (position < foldedSlim.length && !keep[position]) position += 1;
|
|
2197
|
+
const start = input.lines[foldedSlim[runStart].index].originalLine;
|
|
2198
|
+
const end = originalEnd(input.lines, foldedSlim[position - 1].index);
|
|
2199
|
+
if (firstElided === void 0) firstElided = start;
|
|
2200
|
+
elidedLines += end - start + 1;
|
|
2201
|
+
kept.push(elidedRangeMarker(start, end));
|
|
2202
|
+
}
|
|
2203
|
+
const text = fitLines([buildHeader("html-skeleton", firstElided === void 0 ? void 0 : { start: firstElided }), ...kept], input.budgetChars, input.sourceRef);
|
|
2204
|
+
return text === null ? null : {
|
|
2205
|
+
text,
|
|
2206
|
+
reducer: "html-skeleton",
|
|
2207
|
+
lossy: true,
|
|
2208
|
+
elidedLines
|
|
2209
|
+
};
|
|
2210
|
+
}
|
|
2211
|
+
/** Original-event end line of folded entry `lines[index]`. */
|
|
2212
|
+
function originalEnd(lines, index) {
|
|
2213
|
+
const line = lines[index];
|
|
2214
|
+
return line?.originalLineEnd ?? line?.originalLine ?? 0;
|
|
2215
|
+
}
|
|
2216
|
+
/**
|
|
2217
|
+
* Keep a document skeleton: the heading hierarchy, each section's first and
|
|
2218
|
+
* last content line, list-item starts, table headers, and fence markers,
|
|
2219
|
+
* eliding the remaining bodies with R9 line-range markers. Fails open (null)
|
|
2220
|
+
* when nothing is elidable or the budget cannot be met, so the next candidate
|
|
2221
|
+
* takes over.
|
|
2222
|
+
*/
|
|
2223
|
+
function reduceDocSkeleton(input, sectionRanking) {
|
|
2224
|
+
const lines = input.lines;
|
|
2225
|
+
const keep = new Array(lines.length).fill(false);
|
|
2226
|
+
const headingIndex = [];
|
|
2227
|
+
let inFence = false;
|
|
2228
|
+
for (let index = 0; index < lines.length; index++) {
|
|
2229
|
+
const line = lines[index].content;
|
|
2230
|
+
if (FENCE_PATTERN.test(line)) {
|
|
2231
|
+
inFence = !inFence;
|
|
2232
|
+
keep[index] = true;
|
|
2233
|
+
continue;
|
|
2234
|
+
}
|
|
2235
|
+
if (!inFence && MARKDOWN_HEADING_PATTERN.test(line)) {
|
|
2236
|
+
headingIndex.push(index);
|
|
2237
|
+
keep[index] = true;
|
|
2238
|
+
continue;
|
|
2239
|
+
}
|
|
2240
|
+
if (IMPORTANT_PATTERN.test(line)) keep[index] = true;
|
|
2241
|
+
else if (!inFence && LIST_ITEM_PATTERN.test(line)) keep[index] = true;
|
|
1715
2242
|
}
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
if (
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
2243
|
+
let tableRows = 0;
|
|
2244
|
+
let fenceOpen = false;
|
|
2245
|
+
for (let index = 0; index < lines.length; index++) {
|
|
2246
|
+
const line = lines[index].content;
|
|
2247
|
+
if (FENCE_PATTERN.test(line)) {
|
|
2248
|
+
fenceOpen = !fenceOpen;
|
|
2249
|
+
tableRows = 0;
|
|
2250
|
+
continue;
|
|
2251
|
+
}
|
|
2252
|
+
if (fenceOpen || line.trim() === "") continue;
|
|
2253
|
+
if (TABLE_ROW_PATTERN.test(line)) {
|
|
2254
|
+
if (tableRows < 2) keep[index] = true;
|
|
2255
|
+
tableRows += 1;
|
|
2256
|
+
continue;
|
|
2257
|
+
}
|
|
2258
|
+
tableRows = 0;
|
|
2259
|
+
}
|
|
2260
|
+
const sectionStarts = [-1, ...headingIndex];
|
|
2261
|
+
const sectionEnds = [...headingIndex, lines.length];
|
|
2262
|
+
const sections = headingIndex.map((heading, position) => ({
|
|
2263
|
+
id: lines[heading].content.replace(/^#+\s*/, "").trim(),
|
|
2264
|
+
heading,
|
|
2265
|
+
from: heading + 1,
|
|
2266
|
+
to: position + 1 < headingIndex.length ? headingIndex[position + 1] : lines.length
|
|
2267
|
+
}));
|
|
2268
|
+
const floorKeep = new Array(lines.length).fill(false);
|
|
2269
|
+
for (let section = 0; section < sectionStarts.length; section++) {
|
|
2270
|
+
const from = sectionStarts[section] + 1;
|
|
2271
|
+
const to = sectionEnds[section];
|
|
2272
|
+
let first = -1;
|
|
2273
|
+
let last = -1;
|
|
2274
|
+
for (let index = from; index < to; index++) {
|
|
2275
|
+
if (lines[index].content.trim() === "") continue;
|
|
2276
|
+
if (first === -1) first = index;
|
|
2277
|
+
last = index;
|
|
2278
|
+
}
|
|
2279
|
+
if (first !== -1) floorKeep[first] = true;
|
|
2280
|
+
if (last !== -1) floorKeep[last] = true;
|
|
2281
|
+
}
|
|
2282
|
+
/** Emit the skeleton for one keep-set: header, kept lines, R9 range markers. */
|
|
2283
|
+
const assemble = (flags, budget) => {
|
|
2284
|
+
const kept = [];
|
|
2285
|
+
let index = 0;
|
|
2286
|
+
let firstElided;
|
|
2287
|
+
let elidedLines = 0;
|
|
2288
|
+
while (index < lines.length) {
|
|
2289
|
+
if (flags[index]) {
|
|
2290
|
+
kept.push(lines[index].text);
|
|
2291
|
+
index += 1;
|
|
1732
2292
|
continue;
|
|
1733
2293
|
}
|
|
1734
|
-
|
|
2294
|
+
const runStart = index;
|
|
2295
|
+
while (index < lines.length && !flags[index]) index += 1;
|
|
2296
|
+
const start = lines[runStart].originalLine;
|
|
2297
|
+
const end = originalEnd(lines, index - 1);
|
|
2298
|
+
if (firstElided === void 0) firstElided = start;
|
|
2299
|
+
elidedLines += end - start + 1;
|
|
2300
|
+
kept.push(elidedRangeMarker(start, end));
|
|
1735
2301
|
}
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
proposals: keep
|
|
1739
|
-
});
|
|
1740
|
-
return expired;
|
|
1741
|
-
}
|
|
1742
|
-
/**
|
|
1743
|
-
* Settle an approved proposal with its execution receipt and retire it from
|
|
1744
|
-
* the live store. The caller is responsible for auditing the receipt; the
|
|
1745
|
-
* queue only records which proposal left and why.
|
|
1746
|
-
*/
|
|
1747
|
-
recordReceipt(sessionId, id, receipt) {
|
|
1748
|
-
const record = this.sessionRecord(sessionId);
|
|
1749
|
-
const proposal = record.proposals.find((entry) => entry.id === id);
|
|
1750
|
-
if (proposal === void 0 || proposal.status !== "approved") return void 0;
|
|
1751
|
-
this.safeSave(sessionId, {
|
|
1752
|
-
version: 1,
|
|
1753
|
-
proposals: record.proposals.filter((entry) => entry.id !== id)
|
|
1754
|
-
});
|
|
2302
|
+
const hint = firstElided === void 0 ? "" : `,"start_line":${String(firstElided)},"max_lines":${String(RETRIEVE_HINT_MAX_LINES)}`;
|
|
2303
|
+
kept.unshift(`[document compressed by doc-skeleton; source: ${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}"${hint}})]`);
|
|
1755
2304
|
return {
|
|
1756
|
-
|
|
1757
|
-
|
|
2305
|
+
text: fitLines(kept, budget, input.sourceRef),
|
|
2306
|
+
...firstElided === void 0 ? {} : { firstElided },
|
|
2307
|
+
elidedLines
|
|
1758
2308
|
};
|
|
1759
|
-
}
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
if (
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
2309
|
+
};
|
|
2310
|
+
const combine = (base, overlay) => lines.map((_, index) => (base[index] ?? false) || (overlay[index] ?? false));
|
|
2311
|
+
const mechanical = assemble(combine(keep, floorKeep), input.budgetChars);
|
|
2312
|
+
if (mechanical.text === null) return null;
|
|
2313
|
+
if (sectionRanking === void 0 || sectionRanking.length === 0) return {
|
|
2314
|
+
text: mechanical.text,
|
|
2315
|
+
reducer: "doc-skeleton",
|
|
2316
|
+
lossy: true,
|
|
2317
|
+
elidedLines: mechanical.elidedLines
|
|
2318
|
+
};
|
|
2319
|
+
const cap = Math.min(input.budgetChars, codePointLength(input.text) - 1);
|
|
2320
|
+
const rankedFloor = new Array(lines.length).fill(false);
|
|
2321
|
+
for (const section of sections) for (let index = section.from; index < section.to; index++) {
|
|
2322
|
+
if (lines[index].content.trim() === "") continue;
|
|
2323
|
+
rankedFloor[index] = true;
|
|
2324
|
+
break;
|
|
2325
|
+
}
|
|
2326
|
+
/** Exact packed-output size of a keep-set: header + kept lines + markers. */
|
|
2327
|
+
const packedSize = (flags) => {
|
|
2328
|
+
let size = 180;
|
|
2329
|
+
let index = 0;
|
|
2330
|
+
while (index < lines.length) {
|
|
2331
|
+
if (flags[index]) {
|
|
2332
|
+
size += codePointLength(lines[index].text) + 1;
|
|
2333
|
+
index += 1;
|
|
2334
|
+
continue;
|
|
1785
2335
|
}
|
|
2336
|
+
const runStart = index;
|
|
2337
|
+
while (index < lines.length && !flags[index]) index += 1;
|
|
2338
|
+
size += codePointLength(elidedRangeMarker(lines[runStart].originalLine, originalEnd(lines, index - 1))) + 1;
|
|
1786
2339
|
}
|
|
1787
|
-
return
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
2340
|
+
return size;
|
|
2341
|
+
};
|
|
2342
|
+
const fill = new Array(lines.length).fill(false);
|
|
2343
|
+
for (const section of rankedFirst(sections, sectionRanking)) for (let index = section.from; index < section.to; index++) {
|
|
2344
|
+
if (rankedFloor[index] || fill[index] || lines[index].content.trim() === "") continue;
|
|
2345
|
+
fill[index] = true;
|
|
2346
|
+
if (packedSize(combine(combine(keep, rankedFloor), fill)) > cap) {
|
|
2347
|
+
fill[index] = false;
|
|
2348
|
+
break;
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
const ranked = assemble(combine(combine(keep, rankedFloor), fill), cap);
|
|
2352
|
+
return ranked.text === null ? null : {
|
|
2353
|
+
text: ranked.text,
|
|
2354
|
+
reducer: "doc-skeleton",
|
|
2355
|
+
lossy: true,
|
|
2356
|
+
elidedLines: ranked.elidedLines
|
|
2357
|
+
};
|
|
1792
2358
|
}
|
|
1793
|
-
|
|
2359
|
+
/**
|
|
2360
|
+
* Universal prose fallback (R8b, the main force): keep the head AND the tail
|
|
2361
|
+
* of any non-code text and one R9 line-range marker for everything elided in
|
|
2362
|
+
* between. Unstructured prose (85%+ of large results) previously landed on
|
|
2363
|
+
* head-only truncation; a tail keep preserves conclusions and closing state.
|
|
2364
|
+
* Fails open for code-like text and when the budget cannot hold both ends.
|
|
2365
|
+
*/
|
|
2366
|
+
function reduceProseKeep(input) {
|
|
2367
|
+
const lines = input.lines;
|
|
2368
|
+
if (lines.length < 8) return null;
|
|
2369
|
+
if (looksLikeSourceCode(input.contentText)) return null;
|
|
2370
|
+
const first = lines[0];
|
|
2371
|
+
const last = lines[lines.length - 1];
|
|
2372
|
+
const tailLine = last.originalLineEnd ?? last.originalLine;
|
|
2373
|
+
const markerTemplate = elidedRangeMarker(first.originalLine, tailLine);
|
|
2374
|
+
const sourceNoteTemplate = `; source: ${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}"})`;
|
|
2375
|
+
const reserved = codePointLength(markerTemplate) + codePointLength(sourceNoteTemplate) + 2;
|
|
2376
|
+
const bodyBudget = input.budgetChars - reserved;
|
|
2377
|
+
if (bodyBudget <= 0) return null;
|
|
2378
|
+
const headBudget = Math.floor(bodyBudget / 2);
|
|
2379
|
+
const tailBudget = bodyBudget - headBudget;
|
|
2380
|
+
let headCount = 0;
|
|
2381
|
+
let used = 0;
|
|
2382
|
+
while (headCount < lines.length) {
|
|
2383
|
+
const cost = codePointLength(lines[headCount].text) + (headCount === 0 ? 0 : 1);
|
|
2384
|
+
if (used + cost > headBudget) break;
|
|
2385
|
+
used += cost;
|
|
2386
|
+
headCount += 1;
|
|
2387
|
+
}
|
|
2388
|
+
let tailCount = 0;
|
|
2389
|
+
used = 0;
|
|
2390
|
+
while (tailCount < lines.length - headCount) {
|
|
2391
|
+
const index = lines.length - 1 - tailCount;
|
|
2392
|
+
const cost = codePointLength(lines[index].text) + (tailCount === 0 ? 0 : 1);
|
|
2393
|
+
if (used + cost > tailBudget) break;
|
|
2394
|
+
used += cost;
|
|
2395
|
+
tailCount += 1;
|
|
2396
|
+
}
|
|
2397
|
+
if (headCount === 0 || tailCount === 0 || headCount + tailCount >= lines.length) return null;
|
|
2398
|
+
const elidedStart = lines[headCount].originalLine;
|
|
2399
|
+
const elidedEnd = originalEnd(lines, lines.length - tailCount - 1);
|
|
2400
|
+
if (elidedEnd < elidedStart) return null;
|
|
2401
|
+
const sourceNote = `; source: ${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}","start_line":${String(elidedStart)},"max_lines":${String(RETRIEVE_HINT_MAX_LINES)}})`;
|
|
1794
2402
|
return {
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
2403
|
+
text: [
|
|
2404
|
+
...lines.slice(0, headCount).map((line) => line.text),
|
|
2405
|
+
elidedRangeMarker(elidedStart, elidedEnd) + sourceNote,
|
|
2406
|
+
...lines.slice(lines.length - tailCount).map((line) => line.text)
|
|
2407
|
+
].join("\n"),
|
|
2408
|
+
reducer: "prose-keep",
|
|
2409
|
+
lossy: true,
|
|
2410
|
+
elidedLines: elidedEnd - elidedStart + 1
|
|
1799
2411
|
};
|
|
1800
2412
|
}
|
|
1801
|
-
/** Adapter presenting the sync KV face the queue expects over the domain table. */
|
|
1802
|
-
var StorageDomainReviewStore = class {
|
|
1803
|
-
table;
|
|
1804
|
-
constructor(table) {
|
|
1805
|
-
this.table = table;
|
|
1806
|
-
}
|
|
1807
|
-
load(sessionId) {
|
|
1808
|
-
const value = this.table.get(sessionId);
|
|
1809
|
-
return typeof value === "object" && value !== null ? value : void 0;
|
|
1810
|
-
}
|
|
1811
|
-
save(sessionId, record) {
|
|
1812
|
-
this.table.put(sessionId, record).catch(() => void 0);
|
|
1813
|
-
}
|
|
1814
|
-
ids() {
|
|
1815
|
-
return [...this.table.keys()];
|
|
1816
|
-
}
|
|
1817
|
-
};
|
|
1818
|
-
/**
|
|
1819
|
-
* Attempt to open the review storage domain through the optional
|
|
1820
|
-
* `storageDomain` seam.
|
|
1821
|
-
* @param getService - resolved once with the seam name; `undefined` means the
|
|
1822
|
-
* host lacks the service.
|
|
1823
|
-
* @returns the durable store, or `undefined` when the seam is absent or fails
|
|
1824
|
-
* (the caller falls back to the in-memory store and logs one warning).
|
|
1825
|
-
*/
|
|
1826
|
-
async function openReviewStorage(getService) {
|
|
1827
|
-
let service;
|
|
1828
|
-
try {
|
|
1829
|
-
service = getService("storageDomain");
|
|
1830
|
-
} catch {
|
|
1831
|
-
return;
|
|
1832
|
-
}
|
|
1833
|
-
if (service === void 0 || service === null) return void 0;
|
|
1834
|
-
return new StorageDomainReviewStore((await service.open(reviewStorageSpec())).table(REVIEW_STORAGE_TABLE));
|
|
1835
|
-
}
|
|
1836
|
-
//#endregion
|
|
1837
|
-
//#region src/runtime/reducers.ts
|
|
1838
|
-
/** Deterministic, evidence-backed reducers for fresh tool results. */
|
|
1839
|
-
const ANSI_PATTERN = /\u001B(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007]*(?:\u0007|\u001B\\))/gu;
|
|
1840
|
-
const IMPORTANT_PATTERN = new RegExp([
|
|
1841
|
-
String.raw`\b(?:error|failed|failure|fatal|panic|exception|warning|warn|conflict|denied|forbidden|`,
|
|
1842
|
-
String.raw`timeout|timed out|not found|cannot|unable|invalid|exit(?:ed)?\s+(?:code|status)|traceback|`,
|
|
1843
|
-
String.raw`assert(?:ion)?|segmentation fault|oom|out of memory)\b`
|
|
1844
|
-
].join(""), "i");
|
|
1845
|
-
const STATUS_PATTERN = new RegExp([String.raw`\b(?:success|succeeded|passed|installed|added|removed|updated|built|compiled|`, String.raw`tests?\s+(?:passed|failed)|exit(?:ed)?\s+(?:code|status))\b`].join(""), "i");
|
|
1846
|
-
const PATH_LINE_PATTERN = /^(.*?):(\d+)(?::\d+)?(?::|\s+-\s+)(.*)$/;
|
|
1847
|
-
const GIT_STATUS_PATTERN = new RegExp([String.raw`^(?:On branch|Your branch|HEAD detached|Changes |Untracked |Unmerged |\s*(?:modified|deleted|`, String.raw`new file|renamed|both modified):)`].join(""), "i");
|
|
1848
|
-
const CODE_IMPORT_PATTERN = new RegExp([String.raw`^\s*(?:import\b|from\s+[\w.]+\s+import\b|use\s+\w|package\s+|#include\b|`, String.raw`using\s+[\w.]+;|require\s*\(|extern\s+crate\b)`].join(""));
|
|
1849
|
-
const CODE_STRUCTURE_PATTERN = new RegExp([
|
|
1850
|
-
String.raw`^\s*(?:@[\w.]+|export\s+|default\s+|declare\s+|abstract\s+|public\s+|private\s+|protected\s+|`,
|
|
1851
|
-
String.raw`internal\s+|static\s+|final\s+|sealed\s+|override\s+|pub(?:\([^)]*\))?\s+|async\s+|unsafe\s+)*`,
|
|
1852
|
-
String.raw`(?:function\b|class\b|interface\b|enum\b|struct\b|impl\b|trait\b|type\s+\w|fn\s|func\b|`,
|
|
1853
|
-
String.raw`def\s|module\b|namespace\b|sub\s)`
|
|
1854
|
-
].join(""));
|
|
1855
|
-
const PYTHON_STRUCTURE_PATTERN = /^\s*(?:async\s+)?def\s|^\s*class\s/;
|
|
1856
|
-
const CODE_DECORATOR_PATTERN = /^\s*@[\w.]+/;
|
|
1857
|
-
const CODE_COMMENT_PATTERN = /^\s*(?:\/\/|#|\/\*|\*)/;
|
|
1858
2413
|
/**
|
|
1859
|
-
*
|
|
2414
|
+
* Keep a source-file skeleton: imports, decorators, declaration signatures,
|
|
2415
|
+
* comments at brace depth zero, and every error-signalling line, eliding the
|
|
2416
|
+
* remaining bodies with counted markers. Covers brace languages (TS/JS, Rust,
|
|
2417
|
+
* Go, Java, C family) and indent blocks (Python); unknown syntax fails open to
|
|
2418
|
+
* the next candidate. Output is compressed evidence, not required to parse.
|
|
1860
2419
|
* @param input - original result text, recovery source, and output budget.
|
|
1861
|
-
* @returns a verified candidate, or `null` when
|
|
2420
|
+
* @returns a verified candidate, or `null` when the text is not code-like.
|
|
1862
2421
|
*/
|
|
1863
|
-
function
|
|
1864
|
-
const
|
|
1865
|
-
const
|
|
1866
|
-
|
|
1867
|
-
|
|
2422
|
+
function reduceCodeSkeleton(input) {
|
|
2423
|
+
const lines = input.lines.map((line) => line.content);
|
|
2424
|
+
const kept = [];
|
|
2425
|
+
let elided = 0;
|
|
2426
|
+
let elidedTotal = 0;
|
|
2427
|
+
let firstElided;
|
|
2428
|
+
const flushElided = () => {
|
|
2429
|
+
if (elided > 0) {
|
|
2430
|
+
const start = input.lines[index - elided]?.originalLine ?? 0;
|
|
2431
|
+
const end = originalEnd(input.lines, index - 1);
|
|
2432
|
+
if (firstElided === void 0) firstElided = {
|
|
2433
|
+
start,
|
|
2434
|
+
end
|
|
2435
|
+
};
|
|
2436
|
+
kept.push(elidedRangeMarker(start, end));
|
|
2437
|
+
elidedTotal += elided;
|
|
2438
|
+
}
|
|
2439
|
+
elided = 0;
|
|
1868
2440
|
};
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
const
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
return null;
|
|
1886
|
-
}
|
|
1887
|
-
/**
|
|
1888
|
-
* Build a recoverable placeholder for an old tool result.
|
|
1889
|
-
* @param input - tool identity, source reference, size, status, and retained evidence.
|
|
1890
|
-
* @returns a lossy placeholder that cites the immutable source event.
|
|
1891
|
-
*/
|
|
1892
|
-
function historicalPlaceholder(input) {
|
|
1893
|
-
const anchor = input.compact ? "" : importantAnchor(input.text, 360);
|
|
1894
|
-
const lines = [
|
|
1895
|
-
"[Old tool result content cleared from active context]",
|
|
1896
|
-
`tool: ${input.toolName || "unknown"}`,
|
|
1897
|
-
`status: ${input.isError ? "error" : "completed"}`,
|
|
1898
|
-
`original_chars: ${String(input.charsBefore)}`,
|
|
1899
|
-
`source: ${input.sourceRef}`,
|
|
1900
|
-
"retrieve: context_compression_retrieve({\"ref\":\"" + input.sourceRef + "\"})"
|
|
1901
|
-
];
|
|
1902
|
-
if (anchor !== "") lines.push(`retained_anchor: ${anchor}`);
|
|
1903
|
-
return {
|
|
1904
|
-
text: lines.join("\n"),
|
|
1905
|
-
reducer: input.compact ? "pair-preserving-tail-aging" : "historical-tool-result-aging",
|
|
1906
|
-
lossy: true
|
|
2441
|
+
let depth = 0;
|
|
2442
|
+
let index = 0;
|
|
2443
|
+
const elideBraceBody = () => {
|
|
2444
|
+
const startDepth = depth;
|
|
2445
|
+
index += 1;
|
|
2446
|
+
while (index < lines.length && depth > startDepth) {
|
|
2447
|
+
const body = lines[index];
|
|
2448
|
+
if (body === void 0) break;
|
|
2449
|
+
if (IMPORTANT_PATTERN.test(body)) {
|
|
2450
|
+
flushElided();
|
|
2451
|
+
kept.push(body);
|
|
2452
|
+
} else elided += 1;
|
|
2453
|
+
depth += braceDelta(body);
|
|
2454
|
+
index += 1;
|
|
2455
|
+
}
|
|
2456
|
+
flushElided();
|
|
1907
2457
|
};
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
*/
|
|
1928
|
-
function normalizeTerminalText(text) {
|
|
1929
|
-
const logical = text.replace(ANSI_PATTERN, "").split("\n").map((line) => {
|
|
1930
|
-
return line.split("\r").filter((part) => part !== "").at(-1) ?? "";
|
|
1931
|
-
});
|
|
1932
|
-
const folded = [];
|
|
1933
|
-
let previous;
|
|
1934
|
-
let count = 0;
|
|
1935
|
-
const flush = () => {
|
|
1936
|
-
if (previous === void 0) return;
|
|
1937
|
-
folded.push(previous);
|
|
1938
|
-
if (count > 1) folded.push(`[previous line repeated ${String(count - 1)} more times]`);
|
|
2458
|
+
const keepPythonSignature = (signatureLine) => {
|
|
2459
|
+
index += 1;
|
|
2460
|
+
if (/:\s*$/.test(signatureLine)) {
|
|
2461
|
+
elideIndentedBody(leadingIndent(signatureLine));
|
|
2462
|
+
return;
|
|
2463
|
+
}
|
|
2464
|
+
for (let guard = 0; guard < 6 && index < lines.length; guard += 1) {
|
|
2465
|
+
const next = lines[index];
|
|
2466
|
+
if (next === void 0) break;
|
|
2467
|
+
if (next.trim() !== "" && leadingIndent(next) <= leadingIndent(signatureLine)) break;
|
|
2468
|
+
flushElided();
|
|
2469
|
+
kept.push(next);
|
|
2470
|
+
index += 1;
|
|
2471
|
+
if (/:\s*$/.test(next)) {
|
|
2472
|
+
elideIndentedBody(leadingIndent(next));
|
|
2473
|
+
return;
|
|
2474
|
+
}
|
|
2475
|
+
if (next.trim() !== "" && !/[:,(]\s*$/.test(next)) break;
|
|
2476
|
+
}
|
|
1939
2477
|
};
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
2478
|
+
const elideIndentedBody = (indent) => {
|
|
2479
|
+
while (index < lines.length) {
|
|
2480
|
+
const body = lines[index];
|
|
2481
|
+
if (body === void 0) break;
|
|
2482
|
+
if (body.trim() !== "" && leadingIndent(body) <= indent) break;
|
|
2483
|
+
if (IMPORTANT_PATTERN.test(body)) {
|
|
2484
|
+
flushElided();
|
|
2485
|
+
kept.push(body);
|
|
2486
|
+
index += 1;
|
|
2487
|
+
continue;
|
|
2488
|
+
}
|
|
2489
|
+
if (isCodeStructureLine(body) || CODE_DECORATOR_PATTERN.test(body)) {
|
|
2490
|
+
flushElided();
|
|
2491
|
+
kept.push(body);
|
|
2492
|
+
keepPythonSignature(body);
|
|
2493
|
+
continue;
|
|
2494
|
+
}
|
|
2495
|
+
elided += 1;
|
|
2496
|
+
index += 1;
|
|
2497
|
+
}
|
|
2498
|
+
flushElided();
|
|
2499
|
+
};
|
|
2500
|
+
while (index < lines.length) {
|
|
2501
|
+
const line = lines[index];
|
|
2502
|
+
if (line === void 0) break;
|
|
2503
|
+
const delta = braceDelta(line);
|
|
2504
|
+
if (IMPORTANT_PATTERN.test(line)) {
|
|
2505
|
+
flushElided();
|
|
2506
|
+
kept.push(line);
|
|
2507
|
+
depth += delta;
|
|
2508
|
+
index += 1;
|
|
1943
2509
|
continue;
|
|
1944
2510
|
}
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
2511
|
+
if (isCodeStructureLine(line) || CODE_IMPORT_PATTERN.test(line) || CODE_DECORATOR_PATTERN.test(line)) {
|
|
2512
|
+
flushElided();
|
|
2513
|
+
kept.push(line);
|
|
2514
|
+
depth += delta;
|
|
2515
|
+
if (delta > 0) {
|
|
2516
|
+
elideBraceBody();
|
|
2517
|
+
continue;
|
|
2518
|
+
}
|
|
2519
|
+
if (PYTHON_STRUCTURE_PATTERN.test(line)) {
|
|
2520
|
+
keepPythonSignature(line);
|
|
2521
|
+
continue;
|
|
2522
|
+
}
|
|
2523
|
+
let opened = false;
|
|
2524
|
+
for (let guard = 0; guard < 6 && index + 1 < lines.length; guard += 1) {
|
|
2525
|
+
const next = lines[index + 1];
|
|
2526
|
+
if (next === void 0) break;
|
|
2527
|
+
const nextDelta = braceDelta(next);
|
|
2528
|
+
if (nextDelta === 0 && next.trim() !== "" && !/[:,(]\s*$/.test(next)) break;
|
|
2529
|
+
flushElided();
|
|
2530
|
+
kept.push(next);
|
|
2531
|
+
depth += nextDelta;
|
|
2532
|
+
index += 1;
|
|
2533
|
+
if (nextDelta > 0) {
|
|
2534
|
+
opened = true;
|
|
2535
|
+
break;
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
if (opened) elideBraceBody();
|
|
2539
|
+
else index += 1;
|
|
2540
|
+
continue;
|
|
2541
|
+
}
|
|
2542
|
+
if (depth === 0 && CODE_COMMENT_PATTERN.test(line)) {
|
|
2543
|
+
flushElided();
|
|
2544
|
+
kept.push(line);
|
|
2545
|
+
} else elided += 1;
|
|
2546
|
+
depth += delta;
|
|
2547
|
+
index += 1;
|
|
1948
2548
|
}
|
|
1949
|
-
|
|
1950
|
-
return
|
|
2549
|
+
flushElided();
|
|
2550
|
+
return finishSkeleton(kept, lines, input, firstElided, elidedTotal);
|
|
1951
2551
|
}
|
|
1952
|
-
function
|
|
1953
|
-
const
|
|
1954
|
-
const
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
2552
|
+
function finishSkeleton(kept, lines, input, firstElided, elidedLines) {
|
|
2553
|
+
const hint = firstElided === void 0 ? "" : `; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}","start_line":${String(firstElided.start)},"max_lines":${String(RETRIEVE_HINT_MAX_LINES)}})`;
|
|
2554
|
+
const text = fitLines([
|
|
2555
|
+
`[code output compressed by hypa-code-skeleton; source: ${input.sourceRef}${hint}]`,
|
|
2556
|
+
...kept,
|
|
2557
|
+
...lines.slice(-4)
|
|
2558
|
+
], input.budgetChars, input.sourceRef);
|
|
2559
|
+
return text === null ? null : {
|
|
2560
|
+
text,
|
|
2561
|
+
reducer: "hypa-code-skeleton",
|
|
2562
|
+
lossy: true,
|
|
2563
|
+
elidedLines
|
|
1962
2564
|
};
|
|
1963
2565
|
}
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
2566
|
+
/** Net brace delta of one line, ignoring braces inside string literals. */
|
|
2567
|
+
function braceDelta(line) {
|
|
2568
|
+
let delta = 0;
|
|
2569
|
+
let quote = null;
|
|
2570
|
+
for (let position = 0; position < line.length; position += 1) {
|
|
2571
|
+
const char = line[position];
|
|
2572
|
+
if (quote !== null) {
|
|
2573
|
+
if (char === "\\") position += 1;
|
|
2574
|
+
else if (char === quote) quote = null;
|
|
2575
|
+
continue;
|
|
2576
|
+
}
|
|
2577
|
+
if (char === "\"" || char === "'" || char === "`") {
|
|
2578
|
+
quote = char;
|
|
2579
|
+
continue;
|
|
2580
|
+
}
|
|
2581
|
+
if (char === "{") delta += 1;
|
|
2582
|
+
else if (char === "}") delta -= 1;
|
|
2583
|
+
}
|
|
2584
|
+
return delta;
|
|
1975
2585
|
}
|
|
1976
|
-
function
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
2586
|
+
function leadingIndent(line) {
|
|
2587
|
+
return codePointLength(line) - codePointLength(line.trimStart());
|
|
2588
|
+
}
|
|
2589
|
+
function isCodeStructureLine(line) {
|
|
2590
|
+
return CODE_STRUCTURE_PATTERN.test(line) || PYTHON_STRUCTURE_PATTERN.test(line);
|
|
2591
|
+
}
|
|
2592
|
+
/**
|
|
2593
|
+
* Require content evidence of source code: enough declaration, import, or
|
|
2594
|
+
* decorator lines among a bounded prefix. Failing this keeps prose, logs, and
|
|
2595
|
+
* data on their existing reducers.
|
|
2596
|
+
* @param text - normalized result text.
|
|
2597
|
+
* @returns whether the text qualifies as source code.
|
|
2598
|
+
*/
|
|
2599
|
+
function looksLikeSourceCode(text) {
|
|
2600
|
+
const lines = splitLines(text);
|
|
2601
|
+
if (lines.length < 12) return false;
|
|
2602
|
+
let evidence = 0;
|
|
2603
|
+
for (const line of lines.slice(0, 400)) if (isCodeStructureLine(line) || CODE_IMPORT_PATTERN.test(line) || CODE_DECORATOR_PATTERN.test(line)) {
|
|
2604
|
+
evidence += 1;
|
|
2605
|
+
if (evidence >= 3) return true;
|
|
1982
2606
|
}
|
|
1983
|
-
|
|
1984
|
-
if (codePointLength(minified) < codePointLength(input.text) && codePointLength(minified) <= input.budgetChars) return {
|
|
1985
|
-
text: minified,
|
|
1986
|
-
reducer: "json-minify",
|
|
1987
|
-
lossy: false
|
|
1988
|
-
};
|
|
1989
|
-
const envelope = {
|
|
1990
|
-
$dsh_compression: {
|
|
1991
|
-
kind: "json-preview",
|
|
1992
|
-
source: input.sourceRef,
|
|
1993
|
-
original_chars: codePointLength(input.text)
|
|
1994
|
-
},
|
|
1995
|
-
value: shrinkJson(value, 0)
|
|
1996
|
-
};
|
|
1997
|
-
const text = JSON.stringify(envelope, null, 2);
|
|
1998
|
-
if (codePointLength(text) <= input.budgetChars) return {
|
|
1999
|
-
text,
|
|
2000
|
-
reducer: "json-structure-preview",
|
|
2001
|
-
lossy: true
|
|
2002
|
-
};
|
|
2003
|
-
return null;
|
|
2607
|
+
return false;
|
|
2004
2608
|
}
|
|
2005
|
-
function
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2609
|
+
function omissionMarker(input, reducer) {
|
|
2610
|
+
return `[... ${reducer} omitted content; original_chars=${String(codePointLength(input.text))}; source=${input.sourceRef}; retrieve with context_compression_retrieve ...]`;
|
|
2611
|
+
}
|
|
2612
|
+
/**
|
|
2613
|
+
* Continuous-mask marker (R9b): an original-event line range plus a pasteable
|
|
2614
|
+
* retrieve hint whose start_line is the first elided line. Line numbers point
|
|
2615
|
+
* at the RAW event because retrieve reads raw events (D8).
|
|
2616
|
+
*/
|
|
2617
|
+
function rangeOmissionMarker(input, reducer, elidedStart, elidedEnd) {
|
|
2618
|
+
return `[... lines ${String(elidedStart)}-${String(elidedEnd)} elided (${String(elidedEnd - elidedStart + 1)} lines); ${reducer}; original_chars=${String(codePointLength(input.text))}; source=${input.sourceRef}; retrieve with context_compression_retrieve({"ref":"${input.sourceRef}","start_line":${String(elidedStart)},"max_lines":${String(RETRIEVE_HINT_MAX_LINES)}}) ...]`;
|
|
2619
|
+
}
|
|
2620
|
+
/** Scatter-mask note (R9b): totals instead of fragmented per-run ranges. */
|
|
2621
|
+
function scannedTotals(input, kept) {
|
|
2622
|
+
const last = input.lines[input.lines.length - 1];
|
|
2623
|
+
const lastLine = Math.max(last?.originalLineEnd ?? 0, last?.originalLine ?? 0, input.lines.length);
|
|
2624
|
+
return `lines 1-${String(lastLine)} scanned, ${String(kept)} kept`;
|
|
2625
|
+
}
|
|
2626
|
+
function importantAnchor(text, maxChars) {
|
|
2627
|
+
const lines = splitLines(normalizeTerminalText(text));
|
|
2628
|
+
const chosen = lines.find((line) => IMPORTANT_PATTERN.test(line)) ?? lines.at(-1) ?? "";
|
|
2629
|
+
return Array.from(chosen.trim()).slice(0, maxChars).join("");
|
|
2630
|
+
}
|
|
2631
|
+
function fitLines(lines, budgetChars, requiredRef) {
|
|
2632
|
+
const unique = [];
|
|
2633
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2634
|
+
for (const line of lines) {
|
|
2635
|
+
if (line === "" || seen.has(line)) continue;
|
|
2636
|
+
seen.add(line);
|
|
2637
|
+
unique.push(line);
|
|
2010
2638
|
}
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2639
|
+
const output = [];
|
|
2640
|
+
let used = 0;
|
|
2641
|
+
for (const line of unique) {
|
|
2642
|
+
const cost = codePointLength(line) + (output.length === 0 ? 0 : 1);
|
|
2643
|
+
if (used + cost > budgetChars) continue;
|
|
2644
|
+
output.push(line);
|
|
2645
|
+
used += cost;
|
|
2018
2646
|
}
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2647
|
+
const text = output.join("\n");
|
|
2648
|
+
return text.includes(requiredRef) ? text : null;
|
|
2649
|
+
}
|
|
2650
|
+
function takeWholeLinesFromHead(text, budgetChars) {
|
|
2651
|
+
const output = [];
|
|
2652
|
+
let used = 0;
|
|
2653
|
+
for (const line of splitLines(text)) {
|
|
2654
|
+
const cost = codePointLength(line) + (output.length === 0 ? 0 : 1);
|
|
2655
|
+
if (used + cost > budgetChars) break;
|
|
2656
|
+
output.push(line);
|
|
2657
|
+
used += cost;
|
|
2022
2658
|
}
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
const selected = entries.length <= 18 ? entries : [
|
|
2026
|
-
...entries.slice(0, 10),
|
|
2027
|
-
...important.filter((entry) => !entries.slice(0, 10).includes(entry)).slice(0, 6),
|
|
2028
|
-
...entries.slice(-2)
|
|
2029
|
-
];
|
|
2030
|
-
const result = {};
|
|
2031
|
-
for (const [key, entry] of selected) result[key] = shrinkJson(entry, depth + 1);
|
|
2032
|
-
if (selected.length < entries.length) result.$dsh_omitted_keys = entries.length - selected.length;
|
|
2033
|
-
return result;
|
|
2659
|
+
if (output.length === 0) return Array.from(text).slice(0, budgetChars).join("");
|
|
2660
|
+
return output.join("\n");
|
|
2034
2661
|
}
|
|
2035
|
-
function
|
|
2036
|
-
const lines = splitLines(
|
|
2037
|
-
const
|
|
2038
|
-
|
|
2039
|
-
for (
|
|
2040
|
-
const
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
ungrouped.push(row);
|
|
2047
|
-
continue;
|
|
2048
|
-
}
|
|
2049
|
-
const path = match[1] ?? "<unknown>";
|
|
2050
|
-
const bucket = groups.get(path) ?? [];
|
|
2051
|
-
bucket.push(row);
|
|
2052
|
-
groups.set(path, bucket);
|
|
2662
|
+
function takeWholeLinesFromTail(text, budgetChars) {
|
|
2663
|
+
const lines = splitLines(text);
|
|
2664
|
+
const output = [];
|
|
2665
|
+
let used = 0;
|
|
2666
|
+
for (let index = lines.length - 1; index >= 0; index--) {
|
|
2667
|
+
const line = lines[index];
|
|
2668
|
+
if (line === void 0) continue;
|
|
2669
|
+
const cost = codePointLength(line) + (output.length === 0 ? 0 : 1);
|
|
2670
|
+
if (used + cost > budgetChars) break;
|
|
2671
|
+
output.unshift(line);
|
|
2672
|
+
used += cost;
|
|
2053
2673
|
}
|
|
2054
|
-
if (
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2674
|
+
if (output.length === 0) return Array.from(text).slice(-budgetChars).join("");
|
|
2675
|
+
return output.join("\n");
|
|
2676
|
+
}
|
|
2677
|
+
function splitLines(text) {
|
|
2678
|
+
const lines = text.split("\n");
|
|
2679
|
+
if (text.endsWith("\n")) lines.pop();
|
|
2680
|
+
return lines;
|
|
2681
|
+
}
|
|
2682
|
+
/**
|
|
2683
|
+
* Extract the command argument from a tool call's arguments (task_10/AD2):
|
|
2684
|
+
* ONLY `command` / `cmd` / `script` are command keys. `input` was removed —
|
|
2685
|
+
* any MCP tool with a non-empty `input` string parameter would otherwise be
|
|
2686
|
+
* misrouted into the shell reducer, the single largest misroute source.
|
|
2687
|
+
* @param argumentsText - raw JSON arguments of the tool call.
|
|
2688
|
+
* @returns the command string, or '' when absent.
|
|
2689
|
+
*/
|
|
2690
|
+
function extractCommand(argumentsText) {
|
|
2691
|
+
try {
|
|
2692
|
+
const parsed = JSON.parse(argumentsText);
|
|
2693
|
+
if (typeof parsed !== "object" || parsed === null) return "";
|
|
2694
|
+
const record = parsed;
|
|
2695
|
+
for (const key of [
|
|
2696
|
+
"command",
|
|
2697
|
+
"cmd",
|
|
2698
|
+
"script"
|
|
2699
|
+
]) {
|
|
2700
|
+
const value = record[key];
|
|
2701
|
+
if (typeof value === "string") return value;
|
|
2068
2702
|
}
|
|
2069
|
-
|
|
2703
|
+
} catch {
|
|
2704
|
+
return "";
|
|
2070
2705
|
}
|
|
2071
|
-
|
|
2072
|
-
const text = fitLines([`[search results compressed; ${String(omitted)} matches omitted; source: ${input.sourceRef}]`, ...selected], input.budgetChars, input.sourceRef);
|
|
2073
|
-
return text === null ? null : {
|
|
2074
|
-
text,
|
|
2075
|
-
reducer: "search-by-file",
|
|
2076
|
-
lossy: true
|
|
2077
|
-
};
|
|
2706
|
+
return "";
|
|
2078
2707
|
}
|
|
2079
|
-
function
|
|
2080
|
-
const
|
|
2081
|
-
|
|
2082
|
-
let keep;
|
|
2083
|
-
let reducer;
|
|
2084
|
-
if (/\bgit\s+(?:diff|show)\b/.test(lower)) {
|
|
2085
|
-
reducer = "hypa-git-diff";
|
|
2086
|
-
keep = lines.filter((line) => /^(?:diff --git|index |--- |\+\+\+ |@@ |[+-](?![+-]))/.test(line) || IMPORTANT_PATTERN.test(line));
|
|
2087
|
-
} else if (/\bgit\s+(?:status|switch|checkout|merge|rebase|cherry-pick)\b/.test(lower)) {
|
|
2088
|
-
reducer = "hypa-git-status";
|
|
2089
|
-
keep = lines.filter((line) => GIT_STATUS_PATTERN.test(line) || IMPORTANT_PATTERN.test(line));
|
|
2090
|
-
} else {
|
|
2091
|
-
reducer = "hypa-git-log";
|
|
2092
|
-
keep = lines.filter((line) => /^(?:commit\s+[0-9a-f]+|Author:|Date:|[0-9a-f]{7,}\s)/i.test(line) || IMPORTANT_PATTERN.test(line));
|
|
2093
|
-
}
|
|
2094
|
-
if (keep.length === 0) return reduceSalient(input, reducer);
|
|
2095
|
-
const text = fitLines([
|
|
2096
|
-
`[git output compressed; source: ${input.sourceRef}]`,
|
|
2097
|
-
...keep,
|
|
2098
|
-
...lines.slice(-8)
|
|
2099
|
-
], input.budgetChars, input.sourceRef);
|
|
2100
|
-
return text === null ? null : {
|
|
2101
|
-
text,
|
|
2102
|
-
reducer,
|
|
2103
|
-
lossy: true
|
|
2104
|
-
};
|
|
2708
|
+
function looksLikeJson(text) {
|
|
2709
|
+
const trimmed = text.trim();
|
|
2710
|
+
return trimmed.startsWith("{") && trimmed.endsWith("}") || trimmed.startsWith("[") && trimmed.endsWith("]");
|
|
2105
2711
|
}
|
|
2106
|
-
function
|
|
2107
|
-
|
|
2108
|
-
const important = lines.filter((line) => pattern.test(line) || IMPORTANT_PATTERN.test(line) || STATUS_PATTERN.test(line));
|
|
2109
|
-
const text = fitLines([
|
|
2110
|
-
`[command output compressed by ${reducer}; source: ${input.sourceRef}]`,
|
|
2111
|
-
...important,
|
|
2112
|
-
...lines.slice(-20)
|
|
2113
|
-
], input.budgetChars, input.sourceRef);
|
|
2114
|
-
return text === null ? null : {
|
|
2115
|
-
text,
|
|
2116
|
-
reducer,
|
|
2117
|
-
lossy: true
|
|
2118
|
-
};
|
|
2712
|
+
function isGitCommand(name, command) {
|
|
2713
|
+
return name.includes("git") || /(?:^|\s)git\s/.test(command);
|
|
2119
2714
|
}
|
|
2120
|
-
function
|
|
2121
|
-
|
|
2122
|
-
const important = lines.filter((line) => IMPORTANT_PATTERN.test(line));
|
|
2123
|
-
if (important.length === 0) return reduceTail(input, "pi-tail");
|
|
2124
|
-
const text = fitLines([
|
|
2125
|
-
`[shell/log output compressed; source: ${input.sourceRef}]`,
|
|
2126
|
-
...important,
|
|
2127
|
-
"--- final output ---",
|
|
2128
|
-
...lines.slice(-40)
|
|
2129
|
-
], input.budgetChars, input.sourceRef);
|
|
2130
|
-
return text === null ? null : {
|
|
2131
|
-
text,
|
|
2132
|
-
reducer: "shell-salience-tail",
|
|
2133
|
-
lossy: true
|
|
2134
|
-
};
|
|
2715
|
+
function isPackageCommand(command) {
|
|
2716
|
+
return /(?:^|\s)(?:npm|pnpm|yarn|bun|pip|pip3|uv|poetry)\s/.test(command);
|
|
2135
2717
|
}
|
|
2136
|
-
function
|
|
2137
|
-
|
|
2138
|
-
if (lines.length < 3) return reduceHead(input, reducer);
|
|
2139
|
-
const marker = omissionMarker(input, reducer);
|
|
2140
|
-
const headBudget = Math.max(1, Math.floor((input.budgetChars - codePointLength(marker)) * .34));
|
|
2141
|
-
const tailBudget = headBudget;
|
|
2142
|
-
const head = takeWholeLinesFromHead(input.text, headBudget);
|
|
2143
|
-
const tail = takeWholeLinesFromTail(input.text, tailBudget);
|
|
2144
|
-
const text = fitLines([
|
|
2145
|
-
head,
|
|
2146
|
-
...lines.filter((line) => IMPORTANT_PATTERN.test(line) || STATUS_PATTERN.test(line)).slice(0, 24),
|
|
2147
|
-
marker,
|
|
2148
|
-
tail
|
|
2149
|
-
], input.budgetChars, input.sourceRef);
|
|
2150
|
-
return text === null ? null : {
|
|
2151
|
-
text,
|
|
2152
|
-
reducer,
|
|
2153
|
-
lossy: true
|
|
2154
|
-
};
|
|
2718
|
+
function isBuildOrTestCommand(command) {
|
|
2719
|
+
return new RegExp([String.raw`(?:^|\s)(?:tsc|dotnet\s+(?:build|test)|pytest|cargo\s+(?:build|test|check)|go\s+test|mvn\s+test|`, String.raw`gradle|npm\s+(?:test|run\s+build)|pnpm\s+(?:test|build|lint)|yarn\s+(?:test|build|lint))\b`].join("")).test(command);
|
|
2155
2720
|
}
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2721
|
+
function packagePattern() {
|
|
2722
|
+
return new RegExp([String.raw`(?:ERR!|WARN|warning|error|failed|conflict|peer dep|added\s+\d+|removed\s+\d+|installed|success|`, String.raw`up to date|packages?\s+(?:added|removed|changed)|resolution|No matching distribution|Could not find a version)`].join(""), "i");
|
|
2723
|
+
}
|
|
2724
|
+
function buildPattern() {
|
|
2725
|
+
return new RegExp([
|
|
2726
|
+
String.raw`(?:error\s+TS\d+|warning\s+TS\d+|FAILED|FAIL\b|AssertionError|expected|actual|`,
|
|
2727
|
+
String.raw`tests?\s+(?:run|passed|failed|skipped)|Build\s+(?:succeeded|FAILED)|\d+\s+Error\(s\)|`,
|
|
2728
|
+
String.raw`\d+\s+Warning\(s\)|Finished\s+test|compilation failed)`
|
|
2729
|
+
].join(""), "i");
|
|
2730
|
+
}
|
|
2731
|
+
//#endregion
|
|
2732
|
+
//#region src/runtime/tokenpilot/read-state.ts
|
|
2733
|
+
/** Write-style tool names whose success supersedes earlier reads. */
|
|
2734
|
+
const WRITE_TOOLS = /(?:^|[-_])?(?:write|edit|apply_patch|file_write|file_edit|str_replace|replace|multiedit)(?:$|[-_])/i;
|
|
2735
|
+
const PATH_KEYS = ["path", "file_path"];
|
|
2736
|
+
/** Parse one path out of a tool-call arguments JSON blob. */
|
|
2737
|
+
function toolCallPath(argumentsText) {
|
|
2738
|
+
let parsed;
|
|
2739
|
+
try {
|
|
2740
|
+
parsed = JSON.parse(argumentsText);
|
|
2741
|
+
} catch {
|
|
2742
|
+
return;
|
|
2743
|
+
}
|
|
2744
|
+
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
2745
|
+
const record = parsed;
|
|
2746
|
+
for (const key of PATH_KEYS) {
|
|
2747
|
+
const value = record[key];
|
|
2748
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
2749
|
+
}
|
|
2750
|
+
}
|
|
2751
|
+
/**
|
|
2752
|
+
* Decide whether an oversized read result was superseded by a later mutation
|
|
2753
|
+
* of the same file. `readPath` is the read call's target path; events after
|
|
2754
|
+
* `readSeq` are scanned for a write-style call on it.
|
|
2164
2755
|
*/
|
|
2165
|
-
function
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
if (
|
|
2171
|
-
|
|
2756
|
+
function isSupersededRead(events, readSeq, readPath) {
|
|
2757
|
+
if (readPath === void 0) return false;
|
|
2758
|
+
for (let seq = readSeq + 1; seq < events.length; seq += 1) {
|
|
2759
|
+
const event = events[seq];
|
|
2760
|
+
if (event?.type !== "tool/call") continue;
|
|
2761
|
+
if (!WRITE_TOOLS.test(event.data.name)) continue;
|
|
2762
|
+
if (toolCallPath(event.data.arguments) === readPath) return true;
|
|
2763
|
+
}
|
|
2764
|
+
return false;
|
|
2765
|
+
}
|
|
2766
|
+
/** Error/warning/info line classifiers used by the omission summary. */
|
|
2767
|
+
const ERROR_LINE = /\b(error|failed|failure|fatal|exception|traceback|cannot|unable|denied)\b/i;
|
|
2768
|
+
const WARN_LINE = /\b(warn|warning|deprecated)\b/i;
|
|
2769
|
+
const SECTION_HEADING = /^#{1,3}\s+(.{1,80})/;
|
|
2770
|
+
/**
|
|
2771
|
+
* Cluster one omitted line-count into a summary appended to a placeholder
|
|
2772
|
+
* marker, giving the model meta-knowledge about what was dropped. Document
|
|
2773
|
+
* content (R8) swaps the error/warn/info census for a section-heading list —
|
|
2774
|
+
* `0 error, 0 warn, N info` carries no information about a dropped document,
|
|
2775
|
+
* while its heading list does.
|
|
2776
|
+
*/
|
|
2777
|
+
function clusterOmittedLines(text, omittedLines) {
|
|
2778
|
+
if (omittedLines <= 0) return void 0;
|
|
2779
|
+
if (looksLikeDocument(text)) return documentCensus(text, omittedLines);
|
|
2780
|
+
let errors = 0;
|
|
2781
|
+
let warns = 0;
|
|
2782
|
+
let infos = 0;
|
|
2783
|
+
for (const line of text.split("\n")) if (ERROR_LINE.test(line)) errors += 1;
|
|
2784
|
+
else if (WARN_LINE.test(line)) warns += 1;
|
|
2785
|
+
else infos += 1;
|
|
2786
|
+
const parts = [];
|
|
2787
|
+
if (errors > 0) parts.push(`${String(errors)} error`);
|
|
2788
|
+
if (warns > 0) parts.push(`${String(warns)} warn`);
|
|
2789
|
+
if (infos > 0) parts.push(`${String(infos)} info`);
|
|
2790
|
+
if (parts.length === 0) return void 0;
|
|
2791
|
+
return `${String(omittedLines)} lines omitted (${parts.join(", ")})`;
|
|
2792
|
+
}
|
|
2793
|
+
/** Bounded section-heading list for an omitted document (R8 census). */
|
|
2794
|
+
function documentCensus(text, omittedLines) {
|
|
2795
|
+
const titles = [];
|
|
2796
|
+
for (const line of text.split("\n")) {
|
|
2797
|
+
const match = SECTION_HEADING.exec(line);
|
|
2798
|
+
if (match === null) continue;
|
|
2799
|
+
titles.push(match[1].trim());
|
|
2800
|
+
if (titles.length >= 8) break;
|
|
2801
|
+
}
|
|
2802
|
+
if (titles.length === 0) return `${String(omittedLines)} lines omitted (document content)`;
|
|
2803
|
+
let summary = titles.join(" · ");
|
|
2804
|
+
if (summary.length > 240) summary = `${summary.slice(0, 240)}…`;
|
|
2805
|
+
return `${String(omittedLines)} lines omitted (sections: ${summary})`;
|
|
2806
|
+
}
|
|
2807
|
+
//#endregion
|
|
2808
|
+
//#region src/runtime/tokenpilot/sidechannel.ts
|
|
2809
|
+
/** One bound side channel. `ask` resolves `undefined` on ANY failure. */
|
|
2810
|
+
var SideChannel = class {
|
|
2811
|
+
ctx;
|
|
2812
|
+
options;
|
|
2813
|
+
overrides;
|
|
2814
|
+
/**
|
|
2815
|
+
* @param overrides - per-consumer overrides of the estimator-named options.
|
|
2816
|
+
* The estimator itself never passes them (byte-identical behavior); the
|
|
2817
|
+
* advisory advisor passes its own mode/timeout/output budget so both
|
|
2818
|
+
* consumers share one transport without sharing one configuration.
|
|
2819
|
+
*/
|
|
2820
|
+
constructor(ctx, options, overrides) {
|
|
2821
|
+
this.ctx = ctx;
|
|
2822
|
+
this.options = options;
|
|
2823
|
+
this.overrides = overrides;
|
|
2824
|
+
}
|
|
2825
|
+
get mode() {
|
|
2826
|
+
return this.overrides?.mode ?? this.options.estimatorMode ?? "";
|
|
2827
|
+
}
|
|
2828
|
+
get enabled() {
|
|
2829
|
+
return this.mode === "host" || this.mode === "direct";
|
|
2830
|
+
}
|
|
2831
|
+
async ask(request) {
|
|
2832
|
+
const timeoutMs = this.overrides?.timeoutMs ?? this.options.estimatorTimeoutMs ?? 3e3;
|
|
2833
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
2834
|
+
const signal = typeof AbortSignal.any === "function" ? AbortSignal.any([request.signal, timeout]) : timeout;
|
|
2835
|
+
try {
|
|
2836
|
+
if (this.mode === "host") return await this.askHost(request.system, request.user, signal);
|
|
2837
|
+
if (this.mode === "direct") return await this.askDirect(request.system, request.user, signal);
|
|
2838
|
+
return;
|
|
2839
|
+
} catch {
|
|
2840
|
+
return;
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
/** Failure-open wrapper that also records one audit record per call. */
|
|
2844
|
+
async askAudited(request) {
|
|
2845
|
+
const now = Date.now();
|
|
2846
|
+
const text = await this.ask(request);
|
|
2847
|
+
const audit = {
|
|
2848
|
+
ok: text !== void 0,
|
|
2849
|
+
latencyMs: Date.now() - now,
|
|
2850
|
+
...this.identity() !== void 0 ? { channel: this.identity() } : {},
|
|
2851
|
+
...text === void 0 ? { reason: "channel returned no content (timeout, non-2xx, parse failure, or reasoning ate the output-token budget)" } : {}
|
|
2852
|
+
};
|
|
2853
|
+
return {
|
|
2854
|
+
...text === void 0 ? {} : { text },
|
|
2855
|
+
audit
|
|
2856
|
+
};
|
|
2857
|
+
}
|
|
2858
|
+
identity() {
|
|
2859
|
+
if (this.mode === "direct") return `direct:${this.options.estimatorModel ?? ""}`;
|
|
2860
|
+
if (this.mode === "host") {
|
|
2861
|
+
const route = this.resolveHostRoute();
|
|
2862
|
+
return route === void 0 ? "host" : `host:${route.provider}/${route.model}`;
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
/** Same host-route resolution as the estimator: explicit, then host default. */
|
|
2866
|
+
resolveHostRoute() {
|
|
2867
|
+
const provider = this.options.estimatorProvider ?? "";
|
|
2868
|
+
const model = this.options.estimatorModel ?? "";
|
|
2869
|
+
if (provider.length > 0 && model.length > 0) return {
|
|
2870
|
+
provider,
|
|
2871
|
+
model
|
|
2872
|
+
};
|
|
2873
|
+
try {
|
|
2874
|
+
const selected = this.ctx.get("agentDefaultModel")?.currentSelection?.();
|
|
2875
|
+
const selectedProvider = selected?.provider ?? "";
|
|
2876
|
+
const selectedModel = selected?.model ?? "";
|
|
2877
|
+
if (selectedProvider.length > 0 && selectedModel.length > 0) return {
|
|
2878
|
+
provider: provider.length > 0 ? provider : selectedProvider,
|
|
2879
|
+
model: model.length > 0 ? model : selectedModel
|
|
2880
|
+
};
|
|
2881
|
+
} catch {}
|
|
2882
|
+
}
|
|
2883
|
+
async askHost(system, user, signal) {
|
|
2884
|
+
let llm;
|
|
2885
|
+
try {
|
|
2886
|
+
llm = this.ctx.get("llm");
|
|
2887
|
+
} catch {
|
|
2888
|
+
return;
|
|
2889
|
+
}
|
|
2890
|
+
if (llm?.stream === void 0) return void 0;
|
|
2891
|
+
const route = this.resolveHostRoute();
|
|
2892
|
+
if (route === void 0) return void 0;
|
|
2893
|
+
let text = "";
|
|
2894
|
+
const stream = llm.stream({
|
|
2895
|
+
provider: route.provider,
|
|
2896
|
+
model: route.model,
|
|
2897
|
+
messages: [{
|
|
2898
|
+
role: "user",
|
|
2899
|
+
content: [{
|
|
2900
|
+
type: "text",
|
|
2901
|
+
text: user
|
|
2902
|
+
}]
|
|
2903
|
+
}],
|
|
2904
|
+
system,
|
|
2905
|
+
temperature: 0,
|
|
2906
|
+
reasoningEffort: "off",
|
|
2907
|
+
maxTokens: this.overrides?.maxTokens ?? 256,
|
|
2908
|
+
signal
|
|
2909
|
+
});
|
|
2910
|
+
for await (const chunk of stream) if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
2911
|
+
else if (chunk.type === "finish" && chunk.text === void 0) break;
|
|
2912
|
+
return text.trim().length > 0 ? text : void 0;
|
|
2913
|
+
}
|
|
2914
|
+
async askDirect(system, user, signal) {
|
|
2915
|
+
const baseUrl = this.options.estimatorBaseUrl;
|
|
2916
|
+
if (baseUrl === void 0 || baseUrl.length === 0) return void 0;
|
|
2917
|
+
const headers = { "content-type": "application/json" };
|
|
2918
|
+
if (this.options.estimatorApiKey !== void 0 && this.options.estimatorApiKey.length > 0) headers.authorization = `Bearer ${this.options.estimatorApiKey}`;
|
|
2919
|
+
const model = this.options.estimatorModel ?? "";
|
|
2920
|
+
if (model.length === 0) return void 0;
|
|
2921
|
+
const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/chat/completions`, {
|
|
2922
|
+
method: "POST",
|
|
2923
|
+
headers,
|
|
2924
|
+
body: JSON.stringify({
|
|
2925
|
+
model,
|
|
2926
|
+
messages: [{
|
|
2927
|
+
role: "system",
|
|
2928
|
+
content: system
|
|
2929
|
+
}, {
|
|
2930
|
+
role: "user",
|
|
2931
|
+
content: user
|
|
2932
|
+
}],
|
|
2933
|
+
temperature: 0,
|
|
2934
|
+
max_tokens: this.overrides?.maxTokens ?? 256
|
|
2935
|
+
}),
|
|
2936
|
+
signal
|
|
2937
|
+
});
|
|
2938
|
+
if (!response.ok) return void 0;
|
|
2939
|
+
const text = (await response.json()).choices?.[0]?.message?.content;
|
|
2940
|
+
return typeof text === "string" && text.trim().length > 0 ? text : void 0;
|
|
2941
|
+
}
|
|
2942
|
+
};
|
|
2943
|
+
//#endregion
|
|
2944
|
+
//#region src/runtime/tokenpilot/estimator.ts
|
|
2945
|
+
/** Exponential backoff with a 5-minute cap: 1s, 2s, 4s, … */
|
|
2946
|
+
function backoffCooldownMs(failures) {
|
|
2947
|
+
return Math.min(3e5, 1e3 * 2 ** Math.max(0, failures - 1));
|
|
2948
|
+
}
|
|
2949
|
+
function isCoolingDown(state, now) {
|
|
2950
|
+
return state !== void 0 && state.cooldownUntil > now;
|
|
2951
|
+
}
|
|
2952
|
+
function buildEstimatorSystemPrompt() {
|
|
2953
|
+
return [
|
|
2954
|
+
"You are a session residual-utility estimator.",
|
|
2955
|
+
"For each numbered historical file read, decide whether the live agent is likely to",
|
|
2956
|
+
"reference that exact file state again later in the session. Reads whose file was",
|
|
2957
|
+
"already rewritten, or whose task has visibly moved on, are expired.",
|
|
2958
|
+
"Answer with ONLY a JSON array: [{\"seq\":<number>,\"expired\":<boolean>}].",
|
|
2959
|
+
"Optionally, if you can estimate how many user turns remain in this session, answer",
|
|
2960
|
+
"with {\"expectedRemainingTurns\":<number>,\"items\":[{\"seq\":<number>,\"expired\":<boolean>}]}",
|
|
2961
|
+
"instead; omit the field when you cannot estimate it."
|
|
2962
|
+
].join(" ");
|
|
2963
|
+
}
|
|
2964
|
+
function buildEstimatorUserPrompt(samples) {
|
|
2965
|
+
return samples.map((sample) => `{"seq":${String(sample.seq)},"path":${JSON.stringify(sample.path)},"turn":${String(sample.turn)}}`).join("\n");
|
|
2966
|
+
}
|
|
2967
|
+
function parseVerdictArray(value) {
|
|
2968
|
+
if (!Array.isArray(value)) return [];
|
|
2969
|
+
const verdicts = [];
|
|
2970
|
+
for (const entry of value) {
|
|
2971
|
+
if (typeof entry !== "object" || entry === null) continue;
|
|
2972
|
+
const record = entry;
|
|
2973
|
+
if (typeof record.seq !== "number" || typeof record.expired !== "boolean") continue;
|
|
2974
|
+
verdicts.push({
|
|
2975
|
+
seq: record.seq,
|
|
2976
|
+
expired: record.expired
|
|
2977
|
+
});
|
|
2978
|
+
}
|
|
2979
|
+
return verdicts;
|
|
2980
|
+
}
|
|
2981
|
+
/**
|
|
2982
|
+
* Parse the estimator answer including the optional session-level
|
|
2983
|
+
* `expectedRemainingTurns`. Accepts both the legacy bare verdict array and the
|
|
2984
|
+
* extended object form; anything malformed yields no verdicts and no Ŝ.
|
|
2985
|
+
*/
|
|
2986
|
+
function parseEstimatorAnswerDetailed(text) {
|
|
2987
|
+
const objectStart = text.indexOf("{");
|
|
2988
|
+
const objectEnd = text.lastIndexOf("}");
|
|
2989
|
+
if (objectStart >= 0 && objectEnd > objectStart) try {
|
|
2990
|
+
const parsed = JSON.parse(text.slice(objectStart, objectEnd + 1));
|
|
2991
|
+
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
|
|
2992
|
+
const record = parsed;
|
|
2993
|
+
const verdicts = parseVerdictArray(record.verdicts ?? record.items);
|
|
2994
|
+
if (verdicts.length > 0) {
|
|
2995
|
+
const turns = record.expectedRemainingTurns;
|
|
2996
|
+
if (typeof turns === "number" && Number.isFinite(turns) && turns >= 0) return {
|
|
2997
|
+
verdicts,
|
|
2998
|
+
expectedRemainingTurns: Math.floor(turns)
|
|
2999
|
+
};
|
|
3000
|
+
return { verdicts };
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
} catch {}
|
|
3004
|
+
const start = text.indexOf("[");
|
|
3005
|
+
const end = text.lastIndexOf("]");
|
|
3006
|
+
if (start < 0 || end <= start) return { verdicts: [] };
|
|
3007
|
+
try {
|
|
3008
|
+
return { verdicts: parseVerdictArray(JSON.parse(text.slice(start, end + 1))) };
|
|
3009
|
+
} catch {
|
|
3010
|
+
return { verdicts: [] };
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
/** One channel-bound estimator. `ask` resolves undefined on any failure. */
|
|
3014
|
+
var Estimator = class {
|
|
3015
|
+
ctx;
|
|
3016
|
+
options;
|
|
3017
|
+
channel;
|
|
3018
|
+
constructor(ctx, options) {
|
|
3019
|
+
this.ctx = ctx;
|
|
3020
|
+
this.options = options;
|
|
3021
|
+
this.channel = new SideChannel(ctx, options);
|
|
3022
|
+
}
|
|
3023
|
+
get enabled() {
|
|
3024
|
+
return this.options.estimatorMode === "host" || this.options.estimatorMode === "direct";
|
|
3025
|
+
}
|
|
3026
|
+
async ask(system, user, signal) {
|
|
3027
|
+
return this.channel.ask({
|
|
3028
|
+
system,
|
|
3029
|
+
user,
|
|
3030
|
+
signal
|
|
3031
|
+
});
|
|
3032
|
+
}
|
|
3033
|
+
};
|
|
3034
|
+
//#endregion
|
|
3035
|
+
//#region src/runtime/tokenpilot/dedup.ts
|
|
3036
|
+
/**
|
|
3037
|
+
* TokenPilot-inspired A1: byte-identical repeated tool-result dedup.
|
|
3038
|
+
*
|
|
3039
|
+
* Pure helpers behind the ToolResultPruner fresh pass. The per-session table
|
|
3040
|
+
* maps a canonical-content SHA-256 to the first surface seq that produced it;
|
|
3041
|
+
* later identical results may be replaced with a pointer placeholder that the
|
|
3042
|
+
* recovery tool can resolve back to the original full text via the append-only
|
|
3043
|
+
* session log. Only hash+seq metadata is stored — never content.
|
|
3044
|
+
*/
|
|
3045
|
+
/** Per-session dedup index with insertion-order eviction. */
|
|
3046
|
+
var DedupeTable = class {
|
|
3047
|
+
maxEntries;
|
|
3048
|
+
entries = /* @__PURE__ */ new Map();
|
|
3049
|
+
constructor(maxEntries = 2048) {
|
|
3050
|
+
this.maxEntries = maxEntries;
|
|
3051
|
+
}
|
|
3052
|
+
/** Look up the first occurrence for one canonical hash, if any. */
|
|
3053
|
+
get(hash) {
|
|
3054
|
+
return this.entries.get(hash);
|
|
3055
|
+
}
|
|
3056
|
+
/** Record a first occurrence; existing hashes only refresh insertion order. */
|
|
3057
|
+
record(hash, entry) {
|
|
3058
|
+
if (this.entries.has(hash)) return;
|
|
3059
|
+
while (this.entries.size >= this.maxEntries) {
|
|
3060
|
+
const oldest = this.entries.keys().next().value;
|
|
3061
|
+
if (oldest === void 0) break;
|
|
3062
|
+
this.entries.delete(oldest);
|
|
3063
|
+
}
|
|
3064
|
+
this.entries.set(hash, entry);
|
|
3065
|
+
}
|
|
3066
|
+
};
|
|
3067
|
+
/** Canonicalize tool-result text for hashing. */
|
|
3068
|
+
function canonicalizeForDedupe(text, mode) {
|
|
3069
|
+
if (mode === "exact") return text;
|
|
3070
|
+
return text.replace(/[ \t]+\r?\n/g, "\n").replace(/(^\s+)|(\s+$)/g, "");
|
|
3071
|
+
}
|
|
3072
|
+
/** SHA-256 hex of the canonicalized text. */
|
|
3073
|
+
function dedupeHash(text, mode) {
|
|
3074
|
+
return createHash("sha256").update(canonicalizeForDedupe(text, mode), "utf8").digest("hex");
|
|
3075
|
+
}
|
|
3076
|
+
/** Concatenated text of an all-text content block list; null when rich. */
|
|
3077
|
+
function flattenPlainText(content) {
|
|
3078
|
+
let text = "";
|
|
3079
|
+
for (const block of content) {
|
|
3080
|
+
if (block.type !== "text") return void 0;
|
|
3081
|
+
text += block.text;
|
|
3082
|
+
}
|
|
3083
|
+
return text;
|
|
3084
|
+
}
|
|
3085
|
+
/** Pointer placeholder pointing at the first occurrence's original event. */
|
|
3086
|
+
function dedupePlaceholder(entry, originalChars) {
|
|
3087
|
+
return [
|
|
3088
|
+
`[... identical to the earlier ${entry.toolName} result; first seen at ${entry.sourceRef};`,
|
|
3089
|
+
`original_chars=${String(originalChars)};`,
|
|
3090
|
+
"retrieve with context_compression_retrieve({\"ref\":\"" + entry.sourceRef + "\",\"start_line\":1}) if the omitted evidence is necessary.]"
|
|
3091
|
+
].join(" ");
|
|
3092
|
+
}
|
|
3093
|
+
//#endregion
|
|
3094
|
+
//#region src/runtime/tokenpilot/proposal.ts
|
|
3095
|
+
/**
|
|
3096
|
+
* TokenPilot-inspired R4: benefit model for the human-gated review pipeline.
|
|
3097
|
+
*
|
|
3098
|
+
* Pure functions only: the classifier needs no I/O, no session state, and no
|
|
3099
|
+
* host services, so every decision is unit-testable and audit-replayable.
|
|
3100
|
+
*
|
|
3101
|
+
* The cost model follows the TokenPilot paper's cache-accounting view: one
|
|
3102
|
+
* merged mutation pays a one-time tail KV-cache refill penalty of
|
|
3103
|
+
* `(1−α)·tailTokens`, and every later turn recovers the reclaimed tokens at
|
|
3104
|
+
* the cache-hit discount `α`:
|
|
3105
|
+
*
|
|
3106
|
+
* ```
|
|
3107
|
+
* R = Σ(tokensBefore − tokensAfter) // net reclaimed tokens
|
|
3108
|
+
* paybackTurns = (1−α)·tailTokens / (α·R) // one-time refill / per-turn saving
|
|
3109
|
+
* expectedSaving = α·R·max(0, Ŝ − paybackTurns) // Ŝ = estimated remaining turns
|
|
3110
|
+
* ```
|
|
3111
|
+
*
|
|
3112
|
+
* The refill penalty only models mutations of already-cached context. A
|
|
3113
|
+
* fresh-stage batch (shaped before its first request) is exempt via
|
|
3114
|
+
* `refillPenaltyExempt`: payback is 0 and every reclaimed token saves from
|
|
3115
|
+
* the very first turn.
|
|
3116
|
+
*
|
|
3117
|
+
* `expectedSaving` is only produced when Ŝ is known (the estimator answered
|
|
3118
|
+
* with `expectedRemainingTurns`); it is never fabricated from a guess.
|
|
3119
|
+
*/
|
|
3120
|
+
/**
|
|
3121
|
+
* Aggregate the batch-level benefit of a set of reduction candidates.
|
|
3122
|
+
*
|
|
3123
|
+
* Individual candidates whose replacement would grow the context contribute
|
|
3124
|
+
* zero recovery (they never make a batch look better than dropping them).
|
|
3125
|
+
*/
|
|
3126
|
+
function computeBenefit(candidates, input) {
|
|
3127
|
+
const { alpha, tailTokens, remainingTurns } = input;
|
|
3128
|
+
let recoveredTokens = 0;
|
|
3129
|
+
for (const candidate of candidates) recoveredTokens += Math.max(0, candidate.tokensBefore - candidate.tokensAfter);
|
|
3130
|
+
const penaltyTokens = input.refillPenaltyExempt === true ? 0 : (1 - alpha) * tailTokens;
|
|
3131
|
+
const perTurnSaving = alpha * recoveredTokens;
|
|
3132
|
+
if (perTurnSaving <= 0) return remainingTurns === void 0 ? {
|
|
3133
|
+
recoveredTokens,
|
|
3134
|
+
penaltyTokens
|
|
3135
|
+
} : {
|
|
3136
|
+
recoveredTokens,
|
|
3137
|
+
penaltyTokens,
|
|
3138
|
+
expectedSaving: -penaltyTokens
|
|
2172
3139
|
};
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
while (index < lines.length && depth > startDepth) {
|
|
2179
|
-
const body = lines[index];
|
|
2180
|
-
if (body === void 0) break;
|
|
2181
|
-
if (IMPORTANT_PATTERN.test(body)) {
|
|
2182
|
-
flushElided();
|
|
2183
|
-
kept.push(body);
|
|
2184
|
-
} else elided += 1;
|
|
2185
|
-
depth += braceDelta(body);
|
|
2186
|
-
index += 1;
|
|
2187
|
-
}
|
|
2188
|
-
flushElided();
|
|
3140
|
+
const paybackTurns = penaltyTokens / perTurnSaving;
|
|
3141
|
+
if (remainingTurns === void 0) return {
|
|
3142
|
+
recoveredTokens,
|
|
3143
|
+
penaltyTokens,
|
|
3144
|
+
paybackTurns
|
|
2189
3145
|
};
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
3146
|
+
return {
|
|
3147
|
+
recoveredTokens,
|
|
3148
|
+
penaltyTokens,
|
|
3149
|
+
paybackTurns,
|
|
3150
|
+
expectedSaving: perTurnSaving * Math.max(0, remainingTurns - paybackTurns)
|
|
3151
|
+
};
|
|
3152
|
+
}
|
|
3153
|
+
/**
|
|
3154
|
+
* Stable proposal identity: the sha-256 of the serialized item digests, cut to
|
|
3155
|
+
* 12 hex chars. Stable across re-enqueues of the same content so a repeated
|
|
3156
|
+
* classification cannot duplicate a pending proposal.
|
|
3157
|
+
*/
|
|
3158
|
+
function proposalId(itemDigests) {
|
|
3159
|
+
const hash = createHash("sha256");
|
|
3160
|
+
for (const digest of itemDigests) hash.update(digest);
|
|
3161
|
+
hash.update(String(itemDigests.length));
|
|
3162
|
+
return hash.digest("hex").slice(0, 12);
|
|
3163
|
+
}
|
|
3164
|
+
/**
|
|
3165
|
+
* Canonical content digest reused from the dedup hash: plain-text results hash
|
|
3166
|
+
* through the dedupe canonicalization; rich blocks fall back to canonical JSON
|
|
3167
|
+
* so every candidate is freezable.
|
|
3168
|
+
*/
|
|
3169
|
+
function contentDigest(content) {
|
|
3170
|
+
return dedupeHash(flattenPlainText(content) ?? JSON.stringify(content), "trim-eol");
|
|
3171
|
+
}
|
|
3172
|
+
function proposalKindFor(candidate, estimatorSeqs) {
|
|
3173
|
+
if (estimatorSeqs?.has(candidate.sourceSeq) === true) return "estimator";
|
|
3174
|
+
if (candidate.reducer === "dedupe-pointer") return "dedup";
|
|
3175
|
+
return "read-state";
|
|
3176
|
+
}
|
|
3177
|
+
/**
|
|
3178
|
+
* Triage planned replacements into the three review-mode buckets, pricing the
|
|
3179
|
+
* pass as ONE merged mutation (R1): the tail KV-cache refill penalty is a
|
|
3180
|
+
* property of the landing event, not of any single candidate, so it must be
|
|
3181
|
+
* paid exactly once per batch. Pricing per candidate overstates the payback
|
|
3182
|
+
* N-fold and starves every real batch out of the auto path.
|
|
3183
|
+
*
|
|
3184
|
+
* Pipeline: zero/negative-recovery candidates are priced out first (they never
|
|
3185
|
+
* make a batch look better), the surviving batch is priced once through
|
|
3186
|
+
* `computeBenefit`, the verdict is a batch decision, and any high-impact
|
|
3187
|
+
* candidate (`tokensBefore ≥ reviewHighImpactTokens`) covers the whole batch
|
|
3188
|
+
* into review — splitting the batch would pay a second cache break that the
|
|
3189
|
+
* accounting does not model. Review skeletons are grouped one proposal per
|
|
3190
|
+
* kind; a proposal id covers every item digest.
|
|
3191
|
+
*
|
|
3192
|
+
* Batch verdict bands (identical thresholds to the per-candidate model):
|
|
3193
|
+
* - any high-impact candidate, or α too small to price a payback → review;
|
|
3194
|
+
* - `paybackTurns ≤ 1`, or Ŝ known and `paybackTurns ≤ 0.25·Ŝ` → auto;
|
|
3195
|
+
* - Ŝ known and `paybackTurns ∈ (1, 3]` → review;
|
|
3196
|
+
* - everything else (Ŝ unknown with a slow payback) → drop.
|
|
3197
|
+
*
|
|
3198
|
+
* Stage asymmetry: a `'fresh'` batch is exempt from the tail-refill penalty
|
|
3199
|
+
* (`refillPenaltyExempt`) — its content was never served, so compressing it
|
|
3200
|
+
* breaks no cache and payback is 0 — while a `'history'` batch mutates
|
|
3201
|
+
* already-cached context and pays `(1−α)·tailTokens` in full. Without this
|
|
3202
|
+
* exemption every realistic fresh batch prices into the drop band and the
|
|
3203
|
+
* auto bucket stays structurally unreachable.
|
|
3204
|
+
*/
|
|
3205
|
+
function classifyCandidates(candidates, input) {
|
|
3206
|
+
const drop = [];
|
|
3207
|
+
const usable = [];
|
|
3208
|
+
for (const candidate of candidates) {
|
|
3209
|
+
if (Math.max(0, candidate.tokensBefore - candidate.tokensAfter) <= 0) {
|
|
3210
|
+
drop.push(candidate);
|
|
3211
|
+
continue;
|
|
2208
3212
|
}
|
|
3213
|
+
usable.push(candidate);
|
|
3214
|
+
}
|
|
3215
|
+
if (usable.length === 0) return {
|
|
3216
|
+
auto: [],
|
|
3217
|
+
review: [],
|
|
3218
|
+
drop
|
|
2209
3219
|
};
|
|
2210
|
-
const
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
3220
|
+
const benefit = computeBenefit(usable, {
|
|
3221
|
+
alpha: input.alpha,
|
|
3222
|
+
tailTokens: input.tailTokens,
|
|
3223
|
+
...input.remainingTurns !== void 0 ? { remainingTurns: input.remainingTurns } : {},
|
|
3224
|
+
refillPenaltyExempt: input.stage === "fresh"
|
|
3225
|
+
});
|
|
3226
|
+
const payback = benefit.paybackTurns;
|
|
3227
|
+
const highImpact = usable.some((candidate) => candidate.tokensBefore >= input.reviewHighImpactTokens);
|
|
3228
|
+
let verdict;
|
|
3229
|
+
if (highImpact || payback === void 0) verdict = "review";
|
|
3230
|
+
else if (payback <= 1 || input.remainingTurns !== void 0 && payback <= .25 * input.remainingTurns) verdict = "auto";
|
|
3231
|
+
else if (input.remainingTurns !== void 0 && payback <= 3) verdict = "review";
|
|
3232
|
+
else verdict = "drop";
|
|
3233
|
+
if (verdict === "auto") return {
|
|
3234
|
+
auto: usable,
|
|
3235
|
+
review: [],
|
|
3236
|
+
drop
|
|
3237
|
+
};
|
|
3238
|
+
if (verdict === "drop") return {
|
|
3239
|
+
auto: [],
|
|
3240
|
+
review: [],
|
|
3241
|
+
drop: [...drop, ...usable]
|
|
3242
|
+
};
|
|
3243
|
+
const itemsByKind = /* @__PURE__ */ new Map();
|
|
3244
|
+
for (const candidate of usable) {
|
|
3245
|
+
const item = {
|
|
3246
|
+
seq: candidate.sourceSeq,
|
|
3247
|
+
component: candidate.component,
|
|
3248
|
+
kind: proposalKindFor(candidate, input.estimatorSeqs),
|
|
3249
|
+
tokensBefore: candidate.tokensBefore,
|
|
3250
|
+
tokensAfter: candidate.tokensAfter,
|
|
3251
|
+
digest: contentDigest(candidate.content)
|
|
3252
|
+
};
|
|
3253
|
+
const bucket = itemsByKind.get(item.kind) ?? [];
|
|
3254
|
+
bucket.push(item);
|
|
3255
|
+
itemsByKind.set(item.kind, bucket);
|
|
3256
|
+
}
|
|
3257
|
+
const review = [];
|
|
3258
|
+
for (const [kind, items] of itemsByKind) review.push({
|
|
3259
|
+
id: proposalId(items.map((item) => item.digest)),
|
|
3260
|
+
kind,
|
|
3261
|
+
items,
|
|
3262
|
+
benefit
|
|
3263
|
+
});
|
|
3264
|
+
return {
|
|
3265
|
+
auto: [],
|
|
3266
|
+
review,
|
|
3267
|
+
drop
|
|
3268
|
+
};
|
|
3269
|
+
}
|
|
3270
|
+
//#endregion
|
|
3271
|
+
//#region src/runtime/tokenpilot/review-storage.ts
|
|
3272
|
+
/** Domain name — `UNIT_NAME_RE` (`/^[a-z][a-z0-9_]*$/`) allows no hyphens. */
|
|
3273
|
+
const REVIEW_STORAGE_DOMAIN = "context_compression_review";
|
|
3274
|
+
/** The one declared table: one record per session id. */
|
|
3275
|
+
const REVIEW_STORAGE_TABLE = "sessions";
|
|
3276
|
+
/** Structural validator: accepts exactly the shape this module persists. */
|
|
3277
|
+
function reviewSessionRecordValidator() {
|
|
3278
|
+
return { safeParse(value) {
|
|
3279
|
+
if (typeof value !== "object" || value === null) return { success: false };
|
|
3280
|
+
const record = value;
|
|
3281
|
+
if (record.version !== 1 || !Array.isArray(record.proposals)) return { success: false };
|
|
3282
|
+
for (const proposal of record.proposals) {
|
|
3283
|
+
if (typeof proposal !== "object" || proposal === null) return { success: false };
|
|
3284
|
+
const entry = proposal;
|
|
3285
|
+
if (typeof entry.id !== "string" || typeof entry.sessionId !== "string") return { success: false };
|
|
3286
|
+
if (entry.kind !== "estimator" && entry.kind !== "dedup" && entry.kind !== "read-state") return { success: false };
|
|
3287
|
+
if (entry.status !== "pending" && entry.status !== "approved") return { success: false };
|
|
3288
|
+
if (!Number.isSafeInteger(entry.enqueuedTurn) || !Number.isSafeInteger(entry.lastTurnIndex)) return { success: false };
|
|
3289
|
+
if (!Array.isArray(entry.items) || typeof entry.benefit !== "object" || entry.benefit === null) return { success: false };
|
|
3290
|
+
for (const item of entry.items) {
|
|
3291
|
+
if (typeof item !== "object" || item === null) return { success: false };
|
|
3292
|
+
const one = item;
|
|
3293
|
+
if (!Number.isSafeInteger(one.seq) || typeof one.digest !== "string") return { success: false };
|
|
2226
3294
|
}
|
|
2227
|
-
elided += 1;
|
|
2228
|
-
index += 1;
|
|
2229
3295
|
}
|
|
2230
|
-
|
|
3296
|
+
return {
|
|
3297
|
+
success: true,
|
|
3298
|
+
data: value
|
|
3299
|
+
};
|
|
3300
|
+
} };
|
|
3301
|
+
}
|
|
3302
|
+
function reviewStorageSpec() {
|
|
3303
|
+
return {
|
|
3304
|
+
name: REVIEW_STORAGE_DOMAIN,
|
|
3305
|
+
version: 1,
|
|
3306
|
+
layout: "per-record",
|
|
3307
|
+
tables: { [REVIEW_STORAGE_TABLE]: { valueSchema: reviewSessionRecordValidator() } }
|
|
2231
3308
|
};
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
3309
|
+
}
|
|
3310
|
+
/** Adapter presenting the sync KV face the queue expects over the domain table. */
|
|
3311
|
+
var StorageDomainReviewStore = class {
|
|
3312
|
+
table;
|
|
3313
|
+
constructor(table) {
|
|
3314
|
+
this.table = table;
|
|
3315
|
+
}
|
|
3316
|
+
load(sessionId) {
|
|
3317
|
+
const value = this.table.get(sessionId);
|
|
3318
|
+
return typeof value === "object" && value !== null ? value : void 0;
|
|
3319
|
+
}
|
|
3320
|
+
save(sessionId, record) {
|
|
3321
|
+
this.table.put(sessionId, record).catch(() => void 0);
|
|
3322
|
+
}
|
|
3323
|
+
ids() {
|
|
3324
|
+
return [...this.table.keys()];
|
|
3325
|
+
}
|
|
3326
|
+
};
|
|
3327
|
+
/**
|
|
3328
|
+
* Attempt to open the review storage domain through the optional
|
|
3329
|
+
* `storageDomain` seam.
|
|
3330
|
+
* @param getService - resolved once with the seam name; `undefined` means the
|
|
3331
|
+
* host lacks the service.
|
|
3332
|
+
* @returns the durable store, or `undefined` when the seam is absent or fails
|
|
3333
|
+
* (the caller falls back to the in-memory store and logs one warning).
|
|
3334
|
+
*/
|
|
3335
|
+
async function openReviewStorage(getService) {
|
|
3336
|
+
let service;
|
|
3337
|
+
try {
|
|
3338
|
+
service = getService("storageDomain");
|
|
3339
|
+
} catch {
|
|
3340
|
+
return;
|
|
3341
|
+
}
|
|
3342
|
+
if (service === void 0 || service === null) return void 0;
|
|
3343
|
+
return new StorageDomainReviewStore((await service.open(reviewStorageSpec())).table(REVIEW_STORAGE_TABLE));
|
|
3344
|
+
}
|
|
3345
|
+
//#endregion
|
|
3346
|
+
//#region src/runtime/tokenpilot/advisor-prompt.ts
|
|
3347
|
+
function buildAdvisorSummarySystemPrompt() {
|
|
3348
|
+
return [
|
|
3349
|
+
"You summarize what an agent session is working on, for relevance statistics only.",
|
|
3350
|
+
"Input: the session todolist snapshot and a recent tail of assistant narration.",
|
|
3351
|
+
"Answer with ONLY one JSON object:",
|
|
3352
|
+
"{\"overallTask\":\"<one sentence>\",\"activeSubtasks\":[\"<subtask>\"],\"keywords\":[\"<task keyword>\"]}.",
|
|
3353
|
+
"keywords must be 3-10 short distinctive words describing the CURRENT task.",
|
|
3354
|
+
"Never add commentary; never invent tasks that the input does not support."
|
|
3355
|
+
].join(" ");
|
|
3356
|
+
}
|
|
3357
|
+
function buildAdvisorSummaryUserPrompt(taskText, tailText) {
|
|
3358
|
+
return [`todolist:\n${taskText}`, tailText.trim().length > 0 ? `recent tail:\n${tailText.trim()}` : "recent tail: (none)"].join("\n\n");
|
|
3359
|
+
}
|
|
3360
|
+
function buildAdvisorScoringSystemPrompt() {
|
|
3361
|
+
return [
|
|
3362
|
+
"You score how relevant each historical session artifact is to the current task,",
|
|
3363
|
+
"for statistics only. Relevance covers both the artifact content and its comments",
|
|
3364
|
+
"(comment semantics count too). 0 means unrelated, 1 means the live agent will",
|
|
3365
|
+
"very likely need this exact content again.",
|
|
3366
|
+
"Answer with ONLY one JSON object per input line:",
|
|
3367
|
+
"{\"seq\":<number>,\"score\":<number between 0 and 1>,\"reason\":\"<short>\"}",
|
|
3368
|
+
"one per line, same order as the input. Never invent seq values; never add commentary."
|
|
3369
|
+
].join(" ");
|
|
3370
|
+
}
|
|
3371
|
+
function buildAdvisorScoringUserPrompt(taskText, activeSubtasks, candidates) {
|
|
3372
|
+
return [
|
|
3373
|
+
`task: ${taskText.replace(/\s+/gu, " ").slice(0, 600)}`,
|
|
3374
|
+
activeSubtasks.length > 0 ? `active subtasks: ${activeSubtasks.join("; ").slice(0, 300)}` : "active subtasks: (none)",
|
|
3375
|
+
"",
|
|
3376
|
+
...candidates.map((candidate) => `seq=${String(candidate.seq)} | ${candidate.preview.replace(/\s+/gu, " ")}`)
|
|
3377
|
+
].join("\n");
|
|
3378
|
+
}
|
|
3379
|
+
/** Pull the first balanced JSON object out of a possibly chatty answer. */
|
|
3380
|
+
function firstJsonObject(text) {
|
|
3381
|
+
const start = text.indexOf("{");
|
|
3382
|
+
if (start < 0) return void 0;
|
|
3383
|
+
let depth = 0;
|
|
3384
|
+
let inString = false;
|
|
3385
|
+
let escaped = false;
|
|
3386
|
+
for (let index = start; index < text.length; index += 1) {
|
|
3387
|
+
const char = text[index];
|
|
3388
|
+
if (inString) {
|
|
3389
|
+
if (escaped) escaped = false;
|
|
3390
|
+
else if (char === "\\") escaped = true;
|
|
3391
|
+
else if (char === "\"") inString = false;
|
|
2272
3392
|
continue;
|
|
2273
3393
|
}
|
|
2274
|
-
if (
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
3394
|
+
if (char === "\"") inString = true;
|
|
3395
|
+
else if (char === "{") depth += 1;
|
|
3396
|
+
else if (char === "}") {
|
|
3397
|
+
depth -= 1;
|
|
3398
|
+
if (depth === 0) try {
|
|
3399
|
+
const parsed = JSON.parse(text.slice(start, index + 1));
|
|
3400
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : void 0;
|
|
3401
|
+
} catch {
|
|
3402
|
+
return;
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
2280
3405
|
}
|
|
2281
|
-
flushElided();
|
|
2282
|
-
return finishSkeleton(kept, lines, input);
|
|
2283
3406
|
}
|
|
2284
|
-
function
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
3407
|
+
function stringList(value, limit) {
|
|
3408
|
+
if (!Array.isArray(value)) return [];
|
|
3409
|
+
return value.filter((entry) => typeof entry === "string" && entry.trim().length > 0).slice(0, limit).map((entry) => entry.trim());
|
|
3410
|
+
}
|
|
3411
|
+
/**
|
|
3412
|
+
* Parse one summary answer. Fail-open: `undefined` on any malformed or
|
|
3413
|
+
* missing field, so a broken channel can never poison the cached summary.
|
|
3414
|
+
*/
|
|
3415
|
+
function parseAdvisorSummary(text) {
|
|
3416
|
+
if (text === void 0 || text.trim().length === 0) return void 0;
|
|
3417
|
+
const object = firstJsonObject(text);
|
|
3418
|
+
if (object === void 0) return void 0;
|
|
3419
|
+
const overallTask = object.overallTask;
|
|
3420
|
+
if (typeof overallTask !== "string" || overallTask.trim().length === 0) return void 0;
|
|
3421
|
+
const activeSubtasks = stringList(object.activeSubtasks, 12);
|
|
3422
|
+
const keywords = stringList(object.keywords, 12);
|
|
3423
|
+
if (keywords.length === 0 && activeSubtasks.length === 0) return void 0;
|
|
3424
|
+
return {
|
|
3425
|
+
overallTask: overallTask.trim(),
|
|
3426
|
+
activeSubtasks,
|
|
3427
|
+
keywords
|
|
2294
3428
|
};
|
|
2295
3429
|
}
|
|
2296
|
-
/**
|
|
2297
|
-
function
|
|
2298
|
-
|
|
2299
|
-
let
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
3430
|
+
/** Extract every balanced JSON object from a JSON-lines or chatty answer. */
|
|
3431
|
+
function jsonObjects(text) {
|
|
3432
|
+
const objects = [];
|
|
3433
|
+
let depth = 0;
|
|
3434
|
+
let start = -1;
|
|
3435
|
+
let inString = false;
|
|
3436
|
+
let escaped = false;
|
|
3437
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
3438
|
+
const char = text[index];
|
|
3439
|
+
if (inString) {
|
|
3440
|
+
if (escaped) escaped = false;
|
|
3441
|
+
else if (char === "\\") escaped = true;
|
|
3442
|
+
else if (char === "\"") inString = false;
|
|
2305
3443
|
continue;
|
|
2306
3444
|
}
|
|
2307
|
-
if (char === "\""
|
|
2308
|
-
|
|
2309
|
-
|
|
3445
|
+
if (char === "\"") inString = true;
|
|
3446
|
+
else if (char === "{") {
|
|
3447
|
+
if (depth === 0) start = index;
|
|
3448
|
+
depth += 1;
|
|
3449
|
+
} else if (char === "}") {
|
|
3450
|
+
depth -= 1;
|
|
3451
|
+
if (depth === 0 && start >= 0) {
|
|
3452
|
+
try {
|
|
3453
|
+
const parsed = JSON.parse(text.slice(start, index + 1));
|
|
3454
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) objects.push(parsed);
|
|
3455
|
+
} catch {}
|
|
3456
|
+
start = -1;
|
|
3457
|
+
}
|
|
2310
3458
|
}
|
|
2311
|
-
if (char === "{") delta += 1;
|
|
2312
|
-
else if (char === "}") delta -= 1;
|
|
2313
3459
|
}
|
|
2314
|
-
return
|
|
2315
|
-
}
|
|
2316
|
-
function leadingIndent(line) {
|
|
2317
|
-
return codePointLength(line) - codePointLength(line.trimStart());
|
|
2318
|
-
}
|
|
2319
|
-
function isCodeStructureLine(line) {
|
|
2320
|
-
return CODE_STRUCTURE_PATTERN.test(line) || PYTHON_STRUCTURE_PATTERN.test(line);
|
|
3460
|
+
return objects;
|
|
2321
3461
|
}
|
|
2322
3462
|
/**
|
|
2323
|
-
*
|
|
2324
|
-
*
|
|
2325
|
-
*
|
|
2326
|
-
* @param text - normalized result text.
|
|
2327
|
-
* @returns whether the text qualifies as source code.
|
|
3463
|
+
* Parse one scoring answer. Fail-open: returns the valid rows it could read
|
|
3464
|
+
* (`undefined` when nothing valid remains) — a partially garbage answer still
|
|
3465
|
+
* contributes its good rows, mirroring the estimator's per-item tolerance.
|
|
2328
3466
|
*/
|
|
2329
|
-
function
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
if (
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
}
|
|
2342
|
-
|
|
2343
|
-
const lines = splitLines(normalizeTerminalText(text));
|
|
2344
|
-
const chosen = lines.find((line) => IMPORTANT_PATTERN.test(line)) ?? lines.at(-1) ?? "";
|
|
2345
|
-
return Array.from(chosen.trim()).slice(0, maxChars).join("");
|
|
2346
|
-
}
|
|
2347
|
-
function fitLines(lines, budgetChars, requiredRef) {
|
|
2348
|
-
const unique = [];
|
|
2349
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2350
|
-
for (const line of lines) {
|
|
2351
|
-
if (line === "" || seen.has(line)) continue;
|
|
2352
|
-
seen.add(line);
|
|
2353
|
-
unique.push(line);
|
|
2354
|
-
}
|
|
2355
|
-
const output = [];
|
|
2356
|
-
let used = 0;
|
|
2357
|
-
for (const line of unique) {
|
|
2358
|
-
const cost = codePointLength(line) + (output.length === 0 ? 0 : 1);
|
|
2359
|
-
if (used + cost > budgetChars) continue;
|
|
2360
|
-
output.push(line);
|
|
2361
|
-
used += cost;
|
|
3467
|
+
function parseAdvisorScores(text, validSeqs) {
|
|
3468
|
+
if (text === void 0 || text.trim().length === 0) return void 0;
|
|
3469
|
+
const scores = /* @__PURE__ */ new Map();
|
|
3470
|
+
for (const object of jsonObjects(text)) {
|
|
3471
|
+
const seq = object.seq;
|
|
3472
|
+
const score = object.score;
|
|
3473
|
+
if (typeof seq !== "number" || !Number.isSafeInteger(seq) || !validSeqs.has(seq)) continue;
|
|
3474
|
+
if (typeof score !== "number" || !Number.isFinite(score) || score < 0 || score > 1) continue;
|
|
3475
|
+
const reason = typeof object.reason === "string" && object.reason.trim().length > 0 ? object.reason.trim() : void 0;
|
|
3476
|
+
scores.set(seq, {
|
|
3477
|
+
seq,
|
|
3478
|
+
score,
|
|
3479
|
+
...reason !== void 0 ? { reason } : {}
|
|
3480
|
+
});
|
|
2362
3481
|
}
|
|
2363
|
-
|
|
2364
|
-
return text.includes(requiredRef) ? text : null;
|
|
3482
|
+
return scores.size > 0 ? scores : void 0;
|
|
2365
3483
|
}
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
return
|
|
3484
|
+
//#endregion
|
|
3485
|
+
//#region src/runtime/tokenpilot/advisor.ts
|
|
3486
|
+
/** Character cap for the recent-text fallback and the tail-text summary input. */
|
|
3487
|
+
const TAIL_TEXT_CHAR_BUDGET = 4e3;
|
|
3488
|
+
/** Score the advisor assigns to candidates it has no answer for. */
|
|
3489
|
+
const NEUTRAL_RELEVANCE = .5;
|
|
3490
|
+
/** Deterministic djb2-derived hex digest for task-semantics versioning. */
|
|
3491
|
+
function versionDigest(text) {
|
|
3492
|
+
let hash = 5381;
|
|
3493
|
+
for (let index = 0; index < text.length; index += 1) hash = (hash * 33 ^ text.charCodeAt(index)) >>> 0;
|
|
3494
|
+
return hash.toString(16).padStart(8, "0");
|
|
3495
|
+
}
|
|
3496
|
+
/** Truncate on the character basis (Unicode code points), never UTF-16 units. */
|
|
3497
|
+
function truncateChars(text, budget) {
|
|
3498
|
+
if (codePointLength(text) <= budget) return text;
|
|
3499
|
+
return Array.from(text).slice(0, budget).join("");
|
|
3500
|
+
}
|
|
3501
|
+
/** Character cap of one candidate preview line offered to the scoring prompt. */
|
|
3502
|
+
const PREVIEW_CHAR_BUDGET = 200;
|
|
3503
|
+
/**
|
|
3504
|
+
* One candidate face for the scoring prompt: tool-call name plus the head of
|
|
3505
|
+
* the result text. Pure and shape-defensive.
|
|
3506
|
+
*/
|
|
3507
|
+
function advisorCandidatePreview(callName, blocks) {
|
|
3508
|
+
return truncateChars(`${callName} ${textBlocks(blocks)}`.trim(), PREVIEW_CHAR_BUDGET);
|
|
2377
3509
|
}
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
3510
|
+
/**
|
|
3511
|
+
* Collect the recent assistant narration tail (bounded, oldest-first join) as
|
|
3512
|
+
* summary-prompt context. Pure.
|
|
3513
|
+
*/
|
|
3514
|
+
function collectTailText(events, budget = TAIL_TEXT_CHAR_BUDGET) {
|
|
3515
|
+
const parts = [];
|
|
3516
|
+
let size = 0;
|
|
3517
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
3518
|
+
const event = events[index];
|
|
3519
|
+
if (event?.type !== "assistant/message") continue;
|
|
3520
|
+
const text = textBlocks(event.data).trim();
|
|
3521
|
+
if (text.length === 0) continue;
|
|
3522
|
+
parts.unshift(text);
|
|
3523
|
+
size += codePointLength(text);
|
|
3524
|
+
if (size >= budget) break;
|
|
3525
|
+
}
|
|
3526
|
+
return truncateChars(parts.join("\n"), budget);
|
|
3527
|
+
}
|
|
3528
|
+
function textBlocks(data) {
|
|
3529
|
+
const content = data?.content;
|
|
3530
|
+
if (!Array.isArray(content)) return "";
|
|
3531
|
+
const parts = [];
|
|
3532
|
+
for (const block of content) if (block?.type === "text" && typeof block.text === "string") parts.push(block.text);
|
|
3533
|
+
return parts.join("\n");
|
|
3534
|
+
}
|
|
3535
|
+
/** Structured probe of one `todo/write` payload: the list of task strings, or undefined. */
|
|
3536
|
+
function extractTodoItems(data) {
|
|
3537
|
+
const todos = data?.todos;
|
|
3538
|
+
const list = Array.isArray(todos) ? todos : Array.isArray(data) ? data : void 0;
|
|
3539
|
+
if (list === void 0 || list.length === 0) return void 0;
|
|
3540
|
+
const items = [];
|
|
3541
|
+
for (const entry of list) {
|
|
3542
|
+
if (typeof entry === "string") {
|
|
3543
|
+
if (entry.trim().length > 0) items.push(entry.trim());
|
|
3544
|
+
continue;
|
|
3545
|
+
}
|
|
3546
|
+
if (entry !== null && typeof entry === "object") {
|
|
3547
|
+
const record = entry;
|
|
3548
|
+
const text = [
|
|
3549
|
+
record.content,
|
|
3550
|
+
record.text,
|
|
3551
|
+
record.title,
|
|
3552
|
+
record.name
|
|
3553
|
+
].find((candidate) => typeof candidate === "string" && candidate.trim().length > 0);
|
|
3554
|
+
if (typeof text === "string") {
|
|
3555
|
+
items.push(text.trim());
|
|
3556
|
+
continue;
|
|
3557
|
+
}
|
|
3558
|
+
items.push(JSON.stringify(record));
|
|
3559
|
+
}
|
|
2389
3560
|
}
|
|
2390
|
-
|
|
2391
|
-
return output.join("\n");
|
|
2392
|
-
}
|
|
2393
|
-
function splitLines(text) {
|
|
2394
|
-
const lines = text.split("\n");
|
|
2395
|
-
if (text.endsWith("\n")) lines.pop();
|
|
2396
|
-
return lines;
|
|
3561
|
+
return items.length > 0 ? items : void 0;
|
|
2397
3562
|
}
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
3563
|
+
/**
|
|
3564
|
+
* Harvest task semantics for the summary/scoring prompts: the most recent
|
|
3565
|
+
* `todo/write` event (structured probe first, then the raw JSON string),
|
|
3566
|
+
* falling back to recent user/message text. Pure — log in, semantics out.
|
|
3567
|
+
*/
|
|
3568
|
+
function collectTaskSemantics(events) {
|
|
3569
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
3570
|
+
const event = events[index];
|
|
3571
|
+
if (event === void 0 || event.type !== "todo/write") continue;
|
|
3572
|
+
const data = event.data;
|
|
3573
|
+
const items = extractTodoItems(data);
|
|
3574
|
+
if (items !== void 0) {
|
|
3575
|
+
const taskText = truncateChars(items.join("\n"), TAIL_TEXT_CHAR_BUDGET);
|
|
3576
|
+
return {
|
|
3577
|
+
source: "todos",
|
|
3578
|
+
todoVersion: versionDigest(taskText),
|
|
3579
|
+
taskText
|
|
3580
|
+
};
|
|
2411
3581
|
}
|
|
2412
|
-
|
|
2413
|
-
return
|
|
3582
|
+
const raw = truncateChars(JSON.stringify(event.data) ?? "", TAIL_TEXT_CHAR_BUDGET);
|
|
3583
|
+
if (raw.length > 2) return {
|
|
3584
|
+
source: "raw-todo",
|
|
3585
|
+
todoVersion: versionDigest(raw),
|
|
3586
|
+
taskText: raw
|
|
3587
|
+
};
|
|
3588
|
+
}
|
|
3589
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
3590
|
+
const event = events[index];
|
|
3591
|
+
if (event?.type !== "user/message") continue;
|
|
3592
|
+
const text = truncateChars(textBlocks(event.data).trim(), TAIL_TEXT_CHAR_BUDGET);
|
|
3593
|
+
if (text.length === 0) continue;
|
|
3594
|
+
return {
|
|
3595
|
+
source: "messages",
|
|
3596
|
+
todoVersion: versionDigest(text),
|
|
3597
|
+
taskText: text
|
|
3598
|
+
};
|
|
2414
3599
|
}
|
|
2415
|
-
return "";
|
|
2416
|
-
}
|
|
2417
|
-
function looksLikeJson(text) {
|
|
2418
|
-
const trimmed = text.trim();
|
|
2419
|
-
return trimmed.startsWith("{") && trimmed.endsWith("}") || trimmed.startsWith("[") && trimmed.endsWith("]");
|
|
2420
|
-
}
|
|
2421
|
-
function isReadTool(name) {
|
|
2422
|
-
return /(?:^|[-_/])(?:read|cat|view|open_file)(?:$|[-_/])/.test(name);
|
|
2423
3600
|
}
|
|
2424
|
-
|
|
2425
|
-
|
|
3601
|
+
/**
|
|
3602
|
+
* Prefix-decay figure: 1 minus the character-pressure-weighted mean relevance
|
|
3603
|
+
* of the prefix candidates. Unscored candidates count as neutral 0.5. Pure,
|
|
3604
|
+
* deterministic, no LLM and no I/O.
|
|
3605
|
+
*/
|
|
3606
|
+
function prefixDecay(candidates, scores) {
|
|
3607
|
+
let totalWeight = 0;
|
|
3608
|
+
let weightedRelevance = 0;
|
|
3609
|
+
for (const candidate of candidates) {
|
|
3610
|
+
const weight = candidate.characterPressure > 0 ? candidate.characterPressure : 0;
|
|
3611
|
+
if (weight === 0) continue;
|
|
3612
|
+
totalWeight += weight;
|
|
3613
|
+
weightedRelevance += weight * (scores.get(candidate.seq)?.score ?? NEUTRAL_RELEVANCE);
|
|
3614
|
+
}
|
|
3615
|
+
if (totalWeight === 0) return {
|
|
3616
|
+
decay: 0,
|
|
3617
|
+
weightedChars: 0
|
|
3618
|
+
};
|
|
3619
|
+
return {
|
|
3620
|
+
decay: 1 - weightedRelevance / totalWeight,
|
|
3621
|
+
weightedChars: totalWeight
|
|
3622
|
+
};
|
|
2426
3623
|
}
|
|
2427
|
-
|
|
2428
|
-
|
|
3624
|
+
/** Lowercase word tokens used by the local keyword-overlap prescreen. */
|
|
3625
|
+
function keywordsOf(text) {
|
|
3626
|
+
const matches = text.toLowerCase().match(/[\p{L}\p{N}_-]{3,}/gu) ?? [];
|
|
3627
|
+
return new Set(matches);
|
|
2429
3628
|
}
|
|
2430
|
-
function
|
|
2431
|
-
|
|
3629
|
+
function overlapCount(left, right) {
|
|
3630
|
+
let count = 0;
|
|
3631
|
+
for (const token of right) if (left.has(token)) count += 1;
|
|
3632
|
+
return count;
|
|
2432
3633
|
}
|
|
2433
|
-
|
|
2434
|
-
|
|
3634
|
+
/**
|
|
3635
|
+
* Incremental scoring selection: candidates newer than the watermark whose
|
|
3636
|
+
* character pressure reaches the token-named floor, ranked by local keyword
|
|
3637
|
+
* overlap with the task semantics and cut at the sample limit. When the task
|
|
3638
|
+
* semantics changed, the watermark is ignored so every eligible candidate can
|
|
3639
|
+
* rescore. Pure.
|
|
3640
|
+
*/
|
|
3641
|
+
function selectScoringCandidates(candidates, state, input) {
|
|
3642
|
+
const eligible = [];
|
|
3643
|
+
for (const candidate of candidates) {
|
|
3644
|
+
if (!input.taskChanged && candidate.seq <= state.watermarkSeq) continue;
|
|
3645
|
+
if (candidate.characterPressure < input.minChars) continue;
|
|
3646
|
+
eligible.push({
|
|
3647
|
+
...candidate,
|
|
3648
|
+
overlap: overlapCount(input.taskKeywords, keywordsOf(candidate.preview))
|
|
3649
|
+
});
|
|
3650
|
+
}
|
|
3651
|
+
eligible.sort((left, right) => right.overlap - left.overlap || right.characterPressure - left.characterPressure);
|
|
3652
|
+
return eligible.slice(0, input.sampleLimit).map(({ overlap: _overlap, ...candidate }) => candidate);
|
|
2435
3653
|
}
|
|
2436
|
-
function
|
|
2437
|
-
return
|
|
3654
|
+
function advisorAudit(input, phase, fields) {
|
|
3655
|
+
return {
|
|
3656
|
+
schemaVersion: 1,
|
|
3657
|
+
kind: "advisor-outcome",
|
|
3658
|
+
sessionId: input.sessionId,
|
|
3659
|
+
phase,
|
|
3660
|
+
turnIndex: input.turn,
|
|
3661
|
+
...fields
|
|
3662
|
+
};
|
|
2438
3663
|
}
|
|
2439
|
-
|
|
2440
|
-
|
|
3664
|
+
/**
|
|
3665
|
+
* One full advisor pass: summary refresh (todo change or every refreshTurns),
|
|
3666
|
+
* incremental batch scoring with recertification marks, then the decay
|
|
3667
|
+
* figure. State is written only on success; any failure leaves state
|
|
3668
|
+
* untouched, emits ok:false audits with reason codes, and never throws.
|
|
3669
|
+
*/
|
|
3670
|
+
async function runAdvisorPass(state, channel, emit, input) {
|
|
3671
|
+
if (input.task === void 0) return void 0;
|
|
3672
|
+
const taskChanged = invalidateOnTaskChange(state, input.task.todoVersion);
|
|
3673
|
+
const turn = input.turn;
|
|
3674
|
+
if (state.summary === void 0 || turn - state.lastSummaryTurn >= input.advisor.refreshTurns) {
|
|
3675
|
+
const summaryStarted = Date.now();
|
|
3676
|
+
const summaryText = await channel.ask({
|
|
3677
|
+
system: buildAdvisorSummarySystemPrompt(),
|
|
3678
|
+
user: buildAdvisorSummaryUserPrompt(input.task.taskText, input.tailText),
|
|
3679
|
+
signal: input.signal
|
|
3680
|
+
});
|
|
3681
|
+
const summaryLatencyMs = Date.now() - summaryStarted;
|
|
3682
|
+
const summary = input.signal.aborted ? void 0 : parseAdvisorSummary(summaryText);
|
|
3683
|
+
if (summary === void 0) {
|
|
3684
|
+
emit(advisorAudit(input, "summary", {
|
|
3685
|
+
ok: false,
|
|
3686
|
+
latencyMs: summaryLatencyMs,
|
|
3687
|
+
...input.signal.aborted ? { reason: "aborted" } : summaryText === void 0 ? { reason: "channel-empty" } : { reason: "parse-failed" }
|
|
3688
|
+
}));
|
|
3689
|
+
return;
|
|
3690
|
+
}
|
|
3691
|
+
state.summary = {
|
|
3692
|
+
...summary,
|
|
3693
|
+
todoVersion: input.task.todoVersion,
|
|
3694
|
+
turn
|
|
3695
|
+
};
|
|
3696
|
+
state.lastSummaryTurn = turn;
|
|
3697
|
+
emit(advisorAudit(input, "summary", {
|
|
3698
|
+
ok: true,
|
|
3699
|
+
latencyMs: summaryLatencyMs
|
|
3700
|
+
}));
|
|
3701
|
+
}
|
|
3702
|
+
const summary = state.summary;
|
|
3703
|
+
if (summary === void 0) return void 0;
|
|
3704
|
+
const sampled = selectScoringCandidates(input.candidates, state, {
|
|
3705
|
+
taskKeywords: keywordsOf(`${input.task.taskText}\n${summary.keywords.join(" ")}`),
|
|
3706
|
+
minChars: charsForTokens(input.advisor.minTokens),
|
|
3707
|
+
sampleLimit: input.advisor.sampleLimit,
|
|
3708
|
+
taskChanged
|
|
3709
|
+
});
|
|
3710
|
+
let scored = 0;
|
|
3711
|
+
let highestScored = 0;
|
|
3712
|
+
if (sampled.length > 0) {
|
|
3713
|
+
const scoringStarted = Date.now();
|
|
3714
|
+
const scoresText = await channel.ask({
|
|
3715
|
+
system: buildAdvisorScoringSystemPrompt(),
|
|
3716
|
+
user: buildAdvisorScoringUserPrompt(input.task.taskText, summary.activeSubtasks, sampled),
|
|
3717
|
+
signal: input.signal
|
|
3718
|
+
});
|
|
3719
|
+
const scoringLatencyMs = Date.now() - scoringStarted;
|
|
3720
|
+
const scores = input.signal.aborted ? void 0 : parseAdvisorScores(scoresText, new Set(sampled.map((item) => item.seq)));
|
|
3721
|
+
if (scores === void 0 || scores.size === 0) {
|
|
3722
|
+
emit(advisorAudit(input, "scoring", {
|
|
3723
|
+
ok: false,
|
|
3724
|
+
sampledCount: sampled.length,
|
|
3725
|
+
latencyMs: scoringLatencyMs,
|
|
3726
|
+
...input.signal.aborted ? { reason: "aborted" } : scoresText === void 0 ? { reason: "channel-empty" } : { reason: "parse-failed" }
|
|
3727
|
+
}));
|
|
3728
|
+
return;
|
|
3729
|
+
}
|
|
3730
|
+
for (const candidate of sampled) {
|
|
3731
|
+
const answer = scores.get(candidate.seq);
|
|
3732
|
+
if (answer === void 0) continue;
|
|
3733
|
+
recordScore(state, candidate.seq, {
|
|
3734
|
+
score: answer.score,
|
|
3735
|
+
turn
|
|
3736
|
+
});
|
|
3737
|
+
scored += 1;
|
|
3738
|
+
if (candidate.seq > highestScored) highestScored = candidate.seq;
|
|
3739
|
+
if (answer.score < input.advisor.scoreThreshold) recordRecertified(state, candidate.seq, turn);
|
|
3740
|
+
}
|
|
3741
|
+
emit(advisorAudit(input, "scoring", {
|
|
3742
|
+
ok: true,
|
|
3743
|
+
sampledCount: sampled.length,
|
|
3744
|
+
latencyMs: scoringLatencyMs
|
|
3745
|
+
}));
|
|
3746
|
+
if (highestScored > state.watermarkSeq) state.watermarkSeq = highestScored;
|
|
3747
|
+
}
|
|
3748
|
+
const decay = prefixDecay(input.candidates, state.scores);
|
|
3749
|
+
state.lastDecay = {
|
|
3750
|
+
decay: decay.decay,
|
|
3751
|
+
weightedChars: decay.weightedChars,
|
|
3752
|
+
turn
|
|
3753
|
+
};
|
|
3754
|
+
emit(advisorAudit(input, "decay", {
|
|
3755
|
+
ok: true,
|
|
3756
|
+
sampledCount: scored,
|
|
3757
|
+
decay: decay.decay,
|
|
3758
|
+
weightedChars: decay.weightedChars,
|
|
3759
|
+
latencyMs: 0
|
|
3760
|
+
}));
|
|
3761
|
+
return {
|
|
3762
|
+
decay: decay.decay,
|
|
3763
|
+
weightedChars: decay.weightedChars,
|
|
3764
|
+
sampled: sampled.length
|
|
3765
|
+
};
|
|
2441
3766
|
}
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
3767
|
+
/**
|
|
3768
|
+
* Convenience entry used by the pruner: fetch-or-create the session state and
|
|
3769
|
+
* run one pass against it.
|
|
3770
|
+
*/
|
|
3771
|
+
async function runSessionAdvisorPass(session, channel, emit, input) {
|
|
3772
|
+
const state = getAdvisorState(session);
|
|
3773
|
+
if (state.inFlight) return void 0;
|
|
3774
|
+
state.inFlight = true;
|
|
3775
|
+
try {
|
|
3776
|
+
return await runAdvisorPass(state, channel, emit, {
|
|
3777
|
+
...input,
|
|
3778
|
+
sessionId: input.sessionId ?? String(session.id)
|
|
3779
|
+
});
|
|
3780
|
+
} finally {
|
|
3781
|
+
state.inFlight = false;
|
|
3782
|
+
}
|
|
2448
3783
|
}
|
|
2449
3784
|
//#endregion
|
|
2450
3785
|
//#region src/runtime/deepseek-official-pricing.ts
|
|
@@ -2727,18 +4062,25 @@ function deriveAdaptiveTokenBounds(input) {
|
|
|
2727
4062
|
if (reclaimedLowerBoundTokens === 0) return unknown("reclaim-not-positive-after-margin");
|
|
2728
4063
|
let identity;
|
|
2729
4064
|
let exactPrefixLowerBoundTokens = 0;
|
|
4065
|
+
let characterDerivedPrefix = false;
|
|
2730
4066
|
const seen = /* @__PURE__ */ new Set();
|
|
2731
4067
|
for (const node of input.measuredNodes) {
|
|
2732
4068
|
if (!isCount(node.seq) || seen.has(node.seq)) return unknown("invalid-measured-node-sequence");
|
|
2733
4069
|
seen.add(node.seq);
|
|
2734
|
-
if (node.seq >= input.earliestChangedSeq
|
|
2735
|
-
if (
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
4070
|
+
if (node.seq >= input.earliestChangedSeq) continue;
|
|
4071
|
+
if (node.count.kind === "exact-tokenizer") {
|
|
4072
|
+
if (!isCount(node.count.tokens)) return unknown("invalid-exact-prefix-count");
|
|
4073
|
+
if (node.count.tokenizerRevision !== input.expectedTokenizerRevision) return unknown("exact-prefix-tokenizer-revision-mismatch");
|
|
4074
|
+
if (identity !== void 0 && (identity.tokenizerId !== node.count.tokenizerId || identity.tokenizerRevision !== node.count.tokenizerRevision)) return unknown("exact-prefix-tokenizer-identity-mismatch");
|
|
4075
|
+
identity ??= node.count;
|
|
4076
|
+
exactPrefixLowerBoundTokens += node.count.tokens;
|
|
4077
|
+
} else {
|
|
4078
|
+
exactPrefixLowerBoundTokens += charsToTokens(node.characterPressure);
|
|
4079
|
+
characterDerivedPrefix = true;
|
|
4080
|
+
}
|
|
2740
4081
|
if (!isCount(exactPrefixLowerBoundTokens)) return unknown("exact-prefix-overflow");
|
|
2741
4082
|
}
|
|
4083
|
+
if (characterDerivedPrefix) measurementKind = "characters";
|
|
2742
4084
|
const accounted = exactPrefixLowerBoundTokens + reclaimedLowerBoundTokens;
|
|
2743
4085
|
if (!isCount(accounted) || accounted > input.previousPromptTokens) return unknown("adaptive-bounds-exceed-previous-prompt");
|
|
2744
4086
|
return {
|
|
@@ -2870,12 +4212,14 @@ var ToolResultPruner = class extends Service {
|
|
|
2870
4212
|
activeRequestBoundaries: /* @__PURE__ */ new WeakMap(),
|
|
2871
4213
|
tailTrimBoundaryAttempts: /* @__PURE__ */ new WeakMap(),
|
|
2872
4214
|
policyResolutionAudits: /* @__PURE__ */ new WeakMap(),
|
|
2873
|
-
reviewStore:
|
|
4215
|
+
reviewStore: sharedReviewStore(),
|
|
2874
4216
|
reviewQueues: /* @__PURE__ */ new WeakMap(),
|
|
2875
4217
|
reviewClocks: /* @__PURE__ */ new WeakMap(),
|
|
2876
4218
|
estimatorRemainingTurns: /* @__PURE__ */ new WeakMap(),
|
|
4219
|
+
advisorChannels: /* @__PURE__ */ new WeakMap(),
|
|
2877
4220
|
reviewSummaries: /* @__PURE__ */ new WeakMap()
|
|
2878
4221
|
};
|
|
4222
|
+
ctx.effect(() => registerReviewPruner(this), "contextCompressionSelector.reviewRegistry()");
|
|
2879
4223
|
openReviewStorage((name) => this.ctx.get(name)).then((store) => {
|
|
2880
4224
|
if (store !== void 0) this.state.reviewStore = store;
|
|
2881
4225
|
}).catch(() => {
|
|
@@ -2928,6 +4272,7 @@ var ToolResultPruner = class extends Service {
|
|
|
2928
4272
|
ctx.logger.warn("context-compression review turn-boundary pass failed open: %o", error);
|
|
2929
4273
|
}
|
|
2930
4274
|
this.postflightEstimatorPass(agent.session, signal).catch(() => void 0);
|
|
4275
|
+
this.postflightAdvisorPass(agent.session, turn, signal).catch(() => void 0);
|
|
2931
4276
|
});
|
|
2932
4277
|
}
|
|
2933
4278
|
/**
|
|
@@ -2964,15 +4309,14 @@ var ToolResultPruner = class extends Service {
|
|
|
2964
4309
|
const landed = [];
|
|
2965
4310
|
if (policy.nativeToolResultEnabled) {
|
|
2966
4311
|
const eligible = this.snapshot(session, view).filter((candidate) => !this.isRecoveryExempt(session, candidate));
|
|
2967
|
-
|
|
2968
|
-
if (exactUnavailable) this.warnExactUnavailable(session, view, "native");
|
|
4312
|
+
if (eligible.some((candidate) => candidate.count.kind !== "exact-tokenizer")) this.warnExactUnavailable(session, view, "native");
|
|
2969
4313
|
const planned = eligible.map((candidate) => this.planNative(candidate, session, stage, policy, view)).filter((entry) => entry !== null);
|
|
2970
|
-
landed.push(...this.landAll(session, this.triageForReview(session, policy, planned)));
|
|
4314
|
+
landed.push(...this.landAll(session, this.triageForReview(session, policy, planned, "history")));
|
|
2971
4315
|
if (landed.length === 0) {
|
|
2972
|
-
const
|
|
2973
|
-
this.auditComponent(session, policy, "native-tool-result", "pressure", "skipped",
|
|
2974
|
-
measurementKind:
|
|
2975
|
-
...
|
|
4316
|
+
const chars = eligible.map((candidate) => candidate.characterPressure);
|
|
4317
|
+
this.auditComponent(session, policy, "native-tool-result", "pressure", "skipped", chars.length === 0 ? "no-tool-result-candidates" : Math.max(...chars) <= charsForTokens(policy.nativeTriggerTokens) ? "at-or-below-trigger" : planned.length === 0 ? "no-valid-reduction" : "recovery-tool-unavailable", {
|
|
4318
|
+
measurementKind: "characters",
|
|
4319
|
+
...chars.length === 0 ? {} : { currentTokens: charsToTokens(Math.max(...chars)) },
|
|
2976
4320
|
triggerTokens: policy.nativeTriggerTokens,
|
|
2977
4321
|
targetTokens: policy.nativeTargetTokens
|
|
2978
4322
|
});
|
|
@@ -2989,13 +4333,13 @@ var ToolResultPruner = class extends Service {
|
|
|
2989
4333
|
if (historyOutcome.kind === "planned") {
|
|
2990
4334
|
const capacityPressure = this.capacityPressureActive(session, view, policy);
|
|
2991
4335
|
historyAllowed = this.adaptiveHistoryAllowed(session, view, historyOutcome.plans, capacityPressure);
|
|
2992
|
-
if (historyAllowed) landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans)));
|
|
4336
|
+
if (historyAllowed) landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans, "history")));
|
|
2993
4337
|
}
|
|
2994
4338
|
} else {
|
|
2995
4339
|
historyAllowed = this.historyAllowed(session, policy, view);
|
|
2996
4340
|
if (historyAllowed) {
|
|
2997
4341
|
historyOutcome = this.planHistoricalAging(session, policy, view);
|
|
2998
|
-
if (historyOutcome.kind === "planned") landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans)));
|
|
4342
|
+
if (historyOutcome.kind === "planned") landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans, "history")));
|
|
2999
4343
|
}
|
|
3000
4344
|
}
|
|
3001
4345
|
if (!landed.some((entry) => entry.stage === "pressure")) this.auditHistoryEvaluation(session, policy, view, historyAllowed, historyOutcome);
|
|
@@ -3127,8 +4471,7 @@ var ToolResultPruner = class extends Service {
|
|
|
3127
4471
|
for (const candidate of this.snapshot(session, measureForCompaction(this.ctx, session))) {
|
|
3128
4472
|
if (samples.length >= 3) break;
|
|
3129
4473
|
if (candidate.event.data.turn === void 0) continue;
|
|
3130
|
-
|
|
3131
|
-
if (tokens === void 0 || tokens <= policy.freshTriggerTokens) continue;
|
|
4474
|
+
if (candidate.characterPressure <= charsForTokens(policy.freshTriggerTokens)) continue;
|
|
3132
4475
|
const path = toolCallPath(candidate.call.arguments);
|
|
3133
4476
|
if (path === void 0) continue;
|
|
3134
4477
|
if (isSupersededRead(events, candidate.seq, path)) continue;
|
|
@@ -3177,6 +4520,88 @@ var ToolResultPruner = class extends Service {
|
|
|
3177
4520
|
});
|
|
3178
4521
|
}
|
|
3179
4522
|
/**
|
|
4523
|
+
* Advisory advisor pass at the turn boundary, strictly fire-and-forget.
|
|
4524
|
+
* Produces todolist-bound tail-task summaries, incremental relevance
|
|
4525
|
+
* scores, and a prefix-decay figure — all observational. Every short
|
|
4526
|
+
* circuit below (mode off, re-entry, cooldown, no task semantics, no
|
|
4527
|
+
* direct endpoint) returns without touching any state the pruning chain
|
|
4528
|
+
* reads, so the default configuration adds exactly zero behavior.
|
|
4529
|
+
*/
|
|
4530
|
+
async postflightAdvisorPass(session, turn, signal) {
|
|
4531
|
+
const policy = this.activePolicy(session);
|
|
4532
|
+
const presetOptions = policy?.presetOptions;
|
|
4533
|
+
const advisor = presetOptions?.advisor;
|
|
4534
|
+
if (policy === void 0 || presetOptions === void 0 || advisor === void 0 || advisor.mode === "") return;
|
|
4535
|
+
const advisorState = getAdvisorState(session);
|
|
4536
|
+
if (advisorState.inFlight) return;
|
|
4537
|
+
if (isCoolingDown(advisorState.failures, Date.now())) return;
|
|
4538
|
+
const events = sessionEvents(session);
|
|
4539
|
+
const task = collectTaskSemantics(events);
|
|
4540
|
+
if (task === void 0) return;
|
|
4541
|
+
const settings = this.activeSettings(session).presetOptions ?? {};
|
|
4542
|
+
if (advisor.mode === "direct" && (settings.estimatorBaseUrl === void 0 || settings.estimatorBaseUrl.length === 0 || settings.estimatorModel === void 0 || settings.estimatorModel.length === 0)) {
|
|
4543
|
+
emitCompressionAudit(this.ctx.logger, {
|
|
4544
|
+
schemaVersion: 1,
|
|
4545
|
+
kind: "advisor-outcome",
|
|
4546
|
+
sessionId: String(session.id),
|
|
4547
|
+
phase: "summary",
|
|
4548
|
+
channel: "direct",
|
|
4549
|
+
ok: false,
|
|
4550
|
+
turnIndex: turn,
|
|
4551
|
+
reason: "no-direct-endpoint",
|
|
4552
|
+
latencyMs: 0
|
|
4553
|
+
});
|
|
4554
|
+
advisorState.failures = {
|
|
4555
|
+
failures: (advisorState.failures?.failures ?? 0) + 1,
|
|
4556
|
+
cooldownUntil: Date.now() + backoffCooldownMs((advisorState.failures?.failures ?? 0) + 1)
|
|
4557
|
+
};
|
|
4558
|
+
return;
|
|
4559
|
+
}
|
|
4560
|
+
let channel = this.state.advisorChannels.get(session);
|
|
4561
|
+
if (channel === void 0) {
|
|
4562
|
+
channel = new SideChannel(this.ctx, settings, {
|
|
4563
|
+
mode: advisor.mode,
|
|
4564
|
+
timeoutMs: advisor.timeoutMs,
|
|
4565
|
+
maxTokens: 512
|
|
4566
|
+
});
|
|
4567
|
+
this.state.advisorChannels.set(session, channel);
|
|
4568
|
+
}
|
|
4569
|
+
const view = measureForCompaction(this.ctx, session);
|
|
4570
|
+
const candidates = this.snapshot(session, view).filter((candidate) => !this.isRecoveryExempt(session, candidate)).map((candidate) => ({
|
|
4571
|
+
seq: candidate.seq,
|
|
4572
|
+
characterPressure: candidate.characterPressure,
|
|
4573
|
+
preview: advisorCandidatePreview(candidate.call.name, candidate.event.data.message.content)
|
|
4574
|
+
}));
|
|
4575
|
+
let sawFailure = false;
|
|
4576
|
+
const outcome = await runSessionAdvisorPass(session, channel, (record) => {
|
|
4577
|
+
if (record.ok === false) sawFailure = true;
|
|
4578
|
+
emitCompressionAudit(this.ctx.logger, record);
|
|
4579
|
+
}, {
|
|
4580
|
+
profile: policy.profile,
|
|
4581
|
+
sessionId: String(session.id),
|
|
4582
|
+
turn,
|
|
4583
|
+
candidates,
|
|
4584
|
+
task: {
|
|
4585
|
+
source: task.source,
|
|
4586
|
+
todoVersion: task.todoVersion,
|
|
4587
|
+
taskText: task.taskText
|
|
4588
|
+
},
|
|
4589
|
+
advisor: {
|
|
4590
|
+
refreshTurns: advisor.refreshTurns,
|
|
4591
|
+
scoreThreshold: advisor.scoreThreshold,
|
|
4592
|
+
sampleLimit: advisor.sampleLimit,
|
|
4593
|
+
minTokens: advisor.minTokens
|
|
4594
|
+
},
|
|
4595
|
+
tailText: collectTailText(events),
|
|
4596
|
+
signal
|
|
4597
|
+
});
|
|
4598
|
+
if (outcome === void 0 && sawFailure && signal.aborted === false) advisorState.failures = {
|
|
4599
|
+
failures: (advisorState.failures?.failures ?? 0) + 1,
|
|
4600
|
+
cooldownUntil: Date.now() + backoffCooldownMs((advisorState.failures?.failures ?? 0) + 1)
|
|
4601
|
+
};
|
|
4602
|
+
else if (outcome !== void 0) advisorState.failures = void 0;
|
|
4603
|
+
}
|
|
4604
|
+
/**
|
|
3180
4605
|
* The per-session review queue, or `undefined` while review mode is off
|
|
3181
4606
|
* (every review path must then behave exactly like before).
|
|
3182
4607
|
*/
|
|
@@ -3228,7 +4653,7 @@ var ToolResultPruner = class extends Service {
|
|
|
3228
4653
|
* The digest freezes each candidate's ORIGINAL surface content, so the apply
|
|
3229
4654
|
* point can prove "what is removed now is what was approved then".
|
|
3230
4655
|
*/
|
|
3231
|
-
triageForReview(session, policy, plans) {
|
|
4656
|
+
triageForReview(session, policy, plans, stage = "history") {
|
|
3232
4657
|
const queue = this.reviewQueueFor(session, policy);
|
|
3233
4658
|
if (queue === void 0 || plans.length === 0) return plans;
|
|
3234
4659
|
const presetOptions = policy.presetOptions;
|
|
@@ -3239,7 +4664,8 @@ var ToolResultPruner = class extends Service {
|
|
|
3239
4664
|
tailTokens: Math.max(1, policy.historyKeepRecentTokens),
|
|
3240
4665
|
reviewHighImpactTokens: presetOptions.reviewHighImpactTokens,
|
|
3241
4666
|
...this.state.estimatorRemainingTurns.get(session) === void 0 ? {} : { remainingTurns: this.state.estimatorRemainingTurns.get(session) },
|
|
3242
|
-
estimatorSeqs
|
|
4667
|
+
estimatorSeqs,
|
|
4668
|
+
stage
|
|
3243
4669
|
};
|
|
3244
4670
|
const classified = classifyCandidates(plans.map((plan) => ({
|
|
3245
4671
|
sourceSeq: plan.sourceSeq,
|
|
@@ -3744,9 +5170,8 @@ var ToolResultPruner = class extends Service {
|
|
|
3744
5170
|
const plans = /* @__PURE__ */ new Map();
|
|
3745
5171
|
let freshPlanned = 0;
|
|
3746
5172
|
const dedupeEnabled = policy.presetOptions?.dedupeToolResults === true;
|
|
3747
|
-
const
|
|
3748
|
-
const
|
|
3749
|
-
const maxCandidateTokens = exactAvailable ? Math.max(...exactCandidateTokens) : void 0;
|
|
5173
|
+
const candidateChars = candidates.map((candidate) => candidate.characterPressure);
|
|
5174
|
+
const maxCandidateChars = candidateChars.length === 0 ? void 0 : Math.max(...candidateChars);
|
|
3750
5175
|
if (policy.freshEnabled) {
|
|
3751
5176
|
if (candidates.some((candidate) => candidate.call.name !== "context_compression_retrieve" && candidate.count.kind !== "exact-tokenizer")) this.warnExactUnavailable(session, view, "fresh");
|
|
3752
5177
|
for (const candidate of candidates) {
|
|
@@ -3765,41 +5190,39 @@ var ToolResultPruner = class extends Service {
|
|
|
3765
5190
|
}
|
|
3766
5191
|
}
|
|
3767
5192
|
}
|
|
3768
|
-
let
|
|
5193
|
+
let aggregateInputChars;
|
|
3769
5194
|
let aggregatePlanned = 0;
|
|
3770
5195
|
if (policy.aggregateEnabled) {
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
if (aggregateAvailable && total > policy.aggregateTriggerTokens) {
|
|
3776
|
-
const remaining = candidates.filter((candidate) => !this.isRecoveryExempt(session, candidate)).sort((a, b) => Number(isError(a)) - Number(isError(b)) || (plans.get(b.seq)?.tokensAfter ?? exactTokens(b.count) ?? 0) - (plans.get(a.seq)?.tokensAfter ?? exactTokens(a.count) ?? 0));
|
|
5196
|
+
let total = candidates.reduce((sum, candidate) => sum + (plans.get(candidate.seq)?.charsAfter ?? candidate.characterPressure), 0);
|
|
5197
|
+
aggregateInputChars = total;
|
|
5198
|
+
if (total > charsForTokens(policy.aggregateTriggerTokens)) {
|
|
5199
|
+
const remaining = candidates.filter((candidate) => !this.isRecoveryExempt(session, candidate)).sort((a, b) => Number(isError(a)) - Number(isError(b)) || (plans.get(b.seq)?.charsAfter ?? b.characterPressure) - (plans.get(a.seq)?.charsAfter ?? a.characterPressure));
|
|
3777
5200
|
for (const candidate of remaining) {
|
|
3778
5201
|
const previous = plans.get(candidate.seq);
|
|
3779
5202
|
const plan = this.planAggregate(candidate, session, view);
|
|
3780
|
-
const
|
|
3781
|
-
if (plan === null || plan.
|
|
5203
|
+
const previousChars = previous?.charsAfter ?? candidate.characterPressure;
|
|
5204
|
+
if (plan === null || plan.charsAfter >= previousChars) continue;
|
|
3782
5205
|
plans.set(candidate.seq, plan);
|
|
3783
5206
|
aggregatePlanned += 1;
|
|
3784
|
-
total -=
|
|
3785
|
-
if (total <= policy.aggregateTargetTokens) break;
|
|
5207
|
+
total -= previousChars - plan.charsAfter;
|
|
5208
|
+
if (total <= charsForTokens(policy.aggregateTargetTokens)) break;
|
|
3786
5209
|
}
|
|
3787
|
-
if (total > policy.aggregateTargetTokens) this.ctx.logger.warn("context-compression fresh aggregate residual: %d
|
|
5210
|
+
if (total > charsForTokens(policy.aggregateTargetTokens)) this.ctx.logger.warn("context-compression fresh aggregate residual: %d characters exceed target %d", total, charsForTokens(policy.aggregateTargetTokens));
|
|
3788
5211
|
}
|
|
3789
5212
|
}
|
|
3790
5213
|
const freshCandidates = candidates.map((candidate) => plans.get(candidate.seq)).filter((plan) => plan !== void 0);
|
|
3791
|
-
const landed = this.landAll(session, this.triageForReview(session, policy, freshCandidates));
|
|
5214
|
+
const landed = this.landAll(session, this.triageForReview(session, policy, freshCandidates, "fresh"));
|
|
3792
5215
|
const freshLanded = landed.some((entry) => entry.stage === "fresh" && plans.get(entry.originalSeq)?.component === "fresh");
|
|
3793
5216
|
const aggregateLanded = landed.some((entry) => entry.stage === "fresh" && plans.get(entry.originalSeq)?.component === "aggregate");
|
|
3794
|
-
if (!freshLanded) this.auditComponent(session, policy, "fresh", "fresh", policy.freshEnabled ? "skipped" : "disabled", !policy.freshEnabled ? "profile-policy" :
|
|
3795
|
-
measurementKind:
|
|
3796
|
-
...
|
|
5217
|
+
if (!freshLanded) this.auditComponent(session, policy, "fresh", "fresh", policy.freshEnabled ? "skipped" : "disabled", !policy.freshEnabled ? "profile-policy" : (maxCandidateChars ?? 0) <= charsForTokens(policy.freshTriggerTokens) ? "at-or-below-trigger" : freshPlanned > 0 && aggregatePlanned > 0 ? "superseded-by-aggregate" : freshPlanned === 0 ? "no-valid-reduction" : "recovery-tool-unavailable", {
|
|
5218
|
+
measurementKind: "characters",
|
|
5219
|
+
...maxCandidateChars === void 0 ? {} : { currentTokens: charsToTokens(maxCandidateChars) },
|
|
3797
5220
|
triggerTokens: policy.freshTriggerTokens,
|
|
3798
5221
|
targetTokens: policy.freshTargetTokens
|
|
3799
5222
|
});
|
|
3800
|
-
if (!aggregateLanded) this.auditComponent(session, policy, "aggregate", "fresh", policy.aggregateEnabled ? "skipped" : "disabled", !policy.aggregateEnabled ? "profile-policy" :
|
|
3801
|
-
measurementKind:
|
|
3802
|
-
...
|
|
5223
|
+
if (!aggregateLanded) this.auditComponent(session, policy, "aggregate", "fresh", policy.aggregateEnabled ? "skipped" : "disabled", !policy.aggregateEnabled ? "profile-policy" : (aggregateInputChars ?? 0) <= charsForTokens(policy.aggregateTriggerTokens) ? "at-or-below-trigger" : aggregatePlanned === 0 ? "no-valid-reduction" : "recovery-tool-unavailable", {
|
|
5224
|
+
measurementKind: "characters",
|
|
5225
|
+
...aggregateInputChars === void 0 ? {} : { currentTokens: charsToTokens(aggregateInputChars) },
|
|
3803
5226
|
triggerTokens: policy.aggregateTriggerTokens,
|
|
3804
5227
|
targetTokens: policy.aggregateTargetTokens
|
|
3805
5228
|
});
|
|
@@ -3838,16 +5261,15 @@ var ToolResultPruner = class extends Service {
|
|
|
3838
5261
|
}
|
|
3839
5262
|
planNative(candidate, session, stage, policy, view) {
|
|
3840
5263
|
if (this.isRecoveryExempt(session, candidate)) return null;
|
|
3841
|
-
|
|
3842
|
-
if (tokensBefore === void 0 || tokensBefore <= policy.nativeTriggerTokens) return null;
|
|
5264
|
+
if (candidate.characterPressure <= charsForTokens(policy.nativeTriggerTokens)) return null;
|
|
3843
5265
|
const result = candidate.event.data.message.content[0];
|
|
3844
5266
|
if (onlyTextBlocks(result.content) === null) return null;
|
|
3845
5267
|
const sourceSeq = rootToolResultSeq(session, candidate.seq);
|
|
3846
|
-
const marker = recoveryMarker(sourceRef(session, sourceSeq), "tool result middle pruned");
|
|
5268
|
+
const marker = (startLine) => recoveryMarker(sourceRef(session, sourceSeq), "tool result middle pruned", startLine);
|
|
3847
5269
|
let head = this.state.config.headChars;
|
|
3848
5270
|
let tail = this.state.config.tailChars;
|
|
3849
5271
|
for (let attempt = 0; attempt < 10; attempt += 1) {
|
|
3850
|
-
const threshold = head + codePointLength(marker) + tail;
|
|
5272
|
+
const threshold = head + codePointLength(marker(1)) + tail;
|
|
3851
5273
|
const content = nativePruneContent(result.content, threshold, head, tail, marker);
|
|
3852
5274
|
if (content !== null) {
|
|
3853
5275
|
const plan = this.plan(candidate, content, sourceSeq, "native-head-tail", stage, "native-tool-result", void 0, view);
|
|
@@ -3870,8 +5292,7 @@ var ToolResultPruner = class extends Service {
|
|
|
3870
5292
|
const result = candidate.event.data.message.content[0];
|
|
3871
5293
|
const text = flattenPlainText(result.content);
|
|
3872
5294
|
if (text === void 0) return null;
|
|
3873
|
-
|
|
3874
|
-
if (tokensBefore === void 0 || tokensBefore <= policy.freshTriggerTokens) return null;
|
|
5295
|
+
if (candidate.characterPressure <= charsForTokens(policy.freshTriggerTokens)) return null;
|
|
3875
5296
|
let table = this.state.dedupeTables.get(session);
|
|
3876
5297
|
if (table === void 0) {
|
|
3877
5298
|
table = new DedupeTable();
|
|
@@ -3899,8 +5320,7 @@ var ToolResultPruner = class extends Service {
|
|
|
3899
5320
|
planFresh(candidate, session, policy, view) {
|
|
3900
5321
|
if (typeof candidate.event.surfaceOp === "object") return null;
|
|
3901
5322
|
const result = candidate.event.data.message.content[0];
|
|
3902
|
-
|
|
3903
|
-
if (tokensBefore === void 0 || tokensBefore <= policy.freshTriggerTokens) return null;
|
|
5323
|
+
if (candidate.characterPressure <= charsForTokens(policy.freshTriggerTokens)) return null;
|
|
3904
5324
|
const sourceSeq = candidate.seq;
|
|
3905
5325
|
const sourceRef$1 = sourceRef(session, sourceSeq);
|
|
3906
5326
|
const textBlock = onlyTextBlock(result.content);
|
|
@@ -3921,8 +5341,11 @@ var ToolResultPruner = class extends Service {
|
|
|
3921
5341
|
const plan = this.plan(candidate, [{
|
|
3922
5342
|
...textBlock,
|
|
3923
5343
|
text: output.text
|
|
3924
|
-
}], sourceSeq, output.reducer, "fresh", "fresh", void 0, view, {
|
|
3925
|
-
|
|
5344
|
+
}], sourceSeq, output.reducer, "fresh", "fresh", void 0, view, {
|
|
5345
|
+
noNetSavingsGuard: policy.presetOptions?.noNetSavingsGuard === true,
|
|
5346
|
+
...output.elidedLines === void 0 ? {} : { elidedLines: output.elidedLines }
|
|
5347
|
+
});
|
|
5348
|
+
if (plan !== null && plan.charsAfter <= charsForTokens(policy.freshTargetTokens)) return plan;
|
|
3926
5349
|
}
|
|
3927
5350
|
if (budgetChars === 1) break;
|
|
3928
5351
|
budgetChars = Math.max(1, Math.floor(budgetChars / 2));
|
|
@@ -3934,6 +5357,8 @@ var ToolResultPruner = class extends Service {
|
|
|
3934
5357
|
if (isError(candidate)) return this.planErrorEvidence(candidate, session, view, stage, targetTokens, component, historyMode);
|
|
3935
5358
|
const sourceSeq = rootToolResultSeq(session, candidate.seq);
|
|
3936
5359
|
const sourceRef$2 = sourceRef(session, sourceSeq);
|
|
5360
|
+
const redacted = candidate.event.data.message.content[0];
|
|
5361
|
+
if (onlyTextBlocks(redacted.content) === null) return null;
|
|
3937
5362
|
const text = [
|
|
3938
5363
|
"[Tool result reduced to satisfy the completed-step aggregate budget]",
|
|
3939
5364
|
`tool: ${candidate.call.name}`,
|
|
@@ -3980,17 +5405,8 @@ var ToolResultPruner = class extends Service {
|
|
|
3980
5405
|
planHistoricalAging(session, policy, view) {
|
|
3981
5406
|
const candidates = this.snapshot(session, view);
|
|
3982
5407
|
const events = sessionEvents(session);
|
|
3983
|
-
const
|
|
3984
|
-
|
|
3985
|
-
const tokens = exactTokens(candidate.count);
|
|
3986
|
-
if (tokens === void 0) {
|
|
3987
|
-
this.warnExactUnavailable(session, view, "history");
|
|
3988
|
-
return { kind: "exact-tokenizer-unavailable" };
|
|
3989
|
-
}
|
|
3990
|
-
exact.push(tokens);
|
|
3991
|
-
}
|
|
3992
|
-
const total = exact.reduce((sum, tokens) => sum + tokens, 0);
|
|
3993
|
-
const trigger = policy.historyTriggerTokens;
|
|
5408
|
+
const total = candidates.map((candidate) => candidate.characterPressure).reduce((sum, charsOfNode) => sum + charsOfNode, 0);
|
|
5409
|
+
const trigger = charsForTokens(policy.historyTriggerTokens);
|
|
3994
5410
|
const deadline = policy.microDeadlineTokens;
|
|
3995
5411
|
const lastChance = deadline !== void 0 && view.totalTokens >= deadline;
|
|
3996
5412
|
if (total <= trigger && !lastChance) return { kind: "below-profile-trigger" };
|
|
@@ -4005,9 +5421,10 @@ var ToolResultPruner = class extends Service {
|
|
|
4005
5421
|
if (eligible.length === 0) return safe.length === 0 ? { kind: "no-safe-candidates" } : { kind: "protected-working-set" };
|
|
4006
5422
|
const planned = [];
|
|
4007
5423
|
let reclaim = 0;
|
|
4008
|
-
const
|
|
4009
|
-
const
|
|
4010
|
-
const
|
|
5424
|
+
const minReclaimChars = charsForTokens(policy.historyMinReclaimTokens);
|
|
5425
|
+
const microTarget = deadline === void 0 ? void 0 : Math.max(0, charsForTokens(deadline) - minReclaimChars);
|
|
5426
|
+
const required = Math.max(minReclaimChars, total - trigger, ...microTarget === void 0 ? [] : [charsForTokens(view.totalTokens) - microTarget]);
|
|
5427
|
+
const batchTarget = microTarget === void 0 ? minReclaimChars : required;
|
|
4011
5428
|
for (const candidate of eligible) {
|
|
4012
5429
|
const result = candidate.event.data.message.content[0];
|
|
4013
5430
|
const block = onlyTextBlock(result.content);
|
|
@@ -4018,7 +5435,7 @@ var ToolResultPruner = class extends Service {
|
|
|
4018
5435
|
const plan = this.planAggregate(candidate, session, view, "superseded-read-whole-result", "pressure", void 0, "history", policy.historyMode);
|
|
4019
5436
|
if (plan === null) continue;
|
|
4020
5437
|
planned.push(plan);
|
|
4021
|
-
reclaim += plan.
|
|
5438
|
+
reclaim += plan.charsBefore - plan.charsAfter;
|
|
4022
5439
|
if (reclaim >= required) break;
|
|
4023
5440
|
continue;
|
|
4024
5441
|
}
|
|
@@ -4028,7 +5445,7 @@ var ToolResultPruner = class extends Service {
|
|
|
4028
5445
|
const plan = this.planAggregate(candidate, session, view, "historical-rich-whole-result", "pressure", void 0, "history", policy.historyMode);
|
|
4029
5446
|
if (plan === null) continue;
|
|
4030
5447
|
planned.push(plan);
|
|
4031
|
-
reclaim += plan.
|
|
5448
|
+
reclaim += plan.charsBefore - plan.charsAfter;
|
|
4032
5449
|
if (reclaim >= required) break;
|
|
4033
5450
|
continue;
|
|
4034
5451
|
}
|
|
@@ -4058,10 +5475,10 @@ var ToolResultPruner = class extends Service {
|
|
|
4058
5475
|
const plan = this.plan(candidate, [{
|
|
4059
5476
|
...block,
|
|
4060
5477
|
text: replacementText
|
|
4061
|
-
}], sourceSeq, output.reducer, "pressure", "history", policy.historyMode, view);
|
|
5478
|
+
}], sourceSeq, output.reducer, "pressure", "history", policy.historyMode, view, { ...output.elidedLines === void 0 ? {} : { elidedLines: output.elidedLines } });
|
|
4062
5479
|
if (plan === null) continue;
|
|
4063
5480
|
planned.push(plan);
|
|
4064
|
-
reclaim += plan.
|
|
5481
|
+
reclaim += plan.charsBefore - plan.charsAfter;
|
|
4065
5482
|
if (reclaim >= required) break;
|
|
4066
5483
|
}
|
|
4067
5484
|
if (reclaim >= batchTarget && planned.length > 0) return historyOutcome(planned);
|
|
@@ -4077,7 +5494,6 @@ var ToolResultPruner = class extends Service {
|
|
|
4077
5494
|
}
|
|
4078
5495
|
protectedHistoryResultSeqs(session, policy, view) {
|
|
4079
5496
|
const candidates = this.snapshot(session, view);
|
|
4080
|
-
if (candidates.some((candidate) => exactTokens(candidate.count) === void 0)) return null;
|
|
4081
5497
|
return this.protectedHistoryCandidateSeqs(candidates, policy);
|
|
4082
5498
|
}
|
|
4083
5499
|
/** Select the newest completed tool calls and token tail for History-derived stages. */
|
|
@@ -4087,12 +5503,12 @@ var ToolResultPruner = class extends Service {
|
|
|
4087
5503
|
const candidate = candidates[index];
|
|
4088
5504
|
if (candidate !== void 0) protectedSeqs.add(candidate.seq);
|
|
4089
5505
|
}
|
|
4090
|
-
let
|
|
4091
|
-
for (let index = candidates.length - 1; index >= 0 &&
|
|
5506
|
+
let recentChars = 0;
|
|
5507
|
+
for (let index = candidates.length - 1; index >= 0 && recentChars < charsForTokens(policy.historyKeepRecentTokens); index--) {
|
|
4092
5508
|
const candidate = candidates[index];
|
|
4093
5509
|
if (candidate === void 0) continue;
|
|
4094
5510
|
protectedSeqs.add(candidate.seq);
|
|
4095
|
-
|
|
5511
|
+
recentChars += candidate.characterPressure;
|
|
4096
5512
|
}
|
|
4097
5513
|
return protectedSeqs;
|
|
4098
5514
|
}
|
|
@@ -4101,42 +5517,35 @@ var ToolResultPruner = class extends Service {
|
|
|
4101
5517
|
const tailTrim = policy.tailTrim;
|
|
4102
5518
|
if (tailTrim?.enabled !== true) return;
|
|
4103
5519
|
const events = sessionEvents(session);
|
|
4104
|
-
if (view.
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
...view.currentSurface.kind === "exact-tokenizer" ? { currentTokens: view.currentSurface.tokens } : {},
|
|
5520
|
+
if (view.currentSurfaceChars <= charsForTokens(tailTrim.triggerTokens)) {
|
|
5521
|
+
this.auditComponent(session, policy, "tail-trim", "pressure", "skipped", "at-or-below-trigger", {
|
|
5522
|
+
measurementKind: "characters",
|
|
5523
|
+
currentTokens: charsToTokens(view.currentSurfaceChars),
|
|
4109
5524
|
triggerTokens: tailTrim.triggerTokens
|
|
4110
5525
|
});
|
|
4111
5526
|
return;
|
|
4112
5527
|
}
|
|
4113
5528
|
const surfaceCount = view.currentSurface;
|
|
5529
|
+
const exactSurface = surfaceCount.kind === "exact-tokenizer" ? surfaceCount : void 0;
|
|
4114
5530
|
if (!this.hasRecoveryTool(session)) {
|
|
4115
5531
|
this.auditComponent(session, policy, "tail-trim", "pressure", "skipped", "recovery-tool-unavailable", {
|
|
4116
|
-
measurementKind: "
|
|
4117
|
-
currentTokens:
|
|
5532
|
+
measurementKind: "characters",
|
|
5533
|
+
currentTokens: charsToTokens(view.currentSurfaceChars),
|
|
4118
5534
|
triggerTokens: tailTrim.triggerTokens
|
|
4119
5535
|
});
|
|
4120
5536
|
return;
|
|
4121
5537
|
}
|
|
4122
5538
|
if (!hasOpenTurn(session)) {
|
|
4123
5539
|
this.auditComponent(session, policy, "tail-trim", "pressure", "skipped", "no-open-turn", {
|
|
4124
|
-
measurementKind: "
|
|
4125
|
-
currentTokens:
|
|
5540
|
+
measurementKind: "characters",
|
|
5541
|
+
currentTokens: charsToTokens(view.currentSurfaceChars),
|
|
4126
5542
|
triggerTokens: tailTrim.triggerTokens
|
|
4127
5543
|
});
|
|
4128
5544
|
return;
|
|
4129
5545
|
}
|
|
4130
5546
|
const protectedResults = this.protectedHistoryResultSeqs(session, policy, view);
|
|
4131
|
-
if (protectedResults === null) {
|
|
4132
|
-
this.auditComponent(session, policy, "tail-trim", "pressure", "skipped", "exact-tokenizer-unavailable-in-protected-set", {
|
|
4133
|
-
measurementKind: "unavailable",
|
|
4134
|
-
currentTokens: surfaceCount.tokens,
|
|
4135
|
-
triggerTokens: tailTrim.triggerTokens
|
|
4136
|
-
});
|
|
4137
|
-
return;
|
|
4138
|
-
}
|
|
4139
5547
|
const measured = new Map(view.measuredNodes.map((node) => [node.seq, node.count]));
|
|
5548
|
+
const nodeChars = new Map(view.measuredNodes.map((node) => [node.seq, node.characterPressure]));
|
|
4140
5549
|
const heuristic = new Map(view.nodes.map((node) => [node.seq, node.tokens]));
|
|
4141
5550
|
const completedTurns = /* @__PURE__ */ new Set();
|
|
4142
5551
|
const completedSteps = /* @__PURE__ */ new Set();
|
|
@@ -4171,17 +5580,35 @@ var ToolResultPruner = class extends Service {
|
|
|
4171
5580
|
if (roots.some((root) => root === null)) continue;
|
|
4172
5581
|
const sourceEventSeqs = roots;
|
|
4173
5582
|
if (new Set(sourceEventSeqs).size !== sourceEventSeqs.length) continue;
|
|
4174
|
-
|
|
4175
|
-
if (
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
5583
|
+
let exactTokensBefore;
|
|
5584
|
+
if (exactSurface !== void 0) {
|
|
5585
|
+
let sum = 0;
|
|
5586
|
+
let allExact = true;
|
|
5587
|
+
for (const seq of shadowedSeqs) {
|
|
5588
|
+
const count = measured.get(seq);
|
|
5589
|
+
if (count?.kind !== "exact-tokenizer" || count.tokenizerId !== exactSurface.tokenizerId || count.tokenizerRevision !== exactSurface.tokenizerRevision) {
|
|
5590
|
+
allExact = false;
|
|
5591
|
+
break;
|
|
5592
|
+
}
|
|
5593
|
+
sum += count.tokens;
|
|
5594
|
+
}
|
|
5595
|
+
if (allExact) exactTokensBefore = sum;
|
|
5596
|
+
}
|
|
5597
|
+
const charsBefore = shadowedSeqs.reduce((sum, seq) => sum + (nodeChars.get(seq) ?? 0), 0);
|
|
4179
5598
|
const manifestSeq = events.length;
|
|
4180
5599
|
const ref = tailTrimRef(String(session.id), manifestSeq);
|
|
4181
5600
|
const stub = tailTrimStub(ref, calls.map((call) => call.name), sourceEventSeqs);
|
|
4182
5601
|
if (stub === null) continue;
|
|
4183
|
-
const
|
|
4184
|
-
if (
|
|
5602
|
+
const stubChars = codePointLength(stub);
|
|
5603
|
+
if (stubChars <= 0 || charsBefore - stubChars < charsForTokens(policy.historyMinReclaimTokens)) continue;
|
|
5604
|
+
let exactTokensAfter;
|
|
5605
|
+
if (exactTokensBefore !== void 0 && exactSurface !== void 0) {
|
|
5606
|
+
const stubCount = countExactCanonicalTextFields([stub], (candidate) => view.countCanonicalText(candidate), "TailTrim group stub");
|
|
5607
|
+
if (stubCount.kind === "exact-tokenizer" && stubCount.tokenizerId === exactSurface.tokenizerId && stubCount.tokenizerRevision === exactSurface.tokenizerRevision) exactTokensAfter = stubCount.tokens;
|
|
5608
|
+
}
|
|
5609
|
+
const exact = exactTokensBefore !== void 0 && exactTokensAfter !== void 0;
|
|
5610
|
+
const tokensBefore = exactTokensBefore ?? charsToTokens(charsBefore);
|
|
5611
|
+
const tokensAfter = exactTokensAfter ?? charsToTokens(stubChars);
|
|
4185
5612
|
const heuristicTokens = shadowedSeqs.reduce((sum, seq) => sum + (heuristic.get(seq) ?? 0), 0);
|
|
4186
5613
|
const range = {
|
|
4187
5614
|
start: SessionSeq(assistantSeq),
|
|
@@ -4194,8 +5621,8 @@ var ToolResultPruner = class extends Service {
|
|
|
4194
5621
|
};
|
|
4195
5622
|
if (!this.reserveTailTrimBoundaryAttempt(session)) {
|
|
4196
5623
|
this.auditComponent(session, policy, "tail-trim", "pressure", "skipped", "already-attempted-at-request-boundary", {
|
|
4197
|
-
measurementKind: "
|
|
4198
|
-
currentTokens:
|
|
5624
|
+
measurementKind: "characters",
|
|
5625
|
+
currentTokens: charsToTokens(view.currentSurfaceChars),
|
|
4199
5626
|
triggerTokens: tailTrim.triggerTokens
|
|
4200
5627
|
});
|
|
4201
5628
|
return;
|
|
@@ -4228,16 +5655,17 @@ var ToolResultPruner = class extends Service {
|
|
|
4228
5655
|
replacementSeq: replacement.seq,
|
|
4229
5656
|
sourceSeqs: sourceEventSeqs,
|
|
4230
5657
|
tokensBefore,
|
|
4231
|
-
tokensAfter
|
|
4232
|
-
tokensRemoved: tokensBefore -
|
|
4233
|
-
|
|
4234
|
-
|
|
5658
|
+
tokensAfter,
|
|
5659
|
+
tokensRemoved: tokensBefore - tokensAfter,
|
|
5660
|
+
measurementBasis: exact ? "exact-tokenizer" : "characters",
|
|
5661
|
+
tokenizerId: exact === true && exactSurface !== void 0 ? exactSurface.tokenizerId : "characters",
|
|
5662
|
+
tokenizerRevision: exact === true && exactSurface !== void 0 ? exactSurface.tokenizerRevision : "chars-per-token-4.0"
|
|
4235
5663
|
});
|
|
4236
5664
|
return;
|
|
4237
5665
|
}
|
|
4238
5666
|
this.auditComponent(session, policy, "tail-trim", "pressure", "skipped", "no-safe-eligible-tool-group", {
|
|
4239
|
-
measurementKind: "
|
|
4240
|
-
currentTokens:
|
|
5667
|
+
measurementKind: "characters",
|
|
5668
|
+
currentTokens: charsToTokens(view.currentSurfaceChars),
|
|
4241
5669
|
triggerTokens: tailTrim.triggerTokens
|
|
4242
5670
|
});
|
|
4243
5671
|
}
|
|
@@ -4277,13 +5705,14 @@ var ToolResultPruner = class extends Service {
|
|
|
4277
5705
|
return roots.size === 1 ? [...roots][0] ?? null : null;
|
|
4278
5706
|
}
|
|
4279
5707
|
plan(candidate, content, sourceSeq, reducer, stage, component, historyMode, view, options = {}) {
|
|
5708
|
+
const charsBefore = candidate.characterPressure;
|
|
5709
|
+
const charsAfter = pressureCost(content);
|
|
5710
|
+
if (charsAfter <= 0 || charsAfter >= charsBefore) return null;
|
|
4280
5711
|
const countBefore = candidate.count;
|
|
4281
|
-
if (countBefore.kind !== "exact-tokenizer") return null;
|
|
4282
5712
|
const countAfter = countToolContent(content, view);
|
|
4283
|
-
|
|
4284
|
-
const tokensBefore = countBefore.tokens;
|
|
4285
|
-
const tokensAfter = countAfter.tokens;
|
|
4286
|
-
if (tokensAfter <= 0 || tokensAfter >= tokensBefore) return null;
|
|
5713
|
+
const exact = countBefore.kind === "exact-tokenizer" && countAfter.kind === "exact-tokenizer" && countAfter.tokenizerId === countBefore.tokenizerId && countAfter.tokenizerRevision === countBefore.tokenizerRevision;
|
|
5714
|
+
const tokensBefore = exact ? countBefore.tokens : charsToTokens(charsBefore);
|
|
5715
|
+
const tokensAfter = exact ? countAfter.tokens : charsToTokens(charsAfter);
|
|
4287
5716
|
if (options.noNetSavingsGuard === true) {
|
|
4288
5717
|
const originalBlocks = onlyTextBlocks(candidate.event.data.message.content[0].content);
|
|
4289
5718
|
const replacementBlocks = onlyTextBlocks(content);
|
|
@@ -4292,8 +5721,6 @@ var ToolResultPruner = class extends Service {
|
|
|
4292
5721
|
if (replacementBlocks.reduce((sum, block) => sum + codePointLength(block.text), 0) >= originalChars) return null;
|
|
4293
5722
|
}
|
|
4294
5723
|
}
|
|
4295
|
-
const charsBefore = candidate.characterPressure;
|
|
4296
|
-
const charsAfter = pressureCost(content);
|
|
4297
5724
|
return {
|
|
4298
5725
|
candidate,
|
|
4299
5726
|
content,
|
|
@@ -4306,8 +5733,10 @@ var ToolResultPruner = class extends Service {
|
|
|
4306
5733
|
charsAfter,
|
|
4307
5734
|
tokensBefore,
|
|
4308
5735
|
tokensAfter,
|
|
4309
|
-
|
|
4310
|
-
|
|
5736
|
+
measurementBasis: exact ? "exact-tokenizer" : "characters",
|
|
5737
|
+
tokenizerId: exact ? countBefore.tokenizerId : "characters",
|
|
5738
|
+
tokenizerRevision: exact ? countBefore.tokenizerRevision : "chars-per-token-4.0",
|
|
5739
|
+
...options.elidedLines === void 0 ? {} : { elidedLines: options.elidedLines }
|
|
4311
5740
|
};
|
|
4312
5741
|
}
|
|
4313
5742
|
land(session, plan) {
|
|
@@ -4361,8 +5790,10 @@ var ToolResultPruner = class extends Service {
|
|
|
4361
5790
|
tokensBefore: plan.tokensBefore,
|
|
4362
5791
|
tokensAfter: plan.tokensAfter,
|
|
4363
5792
|
tokensRemoved: plan.tokensBefore - plan.tokensAfter,
|
|
5793
|
+
measurementBasis: plan.measurementBasis,
|
|
4364
5794
|
tokenizerId: plan.tokenizerId,
|
|
4365
|
-
tokenizerRevision: plan.tokenizerRevision
|
|
5795
|
+
tokenizerRevision: plan.tokenizerRevision,
|
|
5796
|
+
...plan.elidedLines === void 0 ? {} : { elidedLines: plan.elidedLines }
|
|
4366
5797
|
});
|
|
4367
5798
|
if (plan.reducer !== "review-approved-whole-result") this.reviewSummaryFor(session).autoApplied += 1;
|
|
4368
5799
|
return {
|
|
@@ -4410,7 +5841,7 @@ var ToolResultPruner = class extends Service {
|
|
|
4410
5841
|
const capacityTrigger = deadlineTrigger !== void 0 ? deadlineTrigger : Number.isSafeInteger(capacity) && capacity !== void 0 && capacity > 0 ? Math.floor(capacity * CAPACITY_PRESSURE_RATIO) : void 0;
|
|
4411
5842
|
this.auditComponent(session, policy, "history", "pressure", "skipped", policy.historyMode === "capacity-pressure" ? "below-micro-deadline" : "adaptive-cost-rejected", {
|
|
4412
5843
|
historyMode: policy.historyMode,
|
|
4413
|
-
measurementKind:
|
|
5844
|
+
measurementKind: "characters",
|
|
4414
5845
|
currentTokens: view.totalTokens,
|
|
4415
5846
|
...capacityTrigger === void 0 ? {} : { triggerTokens: capacityTrigger }
|
|
4416
5847
|
});
|
|
@@ -4420,7 +5851,7 @@ var ToolResultPruner = class extends Service {
|
|
|
4420
5851
|
const lastChance = deadline !== void 0 && view.totalTokens >= deadline;
|
|
4421
5852
|
const detail = (extra = {}) => ({
|
|
4422
5853
|
historyMode: policy.historyMode,
|
|
4423
|
-
measurementKind:
|
|
5854
|
+
measurementKind: "characters",
|
|
4424
5855
|
currentTokens: view.totalTokens,
|
|
4425
5856
|
...outcome.kind === "insufficient-reclaim" || outcome.kind === "cannot-reach-deadline-target" ? {
|
|
4426
5857
|
reclaimTokens: outcome.reclaim,
|
|
@@ -4429,9 +5860,6 @@ var ToolResultPruner = class extends Service {
|
|
|
4429
5860
|
...extra
|
|
4430
5861
|
});
|
|
4431
5862
|
switch (outcome.kind) {
|
|
4432
|
-
case "exact-tokenizer-unavailable":
|
|
4433
|
-
this.auditComponent(session, policy, "history", "pressure", "skipped", "exact-tokenizer-unavailable", detail({ triggerTokens: policy.historyTriggerTokens }));
|
|
4434
|
-
return;
|
|
4435
5863
|
case "below-profile-trigger":
|
|
4436
5864
|
this.auditComponent(session, policy, "history", "pressure", "skipped", "below-profile-trigger", detail({ triggerTokens: policy.historyTriggerTokens }));
|
|
4437
5865
|
return;
|
|
@@ -4524,7 +5952,7 @@ var ToolResultPruner = class extends Service {
|
|
|
4524
5952
|
warnExactUnavailable(session, view, gate) {
|
|
4525
5953
|
const provider = view.providerRoute ?? "unbound-provider";
|
|
4526
5954
|
const model = view.modelId ?? "unbound-model";
|
|
4527
|
-
this.warnOnce(session, `exact-tokenizer:${gate}:${provider}\0${model}`, "context-compression %s
|
|
5955
|
+
this.warnOnce(session, `exact-tokenizer:${gate}:${provider}\0${model}`, "context-compression %s is measuring on the character basis: exact tokenizer counts are unavailable for %s/%s", gate, provider, model);
|
|
4528
5956
|
}
|
|
4529
5957
|
warnOnce(session, key, message, ...args) {
|
|
4530
5958
|
let warned = this.state.warnedFailures.get(session);
|