@retaw106/mmc-oc 1.2.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +7095 -0
- package/dist/config/index.d.ts +0 -12
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/legacy-harness-blocks.d.ts +27 -0
- package/dist/config/legacy-harness-blocks.d.ts.map +1 -0
- package/dist/config/live-run-config.d.ts.map +1 -1
- package/dist/config/live-snapshot.d.ts +2 -1
- package/dist/config/live-snapshot.d.ts.map +1 -1
- package/dist/config/project-security.d.ts +4 -5
- package/dist/config/project-security.d.ts.map +1 -1
- package/dist/config/raw-loader.d.ts +7 -26
- package/dist/config/raw-loader.d.ts.map +1 -1
- package/dist/config/schema/magic-context.d.ts +164 -259
- package/dist/config/schema/magic-context.d.ts.map +1 -1
- package/dist/config/variable.d.ts +8 -0
- package/dist/config/variable.d.ts.map +1 -1
- package/dist/config-init.js +5595 -0
- package/dist/{context-a6px04s0.js → context-95zkksn3.js} +2471 -2438
- package/dist/{derive-budgets-fhk00zv8.js → derive-budgets-rjjz5f7y.js} +2 -2
- package/dist/features/magic-context/compaction-marker.d.ts.map +1 -1
- package/dist/features/magic-context/message-index.d.ts +2 -0
- package/dist/features/magic-context/message-index.d.ts.map +1 -1
- package/dist/features/magic-context/migration-v3-perf-schema.d.ts +8 -0
- package/dist/features/magic-context/migration-v3-perf-schema.d.ts.map +1 -0
- package/dist/features/magic-context/migration-v4-temporal-decisions.d.ts +12 -0
- package/dist/features/magic-context/migration-v4-temporal-decisions.d.ts.map +1 -0
- package/dist/features/magic-context/migrations.d.ts.map +1 -1
- package/dist/features/magic-context/overflow-detection.d.ts +6 -0
- package/dist/features/magic-context/overflow-detection.d.ts.map +1 -1
- package/dist/features/magic-context/reclaim-protection.d.ts +15 -10
- package/dist/features/magic-context/reclaim-protection.d.ts.map +1 -1
- package/dist/features/magic-context/session-cache-ttl.d.ts +4 -2
- package/dist/features/magic-context/session-cache-ttl.d.ts.map +1 -1
- package/dist/features/magic-context/storage-db.d.ts +1 -1
- package/dist/features/magic-context/storage-db.d.ts.map +1 -1
- package/dist/features/magic-context/storage-historian-runs.d.ts +2 -0
- package/dist/features/magic-context/storage-historian-runs.d.ts.map +1 -1
- package/dist/features/magic-context/storage-ops.d.ts +2 -0
- package/dist/features/magic-context/storage-ops.d.ts.map +1 -1
- package/dist/features/magic-context/storage-session-tables.d.ts.map +1 -1
- package/dist/features/magic-context/storage-tags.d.ts +15 -2
- package/dist/features/magic-context/storage-tags.d.ts.map +1 -1
- package/dist/features/magic-context/storage.d.ts +2 -2
- package/dist/features/magic-context/storage.d.ts.map +1 -1
- package/dist/features/magic-context/temporal-decisions.d.ts +10 -0
- package/dist/features/magic-context/temporal-decisions.d.ts.map +1 -0
- package/dist/features/magic-context/tool-definition-tokens.d.ts +3 -0
- package/dist/features/magic-context/tool-definition-tokens.d.ts.map +1 -1
- package/dist/hooks/magic-context/active-anthropic-turn.d.ts +8 -0
- package/dist/hooks/magic-context/active-anthropic-turn.d.ts.map +1 -0
- package/dist/hooks/magic-context/apply-operations.d.ts +15 -0
- package/dist/hooks/magic-context/apply-operations.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-deadline.d.ts +10 -3
- package/dist/hooks/magic-context/auto-search-deadline.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-runner.d.ts +1 -1
- package/dist/hooks/magic-context/auto-search-runner.d.ts.map +1 -1
- package/dist/hooks/magic-context/caveman-cleanup.d.ts.map +1 -1
- package/dist/hooks/magic-context/channel2-cycle.d.ts +2 -0
- package/dist/hooks/magic-context/channel2-cycle.d.ts.map +1 -1
- package/dist/hooks/magic-context/compaction-marker-manager.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-render-epoch.d.ts +8 -0
- package/dist/hooks/magic-context/compartment-render-epoch.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-partial-recomp.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-recomp.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts +3 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts.map +1 -1
- package/dist/hooks/magic-context/ctx-reduce-availability.d.ts +12 -0
- package/dist/hooks/magic-context/ctx-reduce-availability.d.ts.map +1 -1
- package/dist/hooks/magic-context/decision-calibration.d.ts +3 -0
- package/dist/hooks/magic-context/decision-calibration.d.ts.map +1 -1
- package/dist/hooks/magic-context/emergency-drop.d.ts +2 -0
- package/dist/hooks/magic-context/emergency-drop.d.ts.map +1 -1
- package/dist/hooks/magic-context/emergency-fail-closed.d.ts +21 -1
- package/dist/hooks/magic-context/emergency-fail-closed.d.ts.map +1 -1
- package/dist/hooks/magic-context/final-wire-token-estimate.d.ts +30 -1
- package/dist/hooks/magic-context/final-wire-token-estimate.d.ts.map +1 -1
- package/dist/hooks/magic-context/heuristic-cleanup.d.ts +2 -0
- package/dist/hooks/magic-context/heuristic-cleanup.d.ts.map +1 -1
- package/dist/hooks/magic-context/inject-compartments.d.ts +10 -8
- package/dist/hooks/magic-context/inject-compartments.d.ts.map +1 -1
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts +25 -0
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts.map +1 -0
- package/dist/hooks/magic-context/lkg-measured-request.d.ts +69 -0
- package/dist/hooks/magic-context/lkg-measured-request.d.ts.map +1 -0
- package/dist/hooks/magic-context/read-session-formatting.d.ts +3 -0
- package/dist/hooks/magic-context/read-session-formatting.d.ts.map +1 -1
- package/dist/hooks/magic-context/reasoning-budget-status.d.ts +15 -0
- package/dist/hooks/magic-context/reasoning-budget-status.d.ts.map +1 -0
- package/dist/hooks/magic-context/reasoning-budget.d.ts +40 -0
- package/dist/hooks/magic-context/reasoning-budget.d.ts.map +1 -0
- package/dist/hooks/magic-context/reasoning-removal.d.ts +6 -6
- package/dist/hooks/magic-context/reasoning-removal.d.ts.map +1 -1
- package/dist/hooks/magic-context/recomp-orchestrator.d.ts +2 -0
- package/dist/hooks/magic-context/recomp-orchestrator.d.ts.map +1 -1
- package/dist/hooks/magic-context/sentinel.d.ts +2 -0
- package/dist/hooks/magic-context/sentinel.d.ts.map +1 -1
- package/dist/hooks/magic-context/strip-content.d.ts +7 -3
- package/dist/hooks/magic-context/strip-content.d.ts.map +1 -1
- package/dist/hooks/magic-context/supersession-reclaim.d.ts +4 -0
- package/dist/hooks/magic-context/supersession-reclaim.d.ts.map +1 -1
- package/dist/hooks/magic-context/tag-messages.d.ts +13 -0
- package/dist/hooks/magic-context/tag-messages.d.ts.map +1 -1
- package/dist/hooks/magic-context/tail-hygiene-walk.d.ts +5 -0
- package/dist/hooks/magic-context/tail-hygiene-walk.d.ts.map +1 -1
- package/dist/hooks/magic-context/temporal-awareness.d.ts +6 -3
- package/dist/hooks/magic-context/temporal-awareness.d.ts.map +1 -1
- package/dist/hooks/magic-context/temporal-served-projection.d.ts +4 -0
- package/dist/hooks/magic-context/temporal-served-projection.d.ts.map +1 -0
- package/dist/hooks/magic-context/tool-reclaim.d.ts +2 -0
- package/dist/hooks/magic-context/tool-reclaim.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts +34 -12
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform.d.ts +11 -5
- package/dist/hooks/magic-context/transform.d.ts.map +1 -1
- package/dist/hooks/magic-context/wrapup-orchestrator.d.ts.map +1 -1
- package/dist/{index-wgt4gamw.js → index-1mjhzqq7.js} +422 -31
- package/dist/index-33gprt9g.js +30 -0
- package/dist/{index-zx63j2e2.js → index-5cep2r9n.js} +3 -2
- package/dist/{index-by6h2pav.js → index-bdt9my9y.js} +374 -239
- package/dist/{index-3wagyrnj.js → index-x37vg474.js} +5 -1
- package/dist/{index-1vs6anwz.js → index-xt7y10x6.js} +2 -1
- package/dist/{index-me3zcj5t.js → index-z2g20msd.js} +4 -8
- package/dist/index.js +2 -2
- package/dist/{message-time-backfill-5ewk0axe.js → message-time-backfill-zfernxbw.js} +1 -1
- package/dist/{model-resolution-5haaxnpt.js → model-resolution-wazv26n0.js} +1 -1
- package/dist/plugin/rpc-handlers.d.ts +4 -0
- package/dist/plugin/rpc-handlers.d.ts.map +1 -1
- package/dist/{read-session-chunk-apjwrttc.js → read-session-chunk-hp464g13.js} +2 -2
- package/dist/{recomp-orchestrator-gyeevxaw.js → recomp-orchestrator-3qxeqwb9.js} +6 -6
- package/dist/shared/auto-search-lifecycle.d.ts +7 -0
- package/dist/shared/auto-search-lifecycle.d.ts.map +1 -0
- package/dist/shared/cache-ttl-display.d.ts +6 -4
- package/dist/shared/cache-ttl-display.d.ts.map +1 -1
- package/dist/shared/model-cache-ttl.d.ts +1 -1
- package/dist/shared/model-cache-ttl.d.ts.map +1 -1
- package/dist/shared/model-resolution.d.ts +5 -2
- package/dist/shared/model-resolution.d.ts.map +1 -1
- package/dist/shared/model-suggestion-retry.d.ts +2 -0
- package/dist/shared/model-suggestion-retry.d.ts.map +1 -1
- package/dist/shared/protected-tools-policy.d.ts +6 -0
- package/dist/shared/protected-tools-policy.d.ts.map +1 -0
- package/dist/shared/provider-output-failure.d.ts +19 -0
- package/dist/shared/provider-output-failure.d.ts.map +1 -0
- package/dist/shared/temporal-upgrade-fixture.d.ts +19 -0
- package/dist/shared/temporal-upgrade-fixture.d.ts.map +1 -0
- package/dist/shared/window-geometry.d.ts +4 -0
- package/dist/shared/window-geometry.d.ts.map +1 -1
- package/dist/shared/write-transaction-timing.d.ts +8 -1
- package/dist/shared/write-transaction-timing.d.ts.map +1 -1
- package/dist/tools/ctx-reduce/tools.d.ts +2 -0
- package/dist/tools/ctx-reduce/tools.d.ts.map +1 -1
- package/dist/v2/hidden-child-native.d.ts +7 -0
- package/dist/v2/hidden-child-native.d.ts.map +1 -1
- package/dist/v2/hidden-child-record.d.ts +13 -0
- package/dist/v2/hidden-child-record.d.ts.map +1 -1
- package/dist/v2/hidden-completion.d.ts.map +1 -1
- package/dist/v2/hooks/context.d.ts.map +1 -1
- package/dist/v2/hooks/hidden-child.d.ts +46 -2
- package/dist/v2/hooks/hidden-child.d.ts.map +1 -1
- package/dist/v2/hooks/payload.d.ts.map +1 -1
- package/dist/v2/hooks/store.d.ts.map +1 -1
- package/dist/v2/hooks/tools.d.ts.map +1 -1
- package/dist/v2/hooks/types.d.ts +8 -0
- package/dist/v2/hooks/types.d.ts.map +1 -1
- package/dist/v2/server.js +4540 -3889
- package/dist/v2/tool-result.d.ts +8 -0
- package/dist/v2/tool-result.d.ts.map +1 -0
- package/dist/web/api.d.ts +6 -0
- package/dist/web/api.d.ts.map +1 -1
- package/dist/web/context-store.d.ts +13 -0
- package/dist/web/context-store.d.ts.map +1 -1
- package/dist/web/display-path.d.ts +28 -0
- package/dist/web/display-path.d.ts.map +1 -0
- package/dist/web/entry-args.d.ts +33 -0
- package/dist/web/entry-args.d.ts.map +1 -0
- package/dist/web/host-pi.d.ts +2 -0
- package/dist/web/host-pi.d.ts.map +1 -1
- package/dist/web/main.d.ts +4 -6
- package/dist/web/main.d.ts.map +1 -1
- package/dist/web/main.js +772 -77
- package/dist/web/message-stats.d.ts +45 -0
- package/dist/web/message-stats.d.ts.map +1 -0
- package/dist/web/ui/index.html +1 -0
- package/dist/web/ui/panel.css +5 -0
- package/dist/web/ui/panel.js +220 -70
- package/dist/web/views.d.ts +23 -0
- package/dist/web/views.d.ts.map +1 -1
- package/dist/{wrapup-orchestrator-z9sqpqjf.js → wrapup-orchestrator-wgb2nd19.js} +7 -6
- package/package.json +7 -2
- package/src/config/index.test.ts +257 -126
- package/src/config/index.ts +45 -39
- package/src/config/legacy-harness-blocks.test.ts +78 -0
- package/src/config/legacy-harness-blocks.ts +71 -0
- package/src/config/live-run-config.test.ts +49 -5
- package/src/config/live-run-config.ts +1 -0
- package/src/config/live-snapshot.ts +5 -1
- package/src/config/profiles.ts +1 -1
- package/src/config/project-config-trust-boundary.test.ts +2 -2
- package/src/config/project-security.hostile-matrix.test.ts +51 -65
- package/src/config/project-security.test.ts +6 -1
- package/src/config/project-security.ts +13 -12
- package/src/config/protected-tokens.test.ts +1 -1
- package/src/config/raw-loader.test.ts +29 -352
- package/src/config/raw-loader.ts +17 -406
- package/src/config/removed-config-keys.test.ts +1 -1
- package/src/config/schema/magic-context.test.ts +60 -64
- package/src/config/schema/magic-context.ts +105 -137
- package/src/config/variable.ts +21 -12
- package/src/shared/auto-search-lifecycle.ts +27 -0
- package/src/shared/cache-ttl-display.test.ts +1 -1
- package/src/shared/cache-ttl-display.ts +17 -16
- package/src/shared/model-cache-ttl.test.ts +2 -2
- package/src/shared/model-cache-ttl.ts +8 -4
- package/src/shared/model-resolution.test.ts +22 -32
- package/src/shared/model-resolution.ts +13 -18
- package/src/shared/model-suggestion-retry.ts +63 -7
- package/src/shared/protected-tools-policy.ts +25 -0
- package/src/shared/provider-output-failure.ts +176 -0
- package/src/shared/status-detail-text.test.ts +1 -1
- package/src/shared/status-view.test.ts +1 -1
- package/src/shared/temporal-upgrade-fixture.ts +50 -0
- package/src/shared/window-geometry.ts +10 -0
- package/src/shared/write-transaction-timing.ts +25 -1
- package/dist/features/magic-context/window-report-ledger.d.ts +0 -70
- package/dist/features/magic-context/window-report-ledger.d.ts.map +0 -1
- package/dist/index-e1q8002h.js +0 -16
- package/src/config/misplaced-agent-model.test.ts +0 -55
|
@@ -1,23 +1,65 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
canonicalModelIdentity,
|
|
3
|
+
UNKNOWN_FIT_RATIO,
|
|
4
|
+
resolveDecisionCalibration,
|
|
5
|
+
hasMeasuredDecisionCalibration,
|
|
6
|
+
providerMass
|
|
7
|
+
} from "./index-x37vg474.js";
|
|
8
|
+
import {
|
|
9
|
+
getHarness,
|
|
10
|
+
sessionLog
|
|
3
11
|
} from "./index-qbkzdkk0.js";
|
|
4
12
|
import {
|
|
13
|
+
isKnownAutocommit,
|
|
5
14
|
openCodeDbExists,
|
|
6
15
|
withReadOnlySessionDb,
|
|
7
16
|
getRawSessionMessageCountFromDb,
|
|
8
17
|
getMessageTimesFromOpenCodeDb
|
|
9
18
|
} from "./index-977gxfz0.js";
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}
|
|
19
|
+
|
|
20
|
+
// src/shared/protected-tools-policy.ts
|
|
21
|
+
var DEFAULT_PROTECTED_TOOLS = {
|
|
22
|
+
todowrite: 1,
|
|
23
|
+
mmc_reduce: 3
|
|
24
|
+
};
|
|
25
|
+
function normalizeProtectedToolName(name) {
|
|
26
|
+
const lowered = (name ?? "").toLowerCase().replace(/^mcp_/, "");
|
|
27
|
+
return lowered === "ctx_reduce" ? "mmc_reduce" : lowered;
|
|
28
|
+
}
|
|
29
|
+
function mergeProtectedTools(...maps) {
|
|
30
|
+
return Object.fromEntries([DEFAULT_PROTECTED_TOOLS, ...maps].flatMap((map) => Object.entries(map ?? {}).map(([name, count]) => [
|
|
31
|
+
normalizeProtectedToolName(name),
|
|
32
|
+
count
|
|
33
|
+
])));
|
|
34
|
+
}
|
|
16
35
|
|
|
17
36
|
// src/features/magic-context/reclaim-protection.ts
|
|
18
|
-
var CTX_REDUCE_KEEP =
|
|
19
|
-
function
|
|
20
|
-
|
|
37
|
+
var CTX_REDUCE_KEEP = DEFAULT_PROTECTED_TOOLS.mmc_reduce;
|
|
38
|
+
function adoptedProtectedToolsPolicy(current, previous, rebuilding, hasBaseline) {
|
|
39
|
+
if (rebuilding || !hasBaseline)
|
|
40
|
+
return mergeProtectedTools(current);
|
|
41
|
+
return { ...previous ?? { todowrite: 0, mmc_reduce: 3 } };
|
|
42
|
+
}
|
|
43
|
+
function protectedToolTagNumbers(tags, protectedTools) {
|
|
44
|
+
const counts = mergeProtectedTools(protectedTools);
|
|
45
|
+
const protectedTags = new Set;
|
|
46
|
+
for (const tag of [...tags].sort((left, right) => right.tagNumber - left.tagNumber)) {
|
|
47
|
+
if (tag.status !== undefined && tag.status !== "active")
|
|
48
|
+
continue;
|
|
49
|
+
if (tag.type !== undefined && tag.type !== "tool")
|
|
50
|
+
continue;
|
|
51
|
+
const name = normalizeProtectedToolName(tag.toolName);
|
|
52
|
+
if (!Object.hasOwn(counts, name) || counts[name] <= 0)
|
|
53
|
+
continue;
|
|
54
|
+
protectedTags.add(tag.tagNumber);
|
|
55
|
+
counts[name] -= 1;
|
|
56
|
+
}
|
|
57
|
+
return protectedTags;
|
|
58
|
+
}
|
|
59
|
+
function protectedToolTokenCount(tags, counts, calibration) {
|
|
60
|
+
const protectedTags = protectedToolTagNumbers(tags, counts);
|
|
61
|
+
const tokens = tags.reduce((sum, tag) => sum + (protectedTags.has(tag.tagNumber) ? tag.tokenCount ?? 0 : 0), 0);
|
|
62
|
+
return calibration && hasMeasuredDecisionCalibration(calibration) ? providerMass({ tools: tokens }, calibration) : 0;
|
|
21
63
|
}
|
|
22
64
|
|
|
23
65
|
// src/hooks/magic-context/read-session-formatting.ts
|
|
@@ -324,6 +366,12 @@ function getTokenizer() {
|
|
|
324
366
|
function estimateTokensHeuristically(text) {
|
|
325
367
|
return Math.ceil(text.length / 3.5);
|
|
326
368
|
}
|
|
369
|
+
function hasTokenizerForFit() {
|
|
370
|
+
return getTokenizer() !== undefined;
|
|
371
|
+
}
|
|
372
|
+
function tokenCountUsesByteBound(text) {
|
|
373
|
+
return text.length > 1024 * 1024 || /[\p{L}\p{N}]{16385}/u.test(text);
|
|
374
|
+
}
|
|
327
375
|
function estimateTokens(text) {
|
|
328
376
|
if (!text)
|
|
329
377
|
return 0;
|
|
@@ -1064,10 +1112,10 @@ function messageIdForTag(tag) {
|
|
|
1064
1112
|
return tag.toolOwnerMessageId;
|
|
1065
1113
|
return contentTagOwnerMessageId(tag.messageId);
|
|
1066
1114
|
}
|
|
1067
|
-
function resolveProtectedTagNumbers(tags, protectedTagNumbersInput) {
|
|
1115
|
+
function resolveProtectedTagNumbers(tags, protectedTagNumbersInput, protectedToolTags) {
|
|
1068
1116
|
const protectedNumbers = new Set(protectedTagNumbersInput);
|
|
1069
|
-
const
|
|
1070
|
-
for (const tagNumber of
|
|
1117
|
+
const protectedExemplars = protectedToolTags ?? protectedToolTagNumbers(tags.filter((tag) => tag.status === "active" && tag.type === "tool"));
|
|
1118
|
+
for (const tagNumber of protectedExemplars)
|
|
1071
1119
|
protectedNumbers.add(tagNumber);
|
|
1072
1120
|
return protectedNumbers;
|
|
1073
1121
|
}
|
|
@@ -1089,7 +1137,7 @@ function neighborhoodConsistent(args) {
|
|
|
1089
1137
|
return args.orphanTagNumber >= previousMax && args.orphanTagNumber <= nextMin;
|
|
1090
1138
|
}
|
|
1091
1139
|
function buildTagAttribution(args) {
|
|
1092
|
-
const protectedNumbers = resolveProtectedTagNumbers(args.tags, args.protectedTagNumbers);
|
|
1140
|
+
const protectedNumbers = resolveProtectedTagNumbers(args.tags, args.protectedTagNumbers, args.protectedToolTags);
|
|
1093
1141
|
const messageTags = new Map;
|
|
1094
1142
|
const exactToolTags = new Map;
|
|
1095
1143
|
const orphanTagsByCall = new Map;
|
|
@@ -1261,7 +1309,8 @@ function measureWithReplay(input, replay) {
|
|
|
1261
1309
|
messages: input.messages,
|
|
1262
1310
|
tags: input.tags,
|
|
1263
1311
|
toolIdentities,
|
|
1264
|
-
protectedTagNumbers: input.protectedTagNumbers
|
|
1312
|
+
protectedTagNumbers: input.protectedTagNumbers,
|
|
1313
|
+
protectedToolTags: input.protectedToolTags
|
|
1265
1314
|
});
|
|
1266
1315
|
const toolOutputs = input.messages.map((message, index) => message.parts.map((part, partIndex) => unchanged[index] ? replay?.measured.toolOutputs[index][partIndex] ?? null : isRecord(part) ? toolOutputText(part) : null));
|
|
1267
1316
|
const outputByPart = new Map;
|
|
@@ -1632,6 +1681,13 @@ function freezeTailHygieneMeasurement(measured) {
|
|
|
1632
1681
|
};
|
|
1633
1682
|
}
|
|
1634
1683
|
function refreshTailHygieneBaseline(input) {
|
|
1684
|
+
const protectedToolsPolicy = adoptedProtectedToolsPolicy(input.protectedTools, input.previous?.protectedToolsPolicy, input.cacheBusting, !!input.previous);
|
|
1685
|
+
const protectedToolTags = input.protectedToolTags ?? protectedToolTagNumbers(input.tags, protectedToolsPolicy);
|
|
1686
|
+
input = {
|
|
1687
|
+
...input,
|
|
1688
|
+
protectedToolTags,
|
|
1689
|
+
protectedTagNumbers: new Set([...input.protectedTagNumbers, ...protectedToolTags])
|
|
1690
|
+
};
|
|
1635
1691
|
const pendingDropTagNumbers = input.pendingDropTagNumbers ?? new Set;
|
|
1636
1692
|
const cached = input.previous ? baselineMeasurementMemo.get(input.previous.baselineParts) : undefined;
|
|
1637
1693
|
const hit = !input.cacheBusting && cached && sameReplayMessages(cached.messages, input.messages) && sameReplayValue(cached.tags, input.tags) && sameNumbers(cached.protectedTagNumbers, input.protectedTagNumbers) && sameNumbers(cached.pendingDropTagNumbers, pendingDropTagNumbers);
|
|
@@ -1694,7 +1750,8 @@ function refreshTailHygieneBaseline(input) {
|
|
|
1694
1750
|
generationInvalidated: false,
|
|
1695
1751
|
contentSignature: measured.contentSignature,
|
|
1696
1752
|
channel1PostReduceGrace: input.previous?.channel1PostReduceGrace,
|
|
1697
|
-
lastPrefixMismatch: mismatch
|
|
1753
|
+
lastPrefixMismatch: mismatch,
|
|
1754
|
+
protectedToolsPolicy
|
|
1698
1755
|
};
|
|
1699
1756
|
};
|
|
1700
1757
|
if (input.cacheBusting || !input.previous)
|
|
@@ -2126,7 +2183,7 @@ function planEmergencyDrop(input) {
|
|
|
2126
2183
|
reserved.add(nums[i]);
|
|
2127
2184
|
}
|
|
2128
2185
|
}
|
|
2129
|
-
const
|
|
2186
|
+
const protectedTags = input.protectedToolTags ?? protectedToolTagNumbers(floorTags, input.protectedTools);
|
|
2130
2187
|
const byTier = { 1: [], 2: [], 3: [] };
|
|
2131
2188
|
for (const tag of tags) {
|
|
2132
2189
|
if (tag.status !== "active" || tag.type !== "tool")
|
|
@@ -2137,7 +2194,7 @@ function planEmergencyDrop(input) {
|
|
|
2137
2194
|
continue;
|
|
2138
2195
|
}
|
|
2139
2196
|
}
|
|
2140
|
-
if (
|
|
2197
|
+
if (protectedTags.has(tag.tagNumber))
|
|
2141
2198
|
continue;
|
|
2142
2199
|
const tier = resolveToolTier(tag.toolName);
|
|
2143
2200
|
if ((tier === 1 || tier === 2) && reserved.has(tag.tagNumber))
|
|
@@ -2182,6 +2239,158 @@ function measureEmergencyTag(tag, target, calibration, skeleton) {
|
|
|
2182
2239
|
return { ...tag, servedTokens: served, reclaimableTokens: 0 };
|
|
2183
2240
|
}
|
|
2184
2241
|
|
|
2242
|
+
// src/features/magic-context/migration-v3-perf-schema.ts
|
|
2243
|
+
var V3_REDUNDANT_INDEXES = [
|
|
2244
|
+
"idx_tags_session_tag_number",
|
|
2245
|
+
"idx_compartments_session",
|
|
2246
|
+
"idx_pending_ops_session",
|
|
2247
|
+
"idx_source_contents_session",
|
|
2248
|
+
"idx_compression_depth_session",
|
|
2249
|
+
"idx_transform_decisions_session_harness",
|
|
2250
|
+
"idx_project_key_files_project"
|
|
2251
|
+
];
|
|
2252
|
+
var indexes = [
|
|
2253
|
+
["message_fts_rowid_map", "idx_message_fts_rowid_map_session_rowid", "session_id, fts_rowid"]
|
|
2254
|
+
];
|
|
2255
|
+
function installV3PerfSchema(db) {
|
|
2256
|
+
const schema = new Map(db.prepare("SELECT name, sql FROM sqlite_master").all().map((row) => [row.name, row.sql]));
|
|
2257
|
+
for (const [table, name, columns] of indexes) {
|
|
2258
|
+
if (schema.has(table) && !schema.has(name))
|
|
2259
|
+
db.exec(`CREATE INDEX ${name} ON ${table}(${columns})`);
|
|
2260
|
+
}
|
|
2261
|
+
for (const name of V3_REDUNDANT_INDEXES) {
|
|
2262
|
+
if (schema.has(name))
|
|
2263
|
+
db.exec(`DROP INDEX ${name}`);
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
var tagOrderIndexes = new WeakMap;
|
|
2267
|
+
function tagOrderConstraintIndex(db) {
|
|
2268
|
+
const version = db.prepare("PRAGMA schema_version").get().schema_version;
|
|
2269
|
+
const cacheable = isKnownAutocommit(db);
|
|
2270
|
+
if (!cacheable)
|
|
2271
|
+
tagOrderIndexes.delete(db);
|
|
2272
|
+
const cached = tagOrderIndexes.get(db);
|
|
2273
|
+
if (cached?.schemaVersion === version)
|
|
2274
|
+
return cached.name;
|
|
2275
|
+
const candidates = db.prepare("PRAGMA index_list(tags)").all();
|
|
2276
|
+
for (const candidate of candidates) {
|
|
2277
|
+
if (!candidate.unique || candidate.origin !== "u")
|
|
2278
|
+
continue;
|
|
2279
|
+
const quoted = `"${candidate.name.replaceAll('"', '""')}"`;
|
|
2280
|
+
const columns = db.prepare(`PRAGMA index_info(${quoted})`).all();
|
|
2281
|
+
if (columns.length !== 2 || columns[0]?.name !== "session_id" || columns[1]?.name !== "tag_number")
|
|
2282
|
+
continue;
|
|
2283
|
+
if (cacheable)
|
|
2284
|
+
tagOrderIndexes.set(db, { schemaVersion: version, name: quoted });
|
|
2285
|
+
return quoted;
|
|
2286
|
+
}
|
|
2287
|
+
throw new Error("tags lacks its UNIQUE(session_id, tag_number) ordering constraint");
|
|
2288
|
+
}
|
|
2289
|
+
|
|
2290
|
+
// src/features/magic-context/storage-ops.ts
|
|
2291
|
+
var queuePendingOpStatements = new WeakMap;
|
|
2292
|
+
var queueUnchangedDropStatements = new WeakMap;
|
|
2293
|
+
var getPendingOpsStatements = new WeakMap;
|
|
2294
|
+
var getPendingOpsCountStatements = new WeakMap;
|
|
2295
|
+
var clearPendingOpsStatements = new WeakMap;
|
|
2296
|
+
var removePendingOpStatements = new WeakMap;
|
|
2297
|
+
var hasPendingDropOpsStatements = new WeakMap;
|
|
2298
|
+
function getQueuePendingOpStatement(db) {
|
|
2299
|
+
let stmt = queuePendingOpStatements.get(db);
|
|
2300
|
+
if (!stmt) {
|
|
2301
|
+
stmt = db.prepare(`INSERT INTO pending_ops (session_id, tag_id, operation, queued_at, harness)
|
|
2302
|
+
SELECT ?, ?, ?, ?, ? WHERE NOT EXISTS (
|
|
2303
|
+
SELECT 1 FROM pending_ops WHERE session_id = ? AND tag_id = ? AND operation = ?
|
|
2304
|
+
)`);
|
|
2305
|
+
queuePendingOpStatements.set(db, stmt);
|
|
2306
|
+
}
|
|
2307
|
+
return stmt;
|
|
2308
|
+
}
|
|
2309
|
+
function getPendingOpsStatement(db) {
|
|
2310
|
+
let stmt = getPendingOpsStatements.get(db);
|
|
2311
|
+
if (!stmt) {
|
|
2312
|
+
stmt = db.prepare("SELECT id, session_id, tag_id, operation, queued_at FROM pending_ops WHERE session_id = ? ORDER BY queued_at ASC, id ASC");
|
|
2313
|
+
getPendingOpsStatements.set(db, stmt);
|
|
2314
|
+
}
|
|
2315
|
+
return stmt;
|
|
2316
|
+
}
|
|
2317
|
+
function getClearPendingOpsStatement(db) {
|
|
2318
|
+
let stmt = clearPendingOpsStatements.get(db);
|
|
2319
|
+
if (!stmt) {
|
|
2320
|
+
stmt = db.prepare("DELETE FROM pending_ops WHERE session_id = ?");
|
|
2321
|
+
clearPendingOpsStatements.set(db, stmt);
|
|
2322
|
+
}
|
|
2323
|
+
return stmt;
|
|
2324
|
+
}
|
|
2325
|
+
function getRemovePendingOpStatement(db) {
|
|
2326
|
+
let stmt = removePendingOpStatements.get(db);
|
|
2327
|
+
if (!stmt) {
|
|
2328
|
+
stmt = db.prepare("DELETE FROM pending_ops WHERE session_id = ? AND tag_id = ?");
|
|
2329
|
+
removePendingOpStatements.set(db, stmt);
|
|
2330
|
+
}
|
|
2331
|
+
return stmt;
|
|
2332
|
+
}
|
|
2333
|
+
function isPendingOpRow(row) {
|
|
2334
|
+
if (row === null || typeof row !== "object")
|
|
2335
|
+
return false;
|
|
2336
|
+
const r = row;
|
|
2337
|
+
return typeof r.id === "number" && typeof r.session_id === "string" && typeof r.tag_id === "number" && typeof r.operation === "string" && typeof r.queued_at === "number";
|
|
2338
|
+
}
|
|
2339
|
+
function toPendingOp(row) {
|
|
2340
|
+
if (row.operation !== "drop") {
|
|
2341
|
+
sessionLog(row.session_id, `unknown pending operation "${row.operation}"; ignoring`);
|
|
2342
|
+
return null;
|
|
2343
|
+
}
|
|
2344
|
+
return {
|
|
2345
|
+
id: row.id,
|
|
2346
|
+
sessionId: row.session_id,
|
|
2347
|
+
tagId: row.tag_id,
|
|
2348
|
+
operation: row.operation,
|
|
2349
|
+
queuedAt: row.queued_at
|
|
2350
|
+
};
|
|
2351
|
+
}
|
|
2352
|
+
function queuePendingOp(db, sessionId, tagId, operation, queuedAt = Date.now()) {
|
|
2353
|
+
getQueuePendingOpStatement(db).run(sessionId, tagId, operation, queuedAt, getHarness(), sessionId, tagId, operation);
|
|
2354
|
+
}
|
|
2355
|
+
function queuePendingDropForUnchangedTag(db, sessionId, tag) {
|
|
2356
|
+
let statement = queueUnchangedDropStatements.get(db);
|
|
2357
|
+
if (!statement) {
|
|
2358
|
+
statement = db.prepare(`INSERT INTO pending_ops (session_id, tag_id, operation, queued_at, harness)
|
|
2359
|
+
SELECT session_id, tag_number, 'drop', ?, ? FROM tags
|
|
2360
|
+
WHERE id = ? AND session_id = ? AND tag_number = ? AND message_id = ?
|
|
2361
|
+
AND type = ? AND tool_owner_message_id IS ? AND status = 'active'
|
|
2362
|
+
AND NOT EXISTS (
|
|
2363
|
+
SELECT 1 FROM pending_ops WHERE session_id = tags.session_id
|
|
2364
|
+
AND tag_id = tags.tag_number AND operation = 'drop'
|
|
2365
|
+
)`);
|
|
2366
|
+
queueUnchangedDropStatements.set(db, statement);
|
|
2367
|
+
}
|
|
2368
|
+
return statement.run(Date.now(), getHarness(), tag.id ?? null, sessionId, tag.tagNumber, tag.messageId, tag.type, tag.toolOwnerMessageId).changes > 0;
|
|
2369
|
+
}
|
|
2370
|
+
function getPendingOps(db, sessionId) {
|
|
2371
|
+
const rows = getPendingOpsStatement(db).all(sessionId).filter(isPendingOpRow);
|
|
2372
|
+
return rows.map(toPendingOp).filter((op) => op !== null);
|
|
2373
|
+
}
|
|
2374
|
+
function getPendingOpsCount(db, sessionId) {
|
|
2375
|
+
try {
|
|
2376
|
+
let statement = getPendingOpsCountStatements.get(db);
|
|
2377
|
+
if (!statement) {
|
|
2378
|
+
statement = db.prepare("SELECT COUNT(*) AS count FROM pending_ops WHERE session_id = ?");
|
|
2379
|
+
getPendingOpsCountStatements.set(db, statement);
|
|
2380
|
+
}
|
|
2381
|
+
const row = statement.get(sessionId);
|
|
2382
|
+
return typeof row?.count === "number" && Number.isFinite(row.count) ? row.count : null;
|
|
2383
|
+
} catch {
|
|
2384
|
+
return null;
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
function clearPendingOps(db, sessionId) {
|
|
2388
|
+
getClearPendingOpsStatement(db).run(sessionId);
|
|
2389
|
+
}
|
|
2390
|
+
function removePendingOp(db, sessionId, tagId) {
|
|
2391
|
+
getRemovePendingOpStatement(db).run(sessionId, tagId);
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2185
2394
|
// src/features/magic-context/storage-tags.ts
|
|
2186
2395
|
var insertTagStatements = new WeakMap;
|
|
2187
2396
|
var updateTagStatusStatements = new WeakMap;
|
|
@@ -2328,8 +2537,7 @@ function getActiveToolTagsForAgeReclaim(db, sessionId) {
|
|
|
2328
2537
|
reclaimableTokens: outputTokens === null && inputTokens === null ? null : (outputTokens ?? 0) + (inputTokens ?? 0)
|
|
2329
2538
|
};
|
|
2330
2539
|
});
|
|
2331
|
-
|
|
2332
|
-
return tags.filter((tag) => !protectedCtxReduceTags.has(tag.tagNumber));
|
|
2540
|
+
return tags;
|
|
2333
2541
|
}
|
|
2334
2542
|
function getTriggerTagTokenUpperBound(db, sessionId, floor = 0) {
|
|
2335
2543
|
const sql = floor > 0 ? `SELECT
|
|
@@ -2429,6 +2637,17 @@ function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration =
|
|
|
2429
2637
|
}
|
|
2430
2638
|
return { totals, nullMessageIds };
|
|
2431
2639
|
}
|
|
2640
|
+
function getReasoningTokenEstimatesByMessage(db, sessionId, proseRatio) {
|
|
2641
|
+
const rows = db.prepare(`SELECT type, message_id, tool_owner_message_id, reasoning_token_count FROM tags WHERE session_id = ? AND reasoning_token_count IS NOT NULL`).all(sessionId);
|
|
2642
|
+
const totals = new Map;
|
|
2643
|
+
for (const row of rows) {
|
|
2644
|
+
if (row.type !== "message")
|
|
2645
|
+
continue;
|
|
2646
|
+
const id = ownerMessageIdForTagRow(row);
|
|
2647
|
+
totals.set(id, Math.max(totals.get(id) ?? 0, row.reasoning_token_count * proseRatio));
|
|
2648
|
+
}
|
|
2649
|
+
return totals;
|
|
2650
|
+
}
|
|
2432
2651
|
function tagTokenCountIsNull(db, sessionId, tagNumber) {
|
|
2433
2652
|
const row = db.prepare("SELECT token_count FROM tags WHERE session_id = ? AND tag_number = ?").get(sessionId, tagNumber);
|
|
2434
2653
|
return row != null && row.token_count === null;
|
|
@@ -2765,9 +2984,9 @@ function foldShrunkPartTags(db, sessionId, messages) {
|
|
|
2765
2984
|
result.foldedTagNumbers.sort((a, b) => a - b);
|
|
2766
2985
|
return result;
|
|
2767
2986
|
}
|
|
2768
|
-
function markTagsCompactedByMessageIds(db, sessionId, messageIds) {
|
|
2987
|
+
function markTagsCompactedByMessageIds(db, sessionId, messageIds, eligibleTagNumbers) {
|
|
2769
2988
|
const ids = new Set(messageIds);
|
|
2770
|
-
if (ids.size === 0)
|
|
2989
|
+
if (ids.size === 0 || eligibleTagNumbers?.size === 0)
|
|
2771
2990
|
return 0;
|
|
2772
2991
|
const asciiLower = (value) => value.replace(/[A-Z]/g, (char) => char.toLowerCase());
|
|
2773
2992
|
const foldedIds = new Set(Array.from(ids, asciiLower));
|
|
@@ -2793,7 +3012,7 @@ function markTagsCompactedByMessageIds(db, sessionId, messageIds) {
|
|
|
2793
3012
|
const rows = db.prepare(`WITH ids AS MATERIALIZED (
|
|
2794
3013
|
SELECT value AS source_id, lower(value) AS folded_id FROM json_each(?)
|
|
2795
3014
|
)
|
|
2796
|
-
SELECT id, message_id, tool_owner_message_id FROM tags INDEXED BY
|
|
3015
|
+
SELECT id, tag_number, message_id, tool_owner_message_id FROM tags INDEXED BY ${tagOrderConstraintIndex(db)}
|
|
2797
3016
|
WHERE session_id = ? AND status IN ('active', 'dropped') AND (
|
|
2798
3017
|
message_id IN (SELECT source_id FROM ids)
|
|
2799
3018
|
OR tool_owner_message_id IN (SELECT source_id FROM ids)
|
|
@@ -2804,14 +3023,14 @@ function markTagsCompactedByMessageIds(db, sessionId, messageIds) {
|
|
|
2804
3023
|
OR (? AND instr(message_id, ':') > 0)
|
|
2805
3024
|
OR ?
|
|
2806
3025
|
)`).all(JSON.stringify(Array.from(ids)), sessionId, Array.from(ids).some((id) => id.includes(":")) ? 1 : 0, nulPrefixes.size > 0 ? 1 : 0);
|
|
2807
|
-
const candidates = rows.filter((row) => matches(row.message_id, row.tool_owner_message_id));
|
|
3026
|
+
const candidates = rows.filter((row) => (eligibleTagNumbers === undefined || eligibleTagNumbers.has(row.tag_number)) && matches(row.message_id, row.tool_owner_message_id));
|
|
2808
3027
|
if (candidates.length === 0)
|
|
2809
3028
|
return 0;
|
|
2810
3029
|
const update = db.prepare(`UPDATE tags SET status = 'compacted'
|
|
2811
3030
|
WHERE id = ? AND session_id = ?
|
|
2812
3031
|
AND status IN ('active', 'dropped')
|
|
2813
3032
|
AND message_id IS ? AND tool_owner_message_id IS ?
|
|
2814
|
-
RETURNING
|
|
3033
|
+
RETURNING tag_number`);
|
|
2815
3034
|
let cursor = 0;
|
|
2816
3035
|
let changed = 0;
|
|
2817
3036
|
const batch = db.transaction(() => {
|
|
@@ -2819,8 +3038,11 @@ function markTagsCompactedByMessageIds(db, sessionId, messageIds) {
|
|
|
2819
3038
|
let processed = 0;
|
|
2820
3039
|
do {
|
|
2821
3040
|
const row = candidates[cursor++];
|
|
2822
|
-
|
|
3041
|
+
const retired = update.get(row.id, sessionId, row.message_id, row.tool_owner_message_id);
|
|
3042
|
+
if (retired) {
|
|
3043
|
+
removePendingOp(db, sessionId, retired.tag_number);
|
|
2823
3044
|
changed++;
|
|
3045
|
+
}
|
|
2824
3046
|
processed++;
|
|
2825
3047
|
} while (cursor < candidates.length && processed < 128 && performance.now() - start < 8);
|
|
2826
3048
|
});
|
|
@@ -2948,6 +3170,24 @@ function getTagsByNumbers(db, sessionId, tagNumbers) {
|
|
|
2948
3170
|
const rows = db.prepare(`SELECT ${TAG_SELECT_COLUMNS} FROM tags WHERE session_id = ? AND tag_number IN (${placeholders}) ORDER BY tag_number ASC, id ASC`).all(sessionId, ...tagNumbers).filter(isTagRow);
|
|
2949
3171
|
return rows.map(toTagEntry);
|
|
2950
3172
|
}
|
|
3173
|
+
var newestToolTagNumberStatements = new WeakMap;
|
|
3174
|
+
function getNewestToolTagNumbers(db, sessionId, limit) {
|
|
3175
|
+
if (limit <= 0)
|
|
3176
|
+
return [];
|
|
3177
|
+
let statement = newestToolTagNumberStatements.get(db);
|
|
3178
|
+
if (!statement) {
|
|
3179
|
+
statement = db.prepare(`SELECT tag_number FROM tags
|
|
3180
|
+
WHERE session_id = ? AND type = 'tool'
|
|
3181
|
+
AND typeof(message_id) = 'text'
|
|
3182
|
+
AND typeof(status) = 'text'
|
|
3183
|
+
AND typeof(byte_size) IN ('integer', 'real')
|
|
3184
|
+
AND typeof(tag_number) IN ('integer', 'real')
|
|
3185
|
+
ORDER BY tag_number DESC
|
|
3186
|
+
LIMIT ?`);
|
|
3187
|
+
newestToolTagNumberStatements.set(db, statement);
|
|
3188
|
+
}
|
|
3189
|
+
return statement.all(sessionId, limit).map((row) => row.tag_number);
|
|
3190
|
+
}
|
|
2951
3191
|
var droppedNumberStatements = new WeakMap;
|
|
2952
3192
|
function getDroppedTagsByNumbers(db, sessionId, tagNumbers) {
|
|
2953
3193
|
if (tagNumbers.length === 0)
|
|
@@ -4587,6 +4827,14 @@ function getCurrentToolSetHash(providerID, modelID, agentName) {
|
|
|
4587
4827
|
return "";
|
|
4588
4828
|
return createHash2("sha256").update(Array.from(toolIds.keys()).sort().join("\x00")).digest("hex");
|
|
4589
4829
|
}
|
|
4830
|
+
function getToolDefinitionMeasurementSignature(providerID, modelID, agentName) {
|
|
4831
|
+
const key = keyFor(providerID, modelID, agentName);
|
|
4832
|
+
const measured = measurements.get(key);
|
|
4833
|
+
const observed = fingerprints.get(key);
|
|
4834
|
+
if (!measured || !observed || [...measured.keys()].some((id) => !observed.has(id)))
|
|
4835
|
+
return;
|
|
4836
|
+
return createHash2("sha256").update(JSON.stringify([...measured.keys()].sort().map((id) => [id, observed.get(id)]))).digest("hex");
|
|
4837
|
+
}
|
|
4590
4838
|
function setDatabase(db) {
|
|
4591
4839
|
persistenceDb = db;
|
|
4592
4840
|
cachedInsertStmt = null;
|
|
@@ -4700,8 +4948,10 @@ function serializedText(value) {
|
|
|
4700
4948
|
return (typeof value === "string" ? value : JSON.stringify(value)) ?? "";
|
|
4701
4949
|
}
|
|
4702
4950
|
var wireTextTokens = createTokenCountMemo(1e5, 128 * 1024 * 1024);
|
|
4703
|
-
function serializedTokens(value) {
|
|
4951
|
+
function serializedTokens(value, bounded = false) {
|
|
4704
4952
|
const serialized = serializedText(value);
|
|
4953
|
+
if (bounded && tokenCountUsesByteBound(serialized))
|
|
4954
|
+
return Buffer.byteLength(serialized);
|
|
4705
4955
|
return serialized ? wireTextTokens(serialized) : 0;
|
|
4706
4956
|
}
|
|
4707
4957
|
function visitWireContent(message, visit) {
|
|
@@ -4762,11 +5012,11 @@ function visitWireContent(message, visit) {
|
|
|
4762
5012
|
}
|
|
4763
5013
|
}
|
|
4764
5014
|
}
|
|
4765
|
-
function estimateMessageTokens(message) {
|
|
5015
|
+
function estimateMessageTokens(message, bounded = false) {
|
|
4766
5016
|
const total = { conversation: 0, toolCall: 0 };
|
|
4767
5017
|
visitWireContent(message, {
|
|
4768
5018
|
text: (bucket, value) => {
|
|
4769
|
-
total[bucket] += serializedTokens(value);
|
|
5019
|
+
total[bucket] += serializedTokens(value, bounded);
|
|
4770
5020
|
},
|
|
4771
5021
|
image: (bucket, tokens) => {
|
|
4772
5022
|
total[bucket] += tokens;
|
|
@@ -4774,9 +5024,122 @@ function estimateMessageTokens(message) {
|
|
|
4774
5024
|
});
|
|
4775
5025
|
return total;
|
|
4776
5026
|
}
|
|
5027
|
+
function wireContentBytes(messages, abortAboveBytes, bytesPerToken) {
|
|
5028
|
+
let bytes = 0;
|
|
5029
|
+
try {
|
|
5030
|
+
for (const message of messages) {
|
|
5031
|
+
visitWireContent(message, {
|
|
5032
|
+
text: (_bucket, value) => {
|
|
5033
|
+
bytes += Buffer.byteLength(serializedText(value));
|
|
5034
|
+
},
|
|
5035
|
+
image: (_bucket, tokens) => {
|
|
5036
|
+
bytes += tokens * bytesPerToken;
|
|
5037
|
+
}
|
|
5038
|
+
});
|
|
5039
|
+
if (bytes > abortAboveBytes)
|
|
5040
|
+
return { bytes, aborted: true };
|
|
5041
|
+
}
|
|
5042
|
+
} catch {
|
|
5043
|
+
return null;
|
|
5044
|
+
}
|
|
5045
|
+
return { bytes, aborted: false };
|
|
5046
|
+
}
|
|
5047
|
+
function createFinalWireUsageTracker() {
|
|
5048
|
+
const served = new Map;
|
|
5049
|
+
const wireRows = (input) => input.messages.filter((message) => message.parts.some((part) => !part || typeof part !== "object" || !["step-start", "step-finish"].includes(String(part.type))));
|
|
5050
|
+
const rowBytes = (message) => JSON.stringify({ id: message.info.id, role: message.info.role, parts: message.parts });
|
|
5051
|
+
const envelopeFor = (input) => {
|
|
5052
|
+
if (!input.providerID || !input.modelID || !input.systemPromptHash)
|
|
5053
|
+
return;
|
|
5054
|
+
const tools = getToolDefinitionMeasurementSignature(input.providerID, input.modelID, input.agentName);
|
|
5055
|
+
if (!tools)
|
|
5056
|
+
return;
|
|
5057
|
+
return JSON.stringify([
|
|
5058
|
+
input.providerID,
|
|
5059
|
+
input.modelID,
|
|
5060
|
+
input.agentName,
|
|
5061
|
+
input.systemPromptHash,
|
|
5062
|
+
input.systemPromptTokens,
|
|
5063
|
+
tools
|
|
5064
|
+
]);
|
|
5065
|
+
};
|
|
5066
|
+
return {
|
|
5067
|
+
estimate(sessionId, input, limit) {
|
|
5068
|
+
let measuredPrefix;
|
|
5069
|
+
try {
|
|
5070
|
+
const previous = served.get(sessionId);
|
|
5071
|
+
const rows = wireRows(input);
|
|
5072
|
+
if (previous && previous.envelope === envelopeFor(input) && rows.length > previous.prefix.length && previous.prefix.every((bytes, i) => rowBytes(rows[i]) === bytes)) {
|
|
5073
|
+
const reply = rows[previous.prefix.length]?.info;
|
|
5074
|
+
const tokens = reply.tokens;
|
|
5075
|
+
const time = reply.time;
|
|
5076
|
+
const counts = [tokens?.input, tokens?.cache?.read, tokens?.cache?.write];
|
|
5077
|
+
if (reply.role === "assistant" && reply.parentID === previous.parent && reply.providerID === input.providerID && reply.modelID === input.modelID && ["stop", "tool-calls", "length"].includes(String(reply.finish)) && !reply.error && typeof time?.completed === "number" && time.completed >= previous.at && counts.every((n) => typeof n === "number" && Number.isSafeInteger(n) && n >= 0)) {
|
|
5078
|
+
const inputTokens = counts.reduce((sum, n) => sum + (n ?? 0), 0);
|
|
5079
|
+
if (inputTokens > 0)
|
|
5080
|
+
measuredPrefix = {
|
|
5081
|
+
inputTokens,
|
|
5082
|
+
appendedMessages: rows.slice(previous.prefix.length)
|
|
5083
|
+
};
|
|
5084
|
+
}
|
|
5085
|
+
}
|
|
5086
|
+
} catch {}
|
|
5087
|
+
return estimateOutgoingWireForRefusal({ ...input, measuredPrefix }, limit);
|
|
5088
|
+
},
|
|
5089
|
+
capture(sessionId, input) {
|
|
5090
|
+
served.delete(sessionId);
|
|
5091
|
+
try {
|
|
5092
|
+
const envelope = envelopeFor(input);
|
|
5093
|
+
const rows = wireRows(input);
|
|
5094
|
+
const parent = [...rows].reverse().find((row) => row.info.role === "user")?.info.id;
|
|
5095
|
+
if (!envelope || typeof parent !== "string" || !rows.every(hasCountableParts))
|
|
5096
|
+
return;
|
|
5097
|
+
const prefix = rows.map(rowBytes);
|
|
5098
|
+
if (prefix.reduce((sum, text) => sum + text.length, 0) > 4 * 1024 * 1024)
|
|
5099
|
+
return;
|
|
5100
|
+
if (served.size >= 16) {
|
|
5101
|
+
const oldest = served.keys().next().value;
|
|
5102
|
+
if (oldest !== undefined)
|
|
5103
|
+
served.delete(oldest);
|
|
5104
|
+
}
|
|
5105
|
+
served.set(sessionId, { envelope, prefix, parent, at: Date.now() });
|
|
5106
|
+
} catch {}
|
|
5107
|
+
}
|
|
5108
|
+
};
|
|
5109
|
+
}
|
|
5110
|
+
function estimateOutgoingWireForRefusal(input, limit) {
|
|
5111
|
+
if (typeof limit === "number" && Number.isFinite(limit) && limit > 0) {
|
|
5112
|
+
const measured = input.measuredPrefix;
|
|
5113
|
+
const baseline = measured?.inputTokens ?? 0;
|
|
5114
|
+
const pricedMessages = measured?.appendedMessages ?? input.messages;
|
|
5115
|
+
const bytes = wireContentBytes(pricedMessages, Math.max(0, limit - baseline) * 4, 4);
|
|
5116
|
+
if (bytes) {
|
|
5117
|
+
const lower = baseline + Math.ceil(bytes.bytes / 4);
|
|
5118
|
+
const calibration = resolveDecisionCalibration(input.providerID, input.modelID);
|
|
5119
|
+
const tools = input.providerID && input.modelID ? getMeasuredToolDefinitionTokens(input.providerID, input.modelID, input.agentName) : undefined;
|
|
5120
|
+
const envelope = measured ? baseline : tools !== undefined && Number.isFinite(input.systemPromptTokens) && input.systemPromptTokens > 0 ? providerMass({ system: input.systemPromptTokens, tools }, calibration, true) : undefined;
|
|
5121
|
+
const upper = envelope === undefined ? undefined : envelope + Math.ceil(bytes.bytes * Math.max(UNKNOWN_FIT_RATIO, calibration.toolsRatio, calibration.proseRatio));
|
|
5122
|
+
if (bytes.aborted || lower > limit || upper !== undefined && upper <= limit) {
|
|
5123
|
+
const over = bytes.aborted || lower > limit;
|
|
5124
|
+
return {
|
|
5125
|
+
tokens: over ? lower : upper ?? lower,
|
|
5126
|
+
trusted: !over,
|
|
5127
|
+
refusalGrade: over && envelope !== undefined,
|
|
5128
|
+
refusalTokens: over ? lower : undefined,
|
|
5129
|
+
refusalBasis: "byte-bound",
|
|
5130
|
+
messageTokens: { conversation: 0, toolCall: 0 },
|
|
5131
|
+
systemTokens: input.systemPromptTokens,
|
|
5132
|
+
toolDefinitionTokens: tools,
|
|
5133
|
+
toolDefinitionsMeasured: tools !== undefined
|
|
5134
|
+
};
|
|
5135
|
+
}
|
|
5136
|
+
}
|
|
5137
|
+
}
|
|
5138
|
+
return estimateFinalWireInputTokens({ ...input, boundedTokenization: true });
|
|
5139
|
+
}
|
|
4777
5140
|
function estimateFinalWireInputTokens(input) {
|
|
4778
5141
|
const messageTokens = input.messages.reduce((total, message) => {
|
|
4779
|
-
const next = estimateMessageTokens(message);
|
|
5142
|
+
const next = estimateMessageTokens(message, input.boundedTokenization);
|
|
4780
5143
|
total.conversation += next.conversation;
|
|
4781
5144
|
total.toolCall += next.toolCall;
|
|
4782
5145
|
return total;
|
|
@@ -4792,11 +5155,25 @@ function estimateFinalWireInputTokens(input) {
|
|
|
4792
5155
|
};
|
|
4793
5156
|
const tokens = providerMass(rawComponents, calibration, true);
|
|
4794
5157
|
const complete = Number.isFinite(tokens) && tokens > 0 && Number.isFinite(input.systemPromptTokens) && input.systemPromptTokens > 0 && toolDefinitions !== undefined && input.messages.every(hasCountableParts);
|
|
5158
|
+
const refusalGrade = complete && hasTokenizerForFit() && measuredToolDefinitions !== undefined && hasMeasuredDecisionCalibration(calibration) && input.messages.every(hasRefusalCountableParts);
|
|
5159
|
+
const measured = input.measuredPrefix;
|
|
5160
|
+
const useMeasured = refusalGrade && measured && Number.isSafeInteger(measured.inputTokens) && measured.inputTokens > 0 && measured.appendedMessages.every(hasCountableParts);
|
|
5161
|
+
const tail = useMeasured ? measured.appendedMessages.reduce((sum, message) => {
|
|
5162
|
+
const next = estimateMessageTokens(message, input.boundedTokenization);
|
|
5163
|
+
return {
|
|
5164
|
+
conversation: sum.conversation + next.conversation,
|
|
5165
|
+
toolCall: sum.toolCall + next.toolCall
|
|
5166
|
+
};
|
|
5167
|
+
}, { conversation: 0, toolCall: 0 }) : undefined;
|
|
5168
|
+
const refusalTokens = refusalGrade ? tail && measured ? measured.inputTokens + providerMass({ tools: tail.toolCall, prose: tail.conversation }, calibration) : providerMass(rawComponents, calibration) : undefined;
|
|
4795
5169
|
const systemTokens = Math.round(Math.max(0, input.systemPromptTokens) * calibration.systemRatio);
|
|
4796
5170
|
const toolDefinitionTokens = toolDefinitions === undefined ? undefined : Math.round(toolDefinitions * (calibration.seeded ? calibration.toolsRatio : UNKNOWN_FIT_RATIO));
|
|
4797
5171
|
return {
|
|
4798
5172
|
tokens,
|
|
4799
5173
|
trusted: complete,
|
|
5174
|
+
refusalGrade,
|
|
5175
|
+
refusalTokens,
|
|
5176
|
+
refusalBasis: refusalGrade ? useMeasured ? "provider-prefix" : "calibrated" : undefined,
|
|
4800
5177
|
rawTokens: rawComponents.system + rawComponents.tools + rawComponents.prose,
|
|
4801
5178
|
rawComponents,
|
|
4802
5179
|
completeness: complete ? "complete" : "partial",
|
|
@@ -4807,6 +5184,17 @@ function estimateFinalWireInputTokens(input) {
|
|
|
4807
5184
|
toolDefinitionsMeasured: measuredToolDefinitions !== undefined
|
|
4808
5185
|
};
|
|
4809
5186
|
}
|
|
5187
|
+
function hasRefusalCountableParts(message) {
|
|
5188
|
+
let exact = true;
|
|
5189
|
+
visitWireContent(message, {
|
|
5190
|
+
text: (_bucket, value) => {
|
|
5191
|
+
if (tokenCountUsesByteBound(serializedText(value)))
|
|
5192
|
+
exact = false;
|
|
5193
|
+
},
|
|
5194
|
+
image: () => {}
|
|
5195
|
+
});
|
|
5196
|
+
return exact;
|
|
5197
|
+
}
|
|
4810
5198
|
function hasCountableParts(message) {
|
|
4811
5199
|
return message.parts.every((part) => {
|
|
4812
5200
|
if (!part || typeof part !== "object")
|
|
@@ -5067,6 +5455,9 @@ function restoreNeutralizedReasoningPart(part, legacyCleared) {
|
|
|
5067
5455
|
function isNeutralizedReasoningPart(part) {
|
|
5068
5456
|
return isRecord(part) && part[REMOVED_REASONING_MARK] === true;
|
|
5069
5457
|
}
|
|
5458
|
+
function neutralizedReasoningSource(part) {
|
|
5459
|
+
return isRecord(part) ? NEUTRALIZED_ORIGINALS.get(part) ?? part : part;
|
|
5460
|
+
}
|
|
5070
5461
|
var VARIANT_CACHE_PRESERVING_MODELS = {
|
|
5071
5462
|
"anthropic/claude-fable-5-1": "2026-09-02",
|
|
5072
5463
|
"openai/gpt-6-astra": "2026-09-05",
|
|
@@ -6385,4 +6776,4 @@ function readSessionChunk2(sessionId, tokenBudget, offset = 1, eligibleEndOrdina
|
|
|
6385
6776
|
toolResultBoundaries
|
|
6386
6777
|
};
|
|
6387
6778
|
}
|
|
6388
|
-
export { toolTemplateError, isRecord, isPrefixBoundThinkingModel, detectOverflow,
|
|
6779
|
+
export { toolTemplateError, DEFAULT_PROTECTED_TOOLS, normalizeProtectedToolName, mergeProtectedTools, isRecord, isPrefixBoundThinkingModel, detectOverflow, protectedToolTagNumbers, protectedToolTokenCount, textMentionsRecentCommit, removeSystemReminders, hasMeaningfulUserText2, extractTexts2, getTokenizerNativeMemoryStats, preloadTokenizer, estimateTokens, estimateFixedPromptTokens, stripWellFormedLeadingTagPrefix, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, isThinkingPart, stableStringify, contentTagOwnerMessageId, estimateImageTokensFromDataUrl, estimateToolAttachmentImageTokens, normalizeTodoStateJson, buildSyntheticTodoPart, isSyntheticTodoPart, countRealUserMessages, tailHygieneStructuralSignature, sameTailHygieneStructuralSignature, formatTailHygienePrefixMismatch, refreshTailHygieneBaseline, effectiveTailHygiene, assertTailHygieneContentUnchangedIfEnabled, CHANNEL1_SENTINEL, CHANNEL1_FLOOR_TOKENS, isDroppedToolOutput, toolOutputTokens, decideChannel1, evaluateChannel2, formatChannel1Evaluation, formatChannel2Evaluation, reclaimableToolOutputCount, buildChannel2Reminder, buildChannel1Reminder, estimateEmergencyDropReclaimTokens, planEmergencyDrop, measureEmergencyTag, installV3PerfSchema, queuePendingOp, queuePendingDropForUnchangedTag, getPendingOps, getPendingOpsCount, clearPendingOps, removePendingOp, getRecentTagOwnerMessageIds, AGE_RECLAIM_MIN_TOKENS, getOldestActiveUnprotectedToolTags, getActiveToolTagsForAgeReclaim, getTriggerTagTokenUpperBound, getActiveTagTokenTotalsByMessage, getAllStatusTagTokenTotalsFlat, getReasoningTokenEstimatesByMessage, tagTokenCountIsNull, backfillTagTokenCounts, insertTag, updateTagStatus, markWhitespaceAssistantTagInert, getInertWhitespaceAssistantTags, updateTagDropMode, updateCavemanDepth, updateTagMessageId, foldShrunkPartTags, markTagsCompactedByMessageIds, getMaxTagNumberBySession, getTagNumberByMessageId, getAssignableTagNumberByMessageId, deriveTagLoadFloor, TAG_SELECT_COLUMNS, getTagsBySession, getActiveTagsBySession, getTailHygieneTags, getTagsByNumbers, getNewestToolTagNumbers, getDroppedTagsByNumbers, getMaxDroppedTagNumber, getTagById, getToolTagNumberByOwner, getNullOwnerToolTag, adoptNullOwnerToolTag, getCandidateToolOwners, pickNearestPriorOwner, expandToolPart, markHostUnservedRow, retreatPastHostUnservedRows, snapTerminalCompartmentToServedRow, isHostRenderedSystemMessage, isStrictGapHealingMessage, extractInMemoryMessageViews, buildInMemoryTailRawMessages, completedToolArcCrossesBoundary, estimateTrueRawMessageTokens, buildToolArcs, fenceBoundaryForCompletedToolArcs, fenceBoundaryForToolArcs, buildTrueRawTokenIndex, computeRawRangeFingerprint, isTextPart, isToolPartWithOutput, isFilePart, buildFileSourceContent, toolPartHasUserAnswer, recordToolParameters, createDroppedInputGuard, assertExecutableToolInput, isEditTool, getCurrentToolSetHash, setDatabase, loadToolDefinitionMeasurements, recordToolDefinition, getMeasuredToolDefinitionTokens, describeFinalWireTail, estimateMessageTokens, createFinalWireUsageTracker, estimateFinalWireInputTokens, modelAcceptsEmptyContent, isAnthropicFamilyRoute, neutralizeDroppedReasoningPart, restoreNeutralizedReasoningPart, isNeutralizedReasoningPart, neutralizedReasoningSource, variantChangeBustsProviderCache, makeSentinel, makeWholeMessageSentinel, isSentinel, replaySentinelByMessageIds, SKELETON_REAL_INPUT_MAX_BYTES, partHasCompletedResult, extractToolCallObservation, ToolMutationBatch, createToolDropTarget, resolveHostServedBoundaryId2, hasRawMessageProvider2, setBoundedRawMessageProvider2, cleanUserText2, withRawSessionMessageCache2, readRawSessionMessages2, readRawSessionMessagePage2, getRawSessionMessageOrdinalCount2, visitRawSessionMessages2, readRawSessionMessageRange2, primeTailRawMessageCache2, getCachedAbsoluteMessageCount2, primeInMemoryTailRawMessageCache2, readRawSessionMessageIdOrdinalsForRange2, hasRawSessionMessageById2, readRawSessionMessageOrdinalById2, compareRawSessionMessageOrder2, readRawSessionMessageById2, getRawSessionMessageCount2, getRawSessionTagKeysThrough2, getLegacyProtectedTailStartOrdinal2, readSessionChunk2 };
|