@danypops/jittor 0.16.2 → 0.17.0

package/src/{domain → observability}/context-telemetry.ts

@@ -7,6 +7,7 @@ import {
 	PAPYRUS_CONTEXT_INJECTION_SCHEMA,
 } from "../constants.ts";
 import type { MetricObservation, StoredMetricObservation } from "./metric.ts";
+import type { TokenMeasurementProvenance } from "./token-measurement.ts";
 
 interface PayloadSize {
 	characters: number;
@@ -133,16 +134,50 @@ export function papyrusContextMetric(observation: PapyrusContextInjection): Metr
 	};
 }
 
+export type CompactionMechanism = "pi-native" | "provider-side" | "extension";
+
 export interface CompactionStart {
 	reason: "manual" | "threshold" | "overflow";
 	willRetry: boolean;
+	mechanism?: CompactionMechanism;
 	contextPercent?: number;
 	contextTokens?: number;
+	contextProvenance?: TokenMeasurementProvenance;
+	provider?: string;
+	model?: string;
+}
+
+export interface CompactionCompletion extends Pick<CompactionStart, "reason" | "willRetry"> {
+	mechanism?: CompactionMechanism;
+	summaryTokens?: number;
+	summaryProvenance?: TokenMeasurementProvenance;
 }
 
 interface OpenCompaction extends CompactionStart {
+	mechanism: CompactionMechanism;
 	startedAt: number;
 }
+
+interface PendingEffectiveness {
+	preContextTokens: number;
+	preContextProvenance: TokenMeasurementProvenance;
+	mechanism: CompactionMechanism;
+	provider?: string;
+	model?: string;
+	summaryTokens?: number;
+	summaryProvenance?: TokenMeasurementProvenance;
+	completedAt: number;
+}
+
+interface RegrowthState {
+	preContextTokens: number;
+	completedAt: number;
+	turns: number;
+	emitted: Set<number>;
+	mechanism: CompactionMechanism;
+	provider?: string;
+	model?: string;
+}
 interface UsageCounters {
 	turns: number;
 	injectedCharacters: number;
@@ -150,22 +185,42 @@ interface UsageCounters {
 	providerTokens: number;
 	cacheReadTokens: number;
 	cacheWriteTokens: number;
+	toolCalls: number;
+	toolFailures: number;
+	toolClasses: Map<string, number>;
+	acceptedOutcomes: number;
+	rejectedOutcomes: number;
 }
 
 function emptyCounters(): UsageCounters {
-	return { turns: 0, injectedCharacters: 0, estimatedInjectedTokens: 0, providerTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
+	return {
+		turns: 0,
+		injectedCharacters: 0,
+		estimatedInjectedTokens: 0,
+		providerTokens: 0,
+		cacheReadTokens: 0,
+		cacheWriteTokens: 0,
+		toolCalls: 0,
+		toolFailures: 0,
+		toolClasses: new Map(),
+		acceptedOutcomes: 0,
+		rejectedOutcomes: 0,
+	};
 }
 
 export class CompactionTelemetry {
 	private open: OpenCompaction | undefined;
 	private counters = emptyCounters();
 	private previousCompletedAt: number | undefined;
+	private pendingEffectiveness: PendingEffectiveness | undefined;
+	private regrowth: RegrowthState | undefined;
 
 	hasOpenCompaction(): boolean {
 		return this.open !== undefined;
 	}
 	observeTurn(): void {
 		this.counters.turns += 1;
+		if (this.regrowth) this.regrowth.turns += 1;
 	}
 	observeInjection(characters: number, estimatedTokens: number): void {
 		this.counters.injectedCharacters += Math.max(0, characters);
@@ -176,9 +231,31 @@ export class CompactionTelemetry {
 		this.counters.cacheReadTokens += Math.max(0, usage.cacheRead);
 		this.counters.cacheWriteTokens += Math.max(0, usage.cacheWrite);
 	}
+	observeToolClass(toolClass: string, failed: boolean): void {
+		if (!/^[a-z][a-z0-9-]{0,31}$/.test(toolClass)) return;
+		this.counters.toolCalls += 1;
+		if (failed) this.counters.toolFailures += 1;
+		if (this.counters.toolClasses.size < 16 || this.counters.toolClasses.has(toolClass))
+			this.counters.toolClasses.set(toolClass, (this.counters.toolClasses.get(toolClass) ?? 0) + 1);
+	}
+	observeOutcome(outcome: "accepted" | "rejected"): void {
+		if (outcome === "accepted") this.counters.acceptedOutcomes += 1;
+		else this.counters.rejectedOutcomes += 1;
+	}
 
 	begin(input: CompactionStart, now = Date.now()): MetricObservation {
-		this.open = { ...input, startedAt: now };
+		if (this.open) {
+			return {
+				source: "pi-context",
+				scope: "compaction",
+				metric: "compaction-overlap",
+				value: 1,
+				unit: "count",
+				observedAt: now,
+				attributes: { reason: input.reason, willRetry: input.willRetry, openReason: this.open.reason },
+			};
+		}
+		this.open = { ...input, mechanism: input.mechanism ?? "pi-native", startedAt: now };
 		return {
 			source: "pi-context",
 			scope: "compaction",
@@ -190,7 +267,7 @@ export class CompactionTelemetry {
 		};
 	}
 
-	complete(input: Pick<CompactionStart, "reason" | "willRetry">, now = Date.now()): MetricObservation {
+	complete(input: CompactionCompletion, now = Date.now()): MetricObservation {
 		if (!this.open)
 			return {
 				source: "pi-context",
@@ -206,6 +283,18 @@ export class CompactionTelemetry {
 		const attributes = this.intervalAttributes(open, now);
 		this.previousCompletedAt = now;
 		this.counters = emptyCounters();
+		if (typeof open.contextTokens === "number" && Number.isSafeInteger(open.contextTokens) && open.contextTokens > 0) {
+			this.pendingEffectiveness = {
+				preContextTokens: open.contextTokens,
+				preContextProvenance: open.contextProvenance ?? "provider-reported",
+				mechanism: input.mechanism ?? open.mechanism,
+				...(open.provider === undefined ? {} : { provider: open.provider }),
+				...(open.model === undefined ? {} : { model: open.model }),
+				...(input.summaryTokens === undefined ? {} : { summaryTokens: Math.max(0, Math.round(input.summaryTokens)) }),
+				...(input.summaryProvenance === undefined ? {} : { summaryProvenance: input.summaryProvenance }),
+				completedAt: now,
+			};
+		}
 		return {
 			source: "pi-context",
 			scope: "compaction",
@@ -213,10 +302,108 @@ export class CompactionTelemetry {
 			value: Math.max(0, now - open.startedAt),
 			unit: "milliseconds",
 			observedAt: now,
-			attributes: { ...attributes, reason: input.reason, willRetry: input.willRetry },
+			attributes: {
+				...attributes,
+				reason: input.reason,
+				willRetry: input.willRetry,
+				mechanism: input.mechanism ?? open.mechanism,
+				...(input.summaryTokens === undefined ? {} : { summaryTokens: Math.max(0, Math.round(input.summaryTokens)) }),
+				...(input.summaryProvenance === undefined ? {} : { summaryProvenance: input.summaryProvenance }),
+			},
 		};
 	}
 
+	/** Correlates the first post-compaction request, then reports bounded 50/80/100% regrowth milestones once each. */
+	observeContextSnapshot(
+		tokens: number,
+		provenance: TokenMeasurementProvenance,
+		now = Date.now(),
+		identity?: { provider: string; model: string },
+	): MetricObservation[] {
+		if (!Number.isSafeInteger(tokens) || tokens < 0) return [];
+		const observations: MetricObservation[] = [];
+		if (this.pendingEffectiveness) {
+			const pending = this.pendingEffectiveness;
+			this.pendingEffectiveness = undefined;
+			const identityChange =
+				identity && pending.provider && identity.provider !== pending.provider
+					? "provider-changed"
+					: identity && pending.model && identity.model !== pending.model
+						? "model-changed"
+						: null;
+			if (identityChange) {
+				this.regrowth = undefined;
+				return [
+					{
+						source: "pi-context",
+						scope: "compaction",
+						metric: "compaction-effectiveness-unavailable",
+						value: 1,
+						unit: "count",
+						observedAt: now,
+						attributes: { reason: identityChange, mechanism: pending.mechanism },
+					},
+				];
+			}
+			const reduction = (pending.preContextTokens - tokens) / pending.preContextTokens;
+			observations.push({
+				source: "pi-context",
+				scope: "compaction",
+				metric: "compaction-effectiveness",
+				value: reduction,
+				unit: "ratio",
+				observedAt: now,
+				attributes: {
+					mechanism: pending.mechanism,
+					preContextTokens: pending.preContextTokens,
+					postContextTokens: tokens,
+					preContextProvenance: pending.preContextProvenance,
+					postContextProvenance: provenance,
+					...(pending.provider === undefined ? {} : { provider: pending.provider }),
+					...(pending.model === undefined ? {} : { model: pending.model }),
+					...(pending.summaryTokens === undefined ? {} : { summaryTokens: pending.summaryTokens }),
+					...(pending.summaryProvenance === undefined ? {} : { summaryProvenance: pending.summaryProvenance }),
+				},
+			});
+			this.regrowth = {
+				preContextTokens: pending.preContextTokens,
+				completedAt: pending.completedAt,
+				turns: 0,
+				emitted: new Set(),
+				mechanism: pending.mechanism,
+				...(pending.provider === undefined ? {} : { provider: pending.provider }),
+				...(pending.model === undefined ? {} : { model: pending.model }),
+			};
+			return observations;
+		}
+		const regrowth = this.regrowth;
+		if (!regrowth) return observations;
+		const fraction = tokens / regrowth.preContextTokens;
+		for (const threshold of [0.5, 0.8, 1]) {
+			if (fraction < threshold || regrowth.emitted.has(threshold)) continue;
+			regrowth.emitted.add(threshold);
+			observations.push({
+				source: "pi-context",
+				scope: "compaction",
+				metric: "compaction-regrowth",
+				value: threshold,
+				unit: "ratio",
+				observedAt: now,
+				attributes: {
+					threshold,
+					contextTokens: tokens,
+					contextProvenance: provenance,
+					turnsSinceCompaction: regrowth.turns,
+					elapsedSinceCompactionMs: Math.max(0, now - regrowth.completedAt),
+					mechanism: regrowth.mechanism,
+					...(regrowth.provider === undefined ? {} : { provider: regrowth.provider }),
+					...(regrowth.model === undefined ? {} : { model: regrowth.model }),
+				},
+			});
+		}
+		return observations;
+	}
+
 	abort(now = Date.now(), abortReason = "aborted"): MetricObservation {
 		const open = this.open;
 		this.open = undefined;
@@ -247,6 +434,10 @@ export class CompactionTelemetry {
 		return {
 			reason: open.reason,
 			willRetry: open.willRetry,
+			mechanism: open.mechanism,
+			...(open.contextProvenance === undefined ? {} : { contextProvenance: open.contextProvenance }),
+			...(open.provider === undefined ? {} : { provider: open.provider }),
+			...(open.model === undefined ? {} : { model: open.model }),
 			...(open.contextPercent === undefined ? {} : { contextPercent: open.contextPercent }),
 			...(open.contextTokens === undefined ? {} : { contextTokens: open.contextTokens }),
 			turnsSincePrevious: this.counters.turns,
@@ -255,6 +446,13 @@ export class CompactionTelemetry {
 			providerTokensSincePrevious: this.counters.providerTokens,
 			cacheReadTokensSincePrevious: this.counters.cacheReadTokens,
 			cacheWriteTokensSincePrevious: this.counters.cacheWriteTokens,
+			toolCallsSincePrevious: this.counters.toolCalls,
+			toolFailuresSincePrevious: this.counters.toolFailures,
+			repeatedToolClassesSincePrevious: [...this.counters.toolClasses.entries()]
+				.filter(([, count]) => count > 1)
+				.map(([name, count]) => `${name}:${count}`),
+			acceptedOutcomesSincePrevious: this.counters.acceptedOutcomes,
+			rejectedOutcomesSincePrevious: this.counters.rejectedOutcomes,
 			...(this.previousCompletedAt === undefined ? {} : { elapsedSincePreviousMs: Math.max(0, now - this.previousCompletedAt) }),
 		};
 	}
@@ -285,6 +483,12 @@ export interface ContextAssessment {
 		averageElapsedMsBetween: number | null;
 		averageProviderTokensBetween: number | null;
 		averageCacheReadTokensBetween: number | null;
+		effectivenessSamples: number;
+		averageReductionRatio: number | null;
+		averagePreContextTokens: number | null;
+		averagePostContextTokens: number | null;
+		mechanisms: Record<CompactionMechanism, number>;
+		regrowth: Record<"50" | "80" | "100", { samples: number; averageTurns: number | null; averageElapsedMs: number | null } | null>;
 		reasons: Record<"manual" | "threshold" | "overflow", number>;
 	};
 }
@@ -315,6 +519,24 @@ export function assessContextTelemetry(
 	const unchanged = injections.filter((row) => row.attributes.unchanged === true).length;
 	const completed = compactions.filter((row) => row.metric === "compaction-duration" && typeof row.value === "number");
 	const aborted = compactions.filter((row) => row.metric === "compaction-aborted");
+	const effectiveness = compactions.filter(
+		(row) => row.metric === "compaction-effectiveness" && typeof row.value === "number" && Number.isFinite(row.value),
+	);
+	const regrowthRows = compactions.filter((row) => row.metric === "compaction-regrowth");
+	const mechanisms: Record<CompactionMechanism, number> = { "pi-native": 0, "provider-side": 0, extension: 0 };
+	for (const row of effectiveness) {
+		const mechanism = row.attributes.mechanism;
+		if (mechanism === "pi-native" || mechanism === "provider-side" || mechanism === "extension") mechanisms[mechanism] += 1;
+	}
+	const regrowth = (threshold: number): { samples: number; averageTurns: number | null; averageElapsedMs: number | null } | null => {
+		const rows = regrowthRows.filter((row) => row.attributes.threshold === threshold);
+		if (rows.length === 0) return null;
+		return {
+			samples: rows.length,
+			averageTurns: average(rows.flatMap((row) => numericAttribute(row, "turnsSinceCompaction") ?? [])),
+			averageElapsedMs: average(rows.flatMap((row) => numericAttribute(row, "elapsedSinceCompactionMs") ?? [])),
+		};
+	};
 	const reasons = { manual: 0, threshold: 0, overflow: 0 };
 	for (const row of completed) {
 		const reason = row.attributes.reason;
@@ -347,6 +569,12 @@ export function assessContextTelemetry(
 			averageElapsedMsBetween: average(completed.flatMap((row) => numericAttribute(row, "elapsedSincePreviousMs") ?? [])),
 			averageProviderTokensBetween: average(completed.flatMap((row) => numericAttribute(row, "providerTokensSincePrevious") ?? [])),
 			averageCacheReadTokensBetween: average(completed.flatMap((row) => numericAttribute(row, "cacheReadTokensSincePrevious") ?? [])),
+			effectivenessSamples: effectiveness.length,
+			averageReductionRatio: average(effectiveness.map((row) => row.value as number)),
+			averagePreContextTokens: average(effectiveness.flatMap((row) => numericAttribute(row, "preContextTokens") ?? [])),
+			averagePostContextTokens: average(effectiveness.flatMap((row) => numericAttribute(row, "postContextTokens") ?? [])),
+			mechanisms,
+			regrowth: { "50": regrowth(0.5), "80": regrowth(0.8), "100": regrowth(1) },
 			reasons,
 		},
 	};

package/src/observability/model-identity.ts

@@ -0,0 +1,42 @@
+import { BENCHMARK_IDENTITY_MAX_CHARACTERS } from "../constants.ts";
+
+const VERSION_SUFFIX = /-(\d{4}-\d{2}-\d{2})$/;
+
+export interface ModelIdentity {
+	provider: string;
+	model: string;
+	version: string | null;
+	canonical: string;
+	aliases: string[];
+}
+
+function identityText(value: string, name: string): string {
+	if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${name} is required`);
+	const normalized = value.trim().toLowerCase();
+	if (normalized.length > BENCHMARK_IDENTITY_MAX_CHARACTERS) throw new Error(`${name} exceeds the length limit`);
+	if (/\p{Cc}/u.test(normalized)) throw new Error(`${name} contains control characters`);
+	return normalized;
+}
+
+function identityPart(value: string, name: string, allowPath = false): string {
+	const normalized = identityText(value, name);
+	const pattern = allowPath ? /^[a-z0-9][a-z0-9._:+/-]*$/ : /^[a-z0-9][a-z0-9._:+-]*$/;
+	if (!pattern.test(normalized) || normalized.includes("//")) throw new Error(`${name} is invalid`);
+	return normalized;
+}
+
+export function normalizeModelIdentity(provider: string, model: string, aliases: string[] = []): ModelIdentity {
+	const normalizedProvider = identityPart(provider, "provider");
+	const normalizedModel = identityPart(model, "model", true);
+	const canonical = `${normalizedProvider}/${normalizedModel}`;
+	const normalizedAliases = [...new Set(aliases.map((alias) => identityText(alias, "alias")))]
+		.filter((alias) => alias !== canonical)
+		.sort();
+	return {
+		provider: normalizedProvider,
+		model: normalizedModel,
+		version: VERSION_SUFFIX.exec(normalizedModel)?.[1] ?? null,
+		canonical,
+		aliases: normalizedAliases,
+	};
+}

package/src/{domain → observability}/model-observation.ts

@@ -4,8 +4,9 @@ import {
 	MODEL_OBSERVATION_FRESH_MS,
 	MODEL_OBSERVATION_IDENTITY_MAX_CHARACTERS,
 } from "../constants.ts";
-import { normalizeModelIdentity } from "./benchmark.ts";
 import type { MetricObservation, MetricUnit, StoredMetricObservation } from "./metric.ts";
+import { normalizeModelIdentity } from "./model-identity.ts";
+import type { TokenMeasurementScope } from "./token-measurement.ts";
 
 /**
  * Two independent axes, not one flat class: "coding" is a subject-matter domain (which
@@ -131,10 +132,10 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
 		startedAt,
 		firstTokenAt,
 		completedAt,
-		inputTokens: nonNegative(input.inputTokens, "input tokens"),
-		outputTokens: nonNegative(input.outputTokens, "output tokens"),
-		cacheReadTokens: nonNegative(input.cacheReadTokens, "cache read tokens"),
-		cacheWriteTokens: nonNegative(input.cacheWriteTokens, "cache write tokens"),
+		inputTokens: nonNegative(input.inputTokens, "input tokens", true),
+		outputTokens: nonNegative(input.outputTokens, "output tokens", true),
+		cacheReadTokens: nonNegative(input.cacheReadTokens, "cache read tokens", true),
+		cacheWriteTokens: nonNegative(input.cacheWriteTokens, "cache write tokens", true),
 		costUsd: nonNegative(input.costUsd, "cost"),
 		providerResponses,
 		toolCalls,
@@ -174,14 +175,28 @@ export function modelRunMetrics(value: ModelRunObservation): MetricObservation[]
 		type: run.type,
 		runId: run.runId,
 	};
-	const metric = (name: string, amount: number, unit: MetricUnit): MetricObservation => ({
+	const metric = (name: string, amount: number, unit: MetricUnit, tokenScope?: TokenMeasurementScope): MetricObservation => ({
 		source: "local-model",
 		scope,
 		metric: name,
 		value: amount,
 		unit,
 		observedAt: run.completedAt,
-		attributes,
+		attributes: {
+			...attributes,
+			...(tokenScope
+				? {
+						tokenMeasurement: {
+							tokens: amount,
+							scope: tokenScope,
+							provenance: "provider-reported",
+							method: "pi-assistant-usage",
+							provider: run.provider,
+							model: run.model,
+						},
+					}
+				: {}),
+		},
 	});
 	const wallMs = run.completedAt - run.startedAt;
 	const totalInput = run.inputTokens + run.cacheReadTokens;
@@ -190,10 +205,10 @@ export function modelRunMetrics(value: ModelRunObservation): MetricObservation[]
 	metrics.push(
 		metric("wall-latency", wallMs, "milliseconds"),
 		metric("output-throughput", wallMs === 0 ? 0 : run.outputTokens / (wallMs / 1_000), "tokens-per-second"),
-		metric("input-tokens", run.inputTokens, "tokens"),
-		metric("output-tokens", run.outputTokens, "tokens"),
-		metric("cache-read-tokens", run.cacheReadTokens, "tokens"),
-		metric("cache-write-tokens", run.cacheWriteTokens, "tokens"),
+		metric("input-tokens", run.inputTokens, "tokens", "request-input"),
+		metric("output-tokens", run.outputTokens, "tokens", "response-output"),
+		metric("cache-read-tokens", run.cacheReadTokens, "tokens", "cache-read"),
+		metric("cache-write-tokens", run.cacheWriteTokens, "tokens", "cache-write"),
 		metric("cache-read-ratio", totalInput === 0 ? 0 : run.cacheReadTokens / totalInput, "ratio"),
 		metric("cost", run.costUsd, "usd"),
 		metric("provider-responses", run.providerResponses, "count"),

package/src/{ports/metric-store.ts → observability/store.ts}

@@ -1,5 +1,5 @@
-import type { MetricObservation, MetricQuery, StoredMetricObservation } from "../domain/metric.ts";
-import type { UsageAggregateRow } from "../domain/usage.ts";
+import type { MetricObservation, MetricQuery, StoredMetricObservation } from "./metric.ts";
+import type { UsageAggregateRow } from "./usage.ts";
 
 export interface DistinctScopesFilter {
 	source: string;

package/src/{ports → observability}/telemetry-source.ts

@@ -1,5 +1,5 @@
-import type { MetricObservation } from "../domain/metric.ts";
-import type { BudgetWindow } from "../policy.ts";
+import type { BudgetWindow } from "./budget.ts";
+import type { MetricObservation } from "./metric.ts";
 
 export interface TelemetryBatch {
 	observedAt: number;

package/src/observability/token-measurement.ts

@@ -0,0 +1,194 @@
+import { CONTEXT_ESTIMATE_CHARACTERS_PER_TOKEN } from "../constants.ts";
+import { normalizeModelIdentity } from "./model-identity.ts";
+
+export const TOKEN_MEASUREMENT_PROVENANCES = [
+	"provider-reported",
+	"provider-count-api",
+	"tokenizer-exact-text",
+	"structural-estimate",
+] as const;
+export type TokenMeasurementProvenance = (typeof TOKEN_MEASUREMENT_PROVENANCES)[number];
+
+export const TOKEN_MEASUREMENT_SCOPES = [
+	"request-context",
+	"request-input",
+	"response-output",
+	"cache-read",
+	"cache-write",
+	"context-item",
+	"unattributed-residual",
+] as const;
+export type TokenMeasurementScope = (typeof TOKEN_MEASUREMENT_SCOPES)[number];
+
+export interface TokenMeasurement {
+	tokens: number;
+	scope: TokenMeasurementScope;
+	provenance: TokenMeasurementProvenance;
+	/** Bounded implementation/counting identity, never content (for example `char/4` or `gpt-tokenizer:o200k_base`). */
+	method: string;
+	provider?: string;
+	model?: string;
+}
+
+const TOKEN_MEASUREMENT_FIELDS = new Set<keyof TokenMeasurement>(["tokens", "scope", "provenance", "method", "provider", "model"]);
+const TOKEN_MEASUREMENT_METHOD_MAX_CHARACTERS = 120;
+
+function boundedMethod(value: unknown): string {
+	if (typeof value !== "string" || value.length === 0 || value.length > TOKEN_MEASUREMENT_METHOD_MAX_CHARACTERS || /\p{Cc}/u.test(value))
+		throw new Error("token measurement method is invalid");
+	return value;
+}
+
+export function validateTokenMeasurement(value: unknown): TokenMeasurement {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("token measurement must be an object");
+	const input = value as Record<string, unknown>;
+	for (const key of Object.keys(input)) {
+		if (!TOKEN_MEASUREMENT_FIELDS.has(key as keyof TokenMeasurement)) throw new Error(`unsupported field: ${key}`);
+	}
+	if (typeof input.tokens !== "number" || !Number.isSafeInteger(input.tokens) || input.tokens < 0)
+		throw new Error("token measurement tokens are invalid");
+	if (!TOKEN_MEASUREMENT_SCOPES.includes(input.scope as TokenMeasurementScope)) throw new Error("token measurement scope is invalid");
+	if (!TOKEN_MEASUREMENT_PROVENANCES.includes(input.provenance as TokenMeasurementProvenance))
+		throw new Error("token measurement provenance is invalid");
+	if ((input.provider === undefined) !== (input.model === undefined))
+		throw new Error("token measurement provider/model identity is incomplete");
+	if (
+		(input.provider !== undefined && typeof input.provider !== "string") ||
+		(input.model !== undefined && typeof input.model !== "string")
+	)
+		throw new Error("token measurement provider/model identity is invalid");
+	const identity =
+		input.provider === undefined || input.model === undefined ? undefined : normalizeModelIdentity(input.provider, input.model);
+	return {
+		tokens: input.tokens,
+		scope: input.scope as TokenMeasurementScope,
+		provenance: input.provenance as TokenMeasurementProvenance,
+		method: boundedMethod(input.method),
+		...(identity ? { provider: identity.provider, model: identity.model } : {}),
+	};
+}
+
+export interface TextTokenCountInput {
+	/** Ephemeral input. Implementations must not retain or include it in diagnostics/measurements. */
+	text: string;
+	provider?: string;
+	model?: string;
+	scope: "context-item" | "request-input" | "response-output";
+}
+
+/** A replaceable, content-ephemeral text counting boundary. Unsupported provider/model identities return null. */
+export interface TextTokenCounter {
+	countText(input: TextTokenCountInput): TokenMeasurement | null;
+}
+
+export class StructuralTextTokenCounter implements TextTokenCounter {
+	countText(input: TextTokenCountInput): TokenMeasurement {
+		const identity =
+			input.provider === undefined || input.model === undefined ? undefined : normalizeModelIdentity(input.provider, input.model);
+		return {
+			tokens: Math.ceil(input.text.length / CONTEXT_ESTIMATE_CHARACTERS_PER_TOKEN),
+			scope: input.scope,
+			provenance: "structural-estimate",
+			method: `char/${CONTEXT_ESTIMATE_CHARACTERS_PER_TOKEN}`,
+			...(identity ? { provider: identity.provider, model: identity.model } : {}),
+		};
+	}
+}
+
+/** Uses the first supporting model-aware counter and fails open to the explicit structural fallback. */
+export function countTextWithFallback(
+	input: TextTokenCountInput,
+	counters: readonly TextTokenCounter[],
+	fallback: TextTokenCounter = new StructuralTextTokenCounter(),
+): TokenMeasurement {
+	for (const counter of counters) {
+		try {
+			const measurement = counter.countText(input);
+			if (measurement) return validateTokenMeasurement(measurement);
+		} catch {
+			// Counting is observational. Unknown mappings/tokenizer failures must remain visibly estimated, never block the request.
+		}
+	}
+	const measurement = fallback.countText(input);
+	if (!measurement) throw new Error("structural token counter did not return a measurement");
+	return validateTokenMeasurement(measurement);
+}
+
+export interface RequestTokenReconciliation {
+	aggregate: TokenMeasurement;
+	attributedTokens: number;
+	residual: TokenMeasurement;
+	overshootTokens: number;
+}
+
+const REQUEST_RECONCILIATION_FIELDS = new Set<keyof RequestTokenReconciliation>([
+	"aggregate",
+	"attributedTokens",
+	"residual",
+	"overshootTokens",
+]);
+
+export function validateRequestTokenReconciliation(value: unknown): RequestTokenReconciliation {
+	if (typeof value !== "object" || value === null || Array.isArray(value))
+		throw new Error("request token reconciliation must be an object");
+	const input = value as Record<string, unknown>;
+	for (const key of Object.keys(input)) {
+		if (!REQUEST_RECONCILIATION_FIELDS.has(key as keyof RequestTokenReconciliation))
+			throw new Error(`request token reconciliation contains unsupported field: ${key}`);
+	}
+	const aggregate = validateTokenMeasurement(input.aggregate);
+	const residual = validateTokenMeasurement(input.residual);
+	if (aggregate.scope !== "request-context" && aggregate.scope !== "request-input")
+		throw new Error("request token aggregate scope is invalid");
+	if (
+		residual.scope !== "unattributed-residual" ||
+		residual.provenance !== "structural-estimate" ||
+		residual.method !== "aggregate-minus-attributed"
+	)
+		throw new Error("request token residual provenance is invalid");
+	if (residual.provider !== aggregate.provider || residual.model !== aggregate.model)
+		throw new Error("request token residual identity does not match aggregate identity");
+	const attributedTokens = input.attributedTokens;
+	const overshootTokens = input.overshootTokens;
+	if (typeof attributedTokens !== "number" || !Number.isSafeInteger(attributedTokens) || attributedTokens < 0)
+		throw new Error("request token attributed count is invalid");
+	if (typeof overshootTokens !== "number" || !Number.isSafeInteger(overshootTokens) || overshootTokens < 0)
+		throw new Error("request token overshoot count is invalid");
+	if (residual.tokens !== Math.max(0, aggregate.tokens - attributedTokens)) throw new Error("request token residual is inconsistent");
+	if (overshootTokens !== Math.max(0, attributedTokens - aggregate.tokens)) throw new Error("request token overshoot is inconsistent");
+	return { aggregate, attributedTokens, residual, overshootTokens };
+}
+
+/**
+ * Reconciles local item measurements against an authoritative request aggregate without spreading
+ * its remainder across those items. The residual stays explicit and estimate-provenance because
+ * provider envelope/media/rewrites and mixed local counting methods prevent exact attribution.
+ */
+export function reconcileRequestTokens(
+	aggregateValue: TokenMeasurement,
+	attributedValues: readonly TokenMeasurement[],
+): RequestTokenReconciliation {
+	const aggregate = validateTokenMeasurement(aggregateValue);
+	if (aggregate.scope !== "request-context" && aggregate.scope !== "request-input")
+		throw new Error("request token aggregate scope is invalid");
+	const attributed = attributedValues.map(validateTokenMeasurement);
+	if (attributed.some((measurement) => measurement.scope !== "context-item")) throw new Error("request token attribution scope is invalid");
+	const attributedTokens = attributed.reduce((sum, measurement) => {
+		const next = sum + measurement.tokens;
+		if (!Number.isSafeInteger(next)) throw new Error("request token attribution sum is invalid");
+		return next;
+	}, 0);
+	const difference = aggregate.tokens - attributedTokens;
+	return {
+		aggregate,
+		attributedTokens,
+		residual: {
+			tokens: Math.max(0, difference),
+			scope: "unattributed-residual",
+			provenance: "structural-estimate",
+			method: "aggregate-minus-attributed",
+			...(aggregate.provider && aggregate.model ? { provider: aggregate.provider, model: aggregate.model } : {}),
+		},
+		overshootTokens: Math.max(0, -difference),
+	};
+}