dsh-context-compression-improved 0.4.0-beta.1 → 0.5.0

package/packages/selector/src/pruner.ts

@@ -21,7 +21,6 @@ import type {} from '@deepseek-ai/dsh-settings'
 import type {
   CompactionTokenView,
   ObservedPromptUsage,
-  TokenCount,
 } from './runtime/measurement.ts'
 import { measureForCompaction } from './runtime/measurement.ts'
 import { eventBySeq, sessionEvents } from './runtime/session-events.ts'
@@ -40,7 +39,6 @@ import {
   onlyTextBlock,
   onlyTextBlocks,
   countToolContent,
-  exactTokens,
   sameProviderMeasurementKey,
   unavailableCount,
   recoveryMarker,
@@ -76,13 +74,20 @@ import {
   contentDigest,
 } from './runtime/tokenpilot/proposal.ts'
 import {
-  MemoryReviewStore,
   ReviewQueue,
   type ReviewProposalRecord,
-  type ReviewQueueStore,
   type ReviewReceipt,
 } from './runtime/tokenpilot/review-queue.ts'
+import { registerReviewPruner, sharedReviewStore } from './runtime/tokenpilot/review-registry.ts'
 import { openReviewStorage } from './runtime/tokenpilot/review-storage.ts'
+import { SideChannel } from './runtime/tokenpilot/sidechannel.ts'
+import {
+  advisorCandidatePreview,
+  collectTailText,
+  collectTaskSemantics,
+  runSessionAdvisorPass,
+} from './runtime/tokenpilot/advisor.ts'
+import { getAdvisorState } from './runtime/tokenpilot/advisor-state.ts'
 
 import {
   DedupeTable,
@@ -91,6 +96,8 @@ import {
   flattenPlainText,
 } from './runtime/tokenpilot/dedup.ts'
 import {
+  charsForTokens,
+  charsToTokens,
   codePointLength,
   CONTEXT_COMPRESSION_SETTINGS_NAMESPACE,
   ContextCompressionSettingsSchema,
@@ -234,13 +241,23 @@ export class ToolResultPruner extends Service {
       activeRequestBoundaries: new WeakMap(),
       tailTrimBoundaryAttempts: new WeakMap(),
       policyResolutionAudits: new WeakMap(),
-      reviewStore: new MemoryReviewStore(),
+      reviewStore: sharedReviewStore(),
       reviewQueues: new WeakMap(),
       reviewClocks: new WeakMap(),
       estimatorRemainingTurns: new WeakMap(),
+      advisorChannels: new WeakMap(),
       reviewSummaries: new WeakMap(),
     }
 
+    // Publish this instance for the top-level R4 routes. They are registered on
+    // the plugin's top-level fiber while every pruner lives inside an agent
+    // preset's isolated group, so a top-level `ctx.get` can never find one and
+    // the queue route could only answer 503.
+    ctx.effect(
+      () => registerReviewPruner(this),
+      'contextCompressionSelector.reviewRegistry()',
+    )
+
     // TokenPilot-inspired R4: upgrade the review queue to durable storage when
     // the optional storageDomain seam is available; the memory fallback above
     // serves every session until (and unless) that open succeeds.
@@ -325,6 +342,10 @@ export class ToolResultPruner extends Service {
       // TokenPilot-inspired E1: advisory estimator pass, strictly off the
       // synchronous chain. Verdicts only feed the next pressure pass.
       void this.postflightEstimatorPass(agent.session, signal).catch(() => undefined)
+      // Advisory relevance advisor: statistics and suggestions only — its
+      // summaries, scores, and decay figure never touch any decision path.
+      // Strictly fire-and-forget, with its own backoff state.
+      void this.postflightAdvisorPass(agent.session, turn, signal).catch(() => undefined)
     })
   }
 
@@ -380,18 +401,16 @@ export class ToolResultPruner extends Service {
       const planned = eligible
         .map(candidate => this.planNative(candidate, session, stage, policy, view))
         .filter((entry): entry is PlannedReplacement => entry !== null)
-      landed.push(...this.landAll(session, this.triageForReview(session, policy, planned)))
+      landed.push(...this.landAll(session, this.triageForReview(session, policy, planned, 'history')))
       if (landed.length === 0) {
-        const exact = eligible.flatMap(candidate => candidate.count.kind === 'exact-tokenizer'
-          ? [candidate.count.tokens] : [])
+        const chars = eligible.map(candidate => candidate.characterPressure)
         this.auditComponent(session, policy, 'native-tool-result', 'pressure', 'skipped',
-          exactUnavailable ? 'exact-tokenizer-unavailable'
-            : exact.length === 0 ? 'no-tool-result-candidates'
-              : Math.max(...exact) <= policy.nativeTriggerTokens ? 'at-or-below-trigger'
-                : planned.length === 0 ? 'no-valid-reduction'
-                  : 'recovery-tool-unavailable', {
-            measurementKind: exactUnavailable ? 'unavailable' : 'exact-tokenizer',
-            ...(exact.length === 0 ? {} : { currentTokens: Math.max(...exact) }),
+          chars.length === 0 ? 'no-tool-result-candidates'
+            : Math.max(...chars) <= charsForTokens(policy.nativeTriggerTokens) ? 'at-or-below-trigger'
+              : planned.length === 0 ? 'no-valid-reduction'
+                : 'recovery-tool-unavailable', {
+            measurementKind: 'characters',
+            ...(chars.length === 0 ? {} : { currentTokens: charsToTokens(Math.max(...chars)) }),
             triggerTokens: policy.nativeTriggerTokens,
             targetTokens: policy.nativeTargetTokens,
           })
@@ -416,7 +435,7 @@ export class ToolResultPruner extends Service {
           capacityPressure,
         )
         if (historyAllowed) {
-          landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans)))
+          landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans, 'history')))
         }
       }
     } else {
@@ -424,7 +443,7 @@ export class ToolResultPruner extends Service {
       if (historyAllowed) {
         historyOutcome = this.planHistoricalAging(session, policy, view)
         if (historyOutcome.kind === 'planned') {
-          landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans)))
+          landed.push(...this.landAll(session, this.triageForReview(session, policy, historyOutcome.plans, 'history')))
         }
       }
     }
@@ -587,8 +606,7 @@ export class ToolResultPruner extends Service {
     for (const candidate of this.snapshot(session, measureForCompaction(this.ctx, session))) {
       if (samples.length >= 3) break
       if (candidate.event.data.turn === undefined) continue
-      const tokens = exactTokens(candidate.count)
-      if (tokens === undefined || tokens <= policy.freshTriggerTokens) continue
+      if (candidate.characterPressure <= charsForTokens(policy.freshTriggerTokens)) continue
       const path = toolCallPath(candidate.call.arguments)
       if (path === undefined) continue
       if (isSupersededRead(events, candidate.seq, path)) continue
@@ -639,6 +657,106 @@ export class ToolResultPruner extends Service {
     })
   }
 
+  // ─────────── Advisory relevance advisor (statistics & suggestions only) ───────────
+
+  /**
+   * Advisory advisor pass at the turn boundary, strictly fire-and-forget.
+   * Produces todolist-bound tail-task summaries, incremental relevance
+   * scores, and a prefix-decay figure — all observational. Every short
+   * circuit below (mode off, re-entry, cooldown, no task semantics, no
+   * direct endpoint) returns without touching any state the pruning chain
+   * reads, so the default configuration adds exactly zero behavior.
+   */
+  private async postflightAdvisorPass(session: Session, turn: number, signal: AbortSignal): Promise<void> {
+    const policy = this.activePolicy(session)
+    const presetOptions = policy?.presetOptions
+    const advisor = presetOptions?.advisor
+    if (policy === undefined || presetOptions === undefined || advisor === undefined || advisor.mode === '') return
+    const advisorState = getAdvisorState(session)
+    if (advisorState.inFlight) return
+    if (isCoolingDown(advisorState.failures, Date.now())) return
+
+    const events = sessionEvents(session)
+    const task = collectTaskSemantics(events)
+    if (task === undefined) return
+
+    const settings = this.activeSettings(session).presetOptions ?? {}
+    if (advisor.mode === 'direct'
+      && (settings.estimatorBaseUrl === undefined || settings.estimatorBaseUrl.length === 0
+        || settings.estimatorModel === undefined || settings.estimatorModel.length === 0)) {
+      // The advisor reuses the estimator's direct endpoint; when it is not
+      // configured there is nothing to ask, so record the aligned reason and
+      // back off instead of re-emitting the audit at every turn.
+      emitCompressionAudit(this.ctx.logger, {
+        schemaVersion: 1,
+        kind: 'advisor-outcome',
+        sessionId: String(session.id),
+        phase: 'summary',
+        channel: 'direct',
+        ok: false,
+        turnIndex: turn,
+        reason: 'no-direct-endpoint',
+        latencyMs: 0,
+      })
+      advisorState.failures = {
+        failures: (advisorState.failures?.failures ?? 0) + 1,
+        cooldownUntil: Date.now() + backoffCooldownMs((advisorState.failures?.failures ?? 0) + 1),
+      }
+      return
+    }
+
+    let channel = this.state.advisorChannels.get(session)
+    if (channel === undefined) {
+      channel = new SideChannel(this.ctx, settings, {
+        mode: advisor.mode,
+        timeoutMs: advisor.timeoutMs,
+        maxTokens: 512,
+      })
+      this.state.advisorChannels.set(session, channel)
+    }
+
+    const view = measureForCompaction(this.ctx, session)
+    const candidates = this.snapshot(session, view)
+      .filter(candidate => !this.isRecoveryExempt(session, candidate))
+      .map(candidate => ({
+        seq: candidate.seq,
+        characterPressure: candidate.characterPressure,
+        preview: advisorCandidatePreview(candidate.call.name, candidate.event.data.message.content),
+      }))
+
+    let sawFailure = false
+    const outcome = await runSessionAdvisorPass(session, channel, record => {
+      if (record.ok === false) sawFailure = true
+      emitCompressionAudit(this.ctx.logger, record)
+    }, {
+      profile: policy.profile,
+      sessionId: String(session.id),
+      turn,
+      candidates,
+      task: {
+        source: task.source,
+        todoVersion: task.todoVersion,
+        taskText: task.taskText,
+      },
+      advisor: {
+        refreshTurns: advisor.refreshTurns,
+        scoreThreshold: advisor.scoreThreshold,
+        sampleLimit: advisor.sampleLimit,
+        minTokens: advisor.minTokens,
+      },
+      tailText: collectTailText(events),
+      signal,
+    })
+    if (outcome === undefined && sawFailure && signal.aborted === false) {
+      advisorState.failures = {
+        failures: (advisorState.failures?.failures ?? 0) + 1,
+        cooldownUntil: Date.now() + backoffCooldownMs((advisorState.failures?.failures ?? 0) + 1),
+      }
+    } else if (outcome !== undefined) {
+      advisorState.failures = undefined
+    }
+  }
+
   // ─────────── TokenPilot-inspired R4: human-gated review pipeline ───────────
 
   /**
@@ -710,6 +828,7 @@ export class ToolResultPruner extends Service {
     session: Session,
     policy: CompressionPolicy,
     plans: readonly PlannedReplacement[],
+    stage: 'fresh' | 'history' = 'history',
   ): readonly PlannedReplacement[] {
     const queue = this.reviewQueueFor(session, policy)
     if (queue === undefined || plans.length === 0) return plans
@@ -728,6 +847,10 @@ export class ToolResultPruner extends Service {
         ? {}
         : { remainingTurns: this.state.estimatorRemainingTurns.get(session) },
       estimatorSeqs,
+      // Fresh plans shape content before its first request — it is not in the
+      // KV cache yet, so no cache break occurs and the refill penalty would
+      // be a phantom cost pricing every realistic fresh batch into drop.
+      stage,
     }
     const classified = classifyCandidates(plans.map(plan => ({
       sourceSeq: plan.sourceSeq,
@@ -773,7 +896,6 @@ export class ToolResultPruner extends Service {
     try {
       const view = measureForCompaction(this.ctx, session)
       const candidatesBySeq = new Map(this.snapshot(session, view).map(candidate => [candidate.seq, candidate]))
-      const plans: PlannedReplacement[] = []
       const settled: {
         proposal: ReviewProposalRecord
         auditItems?: ReviewProposalRecord['items']
@@ -1404,11 +1526,8 @@ export class ToolResultPruner extends Service {
     const plans = new Map<number, PlannedReplacement>()
     let freshPlanned = 0
     const dedupeEnabled = policy.presetOptions?.dedupeToolResults === true
-    const exactCandidateTokens = candidates.map(candidate => exactTokens(candidate.count))
-    const exactAvailable = exactCandidateTokens.every(tokens => tokens !== undefined)
-    const maxCandidateTokens = exactAvailable
-      ? Math.max(...exactCandidateTokens as number[])
-      : undefined
+    const candidateChars = candidates.map(candidate => candidate.characterPressure)
+    const maxCandidateChars = candidateChars.length === 0 ? undefined : Math.max(...candidateChars)
     if (policy.freshEnabled) {
       if (candidates.some(candidate => candidate.call.name !== 'context_compression_retrieve'
         && candidate.count.kind !== 'exact-tokenizer')) {
@@ -1430,37 +1549,33 @@ export class ToolResultPruner extends Service {
         }
       }
     }
-    let aggregateInputTokens: number | undefined
+    let aggregateInputChars: number | undefined
     let aggregatePlanned = 0
     if (policy.aggregateEnabled) {
-      const aggregateAvailable = exactAvailable
-      if (!aggregateAvailable) this.warnExactUnavailable(session, view, 'aggregate')
-      let total = aggregateAvailable
-        ? candidates.reduce((sum, candidate) => sum + (plans.get(candidate.seq)?.tokensAfter
-          ?? exactTokens(candidate.count) ?? 0), 0)
-        : 0
-      if (aggregateAvailable) aggregateInputTokens = total
-      if (aggregateAvailable && total > policy.aggregateTriggerTokens) {
+      let total = candidates.reduce((sum, candidate) => sum
+        + (plans.get(candidate.seq)?.charsAfter ?? candidate.characterPressure), 0)
+      aggregateInputChars = total
+      if (total > charsForTokens(policy.aggregateTriggerTokens)) {
         const remaining = candidates
           .filter(candidate => !this.isRecoveryExempt(session, candidate))
           .sort((a, b) => Number(isError(a)) - Number(isError(b))
-            || (plans.get(b.seq)?.tokensAfter ?? exactTokens(b.count) ?? 0)
-              - (plans.get(a.seq)?.tokensAfter ?? exactTokens(a.count) ?? 0))
+            || (plans.get(b.seq)?.charsAfter ?? b.characterPressure)
+              - (plans.get(a.seq)?.charsAfter ?? a.characterPressure))
         for (const candidate of remaining) {
           const previous = plans.get(candidate.seq)
           const plan = this.planAggregate(candidate, session, view)
-          const previousTokens = previous?.tokensAfter ?? exactTokens(candidate.count) ?? 0
-          if (plan === null || plan.tokensAfter >= previousTokens) continue
+          const previousChars = previous?.charsAfter ?? candidate.characterPressure
+          if (plan === null || plan.charsAfter >= previousChars) continue
           plans.set(candidate.seq, plan)
           aggregatePlanned += 1
-          total -= previousTokens - plan.tokensAfter
-          if (total <= policy.aggregateTargetTokens) break
+          total -= previousChars - plan.charsAfter
+          if (total <= charsForTokens(policy.aggregateTargetTokens)) break
         }
-        if (total > policy.aggregateTargetTokens) {
+        if (total > charsForTokens(policy.aggregateTargetTokens)) {
           this.ctx.logger.warn(
-            'context-compression fresh aggregate residual: %d tokens exceed target %d',
+            'context-compression fresh aggregate residual: %d characters exceed target %d',
             total,
-            policy.aggregateTargetTokens,
+            charsForTokens(policy.aggregateTargetTokens),
           )
         }
       }
@@ -1469,7 +1584,7 @@ export class ToolResultPruner extends Service {
     const freshCandidates = candidates
       .map(candidate => plans.get(candidate.seq))
       .filter((plan): plan is PlannedReplacement => plan !== undefined)
-    const landed = this.landAll(session, this.triageForReview(session, policy, freshCandidates))
+    const landed = this.landAll(session, this.triageForReview(session, policy, freshCandidates, 'fresh'))
     const freshLanded = landed.some(entry => entry.stage === 'fresh'
       && plans.get(entry.originalSeq)?.component === 'fresh')
     const aggregateLanded = landed.some(entry => entry.stage === 'fresh'
@@ -1478,13 +1593,12 @@ export class ToolResultPruner extends Service {
       this.auditComponent(session, policy, 'fresh', 'fresh',
         policy.freshEnabled ? 'skipped' : 'disabled',
         !policy.freshEnabled ? 'profile-policy'
-          : !exactAvailable ? 'exact-tokenizer-unavailable'
-            : (maxCandidateTokens ?? 0) <= policy.freshTriggerTokens ? 'at-or-below-trigger'
-              : freshPlanned > 0 && aggregatePlanned > 0 ? 'superseded-by-aggregate'
-                : freshPlanned === 0 ? 'no-valid-reduction'
-                  : 'recovery-tool-unavailable', {
-          measurementKind: exactAvailable ? 'exact-tokenizer' : 'unavailable',
-          ...(maxCandidateTokens === undefined ? {} : { currentTokens: maxCandidateTokens }),
+          : (maxCandidateChars ?? 0) <= charsForTokens(policy.freshTriggerTokens) ? 'at-or-below-trigger'
+            : freshPlanned > 0 && aggregatePlanned > 0 ? 'superseded-by-aggregate'
+              : freshPlanned === 0 ? 'no-valid-reduction'
+                : 'recovery-tool-unavailable', {
+          measurementKind: 'characters',
+          ...(maxCandidateChars === undefined ? {} : { currentTokens: charsToTokens(maxCandidateChars) }),
           triggerTokens: policy.freshTriggerTokens,
           targetTokens: policy.freshTargetTokens,
         })
@@ -1493,12 +1607,11 @@ export class ToolResultPruner extends Service {
       this.auditComponent(session, policy, 'aggregate', 'fresh',
         policy.aggregateEnabled ? 'skipped' : 'disabled',
         !policy.aggregateEnabled ? 'profile-policy'
-          : !exactAvailable ? 'exact-tokenizer-unavailable'
-            : (aggregateInputTokens ?? 0) <= policy.aggregateTriggerTokens ? 'at-or-below-trigger'
-              : aggregatePlanned === 0 ? 'no-valid-reduction'
-                : 'recovery-tool-unavailable', {
-          measurementKind: exactAvailable ? 'exact-tokenizer' : 'unavailable',
-          ...(aggregateInputTokens === undefined ? {} : { currentTokens: aggregateInputTokens }),
+          : (aggregateInputChars ?? 0) <= charsForTokens(policy.aggregateTriggerTokens) ? 'at-or-below-trigger'
+            : aggregatePlanned === 0 ? 'no-valid-reduction'
+              : 'recovery-tool-unavailable', {
+          measurementKind: 'characters',
+          ...(aggregateInputChars === undefined ? {} : { currentTokens: charsToTokens(aggregateInputChars) }),
           triggerTokens: policy.aggregateTriggerTokens,
           targetTokens: policy.aggregateTargetTokens,
         })
@@ -1548,16 +1661,17 @@ export class ToolResultPruner extends Service {
     view: CompactionTokenView,
   ): PlannedReplacement | null {
     if (this.isRecoveryExempt(session, candidate)) return null
-    const tokensBefore = exactTokens(candidate.count)
-    if (tokensBefore === undefined || tokensBefore <= policy.nativeTriggerTokens) return null
+    if (candidate.characterPressure <= charsForTokens(policy.nativeTriggerTokens)) return null
     const result = candidate.event.data.message.content[0]
     if (onlyTextBlocks(result.content) === null) return null
     const sourceSeq = rootToolResultSeq(session, candidate.seq)
-    const marker = recoveryMarker(sourceRefFn(session, sourceSeq), 'tool result middle pruned')
+    // R9b site: the marker's retrieve hint starts at the event line right
+    // after the retained head, computed inside nativePruneContent.
+    const marker = (startLine: number): string => recoveryMarker(sourceRefFn(session, sourceSeq), 'tool result middle pruned', startLine)
     let head = this.state.config.headChars
     let tail = this.state.config.tailChars
     for (let attempt = 0; attempt < 10; attempt += 1) {
-      const threshold = head + codePointLength(marker) + tail
+      const threshold = head + codePointLength(marker(1)) + tail
       const content = nativePruneContent(result.content, threshold, head, tail, marker)
       if (content !== null) {
         const plan = this.plan(
@@ -1603,8 +1717,7 @@ export class ToolResultPruner extends Service {
     const result = candidate.event.data.message.content[0]
     const text = flattenPlainText(result.content)
     if (text === undefined) return null
-    const tokensBefore = exactTokens(candidate.count)
-    if (tokensBefore === undefined || tokensBefore <= policy.freshTriggerTokens) return null
+    if (candidate.characterPressure <= charsForTokens(policy.freshTriggerTokens)) return null
     let table = this.state.dedupeTables.get(session)
     if (table === undefined) {
       table = new DedupeTable()
@@ -1650,8 +1763,7 @@ export class ToolResultPruner extends Service {
     // being reconsidered after their first request.
     if (typeof candidate.event.surfaceOp === 'object') return null
     const result = candidate.event.data.message.content[0]
-    const tokensBefore = exactTokens(candidate.count)
-    if (tokensBefore === undefined || tokensBefore <= policy.freshTriggerTokens) return null
+    if (candidate.characterPressure <= charsForTokens(policy.freshTriggerTokens)) return null
     const sourceSeq = candidate.seq
     const sourceRef = sourceRefFn(session, sourceSeq)
     const textBlock = onlyTextBlock(result.content)
@@ -1678,9 +1790,12 @@ export class ToolResultPruner extends Service {
             'fresh',
             undefined,
             view,
-            { noNetSavingsGuard: policy.presetOptions?.noNetSavingsGuard === true },
+            {
+              noNetSavingsGuard: policy.presetOptions?.noNetSavingsGuard === true,
+              ...(output.elidedLines === undefined ? {} : { elidedLines: output.elidedLines }),
+            },
           )
-          if (plan !== null && plan.tokensAfter <= policy.freshTargetTokens) return plan
+          if (plan !== null && plan.charsAfter <= charsForTokens(policy.freshTargetTokens)) return plan
         }
         if (budgetChars === 1) break
         budgetChars = Math.max(1, Math.floor(budgetChars / 2))
@@ -1721,6 +1836,12 @@ export class ToolResultPruner extends Service {
     }
     const sourceSeq = rootToolResultSeq(session, candidate.seq)
     const sourceRef = sourceRefFn(session, sourceSeq)
+    // The placeholder below is a single text block, so a rich tool result (for
+    // example one carrying an image) must never reach it. The character basis
+    // no longer inherits the exact-tokenizer precondition that used to reject
+    // this path implicitly, so the guard has to be explicit.
+    const redacted = candidate.event.data.message.content[0]
+    if (onlyTextBlocks(redacted.content) === null) return null
     const text = [
       '[Tool result reduced to satisfy the completed-step aggregate budget]',
       `tool: ${candidate.call.name}`,
@@ -1798,17 +1919,9 @@ export class ToolResultPruner extends Service {
   ): HistoryPlanOutcome {
     const candidates = this.snapshot(session, view)
     const events = sessionEvents(session)
-    const exact: number[] = []
-    for (const candidate of candidates) {
-      const tokens = exactTokens(candidate.count)
-      if (tokens === undefined) {
-        this.warnExactUnavailable(session, view, 'history')
-        return { kind: 'exact-tokenizer-unavailable' }
-      }
-      exact.push(tokens)
-    }
-    const total = exact.reduce((sum, tokens) => sum + tokens, 0)
-    const trigger = policy.historyTriggerTokens
+    const chars = candidates.map(candidate => candidate.characterPressure)
+    const total = chars.reduce((sum, charsOfNode) => sum + charsOfNode, 0)
+    const trigger = charsForTokens(policy.historyTriggerTokens)
     // Full-request last chance: ordinary prose, images, prompts, or schemas can
     // push the complete request past the Auto Compact deadline before the tool
     // results alone cross the profile trigger.
@@ -1835,19 +1948,18 @@ export class ToolResultPruner extends Service {
     }
     const planned: PlannedReplacement[] = []
     let reclaim = 0
-    // With a frozen Auto Compact deadline, microTarget = max(0, D - M) and one
-    // batch must justify its cache break by pulling the complete request back
-    // below the deadline. Without linkage (Custom manual, or no resolved
-    // routed capacity) the loop keeps its traditional target and the batch
-    // still commits once it reaches the minimum reclaim.
-    const microTarget = deadline === undefined ? undefined : Math.max(0, deadline - policy.historyMinReclaimTokens)
+    // Linked batches must reach the deadline target; unlinked batches keep
+    // the traditional minimum-reclaim commit threshold. All arithmetic runs on
+    // the character basis: token-named thresholds enter via charsForTokens.
+    const minReclaimChars = charsForTokens(policy.historyMinReclaimTokens)
+    const microTarget = deadline === undefined ? undefined : Math.max(0, charsForTokens(deadline) - minReclaimChars)
     const required = Math.max(
-      policy.historyMinReclaimTokens,
+      minReclaimChars,
       total - trigger,
-      ...(microTarget === undefined ? [] : [view.totalTokens - microTarget]),
+      ...(microTarget === undefined ? [] : [charsForTokens(view.totalTokens) - microTarget]),
     )
     const batchTarget = microTarget === undefined
-      ? policy.historyMinReclaimTokens
+      ? minReclaimChars
       : required
     for (const candidate of eligible) {
       const result = candidate.event.data.message.content[0]
@@ -1872,7 +1984,7 @@ export class ToolResultPruner extends Service {
           )
           if (plan === null) continue
           planned.push(plan)
-          reclaim += plan.tokensBefore - plan.tokensAfter
+          reclaim += plan.charsBefore - plan.charsAfter
           if (reclaim >= required) break
           continue
         }
@@ -1891,7 +2003,7 @@ export class ToolResultPruner extends Service {
         )
         if (plan === null) continue
         planned.push(plan)
-        reclaim += plan.tokensBefore - plan.tokensAfter
+        reclaim += plan.charsBefore - plan.charsAfter
         if (reclaim >= required) break
         continue
       }
@@ -1931,10 +2043,11 @@ export class ToolResultPruner extends Service {
         'history',
         policy.historyMode,
         view,
+        { ...(output.elidedLines === undefined ? {} : { elidedLines: output.elidedLines }) },
       )
       if (plan === null) continue
       planned.push(plan)
-      reclaim += plan.tokensBefore - plan.tokensAfter
+      reclaim += plan.charsBefore - plan.charsAfter
       if (reclaim >= required) break
     }
     // Linked batches must reach the deadline target; unlinked batches keep
@@ -1949,9 +2062,8 @@ export class ToolResultPruner extends Service {
     session: Session,
     policy: CompressionPolicy,
     view: CompactionTokenView,
-  ): Set<number> | null {
+  ): Set<number> {
     const candidates = this.snapshot(session, view)
-    if (candidates.some(candidate => exactTokens(candidate.count) === undefined)) return null
     return this.protectedHistoryCandidateSeqs(candidates, policy)
   }
 
@@ -1967,14 +2079,14 @@ export class ToolResultPruner extends Service {
       const candidate = candidates[index]
       if (candidate !== undefined) protectedSeqs.add(candidate.seq)
     }
-    let recentTokens = 0
+    let recentChars = 0
     for (let index = candidates.length - 1;
-      index >= 0 && recentTokens < policy.historyKeepRecentTokens;
+      index >= 0 && recentChars < charsForTokens(policy.historyKeepRecentTokens);
       index--) {
       const candidate = candidates[index]
       if (candidate === undefined) continue
       protectedSeqs.add(candidate.seq)
-      recentTokens += exactTokens(candidate.count) ?? 0
+      recentChars += candidate.characterPressure
     }
     return protectedSeqs
   }
@@ -1988,26 +2100,22 @@ export class ToolResultPruner extends Service {
     const tailTrim = policy.tailTrim
     if (tailTrim?.enabled !== true) return
     const events = sessionEvents(session)
-    if (view.currentSurface.kind !== 'exact-tokenizer'
-      || view.currentSurface.tokens <= tailTrim.triggerTokens) {
-      if (view.currentSurface.kind !== 'exact-tokenizer') this.warnExactUnavailable(session, view, 'tailtrim')
+    if (view.currentSurfaceChars <= charsForTokens(tailTrim.triggerTokens)) {
       this.auditComponent(session, policy, 'tail-trim', 'pressure', 'skipped',
-        view.currentSurface.kind !== 'exact-tokenizer'
-          ? 'exact-tokenizer-unavailable' : 'at-or-below-trigger', {
-          measurementKind: view.currentSurface.kind,
-          ...(view.currentSurface.kind === 'exact-tokenizer'
-            ? { currentTokens: view.currentSurface.tokens }
-            : {}),
+        'at-or-below-trigger', {
+          measurementKind: 'characters',
+          currentTokens: charsToTokens(view.currentSurfaceChars),
           triggerTokens: tailTrim.triggerTokens,
         })
       return
     }
     const surfaceCount = view.currentSurface
+    const exactSurface = surfaceCount.kind === 'exact-tokenizer' ? surfaceCount : undefined
     if (!this.hasRecoveryTool(session)) {
       this.auditComponent(session, policy, 'tail-trim', 'pressure', 'skipped',
         'recovery-tool-unavailable', {
-          measurementKind: 'exact-tokenizer',
-          currentTokens: surfaceCount.tokens,
+          measurementKind: 'characters',
+          currentTokens: charsToTokens(view.currentSurfaceChars),
           triggerTokens: tailTrim.triggerTokens,
         })
       return
@@ -2015,23 +2123,15 @@ export class ToolResultPruner extends Service {
     if (!hasOpenTurn(session)) {
       this.auditComponent(session, policy, 'tail-trim', 'pressure', 'skipped',
         'no-open-turn', {
-          measurementKind: 'exact-tokenizer',
-          currentTokens: surfaceCount.tokens,
+          measurementKind: 'characters',
+          currentTokens: charsToTokens(view.currentSurfaceChars),
           triggerTokens: tailTrim.triggerTokens,
         })
       return
     }
     const protectedResults = this.protectedHistoryResultSeqs(session, policy, view)
-    if (protectedResults === null) {
-      this.auditComponent(session, policy, 'tail-trim', 'pressure', 'skipped',
-        'exact-tokenizer-unavailable-in-protected-set', {
-          measurementKind: 'unavailable',
-          currentTokens: surfaceCount.tokens,
-          triggerTokens: tailTrim.triggerTokens,
-        })
-      return
-    }
     const measured = new Map(view.measuredNodes.map(node => [node.seq, node.count]))
+    const nodeChars = new Map(view.measuredNodes.map(node => [node.seq, node.characterPressure]))
     const heuristic = new Map(view.nodes.map(node => [node.seq, node.tokens]))
     const completedTurns = new Set<number>()
     const completedSteps = new Set<string>()
@@ -2090,34 +2190,58 @@ export class ToolResultPruner extends Service {
       if (roots.some(root => root === null)) continue
       const sourceEventSeqs = roots as number[]
       if (new Set(sourceEventSeqs).size !== sourceEventSeqs.length) continue
-      const counts = shadowedSeqs.map(seq => measured.get(seq))
-      if (counts.some(count => count?.kind !== 'exact-tokenizer')) continue
-      const exactCounts = counts as Extract<TokenCount, { kind: 'exact-tokenizer' }>[]
-      if (exactCounts.some(count => count.tokenizerId !== surfaceCount.tokenizerId
-        || count.tokenizerRevision !== surfaceCount.tokenizerRevision)) continue
-      const tokensBefore = exactCounts.reduce((sum, count) => sum + count.tokens, 0)
+      // Exact tokens stay telemetry-only: they are recorded when every
+      // shadowed node shares the surface tokenizer identity, and derived from
+      // characters otherwise. The skip decisions above and below are all
+      // character-based.
+      let exactTokensBefore: number | undefined
+      if (exactSurface !== undefined) {
+        let sum = 0
+        let allExact = true
+        for (const seq of shadowedSeqs) {
+          const count = measured.get(seq)
+          if (count?.kind !== 'exact-tokenizer'
+            || count.tokenizerId !== exactSurface.tokenizerId
+            || count.tokenizerRevision !== exactSurface.tokenizerRevision) {
+            allExact = false
+            break
+          }
+          sum += count.tokens
+        }
+        if (allExact) exactTokensBefore = sum
+      }
+      const charsBefore = shadowedSeqs.reduce((sum, seq) => sum + (nodeChars.get(seq) ?? 0), 0)
       const manifestSeq = events.length
       const ref = tailTrimRef(String(session.id), manifestSeq)
       const stub = tailTrimStub(ref, calls.map(call => call.name), sourceEventSeqs)
       if (stub === null) continue
-      const stubCount = countExactCanonicalTextFields(
-        [stub],
-        candidate => view.countCanonicalText(candidate),
-        'TailTrim group stub',
-      )
-      if (stubCount.kind !== 'exact-tokenizer'
-        || stubCount.tokenizerId !== surfaceCount.tokenizerId
-        || stubCount.tokenizerRevision !== surfaceCount.tokenizerRevision
-        || stubCount.tokens <= 0
-        || tokensBefore - stubCount.tokens < policy.historyMinReclaimTokens) continue
+      const stubChars = codePointLength(stub)
+      if (stubChars <= 0
+        || charsBefore - stubChars < charsForTokens(policy.historyMinReclaimTokens)) continue
+      let exactTokensAfter: number | undefined
+      if (exactTokensBefore !== undefined && exactSurface !== undefined) {
+        const stubCount = countExactCanonicalTextFields(
+          [stub],
+          candidate => view.countCanonicalText(candidate),
+          'TailTrim group stub',
+        )
+        if (stubCount.kind === 'exact-tokenizer'
+          && stubCount.tokenizerId === exactSurface.tokenizerId
+          && stubCount.tokenizerRevision === exactSurface.tokenizerRevision) {
+          exactTokensAfter = stubCount.tokens
+        }
+      }
+      const exact = exactTokensBefore !== undefined && exactTokensAfter !== undefined
+      const tokensBefore = exactTokensBefore ?? charsToTokens(charsBefore)
+      const tokensAfter = exactTokensAfter ?? charsToTokens(stubChars)
       const heuristicTokens = shadowedSeqs.reduce((sum, seq) => sum + (heuristic.get(seq) ?? 0), 0)
       const range = { start: SessionSeq(assistantSeq), end: SessionSeq(resultSeqs.at(-1) ?? assistantSeq) }
       const surfaceRange = { op: 'replace' as const, startSeq: range.start, endSeq: range.end }
       if (!this.reserveTailTrimBoundaryAttempt(session)) {
         this.auditComponent(session, policy, 'tail-trim', 'pressure', 'skipped',
           'already-attempted-at-request-boundary', {
-            measurementKind: 'exact-tokenizer',
-            currentTokens: surfaceCount.tokens,
+            measurementKind: 'characters',
+            currentTokens: charsToTokens(view.currentSurfaceChars),
             triggerTokens: tailTrim.triggerTokens,
           })
         return
@@ -2156,17 +2280,22 @@ export class ToolResultPruner extends Service {
         replacementSeq: replacement.seq,
         sourceSeqs: sourceEventSeqs,
         tokensBefore,
-        tokensAfter: stubCount.tokens,
-        tokensRemoved: tokensBefore - stubCount.tokens,
-        tokenizerId: stubCount.tokenizerId,
-        tokenizerRevision: stubCount.tokenizerRevision,
+        tokensAfter,
+        tokensRemoved: tokensBefore - tokensAfter,
+        measurementBasis: exact ? 'exact-tokenizer' : 'characters',
+        tokenizerId: exact === true && exactSurface !== undefined
+          ? exactSurface.tokenizerId
+          : 'characters',
+        tokenizerRevision: exact === true && exactSurface !== undefined
+          ? exactSurface.tokenizerRevision
+          : 'chars-per-token-4.0',
       })
       return
     }
     this.auditComponent(session, policy, 'tail-trim', 'pressure', 'skipped',
       'no-safe-eligible-tool-group', {
-        measurementKind: 'exact-tokenizer',
-        currentTokens: surfaceCount.tokens,
+        measurementKind: 'characters',
+        currentTokens: charsToTokens(view.currentSurfaceChars),
         triggerTokens: tailTrim.triggerTokens,
       })
   }
@@ -2211,17 +2340,23 @@ export class ToolResultPruner extends Service {
     component: CompressionAuditComponent,
     historyMode: HistoryMode | undefined,
     view: CompactionTokenView,
-    options: { readonly noNetSavingsGuard?: boolean } = {},
+    options: { readonly noNetSavingsGuard?: boolean, readonly elidedLines?: number } = {},
   ): PlannedReplacement | null {
+    const charsBefore = candidate.characterPressure
+    const charsAfter = pressureCost(content)
+    // Character proof replaces the exact-tokenizer precondition: a reduction
+    // must shrink the decision surface regardless of the routed model id.
+    if (charsAfter <= 0 || charsAfter >= charsBefore) return null
+    // Tokens are telemetry only: exact when both sides share one bundled
+    // tokenizer identity, otherwise derived from the character measurement.
     const countBefore = candidate.count
-    if (countBefore.kind !== 'exact-tokenizer') return null
     const countAfter = countToolContent(content, view)
-    if (countAfter.kind !== 'exact-tokenizer'
-      || countAfter.tokenizerId !== countBefore.tokenizerId
-      || countAfter.tokenizerRevision !== countBefore.tokenizerRevision) return null
-    const tokensBefore = countBefore.tokens
-    const tokensAfter = countAfter.tokens
-    if (tokensAfter <= 0 || tokensAfter >= tokensBefore) return null
+    const exact = countBefore.kind === 'exact-tokenizer'
+      && countAfter.kind === 'exact-tokenizer'
+      && countAfter.tokenizerId === countBefore.tokenizerId
+      && countAfter.tokenizerRevision === countBefore.tokenizerRevision
+    const tokensBefore = exact ? countBefore.tokens : charsToTokens(charsBefore)
+    const tokensAfter = exact ? countAfter.tokens : charsToTokens(charsAfter)
     // TokenPilot-style no-net-savings: even when the exact tokenizer reports a
     // saving, a replacement whose text is not smaller than its original adds
     // noise without reclaiming context. Text-level because the placeholder
@@ -2235,8 +2370,6 @@ export class ToolResultPruner extends Service {
         if (replacementChars >= originalChars) return null
       }
     }
-    const charsBefore = candidate.characterPressure
-    const charsAfter = pressureCost(content)
     return {
       candidate,
       content,
@@ -2249,8 +2382,10 @@ export class ToolResultPruner extends Service {
       charsAfter,
       tokensBefore,
       tokensAfter,
-      tokenizerId: countBefore.tokenizerId,
-      tokenizerRevision: countBefore.tokenizerRevision,
+      measurementBasis: exact ? 'exact-tokenizer' : 'characters',
+      tokenizerId: exact ? countBefore.tokenizerId : 'characters',
+      tokenizerRevision: exact ? countBefore.tokenizerRevision : 'chars-per-token-4.0',
+      ...options.elidedLines === undefined ? {} : { elidedLines: options.elidedLines },
     }
   }
 
@@ -2301,8 +2436,12 @@ export class ToolResultPruner extends Service {
       tokensBefore: plan.tokensBefore,
       tokensAfter: plan.tokensAfter,
       tokensRemoved: plan.tokensBefore - plan.tokensAfter,
+      measurementBasis: plan.measurementBasis,
       tokenizerId: plan.tokenizerId,
       tokenizerRevision: plan.tokenizerRevision,
+      // task_4c/G7 telemetry: original-event lines the reducer elided. Audit
+      // record ONLY — the replacement content is untouched by this field.
+      ...plan.elidedLines === undefined ? {} : { elidedLines: plan.elidedLines },
     })
     // R4: the four-state summary counts automatic-path rewrites at the single
     // landing chokepoint; the review-approved batch settles its own counters
@@ -2380,7 +2519,7 @@ export class ToolResultPruner extends Service {
         policy.historyMode === 'capacity-pressure'
           ? 'below-micro-deadline' : 'adaptive-cost-rejected', {
           historyMode: policy.historyMode,
-          measurementKind: view.currentSurface.kind,
+          measurementKind: 'characters',
           currentTokens: view.totalTokens,
           ...(capacityTrigger === undefined ? {} : { triggerTokens: capacityTrigger }),
         })
@@ -2390,14 +2529,14 @@ export class ToolResultPruner extends Service {
     const lastChance = deadline !== undefined && view.totalTokens >= deadline
     const detail = (extra: Readonly<Record<string, number>> = {}): Readonly<{
       historyMode?: HistoryMode
-      measurementKind?: 'exact-tokenizer' | 'tokenizer-estimate' | 'unavailable'
+      measurementKind?: 'exact-tokenizer' | 'tokenizer-estimate' | 'characters' | 'unavailable'
       currentTokens?: number
       triggerTokens?: number
       reclaimTokens?: number
       requiredTokens?: number
     }> => ({
       historyMode: policy.historyMode,
-      measurementKind: outcome.kind === 'exact-tokenizer-unavailable' ? 'unavailable' : 'exact-tokenizer',
+      measurementKind: 'characters',
       currentTokens: view.totalTokens,
       ...(outcome.kind === 'insufficient-reclaim' || outcome.kind === 'cannot-reach-deadline-target'
         ? { reclaimTokens: outcome.reclaim, requiredTokens: outcome.required }
@@ -2405,10 +2544,6 @@ export class ToolResultPruner extends Service {
       ...extra,
     })
     switch (outcome.kind) {
-      case 'exact-tokenizer-unavailable':
-        this.auditComponent(session, policy, 'history', 'pressure', 'skipped',
-          'exact-tokenizer-unavailable', detail({ triggerTokens: policy.historyTriggerTokens }))
-        return
       case 'below-profile-trigger':
         this.auditComponent(session, policy, 'history', 'pressure', 'skipped',
           'below-profile-trigger', detail({ triggerTokens: policy.historyTriggerTokens }))
@@ -2454,7 +2589,7 @@ export class ToolResultPruner extends Service {
     reason: string,
     detail: Readonly<{
       historyMode?: HistoryMode
-      measurementKind?: 'exact-tokenizer' | 'tokenizer-estimate' | 'unavailable'
+      measurementKind?: 'exact-tokenizer' | 'tokenizer-estimate' | 'characters' | 'unavailable'
       currentTokens?: number
       triggerTokens?: number
       targetTokens?: number
@@ -2567,7 +2702,7 @@ export class ToolResultPruner extends Service {
     this.warnOnce(
       session,
       `exact-tokenizer:${gate}:${provider}\0${model}`,
-      'context-compression %s kept original tool results because exact tokenizer counts are unavailable for %s/%s',
+      'context-compression %s is measuring on the character basis: exact tokenizer counts are unavailable for %s/%s',
       gate,
       provider,
       model,