dsh-context-compression-improved 0.4.0-beta.1 → 0.5.0

package/packages/selector/src/runtime/adaptive-cost.ts

@@ -5,6 +5,7 @@ import type {
   TokenCount,
 } from './measurement.ts'
 import { decimalRateNanoUnits } from './deepseek-official-pricing.ts'
+import { charsToTokens } from './config.ts'
 
 /** Inputs that conservatively bound one already-planned Adaptive History batch. */
 export interface AdaptiveTokenBoundsInput {
@@ -25,7 +26,7 @@ export interface AdaptiveTokenBoundsInput {
 /** Available lower-benefit and upper-risk bounds for one History batch. */
 export interface AvailableAdaptiveTokenBounds {
   readonly kind: 'available'
-  readonly measurementKind: 'exact-tokenizer' | 'tokenizer-estimate'
+  readonly measurementKind: 'exact-tokenizer' | 'tokenizer-estimate' | 'characters'
   /** D: conservative lower bound of input tokens removed by the plan. */
   readonly reclaimedLowerBoundTokens: number
   /** A: conservative upper bound of retained suffix tokens that may lose a hit. */
@@ -89,24 +90,34 @@ export function deriveAdaptiveTokenBounds(input: AdaptiveTokenBoundsInput): Adap
 
   let identity: { readonly tokenizerId: string; readonly tokenizerRevision: string } | undefined
   let exactPrefixLowerBoundTokens = 0
+  let characterDerivedPrefix = false
   const seen = new Set<number>()
   for (const node of input.measuredNodes) {
     if (!isCount(node.seq) || seen.has(node.seq)) return unknown('invalid-measured-node-sequence')
     seen.add(node.seq)
-    if (node.seq >= input.earliestChangedSeq || node.count.kind !== 'exact-tokenizer') continue
-    if (!isCount(node.count.tokens)) return unknown('invalid-exact-prefix-count')
-    if (node.count.tokenizerRevision !== input.expectedTokenizerRevision) {
-      return unknown('exact-prefix-tokenizer-revision-mismatch')
+    if (node.seq >= input.earliestChangedSeq) continue
+    if (node.count.kind === 'exact-tokenizer') {
+      if (!isCount(node.count.tokens)) return unknown('invalid-exact-prefix-count')
+      if (node.count.tokenizerRevision !== input.expectedTokenizerRevision) {
+        return unknown('exact-prefix-tokenizer-revision-mismatch')
+      }
+      if (identity !== undefined
+        && (identity.tokenizerId !== node.count.tokenizerId
+          || identity.tokenizerRevision !== node.count.tokenizerRevision)) {
+        return unknown('exact-prefix-tokenizer-identity-mismatch')
+      }
+      identity ??= node.count
+      exactPrefixLowerBoundTokens += node.count.tokens
+    } else {
+      // Character-basis degrade: a prefix node without an exact count still
+      // occupies retained-request space, so charge its conservative
+      // character→token derivation instead of skipping it silently.
+      exactPrefixLowerBoundTokens += charsToTokens(node.characterPressure)
+      characterDerivedPrefix = true
     }
-    if (identity !== undefined
-      && (identity.tokenizerId !== node.count.tokenizerId
-        || identity.tokenizerRevision !== node.count.tokenizerRevision)) {
-      return unknown('exact-prefix-tokenizer-identity-mismatch')
-    }
-    identity ??= node.count
-    exactPrefixLowerBoundTokens += node.count.tokens
     if (!isCount(exactPrefixLowerBoundTokens)) return unknown('exact-prefix-overflow')
   }
+  if (characterDerivedPrefix) measurementKind = 'characters'
 
   const accounted = exactPrefixLowerBoundTokens + reclaimedLowerBoundTokens
   if (!isCount(accounted) || accounted > input.previousPromptTokens) {

package/packages/selector/src/runtime/audit.ts

@@ -79,7 +79,7 @@ export interface CompressionPolicyResolvedAuditRecord extends CompressionAuditBa
   readonly tokenizer?: CompressionTokenizerAuditFact
 }
 
-/** One committed model-free surface rewrite and its exact token accounting. */
+/** One committed model-free surface rewrite and its accounting. */
 export interface CompressionRewriteAuditRecord extends CompressionAuditBase {
   readonly kind: 'rewrite'
   readonly profile: CompressionProfile
@@ -94,8 +94,24 @@ export interface CompressionRewriteAuditRecord extends CompressionAuditBase {
   readonly tokensBefore: number
   readonly tokensAfter: number
   readonly tokensRemoved: number
+  /**
+   * Which measurement backed this record's decision and token figures:
+   * 'exact-tokenizer' when both sides of the reduction carry one bundled
+   * tokenizer identity, 'characters' when the proof ran on Unicode code
+   * points and `tokensBefore/After` were derived at 4.0 chars per token
+   * (`tokenizerId: 'characters'`, `tokenizerRevision: 'chars-per-token-4.0'`).
+   */
+  readonly measurementBasis: 'exact-tokenizer' | 'characters'
   readonly tokenizerId: string
   readonly tokenizerRevision: string
+  /**
+   * Original-event lines the reducer elided (task_4c/G7, from
+   * `ReducerOutput.elidedLines` via the plan). Present when the landing
+   * reducer reports it; absent for placeholder/trim rewrites. Exists so the
+   * compress→retrieve M/N ratio is computable from session logs alone —
+   * never printed into replacement content.
+   */
+  readonly elidedLines?: number
 }
 
 /** Why an enabled component did not rewrite, or why it was disabled. */
@@ -107,7 +123,7 @@ export interface CompressionComponentEvaluationAuditRecord extends CompressionAu
   readonly status: CompressionAuditEvaluationStatus
   readonly reason: string
   readonly historyMode?: HistoryMode
-  readonly measurementKind?: 'exact-tokenizer' | 'tokenizer-estimate' | 'unavailable'
+  readonly measurementKind?: 'exact-tokenizer' | 'tokenizer-estimate' | 'characters' | 'unavailable'
   readonly currentTokens?: number
   readonly triggerTokens?: number
   readonly targetTokens?: number
@@ -167,6 +183,27 @@ export interface EstimatorOutcomeAuditRecord extends CompressionAuditBase {
   readonly ok: boolean
 }
 
+/** One background relevance-advisor pass. Only numeric metadata — never prompts, keys, or content. */
+export interface AdvisorOutcomeAuditRecord extends CompressionAuditBase {
+  readonly kind: 'advisor-outcome'
+  /** Which advisory stage produced this record. */
+  readonly phase: 'summary' | 'scoring' | 'decay'
+  /** LLM channel; omitted for the locally computed 'decay' phase. */
+  readonly channel?: 'host' | 'direct'
+  readonly ok: boolean
+  /** Whether the pass found candidates to sample (scoring phase). */
+  readonly sampledCount?: number
+  /** Prefix-decay figure in [0, 1] (decay phase; also emitted by summary/scoring on success). */
+  readonly decay?: number
+  /** Weighted surface the decay was computed over, in Unicode code points. */
+  readonly weightedChars?: number
+  /** Turn index the pass ran at. */
+  readonly turnIndex?: number
+  /** Aligned failure/skip reason code (e.g. 'no-direct-endpoint', 'parse-failed'). */
+  readonly reason?: string
+  readonly latencyMs: number
+}
+
 /** Lifecycle of one human-gated review proposal. Only numeric and enum fields — never content. */
 export interface ReviewOutcomeAuditRecord extends CompressionAuditBase {
   readonly kind: 'review-outcome'
@@ -203,6 +240,7 @@ export type CompressionAuditRecord =
   | NativeAutoCompactAuditRecord
   | SummaryLocatorAuditRecord
   | EstimatorOutcomeAuditRecord
+  | AdvisorOutcomeAuditRecord
   | ReviewOutcomeAuditRecord
 
 /** Minimal logger method consumed by the audit publisher. */

package/packages/selector/src/runtime/config.ts

@@ -128,6 +128,8 @@ export function parsePresetOptionsSettings(value: unknown): PresetOptionsSetting
     'dedupeToolResults', 'summaryLocator', 'prefixStabilizer', 'readState', 'estimatorMode',
     'estimatorProvider', 'estimatorModel', 'estimatorBaseUrl', 'estimatorApiKey', 'estimatorTimeoutMs',
     'reviewMode', 'reviewTimeoutTurns', 'cacheHitDiscountAlpha', 'reviewHighImpactTokens',
+    'advisorMode', 'advisorTimeoutMs', 'advisorRefreshTurns', 'advisorScoreThreshold', 'advisorSampleLimit',
+    'advisorMinTokens',
   ])
   const unknown = Object.keys(value).find(key => !allowed.has(key))
   if (unknown !== undefined) {
@@ -144,6 +146,39 @@ export function parsePresetOptionsSettings(value: unknown): PresetOptionsSetting
   if (estimatorMode !== undefined && estimatorMode !== '' && estimatorMode !== 'host' && estimatorMode !== 'direct') {
     throw new TypeError('Context-compression presetOptions.estimatorMode must be "", "host", or "direct"')
   }
+  const advisorMode = value.advisorMode
+  if (advisorMode !== undefined && advisorMode !== '' && advisorMode !== 'host' && advisorMode !== 'direct') {
+    throw new TypeError('Context-compression presetOptions.advisorMode must be "", "host", or "direct"')
+  }
+  const advisorTimeoutMs = value.advisorTimeoutMs
+  if (advisorTimeoutMs !== undefined
+    && (typeof advisorTimeoutMs !== 'number' || !Number.isSafeInteger(advisorTimeoutMs)
+      || advisorTimeoutMs < 100 || advisorTimeoutMs > 60_000)) {
+    throw new TypeError('Context-compression presetOptions.advisorTimeoutMs must be an integer between 100 and 60000')
+  }
+  const advisorRefreshTurns = value.advisorRefreshTurns
+  if (advisorRefreshTurns !== undefined
+    && (typeof advisorRefreshTurns !== 'number' || !Number.isSafeInteger(advisorRefreshTurns)
+      || advisorRefreshTurns < 1)) {
+    throw new TypeError('Context-compression presetOptions.advisorRefreshTurns must be an integer of at least 1')
+  }
+  const advisorScoreThreshold = value.advisorScoreThreshold
+  if (advisorScoreThreshold !== undefined
+    && (typeof advisorScoreThreshold !== 'number' || !Number.isFinite(advisorScoreThreshold)
+      || advisorScoreThreshold <= 0 || advisorScoreThreshold >= 1)) {
+    throw new TypeError('Context-compression presetOptions.advisorScoreThreshold must be a number strictly between 0 and 1')
+  }
+  const advisorSampleLimit = value.advisorSampleLimit
+  if (advisorSampleLimit !== undefined
+    && (typeof advisorSampleLimit !== 'number' || !Number.isSafeInteger(advisorSampleLimit)
+      || advisorSampleLimit < 1 || advisorSampleLimit > 64)) {
+    throw new TypeError('Context-compression presetOptions.advisorSampleLimit must be an integer between 1 and 64')
+  }
+  const advisorMinTokens = value.advisorMinTokens
+  if (advisorMinTokens !== undefined
+    && (typeof advisorMinTokens !== 'number' || !Number.isSafeInteger(advisorMinTokens) || advisorMinTokens < 1)) {
+    throw new TypeError('Context-compression presetOptions.advisorMinTokens must be a positive integer')
+  }
   const estimatorTimeoutMs = value.estimatorTimeoutMs
   if (estimatorTimeoutMs !== undefined
     && (typeof estimatorTimeoutMs !== 'number' || !Number.isSafeInteger(estimatorTimeoutMs)
@@ -190,6 +225,12 @@ export function parsePresetOptionsSettings(value: unknown): PresetOptionsSetting
   if (reviewTimeoutTurns !== undefined) result.reviewTimeoutTurns = reviewTimeoutTurns as number
   if (cacheHitDiscountAlpha !== undefined) result.cacheHitDiscountAlpha = cacheHitDiscountAlpha as number
   if (reviewHighImpactTokens !== undefined) result.reviewHighImpactTokens = reviewHighImpactTokens as number
+  if (advisorMode !== undefined) result.advisorMode = advisorMode as '' | 'host' | 'direct'
+  if (advisorTimeoutMs !== undefined) result.advisorTimeoutMs = advisorTimeoutMs as number
+  if (advisorRefreshTurns !== undefined) result.advisorRefreshTurns = advisorRefreshTurns as number
+  if (advisorScoreThreshold !== undefined) result.advisorScoreThreshold = advisorScoreThreshold as number
+  if (advisorSampleLimit !== undefined) result.advisorSampleLimit = advisorSampleLimit as number
+  if (advisorMinTokens !== undefined) result.advisorMinTokens = advisorMinTokens as number
   return result
 }
 
@@ -308,6 +349,7 @@ const CONFIG_KEYS: ReadonlySet<string> = new Set([
   'historyKeepRecentToolCalls',
   'historyKeepRecentTokens',
   'historyMinReclaimTokens',
+  'readInputCapChars',
   'autoCompactThresholdPercent',
   'presetOptions',
 ])
@@ -335,6 +377,24 @@ export function codePointLength(text: string): number {
   return length
 }
 
+/**
+ * Conservative upper bound on characters per token. Same convention the
+ * readInputCapChars invariant already uses (see resolvePolicy): English
+ * code runs approximately four characters per token at the upper bound.
+ */
+export const CHARS_PER_TOKEN = 4.0
+
+/** Express one token-named policy gate on the character basis. */
+export function charsForTokens(tokens: number): number {
+  return tokens * CHARS_PER_TOKEN
+}
+
+/** Derive the telemetry-only token figure from a character measurement. */
+export function charsToTokens(chars: number): number {
+  if (!Number.isFinite(chars) || chars <= 0) return 0
+  return Math.max(1, Math.round(chars / CHARS_PER_TOKEN))
+}
+
 /**
  * Test whether a settings value names a supported compression profile.
  * @param value - untrusted settings value.
@@ -376,6 +436,7 @@ export function resolveConfig(config: ToolResultPruneConfig = {}): ResolvedConfi
     ...config.historyKeepRecentToolCalls === undefined ? {} : { historyKeepRecentToolCalls: config.historyKeepRecentToolCalls },
     ...config.historyKeepRecentTokens === undefined ? {} : { historyKeepRecentTokens: config.historyKeepRecentTokens },
     ...config.historyMinReclaimTokens === undefined ? {} : { historyMinReclaimTokens: config.historyMinReclaimTokens },
+    ...config.readInputCapChars === undefined ? {} : { readInputCapChars: config.readInputCapChars },
     ...config.autoCompactThresholdPercent === undefined ? {} : { autoCompactThresholdPercent: config.autoCompactThresholdPercent },
     ...config.presetOptions === undefined ? {} : { presetOptions: config.presetOptions },
   }
@@ -387,7 +448,7 @@ export function resolveConfig(config: ToolResultPruneConfig = {}): ResolvedConfi
   for (const key of [
     'nativeTriggerTokens', 'nativeTargetTokens', 'freshTriggerTokens', 'freshTargetTokens',
     'aggregateTriggerTokens', 'aggregateTargetTokens', 'historyTriggerTokens',
-    'historyMinReclaimTokens',
+    'historyMinReclaimTokens', 'readInputCapChars',
   ] as const) {
     const value = resolved[key]
     if (value !== undefined) assertPositiveInteger(key, value)
@@ -443,6 +504,15 @@ const PRESET_OPTION_DEFAULTS: PresetOptions = deepFreeze({
   reviewTimeoutTurns: 6,
   cacheHitDiscountAlpha: 0.1,
   reviewHighImpactTokens: 4000,
+  // Advisory advisor ships off: statistics and suggestions only, never a gate.
+  advisor: {
+    mode: '',
+    timeoutMs: 8_000,
+    refreshTurns: 8,
+    scoreThreshold: 0.35,
+    sampleLimit: 16,
+    minTokens: 250,
+  },
 })
 
 /**
@@ -464,6 +534,14 @@ function mergePresetOptions(overrides: PresetOptionsSettings | undefined): Prese
     reviewTimeoutTurns: overrides.reviewTimeoutTurns ?? PRESET_OPTION_DEFAULTS.reviewTimeoutTurns,
     cacheHitDiscountAlpha: overrides.cacheHitDiscountAlpha ?? PRESET_OPTION_DEFAULTS.cacheHitDiscountAlpha,
     reviewHighImpactTokens: overrides.reviewHighImpactTokens ?? PRESET_OPTION_DEFAULTS.reviewHighImpactTokens,
+    advisor: {
+      mode: overrides.advisorMode ?? PRESET_OPTION_DEFAULTS.advisor.mode,
+      timeoutMs: overrides.advisorTimeoutMs ?? PRESET_OPTION_DEFAULTS.advisor.timeoutMs,
+      refreshTurns: overrides.advisorRefreshTurns ?? PRESET_OPTION_DEFAULTS.advisor.refreshTurns,
+      scoreThreshold: overrides.advisorScoreThreshold ?? PRESET_OPTION_DEFAULTS.advisor.scoreThreshold,
+      sampleLimit: overrides.advisorSampleLimit ?? PRESET_OPTION_DEFAULTS.advisor.sampleLimit,
+      minTokens: overrides.advisorMinTokens ?? PRESET_OPTION_DEFAULTS.advisor.minTokens,
+    },
   })
 }
 
@@ -599,6 +677,7 @@ export function resolvePolicy(
       ?? linkage?.historyKeepRecentTokens ?? preset.historyKeepRecentTokens,
     historyMinReclaimTokens: config.historyMinReclaimTokens
       ?? linkage?.historyMinReclaimTokens ?? preset.historyMinReclaimTokens,
+    ...config.readInputCapChars === undefined ? {} : { readInputCapChars: config.readInputCapChars },
     ...linkage === undefined ? {} : {
       autoCompactTokens: linkage.autoCompactTokens,
       microDeadlineTokens: linkage.microDeadlineTokens,
@@ -616,6 +695,14 @@ export function resolvePolicy(
   if (policy.aggregateTargetTokens >= policy.aggregateTriggerTokens && policy.freshEnabled) {
     throw new Error('context compression policy: aggregate target must be below trigger')
   }
+  // G2 invariant: the fresh trigger is in TOKENS while a read input cap is in
+  // CHARACTERS. English code runs ≈4.0 chars/token at the conservative upper
+  // bound, so a cap at or below `freshTriggerTokens × 4.0` would truncate every
+  // read result below the trigger and silently silence the fresh path.
+  if (policy.readInputCapChars !== undefined
+    && policy.readInputCapChars <= policy.freshTriggerTokens * 4.0) {
+    throw new Error('context compression policy: read input cap would silence the fresh path')
+  }
   return deepFreeze(policy)
 }
 

package/packages/selector/src/runtime/measurement.ts

@@ -16,6 +16,7 @@ import {
   estimateDeepSeekVisionImageTokens,
 } from './deepseek-v4-vision-tokens.ts'
 import { unavailableTokenCount } from './token-count.ts'
+import { pressureCost } from '../pruner/content.ts'
 import type {
   CanonicalTextTokenCounter,
   ExactTokenizerTokenCount,
@@ -25,6 +26,19 @@ import type {
 
 export type { TokenCount } from './token-count.ts'
 
+/**
+ * Character pressure of one model-visible content array, measured on the same
+ * level `SnapshotCandidate.characterPressure` uses: a tool/result message is
+ * measured by its inner tool-result content, never by the wrapper block, so the
+ * node-level and candidate-level figures stay directly comparable.
+ * @param content - the node's model-visible content blocks.
+ * @returns character pressure in Unicode code points, plus rich-block costs.
+ */
+function nodeCharacterPressure(content: readonly ContentBlock[]): number {
+  const only = content.length === 1 ? content[0] : undefined
+  return pressureCost(only?.type === 'tool-result' ? only.content : content)
+}
+
 /** Request identity retained only when every dimension is publicly known. */
 export interface ProviderMeasurementKey {
   readonly provider: string
@@ -77,6 +91,11 @@ export interface IntrinsicImageBlockDiagnostic {
 export interface MeasuredTokenSurfaceNode {
   readonly seq: number
   readonly count: TokenCount
+  /**
+   * Authoritative decision metric for this node: character pressure in
+   * Unicode code points (same convention as SnapshotCandidate.characterPressure).
+   */
+  readonly characterPressure: number
   /** Intrinsic-grid diagnostic when usable image dimensions were available. */
   readonly intrinsicImageBlockEstimate?: IntrinsicImageBlockDiagnostic
 }
@@ -87,6 +106,11 @@ export interface CompactionTokenView extends TokenMeasurement {
   readonly modelId?: string
   readonly measuredNodes: readonly MeasuredTokenSurfaceNode[]
   readonly currentSurface: TokenCount
+  /**
+   * Exact character analogue of `currentSurface`: sum of per-node
+   * `characterPressure` over the same nodes `currentSurface` covers.
+   */
+  readonly currentSurfaceChars: number
   /** Sum of per-node intrinsic padding minima; a diagnostic, not a token bound. */
   readonly intrinsicImageBlockEstimateTokens: number
   readonly latestEnvelopeKey?: ProviderMeasurementKey
@@ -120,11 +144,13 @@ export function measureForCompaction(ctx: Context, session: Session): Compaction
   const measuredNodes = measurement.nodes.map((node): MeasuredTokenSurfaceNode => {
     const event = eventsBySeq.get(Number(node.seq))
     if (event === undefined) {
-      return { seq: node.seq, count: unavailableTokenCount(`surface node ${String(node.seq)} is missing`) }
+      // The node's content is not model-visible, so it carries no character
+      // pressure on the decision surface.
+      return { seq: node.seq, count: unavailableTokenCount(`surface node ${String(node.seq)} is missing`), characterPressure: 0 }
     }
     const message = deriveEventMessage(event)
     if (message === null) {
-      return { seq: node.seq, count: unavailableTokenCount(`surface node ${String(node.seq)} is not model-visible`) }
+      return { seq: node.seq, count: unavailableTokenCount(`surface node ${String(node.seq)} is not model-visible`), characterPressure: 0 }
     }
     const count = countCanonicalContent(message.content, counter, `surface node ${String(node.seq)}`)
     const intrinsicImageBlockEstimate = count.kind === 'tokenizer-estimate'
@@ -133,6 +159,7 @@ export function measureForCompaction(ctx: Context, session: Session): Compaction
     return {
       seq: node.seq,
       count,
+      characterPressure: nodeCharacterPressure(message.content),
       ...intrinsicImageBlockEstimate === undefined ? {} : { intrinsicImageBlockEstimate },
     }
   })
@@ -140,6 +167,7 @@ export function measureForCompaction(ctx: Context, session: Session): Compaction
     measuredNodes.map(node => node.count),
     'current surface',
   )
+  const currentSurfaceChars = measuredNodes.reduce((sum, node) => sum + node.characterPressure, 0)
   const intrinsicImageBlockEstimateTokens = measuredNodes.reduce(
     (sum, node) => sum + (node.intrinsicImageBlockEstimate?.paddingMinimumTokens ?? 0),
     0,
@@ -149,6 +177,7 @@ export function measureForCompaction(ctx: Context, session: Session): Compaction
     ...(target === undefined ? {} : { providerRoute: target.provider, modelId: target.model }),
     measuredNodes: Object.freeze(measuredNodes),
     currentSurface,
+    currentSurfaceChars,
     intrinsicImageBlockEstimateTokens,
     countCanonicalText: counter.countText,
   })