dsh-context-compression-improved 0.4.0-beta.1 → 0.5.0

package/packages/selector/src/client/ReviewOverlay.tsx

@@ -210,7 +210,7 @@ export function ReviewOverlay({ scope, t }: ReviewOverlayProps) {
       }
     }
     pull()
-    let unsubscribe = (): void => {}
+    let unsubscribe: () => void
     try {
       unsubscribe = scope.subscribe(pull)
     } catch {

package/packages/selector/src/client/index.ts

@@ -12,7 +12,7 @@ import { DEFAULT_CUSTOM_COMPRESSION_POLICY } from '../profiles.ts'
 import { decodeSettings } from './decode.ts'
 import { en, zh } from './locales.ts'
 import { planPresetOptionsOps, presetOptionsOpsAccepted } from './preset-options.ts'
-import { ReviewOverlay, renderReviewOverlay } from './ReviewOverlay.tsx'
+import { renderReviewOverlay } from './ReviewOverlay.tsx'
 
 /**
  * Harness 0.1.5 mounts the web core's `slots` service on the client context

package/packages/selector/src/client/preset-options.ts

@@ -25,6 +25,8 @@ const PRESET_OPTION_KEYS = [
   'dedupeToolResults', 'summaryLocator', 'prefixStabilizer', 'readState', 'estimatorMode',
   'estimatorProvider', 'estimatorModel', 'estimatorBaseUrl', 'estimatorApiKey', 'estimatorTimeoutMs',
   'reviewMode', 'reviewTimeoutTurns', 'cacheHitDiscountAlpha', 'reviewHighImpactTokens',
+  'advisorMode', 'advisorTimeoutMs', 'advisorRefreshTurns', 'advisorScoreThreshold', 'advisorSampleLimit',
+  'advisorMinTokens',
 ] as const
 
 /** One partial edit of `presetOptions`; `undefined` clears the named field. */

package/packages/selector/src/index.ts

@@ -11,6 +11,8 @@ import {
   CONTEXT_COMPRESSION_SETTINGS_NAMESPACE,
   ContextCompressionSettingsSchema,
 } from './runtime/config.ts'
+import { resolveReviewPruner, type ReviewPrunerFace } from './runtime/tokenpilot/review-registry.ts'
+import { getAdvisorState } from './runtime/tokenpilot/advisor-state.ts'
 
 // The settings namespace literal and the settings schema are owned by the
 // runtime config module. Both were once inlined/replaced here to dodge a
@@ -44,66 +46,43 @@ const REVIEW_DECIDE_ROUTES = [
   '/api/dsh-context-compression-improved/review-decide',
 ] as const
 
-/** The review faces of the pruner service the routes consume. */
-interface ReviewPrunerLike {
-  listReviewProposals(session: unknown): readonly {
-    readonly id: string
-    readonly kind: string
-    readonly items: readonly {
-      readonly seq: number
-      readonly kind: string
-      readonly component: string
-      readonly tokensBefore: number
-      readonly tokensAfter: number
-    }[]
-    readonly benefit: {
-      readonly recoveredTokens: number
-      readonly penaltyTokens: number
-      readonly paybackTurns?: number
-      readonly expectedSaving?: number
-    }
-    readonly enqueuedTurn: number
-    readonly lastTurnIndex: number
-  }[]
-  decideReviewProposal(
-    session: unknown,
-    proposalId: string,
-    decision: 'approved' | 'rejected' | 'ignored',
-  ): { ok: true } | { ok: false, reason: string } | undefined
-  /** Aggregate pending read; absent on older builds (routes then degrade to 503). */
-  listAllReviewProposals?(): readonly {
-    readonly sessionId: string
-    readonly proposals: readonly {
-      readonly id: string
-      readonly kind: string
-      readonly items: readonly { readonly seq: number, readonly kind: string, readonly component: string, readonly tokensBefore: number, readonly tokensAfter: number }[]
-      readonly benefit: { readonly recoveredTokens: number, readonly penaltyTokens: number, readonly paybackTurns?: number, readonly expectedSaving?: number }
-      readonly enqueuedTurn: number
-      readonly lastTurnIndex: number
-    }[]
-  }[]
-  reviewSummary?(session: unknown): {
-    readonly autoApplied: number
-    readonly reviewApplied: number
-    readonly expired: number
-    readonly voided: number
-  }
-}
+// Advisory advisor: one read-only report route, dual prefixed like the others.
+const ADVISOR_REPORT_ROUTES = [
+  '/endpoint/dsh-context-compression-improved/advisor-report',
+  '/api/dsh-context-compression-improved/advisor-report',
+] as const
+
+/**
+ * The review faces of the pruner service the routes consume. Owned by the
+ * review registry, which the routes also fall back to when no top-level
+ * `toolResultPruner` service exists — the preset-scoped case in production.
+ */
+type ReviewPrunerLike = ReviewPrunerFace
 
 /** Minimal face of the agents service: session id → agent (carrying the session). */
 interface AgentsServiceLike {
   get?(id: unknown): { session?: unknown } | undefined
 }
 
+/**
+ * Resolve the review pipeline for the top-level routes.
+ *
+ * A top-level `toolResultPruner` service wins when a deployment actually mounts
+ * one, but in production every pruner lives inside an agent preset's isolated
+ * group, so the registry is the path that resolves. Without the fallback the
+ * queue route answered 503 "review pipeline unavailable" on every request while
+ * the review pipeline itself was running normally.
+ */
 function reviewPrunerOf(readService: (name: string) => unknown): ReviewPrunerLike | undefined {
   const candidate = readService('toolResultPruner') as {
     listReviewProposals?: unknown
     decideReviewProposal?: unknown
   } | undefined
-  return typeof candidate?.listReviewProposals === 'function'
-    && typeof candidate?.decideReviewProposal === 'function'
-    ? candidate as unknown as ReviewPrunerLike
-    : undefined
+  if (typeof candidate?.listReviewProposals === 'function'
+    && typeof candidate?.decideReviewProposal === 'function') {
+    return candidate as unknown as ReviewPrunerLike
+  }
+  return resolveReviewPruner()
 }
 
 function sessionFor(readService: (name: string) => unknown, sessionId: string): unknown {
@@ -329,6 +308,96 @@ function registerReviewQueueRoutes(ctx: Context): void {
   log('warn', 'context-compression webServer not active yet — review routes pending: %s', REVIEW_QUEUE_ROUTES.join(', '))
 }
 
+/**
+ * Serve the advisory advisor's read-only report route (same registration
+ * skeleton as the review routes):
+ *
+ * `GET .../advisor-report?sessionId=…` → the session's prefix-decay figure,
+ * the todolist-bound task summary, and the score distribution. Content-free
+ * by construction: task semantics are LLM-derived summaries, never message
+ * text, and no score reason or candidate preview is ever returned.
+ * Unknown session → 404; no agents service → 503. A session whose advisor
+ * never ran reports nulls and empty arrays, not an error.
+ */
+function registerAdvisorReportRoute(ctx: Context): void {
+  const readService = (name: string): unknown => {
+    try {
+      return (ctx as unknown as { get: (service: string) => unknown }).get(name)
+    } catch {
+      return undefined
+    }
+  }
+  const log = (level: 'info' | 'warn', message: string, ...args: unknown[]): void => {
+    console[level](message, ...args)
+  }
+
+  const getHandler = (req: unknown, res: unknown): void => {
+    const agents = readService('agents') as AgentsServiceLike | undefined
+    if (typeof agents?.get !== 'function') {
+      reviewJson(res, 503, { ok: false, error: 'advisor report unavailable' })
+      return
+    }
+    let sessionId = ''
+    try {
+      const url = new URL(String((req as { url?: string }).url ?? ''), 'http://localhost')
+      sessionId = url.searchParams.get('sessionId') ?? ''
+    } catch {
+      // Malformed URL: fall through with the empty sessionId already set.
+    }
+    if (sessionId === '') {
+      reviewJson(res, 400, { ok: false, error: 'sessionId is required' })
+      return
+    }
+    const session = sessionFor(readService, sessionId)
+    if (session === undefined) {
+      reviewJson(res, 404, { ok: false, error: 'unknown session' })
+      return
+    }
+    const state = getAdvisorState(session as Parameters<typeof getAdvisorState>[0])
+    reviewJson(res, 200, {
+      ok: true,
+      sessionId,
+      advisor: {
+        summary: state.summary ?? null,
+        decay: state.lastDecay?.decay ?? null,
+        weightedChars: state.lastDecay?.weightedChars ?? null,
+        decayTurn: state.lastDecay?.turn ?? null,
+        scores: [...state.scores].map(([seq, entry]) => ({ seq, score: entry.score, turn: entry.turn })),
+        lowRelevanceSeqs: [...state.recertified.keys()],
+      },
+    })
+  }
+
+  const register = (webServer: WebServerLike): void => {
+    const table = [...ADVISOR_REPORT_ROUTES].map(path => ({ path, handler: getHandler }))
+    const disposers = table
+      .map(entry => webServer.register({ kind: 'exact', path: entry.path, handler: entry.handler }))
+      .filter((off): off is () => void => typeof off === 'function')
+    ctx.effect(
+      () => () => { for (const off of disposers) off() },
+      'contextCompressionSelector.advisor report route',
+    )
+    log('info', 'context-compression advisor report route registered: %s', ADVISOR_REPORT_ROUTES.join(', '))
+  }
+
+  const active = asWebServer(readService('webServer'))
+  if (active !== undefined) {
+    register(active)
+    return
+  }
+
+  ctx.inject(['webServer'], (injected) => {
+    const webServer = asWebServer((injected as { webServer?: unknown }).webServer)
+    if (webServer === undefined) {
+      log('warn', 'context-compression webServer exposes no register() — advisor report route not registered')
+      return
+    }
+    register(webServer)
+  })
+
+  log('warn', 'context-compression webServer not active yet — advisor report route pending: %s', ADVISOR_REPORT_ROUTES.join(', '))
+}
+
 /**
  * The one service the catalog route actually needs. `llm` and
  * `agentDefaultModel` are payload enrichment the handler resolves per request,
@@ -512,6 +581,13 @@ export interface Config {
    * transport and simply never appears.
    */
   reviewQueueRoute?: boolean
+  /**
+   * Register the advisory advisor's read-only HTTP report route (decay
+   * figure, task summary, score distribution). Same Bundle opt-in semantics
+   * as `reviewQueueRoute`; the advisor itself stays off until the user turns
+   * it on through the `presetOptions.advisor*` settings keys.
+   */
+  advisorReportRoute?: boolean
 }
 
 /** Loader validation for the standalone Bundle opt-in. */
@@ -519,6 +595,7 @@ export const Config: z<Config> = z.object({
   presetOverlay: z.boolean().default(false),
   estimatorCatalogRoute: z.boolean().default(false),
   reviewQueueRoute: z.boolean().default(false),
+  advisorReportRoute: z.boolean().default(false),
 })
 
 /** Register the persisted default read by the currently mounted root pruner. */
@@ -542,6 +619,9 @@ export function apply(ctx: Context, config: Config = {}): void {
     // config JSDoc for the opt-in semantics).
     if (config.reviewQueueRoute === true) registerReviewQueueRoutes(ctx)
 
+    // Advisory advisor: read-only decay/score report (opt-in, like review).
+    if (config.advisorReportRoute === true) registerAdvisorReportRoute(ctx)
+
     if (config.presetOverlay !== true) return
 
     ctx.inject(['agentPresets'], (presetsCtx) => {

package/packages/selector/src/profiles.ts

@@ -140,6 +140,13 @@ export interface PresetOptionsSettings {
   readonly reviewTimeoutTurns?: number
   readonly cacheHitDiscountAlpha?: number
   readonly reviewHighImpactTokens?: number
+  /** Advisory advisor channel; `''` (the default) keeps the advisor off. */
+  readonly advisorMode?: '' | 'host' | 'direct'
+  readonly advisorTimeoutMs?: number
+  readonly advisorRefreshTurns?: number
+  readonly advisorScoreThreshold?: number
+  readonly advisorSampleLimit?: number
+  readonly advisorMinTokens?: number
 }
 
 /**
@@ -154,6 +161,8 @@ export function decodePresetOptionsSettings(value: unknown): PresetOptionsSettin
     'dedupeToolResults', 'summaryLocator', 'prefixStabilizer', 'readState', 'estimatorMode',
     'estimatorProvider', 'estimatorModel', 'estimatorBaseUrl', 'estimatorApiKey', 'estimatorTimeoutMs',
     'reviewMode', 'reviewTimeoutTurns', 'cacheHitDiscountAlpha', 'reviewHighImpactTokens',
+    'advisorMode', 'advisorTimeoutMs', 'advisorRefreshTurns', 'advisorScoreThreshold', 'advisorSampleLimit',
+    'advisorMinTokens',
   ])
   if (Object.keys(value).some(key => !allowed.has(key))) return undefined
   for (const key of ['dedupeToolResults', 'summaryLocator', 'prefixStabilizer', 'readState', 'reviewMode'] as const) {
@@ -164,6 +173,10 @@ export function decodePresetOptionsSettings(value: unknown): PresetOptionsSettin
   if (estimatorMode !== undefined && estimatorMode !== '' && estimatorMode !== 'host' && estimatorMode !== 'direct') {
     return undefined
   }
+  const advisorMode = value.advisorMode
+  if (advisorMode !== undefined && advisorMode !== '' && advisorMode !== 'host' && advisorMode !== 'direct') {
+    return undefined
+  }
   for (const key of ['estimatorProvider', 'estimatorModel', 'estimatorBaseUrl', 'estimatorApiKey'] as const) {
     const entry = value[key]
     if (entry !== undefined && typeof entry !== 'string') return undefined
@@ -191,6 +204,35 @@ export function decodePresetOptionsSettings(value: unknown): PresetOptionsSettin
       || reviewHighImpactTokens < 0)) {
     return undefined
   }
+  const advisorTimeoutMs = value.advisorTimeoutMs
+  if (advisorTimeoutMs !== undefined
+    && (typeof advisorTimeoutMs !== 'number' || !Number.isSafeInteger(advisorTimeoutMs)
+      || advisorTimeoutMs < 100 || advisorTimeoutMs > 60_000)) {
+    return undefined
+  }
+  const advisorRefreshTurns = value.advisorRefreshTurns
+  if (advisorRefreshTurns !== undefined
+    && (typeof advisorRefreshTurns !== 'number' || !Number.isSafeInteger(advisorRefreshTurns)
+      || advisorRefreshTurns < 1)) {
+    return undefined
+  }
+  const advisorScoreThreshold = value.advisorScoreThreshold
+  if (advisorScoreThreshold !== undefined
+    && (typeof advisorScoreThreshold !== 'number' || !Number.isFinite(advisorScoreThreshold)
+      || advisorScoreThreshold <= 0 || advisorScoreThreshold >= 1)) {
+    return undefined
+  }
+  const advisorSampleLimit = value.advisorSampleLimit
+  if (advisorSampleLimit !== undefined
+    && (typeof advisorSampleLimit !== 'number' || !Number.isSafeInteger(advisorSampleLimit)
+      || advisorSampleLimit < 1 || advisorSampleLimit > 64)) {
+    return undefined
+  }
+  const advisorMinTokens = value.advisorMinTokens
+  if (advisorMinTokens !== undefined
+    && (typeof advisorMinTokens !== 'number' || !Number.isSafeInteger(advisorMinTokens) || advisorMinTokens < 1)) {
+    return undefined
+  }
   const decoded: {
     -readonly [K in keyof PresetOptionsSettings]: PresetOptionsSettings[K]
   } = {}
@@ -208,6 +250,12 @@ export function decodePresetOptionsSettings(value: unknown): PresetOptionsSettin
   if (reviewTimeoutTurns !== undefined) decoded.reviewTimeoutTurns = reviewTimeoutTurns as number
   if (cacheHitDiscountAlpha !== undefined) decoded.cacheHitDiscountAlpha = cacheHitDiscountAlpha as number
   if (reviewHighImpactTokens !== undefined) decoded.reviewHighImpactTokens = reviewHighImpactTokens as number
+  if (advisorMode !== undefined) decoded.advisorMode = advisorMode as '' | 'host' | 'direct'
+  if (advisorTimeoutMs !== undefined) decoded.advisorTimeoutMs = advisorTimeoutMs as number
+  if (advisorRefreshTurns !== undefined) decoded.advisorRefreshTurns = advisorRefreshTurns as number
+  if (advisorScoreThreshold !== undefined) decoded.advisorScoreThreshold = advisorScoreThreshold as number
+  if (advisorSampleLimit !== undefined) decoded.advisorSampleLimit = advisorSampleLimit as number
+  if (advisorMinTokens !== undefined) decoded.advisorMinTokens = advisorMinTokens as number
   return decoded
 }
 

package/packages/selector/src/pruner/content.ts

@@ -70,8 +70,11 @@ function unavailableCount(reason: string): TokenCount {
 
 // ── Recovery markers ────────────────────────────────────────────────────
 
-function recoveryMarker(sourceRef: string, label: string): string {
-  return `\n\n[... ${label}; source=${sourceRef}; use context_compression_retrieve if needed ...]\n\n`
+function recoveryMarker(sourceRef: string, label: string, startLine?: number): string {
+  const hint = startLine === undefined
+    ? 'use context_compression_retrieve if needed'
+    : `retrieve with context_compression_retrieve({"ref":"${sourceRef}","start_line":${String(startLine)},"max_lines":80})`
+  return `\n\n[... ${label}; source=${sourceRef}; ${hint} ...]\n\n`
 }
 
 // ── Content measurement ─────────────────────────────────────────────────
@@ -124,11 +127,11 @@ function nativePruneContent(
   thresholdChars: number,
   headChars: number,
   tailChars: number,
-  marker: string = PRUNE_MARKER,
+  marker: string | ((startLine: number) => string) = PRUNE_MARKER,
 ): ContentBlock[] | null {
   const totalChars = measureContent(blocks)
   if (totalChars <= thresholdChars) return null
-  const markerChars = codePointLength(marker)
+  const markerChars = codePointLength(typeof marker === 'function' ? marker(1) : marker)
   const safeHead = Math.max(0, Math.min(headChars, thresholdChars - markerChars))
   const safeTail = Math.max(0, Math.min(tailChars, thresholdChars - markerChars - safeHead))
   const removedStart = safeHead
@@ -136,9 +139,14 @@ function nativePruneContent(
   const pruned: ContentBlock[] = []
   let consumed = 0
   let markerInserted = false
+  // '\n' count in the RAW event text before the current block (retrieve scans
+  // the original event; blocks are joined with '\n'). Line numbers therefore
+  // never depend on what the pruning removed.
+  let newlinesBefore = 0
   for (const block of blocks) {
     if (block.type !== 'text') {
       pruned.push(block)
+      newlinesBefore += 1
       continue
     }
     const points = Array.from(block.text)
@@ -147,10 +155,15 @@ function nativePruneContent(
     const headEnd = Math.min(points.length, Math.max(0, removedStart - blockStart))
     const tailStart = Math.min(points.length, Math.max(0, removedEnd - blockStart))
     const intersectsRemoved = blockStart < removedEnd && blockEnd > removedStart
-    const insertion = intersectsRemoved && !markerInserted ? marker : ''
+    // The elided region starts on the event line after the retained head.
+    const headText = points.slice(0, headEnd).join('')
+    const insertion = intersectsRemoved && !markerInserted && typeof marker === 'function'
+      ? marker(1 + newlinesBefore + headText.split('\n').length - 1)
+      : intersectsRemoved && !markerInserted ? marker : ''
     if (insertion !== '') markerInserted = true
     const text = points.slice(0, headEnd).join('') + insertion + points.slice(tailStart).join('')
     if (text !== '') pruned.push({ ...block, text })
+    newlinesBefore += block.text.split('\n').length - 1 + 1
     consumed = blockEnd
   }
   if (!markerInserted) return null

package/packages/selector/src/pruner/state.ts

@@ -9,6 +9,7 @@ import type { Session } from '@deepseek-ai/dsh-session'
 import type { DedupeTable } from '../runtime/tokenpilot/dedup.ts'
 import type { EstimatorFailures } from '../runtime/tokenpilot/estimator.ts'
 import type { ReviewQueue, ReviewQueueStore } from '../runtime/tokenpilot/review-queue.ts'
+import type { SideChannel } from '../runtime/tokenpilot/sidechannel.ts'
 import type {
   ContextCompressionSettings,
   ResolvedConfig,
@@ -65,6 +66,8 @@ export interface PrunerState {
   readonly reviewClocks: WeakMap<Session, number>
   /** Estimator-reported remaining turns Ŝ per Session; advisory only. */
   readonly estimatorRemainingTurns: WeakMap<Session, number>
+  /** Per-session advisor side channel, constructed once with the advisor overrides. */
+  readonly advisorChannels: WeakMap<Session, SideChannel>
   /** Four-state outcome counters per Session (floating-window summary row). */
   readonly reviewSummaries: WeakMap<Session, ReviewSessionSummary>
 }

package/packages/selector/src/pruner/types.ts

@@ -20,11 +20,16 @@ export interface SnapshotCandidate {
   readonly seq: number
   readonly event: SessionEvent<'tool/result'>
   readonly call: ToolCallInfo
-  /** Authoritative same-revision exact canonical content count. */
+  /** Authoritative same-revision exact canonical content count (telemetry only). */
   readonly count: TokenCount
   /** Same-revision legacy heuristic price used only by bounded projections. */
   readonly shadowedHeuristicTokenCount: number
-  /** Character pressure is candidate-shape telemetry only, never a gate. */
+  /**
+   * Authoritative decision metric: character pressure in Unicode code points.
+   * Every eligibility and acceptance gate decides on this number so the plugin
+   * never depends on the routed model id owning a bundled exact tokenizer;
+   * `count` is retained as diagnostic telemetry only.
+   */
   readonly characterPressure: number
 }
 
@@ -40,19 +45,32 @@ export interface PlannedReplacement {
   readonly charsAfter: number
   readonly tokensBefore: number
   readonly tokensAfter: number
+  /**
+   * Which measurement produced this plan's decision: 'exact-tokenizer' when
+   * both sides of the reduction carry one bundled tokenizer identity,
+   * 'characters' when the proof ran on code points alone.
+   */
+  readonly measurementBasis: 'exact-tokenizer' | 'characters'
   readonly tokenizerId: string
   readonly tokenizerRevision: string
+  /**
+   * Original-event lines the reducer elided (task_4c/G7 telemetry, from
+   * `ReducerOutput.elidedLines`). Rides the rewrite AUDIT record only — it is
+   * never printed into replacement content.
+   */
+  readonly elidedLines?: number
 }
 
 /**
  * Discriminated History planning outcome: every skip path carries its own
  * reason instead of collapsing into one merged "no eligible minimum reclaim"
- * audit, so operators can tell a protected working set from missing exact
- * counts or an unreachable reclaim target.
+ * audit, so operators can tell a protected working set from an unreachable
+ * reclaim target. The former 'exact-tokenizer-unavailable' member is gone:
+ * planning decides on the character basis and can no longer refuse for a
+ * missing exact count.
  */
 export type HistoryPlanOutcome =
   | { readonly kind: 'planned', readonly plans: PlannedReplacement[] }
-  | { readonly kind: 'exact-tokenizer-unavailable' }
   | { readonly kind: 'below-profile-trigger' }
   | { readonly kind: 'no-safe-candidates' }
   | { readonly kind: 'protected-working-set' }