dsh-context-compression-improved 0.1.1 → 0.2.0

package/packages/selector/src/runtime/token-count.ts

@@ -0,0 +1,72 @@
+/** Token counts owned by the standalone compression runtime. */
+
+/** Exact count of one canonical value under a pinned tokenizer artifact. */
+export interface ExactTokenizerTokenCount {
+  readonly kind: 'exact-tokenizer'
+  readonly tokens: number
+  readonly tokenizerId: string
+  readonly tokenizerRevision: string
+}
+
+/** A value that the bundled tokenizer cannot safely count. */
+export interface UnavailableTokenCount {
+  readonly kind: 'unavailable'
+  readonly reason: string
+}
+
+/** Best-effort estimate carrying a conservative upper value when available. */
+export interface TokenizerEstimateTokenCount {
+  readonly kind: 'tokenizer-estimate'
+  readonly tokens: number
+  readonly upperBoundTokens: number
+  readonly estimatorId: string
+  readonly estimatorRevision: string
+  readonly calibration?: Readonly<{
+    readonly sampleCount: number
+    readonly conservativeMarginTokens: number
+  }>
+}
+
+/** Exact, conservative request estimate, or an explicit refusal. */
+export type TokenCount = ExactTokenizerTokenCount | TokenizerEstimateTokenCount | UnavailableTokenCount
+
+/** Bound exact text counter for one durable provider/model request target. */
+export type CanonicalTextTokenCounter = (text: string) => TokenCount
+
+/** Build an explicit unavailable result without inventing an estimate. */
+export function unavailableTokenCount(reason: string): UnavailableTokenCount {
+  if (reason.length === 0) throw new TypeError('unavailable token count requires a reason')
+  return Object.freeze({ kind: 'unavailable', reason })
+}
+
+/** Sum independent canonical fields only when every count has one identity. */
+export function countExactCanonicalTextFields(
+  fields: readonly string[],
+  counter: CanonicalTextTokenCounter,
+  subject: string,
+): TokenCount {
+  if (subject.length === 0) throw new TypeError('canonical text field count requires a subject')
+  const values = fields.length === 0 ? [''] : fields
+  let identity: ExactTokenizerTokenCount | undefined
+  let tokens = 0
+  for (const value of values) {
+    const count = counter(value)
+    if (count.kind !== 'exact-tokenizer') {
+      return unavailableTokenCount(`${subject}: ${count.kind === 'unavailable'
+        ? count.reason
+        : 'canonical content requires an exact tokenizer count'}`)
+    }
+    if (identity !== undefined
+      && (identity.tokenizerId !== count.tokenizerId
+        || identity.tokenizerRevision !== count.tokenizerRevision)) {
+      return unavailableTokenCount(`${subject}: tokenizer identity changed within one measurement`)
+    }
+    identity ??= count
+    tokens += count.tokens
+    if (!Number.isSafeInteger(tokens) || tokens < 0) {
+      return unavailableTokenCount(`${subject}: token sum is outside the safe integer range`)
+    }
+  }
+  if (identity === undefined) return unavailableTokenCount(`${subject}: no tokenizer identity`)
+  return Object.freeze({ ...identity, tokens })
+}

package/packages/selector/src/runtime/tokenpilot/dedup.ts

@@ -0,0 +1,81 @@
+/**
+ * TokenPilot-inspired A1: byte-identical repeated tool-result dedup.
+ *
+ * Pure helpers behind the ToolResultPruner fresh pass. The per-session table
+ * maps a canonical-content SHA-256 to the first surface seq that produced it;
+ * later identical results may be replaced with a pointer placeholder that the
+ * recovery tool can resolve back to the original full text via the append-only
+ * session log. Only hash+seq metadata is stored — never content.
+ */
+import { createHash } from 'node:crypto'
+import type { ContentBlock } from '@deepseek-ai/dsh-llm'
+
+/** Canonicalization modes for content hashing. */
+export type DedupeNormalization = 'trim-eol' | 'exact'
+
+/** Entry stored per first-seen content hash. */
+export interface DedupeTableEntry {
+  /** First surface seq carrying this canonical content. */
+  readonly seq: number
+  /** Session source reference of the first occurrence. */
+  readonly sourceRef: string
+  /** Tool name of the first occurrence. */
+  readonly toolName: string
+  /** Original full-text code points of the first occurrence. */
+  readonly originalChars: number
+}
+
+/** Per-session dedup index with insertion-order eviction. */
+export class DedupeTable {
+  private readonly entries = new Map<string, DedupeTableEntry>()
+
+  constructor(private readonly maxEntries = 2_048) {}
+
+  /** Look up the first occurrence for one canonical hash, if any. */
+  get(hash: string): DedupeTableEntry | undefined {
+    return this.entries.get(hash)
+  }
+
+  /** Record a first occurrence; existing hashes only refresh insertion order. */
+  record(hash: string, entry: DedupeTableEntry): void {
+    if (this.entries.has(hash)) return
+    while (this.entries.size >= this.maxEntries) {
+      const oldest = this.entries.keys().next().value
+      if (oldest === undefined) break
+      this.entries.delete(oldest)
+    }
+    this.entries.set(hash, entry)
+  }
+}
+
+/** Canonicalize tool-result text for hashing. */
+export function canonicalizeForDedupe(text: string, mode: DedupeNormalization): string {
+  if (mode === 'exact') return text
+  return text
+    .replace(/[ \t]+\r?\n/g, '\n')
+    .replace(/(^\s+)|(\s+$)/g, '')
+}
+
+/** SHA-256 hex of the canonicalized text. */
+export function dedupeHash(text: string, mode: DedupeNormalization): string {
+  return createHash('sha256').update(canonicalizeForDedupe(text, mode), 'utf8').digest('hex')
+}
+
+/** Concatenated text of an all-text content block list; null when rich. */
+export function flattenPlainText(content: readonly ContentBlock[]): string | undefined {
+  let text = ''
+  for (const block of content) {
+    if (block.type !== 'text') return undefined
+    text += block.text
+  }
+  return text
+}
+
+/** Pointer placeholder pointing at the first occurrence's original event. */
+export function dedupePlaceholder(entry: DedupeTableEntry, originalChars: number): string {
+  return [
+    `[... identical to the earlier ${entry.toolName} result; first seen at ${entry.sourceRef};`,
+    `original_chars=${String(originalChars)};`,
+    'use context_compression_retrieve with this source if the omitted evidence is necessary.]',
+  ].join(' ')
+}

package/packages/selector/src/runtime/tokenpilot/estimator.ts

@@ -0,0 +1,183 @@
+/**
+ * TokenPilot-inspired E1: lightweight zero-shot estimator channel.
+ *
+ * A small auxiliary model (TokenPilot uses a Qwen3.5-35B-A3B-class estimator)
+ * judges residual utility of oversized historical reads: "will the model still
+ * reference this file state?" Verdicts are advisory — they only extend the
+ * rule-only superseded classification and never block or roll back a landed
+ * rewrite. Any failure is fail-open.
+ *
+ * Channels: `''` (off — every consumer stays on rule-only fallbacks), `host`
+ * (the Harness `llm` service with the user's configured providers), `direct`
+ * (native OpenAI-compatible HTTP endpoint). The API key lives in session
+ * settings only and is never logged or audited.
+ */
+import type { Context } from '@deepseek-ai/cordis'
+import type { PresetOptionsSettings } from '../types.ts'
+
+/** The minimal face of the Harness `llm` service this module consumes. */
+export interface HostLlmLike {
+  stream(request: {
+    provider: string
+    model: string
+    messages: readonly { readonly role: 'user', readonly content: readonly { readonly type: 'text', readonly text: string }[] }[]
+    system?: string
+    temperature?: number
+    reasoningEffort?: string
+    maxTokens?: number
+    signal?: AbortSignal
+  }): AsyncIterable<{ readonly type: string, readonly text?: string }>
+}
+
+/** One sampled historical read offered to the estimator. */
+export interface EstimatorSample {
+  readonly seq: number
+  readonly path: string
+  readonly turn: number
+}
+
+/** One estimator verdict for a sampled read. */
+export interface EstimatorVerdict {
+  readonly seq: number
+  readonly expired: boolean
+}
+
+/** Per-session estimator failure bookkeeping for exponential backoff. */
+export interface EstimatorFailures {
+  failures: number
+  cooldownUntil: number
+}
+
+/** Exponential backoff with a 5-minute cap: 1s, 2s, 4s, … */
+export function backoffCooldownMs(failures: number): number {
+  return Math.min(5 * 60_000, 1_000 * 2 ** Math.max(0, failures - 1))
+}
+
+export function isCoolingDown(state: EstimatorFailures | undefined, now: number): boolean {
+  return state !== undefined && state.cooldownUntil > now
+}
+
+export function buildEstimatorSystemPrompt(): string {
+  return [
+    'You are a session residual-utility estimator.',
+    'For each numbered historical file read, decide whether the live agent is likely to',
+    'reference that exact file state again later in the session. Reads whose file was',
+    'already rewritten, or whose task has visibly moved on, are expired.',
+    'Answer with ONLY a JSON array: [{"seq":<number>,"expired":<boolean>}].',
+  ].join(' ')
+}
+
+export function buildEstimatorUserPrompt(samples: readonly EstimatorSample[]): string {
+  const lines = samples.map(sample =>
+    `{"seq":${String(sample.seq)},"path":${JSON.stringify(sample.path)},"turn":${String(sample.turn)}}`)
+  return lines.join('\n')
+}
+
+/** Parse the estimator answer; anything malformed yields no verdicts. */
+export function parseEstimatorAnswer(text: string): EstimatorVerdict[] {
+  const start = text.indexOf('[')
+  const end = text.lastIndexOf(']')
+  if (start < 0 || end <= start) return []
+  try {
+    const parsed: unknown = JSON.parse(text.slice(start, end + 1))
+    if (!Array.isArray(parsed)) return []
+    const verdicts: EstimatorVerdict[] = []
+    for (const entry of parsed) {
+      if (typeof entry !== 'object' || entry === null) continue
+      const record = entry as { seq?: unknown, expired?: unknown }
+      if (typeof record.seq !== 'number' || typeof record.expired !== 'boolean') continue
+      verdicts.push({ seq: record.seq, expired: record.expired })
+    }
+    return verdicts
+  } catch {
+    return []
+  }
+}
+
+/** One channel-bound estimator. `ask` resolves undefined on any failure. */
+export class Estimator {
+  constructor(
+    private readonly ctx: Context,
+    private readonly options: PresetOptionsSettings,
+  ) {}
+
+  get enabled(): boolean {
+    return this.options.estimatorMode === 'host' || this.options.estimatorMode === 'direct'
+  }
+
+  async ask(system: string, user: string, signal: AbortSignal): Promise<string | undefined> {
+    const timeoutMs = this.options.estimatorTimeoutMs ?? 3_000
+    const timeout = AbortSignal.timeout(timeoutMs)
+    const signal2 = typeof AbortSignal.any === 'function' ? AbortSignal.any([signal, timeout]) : timeout
+    try {
+      if (this.options.estimatorMode === 'host') return await this.askHost(system, user, signal2)
+      if (this.options.estimatorMode === 'direct') return await this.askDirect(system, user, signal2)
+      return undefined
+    } catch {
+      return undefined
+    }
+  }
+
+  private async askHost(system: string, user: string, signal: AbortSignal): Promise<string | undefined> {
+    let llm: HostLlmLike | undefined
+    try {
+      llm = this.ctx.get('llm' as never) as HostLlmLike | undefined
+    } catch {
+      return undefined
+    }
+    if (llm?.stream === undefined) return undefined
+    const provider = this.options.estimatorProvider ?? ''
+    const model = this.options.estimatorModel ?? ''
+    if (provider.length === 0 || model.length === 0) return undefined
+    let text = ''
+    const stream = llm.stream({
+      provider,
+      model,
+      messages: [{ role: 'user', content: [{ type: 'text', text: user }] }],
+      system,
+      temperature: 0,
+      reasoningEffort: 'off',
+      maxTokens: 256,
+      signal,
+    })
+    for await (const chunk of stream) {
+      if ((chunk.type === 'text-delta' || chunk.type === 'reasoning-delta') && typeof chunk.text === 'string') {
+        text += chunk.text
+      } else if (chunk.type === 'finish' && chunk.text === undefined) {
+        break
+      }
+    }
+    return text.trim().length > 0 ? text : undefined
+  }
+
+  private async askDirect(system: string, user: string, signal: AbortSignal): Promise<string | undefined> {
+    const baseUrl = this.options.estimatorBaseUrl
+    if (baseUrl === undefined || baseUrl.length === 0) return undefined
+    const headers: Record<string, string> = { 'content-type': 'application/json' }
+    if (this.options.estimatorApiKey !== undefined && this.options.estimatorApiKey.length > 0) {
+      headers.authorization = `Bearer ${this.options.estimatorApiKey}`
+    }
+    const model = this.options.estimatorModel ?? ''
+    if (model.length === 0) return undefined
+    const response = await fetch(`${baseUrl.replace(/\/+$/, '')}/chat/completions`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        model,
+        messages: [
+          { role: 'system', content: system },
+          { role: 'user', content: user },
+        ],
+        temperature: 0,
+        max_tokens: 256,
+      }),
+      signal,
+    })
+    if (!response.ok) return undefined
+    const payload = (await response.json()) as {
+      choices?: { message?: { content?: string } }[]
+    }
+    const text = payload.choices?.[0]?.message?.content
+    return typeof text === 'string' && text.trim().length > 0 ? text : undefined
+  }
+}

package/packages/selector/src/runtime/tokenpilot/locator.ts

@@ -0,0 +1,128 @@
+/**
+ * TokenPilot-inspired A2: Exact Sources locator block appended to the Auto
+ * Compact summary checkpoint AFTER compaction/end.
+ *
+ * The block carries three kinds of locators over the shadowed range — the seq
+ * range itself, spill files named by the Harness spill notices, and files
+ * touched by read/grep-style tool calls — so details removed by the LLM
+ * summary remain recoverable through ordinary file reads, the recovery tool,
+ * or session event references. The block is only a few hundred bytes and is
+ * skipped entirely when it would locate nothing concrete.
+ */
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+
+/** One resolved compaction transaction to annotate. */
+export interface CompactionTrace {
+  readonly compactionId: string
+  readonly summarySeq: number
+  readonly summaryShadowedRange: { readonly start: number, readonly end: number }
+}
+
+/** Files touched by read/grep-style tool calls inside the range. */
+const TOUCHED_FILE_TOOL = /(?:^|[-_])?(?:read|write|edit|glob|grep|view|str_replace_editor)(?:$|[-_])/i
+/** Spill notice paths emitted by the Harness output-retention policy. */
+const SPILL_PATH = /stored at:\s*([^\s)\]]+)/g
+/** Tool call arguments keys that commonly carry a file path. */
+const PATH_KEYS = ['path', 'file_path'] as const
+
+/**
+ * Find the latest compaction/summary event matching the compaction id of a
+ * compaction/end event. Returns undefined when the transaction cannot be
+ * identified — the caller must skip rather than guess.
+ */
+export function findCompactionTrace(
+  events: readonly SessionEvent[],
+  compactionId: string,
+): CompactionTrace | undefined {
+  let trace: CompactionTrace | undefined
+  for (const event of events) {
+    if (event.type === 'compaction/summary' && event.data.compactionId === compactionId) {
+      trace = {
+        compactionId,
+        summarySeq: event.seq,
+        summaryShadowedRange: event.data.shadowedRange,
+      }
+    }
+  }
+  return trace
+}
+
+/** Extract spill file paths from one text chunk. */
+export function extractSpillPaths(text: string): string[] {
+  const paths: string[] = []
+  for (const match of text.matchAll(SPILL_PATH)) {
+    const path = match[1]?.replace(/[.,;]+$/, '')
+    if (path !== undefined && path.length > 0) paths.push(path)
+  }
+  return paths
+}
+
+/** Extract touched file paths from one tool/call event's arguments. */
+export function extractTouchedPath(name: string, argumentsText: string): string | undefined {
+  if (!TOUCHED_FILE_TOOL.test(name)) return undefined
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(argumentsText)
+  } catch {
+    return undefined
+  }
+  if (typeof parsed !== 'object' || parsed === null) return undefined
+  const record = parsed as Record<string, unknown>
+  for (const key of PATH_KEYS) {
+    const value = record[key]
+    if (typeof value === 'string' && value.length > 0) return value
+  }
+  return undefined
+}
+
+/** Result of one locator block build: text plus locator census for auditing. */
+export interface LocatorBlock {
+  readonly text: string
+  readonly spillFiles: number
+  readonly touchedFiles: number
+}
+
+/**
+ * Build the Exact Sources block for one shadowed range, or null when the
+ * range locates nothing concrete (no spill files and no touched files).
+ */
+export function buildLocatorBlock(
+  events: readonly SessionEvent[],
+  shadowedRange: { readonly start: number, readonly end: number },
+): LocatorBlock | null {
+  const spillFiles = new Set<string>()
+  const touchedFiles = new Set<string>()
+  for (let seq = shadowedRange.start; seq <= shadowedRange.end && seq < events.length; seq += 1) {
+    const event = events[seq]
+    if (event === undefined) continue
+    if (event.type === 'tool/call') {
+      const path = extractTouchedPath(event.data.name, event.data.arguments)
+      if (path !== undefined) touchedFiles.add(path)
+      continue
+    }
+    if (event.type === 'tool/result' || event.type === 'user/message') {
+      // tool/result nests content under data.message; user/message carries it directly.
+      const data = event.data as { content?: unknown, message?: { content?: unknown } }
+      const content = Array.isArray(data.content) ? data.content : data.message?.content
+      if (!Array.isArray(content)) continue
+      for (const block of content) {
+        if (block.type === 'text') {
+          for (const path of extractSpillPaths(block.text)) spillFiles.add(path)
+        }
+      }
+    }
+  }
+  if (spillFiles.size === 0 && touchedFiles.size === 0) return null
+  const lines = [
+    '## Exact Sources (locators)',
+    `- seq range: ${String(shadowedRange.start)}-${String(shadowedRange.end)}`,
+    ...[...spillFiles].map(path => `- spill file: ${path}`),
+    ...[...touchedFiles].map(path => `- file touched: ${path}`),
+    '(Use `read <spill file>` or `context_compression_retrieve` with a `session://` source to restore exact text.)',
+  ]
+  return {
+    text: lines.join('\n'),
+    spillFiles: spillFiles.size,
+    touchedFiles: touchedFiles.size,
+  }
+}

package/packages/selector/src/runtime/tokenpilot/read-state.ts

@@ -0,0 +1,77 @@
+/**
+ * TokenPilot-inspired R2/R3: read-state semantics and clustered omission
+ * markers.
+ *
+ * A read result is `superseded` when a later write-style tool call mutated the
+ * same file: its full text can no longer match the file the model would read
+ * again, so aggressive aging is safe. Superseded reads may compress to a small
+ * aggregate placeholder instead of the ordinary historical placeholder.
+ */
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+
+/** Write-style tool names whose success supersedes earlier reads. */
+const WRITE_TOOLS = /(?:^|[-_])?(?:write|edit|apply_patch|file_write|file_edit|str_replace|replace|multiedit)(?:$|[-_])/i
+const PATH_KEYS = ['path', 'file_path'] as const
+
+/** Parse one path out of a tool-call arguments JSON blob. */
+export function toolCallPath(argumentsText: string): string | undefined {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(argumentsText)
+  } catch {
+    return undefined
+  }
+  if (typeof parsed !== 'object' || parsed === null) return undefined
+  const record = parsed as Record<string, unknown>
+  for (const key of PATH_KEYS) {
+    const value = record[key]
+    if (typeof value === 'string' && value.length > 0) return value
+  }
+  return undefined
+}
+
+/**
+ * Decide whether an oversized read result was superseded by a later mutation
+ * of the same file. `readPath` is the read call's target path; events after
+ * `readSeq` are scanned for a write-style call on it.
+ */
+export function isSupersededRead(
+  events: readonly SessionEvent[],
+  readSeq: number,
+  readPath: string | undefined,
+): boolean {
+  if (readPath === undefined) return false
+  for (let seq = readSeq + 1; seq < events.length; seq += 1) {
+    const event = events[seq]
+    if (event?.type !== 'tool/call') continue
+    if (!WRITE_TOOLS.test(event.data.name)) continue
+    if (toolCallPath(event.data.arguments) === readPath) return true
+  }
+  return false
+}
+
+/** Error/warning/info line classifiers used by the omission summary. */
+const ERROR_LINE = /\b(error|failed|failure|fatal|exception|traceback|cannot|unable|denied)\b/i
+const WARN_LINE = /\b(warn|warning|deprecated)\b/i
+
+/**
+ * Cluster one omitted line-count into an error/warn/info census appended to a
+ * placeholder marker, giving the model meta-knowledge about what was dropped.
+ */
+export function clusterOmittedLines(text: string, omittedLines: number): string | undefined {
+  if (omittedLines <= 0) return undefined
+  let errors = 0
+  let warns = 0
+  let infos = 0
+  for (const line of text.split('\n')) {
+    if (ERROR_LINE.test(line)) errors += 1
+    else if (WARN_LINE.test(line)) warns += 1
+    else infos += 1
+  }
+  const parts: string[] = []
+  if (errors > 0) parts.push(`${String(errors)} error`)
+  if (warns > 0) parts.push(`${String(warns)} warn`)
+  if (infos > 0) parts.push(`${String(infos)} info`)
+  if (parts.length === 0) return undefined
+  return `${String(omittedLines)} lines omitted (${parts.join(', ')})`
+}