dsh-context-compression-improved 0.1.1 → 0.2.1

package/packages/selector/tests/runtime/session-events.spec.ts

@@ -0,0 +1,27 @@
+import { describe, expect, it, vi } from 'vitest'
+import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
+import { sessionEvents } from '../../src/runtime/session-events.ts'
+
+const EVENTS = Object.freeze([
+  Object.freeze({ seq: 0, time: 1, type: 'turn/start', data: { turn: 1 } }),
+]) as unknown as readonly SessionEvent[]
+
+describe('sessionEvents', () => {
+  it('retains the rc.2 events accessor fallback', () => {
+    const session = { events: EVENTS } as Session
+    expect(sessionEvents(session)).toBe(EVENTS)
+  })
+
+  it('prefers the Alpha 5 public snapshotEvents API and preserves its receiver', () => {
+    const modern = {
+      marker: EVENTS,
+      events: undefined,
+      snapshotEvents(this: { marker: readonly SessionEvent[] }) {
+        return this.marker
+      },
+    }
+    const snapshot = vi.spyOn(modern, 'snapshotEvents')
+    expect(sessionEvents(modern as unknown as Session)).toBe(EVENTS)
+    expect(snapshot).toHaveBeenCalledOnce()
+  })
+})

package/packages/selector/tests/runtime/tokenizer-golden.spec.ts

@@ -0,0 +1,53 @@
+/** Node loader equivalence against Hugging Face Python tokenizer golden counts. */
+
+import { readFileSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { describe, expect, it } from 'vitest'
+import { deepSeekV4TokenizerForModel } from '../../src/deepseek-v4-tokenizer.ts'
+
+interface GoldenTokenizerCase {
+  readonly label: string
+  readonly text: string
+  readonly counts: Readonly<Record<string, number>>
+}
+
+interface GoldenFixture {
+  readonly generator: string
+  readonly revision: string
+  readonly tokenizers: Readonly<Record<string, string>>
+  readonly cases: readonly GoldenTokenizerCase[]
+}
+
+const fixture = JSON.parse(readFileSync(
+  fileURLToPath(new URL('fixtures/tokenizer-golden.json', import.meta.url)),
+  'utf8',
+)) as GoldenFixture
+
+const MODEL_BY_TOKENIZER: Readonly<Record<string, string>> = {
+  'deepseek-ai/DeepSeek-V4-Pro': 'deepseek-v4-pro',
+  'deepseek-ai/DeepSeek-V4-Flash-Vision-Exp': 'deepseek-v4-flash-vision-exp',
+}
+
+describe('bundled tokenizers match Hugging Face Python golden counts', () => {
+  it('carries cases for both pinned artifacts with distinct vocabularies', () => {
+    expect(Object.keys(fixture.tokenizers).sort()).toEqual(Object.keys(MODEL_BY_TOKENIZER).sort())
+    // At least one case must prove the vocabularies differ (vision-only special token).
+    const imageCase = fixture.cases.find(entry => entry.text.includes('deepseek_image'))
+    expect(imageCase).toBeDefined()
+    expect(imageCase?.counts['deepseek-ai/DeepSeek-V4-Pro'])
+      .not.toBe(imageCase?.counts['deepseek-ai/DeepSeek-V4-Flash-Vision-Exp'])
+  })
+
+  it.each(fixture.cases)('counts "$label" identically in every tokenizer', (entry) => {
+    for (const [repository, expected] of Object.entries(entry.counts)) {
+      const modelId = MODEL_BY_TOKENIZER[repository]
+      if (modelId === undefined) throw new Error(`golden fixture cites unknown tokenizer ${repository}`)
+      const count = deepSeekV4TokenizerForModel(modelId)?.countText(entry.text)
+      expect(count, `${repository} must resolve for ${modelId}`).toMatchObject({
+        kind: 'exact-tokenizer',
+        tokenizerId: repository,
+      })
+      expect(count?.tokens).toBe(expected)
+    }
+  })
+})

package/packages/selector/tests/runtime/tokenpilot/dedup.spec.ts

@@ -0,0 +1,52 @@
+/**
+ * TokenPilot-inspired A1 unit coverage: dedup hashing, canonicalization,
+ * table eviction, and pointer placeholder shape.
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  DedupeTable,
+  canonicalizeForDedupe,
+  dedupeHash,
+  dedupePlaceholder,
+  flattenPlainText,
+} from '../../../src/runtime/tokenpilot/dedup.ts'
+
+describe('tokenpilot dedup helpers', () => {
+  it('canonicalizes trim-eol without touching inner content', () => {
+    expect(canonicalizeForDedupe('a \n b  \n\n', 'trim-eol')).toBe('a\n b')
+    expect(canonicalizeForDedupe('a\nb', 'exact')).toBe('a\nb')
+    expect(dedupeHash('a \n', 'trim-eol')).toBe(dedupeHash('a', 'trim-eol'))
+    expect(dedupeHash('a \n', 'exact')).not.toBe(dedupeHash('a', 'exact'))
+  })
+
+  it('flattens all-text content and rejects rich blocks', () => {
+    expect(flattenPlainText([{ type: 'text', text: 'a' }, { type: 'text', text: 'b' }])).toBe('ab')
+    expect(flattenPlainText([{ type: 'text', text: 'a' }, { type: 'image', source: {} } as never])).toBeUndefined()
+  })
+
+  it('records first occurrences, keeps them, and evicts oldest beyond the cap', () => {
+    const table = new DedupeTable(2)
+    table.record('h1', { seq: 10, sourceRef: 'session://s/event/10', toolName: 'bash', originalChars: 100 })
+    expect(table.get('h1')?.seq).toBe(10)
+    // Re-recording the same hash never replaces the first occurrence.
+    table.record('h1', { seq: 99, sourceRef: 'session://s/event/99', toolName: 'bash', originalChars: 5 })
+    expect(table.get('h1')?.seq).toBe(10)
+    table.record('h2', { seq: 20, sourceRef: 'session://s/event/20', toolName: 'bash', originalChars: 1 })
+    table.record('h3', { seq: 30, sourceRef: 'session://s/event/30', toolName: 'bash', originalChars: 1 })
+    // Insertion-order eviction drops h1, the oldest.
+    expect(table.get('h1')).toBeUndefined()
+    expect(table.get('h2')?.seq).toBe(20)
+    expect(table.get('h3')?.seq).toBe(30)
+  })
+
+  it('points the placeholder at the first occurrence with a retrieval hint', () => {
+    const text = dedupePlaceholder(
+      { seq: 10, sourceRef: 'session://s1/event/10', toolName: 'git_status', originalChars: 4096 },
+      4096,
+    )
+    expect(text).toContain('identical to the earlier git_status result')
+    expect(text).toContain('session://s1/event/10')
+    expect(text).toContain('original_chars=4096')
+    expect(text).toContain('context_compression_retrieve')
+  })
+})

package/packages/selector/tests/runtime/tokenpilot/estimator.spec.ts

@@ -0,0 +1,56 @@
+/**
+ * TokenPilot-inspired E1 unit coverage: pure estimator helpers. The channels
+ * themselves are integration-level (host/direct transports) and are covered
+ * by the fail-open contract — every malformed answer resolves to no verdicts.
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  backoffCooldownMs,
+  buildEstimatorUserPrompt,
+  isCoolingDown,
+  parseEstimatorAnswer,
+} from '../../../src/runtime/tokenpilot/estimator.ts'
+
+describe('tokenpilot estimator helpers', () => {
+  it('parses well-formed verdict arrays, tolerating surrounding prose', () => {
+    expect(parseEstimatorAnswer('[{"seq":12,"expired":true},{"seq":13,"expired":false}]')).toEqual([
+      { seq: 12, expired: true },
+      { seq: 13, expired: false },
+    ])
+    expect(parseEstimatorAnswer('Verdicts: [{"seq":7,"expired":true}] done')).toEqual([
+      { seq: 7, expired: true },
+    ])
+  })
+
+  it('yields no verdicts for malformed answers', () => {
+    expect(parseEstimatorAnswer('')).toEqual([])
+    expect(parseEstimatorAnswer('no json here')).toEqual([])
+    expect(parseEstimatorAnswer('[{"seq":"12","expired":true}]')).toEqual([])
+    expect(parseEstimatorAnswer('[{"seq":12}]')).toEqual([])
+    expect(parseEstimatorAnswer('not json [')).toEqual([])
+  })
+
+  it('builds a path-only user prompt without file content', () => {
+    const prompt = buildEstimatorUserPrompt([
+      { seq: 4, path: 'src/a.ts', turn: 2 },
+      { seq: 9, path: 'docs/b.md', turn: 3 },
+    ])
+    expect(prompt).toContain('"seq":4')
+    expect(prompt).toContain('"path":"src/a.ts"')
+    expect(prompt).not.toMatch(/content/i)
+  })
+
+  it('backs off exponentially with a five-minute cap', () => {
+    expect(backoffCooldownMs(0)).toBe(1_000)
+    expect(backoffCooldownMs(1)).toBe(1_000)
+    expect(backoffCooldownMs(2)).toBe(2_000)
+    expect(backoffCooldownMs(3)).toBe(4_000)
+    expect(backoffCooldownMs(20)).toBe(300_000)
+  })
+
+  it('treats missing failure state as never cooling down', () => {
+    expect(isCoolingDown(undefined, Date.now())).toBe(false)
+    expect(isCoolingDown({ failures: 1, cooldownUntil: Date.now() - 1 }, Date.now())).toBe(false)
+    expect(isCoolingDown({ failures: 1, cooldownUntil: Date.now() + 60_000 }, Date.now())).toBe(true)
+  })
+})

package/packages/selector/tests/runtime/tokenpilot/locator.spec.ts

@@ -0,0 +1,76 @@
+/**
+ * TokenPilot-inspired A2 unit coverage: compaction trace resolution, spill and
+ * touched-file extraction, and the Exact Sources block shape.
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  buildLocatorBlock,
+  extractSpillPaths,
+  extractTouchedPath,
+  findCompactionTrace,
+} from '../../../src/runtime/tokenpilot/locator.ts'
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+
+/** Minimal event literal; the locator only reads seq/type/data fields. */
+function event(seq: number, data: Record<string, unknown>): SessionEvent {
+  return { seq, type: 'tool/call', data } as unknown as SessionEvent
+}
+
+const SHADOWED = { start: 3, end: 9 }
+
+const EVENTS: SessionEvent[] = [
+  event(0, {}),
+  event(1, {}),
+  { seq: 2, type: 'compaction/start', data: { compactionId: 'c1' } } as unknown as SessionEvent,
+  event(3, { name: 'read', arguments: '{"path":"src/a.ts"}' }),
+  { seq: 4, type: 'tool/result', data: { message: { content: [{ type: 'text', text: 'omitted 100 bytes. Full formatted result stored at: /tmp/spill/a.txt. Use read to restore.' }] } } } as unknown as SessionEvent,
+  event(5, { name: 'grep', arguments: '{"path":"src/a.ts"}' }),
+  event(6, { name: 'edit', arguments: '{"file_path":"src/a.ts"}' }),
+  { seq: 7, type: 'user/message', data: { content: [{ type: 'text', text: 'stored at: /tmp/spill/b.txt' }] } } as unknown as SessionEvent,
+  event(8, { name: 'bash', arguments: '{"command":"ls"}' }),
+  { seq: 9, type: 'compaction/summary', data: { compactionId: 'c1', shadowedRange: SHADOWED, shadowedTokenCount: 100, provider: 'deepseek', model: 'v4' } } as unknown as SessionEvent,
+  { seq: 10, type: 'compaction/end', data: { compactionId: 'c1' } } as unknown as SessionEvent,
+]
+
+describe('tokenpilot summary locator helpers', () => {
+  it('resolves the latest compaction/summary trace for one compaction id', () => {
+    const trace = findCompactionTrace(EVENTS, 'c1')
+    expect(trace?.summarySeq).toBe(9)
+    expect(trace?.summaryShadowedRange).toEqual(SHADOWED)
+    expect(findCompactionTrace(EVENTS, 'unknown')).toBeUndefined()
+  })
+
+  it('extracts spill paths, stripping trailing punctuation and deduplicating', () => {
+    expect(extractSpillPaths('stored at: /tmp/a.txt. Use read')).toEqual(['/tmp/a.txt'])
+    expect(extractSpillPaths('stored at: /tmp/a.txt, stored at: /tmp/a.txt')).toEqual(['/tmp/a.txt', '/tmp/a.txt'])
+    expect(extractSpillPaths('nothing here')).toEqual([])
+  })
+
+  it('extracts touched paths only from read/grep-style calls', () => {
+    expect(extractTouchedPath('read', '{"path":"src/a.ts"}')).toBe('src/a.ts')
+    expect(extractTouchedPath('str_replace_editor', '{"file_path":"src/b.ts"}')).toBe('src/b.ts')
+    expect(extractTouchedPath('bash', '{"path":"src/a.ts"}')).toBeUndefined()
+    expect(extractTouchedPath('read', 'not json')).toBeUndefined()
+  })
+
+  it('builds an Exact Sources block over the shadowed range', () => {
+    const block = buildLocatorBlock(EVENTS, SHADOWED)
+    expect(block).not.toBeNull()
+    expect(block?.text).toContain('## Exact Sources (locators)')
+    expect(block?.text).toContain('- seq range: 3-9')
+    expect(block?.text).toContain('- spill file: /tmp/spill/a.txt')
+    expect(block?.text).toContain('- file touched: src/a.ts')
+    expect(block?.text).toContain('context_compression_retrieve')
+    expect(block?.spillFiles).toBeGreaterThanOrEqual(1)
+    expect(block?.touchedFiles).toBeGreaterThanOrEqual(1)
+  })
+
+  it('returns null when the range locates nothing concrete', () => {
+    const empty: SessionEvent[] = [
+      { seq: 0, type: 'compaction/summary', data: { compactionId: 'c1', shadowedRange: { start: 1, end: 2 }, shadowedTokenCount: 1, provider: 'p', model: 'm' } } as unknown as SessionEvent,
+      event(1, { name: 'bash', arguments: '{"command":"ls"}' }),
+      event(2, {}),
+    ]
+    expect(buildLocatorBlock(empty, { start: 1, end: 2 })).toBeNull()
+  })
+})

package/packages/selector/tests/runtime/tokenpilot/profile-baseline.spec.ts

@@ -0,0 +1,100 @@
+/**
+ * TokenPilot-inspired preset: policy derivation plus the backward-compat
+ * golden. The fixture freezes the resolvePolicy output of every pre-existing
+ * profile captured BEFORE this preset existed; any byte difference is a
+ * compatibility regression, however intentional it feels.
+ */
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import {
+  parseContextCompressionSettings,
+  resolveConfig,
+  resolvePolicy,
+} from '../../../src/pruner.ts'
+
+const baseline = JSON.parse(readFileSync(
+  join(import.meta.dirname, '..', 'fixtures', 'profile-baseline.json'),
+  'utf8',
+)) as {
+  profiles: Record<string, Record<string, unknown>>
+}
+
+const OLD_PROFILES = ['off', 'native', 'balanced', 'cache-strict', 'savings', 'adaptive', 'custom'] as const
+const OPTION_CASES = [
+  { label: 'defaults', options: {} },
+  { label: 'linkage', options: { contextWindowTokens: 128_000, autoCompactThresholdPercent: 80 } },
+] as const
+
+describe('tokenpilot-inspired preset', () => {
+  it('keeps every pre-existing profile byte-identical to the pre-preset baseline', () => {
+    const config = resolveConfig({})
+    for (const profile of OLD_PROFILES) {
+      for (const optionCase of OPTION_CASES) {
+        const key = optionCase.label
+        expect(
+          resolvePolicy(config, profile, undefined, optionCase.options),
+          `${profile}/${key}`,
+        ).toStrictEqual(baseline.profiles[profile]?.[key])
+      }
+    }
+  })
+
+  it('derives from the balanced baseline and injects the capability matrix', () => {
+    const config = resolveConfig({})
+    const policy = resolvePolicy(config, 'tokenpilot-inspired')
+    expect(policy.profile).toBe('tokenpilot-inspired')
+    expect(policy.freshEnabled).toBe(true)
+    expect(policy.aggregateEnabled).toBe(true)
+    expect(policy.historyMode).toBe('routine')
+    expect(policy.freshTriggerTokens).toBe(8_192)
+    expect(policy.freshTargetTokens).toBe(3_072)
+    expect(policy.aggregateTriggerTokens).toBe(32_768)
+    expect(policy.aggregateTargetTokens).toBe(12_288)
+    expect(policy.historyTriggerTokens).toBe(500_000)
+    expect(policy.historyMinReclaimTokens).toBe(96_000)
+    expect(policy.presetOptions).toStrictEqual({
+      noNetSavingsGuard: true,
+      skipReductionRecovery: true,
+      dedupeToolResults: true,
+      summaryLocator: true,
+      prefixStabilizer: true,
+      readState: true,
+      estimator: { mode: '' },
+    })
+    // Other profiles never carry the capability matrix.
+    expect(resolvePolicy(config, 'balanced').presetOptions).toBeUndefined()
+  })
+
+  it('links Auto Compact watermarks like balanced does', () => {
+    const config = resolveConfig({})
+    const policy = resolvePolicy(config, 'tokenpilot-inspired', undefined, {
+      contextWindowTokens: 128_000,
+      autoCompactThresholdPercent: 80,
+    })
+    expect(policy.autoCompactTokens).toBe(102_400)
+    expect(policy.microDeadlineTokens).toBe(89_600)
+    expect(policy.historyTriggerTokens).toBe(64_000)
+    expect(policy.historyMinReclaimTokens).toBe(12_288)
+    expect(policy.historyKeepRecentTokens).toBe(8_192)
+  })
+
+  it('accepts persisted presetOptions overrides and rejects unknown keys', () => {
+    const base = {
+      profile: 'balanced',
+      custom: { version: 3, unit: 'tokens', fresh: { enabled: true, trigger: 2, target: 1 }, aggregate: { enabled: false, trigger: 2, target: 1 }, history: { enabled: false, trigger: 2, keepRecentToolCalls: 1, keepRecentTokens: 1, minReclaim: 1 }, prefixPolicy: 'preserve', tailTrim: { enabled: false, trigger: 2 } },
+    }
+    const doc = {
+      ...base,
+      presetOptions: { dedupeToolResults: false, estimatorMode: 'host' },
+    }
+    expect((parseContextCompressionSettings(structuredClone(doc)) as { presetOptions?: unknown }).presetOptions).toStrictEqual({
+      dedupeToolResults: false,
+      estimatorMode: 'host',
+    })
+    expect(() => parseContextCompressionSettings({
+      ...base,
+      presetOptions: { nope: true },
+    })).toThrow()
+  })
+})

package/packages/selector/tests/runtime/tokenpilot/read-state.spec.ts

@@ -0,0 +1,58 @@
+/**
+ * TokenPilot-inspired R2/R3 unit coverage: superseded-read classification and
+ * clustered omission summaries.
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  clusterOmittedLines,
+  isSupersededRead,
+  toolCallPath,
+} from '../../../src/runtime/tokenpilot/read-state.ts'
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+
+function callEvent(seq: number, name: string, args: Record<string, unknown>): SessionEvent {
+  return { seq, type: 'tool/call', data: { name, arguments: JSON.stringify(args) } } as unknown as SessionEvent
+}
+
+function resultEvent(seq: number): SessionEvent {
+  return { seq, type: 'tool/result', data: { message: { content: [{ type: 'text', text: 'x' }] } } } as unknown as SessionEvent
+}
+
+describe('tokenpilot read-state helpers', () => {
+  it('parses paths from path and file_path argument keys', () => {
+    expect(toolCallPath('{"path":"a.ts"}')).toBe('a.ts')
+    expect(toolCallPath('{"file_path":"b.ts"}')).toBe('b.ts')
+    expect(toolCallPath('not json')).toBeUndefined()
+    expect(toolCallPath('{"path":42}')).toBeUndefined()
+  })
+
+  it('marks a read superseded only by a later write-style call on the same path', () => {
+    const events: SessionEvent[] = [
+      resultEvent(1),
+      callEvent(2, 'read', { path: 'src/a.ts' }),
+      callEvent(3, 'bash', { command: 'echo write src/a.ts' }),
+      callEvent(4, 'edit', { path: 'src/other.ts' }),
+    ]
+    expect(isSupersededRead(events, 2, 'src/a.ts')).toBe(false)
+    const events2: SessionEvent[] = [...events, callEvent(5, 'file_edit', { path: 'src/a.ts' })]
+    expect(isSupersededRead(events2, 2, 'src/a.ts')).toBe(true)
+    expect(isSupersededRead(events, 2, undefined)).toBe(false)
+    // Mutations before the read never supersede it.
+    const events3: SessionEvent[] = [callEvent(0, 'edit', { path: 'src/a.ts' }), resultEvent(1), callEvent(2, 'read', { path: 'src/a.ts' })]
+    expect(isSupersededRead(events3, 2, 'src/a.ts')).toBe(false)
+  })
+
+  it('clusters omitted lines into an error/warn/info census', () => {
+    const text = [
+      'ok line',
+      'Error: bad thing',
+      'warning: careful',
+      'another ok',
+      'FATAL: worse',
+      'warn: also',
+    ].join('\n')
+    expect(clusterOmittedLines(text, 6)).toBe('6 lines omitted (2 error, 2 warn, 2 info)')
+    expect(clusterOmittedLines('fine', 1)).toBe('1 lines omitted (1 info)')
+    expect(clusterOmittedLines('fine', 0)).toBeUndefined()
+  })
+})

package/packages/selector/tests/runtime/value.spec.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest'
+import { assertNever, deepFreeze } from '../../src/runtime/value.ts'
+
+describe('version-neutral runtime values', () => {
+  it('freezes nested and cyclic data without freezing a live AbortSignal', () => {
+    const controller = new AbortController()
+    const value: { nested: { value: number }, self?: unknown, signal: AbortSignal } = {
+      nested: { value: 1 },
+      signal: controller.signal,
+    }
+    value.self = value
+
+    expect(deepFreeze(value)).toBe(value)
+    expect(Object.isFrozen(value)).toBe(true)
+    expect(Object.isFrozen(value.nested)).toBe(true)
+    expect(Object.isFrozen(controller.signal)).toBe(false)
+  })
+
+  it('reports an escaped closed-union member with its switch context', () => {
+    expect(() => assertNever({ kind: 'unexpected' } as never, 'value test'))
+      .toThrow('unreachable variant in value test: {"kind":"unexpected"}')
+  })
+})