dsh-context-compression-improved 0.2.1 → 0.3.1

package/scripts/{verify-release.ts → verify-release.mjs}

@@ -1,4 +1,3 @@
-import { execFileSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
 import { readdir, readFile, stat } from 'node:fs/promises'
 import { join, relative } from 'node:path'
@@ -6,42 +5,62 @@ import { fileURLToPath } from 'node:url'
 
 const root = fileURLToPath(new URL('..', import.meta.url))
 
-const fail = (message: string): never => {
+const fail = (message) => {
   throw new Error(`release verification: ${message}`)
 }
 
-const json = async (path: string) => JSON.parse(await readFile(path, 'utf8')) as Record<string, unknown>
+const json = async (path) => JSON.parse(await readFile(path, 'utf8'))
 const rootPackage = await json(join(root, 'package.json'))
-const selectorPackage = await json(join(root, 'packages/selector/package.json')) as Record<string, Record<string, unknown>>
+const selectorPackage = await json(join(root, 'packages/selector/package.json'))
 
-const rootScripts = rootPackage.scripts as Record<string, string> | undefined
-const selectorScripts = selectorPackage.scripts as Record<string, string> | undefined
-
-if (!(rootScripts?.typecheck as string)?.includes('pnpm run typecheck:tests')
-  || rootScripts?.['typecheck:tests'] !== 'tsc --noEmit -p tsconfig.tests.json') {
+if (!rootPackage.scripts?.typecheck?.includes('pnpm run typecheck:tests')
+  || rootPackage.scripts?.['typecheck:tests'] !== 'tsc --noEmit -p tsconfig.tests.json') {
   fail('root typecheck must include the strict active-test TypeScript gate')
 }
-if (selectorScripts?.test
+// The root manifest is the install surface for git installs: one package
+// carries the runtime, the settings UI and the bundle patch.
+if (rootPackage.name !== 'dsh-context-compression-improved') fail('unexpected root package name')
+for (const specifier of ['.', './invariant', './pruner', './client']) {
+  if (rootPackage.exports?.[specifier] === undefined) {
+    fail(`root exports are missing the ${specifier} specifier`)
+  }
+}
+if (rootPackage.dsh?.bundle?.patch !== './packages/selector/cordis.patch.yml') {
+  fail('root must declare the DSH Bundle patch of the selector package')
+}
+for (const dependency of ['@huggingface/tokenizers', 'js-yaml']) {
+  if (rootPackage.dependencies?.[dependency] === undefined) {
+    fail(`root dependencies are missing ${dependency}`)
+  }
+}
+for (const specifier of ['.', './invariant', './pruner', './client']) {
+  if (selectorPackage.exports?.[specifier] === undefined) {
+    fail(`selector exports are missing the ${specifier} specifier`)
+  }
+}
+if (selectorPackage.scripts?.test
   !== 'vitest run --root ../.. --config vitest.config.ts --project runtime --project selector-host --project selector-client') {
-  fail('Package-local test script is not the verified root project command')
+  fail('Selector package-local test script is not the verified root project command')
 }
 const ci = await readFile(join(root, '.github/workflows/ci.yml'), 'utf8')
-if (!ci.includes('pnpm --filter dsh-context-compression-improved test')) {
-  fail('CI lacks the package-local gate: pnpm --filter dsh-context-compression-improved test')
+for (const required of [
+  'pnpm --filter dsh-context-compression-improved test',
+]) {
+  if (!ci.includes(required)) fail(`CI lacks package-local gate: ${required}`)
 }
 // The packed release E2E is the release gate that actually installs the
 // tarballs: verify it cannot be silently dropped or defanged. It must exist
 // as a root script, CI must run exactly that script, the script must default
 // to fail-closed release mode, and that mode must refuse every skip/null
 // lifecycle outcome.
-if (rootScripts?.['test:e2e:packed'] !== 'node scripts-dist/packed-install-e2e.js') {
+if (rootPackage.scripts?.['test:e2e:packed'] !== 'node scripts/packed-install-e2e.mjs') {
   fail('root test:e2e:packed script is missing or does not run the packed E2E directly')
 }
 if (!ci.includes('pnpm run test:e2e:packed') && !ci.includes('pnpm test:e2e:packed')) {
   fail('CI does not run the packed release E2E gate')
 }
-const packedE2e = await readFile(join(root, 'scripts/packed-install-e2e.ts'), 'utf8')
-if (!/const e2eMode\s*(:\s*'dev'\s*\|\s*'release')?\s*=\s*process\.env\.DSH_E2E_MODE === 'dev' \? 'dev' : 'release'/u.test(packedE2e)) {
+const packedE2e = await readFile(join(root, 'scripts/packed-install-e2e.mjs'), 'utf8')
+if (!/const e2eMode = process\.env\.DSH_E2E_MODE === 'dev' \? 'dev' : 'release'/u.test(packedE2e)) {
   fail('packed E2E must default to release mode (dev only via an explicit DSH_E2E_MODE)')
 }
 for (const failClosed of [
@@ -52,7 +71,7 @@ for (const failClosed of [
     fail(`packed E2E release mode lost its fail-closed guard: ${failClosed}`)
   }
 }
-const packedComponents = await readFile(join(root, 'scripts/packed-components-smoke.ts'), 'utf8')
+const packedComponents = await readFile(join(root, 'scripts/packed-components-smoke.mjs'), 'utf8')
 for (const required of [
   'Runtime.measureForCompaction(visionCtx, visionImage)',
   "estimatedImageCount?.kind === 'tokenizer-estimate'",
@@ -77,36 +96,30 @@ for (const required of [
 }
 await stat(join(root, 'tsconfig.tests.json'))
 
-if ((selectorPackage.dsh as Record<string, Record<string, string>>)?.bundle?.patch !== './cordis.patch.yml') {
+if (selectorPackage.dsh?.bundle?.patch !== './cordis.patch.yml') {
   fail('selector must declare the DSH Bundle patch')
 }
-if ((selectorPackage.files as string[] | undefined)?.some((entry: string) => entry.endsWith('.css'))) {
+if (selectorPackage.files?.some((entry) => entry.endsWith('.css'))) {
   fail('selector must not rely on separately served CSS assets')
 }
-if ((selectorPackage.name as unknown as string) !== 'dsh-context-compression-improved') fail('unexpected package name')
-if ((selectorPackage.publishConfig as Record<string, string>)?.access !== 'public') fail('publish access is not public')
-if ((selectorPackage.publishConfig as Record<string, string>)?.tag !== 'latest') fail('publish tag is not latest')
+if (selectorPackage.name !== 'dsh-context-compression-improved') fail('unexpected selector package name')
+if (selectorPackage.publishConfig?.access !== 'public') fail('selector publish access is not public')
+if (selectorPackage.publishConfig?.tag !== 'latest') fail('selector publish tag is not latest')
 for (const peer of [
   '@deepseek-ai/dsh-command-compact',
   '@deepseek-ai/dsh-compaction-basic',
 ]) {
-  if (((selectorPackage.peerDependencies as Record<string, string>) ?? {})[peer] !== '>=0.1.1-rc.2 <0.2.0') {
+  if (selectorPackage.peerDependencies?.[peer] !== '>=0.1.5-rc.2 <0.2.0-0') {
     fail(`selector peer ${peer} is missing or outside the verified range`)
   }
 }
-const notice = await readFile(join(root, 'packages/selector/THIRD_PARTY_NOTICES.md'), 'utf8')
+const packageRoot = join(root, 'packages/selector')
+const notice = await readFile(join(packageRoot, 'THIRD_PARTY_NOTICES.md'), 'utf8')
 if (!notice.includes('Copyright (c) 2026 DeepSeek')) {
   fail('packages/selector does not carry the full DeepSeek Harness MIT notice')
 }
 
-interface AssetManifest {
-  directory: string
-  repository: string
-  modelIds: string
-  revision?: string
-}
-
-const assetManifests: AssetManifest[] = [
+const assetManifests = [
   {
     directory: 'deepseek-v4',
     repository: 'deepseek-ai/DeepSeek-V4-Pro',
@@ -121,12 +134,7 @@ const assetManifests: AssetManifest[] = [
 ]
 for (const expected of assetManifests) {
   const assetRoot = join(root, 'packages/selector/assets', expected.directory)
-  const manifest = await json(join(assetRoot, 'manifest.json')) as {
-    repository: string
-    modelIds: string[]
-    revision?: string
-    files: Record<string, { bytes: number; sha256: string }>
-  }
+  const manifest = await json(join(assetRoot, 'manifest.json'))
   if (manifest.repository !== expected.repository) fail(`${expected.directory} manifest repository differs`)
   if (!JSON.stringify(manifest.modelIds).includes(expected.modelIds)) {
     fail(`${expected.directory} manifest model ids differ`)
@@ -139,11 +147,15 @@ for (const expected of assetManifests) {
     const hash = createHash('sha256').update(bytes).digest('hex')
     if (bytes.byteLength !== descriptor.bytes) fail(`${expected.directory}/${name} byte length differs from manifest`)
     if (hash !== descriptor.sha256) fail(`${expected.directory}/${name} SHA-256 differs from manifest`)
+    // A checkout with core.autocrlf=true can rewrite these pinned bytes as CRLF;
+    // the `-text` attribute in .gitattributes prevents it, but only while the
+    // pattern still matches the path. Catch it here rather than in a consumer
+    // whose tokenizer would fail its own integrity check at run time.
+    if (bytes.includes(0x0d)) fail(`${expected.directory}/${name} contains CR bytes; the asset was rewritten`)
   }
-  if (!(selectorPackage.files as string[] | undefined)?.some(
-    entry => entry === 'assets' || entry === `assets/${expected.directory}/*`,
-  )) {
-    fail(`package files list omits assets for ${expected.directory}`)
+  const assetEntries = selectorPackage.files ?? []
+  if (!assetEntries.some(entry => entry === 'assets' || entry.startsWith('assets/'))) {
+    fail('selector package files list omits the tokenizer assets')
   }
 }
 
@@ -153,11 +165,13 @@ const forbidden = [
   { pattern: /@deepseek-ai\/[^'"\s]+\/src(?:\/|['"])/u, label: 'Harness source subpath import' },
   { pattern: /(?:\/home\/|[A-Za-z]:\\Users\\)/u, label: 'developer absolute path' },
   { pattern: /\.\.\/\.\.\/\.\.\/(?:core|packages)\//u, label: 'monorepo-relative source import' },
+  // The single-package merge must not leave a cross-package import behind.
+  { pattern: /from\s*['"]dsh-context-compression-improved-runtime/u, label: 'cross-package runtime import' },
 ]
 
-const walk = async (directory: string): Promise<string[]> => {
+const walk = async (directory) => {
   const entries = await readdir(directory, { withFileTypes: true })
-  const files: string[] = []
+  const files = []
   for (const entry of entries) {
     const path = join(directory, entry.name)
     if (entry.isDirectory()) files.push(...await walk(path))
@@ -175,52 +189,15 @@ for (const sourceRoot of sourceRoots) {
   }
 }
 
-const lib = join(root, 'packages/selector/lib')
+const lib = join(packageRoot, 'lib')
 if (!(await stat(lib)).isDirectory()) fail(`${relative(root, lib)} is missing; run build first`)
 for (const path of await walk(lib)) {
   if (path.endsWith('.map')) fail(`${relative(root, path)} is a source map`)
-}
-
-// A git install ships exactly the tracked tree, so the artifact graph the
-// entries import must be complete AND committed. tsdown splits a shared chunk
-// out of every entry that reuses a module, and an untracked chunk makes the
-// installed entry crash at module-load time with ERR_MODULE_NOT_FOUND — the
-// same failure class as a cross-package import, so it gets the same gate.
-// Only the import graph is gated: build by-products nothing imports (for
-// example lib/style.css, whose rules the client artifact already carries
-// inline) are neither shipped nor required.
-const libArtifacts = await walk(lib)
-const libNames = new Set(libArtifacts.map(path => relative(lib, path).replaceAll('\\', '/')))
-const libScripts = [...libNames].filter(name => name.endsWith('.js')).sort()
-const libAllowedSuffixes = ((selectorPackage.files as unknown as string[]) ?? [])
-  .filter(pattern => pattern.startsWith('lib/'))
-  .map(pattern => pattern.slice(pattern.lastIndexOf('*') + 1))
-if (libAllowedSuffixes.length === 0) fail('selector package files allowlist covers no lib artifact')
-const isAllowed = (name: string) => libAllowedSuffixes.some(suffix => name.endsWith(suffix))
-const importedArtifacts = new Set<string>()
-for (const name of libScripts) {
-  const text = await readFile(join(lib, name), 'utf8')
-  for (const match of text.matchAll(/from\s*["'](\.\/[^"']+)["']/gu)) {
-    const target = match[1]!.slice(2)
-    if (!libNames.has(target)) fail(`lib/${name} imports missing artifact ${match[1]}`)
-    importedArtifacts.add(target)
-  }
-}
-for (const name of [...libScripts, ...importedArtifacts].sort()) {
-  if (!isAllowed(name)) fail(`lib/${name} is part of the load graph but the package files allowlist would drop it`)
-}
-let trackedLib: Set<string> | undefined
-try {
-  trackedLib = new Set(execFileSync('git', ['ls-files', '--', 'packages/selector/lib'], { cwd: root, encoding: 'utf8' })
-    .split('\n').map(line => line.trim()).filter(Boolean))
-} catch (cause: unknown) {
-  fail(`cannot read the tracked artifact list with git: ${(cause as Error).message}`)
-}
-if (trackedLib === undefined || trackedLib.size === 0) fail('git tracks no packages/selector/lib artifact; a git install would ship an unbuilt package')
-for (const name of [...libScripts, ...importedArtifacts].sort()) {
-  const tracked = `packages/selector/lib/${name}`
-  if (!trackedLib!.has(tracked)) {
-    fail(`${tracked} is not committed; a git install would fetch an incomplete artifact graph`)
+  if (path.endsWith('.js')) {
+    const text = await readFile(path, 'utf8')
+    if (/from\s*["']dsh-context-compression-improved-runtime["']/u.test(text)) {
+      fail(`${relative(root, path)} still imports the removed runtime package`)
+    }
   }
 }
 
@@ -234,7 +211,7 @@ if (!clientArtifact.includes('data-plugin-css') || !clientArtifact.includes('doc
 if (/^\s*(?:import|export)\s/mu.test(clientArtifact)) fail('client.js contains ESM syntax')
 if (/(?:\/home\/|[A-Za-z]:\\Users\\)/u.test(clientArtifact)) fail('client.js contains a developer absolute path')
 if (clientArtifact.includes('sourceMappingURL')) fail('client.js contains a source map reference')
-const clientRequires = [...clientArtifact.matchAll(/require\("([^"]+)"\)/gu)].map(match => match[1]!).filter(Boolean)
+const clientRequires = [...clientArtifact.matchAll(/require\("([^"]+)"\)/gu)].map(match => match[1])
 const allowedClientRequires = new Set([
   'react',
   'react/jsx-runtime',
@@ -244,7 +221,7 @@ for (const dependency of clientRequires) {
   if (!allowedClientRequires.has(dependency)) fail(`client.js has unexpected external dependency ${dependency}`)
 }
 
-const collectSpecs = async (directory: string) => (await walk(directory))
+const collectSpecs = async (directory) => (await walk(directory))
   .filter(path => /\.spec\.tsx?$/u.test(path))
   .map(path => relative(root, path).replaceAll('\\', '/'))
   .sort()
@@ -289,11 +266,10 @@ for (const path of [...runtimeSpecs, ...selectorSpecs]) {
   }
 }
 
-const runtime = await import(new URL('../packages/selector/lib/pruner.js', import.meta.url).href)
-const defaults = runtime.DEFAULT_CUSTOM_COMPRESSION_POLICY as Record<string, Record<string, unknown>> | undefined
-if ((defaults?.history as Record<string, number>)?.trigger !== 500_000) fail('Custom History default is not 500000')
-if ((defaults?.tailTrim as Record<string, unknown>)?.trigger !== 700_000
-  || (defaults?.tailTrim as Record<string, unknown>)?.enabled !== false) {
+const runtime = await import(new URL('../packages/selector/lib/pruner.js', import.meta.url))
+const defaults = runtime.DEFAULT_CUSTOM_COMPRESSION_POLICY
+if (defaults?.history?.trigger !== 500_000) fail('Custom History default is not 500000')
+if (defaults?.tailTrim?.trigger !== 700_000 || defaults?.tailTrim?.enabled !== false) {
   fail('Custom TailTrim default is not disabled at 700000')
 }
 

package/tests/TEST_INVENTORY.md

@@ -5,17 +5,16 @@ legacy-test allowlist.
 
 ## Root suite
 
-- Runtime: `packages/selector/tests/runtime/**/*.spec.ts` under the `runtime`
-  project.
-- Selector Host: `cache-prefix-audit.spec.ts`, `estimator-catalog.spec.ts` plus
-  every `*.host.spec.ts` under the `selector-host` project.
+- Runtime: `packages/runtime/tests/**/*.spec.ts` under the `runtime` project.
+- Selector Host: `cache-prefix-audit.spec.ts` plus every `*.host.spec.ts` under
+  the `selector-host` project.
 - Selector client: every `*.client.spec.ts` or `*.client.spec.tsx` under the
   `selector-client` project.
 - Built browser artifact: `packages/selector/tests/built/client-artifact.spec.ts`
   under the separate `vitest.built.config.ts` gate, after `pnpm build`.
 
-The root `pnpm test`, the package-local `pnpm test` command, and
-`pnpm test:built` are release gates. `scripts-dist/verify-release.js` (compiled from `scripts/verify-release.ts`) compares the
+The root `pnpm test`, both package-local `pnpm test` commands, and
+`pnpm test:built` are release gates. `scripts/verify-release.mjs` compares the
 checked-in spec inventory with these project rules so a new test cannot be
 silently excluded.
 
@@ -37,6 +36,6 @@ Their supported behavior is covered by active public tests:
   service identity: `packages/selector/tests/preset-overlay-loader.e2e.host.spec.ts`;
 - package/export/tarball contract:
   `packages/selector/tests/public/package-contract.client.spec.ts` and
-  `scripts-dist/packed-install-e2e.js` (compiled from `scripts/packed-install-e2e.ts`);
+  `scripts/packed-install-e2e.mjs`;
 - profile and Custom editor behavior:
   `packages/selector/tests/profiles.client.spec.tsx`.

package/scripts/capture-profile-baseline.ts

@@ -1,80 +0,0 @@
-/**
- * One-off baseline capture: freeze the resolvePolicy output for the pre-existing
- * profiles (before 'tokenpilot-inspired' is added) into a JSON fixture used by
- * tests/runtime/tokenpilot/profile-baseline.spec.ts as the backward-compat golden.
- *
- * Usage: node scripts-dist/capture-profile-baseline.js [baseline-file]
- */
-import { writeFileSync } from 'node:fs'
-import { createRequire } from 'node:module'
-
-/** The slice of the built pruner entry this capture consumes. */
-interface PrunerRuntime {
-  resolveConfig: (input: Record<string, unknown>) => Record<string, unknown>
-  resolvePolicy: (
-    config: Record<string, unknown>,
-    profile: string,
-    custom: unknown,
-    options: Record<string, unknown>,
-  ) => Record<string, unknown>
-}
-
-/** One option matrix row. */
-interface OptionCase {
-  label: string
-  options: Record<string, unknown>
-}
-
-/** One custom-document matrix row. */
-interface CustomDocCase {
-  label: string
-  custom: unknown
-}
-
-/** The fixture shape `profile-baseline.spec.ts` reads back. */
-interface ProfileBaseline {
-  capturedAt: string
-  profiles: Record<string, Record<string, Record<string, unknown>>>
-}
-
-const require = createRequire(import.meta.url)
-const runtime = require('../packages/selector/lib/pruner.js') as PrunerRuntime
-
-const OLD_PROFILES: readonly string[] = [
-  'off',
-  'native',
-  'balanced',
-  'cache-strict',
-  'savings',
-  'adaptive',
-  'custom',
-]
-const baseConfig = runtime.resolveConfig({})
-const OPTION_CASES: readonly OptionCase[] = [
-  { label: 'defaults', options: {} },
-  { label: 'linkage', options: { contextWindowTokens: 128_000, autoCompactThresholdPercent: 80 } },
-]
-const CUSTOM_DOCS: readonly CustomDocCase[] = [
-  { label: 'default-custom', custom: undefined },
-]
-
-const snapshot: ProfileBaseline = { capturedAt: 'pre-tokenpilot-inspired', profiles: {} }
-for (const profile of OLD_PROFILES) {
-  const byCase: Record<string, Record<string, unknown>> = {}
-  snapshot.profiles[profile] = byCase
-  for (const optionCase of OPTION_CASES) {
-    for (const customCase of CUSTOM_DOCS) {
-      const key = `${optionCase.label}${customCase.label === 'default-custom' ? '' : `/${customCase.label}`}`
-      byCase[key] = runtime.resolvePolicy(
-        baseConfig,
-        profile,
-        customCase.custom,
-        optionCase.options,
-      )
-    }
-  }
-}
-
-const out = process.argv[2] ?? 'packages/selector/tests/runtime/fixtures/profile-baseline.json'
-writeFileSync(out, `${JSON.stringify(snapshot, null, 2)}\n`)
-console.log(`baseline written: ${out}`)

package/tsconfig.scripts.json

@@ -1,13 +0,0 @@
-{
-  "extends": "./tsconfig.base.json",
-  "compilerOptions": {
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "allowImportingTsExtensions": false,
-    "rootDir": "scripts",
-    "outDir": "scripts-dist",
-    "noEmit": false,
-    "types": ["node"]
-  },
-  "include": ["scripts/**/*.ts"]
-}