@anionex/dsh-tool-search 0.1.0

package/scripts/profile-e2e.mjs

@@ -0,0 +1,366 @@
+import { spawn } from 'node:child_process'
+import { access, mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'
+import { createServer } from 'node:http'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const root = dirname(dirname(fileURLToPath(import.meta.url)))
+const dsh = process.env.DSH_BIN ?? 'dsh'
+const keepTemp = process.argv.includes('--keep-temp')
+const timeoutMs = Number(process.env.DSH_TOOL_SEARCH_E2E_TIMEOUT_MS ?? 180_000)
+
+function run(command, args, options = {}) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd ?? root,
+      env: { ...process.env, ...options.env },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+    const stdout = []
+    const stderr = []
+    child.stdout.on('data', chunk => stdout.push(Buffer.from(chunk)))
+    child.stderr.on('data', chunk => stderr.push(Buffer.from(chunk)))
+    const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs)
+    child.once('error', error => {
+      clearTimeout(timer)
+      reject(error)
+    })
+    child.once('close', code => {
+      clearTimeout(timer)
+      resolve({
+        code: code ?? -1,
+        stdout: Buffer.concat(stdout).toString('utf8'),
+        stderr: Buffer.concat(stderr).toString('utf8'),
+      })
+    })
+  })
+}
+
+async function checked(command, args, options) {
+  const result = await run(command, args, options)
+  if (result.code !== 0) {
+    throw new Error(`${command} ${args.join(' ')} failed (${result.code})\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`)
+  }
+  return result
+}
+
+async function pack(directory, destination) {
+  const result = await checked('npm', ['pack', '--ignore-scripts', '--pack-destination', destination, '--json'], {
+    cwd: directory,
+  })
+  const filename = JSON.parse(result.stdout)[0]?.filename
+  if (typeof filename !== 'string') throw new Error(`npm pack returned no filename: ${result.stdout}`)
+  return join(destination, filename)
+}
+
+async function fixturePackage(directory) {
+  await mkdir(join(directory, 'lib'), { recursive: true })
+  await writeFile(join(directory, 'package.json'), `${JSON.stringify({
+    name: '@dsh-tool-search/e2e-fixture',
+    version: '1.0.0',
+    type: 'module',
+    main: './lib/index.js',
+    files: ['lib', 'cordis.patch.yml'],
+    dsh: { bundle: { patch: './cordis.patch.yml' } },
+    peerDependencies: {
+      '@deepseek-ai/dsh-tools': '^0.1.0-rc.8',
+    },
+  }, null, 2)}\n`)
+  await writeFile(join(directory, 'cordis.patch.yml'), [
+    '- insert:',
+    '    - id: tool-search-e2e-fixture',
+    "      name: '@dsh-tool-search/e2e-fixture'",
+    '',
+  ].join('\n'))
+  await writeFile(join(directory, 'lib', 'index.js'), [
+    "import { defineTool } from '@deepseek-ai/dsh-tools'",
+    "export const name = '@dsh-tool-search/e2e-fixture'",
+    "export const inject = ['tools']",
+    'export function apply(ctx) {',
+    '  ctx.tools.register(defineTool({',
+    "    name: 'fixture_echo',",
+    "    description: 'Echo a text value for deferred tool Agent acceptance.',",
+    "    parameters: { value: { type: 'string', required: true, description: 'Text value to echo.' } },",
+    '    output: {',
+    "      schema: { type: 'object', additionalProperties: false, properties: { echo: { type: 'string', required: true } } },",
+    "      render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }],",
+    '    },',
+    '    execute: args => Promise.resolve({ echo: args.value }),',
+    '  }))',
+    '}',
+    '',
+  ].join('\n'))
+}
+
+async function startLlm(script = [
+  { kind: 'tool', name: 'tool_search', arguments: '{"query":"fixture_echo","limit":1}' },
+  { kind: 'tool', name: 'fixture_echo', arguments: '{"value":"real-agent-ok"}' },
+  { kind: 'text', text: 'tool search e2e done' },
+]) {
+  const requests = []
+  let index = 0
+  const server = createServer((request, response) => {
+    if (request.method !== 'POST' || !request.url?.endsWith('/chat/completions')) {
+      response.writeHead(404, { 'content-type': 'application/json' })
+      response.end('{"error":"not-found"}')
+      return
+    }
+    const chunks = []
+    request.on('data', chunk => chunks.push(Buffer.from(chunk)))
+    request.on('end', () => {
+      let body
+      try {
+        body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
+      } catch {
+        response.writeHead(400, { 'content-type': 'application/json' })
+        response.end('{"error":"invalid-json"}')
+        return
+      }
+      requests.push(body)
+      const step = script[index++]
+      if (step === undefined) {
+        response.writeHead(500, { 'content-type': 'application/json' })
+        response.end('{"error":{"message":"script-exhausted"}}')
+        return
+      }
+      response.writeHead(200, {
+        'content-type': 'text/event-stream; charset=utf-8',
+        'cache-control': 'no-cache',
+        connection: 'keep-alive',
+      })
+      const send = payload => response.write(`data: ${typeof payload === 'string' ? payload : JSON.stringify(payload)}\n\n`)
+      if (step.kind === 'tool') {
+        send({
+          choices: [{
+            index: 0,
+            delta: {
+              tool_calls: [{
+                index: 0,
+                id: `tool-search-e2e-${index}`,
+                type: 'function',
+                function: { name: step.name, arguments: step.arguments },
+              }],
+            },
+            finish_reason: null,
+          }],
+        })
+        send({
+          choices: [{ index: 0, delta: { content: '' }, finish_reason: 'tool_calls' }],
+          usage: { prompt_tokens: 10, completion_tokens: 5 },
+        })
+      } else {
+        send({ choices: [{ index: 0, delta: { content: step.text }, finish_reason: null }] })
+        send({
+          choices: [{ index: 0, delta: { content: '' }, finish_reason: 'stop' }],
+          usage: { prompt_tokens: 10, completion_tokens: 5 },
+        })
+      }
+      send('[DONE]')
+      response.end()
+    })
+  })
+  await new Promise((resolve, reject) => {
+    server.once('error', reject)
+    server.listen(0, '127.0.0.1', resolve)
+  })
+  const address = server.address()
+  if (address === null || typeof address === 'string') throw new Error('mock LLM did not bind a TCP port')
+  return {
+    baseUrl: `http://127.0.0.1:${address.port}/v1`,
+    requests,
+    close: () => new Promise((resolve, reject) => {
+      server.close(error => error === undefined ? resolve() : reject(error))
+      server.closeAllConnections()
+    }),
+  }
+}
+
+function requestToolNames(body) {
+  return body?.tools?.map(tool => tool?.function?.name).filter(name => typeof name === 'string') ?? []
+}
+
+function toolResultTexts(body) {
+  return body?.messages
+    ?.filter(message => message?.role === 'tool')
+    .map(message => typeof message.content === 'string' ? message.content : JSON.stringify(message.content)) ?? []
+}
+
+function assert(condition, message) {
+  if (!condition) throw new Error(message)
+}
+
+async function exposedDshScript(command) {
+  if (process.env.DSH_BIN === undefined) {
+    return join(root, 'node_modules', '@deepseek-ai', 'dsh', 'lib', 'bin.js')
+  }
+  const profileCandidate = join(dirname(dirname(command)), '@deepseek-ai', 'dsh', 'lib', 'bin.js')
+  try {
+    await access(profileCandidate)
+    return profileCandidate
+  } catch {
+    return await realpath(command)
+  }
+}
+
+const home = await mkdtemp(join(tmpdir(), 'dsh-tool-search-e2e-'))
+const packageDirectory = join(home, 'packages')
+const fixtureDirectory = join(home, 'fixture')
+const workspaceDirectory = join(home, 'workspace')
+await mkdir(packageDirectory)
+await mkdir(workspaceDirectory)
+const llm = await startLlm()
+
+try {
+  const dshVersion = (await checked(dsh, ['--version'])).stdout.trim()
+  const profileModulePrefix = dshVersion.startsWith('0.1.1-')
+    ? './node_modules'
+    : './profiles/headless/node_modules'
+  const dshExecutable = dsh.includes('/')
+    ? dsh
+    : (await checked('/usr/bin/which', [dsh])).stdout.trim()
+  const dshScript = await exposedDshScript(dshExecutable)
+  const nodeExecutable = (await checked('/usr/bin/which', ['node'])).stdout.trim()
+  const runDshExposed = (args, options) => checked(
+    nodeExecutable,
+    ['--expose-internals', dshScript, ...args],
+    options,
+  )
+  let runDsh = (args, options) => checked(dsh, args, options)
+  if (dshVersion.startsWith('0.1.1-')) {
+    runDsh = runDshExposed
+  }
+  await fixturePackage(fixtureDirectory)
+  const pluginTarball = await pack(root, packageDirectory)
+  const fixtureTarball = await pack(fixtureDirectory, packageDirectory)
+  await runDsh(['plugin', '--profile', 'headless', 'add', pluginTarball], {
+    env: { DSH_HOME: home },
+  })
+  await runDsh(['plugin', '--profile', 'headless', 'add', fixtureTarball], {
+    env: { DSH_HOME: home },
+  })
+  const dump = await runDsh(['--profile', 'headless', '--dump-config'], {
+    env: { DSH_HOME: home },
+  })
+  assert(
+    dump.stdout.includes('@anionex/dsh-tool-search'),
+    `installed profile omits dsh-tool-search\nstdout:\n${dump.stdout}\nstderr:\n${dump.stderr}`,
+  )
+  assert(
+    dump.stdout.includes('@dsh-tool-search/e2e-fixture'),
+    `installed profile omits E2E fixture\nstdout:\n${dump.stdout}\nstderr:\n${dump.stderr}`,
+  )
+
+  let entryMode = dshVersion.startsWith('0.1.1-') ? 'bare-exposed-internals' : 'bare'
+  let bareFailure
+  const bareLlm = await startLlm([{ kind: 'text', text: 'bare package boot ok' }])
+  const bareArgs = [
+    '--profile', 'headless',
+    'Verify standard package loading.',
+  ]
+  const bareOptions = {
+    cwd: workspaceDirectory,
+    env: {
+      DSH_HOME: home,
+      DSH_TELEMETRY_DISABLED: '1',
+      DSH_PERMISSION_MODE: 'danger-full-access',
+      DEEPSEEK_API_KEY: 'tool-search-e2e-key',
+      DEEPSEEK_BASE_URL: bareLlm.baseUrl,
+    },
+  }
+  try {
+    const bareResult = await runDsh(bareArgs, bareOptions)
+    assert(bareResult.stdout.trim() === 'bare package boot ok', `unexpected bare-load smoke output: ${bareResult.stdout}`)
+  } catch (error) {
+    if (!String(error).includes('Cannot find package')) throw error
+    bareFailure = /Cannot find package '[^']+'/u.exec(String(error))?.[0]
+    if (bareLlm.requests.length === 0 && runDsh !== runDshExposed) {
+      try {
+        const exposedResult = await runDshExposed(bareArgs, bareOptions)
+        assert(exposedResult.stdout.trim() === 'bare package boot ok', `unexpected exposed-internals smoke output: ${exposedResult.stdout}`)
+        runDsh = runDshExposed
+        entryMode = 'bare-exposed-internals'
+      } catch {
+        entryMode = 'relative-compat'
+      }
+    } else {
+      entryMode = 'relative-compat'
+    }
+    if (entryMode !== 'bare' && process.env.DSH_TOOL_SEARCH_REQUIRE_BARE === '1') {
+      throw new Error(`standard installed package loading is required\n${String(error)}`)
+    }
+    if (entryMode === 'relative-compat' && process.env.DSH_TOOL_SEARCH_ALLOW_RELATIVE_COMPAT !== '1') {
+      throw new Error(`installed package loading failed even with Node internals exposed\n${String(error)}`)
+    }
+  } finally {
+    await bareLlm.close()
+  }
+
+  const patch = join(home, 'e2e.patch.yml')
+  const patchLines = [
+    '# Disable unrelated title generation for deterministic model-call accounting.',
+    '- id: session-title-llm',
+    '  disabled: true',
+  ]
+  if (entryMode === 'relative-compat') {
+    patchLines.unshift(
+      '# CLI-only relative paths work around bare-specifier resolution in standalone dsh launchers.',
+      '- id: dsh-tool-search',
+      "  name: '@anionex/dsh-tool-search'",
+      '  disabled: true',
+      '- id: tool-search-e2e-fixture',
+      "  name: '@dsh-tool-search/e2e-fixture'",
+      '  disabled: true',
+      '- insert:',
+      '    - id: dsh-tool-search-e2e-relative',
+      `      name: ${profileModulePrefix}/@anionex/dsh-tool-search/lib/index.js`,
+      '    - id: tool-search-e2e-fixture-relative',
+      `      name: ${profileModulePrefix}/@dsh-tool-search/e2e-fixture/lib/index.js`,
+    )
+  }
+  patchLines.push('')
+  await writeFile(patch, patchLines.join('\n'))
+  const runResult = await runDsh([
+    '--profile', 'headless',
+    '--patch', patch,
+    'Find and call the fixture echo tool.',
+  ], {
+    cwd: workspaceDirectory,
+    env: {
+      DSH_HOME: home,
+      DSH_TELEMETRY_DISABLED: '1',
+      DSH_PERMISSION_MODE: 'danger-full-access',
+      DEEPSEEK_API_KEY: 'tool-search-e2e-key',
+      DEEPSEEK_BASE_URL: llm.baseUrl,
+    },
+  })
+
+  assert(runResult.stdout.trim() === 'tool search e2e done', `unexpected Agent output: ${runResult.stdout}`)
+  assert(llm.requests.length === 3, `expected 3 model calls, received ${llm.requests.length}`)
+  const initialNames = requestToolNames(llm.requests[0])
+  const searchedNames = requestToolNames(llm.requests[1])
+  assert(initialNames.includes('tool_search'), 'initial model call omits tool_search')
+  assert(!initialNames.includes('fixture_echo'), 'initial model call exposes deferred fixture_echo')
+  assert(searchedNames.includes('fixture_echo'), 'next model call omits selected fixture_echo')
+  const searchResult = toolResultTexts(llm.requests[1]).at(-1) ?? ''
+  assert(searchResult.includes('fixture_echo'), 'tool_search result omits fixture_echo summary')
+  assert(!searchResult.includes('"parameters"'), 'tool_search result duplicates a full schema')
+  const echoResult = toolResultTexts(llm.requests[2]).at(-1) ?? ''
+  assert(echoResult.includes('real-agent-ok'), 'fixture_echo did not execute through the real Agent loop')
+
+  process.stdout.write(`${JSON.stringify({
+    ok: true,
+    dsh: dshVersion,
+    entryMode,
+    bareFailure,
+    modelCalls: llm.requests.length,
+    initialToolCount: initialNames.length,
+    selectedToolCount: searchedNames.length,
+    selectedOnNextCall: searchedNames.includes('fixture_echo'),
+    package: '@anionex/dsh-tool-search@0.1.0',
+    temporaryHome: keepTemp ? home : undefined,
+  }, null, 2)}\n`)
+} finally {
+  await llm.close()
+  if (!keepTemp) await rm(home, { recursive: true, force: true })
+}

package/scripts/validate.mjs

@@ -0,0 +1,170 @@
+import { readFile, stat } from 'node:fs/promises'
+import { spawnSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { createRequire } from 'node:module'
+import { dirname, join } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { runInNewContext } from 'node:vm'
+
+const root = dirname(dirname(fileURLToPath(import.meta.url)))
+const require = createRequire(import.meta.url)
+const errors = []
+
+function check(condition, message) {
+  if (!condition) errors.push(message)
+}
+
+async function exists(path) {
+  try {
+    return (await stat(join(root, path))).isFile()
+  } catch {
+    return false
+  }
+}
+
+const pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'))
+check(pkg.name === '@anionex/dsh-tool-search', 'package name is incorrect')
+check(pkg.version === '0.1.0', 'release version is incorrect')
+check(pkg.main === './lib/index.js', 'main entrypoint is incorrect')
+check(pkg.types === './lib/types/index.d.ts', 'types entrypoint is incorrect')
+check(pkg.exports?.['./client']?.default === './lib/client.js', 'client export is incorrect')
+check(pkg.dsh?.bundle?.patch === './cordis.patch.yml', 'bundle patch is not declared')
+check(pkg.dsh?.client?.platform === 'web', 'Web client is not declared')
+for (const edge of [
+  '@deepseek-ai/dsh-client-locale',
+  '@deepseek-ai/dsh-client-ui-renderer',
+  '@deepseek-ai/dsh-client-ui-settings',
+]) {
+  check(pkg.dsh?.client?.inject?.includes(edge), `Web client injection edge is missing: ${edge}`)
+}
+check(pkg.dependencies?.['wink-porter2-stemmer'] === '^2.0.1', 'runtime stemmer dependency is missing')
+check(pkg.peerDependenciesMeta?.['@deepseek-ai/dsh-host-webserver']?.optional === true, 'Web server peer must remain optional')
+
+for (const group of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) {
+  for (const [name, spec] of Object.entries(pkg[group] ?? {})) {
+    check(typeof spec === 'string', `${group}.${name} must be a string`)
+    if (typeof spec !== 'string') continue
+    check(!/^(?:file:|link:|workspace:|\/|[A-Za-z]:[\\/])/u.test(spec), `${group}.${name} is machine-local: ${spec}`)
+  }
+}
+
+for (const path of [
+  'LICENSE',
+  'README.md',
+  'README.zh.md',
+  'cordis.patch.yml',
+  'pnpm-lock.yaml',
+  'lib/index.js',
+  'lib/types/index.d.ts',
+  'lib/client.js',
+  'lib/types/client/index.d.ts',
+  'docs/benchmark-results.json',
+  'scripts/benchmark.mjs',
+  'scripts/profile-e2e.mjs',
+  'scripts/validate.mjs',
+  'tests/runtime.spec.ts',
+  'pnpm-lock.yaml',
+  'pnpm-workspace.yaml',
+  'tsconfig.json',
+  'tsconfig.client.json',
+  'tsconfig.test.json',
+  'tsdown.config.mjs',
+]) check(await exists(path), `required package file is missing: ${path}`)
+
+if (await exists('docs/benchmark-results.json')) {
+  try {
+    const benchmark = JSON.parse(await readFile(join(root, 'docs/benchmark-results.json'), 'utf8'))
+    const generatorPath = join(root, benchmark.fixture?.generator ?? '')
+    const generatorHash = createHash('sha256').update(await readFile(generatorPath)).digest('hex')
+    check(benchmark.schemaVersion === 2, 'benchmark schema version is incorrect')
+    check(benchmark.packageVersion === pkg.version, 'benchmark package version is stale')
+    check(benchmark.fixture?.generatorSha256 === generatorHash, 'benchmark generator hash is stale')
+    check(JSON.stringify(benchmark.fixture?.sizes) === JSON.stringify([10, 30, 50, 100]), 'benchmark fixture sizes are incorrect')
+    const rows = benchmark.rows.map(row => (
+      `| ${row.tools} | ${row.fullTokens.toLocaleString('en-US')} | ${row.deferredInitialTokens.toLocaleString('en-US')} | ${row.initialReductionPercent}% | ${row.afterTop5Tokens.toLocaleString('en-US')} |`
+    ))
+    for (const readme of ['README.md', 'README.zh.md']) {
+      const text = await readFile(join(root, readme), 'utf8')
+      for (const row of rows) check(text.includes(row), `${readme} benchmark table is stale: ${row}`)
+    }
+  } catch (error) {
+    errors.push(`benchmark artifact validation failed: ${String(error)}`)
+  }
+}
+
+if (await exists('lib/client.js')) {
+  const client = await readFile(join(root, 'lib/client.js'), 'utf8')
+  check(client.startsWith('window.__ModuleLoader__.load('), 'client artifact is not wrapped for the DSH module loader')
+  check(client.includes('@anionex/dsh-tool-search'), 'client artifact is missing its plugin id')
+  let descriptor
+  try {
+    runInNewContext(client, {
+      window: {
+        __ModuleLoader__: {
+          load(value) { descriptor = value },
+        },
+      },
+    })
+    check(descriptor?.id === '@anionex/dsh-tool-search', 'client loader descriptor id is incorrect')
+    check(typeof descriptor?.factory === 'function', 'client loader descriptor has no factory')
+    const exports = descriptor?.factory(name => require(name))
+    check(typeof exports?.apply === 'function', 'client loader factory has no apply() export')
+  } catch (error) {
+    errors.push(`client loader execution failed: ${String(error)}`)
+  }
+}
+
+if (await exists('cordis.patch.yml')) {
+  const patch = await readFile(join(root, 'cordis.patch.yml'), 'utf8')
+  check(patch.includes('id: dsh-tool-search'), 'bundle patch does not declare dsh-tool-search')
+  check(patch.includes("name: '@anionex/dsh-tool-search'"), 'bundle patch package name is incorrect')
+}
+
+if (await exists('lib/index.js')) {
+  const runtime = await import(`${pathToFileURL(join(root, 'lib/index.js')).href}?validate=${Date.now()}`)
+  check(runtime.name === '@anionex/dsh-tool-search', 'built Host plugin id is incorrect')
+  check(typeof runtime.apply === 'function', 'built Host plugin has no apply()')
+  check(typeof runtime.Bm25Index === 'function', 'built Host plugin has no BM25 export')
+}
+
+const packed = spawnSync('npm', ['pack', '--ignore-scripts', '--dry-run', '--json'], {
+  cwd: root,
+  encoding: 'utf8',
+  timeout: 120_000,
+})
+check(packed.status === 0, `npm pack --dry-run failed: ${packed.stderr}`)
+if (packed.status === 0) {
+  let rows
+  try {
+    rows = JSON.parse(packed.stdout)
+  } catch {
+    errors.push('npm pack --dry-run did not return JSON')
+  }
+  const files = new Set(rows?.[0]?.files?.map(file => file.path) ?? [])
+  for (const path of [
+    'lib/index.js',
+    'lib/client.js',
+    'lib/types/index.d.ts',
+    'src/runtime.ts',
+    'scripts/benchmark.mjs',
+    'tests/runtime.spec.ts',
+    'tsconfig.json',
+    'docs/benchmark-results.json',
+    'cordis.patch.yml',
+  ]) {
+    check(files.has(path), `packed tarball is missing ${path}`)
+  }
+  check(!files.has('node_modules'), 'packed tarball includes node_modules')
+  check(![...files].some(path => path.endsWith('.env')), 'packed tarball includes an .env file')
+}
+
+if (errors.length > 0) {
+  process.stderr.write(`${errors.map(error => `- ${error}`).join('\n')}\n`)
+  process.exitCode = 1
+} else {
+  process.stdout.write(JSON.stringify({
+    ok: true,
+    package: `${pkg.name}@${pkg.version}`,
+    packedFiles: packed.status === 0 ? JSON.parse(packed.stdout)[0].files.length : 0,
+  }, null, 2) + '\n')
+}

package/src/bm25.ts

@@ -0,0 +1,104 @@
+/** Deterministic local BM25, aligned with Codex's bm25 2.3.2 defaults. */
+
+import stem from 'wink-porter2-stemmer'
+
+const K1 = 1.2
+const B = 0.75
+
+// stop-words 0.9.0 uses the NLTK English list before stemming.
+const ENGLISH_STOP_WORDS = new Set([
+  'a', 'about', 'above', 'after', 'again', 'against', 'ain', 'all', 'am', 'an', 'and', 'any', 'are',
+  'aren', "aren't", 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both',
+  'but', 'by', 'can', 'couldn', "couldn't", 'd', 'did', 'didn', "didn't", 'do', 'does', 'doesn',
+  "doesn't", 'doing', 'don', "don't", 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had',
+  'hadn', "hadn't", 'has', 'hasn', "hasn't", 'have', 'haven', "haven't", 'having', 'he', 'her', 'here',
+  'hers', 'herself', 'him', 'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'isn', "isn't",
+  'it', "it's", 'its', 'itself', 'just', 'll', 'm', 'ma', 'me', 'mightn', "mightn't", 'more', 'most',
+  'mustn', "mustn't", 'my', 'myself', 'needn', "needn't", 'no', 'nor', 'not', 'now', 'o', 'of', 'off',
+  'on', 'once', 'only', 'or', 'other', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 're', 's',
+  'same', 'shan', "shan't", 'she', "she's", 'should', "should've", 'shouldn', "shouldn't", 'so', 'some',
+  'such', 't', 'than', 'that', "that'll", 'the', 'their', 'theirs', 'them', 'themselves', 'then',
+  'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 've',
+  'very', 'was', 'wasn', "wasn't", 'we', 'were', 'weren', "weren't", 'what', 'when', 'where', 'which',
+  'while', 'who', 'whom', 'why', 'will', 'with', 'won', "won't", 'wouldn', "wouldn't", 'y', 'you',
+  "you'd", "you'll", "you're", "you've", 'your', 'yours', 'yourself', 'yourselves',
+])
+
+function asciiFold(value: string): string {
+  return value.normalize('NFKD').replace(/\p{M}+/gu, '')
+}
+
+export function tokenize(value: string): string[] {
+  const matches = asciiFold(value).toLowerCase().match(/[\p{L}\p{N}]+(?:['.][\p{L}\p{N}]+)*/gu) ?? []
+  return matches.filter(token => !ENGLISH_STOP_WORDS.has(token)).map(token => stem(token))
+}
+
+export interface Bm25Document<T> {
+  id: T
+  name: string
+  text: string
+}
+
+export interface Bm25Match<T> {
+  id: T
+  name: string
+  score: number
+}
+
+interface IndexedDocument<T> extends Bm25Document<T> {
+  tokens: string[]
+  counts: Map<string, number>
+  order: number
+}
+
+export class Bm25Index<T> {
+  private readonly documents: IndexedDocument<T>[]
+  private readonly frequencies = new Map<string, number>()
+  private readonly averageLength: number
+
+  constructor(documents: readonly Bm25Document<T>[]) {
+    this.documents = documents.map((document, order) => {
+      const tokens = tokenize(document.text)
+      const counts = new Map<string, number>()
+      for (const token of tokens) counts.set(token, (counts.get(token) ?? 0) + 1)
+      for (const token of counts.keys()) this.frequencies.set(token, (this.frequencies.get(token) ?? 0) + 1)
+      return { ...document, tokens, counts, order }
+    })
+    this.averageLength = this.documents.length === 0
+      ? 256
+      : this.documents.reduce((total, document) => total + document.tokens.length, 0) / this.documents.length
+  }
+
+  search(query: string, limit: number): Bm25Match<T>[] {
+    if (limit <= 0 || this.documents.length === 0) return []
+    const queryTokens = tokenize(query)
+    const normalizedQuery = query.trim()
+    const matches: Array<Bm25Match<T> & { exact: boolean; order: number }> = []
+
+    for (const document of this.documents) {
+      let score = 0
+      for (const token of queryTokens) {
+        const termFrequency = document.counts.get(token) ?? 0
+        if (termFrequency === 0) continue
+        const documentFrequency = this.frequencies.get(token) ?? 0
+        const inverseDocumentFrequency = Math.log1p(
+          (this.documents.length - documentFrequency + 0.5) / (documentFrequency + 0.5),
+        )
+        const lengthRatio = document.tokens.length / this.averageLength
+        const weight = (termFrequency * (K1 + 1))
+          / (termFrequency + K1 * (1 - B + B * lengthRatio))
+        score += inverseDocumentFrequency * weight
+      }
+      const exact = document.name === normalizedQuery
+      if (score > 0 || exact) matches.push({ id: document.id, name: document.name, score, exact, order: document.order })
+    }
+
+    matches.sort((left, right) => {
+      if (left.exact !== right.exact) return left.exact ? -1 : 1
+      if (left.score !== right.score) return right.score - left.score
+      if (left.name !== right.name) return left.name < right.name ? -1 : 1
+      return left.order - right.order
+    })
+    return matches.slice(0, limit).map(({ id, name, score }) => ({ id, name, score }))
+  }
+}