docspack 0.2.0 → 0.3.0

package/src/cli.ts

@@ -7,15 +7,10 @@ import { defaultStorePath, Store, silenceSqliteWarning } from "./db.js";
 import { discoverPackages } from "./discovery.js";
 import { type DoctorReport, runDoctor } from "./doctor.js";
 import { DocspackError } from "./errors.js";
+import { commandHelpText, findCommandHelp, globalHelpText } from "./help.js";
 import { renderTree } from "./init/write.js";
 import { previewPackage } from "./preview.js";
-import {
-  DEFAULT_LIMIT,
-  DEFAULT_MAX_TOKENS,
-  type QueryResult,
-  queryDocs,
-  renderAnswer,
-} from "./search.js";
+import { type QueryResult, queryDocs, renderAnswer } from "./search.js";
 import { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
 import { syncProject } from "./sync.js";
 import { verifyProject } from "./verify.js";
@@ -42,6 +37,10 @@ const OPTIONS = {
   name: { type: "string" },
   "pkg-version": { type: "string" },
   pages: { type: "string" },
+  "max-chunk-tokens": { type: "string" },
+  "min-chunk-tokens": { type: "string" },
+  documents: { type: "string", multiple: true },
+  "min-hit-rate": { type: "string" },
   mirror: { type: "string" },
   mode: { type: "string" },
   template: { type: "string" },
@@ -70,138 +69,14 @@ const OPTIONS = {
 const EXIT_NOT_INDEXED = 3;
 const EXIT_NO_MATCH = 4;
 
-const HELP = `docspack — local, version-locked documentation for AI agents
-
-Usage
-  docspack <command> [options]
-
-Commands
-  sync                 Index the docs packages this project depends on
-  ask <question>       Answer from the local index — the command to give an agent
-  search <query>       Same index, formatted for a human reading the terminal
-  list                 Show this project's docs packages and their index state
-  verify               Check the docs still describe the code you installed
-  feedback <sub>       Record documentation problems: add, list, submit, remove
-  mcp                  Serve the index over MCP instead, as a long-lived process
-  sources              List curated sources that \`docspack build\` can fetch
-
-Authoring
-  init                 Scaffold a documentation package, then build and check it
-  build [source]       Generate the .llms/ payload for publishing
-  doctor               Check a package the way the indexer and a reviewer would
-  preview <query>      Answer a query from the local package, as an agent would
-
-How it works
-  Documentation ships as npm packages named @vendor/docspack or
-  @docspack-community/<name>. Add them to package.json, run \`docspack sync\`, and
-  every chunk is indexed into a shared SQLite database with FTS5. Agents query
-  that index locally: no network, no scraping, always the installed version.
-
-Giving an agent access
-  Any agent with a shell can run \`docspack ask\`, so one line in AGENTS.md or
-  CLAUDE.md is the whole setup — no server, no per-agent configuration, and
-  nothing resident when nobody is asking:
-
-${AGENTS_SNIPPET.map((line) => `    ${line}`).join("\n")}
-
-  Add the block below as well if you want the agent to record documentation
-  problems it runs into. It writes to a local file; nothing is sent:
-
-${FEEDBACK_SNIPPET.map((line) => `    ${line}`).join("\n")}
-
-  \`docspack mcp\` serves the same index over the Model Context Protocol for
-  clients that prefer a tool definition. Both return identical text.
-
-Recording documentation problems
-  \`docspack feedback add\` writes a finding to .docspack/feedback.jsonl in this
-  project. Nothing is transmitted: docspack contains no code that can send a
-  report anywhere, and it never will.
-
-    docspack feedback add --chunk @acme/docspack@1.4.0/api-auth \\
-      --kind drift --evidence "client.setKey is not exported; setApiKey is"
-
-  A finding has to be falsifiable. \`drift\` must name the identifier that
-  drifted. \`incorrect\` and \`missing\` must carry --expected, --actual and
-  --repro, so a claim that cannot show its work cannot be recorded at all.
-  Recording the same problem again increments a counter instead of adding a
-  second entry.
-
-  \`docspack feedback submit\` prints a prefilled GitHub issue URL for every
-  finding whose vendor asked to receive it, and routes nothing anywhere else.
-  A vendor opts in from their own package.json:
-
-    "docspack": {
-      "feedback": {
-        "github": "acme/sdk",
-        "labels": ["documentation"],
-        "accepts": ["drift"]
-      }
-    }
-
-  No \`feedback\` block means no channel and no route. \`accepts\` narrows which
-  kinds reach them. A human opens the link, reads it, and files it.
-
-Options
-  -y, --yes          init: skip prompts and use flags plus detected defaults
-  --dry-run          init: print the file tree and write nothing
-  --mirror <id|url>  init: bootstrap from a published llms.txt
-  --community        init: scaffold under @docspack-community
-  --no-workflow      init: skip the release workflow
-  --no-build         init: scaffold only
-  --template <t>     init: full (default) or minimal
-  --strict           doctor: treat structural warnings as failures
-  --pedantic         doctor: --strict, and fail on prose style as well
-  --package-dir <d>  doctor, preview, verify: the package to inspect (default: .)
-  --chunk <id>       feedback add: the chunk the problem is in
-  --kind <k>         feedback add: drift, incorrect or missing
-  --evidence <text>  feedback add: the claim, in one line
-  --expected <text>  feedback add: what the documentation led you to expect
-  --actual <text>    feedback add: what happened instead
-  --repro <code>     feedback add: code that demonstrates it
-  --force            sync: re-index packages already in the store; init: overwrite files
-  -p, --package <s>  ask, search: only packages whose name contains this text
-  --limit <n>        ask, search, preview: maximum chunks to return (default ${DEFAULT_LIMIT})
-  --max-tokens <n>   ask, search, preview: token ceiling for the result set
-                     (default ${DEFAULT_MAX_TOKENS})
-  --all              ask, search: the whole store, not just this project;
-                     feedback remove: every recorded finding
-  --from <dir>       build: directory of Markdown to package
-  --openapi <file>   build: OpenAPI JSON to package, one chunk per operation
-  --name <name>      build: package name, e.g. @acme/docspack
-  --pkg-version <v>  build: package version
-  --out <dir>        build: output directory (default: the current directory)
-  --pages <n>        build: maximum documents to fetch from a remote source
-  --store <path>     Use a different index (default: ${defaultStorePath()})
-  --cwd <dir>        Run in a different directory
-  --json             Machine-readable output
-  -q, --quiet        Only print errors
-  -h, --help         Show this help
-  -v, --version      Show the version
-
-Exit codes
-  \`ask\` and \`search\` answer with an exit code a script can branch on:
-
-    0  chunks were returned
-    ${EXIT_NOT_INDEXED}  nothing matched, and a docs package is installed but not indexed —
-       run \`docspack sync\`
-    ${EXIT_NO_MATCH}  nothing matched, and everything installed is already indexed
-    1  the command failed; 2  the command was used wrongly
-
-Examples
-  npx docspack sync
-  npx docspack ask "how do I verify a webhook signature"
-  npx docspack search "webhook signature" --package stripe
-  npx docspack init
-  npx docspack init --name @acme/docspack --from ./docs --yes
-  npx docspack doctor --strict
-  npx docspack preview "how do I authenticate"
-  npx docspack verify
-  npx docspack feedback list
-  npx docspack feedback submit
-`;
+const HELP = globalHelpText(defaultStorePath(), EXIT_NOT_INDEXED, EXIT_NO_MATCH);
 
 type Values = {
-  [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K]["type"] extends "boolean" ? boolean : string;
+  [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K] extends { multiple: true }
+    ? string[]
+    : (typeof OPTIONS)[K]["type"] extends "boolean"
+      ? boolean
+      : string;
 };
 
 const color = process.env.NO_COLOR === undefined && process.stdout.isTTY === true;
@@ -295,8 +170,21 @@ async function main(argv: readonly string[]): Promise<number> {
     return 0;
   }
 
+  // `docspack doctor --help` and `docspack help doctor` are the same question, and the answer to
+  // both is that command's flags rather than the global page.
   const [command = "help", ...rest] = positionals;
   if (values.help === true || command === "help") {
+    const wanted = command === "help" ? rest[0] : command;
+    const subject = wanted === undefined ? undefined : findCommandHelp(wanted);
+    if (subject !== undefined) {
+      process.stdout.write(commandHelpText(subject));
+      return 0;
+    }
+    if (command === "help" && wanted !== undefined) {
+      process.stderr.write(`${red("error")} unknown command "${wanted}"\n\n`);
+      process.stderr.write(HELP);
+      return 2;
+    }
     process.stdout.write(HELP);
     return 0;
   }
@@ -508,8 +396,13 @@ async function main(argv: readonly string[]): Promise<number> {
 
         const mark = pkg.findings.length === 0 ? green("ok") : red("drift");
         process.stdout.write(
-          `${mark} ${bold(pkg.id)} ${dim(`documents ${pkg.documents ?? ""} · ${plural(pkg.checked, "name")} checked, ${pkg.matched} declared`)}\n`,
+          `${mark} ${bold(pkg.id)} ${dim(`documents ${(pkg.documents ?? []).join(", ")} · ${plural(pkg.checked, "name")} checked, ${pkg.matched} declared`)}\n`,
         );
+        if (pkg.unchecked !== undefined && pkg.unchecked.length > 0) {
+          process.stderr.write(
+            `  ${yellow("!")} ${pkg.unchecked.join(", ")} could not be read, so the check ran against less than the package documents\n`,
+          );
+        }
         for (const finding of pkg.findings) {
           process.stderr.write(
             `  ${yellow(finding.entity)} is not declared; did the API become ${bold(finding.suggestion)}?\n`,
@@ -719,6 +612,15 @@ async function main(argv: readonly string[]): Promise<number> {
           const pages = integer(values.pages, "--pages");
           return pages === undefined ? {} : { pages };
         })(),
+        ...(() => {
+          const max = integer(values["max-chunk-tokens"], "--max-chunk-tokens");
+          return max === undefined ? {} : { maxChunkTokens: max };
+        })(),
+        ...(() => {
+          const min = integer(values["min-chunk-tokens"], "--min-chunk-tokens");
+          return min === undefined ? {} : { minChunkTokens: min };
+        })(),
+        ...(values.documents === undefined ? {} : { documents: values.documents }),
         ...(quiet || json
           ? {}
           : {
@@ -882,6 +784,57 @@ async function main(argv: readonly string[]): Promise<number> {
       return 0;
     }
 
+    case "eval": {
+      const file = rest[0];
+      if (file === undefined) throw new DocspackError("Usage: docspack eval <queries.json>");
+
+      const { evaluatePackage, readEvalSet } = await import("./eval.js");
+      const report = await evaluatePackage({
+        dir: values["package-dir"] ?? cwd,
+        cases: await readEvalSet(file),
+        ...(() => {
+          const limit = integer(values.limit, "--limit");
+          return limit === undefined ? {} : { limit };
+        })(),
+        ...(() => {
+          const maxTokens = integer(values["max-tokens"], "--max-tokens");
+          return maxTokens === undefined ? {} : { maxTokens };
+        })(),
+        ...(() => {
+          const rate = integer(values["min-hit-rate"], "--min-hit-rate");
+          return rate === undefined ? {} : { minHitRate: rate };
+        })(),
+      });
+
+      if (json) {
+        process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+        return report.ok ? 0 : 1;
+      }
+
+      for (const entry of report.cases) {
+        // A miss is only actionable next to what came back instead of the expected chunk.
+        const mark =
+          entry.rank === 1 ? green("1") : entry.rank > 0 ? yellow(String(entry.rank)) : red("—");
+        process.stdout.write(`${mark}  ${entry.query}\n`);
+        if (entry.rank !== 1) {
+          process.stdout.write(`   ${dim(`expected ${entry.expect.join(" | ")}`)}\n`);
+          process.stdout.write(
+            `   ${dim(`returned ${entry.returned.length === 0 ? "nothing" : entry.returned.join(", ")}`)}\n`,
+          );
+        }
+      }
+
+      const percent = (count: number): string =>
+        `${count}/${report.total} (${Math.round((count / report.total) * 100)}%)`;
+      process.stdout.write(
+        `\n${bold(`top-${report.limit}`)} ${percent(report.hits)}   ${bold("top-1")} ${percent(report.top1)}   ${dim(`~${report.averageTokens} tokens per answer`)}\n`,
+      );
+      if (!report.ok) {
+        process.stderr.write(`${red("failed")} below the required hit rate\n`);
+      }
+      return report.ok ? 0 : 1;
+    }
+
     case "sources": {
       const { entries: registryEntries } = await import("@docspack/registry");
       if (json) {

package/src/config.ts

@@ -28,10 +28,15 @@ export interface BuildConfig {
   readonly from?: string;
   readonly openapi?: string;
   readonly source?: string;
-  /** The library this package documents, e.g. `@acme/sdk`. What `docspack verify` checks against. */
-  readonly documents?: string;
+  /**
+   * The libraries this package documents, each `name` or `name@version`. What `docspack verify`
+   * checks against, and what the manifest records so a consumer can say which release an answer
+   * describes. A monorepo documents several from one surface, so this is a list.
+   */
+  readonly documents?: readonly string[];
   readonly feedback?: FeedbackChannel;
   readonly maxChunkTokens?: number;
+  readonly minChunkTokens?: number;
   readonly pages?: number;
 }
 
@@ -64,7 +69,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
   }
   const raw = value as Record<string, unknown>;
 
-  const text = (key: "from" | "openapi" | "source" | "documents"): string | undefined => {
+  const text = (key: "from" | "openapi" | "source"): string | undefined => {
     const found = raw[key];
     if (found === undefined) return undefined;
     if (typeof found !== "string" || found.length === 0) {
@@ -73,7 +78,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
     return found;
   };
 
-  const count = (key: "maxChunkTokens" | "pages"): number | undefined => {
+  const count = (key: "maxChunkTokens" | "minChunkTokens" | "pages"): number | undefined => {
     const found = raw[key];
     if (found === undefined) return undefined;
     if (!Number.isInteger(found) || (found as number) < 1) {
@@ -85,9 +90,10 @@ export function parseBuildConfig(value: unknown): BuildConfig {
   const from = text("from");
   const openapi = text("openapi");
   const source = text("source");
-  const documents = text("documents");
+  const documents = parseDocuments(raw.documents);
   const feedback = parseFeedbackChannel(raw.feedback);
   const maxChunkTokens = count("maxChunkTokens");
+  const minChunkTokens = count("minChunkTokens");
   const pages = count("pages");
 
   return {
@@ -97,10 +103,34 @@ export function parseBuildConfig(value: unknown): BuildConfig {
     ...(documents === undefined ? {} : { documents }),
     ...(feedback === undefined ? {} : { feedback }),
     ...(maxChunkTokens === undefined ? {} : { maxChunkTokens }),
+    ...(minChunkTokens === undefined ? {} : { minChunkTokens }),
     ...(pages === undefined ? {} : { pages }),
   };
 }
 
+/**
+ * `"acme"`, `"acme@1.4.0"` or a list of either. A single string stays valid: one library is the
+ * common case, and every package written against the earlier field shape still reads.
+ */
+function parseDocuments(value: unknown): readonly string[] | undefined {
+  if (value === undefined) return undefined;
+  const where = `${CONFIG_KEY}.documents`;
+  const specs = Array.isArray(value) ? value : [value];
+
+  if (
+    specs.length === 0 ||
+    specs.some((entry) => typeof entry !== "string" || entry.length === 0)
+  ) {
+    throw new DocspackError(
+      `package.json: "${where}" must be a package name, or an array of them`,
+      {
+        hint: 'Add a version to scope it: "acme@1.4.0".',
+      },
+    );
+  }
+  return specs as string[];
+}
+
 /** `owner/repo`, the only shape a prefilled issue URL can be built from. */
 const GITHUB_REPO = /^[\w.-]+\/[\w.-]+$/;
 

package/src/db.ts

@@ -4,6 +4,7 @@ import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import type { DatabaseSync } from "node:sqlite";
 import { DocspackError } from "./errors.js";
+import { STOPWORDS } from "./stopwords.js";
 
 const require = createRequire(import.meta.url);
 
@@ -102,10 +103,14 @@ export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
  * Turns free text into an FTS5 MATCH expression. Every term is quoted, so punctuation in a user
  * query can never be interpreted as FTS syntax.
  *
- * Terms of one or two characters are dropped. Questions are mostly `how`, `do`, `i`, `in`, and
- * every chunk containing one becomes a candidate — which is how a chunk that merely quotes an
- * example question outranks the chunk that answers it. Two characters is the cut because it
- * leaves real short names (`id`, `db`) out and keeps three-letter ones (`api`, `key`) in.
+ * Terms of one or two characters are dropped, as are function words of any length. Questions are
+ * mostly `how`, `do`, `i`, `in`, and every chunk containing one becomes a candidate — which is
+ * how a chunk that merely quotes an example question outranks the chunk that answers it. Two
+ * characters is the cut because it leaves real short names (`id`, `db`) out and keeps
+ * three-letter ones (`api`, `key`) in; `how` and `what` need the word list as well.
+ *
+ * Each narrowing falls back to the previous one, so a query made entirely of short or common
+ * words still searches for something rather than failing.
  */
 export function toFtsQuery(raw: string): string {
   const terms = raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
@@ -114,9 +119,10 @@ export function toFtsQuery(raw: string): string {
       hint: "Search for words, for example: docspack search webhook signature",
     });
   }
-  const meaningful = terms.filter((term) => term.length > 2);
-  // A query made only of short terms still has to search for something.
-  return (meaningful.length > 0 ? meaningful : terms).map((term) => `"${term}"`).join(" OR ");
+  const long = terms.filter((term) => term.length > 2);
+  const meaningful = long.filter((term) => !STOPWORDS.has(term));
+  const chosen = meaningful.length > 0 ? meaningful : long.length > 0 ? long : terms;
+  return chosen.map((term) => `"${term}"`).join(" OR ");
 }
 
 /** The global chunk index: one SQLite database shared by every project on the machine. */

package/src/doctor.ts

@@ -104,7 +104,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
       continue;
     }
 
-    const size = chunk.tokens > 0 ? chunk.tokens : estimateTokens(text);
+    const size = chunk.tokens ?? estimateTokens(text);
     tokens += size;
 
     if (size > MAX_CHUNK_TOKENS) {

package/src/eval.ts

@@ -0,0 +1,149 @@
+import { readFile } from "node:fs/promises";
+import { DocspackError } from "./errors.js";
+import { previewPackage } from "./preview.js";
+import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS } from "./search.js";
+
+/** One question, and the chunk ids that would be a correct answer to it. */
+export interface EvalCase {
+  readonly query: string;
+  /** Bare chunk ids, as they appear in the manifest. Any one of them counts as a hit. */
+  readonly expect: readonly string[];
+}
+
+export interface EvalCaseResult extends EvalCase {
+  /** Chunk ids returned, in rank order. */
+  readonly returned: readonly string[];
+  /** 1-based rank of the first expected chunk, or 0 when none of them came back. */
+  readonly rank: number;
+  readonly tokens: number;
+}
+
+export interface EvalReport {
+  readonly cases: readonly EvalCaseResult[];
+  /** Cases where an expected chunk came back at all, within `limit`. */
+  readonly hits: number;
+  /** Cases where an expected chunk came back first. */
+  readonly top1: number;
+  readonly total: number;
+  /** Mean tokens per answer — what tuning the chunk budget trades retrieval against. */
+  readonly averageTokens: number;
+  readonly limit: number;
+  readonly ok: boolean;
+}
+
+export interface EvalOptions {
+  readonly dir: string;
+  readonly cases: readonly EvalCase[];
+  readonly limit?: number;
+  readonly maxTokens?: number;
+  /** Percentage of cases that must return an expected chunk. Below it, `ok` is false. */
+  readonly minHitRate?: number;
+}
+
+/**
+ * Answers a set of questions from a package on disk and reports how often the right chunk came
+ * back.
+ *
+ * `doctor` cannot see this. Every chunk can be well-formed, well-tagged and the right size while
+ * the package still answers the wrong question, and the highest-leverage authoring decision —
+ * the chunk budget — trades two things off against each other in a way only measurement settles:
+ * bm25 normalizes for length, so retrieval prefers small chunks, while an answer capped at three
+ * of them prefers large ones.
+ */
+export async function evaluatePackage(options: EvalOptions): Promise<EvalReport> {
+  if (options.cases.length === 0) {
+    throw new DocspackError("The evaluation set is empty", {
+      hint: "Add at least one { query, expect } entry.",
+    });
+  }
+
+  const limit = options.limit ?? DEFAULT_LIMIT;
+  const cases: EvalCaseResult[] = [];
+
+  for (const entry of options.cases) {
+    const result = await previewPackage({
+      dir: options.dir,
+      query: entry.query,
+      limit,
+      maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
+    });
+
+    // A hit is compared on the bare id: an eval set written by the author must not have to
+    // restate the package name and version it is already sitting inside.
+    const returned = result.hits.map((hit) => bareChunkId(hit.chunkId));
+    const expected = new Set(entry.expect);
+    const rank = returned.findIndex((id) => expected.has(id)) + 1;
+
+    cases.push({ ...entry, returned, rank, tokens: result.tokens });
+  }
+
+  const hits = cases.filter((entry) => entry.rank > 0).length;
+  const total = cases.length;
+  const rate = (hits / total) * 100;
+
+  return {
+    cases,
+    hits,
+    top1: cases.filter((entry) => entry.rank === 1).length,
+    total,
+    averageTokens: Math.round(cases.reduce((sum, entry) => sum + entry.tokens, 0) / total),
+    limit,
+    ok: options.minHitRate === undefined || rate >= options.minHitRate,
+  };
+}
+
+/** `@acme/docspack@1.4.0/api-auth` -> `api-auth`. */
+function bareChunkId(id: string): string {
+  const slash = id.lastIndexOf("/");
+  return slash === -1 ? id : id.slice(slash + 1);
+}
+
+/**
+ * Reads an evaluation set: a JSON array of `{ query, expect }`, where `expect` is a chunk id or
+ * an array of them. Malformed entries are an error rather than a skip — a gate that silently
+ * evaluates nine of ten questions reports a hit rate that is not about the set it was given.
+ */
+export async function readEvalSet(file: string): Promise<readonly EvalCase[]> {
+  let raw: string;
+  try {
+    raw = await readFile(file, "utf8");
+  } catch {
+    throw new DocspackError(`Could not read the evaluation set ${file}`, {
+      hint: 'It is a JSON array of { "query": "…", "expect": "chunk-id" }.',
+    });
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    throw new DocspackError(`${file} is not valid JSON`, { cause: error });
+  }
+
+  return parseEvalSet(parsed, file);
+}
+
+export function parseEvalSet(value: unknown, where: string): readonly EvalCase[] {
+  const fail = (message: string): never => {
+    throw new DocspackError(`${where}: ${message}`, {
+      hint: 'Each entry is { "query": "how do I …", "expect": "chunk-id" }.',
+    });
+  };
+
+  if (!Array.isArray(value)) fail("expected a JSON array of evaluation cases");
+
+  return (value as unknown[]).map((entry, index): EvalCase => {
+    if (typeof entry !== "object" || entry === null) return fail(`case #${index} is not an object`);
+    const record = entry as { query?: unknown; expect?: unknown };
+
+    if (typeof record.query !== "string" || record.query.trim().length === 0) {
+      return fail(`case #${index} needs a non-empty "query"`);
+    }
+    const expect = Array.isArray(record.expect) ? record.expect : [record.expect];
+    if (expect.length === 0 || expect.some((id) => typeof id !== "string" || id.length === 0)) {
+      return fail(`case "${record.query}" needs "expect": a chunk id, or an array of them`);
+    }
+
+    return { query: record.query, expect: expect as string[] };
+  });
+}