docspack 0.2.0 → 0.4.0

package/src/eval.ts

@@ -0,0 +1,149 @@
+import { readFile } from "node:fs/promises";
+import { DocspackError } from "./errors.js";
+import { previewPackage } from "./preview.js";
+import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS } from "./search.js";
+
+/** One question, and the chunk ids that would be a correct answer to it. */
+export interface EvalCase {
+  readonly query: string;
+  /** Bare chunk ids, as they appear in the manifest. Any one of them counts as a hit. */
+  readonly expect: readonly string[];
+}
+
+export interface EvalCaseResult extends EvalCase {
+  /** Chunk ids returned, in rank order. */
+  readonly returned: readonly string[];
+  /** 1-based rank of the first expected chunk, or 0 when none of them came back. */
+  readonly rank: number;
+  readonly tokens: number;
+}
+
+export interface EvalReport {
+  readonly cases: readonly EvalCaseResult[];
+  /** Cases where an expected chunk came back at all, within `limit`. */
+  readonly hits: number;
+  /** Cases where an expected chunk came back first. */
+  readonly top1: number;
+  readonly total: number;
+  /** Mean tokens per answer — what tuning the chunk budget trades retrieval against. */
+  readonly averageTokens: number;
+  readonly limit: number;
+  readonly ok: boolean;
+}
+
+export interface EvalOptions {
+  readonly dir: string;
+  readonly cases: readonly EvalCase[];
+  readonly limit?: number;
+  readonly maxTokens?: number;
+  /** Percentage of cases that must return an expected chunk. Below it, `ok` is false. */
+  readonly minHitRate?: number;
+}
+
+/**
+ * Answers a set of questions from a package on disk and reports how often the right chunk came
+ * back.
+ *
+ * `doctor` cannot see this. Every chunk can be well-formed, well-tagged and the right size while
+ * the package still answers the wrong question, and the highest-leverage authoring decision —
+ * the chunk budget — trades two things off against each other in a way only measurement settles:
+ * bm25 normalizes for length, so retrieval prefers small chunks, while an answer capped at three
+ * of them prefers large ones.
+ */
+export async function evaluatePackage(options: EvalOptions): Promise<EvalReport> {
+  if (options.cases.length === 0) {
+    throw new DocspackError("The evaluation set is empty", {
+      hint: "Add at least one { query, expect } entry.",
+    });
+  }
+
+  const limit = options.limit ?? DEFAULT_LIMIT;
+  const cases: EvalCaseResult[] = [];
+
+  for (const entry of options.cases) {
+    const result = await previewPackage({
+      dir: options.dir,
+      query: entry.query,
+      limit,
+      maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
+    });
+
+    // A hit is compared on the bare id: an eval set written by the author must not have to
+    // restate the package name and version it is already sitting inside.
+    const returned = result.hits.map((hit) => bareChunkId(hit.chunkId));
+    const expected = new Set(entry.expect);
+    const rank = returned.findIndex((id) => expected.has(id)) + 1;
+
+    cases.push({ ...entry, returned, rank, tokens: result.tokens });
+  }
+
+  const hits = cases.filter((entry) => entry.rank > 0).length;
+  const total = cases.length;
+  const rate = (hits / total) * 100;
+
+  return {
+    cases,
+    hits,
+    top1: cases.filter((entry) => entry.rank === 1).length,
+    total,
+    averageTokens: Math.round(cases.reduce((sum, entry) => sum + entry.tokens, 0) / total),
+    limit,
+    ok: options.minHitRate === undefined || rate >= options.minHitRate,
+  };
+}
+
+/** `@acme/docspack@1.4.0/api-auth` -> `api-auth`. */
+function bareChunkId(id: string): string {
+  const slash = id.lastIndexOf("/");
+  return slash === -1 ? id : id.slice(slash + 1);
+}
+
+/**
+ * Reads an evaluation set: a JSON array of `{ query, expect }`, where `expect` is a chunk id or
+ * an array of them. Malformed entries are an error rather than a skip — a gate that silently
+ * evaluates nine of ten questions reports a hit rate that is not about the set it was given.
+ */
+export async function readEvalSet(file: string): Promise<readonly EvalCase[]> {
+  let raw: string;
+  try {
+    raw = await readFile(file, "utf8");
+  } catch {
+    throw new DocspackError(`Could not read the evaluation set ${file}`, {
+      hint: 'It is a JSON array of { "query": "…", "expect": "chunk-id" }.',
+    });
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    throw new DocspackError(`${file} is not valid JSON`, { cause: error });
+  }
+
+  return parseEvalSet(parsed, file);
+}
+
+export function parseEvalSet(value: unknown, where: string): readonly EvalCase[] {
+  const fail = (message: string): never => {
+    throw new DocspackError(`${where}: ${message}`, {
+      hint: 'Each entry is { "query": "how do I …", "expect": "chunk-id" }.',
+    });
+  };
+
+  if (!Array.isArray(value)) fail("expected a JSON array of evaluation cases");
+
+  return (value as unknown[]).map((entry, index): EvalCase => {
+    if (typeof entry !== "object" || entry === null) return fail(`case #${index} is not an object`);
+    const record = entry as { query?: unknown; expect?: unknown };
+
+    if (typeof record.query !== "string" || record.query.trim().length === 0) {
+      return fail(`case #${index} needs a non-empty "query"`);
+    }
+    const expect = Array.isArray(record.expect) ? record.expect : [record.expect];
+    if (expect.length === 0 || expect.some((id) => typeof id !== "string" || id.length === 0)) {
+      return fail(`case "${record.query}" needs "expect": a chunk id, or an array of them`);
+    }
+
+    return { query: record.query, expect: expect as string[] };
+  });
+}

package/src/help.ts

@@ -0,0 +1,382 @@
+import { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
+
+/**
+ * The help text, per command.
+ *
+ * `docspack <command> --help` printed the global help, so no flag was discoverable from the CLI
+ * at all — `doctor --pedantic`, `ask --max-tokens` and `init --mirror` were findable only by
+ * reading the source. That matters more than usual for a tool whose main user is an agent that
+ * will not open a browser, and it is why the global page now points at these rather than trying
+ * to list every flag of every command in one block.
+ */
+export interface CommandHelp {
+  readonly name: string;
+  /** The line this command shows in the global help and on the site. */
+  readonly summary: string;
+  readonly group: "core" | "authoring";
+  readonly usage: string;
+  /** Flags this command reads, beyond the global ones. */
+  readonly options: readonly (readonly [string, string])[];
+  readonly detail?: readonly string[];
+  readonly examples: readonly string[];
+}
+
+const LIMIT = 3;
+const MAX_TOKENS = 3000;
+
+const QUERY_OPTIONS: readonly (readonly [string, string])[] = [
+  ["-p, --package <s>", "only packages whose name contains this text"],
+  ["--limit <n>", `maximum chunks to return (default ${LIMIT})`],
+  ["--max-tokens <n>", `token ceiling for the result set (default ${MAX_TOKENS})`],
+  ["--all", "search the whole store, not just this project"],
+];
+
+export const COMMANDS: readonly CommandHelp[] = [
+  {
+    name: "sync",
+    summary: "Index the docs packages this project depends on",
+    group: "core",
+    usage: "docspack sync [options]",
+    options: [["--force", "re-index packages already in the store"]],
+    detail: [
+      "Reads node_modules and indexes every @vendor/docspack and @docspack-community/<name>",
+      "package this project declares. Makes no network requests.",
+    ],
+    examples: ["docspack sync", "docspack sync --force"],
+  },
+  {
+    name: "ask",
+    summary: "Answer from the local index — the command to give an agent",
+    group: "core",
+    usage: 'docspack ask "<question>"',
+    options: QUERY_OPTIONS,
+    detail: [
+      "Answers from the installed versions, in the Markdown an agent should read. Exits 0 when",
+      "chunks were returned, 3 when a docs package is installed but not indexed, and 4 when",
+      "everything installed is indexed and nothing matched.",
+    ],
+    examples: [
+      'docspack ask "how do I verify a webhook signature"',
+      'docspack ask "webhook signature" --package stripe --limit 5',
+    ],
+  },
+  {
+    name: "search",
+    summary: "Same index, formatted for a human reading the terminal",
+    group: "core",
+    usage: "docspack search <query>",
+    options: QUERY_OPTIONS,
+    detail: ["The same index and ranking as `ask`, printed as a list of hits with a preview."],
+    examples: ["docspack search webhook signature"],
+  },
+  {
+    name: "list",
+    summary: "Show this project's docs packages and their index state",
+    group: "core",
+    usage: "docspack list",
+    options: [],
+    examples: ["docspack list", "docspack list --json"],
+  },
+  {
+    name: "verify",
+    summary: "Check the docs still describe the code you installed",
+    group: "core",
+    usage: "docspack verify [options]",
+    options: [["--package-dir <d>", "verify one package directory instead of what is installed"]],
+    detail: [
+      "Compares the identifiers the chunks name against what the documented libraries declare,",
+      "read from their .d.ts files. Nothing you depend on is imported or executed. The libraries",
+      'come from the manifest\'s "documents", or from the docspack key when the manifest has none.',
+      "Exits 1 when anything was reported.",
+    ],
+    examples: ["docspack verify", "docspack verify --json"],
+  },
+  {
+    name: "feedback",
+    summary: "Record documentation problems: add, list, submit, remove",
+    group: "core",
+    usage: "docspack feedback <add|list|submit|remove> [options]",
+    options: [
+      ["--chunk <id>", "add: the chunk the problem is in"],
+      ["--kind <k>", "add: drift, incorrect or missing"],
+      ["--evidence <text>", "add: the claim, in one line"],
+      ["--expected <text>", "add: what the documentation led you to expect"],
+      ["--actual <text>", "add: what happened instead"],
+      ["--repro <code>", "add: code that demonstrates it"],
+      ["-p, --package <s>", "submit: only findings from packages matching this text"],
+      ["--all", "remove: every recorded finding"],
+    ],
+    detail: [
+      "Writes to .docspack/feedback.jsonl in this project. Nothing is transmitted: docspack",
+      "contains no code that can send a report anywhere.",
+      "",
+      "`drift` must name the identifier that drifted. `incorrect` and `missing` must carry",
+      "--expected, --actual and --repro, so a claim that cannot show its work cannot be",
+      "recorded. `submit` prints a prefilled GitHub issue URL for a human to open and file.",
+    ],
+    examples: [
+      'docspack feedback add --chunk @acme/docspack@1.4.0/api-auth --kind drift --evidence "client.setKey is not exported; setApiKey is"',
+      "docspack feedback list",
+      "docspack feedback submit",
+      "docspack feedback remove <fingerprint>",
+    ],
+  },
+  {
+    name: "mcp",
+    summary: "Serve the index over MCP instead, as a long-lived process",
+    group: "core",
+    usage: "docspack mcp",
+    options: [],
+    detail: [
+      "Serves the same index over the Model Context Protocol, for clients that prefer a tool",
+      "definition to a shell command. `query_local_docs` returns exactly what `ask` prints.",
+      "stdout is the protocol channel, so nothing else is written to it.",
+    ],
+    examples: ["docspack mcp"],
+  },
+  {
+    name: "sources",
+    summary: "List curated sources that `docspack build` can fetch",
+    group: "core",
+    usage: "docspack sources",
+    options: [],
+    examples: ["docspack sources", "docspack build hono --name @docspack-community/hono"],
+  },
+  {
+    name: "init",
+    summary: "Scaffold a documentation package, then build and check it",
+    group: "authoring",
+    usage: "docspack init [name] [options]",
+    options: [
+      ["-y, --yes", "skip the prompts and use flags plus detected defaults"],
+      ["--dry-run", "print the file tree and write nothing"],
+      ["--mirror <id|url>", "bootstrap from a published llms.txt"],
+      ["--community", "scaffold under @docspack-community"],
+      ["--template <t>", "full (default) or minimal"],
+      ["--mode <m>", "standalone or in-repo"],
+      ["--no-workflow", "skip the release workflow"],
+      ["--no-build", "scaffold only"],
+      ["--name <name>", "package name, e.g. @acme/docspack"],
+      ["--pkg-version <v>", "package version"],
+      ["--from <dir>", "directory of Markdown to package"],
+      ["--openapi <file>", "OpenAPI JSON to package"],
+      ["--out <dir>", "where to scaffold (default: ./docspack)"],
+      ["--force", "overwrite files that already exist"],
+    ],
+    detail: [
+      "Reads the surrounding project first — name, version, a docs directory, an OpenAPI",
+      "document, the git remote, the license — and proposes a package built from what it found.",
+    ],
+    examples: [
+      "docspack init",
+      "docspack init --name @acme/docspack --from ./docs --yes",
+      "docspack init --mirror hono --community --yes",
+    ],
+  },
+  {
+    name: "build",
+    summary: "Generate the .llms/ payload for publishing",
+    group: "authoring",
+    usage: "docspack build [source] [options]",
+    options: [
+      ["--from <dir>", "directory of Markdown to package"],
+      ["--openapi <file>", "OpenAPI JSON to package, one chunk per operation"],
+      ["--name <name>", "package name, e.g. @acme/docspack"],
+      ["--pkg-version <v>", "package version"],
+      ["--out <dir>", "output directory (default: the current directory)"],
+      ["--pages <n>", "maximum documents to fetch from a remote source"],
+      ["--max-chunk-tokens <n>", "split a section above this size (default 800)"],
+      ["--min-chunk-tokens <n>", "pack adjacent sections up to this size before splitting"],
+      ["--documents <p>", "library this package documents, repeatable: acme@1.4.0"],
+    ],
+    detail: [
+      "Splits each document at ## headings, then ###, then paragraphs, until every chunk fits",
+      "the budget. --min-chunk-tokens packs the other way first, merging adjacent sections up to",
+      "the budget: use it for generated reference, where a heading is a field name and one chunk",
+      "per heading is hundreds of chunks too small to answer anything.",
+      "",
+      "With no flags, settings are read from the docspack key of the package.json in --out.",
+    ],
+    examples: [
+      "docspack build",
+      "docspack build --from ./docs --name @acme/docspack --pkg-version 1.4.0",
+      "docspack build --from ./reference --min-chunk-tokens 400 --max-chunk-tokens 900",
+      "docspack build --openapi ./openapi.json",
+    ],
+  },
+  {
+    name: "doctor",
+    summary: "Check a package the way the indexer and a reviewer would",
+    group: "authoring",
+    usage: "docspack doctor [options]",
+    options: [
+      ["--strict", "treat structural warnings as failures"],
+      ["--pedantic", "--strict, and fail on prose style as well"],
+      ["--package-dir <d>", "the package to inspect (default: .)"],
+    ],
+    detail: [
+      "Errors are packages that will not index. Warnings are packages that will index and",
+      "retrieve badly. Prose style is a note unless --pedantic, because writing by hand is not a",
+      "reason to block a first publish. Exits 1 when the package is not ready to publish.",
+    ],
+    examples: ["docspack doctor", "docspack doctor --strict", "docspack doctor --json"],
+  },
+  {
+    name: "preview",
+    summary: "Answer a query from the local package, as an agent would",
+    group: "authoring",
+    usage: "docspack preview <query> [options]",
+    options: [
+      ["--package-dir <d>", "the package to read (default: .)"],
+      ["--limit <n>", `maximum chunks to return (default ${LIMIT})`],
+      ["--max-tokens <n>", `token ceiling for the result set (default ${MAX_TOKENS})`],
+    ],
+    detail: [
+      "Indexes the package in memory and answers through the same ranking and token budget an",
+      "agent gets. Nothing is published, installed, or written to the global store.",
+    ],
+    examples: ['docspack preview "how do I authenticate"'],
+  },
+  {
+    name: "eval",
+    summary: "Measure retrieval against a set of questions",
+    group: "authoring",
+    usage: "docspack eval <queries.json> [options]",
+    options: [
+      ["--package-dir <d>", "the package to read (default: .)"],
+      ["--limit <n>", `chunks per answer, the window a hit must fall in (default ${LIMIT})`],
+      ["--max-tokens <n>", `token ceiling for each answer (default ${MAX_TOKENS})`],
+      ["--min-hit-rate <n>", "exit 1 below this percentage of questions answered"],
+    ],
+    detail: [
+      "The evaluation set is a JSON array of questions and the chunk ids that would answer them:",
+      "",
+      '  [{ "query": "how do I verify a webhook signature", "expect": "webhooks-signing" },',
+      '   { "query": "rate limits", "expect": ["rate-limits", "errors"] }]',
+      "",
+      "Reports hit rate, top-1 rate and mean answer size — the numbers that settle the chunk",
+      "budget, which no structural check can see. With --min-hit-rate it gates a publish on",
+      "retrieval quality the way `doctor --strict` gates it on structure.",
+    ],
+    examples: [
+      "docspack eval ./eval/queries.json",
+      "docspack eval ./eval/queries.json --min-hit-rate 90",
+      "docspack eval ./eval/queries.json --limit 1 --json",
+    ],
+  },
+];
+
+const GLOBAL_OPTIONS: readonly (readonly [string, string])[] = [
+  ["--store <path>", "Use a different index"],
+  ["--cwd <dir>", "Run in a different directory"],
+  ["--json", "Machine-readable output"],
+  ["-q, --quiet", "Only print errors"],
+  ["-h, --help", "Show this help, or a command's help"],
+  ["-v, --version", "Show the version"],
+];
+
+export function findCommandHelp(name: string): CommandHelp | undefined {
+  return COMMANDS.find((command) => command.name === name);
+}
+
+const WIDTH = 21;
+
+function row([flag, description]: readonly [string, string]): string {
+  return `  ${flag.padEnd(WIDTH)} ${description}`;
+}
+
+/** The page one command prints for `docspack <command> --help`. */
+export function commandHelpText(command: CommandHelp): string {
+  const blocks: string[] = [
+    `docspack ${command.name} — ${command.summary}`,
+    "",
+    "Usage",
+    `  ${command.usage}`,
+  ];
+
+  if (command.detail !== undefined) {
+    blocks.push("", ...command.detail.map((line) => (line.length === 0 ? "" : `  ${line}`)));
+  }
+  if (command.options.length > 0) {
+    blocks.push("", "Options", ...command.options.map(row));
+  }
+  blocks.push(
+    "",
+    "Global options",
+    ...GLOBAL_OPTIONS.map(row),
+    "",
+    "Examples",
+    ...command.examples.map((example) => `  ${example}`),
+    "",
+  );
+  return blocks.join("\n");
+}
+
+/**
+ * The global page. It lists the commands and the options that apply to all of them, and points
+ * at the per-command pages for the rest — one list of every flag of every command was what made
+ * the flags unfindable in the first place.
+ */
+export function globalHelpText(storePath: string, notIndexed: number, noMatch: number): string {
+  const group = (which: CommandHelp["group"]): string =>
+    COMMANDS.filter((command) => command.group === which)
+      .map((command) => row([command.name, command.summary]))
+      .join("\n");
+
+  return `docspack — local, version-locked documentation for AI agents
+
+Usage
+  docspack <command> [options]
+  docspack <command> --help      every flag that command takes
+
+Commands
+${group("core")}
+
+Authoring
+${group("authoring")}
+
+How it works
+  Documentation ships as npm packages named @vendor/docspack or
+  @docspack-community/<name>. Add them to package.json, run \`docspack sync\`, and
+  every chunk is indexed into a shared SQLite database with FTS5. Agents query
+  that index locally: no network, no scraping, always the installed version.
+
+Giving an agent access
+  Any agent with a shell can run \`docspack ask\`, so one line in AGENTS.md or
+  CLAUDE.md is the whole setup — no server, no per-agent configuration, and
+  nothing resident when nobody is asking:
+
+${AGENTS_SNIPPET.map((line) => `    ${line}`).join("\n")}
+
+  Add the block below as well if you want the agent to record documentation
+  problems it runs into. It writes to a local file; nothing is sent:
+
+${FEEDBACK_SNIPPET.map((line) => `    ${line}`).join("\n")}
+
+  \`docspack mcp\` serves the same index over the Model Context Protocol for
+  clients that prefer a tool definition. Both return identical text.
+
+Global options
+${GLOBAL_OPTIONS.map(row).join("\n")}
+
+  The default index is ${storePath}.
+
+Exit codes
+  \`ask\` and \`search\` answer with an exit code a script can branch on:
+
+    0  chunks were returned
+    ${notIndexed}  nothing matched, and a docs package is installed but not indexed —
+       run \`docspack sync\`
+    ${noMatch}  nothing matched, and everything installed is already indexed
+    1  the command failed; 2  the command was used wrongly
+
+Examples
+  npx docspack sync
+  npx docspack ask "how do I verify a webhook signature"
+  npx docspack init
+  npx docspack doctor --strict
+  npx docspack preview "how do I authenticate"
+  npx docspack eval ./eval/queries.json --min-hit-rate 90
+  npx docspack feedback submit
+`;
+}

package/src/index.ts

@@ -32,6 +32,15 @@ export {
   type Severity,
 } from "./doctor.js";
 export { DocspackError } from "./errors.js";
+export {
+  type EvalCase,
+  type EvalCaseResult,
+  type EvalOptions,
+  type EvalReport,
+  evaluatePackage,
+  parseEvalSet,
+  readEvalSet,
+} from "./eval.js";
 export { type ExportSurface, readExportSurface } from "./exports.js";
 export {
   type AddOptions,
@@ -46,6 +55,13 @@ export {
   readFeedback,
   removeFindings,
 } from "./feedback.js";
+export {
+  COMMANDS,
+  type CommandHelp,
+  commandHelpText,
+  findCommandHelp,
+  globalHelpText,
+} from "./help.js";
 export { type FetchLike, HttpClient } from "./http.js";
 export { type Detected, detectProject, type PackageManager } from "./init/detect.js";
 export {
@@ -92,7 +108,9 @@ export {
   CHUNKS_DIR,
   type ChunkSpec,
   chunkId,
+  type DocumentedLibrary,
   estimateTokens,
+  formatDocumentedLibrary,
   isCommunityPackage,
   isDocsPackage,
   isVendorPackage,
@@ -100,12 +118,14 @@ export {
   MANIFEST_FILE,
   type PackageManifest,
   packageId,
+  parseDocumentedLibrary,
   parseManifest,
   resolveChunkFile,
   SCHEMA_URL,
   SPEC_URL,
   serializeManifest,
 } from "./spec.js";
+export { STOPWORDS } from "./stopwords.js";
 export {
   type BlockedFinding,
   type BlockedReason,

package/src/preview.ts

@@ -6,6 +6,7 @@ import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS, type QueryHit } from "./search.js";
 import {
   chunkId,
   estimateTokens,
+  formatDocumentedLibrary,
   isCommunityPackage,
   LLMS_DIR,
   MANIFEST_FILE,
@@ -45,6 +46,7 @@ export async function previewPackage(options: PreviewOptions): Promise<PreviewRe
 
   const manifest = parseManifest(JSON.parse(raw), `${LLMS_DIR}/${MANIFEST_FILE}`);
   const id = packageId(manifest.name, manifest.version);
+  const documents = (manifest.documents ?? []).map(formatDocumentedLibrary);
 
   const chunks: IndexedChunk[] = [];
   for (const chunk of manifest.chunks) {
@@ -58,7 +60,7 @@ export async function previewPackage(options: PreviewOptions): Promise<PreviewRe
     chunks.push({
       chunkId: chunkId(id, chunk.id),
       filePath: chunk.file,
-      tokens: chunk.tokens > 0 ? chunk.tokens : estimateTokens(contents),
+      tokens: chunk.tokens ?? estimateTokens(contents),
       content: contents,
       tags: [...chunk.tags, ...chunk.entities],
     });
@@ -84,6 +86,7 @@ export async function previewPackage(options: PreviewOptions): Promise<PreviewRe
           name: manifest.name,
           version: manifest.version,
           trusted: !isCommunityPackage(manifest.name),
+          ...(documents.length === 0 ? {} : { documents }),
         }),
       );