docspack 0.4.0 → 1.1.0

package/src/db.ts

@@ -8,13 +8,57 @@ import { STOPWORDS } from "./stopwords.js";
 
 const require = createRequire(import.meta.url);
 
+/**
+ * Where a package's text came from. `docs` is written by a human and published as a docs
+ * package; `artifact` is derived from an installed library's own type declarations; `local` is a
+ * working corpus indexed from this project's own sources and never published. The
+ * distinction is not cosmetic — a signature is a fact about the build, prose is a claim by its
+ * author, and a working corpus is a copy that can go stale under the reader — so it is stored
+ * rather than inferred from the name.
+ */
+export type PackageKind = "docs" | "artifact" | "local";
+
+/**
+ * Reads the stored kind, defaulting an unrecognized one to `docs`.
+ *
+ * Written once rather than inline at each read: a store may have been written by a newer version
+ * that knows a kind this one does not, and silently relabelling it as something it is not is how
+ * a local corpus would end up presented as published documentation.
+ */
+export function toPackageKind(value: string): PackageKind {
+  return value === "artifact" || value === "local" ? value : "docs";
+}
+
 export interface IndexedPackage {
   readonly id: string;
   readonly name: string;
   readonly version: string;
+  readonly kind?: PackageKind;
   readonly indexedAt?: string;
 }
 
+/** An exported name and the chunk carrying its declaration. */
+export interface SymbolHit {
+  readonly name: string;
+  readonly packageId: string;
+  readonly chunkId: string;
+}
+
+/**
+ * A file an indexed package was built from, as it was at index time.
+ *
+ * Only a working corpus records these. Published documentation is immutable for the life of its
+ * version, so there is nothing to detect; a corpus of the reader's own files is edited underneath
+ * the index, and an answer quoted from a superseded source is worse than no answer.
+ */
+export interface IndexedSource {
+  readonly path: string;
+  readonly size: number;
+  /** ISO timestamp, as text, because that is what survives a round trip through SQLite. */
+  readonly mtime: string;
+  readonly hash: string;
+}
+
 export interface IndexedChunk {
   readonly chunkId: string;
   readonly filePath: string;
@@ -39,15 +83,21 @@ export interface SearchOptions {
   readonly limit?: number;
   /** Stop returning chunks once this many tokens have been collected. */
   readonly maxTokens?: number;
+  /** Restrict to packages of these kinds. Defaults to every kind. */
+  readonly kinds?: readonly PackageKind[];
 }
 
-const SCHEMA_VERSION = 1;
+/** Where per-project state lives, alongside the feedback file already written there. */
+export const LOCAL_STORE_DIR = ".docspack";
+
+const SCHEMA_VERSION = 3;
 
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS packages (
     id TEXT PRIMARY KEY,
     name TEXT NOT NULL,
     version TEXT NOT NULL,
+    kind TEXT NOT NULL DEFAULT 'docs',
     indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP
 );
 
@@ -61,6 +111,24 @@ CREATE TABLE IF NOT EXISTS chunks (
 
 CREATE INDEX IF NOT EXISTS chunks_by_package ON chunks(package_id);
 
+CREATE TABLE IF NOT EXISTS symbols (
+    package_id TEXT NOT NULL REFERENCES packages(id),
+    name TEXT NOT NULL,
+    chunk_id TEXT NOT NULL,
+    PRIMARY KEY (package_id, name)
+);
+
+CREATE INDEX IF NOT EXISTS symbols_by_name ON symbols(name);
+
+CREATE TABLE IF NOT EXISTS sources (
+    package_id TEXT NOT NULL REFERENCES packages(id),
+    path TEXT NOT NULL,
+    size INTEGER NOT NULL,
+    mtime TEXT NOT NULL,
+    hash TEXT NOT NULL,
+    PRIMARY KEY (package_id, path)
+);
+
 CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
     content,
     tags,
@@ -92,6 +160,17 @@ function loadSqlite(): typeof import("node:sqlite") {
   return require("node:sqlite") as typeof import("node:sqlite");
 }
 
+/**
+ * Where a project's own working corpus is indexed.
+ *
+ * Deliberately not the global store: these chunks are a plaintext copy of one project's sources,
+ * they are worthless to any other project, and a machine-wide store would accumulate them from
+ * every checkout with nothing to evict them.
+ */
+export function localStorePath(cwd: string): string {
+  return join(cwd, LOCAL_STORE_DIR, "local.db");
+}
+
 /** Default location of the shared index, mirroring pnpm's global store. */
 export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
   const override = env.DOCSPACK_STORE;
@@ -133,9 +212,22 @@ export class Store {
     this.#db = db;
     this.#db.exec("PRAGMA journal_mode = WAL");
     this.#db.exec(SCHEMA);
+    this.#migrate();
     this.#db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
   }
 
+  /**
+   * Brings a store written by an older version forward. `CREATE TABLE IF NOT EXISTS` covers a
+   * new table, but not a new column on one that already exists, and rebuilding the whole index
+   * to add one would make an upgrade re-read every package on the machine.
+   */
+  #migrate(): void {
+    const columns = this.#db.prepare("PRAGMA table_info(packages)").all() as { name: string }[];
+    if (!columns.some((column) => column.name === "kind")) {
+      this.#db.exec("ALTER TABLE packages ADD COLUMN kind TEXT NOT NULL DEFAULT 'docs'");
+    }
+  }
+
   static open(path: string = defaultStorePath()): Store {
     if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
     try {
@@ -154,16 +246,115 @@ export class Store {
 
   listPackages(): IndexedPackage[] {
     const rows = this.#db
-      .prepare("SELECT id, name, version, indexed_at FROM packages ORDER BY name, version")
-      .all() as { id: string; name: string; version: string; indexed_at: string }[];
+      .prepare("SELECT id, name, version, kind, indexed_at FROM packages ORDER BY name, version")
+      .all() as {
+      id: string;
+      name: string;
+      version: string;
+      kind: string;
+      indexed_at: string;
+    }[];
     return rows.map((row) => ({
       id: row.id,
       name: row.name,
       version: row.version,
+      kind: toPackageKind(row.kind),
       indexedAt: row.indexed_at,
     }));
   }
 
+  /** Every version of one library in the store, newest indexing first. */
+  versionsOf(name: string): IndexedPackage[] {
+    return this.listPackages().filter((pkg) => pkg.name === name);
+  }
+
+  /**
+   * Chunks declaring an exported name, by exact match rather than by ranking.
+   *
+   * A symbol is a key, not a phrase. Ranking one is the mistake that made `docspack ask` unable
+   * to tell "the documentation does not mention this" from "nothing matched".
+   */
+  lookupSymbol(name: string, packageIds?: readonly string[]): SymbolHit[] {
+    if (packageIds !== undefined && packageIds.length === 0) return [];
+    const scope =
+      packageIds === undefined
+        ? ""
+        : ` AND package_id IN (${packageIds.map(() => "?").join(", ")})`;
+    const rows = this.#db
+      .prepare(
+        // A package that declares the name comes first: an empty chunk id means the name is
+        // exported but its declaration was not readable, which answers less.
+        `SELECT name, package_id AS packageId, chunk_id AS chunkId
+           FROM symbols WHERE name = ?${scope}
+          ORDER BY (chunk_id = '') ASC, package_id`,
+      )
+      .all(name, ...(packageIds ?? [])) as {
+      name: string;
+      packageId: string;
+      chunkId: string;
+    }[];
+    return rows.map((row) => ({ name: row.name, packageId: row.packageId, chunkId: row.chunkId }));
+  }
+
+  /** Every chunk of one package, for a check that has to read the whole corpus. */
+  chunksOf(packageId: string): SearchHit[] {
+    const rows = this.#db
+      .prepare(
+        `SELECT chunk_id AS chunkId, package_id AS packageId, file_path AS filePath,
+                tokens, content
+           FROM chunks WHERE package_id = ?`,
+      )
+      .all(packageId) as {
+      chunkId: string;
+      packageId: string;
+      filePath: string;
+      tokens: number;
+      content: string;
+    }[];
+    return rows.map((row) => ({
+      chunkId: row.chunkId,
+      packageId: row.packageId,
+      filePath: row.filePath,
+      tokens: Number(row.tokens ?? 0),
+      content: row.content,
+    }));
+  }
+
+  /** Every exported name recorded for one package. */
+  symbolsOf(packageId: string): string[] {
+    const rows = this.#db
+      .prepare("SELECT name FROM symbols WHERE package_id = ? ORDER BY name")
+      .all(packageId) as { name: string }[];
+    return rows.map((row) => row.name);
+  }
+
+  /** One chunk by its id, however it was found. */
+  chunk(chunkId: string): SearchHit | undefined {
+    const row = this.#db
+      .prepare(
+        `SELECT chunk_id AS chunkId, package_id AS packageId, file_path AS filePath,
+                tokens, content
+           FROM chunks WHERE chunk_id = ?`,
+      )
+      .get(chunkId) as
+      | {
+          chunkId: string;
+          packageId: string;
+          filePath: string;
+          tokens: number;
+          content: string;
+        }
+      | undefined;
+    if (row === undefined) return undefined;
+    return {
+      chunkId: row.chunkId,
+      packageId: row.packageId,
+      filePath: row.filePath,
+      tokens: Number(row.tokens ?? 0),
+      content: row.content,
+    };
+  }
+
   countChunks(packageId: string): number {
     const row = this.#db
       .prepare("SELECT COUNT(*) AS total FROM chunks WHERE package_id = ?")
@@ -171,10 +362,19 @@ export class Store {
     return row.total;
   }
 
-  /** Replaces a package and all of its chunks in a single transaction. */
-  indexPackage(pkg: IndexedPackage, chunks: readonly IndexedChunk[]): void {
+  /**
+   * Replaces a package, all of its chunks and its symbol table in a single transaction.
+   *
+   * `symbols` maps an exported name to the chunk declaring it, and is how a query about a name
+   * is answered without ranking anything.
+   */
+  indexPackage(
+    pkg: IndexedPackage,
+    chunks: readonly IndexedChunk[],
+    symbols?: ReadonlyMap<string, string>,
+  ): void {
     const insertPackage = this.#db.prepare(
-      "INSERT OR REPLACE INTO packages (id, name, version) VALUES (?, ?, ?)",
+      "INSERT OR REPLACE INTO packages (id, name, version, kind) VALUES (?, ?, ?, ?)",
     );
     const insertChunk = this.#db.prepare(
       "INSERT OR REPLACE INTO chunks (chunk_id, package_id, file_path, tokens, content) VALUES (?, ?, ?, ?, ?)",
@@ -182,15 +382,37 @@ export class Store {
     const insertFts = this.#db.prepare(
       "INSERT INTO chunks_fts (content, tags, package_id, chunk_id) VALUES (?, ?, ?, ?)",
     );
+    const insertSymbol = this.#db.prepare(
+      "INSERT OR REPLACE INTO symbols (package_id, name, chunk_id) VALUES (?, ?, ?)",
+    );
 
     this.#db.exec("BEGIN");
     try {
       this.#deleteChunks(pkg.id);
-      insertPackage.run(pkg.id, pkg.name, pkg.version);
+      insertPackage.run(pkg.id, pkg.name, pkg.version, pkg.kind ?? "docs");
       for (const chunk of chunks) {
         insertChunk.run(chunk.chunkId, pkg.id, chunk.filePath, chunk.tokens, chunk.content);
         insertFts.run(chunk.content, chunk.tags.join(" "), pkg.id, chunk.chunkId);
       }
+      for (const [name, chunk] of symbols ?? []) insertSymbol.run(pkg.id, name, chunk);
+      this.#db.exec("COMMIT");
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+
+  /** Replaces the record of what a package was built from. */
+  recordSources(packageId: string, sources: readonly IndexedSource[]): void {
+    const insert = this.#db.prepare(
+      "INSERT OR REPLACE INTO sources (package_id, path, size, mtime, hash) VALUES (?, ?, ?, ?, ?)",
+    );
+    this.#db.exec("BEGIN");
+    try {
+      this.#db.prepare("DELETE FROM sources WHERE package_id = ?").run(packageId);
+      for (const source of sources) {
+        insert.run(packageId, source.path, source.size, source.mtime, source.hash);
+      }
       this.#db.exec("COMMIT");
     } catch (error) {
       this.#db.exec("ROLLBACK");
@@ -198,6 +420,18 @@ export class Store {
     }
   }
 
+  sourcesOf(packageId: string): IndexedSource[] {
+    const rows = this.#db
+      .prepare("SELECT path, size, mtime, hash FROM sources WHERE package_id = ? ORDER BY path")
+      .all(packageId) as { path: string; size: number; mtime: string; hash: string }[];
+    return rows.map((row) => ({
+      path: row.path,
+      size: Number(row.size),
+      mtime: row.mtime,
+      hash: row.hash,
+    }));
+  }
+
   removePackage(id: string): void {
     this.#db.exec("BEGIN");
     try {
@@ -228,6 +462,13 @@ export class Store {
       conditions.push("f.package_id LIKE ?");
       parameters.push(options.packageFilter);
     }
+    if (options.kinds !== undefined) {
+      if (options.kinds.length === 0) return [];
+      conditions.push(
+        `f.package_id IN (SELECT id FROM packages WHERE kind IN (${options.kinds.map(() => "?").join(", ")}))`,
+      );
+      parameters.push(...options.kinds);
+    }
 
     const rows = this.#db
       .prepare(
@@ -270,7 +511,9 @@ export class Store {
   }
 
   #deleteChunks(packageId: string): void {
+    this.#db.prepare("DELETE FROM sources WHERE package_id = ?").run(packageId);
     this.#db.prepare("DELETE FROM chunks_fts WHERE package_id = ?").run(packageId);
     this.#db.prepare("DELETE FROM chunks WHERE package_id = ?").run(packageId);
+    this.#db.prepare("DELETE FROM symbols WHERE package_id = ?").run(packageId);
   }
 }

package/src/discovery.ts

@@ -23,6 +23,14 @@ export interface DiscoveredPackage {
   readonly trusted: boolean;
 }
 
+/** An ordinary dependency: a library this project installed, whatever documents it. */
+export interface DiscoveredLibrary {
+  readonly name: string;
+  readonly version: string;
+  /** Root of the installed package. */
+  readonly dir: string;
+}
+
 export interface Discovery {
   readonly packages: readonly DiscoveredPackage[];
   /** Human-readable problems that did not stop discovery, e.g. a declared but uninstalled package. */
@@ -90,19 +98,48 @@ export async function discoverPackages(cwd: string): Promise<Discovery> {
   return { packages, problems };
 }
 
+/**
+ * Every library this project depends on directly, installed and readable.
+ *
+ * Documentation packages are excluded: those are handled by `discoverPackages`, and a docs
+ * package's own type declarations describe nothing anybody asks about.
+ */
+export async function discoverLibraries(cwd: string): Promise<readonly DiscoveredLibrary[]> {
+  const root = resolve(cwd);
+  const names = (await declaredDependencies(root)).filter((name) => !isDocsPackage(name));
+  const libraries: DiscoveredLibrary[] = [];
+
+  for (const name of names) {
+    const dir = await resolvePackageDir(name, root);
+    if (dir === undefined) continue;
+    try {
+      libraries.push({ name, version: await installedVersion(name, dir), dir });
+    } catch {
+      // A dependency with no version in its manifest is not one to index.
+    }
+  }
+
+  return libraries;
+}
+
 /** Package ids installed in this project, used to scope queries to the versions actually in use. */
 export async function projectPackageIds(cwd: string): Promise<string[]> {
   const { packages } = await discoverPackages(cwd);
   return packages.map((pkg) => pkg.id);
 }
 
+/** Docs packages this project depends on, the input to `discoverPackages`. */
+async function declaredDocsPackages(cwd: string): Promise<string[]> {
+  return (await declaredDependencies(cwd)).filter(isDocsPackage);
+}
+
 /**
- * Every docs package declared by this directory or by an ancestor of it. A workspace declares
+ * Every dependency declared by this directory or by an ancestor of it. A workspace declares
  * shared tooling in the repository root and its members inherit the install, so reading only
  * `cwd/package.json` answers "this project has no documentation" for most monorepos — the same
  * upward walk `resolvePackageDir` already does for the install.
  */
-async function declaredDocsPackages(cwd: string): Promise<string[]> {
+async function declaredDependencies(cwd: string): Promise<string[]> {
   const names = new Set<string>();
   let found = false;
   let dir = cwd;
@@ -131,9 +168,7 @@ async function declaredDocsPackages(cwd: string): Promise<string[]> {
     const manifest = parsed as { dependencies?: unknown; devDependencies?: unknown };
     for (const field of [manifest.dependencies, manifest.devDependencies]) {
       if (typeof field !== "object" || field === null) continue;
-      for (const name of Object.keys(field as Record<string, unknown>)) {
-        if (isDocsPackage(name)) names.add(name);
-      }
+      for (const name of Object.keys(field as Record<string, unknown>)) names.add(name);
     }
 
     const parent = dirname(dir);

package/src/doctor.ts

@@ -2,6 +2,7 @@ import type { Dirent } from "node:fs";
 import { readdir, readFile, stat } from "node:fs/promises";
 import { join } from "node:path";
 import { readBuildConfig } from "./config.js";
+import { measureCoverage } from "./coverage.js";
 import { PLACEHOLDER } from "./init/templates.js";
 import {
   estimateTokens,
@@ -48,6 +49,13 @@ export interface DoctorOptions {
   readonly pedantic?: boolean;
 }
 
+/**
+ * Below this share of a library's exported names, a package is documenting a fraction of what
+ * the library publishes. Reported, never gated on: a page listing every export and explaining
+ * none would score full marks, so this is a number for an author to read, not a wall.
+ */
+const THIN_COVERAGE = 0.6;
+
 /** Over this, a chunk crowds out the response budget; under it, a chunk answers nothing. */
 const MAX_CHUNK_TOKENS = 1500;
 const MIN_CHUNK_TOKENS = 30;
@@ -199,6 +207,8 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
     });
   }
 
+  findings.push(...(await coverageFindings(options.dir, llmsDir, manifest)));
+
   if (manifest.chunks.every((chunk) => chunk.entities.length === 0)) {
     findings.push({
       check: "no-entities",
@@ -352,21 +362,29 @@ function checkPackageJson(
   }
 }
 
-/** Newest mtime across the configured documentation source, used to spot a stale payload. */
+/**
+ * Newest mtime across every configured documentation source, used to spot a stale payload.
+ *
+ * Both sources are checked, not the first one found: a package configured with prose *and* an
+ * OpenAPI document would otherwise report a fresh payload after the document changed, which is the
+ * one case this check exists to catch.
+ */
 async function newestSourceMtime(dir: string): Promise<number> {
   const config = await readBuildConfig(dir);
-  const source = config.from ?? config.openapi;
-  if (source === undefined) return 0;
-
-  const target = join(dir, source);
-  const single = await mtime(target);
-  if (single > 0 && config.openapi !== undefined) return single;
+  const newest = await Promise.all([
+    config.openapi === undefined ? 0 : mtime(join(dir, config.openapi)),
+    config.from === undefined ? 0 : newestInTree(join(dir, config.from)),
+  ]);
+  return Math.max(...newest);
+}
 
+/** Newest mtime in a directory tree, or the file's own when the path is a file. */
+async function newestInTree(target: string): Promise<number> {
   let found: Dirent[];
   try {
     found = await readdir(target, { recursive: true, withFileTypes: true });
   } catch {
-    return single;
+    return mtime(target);
   }
 
   let newest = 0;
@@ -385,6 +403,48 @@ async function mtime(path: string): Promise<number> {
   }
 }
 
+/**
+ * How much of the documented library's public surface the package mentions.
+ *
+ * A documentation site is written page by page around tasks and an API grows name by name, so
+ * they drift apart with nobody noticing. This is the check that notices — mechanical, from two
+ * files, with no model and no judgement in it.
+ */
+async function coverageFindings(
+  dir: string,
+  llmsDir: string,
+  manifest: PackageManifest,
+): Promise<Finding[]> {
+  const findings: Finding[] = [];
+  let report: Awaited<ReturnType<typeof measureCoverage>>;
+  try {
+    report = await measureCoverage({ packageDir: dir, llmsDir, manifest, cwd: dir });
+  } catch {
+    // Coverage is a note about the documentation, never a reason a package fails to check.
+    return findings;
+  }
+
+  for (const library of report.libraries) {
+    if (library.names === 0) continue;
+    const share = library.documented / library.names;
+    const percent = Math.round(100 * share);
+    findings.push({
+      check: "coverage",
+      severity: "info",
+      message:
+        `mentions ${library.documented} of ${library.names} names ${library.library} exports (${percent}%)` +
+        (library.uncovered.length === 0 ? "" : `; missing ${library.uncovered.join(", ")}`),
+      ...(share < THIN_COVERAGE
+        ? {
+            fix: "Document the exports nobody has written about, or narrow what the package claims to document.",
+          }
+        : {}),
+    });
+  }
+
+  return findings;
+}
+
 async function readJson(path: string): Promise<Record<string, unknown> | undefined> {
   try {
     const parsed: unknown = JSON.parse(await readFile(path, "utf8"));