docspack 0.2.0 → 0.4.0

package/src/search.ts

@@ -1,6 +1,6 @@
 import type { SearchHit, Store } from "./db.js";
 import { discoverPackages } from "./discovery.js";
-import { isCommunityPackage } from "./spec.js";
+import { chunkId, formatDocumentedLibrary, isCommunityPackage } from "./spec.js";
 
 /** Ceiling on how much context one query may return, per the blueprint's context-exhaustion rule. */
 export const DEFAULT_MAX_TOKENS = 3000;
@@ -25,6 +25,13 @@ export interface QueryHit extends SearchHit {
   readonly name: string;
   readonly version: string;
   readonly trusted: boolean;
+  /**
+   * Libraries this chunk documents, from its manifest. A docs package version cannot always
+   * imply this — a monorepo documents many libraries at many versions from one surface — so the
+   * answer states it rather than leaving the reader to infer it from the package name. A chunk
+   * that names its own libraries answers for those; otherwise the package's list stands.
+   */
+  readonly documents?: readonly string[];
 }
 
 export interface QueryResult {
@@ -48,6 +55,22 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
     .filter((pkg) => !options.store.hasPackage(pkg.id))
     .map((pkg) => pkg.id);
 
+  // Read from the installed manifests rather than the index: the store is a cache of chunk text,
+  // and adding a column to it would make every existing store need a rebuild to answer this.
+  // Keyed by package id and by chunk id in the same map, because a chunk id already carries its
+  // package: `@acme/docspack@1.4.0/api-auth` cannot collide with `@acme/docspack@1.4.0`.
+  const documented = new Map<string, readonly string[]>();
+  for (const pkg of installed ?? []) {
+    const documents = pkg.manifest.documents;
+    if (documents !== undefined && documents.length > 0) {
+      documented.set(pkg.id, documents.map(formatDocumentedLibrary));
+    }
+    for (const chunk of pkg.manifest.chunks) {
+      if (chunk.documents === undefined || chunk.documents.length === 0) continue;
+      documented.set(chunkId(pkg.id, chunk.id), chunk.documents.map(formatDocumentedLibrary));
+    }
+  }
+
   const hits = options.store
     .search(options.query, {
       ...(packageIds === undefined ? {} : { packageIds }),
@@ -59,7 +82,15 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
     })
     .map((hit): QueryHit => {
       const { name, version } = splitPackageId(hit.packageId);
-      return { ...hit, name, version, trusted: !isCommunityPackage(name) };
+      // The chunk's own libraries win: they are the narrower, and therefore the truer, claim.
+      const documents = documented.get(hit.chunkId) ?? documented.get(hit.packageId);
+      return {
+        ...hit,
+        name,
+        version,
+        trusted: !isCommunityPackage(name),
+        ...(documents === undefined ? {} : { documents }),
+      };
     });
 
   return {
@@ -90,9 +121,16 @@ export function renderAnswer(result: QueryResult, query: string): string {
 
   const sections = result.hits.map((hit) => {
     const trust = hit.trusted ? "" : " (community)";
+    // Which release the chunk describes, on the line that already carries provenance. An agent
+    // otherwise has to assume the docs package version is the library version, which a
+    // repository publishing eighteen packages from one docs surface cannot make true.
+    const describes =
+      hit.documents === undefined || hit.documents.length === 0
+        ? ""
+        : ` · documents ${hit.documents.join(", ")}`;
     return [
       `## ${hit.chunkId}${trust}`,
-      `Source: ${hit.name}@${hit.version} — ${hit.filePath}`,
+      `Source: ${hit.name}@${hit.version} — ${hit.filePath}${describes}`,
       "",
       hit.content,
     ].join("\n");

package/src/spec.ts

@@ -14,14 +14,33 @@ export interface ChunkSpec {
   readonly id: string;
   /** Path relative to the package's `.llms/` directory. */
   readonly file: string;
-  readonly tokens: number;
+  /** Absent means "estimate it". Never 0: an empty chunk is a different problem. */
+  readonly tokens?: number;
   readonly tags: readonly string[];
   readonly entities: readonly string[];
+  /**
+   * What this chunk describes, when that is narrower than what the package describes. A
+   * repository publishing eighteen libraries at four versions from one documentation surface
+   * has a package-level answer that is true of the pack and useless about any one chunk.
+   */
+  readonly documents?: readonly DocumentedLibrary[];
+}
+
+/** A library release the package documents, e.g. `acme` at `1.4.0`. */
+export interface DocumentedLibrary {
+  readonly name: string;
+  /** A version or a range, as the author wrote it. Absent when the docs are not release-scoped. */
+  readonly version?: string;
 }
 
 export interface PackageManifest {
   readonly name: string;
   readonly version: string;
+  /**
+   * What the package documents, which the package's own version cannot always say: a monorepo
+   * publishes many libraries at many versions from one documentation surface.
+   */
+  readonly documents?: readonly DocumentedLibrary[];
   readonly chunks: readonly ChunkSpec[];
 }
 
@@ -50,6 +69,21 @@ export function chunkId(pkgId: string, chunk: string): string {
   return `${pkgId}/${chunk}`;
 }
 
+/**
+ * Reads `acme`, `acme@1.4.0` or `@acme/sdk@^1.4.0` as a library and an optional version. The
+ * separator is the last `@`, since a scope puts one at the start of the name as well.
+ */
+export function parseDocumentedLibrary(spec: string): DocumentedLibrary {
+  const at = spec.lastIndexOf("@");
+  if (at <= 0) return { name: spec };
+  return { name: spec.slice(0, at), version: spec.slice(at + 1) };
+}
+
+/** `acme@1.4.0`, or just the name when no version is scoped. The form an answer is labelled with. */
+export function formatDocumentedLibrary(library: DocumentedLibrary): string {
+  return library.version === undefined ? library.name : `${library.name}@${library.version}`;
+}
+
 /**
  * Resolves a manifest `file` entry inside the package's `.llms/` directory, refusing anything
  * that escapes it. Manifests are third-party input, so this is a security boundary, not a
@@ -88,6 +122,7 @@ export function parseManifest(raw: unknown, where: string): PackageManifest {
   if (typeof version !== "string" || version.length === 0) fail('missing string field "version"');
   if (!Array.isArray(root.chunks)) fail('missing "chunks" array');
 
+  const packageDocuments = parseDocuments(root.documents, fail);
   const seen = new Set<string>();
   const chunks = (root.chunks as unknown[]).map((entry, index): ChunkSpec => {
     if (typeof entry !== "object" || entry === null)
@@ -106,21 +141,48 @@ export function parseManifest(raw: unknown, where: string): PackageManifest {
       return fail(`chunk "${id}" is missing its "file"`);
     }
 
+    // 0 is refused rather than read as "estimate it": the schema accepts it as a deliberate
+    // count, the reader would treat it as absent, and no chunk is genuinely 0 tokens.
     const tokens = chunk.tokens;
-    if (tokens !== undefined && (!Number.isInteger(tokens) || (tokens as number) < 0)) {
-      return fail(`chunk "${id}" has an invalid "tokens" value`);
+    if (tokens !== undefined && (!Number.isInteger(tokens) || (tokens as number) < 1)) {
+      return fail(`chunk "${id}" has an invalid "tokens" value; omit it to have it estimated`);
     }
 
+    const documents = parseDocuments(chunk.documents, fail);
+
     return {
       id,
       file,
-      tokens: typeof tokens === "number" ? tokens : 0,
+      ...(typeof tokens === "number" ? { tokens } : {}),
       tags: stringArray(chunk.tags, `chunk "${id}" field "tags"`, fail),
       entities: stringArray(chunk.entities, `chunk "${id}" field "entities"`, fail),
+      ...(documents === undefined ? {} : { documents }),
     };
   });
 
-  return { name, version, chunks };
+  return {
+    name,
+    version,
+    ...(packageDocuments === undefined ? {} : { documents: packageDocuments }),
+    chunks,
+  };
+}
+
+/**
+ * `["acme@1.4.0", "@acme/sdk@^2"]`. An array even for one library, because that is what the
+ * schema declares — a reader that accepts a shape the validator rejects is the same defect as a
+ * validator that accepts a value the reader reinterprets.
+ */
+function parseDocuments(
+  value: unknown,
+  fail: (message: string) => never,
+): readonly DocumentedLibrary[] | undefined {
+  if (value === undefined) return undefined;
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string" || entry === "")) {
+    fail('"documents" must be an array of package names, optionally with a version');
+  }
+  const specs = value as string[];
+  return specs.length === 0 ? undefined : specs.map(parseDocumentedLibrary);
 }
 
 function stringArray(
@@ -135,6 +197,22 @@ function stringArray(
   return value as string[];
 }
 
+/** Writes the manifest, with `documents` back in the string form the schema declares. */
 export function serializeManifest(manifest: PackageManifest): string {
-  return `${JSON.stringify({ $schema: SCHEMA_URL, ...manifest }, null, 2)}\n`;
+  const documents = manifest.documents ?? [];
+  return `${JSON.stringify(
+    {
+      $schema: SCHEMA_URL,
+      name: manifest.name,
+      version: manifest.version,
+      ...(documents.length === 0 ? {} : { documents: documents.map(formatDocumentedLibrary) }),
+      chunks: manifest.chunks.map((chunk) =>
+        chunk.documents === undefined
+          ? chunk
+          : { ...chunk, documents: chunk.documents.map(formatDocumentedLibrary) },
+      ),
+    },
+    null,
+    2,
+  )}\n`;
 }

package/src/stopwords.ts

@@ -0,0 +1,120 @@
+/**
+ * Words that carry no discrimination in a documentation corpus.
+ *
+ * Heading words become tags, and tags are weighted 3× against prose in the ranking. A section
+ * titled "How to use it" therefore carried `how`, `to` and `it` as high-weight terms, and every
+ * question beginning "how do I…" matched it — ahead of the chunk that answers the question.
+ *
+ * The list is closed-class only: articles, pronouns, prepositions, conjunctions, auxiliaries,
+ * modals, demonstratives and question words. No content word is in it, however generic it
+ * looks. `use`, `add`, `get` and `set` all name real APIs, and a list that grows past the
+ * function words starts deciding which of an author's own words count.
+ */
+export const STOPWORDS: ReadonlySet<string> = new Set([
+  "about",
+  "after",
+  "all",
+  "also",
+  "an",
+  "and",
+  "any",
+  "are",
+  "as",
+  "at",
+  "be",
+  "been",
+  "before",
+  "being",
+  "between",
+  "both",
+  "but",
+  "by",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "doing",
+  "each",
+  "for",
+  "from",
+  "had",
+  "has",
+  "have",
+  "he",
+  "her",
+  "here",
+  "hers",
+  "him",
+  "his",
+  "how",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "its",
+  "may",
+  "me",
+  "might",
+  "must",
+  "my",
+  "no",
+  "nor",
+  "not",
+  "of",
+  "off",
+  "on",
+  "once",
+  "only",
+  "or",
+  "other",
+  "our",
+  "ours",
+  "out",
+  "over",
+  "own",
+  "same",
+  "shall",
+  "she",
+  "should",
+  "so",
+  "some",
+  "such",
+  "than",
+  "that",
+  "the",
+  "their",
+  "theirs",
+  "them",
+  "then",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "through",
+  "to",
+  "too",
+  "under",
+  "until",
+  "up",
+  "very",
+  "was",
+  "we",
+  "were",
+  "what",
+  "when",
+  "where",
+  "which",
+  "while",
+  "who",
+  "whom",
+  "why",
+  "will",
+  "with",
+  "would",
+  "you",
+  "your",
+  "yours",
+]);

package/src/sync.ts

@@ -102,7 +102,7 @@ async function readChunks(
     chunks.push({
       chunkId: chunkId(pkg.id, chunk.id),
       filePath: chunk.file,
-      tokens: chunk.tokens > 0 ? chunk.tokens : estimateTokens(text),
+      tokens: chunk.tokens ?? estimateTokens(text),
       content: text,
       tags: [...chunk.tags, ...chunk.entities],
     });

package/src/verify.ts

@@ -3,8 +3,17 @@ import { join } from "node:path";
 import { readBuildConfig } from "./config.js";
 import { discoverPackages, resolvePackageDir } from "./discovery.js";
 import { DocspackError } from "./errors.js";
-import { type ExportSurface, readExportSurface } from "./exports.js";
-import { chunkId, LLMS_DIR, MANIFEST_FILE, type PackageManifest, parseManifest } from "./spec.js";
+import { readExportSurface } from "./exports.js";
+import {
+  chunkId,
+  type DocumentedLibrary,
+  formatDocumentedLibrary,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  type PackageManifest,
+  parseDocumentedLibrary,
+  parseManifest,
+} from "./spec.js";
 
 export interface DriftFinding {
   readonly chunkId: string;
@@ -22,8 +31,14 @@ export type VerifyStatus = "verified" | "skipped";
 export interface VerifiedPackage {
   readonly id: string;
   readonly status: VerifyStatus;
-  /** Library the package declares it documents. */
-  readonly documents?: string;
+  /** Libraries the package declares it documents, as written. */
+  readonly documents?: readonly string[];
+  /**
+   * Documented libraries whose declarations could not be read, when others could. The check ran
+   * against a smaller surface than the package claims, which is worth saying: an entity those
+   * libraries export is one this run could have accused.
+   */
+  readonly unchecked?: readonly string[];
   /** Why nothing was checked, when status is `skipped`. */
   readonly reason?: string;
   readonly checked: number;
@@ -181,38 +196,65 @@ async function verifyPackage(
 ): Promise<VerifiedPackage> {
   const empty = { id, checked: 0, matched: 0, unrecognised: 0, findings: [] };
 
-  const { documents } = await readBuildConfig(packageDir);
-  if (documents === undefined) {
+  const packageLibraries = await documentedLibraries(packageDir, manifest);
+  // A chunk may name its own libraries, which is how a monorepo says that one chunk describes
+  // `@acme/api@1.2.0` and its neighbour describes `@acme/cli@3.0.0`. Those are what that chunk
+  // is checked against; the package's list covers every chunk that names none.
+  const scopes = manifest.chunks.map((chunk) => chunk.documents ?? packageLibraries);
+  if (packageLibraries.length === 0 && scopes.every((scope) => scope.length === 0)) {
     return {
       ...empty,
       status: "skipped",
-      reason: `declares no "docspack.documents", so there is nothing to check against`,
+      reason: `declares no "documents", so there is nothing to check against`,
     };
   }
 
-  const libraryDir =
-    (await resolvePackageDir(documents, packageDir)) ?? (await resolvePackageDir(documents, cwd));
-  if (libraryDir === undefined) {
-    return { ...empty, status: "skipped", documents, reason: `${documents} is not installed` };
-  }
-
-  const surface = await readExportSurface(libraryDir);
-  if (surface === undefined || surface.names.size === 0) {
-    return {
-      ...empty,
-      status: "skipped",
-      documents,
-      reason: `${documents} ships no type declarations to check against`,
-    };
-  }
+  const documents = [
+    ...new Set([...packageLibraries, ...scopes.flat()].map(formatDocumentedLibrary)),
+  ];
+
+  // One resolution per library, however many chunks name it: reading a type surface is the
+  // expensive part, and a monorepo pack names the same eighteen libraries throughout.
+  const surfaces = new Map<string, ReadonlySet<string>>();
+  const resolve = async (library: DocumentedLibrary): Promise<ReadonlySet<string>> => {
+    const cached = surfaces.get(library.name);
+    if (cached !== undefined) return cached;
+
+    const libraryDir =
+      (await resolvePackageDir(library.name, packageDir)) ??
+      (await resolvePackageDir(library.name, cwd));
+    const surface = libraryDir === undefined ? undefined : await readExportSurface(libraryDir);
+    const names = surface?.names ?? new Set<string>();
+    surfaces.set(library.name, names);
+    return names;
+  };
 
-  const declared = new Set([...surface.names].map(normalize));
   const findings: DriftFinding[] = [];
+  const missing = new Set<string>();
   let checked = 0;
   let matched = 0;
   let unrecognised = 0;
+  let readAny = false;
+
+  for (const [index, chunk] of manifest.chunks.entries()) {
+    const scope = scopes[index] ?? packageLibraries;
+    if (scope.length === 0) continue;
+
+    // An entity is fine if any library in scope declares it: a monorepo documents eighteen
+    // packages from one surface, and a name belongs to whichever of them exports it.
+    const names = new Set<string>();
+    for (const library of scope) {
+      const surface = await resolve(library);
+      if (surface.size === 0) {
+        missing.add(library.name);
+        continue;
+      }
+      for (const name of surface) names.add(name);
+    }
+    if (names.size === 0) continue;
+    readAny = true;
 
-  for (const chunk of manifest.chunks) {
+    const declared = new Set([...names].map(normalize));
     for (const entity of chunk.entities) {
       const symbol = checkableSymbol(entity);
       if (symbol === undefined) continue;
@@ -225,7 +267,7 @@ async function verifyPackage(
         continue;
       }
 
-      const suggestion = nearestName(symbol, surface);
+      const suggestion = nearestName(symbol, names);
       if (suggestion === undefined) {
         unrecognised += 1;
         continue;
@@ -241,7 +283,45 @@ async function verifyPackage(
     }
   }
 
-  return { id, status: "verified", documents, checked, matched, unrecognised, findings };
+  if (!readAny) {
+    const unread = [...missing];
+    return {
+      ...empty,
+      status: "skipped",
+      documents,
+      reason:
+        unread.length === 0
+          ? "has no chunk naming a library to check against"
+          : `${unread.join(", ")} ${unread.length === 1 ? "is" : "are"} not installed, or ship no type declarations`,
+    };
+  }
+
+  return {
+    id,
+    status: "verified",
+    documents,
+    ...(missing.size === 0 ? {} : { unchecked: [...missing] }),
+    checked,
+    matched,
+    unrecognised,
+    findings,
+  };
+}
+
+/**
+ * What a package says it documents.
+ *
+ * The manifest is the answer when it carries one, because that is the only copy a consumer of an
+ * installed package can see. The `docspack` key is read as a fallback: it is build configuration
+ * rather than payload, and a package built before `documents` reached the manifest has only that.
+ */
+async function documentedLibraries(
+  packageDir: string,
+  manifest: PackageManifest,
+): Promise<readonly DocumentedLibrary[]> {
+  if (manifest.documents !== undefined && manifest.documents.length > 0) return manifest.documents;
+  const { documents } = await readBuildConfig(packageDir);
+  return (documents ?? []).map(parseDocumentedLibrary);
 }
 
 /**
@@ -275,13 +355,13 @@ function normalize(name: string): string {
  * is left alone. In particular a differing first word is never a match: `includeLanguages` and
  * `excludeLanguages` are two edits apart and mean opposite things.
  */
-function nearestName(symbol: string, surface: ExportSurface): string | undefined {
+function nearestName(symbol: string, names: ReadonlySet<string>): string | undefined {
   const lower = normalize(symbol);
   const words = camelWords(symbol);
   let best: string | undefined;
   let bestDistance = Number.POSITIVE_INFINITY;
 
-  for (const candidate of surface.names) {
+  for (const candidate of names) {
     if (candidate.length < MIN_SYMBOL) continue;
     const candidateWords = camelWords(candidate);
     if (isRequalified(words, candidateWords)) return candidate;