docspack 0.2.0 → 0.3.0

package/src/stopwords.ts

@@ -0,0 +1,120 @@
+/**
+ * Words that carry no discrimination in a documentation corpus.
+ *
+ * Heading words become tags, and tags are weighted 3× against prose in the ranking. A section
+ * titled "How to use it" therefore carried `how`, `to` and `it` as high-weight terms, and every
+ * question beginning "how do I…" matched it — ahead of the chunk that answers the question.
+ *
+ * The list is closed-class only: articles, pronouns, prepositions, conjunctions, auxiliaries,
+ * modals, demonstratives and question words. No content word is in it, however generic it
+ * looks. `use`, `add`, `get` and `set` all name real APIs, and a list that grows past the
+ * function words starts deciding which of an author's own words count.
+ */
+export const STOPWORDS: ReadonlySet<string> = new Set([
+  "about",
+  "after",
+  "all",
+  "also",
+  "an",
+  "and",
+  "any",
+  "are",
+  "as",
+  "at",
+  "be",
+  "been",
+  "before",
+  "being",
+  "between",
+  "both",
+  "but",
+  "by",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "doing",
+  "each",
+  "for",
+  "from",
+  "had",
+  "has",
+  "have",
+  "he",
+  "her",
+  "here",
+  "hers",
+  "him",
+  "his",
+  "how",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "its",
+  "may",
+  "me",
+  "might",
+  "must",
+  "my",
+  "no",
+  "nor",
+  "not",
+  "of",
+  "off",
+  "on",
+  "once",
+  "only",
+  "or",
+  "other",
+  "our",
+  "ours",
+  "out",
+  "over",
+  "own",
+  "same",
+  "shall",
+  "she",
+  "should",
+  "so",
+  "some",
+  "such",
+  "than",
+  "that",
+  "the",
+  "their",
+  "theirs",
+  "them",
+  "then",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "through",
+  "to",
+  "too",
+  "under",
+  "until",
+  "up",
+  "very",
+  "was",
+  "we",
+  "were",
+  "what",
+  "when",
+  "where",
+  "which",
+  "while",
+  "who",
+  "whom",
+  "why",
+  "will",
+  "with",
+  "would",
+  "you",
+  "your",
+  "yours",
+]);

package/src/sync.ts

@@ -102,7 +102,7 @@ async function readChunks(
     chunks.push({
       chunkId: chunkId(pkg.id, chunk.id),
       filePath: chunk.file,
-      tokens: chunk.tokens > 0 ? chunk.tokens : estimateTokens(text),
+      tokens: chunk.tokens ?? estimateTokens(text),
       content: text,
       tags: [...chunk.tags, ...chunk.entities],
     });

package/src/verify.ts

@@ -3,8 +3,17 @@ import { join } from "node:path";
 import { readBuildConfig } from "./config.js";
 import { discoverPackages, resolvePackageDir } from "./discovery.js";
 import { DocspackError } from "./errors.js";
-import { type ExportSurface, readExportSurface } from "./exports.js";
-import { chunkId, LLMS_DIR, MANIFEST_FILE, type PackageManifest, parseManifest } from "./spec.js";
+import { readExportSurface } from "./exports.js";
+import {
+  chunkId,
+  type DocumentedLibrary,
+  formatDocumentedLibrary,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  type PackageManifest,
+  parseDocumentedLibrary,
+  parseManifest,
+} from "./spec.js";
 
 export interface DriftFinding {
   readonly chunkId: string;
@@ -22,8 +31,14 @@ export type VerifyStatus = "verified" | "skipped";
 export interface VerifiedPackage {
   readonly id: string;
   readonly status: VerifyStatus;
-  /** Library the package declares it documents. */
-  readonly documents?: string;
+  /** Libraries the package declares it documents, as written. */
+  readonly documents?: readonly string[];
+  /**
+   * Documented libraries whose declarations could not be read, when others could. The check ran
+   * against a smaller surface than the package claims, which is worth saying: an entity those
+   * libraries export is one this run could have accused.
+   */
+  readonly unchecked?: readonly string[];
   /** Why nothing was checked, when status is `skipped`. */
   readonly reason?: string;
   readonly checked: number;
@@ -181,32 +196,42 @@ async function verifyPackage(
 ): Promise<VerifiedPackage> {
   const empty = { id, checked: 0, matched: 0, unrecognised: 0, findings: [] };
 
-  const { documents } = await readBuildConfig(packageDir);
-  if (documents === undefined) {
+  const declaredLibraries = await documentedLibraries(packageDir, manifest);
+  if (declaredLibraries.length === 0) {
     return {
       ...empty,
       status: "skipped",
-      reason: `declares no "docspack.documents", so there is nothing to check against`,
+      reason: `declares no "documents", so there is nothing to check against`,
     };
   }
 
-  const libraryDir =
-    (await resolvePackageDir(documents, packageDir)) ?? (await resolvePackageDir(documents, cwd));
-  if (libraryDir === undefined) {
-    return { ...empty, status: "skipped", documents, reason: `${documents} is not installed` };
+  const documents = declaredLibraries.map(formatDocumentedLibrary);
+  // An entity is fine if any documented library declares it: a monorepo documents eighteen
+  // packages from one surface, and a name belongs to whichever of them exports it.
+  const names = new Set<string>();
+  const missing: string[] = [];
+  for (const library of declaredLibraries) {
+    const libraryDir =
+      (await resolvePackageDir(library.name, packageDir)) ??
+      (await resolvePackageDir(library.name, cwd));
+    const surface = libraryDir === undefined ? undefined : await readExportSurface(libraryDir);
+    if (surface === undefined || surface.names.size === 0) {
+      missing.push(library.name);
+      continue;
+    }
+    for (const name of surface.names) names.add(name);
   }
 
-  const surface = await readExportSurface(libraryDir);
-  if (surface === undefined || surface.names.size === 0) {
+  if (names.size === 0) {
     return {
       ...empty,
       status: "skipped",
       documents,
-      reason: `${documents} ships no type declarations to check against`,
+      reason: `${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not installed, or ship no type declarations`,
     };
   }
 
-  const declared = new Set([...surface.names].map(normalize));
+  const declared = new Set([...names].map(normalize));
   const findings: DriftFinding[] = [];
   let checked = 0;
   let matched = 0;
@@ -225,7 +250,7 @@ async function verifyPackage(
         continue;
       }
 
-      const suggestion = nearestName(symbol, surface);
+      const suggestion = nearestName(symbol, names);
       if (suggestion === undefined) {
         unrecognised += 1;
         continue;
@@ -241,7 +266,32 @@ async function verifyPackage(
     }
   }
 
-  return { id, status: "verified", documents, checked, matched, unrecognised, findings };
+  return {
+    id,
+    status: "verified",
+    documents,
+    ...(missing.length === 0 ? {} : { unchecked: missing }),
+    checked,
+    matched,
+    unrecognised,
+    findings,
+  };
+}
+
+/**
+ * What a package says it documents.
+ *
+ * The manifest is the answer when it carries one, because that is the only copy a consumer of an
+ * installed package can see. The `docspack` key is read as a fallback: it is build configuration
+ * rather than payload, and a package built before `documents` reached the manifest has only that.
+ */
+async function documentedLibraries(
+  packageDir: string,
+  manifest: PackageManifest,
+): Promise<readonly DocumentedLibrary[]> {
+  if (manifest.documents !== undefined && manifest.documents.length > 0) return manifest.documents;
+  const { documents } = await readBuildConfig(packageDir);
+  return (documents ?? []).map(parseDocumentedLibrary);
 }
 
 /**
@@ -275,13 +325,13 @@ function normalize(name: string): string {
  * is left alone. In particular a differing first word is never a match: `includeLanguages` and
  * `excludeLanguages` are two edits apart and mean opposite things.
  */
-function nearestName(symbol: string, surface: ExportSurface): string | undefined {
+function nearestName(symbol: string, names: ReadonlySet<string>): string | undefined {
   const lower = normalize(symbol);
   const words = camelWords(symbol);
   let best: string | undefined;
   let bestDistance = Number.POSITIVE_INFINITY;
 
-  for (const candidate of surface.names) {
+  for (const candidate of names) {
     if (candidate.length < MIN_SYMBOL) continue;
     const candidateWords = camelWords(candidate);
     if (isRequalified(words, candidateWords)) return candidate;