docspack 0.1.1 → 0.3.0

package/src/search.ts

@@ -1,6 +1,6 @@
 import type { SearchHit, Store } from "./db.js";
 import { discoverPackages } from "./discovery.js";
-import { isCommunityPackage } from "./spec.js";
+import { formatDocumentedLibrary, isCommunityPackage } from "./spec.js";
 
 /** Ceiling on how much context one query may return, per the blueprint's context-exhaustion rule. */
 export const DEFAULT_MAX_TOKENS = 3000;
@@ -25,6 +25,12 @@ export interface QueryHit extends SearchHit {
   readonly name: string;
   readonly version: string;
   readonly trusted: boolean;
+  /**
+   * Libraries the package documents, from its manifest. A docs package version cannot always
+   * imply this — a monorepo documents many libraries at many versions from one surface — so the
+   * answer states it rather than leaving the reader to infer it from the package name.
+   */
+  readonly documents?: readonly string[];
 }
 
 export interface QueryResult {
@@ -32,13 +38,31 @@ export interface QueryResult {
   readonly tokens: number;
   /** True when any hit came from an unvetted community package. */
   readonly untrusted: boolean;
+  /**
+   * Packages this project depends on that are installed but absent from the store. Nothing they
+   * document can match until `docspack sync` runs, which is a different answer from "nothing
+   * covers this question".
+   */
+  readonly unindexed: readonly string[];
 }
 
 export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
   const scoped = options.scoped !== false;
-  const packageIds = scoped
-    ? (await discoverPackages(options.cwd)).packages.map((pkg) => pkg.id)
-    : undefined;
+  const installed = scoped ? (await discoverPackages(options.cwd)).packages : undefined;
+  const packageIds = installed?.map((pkg) => pkg.id);
+  const unindexed = (installed ?? [])
+    .filter((pkg) => !options.store.hasPackage(pkg.id))
+    .map((pkg) => pkg.id);
+
+  // Read from the installed manifests rather than the index: the store is a cache of chunk text,
+  // and adding a column to it would make every existing store need a rebuild to answer this.
+  const documented = new Map<string, readonly string[]>();
+  for (const pkg of installed ?? []) {
+    const documents = pkg.manifest.documents;
+    if (documents !== undefined && documents.length > 0) {
+      documented.set(pkg.id, documents.map(formatDocumentedLibrary));
+    }
+  }
 
   const hits = options.store
     .search(options.query, {
@@ -51,13 +75,21 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
     })
     .map((hit): QueryHit => {
       const { name, version } = splitPackageId(hit.packageId);
-      return { ...hit, name, version, trusted: !isCommunityPackage(name) };
+      const documents = documented.get(hit.packageId);
+      return {
+        ...hit,
+        name,
+        version,
+        trusted: !isCommunityPackage(name),
+        ...(documents === undefined ? {} : { documents }),
+      };
     });
 
   return {
     hits,
     tokens: hits.reduce((total, hit) => total + hit.tokens, 0),
     untrusted: hits.some((hit) => !hit.trusted),
+    unindexed,
   };
 }
 
@@ -71,14 +103,26 @@ const UNTRUSTED_NOTICE =
  */
 export function renderAnswer(result: QueryResult, query: string): string {
   if (result.hits.length === 0) {
+    // Forgetting `sync` is the likeliest first mistake, and the tool can tell it apart from a
+    // question nothing covers: it can see what is installed and what is in the store.
+    if (result.unindexed.length > 0) {
+      return `No local documentation matched "${query}", and ${result.unindexed.join(", ")} ${result.unindexed.length === 1 ? "is" : "are"} installed but not indexed. Run \`docspack sync\`, then ask again.`;
+    }
     return `No local documentation matched "${query}". The project may not depend on a docspack package covering it.`;
   }
 
   const sections = result.hits.map((hit) => {
     const trust = hit.trusted ? "" : " (community)";
+    // Which release the chunk describes, on the line that already carries provenance. An agent
+    // otherwise has to assume the docs package version is the library version, which a
+    // repository publishing eighteen packages from one docs surface cannot make true.
+    const describes =
+      hit.documents === undefined || hit.documents.length === 0
+        ? ""
+        : ` · documents ${hit.documents.join(", ")}`;
     return [
       `## ${hit.chunkId}${trust}`,
-      `Source: ${hit.name}@${hit.version} — ${hit.filePath}`,
+      `Source: ${hit.name}@${hit.version} — ${hit.filePath}${describes}`,
       "",
       hit.content,
     ].join("\n");

package/src/spec.ts

@@ -6,20 +6,35 @@ export const LLMS_DIR = ".llms";
 export const MANIFEST_FILE = "manifest.json";
 export const CHUNKS_DIR = "chunks";
 export const SCHEMA_URL = "https://docspack.dev/schema/v1.json";
+/** Prose specification of the package format. The hint on a validation failure points here. */
+export const SPEC_URL = "https://docspack.dev/spec";
 
 /** One retrievable unit of documentation. */
 export interface ChunkSpec {
   readonly id: string;
   /** Path relative to the package's `.llms/` directory. */
   readonly file: string;
-  readonly tokens: number;
+  /** Absent means "estimate it". Never 0: an empty chunk is a different problem. */
+  readonly tokens?: number;
   readonly tags: readonly string[];
   readonly entities: readonly string[];
 }
 
+/** A library release the package documents, e.g. `acme` at `1.4.0`. */
+export interface DocumentedLibrary {
+  readonly name: string;
+  /** A version or a range, as the author wrote it. Absent when the docs are not release-scoped. */
+  readonly version?: string;
+}
+
 export interface PackageManifest {
   readonly name: string;
   readonly version: string;
+  /**
+   * What the package documents, which the package's own version cannot always say: a monorepo
+   * publishes many libraries at many versions from one documentation surface.
+   */
+  readonly documents?: readonly DocumentedLibrary[];
   readonly chunks: readonly ChunkSpec[];
 }
 
@@ -48,6 +63,21 @@ export function chunkId(pkgId: string, chunk: string): string {
   return `${pkgId}/${chunk}`;
 }
 
+/**
+ * Reads `acme`, `acme@1.4.0` or `@acme/sdk@^1.4.0` as a library and an optional version. The
+ * separator is the last `@`, since a scope puts one at the start of the name as well.
+ */
+export function parseDocumentedLibrary(spec: string): DocumentedLibrary {
+  const at = spec.lastIndexOf("@");
+  if (at <= 0) return { name: spec };
+  return { name: spec.slice(0, at), version: spec.slice(at + 1) };
+}
+
+/** `acme@1.4.0`, or just the name when no version is scoped. The form an answer is labelled with. */
+export function formatDocumentedLibrary(library: DocumentedLibrary): string {
+  return library.version === undefined ? library.name : `${library.name}@${library.version}`;
+}
+
 /**
  * Resolves a manifest `file` entry inside the package's `.llms/` directory, refusing anything
  * that escapes it. Manifests are third-party input, so this is a security boundary, not a
@@ -73,7 +103,7 @@ export function parseManifest(raw: unknown, where: string): PackageManifest {
   // The explicit annotation is what lets TypeScript treat a `fail(...)` call as unreachable-after.
   const fail: (message: string) => never = (message: string): never => {
     throw new DocspackError(`${where}: ${message}`, {
-      hint: `See the package specification: ${SCHEMA_URL}`,
+      hint: `See the package specification: ${SPEC_URL}`,
     });
   };
 
@@ -86,6 +116,7 @@ export function parseManifest(raw: unknown, where: string): PackageManifest {
   if (typeof version !== "string" || version.length === 0) fail('missing string field "version"');
   if (!Array.isArray(root.chunks)) fail('missing "chunks" array');
 
+  const documents = parseDocuments(root.documents, fail);
   const seen = new Set<string>();
   const chunks = (root.chunks as unknown[]).map((entry, index): ChunkSpec => {
     if (typeof entry !== "object" || entry === null)
@@ -104,21 +135,40 @@ export function parseManifest(raw: unknown, where: string): PackageManifest {
       return fail(`chunk "${id}" is missing its "file"`);
     }
 
+    // 0 is refused rather than read as "estimate it": the schema accepts it as a deliberate
+    // count, the reader would treat it as absent, and no chunk is genuinely 0 tokens.
     const tokens = chunk.tokens;
-    if (tokens !== undefined && (!Number.isInteger(tokens) || (tokens as number) < 0)) {
-      return fail(`chunk "${id}" has an invalid "tokens" value`);
+    if (tokens !== undefined && (!Number.isInteger(tokens) || (tokens as number) < 1)) {
+      return fail(`chunk "${id}" has an invalid "tokens" value; omit it to have it estimated`);
     }
 
     return {
       id,
       file,
-      tokens: typeof tokens === "number" ? tokens : 0,
+      ...(typeof tokens === "number" ? { tokens } : {}),
       tags: stringArray(chunk.tags, `chunk "${id}" field "tags"`, fail),
       entities: stringArray(chunk.entities, `chunk "${id}" field "entities"`, fail),
     };
   });
 
-  return { name, version, chunks };
+  return { name, version, ...(documents === undefined ? {} : { documents }), chunks };
+}
+
+/**
+ * `["acme@1.4.0", "@acme/sdk@^2"]`. An array even for one library, because that is what the
+ * schema declares — a reader that accepts a shape the validator rejects is the same defect as a
+ * validator that accepts a value the reader reinterprets.
+ */
+function parseDocuments(
+  value: unknown,
+  fail: (message: string) => never,
+): readonly DocumentedLibrary[] | undefined {
+  if (value === undefined) return undefined;
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string" || entry === "")) {
+    fail('"documents" must be an array of package names, optionally with a version');
+  }
+  const specs = value as string[];
+  return specs.length === 0 ? undefined : specs.map(parseDocumentedLibrary);
 }
 
 function stringArray(
@@ -133,6 +183,18 @@ function stringArray(
   return value as string[];
 }
 
+/** Writes the manifest, with `documents` back in the string form the schema declares. */
 export function serializeManifest(manifest: PackageManifest): string {
-  return `${JSON.stringify({ $schema: SCHEMA_URL, ...manifest }, null, 2)}\n`;
+  const documents = manifest.documents ?? [];
+  return `${JSON.stringify(
+    {
+      $schema: SCHEMA_URL,
+      name: manifest.name,
+      version: manifest.version,
+      ...(documents.length === 0 ? {} : { documents: documents.map(formatDocumentedLibrary) }),
+      chunks: manifest.chunks,
+    },
+    null,
+    2,
+  )}\n`;
 }

package/src/stopwords.ts

@@ -0,0 +1,120 @@
+/**
+ * Words that carry no discrimination in a documentation corpus.
+ *
+ * Heading words become tags, and tags are weighted 3× against prose in the ranking. A section
+ * titled "How to use it" therefore carried `how`, `to` and `it` as high-weight terms, and every
+ * question beginning "how do I…" matched it — ahead of the chunk that answers the question.
+ *
+ * The list is closed-class only: articles, pronouns, prepositions, conjunctions, auxiliaries,
+ * modals, demonstratives and question words. No content word is in it, however generic it
+ * looks. `use`, `add`, `get` and `set` all name real APIs, and a list that grows past the
+ * function words starts deciding which of an author's own words count.
+ */
+export const STOPWORDS: ReadonlySet<string> = new Set([
+  "about",
+  "after",
+  "all",
+  "also",
+  "an",
+  "and",
+  "any",
+  "are",
+  "as",
+  "at",
+  "be",
+  "been",
+  "before",
+  "being",
+  "between",
+  "both",
+  "but",
+  "by",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "doing",
+  "each",
+  "for",
+  "from",
+  "had",
+  "has",
+  "have",
+  "he",
+  "her",
+  "here",
+  "hers",
+  "him",
+  "his",
+  "how",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "its",
+  "may",
+  "me",
+  "might",
+  "must",
+  "my",
+  "no",
+  "nor",
+  "not",
+  "of",
+  "off",
+  "on",
+  "once",
+  "only",
+  "or",
+  "other",
+  "our",
+  "ours",
+  "out",
+  "over",
+  "own",
+  "same",
+  "shall",
+  "she",
+  "should",
+  "so",
+  "some",
+  "such",
+  "than",
+  "that",
+  "the",
+  "their",
+  "theirs",
+  "them",
+  "then",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "through",
+  "to",
+  "too",
+  "under",
+  "until",
+  "up",
+  "very",
+  "was",
+  "we",
+  "were",
+  "what",
+  "when",
+  "where",
+  "which",
+  "while",
+  "who",
+  "whom",
+  "why",
+  "will",
+  "with",
+  "would",
+  "you",
+  "your",
+  "yours",
+]);

package/src/style.ts

@@ -34,6 +34,11 @@ const SENTENCE_SPLIT = /(?<=[.!?])\s+/;
 /** A heading or list item starts a new unit, whatever the previous line ended with. */
 const BLOCK_START = /\n(?=\s*(?:[-*+]\s|\d+\.\s|#{1,6}\s|\|))/;
 const LONG_SENTENCE_WORDS = 40;
+/**
+ * A word carries a letter or a digit. Stripped code leaves its punctuation behind, so counting
+ * everything between spaces reports an enumeration of forty code spans as a forty-word sentence.
+ */
+const WORD = /[\p{L}\p{N}]/u;
 
 export interface FillerHit {
   readonly phrase: string;
@@ -68,7 +73,7 @@ export function findLongSentences(text: string, maxWords = LONG_SENTENCE_WORDS):
     .flatMap((block) => block.split(BLOCK_START))
     .flatMap((block) => block.split(SENTENCE_SPLIT))
     .map((sentence) => sentence.replace(/\s+/g, " ").trim())
-    .filter((sentence) => sentence.split(" ").filter(Boolean).length > maxWords);
+    .filter((sentence) => sentence.split(" ").filter((word) => WORD.test(word)).length > maxWords);
 }
 
 /**
@@ -94,11 +99,16 @@ export function contentFingerprint(text: string): string {
     .trim();
 }
 
+/** An authored directive, e.g. `<!-- docspack: tags=grid -->`. Metadata, not site boilerplate. */
+const DIRECTIVE_LINE = /^\s*<!--\s*docspack:/i;
+
 /** Removes lines a documentation site wraps around its content. */
 export function stripBoilerplate(text: string): string {
   return text
     .split("\n")
-    .filter((line) => !BOILERPLATE.some((pattern) => pattern.test(line)))
+    .filter(
+      (line) => DIRECTIVE_LINE.test(line) || !BOILERPLATE.some((pattern) => pattern.test(line)),
+    )
     .join("\n");
 }
 

package/src/sync.ts

@@ -102,7 +102,7 @@ async function readChunks(
     chunks.push({
       chunkId: chunkId(pkg.id, chunk.id),
       filePath: chunk.file,
-      tokens: chunk.tokens > 0 ? chunk.tokens : estimateTokens(text),
+      tokens: chunk.tokens ?? estimateTokens(text),
       content: text,
       tags: [...chunk.tags, ...chunk.entities],
     });

package/src/verify.ts

@@ -3,8 +3,17 @@ import { join } from "node:path";
 import { readBuildConfig } from "./config.js";
 import { discoverPackages, resolvePackageDir } from "./discovery.js";
 import { DocspackError } from "./errors.js";
-import { type ExportSurface, readExportSurface } from "./exports.js";
-import { chunkId, LLMS_DIR, MANIFEST_FILE, type PackageManifest, parseManifest } from "./spec.js";
+import { readExportSurface } from "./exports.js";
+import {
+  chunkId,
+  type DocumentedLibrary,
+  formatDocumentedLibrary,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  type PackageManifest,
+  parseDocumentedLibrary,
+  parseManifest,
+} from "./spec.js";
 
 export interface DriftFinding {
   readonly chunkId: string;
@@ -22,8 +31,14 @@ export type VerifyStatus = "verified" | "skipped";
 export interface VerifiedPackage {
   readonly id: string;
   readonly status: VerifyStatus;
-  /** Library the package declares it documents. */
-  readonly documents?: string;
+  /** Libraries the package declares it documents, as written. */
+  readonly documents?: readonly string[];
+  /**
+   * Documented libraries whose declarations could not be read, when others could. The check ran
+   * against a smaller surface than the package claims, which is worth saying: an entity those
+   * libraries export is one this run could have accused.
+   */
+  readonly unchecked?: readonly string[];
   /** Why nothing was checked, when status is `skipped`. */
   readonly reason?: string;
   readonly checked: number;
@@ -181,32 +196,42 @@ async function verifyPackage(
 ): Promise<VerifiedPackage> {
   const empty = { id, checked: 0, matched: 0, unrecognised: 0, findings: [] };
 
-  const { documents } = await readBuildConfig(packageDir);
-  if (documents === undefined) {
+  const declaredLibraries = await documentedLibraries(packageDir, manifest);
+  if (declaredLibraries.length === 0) {
     return {
       ...empty,
       status: "skipped",
-      reason: `declares no "docspack.documents", so there is nothing to check against`,
+      reason: `declares no "documents", so there is nothing to check against`,
     };
   }
 
-  const libraryDir =
-    (await resolvePackageDir(documents, packageDir)) ?? (await resolvePackageDir(documents, cwd));
-  if (libraryDir === undefined) {
-    return { ...empty, status: "skipped", documents, reason: `${documents} is not installed` };
+  const documents = declaredLibraries.map(formatDocumentedLibrary);
+  // An entity is fine if any documented library declares it: a monorepo documents eighteen
+  // packages from one surface, and a name belongs to whichever of them exports it.
+  const names = new Set<string>();
+  const missing: string[] = [];
+  for (const library of declaredLibraries) {
+    const libraryDir =
+      (await resolvePackageDir(library.name, packageDir)) ??
+      (await resolvePackageDir(library.name, cwd));
+    const surface = libraryDir === undefined ? undefined : await readExportSurface(libraryDir);
+    if (surface === undefined || surface.names.size === 0) {
+      missing.push(library.name);
+      continue;
+    }
+    for (const name of surface.names) names.add(name);
   }
 
-  const surface = await readExportSurface(libraryDir);
-  if (surface === undefined || surface.names.size === 0) {
+  if (names.size === 0) {
     return {
       ...empty,
       status: "skipped",
       documents,
-      reason: `${documents} ships no type declarations to check against`,
+      reason: `${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not installed, or ship no type declarations`,
     };
   }
 
-  const declared = new Set([...surface.names].map(normalize));
+  const declared = new Set([...names].map(normalize));
   const findings: DriftFinding[] = [];
   let checked = 0;
   let matched = 0;
@@ -225,7 +250,7 @@ async function verifyPackage(
         continue;
       }
 
-      const suggestion = nearestName(symbol, surface);
+      const suggestion = nearestName(symbol, names);
       if (suggestion === undefined) {
         unrecognised += 1;
         continue;
@@ -241,7 +266,32 @@ async function verifyPackage(
     }
   }
 
-  return { id, status: "verified", documents, checked, matched, unrecognised, findings };
+  return {
+    id,
+    status: "verified",
+    documents,
+    ...(missing.length === 0 ? {} : { unchecked: missing }),
+    checked,
+    matched,
+    unrecognised,
+    findings,
+  };
+}
+
+/**
+ * What a package says it documents.
+ *
+ * The manifest is the answer when it carries one, because that is the only copy a consumer of an
+ * installed package can see. The `docspack` key is read as a fallback: it is build configuration
+ * rather than payload, and a package built before `documents` reached the manifest has only that.
+ */
+async function documentedLibraries(
+  packageDir: string,
+  manifest: PackageManifest,
+): Promise<readonly DocumentedLibrary[]> {
+  if (manifest.documents !== undefined && manifest.documents.length > 0) return manifest.documents;
+  const { documents } = await readBuildConfig(packageDir);
+  return (documents ?? []).map(parseDocumentedLibrary);
 }
 
 /**
@@ -275,13 +325,13 @@ function normalize(name: string): string {
  * is left alone. In particular a differing first word is never a match: `includeLanguages` and
  * `excludeLanguages` are two edits apart and mean opposite things.
  */
-function nearestName(symbol: string, surface: ExportSurface): string | undefined {
+function nearestName(symbol: string, names: ReadonlySet<string>): string | undefined {
   const lower = normalize(symbol);
   const words = camelWords(symbol);
   let best: string | undefined;
   let bestDistance = Number.POSITIVE_INFINITY;
 
-  for (const candidate of surface.names) {
+  for (const candidate of names) {
     if (candidate.length < MIN_SYMBOL) continue;
     const candidateWords = camelWords(candidate);
     if (isRequalified(words, candidateWords)) return candidate;