docspack 0.3.0 → 1.0.0

package/src/surface.ts

@@ -0,0 +1,265 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { dirname, join, relative, resolve, sep } from "node:path";
+
+/**
+ * One name a library publishes, and how a caller reaches it.
+ *
+ * This is a narrower reading of a package than [`readExportSurface`](./exports.ts), which
+ * collects every identifier in every `.d.ts` so that `verify` accuses as little as possible.
+ * Here the opposite is wanted: only what an entry point actually exports, because these names
+ * become answers, and an answer about a library's private type helps nobody.
+ */
+export interface ExportedSymbol {
+  readonly name: string;
+  /** What a caller writes to import it, e.g. `hono/jwt`. */
+  readonly from: string;
+  /** Declaration file the name was read from, relative to the package root. */
+  readonly file: string;
+}
+
+export interface PublicSurface {
+  readonly name: string;
+  readonly version: string;
+  /** Subpaths of the `exports` map that resolve to a declaration file. */
+  readonly entryPoints: number;
+  readonly symbols: ReadonlyMap<string, ExportedSymbol>;
+}
+
+/** How far `export … from` chains are followed out of an entry point. */
+const MAX_DEPTH = 6;
+
+/** A package this large is generated, and reading all of it would dominate a sync. */
+const MAX_FILES = 600;
+
+const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
+
+const DECLARATION =
+  /export\s+(?:declare\s+)?(?:abstract\s+)?(?:class|function|const|let|var|interface|type|enum)\s+([A-Za-z_$][\w$]*)/g;
+/**
+ * The same declaration, with the JSDoc comment above it captured as well.
+ *
+ * The comment body is written as "anything that is not the comment's end" rather than as a lazy
+ * run of any character. A lazy run expands across dozens of declarations to reach a distant
+ * comment terminator, and in a single global pass that swallows everything in between: on hono's
+ * 75kB type file it found 28 declarations where there are hundreds.
+ */
+const DECLARED =
+  /(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+(?:declare\s+)?(?:abstract\s+)?(?:class|function|const|let|var|interface|type|enum)\s+([A-Za-z_$][\w$]*)/g;
+
+/** How much of a declaration is carried: long enough for a signature, short enough to rank. */
+const DECLARATION_LIMIT = 400;
+const EXPORT_LIST = /export\s*\{([^}]*)\}/g;
+const REEXPORT =
+  /export\s+(?:\*|\{[^}]*\})\s*(?:as\s+[A-Za-z_$][\w$]*\s*)?from\s+['"]([^'"]+)['"]/g;
+
+/**
+ * Reads what a library exports, starting from its `exports` map. Returns nothing when the
+ * package ships no type declarations, which is not an error: most of a project's dependencies
+ * are not the ones anybody asks questions about.
+ */
+export async function readPublicSurface(dir: string): Promise<PublicSurface | undefined> {
+  const manifest = await readJson(join(dir, "package.json"));
+  if (manifest === undefined) return undefined;
+
+  const symbols = new Map<string, ExportedSymbol>();
+  const name = typeof manifest.name === "string" ? manifest.name : "";
+  const budget = { files: MAX_FILES };
+  let entryPoints = 0;
+
+  for (const entry of entryPointsOf(manifest, dir, name)) {
+    entryPoints += 1;
+    await collect(entry.file, entry.specifier, dir, 0, new Set<string>(), symbols, budget);
+  }
+
+  if (entryPoints === 0) return undefined;
+
+  return {
+    name,
+    version: typeof manifest.version === "string" ? manifest.version : "",
+    entryPoints,
+    symbols,
+  };
+}
+
+/**
+ * Every declaration in one file, with the JSDoc comment above it when there is one, keyed by the
+ * name it declares.
+ *
+ * One pass over the file rather than one per name. A generated declaration file can be megabytes
+ * and export hundreds of names, and scanning it once per name made indexing TypeScript itself
+ * take longer than everything else in a sync put together.
+ */
+export function declarationsIn(source: string, limit = DECLARATION_LIMIT): Map<string, string> {
+  const found = new Map<string, string>();
+  DECLARED.lastIndex = 0;
+  for (const match of source.matchAll(DECLARED)) {
+    const name = match[2];
+    if (name === undefined || found.has(name) || match.index === undefined) continue;
+    const head = match[0];
+    const start = match.index + head.length;
+    const body = source.slice(start, start + limit);
+    found.set(name, head + trimToDeclaration(body, body.length === limit));
+  }
+  return found;
+}
+
+/**
+ * Cuts a declaration's body where the declaration ends.
+ *
+ * A fixed window runs into whatever follows it, so the body is cut at the next thing that starts
+ * at the left margin — another declaration, or the comment above one. What is left is a
+ * signature rather than a signature plus the first half of its neighbour.
+ */
+function trimToDeclaration(body: string, truncated: boolean): string {
+  // A declaration's continuation lines are indented, so an unindented keyword is a new statement.
+  const next = body.search(
+    /\n(?:export|declare|type|interface|function|class|const|let|var|enum)\s|\n\/\*\*/,
+  );
+  if (next !== -1) return body.slice(0, next);
+  if (!truncated) return body;
+  // The window ran out mid-declaration. Ending on a whole line beats ending mid-identifier.
+  const lastLine = body.lastIndexOf("\n");
+  return lastLine === -1 ? body : `${body.slice(0, lastLine)}\n…`;
+}
+
+/**
+ * The declaration of one name, with the JSDoc comment above it when there is one. This is the
+ * text that answers a question the prose never covered.
+ */
+export function declarationOf(
+  source: string,
+  name: string,
+  limit = DECLARATION_LIMIT,
+): string | undefined {
+  return declarationsIn(source, limit).get(name);
+}
+
+/** True when the declaration `declarationOf` returned carries a JSDoc comment. */
+export function hasJsdoc(declaration: string): boolean {
+  return declaration.startsWith("/**");
+}
+
+interface EntryPoint {
+  readonly file: string;
+  /** The specifier a caller imports, e.g. `hono` or `hono/jwt`. */
+  readonly specifier: string;
+}
+
+function* entryPointsOf(
+  manifest: Record<string, unknown>,
+  dir: string,
+  name: string,
+): Generator<EntryPoint> {
+  const exports = manifest.exports;
+  const entries: [string, unknown][] =
+    typeof exports === "object" && exports !== null
+      ? Object.entries(exports as Record<string, unknown>)
+      : [[".", { types: manifest.types ?? manifest.typings }]];
+
+  for (const [subpath, entry] of entries) {
+    const types = declarationTarget(entry);
+    if (types === undefined) continue;
+    // A wildcard subpath names no single module a caller could import.
+    if (subpath.includes("*")) continue;
+    const file = resolve(dir, types);
+    if (!existsSync(file)) continue;
+    yield { file, specifier: specifierFor(name, subpath) };
+  }
+}
+
+function specifierFor(name: string, subpath: string): string {
+  if (subpath === "." || subpath === "") return name;
+  return `${name}/${subpath.replace(/^\.\//, "")}`;
+}
+
+function declarationTarget(entry: unknown): string | undefined {
+  if (typeof entry !== "object" || entry === null) return undefined;
+  const record = entry as Record<string, unknown>;
+  if (typeof record.types === "string") return record.types;
+  for (const key of ["import", "require", "default"] as const) {
+    const nested = record[key];
+    if (typeof nested === "object" && nested !== null) {
+      const types = (nested as Record<string, unknown>).types;
+      if (typeof types === "string") return types;
+    }
+  }
+  return undefined;
+}
+
+async function collect(
+  file: string,
+  specifier: string,
+  root: string,
+  depth: number,
+  seen: Set<string>,
+  into: Map<string, ExportedSymbol>,
+  budget: { files: number },
+): Promise<void> {
+  if (depth > MAX_DEPTH || budget.files <= 0 || seen.has(file) || !existsSync(file)) return;
+  seen.add(file);
+  budget.files -= 1;
+
+  let source: string;
+  try {
+    source = await readFile(file, "utf8");
+  } catch {
+    return;
+  }
+
+  // The first entry point that publishes a name wins, so a name re-exported from several
+  // subpaths is attributed to the one listed first — which is the package's own ordering.
+  const record = (name: string): void => {
+    if (!IDENTIFIER.test(name) || into.has(name)) return;
+    into.set(name, { name, from: specifier, file: relativePath(root, file) });
+  };
+
+  for (const match of source.matchAll(DECLARATION)) {
+    if (match[1] !== undefined) record(match[1]);
+  }
+
+  for (const match of source.matchAll(EXPORT_LIST)) {
+    for (const entry of (match[1] ?? "").split(",")) {
+      // `export { internal as public }` publishes the second name.
+      const parts = entry.split(/\s+as\s+/);
+      record((parts[parts.length - 1] ?? "").trim().replace(/^type\s+/, ""));
+    }
+  }
+
+  for (const match of source.matchAll(REEXPORT)) {
+    const target = match[1];
+    // A bare specifier points into another package, whose names are not this one's to publish.
+    if (target === undefined || !target.startsWith(".")) continue;
+    const next = resolveDeclaration(resolve(dirname(file), target));
+    if (next !== undefined) await collect(next, specifier, root, depth + 1, seen, into, budget);
+  }
+}
+
+/** `./external.cjs` inside a declaration file means `./external.d.cts`, not a JavaScript file. */
+function resolveDeclaration(specifier: string): string | undefined {
+  const base = specifier.replace(/\.(js|cjs|mjs)$/, "");
+  for (const candidate of [
+    `${base}.d.ts`,
+    `${base}.d.cts`,
+    `${base}.d.mts`,
+    join(base, "index.d.ts"),
+    join(base, "index.d.cts"),
+  ]) {
+    if (existsSync(candidate)) return candidate;
+  }
+  return undefined;
+}
+
+function relativePath(root: string, file: string): string {
+  return relative(root, file).split(sep).join("/");
+}
+
+async function readJson(file: string): Promise<Record<string, unknown> | undefined> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(file, "utf8"));
+    if (typeof parsed !== "object" || parsed === null) return undefined;
+    return parsed as Record<string, unknown>;
+  } catch {
+    return undefined;
+  }
+}

package/src/sync.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
-import type { IndexedChunk, Store } from "./db.js";
-import { type DiscoveredPackage, discoverPackages } from "./discovery.js";
+import { readArtifact } from "./artifact.js";
+import type { IndexedChunk, PackageKind, Store } from "./db.js";
+import { type DiscoveredPackage, discoverLibraries, discoverPackages } from "./discovery.js";
 import { chunkId, estimateTokens, resolveChunkFile } from "./spec.js";
 
 export interface SyncOptions {
@@ -8,6 +9,12 @@ export interface SyncOptions {
   readonly store: Store;
   /** Re-index packages that are already in the store. */
   readonly force?: boolean;
+  /**
+   * Also derive an index from each installed library's own type declarations. On by default:
+   * half of a library's exported names appear in no documentation anyone published, and the
+   * declarations are the only local answer for those.
+   */
+  readonly artifacts?: boolean;
   readonly onProgress?: (message: string) => void;
 }
 
@@ -21,6 +28,7 @@ export interface SyncedPackage {
   readonly tokens: number;
   readonly status: SyncStatus;
   readonly trusted: boolean;
+  readonly kind: PackageKind;
 }
 
 export interface SyncResult {
@@ -48,6 +56,7 @@ export async function syncProject(options: SyncOptions): Promise<SyncResult> {
         tokens: 0,
         status: "cached",
         trusted: pkg.trusted,
+        kind: "docs",
       });
       continue;
     }
@@ -70,12 +79,67 @@ export async function syncProject(options: SyncOptions): Promise<SyncResult> {
       tokens: chunks.reduce((total, chunk) => total + chunk.tokens, 0),
       status: "indexed",
       trusted: pkg.trusted,
+      kind: "docs",
     });
   }
 
+  if (options.artifacts !== false) synced.push(...(await syncArtifacts(options)));
+
   return { packages: synced, problems };
 }
 
+/**
+ * Indexes each installed library's exported declarations.
+ *
+ * These are not ranked alongside prose. They are addressed by name, so a question that names a
+ * symbol can be answered from the installed build even when nobody documented it — and a corpus
+ * of generated type machinery cannot crowd out the documentation that does exist.
+ */
+async function syncArtifacts(options: SyncOptions): Promise<SyncedPackage[]> {
+  const synced: SyncedPackage[] = [];
+
+  for (const library of await discoverLibraries(options.cwd)) {
+    const id = `${library.name}@${library.version}`;
+    if (options.force !== true && options.store.hasPackage(id)) {
+      synced.push({
+        id,
+        name: library.name,
+        version: library.version,
+        chunks: options.store.countChunks(id),
+        tokens: 0,
+        status: "cached",
+        trusted: true,
+        kind: "artifact",
+      });
+      continue;
+    }
+
+    options.onProgress?.(`reading ${id}`);
+    const artifact = await readArtifact(library.dir);
+    // A dependency that ships no type declarations has nothing to derive, which is ordinary
+    // rather than a problem worth reporting on every sync.
+    if (artifact === undefined) continue;
+
+    options.store.indexPackage(
+      { id: artifact.id, name: artifact.name, version: artifact.version, kind: "artifact" },
+      artifact.chunks,
+      artifact.symbols,
+    );
+    synced.push({
+      id: artifact.id,
+      name: artifact.name,
+      version: artifact.version,
+      chunks: artifact.chunks.length,
+      tokens: artifact.chunks.reduce((total, chunk) => total + chunk.tokens, 0),
+      status: "indexed",
+      trusted: true,
+      kind: "artifact",
+    });
+  }
+
+  return synced;
+}
+
 async function readChunks(
   pkg: DiscoveredPackage,
 ): Promise<{ chunks: IndexedChunk[]; problems: string[] }> {

package/src/verify.ts

@@ -196,8 +196,12 @@ async function verifyPackage(
 ): Promise<VerifiedPackage> {
   const empty = { id, checked: 0, matched: 0, unrecognised: 0, findings: [] };
 
-  const declaredLibraries = await documentedLibraries(packageDir, manifest);
-  if (declaredLibraries.length === 0) {
+  const packageLibraries = await documentedLibraries(packageDir, manifest);
+  // A chunk may name its own libraries, which is how a monorepo says that one chunk describes
+  // `@acme/api@1.2.0` and its neighbour describes `@acme/cli@3.0.0`. Those are what that chunk
+  // is checked against; the package's list covers every chunk that names none.
+  const scopes = manifest.chunks.map((chunk) => chunk.documents ?? packageLibraries);
+  if (packageLibraries.length === 0 && scopes.every((scope) => scope.length === 0)) {
     return {
       ...empty,
       status: "skipped",
@@ -205,39 +209,52 @@ async function verifyPackage(
     };
   }
 
-  const documents = declaredLibraries.map(formatDocumentedLibrary);
-  // An entity is fine if any documented library declares it: a monorepo documents eighteen
-  // packages from one surface, and a name belongs to whichever of them exports it.
-  const names = new Set<string>();
-  const missing: string[] = [];
-  for (const library of declaredLibraries) {
+  const documents = [
+    ...new Set([...packageLibraries, ...scopes.flat()].map(formatDocumentedLibrary)),
+  ];
+
+  // One resolution per library, however many chunks name it: reading a type surface is the
+  // expensive part, and a monorepo pack names the same eighteen libraries throughout.
+  const surfaces = new Map<string, ReadonlySet<string>>();
+  const resolve = async (library: DocumentedLibrary): Promise<ReadonlySet<string>> => {
+    const cached = surfaces.get(library.name);
+    if (cached !== undefined) return cached;
+
     const libraryDir =
       (await resolvePackageDir(library.name, packageDir)) ??
       (await resolvePackageDir(library.name, cwd));
     const surface = libraryDir === undefined ? undefined : await readExportSurface(libraryDir);
-    if (surface === undefined || surface.names.size === 0) {
-      missing.push(library.name);
-      continue;
-    }
-    for (const name of surface.names) names.add(name);
-  }
-
-  if (names.size === 0) {
-    return {
-      ...empty,
-      status: "skipped",
-      documents,
-      reason: `${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not installed, or ship no type declarations`,
-    };
-  }
+    const names = surface?.names ?? new Set<string>();
+    surfaces.set(library.name, names);
+    return names;
+  };
 
-  const declared = new Set([...names].map(normalize));
   const findings: DriftFinding[] = [];
+  const missing = new Set<string>();
   let checked = 0;
   let matched = 0;
   let unrecognised = 0;
+  let readAny = false;
+
+  for (const [index, chunk] of manifest.chunks.entries()) {
+    const scope = scopes[index] ?? packageLibraries;
+    if (scope.length === 0) continue;
+
+    // An entity is fine if any library in scope declares it: a monorepo documents eighteen
+    // packages from one surface, and a name belongs to whichever of them exports it.
+    const names = new Set<string>();
+    for (const library of scope) {
+      const surface = await resolve(library);
+      if (surface.size === 0) {
+        missing.add(library.name);
+        continue;
+      }
+      for (const name of surface) names.add(name);
+    }
+    if (names.size === 0) continue;
+    readAny = true;
 
-  for (const chunk of manifest.chunks) {
+    const declared = new Set([...names].map(normalize));
     for (const entity of chunk.entities) {
       const symbol = checkableSymbol(entity);
       if (symbol === undefined) continue;
@@ -266,11 +283,24 @@ async function verifyPackage(
     }
   }
 
+  if (!readAny) {
+    const unread = [...missing];
+    return {
+      ...empty,
+      status: "skipped",
+      documents,
+      reason:
+        unread.length === 0
+          ? "has no chunk naming a library to check against"
+          : `${unread.join(", ")} ${unread.length === 1 ? "is" : "are"} not installed, or ship no type declarations`,
+    };
+  }
+
   return {
     id,
     status: "verified",
     documents,
-    ...(missing.length === 0 ? {} : { unchecked: missing }),
+    ...(missing.size === 0 ? {} : { unchecked: [...missing] }),
     checked,
     matched,
     unrecognised,
@@ -285,7 +315,7 @@ async function verifyPackage(
  * installed package can see. The `docspack` key is read as a fallback: it is build configuration
  * rather than payload, and a package built before `documents` reached the manifest has only that.
  */
-async function documentedLibraries(
+export async function documentedLibraries(
   packageDir: string,
   manifest: PackageManifest,
 ): Promise<readonly DocumentedLibrary[]> {