docspack 0.4.0 → 1.0.0

package/src/surface.ts

@@ -0,0 +1,265 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { dirname, join, relative, resolve, sep } from "node:path";
+
+/**
+ * One name a library publishes, and how a caller reaches it.
+ *
+ * This is a narrower reading of a package than [`readExportSurface`](./exports.ts), which
+ * collects every identifier in every `.d.ts` so that `verify` accuses as little as possible.
+ * Here the opposite is wanted: only what an entry point actually exports, because these names
+ * become answers, and an answer about a library's private type helps nobody.
+ */
+export interface ExportedSymbol {
+  readonly name: string;
+  /** What a caller writes to import it, e.g. `hono/jwt`. */
+  readonly from: string;
+  /** Declaration file the name was read from, relative to the package root. */
+  readonly file: string;
+}
+
+export interface PublicSurface {
+  readonly name: string;
+  readonly version: string;
+  /** Subpaths of the `exports` map that resolve to a declaration file. */
+  readonly entryPoints: number;
+  readonly symbols: ReadonlyMap<string, ExportedSymbol>;
+}
+
+/** How far `export … from` chains are followed out of an entry point. */
+const MAX_DEPTH = 6;
+
+/** A package this large is generated, and reading all of it would dominate a sync. */
+const MAX_FILES = 600;
+
+const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
+
+const DECLARATION =
+  /export\s+(?:declare\s+)?(?:abstract\s+)?(?:class|function|const|let|var|interface|type|enum)\s+([A-Za-z_$][\w$]*)/g;
+/**
+ * The same declaration, with the JSDoc comment above it captured as well.
+ *
+ * The comment body is written as "anything that is not the comment's end" rather than as a lazy
+ * run of any character. A lazy run expands across dozens of declarations to reach a distant
+ * comment terminator, and in a single global pass that swallows everything in between: on hono's
+ * 75kB type file it found 28 declarations where there are hundreds.
+ */
+const DECLARED =
+  /(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+(?:declare\s+)?(?:abstract\s+)?(?:class|function|const|let|var|interface|type|enum)\s+([A-Za-z_$][\w$]*)/g;
+
+/** How much of a declaration is carried: long enough for a signature, short enough to rank. */
+const DECLARATION_LIMIT = 400;
+const EXPORT_LIST = /export\s*\{([^}]*)\}/g;
+const REEXPORT =
+  /export\s+(?:\*|\{[^}]*\})\s*(?:as\s+[A-Za-z_$][\w$]*\s*)?from\s+['"]([^'"]+)['"]/g;
+
+/**
+ * Reads what a library exports, starting from its `exports` map. Returns nothing when the
+ * package ships no type declarations, which is not an error: most of a project's dependencies
+ * are not the ones anybody asks questions about.
+ */
+export async function readPublicSurface(dir: string): Promise<PublicSurface | undefined> {
+  const manifest = await readJson(join(dir, "package.json"));
+  if (manifest === undefined) return undefined;
+
+  const symbols = new Map<string, ExportedSymbol>();
+  const name = typeof manifest.name === "string" ? manifest.name : "";
+  const budget = { files: MAX_FILES };
+  let entryPoints = 0;
+
+  for (const entry of entryPointsOf(manifest, dir, name)) {
+    entryPoints += 1;
+    await collect(entry.file, entry.specifier, dir, 0, new Set<string>(), symbols, budget);
+  }
+
+  if (entryPoints === 0) return undefined;
+
+  return {
+    name,
+    version: typeof manifest.version === "string" ? manifest.version : "",
+    entryPoints,
+    symbols,
+  };
+}
+
+/**
+ * Every declaration in one file, with the JSDoc comment above it when there is one, keyed by the
+ * name it declares.
+ *
+ * One pass over the file rather than one per name. A generated declaration file can be megabytes
+ * and export hundreds of names, and scanning it once per name made indexing TypeScript itself
+ * take longer than everything else in a sync put together.
+ */
+export function declarationsIn(source: string, limit = DECLARATION_LIMIT): Map<string, string> {
+  const found = new Map<string, string>();
+  DECLARED.lastIndex = 0;
+  for (const match of source.matchAll(DECLARED)) {
+    const name = match[2];
+    if (name === undefined || found.has(name) || match.index === undefined) continue;
+    const head = match[0];
+    const start = match.index + head.length;
+    const body = source.slice(start, start + limit);
+    found.set(name, head + trimToDeclaration(body, body.length === limit));
+  }
+  return found;
+}
+
+/**
+ * Cuts a declaration's body where the declaration ends.
+ *
+ * A fixed window runs into whatever follows it, so the body is cut at the next thing that starts
+ * at the left margin — another declaration, or the comment above one. What is left is a
+ * signature rather than a signature plus the first half of its neighbour.
+ */
+function trimToDeclaration(body: string, truncated: boolean): string {
+  // A declaration's continuation lines are indented, so an unindented keyword is a new statement.
+  const next = body.search(
+    /\n(?:export|declare|type|interface|function|class|const|let|var|enum)\s|\n\/\*\*/,
+  );
+  if (next !== -1) return body.slice(0, next);
+  if (!truncated) return body;
+  // The window ran out mid-declaration. Ending on a whole line beats ending mid-identifier.
+  const lastLine = body.lastIndexOf("\n");
+  return lastLine === -1 ? body : `${body.slice(0, lastLine)}\n…`;
+}
+
+/**
+ * The declaration of one name, with the JSDoc comment above it when there is one. This is the
+ * text that answers a question the prose never covered.
+ */
+export function declarationOf(
+  source: string,
+  name: string,
+  limit = DECLARATION_LIMIT,
+): string | undefined {
+  return declarationsIn(source, limit).get(name);
+}
+
+/** True when the declaration `declarationOf` returned carries a JSDoc comment. */
+export function hasJsdoc(declaration: string): boolean {
+  return declaration.startsWith("/**");
+}
+
+interface EntryPoint {
+  readonly file: string;
+  /** The specifier a caller imports, e.g. `hono` or `hono/jwt`. */
+  readonly specifier: string;
+}
+
+function* entryPointsOf(
+  manifest: Record<string, unknown>,
+  dir: string,
+  name: string,
+): Generator<EntryPoint> {
+  const exports = manifest.exports;
+  const entries: [string, unknown][] =
+    typeof exports === "object" && exports !== null
+      ? Object.entries(exports as Record<string, unknown>)
+      : [[".", { types: manifest.types ?? manifest.typings }]];
+
+  for (const [subpath, entry] of entries) {
+    const types = declarationTarget(entry);
+    if (types === undefined) continue;
+    // A wildcard subpath names no single module a caller could import.
+    if (subpath.includes("*")) continue;
+    const file = resolve(dir, types);
+    if (!existsSync(file)) continue;
+    yield { file, specifier: specifierFor(name, subpath) };
+  }
+}
+
+function specifierFor(name: string, subpath: string): string {
+  if (subpath === "." || subpath === "") return name;
+  return `${name}/${subpath.replace(/^\.\//, "")}`;
+}
+
+function declarationTarget(entry: unknown): string | undefined {
+  if (typeof entry !== "object" || entry === null) return undefined;
+  const record = entry as Record<string, unknown>;
+  if (typeof record.types === "string") return record.types;
+  for (const key of ["import", "require", "default"] as const) {
+    const nested = record[key];
+    if (typeof nested === "object" && nested !== null) {
+      const types = (nested as Record<string, unknown>).types;
+      if (typeof types === "string") return types;
+    }
+  }
+  return undefined;
+}
+
+async function collect(
+  file: string,
+  specifier: string,
+  root: string,
+  depth: number,
+  seen: Set<string>,
+  into: Map<string, ExportedSymbol>,
+  budget: { files: number },
+): Promise<void> {
+  if (depth > MAX_DEPTH || budget.files <= 0 || seen.has(file) || !existsSync(file)) return;
+  seen.add(file);
+  budget.files -= 1;
+
+  let source: string;
+  try {
+    source = await readFile(file, "utf8");
+  } catch {
+    return;
+  }
+
+  // The first entry point that publishes a name wins, so a name re-exported from several
+  // subpaths is attributed to the one listed first — which is the package's own ordering.
+  const record = (name: string): void => {
+    if (!IDENTIFIER.test(name) || into.has(name)) return;
+    into.set(name, { name, from: specifier, file: relativePath(root, file) });
+  };
+
+  for (const match of source.matchAll(DECLARATION)) {
+    if (match[1] !== undefined) record(match[1]);
+  }
+
+  for (const match of source.matchAll(EXPORT_LIST)) {
+    for (const entry of (match[1] ?? "").split(",")) {
+      // `export { internal as public }` publishes the second name.
+      const parts = entry.split(/\s+as\s+/);
+      record((parts[parts.length - 1] ?? "").trim().replace(/^type\s+/, ""));
+    }
+  }
+
+  for (const match of source.matchAll(REEXPORT)) {
+    const target = match[1];
+    // A bare specifier points into another package, whose names are not this one's to publish.
+    if (target === undefined || !target.startsWith(".")) continue;
+    const next = resolveDeclaration(resolve(dirname(file), target));
+    if (next !== undefined) await collect(next, specifier, root, depth + 1, seen, into, budget);
+  }
+}
+
+/** `./external.cjs` inside a declaration file means `./external.d.cts`, not a JavaScript file. */
+function resolveDeclaration(specifier: string): string | undefined {
+  const base = specifier.replace(/\.(js|cjs|mjs)$/, "");
+  for (const candidate of [
+    `${base}.d.ts`,
+    `${base}.d.cts`,
+    `${base}.d.mts`,
+    join(base, "index.d.ts"),
+    join(base, "index.d.cts"),
+  ]) {
+    if (existsSync(candidate)) return candidate;
+  }
+  return undefined;
+}
+
+function relativePath(root: string, file: string): string {
+  return relative(root, file).split(sep).join("/");
+}
+
+async function readJson(file: string): Promise<Record<string, unknown> | undefined> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(file, "utf8"));
+    if (typeof parsed !== "object" || parsed === null) return undefined;
+    return parsed as Record<string, unknown>;
+  } catch {
+    return undefined;
+  }
+}

package/src/sync.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
-import type { IndexedChunk, Store } from "./db.js";
-import { type DiscoveredPackage, discoverPackages } from "./discovery.js";
+import { readArtifact } from "./artifact.js";
+import type { IndexedChunk, PackageKind, Store } from "./db.js";
+import { type DiscoveredPackage, discoverLibraries, discoverPackages } from "./discovery.js";
 import { chunkId, estimateTokens, resolveChunkFile } from "./spec.js";
 
 export interface SyncOptions {
@@ -8,6 +9,12 @@ export interface SyncOptions {
   readonly store: Store;
   /** Re-index packages that are already in the store. */
   readonly force?: boolean;
+  /**
+   * Also derive an index from each installed library's own type declarations. On by default:
+   * half of a library's exported names appear in no documentation anyone published, and the
+   * declarations are the only local answer for those.
+   */
+  readonly artifacts?: boolean;
   readonly onProgress?: (message: string) => void;
 }
 
@@ -21,6 +28,7 @@ export interface SyncedPackage {
   readonly tokens: number;
   readonly status: SyncStatus;
   readonly trusted: boolean;
+  readonly kind: PackageKind;
 }
 
 export interface SyncResult {
@@ -48,6 +56,7 @@ export async function syncProject(options: SyncOptions): Promise<SyncResult> {
         tokens: 0,
         status: "cached",
         trusted: pkg.trusted,
+        kind: "docs",
       });
       continue;
     }
@@ -70,12 +79,67 @@ export async function syncProject(options: SyncOptions): Promise<SyncResult> {
       tokens: chunks.reduce((total, chunk) => total + chunk.tokens, 0),
       status: "indexed",
       trusted: pkg.trusted,
+      kind: "docs",
     });
   }
 
+  if (options.artifacts !== false) synced.push(...(await syncArtifacts(options)));
+
   return { packages: synced, problems };
 }
 
+/**
+ * Indexes each installed library's exported declarations.
+ *
+ * These are not ranked alongside prose. They are addressed by name, so a question that names a
+ * symbol can be answered from the installed build even when nobody documented it — and a corpus
+ * of generated type machinery cannot crowd out the documentation that does exist.
+ */
+async function syncArtifacts(options: SyncOptions): Promise<SyncedPackage[]> {
+  const synced: SyncedPackage[] = [];
+
+  for (const library of await discoverLibraries(options.cwd)) {
+    const id = `${library.name}@${library.version}`;
+    if (options.force !== true && options.store.hasPackage(id)) {
+      synced.push({
+        id,
+        name: library.name,
+        version: library.version,
+        chunks: options.store.countChunks(id),
+        tokens: 0,
+        status: "cached",
+        trusted: true,
+        kind: "artifact",
+      });
+      continue;
+    }
+
+    options.onProgress?.(`reading ${id}`);
+    const artifact = await readArtifact(library.dir);
+    // A dependency that ships no type declarations has nothing to derive, which is ordinary
+    // rather than a problem worth reporting on every sync.
+    if (artifact === undefined) continue;
+
+    options.store.indexPackage(
+      { id: artifact.id, name: artifact.name, version: artifact.version, kind: "artifact" },
+      artifact.chunks,
+      artifact.symbols,
+    );
+    synced.push({
+      id: artifact.id,
+      name: artifact.name,
+      version: artifact.version,
+      chunks: artifact.chunks.length,
+      tokens: artifact.chunks.reduce((total, chunk) => total + chunk.tokens, 0),
+      status: "indexed",
+      trusted: true,
+      kind: "artifact",
+    });
+  }
+
+  return synced;
+}
+
 async function readChunks(
   pkg: DiscoveredPackage,
 ): Promise<{ chunks: IndexedChunk[]; problems: string[] }> {

package/src/verify.ts

@@ -315,7 +315,7 @@ async function verifyPackage(
  * installed package can see. The `docspack` key is read as a fallback: it is build configuration
  * rather than payload, and a package built before `documents` reached the manifest has only that.
  */
-async function documentedLibraries(
+export async function documentedLibraries(
   packageDir: string,
   manifest: PackageManifest,
 ): Promise<readonly DocumentedLibrary[]> {