docspack 0.0.1 → 0.1.0

package/src/verify.ts

@@ -0,0 +1,355 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { readBuildConfig } from "./config.js";
+import { discoverPackages, resolvePackageDir } from "./discovery.js";
+import { DocspackError } from "./errors.js";
+import { type ExportSurface, readExportSurface } from "./exports.js";
+import { chunkId, LLMS_DIR, MANIFEST_FILE, type PackageManifest, parseManifest } from "./spec.js";
+
+export interface DriftFinding {
+  readonly chunkId: string;
+  readonly file: string;
+  /** The entity as the documentation wrote it, e.g. `client.setKey`. */
+  readonly entity: string;
+  /** The part that was checked: the last segment. */
+  readonly symbol: string;
+  /** The declared name it most closely resembles. */
+  readonly suggestion: string;
+}
+
+export type VerifyStatus = "verified" | "skipped";
+
+export interface VerifiedPackage {
+  readonly id: string;
+  readonly status: VerifyStatus;
+  /** Library the package declares it documents. */
+  readonly documents?: string;
+  /** Why nothing was checked, when status is `skipped`. */
+  readonly reason?: string;
+  readonly checked: number;
+  /** Entities that matched a declared name. */
+  readonly matched: number;
+  /** Entities absent from the type surface with nothing similar — ignored, not reported. */
+  readonly unrecognised: number;
+  readonly findings: readonly DriftFinding[];
+}
+
+export interface VerifyReport {
+  readonly packages: readonly VerifiedPackage[];
+  readonly ok: boolean;
+}
+
+export interface VerifyOptions {
+  readonly cwd: string;
+  /** Verify one package directory instead of the project's installed docs packages. */
+  readonly packageDir?: string;
+}
+
+/** Below this length a name is too generic to reason about. */
+const MIN_SYMBOL = 4;
+
+/**
+ * Names that appear in documentation everywhere and belong to the language or the platform
+ * rather than to any library.
+ */
+const UNIVERSAL = new Set([
+  "console",
+  "log",
+  "error",
+  "warn",
+  "then",
+  "catch",
+  "finally",
+  "await",
+  "async",
+  "json",
+  "stringify",
+  "parse",
+  "length",
+  "process",
+  "env",
+  "module",
+  "exports",
+  "require",
+  "import",
+  "default",
+  "window",
+  "document",
+  "fetch",
+  "response",
+  "request",
+]);
+
+/**
+ * Receivers that belong to the language or the runtime. `document.cookie` and `stream.Duplex`
+ * are documentation about the platform, and the library they appear in did not declare them.
+ */
+const PLATFORM = new Set([
+  "array",
+  "buffer",
+  "console",
+  "crypto",
+  "document",
+  "fs",
+  "global",
+  "globalthis",
+  "intl",
+  "json",
+  "math",
+  "navigator",
+  "number",
+  "object",
+  "os",
+  "path",
+  "process",
+  "promise",
+  "reflect",
+  "regexp",
+  "stream",
+  "string",
+  "symbol",
+  "url",
+  "window",
+]);
+
+/** `wrangler.jsonc` is a filename that survives the entity regex, not a member access. */
+const EXTENSIONS = new Set([
+  "cjs",
+  "css",
+  "html",
+  "js",
+  "json",
+  "jsonc",
+  "lock",
+  "md",
+  "mdx",
+  "mjs",
+  "toml",
+  "ts",
+  "tsx",
+  "txt",
+  "yaml",
+  "yml",
+]);
+
+/**
+ * Checks the identifiers documentation names against what the documented library declares.
+ *
+ * A finding is only produced when a name is absent *and* a very similar name exists — that is
+ * the signature of a rename. An absent name with nothing like it is far more likely to come
+ * from an example about some other library, so it is counted and ignored rather than reported.
+ * Precision matters more than recall here: a check that cries wolf is worse than no check.
+ */
+export async function verifyProject(options: VerifyOptions): Promise<VerifyReport> {
+  const packages =
+    options.packageDir === undefined
+      ? await verifyInstalled(options.cwd)
+      : [await verifyOne(options.cwd, options.packageDir)];
+
+  return { packages, ok: packages.every((pkg) => pkg.findings.length === 0) };
+}
+
+async function verifyInstalled(cwd: string): Promise<VerifiedPackage[]> {
+  const { packages } = await discoverPackages(cwd);
+  const results: VerifiedPackage[] = [];
+
+  for (const pkg of packages) {
+    results.push(await verifyPackage(cwd, pkg.id, pkg.dir, pkg.manifest));
+  }
+  return results;
+}
+
+async function verifyOne(cwd: string, packageDir: string): Promise<VerifiedPackage> {
+  let manifest: PackageManifest;
+  try {
+    const raw = await readFile(join(packageDir, LLMS_DIR, MANIFEST_FILE), "utf8");
+    manifest = parseManifest(JSON.parse(raw), `${LLMS_DIR}/${MANIFEST_FILE}`);
+  } catch {
+    throw new DocspackError(`No ${LLMS_DIR}/${MANIFEST_FILE} in ${packageDir}`, {
+      hint: "Run `docspack build` first.",
+    });
+  }
+
+  return verifyPackage(cwd, `${manifest.name}@${manifest.version}`, packageDir, manifest);
+}
+
+async function verifyPackage(
+  cwd: string,
+  id: string,
+  packageDir: string,
+  manifest: PackageManifest,
+): Promise<VerifiedPackage> {
+  const empty = { id, checked: 0, matched: 0, unrecognised: 0, findings: [] };
+
+  const { documents } = await readBuildConfig(packageDir);
+  if (documents === undefined) {
+    return {
+      ...empty,
+      status: "skipped",
+      reason: `declares no "docspack.documents", so there is nothing to check against`,
+    };
+  }
+
+  const libraryDir =
+    (await resolvePackageDir(documents, packageDir)) ?? (await resolvePackageDir(documents, cwd));
+  if (libraryDir === undefined) {
+    return { ...empty, status: "skipped", documents, reason: `${documents} is not installed` };
+  }
+
+  const surface = await readExportSurface(libraryDir);
+  if (surface === undefined || surface.names.size === 0) {
+    return {
+      ...empty,
+      status: "skipped",
+      documents,
+      reason: `${documents} ships no type declarations to check against`,
+    };
+  }
+
+  const declared = new Set([...surface.names].map(normalize));
+  const findings: DriftFinding[] = [];
+  let checked = 0;
+  let matched = 0;
+  let unrecognised = 0;
+
+  for (const chunk of manifest.chunks) {
+    for (const entity of chunk.entities) {
+      const symbol = checkableSymbol(entity);
+      if (symbol === undefined) continue;
+
+      checked += 1;
+      // Case and leading underscores differ between documentation and declarations often
+      // enough that treating them as drift is wrong.
+      if (declared.has(normalize(symbol))) {
+        matched += 1;
+        continue;
+      }
+
+      const suggestion = nearestName(symbol, surface);
+      if (suggestion === undefined) {
+        unrecognised += 1;
+        continue;
+      }
+
+      findings.push({
+        chunkId: chunkId(id, chunk.id),
+        file: chunk.file,
+        entity,
+        symbol,
+        suggestion,
+      });
+    }
+  }
+
+  return { id, status: "verified", documents, checked, matched, unrecognised, findings };
+}
+
+/**
+ * The name worth checking in an entity like `client.setKey()`, or nothing when the entity is
+ * not about the library at all: a filename, a platform API, or a name too generic to place.
+ */
+function checkableSymbol(entity: string): string | undefined {
+  const parts = entity.replace(/\(\)$/, "").split(".");
+  const symbol = parts[parts.length - 1] ?? entity;
+  const receiver = parts.length > 1 ? (parts[parts.length - 2] ?? "") : "";
+
+  if (symbol.length < MIN_SYMBOL) return undefined;
+  if (EXTENSIONS.has(symbol.toLowerCase())) return undefined;
+  if (UNIVERSAL.has(symbol.toLowerCase())) return undefined;
+  if (PLATFORM.has(receiver.toLowerCase())) return undefined;
+  return symbol;
+}
+
+/** Case and leading underscores are styling, not identity, when comparing two names. */
+function normalize(name: string): string {
+  return name.replace(/^_+/, "").toLowerCase();
+}
+
+/**
+ * The declared name a symbol was most likely renamed from, or nothing if no declared name
+ * resembles it closely enough to say so.
+ *
+ * Two shapes count, and only two. A near-identical spelling of the same leading word, which
+ * catches typos and small edits; and a qualifier inserted between the same first and last word
+ * — `setKey` against `setApiKey` — which is what most real renames look like. Everything else
+ * is left alone. In particular a differing first word is never a match: `includeLanguages` and
+ * `excludeLanguages` are two edits apart and mean opposite things.
+ */
+function nearestName(symbol: string, surface: ExportSurface): string | undefined {
+  const lower = normalize(symbol);
+  const words = camelWords(symbol);
+  let best: string | undefined;
+  let bestDistance = Number.POSITIVE_INFINITY;
+
+  for (const candidate of surface.names) {
+    if (candidate.length < MIN_SYMBOL) continue;
+    const candidateWords = camelWords(candidate);
+    if (isRequalified(words, candidateWords)) return candidate;
+
+    if (candidateWords[0] !== words[0]) continue;
+    const other = normalize(candidate);
+    if (Math.abs(other.length - lower.length) > 4) continue;
+
+    const distance = editDistance(lower, other, 3);
+    if (distance < bestDistance) {
+      bestDistance = distance;
+      best = candidate;
+    }
+  }
+
+  if (best === undefined || bestDistance > 2) return undefined;
+  // A two-character difference between two short names is coincidence more often than a rename.
+  if (bestDistance === 2 && symbol.length < 8) return undefined;
+  return best;
+}
+
+/** `setApiKey` becomes `["set", "api", "key"]`. */
+function camelWords(name: string): string[] {
+  return name
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .split(/[\s_$-]+/)
+    .filter((word) => word.length > 0)
+    .map((word) => word.toLowerCase());
+}
+
+/**
+ * True when the two names are the same multi-word name with a qualifier inserted, in either
+ * direction. Requiring the first and last word to match, and both names to have at least two
+ * words, is what keeps this from firing on every method that happens to start with `get`.
+ */
+function isRequalified(a: readonly string[], b: readonly string[]): boolean {
+  const [shorter, longer] = a.length <= b.length ? [a, b] : [b, a];
+  if (shorter.length < 2 || longer.length <= shorter.length) return false;
+  if (shorter[0] !== longer[0]) return false;
+  if (shorter[shorter.length - 1] !== longer[longer.length - 1]) return false;
+
+  let index = 0;
+  for (const word of longer) {
+    if (word === shorter[index]) index += 1;
+  }
+  return index === shorter.length;
+}
+
+/** Levenshtein distance, abandoned once it passes `limit`. */
+function editDistance(a: string, b: string, limit: number): number {
+  if (Math.abs(a.length - b.length) > limit) return limit + 1;
+
+  let previous = Array.from({ length: b.length + 1 }, (_, index) => index);
+  for (let i = 1; i <= a.length; i += 1) {
+    const current = [i];
+    let rowBest = i;
+    for (let j = 1; j <= b.length; j += 1) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      const value = Math.min(
+        (current[j - 1] ?? 0) + 1,
+        (previous[j] ?? 0) + 1,
+        (previous[j - 1] ?? 0) + cost,
+      );
+      current.push(value);
+      rowBest = Math.min(rowBest, value);
+    }
+    if (rowBest > limit) return limit + 1;
+    previous = current;
+  }
+  return previous[b.length] ?? limit + 1;
+}

package/bin/cli.js

@@ -1,2 +0,0 @@
-#!/usr/bin/env node
-console.log("docspack is currently in development.");