docspack 0.0.1 → 0.1.0

package/src/doctor.ts

@@ -0,0 +1,344 @@
+import type { Dirent } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { readBuildConfig } from "./config.js";
+import { PLACEHOLDER } from "./init/templates.js";
+import {
+  estimateTokens,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  type PackageManifest,
+  parseManifest,
+  resolveChunkFile,
+} from "./spec.js";
+import {
+  contentFingerprint,
+  findFiller,
+  findLongSentences,
+  hasConcreteContent,
+  withoutCode,
+} from "./style.js";
+
+export type Severity = "error" | "warn" | "info";
+
+export interface Finding {
+  readonly check: string;
+  readonly severity: Severity;
+  readonly message: string;
+  readonly where?: string;
+  readonly fix?: string;
+}
+
+export interface DoctorReport {
+  readonly ok: boolean;
+  readonly findings: readonly Finding[];
+  readonly chunks: number;
+  readonly tokens: number;
+}
+
+export interface DoctorOptions {
+  readonly dir: string;
+  /** Treat warnings as failures. Used by prepublishOnly and CI. */
+  readonly strict?: boolean;
+}
+
+/** Over this, a chunk crowds out the response budget; under it, a chunk answers nothing. */
+const MAX_CHUNK_TOKENS = 1500;
+const MIN_CHUNK_TOKENS = 30;
+
+/**
+ * Checks a documentation package the way the indexer and a reviewer would. A broken docs package
+ * still installs and still indexes, so every problem here is one that would otherwise be silent.
+ */
+export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
+  const findings: Finding[] = [];
+  const llmsDir = join(options.dir, LLMS_DIR);
+
+  const pkg = await readJson(join(options.dir, "package.json"));
+  const manifest = await loadManifest(llmsDir, findings);
+  if (manifest === undefined) {
+    return { ok: false, findings, chunks: 0, tokens: 0 };
+  }
+
+  checkPackageJson(pkg, manifest, findings);
+
+  let tokens = 0;
+  let newestSource = 0;
+  const seen = new Map<string, string>();
+  const abstract: string[] = [];
+  for (const chunk of manifest.chunks) {
+    let contents: string;
+    try {
+      contents = await readFile(resolveChunkFile(llmsDir, chunk.file), "utf8");
+    } catch (error) {
+      findings.push({
+        check: /outside of/.test(String(error)) ? "chunk-escapes" : "chunk-missing",
+        severity: "error",
+        message: /outside of/.test(String(error))
+          ? `chunk "${chunk.id}" points outside ${LLMS_DIR}/`
+          : `chunk "${chunk.id}" is missing its file`,
+        where: chunk.file,
+        fix: "Rebuild with `docspack build`.",
+      });
+      continue;
+    }
+
+    const text = contents.trim();
+    if (text.length === 0) {
+      findings.push({
+        check: "chunk-empty",
+        severity: "error",
+        message: `chunk "${chunk.id}" is empty`,
+        where: chunk.file,
+        fix: "Write the section, or remove it from the manifest.",
+      });
+      continue;
+    }
+
+    const size = chunk.tokens > 0 ? chunk.tokens : estimateTokens(text);
+    tokens += size;
+
+    if (size > MAX_CHUNK_TOKENS) {
+      findings.push({
+        check: "chunk-too-large",
+        severity: "warn",
+        message: `chunk "${chunk.id}" is ~${size} tokens; it will crowd out a response`,
+        where: chunk.file,
+        fix: "Split the section with more headings, or lower maxChunkTokens.",
+      });
+    } else if (size < MIN_CHUNK_TOKENS) {
+      findings.push({
+        check: "chunk-too-small",
+        severity: "warn",
+        message: `chunk "${chunk.id}" is ~${size} tokens; too small to answer anything`,
+        where: chunk.file,
+        fix: "Merge it into a neighbouring section.",
+      });
+    }
+
+    if (chunk.tags.length === 0 && chunk.entities.length === 0) {
+      findings.push({
+        check: "untagged",
+        severity: "warn",
+        message: `chunk "${chunk.id}" has no tags or entities; it is findable only by its prose`,
+        where: chunk.file,
+        fix: "Name the API in `inline code`, or add tags to the manifest.",
+      });
+    }
+
+    // Prose that mentions the marker inside `code` is documenting it, not leaving one behind.
+    if (withoutCode(text).includes(PLACEHOLDER)) {
+      findings.push({
+        check: "placeholder",
+        severity: "warn",
+        message: `chunk "${chunk.id}" still contains ${PLACEHOLDER} markers from the template`,
+        where: chunk.file,
+        fix: "Replace the template text with real documentation.",
+      });
+    }
+
+    const filler = findFiller(text);
+    if (filler.length > 0) {
+      const listed = filler
+        .slice(0, 3)
+        .map((hit) => `"${hit.phrase}"`)
+        .join(", ");
+      findings.push({
+        check: "filler",
+        severity: "warn",
+        message: `chunk "${chunk.id}" contains narration a model does not need: ${listed}`,
+        where: chunk.file,
+        fix: "Delete it. Nobody searches for these words and they carry no fact.",
+      });
+    }
+
+    const long = findLongSentences(text);
+    if (long.length > 0) {
+      findings.push({
+        check: "long-sentence",
+        severity: "warn",
+        message: `chunk "${chunk.id}" has ${plural(long.length, "sentence")} over 40 words`,
+        where: chunk.file,
+        fix: "Split them. One claim per sentence retrieves and reads better.",
+      });
+    }
+
+    if (!hasConcreteContent(text)) abstract.push(chunk.id);
+
+    const fingerprint = contentFingerprint(text);
+    const twin = seen.get(fingerprint);
+    if (twin === undefined) seen.set(fingerprint, chunk.id);
+    else {
+      findings.push({
+        check: "duplicate-chunk",
+        severity: "warn",
+        message: `chunk "${chunk.id}" says the same thing as "${twin}"`,
+        where: chunk.file,
+        fix: "Remove one. Duplicates compete for the same query and waste the response budget.",
+      });
+    }
+  }
+
+  if (abstract.length > 0) {
+    findings.push({
+      check: "no-example",
+      severity: "info",
+      message: `${plural(abstract.length, "chunk")} of ${manifest.chunks.length} have no code, list or table: ${abstract.slice(0, 3).join(", ")}${abstract.length > 3 ? ", …" : ""}`,
+      fix: "Agents act on concrete text. Add the call, the signature or the values.",
+    });
+  }
+
+  if (manifest.chunks.every((chunk) => chunk.entities.length === 0)) {
+    findings.push({
+      check: "no-entities",
+      severity: "info",
+      message:
+        "no chunk names an identifier; agents searching for an API name may miss this package",
+      fix: "Put API names in `inline code` in the source documentation.",
+    });
+  }
+
+  newestSource = await newestSourceMtime(options.dir);
+  const manifestMtime = await mtime(join(llmsDir, MANIFEST_FILE));
+  if (newestSource > 0 && manifestMtime > 0 && newestSource > manifestMtime) {
+    findings.push({
+      check: "payload-stale",
+      severity: "warn",
+      message: "the documentation source is newer than the generated payload",
+      fix: "Run `docspack build`.",
+    });
+  }
+
+  const failed = findings.some(
+    (finding) =>
+      finding.severity === "error" || (options.strict === true && finding.severity === "warn"),
+  );
+
+  return { ok: !failed, findings, chunks: manifest.chunks.length, tokens };
+}
+
+function plural(count: number, noun: string): string {
+  return `${count} ${noun}${count === 1 ? "" : "s"}`;
+}
+
+async function loadManifest(
+  llmsDir: string,
+  findings: Finding[],
+): Promise<PackageManifest | undefined> {
+  let raw: string;
+  try {
+    raw = await readFile(join(llmsDir, MANIFEST_FILE), "utf8");
+  } catch {
+    findings.push({
+      check: "manifest-invalid",
+      severity: "error",
+      message: `no ${LLMS_DIR}/${MANIFEST_FILE}`,
+      fix: "Run `docspack build` to generate the payload.",
+    });
+    return undefined;
+  }
+
+  try {
+    return parseManifest(JSON.parse(raw), `${LLMS_DIR}/${MANIFEST_FILE}`);
+  } catch (error) {
+    findings.push({
+      check: "manifest-invalid",
+      severity: "error",
+      message: error instanceof Error ? error.message : String(error),
+      where: `${LLMS_DIR}/${MANIFEST_FILE}`,
+      fix: "Run `docspack build` to regenerate it.",
+    });
+    return undefined;
+  }
+}
+
+function checkPackageJson(
+  pkg: Record<string, unknown> | undefined,
+  manifest: PackageManifest,
+  findings: Finding[],
+): void {
+  if (pkg === undefined) {
+    findings.push({
+      check: "manifest-invalid",
+      severity: "error",
+      message: "no package.json next to the payload",
+      fix: "Run `docspack init` or add a package.json.",
+    });
+    return;
+  }
+
+  if (typeof pkg.version === "string" && pkg.version !== manifest.version) {
+    findings.push({
+      check: "version-mismatch",
+      severity: "error",
+      message: `package.json is ${pkg.version} but the manifest says ${manifest.version}`,
+      fix: "Run `docspack build` after bumping the version.",
+    });
+  }
+  if (typeof pkg.name === "string" && pkg.name !== manifest.name) {
+    findings.push({
+      check: "version-mismatch",
+      severity: "error",
+      message: `package.json is ${pkg.name} but the manifest says ${manifest.name}`,
+      fix: "Run `docspack build` to regenerate the manifest.",
+    });
+  }
+
+  const files = Array.isArray(pkg.files) ? pkg.files.map(String) : undefined;
+  if (
+    files !== undefined &&
+    !files.some((entry) => entry.replace(/^\.\//, "").startsWith(LLMS_DIR))
+  ) {
+    findings.push({
+      check: "files-missing-payload",
+      severity: "error",
+      message: `"files" does not include ${LLMS_DIR}; publishing would ship a package with no documentation`,
+      where: "package.json",
+      fix: `Add "${LLMS_DIR}" to the files array.`,
+    });
+  }
+}
+
+/** Newest mtime across the configured documentation source, used to spot a stale payload. */
+async function newestSourceMtime(dir: string): Promise<number> {
+  const config = await readBuildConfig(dir);
+  const source = config.from ?? config.openapi;
+  if (source === undefined) return 0;
+
+  const target = join(dir, source);
+  const single = await mtime(target);
+  if (single > 0 && config.openapi !== undefined) return single;
+
+  let found: Dirent[];
+  try {
+    found = await readdir(target, { recursive: true, withFileTypes: true });
+  } catch {
+    return single;
+  }
+
+  let newest = 0;
+  for (const entry of found) {
+    if (!entry.isFile()) continue;
+    newest = Math.max(newest, await mtime(join(entry.parentPath, entry.name)));
+  }
+  return newest;
+}
+
+async function mtime(path: string): Promise<number> {
+  try {
+    return (await stat(path)).mtimeMs;
+  } catch {
+    return 0;
+  }
+}
+
+async function readJson(path: string): Promise<Record<string, unknown> | undefined> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+    return typeof parsed === "object" && parsed !== null
+      ? (parsed as Record<string, unknown>)
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}

package/src/document.ts

@@ -0,0 +1,59 @@
+import { stripBoilerplate } from "./style.js";
+
+export interface CleanedDocument {
+  readonly text: string;
+  /** Title taken from front matter, when the document declares one. */
+  readonly title?: string;
+  /** Keywords taken from front matter, indexed alongside the prose. */
+  readonly tags: readonly string[];
+}
+
+const FRONT_MATTER = /^?---\r?\n([\s\S]*?)\r?\n---\r?\n?/;
+
+/**
+ * Prepares a source document for chunking: lifts front matter into metadata the index can use,
+ * and drops site boilerplate. Prose is left alone — the index matches on it.
+ */
+export function cleanDocument(raw: string): CleanedDocument {
+  const match = FRONT_MATTER.exec(raw);
+  const body = stripBoilerplate(match === null ? raw : raw.slice(match[0].length)).trim();
+  if (match?.[1] === undefined) return { text: body, tags: [] };
+
+  const meta = parseFrontMatter(match[1]);
+  return {
+    text: body,
+    ...(meta.title === undefined ? {} : { title: meta.title }),
+    tags: meta.tags,
+  };
+}
+
+/**
+ * Reads the two front matter fields worth indexing. This is deliberately not a YAML parser:
+ * anything more complex is not metadata a retrieval index can use.
+ */
+function parseFrontMatter(block: string): { title?: string; tags: string[] } {
+  let title: string | undefined;
+  const tags: string[] = [];
+
+  for (const line of block.split("\n")) {
+    const entry = /^\s*([A-Za-z_][\w-]*)\s*:\s*(.*)$/.exec(line);
+    if (entry?.[1] === undefined || entry[2] === undefined) continue;
+
+    const key = entry[1].toLowerCase();
+    const value = entry[2].trim().replace(/^["']|["']$/g, "");
+    if (key === "title" && value.length > 0) title = value;
+    if ((key === "tags" || key === "keywords") && value.startsWith("[")) {
+      tags.push(...splitList(value));
+    }
+  }
+
+  return { ...(title === undefined ? {} : { title }), tags };
+}
+
+function splitList(value: string): string[] {
+  return value
+    .replace(/^\[|\]$/g, "")
+    .split(",")
+    .map((item) => item.trim().replace(/^["']|["']$/g, ""))
+    .filter((item) => item.length > 0);
+}

package/src/errors.ts

@@ -0,0 +1,10 @@
+/** An error with a message safe to show to the user, plus an optional next step. */
+export class DocspackError extends Error {
+  readonly hint: string | undefined;
+
+  constructor(message: string, options: { hint?: string; cause?: unknown } = {}) {
+    super(message, options.cause === undefined ? undefined : { cause: options.cause });
+    this.name = "DocspackError";
+    this.hint = options.hint;
+  }
+}

package/src/exports.ts

@@ -0,0 +1,120 @@
+import type { Dirent } from "node:fs";
+import { readdir, readFile } from "node:fs/promises";
+import { join, relative, sep } from "node:path";
+
+/**
+ * The names a package declares, read statically from its type declarations.
+ *
+ * Importing the package and reading its keys would be more accurate and would execute
+ * third-party code, which is not acceptable for a check that runs over every dependency.
+ */
+export interface ExportSurface {
+  readonly name: string;
+  readonly version: string;
+  /** Every identifier declared anywhere in the package's `.d.ts` files. */
+  readonly names: ReadonlySet<string>;
+  readonly files: number;
+}
+
+/** Declaration forms that introduce a name a caller could write. */
+const DECLARATIONS = [
+  /(?:^|\s)(?:export\s+)?(?:declare\s+)?(?:async\s+)?(?:function|const|let|var|class|interface|type|enum|namespace|module)\s+([A-Za-z_$][\w$]*)/g,
+  /(?:^|\s)(?:readonly\s+)?(?:get|set)\s+([A-Za-z_$][\w$]*)\s*\(/g,
+  /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\??\s*[(<:]/gm,
+];
+const EXPORT_LIST = /export\s*(?:type\s*)?\{([^}]*)\}/g;
+
+const MAX_FILES = 400;
+const MAX_BYTES = 8_000_000;
+
+/**
+ * Reads every `.d.ts` in an installed package. Reading all of them rather than following the
+ * entry points is deliberate: a wider set of known names means fewer false accusations, which
+ * matters more here than catching every last rename.
+ */
+export async function readExportSurface(dir: string): Promise<ExportSurface | undefined> {
+  const manifest = await readJson(join(dir, "package.json"));
+  if (manifest === undefined) return undefined;
+
+  const declarations = await findDeclarationFiles(dir);
+  if (declarations.length === 0) return undefined;
+
+  const names = new Set<string>();
+  let budget = MAX_BYTES;
+
+  for (const file of declarations) {
+    if (budget <= 0) break;
+    let source: string;
+    try {
+      source = await readFile(file, "utf8");
+    } catch {
+      continue;
+    }
+    budget -= source.length;
+    collectNames(stripComments(source), names);
+  }
+
+  return {
+    name: typeof manifest.name === "string" ? manifest.name : "",
+    version: typeof manifest.version === "string" ? manifest.version : "",
+    names,
+    files: declarations.length,
+  };
+}
+
+function collectNames(source: string, into: Set<string>): void {
+  for (const pattern of DECLARATIONS) {
+    pattern.lastIndex = 0;
+    for (const match of source.matchAll(pattern)) {
+      if (match[1] !== undefined) into.add(match[1]);
+    }
+  }
+
+  EXPORT_LIST.lastIndex = 0;
+  for (const match of source.matchAll(EXPORT_LIST)) {
+    for (const entry of (match[1] ?? "").split(",")) {
+      // `export { internalName as publicName }` — both names are worth knowing.
+      for (const part of entry.split(/\s+as\s+/)) {
+        const name = part.trim().replace(/^type\s+/, "");
+        if (/^[A-Za-z_$][\w$]*$/.test(name)) into.add(name);
+      }
+    }
+  }
+}
+
+function stripComments(source: string): string {
+  return source.replace(/\/\*[\s\S]*?\*\//g, " ").replace(/(^|\s)\/\/[^\n]*/g, "$1");
+}
+
+async function findDeclarationFiles(dir: string): Promise<string[]> {
+  let found: Dirent[];
+  try {
+    found = await readdir(dir, { recursive: true, withFileTypes: true });
+  } catch {
+    return [];
+  }
+
+  return found
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".d.ts") && !isVendored(dir, entry))
+    .map((entry) => join(entry.parentPath, entry.name))
+    .slice(0, MAX_FILES);
+}
+
+/**
+ * True for a file under a nested `node_modules`. The package's own path is almost always inside
+ * one, so the check has to be relative to the package root rather than absolute.
+ */
+function isVendored(dir: string, entry: Dirent): boolean {
+  return relative(dir, entry.parentPath).split(sep).includes("node_modules");
+}
+
+async function readJson(path: string): Promise<Record<string, unknown> | undefined> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+    return typeof parsed === "object" && parsed !== null
+      ? (parsed as Record<string, unknown>)
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}

package/src/html.ts

@@ -0,0 +1,24 @@
+import TurndownService from "turndown";
+
+// Documentation pages wrap their prose in <main> or <article>; everything outside is chrome.
+const MAIN_CONTENT = /<(main|article)\b[^>]*>([\s\S]*)<\/\1>/i;
+
+const turndown = new TurndownService({
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  bulletListMarker: "-",
+  hr: "---",
+});
+turndown.remove(["script", "style", "noscript", "nav", "header", "footer", "aside", "form", "svg"]);
+
+export function isHtml(contentType: string, body: string): boolean {
+  if (/\btext\/html\b|\bapplication\/xhtml\+xml\b/i.test(contentType)) return true;
+  if (/\btext\/(markdown|plain)\b/i.test(contentType)) return false;
+  return /^\s*(?:<!doctype html|<html\b)/i.test(body);
+}
+
+/** Best-effort HTML to Markdown conversion for sources that do not serve Markdown. */
+export function htmlToMarkdown(html: string): string {
+  const match = MAIN_CONTENT.exec(html);
+  return turndown.turndown(match?.[2] ?? html).trim();
+}