docspack 0.4.0 → 1.1.0

package/src/build.ts

@@ -1,6 +1,16 @@
+import { createHash } from "node:crypto";
 import type { Dirent } from "node:fs";
 import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { join, relative, sep } from "node:path";
+import { basename, join, relative, sep } from "node:path";
+import { readApiFile } from "@docspack/lapis/read";
+import {
+  type ApiDocument,
+  OpenApiError,
+  operationDigest,
+  operationEntities,
+  operationTags,
+  operationTitle,
+} from "@docspack/openapi";
 import { findEntry } from "@docspack/registry";
 import { readBuildConfig } from "./config.js";
 import { cleanDocument } from "./document.js";
@@ -33,6 +43,17 @@ export interface BuildOptions {
   readonly openapi?: string;
   /** Registry id or llms.txt URL to fetch and package. */
   readonly source?: string;
+  /**
+   * JSON records to package, or `-` for standard input. One chunk per record, so anything a query
+   * can produce rows for — a table, an export, an API response — reaches the index without this
+   * tool needing a driver for it.
+   */
+  readonly json?: string;
+  /**
+   * Build a working corpus rather than a publishable package: identity is derived instead of
+   * demanded, and no `package.json` is written.
+   */
+  readonly local?: boolean;
   readonly pages?: number;
   readonly maxChunkTokens?: number;
   /**
@@ -58,6 +79,16 @@ export interface BuildResult {
 
 /** A document before it is split into chunks. */
 interface SourceDocument {
+  /**
+   * Chunk id to use instead of one derived from the title. Atomic documents only.
+   *
+   * A derived id is a slug of the prose, which is right for a Markdown page — the heading is the
+   * stable thing about it. It is wrong for a generated reference, where the title carries a summary
+   * somebody will reword: `POST /v1/charges — Create a charge` becoming `…— Creates a charge`
+   * renames the chunk, and a chunk id is what `feedback add --chunk` pins and what a published link
+   * resolves to.
+   */
+  readonly id?: string;
   readonly title: string;
   /** File path or URL the text came from, recorded in the chunk for provenance. */
   readonly origin: string;
@@ -71,8 +102,8 @@ interface SourceDocument {
 }
 
 const DEFAULT_MAX_CHUNK_TOKENS = 800;
-const MARKDOWN = /\.(md|mdx|markdown)$/i;
-const HTTP_METHODS = ["get", "post", "put", "patch", "delete", "head", "options"] as const;
+/** The source extensions a corpus is collected from. */
+export const MARKDOWN = /\.(md|mdx|markdown)$/i;
 
 /**
  * Generates a docs package: `.llms/manifest.json`, `.llms/chunks/*.md` and an `llms.txt` table of
@@ -202,6 +233,16 @@ async function resolveIdentity(
   const version =
     options.version ?? (typeof existing.version === "string" ? existing.version : undefined);
 
+  // A working corpus has no publisher to name it and no release to version, so demanding both is
+  // friction with nothing on the other side of it. A publishable package still fails below.
+  if (options.local === true) {
+    return {
+      name: name ?? localPackageName(options),
+      version: version ?? LOCAL_VERSION,
+      createPackageJson: false,
+    };
+  }
+
   if (name === undefined) {
     throw new DocspackError("The package needs a name", {
       hint: "Pass --name @vendor/docspack, or run inside a directory that has a package.json.",
@@ -214,17 +255,134 @@ async function resolveIdentity(
   return { name, version, createPackageJson: !found };
 }
 
+/**
+ * A stable version for a working corpus, rather than a hash of its contents.
+ *
+ * A content hash would look like the careful choice and is the wrong one: it changes the store key
+ * on every edit, so each save leaves the previous index behind as garbage that nothing evicts.
+ * Freshness is a property of each source, tracked per source, not of the corpus's name.
+ */
+export const LOCAL_VERSION = "0.0.0";
+
+/** Named after where the text came from, which is the only thing that distinguishes one corpus. */
+export function localPackageName(options: Pick<BuildOptions, "from" | "json">): string {
+  const source =
+    options.from ?? (options.json !== undefined && options.json !== "-" ? options.json : undefined);
+  const base = source === undefined ? "corpus" : basename(source).replace(/\.[^.]+$/, "");
+  return `@local/${slugify(base, "corpus")}`;
+}
+
+/**
+ * Records as JSON, from a file or standard input: `[{ "id", "title", "text", "tags", "entities" }]`.
+ *
+ * This is the whole of the non-file input, and it is deliberately not a database adapter. A driver
+ * would put a native dependency into a CLI whose startup cost is the reason it is usable from an
+ * agent at all, and every store worth indexing can already emit JSON — `psql --json`,
+ * `sqlite3 -json`, an API response saved to a file. The conversion belongs to whoever owns the
+ * query.
+ *
+ * A record carrying an `id` is atomic: its id is its key, and splitting it would either duplicate
+ * that key across sections or discard it. A record without one is split like any other document.
+ */
+async function collectFromJson(source: string): Promise<SourceDocument[]> {
+  const origin = source === "-" ? "stdin" : source;
+  const raw = source === "-" ? await readStdin() : await readFile(source, "utf8");
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    throw new DocspackError(`${origin} is not valid JSON`, {
+      hint: 'Expected an array of records: [{ "title": "…", "text": "…" }].',
+      cause: error,
+    });
+  }
+  if (!Array.isArray(parsed)) {
+    throw new DocspackError(`${origin} is not a JSON array`, {
+      hint: 'Expected [{ "title": "…", "text": "…" }], one entry per record.',
+    });
+  }
+  if (parsed.length === 0) {
+    throw new DocspackError(`${origin} contains no records`, {
+      hint: "A query that returned nothing indexes nothing; check the query first.",
+    });
+  }
+
+  return parsed.map((entry, index): SourceDocument => {
+    const where = `${origin} record ${index}`;
+    if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
+      throw new DocspackError(`${where} is not an object`);
+    }
+    const record = entry as Record<string, unknown>;
+    const title = field(record.title, `${where}.title`);
+    const text = field(record.text, `${where}.text`);
+    const id = record.id === undefined ? undefined : field(record.id, `${where}.id`);
+
+    return {
+      title,
+      text,
+      origin,
+      ...(id === undefined ? {} : { id: slugify(id, `record-${index}`), atomic: true }),
+      ...(record.tags === undefined ? {} : { tags: stringList(record.tags, `${where}.tags`) }),
+      ...(record.entities === undefined
+        ? {}
+        : { entities: stringList(record.entities, `${where}.entities`) }),
+    };
+  });
+}
+
+function field(value: unknown, where: string): string {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new DocspackError(`${where} must be a non-empty string`);
+  }
+  return value;
+}
+
+function stringList(value: unknown, where: string): string[] {
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
+    throw new DocspackError(`${where} must be an array of strings`);
+  }
+  return value as string[];
+}
+
+async function readStdin(): Promise<string> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
+  return Buffer.concat(chunks).toString("utf8");
+}
+
+/**
+ * Prose and an API document, in that order, or a mirror on its own.
+ *
+ * `--from` and `--openapi` combine, because a library with an HTTP API usually has both and
+ * documents them in the same package — refusing the combination meant a vendor had to pick which
+ * half of their documentation to publish. A mirror does not combine with either: it is somebody
+ * else's whole documentation set, fetched, and merging our own prose into it would produce a package
+ * claiming to be a mirror of something it is not.
+ *
+ * Operations come first so that their ids win a collision. A prose chunk's id is derived and may
+ * take a numeric suffix without anything breaking; an operation's is pinned to its endpoint, and a
+ * suffix there is the stable address moving.
+ */
 async function collect(options: BuildOptions, warnings: string[]): Promise<SourceDocument[]> {
-  const chosen = [options.from, options.openapi, options.source].filter(
-    (value) => value !== undefined,
-  );
-  if (chosen.length > 1) {
-    throw new DocspackError("Choose one input: --from, --openapi or a source");
+  if (options.source !== undefined) {
+    if (options.from !== undefined || options.openapi !== undefined || options.json !== undefined) {
+      throw new DocspackError("A source cannot be combined with --from, --openapi or --from-json", {
+        hint: "A mirror is a whole documentation set on its own.",
+      });
+    }
+    return collectFromSource(options, warnings);
   }
 
-  if (options.openapi !== undefined) return collectFromOpenApi(options.openapi);
-  if (options.source !== undefined) return collectFromSource(options, warnings);
-  return collectFromDirectory(options.from ?? join(options.out, "docs"));
+  const documents: SourceDocument[] = [];
+  if (options.openapi !== undefined) documents.push(...(await collectFromOpenApi(options.openapi)));
+  if (options.json !== undefined) documents.push(...(await collectFromJson(options.json)));
+  // The default `docs/` directory is only read when nothing else was named: a build given only an
+  // OpenAPI document or a set of records should not fail on a `docs/` directory that does not exist.
+  if (options.from !== undefined || (options.openapi === undefined && options.json === undefined)) {
+    documents.push(...(await collectFromDirectory(options.from ?? join(options.out, "docs"))));
+  }
+  return documents;
 }
 
 async function collectFromDirectory(dir: string): Promise<SourceDocument[]> {
@@ -281,6 +439,13 @@ async function collectFromSource(
 
   const documents: SourceDocument[] = [];
   const links = fetchableLinks(parsed).slice(0, options.pages ?? 50);
+  // Two links can name the same document. `fetchableLinks` already drops a repeated URL and a
+  // repeated fragment, but a site whose index links anchors as a query — `/v4?id=codecs` — gets
+  // one entry per anchor, and each one fetches the whole page again. Left alone that packages
+  // the same text dozens of times: an index mostly made of duplicates, which is slower to search
+  // and ranks worse, because copies of one page crowd out the rest of the corpus.
+  const seen = new Set<string>();
+  let duplicates = 0;
 
   for (const link of links) {
     options.onProgress?.(`fetching ${link.url}`);
@@ -293,6 +458,12 @@ async function collectFromSource(
         warnings.push(`skipped ${link.url}: document was empty`);
         continue;
       }
+      const fingerprint = createHash("sha256").update(text).digest("hex");
+      if (seen.has(fingerprint)) {
+        duplicates += 1;
+        continue;
+      }
+      seen.add(fingerprint);
       documents.push({
         title: link.title.length > 0 ? link.title : (firstHeading(text) ?? link.url),
         origin: link.url,
@@ -306,6 +477,11 @@ async function collectFromSource(
     }
   }
 
+  if (duplicates > 0) {
+    warnings.push(
+      `skipped ${duplicates} of ${links.length} linked documents whose text repeated one already packaged`,
+    );
+  }
   if (fetchableLinks(parsed).length > links.length) {
     warnings.push(
       `packaged ${links.length} of ${fetchableLinks(parsed).length} linked documents (raise with --pages)`,
@@ -314,121 +490,69 @@ async function collectFromSource(
   return documents;
 }
 
+/**
+ * One chunk per operation, plus the document's own overview.
+ *
+ * What each chunk carries is `@docspack/openapi`'s digest: the base URL, the credential, the inputs
+ * with their types, the body shape, the response, the failures, the types it reaches, and a runnable
+ * cURL call — in LAPIS notation, which is an open format for describing an API to a model. That
+ * package's README has the measurement; the short version is that a digest is about 72% smaller than
+ * the smallest correct slice of OpenAPI JSON for the same operation, and a rounding error against
+ * the whole document, which is what an agent without retrieval is handed instead.
+ *
+ * This is deliberately not a summary. An agent that retrieved the chunk for `POST /v1/charges` can
+ * make the call from it without reading anything else, which is the only test a reference chunk has
+ * to pass.
+ */
 async function collectFromOpenApi(file: string): Promise<SourceDocument[]> {
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(await readFile(file, "utf8"));
-  } catch (error) {
-    throw new DocspackError(`Could not read the OpenAPI document ${file}`, {
-      hint: "Only JSON is supported. Convert YAML with a tool such as `yq -o json`.",
-      cause: error,
-    });
-  }
-
-  const spec = parsed as {
-    info?: { title?: unknown; description?: unknown; version?: unknown };
-    paths?: Record<string, unknown>;
-  };
+  // JSON or YAML, decided by what the file contains rather than by its extension. The reader is a
+  // separate entry point of `@docspack/lapis` so that only the callers who need a YAML parser load
+  // one; this is a CLI, so it is one of them.
+  const document = await readOpenApi(file);
   const documents: SourceDocument[] = [];
-  const apiTitle = typeof spec.info?.title === "string" ? spec.info.title : "API";
 
-  if (typeof spec.info?.description === "string" && spec.info.description.trim().length > 0) {
+  if (document.description.length > 0) {
     documents.push({
-      title: `${apiTitle} overview`,
+      id: "api-overview",
+      title: `${document.title} overview`,
       origin: file,
-      text: spec.info.description.trim(),
+      text: document.description,
       atomic: true,
-      tags: ["overview", "introduction"],
+      tags: ["overview", "introduction", "api"],
     });
   }
 
-  for (const [path, item] of Object.entries(spec.paths ?? {})) {
-    if (typeof item !== "object" || item === null) continue;
-    const pathItem = item as Record<string, unknown>;
-
-    for (const method of HTTP_METHODS) {
-      const operation = pathItem[method];
-      if (typeof operation !== "object" || operation === null) continue;
-      documents.push(renderOperation(method, path, operation as Record<string, unknown>, file));
-    }
-  }
-
-  if (documents.length === 0) {
-    throw new DocspackError(`No operations found in ${file}`, {
-      hint: "The document needs a `paths` object with at least one operation.",
+  for (const operation of document.operations) {
+    documents.push({
+      // The endpoint, not the operation id: a vendor renames `createCharge` to `postCharges` and
+      // the endpoint is still `POST /v1/charges`. Both are recorded as entities, so either spelling
+      // still finds the chunk.
+      id: slugify(`${operation.method} ${operation.path}`, "operation"),
+      title: operationTitle(operation),
+      origin: `${file}#${operation.method} ${operation.path}`,
+      text: operationDigest(document, operation),
+      atomic: true,
+      tags: operationTags(operation),
+      entities: operationEntities(operation),
     });
   }
+
   return documents;
 }
 
-function renderOperation(
-  method: string,
-  path: string,
-  operation: Record<string, unknown>,
-  file: string,
-): SourceDocument {
-  const summary = typeof operation.summary === "string" ? operation.summary : "";
-  const description = typeof operation.description === "string" ? operation.description : "";
-  const operationId = typeof operation.operationId === "string" ? operation.operationId : undefined;
-  const tags = Array.isArray(operation.tags)
-    ? operation.tags.filter((tag): tag is string => typeof tag === "string")
-    : [];
-
-  const lines = [`${method.toUpperCase()} ${path}`, ""];
-  if (summary.length > 0) lines.push(summary, "");
-  if (description.length > 0) lines.push(description, "");
-  if (operationId !== undefined) lines.push(`Operation: \`${operationId}\``, "");
-
-  const parameters = Array.isArray(operation.parameters) ? operation.parameters : [];
-  if (parameters.length > 0) {
-    lines.push("## Parameters", "");
-    for (const raw of parameters) {
-      if (typeof raw !== "object" || raw === null) continue;
-      const parameter = raw as {
-        name?: unknown;
-        in?: unknown;
-        required?: unknown;
-        description?: unknown;
-      };
-      if (typeof parameter.name !== "string") continue;
-      const location = typeof parameter.in === "string" ? parameter.in : "query";
-      const required = parameter.required === true ? ", required" : "";
-      const note =
-        typeof parameter.description === "string" && parameter.description.length > 0
-          ? ` — ${parameter.description}`
-          : "";
-      lines.push(`- \`${parameter.name}\` (${location}${required})${note}`);
-    }
-    lines.push("");
-  }
-
-  const responses = operation.responses;
-  if (typeof responses === "object" && responses !== null) {
-    lines.push("## Responses", "");
-    for (const [status, raw] of Object.entries(responses as Record<string, unknown>)) {
-      const detail =
-        typeof raw === "object" &&
-        raw !== null &&
-        typeof (raw as { description?: unknown }).description === "string"
-          ? ` — ${(raw as { description: string }).description}`
-          : "";
-      lines.push(`- \`${status}\`${detail}`);
+/** The reader's failures, re-reported as the CLI's own error so the hint survives. */
+async function readOpenApi(file: string): Promise<ApiDocument> {
+  try {
+    return (await readApiFile(file)).document;
+  } catch (error) {
+    if (error instanceof OpenApiError) {
+      throw new DocspackError(error.message, {
+        ...(error.hint === undefined ? {} : { hint: error.hint }),
+        cause: error,
+      });
     }
-    lines.push("");
+    throw error;
   }
-
-  return {
-    title: summary.length > 0 ? summary : `${method.toUpperCase()} ${path}`,
-    origin: `${file}#${method} ${path}`,
-    text: lines.join("\n").trim(),
-    atomic: true,
-    tags: [
-      method,
-      ...tags,
-      ...path.split("/").filter((part) => part.length > 0 && !part.startsWith("{")),
-    ],
-    ...(operationId === undefined ? {} : { entities: [operationId] }),
-  };
 }
 
 function chunkDocuments(
@@ -455,7 +579,12 @@ function chunkDocuments(
     for (const section of sections) {
       const directives = readDirectives(section.body);
       const body = withoutDuplicateHeading(directives.body, section.heading);
-      const derived = chunkSlug(document.title, section.heading);
+      // Only for an atomic document: a pinned id on a document that splits would give every
+      // section the same id.
+      const derived =
+        document.atomic === true && document.id !== undefined
+          ? document.id
+          : chunkSlug(document.title, section.heading);
       const id = uniqueId(derived, taken);
       if (id !== derived) collisions.push(derived);
       const libraries = directives.documents.length > 0 ? directives.documents : document.documents;

package/src/changed.ts

@@ -0,0 +1,99 @@
+import { identifiers } from "./coverage.js";
+import type { Store } from "./db.js";
+import { discoverLibraries, discoverPackages } from "./discovery.js";
+import { DocspackError } from "./errors.js";
+import { packageId } from "./spec.js";
+
+export interface SurfaceChange {
+  readonly library: string;
+  readonly from: string;
+  readonly to: string;
+  /** Names the newer release exports that the older one did not. */
+  readonly added: readonly string[];
+  /** Names the older release exported that the newer one does not. */
+  readonly removed: readonly string[];
+  /**
+   * Added names that no documentation package installed here mentions. These are the ones a
+   * model cannot know: too new for its training data, and absent from what the vendor published.
+   */
+  readonly undocumented: readonly string[];
+}
+
+export interface ChangedOptions {
+  readonly cwd: string;
+  readonly store: Store;
+  /** Library name, optionally with the version to compare against: `hono` or `hono@4.0.0`. */
+  readonly library: string;
+}
+
+/**
+ * What changed in a library's exported surface between two versions in the store.
+ *
+ * Both versions come from the local index, never the network: the store is global, so a version
+ * indexed for another project on this machine is already here. Measured across two libraries,
+ * upgrades are overwhelmingly additive — hono 3.12.12 to 4.13.3 removed three names and added
+ * 243 — so the useful report is what exists now that an older release did not have.
+ */
+export async function changedSurface(options: ChangedOptions): Promise<SurfaceChange> {
+  const [name, requested] = split(options.library);
+
+  const installed = (await discoverLibraries(options.cwd)).find((library) => library.name === name);
+  const indexed = options.store.versionsOf(name).filter((pkg) => pkg.kind === "artifact");
+  if (indexed.length === 0) {
+    throw new DocspackError(`No version of ${name} is indexed`, {
+      hint:
+        installed === undefined
+          ? `${name} is not a dependency of this project.`
+          : "Run `docspack sync` to index the installed version, then ask again.",
+    });
+  }
+
+  const to = installed?.version ?? indexed[indexed.length - 1]?.version;
+  if (to === undefined) throw new DocspackError(`No version of ${name} is indexed`);
+
+  const others = indexed.map((pkg) => pkg.version).filter((version) => version !== to);
+  const from = requested ?? others[others.length - 1];
+  if (from === undefined) {
+    throw new DocspackError(`Only ${name}@${to} is indexed, so there is nothing to compare`, {
+      hint: "Index another version by running `docspack sync` in a project that installs it. The store is shared across projects on this machine.",
+    });
+  }
+  if (!options.store.hasPackage(packageId(name, from))) {
+    throw new DocspackError(`${name}@${from} is not in the store`, {
+      hint: `Indexed versions: ${indexed.map((pkg) => pkg.version).join(", ")}`,
+    });
+  }
+
+  const before = new Set(options.store.symbolsOf(packageId(name, from)));
+  const after = new Set(options.store.symbolsOf(packageId(name, to)));
+  const added = [...after].filter((symbol) => !before.has(symbol)).sort();
+  const removed = [...before].filter((symbol) => !after.has(symbol)).sort();
+
+  const documented = await documentedNames(options);
+  return {
+    library: name,
+    from,
+    to,
+    added,
+    removed,
+    undocumented: added.filter((symbol) => !documented.has(symbol)),
+  };
+}
+
+/** Every identifier any documentation package installed here mentions. */
+async function documentedNames(options: ChangedOptions): Promise<Set<string>> {
+  const names = new Set<string>();
+  for (const pkg of (await discoverPackages(options.cwd)).packages) {
+    for (const chunk of options.store.chunksOf(pkg.id)) {
+      for (const identifier of identifiers(chunk.content)) names.add(identifier);
+    }
+  }
+  return names;
+}
+
+/** `hono@4.0.0` into its parts, keeping a scope's leading `@` out of it. */
+function split(spec: string): [name: string, version: string | undefined] {
+  const at = spec.lastIndexOf("@");
+  if (at <= 0) return [spec, undefined];
+  return [spec.slice(0, at), spec.slice(at + 1)];
+}