docspack 1.0.0 → 1.2.0

package/src/build.ts

@@ -1,7 +1,16 @@
 import { createHash } from "node:crypto";
 import type { Dirent } from "node:fs";
 import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { join, relative, sep } from "node:path";
+import { basename, join, relative, sep } from "node:path";
+import { readApiFile } from "@docspack/lapis/read";
+import {
+  type ApiDocument,
+  OpenApiError,
+  operationDigest,
+  operationEntities,
+  operationTags,
+  operationTitle,
+} from "@docspack/openapi";
 import { findEntry } from "@docspack/registry";
 import { readBuildConfig } from "./config.js";
 import { cleanDocument } from "./document.js";
@@ -34,6 +43,17 @@ export interface BuildOptions {
   readonly openapi?: string;
   /** Registry id or llms.txt URL to fetch and package. */
   readonly source?: string;
+  /**
+   * JSON records to package, or `-` for standard input. One chunk per record, so anything a query
+   * can produce rows for — a table, an export, an API response — reaches the index without this
+   * tool needing a driver for it.
+   */
+  readonly json?: string;
+  /**
+   * Build a working corpus rather than a publishable package: identity is derived instead of
+   * demanded, and no `package.json` is written.
+   */
+  readonly local?: boolean;
   readonly pages?: number;
   readonly maxChunkTokens?: number;
   /**
@@ -59,6 +79,16 @@ export interface BuildResult {
 
 /** A document before it is split into chunks. */
 interface SourceDocument {
+  /**
+   * Chunk id to use instead of one derived from the title. Atomic documents only.
+   *
+   * A derived id is a slug of the prose, which is right for a Markdown page — the heading is the
+   * stable thing about it. It is wrong for a generated reference, where the title carries a summary
+   * somebody will reword: `POST /v1/charges — Create a charge` becoming `…— Creates a charge`
+   * renames the chunk, and a chunk id is what `feedback add --chunk` pins and what a published link
+   * resolves to.
+   */
+  readonly id?: string;
   readonly title: string;
   /** File path or URL the text came from, recorded in the chunk for provenance. */
   readonly origin: string;
@@ -72,8 +102,8 @@ interface SourceDocument {
 }
 
 const DEFAULT_MAX_CHUNK_TOKENS = 800;
-const MARKDOWN = /\.(md|mdx|markdown)$/i;
-const HTTP_METHODS = ["get", "post", "put", "patch", "delete", "head", "options"] as const;
+/** The source extensions a corpus is collected from. */
+export const MARKDOWN = /\.(md|mdx|markdown)$/i;
 
 /**
  * Generates a docs package: `.llms/manifest.json`, `.llms/chunks/*.md` and an `llms.txt` table of
@@ -203,6 +233,16 @@ async function resolveIdentity(
   const version =
     options.version ?? (typeof existing.version === "string" ? existing.version : undefined);
 
+  // A working corpus has no publisher to name it and no release to version, so demanding both is
+  // friction with nothing on the other side of it. A publishable package still fails below.
+  if (options.local === true) {
+    return {
+      name: name ?? localPackageName(options),
+      version: version ?? LOCAL_VERSION,
+      createPackageJson: false,
+    };
+  }
+
   if (name === undefined) {
     throw new DocspackError("The package needs a name", {
       hint: "Pass --name @vendor/docspack, or run inside a directory that has a package.json.",
@@ -215,17 +255,134 @@ async function resolveIdentity(
   return { name, version, createPackageJson: !found };
 }
 
+/**
+ * A stable version for a working corpus, rather than a hash of its contents.
+ *
+ * A content hash would look like the careful choice and is the wrong one: it changes the store key
+ * on every edit, so each save leaves the previous index behind as garbage that nothing evicts.
+ * Freshness is a property of each source, tracked per source, not of the corpus's name.
+ */
+export const LOCAL_VERSION = "0.0.0";
+
+/** Named after where the text came from, which is the only thing that distinguishes one corpus. */
+export function localPackageName(options: Pick<BuildOptions, "from" | "json">): string {
+  const source =
+    options.from ?? (options.json !== undefined && options.json !== "-" ? options.json : undefined);
+  const base = source === undefined ? "corpus" : basename(source).replace(/\.[^.]+$/, "");
+  return `@local/${slugify(base, "corpus")}`;
+}
+
+/**
+ * Records as JSON, from a file or standard input: `[{ "id", "title", "text", "tags", "entities" }]`.
+ *
+ * This is the whole of the non-file input, and it is deliberately not a database adapter. A driver
+ * would put a native dependency into a CLI whose startup cost is the reason it is usable from an
+ * agent at all, and every store worth indexing can already emit JSON — `psql --json`,
+ * `sqlite3 -json`, an API response saved to a file. The conversion belongs to whoever owns the
+ * query.
+ *
+ * A record carrying an `id` is atomic: its id is its key, and splitting it would either duplicate
+ * that key across sections or discard it. A record without one is split like any other document.
+ */
+async function collectFromJson(source: string): Promise<SourceDocument[]> {
+  const origin = source === "-" ? "stdin" : source;
+  const raw = source === "-" ? await readStdin() : await readFile(source, "utf8");
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    throw new DocspackError(`${origin} is not valid JSON`, {
+      hint: 'Expected an array of records: [{ "title": "…", "text": "…" }].',
+      cause: error,
+    });
+  }
+  if (!Array.isArray(parsed)) {
+    throw new DocspackError(`${origin} is not a JSON array`, {
+      hint: 'Expected [{ "title": "…", "text": "…" }], one entry per record.',
+    });
+  }
+  if (parsed.length === 0) {
+    throw new DocspackError(`${origin} contains no records`, {
+      hint: "A query that returned nothing indexes nothing; check the query first.",
+    });
+  }
+
+  return parsed.map((entry, index): SourceDocument => {
+    const where = `${origin} record ${index}`;
+    if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
+      throw new DocspackError(`${where} is not an object`);
+    }
+    const record = entry as Record<string, unknown>;
+    const title = field(record.title, `${where}.title`);
+    const text = field(record.text, `${where}.text`);
+    const id = record.id === undefined ? undefined : field(record.id, `${where}.id`);
+
+    return {
+      title,
+      text,
+      origin,
+      ...(id === undefined ? {} : { id: slugify(id, `record-${index}`), atomic: true }),
+      ...(record.tags === undefined ? {} : { tags: stringList(record.tags, `${where}.tags`) }),
+      ...(record.entities === undefined
+        ? {}
+        : { entities: stringList(record.entities, `${where}.entities`) }),
+    };
+  });
+}
+
+function field(value: unknown, where: string): string {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new DocspackError(`${where} must be a non-empty string`);
+  }
+  return value;
+}
+
+function stringList(value: unknown, where: string): string[] {
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
+    throw new DocspackError(`${where} must be an array of strings`);
+  }
+  return value as string[];
+}
+
+async function readStdin(): Promise<string> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
+  return Buffer.concat(chunks).toString("utf8");
+}
+
+/**
+ * Prose and an API document, in that order, or a mirror on its own.
+ *
+ * `--from` and `--openapi` combine, because a library with an HTTP API usually has both and
+ * documents them in the same package — refusing the combination meant a vendor had to pick which
+ * half of their documentation to publish. A mirror does not combine with either: it is somebody
+ * else's whole documentation set, fetched, and merging our own prose into it would produce a package
+ * claiming to be a mirror of something it is not.
+ *
+ * Operations come first so that their ids win a collision. A prose chunk's id is derived and may
+ * take a numeric suffix without anything breaking; an operation's is pinned to its endpoint, and a
+ * suffix there is the stable address moving.
+ */
 async function collect(options: BuildOptions, warnings: string[]): Promise<SourceDocument[]> {
-  const chosen = [options.from, options.openapi, options.source].filter(
-    (value) => value !== undefined,
-  );
-  if (chosen.length > 1) {
-    throw new DocspackError("Choose one input: --from, --openapi or a source");
+  if (options.source !== undefined) {
+    if (options.from !== undefined || options.openapi !== undefined || options.json !== undefined) {
+      throw new DocspackError("A source cannot be combined with --from, --openapi or --from-json", {
+        hint: "A mirror is a whole documentation set on its own.",
+      });
+    }
+    return collectFromSource(options, warnings);
   }
 
-  if (options.openapi !== undefined) return collectFromOpenApi(options.openapi);
-  if (options.source !== undefined) return collectFromSource(options, warnings);
-  return collectFromDirectory(options.from ?? join(options.out, "docs"));
+  const documents: SourceDocument[] = [];
+  if (options.openapi !== undefined) documents.push(...(await collectFromOpenApi(options.openapi)));
+  if (options.json !== undefined) documents.push(...(await collectFromJson(options.json)));
+  // The default `docs/` directory is only read when nothing else was named: a build given only an
+  // OpenAPI document or a set of records should not fail on a `docs/` directory that does not exist.
+  if (options.from !== undefined || (options.openapi === undefined && options.json === undefined)) {
+    documents.push(...(await collectFromDirectory(options.from ?? join(options.out, "docs"))));
+  }
+  return documents;
 }
 
 async function collectFromDirectory(dir: string): Promise<SourceDocument[]> {
@@ -333,121 +490,69 @@ async function collectFromSource(
   return documents;
 }
 
+/**
+ * One chunk per operation, plus the document's own overview.
+ *
+ * What each chunk carries is `@docspack/openapi`'s digest: the base URL, the credential, the inputs
+ * with their types, the body shape, the response, the failures, the types it reaches, and a runnable
+ * cURL call — in LAPIS notation, which is an open format for describing an API to a model. That
+ * package's README has the measurement; the short version is that a digest is about 72% smaller than
+ * the smallest correct slice of OpenAPI JSON for the same operation, and a rounding error against
+ * the whole document, which is what an agent without retrieval is handed instead.
+ *
+ * This is deliberately not a summary. An agent that retrieved the chunk for `POST /v1/charges` can
+ * make the call from it without reading anything else, which is the only test a reference chunk has
+ * to pass.
+ */
 async function collectFromOpenApi(file: string): Promise<SourceDocument[]> {
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(await readFile(file, "utf8"));
-  } catch (error) {
-    throw new DocspackError(`Could not read the OpenAPI document ${file}`, {
-      hint: "Only JSON is supported. Convert YAML with a tool such as `yq -o json`.",
-      cause: error,
-    });
-  }
-
-  const spec = parsed as {
-    info?: { title?: unknown; description?: unknown; version?: unknown };
-    paths?: Record<string, unknown>;
-  };
+  // JSON or YAML, decided by what the file contains rather than by its extension. The reader is a
+  // separate entry point of `@docspack/lapis` so that only the callers who need a YAML parser load
+  // one; this is a CLI, so it is one of them.
+  const document = await readOpenApi(file);
   const documents: SourceDocument[] = [];
-  const apiTitle = typeof spec.info?.title === "string" ? spec.info.title : "API";
 
-  if (typeof spec.info?.description === "string" && spec.info.description.trim().length > 0) {
+  if (document.description.length > 0) {
     documents.push({
-      title: `${apiTitle} overview`,
+      id: "api-overview",
+      title: `${document.title} overview`,
       origin: file,
-      text: spec.info.description.trim(),
+      text: document.description,
       atomic: true,
-      tags: ["overview", "introduction"],
+      tags: ["overview", "introduction", "api"],
     });
   }
 
-  for (const [path, item] of Object.entries(spec.paths ?? {})) {
-    if (typeof item !== "object" || item === null) continue;
-    const pathItem = item as Record<string, unknown>;
-
-    for (const method of HTTP_METHODS) {
-      const operation = pathItem[method];
-      if (typeof operation !== "object" || operation === null) continue;
-      documents.push(renderOperation(method, path, operation as Record<string, unknown>, file));
-    }
-  }
-
-  if (documents.length === 0) {
-    throw new DocspackError(`No operations found in ${file}`, {
-      hint: "The document needs a `paths` object with at least one operation.",
+  for (const operation of document.operations) {
+    documents.push({
+      // The endpoint, not the operation id: a vendor renames `createCharge` to `postCharges` and
+      // the endpoint is still `POST /v1/charges`. Both are recorded as entities, so either spelling
+      // still finds the chunk.
+      id: slugify(`${operation.method} ${operation.path}`, "operation"),
+      title: operationTitle(operation),
+      origin: `${file}#${operation.method} ${operation.path}`,
+      text: operationDigest(document, operation),
+      atomic: true,
+      tags: operationTags(operation),
+      entities: operationEntities(operation),
     });
   }
+
   return documents;
 }
 
-function renderOperation(
-  method: string,
-  path: string,
-  operation: Record<string, unknown>,
-  file: string,
-): SourceDocument {
-  const summary = typeof operation.summary === "string" ? operation.summary : "";
-  const description = typeof operation.description === "string" ? operation.description : "";
-  const operationId = typeof operation.operationId === "string" ? operation.operationId : undefined;
-  const tags = Array.isArray(operation.tags)
-    ? operation.tags.filter((tag): tag is string => typeof tag === "string")
-    : [];
-
-  const lines = [`${method.toUpperCase()} ${path}`, ""];
-  if (summary.length > 0) lines.push(summary, "");
-  if (description.length > 0) lines.push(description, "");
-  if (operationId !== undefined) lines.push(`Operation: \`${operationId}\``, "");
-
-  const parameters = Array.isArray(operation.parameters) ? operation.parameters : [];
-  if (parameters.length > 0) {
-    lines.push("## Parameters", "");
-    for (const raw of parameters) {
-      if (typeof raw !== "object" || raw === null) continue;
-      const parameter = raw as {
-        name?: unknown;
-        in?: unknown;
-        required?: unknown;
-        description?: unknown;
-      };
-      if (typeof parameter.name !== "string") continue;
-      const location = typeof parameter.in === "string" ? parameter.in : "query";
-      const required = parameter.required === true ? ", required" : "";
-      const note =
-        typeof parameter.description === "string" && parameter.description.length > 0
-          ? ` — ${parameter.description}`
-          : "";
-      lines.push(`- \`${parameter.name}\` (${location}${required})${note}`);
-    }
-    lines.push("");
-  }
-
-  const responses = operation.responses;
-  if (typeof responses === "object" && responses !== null) {
-    lines.push("## Responses", "");
-    for (const [status, raw] of Object.entries(responses as Record<string, unknown>)) {
-      const detail =
-        typeof raw === "object" &&
-        raw !== null &&
-        typeof (raw as { description?: unknown }).description === "string"
-          ? ` — ${(raw as { description: string }).description}`
-          : "";
-      lines.push(`- \`${status}\`${detail}`);
+/** The reader's failures, re-reported as the CLI's own error so the hint survives. */
+async function readOpenApi(file: string): Promise<ApiDocument> {
+  try {
+    return (await readApiFile(file)).document;
+  } catch (error) {
+    if (error instanceof OpenApiError) {
+      throw new DocspackError(error.message, {
+        ...(error.hint === undefined ? {} : { hint: error.hint }),
+        cause: error,
+      });
     }
-    lines.push("");
+    throw error;
   }
-
-  return {
-    title: summary.length > 0 ? summary : `${method.toUpperCase()} ${path}`,
-    origin: `${file}#${method} ${path}`,
-    text: lines.join("\n").trim(),
-    atomic: true,
-    tags: [
-      method,
-      ...tags,
-      ...path.split("/").filter((part) => part.length > 0 && !part.startsWith("{")),
-    ],
-    ...(operationId === undefined ? {} : { entities: [operationId] }),
-  };
 }
 
 function chunkDocuments(
@@ -474,7 +579,12 @@ function chunkDocuments(
     for (const section of sections) {
       const directives = readDirectives(section.body);
       const body = withoutDuplicateHeading(directives.body, section.heading);
-      const derived = chunkSlug(document.title, section.heading);
+      // Only for an atomic document: a pinned id on a document that splits would give every
+      // section the same id.
+      const derived =
+        document.atomic === true && document.id !== undefined
+          ? document.id
+          : chunkSlug(document.title, section.heading);
       const id = uniqueId(derived, taken);
       if (id !== derived) collisions.push(derived);
       const libraries = directives.documents.length > 0 ? directives.documents : document.documents;

package/src/cli.ts

@@ -6,7 +6,7 @@ import { parseArgs } from "node:util";
 import { applyAgentSetup, planAgentSetup } from "./agent.js";
 import { changedSurface } from "./changed.js";
 import { measureCoverage } from "./coverage.js";
-import { defaultStorePath, Store, silenceSqliteWarning } from "./db.js";
+import { defaultStorePath, localStorePath, Store, silenceSqliteWarning } from "./db.js";
 import { discoverPackages } from "./discovery.js";
 import { type DoctorReport, runDoctor } from "./doctor.js";
 import { DocspackError } from "./errors.js";
@@ -40,6 +40,7 @@ const OPTIONS = {
   "max-tokens": { type: "string" },
   all: { type: "boolean" },
   from: { type: "string" },
+  "from-json": { type: "string" },
   openapi: { type: "string" },
   out: { type: "string" },
   name: { type: "string" },
@@ -47,6 +48,7 @@ const OPTIONS = {
   pages: { type: "string" },
   "max-chunk-tokens": { type: "string" },
   "min-chunk-tokens": { type: "string" },
+  local: { type: "boolean" },
   documents: { type: "string", multiple: true },
   "min-hit-rate": { type: "string" },
   mirror: { type: "string" },
@@ -164,6 +166,11 @@ function openStore(values: Values): Store {
   return Store.open(values.store ?? defaultStorePath());
 }
 
+/** The project's own corpus, which is never the machine-wide store unless asked for by path. */
+function openLocalStore(values: Values, cwd: string): Store {
+  return Store.open(values.store ?? localStorePath(cwd));
+}
+
 async function main(argv: readonly string[]): Promise<number> {
   let values: Values;
   let positionals: string[];
@@ -302,6 +309,83 @@ async function main(argv: readonly string[]): Promise<number> {
       }
     }
 
+    case "index": {
+      const { indexLocal } = await import("./local.js");
+      const store = openLocalStore(values, cwd);
+      try {
+        const result = await indexLocal({
+          cwd,
+          store,
+          ...(values.from === undefined ? {} : { from: values.from }),
+          ...(values["from-json"] === undefined ? {} : { json: values["from-json"] }),
+          ...(values.name === undefined ? {} : { name: values.name }),
+          ...(values.force === true ? { force: true } : {}),
+          ...(quiet || json
+            ? {}
+            : {
+                onProgress: (message: string): void => {
+                  process.stderr.write(`${dim(message)}\n`);
+                },
+              }),
+        });
+
+        if (json) {
+          process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+          return 0;
+        }
+        const state =
+          result.status === "unchanged"
+            ? dim("unchanged")
+            : `${String(result.chunks)} chunks  ${dim(`~${String(result.tokens)} tokens`)}`;
+        process.stdout.write(`${green("+")} ${bold(result.name)}  ${state}\n`);
+        for (const warning of result.warnings) process.stderr.write(`${yellow("!")} ${warning}\n`);
+        if (!quiet) {
+          process.stdout.write(
+            result.streamed
+              ? `\nIndexed from a stream, so nothing can tell later whether it is still current.\nAsk it with \`docspack recall "<question>"\`.\n`
+              : `\nAsk it with \`docspack recall "<question>"\`. Re-run this after editing the sources.\n`,
+          );
+        }
+        return 0;
+      } finally {
+        store.close();
+      }
+    }
+
+    case "recall": {
+      const question = rest.join(" ");
+      if (question.length === 0) {
+        throw new DocspackError('Usage: docspack recall "<question>"');
+      }
+
+      const { recallLocal, renderRecall } = await import("./local.js");
+      const store = openLocalStore(values, cwd);
+      try {
+        const result = await recallLocal({
+          cwd,
+          store,
+          query: question,
+          ...(() => {
+            const limit = integer(values.limit, "--limit");
+            return limit === undefined ? {} : { limit };
+          })(),
+          ...(() => {
+            const maxTokens = integer(values["max-tokens"], "--max-tokens");
+            return maxTokens === undefined ? {} : { maxTokens };
+          })(),
+        });
+
+        if (json) {
+          process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        } else {
+          process.stdout.write(`${renderRecall(result, question)}\n`);
+        }
+        return result.hits.length === 0 ? 1 : 0;
+      } finally {
+        store.close();
+      }
+    }
+
     case "search": {
       const query = rest.join(" ");
       if (query.length === 0) throw new DocspackError("Usage: docspack search <query>");
@@ -348,6 +432,11 @@ async function main(argv: readonly string[]): Promise<number> {
           }
           process.stdout.write("\n");
         }
+        if (result.endpoints.length > 0) {
+          // Which hits were addressed rather than ranked. A reader who asked about a concrete URL
+          // and got a chunk about a template path needs to see that the match was the template.
+          process.stdout.write(dim(`matched by endpoint: ${result.endpoints.join(", ")}\n`));
+        }
         process.stdout.write(
           dim(`${result.tokens} tokens across ${plural(result.hits.length, "chunk")}\n`),
         );
@@ -762,7 +851,9 @@ async function main(argv: readonly string[]): Promise<number> {
         out: values.out ?? cwd,
         ...(rest[0] === undefined ? {} : { source: rest[0] }),
         ...(values.from === undefined ? {} : { from: values.from }),
+        ...(values["from-json"] === undefined ? {} : { json: values["from-json"] }),
         ...(values.openapi === undefined ? {} : { openapi: values.openapi }),
+        ...(values.local === true ? { local: true } : {}),
         ...(values.name === undefined ? {} : { name: values.name }),
         ...(values["pkg-version"] === undefined ? {} : { version: values["pkg-version"] }),
         ...(() => {
@@ -796,8 +887,11 @@ async function main(argv: readonly string[]): Promise<number> {
       );
       for (const warning of result.warnings) process.stderr.write(`${yellow("!")} ${warning}\n`);
       if (!quiet) {
+        // A local build is not going to npm, so the publishing instruction would be wrong advice.
         process.stdout.write(
-          `\nWrote ${result.dir}/.llms and ${result.dir}/llms.txt.\nPublish with \`npm publish\`, then add it to a project and run \`docspack sync\`.\n`,
+          values.local === true
+            ? `\nWrote ${result.dir}/.llms and ${result.dir}/llms.txt.\nTo index a corpus into this project instead, use \`docspack index\`.\n`
+            : `\nWrote ${result.dir}/.llms and ${result.dir}/llms.txt.\nPublish with \`npm publish\`, then add it to a project and run \`docspack sync\`.\n`,
         );
       }
       return 0;