docspack 1.1.0 → 1.3.0

package/src/cli.ts

@@ -3,14 +3,16 @@ import { readFileSync } from "node:fs";
 import { posix } from "node:path";
 import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
+import { parseArgsConfig, type ResolvedCommand, resolveDocument } from "@docspack/cmdspec";
 import { applyAgentSetup, planAgentSetup } from "./agent.js";
 import { changedSurface } from "./changed.js";
+import { CLI } from "./cli-spec.js";
 import { measureCoverage } from "./coverage.js";
 import { defaultStorePath, localStorePath, Store, silenceSqliteWarning } from "./db.js";
 import { discoverPackages } from "./discovery.js";
 import { type DoctorReport, runDoctor } from "./doctor.js";
 import { DocspackError } from "./errors.js";
-import { commandHelpText, findCommandHelp, globalHelpText } from "./help.js";
+import { cliDocument, commandHelpText, findCommandHelp, globalHelpText } from "./help.js";
 import { renderTree } from "./init/write.js";
 import { previewPackage } from "./preview.js";
 import { type QueryResult, queryDocs, renderAnswer } from "./search.js";
@@ -22,11 +24,16 @@ import { verifyProject } from "./verify.js";
 // answer a query, so the commands that need them import them at the point of use. Every
 // `docspack ask` pays for what it uses and nothing else.
 
-const OPTIONS = {
-  help: { type: "boolean", short: "h" },
-  version: { type: "boolean", short: "v" },
+/**
+ * What each option's value is, by the key `parseArgs` reports it under — for the type of `values`
+ * only. Which command accepts which option is the CLI's cmdspec document (`cli-spec.ts`), and
+ * `tests/cli-spec.test.ts` fails if this map and that document disagree about a key or a type.
+ */
+export const OPTION_TYPES = {
+  help: { type: "boolean" },
+  version: { type: "boolean" },
   json: { type: "boolean" },
-  quiet: { type: "boolean", short: "q" },
+  quiet: { type: "boolean" },
   cwd: { type: "string" },
   store: { type: "string" },
   force: { type: "boolean" },
@@ -35,13 +42,14 @@ const OPTIONS = {
   hooks: { type: "boolean" },
   mcp: { type: "boolean" },
   feedback: { type: "boolean" },
-  package: { type: "string", short: "p" },
+  package: { type: "string" },
   limit: { type: "string" },
   "max-tokens": { type: "string" },
   all: { type: "boolean" },
   from: { type: "string" },
   "from-json": { type: "string" },
   openapi: { type: "string" },
+  cmdspec: { type: "string" },
   out: { type: "string" },
   name: { type: "string" },
   "pkg-version": { type: "string" },
@@ -56,9 +64,8 @@ const OPTIONS = {
   template: { type: "string" },
   community: { type: "boolean" },
   workflow: { type: "boolean" },
-  "no-workflow": { type: "boolean" },
-  "no-build": { type: "boolean" },
-  yes: { type: "boolean", short: "y" },
+  build: { type: "boolean" },
+  yes: { type: "boolean" },
   "dry-run": { type: "boolean" },
   strict: { type: "boolean" },
   pedantic: { type: "boolean" },
@@ -82,13 +89,55 @@ const EXIT_NO_MATCH = 4;
 const HELP = globalHelpText(defaultStorePath(), EXIT_NOT_INDEXED, EXIT_NO_MATCH);
 
 type Values = {
-  [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K] extends { multiple: true }
+  [K in keyof typeof OPTION_TYPES]?: (typeof OPTION_TYPES)[K] extends { multiple: true }
     ? string[]
-    : (typeof OPTIONS)[K]["type"] extends "boolean"
+    : (typeof OPTION_TYPES)[K]["type"] extends "boolean"
       ? boolean
       : string;
 };
 
+const SPEC = resolveDocument(CLI);
+
+/**
+ * The command an argument list invokes: its leading words, walked down the CLI's command tree.
+ * Options before or between the words are skipped, along with the value of any that takes one,
+ * so `docspack --store ./s.db agent install` finds `agent install`.
+ */
+function invokedCommand(argv: readonly string[]): ResolvedCommand {
+  let current = SPEC.commands[0] as ResolvedCommand;
+  let skipValue = false;
+  for (const token of argv) {
+    if (skipValue) {
+      skipValue = false;
+      continue;
+    }
+    if (token === "--") break;
+    if (token.startsWith("-")) {
+      const spelling = token.split("=")[0] ?? token;
+      const option = [...current.inherited, ...current.options].find(
+        (candidate) => candidate.name === spelling || (candidate.aliases ?? []).includes(spelling),
+      );
+      skipValue = !token.includes("=") && option?.value !== undefined;
+      continue;
+    }
+    const next = SPEC.commands.find(
+      (candidate) =>
+        candidate.path.length === current.path.length + 1 &&
+        candidate.path[candidate.path.length - 1] === token &&
+        candidate.path.slice(0, -1).join(" ") === current.path.join(" "),
+    );
+    if (next === undefined) break;
+    current = next;
+  }
+  return current;
+}
+
+/** The help page for a command, or the global one for the root and for `help`. */
+function helpFor(command: ResolvedCommand): string {
+  const subject = command.path.length > 1 ? findCommandHelp(command.path[1] as string) : undefined;
+  return subject === undefined ? HELP : commandHelpText(subject);
+}
+
 const color = process.env.NO_COLOR === undefined && process.stdout.isTTY === true;
 const paint = (code: string, text: string): string =>
   color ? `\u001b[${code}m${text}\u001b[0m` : text;
@@ -172,20 +221,35 @@ function openLocalStore(values: Values, cwd: string): Store {
 }
 
 async function main(argv: readonly string[]): Promise<number> {
+  // Each command is parsed with the options its document gives it — its own and those it
+  // inherits — so a flag another command reads is an error here rather than silently ignored.
+  const invoked = invokedCommand(argv);
+  const config = parseArgsConfig(invoked);
   let values: Values;
   let positionals: string[];
   try {
-    const parsed = parseArgs({ args: [...argv], options: OPTIONS, allowPositionals: true });
-    values = parsed.values;
+    const parsed = parseArgs({
+      args: [...argv],
+      options: config.options,
+      allowPositionals: true,
+      allowNegative: config.allowNegative,
+      strict: true,
+    });
+    values = parsed.values as Values;
     positionals = parsed.positionals;
   } catch (error) {
     process.stderr.write(
       `${red("error")} ${error instanceof Error ? error.message : String(error)}\n\n`,
     );
-    process.stderr.write(HELP);
+    process.stderr.write(helpFor(invoked));
     return 2;
   }
 
+  // The root's `--cmdspec` is a switch; `build`'s takes a file. Only the root's prints.
+  if (invoked.path.length === 1 && (values as { cmdspec?: unknown }).cmdspec === true) {
+    process.stdout.write(`${JSON.stringify(cliDocument(version()), null, 2)}\n`);
+    return 0;
+  }
   if (values.version === true) {
     process.stdout.write(`${version()}\n`);
     return 0;
@@ -242,7 +306,12 @@ async function main(argv: readonly string[]): Promise<number> {
           const trust = pkg.trusted ? "" : ` ${yellow("(community)")}`;
           // Declarations read from an installed build are not documentation somebody wrote, and
           // the listing says so rather than letting them pass for it.
-          const kind = pkg.kind === "artifact" ? ` ${dim("(declarations)")}` : "";
+          const kind =
+            pkg.kind === "artifact"
+              ? ` ${dim("(declarations)")}`
+              : pkg.kind === "cli"
+                ? ` ${dim("(command line)")}`
+                : "";
           process.stdout.write(
             `${mark} ${bold(pkg.id)}  ${pkg.chunks} chunks  ${dim(pkg.status)}${trust}${kind}\n`,
           );
@@ -437,6 +506,9 @@ async function main(argv: readonly string[]): Promise<number> {
           // and got a chunk about a template path needs to see that the match was the template.
           process.stdout.write(dim(`matched by endpoint: ${result.endpoints.join(", ")}\n`));
         }
+        if (result.commands.length > 0) {
+          process.stdout.write(dim(`matched by command: ${result.commands.join(", ")}\n`));
+        }
         process.stdout.write(
           dim(`${result.tokens} tokens across ${plural(result.hits.length, "chunk")}\n`),
         );
@@ -853,6 +925,7 @@ async function main(argv: readonly string[]): Promise<number> {
         ...(values.from === undefined ? {} : { from: values.from }),
         ...(values["from-json"] === undefined ? {} : { json: values["from-json"] }),
         ...(values.openapi === undefined ? {} : { openapi: values.openapi }),
+        ...(values.cmdspec === undefined ? {} : { cmdspec: values.cmdspec }),
         ...(values.local === true ? { local: true } : {}),
         ...(values.name === undefined ? {} : { name: values.name }),
         ...(values["pkg-version"] === undefined ? {} : { version: values["pkg-version"] }),
@@ -914,11 +987,11 @@ async function main(argv: readonly string[]): Promise<number> {
           ? {}
           : { template: values.template as "full" | "minimal" }),
         ...(values.community === true ? { community: true } : {}),
-        workflow: values["no-workflow"] === true ? false : values.workflow !== false,
+        workflow: values.workflow !== false,
         ...(values.yes === true || json ? { yes: true } : {}),
         ...(values["dry-run"] === true ? { dryRun: true } : {}),
         ...(values.force === true ? { force: true } : {}),
-        ...(values["no-build"] === true ? { build: false } : {}),
+        ...(values.build === false ? { build: false } : {}),
         ...(quiet || json
           ? {}
           : {

package/src/commands.ts

@@ -0,0 +1,298 @@
+import {
+  type CmdspecDocument,
+  commandDigest,
+  commandMap,
+  commandPath,
+  commandSummary,
+  programName,
+  type ResolvedCommand,
+  type ResolvedDocument,
+  resolveDocument,
+} from "@docspack/cmdspec";
+import { estimateTokens } from "./spec.js";
+
+/**
+ * A command-line interface as chunks, and a question that names a command answered by address.
+ *
+ * The same two halves `build --openapi` and `endpoints.ts` are for an HTTP API, applied to a CLI
+ * described in cmdspec (`packages/cmdspec`, `docs/design/cmdspec.md`): one chunk per command
+ * carrying `@docspack/cmdspec`'s digest — synopsis, inputs with their types and defaults, effects,
+ * outputs, exit statuses, examples — and the command's path recorded as an entity, so that
+ * `docspack ask "git remote add"` pins that chunk rather than ranking every chunk that says
+ * "remote".
+ */
+
+/**
+ * How a chunk entity says it is a command path: `$ git remote add`. A shell prompt, because no
+ * identifier or endpoint can start with one, so the three kinds of entity never collide.
+ */
+export const COMMAND_ENTITY = "$ ";
+
+/**
+ * How many commands one answer may pin. Two, like endpoints: a question names one command,
+ * occasionally two ("init, then build"), and past that the names are incidental.
+ */
+export const MAX_COMMANDS = 2;
+
+/**
+ * The most the overview's command list may cost. Past it the overview lists the top-level
+ * commands only: mise's 290 commands, one line each, are about 8,000 tokens — more than two whole
+ * answers — and the full map is what `cmdspec map` is for.
+ */
+const OVERVIEW_MAP_TOKENS = 800;
+
+/** One chunk's worth of a CLI, before it is a file or a row. */
+export interface CommandChunk {
+  readonly id: string;
+  /** The command's path — `git remote add` — or undefined for the overview. */
+  readonly path?: string;
+  readonly title: string;
+  readonly text: string;
+  readonly tags: readonly string[];
+  readonly entities: readonly string[];
+}
+
+/**
+ * The chunks a CLI becomes: an overview, then one per command a caller runs.
+ *
+ * The overview carries the program's prose, the root's digest — the options every command
+ * accepts, with what they do, which a subcommand's digest only names — and the list of commands.
+ * The root has a chunk of its own only when it is itself a command a caller runs, as ffmpeg's is.
+ * A command that only groups others gets none unless it takes something its subcommands do not
+ * inherit, and a hidden command gets none for the reason it is hidden.
+ */
+export function commandChunks(document: CmdspecDocument): CommandChunk[] {
+  const resolved = resolveDocument(document);
+  const chunks: CommandChunk[] = [overview(resolved)];
+  for (const command of resolved.commands) {
+    if (!isChunked(command)) continue;
+    const path = commandPath(command);
+    const summary = commandSummary(command);
+    const prose = command.declared.description?.trim();
+    const text = [
+      prose === undefined || prose.length === 0 ? undefined : prose,
+      `\`\`\`text\n${commandDigest(command)}\n\`\`\``,
+    ]
+      .filter((part) => part !== undefined)
+      .join("\n\n");
+    chunks.push({
+      id: commandChunkId(command.path),
+      path,
+      title: summary === undefined ? path : `${path} — ${summary}`,
+      text,
+      tags: [
+        "cli",
+        "command",
+        ...command.path,
+        ...(command.declared.group === undefined ? [] : [command.declared.group]),
+      ],
+      entities: commandEntities(command),
+    });
+  }
+  return chunks;
+}
+
+/** A chunk id for a command path: `cli-git-remote-add`. Stable for as long as the path is. */
+export function commandChunkId(path: readonly string[]): string {
+  const slug = path
+    .join("-")
+    .toLowerCase()
+    .normalize("NFKD")
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 80)
+    .replace(/-+$/g, "");
+  return `cli-${slug.length > 0 ? slug : "command"}`;
+}
+
+/**
+ * The checks a CLI description can fail without being invalid — what makes its chunks answer
+ * worse, not what stops them existing. `build` prints them and `doctor` runs them against the
+ * package's configured `docspack.cmdspec`.
+ */
+export function commandFindings(document: CmdspecDocument): CommandFinding[] {
+  const resolved = resolveDocument(document);
+  const chunked = resolved.commands.filter(isChunked);
+  const findings: CommandFinding[] = [];
+
+  const unsummarized = chunked.filter((command) => commandSummary(command) === undefined);
+  if (unsummarized.length > 0) {
+    findings.push({
+      severity: "warn",
+      message: `${unsummarized.length} of ${chunked.length} commands have no summary or description: ${sample(unsummarized)}. The summary is the line a question is matched against.`,
+    });
+  }
+  // A command that only dispatches to its subcommands is run through them; it has no example or
+  // effect of its own to state.
+  const runnable = chunked.filter((command) => command.declared.subcommandRequired !== true);
+  const unexampled = runnable.filter((command) => (command.declared.examples ?? []).length === 0);
+  if (unexampled.length > 0) {
+    findings.push({
+      severity: "info",
+      message: `${unexampled.length} of ${runnable.length} commands have no example: ${sample(unexampled)}. An example is the line an agent copies.`,
+    });
+  }
+  if (chunked.every((command) => Object.keys(command.exits).length === 0)) {
+    findings.push({
+      severity: "info",
+      message:
+        "No exit status is described. Declare at least what 0, 1 and 2 mean on the root, which every command inherits.",
+    });
+  }
+  const unstated = runnable.filter((command) => command.declared.effects === undefined);
+  if (unstated.length > 0) {
+    findings.push({
+      severity: "info",
+      message: `${unstated.length} of ${runnable.length} commands do not state their effects: ${sample(unstated)}. An agent reads that as unknown, not as safe.`,
+    });
+  }
+  return findings;
+}
+
+/**
+ * Something a CLI description could say better. A missing summary is a warning — the summary is
+ * what a question is matched against — and the rest are notes.
+ */
+export interface CommandFinding {
+  readonly severity: "warn" | "info";
+  readonly message: string;
+}
+
+/** A command path the question names, and the chunk that describes it. */
+export interface CommandEntry {
+  readonly chunkId: string;
+  /** The path as written: `git remote add`. */
+  readonly path: string;
+}
+
+/** Collects the command paths a set of chunks declares, from their entities. */
+export function commandIndex(
+  chunks: Iterable<{ readonly chunkId: string; readonly entities: readonly string[] }>,
+): CommandEntry[] {
+  const entries: CommandEntry[] = [];
+  for (const chunk of chunks) {
+    for (const entity of chunk.entities) {
+      if (entity.startsWith(COMMAND_ENTITY)) {
+        entries.push({ chunkId: chunk.chunkId, path: entity.slice(COMMAND_ENTITY.length) });
+      }
+    }
+  }
+  return entries;
+}
+
+/**
+ * The commands a question names, in the order it names them.
+ *
+ * A path matches where its words appear in the question in order and next to each other, exactly
+ * as written — command names are case-sensitive, and `Git Remote` is prose. Where two paths match
+ * at the same place the longer wins, so `git remote add origin …` pins `git remote add`, not
+ * `git remote` as well.
+ */
+export function matchCommands(
+  query: string,
+  index: readonly CommandEntry[],
+  limit = MAX_COMMANDS,
+): CommandEntry[] {
+  if (index.length === 0) return [];
+  const words = query
+    .split(/\s+/)
+    .map((word) => word.replace(/^[`'"(]+|[`'".,;:!?)]+$/g, ""))
+    .filter((word) => word.length > 0);
+
+  const matches: { readonly entry: CommandEntry; readonly at: number; readonly length: number }[] =
+    [];
+  for (const entry of index) {
+    const path = entry.path.split(" ");
+    for (let at = 0; at + path.length <= words.length; at += 1) {
+      if (path.every((word, offset) => words[at + offset] === word)) {
+        matches.push({ entry, at, length: path.length });
+      }
+    }
+  }
+
+  const longest = matches.filter(
+    (match) =>
+      !matches.some(
+        (other) =>
+          other.at <= match.at &&
+          other.at + other.length >= match.at + match.length &&
+          other.length > match.length,
+      ),
+  );
+  longest.sort((a, b) => a.at - b.at);
+
+  const found: CommandEntry[] = [];
+  const seen = new Set<string>();
+  for (const { entry } of longest) {
+    if (found.length >= limit) break;
+    if (seen.has(entry.chunkId)) continue;
+    seen.add(entry.chunkId);
+    found.push(entry);
+  }
+  return found;
+}
+
+function overview(resolved: ResolvedDocument): CommandChunk {
+  const root = resolved.commands[0] as ResolvedCommand;
+  const { document } = resolved;
+  const program = programName(document);
+  const map = commandMap(resolved);
+  const topLevel = resolved.commands.filter(
+    (command) => command.path.length === 2 && !command.hidden,
+  );
+  const fits = estimateTokens(map) <= OVERVIEW_MAP_TOKENS;
+  const list = fits
+    ? map
+    : topLevel
+        .map((command) => `${commandPath(command)}  # ${commandSummary(command) ?? ""}`)
+        .join("\n");
+  const more = fits
+    ? ""
+    : `\n\nThe ${resolved.commands.length - 1} commands are too many to list here; ask for one by its path, such as \`${commandPath(topLevel[0] ?? (resolved.commands[0] as ResolvedCommand))}\`.`;
+  const prose = [document.info.summary, document.info.description]
+    .filter((part) => part !== undefined && part.trim().length > 0)
+    .join("\n\n");
+  return {
+    id: "cli",
+    title: `${program} ${document.info.version} — commands`,
+    text: [
+      prose.length > 0 ? prose : undefined,
+      isChunked(root) ? undefined : `\`\`\`text\n${commandDigest(root)}\n\`\`\``,
+      `Commands:\n\n\`\`\`text\n${list}\n\`\`\`${more}`,
+    ]
+      .filter((part) => part !== undefined)
+      .join("\n\n"),
+    tags: ["cli", "commands", "overview", program],
+    entities: [],
+  };
+}
+
+function isChunked(command: ResolvedCommand): boolean {
+  // Hidden itself or under a hidden command: mise's legacy `macos-defaults` group repeats the
+  // commands of `macos defaults`, and its subcommands do not each say they are hidden.
+  if (command.hidden) return false;
+  if (command.declared.subcommandRequired !== true) return true;
+  // The root's digest is in the overview, so a root that only dispatches needs nothing more.
+  if (command.path.length === 1) return false;
+  return (
+    command.arguments.length > 0 || command.options.some((option) => option.inherited !== true)
+  );
+}
+
+/** The command's path, and the path through each of its own aliases: `git remote rm`. */
+function commandEntities(command: ResolvedCommand): string[] {
+  const path = commandPath(command);
+  const parent = command.path.slice(0, -1).join(" ");
+  const aliases =
+    command.path.length > 1 && "aliases" in command.declared
+      ? ((command.declared as { readonly aliases?: readonly string[] }).aliases ?? [])
+      : [];
+  return [path, ...aliases.map((alias) => `${parent} ${alias}`)].map(
+    (entry) => `${COMMAND_ENTITY}${entry}`,
+  );
+}
+
+function sample(commands: readonly ResolvedCommand[]): string {
+  const shown = commands.slice(0, 3).map(commandPath).join(", ");
+  return commands.length > 3 ? `${shown} and ${commands.length - 3} more` : shown;
+}

package/src/config.ts

@@ -27,6 +27,7 @@ export interface FeedbackChannel {
 export interface BuildConfig {
   readonly from?: string;
   readonly openapi?: string;
+  readonly cmdspec?: string;
   readonly source?: string;
   /**
    * The libraries this package documents, each `name` or `name@version`. What `docspack verify`
@@ -69,7 +70,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
   }
   const raw = value as Record<string, unknown>;
 
-  const text = (key: "from" | "openapi" | "source"): string | undefined => {
+  const text = (key: "from" | "openapi" | "cmdspec" | "source"): string | undefined => {
     const found = raw[key];
     if (found === undefined) return undefined;
     if (typeof found !== "string" || found.length === 0) {
@@ -89,6 +90,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
 
   const from = text("from");
   const openapi = text("openapi");
+  const cmdspec = text("cmdspec");
   const source = text("source");
   const documents = parseDocuments(raw.documents);
   const feedback = parseFeedbackChannel(raw.feedback);
@@ -99,6 +101,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
   return {
     ...(from === undefined ? {} : { from }),
     ...(openapi === undefined ? {} : { openapi }),
+    ...(cmdspec === undefined ? {} : { cmdspec }),
     ...(source === undefined ? {} : { source }),
     ...(documents === undefined ? {} : { documents }),
     ...(feedback === undefined ? {} : { feedback }),

package/src/db.ts

@@ -16,7 +16,7 @@ const require = createRequire(import.meta.url);
  * author, and a working corpus is a copy that can go stale under the reader — so it is stored
  * rather than inferred from the name.
  */
-export type PackageKind = "docs" | "artifact" | "local";
+export type PackageKind = "docs" | "artifact" | "local" | "cli";
 
 /**
  * Reads the stored kind, defaulting an unrecognized one to `docs`.
@@ -26,7 +26,7 @@ export type PackageKind = "docs" | "artifact" | "local";
  * a local corpus would end up presented as published documentation.
  */
 export function toPackageKind(value: string): PackageKind {
-  return value === "artifact" || value === "local" ? value : "docs";
+  return value === "artifact" || value === "local" || value === "cli" ? value : "docs";
 }
 
 export interface IndexedPackage {
@@ -328,6 +328,13 @@ export class Store {
     return rows.map((row) => row.name);
   }
 
+  /** One package's names with the chunk each points at — a CLI's command paths, for one. */
+  symbolTable(packageId: string): { name: string; chunkId: string }[] {
+    return this.#db
+      .prepare("SELECT name, chunk_id AS chunkId FROM symbols WHERE package_id = ? ORDER BY name")
+      .all(packageId) as { name: string; chunkId: string }[];
+  }
+
   /** One chunk by its id, however it was found. */
   chunk(chunkId: string): SearchHit | undefined {
     const row = this.#db