docspack 1.1.0 → 1.3.0

package/src/preview.ts

@@ -1,5 +1,6 @@
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
+import { commandIndex, matchCommands } from "./commands.js";
 import { type IndexedChunk, type SearchHit, Store } from "./db.js";
 import { endpointIndex, matchEndpoints } from "./endpoints.js";
 import { DocspackError } from "./errors.js";
@@ -86,32 +87,32 @@ export async function previewPackage(options: PreviewOptions): Promise<PreviewRe
       ...(documents.length === 0 ? {} : { documents }),
     });
 
-    // The same endpoint pinning `queryDocs` does, because `preview` exists to show an author what an
+    // The same endpoint and command pinning `queryDocs` does, because `preview` exists to show an author what an
     // agent would receive. A preview that ranked where the real query path pins would send them
     // chasing a retrieval problem they do not have.
     const limit = options.limit ?? DEFAULT_LIMIT;
     const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
-    const pinned = matchEndpoints(
-      options.query,
-      endpointIndex(
-        manifest.chunks.map((chunk) => ({
-          chunkId: chunkId(id, chunk.id),
-          entities: chunk.entities,
-        })),
-      ),
-    )
-      .map((match) => store.chunk(match.chunkId))
+    const addressable = manifest.chunks.map((chunk) => ({
+      chunkId: chunkId(id, chunk.id),
+      entities: chunk.entities,
+    }));
+    const pinnedIds = [
+      ...matchEndpoints(options.query, endpointIndex(addressable)),
+      ...matchCommands(options.query, commandIndex(addressable)),
+    ].map((match) => match.chunkId);
+    const pinned = [...new Set(pinnedIds)]
+      .map((pinnedId) => store.chunk(pinnedId))
       .filter((chunk): chunk is SearchHit => chunk !== undefined)
       .map(describe);
     const spent = pinned.reduce((total, hit) => total + hit.tokens, 0);
-    const pinnedIds = new Set(pinned.map((hit) => hit.chunkId));
+    const pinnedSet = new Set(pinned.map((hit) => hit.chunkId));
 
     const ranked = store
       .search(options.query, {
         limit: Math.max(1, limit - pinned.length),
         maxTokens: Math.max(1, maxTokens - spent),
       })
-      .filter((hit) => !pinnedIds.has(hit.chunkId))
+      .filter((hit) => !pinnedSet.has(hit.chunkId))
       .map(describe);
 
     const hits = [...pinned, ...ranked];

package/src/search.ts

@@ -1,7 +1,13 @@
+import { COMMAND_ENTITY, type CommandEntry, commandIndex, matchCommands } from "./commands.js";
 import type { PackageKind, SearchHit, Store } from "./db.js";
-import { discoverLibraries, discoverPackages } from "./discovery.js";
+import {
+  type DiscoveredLibrary,
+  discoverCommandLines,
+  discoverLibraries,
+  discoverPackages,
+} from "./discovery.js";
 import { endpointIndex, matchEndpoints } from "./endpoints.js";
-import { chunkId, formatDocumentedLibrary, isCommunityPackage } from "./spec.js";
+import { CLI_SUFFIX, chunkId, formatDocumentedLibrary, isCommunityPackage } from "./spec.js";
 
 /** Ceiling on how much context one query may return, per the blueprint's context-exhaustion rule. */
 export const DEFAULT_MAX_TOKENS = 3000;
@@ -76,15 +82,27 @@ export interface QueryResult {
    * chunk, and has to be told that the match was against a template.
    */
   readonly endpoints: readonly string[];
+  /** Command paths the question named that a described command answers, pinned likewise. */
+  readonly commands: readonly string[];
 }
 
 export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
   const scoped = options.scoped !== false;
   const installed = scoped ? (await discoverPackages(options.cwd)).packages : undefined;
-  const packageIds = installed?.map((pkg) => pkg.id);
-  const unindexed = (installed ?? [])
-    .filter((pkg) => !options.store.hasPackage(pkg.id))
-    .map((pkg) => pkg.id);
+  // Read once: the libraries scope both the declarations and the CLIs a question can reach.
+  const libraries = scoped ? await discoverLibraries(options.cwd) : undefined;
+  const commandLines = scoped
+    ? (await discoverCommandLines(options.cwd, libraries)).map((cli) => cli.id)
+    : options.store
+        .listPackages()
+        .filter((pkg) => pkg.kind === "cli")
+        .map((pkg) => pkg.id);
+  const packageIds =
+    installed === undefined ? undefined : [...installed.map((pkg) => pkg.id), ...commandLines];
+  const unindexed = [
+    ...(installed ?? []).map((pkg) => pkg.id),
+    ...(scoped ? commandLines : []),
+  ].filter((id) => !options.store.hasPackage(id));
 
   // Read from the installed manifests rather than the index: the store is a cache of chunk text,
   // and adding a column to it would make every existing store need a rebuild to answer this.
@@ -107,9 +125,25 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
   const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
   const limit = options.limit ?? DEFAULT_LIMIT;
 
-  // Endpoints first, and the ranker is then asked for less: a pinned operation is the answer, and
-  // three ranked chunks beside it would spend the budget restating what it already says.
-  const { hits: pinned, endpoints } = findEndpoints(options, operations, documented, maxTokens);
+  // A described CLI's command paths: from a docs package's manifest entities, and from the name
+  // table of each CLI read out of an installed package by `sync`.
+  const commandEntries = [
+    ...commandIndex(operations),
+    ...commandLines.flatMap((id) =>
+      options.store
+        .symbolTable(id)
+        .filter((row) => row.name.startsWith(COMMAND_ENTITY))
+        .map((row) => ({ chunkId: row.chunkId, path: row.name.slice(COMMAND_ENTITY.length) })),
+    ),
+  ];
+
+  // Addressed chunks first, and the ranker is then asked for less: a pinned operation or command
+  // is the answer, and three ranked chunks beside it would spend the budget restating it.
+  const {
+    hits: pinned,
+    endpoints,
+    commands,
+  } = findAddressed(options, operations, commandEntries, documented, maxTokens);
   const spent = pinned.reduce((total, hit) => total + hit.tokens, 0);
   const pinnedIds = new Set(pinned.map((hit) => hit.chunkId));
 
@@ -123,8 +157,9 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
       maxTokens: Math.max(1, maxTokens - spent),
       // Declarations are addressed by name, never ranked against prose: a library exports far
       // more names than its documentation has pages, and ranking them together would answer
-      // every question with type machinery.
-      kinds: ["docs"],
+      // every question with type machinery. A CLI's command chunks are prose about what to type,
+      // one per command, and are ranked like the documentation they are.
+      kinds: ["docs", "cli"],
     })
     .filter((hit) => !pinnedIds.has(hit.chunkId))
     .map((hit) => toQueryHit(hit, "docs", documented));
@@ -132,7 +167,7 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
   const { declarations, undocumented } =
     options.artifacts === false
       ? { declarations: [] as QueryHit[], undocumented: [] as string[] }
-      : await findDeclarations(options, [...pinned, ...prose], maxTokens - spent);
+      : await findDeclarations(options, [...pinned, ...prose], maxTokens - spent, libraries);
 
   // Pinned operations, then declarations, then the ranked prose. Both of the first two were
   // addressed by name rather than guessed at, and for an agent about to write a call the exact
@@ -146,39 +181,55 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
     unindexed,
     undocumented,
     endpoints,
+    commands,
   };
 }
 
 /**
- * Operation chunks the question addressed.
+ * Operation and command chunks the question addressed — operations first, then commands.
  *
  * Bounded by the same token budget as everything else, and the first match is always kept: an answer
- * whose one pinned operation did not fit would have pinned nothing and said nothing about it.
+ * whose one pinned chunk did not fit would have pinned nothing and said nothing about it.
  */
-function findEndpoints(
+function findAddressed(
   options: QueryOptions,
   operations: readonly { chunkId: string; entities: readonly string[] }[],
+  commandEntries: readonly CommandEntry[],
   documented: ReadonlyMap<string, readonly string[]>,
   maxTokens: number,
-): { hits: QueryHit[]; endpoints: string[] } {
-  const matches = matchEndpoints(options.query, endpointIndex(operations));
+): { hits: QueryHit[]; endpoints: string[]; commands: string[] } {
+  const matches = [
+    ...matchEndpoints(options.query, endpointIndex(operations)).map((match) => ({
+      chunkId: match.chunkId,
+      endpoint: `${match.method} ${match.path}`,
+    })),
+    ...matchCommands(options.query, commandEntries).map((match) => ({
+      chunkId: match.chunkId,
+      command: match.path,
+    })),
+  ];
   const hits: QueryHit[] = [];
   const endpoints: string[] = [];
+  const commands: string[] = [];
+  const seen = new Set<string>();
   let spent = 0;
 
   for (const match of matches) {
+    if (seen.has(match.chunkId)) continue;
     if (options.packageFilter !== undefined && !match.chunkId.includes(options.packageFilter)) {
       continue;
     }
     const chunk = options.store.chunk(match.chunkId);
     if (chunk === undefined) continue;
     if (hits.length > 0 && spent + chunk.tokens > maxTokens) break;
+    seen.add(match.chunkId);
     hits.push(toQueryHit(chunk, "docs", documented));
-    endpoints.push(`${match.method} ${match.path}`);
+    if ("endpoint" in match) endpoints.push(match.endpoint);
+    else commands.push(match.command);
     spent += chunk.tokens;
   }
 
-  return { hits, endpoints };
+  return { hits, endpoints, commands };
 }
 
 /**
@@ -192,11 +243,12 @@ async function findDeclarations(
   options: QueryOptions,
   prose: readonly QueryHit[],
   maxTokens: number,
+  libraries: readonly DiscoveredLibrary[] | undefined,
 ): Promise<{ declarations: QueryHit[]; undocumented: string[] }> {
   const scope =
     options.scoped === false
       ? undefined
-      : (await discoverLibraries(options.cwd)).map(
+      : (libraries ?? (await discoverLibraries(options.cwd))).map(
           (library) => `${library.name}@${library.version}`,
         );
 
@@ -261,7 +313,8 @@ function toQueryHit(
     ...hit,
     name,
     version,
-    kind,
+    // A CLI's chunks are found by ranking and by address alike; the id says which kind they are.
+    kind: hit.packageId.endsWith(CLI_SUFFIX) ? "cli" : kind,
     trusted: !isCommunityPackage(name),
     ...(documents === undefined ? {} : { documents }),
   };
@@ -297,7 +350,9 @@ export function renderAnswer(result: QueryResult, query: string): string {
     const source =
       hit.kind === "artifact"
         ? `Source: ${hit.name}@${hit.version} — declared in ${hit.filePath}, read from the installed package`
-        : `Source: ${hit.name}@${hit.version} — ${hit.filePath}${describes}`;
+        : hit.kind === "cli"
+          ? `Source: ${hit.name}@${hit.version} — ${hit.filePath}, the installed package's own description of its command line`
+          : `Source: ${hit.name}@${hit.version} — ${hit.filePath}${describes}`;
     return [`## ${hit.chunkId}${trust}`, source, "", hit.content].join("\n");
   });
 
@@ -322,6 +377,8 @@ function undocumentedNotice(result: QueryResult): string {
 
 /** Splits `@stripe/docspack@2025.4.1` into its name and version. */
 export function splitPackageId(id: string): { name: string; version: string } {
+  // A CLI's commands are indexed under the library's id with a marker; the release is the same.
+  if (id.endsWith(CLI_SUFFIX)) return splitPackageId(id.slice(0, -CLI_SUFFIX.length));
   const at = id.lastIndexOf("@");
   if (at <= 0) return { name: id, version: "" };
   return { name: id.slice(0, at), version: id.slice(at + 1) };

package/src/spec.ts

@@ -46,9 +46,21 @@ export interface PackageManifest {
 
 const CHUNK_ID = /^[a-z0-9][a-z0-9._-]*$/i;
 
-/** Official vendor packages: `@stripe/docspack`. */
+/**
+ * Official vendor packages: `@stripe/docspack`, and its siblings `@stripe/openapi-docspack`.
+ *
+ * The suffixed form is for a vendor that also redistributes somebody else's documentation, where
+ * licence, version, cadence and attribution belong to a different upstream and cannot be merged
+ * into one package. It costs nothing the single name was protecting: discovery stays a pure name
+ * check against `package.json`, and npm scope ownership is the boundary trust rests on — whoever
+ * can publish `@stripe/docspack` can publish `@stripe/openapi-docspack` and nobody else can.
+ *
+ * Derivation is a separate question and unchanged: `@stripe/docspack` is the one name inferable
+ * from a dependency on `@stripe/sdk`, so that is still the only name `init` suggests. Derive to
+ * suggest; match to discover.
+ */
 export function isVendorPackage(name: string): boolean {
-  return /^@[^/]+\/docspack$/.test(name);
+  return /^@[^/]+\/(?:[^/]+-)?docspack$/.test(name);
 }
 
 /** Community packages: `@docspack-community/jira`. */
@@ -60,11 +72,25 @@ export function isDocsPackage(name: string): boolean {
   return isVendorPackage(name) || isCommunityPackage(name);
 }
 
+/** The discoverable name shapes, for an error that has to say what it would have accepted. */
+export const DISCOVERABLE_NAMES =
+  "@vendor/docspack, @vendor/<name>-docspack or @docspack-community/<name>";
+
 /** Stable identifier used as the primary key in the store: `@stripe/docspack@2025.4.1`. */
 export function packageId(name: string, version: string): string {
   return `${name}@${version}`;
 }
 
+/**
+ * What marks a package of command chunks derived from an installed CLI's own description. Its id
+ * cannot be the library's, which its type declarations already hold.
+ */
+export const CLI_SUFFIX = "#cli";
+
+export function cliPackageId(name: string, version: string): string {
+  return `${packageId(name, version)}${CLI_SUFFIX}`;
+}
+
 export function chunkId(pkgId: string, chunk: string): string {
   return `${pkgId}/${chunk}`;
 }

package/src/sync.ts

@@ -1,7 +1,15 @@
 import { readFile } from "node:fs/promises";
+import { relative } from "node:path";
 import { readArtifact } from "./artifact.js";
+import { COMMAND_ENTITY, commandChunks } from "./commands.js";
 import type { IndexedChunk, PackageKind, Store } from "./db.js";
-import { type DiscoveredPackage, discoverLibraries, discoverPackages } from "./discovery.js";
+import {
+  type DiscoveredCommandLine,
+  type DiscoveredPackage,
+  discoverCommandLines,
+  discoverLibraries,
+  discoverPackages,
+} from "./discovery.js";
 import { chunkId, estimateTokens, resolveChunkFile } from "./spec.js";
 
 export interface SyncOptions {
@@ -84,10 +92,121 @@ export async function syncProject(options: SyncOptions): Promise<SyncResult> {
   }
 
   if (options.artifacts !== false) synced.push(...(await syncArtifacts(options)));
+  synced.push(...(await syncCommandLines(options, problems)));
 
   return { packages: synced, problems };
 }
 
+/**
+ * Indexes the command-line interface each installed library describes in its own `package.json`
+ * (`"cmdspec"`, `packages/cmdspec/SPEC.md` §13): one chunk per command, ranked like documentation
+ * and addressed by command path, for exactly the version installed.
+ *
+ * Not tied to `--no-artifacts`. That flag skips the declarations, which are many and machine-made;
+ * a CLI's description is one document its authors wrote, and leaving it out would make `ask` report
+ * the package as installed but not indexed on every question.
+ */
+async function syncCommandLines(
+  options: SyncOptions,
+  problems: string[],
+): Promise<SyncedPackage[]> {
+  const commandLines = await discoverCommandLines(options.cwd);
+  if (commandLines.length === 0) return [];
+  const synced: SyncedPackage[] = [];
+
+  for (const cli of commandLines) {
+    if (options.force !== true && options.store.hasPackage(cli.id)) {
+      synced.push({
+        id: cli.id,
+        name: cli.name,
+        version: cli.version,
+        chunks: options.store.countChunks(cli.id),
+        tokens: 0,
+        status: "cached",
+        trusted: true,
+        kind: "cli",
+      });
+      continue;
+    }
+
+    options.onProgress?.(`reading ${cli.id}`);
+    const { chunks, symbols } = await readCommandLine(cli, problems);
+    if (chunks.length === 0) continue;
+    options.store.indexPackage(
+      { id: cli.id, name: cli.name, version: cli.version, kind: "cli" },
+      chunks,
+      symbols,
+    );
+    synced.push({
+      id: cli.id,
+      name: cli.name,
+      version: cli.version,
+      chunks: chunks.length,
+      tokens: chunks.reduce((total, chunk) => total + chunk.tokens, 0),
+      status: "indexed",
+      trusted: true,
+      kind: "cli",
+    });
+  }
+  return synced;
+}
+
+/**
+ * A library's CLI descriptions as chunks, and its command paths as names pointing at them.
+ *
+ * A description that is invalid, or that describes another version than the one installed, is
+ * reported and skipped: indexing it would answer with a command line the installed program does
+ * not accept, which is the one failure this tool exists to prevent.
+ */
+async function readCommandLine(
+  cli: DiscoveredCommandLine,
+  problems: string[],
+): Promise<{ chunks: IndexedChunk[]; symbols: Map<string, string> }> {
+  const { readFile: readDocument } = await import("@docspack/cmdspec/read");
+  const chunks: IndexedChunk[] = [];
+  const symbols = new Map<string, string>();
+  const taken = new Set<string>();
+
+  for (const path of cli.documents) {
+    const where = relative(cli.dir, path);
+    let document: Awaited<ReturnType<typeof readDocument>>["document"];
+    try {
+      ({ document } = await readDocument(path));
+    } catch (error) {
+      problems.push(
+        `${cli.name}: ${where} ${error instanceof Error ? error.message : String(error)}`,
+      );
+      continue;
+    }
+    if (document.info.version !== cli.version) {
+      problems.push(
+        `${cli.name}: ${where} describes version ${document.info.version}, but ${cli.version} is installed — not indexed`,
+      );
+      continue;
+    }
+
+    for (const chunk of commandChunks(document)) {
+      // Two executables in one package each have an overview called `cli`.
+      let id = chunk.id;
+      for (let suffix = 2; taken.has(id); suffix += 1) id = `${chunk.id}-${suffix}`;
+      taken.add(id);
+      const content = `# ${chunk.title}\n\n${chunk.text}`;
+      const full = chunkId(cli.id, id);
+      chunks.push({
+        chunkId: full,
+        filePath: chunk.path === undefined ? where : `${where}#${chunk.path}`,
+        tokens: estimateTokens(content),
+        content,
+        tags: [...chunk.tags, ...chunk.entities],
+      });
+      for (const entity of chunk.entities) {
+        if (entity.startsWith(COMMAND_ENTITY)) symbols.set(entity, full);
+      }
+    }
+  }
+  return { chunks, symbols };
+}
+
 /**
  * Indexes each installed library's exported declarations.
  *