docspack 0.4.0 → 1.1.0

package/src/endpoints.ts

@@ -0,0 +1,184 @@
+import { HTTP_METHODS } from "@docspack/openapi";
+
+/**
+ * Answering a question that names an HTTP request, by address rather than by ranking.
+ *
+ * This is the same argument `search.ts` makes for exported symbols, applied to endpoints. A query
+ * like `POST /v1/charges` is not a phrase, it is a key: ranked as prose it matches every chunk that
+ * mentions charges, and the overview page — which says "charges" six times — outranks the one chunk
+ * that describes the operation. `/v1` and `charges` are also the two worst possible search terms in
+ * a document where every path starts with `/v1`.
+ *
+ * So an endpoint an operation chunk declares is looked up exactly, and the matching chunk is pinned
+ * ahead of anything the ranker found. `docspack build --openapi` records both spellings of every
+ * operation in its chunk's `entities` — `POST /v1/charges` and `createCharge` — and the second one
+ * already worked, because an operation id is identifier-shaped and the symbol path handles it. The
+ * first one could not match anything at all.
+ */
+
+/** An operation a chunk declares. */
+export interface EndpointEntry {
+  readonly chunkId: string;
+  readonly method: string;
+  /** Template path as the document wrote it, braces included. */
+  readonly path: string;
+}
+
+/**
+ * How many endpoints one answer may pin.
+ *
+ * Two, matching `MAX_DECLARATIONS`. A question names one request, occasionally two — "how do I
+ * create a charge and then refund it". More than that and the addresses are incidental, and pinning
+ * them spends the budget on operations nobody asked about.
+ */
+export const MAX_ENDPOINTS = 2;
+
+const METHODS = new Set<string>(HTTP_METHODS);
+
+/**
+ * An endpoint reference in a query.
+ *
+ * The path is required to contain a `/`, which is what keeps `GET the newest invoice` from being
+ * read as a request for a path called `the`. A full URL is accepted and reduced to its path, because
+ * that is what an agent has in front of it when it is reading a log line or a network tab.
+ */
+const REFERENCE = /\b([A-Za-z]+)\s+(\S*\/\S*)/g;
+
+/** A bare path, for a question that names an endpoint without naming a method. */
+const BARE_PATH = /(?:^|\s)(\/[A-Za-z0-9{}._~%\-/]*)/g;
+
+/** Collects the endpoints a set of chunks declares, from their manifest entities. */
+export function endpointIndex(
+  chunks: Iterable<{ readonly chunkId: string; readonly entities: readonly string[] }>,
+): EndpointEntry[] {
+  const entries: EndpointEntry[] = [];
+  for (const chunk of chunks) {
+    for (const entity of chunk.entities) {
+      const parsed = parseEndpoint(entity);
+      if (parsed !== undefined) entries.push({ chunkId: chunk.chunkId, ...parsed });
+    }
+  }
+  return entries;
+}
+
+/** Reads `POST /v1/charges` as a method and a path. */
+function parseEndpoint(entity: string): { method: string; path: string } | undefined {
+  const match = /^([A-Z]+) (\/\S*)$/.exec(entity);
+  const method = match?.[1];
+  const path = match?.[2];
+  if (method === undefined || path === undefined || !METHODS.has(method)) return undefined;
+  return { method, path };
+}
+
+/**
+ * The operations a query addresses, in the order the query named them.
+ *
+ * An exact match wins. Failing that, the path is matched against the template a document declared,
+ * so an agent holding a real URL — `GET /v1/charges/ch_3Ox7`, copied out of a log — still reaches
+ * `GET /v1/charges/{charge}`. That is the case this exists for: a concrete request is what somebody
+ * actually has, and a template is what the document contains.
+ */
+export function matchEndpoints(
+  query: string,
+  index: readonly EndpointEntry[],
+  limit = MAX_ENDPOINTS,
+): EndpointEntry[] {
+  if (index.length === 0) return [];
+
+  const found: EndpointEntry[] = [];
+  const seen = new Set<string>();
+  const take = (entry: EndpointEntry): void => {
+    if (seen.has(entry.chunkId) || found.length >= limit) return;
+    seen.add(entry.chunkId);
+    found.push(entry);
+  };
+
+  for (const reference of references(query)) {
+    if (found.length >= limit) break;
+    const exact = index.filter(
+      (entry) => entry.method === reference.method && entry.path === reference.path,
+    );
+    if (exact.length > 0) {
+      for (const entry of exact) take(entry);
+      continue;
+    }
+    for (const entry of index) {
+      if (entry.method !== reference.method) continue;
+      if (templateMatches(entry.path, reference.path)) take(entry);
+    }
+  }
+
+  // Only when no method was named anywhere: otherwise a question that spells out `POST /charges`
+  // would also pin the `GET` on the same path, which is not what it asked for.
+  if (found.length === 0) {
+    for (const path of barePaths(query)) {
+      for (const entry of index) {
+        if (entry.path === path || templateMatches(entry.path, path)) take(entry);
+      }
+    }
+  }
+
+  return found;
+}
+
+/** Method-and-path references in a query, normalized. */
+function references(query: string): { method: string; path: string }[] {
+  const found: { method: string; path: string }[] = [];
+  for (const match of query.matchAll(REFERENCE)) {
+    const method = match[1]?.toUpperCase();
+    const raw = match[2];
+    if (method === undefined || raw === undefined || !METHODS.has(method)) continue;
+    const path = toPath(raw);
+    if (path !== undefined) found.push({ method, path });
+  }
+  return found;
+}
+
+function barePaths(query: string): string[] {
+  const found: string[] = [];
+  for (const match of query.matchAll(BARE_PATH)) {
+    const path = match[1] === undefined ? undefined : toPath(match[1]);
+    // A lone `/` addresses nothing, and every sentence with a slash in it would otherwise be read
+    // as naming the document's root.
+    if (path !== undefined && path.length > 1) found.push(path);
+  }
+  return found;
+}
+
+/**
+ * Reduces what someone wrote to a path.
+ *
+ * A full URL keeps only its path. A query string and trailing sentence punctuation are dropped: the
+ * query string is values rather than address, and a question mark at the end of `GET /charges?` is
+ * almost never an empty query string.
+ */
+function toPath(raw: string): string | undefined {
+  let value = raw.replace(/[),.;:'"`]+$/, "");
+  const scheme = /^[a-z][a-z0-9+.-]*:\/\//i.exec(value);
+  if (scheme !== null) {
+    try {
+      value = new URL(value).pathname;
+    } catch {
+      return undefined;
+    }
+  }
+  const queryAt = value.indexOf("?");
+  if (queryAt >= 0) value = value.slice(0, queryAt);
+  if (!value.startsWith("/")) return undefined;
+  // Repeated and trailing slashes are noise a URL bar forgives, and a path whose segment count is
+  // off by a stray slash matches no template at all — which reads as "the endpoint is undocumented".
+  return value.replace(/\/{2,}/g, "/").replace(/\/+$/, "") || "/";
+}
+
+/** Whether a concrete path fills in a template: `/v1/charges/{charge}` against `/v1/charges/ch_3`. */
+function templateMatches(template: string, concrete: string): boolean {
+  const left = template.split("/");
+  const right = concrete.split("/");
+  if (left.length !== right.length) return false;
+  return left.every((segment, index) => {
+    // A template segment matches any one segment, but not an empty one: `/charges/` is not a
+    // request for a charge.
+    if (segment.startsWith("{") && segment.endsWith("}")) return (right[index] ?? "").length > 0;
+    return segment === right[index];
+  });
+}

package/src/help.ts

@@ -37,12 +37,21 @@ export const COMMANDS: readonly CommandHelp[] = [
     summary: "Index the docs packages this project depends on",
     group: "core",
     usage: "docspack sync [options]",
-    options: [["--force", "re-index packages already in the store"]],
+    options: [
+      ["--force", "re-index packages already in the store"],
+      ["--no-artifacts", "skip the declarations derived from installed libraries"],
+    ],
     detail: [
       "Reads node_modules and indexes every @vendor/docspack and @docspack-community/<name>",
       "package this project declares. Makes no network requests.",
+      "",
+      "It also reads each installed library's own type declarations and indexes one entry per",
+      "exported name. Half of a well-documented library's exports are mentioned in no",
+      "documentation anyone published, and those declarations are the only local answer for",
+      "them. They are looked up by name, never ranked against prose, so they cannot crowd out",
+      "the documentation that does exist.",
     ],
-    examples: ["docspack sync", "docspack sync --force"],
+    examples: ["docspack sync", "docspack sync --force", "docspack sync --no-artifacts"],
   },
   {
     name: "ask",
@@ -54,12 +63,64 @@ export const COMMANDS: readonly CommandHelp[] = [
       "Answers from the installed versions, in the Markdown an agent should read. Exits 0 when",
       "chunks were returned, 3 when a docs package is installed but not indexed, and 4 when",
       "everything installed is indexed and nothing matched.",
+      "",
+      "When the question names something an installed library exports and no documentation",
+      "mentions, the answer leads with that name's declaration from the installed build and says",
+      "the documentation does not cover it. Ranking alone cannot tell that apart from a match.",
     ],
     examples: [
       'docspack ask "how do I verify a webhook signature"',
       'docspack ask "webhook signature" --package stripe --limit 5',
     ],
   },
+  {
+    name: "index",
+    summary: "Index this project's own sources, so an agent can ask them instead of reading them",
+    group: "core",
+    usage: "docspack index [--from <dir>] [--from-json <file|->]",
+    options: [
+      ["--from <dir>", "directory of Markdown to index"],
+      ["--from-json <f>", "JSON records to index, or `-` for standard input"],
+      ["--name <s>", "name for the corpus (default: derived from the source)"],
+      ["--force", "re-index even when no source has changed"],
+    ],
+    detail: [
+      "For a corpus this project already has rather than one somebody published: notes, an",
+      "export, rows out of a query. The payload is built in a temporary directory and thrown",
+      "away; what is kept is the index, in `.docspack/local.db`, which is a plaintext copy of",
+      "whatever was indexed and is gitignored on the tool's behalf.",
+      "",
+      "Records arrive as JSON so no database driver is needed: `sqlite3 -json … | docspack index",
+      "--from-json -`. A record with an `id` becomes one chunk under that id, because a row's",
+      "identity is its key.",
+      "",
+      "What was indexed is recorded with each source's size, mtime and hash, so a re-run does",
+      "nothing when nothing has changed and `recall` can say when an answer may be superseded.",
+    ],
+    examples: [
+      "docspack index --from ./notes",
+      "sqlite3 -json shop.db 'select id, title, body as text from posts' | docspack index --from-json -",
+    ],
+  },
+  {
+    name: "recall",
+    summary: "Answer from this project's indexed corpus, not from its dependencies",
+    group: "core",
+    usage: 'docspack recall "<question>"',
+    options: [
+      ["--limit <n>", `maximum chunks to return (default ${LIMIT})`],
+      ["--max-tokens <n>", `token ceiling for the result set (default ${MAX_TOKENS})`],
+    ],
+    detail: [
+      "Separate from `ask` on purpose. `ask` answers from the versions this project installed,",
+      "and a working corpus is never one of them — so a corpus cannot reach an answer about a",
+      "dependency, and a dependency cannot reach an answer about your notes.",
+      "",
+      "An answer leads with a warning when a source has changed since it was indexed. Exits 1",
+      "when nothing matched.",
+    ],
+    examples: ['docspack recall "what did we decide about retries"'],
+  },
   {
     name: "search",
     summary: "Same index, formatted for a human reading the terminal",
@@ -73,9 +134,57 @@ export const COMMANDS: readonly CommandHelp[] = [
     name: "list",
     summary: "Show this project's docs packages and their index state",
     group: "core",
-    usage: "docspack list",
+    usage: "docspack list [options]",
+    options: [["--coverage", "how much of each documented library's exports the prose mentions"]],
+    detail: [
+      "Coverage is mechanical: the exported names a library declares, against the names its",
+      "documentation mentions anywhere. It is reported, never gated on — a page listing every",
+      "export and explaining none would score full marks.",
+    ],
+    examples: ["docspack list", "docspack list --coverage", "docspack list --json"],
+  },
+  {
+    name: "agent",
+    summary: "Wire docspack into the agent tooling this project already uses",
+    group: "core",
+    usage: "docspack agent <install|check> [options]",
+    options: [
+      ["--feedback", "also include recording documentation problems"],
+      ["--hooks", "add a SessionStart hook that keeps the index in step"],
+      ["--mcp", "add the MCP server to .mcp.json"],
+      ["--dry-run", "print what would be written and write nothing"],
+    ],
+    detail: [
+      "`install` writes a marked block into AGENTS.md or CLAUDE.md — whichever the project",
+      "already has — and a skill into .claude/skills/docspack/ when the project uses Claude",
+      "Code. Everything outside the markers is left alone, and re-running rewrites the block",
+      "rather than appending a second copy.",
+      "",
+      "`check` writes nothing and exits non-zero when the wiring is missing or out of date, so",
+      "CI notices a pasted instruction that has drifted from what the tool now does.",
+    ],
+    examples: [
+      "docspack agent install",
+      "docspack agent install --feedback --hooks",
+      "docspack agent check",
+    ],
+  },
+  {
+    name: "changed",
+    summary: "What a library's exports gained and lost between two versions",
+    group: "core",
+    usage: "docspack changed <library>[@version]",
     options: [],
-    examples: ["docspack list", "docspack list --json"],
+    detail: [
+      "Compares two versions already in the global store, which is shared by every project on",
+      "this machine, so nothing is fetched. Without a version it compares what is installed here",
+      "against the most recently indexed other version.",
+      "",
+      "Upgrades are overwhelmingly additive: the useful answer is what exists now that an older",
+      "release did not have, and which of those names no documentation here mentions — the ones",
+      "a model can know from neither its training data nor the vendor's pages.",
+    ],
+    examples: ["docspack changed hono", "docspack changed hono@4.0.0"],
   },
   {
     name: "verify",
@@ -180,7 +289,7 @@ export const COMMANDS: readonly CommandHelp[] = [
     usage: "docspack build [source] [options]",
     options: [
       ["--from <dir>", "directory of Markdown to package"],
-      ["--openapi <file>", "OpenAPI JSON to package, one chunk per operation"],
+      ["--openapi <file>", "OpenAPI 3 document (JSON or YAML), one chunk per operation"],
       ["--name <name>", "package name, e.g. @acme/docspack"],
       ["--pkg-version <v>", "package version"],
       ["--out <dir>", "output directory (default: the current directory)"],
@@ -195,6 +304,11 @@ export const COMMANDS: readonly CommandHelp[] = [
       "the budget: use it for generated reference, where a heading is a field name and one chunk",
       "per heading is hundreds of chunks too small to answer anything.",
       "",
+      "--openapi writes one chunk per operation: the base URL, the credential, the inputs with",
+      "their types, the response, the failures and a runnable curl call, in LAPIS notation. Each",
+      "chunk answers to both `POST /v1/charges` and the operationId, so `docspack ask` can be",
+      "given either. Combine it with --from to publish prose and an API in one package.",
+      "",
       "With no flags, settings are read from the docspack key of the package.json in --out.",
     ],
     examples: [
@@ -202,6 +316,7 @@ export const COMMANDS: readonly CommandHelp[] = [
       "docspack build --from ./docs --name @acme/docspack --pkg-version 1.4.0",
       "docspack build --from ./reference --min-chunk-tokens 400 --max-chunk-tokens 900",
       "docspack build --openapi ./openapi.json",
+      "docspack build --from ./docs --openapi ./openapi.json",
     ],
   },
   {

package/src/index.ts

@@ -1,4 +1,21 @@
-export { type BuildOptions, type BuildResult, buildPackage } from "./build.js";
+export {
+  type AgentFile,
+  type AgentOptions,
+  type AgentPlan,
+  applyAgentSetup,
+  planAgentSetup,
+  type SurfaceKind,
+  withBlock,
+} from "./agent.js";
+export { type ArtifactPackage, readArtifact } from "./artifact.js";
+export {
+  type BuildOptions,
+  type BuildResult,
+  buildPackage,
+  LOCAL_VERSION,
+  localPackageName,
+} from "./build.js";
+export { type ChangedOptions, changedSurface, type SurfaceChange } from "./changed.js";
 export {
   type BuildConfig,
   CONFIG_KEY,
@@ -7,19 +24,34 @@ export {
   parseFeedbackChannel,
   readBuildConfig,
 } from "./config.js";
+export {
+  type CoverageOptions,
+  type CoverageReport,
+  identifiers,
+  type LibraryCoverage,
+  measureCoverage,
+} from "./coverage.js";
 export {
   defaultStorePath,
   type IndexedChunk,
   type IndexedPackage,
+  type IndexedSource,
+  LOCAL_STORE_DIR,
+  localStorePath,
+  type PackageKind,
   type SearchHit,
   type SearchOptions,
   Store,
+  type SymbolHit,
   silenceSqliteWarning,
   toFtsQuery,
+  toPackageKind,
 } from "./db.js";
 export {
+  type DiscoveredLibrary,
   type DiscoveredPackage,
   type Discovery,
+  discoverLibraries,
   discoverPackages,
   projectPackageIds,
   resolvePackageDir,
@@ -90,6 +122,17 @@ export {
   type LlmsTxtSection,
   parseLlmsTxt,
 } from "./llms-txt.js";
+export {
+  type IndexLocalOptions,
+  type IndexLocalResult,
+  indexLocal,
+  type RecallHit,
+  type RecallOptions,
+  type RecallResult,
+  recallLocal,
+  renderRecall,
+  type StaleSource,
+} from "./local.js";
 export { createMcpServer, type McpOptions, startMcpServer } from "./mcp.js";
 export { type PreviewOptions, type PreviewResult, previewPackage } from "./preview.js";
 export { type Choice, defaultStreams, Prompter, type PromptStreams } from "./prompt.js";
@@ -136,6 +179,13 @@ export {
   type SubmitOptions,
   type SubmitReport,
 } from "./submit.js";
+export {
+  declarationOf,
+  type ExportedSymbol,
+  hasJsdoc,
+  type PublicSurface,
+  readPublicSurface,
+} from "./surface.js";
 export {
   type SyncedPackage,
   type SyncOptions,
@@ -145,6 +195,7 @@ export {
 } from "./sync.js";
 export {
   type DriftFinding,
+  documentedLibraries,
   type VerifiedPackage,
   type VerifyOptions,
   type VerifyReport,