docspack 1.0.0 → 1.1.0

package/src/db.ts

@@ -10,11 +10,24 @@ const require = createRequire(import.meta.url);
 
 /**
  * Where a package's text came from. `docs` is written by a human and published as a docs
- * package; `artifact` is derived from an installed library's own type declarations. The
+ * package; `artifact` is derived from an installed library's own type declarations; `local` is a
+ * working corpus indexed from this project's own sources and never published. The
  * distinction is not cosmetic — a signature is a fact about the build, prose is a claim by its
- * author — so it is stored rather than inferred from the name.
+ * author, and a working corpus is a copy that can go stale under the reader — so it is stored
+ * rather than inferred from the name.
  */
-export type PackageKind = "docs" | "artifact";
+export type PackageKind = "docs" | "artifact" | "local";
+
+/**
+ * Reads the stored kind, defaulting an unrecognized one to `docs`.
+ *
+ * Written once rather than inline at each read: a store may have been written by a newer version
+ * that knows a kind this one does not, and silently relabelling it as something it is not is how
+ * a local corpus would end up presented as published documentation.
+ */
+export function toPackageKind(value: string): PackageKind {
+  return value === "artifact" || value === "local" ? value : "docs";
+}
 
 export interface IndexedPackage {
   readonly id: string;
@@ -31,6 +44,21 @@ export interface SymbolHit {
   readonly chunkId: string;
 }
 
+/**
+ * A file an indexed package was built from, as it was at index time.
+ *
+ * Only a working corpus records these. Published documentation is immutable for the life of its
+ * version, so there is nothing to detect; a corpus of the reader's own files is edited underneath
+ * the index, and an answer quoted from a superseded source is worse than no answer.
+ */
+export interface IndexedSource {
+  readonly path: string;
+  readonly size: number;
+  /** ISO timestamp, as text, because that is what survives a round trip through SQLite. */
+  readonly mtime: string;
+  readonly hash: string;
+}
+
 export interface IndexedChunk {
   readonly chunkId: string;
   readonly filePath: string;
@@ -59,7 +87,10 @@ export interface SearchOptions {
   readonly kinds?: readonly PackageKind[];
 }
 
-const SCHEMA_VERSION = 2;
+/** Where per-project state lives, alongside the feedback file already written there. */
+export const LOCAL_STORE_DIR = ".docspack";
+
+const SCHEMA_VERSION = 3;
 
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS packages (
@@ -89,6 +120,15 @@ CREATE TABLE IF NOT EXISTS symbols (
 
 CREATE INDEX IF NOT EXISTS symbols_by_name ON symbols(name);
 
+CREATE TABLE IF NOT EXISTS sources (
+    package_id TEXT NOT NULL REFERENCES packages(id),
+    path TEXT NOT NULL,
+    size INTEGER NOT NULL,
+    mtime TEXT NOT NULL,
+    hash TEXT NOT NULL,
+    PRIMARY KEY (package_id, path)
+);
+
 CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
     content,
     tags,
@@ -120,6 +160,17 @@ function loadSqlite(): typeof import("node:sqlite") {
   return require("node:sqlite") as typeof import("node:sqlite");
 }
 
+/**
+ * Where a project's own working corpus is indexed.
+ *
+ * Deliberately not the global store: these chunks are a plaintext copy of one project's sources,
+ * they are worthless to any other project, and a machine-wide store would accumulate them from
+ * every checkout with nothing to evict them.
+ */
+export function localStorePath(cwd: string): string {
+  return join(cwd, LOCAL_STORE_DIR, "local.db");
+}
+
 /** Default location of the shared index, mirroring pnpm's global store. */
 export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
   const override = env.DOCSPACK_STORE;
@@ -207,7 +258,7 @@ export class Store {
       id: row.id,
       name: row.name,
       version: row.version,
-      kind: row.kind === "artifact" ? "artifact" : "docs",
+      kind: toPackageKind(row.kind),
       indexedAt: row.indexed_at,
     }));
   }
@@ -351,6 +402,36 @@ export class Store {
     }
   }
 
+  /** Replaces the record of what a package was built from. */
+  recordSources(packageId: string, sources: readonly IndexedSource[]): void {
+    const insert = this.#db.prepare(
+      "INSERT OR REPLACE INTO sources (package_id, path, size, mtime, hash) VALUES (?, ?, ?, ?, ?)",
+    );
+    this.#db.exec("BEGIN");
+    try {
+      this.#db.prepare("DELETE FROM sources WHERE package_id = ?").run(packageId);
+      for (const source of sources) {
+        insert.run(packageId, source.path, source.size, source.mtime, source.hash);
+      }
+      this.#db.exec("COMMIT");
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+
+  sourcesOf(packageId: string): IndexedSource[] {
+    const rows = this.#db
+      .prepare("SELECT path, size, mtime, hash FROM sources WHERE package_id = ? ORDER BY path")
+      .all(packageId) as { path: string; size: number; mtime: string; hash: string }[];
+    return rows.map((row) => ({
+      path: row.path,
+      size: Number(row.size),
+      mtime: row.mtime,
+      hash: row.hash,
+    }));
+  }
+
   removePackage(id: string): void {
     this.#db.exec("BEGIN");
     try {
@@ -430,6 +511,7 @@ export class Store {
   }
 
   #deleteChunks(packageId: string): void {
+    this.#db.prepare("DELETE FROM sources WHERE package_id = ?").run(packageId);
     this.#db.prepare("DELETE FROM chunks_fts WHERE package_id = ?").run(packageId);
     this.#db.prepare("DELETE FROM chunks WHERE package_id = ?").run(packageId);
     this.#db.prepare("DELETE FROM symbols WHERE package_id = ?").run(packageId);

package/src/doctor.ts

@@ -362,21 +362,29 @@ function checkPackageJson(
   }
 }
 
-/** Newest mtime across the configured documentation source, used to spot a stale payload. */
+/**
+ * Newest mtime across every configured documentation source, used to spot a stale payload.
+ *
+ * Both sources are checked, not the first one found: a package configured with prose *and* an
+ * OpenAPI document would otherwise report a fresh payload after the document changed, which is the
+ * one case this check exists to catch.
+ */
 async function newestSourceMtime(dir: string): Promise<number> {
   const config = await readBuildConfig(dir);
-  const source = config.from ?? config.openapi;
-  if (source === undefined) return 0;
-
-  const target = join(dir, source);
-  const single = await mtime(target);
-  if (single > 0 && config.openapi !== undefined) return single;
+  const newest = await Promise.all([
+    config.openapi === undefined ? 0 : mtime(join(dir, config.openapi)),
+    config.from === undefined ? 0 : newestInTree(join(dir, config.from)),
+  ]);
+  return Math.max(...newest);
+}
 
+/** Newest mtime in a directory tree, or the file's own when the path is a file. */
+async function newestInTree(target: string): Promise<number> {
   let found: Dirent[];
   try {
     found = await readdir(target, { recursive: true, withFileTypes: true });
   } catch {
-    return single;
+    return mtime(target);
   }
 
   let newest = 0;

package/src/endpoints.ts

@@ -0,0 +1,184 @@
+import { HTTP_METHODS } from "@docspack/openapi";
+
+/**
+ * Answering a question that names an HTTP request, by address rather than by ranking.
+ *
+ * This is the same argument `search.ts` makes for exported symbols, applied to endpoints. A query
+ * like `POST /v1/charges` is not a phrase, it is a key: ranked as prose it matches every chunk that
+ * mentions charges, and the overview page — which says "charges" six times — outranks the one chunk
+ * that describes the operation. `/v1` and `charges` are also the two worst possible search terms in
+ * a document where every path starts with `/v1`.
+ *
+ * So an endpoint an operation chunk declares is looked up exactly, and the matching chunk is pinned
+ * ahead of anything the ranker found. `docspack build --openapi` records both spellings of every
+ * operation in its chunk's `entities` — `POST /v1/charges` and `createCharge` — and the second one
+ * already worked, because an operation id is identifier-shaped and the symbol path handles it. The
+ * first one could not match anything at all.
+ */
+
+/** An operation a chunk declares. */
+export interface EndpointEntry {
+  readonly chunkId: string;
+  readonly method: string;
+  /** Template path as the document wrote it, braces included. */
+  readonly path: string;
+}
+
+/**
+ * How many endpoints one answer may pin.
+ *
+ * Two, matching `MAX_DECLARATIONS`. A question names one request, occasionally two — "how do I
+ * create a charge and then refund it". More than that and the addresses are incidental, and pinning
+ * them spends the budget on operations nobody asked about.
+ */
+export const MAX_ENDPOINTS = 2;
+
+const METHODS = new Set<string>(HTTP_METHODS);
+
+/**
+ * An endpoint reference in a query.
+ *
+ * The path is required to contain a `/`, which is what keeps `GET the newest invoice` from being
+ * read as a request for a path called `the`. A full URL is accepted and reduced to its path, because
+ * that is what an agent has in front of it when it is reading a log line or a network tab.
+ */
+const REFERENCE = /\b([A-Za-z]+)\s+(\S*\/\S*)/g;
+
+/** A bare path, for a question that names an endpoint without naming a method. */
+const BARE_PATH = /(?:^|\s)(\/[A-Za-z0-9{}._~%\-/]*)/g;
+
+/** Collects the endpoints a set of chunks declares, from their manifest entities. */
+export function endpointIndex(
+  chunks: Iterable<{ readonly chunkId: string; readonly entities: readonly string[] }>,
+): EndpointEntry[] {
+  const entries: EndpointEntry[] = [];
+  for (const chunk of chunks) {
+    for (const entity of chunk.entities) {
+      const parsed = parseEndpoint(entity);
+      if (parsed !== undefined) entries.push({ chunkId: chunk.chunkId, ...parsed });
+    }
+  }
+  return entries;
+}
+
+/** Reads `POST /v1/charges` as a method and a path. */
+function parseEndpoint(entity: string): { method: string; path: string } | undefined {
+  const match = /^([A-Z]+) (\/\S*)$/.exec(entity);
+  const method = match?.[1];
+  const path = match?.[2];
+  if (method === undefined || path === undefined || !METHODS.has(method)) return undefined;
+  return { method, path };
+}
+
+/**
+ * The operations a query addresses, in the order the query named them.
+ *
+ * An exact match wins. Failing that, the path is matched against the template a document declared,
+ * so an agent holding a real URL — `GET /v1/charges/ch_3Ox7`, copied out of a log — still reaches
+ * `GET /v1/charges/{charge}`. That is the case this exists for: a concrete request is what somebody
+ * actually has, and a template is what the document contains.
+ */
+export function matchEndpoints(
+  query: string,
+  index: readonly EndpointEntry[],
+  limit = MAX_ENDPOINTS,
+): EndpointEntry[] {
+  if (index.length === 0) return [];
+
+  const found: EndpointEntry[] = [];
+  const seen = new Set<string>();
+  const take = (entry: EndpointEntry): void => {
+    if (seen.has(entry.chunkId) || found.length >= limit) return;
+    seen.add(entry.chunkId);
+    found.push(entry);
+  };
+
+  for (const reference of references(query)) {
+    if (found.length >= limit) break;
+    const exact = index.filter(
+      (entry) => entry.method === reference.method && entry.path === reference.path,
+    );
+    if (exact.length > 0) {
+      for (const entry of exact) take(entry);
+      continue;
+    }
+    for (const entry of index) {
+      if (entry.method !== reference.method) continue;
+      if (templateMatches(entry.path, reference.path)) take(entry);
+    }
+  }
+
+  // Only when no method was named anywhere: otherwise a question that spells out `POST /charges`
+  // would also pin the `GET` on the same path, which is not what it asked for.
+  if (found.length === 0) {
+    for (const path of barePaths(query)) {
+      for (const entry of index) {
+        if (entry.path === path || templateMatches(entry.path, path)) take(entry);
+      }
+    }
+  }
+
+  return found;
+}
+
+/** Method-and-path references in a query, normalized. */
+function references(query: string): { method: string; path: string }[] {
+  const found: { method: string; path: string }[] = [];
+  for (const match of query.matchAll(REFERENCE)) {
+    const method = match[1]?.toUpperCase();
+    const raw = match[2];
+    if (method === undefined || raw === undefined || !METHODS.has(method)) continue;
+    const path = toPath(raw);
+    if (path !== undefined) found.push({ method, path });
+  }
+  return found;
+}
+
+function barePaths(query: string): string[] {
+  const found: string[] = [];
+  for (const match of query.matchAll(BARE_PATH)) {
+    const path = match[1] === undefined ? undefined : toPath(match[1]);
+    // A lone `/` addresses nothing, and every sentence with a slash in it would otherwise be read
+    // as naming the document's root.
+    if (path !== undefined && path.length > 1) found.push(path);
+  }
+  return found;
+}
+
+/**
+ * Reduces what someone wrote to a path.
+ *
+ * A full URL keeps only its path. A query string and trailing sentence punctuation are dropped: the
+ * query string is values rather than address, and a question mark at the end of `GET /charges?` is
+ * almost never an empty query string.
+ */
+function toPath(raw: string): string | undefined {
+  let value = raw.replace(/[),.;:'"`]+$/, "");
+  const scheme = /^[a-z][a-z0-9+.-]*:\/\//i.exec(value);
+  if (scheme !== null) {
+    try {
+      value = new URL(value).pathname;
+    } catch {
+      return undefined;
+    }
+  }
+  const queryAt = value.indexOf("?");
+  if (queryAt >= 0) value = value.slice(0, queryAt);
+  if (!value.startsWith("/")) return undefined;
+  // Repeated and trailing slashes are noise a URL bar forgives, and a path whose segment count is
+  // off by a stray slash matches no template at all — which reads as "the endpoint is undocumented".
+  return value.replace(/\/{2,}/g, "/").replace(/\/+$/, "") || "/";
+}
+
+/** Whether a concrete path fills in a template: `/v1/charges/{charge}` against `/v1/charges/ch_3`. */
+function templateMatches(template: string, concrete: string): boolean {
+  const left = template.split("/");
+  const right = concrete.split("/");
+  if (left.length !== right.length) return false;
+  return left.every((segment, index) => {
+    // A template segment matches any one segment, but not an empty one: `/charges/` is not a
+    // request for a charge.
+    if (segment.startsWith("{") && segment.endsWith("}")) return (right[index] ?? "").length > 0;
+    return segment === right[index];
+  });
+}

package/src/help.ts

@@ -73,6 +73,54 @@ export const COMMANDS: readonly CommandHelp[] = [
       'docspack ask "webhook signature" --package stripe --limit 5',
     ],
   },
+  {
+    name: "index",
+    summary: "Index this project's own sources, so an agent can ask them instead of reading them",
+    group: "core",
+    usage: "docspack index [--from <dir>] [--from-json <file|->]",
+    options: [
+      ["--from <dir>", "directory of Markdown to index"],
+      ["--from-json <f>", "JSON records to index, or `-` for standard input"],
+      ["--name <s>", "name for the corpus (default: derived from the source)"],
+      ["--force", "re-index even when no source has changed"],
+    ],
+    detail: [
+      "For a corpus this project already has rather than one somebody published: notes, an",
+      "export, rows out of a query. The payload is built in a temporary directory and thrown",
+      "away; what is kept is the index, in `.docspack/local.db`, which is a plaintext copy of",
+      "whatever was indexed and is gitignored on the tool's behalf.",
+      "",
+      "Records arrive as JSON so no database driver is needed: `sqlite3 -json … | docspack index",
+      "--from-json -`. A record with an `id` becomes one chunk under that id, because a row's",
+      "identity is its key.",
+      "",
+      "What was indexed is recorded with each source's size, mtime and hash, so a re-run does",
+      "nothing when nothing has changed and `recall` can say when an answer may be superseded.",
+    ],
+    examples: [
+      "docspack index --from ./notes",
+      "sqlite3 -json shop.db 'select id, title, body as text from posts' | docspack index --from-json -",
+    ],
+  },
+  {
+    name: "recall",
+    summary: "Answer from this project's indexed corpus, not from its dependencies",
+    group: "core",
+    usage: 'docspack recall "<question>"',
+    options: [
+      ["--limit <n>", `maximum chunks to return (default ${LIMIT})`],
+      ["--max-tokens <n>", `token ceiling for the result set (default ${MAX_TOKENS})`],
+    ],
+    detail: [
+      "Separate from `ask` on purpose. `ask` answers from the versions this project installed,",
+      "and a working corpus is never one of them — so a corpus cannot reach an answer about a",
+      "dependency, and a dependency cannot reach an answer about your notes.",
+      "",
+      "An answer leads with a warning when a source has changed since it was indexed. Exits 1",
+      "when nothing matched.",
+    ],
+    examples: ['docspack recall "what did we decide about retries"'],
+  },
   {
     name: "search",
     summary: "Same index, formatted for a human reading the terminal",
@@ -241,7 +289,7 @@ export const COMMANDS: readonly CommandHelp[] = [
     usage: "docspack build [source] [options]",
     options: [
       ["--from <dir>", "directory of Markdown to package"],
-      ["--openapi <file>", "OpenAPI JSON to package, one chunk per operation"],
+      ["--openapi <file>", "OpenAPI 3 document (JSON or YAML), one chunk per operation"],
       ["--name <name>", "package name, e.g. @acme/docspack"],
       ["--pkg-version <v>", "package version"],
       ["--out <dir>", "output directory (default: the current directory)"],
@@ -256,6 +304,11 @@ export const COMMANDS: readonly CommandHelp[] = [
       "the budget: use it for generated reference, where a heading is a field name and one chunk",
       "per heading is hundreds of chunks too small to answer anything.",
       "",
+      "--openapi writes one chunk per operation: the base URL, the credential, the inputs with",
+      "their types, the response, the failures and a runnable curl call, in LAPIS notation. Each",
+      "chunk answers to both `POST /v1/charges` and the operationId, so `docspack ask` can be",
+      "given either. Combine it with --from to publish prose and an API in one package.",
+      "",
       "With no flags, settings are read from the docspack key of the package.json in --out.",
     ],
     examples: [
@@ -263,6 +316,7 @@ export const COMMANDS: readonly CommandHelp[] = [
       "docspack build --from ./docs --name @acme/docspack --pkg-version 1.4.0",
       "docspack build --from ./reference --min-chunk-tokens 400 --max-chunk-tokens 900",
       "docspack build --openapi ./openapi.json",
+      "docspack build --from ./docs --openapi ./openapi.json",
     ],
   },
   {

package/src/index.ts

@@ -8,7 +8,13 @@ export {
   withBlock,
 } from "./agent.js";
 export { type ArtifactPackage, readArtifact } from "./artifact.js";
-export { type BuildOptions, type BuildResult, buildPackage } from "./build.js";
+export {
+  type BuildOptions,
+  type BuildResult,
+  buildPackage,
+  LOCAL_VERSION,
+  localPackageName,
+} from "./build.js";
 export { type ChangedOptions, changedSurface, type SurfaceChange } from "./changed.js";
 export {
   type BuildConfig,
@@ -29,6 +35,9 @@ export {
   defaultStorePath,
   type IndexedChunk,
   type IndexedPackage,
+  type IndexedSource,
+  LOCAL_STORE_DIR,
+  localStorePath,
   type PackageKind,
   type SearchHit,
   type SearchOptions,
@@ -36,6 +45,7 @@ export {
   type SymbolHit,
   silenceSqliteWarning,
   toFtsQuery,
+  toPackageKind,
 } from "./db.js";
 export {
   type DiscoveredLibrary,
@@ -112,6 +122,17 @@ export {
   type LlmsTxtSection,
   parseLlmsTxt,
 } from "./llms-txt.js";
+export {
+  type IndexLocalOptions,
+  type IndexLocalResult,
+  indexLocal,
+  type RecallHit,
+  type RecallOptions,
+  type RecallResult,
+  recallLocal,
+  renderRecall,
+  type StaleSource,
+} from "./local.js";
 export { createMcpServer, type McpOptions, startMcpServer } from "./mcp.js";
 export { type PreviewOptions, type PreviewResult, previewPackage } from "./preview.js";
 export { type Choice, defaultStreams, Prompter, type PromptStreams } from "./prompt.js";