docspack 1.0.0 → 1.1.0

package/src/local.ts

@@ -0,0 +1,335 @@
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { buildPackage, LOCAL_VERSION, localPackageName, MARKDOWN } from "./build.js";
+import {
+  type IndexedChunk,
+  type IndexedSource,
+  LOCAL_STORE_DIR,
+  type SearchHit,
+  type Store,
+} from "./db.js";
+import { DocspackError } from "./errors.js";
+import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS } from "./search.js";
+import {
+  chunkId,
+  estimateTokens,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  packageId,
+  parseManifest,
+  resolveChunkFile,
+} from "./spec.js";
+
+export interface IndexLocalOptions {
+  readonly cwd: string;
+  readonly store: Store;
+  /** Directory of Markdown to index. */
+  readonly from?: string;
+  /** JSON records to index, or `-` for standard input. */
+  readonly json?: string;
+  readonly name?: string;
+  /** Re-index even when no source has changed. */
+  readonly force?: boolean;
+  readonly onProgress?: (message: string) => void;
+}
+
+export interface IndexLocalResult {
+  readonly id: string;
+  readonly name: string;
+  readonly chunks: number;
+  readonly tokens: number;
+  readonly status: "indexed" | "unchanged";
+  readonly sources: number;
+  /** A corpus read from a stream: nothing on disk to compare it against later. */
+  readonly streamed: boolean;
+  readonly warnings: readonly string[];
+}
+
+/**
+ * Indexes a working corpus into this project's own store.
+ *
+ * The payload is built in a temporary directory and thrown away. A published package is an
+ * artifact somebody installs; this is a cache of the reader's own files, and leaving a copy of it
+ * in the project would be one more thing to gitignore, review and let go stale.
+ */
+export async function indexLocal(options: IndexLocalOptions): Promise<IndexLocalResult> {
+  if (options.from === undefined && options.json === undefined) {
+    throw new DocspackError("Nothing to index", {
+      hint: "Pass --from <dir> for Markdown, or --from-json <file|-> for records.",
+    });
+  }
+
+  const name = options.name ?? localPackageName(options);
+  const id = packageId(name, LOCAL_VERSION);
+  const streamed = options.json === "-";
+  const sources = streamed ? [] : await fingerprint(options);
+
+  if (
+    options.force !== true &&
+    !streamed &&
+    options.store.hasPackage(id) &&
+    unchanged(options.store.sourcesOf(id), sources)
+  ) {
+    return {
+      id,
+      name,
+      chunks: options.store.countChunks(id),
+      tokens: 0,
+      status: "unchanged",
+      sources: sources.length,
+      streamed,
+      warnings: [],
+    };
+  }
+
+  const work = await mkdtemp(join(tmpdir(), "docspack-local-"));
+  try {
+    options.onProgress?.(`indexing ${name}`);
+    const built = await buildPackage({
+      out: work,
+      local: true,
+      name,
+      version: LOCAL_VERSION,
+      ...(options.from === undefined ? {} : { from: options.from }),
+      ...(options.json === undefined ? {} : { json: options.json }),
+    });
+
+    const chunks = await readChunks(work, id);
+    options.store.indexPackage({ id, name, version: LOCAL_VERSION, kind: "local" }, chunks);
+    options.store.recordSources(id, sources);
+    await protectStore(options.cwd);
+
+    return {
+      id,
+      name,
+      chunks: chunks.length,
+      tokens: built.tokens,
+      status: "indexed",
+      sources: sources.length,
+      streamed,
+      warnings: built.warnings,
+    };
+  } finally {
+    await rm(work, { recursive: true, force: true });
+  }
+}
+
+export interface RecallHit extends SearchHit {
+  readonly name: string;
+}
+
+/** A source that no longer matches what was indexed from it. */
+export interface StaleSource {
+  readonly path: string;
+  readonly reason: "changed" | "missing";
+}
+
+export interface RecallResult {
+  readonly hits: readonly RecallHit[];
+  readonly tokens: number;
+  /** Corpora present in the local store, whether or not they matched. */
+  readonly corpora: readonly string[];
+  readonly stale: readonly StaleSource[];
+  /** Corpora read from a stream, whose freshness cannot be checked. */
+  readonly streamed: readonly string[];
+}
+
+export interface RecallOptions {
+  readonly cwd: string;
+  readonly store: Store;
+  readonly query: string;
+  readonly limit?: number;
+  readonly maxTokens?: number;
+}
+
+/**
+ * Answers from the working corpora in this project's store, and from nothing else.
+ *
+ * Kept separate from `queryDocs` rather than added to it as a flag. `ask` promises an answer from
+ * the versions this project installed, and the one line that keeps that promise is the `kinds`
+ * filter it passes; a flag that flipped it would put a corpus of somebody's notes one typo away
+ * from an answer about a dependency.
+ */
+export async function recallLocal(options: RecallOptions): Promise<RecallResult> {
+  const corpora = options.store.listPackages().filter((pkg) => pkg.kind === "local");
+  const hits = options.store
+    .search(options.query, {
+      kinds: ["local"],
+      limit: options.limit ?? DEFAULT_LIMIT,
+      maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
+    })
+    .map((hit): RecallHit => ({ ...hit, name: splitLocalName(hit.packageId) }));
+
+  const stale: StaleSource[] = [];
+  const streamed: string[] = [];
+  for (const corpus of corpora) {
+    const recorded = options.store.sourcesOf(corpus.id);
+    if (recorded.length === 0) {
+      streamed.push(corpus.name);
+      continue;
+    }
+    stale.push(...(await drifted(recorded)));
+  }
+
+  return {
+    hits,
+    tokens: hits.reduce((total, hit) => total + hit.tokens, 0),
+    corpora: corpora.map((corpus) => corpus.name),
+    stale,
+    streamed,
+  };
+}
+
+export function renderRecall(result: RecallResult, query: string): string {
+  if (result.hits.length === 0) {
+    if (result.corpora.length === 0) {
+      return `Nothing has been indexed into this project's corpus. Run \`docspack index --from <dir>\` first.`;
+    }
+    return `Nothing in ${result.corpora.join(", ")} matched "${query}".`;
+  }
+
+  const sections = result.hits.map((hit) =>
+    [`## ${hit.chunkId}`, `Source: ${hit.name} — ${hit.filePath}`, "", hit.content].join("\n"),
+  );
+
+  // Before the passages, not after: a reader who acts on the first thing they read has to be told
+  // the text may be superseded before they read it, not once they already have.
+  if (result.stale.length > 0) sections.unshift(staleNotice(result.stale));
+  if (result.streamed.length > 0) {
+    sections.push(
+      `NOTE: ${result.streamed.join(", ")} was indexed from a stream, so whether it still matches its source cannot be checked here.`,
+    );
+  }
+  return sections.join("\n\n---\n\n");
+}
+
+function staleNotice(stale: readonly StaleSource[]): string {
+  const shown = stale.slice(0, 5).map((source) => source.path);
+  const rest = stale.length - shown.length;
+  const missing = stale.filter((source) => source.reason === "missing").length;
+  const tail = rest > 0 ? `, and ${String(rest)} more` : "";
+  const gone = missing > 0 ? ` ${String(missing)} of them no longer exist.` : "";
+  return `NOTE: the corpus is out of date. ${String(stale.length)} indexed ${stale.length === 1 ? "source has" : "sources have"} changed since it was built: ${shown.join(", ")}${tail}.${gone} The passages below may be superseded — run \`docspack index\` again before relying on them.`;
+}
+
+/** Which recorded sources no longer match what is on disk. */
+async function drifted(recorded: readonly IndexedSource[]): Promise<StaleSource[]> {
+  const stale: StaleSource[] = [];
+  for (const source of recorded) {
+    let info: Awaited<ReturnType<typeof stat>>;
+    try {
+      info = await stat(source.path);
+    } catch {
+      stale.push({ path: source.path, reason: "missing" });
+      continue;
+    }
+    // Size and mtime first, and the hash only when one of them moved: a checkout or a formatter
+    // touches every file's mtime without changing a byte, and hashing a whole corpus on every
+    // query to find that out would make the cheap path the expensive one.
+    if (info.size === source.size && info.mtimeMs === Date.parse(source.mtime)) continue;
+    if ((await hashOf(source.path)) === source.hash) continue;
+    stale.push({ path: source.path, reason: "changed" });
+  }
+  return stale;
+}
+
+/** What a corpus was built from, as it is right now. */
+async function fingerprint(options: IndexLocalOptions): Promise<IndexedSource[]> {
+  const paths: string[] = [];
+  if (options.from !== undefined) {
+    const root = resolve(options.from);
+    const found = await readdir(root, { recursive: true, withFileTypes: true }).catch(() => {
+      throw new DocspackError(`No such directory: ${options.from ?? ""}`, {
+        hint: "Pass --from <dir> to point at the Markdown you want indexed.",
+      });
+    });
+    paths.push(
+      ...found
+        .filter((entry) => entry.isFile() && MARKDOWN.test(entry.name))
+        .map((entry) => join(entry.parentPath, entry.name))
+        .sort(),
+    );
+  }
+  if (options.json !== undefined && options.json !== "-") paths.push(resolve(options.json));
+
+  const sources: IndexedSource[] = [];
+  for (const path of paths) {
+    const info = await stat(path);
+    sources.push({
+      path,
+      size: info.size,
+      mtime: new Date(info.mtimeMs).toISOString(),
+      hash: await hashOf(path),
+    });
+  }
+  return sources;
+}
+
+function unchanged(recorded: readonly IndexedSource[], current: readonly IndexedSource[]): boolean {
+  if (recorded.length !== current.length) return false;
+  const before = new Map(recorded.map((source) => [source.path, source.hash]));
+  return current.every((source) => before.get(source.path) === source.hash);
+}
+
+async function hashOf(path: string): Promise<string> {
+  return createHash("sha256")
+    .update(await readFile(path))
+    .digest("hex");
+}
+
+async function readChunks(dir: string, id: string): Promise<IndexedChunk[]> {
+  const llmsDir = join(dir, LLMS_DIR);
+  const manifest = parseManifest(
+    JSON.parse(await readFile(join(llmsDir, MANIFEST_FILE), "utf8")),
+    `${LLMS_DIR}/${MANIFEST_FILE}`,
+  );
+
+  const chunks: IndexedChunk[] = [];
+  for (const chunk of manifest.chunks) {
+    const content = (await readFile(resolveChunkFile(llmsDir, chunk.file), "utf8")).trim();
+    if (content.length === 0) continue;
+    chunks.push({
+      chunkId: chunkId(id, chunk.id),
+      filePath: chunk.file,
+      tokens: chunk.tokens ?? estimateTokens(content),
+      content,
+      tags: [...chunk.tags, ...chunk.entities],
+    });
+  }
+  if (chunks.length === 0) {
+    throw new DocspackError("Nothing readable to index", {
+      hint: "The sources produced no chunks. Check they contain text.",
+    });
+  }
+  return chunks;
+}
+
+/**
+ * Keeps the store out of version control.
+ *
+ * The index is a plaintext copy of whatever was indexed, so committing it would put the corpus in
+ * the repository a second time — and for notes, an export or a query result, somewhere nobody
+ * thought to look for it. Written next to the store rather than in the project's own
+ * `.gitignore`, which is the project's file and not this tool's to edit.
+ */
+async function protectStore(cwd: string): Promise<void> {
+  const dir = join(cwd, LOCAL_STORE_DIR);
+  await mkdir(dir, { recursive: true });
+  const ignore = join(dir, ".gitignore");
+  try {
+    await readFile(ignore, "utf8");
+  } catch {
+    await writeFile(
+      ignore,
+      "# The local index is a copy of your own sources. Do not commit it.\n*\n",
+    );
+  }
+}
+
+/** `@local/notes@0.0.0/chunk` carries the corpus name; the version is never interesting. */
+function splitLocalName(packageId: string): string {
+  const at = packageId.lastIndexOf("@");
+  return at <= 0 ? packageId : packageId.slice(0, at);
+}

package/src/preview.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
-import { type IndexedChunk, Store } from "./db.js";
+import { type IndexedChunk, type SearchHit, Store } from "./db.js";
+import { endpointIndex, matchEndpoints } from "./endpoints.js";
 import { DocspackError } from "./errors.js";
 import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS, type QueryHit } from "./search.js";
 import {
@@ -75,22 +76,45 @@ export async function previewPackage(options: PreviewOptions): Promise<PreviewRe
   const store = Store.open(":memory:");
   try {
     store.indexPackage({ id, name: manifest.name, version: manifest.version }, chunks);
-    const hits = store
+
+    const describe = (hit: SearchHit): QueryHit => ({
+      ...hit,
+      name: manifest.name,
+      version: manifest.version,
+      kind: "docs",
+      trusted: !isCommunityPackage(manifest.name),
+      ...(documents.length === 0 ? {} : { documents }),
+    });
+
+    // The same endpoint pinning `queryDocs` does, because `preview` exists to show an author what an
+    // agent would receive. A preview that ranked where the real query path pins would send them
+    // chasing a retrieval problem they do not have.
+    const limit = options.limit ?? DEFAULT_LIMIT;
+    const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+    const pinned = matchEndpoints(
+      options.query,
+      endpointIndex(
+        manifest.chunks.map((chunk) => ({
+          chunkId: chunkId(id, chunk.id),
+          entities: chunk.entities,
+        })),
+      ),
+    )
+      .map((match) => store.chunk(match.chunkId))
+      .filter((chunk): chunk is SearchHit => chunk !== undefined)
+      .map(describe);
+    const spent = pinned.reduce((total, hit) => total + hit.tokens, 0);
+    const pinnedIds = new Set(pinned.map((hit) => hit.chunkId));
+
+    const ranked = store
       .search(options.query, {
-        limit: options.limit ?? DEFAULT_LIMIT,
-        maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
+        limit: Math.max(1, limit - pinned.length),
+        maxTokens: Math.max(1, maxTokens - spent),
       })
-      .map(
-        (hit): QueryHit => ({
-          ...hit,
-          name: manifest.name,
-          version: manifest.version,
-          kind: "docs",
-          trusted: !isCommunityPackage(manifest.name),
-          ...(documents.length === 0 ? {} : { documents }),
-        }),
-      );
+      .filter((hit) => !pinnedIds.has(hit.chunkId))
+      .map(describe);
 
+    const hits = [...pinned, ...ranked];
     return {
       hits,
       tokens: hits.reduce((total, hit) => total + hit.tokens, 0),

package/src/search.ts

@@ -1,5 +1,6 @@
 import type { PackageKind, SearchHit, Store } from "./db.js";
 import { discoverLibraries, discoverPackages } from "./discovery.js";
+import { endpointIndex, matchEndpoints } from "./endpoints.js";
 import { chunkId, formatDocumentedLibrary, isCommunityPackage } from "./spec.js";
 
 /** Ceiling on how much context one query may return, per the blueprint's context-exhaustion rule. */
@@ -67,6 +68,14 @@ export interface QueryResult {
    * matched", which a ranker alone cannot tell apart — it always returns its best three.
    */
   readonly undocumented: readonly string[];
+  /**
+   * Endpoints the question named that a documented operation answers, pinned rather than ranked.
+   *
+   * Reported so an answer can say *why* a chunk is first. A reader who asked about
+   * `GET /v1/charges/ch_3Ox7` and is shown `GET /v1/charges/{charge}` has been given the right
+   * chunk, and has to be told that the match was against a template.
+   */
+  readonly endpoints: readonly string[];
 }
 
 export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
@@ -82,41 +91,53 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
   // Keyed by package id and by chunk id in the same map, because a chunk id already carries its
   // package: `@acme/docspack@1.4.0/api-auth` cannot collide with `@acme/docspack@1.4.0`.
   const documented = new Map<string, readonly string[]>();
+  const operations: { chunkId: string; entities: readonly string[] }[] = [];
   for (const pkg of installed ?? []) {
     const documents = pkg.manifest.documents;
     if (documents !== undefined && documents.length > 0) {
       documented.set(pkg.id, documents.map(formatDocumentedLibrary));
     }
     for (const chunk of pkg.manifest.chunks) {
+      operations.push({ chunkId: chunkId(pkg.id, chunk.id), entities: chunk.entities });
       if (chunk.documents === undefined || chunk.documents.length === 0) continue;
       documented.set(chunkId(pkg.id, chunk.id), chunk.documents.map(formatDocumentedLibrary));
     }
   }
 
   const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+  const limit = options.limit ?? DEFAULT_LIMIT;
+
+  // Endpoints first, and the ranker is then asked for less: a pinned operation is the answer, and
+  // three ranked chunks beside it would spend the budget restating what it already says.
+  const { hits: pinned, endpoints } = findEndpoints(options, operations, documented, maxTokens);
+  const spent = pinned.reduce((total, hit) => total + hit.tokens, 0);
+  const pinnedIds = new Set(pinned.map((hit) => hit.chunkId));
+
   const prose = options.store
     .search(options.query, {
       ...(packageIds === undefined ? {} : { packageIds }),
       ...(options.packageFilter === undefined
         ? {}
         : { packageFilter: `%${options.packageFilter}%` }),
-      limit: options.limit ?? DEFAULT_LIMIT,
-      maxTokens,
+      limit: Math.max(1, limit - pinned.length),
+      maxTokens: Math.max(1, maxTokens - spent),
       // Declarations are addressed by name, never ranked against prose: a library exports far
       // more names than its documentation has pages, and ranking them together would answer
       // every question with type machinery.
       kinds: ["docs"],
     })
+    .filter((hit) => !pinnedIds.has(hit.chunkId))
     .map((hit) => toQueryHit(hit, "docs", documented));
 
   const { declarations, undocumented } =
     options.artifacts === false
       ? { declarations: [] as QueryHit[], undocumented: [] as string[] }
-      : await findDeclarations(options, prose, maxTokens);
+      : await findDeclarations(options, [...pinned, ...prose], maxTokens - spent);
 
-  // Declarations first: for an agent about to write a call, the signature at the installed
-  // version is the load-bearing line, and the prose explains it afterwards.
-  const hits = [...declarations, ...prose];
+  // Pinned operations, then declarations, then the ranked prose. Both of the first two were
+  // addressed by name rather than guessed at, and for an agent about to write a call the exact
+  // thing it named is the load-bearing line.
+  const hits = [...pinned, ...declarations, ...prose];
 
   return {
     hits,
@@ -124,9 +145,42 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
     untrusted: hits.some((hit) => !hit.trusted),
     unindexed,
     undocumented,
+    endpoints,
   };
 }
 
+/**
+ * Operation chunks the question addressed.
+ *
+ * Bounded by the same token budget as everything else, and the first match is always kept: an answer
+ * whose one pinned operation did not fit would have pinned nothing and said nothing about it.
+ */
+function findEndpoints(
+  options: QueryOptions,
+  operations: readonly { chunkId: string; entities: readonly string[] }[],
+  documented: ReadonlyMap<string, readonly string[]>,
+  maxTokens: number,
+): { hits: QueryHit[]; endpoints: string[] } {
+  const matches = matchEndpoints(options.query, endpointIndex(operations));
+  const hits: QueryHit[] = [];
+  const endpoints: string[] = [];
+  let spent = 0;
+
+  for (const match of matches) {
+    if (options.packageFilter !== undefined && !match.chunkId.includes(options.packageFilter)) {
+      continue;
+    }
+    const chunk = options.store.chunk(match.chunkId);
+    if (chunk === undefined) continue;
+    if (hits.length > 0 && spent + chunk.tokens > maxTokens) break;
+    hits.push(toQueryHit(chunk, "docs", documented));
+    endpoints.push(`${match.method} ${match.path}`);
+    spent += chunk.tokens;
+  }
+
+  return { hits, endpoints };
+}
+
 /**
  * Looks up the names a question used against the symbols the installed libraries export.
  *