docspack 0.4.0 → 1.0.0

package/src/agent.ts

@@ -0,0 +1,308 @@
+import { existsSync } from "node:fs";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { DocspackError } from "./errors.js";
+import { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
+
+/**
+ * Wiring docspack into the agent tooling a project already uses.
+ *
+ * The documented setup is "paste two lines into AGENTS.md", which is manual, easy to forget, and
+ * drifts: a line pasted once says whatever docspack did at the time. This writes the same thing
+ * from the code that defines it, into a marked block that can be rewritten in place.
+ */
+export type SurfaceKind = "instructions" | "skill" | "hook" | "mcp";
+
+export interface AgentFile {
+  /** Path relative to the project root. */
+  readonly path: string;
+  readonly kind: SurfaceKind;
+  readonly contents: string;
+  readonly status: "create" | "update" | "unchanged";
+  /** Why this file is in the plan, for the line the CLI prints. */
+  readonly reason: string;
+}
+
+export interface AgentPlan {
+  readonly files: readonly AgentFile[];
+}
+
+export interface AgentOptions {
+  readonly cwd: string;
+  /**
+   * The three surfaces beyond the instruction block. Left unset, each is inferred from what is
+   * already on disk, so `docspack agent check` needs no flags to say whether what a project
+   * chose is still current — which is the only form of the check CI can run.
+   */
+  readonly feedback?: boolean;
+  /** Write a SessionStart hook that keeps the index in step with the lockfile. */
+  readonly hooks?: boolean;
+  /** Write the MCP server into `.mcp.json`. */
+  readonly mcp?: boolean;
+}
+
+const START = "<!-- docspack:start -->";
+const END = "<!-- docspack:end -->";
+
+/** Files an agent reads as standing instructions, in the order they are checked. */
+const INSTRUCTION_FILES = ["AGENTS.md", "CLAUDE.md"] as const;
+
+/**
+ * What to write, and where. Nothing is written for a tool the project does not use: the presence
+ * of `.claude/` is what says a Claude Code user works here, and a project with neither an
+ * AGENTS.md nor a CLAUDE.md gets AGENTS.md, which is the cross-tool convention.
+ */
+export async function planAgentSetup(options: AgentOptions): Promise<AgentPlan> {
+  const root = resolve(options.cwd);
+  if (!existsSync(join(root, "package.json"))) {
+    throw new DocspackError(`No package.json found in ${root}`, {
+      hint: "Run docspack from a project directory, or pass --cwd <dir>.",
+    });
+  }
+
+  const existing = INSTRUCTION_FILES.filter((name) => existsSync(join(root, name)));
+  const feedback = options.feedback ?? (await wantsFeedback(root, existing));
+  const files: AgentFile[] = [];
+  const block = instructionBlock(feedback);
+
+  const targets = existing.length > 0 ? existing : ["AGENTS.md"];
+  for (const name of targets) {
+    const before = await read(join(root, name));
+    const after = withBlock(before ?? "", block);
+    files.push({
+      path: name,
+      kind: "instructions",
+      contents: after,
+      status: before === undefined ? "create" : after === before ? "unchanged" : "update",
+      reason:
+        before === undefined
+          ? "the cross-tool convention for standing agent instructions"
+          : "already read by the agents working here",
+    });
+  }
+
+  if (existsSync(join(root, ".claude"))) {
+    const path = join(".claude", "skills", "docspack", "SKILL.md");
+    const before = await read(join(root, path));
+    const contents = skill(feedback);
+    files.push({
+      path,
+      kind: "skill",
+      contents,
+      status: before === undefined ? "create" : before === contents ? "unchanged" : "update",
+      // A skill's body is loaded when it is used; instructions are loaded every session. Both
+      // are written, because only one of them survives a session that never asks a question.
+      reason: "loaded on demand, so it costs nothing in a session that asks no questions",
+    });
+  }
+
+  const settings = await read(join(root, ".claude", "settings.json"));
+  const wantsHook = options.hooks ?? (settings ?? "").includes("docspack sync");
+  if (wantsHook) files.push(await hook(root));
+
+  const mcpConfig = await read(join(root, ".mcp.json"));
+  const wantsMcp = options.mcp ?? /"docspack"\s*:/.test(mcpConfig ?? "");
+  if (wantsMcp) files.push(await mcpEntry(root));
+
+  return { files };
+}
+
+/** Writes a plan. A file that already carries the current text is left alone. */
+export async function applyAgentSetup(
+  options: AgentOptions,
+  plan: AgentPlan,
+): Promise<readonly AgentFile[]> {
+  const root = resolve(options.cwd);
+  const written: AgentFile[] = [];
+
+  for (const file of plan.files) {
+    if (file.status === "unchanged") continue;
+    const target = join(root, file.path);
+    await mkdir(dirname(target), { recursive: true });
+    await writeFile(target, file.contents, "utf8");
+    written.push(file);
+  }
+
+  return written;
+}
+
+/**
+ * Replaces the marked block, or appends one. Everything outside the markers is left exactly as
+ * it was: these files are written by people, and a tool that rewrites them wholesale is a tool
+ * nobody runs twice.
+ */
+export function withBlock(document: string, block: string): string {
+  const start = document.indexOf(START);
+  const end = document.indexOf(END);
+
+  if (start !== -1 && end > start) {
+    return `${document.slice(0, start)}${block}${document.slice(end + END.length)}`;
+  }
+
+  const body = document.trimEnd();
+  return body.length === 0 ? `${block}\n` : `${body}\n\n${block}\n`;
+}
+
+/** Whether a project already asked for the feedback half, read from what it has on disk. */
+async function wantsFeedback(root: string, files: readonly string[]): Promise<boolean> {
+  for (const name of files) {
+    if ((await read(join(root, name)))?.includes("docspack feedback add") === true) return true;
+  }
+  const skillFile = await read(join(root, ".claude", "skills", "docspack", "SKILL.md"));
+  return skillFile?.includes("docspack feedback add") === true;
+}
+
+function instructionBlock(feedback: boolean): string {
+  const lines = [
+    START,
+    "## Documentation for this project's dependencies",
+    "",
+    ...AGENTS_SNIPPET,
+    ...(feedback ? ["", ...FEEDBACK_SNIPPET] : []),
+    "",
+    "<!-- Written by `docspack agent install`. Re-run it to update this block. -->",
+    END,
+  ];
+  return lines.join("\n");
+}
+
+/**
+ * The Claude Code skill. Its description is what decides whether the skill is reached for, and
+ * the body is loaded only once it is — so the description names the situations rather than the
+ * tool, and the body carries what the two-line instruction cannot afford to.
+ */
+function skill(feedback: boolean): string {
+  return [
+    "---",
+    "name: docspack",
+    "description: >-",
+    "  Answer questions about this project's installed dependencies from local, version-matched",
+    "  documentation instead of memory. Use before writing or reviewing code that calls a",
+    "  dependency, when an API's shape is uncertain, when an import or a signature may have",
+    "  changed between versions, or when a name is not in any documentation at all.",
+    "allowed-tools: Bash",
+    "---",
+    "",
+    "# Documentation for installed dependencies",
+    "",
+    'Run `docspack ask "<question>"` for anything about a dependency of this project. It answers',
+    "from the versions in `node_modules`, offline, in about 100ms, and bounded to 3,000 tokens.",
+    "",
+    "Prefer it over recalling an API from training data. The installed version is usually newer",
+    "than the training data and the answer states which release it came from.",
+    "",
+    "## What the answer can tell you",
+    "",
+    "- A passage from a documentation package, headed by its chunk id and the version it documents.",
+    "- A declaration read from the installed build, when the question names something the",
+    "  documentation never mentions. It carries the import to write and the signature as installed.",
+    "- A note saying the name is exported but documented nowhere. That is a real answer: it means",
+    "  no prose exists, not that the search failed.",
+    "",
+    "## Other commands",
+    "",
+    "- `docspack changed <library>` — what a library's exports gained and lost between two",
+    "  installed versions. Use it when an upgrade broke something, or before writing code against",
+    "  a version newer than what you remember.",
+    "- `docspack list` — which packages are indexed here.",
+    "- `docspack sync` — index them. Run it if `ask` reports something installed but not indexed.",
+    ...(feedback
+      ? [
+          "",
+          "## Recording a problem",
+          "",
+          "If the documentation contradicts the installed package, record it:",
+          "",
+          "```bash",
+          'docspack feedback add --chunk <id> --kind <drift|incorrect|missing> --evidence "<claim>"',
+          "```",
+          "",
+          "Only claims that can be shown false. `incorrect` and `missing` also need `--expected`,",
+          "`--actual` and `--repro`, so record those only after running something. It appends to a",
+          "local file for a human to review and sends nothing.",
+        ]
+      : []),
+    "",
+  ].join("\n");
+}
+
+/**
+ * A SessionStart hook, so the index follows the lockfile without anyone remembering to sync.
+ *
+ * `--quiet` keeps it silent: a hook that prints on every session start spends context on a
+ * message saying nothing changed.
+ */
+async function hook(root: string): Promise<AgentFile> {
+  const path = join(".claude", "settings.json");
+  const before = await read(join(root, path));
+  const settings = parseJson(before, path);
+  // Silent and never failing: a session must not break, or print, because an index is stale.
+  const command = "npx --no-install docspack sync --quiet >/dev/null 2>&1 || true";
+
+  const hooks = (settings.hooks ?? {}) as Record<string, unknown>;
+  const sessionStart = Array.isArray(hooks.SessionStart) ? [...hooks.SessionStart] : [];
+  const already = JSON.stringify(sessionStart).includes("docspack sync");
+  if (!already) sessionStart.push({ hooks: [{ type: "command", command }] });
+
+  const contents = `${JSON.stringify(
+    { ...settings, hooks: { ...hooks, SessionStart: sessionStart } },
+    null,
+    2,
+  )}\n`;
+
+  return {
+    path,
+    kind: "hook",
+    contents,
+    status: before === undefined ? "create" : before === contents ? "unchanged" : "update",
+    reason: "keeps the index in step with the lockfile without anyone remembering to sync",
+  };
+}
+
+async function mcpEntry(root: string): Promise<AgentFile> {
+  const path = ".mcp.json";
+  const before = await read(join(root, path));
+  const config = parseJson(before, path);
+  const servers = (config.mcpServers ?? {}) as Record<string, unknown>;
+  const contents = `${JSON.stringify(
+    {
+      ...config,
+      mcpServers: {
+        ...servers,
+        docspack: { command: "npx", args: ["-y", "docspack", "mcp"] },
+      },
+    },
+    null,
+    2,
+  )}\n`;
+
+  return {
+    path,
+    kind: "mcp",
+    contents,
+    status: before === undefined ? "create" : before === contents ? "unchanged" : "update",
+    reason: "a declared tool, for clients that would rather have one than a shell command",
+  };
+}
+
+function parseJson(raw: string | undefined, path: string): Record<string, unknown> {
+  if (raw === undefined || raw.trim().length === 0) return {};
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    if (typeof parsed !== "object" || parsed === null) throw new Error("not an object");
+    return parsed as Record<string, unknown>;
+  } catch (error) {
+    throw new DocspackError(`${path} is not valid JSON`, {
+      hint: "Fix it, or move it aside and run this again.",
+      cause: error,
+    });
+  }
+}
+
+async function read(path: string): Promise<string | undefined> {
+  try {
+    return await readFile(path, "utf8");
+  } catch {
+    return undefined;
+  }
+}

package/src/artifact.ts

@@ -0,0 +1,110 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { IndexedChunk } from "./db.js";
+import { chunkId, estimateTokens, packageId } from "./spec.js";
+import { declarationsIn, type ExportedSymbol, readPublicSurface } from "./surface.js";
+
+/**
+ * A package derived from an installed library rather than published by its author.
+ *
+ * Half of a well-documented library's exported names are mentioned nowhere in its own
+ * documentation (see `docs/design/beyond-retrieval.md`), and almost all of them are declared in
+ * the type declarations sitting in `node_modules`. Those declarations are the answer a
+ * documentation site cannot serve, because the site is not the artifact.
+ */
+export interface ArtifactPackage {
+  readonly id: string;
+  readonly name: string;
+  readonly version: string;
+  readonly chunks: readonly IndexedChunk[];
+  /**
+   * Every exported name, mapped to the chunk carrying its declaration — or to an empty string
+   * when the package exports the name without declaring it anywhere this can read.
+   *
+   * Recording those too is what keeps `changed` honest: a name re-exported from a file with no
+   * declaration in it would otherwise look like a name the release removed.
+   */
+  readonly symbols: ReadonlyMap<string, string>;
+}
+
+/** Beyond this a package is generated machinery, and indexing it would drown the prose. */
+const MAX_SYMBOLS = 2000;
+
+/** How much of a declaration is carried. Long enough for a signature, short enough to rank. */
+const DECLARATION_LIMIT = 400;
+
+/**
+ * Derives chunks from an installed library: one per exported name, carrying the import a caller
+ * writes and the declaration as the installed build states it.
+ *
+ * Returns nothing when the package ships no type declarations. That is the common case across a
+ * dependency tree and is not a problem worth reporting.
+ */
+export async function readArtifact(dir: string): Promise<ArtifactPackage | undefined> {
+  const surface = await readPublicSurface(dir);
+  if (surface === undefined || surface.name.length === 0 || surface.version.length === 0) {
+    return undefined;
+  }
+
+  const id = packageId(surface.name, surface.version);
+  const declared = new Map<string, Map<string, string>>();
+  const chunks: IndexedChunk[] = [];
+  const symbols = new Map<string, string>();
+
+  for (const symbol of surface.symbols.values()) {
+    symbols.set(symbol.name, "");
+    if (chunks.length >= MAX_SYMBOLS) continue;
+
+    let declarations = declared.get(symbol.file);
+    if (declarations === undefined) {
+      try {
+        declarations = declarationsIn(
+          await readFile(join(dir, symbol.file), "utf8"),
+          DECLARATION_LIMIT,
+        );
+      } catch {
+        declarations = new Map();
+      }
+      declared.set(symbol.file, declarations);
+    }
+
+    const declaration = declarations.get(symbol.name);
+    // A name that only appears in an `export { … }` list has no declaration to show. The import
+    // line alone would say nothing the name did not already say, so it is left out.
+    if (declaration === undefined) continue;
+
+    const content = render(symbol, declaration);
+    const chunk = chunkId(id, symbol.name);
+    chunks.push({
+      chunkId: chunk,
+      filePath: symbol.file,
+      tokens: estimateTokens(content),
+      content,
+      // The name is the tag, and tags are weighted 3× against prose in the ranking.
+      tags: [symbol.name, symbol.from],
+    });
+    symbols.set(symbol.name, chunk);
+  }
+
+  if (symbols.size === 0) return undefined;
+  return { id, name: surface.name, version: surface.version, chunks, symbols };
+}
+
+/**
+ * What an agent receives. The import line is first because it is the part a caller gets wrong
+ * most often — a name that exists at a subpath nobody guessed is the same failure as a name that
+ * does not exist.
+ */
+function render(symbol: ExportedSymbol, declaration: string): string {
+  return [
+    `# ${symbol.name}`,
+    "",
+    `Declared by \`${symbol.from}\`. This is the installed build's own declaration, not prose.`,
+    "",
+    "```ts",
+    `import { ${symbol.name} } from "${symbol.from}";`,
+    "",
+    declaration.trim(),
+    "```",
+  ].join("\n");
+}

package/src/build.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import type { Dirent } from "node:fs";
 import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import { join, relative, sep } from "node:path";
@@ -281,6 +282,13 @@ async function collectFromSource(
 
   const documents: SourceDocument[] = [];
   const links = fetchableLinks(parsed).slice(0, options.pages ?? 50);
+  // Two links can name the same document. `fetchableLinks` already drops a repeated URL and a
+  // repeated fragment, but a site whose index links anchors as a query — `/v4?id=codecs` — gets
+  // one entry per anchor, and each one fetches the whole page again. Left alone that packages
+  // the same text dozens of times: an index mostly made of duplicates, which is slower to search
+  // and ranks worse, because copies of one page crowd out the rest of the corpus.
+  const seen = new Set<string>();
+  let duplicates = 0;
 
   for (const link of links) {
     options.onProgress?.(`fetching ${link.url}`);
@@ -293,6 +301,12 @@ async function collectFromSource(
         warnings.push(`skipped ${link.url}: document was empty`);
         continue;
       }
+      const fingerprint = createHash("sha256").update(text).digest("hex");
+      if (seen.has(fingerprint)) {
+        duplicates += 1;
+        continue;
+      }
+      seen.add(fingerprint);
       documents.push({
         title: link.title.length > 0 ? link.title : (firstHeading(text) ?? link.url),
         origin: link.url,
@@ -306,6 +320,11 @@ async function collectFromSource(
     }
   }
 
+  if (duplicates > 0) {
+    warnings.push(
+      `skipped ${duplicates} of ${links.length} linked documents whose text repeated one already packaged`,
+    );
+  }
   if (fetchableLinks(parsed).length > links.length) {
     warnings.push(
       `packaged ${links.length} of ${fetchableLinks(parsed).length} linked documents (raise with --pages)`,

package/src/changed.ts

@@ -0,0 +1,99 @@
+import { identifiers } from "./coverage.js";
+import type { Store } from "./db.js";
+import { discoverLibraries, discoverPackages } from "./discovery.js";
+import { DocspackError } from "./errors.js";
+import { packageId } from "./spec.js";
+
+export interface SurfaceChange {
+  readonly library: string;
+  readonly from: string;
+  readonly to: string;
+  /** Names the newer release exports that the older one did not. */
+  readonly added: readonly string[];
+  /** Names the older release exported that the newer one does not. */
+  readonly removed: readonly string[];
+  /**
+   * Added names that no documentation package installed here mentions. These are the ones a
+   * model cannot know: too new for its training data, and absent from what the vendor published.
+   */
+  readonly undocumented: readonly string[];
+}
+
+export interface ChangedOptions {
+  readonly cwd: string;
+  readonly store: Store;
+  /** Library name, optionally with the version to compare against: `hono` or `hono@4.0.0`. */
+  readonly library: string;
+}
+
+/**
+ * What changed in a library's exported surface between two versions in the store.
+ *
+ * Both versions come from the local index, never the network: the store is global, so a version
+ * indexed for another project on this machine is already here. Measured across two libraries,
+ * upgrades are overwhelmingly additive — hono 3.12.12 to 4.13.3 removed three names and added
+ * 243 — so the useful report is what exists now that an older release did not have.
+ */
+export async function changedSurface(options: ChangedOptions): Promise<SurfaceChange> {
+  const [name, requested] = split(options.library);
+
+  const installed = (await discoverLibraries(options.cwd)).find((library) => library.name === name);
+  const indexed = options.store.versionsOf(name).filter((pkg) => pkg.kind === "artifact");
+  if (indexed.length === 0) {
+    throw new DocspackError(`No version of ${name} is indexed`, {
+      hint:
+        installed === undefined
+          ? `${name} is not a dependency of this project.`
+          : "Run `docspack sync` to index the installed version, then ask again.",
+    });
+  }
+
+  const to = installed?.version ?? indexed[indexed.length - 1]?.version;
+  if (to === undefined) throw new DocspackError(`No version of ${name} is indexed`);
+
+  const others = indexed.map((pkg) => pkg.version).filter((version) => version !== to);
+  const from = requested ?? others[others.length - 1];
+  if (from === undefined) {
+    throw new DocspackError(`Only ${name}@${to} is indexed, so there is nothing to compare`, {
+      hint: "Index another version by running `docspack sync` in a project that installs it. The store is shared across projects on this machine.",
+    });
+  }
+  if (!options.store.hasPackage(packageId(name, from))) {
+    throw new DocspackError(`${name}@${from} is not in the store`, {
+      hint: `Indexed versions: ${indexed.map((pkg) => pkg.version).join(", ")}`,
+    });
+  }
+
+  const before = new Set(options.store.symbolsOf(packageId(name, from)));
+  const after = new Set(options.store.symbolsOf(packageId(name, to)));
+  const added = [...after].filter((symbol) => !before.has(symbol)).sort();
+  const removed = [...before].filter((symbol) => !after.has(symbol)).sort();
+
+  const documented = await documentedNames(options);
+  return {
+    library: name,
+    from,
+    to,
+    added,
+    removed,
+    undocumented: added.filter((symbol) => !documented.has(symbol)),
+  };
+}
+
+/** Every identifier any documentation package installed here mentions. */
+async function documentedNames(options: ChangedOptions): Promise<Set<string>> {
+  const names = new Set<string>();
+  for (const pkg of (await discoverPackages(options.cwd)).packages) {
+    for (const chunk of options.store.chunksOf(pkg.id)) {
+      for (const identifier of identifiers(chunk.content)) names.add(identifier);
+    }
+  }
+  return names;
+}
+
+/** `hono@4.0.0` into its parts, keeping a scope's leading `@` out of it. */
+function split(spec: string): [name: string, version: string | undefined] {
+  const at = spec.lastIndexOf("@");
+  if (at <= 0) return [spec, undefined];
+  return [spec.slice(0, at), spec.slice(at + 1)];
+}