docspack 0.0.1 → 0.1.0

package/src/config.ts

@@ -0,0 +1,158 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { DocspackError } from "./errors.js";
+import { type FindingKind, isFindingKind, KINDS } from "./kinds.js";
+
+/**
+ * Where a vendor has asked to receive documentation problems.
+ *
+ * Declaring this is the only thing that lets `docspack feedback submit` route a finding
+ * anywhere. A package without it collects findings locally and routes nothing, which is the
+ * correct default: a maintainer who has not asked for this gets nothing.
+ */
+export interface FeedbackChannel {
+  /** `owner/repo` on GitHub. Reports become a prefilled issue URL for a human to open. */
+  readonly github: string;
+  readonly labels?: readonly string[];
+  /** Kinds this vendor will accept. Omitted means all of them. */
+  readonly accepts?: readonly FindingKind[];
+  /** The vendor's own rules for reporting, shown before anything is filed. */
+  readonly policy?: string;
+}
+
+/**
+ * Build settings read from the `docspack` key of a package.json. Having them on disk keeps the
+ * build script, the release workflow and `prepublishOnly` from drifting apart.
+ */
+export interface BuildConfig {
+  readonly from?: string;
+  readonly openapi?: string;
+  readonly source?: string;
+  /** The library this package documents, e.g. `@acme/sdk`. What `docspack verify` checks against. */
+  readonly documents?: string;
+  readonly feedback?: FeedbackChannel;
+  readonly maxChunkTokens?: number;
+  readonly pages?: number;
+}
+
+export const CONFIG_KEY = "docspack";
+
+export async function readBuildConfig(dir: string): Promise<BuildConfig> {
+  let raw: string;
+  try {
+    raw = await readFile(join(dir, "package.json"), "utf8");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return {};
+    throw error;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return {};
+  }
+
+  const value = (parsed as Record<string, unknown>)[CONFIG_KEY];
+  if (value === undefined) return {};
+  return parseBuildConfig(value);
+}
+
+export function parseBuildConfig(value: unknown): BuildConfig {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new DocspackError(`package.json: "${CONFIG_KEY}" must be an object`);
+  }
+  const raw = value as Record<string, unknown>;
+
+  const text = (key: "from" | "openapi" | "source" | "documents"): string | undefined => {
+    const found = raw[key];
+    if (found === undefined) return undefined;
+    if (typeof found !== "string" || found.length === 0) {
+      throw new DocspackError(`package.json: "${CONFIG_KEY}.${key}" must be a non-empty string`);
+    }
+    return found;
+  };
+
+  const count = (key: "maxChunkTokens" | "pages"): number | undefined => {
+    const found = raw[key];
+    if (found === undefined) return undefined;
+    if (!Number.isInteger(found) || (found as number) < 1) {
+      throw new DocspackError(`package.json: "${CONFIG_KEY}.${key}" must be a positive integer`);
+    }
+    return found as number;
+  };
+
+  const from = text("from");
+  const openapi = text("openapi");
+  const source = text("source");
+  const documents = text("documents");
+  const feedback = parseFeedbackChannel(raw.feedback);
+  const maxChunkTokens = count("maxChunkTokens");
+  const pages = count("pages");
+
+  return {
+    ...(from === undefined ? {} : { from }),
+    ...(openapi === undefined ? {} : { openapi }),
+    ...(source === undefined ? {} : { source }),
+    ...(documents === undefined ? {} : { documents }),
+    ...(feedback === undefined ? {} : { feedback }),
+    ...(maxChunkTokens === undefined ? {} : { maxChunkTokens }),
+    ...(pages === undefined ? {} : { pages }),
+  };
+}
+
+/** `owner/repo`, the only shape a prefilled issue URL can be built from. */
+const GITHUB_REPO = /^[\w.-]+\/[\w.-]+$/;
+
+/**
+ * A malformed channel is an error rather than a shrug. Getting this wrong silently would
+ * either route a report to the wrong place or quietly route nothing, and both are worse than
+ * refusing to read the package.
+ */
+export function parseFeedbackChannel(value: unknown): FeedbackChannel | undefined {
+  if (value === undefined) return undefined;
+
+  const where = `${CONFIG_KEY}.feedback`;
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new DocspackError(`package.json: "${where}" must be an object`);
+  }
+  const raw = value as Record<string, unknown>;
+
+  const github = raw.github;
+  if (typeof github !== "string" || !GITHUB_REPO.test(github)) {
+    throw new DocspackError(`package.json: "${where}.github" must look like "owner/repo"`, {
+      hint: "GitHub is the only channel docspack can build a prefilled issue URL for.",
+    });
+  }
+
+  const labels = stringList(raw.labels, `${where}.labels`);
+  const policy = raw.policy;
+  if (policy !== undefined && (typeof policy !== "string" || policy.length === 0)) {
+    throw new DocspackError(`package.json: "${where}.policy" must be a URL`);
+  }
+
+  const accepts = stringList(raw.accepts, `${where}.accepts`);
+  if (accepts !== undefined) {
+    const unknown = accepts.filter((kind) => !isFindingKind(kind));
+    if (unknown.length > 0) {
+      throw new DocspackError(`package.json: "${where}.accepts" has unknown kinds: ${unknown}`, {
+        hint: `Use any of: ${KINDS.join(", ")}.`,
+      });
+    }
+  }
+
+  return {
+    github,
+    ...(labels === undefined ? {} : { labels }),
+    ...(accepts === undefined ? {} : { accepts: accepts as FindingKind[] }),
+    ...(policy === undefined ? {} : { policy }),
+  };
+}
+
+function stringList(value: unknown, where: string): string[] | undefined {
+  if (value === undefined) return undefined;
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
+    throw new DocspackError(`package.json: "${where}" must be an array of strings`);
+  }
+  return value as string[];
+}

package/src/db.ts

@@ -0,0 +1,261 @@
+import { mkdirSync } from "node:fs";
+import { createRequire } from "node:module";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import type { DatabaseSync } from "node:sqlite";
+import { DocspackError } from "./errors.js";
+
+const require = createRequire(import.meta.url);
+
+export interface IndexedPackage {
+  readonly id: string;
+  readonly name: string;
+  readonly version: string;
+  readonly indexedAt?: string;
+}
+
+export interface IndexedChunk {
+  readonly chunkId: string;
+  readonly filePath: string;
+  readonly tokens: number;
+  readonly content: string;
+  readonly tags: readonly string[];
+}
+
+export interface SearchHit {
+  readonly chunkId: string;
+  readonly packageId: string;
+  readonly filePath: string;
+  readonly tokens: number;
+  readonly content: string;
+}
+
+export interface SearchOptions {
+  /** Restrict to these package ids. An empty array matches nothing. */
+  readonly packageIds?: readonly string[];
+  /** SQL LIKE pattern applied to package_id, e.g. `@stripe/%`. */
+  readonly packageFilter?: string;
+  readonly limit?: number;
+  /** Stop returning chunks once this many tokens have been collected. */
+  readonly maxTokens?: number;
+}
+
+const SCHEMA_VERSION = 1;
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS packages (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    version TEXT NOT NULL,
+    indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    chunk_id TEXT PRIMARY KEY,
+    package_id TEXT REFERENCES packages(id),
+    file_path TEXT NOT NULL,
+    tokens INTEGER,
+    content TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS chunks_by_package ON chunks(package_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+    content,
+    tags,
+    package_id UNINDEXED,
+    chunk_id UNINDEXED,
+    tokenize="porter unicode61"
+);
+`;
+
+/**
+ * `node:sqlite` is still flagged experimental on Node 22 and warns when it is loaded. The database
+ * is an implementation detail of this tool, so the warning is noise for its users.
+ */
+export function silenceSqliteWarning(): void {
+  const original = process.emitWarning.bind(process);
+  process.emitWarning = ((warning: string | Error, ...rest: unknown[]): void => {
+    const message = typeof warning === "string" ? warning : warning.message;
+    if (message.includes("SQLite is an experimental feature")) return;
+    (original as (...args: unknown[]) => void)(warning, ...rest);
+  }) as typeof process.emitWarning;
+}
+
+/**
+ * Loads `node:sqlite` on first use. The warning above is emitted when the module loads, so this
+ * must not be a static import: that would run before anything has a chance to silence it.
+ */
+function loadSqlite(): typeof import("node:sqlite") {
+  silenceSqliteWarning();
+  return require("node:sqlite") as typeof import("node:sqlite");
+}
+
+/** Default location of the shared index, mirroring pnpm's global store. */
+export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
+  const override = env.DOCSPACK_STORE;
+  if (override !== undefined && override.length > 0) return override;
+  return join(homedir(), ".docspack", "store.db");
+}
+
+/**
+ * Turns free text into an FTS5 MATCH expression. Every term is quoted, so punctuation in a user
+ * query can never be interpreted as FTS syntax.
+ */
+export function toFtsQuery(raw: string): string {
+  const terms = raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
+  if (terms.length === 0) {
+    throw new DocspackError(`"${raw}" contains nothing searchable`, {
+      hint: "Search for words, for example: docspack search webhook signature",
+    });
+  }
+  return terms.map((term) => `"${term}"`).join(" OR ");
+}
+
+/** The global chunk index: one SQLite database shared by every project on the machine. */
+export class Store {
+  readonly #db: DatabaseSync;
+
+  private constructor(db: DatabaseSync) {
+    this.#db = db;
+    this.#db.exec("PRAGMA journal_mode = WAL");
+    this.#db.exec(SCHEMA);
+    this.#db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
+  }
+
+  static open(path: string = defaultStorePath()): Store {
+    if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
+    try {
+      return new Store(new (loadSqlite().DatabaseSync)(path));
+    } catch (error) {
+      throw new DocspackError(`Could not open the docspack store at ${path}`, {
+        hint: "Check the path is writable, or set DOCSPACK_STORE to another location.",
+        cause: error,
+      });
+    }
+  }
+
+  hasPackage(id: string): boolean {
+    return this.#db.prepare("SELECT 1 FROM packages WHERE id = ?").get(id) !== undefined;
+  }
+
+  listPackages(): IndexedPackage[] {
+    const rows = this.#db
+      .prepare("SELECT id, name, version, indexed_at FROM packages ORDER BY name, version")
+      .all() as { id: string; name: string; version: string; indexed_at: string }[];
+    return rows.map((row) => ({
+      id: row.id,
+      name: row.name,
+      version: row.version,
+      indexedAt: row.indexed_at,
+    }));
+  }
+
+  countChunks(packageId: string): number {
+    const row = this.#db
+      .prepare("SELECT COUNT(*) AS total FROM chunks WHERE package_id = ?")
+      .get(packageId) as { total: number };
+    return row.total;
+  }
+
+  /** Replaces a package and all of its chunks in a single transaction. */
+  indexPackage(pkg: IndexedPackage, chunks: readonly IndexedChunk[]): void {
+    const insertPackage = this.#db.prepare(
+      "INSERT OR REPLACE INTO packages (id, name, version) VALUES (?, ?, ?)",
+    );
+    const insertChunk = this.#db.prepare(
+      "INSERT OR REPLACE INTO chunks (chunk_id, package_id, file_path, tokens, content) VALUES (?, ?, ?, ?, ?)",
+    );
+    const insertFts = this.#db.prepare(
+      "INSERT INTO chunks_fts (content, tags, package_id, chunk_id) VALUES (?, ?, ?, ?)",
+    );
+
+    this.#db.exec("BEGIN");
+    try {
+      this.#deleteChunks(pkg.id);
+      insertPackage.run(pkg.id, pkg.name, pkg.version);
+      for (const chunk of chunks) {
+        insertChunk.run(chunk.chunkId, pkg.id, chunk.filePath, chunk.tokens, chunk.content);
+        insertFts.run(chunk.content, chunk.tags.join(" "), pkg.id, chunk.chunkId);
+      }
+      this.#db.exec("COMMIT");
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+
+  removePackage(id: string): void {
+    this.#db.exec("BEGIN");
+    try {
+      this.#deleteChunks(id);
+      this.#db.prepare("DELETE FROM packages WHERE id = ?").run(id);
+      this.#db.exec("COMMIT");
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+
+  /**
+   * Full-text search, ranked by bm25. Results stop early once `maxTokens` is reached, so a caller
+   * can bound how much context it hands to a model.
+   */
+  search(query: string, options: SearchOptions = {}): SearchHit[] {
+    const limit = options.limit ?? 3;
+    const conditions = ["chunks_fts MATCH ?"];
+    const parameters: (string | number)[] = [toFtsQuery(query)];
+
+    if (options.packageIds !== undefined) {
+      if (options.packageIds.length === 0) return [];
+      conditions.push(`f.package_id IN (${options.packageIds.map(() => "?").join(", ")})`);
+      parameters.push(...options.packageIds);
+    }
+    if (options.packageFilter !== undefined) {
+      conditions.push("f.package_id LIKE ?");
+      parameters.push(options.packageFilter);
+    }
+
+    const rows = this.#db
+      .prepare(
+        `SELECT f.chunk_id AS chunkId, f.package_id AS packageId, c.file_path AS filePath,
+                c.tokens AS tokens, c.content AS content
+           FROM chunks_fts f
+           JOIN chunks c ON c.chunk_id = f.chunk_id
+          WHERE ${conditions.join(" AND ")}
+          ORDER BY bm25(chunks_fts)
+          LIMIT ?`,
+      )
+      .all(...parameters, limit)
+      .map(
+        (row): SearchHit => ({
+          chunkId: String(row.chunkId),
+          packageId: String(row.packageId),
+          filePath: String(row.filePath),
+          tokens: Number(row.tokens ?? 0),
+          content: String(row.content),
+        }),
+      );
+
+    if (options.maxTokens === undefined) return rows;
+
+    const budgeted: SearchHit[] = [];
+    let spent = 0;
+    for (const row of rows) {
+      const tokens = row.tokens > 0 ? row.tokens : 1;
+      if (budgeted.length > 0 && spent + tokens > options.maxTokens) break;
+      budgeted.push(row);
+      spent += tokens;
+    }
+    return budgeted;
+  }
+
+  close(): void {
+    this.#db.close();
+  }
+
+  #deleteChunks(packageId: string): void {
+    this.#db.prepare("DELETE FROM chunks_fts WHERE package_id = ?").run(packageId);
+    this.#db.prepare("DELETE FROM chunks WHERE package_id = ?").run(packageId);
+  }
+}

package/src/discovery.ts

@@ -0,0 +1,161 @@
+import { readFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { DocspackError } from "./errors.js";
+import {
+  isCommunityPackage,
+  isDocsPackage,
+  LLMS_DIR,
+  MANIFEST_FILE,
+  type PackageManifest,
+  packageId,
+  parseManifest,
+} from "./spec.js";
+
+export interface DiscoveredPackage {
+  readonly id: string;
+  readonly name: string;
+  readonly version: string;
+  /** Root of the installed package. */
+  readonly dir: string;
+  readonly llmsDir: string;
+  readonly manifest: PackageManifest;
+  /** False for `@docspack-community/*`, whose contents nobody has vetted. */
+  readonly trusted: boolean;
+}
+
+export interface Discovery {
+  readonly packages: readonly DiscoveredPackage[];
+  /** Human-readable problems that did not stop discovery, e.g. a declared but uninstalled package. */
+  readonly problems: readonly string[];
+}
+
+/**
+ * Finds the documentation packages a project depends on: every `@vendor/docspack` or
+ * `@docspack-community/*` entry in `dependencies` or `devDependencies` that is installed and
+ * ships a `.llms/manifest.json`.
+ */
+export async function discoverPackages(cwd: string): Promise<Discovery> {
+  const root = resolve(cwd);
+  const declared = await declaredDocsPackages(root);
+  const packages: DiscoveredPackage[] = [];
+  const problems: string[] = [];
+
+  for (const name of declared) {
+    const dir = await resolvePackageDir(name, root);
+    if (dir === undefined) {
+      problems.push(`${name} is declared in package.json but is not installed`);
+      continue;
+    }
+
+    let version: string;
+    try {
+      version = await installedVersion(name, dir);
+    } catch (error) {
+      problems.push(describe(error));
+      continue;
+    }
+
+    const llmsDir = join(dir, LLMS_DIR);
+    let manifest: PackageManifest;
+    try {
+      const raw = await readFile(join(llmsDir, MANIFEST_FILE), "utf8");
+      manifest = parseManifest(JSON.parse(raw), `${name} ${LLMS_DIR}/${MANIFEST_FILE}`);
+    } catch (error) {
+      problems.push(
+        (error as NodeJS.ErrnoException).code === "ENOENT"
+          ? `${name} has no ${LLMS_DIR}/${MANIFEST_FILE} and cannot be indexed`
+          : describe(error),
+      );
+      continue;
+    }
+
+    if (manifest.version !== version) {
+      problems.push(
+        `${name}: manifest says version ${manifest.version}, package.json says ${version}; using ${version}`,
+      );
+    }
+
+    packages.push({
+      id: packageId(name, version),
+      name,
+      version,
+      dir,
+      llmsDir,
+      manifest,
+      trusted: !isCommunityPackage(name),
+    });
+  }
+
+  packages.sort((a, b) => a.name.localeCompare(b.name));
+  return { packages, problems };
+}
+
+/** Package ids installed in this project, used to scope queries to the versions actually in use. */
+export async function projectPackageIds(cwd: string): Promise<string[]> {
+  const { packages } = await discoverPackages(cwd);
+  return packages.map((pkg) => pkg.id);
+}
+
+async function declaredDocsPackages(cwd: string): Promise<string[]> {
+  let raw: string;
+  try {
+    raw = await readFile(join(cwd, "package.json"), "utf8");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      throw new DocspackError(`No package.json found in ${cwd}`, {
+        hint: "Run docspack from a project directory, or pass --cwd <dir>.",
+      });
+    }
+    throw error;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    throw new DocspackError(`${join(cwd, "package.json")} is not valid JSON`, { cause: error });
+  }
+
+  const root = parsed as { dependencies?: unknown; devDependencies?: unknown };
+  const names = new Set<string>();
+  for (const field of [root.dependencies, root.devDependencies]) {
+    if (typeof field !== "object" || field === null) continue;
+    for (const name of Object.keys(field as Record<string, unknown>)) {
+      if (isDocsPackage(name)) names.add(name);
+    }
+  }
+  return [...names].sort();
+}
+
+/**
+ * Walks `node_modules` upwards from the project, which resolves npm, pnpm (through its symlinks)
+ * and workspace layouts without depending on a package's export map.
+ */
+export async function resolvePackageDir(name: string, cwd: string): Promise<string | undefined> {
+  let dir = cwd;
+  while (true) {
+    const candidate = join(dir, "node_modules", ...name.split("/"));
+    try {
+      await readFile(join(candidate, "package.json"), "utf8");
+      return candidate;
+    } catch {
+      // Not here; keep walking up.
+    }
+    const parent = dirname(dir);
+    if (parent === dir) return undefined;
+    dir = parent;
+  }
+}
+
+async function installedVersion(name: string, dir: string): Promise<string> {
+  const parsed: unknown = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
+  const version = (parsed as { version?: unknown }).version;
+  if (typeof version !== "string" || version.length === 0) {
+    throw new DocspackError(`${name} has no version in its package.json`);
+  }
+  return version;
+}
+
+function describe(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}