docspack 0.1.1 → 0.2.0

package/src/cli.ts

@@ -9,7 +9,13 @@ import { type DoctorReport, runDoctor } from "./doctor.js";
 import { DocspackError } from "./errors.js";
 import { renderTree } from "./init/write.js";
 import { previewPackage } from "./preview.js";
-import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS, queryDocs, renderAnswer } from "./search.js";
+import {
+  DEFAULT_LIMIT,
+  DEFAULT_MAX_TOKENS,
+  type QueryResult,
+  queryDocs,
+  renderAnswer,
+} from "./search.js";
 import { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
 import { syncProject } from "./sync.js";
 import { verifyProject } from "./verify.js";
@@ -46,6 +52,7 @@ const OPTIONS = {
   yes: { type: "boolean", short: "y" },
   "dry-run": { type: "boolean" },
   strict: { type: "boolean" },
+  pedantic: { type: "boolean" },
   "package-dir": { type: "string" },
   chunk: { type: "string" },
   kind: { type: "string" },
@@ -55,6 +62,14 @@ const OPTIONS = {
   repro: { type: "string" },
 } as const;
 
+/**
+ * A query that returned nothing is not a failure, but "you have not indexed anything yet" and
+ * "the documentation does not cover this" are different answers and a wrapper has to tell them
+ * apart. Both are non-zero; neither collides with 1 (failed) or 2 (used wrongly).
+ */
+const EXIT_NOT_INDEXED = 3;
+const EXIT_NO_MATCH = 4;
+
 const HELP = `docspack — local, version-locked documentation for AI agents
 
 Usage
@@ -134,7 +149,8 @@ Options
   --no-workflow      init: skip the release workflow
   --no-build         init: scaffold only
   --template <t>     init: full (default) or minimal
-  --strict           doctor: treat warnings as failures
+  --strict           doctor: treat structural warnings as failures
+  --pedantic         doctor: --strict, and fail on prose style as well
   --package-dir <d>  doctor, preview, verify: the package to inspect (default: .)
   --chunk <id>       feedback add: the chunk the problem is in
   --kind <k>         feedback add: drift, incorrect or missing
@@ -143,10 +159,11 @@ Options
   --actual <text>    feedback add: what happened instead
   --repro <code>     feedback add: code that demonstrates it
   --force            sync: re-index packages already in the store; init: overwrite files
-  -p, --package <s>  search: only packages whose name contains this text
-  --limit <n>        search: maximum chunks to return (default ${DEFAULT_LIMIT})
-  --max-tokens <n>   search: token ceiling for the result set (default ${DEFAULT_MAX_TOKENS})
-  --all              search: the whole store, not just this project;
+  -p, --package <s>  ask, search: only packages whose name contains this text
+  --limit <n>        ask, search, preview: maximum chunks to return (default ${DEFAULT_LIMIT})
+  --max-tokens <n>   ask, search, preview: token ceiling for the result set
+                     (default ${DEFAULT_MAX_TOKENS})
+  --all              ask, search: the whole store, not just this project;
                      feedback remove: every recorded finding
   --from <dir>       build: directory of Markdown to package
   --openapi <file>   build: OpenAPI JSON to package, one chunk per operation
@@ -161,6 +178,15 @@ Options
   -h, --help         Show this help
   -v, --version      Show the version
 
+Exit codes
+  \`ask\` and \`search\` answer with an exit code a script can branch on:
+
+    0  chunks were returned
+    ${EXIT_NOT_INDEXED}  nothing matched, and a docs package is installed but not indexed —
+       run \`docspack sync\`
+    ${EXIT_NO_MATCH}  nothing matched, and everything installed is already indexed
+    1  the command failed; 2  the command was used wrongly
+
 Examples
   npx docspack sync
   npx docspack ask "how do I verify a webhook signature"
@@ -239,6 +265,12 @@ function reportDoctor(report: DoctorReport, quiet: boolean): void {
   }
 }
 
+/** 0 when the query was answered; otherwise which of the two empty answers this was. */
+function queryExit(result: QueryResult): number {
+  if (result.hits.length > 0) return 0;
+  return result.unindexed.length > 0 ? EXIT_NOT_INDEXED : EXIT_NO_MATCH;
+}
+
 function openStore(values: Values): Store {
   return Store.open(values.store ?? defaultStorePath());
 }
@@ -358,8 +390,7 @@ async function main(argv: readonly string[]): Promise<number> {
           // Exactly what the MCP tool returns, so both interfaces answer identically.
           process.stdout.write(`${renderAnswer(result, question)}\n`);
         }
-        // Answering "nothing matched" is a successful answer, not a failed command.
-        return 0;
+        return queryExit(result);
       } finally {
         store.close();
       }
@@ -389,11 +420,16 @@ async function main(argv: readonly string[]): Promise<number> {
 
         if (json) {
           process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
-          return 0;
+          return queryExit(result);
         }
         if (result.hits.length === 0) {
           process.stdout.write(`No local documentation matched "${query}".\n`);
-          return 1;
+          if (result.unindexed.length > 0) {
+            process.stderr.write(
+              `${yellow("!")} ${result.unindexed.join(", ")} installed but not indexed. Run \`docspack sync\`.\n`,
+            );
+          }
+          return queryExit(result);
         }
 
         for (const hit of result.hits) {
@@ -793,6 +829,7 @@ async function main(argv: readonly string[]): Promise<number> {
       const report = await runDoctor({
         dir,
         ...(values.strict === true ? { strict: true } : {}),
+        ...(values.pedantic === true ? { pedantic: true } : {}),
       });
 
       if (json) {

package/src/db.ts

@@ -101,6 +101,11 @@ export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
 /**
  * Turns free text into an FTS5 MATCH expression. Every term is quoted, so punctuation in a user
  * query can never be interpreted as FTS syntax.
+ *
+ * Terms of one or two characters are dropped. Questions are mostly `how`, `do`, `i`, `in`, and
+ * every chunk containing one becomes a candidate — which is how a chunk that merely quotes an
+ * example question outranks the chunk that answers it. Two characters is the cut because it
+ * leaves real short names (`id`, `db`) out and keeps three-letter ones (`api`, `key`) in.
  */
 export function toFtsQuery(raw: string): string {
   const terms = raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
@@ -109,7 +114,9 @@ export function toFtsQuery(raw: string): string {
       hint: "Search for words, for example: docspack search webhook signature",
     });
   }
-  return terms.map((term) => `"${term}"`).join(" OR ");
+  const meaningful = terms.filter((term) => term.length > 2);
+  // A query made only of short terms still has to search for something.
+  return (meaningful.length > 0 ? meaningful : terms).map((term) => `"${term}"`).join(" OR ");
 }
 
 /** The global chunk index: one SQLite database shared by every project on the machine. */
@@ -223,7 +230,9 @@ export class Store {
            FROM chunks_fts f
            JOIN chunks c ON c.chunk_id = f.chunk_id
           WHERE ${conditions.join(" AND ")}
-          ORDER BY bm25(chunks_fts)
+          -- Tags are what an author writes to aim a chunk, and front matter did nothing at all
+          -- while they scored the same as prose. Weights are (content, tags).
+          ORDER BY bm25(chunks_fts, 1.0, 3.0)
           LIMIT ?`,
       )
       .all(...parameters, limit)

package/src/discovery.ts

@@ -96,33 +96,55 @@ export async function projectPackageIds(cwd: string): Promise<string[]> {
   return packages.map((pkg) => pkg.id);
 }
 
+/**
+ * Every docs package declared by this directory or by an ancestor of it. A workspace declares
+ * shared tooling in the repository root and its members inherit the install, so reading only
+ * `cwd/package.json` answers "this project has no documentation" for most monorepos — the same
+ * upward walk `resolvePackageDir` already does for the install.
+ */
 async function declaredDocsPackages(cwd: string): Promise<string[]> {
-  let raw: string;
-  try {
-    raw = await readFile(join(cwd, "package.json"), "utf8");
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      throw new DocspackError(`No package.json found in ${cwd}`, {
-        hint: "Run docspack from a project directory, or pass --cwd <dir>.",
-      });
+  const names = new Set<string>();
+  let found = false;
+  let dir = cwd;
+
+  while (true) {
+    const path = join(dir, "package.json");
+    let raw: string;
+    try {
+      raw = await readFile(path, "utf8");
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      const parent = dirname(dir);
+      if (parent === dir) break;
+      dir = parent;
+      continue;
     }
-    throw error;
-  }
 
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(raw);
-  } catch (error) {
-    throw new DocspackError(`${join(cwd, "package.json")} is not valid JSON`, { cause: error });
-  }
+    found = true;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch (error) {
+      throw new DocspackError(`${path} is not valid JSON`, { cause: error });
+    }
 
-  const root = parsed as { dependencies?: unknown; devDependencies?: unknown };
-  const names = new Set<string>();
-  for (const field of [root.dependencies, root.devDependencies]) {
-    if (typeof field !== "object" || field === null) continue;
-    for (const name of Object.keys(field as Record<string, unknown>)) {
-      if (isDocsPackage(name)) names.add(name);
+    const manifest = parsed as { dependencies?: unknown; devDependencies?: unknown };
+    for (const field of [manifest.dependencies, manifest.devDependencies]) {
+      if (typeof field !== "object" || field === null) continue;
+      for (const name of Object.keys(field as Record<string, unknown>)) {
+        if (isDocsPackage(name)) names.add(name);
+      }
     }
+
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+
+  if (!found) {
+    throw new DocspackError(`No package.json found in ${cwd}`, {
+      hint: "Run docspack from a project directory, or pass --cwd <dir>.",
+    });
   }
   return [...names].sort();
 }

package/src/doctor.ts

@@ -40,6 +40,12 @@ export interface DoctorOptions {
   readonly dir: string;
   /** Treat warnings as failures. Used by prepublishOnly and CI. */
   readonly strict?: boolean;
+  /**
+   * Also fail on prose style — filler and long sentences. Implies `strict`. Off by default
+   * because `--strict` is the gate `init` scaffolds, and prose written by a human is not a
+   * reason to block a first publish.
+   */
+  readonly pedantic?: boolean;
 }
 
 /** Over this, a chunk crowds out the response budget; under it, a chunk answers nothing. */
@@ -53,6 +59,9 @@ const MIN_CHUNK_TOKENS = 30;
 export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
   const findings: Finding[] = [];
   const llmsDir = join(options.dir, LLMS_DIR);
+  // Structural checks describe a package that will not retrieve properly; the two prose checks
+  // are opinions about writing. Only the first kind may block a publish by default.
+  const prose: Severity = options.pedantic === true ? "warn" : "info";
 
   const pkg = await readJson(join(options.dir, "package.json"));
   const manifest = await loadManifest(llmsDir, findings);
@@ -145,7 +154,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
         .join(", ");
       findings.push({
         check: "filler",
-        severity: "warn",
+        severity: prose,
         message: `chunk "${chunk.id}" contains narration a model does not need: ${listed}`,
         where: chunk.file,
         fix: "Delete it. Nobody searches for these words and they carry no fact.",
@@ -156,7 +165,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
     if (long.length > 0) {
       findings.push({
         check: "long-sentence",
-        severity: "warn",
+        severity: prose,
         message: `chunk "${chunk.id}" has ${plural(long.length, "sentence")} over 40 words`,
         where: chunk.file,
         fix: "Split them. One claim per sentence retrieves and reads better.",
@@ -209,9 +218,9 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
     });
   }
 
+  const strict = options.strict === true || options.pedantic === true;
   const failed = findings.some(
-    (finding) =>
-      finding.severity === "error" || (options.strict === true && finding.severity === "warn"),
+    (finding) => finding.severity === "error" || (strict && finding.severity === "warn"),
   );
 
   return { ok: !failed, findings, chunks: manifest.chunks.length, tokens };

package/src/index.ts

@@ -103,6 +103,7 @@ export {
   parseManifest,
   resolveChunkFile,
   SCHEMA_URL,
+  SPEC_URL,
   serializeManifest,
 } from "./spec.js";
 export {

package/src/search.ts

@@ -32,13 +32,21 @@ export interface QueryResult {
   readonly tokens: number;
   /** True when any hit came from an unvetted community package. */
   readonly untrusted: boolean;
+  /**
+   * Packages this project depends on that are installed but absent from the store. Nothing they
+   * document can match until `docspack sync` runs, which is a different answer from "nothing
+   * covers this question".
+   */
+  readonly unindexed: readonly string[];
 }
 
 export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
   const scoped = options.scoped !== false;
-  const packageIds = scoped
-    ? (await discoverPackages(options.cwd)).packages.map((pkg) => pkg.id)
-    : undefined;
+  const installed = scoped ? (await discoverPackages(options.cwd)).packages : undefined;
+  const packageIds = installed?.map((pkg) => pkg.id);
+  const unindexed = (installed ?? [])
+    .filter((pkg) => !options.store.hasPackage(pkg.id))
+    .map((pkg) => pkg.id);
 
   const hits = options.store
     .search(options.query, {
@@ -58,6 +66,7 @@ export async function queryDocs(options: QueryOptions): Promise<QueryResult> {
     hits,
     tokens: hits.reduce((total, hit) => total + hit.tokens, 0),
     untrusted: hits.some((hit) => !hit.trusted),
+    unindexed,
   };
 }
 
@@ -71,6 +80,11 @@ const UNTRUSTED_NOTICE =
  */
 export function renderAnswer(result: QueryResult, query: string): string {
   if (result.hits.length === 0) {
+    // Forgetting `sync` is the likeliest first mistake, and the tool can tell it apart from a
+    // question nothing covers: it can see what is installed and what is in the store.
+    if (result.unindexed.length > 0) {
+      return `No local documentation matched "${query}", and ${result.unindexed.join(", ")} ${result.unindexed.length === 1 ? "is" : "are"} installed but not indexed. Run \`docspack sync\`, then ask again.`;
+    }
     return `No local documentation matched "${query}". The project may not depend on a docspack package covering it.`;
   }
 

package/src/spec.ts

@@ -6,6 +6,8 @@ export const LLMS_DIR = ".llms";
 export const MANIFEST_FILE = "manifest.json";
 export const CHUNKS_DIR = "chunks";
 export const SCHEMA_URL = "https://docspack.dev/schema/v1.json";
+/** Prose specification of the package format. The hint on a validation failure points here. */
+export const SPEC_URL = "https://docspack.dev/spec";
 
 /** One retrievable unit of documentation. */
 export interface ChunkSpec {
@@ -73,7 +75,7 @@ export function parseManifest(raw: unknown, where: string): PackageManifest {
   // The explicit annotation is what lets TypeScript treat a `fail(...)` call as unreachable-after.
   const fail: (message: string) => never = (message: string): never => {
     throw new DocspackError(`${where}: ${message}`, {
-      hint: `See the package specification: ${SCHEMA_URL}`,
+      hint: `See the package specification: ${SPEC_URL}`,
     });
   };
 

package/src/style.ts

@@ -34,6 +34,11 @@ const SENTENCE_SPLIT = /(?<=[.!?])\s+/;
 /** A heading or list item starts a new unit, whatever the previous line ended with. */
 const BLOCK_START = /\n(?=\s*(?:[-*+]\s|\d+\.\s|#{1,6}\s|\|))/;
 const LONG_SENTENCE_WORDS = 40;
+/**
+ * A word carries a letter or a digit. Stripped code leaves its punctuation behind, so counting
+ * everything between spaces reports an enumeration of forty code spans as a forty-word sentence.
+ */
+const WORD = /[\p{L}\p{N}]/u;
 
 export interface FillerHit {
   readonly phrase: string;
@@ -68,7 +73,7 @@ export function findLongSentences(text: string, maxWords = LONG_SENTENCE_WORDS):
     .flatMap((block) => block.split(BLOCK_START))
     .flatMap((block) => block.split(SENTENCE_SPLIT))
     .map((sentence) => sentence.replace(/\s+/g, " ").trim())
-    .filter((sentence) => sentence.split(" ").filter(Boolean).length > maxWords);
+    .filter((sentence) => sentence.split(" ").filter((word) => WORD.test(word)).length > maxWords);
 }
 
 /**
@@ -94,11 +99,16 @@ export function contentFingerprint(text: string): string {
     .trim();
 }
 
+/** An authored directive, e.g. `<!-- docspack: tags=grid -->`. Metadata, not site boilerplate. */
+const DIRECTIVE_LINE = /^\s*<!--\s*docspack:/i;
+
 /** Removes lines a documentation site wraps around its content. */
 export function stripBoilerplate(text: string): string {
   return text
     .split("\n")
-    .filter((line) => !BOILERPLATE.some((pattern) => pattern.test(line)))
+    .filter(
+      (line) => DIRECTIVE_LINE.test(line) || !BOILERPLATE.some((pattern) => pattern.test(line)),
+    )
     .join("\n");
 }