docspack 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +9 -3
  2. package/dist/build.d.ts +21 -0
  3. package/dist/build.d.ts.map +1 -1
  4. package/dist/build.js +194 -31
  5. package/dist/build.js.map +1 -1
  6. package/dist/cli.js +74 -131
  7. package/dist/cli.js.map +1 -1
  8. package/dist/config.d.ts +7 -2
  9. package/dist/config.d.ts.map +1 -1
  10. package/dist/config.js +20 -1
  11. package/dist/config.js.map +1 -1
  12. package/dist/db.d.ts +8 -4
  13. package/dist/db.d.ts.map +1 -1
  14. package/dist/db.js +13 -7
  15. package/dist/db.js.map +1 -1
  16. package/dist/doctor.d.ts.map +1 -1
  17. package/dist/doctor.js +47 -4
  18. package/dist/doctor.js.map +1 -1
  19. package/dist/document.d.ts +2 -0
  20. package/dist/document.d.ts.map +1 -1
  21. package/dist/document.js +7 -3
  22. package/dist/document.js.map +1 -1
  23. package/dist/eval.d.ts +52 -0
  24. package/dist/eval.d.ts.map +1 -0
  25. package/dist/eval.js +101 -0
  26. package/dist/eval.js.map +1 -0
  27. package/dist/help.d.ts +31 -0
  28. package/dist/help.d.ts.map +1 -0
  29. package/dist/help.js +342 -0
  30. package/dist/help.js.map +1 -0
  31. package/dist/index.d.ts +4 -1
  32. package/dist/index.d.ts.map +1 -1
  33. package/dist/index.js +4 -1
  34. package/dist/index.js.map +1 -1
  35. package/dist/preview.d.ts.map +1 -1
  36. package/dist/preview.js +4 -2
  37. package/dist/preview.js.map +1 -1
  38. package/dist/search.d.ts +7 -0
  39. package/dist/search.d.ts.map +1 -1
  40. package/dist/search.js +33 -3
  41. package/dist/search.js.map +1 -1
  42. package/dist/spec.d.ts +27 -1
  43. package/dist/spec.d.ts.map +1 -1
  44. package/dist/spec.js +53 -5
  45. package/dist/spec.js.map +1 -1
  46. package/dist/stopwords.d.ts +14 -0
  47. package/dist/stopwords.d.ts.map +1 -0
  48. package/dist/stopwords.js +121 -0
  49. package/dist/stopwords.js.map +1 -0
  50. package/dist/sync.js +1 -1
  51. package/dist/sync.js.map +1 -1
  52. package/dist/verify.d.ts +8 -2
  53. package/dist/verify.d.ts.map +1 -1
  54. package/dist/verify.js +84 -23
  55. package/dist/verify.js.map +1 -1
  56. package/package.json +1 -1
  57. package/src/build.ts +261 -33
  58. package/src/cli.ts +91 -138
  59. package/src/config.ts +35 -5
  60. package/src/db.ts +13 -7
  61. package/src/doctor.ts +49 -5
  62. package/src/document.ts +13 -4
  63. package/src/eval.ts +149 -0
  64. package/src/help.ts +382 -0
  65. package/src/index.ts +20 -0
  66. package/src/preview.ts +4 -1
  67. package/src/search.ts +41 -3
  68. package/src/spec.ts +84 -6
  69. package/src/stopwords.ts +120 -0
  70. package/src/sync.ts +1 -1
  71. package/src/verify.ts +108 -28
package/src/cli.ts CHANGED
@@ -7,15 +7,10 @@ import { defaultStorePath, Store, silenceSqliteWarning } from "./db.js";
7
7
  import { discoverPackages } from "./discovery.js";
8
8
  import { type DoctorReport, runDoctor } from "./doctor.js";
9
9
  import { DocspackError } from "./errors.js";
10
+ import { commandHelpText, findCommandHelp, globalHelpText } from "./help.js";
10
11
  import { renderTree } from "./init/write.js";
11
12
  import { previewPackage } from "./preview.js";
12
- import {
13
- DEFAULT_LIMIT,
14
- DEFAULT_MAX_TOKENS,
15
- type QueryResult,
16
- queryDocs,
17
- renderAnswer,
18
- } from "./search.js";
13
+ import { type QueryResult, queryDocs, renderAnswer } from "./search.js";
19
14
  import { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
20
15
  import { syncProject } from "./sync.js";
21
16
  import { verifyProject } from "./verify.js";
@@ -42,6 +37,10 @@ const OPTIONS = {
42
37
  name: { type: "string" },
43
38
  "pkg-version": { type: "string" },
44
39
  pages: { type: "string" },
40
+ "max-chunk-tokens": { type: "string" },
41
+ "min-chunk-tokens": { type: "string" },
42
+ documents: { type: "string", multiple: true },
43
+ "min-hit-rate": { type: "string" },
45
44
  mirror: { type: "string" },
46
45
  mode: { type: "string" },
47
46
  template: { type: "string" },
@@ -70,138 +69,14 @@ const OPTIONS = {
70
69
  const EXIT_NOT_INDEXED = 3;
71
70
  const EXIT_NO_MATCH = 4;
72
71
 
73
- const HELP = `docspack — local, version-locked documentation for AI agents
74
-
75
- Usage
76
- docspack <command> [options]
77
-
78
- Commands
79
- sync Index the docs packages this project depends on
80
- ask <question> Answer from the local index — the command to give an agent
81
- search <query> Same index, formatted for a human reading the terminal
82
- list Show this project's docs packages and their index state
83
- verify Check the docs still describe the code you installed
84
- feedback <sub> Record documentation problems: add, list, submit, remove
85
- mcp Serve the index over MCP instead, as a long-lived process
86
- sources List curated sources that \`docspack build\` can fetch
87
-
88
- Authoring
89
- init Scaffold a documentation package, then build and check it
90
- build [source] Generate the .llms/ payload for publishing
91
- doctor Check a package the way the indexer and a reviewer would
92
- preview <query> Answer a query from the local package, as an agent would
93
-
94
- How it works
95
- Documentation ships as npm packages named @vendor/docspack or
96
- @docspack-community/<name>. Add them to package.json, run \`docspack sync\`, and
97
- every chunk is indexed into a shared SQLite database with FTS5. Agents query
98
- that index locally: no network, no scraping, always the installed version.
99
-
100
- Giving an agent access
101
- Any agent with a shell can run \`docspack ask\`, so one line in AGENTS.md or
102
- CLAUDE.md is the whole setup — no server, no per-agent configuration, and
103
- nothing resident when nobody is asking:
104
-
105
- ${AGENTS_SNIPPET.map((line) => ` ${line}`).join("\n")}
106
-
107
- Add the block below as well if you want the agent to record documentation
108
- problems it runs into. It writes to a local file; nothing is sent:
109
-
110
- ${FEEDBACK_SNIPPET.map((line) => ` ${line}`).join("\n")}
111
-
112
- \`docspack mcp\` serves the same index over the Model Context Protocol for
113
- clients that prefer a tool definition. Both return identical text.
114
-
115
- Recording documentation problems
116
- \`docspack feedback add\` writes a finding to .docspack/feedback.jsonl in this
117
- project. Nothing is transmitted: docspack contains no code that can send a
118
- report anywhere, and it never will.
119
-
120
- docspack feedback add --chunk @acme/docspack@1.4.0/api-auth \\
121
- --kind drift --evidence "client.setKey is not exported; setApiKey is"
122
-
123
- A finding has to be falsifiable. \`drift\` must name the identifier that
124
- drifted. \`incorrect\` and \`missing\` must carry --expected, --actual and
125
- --repro, so a claim that cannot show its work cannot be recorded at all.
126
- Recording the same problem again increments a counter instead of adding a
127
- second entry.
128
-
129
- \`docspack feedback submit\` prints a prefilled GitHub issue URL for every
130
- finding whose vendor asked to receive it, and routes nothing anywhere else.
131
- A vendor opts in from their own package.json:
132
-
133
- "docspack": {
134
- "feedback": {
135
- "github": "acme/sdk",
136
- "labels": ["documentation"],
137
- "accepts": ["drift"]
138
- }
139
- }
140
-
141
- No \`feedback\` block means no channel and no route. \`accepts\` narrows which
142
- kinds reach them. A human opens the link, reads it, and files it.
143
-
144
- Options
145
- -y, --yes init: skip prompts and use flags plus detected defaults
146
- --dry-run init: print the file tree and write nothing
147
- --mirror <id|url> init: bootstrap from a published llms.txt
148
- --community init: scaffold under @docspack-community
149
- --no-workflow init: skip the release workflow
150
- --no-build init: scaffold only
151
- --template <t> init: full (default) or minimal
152
- --strict doctor: treat structural warnings as failures
153
- --pedantic doctor: --strict, and fail on prose style as well
154
- --package-dir <d> doctor, preview, verify: the package to inspect (default: .)
155
- --chunk <id> feedback add: the chunk the problem is in
156
- --kind <k> feedback add: drift, incorrect or missing
157
- --evidence <text> feedback add: the claim, in one line
158
- --expected <text> feedback add: what the documentation led you to expect
159
- --actual <text> feedback add: what happened instead
160
- --repro <code> feedback add: code that demonstrates it
161
- --force sync: re-index packages already in the store; init: overwrite files
162
- -p, --package <s> ask, search: only packages whose name contains this text
163
- --limit <n> ask, search, preview: maximum chunks to return (default ${DEFAULT_LIMIT})
164
- --max-tokens <n> ask, search, preview: token ceiling for the result set
165
- (default ${DEFAULT_MAX_TOKENS})
166
- --all ask, search: the whole store, not just this project;
167
- feedback remove: every recorded finding
168
- --from <dir> build: directory of Markdown to package
169
- --openapi <file> build: OpenAPI JSON to package, one chunk per operation
170
- --name <name> build: package name, e.g. @acme/docspack
171
- --pkg-version <v> build: package version
172
- --out <dir> build: output directory (default: the current directory)
173
- --pages <n> build: maximum documents to fetch from a remote source
174
- --store <path> Use a different index (default: ${defaultStorePath()})
175
- --cwd <dir> Run in a different directory
176
- --json Machine-readable output
177
- -q, --quiet Only print errors
178
- -h, --help Show this help
179
- -v, --version Show the version
180
-
181
- Exit codes
182
- \`ask\` and \`search\` answer with an exit code a script can branch on:
183
-
184
- 0 chunks were returned
185
- ${EXIT_NOT_INDEXED} nothing matched, and a docs package is installed but not indexed —
186
- run \`docspack sync\`
187
- ${EXIT_NO_MATCH} nothing matched, and everything installed is already indexed
188
- 1 the command failed; 2 the command was used wrongly
189
-
190
- Examples
191
- npx docspack sync
192
- npx docspack ask "how do I verify a webhook signature"
193
- npx docspack search "webhook signature" --package stripe
194
- npx docspack init
195
- npx docspack init --name @acme/docspack --from ./docs --yes
196
- npx docspack doctor --strict
197
- npx docspack preview "how do I authenticate"
198
- npx docspack verify
199
- npx docspack feedback list
200
- npx docspack feedback submit
201
- `;
72
+ const HELP = globalHelpText(defaultStorePath(), EXIT_NOT_INDEXED, EXIT_NO_MATCH);
202
73
 
203
74
  type Values = {
204
- [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K]["type"] extends "boolean" ? boolean : string;
75
+ [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K] extends { multiple: true }
76
+ ? string[]
77
+ : (typeof OPTIONS)[K]["type"] extends "boolean"
78
+ ? boolean
79
+ : string;
205
80
  };
206
81
 
207
82
  const color = process.env.NO_COLOR === undefined && process.stdout.isTTY === true;
@@ -295,8 +170,21 @@ async function main(argv: readonly string[]): Promise<number> {
295
170
  return 0;
296
171
  }
297
172
 
173
+ // `docspack doctor --help` and `docspack help doctor` are the same question, and the answer to
174
+ // both is that command's flags rather than the global page.
298
175
  const [command = "help", ...rest] = positionals;
299
176
  if (values.help === true || command === "help") {
177
+ const wanted = command === "help" ? rest[0] : command;
178
+ const subject = wanted === undefined ? undefined : findCommandHelp(wanted);
179
+ if (subject !== undefined) {
180
+ process.stdout.write(commandHelpText(subject));
181
+ return 0;
182
+ }
183
+ if (command === "help" && wanted !== undefined) {
184
+ process.stderr.write(`${red("error")} unknown command "${wanted}"\n\n`);
185
+ process.stderr.write(HELP);
186
+ return 2;
187
+ }
300
188
  process.stdout.write(HELP);
301
189
  return 0;
302
190
  }
@@ -508,8 +396,13 @@ async function main(argv: readonly string[]): Promise<number> {
508
396
 
509
397
  const mark = pkg.findings.length === 0 ? green("ok") : red("drift");
510
398
  process.stdout.write(
511
- `${mark} ${bold(pkg.id)} ${dim(`documents ${pkg.documents ?? ""} · ${plural(pkg.checked, "name")} checked, ${pkg.matched} declared`)}\n`,
399
+ `${mark} ${bold(pkg.id)} ${dim(`documents ${(pkg.documents ?? []).join(", ")} · ${plural(pkg.checked, "name")} checked, ${pkg.matched} declared`)}\n`,
512
400
  );
401
+ if (pkg.unchecked !== undefined && pkg.unchecked.length > 0) {
402
+ process.stderr.write(
403
+ ` ${yellow("!")} ${pkg.unchecked.join(", ")} could not be read, so the check ran against less than the package documents\n`,
404
+ );
405
+ }
513
406
  for (const finding of pkg.findings) {
514
407
  process.stderr.write(
515
408
  ` ${yellow(finding.entity)} is not declared; did the API become ${bold(finding.suggestion)}?\n`,
@@ -719,6 +612,15 @@ async function main(argv: readonly string[]): Promise<number> {
719
612
  const pages = integer(values.pages, "--pages");
720
613
  return pages === undefined ? {} : { pages };
721
614
  })(),
615
+ ...(() => {
616
+ const max = integer(values["max-chunk-tokens"], "--max-chunk-tokens");
617
+ return max === undefined ? {} : { maxChunkTokens: max };
618
+ })(),
619
+ ...(() => {
620
+ const min = integer(values["min-chunk-tokens"], "--min-chunk-tokens");
621
+ return min === undefined ? {} : { minChunkTokens: min };
622
+ })(),
623
+ ...(values.documents === undefined ? {} : { documents: values.documents }),
722
624
  ...(quiet || json
723
625
  ? {}
724
626
  : {
@@ -882,6 +784,57 @@ async function main(argv: readonly string[]): Promise<number> {
882
784
  return 0;
883
785
  }
884
786
 
787
+ case "eval": {
788
+ const file = rest[0];
789
+ if (file === undefined) throw new DocspackError("Usage: docspack eval <queries.json>");
790
+
791
+ const { evaluatePackage, readEvalSet } = await import("./eval.js");
792
+ const report = await evaluatePackage({
793
+ dir: values["package-dir"] ?? cwd,
794
+ cases: await readEvalSet(file),
795
+ ...(() => {
796
+ const limit = integer(values.limit, "--limit");
797
+ return limit === undefined ? {} : { limit };
798
+ })(),
799
+ ...(() => {
800
+ const maxTokens = integer(values["max-tokens"], "--max-tokens");
801
+ return maxTokens === undefined ? {} : { maxTokens };
802
+ })(),
803
+ ...(() => {
804
+ const rate = integer(values["min-hit-rate"], "--min-hit-rate");
805
+ return rate === undefined ? {} : { minHitRate: rate };
806
+ })(),
807
+ });
808
+
809
+ if (json) {
810
+ process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
811
+ return report.ok ? 0 : 1;
812
+ }
813
+
814
+ for (const entry of report.cases) {
815
+ // A miss is only actionable next to what came back instead of the expected chunk.
816
+ const mark =
817
+ entry.rank === 1 ? green("1") : entry.rank > 0 ? yellow(String(entry.rank)) : red("—");
818
+ process.stdout.write(`${mark} ${entry.query}\n`);
819
+ if (entry.rank !== 1) {
820
+ process.stdout.write(` ${dim(`expected ${entry.expect.join(" | ")}`)}\n`);
821
+ process.stdout.write(
822
+ ` ${dim(`returned ${entry.returned.length === 0 ? "nothing" : entry.returned.join(", ")}`)}\n`,
823
+ );
824
+ }
825
+ }
826
+
827
+ const percent = (count: number): string =>
828
+ `${count}/${report.total} (${Math.round((count / report.total) * 100)}%)`;
829
+ process.stdout.write(
830
+ `\n${bold(`top-${report.limit}`)} ${percent(report.hits)} ${bold("top-1")} ${percent(report.top1)} ${dim(`~${report.averageTokens} tokens per answer`)}\n`,
831
+ );
832
+ if (!report.ok) {
833
+ process.stderr.write(`${red("failed")} below the required hit rate\n`);
834
+ }
835
+ return report.ok ? 0 : 1;
836
+ }
837
+
885
838
  case "sources": {
886
839
  const { entries: registryEntries } = await import("@docspack/registry");
887
840
  if (json) {
package/src/config.ts CHANGED
@@ -28,10 +28,15 @@ export interface BuildConfig {
28
28
  readonly from?: string;
29
29
  readonly openapi?: string;
30
30
  readonly source?: string;
31
- /** The library this package documents, e.g. `@acme/sdk`. What `docspack verify` checks against. */
32
- readonly documents?: string;
31
+ /**
32
+ * The libraries this package documents, each `name` or `name@version`. What `docspack verify`
33
+ * checks against, and what the manifest records so a consumer can say which release an answer
34
+ * describes. A monorepo documents several from one surface, so this is a list.
35
+ */
36
+ readonly documents?: readonly string[];
33
37
  readonly feedback?: FeedbackChannel;
34
38
  readonly maxChunkTokens?: number;
39
+ readonly minChunkTokens?: number;
35
40
  readonly pages?: number;
36
41
  }
37
42
 
@@ -64,7 +69,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
64
69
  }
65
70
  const raw = value as Record<string, unknown>;
66
71
 
67
- const text = (key: "from" | "openapi" | "source" | "documents"): string | undefined => {
72
+ const text = (key: "from" | "openapi" | "source"): string | undefined => {
68
73
  const found = raw[key];
69
74
  if (found === undefined) return undefined;
70
75
  if (typeof found !== "string" || found.length === 0) {
@@ -73,7 +78,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
73
78
  return found;
74
79
  };
75
80
 
76
- const count = (key: "maxChunkTokens" | "pages"): number | undefined => {
81
+ const count = (key: "maxChunkTokens" | "minChunkTokens" | "pages"): number | undefined => {
77
82
  const found = raw[key];
78
83
  if (found === undefined) return undefined;
79
84
  if (!Number.isInteger(found) || (found as number) < 1) {
@@ -85,9 +90,10 @@ export function parseBuildConfig(value: unknown): BuildConfig {
85
90
  const from = text("from");
86
91
  const openapi = text("openapi");
87
92
  const source = text("source");
88
- const documents = text("documents");
93
+ const documents = parseDocuments(raw.documents);
89
94
  const feedback = parseFeedbackChannel(raw.feedback);
90
95
  const maxChunkTokens = count("maxChunkTokens");
96
+ const minChunkTokens = count("minChunkTokens");
91
97
  const pages = count("pages");
92
98
 
93
99
  return {
@@ -97,10 +103,34 @@ export function parseBuildConfig(value: unknown): BuildConfig {
97
103
  ...(documents === undefined ? {} : { documents }),
98
104
  ...(feedback === undefined ? {} : { feedback }),
99
105
  ...(maxChunkTokens === undefined ? {} : { maxChunkTokens }),
106
+ ...(minChunkTokens === undefined ? {} : { minChunkTokens }),
100
107
  ...(pages === undefined ? {} : { pages }),
101
108
  };
102
109
  }
103
110
 
111
+ /**
112
+ * `"acme"`, `"acme@1.4.0"` or a list of either. A single string stays valid: one library is the
113
+ * common case, and every package written against the earlier field shape still reads.
114
+ */
115
+ function parseDocuments(value: unknown): readonly string[] | undefined {
116
+ if (value === undefined) return undefined;
117
+ const where = `${CONFIG_KEY}.documents`;
118
+ const specs = Array.isArray(value) ? value : [value];
119
+
120
+ if (
121
+ specs.length === 0 ||
122
+ specs.some((entry) => typeof entry !== "string" || entry.length === 0)
123
+ ) {
124
+ throw new DocspackError(
125
+ `package.json: "${where}" must be a package name, or an array of them`,
126
+ {
127
+ hint: 'Add a version to scope it: "acme@1.4.0".',
128
+ },
129
+ );
130
+ }
131
+ return specs as string[];
132
+ }
133
+
104
134
  /** `owner/repo`, the only shape a prefilled issue URL can be built from. */
105
135
  const GITHUB_REPO = /^[\w.-]+\/[\w.-]+$/;
106
136
 
package/src/db.ts CHANGED
@@ -4,6 +4,7 @@ import { homedir } from "node:os";
4
4
  import { dirname, join } from "node:path";
5
5
  import type { DatabaseSync } from "node:sqlite";
6
6
  import { DocspackError } from "./errors.js";
7
+ import { STOPWORDS } from "./stopwords.js";
7
8
 
8
9
  const require = createRequire(import.meta.url);
9
10
 
@@ -102,10 +103,14 @@ export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
102
103
  * Turns free text into an FTS5 MATCH expression. Every term is quoted, so punctuation in a user
103
104
  * query can never be interpreted as FTS syntax.
104
105
  *
105
- * Terms of one or two characters are dropped. Questions are mostly `how`, `do`, `i`, `in`, and
106
- * every chunk containing one becomes a candidate — which is how a chunk that merely quotes an
107
- * example question outranks the chunk that answers it. Two characters is the cut because it
108
- * leaves real short names (`id`, `db`) out and keeps three-letter ones (`api`, `key`) in.
106
+ * Terms of one or two characters are dropped, as are function words of any length. Questions are
107
+ * mostly `how`, `do`, `i`, `in`, and every chunk containing one becomes a candidate — which is
108
+ * how a chunk that merely quotes an example question outranks the chunk that answers it. Two
109
+ * characters is the cut because it leaves real short names (`id`, `db`) out and keeps
110
+ * three-letter ones (`api`, `key`) in; `how` and `what` need the word list as well.
111
+ *
112
+ * Each narrowing falls back to the previous one, so a query made entirely of short or common
113
+ * words still searches for something rather than failing.
109
114
  */
110
115
  export function toFtsQuery(raw: string): string {
111
116
  const terms = raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
@@ -114,9 +119,10 @@ export function toFtsQuery(raw: string): string {
114
119
  hint: "Search for words, for example: docspack search webhook signature",
115
120
  });
116
121
  }
117
- const meaningful = terms.filter((term) => term.length > 2);
118
- // A query made only of short terms still has to search for something.
119
- return (meaningful.length > 0 ? meaningful : terms).map((term) => `"${term}"`).join(" OR ");
122
+ const long = terms.filter((term) => term.length > 2);
123
+ const meaningful = long.filter((term) => !STOPWORDS.has(term));
124
+ const chosen = meaningful.length > 0 ? meaningful : long.length > 0 ? long : terms;
125
+ return chosen.map((term) => `"${term}"`).join(" OR ");
120
126
  }
121
127
 
122
128
  /** The global chunk index: one SQLite database shared by every project on the machine. */
package/src/doctor.ts CHANGED
@@ -64,12 +64,14 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
64
64
  const prose: Severity = options.pedantic === true ? "warn" : "info";
65
65
 
66
66
  const pkg = await readJson(join(options.dir, "package.json"));
67
- const manifest = await loadManifest(llmsDir, findings);
68
- if (manifest === undefined) {
67
+ const loaded = await loadManifest(llmsDir, findings);
68
+ if (loaded === undefined) {
69
69
  return { ok: false, findings, chunks: 0, tokens: 0 };
70
70
  }
71
+ const { manifest, raw } = loaded;
71
72
 
72
73
  checkPackageJson(pkg, manifest, findings);
74
+ checkUnknownFields(raw, findings);
73
75
 
74
76
  let tokens = 0;
75
77
  let newestSource = 0;
@@ -104,7 +106,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
104
106
  continue;
105
107
  }
106
108
 
107
- const size = chunk.tokens > 0 ? chunk.tokens : estimateTokens(text);
109
+ const size = chunk.tokens ?? estimateTokens(text);
108
110
  tokens += size;
109
111
 
110
112
  if (size > MAX_CHUNK_TOKENS) {
@@ -233,7 +235,7 @@ function plural(count: number, noun: string): string {
233
235
  async function loadManifest(
234
236
  llmsDir: string,
235
237
  findings: Finding[],
236
- ): Promise<PackageManifest | undefined> {
238
+ ): Promise<{ manifest: PackageManifest; raw: unknown } | undefined> {
237
239
  let raw: string;
238
240
  try {
239
241
  raw = await readFile(join(llmsDir, MANIFEST_FILE), "utf8");
@@ -248,7 +250,8 @@ async function loadManifest(
248
250
  }
249
251
 
250
252
  try {
251
- return parseManifest(JSON.parse(raw), `${LLMS_DIR}/${MANIFEST_FILE}`);
253
+ const parsed: unknown = JSON.parse(raw);
254
+ return { manifest: parseManifest(parsed, `${LLMS_DIR}/${MANIFEST_FILE}`), raw: parsed };
252
255
  } catch (error) {
253
256
  findings.push({
254
257
  check: "manifest-invalid",
@@ -261,6 +264,47 @@ async function loadManifest(
261
264
  }
262
265
  }
263
266
 
267
+ /** Every key the format defines. `additionalProperties` is true, so nothing else is read. */
268
+ const MANIFEST_KEYS = new Set(["$schema", "name", "version", "documents", "chunks"]);
269
+ const CHUNK_KEYS = new Set(["id", "file", "tokens", "tags", "entities", "documents"]);
270
+
271
+ /**
272
+ * Reports keys outside the closed set.
273
+ *
274
+ * Unknown fields are accepted and ignored, which is the forward-compatibility rule and worth
275
+ * keeping — but it also means `"tokens"` written as `"token"` validates, publishes, and does
276
+ * nothing. A publisher is the one party who can still tell a typo from an extension, so the
277
+ * check belongs here rather than in the reader, at a severity `--strict` fails on.
278
+ */
279
+ function checkUnknownFields(raw: unknown, findings: Finding[]): void {
280
+ const root = raw as Record<string, unknown>;
281
+ const unknown = Object.keys(root).filter((key) => !MANIFEST_KEYS.has(key));
282
+ if (unknown.length > 0) {
283
+ findings.push({
284
+ check: "unknown-field",
285
+ severity: "warn",
286
+ message: `the manifest has ${unknown.length === 1 ? "a field" : "fields"} nothing reads: ${unknown.join(", ")}`,
287
+ where: `${LLMS_DIR}/${MANIFEST_FILE}`,
288
+ fix: "Remove it, or correct the spelling. Unknown fields are accepted and ignored.",
289
+ });
290
+ }
291
+
292
+ const chunks = Array.isArray(root.chunks) ? root.chunks : [];
293
+ for (const entry of chunks) {
294
+ if (typeof entry !== "object" || entry === null) continue;
295
+ const chunk = entry as Record<string, unknown>;
296
+ const extra = Object.keys(chunk).filter((key) => !CHUNK_KEYS.has(key));
297
+ if (extra.length === 0) continue;
298
+ findings.push({
299
+ check: "unknown-field",
300
+ severity: "warn",
301
+ message: `chunk "${String(chunk.id)}" has ${extra.length === 1 ? "a field" : "fields"} nothing reads: ${extra.join(", ")}`,
302
+ where: `${LLMS_DIR}/${MANIFEST_FILE}`,
303
+ fix: "Remove it, or correct the spelling. Unknown fields are accepted and ignored.",
304
+ });
305
+ }
306
+ }
307
+
264
308
  function checkPackageJson(
265
309
  pkg: Record<string, unknown> | undefined,
266
310
  manifest: PackageManifest,
package/src/document.ts CHANGED
@@ -6,6 +6,8 @@ export interface CleanedDocument {
6
6
  readonly title?: string;
7
7
  /** Keywords taken from front matter, indexed alongside the prose. */
8
8
  readonly tags: readonly string[];
9
+ /** Libraries this document describes, when it describes fewer than the whole package. */
10
+ readonly documents: readonly string[];
9
11
  }
10
12
 
11
13
  const FRONT_MATTER = /^?---\r?\n([\s\S]*?)\r?\n---\r?\n?/;
@@ -17,23 +19,29 @@ const FRONT_MATTER = /^?---\r?\n([\s\S]*?)\r?\n---\r?\n?/;
17
19
  export function cleanDocument(raw: string): CleanedDocument {
18
20
  const match = FRONT_MATTER.exec(raw);
19
21
  const body = stripBoilerplate(match === null ? raw : raw.slice(match[0].length)).trim();
20
- if (match?.[1] === undefined) return { text: body, tags: [] };
22
+ if (match?.[1] === undefined) return { text: body, tags: [], documents: [] };
21
23
 
22
24
  const meta = parseFrontMatter(match[1]);
23
25
  return {
24
26
  text: body,
25
27
  ...(meta.title === undefined ? {} : { title: meta.title }),
26
28
  tags: meta.tags,
29
+ documents: meta.documents,
27
30
  };
28
31
  }
29
32
 
30
33
  /**
31
- * Reads the two front matter fields worth indexing. This is deliberately not a YAML parser:
34
+ * Reads the three front matter fields worth reading. This is deliberately not a YAML parser:
32
35
  * anything more complex is not metadata a retrieval index can use.
33
36
  */
34
- function parseFrontMatter(block: string): { title?: string; tags: string[] } {
37
+ function parseFrontMatter(block: string): {
38
+ title?: string;
39
+ tags: string[];
40
+ documents: string[];
41
+ } {
35
42
  let title: string | undefined;
36
43
  const tags: string[] = [];
44
+ const documents: string[] = [];
37
45
 
38
46
  for (const line of block.split("\n")) {
39
47
  const entry = /^\s*([A-Za-z_][\w-]*)\s*:\s*(.*)$/.exec(line);
@@ -45,9 +53,10 @@ function parseFrontMatter(block: string): { title?: string; tags: string[] } {
45
53
  if ((key === "tags" || key === "keywords") && value.startsWith("[")) {
46
54
  tags.push(...splitList(value));
47
55
  }
56
+ if (key === "documents" && value.startsWith("[")) documents.push(...splitList(value));
48
57
  }
49
58
 
50
- return { ...(title === undefined ? {} : { title }), tags };
59
+ return { ...(title === undefined ? {} : { title }), tags, documents };
51
60
  }
52
61
 
53
62
  function splitList(value: string): string[] {