docspack 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +11 -4
  2. package/dist/build.d.ts +21 -0
  3. package/dist/build.d.ts.map +1 -1
  4. package/dist/build.js +226 -29
  5. package/dist/build.js.map +1 -1
  6. package/dist/cli.js +95 -124
  7. package/dist/cli.js.map +1 -1
  8. package/dist/config.d.ts +7 -2
  9. package/dist/config.d.ts.map +1 -1
  10. package/dist/config.js +20 -1
  11. package/dist/config.js.map +1 -1
  12. package/dist/db.d.ts +9 -0
  13. package/dist/db.d.ts.map +1 -1
  14. package/dist/db.js +17 -2
  15. package/dist/db.js.map +1 -1
  16. package/dist/discovery.d.ts.map +1 -1
  17. package/dist/discovery.js +47 -25
  18. package/dist/discovery.js.map +1 -1
  19. package/dist/doctor.d.ts +6 -0
  20. package/dist/doctor.d.ts.map +1 -1
  21. package/dist/doctor.js +8 -4
  22. package/dist/doctor.js.map +1 -1
  23. package/dist/eval.d.ts +52 -0
  24. package/dist/eval.d.ts.map +1 -0
  25. package/dist/eval.js +101 -0
  26. package/dist/eval.js.map +1 -0
  27. package/dist/help.d.ts +31 -0
  28. package/dist/help.d.ts.map +1 -0
  29. package/dist/help.js +342 -0
  30. package/dist/help.js.map +1 -0
  31. package/dist/index.d.ts +4 -1
  32. package/dist/index.d.ts.map +1 -1
  33. package/dist/index.js +4 -1
  34. package/dist/index.js.map +1 -1
  35. package/dist/preview.d.ts.map +1 -1
  36. package/dist/preview.js +4 -2
  37. package/dist/preview.js.map +1 -1
  38. package/dist/search.d.ts +12 -0
  39. package/dist/search.d.ts.map +1 -1
  40. package/dist/search.js +36 -6
  41. package/dist/search.js.map +1 -1
  42. package/dist/spec.d.ts +23 -1
  43. package/dist/spec.d.ts.map +1 -1
  44. package/dist/spec.js +47 -6
  45. package/dist/spec.js.map +1 -1
  46. package/dist/stopwords.d.ts +14 -0
  47. package/dist/stopwords.d.ts.map +1 -0
  48. package/dist/stopwords.js +121 -0
  49. package/dist/stopwords.js.map +1 -0
  50. package/dist/style.d.ts.map +1 -1
  51. package/dist/style.js +9 -2
  52. package/dist/style.js.map +1 -1
  53. package/dist/sync.js +1 -1
  54. package/dist/sync.js.map +1 -1
  55. package/dist/verify.d.ts +8 -2
  56. package/dist/verify.d.ts.map +1 -1
  57. package/dist/verify.js +48 -15
  58. package/dist/verify.js.map +1 -1
  59. package/package.json +1 -1
  60. package/src/build.ts +274 -29
  61. package/src/cli.ts +114 -124
  62. package/src/config.ts +35 -5
  63. package/src/db.ts +17 -2
  64. package/src/discovery.ts +44 -22
  65. package/src/doctor.ts +14 -5
  66. package/src/eval.ts +149 -0
  67. package/src/help.ts +382 -0
  68. package/src/index.ts +21 -0
  69. package/src/preview.ts +4 -1
  70. package/src/search.ts +50 -6
  71. package/src/spec.ts +69 -7
  72. package/src/stopwords.ts +120 -0
  73. package/src/style.ts +12 -2
  74. package/src/sync.ts +1 -1
  75. package/src/verify.ts +69 -19
package/src/cli.ts CHANGED
@@ -7,9 +7,10 @@ import { defaultStorePath, Store, silenceSqliteWarning } from "./db.js";
7
7
  import { discoverPackages } from "./discovery.js";
8
8
  import { type DoctorReport, runDoctor } from "./doctor.js";
9
9
  import { DocspackError } from "./errors.js";
10
+ import { commandHelpText, findCommandHelp, globalHelpText } from "./help.js";
10
11
  import { renderTree } from "./init/write.js";
11
12
  import { previewPackage } from "./preview.js";
12
- import { DEFAULT_LIMIT, DEFAULT_MAX_TOKENS, queryDocs, renderAnswer } from "./search.js";
13
+ import { type QueryResult, queryDocs, renderAnswer } from "./search.js";
13
14
  import { AGENTS_SNIPPET, FEEDBACK_SNIPPET } from "./snippet.js";
14
15
  import { syncProject } from "./sync.js";
15
16
  import { verifyProject } from "./verify.js";
@@ -36,6 +37,10 @@ const OPTIONS = {
36
37
  name: { type: "string" },
37
38
  "pkg-version": { type: "string" },
38
39
  pages: { type: "string" },
40
+ "max-chunk-tokens": { type: "string" },
41
+ "min-chunk-tokens": { type: "string" },
42
+ documents: { type: "string", multiple: true },
43
+ "min-hit-rate": { type: "string" },
39
44
  mirror: { type: "string" },
40
45
  mode: { type: "string" },
41
46
  template: { type: "string" },
@@ -46,6 +51,7 @@ const OPTIONS = {
46
51
  yes: { type: "boolean", short: "y" },
47
52
  "dry-run": { type: "boolean" },
48
53
  strict: { type: "boolean" },
54
+ pedantic: { type: "boolean" },
49
55
  "package-dir": { type: "string" },
50
56
  chunk: { type: "string" },
51
57
  kind: { type: "string" },
@@ -55,127 +61,22 @@ const OPTIONS = {
55
61
  repro: { type: "string" },
56
62
  } as const;
57
63
 
58
- const HELP = `docspack — local, version-locked documentation for AI agents
59
-
60
- Usage
61
- docspack <command> [options]
62
-
63
- Commands
64
- sync Index the docs packages this project depends on
65
- ask <question> Answer from the local index — the command to give an agent
66
- search <query> Same index, formatted for a human reading the terminal
67
- list Show this project's docs packages and their index state
68
- verify Check the docs still describe the code you installed
69
- feedback <sub> Record documentation problems: add, list, submit, remove
70
- mcp Serve the index over MCP instead, as a long-lived process
71
- sources List curated sources that \`docspack build\` can fetch
72
-
73
- Authoring
74
- init Scaffold a documentation package, then build and check it
75
- build [source] Generate the .llms/ payload for publishing
76
- doctor Check a package the way the indexer and a reviewer would
77
- preview <query> Answer a query from the local package, as an agent would
78
-
79
- How it works
80
- Documentation ships as npm packages named @vendor/docspack or
81
- @docspack-community/<name>. Add them to package.json, run \`docspack sync\`, and
82
- every chunk is indexed into a shared SQLite database with FTS5. Agents query
83
- that index locally: no network, no scraping, always the installed version.
84
-
85
- Giving an agent access
86
- Any agent with a shell can run \`docspack ask\`, so one line in AGENTS.md or
87
- CLAUDE.md is the whole setup — no server, no per-agent configuration, and
88
- nothing resident when nobody is asking:
89
-
90
- ${AGENTS_SNIPPET.map((line) => ` ${line}`).join("\n")}
91
-
92
- Add the block below as well if you want the agent to record documentation
93
- problems it runs into. It writes to a local file; nothing is sent:
94
-
95
- ${FEEDBACK_SNIPPET.map((line) => ` ${line}`).join("\n")}
96
-
97
- \`docspack mcp\` serves the same index over the Model Context Protocol for
98
- clients that prefer a tool definition. Both return identical text.
99
-
100
- Recording documentation problems
101
- \`docspack feedback add\` writes a finding to .docspack/feedback.jsonl in this
102
- project. Nothing is transmitted: docspack contains no code that can send a
103
- report anywhere, and it never will.
104
-
105
- docspack feedback add --chunk @acme/docspack@1.4.0/api-auth \\
106
- --kind drift --evidence "client.setKey is not exported; setApiKey is"
107
-
108
- A finding has to be falsifiable. \`drift\` must name the identifier that
109
- drifted. \`incorrect\` and \`missing\` must carry --expected, --actual and
110
- --repro, so a claim that cannot show its work cannot be recorded at all.
111
- Recording the same problem again increments a counter instead of adding a
112
- second entry.
113
-
114
- \`docspack feedback submit\` prints a prefilled GitHub issue URL for every
115
- finding whose vendor asked to receive it, and routes nothing anywhere else.
116
- A vendor opts in from their own package.json:
117
-
118
- "docspack": {
119
- "feedback": {
120
- "github": "acme/sdk",
121
- "labels": ["documentation"],
122
- "accepts": ["drift"]
123
- }
124
- }
64
+ /**
65
+ * A query that returned nothing is not a failure, but "you have not indexed anything yet" and
66
+ * "the documentation does not cover this" are different answers and a wrapper has to tell them
67
+ * apart. Both are non-zero; neither collides with 1 (failed) or 2 (used wrongly).
68
+ */
69
+ const EXIT_NOT_INDEXED = 3;
70
+ const EXIT_NO_MATCH = 4;
125
71
 
126
- No \`feedback\` block means no channel and no route. \`accepts\` narrows which
127
- kinds reach them. A human opens the link, reads it, and files it.
128
-
129
- Options
130
- -y, --yes init: skip prompts and use flags plus detected defaults
131
- --dry-run init: print the file tree and write nothing
132
- --mirror <id|url> init: bootstrap from a published llms.txt
133
- --community init: scaffold under @docspack-community
134
- --no-workflow init: skip the release workflow
135
- --no-build init: scaffold only
136
- --template <t> init: full (default) or minimal
137
- --strict doctor: treat warnings as failures
138
- --package-dir <d> doctor, preview, verify: the package to inspect (default: .)
139
- --chunk <id> feedback add: the chunk the problem is in
140
- --kind <k> feedback add: drift, incorrect or missing
141
- --evidence <text> feedback add: the claim, in one line
142
- --expected <text> feedback add: what the documentation led you to expect
143
- --actual <text> feedback add: what happened instead
144
- --repro <code> feedback add: code that demonstrates it
145
- --force sync: re-index packages already in the store; init: overwrite files
146
- -p, --package <s> search: only packages whose name contains this text
147
- --limit <n> search: maximum chunks to return (default ${DEFAULT_LIMIT})
148
- --max-tokens <n> search: token ceiling for the result set (default ${DEFAULT_MAX_TOKENS})
149
- --all search: the whole store, not just this project;
150
- feedback remove: every recorded finding
151
- --from <dir> build: directory of Markdown to package
152
- --openapi <file> build: OpenAPI JSON to package, one chunk per operation
153
- --name <name> build: package name, e.g. @acme/docspack
154
- --pkg-version <v> build: package version
155
- --out <dir> build: output directory (default: the current directory)
156
- --pages <n> build: maximum documents to fetch from a remote source
157
- --store <path> Use a different index (default: ${defaultStorePath()})
158
- --cwd <dir> Run in a different directory
159
- --json Machine-readable output
160
- -q, --quiet Only print errors
161
- -h, --help Show this help
162
- -v, --version Show the version
163
-
164
- Examples
165
- npx docspack sync
166
- npx docspack ask "how do I verify a webhook signature"
167
- npx docspack search "webhook signature" --package stripe
168
- npx docspack init
169
- npx docspack init --name @acme/docspack --from ./docs --yes
170
- npx docspack doctor --strict
171
- npx docspack preview "how do I authenticate"
172
- npx docspack verify
173
- npx docspack feedback list
174
- npx docspack feedback submit
175
- `;
72
+ const HELP = globalHelpText(defaultStorePath(), EXIT_NOT_INDEXED, EXIT_NO_MATCH);
176
73
 
177
74
  type Values = {
178
- [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K]["type"] extends "boolean" ? boolean : string;
75
+ [K in keyof typeof OPTIONS]?: (typeof OPTIONS)[K] extends { multiple: true }
76
+ ? string[]
77
+ : (typeof OPTIONS)[K]["type"] extends "boolean"
78
+ ? boolean
79
+ : string;
179
80
  };
180
81
 
181
82
  const color = process.env.NO_COLOR === undefined && process.stdout.isTTY === true;
@@ -239,6 +140,12 @@ function reportDoctor(report: DoctorReport, quiet: boolean): void {
239
140
  }
240
141
  }
241
142
 
143
+ /** 0 when the query was answered; otherwise which of the two empty answers this was. */
144
+ function queryExit(result: QueryResult): number {
145
+ if (result.hits.length > 0) return 0;
146
+ return result.unindexed.length > 0 ? EXIT_NOT_INDEXED : EXIT_NO_MATCH;
147
+ }
148
+
242
149
  function openStore(values: Values): Store {
243
150
  return Store.open(values.store ?? defaultStorePath());
244
151
  }
@@ -263,8 +170,21 @@ async function main(argv: readonly string[]): Promise<number> {
263
170
  return 0;
264
171
  }
265
172
 
173
+ // `docspack doctor --help` and `docspack help doctor` are the same question, and the answer to
174
+ // both is that command's flags rather than the global page.
266
175
  const [command = "help", ...rest] = positionals;
267
176
  if (values.help === true || command === "help") {
177
+ const wanted = command === "help" ? rest[0] : command;
178
+ const subject = wanted === undefined ? undefined : findCommandHelp(wanted);
179
+ if (subject !== undefined) {
180
+ process.stdout.write(commandHelpText(subject));
181
+ return 0;
182
+ }
183
+ if (command === "help" && wanted !== undefined) {
184
+ process.stderr.write(`${red("error")} unknown command "${wanted}"\n\n`);
185
+ process.stderr.write(HELP);
186
+ return 2;
187
+ }
268
188
  process.stdout.write(HELP);
269
189
  return 0;
270
190
  }
@@ -358,8 +278,7 @@ async function main(argv: readonly string[]): Promise<number> {
358
278
  // Exactly what the MCP tool returns, so both interfaces answer identically.
359
279
  process.stdout.write(`${renderAnswer(result, question)}\n`);
360
280
  }
361
- // Answering "nothing matched" is a successful answer, not a failed command.
362
- return 0;
281
+ return queryExit(result);
363
282
  } finally {
364
283
  store.close();
365
284
  }
@@ -389,11 +308,16 @@ async function main(argv: readonly string[]): Promise<number> {
389
308
 
390
309
  if (json) {
391
310
  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
392
- return 0;
311
+ return queryExit(result);
393
312
  }
394
313
  if (result.hits.length === 0) {
395
314
  process.stdout.write(`No local documentation matched "${query}".\n`);
396
- return 1;
315
+ if (result.unindexed.length > 0) {
316
+ process.stderr.write(
317
+ `${yellow("!")} ${result.unindexed.join(", ")} installed but not indexed. Run \`docspack sync\`.\n`,
318
+ );
319
+ }
320
+ return queryExit(result);
397
321
  }
398
322
 
399
323
  for (const hit of result.hits) {
@@ -472,8 +396,13 @@ async function main(argv: readonly string[]): Promise<number> {
472
396
 
473
397
  const mark = pkg.findings.length === 0 ? green("ok") : red("drift");
474
398
  process.stdout.write(
475
- `${mark} ${bold(pkg.id)} ${dim(`documents ${pkg.documents ?? ""} · ${plural(pkg.checked, "name")} checked, ${pkg.matched} declared`)}\n`,
399
+ `${mark} ${bold(pkg.id)} ${dim(`documents ${(pkg.documents ?? []).join(", ")} · ${plural(pkg.checked, "name")} checked, ${pkg.matched} declared`)}\n`,
476
400
  );
401
+ if (pkg.unchecked !== undefined && pkg.unchecked.length > 0) {
402
+ process.stderr.write(
403
+ ` ${yellow("!")} ${pkg.unchecked.join(", ")} could not be read, so the check ran against less than the package documents\n`,
404
+ );
405
+ }
477
406
  for (const finding of pkg.findings) {
478
407
  process.stderr.write(
479
408
  ` ${yellow(finding.entity)} is not declared; did the API become ${bold(finding.suggestion)}?\n`,
@@ -683,6 +612,15 @@ async function main(argv: readonly string[]): Promise<number> {
683
612
  const pages = integer(values.pages, "--pages");
684
613
  return pages === undefined ? {} : { pages };
685
614
  })(),
615
+ ...(() => {
616
+ const max = integer(values["max-chunk-tokens"], "--max-chunk-tokens");
617
+ return max === undefined ? {} : { maxChunkTokens: max };
618
+ })(),
619
+ ...(() => {
620
+ const min = integer(values["min-chunk-tokens"], "--min-chunk-tokens");
621
+ return min === undefined ? {} : { minChunkTokens: min };
622
+ })(),
623
+ ...(values.documents === undefined ? {} : { documents: values.documents }),
686
624
  ...(quiet || json
687
625
  ? {}
688
626
  : {
@@ -793,6 +731,7 @@ async function main(argv: readonly string[]): Promise<number> {
793
731
  const report = await runDoctor({
794
732
  dir,
795
733
  ...(values.strict === true ? { strict: true } : {}),
734
+ ...(values.pedantic === true ? { pedantic: true } : {}),
796
735
  });
797
736
 
798
737
  if (json) {
@@ -845,6 +784,57 @@ async function main(argv: readonly string[]): Promise<number> {
845
784
  return 0;
846
785
  }
847
786
 
787
+ case "eval": {
788
+ const file = rest[0];
789
+ if (file === undefined) throw new DocspackError("Usage: docspack eval <queries.json>");
790
+
791
+ const { evaluatePackage, readEvalSet } = await import("./eval.js");
792
+ const report = await evaluatePackage({
793
+ dir: values["package-dir"] ?? cwd,
794
+ cases: await readEvalSet(file),
795
+ ...(() => {
796
+ const limit = integer(values.limit, "--limit");
797
+ return limit === undefined ? {} : { limit };
798
+ })(),
799
+ ...(() => {
800
+ const maxTokens = integer(values["max-tokens"], "--max-tokens");
801
+ return maxTokens === undefined ? {} : { maxTokens };
802
+ })(),
803
+ ...(() => {
804
+ const rate = integer(values["min-hit-rate"], "--min-hit-rate");
805
+ return rate === undefined ? {} : { minHitRate: rate };
806
+ })(),
807
+ });
808
+
809
+ if (json) {
810
+ process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
811
+ return report.ok ? 0 : 1;
812
+ }
813
+
814
+ for (const entry of report.cases) {
815
+ // A miss is only actionable next to what came back instead of the expected chunk.
816
+ const mark =
817
+ entry.rank === 1 ? green("1") : entry.rank > 0 ? yellow(String(entry.rank)) : red("—");
818
+ process.stdout.write(`${mark} ${entry.query}\n`);
819
+ if (entry.rank !== 1) {
820
+ process.stdout.write(` ${dim(`expected ${entry.expect.join(" | ")}`)}\n`);
821
+ process.stdout.write(
822
+ ` ${dim(`returned ${entry.returned.length === 0 ? "nothing" : entry.returned.join(", ")}`)}\n`,
823
+ );
824
+ }
825
+ }
826
+
827
+ const percent = (count: number): string =>
828
+ `${count}/${report.total} (${Math.round((count / report.total) * 100)}%)`;
829
+ process.stdout.write(
830
+ `\n${bold(`top-${report.limit}`)} ${percent(report.hits)} ${bold("top-1")} ${percent(report.top1)} ${dim(`~${report.averageTokens} tokens per answer`)}\n`,
831
+ );
832
+ if (!report.ok) {
833
+ process.stderr.write(`${red("failed")} below the required hit rate\n`);
834
+ }
835
+ return report.ok ? 0 : 1;
836
+ }
837
+
848
838
  case "sources": {
849
839
  const { entries: registryEntries } = await import("@docspack/registry");
850
840
  if (json) {
package/src/config.ts CHANGED
@@ -28,10 +28,15 @@ export interface BuildConfig {
28
28
  readonly from?: string;
29
29
  readonly openapi?: string;
30
30
  readonly source?: string;
31
- /** The library this package documents, e.g. `@acme/sdk`. What `docspack verify` checks against. */
32
- readonly documents?: string;
31
+ /**
32
+ * The libraries this package documents, each `name` or `name@version`. What `docspack verify`
33
+ * checks against, and what the manifest records so a consumer can say which release an answer
34
+ * describes. A monorepo documents several from one surface, so this is a list.
35
+ */
36
+ readonly documents?: readonly string[];
33
37
  readonly feedback?: FeedbackChannel;
34
38
  readonly maxChunkTokens?: number;
39
+ readonly minChunkTokens?: number;
35
40
  readonly pages?: number;
36
41
  }
37
42
 
@@ -64,7 +69,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
64
69
  }
65
70
  const raw = value as Record<string, unknown>;
66
71
 
67
- const text = (key: "from" | "openapi" | "source" | "documents"): string | undefined => {
72
+ const text = (key: "from" | "openapi" | "source"): string | undefined => {
68
73
  const found = raw[key];
69
74
  if (found === undefined) return undefined;
70
75
  if (typeof found !== "string" || found.length === 0) {
@@ -73,7 +78,7 @@ export function parseBuildConfig(value: unknown): BuildConfig {
73
78
  return found;
74
79
  };
75
80
 
76
- const count = (key: "maxChunkTokens" | "pages"): number | undefined => {
81
+ const count = (key: "maxChunkTokens" | "minChunkTokens" | "pages"): number | undefined => {
77
82
  const found = raw[key];
78
83
  if (found === undefined) return undefined;
79
84
  if (!Number.isInteger(found) || (found as number) < 1) {
@@ -85,9 +90,10 @@ export function parseBuildConfig(value: unknown): BuildConfig {
85
90
  const from = text("from");
86
91
  const openapi = text("openapi");
87
92
  const source = text("source");
88
- const documents = text("documents");
93
+ const documents = parseDocuments(raw.documents);
89
94
  const feedback = parseFeedbackChannel(raw.feedback);
90
95
  const maxChunkTokens = count("maxChunkTokens");
96
+ const minChunkTokens = count("minChunkTokens");
91
97
  const pages = count("pages");
92
98
 
93
99
  return {
@@ -97,10 +103,34 @@ export function parseBuildConfig(value: unknown): BuildConfig {
97
103
  ...(documents === undefined ? {} : { documents }),
98
104
  ...(feedback === undefined ? {} : { feedback }),
99
105
  ...(maxChunkTokens === undefined ? {} : { maxChunkTokens }),
106
+ ...(minChunkTokens === undefined ? {} : { minChunkTokens }),
100
107
  ...(pages === undefined ? {} : { pages }),
101
108
  };
102
109
  }
103
110
 
111
+ /**
112
+ * `"acme"`, `"acme@1.4.0"` or a list of either. A single string stays valid: one library is the
113
+ * common case, and every package written against the earlier field shape still reads.
114
+ */
115
+ function parseDocuments(value: unknown): readonly string[] | undefined {
116
+ if (value === undefined) return undefined;
117
+ const where = `${CONFIG_KEY}.documents`;
118
+ const specs = Array.isArray(value) ? value : [value];
119
+
120
+ if (
121
+ specs.length === 0 ||
122
+ specs.some((entry) => typeof entry !== "string" || entry.length === 0)
123
+ ) {
124
+ throw new DocspackError(
125
+ `package.json: "${where}" must be a package name, or an array of them`,
126
+ {
127
+ hint: 'Add a version to scope it: "acme@1.4.0".',
128
+ },
129
+ );
130
+ }
131
+ return specs as string[];
132
+ }
133
+
104
134
  /** `owner/repo`, the only shape a prefilled issue URL can be built from. */
105
135
  const GITHUB_REPO = /^[\w.-]+\/[\w.-]+$/;
106
136
 
package/src/db.ts CHANGED
@@ -4,6 +4,7 @@ import { homedir } from "node:os";
4
4
  import { dirname, join } from "node:path";
5
5
  import type { DatabaseSync } from "node:sqlite";
6
6
  import { DocspackError } from "./errors.js";
7
+ import { STOPWORDS } from "./stopwords.js";
7
8
 
8
9
  const require = createRequire(import.meta.url);
9
10
 
@@ -101,6 +102,15 @@ export function defaultStorePath(env: NodeJS.ProcessEnv = process.env): string {
101
102
  /**
102
103
  * Turns free text into an FTS5 MATCH expression. Every term is quoted, so punctuation in a user
103
104
  * query can never be interpreted as FTS syntax.
105
+ *
106
+ * Terms of one or two characters are dropped, as are function words of any length. Questions are
107
+ * mostly `how`, `do`, `i`, `in`, and every chunk containing one becomes a candidate — which is
108
+ * how a chunk that merely quotes an example question outranks the chunk that answers it. Two
109
+ * characters is the cut because it leaves real short names (`id`, `db`) out and keeps
110
+ * three-letter ones (`api`, `key`) in; `how` and `what` need the word list as well.
111
+ *
112
+ * Each narrowing falls back to the previous one, so a query made entirely of short or common
113
+ * words still searches for something rather than failing.
104
114
  */
105
115
  export function toFtsQuery(raw: string): string {
106
116
  const terms = raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
@@ -109,7 +119,10 @@ export function toFtsQuery(raw: string): string {
109
119
  hint: "Search for words, for example: docspack search webhook signature",
110
120
  });
111
121
  }
112
- return terms.map((term) => `"${term}"`).join(" OR ");
122
+ const long = terms.filter((term) => term.length > 2);
123
+ const meaningful = long.filter((term) => !STOPWORDS.has(term));
124
+ const chosen = meaningful.length > 0 ? meaningful : long.length > 0 ? long : terms;
125
+ return chosen.map((term) => `"${term}"`).join(" OR ");
113
126
  }
114
127
 
115
128
  /** The global chunk index: one SQLite database shared by every project on the machine. */
@@ -223,7 +236,9 @@ export class Store {
223
236
  FROM chunks_fts f
224
237
  JOIN chunks c ON c.chunk_id = f.chunk_id
225
238
  WHERE ${conditions.join(" AND ")}
226
- ORDER BY bm25(chunks_fts)
239
+ -- Tags are what an author writes to aim a chunk, and front matter did nothing at all
240
+ -- while they scored the same as prose. Weights are (content, tags).
241
+ ORDER BY bm25(chunks_fts, 1.0, 3.0)
227
242
  LIMIT ?`,
228
243
  )
229
244
  .all(...parameters, limit)
package/src/discovery.ts CHANGED
@@ -96,33 +96,55 @@ export async function projectPackageIds(cwd: string): Promise<string[]> {
96
96
  return packages.map((pkg) => pkg.id);
97
97
  }
98
98
 
99
+ /**
100
+ * Every docs package declared by this directory or by an ancestor of it. A workspace declares
101
+ * shared tooling in the repository root and its members inherit the install, so reading only
102
+ * `cwd/package.json` answers "this project has no documentation" for most monorepos — the same
103
+ * upward walk `resolvePackageDir` already does for the install.
104
+ */
99
105
  async function declaredDocsPackages(cwd: string): Promise<string[]> {
100
- let raw: string;
101
- try {
102
- raw = await readFile(join(cwd, "package.json"), "utf8");
103
- } catch (error) {
104
- if ((error as NodeJS.ErrnoException).code === "ENOENT") {
105
- throw new DocspackError(`No package.json found in ${cwd}`, {
106
- hint: "Run docspack from a project directory, or pass --cwd <dir>.",
107
- });
106
+ const names = new Set<string>();
107
+ let found = false;
108
+ let dir = cwd;
109
+
110
+ while (true) {
111
+ const path = join(dir, "package.json");
112
+ let raw: string;
113
+ try {
114
+ raw = await readFile(path, "utf8");
115
+ } catch (error) {
116
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
117
+ const parent = dirname(dir);
118
+ if (parent === dir) break;
119
+ dir = parent;
120
+ continue;
108
121
  }
109
- throw error;
110
- }
111
122
 
112
- let parsed: unknown;
113
- try {
114
- parsed = JSON.parse(raw);
115
- } catch (error) {
116
- throw new DocspackError(`${join(cwd, "package.json")} is not valid JSON`, { cause: error });
117
- }
123
+ found = true;
124
+ let parsed: unknown;
125
+ try {
126
+ parsed = JSON.parse(raw);
127
+ } catch (error) {
128
+ throw new DocspackError(`${path} is not valid JSON`, { cause: error });
129
+ }
118
130
 
119
- const root = parsed as { dependencies?: unknown; devDependencies?: unknown };
120
- const names = new Set<string>();
121
- for (const field of [root.dependencies, root.devDependencies]) {
122
- if (typeof field !== "object" || field === null) continue;
123
- for (const name of Object.keys(field as Record<string, unknown>)) {
124
- if (isDocsPackage(name)) names.add(name);
131
+ const manifest = parsed as { dependencies?: unknown; devDependencies?: unknown };
132
+ for (const field of [manifest.dependencies, manifest.devDependencies]) {
133
+ if (typeof field !== "object" || field === null) continue;
134
+ for (const name of Object.keys(field as Record<string, unknown>)) {
135
+ if (isDocsPackage(name)) names.add(name);
136
+ }
125
137
  }
138
+
139
+ const parent = dirname(dir);
140
+ if (parent === dir) break;
141
+ dir = parent;
142
+ }
143
+
144
+ if (!found) {
145
+ throw new DocspackError(`No package.json found in ${cwd}`, {
146
+ hint: "Run docspack from a project directory, or pass --cwd <dir>.",
147
+ });
126
148
  }
127
149
  return [...names].sort();
128
150
  }
package/src/doctor.ts CHANGED
@@ -40,6 +40,12 @@ export interface DoctorOptions {
40
40
  readonly dir: string;
41
41
  /** Treat warnings as failures. Used by prepublishOnly and CI. */
42
42
  readonly strict?: boolean;
43
+ /**
44
+ * Also fail on prose style — filler and long sentences. Implies `strict`. Off by default
45
+ * because `--strict` is the gate `init` scaffolds, and prose written by a human is not a
46
+ * reason to block a first publish.
47
+ */
48
+ readonly pedantic?: boolean;
43
49
  }
44
50
 
45
51
  /** Over this, a chunk crowds out the response budget; under it, a chunk answers nothing. */
@@ -53,6 +59,9 @@ const MIN_CHUNK_TOKENS = 30;
53
59
  export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
54
60
  const findings: Finding[] = [];
55
61
  const llmsDir = join(options.dir, LLMS_DIR);
62
+ // Structural checks describe a package that will not retrieve properly; the two prose checks
63
+ // are opinions about writing. Only the first kind may block a publish by default.
64
+ const prose: Severity = options.pedantic === true ? "warn" : "info";
56
65
 
57
66
  const pkg = await readJson(join(options.dir, "package.json"));
58
67
  const manifest = await loadManifest(llmsDir, findings);
@@ -95,7 +104,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
95
104
  continue;
96
105
  }
97
106
 
98
- const size = chunk.tokens > 0 ? chunk.tokens : estimateTokens(text);
107
+ const size = chunk.tokens ?? estimateTokens(text);
99
108
  tokens += size;
100
109
 
101
110
  if (size > MAX_CHUNK_TOKENS) {
@@ -145,7 +154,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
145
154
  .join(", ");
146
155
  findings.push({
147
156
  check: "filler",
148
- severity: "warn",
157
+ severity: prose,
149
158
  message: `chunk "${chunk.id}" contains narration a model does not need: ${listed}`,
150
159
  where: chunk.file,
151
160
  fix: "Delete it. Nobody searches for these words and they carry no fact.",
@@ -156,7 +165,7 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
156
165
  if (long.length > 0) {
157
166
  findings.push({
158
167
  check: "long-sentence",
159
- severity: "warn",
168
+ severity: prose,
160
169
  message: `chunk "${chunk.id}" has ${plural(long.length, "sentence")} over 40 words`,
161
170
  where: chunk.file,
162
171
  fix: "Split them. One claim per sentence retrieves and reads better.",
@@ -209,9 +218,9 @@ export async function runDoctor(options: DoctorOptions): Promise<DoctorReport> {
209
218
  });
210
219
  }
211
220
 
221
+ const strict = options.strict === true || options.pedantic === true;
212
222
  const failed = findings.some(
213
- (finding) =>
214
- finding.severity === "error" || (options.strict === true && finding.severity === "warn"),
223
+ (finding) => finding.severity === "error" || (strict && finding.severity === "warn"),
215
224
  );
216
225
 
217
226
  return { ok: !failed, findings, chunks: manifest.chunks.length, tokens };