docspack 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +48 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +243 -0
- package/dist/agent.js.map +1 -0
- package/dist/artifact.d.ts +32 -0
- package/dist/artifact.d.ts.map +1 -0
- package/dist/artifact.js +78 -0
- package/dist/artifact.js.map +1 -0
- package/dist/build.d.ts.map +1 -1
- package/dist/build.js +17 -0
- package/dist/build.js.map +1 -1
- package/dist/changed.d.ts +31 -0
- package/dist/changed.d.ts.map +1 -0
- package/dist/changed.js +71 -0
- package/dist/changed.js.map +1 -0
- package/dist/cli.js +131 -10
- package/dist/cli.js.map +1 -1
- package/dist/coverage.d.ts +35 -0
- package/dist/coverage.d.ts.map +1 -0
- package/dist/coverage.js +64 -0
- package/dist/coverage.js.map +1 -0
- package/dist/db.d.ts +38 -2
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +109 -6
- package/dist/db.js.map +1 -1
- package/dist/discovery.d.ts +14 -0
- package/dist/discovery.d.ts.map +1 -1
- package/dist/discovery.js +31 -6
- package/dist/discovery.js.map +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +44 -0
- package/dist/doctor.js.map +1 -1
- package/dist/help.d.ts.map +1 -1
- package/dist/help.js +65 -4
- package/dist/help.js.map +1 -1
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +3 -0
- package/dist/mcp.js.map +1 -1
- package/dist/preview.d.ts.map +1 -1
- package/dist/preview.js +1 -0
- package/dist/preview.js.map +1 -1
- package/dist/search.d.ts +14 -1
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +108 -21
- package/dist/search.js.map +1 -1
- package/dist/surface.d.ts +45 -0
- package/dist/surface.d.ts.map +1 -0
- package/dist/surface.js +208 -0
- package/dist/surface.js.map +1 -0
- package/dist/sync.d.ts +8 -1
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +50 -1
- package/dist/sync.js.map +1 -1
- package/dist/verify.d.ts +9 -0
- package/dist/verify.d.ts.map +1 -1
- package/dist/verify.js +1 -1
- package/dist/verify.js.map +1 -1
- package/package.json +1 -1
- package/src/agent.ts +308 -0
- package/src/artifact.ts +110 -0
- package/src/build.ts +19 -0
- package/src/changed.ts +99 -0
- package/src/cli.ts +167 -10
- package/src/coverage.ts +96 -0
- package/src/db.ts +168 -7
- package/src/discovery.ts +40 -5
- package/src/doctor.ts +52 -0
- package/src/help.ts +65 -4
- package/src/index.ts +30 -0
- package/src/mcp.ts +3 -0
- package/src/preview.ts +1 -0
- package/src/search.ts +148 -22
- package/src/surface.ts +265 -0
- package/src/sync.ts +66 -2
- package/src/verify.ts +1 -1
package/src/cli.ts
CHANGED
|
@@ -3,6 +3,9 @@ import { readFileSync } from "node:fs";
|
|
|
3
3
|
import { posix } from "node:path";
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { parseArgs } from "node:util";
|
|
6
|
+
import { applyAgentSetup, planAgentSetup } from "./agent.js";
|
|
7
|
+
import { changedSurface } from "./changed.js";
|
|
8
|
+
import { measureCoverage } from "./coverage.js";
|
|
6
9
|
import { defaultStorePath, Store, silenceSqliteWarning } from "./db.js";
|
|
7
10
|
import { discoverPackages } from "./discovery.js";
|
|
8
11
|
import { type DoctorReport, runDoctor } from "./doctor.js";
|
|
@@ -27,6 +30,11 @@ const OPTIONS = {
|
|
|
27
30
|
cwd: { type: "string" },
|
|
28
31
|
store: { type: "string" },
|
|
29
32
|
force: { type: "boolean" },
|
|
33
|
+
"no-artifacts": { type: "boolean" },
|
|
34
|
+
coverage: { type: "boolean" },
|
|
35
|
+
hooks: { type: "boolean" },
|
|
36
|
+
mcp: { type: "boolean" },
|
|
37
|
+
feedback: { type: "boolean" },
|
|
30
38
|
package: { type: "string", short: "p" },
|
|
31
39
|
limit: { type: "string" },
|
|
32
40
|
"max-tokens": { type: "string" },
|
|
@@ -141,6 +149,12 @@ function reportDoctor(report: DoctorReport, quiet: boolean): void {
|
|
|
141
149
|
}
|
|
142
150
|
|
|
143
151
|
/** 0 when the query was answered; otherwise which of the two empty answers this was. */
|
|
152
|
+
/** Names, capped. A hundred identifiers on one line is a wall, and `--json` has them all. */
|
|
153
|
+
function listed(names: readonly string[], limit = 20): string {
|
|
154
|
+
if (names.length <= limit) return names.join(", ");
|
|
155
|
+
return `${names.slice(0, limit).join(", ")} … and ${names.length - limit} more (--json for all)`;
|
|
156
|
+
}
|
|
157
|
+
|
|
144
158
|
function queryExit(result: QueryResult): number {
|
|
145
159
|
if (result.hits.length > 0) return 0;
|
|
146
160
|
return result.unindexed.length > 0 ? EXIT_NOT_INDEXED : EXIT_NO_MATCH;
|
|
@@ -201,6 +215,7 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
201
215
|
cwd,
|
|
202
216
|
store,
|
|
203
217
|
...(values.force === true ? { force: true } : {}),
|
|
218
|
+
...(values["no-artifacts"] === true ? { artifacts: false } : {}),
|
|
204
219
|
...(quiet || json
|
|
205
220
|
? {}
|
|
206
221
|
: {
|
|
@@ -218,8 +233,11 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
218
233
|
for (const pkg of result.packages) {
|
|
219
234
|
const mark = pkg.status === "indexed" ? green("+") : dim("=");
|
|
220
235
|
const trust = pkg.trusted ? "" : ` ${yellow("(community)")}`;
|
|
236
|
+
// Declarations read from an installed build are not documentation somebody wrote, and
|
|
237
|
+
// the listing says so rather than letting them pass for it.
|
|
238
|
+
const kind = pkg.kind === "artifact" ? ` ${dim("(declarations)")}` : "";
|
|
221
239
|
process.stdout.write(
|
|
222
|
-
`${mark} ${bold(pkg.id)} ${pkg.chunks} chunks ${dim(pkg.status)}${trust}\n`,
|
|
240
|
+
`${mark} ${bold(pkg.id)} ${pkg.chunks} chunks ${dim(pkg.status)}${trust}${kind}\n`,
|
|
223
241
|
);
|
|
224
242
|
}
|
|
225
243
|
for (const problem of result.problems) {
|
|
@@ -343,17 +361,38 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
343
361
|
const store = openStore(values);
|
|
344
362
|
try {
|
|
345
363
|
const { packages, problems } = await discoverPackages(cwd);
|
|
346
|
-
const
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
364
|
+
const wantCoverage = values.coverage === true || json;
|
|
365
|
+
const rows = [];
|
|
366
|
+
for (const pkg of packages) {
|
|
367
|
+
rows.push({
|
|
368
|
+
id: pkg.id,
|
|
369
|
+
name: pkg.name,
|
|
370
|
+
version: pkg.version,
|
|
371
|
+
chunks: pkg.manifest.chunks.length,
|
|
372
|
+
trusted: pkg.trusted,
|
|
373
|
+
indexed: store.hasPackage(pkg.id),
|
|
374
|
+
...(wantCoverage
|
|
375
|
+
? {
|
|
376
|
+
coverage: await measureCoverage({
|
|
377
|
+
packageDir: pkg.dir,
|
|
378
|
+
llmsDir: pkg.llmsDir,
|
|
379
|
+
manifest: pkg.manifest,
|
|
380
|
+
cwd,
|
|
381
|
+
}),
|
|
382
|
+
}
|
|
383
|
+
: {}),
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
const declarations = store
|
|
388
|
+
.listPackages()
|
|
389
|
+
.filter((pkg) => pkg.kind === "artifact")
|
|
390
|
+
.map((pkg) => ({ id: pkg.id, chunks: store.countChunks(pkg.id) }));
|
|
354
391
|
|
|
355
392
|
if (json) {
|
|
356
|
-
process.stdout.write(
|
|
393
|
+
process.stdout.write(
|
|
394
|
+
`${JSON.stringify({ packages: rows, declarations, problems }, null, 2)}\n`,
|
|
395
|
+
);
|
|
357
396
|
return 0;
|
|
358
397
|
}
|
|
359
398
|
if (rows.length === 0) {
|
|
@@ -363,6 +402,24 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
363
402
|
const state = row.indexed ? green("indexed") : yellow("not indexed");
|
|
364
403
|
const trust = row.trusted ? "" : ` ${yellow("(community)")}`;
|
|
365
404
|
process.stdout.write(`${bold(row.id)} ${row.chunks} chunks ${state}${trust}\n`);
|
|
405
|
+
for (const library of row.coverage?.libraries ?? []) {
|
|
406
|
+
const percent =
|
|
407
|
+
library.names === 0 ? 0 : Math.round((100 * library.documented) / library.names);
|
|
408
|
+
process.stdout.write(
|
|
409
|
+
` ${dim(`documents ${library.documented}/${library.names} (${percent}%) of ${library.library}'s exports`)}\n`,
|
|
410
|
+
);
|
|
411
|
+
if (library.uncovered.length > 0) {
|
|
412
|
+
process.stdout.write(` ${dim(`missing: ${library.uncovered.join(", ")}`)}\n`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (declarations.length > 0 && !quiet) {
|
|
417
|
+
const chunks = declarations.reduce((total, row) => total + row.chunks, 0);
|
|
418
|
+
process.stdout.write(
|
|
419
|
+
dim(
|
|
420
|
+
`\n${declarations.length} installed libraries indexed by declaration, ${chunks} names in all.\n`,
|
|
421
|
+
),
|
|
422
|
+
);
|
|
366
423
|
}
|
|
367
424
|
for (const problem of problems) process.stderr.write(`${yellow("!")} ${problem}\n`);
|
|
368
425
|
return 0;
|
|
@@ -371,6 +428,106 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
371
428
|
}
|
|
372
429
|
}
|
|
373
430
|
|
|
431
|
+
case "agent": {
|
|
432
|
+
const [sub = "install"] = rest;
|
|
433
|
+
if (sub !== "install" && sub !== "check") {
|
|
434
|
+
throw new DocspackError(`Unknown subcommand "${sub}"`, {
|
|
435
|
+
hint: "Usage: docspack agent <install|check>",
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
const plan = await planAgentSetup({
|
|
440
|
+
cwd,
|
|
441
|
+
...(values.feedback === true ? { feedback: true } : {}),
|
|
442
|
+
...(values.hooks === true ? { hooks: true } : {}),
|
|
443
|
+
...(values.mcp === true ? { mcp: true } : {}),
|
|
444
|
+
});
|
|
445
|
+
const pending = plan.files.filter((file) => file.status !== "unchanged");
|
|
446
|
+
|
|
447
|
+
if (json) {
|
|
448
|
+
process.stdout.write(
|
|
449
|
+
`${JSON.stringify({ files: plan.files.map(({ contents, ...rest }) => rest) }, null, 2)}\n`,
|
|
450
|
+
);
|
|
451
|
+
return sub === "check" && pending.length > 0 ? 1 : 0;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// `check` is the CI half: it writes nothing and fails when the wiring is missing or stale,
|
|
455
|
+
// which is the only way a pasted instruction ever gets noticed after it drifts.
|
|
456
|
+
if (sub === "check") {
|
|
457
|
+
for (const file of plan.files) {
|
|
458
|
+
const mark = file.status === "unchanged" ? green("ok") : yellow(file.status);
|
|
459
|
+
process.stdout.write(`${mark} ${file.path}\n`);
|
|
460
|
+
}
|
|
461
|
+
if (pending.length > 0) {
|
|
462
|
+
process.stderr.write(
|
|
463
|
+
`\n${yellow("!")} ${plural(pending.length, "file")} out of date. Run \`docspack agent install\`.\n`,
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
return pending.length > 0 ? 1 : 0;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (values["dry-run"] === true) {
|
|
470
|
+
for (const file of plan.files) {
|
|
471
|
+
process.stdout.write(
|
|
472
|
+
`${file.status === "unchanged" ? dim("=") : green("+")} ${bold(file.path)} ${dim(file.reason)}\n`,
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
return 0;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
const written = await applyAgentSetup({ cwd }, plan);
|
|
479
|
+
for (const file of plan.files) {
|
|
480
|
+
const mark = file.status === "unchanged" ? dim("=") : green("+");
|
|
481
|
+
process.stdout.write(
|
|
482
|
+
`${mark} ${bold(file.path)} ${dim(file.status === "unchanged" ? "unchanged" : file.reason)}\n`,
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
if (written.length > 0 && !quiet) {
|
|
486
|
+
process.stdout.write(
|
|
487
|
+
`\n${dim("Commit these. Every agent working in this repository can now read the")}\n${dim("documentation of its dependencies, without anyone being told the command exists.")}\n`,
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
return 0;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
case "changed": {
|
|
494
|
+
const library = rest[0];
|
|
495
|
+
if (library === undefined) {
|
|
496
|
+
throw new DocspackError("Usage: docspack changed <library>[@version]");
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
const store = openStore(values);
|
|
500
|
+
try {
|
|
501
|
+
const change = await changedSurface({ cwd, store, library });
|
|
502
|
+
if (json) {
|
|
503
|
+
process.stdout.write(`${JSON.stringify(change, null, 2)}\n`);
|
|
504
|
+
return 0;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
process.stdout.write(
|
|
508
|
+
`${bold(`${change.library} ${change.from} → ${change.to}`)} ${dim(
|
|
509
|
+
`${change.added.length} added, ${change.removed.length} removed`,
|
|
510
|
+
)}\n`,
|
|
511
|
+
);
|
|
512
|
+
if (change.removed.length > 0) {
|
|
513
|
+
process.stdout.write(`\n${yellow("gone")} ${listed(change.removed)}\n`);
|
|
514
|
+
}
|
|
515
|
+
if (change.added.length > 0) {
|
|
516
|
+
process.stdout.write(`\n${green("new")} ${listed(change.added)}\n`);
|
|
517
|
+
}
|
|
518
|
+
if (change.undocumented.length > 0) {
|
|
519
|
+
process.stdout.write(
|
|
520
|
+
`\n${dim(
|
|
521
|
+
`${plural(change.undocumented.length, "new name")} that no documentation package here mentions:`,
|
|
522
|
+
)}\n${listed(change.undocumented)}\n`,
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
return 0;
|
|
526
|
+
} finally {
|
|
527
|
+
store.close();
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
374
531
|
case "verify": {
|
|
375
532
|
const report = await verifyProject({
|
|
376
533
|
cwd,
|
package/src/coverage.ts
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { resolvePackageDir } from "./discovery.js";
|
|
3
|
+
import { formatDocumentedLibrary, type PackageManifest, resolveChunkFile } from "./spec.js";
|
|
4
|
+
import { readPublicSurface } from "./surface.js";
|
|
5
|
+
import { documentedLibraries } from "./verify.js";
|
|
6
|
+
|
|
7
|
+
/** How much of one library's public surface the documentation mentions. */
|
|
8
|
+
export interface LibraryCoverage {
|
|
9
|
+
readonly library: string;
|
|
10
|
+
/** Exported names long enough to search prose for. */
|
|
11
|
+
readonly names: number;
|
|
12
|
+
/** Those the documentation mentions at least once, anywhere. */
|
|
13
|
+
readonly documented: number;
|
|
14
|
+
/** A sample of what it does not mention, for an author to act on. */
|
|
15
|
+
readonly uncovered: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface CoverageReport {
|
|
19
|
+
readonly libraries: readonly LibraryCoverage[];
|
|
20
|
+
/** Libraries the package documents whose declarations could not be read. */
|
|
21
|
+
readonly unreadable: readonly string[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* A name shorter than this is too generic to look for in prose, and its absence would say
|
|
26
|
+
* nothing: `Env`, `z`, `app`.
|
|
27
|
+
*/
|
|
28
|
+
const MIN_NAME = 4;
|
|
29
|
+
|
|
30
|
+
/** How many uncovered names are named in the report. */
|
|
31
|
+
const SAMPLE = 12;
|
|
32
|
+
|
|
33
|
+
export interface CoverageOptions {
|
|
34
|
+
readonly packageDir: string;
|
|
35
|
+
readonly llmsDir: string;
|
|
36
|
+
readonly manifest: PackageManifest;
|
|
37
|
+
/** Project the library is resolved from when the docs package does not depend on it itself. */
|
|
38
|
+
readonly cwd: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Compares what a documentation package says against what the library it documents exports.
|
|
43
|
+
*
|
|
44
|
+
* A documentation site is written page by page around tasks and an API grows name by name, so
|
|
45
|
+
* the two drift apart with nobody noticing — measured across two well-documented projects, half
|
|
46
|
+
* of the exported surface is mentioned nowhere. This is the number that makes that visible, and
|
|
47
|
+
* it is mechanical: two files, no model, no judgement.
|
|
48
|
+
*/
|
|
49
|
+
export async function measureCoverage(options: CoverageOptions): Promise<CoverageReport> {
|
|
50
|
+
const libraries = await documentedLibraries(options.packageDir, options.manifest);
|
|
51
|
+
if (libraries.length === 0) return { libraries: [], unreadable: [] };
|
|
52
|
+
|
|
53
|
+
const text = await readCorpus(options);
|
|
54
|
+
const mentioned = identifiers(text);
|
|
55
|
+
const covered: LibraryCoverage[] = [];
|
|
56
|
+
const unreadable: string[] = [];
|
|
57
|
+
|
|
58
|
+
for (const library of libraries) {
|
|
59
|
+
const dir =
|
|
60
|
+
(await resolvePackageDir(library.name, options.packageDir)) ??
|
|
61
|
+
(await resolvePackageDir(library.name, options.cwd));
|
|
62
|
+
const surface = dir === undefined ? undefined : await readPublicSurface(dir);
|
|
63
|
+
if (surface === undefined) {
|
|
64
|
+
unreadable.push(formatDocumentedLibrary(library));
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const names = [...surface.symbols.keys()].filter((name) => name.length >= MIN_NAME);
|
|
69
|
+
const uncovered = names.filter((name) => !mentioned.has(name));
|
|
70
|
+
covered.push({
|
|
71
|
+
library: formatDocumentedLibrary(library),
|
|
72
|
+
names: names.length,
|
|
73
|
+
documented: names.length - uncovered.length,
|
|
74
|
+
uncovered: uncovered.slice(0, SAMPLE),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return { libraries: covered, unreadable };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Splits text into the identifier-shaped tokens an exact mention would produce. */
|
|
82
|
+
export function identifiers(text: string): Set<string> {
|
|
83
|
+
return new Set(text.split(/[^A-Za-z0-9_$]+/).filter((token) => token.length > 0));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function readCorpus(options: CoverageOptions): Promise<string> {
|
|
87
|
+
const parts: string[] = [];
|
|
88
|
+
for (const chunk of options.manifest.chunks) {
|
|
89
|
+
try {
|
|
90
|
+
parts.push(await readFile(resolveChunkFile(options.llmsDir, chunk.file), "utf8"));
|
|
91
|
+
} catch {
|
|
92
|
+
// A chunk that cannot be read is doctor's finding to report, not coverage's.
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return parts.join("\n");
|
|
96
|
+
}
|
package/src/db.ts
CHANGED
|
@@ -8,13 +8,29 @@ import { STOPWORDS } from "./stopwords.js";
|
|
|
8
8
|
|
|
9
9
|
const require = createRequire(import.meta.url);
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Where a package's text came from. `docs` is written by a human and published as a docs
|
|
13
|
+
* package; `artifact` is derived from an installed library's own type declarations. The
|
|
14
|
+
* distinction is not cosmetic — a signature is a fact about the build, prose is a claim by its
|
|
15
|
+
* author — so it is stored rather than inferred from the name.
|
|
16
|
+
*/
|
|
17
|
+
export type PackageKind = "docs" | "artifact";
|
|
18
|
+
|
|
11
19
|
export interface IndexedPackage {
|
|
12
20
|
readonly id: string;
|
|
13
21
|
readonly name: string;
|
|
14
22
|
readonly version: string;
|
|
23
|
+
readonly kind?: PackageKind;
|
|
15
24
|
readonly indexedAt?: string;
|
|
16
25
|
}
|
|
17
26
|
|
|
27
|
+
/** An exported name and the chunk carrying its declaration. */
|
|
28
|
+
export interface SymbolHit {
|
|
29
|
+
readonly name: string;
|
|
30
|
+
readonly packageId: string;
|
|
31
|
+
readonly chunkId: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
18
34
|
export interface IndexedChunk {
|
|
19
35
|
readonly chunkId: string;
|
|
20
36
|
readonly filePath: string;
|
|
@@ -39,15 +55,18 @@ export interface SearchOptions {
|
|
|
39
55
|
readonly limit?: number;
|
|
40
56
|
/** Stop returning chunks once this many tokens have been collected. */
|
|
41
57
|
readonly maxTokens?: number;
|
|
58
|
+
/** Restrict to packages of these kinds. Defaults to every kind. */
|
|
59
|
+
readonly kinds?: readonly PackageKind[];
|
|
42
60
|
}
|
|
43
61
|
|
|
44
|
-
const SCHEMA_VERSION =
|
|
62
|
+
const SCHEMA_VERSION = 2;
|
|
45
63
|
|
|
46
64
|
const SCHEMA = `
|
|
47
65
|
CREATE TABLE IF NOT EXISTS packages (
|
|
48
66
|
id TEXT PRIMARY KEY,
|
|
49
67
|
name TEXT NOT NULL,
|
|
50
68
|
version TEXT NOT NULL,
|
|
69
|
+
kind TEXT NOT NULL DEFAULT 'docs',
|
|
51
70
|
indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
52
71
|
);
|
|
53
72
|
|
|
@@ -61,6 +80,15 @@ CREATE TABLE IF NOT EXISTS chunks (
|
|
|
61
80
|
|
|
62
81
|
CREATE INDEX IF NOT EXISTS chunks_by_package ON chunks(package_id);
|
|
63
82
|
|
|
83
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
84
|
+
package_id TEXT NOT NULL REFERENCES packages(id),
|
|
85
|
+
name TEXT NOT NULL,
|
|
86
|
+
chunk_id TEXT NOT NULL,
|
|
87
|
+
PRIMARY KEY (package_id, name)
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
CREATE INDEX IF NOT EXISTS symbols_by_name ON symbols(name);
|
|
91
|
+
|
|
64
92
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
65
93
|
content,
|
|
66
94
|
tags,
|
|
@@ -133,9 +161,22 @@ export class Store {
|
|
|
133
161
|
this.#db = db;
|
|
134
162
|
this.#db.exec("PRAGMA journal_mode = WAL");
|
|
135
163
|
this.#db.exec(SCHEMA);
|
|
164
|
+
this.#migrate();
|
|
136
165
|
this.#db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
|
|
137
166
|
}
|
|
138
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Brings a store written by an older version forward. `CREATE TABLE IF NOT EXISTS` covers a
|
|
170
|
+
* new table, but not a new column on one that already exists, and rebuilding the whole index
|
|
171
|
+
* to add one would make an upgrade re-read every package on the machine.
|
|
172
|
+
*/
|
|
173
|
+
#migrate(): void {
|
|
174
|
+
const columns = this.#db.prepare("PRAGMA table_info(packages)").all() as { name: string }[];
|
|
175
|
+
if (!columns.some((column) => column.name === "kind")) {
|
|
176
|
+
this.#db.exec("ALTER TABLE packages ADD COLUMN kind TEXT NOT NULL DEFAULT 'docs'");
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
139
180
|
static open(path: string = defaultStorePath()): Store {
|
|
140
181
|
if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
|
|
141
182
|
try {
|
|
@@ -154,16 +195,115 @@ export class Store {
|
|
|
154
195
|
|
|
155
196
|
listPackages(): IndexedPackage[] {
|
|
156
197
|
const rows = this.#db
|
|
157
|
-
.prepare("SELECT id, name, version, indexed_at FROM packages ORDER BY name, version")
|
|
158
|
-
.all() as {
|
|
198
|
+
.prepare("SELECT id, name, version, kind, indexed_at FROM packages ORDER BY name, version")
|
|
199
|
+
.all() as {
|
|
200
|
+
id: string;
|
|
201
|
+
name: string;
|
|
202
|
+
version: string;
|
|
203
|
+
kind: string;
|
|
204
|
+
indexed_at: string;
|
|
205
|
+
}[];
|
|
159
206
|
return rows.map((row) => ({
|
|
160
207
|
id: row.id,
|
|
161
208
|
name: row.name,
|
|
162
209
|
version: row.version,
|
|
210
|
+
kind: row.kind === "artifact" ? "artifact" : "docs",
|
|
163
211
|
indexedAt: row.indexed_at,
|
|
164
212
|
}));
|
|
165
213
|
}
|
|
166
214
|
|
|
215
|
+
/** Every version of one library in the store, newest indexing first. */
|
|
216
|
+
versionsOf(name: string): IndexedPackage[] {
|
|
217
|
+
return this.listPackages().filter((pkg) => pkg.name === name);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Chunks declaring an exported name, by exact match rather than by ranking.
|
|
222
|
+
*
|
|
223
|
+
* A symbol is a key, not a phrase. Ranking one is the mistake that made `docspack ask` unable
|
|
224
|
+
* to tell "the documentation does not mention this" from "nothing matched".
|
|
225
|
+
*/
|
|
226
|
+
lookupSymbol(name: string, packageIds?: readonly string[]): SymbolHit[] {
|
|
227
|
+
if (packageIds !== undefined && packageIds.length === 0) return [];
|
|
228
|
+
const scope =
|
|
229
|
+
packageIds === undefined
|
|
230
|
+
? ""
|
|
231
|
+
: ` AND package_id IN (${packageIds.map(() => "?").join(", ")})`;
|
|
232
|
+
const rows = this.#db
|
|
233
|
+
.prepare(
|
|
234
|
+
// A package that declares the name comes first: an empty chunk id means the name is
|
|
235
|
+
// exported but its declaration was not readable, which answers less.
|
|
236
|
+
`SELECT name, package_id AS packageId, chunk_id AS chunkId
|
|
237
|
+
FROM symbols WHERE name = ?${scope}
|
|
238
|
+
ORDER BY (chunk_id = '') ASC, package_id`,
|
|
239
|
+
)
|
|
240
|
+
.all(name, ...(packageIds ?? [])) as {
|
|
241
|
+
name: string;
|
|
242
|
+
packageId: string;
|
|
243
|
+
chunkId: string;
|
|
244
|
+
}[];
|
|
245
|
+
return rows.map((row) => ({ name: row.name, packageId: row.packageId, chunkId: row.chunkId }));
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Every chunk of one package, for a check that has to read the whole corpus. */
|
|
249
|
+
chunksOf(packageId: string): SearchHit[] {
|
|
250
|
+
const rows = this.#db
|
|
251
|
+
.prepare(
|
|
252
|
+
`SELECT chunk_id AS chunkId, package_id AS packageId, file_path AS filePath,
|
|
253
|
+
tokens, content
|
|
254
|
+
FROM chunks WHERE package_id = ?`,
|
|
255
|
+
)
|
|
256
|
+
.all(packageId) as {
|
|
257
|
+
chunkId: string;
|
|
258
|
+
packageId: string;
|
|
259
|
+
filePath: string;
|
|
260
|
+
tokens: number;
|
|
261
|
+
content: string;
|
|
262
|
+
}[];
|
|
263
|
+
return rows.map((row) => ({
|
|
264
|
+
chunkId: row.chunkId,
|
|
265
|
+
packageId: row.packageId,
|
|
266
|
+
filePath: row.filePath,
|
|
267
|
+
tokens: Number(row.tokens ?? 0),
|
|
268
|
+
content: row.content,
|
|
269
|
+
}));
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** Every exported name recorded for one package. */
|
|
273
|
+
symbolsOf(packageId: string): string[] {
|
|
274
|
+
const rows = this.#db
|
|
275
|
+
.prepare("SELECT name FROM symbols WHERE package_id = ? ORDER BY name")
|
|
276
|
+
.all(packageId) as { name: string }[];
|
|
277
|
+
return rows.map((row) => row.name);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** One chunk by its id, however it was found. */
|
|
281
|
+
chunk(chunkId: string): SearchHit | undefined {
|
|
282
|
+
const row = this.#db
|
|
283
|
+
.prepare(
|
|
284
|
+
`SELECT chunk_id AS chunkId, package_id AS packageId, file_path AS filePath,
|
|
285
|
+
tokens, content
|
|
286
|
+
FROM chunks WHERE chunk_id = ?`,
|
|
287
|
+
)
|
|
288
|
+
.get(chunkId) as
|
|
289
|
+
| {
|
|
290
|
+
chunkId: string;
|
|
291
|
+
packageId: string;
|
|
292
|
+
filePath: string;
|
|
293
|
+
tokens: number;
|
|
294
|
+
content: string;
|
|
295
|
+
}
|
|
296
|
+
| undefined;
|
|
297
|
+
if (row === undefined) return undefined;
|
|
298
|
+
return {
|
|
299
|
+
chunkId: row.chunkId,
|
|
300
|
+
packageId: row.packageId,
|
|
301
|
+
filePath: row.filePath,
|
|
302
|
+
tokens: Number(row.tokens ?? 0),
|
|
303
|
+
content: row.content,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
|
|
167
307
|
countChunks(packageId: string): number {
|
|
168
308
|
const row = this.#db
|
|
169
309
|
.prepare("SELECT COUNT(*) AS total FROM chunks WHERE package_id = ?")
|
|
@@ -171,10 +311,19 @@ export class Store {
|
|
|
171
311
|
return row.total;
|
|
172
312
|
}
|
|
173
313
|
|
|
174
|
-
/**
|
|
175
|
-
|
|
314
|
+
/**
|
|
315
|
+
* Replaces a package, all of its chunks and its symbol table in a single transaction.
|
|
316
|
+
*
|
|
317
|
+
* `symbols` maps an exported name to the chunk declaring it, and is how a query about a name
|
|
318
|
+
* is answered without ranking anything.
|
|
319
|
+
*/
|
|
320
|
+
indexPackage(
|
|
321
|
+
pkg: IndexedPackage,
|
|
322
|
+
chunks: readonly IndexedChunk[],
|
|
323
|
+
symbols?: ReadonlyMap<string, string>,
|
|
324
|
+
): void {
|
|
176
325
|
const insertPackage = this.#db.prepare(
|
|
177
|
-
"INSERT OR REPLACE INTO packages (id, name, version) VALUES (?, ?, ?)",
|
|
326
|
+
"INSERT OR REPLACE INTO packages (id, name, version, kind) VALUES (?, ?, ?, ?)",
|
|
178
327
|
);
|
|
179
328
|
const insertChunk = this.#db.prepare(
|
|
180
329
|
"INSERT OR REPLACE INTO chunks (chunk_id, package_id, file_path, tokens, content) VALUES (?, ?, ?, ?, ?)",
|
|
@@ -182,15 +331,19 @@ export class Store {
|
|
|
182
331
|
const insertFts = this.#db.prepare(
|
|
183
332
|
"INSERT INTO chunks_fts (content, tags, package_id, chunk_id) VALUES (?, ?, ?, ?)",
|
|
184
333
|
);
|
|
334
|
+
const insertSymbol = this.#db.prepare(
|
|
335
|
+
"INSERT OR REPLACE INTO symbols (package_id, name, chunk_id) VALUES (?, ?, ?)",
|
|
336
|
+
);
|
|
185
337
|
|
|
186
338
|
this.#db.exec("BEGIN");
|
|
187
339
|
try {
|
|
188
340
|
this.#deleteChunks(pkg.id);
|
|
189
|
-
insertPackage.run(pkg.id, pkg.name, pkg.version);
|
|
341
|
+
insertPackage.run(pkg.id, pkg.name, pkg.version, pkg.kind ?? "docs");
|
|
190
342
|
for (const chunk of chunks) {
|
|
191
343
|
insertChunk.run(chunk.chunkId, pkg.id, chunk.filePath, chunk.tokens, chunk.content);
|
|
192
344
|
insertFts.run(chunk.content, chunk.tags.join(" "), pkg.id, chunk.chunkId);
|
|
193
345
|
}
|
|
346
|
+
for (const [name, chunk] of symbols ?? []) insertSymbol.run(pkg.id, name, chunk);
|
|
194
347
|
this.#db.exec("COMMIT");
|
|
195
348
|
} catch (error) {
|
|
196
349
|
this.#db.exec("ROLLBACK");
|
|
@@ -228,6 +381,13 @@ export class Store {
|
|
|
228
381
|
conditions.push("f.package_id LIKE ?");
|
|
229
382
|
parameters.push(options.packageFilter);
|
|
230
383
|
}
|
|
384
|
+
if (options.kinds !== undefined) {
|
|
385
|
+
if (options.kinds.length === 0) return [];
|
|
386
|
+
conditions.push(
|
|
387
|
+
`f.package_id IN (SELECT id FROM packages WHERE kind IN (${options.kinds.map(() => "?").join(", ")}))`,
|
|
388
|
+
);
|
|
389
|
+
parameters.push(...options.kinds);
|
|
390
|
+
}
|
|
231
391
|
|
|
232
392
|
const rows = this.#db
|
|
233
393
|
.prepare(
|
|
@@ -272,5 +432,6 @@ export class Store {
|
|
|
272
432
|
#deleteChunks(packageId: string): void {
|
|
273
433
|
this.#db.prepare("DELETE FROM chunks_fts WHERE package_id = ?").run(packageId);
|
|
274
434
|
this.#db.prepare("DELETE FROM chunks WHERE package_id = ?").run(packageId);
|
|
435
|
+
this.#db.prepare("DELETE FROM symbols WHERE package_id = ?").run(packageId);
|
|
275
436
|
}
|
|
276
437
|
}
|
package/src/discovery.ts
CHANGED
|
@@ -23,6 +23,14 @@ export interface DiscoveredPackage {
|
|
|
23
23
|
readonly trusted: boolean;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
/** An ordinary dependency: a library this project installed, whatever documents it. */
|
|
27
|
+
export interface DiscoveredLibrary {
|
|
28
|
+
readonly name: string;
|
|
29
|
+
readonly version: string;
|
|
30
|
+
/** Root of the installed package. */
|
|
31
|
+
readonly dir: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
26
34
|
export interface Discovery {
|
|
27
35
|
readonly packages: readonly DiscoveredPackage[];
|
|
28
36
|
/** Human-readable problems that did not stop discovery, e.g. a declared but uninstalled package. */
|
|
@@ -90,19 +98,48 @@ export async function discoverPackages(cwd: string): Promise<Discovery> {
|
|
|
90
98
|
return { packages, problems };
|
|
91
99
|
}
|
|
92
100
|
|
|
101
|
+
/**
|
|
102
|
+
* Every library this project depends on directly, installed and readable.
|
|
103
|
+
*
|
|
104
|
+
* Documentation packages are excluded: those are handled by `discoverPackages`, and a docs
|
|
105
|
+
* package's own type declarations describe nothing anybody asks about.
|
|
106
|
+
*/
|
|
107
|
+
export async function discoverLibraries(cwd: string): Promise<readonly DiscoveredLibrary[]> {
|
|
108
|
+
const root = resolve(cwd);
|
|
109
|
+
const names = (await declaredDependencies(root)).filter((name) => !isDocsPackage(name));
|
|
110
|
+
const libraries: DiscoveredLibrary[] = [];
|
|
111
|
+
|
|
112
|
+
for (const name of names) {
|
|
113
|
+
const dir = await resolvePackageDir(name, root);
|
|
114
|
+
if (dir === undefined) continue;
|
|
115
|
+
try {
|
|
116
|
+
libraries.push({ name, version: await installedVersion(name, dir), dir });
|
|
117
|
+
} catch {
|
|
118
|
+
// A dependency with no version in its manifest is not one to index.
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return libraries;
|
|
123
|
+
}
|
|
124
|
+
|
|
93
125
|
/** Package ids installed in this project, used to scope queries to the versions actually in use. */
|
|
94
126
|
export async function projectPackageIds(cwd: string): Promise<string[]> {
|
|
95
127
|
const { packages } = await discoverPackages(cwd);
|
|
96
128
|
return packages.map((pkg) => pkg.id);
|
|
97
129
|
}
|
|
98
130
|
|
|
131
|
+
/** Docs packages this project depends on, the input to `discoverPackages`. */
|
|
132
|
+
async function declaredDocsPackages(cwd: string): Promise<string[]> {
|
|
133
|
+
return (await declaredDependencies(cwd)).filter(isDocsPackage);
|
|
134
|
+
}
|
|
135
|
+
|
|
99
136
|
/**
|
|
100
|
-
* Every
|
|
137
|
+
* Every dependency declared by this directory or by an ancestor of it. A workspace declares
|
|
101
138
|
* shared tooling in the repository root and its members inherit the install, so reading only
|
|
102
139
|
* `cwd/package.json` answers "this project has no documentation" for most monorepos — the same
|
|
103
140
|
* upward walk `resolvePackageDir` already does for the install.
|
|
104
141
|
*/
|
|
105
|
-
async function
|
|
142
|
+
async function declaredDependencies(cwd: string): Promise<string[]> {
|
|
106
143
|
const names = new Set<string>();
|
|
107
144
|
let found = false;
|
|
108
145
|
let dir = cwd;
|
|
@@ -131,9 +168,7 @@ async function declaredDocsPackages(cwd: string): Promise<string[]> {
|
|
|
131
168
|
const manifest = parsed as { dependencies?: unknown; devDependencies?: unknown };
|
|
132
169
|
for (const field of [manifest.dependencies, manifest.devDependencies]) {
|
|
133
170
|
if (typeof field !== "object" || field === null) continue;
|
|
134
|
-
for (const name of Object.keys(field as Record<string, unknown>))
|
|
135
|
-
if (isDocsPackage(name)) names.add(name);
|
|
136
|
-
}
|
|
171
|
+
for (const name of Object.keys(field as Record<string, unknown>)) names.add(name);
|
|
137
172
|
}
|
|
138
173
|
|
|
139
174
|
const parent = dirname(dir);
|