docspack 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +48 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +243 -0
- package/dist/agent.js.map +1 -0
- package/dist/artifact.d.ts +32 -0
- package/dist/artifact.d.ts.map +1 -0
- package/dist/artifact.js +78 -0
- package/dist/artifact.js.map +1 -0
- package/dist/build.d.ts.map +1 -1
- package/dist/build.js +75 -9
- package/dist/build.js.map +1 -1
- package/dist/changed.d.ts +31 -0
- package/dist/changed.d.ts.map +1 -0
- package/dist/changed.js +71 -0
- package/dist/changed.js.map +1 -0
- package/dist/cli.js +131 -10
- package/dist/cli.js.map +1 -1
- package/dist/coverage.d.ts +35 -0
- package/dist/coverage.d.ts.map +1 -0
- package/dist/coverage.js +64 -0
- package/dist/coverage.js.map +1 -0
- package/dist/db.d.ts +38 -2
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +109 -6
- package/dist/db.js.map +1 -1
- package/dist/discovery.d.ts +14 -0
- package/dist/discovery.d.ts.map +1 -1
- package/dist/discovery.js +31 -6
- package/dist/discovery.js.map +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +90 -3
- package/dist/doctor.js.map +1 -1
- package/dist/document.d.ts +2 -0
- package/dist/document.d.ts.map +1 -1
- package/dist/document.js +7 -3
- package/dist/document.js.map +1 -1
- package/dist/help.d.ts.map +1 -1
- package/dist/help.js +65 -4
- package/dist/help.js.map +1 -1
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +3 -0
- package/dist/mcp.js.map +1 -1
- package/dist/preview.d.ts.map +1 -1
- package/dist/preview.js +1 -0
- package/dist/preview.js.map +1 -1
- package/dist/search.d.ts +17 -3
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +116 -21
- package/dist/search.js.map +1 -1
- package/dist/spec.d.ts +6 -0
- package/dist/spec.d.ts.map +1 -1
- package/dist/spec.js +12 -3
- package/dist/spec.js.map +1 -1
- package/dist/surface.d.ts +45 -0
- package/dist/surface.d.ts.map +1 -0
- package/dist/surface.js +208 -0
- package/dist/surface.js.map +1 -0
- package/dist/sync.d.ts +8 -1
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +50 -1
- package/dist/sync.js.map +1 -1
- package/dist/verify.d.ts +9 -0
- package/dist/verify.d.ts.map +1 -1
- package/dist/verify.js +55 -27
- package/dist/verify.js.map +1 -1
- package/package.json +1 -1
- package/src/agent.ts +308 -0
- package/src/artifact.ts +110 -0
- package/src/build.ts +103 -10
- package/src/changed.ts +99 -0
- package/src/cli.ts +167 -10
- package/src/coverage.ts +96 -0
- package/src/db.ts +168 -7
- package/src/discovery.ts +40 -5
- package/src/doctor.ts +100 -4
- package/src/document.ts +13 -4
- package/src/help.ts +65 -4
- package/src/index.ts +30 -0
- package/src/mcp.ts +3 -0
- package/src/preview.ts +1 -0
- package/src/search.ts +158 -24
- package/src/spec.ts +21 -3
- package/src/surface.ts +265 -0
- package/src/sync.ts +66 -2
- package/src/verify.ts +57 -27
package/src/artifact.ts
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { IndexedChunk } from "./db.js";
|
|
4
|
+
import { chunkId, estimateTokens, packageId } from "./spec.js";
|
|
5
|
+
import { declarationsIn, type ExportedSymbol, readPublicSurface } from "./surface.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A package derived from an installed library rather than published by its author.
|
|
9
|
+
*
|
|
10
|
+
* Half of a well-documented library's exported names are mentioned nowhere in its own
|
|
11
|
+
* documentation (see `docs/design/beyond-retrieval.md`), and almost all of them are declared in
|
|
12
|
+
* the type declarations sitting in `node_modules`. Those declarations are the answer a
|
|
13
|
+
* documentation site cannot serve, because the site is not the artifact.
|
|
14
|
+
*/
|
|
15
|
+
export interface ArtifactPackage {
|
|
16
|
+
readonly id: string;
|
|
17
|
+
readonly name: string;
|
|
18
|
+
readonly version: string;
|
|
19
|
+
readonly chunks: readonly IndexedChunk[];
|
|
20
|
+
/**
|
|
21
|
+
* Every exported name, mapped to the chunk carrying its declaration — or to an empty string
|
|
22
|
+
* when the package exports the name without declaring it anywhere this can read.
|
|
23
|
+
*
|
|
24
|
+
* Recording those too is what keeps `changed` honest: a name re-exported from a file with no
|
|
25
|
+
* declaration in it would otherwise look like a name the release removed.
|
|
26
|
+
*/
|
|
27
|
+
readonly symbols: ReadonlyMap<string, string>;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Beyond this a package is generated machinery, and indexing it would drown the prose. */
|
|
31
|
+
const MAX_SYMBOLS = 2000;
|
|
32
|
+
|
|
33
|
+
/** How much of a declaration is carried. Long enough for a signature, short enough to rank. */
|
|
34
|
+
const DECLARATION_LIMIT = 400;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Derives chunks from an installed library: one per exported name, carrying the import a caller
|
|
38
|
+
* writes and the declaration as the installed build states it.
|
|
39
|
+
*
|
|
40
|
+
* Returns nothing when the package ships no type declarations. That is the common case across a
|
|
41
|
+
* dependency tree and is not a problem worth reporting.
|
|
42
|
+
*/
|
|
43
|
+
export async function readArtifact(dir: string): Promise<ArtifactPackage | undefined> {
|
|
44
|
+
const surface = await readPublicSurface(dir);
|
|
45
|
+
if (surface === undefined || surface.name.length === 0 || surface.version.length === 0) {
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const id = packageId(surface.name, surface.version);
|
|
50
|
+
const declared = new Map<string, Map<string, string>>();
|
|
51
|
+
const chunks: IndexedChunk[] = [];
|
|
52
|
+
const symbols = new Map<string, string>();
|
|
53
|
+
|
|
54
|
+
for (const symbol of surface.symbols.values()) {
|
|
55
|
+
symbols.set(symbol.name, "");
|
|
56
|
+
if (chunks.length >= MAX_SYMBOLS) continue;
|
|
57
|
+
|
|
58
|
+
let declarations = declared.get(symbol.file);
|
|
59
|
+
if (declarations === undefined) {
|
|
60
|
+
try {
|
|
61
|
+
declarations = declarationsIn(
|
|
62
|
+
await readFile(join(dir, symbol.file), "utf8"),
|
|
63
|
+
DECLARATION_LIMIT,
|
|
64
|
+
);
|
|
65
|
+
} catch {
|
|
66
|
+
declarations = new Map();
|
|
67
|
+
}
|
|
68
|
+
declared.set(symbol.file, declarations);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const declaration = declarations.get(symbol.name);
|
|
72
|
+
// A name that only appears in an `export { … }` list has no declaration to show. The import
|
|
73
|
+
// line alone would say nothing the name did not already say, so it is left out.
|
|
74
|
+
if (declaration === undefined) continue;
|
|
75
|
+
|
|
76
|
+
const content = render(symbol, declaration);
|
|
77
|
+
const chunk = chunkId(id, symbol.name);
|
|
78
|
+
chunks.push({
|
|
79
|
+
chunkId: chunk,
|
|
80
|
+
filePath: symbol.file,
|
|
81
|
+
tokens: estimateTokens(content),
|
|
82
|
+
content,
|
|
83
|
+
// The name is the tag, and tags are weighted 3× against prose in the ranking.
|
|
84
|
+
tags: [symbol.name, symbol.from],
|
|
85
|
+
});
|
|
86
|
+
symbols.set(symbol.name, chunk);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (symbols.size === 0) return undefined;
|
|
90
|
+
return { id, name: surface.name, version: surface.version, chunks, symbols };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* What an agent receives. The import line is first because it is the part a caller gets wrong
|
|
95
|
+
* most often — a name that exists at a subpath nobody guessed is the same failure as a name that
|
|
96
|
+
* does not exist.
|
|
97
|
+
*/
|
|
98
|
+
function render(symbol: ExportedSymbol, declaration: string): string {
|
|
99
|
+
return [
|
|
100
|
+
`# ${symbol.name}`,
|
|
101
|
+
"",
|
|
102
|
+
`Declared by \`${symbol.from}\`. This is the installed build's own declaration, not prose.`,
|
|
103
|
+
"",
|
|
104
|
+
"```ts",
|
|
105
|
+
`import { ${symbol.name} } from "${symbol.from}";`,
|
|
106
|
+
"",
|
|
107
|
+
declaration.trim(),
|
|
108
|
+
"```",
|
|
109
|
+
].join("\n");
|
|
110
|
+
}
|
package/src/build.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import type { Dirent } from "node:fs";
|
|
2
3
|
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
4
|
import { join, relative, sep } from "node:path";
|
|
@@ -15,7 +16,9 @@ import {
|
|
|
15
16
|
estimateTokens,
|
|
16
17
|
LLMS_DIR,
|
|
17
18
|
MANIFEST_FILE,
|
|
19
|
+
type PackageManifest,
|
|
18
20
|
parseDocumentedLibrary,
|
|
21
|
+
parseManifest,
|
|
19
22
|
serializeManifest,
|
|
20
23
|
} from "./spec.js";
|
|
21
24
|
import { STOPWORDS } from "./stopwords.js";
|
|
@@ -64,6 +67,8 @@ interface SourceDocument {
|
|
|
64
67
|
readonly atomic?: boolean;
|
|
65
68
|
readonly tags?: readonly string[];
|
|
66
69
|
readonly entities?: readonly string[];
|
|
70
|
+
/** Libraries this document describes, when that is narrower than the package's `documents`. */
|
|
71
|
+
readonly documents?: readonly string[];
|
|
67
72
|
}
|
|
68
73
|
|
|
69
74
|
const DEFAULT_MAX_CHUNK_TOKENS = 800;
|
|
@@ -94,9 +99,12 @@ export async function buildPackage(input: BuildOptions): Promise<BuildResult> {
|
|
|
94
99
|
{ hint: "A chunk cannot be required to be bigger than it is allowed to be." },
|
|
95
100
|
);
|
|
96
101
|
}
|
|
97
|
-
const { chunks, files } = chunkDocuments(documents, maxTokens, minTokens);
|
|
102
|
+
const { chunks, files, collisions } = chunkDocuments(documents, maxTokens, minTokens);
|
|
98
103
|
|
|
99
104
|
const llmsDir = join(options.out, LLMS_DIR);
|
|
105
|
+
// Read before the payload is removed: a chunk id is what `feedback add --chunk` pins and what
|
|
106
|
+
// an answer is headed with, so a rebuild that renames one silently orphans both.
|
|
107
|
+
warnings.push(...idWarnings(collisions, await departedChunkIds(llmsDir, chunks)));
|
|
100
108
|
await rm(llmsDir, { recursive: true, force: true });
|
|
101
109
|
await mkdir(join(llmsDir, CHUNKS_DIR), { recursive: true });
|
|
102
110
|
|
|
@@ -248,6 +256,7 @@ async function collectFromDirectory(dir: string): Promise<SourceDocument[]> {
|
|
|
248
256
|
origin: relativePath,
|
|
249
257
|
text: cleaned.text,
|
|
250
258
|
...(cleaned.tags.length === 0 ? {} : { tags: cleaned.tags }),
|
|
259
|
+
...(cleaned.documents.length === 0 ? {} : { documents: cleaned.documents }),
|
|
251
260
|
});
|
|
252
261
|
}
|
|
253
262
|
return documents;
|
|
@@ -273,6 +282,13 @@ async function collectFromSource(
|
|
|
273
282
|
|
|
274
283
|
const documents: SourceDocument[] = [];
|
|
275
284
|
const links = fetchableLinks(parsed).slice(0, options.pages ?? 50);
|
|
285
|
+
// Two links can name the same document. `fetchableLinks` already drops a repeated URL and a
|
|
286
|
+
// repeated fragment, but a site whose index links anchors as a query — `/v4?id=codecs` — gets
|
|
287
|
+
// one entry per anchor, and each one fetches the whole page again. Left alone that packages
|
|
288
|
+
// the same text dozens of times: an index mostly made of duplicates, which is slower to search
|
|
289
|
+
// and ranks worse, because copies of one page crowd out the rest of the corpus.
|
|
290
|
+
const seen = new Set<string>();
|
|
291
|
+
let duplicates = 0;
|
|
276
292
|
|
|
277
293
|
for (const link of links) {
|
|
278
294
|
options.onProgress?.(`fetching ${link.url}`);
|
|
@@ -285,6 +301,12 @@ async function collectFromSource(
|
|
|
285
301
|
warnings.push(`skipped ${link.url}: document was empty`);
|
|
286
302
|
continue;
|
|
287
303
|
}
|
|
304
|
+
const fingerprint = createHash("sha256").update(text).digest("hex");
|
|
305
|
+
if (seen.has(fingerprint)) {
|
|
306
|
+
duplicates += 1;
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
seen.add(fingerprint);
|
|
288
310
|
documents.push({
|
|
289
311
|
title: link.title.length > 0 ? link.title : (firstHeading(text) ?? link.url),
|
|
290
312
|
origin: link.url,
|
|
@@ -298,6 +320,11 @@ async function collectFromSource(
|
|
|
298
320
|
}
|
|
299
321
|
}
|
|
300
322
|
|
|
323
|
+
if (duplicates > 0) {
|
|
324
|
+
warnings.push(
|
|
325
|
+
`skipped ${duplicates} of ${links.length} linked documents whose text repeated one already packaged`,
|
|
326
|
+
);
|
|
327
|
+
}
|
|
301
328
|
if (fetchableLinks(parsed).length > links.length) {
|
|
302
329
|
warnings.push(
|
|
303
330
|
`packaged ${links.length} of ${fetchableLinks(parsed).length} linked documents (raise with --pages)`,
|
|
@@ -427,9 +454,14 @@ function chunkDocuments(
|
|
|
427
454
|
documents: readonly SourceDocument[],
|
|
428
455
|
maxTokens: number,
|
|
429
456
|
minTokens: number | undefined,
|
|
430
|
-
): {
|
|
457
|
+
): {
|
|
458
|
+
chunks: ChunkSpec[];
|
|
459
|
+
files: { path: string; contents: string }[];
|
|
460
|
+
collisions: string[];
|
|
461
|
+
} {
|
|
431
462
|
const chunks: ChunkSpec[] = [];
|
|
432
463
|
const files: { path: string; contents: string }[] = [];
|
|
464
|
+
const collisions: string[] = [];
|
|
433
465
|
const taken = new Set<string>();
|
|
434
466
|
|
|
435
467
|
for (const document of documents) {
|
|
@@ -442,7 +474,10 @@ function chunkDocuments(
|
|
|
442
474
|
for (const section of sections) {
|
|
443
475
|
const directives = readDirectives(section.body);
|
|
444
476
|
const body = withoutDuplicateHeading(directives.body, section.heading);
|
|
445
|
-
const
|
|
477
|
+
const derived = chunkSlug(document.title, section.heading);
|
|
478
|
+
const id = uniqueId(derived, taken);
|
|
479
|
+
if (id !== derived) collisions.push(derived);
|
|
480
|
+
const libraries = directives.documents.length > 0 ? directives.documents : document.documents;
|
|
446
481
|
const contents = `# ${section.heading}\n\n<!-- docspack: from ${document.origin} -->\n\n${body}\n`;
|
|
447
482
|
const file = `${CHUNKS_DIR}/${id}.md`;
|
|
448
483
|
|
|
@@ -463,11 +498,59 @@ function chunkDocuments(
|
|
|
463
498
|
...extractEntities(body),
|
|
464
499
|
]),
|
|
465
500
|
],
|
|
501
|
+
...(libraries === undefined || libraries.length === 0
|
|
502
|
+
? {}
|
|
503
|
+
: { documents: libraries.map(parseDocumentedLibrary) }),
|
|
466
504
|
});
|
|
467
505
|
}
|
|
468
506
|
}
|
|
469
507
|
|
|
470
|
-
return { chunks, files };
|
|
508
|
+
return { chunks, files, collisions };
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* The two ways a chunk id moves without anyone deciding it should. Both are reported as one
|
|
513
|
+
* line each, however many ids are involved: a large package collides in dozens of places, and a
|
|
514
|
+
* warning per id is a wall of text nobody reads.
|
|
515
|
+
*/
|
|
516
|
+
function idWarnings(collisions: readonly string[], gone: readonly string[]): string[] {
|
|
517
|
+
const warnings: string[] = [];
|
|
518
|
+
if (collisions.length > 0) {
|
|
519
|
+
warnings.push(
|
|
520
|
+
`${plural(collisions.length, "derived chunk id")} collided and ${collisions.length === 1 ? "was" : "were"} given a numeric suffix: ${list(collisions)} — which section keeps the bare id depends on the order the sources were read in`,
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
if (gone.length > 0) {
|
|
524
|
+
warnings.push(
|
|
525
|
+
`${plural(gone.length, "chunk id")} the previous build published ${gone.length === 1 ? "is" : "are"} gone: ${list(gone)} — recorded feedback and published links pinned to ${gone.length === 1 ? "it" : "them"} no longer resolve`,
|
|
526
|
+
);
|
|
527
|
+
}
|
|
528
|
+
return warnings;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
const plural = (count: number, noun: string): string => `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
532
|
+
|
|
533
|
+
const list = (ids: readonly string[]): string =>
|
|
534
|
+
`${ids.slice(0, 5).join(", ")}${ids.length > 5 ? `, and ${ids.length - 5} more` : ""}`;
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* Chunk ids the payload on disk carries that this build does not, or nothing when there is no
|
|
538
|
+
* previous payload to compare with. An unreadable manifest is not a finding: `build` is what
|
|
539
|
+
* replaces it.
|
|
540
|
+
*/
|
|
541
|
+
async function departedChunkIds(llmsDir: string, chunks: readonly ChunkSpec[]): Promise<string[]> {
|
|
542
|
+
let previous: PackageManifest;
|
|
543
|
+
try {
|
|
544
|
+
previous = parseManifest(
|
|
545
|
+
JSON.parse(await readFile(join(llmsDir, MANIFEST_FILE), "utf8")),
|
|
546
|
+
`${LLMS_DIR}/${MANIFEST_FILE}`,
|
|
547
|
+
);
|
|
548
|
+
} catch {
|
|
549
|
+
return [];
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
const current = new Set(chunks.map((chunk) => chunk.id));
|
|
553
|
+
return previous.chunks.map((chunk) => chunk.id).filter((id) => !current.has(id));
|
|
471
554
|
}
|
|
472
555
|
|
|
473
556
|
/** One piece of a document, before it becomes a chunk. */
|
|
@@ -661,26 +744,36 @@ function matchAll(text: string, pattern: RegExp): string[] {
|
|
|
661
744
|
}
|
|
662
745
|
|
|
663
746
|
/** One authored override under a heading: `<!-- docspack: tags=grid,columns -->`. */
|
|
664
|
-
const DIRECTIVE =
|
|
747
|
+
const DIRECTIVE =
|
|
748
|
+
/^[ \t]*<!--[ \t]*docspack:[ \t]*(tags|entities|documents)[ \t]*=([^>]*?)-->[ \t]*$/gim;
|
|
665
749
|
|
|
666
750
|
/**
|
|
667
751
|
* Reads per-section directives and removes them from the body. Front matter belongs to a whole
|
|
668
752
|
* document, so a page of eighty sections cannot aim any one of them; this is that lever.
|
|
669
753
|
*/
|
|
670
|
-
function readDirectives(body: string): {
|
|
671
|
-
|
|
672
|
-
|
|
754
|
+
function readDirectives(body: string): {
|
|
755
|
+
body: string;
|
|
756
|
+
tags: string[];
|
|
757
|
+
entities: string[];
|
|
758
|
+
documents: string[];
|
|
759
|
+
} {
|
|
760
|
+
const collected: Record<string, string[]> = { tags: [], entities: [], documents: [] };
|
|
673
761
|
|
|
674
762
|
const text = body.replace(DIRECTIVE, (_line, key: string, value: string) => {
|
|
675
763
|
const values = value
|
|
676
764
|
.split(",")
|
|
677
765
|
.map((item) => item.trim())
|
|
678
766
|
.filter((item) => item.length > 0);
|
|
679
|
-
|
|
767
|
+
collected[key.toLowerCase()]?.push(...values);
|
|
680
768
|
return "";
|
|
681
769
|
});
|
|
682
770
|
|
|
683
|
-
return {
|
|
771
|
+
return {
|
|
772
|
+
body: text.replace(/\n{3,}/g, "\n\n").trim(),
|
|
773
|
+
tags: collected.tags ?? [],
|
|
774
|
+
entities: collected.entities ?? [],
|
|
775
|
+
documents: collected.documents ?? [],
|
|
776
|
+
};
|
|
684
777
|
}
|
|
685
778
|
|
|
686
779
|
/**
|
package/src/changed.ts
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { identifiers } from "./coverage.js";
|
|
2
|
+
import type { Store } from "./db.js";
|
|
3
|
+
import { discoverLibraries, discoverPackages } from "./discovery.js";
|
|
4
|
+
import { DocspackError } from "./errors.js";
|
|
5
|
+
import { packageId } from "./spec.js";
|
|
6
|
+
|
|
7
|
+
export interface SurfaceChange {
|
|
8
|
+
readonly library: string;
|
|
9
|
+
readonly from: string;
|
|
10
|
+
readonly to: string;
|
|
11
|
+
/** Names the newer release exports that the older one did not. */
|
|
12
|
+
readonly added: readonly string[];
|
|
13
|
+
/** Names the older release exported that the newer one does not. */
|
|
14
|
+
readonly removed: readonly string[];
|
|
15
|
+
/**
|
|
16
|
+
* Added names that no documentation package installed here mentions. These are the ones a
|
|
17
|
+
* model cannot know: too new for its training data, and absent from what the vendor published.
|
|
18
|
+
*/
|
|
19
|
+
readonly undocumented: readonly string[];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ChangedOptions {
|
|
23
|
+
readonly cwd: string;
|
|
24
|
+
readonly store: Store;
|
|
25
|
+
/** Library name, optionally with the version to compare against: `hono` or `hono@4.0.0`. */
|
|
26
|
+
readonly library: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* What changed in a library's exported surface between two versions in the store.
|
|
31
|
+
*
|
|
32
|
+
* Both versions come from the local index, never the network: the store is global, so a version
|
|
33
|
+
* indexed for another project on this machine is already here. Measured across two libraries,
|
|
34
|
+
* upgrades are overwhelmingly additive — hono 3.12.12 to 4.13.3 removed three names and added
|
|
35
|
+
* 243 — so the useful report is what exists now that an older release did not have.
|
|
36
|
+
*/
|
|
37
|
+
export async function changedSurface(options: ChangedOptions): Promise<SurfaceChange> {
|
|
38
|
+
const [name, requested] = split(options.library);
|
|
39
|
+
|
|
40
|
+
const installed = (await discoverLibraries(options.cwd)).find((library) => library.name === name);
|
|
41
|
+
const indexed = options.store.versionsOf(name).filter((pkg) => pkg.kind === "artifact");
|
|
42
|
+
if (indexed.length === 0) {
|
|
43
|
+
throw new DocspackError(`No version of ${name} is indexed`, {
|
|
44
|
+
hint:
|
|
45
|
+
installed === undefined
|
|
46
|
+
? `${name} is not a dependency of this project.`
|
|
47
|
+
: "Run `docspack sync` to index the installed version, then ask again.",
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const to = installed?.version ?? indexed[indexed.length - 1]?.version;
|
|
52
|
+
if (to === undefined) throw new DocspackError(`No version of ${name} is indexed`);
|
|
53
|
+
|
|
54
|
+
const others = indexed.map((pkg) => pkg.version).filter((version) => version !== to);
|
|
55
|
+
const from = requested ?? others[others.length - 1];
|
|
56
|
+
if (from === undefined) {
|
|
57
|
+
throw new DocspackError(`Only ${name}@${to} is indexed, so there is nothing to compare`, {
|
|
58
|
+
hint: "Index another version by running `docspack sync` in a project that installs it. The store is shared across projects on this machine.",
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
if (!options.store.hasPackage(packageId(name, from))) {
|
|
62
|
+
throw new DocspackError(`${name}@${from} is not in the store`, {
|
|
63
|
+
hint: `Indexed versions: ${indexed.map((pkg) => pkg.version).join(", ")}`,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const before = new Set(options.store.symbolsOf(packageId(name, from)));
|
|
68
|
+
const after = new Set(options.store.symbolsOf(packageId(name, to)));
|
|
69
|
+
const added = [...after].filter((symbol) => !before.has(symbol)).sort();
|
|
70
|
+
const removed = [...before].filter((symbol) => !after.has(symbol)).sort();
|
|
71
|
+
|
|
72
|
+
const documented = await documentedNames(options);
|
|
73
|
+
return {
|
|
74
|
+
library: name,
|
|
75
|
+
from,
|
|
76
|
+
to,
|
|
77
|
+
added,
|
|
78
|
+
removed,
|
|
79
|
+
undocumented: added.filter((symbol) => !documented.has(symbol)),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Every identifier any documentation package installed here mentions. */
|
|
84
|
+
async function documentedNames(options: ChangedOptions): Promise<Set<string>> {
|
|
85
|
+
const names = new Set<string>();
|
|
86
|
+
for (const pkg of (await discoverPackages(options.cwd)).packages) {
|
|
87
|
+
for (const chunk of options.store.chunksOf(pkg.id)) {
|
|
88
|
+
for (const identifier of identifiers(chunk.content)) names.add(identifier);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return names;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** `hono@4.0.0` into its parts, keeping a scope's leading `@` out of it. */
|
|
95
|
+
function split(spec: string): [name: string, version: string | undefined] {
|
|
96
|
+
const at = spec.lastIndexOf("@");
|
|
97
|
+
if (at <= 0) return [spec, undefined];
|
|
98
|
+
return [spec.slice(0, at), spec.slice(at + 1)];
|
|
99
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -3,6 +3,9 @@ import { readFileSync } from "node:fs";
|
|
|
3
3
|
import { posix } from "node:path";
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { parseArgs } from "node:util";
|
|
6
|
+
import { applyAgentSetup, planAgentSetup } from "./agent.js";
|
|
7
|
+
import { changedSurface } from "./changed.js";
|
|
8
|
+
import { measureCoverage } from "./coverage.js";
|
|
6
9
|
import { defaultStorePath, Store, silenceSqliteWarning } from "./db.js";
|
|
7
10
|
import { discoverPackages } from "./discovery.js";
|
|
8
11
|
import { type DoctorReport, runDoctor } from "./doctor.js";
|
|
@@ -27,6 +30,11 @@ const OPTIONS = {
|
|
|
27
30
|
cwd: { type: "string" },
|
|
28
31
|
store: { type: "string" },
|
|
29
32
|
force: { type: "boolean" },
|
|
33
|
+
"no-artifacts": { type: "boolean" },
|
|
34
|
+
coverage: { type: "boolean" },
|
|
35
|
+
hooks: { type: "boolean" },
|
|
36
|
+
mcp: { type: "boolean" },
|
|
37
|
+
feedback: { type: "boolean" },
|
|
30
38
|
package: { type: "string", short: "p" },
|
|
31
39
|
limit: { type: "string" },
|
|
32
40
|
"max-tokens": { type: "string" },
|
|
@@ -141,6 +149,12 @@ function reportDoctor(report: DoctorReport, quiet: boolean): void {
|
|
|
141
149
|
}
|
|
142
150
|
|
|
143
151
|
/** 0 when the query was answered; otherwise which of the two empty answers this was. */
|
|
152
|
+
/** Names, capped. A hundred identifiers on one line is a wall, and `--json` has them all. */
|
|
153
|
+
function listed(names: readonly string[], limit = 20): string {
|
|
154
|
+
if (names.length <= limit) return names.join(", ");
|
|
155
|
+
return `${names.slice(0, limit).join(", ")} … and ${names.length - limit} more (--json for all)`;
|
|
156
|
+
}
|
|
157
|
+
|
|
144
158
|
function queryExit(result: QueryResult): number {
|
|
145
159
|
if (result.hits.length > 0) return 0;
|
|
146
160
|
return result.unindexed.length > 0 ? EXIT_NOT_INDEXED : EXIT_NO_MATCH;
|
|
@@ -201,6 +215,7 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
201
215
|
cwd,
|
|
202
216
|
store,
|
|
203
217
|
...(values.force === true ? { force: true } : {}),
|
|
218
|
+
...(values["no-artifacts"] === true ? { artifacts: false } : {}),
|
|
204
219
|
...(quiet || json
|
|
205
220
|
? {}
|
|
206
221
|
: {
|
|
@@ -218,8 +233,11 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
218
233
|
for (const pkg of result.packages) {
|
|
219
234
|
const mark = pkg.status === "indexed" ? green("+") : dim("=");
|
|
220
235
|
const trust = pkg.trusted ? "" : ` ${yellow("(community)")}`;
|
|
236
|
+
// Declarations read from an installed build are not documentation somebody wrote, and
|
|
237
|
+
// the listing says so rather than letting them pass for it.
|
|
238
|
+
const kind = pkg.kind === "artifact" ? ` ${dim("(declarations)")}` : "";
|
|
221
239
|
process.stdout.write(
|
|
222
|
-
`${mark} ${bold(pkg.id)} ${pkg.chunks} chunks ${dim(pkg.status)}${trust}\n`,
|
|
240
|
+
`${mark} ${bold(pkg.id)} ${pkg.chunks} chunks ${dim(pkg.status)}${trust}${kind}\n`,
|
|
223
241
|
);
|
|
224
242
|
}
|
|
225
243
|
for (const problem of result.problems) {
|
|
@@ -343,17 +361,38 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
343
361
|
const store = openStore(values);
|
|
344
362
|
try {
|
|
345
363
|
const { packages, problems } = await discoverPackages(cwd);
|
|
346
|
-
const
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
364
|
+
const wantCoverage = values.coverage === true || json;
|
|
365
|
+
const rows = [];
|
|
366
|
+
for (const pkg of packages) {
|
|
367
|
+
rows.push({
|
|
368
|
+
id: pkg.id,
|
|
369
|
+
name: pkg.name,
|
|
370
|
+
version: pkg.version,
|
|
371
|
+
chunks: pkg.manifest.chunks.length,
|
|
372
|
+
trusted: pkg.trusted,
|
|
373
|
+
indexed: store.hasPackage(pkg.id),
|
|
374
|
+
...(wantCoverage
|
|
375
|
+
? {
|
|
376
|
+
coverage: await measureCoverage({
|
|
377
|
+
packageDir: pkg.dir,
|
|
378
|
+
llmsDir: pkg.llmsDir,
|
|
379
|
+
manifest: pkg.manifest,
|
|
380
|
+
cwd,
|
|
381
|
+
}),
|
|
382
|
+
}
|
|
383
|
+
: {}),
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
const declarations = store
|
|
388
|
+
.listPackages()
|
|
389
|
+
.filter((pkg) => pkg.kind === "artifact")
|
|
390
|
+
.map((pkg) => ({ id: pkg.id, chunks: store.countChunks(pkg.id) }));
|
|
354
391
|
|
|
355
392
|
if (json) {
|
|
356
|
-
process.stdout.write(
|
|
393
|
+
process.stdout.write(
|
|
394
|
+
`${JSON.stringify({ packages: rows, declarations, problems }, null, 2)}\n`,
|
|
395
|
+
);
|
|
357
396
|
return 0;
|
|
358
397
|
}
|
|
359
398
|
if (rows.length === 0) {
|
|
@@ -363,6 +402,24 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
363
402
|
const state = row.indexed ? green("indexed") : yellow("not indexed");
|
|
364
403
|
const trust = row.trusted ? "" : ` ${yellow("(community)")}`;
|
|
365
404
|
process.stdout.write(`${bold(row.id)} ${row.chunks} chunks ${state}${trust}\n`);
|
|
405
|
+
for (const library of row.coverage?.libraries ?? []) {
|
|
406
|
+
const percent =
|
|
407
|
+
library.names === 0 ? 0 : Math.round((100 * library.documented) / library.names);
|
|
408
|
+
process.stdout.write(
|
|
409
|
+
` ${dim(`documents ${library.documented}/${library.names} (${percent}%) of ${library.library}'s exports`)}\n`,
|
|
410
|
+
);
|
|
411
|
+
if (library.uncovered.length > 0) {
|
|
412
|
+
process.stdout.write(` ${dim(`missing: ${library.uncovered.join(", ")}`)}\n`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (declarations.length > 0 && !quiet) {
|
|
417
|
+
const chunks = declarations.reduce((total, row) => total + row.chunks, 0);
|
|
418
|
+
process.stdout.write(
|
|
419
|
+
dim(
|
|
420
|
+
`\n${declarations.length} installed libraries indexed by declaration, ${chunks} names in all.\n`,
|
|
421
|
+
),
|
|
422
|
+
);
|
|
366
423
|
}
|
|
367
424
|
for (const problem of problems) process.stderr.write(`${yellow("!")} ${problem}\n`);
|
|
368
425
|
return 0;
|
|
@@ -371,6 +428,106 @@ async function main(argv: readonly string[]): Promise<number> {
|
|
|
371
428
|
}
|
|
372
429
|
}
|
|
373
430
|
|
|
431
|
+
case "agent": {
|
|
432
|
+
const [sub = "install"] = rest;
|
|
433
|
+
if (sub !== "install" && sub !== "check") {
|
|
434
|
+
throw new DocspackError(`Unknown subcommand "${sub}"`, {
|
|
435
|
+
hint: "Usage: docspack agent <install|check>",
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
const plan = await planAgentSetup({
|
|
440
|
+
cwd,
|
|
441
|
+
...(values.feedback === true ? { feedback: true } : {}),
|
|
442
|
+
...(values.hooks === true ? { hooks: true } : {}),
|
|
443
|
+
...(values.mcp === true ? { mcp: true } : {}),
|
|
444
|
+
});
|
|
445
|
+
const pending = plan.files.filter((file) => file.status !== "unchanged");
|
|
446
|
+
|
|
447
|
+
if (json) {
|
|
448
|
+
process.stdout.write(
|
|
449
|
+
`${JSON.stringify({ files: plan.files.map(({ contents, ...rest }) => rest) }, null, 2)}\n`,
|
|
450
|
+
);
|
|
451
|
+
return sub === "check" && pending.length > 0 ? 1 : 0;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// `check` is the CI half: it writes nothing and fails when the wiring is missing or stale,
|
|
455
|
+
// which is the only way a pasted instruction ever gets noticed after it drifts.
|
|
456
|
+
if (sub === "check") {
|
|
457
|
+
for (const file of plan.files) {
|
|
458
|
+
const mark = file.status === "unchanged" ? green("ok") : yellow(file.status);
|
|
459
|
+
process.stdout.write(`${mark} ${file.path}\n`);
|
|
460
|
+
}
|
|
461
|
+
if (pending.length > 0) {
|
|
462
|
+
process.stderr.write(
|
|
463
|
+
`\n${yellow("!")} ${plural(pending.length, "file")} out of date. Run \`docspack agent install\`.\n`,
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
return pending.length > 0 ? 1 : 0;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (values["dry-run"] === true) {
|
|
470
|
+
for (const file of plan.files) {
|
|
471
|
+
process.stdout.write(
|
|
472
|
+
`${file.status === "unchanged" ? dim("=") : green("+")} ${bold(file.path)} ${dim(file.reason)}\n`,
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
return 0;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
const written = await applyAgentSetup({ cwd }, plan);
|
|
479
|
+
for (const file of plan.files) {
|
|
480
|
+
const mark = file.status === "unchanged" ? dim("=") : green("+");
|
|
481
|
+
process.stdout.write(
|
|
482
|
+
`${mark} ${bold(file.path)} ${dim(file.status === "unchanged" ? "unchanged" : file.reason)}\n`,
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
if (written.length > 0 && !quiet) {
|
|
486
|
+
process.stdout.write(
|
|
487
|
+
`\n${dim("Commit these. Every agent working in this repository can now read the")}\n${dim("documentation of its dependencies, without anyone being told the command exists.")}\n`,
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
return 0;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
case "changed": {
|
|
494
|
+
const library = rest[0];
|
|
495
|
+
if (library === undefined) {
|
|
496
|
+
throw new DocspackError("Usage: docspack changed <library>[@version]");
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
const store = openStore(values);
|
|
500
|
+
try {
|
|
501
|
+
const change = await changedSurface({ cwd, store, library });
|
|
502
|
+
if (json) {
|
|
503
|
+
process.stdout.write(`${JSON.stringify(change, null, 2)}\n`);
|
|
504
|
+
return 0;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
process.stdout.write(
|
|
508
|
+
`${bold(`${change.library} ${change.from} → ${change.to}`)} ${dim(
|
|
509
|
+
`${change.added.length} added, ${change.removed.length} removed`,
|
|
510
|
+
)}\n`,
|
|
511
|
+
);
|
|
512
|
+
if (change.removed.length > 0) {
|
|
513
|
+
process.stdout.write(`\n${yellow("gone")} ${listed(change.removed)}\n`);
|
|
514
|
+
}
|
|
515
|
+
if (change.added.length > 0) {
|
|
516
|
+
process.stdout.write(`\n${green("new")} ${listed(change.added)}\n`);
|
|
517
|
+
}
|
|
518
|
+
if (change.undocumented.length > 0) {
|
|
519
|
+
process.stdout.write(
|
|
520
|
+
`\n${dim(
|
|
521
|
+
`${plural(change.undocumented.length, "new name")} that no documentation package here mentions:`,
|
|
522
|
+
)}\n${listed(change.undocumented)}\n`,
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
return 0;
|
|
526
|
+
} finally {
|
|
527
|
+
store.close();
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
374
531
|
case "verify": {
|
|
375
532
|
const report = await verifyProject({
|
|
376
533
|
cwd,
|