@bendyline/gezel-knowledge 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +46 -11
- package/dist/index.js +734 -84
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -209,6 +209,26 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
209
209
|
* builds must not set it: the constant IS the format's scale design.
|
|
210
210
|
*/
|
|
211
211
|
shardTargetChunks?: number;
|
|
212
|
+
/**
|
|
213
|
+
* How a multi-shard catalog's documents are assigned to shards.
|
|
214
|
+
*
|
|
215
|
+
* - `'topic'` (default; the gezk spec §8.1): greedy fill in
|
|
216
|
+
* `(topicPathKey, documentId)` order, so shards are slices of the table
|
|
217
|
+
* of contents.
|
|
218
|
+
* - `'semantic'`: embed every chunk first, then fill shards by balanced
|
|
219
|
+
* k-means over each document's mean chunk vector, so documents that
|
|
220
|
+
* answer the same queries share a shard and routing (§8.4) reaches them
|
|
221
|
+
* within its budget. Topic slices put an artist, their albums and their
|
|
222
|
+
* songs in different shards. Within a shard documents keep
|
|
223
|
+
* `(topicPathKey, documentId)` order. Every chunk is still embedded
|
|
224
|
+
* exactly once: vectors are staged in `workDir` between the passes.
|
|
225
|
+
*
|
|
226
|
+
* Measured on a 1.85M-chunk Wikipedia music catalog (155 queries,
|
|
227
|
+
* routed recall@8 vs an exact scan): 10 shards, S=6 82.6% → 85.7% and
|
|
228
|
+
* S=3 52.1% → 63.9%; recut to 37 shards, S=6 36.0% → 55.5%. The gain
|
|
229
|
+
* grows with the shard count.
|
|
230
|
+
*/
|
|
231
|
+
shardFill?: 'topic' | 'semantic';
|
|
212
232
|
}
|
|
213
233
|
interface CompileReport {
|
|
214
234
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -944,9 +964,19 @@ interface ProfileEmbedder {
|
|
|
944
964
|
countTokens(text: string): number;
|
|
945
965
|
dispose(): Promise<void>;
|
|
946
966
|
}
|
|
967
|
+
/**
|
|
968
|
+
* The `@huggingface/transformers` range this package supports, the same range
|
|
969
|
+
* as its optional peer dependency (a test holds the two together). Error
|
|
970
|
+
* messages quote it so the install command they print is exact.
|
|
971
|
+
*/
|
|
972
|
+
declare const TRANSFORMERS_PEER_RANGE = "^3.8.1";
|
|
947
973
|
declare class EmbedderUnavailableError extends Error {
|
|
948
974
|
readonly isActionable = true;
|
|
949
|
-
|
|
975
|
+
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
976
|
+
readonly runtimeMissing: boolean;
|
|
977
|
+
constructor(message: string, options?: {
|
|
978
|
+
runtimeMissing?: boolean;
|
|
979
|
+
});
|
|
950
980
|
}
|
|
951
981
|
/** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
|
|
952
982
|
interface TransformersModule {
|
|
@@ -1022,7 +1052,7 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
1022
1052
|
* files, never the disk, so an outline can be checked in a test without a
|
|
1023
1053
|
* tree on disk.
|
|
1024
1054
|
*/
|
|
1025
|
-
type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book';
|
|
1055
|
+
type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book' | 'docfx';
|
|
1026
1056
|
type OutlineFormat = Exclude<TableOfContentsFormat, 'folders' | 'hugo'>;
|
|
1027
1057
|
interface OutlineEntry {
|
|
1028
1058
|
/** Root-relative POSIX path of the Markdown file, extension included. */
|
|
@@ -1101,10 +1131,11 @@ declare function parseJupyterBookToc(doc: unknown, tocDir: string, files: readon
|
|
|
1101
1131
|
* gets the single root topic the compiler demands.
|
|
1102
1132
|
*
|
|
1103
1133
|
* A tree that already carries an outline keeps it: GitBook's `SUMMARY.md`,
|
|
1104
|
-
* an `mkdocs.yml` nav
|
|
1105
|
-
*
|
|
1106
|
-
*
|
|
1107
|
-
*
|
|
1134
|
+
* an `mkdocs.yml` nav, Jupyter Book's `_toc.yml` and docfx's per-folder
|
|
1135
|
+
* `toc.yml` files are read into the same topic tree (`outline.ts`,
|
|
1136
|
+
* `docfx.ts`), and Hugo's conventions — `_index.md` section pages,
|
|
1137
|
+
* `weight`, `draft` — are honored on top of the folders. A file an outline
|
|
1138
|
+
* omits stays in its folder, with a warning, so nothing is lost.
|
|
1108
1139
|
*
|
|
1109
1140
|
* Front matter carries what a documentation tree knows about itself: the
|
|
1110
1141
|
* title and summary, an explicit `id`, an `order` (the listing ordinal), a
|
|
@@ -1136,7 +1167,10 @@ interface TableOfContentsOptions {
|
|
|
1136
1167
|
/**
|
|
1137
1168
|
* The outline file for `gitbook` (`SUMMARY.md`), `mkdocs` (`mkdocs.yml`)
|
|
1138
1169
|
* and `jupyter-book` (`_toc.yml`), absolute or relative to the root. By
|
|
1139
|
-
* default the root is searched, and for mkdocs its parent as well.
|
|
1170
|
+
* default the root is searched, and for mkdocs its parent as well. For
|
|
1171
|
+
* `docfx` it is the entry TOC (a breadcrumb such as `bread/toc.yml`, or a
|
|
1172
|
+
* root `toc.yml`); by default docfx.json's `breadcrumb_path`, else the
|
|
1173
|
+
* root `toc.yml`.
|
|
1140
1174
|
*/
|
|
1141
1175
|
path?: string;
|
|
1142
1176
|
}
|
|
@@ -1162,7 +1196,7 @@ interface LoadMarkdownCatalogOptions {
|
|
|
1162
1196
|
publisherId: string;
|
|
1163
1197
|
catalogId: string;
|
|
1164
1198
|
};
|
|
1165
|
-
/** What to do with an image link whose file is missing (default `error`). */
|
|
1199
|
+
/** What to do with an image link whose file is missing (default `error`; `warn` for docfx). */
|
|
1166
1200
|
missingAssets?: 'error' | 'warn';
|
|
1167
1201
|
onWarning?: (message: string) => void;
|
|
1168
1202
|
}
|
|
@@ -1171,8 +1205,9 @@ declare function readMkdocsDocsDir(mkdocsPath: string): Promise<string>;
|
|
|
1171
1205
|
/**
|
|
1172
1206
|
* Which outline a tree carries: GitBook's `SUMMARY.md` or Jupyter Book's
|
|
1173
1207
|
* `_toc.yml` at the root, an `mkdocs.yml` at the root, the project folder
|
|
1174
|
-
* or the root's parent,
|
|
1175
|
-
* tree — else the
|
|
1208
|
+
* or the root's parent, a docfx project (`docfx.json` or a `toc.yml` at the
|
|
1209
|
+
* root), Hugo's `_index.md` section pages anywhere in the tree — else the
|
|
1210
|
+
* folders themselves.
|
|
1176
1211
|
*/
|
|
1177
1212
|
declare function detectTableOfContents(rootDir: string, projectDir?: string): Promise<TableOfContentsOptions>;
|
|
1178
1213
|
/**
|
|
@@ -1262,4 +1297,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1262
1297
|
|
|
1263
1298
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1264
1299
|
|
|
1265
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1300
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -181,7 +181,16 @@ function splitOversizeLine(line, opts) {
|
|
|
181
181
|
|
|
182
182
|
// src/compiler/compile.ts
|
|
183
183
|
import { createHash } from "crypto";
|
|
184
|
-
import {
|
|
184
|
+
import {
|
|
185
|
+
closeSync,
|
|
186
|
+
mkdirSync,
|
|
187
|
+
openSync,
|
|
188
|
+
readFileSync,
|
|
189
|
+
readSync,
|
|
190
|
+
rmSync,
|
|
191
|
+
statSync,
|
|
192
|
+
writeSync
|
|
193
|
+
} from "fs";
|
|
185
194
|
import { stat } from "fs/promises";
|
|
186
195
|
import { join } from "path";
|
|
187
196
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
@@ -310,10 +319,47 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
310
319
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
311
320
|
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
312
321
|
const embedded = totalChunks <= shardTarget;
|
|
322
|
+
const semanticFill = opts.shardFill === "semantic" && !embedded;
|
|
323
|
+
const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
|
|
324
|
+
p.doc.title,
|
|
325
|
+
chunk.headingPath,
|
|
326
|
+
opts.chunkingProfile.contextHeader.max,
|
|
327
|
+
opts.countTokens
|
|
328
|
+
)}${chunk.text}`;
|
|
329
|
+
rmSync(opts.workDir, { recursive: true, force: true });
|
|
330
|
+
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
331
|
+
const stagedPath = join(opts.workDir, "staged-vectors.f32");
|
|
332
|
+
let staged = null;
|
|
333
|
+
let docVectors = [];
|
|
334
|
+
if (semanticFill) {
|
|
335
|
+
docVectors = await stageVectors({
|
|
336
|
+
prepared,
|
|
337
|
+
stagedPath,
|
|
338
|
+
dimensions: profile.dimensions,
|
|
339
|
+
batchSize: opts.embedBatchSize ?? 32,
|
|
340
|
+
embed: opts.embed,
|
|
341
|
+
embedInput,
|
|
342
|
+
onProgress: (done) => opts.onProgress?.({ phase: "embed", done, total: totalChunks })
|
|
343
|
+
});
|
|
344
|
+
staged = new StagedVectorReader(stagedPath, profile.dimensions);
|
|
345
|
+
}
|
|
313
346
|
const shardOf = /* @__PURE__ */ new Map();
|
|
314
347
|
let shardCount = 1;
|
|
315
348
|
if (embedded) {
|
|
316
349
|
for (const p of prepared) shardOf.set(p.doc.id, 0);
|
|
350
|
+
} else if (semanticFill) {
|
|
351
|
+
shardCount = Math.ceil(totalChunks / shardTarget);
|
|
352
|
+
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
|
|
353
|
+
const assignment = semanticShards(
|
|
354
|
+
docVectors,
|
|
355
|
+
prepared.map((p) => p.chunks.length),
|
|
356
|
+
shardCount,
|
|
357
|
+
capacity,
|
|
358
|
+
seedFor(`${opts.catalog.id}#shard-fill`, 0)
|
|
359
|
+
);
|
|
360
|
+
prepared.forEach((p, i) => shardOf.set(p.doc.id, assignment[i]));
|
|
361
|
+
shardCount = new Set(assignment).size;
|
|
362
|
+
docVectors = [];
|
|
317
363
|
} else {
|
|
318
364
|
let current = 0;
|
|
319
365
|
let filled = 0;
|
|
@@ -327,8 +373,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
327
373
|
}
|
|
328
374
|
shardCount = current + 1;
|
|
329
375
|
}
|
|
330
|
-
rmSync(opts.workDir, { recursive: true, force: true });
|
|
331
|
-
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
332
376
|
const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
|
|
333
377
|
const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
|
|
334
378
|
const shardDbs = /* @__PURE__ */ new Map();
|
|
@@ -467,7 +511,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
467
511
|
let processedChunks = 0;
|
|
468
512
|
const flushEmbeds = async () => {
|
|
469
513
|
if (pendingTexts.length === 0) return;
|
|
470
|
-
const vectors = await opts.embed(pendingTexts);
|
|
514
|
+
const vectors = staged ? staged.next(pendingTexts.length) : await opts.embed(pendingTexts);
|
|
471
515
|
if (vectors.length !== pendingTexts.length) {
|
|
472
516
|
throw new Error(
|
|
473
517
|
`embedder returned ${vectors.length} vectors for ${pendingTexts.length} texts`
|
|
@@ -492,7 +536,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
492
536
|
if ((row.chunkId - 1) % stride === 0) samples.push(unit);
|
|
493
537
|
}
|
|
494
538
|
processedChunks += pendingTexts.length;
|
|
495
|
-
opts.onProgress?.({
|
|
539
|
+
opts.onProgress?.({
|
|
540
|
+
phase: staged ? "write" : "embed",
|
|
541
|
+
done: processedChunks,
|
|
542
|
+
total: totalChunks
|
|
543
|
+
});
|
|
496
544
|
pendingTexts = [];
|
|
497
545
|
pendingRows = [];
|
|
498
546
|
};
|
|
@@ -530,18 +578,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
530
578
|
const chunkId = Number(row.id);
|
|
531
579
|
insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, chunk.text);
|
|
532
580
|
shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
|
|
533
|
-
|
|
534
|
-
p.doc.title,
|
|
535
|
-
chunk.headingPath,
|
|
536
|
-
opts.chunkingProfile.contextHeader.max,
|
|
537
|
-
opts.countTokens
|
|
538
|
-
);
|
|
539
|
-
pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
|
|
581
|
+
pendingTexts.push(embedInput(p, chunk));
|
|
540
582
|
pendingRows.push({ shardId, chunkId });
|
|
541
583
|
if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
|
|
542
584
|
}
|
|
543
585
|
}
|
|
544
586
|
await flushEmbeds();
|
|
587
|
+
staged?.close();
|
|
588
|
+
rmSync(stagedPath, { force: true });
|
|
545
589
|
for (let id = 0; id < shardCount; id++) {
|
|
546
590
|
if (!embedded)
|
|
547
591
|
putMeta(shardDb(id), {
|
|
@@ -913,6 +957,158 @@ ${path.join(" > ")}
|
|
|
913
957
|
}
|
|
914
958
|
return header;
|
|
915
959
|
}
|
|
960
|
+
async function stageVectors(args) {
|
|
961
|
+
const { prepared, dimensions } = args;
|
|
962
|
+
const docVectors = prepared.map(() => new Float32Array(dimensions));
|
|
963
|
+
const fd = openSync(args.stagedPath, "w");
|
|
964
|
+
let texts = [];
|
|
965
|
+
let owners = [];
|
|
966
|
+
let done = 0;
|
|
967
|
+
const flush = async () => {
|
|
968
|
+
if (texts.length === 0) return;
|
|
969
|
+
const vectors = await args.embed(texts);
|
|
970
|
+
if (vectors.length !== texts.length) {
|
|
971
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${texts.length} texts`);
|
|
972
|
+
}
|
|
973
|
+
const block = new Float32Array(texts.length * dimensions);
|
|
974
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
975
|
+
const unit = l2Normalize(vectors[i]);
|
|
976
|
+
if (unit.length !== dimensions) {
|
|
977
|
+
throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
|
|
978
|
+
}
|
|
979
|
+
block.set(unit, i * dimensions);
|
|
980
|
+
const sum = docVectors[owners[i]];
|
|
981
|
+
for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
|
|
982
|
+
}
|
|
983
|
+
writeSync(fd, new Uint8Array(block.buffer));
|
|
984
|
+
done += texts.length;
|
|
985
|
+
args.onProgress(done);
|
|
986
|
+
texts = [];
|
|
987
|
+
owners = [];
|
|
988
|
+
};
|
|
989
|
+
try {
|
|
990
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
991
|
+
const p = prepared[i];
|
|
992
|
+
for (const chunk of p.chunks) {
|
|
993
|
+
texts.push(args.embedInput(p, chunk));
|
|
994
|
+
owners.push(i);
|
|
995
|
+
if (texts.length >= args.batchSize) await flush();
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
await flush();
|
|
999
|
+
} finally {
|
|
1000
|
+
closeSync(fd);
|
|
1001
|
+
}
|
|
1002
|
+
return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
|
|
1003
|
+
}
|
|
1004
|
+
var StagedVectorReader = class {
|
|
1005
|
+
constructor(path, dimensions) {
|
|
1006
|
+
this.dimensions = dimensions;
|
|
1007
|
+
this.fd = openSync(path, "r");
|
|
1008
|
+
}
|
|
1009
|
+
dimensions;
|
|
1010
|
+
fd;
|
|
1011
|
+
position = 0;
|
|
1012
|
+
next(count) {
|
|
1013
|
+
const bytes = count * this.dimensions * 4;
|
|
1014
|
+
const buffer = Buffer.alloc(bytes);
|
|
1015
|
+
let read = 0;
|
|
1016
|
+
while (read < bytes) {
|
|
1017
|
+
const n = readSync(this.fd, buffer, read, bytes - read, this.position + read);
|
|
1018
|
+
if (n === 0)
|
|
1019
|
+
throw new Error("staged vectors ended early: the write pass and pass 1 disagree");
|
|
1020
|
+
read += n;
|
|
1021
|
+
}
|
|
1022
|
+
this.position += bytes;
|
|
1023
|
+
const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * this.dimensions);
|
|
1024
|
+
const out = [];
|
|
1025
|
+
for (let i = 0; i < count; i++) {
|
|
1026
|
+
out.push(Array.from(floats.subarray(i * this.dimensions, (i + 1) * this.dimensions)));
|
|
1027
|
+
}
|
|
1028
|
+
return out;
|
|
1029
|
+
}
|
|
1030
|
+
close() {
|
|
1031
|
+
closeSync(this.fd);
|
|
1032
|
+
}
|
|
1033
|
+
};
|
|
1034
|
+
var SEMANTIC_FILL_ROUNDS = 6;
|
|
1035
|
+
function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
1036
|
+
const n = docVectors.length;
|
|
1037
|
+
const dim = docVectors[0].length;
|
|
1038
|
+
const dot = (a, b) => {
|
|
1039
|
+
let sum = 0;
|
|
1040
|
+
for (let d = 0; d < dim; d++) sum += a[d] * b[d];
|
|
1041
|
+
return sum;
|
|
1042
|
+
};
|
|
1043
|
+
const stride = Math.max(1, Math.ceil(n / 2e4));
|
|
1044
|
+
const sample = docVectors.filter((_, i) => i % stride === 0);
|
|
1045
|
+
const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
|
|
1046
|
+
for (let i = 0; centers.length < shardCount; i++) {
|
|
1047
|
+
centers.push(
|
|
1048
|
+
Float32Array.from(docVectors[Math.floor(i * n / shardCount) % n])
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
let assignment = new Array(n).fill(-1);
|
|
1052
|
+
for (let round = 0; round < SEMANTIC_FILL_ROUNDS; round++) {
|
|
1053
|
+
const best = new Int32Array(n);
|
|
1054
|
+
const margin = new Float64Array(n);
|
|
1055
|
+
for (let i = 0; i < n; i++) {
|
|
1056
|
+
let first = Number.NEGATIVE_INFINITY;
|
|
1057
|
+
let second = Number.NEGATIVE_INFINITY;
|
|
1058
|
+
let arg = 0;
|
|
1059
|
+
for (let c = 0; c < shardCount; c++) {
|
|
1060
|
+
const s = dot(docVectors[i], centers[c]);
|
|
1061
|
+
if (s > first) {
|
|
1062
|
+
second = first;
|
|
1063
|
+
first = s;
|
|
1064
|
+
arg = c;
|
|
1065
|
+
} else if (s > second) {
|
|
1066
|
+
second = s;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
best[i] = arg;
|
|
1070
|
+
margin[i] = first - (Number.isFinite(second) ? second : first);
|
|
1071
|
+
}
|
|
1072
|
+
const order = Array.from({ length: n }, (_, i) => i).sort(
|
|
1073
|
+
(a, b) => margin[b] - margin[a] || a - b
|
|
1074
|
+
);
|
|
1075
|
+
const load = new Array(shardCount).fill(0);
|
|
1076
|
+
const next = new Array(n).fill(-1);
|
|
1077
|
+
for (const i of order) {
|
|
1078
|
+
const size = docChunks[i];
|
|
1079
|
+
let shard = best[i];
|
|
1080
|
+
if (load[shard] + size > capacity) {
|
|
1081
|
+
const ranked = centers.map((c, id) => ({ id, s: dot(docVectors[i], c) })).sort((a, b) => b.s - a.s || a.id - b.id);
|
|
1082
|
+
const fit = ranked.find((r) => load[r.id] + size <= capacity);
|
|
1083
|
+
shard = fit ? fit.id : load.reduce((min, l, id) => l < load[min] ? id : min, 0);
|
|
1084
|
+
}
|
|
1085
|
+
next[i] = shard;
|
|
1086
|
+
load[shard] = load[shard] + size;
|
|
1087
|
+
}
|
|
1088
|
+
const moved = next.some((s, i) => s !== assignment[i]);
|
|
1089
|
+
assignment = next;
|
|
1090
|
+
if (!moved) break;
|
|
1091
|
+
const sums = centers.map(() => new Float64Array(dim));
|
|
1092
|
+
for (let i = 0; i < n; i++) {
|
|
1093
|
+
const sum = sums[assignment[i]];
|
|
1094
|
+
const v = docVectors[i];
|
|
1095
|
+
const w = docChunks[i];
|
|
1096
|
+
for (let d = 0; d < dim; d++) sum[d] = sum[d] + v[d] * w;
|
|
1097
|
+
}
|
|
1098
|
+
for (let c = 0; c < shardCount; c++) {
|
|
1099
|
+
const sum = sums[c];
|
|
1100
|
+
let norm = 0;
|
|
1101
|
+
for (let d = 0; d < dim; d++) norm += sum[d] ** 2;
|
|
1102
|
+
if (norm === 0) continue;
|
|
1103
|
+
norm = Math.sqrt(norm);
|
|
1104
|
+
const center = centers[c];
|
|
1105
|
+
for (let d = 0; d < dim; d++) center[d] = sum[d] / norm;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const renumber = /* @__PURE__ */ new Map();
|
|
1109
|
+
for (const s of assignment) if (!renumber.has(s)) renumber.set(s, renumber.size);
|
|
1110
|
+
return assignment.map((s) => renumber.get(s));
|
|
1111
|
+
}
|
|
916
1112
|
function seedFor(catalogId, shardId) {
|
|
917
1113
|
const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
918
1114
|
return digest.readBigUInt64LE(24);
|
|
@@ -3472,11 +3668,15 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
3472
3668
|
// src/embedding/profile-embedder.ts
|
|
3473
3669
|
var MAX_BATCH = 8;
|
|
3474
3670
|
var MAX_CHARS = 8e3;
|
|
3671
|
+
var TRANSFORMERS_PEER_RANGE = "^3.8.1";
|
|
3475
3672
|
var EmbedderUnavailableError = class extends Error {
|
|
3476
3673
|
isActionable = true;
|
|
3477
|
-
|
|
3674
|
+
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
3675
|
+
runtimeMissing;
|
|
3676
|
+
constructor(message, options = {}) {
|
|
3478
3677
|
super(message);
|
|
3479
3678
|
this.name = "EmbedderUnavailableError";
|
|
3679
|
+
this.runtimeMissing = options.runtimeMissing ?? false;
|
|
3480
3680
|
}
|
|
3481
3681
|
};
|
|
3482
3682
|
function aggregateDownloadProgress(onProgress) {
|
|
@@ -3545,7 +3745,8 @@ async function loadTransformers() {
|
|
|
3545
3745
|
const message = err instanceof Error ? err.message : String(err);
|
|
3546
3746
|
if (/Cannot find (module|package)/i.test(message)) {
|
|
3547
3747
|
throw new EmbedderUnavailableError(
|
|
3548
|
-
|
|
3748
|
+
`the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer @huggingface/transformers@${TRANSFORMERS_PEER_RANGE}; install it alongside @bendyline/gezel-knowledge`,
|
|
3749
|
+
{ runtimeMissing: true }
|
|
3549
3750
|
);
|
|
3550
3751
|
}
|
|
3551
3752
|
throw err;
|
|
@@ -3616,52 +3817,19 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3616
3817
|
// src/markdown-adapter/load.ts
|
|
3617
3818
|
import { existsSync } from "fs";
|
|
3618
3819
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
3619
|
-
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as
|
|
3820
|
+
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
|
|
3620
3821
|
import {
|
|
3621
3822
|
CatalogDocumentSchema,
|
|
3622
3823
|
KnowledgeDocumentIdSchema,
|
|
3623
3824
|
KnowledgeOrdinalSchema,
|
|
3825
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
|
|
3624
3826
|
assetExtension as assetExtension3,
|
|
3625
3827
|
formatKnowledgeUri,
|
|
3626
3828
|
topicSortKeyForOrder
|
|
3627
3829
|
} from "@bendyline/gezk";
|
|
3628
3830
|
|
|
3629
|
-
// src/markdown-adapter/
|
|
3630
|
-
import {
|
|
3631
|
-
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
3632
|
-
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
3633
|
-
function parseYaml(text, opts) {
|
|
3634
|
-
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
3635
|
-
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
3636
|
-
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
3637
|
-
}
|
|
3638
|
-
const doc = parseDocument(text, {
|
|
3639
|
-
version: "1.2",
|
|
3640
|
-
schema: "core",
|
|
3641
|
-
uniqueKeys: true,
|
|
3642
|
-
logLevel: "silent"
|
|
3643
|
-
});
|
|
3644
|
-
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
3645
|
-
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
3646
|
-
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
3647
|
-
}
|
|
3648
|
-
var OPEN = /^---[ \t]*\r?\n/;
|
|
3649
|
-
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
3650
|
-
function parseMarkdownFrontMatter(source) {
|
|
3651
|
-
const open = OPEN.exec(source);
|
|
3652
|
-
if (!open) return { data: {}, body: source };
|
|
3653
|
-
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
3654
|
-
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
3655
|
-
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
3656
|
-
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
3657
|
-
const body = lines.slice(closeAt + 1).join("\n");
|
|
3658
|
-
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
3659
|
-
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
3660
|
-
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3661
|
-
throw new Error("front matter must be a mapping");
|
|
3662
|
-
}
|
|
3663
|
-
return { data: parsed, body };
|
|
3664
|
-
}
|
|
3831
|
+
// src/markdown-adapter/docfx.ts
|
|
3832
|
+
import { posix as posix2 } from "path";
|
|
3665
3833
|
|
|
3666
3834
|
// src/markdown-adapter/outline.ts
|
|
3667
3835
|
import { posix } from "path";
|
|
@@ -3958,14 +4126,400 @@ function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
|
|
|
3958
4126
|
return outline;
|
|
3959
4127
|
}
|
|
3960
4128
|
|
|
4129
|
+
// src/markdown-adapter/docfx.ts
|
|
4130
|
+
var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
|
|
4131
|
+
var MARKDOWN_EXT2 = /\.(md|markdown)$/i;
|
|
4132
|
+
var TOC_FILE = /^toc\.(ya?ml|json|md)$/i;
|
|
4133
|
+
var LANDING_PAGE = /(^|\/)index\.(ya?ml|md)$/i;
|
|
4134
|
+
var WARNING_EXAMPLES = 5;
|
|
4135
|
+
function isRecord2(value) {
|
|
4136
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4137
|
+
}
|
|
4138
|
+
function normalizeFolder(value) {
|
|
4139
|
+
if (typeof value !== "string") return "";
|
|
4140
|
+
const cleaned = posix2.normalize(value.replace(/\\/g, "/")).replace(/^\.?\/+|\/+$/g, "");
|
|
4141
|
+
return cleaned === "." ? "" : cleaned;
|
|
4142
|
+
}
|
|
4143
|
+
function dirOf(file) {
|
|
4144
|
+
const dir = posix2.dirname(file);
|
|
4145
|
+
return dir === "." ? "" : dir;
|
|
4146
|
+
}
|
|
4147
|
+
function parentOf(dir) {
|
|
4148
|
+
return dir === "" ? null : dirOf(dir);
|
|
4149
|
+
}
|
|
4150
|
+
function isUnder(file, dir) {
|
|
4151
|
+
return dir === "" || file.startsWith(`${dir}/`);
|
|
4152
|
+
}
|
|
4153
|
+
function escapeRegExp(text) {
|
|
4154
|
+
return text.replace(/[.+^$()|[\]\\]/g, "\\$&");
|
|
4155
|
+
}
|
|
4156
|
+
function docfxGlob(pattern) {
|
|
4157
|
+
const p = pattern.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
4158
|
+
let source = "";
|
|
4159
|
+
for (let i = 0; i < p.length; i++) {
|
|
4160
|
+
const ch = p[i];
|
|
4161
|
+
if (ch === "*") {
|
|
4162
|
+
if (p[i + 1] === "*") {
|
|
4163
|
+
if (p[i + 2] === "/") {
|
|
4164
|
+
source += "(?:.*/)?";
|
|
4165
|
+
i += 2;
|
|
4166
|
+
} else {
|
|
4167
|
+
source += ".*";
|
|
4168
|
+
i += 1;
|
|
4169
|
+
}
|
|
4170
|
+
} else source += "[^/]*";
|
|
4171
|
+
} else if (ch === "?") {
|
|
4172
|
+
source += "[^/]";
|
|
4173
|
+
} else if (ch === "{" && p.indexOf("}", i) > i) {
|
|
4174
|
+
const close = p.indexOf("}", i);
|
|
4175
|
+
source += `(?:${p.slice(i + 1, close).split(",").map(escapeRegExp).join("|")})`;
|
|
4176
|
+
i = close;
|
|
4177
|
+
} else {
|
|
4178
|
+
source += escapeRegExp(ch);
|
|
4179
|
+
}
|
|
4180
|
+
}
|
|
4181
|
+
return new RegExp(`^${source}$`, "i");
|
|
4182
|
+
}
|
|
4183
|
+
function globList(value) {
|
|
4184
|
+
const list = typeof value === "string" ? [value] : Array.isArray(value) ? value : [];
|
|
4185
|
+
return list.filter((g) => typeof g === "string" && g.trim() !== "").map(docfxGlob);
|
|
4186
|
+
}
|
|
4187
|
+
function parseDocfxJson(config, configFile = "docfx.json") {
|
|
4188
|
+
if (!isRecord2(config)) throw new Error(`${configFile}: not a JSON object`);
|
|
4189
|
+
const build = isRecord2(config.build) ? config.build : {};
|
|
4190
|
+
const content = [];
|
|
4191
|
+
for (const group of Array.isArray(build.content) ? build.content : []) {
|
|
4192
|
+
if (!isRecord2(group)) continue;
|
|
4193
|
+
const include = globList(group.files);
|
|
4194
|
+
if (include.length === 0) continue;
|
|
4195
|
+
content.push({
|
|
4196
|
+
src: normalizeFolder(group.src),
|
|
4197
|
+
dest: normalizeFolder(group.dest),
|
|
4198
|
+
include,
|
|
4199
|
+
exclude: globList(group.exclude)
|
|
4200
|
+
});
|
|
4201
|
+
}
|
|
4202
|
+
const meta = isRecord2(build.globalMetadata) ? build.globalMetadata : {};
|
|
4203
|
+
const breadcrumb = meta.breadcrumb_path;
|
|
4204
|
+
return {
|
|
4205
|
+
content,
|
|
4206
|
+
...typeof breadcrumb === "string" && breadcrumb.trim() ? { breadcrumbPath: breadcrumb.trim() } : {}
|
|
4207
|
+
};
|
|
4208
|
+
}
|
|
4209
|
+
function docfxPublishes(project, file) {
|
|
4210
|
+
if (project.content.length === 0) return true;
|
|
4211
|
+
return project.content.some((group) => {
|
|
4212
|
+
if (!isUnder(file, group.src)) return false;
|
|
4213
|
+
const rel = group.src === "" ? file : file.slice(group.src.length + 1);
|
|
4214
|
+
return group.include.some((r) => r.test(rel)) && !group.exclude.some((r) => r.test(rel));
|
|
4215
|
+
});
|
|
4216
|
+
}
|
|
4217
|
+
function tocItems(doc) {
|
|
4218
|
+
if (Array.isArray(doc)) return doc;
|
|
4219
|
+
if (isRecord2(doc) && Array.isArray(doc.items)) return doc.items;
|
|
4220
|
+
return [];
|
|
4221
|
+
}
|
|
4222
|
+
function isBreadcrumbToc(doc) {
|
|
4223
|
+
let tocHref = false;
|
|
4224
|
+
const visit = (items) => {
|
|
4225
|
+
for (const item of items) {
|
|
4226
|
+
if (!isRecord2(item)) continue;
|
|
4227
|
+
if (typeof item.href === "string") return false;
|
|
4228
|
+
if (typeof item.tocHref === "string") tocHref = true;
|
|
4229
|
+
if (Array.isArray(item.items) && !visit(item.items)) return false;
|
|
4230
|
+
}
|
|
4231
|
+
return true;
|
|
4232
|
+
};
|
|
4233
|
+
return visit(tocItems(doc)) && tocHref;
|
|
4234
|
+
}
|
|
4235
|
+
function basePrefixes(url) {
|
|
4236
|
+
const segments = url.split("/").filter(Boolean);
|
|
4237
|
+
const out = ["/"];
|
|
4238
|
+
for (let i = 1; i < segments.length; i++) out.push(`/${segments.slice(0, i).join("/")}/`);
|
|
4239
|
+
return out;
|
|
4240
|
+
}
|
|
4241
|
+
function parseDocfxToc(input) {
|
|
4242
|
+
const outline = {
|
|
4243
|
+
format: "docfx",
|
|
4244
|
+
root: newOutlineTopic(""),
|
|
4245
|
+
consumed: input.files.filter((file) => TOC_FILE.test(posix2.basename(file))),
|
|
4246
|
+
warnings: []
|
|
4247
|
+
};
|
|
4248
|
+
const known = new Set(input.files.map((file) => file.normalize("NFC")));
|
|
4249
|
+
const groups = input.project?.content.length ? [...input.project.content].sort((a, b) => b.dest.length - a.dest.length) : [{ src: "", dest: "", include: [], exclude: [] }];
|
|
4250
|
+
const tocByDir = /* @__PURE__ */ new Map();
|
|
4251
|
+
for (const file of [...input.tocs.keys()].sort()) {
|
|
4252
|
+
if (/\.ya?ml$/i.test(file) && !tocByDir.has(dirOf(file))) tocByDir.set(dirOf(file), file);
|
|
4253
|
+
}
|
|
4254
|
+
const sourcesOf = (sitePath) => {
|
|
4255
|
+
const out = [];
|
|
4256
|
+
for (const group of groups) {
|
|
4257
|
+
if (group.dest !== "" && sitePath !== group.dest && !sitePath.startsWith(`${group.dest}/`))
|
|
4258
|
+
continue;
|
|
4259
|
+
const rest = sitePath.slice(group.dest.length).replace(/^\/+/, "");
|
|
4260
|
+
out.push(group.src && rest ? `${group.src}/${rest}` : group.src || rest);
|
|
4261
|
+
}
|
|
4262
|
+
return out;
|
|
4263
|
+
};
|
|
4264
|
+
const tocAt = (path) => {
|
|
4265
|
+
const trimmed = path.replace(/\/+$/, "");
|
|
4266
|
+
return TOC_FILE.test(posix2.basename(trimmed)) ? tocByDir.get(dirOf(trimmed)) : tocByDir.get(trimmed);
|
|
4267
|
+
};
|
|
4268
|
+
const pageAt = (path) => {
|
|
4269
|
+
const candidates = path === "" || path.endsWith("/") ? [`${path}index.md`] : MARKDOWN_EXT2.test(path) ? [path] : [`${path}.md`, `${path}/index.md`];
|
|
4270
|
+
return candidates.find((candidate) => known.has(candidate.normalize("NFC")));
|
|
4271
|
+
};
|
|
4272
|
+
const resolveSite = (url, base2) => {
|
|
4273
|
+
const path = url.replace(/[?#].*$/, "");
|
|
4274
|
+
let below;
|
|
4275
|
+
if (`${path}/` === base2) below = "";
|
|
4276
|
+
else if (path.startsWith(base2)) below = path.slice(base2.length);
|
|
4277
|
+
else return null;
|
|
4278
|
+
for (const source of sourcesOf(below)) {
|
|
4279
|
+
const page = pageAt(source);
|
|
4280
|
+
if (page) return { kind: "page", file: page };
|
|
4281
|
+
const toc = tocAt(source);
|
|
4282
|
+
if (toc) return { kind: "toc", file: toc };
|
|
4283
|
+
}
|
|
4284
|
+
return null;
|
|
4285
|
+
};
|
|
4286
|
+
let entry = input.entry;
|
|
4287
|
+
let base;
|
|
4288
|
+
const breadcrumb = input.project?.breadcrumbPath;
|
|
4289
|
+
if (!entry && breadcrumb) {
|
|
4290
|
+
for (const candidate of basePrefixes(breadcrumb)) {
|
|
4291
|
+
const target = resolveSite(breadcrumb, candidate);
|
|
4292
|
+
if (target?.kind === "toc") {
|
|
4293
|
+
entry = target.file;
|
|
4294
|
+
base = candidate;
|
|
4295
|
+
break;
|
|
4296
|
+
}
|
|
4297
|
+
}
|
|
4298
|
+
if (!entry) {
|
|
4299
|
+
outline.warnings.push(
|
|
4300
|
+
`docfx.json: breadcrumb_path '${breadcrumb}' names no toc.yml in the tree; ignored`
|
|
4301
|
+
);
|
|
4302
|
+
}
|
|
4303
|
+
}
|
|
4304
|
+
entry ??= tocByDir.get("");
|
|
4305
|
+
if (entry && base === void 0) {
|
|
4306
|
+
const urls = [];
|
|
4307
|
+
const collect = (items) => {
|
|
4308
|
+
for (const item of items) {
|
|
4309
|
+
if (!isRecord2(item)) continue;
|
|
4310
|
+
for (const key of ["tocHref", "topicHref", "href"]) {
|
|
4311
|
+
const value = item[key];
|
|
4312
|
+
if (typeof value === "string" && value.startsWith("/") && !value.startsWith("//")) {
|
|
4313
|
+
urls.push(value);
|
|
4314
|
+
}
|
|
4315
|
+
}
|
|
4316
|
+
if (Array.isArray(item.items)) collect(item.items);
|
|
4317
|
+
}
|
|
4318
|
+
};
|
|
4319
|
+
collect(tocItems(input.tocs.get(entry)));
|
|
4320
|
+
let best = { base: "/", hits: 0 };
|
|
4321
|
+
for (const candidate of [...new Set(urls.flatMap(basePrefixes))].sort(
|
|
4322
|
+
(a, b) => a.length - b.length
|
|
4323
|
+
)) {
|
|
4324
|
+
const hits = urls.filter((url) => resolveSite(url, candidate) !== null).length;
|
|
4325
|
+
if (hits > best.hits) best = { base: candidate, hits };
|
|
4326
|
+
}
|
|
4327
|
+
base = best.base;
|
|
4328
|
+
}
|
|
4329
|
+
const siteBase = base ?? "/";
|
|
4330
|
+
const classify = (href, tocDir) => {
|
|
4331
|
+
const value = href.trim();
|
|
4332
|
+
if (value === "" || value.startsWith("#") || HAS_SCHEME2.test(value) || value.startsWith("//"))
|
|
4333
|
+
return null;
|
|
4334
|
+
if (value.startsWith("/")) return resolveSite(value, siteBase);
|
|
4335
|
+
const path = resolveOutlinePath(tocDir, value);
|
|
4336
|
+
if (path === null) return null;
|
|
4337
|
+
if (MARKDOWN_EXT2.test(path)) return { kind: "page", file: path };
|
|
4338
|
+
const toc = tocAt(path);
|
|
4339
|
+
return toc ? { kind: "toc", file: toc } : null;
|
|
4340
|
+
};
|
|
4341
|
+
const crossLinks = /* @__PURE__ */ new WeakSet();
|
|
4342
|
+
const included = /* @__PURE__ */ new Set();
|
|
4343
|
+
const homes = /* @__PURE__ */ new Map();
|
|
4344
|
+
const pageEntry = (file, tocDir) => {
|
|
4345
|
+
const entry2 = { file };
|
|
4346
|
+
if (!isUnder(file, tocDir)) crossLinks.add(entry2);
|
|
4347
|
+
return entry2;
|
|
4348
|
+
};
|
|
4349
|
+
const walkItems = (items, tocFile, container, followTocHref) => {
|
|
4350
|
+
const tocDir = dirOf(tocFile);
|
|
4351
|
+
for (const item of items) {
|
|
4352
|
+
if (!isRecord2(item)) continue;
|
|
4353
|
+
const name = typeof item.name === "string" ? item.name.trim() : "";
|
|
4354
|
+
const href = typeof item.href === "string" ? classify(item.href, tocDir) : null;
|
|
4355
|
+
const topicHref = typeof item.topicHref === "string" ? classify(item.topicHref, tocDir) : null;
|
|
4356
|
+
const tocHref = followTocHref && typeof item.tocHref === "string" ? classify(item.tocHref, tocDir) : null;
|
|
4357
|
+
const page = [href, topicHref].find((target) => target?.kind === "page");
|
|
4358
|
+
const nested = [href, tocHref].find(
|
|
4359
|
+
(target) => target?.kind === "toc" && !included.has(target.file)
|
|
4360
|
+
);
|
|
4361
|
+
const children = Array.isArray(item.items) && item.items.length > 0 ? item.items : null;
|
|
4362
|
+
if (!nested && !children) {
|
|
4363
|
+
if (page) container.entries.push(pageEntry(page.file, tocDir));
|
|
4364
|
+
continue;
|
|
4365
|
+
}
|
|
4366
|
+
const topic = newOutlineTopic(
|
|
4367
|
+
name || (nested ? titleFromFolderName(posix2.basename(dirOf(nested.file))) : "Untitled")
|
|
4368
|
+
);
|
|
4369
|
+
if (!name && page) topic.titleFromFile = page.file;
|
|
4370
|
+
if (page) topic.entries.push(pageEntry(page.file, tocDir));
|
|
4371
|
+
if (nested) includeToc(nested.file, topic, false);
|
|
4372
|
+
if (children) walkItems(children, tocFile, topic, followTocHref);
|
|
4373
|
+
container.children.push(topic);
|
|
4374
|
+
}
|
|
4375
|
+
};
|
|
4376
|
+
const includeToc = (file, container, followTocHref) => {
|
|
4377
|
+
if (included.has(file)) return;
|
|
4378
|
+
included.add(file);
|
|
4379
|
+
const dir = dirOf(file);
|
|
4380
|
+
if (container !== outline.root && !homes.has(dir)) homes.set(dir, container);
|
|
4381
|
+
walkItems(tocItems(input.tocs.get(file)), file, container, followTocHref);
|
|
4382
|
+
};
|
|
4383
|
+
const homeOf = (dir) => {
|
|
4384
|
+
for (let d = dir; d !== null; d = parentOf(d)) {
|
|
4385
|
+
const home = homes.get(d);
|
|
4386
|
+
if (home) return home;
|
|
4387
|
+
}
|
|
4388
|
+
return void 0;
|
|
4389
|
+
};
|
|
4390
|
+
if (entry) {
|
|
4391
|
+
outline.entry = entry;
|
|
4392
|
+
includeToc(entry, outline.root, true);
|
|
4393
|
+
}
|
|
4394
|
+
const lone = outline.root.entries.length === 0 && outline.root.children.length === 1 ? outline.root.children[0] : void 0;
|
|
4395
|
+
const top = lone ?? outline.root;
|
|
4396
|
+
const orphans = [...input.tocs.keys()].filter((file) => !included.has(file) && /\.ya?ml$/i.test(file)).sort((a, b) => a.split("/").length - b.split("/").length || (a < b ? -1 : a > b ? 1 : 0));
|
|
4397
|
+
for (const file of orphans) {
|
|
4398
|
+
if (included.has(file)) continue;
|
|
4399
|
+
const doc = input.tocs.get(file);
|
|
4400
|
+
if (isBreadcrumbToc(doc)) continue;
|
|
4401
|
+
const parent = parentOf(dirOf(file));
|
|
4402
|
+
const topic = newOutlineTopic(orphanTopicName(file, doc));
|
|
4403
|
+
((parent === null ? void 0 : homeOf(parent)) ?? top).children.push(topic);
|
|
4404
|
+
includeToc(file, topic, false);
|
|
4405
|
+
}
|
|
4406
|
+
const listed = /* @__PURE__ */ new Set();
|
|
4407
|
+
const collectListed = (topic) => {
|
|
4408
|
+
for (const entry2 of topic.entries) {
|
|
4409
|
+
if (!crossLinks.has(entry2)) listed.add(entry2.file.normalize("NFC"));
|
|
4410
|
+
}
|
|
4411
|
+
topic.children.forEach(collectListed);
|
|
4412
|
+
};
|
|
4413
|
+
collectListed(outline.root);
|
|
4414
|
+
const placed = /* @__PURE__ */ new Set();
|
|
4415
|
+
const missing = [];
|
|
4416
|
+
const duplicates = [];
|
|
4417
|
+
const settle = (topic) => {
|
|
4418
|
+
topic.entries = topic.entries.filter((entry2) => {
|
|
4419
|
+
const key = entry2.file.normalize("NFC");
|
|
4420
|
+
if (!known.has(key)) {
|
|
4421
|
+
missing.push(entry2.file);
|
|
4422
|
+
return false;
|
|
4423
|
+
}
|
|
4424
|
+
if (crossLinks.has(entry2) && listed.has(key)) return false;
|
|
4425
|
+
if (placed.has(key)) {
|
|
4426
|
+
if (!crossLinks.has(entry2)) duplicates.push(entry2.file);
|
|
4427
|
+
return false;
|
|
4428
|
+
}
|
|
4429
|
+
placed.add(key);
|
|
4430
|
+
return true;
|
|
4431
|
+
});
|
|
4432
|
+
topic.children.forEach(settle);
|
|
4433
|
+
};
|
|
4434
|
+
settle(outline.root);
|
|
4435
|
+
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
4436
|
+
const homed = [];
|
|
4437
|
+
for (const file of input.files) {
|
|
4438
|
+
const key = file.normalize("NFC");
|
|
4439
|
+
if (placed.has(key) || consumed.has(key)) continue;
|
|
4440
|
+
const home = homeOf(dirOf(file));
|
|
4441
|
+
if (!home) continue;
|
|
4442
|
+
home.entries.push({ file });
|
|
4443
|
+
homed.push(file);
|
|
4444
|
+
}
|
|
4445
|
+
if (lone) {
|
|
4446
|
+
outline.root.entries = lone.entries;
|
|
4447
|
+
outline.root.children = lone.children;
|
|
4448
|
+
}
|
|
4449
|
+
assignOutlineOrders(outline.root);
|
|
4450
|
+
const report = (files, what) => {
|
|
4451
|
+
if (files.length === 0) return;
|
|
4452
|
+
const named = files.slice(0, WARNING_EXAMPLES).join(", ");
|
|
4453
|
+
const more = files.length > WARNING_EXAMPLES ? `, and ${files.length - WARNING_EXAMPLES} more` : "";
|
|
4454
|
+
outline.warnings.push(`${files.length} ${what}: ${named}${more}`);
|
|
4455
|
+
};
|
|
4456
|
+
report(
|
|
4457
|
+
[...new Set(missing)],
|
|
4458
|
+
"pages named by a toc.yml are not among the published Markdown files"
|
|
4459
|
+
);
|
|
4460
|
+
report([...new Set(duplicates)], "pages are listed more than once; the first place wins");
|
|
4461
|
+
report(homed, "pages are in no toc.yml; filed with their folder's section");
|
|
4462
|
+
return outline;
|
|
4463
|
+
}
|
|
4464
|
+
function orphanTopicName(file, doc) {
|
|
4465
|
+
for (const item of tocItems(doc)) {
|
|
4466
|
+
if (!isRecord2(item) || typeof item.name !== "string" || typeof item.href !== "string") continue;
|
|
4467
|
+
if (!LANDING_PAGE.test(item.href.replace(/[?#].*$/, ""))) continue;
|
|
4468
|
+
const name = item.name.trim().replace(/\s+(documentation|docs)$/i, "").trim();
|
|
4469
|
+
if (name) return name;
|
|
4470
|
+
}
|
|
4471
|
+
const folder = posix2.basename(dirOf(file));
|
|
4472
|
+
return folder ? titleFromFolderName(folder) : "Documentation";
|
|
4473
|
+
}
|
|
4474
|
+
|
|
4475
|
+
// src/markdown-adapter/frontmatter.ts
|
|
4476
|
+
import { parseDocument } from "yaml";
|
|
4477
|
+
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
4478
|
+
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
4479
|
+
function parseYaml(text, opts) {
|
|
4480
|
+
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
4481
|
+
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
4482
|
+
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
4483
|
+
}
|
|
4484
|
+
const doc = parseDocument(text, {
|
|
4485
|
+
version: "1.2",
|
|
4486
|
+
schema: "core",
|
|
4487
|
+
uniqueKeys: true,
|
|
4488
|
+
logLevel: "silent"
|
|
4489
|
+
});
|
|
4490
|
+
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
4491
|
+
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
4492
|
+
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
4493
|
+
}
|
|
4494
|
+
var OPEN = /^---[ \t]*\r?\n/;
|
|
4495
|
+
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
4496
|
+
function parseMarkdownFrontMatter(source) {
|
|
4497
|
+
const open = OPEN.exec(source);
|
|
4498
|
+
if (!open) return { data: {}, body: source };
|
|
4499
|
+
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
4500
|
+
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
4501
|
+
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
4502
|
+
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
4503
|
+
const body = lines.slice(closeAt + 1).join("\n");
|
|
4504
|
+
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
4505
|
+
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
4506
|
+
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
4507
|
+
throw new Error("front matter must be a mapping");
|
|
4508
|
+
}
|
|
4509
|
+
return { data: parsed, body };
|
|
4510
|
+
}
|
|
4511
|
+
|
|
3961
4512
|
// src/markdown-adapter/load.ts
|
|
3962
4513
|
var ROOT_TOPIC_ID = "general";
|
|
3963
4514
|
var SUMMARY_MAX_CHARS = 280;
|
|
3964
4515
|
var TOPIC_SIDECAR = "_topic.yaml";
|
|
3965
4516
|
var HUGO_SECTION_PAGE = "_index";
|
|
4517
|
+
var DOCFX_JSON = "docfx.json";
|
|
4518
|
+
var WARNING_EXAMPLES2 = 20;
|
|
3966
4519
|
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
3967
4520
|
var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
|
|
3968
|
-
function readFrontMatter(raw, file,
|
|
4521
|
+
function readFrontMatter(raw, file, conventions) {
|
|
4522
|
+
const { hugo } = conventions;
|
|
3969
4523
|
let data;
|
|
3970
4524
|
let body;
|
|
3971
4525
|
try {
|
|
@@ -4030,8 +4584,8 @@ function readFrontMatter(raw, file, hugo) {
|
|
|
4030
4584
|
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
4031
4585
|
if (weight.data > 0) out.weight = weight.data;
|
|
4032
4586
|
}
|
|
4033
|
-
out.description = str("description");
|
|
4034
4587
|
}
|
|
4588
|
+
if (conventions.description) out.description = str("description");
|
|
4035
4589
|
for (const [key, value] of Object.entries(data)) {
|
|
4036
4590
|
if (RESERVED_KEYS.has(key) || value === void 0) continue;
|
|
4037
4591
|
out.meta[key] = value;
|
|
@@ -4060,7 +4614,7 @@ function topicIdFor(name, taken) {
|
|
|
4060
4614
|
taken.add(id);
|
|
4061
4615
|
return id;
|
|
4062
4616
|
}
|
|
4063
|
-
async function
|
|
4617
|
+
async function walkFiles(root, matches) {
|
|
4064
4618
|
const out = [];
|
|
4065
4619
|
const visit = async (dir) => {
|
|
4066
4620
|
const entries = (await readdir(dir, { withFileTypes: true })).sort(
|
|
@@ -4070,13 +4624,20 @@ async function walkMarkdownFiles(root) {
|
|
|
4070
4624
|
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
4071
4625
|
const abs = join6(dir, entry.name);
|
|
4072
4626
|
if (entry.isDirectory()) await visit(abs);
|
|
4073
|
-
else if (entry.isFile() &&
|
|
4627
|
+
else if (entry.isFile() && matches(entry.name)) out.push(abs);
|
|
4074
4628
|
}
|
|
4075
4629
|
};
|
|
4076
4630
|
await visit(root);
|
|
4077
4631
|
return out;
|
|
4078
4632
|
}
|
|
4079
|
-
function
|
|
4633
|
+
function walkMarkdownFiles(root) {
|
|
4634
|
+
return walkFiles(root, (name) => /\.(md|markdown)$/i.test(name));
|
|
4635
|
+
}
|
|
4636
|
+
function warnCapped(warn, items, each, rest) {
|
|
4637
|
+
for (const item of items.slice(0, WARNING_EXAMPLES2)) warn(each(item));
|
|
4638
|
+
if (items.length > WARNING_EXAMPLES2) warn(rest(items.length - WARNING_EXAMPLES2));
|
|
4639
|
+
}
|
|
4640
|
+
function isRecord3(value) {
|
|
4080
4641
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4081
4642
|
}
|
|
4082
4643
|
async function readTopicSidecar(dir) {
|
|
@@ -4085,7 +4646,7 @@ async function readTopicSidecar(dir) {
|
|
|
4085
4646
|
const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
4086
4647
|
const data = parseYaml(raw, { what: path });
|
|
4087
4648
|
if (data === null || data === void 0) return {};
|
|
4088
|
-
if (!
|
|
4649
|
+
if (!isRecord3(data)) throw new Error(`${path}: must be a mapping`);
|
|
4089
4650
|
const out = {};
|
|
4090
4651
|
if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
|
|
4091
4652
|
if (typeof data.description === "string" && data.description.trim()) {
|
|
@@ -4119,7 +4680,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
|
|
|
4119
4680
|
tolerateUnknownTags: true,
|
|
4120
4681
|
maxAliasCount: 100
|
|
4121
4682
|
});
|
|
4122
|
-
const docsDir =
|
|
4683
|
+
const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
4123
4684
|
return resolve4(dirname3(mkdocsPath), docsDir);
|
|
4124
4685
|
}
|
|
4125
4686
|
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
@@ -4134,6 +4695,10 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
|
4134
4695
|
])
|
|
4135
4696
|
);
|
|
4136
4697
|
if (mkdocs) return { format: "mkdocs", path: mkdocs };
|
|
4698
|
+
const docfx = firstExisting(
|
|
4699
|
+
[DOCFX_JSON, "toc.yml", "TOC.yml", "toc.yaml"].map((name) => join6(rootDir, name))
|
|
4700
|
+
);
|
|
4701
|
+
if (docfx) return { format: "docfx" };
|
|
4137
4702
|
const files = await walkMarkdownFiles(rootDir);
|
|
4138
4703
|
if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
|
|
4139
4704
|
return { format: "hugo" };
|
|
@@ -4153,6 +4718,7 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
4153
4718
|
if (explicit && !existsSync(explicit)) {
|
|
4154
4719
|
throw new Error(`${toc.format} table of contents not found: ${explicit}`);
|
|
4155
4720
|
}
|
|
4721
|
+
if (toc.format === "docfx") return loadDocfxOutline(rootDir, explicit, files);
|
|
4156
4722
|
if (toc.format === "gitbook") {
|
|
4157
4723
|
const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
|
|
4158
4724
|
const rel = insideRoot(rootDir, path2);
|
|
@@ -4195,16 +4761,68 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
4195
4761
|
});
|
|
4196
4762
|
return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
|
|
4197
4763
|
}
|
|
4764
|
+
async function loadDocfxOutline(rootDir, explicit, files) {
|
|
4765
|
+
const configPath = join6(rootDir, DOCFX_JSON);
|
|
4766
|
+
let project;
|
|
4767
|
+
if (existsSync(configPath)) {
|
|
4768
|
+
let config;
|
|
4769
|
+
try {
|
|
4770
|
+
config = JSON.parse(await readOutlineText(configPath));
|
|
4771
|
+
} catch (error) {
|
|
4772
|
+
throw new Error(`${DOCFX_JSON}: ${error instanceof Error ? error.message : String(error)}`);
|
|
4773
|
+
}
|
|
4774
|
+
project = parseDocfxJson(config, DOCFX_JSON);
|
|
4775
|
+
}
|
|
4776
|
+
const entry = explicit === void 0 ? void 0 : insideRoot(rootDir, explicit);
|
|
4777
|
+
if (entry === null) {
|
|
4778
|
+
throw new Error(`${basename(explicit)} must live inside the content root ${rootDir}`);
|
|
4779
|
+
}
|
|
4780
|
+
const readToc = async (abs, rel) => parseYaml(await readOutlineText(abs), {
|
|
4781
|
+
what: rel,
|
|
4782
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
4783
|
+
tolerateUnknownTags: true,
|
|
4784
|
+
maxAliasCount: 100
|
|
4785
|
+
});
|
|
4786
|
+
const tocs = /* @__PURE__ */ new Map();
|
|
4787
|
+
const warnings = [];
|
|
4788
|
+
for (const abs of await walkFiles(rootDir, (name) => /^toc\.ya?ml$/i.test(name))) {
|
|
4789
|
+
const rel = insideRoot(rootDir, abs);
|
|
4790
|
+
try {
|
|
4791
|
+
tocs.set(rel, await readToc(abs, rel));
|
|
4792
|
+
} catch (error) {
|
|
4793
|
+
if (rel === entry) throw error;
|
|
4794
|
+
warnings.push(`${rel}: ${error instanceof Error ? error.message : String(error)}; skipped`);
|
|
4795
|
+
}
|
|
4796
|
+
}
|
|
4797
|
+
if (entry !== void 0 && explicit !== void 0 && !tocs.has(entry)) {
|
|
4798
|
+
tocs.set(entry, await readToc(explicit, entry));
|
|
4799
|
+
}
|
|
4800
|
+
const publishes = project ? (file) => docfxPublishes(project, file) : void 0;
|
|
4801
|
+
const outline = parseDocfxToc({
|
|
4802
|
+
tocs,
|
|
4803
|
+
files: publishes ? files.filter(publishes) : files,
|
|
4804
|
+
...project ? { project } : {},
|
|
4805
|
+
...entry !== void 0 ? { entry } : {}
|
|
4806
|
+
});
|
|
4807
|
+
outline.warnings.unshift(...warnings);
|
|
4808
|
+
const path = outline.entry ? join6(rootDir, ...outline.entry.split("/")) : void 0;
|
|
4809
|
+
return {
|
|
4810
|
+
outline,
|
|
4811
|
+
...path ? { path } : project ? { path: configPath } : {},
|
|
4812
|
+
...publishes ? { publishes } : {}
|
|
4813
|
+
};
|
|
4814
|
+
}
|
|
4198
4815
|
var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
|
|
4199
|
-
var
|
|
4816
|
+
var HAS_SCHEME3 = /^[a-z][a-z0-9+.-]*:/i;
|
|
4200
4817
|
function resolveInTree(ctx, target) {
|
|
4201
|
-
const
|
|
4202
|
-
|
|
4818
|
+
const fromRoot = ctx.tildeIsRoot && target.startsWith("~/");
|
|
4819
|
+
const resolved = posix3.normalize(fromRoot ? target.slice(2) : posix3.join(ctx.docDir, target));
|
|
4820
|
+
if (resolved === ".." || resolved.startsWith("../") || posix3.isAbsolute(resolved)) return null;
|
|
4203
4821
|
return resolved === "." ? "" : resolved;
|
|
4204
4822
|
}
|
|
4205
4823
|
function rewriteTarget(ctx, isImage, rawTarget) {
|
|
4206
4824
|
const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
|
|
4207
|
-
if (target === "" ||
|
|
4825
|
+
if (target === "" || HAS_SCHEME3.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
|
|
4208
4826
|
return null;
|
|
4209
4827
|
}
|
|
4210
4828
|
const hashAt = target.search(/[#?]/);
|
|
@@ -4278,11 +4896,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4278
4896
|
});
|
|
4279
4897
|
const toc = opts.toc ?? { format: "folders" };
|
|
4280
4898
|
const hugo = toc.format === "hugo";
|
|
4899
|
+
const docfx = toc.format === "docfx";
|
|
4900
|
+
const conventions = { hugo, description: hugo || docfx };
|
|
4281
4901
|
const ignore = new Set(opts.ignore ?? []);
|
|
4282
4902
|
const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
|
|
4283
4903
|
let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
|
|
4284
|
-
const {
|
|
4904
|
+
const {
|
|
4905
|
+
outline,
|
|
4906
|
+
path: outlinePath,
|
|
4907
|
+
publishes
|
|
4908
|
+
} = await loadOutline(rootDir, toc, files.map(relOf));
|
|
4285
4909
|
for (const message of outline?.warnings ?? []) warn(message);
|
|
4910
|
+
if (publishes) files = files.filter((abs) => publishes(relOf(abs)));
|
|
4286
4911
|
if (outline) {
|
|
4287
4912
|
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
4288
4913
|
files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
|
|
@@ -4330,29 +4955,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4330
4955
|
};
|
|
4331
4956
|
const placements = /* @__PURE__ */ new Map();
|
|
4332
4957
|
if (outline) {
|
|
4958
|
+
const duplicates = [];
|
|
4333
4959
|
const walk = (topic, chain) => {
|
|
4334
4960
|
for (const entry of topic.entries) {
|
|
4335
4961
|
const key = entry.file.normalize("NFC");
|
|
4336
|
-
if (placements.has(key))
|
|
4337
|
-
|
|
4338
|
-
`${entry.file}: listed more than once in the table of contents; the first place wins`
|
|
4339
|
-
);
|
|
4340
|
-
} else {
|
|
4341
|
-
placements.set(key, { chain, entry });
|
|
4342
|
-
}
|
|
4962
|
+
if (placements.has(key)) duplicates.push(entry.file);
|
|
4963
|
+
else placements.set(key, { chain, entry });
|
|
4343
4964
|
}
|
|
4344
4965
|
for (const child of topic.children) walk(child, [...chain, child]);
|
|
4345
4966
|
};
|
|
4346
4967
|
walk(outline.root, []);
|
|
4968
|
+
warnCapped(
|
|
4969
|
+
warn,
|
|
4970
|
+
duplicates,
|
|
4971
|
+
(file) => `${file}: listed more than once in the table of contents; the first place wins`,
|
|
4972
|
+
(more) => `${more} more files are listed more than once in the table of contents`
|
|
4973
|
+
);
|
|
4347
4974
|
}
|
|
4348
4975
|
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
4976
|
+
let outlineTooDeep = false;
|
|
4349
4977
|
const ensureOutlineChain = (chain) => {
|
|
4350
4978
|
if (chain.length === 0) {
|
|
4351
4979
|
rootTopicUsed = true;
|
|
4352
4980
|
return [rootTopicId];
|
|
4353
4981
|
}
|
|
4982
|
+
if (chain.length > MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1) outlineTooDeep = true;
|
|
4354
4983
|
const ids = [];
|
|
4355
|
-
for (const node of chain) {
|
|
4984
|
+
for (const node of chain.slice(0, MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1)) {
|
|
4356
4985
|
let id = outlineTopicIds.get(node);
|
|
4357
4986
|
if (!id) {
|
|
4358
4987
|
id = topicIdFor(node.name, takenTopicIds);
|
|
@@ -4393,12 +5022,13 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4393
5022
|
const fileById = /* @__PURE__ */ new Map();
|
|
4394
5023
|
const idByRel = /* @__PURE__ */ new Map();
|
|
4395
5024
|
const seenRel = /* @__PURE__ */ new Set();
|
|
5025
|
+
const unlisted = [];
|
|
4396
5026
|
for (const abs of files) {
|
|
4397
5027
|
const rel = relOf(abs);
|
|
4398
5028
|
const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
4399
5029
|
const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
4400
5030
|
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
4401
|
-
const fm = readFrontMatter(raw, rel,
|
|
5031
|
+
const fm = readFrontMatter(raw, rel, conventions);
|
|
4402
5032
|
if (fm.unpublished) {
|
|
4403
5033
|
warn(`${rel}: skipped, ${fm.unpublished} page`);
|
|
4404
5034
|
continue;
|
|
@@ -4412,7 +5042,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4412
5042
|
fileById.set(id, rel);
|
|
4413
5043
|
idByRel.set(relNoExt, id);
|
|
4414
5044
|
const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
4415
|
-
let title = fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
5045
|
+
let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
4416
5046
|
let ordinal = fm.order;
|
|
4417
5047
|
let topicPath;
|
|
4418
5048
|
const placement = placements.get(rel.normalize("NFC"));
|
|
@@ -4421,8 +5051,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4421
5051
|
if (placement.entry.title) title = placement.entry.title;
|
|
4422
5052
|
if (placement.entry.order !== void 0) ordinal = placement.entry.order;
|
|
4423
5053
|
} else {
|
|
4424
|
-
if (outline)
|
|
4425
|
-
warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
|
|
5054
|
+
if (outline) unlisted.push(rel);
|
|
4426
5055
|
topicPath = await ensureTopicChain(relDir);
|
|
4427
5056
|
if (hugo && ordinal === void 0) ordinal = fm.weight;
|
|
4428
5057
|
}
|
|
@@ -4446,7 +5075,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4446
5075
|
const parentId = topicPath[topicPath.length - 1];
|
|
4447
5076
|
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
4448
5077
|
}
|
|
4449
|
-
const summary = fm.summary ??
|
|
5078
|
+
const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
|
|
4450
5079
|
loaded.push({
|
|
4451
5080
|
rel,
|
|
4452
5081
|
file: rel,
|
|
@@ -4478,12 +5107,22 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4478
5107
|
}
|
|
4479
5108
|
};
|
|
4480
5109
|
nameFromPages(outline.root);
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
5110
|
+
warnCapped(
|
|
5111
|
+
warn,
|
|
5112
|
+
unlisted,
|
|
5113
|
+
(file) => `${file}: not in the ${outline.format} table of contents; filed under its folder`,
|
|
5114
|
+
(more) => `${more} more files are not in the ${outline.format} table of contents; filed under their folders`
|
|
5115
|
+
);
|
|
5116
|
+
warnCapped(
|
|
5117
|
+
warn,
|
|
5118
|
+
[...placements].filter(([file]) => !seenRel.has(file)).map(([, p]) => p.entry.file),
|
|
5119
|
+
(file) => `${file}: named by the table of contents but not found among the Markdown files`,
|
|
5120
|
+
(more) => `${more} more files named by the table of contents are not among the Markdown files`
|
|
5121
|
+
);
|
|
5122
|
+
if (outlineTooDeep) {
|
|
5123
|
+
warn(
|
|
5124
|
+
`the table of contents nests deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1} levels; deeper sections are folded into their ancestors`
|
|
5125
|
+
);
|
|
4487
5126
|
}
|
|
4488
5127
|
}
|
|
4489
5128
|
if (rootTopicUsed) {
|
|
@@ -4501,6 +5140,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4501
5140
|
}
|
|
4502
5141
|
const assets = /* @__PURE__ */ new Map();
|
|
4503
5142
|
const documents = [];
|
|
5143
|
+
const linkWarnings = [];
|
|
4504
5144
|
for (const entry of loaded) {
|
|
4505
5145
|
const ctx = {
|
|
4506
5146
|
rootDir,
|
|
@@ -4509,8 +5149,11 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4509
5149
|
idByRel,
|
|
4510
5150
|
uri: opts.uri,
|
|
4511
5151
|
assets,
|
|
4512
|
-
|
|
4513
|
-
|
|
5152
|
+
// A docfx docset draws on dependent repositories (`~/reusable-content`)
|
|
5153
|
+
// that a checkout never holds, so a missing image is expected there.
|
|
5154
|
+
missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
|
|
5155
|
+
tildeIsRoot: docfx,
|
|
5156
|
+
warn: (message) => linkWarnings.push(message)
|
|
4514
5157
|
};
|
|
4515
5158
|
documents.push(
|
|
4516
5159
|
CatalogDocumentSchema.parse({
|
|
@@ -4519,6 +5162,12 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4519
5162
|
})
|
|
4520
5163
|
);
|
|
4521
5164
|
}
|
|
5165
|
+
warnCapped(
|
|
5166
|
+
warn,
|
|
5167
|
+
linkWarnings,
|
|
5168
|
+
(message) => message,
|
|
5169
|
+
(more) => `${more} more link and image warnings`
|
|
5170
|
+
);
|
|
4522
5171
|
return {
|
|
4523
5172
|
topics,
|
|
4524
5173
|
documents,
|
|
@@ -4691,6 +5340,7 @@ export {
|
|
|
4691
5340
|
SHARD_MAX_CHUNKS,
|
|
4692
5341
|
SHARD_TARGET_CHUNKS,
|
|
4693
5342
|
SOURCE_NOTICES_PATH,
|
|
5343
|
+
TRANSFORMERS_PEER_RANGE,
|
|
4694
5344
|
ZIP_FIXED_MTIME,
|
|
4695
5345
|
assertDuckdbCli,
|
|
4696
5346
|
assetReferences,
|
package/package.json
CHANGED