@bendyline/gezel-knowledge 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -209,6 +209,26 @@ interface CompileKnowledgeCatalogOptions {
209
209
  * builds must not set it: the constant IS the format's scale design.
210
210
  */
211
211
  shardTargetChunks?: number;
212
+ /**
213
+ * How a multi-shard catalog's documents are assigned to shards.
214
+ *
215
+ * - `'topic'` (default; the gezk spec §8.1): greedy fill in
216
+ * `(topicPathKey, documentId)` order, so shards are slices of the table
217
+ * of contents.
218
+ * - `'semantic'`: embed every chunk first, then fill shards by balanced
219
+ * k-means over each document's mean chunk vector, so documents that
220
+ * answer the same queries share a shard and routing (§8.4) reaches them
221
+ * within its budget. Topic slices put an artist, their albums and their
222
+ * songs in different shards. Within a shard documents keep
223
+ * `(topicPathKey, documentId)` order. Every chunk is still embedded
224
+ * exactly once: vectors are staged in `workDir` between the passes.
225
+ *
226
+ * Measured on a 1.85M-chunk Wikipedia music catalog (155 queries,
227
+ * routed recall@8 vs an exact scan): 10 shards, S=6 82.6% → 85.7% and
228
+ * S=3 52.1% → 63.9%; recut to 37 shards, S=6 36.0% → 55.5%. The gain
229
+ * grows with the shard count.
230
+ */
231
+ shardFill?: 'topic' | 'semantic';
212
232
  }
213
233
  interface CompileReport {
214
234
  manifest: KnowledgeCatalogManifest;
@@ -944,9 +964,19 @@ interface ProfileEmbedder {
944
964
  countTokens(text: string): number;
945
965
  dispose(): Promise<void>;
946
966
  }
967
+ /**
968
+ * The `@huggingface/transformers` range this package supports, the same range
969
+ * as its optional peer dependency (a test holds the two together). Error
970
+ * messages quote it so the install command they print is exact.
971
+ */
972
+ declare const TRANSFORMERS_PEER_RANGE = "^3.8.1";
947
973
  declare class EmbedderUnavailableError extends Error {
948
974
  readonly isActionable = true;
949
- constructor(message: string);
975
+ /** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
976
+ readonly runtimeMissing: boolean;
977
+ constructor(message: string, options?: {
978
+ runtimeMissing?: boolean;
979
+ });
950
980
  }
951
981
  /** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
952
982
  interface TransformersModule {
@@ -1022,7 +1052,7 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
1022
1052
  * files, never the disk, so an outline can be checked in a test without a
1023
1053
  * tree on disk.
1024
1054
  */
1025
- type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book';
1055
+ type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book' | 'docfx';
1026
1056
  type OutlineFormat = Exclude<TableOfContentsFormat, 'folders' | 'hugo'>;
1027
1057
  interface OutlineEntry {
1028
1058
  /** Root-relative POSIX path of the Markdown file, extension included. */
@@ -1101,10 +1131,11 @@ declare function parseJupyterBookToc(doc: unknown, tocDir: string, files: readon
1101
1131
  * gets the single root topic the compiler demands.
1102
1132
  *
1103
1133
  * A tree that already carries an outline keeps it: GitBook's `SUMMARY.md`,
1104
- * an `mkdocs.yml` nav and Jupyter Book's `_toc.yml` are read into the same
1105
- * topic tree (`outline.ts`), and Hugo's conventions — `_index.md` section
1106
- * pages, `weight`, `draft` — are honored on top of the folders. A file an
1107
- * outline omits stays in its folder, with a warning, so nothing is lost.
1134
+ * an `mkdocs.yml` nav, Jupyter Book's `_toc.yml` and docfx's per-folder
1135
+ * `toc.yml` files are read into the same topic tree (`outline.ts`,
1136
+ * `docfx.ts`), and Hugo's conventions — `_index.md` section pages,
1137
+ * `weight`, `draft` — are honored on top of the folders. A file an outline
1138
+ * omits stays in its folder, with a warning, so nothing is lost.
1108
1139
  *
1109
1140
  * Front matter carries what a documentation tree knows about itself: the
1110
1141
  * title and summary, an explicit `id`, an `order` (the listing ordinal), a
@@ -1136,7 +1167,10 @@ interface TableOfContentsOptions {
1136
1167
  /**
1137
1168
  * The outline file for `gitbook` (`SUMMARY.md`), `mkdocs` (`mkdocs.yml`)
1138
1169
  * and `jupyter-book` (`_toc.yml`), absolute or relative to the root. By
1139
- * default the root is searched, and for mkdocs its parent as well.
1170
+ * default the root is searched, and for mkdocs its parent as well. For
1171
+ * `docfx` it is the entry TOC (a breadcrumb such as `bread/toc.yml`, or a
1172
+ * root `toc.yml`); by default docfx.json's `breadcrumb_path`, else the
1173
+ * root `toc.yml`.
1140
1174
  */
1141
1175
  path?: string;
1142
1176
  }
@@ -1162,7 +1196,7 @@ interface LoadMarkdownCatalogOptions {
1162
1196
  publisherId: string;
1163
1197
  catalogId: string;
1164
1198
  };
1165
- /** What to do with an image link whose file is missing (default `error`). */
1199
+ /** What to do with an image link whose file is missing (default `error`; `warn` for docfx). */
1166
1200
  missingAssets?: 'error' | 'warn';
1167
1201
  onWarning?: (message: string) => void;
1168
1202
  }
@@ -1171,8 +1205,9 @@ declare function readMkdocsDocsDir(mkdocsPath: string): Promise<string>;
1171
1205
  /**
1172
1206
  * Which outline a tree carries: GitBook's `SUMMARY.md` or Jupyter Book's
1173
1207
  * `_toc.yml` at the root, an `mkdocs.yml` at the root, the project folder
1174
- * or the root's parent, Hugo's `_index.md` section pages anywhere in the
1175
- * tree — else the folders themselves.
1208
+ * or the root's parent, a docfx project (`docfx.json` or a `toc.yml` at the
1209
+ * root), Hugo's `_index.md` section pages anywhere in the tree — else the
1210
+ * folders themselves.
1176
1211
  */
1177
1212
  declare function detectTableOfContents(rootDir: string, projectDir?: string): Promise<TableOfContentsOptions>;
1178
1213
  /**
@@ -1262,4 +1297,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1262
1297
 
1263
1298
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1264
1299
 
1265
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
1300
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -181,7 +181,16 @@ function splitOversizeLine(line, opts) {
181
181
 
182
182
  // src/compiler/compile.ts
183
183
  import { createHash } from "crypto";
184
- import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
184
+ import {
185
+ closeSync,
186
+ mkdirSync,
187
+ openSync,
188
+ readFileSync,
189
+ readSync,
190
+ rmSync,
191
+ statSync,
192
+ writeSync
193
+ } from "fs";
185
194
  import { stat } from "fs/promises";
186
195
  import { join } from "path";
187
196
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
@@ -310,10 +319,47 @@ async function compileKnowledgeCatalog(opts) {
310
319
  const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
311
320
  const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
312
321
  const embedded = totalChunks <= shardTarget;
322
+ const semanticFill = opts.shardFill === "semantic" && !embedded;
323
+ const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
324
+ p.doc.title,
325
+ chunk.headingPath,
326
+ opts.chunkingProfile.contextHeader.max,
327
+ opts.countTokens
328
+ )}${chunk.text}`;
329
+ rmSync(opts.workDir, { recursive: true, force: true });
330
+ mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
331
+ const stagedPath = join(opts.workDir, "staged-vectors.f32");
332
+ let staged = null;
333
+ let docVectors = [];
334
+ if (semanticFill) {
335
+ docVectors = await stageVectors({
336
+ prepared,
337
+ stagedPath,
338
+ dimensions: profile.dimensions,
339
+ batchSize: opts.embedBatchSize ?? 32,
340
+ embed: opts.embed,
341
+ embedInput,
342
+ onProgress: (done) => opts.onProgress?.({ phase: "embed", done, total: totalChunks })
343
+ });
344
+ staged = new StagedVectorReader(stagedPath, profile.dimensions);
345
+ }
313
346
  const shardOf = /* @__PURE__ */ new Map();
314
347
  let shardCount = 1;
315
348
  if (embedded) {
316
349
  for (const p of prepared) shardOf.set(p.doc.id, 0);
350
+ } else if (semanticFill) {
351
+ shardCount = Math.ceil(totalChunks / shardTarget);
352
+ const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
353
+ const assignment = semanticShards(
354
+ docVectors,
355
+ prepared.map((p) => p.chunks.length),
356
+ shardCount,
357
+ capacity,
358
+ seedFor(`${opts.catalog.id}#shard-fill`, 0)
359
+ );
360
+ prepared.forEach((p, i) => shardOf.set(p.doc.id, assignment[i]));
361
+ shardCount = new Set(assignment).size;
362
+ docVectors = [];
317
363
  } else {
318
364
  let current = 0;
319
365
  let filled = 0;
@@ -327,8 +373,6 @@ async function compileKnowledgeCatalog(opts) {
327
373
  }
328
374
  shardCount = current + 1;
329
375
  }
330
- rmSync(opts.workDir, { recursive: true, force: true });
331
- mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
332
376
  const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
333
377
  const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
334
378
  const shardDbs = /* @__PURE__ */ new Map();
@@ -467,7 +511,7 @@ async function compileKnowledgeCatalog(opts) {
467
511
  let processedChunks = 0;
468
512
  const flushEmbeds = async () => {
469
513
  if (pendingTexts.length === 0) return;
470
- const vectors = await opts.embed(pendingTexts);
514
+ const vectors = staged ? staged.next(pendingTexts.length) : await opts.embed(pendingTexts);
471
515
  if (vectors.length !== pendingTexts.length) {
472
516
  throw new Error(
473
517
  `embedder returned ${vectors.length} vectors for ${pendingTexts.length} texts`
@@ -492,7 +536,11 @@ async function compileKnowledgeCatalog(opts) {
492
536
  if ((row.chunkId - 1) % stride === 0) samples.push(unit);
493
537
  }
494
538
  processedChunks += pendingTexts.length;
495
- opts.onProgress?.({ phase: "embed", done: processedChunks, total: totalChunks });
539
+ opts.onProgress?.({
540
+ phase: staged ? "write" : "embed",
541
+ done: processedChunks,
542
+ total: totalChunks
543
+ });
496
544
  pendingTexts = [];
497
545
  pendingRows = [];
498
546
  };
@@ -530,18 +578,14 @@ async function compileKnowledgeCatalog(opts) {
530
578
  const chunkId = Number(row.id);
531
579
  insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, chunk.text);
532
580
  shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
533
- const header = buildContextHeader(
534
- p.doc.title,
535
- chunk.headingPath,
536
- opts.chunkingProfile.contextHeader.max,
537
- opts.countTokens
538
- );
539
- pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
581
+ pendingTexts.push(embedInput(p, chunk));
540
582
  pendingRows.push({ shardId, chunkId });
541
583
  if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
542
584
  }
543
585
  }
544
586
  await flushEmbeds();
587
+ staged?.close();
588
+ rmSync(stagedPath, { force: true });
545
589
  for (let id = 0; id < shardCount; id++) {
546
590
  if (!embedded)
547
591
  putMeta(shardDb(id), {
@@ -913,6 +957,158 @@ ${path.join(" > ")}
913
957
  }
914
958
  return header;
915
959
  }
960
+ async function stageVectors(args) {
961
+ const { prepared, dimensions } = args;
962
+ const docVectors = prepared.map(() => new Float32Array(dimensions));
963
+ const fd = openSync(args.stagedPath, "w");
964
+ let texts = [];
965
+ let owners = [];
966
+ let done = 0;
967
+ const flush = async () => {
968
+ if (texts.length === 0) return;
969
+ const vectors = await args.embed(texts);
970
+ if (vectors.length !== texts.length) {
971
+ throw new Error(`embedder returned ${vectors.length} vectors for ${texts.length} texts`);
972
+ }
973
+ const block = new Float32Array(texts.length * dimensions);
974
+ for (let i = 0; i < vectors.length; i++) {
975
+ const unit = l2Normalize(vectors[i]);
976
+ if (unit.length !== dimensions) {
977
+ throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
978
+ }
979
+ block.set(unit, i * dimensions);
980
+ const sum = docVectors[owners[i]];
981
+ for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
982
+ }
983
+ writeSync(fd, new Uint8Array(block.buffer));
984
+ done += texts.length;
985
+ args.onProgress(done);
986
+ texts = [];
987
+ owners = [];
988
+ };
989
+ try {
990
+ for (let i = 0; i < prepared.length; i++) {
991
+ const p = prepared[i];
992
+ for (const chunk of p.chunks) {
993
+ texts.push(args.embedInput(p, chunk));
994
+ owners.push(i);
995
+ if (texts.length >= args.batchSize) await flush();
996
+ }
997
+ }
998
+ await flush();
999
+ } finally {
1000
+ closeSync(fd);
1001
+ }
1002
+ return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
1003
+ }
1004
+ var StagedVectorReader = class {
1005
+ constructor(path, dimensions) {
1006
+ this.dimensions = dimensions;
1007
+ this.fd = openSync(path, "r");
1008
+ }
1009
+ dimensions;
1010
+ fd;
1011
+ position = 0;
1012
+ next(count) {
1013
+ const bytes = count * this.dimensions * 4;
1014
+ const buffer = Buffer.alloc(bytes);
1015
+ let read = 0;
1016
+ while (read < bytes) {
1017
+ const n = readSync(this.fd, buffer, read, bytes - read, this.position + read);
1018
+ if (n === 0)
1019
+ throw new Error("staged vectors ended early: the write pass and pass 1 disagree");
1020
+ read += n;
1021
+ }
1022
+ this.position += bytes;
1023
+ const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * this.dimensions);
1024
+ const out = [];
1025
+ for (let i = 0; i < count; i++) {
1026
+ out.push(Array.from(floats.subarray(i * this.dimensions, (i + 1) * this.dimensions)));
1027
+ }
1028
+ return out;
1029
+ }
1030
+ close() {
1031
+ closeSync(this.fd);
1032
+ }
1033
+ };
1034
+ var SEMANTIC_FILL_ROUNDS = 6;
1035
+ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1036
+ const n = docVectors.length;
1037
+ const dim = docVectors[0].length;
1038
+ const dot = (a, b) => {
1039
+ let sum = 0;
1040
+ for (let d = 0; d < dim; d++) sum += a[d] * b[d];
1041
+ return sum;
1042
+ };
1043
+ const stride = Math.max(1, Math.ceil(n / 2e4));
1044
+ const sample = docVectors.filter((_, i) => i % stride === 0);
1045
+ const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
1046
+ for (let i = 0; centers.length < shardCount; i++) {
1047
+ centers.push(
1048
+ Float32Array.from(docVectors[Math.floor(i * n / shardCount) % n])
1049
+ );
1050
+ }
1051
+ let assignment = new Array(n).fill(-1);
1052
+ for (let round = 0; round < SEMANTIC_FILL_ROUNDS; round++) {
1053
+ const best = new Int32Array(n);
1054
+ const margin = new Float64Array(n);
1055
+ for (let i = 0; i < n; i++) {
1056
+ let first = Number.NEGATIVE_INFINITY;
1057
+ let second = Number.NEGATIVE_INFINITY;
1058
+ let arg = 0;
1059
+ for (let c = 0; c < shardCount; c++) {
1060
+ const s = dot(docVectors[i], centers[c]);
1061
+ if (s > first) {
1062
+ second = first;
1063
+ first = s;
1064
+ arg = c;
1065
+ } else if (s > second) {
1066
+ second = s;
1067
+ }
1068
+ }
1069
+ best[i] = arg;
1070
+ margin[i] = first - (Number.isFinite(second) ? second : first);
1071
+ }
1072
+ const order = Array.from({ length: n }, (_, i) => i).sort(
1073
+ (a, b) => margin[b] - margin[a] || a - b
1074
+ );
1075
+ const load = new Array(shardCount).fill(0);
1076
+ const next = new Array(n).fill(-1);
1077
+ for (const i of order) {
1078
+ const size = docChunks[i];
1079
+ let shard = best[i];
1080
+ if (load[shard] + size > capacity) {
1081
+ const ranked = centers.map((c, id) => ({ id, s: dot(docVectors[i], c) })).sort((a, b) => b.s - a.s || a.id - b.id);
1082
+ const fit = ranked.find((r) => load[r.id] + size <= capacity);
1083
+ shard = fit ? fit.id : load.reduce((min, l, id) => l < load[min] ? id : min, 0);
1084
+ }
1085
+ next[i] = shard;
1086
+ load[shard] = load[shard] + size;
1087
+ }
1088
+ const moved = next.some((s, i) => s !== assignment[i]);
1089
+ assignment = next;
1090
+ if (!moved) break;
1091
+ const sums = centers.map(() => new Float64Array(dim));
1092
+ for (let i = 0; i < n; i++) {
1093
+ const sum = sums[assignment[i]];
1094
+ const v = docVectors[i];
1095
+ const w = docChunks[i];
1096
+ for (let d = 0; d < dim; d++) sum[d] = sum[d] + v[d] * w;
1097
+ }
1098
+ for (let c = 0; c < shardCount; c++) {
1099
+ const sum = sums[c];
1100
+ let norm = 0;
1101
+ for (let d = 0; d < dim; d++) norm += sum[d] ** 2;
1102
+ if (norm === 0) continue;
1103
+ norm = Math.sqrt(norm);
1104
+ const center = centers[c];
1105
+ for (let d = 0; d < dim; d++) center[d] = sum[d] / norm;
1106
+ }
1107
+ }
1108
+ const renumber = /* @__PURE__ */ new Map();
1109
+ for (const s of assignment) if (!renumber.has(s)) renumber.set(s, renumber.size);
1110
+ return assignment.map((s) => renumber.get(s));
1111
+ }
916
1112
  function seedFor(catalogId, shardId) {
917
1113
  const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
918
1114
  return digest.readBigUInt64LE(24);
@@ -3472,11 +3668,15 @@ async function verifyProfileArtifacts(profile, opts) {
3472
3668
  // src/embedding/profile-embedder.ts
3473
3669
  var MAX_BATCH = 8;
3474
3670
  var MAX_CHARS = 8e3;
3671
+ var TRANSFORMERS_PEER_RANGE = "^3.8.1";
3475
3672
  var EmbedderUnavailableError = class extends Error {
3476
3673
  isActionable = true;
3477
- constructor(message) {
3674
+ /** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
3675
+ runtimeMissing;
3676
+ constructor(message, options = {}) {
3478
3677
  super(message);
3479
3678
  this.name = "EmbedderUnavailableError";
3679
+ this.runtimeMissing = options.runtimeMissing ?? false;
3480
3680
  }
3481
3681
  };
3482
3682
  function aggregateDownloadProgress(onProgress) {
@@ -3545,7 +3745,8 @@ async function loadTransformers() {
3545
3745
  const message = err instanceof Error ? err.message : String(err);
3546
3746
  if (/Cannot find (module|package)/i.test(message)) {
3547
3747
  throw new EmbedderUnavailableError(
3548
- "the embedding runtime (@huggingface/transformers) is not installed \u2014 knowledge builds and semantic search need it; install it or run inside a gezel install that ships it"
3748
+ `the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer @huggingface/transformers@${TRANSFORMERS_PEER_RANGE}; install it alongside @bendyline/gezel-knowledge`,
3749
+ { runtimeMissing: true }
3549
3750
  );
3550
3751
  }
3551
3752
  throw err;
@@ -3616,52 +3817,19 @@ async function createProfileEmbedder(profile, opts = {}) {
3616
3817
  // src/markdown-adapter/load.ts
3617
3818
  import { existsSync } from "fs";
3618
3819
  import { readFile as readFile3, readdir } from "fs/promises";
3619
- import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
3820
+ import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
3620
3821
  import {
3621
3822
  CatalogDocumentSchema,
3622
3823
  KnowledgeDocumentIdSchema,
3623
3824
  KnowledgeOrdinalSchema,
3825
+ MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
3624
3826
  assetExtension as assetExtension3,
3625
3827
  formatKnowledgeUri,
3626
3828
  topicSortKeyForOrder
3627
3829
  } from "@bendyline/gezk";
3628
3830
 
3629
- // src/markdown-adapter/frontmatter.ts
3630
- import { parseDocument } from "yaml";
3631
- var FRONTMATTER_MAX_BYTES = 64 * 1024;
3632
- var OUTLINE_MAX_BYTES = 1024 * 1024;
3633
- function parseYaml(text, opts) {
3634
- const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
3635
- if (Buffer.byteLength(text, "utf8") > maxBytes) {
3636
- throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
3637
- }
3638
- const doc = parseDocument(text, {
3639
- version: "1.2",
3640
- schema: "core",
3641
- uniqueKeys: true,
3642
- logLevel: "silent"
3643
- });
3644
- const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
3645
- if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
3646
- return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
3647
- }
3648
- var OPEN = /^---[ \t]*\r?\n/;
3649
- var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
3650
- function parseMarkdownFrontMatter(source) {
3651
- const open = OPEN.exec(source);
3652
- if (!open) return { data: {}, body: source };
3653
- const lines = source.slice(open[0].length).split(/\r?\n/);
3654
- const closeAt = lines.findIndex((line) => CLOSE.test(line));
3655
- if (closeAt === -1) throw new Error("front matter is not closed with ---");
3656
- const yamlText = lines.slice(0, closeAt).join("\n");
3657
- const body = lines.slice(closeAt + 1).join("\n");
3658
- const parsed = parseYaml(yamlText, { what: "front matter" });
3659
- if (parsed === null || parsed === void 0) return { data: {}, body };
3660
- if (typeof parsed !== "object" || Array.isArray(parsed)) {
3661
- throw new Error("front matter must be a mapping");
3662
- }
3663
- return { data: parsed, body };
3664
- }
3831
+ // src/markdown-adapter/docfx.ts
3832
+ import { posix as posix2 } from "path";
3665
3833
 
3666
3834
  // src/markdown-adapter/outline.ts
3667
3835
  import { posix } from "path";
@@ -3958,14 +4126,400 @@ function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
3958
4126
  return outline;
3959
4127
  }
3960
4128
 
4129
+ // src/markdown-adapter/docfx.ts
4130
+ var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
4131
+ var MARKDOWN_EXT2 = /\.(md|markdown)$/i;
4132
+ var TOC_FILE = /^toc\.(ya?ml|json|md)$/i;
4133
+ var LANDING_PAGE = /(^|\/)index\.(ya?ml|md)$/i;
4134
+ var WARNING_EXAMPLES = 5;
4135
+ function isRecord2(value) {
4136
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4137
+ }
4138
+ function normalizeFolder(value) {
4139
+ if (typeof value !== "string") return "";
4140
+ const cleaned = posix2.normalize(value.replace(/\\/g, "/")).replace(/^\.?\/+|\/+$/g, "");
4141
+ return cleaned === "." ? "" : cleaned;
4142
+ }
4143
+ function dirOf(file) {
4144
+ const dir = posix2.dirname(file);
4145
+ return dir === "." ? "" : dir;
4146
+ }
4147
+ function parentOf(dir) {
4148
+ return dir === "" ? null : dirOf(dir);
4149
+ }
4150
+ function isUnder(file, dir) {
4151
+ return dir === "" || file.startsWith(`${dir}/`);
4152
+ }
4153
+ function escapeRegExp(text) {
4154
+ return text.replace(/[.+^$()|[\]\\]/g, "\\$&");
4155
+ }
4156
+ function docfxGlob(pattern) {
4157
+ const p = pattern.replace(/\\/g, "/").replace(/^\.\//, "");
4158
+ let source = "";
4159
+ for (let i = 0; i < p.length; i++) {
4160
+ const ch = p[i];
4161
+ if (ch === "*") {
4162
+ if (p[i + 1] === "*") {
4163
+ if (p[i + 2] === "/") {
4164
+ source += "(?:.*/)?";
4165
+ i += 2;
4166
+ } else {
4167
+ source += ".*";
4168
+ i += 1;
4169
+ }
4170
+ } else source += "[^/]*";
4171
+ } else if (ch === "?") {
4172
+ source += "[^/]";
4173
+ } else if (ch === "{" && p.indexOf("}", i) > i) {
4174
+ const close = p.indexOf("}", i);
4175
+ source += `(?:${p.slice(i + 1, close).split(",").map(escapeRegExp).join("|")})`;
4176
+ i = close;
4177
+ } else {
4178
+ source += escapeRegExp(ch);
4179
+ }
4180
+ }
4181
+ return new RegExp(`^${source}$`, "i");
4182
+ }
4183
+ function globList(value) {
4184
+ const list = typeof value === "string" ? [value] : Array.isArray(value) ? value : [];
4185
+ return list.filter((g) => typeof g === "string" && g.trim() !== "").map(docfxGlob);
4186
+ }
4187
+ function parseDocfxJson(config, configFile = "docfx.json") {
4188
+ if (!isRecord2(config)) throw new Error(`${configFile}: not a JSON object`);
4189
+ const build = isRecord2(config.build) ? config.build : {};
4190
+ const content = [];
4191
+ for (const group of Array.isArray(build.content) ? build.content : []) {
4192
+ if (!isRecord2(group)) continue;
4193
+ const include = globList(group.files);
4194
+ if (include.length === 0) continue;
4195
+ content.push({
4196
+ src: normalizeFolder(group.src),
4197
+ dest: normalizeFolder(group.dest),
4198
+ include,
4199
+ exclude: globList(group.exclude)
4200
+ });
4201
+ }
4202
+ const meta = isRecord2(build.globalMetadata) ? build.globalMetadata : {};
4203
+ const breadcrumb = meta.breadcrumb_path;
4204
+ return {
4205
+ content,
4206
+ ...typeof breadcrumb === "string" && breadcrumb.trim() ? { breadcrumbPath: breadcrumb.trim() } : {}
4207
+ };
4208
+ }
4209
+ function docfxPublishes(project, file) {
4210
+ if (project.content.length === 0) return true;
4211
+ return project.content.some((group) => {
4212
+ if (!isUnder(file, group.src)) return false;
4213
+ const rel = group.src === "" ? file : file.slice(group.src.length + 1);
4214
+ return group.include.some((r) => r.test(rel)) && !group.exclude.some((r) => r.test(rel));
4215
+ });
4216
+ }
4217
+ function tocItems(doc) {
4218
+ if (Array.isArray(doc)) return doc;
4219
+ if (isRecord2(doc) && Array.isArray(doc.items)) return doc.items;
4220
+ return [];
4221
+ }
4222
+ function isBreadcrumbToc(doc) {
4223
+ let tocHref = false;
4224
+ const visit = (items) => {
4225
+ for (const item of items) {
4226
+ if (!isRecord2(item)) continue;
4227
+ if (typeof item.href === "string") return false;
4228
+ if (typeof item.tocHref === "string") tocHref = true;
4229
+ if (Array.isArray(item.items) && !visit(item.items)) return false;
4230
+ }
4231
+ return true;
4232
+ };
4233
+ return visit(tocItems(doc)) && tocHref;
4234
+ }
4235
+ function basePrefixes(url) {
4236
+ const segments = url.split("/").filter(Boolean);
4237
+ const out = ["/"];
4238
+ for (let i = 1; i < segments.length; i++) out.push(`/${segments.slice(0, i).join("/")}/`);
4239
+ return out;
4240
+ }
4241
+ function parseDocfxToc(input) {
4242
+ const outline = {
4243
+ format: "docfx",
4244
+ root: newOutlineTopic(""),
4245
+ consumed: input.files.filter((file) => TOC_FILE.test(posix2.basename(file))),
4246
+ warnings: []
4247
+ };
4248
+ const known = new Set(input.files.map((file) => file.normalize("NFC")));
4249
+ const groups = input.project?.content.length ? [...input.project.content].sort((a, b) => b.dest.length - a.dest.length) : [{ src: "", dest: "", include: [], exclude: [] }];
4250
+ const tocByDir = /* @__PURE__ */ new Map();
4251
+ for (const file of [...input.tocs.keys()].sort()) {
4252
+ if (/\.ya?ml$/i.test(file) && !tocByDir.has(dirOf(file))) tocByDir.set(dirOf(file), file);
4253
+ }
4254
+ const sourcesOf = (sitePath) => {
4255
+ const out = [];
4256
+ for (const group of groups) {
4257
+ if (group.dest !== "" && sitePath !== group.dest && !sitePath.startsWith(`${group.dest}/`))
4258
+ continue;
4259
+ const rest = sitePath.slice(group.dest.length).replace(/^\/+/, "");
4260
+ out.push(group.src && rest ? `${group.src}/${rest}` : group.src || rest);
4261
+ }
4262
+ return out;
4263
+ };
4264
+ const tocAt = (path) => {
4265
+ const trimmed = path.replace(/\/+$/, "");
4266
+ return TOC_FILE.test(posix2.basename(trimmed)) ? tocByDir.get(dirOf(trimmed)) : tocByDir.get(trimmed);
4267
+ };
4268
+ const pageAt = (path) => {
4269
+ const candidates = path === "" || path.endsWith("/") ? [`${path}index.md`] : MARKDOWN_EXT2.test(path) ? [path] : [`${path}.md`, `${path}/index.md`];
4270
+ return candidates.find((candidate) => known.has(candidate.normalize("NFC")));
4271
+ };
4272
+ const resolveSite = (url, base2) => {
4273
+ const path = url.replace(/[?#].*$/, "");
4274
+ let below;
4275
+ if (`${path}/` === base2) below = "";
4276
+ else if (path.startsWith(base2)) below = path.slice(base2.length);
4277
+ else return null;
4278
+ for (const source of sourcesOf(below)) {
4279
+ const page = pageAt(source);
4280
+ if (page) return { kind: "page", file: page };
4281
+ const toc = tocAt(source);
4282
+ if (toc) return { kind: "toc", file: toc };
4283
+ }
4284
+ return null;
4285
+ };
4286
+ let entry = input.entry;
4287
+ let base;
4288
+ const breadcrumb = input.project?.breadcrumbPath;
4289
+ if (!entry && breadcrumb) {
4290
+ for (const candidate of basePrefixes(breadcrumb)) {
4291
+ const target = resolveSite(breadcrumb, candidate);
4292
+ if (target?.kind === "toc") {
4293
+ entry = target.file;
4294
+ base = candidate;
4295
+ break;
4296
+ }
4297
+ }
4298
+ if (!entry) {
4299
+ outline.warnings.push(
4300
+ `docfx.json: breadcrumb_path '${breadcrumb}' names no toc.yml in the tree; ignored`
4301
+ );
4302
+ }
4303
+ }
4304
+ entry ??= tocByDir.get("");
4305
+ if (entry && base === void 0) {
4306
+ const urls = [];
4307
+ const collect = (items) => {
4308
+ for (const item of items) {
4309
+ if (!isRecord2(item)) continue;
4310
+ for (const key of ["tocHref", "topicHref", "href"]) {
4311
+ const value = item[key];
4312
+ if (typeof value === "string" && value.startsWith("/") && !value.startsWith("//")) {
4313
+ urls.push(value);
4314
+ }
4315
+ }
4316
+ if (Array.isArray(item.items)) collect(item.items);
4317
+ }
4318
+ };
4319
+ collect(tocItems(input.tocs.get(entry)));
4320
+ let best = { base: "/", hits: 0 };
4321
+ for (const candidate of [...new Set(urls.flatMap(basePrefixes))].sort(
4322
+ (a, b) => a.length - b.length
4323
+ )) {
4324
+ const hits = urls.filter((url) => resolveSite(url, candidate) !== null).length;
4325
+ if (hits > best.hits) best = { base: candidate, hits };
4326
+ }
4327
+ base = best.base;
4328
+ }
4329
+ const siteBase = base ?? "/";
4330
+ const classify = (href, tocDir) => {
4331
+ const value = href.trim();
4332
+ if (value === "" || value.startsWith("#") || HAS_SCHEME2.test(value) || value.startsWith("//"))
4333
+ return null;
4334
+ if (value.startsWith("/")) return resolveSite(value, siteBase);
4335
+ const path = resolveOutlinePath(tocDir, value);
4336
+ if (path === null) return null;
4337
+ if (MARKDOWN_EXT2.test(path)) return { kind: "page", file: path };
4338
+ const toc = tocAt(path);
4339
+ return toc ? { kind: "toc", file: toc } : null;
4340
+ };
4341
+ const crossLinks = /* @__PURE__ */ new WeakSet();
4342
+ const included = /* @__PURE__ */ new Set();
4343
+ const homes = /* @__PURE__ */ new Map();
4344
+ const pageEntry = (file, tocDir) => {
4345
+ const entry2 = { file };
4346
+ if (!isUnder(file, tocDir)) crossLinks.add(entry2);
4347
+ return entry2;
4348
+ };
4349
+ const walkItems = (items, tocFile, container, followTocHref) => {
4350
+ const tocDir = dirOf(tocFile);
4351
+ for (const item of items) {
4352
+ if (!isRecord2(item)) continue;
4353
+ const name = typeof item.name === "string" ? item.name.trim() : "";
4354
+ const href = typeof item.href === "string" ? classify(item.href, tocDir) : null;
4355
+ const topicHref = typeof item.topicHref === "string" ? classify(item.topicHref, tocDir) : null;
4356
+ const tocHref = followTocHref && typeof item.tocHref === "string" ? classify(item.tocHref, tocDir) : null;
4357
+ const page = [href, topicHref].find((target) => target?.kind === "page");
4358
+ const nested = [href, tocHref].find(
4359
+ (target) => target?.kind === "toc" && !included.has(target.file)
4360
+ );
4361
+ const children = Array.isArray(item.items) && item.items.length > 0 ? item.items : null;
4362
+ if (!nested && !children) {
4363
+ if (page) container.entries.push(pageEntry(page.file, tocDir));
4364
+ continue;
4365
+ }
4366
+ const topic = newOutlineTopic(
4367
+ name || (nested ? titleFromFolderName(posix2.basename(dirOf(nested.file))) : "Untitled")
4368
+ );
4369
+ if (!name && page) topic.titleFromFile = page.file;
4370
+ if (page) topic.entries.push(pageEntry(page.file, tocDir));
4371
+ if (nested) includeToc(nested.file, topic, false);
4372
+ if (children) walkItems(children, tocFile, topic, followTocHref);
4373
+ container.children.push(topic);
4374
+ }
4375
+ };
4376
+ const includeToc = (file, container, followTocHref) => {
4377
+ if (included.has(file)) return;
4378
+ included.add(file);
4379
+ const dir = dirOf(file);
4380
+ if (container !== outline.root && !homes.has(dir)) homes.set(dir, container);
4381
+ walkItems(tocItems(input.tocs.get(file)), file, container, followTocHref);
4382
+ };
4383
+ const homeOf = (dir) => {
4384
+ for (let d = dir; d !== null; d = parentOf(d)) {
4385
+ const home = homes.get(d);
4386
+ if (home) return home;
4387
+ }
4388
+ return void 0;
4389
+ };
4390
+ if (entry) {
4391
+ outline.entry = entry;
4392
+ includeToc(entry, outline.root, true);
4393
+ }
4394
+ const lone = outline.root.entries.length === 0 && outline.root.children.length === 1 ? outline.root.children[0] : void 0;
4395
+ const top = lone ?? outline.root;
4396
+ const orphans = [...input.tocs.keys()].filter((file) => !included.has(file) && /\.ya?ml$/i.test(file)).sort((a, b) => a.split("/").length - b.split("/").length || (a < b ? -1 : a > b ? 1 : 0));
4397
+ for (const file of orphans) {
4398
+ if (included.has(file)) continue;
4399
+ const doc = input.tocs.get(file);
4400
+ if (isBreadcrumbToc(doc)) continue;
4401
+ const parent = parentOf(dirOf(file));
4402
+ const topic = newOutlineTopic(orphanTopicName(file, doc));
4403
+ ((parent === null ? void 0 : homeOf(parent)) ?? top).children.push(topic);
4404
+ includeToc(file, topic, false);
4405
+ }
4406
+ const listed = /* @__PURE__ */ new Set();
4407
+ const collectListed = (topic) => {
4408
+ for (const entry2 of topic.entries) {
4409
+ if (!crossLinks.has(entry2)) listed.add(entry2.file.normalize("NFC"));
4410
+ }
4411
+ topic.children.forEach(collectListed);
4412
+ };
4413
+ collectListed(outline.root);
4414
+ const placed = /* @__PURE__ */ new Set();
4415
+ const missing = [];
4416
+ const duplicates = [];
4417
+ const settle = (topic) => {
4418
+ topic.entries = topic.entries.filter((entry2) => {
4419
+ const key = entry2.file.normalize("NFC");
4420
+ if (!known.has(key)) {
4421
+ missing.push(entry2.file);
4422
+ return false;
4423
+ }
4424
+ if (crossLinks.has(entry2) && listed.has(key)) return false;
4425
+ if (placed.has(key)) {
4426
+ if (!crossLinks.has(entry2)) duplicates.push(entry2.file);
4427
+ return false;
4428
+ }
4429
+ placed.add(key);
4430
+ return true;
4431
+ });
4432
+ topic.children.forEach(settle);
4433
+ };
4434
+ settle(outline.root);
4435
+ const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
4436
+ const homed = [];
4437
+ for (const file of input.files) {
4438
+ const key = file.normalize("NFC");
4439
+ if (placed.has(key) || consumed.has(key)) continue;
4440
+ const home = homeOf(dirOf(file));
4441
+ if (!home) continue;
4442
+ home.entries.push({ file });
4443
+ homed.push(file);
4444
+ }
4445
+ if (lone) {
4446
+ outline.root.entries = lone.entries;
4447
+ outline.root.children = lone.children;
4448
+ }
4449
+ assignOutlineOrders(outline.root);
4450
+ const report = (files, what) => {
4451
+ if (files.length === 0) return;
4452
+ const named = files.slice(0, WARNING_EXAMPLES).join(", ");
4453
+ const more = files.length > WARNING_EXAMPLES ? `, and ${files.length - WARNING_EXAMPLES} more` : "";
4454
+ outline.warnings.push(`${files.length} ${what}: ${named}${more}`);
4455
+ };
4456
+ report(
4457
+ [...new Set(missing)],
4458
+ "pages named by a toc.yml are not among the published Markdown files"
4459
+ );
4460
+ report([...new Set(duplicates)], "pages are listed more than once; the first place wins");
4461
+ report(homed, "pages are in no toc.yml; filed with their folder's section");
4462
+ return outline;
4463
+ }
4464
+ function orphanTopicName(file, doc) {
4465
+ for (const item of tocItems(doc)) {
4466
+ if (!isRecord2(item) || typeof item.name !== "string" || typeof item.href !== "string") continue;
4467
+ if (!LANDING_PAGE.test(item.href.replace(/[?#].*$/, ""))) continue;
4468
+ const name = item.name.trim().replace(/\s+(documentation|docs)$/i, "").trim();
4469
+ if (name) return name;
4470
+ }
4471
+ const folder = posix2.basename(dirOf(file));
4472
+ return folder ? titleFromFolderName(folder) : "Documentation";
4473
+ }
4474
+
4475
+ // src/markdown-adapter/frontmatter.ts
4476
+ import { parseDocument } from "yaml";
4477
+ var FRONTMATTER_MAX_BYTES = 64 * 1024;
4478
+ var OUTLINE_MAX_BYTES = 1024 * 1024;
4479
+ function parseYaml(text, opts) {
4480
+ const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
4481
+ if (Buffer.byteLength(text, "utf8") > maxBytes) {
4482
+ throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
4483
+ }
4484
+ const doc = parseDocument(text, {
4485
+ version: "1.2",
4486
+ schema: "core",
4487
+ uniqueKeys: true,
4488
+ logLevel: "silent"
4489
+ });
4490
+ const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
4491
+ if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
4492
+ return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
4493
+ }
4494
+ var OPEN = /^---[ \t]*\r?\n/;
4495
+ var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
4496
+ function parseMarkdownFrontMatter(source) {
4497
+ const open = OPEN.exec(source);
4498
+ if (!open) return { data: {}, body: source };
4499
+ const lines = source.slice(open[0].length).split(/\r?\n/);
4500
+ const closeAt = lines.findIndex((line) => CLOSE.test(line));
4501
+ if (closeAt === -1) throw new Error("front matter is not closed with ---");
4502
+ const yamlText = lines.slice(0, closeAt).join("\n");
4503
+ const body = lines.slice(closeAt + 1).join("\n");
4504
+ const parsed = parseYaml(yamlText, { what: "front matter" });
4505
+ if (parsed === null || parsed === void 0) return { data: {}, body };
4506
+ if (typeof parsed !== "object" || Array.isArray(parsed)) {
4507
+ throw new Error("front matter must be a mapping");
4508
+ }
4509
+ return { data: parsed, body };
4510
+ }
4511
+
3961
4512
  // src/markdown-adapter/load.ts
3962
4513
  var ROOT_TOPIC_ID = "general";
3963
4514
  var SUMMARY_MAX_CHARS = 280;
3964
4515
  var TOPIC_SIDECAR = "_topic.yaml";
3965
4516
  var HUGO_SECTION_PAGE = "_index";
4517
+ var DOCFX_JSON = "docfx.json";
4518
+ var WARNING_EXAMPLES2 = 20;
3966
4519
  var SECTION_PAGE_ORDINAL = -2147483648;
3967
4520
  var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
3968
- function readFrontMatter(raw, file, hugo) {
4521
+ function readFrontMatter(raw, file, conventions) {
4522
+ const { hugo } = conventions;
3969
4523
  let data;
3970
4524
  let body;
3971
4525
  try {
@@ -4030,8 +4584,8 @@ function readFrontMatter(raw, file, hugo) {
4030
4584
  if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
4031
4585
  if (weight.data > 0) out.weight = weight.data;
4032
4586
  }
4033
- out.description = str("description");
4034
4587
  }
4588
+ if (conventions.description) out.description = str("description");
4035
4589
  for (const [key, value] of Object.entries(data)) {
4036
4590
  if (RESERVED_KEYS.has(key) || value === void 0) continue;
4037
4591
  out.meta[key] = value;
@@ -4060,7 +4614,7 @@ function topicIdFor(name, taken) {
4060
4614
  taken.add(id);
4061
4615
  return id;
4062
4616
  }
4063
- async function walkMarkdownFiles(root) {
4617
+ async function walkFiles(root, matches) {
4064
4618
  const out = [];
4065
4619
  const visit = async (dir) => {
4066
4620
  const entries = (await readdir(dir, { withFileTypes: true })).sort(
@@ -4070,13 +4624,20 @@ async function walkMarkdownFiles(root) {
4070
4624
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
4071
4625
  const abs = join6(dir, entry.name);
4072
4626
  if (entry.isDirectory()) await visit(abs);
4073
- else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
4627
+ else if (entry.isFile() && matches(entry.name)) out.push(abs);
4074
4628
  }
4075
4629
  };
4076
4630
  await visit(root);
4077
4631
  return out;
4078
4632
  }
4079
- function isRecord2(value) {
4633
+ function walkMarkdownFiles(root) {
4634
+ return walkFiles(root, (name) => /\.(md|markdown)$/i.test(name));
4635
+ }
4636
+ function warnCapped(warn, items, each, rest) {
4637
+ for (const item of items.slice(0, WARNING_EXAMPLES2)) warn(each(item));
4638
+ if (items.length > WARNING_EXAMPLES2) warn(rest(items.length - WARNING_EXAMPLES2));
4639
+ }
4640
+ function isRecord3(value) {
4080
4641
  return typeof value === "object" && value !== null && !Array.isArray(value);
4081
4642
  }
4082
4643
  async function readTopicSidecar(dir) {
@@ -4085,7 +4646,7 @@ async function readTopicSidecar(dir) {
4085
4646
  const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
4086
4647
  const data = parseYaml(raw, { what: path });
4087
4648
  if (data === null || data === void 0) return {};
4088
- if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
4649
+ if (!isRecord3(data)) throw new Error(`${path}: must be a mapping`);
4089
4650
  const out = {};
4090
4651
  if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
4091
4652
  if (typeof data.description === "string" && data.description.trim()) {
@@ -4119,7 +4680,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
4119
4680
  tolerateUnknownTags: true,
4120
4681
  maxAliasCount: 100
4121
4682
  });
4122
- const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
4683
+ const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
4123
4684
  return resolve4(dirname3(mkdocsPath), docsDir);
4124
4685
  }
4125
4686
  async function detectTableOfContents(rootDir, projectDir = rootDir) {
@@ -4134,6 +4695,10 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
4134
4695
  ])
4135
4696
  );
4136
4697
  if (mkdocs) return { format: "mkdocs", path: mkdocs };
4698
+ const docfx = firstExisting(
4699
+ [DOCFX_JSON, "toc.yml", "TOC.yml", "toc.yaml"].map((name) => join6(rootDir, name))
4700
+ );
4701
+ if (docfx) return { format: "docfx" };
4137
4702
  const files = await walkMarkdownFiles(rootDir);
4138
4703
  if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
4139
4704
  return { format: "hugo" };
@@ -4153,6 +4718,7 @@ async function loadOutline(rootDir, toc, files) {
4153
4718
  if (explicit && !existsSync(explicit)) {
4154
4719
  throw new Error(`${toc.format} table of contents not found: ${explicit}`);
4155
4720
  }
4721
+ if (toc.format === "docfx") return loadDocfxOutline(rootDir, explicit, files);
4156
4722
  if (toc.format === "gitbook") {
4157
4723
  const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
4158
4724
  const rel = insideRoot(rootDir, path2);
@@ -4195,16 +4761,68 @@ async function loadOutline(rootDir, toc, files) {
4195
4761
  });
4196
4762
  return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
4197
4763
  }
4764
+ async function loadDocfxOutline(rootDir, explicit, files) {
4765
+ const configPath = join6(rootDir, DOCFX_JSON);
4766
+ let project;
4767
+ if (existsSync(configPath)) {
4768
+ let config;
4769
+ try {
4770
+ config = JSON.parse(await readOutlineText(configPath));
4771
+ } catch (error) {
4772
+ throw new Error(`${DOCFX_JSON}: ${error instanceof Error ? error.message : String(error)}`);
4773
+ }
4774
+ project = parseDocfxJson(config, DOCFX_JSON);
4775
+ }
4776
+ const entry = explicit === void 0 ? void 0 : insideRoot(rootDir, explicit);
4777
+ if (entry === null) {
4778
+ throw new Error(`${basename(explicit)} must live inside the content root ${rootDir}`);
4779
+ }
4780
+ const readToc = async (abs, rel) => parseYaml(await readOutlineText(abs), {
4781
+ what: rel,
4782
+ maxBytes: OUTLINE_MAX_BYTES,
4783
+ tolerateUnknownTags: true,
4784
+ maxAliasCount: 100
4785
+ });
4786
+ const tocs = /* @__PURE__ */ new Map();
4787
+ const warnings = [];
4788
+ for (const abs of await walkFiles(rootDir, (name) => /^toc\.ya?ml$/i.test(name))) {
4789
+ const rel = insideRoot(rootDir, abs);
4790
+ try {
4791
+ tocs.set(rel, await readToc(abs, rel));
4792
+ } catch (error) {
4793
+ if (rel === entry) throw error;
4794
+ warnings.push(`${rel}: ${error instanceof Error ? error.message : String(error)}; skipped`);
4795
+ }
4796
+ }
4797
+ if (entry !== void 0 && explicit !== void 0 && !tocs.has(entry)) {
4798
+ tocs.set(entry, await readToc(explicit, entry));
4799
+ }
4800
+ const publishes = project ? (file) => docfxPublishes(project, file) : void 0;
4801
+ const outline = parseDocfxToc({
4802
+ tocs,
4803
+ files: publishes ? files.filter(publishes) : files,
4804
+ ...project ? { project } : {},
4805
+ ...entry !== void 0 ? { entry } : {}
4806
+ });
4807
+ outline.warnings.unshift(...warnings);
4808
+ const path = outline.entry ? join6(rootDir, ...outline.entry.split("/")) : void 0;
4809
+ return {
4810
+ outline,
4811
+ ...path ? { path } : project ? { path: configPath } : {},
4812
+ ...publishes ? { publishes } : {}
4813
+ };
4814
+ }
4198
4815
  var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
4199
- var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
4816
+ var HAS_SCHEME3 = /^[a-z][a-z0-9+.-]*:/i;
4200
4817
  function resolveInTree(ctx, target) {
4201
- const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
4202
- if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
4818
+ const fromRoot = ctx.tildeIsRoot && target.startsWith("~/");
4819
+ const resolved = posix3.normalize(fromRoot ? target.slice(2) : posix3.join(ctx.docDir, target));
4820
+ if (resolved === ".." || resolved.startsWith("../") || posix3.isAbsolute(resolved)) return null;
4203
4821
  return resolved === "." ? "" : resolved;
4204
4822
  }
4205
4823
  function rewriteTarget(ctx, isImage, rawTarget) {
4206
4824
  const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
4207
- if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
4825
+ if (target === "" || HAS_SCHEME3.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
4208
4826
  return null;
4209
4827
  }
4210
4828
  const hashAt = target.search(/[#?]/);
@@ -4278,11 +4896,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
4278
4896
  });
4279
4897
  const toc = opts.toc ?? { format: "folders" };
4280
4898
  const hugo = toc.format === "hugo";
4899
+ const docfx = toc.format === "docfx";
4900
+ const conventions = { hugo, description: hugo || docfx };
4281
4901
  const ignore = new Set(opts.ignore ?? []);
4282
4902
  const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
4283
4903
  let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
4284
- const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
4904
+ const {
4905
+ outline,
4906
+ path: outlinePath,
4907
+ publishes
4908
+ } = await loadOutline(rootDir, toc, files.map(relOf));
4285
4909
  for (const message of outline?.warnings ?? []) warn(message);
4910
+ if (publishes) files = files.filter((abs) => publishes(relOf(abs)));
4286
4911
  if (outline) {
4287
4912
  const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
4288
4913
  files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
@@ -4330,29 +4955,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
4330
4955
  };
4331
4956
  const placements = /* @__PURE__ */ new Map();
4332
4957
  if (outline) {
4958
+ const duplicates = [];
4333
4959
  const walk = (topic, chain) => {
4334
4960
  for (const entry of topic.entries) {
4335
4961
  const key = entry.file.normalize("NFC");
4336
- if (placements.has(key)) {
4337
- warn(
4338
- `${entry.file}: listed more than once in the table of contents; the first place wins`
4339
- );
4340
- } else {
4341
- placements.set(key, { chain, entry });
4342
- }
4962
+ if (placements.has(key)) duplicates.push(entry.file);
4963
+ else placements.set(key, { chain, entry });
4343
4964
  }
4344
4965
  for (const child of topic.children) walk(child, [...chain, child]);
4345
4966
  };
4346
4967
  walk(outline.root, []);
4968
+ warnCapped(
4969
+ warn,
4970
+ duplicates,
4971
+ (file) => `${file}: listed more than once in the table of contents; the first place wins`,
4972
+ (more) => `${more} more files are listed more than once in the table of contents`
4973
+ );
4347
4974
  }
4348
4975
  const outlineTopicIds = /* @__PURE__ */ new Map();
4976
+ let outlineTooDeep = false;
4349
4977
  const ensureOutlineChain = (chain) => {
4350
4978
  if (chain.length === 0) {
4351
4979
  rootTopicUsed = true;
4352
4980
  return [rootTopicId];
4353
4981
  }
4982
+ if (chain.length > MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1) outlineTooDeep = true;
4354
4983
  const ids = [];
4355
- for (const node of chain) {
4984
+ for (const node of chain.slice(0, MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1)) {
4356
4985
  let id = outlineTopicIds.get(node);
4357
4986
  if (!id) {
4358
4987
  id = topicIdFor(node.name, takenTopicIds);
@@ -4393,12 +5022,13 @@ async function loadMarkdownCatalog(rootDir, opts) {
4393
5022
  const fileById = /* @__PURE__ */ new Map();
4394
5023
  const idByRel = /* @__PURE__ */ new Map();
4395
5024
  const seenRel = /* @__PURE__ */ new Set();
5025
+ const unlisted = [];
4396
5026
  for (const abs of files) {
4397
5027
  const rel = relOf(abs);
4398
5028
  const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
4399
5029
  const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
4400
5030
  const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
4401
- const fm = readFrontMatter(raw, rel, hugo);
5031
+ const fm = readFrontMatter(raw, rel, conventions);
4402
5032
  if (fm.unpublished) {
4403
5033
  warn(`${rel}: skipped, ${fm.unpublished} page`);
4404
5034
  continue;
@@ -4412,7 +5042,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
4412
5042
  fileById.set(id, rel);
4413
5043
  idByRel.set(relNoExt, id);
4414
5044
  const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
4415
- let title = fm.title ?? firstHeading(fm.body) ?? stem2;
5045
+ let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
4416
5046
  let ordinal = fm.order;
4417
5047
  let topicPath;
4418
5048
  const placement = placements.get(rel.normalize("NFC"));
@@ -4421,8 +5051,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
4421
5051
  if (placement.entry.title) title = placement.entry.title;
4422
5052
  if (placement.entry.order !== void 0) ordinal = placement.entry.order;
4423
5053
  } else {
4424
- if (outline)
4425
- warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
5054
+ if (outline) unlisted.push(rel);
4426
5055
  topicPath = await ensureTopicChain(relDir);
4427
5056
  if (hugo && ordinal === void 0) ordinal = fm.weight;
4428
5057
  }
@@ -4446,7 +5075,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
4446
5075
  const parentId = topicPath[topicPath.length - 1];
4447
5076
  topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
4448
5077
  }
4449
- const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
5078
+ const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
4450
5079
  loaded.push({
4451
5080
  rel,
4452
5081
  file: rel,
@@ -4478,12 +5107,22 @@ async function loadMarkdownCatalog(rootDir, opts) {
4478
5107
  }
4479
5108
  };
4480
5109
  nameFromPages(outline.root);
4481
- for (const [file, placement] of placements) {
4482
- if (!seenRel.has(file)) {
4483
- warn(
4484
- `${placement.entry.file}: named by the table of contents but not found among the Markdown files`
4485
- );
4486
- }
5110
+ warnCapped(
5111
+ warn,
5112
+ unlisted,
5113
+ (file) => `${file}: not in the ${outline.format} table of contents; filed under its folder`,
5114
+ (more) => `${more} more files are not in the ${outline.format} table of contents; filed under their folders`
5115
+ );
5116
+ warnCapped(
5117
+ warn,
5118
+ [...placements].filter(([file]) => !seenRel.has(file)).map(([, p]) => p.entry.file),
5119
+ (file) => `${file}: named by the table of contents but not found among the Markdown files`,
5120
+ (more) => `${more} more files named by the table of contents are not among the Markdown files`
5121
+ );
5122
+ if (outlineTooDeep) {
5123
+ warn(
5124
+ `the table of contents nests deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1} levels; deeper sections are folded into their ancestors`
5125
+ );
4487
5126
  }
4488
5127
  }
4489
5128
  if (rootTopicUsed) {
@@ -4501,6 +5140,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
4501
5140
  }
4502
5141
  const assets = /* @__PURE__ */ new Map();
4503
5142
  const documents = [];
5143
+ const linkWarnings = [];
4504
5144
  for (const entry of loaded) {
4505
5145
  const ctx = {
4506
5146
  rootDir,
@@ -4509,8 +5149,11 @@ async function loadMarkdownCatalog(rootDir, opts) {
4509
5149
  idByRel,
4510
5150
  uri: opts.uri,
4511
5151
  assets,
4512
- missingAssets: opts.missingAssets ?? "error",
4513
- warn
5152
+ // A docfx docset draws on dependent repositories (`~/reusable-content`)
5153
+ // that a checkout never holds, so a missing image is expected there.
5154
+ missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
5155
+ tildeIsRoot: docfx,
5156
+ warn: (message) => linkWarnings.push(message)
4514
5157
  };
4515
5158
  documents.push(
4516
5159
  CatalogDocumentSchema.parse({
@@ -4519,6 +5162,12 @@ async function loadMarkdownCatalog(rootDir, opts) {
4519
5162
  })
4520
5163
  );
4521
5164
  }
5165
+ warnCapped(
5166
+ warn,
5167
+ linkWarnings,
5168
+ (message) => message,
5169
+ (more) => `${more} more link and image warnings`
5170
+ );
4522
5171
  return {
4523
5172
  topics,
4524
5173
  documents,
@@ -4691,6 +5340,7 @@ export {
4691
5340
  SHARD_MAX_CHUNKS,
4692
5341
  SHARD_TARGET_CHUNKS,
4693
5342
  SOURCE_NOTICES_PATH,
5343
+ TRANSFORMERS_PEER_RANGE,
4694
5344
  ZIP_FIXED_MTIME,
4695
5345
  assertDuckdbCli,
4696
5346
  assetReferences,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-knowledge",
3
- "version": "1.2.0",
3
+ "version": "1.2.2",
4
4
  "description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [