@bendyline/gezel-knowledge 1.2.5 → 1.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +34 -4
- package/dist/index.js +126 -70
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -475,8 +475,8 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
|
475
475
|
* Read-only document point index. The router's ordinary SQLite latitude and
|
|
476
476
|
* longitude indexes provide conservative candidates; the portable gezk
|
|
477
477
|
* spherical predicate decides membership before ranking or pagination.
|
|
478
|
-
*
|
|
479
|
-
*
|
|
478
|
+
* Archives without a `document_locations` table (before 0.7) have no
|
|
479
|
+
* locations: opaque producer metadata is never read as coordinates.
|
|
480
480
|
*/
|
|
481
481
|
|
|
482
482
|
interface SpatialMatch {
|
|
@@ -517,6 +517,14 @@ interface CatalogDocumentMeta {
|
|
|
517
517
|
meta: Record<string, unknown> | null;
|
|
518
518
|
locations?: KnowledgeLocation[];
|
|
519
519
|
}
|
|
520
|
+
/** One slug initial of a listing: where it first appears and how many documents share it. */
|
|
521
|
+
interface CatalogDocumentInitial {
|
|
522
|
+
/** The slug's first character, as stored (slugs are usually lowercase ASCII). */
|
|
523
|
+
initial: string;
|
|
524
|
+
/** Zero-based position of its first document in `documentsPage` order. */
|
|
525
|
+
offset: number;
|
|
526
|
+
count: number;
|
|
527
|
+
}
|
|
520
528
|
interface CatalogAssetInfo {
|
|
521
529
|
/** Archive path under `assets/`. */
|
|
522
530
|
path: string;
|
|
@@ -671,6 +679,26 @@ declare class CatalogHandle {
|
|
|
671
679
|
documents: CatalogDocumentMeta[];
|
|
672
680
|
total: number;
|
|
673
681
|
};
|
|
682
|
+
/**
|
|
683
|
+
* Where each slug initial first appears in `documentsPage` order, for an
|
|
684
|
+
* A–Z index over the listing. Only the alphabetical part counts: documents
|
|
685
|
+
* with an ordinal are listed first in their curated order, so a fully
|
|
686
|
+
* curated listing has no initials.
|
|
687
|
+
*/
|
|
688
|
+
documentInitials(opts?: {
|
|
689
|
+
topicId?: string;
|
|
690
|
+
descendants?: boolean;
|
|
691
|
+
}): CatalogDocumentInitial[];
|
|
692
|
+
/** A document's zero-based position in `documentsPage` order; null when the listing lacks it. */
|
|
693
|
+
documentPosition(documentId: string, opts?: {
|
|
694
|
+
topicId?: string;
|
|
695
|
+
descendants?: boolean;
|
|
696
|
+
}): number | null;
|
|
697
|
+
/** A listing's documents as `id, slug, ordinal` rows, ordered by LISTING_ORDER. */
|
|
698
|
+
private listingRows;
|
|
699
|
+
private plainScope;
|
|
700
|
+
/** The topic's placements (schema 4) as `scoped`; binds the topic id once. */
|
|
701
|
+
private sharedScope;
|
|
674
702
|
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
675
703
|
private sharedDocumentsPage;
|
|
676
704
|
/** Integrity facts kept separate from browse counts, which include shared references. */
|
|
@@ -957,7 +985,7 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
957
985
|
* bytes.
|
|
958
986
|
*/
|
|
959
987
|
|
|
960
|
-
/** 4: document-locations companion
|
|
988
|
+
/** 4: document-locations companion. */
|
|
961
989
|
declare const PARQUET_EXPORT_VERSION = 4;
|
|
962
990
|
type ParquetExportSource = {
|
|
963
991
|
archivePath: string;
|
|
@@ -1062,6 +1090,8 @@ declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
|
|
|
1062
1090
|
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
1063
1091
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
1064
1092
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
1093
|
+
/** What a profile's query model costs to download, or null for an unregistered profile. */
|
|
1094
|
+
declare function knowledgeProfileQueryModelBytes(id: string): number | null;
|
|
1065
1095
|
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
1066
1096
|
declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
1067
1097
|
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
@@ -1763,4 +1793,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1763
1793
|
|
|
1764
1794
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1765
1795
|
|
|
1766
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
|
1796
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -2195,23 +2195,19 @@ import {
|
|
|
2195
2195
|
KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
|
|
2196
2196
|
KnowledgeRadiusSchema,
|
|
2197
2197
|
locationDistanceMeters,
|
|
2198
|
-
locationInBounds,
|
|
2199
2198
|
normalizeLongitude as normalizeLongitude2,
|
|
2200
2199
|
radiusBounds,
|
|
2201
2200
|
spatialManifest as spatialManifest2
|
|
2202
2201
|
} from "@bendyline/gezk";
|
|
2203
2202
|
var DocumentSpatialIndex = class {
|
|
2204
|
-
constructor(db
|
|
2203
|
+
constructor(db) {
|
|
2205
2204
|
this.db = db;
|
|
2206
|
-
this.allowLegacy = allowLegacy;
|
|
2207
2205
|
this.hasTable = Boolean(
|
|
2208
2206
|
db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
|
|
2209
2207
|
);
|
|
2210
2208
|
}
|
|
2211
2209
|
db;
|
|
2212
|
-
allowLegacy;
|
|
2213
2210
|
hasTable;
|
|
2214
|
-
legacy;
|
|
2215
2211
|
decode(row) {
|
|
2216
2212
|
const provenance = row.provenance_json;
|
|
2217
2213
|
if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
|
|
@@ -2227,39 +2223,13 @@ var DocumentSpatialIndex = class {
|
|
|
2227
2223
|
})
|
|
2228
2224
|
};
|
|
2229
2225
|
}
|
|
2230
|
-
legacyRows() {
|
|
2231
|
-
if (this.legacy) return this.legacy;
|
|
2232
|
-
this.legacy = [];
|
|
2233
|
-
if (!this.allowLegacy) return this.legacy;
|
|
2234
|
-
for (const row of this.db.prepare(
|
|
2235
|
-
"SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
|
|
2236
|
-
).all()) {
|
|
2237
|
-
try {
|
|
2238
|
-
const meta = JSON.parse(String(row.meta_json));
|
|
2239
|
-
if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
|
|
2240
|
-
continue;
|
|
2241
|
-
const point = KnowledgeLocationSchema.safeParse({
|
|
2242
|
-
id: "qualla-coordinate",
|
|
2243
|
-
role: "subject",
|
|
2244
|
-
latitude: meta.coordinates.lat,
|
|
2245
|
-
longitude: meta.coordinates.lng,
|
|
2246
|
-
provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
|
|
2247
|
-
});
|
|
2248
|
-
if (point.success)
|
|
2249
|
-
this.legacy.push({
|
|
2250
|
-
documentId: String(row.id),
|
|
2251
|
-
location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
|
|
2252
|
-
});
|
|
2253
|
-
} catch {
|
|
2254
|
-
}
|
|
2255
|
-
}
|
|
2256
|
-
return this.legacy;
|
|
2257
|
-
}
|
|
2258
2226
|
rows() {
|
|
2259
|
-
|
|
2227
|
+
if (!this.hasTable) return [];
|
|
2228
|
+
return this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row));
|
|
2260
2229
|
}
|
|
2261
2230
|
locations(documentId) {
|
|
2262
|
-
|
|
2231
|
+
if (!this.hasTable) return [];
|
|
2232
|
+
return this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location);
|
|
2263
2233
|
}
|
|
2264
2234
|
integrity() {
|
|
2265
2235
|
const rows = this.rows();
|
|
@@ -2278,14 +2248,12 @@ var DocumentSpatialIndex = class {
|
|
|
2278
2248
|
}
|
|
2279
2249
|
matches(raw) {
|
|
2280
2250
|
const radius = KnowledgeRadiusSchema.parse(raw);
|
|
2281
|
-
const
|
|
2282
|
-
|
|
2251
|
+
const matches = /* @__PURE__ */ new Map();
|
|
2252
|
+
if (!this.hasTable) return matches;
|
|
2253
|
+
const candidates = radiusBounds(radius).flatMap(
|
|
2283
2254
|
(box) => this.db.prepare(`SELECT * FROM document_locations
|
|
2284
2255
|
WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
|
|
2285
|
-
) : this.legacyRows().filter(
|
|
2286
|
-
(row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
|
|
2287
2256
|
);
|
|
2288
|
-
const matches = /* @__PURE__ */ new Map();
|
|
2289
2257
|
for (const { documentId, location } of candidates) {
|
|
2290
2258
|
const distanceMeters = locationDistanceMeters(radius, location);
|
|
2291
2259
|
if (distanceMeters > radius.radiusMeters) continue;
|
|
@@ -2300,6 +2268,11 @@ var DocumentSpatialIndex = class {
|
|
|
2300
2268
|
|
|
2301
2269
|
// src/reader/catalog-handle.ts
|
|
2302
2270
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
2271
|
+
var LISTING_ORDER = "(ordinal IS NULL), ordinal, slug, id";
|
|
2272
|
+
var SHARED_PLACEMENT_RANK = `ranked AS (
|
|
2273
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2274
|
+
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2275
|
+
)`;
|
|
2303
2276
|
var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
|
|
2304
2277
|
var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
|
|
2305
2278
|
function mediaOf(row) {
|
|
@@ -2342,7 +2315,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2342
2315
|
this.rootDir = rootDir;
|
|
2343
2316
|
this.router = router;
|
|
2344
2317
|
this.schemaVersion = router.schemaVersion;
|
|
2345
|
-
this.spatial = new DocumentSpatialIndex(router.db
|
|
2318
|
+
this.spatial = new DocumentSpatialIndex(router.db);
|
|
2346
2319
|
this.meta = {};
|
|
2347
2320
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
2348
2321
|
this.meta[row.key] = row.value;
|
|
@@ -2690,22 +2663,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2690
2663
|
if (this.schemaVersion >= 4 && opts.topicId) {
|
|
2691
2664
|
return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
|
|
2692
2665
|
}
|
|
2693
|
-
|
|
2694
|
-
let where = "";
|
|
2695
|
-
const params = [];
|
|
2696
|
-
if (opts.topicId && descendants) {
|
|
2697
|
-
scope = `WITH RECURSIVE sub(id, depth) AS (
|
|
2698
|
-
SELECT ?, 0
|
|
2699
|
-
UNION ALL
|
|
2700
|
-
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
2701
|
-
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
2702
|
-
)`;
|
|
2703
|
-
where = "WHERE topic_id IN (SELECT id FROM sub)";
|
|
2704
|
-
params.push(opts.topicId);
|
|
2705
|
-
} else if (opts.topicId) {
|
|
2706
|
-
where = "WHERE topic_id = ?";
|
|
2707
|
-
params.push(opts.topicId);
|
|
2708
|
-
}
|
|
2666
|
+
const { scope, where, params } = this.plainScope(opts.topicId, descendants);
|
|
2709
2667
|
const total = Number(
|
|
2710
2668
|
this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
|
|
2711
2669
|
);
|
|
@@ -2715,23 +2673,88 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2715
2673
|
).all(...params, limit, offset).map((row) => this.documentMeta(row));
|
|
2716
2674
|
return { documents, total };
|
|
2717
2675
|
}
|
|
2718
|
-
/**
|
|
2719
|
-
|
|
2676
|
+
/**
|
|
2677
|
+
* Where each slug initial first appears in `documentsPage` order, for an
|
|
2678
|
+
* A–Z index over the listing. Only the alphabetical part counts: documents
|
|
2679
|
+
* with an ordinal are listed first in their curated order, so a fully
|
|
2680
|
+
* curated listing has no initials.
|
|
2681
|
+
*/
|
|
2682
|
+
documentInitials(opts = {}) {
|
|
2683
|
+
const { sql, params } = this.listingRows(opts.topicId, opts.descendants ?? true);
|
|
2684
|
+
const rows = this.router.db.prepare(
|
|
2685
|
+
`SELECT initial, MIN(position) AS position, COUNT(*) AS n FROM (
|
|
2686
|
+
SELECT substr(slug, 1, 1) AS initial, ordinal,
|
|
2687
|
+
ROW_NUMBER() OVER (ORDER BY ${LISTING_ORDER}) - 1 AS position
|
|
2688
|
+
FROM (${sql})
|
|
2689
|
+
) WHERE ordinal IS NULL GROUP BY initial ORDER BY position`
|
|
2690
|
+
).all(...params);
|
|
2691
|
+
return rows.map((r) => ({
|
|
2692
|
+
initial: r.initial,
|
|
2693
|
+
offset: Number(r.position),
|
|
2694
|
+
count: Number(r.n)
|
|
2695
|
+
}));
|
|
2696
|
+
}
|
|
2697
|
+
/** A document's zero-based position in `documentsPage` order; null when the listing lacks it. */
|
|
2698
|
+
documentPosition(documentId, opts = {}) {
|
|
2699
|
+
const { sql, params } = this.listingRows(opts.topicId, opts.descendants ?? true);
|
|
2700
|
+
const row = this.router.db.prepare(
|
|
2701
|
+
`SELECT position FROM (
|
|
2702
|
+
SELECT id, ROW_NUMBER() OVER (ORDER BY ${LISTING_ORDER}) - 1 AS position FROM (${sql})
|
|
2703
|
+
) WHERE id = ?`
|
|
2704
|
+
).get(...params, documentId);
|
|
2705
|
+
return row ? Number(row.position) : null;
|
|
2706
|
+
}
|
|
2707
|
+
/** A listing's documents as `id, slug, ordinal` rows, ordered by LISTING_ORDER. */
|
|
2708
|
+
listingRows(topicId, descendants) {
|
|
2709
|
+
if (this.schemaVersion >= 4 && topicId) {
|
|
2710
|
+
return {
|
|
2711
|
+
sql: `${this.sharedScope(descendants)}, ${SHARED_PLACEMENT_RANK}
|
|
2712
|
+
SELECT d.id AS id, d.slug AS slug, p.ordinal AS ordinal
|
|
2713
|
+
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1`,
|
|
2714
|
+
params: [topicId]
|
|
2715
|
+
};
|
|
2716
|
+
}
|
|
2717
|
+
const { scope, where, params } = this.plainScope(topicId, descendants);
|
|
2718
|
+
const ordinal = this.schemaVersion >= 3 ? "ordinal" : "NULL";
|
|
2719
|
+
return {
|
|
2720
|
+
sql: `${scope} SELECT id, slug, ${ordinal} AS ordinal FROM documents ${where}`,
|
|
2721
|
+
params
|
|
2722
|
+
};
|
|
2723
|
+
}
|
|
2724
|
+
plainScope(topicId, descendants) {
|
|
2725
|
+
if (topicId && descendants) {
|
|
2726
|
+
return {
|
|
2727
|
+
scope: `WITH RECURSIVE sub(id, depth) AS (
|
|
2728
|
+
SELECT ?, 0
|
|
2729
|
+
UNION ALL
|
|
2730
|
+
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
2731
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
2732
|
+
)`,
|
|
2733
|
+
where: "WHERE topic_id IN (SELECT id FROM sub)",
|
|
2734
|
+
params: [topicId]
|
|
2735
|
+
};
|
|
2736
|
+
}
|
|
2737
|
+
if (topicId) return { scope: "", where: "WHERE topic_id = ?", params: [topicId] };
|
|
2738
|
+
return { scope: "", where: "", params: [] };
|
|
2739
|
+
}
|
|
2740
|
+
/** The topic's placements (schema 4) as `scoped`; binds the topic id once. */
|
|
2741
|
+
sharedScope(descendants) {
|
|
2720
2742
|
const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
|
|
2721
2743
|
JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
|
|
2722
|
-
|
|
2744
|
+
return `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
|
|
2723
2745
|
SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
|
|
2724
2746
|
FROM topic_documents td JOIN topics t ON t.id = td.topic_id
|
|
2725
2747
|
WHERE td.topic_id IN (SELECT id FROM sub)
|
|
2726
2748
|
)`;
|
|
2749
|
+
}
|
|
2750
|
+
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
2751
|
+
sharedDocumentsPage(topicId, descendants, limit, offset) {
|
|
2752
|
+
const scope = this.sharedScope(descendants);
|
|
2727
2753
|
const total = Number(
|
|
2728
2754
|
this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
|
|
2729
2755
|
);
|
|
2730
2756
|
const documents = this.router.db.prepare(
|
|
2731
|
-
`${scope},
|
|
2732
|
-
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2733
|
-
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2734
|
-
) SELECT ${this.documentColumns("d", "p")}
|
|
2757
|
+
`${scope}, ${SHARED_PLACEMENT_RANK} SELECT ${this.documentColumns("d", "p")}
|
|
2735
2758
|
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
|
|
2736
2759
|
ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
|
|
2737
2760
|
).all(topicId, limit, offset).map((row) => this.documentMeta(row));
|
|
@@ -3725,9 +3748,9 @@ async function exportCatalogParquet(opts) {
|
|
|
3725
3748
|
onProgress: opts.onProgress
|
|
3726
3749
|
})
|
|
3727
3750
|
);
|
|
3728
|
-
const spatial = new DocumentSpatialIndex(router.db
|
|
3751
|
+
const spatial = new DocumentSpatialIndex(router.db);
|
|
3729
3752
|
const locations = spatial.rows();
|
|
3730
|
-
if (spatial.hasTable
|
|
3753
|
+
if (spatial.hasTable) {
|
|
3731
3754
|
const path = join4(staging, "document-locations.ndjson");
|
|
3732
3755
|
const out = ndjsonWriter(path);
|
|
3733
3756
|
for (const { documentId, location } of locations)
|
|
@@ -5183,6 +5206,15 @@ var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
|
5183
5206
|
function knowledgeEmbeddingProfile(id) {
|
|
5184
5207
|
return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
|
|
5185
5208
|
}
|
|
5209
|
+
var QUERY_MODEL_DOWNLOAD_BYTES = {
|
|
5210
|
+
"embeddinggemma-2-512@1": 346397233,
|
|
5211
|
+
"multilingual-e5-small@2": 487352364,
|
|
5212
|
+
"multilingual-e5-small@1": 487352364,
|
|
5213
|
+
"bge-small-en-v1.5@1": 133805935
|
|
5214
|
+
};
|
|
5215
|
+
function knowledgeProfileQueryModelBytes(id) {
|
|
5216
|
+
return QUERY_MODEL_DOWNLOAD_BYTES[id] ?? null;
|
|
5217
|
+
}
|
|
5186
5218
|
var MARKDOWN_CHUNKS_2 = {
|
|
5187
5219
|
id: "markdown-chunks@2",
|
|
5188
5220
|
unit: "tokens",
|
|
@@ -5406,6 +5438,27 @@ async function prefetchExternalData(profile, opts) {
|
|
|
5406
5438
|
return fetched;
|
|
5407
5439
|
}
|
|
5408
5440
|
|
|
5441
|
+
// src/embedding/transformers-import.ts
|
|
5442
|
+
var TRANSFORMERS_MODULE = "@huggingface/transformers";
|
|
5443
|
+
function unresolvedSpecifier(err) {
|
|
5444
|
+
if (!(err instanceof Error)) return null;
|
|
5445
|
+
const { code } = err;
|
|
5446
|
+
if (code !== "ERR_MODULE_NOT_FOUND" && code !== "MODULE_NOT_FOUND") return null;
|
|
5447
|
+
return /Cannot find (?:package|module) '([^']+)'/.exec(err.message)?.[1] ?? null;
|
|
5448
|
+
}
|
|
5449
|
+
var TransformersDependencyMissingError = class extends Error {
|
|
5450
|
+
isActionable = true;
|
|
5451
|
+
missing;
|
|
5452
|
+
constructor(missing, peerRange, cause) {
|
|
5453
|
+
super(
|
|
5454
|
+
`${TRANSFORMERS_MODULE} is installed but cannot load: Node cannot find '${missing}', which it imports. Reinstall ${TRANSFORMERS_MODULE}@${peerRange} together with its dependencies`,
|
|
5455
|
+
{ cause }
|
|
5456
|
+
);
|
|
5457
|
+
this.name = "TransformersDependencyMissingError";
|
|
5458
|
+
this.missing = missing;
|
|
5459
|
+
}
|
|
5460
|
+
};
|
|
5461
|
+
|
|
5409
5462
|
// src/embedding/profile-embedder.ts
|
|
5410
5463
|
var MAX_BATCH = 8;
|
|
5411
5464
|
var MAX_CHARS = 8e3;
|
|
@@ -5515,16 +5568,18 @@ function encoderRows(outputs, attentionMask) {
|
|
|
5515
5568
|
}
|
|
5516
5569
|
async function loadTransformers() {
|
|
5517
5570
|
try {
|
|
5518
|
-
const specifier =
|
|
5571
|
+
const specifier = TRANSFORMERS_MODULE;
|
|
5519
5572
|
return await import(specifier);
|
|
5520
5573
|
} catch (err) {
|
|
5521
|
-
const
|
|
5522
|
-
if (
|
|
5574
|
+
const missing = unresolvedSpecifier(err);
|
|
5575
|
+
if (missing === TRANSFORMERS_MODULE) {
|
|
5523
5576
|
throw new EmbedderUnavailableError(
|
|
5524
|
-
`the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer
|
|
5577
|
+
`the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer ${TRANSFORMERS_MODULE}@${TRANSFORMERS_PEER_RANGE}; install it alongside @bendyline/gezel-knowledge`,
|
|
5525
5578
|
{ runtimeMissing: true }
|
|
5526
5579
|
);
|
|
5527
5580
|
}
|
|
5581
|
+
if (missing)
|
|
5582
|
+
throw new TransformersDependencyMissingError(missing, TRANSFORMERS_PEER_RANGE, err);
|
|
5528
5583
|
throw err;
|
|
5529
5584
|
}
|
|
5530
5585
|
}
|
|
@@ -7238,6 +7293,7 @@ export {
|
|
|
7238
7293
|
isSupportedIndexSchemaVersion,
|
|
7239
7294
|
knowledgeEmbeddingProfile,
|
|
7240
7295
|
knowledgeKeyId,
|
|
7296
|
+
knowledgeProfileQueryModelBytes,
|
|
7241
7297
|
l2Normalize,
|
|
7242
7298
|
loadMarkdownCatalog,
|
|
7243
7299
|
manifestSigningPayload,
|
package/package.json
CHANGED