@equationalapplications/core-llm-wiki 6.3.0 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/{chunk-YDTJ2OO4.mjs → chunk-DPFM33WV.mjs} +158 -13
- package/dist/chunk-DPFM33WV.mjs.map +1 -0
- package/dist/index.d.mts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +158 -10
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BPwT0171.d.mts → testing-DWpAgNLt.d.mts} +39 -1
- package/dist/{testing-BPwT0171.d.ts → testing-DWpAgNLt.d.ts} +39 -1
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +155 -10
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-YDTJ2OO4.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DWpAgNLt.mjs';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiInvalidReadOptions, Z as WikiMemoryTestAccess, _ as WikiOutboxEvent, $ as WikiParseError, a0 as WikiSourceRefHashCollision, a1 as WikiStrictOntologyViolation, a2 as WikiTask, a3 as WikiTransactionError } from './testing-DWpAgNLt.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -119,6 +119,18 @@ declare const DEFAULT_MAX_CHUNK_LENGTH = 12000;
|
|
|
119
119
|
*/
|
|
120
120
|
declare const DEFAULT_CHUNK_OVERLAP = 400;
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Default clamp on embed input length, in characters (~1,500 tokens).
|
|
124
|
+
* Chosen to fit common embedding windows including nomic-embed-text's 2,048 tokens.
|
|
125
|
+
*/
|
|
126
|
+
declare const DEFAULT_MAX_EMBED_CHARS = 6000;
|
|
127
|
+
/**
|
|
128
|
+
* Hard upper bound on embed input length, in characters.
|
|
129
|
+
* An unbounded-input guard from `5e04bad fix(core): security hardening phase 2`.
|
|
130
|
+
* Configuration may lower the effective limit but must never raise it past this.
|
|
131
|
+
*/
|
|
132
|
+
declare const EMBED_CHARS_CEILING = 16000;
|
|
133
|
+
|
|
122
134
|
interface LibrarianOptions {
|
|
123
135
|
/** If provided, replaces the default Librarian system instructions. */
|
|
124
136
|
systemPrompt?: string;
|
|
@@ -145,4 +157,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
145
157
|
|
|
146
158
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
147
159
|
|
|
148
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
160
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DWpAgNLt.js';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiInvalidReadOptions, Z as WikiMemoryTestAccess, _ as WikiOutboxEvent, $ as WikiParseError, a0 as WikiSourceRefHashCollision, a1 as WikiStrictOntologyViolation, a2 as WikiTask, a3 as WikiTransactionError } from './testing-DWpAgNLt.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -119,6 +119,18 @@ declare const DEFAULT_MAX_CHUNK_LENGTH = 12000;
|
|
|
119
119
|
*/
|
|
120
120
|
declare const DEFAULT_CHUNK_OVERLAP = 400;
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Default clamp on embed input length, in characters (~1,500 tokens).
|
|
124
|
+
* Chosen to fit common embedding windows including nomic-embed-text's 2,048 tokens.
|
|
125
|
+
*/
|
|
126
|
+
declare const DEFAULT_MAX_EMBED_CHARS = 6000;
|
|
127
|
+
/**
|
|
128
|
+
* Hard upper bound on embed input length, in characters.
|
|
129
|
+
* An unbounded-input guard from `5e04bad fix(core): security hardening phase 2`.
|
|
130
|
+
* Configuration may lower the effective limit but must never raise it past this.
|
|
131
|
+
*/
|
|
132
|
+
declare const EMBED_CHARS_CEILING = 16000;
|
|
133
|
+
|
|
122
134
|
interface LibrarianOptions {
|
|
123
135
|
/** If provided, replaces the default Librarian system instructions. */
|
|
124
136
|
systemPrompt?: string;
|
|
@@ -145,4 +157,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
145
157
|
|
|
146
158
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
147
159
|
|
|
148
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
160
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.js
CHANGED
|
@@ -306,6 +306,15 @@ var WikiIngestEmptyError = class extends Error {
|
|
|
306
306
|
this.chunks = params.chunks;
|
|
307
307
|
}
|
|
308
308
|
};
|
|
309
|
+
var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
310
|
+
constructor(field, reason) {
|
|
311
|
+
super(`Invalid ReadOptions.${field}: ${reason}`);
|
|
312
|
+
this.name = "WikiInvalidReadOptions";
|
|
313
|
+
this.field = field;
|
|
314
|
+
this.reason = reason;
|
|
315
|
+
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
316
|
+
}
|
|
317
|
+
};
|
|
309
318
|
|
|
310
319
|
// src/db/serializedAdapter.ts
|
|
311
320
|
function isDriverError(err) {
|
|
@@ -6141,6 +6150,10 @@ After running the migration SQL, restart your application.`
|
|
|
6141
6150
|
}
|
|
6142
6151
|
};
|
|
6143
6152
|
|
|
6153
|
+
// src/utils/embedDefaults.ts
|
|
6154
|
+
var DEFAULT_MAX_EMBED_CHARS = 6e3;
|
|
6155
|
+
var EMBED_CHARS_CEILING = 16e3;
|
|
6156
|
+
|
|
6144
6157
|
// src/services/EmbeddingService.ts
|
|
6145
6158
|
var EmbeddingService = class {
|
|
6146
6159
|
constructor(db, options, entryRepo, metadataRepo) {
|
|
@@ -6188,7 +6201,9 @@ var EmbeddingService = class {
|
|
|
6188
6201
|
tagsStr = fact.tags;
|
|
6189
6202
|
}
|
|
6190
6203
|
}
|
|
6191
|
-
const
|
|
6204
|
+
const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
|
|
6205
|
+
const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
|
|
6206
|
+
const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
|
|
6192
6207
|
try {
|
|
6193
6208
|
const vector = await embedFn(text);
|
|
6194
6209
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
@@ -6299,6 +6314,63 @@ function applyTierWeight(score, entityId, sanitizedTierWeights) {
|
|
|
6299
6314
|
function shouldExposeReadMetadata(entityId) {
|
|
6300
6315
|
return Array.isArray(entityId);
|
|
6301
6316
|
}
|
|
6317
|
+
function validateTierFloors(entityIds, tierFloors, sanitizedTierWeights, includeZeroWeightEntities, maxResults) {
|
|
6318
|
+
if (tierFloors === void 0) return void 0;
|
|
6319
|
+
const known = new Set(entityIds);
|
|
6320
|
+
for (const key of Object.keys(tierFloors)) {
|
|
6321
|
+
const raw = tierFloors[key];
|
|
6322
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
6323
|
+
if (!known.has(key)) {
|
|
6324
|
+
throw new WikiInvalidReadOptions(
|
|
6325
|
+
"tierFloors",
|
|
6326
|
+
`"${key}" is not one of the entity IDs passed to read(); a floor on an unrequested entity can never be satisfied`
|
|
6327
|
+
);
|
|
6328
|
+
}
|
|
6329
|
+
}
|
|
6330
|
+
const sanitized = /* @__PURE__ */ Object.create(null);
|
|
6331
|
+
let total = 0;
|
|
6332
|
+
for (const entityId of entityIds) {
|
|
6333
|
+
const raw = tierFloors[entityId];
|
|
6334
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
6335
|
+
const floor = Math.max(0, Math.trunc(raw));
|
|
6336
|
+
if (floor > 0 && includeZeroWeightEntities !== true && sanitizedTierWeights?.[entityId] === 0) {
|
|
6337
|
+
throw new WikiInvalidReadOptions(
|
|
6338
|
+
"tierFloors",
|
|
6339
|
+
`"${entityId}" has tierWeight 0 and includeZeroWeightEntities is not set, so it is excluded from retrieval and its floor of ${floor} can never be met`
|
|
6340
|
+
);
|
|
6341
|
+
}
|
|
6342
|
+
sanitized[entityId] = floor;
|
|
6343
|
+
total += floor;
|
|
6344
|
+
}
|
|
6345
|
+
if (total > maxResults) {
|
|
6346
|
+
throw new WikiInvalidReadOptions(
|
|
6347
|
+
"tierFloors",
|
|
6348
|
+
`floors sum to ${total}, which exceeds maxResults (${maxResults})`
|
|
6349
|
+
);
|
|
6350
|
+
}
|
|
6351
|
+
return sanitized;
|
|
6352
|
+
}
|
|
6353
|
+
function selectWithFloors(sortedRows, floors, maxResults) {
|
|
6354
|
+
if (maxResults <= 0) return [];
|
|
6355
|
+
if (floors === void 0 || Object.keys(floors).length === 0) {
|
|
6356
|
+
return sortedRows.slice(0, maxResults);
|
|
6357
|
+
}
|
|
6358
|
+
const selected = /* @__PURE__ */ new Set();
|
|
6359
|
+
const taken = /* @__PURE__ */ Object.create(null);
|
|
6360
|
+
for (let i = 0; i < sortedRows.length; i++) {
|
|
6361
|
+
const entityId = sortedRows[i].entity_id;
|
|
6362
|
+
const floor = floors[entityId] ?? 0;
|
|
6363
|
+
if (floor <= 0) continue;
|
|
6364
|
+
const count = taken[entityId] ?? 0;
|
|
6365
|
+
if (count >= floor) continue;
|
|
6366
|
+
taken[entityId] = count + 1;
|
|
6367
|
+
selected.add(i);
|
|
6368
|
+
}
|
|
6369
|
+
for (let i = 0; i < sortedRows.length && selected.size < maxResults; i++) {
|
|
6370
|
+
if (!selected.has(i)) selected.add(i);
|
|
6371
|
+
}
|
|
6372
|
+
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
6373
|
+
}
|
|
6302
6374
|
|
|
6303
6375
|
// src/services/RetrievalService.ts
|
|
6304
6376
|
var RetrievalService = class {
|
|
@@ -6333,13 +6405,20 @@ var RetrievalService = class {
|
|
|
6333
6405
|
}
|
|
6334
6406
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
6335
6407
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
6408
|
+
const trimmedQuery = query.trim();
|
|
6409
|
+
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
6410
|
+
entityIds,
|
|
6411
|
+
options?.tierFloors,
|
|
6412
|
+
sanitizedTierWeights,
|
|
6413
|
+
options?.includeZeroWeightEntities,
|
|
6414
|
+
maxResults
|
|
6415
|
+
) : void 0;
|
|
6336
6416
|
const rawPreFilterLimit = options?.preFilterLimit === null ? void 0 : options?.preFilterLimit ?? config?.preFilterLimit;
|
|
6337
6417
|
const effectivePreFilterLimit = rawPreFilterLimit === void 0 ? void 0 : Number.isFinite(rawPreFilterLimit) ? Math.max(0, Math.trunc(rawPreFilterLimit)) : void 0;
|
|
6338
6418
|
const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
|
|
6339
6419
|
const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
|
|
6340
6420
|
const skipEmbed = weight === 0;
|
|
6341
6421
|
const embedFn = this.options.llmProvider.embed;
|
|
6342
|
-
const trimmedQuery = query.trim();
|
|
6343
6422
|
let facts = [];
|
|
6344
6423
|
let scoreByFactId;
|
|
6345
6424
|
if (maxResults === 0) ; else if (trimmedQuery) {
|
|
@@ -6450,13 +6529,74 @@ var RetrievalService = class {
|
|
|
6450
6529
|
access_count: metadata?.access_count ?? null
|
|
6451
6530
|
};
|
|
6452
6531
|
});
|
|
6532
|
+
const floorShortfallByEntity = /* @__PURE__ */ Object.create(null);
|
|
6533
|
+
let totalFloorShortfall = 0;
|
|
6534
|
+
if (sanitizedTierFloors) {
|
|
6535
|
+
const scoredCounts = /* @__PURE__ */ new Map();
|
|
6536
|
+
for (const s of scored) {
|
|
6537
|
+
scoredCounts.set(s.entity_id, (scoredCounts.get(s.entity_id) ?? 0) + 1);
|
|
6538
|
+
}
|
|
6539
|
+
for (const [entityId2, floor] of Object.entries(sanitizedTierFloors)) {
|
|
6540
|
+
if (floor <= 0) continue;
|
|
6541
|
+
const currentCount = scoredCounts.get(entityId2) ?? 0;
|
|
6542
|
+
const shortfall = Math.max(0, floor - currentCount);
|
|
6543
|
+
if (shortfall > 0) {
|
|
6544
|
+
floorShortfallByEntity[entityId2] = shortfall;
|
|
6545
|
+
totalFloorShortfall += shortfall;
|
|
6546
|
+
}
|
|
6547
|
+
}
|
|
6548
|
+
}
|
|
6549
|
+
const reservedIds = /* @__PURE__ */ new Set();
|
|
6550
|
+
const reserved = [];
|
|
6551
|
+
if (totalFloorShortfall > 0) {
|
|
6552
|
+
for (const [entityId2, shortfall] of Object.entries(floorShortfallByEntity)) {
|
|
6553
|
+
const entityOmitted = [];
|
|
6554
|
+
for (const row of candidateRows) {
|
|
6555
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
6556
|
+
if (row.entity_id !== entityId2) continue;
|
|
6557
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
6558
|
+
entityOmitted.push({ row, kwScore });
|
|
6559
|
+
}
|
|
6560
|
+
entityOmitted.sort((a, b) => this._compareScoredRows(
|
|
6561
|
+
{ id: a.row.id, score: a.kwScore, updated_at: a.row.updated_at, access_count: a.row.access_count },
|
|
6562
|
+
{ id: b.row.id, score: b.kwScore, updated_at: b.row.updated_at, access_count: b.row.access_count }
|
|
6563
|
+
));
|
|
6564
|
+
for (let i = 0; i < Math.min(shortfall, entityOmitted.length); i++) {
|
|
6565
|
+
reserved.push(entityOmitted[i]);
|
|
6566
|
+
reservedIds.add(entityOmitted[i].row.id);
|
|
6567
|
+
}
|
|
6568
|
+
}
|
|
6569
|
+
}
|
|
6453
6570
|
const isHybrid = weight !== void 0 && weight < 1;
|
|
6571
|
+
if (reserved.length > 0) {
|
|
6572
|
+
if (isHybrid) {
|
|
6573
|
+
for (const { row, kwScore } of reserved) {
|
|
6574
|
+
scored.push({
|
|
6575
|
+
id: row.id,
|
|
6576
|
+
entity_id: row.entity_id,
|
|
6577
|
+
score: (1 - weight) * kwScore,
|
|
6578
|
+
updated_at: row.updated_at,
|
|
6579
|
+
access_count: row.access_count
|
|
6580
|
+
});
|
|
6581
|
+
}
|
|
6582
|
+
} else {
|
|
6583
|
+
for (const { row } of reserved) {
|
|
6584
|
+
scored.push({
|
|
6585
|
+
id: row.id,
|
|
6586
|
+
entity_id: row.entity_id,
|
|
6587
|
+
score: -2,
|
|
6588
|
+
updated_at: row.updated_at,
|
|
6589
|
+
access_count: row.access_count
|
|
6590
|
+
});
|
|
6591
|
+
}
|
|
6592
|
+
}
|
|
6593
|
+
}
|
|
6454
6594
|
const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
|
|
6455
6595
|
if (maxBackfill > 0) {
|
|
6456
6596
|
if (isHybrid) {
|
|
6457
6597
|
const topK = [];
|
|
6458
6598
|
for (const row of candidateRows) {
|
|
6459
|
-
if (scoredIds.has(row.id)) continue;
|
|
6599
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
6460
6600
|
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
6461
6601
|
const candidate = { row, kwScore };
|
|
6462
6602
|
if (topK.length < maxBackfill) {
|
|
@@ -6536,7 +6676,7 @@ var RetrievalService = class {
|
|
|
6536
6676
|
} else {
|
|
6537
6677
|
const omitted = [];
|
|
6538
6678
|
for (const row of candidateRows) {
|
|
6539
|
-
if (scoredIds.has(row.id)) continue;
|
|
6679
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
6540
6680
|
omitted.push({ id: row.id, entity_id: row.entity_id, score: -2, updated_at: row.updated_at, access_count: row.access_count });
|
|
6541
6681
|
}
|
|
6542
6682
|
if (omitted.length > 0) {
|
|
@@ -6584,8 +6724,11 @@ var RetrievalService = class {
|
|
|
6584
6724
|
// read() re-sorts after applying tier weights
|
|
6585
6725
|
});
|
|
6586
6726
|
} else if (policy === "keyword") {
|
|
6587
|
-
const
|
|
6588
|
-
const
|
|
6727
|
+
const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
6728
|
+
const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
6729
|
+
const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
|
|
6730
|
+
const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
|
|
6731
|
+
const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
|
|
6589
6732
|
const topResultIds = new Set(topResults.map((r) => r.id));
|
|
6590
6733
|
const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
|
|
6591
6734
|
scored = topResults.map((result) => {
|
|
@@ -6610,7 +6753,7 @@ var RetrievalService = class {
|
|
|
6610
6753
|
}
|
|
6611
6754
|
}
|
|
6612
6755
|
} else {
|
|
6613
|
-
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1);
|
|
6756
|
+
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1) || sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
6614
6757
|
scored = await this.searchService.rankSemantic({
|
|
6615
6758
|
entityId: entityCacheKey,
|
|
6616
6759
|
queryVec,
|
|
@@ -6629,7 +6772,7 @@ var RetrievalService = class {
|
|
|
6629
6772
|
score: applyTierWeight(row.score, row.entity_id, sanitizedTierWeights)
|
|
6630
6773
|
}));
|
|
6631
6774
|
this._tieBreakSort(scored);
|
|
6632
|
-
const selectedScored = scored
|
|
6775
|
+
const selectedScored = selectWithFloors(scored, sanitizedTierFloors, maxResults);
|
|
6633
6776
|
const topIds = selectedScored.map((s) => s.id);
|
|
6634
6777
|
if (exposeMetadata && trimmedQuery) {
|
|
6635
6778
|
scoreByFactId = new Map(selectedScored.map((s) => [s.id, Number.isFinite(s.score) ? s.score : 0]));
|
|
@@ -6684,7 +6827,8 @@ var RetrievalService = class {
|
|
|
6684
6827
|
}
|
|
6685
6828
|
}
|
|
6686
6829
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
6687
|
-
const
|
|
6830
|
+
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
6831
|
+
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
6688
6832
|
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
|
|
6689
6833
|
const candidates = results.map((r) => ({
|
|
6690
6834
|
id: r.id,
|
|
@@ -6694,7 +6838,7 @@ var RetrievalService = class {
|
|
|
6694
6838
|
access_count: null
|
|
6695
6839
|
}));
|
|
6696
6840
|
this._tieBreakSort(candidates);
|
|
6697
|
-
const topCandidates = candidates
|
|
6841
|
+
const topCandidates = selectWithFloors(candidates, sanitizedTierFloors, maxResults);
|
|
6698
6842
|
const topIds = topCandidates.map((c) => c.id);
|
|
6699
6843
|
if (topIds.length > 0) {
|
|
6700
6844
|
facts = await this._hydrateFactsByIds(topIds, entityIds);
|
|
@@ -6724,6 +6868,7 @@ var RetrievalService = class {
|
|
|
6724
6868
|
if (exposeMetadata) {
|
|
6725
6869
|
bundle.metadata = { query, entityIds };
|
|
6726
6870
|
if (sanitizedTierWeights && Object.keys(sanitizedTierWeights).length > 0) bundle.metadata.tierWeights = sanitizedTierWeights;
|
|
6871
|
+
if (sanitizedTierFloors && Object.keys(sanitizedTierFloors).length > 0) bundle.metadata.tierFloors = sanitizedTierFloors;
|
|
6727
6872
|
if (factScores && Object.keys(factScores).length > 0) bundle.factScores = factScores;
|
|
6728
6873
|
}
|
|
6729
6874
|
return bundle;
|
|
@@ -8722,6 +8867,8 @@ function createWiki(db, options) {
|
|
|
8722
8867
|
exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
|
|
8723
8868
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
8724
8869
|
exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
|
|
8870
|
+
exports.DEFAULT_MAX_EMBED_CHARS = DEFAULT_MAX_EMBED_CHARS;
|
|
8871
|
+
exports.EMBED_CHARS_CEILING = EMBED_CHARS_CEILING;
|
|
8725
8872
|
exports.HEAL_ANCHORS_PER_CANDIDATE = HEAL_ANCHORS_PER_CANDIDATE;
|
|
8726
8873
|
exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;
|
|
8727
8874
|
exports.HEAL_MAX_FACT_BODY_CHARS_L3 = HEAL_MAX_FACT_BODY_CHARS_L3;
|
|
@@ -8737,6 +8884,7 @@ exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
|
8737
8884
|
exports.WikiBusyError = WikiBusyError;
|
|
8738
8885
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
8739
8886
|
exports.WikiIngestEmptyError = WikiIngestEmptyError;
|
|
8887
|
+
exports.WikiInvalidReadOptions = WikiInvalidReadOptions;
|
|
8740
8888
|
exports.WikiMemory = WikiMemory;
|
|
8741
8889
|
exports.WikiParseError = WikiParseError;
|
|
8742
8890
|
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|