@equationalapplications/core-llm-wiki 6.2.0 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/dist/{chunk-YEESP6J2.mjs → chunk-DPFM33WV.mjs} +175 -16
- package/dist/chunk-DPFM33WV.mjs.map +1 -0
- package/dist/index.d.mts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +231 -18
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +58 -7
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-D0m2V9-E.d.mts → testing-DWpAgNLt.d.mts} +87 -2
- package/dist/{testing-D0m2V9-E.d.ts → testing-DWpAgNLt.d.ts} +87 -2
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +155 -10
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-YEESP6J2.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DWpAgNLt.mjs';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiInvalidReadOptions, Z as WikiMemoryTestAccess, _ as WikiOutboxEvent, $ as WikiParseError, a0 as WikiSourceRefHashCollision, a1 as WikiStrictOntologyViolation, a2 as WikiTask, a3 as WikiTransactionError } from './testing-DWpAgNLt.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -119,6 +119,18 @@ declare const DEFAULT_MAX_CHUNK_LENGTH = 12000;
|
|
|
119
119
|
*/
|
|
120
120
|
declare const DEFAULT_CHUNK_OVERLAP = 400;
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Default clamp on embed input length, in characters (~1,500 tokens).
|
|
124
|
+
* Chosen to fit common embedding windows including nomic-embed-text's 2,048 tokens.
|
|
125
|
+
*/
|
|
126
|
+
declare const DEFAULT_MAX_EMBED_CHARS = 6000;
|
|
127
|
+
/**
|
|
128
|
+
* Hard upper bound on embed input length, in characters.
|
|
129
|
+
* An unbounded-input guard from `5e04bad fix(core): security hardening phase 2`.
|
|
130
|
+
* Configuration may lower the effective limit but must never raise it past this.
|
|
131
|
+
*/
|
|
132
|
+
declare const EMBED_CHARS_CEILING = 16000;
|
|
133
|
+
|
|
122
134
|
interface LibrarianOptions {
|
|
123
135
|
/** If provided, replaces the default Librarian system instructions. */
|
|
124
136
|
systemPrompt?: string;
|
|
@@ -145,4 +157,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
145
157
|
|
|
146
158
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
147
159
|
|
|
148
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
160
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-DWpAgNLt.js';
|
|
2
|
+
export { C as ChunkFailure, D as DegradedRecord, E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, I as IngestDocumentResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, J as WikiCheckpoint, K as WikiConfig, N as WikiDuplicateHashError, Q as WikiEdge, T as WikiEvent, U as WikiFact, X as WikiIngestEmptyError, Y as WikiInvalidReadOptions, Z as WikiMemoryTestAccess, _ as WikiOutboxEvent, $ as WikiParseError, a0 as WikiSourceRefHashCollision, a1 as WikiStrictOntologyViolation, a2 as WikiTask, a3 as WikiTransactionError } from './testing-DWpAgNLt.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -119,6 +119,18 @@ declare const DEFAULT_MAX_CHUNK_LENGTH = 12000;
|
|
|
119
119
|
*/
|
|
120
120
|
declare const DEFAULT_CHUNK_OVERLAP = 400;
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Default clamp on embed input length, in characters (~1,500 tokens).
|
|
124
|
+
* Chosen to fit common embedding windows including nomic-embed-text's 2,048 tokens.
|
|
125
|
+
*/
|
|
126
|
+
declare const DEFAULT_MAX_EMBED_CHARS = 6000;
|
|
127
|
+
/**
|
|
128
|
+
* Hard upper bound on embed input length, in characters.
|
|
129
|
+
* An unbounded-input guard from `5e04bad fix(core): security hardening phase 2`.
|
|
130
|
+
* Configuration may lower the effective limit but must never raise it past this.
|
|
131
|
+
*/
|
|
132
|
+
declare const EMBED_CHARS_CEILING = 16000;
|
|
133
|
+
|
|
122
134
|
interface LibrarianOptions {
|
|
123
135
|
/** If provided, replaces the default Librarian system instructions. */
|
|
124
136
|
systemPrompt?: string;
|
|
@@ -145,4 +157,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
|
|
|
145
157
|
|
|
146
158
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
147
159
|
|
|
148
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
|
160
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, FormatContextOptions, type FormatOkfBundleOptions, FormattedMemoryDump, GraphNeighborhood, HEAL_ANCHORS_PER_CANDIDATE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfFormatProfile, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, chunkText, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, safeSlice, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.js
CHANGED
|
@@ -306,6 +306,15 @@ var WikiIngestEmptyError = class extends Error {
|
|
|
306
306
|
this.chunks = params.chunks;
|
|
307
307
|
}
|
|
308
308
|
};
|
|
309
|
+
var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
310
|
+
constructor(field, reason) {
|
|
311
|
+
super(`Invalid ReadOptions.${field}: ${reason}`);
|
|
312
|
+
this.name = "WikiInvalidReadOptions";
|
|
313
|
+
this.field = field;
|
|
314
|
+
this.reason = reason;
|
|
315
|
+
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
316
|
+
}
|
|
317
|
+
};
|
|
309
318
|
|
|
310
319
|
// src/db/serializedAdapter.ts
|
|
311
320
|
function isDriverError(err) {
|
|
@@ -2784,15 +2793,29 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2784
2793
|
manifest
|
|
2785
2794
|
};
|
|
2786
2795
|
}
|
|
2787
|
-
|
|
2796
|
+
/**
|
|
2797
|
+
* Write an entity's manifest.
|
|
2798
|
+
*
|
|
2799
|
+
* `opts.ifAbsent` selects the conflict behaviour:
|
|
2800
|
+
* - omitted / `false` — upsert. An existing row is replaced.
|
|
2801
|
+
* - `true` — create-if-absent. An existing row is left exactly as it is.
|
|
2802
|
+
*
|
|
2803
|
+
* Returns whether this call actually wrote a row. Under `ifAbsent` a `false`
|
|
2804
|
+
* result means another writer got there first; the check and the write are one
|
|
2805
|
+
* statement, so there is no window between them.
|
|
2806
|
+
*/
|
|
2807
|
+
async setManifest(entityId, data, tx, opts) {
|
|
2788
2808
|
validateManifest(data.manifest);
|
|
2789
2809
|
const executor = this.getExecutor(tx);
|
|
2790
|
-
|
|
2810
|
+
const conflictClause = opts?.ifAbsent ? "ON CONFLICT(entity_id) DO NOTHING" : `ON CONFLICT(entity_id) DO UPDATE SET
|
|
2811
|
+
mode = excluded.mode, manifest_json = excluded.manifest_json, updated_at = excluded.updated_at`;
|
|
2812
|
+
const result = await executor.runAsync(
|
|
2791
2813
|
`INSERT INTO ${this.prefix}entity_manifests (entity_id, mode, manifest_json, updated_at)
|
|
2792
2814
|
VALUES (?, ?, ?, ?)
|
|
2793
|
-
|
|
2815
|
+
${conflictClause}`,
|
|
2794
2816
|
[entityId, data.mode, JSON.stringify(data.manifest), Date.now()]
|
|
2795
2817
|
);
|
|
2818
|
+
return result.changes > 0;
|
|
2796
2819
|
}
|
|
2797
2820
|
async mergeManifestUpdates(entityId, updates, tx) {
|
|
2798
2821
|
const current = await this.getManifest(entityId, tx) ?? {
|
|
@@ -6127,6 +6150,10 @@ After running the migration SQL, restart your application.`
|
|
|
6127
6150
|
}
|
|
6128
6151
|
};
|
|
6129
6152
|
|
|
6153
|
+
// src/utils/embedDefaults.ts
|
|
6154
|
+
var DEFAULT_MAX_EMBED_CHARS = 6e3;
|
|
6155
|
+
var EMBED_CHARS_CEILING = 16e3;
|
|
6156
|
+
|
|
6130
6157
|
// src/services/EmbeddingService.ts
|
|
6131
6158
|
var EmbeddingService = class {
|
|
6132
6159
|
constructor(db, options, entryRepo, metadataRepo) {
|
|
@@ -6174,7 +6201,9 @@ var EmbeddingService = class {
|
|
|
6174
6201
|
tagsStr = fact.tags;
|
|
6175
6202
|
}
|
|
6176
6203
|
}
|
|
6177
|
-
const
|
|
6204
|
+
const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
|
|
6205
|
+
const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
|
|
6206
|
+
const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
|
|
6178
6207
|
try {
|
|
6179
6208
|
const vector = await embedFn(text);
|
|
6180
6209
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
@@ -6285,6 +6314,63 @@ function applyTierWeight(score, entityId, sanitizedTierWeights) {
|
|
|
6285
6314
|
function shouldExposeReadMetadata(entityId) {
|
|
6286
6315
|
return Array.isArray(entityId);
|
|
6287
6316
|
}
|
|
6317
|
+
function validateTierFloors(entityIds, tierFloors, sanitizedTierWeights, includeZeroWeightEntities, maxResults) {
|
|
6318
|
+
if (tierFloors === void 0) return void 0;
|
|
6319
|
+
const known = new Set(entityIds);
|
|
6320
|
+
for (const key of Object.keys(tierFloors)) {
|
|
6321
|
+
const raw = tierFloors[key];
|
|
6322
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
6323
|
+
if (!known.has(key)) {
|
|
6324
|
+
throw new WikiInvalidReadOptions(
|
|
6325
|
+
"tierFloors",
|
|
6326
|
+
`"${key}" is not one of the entity IDs passed to read(); a floor on an unrequested entity can never be satisfied`
|
|
6327
|
+
);
|
|
6328
|
+
}
|
|
6329
|
+
}
|
|
6330
|
+
const sanitized = /* @__PURE__ */ Object.create(null);
|
|
6331
|
+
let total = 0;
|
|
6332
|
+
for (const entityId of entityIds) {
|
|
6333
|
+
const raw = tierFloors[entityId];
|
|
6334
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
6335
|
+
const floor = Math.max(0, Math.trunc(raw));
|
|
6336
|
+
if (floor > 0 && includeZeroWeightEntities !== true && sanitizedTierWeights?.[entityId] === 0) {
|
|
6337
|
+
throw new WikiInvalidReadOptions(
|
|
6338
|
+
"tierFloors",
|
|
6339
|
+
`"${entityId}" has tierWeight 0 and includeZeroWeightEntities is not set, so it is excluded from retrieval and its floor of ${floor} can never be met`
|
|
6340
|
+
);
|
|
6341
|
+
}
|
|
6342
|
+
sanitized[entityId] = floor;
|
|
6343
|
+
total += floor;
|
|
6344
|
+
}
|
|
6345
|
+
if (total > maxResults) {
|
|
6346
|
+
throw new WikiInvalidReadOptions(
|
|
6347
|
+
"tierFloors",
|
|
6348
|
+
`floors sum to ${total}, which exceeds maxResults (${maxResults})`
|
|
6349
|
+
);
|
|
6350
|
+
}
|
|
6351
|
+
return sanitized;
|
|
6352
|
+
}
|
|
6353
|
+
function selectWithFloors(sortedRows, floors, maxResults) {
|
|
6354
|
+
if (maxResults <= 0) return [];
|
|
6355
|
+
if (floors === void 0 || Object.keys(floors).length === 0) {
|
|
6356
|
+
return sortedRows.slice(0, maxResults);
|
|
6357
|
+
}
|
|
6358
|
+
const selected = /* @__PURE__ */ new Set();
|
|
6359
|
+
const taken = /* @__PURE__ */ Object.create(null);
|
|
6360
|
+
for (let i = 0; i < sortedRows.length; i++) {
|
|
6361
|
+
const entityId = sortedRows[i].entity_id;
|
|
6362
|
+
const floor = floors[entityId] ?? 0;
|
|
6363
|
+
if (floor <= 0) continue;
|
|
6364
|
+
const count = taken[entityId] ?? 0;
|
|
6365
|
+
if (count >= floor) continue;
|
|
6366
|
+
taken[entityId] = count + 1;
|
|
6367
|
+
selected.add(i);
|
|
6368
|
+
}
|
|
6369
|
+
for (let i = 0; i < sortedRows.length && selected.size < maxResults; i++) {
|
|
6370
|
+
if (!selected.has(i)) selected.add(i);
|
|
6371
|
+
}
|
|
6372
|
+
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
6373
|
+
}
|
|
6288
6374
|
|
|
6289
6375
|
// src/services/RetrievalService.ts
|
|
6290
6376
|
var RetrievalService = class {
|
|
@@ -6319,13 +6405,20 @@ var RetrievalService = class {
|
|
|
6319
6405
|
}
|
|
6320
6406
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
6321
6407
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
6408
|
+
const trimmedQuery = query.trim();
|
|
6409
|
+
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
6410
|
+
entityIds,
|
|
6411
|
+
options?.tierFloors,
|
|
6412
|
+
sanitizedTierWeights,
|
|
6413
|
+
options?.includeZeroWeightEntities,
|
|
6414
|
+
maxResults
|
|
6415
|
+
) : void 0;
|
|
6322
6416
|
const rawPreFilterLimit = options?.preFilterLimit === null ? void 0 : options?.preFilterLimit ?? config?.preFilterLimit;
|
|
6323
6417
|
const effectivePreFilterLimit = rawPreFilterLimit === void 0 ? void 0 : Number.isFinite(rawPreFilterLimit) ? Math.max(0, Math.trunc(rawPreFilterLimit)) : void 0;
|
|
6324
6418
|
const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
|
|
6325
6419
|
const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
|
|
6326
6420
|
const skipEmbed = weight === 0;
|
|
6327
6421
|
const embedFn = this.options.llmProvider.embed;
|
|
6328
|
-
const trimmedQuery = query.trim();
|
|
6329
6422
|
let facts = [];
|
|
6330
6423
|
let scoreByFactId;
|
|
6331
6424
|
if (maxResults === 0) ; else if (trimmedQuery) {
|
|
@@ -6436,13 +6529,74 @@ var RetrievalService = class {
|
|
|
6436
6529
|
access_count: metadata?.access_count ?? null
|
|
6437
6530
|
};
|
|
6438
6531
|
});
|
|
6532
|
+
const floorShortfallByEntity = /* @__PURE__ */ Object.create(null);
|
|
6533
|
+
let totalFloorShortfall = 0;
|
|
6534
|
+
if (sanitizedTierFloors) {
|
|
6535
|
+
const scoredCounts = /* @__PURE__ */ new Map();
|
|
6536
|
+
for (const s of scored) {
|
|
6537
|
+
scoredCounts.set(s.entity_id, (scoredCounts.get(s.entity_id) ?? 0) + 1);
|
|
6538
|
+
}
|
|
6539
|
+
for (const [entityId2, floor] of Object.entries(sanitizedTierFloors)) {
|
|
6540
|
+
if (floor <= 0) continue;
|
|
6541
|
+
const currentCount = scoredCounts.get(entityId2) ?? 0;
|
|
6542
|
+
const shortfall = Math.max(0, floor - currentCount);
|
|
6543
|
+
if (shortfall > 0) {
|
|
6544
|
+
floorShortfallByEntity[entityId2] = shortfall;
|
|
6545
|
+
totalFloorShortfall += shortfall;
|
|
6546
|
+
}
|
|
6547
|
+
}
|
|
6548
|
+
}
|
|
6549
|
+
const reservedIds = /* @__PURE__ */ new Set();
|
|
6550
|
+
const reserved = [];
|
|
6551
|
+
if (totalFloorShortfall > 0) {
|
|
6552
|
+
for (const [entityId2, shortfall] of Object.entries(floorShortfallByEntity)) {
|
|
6553
|
+
const entityOmitted = [];
|
|
6554
|
+
for (const row of candidateRows) {
|
|
6555
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
6556
|
+
if (row.entity_id !== entityId2) continue;
|
|
6557
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
6558
|
+
entityOmitted.push({ row, kwScore });
|
|
6559
|
+
}
|
|
6560
|
+
entityOmitted.sort((a, b) => this._compareScoredRows(
|
|
6561
|
+
{ id: a.row.id, score: a.kwScore, updated_at: a.row.updated_at, access_count: a.row.access_count },
|
|
6562
|
+
{ id: b.row.id, score: b.kwScore, updated_at: b.row.updated_at, access_count: b.row.access_count }
|
|
6563
|
+
));
|
|
6564
|
+
for (let i = 0; i < Math.min(shortfall, entityOmitted.length); i++) {
|
|
6565
|
+
reserved.push(entityOmitted[i]);
|
|
6566
|
+
reservedIds.add(entityOmitted[i].row.id);
|
|
6567
|
+
}
|
|
6568
|
+
}
|
|
6569
|
+
}
|
|
6439
6570
|
const isHybrid = weight !== void 0 && weight < 1;
|
|
6571
|
+
if (reserved.length > 0) {
|
|
6572
|
+
if (isHybrid) {
|
|
6573
|
+
for (const { row, kwScore } of reserved) {
|
|
6574
|
+
scored.push({
|
|
6575
|
+
id: row.id,
|
|
6576
|
+
entity_id: row.entity_id,
|
|
6577
|
+
score: (1 - weight) * kwScore,
|
|
6578
|
+
updated_at: row.updated_at,
|
|
6579
|
+
access_count: row.access_count
|
|
6580
|
+
});
|
|
6581
|
+
}
|
|
6582
|
+
} else {
|
|
6583
|
+
for (const { row } of reserved) {
|
|
6584
|
+
scored.push({
|
|
6585
|
+
id: row.id,
|
|
6586
|
+
entity_id: row.entity_id,
|
|
6587
|
+
score: -2,
|
|
6588
|
+
updated_at: row.updated_at,
|
|
6589
|
+
access_count: row.access_count
|
|
6590
|
+
});
|
|
6591
|
+
}
|
|
6592
|
+
}
|
|
6593
|
+
}
|
|
6440
6594
|
const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
|
|
6441
6595
|
if (maxBackfill > 0) {
|
|
6442
6596
|
if (isHybrid) {
|
|
6443
6597
|
const topK = [];
|
|
6444
6598
|
for (const row of candidateRows) {
|
|
6445
|
-
if (scoredIds.has(row.id)) continue;
|
|
6599
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
6446
6600
|
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
6447
6601
|
const candidate = { row, kwScore };
|
|
6448
6602
|
if (topK.length < maxBackfill) {
|
|
@@ -6522,7 +6676,7 @@ var RetrievalService = class {
|
|
|
6522
6676
|
} else {
|
|
6523
6677
|
const omitted = [];
|
|
6524
6678
|
for (const row of candidateRows) {
|
|
6525
|
-
if (scoredIds.has(row.id)) continue;
|
|
6679
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
6526
6680
|
omitted.push({ id: row.id, entity_id: row.entity_id, score: -2, updated_at: row.updated_at, access_count: row.access_count });
|
|
6527
6681
|
}
|
|
6528
6682
|
if (omitted.length > 0) {
|
|
@@ -6570,8 +6724,11 @@ var RetrievalService = class {
|
|
|
6570
6724
|
// read() re-sorts after applying tier weights
|
|
6571
6725
|
});
|
|
6572
6726
|
} else if (policy === "keyword") {
|
|
6573
|
-
const
|
|
6574
|
-
const
|
|
6727
|
+
const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
6728
|
+
const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
6729
|
+
const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
|
|
6730
|
+
const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
|
|
6731
|
+
const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
|
|
6575
6732
|
const topResultIds = new Set(topResults.map((r) => r.id));
|
|
6576
6733
|
const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
|
|
6577
6734
|
scored = topResults.map((result) => {
|
|
@@ -6596,7 +6753,7 @@ var RetrievalService = class {
|
|
|
6596
6753
|
}
|
|
6597
6754
|
}
|
|
6598
6755
|
} else {
|
|
6599
|
-
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1);
|
|
6756
|
+
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1) || sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
6600
6757
|
scored = await this.searchService.rankSemantic({
|
|
6601
6758
|
entityId: entityCacheKey,
|
|
6602
6759
|
queryVec,
|
|
@@ -6615,7 +6772,7 @@ var RetrievalService = class {
|
|
|
6615
6772
|
score: applyTierWeight(row.score, row.entity_id, sanitizedTierWeights)
|
|
6616
6773
|
}));
|
|
6617
6774
|
this._tieBreakSort(scored);
|
|
6618
|
-
const selectedScored = scored
|
|
6775
|
+
const selectedScored = selectWithFloors(scored, sanitizedTierFloors, maxResults);
|
|
6619
6776
|
const topIds = selectedScored.map((s) => s.id);
|
|
6620
6777
|
if (exposeMetadata && trimmedQuery) {
|
|
6621
6778
|
scoreByFactId = new Map(selectedScored.map((s) => [s.id, Number.isFinite(s.score) ? s.score : 0]));
|
|
@@ -6670,7 +6827,8 @@ var RetrievalService = class {
|
|
|
6670
6827
|
}
|
|
6671
6828
|
}
|
|
6672
6829
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
6673
|
-
const
|
|
6830
|
+
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
6831
|
+
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
6674
6832
|
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
|
|
6675
6833
|
const candidates = results.map((r) => ({
|
|
6676
6834
|
id: r.id,
|
|
@@ -6680,7 +6838,7 @@ var RetrievalService = class {
|
|
|
6680
6838
|
access_count: null
|
|
6681
6839
|
}));
|
|
6682
6840
|
this._tieBreakSort(candidates);
|
|
6683
|
-
const topCandidates = candidates
|
|
6841
|
+
const topCandidates = selectWithFloors(candidates, sanitizedTierFloors, maxResults);
|
|
6684
6842
|
const topIds = topCandidates.map((c) => c.id);
|
|
6685
6843
|
if (topIds.length > 0) {
|
|
6686
6844
|
facts = await this._hydrateFactsByIds(topIds, entityIds);
|
|
@@ -6710,6 +6868,7 @@ var RetrievalService = class {
|
|
|
6710
6868
|
if (exposeMetadata) {
|
|
6711
6869
|
bundle.metadata = { query, entityIds };
|
|
6712
6870
|
if (sanitizedTierWeights && Object.keys(sanitizedTierWeights).length > 0) bundle.metadata.tierWeights = sanitizedTierWeights;
|
|
6871
|
+
if (sanitizedTierFloors && Object.keys(sanitizedTierFloors).length > 0) bundle.metadata.tierFloors = sanitizedTierFloors;
|
|
6713
6872
|
if (factScores && Object.keys(factScores).length > 0) bundle.factScores = factScores;
|
|
6714
6873
|
}
|
|
6715
6874
|
return bundle;
|
|
@@ -7701,13 +7860,64 @@ var WikiMemory = class {
|
|
|
7701
7860
|
* Seeds or replaces an entity's ontology manifest and optional mode override.
|
|
7702
7861
|
* Validates manifest invariants (unique type slugs, edge endpoints reference node types).
|
|
7703
7862
|
* Invalidates the in-memory ontology cache for this entity.
|
|
7863
|
+
*
|
|
7864
|
+
* Delegates to `setOntologyManifests` so there is a single write path. The
|
|
7865
|
+
* observable contract is unchanged: same upsert semantics, same cache
|
|
7866
|
+
* invalidation, same `void` result.
|
|
7704
7867
|
*/
|
|
7705
7868
|
async setOntologyManifest(entityId, manifest, options) {
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
|
|
7709
|
-
|
|
7710
|
-
|
|
7869
|
+
await this.setOntologyManifests([{ entityId, manifest, mode: options?.mode }]);
|
|
7870
|
+
}
|
|
7871
|
+
/**
|
|
7872
|
+
* Write several entities' ontology manifests in a single transaction.
|
|
7873
|
+
*
|
|
7874
|
+
* All succeed or none do. The transaction is opened on this instance's
|
|
7875
|
+
* serialized adapter — callers pass data, never a transaction handle, because
|
|
7876
|
+
* a consumer holds the unwrapped adapter and a transaction opened on it would
|
|
7877
|
+
* bypass the serialization mutex applied in the constructor.
|
|
7878
|
+
*
|
|
7879
|
+
* Returns which entities were written. `skipped` is only ever non-empty under
|
|
7880
|
+
* `opts.ifAbsent`.
|
|
7881
|
+
*
|
|
7882
|
+
* Sharp edge: `ifAbsent` checks for a PERSISTED row, not for an effective
|
|
7883
|
+
* manifest. An entity whose manifest currently comes from
|
|
7884
|
+
* `WikiConfig.ontology.seedManifests` has no row until the seed is
|
|
7885
|
+
* materialized on first access that supplies a transaction (in practice, an
|
|
7886
|
+
* ingest). Until then `ifAbsent` treats the entity as absent, writes the new
|
|
7887
|
+
* manifest over the configured seed, and reports it in `written`; the same
|
|
7888
|
+
* call after the seed has been materialized reports it in `skipped`. See the
|
|
7889
|
+
* `ifAbsent` notes in the package README.
|
|
7890
|
+
*/
|
|
7891
|
+
async setOntologyManifests(entries, opts) {
|
|
7892
|
+
if (entries.length === 0) return { written: [], skipped: [] };
|
|
7893
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7894
|
+
for (const entry of entries) {
|
|
7895
|
+
if (seen.has(entry.entityId)) {
|
|
7896
|
+
throw new Error(`Duplicate entityId in batch: ${entry.entityId}`);
|
|
7897
|
+
}
|
|
7898
|
+
seen.add(entry.entityId);
|
|
7899
|
+
}
|
|
7900
|
+
for (const entry of entries) {
|
|
7901
|
+
validateManifest(entry.manifest);
|
|
7902
|
+
}
|
|
7903
|
+
const written = [];
|
|
7904
|
+
const skipped = [];
|
|
7905
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
7906
|
+
for (const entry of entries) {
|
|
7907
|
+
const mode = entry.mode ?? this.ontologyService.resolveMode();
|
|
7908
|
+
const wrote = await this.metadataRepo.setManifest(
|
|
7909
|
+
entry.entityId,
|
|
7910
|
+
{ mode, manifest: entry.manifest },
|
|
7911
|
+
tx,
|
|
7912
|
+
{ ifAbsent: opts?.ifAbsent === true }
|
|
7913
|
+
);
|
|
7914
|
+
(wrote ? written : skipped).push(entry.entityId);
|
|
7915
|
+
}
|
|
7916
|
+
});
|
|
7917
|
+
for (const entry of entries) {
|
|
7918
|
+
this.ontologyService.invalidateCache(entry.entityId);
|
|
7919
|
+
}
|
|
7920
|
+
return { written, skipped };
|
|
7711
7921
|
}
|
|
7712
7922
|
/** Append a verification event to a fact. Does NOT touch `updated_at`. */
|
|
7713
7923
|
async writeOkfTrust(entryId, entityId, verified) {
|
|
@@ -8657,6 +8867,8 @@ function createWiki(db, options) {
|
|
|
8657
8867
|
exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
|
|
8658
8868
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
8659
8869
|
exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
|
|
8870
|
+
exports.DEFAULT_MAX_EMBED_CHARS = DEFAULT_MAX_EMBED_CHARS;
|
|
8871
|
+
exports.EMBED_CHARS_CEILING = EMBED_CHARS_CEILING;
|
|
8660
8872
|
exports.HEAL_ANCHORS_PER_CANDIDATE = HEAL_ANCHORS_PER_CANDIDATE;
|
|
8661
8873
|
exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;
|
|
8662
8874
|
exports.HEAL_MAX_FACT_BODY_CHARS_L3 = HEAL_MAX_FACT_BODY_CHARS_L3;
|
|
@@ -8672,6 +8884,7 @@ exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
|
8672
8884
|
exports.WikiBusyError = WikiBusyError;
|
|
8673
8885
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
8674
8886
|
exports.WikiIngestEmptyError = WikiIngestEmptyError;
|
|
8887
|
+
exports.WikiInvalidReadOptions = WikiInvalidReadOptions;
|
|
8675
8888
|
exports.WikiMemory = WikiMemory;
|
|
8676
8889
|
exports.WikiParseError = WikiParseError;
|
|
8677
8890
|
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|