@equationalapplications/core-llm-wiki 6.2.0 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/dist/{chunk-YEESP6J2.mjs → chunk-DPFM33WV.mjs} +175 -16
- package/dist/chunk-DPFM33WV.mjs.map +1 -0
- package/dist/index.d.mts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +231 -18
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +58 -7
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-D0m2V9-E.d.mts → testing-DWpAgNLt.d.mts} +87 -2
- package/dist/{testing-D0m2V9-E.d.ts → testing-DWpAgNLt.d.ts} +87 -2
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +155 -10
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-YEESP6J2.mjs.map +0 -1
|
@@ -195,6 +195,16 @@ interface WikiConfig {
|
|
|
195
195
|
maxChunkLength?: number;
|
|
196
196
|
chunkOverlap?: number;
|
|
197
197
|
chunkConcurrency?: number;
|
|
198
|
+
/**
|
|
199
|
+
* Maximum characters of `title + body + tags` passed to `llmProvider.embed()`
|
|
200
|
+
* when embedding a fact. Longer input is clipped, not rejected.
|
|
201
|
+
*
|
|
202
|
+
* Default 6000 (~1,500 tokens), which fits common embedding windows including
|
|
203
|
+
* nomic-embed-text's 2,048 tokens with headroom. Values above the 16,000
|
|
204
|
+
* hard ceiling are clamped to it; the ceiling is a security control and
|
|
205
|
+
* configuration cannot raise it.
|
|
206
|
+
*/
|
|
207
|
+
maxEmbedChars?: number;
|
|
198
208
|
/**
|
|
199
209
|
* Max MiniSearch candidates passed to cosine scoring.
|
|
200
210
|
* When set, MiniSearch pre-filters before the cosine scan.
|
|
@@ -250,6 +260,23 @@ interface ReadOptions {
|
|
|
250
260
|
* Only meaningful when `entityId` is an array; ignored for single-string calls.
|
|
251
261
|
*/
|
|
252
262
|
includeZeroWeightEntities?: boolean;
|
|
263
|
+
/**
|
|
264
|
+
* Minimum number of results to retain from each named entity before the global
|
|
265
|
+
* `maxResults` cut. Applied after scoring: the top-N scored results from each
|
|
266
|
+
* named entity are reserved, then remaining slots fill by score as usual.
|
|
267
|
+
*
|
|
268
|
+
* An entity with fewer than N matching results contributes what it has — this is
|
|
269
|
+
* not an error. Floors cannot resurrect a result excluded by `preFilterLimit`.
|
|
270
|
+
*
|
|
271
|
+
* Only meaningful when `entityId` is an array; ignored for single-string calls.
|
|
272
|
+
* Ignored when `query` is empty — that path uses recency ordering and ignores
|
|
273
|
+
* `tierWeights` as well.
|
|
274
|
+
*
|
|
275
|
+
* Throws `WikiInvalidReadOptions` when a floor cannot be satisfied by
|
|
276
|
+
* construction: floors summing above `maxResults`, a floor on an entity excluded
|
|
277
|
+
* by `tierWeights: 0`, or a floor keyed to an entity not in `entityId`.
|
|
278
|
+
*/
|
|
279
|
+
tierFloors?: Record<string, number>;
|
|
253
280
|
}
|
|
254
281
|
interface WikiFact {
|
|
255
282
|
id: string;
|
|
@@ -588,6 +615,8 @@ interface MemoryBundle {
|
|
|
588
615
|
query: string;
|
|
589
616
|
entityIds: string[];
|
|
590
617
|
tierWeights?: Record<string, number>;
|
|
618
|
+
/** Sanitized per-entity result floors, when `tierFloors` was supplied and non-empty. */
|
|
619
|
+
tierFloors?: Record<string, number>;
|
|
591
620
|
};
|
|
592
621
|
}
|
|
593
622
|
interface MemoryDump {
|
|
@@ -787,6 +816,15 @@ declare class WikiIngestEmptyError extends Error {
|
|
|
787
816
|
chunks: number;
|
|
788
817
|
});
|
|
789
818
|
}
|
|
819
|
+
/**
|
|
820
|
+
* Thrown by `read()` when `ReadOptions` contains a request that cannot be
|
|
821
|
+
* satisfied by construction — as opposed to one that merely finds no data.
|
|
822
|
+
*/
|
|
823
|
+
declare class WikiInvalidReadOptions extends Error {
|
|
824
|
+
readonly field: string;
|
|
825
|
+
readonly reason: string;
|
|
826
|
+
constructor(field: string, reason: string);
|
|
827
|
+
}
|
|
790
828
|
|
|
791
829
|
interface WikiOutboxEvent<T = unknown> {
|
|
792
830
|
id: string;
|
|
@@ -1244,10 +1282,23 @@ declare class MetadataRepository extends BaseRepository {
|
|
|
1244
1282
|
mode: OntologyMode;
|
|
1245
1283
|
manifest: OntologyManifest;
|
|
1246
1284
|
} | null>;
|
|
1285
|
+
/**
|
|
1286
|
+
* Write an entity's manifest.
|
|
1287
|
+
*
|
|
1288
|
+
* `opts.ifAbsent` selects the conflict behaviour:
|
|
1289
|
+
* - omitted / `false` — upsert. An existing row is replaced.
|
|
1290
|
+
* - `true` — create-if-absent. An existing row is left exactly as it is.
|
|
1291
|
+
*
|
|
1292
|
+
* Returns whether this call actually wrote a row. Under `ifAbsent` a `false`
|
|
1293
|
+
* result means another writer got there first; the check and the write are one
|
|
1294
|
+
* statement, so there is no window between them.
|
|
1295
|
+
*/
|
|
1247
1296
|
setManifest(entityId: string, data: {
|
|
1248
1297
|
mode: OntologyMode;
|
|
1249
1298
|
manifest: OntologyManifest;
|
|
1250
|
-
}, tx: SQLiteAdapter
|
|
1299
|
+
}, tx: SQLiteAdapter, opts?: {
|
|
1300
|
+
ifAbsent?: boolean;
|
|
1301
|
+
}): Promise<boolean>;
|
|
1251
1302
|
mergeManifestUpdates(entityId: string, updates: OntologyUpdates, tx: SQLiteAdapter): Promise<OntologyManifest>;
|
|
1252
1303
|
}
|
|
1253
1304
|
|
|
@@ -2339,10 +2390,44 @@ declare class WikiMemory {
|
|
|
2339
2390
|
* Seeds or replaces an entity's ontology manifest and optional mode override.
|
|
2340
2391
|
* Validates manifest invariants (unique type slugs, edge endpoints reference node types).
|
|
2341
2392
|
* Invalidates the in-memory ontology cache for this entity.
|
|
2393
|
+
*
|
|
2394
|
+
* Delegates to `setOntologyManifests` so there is a single write path. The
|
|
2395
|
+
* observable contract is unchanged: same upsert semantics, same cache
|
|
2396
|
+
* invalidation, same `void` result.
|
|
2342
2397
|
*/
|
|
2343
2398
|
setOntologyManifest(entityId: string, manifest: OntologyManifest, options?: {
|
|
2344
2399
|
mode?: OntologyMode;
|
|
2345
2400
|
}): Promise<void>;
|
|
2401
|
+
/**
|
|
2402
|
+
* Write several entities' ontology manifests in a single transaction.
|
|
2403
|
+
*
|
|
2404
|
+
* All succeed or none do. The transaction is opened on this instance's
|
|
2405
|
+
* serialized adapter — callers pass data, never a transaction handle, because
|
|
2406
|
+
* a consumer holds the unwrapped adapter and a transaction opened on it would
|
|
2407
|
+
* bypass the serialization mutex applied in the constructor.
|
|
2408
|
+
*
|
|
2409
|
+
* Returns which entities were written. `skipped` is only ever non-empty under
|
|
2410
|
+
* `opts.ifAbsent`.
|
|
2411
|
+
*
|
|
2412
|
+
* Sharp edge: `ifAbsent` checks for a PERSISTED row, not for an effective
|
|
2413
|
+
* manifest. An entity whose manifest currently comes from
|
|
2414
|
+
* `WikiConfig.ontology.seedManifests` has no row until the seed is
|
|
2415
|
+
* materialized on first access that supplies a transaction (in practice, an
|
|
2416
|
+
* ingest). Until then `ifAbsent` treats the entity as absent, writes the new
|
|
2417
|
+
* manifest over the configured seed, and reports it in `written`; the same
|
|
2418
|
+
* call after the seed has been materialized reports it in `skipped`. See the
|
|
2419
|
+
* `ifAbsent` notes in the package README.
|
|
2420
|
+
*/
|
|
2421
|
+
setOntologyManifests(entries: Array<{
|
|
2422
|
+
entityId: string;
|
|
2423
|
+
manifest: OntologyManifest;
|
|
2424
|
+
mode?: OntologyMode;
|
|
2425
|
+
}>, opts?: {
|
|
2426
|
+
ifAbsent?: boolean;
|
|
2427
|
+
}): Promise<{
|
|
2428
|
+
written: string[];
|
|
2429
|
+
skipped: string[];
|
|
2430
|
+
}>;
|
|
2346
2431
|
/** Append a verification event to a fact. Does NOT touch `updated_at`. */
|
|
2347
2432
|
writeOkfTrust(entryId: string, entityId: string, verified: {
|
|
2348
2433
|
by: string;
|
|
@@ -2373,4 +2458,4 @@ declare class WikiMemory {
|
|
|
2373
2458
|
setGeneratedByTask(taskId: string, entityId: string, actor: string): Promise<void>;
|
|
2374
2459
|
}
|
|
2375
2460
|
|
|
2376
|
-
export {
|
|
2461
|
+
export { WikiParseError as $, WikiBusyError as A, type WikiBusyOperation as B, type ChunkFailure as C, type DegradedRecord as D, type EntityStatus as E, type FormatContextOptions as F, type GraphNeighborhood as G, HEAL_BATCH_SIZE as H, type IngestDocumentResult as I, type WikiCheckpoint as J, type WikiConfig as K, type LLMProvider as L, type MemoryBundle as M, WikiDuplicateHashError as N, type OntologyManifest as O, type PromptOverrides as P, type WikiEdge as Q, type ReadOptions as R, type SQLiteAdapter as S, type WikiEvent as T, type WikiFact as U, type VectorRanker as V, type WikiOptions as W, WikiIngestEmptyError as X, WikiInvalidReadOptions as Y, type WikiMemoryTestAccess as Z, type WikiOutboxEvent as _, type MemoryDump as a, WikiSourceRefHashCollision as a0, WikiStrictOntologyViolation as a1, type WikiTask as a2, WikiTransactionError as a3, EmbeddingService as a4, ImportExportService as a5, IngestionService as a6, JobManager as a7, MaintenanceService as a8, RetrievalService as a9, SearchService as aa, WriteService as ab, type FormattedMemoryDump as b, WikiMemory as c, type ExtractedFact as d, type ExtractedFactEdge as e, type ExtractedFactWithOntology as f, type ExtractedTask as g, type GraphTraversalOptions as h, HEAL_RECHECK_MS as i, HOOK_TIMEOUT_MARKER as j, type HealResult as k, ONTOLOGY_BACKFILL_BATCH_SIZE as l, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS as m, ONTOLOGY_BACKFILL_RECHECK_MS as n, type OntologyBackfillResult as o, type OntologyConfig as p, type OntologyEdgeType as q, type OntologyMode as r, type OntologyNodeType as s, type OntologyPromptContext as t, type OntologyUpdates as u, PromptService as v, PrunePartialFailureError as w, type VectorRankerFallback as x, type VectorRankerRankArgs as y, type VectorRankerSemanticResult as z };
|
|
@@ -195,6 +195,16 @@ interface WikiConfig {
|
|
|
195
195
|
maxChunkLength?: number;
|
|
196
196
|
chunkOverlap?: number;
|
|
197
197
|
chunkConcurrency?: number;
|
|
198
|
+
/**
|
|
199
|
+
* Maximum characters of `title + body + tags` passed to `llmProvider.embed()`
|
|
200
|
+
* when embedding a fact. Longer input is clipped, not rejected.
|
|
201
|
+
*
|
|
202
|
+
* Default 6000 (~1,500 tokens), which fits common embedding windows including
|
|
203
|
+
* nomic-embed-text's 2,048 tokens with headroom. Values above the 16,000
|
|
204
|
+
* hard ceiling are clamped to it; the ceiling is a security control and
|
|
205
|
+
* configuration cannot raise it.
|
|
206
|
+
*/
|
|
207
|
+
maxEmbedChars?: number;
|
|
198
208
|
/**
|
|
199
209
|
* Max MiniSearch candidates passed to cosine scoring.
|
|
200
210
|
* When set, MiniSearch pre-filters before the cosine scan.
|
|
@@ -250,6 +260,23 @@ interface ReadOptions {
|
|
|
250
260
|
* Only meaningful when `entityId` is an array; ignored for single-string calls.
|
|
251
261
|
*/
|
|
252
262
|
includeZeroWeightEntities?: boolean;
|
|
263
|
+
/**
|
|
264
|
+
* Minimum number of results to retain from each named entity before the global
|
|
265
|
+
* `maxResults` cut. Applied after scoring: the top-N scored results from each
|
|
266
|
+
* named entity are reserved, then remaining slots fill by score as usual.
|
|
267
|
+
*
|
|
268
|
+
* An entity with fewer than N matching results contributes what it has — this is
|
|
269
|
+
* not an error. Floors cannot resurrect a result excluded by `preFilterLimit`.
|
|
270
|
+
*
|
|
271
|
+
* Only meaningful when `entityId` is an array; ignored for single-string calls.
|
|
272
|
+
* Ignored when `query` is empty — that path uses recency ordering and ignores
|
|
273
|
+
* `tierWeights` as well.
|
|
274
|
+
*
|
|
275
|
+
* Throws `WikiInvalidReadOptions` when a floor cannot be satisfied by
|
|
276
|
+
* construction: floors summing above `maxResults`, a floor on an entity excluded
|
|
277
|
+
* by `tierWeights: 0`, or a floor keyed to an entity not in `entityId`.
|
|
278
|
+
*/
|
|
279
|
+
tierFloors?: Record<string, number>;
|
|
253
280
|
}
|
|
254
281
|
interface WikiFact {
|
|
255
282
|
id: string;
|
|
@@ -588,6 +615,8 @@ interface MemoryBundle {
|
|
|
588
615
|
query: string;
|
|
589
616
|
entityIds: string[];
|
|
590
617
|
tierWeights?: Record<string, number>;
|
|
618
|
+
/** Sanitized per-entity result floors, when `tierFloors` was supplied and non-empty. */
|
|
619
|
+
tierFloors?: Record<string, number>;
|
|
591
620
|
};
|
|
592
621
|
}
|
|
593
622
|
interface MemoryDump {
|
|
@@ -787,6 +816,15 @@ declare class WikiIngestEmptyError extends Error {
|
|
|
787
816
|
chunks: number;
|
|
788
817
|
});
|
|
789
818
|
}
|
|
819
|
+
/**
|
|
820
|
+
* Thrown by `read()` when `ReadOptions` contains a request that cannot be
|
|
821
|
+
* satisfied by construction — as opposed to one that merely finds no data.
|
|
822
|
+
*/
|
|
823
|
+
declare class WikiInvalidReadOptions extends Error {
|
|
824
|
+
readonly field: string;
|
|
825
|
+
readonly reason: string;
|
|
826
|
+
constructor(field: string, reason: string);
|
|
827
|
+
}
|
|
790
828
|
|
|
791
829
|
interface WikiOutboxEvent<T = unknown> {
|
|
792
830
|
id: string;
|
|
@@ -1244,10 +1282,23 @@ declare class MetadataRepository extends BaseRepository {
|
|
|
1244
1282
|
mode: OntologyMode;
|
|
1245
1283
|
manifest: OntologyManifest;
|
|
1246
1284
|
} | null>;
|
|
1285
|
+
/**
|
|
1286
|
+
* Write an entity's manifest.
|
|
1287
|
+
*
|
|
1288
|
+
* `opts.ifAbsent` selects the conflict behaviour:
|
|
1289
|
+
* - omitted / `false` — upsert. An existing row is replaced.
|
|
1290
|
+
* - `true` — create-if-absent. An existing row is left exactly as it is.
|
|
1291
|
+
*
|
|
1292
|
+
* Returns whether this call actually wrote a row. Under `ifAbsent` a `false`
|
|
1293
|
+
* result means another writer got there first; the check and the write are one
|
|
1294
|
+
* statement, so there is no window between them.
|
|
1295
|
+
*/
|
|
1247
1296
|
setManifest(entityId: string, data: {
|
|
1248
1297
|
mode: OntologyMode;
|
|
1249
1298
|
manifest: OntologyManifest;
|
|
1250
|
-
}, tx: SQLiteAdapter
|
|
1299
|
+
}, tx: SQLiteAdapter, opts?: {
|
|
1300
|
+
ifAbsent?: boolean;
|
|
1301
|
+
}): Promise<boolean>;
|
|
1251
1302
|
mergeManifestUpdates(entityId: string, updates: OntologyUpdates, tx: SQLiteAdapter): Promise<OntologyManifest>;
|
|
1252
1303
|
}
|
|
1253
1304
|
|
|
@@ -2339,10 +2390,44 @@ declare class WikiMemory {
|
|
|
2339
2390
|
* Seeds or replaces an entity's ontology manifest and optional mode override.
|
|
2340
2391
|
* Validates manifest invariants (unique type slugs, edge endpoints reference node types).
|
|
2341
2392
|
* Invalidates the in-memory ontology cache for this entity.
|
|
2393
|
+
*
|
|
2394
|
+
* Delegates to `setOntologyManifests` so there is a single write path. The
|
|
2395
|
+
* observable contract is unchanged: same upsert semantics, same cache
|
|
2396
|
+
* invalidation, same `void` result.
|
|
2342
2397
|
*/
|
|
2343
2398
|
setOntologyManifest(entityId: string, manifest: OntologyManifest, options?: {
|
|
2344
2399
|
mode?: OntologyMode;
|
|
2345
2400
|
}): Promise<void>;
|
|
2401
|
+
/**
|
|
2402
|
+
* Write several entities' ontology manifests in a single transaction.
|
|
2403
|
+
*
|
|
2404
|
+
* All succeed or none do. The transaction is opened on this instance's
|
|
2405
|
+
* serialized adapter — callers pass data, never a transaction handle, because
|
|
2406
|
+
* a consumer holds the unwrapped adapter and a transaction opened on it would
|
|
2407
|
+
* bypass the serialization mutex applied in the constructor.
|
|
2408
|
+
*
|
|
2409
|
+
* Returns which entities were written. `skipped` is only ever non-empty under
|
|
2410
|
+
* `opts.ifAbsent`.
|
|
2411
|
+
*
|
|
2412
|
+
* Sharp edge: `ifAbsent` checks for a PERSISTED row, not for an effective
|
|
2413
|
+
* manifest. An entity whose manifest currently comes from
|
|
2414
|
+
* `WikiConfig.ontology.seedManifests` has no row until the seed is
|
|
2415
|
+
* materialized on first access that supplies a transaction (in practice, an
|
|
2416
|
+
* ingest). Until then `ifAbsent` treats the entity as absent, writes the new
|
|
2417
|
+
* manifest over the configured seed, and reports it in `written`; the same
|
|
2418
|
+
* call after the seed has been materialized reports it in `skipped`. See the
|
|
2419
|
+
* `ifAbsent` notes in the package README.
|
|
2420
|
+
*/
|
|
2421
|
+
setOntologyManifests(entries: Array<{
|
|
2422
|
+
entityId: string;
|
|
2423
|
+
manifest: OntologyManifest;
|
|
2424
|
+
mode?: OntologyMode;
|
|
2425
|
+
}>, opts?: {
|
|
2426
|
+
ifAbsent?: boolean;
|
|
2427
|
+
}): Promise<{
|
|
2428
|
+
written: string[];
|
|
2429
|
+
skipped: string[];
|
|
2430
|
+
}>;
|
|
2346
2431
|
/** Append a verification event to a fact. Does NOT touch `updated_at`. */
|
|
2347
2432
|
writeOkfTrust(entryId: string, entityId: string, verified: {
|
|
2348
2433
|
by: string;
|
|
@@ -2373,4 +2458,4 @@ declare class WikiMemory {
|
|
|
2373
2458
|
setGeneratedByTask(taskId: string, entityId: string, actor: string): Promise<void>;
|
|
2374
2459
|
}
|
|
2375
2460
|
|
|
2376
|
-
export {
|
|
2461
|
+
export { WikiParseError as $, WikiBusyError as A, type WikiBusyOperation as B, type ChunkFailure as C, type DegradedRecord as D, type EntityStatus as E, type FormatContextOptions as F, type GraphNeighborhood as G, HEAL_BATCH_SIZE as H, type IngestDocumentResult as I, type WikiCheckpoint as J, type WikiConfig as K, type LLMProvider as L, type MemoryBundle as M, WikiDuplicateHashError as N, type OntologyManifest as O, type PromptOverrides as P, type WikiEdge as Q, type ReadOptions as R, type SQLiteAdapter as S, type WikiEvent as T, type WikiFact as U, type VectorRanker as V, type WikiOptions as W, WikiIngestEmptyError as X, WikiInvalidReadOptions as Y, type WikiMemoryTestAccess as Z, type WikiOutboxEvent as _, type MemoryDump as a, WikiSourceRefHashCollision as a0, WikiStrictOntologyViolation as a1, type WikiTask as a2, WikiTransactionError as a3, EmbeddingService as a4, ImportExportService as a5, IngestionService as a6, JobManager as a7, MaintenanceService as a8, RetrievalService as a9, SearchService as aa, WriteService as ab, type FormattedMemoryDump as b, WikiMemory as c, type ExtractedFact as d, type ExtractedFactEdge as e, type ExtractedFactWithOntology as f, type ExtractedTask as g, type GraphTraversalOptions as h, HEAL_RECHECK_MS as i, HOOK_TIMEOUT_MARKER as j, type HealResult as k, ONTOLOGY_BACKFILL_BATCH_SIZE as l, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS as m, ONTOLOGY_BACKFILL_RECHECK_MS as n, type OntologyBackfillResult as o, type OntologyConfig as p, type OntologyEdgeType as q, type OntologyMode as r, type OntologyNodeType as s, type OntologyPromptContext as t, type OntologyUpdates as u, PromptService as v, PrunePartialFailureError as w, type VectorRankerFallback as x, type VectorRankerRankArgs as y, type VectorRankerSemanticResult as z };
|
package/dist/testing.d.mts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { a4 as EmbeddingService, a5 as ImportExportService, a6 as IngestionService, a7 as JobManager, a7 as JobManagerType, a8 as MaintenanceService, a9 as RetrievalService, aa as SearchService, aa as SearchServiceType, Z as WikiMemoryTestAccess, ab as WriteService } from './testing-DWpAgNLt.mjs';
|
|
2
2
|
import '@equationalapplications/core-okf';
|
|
3
3
|
import 'minisearch';
|
package/dist/testing.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { a4 as EmbeddingService, a5 as ImportExportService, a6 as IngestionService, a7 as JobManager, a7 as JobManagerType, a8 as MaintenanceService, a9 as RetrievalService, aa as SearchService, aa as SearchServiceType, Z as WikiMemoryTestAccess, ab as WriteService } from './testing-DWpAgNLt.js';
|
|
2
2
|
import '@equationalapplications/core-okf';
|
|
3
3
|
import 'minisearch';
|
package/dist/testing.js
CHANGED
|
@@ -129,6 +129,15 @@ var WikiIngestEmptyError = class extends Error {
|
|
|
129
129
|
this.chunks = params.chunks;
|
|
130
130
|
}
|
|
131
131
|
};
|
|
132
|
+
var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
133
|
+
constructor(field, reason) {
|
|
134
|
+
super(`Invalid ReadOptions.${field}: ${reason}`);
|
|
135
|
+
this.name = "WikiInvalidReadOptions";
|
|
136
|
+
this.field = field;
|
|
137
|
+
this.reason = reason;
|
|
138
|
+
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
139
|
+
}
|
|
140
|
+
};
|
|
132
141
|
|
|
133
142
|
// src/utils/pure.ts
|
|
134
143
|
var MAX_REPAIR_CANDIDATES = 5;
|
|
@@ -616,6 +625,10 @@ function jaccardScore(a, b) {
|
|
|
616
625
|
return intersection.size / union.size;
|
|
617
626
|
}
|
|
618
627
|
|
|
628
|
+
// src/utils/embedDefaults.ts
|
|
629
|
+
var DEFAULT_MAX_EMBED_CHARS = 6e3;
|
|
630
|
+
var EMBED_CHARS_CEILING = 16e3;
|
|
631
|
+
|
|
619
632
|
// src/services/EmbeddingService.ts
|
|
620
633
|
var EmbeddingService = class {
|
|
621
634
|
constructor(db, options, entryRepo, metadataRepo) {
|
|
@@ -663,7 +676,9 @@ var EmbeddingService = class {
|
|
|
663
676
|
tagsStr = fact.tags;
|
|
664
677
|
}
|
|
665
678
|
}
|
|
666
|
-
const
|
|
679
|
+
const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
|
|
680
|
+
const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
|
|
681
|
+
const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
|
|
667
682
|
try {
|
|
668
683
|
const vector = await embedFn(text);
|
|
669
684
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
@@ -2989,6 +3004,63 @@ function applyTierWeight(score, entityId, sanitizedTierWeights) {
|
|
|
2989
3004
|
function shouldExposeReadMetadata(entityId) {
|
|
2990
3005
|
return Array.isArray(entityId);
|
|
2991
3006
|
}
|
|
3007
|
+
function validateTierFloors(entityIds, tierFloors, sanitizedTierWeights, includeZeroWeightEntities, maxResults) {
|
|
3008
|
+
if (tierFloors === void 0) return void 0;
|
|
3009
|
+
const known = new Set(entityIds);
|
|
3010
|
+
for (const key of Object.keys(tierFloors)) {
|
|
3011
|
+
const raw = tierFloors[key];
|
|
3012
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
3013
|
+
if (!known.has(key)) {
|
|
3014
|
+
throw new WikiInvalidReadOptions(
|
|
3015
|
+
"tierFloors",
|
|
3016
|
+
`"${key}" is not one of the entity IDs passed to read(); a floor on an unrequested entity can never be satisfied`
|
|
3017
|
+
);
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
3020
|
+
const sanitized = /* @__PURE__ */ Object.create(null);
|
|
3021
|
+
let total = 0;
|
|
3022
|
+
for (const entityId of entityIds) {
|
|
3023
|
+
const raw = tierFloors[entityId];
|
|
3024
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
3025
|
+
const floor = Math.max(0, Math.trunc(raw));
|
|
3026
|
+
if (floor > 0 && includeZeroWeightEntities !== true && sanitizedTierWeights?.[entityId] === 0) {
|
|
3027
|
+
throw new WikiInvalidReadOptions(
|
|
3028
|
+
"tierFloors",
|
|
3029
|
+
`"${entityId}" has tierWeight 0 and includeZeroWeightEntities is not set, so it is excluded from retrieval and its floor of ${floor} can never be met`
|
|
3030
|
+
);
|
|
3031
|
+
}
|
|
3032
|
+
sanitized[entityId] = floor;
|
|
3033
|
+
total += floor;
|
|
3034
|
+
}
|
|
3035
|
+
if (total > maxResults) {
|
|
3036
|
+
throw new WikiInvalidReadOptions(
|
|
3037
|
+
"tierFloors",
|
|
3038
|
+
`floors sum to ${total}, which exceeds maxResults (${maxResults})`
|
|
3039
|
+
);
|
|
3040
|
+
}
|
|
3041
|
+
return sanitized;
|
|
3042
|
+
}
|
|
3043
|
+
function selectWithFloors(sortedRows, floors, maxResults) {
|
|
3044
|
+
if (maxResults <= 0) return [];
|
|
3045
|
+
if (floors === void 0 || Object.keys(floors).length === 0) {
|
|
3046
|
+
return sortedRows.slice(0, maxResults);
|
|
3047
|
+
}
|
|
3048
|
+
const selected = /* @__PURE__ */ new Set();
|
|
3049
|
+
const taken = /* @__PURE__ */ Object.create(null);
|
|
3050
|
+
for (let i = 0; i < sortedRows.length; i++) {
|
|
3051
|
+
const entityId = sortedRows[i].entity_id;
|
|
3052
|
+
const floor = floors[entityId] ?? 0;
|
|
3053
|
+
if (floor <= 0) continue;
|
|
3054
|
+
const count = taken[entityId] ?? 0;
|
|
3055
|
+
if (count >= floor) continue;
|
|
3056
|
+
taken[entityId] = count + 1;
|
|
3057
|
+
selected.add(i);
|
|
3058
|
+
}
|
|
3059
|
+
for (let i = 0; i < sortedRows.length && selected.size < maxResults; i++) {
|
|
3060
|
+
if (!selected.has(i)) selected.add(i);
|
|
3061
|
+
}
|
|
3062
|
+
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
3063
|
+
}
|
|
2992
3064
|
|
|
2993
3065
|
// src/services/RetrievalService.ts
|
|
2994
3066
|
var RetrievalService = class {
|
|
@@ -3023,13 +3095,20 @@ var RetrievalService = class {
|
|
|
3023
3095
|
}
|
|
3024
3096
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
3025
3097
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
3098
|
+
const trimmedQuery = query.trim();
|
|
3099
|
+
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
3100
|
+
entityIds,
|
|
3101
|
+
options?.tierFloors,
|
|
3102
|
+
sanitizedTierWeights,
|
|
3103
|
+
options?.includeZeroWeightEntities,
|
|
3104
|
+
maxResults
|
|
3105
|
+
) : void 0;
|
|
3026
3106
|
const rawPreFilterLimit = options?.preFilterLimit === null ? void 0 : options?.preFilterLimit ?? config?.preFilterLimit;
|
|
3027
3107
|
const effectivePreFilterLimit = rawPreFilterLimit === void 0 ? void 0 : Number.isFinite(rawPreFilterLimit) ? Math.max(0, Math.trunc(rawPreFilterLimit)) : void 0;
|
|
3028
3108
|
const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
|
|
3029
3109
|
const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
|
|
3030
3110
|
const skipEmbed = weight === 0;
|
|
3031
3111
|
const embedFn = this.options.llmProvider.embed;
|
|
3032
|
-
const trimmedQuery = query.trim();
|
|
3033
3112
|
let facts = [];
|
|
3034
3113
|
let scoreByFactId;
|
|
3035
3114
|
if (maxResults === 0) ; else if (trimmedQuery) {
|
|
@@ -3140,13 +3219,74 @@ var RetrievalService = class {
|
|
|
3140
3219
|
access_count: metadata?.access_count ?? null
|
|
3141
3220
|
};
|
|
3142
3221
|
});
|
|
3222
|
+
const floorShortfallByEntity = /* @__PURE__ */ Object.create(null);
|
|
3223
|
+
let totalFloorShortfall = 0;
|
|
3224
|
+
if (sanitizedTierFloors) {
|
|
3225
|
+
const scoredCounts = /* @__PURE__ */ new Map();
|
|
3226
|
+
for (const s of scored) {
|
|
3227
|
+
scoredCounts.set(s.entity_id, (scoredCounts.get(s.entity_id) ?? 0) + 1);
|
|
3228
|
+
}
|
|
3229
|
+
for (const [entityId2, floor] of Object.entries(sanitizedTierFloors)) {
|
|
3230
|
+
if (floor <= 0) continue;
|
|
3231
|
+
const currentCount = scoredCounts.get(entityId2) ?? 0;
|
|
3232
|
+
const shortfall = Math.max(0, floor - currentCount);
|
|
3233
|
+
if (shortfall > 0) {
|
|
3234
|
+
floorShortfallByEntity[entityId2] = shortfall;
|
|
3235
|
+
totalFloorShortfall += shortfall;
|
|
3236
|
+
}
|
|
3237
|
+
}
|
|
3238
|
+
}
|
|
3239
|
+
const reservedIds = /* @__PURE__ */ new Set();
|
|
3240
|
+
const reserved = [];
|
|
3241
|
+
if (totalFloorShortfall > 0) {
|
|
3242
|
+
for (const [entityId2, shortfall] of Object.entries(floorShortfallByEntity)) {
|
|
3243
|
+
const entityOmitted = [];
|
|
3244
|
+
for (const row of candidateRows) {
|
|
3245
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
3246
|
+
if (row.entity_id !== entityId2) continue;
|
|
3247
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
3248
|
+
entityOmitted.push({ row, kwScore });
|
|
3249
|
+
}
|
|
3250
|
+
entityOmitted.sort((a, b) => this._compareScoredRows(
|
|
3251
|
+
{ id: a.row.id, score: a.kwScore, updated_at: a.row.updated_at, access_count: a.row.access_count },
|
|
3252
|
+
{ id: b.row.id, score: b.kwScore, updated_at: b.row.updated_at, access_count: b.row.access_count }
|
|
3253
|
+
));
|
|
3254
|
+
for (let i = 0; i < Math.min(shortfall, entityOmitted.length); i++) {
|
|
3255
|
+
reserved.push(entityOmitted[i]);
|
|
3256
|
+
reservedIds.add(entityOmitted[i].row.id);
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
}
|
|
3143
3260
|
const isHybrid = weight !== void 0 && weight < 1;
|
|
3261
|
+
if (reserved.length > 0) {
|
|
3262
|
+
if (isHybrid) {
|
|
3263
|
+
for (const { row, kwScore } of reserved) {
|
|
3264
|
+
scored.push({
|
|
3265
|
+
id: row.id,
|
|
3266
|
+
entity_id: row.entity_id,
|
|
3267
|
+
score: (1 - weight) * kwScore,
|
|
3268
|
+
updated_at: row.updated_at,
|
|
3269
|
+
access_count: row.access_count
|
|
3270
|
+
});
|
|
3271
|
+
}
|
|
3272
|
+
} else {
|
|
3273
|
+
for (const { row } of reserved) {
|
|
3274
|
+
scored.push({
|
|
3275
|
+
id: row.id,
|
|
3276
|
+
entity_id: row.entity_id,
|
|
3277
|
+
score: -2,
|
|
3278
|
+
updated_at: row.updated_at,
|
|
3279
|
+
access_count: row.access_count
|
|
3280
|
+
});
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3144
3284
|
const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
|
|
3145
3285
|
if (maxBackfill > 0) {
|
|
3146
3286
|
if (isHybrid) {
|
|
3147
3287
|
const topK = [];
|
|
3148
3288
|
for (const row of candidateRows) {
|
|
3149
|
-
if (scoredIds.has(row.id)) continue;
|
|
3289
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
3150
3290
|
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
3151
3291
|
const candidate = { row, kwScore };
|
|
3152
3292
|
if (topK.length < maxBackfill) {
|
|
@@ -3226,7 +3366,7 @@ var RetrievalService = class {
|
|
|
3226
3366
|
} else {
|
|
3227
3367
|
const omitted = [];
|
|
3228
3368
|
for (const row of candidateRows) {
|
|
3229
|
-
if (scoredIds.has(row.id)) continue;
|
|
3369
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
3230
3370
|
omitted.push({ id: row.id, entity_id: row.entity_id, score: -2, updated_at: row.updated_at, access_count: row.access_count });
|
|
3231
3371
|
}
|
|
3232
3372
|
if (omitted.length > 0) {
|
|
@@ -3274,8 +3414,11 @@ var RetrievalService = class {
|
|
|
3274
3414
|
// read() re-sorts after applying tier weights
|
|
3275
3415
|
});
|
|
3276
3416
|
} else if (policy === "keyword") {
|
|
3277
|
-
const
|
|
3278
|
-
const
|
|
3417
|
+
const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
3418
|
+
const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
3419
|
+
const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
|
|
3420
|
+
const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
|
|
3421
|
+
const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
|
|
3279
3422
|
const topResultIds = new Set(topResults.map((r) => r.id));
|
|
3280
3423
|
const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
|
|
3281
3424
|
scored = topResults.map((result) => {
|
|
@@ -3300,7 +3443,7 @@ var RetrievalService = class {
|
|
|
3300
3443
|
}
|
|
3301
3444
|
}
|
|
3302
3445
|
} else {
|
|
3303
|
-
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1);
|
|
3446
|
+
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1) || sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
3304
3447
|
scored = await this.searchService.rankSemantic({
|
|
3305
3448
|
entityId: entityCacheKey,
|
|
3306
3449
|
queryVec,
|
|
@@ -3319,7 +3462,7 @@ var RetrievalService = class {
|
|
|
3319
3462
|
score: applyTierWeight(row.score, row.entity_id, sanitizedTierWeights)
|
|
3320
3463
|
}));
|
|
3321
3464
|
this._tieBreakSort(scored);
|
|
3322
|
-
const selectedScored = scored
|
|
3465
|
+
const selectedScored = selectWithFloors(scored, sanitizedTierFloors, maxResults);
|
|
3323
3466
|
const topIds = selectedScored.map((s) => s.id);
|
|
3324
3467
|
if (exposeMetadata && trimmedQuery) {
|
|
3325
3468
|
scoreByFactId = new Map(selectedScored.map((s) => [s.id, Number.isFinite(s.score) ? s.score : 0]));
|
|
@@ -3374,7 +3517,8 @@ var RetrievalService = class {
|
|
|
3374
3517
|
}
|
|
3375
3518
|
}
|
|
3376
3519
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
3377
|
-
const
|
|
3520
|
+
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
3521
|
+
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
3378
3522
|
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
|
|
3379
3523
|
const candidates = results.map((r) => ({
|
|
3380
3524
|
id: r.id,
|
|
@@ -3384,7 +3528,7 @@ var RetrievalService = class {
|
|
|
3384
3528
|
access_count: null
|
|
3385
3529
|
}));
|
|
3386
3530
|
this._tieBreakSort(candidates);
|
|
3387
|
-
const topCandidates = candidates
|
|
3531
|
+
const topCandidates = selectWithFloors(candidates, sanitizedTierFloors, maxResults);
|
|
3388
3532
|
const topIds = topCandidates.map((c) => c.id);
|
|
3389
3533
|
if (topIds.length > 0) {
|
|
3390
3534
|
facts = await this._hydrateFactsByIds(topIds, entityIds);
|
|
@@ -3414,6 +3558,7 @@ var RetrievalService = class {
|
|
|
3414
3558
|
if (exposeMetadata) {
|
|
3415
3559
|
bundle.metadata = { query, entityIds };
|
|
3416
3560
|
if (sanitizedTierWeights && Object.keys(sanitizedTierWeights).length > 0) bundle.metadata.tierWeights = sanitizedTierWeights;
|
|
3561
|
+
if (sanitizedTierFloors && Object.keys(sanitizedTierFloors).length > 0) bundle.metadata.tierFloors = sanitizedTierFloors;
|
|
3417
3562
|
if (factScores && Object.keys(factScores).length > 0) bundle.factScores = factScores;
|
|
3418
3563
|
}
|
|
3419
3564
|
return bundle;
|