@equationalapplications/core-llm-wiki 6.2.0 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/dist/{chunk-YEESP6J2.mjs → chunk-DPFM33WV.mjs} +175 -16
- package/dist/chunk-DPFM33WV.mjs.map +1 -0
- package/dist/index.d.mts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +231 -18
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +58 -7
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-D0m2V9-E.d.mts → testing-DWpAgNLt.d.mts} +87 -2
- package/dist/{testing-D0m2V9-E.d.ts → testing-DWpAgNLt.d.ts} +87 -2
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +155 -10
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-YEESP6J2.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -16,8 +16,8 @@ Platform-agnostic TypeScript engine for hybrid LLM memory. Features episodic fac
|
|
|
16
16
|
- **Platform-agnostic** — Zero runtime dependencies; works with any SQLite driver via the `SQLiteAdapter` interface
|
|
17
17
|
- **Semantic search** — Vector embeddings via your LLM's `embed` function, ranked by cosine similarity
|
|
18
18
|
- **Keyword fallback** — [MiniSearch](https://github.com/lucaong/minisearch) in-memory index for offline/degraded scenarios when embeddings unavailable
|
|
19
|
-
- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, `hybridWeight`, `tierWeights`, and `includeZeroWeightEntities`
|
|
20
|
-
- **Multi-entity reads** — Search across multiple `entity_id` namespaces in one pass with per-entity score multipliers (`tierWeights`); optional `factScores` and `metadata` for explainability
|
|
19
|
+
- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, `hybridWeight`, `tierWeights`, `tierFloors`, and `includeZeroWeightEntities`
|
|
20
|
+
- **Multi-entity reads** — Search across multiple `entity_id` namespaces in one pass with per-entity score multipliers (`tierWeights`); `tierFloors` reserves each namespace's top-N matching results; optional `factScores` and `metadata` for explainability
|
|
21
21
|
- **Immutable vs mutable facts** — Use `WikiFact.source_type` to distinguish document-sourced facts (`immutable_document`) from derived or user-provided facts (`librarian_inferred`, `user_stated`, `user_confirmed`). Immutable document facts are not rewritten by `runLibrarian()` or `runHeal()` and can only be removed by `forget()` or re-ingesting.
|
|
22
22
|
- **Full-featured memory** — Facts, tasks, events, maintenance jobs (librarian, heal, reembed, prune)
|
|
23
23
|
- **Type-safe** — Built with TypeScript, full type exports
|
|
@@ -135,6 +135,7 @@ const wikiMemory = new WikiMemory(db, {
|
|
|
135
135
|
maxChunkLength: 12000, // default: 12000 (char count per ingestDocument chunk; exported as DEFAULT_MAX_CHUNK_LENGTH)
|
|
136
136
|
chunkOverlap: 400, // default: 400 (overlap between chunks in characters; exported as DEFAULT_CHUNK_OVERLAP)
|
|
137
137
|
chunkConcurrency: 1, // default: 1 (parallel LLM calls per ingestDocument)
|
|
138
|
+
maxEmbedChars: 6000, // default: 6000 (chars of title+body+tags sent to embed(); hard ceiling 16000; exported as DEFAULT_MAX_EMBED_CHARS)
|
|
138
139
|
pruneRetainSoftDeletedFor: 7, // default: 7 (days before hard-deleting soft-deleted facts)
|
|
139
140
|
pruneEventsAfter: 30, // default: 30 (days before hard-deleting old events)
|
|
140
141
|
orphanAfterDays: 30, // default: 30 (days before runHeal flags sourceless facts; null to disable)
|
|
@@ -261,7 +262,7 @@ const multiMemory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_wor
|
|
|
261
262
|
// includeZeroWeightEntities: true — include 0-weight entities as bottom-ranked filler
|
|
262
263
|
});
|
|
263
264
|
// multiMemory.factScores — optional Record<factId, weightedScore> for returned facts; may be absent/undefined
|
|
264
|
-
// multiMemory.metadata — optional { query, entityIds, tierWeights }; may be absent/undefined
|
|
265
|
+
// multiMemory.metadata — optional { query, entityIds, tierWeights, tierFloors }; may be absent/undefined
|
|
265
266
|
```
|
|
266
267
|
|
|
267
268
|
**Hybrid scoring blends:**
|
|
@@ -276,6 +277,7 @@ True cosine-range pure semantic ranking (including negative cosine values) is us
|
|
|
276
277
|
- Missing weights default to `1.0`. Negative weights clamp to `0`. Non-finite weights default to `1.0`.
|
|
277
278
|
- `tierWeights[entity] = 0` skips that entity's scored retrieval branch (no compute cost).
|
|
278
279
|
- `includeZeroWeightEntities: true` includes zero-weight entities as bottom-ranked filler instead of skipping them.
|
|
280
|
+
- `tierFloors: Record<string, number>` reserves that entity's top-N scored results before the global `maxResults` cut — a floor on entity `X` means at least `N` of the returned facts come from `X`. Floors apply *after* scoring (including `tierWeights`) and *after* `preFilterLimit`'s candidate selection, so a row excluded by `preFilterLimit` cannot be resurrected by a floor. An entity with fewer matching facts than its floor contributes what it has — this is not an error. Only meaningful when `entityId` is an array; ignored for single-string calls and for empty-query ("recent facts") reads. Throws `WikiInvalidReadOptions` when a floor cannot be satisfied by construction: floors summing above `maxResults`, a floor keyed to an entity not in `entityId`, or a floor on an entity excluded by `tierWeights: 0` when `includeZeroWeightEntities` is not set. With an external `vectorRanker`, ranker-omitted rows (typically un-embedded facts) for a floored entity are pre-reserved before the global backfill budget is spent, so an unembedded floored entity still satisfies its floor.
|
|
279
281
|
- `factScores` is present for array-shaped `entityId` calls only when the query is non-empty and at least one fact is scored; empty-query ("recent facts") reads leave it absent even when `entityId` is an array. Plain string calls never expose it. `metadata` is present for all array-shaped calls regardless of query.
|
|
280
282
|
- `maxResults` applies globally across all requested entities.
|
|
281
283
|
- Tasks are capped at `min(20 × entityCount, 200)`; events at `min(10 × entityCount, 100)` for multi-entity reads.
|
|
@@ -535,7 +537,11 @@ const wikiMemory = new WikiMemory(db, {
|
|
|
535
537
|
});
|
|
536
538
|
```
|
|
537
539
|
|
|
538
|
-
`seedManifests` entries are written to SQLite
|
|
540
|
+
`seedManifests` entries are written to SQLite the first time an entity's ontology is
|
|
541
|
+
resolved *inside a transaction* (ingest, heal) and no row exists for it. A read outside a
|
|
542
|
+
transaction — including `getOntologyManifest` — resolves the seed without persisting it, so
|
|
543
|
+
a configured entity can report a manifest while still having no row in `entity_manifests`.
|
|
544
|
+
That distinction matters for `ifAbsent` below.
|
|
539
545
|
|
|
540
546
|
### Public API
|
|
541
547
|
|
|
@@ -559,6 +565,37 @@ await wikiMemory.setOntologyManifest('team-alpha', {
|
|
|
559
565
|
}, { mode: 'strict' });
|
|
560
566
|
```
|
|
561
567
|
|
|
568
|
+
Seed several entities atomically — all manifests are written in one transaction,
|
|
569
|
+
so a failure partway through leaves none of them behind:
|
|
570
|
+
|
|
571
|
+
```typescript
|
|
572
|
+
const { written, skipped } = await wikiMemory.setOntologyManifests(
|
|
573
|
+
[
|
|
574
|
+
{ entityId: 'tier_fact', manifest, mode: 'strict' },
|
|
575
|
+
{ entityId: 'tier_wisdom', manifest, mode: 'strict' },
|
|
576
|
+
],
|
|
577
|
+
{ ifAbsent: true },
|
|
578
|
+
);
|
|
579
|
+
// written: entities whose manifest this call wrote
|
|
580
|
+
// skipped: entities that already had a persisted manifest (only under `ifAbsent`)
|
|
581
|
+
```
|
|
582
|
+
|
|
583
|
+
`ifAbsent` makes each write create-if-absent rather than an upsert, so a
|
|
584
|
+
concurrent initializer loses the race by writing nothing instead of overwriting
|
|
585
|
+
a manifest it never read. Omit it for replace-on-conflict, which is what
|
|
586
|
+
`setOntologyManifest` does.
|
|
587
|
+
|
|
588
|
+
**`ifAbsent` tests for a persisted row, not for an effective manifest.** An entity
|
|
589
|
+
whose manifest comes from `WikiConfig.ontology.seedManifests` has no row until an
|
|
590
|
+
ingest materializes one, so `ifAbsent` writes over the configured seed and reports
|
|
591
|
+
the entity in `written`; after an ingest has run for that entity the same call
|
|
592
|
+
reports it in `skipped`. Don't mix the two seeding routes for one entity: seed it
|
|
593
|
+
through the config, or through this method, not both.
|
|
594
|
+
|
|
595
|
+
The method takes data, never a transaction handle: `WikiMemory` serializes
|
|
596
|
+
transactions on the adapter it is given, so a transaction opened on the adapter
|
|
597
|
+
you passed to `createWiki` would not participate in that serialization.
|
|
598
|
+
|
|
562
599
|
### Fact Shape Extensions
|
|
563
600
|
|
|
564
601
|
In **Strict** and **Emergent** modes, librarian and ingest JSON may include typed facts with inline edges:
|
|
@@ -144,6 +144,15 @@ var WikiIngestEmptyError = class extends Error {
|
|
|
144
144
|
this.chunks = params.chunks;
|
|
145
145
|
}
|
|
146
146
|
};
|
|
147
|
+
var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
148
|
+
constructor(field, reason) {
|
|
149
|
+
super(`Invalid ReadOptions.${field}: ${reason}`);
|
|
150
|
+
this.name = "WikiInvalidReadOptions";
|
|
151
|
+
this.field = field;
|
|
152
|
+
this.reason = reason;
|
|
153
|
+
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
147
156
|
|
|
148
157
|
// src/utils/ids.ts
|
|
149
158
|
var _injectedGetRandomValues = null;
|
|
@@ -2337,15 +2346,29 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2337
2346
|
manifest
|
|
2338
2347
|
};
|
|
2339
2348
|
}
|
|
2340
|
-
|
|
2349
|
+
/**
|
|
2350
|
+
* Write an entity's manifest.
|
|
2351
|
+
*
|
|
2352
|
+
* `opts.ifAbsent` selects the conflict behaviour:
|
|
2353
|
+
* - omitted / `false` — upsert. An existing row is replaced.
|
|
2354
|
+
* - `true` — create-if-absent. An existing row is left exactly as it is.
|
|
2355
|
+
*
|
|
2356
|
+
* Returns whether this call actually wrote a row. Under `ifAbsent` a `false`
|
|
2357
|
+
* result means another writer got there first; the check and the write are one
|
|
2358
|
+
* statement, so there is no window between them.
|
|
2359
|
+
*/
|
|
2360
|
+
async setManifest(entityId, data, tx, opts) {
|
|
2341
2361
|
validateManifest(data.manifest);
|
|
2342
2362
|
const executor = this.getExecutor(tx);
|
|
2343
|
-
|
|
2363
|
+
const conflictClause = opts?.ifAbsent ? "ON CONFLICT(entity_id) DO NOTHING" : `ON CONFLICT(entity_id) DO UPDATE SET
|
|
2364
|
+
mode = excluded.mode, manifest_json = excluded.manifest_json, updated_at = excluded.updated_at`;
|
|
2365
|
+
const result = await executor.runAsync(
|
|
2344
2366
|
`INSERT INTO ${this.prefix}entity_manifests (entity_id, mode, manifest_json, updated_at)
|
|
2345
2367
|
VALUES (?, ?, ?, ?)
|
|
2346
|
-
|
|
2368
|
+
${conflictClause}`,
|
|
2347
2369
|
[entityId, data.mode, JSON.stringify(data.manifest), Date.now()]
|
|
2348
2370
|
);
|
|
2371
|
+
return result.changes > 0;
|
|
2349
2372
|
}
|
|
2350
2373
|
async mergeManifestUpdates(entityId, updates, tx) {
|
|
2351
2374
|
const current = await this.getManifest(entityId, tx) ?? {
|
|
@@ -3827,6 +3850,10 @@ After running the migration SQL, restart your application.`
|
|
|
3827
3850
|
}
|
|
3828
3851
|
};
|
|
3829
3852
|
|
|
3853
|
+
// src/utils/embedDefaults.ts
|
|
3854
|
+
var DEFAULT_MAX_EMBED_CHARS = 6e3;
|
|
3855
|
+
var EMBED_CHARS_CEILING = 16e3;
|
|
3856
|
+
|
|
3830
3857
|
// src/services/EmbeddingService.ts
|
|
3831
3858
|
var EmbeddingService = class {
|
|
3832
3859
|
constructor(db, options, entryRepo, metadataRepo) {
|
|
@@ -3874,7 +3901,9 @@ var EmbeddingService = class {
|
|
|
3874
3901
|
tagsStr = fact.tags;
|
|
3875
3902
|
}
|
|
3876
3903
|
}
|
|
3877
|
-
const
|
|
3904
|
+
const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
|
|
3905
|
+
const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
|
|
3906
|
+
const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
|
|
3878
3907
|
try {
|
|
3879
3908
|
const vector = await embedFn(text);
|
|
3880
3909
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
@@ -3985,6 +4014,63 @@ function applyTierWeight(score, entityId, sanitizedTierWeights) {
|
|
|
3985
4014
|
function shouldExposeReadMetadata(entityId) {
|
|
3986
4015
|
return Array.isArray(entityId);
|
|
3987
4016
|
}
|
|
4017
|
+
function validateTierFloors(entityIds, tierFloors, sanitizedTierWeights, includeZeroWeightEntities, maxResults) {
|
|
4018
|
+
if (tierFloors === void 0) return void 0;
|
|
4019
|
+
const known = new Set(entityIds);
|
|
4020
|
+
for (const key of Object.keys(tierFloors)) {
|
|
4021
|
+
const raw = tierFloors[key];
|
|
4022
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
4023
|
+
if (!known.has(key)) {
|
|
4024
|
+
throw new WikiInvalidReadOptions(
|
|
4025
|
+
"tierFloors",
|
|
4026
|
+
`"${key}" is not one of the entity IDs passed to read(); a floor on an unrequested entity can never be satisfied`
|
|
4027
|
+
);
|
|
4028
|
+
}
|
|
4029
|
+
}
|
|
4030
|
+
const sanitized = /* @__PURE__ */ Object.create(null);
|
|
4031
|
+
let total = 0;
|
|
4032
|
+
for (const entityId of entityIds) {
|
|
4033
|
+
const raw = tierFloors[entityId];
|
|
4034
|
+
if (raw === void 0 || !Number.isFinite(raw)) continue;
|
|
4035
|
+
const floor = Math.max(0, Math.trunc(raw));
|
|
4036
|
+
if (floor > 0 && includeZeroWeightEntities !== true && sanitizedTierWeights?.[entityId] === 0) {
|
|
4037
|
+
throw new WikiInvalidReadOptions(
|
|
4038
|
+
"tierFloors",
|
|
4039
|
+
`"${entityId}" has tierWeight 0 and includeZeroWeightEntities is not set, so it is excluded from retrieval and its floor of ${floor} can never be met`
|
|
4040
|
+
);
|
|
4041
|
+
}
|
|
4042
|
+
sanitized[entityId] = floor;
|
|
4043
|
+
total += floor;
|
|
4044
|
+
}
|
|
4045
|
+
if (total > maxResults) {
|
|
4046
|
+
throw new WikiInvalidReadOptions(
|
|
4047
|
+
"tierFloors",
|
|
4048
|
+
`floors sum to ${total}, which exceeds maxResults (${maxResults})`
|
|
4049
|
+
);
|
|
4050
|
+
}
|
|
4051
|
+
return sanitized;
|
|
4052
|
+
}
|
|
4053
|
+
function selectWithFloors(sortedRows, floors, maxResults) {
|
|
4054
|
+
if (maxResults <= 0) return [];
|
|
4055
|
+
if (floors === void 0 || Object.keys(floors).length === 0) {
|
|
4056
|
+
return sortedRows.slice(0, maxResults);
|
|
4057
|
+
}
|
|
4058
|
+
const selected = /* @__PURE__ */ new Set();
|
|
4059
|
+
const taken = /* @__PURE__ */ Object.create(null);
|
|
4060
|
+
for (let i = 0; i < sortedRows.length; i++) {
|
|
4061
|
+
const entityId = sortedRows[i].entity_id;
|
|
4062
|
+
const floor = floors[entityId] ?? 0;
|
|
4063
|
+
if (floor <= 0) continue;
|
|
4064
|
+
const count = taken[entityId] ?? 0;
|
|
4065
|
+
if (count >= floor) continue;
|
|
4066
|
+
taken[entityId] = count + 1;
|
|
4067
|
+
selected.add(i);
|
|
4068
|
+
}
|
|
4069
|
+
for (let i = 0; i < sortedRows.length && selected.size < maxResults; i++) {
|
|
4070
|
+
if (!selected.has(i)) selected.add(i);
|
|
4071
|
+
}
|
|
4072
|
+
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
4073
|
+
}
|
|
3988
4074
|
|
|
3989
4075
|
// src/services/RetrievalService.ts
|
|
3990
4076
|
var RetrievalService = class {
|
|
@@ -4019,13 +4105,20 @@ var RetrievalService = class {
|
|
|
4019
4105
|
}
|
|
4020
4106
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
4021
4107
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
4108
|
+
const trimmedQuery = query.trim();
|
|
4109
|
+
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
4110
|
+
entityIds,
|
|
4111
|
+
options?.tierFloors,
|
|
4112
|
+
sanitizedTierWeights,
|
|
4113
|
+
options?.includeZeroWeightEntities,
|
|
4114
|
+
maxResults
|
|
4115
|
+
) : void 0;
|
|
4022
4116
|
const rawPreFilterLimit = options?.preFilterLimit === null ? void 0 : options?.preFilterLimit ?? config?.preFilterLimit;
|
|
4023
4117
|
const effectivePreFilterLimit = rawPreFilterLimit === void 0 ? void 0 : Number.isFinite(rawPreFilterLimit) ? Math.max(0, Math.trunc(rawPreFilterLimit)) : void 0;
|
|
4024
4118
|
const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
|
|
4025
4119
|
const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
|
|
4026
4120
|
const skipEmbed = weight === 0;
|
|
4027
4121
|
const embedFn = this.options.llmProvider.embed;
|
|
4028
|
-
const trimmedQuery = query.trim();
|
|
4029
4122
|
let facts = [];
|
|
4030
4123
|
let scoreByFactId;
|
|
4031
4124
|
if (maxResults === 0) ; else if (trimmedQuery) {
|
|
@@ -4136,13 +4229,74 @@ var RetrievalService = class {
|
|
|
4136
4229
|
access_count: metadata?.access_count ?? null
|
|
4137
4230
|
};
|
|
4138
4231
|
});
|
|
4232
|
+
const floorShortfallByEntity = /* @__PURE__ */ Object.create(null);
|
|
4233
|
+
let totalFloorShortfall = 0;
|
|
4234
|
+
if (sanitizedTierFloors) {
|
|
4235
|
+
const scoredCounts = /* @__PURE__ */ new Map();
|
|
4236
|
+
for (const s of scored) {
|
|
4237
|
+
scoredCounts.set(s.entity_id, (scoredCounts.get(s.entity_id) ?? 0) + 1);
|
|
4238
|
+
}
|
|
4239
|
+
for (const [entityId2, floor] of Object.entries(sanitizedTierFloors)) {
|
|
4240
|
+
if (floor <= 0) continue;
|
|
4241
|
+
const currentCount = scoredCounts.get(entityId2) ?? 0;
|
|
4242
|
+
const shortfall = Math.max(0, floor - currentCount);
|
|
4243
|
+
if (shortfall > 0) {
|
|
4244
|
+
floorShortfallByEntity[entityId2] = shortfall;
|
|
4245
|
+
totalFloorShortfall += shortfall;
|
|
4246
|
+
}
|
|
4247
|
+
}
|
|
4248
|
+
}
|
|
4249
|
+
const reservedIds = /* @__PURE__ */ new Set();
|
|
4250
|
+
const reserved = [];
|
|
4251
|
+
if (totalFloorShortfall > 0) {
|
|
4252
|
+
for (const [entityId2, shortfall] of Object.entries(floorShortfallByEntity)) {
|
|
4253
|
+
const entityOmitted = [];
|
|
4254
|
+
for (const row of candidateRows) {
|
|
4255
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
4256
|
+
if (row.entity_id !== entityId2) continue;
|
|
4257
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
4258
|
+
entityOmitted.push({ row, kwScore });
|
|
4259
|
+
}
|
|
4260
|
+
entityOmitted.sort((a, b) => this._compareScoredRows(
|
|
4261
|
+
{ id: a.row.id, score: a.kwScore, updated_at: a.row.updated_at, access_count: a.row.access_count },
|
|
4262
|
+
{ id: b.row.id, score: b.kwScore, updated_at: b.row.updated_at, access_count: b.row.access_count }
|
|
4263
|
+
));
|
|
4264
|
+
for (let i = 0; i < Math.min(shortfall, entityOmitted.length); i++) {
|
|
4265
|
+
reserved.push(entityOmitted[i]);
|
|
4266
|
+
reservedIds.add(entityOmitted[i].row.id);
|
|
4267
|
+
}
|
|
4268
|
+
}
|
|
4269
|
+
}
|
|
4139
4270
|
const isHybrid = weight !== void 0 && weight < 1;
|
|
4271
|
+
if (reserved.length > 0) {
|
|
4272
|
+
if (isHybrid) {
|
|
4273
|
+
for (const { row, kwScore } of reserved) {
|
|
4274
|
+
scored.push({
|
|
4275
|
+
id: row.id,
|
|
4276
|
+
entity_id: row.entity_id,
|
|
4277
|
+
score: (1 - weight) * kwScore,
|
|
4278
|
+
updated_at: row.updated_at,
|
|
4279
|
+
access_count: row.access_count
|
|
4280
|
+
});
|
|
4281
|
+
}
|
|
4282
|
+
} else {
|
|
4283
|
+
for (const { row } of reserved) {
|
|
4284
|
+
scored.push({
|
|
4285
|
+
id: row.id,
|
|
4286
|
+
entity_id: row.entity_id,
|
|
4287
|
+
score: -2,
|
|
4288
|
+
updated_at: row.updated_at,
|
|
4289
|
+
access_count: row.access_count
|
|
4290
|
+
});
|
|
4291
|
+
}
|
|
4292
|
+
}
|
|
4293
|
+
}
|
|
4140
4294
|
const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
|
|
4141
4295
|
if (maxBackfill > 0) {
|
|
4142
4296
|
if (isHybrid) {
|
|
4143
4297
|
const topK = [];
|
|
4144
4298
|
for (const row of candidateRows) {
|
|
4145
|
-
if (scoredIds.has(row.id)) continue;
|
|
4299
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
4146
4300
|
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
4147
4301
|
const candidate = { row, kwScore };
|
|
4148
4302
|
if (topK.length < maxBackfill) {
|
|
@@ -4222,7 +4376,7 @@ var RetrievalService = class {
|
|
|
4222
4376
|
} else {
|
|
4223
4377
|
const omitted = [];
|
|
4224
4378
|
for (const row of candidateRows) {
|
|
4225
|
-
if (scoredIds.has(row.id)) continue;
|
|
4379
|
+
if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
|
|
4226
4380
|
omitted.push({ id: row.id, entity_id: row.entity_id, score: -2, updated_at: row.updated_at, access_count: row.access_count });
|
|
4227
4381
|
}
|
|
4228
4382
|
if (omitted.length > 0) {
|
|
@@ -4270,8 +4424,11 @@ var RetrievalService = class {
|
|
|
4270
4424
|
// read() re-sorts after applying tier weights
|
|
4271
4425
|
});
|
|
4272
4426
|
} else if (policy === "keyword") {
|
|
4273
|
-
const
|
|
4274
|
-
const
|
|
4427
|
+
const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
4428
|
+
const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
4429
|
+
const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
|
|
4430
|
+
const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
|
|
4431
|
+
const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
|
|
4275
4432
|
const topResultIds = new Set(topResults.map((r) => r.id));
|
|
4276
4433
|
const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
|
|
4277
4434
|
scored = topResults.map((result) => {
|
|
@@ -4296,7 +4453,7 @@ var RetrievalService = class {
|
|
|
4296
4453
|
}
|
|
4297
4454
|
}
|
|
4298
4455
|
} else {
|
|
4299
|
-
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1);
|
|
4456
|
+
const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1) || sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
4300
4457
|
scored = await this.searchService.rankSemantic({
|
|
4301
4458
|
entityId: entityCacheKey,
|
|
4302
4459
|
queryVec,
|
|
@@ -4315,7 +4472,7 @@ var RetrievalService = class {
|
|
|
4315
4472
|
score: applyTierWeight(row.score, row.entity_id, sanitizedTierWeights)
|
|
4316
4473
|
}));
|
|
4317
4474
|
this._tieBreakSort(scored);
|
|
4318
|
-
const selectedScored = scored
|
|
4475
|
+
const selectedScored = selectWithFloors(scored, sanitizedTierFloors, maxResults);
|
|
4319
4476
|
const topIds = selectedScored.map((s) => s.id);
|
|
4320
4477
|
if (exposeMetadata && trimmedQuery) {
|
|
4321
4478
|
scoreByFactId = new Map(selectedScored.map((s) => [s.id, Number.isFinite(s.score) ? s.score : 0]));
|
|
@@ -4370,7 +4527,8 @@ var RetrievalService = class {
|
|
|
4370
4527
|
}
|
|
4371
4528
|
}
|
|
4372
4529
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
4373
|
-
const
|
|
4530
|
+
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
4531
|
+
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
|
|
4374
4532
|
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
|
|
4375
4533
|
const candidates = results.map((r) => ({
|
|
4376
4534
|
id: r.id,
|
|
@@ -4380,7 +4538,7 @@ var RetrievalService = class {
|
|
|
4380
4538
|
access_count: null
|
|
4381
4539
|
}));
|
|
4382
4540
|
this._tieBreakSort(candidates);
|
|
4383
|
-
const topCandidates = candidates
|
|
4541
|
+
const topCandidates = selectWithFloors(candidates, sanitizedTierFloors, maxResults);
|
|
4384
4542
|
const topIds = topCandidates.map((c) => c.id);
|
|
4385
4543
|
if (topIds.length > 0) {
|
|
4386
4544
|
facts = await this._hydrateFactsByIds(topIds, entityIds);
|
|
@@ -4410,6 +4568,7 @@ var RetrievalService = class {
|
|
|
4410
4568
|
if (exposeMetadata) {
|
|
4411
4569
|
bundle.metadata = { query, entityIds };
|
|
4412
4570
|
if (sanitizedTierWeights && Object.keys(sanitizedTierWeights).length > 0) bundle.metadata.tierWeights = sanitizedTierWeights;
|
|
4571
|
+
if (sanitizedTierFloors && Object.keys(sanitizedTierFloors).length > 0) bundle.metadata.tierFloors = sanitizedTierFloors;
|
|
4413
4572
|
if (factScores && Object.keys(factScores).length > 0) bundle.factScores = factScores;
|
|
4414
4573
|
}
|
|
4415
4574
|
return bundle;
|
|
@@ -4619,6 +4778,6 @@ var WriteService = class {
|
|
|
4619
4778
|
}
|
|
4620
4779
|
};
|
|
4621
4780
|
|
|
4622
|
-
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
4623
|
-
//# sourceMappingURL=chunk-
|
|
4624
|
-
//# sourceMappingURL=chunk-
|
|
4781
|
+
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
4782
|
+
//# sourceMappingURL=chunk-DPFM33WV.mjs.map
|
|
4783
|
+
//# sourceMappingURL=chunk-DPFM33WV.mjs.map
|