@equationalapplications/core-llm-wiki 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/index.d.mts +57 -1
- package/dist/index.d.ts +57 -1
- package/dist/index.js +173 -30
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +173 -31
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -302,6 +302,39 @@ wikiMemory.clearVectorCache();
|
|
|
302
302
|
|
|
303
303
|
The cache is also automatically invalidated on any mutation (`runLibrarian`, `runHeal`, `runPrune`, `runReembed`, `ingestDocument`, `importDump`, `forget`).
|
|
304
304
|
|
|
305
|
+
## Security
|
|
306
|
+
|
|
307
|
+
`@equationalapplications/core-llm-wiki` enforces multiple security layers:
|
|
308
|
+
|
|
309
|
+
### VectorRanker Adapter Security
|
|
310
|
+
|
|
311
|
+
If implementing a custom `VectorRanker`:
|
|
312
|
+
|
|
313
|
+
- **SQL Injection**: ALWAYS use parameterized queries for `entityId`, `factId`, `candidateIds`. Never concatenate into SQL strings.
|
|
314
|
+
- **Entity Isolation**: Filter by `entityId` in all queries to prevent cross-tenant data leaks.
|
|
315
|
+
- **Credential Scrubbing**: Strip API keys, tokens, connection strings from thrown errors before surfacing to host.
|
|
316
|
+
- **Resource Limits**: Cap `limit` and `candidateIds.length` to prevent DoS. Do NOT retain `vector` references beyond callback scope — blocks GC.
|
|
317
|
+
|
|
318
|
+
See [SECURITY.md](../../SECURITY.md) for complete adapter security guidance and code examples.
|
|
319
|
+
|
|
320
|
+
### Host Application Security
|
|
321
|
+
|
|
322
|
+
When using `VectorRanker`:
|
|
323
|
+
|
|
324
|
+
- **Error Sanitization**: `sanitizeRankerErrors: true` (default) scrubs ranker errors before mirroring via `error.cause`.
|
|
325
|
+
- **Fallback Policy**: Choose `vectorRankerFallback` based on availability vs consistency requirements:
|
|
326
|
+
- `'js-cosine'` (default): Best availability
|
|
327
|
+
- `'keyword'`: Fast fallback without semantic ranking
|
|
328
|
+
- `'empty'`: Strict consistency (no facts on failure)
|
|
329
|
+
- `'throw'`: Fail-fast error propagation
|
|
330
|
+
- **Deletion Hook Contract**: `forget()` / `runPrune()` reject on hook timeout/failure. Prevents GDPR violations (deleted vectors still retrievable). Handle failures with retry or queue for reconciliation.
|
|
331
|
+
- **Timeout Tuning**: Set `deletionHookTimeoutMs` per deployment (default 30s). Interactive UX: 5s. Background jobs: 60s.
|
|
332
|
+
|
|
333
|
+
Core WikiMemory provides:
|
|
334
|
+
- **Defensive Copies**: Query/embedding vectors copied before ranker/hook calls
|
|
335
|
+
- **Input Validation**: `sourceRef`/`sourceHash` normalized; embedding dimensions validated
|
|
336
|
+
- **Parameterized Queries**: All SQL uses bind parameters
|
|
337
|
+
|
|
305
338
|
## Usage
|
|
306
339
|
|
|
307
340
|
```typescript
|
package/dist/index.d.mts
CHANGED
|
@@ -145,6 +145,11 @@ interface VectorRankerSemanticResult {
|
|
|
145
145
|
*/
|
|
146
146
|
interface VectorRankerRankArgs {
|
|
147
147
|
entityId: string;
|
|
148
|
+
/**
|
|
149
|
+
* Query embedding. Treat as readonly — core provides a defensive copy,
|
|
150
|
+
* but adapters MUST NOT mutate this array. Mutation can corrupt
|
|
151
|
+
* WikiMemory's internal vector cache and JS-cosine fallback path.
|
|
152
|
+
*/
|
|
148
153
|
queryVec: Float32Array | number[];
|
|
149
154
|
/**
|
|
150
155
|
* When set (MiniSearch pre-filter path): ranker MUST only produce results for ids in this set.
|
|
@@ -172,6 +177,13 @@ interface VectorRanker {
|
|
|
172
177
|
/**
|
|
173
178
|
* Called after a fact's embedding is successfully persisted to embedding_blob (or cleared).
|
|
174
179
|
* Hosts use this to keep sqlite-vec / external indexes consistent with SQLite as source of truth.
|
|
180
|
+
*
|
|
181
|
+
* On deletion paths (forget, prune, hard-delete), core awaits this hook to ensure ANN cleanup
|
|
182
|
+
* completes before the deletion call resolves (GDPR compliance). Hook failures or timeouts on
|
|
183
|
+
* those paths reject the deletion call.
|
|
184
|
+
*
|
|
185
|
+
* Treat `vector` as readonly — core provides a defensive copy, but adapters MUST NOT mutate.
|
|
186
|
+
*
|
|
175
187
|
* Optional: if omitted, hosts MUST document "index rebuilt separately" and accept stale ANN until rebuild.
|
|
176
188
|
*/
|
|
177
189
|
onEmbeddingPersisted?(event: {
|
|
@@ -226,6 +238,29 @@ interface WikiOptions {
|
|
|
226
238
|
* Ignored when vectorRankerFallback is 'throw'. Default false.
|
|
227
239
|
*/
|
|
228
240
|
propagateRankerFailureToRetrievalFallback?: boolean;
|
|
241
|
+
/**
|
|
242
|
+
* When true (default), sanitize ranker errors before exposing via error.cause
|
|
243
|
+
* to prevent credential leakage in host telemetry. Disable only when you
|
|
244
|
+
* control the ranker implementation.
|
|
245
|
+
*
|
|
246
|
+
* Sanitization replaces error message/stack with a generic message preserving
|
|
247
|
+
* only the error type (constructor name).
|
|
248
|
+
*/
|
|
249
|
+
sanitizeRankerErrors?: boolean;
|
|
250
|
+
/**
|
|
251
|
+
* Timeout (ms) for onEmbeddingPersisted hook on GDPR deletion paths
|
|
252
|
+
* (forget, _doPrune). Hook must complete within this window or the
|
|
253
|
+
* deletion operation rejects. Default 30000.
|
|
254
|
+
* Lower for interactive deletes; raise for slow remote ANN backends.
|
|
255
|
+
*/
|
|
256
|
+
deletionHookTimeoutMs?: number;
|
|
257
|
+
/**
|
|
258
|
+
* Escape hatch: skip onEmbeddingPersisted on deletion paths entirely.
|
|
259
|
+
* Use ONLY when the ANN backend is permanently decommissioned. Vectors
|
|
260
|
+
* orphaned in the (unreachable) external index are accepted as a tradeoff.
|
|
261
|
+
* NOT GDPR-safe for live indexes. Default false.
|
|
262
|
+
*/
|
|
263
|
+
forceDeleteIgnoreRankerHook?: boolean;
|
|
229
264
|
}
|
|
230
265
|
interface MemoryBundle {
|
|
231
266
|
facts: WikiFact[];
|
|
@@ -278,6 +313,15 @@ declare class WikiBusyError extends Error {
|
|
|
278
313
|
readonly entityId: string;
|
|
279
314
|
constructor(operation: WikiBusyOperation, entityId: string);
|
|
280
315
|
}
|
|
316
|
+
declare class PrunePartialFailureError extends Error {
|
|
317
|
+
readonly deleted: number;
|
|
318
|
+
readonly failedAt: string;
|
|
319
|
+
readonly remaining: number;
|
|
320
|
+
readonly deletedTasks: number;
|
|
321
|
+
readonly deletedEvents: number;
|
|
322
|
+
readonly cause: Error;
|
|
323
|
+
constructor(deleted: number, failedAt: string, remaining: number, cause: Error, deletedTasks?: number, deletedEvents?: number);
|
|
324
|
+
}
|
|
281
325
|
|
|
282
326
|
declare class WikiMemory {
|
|
283
327
|
private db;
|
|
@@ -315,6 +359,12 @@ declare class WikiMemory {
|
|
|
315
359
|
private _healKey;
|
|
316
360
|
private _warnCrossEntityCollision;
|
|
317
361
|
private _notifyEmbeddingPersisted;
|
|
362
|
+
/**
|
|
363
|
+
* GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
|
|
364
|
+
* Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
|
|
365
|
+
* calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
|
|
366
|
+
*/
|
|
367
|
+
private _notifyEmbeddingPersistedOrThrow;
|
|
318
368
|
constructor(db: SQLiteAdapter, options: WikiOptions);
|
|
319
369
|
setup(): Promise<void>;
|
|
320
370
|
hasChanged(entityId: string, sourceRef: string, sourceHash: string): Promise<boolean>;
|
|
@@ -351,6 +401,12 @@ declare class WikiMemory {
|
|
|
351
401
|
* Negative return means "a ranks ahead of b" for descending score order.
|
|
352
402
|
*/
|
|
353
403
|
private _compareScoredRows;
|
|
404
|
+
/**
|
|
405
|
+
* Strip potentially sensitive data from ranker errors before exposing to host callbacks.
|
|
406
|
+
* Preserves error type for debugging but removes message/stack that may contain credentials.
|
|
407
|
+
* Recursively sanitizes one level of .cause; deeper chains collapse to type only.
|
|
408
|
+
*/
|
|
409
|
+
private _sanitizeRankerError;
|
|
354
410
|
/**
|
|
355
411
|
* Score candidate rows using in-process JS cosine similarity.
|
|
356
412
|
* Applies hybrid blending (if weight set) and tie-break sorting before returning.
|
|
@@ -416,4 +472,4 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
|
|
|
416
472
|
|
|
417
473
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
418
474
|
|
|
419
|
-
export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
|
|
475
|
+
export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
|
package/dist/index.d.ts
CHANGED
|
@@ -145,6 +145,11 @@ interface VectorRankerSemanticResult {
|
|
|
145
145
|
*/
|
|
146
146
|
interface VectorRankerRankArgs {
|
|
147
147
|
entityId: string;
|
|
148
|
+
/**
|
|
149
|
+
* Query embedding. Treat as readonly — core provides a defensive copy,
|
|
150
|
+
* but adapters MUST NOT mutate this array. Mutation can corrupt
|
|
151
|
+
* WikiMemory's internal vector cache and JS-cosine fallback path.
|
|
152
|
+
*/
|
|
148
153
|
queryVec: Float32Array | number[];
|
|
149
154
|
/**
|
|
150
155
|
* When set (MiniSearch pre-filter path): ranker MUST only produce results for ids in this set.
|
|
@@ -172,6 +177,13 @@ interface VectorRanker {
|
|
|
172
177
|
/**
|
|
173
178
|
* Called after a fact's embedding is successfully persisted to embedding_blob (or cleared).
|
|
174
179
|
* Hosts use this to keep sqlite-vec / external indexes consistent with SQLite as source of truth.
|
|
180
|
+
*
|
|
181
|
+
* On deletion paths (forget, prune, hard-delete), core awaits this hook to ensure ANN cleanup
|
|
182
|
+
* completes before the deletion call resolves (GDPR compliance). Hook failures or timeouts on
|
|
183
|
+
* those paths reject the deletion call.
|
|
184
|
+
*
|
|
185
|
+
* Treat `vector` as readonly — core provides a defensive copy, but adapters MUST NOT mutate.
|
|
186
|
+
*
|
|
175
187
|
* Optional: if omitted, hosts MUST document "index rebuilt separately" and accept stale ANN until rebuild.
|
|
176
188
|
*/
|
|
177
189
|
onEmbeddingPersisted?(event: {
|
|
@@ -226,6 +238,29 @@ interface WikiOptions {
|
|
|
226
238
|
* Ignored when vectorRankerFallback is 'throw'. Default false.
|
|
227
239
|
*/
|
|
228
240
|
propagateRankerFailureToRetrievalFallback?: boolean;
|
|
241
|
+
/**
|
|
242
|
+
* When true (default), sanitize ranker errors before exposing via error.cause
|
|
243
|
+
* to prevent credential leakage in host telemetry. Disable only when you
|
|
244
|
+
* control the ranker implementation.
|
|
245
|
+
*
|
|
246
|
+
* Sanitization replaces error message/stack with a generic message preserving
|
|
247
|
+
* only the error type (constructor name).
|
|
248
|
+
*/
|
|
249
|
+
sanitizeRankerErrors?: boolean;
|
|
250
|
+
/**
|
|
251
|
+
* Timeout (ms) for onEmbeddingPersisted hook on GDPR deletion paths
|
|
252
|
+
* (forget, _doPrune). Hook must complete within this window or the
|
|
253
|
+
* deletion operation rejects. Default 30000.
|
|
254
|
+
* Lower for interactive deletes; raise for slow remote ANN backends.
|
|
255
|
+
*/
|
|
256
|
+
deletionHookTimeoutMs?: number;
|
|
257
|
+
/**
|
|
258
|
+
* Escape hatch: skip onEmbeddingPersisted on deletion paths entirely.
|
|
259
|
+
* Use ONLY when the ANN backend is permanently decommissioned. Vectors
|
|
260
|
+
* orphaned in the (unreachable) external index are accepted as a tradeoff.
|
|
261
|
+
* NOT GDPR-safe for live indexes. Default false.
|
|
262
|
+
*/
|
|
263
|
+
forceDeleteIgnoreRankerHook?: boolean;
|
|
229
264
|
}
|
|
230
265
|
interface MemoryBundle {
|
|
231
266
|
facts: WikiFact[];
|
|
@@ -278,6 +313,15 @@ declare class WikiBusyError extends Error {
|
|
|
278
313
|
readonly entityId: string;
|
|
279
314
|
constructor(operation: WikiBusyOperation, entityId: string);
|
|
280
315
|
}
|
|
316
|
+
declare class PrunePartialFailureError extends Error {
|
|
317
|
+
readonly deleted: number;
|
|
318
|
+
readonly failedAt: string;
|
|
319
|
+
readonly remaining: number;
|
|
320
|
+
readonly deletedTasks: number;
|
|
321
|
+
readonly deletedEvents: number;
|
|
322
|
+
readonly cause: Error;
|
|
323
|
+
constructor(deleted: number, failedAt: string, remaining: number, cause: Error, deletedTasks?: number, deletedEvents?: number);
|
|
324
|
+
}
|
|
281
325
|
|
|
282
326
|
declare class WikiMemory {
|
|
283
327
|
private db;
|
|
@@ -315,6 +359,12 @@ declare class WikiMemory {
|
|
|
315
359
|
private _healKey;
|
|
316
360
|
private _warnCrossEntityCollision;
|
|
317
361
|
private _notifyEmbeddingPersisted;
|
|
362
|
+
/**
|
|
363
|
+
* GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
|
|
364
|
+
* Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
|
|
365
|
+
* calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
|
|
366
|
+
*/
|
|
367
|
+
private _notifyEmbeddingPersistedOrThrow;
|
|
318
368
|
constructor(db: SQLiteAdapter, options: WikiOptions);
|
|
319
369
|
setup(): Promise<void>;
|
|
320
370
|
hasChanged(entityId: string, sourceRef: string, sourceHash: string): Promise<boolean>;
|
|
@@ -351,6 +401,12 @@ declare class WikiMemory {
|
|
|
351
401
|
* Negative return means "a ranks ahead of b" for descending score order.
|
|
352
402
|
*/
|
|
353
403
|
private _compareScoredRows;
|
|
404
|
+
/**
|
|
405
|
+
* Strip potentially sensitive data from ranker errors before exposing to host callbacks.
|
|
406
|
+
* Preserves error type for debugging but removes message/stack that may contain credentials.
|
|
407
|
+
* Recursively sanitizes one level of .cause; deeper chains collapse to type only.
|
|
408
|
+
*/
|
|
409
|
+
private _sanitizeRankerError;
|
|
354
410
|
/**
|
|
355
411
|
* Score candidate rows using in-process JS cosine similarity.
|
|
356
412
|
* Applies hybrid blending (if weight set) and tie-break sorting before returning.
|
|
@@ -416,4 +472,4 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
|
|
|
416
472
|
|
|
417
473
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
418
474
|
|
|
419
|
-
export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
|
|
475
|
+
export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
|
package/dist/index.js
CHANGED
|
@@ -132,6 +132,18 @@ var WikiBusyError = class extends Error {
|
|
|
132
132
|
this.entityId = entityId;
|
|
133
133
|
}
|
|
134
134
|
};
|
|
135
|
+
var PrunePartialFailureError = class extends Error {
|
|
136
|
+
constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
|
|
137
|
+
super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
|
|
138
|
+
this.name = "PrunePartialFailureError";
|
|
139
|
+
this.deleted = deleted;
|
|
140
|
+
this.failedAt = failedAt;
|
|
141
|
+
this.remaining = remaining;
|
|
142
|
+
this.deletedTasks = deletedTasks;
|
|
143
|
+
this.deletedEvents = deletedEvents;
|
|
144
|
+
this.cause = cause;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
135
147
|
|
|
136
148
|
// src/prompts.ts
|
|
137
149
|
var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
|
|
@@ -198,6 +210,7 @@ function parseEmbedding(blob, text) {
|
|
|
198
210
|
}
|
|
199
211
|
|
|
200
212
|
// src/WikiMemory.ts
|
|
213
|
+
var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
|
|
201
214
|
function parseJsonResponse(text) {
|
|
202
215
|
const firstBrace = text.indexOf("{");
|
|
203
216
|
const firstBracket = text.indexOf("[");
|
|
@@ -581,7 +594,55 @@ var _WikiMemory = class _WikiMemory {
|
|
|
581
594
|
console.warn(`[WikiMemory] importDump: ${type} id "${id}" already belongs to entity "${existingEntityId}"; skipping for entity "${targetEntityId}"`);
|
|
582
595
|
}
|
|
583
596
|
async _notifyEmbeddingPersisted(entityId, factId, vector) {
|
|
584
|
-
|
|
597
|
+
if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
|
|
598
|
+
const vectorCopy = vector ? vector.slice() : null;
|
|
599
|
+
await this.options.vectorRanker.onEmbeddingPersisted({
|
|
600
|
+
entityId,
|
|
601
|
+
factId,
|
|
602
|
+
vector: vectorCopy
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
|
|
607
|
+
* Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
|
|
608
|
+
* calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
|
|
609
|
+
*/
|
|
610
|
+
async _notifyEmbeddingPersistedOrThrow(entityId, factId, vector) {
|
|
611
|
+
if (!this.options.vectorRanker?.onEmbeddingPersisted) return;
|
|
612
|
+
if (this.options.forceDeleteIgnoreRankerHook === true) return;
|
|
613
|
+
const vectorCopy = vector ? vector.slice() : null;
|
|
614
|
+
const rawTimeout = this.options.deletionHookTimeoutMs ?? 3e4;
|
|
615
|
+
if (typeof rawTimeout !== "number" || !Number.isFinite(rawTimeout) || rawTimeout <= 0) {
|
|
616
|
+
throw new Error("Invalid deletionHookTimeoutMs: must be a positive finite number");
|
|
617
|
+
}
|
|
618
|
+
const timeoutMs = rawTimeout;
|
|
619
|
+
let timeoutHandle;
|
|
620
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
621
|
+
timeoutHandle = setTimeout(
|
|
622
|
+
() => {
|
|
623
|
+
const timeoutError = new Error(`onEmbeddingPersisted timed out after ${timeoutMs}ms`);
|
|
624
|
+
timeoutError[HOOK_TIMEOUT_MARKER] = true;
|
|
625
|
+
reject(timeoutError);
|
|
626
|
+
},
|
|
627
|
+
timeoutMs
|
|
628
|
+
);
|
|
629
|
+
});
|
|
630
|
+
const hookPromise = Promise.resolve(
|
|
631
|
+
this.options.vectorRanker.onEmbeddingPersisted({
|
|
632
|
+
entityId,
|
|
633
|
+
factId,
|
|
634
|
+
vector: vectorCopy
|
|
635
|
+
})
|
|
636
|
+
);
|
|
637
|
+
try {
|
|
638
|
+
await Promise.race([hookPromise, timeoutPromise]);
|
|
639
|
+
} catch (err) {
|
|
640
|
+
hookPromise.catch(() => {
|
|
641
|
+
});
|
|
642
|
+
throw err;
|
|
643
|
+
} finally {
|
|
644
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
645
|
+
}
|
|
585
646
|
}
|
|
586
647
|
async setup() {
|
|
587
648
|
const entriesExistedBeforeSetup = await this.db.getFirstAsync(
|
|
@@ -765,27 +826,71 @@ var _WikiMemory = class _WikiMemory {
|
|
|
765
826
|
let deletedEntries = 0;
|
|
766
827
|
let deletedTasks = 0;
|
|
767
828
|
let deletedEvents = 0;
|
|
768
|
-
const deletedEntryIds = [];
|
|
769
829
|
if (retainSoftDeletedFor !== null) {
|
|
770
830
|
const cutoff = now - retainSoftDeletedFor * 864e5;
|
|
771
831
|
const entriesToDelete = await this.db.getAllAsync(
|
|
772
|
-
`SELECT id FROM ${this.prefix}entries
|
|
832
|
+
`SELECT id, entity_id FROM ${this.prefix}entries
|
|
773
833
|
WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
|
|
774
834
|
[entityId, cutoff]
|
|
775
835
|
);
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
836
|
+
const succeeded = [];
|
|
837
|
+
let failure = null;
|
|
838
|
+
for (const row of entriesToDelete) {
|
|
839
|
+
try {
|
|
840
|
+
await this._notifyEmbeddingPersistedOrThrow(row.entity_id, row.id, null);
|
|
841
|
+
succeeded.push({ entity_id: row.entity_id, id: row.id });
|
|
842
|
+
} catch (err) {
|
|
843
|
+
failure = { factId: row.id, cause: err };
|
|
844
|
+
break;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
if (succeeded.length > 0) {
|
|
848
|
+
const chunkSize = 500;
|
|
849
|
+
for (let i = 0; i < succeeded.length; i += chunkSize) {
|
|
850
|
+
const chunk = succeeded.slice(i, i + chunkSize);
|
|
851
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
852
|
+
const entryResult = await this.db.runAsync(
|
|
853
|
+
`DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ? AND id IN (${placeholders})`,
|
|
854
|
+
[entityId, cutoff, ...chunk.map((r) => r.id)]
|
|
855
|
+
);
|
|
856
|
+
deletedEntries += entryResult.changes;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
783
859
|
const taskResult = await this.db.runAsync(
|
|
784
860
|
`DELETE FROM ${this.prefix}tasks
|
|
785
861
|
WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at < ?`,
|
|
786
862
|
[entityId, cutoff]
|
|
787
863
|
);
|
|
788
864
|
deletedTasks = taskResult.changes;
|
|
865
|
+
if (failure) {
|
|
866
|
+
await this.rebuildMiniSearchIndex(entityId);
|
|
867
|
+
this.vectorCache.delete(entityId);
|
|
868
|
+
const remaining = entriesToDelete.length - succeeded.length - 1;
|
|
869
|
+
const isTimeout = failure.cause?.[HOOK_TIMEOUT_MARKER] === true;
|
|
870
|
+
if (isTimeout) {
|
|
871
|
+
throw new PrunePartialFailureError(
|
|
872
|
+
succeeded.length,
|
|
873
|
+
failure.factId,
|
|
874
|
+
remaining,
|
|
875
|
+
new Error("Deletion hook timed out"),
|
|
876
|
+
deletedTasks,
|
|
877
|
+
0
|
|
878
|
+
// events not yet deleted at this point
|
|
879
|
+
);
|
|
880
|
+
}
|
|
881
|
+
const errMsg = failure.cause?.message ?? "";
|
|
882
|
+
const isValidationError = errMsg.startsWith("Invalid deletionHookTimeoutMs");
|
|
883
|
+
const sanitizedCause = isValidationError ? failure.cause : this._sanitizeRankerError(failure.cause);
|
|
884
|
+
throw new PrunePartialFailureError(
|
|
885
|
+
succeeded.length,
|
|
886
|
+
failure.factId,
|
|
887
|
+
remaining,
|
|
888
|
+
sanitizedCause,
|
|
889
|
+
deletedTasks,
|
|
890
|
+
0
|
|
891
|
+
// events not yet deleted at this point
|
|
892
|
+
);
|
|
893
|
+
}
|
|
789
894
|
}
|
|
790
895
|
if (retainEventsFor !== null) {
|
|
791
896
|
const cutoff = now - retainEventsFor * 864e5;
|
|
@@ -802,14 +907,6 @@ var _WikiMemory = class _WikiMemory {
|
|
|
802
907
|
}
|
|
803
908
|
await this.rebuildMiniSearchIndex(entityId);
|
|
804
909
|
this.vectorCache.delete(entityId);
|
|
805
|
-
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
806
|
-
for (const factId of uniqueDeletedIds) {
|
|
807
|
-
try {
|
|
808
|
-
await this._notifyEmbeddingPersisted(entityId, factId, null);
|
|
809
|
-
} catch (hookErr) {
|
|
810
|
-
console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during prune for ${factId}:`, hookErr);
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
910
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
814
911
|
} finally {
|
|
815
912
|
this.activeMaintenanceJobs.delete(pruneKey);
|
|
@@ -1063,7 +1160,10 @@ var _WikiMemory = class _WikiMemory {
|
|
|
1063
1160
|
} catch (rankerErr) {
|
|
1064
1161
|
const rankerError = rankerErr instanceof Error ? rankerErr : new Error(String(rankerErr));
|
|
1065
1162
|
const policy = this.options.vectorRankerFallback ?? "js-cosine";
|
|
1066
|
-
this.options.onVectorRankerFallback?.({
|
|
1163
|
+
this.options.onVectorRankerFallback?.({
|
|
1164
|
+
error: this._sanitizeRankerError(rankerError),
|
|
1165
|
+
policy
|
|
1166
|
+
});
|
|
1067
1167
|
if (policy === "throw") {
|
|
1068
1168
|
rankerShouldRethrow = true;
|
|
1069
1169
|
throw rankerError;
|
|
@@ -1127,8 +1227,9 @@ var _WikiMemory = class _WikiMemory {
|
|
|
1127
1227
|
scored = [];
|
|
1128
1228
|
}
|
|
1129
1229
|
if (this.options.propagateRankerFailureToRetrievalFallback) {
|
|
1130
|
-
const mirrored = new Error("Vector ranker failed, falling back"
|
|
1131
|
-
|
|
1230
|
+
const mirrored = new Error("Vector ranker failed, falling back", {
|
|
1231
|
+
cause: this._sanitizeRankerError(rankerErr)
|
|
1232
|
+
});
|
|
1132
1233
|
pendingRankerFallbackError = mirrored;
|
|
1133
1234
|
}
|
|
1134
1235
|
}
|
|
@@ -1288,12 +1389,31 @@ var _WikiMemory = class _WikiMemory {
|
|
|
1288
1389
|
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
1289
1390
|
return a.id.localeCompare(b.id);
|
|
1290
1391
|
}
|
|
1392
|
+
/**
|
|
1393
|
+
* Strip potentially sensitive data from ranker errors before exposing to host callbacks.
|
|
1394
|
+
* Preserves error type for debugging but removes message/stack that may contain credentials.
|
|
1395
|
+
* Recursively sanitizes one level of .cause; deeper chains collapse to type only.
|
|
1396
|
+
*/
|
|
1397
|
+
_sanitizeRankerError(err) {
|
|
1398
|
+
if (this.options.sanitizeRankerErrors === false) {
|
|
1399
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
1400
|
+
}
|
|
1401
|
+
const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
|
|
1402
|
+
const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
|
|
1403
|
+
const sanitized = new Error(
|
|
1404
|
+
`VectorRanker ${typeName} (message scrubbed for security)`,
|
|
1405
|
+
innerCause ? { cause: innerCause } : void 0
|
|
1406
|
+
);
|
|
1407
|
+
sanitized.name = typeName;
|
|
1408
|
+
return sanitized;
|
|
1409
|
+
}
|
|
1291
1410
|
/**
|
|
1292
1411
|
* Score candidate rows using in-process JS cosine similarity.
|
|
1293
1412
|
* Applies hybrid blending (if weight set) and tie-break sorting before returning.
|
|
1294
1413
|
*/
|
|
1295
1414
|
async _rankWithJsCosine(args) {
|
|
1296
|
-
const
|
|
1415
|
+
const queryVec = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
|
|
1416
|
+
const { entityId, candidateRows, weight, miniSearchScores, populateCache, limit } = args;
|
|
1297
1417
|
let entityCache = this.vectorCache.get(entityId);
|
|
1298
1418
|
const tooLarge = populateCache && candidateRows.length > _WikiMemory.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
|
|
1299
1419
|
if (tooLarge && entityCache) {
|
|
@@ -1344,14 +1464,15 @@ var _WikiMemory = class _WikiMemory {
|
|
|
1344
1464
|
* Returns scored results ready for hybrid blending and tie-break sorting.
|
|
1345
1465
|
*/
|
|
1346
1466
|
async _rankWithVectorRanker(args) {
|
|
1347
|
-
const { entityId,
|
|
1467
|
+
const { entityId, candidateIds, weight, miniSearchScores, limit } = args;
|
|
1348
1468
|
const ranker = this.options.vectorRanker;
|
|
1349
1469
|
if (!ranker) {
|
|
1350
1470
|
throw new Error("vectorRanker not configured");
|
|
1351
1471
|
}
|
|
1472
|
+
const queryVecCopy = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
|
|
1352
1473
|
const rankerResults = await ranker.rankBySimilarity({
|
|
1353
1474
|
entityId,
|
|
1354
|
-
queryVec,
|
|
1475
|
+
queryVec: queryVecCopy,
|
|
1355
1476
|
candidateIds,
|
|
1356
1477
|
limit
|
|
1357
1478
|
});
|
|
@@ -2171,11 +2292,15 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2171
2292
|
let deletedTasks = 0;
|
|
2172
2293
|
const deletedEntryIds = [];
|
|
2173
2294
|
if (params.clearAll) {
|
|
2174
|
-
const
|
|
2295
|
+
const newDeletions = await this.db.getAllAsync(
|
|
2175
2296
|
`SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
2176
2297
|
[entityId]
|
|
2177
2298
|
);
|
|
2178
|
-
|
|
2299
|
+
const alreadySoftDeleted = await this.db.getAllAsync(
|
|
2300
|
+
`SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL`,
|
|
2301
|
+
[entityId]
|
|
2302
|
+
);
|
|
2303
|
+
deletedEntryIds.push(...newDeletions.map((e) => e.id), ...alreadySoftDeleted.map((e) => e.id));
|
|
2179
2304
|
const [entriesRes, tasksRes] = await Promise.all([
|
|
2180
2305
|
this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId]),
|
|
2181
2306
|
this.db.runAsync(`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`, [now, now, entityId])
|
|
@@ -2195,13 +2320,13 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2195
2320
|
if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
2196
2321
|
if (params.entryId) {
|
|
2197
2322
|
const entry = await this.db.getFirstAsync(
|
|
2198
|
-
`SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id =
|
|
2323
|
+
`SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
|
|
2199
2324
|
[params.entryId, entityId]
|
|
2200
2325
|
);
|
|
2201
2326
|
if (entry) deletedEntryIds.push(entry.id);
|
|
2202
2327
|
}
|
|
2203
2328
|
if (sourceRef || sourceHash) {
|
|
2204
|
-
let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id =
|
|
2329
|
+
let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
|
|
2205
2330
|
const args = [entityId];
|
|
2206
2331
|
if (sourceRef) {
|
|
2207
2332
|
q += ` AND source_ref = ?`;
|
|
@@ -2244,9 +2369,26 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2244
2369
|
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
2245
2370
|
for (const factId of uniqueDeletedIds) {
|
|
2246
2371
|
try {
|
|
2247
|
-
await this.
|
|
2372
|
+
await this._notifyEmbeddingPersistedOrThrow(entityId, factId, null);
|
|
2248
2373
|
} catch (hookErr) {
|
|
2249
|
-
|
|
2374
|
+
const isTimeout = hookErr?.[HOOK_TIMEOUT_MARKER] === true;
|
|
2375
|
+
if (isTimeout) {
|
|
2376
|
+
throw new Error(
|
|
2377
|
+
`forget(${entityId}/${factId}) failed: ${hookErr.message}`
|
|
2378
|
+
);
|
|
2379
|
+
}
|
|
2380
|
+
const errMsg = hookErr?.message ?? "";
|
|
2381
|
+
const isValidationError = errMsg.startsWith("Invalid deletionHookTimeoutMs");
|
|
2382
|
+
if (isValidationError) {
|
|
2383
|
+
throw new Error(
|
|
2384
|
+
`forget(${entityId}/${factId}) failed: ${errMsg}`,
|
|
2385
|
+
{ cause: hookErr }
|
|
2386
|
+
);
|
|
2387
|
+
}
|
|
2388
|
+
throw new Error(
|
|
2389
|
+
`forget(${entityId}/${factId}) failed: ANN cleanup hook rejected`,
|
|
2390
|
+
{ cause: this._sanitizeRankerError(hookErr) }
|
|
2391
|
+
);
|
|
2250
2392
|
}
|
|
2251
2393
|
}
|
|
2252
2394
|
return { deleted: { entries: deletedEntries, tasks: deletedTasks } };
|
|
@@ -2595,6 +2737,7 @@ function createWiki(db, options) {
|
|
|
2595
2737
|
return new WikiMemory(db, options);
|
|
2596
2738
|
}
|
|
2597
2739
|
|
|
2740
|
+
exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
2598
2741
|
exports.WikiBusyError = WikiBusyError;
|
|
2599
2742
|
exports.WikiMemory = WikiMemory;
|
|
2600
2743
|
exports.createWiki = createWiki;
|