@wrongstack/vector-memory 0.308.0 → 0.308.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -0
- package/dist/index.d.ts +7 -1
- package/dist/index.js +926 -67
- package/dist/sage-event-mirror.d.ts +47 -0
- package/dist/sage-fusion.d.ts +87 -0
- package/dist/sage-port-wrapper.d.ts +53 -0
- package/dist/sage-sync-source.d.ts +39 -0
- package/dist/sage-sync.d.ts +55 -0
- package/dist/schema.d.ts +29 -5
- package/dist/search-race.d.ts +75 -0
- package/dist/store.d.ts +78 -1
- package/dist/types.d.ts +15 -0
- package/package.json +3 -3
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { MemoryPort } from '@wrongstack/core/types';
|
|
2
|
+
import type { VectorMemoryStore } from './store.js';
|
|
3
|
+
export interface VectorMemoryMirrorOptions {
|
|
4
|
+
store: VectorMemoryStore;
|
|
5
|
+
memoryStore: MemoryPort;
|
|
6
|
+
logger?: {
|
|
7
|
+
debug?(msg: string, ctx?: unknown): void | undefined;
|
|
8
|
+
warn?(msg: string, ctx?: unknown): void | undefined;
|
|
9
|
+
} | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* Disable the listener (e.g. in tests where the bus is shared across
|
|
12
|
+
* suites). Returns a no-op disposer.
|
|
13
|
+
*/
|
|
14
|
+
enabled?: boolean | undefined;
|
|
15
|
+
}
|
|
16
|
+
export interface VectorMemoryMirrorHandle {
|
|
17
|
+
/** Idempotent — calling twice is a no-op. */
|
|
18
|
+
dispose: () => void;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Subscribe the vector store to the SAGE event bus so live writes are
|
|
22
|
+
* mirrored. Returns a handle whose `dispose()` removes all listeners.
|
|
23
|
+
*
|
|
24
|
+
* Idempotency:
|
|
25
|
+
* - `remember()` content-hash dedupes, so a re-fire of `memory.accepted`
|
|
26
|
+
* with identical text is a no-op.
|
|
27
|
+
* - The mirror skips session-scoped memories (privacy parity with
|
|
28
|
+
* `createSageSurfaceSyncSource`).
|
|
29
|
+
* - Errors are logged and swallowed; the SAGE write path is never blocked.
|
|
30
|
+
*/
|
|
31
|
+
export declare function subscribeVectorMemoryToSage(opts: VectorMemoryMirrorOptions): VectorMemoryMirrorHandle;
|
|
32
|
+
/**
|
|
33
|
+
* Garbage-collect vector entries whose underlying SAGE memory is gone.
|
|
34
|
+
* Walks the store, looks up each `metadata.sageId` in the SAGE surface,
|
|
35
|
+
* and forgets entries whose SAGE id no longer resolves.
|
|
36
|
+
*
|
|
37
|
+
* Useful after bulk operations (`memory.cleared`, `hygiene.purge_deleted`)
|
|
38
|
+
* that emit a single top-level event but may not emit per-memory
|
|
39
|
+
* `memory.deleted` events.
|
|
40
|
+
*/
|
|
41
|
+
export declare function forgetStaleSageMirrors(store: VectorMemoryStore, memoryStore: MemoryPort, logger?: {
|
|
42
|
+
warn?(msg: string, ctx?: unknown): void | undefined;
|
|
43
|
+
}): Promise<{
|
|
44
|
+
scanned: number;
|
|
45
|
+
removed: number;
|
|
46
|
+
}>;
|
|
47
|
+
//# sourceMappingURL=sage-event-mirror.d.ts.map
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SAGE ⇄ Vector Memory fusion — combine a SAGE lexical candidate set with
|
|
3
|
+
* a parallel semantic recall from the vector store. Operates without
|
|
4
|
+
* modifying the SAGE store; the caller supplies both inputs and gets back
|
|
5
|
+
* a ranked, deduplicated list keyed by the SAGE memory id.
|
|
6
|
+
*
|
|
7
|
+
* The fusion is a Reciprocal Rank Fusion (RRF)-style blend rather than a
|
|
8
|
+
* raw cosine-blend, so it stays robust when one side returns nothing
|
|
9
|
+
* (e.g. embedding provider unavailable). All inputs are optional:
|
|
10
|
+
* - `lexical` may be `[]` → fusion falls back to pure vector ranking.
|
|
11
|
+
* - `vector` may be `[]` → fusion falls back to pure lexical ranking.
|
|
12
|
+
* - either may be `undefined` → fusion silently drops that channel.
|
|
13
|
+
*
|
|
14
|
+
* The vector store is queried separately via `vectorStore.search(query)`,
|
|
15
|
+
* so callers that have already done so can pass the hits directly. When
|
|
16
|
+
* `vectorHits` is omitted, the function queries the store itself.
|
|
17
|
+
*/
|
|
18
|
+
import type { Sage, VectorRecallProvider } from '@wrongstack/sage';
|
|
19
|
+
import type { VectorMemoryStore } from './store.js';
|
|
20
|
+
import type { VectorSearchHit } from './types.js';
|
|
21
|
+
export interface SageFusionOptions {
|
|
22
|
+
/** Vector store queried for additional semantic recall. Optional. */
|
|
23
|
+
store?: VectorMemoryStore | undefined;
|
|
24
|
+
/** Pre-computed vector hits. Skip the embedded `store.search()` call. */
|
|
25
|
+
vectorHits?: readonly VectorSearchHit[] | undefined;
|
|
26
|
+
/**
|
|
27
|
+
* Weight of the vector channel in the final score. 0 = pure lexical
|
|
28
|
+
* order, 1 = pure vector order. Default 0.3.
|
|
29
|
+
*/
|
|
30
|
+
vectorWeight?: number | undefined;
|
|
31
|
+
/**
|
|
32
|
+
* Reciprocal-Rank-Fusion k constant. Lower values amplify the top hits
|
|
33
|
+
* of each channel; higher values flatten the curve. Default 60.
|
|
34
|
+
*/
|
|
35
|
+
rrfK?: number | undefined;
|
|
36
|
+
/** Cosine threshold forwarded to `store.search()` when querying internally. */
|
|
37
|
+
threshold?: number | undefined;
|
|
38
|
+
/** Hard cap on the final fused result list. */
|
|
39
|
+
limit?: number | undefined;
|
|
40
|
+
/**
|
|
41
|
+
* Cosine threshold below which a vector-only hit is dropped (no lexical
|
|
42
|
+
* counterpart to lift it). Default 0 — keep all, let RRF decide.
|
|
43
|
+
*/
|
|
44
|
+
vectorOnlyThreshold?: number | undefined;
|
|
45
|
+
}
|
|
46
|
+
export interface SageFusionHit {
|
|
47
|
+
memory: Sage;
|
|
48
|
+
/** 0..1, 0 if the candidate only appeared in the lexical channel. */
|
|
49
|
+
vectorScore: number | null;
|
|
50
|
+
/** 0..1, 0 if the candidate only appeared in the vector channel. */
|
|
51
|
+
lexicalScore: number | null;
|
|
52
|
+
/** 0..1, monotonically higher = better. */
|
|
53
|
+
finalScore: number;
|
|
54
|
+
/** Where the candidate came from. */
|
|
55
|
+
source: 'lexical' | 'vector' | 'both';
|
|
56
|
+
}
|
|
57
|
+
declare function lexicalRankScore(index: number, total: number): number;
|
|
58
|
+
declare function vectorRankScore(index: number, total: number): number;
|
|
59
|
+
/**
|
|
60
|
+
* Fuse lexical and vector recall into a single ranked list. The vector
|
|
61
|
+
* channel pulls additional memories the lexical index missed (semantic
|
|
62
|
+
* recall), and the lexical channel pulls precise matches the embedding
|
|
63
|
+
* model would under-rank (rare-token recall).
|
|
64
|
+
*
|
|
65
|
+
* Returned order is `finalScore` descending, capped at `limit`.
|
|
66
|
+
*/
|
|
67
|
+
export declare function fuseWithVectorMemory(query: string, lexical: readonly Sage[], options?: SageFusionOptions): Promise<SageFusionHit[]>;
|
|
68
|
+
/**
|
|
69
|
+
* Wrap a `VectorMemoryStore` as a `VectorRecallProvider` so it can be
|
|
70
|
+
* passed to `searchSage({ vectorRecall })` and friends. The adapter is
|
|
71
|
+
* intentionally thin: it forwards `search` to `store.search()` and
|
|
72
|
+
* flattens the result into the structural shape the sage retrieval
|
|
73
|
+
* module expects.
|
|
74
|
+
*
|
|
75
|
+
* Caller responsibility: ensure the vector store has been warm-started
|
|
76
|
+
* via `startFirstBootSageSync()` (or equivalent) so vector hits carry
|
|
77
|
+
* `metadata.sageId` and can be fused with the lexical list.
|
|
78
|
+
*/
|
|
79
|
+
export declare function asVectorRecallProvider(store: VectorMemoryStore): VectorRecallProvider;
|
|
80
|
+
declare function clamp01(value: number): number;
|
|
81
|
+
export declare const __testing: {
|
|
82
|
+
lexicalRankScore: typeof lexicalRankScore;
|
|
83
|
+
vectorRankScore: typeof vectorRankScore;
|
|
84
|
+
clamp01: typeof clamp01;
|
|
85
|
+
};
|
|
86
|
+
export {};
|
|
87
|
+
//# sourceMappingURL=sage-fusion.d.ts.map
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wrap an existing SAGE `MemoryPort` so that every `searchSage` /
|
|
3
|
+
* `unifiedSearch` / `retrieveForAudience` call automatically injects
|
|
4
|
+
* the supplied `VectorRecallProvider`. The wrapper preserves the
|
|
5
|
+
* underlying port's identity for callers that compare ports, but routes
|
|
6
|
+
* the read-side capability methods through a vector-augmented
|
|
7
|
+
* `searchSage`.
|
|
8
|
+
*
|
|
9
|
+
* Why a wrapper and not a direct constructor change:
|
|
10
|
+
* - non-invasive: no migration needed for existing host construction
|
|
11
|
+
* - opt-in: hosts that don't want vector augmentation just don't wrap
|
|
12
|
+
* - testable: easy to mock the wrapper in unit tests
|
|
13
|
+
*
|
|
14
|
+
* The wrapper only augments paths that go through the read-side
|
|
15
|
+
* capability (`getCapability(SAGE_RETRIEVAL_CAPABILITY)` /
|
|
16
|
+
* `getCapability(SAGE_SURFACE_CAPABILITY)`). Other capabilities
|
|
17
|
+
* (write-side, hygiene, audit) pass through unchanged so the wrapper
|
|
18
|
+
* never widens the trust boundary.
|
|
19
|
+
*/
|
|
20
|
+
import type { MemoryPort } from '@wrongstack/core/types';
|
|
21
|
+
import { type VectorRecallProvider } from '@wrongstack/sage';
|
|
22
|
+
import type { VectorMemoryStore } from './store.js';
|
|
23
|
+
export interface VectorPortWrappingOptions {
|
|
24
|
+
/** Vector store. The wrapper adapts it to the SAGE recall contract. */
|
|
25
|
+
store: VectorMemoryStore;
|
|
26
|
+
/**
|
|
27
|
+
* Optional pre-built provider. When omitted, the wrapper builds one via
|
|
28
|
+
* `asVectorRecallProvider(store)`.
|
|
29
|
+
*/
|
|
30
|
+
vectorRecall?: VectorRecallProvider | undefined;
|
|
31
|
+
/**
|
|
32
|
+
* Cosine threshold forwarded to the vector backend. 0 = no threshold
|
|
33
|
+
* (keep all hits, let RRF decide).
|
|
34
|
+
*/
|
|
35
|
+
threshold?: number | undefined;
|
|
36
|
+
/**
|
|
37
|
+
* Weight of the vector channel in the RRF blend. Default 0.3.
|
|
38
|
+
*/
|
|
39
|
+
weight?: number | undefined;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Adapt a `VectorMemoryStore` to the SAGE `VectorRecallProvider` contract.
|
|
43
|
+
* Exported so the wrapper can be used directly by hosts that want a
|
|
44
|
+
* custom provider without wrapping the whole port.
|
|
45
|
+
*/
|
|
46
|
+
export declare function asVectorRecallProviderAdapter(store: VectorMemoryStore): VectorRecallProvider;
|
|
47
|
+
/**
|
|
48
|
+
* Return a new `MemoryPort` that routes `searchSage` calls through the
|
|
49
|
+
* supplied `VectorRecallProvider`. All other capabilities are passed
|
|
50
|
+
* through unchanged.
|
|
51
|
+
*/
|
|
52
|
+
export declare function wrapMemoryPortWithVectorRecall(port: MemoryPort, options: VectorPortWrappingOptions): MemoryPort;
|
|
53
|
+
//# sourceMappingURL=sage-port-wrapper.d.ts.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SageSyncSource adapter over a SAGE `SageSurface`.
|
|
3
|
+
*
|
|
4
|
+
* Cursor-walks `listSagePage({ statuses: ['active'] })` so the whole active
|
|
5
|
+
* corpus can be mirrored into the vector store via `syncFromSage()` without
|
|
6
|
+
* the caller hand-rolling pagination. The walk terminates naturally when
|
|
7
|
+
* `nextCursor` is empty (the canonical "end of corpus" signal) — the
|
|
8
|
+
* historical 5 000 hard cap was removed because it silently truncated
|
|
9
|
+
* large projects, and `nextCursor` already bounds the iteration.
|
|
10
|
+
*
|
|
11
|
+
* A `maxTotal` cap remains available for tests and pathological cases
|
|
12
|
+
* (e.g. a runaway enumerator that never reports `nextCursor: null`); the
|
|
13
|
+
* default is to walk the whole corpus.
|
|
14
|
+
*
|
|
15
|
+
* Privacy note: `sessionId`/`includeAllSessions` are deliberately NOT set.
|
|
16
|
+
* Every SAGE retrieval surface hides owned session-scoped memories from an
|
|
17
|
+
* enumerator that omits them — this adapter keeps that under-report default
|
|
18
|
+
* so one session's private records are never mirrored into the shared,
|
|
19
|
+
* project-scoped vector store.
|
|
20
|
+
*/
|
|
21
|
+
import type { SageSurface } from '@wrongstack/sage';
|
|
22
|
+
import type { SageSyncSource } from './store.js';
|
|
23
|
+
export interface SageSurfaceSyncOptions {
|
|
24
|
+
/**
|
|
25
|
+
* Hard cap on total mirrored memories. Omit (or set to `Infinity`) to
|
|
26
|
+
* walk the entire active corpus. Useful for tests that want a bounded
|
|
27
|
+
* walk without depending on `nextCursor` termination.
|
|
28
|
+
*/
|
|
29
|
+
maxTotal?: number | undefined;
|
|
30
|
+
/** Page size per `listSagePage` call. Clamped to [1, 500]. */
|
|
31
|
+
pageSize?: number | undefined;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Build a `SageSyncSource` from any SAGE surface exposing `listSagePage`.
|
|
35
|
+
* The surface capability is obtained via `getSageSurface(memoryPort)` from
|
|
36
|
+
* `@wrongstack/sage`; a `Pick` keeps this adapter honest about what it calls.
|
|
37
|
+
*/
|
|
38
|
+
export declare function createSageSurfaceSyncSource(sage: Pick<SageSurface, 'listSagePage'>, opts?: SageSurfaceSyncOptions): SageSyncSource;
|
|
39
|
+
//# sourceMappingURL=sage-sync-source.d.ts.map
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { MemoryPort } from '@wrongstack/core/types';
|
|
2
|
+
import type { VectorMemoryStore } from './store.js';
|
|
3
|
+
/** Sync marker file written next to the vector db (`running` → `complete` phases). */
|
|
4
|
+
export declare const SAGE_SYNC_MARKER_FILENAME = "sage-sync.complete.json";
|
|
5
|
+
export interface SageSyncMarker {
|
|
6
|
+
phase: 'running' | 'complete';
|
|
7
|
+
pid?: number;
|
|
8
|
+
startedAt?: string;
|
|
9
|
+
completedAt?: string;
|
|
10
|
+
providerId?: string;
|
|
11
|
+
scanned?: number;
|
|
12
|
+
indexed?: number;
|
|
13
|
+
skipped?: number;
|
|
14
|
+
failed?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface FirstBootSageSyncOptions {
|
|
17
|
+
store: VectorMemoryStore;
|
|
18
|
+
/** The host MemoryPort — the SAGE surface capability is read from it. */
|
|
19
|
+
memoryStore: MemoryPort;
|
|
20
|
+
/** Structurally compatible with `@wrongstack/core`'s `Logger`. */
|
|
21
|
+
logger?: {
|
|
22
|
+
debug?(msg: string, ctx?: unknown): void | undefined;
|
|
23
|
+
info?(msg: string, ctx?: unknown): void | undefined;
|
|
24
|
+
warn?(msg: string, ctx?: unknown): void | undefined;
|
|
25
|
+
} | undefined;
|
|
26
|
+
/** Tests: inject a custom staleness window. */
|
|
27
|
+
staleAfterMs?: number | undefined;
|
|
28
|
+
/**
|
|
29
|
+
* Tests / hosts: pid liveness probe used before taking over a `running`
|
|
30
|
+
* marker. Defaults to `process.kill(pid, 0)` — throws for a dead pid.
|
|
31
|
+
*/
|
|
32
|
+
pidAlive?: ((pid: number) => boolean) | undefined;
|
|
33
|
+
/**
|
|
34
|
+
* Force a full re-sync, ignoring any `complete` marker from a prior
|
|
35
|
+
* boot. Used by operator-initiated re-sync (CLI flag, slash command)
|
|
36
|
+
* where the user explicitly opted in to replay. The new sync still
|
|
37
|
+
* respects the cursor-walker's own termination and the completion
|
|
38
|
+
* invariant (every entry has a vector) — `force` only bypasses the
|
|
39
|
+
* "already-complete, skip" decision.
|
|
40
|
+
*/
|
|
41
|
+
force?: boolean | undefined;
|
|
42
|
+
}
|
|
43
|
+
export interface FirstBootSageSyncResult {
|
|
44
|
+
synced: boolean;
|
|
45
|
+
reason: string;
|
|
46
|
+
marker?: SageSyncMarker | undefined;
|
|
47
|
+
}
|
|
48
|
+
export declare function startFirstBootSageSync(opts: FirstBootSageSyncOptions): Promise<FirstBootSageSyncResult>;
|
|
49
|
+
interface SyncDecision {
|
|
50
|
+
run: boolean;
|
|
51
|
+
reason: string;
|
|
52
|
+
}
|
|
53
|
+
export declare function decideWhetherToSync(store: VectorMemoryStore, staleAfterMs: number, now?: Date, pidAlive?: (pid: number) => boolean): SyncDecision;
|
|
54
|
+
export {};
|
|
55
|
+
//# sourceMappingURL=sage-sync.d.ts.map
|
package/dist/schema.d.ts
CHANGED
|
@@ -1,19 +1,43 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* SQLite schema for the vector memory store.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* - `entries` — text + metadata, no embedding column
|
|
6
|
-
* - `vectors` — (entry_id, provider_id) PK, raw float32 blob
|
|
4
|
+
* Tables:
|
|
5
|
+
* - `entries` — text + metadata, no embedding column.
|
|
6
|
+
* - `vectors` — (entry_id, provider_id) PK, raw float32 blob.
|
|
7
|
+
* - `embedding_cache` — provider-level text→vector cache so repeated
|
|
8
|
+
* `embed()` calls for the same text skip the ONNX forward pass. Keyed
|
|
9
|
+
* by (content_hash, provider_id, model_dimensions) so a model swap
|
|
10
|
+
* invalidates only the old provider rows on insert — no mixed-vector
|
|
11
|
+
* search. Independent of `entries`, so the cache survives entry deletes.
|
|
12
|
+
* - `schema_meta` — active provider id and dimensions.
|
|
7
13
|
*
|
|
8
14
|
* The vectors table is keyed by (entry_id, provider_id) so a model swap
|
|
9
15
|
* invalidates only the old provider rows on insert — no mixed-vector search.
|
|
10
|
-
* A companion `schema_meta` table records the active provider id and dims.
|
|
11
16
|
*/
|
|
12
17
|
import type { DatabaseSync } from 'node:sqlite';
|
|
13
|
-
export declare const VECTOR_SCHEMA_VERSION =
|
|
18
|
+
export declare const VECTOR_SCHEMA_VERSION = 2;
|
|
14
19
|
export declare const VECTOR_PROVIDER_KEY = "active_provider_id";
|
|
15
20
|
export declare const VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
|
|
16
21
|
export declare function initVectorSchema(db: DatabaseSync): void;
|
|
22
|
+
/**
|
|
23
|
+
* Upsert a vector into the embedding cache. `use_count` increments on
|
|
24
|
+
* conflict so hot texts can be identified later. `last_used_at` advances
|
|
25
|
+
* to the current timestamp on every hit so LRU eviction has something to
|
|
26
|
+
* read.
|
|
27
|
+
*/
|
|
28
|
+
export declare function upsertEmbeddingCache(db: DatabaseSync, row: {
|
|
29
|
+
contentHash: string;
|
|
30
|
+
providerId: string;
|
|
31
|
+
dimensions: number;
|
|
32
|
+
vector: Buffer;
|
|
33
|
+
text: string;
|
|
34
|
+
now: string;
|
|
35
|
+
}): void;
|
|
36
|
+
/**
|
|
37
|
+
* Look up a vector in the cache. Returns the decoded vector and bumps
|
|
38
|
+
* `last_used_at` / `use_count` atomically. Returns undefined on miss.
|
|
39
|
+
*/
|
|
40
|
+
export declare function lookupEmbeddingCache(db: DatabaseSync, contentHash: string, providerId: string, dimensions: number, now: string): Float32Array | undefined;
|
|
17
41
|
/** Encode a Float32Array to a SQLite BLOB (Buffer). */
|
|
18
42
|
export declare function encodeVector(vec: Float32Array): Buffer;
|
|
19
43
|
/** Decode a SQLite BLOB (Buffer or Uint8Array) back to a Float32Array. */
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search-channel race — run a query through SAGE (lexical) and the
|
|
3
|
+
* vector store (semantic) independently and surface the overlap and
|
|
4
|
+
* the channel-specific misses. The point of the dual system is
|
|
5
|
+
* precisely this comparison: the operator can SEE that one channel
|
|
6
|
+
* caught a memory the other missed, and tune the RRF weight
|
|
7
|
+
* accordingly.
|
|
8
|
+
*
|
|
9
|
+
* The output is structured for two consumers:
|
|
10
|
+
* - The WebUI's memory panel uses the `channels` / `overlap` arrays
|
|
11
|
+
* to render a side-by-side breakdown.
|
|
12
|
+
* - The CLI `/memory race <query>` command renders a human-readable
|
|
13
|
+
* summary via `formatSearchRace` in `@wrongstack/cli`.
|
|
14
|
+
*
|
|
15
|
+
* Race is read-only. No writes to either store.
|
|
16
|
+
*/
|
|
17
|
+
import type { Sage } from '@wrongstack/sage';
|
|
18
|
+
import type { VectorMemoryStore } from './store.js';
|
|
19
|
+
export interface SearchRaceChannelHit {
|
|
20
|
+
/** SAGE memory id (the join key — both channels index the same corpus). */
|
|
21
|
+
id: string;
|
|
22
|
+
/** Lexical rank score (0..1) — null on the vector channel. */
|
|
23
|
+
lexicalScore: number | null;
|
|
24
|
+
/** Vector cosine (0..1) — null on the lexical channel. */
|
|
25
|
+
vectorScore: number | null;
|
|
26
|
+
/** Truncated text preview, for human-readable output. */
|
|
27
|
+
preview: string;
|
|
28
|
+
}
|
|
29
|
+
export interface SearchRaceResult {
|
|
30
|
+
query: string;
|
|
31
|
+
lexicalOnly: SearchRaceChannelHit[];
|
|
32
|
+
vectorOnly: SearchRaceChannelHit[];
|
|
33
|
+
overlap: Array<{
|
|
34
|
+
id: string;
|
|
35
|
+
lexicalScore: number;
|
|
36
|
+
vectorScore: number;
|
|
37
|
+
preview: string;
|
|
38
|
+
}>;
|
|
39
|
+
/**
|
|
40
|
+
* Summary metrics — the whole point of the race is to surface
|
|
41
|
+
* "what would the user have missed if they had only run one
|
|
42
|
+
* channel?" Each ratio is a 0..1 number.
|
|
43
|
+
*/
|
|
44
|
+
metrics: {
|
|
45
|
+
/** Memories found by the lexical channel (incl. overlap). */
|
|
46
|
+
lexicalCount: number;
|
|
47
|
+
/** Memories found by the vector channel (incl. overlap). */
|
|
48
|
+
vectorCount: number;
|
|
49
|
+
/** Memories found by both. */
|
|
50
|
+
overlapCount: number;
|
|
51
|
+
/** `lexicalOnly / lexicalCount` — how much of lexical recall is invisible to vector. */
|
|
52
|
+
lexicalOnlyRatio: number;
|
|
53
|
+
/** `vectorOnly / vectorCount` — how much of vector recall is invisible to lexical. */
|
|
54
|
+
vectorOnlyRatio: number;
|
|
55
|
+
/** `overlapCount / max(lexicalCount, vectorCount)` — agreement rate. */
|
|
56
|
+
agreementRatio: number;
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
export interface SearchRaceOptions {
|
|
60
|
+
/** Cap on each channel's result set. Default 20. */
|
|
61
|
+
limit?: number | undefined;
|
|
62
|
+
/** Cosine threshold forwarded to `vectorStore.search`. Default 0. */
|
|
63
|
+
threshold?: number | undefined;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Run the same query through both channels and compute the overlap.
|
|
67
|
+
* Lexical hits come from the caller (the SAGE surface). Vector hits
|
|
68
|
+
* are queried from the supplied `vectorStore`. Memories are joined
|
|
69
|
+
* by SAGE id, so the race is only meaningful when the vector store
|
|
70
|
+
* has been warm-started via the SAGE mirror.
|
|
71
|
+
*
|
|
72
|
+
* No writes. No events emitted. Safe to run on every TUI refresh.
|
|
73
|
+
*/
|
|
74
|
+
export declare function runSearchRace(query: string, lexical: readonly Sage[], vectorStore: VectorMemoryStore, options?: SearchRaceOptions): Promise<SearchRaceResult>;
|
|
75
|
+
//# sourceMappingURL=search-race.d.ts.map
|
package/dist/store.d.ts
CHANGED
|
@@ -3,15 +3,72 @@ import type { SageSyncReport, VectorEntry, VectorEntryInput, VectorEntryWithVect
|
|
|
3
3
|
export declare class VectorMemoryStore {
|
|
4
4
|
private readonly db;
|
|
5
5
|
private readonly dbPath;
|
|
6
|
+
private readonly rootDir;
|
|
6
7
|
private readonly provider;
|
|
7
8
|
private closed;
|
|
8
9
|
constructor(opts: VectorMemoryStoreOptions);
|
|
10
|
+
/**
|
|
11
|
+
* Absolute path of the store's data directory (the resolved
|
|
12
|
+
* `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
|
|
13
|
+
* to place sidecar state (e.g. the first-boot SAGE sync marker) next to
|
|
14
|
+
* the db instead of re-deriving the path and drifting from it.
|
|
15
|
+
*/
|
|
16
|
+
get directory(): string;
|
|
17
|
+
/**
|
|
18
|
+
* Absolute path of the SQLite database file. Hosts use this to take a
|
|
19
|
+
* file-level lock that covers all mutating operations (see
|
|
20
|
+
* `withFileLock(this.dbPath + '.lock', …)`).
|
|
21
|
+
*/
|
|
22
|
+
get databasePath(): string;
|
|
23
|
+
/** The lockfile path used to serialize mutating operations. */
|
|
24
|
+
get lockPath(): string;
|
|
9
25
|
get activeProviderId(): string;
|
|
10
26
|
private recordActiveProvider;
|
|
11
27
|
static contentHash(text: string): string;
|
|
28
|
+
/**
|
|
29
|
+
* Look up a vector for `text` in the provider-level embedding cache.
|
|
30
|
+
* Cache hit returns the cached vector (no ONNX pass). Miss returns
|
|
31
|
+
* `undefined`.
|
|
32
|
+
*/
|
|
33
|
+
private cachedVector;
|
|
34
|
+
/** Persist `vec` for `text` to the embedding cache. */
|
|
35
|
+
private cacheVector;
|
|
36
|
+
/**
|
|
37
|
+
* Embed `text`, hitting the provider-level cache first. Cache miss falls
|
|
38
|
+
* through to the configured provider and writes the result back. Returns
|
|
39
|
+
* `undefined` when the provider fails — the caller can persist the entry
|
|
40
|
+
* without a vector (fail-open).
|
|
41
|
+
*/
|
|
42
|
+
private embedWithCache;
|
|
43
|
+
/**
|
|
44
|
+
* Look up an existing entry by `content_hash`. Returns `undefined` when
|
|
45
|
+
* the entry is not present. Used by `remember()` to make writes idempotent
|
|
46
|
+
* and by `syncFromSage()` to skip already-indexed SAGE memories.
|
|
47
|
+
*/
|
|
48
|
+
findByContentHash(contentHash: string): VectorEntryWithVector | undefined;
|
|
49
|
+
/**
|
|
50
|
+
* Look up the entry mirroring a given SAGE memory id (i.e. the entry
|
|
51
|
+
* whose `metadata.sageId` equals `sageId`). Returns `undefined` when
|
|
52
|
+
* no such entry exists. Used by the event-driven mirror to delete
|
|
53
|
+
* vector entries on SAGE delete events (the emitter knows the SAGE
|
|
54
|
+
* id, not the vector entry id).
|
|
55
|
+
*
|
|
56
|
+
* Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
|
|
57
|
+
* column is the JSON blob `syncFromSage` writes, so this avoids a
|
|
58
|
+
* full table scan.
|
|
59
|
+
*/
|
|
60
|
+
findBySageId(sageId: string): VectorEntryWithVector | undefined;
|
|
61
|
+
/**
|
|
62
|
+
* Persist a new entry. Idempotent: if an entry with the same
|
|
63
|
+
* `content_hash` already exists, that entry is returned unchanged
|
|
64
|
+
* instead of inserting a duplicate. Mutating ops are wrapped in
|
|
65
|
+
* `withFileLock` so two processes cannot race the dedup check.
|
|
66
|
+
*/
|
|
12
67
|
remember(input: VectorEntryInput): Promise<VectorEntryWithVector>;
|
|
68
|
+
private rememberUnlocked;
|
|
13
69
|
get(id: string): VectorEntryWithVector | undefined;
|
|
14
|
-
|
|
70
|
+
/** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
|
|
71
|
+
forget(id: string): Promise<boolean>;
|
|
15
72
|
search(query: string, opts?: VectorSearchOptions): Promise<VectorSearchHit[]>;
|
|
16
73
|
list(opts?: {
|
|
17
74
|
limit?: number;
|
|
@@ -23,6 +80,26 @@ export declare class VectorMemoryStore {
|
|
|
23
80
|
errors: number;
|
|
24
81
|
}>;
|
|
25
82
|
stats(): VectorStoreStats;
|
|
83
|
+
/**
|
|
84
|
+
* Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
|
|
85
|
+
* Useful for the WebUI's vector-memory panel and for diagnosing the
|
|
86
|
+
* "why is search slow?" question.
|
|
87
|
+
*/
|
|
88
|
+
cacheStats(): {
|
|
89
|
+
entries: number;
|
|
90
|
+
providers: number;
|
|
91
|
+
totalUseCount: number;
|
|
92
|
+
oldestLastUsedAt: string | null;
|
|
93
|
+
};
|
|
94
|
+
/**
|
|
95
|
+
* LRU-evict the embedding cache down to `keepMostRecent` rows. The
|
|
96
|
+
* `embedding_cache` table is independent of `entries`, so a sweep here
|
|
97
|
+
* only removes cached vectors — never stored entries. Called by hosts
|
|
98
|
+
* that want to bound cache growth on long-lived processes.
|
|
99
|
+
*/
|
|
100
|
+
evictCache(keepMostRecent: number): Promise<{
|
|
101
|
+
removed: number;
|
|
102
|
+
}>;
|
|
26
103
|
close(): void;
|
|
27
104
|
private assertOpen;
|
|
28
105
|
private rowToEntry;
|
package/dist/types.d.ts
CHANGED
|
@@ -48,12 +48,27 @@ export interface VectorSearchOptions {
|
|
|
48
48
|
kind?: VectorKind | undefined;
|
|
49
49
|
/** Provider id override for the query embedding. Defaults to the store's provider. */
|
|
50
50
|
providerId?: string | undefined;
|
|
51
|
+
/**
|
|
52
|
+
* When true, each returned hit also carries the decoded embedding
|
|
53
|
+
* vector. Off by default to keep the response small. Used by the
|
|
54
|
+
* webui-server's `/api/vector-memory/search?similarity=1` route to
|
|
55
|
+
* build the pairwise-similarity heatmap.
|
|
56
|
+
*/
|
|
57
|
+
includeVectors?: boolean | undefined;
|
|
51
58
|
}
|
|
52
59
|
export interface VectorSearchHit {
|
|
53
60
|
entry: VectorEntry;
|
|
54
61
|
/** Cosine similarity in [-1, 1]. The store clamps to [0, 1] before returning. */
|
|
55
62
|
score: number;
|
|
56
63
|
providerId: string;
|
|
64
|
+
/**
|
|
65
|
+
* Decoded embedding vector for the matched entry. Populated only when
|
|
66
|
+
* the caller passes `includeVectors: true` to `store.search()` — keeps
|
|
67
|
+
* the default response small. Useful for downstream consumers that
|
|
68
|
+
* need pairwise similarity (e.g. WebUI heatmap) without a second
|
|
69
|
+
* `get(id)` round-trip.
|
|
70
|
+
*/
|
|
71
|
+
vector?: Float32Array | undefined;
|
|
57
72
|
}
|
|
58
73
|
export interface VectorStoreStats {
|
|
59
74
|
entries: number;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wrongstack/vector-memory",
|
|
3
|
-
"version": "0.308.
|
|
3
|
+
"version": "0.308.2",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "WrongStack Vector Memory — an additional vector-search memory store powered by @huggingface/transformers (local ONNX embeddings), alongside the SAGE lexical memory system.",
|
|
6
6
|
"repository": {
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"README.md"
|
|
28
28
|
],
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@wrongstack/
|
|
31
|
-
"@wrongstack/
|
|
30
|
+
"@wrongstack/sage": "0.308.2",
|
|
31
|
+
"@wrongstack/core": "0.308.2"
|
|
32
32
|
},
|
|
33
33
|
"optionalDependencies": {
|
|
34
34
|
"@huggingface/transformers": "^4.2.0"
|