@monoes/memory 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +902 -0
- package/dist/agent-memory-scope.d.ts +131 -0
- package/dist/agent-memory-scope.d.ts.map +1 -0
- package/dist/agent-memory-scope.js +230 -0
- package/dist/agent-memory-scope.js.map +1 -0
- package/dist/agentdb-adapter.d.ts +165 -0
- package/dist/agentdb-adapter.d.ts.map +1 -0
- package/dist/agentdb-adapter.js +836 -0
- package/dist/agentdb-adapter.js.map +1 -0
- package/dist/agentdb-backend.d.ts +218 -0
- package/dist/agentdb-backend.d.ts.map +1 -0
- package/dist/agentdb-backend.js +879 -0
- package/dist/agentdb-backend.js.map +1 -0
- package/dist/auto-memory-bridge.d.ts +230 -0
- package/dist/auto-memory-bridge.d.ts.map +1 -0
- package/dist/auto-memory-bridge.js +730 -0
- package/dist/auto-memory-bridge.js.map +1 -0
- package/dist/cache-manager.d.ts +134 -0
- package/dist/cache-manager.d.ts.map +1 -0
- package/dist/cache-manager.js +410 -0
- package/dist/cache-manager.js.map +1 -0
- package/dist/checkpointer.d.ts +78 -0
- package/dist/checkpointer.d.ts.map +1 -0
- package/dist/checkpointer.js +174 -0
- package/dist/checkpointer.js.map +1 -0
- package/dist/controller-registry.d.ts +216 -0
- package/dist/controller-registry.d.ts.map +1 -0
- package/dist/controller-registry.js +897 -0
- package/dist/controller-registry.js.map +1 -0
- package/dist/database-provider.d.ts +87 -0
- package/dist/database-provider.d.ts.map +1 -0
- package/dist/database-provider.js +434 -0
- package/dist/database-provider.js.map +1 -0
- package/dist/diskann-backend.d.ts +126 -0
- package/dist/diskann-backend.d.ts.map +1 -0
- package/dist/diskann-backend.js +324 -0
- package/dist/diskann-backend.js.map +1 -0
- package/dist/episodic-store.d.ts +58 -0
- package/dist/episodic-store.d.ts.map +1 -0
- package/dist/episodic-store.js +160 -0
- package/dist/episodic-store.js.map +1 -0
- package/dist/hnsw-index.d.ts +111 -0
- package/dist/hnsw-index.d.ts.map +1 -0
- package/dist/hnsw-index.js +778 -0
- package/dist/hnsw-index.js.map +1 -0
- package/dist/hnsw-lite.d.ts +29 -0
- package/dist/hnsw-lite.d.ts.map +1 -0
- package/dist/hnsw-lite.js +236 -0
- package/dist/hnsw-lite.js.map +1 -0
- package/dist/hybrid-backend.d.ts +299 -0
- package/dist/hybrid-backend.d.ts.map +1 -0
- package/dist/hybrid-backend.js +804 -0
- package/dist/hybrid-backend.js.map +1 -0
- package/dist/index.d.ts +228 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +379 -0
- package/dist/index.js.map +1 -0
- package/dist/knowledge/document-chunker.d.ts +22 -0
- package/dist/knowledge/document-chunker.d.ts.map +1 -0
- package/dist/knowledge/document-chunker.js +52 -0
- package/dist/knowledge/document-chunker.js.map +1 -0
- package/dist/knowledge/index.d.ts +12 -0
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/index.js +9 -0
- package/dist/knowledge/index.js.map +1 -0
- package/dist/knowledge/knowledge-retriever.d.ts +48 -0
- package/dist/knowledge/knowledge-retriever.d.ts.map +1 -0
- package/dist/knowledge/knowledge-retriever.js +80 -0
- package/dist/knowledge/knowledge-retriever.js.map +1 -0
- package/dist/knowledge/knowledge-store.d.ts +58 -0
- package/dist/knowledge/knowledge-store.d.ts.map +1 -0
- package/dist/knowledge/knowledge-store.js +121 -0
- package/dist/knowledge/knowledge-store.js.map +1 -0
- package/dist/learning-bridge.d.ts +162 -0
- package/dist/learning-bridge.d.ts.map +1 -0
- package/dist/learning-bridge.js +522 -0
- package/dist/learning-bridge.js.map +1 -0
- package/dist/memory-graph.d.ts +123 -0
- package/dist/memory-graph.d.ts.map +1 -0
- package/dist/memory-graph.js +412 -0
- package/dist/memory-graph.js.map +1 -0
- package/dist/migration.d.ts +68 -0
- package/dist/migration.d.ts.map +1 -0
- package/dist/migration.js +513 -0
- package/dist/migration.js.map +1 -0
- package/dist/partitioned-hnsw.d.ts +65 -0
- package/dist/partitioned-hnsw.d.ts.map +1 -0
- package/dist/partitioned-hnsw.js +209 -0
- package/dist/partitioned-hnsw.js.map +1 -0
- package/dist/persistent-sona.d.ts +144 -0
- package/dist/persistent-sona.d.ts.map +1 -0
- package/dist/persistent-sona.js +333 -0
- package/dist/persistent-sona.js.map +1 -0
- package/dist/procedural/action-record.d.ts +19 -0
- package/dist/procedural/action-record.d.ts.map +1 -0
- package/dist/procedural/action-record.js +51 -0
- package/dist/procedural/action-record.js.map +1 -0
- package/dist/procedural/action-sequence-extractor.d.ts +25 -0
- package/dist/procedural/action-sequence-extractor.d.ts.map +1 -0
- package/dist/procedural/action-sequence-extractor.js +91 -0
- package/dist/procedural/action-sequence-extractor.js.map +1 -0
- package/dist/procedural/index.d.ts +14 -0
- package/dist/procedural/index.d.ts.map +1 -0
- package/dist/procedural/index.js +13 -0
- package/dist/procedural/index.js.map +1 -0
- package/dist/procedural/learned-skill.d.ts +17 -0
- package/dist/procedural/learned-skill.d.ts.map +1 -0
- package/dist/procedural/learned-skill.js +73 -0
- package/dist/procedural/learned-skill.js.map +1 -0
- package/dist/procedural/procedure-extractor-worker.d.ts +27 -0
- package/dist/procedural/procedure-extractor-worker.d.ts.map +1 -0
- package/dist/procedural/procedure-extractor-worker.js +39 -0
- package/dist/procedural/procedure-extractor-worker.js.map +1 -0
- package/dist/procedural/skill-registry.d.ts +25 -0
- package/dist/procedural/skill-registry.d.ts.map +1 -0
- package/dist/procedural/skill-registry.js +56 -0
- package/dist/procedural/skill-registry.js.map +1 -0
- package/dist/procedural/types.d.ts +67 -0
- package/dist/procedural/types.d.ts.map +1 -0
- package/dist/procedural/types.js +15 -0
- package/dist/procedural/types.js.map +1 -0
- package/dist/prompt-version-store.d.ts +59 -0
- package/dist/prompt-version-store.d.ts.map +1 -0
- package/dist/prompt-version-store.js +175 -0
- package/dist/prompt-version-store.js.map +1 -0
- package/dist/query-builder.d.ts +52 -0
- package/dist/query-builder.d.ts.map +1 -0
- package/dist/query-builder.js +134 -0
- package/dist/query-builder.js.map +1 -0
- package/dist/rvf-backend.d.ts +51 -0
- package/dist/rvf-backend.d.ts.map +1 -0
- package/dist/rvf-backend.js +499 -0
- package/dist/rvf-backend.js.map +1 -0
- package/dist/rvf-learning-store.d.ts +141 -0
- package/dist/rvf-learning-store.d.ts.map +1 -0
- package/dist/rvf-learning-store.js +344 -0
- package/dist/rvf-learning-store.js.map +1 -0
- package/dist/rvf-migration.d.ts +45 -0
- package/dist/rvf-migration.d.ts.map +1 -0
- package/dist/rvf-migration.js +254 -0
- package/dist/rvf-migration.js.map +1 -0
- package/dist/sqlite-backend.d.ts +127 -0
- package/dist/sqlite-backend.d.ts.map +1 -0
- package/dist/sqlite-backend.js +618 -0
- package/dist/sqlite-backend.js.map +1 -0
- package/dist/sqljs-backend.d.ts +128 -0
- package/dist/sqljs-backend.d.ts.map +1 -0
- package/dist/sqljs-backend.js +611 -0
- package/dist/sqljs-backend.js.map +1 -0
- package/dist/tier-manager.d.ts +79 -0
- package/dist/tier-manager.d.ts.map +1 -0
- package/dist/tier-manager.js +169 -0
- package/dist/tier-manager.js.map +1 -0
- package/dist/tiers/contextual.d.ts +43 -0
- package/dist/tiers/contextual.d.ts.map +1 -0
- package/dist/tiers/contextual.js +102 -0
- package/dist/tiers/contextual.js.map +1 -0
- package/dist/tiers/entity.d.ts +46 -0
- package/dist/tiers/entity.d.ts.map +1 -0
- package/dist/tiers/entity.js +106 -0
- package/dist/tiers/entity.js.map +1 -0
- package/dist/tiers/index.d.ts +4 -0
- package/dist/tiers/index.d.ts.map +1 -0
- package/dist/tiers/index.js +4 -0
- package/dist/tiers/index.js.map +1 -0
- package/dist/tiers/short-term.d.ts +42 -0
- package/dist/tiers/short-term.d.ts.map +1 -0
- package/dist/tiers/short-term.js +89 -0
- package/dist/tiers/short-term.js.map +1 -0
- package/dist/types/checkpoint.d.ts +44 -0
- package/dist/types/checkpoint.d.ts.map +1 -0
- package/dist/types/checkpoint.js +8 -0
- package/dist/types/checkpoint.js.map +1 -0
- package/dist/types.d.ts +526 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +58 -0
- package/dist/types.js.map +1 -0
- package/package.json +46 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DiskAnnBackend — SSD-resident ANN graph with in-memory compressed vectors
|
|
3
|
+
*
|
|
4
|
+
* Architecture (faithful to DiskANN / Vamana; source: arXiv:2305.04359):
|
|
5
|
+
* (a) Adjacency-list graph persisted as a separate JSON file on disk
|
|
6
|
+
* (NOT the SQLite rows — a distinct index artifact)
|
|
7
|
+
* (b) Int8-quantised vectors kept in memory for fast candidate beam search
|
|
8
|
+
* (scalar quantisation: float32 × 127 → int8, ~4× memory reduction)
|
|
9
|
+
* (c) Full-precision re-ranking: for each beam candidate, load the original
|
|
10
|
+
* Float32 embedding from the delegate backend and rescore with cosine
|
|
11
|
+
*
|
|
12
|
+
* This backend wraps any IMemoryBackend (delegate) and delegates all CRUD
|
|
13
|
+
* operations to it, overriding only `search()` with the DiskANN algorithm.
|
|
14
|
+
*
|
|
15
|
+
* @module v1/memory/diskann-backend
|
|
16
|
+
*/
|
|
17
|
+
import type { IMemoryBackend, MemoryEntry, MemoryEntryUpdate, MemoryQuery, SearchOptions, SearchResult, BackendStats, HealthCheckResult } from './types.js';
|
|
18
|
+
export interface DiskAnnBackendConfig {
|
|
19
|
+
/**
|
|
20
|
+
* Filesystem path for the persisted adjacency-list graph file.
|
|
21
|
+
* Must be different from the delegate's database path.
|
|
22
|
+
*/
|
|
23
|
+
graphPath: string;
|
|
24
|
+
/**
|
|
25
|
+
* Beam width (ef) — number of candidates to maintain during graph traversal.
|
|
26
|
+
* Higher values trade speed for recall. Default: 32.
|
|
27
|
+
*/
|
|
28
|
+
beamWidth?: number;
|
|
29
|
+
/**
|
|
30
|
+
* Maximum outgoing edges per node (M). Default: 12.
|
|
31
|
+
*/
|
|
32
|
+
maxConnections?: number;
|
|
33
|
+
/**
|
|
34
|
+
* How many beam candidates to fetch before full-precision re-ranking.
|
|
35
|
+
* Final result count is `k`; this is `k × rerankMultiplier`. Default: 3.
|
|
36
|
+
*/
|
|
37
|
+
rerankMultiplier?: number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* DiskAnnBackend
|
|
41
|
+
*
|
|
42
|
+
* Wraps any IMemoryBackend and replaces the `search()` method with a three-phase
|
|
43
|
+
* DiskANN procedure:
|
|
44
|
+
* 1. Quantised beam search (Int8 dot products — fast, approximate)
|
|
45
|
+
* 2. Full-precision re-ranking (Float32 cosine from delegate.get())
|
|
46
|
+
* 3. Threshold filtering and top-k selection
|
|
47
|
+
*
|
|
48
|
+
* All other IMemoryBackend methods delegate directly to the inner backend.
|
|
49
|
+
*/
|
|
50
|
+
export declare class DiskAnnBackend implements IMemoryBackend {
|
|
51
|
+
private readonly delegate;
|
|
52
|
+
/** Adjacency list: nodeId → sorted neighbour IDs */
|
|
53
|
+
private readonly graph;
|
|
54
|
+
/**
|
|
55
|
+
* Int8 scalar-quantised embeddings for beam search.
|
|
56
|
+
* Kept entirely in memory — the critical DiskANN optimisation that avoids
|
|
57
|
+
* loading full-precision vectors during candidate generation.
|
|
58
|
+
*/
|
|
59
|
+
private readonly qvecs;
|
|
60
|
+
/** Random walk entry point into the graph */
|
|
61
|
+
private entryPoint;
|
|
62
|
+
/** Whether the in-memory graph differs from the disk file */
|
|
63
|
+
private graphDirty;
|
|
64
|
+
private readonly graphPath;
|
|
65
|
+
private readonly beamWidth;
|
|
66
|
+
private readonly maxConnections;
|
|
67
|
+
private readonly rerankMultiplier;
|
|
68
|
+
constructor(delegate: IMemoryBackend, config: DiskAnnBackendConfig);
|
|
69
|
+
initialize(): Promise<void>;
|
|
70
|
+
shutdown(): Promise<void>;
|
|
71
|
+
store(entry: MemoryEntry): Promise<void>;
|
|
72
|
+
get(id: string): Promise<MemoryEntry | null>;
|
|
73
|
+
getByKey(namespace: string, key: string): Promise<MemoryEntry | null>;
|
|
74
|
+
update(id: string, update: MemoryEntryUpdate): Promise<MemoryEntry | null>;
|
|
75
|
+
delete(id: string): Promise<boolean>;
|
|
76
|
+
query(q: MemoryQuery): Promise<MemoryEntry[]>;
|
|
77
|
+
bulkInsert(entries: MemoryEntry[]): Promise<void>;
|
|
78
|
+
bulkDelete(ids: string[]): Promise<number>;
|
|
79
|
+
count(namespace?: string): Promise<number>;
|
|
80
|
+
listNamespaces(): Promise<string[]>;
|
|
81
|
+
clearNamespace(namespace: string): Promise<number>;
|
|
82
|
+
getStats(): Promise<BackendStats>;
|
|
83
|
+
healthCheck(): Promise<HealthCheckResult>;
|
|
84
|
+
/**
|
|
85
|
+
* Vector similarity search using the DiskANN beam algorithm.
|
|
86
|
+
*
|
|
87
|
+
* Phase (b): beam traversal over compressed Int8 vectors (fast, no I/O)
|
|
88
|
+
* Phase (c): full-precision cosine re-ranking via delegate.get() (accurate)
|
|
89
|
+
*
|
|
90
|
+
* Falls back to delegate.search() when no entries have been indexed yet.
|
|
91
|
+
*/
|
|
92
|
+
search(queryVec: Float32Array, options: SearchOptions): Promise<SearchResult[]>;
|
|
93
|
+
/** Add or update a node in the in-memory ANN index */
|
|
94
|
+
private indexEntry;
|
|
95
|
+
/** Remove a node and all its references from the index */
|
|
96
|
+
private removeFromIndex;
|
|
97
|
+
/** Scan all existing qvecs and return the k nearest to `qvec` */
|
|
98
|
+
private greedyNearestNeighbors;
|
|
99
|
+
/**
|
|
100
|
+
* Beam search over the compressed-vector graph.
|
|
101
|
+
*
|
|
102
|
+
* Starts from `entryPoint`, expands neighbours BFS-style, and scores each
|
|
103
|
+
* visited node with Int8 dot products. Returns up to `candidateCount` IDs
|
|
104
|
+
* sorted by descending similarity.
|
|
105
|
+
*/
|
|
106
|
+
private beamSearch;
|
|
107
|
+
/** Scalar Int8 quantisation: float32 × 127 → clamped int8 */
|
|
108
|
+
private quantize;
|
|
109
|
+
/** Int8 dot product (used for beam search scoring) */
|
|
110
|
+
private dotProduct;
|
|
111
|
+
/** Full-precision cosine similarity (used for re-ranking) */
|
|
112
|
+
private cosine;
|
|
113
|
+
/**
|
|
114
|
+
* Load the persisted adjacency list from disk.
|
|
115
|
+
* Called during initialize(). A missing or corrupt file is silently ignored;
|
|
116
|
+
* the index will be rebuilt as entries are stored.
|
|
117
|
+
*/
|
|
118
|
+
private loadGraph;
|
|
119
|
+
/**
|
|
120
|
+
* Persist the adjacency list to disk.
|
|
121
|
+
* Only writes when the graph has changed since the last persist.
|
|
122
|
+
* Errors are non-fatal; the graph will be rebuilt on next startup.
|
|
123
|
+
*/
|
|
124
|
+
private persistGraph;
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=diskann-backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diskann-backend.d.ts","sourceRoot":"","sources":["../src/diskann-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EACV,cAAc,EACd,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAMpB,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAiBD;;;;;;;;;;GAUG;AACH,qBAAa,cAAe,YAAW,cAAc;IAuBjD,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAtB3B,oDAAoD;IACpD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IAErD;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IAEtD,6CAA6C;IAC7C,OAAO,CAAC,UAAU,CAAqB;IAEvC,6DAA6D;IAC7D,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAGvB,QAAQ,EAAE,cAAc,EACzC,MAAM,EAAE,oBAAoB;IAYxB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAK3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IASzB,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAI5C,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAIrE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAI1E,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMpC,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAI7C,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjD,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAM1C,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAI1C,cAAc,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAInC,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAUlD,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC;IAIjC,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAQ/C;;;;;;;OAOG;IACG,MAAM,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IA+BrF,sDAAsD;IACtD,OAAO,CAAC,UAAU;IAsBlB,0DAA0D;IAC1D,OAAO,CAAC,eAAe;IAmBvB,iEAAiE;IACjE,OAAO,CAAC,sBAAsB;IAU9B;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IAkClB,6DAA6D;IAC7D,OAAO,CAAC,QAAQ;IAQhB,sDAAsD;IACtD,OAAO,CAAC,UAAU;IAOlB,6DAA6D;IAC7D,OAAO,CAAC,MAAM;IAgBd;;;;OAIG;YACW,SAAS;IAgBvB;;;;OAIG;YACW,YAAY;CAgB3B"}
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DiskAnnBackend — SSD-resident ANN graph with in-memory compressed vectors
|
|
3
|
+
*
|
|
4
|
+
* Architecture (faithful to DiskANN / Vamana; source: arXiv:2305.04359):
|
|
5
|
+
* (a) Adjacency-list graph persisted as a separate JSON file on disk
|
|
6
|
+
* (NOT the SQLite rows — a distinct index artifact)
|
|
7
|
+
* (b) Int8-quantised vectors kept in memory for fast candidate beam search
|
|
8
|
+
* (scalar quantisation: float32 × 127 → int8, ~4× memory reduction)
|
|
9
|
+
* (c) Full-precision re-ranking: for each beam candidate, load the original
|
|
10
|
+
* Float32 embedding from the delegate backend and rescore with cosine
|
|
11
|
+
*
|
|
12
|
+
* This backend wraps any IMemoryBackend (delegate) and delegates all CRUD
|
|
13
|
+
* operations to it, overriding only `search()` with the DiskANN algorithm.
|
|
14
|
+
*
|
|
15
|
+
* @module v1/memory/diskann-backend
|
|
16
|
+
*/
|
|
17
|
+
import { writeFile, readFile, mkdir } from 'node:fs/promises';
|
|
18
|
+
import { existsSync } from 'node:fs';
|
|
19
|
+
import { dirname } from 'node:path';
|
|
20
|
+
// ============================================================
|
|
21
|
+
// DiskAnnBackend
|
|
22
|
+
// ============================================================
|
|
23
|
+
/**
|
|
24
|
+
* DiskAnnBackend
|
|
25
|
+
*
|
|
26
|
+
* Wraps any IMemoryBackend and replaces the `search()` method with a three-phase
|
|
27
|
+
* DiskANN procedure:
|
|
28
|
+
* 1. Quantised beam search (Int8 dot products — fast, approximate)
|
|
29
|
+
* 2. Full-precision re-ranking (Float32 cosine from delegate.get())
|
|
30
|
+
* 3. Threshold filtering and top-k selection
|
|
31
|
+
*
|
|
32
|
+
* All other IMemoryBackend methods delegate directly to the inner backend.
|
|
33
|
+
*/
|
|
34
|
+
export class DiskAnnBackend {
|
|
35
|
+
delegate;
|
|
36
|
+
/** Adjacency list: nodeId → sorted neighbour IDs */
|
|
37
|
+
graph = new Map();
|
|
38
|
+
/**
|
|
39
|
+
* Int8 scalar-quantised embeddings for beam search.
|
|
40
|
+
* Kept entirely in memory — the critical DiskANN optimisation that avoids
|
|
41
|
+
* loading full-precision vectors during candidate generation.
|
|
42
|
+
*/
|
|
43
|
+
qvecs = new Map();
|
|
44
|
+
/** Random walk entry point into the graph */
|
|
45
|
+
entryPoint;
|
|
46
|
+
/** Whether the in-memory graph differs from the disk file */
|
|
47
|
+
graphDirty = false;
|
|
48
|
+
graphPath;
|
|
49
|
+
beamWidth;
|
|
50
|
+
maxConnections;
|
|
51
|
+
rerankMultiplier;
|
|
52
|
+
constructor(delegate, config) {
|
|
53
|
+
this.delegate = delegate;
|
|
54
|
+
this.graphPath = config.graphPath;
|
|
55
|
+
this.beamWidth = config.beamWidth ?? 32;
|
|
56
|
+
this.maxConnections = config.maxConnections ?? 12;
|
|
57
|
+
this.rerankMultiplier = config.rerankMultiplier ?? 3;
|
|
58
|
+
}
|
|
59
|
+
// ----------------------------------------------------------------
|
|
60
|
+
// Lifecycle
|
|
61
|
+
// ----------------------------------------------------------------
|
|
62
|
+
async initialize() {
|
|
63
|
+
await this.delegate.initialize();
|
|
64
|
+
await this.loadGraph();
|
|
65
|
+
}
|
|
66
|
+
async shutdown() {
|
|
67
|
+
await this.persistGraph();
|
|
68
|
+
await this.delegate.shutdown();
|
|
69
|
+
}
|
|
70
|
+
// ----------------------------------------------------------------
|
|
71
|
+
// CRUD — fully delegated
|
|
72
|
+
// ----------------------------------------------------------------
|
|
73
|
+
async store(entry) {
|
|
74
|
+
await this.delegate.store(entry);
|
|
75
|
+
this.indexEntry(entry);
|
|
76
|
+
}
|
|
77
|
+
async get(id) {
|
|
78
|
+
return this.delegate.get(id);
|
|
79
|
+
}
|
|
80
|
+
async getByKey(namespace, key) {
|
|
81
|
+
return this.delegate.getByKey(namespace, key);
|
|
82
|
+
}
|
|
83
|
+
async update(id, update) {
|
|
84
|
+
return this.delegate.update(id, update);
|
|
85
|
+
}
|
|
86
|
+
async delete(id) {
|
|
87
|
+
const ok = await this.delegate.delete(id);
|
|
88
|
+
if (ok)
|
|
89
|
+
this.removeFromIndex(id);
|
|
90
|
+
return ok;
|
|
91
|
+
}
|
|
92
|
+
async query(q) {
|
|
93
|
+
return this.delegate.query(q);
|
|
94
|
+
}
|
|
95
|
+
async bulkInsert(entries) {
|
|
96
|
+
await this.delegate.bulkInsert(entries);
|
|
97
|
+
for (const e of entries)
|
|
98
|
+
this.indexEntry(e);
|
|
99
|
+
}
|
|
100
|
+
async bulkDelete(ids) {
|
|
101
|
+
const n = await this.delegate.bulkDelete(ids);
|
|
102
|
+
for (const id of ids)
|
|
103
|
+
this.removeFromIndex(id);
|
|
104
|
+
return n;
|
|
105
|
+
}
|
|
106
|
+
async count(namespace) {
|
|
107
|
+
return this.delegate.count(namespace);
|
|
108
|
+
}
|
|
109
|
+
async listNamespaces() {
|
|
110
|
+
return this.delegate.listNamespaces();
|
|
111
|
+
}
|
|
112
|
+
async clearNamespace(namespace) {
|
|
113
|
+
const n = await this.delegate.clearNamespace(namespace);
|
|
114
|
+
// Rebuild index — we don't track namespace per node
|
|
115
|
+
this.graph.clear();
|
|
116
|
+
this.qvecs.clear();
|
|
117
|
+
this.entryPoint = undefined;
|
|
118
|
+
this.graphDirty = true;
|
|
119
|
+
return n;
|
|
120
|
+
}
|
|
121
|
+
async getStats() {
|
|
122
|
+
return this.delegate.getStats();
|
|
123
|
+
}
|
|
124
|
+
async healthCheck() {
|
|
125
|
+
return this.delegate.healthCheck();
|
|
126
|
+
}
|
|
127
|
+
// ----------------------------------------------------------------
|
|
128
|
+
// search() — DiskANN three-phase algorithm
|
|
129
|
+
// ----------------------------------------------------------------
|
|
130
|
+
/**
|
|
131
|
+
* Vector similarity search using the DiskANN beam algorithm.
|
|
132
|
+
*
|
|
133
|
+
* Phase (b): beam traversal over compressed Int8 vectors (fast, no I/O)
|
|
134
|
+
* Phase (c): full-precision cosine re-ranking via delegate.get() (accurate)
|
|
135
|
+
*
|
|
136
|
+
* Falls back to delegate.search() when no entries have been indexed yet.
|
|
137
|
+
*/
|
|
138
|
+
async search(queryVec, options) {
|
|
139
|
+
if (this.qvecs.size === 0) {
|
|
140
|
+
return this.delegate.search(queryVec, options);
|
|
141
|
+
}
|
|
142
|
+
const k = options.k;
|
|
143
|
+
const threshold = options.threshold ?? 0;
|
|
144
|
+
// Phase (b): beam search with Int8-quantised vectors
|
|
145
|
+
const qQuery = this.quantize(queryVec);
|
|
146
|
+
const candidateIds = this.beamSearch(qQuery, k * this.rerankMultiplier);
|
|
147
|
+
// Phase (c): full-precision re-ranking by loading embeddings from disk/delegate
|
|
148
|
+
const results = [];
|
|
149
|
+
for (const id of candidateIds) {
|
|
150
|
+
const entry = await this.delegate.get(id);
|
|
151
|
+
if (!entry?.embedding)
|
|
152
|
+
continue;
|
|
153
|
+
const score = this.cosine(queryVec, entry.embedding);
|
|
154
|
+
if (score >= threshold) {
|
|
155
|
+
results.push({ entry, score, distance: 1 - score });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
results.sort((a, b) => b.score - a.score);
|
|
159
|
+
return results.slice(0, k);
|
|
160
|
+
}
|
|
161
|
+
// ----------------------------------------------------------------
|
|
162
|
+
// Index management (private)
|
|
163
|
+
// ----------------------------------------------------------------
|
|
164
|
+
/** Add or update a node in the in-memory ANN index */
|
|
165
|
+
indexEntry(entry) {
|
|
166
|
+
if (!entry.embedding)
|
|
167
|
+
return;
|
|
168
|
+
const qvec = this.quantize(entry.embedding);
|
|
169
|
+
this.qvecs.set(entry.id, qvec);
|
|
170
|
+
// Greedy M-nearest-neighbour insertion (O(n) scan over compressed vectors)
|
|
171
|
+
const neighbors = this.greedyNearestNeighbors(entry.id, qvec, this.maxConnections);
|
|
172
|
+
this.graph.set(entry.id, neighbors);
|
|
173
|
+
// Add reverse edges (backlinks) — keep outdegree bounded
|
|
174
|
+
for (const nid of neighbors) {
|
|
175
|
+
const nEdges = this.graph.get(nid) ?? [];
|
|
176
|
+
if (!nEdges.includes(entry.id) && nEdges.length < this.maxConnections * 2) {
|
|
177
|
+
this.graph.set(nid, [...nEdges, entry.id]);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (!this.entryPoint)
|
|
181
|
+
this.entryPoint = entry.id;
|
|
182
|
+
this.graphDirty = true;
|
|
183
|
+
}
|
|
184
|
+
/** Remove a node and all its references from the index */
|
|
185
|
+
removeFromIndex(id) {
|
|
186
|
+
this.qvecs.delete(id);
|
|
187
|
+
const neighbors = this.graph.get(id) ?? [];
|
|
188
|
+
this.graph.delete(id);
|
|
189
|
+
// Remove backreferences from neighbours
|
|
190
|
+
for (const nid of neighbors) {
|
|
191
|
+
const edges = this.graph.get(nid);
|
|
192
|
+
if (edges) {
|
|
193
|
+
this.graph.set(nid, edges.filter(e => e !== id));
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (this.entryPoint === id) {
|
|
197
|
+
this.entryPoint = this.graph.keys().next().value;
|
|
198
|
+
}
|
|
199
|
+
this.graphDirty = true;
|
|
200
|
+
}
|
|
201
|
+
/** Scan all existing qvecs and return the k nearest to `qvec` */
|
|
202
|
+
greedyNearestNeighbors(excludeId, qvec, k) {
|
|
203
|
+
const scored = [];
|
|
204
|
+
for (const [id, vec] of this.qvecs) {
|
|
205
|
+
if (id === excludeId)
|
|
206
|
+
continue;
|
|
207
|
+
scored.push({ id, score: this.dotProduct(qvec, vec) });
|
|
208
|
+
}
|
|
209
|
+
scored.sort((a, b) => b.score - a.score);
|
|
210
|
+
return scored.slice(0, k).map(s => s.id);
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Beam search over the compressed-vector graph.
|
|
214
|
+
*
|
|
215
|
+
* Starts from `entryPoint`, expands neighbours BFS-style, and scores each
|
|
216
|
+
* visited node with Int8 dot products. Returns up to `candidateCount` IDs
|
|
217
|
+
* sorted by descending similarity.
|
|
218
|
+
*/
|
|
219
|
+
beamSearch(qQuery, candidateCount) {
|
|
220
|
+
if (!this.entryPoint || this.qvecs.size === 0)
|
|
221
|
+
return [];
|
|
222
|
+
const visited = new Set();
|
|
223
|
+
const scored = [];
|
|
224
|
+
const frontier = [this.entryPoint];
|
|
225
|
+
visited.add(this.entryPoint);
|
|
226
|
+
const maxVisit = Math.min(this.beamWidth * 4, this.qvecs.size);
|
|
227
|
+
while (frontier.length > 0 && visited.size <= maxVisit) {
|
|
228
|
+
const current = frontier.shift();
|
|
229
|
+
const qvec = this.qvecs.get(current);
|
|
230
|
+
if (!qvec)
|
|
231
|
+
continue;
|
|
232
|
+
scored.push({ id: current, score: this.dotProduct(qQuery, qvec) });
|
|
233
|
+
// Expand graph neighbours
|
|
234
|
+
for (const nid of (this.graph.get(current) ?? [])) {
|
|
235
|
+
if (!visited.has(nid)) {
|
|
236
|
+
visited.add(nid);
|
|
237
|
+
frontier.push(nid);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
scored.sort((a, b) => b.score - a.score);
|
|
242
|
+
return scored.slice(0, candidateCount).map(s => s.id);
|
|
243
|
+
}
|
|
244
|
+
// ----------------------------------------------------------------
|
|
245
|
+
// Quantisation and distance helpers
|
|
246
|
+
// ----------------------------------------------------------------
|
|
247
|
+
/** Scalar Int8 quantisation: float32 × 127 → clamped int8 */
|
|
248
|
+
quantize(vec) {
|
|
249
|
+
const q = new Int8Array(vec.length);
|
|
250
|
+
for (let i = 0; i < vec.length; i++) {
|
|
251
|
+
q[i] = Math.max(-127, Math.min(127, Math.round(vec[i] * 127)));
|
|
252
|
+
}
|
|
253
|
+
return q;
|
|
254
|
+
}
|
|
255
|
+
/** Int8 dot product (used for beam search scoring) */
|
|
256
|
+
dotProduct(a, b) {
|
|
257
|
+
const len = Math.min(a.length, b.length);
|
|
258
|
+
let sum = 0;
|
|
259
|
+
for (let i = 0; i < len; i++)
|
|
260
|
+
sum += a[i] * b[i];
|
|
261
|
+
return sum;
|
|
262
|
+
}
|
|
263
|
+
/** Full-precision cosine similarity (used for re-ranking) */
|
|
264
|
+
cosine(a, b) {
|
|
265
|
+
const len = Math.min(a.length, b.length);
|
|
266
|
+
let dot = 0, normA = 0, normB = 0;
|
|
267
|
+
for (let i = 0; i < len; i++) {
|
|
268
|
+
dot += a[i] * b[i];
|
|
269
|
+
normA += a[i] * a[i];
|
|
270
|
+
normB += b[i] * b[i];
|
|
271
|
+
}
|
|
272
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
273
|
+
return denom === 0 ? 0 : dot / denom;
|
|
274
|
+
}
|
|
275
|
+
// ----------------------------------------------------------------
|
|
276
|
+
// Disk persistence (graph adjacency list only — NOT the entry data)
|
|
277
|
+
// ----------------------------------------------------------------
|
|
278
|
+
/**
|
|
279
|
+
* Load the persisted adjacency list from disk.
|
|
280
|
+
* Called during initialize(). A missing or corrupt file is silently ignored;
|
|
281
|
+
* the index will be rebuilt as entries are stored.
|
|
282
|
+
*/
|
|
283
|
+
async loadGraph() {
|
|
284
|
+
if (!existsSync(this.graphPath))
|
|
285
|
+
return;
|
|
286
|
+
try {
|
|
287
|
+
const raw = await readFile(this.graphPath, 'utf8');
|
|
288
|
+
const data = JSON.parse(raw);
|
|
289
|
+
this.graph.clear();
|
|
290
|
+
for (const [id, neighbors] of data.graph) {
|
|
291
|
+
this.graph.set(id, neighbors);
|
|
292
|
+
}
|
|
293
|
+
this.entryPoint = data.entryPoint;
|
|
294
|
+
this.graphDirty = false;
|
|
295
|
+
}
|
|
296
|
+
catch {
|
|
297
|
+
// Corrupt file — start with empty graph
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Persist the adjacency list to disk.
|
|
302
|
+
* Only writes when the graph has changed since the last persist.
|
|
303
|
+
* Errors are non-fatal; the graph will be rebuilt on next startup.
|
|
304
|
+
*/
|
|
305
|
+
async persistGraph() {
|
|
306
|
+
if (!this.graphDirty)
|
|
307
|
+
return;
|
|
308
|
+
try {
|
|
309
|
+
await mkdir(dirname(this.graphPath), { recursive: true });
|
|
310
|
+
const data = {
|
|
311
|
+
graph: [...this.graph.entries()],
|
|
312
|
+
entryPoint: this.entryPoint,
|
|
313
|
+
nodeCount: this.graph.size,
|
|
314
|
+
savedAt: Date.now(),
|
|
315
|
+
};
|
|
316
|
+
await writeFile(this.graphPath, JSON.stringify(data), 'utf8');
|
|
317
|
+
this.graphDirty = false;
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
// Non-fatal — graph will be rebuilt on next startup
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
//# sourceMappingURL=diskann-backend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diskann-backend.js","sourceRoot":"","sources":["../src/diskann-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoDpC,+DAA+D;AAC/D,iBAAiB;AACjB,+DAA+D;AAE/D;;;;;;;;;;GAUG;AACH,MAAM,OAAO,cAAc;IAuBN;IAtBnB,oDAAoD;IACnC,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAErD;;;;OAIG;IACc,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEtD,6CAA6C;IACrC,UAAU,CAAqB;IAEvC,6DAA6D;IACrD,UAAU,GAAG,KAAK,CAAC;IAEV,SAAS,CAAS;IAClB,SAAS,CAAS;IAClB,cAAc,CAAS;IACvB,gBAAgB,CAAS;IAE1C,YACmB,QAAwB,EACzC,MAA4B;QADX,aAAQ,GAAR,QAAQ,CAAgB;QAGzC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,mEAAmE;IACnE,YAAY;IACZ,mEAAmE;IAEnE,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;IACjC,CAAC;IAED,mEAAmE;IACnE,yBAAyB;IACzB,mEAAmE;IAEnE,KAAK,CAAC,KAAK,CAAC,KAAkB;QAC5B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,GAAW;QAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,MAAyB;QAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,EAAE;YAAE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QACjC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,CAAc;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAsB;QACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACxC,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAa;QAC5B,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9C,KAAK,MAAM,EAAE,IAAI,GAAG;YAAE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAkB;QAC5B,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB;QACpC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACxD,oDAAoD;QACpD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED,mEAAmE;IACnE,2CAA2C;IAC3C,mEAAmE;IAEnE;;;;;;;OAOG;IACH,KAAK,CAAC,MAAM,CAAC,QAAsB,EAAE,OAAsB;QACzD,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QACpB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;QAEzC,qDAAqD;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAExE,gFAAgF;QAChF,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,KAAK,EAAE,SAAS;gBAAE,SAAS;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,mEAAmE;IACnE,6BAA6B;IAC7B,mEAAmE;IAEnE,sDAAsD;IAC9C,UAAU,CAAC,KAAkB;QACnC,IAAI,CAAC,KAAK,CAAC,SAAS;YAAE,OAAO;QAE7B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAE/B,2EAA2E;QAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QACnF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAEpC,yDAAyD;QACzD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACzC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,0DAA0D;IAClD,eAAe,CAAC,EAAU;QAChC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAEtB,wCAAwC;QACxC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,EAAE,EAAE,CAAC;YAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAA2B,CAAC;QACzE,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,iEAAiE;IACzD,sBAAsB,CAAC,SAAiB,EAAE,IAAe,EAAE,CAAS;QAC1E,MAAM,MAAM,GAAyC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,EAAE,KAAK,SAAS;gBAAE,SAAS;YAC/B,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;OAMG;IACK,UAAU,CAAC,MAAiB,EAAE,cAAsB;QAC1D,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEzD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,MAAM,GAAyC,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE/D,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,EAAG,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAEnE,0BAA0B;YAC1B,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACjB,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,mEAAmE;IACnE,oCAAoC;IACpC,mEAAmE;IAEnE,6DAA6D;IACrD,QAAQ,CAAC,GAAiB;QAChC,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,sDAAsD;IAC9C,UAAU,CAAC,CAAY,EAAE,CAAY;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,6DAA6D;IACrD,MAAM,CAAC,CAAe,EAAE,CAAe;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,GAAG,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;IACvC,CAAC;IAED,mEAAmE;IACnE,oEAAoE;IACpE,mEAAmE;IAEnE;;;;OAIG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAmB,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,KAAK,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YAClC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAmB;gBAC3B,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAChC,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;gBAC1B,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;aACpB,CAAC;YACF,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;YAC9D,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,oDAAoD;QACtD,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EpisodicStore — JSON-lines file-based episodic memory.
|
|
3
|
+
*
|
|
4
|
+
* Stores episodes as one JSON object per line in a .jsonl file,
|
|
5
|
+
* following the same pattern as CostTracker and SwarmCheckpointer.
|
|
6
|
+
*
|
|
7
|
+
* @module v1/memory/episodic-store
|
|
8
|
+
*/
|
|
9
|
+
import type { Episode, EpisodicStoreConfig } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Accumulates agent runs into episodes and persists them as JSON-lines.
|
|
12
|
+
*/
|
|
13
|
+
export declare class EpisodicStore {
|
|
14
|
+
private readonly filePath;
|
|
15
|
+
private readonly maxRunsPerEpisode;
|
|
16
|
+
private readonly summarizer;
|
|
17
|
+
private currentEpisodeId;
|
|
18
|
+
private currentSessionId;
|
|
19
|
+
private currentRunIds;
|
|
20
|
+
private currentAgentSlugs;
|
|
21
|
+
private currentTaskTypes;
|
|
22
|
+
private currentContent;
|
|
23
|
+
private currentStartedAt;
|
|
24
|
+
constructor(config: EpisodicStoreConfig);
|
|
25
|
+
/** Read all episodes from the JSONL file (oldest-first). */
|
|
26
|
+
readAll(): Episode[];
|
|
27
|
+
/** Append a single episode to the file. */
|
|
28
|
+
private append;
|
|
29
|
+
/**
|
|
30
|
+
* Add a run to the current episode. Starts a new episode if none is open.
|
|
31
|
+
* Auto-closes the episode when maxRunsPerEpisode is reached.
|
|
32
|
+
*
|
|
33
|
+
* @returns The closed Episode if auto-close triggered, otherwise null.
|
|
34
|
+
*/
|
|
35
|
+
addRun(runId: string, agentSlug: string, taskType: string, content: string, sessionId?: string): Promise<Episode | null>;
|
|
36
|
+
/**
|
|
37
|
+
* Close the current episode, persist it, and return it.
|
|
38
|
+
* Returns null if there is no open episode.
|
|
39
|
+
*/
|
|
40
|
+
closeEpisode(): Promise<Episode | null>;
|
|
41
|
+
/**
|
|
42
|
+
* Simple substring search on episode summaries.
|
|
43
|
+
*/
|
|
44
|
+
search(query: string, maxEpisodes?: number): Episode[];
|
|
45
|
+
/**
|
|
46
|
+
* Retrieve an episode by its ID.
|
|
47
|
+
*/
|
|
48
|
+
getById(episodeId: string): Episode | undefined;
|
|
49
|
+
/**
|
|
50
|
+
* List all episodes for a given session.
|
|
51
|
+
*/
|
|
52
|
+
listBySession(sessionId: string): Episode[];
|
|
53
|
+
/**
|
|
54
|
+
* Check whether an episode is currently open.
|
|
55
|
+
*/
|
|
56
|
+
hasOpenEpisode(): boolean;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=episodic-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"episodic-store.d.ts","sourceRoot":"","sources":["../src/episodic-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAS/D;;GAEG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuC;IAGlE,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,gBAAgB,CAAc;IACtC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,iBAAiB,CAA0B;IACnD,OAAO,CAAC,gBAAgB,CAA0B;IAClD,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,gBAAgB,CAAa;gBAEzB,MAAM,EAAE,mBAAmB;IAgBvC,4DAA4D;IAC5D,OAAO,IAAI,OAAO,EAAE;IAUpB,2CAA2C;IAC3C,OAAO,CAAC,MAAM;IAQd;;;;;OAKG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IA8B1B;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAkC7C;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,GAAE,MAAW,GAAG,OAAO,EAAE;IAQ1D;;OAEG;IACH,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS;IAK/C;;OAEG;IACH,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,EAAE;IAK3C;;OAEG;IACH,cAAc,IAAI,OAAO;CAG1B"}
|