@monoes/memory 1.0.16 → 1.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/atomic-file.js +1 -1
- package/dist/atomic-file.js.map +1 -1
- package/dist/cache-manager.d.ts +1 -1
- package/dist/cache-manager.d.ts.map +1 -1
- package/dist/cache-manager.js +1 -2
- package/dist/cache-manager.js.map +1 -1
- package/dist/checkpointer.d.ts +6 -6
- package/dist/checkpointer.d.ts.map +1 -1
- package/dist/checkpointer.js +10 -10
- package/dist/checkpointer.js.map +1 -1
- package/dist/hnsw-index.d.ts +32 -1
- package/dist/hnsw-index.d.ts.map +1 -1
- package/dist/hnsw-index.js +73 -29
- package/dist/hnsw-index.js.map +1 -1
- package/dist/index.d.ts +16 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -8
- package/dist/index.js.map +1 -1
- package/dist/knowledge/document-chunker.d.ts.map +1 -1
- package/dist/knowledge/document-chunker.js +9 -3
- package/dist/knowledge/document-chunker.js.map +1 -1
- package/dist/knowledge/index.d.ts +4 -4
- package/dist/knowledge/index.d.ts.map +1 -1
- package/dist/knowledge/index.js +1 -1
- package/dist/knowledge/index.js.map +1 -1
- package/dist/knowledge/knowledge-store.d.ts.map +1 -1
- package/dist/knowledge/knowledge-store.js +2 -2
- package/dist/knowledge/knowledge-store.js.map +1 -1
- package/dist/math-utils.d.ts.map +1 -1
- package/dist/math-utils.js.map +1 -1
- package/dist/migration.d.ts +1 -1
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +11 -22
- package/dist/migration.js.map +1 -1
- package/dist/prompt-version-store.d.ts.map +1 -1
- package/dist/prompt-version-store.js.map +1 -1
- package/dist/query-builder.d.ts.map +1 -1
- package/dist/query-builder.js +12 -4
- package/dist/query-builder.js.map +1 -1
- package/dist/sql-backend.d.ts +103 -5
- package/dist/sql-backend.d.ts.map +1 -1
- package/dist/sql-backend.js +341 -27
- package/dist/sql-backend.js.map +1 -1
- package/dist/sql-schema.d.ts.map +1 -1
- package/dist/sql-schema.js +116 -47
- package/dist/sql-schema.js.map +1 -1
- package/dist/sqlite-backend.d.ts +1 -0
- package/dist/sqlite-backend.d.ts.map +1 -1
- package/dist/sqlite-backend.js +6 -0
- package/dist/sqlite-backend.js.map +1 -1
- package/dist/sqljs-backend.d.ts +9 -0
- package/dist/sqljs-backend.d.ts.map +1 -1
- package/dist/sqljs-backend.js +24 -3
- package/dist/sqljs-backend.js.map +1 -1
- package/dist/types/checkpoint.d.ts +2 -2
- package/dist/types/checkpoint.d.ts.map +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/jsonl-store.js +2 -2
- package/dist/utils/jsonl-store.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -16,11 +16,12 @@
|
|
|
16
16
|
|
|
17
17
|
```bash
|
|
18
18
|
npm install @monoes/memory
|
|
19
|
-
|
|
20
|
-
# optional: native SQLite (faster than the sql.js WASM fallback)
|
|
21
|
-
npm install better-sqlite3
|
|
22
19
|
```
|
|
23
20
|
|
|
21
|
+
`better-sqlite3` ships as a direct dependency (native SQLite; `sql.js` WASM is the fallback storage engine) — no separate install needed.
|
|
22
|
+
|
|
23
|
+
> **Native module install blocked?** If initialization throws `Could not locate the bindings file`, your npm's `allowScripts` policy blocked `better-sqlite3`'s native build — run `npm install-scripts approve better-sqlite3 && npm rebuild better-sqlite3`.
|
|
24
|
+
|
|
24
25
|
## Core Architecture & Schema v3.0.0
|
|
25
26
|
|
|
26
27
|
### Schema Architecture & Storage Engine
|
package/dist/atomic-file.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* not the reverse) and this needs to accept raw Buffer/string data, not
|
|
10
10
|
* just JSON.
|
|
11
11
|
*/
|
|
12
|
-
import {
|
|
12
|
+
import { renameSync, writeFileSync } from 'node:fs';
|
|
13
13
|
/**
|
|
14
14
|
* Write `data` to `filePath` atomically: write to `${filePath}.${pid}.${ts}.tmp`
|
|
15
15
|
* first, then rename over the destination.
|
package/dist/atomic-file.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"atomic-file.js","sourceRoot":"","sources":["../src/atomic-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"atomic-file.js","sourceRoot":"","sources":["../src/atomic-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,IAAqC,EACrC,QAAyB;IAEzB,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;IAC/D,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACzC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,aAAa,CAAC,OAAO,EAAE,IAAW,CAAC,CAAC;IACtC,CAAC;IACD,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAChC,CAAC"}
|
package/dist/cache-manager.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* @module v1/memory/cache-manager
|
|
8
8
|
*/
|
|
9
9
|
import { EventEmitter } from 'node:events';
|
|
10
|
-
import { CacheConfig, CacheStats, MemoryEntry } from './types.js';
|
|
10
|
+
import type { CacheConfig, CacheStats, MemoryEntry } from './types.js';
|
|
11
11
|
/**
|
|
12
12
|
* High-performance LRU Cache with TTL support
|
|
13
13
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache-manager.d.ts","sourceRoot":"","sources":["../src/cache-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,
|
|
1
|
+
{"version":3,"file":"cache-manager.d.ts","sourceRoot":"","sources":["../src/cache-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,WAAW,EAAe,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAcpF;;;;;;;;;;GAUG;AACH,qBAAa,YAAY,CAAC,CAAC,GAAG,WAAW,CAAE,SAAQ,YAAY;IAC7D,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,IAAI,CAA2B;IACvC,OAAO,CAAC,IAAI,CAA2B;IACvC,OAAO,CAAC,aAAa,CAAa;IAGlC,OAAO,CAAC,KAAK,CAYX;IAGF,OAAO,CAAC,eAAe,CAA+B;IAEtD,YAAY,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM,EAI5C;IAED;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CA6BzB;IAED;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CA4D5C;IAED;;OAEG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAY3B;IAED;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAQxB;IAED;;OAEG;IACH,KAAK,IAAI,IAAI,CAQZ;IAED;;OAEG;IACH,QAAQ,IAAI,UAAU,CAUrB;IAED;;OAEG;IACH,IAAI,IAAI,MAAM,EAAE,CAEf;IAED;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;OAEG;IACG,QAAQ,CACZ,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EACnD,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC,CAcf;IAED;;OAEG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAS9E;IAED;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI,CAKnE;IAED;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAalD;IAED;;OAEG;IACH,QAAQ,IAAI,IAAI,CAOf;IAID,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,UAAU;IAelB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,QAAQ;IAchB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,cAAc;CAevB;eAEc,YAAY"}
|
package/dist/cache-manager.js
CHANGED
|
@@ -91,8 +91,7 @@ export class CacheManager extends EventEmitter {
|
|
|
91
91
|
const entryMemory = this.estimateSize(data);
|
|
92
92
|
// Evict entries if needed for memory pressure
|
|
93
93
|
if (this.config.maxMemory) {
|
|
94
|
-
while (this.currentMemory + entryMemory > this.config.maxMemory &&
|
|
95
|
-
this.cache.size > 0) {
|
|
94
|
+
while (this.currentMemory + entryMemory > this.config.maxMemory && this.cache.size > 0) {
|
|
96
95
|
this.evictLRU();
|
|
97
96
|
}
|
|
98
97
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache-manager.js","sourceRoot":"","sources":["../src/cache-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"cache-manager.js","sourceRoot":"","sources":["../src/cache-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAe3C;;;;;;;;;;GAUG;AACH,MAAM,OAAO,YAA8B,SAAQ,YAAY;IACrD,MAAM,CAAc;IACpB,KAAK,GAA4B,IAAI,GAAG,EAAE,CAAC;IAC3C,IAAI,GAAsB,IAAI,CAAC;IAC/B,IAAI,GAAsB,IAAI,CAAC;IAC/B,aAAa,GAAW,CAAC,CAAC;IAElC,aAAa;IACL,KAAK,GAMT;QACF,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,CAAC;KACV,CAAC;IAEF,gBAAgB;IACR,eAAe,GAA0B,IAAI,CAAC;IAEtD,YAAY,MAAM,GAAyB,EAAE;QAC3C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW;QACb,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAEzB,qCAAqC;QACrC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAEhC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW,EAAE,IAAO,EAAE,GAAY;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;QAExC,8BAA8B;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,YAAY,EAAE,CAAC;YACjB,qEAAqE;YACrE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC;YACvD,YAAY,CAAC,SAAS,GAAG,OAAO,CAAC;YACjC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;YAC/B,YAAY,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC;YAClC,YAAY,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,GAAG,QAAQ,CAAC;YAC9C,YAAY,CAAC,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC;YAExC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,iCAAiC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAE5C,8CAA8C;QAC9C,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,aAAa,GAAG,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC;QAED,kBAAkB;QAClB,MAAM,WAAW,GAAmB;YAClC,IAAI;YACJ,QAAQ,EAAE,GAAG;YACb,SAAS,EAAE,GAAG,GAAG,QAAQ;YACzB,cAAc,EAAE,GAAG;YACnB,WAAW,EAAE,CAAC;SACf,CAAC;QAEF,MAAM,IAAI,GAAe;YACvB,GAAG;YACH,KAAK,EAAE,WAAW;YAClB,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,IAAI;YACV,IAAI,EAAE,IAAI;SACX,CAAC;QAEF,eAAe;QACf,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,aAAa,IAAI,WAAW,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAEpB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,GAAW;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC;QAErC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW;QACb,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAEvB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAChD,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACzB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;YAC/B,WAAW,EAAE,IAAI,CAAC,aAAa;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,IAAI;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,IAAc,EACd,MAAmD,EACnD,GAAY;QAEZ,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAErD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,MAAwB,EAAE,GAAY;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,OAAsD;QAC3D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAAwB;QACxC,MAAM,KAAK,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAC1E,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjB,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACpF,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9B,CAAC;IAED,8BAA8B;IAEtB,WAAW,CAAC,MAA4B;QAC9C,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,MAAM,EAAE,oBAAoB;YAC/C,UAAU,EAAE,MAAM,CAAC,UAAU,KAAK,KAAK;YACvC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,KAAK;SAC3C,CAAC;IACJ,CAAC;IAEO,SAAS,CAAC,KAAqB;QACrC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;IACtC,CAAC;IAEO,YAAY,CAAC,IAAO;QAC1B,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,wBAAwB;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,CAAC,uCAAuC;QACtD,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAAgB;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QAEtB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAAgB;QACjC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,IAAgB;QAClC,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI;YAAE,OAAO;QAE/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAEO,QAAQ;QACd,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QAExC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9B,IAAI,CAAC,aAAa,IAAI,WAAW,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAEvB,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IACnD,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;YACtC,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC,EAAE,KAAK,CAAC,CAAC;QACV,oEAAoE;QACpE,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK;YAAE,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/D,CAAC;IAEO,cAAc;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;gBAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;CACF;AAED,eAAe,YAAY,CAAC"}
|
package/dist/checkpointer.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* MonoswarmCheckpointer — JSON-lines file-based graph checkpointing.
|
|
3
3
|
*
|
|
4
4
|
* Each checkpoint is appended as a single JSON line to a file,
|
|
5
5
|
* similar to the CostTracker pattern used elsewhere in the project.
|
|
6
6
|
* No native SQLite dependency required.
|
|
7
7
|
*/
|
|
8
|
-
import type { AgentState,
|
|
9
|
-
export type { AgentState,
|
|
8
|
+
import type { AgentState, CheckpointMeta, SwarmCheckpoint } from './types/checkpoint.js';
|
|
9
|
+
export type { AgentState, CheckpointMeta, SwarmCheckpoint };
|
|
10
10
|
/** Configuration accepted by the checkpointer constructor. */
|
|
11
11
|
export interface CheckpointerConfig {
|
|
12
12
|
dbPath: string;
|
|
13
|
-
|
|
13
|
+
monoswarmId: string;
|
|
14
14
|
sessionId: string;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
@@ -19,10 +19,10 @@ export interface CheckpointerConfig {
|
|
|
19
19
|
* Each line in the file is a self-contained JSON object representing one
|
|
20
20
|
* {@link SwarmCheckpoint}. Newest entries are always appended at the end.
|
|
21
21
|
*/
|
|
22
|
-
export declare class
|
|
22
|
+
export declare class MonoswarmCheckpointer {
|
|
23
23
|
private stepCounter;
|
|
24
24
|
private readonly dbPath;
|
|
25
|
-
private readonly
|
|
25
|
+
private readonly monoswarmId;
|
|
26
26
|
private readonly sessionId;
|
|
27
27
|
private lastCheckpoint;
|
|
28
28
|
private lastKnownMtimeMs;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkpointer.d.ts","sourceRoot":"","sources":["../src/checkpointer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"checkpointer.d.ts","sourceRoot":"","sources":["../src/checkpointer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAEzF,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;AAE5D,8DAA8D;AAC9D,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAanC,OAAO,CAAC,cAAc,CAAgC;IACtD,OAAO,CAAC,gBAAgB,CAAM;IAE9B,YAAY,MAAM,EAAE,kBAAkB,EASrC;IAED,qEAAqE;IACrE,OAAO,CAAC,cAAc;IAQtB,0FAA0F;IAC1F,OAAO,CAAC,eAAe;IAevB,qFAAqF;IACrF,OAAO,CAAC,mBAAmB;IAQ3B,yDAAyD;IACzD,OAAO,CAAC,OAAO;IAUf,qDAAqD;IACrD,OAAO,CAAC,QAAQ;IAMhB,sEAAsE;IACtE,OAAO,CAAC,WAAW;IAanB;;;;OAIG;IACH,QAAQ,CACN,WAAW,EAAE,UAAU,EAAE,EACzB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EACxC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpC,OAAO,EAAE,eAAe,CAAC,SAAS,CAAC,EACnC,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CA4BR;IAED;;;;;OAKG;IACH,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,GAAG,IAAI,CAc3D;IAED;;;;OAIG;IACH,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAGjD;IAED;;;;OAIG;IACH,IAAI,CAAC,KAAK,SAAK,GAAG,cAAc,EAAE,CAejC;IAED;;;;;;;OAOG;IACH,MAAM,IAAI,eAAe,GAAG,IAAI,CAG/B;IAED;;OAEG;IACH,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,GACX;QAAE,WAAW,EAAE,MAAM,EAAE,CAAC;QAAC,aAAa,EAAE,MAAM,EAAE,CAAC;QAAC,aAAa,EAAE,MAAM,EAAE,CAAA;KAAE,CA0B7E;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,SAAI,GAAG,MAAM,CAW/B;CACF"}
|
package/dist/checkpointer.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* MonoswarmCheckpointer — JSON-lines file-based graph checkpointing.
|
|
3
3
|
*
|
|
4
4
|
* Each checkpoint is appended as a single JSON line to a file,
|
|
5
5
|
* similar to the CostTracker pattern used elsewhere in the project.
|
|
6
6
|
* No native SQLite dependency required.
|
|
7
7
|
*/
|
|
8
|
-
import {
|
|
9
|
-
import { existsSync, readFileSync,
|
|
8
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
9
|
+
import { appendFileSync, existsSync, readFileSync, statSync } from 'node:fs';
|
|
10
10
|
import { writeFileAtomicSync } from './atomic-file.js';
|
|
11
11
|
/**
|
|
12
12
|
* Persists and retrieves full swarm checkpoints using a JSON-lines file.
|
|
@@ -14,10 +14,10 @@ import { writeFileAtomicSync } from './atomic-file.js';
|
|
|
14
14
|
* Each line in the file is a self-contained JSON object representing one
|
|
15
15
|
* {@link SwarmCheckpoint}. Newest entries are always appended at the end.
|
|
16
16
|
*/
|
|
17
|
-
export class
|
|
17
|
+
export class MonoswarmCheckpointer {
|
|
18
18
|
stepCounter = 0;
|
|
19
19
|
dbPath;
|
|
20
|
-
|
|
20
|
+
monoswarmId;
|
|
21
21
|
sessionId;
|
|
22
22
|
// Cache of the newest checkpoint this instance knows about. saveIncremental()
|
|
23
23
|
// used to call latest() -> readAll(), which reads and JSON.parses every
|
|
@@ -26,7 +26,7 @@ export class SwarmCheckpointer {
|
|
|
26
26
|
// checkpoint in memory makes latest() (and therefore saveIncremental) O(1)
|
|
27
27
|
// in the common case. This class is a crash-recovery mechanism ("resume
|
|
28
28
|
// after crash" implies a *new* process reading what a *previous* process
|
|
29
|
-
// wrote — see index.ts's GAP-007
|
|
29
|
+
// wrote — see index.ts's GAP-007 MonoswarmCheckpointer wiring), so a second
|
|
30
30
|
// writer touching the same dbPath from another process is an expected
|
|
31
31
|
// scenario, not an edge case — the cache is therefore invalidated via a
|
|
32
32
|
// cheap mtime check (statSync, not a full read+parse) rather than trusted
|
|
@@ -35,7 +35,7 @@ export class SwarmCheckpointer {
|
|
|
35
35
|
lastKnownMtimeMs = -1;
|
|
36
36
|
constructor(config) {
|
|
37
37
|
this.dbPath = config.dbPath;
|
|
38
|
-
this.
|
|
38
|
+
this.monoswarmId = config.monoswarmId;
|
|
39
39
|
this.sessionId = config.sessionId;
|
|
40
40
|
// Continue the step sequence across process restarts: a fresh instance
|
|
41
41
|
// must not restart step numbering from 0 while older, higher-step
|
|
@@ -115,7 +115,7 @@ export class SwarmCheckpointer {
|
|
|
115
115
|
this.stepCounter += 1;
|
|
116
116
|
const checkpoint = {
|
|
117
117
|
checkpointId,
|
|
118
|
-
|
|
118
|
+
monoswarmId: this.monoswarmId,
|
|
119
119
|
sessionId: this.sessionId,
|
|
120
120
|
step: this.stepCounter,
|
|
121
121
|
trigger,
|
|
@@ -126,7 +126,7 @@ export class SwarmCheckpointer {
|
|
|
126
126
|
createdAt: new Date().toISOString(),
|
|
127
127
|
parentCheckpointId,
|
|
128
128
|
};
|
|
129
|
-
appendFileSync(this.dbPath, JSON.stringify(checkpoint)
|
|
129
|
+
appendFileSync(this.dbPath, `${JSON.stringify(checkpoint)}\n`, 'utf-8');
|
|
130
130
|
this.lastCheckpoint = checkpoint;
|
|
131
131
|
this.lastKnownMtimeMs = this.currentMtimeMs();
|
|
132
132
|
return checkpointId;
|
|
@@ -172,7 +172,7 @@ export class SwarmCheckpointer {
|
|
|
172
172
|
.slice(0, limit)
|
|
173
173
|
.map((c) => ({
|
|
174
174
|
checkpointId: c.checkpointId,
|
|
175
|
-
|
|
175
|
+
monoswarmId: c.monoswarmId,
|
|
176
176
|
sessionId: c.sessionId,
|
|
177
177
|
step: c.step,
|
|
178
178
|
trigger: c.trigger,
|
package/dist/checkpointer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkpointer.js","sourceRoot":"","sources":["../src/checkpointer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"checkpointer.js","sourceRoot":"","sources":["../src/checkpointer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAYvD;;;;;GAKG;AACH,MAAM,OAAO,qBAAqB;IACxB,WAAW,GAAG,CAAC,CAAC;IACP,MAAM,CAAS;IACf,WAAW,CAAS;IACpB,SAAS,CAAS;IACnC,8EAA8E;IAC9E,wEAAwE;IACxE,0EAA0E;IAC1E,0EAA0E;IAC1E,2EAA2E;IAC3E,wEAAwE;IACxE,yEAAyE;IACzE,4EAA4E;IAC5E,sEAAsE;IACtE,wEAAwE;IACxE,0EAA0E;IAC1E,mBAAmB;IACX,cAAc,GAA2B,IAAI,CAAC;IAC9C,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAE9B,YAAY,MAA0B;QACpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,uEAAuE;QACvE,kEAAkE;QAClE,sEAAsE;QACtE,4CAA4C;QAC5C,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAED,qEAAqE;IAC7D,cAAc;QACpB,IAAI,CAAC;YACH,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC,CAAC;QACZ,CAAC;IACH,CAAC;IAED,0FAA0F;IAClF,eAAe;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,MAAM,GAA2B,IAAI,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACpB,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;gBAClB,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;gBACb,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC;QAC7B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IAChD,CAAC;IAED,qFAAqF;IAC7E,mBAAmB;QACzB,IAAI,IAAI,CAAC,cAAc,EAAE,KAAK,IAAI,CAAC,gBAAgB;YAAE,IAAI,CAAC,eAAe,EAAE,CAAC;IAC9E,CAAC;IAED,8EAA8E;IAC9E,sBAAsB;IACtB,8EAA8E;IAE9E,yDAAyD;IACjD,OAAO;QACb,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG;aACP,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC;aACf,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAoB,CAAC,CAAC;IAChE,CAAC;IAED,qDAAqD;IAC7C,QAAQ,CAAC,WAA8B;QAC7C,MAAM,IAAI,GACR,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1F,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,sEAAsE;IAC9D,WAAW,CACjB,WAAyB,EACzB,aAAwC,EACxC,WAAoC;QAEpC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,CAAC;QAC5E,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5D,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAE9E;;;;OAIG;IACH,QAAQ,CACN,WAAyB,EACzB,aAAwC,EACxC,WAAoC,EACpC,OAAmC,EACnC,kBAA2B;QAE3B,wEAAwE;QACxE,wEAAwE;QACxE,uEAAuE;QACvE,0EAA0E;QAC1E,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE3B,MAAM,YAAY,GAAG,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAoB;YAClC,YAAY;YACZ,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,OAAO;YACP,WAAW;YACX,aAAa;YACb,WAAW;YACX,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,WAAW,CAAC;YACpE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,kBAAkB;SACnB,CAAC;QAEF,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxE,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;QACjC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC9C,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,OAAe,EAAE,QAAoB;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,GAAG,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;QAChE,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;YACb,WAAW,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;QAE5C,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;IACvF,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,YAAoB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,YAAY,CAAC,IAAI,IAAI,CAAC;IAClE,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,KAAK,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC3B,OAAO,GAAG;aACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;aAC/B,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC,CAAC,WAAW,CAAC,MAAM;YAChC,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,SAAS,EAAE,CAAC,CAAC,SAAS;SACvB,CAAC,CAAC,CAAC;IACR,CAAC;IAED;;;;;;;OAOG;IACH,MAAM;QACJ,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAI,CACF,MAAc,EACd,IAAY;QAEZ,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,WAAW,IAAI,EAAE;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,WAAW,IAAI,EAAE;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAE5D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,KAAK,MAAM,EAAE,IAAI,KAAK;YAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,IAAI,OAAO;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAErE,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;YACf,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC;oBAAE,SAAS,CAAC,qBAAqB;gBAClE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9E,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,GAAG,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,GAAG,UAAU,CAAC;QACvD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM;YAC/B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
package/dist/hnsw-index.d.ts
CHANGED
|
@@ -12,7 +12,24 @@
|
|
|
12
12
|
* @module v1/memory/hnsw-index
|
|
13
13
|
*/
|
|
14
14
|
import { EventEmitter } from 'node:events';
|
|
15
|
-
import { HNSWConfig, HNSWStats } from './types.js';
|
|
15
|
+
import type { HNSWConfig, HNSWStats } from './types.js';
|
|
16
|
+
/**
|
|
17
|
+
* On-disk representation produced by HNSWIndex.serialize() / consumed by
|
|
18
|
+
* HNSWIndex.deserialize(). `version` guards against loading a shape from a
|
|
19
|
+
* future/incompatible format.
|
|
20
|
+
*/
|
|
21
|
+
export interface HNSWSerialized {
|
|
22
|
+
version: 1;
|
|
23
|
+
config: HNSWConfig;
|
|
24
|
+
entryPoint: string | null;
|
|
25
|
+
maxLevel: number;
|
|
26
|
+
nodes: Array<{
|
|
27
|
+
id: string;
|
|
28
|
+
vectorB64: string;
|
|
29
|
+
level: number;
|
|
30
|
+
connections: Array<[number, string[]]>;
|
|
31
|
+
}>;
|
|
32
|
+
}
|
|
16
33
|
/**
|
|
17
34
|
* HNSW Index implementation for ultra-fast vector similarity search
|
|
18
35
|
*
|
|
@@ -59,6 +76,20 @@ export declare class HNSWIndex extends EventEmitter {
|
|
|
59
76
|
id: string;
|
|
60
77
|
vector: Float32Array;
|
|
61
78
|
}>): Promise<void>;
|
|
79
|
+
/**
|
|
80
|
+
* Serialize the built graph (config, entry point, and every node's vector +
|
|
81
|
+
* per-level connections) so a caller can persist it to disk and reconstruct
|
|
82
|
+
* an identical index later without re-inserting every point. Vectors are
|
|
83
|
+
* base64-encoded Float32 bytes rather than JSON number arrays to keep the
|
|
84
|
+
* serialized size and parse cost down for large indexes.
|
|
85
|
+
*
|
|
86
|
+
* Not supported for quantized indexes (quantizer state isn't captured) —
|
|
87
|
+
* throws rather than silently producing a graph that would deserialize
|
|
88
|
+
* with the wrong vector encoding.
|
|
89
|
+
*/
|
|
90
|
+
serialize(): HNSWSerialized;
|
|
91
|
+
/** Reconstruct an index previously produced by serialize(). */
|
|
92
|
+
static deserialize(data: HNSWSerialized): HNSWIndex;
|
|
62
93
|
/**
|
|
63
94
|
* Get index statistics
|
|
64
95
|
*/
|
package/dist/hnsw-index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hnsw-index.d.ts","sourceRoot":"","sources":["../src/hnsw-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,
|
|
1
|
+
{"version":3,"file":"hnsw-index.d.ts","sourceRoot":"","sources":["../src/hnsw-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAsB,MAAM,YAAY,CAAC;AAoK5E;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,CAAC,CAAC;IACX,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;KACxC,CAAC,CAAC;CACJ;AAsBD;;;;;;;GAOG;AACH,qBAAa,SAAU,SAAQ,YAAY;IACzC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,KAAK,CAYX;IAGF,OAAO,CAAC,SAAS,CAA0B;IAE3C,YAAY,MAAM,GAAE,OAAO,CAAC,UAAU,CAAM,EAQ3C;IAED;;OAEG;IACG,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAmD9D;IAED;;OAEG;IACG,MAAM,CACV,KAAK,EAAE,YAAY,EACnB,CAAC,EAAE,MAAM,EACT,EAAE,CAAC,EAAE,MAAM,GACV,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAgElD;IAED;;OAEG;IACG,iBAAiB,CACrB,KAAK,EAAE,YAAY,EACnB,CAAC,EAAE,MAAM,EACT,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,EAC/B,EAAE,CAAC,EAAE,MAAM,GACV,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAMlD;IAED;;OAEG;IACG,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CA2C9C;IAED;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBjF;IAED;;;;;;;;;;OAUG;IACH,SAAS,IAAI,cAAc,CAwB1B;IAED,+DAA+D;IAC/D,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAiBlD;IAED;;OAEG;IACH,QAAQ,IAAI,SAAS,CAiBpB;IAED;;OAEG;IACH,KAAK,IAAI,IAAI,CAWZ;IAED;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAEvB;IAED;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAID,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,cAAc;YAMR,UAAU;IA0DxB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAgE5B,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,gBAAgB;IA6BxB,OAAO,CAAC,QAAQ;IAehB,OAAO,CAAC,cAAc;IAiBtB;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAUhC;;OAEG;IACH,OAAO,CAAC,eAAe;IAmBvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAkBzB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,iBAAiB;CAO1B;eA6Gc,SAAS"}
|
package/dist/hnsw-index.js
CHANGED
|
@@ -203,13 +203,9 @@ export class HNSWIndex extends EventEmitter {
|
|
|
203
203
|
throw new Error('Index is full, cannot add more elements');
|
|
204
204
|
}
|
|
205
205
|
// Quantize if enabled
|
|
206
|
-
const storedVector = this.quantizer
|
|
207
|
-
? this.quantizer.encode(vector)
|
|
208
|
-
: vector;
|
|
206
|
+
const storedVector = this.quantizer ? this.quantizer.encode(vector) : vector;
|
|
209
207
|
// Pre-normalize vector for O(1) cosine similarity
|
|
210
|
-
const normalizedVector = this.config.metric === 'cosine'
|
|
211
|
-
? this.normalizeVector(storedVector)
|
|
212
|
-
: null;
|
|
208
|
+
const normalizedVector = this.config.metric === 'cosine' ? this.normalizeVector(storedVector) : null;
|
|
213
209
|
// Generate random level for new node
|
|
214
210
|
const level = this.getRandomLevel();
|
|
215
211
|
const node = {
|
|
@@ -251,21 +247,17 @@ export class HNSWIndex extends EventEmitter {
|
|
|
251
247
|
}
|
|
252
248
|
const searchEf = ef || Math.max(k, this.config.efConstruction);
|
|
253
249
|
// Quantize query if needed
|
|
254
|
-
const queryVector = this.quantizer
|
|
255
|
-
? this.quantizer.encode(query)
|
|
256
|
-
: query;
|
|
250
|
+
const queryVector = this.quantizer ? this.quantizer.encode(query) : query;
|
|
257
251
|
// Pre-normalize query for O(1) cosine similarity
|
|
258
|
-
const normalizedQuery = this.config.metric === 'cosine'
|
|
259
|
-
? this.normalizeVector(queryVector)
|
|
260
|
-
: null;
|
|
252
|
+
const normalizedQuery = this.config.metric === 'cosine' ? this.normalizeVector(queryVector) : null;
|
|
261
253
|
// Start from entry point and search down the layers
|
|
262
254
|
let currentNode = this.entryPoint;
|
|
263
|
-
let
|
|
255
|
+
let _currentDist = this.distanceOptimized(queryVector, normalizedQuery, this.nodes.get(currentNode));
|
|
264
256
|
// Search through layers from top to 1
|
|
265
257
|
for (let level = this.maxLevel; level > 0; level--) {
|
|
266
258
|
const layerResult = this.searchLayerOptimized(queryVector, normalizedQuery, currentNode, 1, level);
|
|
267
259
|
currentNode = layerResult[0]?.id || currentNode;
|
|
268
|
-
|
|
260
|
+
_currentDist = this.distanceOptimized(queryVector, normalizedQuery, this.nodes.get(currentNode));
|
|
269
261
|
}
|
|
270
262
|
// Search layer 0 with ef candidates using heap-based search
|
|
271
263
|
const candidates = this.searchLayerOptimized(queryVector, normalizedQuery, currentNode, searchEf, 0);
|
|
@@ -283,9 +275,7 @@ export class HNSWIndex extends EventEmitter {
|
|
|
283
275
|
// Over-fetch to account for filtered results
|
|
284
276
|
const overFetchFactor = 3;
|
|
285
277
|
const candidates = await this.search(query, k * overFetchFactor, ef);
|
|
286
|
-
return candidates
|
|
287
|
-
.filter((c) => filter(c.id))
|
|
288
|
-
.slice(0, k);
|
|
278
|
+
return candidates.filter((c) => filter(c.id)).slice(0, k);
|
|
289
279
|
}
|
|
290
280
|
/**
|
|
291
281
|
* Remove a point from the index
|
|
@@ -348,14 +338,67 @@ export class HNSWIndex extends EventEmitter {
|
|
|
348
338
|
buildTime,
|
|
349
339
|
});
|
|
350
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Serialize the built graph (config, entry point, and every node's vector +
|
|
343
|
+
* per-level connections) so a caller can persist it to disk and reconstruct
|
|
344
|
+
* an identical index later without re-inserting every point. Vectors are
|
|
345
|
+
* base64-encoded Float32 bytes rather than JSON number arrays to keep the
|
|
346
|
+
* serialized size and parse cost down for large indexes.
|
|
347
|
+
*
|
|
348
|
+
* Not supported for quantized indexes (quantizer state isn't captured) —
|
|
349
|
+
* throws rather than silently producing a graph that would deserialize
|
|
350
|
+
* with the wrong vector encoding.
|
|
351
|
+
*/
|
|
352
|
+
serialize() {
|
|
353
|
+
if (this.quantizer) {
|
|
354
|
+
throw new Error('HNSWIndex.serialize() does not support quantized indexes');
|
|
355
|
+
}
|
|
356
|
+
const nodes = [];
|
|
357
|
+
for (const node of this.nodes.values()) {
|
|
358
|
+
const buf = Buffer.from(node.vector.buffer, node.vector.byteOffset, node.vector.byteLength);
|
|
359
|
+
nodes.push({
|
|
360
|
+
id: node.id,
|
|
361
|
+
vectorB64: buf.toString('base64'),
|
|
362
|
+
level: node.level,
|
|
363
|
+
connections: Array.from(node.connections.entries()).map(([lvl, set]) => [
|
|
364
|
+
lvl,
|
|
365
|
+
Array.from(set),
|
|
366
|
+
]),
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
return {
|
|
370
|
+
version: 1,
|
|
371
|
+
config: this.config,
|
|
372
|
+
entryPoint: this.entryPoint,
|
|
373
|
+
maxLevel: this.maxLevel,
|
|
374
|
+
nodes,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
/** Reconstruct an index previously produced by serialize(). */
|
|
378
|
+
static deserialize(data) {
|
|
379
|
+
if (data.version !== 1) {
|
|
380
|
+
throw new Error(`HNSWIndex.deserialize: unsupported version ${data.version}`);
|
|
381
|
+
}
|
|
382
|
+
const index = new HNSWIndex(data.config);
|
|
383
|
+
for (const n of data.nodes) {
|
|
384
|
+
const buf = Buffer.from(n.vectorB64, 'base64');
|
|
385
|
+
const vector = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
386
|
+
const normalizedVector = data.config.metric === 'cosine' ? index.normalizeVector(vector) : null;
|
|
387
|
+
const connections = new Map();
|
|
388
|
+
for (const [lvl, ids] of n.connections)
|
|
389
|
+
connections.set(lvl, new Set(ids));
|
|
390
|
+
index.nodes.set(n.id, { id: n.id, vector, normalizedVector, connections, level: n.level });
|
|
391
|
+
}
|
|
392
|
+
index.entryPoint = data.entryPoint;
|
|
393
|
+
index.maxLevel = data.maxLevel;
|
|
394
|
+
return index;
|
|
395
|
+
}
|
|
351
396
|
/**
|
|
352
397
|
* Get index statistics
|
|
353
398
|
*/
|
|
354
399
|
getStats() {
|
|
355
400
|
const vectorCount = this.nodes.size;
|
|
356
|
-
const avgSearchTime = this.stats.searchCount > 0
|
|
357
|
-
? this.stats.totalSearchTime / this.stats.searchCount
|
|
358
|
-
: 0;
|
|
401
|
+
const avgSearchTime = this.stats.searchCount > 0 ? this.stats.totalSearchTime / this.stats.searchCount : 0;
|
|
359
402
|
// Estimate memory usage
|
|
360
403
|
const bytesPerVector = this.config.dimensions * 4; // Float32 = 4 bytes
|
|
361
404
|
const connectionOverhead = this.config.M * 8 * (this.maxLevel + 1); // Approximate
|
|
@@ -431,7 +474,7 @@ export class HNSWIndex extends EventEmitter {
|
|
|
431
474
|
const selectedNeighbors = this.selectNeighbors(node.id, query, neighbors, this.config.M);
|
|
432
475
|
// Add connections
|
|
433
476
|
for (const neighbor of selectedNeighbors) {
|
|
434
|
-
node.connections.get(level)
|
|
477
|
+
node.connections.get(level)?.add(neighbor.id);
|
|
435
478
|
this.nodes.get(neighbor.id)?.connections.get(level)?.add(node.id);
|
|
436
479
|
// Prune connections if over limit
|
|
437
480
|
const neighborNode = this.nodes.get(neighbor.id);
|
|
@@ -507,12 +550,10 @@ export class HNSWIndex extends EventEmitter {
|
|
|
507
550
|
distance: priority,
|
|
508
551
|
}));
|
|
509
552
|
}
|
|
510
|
-
selectNeighbors(nodeId,
|
|
553
|
+
selectNeighbors(nodeId, _query, candidates, M) {
|
|
511
554
|
// candidates arrive distance-ascending from searchLayerOptimized/toSortedArray;
|
|
512
555
|
// filter preserves that order, so no additional sort is needed.
|
|
513
|
-
return candidates
|
|
514
|
-
.filter((c) => c.id !== nodeId)
|
|
515
|
-
.slice(0, M);
|
|
556
|
+
return candidates.filter((c) => c.id !== nodeId).slice(0, M);
|
|
516
557
|
}
|
|
517
558
|
pruneConnections(node, level, maxConnections) {
|
|
518
559
|
const connections = node.connections.get(level);
|
|
@@ -589,16 +630,16 @@ export class HNSWIndex extends EventEmitter {
|
|
|
589
630
|
normalizeVector(vector) {
|
|
590
631
|
let norm = 0;
|
|
591
632
|
for (let i = 0; i < vector.length; i++) {
|
|
592
|
-
const v = isFinite(vector[i]) ? vector[i] : 0;
|
|
633
|
+
const v = Number.isFinite(vector[i]) ? vector[i] : 0;
|
|
593
634
|
norm += v * v;
|
|
594
635
|
}
|
|
595
636
|
norm = Math.sqrt(norm);
|
|
596
|
-
if (norm === 0 || !isFinite(norm)) {
|
|
637
|
+
if (norm === 0 || !Number.isFinite(norm)) {
|
|
597
638
|
return new Float32Array(vector.length); // safe zero vector for non-finite input
|
|
598
639
|
}
|
|
599
640
|
const normalized = new Float32Array(vector.length);
|
|
600
641
|
for (let i = 0; i < vector.length; i++) {
|
|
601
|
-
normalized[i] = (isFinite(vector[i]) ? vector[i] : 0) / norm;
|
|
642
|
+
normalized[i] = (Number.isFinite(vector[i]) ? vector[i] : 0) / norm;
|
|
602
643
|
}
|
|
603
644
|
return normalized;
|
|
604
645
|
}
|
|
@@ -653,6 +694,9 @@ class Quantizer {
|
|
|
653
694
|
* Encode a vector using quantization
|
|
654
695
|
*/
|
|
655
696
|
encode(vector) {
|
|
697
|
+
if (vector.length !== this.dimensions) {
|
|
698
|
+
throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}`);
|
|
699
|
+
}
|
|
656
700
|
switch (this.config.type) {
|
|
657
701
|
case 'binary':
|
|
658
702
|
return this.binaryQuantize(vector);
|
|
@@ -700,7 +744,7 @@ class Quantizer {
|
|
|
700
744
|
}
|
|
701
745
|
const range = max - min || 1;
|
|
702
746
|
const bits = this.config.bits || 8;
|
|
703
|
-
const levels =
|
|
747
|
+
const levels = 2 ** bits;
|
|
704
748
|
// Quantize each value to [0, levels-1] and normalize back to [0, 1]
|
|
705
749
|
// so the resulting vector is compatible with cosine/euclidean distance.
|
|
706
750
|
const quantized = new Float32Array(vector.length);
|