opencode-mem 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/services/api-handlers.d.ts.map +1 -1
- package/dist/services/api-handlers.js +10 -14
- package/dist/services/client.js +2 -2
- package/dist/services/migration-service.d.ts.map +1 -1
- package/dist/services/migration-service.js +1 -1
- package/dist/services/sqlite/connection-manager.d.ts.map +1 -1
- package/dist/services/sqlite/connection-manager.js +0 -17
- package/dist/services/sqlite/hnsw-index.d.ts +35 -0
- package/dist/services/sqlite/hnsw-index.d.ts.map +1 -0
- package/dist/services/sqlite/hnsw-index.js +207 -0
- package/dist/services/sqlite/shard-manager.d.ts +0 -9
- package/dist/services/sqlite/shard-manager.d.ts.map +1 -1
- package/dist/services/sqlite/shard-manager.js +2 -26
- package/dist/services/sqlite/sqlite-bootstrap.d.ts +0 -18
- package/dist/services/sqlite/sqlite-bootstrap.d.ts.map +1 -1
- package/dist/services/sqlite/sqlite-bootstrap.js +4 -119
- package/dist/services/sqlite/vector-search.d.ts +7 -4
- package/dist/services/sqlite/vector-search.d.ts.map +1 -1
- package/dist/services/sqlite/vector-search.js +30 -44
- package/package.json +2 -2
- package/native/darwin-arm64/.gitkeep +0 -0
- package/native/darwin-arm64/libsqlite3.dylib +0 -0
- package/native/darwin-x64/.gitkeep +0 -0
- package/native/darwin-x64/libsqlite3.dylib +0 -0
package/README.md
CHANGED
|
@@ -20,7 +20,7 @@ A persistent memory system for AI coding agents that enables long-term context r
|
|
|
20
20
|
|
|
21
21
|
## Core Features
|
|
22
22
|
|
|
23
|
-
Local vector database with SQLite, persistent project memories, automatic user profile learning, unified memory-prompt timeline, full-featured web UI, intelligent prompt-based memory extraction, multi-provider AI support (OpenAI, Anthropic), 12+ local embedding models, smart deduplication, and built-in privacy protection.
|
|
23
|
+
Local vector database with SQLite + HNSW (hnswlib-wasm), persistent project memories, automatic user profile learning, unified memory-prompt timeline, full-featured web UI, intelligent prompt-based memory extraction, multi-provider AI support (OpenAI, Anthropic), 12+ local embedding models, smart deduplication, and built-in privacy protection.
|
|
24
24
|
|
|
25
25
|
## Getting Started
|
|
26
26
|
|
|
@@ -32,7 +32,7 @@ Add to your OpenCode configuration at `~/.config/opencode/opencode.json`:
|
|
|
32
32
|
}
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
-
The plugin downloads automatically on next startup.
|
|
35
|
+
The plugin downloads automatically on next startup. No additional dependencies required - vector search is powered by pure WASM for cross-platform compatibility.
|
|
36
36
|
|
|
37
37
|
## Usage Examples
|
|
38
38
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api-handlers.d.ts","sourceRoot":"","sources":["../../src/services/api-handlers.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGpD,UAAU,WAAW,CAAC,CAAC,GAAG,GAAG;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,CAAC,CAAC;IACT,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,MAAM;IACd,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,OAAO;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,iBAAiB,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAmDD,wBAAsB,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC;IAAE,OAAO,EAAE,OAAO,EAAE,CAAA;CAAE,CAAC,CAAC,CAiCnF;AAED,wBAAsB,kBAAkB,CACtC,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,GAAE,MAAU,EAChB,QAAQ,GAAE,MAAW,EACrB,cAAc,GAAE,OAAc,GAC7B,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAsIvD;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,OAAO,CAAC,WAAW,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAiDvC;AAED,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,MAAM,EACV,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,WAAW,CAAC;IAAE,aAAa,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC,CA0BlD;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EAAE,EACb,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,WAAW,CAAC;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAa3C;AAED,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,MAAM,EACV,IAAI,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAC7D,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAuD5B;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,KAAK,gBAAgB,GAAG,eAAe,GAAG,eAAe,CAAC;AAE1D,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,GAAE,MAAU,EAChB,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"api-handlers.d.ts","sourceRoot":"","sources":["../../src/services/api-handlers.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGpD,UAAU,WAAW,CAAC,CAAC,GAAG,GAAG;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,CAAC,CAAC;IACT,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,MAAM;IACd,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,OAAO;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,iBAAiB,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAmDD,wBAAsB,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC;IAAE,OAAO,EAAE,OAAO,EAAE,CAAA;CAAE,CAAC,CAAC,CAiCnF;AAED,wBAAsB,kBAAkB,CACtC,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,GAAE,MAAU,EAChB,QAAQ,GAAE,MAAW,EACrB,cAAc,GAAE,OAAc,GAC7B,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAsIvD;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,OAAO,CAAC,WAAW,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAiDvC;AAED,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,MAAM,EACV,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,WAAW,CAAC;IAAE,aAAa,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC,CA0BlD;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EAAE,EACb,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,WAAW,CAAC;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAa3C;AAED,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,MAAM,EACV,IAAI,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAC7D,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAuD5B;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,KAAK,gBAAgB,GAAG,eAAe,GAAG,eAAe,CAAC;AAE1D,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,GAAE,MAAU,EAChB,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CA6J3D;AAED,wBAAsB,WAAW,IAAI,OAAO,CAC1C,WAAW,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC,CAAC,CACH,CA4BA;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAiB5E;AAED,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAiB9E;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAC/C,WAAW,CAAC;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAC/E,CASA;AAED,wBAAsB,sBAAsB,IAAI,OAAO,CACrD,WAAW,CAAC;IAAE,sBAAsB,EAAE,MAAM,CAAC;IAAC,mBAAmB,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,CAC5E,CASA;AAED,wBAAsB,qBAAqB,IAAI,OAAO,CACpD,WAAW,CAAC;IACV,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,GAAG,EAAE,CAAC;CACxB,CAAC,CACH,CASA;AAED,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,UAAU,GAAG,OAAO,CACrF,WAAW,CAAC;IACV,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC,CACH,CASA;AAED,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,MAAM,EACV,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,WAAW,CAAC;IAAE,aAAa,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC,CAgBlD;AAED,wBAAsB,uBAAuB,CAC3C,GAAG,EAAE,MAAM,EAAE,EACb,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,WAAW,CAAC;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAa3C;AAED,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAwCrF;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,MAAU,GAChB,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAkB7B;AAED,wBAAsB,wBAAwB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAoB7F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAsBrF;AAED,wBAAsB,wBAAwB,IAAI,OAAO,CACvD,WAAW,CAAC;IAAE,cAAc,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CACxD,CAeA;AAED,UAAU,iBAAiB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAWD,wBAAsB,6BAA6B,IAAI,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,CAE7F;AAED,wBAAsB,0BAA0B,CAC9C,SAAS,GAAE,MAAU,GACpB,OAAO,CAAC,WAAW,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC,CA+G9E"}
|
|
@@ -258,7 +258,7 @@ export async function handleAddMemory(data) {
|
|
|
258
258
|
metadata: JSON.stringify({ source: "api" }),
|
|
259
259
|
};
|
|
260
260
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
261
|
-
vectorSearch.insertVector(db, record);
|
|
261
|
+
vectorSearch.insertVector(db, record, shard);
|
|
262
262
|
shardManager.incrementVectorCount(shard.id);
|
|
263
263
|
return { success: true, data: { id } };
|
|
264
264
|
}
|
|
@@ -282,7 +282,7 @@ export async function handleDeleteMemory(id, cascade = false) {
|
|
|
282
282
|
if (linkedPromptId)
|
|
283
283
|
userPromptManager.deletePrompt(linkedPromptId);
|
|
284
284
|
}
|
|
285
|
-
vectorSearch.deleteVector(db, id);
|
|
285
|
+
await vectorSearch.deleteVector(db, id, shard);
|
|
286
286
|
shardManager.decrementVectorCount(shard.id);
|
|
287
287
|
return {
|
|
288
288
|
success: true,
|
|
@@ -333,7 +333,7 @@ export async function handleUpdateMemory(id, data) {
|
|
|
333
333
|
if (!foundShard || !existingMemory)
|
|
334
334
|
return { success: false, error: "Memory not found" };
|
|
335
335
|
const db = connectionManager.getConnection(foundShard.dbPath);
|
|
336
|
-
vectorSearch.deleteVector(db, id);
|
|
336
|
+
await vectorSearch.deleteVector(db, id, foundShard);
|
|
337
337
|
shardManager.decrementVectorCount(foundShard.id);
|
|
338
338
|
const newContent = data.content || existingMemory.content;
|
|
339
339
|
const tags = data.tags || (existingMemory.tags ? existingMemory.tags.split(",") : []);
|
|
@@ -360,7 +360,7 @@ export async function handleUpdateMemory(id, data) {
|
|
|
360
360
|
projectName: existingMemory.project_name,
|
|
361
361
|
gitRepoUrl: existingMemory.git_repo_url,
|
|
362
362
|
};
|
|
363
|
-
vectorSearch.insertVector(db, updatedRecord);
|
|
363
|
+
vectorSearch.insertVector(db, updatedRecord, foundShard);
|
|
364
364
|
shardManager.incrementVectorCount(foundShard.id);
|
|
365
365
|
return { success: true };
|
|
366
366
|
}
|
|
@@ -382,7 +382,7 @@ export async function handleSearch(query, tag, page = 1, pageSize = 20) {
|
|
|
382
382
|
const shards = shardManager.getAllShards(scope, hash);
|
|
383
383
|
for (const shard of shards) {
|
|
384
384
|
try {
|
|
385
|
-
const results = vectorSearch.searchInShard(shard, queryVector, tag, pageSize * 2);
|
|
385
|
+
const results = await vectorSearch.searchInShard(shard, queryVector, tag, pageSize * 2);
|
|
386
386
|
memoryResults.push(...results);
|
|
387
387
|
}
|
|
388
388
|
catch (error) {
|
|
@@ -408,7 +408,7 @@ export async function handleSearch(query, tag, page = 1, pageSize = 20) {
|
|
|
408
408
|
const shards = shardManager.getAllShards(scope, hash);
|
|
409
409
|
for (const shard of shards) {
|
|
410
410
|
try {
|
|
411
|
-
const results = vectorSearch.searchInShard(shard, queryVector, containerTag, pageSize);
|
|
411
|
+
const results = await vectorSearch.searchInShard(shard, queryVector, containerTag, pageSize);
|
|
412
412
|
memoryResults.push(...results);
|
|
413
413
|
}
|
|
414
414
|
catch (error) {
|
|
@@ -874,14 +874,10 @@ export async function handleRunTagMigrationBatch(batchSize = 5) {
|
|
|
874
874
|
const vector = await embeddingService.embedWithTimeout(m.content);
|
|
875
875
|
const vectorBuffer = new Uint8Array(vector.buffer);
|
|
876
876
|
db.prepare("UPDATE memories SET vector = ?, updated_at = ? WHERE id = ?").run(vectorBuffer, Date.now(), m.id);
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
const tagsVectorBuffer = new Uint8Array(tagsVector.buffer);
|
|
882
|
-
db.prepare("DELETE FROM vec_tags WHERE memory_id = ?").run(m.id);
|
|
883
|
-
db.prepare("INSERT INTO vec_tags (memory_id, embedding) VALUES (?, ?)").run(m.id, tagsVectorBuffer);
|
|
884
|
-
}
|
|
877
|
+
const index = vectorSearch
|
|
878
|
+
.getIndexManager()
|
|
879
|
+
.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
880
|
+
await index.insert(m.id, vector);
|
|
885
881
|
migrationProgress.processed++;
|
|
886
882
|
batchProcessed++;
|
|
887
883
|
}
|
package/dist/services/client.js
CHANGED
|
@@ -128,7 +128,7 @@ export class LocalMemoryClient {
|
|
|
128
128
|
metadata: Object.keys(dynamicMetadata).length > 0 ? JSON.stringify(dynamicMetadata) : undefined,
|
|
129
129
|
};
|
|
130
130
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
131
|
-
vectorSearch.insertVector(db, record);
|
|
131
|
+
vectorSearch.insertVector(db, record, shard);
|
|
132
132
|
shardManager.incrementVectorCount(shard.id);
|
|
133
133
|
return { success: true, id };
|
|
134
134
|
}
|
|
@@ -148,7 +148,7 @@ export class LocalMemoryClient {
|
|
|
148
148
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
149
149
|
const memory = vectorSearch.getMemoryById(db, memoryId);
|
|
150
150
|
if (memory) {
|
|
151
|
-
vectorSearch.deleteVector(db, memoryId);
|
|
151
|
+
await vectorSearch.deleteVector(db, memoryId, shard);
|
|
152
152
|
shardManager.decrementVectorCount(shard.id);
|
|
153
153
|
return { success: true };
|
|
154
154
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migration-service.d.ts","sourceRoot":"","sources":["../../src/services/migration-service.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,KAAK,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,WAAW,GAAG,cAAc,GAAG,SAAS,GAAG,UAAU,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,UAAU,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,gBAAgB,CAAC,CAAwC;IAE3D,uBAAuB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAoDrD,iBAAiB,CACrB,QAAQ,EAAE,aAAa,GAAG,UAAU,EACpC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,GACvD,OAAO,CAAC,eAAe,CAAC;YA2Cb,mBAAmB;YA8CnB,gBAAgB;
|
|
1
|
+
{"version":3,"file":"migration-service.d.ts","sourceRoot":"","sources":["../../src/services/migration-service.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,KAAK,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,WAAW,GAAG,cAAc,GAAG,SAAS,GAAG,UAAU,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,UAAU,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,gBAAgB,CAAC,CAAwC;IAE3D,uBAAuB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAoDrD,iBAAiB,CACrB,QAAQ,EAAE,aAAa,GAAG,UAAU,EACpC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,GACvD,OAAO,CAAC,eAAe,CAAC;YA2Cb,mBAAmB;YA8CnB,gBAAgB;IA+I9B,OAAO,CAAC,cAAc;IAMtB,SAAS;;;;;CAOV;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}
|
|
@@ -191,7 +191,7 @@ export class MigrationService {
|
|
|
191
191
|
projectPath: memory.projectPath || undefined,
|
|
192
192
|
projectName: memory.projectName || undefined,
|
|
193
193
|
gitRepoUrl: memory.gitRepoUrl || undefined,
|
|
194
|
-
});
|
|
194
|
+
}, newShard);
|
|
195
195
|
if (memory.isPinned === 1) {
|
|
196
196
|
vectorSearch.pinMemory(newDb, memory.id);
|
|
197
197
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/connection-manager.ts"],"names":[],"mappings":"AAMA,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAE/B,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,WAAW,CAAqD;IAExE,OAAO,CAAC,YAAY;
|
|
1
|
+
{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/connection-manager.ts"],"names":[],"mappings":"AAMA,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAE/B,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,WAAW,CAAqD;IAExE,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,aAAa;IAarB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,QAAQ,CAAC,SAAS;IAiBxD,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IASrC,QAAQ,IAAI,IAAI;IAYhB,aAAa,IAAI,IAAI;CAStB;AAED,eAAO,MAAM,iBAAiB,mBAA0B,CAAC"}
|
|
@@ -13,17 +13,6 @@ export class ConnectionManager {
|
|
|
13
13
|
db.run("PRAGMA cache_size = -64000");
|
|
14
14
|
db.run("PRAGMA temp_store = MEMORY");
|
|
15
15
|
db.run("PRAGMA foreign_keys = ON");
|
|
16
|
-
try {
|
|
17
|
-
const result = db.prepare("SELECT vec_version()").all();
|
|
18
|
-
if (!result || result.length === 0) {
|
|
19
|
-
throw new Error("vec_version() returned no result");
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
catch (error) {
|
|
23
|
-
throw new Error(`sqlite-vec extension not available: ${error}\n\n` +
|
|
24
|
-
`The bundled SQLite dylib should have sqlite-vec built-in.\n` +
|
|
25
|
-
`Try reinstalling opencode-mem or report this issue.`);
|
|
26
|
-
}
|
|
27
16
|
this.migrateSchema(db);
|
|
28
17
|
}
|
|
29
18
|
migrateSchema(db) {
|
|
@@ -33,12 +22,6 @@ export class ConnectionManager {
|
|
|
33
22
|
if (!hasTags && columns.length > 0) {
|
|
34
23
|
db.run("ALTER TABLE memories ADD COLUMN tags TEXT");
|
|
35
24
|
}
|
|
36
|
-
db.run(`
|
|
37
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS vec_tags USING vec0(
|
|
38
|
-
memory_id TEXT PRIMARY KEY,
|
|
39
|
-
embedding float32[${CONFIG.embeddingDimensions}] distance_metric=cosine
|
|
40
|
-
)
|
|
41
|
-
`);
|
|
42
25
|
}
|
|
43
26
|
catch (error) {
|
|
44
27
|
log("Schema migration error", { error: String(error) });
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export interface HNSWIndexData {
|
|
2
|
+
id: string;
|
|
3
|
+
vector: Float32Array;
|
|
4
|
+
}
|
|
5
|
+
export declare class HNSWIndex {
|
|
6
|
+
private index;
|
|
7
|
+
private idMap;
|
|
8
|
+
private reverseMap;
|
|
9
|
+
private nextId;
|
|
10
|
+
private dimensions;
|
|
11
|
+
private indexPath;
|
|
12
|
+
private maxElements;
|
|
13
|
+
private initialized;
|
|
14
|
+
constructor(dimensions: number, indexPath: string);
|
|
15
|
+
private ensureInitialized;
|
|
16
|
+
insert(id: string, vector: Float32Array): Promise<void>;
|
|
17
|
+
insertBatch(items: HNSWIndexData[]): Promise<void>;
|
|
18
|
+
search(queryVector: Float32Array, k: number): Promise<{
|
|
19
|
+
id: string;
|
|
20
|
+
distance: number;
|
|
21
|
+
}[]>;
|
|
22
|
+
delete(id: string): Promise<void>;
|
|
23
|
+
save(): Promise<void>;
|
|
24
|
+
getCount(): number;
|
|
25
|
+
}
|
|
26
|
+
export declare class HNSWIndexManager {
|
|
27
|
+
private indexes;
|
|
28
|
+
private baseDir;
|
|
29
|
+
constructor(baseDir: string);
|
|
30
|
+
getIndex(scope: string, scopeHash: string, shardIndex: number): HNSWIndex;
|
|
31
|
+
rebuildFromShard(db: any, scope: string, scopeHash: string, shardIndex: number): Promise<void>;
|
|
32
|
+
deleteIndex(scope: string, scopeHash: string, shardIndex: number): Promise<void>;
|
|
33
|
+
cleanupOrphanedIndexes(validKeys: Set<string>): Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=hnsw-index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-index.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/hnsw-index.ts"],"names":[],"mappings":"AAuBA,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,WAAW,CAAkB;gBAEzB,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;YAKnC,iBAAiB;IAyCzB,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvD,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,MAAM,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAkBzF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB3B,QAAQ,IAAI,MAAM;CAGnB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM;IAO3B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IAWnE,gBAAgB,CACpB,EAAE,EAAE,GAAG,EACP,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAmBV,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAehF,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CA0BpE"}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { mkdirSync, existsSync, writeFileSync, readFileSync, unlinkSync, readdirSync, statSync, } from "node:fs";
|
|
2
|
+
import { join, dirname, basename } from "node:path";
|
|
3
|
+
import { log } from "../logger.js";
|
|
4
|
+
import { CONFIG } from "../../config.js";
|
|
5
|
+
let HNSWLib = null;
|
|
6
|
+
async function loadHNSWLib() {
|
|
7
|
+
if (!HNSWLib) {
|
|
8
|
+
const module = await import("hnswlib-wasm");
|
|
9
|
+
HNSWLib = module;
|
|
10
|
+
}
|
|
11
|
+
return HNSWLib;
|
|
12
|
+
}
|
|
13
|
+
export class HNSWIndex {
|
|
14
|
+
index = null;
|
|
15
|
+
idMap = new Map();
|
|
16
|
+
reverseMap = new Map();
|
|
17
|
+
nextId = 0;
|
|
18
|
+
dimensions;
|
|
19
|
+
indexPath;
|
|
20
|
+
maxElements = 50000;
|
|
21
|
+
initialized = false;
|
|
22
|
+
constructor(dimensions, indexPath) {
|
|
23
|
+
this.dimensions = dimensions;
|
|
24
|
+
this.indexPath = indexPath;
|
|
25
|
+
}
|
|
26
|
+
async ensureInitialized() {
|
|
27
|
+
if (this.initialized)
|
|
28
|
+
return;
|
|
29
|
+
const hnsw = await loadHNSWLib();
|
|
30
|
+
const dir = dirname(this.indexPath);
|
|
31
|
+
if (!existsSync(dir)) {
|
|
32
|
+
mkdirSync(dir, { recursive: true });
|
|
33
|
+
}
|
|
34
|
+
if (existsSync(this.indexPath)) {
|
|
35
|
+
try {
|
|
36
|
+
this.index = new hnsw.HierarchicalNSW("cosine", this.dimensions);
|
|
37
|
+
this.index.readIndex(this.indexPath);
|
|
38
|
+
const metaPath = this.indexPath + ".meta";
|
|
39
|
+
if (existsSync(metaPath)) {
|
|
40
|
+
const meta = JSON.parse(readFileSync(metaPath, "utf-8"));
|
|
41
|
+
this.nextId = meta.nextId || 0;
|
|
42
|
+
this.idMap = new Map(Object.entries(meta.idMap || {}).map(([k, v]) => [Number(k), v]));
|
|
43
|
+
this.reverseMap = new Map(Object.entries(meta.reverseMap || {}));
|
|
44
|
+
}
|
|
45
|
+
log("HNSW index loaded", { path: this.indexPath, count: this.nextId });
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
log("Failed to load HNSW index, creating new", {
|
|
49
|
+
path: this.indexPath,
|
|
50
|
+
error: String(error),
|
|
51
|
+
});
|
|
52
|
+
this.index = new hnsw.HierarchicalNSW("cosine", this.dimensions, this.maxElements);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
this.index = new hnsw.HierarchicalNSW("cosine", this.dimensions, this.maxElements);
|
|
57
|
+
log("HNSW index created", { path: this.indexPath, dimensions: this.dimensions });
|
|
58
|
+
}
|
|
59
|
+
this.initialized = true;
|
|
60
|
+
}
|
|
61
|
+
async insert(id, vector) {
|
|
62
|
+
await this.ensureInitialized();
|
|
63
|
+
if (this.reverseMap.has(id)) {
|
|
64
|
+
const internalId = this.reverseMap.get(id);
|
|
65
|
+
this.index.markDelete(internalId);
|
|
66
|
+
}
|
|
67
|
+
const internalId = this.nextId++;
|
|
68
|
+
this.index.addPoint(vector, internalId);
|
|
69
|
+
this.idMap.set(internalId, id);
|
|
70
|
+
this.reverseMap.set(id, internalId);
|
|
71
|
+
await this.save();
|
|
72
|
+
}
|
|
73
|
+
async insertBatch(items) {
|
|
74
|
+
await this.ensureInitialized();
|
|
75
|
+
for (const item of items) {
|
|
76
|
+
if (this.reverseMap.has(item.id)) {
|
|
77
|
+
const internalId = this.reverseMap.get(item.id);
|
|
78
|
+
this.index.markDelete(internalId);
|
|
79
|
+
}
|
|
80
|
+
const internalId = this.nextId++;
|
|
81
|
+
this.index.addPoint(item.vector, internalId);
|
|
82
|
+
this.idMap.set(internalId, item.id);
|
|
83
|
+
this.reverseMap.set(item.id, internalId);
|
|
84
|
+
}
|
|
85
|
+
await this.save();
|
|
86
|
+
}
|
|
87
|
+
async search(queryVector, k) {
|
|
88
|
+
await this.ensureInitialized();
|
|
89
|
+
try {
|
|
90
|
+
const results = this.index.searchKnn(queryVector, k);
|
|
91
|
+
return results.neighbors
|
|
92
|
+
.map((internalId, idx) => ({
|
|
93
|
+
id: this.idMap.get(internalId) || "",
|
|
94
|
+
distance: results.distances[idx],
|
|
95
|
+
}))
|
|
96
|
+
.filter((r) => r.id);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
log("HNSW search error", { error: String(error) });
|
|
100
|
+
return [];
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
async delete(id) {
|
|
104
|
+
await this.ensureInitialized();
|
|
105
|
+
if (this.reverseMap.has(id)) {
|
|
106
|
+
const internalId = this.reverseMap.get(id);
|
|
107
|
+
this.index.markDelete(internalId);
|
|
108
|
+
this.idMap.delete(internalId);
|
|
109
|
+
this.reverseMap.delete(id);
|
|
110
|
+
await this.save();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async save() {
|
|
114
|
+
if (!this.index)
|
|
115
|
+
return;
|
|
116
|
+
const dir = dirname(this.indexPath);
|
|
117
|
+
if (!existsSync(dir)) {
|
|
118
|
+
mkdirSync(dir, { recursive: true });
|
|
119
|
+
}
|
|
120
|
+
this.index.writeIndex(this.indexPath);
|
|
121
|
+
const metaPath = this.indexPath + ".meta";
|
|
122
|
+
const meta = {
|
|
123
|
+
nextId: this.nextId,
|
|
124
|
+
idMap: Object.fromEntries(this.idMap),
|
|
125
|
+
reverseMap: Object.fromEntries(this.reverseMap),
|
|
126
|
+
};
|
|
127
|
+
writeFileSync(metaPath, JSON.stringify(meta));
|
|
128
|
+
}
|
|
129
|
+
getCount() {
|
|
130
|
+
return this.reverseMap.size;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
export class HNSWIndexManager {
|
|
134
|
+
indexes = new Map();
|
|
135
|
+
baseDir;
|
|
136
|
+
constructor(baseDir) {
|
|
137
|
+
this.baseDir = baseDir;
|
|
138
|
+
if (!existsSync(baseDir)) {
|
|
139
|
+
mkdirSync(baseDir, { recursive: true });
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
getIndex(scope, scopeHash, shardIndex) {
|
|
143
|
+
const key = `${scope}_${scopeHash}_${shardIndex}`;
|
|
144
|
+
if (!this.indexes.has(key)) {
|
|
145
|
+
const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
|
|
146
|
+
this.indexes.set(key, new HNSWIndex(CONFIG.embeddingDimensions, indexPath));
|
|
147
|
+
}
|
|
148
|
+
return this.indexes.get(key);
|
|
149
|
+
}
|
|
150
|
+
async rebuildFromShard(db, scope, scopeHash, shardIndex) {
|
|
151
|
+
const index = this.getIndex(scope, scopeHash, shardIndex);
|
|
152
|
+
const rows = db.prepare("SELECT id, vector FROM memories").all();
|
|
153
|
+
const items = [];
|
|
154
|
+
for (const row of rows) {
|
|
155
|
+
if (row.vector) {
|
|
156
|
+
const vector = new Float32Array(row.vector.buffer);
|
|
157
|
+
items.push({ id: row.id, vector });
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (items.length > 0) {
|
|
161
|
+
await index.insertBatch(items);
|
|
162
|
+
log("HNSW index rebuilt", { scope, scopeHash, shardIndex, count: items.length });
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async deleteIndex(scope, scopeHash, shardIndex) {
|
|
166
|
+
const key = `${scope}_${scopeHash}_${shardIndex}`;
|
|
167
|
+
this.indexes.delete(key);
|
|
168
|
+
const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
|
|
169
|
+
const metaPath = indexPath + ".meta";
|
|
170
|
+
try {
|
|
171
|
+
if (existsSync(indexPath))
|
|
172
|
+
unlinkSync(indexPath);
|
|
173
|
+
if (existsSync(metaPath))
|
|
174
|
+
unlinkSync(metaPath);
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
log("Error deleting HNSW index files", { path: indexPath, error: String(error) });
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
async cleanupOrphanedIndexes(validKeys) {
|
|
181
|
+
const scopeDirs = ["users", "projects"];
|
|
182
|
+
for (const scopeDir of scopeDirs) {
|
|
183
|
+
const dir = join(this.baseDir, scopeDir);
|
|
184
|
+
if (!existsSync(dir))
|
|
185
|
+
continue;
|
|
186
|
+
const files = readdirSync(dir);
|
|
187
|
+
for (const file of files) {
|
|
188
|
+
if (file.endsWith(".hnsw")) {
|
|
189
|
+
const key = basename(file, ".hnsw");
|
|
190
|
+
if (!validKeys.has(key)) {
|
|
191
|
+
const indexPath = join(dir, file);
|
|
192
|
+
const metaPath = indexPath + ".meta";
|
|
193
|
+
try {
|
|
194
|
+
unlinkSync(indexPath);
|
|
195
|
+
if (existsSync(metaPath))
|
|
196
|
+
unlinkSync(metaPath);
|
|
197
|
+
log("Removed orphaned HNSW index", { path: indexPath });
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
log("Error removing orphaned index", { path: indexPath, error: String(error) });
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
@@ -10,16 +10,7 @@ export declare class ShardManager {
|
|
|
10
10
|
getAllShards(scope: "user" | "project", scopeHash: string): ShardInfo[];
|
|
11
11
|
createShard(scope: "user" | "project", scopeHash: string, shardIndex: number): ShardInfo;
|
|
12
12
|
private initShardDb;
|
|
13
|
-
/**
|
|
14
|
-
* Check if the shard DB file exists and contains the required 'memories' table.
|
|
15
|
-
* Returns false if the file is missing or the table doesn't exist.
|
|
16
|
-
*/
|
|
17
13
|
private isShardValid;
|
|
18
|
-
/**
|
|
19
|
-
* Ensure the shard DB has all required tables. If tables are missing,
|
|
20
|
-
* re-initialize them. This handles cases where the DB file exists but
|
|
21
|
-
* was corrupted or partially created.
|
|
22
|
-
*/
|
|
23
14
|
private ensureShardTables;
|
|
24
15
|
getWriteShard(scope: "user" | "project", scopeHash: string): ShardInfo;
|
|
25
16
|
private markShardReadOnly;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shard-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/shard-manager.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAO5C,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAe;IACjC,OAAO,CAAC,YAAY,CAAS;;IAQ7B,OAAO,CAAC,cAAc;IAqBtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,iBAAiB;IAKzB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsB9E,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE;IAgCvE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IA2BxF,OAAO,CAAC,WAAW;
|
|
1
|
+
{"version":3,"file":"shard-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/shard-manager.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAO5C,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAe;IACjC,OAAO,CAAC,YAAY,CAAS;;IAQ7B,OAAO,CAAC,cAAc;IAqBtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,iBAAiB;IAKzB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsB9E,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE;IAgCvE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IA2BxF,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,YAAY;IA4BpB,OAAO,CAAC,iBAAiB;IAYzB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS;IA+BtE,OAAO,CAAC,iBAAiB;IAOzB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAkBhD,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAwBnC;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { getDatabase } from "./sqlite-bootstrap.js";
|
|
2
|
-
import { join, basename
|
|
2
|
+
import { join, basename } from "node:path";
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import { CONFIG } from "../../config.js";
|
|
5
5
|
import { connectionManager } from "./connection-manager.js";
|
|
@@ -133,6 +133,7 @@ export class ShardManager {
|
|
|
133
133
|
id TEXT PRIMARY KEY,
|
|
134
134
|
content TEXT NOT NULL,
|
|
135
135
|
vector BLOB NOT NULL,
|
|
136
|
+
tags_vector BLOB,
|
|
136
137
|
container_tag TEXT NOT NULL,
|
|
137
138
|
tags TEXT,
|
|
138
139
|
type TEXT,
|
|
@@ -147,28 +148,12 @@ export class ShardManager {
|
|
|
147
148
|
git_repo_url TEXT,
|
|
148
149
|
is_pinned INTEGER DEFAULT 0
|
|
149
150
|
)
|
|
150
|
-
`);
|
|
151
|
-
db.run(`
|
|
152
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(
|
|
153
|
-
memory_id TEXT PRIMARY KEY,
|
|
154
|
-
embedding float32[${CONFIG.embeddingDimensions}] distance_metric=cosine
|
|
155
|
-
)
|
|
156
|
-
`);
|
|
157
|
-
db.run(`
|
|
158
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS vec_tags USING vec0(
|
|
159
|
-
memory_id TEXT PRIMARY KEY,
|
|
160
|
-
embedding float32[${CONFIG.embeddingDimensions}] distance_metric=cosine
|
|
161
|
-
)
|
|
162
151
|
`);
|
|
163
152
|
db.run(`CREATE INDEX IF NOT EXISTS idx_container_tag ON memories(container_tag)`);
|
|
164
153
|
db.run(`CREATE INDEX IF NOT EXISTS idx_type ON memories(type)`);
|
|
165
154
|
db.run(`CREATE INDEX IF NOT EXISTS idx_created_at ON memories(created_at DESC)`);
|
|
166
155
|
db.run(`CREATE INDEX IF NOT EXISTS idx_is_pinned ON memories(is_pinned)`);
|
|
167
156
|
}
|
|
168
|
-
/**
|
|
169
|
-
* Check if the shard DB file exists and contains the required 'memories' table.
|
|
170
|
-
* Returns false if the file is missing or the table doesn't exist.
|
|
171
|
-
*/
|
|
172
157
|
isShardValid(shard) {
|
|
173
158
|
if (!existsSync(shard.dbPath)) {
|
|
174
159
|
log("Shard DB file missing", { dbPath: shard.dbPath, shardId: shard.id });
|
|
@@ -196,11 +181,6 @@ export class ShardManager {
|
|
|
196
181
|
return false;
|
|
197
182
|
}
|
|
198
183
|
}
|
|
199
|
-
/**
|
|
200
|
-
* Ensure the shard DB has all required tables. If tables are missing,
|
|
201
|
-
* re-initialize them. This handles cases where the DB file exists but
|
|
202
|
-
* was corrupted or partially created.
|
|
203
|
-
*/
|
|
204
184
|
ensureShardTables(shard) {
|
|
205
185
|
try {
|
|
206
186
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
@@ -218,7 +198,6 @@ export class ShardManager {
|
|
|
218
198
|
if (!shard) {
|
|
219
199
|
return this.createShard(scope, scopeHash, 0);
|
|
220
200
|
}
|
|
221
|
-
// Validate that the shard DB file exists and has required tables
|
|
222
201
|
if (!this.isShardValid(shard)) {
|
|
223
202
|
log("Active shard is invalid, recreating", {
|
|
224
203
|
scope,
|
|
@@ -226,12 +205,9 @@ export class ShardManager {
|
|
|
226
205
|
shardIndex: shard.shardIndex,
|
|
227
206
|
dbPath: shard.dbPath,
|
|
228
207
|
});
|
|
229
|
-
// Close any cached connection to the invalid shard
|
|
230
208
|
connectionManager.closeConnection(shard.dbPath);
|
|
231
|
-
// Remove the stale metadata record
|
|
232
209
|
const deleteStmt = this.metadataDb.prepare(`DELETE FROM shards WHERE id = ?`);
|
|
233
210
|
deleteStmt.run(shard.id);
|
|
234
|
-
// Create a fresh shard with the same index
|
|
235
211
|
return this.createShard(scope, scopeHash, shard.shardIndex);
|
|
236
212
|
}
|
|
237
213
|
if (shard.vectorCount >= CONFIG.maxVectorsPerShard) {
|
|
@@ -1,20 +1,2 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SQLite Bootstrap Module
|
|
3
|
-
*
|
|
4
|
-
* This module MUST be imported before any other module that uses bun:sqlite.
|
|
5
|
-
* It ensures that setCustomSQLite() is called BEFORE the Database class is
|
|
6
|
-
* instantiated, which is required for custom SQLite paths to work on macOS.
|
|
7
|
-
*
|
|
8
|
-
* Issue: https://github.com/tickernelz/opencode-mem/issues/34
|
|
9
|
-
* Issue: https://github.com/tickernelz/opencode-mem/issues/37
|
|
10
|
-
*
|
|
11
|
-
* The bundled dylib has sqlite-vec statically linked, so no extension loading needed.
|
|
12
|
-
* Loading priority:
|
|
13
|
-
* 1. Bundled dylib (native/darwin-{arch}/libsqlite3.dylib) - includes sqlite-vec
|
|
14
|
-
* 2. Homebrew SQLite (auto-detected common paths)
|
|
15
|
-
*/
|
|
16
|
-
export declare function configureSqlite(): void;
|
|
17
1
|
export declare function getDatabase(): typeof import("bun:sqlite").Database;
|
|
18
|
-
export declare function getSqliteSource(): string | null;
|
|
19
|
-
export declare function verifySqliteVec(): void;
|
|
20
2
|
//# sourceMappingURL=sqlite-bootstrap.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite-bootstrap.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/sqlite-bootstrap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sqlite-bootstrap.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/sqlite-bootstrap.ts"],"names":[],"mappings":"AAEA,wBAAgB,WAAW,IAAI,cAAc,YAAY,EAAE,QAAQ,CAMlE"}
|
|
@@ -1,123 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SQLite Bootstrap Module
|
|
3
|
-
*
|
|
4
|
-
* This module MUST be imported before any other module that uses bun:sqlite.
|
|
5
|
-
* It ensures that setCustomSQLite() is called BEFORE the Database class is
|
|
6
|
-
* instantiated, which is required for custom SQLite paths to work on macOS.
|
|
7
|
-
*
|
|
8
|
-
* Issue: https://github.com/tickernelz/opencode-mem/issues/34
|
|
9
|
-
* Issue: https://github.com/tickernelz/opencode-mem/issues/37
|
|
10
|
-
*
|
|
11
|
-
* The bundled dylib has sqlite-vec statically linked, so no extension loading needed.
|
|
12
|
-
* Loading priority:
|
|
13
|
-
* 1. Bundled dylib (native/darwin-{arch}/libsqlite3.dylib) - includes sqlite-vec
|
|
14
|
-
* 2. Homebrew SQLite (auto-detected common paths)
|
|
15
|
-
*/
|
|
16
|
-
import { existsSync } from "node:fs";
|
|
17
|
-
import { join, dirname } from "node:path";
|
|
18
|
-
import { fileURLToPath } from "node:url";
|
|
19
1
|
let Database;
|
|
20
|
-
let sqliteConfigured = false;
|
|
21
|
-
let sqliteSource = null;
|
|
22
|
-
function getBundledSqlitePath() {
|
|
23
|
-
if (process.platform !== "darwin")
|
|
24
|
-
return null;
|
|
25
|
-
const arch = process.arch;
|
|
26
|
-
if (arch !== "x64" && arch !== "arm64")
|
|
27
|
-
return null;
|
|
28
|
-
try {
|
|
29
|
-
const currentDir = dirname(fileURLToPath(import.meta.url));
|
|
30
|
-
const bundledPath = join(currentDir, "..", "..", "..", "native", `darwin-${arch}`, "libsqlite3.dylib");
|
|
31
|
-
if (existsSync(bundledPath)) {
|
|
32
|
-
return bundledPath;
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
catch { }
|
|
36
|
-
return null;
|
|
37
|
-
}
|
|
38
|
-
function getHomebrewSqlitePath() {
|
|
39
|
-
const arch = process.arch;
|
|
40
|
-
const paths = arch === "arm64"
|
|
41
|
-
? ["/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib"]
|
|
42
|
-
: [
|
|
43
|
-
"/usr/local/opt/sqlite/lib/libsqlite3.dylib",
|
|
44
|
-
"/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib",
|
|
45
|
-
];
|
|
46
|
-
for (const path of paths) {
|
|
47
|
-
if (existsSync(path)) {
|
|
48
|
-
return path;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
return null;
|
|
52
|
-
}
|
|
53
|
-
function checkSqliteVec(db) {
|
|
54
|
-
try {
|
|
55
|
-
const result = db.prepare("SELECT vec_version()").all();
|
|
56
|
-
return result && result.length > 0;
|
|
57
|
-
}
|
|
58
|
-
catch {
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
export function configureSqlite() {
|
|
63
|
-
if (sqliteConfigured)
|
|
64
|
-
return;
|
|
65
|
-
const bunSqlite = require("bun:sqlite");
|
|
66
|
-
Database = bunSqlite.Database;
|
|
67
|
-
if (process.platform !== "darwin") {
|
|
68
|
-
sqliteConfigured = true;
|
|
69
|
-
return;
|
|
70
|
-
}
|
|
71
|
-
const trySetCustomSQLite = (path, source) => {
|
|
72
|
-
try {
|
|
73
|
-
Database.setCustomSQLite(path);
|
|
74
|
-
sqliteSource = source;
|
|
75
|
-
return true;
|
|
76
|
-
}
|
|
77
|
-
catch (error) {
|
|
78
|
-
const errorStr = String(error);
|
|
79
|
-
if (errorStr.includes("SQLite already loaded")) {
|
|
80
|
-
sqliteSource = "already-loaded";
|
|
81
|
-
return true;
|
|
82
|
-
}
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
};
|
|
86
|
-
const bundledPath = getBundledSqlitePath();
|
|
87
|
-
if (bundledPath) {
|
|
88
|
-
if (trySetCustomSQLite(bundledPath, "bundled")) {
|
|
89
|
-
sqliteConfigured = true;
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
const homebrewPath = getHomebrewSqlitePath();
|
|
94
|
-
if (homebrewPath) {
|
|
95
|
-
if (trySetCustomSQLite(homebrewPath, "homebrew")) {
|
|
96
|
-
sqliteConfigured = true;
|
|
97
|
-
return;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
throw new Error(`macOS detected but no compatible SQLite library found.\n\n` +
|
|
101
|
-
`The bundled SQLite dylib with sqlite-vec is required.\n` +
|
|
102
|
-
`Try reinstalling opencode-mem.`);
|
|
103
|
-
}
|
|
104
2
|
export function getDatabase() {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
export function getSqliteSource() {
|
|
109
|
-
return sqliteSource;
|
|
110
|
-
}
|
|
111
|
-
export function verifySqliteVec() {
|
|
112
|
-
const db = new Database(":memory:");
|
|
113
|
-
if (!checkSqliteVec(db)) {
|
|
114
|
-
db.close();
|
|
115
|
-
throw new Error(`sqlite-vec extension not available.\n\n` +
|
|
116
|
-
`This plugin requires SQLite with sqlite-vec built-in.\n` +
|
|
117
|
-
`The bundled dylib should have sqlite-vec statically linked.\n` +
|
|
118
|
-
`Try reinstalling opencode-mem.`);
|
|
3
|
+
if (!Database) {
|
|
4
|
+
const bunSqlite = require("bun:sqlite");
|
|
5
|
+
Database = bunSqlite.Database;
|
|
119
6
|
}
|
|
120
|
-
|
|
7
|
+
return Database;
|
|
121
8
|
}
|
|
122
|
-
configureSqlite();
|
|
123
|
-
verifySqliteVec();
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import { HNSWIndexManager } from "./hnsw-index.js";
|
|
1
2
|
import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js";
|
|
2
3
|
declare const Database: typeof import("bun:sqlite").Database;
|
|
3
4
|
type DatabaseType = typeof Database.prototype;
|
|
4
5
|
export declare class VectorSearch {
|
|
5
|
-
insertVector(db: DatabaseType, record: MemoryRecord): void;
|
|
6
|
-
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string): SearchResult[]
|
|
6
|
+
insertVector(db: DatabaseType, record: MemoryRecord, shard?: ShardInfo): void;
|
|
7
|
+
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string): Promise<SearchResult[]>;
|
|
7
8
|
searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string): Promise<SearchResult[]>;
|
|
8
|
-
deleteVector(db: DatabaseType, memoryId: string): void
|
|
9
|
-
updateVector(db: DatabaseType, memoryId: string, vector: Float32Array, tagsVector?: Float32Array): void
|
|
9
|
+
deleteVector(db: DatabaseType, memoryId: string, shard?: ShardInfo): Promise<void>;
|
|
10
|
+
updateVector(db: DatabaseType, memoryId: string, vector: Float32Array, shard?: ShardInfo, tagsVector?: Float32Array): Promise<void>;
|
|
10
11
|
listMemories(db: DatabaseType, containerTag: string, limit: number): any[];
|
|
11
12
|
getAllMemories(db: DatabaseType): any[];
|
|
12
13
|
getMemoryById(db: DatabaseType, memoryId: string): any | null;
|
|
@@ -16,6 +17,8 @@ export declare class VectorSearch {
|
|
|
16
17
|
getDistinctTags(db: DatabaseType): any[];
|
|
17
18
|
pinMemory(db: DatabaseType, memoryId: string): void;
|
|
18
19
|
unpinMemory(db: DatabaseType, memoryId: string): void;
|
|
20
|
+
rebuildHNSWIndex(db: DatabaseType, scope: string, scopeHash: string, shardIndex: number): Promise<void>;
|
|
21
|
+
getIndexManager(): HNSWIndexManager;
|
|
19
22
|
}
|
|
20
23
|
export declare const vectorSearch: VectorSearch;
|
|
21
24
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/vector-search.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExE,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAC/B,KAAK,YAAY,GAAG,OAAO,QAAQ,CAAC,SAAS,CAAC;AAI9C,qBAAa,YAAY;IACvB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,IAAI;IAoCvE,aAAa,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,YAAY,EACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAkEpB,kBAAkB,CACtB,MAAM,EAAE,SAAS,EAAE,EACnB,WAAW,EAAE,YAAY,EACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,MAAM,EAC3B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAiBpB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IASlF,YAAY,CAChB,EAAE,EAAE,YAAY,EAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,KAAK,CAAC,EAAE,SAAS,EACjB,UAAU,CAAC,EAAE,YAAY,GACxB,OAAO,CAAC,IAAI,CAAC;IAUhB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE;IAW1E,cAAc,CAAC,EAAE,EAAE,YAAY,GAAG,GAAG,EAAE;IAKvC,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAK7D,sBAAsB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE;IAgBlE,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAM5D,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG,MAAM;IAMzC,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG,GAAG,EAAE;IAexC,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAKnD,WAAW,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK/C,gBAAgB,CACpB,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAIhB,eAAe,IAAI,gBAAgB;CAGpC;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { getDatabase } from "./sqlite-bootstrap.js";
|
|
2
2
|
import { connectionManager } from "./connection-manager.js";
|
|
3
|
+
import { HNSWIndexManager } from "./hnsw-index.js";
|
|
3
4
|
import { log } from "../logger.js";
|
|
5
|
+
import { CONFIG } from "../../config.js";
|
|
4
6
|
const Database = getDatabase();
|
|
7
|
+
const hnswIndexManager = new HNSWIndexManager(CONFIG.storagePath);
|
|
5
8
|
export class VectorSearch {
|
|
6
|
-
insertVector(db, record) {
|
|
9
|
+
insertVector(db, record, shard) {
|
|
7
10
|
const insertMemory = db.prepare(`
|
|
8
11
|
INSERT INTO memories (
|
|
9
12
|
id, content, vector, container_tag, tags, type, created_at, updated_at,
|
|
@@ -12,43 +15,20 @@ export class VectorSearch {
|
|
|
12
15
|
`);
|
|
13
16
|
const vectorBuffer = new Uint8Array(record.vector.buffer);
|
|
14
17
|
insertMemory.run(record.id, record.content, vectorBuffer, record.containerTag, record.tags || null, record.type || null, record.createdAt, record.updatedAt, record.metadata || null, record.displayName || null, record.userName || null, record.userEmail || null, record.projectPath || null, record.projectName || null, record.gitRepoUrl || null);
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const tagsVectorBuffer = new Uint8Array(record.tagsVector.buffer);
|
|
21
|
-
const insertTagsVec = db.prepare(`
|
|
22
|
-
INSERT INTO vec_tags (memory_id, embedding) VALUES (?, ?)
|
|
23
|
-
`);
|
|
24
|
-
insertTagsVec.run(record.id, tagsVectorBuffer);
|
|
18
|
+
if (shard && record.vector) {
|
|
19
|
+
const index = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
20
|
+
index.insert(record.id, record.vector).catch((err) => {
|
|
21
|
+
log("HNSW insert error", { memoryId: record.id, error: String(err) });
|
|
22
|
+
});
|
|
25
23
|
}
|
|
26
24
|
}
|
|
27
|
-
searchInShard(shard, queryVector, containerTag, limit, queryText) {
|
|
25
|
+
async searchInShard(shard, queryVector, containerTag, limit, queryText) {
|
|
28
26
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
29
|
-
const
|
|
30
|
-
const contentResults =
|
|
31
|
-
.prepare(`
|
|
32
|
-
SELECT memory_id, distance FROM vec_memories
|
|
33
|
-
WHERE embedding MATCH ? AND k = ?
|
|
34
|
-
ORDER BY distance
|
|
35
|
-
`)
|
|
36
|
-
.all(queryBuffer, limit * 4);
|
|
37
|
-
const tagsResults = db
|
|
38
|
-
.prepare(`
|
|
39
|
-
SELECT memory_id, distance FROM vec_tags
|
|
40
|
-
WHERE embedding MATCH ? AND k = ?
|
|
41
|
-
ORDER BY distance
|
|
42
|
-
`)
|
|
43
|
-
.all(queryBuffer, limit * 4);
|
|
27
|
+
const index = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
28
|
+
const contentResults = await index.search(queryVector, limit * 4);
|
|
44
29
|
const scoreMap = new Map();
|
|
45
30
|
for (const r of contentResults) {
|
|
46
|
-
scoreMap.set(r.
|
|
47
|
-
}
|
|
48
|
-
for (const r of tagsResults) {
|
|
49
|
-
const entry = scoreMap.get(r.memory_id) || { contentSim: 0, tagsSim: 0 };
|
|
50
|
-
entry.tagsSim = 1 - r.distance;
|
|
51
|
-
scoreMap.set(r.memory_id, entry);
|
|
31
|
+
scoreMap.set(r.id, { contentSim: 1 - r.distance, tagsSim: 0 });
|
|
52
32
|
}
|
|
53
33
|
const ids = Array.from(scoreMap.keys());
|
|
54
34
|
if (ids.length === 0)
|
|
@@ -97,7 +77,7 @@ export class VectorSearch {
|
|
|
97
77
|
async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText) {
|
|
98
78
|
const shardPromises = shards.map(async (shard) => {
|
|
99
79
|
try {
|
|
100
|
-
return this.searchInShard(shard, queryVector, containerTag, limit, queryText);
|
|
80
|
+
return await this.searchInShard(shard, queryVector, containerTag, limit, queryText);
|
|
101
81
|
}
|
|
102
82
|
catch (error) {
|
|
103
83
|
log("Shard search error", { shardId: shard.id, error: String(error) });
|
|
@@ -109,19 +89,19 @@ export class VectorSearch {
|
|
|
109
89
|
allResults.sort((a, b) => b.similarity - a.similarity);
|
|
110
90
|
return allResults.filter((r) => r.similarity >= similarityThreshold).slice(0, limit);
|
|
111
91
|
}
|
|
112
|
-
deleteVector(db, memoryId) {
|
|
113
|
-
db.prepare(`DELETE FROM vec_memories WHERE memory_id = ?`).run(memoryId);
|
|
114
|
-
db.prepare(`DELETE FROM vec_tags WHERE memory_id = ?`).run(memoryId);
|
|
92
|
+
async deleteVector(db, memoryId, shard) {
|
|
115
93
|
db.prepare(`DELETE FROM memories WHERE id = ?`).run(memoryId);
|
|
94
|
+
if (shard) {
|
|
95
|
+
const index = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
96
|
+
await index.delete(memoryId);
|
|
97
|
+
}
|
|
116
98
|
}
|
|
117
|
-
updateVector(db, memoryId, vector, tagsVector) {
|
|
99
|
+
async updateVector(db, memoryId, vector, shard, tagsVector) {
|
|
118
100
|
const vectorBuffer = new Uint8Array(vector.buffer);
|
|
119
|
-
db.prepare(`
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
db.prepare(`DELETE FROM vec_tags WHERE memory_id = ?`).run(memoryId);
|
|
124
|
-
db.prepare(`INSERT INTO vec_tags (memory_id, embedding) VALUES (?, ?)`).run(memoryId, tagsVectorBuffer);
|
|
101
|
+
db.prepare(`UPDATE memories SET vector = ? WHERE id = ?`).run(vectorBuffer, memoryId);
|
|
102
|
+
if (shard && vector) {
|
|
103
|
+
const index = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
104
|
+
await index.insert(memoryId, vector);
|
|
125
105
|
}
|
|
126
106
|
}
|
|
127
107
|
listMemories(db, containerTag, limit) {
|
|
@@ -186,5 +166,11 @@ export class VectorSearch {
|
|
|
186
166
|
const stmt = db.prepare(`UPDATE memories SET is_pinned = 0 WHERE id = ?`);
|
|
187
167
|
stmt.run(memoryId);
|
|
188
168
|
}
|
|
169
|
+
async rebuildHNSWIndex(db, scope, scopeHash, shardIndex) {
|
|
170
|
+
await hnswIndexManager.rebuildFromShard(db, scope, scopeHash, shardIndex);
|
|
171
|
+
}
|
|
172
|
+
getIndexManager() {
|
|
173
|
+
return hnswIndexManager;
|
|
174
|
+
}
|
|
189
175
|
}
|
|
190
176
|
export const vectorSearch = new VectorSearch();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-mem",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.11.0",
|
|
4
4
|
"description": "OpenCode plugin that gives coding agents persistent memory using local vector database",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/plugin.js",
|
|
@@ -36,6 +36,7 @@
|
|
|
36
36
|
"@opencode-ai/plugin": "^1.0.162",
|
|
37
37
|
"@xenova/transformers": "^2.17.2",
|
|
38
38
|
"franc-min": "^6.2.0",
|
|
39
|
+
"hnswlib-wasm": "^0.8.2",
|
|
39
40
|
"iso-639-3": "^3.0.1"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
@@ -54,7 +55,6 @@
|
|
|
54
55
|
},
|
|
55
56
|
"files": [
|
|
56
57
|
"dist",
|
|
57
|
-
"native",
|
|
58
58
|
"package.json"
|
|
59
59
|
],
|
|
60
60
|
"lint-staged": {
|
|
File without changes
|
|
Binary file
|
|
File without changes
|
|
Binary file
|