opencode-mem 2.11.8 → 2.11.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/README.md +8 -12
  2. package/dist/config.d.ts +1 -0
  3. package/dist/config.d.ts.map +1 -1
  4. package/dist/config.js +2 -0
  5. package/dist/services/api-handlers.js +6 -6
  6. package/dist/services/cleanup-service.js +1 -1
  7. package/dist/services/client.js +1 -1
  8. package/dist/services/deduplication-service.js +1 -1
  9. package/dist/services/migration-service.js +3 -3
  10. package/dist/services/sqlite/shard-manager.d.ts +1 -1
  11. package/dist/services/sqlite/shard-manager.d.ts.map +1 -1
  12. package/dist/services/sqlite/shard-manager.js +12 -1
  13. package/dist/services/sqlite/vector-search.d.ts +8 -4
  14. package/dist/services/sqlite/vector-search.d.ts.map +1 -1
  15. package/dist/services/sqlite/vector-search.js +107 -44
  16. package/dist/services/vector-backends/backend-factory.d.ts +3 -0
  17. package/dist/services/vector-backends/backend-factory.d.ts.map +1 -0
  18. package/dist/services/vector-backends/backend-factory.js +104 -0
  19. package/dist/services/vector-backends/exact-scan-backend.d.ts +39 -0
  20. package/dist/services/vector-backends/exact-scan-backend.d.ts.map +1 -0
  21. package/dist/services/vector-backends/exact-scan-backend.js +63 -0
  22. package/dist/services/vector-backends/types.d.ts +51 -0
  23. package/dist/services/vector-backends/types.d.ts.map +1 -0
  24. package/dist/services/vector-backends/types.js +1 -0
  25. package/dist/services/vector-backends/usearch-backend.d.ts +47 -0
  26. package/dist/services/vector-backends/usearch-backend.d.ts.map +1 -0
  27. package/dist/services/vector-backends/usearch-backend.js +174 -0
  28. package/package.json +3 -3
  29. package/dist/services/sqlite/hnsw-index.d.ts +0 -37
  30. package/dist/services/sqlite/hnsw-index.d.ts.map +0 -1
  31. package/dist/services/sqlite/hnsw-index.js +0 -235
@@ -0,0 +1,63 @@
1
+ export class ExactScanBackend {
2
+ getBackendName() {
3
+ return "exact-scan";
4
+ }
5
+ rankVectors(rows, queryVector, limit) {
6
+ return rows
7
+ .map((row) => ({
8
+ id: row.id,
9
+ distance: 1 - this.cosineSimilarity(row.vector, queryVector),
10
+ }))
11
+ .sort((a, b) => a.distance - b.distance)
12
+ .slice(0, limit);
13
+ }
14
+ async insert(_args) { }
15
+ async insertBatch(_args) { }
16
+ async delete(_args) { }
17
+ async search(args) {
18
+ const column = args.kind === "tags" ? "tags_vector" : "vector";
19
+ const rows = args.db
20
+ .prepare(`SELECT id, ${column} FROM memories WHERE ${column} IS NOT NULL`)
21
+ .all();
22
+ if (rows.length === 0) {
23
+ return [];
24
+ }
25
+ const rankedRows = rows
26
+ .map((row) => ({
27
+ id: row.id,
28
+ vector: this.decodeVector(args.kind === "tags" ? row.tags_vector : row.vector),
29
+ }))
30
+ .filter((row) => row.vector.length > 0);
31
+ return this.rankVectors(rankedRows, args.queryVector, args.limit);
32
+ }
33
+ async rebuildFromShard(_args) { }
34
+ async deleteShardIndexes(_args) { }
35
+ decodeVector(value) {
36
+ if (!value) {
37
+ return new Float32Array();
38
+ }
39
+ if (value instanceof Uint8Array) {
40
+ return new Float32Array(value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength));
41
+ }
42
+ return new Float32Array(value);
43
+ }
44
+ cosineSimilarity(a, b) {
45
+ if (a.length !== b.length) {
46
+ return 0;
47
+ }
48
+ let dot = 0;
49
+ let magA = 0;
50
+ let magB = 0;
51
+ for (let i = 0; i < a.length; i++) {
52
+ const av = a[i] ?? 0;
53
+ const bv = b[i] ?? 0;
54
+ dot += av * bv;
55
+ magA += av * av;
56
+ magB += bv * bv;
57
+ }
58
+ if (magA === 0 || magB === 0) {
59
+ return 0;
60
+ }
61
+ return dot / (Math.sqrt(magA) * Math.sqrt(magB));
62
+ }
63
+ }
@@ -0,0 +1,51 @@
1
+ import type { ShardInfo } from "../sqlite/types.js";
2
+ export type VectorKind = "content" | "tags";
3
+ export interface BackendSearchResult {
4
+ id: string;
5
+ distance: number;
6
+ }
7
+ export interface BackendInsertItem {
8
+ id: string;
9
+ vector: Float32Array;
10
+ }
11
+ export interface VectorBackendSearchParams {
12
+ db: unknown;
13
+ shard: ShardInfo;
14
+ kind: VectorKind;
15
+ queryVector: Float32Array;
16
+ limit: number;
17
+ }
18
+ export interface VectorBackend {
19
+ getBackendName(): string;
20
+ insert(args: {
21
+ id: string;
22
+ vector: Float32Array;
23
+ shard: ShardInfo;
24
+ kind: VectorKind;
25
+ }): Promise<void>;
26
+ insertBatch(args: {
27
+ items: BackendInsertItem[];
28
+ shard: ShardInfo;
29
+ kind: VectorKind;
30
+ }): Promise<void>;
31
+ delete(args: {
32
+ id: string;
33
+ shard: ShardInfo;
34
+ kind: VectorKind;
35
+ }): Promise<void>;
36
+ search(args: VectorBackendSearchParams): Promise<BackendSearchResult[]>;
37
+ rebuildFromShard(args: {
38
+ db: unknown;
39
+ shard: ShardInfo;
40
+ kind: VectorKind;
41
+ }): Promise<void>;
42
+ deleteShardIndexes(args: {
43
+ shard: ShardInfo;
44
+ }): Promise<void>;
45
+ }
46
+ export interface VectorBackendFactoryOptions {
47
+ vectorBackend: "usearch-first" | "usearch" | "exact-scan";
48
+ probeUSearch?: () => Promise<boolean>;
49
+ createUSearchBackend?: () => VectorBackend;
50
+ }
51
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/vector-backends/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,MAAM,CAAC;AAE5C,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,UAAU,CAAC;IACjB,WAAW,EAAE,YAAY,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,IAAI,MAAM,CAAC;IACzB,MAAM,CAAC,IAAI,EAAE;QACX,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,YAAY,CAAC;QACrB,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClB,WAAW,CAAC,IAAI,EAAE;QAChB,KAAK,EAAE,iBAAiB,EAAE,CAAC;QAC3B,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClB,MAAM,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChF,MAAM,CAAC,IAAI,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,gBAAgB,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3F,kBAAkB,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,2BAA2B;IAC1C,aAAa,EAAE,eAAe,GAAG,SAAS,GAAG,YAAY,CAAC;IAC1D,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,oBAAoB,CAAC,EAAE,MAAM,aAAa,CAAC;CAC5C"}
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,47 @@
1
+ import type { BackendInsertItem, BackendSearchResult, VectorBackend, VectorBackendSearchParams, VectorKind } from "./types.js";
2
+ import type { ShardInfo } from "../sqlite/types.js";
3
+ export declare class USearchBackend implements VectorBackend {
4
+ private readonly options;
5
+ private readonly indexes;
6
+ constructor(options: {
7
+ baseDir: string;
8
+ dimensions: number;
9
+ });
10
+ getBackendName(): string;
11
+ insert(args: {
12
+ id: string;
13
+ vector: Float32Array;
14
+ shard: ShardInfo;
15
+ kind: VectorKind;
16
+ }): Promise<void>;
17
+ insertBatch(args: {
18
+ items: BackendInsertItem[];
19
+ shard: ShardInfo;
20
+ kind: VectorKind;
21
+ }): Promise<void>;
22
+ delete(args: {
23
+ id: string;
24
+ shard: ShardInfo;
25
+ kind: VectorKind;
26
+ }): Promise<void>;
27
+ search(args: VectorBackendSearchParams): Promise<BackendSearchResult[]>;
28
+ rebuildFromShard(args: {
29
+ db: unknown;
30
+ shard: ShardInfo;
31
+ kind: VectorKind;
32
+ }): Promise<void>;
33
+ deleteShardIndexes(args: {
34
+ shard: ShardInfo;
35
+ }): Promise<void>;
36
+ insertManyForTest(indexKey: string, items: BackendInsertItem[]): Promise<void>;
37
+ searchForTest(indexKey: string, queryVector: Float32Array, limit: number): Promise<BackendSearchResult[]>;
38
+ private getOrCreateIndex;
39
+ private createEmptyIndex;
40
+ private ensureKey;
41
+ private addItems;
42
+ private upsertItem;
43
+ private decodeVector;
44
+ private getIndexKey;
45
+ private loadUSearch;
46
+ }
47
+ //# sourceMappingURL=usearch-backend.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"usearch-backend.d.ts","sourceRoot":"","sources":["../../../src/services/vector-backends/usearch-backend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,aAAa,EACb,yBAAyB,EACzB,UAAU,EACX,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAcpD,qBAAa,cAAe,YAAW,aAAa;IAIhD,OAAO,CAAC,QAAQ,CAAC,OAAO;IAH1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;gBAGvC,OAAO,EAAE;QACxB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB;IAKH,cAAc,IAAI,MAAM;IAIlB,MAAM,CAAC,IAAI,EAAE;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,YAAY,CAAC;QACrB,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAWX,WAAW,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,iBAAiB,EAAE,CAAC;QAC3B,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAWX,MAAM,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/E,MAAM,CAAC,IAAI,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAsBvE,gBAAgB,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC1F,kBAAkB,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7D,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9E,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,YAAY,EACzB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAqBnB,gBAAgB;YAShB,gBAAgB;IAY9B,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,WAAW;YAIL,WAAW;CAO1B"}
@@ -0,0 +1,174 @@
1
+ export class USearchBackend {
2
+ options;
3
+ indexes = new Map();
4
+ constructor(options) {
5
+ this.options = options;
6
+ void this.options.baseDir;
7
+ }
8
+ getBackendName() {
9
+ return "usearch";
10
+ }
11
+ async insert(args) {
12
+ const indexKey = this.getIndexKey(args.shard, args.kind);
13
+ const cache = await this.getOrCreateIndex(indexKey);
14
+ try {
15
+ this.upsertItem(cache, { id: args.id, vector: args.vector });
16
+ cache.initialized = true;
17
+ }
18
+ catch (error) {
19
+ throw new Error(`USearch insert failed for ${indexKey}: ${String(error)}`);
20
+ }
21
+ }
22
+ async insertBatch(args) {
23
+ const indexKey = this.getIndexKey(args.shard, args.kind);
24
+ const cache = await this.getOrCreateIndex(indexKey);
25
+ try {
26
+ this.addItems(cache, args.items);
27
+ cache.initialized = true;
28
+ }
29
+ catch (error) {
30
+ throw new Error(`USearch batch insert failed for ${indexKey}: ${String(error)}`);
31
+ }
32
+ }
33
+ async delete(args) {
34
+ const cache = await this.getOrCreateIndex(this.getIndexKey(args.shard, args.kind));
35
+ const key = cache.idToKey.get(args.id);
36
+ if (key === undefined)
37
+ return;
38
+ cache.index.remove(key);
39
+ cache.idToKey.delete(args.id);
40
+ cache.keyToId.delete(key);
41
+ }
42
+ async search(args) {
43
+ const indexKey = this.getIndexKey(args.shard, args.kind);
44
+ const cache = await this.getOrCreateIndex(indexKey);
45
+ try {
46
+ const matches = cache.index.search(args.queryVector, args.limit);
47
+ return Array.from(matches.keys, (key, index) => {
48
+ const id = cache.keyToId.get(key);
49
+ if (!id) {
50
+ throw new Error(`USearch index metadata missing for key ${String(key)} in ${cache.indexKey}`);
51
+ }
52
+ return {
53
+ id,
54
+ distance: matches.distances[index] ?? 0,
55
+ };
56
+ });
57
+ }
58
+ catch (error) {
59
+ throw new Error(`USearch search failed for ${indexKey}: ${String(error)}`);
60
+ }
61
+ }
62
+ async rebuildFromShard(args) {
63
+ const indexKey = this.getIndexKey(args.shard, args.kind);
64
+ const existing = this.indexes.get(indexKey);
65
+ if (existing?.initialized) {
66
+ return;
67
+ }
68
+ const column = args.kind === "tags" ? "tags_vector" : "vector";
69
+ const rows = args.db
70
+ .prepare(`SELECT id, ${column} FROM memories WHERE ${column} IS NOT NULL`)
71
+ .all();
72
+ const cache = await this.createEmptyIndex(indexKey);
73
+ this.indexes.set(indexKey, cache);
74
+ for (const row of rows) {
75
+ const raw = args.kind === "tags" ? row.tags_vector : row.vector;
76
+ const vector = this.decodeVector(raw);
77
+ if (vector.length === 0)
78
+ continue;
79
+ this.upsertItem(cache, { id: row.id, vector });
80
+ }
81
+ cache.initialized = true;
82
+ }
83
+ async deleteShardIndexes(args) {
84
+ for (const kind of ["content", "tags"]) {
85
+ const indexKey = this.getIndexKey(args.shard, kind);
86
+ this.indexes.delete(indexKey);
87
+ }
88
+ }
89
+ async insertManyForTest(indexKey, items) {
90
+ const cache = await this.getOrCreateIndex(indexKey);
91
+ this.addItems(cache, items);
92
+ cache.initialized = true;
93
+ }
94
+ async searchForTest(indexKey, queryVector, limit) {
95
+ const cache = await this.getOrCreateIndex(indexKey);
96
+ try {
97
+ const matches = cache.index.search(queryVector, limit);
98
+ return Array.from(matches.keys, (key, index) => {
99
+ const id = cache.keyToId.get(key);
100
+ if (!id) {
101
+ throw new Error(`USearch index metadata missing for key ${String(key)} in ${cache.indexKey}`);
102
+ }
103
+ return {
104
+ id,
105
+ distance: matches.distances[index] ?? 0,
106
+ };
107
+ });
108
+ }
109
+ catch (error) {
110
+ throw new Error(`USearch test search failed for ${indexKey}: ${String(error)}`);
111
+ }
112
+ }
113
+ async getOrCreateIndex(indexKey) {
114
+ const existing = this.indexes.get(indexKey);
115
+ if (existing)
116
+ return existing;
117
+ const cache = await this.createEmptyIndex(indexKey);
118
+ this.indexes.set(indexKey, cache);
119
+ return cache;
120
+ }
121
+ async createEmptyIndex(indexKey) {
122
+ const usearch = await this.loadUSearch();
123
+ return {
124
+ index: new usearch.Index({ dimensions: this.options.dimensions, metric: "cos" }),
125
+ idToKey: new Map(),
126
+ keyToId: new Map(),
127
+ nextKey: 1n,
128
+ indexKey,
129
+ initialized: false,
130
+ };
131
+ }
132
+ ensureKey(cache, id) {
133
+ const existing = cache.idToKey.get(id);
134
+ if (existing !== undefined)
135
+ return existing;
136
+ const key = cache.nextKey;
137
+ cache.nextKey += 1n;
138
+ cache.idToKey.set(id, key);
139
+ cache.keyToId.set(key, id);
140
+ return key;
141
+ }
142
+ addItems(cache, items) {
143
+ for (const item of items) {
144
+ this.upsertItem(cache, item);
145
+ }
146
+ }
147
+ upsertItem(cache, item) {
148
+ const existing = cache.idToKey.get(item.id);
149
+ if (existing !== undefined) {
150
+ cache.index.remove(existing);
151
+ }
152
+ const key = this.ensureKey(cache, item.id);
153
+ cache.index.add(key, item.vector);
154
+ }
155
+ decodeVector(value) {
156
+ if (!value)
157
+ return new Float32Array();
158
+ if (value instanceof Uint8Array) {
159
+ return new Float32Array(value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength));
160
+ }
161
+ return new Float32Array(value);
162
+ }
163
+ getIndexKey(shard, kind) {
164
+ return `${shard.scope}_${shard.scopeHash}_${shard.shardIndex}_${kind}`;
165
+ }
166
+ async loadUSearch() {
167
+ try {
168
+ return await import("usearch");
169
+ }
170
+ catch (error) {
171
+ throw new Error(`Failed to load usearch backend: ${String(error)}`);
172
+ }
173
+ }
174
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-mem",
3
- "version": "2.11.8",
3
+ "version": "2.11.9",
4
4
  "description": "OpenCode plugin that gives coding agents persistent memory using local vector database",
5
5
  "type": "module",
6
6
  "main": "dist/plugin.js",
@@ -36,8 +36,8 @@
36
36
  "@opencode-ai/plugin": "^1.0.162",
37
37
  "@xenova/transformers": "^2.17.2",
38
38
  "franc-min": "^6.2.0",
39
- "hnswlib-wasm": "^0.8.2",
40
- "iso-639-3": "^3.0.1"
39
+ "iso-639-3": "^3.0.1",
40
+ "usearch": "^2.21.4"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/bun": "^1.3.8",
@@ -1,37 +0,0 @@
1
- export interface HNSWIndexData {
2
- id: string;
3
- vector: Float32Array;
4
- }
5
- export declare class HNSWIndex {
6
- private index;
7
- private idMap;
8
- private reverseMap;
9
- private nextId;
10
- private dimensions;
11
- private indexPath;
12
- private maxElements;
13
- private initialized;
14
- constructor(dimensions: number, indexPath: string);
15
- private ensureInitialized;
16
- insert(id: string, vector: Float32Array): Promise<void>;
17
- insertBatch(items: HNSWIndexData[]): Promise<void>;
18
- search(queryVector: Float32Array, k: number): Promise<{
19
- id: string;
20
- distance: number;
21
- }[]>;
22
- delete(id: string): Promise<void>;
23
- save(): Promise<void>;
24
- getCount(): number;
25
- isPopulated(): boolean;
26
- }
27
- export declare class HNSWIndexManager {
28
- private indexes;
29
- private baseDir;
30
- constructor(baseDir: string);
31
- getIndex(scope: string, scopeHash: string, shardIndex: number): HNSWIndex;
32
- getTagsIndex(scope: string, scopeHash: string, shardIndex: number): HNSWIndex;
33
- rebuildFromShard(db: any, scope: string, scopeHash: string, shardIndex: number): Promise<void>;
34
- deleteIndex(scope: string, scopeHash: string, shardIndex: number): Promise<void>;
35
- cleanupOrphanedIndexes(validKeys: Set<string>): Promise<void>;
36
- }
37
- //# sourceMappingURL=hnsw-index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"hnsw-index.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/hnsw-index.ts"],"names":[],"mappings":"AAoBA,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,WAAW,CAAkB;gBAEzB,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;YAKnC,iBAAiB;IAyBzB,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBvD,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBlD,MAAM,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAqBzF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB3B,QAAQ,IAAI,MAAM;IAIlB,WAAW,IAAI,OAAO;CAGvB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM;IAO3B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IAWzE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IAWvE,gBAAgB,CACpB,EAAE,EAAE,GAAG,EACP,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAoCV,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhF,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CA0BpE"}
@@ -1,235 +0,0 @@
1
- import { mkdirSync, existsSync, writeFileSync, unlinkSync, readdirSync } from "node:fs";
2
- import { join, dirname, basename } from "node:path";
3
- import { log } from "../logger.js";
4
- import { CONFIG } from "../../config.js";
5
- let HNSWLib = null;
6
- async function loadHNSWLib() {
7
- if (!HNSWLib) {
8
- // hnswlib-wasm is compiled with Emscripten -sENVIRONMENT=web and requires
9
- // a browser-like global. This monkey-patch allows it to load in Node.js/Bun.
10
- if (typeof globalThis.window === "undefined") {
11
- globalThis.window = globalThis;
12
- }
13
- const { loadHnswlib } = await import("hnswlib-wasm");
14
- HNSWLib = await loadHnswlib();
15
- }
16
- return HNSWLib;
17
- }
18
- export class HNSWIndex {
19
- index = null;
20
- idMap = new Map();
21
- reverseMap = new Map();
22
- nextId = 0;
23
- dimensions;
24
- indexPath;
25
- maxElements = 50000;
26
- initialized = false;
27
- constructor(dimensions, indexPath) {
28
- this.dimensions = dimensions;
29
- this.indexPath = indexPath;
30
- }
31
- async ensureInitialized() {
32
- if (this.initialized)
33
- return;
34
- const hnsw = await loadHNSWLib();
35
- const dir = dirname(this.indexPath);
36
- if (!existsSync(dir)) {
37
- mkdirSync(dir, { recursive: true });
38
- }
39
- // hnswlib-wasm uses Emscripten MEMFS (in-memory virtual FS) and has no
40
- // exported FS API for bridging to real filesystem. HNSW indexes are kept
41
- // purely in-memory and rebuilt from SQLite vectors on process restart.
42
- // Constructor requires 3 args: (spaceName, numDimensions, autoSaveFilename)
43
- this.index = new hnsw.HierarchicalNSW("cosine", this.dimensions, "");
44
- // initIndex requires 4 args: (maxElements, m, efConstruction, randomSeed)
45
- this.index.initIndex(this.maxElements, 16, 200, 100);
46
- this.initialized = true;
47
- log("HNSW index initialized (in-memory)", {
48
- path: this.indexPath,
49
- dimensions: this.dimensions,
50
- });
51
- }
52
- async insert(id, vector) {
53
- await this.ensureInitialized();
54
- if (this.reverseMap.has(id)) {
55
- const internalId = this.reverseMap.get(id);
56
- this.index.markDelete(internalId);
57
- }
58
- const internalId = this.nextId++;
59
- // hnswlib-wasm addPoint requires 3 args: (point, label, replaceDeleted)
60
- this.index.addPoint(vector, internalId, false);
61
- this.idMap.set(internalId, id);
62
- this.reverseMap.set(id, internalId);
63
- await this.save();
64
- }
65
- async insertBatch(items) {
66
- await this.ensureInitialized();
67
- for (const item of items) {
68
- if (this.reverseMap.has(item.id)) {
69
- const internalId = this.reverseMap.get(item.id);
70
- this.index.markDelete(internalId);
71
- }
72
- const internalId = this.nextId++;
73
- // hnswlib-wasm addPoint requires 3 args: (point, label, replaceDeleted)
74
- this.index.addPoint(item.vector, internalId, false);
75
- this.idMap.set(internalId, item.id);
76
- this.reverseMap.set(item.id, internalId);
77
- }
78
- await this.save();
79
- }
80
- async search(queryVector, k) {
81
- await this.ensureInitialized();
82
- try {
83
- // hnswlib-wasm searchKnn requires 3 args: (queryPoint, numNeighbors, filter)
84
- const actualK = Math.min(k, this.reverseMap.size);
85
- if (actualK === 0)
86
- return [];
87
- const results = this.index.searchKnn(queryVector, actualK, null);
88
- return results.neighbors
89
- .map((internalId, idx) => ({
90
- id: this.idMap.get(internalId) || "",
91
- distance: results.distances[idx] ?? 0,
92
- }))
93
- .filter((r) => r.id);
94
- }
95
- catch (error) {
96
- log("HNSW search error", { error: String(error) });
97
- return [];
98
- }
99
- }
100
- async delete(id) {
101
- await this.ensureInitialized();
102
- if (this.reverseMap.has(id)) {
103
- const internalId = this.reverseMap.get(id);
104
- this.index.markDelete(internalId);
105
- this.idMap.delete(internalId);
106
- this.reverseMap.delete(id);
107
- await this.save();
108
- }
109
- }
110
- async save() {
111
- if (!this.index)
112
- return;
113
- const dir = dirname(this.indexPath);
114
- if (!existsSync(dir)) {
115
- mkdirSync(dir, { recursive: true });
116
- }
117
- // Only persist id mapping (.meta file). HNSW index data lives in-memory
118
- // and is rebuilt from SQLite vectors on process restart.
119
- const metaPath = this.indexPath + ".meta";
120
- const meta = {
121
- nextId: this.nextId,
122
- idMap: Object.fromEntries(this.idMap),
123
- reverseMap: Object.fromEntries(this.reverseMap),
124
- };
125
- writeFileSync(metaPath, JSON.stringify(meta));
126
- }
127
- getCount() {
128
- return this.reverseMap.size;
129
- }
130
- isPopulated() {
131
- return this.reverseMap.size > 0;
132
- }
133
- }
134
- export class HNSWIndexManager {
135
- indexes = new Map();
136
- baseDir;
137
- constructor(baseDir) {
138
- this.baseDir = baseDir;
139
- if (!existsSync(baseDir)) {
140
- mkdirSync(baseDir, { recursive: true });
141
- }
142
- }
143
- getIndex(scope, scopeHash, shardIndex) {
144
- const key = `${scope}_${scopeHash}_${shardIndex}`;
145
- if (!this.indexes.has(key)) {
146
- const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
147
- this.indexes.set(key, new HNSWIndex(CONFIG.embeddingDimensions, indexPath));
148
- }
149
- return this.indexes.get(key);
150
- }
151
- getTagsIndex(scope, scopeHash, shardIndex) {
152
- const key = `${scope}_${scopeHash}_${shardIndex}_tags`;
153
- if (!this.indexes.has(key)) {
154
- const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
155
- this.indexes.set(key, new HNSWIndex(CONFIG.embeddingDimensions, indexPath));
156
- }
157
- return this.indexes.get(key);
158
- }
159
- async rebuildFromShard(db, scope, scopeHash, shardIndex) {
160
- const contentIndex = this.getIndex(scope, scopeHash, shardIndex);
161
- const tagsIndex = this.getTagsIndex(scope, scopeHash, shardIndex);
162
- const rows = db.prepare("SELECT id, vector, tags_vector FROM memories").all();
163
- const contentItems = [];
164
- const tagsItems = [];
165
- for (const row of rows) {
166
- if (row.vector) {
167
- const vector = new Float32Array(row.vector.buffer);
168
- contentItems.push({ id: row.id, vector });
169
- }
170
- if (row.tags_vector) {
171
- const tagsVector = new Float32Array(row.tags_vector.buffer);
172
- tagsItems.push({ id: row.id, vector: tagsVector });
173
- }
174
- }
175
- if (contentItems.length > 0) {
176
- await contentIndex.insertBatch(contentItems);
177
- }
178
- if (tagsItems.length > 0) {
179
- await tagsIndex.insertBatch(tagsItems);
180
- }
181
- log("HNSW indexes rebuilt", {
182
- scope,
183
- scopeHash,
184
- shardIndex,
185
- content: contentItems.length,
186
- tags: tagsItems.length,
187
- });
188
- }
189
- async deleteIndex(scope, scopeHash, shardIndex) {
190
- const contentKey = `${scope}_${scopeHash}_${shardIndex}`;
191
- const tagsKey = `${scope}_${scopeHash}_${shardIndex}_tags`;
192
- this.indexes.delete(contentKey);
193
- this.indexes.delete(tagsKey);
194
- for (const key of [contentKey, tagsKey]) {
195
- const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
196
- const metaPath = indexPath + ".meta";
197
- try {
198
- if (existsSync(indexPath))
199
- unlinkSync(indexPath);
200
- if (existsSync(metaPath))
201
- unlinkSync(metaPath);
202
- }
203
- catch (error) {
204
- log("Error deleting HNSW index files", { path: indexPath, error: String(error) });
205
- }
206
- }
207
- }
208
- async cleanupOrphanedIndexes(validKeys) {
209
- const scopeDirs = ["users", "projects"];
210
- for (const scopeDir of scopeDirs) {
211
- const dir = join(this.baseDir, scopeDir);
212
- if (!existsSync(dir))
213
- continue;
214
- const files = readdirSync(dir);
215
- for (const file of files) {
216
- if (file.endsWith(".hnsw")) {
217
- const key = basename(file, ".hnsw");
218
- if (!validKeys.has(key)) {
219
- const indexPath = join(dir, file);
220
- const metaPath = indexPath + ".meta";
221
- try {
222
- unlinkSync(indexPath);
223
- if (existsSync(metaPath))
224
- unlinkSync(metaPath);
225
- log("Removed orphaned HNSW index", { path: indexPath });
226
- }
227
- catch (error) {
228
- log("Error removing orphaned index", { path: indexPath, error: String(error) });
229
- }
230
- }
231
- }
232
- }
233
- }
234
- }
235
- }