opencode-mem 2.11.2 → 2.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -29,6 +29,7 @@ export declare class HNSWIndexManager {
|
|
|
29
29
|
private baseDir;
|
|
30
30
|
constructor(baseDir: string);
|
|
31
31
|
getIndex(scope: string, scopeHash: string, shardIndex: number): HNSWIndex;
|
|
32
|
+
getTagsIndex(scope: string, scopeHash: string, shardIndex: number): HNSWIndex;
|
|
32
33
|
rebuildFromShard(db: any, scope: string, scopeHash: string, shardIndex: number): Promise<void>;
|
|
33
34
|
deleteIndex(scope: string, scopeHash: string, shardIndex: number): Promise<void>;
|
|
34
35
|
cleanupOrphanedIndexes(validKeys: Set<string>): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hnsw-index.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/hnsw-index.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,KAAK,CAAgC;IAC7C,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,WAAW,CAAkB;gBAEzB,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;YAKnC,iBAAiB;IAyC/B,OAAO,CAAC,aAAa;IAIf,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvD,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,MAAM,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAkBzF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB3B,QAAQ,IAAI,MAAM;CAGnB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM;IAO3B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;
|
|
1
|
+
{"version":3,"file":"hnsw-index.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/hnsw-index.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,KAAK,CAAgC;IAC7C,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,WAAW,CAAkB;gBAEzB,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;YAKnC,iBAAiB;IAyC/B,OAAO,CAAC,aAAa;IAIf,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvD,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,MAAM,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAkBzF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB3B,QAAQ,IAAI,MAAM;CAGnB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM;IAO3B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IAWzE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IAWvE,gBAAgB,CACpB,EAAE,EAAE,GAAG,EACP,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAoCV,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhF,sBAAsB,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CA0BpE"}
|
|
@@ -144,34 +144,61 @@ export class HNSWIndexManager {
|
|
|
144
144
|
}
|
|
145
145
|
return this.indexes.get(key);
|
|
146
146
|
}
|
|
147
|
+
getTagsIndex(scope, scopeHash, shardIndex) {
|
|
148
|
+
const key = `${scope}_${scopeHash}_${shardIndex}_tags`;
|
|
149
|
+
if (!this.indexes.has(key)) {
|
|
150
|
+
const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
|
|
151
|
+
this.indexes.set(key, new HNSWIndex(CONFIG.embeddingDimensions, indexPath));
|
|
152
|
+
}
|
|
153
|
+
return this.indexes.get(key);
|
|
154
|
+
}
|
|
147
155
|
async rebuildFromShard(db, scope, scopeHash, shardIndex) {
|
|
148
|
-
const
|
|
149
|
-
const
|
|
150
|
-
const
|
|
156
|
+
const contentIndex = this.getIndex(scope, scopeHash, shardIndex);
|
|
157
|
+
const tagsIndex = this.getTagsIndex(scope, scopeHash, shardIndex);
|
|
158
|
+
const rows = db.prepare("SELECT id, vector, tags_vector FROM memories").all();
|
|
159
|
+
const contentItems = [];
|
|
160
|
+
const tagsItems = [];
|
|
151
161
|
for (const row of rows) {
|
|
152
162
|
if (row.vector) {
|
|
153
163
|
const vector = new Float32Array(row.vector.buffer);
|
|
154
|
-
|
|
164
|
+
contentItems.push({ id: row.id, vector });
|
|
165
|
+
}
|
|
166
|
+
if (row.tags_vector) {
|
|
167
|
+
const tagsVector = new Float32Array(row.tags_vector.buffer);
|
|
168
|
+
tagsItems.push({ id: row.id, vector: tagsVector });
|
|
155
169
|
}
|
|
156
170
|
}
|
|
157
|
-
if (
|
|
158
|
-
await
|
|
159
|
-
log("HNSW index rebuilt", { scope, scopeHash, shardIndex, count: items.length });
|
|
171
|
+
if (contentItems.length > 0) {
|
|
172
|
+
await contentIndex.insertBatch(contentItems);
|
|
160
173
|
}
|
|
174
|
+
if (tagsItems.length > 0) {
|
|
175
|
+
await tagsIndex.insertBatch(tagsItems);
|
|
176
|
+
}
|
|
177
|
+
log("HNSW indexes rebuilt", {
|
|
178
|
+
scope,
|
|
179
|
+
scopeHash,
|
|
180
|
+
shardIndex,
|
|
181
|
+
content: contentItems.length,
|
|
182
|
+
tags: tagsItems.length,
|
|
183
|
+
});
|
|
161
184
|
}
|
|
162
185
|
async deleteIndex(scope, scopeHash, shardIndex) {
|
|
163
|
-
const
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
186
|
+
const contentKey = `${scope}_${scopeHash}_${shardIndex}`;
|
|
187
|
+
const tagsKey = `${scope}_${scopeHash}_${shardIndex}_tags`;
|
|
188
|
+
this.indexes.delete(contentKey);
|
|
189
|
+
this.indexes.delete(tagsKey);
|
|
190
|
+
for (const key of [contentKey, tagsKey]) {
|
|
191
|
+
const indexPath = join(this.baseDir, scope + "s", `${key}.hnsw`);
|
|
192
|
+
const metaPath = indexPath + ".meta";
|
|
193
|
+
try {
|
|
194
|
+
if (existsSync(indexPath))
|
|
195
|
+
unlinkSync(indexPath);
|
|
196
|
+
if (existsSync(metaPath))
|
|
197
|
+
unlinkSync(metaPath);
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
log("Error deleting HNSW index files", { path: indexPath, error: String(error) });
|
|
201
|
+
}
|
|
175
202
|
}
|
|
176
203
|
}
|
|
177
204
|
async cleanupOrphanedIndexes(validKeys) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExE,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAC/B,KAAK,YAAY,GAAG,OAAO,QAAQ,CAAC,SAAS,CAAC;AAI9C,qBAAa,YAAY;IACvB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExE,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAC/B,KAAK,YAAY,GAAG,OAAO,QAAQ,CAAC,SAAS,CAAC;AAI9C,qBAAa,YAAY;IACvB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,IAAI;IAqDvE,aAAa,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,YAAY,EACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IA6EpB,kBAAkB,CACtB,MAAM,EAAE,SAAS,EAAE,EACnB,WAAW,EAAE,YAAY,EACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,MAAM,EAC3B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAiBpB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlF,YAAY,CAChB,EAAE,EAAE,YAAY,EAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,KAAK,CAAC,EAAE,SAAS,EACjB,UAAU,CAAC,EAAE,YAAY,GACxB,OAAO,CAAC,IAAI,CAAC;IA4BhB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE;IAW1E,cAAc,CAAC,EAAE,EAAE,YAAY,GAAG,GAAG,EAAE;IAKvC,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAK7D,sBAAsB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE;IAgBlE,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAM5D,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG,MAAM;IAMzC,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG,GAAG,EAAE;IAexC,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAKnD,WAAW,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK/C,gBAAgB,CACpB,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAIhB,eAAe,IAAI,gBAAgB;CAGpC;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|
|
@@ -9,27 +9,45 @@ export class VectorSearch {
|
|
|
9
9
|
insertVector(db, record, shard) {
|
|
10
10
|
const insertMemory = db.prepare(`
|
|
11
11
|
INSERT INTO memories (
|
|
12
|
-
id, content, vector, container_tag, tags, type, created_at, updated_at,
|
|
12
|
+
id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
|
|
13
13
|
metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url
|
|
14
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
14
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
15
15
|
`);
|
|
16
16
|
const vectorBuffer = new Uint8Array(record.vector.buffer);
|
|
17
|
-
|
|
17
|
+
const tagsVectorBuffer = record.tagsVector ? new Uint8Array(record.tagsVector.buffer) : null;
|
|
18
|
+
insertMemory.run(record.id, record.content, vectorBuffer, tagsVectorBuffer, record.containerTag, record.tags || null, record.type || null, record.createdAt, record.updatedAt, record.metadata || null, record.displayName || null, record.userName || null, record.userEmail || null, record.projectPath || null, record.projectName || null, record.gitRepoUrl || null);
|
|
18
19
|
if (shard && record.vector) {
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
log("HNSW insert error", { memoryId: record.id, error: String(err) });
|
|
20
|
+
const contentIndex = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
21
|
+
contentIndex.insert(record.id, record.vector).catch((err) => {
|
|
22
|
+
log("HNSW content insert error", { memoryId: record.id, error: String(err) });
|
|
22
23
|
});
|
|
24
|
+
if (record.tagsVector) {
|
|
25
|
+
const tagsIndex = hnswIndexManager.getTagsIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
26
|
+
tagsIndex.insert(record.id, record.tagsVector).catch((err) => {
|
|
27
|
+
log("HNSW tags insert error", { memoryId: record.id, error: String(err) });
|
|
28
|
+
});
|
|
29
|
+
}
|
|
23
30
|
}
|
|
24
31
|
}
|
|
25
32
|
async searchInShard(shard, queryVector, containerTag, limit, queryText) {
|
|
26
33
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
27
|
-
const
|
|
28
|
-
const
|
|
34
|
+
const contentIndex = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
35
|
+
const tagsIndex = hnswIndexManager.getTagsIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
36
|
+
const contentResults = await contentIndex.search(queryVector, limit * 4);
|
|
37
|
+
const tagsResults = await tagsIndex.search(queryVector, limit * 4);
|
|
29
38
|
const scoreMap = new Map();
|
|
30
39
|
for (const r of contentResults) {
|
|
31
40
|
scoreMap.set(r.id, { contentSim: 1 - r.distance, tagsSim: 0 });
|
|
32
41
|
}
|
|
42
|
+
for (const r of tagsResults) {
|
|
43
|
+
const entry = scoreMap.get(r.id);
|
|
44
|
+
if (entry) {
|
|
45
|
+
entry.tagsSim = 1 - r.distance;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
scoreMap.set(r.id, { contentSim: 0, tagsSim: 1 - r.distance });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
33
51
|
const ids = Array.from(scoreMap.keys());
|
|
34
52
|
if (ids.length === 0)
|
|
35
53
|
return [];
|
|
@@ -55,8 +73,8 @@ export class VectorSearch {
|
|
|
55
73
|
const matches = queryWords.filter((w) => memoryTags.some((t) => t.includes(w) || w.includes(t))).length;
|
|
56
74
|
exactMatchBoost = matches / Math.max(queryWords.length, 1);
|
|
57
75
|
}
|
|
58
|
-
const
|
|
59
|
-
const similarity =
|
|
76
|
+
const finalTagsSim = Math.max(scores.tagsSim, exactMatchBoost);
|
|
77
|
+
const similarity = scores.contentSim * 0.6 + finalTagsSim * 0.4;
|
|
60
78
|
return {
|
|
61
79
|
id: row.id,
|
|
62
80
|
memory: row.content,
|
|
@@ -92,16 +110,22 @@ export class VectorSearch {
|
|
|
92
110
|
async deleteVector(db, memoryId, shard) {
|
|
93
111
|
db.prepare(`DELETE FROM memories WHERE id = ?`).run(memoryId);
|
|
94
112
|
if (shard) {
|
|
95
|
-
const
|
|
96
|
-
|
|
113
|
+
const contentIndex = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
114
|
+
const tagsIndex = hnswIndexManager.getTagsIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
115
|
+
await Promise.all([contentIndex.delete(memoryId), tagsIndex.delete(memoryId)]);
|
|
97
116
|
}
|
|
98
117
|
}
|
|
99
118
|
async updateVector(db, memoryId, vector, shard, tagsVector) {
|
|
100
119
|
const vectorBuffer = new Uint8Array(vector.buffer);
|
|
101
|
-
|
|
120
|
+
const tagsVectorBuffer = tagsVector ? new Uint8Array(tagsVector.buffer) : null;
|
|
121
|
+
db.prepare(`UPDATE memories SET vector = ?, tags_vector = ? WHERE id = ?`).run(vectorBuffer, tagsVectorBuffer, memoryId);
|
|
102
122
|
if (shard && vector) {
|
|
103
|
-
const
|
|
104
|
-
await
|
|
123
|
+
const contentIndex = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
124
|
+
await contentIndex.insert(memoryId, vector);
|
|
125
|
+
if (tagsVector) {
|
|
126
|
+
const tagsIndex = hnswIndexManager.getTagsIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
127
|
+
await tagsIndex.insert(memoryId, tagsVector);
|
|
128
|
+
}
|
|
105
129
|
}
|
|
106
130
|
}
|
|
107
131
|
listMemories(db, containerTag, limit) {
|