opencode-mem 2.11.7 → 2.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -12
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +2 -0
- package/dist/services/ai/provider-config.d.ts +16 -0
- package/dist/services/ai/provider-config.d.ts.map +1 -0
- package/dist/services/ai/provider-config.js +13 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts.map +1 -1
- package/dist/services/ai/providers/anthropic-messages.js +2 -0
- package/dist/services/ai/providers/google-gemini.d.ts.map +1 -1
- package/dist/services/ai/providers/google-gemini.js +2 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts.map +1 -1
- package/dist/services/ai/providers/openai-chat-completion.js +6 -0
- package/dist/services/ai/providers/openai-responses.d.ts.map +1 -1
- package/dist/services/ai/providers/openai-responses.js +2 -0
- package/dist/services/api-handlers.d.ts.map +1 -1
- package/dist/services/api-handlers.js +9 -11
- package/dist/services/auto-capture.js +2 -7
- package/dist/services/cleanup-service.js +1 -1
- package/dist/services/client.js +1 -1
- package/dist/services/deduplication-service.js +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +20 -10
- package/dist/services/migration-service.js +3 -3
- package/dist/services/sqlite/shard-manager.d.ts +1 -1
- package/dist/services/sqlite/shard-manager.d.ts.map +1 -1
- package/dist/services/sqlite/shard-manager.js +12 -1
- package/dist/services/sqlite/vector-search.d.ts +8 -4
- package/dist/services/sqlite/vector-search.d.ts.map +1 -1
- package/dist/services/sqlite/vector-search.js +107 -44
- package/dist/services/user-memory-learning.js +2 -7
- package/dist/services/vector-backends/backend-factory.d.ts +3 -0
- package/dist/services/vector-backends/backend-factory.d.ts.map +1 -0
- package/dist/services/vector-backends/backend-factory.js +104 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts +39 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/exact-scan-backend.js +63 -0
- package/dist/services/vector-backends/types.d.ts +51 -0
- package/dist/services/vector-backends/types.d.ts.map +1 -0
- package/dist/services/vector-backends/types.js +1 -0
- package/dist/services/vector-backends/usearch-backend.d.ts +47 -0
- package/dist/services/vector-backends/usearch-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/usearch-backend.js +174 -0
- package/package.json +3 -3
- package/dist/services/sqlite/hnsw-index.d.ts +0 -37
- package/dist/services/sqlite/hnsw-index.d.ts.map +0 -1
- package/dist/services/sqlite/hnsw-index.js +0 -235
|
@@ -1,45 +1,93 @@
|
|
|
1
1
|
import { getDatabase } from "./sqlite-bootstrap.js";
|
|
2
2
|
import { connectionManager } from "./connection-manager.js";
|
|
3
|
-
import { HNSWIndexManager } from "./hnsw-index.js";
|
|
4
3
|
import { log } from "../logger.js";
|
|
5
4
|
import { CONFIG } from "../../config.js";
|
|
5
|
+
import { createVectorBackend } from "../vector-backends/backend-factory.js";
|
|
6
|
+
import { ExactScanBackend } from "../vector-backends/exact-scan-backend.js";
|
|
6
7
|
const Database = getDatabase();
|
|
7
|
-
|
|
8
|
+
function toBlob(vector) {
|
|
9
|
+
return vector ? new Uint8Array(vector.buffer) : null;
|
|
10
|
+
}
|
|
8
11
|
export class VectorSearch {
|
|
9
|
-
|
|
12
|
+
backendPromise;
|
|
13
|
+
fallbackBackend;
|
|
14
|
+
constructor(backend, fallbackBackend = new ExactScanBackend()) {
|
|
15
|
+
this.backendPromise = backend
|
|
16
|
+
? Promise.resolve(backend)
|
|
17
|
+
: createVectorBackend({ vectorBackend: CONFIG.vectorBackend });
|
|
18
|
+
this.fallbackBackend = fallbackBackend;
|
|
19
|
+
}
|
|
20
|
+
async getBackend() {
|
|
21
|
+
return this.backendPromise;
|
|
22
|
+
}
|
|
23
|
+
async insertVector(db, record, shard) {
|
|
10
24
|
const insertMemory = db.prepare(`
|
|
11
25
|
INSERT INTO memories (
|
|
12
26
|
id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
|
|
13
27
|
metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url
|
|
14
28
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
15
29
|
`);
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
if (record.tagsVector) {
|
|
25
|
-
const tagsIndex = hnswIndexManager.getTagsIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
26
|
-
tagsIndex.insert(record.id, record.tagsVector).catch((err) => {
|
|
27
|
-
log("HNSW tags insert error", { memoryId: record.id, error: String(err) });
|
|
28
|
-
});
|
|
30
|
+
insertMemory.run(record.id, record.content, toBlob(record.vector), toBlob(record.tagsVector), record.containerTag, record.tags || null, record.type || null, record.createdAt, record.updatedAt, record.metadata || null, record.displayName || null, record.userName || null, record.userEmail || null, record.projectPath || null, record.projectName || null, record.gitRepoUrl || null);
|
|
31
|
+
try {
|
|
32
|
+
if (shard) {
|
|
33
|
+
const backend = await this.getBackend();
|
|
34
|
+
await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" });
|
|
35
|
+
if (record.tagsVector) {
|
|
36
|
+
await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" });
|
|
37
|
+
}
|
|
29
38
|
}
|
|
30
39
|
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
db.prepare(`DELETE FROM memories WHERE id = ?`).run(record.id);
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
31
44
|
}
|
|
32
45
|
async searchInShard(shard, queryVector, containerTag, limit, queryText) {
|
|
33
46
|
const db = connectionManager.getConnection(shard.dbPath);
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
await
|
|
47
|
+
const backend = await this.getBackend();
|
|
48
|
+
let contentResults;
|
|
49
|
+
let tagsResults;
|
|
50
|
+
try {
|
|
51
|
+
await backend.rebuildFromShard({ db, shard, kind: "content" });
|
|
52
|
+
await backend.rebuildFromShard({ db, shard, kind: "tags" });
|
|
53
|
+
contentResults = await backend.search({
|
|
54
|
+
db,
|
|
55
|
+
shard,
|
|
56
|
+
kind: "content",
|
|
57
|
+
queryVector,
|
|
58
|
+
limit: limit * 4,
|
|
59
|
+
});
|
|
60
|
+
tagsResults = await backend.search({
|
|
61
|
+
db,
|
|
62
|
+
shard,
|
|
63
|
+
kind: "tags",
|
|
64
|
+
queryVector,
|
|
65
|
+
limit: limit * 4,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
log("Vector search degraded to exact scan in shard", {
|
|
70
|
+
shardId: shard.id,
|
|
71
|
+
backend: backend.getBackendName(),
|
|
72
|
+
error: String(error),
|
|
73
|
+
});
|
|
74
|
+
await this.fallbackBackend.rebuildFromShard({ db, shard, kind: "content" });
|
|
75
|
+
await this.fallbackBackend.rebuildFromShard({ db, shard, kind: "tags" });
|
|
76
|
+
contentResults = await this.fallbackBackend.search({
|
|
77
|
+
db,
|
|
78
|
+
shard,
|
|
79
|
+
kind: "content",
|
|
80
|
+
queryVector,
|
|
81
|
+
limit: limit * 4,
|
|
82
|
+
});
|
|
83
|
+
tagsResults = await this.fallbackBackend.search({
|
|
84
|
+
db,
|
|
85
|
+
shard,
|
|
86
|
+
kind: "tags",
|
|
87
|
+
queryVector,
|
|
88
|
+
limit: limit * 4,
|
|
89
|
+
});
|
|
40
90
|
}
|
|
41
|
-
const contentResults = await contentIndex.search(queryVector, limit * 4);
|
|
42
|
-
const tagsResults = await tagsIndex.search(queryVector, limit * 4);
|
|
43
91
|
const scoreMap = new Map();
|
|
44
92
|
for (const r of contentResults) {
|
|
45
93
|
scoreMap.set(r.id, { contentSim: 1 - r.distance, tagsSim: 0 });
|
|
@@ -59,7 +107,7 @@ export class VectorSearch {
|
|
|
59
107
|
const placeholders = ids.map(() => "?").join(",");
|
|
60
108
|
const rows = db
|
|
61
109
|
.prepare(`
|
|
62
|
-
SELECT * FROM memories
|
|
110
|
+
SELECT * FROM memories
|
|
63
111
|
WHERE id IN (${placeholders}) AND container_tag = ?
|
|
64
112
|
`)
|
|
65
113
|
.all(...ids, containerTag);
|
|
@@ -69,7 +117,7 @@ export class VectorSearch {
|
|
|
69
117
|
.split(/[\s,]+/)
|
|
70
118
|
.filter((w) => w.length > 1)
|
|
71
119
|
: [];
|
|
72
|
-
|
|
120
|
+
const hydratedResults = rows.map((row) => {
|
|
73
121
|
const scores = scoreMap.get(row.id);
|
|
74
122
|
const memoryTagsStr = row.tags || "";
|
|
75
123
|
const memoryTags = memoryTagsStr.split(",").map((t) => t.trim().toLowerCase());
|
|
@@ -96,6 +144,8 @@ export class VectorSearch {
|
|
|
96
144
|
isPinned: row.is_pinned,
|
|
97
145
|
};
|
|
98
146
|
});
|
|
147
|
+
hydratedResults.sort((a, b) => b.similarity - a.similarity);
|
|
148
|
+
return hydratedResults;
|
|
99
149
|
}
|
|
100
150
|
async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText) {
|
|
101
151
|
const shardPromises = shards.map(async (shard) => {
|
|
@@ -115,27 +165,27 @@ export class VectorSearch {
|
|
|
115
165
|
async deleteVector(db, memoryId, shard) {
|
|
116
166
|
db.prepare(`DELETE FROM memories WHERE id = ?`).run(memoryId);
|
|
117
167
|
if (shard) {
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
await
|
|
168
|
+
const backend = await this.getBackend();
|
|
169
|
+
await backend.delete({ id: memoryId, shard, kind: "content" });
|
|
170
|
+
await backend.delete({ id: memoryId, shard, kind: "tags" });
|
|
121
171
|
}
|
|
122
172
|
}
|
|
123
173
|
async updateVector(db, memoryId, vector, shard, tagsVector) {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
const contentIndex = hnswIndexManager.getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
|
|
129
|
-
await contentIndex.insert(memoryId, vector);
|
|
174
|
+
db.prepare(`UPDATE memories SET vector = ?, tags_vector = ? WHERE id = ?`).run(toBlob(vector), toBlob(tagsVector), memoryId);
|
|
175
|
+
if (shard) {
|
|
176
|
+
const backend = await this.getBackend();
|
|
177
|
+
await backend.insert({ id: memoryId, vector, shard, kind: "content" });
|
|
130
178
|
if (tagsVector) {
|
|
131
|
-
|
|
132
|
-
|
|
179
|
+
await backend.insert({ id: memoryId, vector: tagsVector, shard, kind: "tags" });
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
await backend.delete({ id: memoryId, shard, kind: "tags" });
|
|
133
183
|
}
|
|
134
184
|
}
|
|
135
185
|
}
|
|
136
186
|
listMemories(db, containerTag, limit) {
|
|
137
187
|
const stmt = db.prepare(`
|
|
138
|
-
SELECT * FROM memories
|
|
188
|
+
SELECT * FROM memories
|
|
139
189
|
WHERE container_tag = ?
|
|
140
190
|
ORDER BY created_at DESC
|
|
141
191
|
LIMIT ?
|
|
@@ -152,7 +202,7 @@ export class VectorSearch {
|
|
|
152
202
|
}
|
|
153
203
|
getMemoriesBySessionID(db, sessionID) {
|
|
154
204
|
const stmt = db.prepare(`
|
|
155
|
-
SELECT * FROM memories
|
|
205
|
+
SELECT * FROM memories
|
|
156
206
|
WHERE metadata LIKE ?
|
|
157
207
|
ORDER BY created_at DESC
|
|
158
208
|
`);
|
|
@@ -175,7 +225,7 @@ export class VectorSearch {
|
|
|
175
225
|
}
|
|
176
226
|
getDistinctTags(db) {
|
|
177
227
|
const stmt = db.prepare(`
|
|
178
|
-
SELECT DISTINCT
|
|
228
|
+
SELECT DISTINCT
|
|
179
229
|
container_tag,
|
|
180
230
|
display_name,
|
|
181
231
|
user_name,
|
|
@@ -195,11 +245,24 @@ export class VectorSearch {
|
|
|
195
245
|
const stmt = db.prepare(`UPDATE memories SET is_pinned = 0 WHERE id = ?`);
|
|
196
246
|
stmt.run(memoryId);
|
|
197
247
|
}
|
|
198
|
-
async
|
|
199
|
-
await
|
|
248
|
+
async rebuildIndexForShard(db, scope, scopeHash, shardIndex) {
|
|
249
|
+
const backend = await this.getBackend();
|
|
250
|
+
const shard = {
|
|
251
|
+
id: 0,
|
|
252
|
+
scope: scope,
|
|
253
|
+
scopeHash,
|
|
254
|
+
shardIndex,
|
|
255
|
+
dbPath: "",
|
|
256
|
+
vectorCount: 0,
|
|
257
|
+
isActive: true,
|
|
258
|
+
createdAt: Date.now(),
|
|
259
|
+
};
|
|
260
|
+
await backend.rebuildFromShard({ db, shard, kind: "content" });
|
|
261
|
+
await backend.rebuildFromShard({ db, shard, kind: "tags" });
|
|
200
262
|
}
|
|
201
|
-
|
|
202
|
-
|
|
263
|
+
async deleteShardIndexes(shard) {
|
|
264
|
+
const backend = await this.getBackend();
|
|
265
|
+
await backend.deleteShardIndexes({ shard });
|
|
203
266
|
}
|
|
204
267
|
}
|
|
205
268
|
export const vectorSearch = new VectorSearch();
|
|
@@ -114,13 +114,8 @@ async function analyzeUserProfile(context, existingProfile) {
|
|
|
114
114
|
throw new Error("External API not configured for user memory learning");
|
|
115
115
|
}
|
|
116
116
|
const { AIProviderFactory } = await import("./ai/ai-provider-factory.js");
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
apiUrl: CONFIG.memoryApiUrl,
|
|
120
|
-
apiKey: CONFIG.memoryApiKey,
|
|
121
|
-
maxIterations: CONFIG.autoCaptureMaxIterations,
|
|
122
|
-
iterationTimeout: CONFIG.autoCaptureIterationTimeout,
|
|
123
|
-
};
|
|
117
|
+
const { buildMemoryProviderConfig } = await import("./ai/provider-config.js");
|
|
118
|
+
const providerConfig = buildMemoryProviderConfig(CONFIG);
|
|
124
119
|
const provider = AIProviderFactory.createProvider(CONFIG.memoryProvider, providerConfig);
|
|
125
120
|
const systemPrompt = `You are a user behavior analyst for a coding assistant.
|
|
126
121
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backend-factory.d.ts","sourceRoot":"","sources":["../../../src/services/vector-backends/backend-factory.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,2BAA2B,EAAE,MAAM,YAAY,CAAC;AA4E7E,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,aAAa,CAAC,CAsCxB"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { CONFIG } from "../../config.js";
|
|
2
|
+
import { log } from "../logger.js";
|
|
3
|
+
import { ExactScanBackend } from "./exact-scan-backend.js";
|
|
4
|
+
import { USearchBackend } from "./usearch-backend.js";
|
|
5
|
+
class FallbackAwareBackend {
|
|
6
|
+
strategy;
|
|
7
|
+
primary;
|
|
8
|
+
fallback;
|
|
9
|
+
activeBackend;
|
|
10
|
+
constructor(strategy, primary, fallback) {
|
|
11
|
+
this.strategy = strategy;
|
|
12
|
+
this.primary = primary;
|
|
13
|
+
this.fallback = fallback;
|
|
14
|
+
this.activeBackend = primary;
|
|
15
|
+
}
|
|
16
|
+
getBackendName() {
|
|
17
|
+
return this.activeBackend.getBackendName();
|
|
18
|
+
}
|
|
19
|
+
async insert(args) {
|
|
20
|
+
await this.activeBackend.insert(args);
|
|
21
|
+
}
|
|
22
|
+
async insertBatch(args) {
|
|
23
|
+
await this.activeBackend.insertBatch(args);
|
|
24
|
+
}
|
|
25
|
+
async delete(args) {
|
|
26
|
+
await this.activeBackend.delete(args);
|
|
27
|
+
}
|
|
28
|
+
async search(args) {
|
|
29
|
+
try {
|
|
30
|
+
return await this.activeBackend.search(args);
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
this.logDegrade("search", error);
|
|
34
|
+
this.activeBackend = this.fallback;
|
|
35
|
+
return this.fallback.search(args);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async rebuildFromShard(args) {
|
|
39
|
+
try {
|
|
40
|
+
await this.activeBackend.rebuildFromShard(args);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
this.logDegrade("rebuild", error);
|
|
44
|
+
this.activeBackend = this.fallback;
|
|
45
|
+
await this.fallback.rebuildFromShard(args);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async deleteShardIndexes(args) {
|
|
49
|
+
await this.primary.deleteShardIndexes(args);
|
|
50
|
+
await this.fallback.deleteShardIndexes(args);
|
|
51
|
+
}
|
|
52
|
+
logDegrade(operation, error) {
|
|
53
|
+
log("Vector backend degraded to exact-scan", {
|
|
54
|
+
strategy: this.strategy,
|
|
55
|
+
severity: this.strategy === "usearch" ? "warning" : "info",
|
|
56
|
+
operation,
|
|
57
|
+
error: String(error),
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async function defaultUSearchProbe() {
|
|
62
|
+
try {
|
|
63
|
+
await import("usearch");
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export async function createVectorBackend(options) {
|
|
71
|
+
const exactScanBackend = new ExactScanBackend();
|
|
72
|
+
if (options.vectorBackend === "exact-scan") {
|
|
73
|
+
return exactScanBackend;
|
|
74
|
+
}
|
|
75
|
+
const probeUSearch = options.probeUSearch ?? defaultUSearchProbe;
|
|
76
|
+
if (!(await probeUSearch())) {
|
|
77
|
+
if (options.vectorBackend === "usearch") {
|
|
78
|
+
log("Vector backend degraded to exact-scan", {
|
|
79
|
+
strategy: "usearch",
|
|
80
|
+
severity: "warning",
|
|
81
|
+
operation: "probe",
|
|
82
|
+
error: "USearch unavailable",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return exactScanBackend;
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
const usearchBackend = options.createUSearchBackend?.() ??
|
|
89
|
+
new USearchBackend({
|
|
90
|
+
baseDir: CONFIG.storagePath,
|
|
91
|
+
dimensions: CONFIG.embeddingDimensions,
|
|
92
|
+
});
|
|
93
|
+
return new FallbackAwareBackend(options.vectorBackend, usearchBackend, exactScanBackend);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
log("Vector backend degraded to exact-scan", {
|
|
97
|
+
strategy: options.vectorBackend,
|
|
98
|
+
severity: options.vectorBackend === "usearch" ? "warning" : "info",
|
|
99
|
+
operation: "create",
|
|
100
|
+
error: String(error),
|
|
101
|
+
});
|
|
102
|
+
return exactScanBackend;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { BackendInsertItem, BackendSearchResult, VectorBackend, VectorBackendSearchParams, VectorKind } from "./types.js";
|
|
2
|
+
import type { ShardInfo } from "../sqlite/types.js";
|
|
3
|
+
interface RankedRow {
|
|
4
|
+
id: string;
|
|
5
|
+
vector: Float32Array;
|
|
6
|
+
}
|
|
7
|
+
export declare class ExactScanBackend implements VectorBackend {
|
|
8
|
+
getBackendName(): string;
|
|
9
|
+
rankVectors(rows: RankedRow[], queryVector: Float32Array, limit: number): BackendSearchResult[];
|
|
10
|
+
insert(_args: {
|
|
11
|
+
id: string;
|
|
12
|
+
vector: Float32Array;
|
|
13
|
+
shard: ShardInfo;
|
|
14
|
+
kind: VectorKind;
|
|
15
|
+
}): Promise<void>;
|
|
16
|
+
insertBatch(_args: {
|
|
17
|
+
items: BackendInsertItem[];
|
|
18
|
+
shard: ShardInfo;
|
|
19
|
+
kind: VectorKind;
|
|
20
|
+
}): Promise<void>;
|
|
21
|
+
delete(_args: {
|
|
22
|
+
id: string;
|
|
23
|
+
shard: ShardInfo;
|
|
24
|
+
kind: VectorKind;
|
|
25
|
+
}): Promise<void>;
|
|
26
|
+
search(args: VectorBackendSearchParams): Promise<BackendSearchResult[]>;
|
|
27
|
+
rebuildFromShard(_args: {
|
|
28
|
+
db: unknown;
|
|
29
|
+
shard: ShardInfo;
|
|
30
|
+
kind: VectorKind;
|
|
31
|
+
}): Promise<void>;
|
|
32
|
+
deleteShardIndexes(_args: {
|
|
33
|
+
shard: ShardInfo;
|
|
34
|
+
}): Promise<void>;
|
|
35
|
+
private decodeVector;
|
|
36
|
+
private cosineSimilarity;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
39
|
+
//# sourceMappingURL=exact-scan-backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exact-scan-backend.d.ts","sourceRoot":"","sources":["../../../src/services/vector-backends/exact-scan-backend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,aAAa,EACb,yBAAyB,EACzB,UAAU,EACX,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD,UAAU,SAAS;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAQD,qBAAa,gBAAiB,YAAW,aAAa;IACpD,cAAc,IAAI,MAAM;IAIxB,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,mBAAmB,EAAE;IAUzF,MAAM,CAAC,KAAK,EAAE;QAClB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,YAAY,CAAC;QACrB,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAEX,WAAW,CAAC,KAAK,EAAE;QACvB,KAAK,EAAE,iBAAiB,EAAE,CAAC;QAC3B,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAEX,MAAM,CAAC,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEhF,MAAM,CAAC,IAAI,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAwBvE,gBAAgB,CAAC,KAAK,EAAE;QAC5B,EAAE,EAAE,OAAO,CAAC;QACZ,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAEX,kBAAkB,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEpE,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,gBAAgB;CAuBzB"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export class ExactScanBackend {
|
|
2
|
+
getBackendName() {
|
|
3
|
+
return "exact-scan";
|
|
4
|
+
}
|
|
5
|
+
rankVectors(rows, queryVector, limit) {
|
|
6
|
+
return rows
|
|
7
|
+
.map((row) => ({
|
|
8
|
+
id: row.id,
|
|
9
|
+
distance: 1 - this.cosineSimilarity(row.vector, queryVector),
|
|
10
|
+
}))
|
|
11
|
+
.sort((a, b) => a.distance - b.distance)
|
|
12
|
+
.slice(0, limit);
|
|
13
|
+
}
|
|
14
|
+
async insert(_args) { }
|
|
15
|
+
async insertBatch(_args) { }
|
|
16
|
+
async delete(_args) { }
|
|
17
|
+
async search(args) {
|
|
18
|
+
const column = args.kind === "tags" ? "tags_vector" : "vector";
|
|
19
|
+
const rows = args.db
|
|
20
|
+
.prepare(`SELECT id, ${column} FROM memories WHERE ${column} IS NOT NULL`)
|
|
21
|
+
.all();
|
|
22
|
+
if (rows.length === 0) {
|
|
23
|
+
return [];
|
|
24
|
+
}
|
|
25
|
+
const rankedRows = rows
|
|
26
|
+
.map((row) => ({
|
|
27
|
+
id: row.id,
|
|
28
|
+
vector: this.decodeVector(args.kind === "tags" ? row.tags_vector : row.vector),
|
|
29
|
+
}))
|
|
30
|
+
.filter((row) => row.vector.length > 0);
|
|
31
|
+
return this.rankVectors(rankedRows, args.queryVector, args.limit);
|
|
32
|
+
}
|
|
33
|
+
async rebuildFromShard(_args) { }
|
|
34
|
+
async deleteShardIndexes(_args) { }
|
|
35
|
+
decodeVector(value) {
|
|
36
|
+
if (!value) {
|
|
37
|
+
return new Float32Array();
|
|
38
|
+
}
|
|
39
|
+
if (value instanceof Uint8Array) {
|
|
40
|
+
return new Float32Array(value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength));
|
|
41
|
+
}
|
|
42
|
+
return new Float32Array(value);
|
|
43
|
+
}
|
|
44
|
+
cosineSimilarity(a, b) {
|
|
45
|
+
if (a.length !== b.length) {
|
|
46
|
+
return 0;
|
|
47
|
+
}
|
|
48
|
+
let dot = 0;
|
|
49
|
+
let magA = 0;
|
|
50
|
+
let magB = 0;
|
|
51
|
+
for (let i = 0; i < a.length; i++) {
|
|
52
|
+
const av = a[i] ?? 0;
|
|
53
|
+
const bv = b[i] ?? 0;
|
|
54
|
+
dot += av * bv;
|
|
55
|
+
magA += av * av;
|
|
56
|
+
magB += bv * bv;
|
|
57
|
+
}
|
|
58
|
+
if (magA === 0 || magB === 0) {
|
|
59
|
+
return 0;
|
|
60
|
+
}
|
|
61
|
+
return dot / (Math.sqrt(magA) * Math.sqrt(magB));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { ShardInfo } from "../sqlite/types.js";
|
|
2
|
+
export type VectorKind = "content" | "tags";
|
|
3
|
+
export interface BackendSearchResult {
|
|
4
|
+
id: string;
|
|
5
|
+
distance: number;
|
|
6
|
+
}
|
|
7
|
+
export interface BackendInsertItem {
|
|
8
|
+
id: string;
|
|
9
|
+
vector: Float32Array;
|
|
10
|
+
}
|
|
11
|
+
export interface VectorBackendSearchParams {
|
|
12
|
+
db: unknown;
|
|
13
|
+
shard: ShardInfo;
|
|
14
|
+
kind: VectorKind;
|
|
15
|
+
queryVector: Float32Array;
|
|
16
|
+
limit: number;
|
|
17
|
+
}
|
|
18
|
+
export interface VectorBackend {
|
|
19
|
+
getBackendName(): string;
|
|
20
|
+
insert(args: {
|
|
21
|
+
id: string;
|
|
22
|
+
vector: Float32Array;
|
|
23
|
+
shard: ShardInfo;
|
|
24
|
+
kind: VectorKind;
|
|
25
|
+
}): Promise<void>;
|
|
26
|
+
insertBatch(args: {
|
|
27
|
+
items: BackendInsertItem[];
|
|
28
|
+
shard: ShardInfo;
|
|
29
|
+
kind: VectorKind;
|
|
30
|
+
}): Promise<void>;
|
|
31
|
+
delete(args: {
|
|
32
|
+
id: string;
|
|
33
|
+
shard: ShardInfo;
|
|
34
|
+
kind: VectorKind;
|
|
35
|
+
}): Promise<void>;
|
|
36
|
+
search(args: VectorBackendSearchParams): Promise<BackendSearchResult[]>;
|
|
37
|
+
rebuildFromShard(args: {
|
|
38
|
+
db: unknown;
|
|
39
|
+
shard: ShardInfo;
|
|
40
|
+
kind: VectorKind;
|
|
41
|
+
}): Promise<void>;
|
|
42
|
+
deleteShardIndexes(args: {
|
|
43
|
+
shard: ShardInfo;
|
|
44
|
+
}): Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
export interface VectorBackendFactoryOptions {
|
|
47
|
+
vectorBackend: "usearch-first" | "usearch" | "exact-scan";
|
|
48
|
+
probeUSearch?: () => Promise<boolean>;
|
|
49
|
+
createUSearchBackend?: () => VectorBackend;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/vector-backends/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,MAAM,CAAC;AAE5C,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,UAAU,CAAC;IACjB,WAAW,EAAE,YAAY,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,IAAI,MAAM,CAAC;IACzB,MAAM,CAAC,IAAI,EAAE;QACX,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,YAAY,CAAC;QACrB,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClB,WAAW,CAAC,IAAI,EAAE;QAChB,KAAK,EAAE,iBAAiB,EAAE,CAAC;QAC3B,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClB,MAAM,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChF,MAAM,CAAC,IAAI,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,gBAAgB,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3F,kBAAkB,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,2BAA2B;IAC1C,aAAa,EAAE,eAAe,GAAG,SAAS,GAAG,YAAY,CAAC;IAC1D,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,oBAAoB,CAAC,EAAE,MAAM,aAAa,CAAC;CAC5C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { BackendInsertItem, BackendSearchResult, VectorBackend, VectorBackendSearchParams, VectorKind } from "./types.js";
|
|
2
|
+
import type { ShardInfo } from "../sqlite/types.js";
|
|
3
|
+
export declare class USearchBackend implements VectorBackend {
|
|
4
|
+
private readonly options;
|
|
5
|
+
private readonly indexes;
|
|
6
|
+
constructor(options: {
|
|
7
|
+
baseDir: string;
|
|
8
|
+
dimensions: number;
|
|
9
|
+
});
|
|
10
|
+
getBackendName(): string;
|
|
11
|
+
insert(args: {
|
|
12
|
+
id: string;
|
|
13
|
+
vector: Float32Array;
|
|
14
|
+
shard: ShardInfo;
|
|
15
|
+
kind: VectorKind;
|
|
16
|
+
}): Promise<void>;
|
|
17
|
+
insertBatch(args: {
|
|
18
|
+
items: BackendInsertItem[];
|
|
19
|
+
shard: ShardInfo;
|
|
20
|
+
kind: VectorKind;
|
|
21
|
+
}): Promise<void>;
|
|
22
|
+
delete(args: {
|
|
23
|
+
id: string;
|
|
24
|
+
shard: ShardInfo;
|
|
25
|
+
kind: VectorKind;
|
|
26
|
+
}): Promise<void>;
|
|
27
|
+
search(args: VectorBackendSearchParams): Promise<BackendSearchResult[]>;
|
|
28
|
+
rebuildFromShard(args: {
|
|
29
|
+
db: unknown;
|
|
30
|
+
shard: ShardInfo;
|
|
31
|
+
kind: VectorKind;
|
|
32
|
+
}): Promise<void>;
|
|
33
|
+
deleteShardIndexes(args: {
|
|
34
|
+
shard: ShardInfo;
|
|
35
|
+
}): Promise<void>;
|
|
36
|
+
insertManyForTest(indexKey: string, items: BackendInsertItem[]): Promise<void>;
|
|
37
|
+
searchForTest(indexKey: string, queryVector: Float32Array, limit: number): Promise<BackendSearchResult[]>;
|
|
38
|
+
private getOrCreateIndex;
|
|
39
|
+
private createEmptyIndex;
|
|
40
|
+
private ensureKey;
|
|
41
|
+
private addItems;
|
|
42
|
+
private upsertItem;
|
|
43
|
+
private decodeVector;
|
|
44
|
+
private getIndexKey;
|
|
45
|
+
private loadUSearch;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=usearch-backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"usearch-backend.d.ts","sourceRoot":"","sources":["../../../src/services/vector-backends/usearch-backend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,aAAa,EACb,yBAAyB,EACzB,UAAU,EACX,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAcpD,qBAAa,cAAe,YAAW,aAAa;IAIhD,OAAO,CAAC,QAAQ,CAAC,OAAO;IAH1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;gBAGvC,OAAO,EAAE;QACxB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB;IAKH,cAAc,IAAI,MAAM;IAIlB,MAAM,CAAC,IAAI,EAAE;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,YAAY,CAAC;QACrB,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAWX,WAAW,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,iBAAiB,EAAE,CAAC;QAC3B,KAAK,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,UAAU,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IAWX,MAAM,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/E,MAAM,CAAC,IAAI,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAsBvE,gBAAgB,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC1F,kBAAkB,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7D,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9E,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,YAAY,EACzB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAqBnB,gBAAgB;YAShB,gBAAgB;IAY9B,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,WAAW;YAIL,WAAW;CAO1B"}
|