@xiaoxiamimengfb/my-opencode-mem 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +155 -0
- package/dist/config.d.ts +58 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +411 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +427 -0
- package/dist/plugin.d.ts +5 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +4 -0
- package/dist/services/ai/ai-provider-factory.d.ts +8 -0
- package/dist/services/ai/ai-provider-factory.d.ts.map +1 -0
- package/dist/services/ai/ai-provider-factory.js +28 -0
- package/dist/services/ai/opencode-provider.d.ts +30 -0
- package/dist/services/ai/opencode-provider.d.ts.map +1 -0
- package/dist/services/ai/opencode-provider.js +332 -0
- package/dist/services/ai/provider-config.d.ts +17 -0
- package/dist/services/ai/provider-config.d.ts.map +1 -0
- package/dist/services/ai/provider-config.js +14 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts +12 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts.map +1 -0
- package/dist/services/ai/providers/anthropic-messages.js +184 -0
- package/dist/services/ai/providers/base-provider.d.ts +25 -0
- package/dist/services/ai/providers/base-provider.d.ts.map +1 -0
- package/dist/services/ai/providers/base-provider.js +23 -0
- package/dist/services/ai/providers/google-gemini.d.ts +16 -0
- package/dist/services/ai/providers/google-gemini.d.ts.map +1 -0
- package/dist/services/ai/providers/google-gemini.js +228 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts +13 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts.map +1 -0
- package/dist/services/ai/providers/openai-chat-completion.js +277 -0
- package/dist/services/ai/providers/openai-responses.d.ts +14 -0
- package/dist/services/ai/providers/openai-responses.d.ts.map +1 -0
- package/dist/services/ai/providers/openai-responses.js +182 -0
- package/dist/services/ai/session/ai-session-manager.d.ts +21 -0
- package/dist/services/ai/session/ai-session-manager.d.ts.map +1 -0
- package/dist/services/ai/session/ai-session-manager.js +166 -0
- package/dist/services/ai/session/session-types.d.ts +43 -0
- package/dist/services/ai/session/session-types.d.ts.map +1 -0
- package/dist/services/ai/session/session-types.js +1 -0
- package/dist/services/ai/tools/tool-schema.d.ts +41 -0
- package/dist/services/ai/tools/tool-schema.d.ts.map +1 -0
- package/dist/services/ai/tools/tool-schema.js +24 -0
- package/dist/services/ai/validators/user-profile-validator.d.ts +13 -0
- package/dist/services/ai/validators/user-profile-validator.d.ts.map +1 -0
- package/dist/services/ai/validators/user-profile-validator.js +111 -0
- package/dist/services/api-handlers.d.ts +164 -0
- package/dist/services/api-handlers.d.ts.map +1 -0
- package/dist/services/api-handlers.js +901 -0
- package/dist/services/auto-capture.d.ts +3 -0
- package/dist/services/auto-capture.d.ts.map +1 -0
- package/dist/services/auto-capture.js +306 -0
- package/dist/services/cleanup-service.d.ts +23 -0
- package/dist/services/cleanup-service.d.ts.map +1 -0
- package/dist/services/cleanup-service.js +102 -0
- package/dist/services/client.d.ts +118 -0
- package/dist/services/client.d.ts.map +1 -0
- package/dist/services/client.js +251 -0
- package/dist/services/context.d.ts +11 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +24 -0
- package/dist/services/deduplication-service.d.ts +30 -0
- package/dist/services/deduplication-service.d.ts.map +1 -0
- package/dist/services/deduplication-service.js +124 -0
- package/dist/services/embedding.d.ts +15 -0
- package/dist/services/embedding.d.ts.map +1 -0
- package/dist/services/embedding.js +106 -0
- package/dist/services/jsonc.d.ts +7 -0
- package/dist/services/jsonc.d.ts.map +1 -0
- package/dist/services/jsonc.js +76 -0
- package/dist/services/language-detector.d.ts +3 -0
- package/dist/services/language-detector.d.ts.map +1 -0
- package/dist/services/language-detector.js +16 -0
- package/dist/services/logger.d.ts +2 -0
- package/dist/services/logger.d.ts.map +1 -0
- package/dist/services/logger.js +51 -0
- package/dist/services/migration-service.d.ts +42 -0
- package/dist/services/migration-service.d.ts.map +1 -0
- package/dist/services/migration-service.js +250 -0
- package/dist/services/privacy.d.ts +3 -0
- package/dist/services/privacy.d.ts.map +1 -0
- package/dist/services/privacy.js +7 -0
- package/dist/services/secret-resolver.d.ts +2 -0
- package/dist/services/secret-resolver.d.ts.map +1 -0
- package/dist/services/secret-resolver.js +55 -0
- package/dist/services/sqlite/connection-manager.d.ts +13 -0
- package/dist/services/sqlite/connection-manager.d.ts.map +1 -0
- package/dist/services/sqlite/connection-manager.js +74 -0
- package/dist/services/sqlite/shard-manager.d.ts +23 -0
- package/dist/services/sqlite/shard-manager.d.ts.map +1 -0
- package/dist/services/sqlite/shard-manager.js +288 -0
- package/dist/services/sqlite/sqlite-bootstrap.d.ts +2 -0
- package/dist/services/sqlite/sqlite-bootstrap.d.ts.map +1 -0
- package/dist/services/sqlite/sqlite-bootstrap.js +8 -0
- package/dist/services/sqlite/types.d.ts +42 -0
- package/dist/services/sqlite/types.d.ts.map +1 -0
- package/dist/services/sqlite/types.js +1 -0
- package/dist/services/sqlite/vector-search.d.ts +29 -0
- package/dist/services/sqlite/vector-search.d.ts.map +1 -0
- package/dist/services/sqlite/vector-search.js +268 -0
- package/dist/services/tags.d.ts +24 -0
- package/dist/services/tags.d.ts.map +1 -0
- package/dist/services/tags.js +146 -0
- package/dist/services/user-memory-learning.d.ts +3 -0
- package/dist/services/user-memory-learning.d.ts.map +1 -0
- package/dist/services/user-memory-learning.js +231 -0
- package/dist/services/user-profile/profile-context.d.ts +2 -0
- package/dist/services/user-profile/profile-context.d.ts.map +1 -0
- package/dist/services/user-profile/profile-context.js +40 -0
- package/dist/services/user-profile/profile-utils.d.ts +3 -0
- package/dist/services/user-profile/profile-utils.d.ts.map +1 -0
- package/dist/services/user-profile/profile-utils.js +45 -0
- package/dist/services/user-profile/types.d.ts +46 -0
- package/dist/services/user-profile/types.d.ts.map +1 -0
- package/dist/services/user-profile/types.js +1 -0
- package/dist/services/user-profile/user-profile-manager.d.ts +23 -0
- package/dist/services/user-profile/user-profile-manager.d.ts.map +1 -0
- package/dist/services/user-profile/user-profile-manager.js +292 -0
- package/dist/services/user-prompt/user-prompt-manager.d.ts +41 -0
- package/dist/services/user-prompt/user-prompt-manager.d.ts.map +1 -0
- package/dist/services/user-prompt/user-prompt-manager.js +192 -0
- package/dist/services/vector-backends/backend-factory.d.ts +3 -0
- package/dist/services/vector-backends/backend-factory.d.ts.map +1 -0
- package/dist/services/vector-backends/backend-factory.js +104 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts +39 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/exact-scan-backend.js +63 -0
- package/dist/services/vector-backends/types.d.ts +51 -0
- package/dist/services/vector-backends/types.d.ts.map +1 -0
- package/dist/services/vector-backends/types.js +1 -0
- package/dist/services/vector-backends/usearch-backend.d.ts +47 -0
- package/dist/services/vector-backends/usearch-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/usearch-backend.js +174 -0
- package/dist/services/web-server-worker.d.ts +2 -0
- package/dist/services/web-server-worker.d.ts.map +1 -0
- package/dist/services/web-server-worker.js +283 -0
- package/dist/services/web-server.d.ts +31 -0
- package/dist/services/web-server.d.ts.map +1 -0
- package/dist/services/web-server.js +356 -0
- package/dist/types/index.d.ts +19 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/web/app.d.ts +2 -0
- package/dist/web/app.d.ts.map +1 -0
- package/dist/web/app.js +1194 -0
- package/dist/web/favicon.ico +0 -0
- package/dist/web/i18n.d.ts +2 -0
- package/dist/web/i18n.d.ts.map +1 -0
- package/dist/web/i18n.js +265 -0
- package/dist/web/index.html +284 -0
- package/dist/web/styles.css +1631 -0
- package/package.json +71 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { embeddingService } from "./embedding.js";
|
|
2
|
+
import { shardManager } from "./sqlite/shard-manager.js";
|
|
3
|
+
import { vectorSearch } from "./sqlite/vector-search.js";
|
|
4
|
+
import { connectionManager } from "./sqlite/connection-manager.js";
|
|
5
|
+
import { CONFIG } from "../config.js";
|
|
6
|
+
import { log } from "./logger.js";
|
|
7
|
+
function safeToISOString(timestamp) {
|
|
8
|
+
try {
|
|
9
|
+
if (timestamp === null || timestamp === undefined) {
|
|
10
|
+
return new Date().toISOString();
|
|
11
|
+
}
|
|
12
|
+
const numValue = typeof timestamp === "bigint" ? Number(timestamp) : Number(timestamp);
|
|
13
|
+
if (isNaN(numValue) || numValue < 0) {
|
|
14
|
+
return new Date().toISOString();
|
|
15
|
+
}
|
|
16
|
+
return new Date(numValue).toISOString();
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return new Date().toISOString();
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function safeJSONParse(jsonString) {
|
|
23
|
+
if (!jsonString || typeof jsonString !== "string") {
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
return JSON.parse(jsonString);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function extractScopeFromContainerTag(containerTag) {
|
|
34
|
+
const parts = containerTag.split("_");
|
|
35
|
+
if (parts.length >= 3) {
|
|
36
|
+
const scope = parts[1];
|
|
37
|
+
const hash = parts.slice(2).join("_");
|
|
38
|
+
return { scope, hash };
|
|
39
|
+
}
|
|
40
|
+
return { scope: "user", hash: containerTag };
|
|
41
|
+
}
|
|
42
|
+
export class LocalMemoryClient {
|
|
43
|
+
initPromise = null;
|
|
44
|
+
isInitialized = false;
|
|
45
|
+
constructor() { }
|
|
46
|
+
async initialize() {
|
|
47
|
+
if (this.isInitialized)
|
|
48
|
+
return;
|
|
49
|
+
if (this.initPromise)
|
|
50
|
+
return this.initPromise;
|
|
51
|
+
this.initPromise = (async () => {
|
|
52
|
+
try {
|
|
53
|
+
this.isInitialized = true;
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
this.initPromise = null;
|
|
57
|
+
log("SQLite initialization failed", { error: String(error) });
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
})();
|
|
61
|
+
return this.initPromise;
|
|
62
|
+
}
|
|
63
|
+
async warmup(progressCallback) {
|
|
64
|
+
await this.initialize();
|
|
65
|
+
await embeddingService.warmup(progressCallback);
|
|
66
|
+
}
|
|
67
|
+
async isReady() {
|
|
68
|
+
return this.isInitialized && embeddingService.isWarmedUp;
|
|
69
|
+
}
|
|
70
|
+
getStatus() {
|
|
71
|
+
return {
|
|
72
|
+
dbConnected: this.isInitialized,
|
|
73
|
+
modelLoaded: embeddingService.isWarmedUp,
|
|
74
|
+
ready: this.isInitialized && embeddingService.isWarmedUp,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
close() {
|
|
78
|
+
connectionManager.closeAll();
|
|
79
|
+
}
|
|
80
|
+
async searchMemories(query, containerTag) {
|
|
81
|
+
try {
|
|
82
|
+
await this.initialize();
|
|
83
|
+
const queryVector = await embeddingService.embedWithTimeout(query);
|
|
84
|
+
const { scope, hash } = extractScopeFromContainerTag(containerTag);
|
|
85
|
+
const shards = shardManager.getAllShards(scope, hash);
|
|
86
|
+
if (shards.length === 0) {
|
|
87
|
+
return { success: true, results: [], total: 0, timing: 0 };
|
|
88
|
+
}
|
|
89
|
+
const results = await vectorSearch.searchAcrossShards(shards, queryVector, containerTag, CONFIG.maxMemories, CONFIG.similarityThreshold, query);
|
|
90
|
+
return { success: true, results, total: results.length, timing: 0 };
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
94
|
+
log("searchMemories: error", { error: errorMessage });
|
|
95
|
+
return { success: false, error: errorMessage, results: [], total: 0, timing: 0 };
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async addMemory(content, containerTag, metadata) {
|
|
99
|
+
try {
|
|
100
|
+
await this.initialize();
|
|
101
|
+
const tags = metadata?.tags || [];
|
|
102
|
+
const vector = await embeddingService.embedWithTimeout(content);
|
|
103
|
+
let tagsVector = undefined;
|
|
104
|
+
if (tags.length > 0) {
|
|
105
|
+
tagsVector = await embeddingService.embedWithTimeout(tags.join(", "));
|
|
106
|
+
}
|
|
107
|
+
const { scope, hash } = extractScopeFromContainerTag(containerTag);
|
|
108
|
+
const shard = shardManager.getWriteShard(scope, hash);
|
|
109
|
+
const id = `mem_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
110
|
+
const now = Date.now();
|
|
111
|
+
const { displayName, userName, userEmail, projectPath, projectName, gitRepoUrl, type, tags: _tags, ...dynamicMetadata } = metadata || {};
|
|
112
|
+
const record = {
|
|
113
|
+
id,
|
|
114
|
+
content,
|
|
115
|
+
vector,
|
|
116
|
+
tagsVector,
|
|
117
|
+
containerTag,
|
|
118
|
+
tags: tags.length > 0 ? tags.join(",") : undefined,
|
|
119
|
+
type,
|
|
120
|
+
createdAt: now,
|
|
121
|
+
updatedAt: now,
|
|
122
|
+
displayName,
|
|
123
|
+
userName,
|
|
124
|
+
userEmail,
|
|
125
|
+
projectPath,
|
|
126
|
+
projectName,
|
|
127
|
+
gitRepoUrl,
|
|
128
|
+
metadata: Object.keys(dynamicMetadata).length > 0 ? JSON.stringify(dynamicMetadata) : undefined,
|
|
129
|
+
};
|
|
130
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
131
|
+
await vectorSearch.insertVector(db, record, shard);
|
|
132
|
+
shardManager.incrementVectorCount(shard.id);
|
|
133
|
+
return { success: true, id };
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
137
|
+
log("addMemory: error", { error: errorMessage });
|
|
138
|
+
return { success: false, error: errorMessage };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
async deleteMemory(memoryId) {
|
|
142
|
+
try {
|
|
143
|
+
await this.initialize();
|
|
144
|
+
const userShards = shardManager.getAllShards("user", "");
|
|
145
|
+
const projectShards = shardManager.getAllShards("project", "");
|
|
146
|
+
const allShards = [...userShards, ...projectShards];
|
|
147
|
+
for (const shard of allShards) {
|
|
148
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
149
|
+
const memory = vectorSearch.getMemoryById(db, memoryId);
|
|
150
|
+
if (memory) {
|
|
151
|
+
await vectorSearch.deleteVector(db, memoryId, shard);
|
|
152
|
+
shardManager.decrementVectorCount(shard.id);
|
|
153
|
+
return { success: true };
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return { success: false, error: "Memory not found" };
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
160
|
+
log("deleteMemory: error", { memoryId, error: errorMessage });
|
|
161
|
+
return { success: false, error: errorMessage };
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
async listMemories(containerTag, limit = 20) {
|
|
165
|
+
try {
|
|
166
|
+
await this.initialize();
|
|
167
|
+
const { scope, hash } = extractScopeFromContainerTag(containerTag);
|
|
168
|
+
const shards = shardManager.getAllShards(scope, hash);
|
|
169
|
+
if (shards.length === 0) {
|
|
170
|
+
return {
|
|
171
|
+
success: true,
|
|
172
|
+
memories: [],
|
|
173
|
+
pagination: { currentPage: 1, totalItems: 0, totalPages: 0 },
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
const allMemories = [];
|
|
177
|
+
for (const shard of shards) {
|
|
178
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
179
|
+
const memories = vectorSearch.listMemories(db, containerTag, limit);
|
|
180
|
+
allMemories.push(...memories);
|
|
181
|
+
}
|
|
182
|
+
allMemories.sort((a, b) => Number(b.created_at) - Number(a.created_at));
|
|
183
|
+
const memories = allMemories.slice(0, limit).map((r) => ({
|
|
184
|
+
id: r.id,
|
|
185
|
+
summary: r.content,
|
|
186
|
+
createdAt: safeToISOString(r.created_at),
|
|
187
|
+
metadata: safeJSONParse(r.metadata),
|
|
188
|
+
displayName: r.display_name,
|
|
189
|
+
userName: r.user_name,
|
|
190
|
+
userEmail: r.user_email,
|
|
191
|
+
projectPath: r.project_path,
|
|
192
|
+
projectName: r.project_name,
|
|
193
|
+
gitRepoUrl: r.git_repo_url,
|
|
194
|
+
}));
|
|
195
|
+
return {
|
|
196
|
+
success: true,
|
|
197
|
+
memories,
|
|
198
|
+
pagination: { currentPage: 1, totalItems: memories.length, totalPages: 1 },
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
203
|
+
log("listMemories: error", { error: errorMessage });
|
|
204
|
+
return {
|
|
205
|
+
success: false,
|
|
206
|
+
error: errorMessage,
|
|
207
|
+
memories: [],
|
|
208
|
+
pagination: { currentPage: 1, totalItems: 0, totalPages: 0 },
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
async searchMemoriesBySessionID(sessionID, containerTag, limit = 10) {
|
|
213
|
+
try {
|
|
214
|
+
await this.initialize();
|
|
215
|
+
const { scope, hash } = extractScopeFromContainerTag(containerTag);
|
|
216
|
+
const shards = shardManager.getAllShards(scope, hash);
|
|
217
|
+
if (shards.length === 0) {
|
|
218
|
+
return { success: true, results: [], total: 0, timing: 0 };
|
|
219
|
+
}
|
|
220
|
+
const allMemories = [];
|
|
221
|
+
for (const shard of shards) {
|
|
222
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
223
|
+
const memories = vectorSearch.getMemoriesBySessionID(db, sessionID);
|
|
224
|
+
allMemories.push(...memories);
|
|
225
|
+
}
|
|
226
|
+
allMemories.sort((a, b) => b.created_at - a.created_at);
|
|
227
|
+
const results = allMemories.slice(0, limit).map((row) => ({
|
|
228
|
+
id: row.id,
|
|
229
|
+
memory: row.content,
|
|
230
|
+
similarity: 1.0,
|
|
231
|
+
tags: row.tags || [],
|
|
232
|
+
metadata: row.metadata || {},
|
|
233
|
+
containerTag: row.container_tag,
|
|
234
|
+
displayName: row.display_name,
|
|
235
|
+
userName: row.user_name,
|
|
236
|
+
userEmail: row.user_email,
|
|
237
|
+
projectPath: row.project_path,
|
|
238
|
+
projectName: row.project_name,
|
|
239
|
+
gitRepoUrl: row.git_repo_url,
|
|
240
|
+
createdAt: row.created_at,
|
|
241
|
+
}));
|
|
242
|
+
return { success: true, results, total: results.length, timing: 0 };
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
246
|
+
log("searchMemoriesBySessionID: error", { error: errorMessage });
|
|
247
|
+
return { success: false, error: errorMessage, results: [], total: 0, timing: 0 };
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
export const memoryClient = new LocalMemoryClient();
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
interface MemoryResultMinimal {
|
|
2
|
+
similarity: number;
|
|
3
|
+
memory?: string;
|
|
4
|
+
chunk?: string;
|
|
5
|
+
}
|
|
6
|
+
interface MemoriesResponseMinimal {
|
|
7
|
+
results?: MemoryResultMinimal[];
|
|
8
|
+
}
|
|
9
|
+
export declare function formatContextForPrompt(userId: string | null, projectMemories: MemoriesResponseMinimal): string;
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=context.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/services/context.ts"],"names":[],"mappings":"AAGA,UAAU,mBAAmB;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,uBAAuB;IAC/B,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACjC;AAED,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,eAAe,EAAE,uBAAuB,GACvC,MAAM,CAyBR"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { CONFIG } from "../config.js";
|
|
2
|
+
import { getUserProfileContext } from "./user-profile/profile-context.js";
|
|
3
|
+
export function formatContextForPrompt(userId, projectMemories) {
|
|
4
|
+
const parts = ["[MEMORY]"];
|
|
5
|
+
if (CONFIG.injectProfile && userId) {
|
|
6
|
+
const profileContext = getUserProfileContext(userId);
|
|
7
|
+
if (profileContext) {
|
|
8
|
+
parts.push("\n" + profileContext);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
const projectResults = projectMemories.results || [];
|
|
12
|
+
if (projectResults.length > 0) {
|
|
13
|
+
parts.push("\nProject Knowledge:");
|
|
14
|
+
projectResults.forEach((mem) => {
|
|
15
|
+
const similarity = Math.round(mem.similarity * 100);
|
|
16
|
+
const content = mem.memory || mem.chunk || "";
|
|
17
|
+
parts.push(`- [${similarity}%] ${content}`);
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
if (parts.length === 1) {
|
|
21
|
+
return "";
|
|
22
|
+
}
|
|
23
|
+
return parts.join("\n");
|
|
24
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
interface DuplicateGroup {
|
|
2
|
+
representative: {
|
|
3
|
+
id: string;
|
|
4
|
+
content: string;
|
|
5
|
+
containerTag: string;
|
|
6
|
+
createdAt: number;
|
|
7
|
+
};
|
|
8
|
+
duplicates: Array<{
|
|
9
|
+
id: string;
|
|
10
|
+
content: string;
|
|
11
|
+
similarity: number;
|
|
12
|
+
}>;
|
|
13
|
+
}
|
|
14
|
+
interface DeduplicationResult {
|
|
15
|
+
exactDuplicatesDeleted: number;
|
|
16
|
+
nearDuplicateGroups: DuplicateGroup[];
|
|
17
|
+
}
|
|
18
|
+
export declare class DeduplicationService {
|
|
19
|
+
private isRunning;
|
|
20
|
+
detectAndRemoveDuplicates(): Promise<DeduplicationResult>;
|
|
21
|
+
private cosineSimilarity;
|
|
22
|
+
getStatus(): {
|
|
23
|
+
enabled: boolean;
|
|
24
|
+
threshold: number;
|
|
25
|
+
isRunning: boolean;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export declare const deduplicationService: DeduplicationService;
|
|
29
|
+
export {};
|
|
30
|
+
//# sourceMappingURL=deduplication-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deduplication-service.d.ts","sourceRoot":"","sources":["../../src/services/deduplication-service.ts"],"names":[],"mappings":"AAMA,UAAU,cAAc;IACtB,cAAc,EAAE;QACd,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,EAAE,KAAK,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,UAAU,mBAAmB;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,mBAAmB,EAAE,cAAc,EAAE,CAAC;CACvC;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,SAAS,CAAkB;IAE7B,yBAAyB,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAwG/D,OAAO,CAAC,gBAAgB;IAoBxB,SAAS;;;;;CAOV;AAED,eAAO,MAAM,oBAAoB,sBAA6B,CAAC"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { shardManager } from "./sqlite/shard-manager.js";
|
|
2
|
+
import { vectorSearch } from "./sqlite/vector-search.js";
|
|
3
|
+
import { connectionManager } from "./sqlite/connection-manager.js";
|
|
4
|
+
import { CONFIG } from "../config.js";
|
|
5
|
+
import { log } from "./logger.js";
|
|
6
|
+
export class DeduplicationService {
|
|
7
|
+
isRunning = false;
|
|
8
|
+
async detectAndRemoveDuplicates() {
|
|
9
|
+
if (this.isRunning) {
|
|
10
|
+
throw new Error("Deduplication already running");
|
|
11
|
+
}
|
|
12
|
+
if (!CONFIG.deduplicationEnabled) {
|
|
13
|
+
throw new Error("Deduplication is disabled in config");
|
|
14
|
+
}
|
|
15
|
+
this.isRunning = true;
|
|
16
|
+
try {
|
|
17
|
+
const userShards = shardManager.getAllShards("user", "");
|
|
18
|
+
const projectShards = shardManager.getAllShards("project", "");
|
|
19
|
+
const allShards = [...userShards, ...projectShards];
|
|
20
|
+
let exactDeleted = 0;
|
|
21
|
+
const nearDuplicateGroups = [];
|
|
22
|
+
for (const shard of allShards) {
|
|
23
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
24
|
+
const memories = vectorSearch.getAllMemories(db);
|
|
25
|
+
const contentMap = new Map();
|
|
26
|
+
for (const memory of memories) {
|
|
27
|
+
const key = `${memory.container_tag}:${memory.content}`;
|
|
28
|
+
if (!contentMap.has(key)) {
|
|
29
|
+
contentMap.set(key, []);
|
|
30
|
+
}
|
|
31
|
+
contentMap.get(key).push(memory);
|
|
32
|
+
}
|
|
33
|
+
for (const [, duplicates] of contentMap) {
|
|
34
|
+
if (duplicates.length > 1) {
|
|
35
|
+
duplicates.sort((a, b) => Number(b.created_at) - Number(a.created_at));
|
|
36
|
+
const toDelete = duplicates.slice(1);
|
|
37
|
+
for (const dup of toDelete) {
|
|
38
|
+
try {
|
|
39
|
+
await vectorSearch.deleteVector(db, dup.id, shard);
|
|
40
|
+
shardManager.decrementVectorCount(shard.id);
|
|
41
|
+
exactDeleted++;
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
log("Deduplication: delete error", {
|
|
45
|
+
memoryId: dup.id,
|
|
46
|
+
error: String(error),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const uniqueMemories = Array.from(contentMap.values()).map((arr) => arr[0]);
|
|
53
|
+
const processedIds = new Set();
|
|
54
|
+
for (let i = 0; i < uniqueMemories.length; i++) {
|
|
55
|
+
const mem1 = uniqueMemories[i];
|
|
56
|
+
if (!mem1.vector || processedIds.has(mem1.id))
|
|
57
|
+
continue;
|
|
58
|
+
const vector1 = new Float32Array(new Uint8Array(mem1.vector).buffer);
|
|
59
|
+
const similarGroup = {
|
|
60
|
+
representative: {
|
|
61
|
+
id: mem1.id,
|
|
62
|
+
content: mem1.content,
|
|
63
|
+
containerTag: mem1.container_tag,
|
|
64
|
+
createdAt: mem1.created_at,
|
|
65
|
+
},
|
|
66
|
+
duplicates: [],
|
|
67
|
+
};
|
|
68
|
+
for (let j = i + 1; j < uniqueMemories.length; j++) {
|
|
69
|
+
const mem2 = uniqueMemories[j];
|
|
70
|
+
if (!mem2.vector || processedIds.has(mem2.id))
|
|
71
|
+
continue;
|
|
72
|
+
if (mem1.container_tag !== mem2.container_tag)
|
|
73
|
+
continue;
|
|
74
|
+
const vector2 = new Float32Array(new Uint8Array(mem2.vector).buffer);
|
|
75
|
+
const similarity = this.cosineSimilarity(vector1, vector2);
|
|
76
|
+
if (similarity >= CONFIG.deduplicationSimilarityThreshold && similarity < 1.0) {
|
|
77
|
+
similarGroup.duplicates.push({
|
|
78
|
+
id: mem2.id,
|
|
79
|
+
content: mem2.content,
|
|
80
|
+
similarity,
|
|
81
|
+
});
|
|
82
|
+
processedIds.add(mem2.id);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (similarGroup.duplicates.length > 0) {
|
|
86
|
+
nearDuplicateGroups.push(similarGroup);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
exactDuplicatesDeleted: exactDeleted,
|
|
92
|
+
nearDuplicateGroups,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
finally {
|
|
96
|
+
this.isRunning = false;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
cosineSimilarity(a, b) {
|
|
100
|
+
if (a.length !== b.length)
|
|
101
|
+
return 0;
|
|
102
|
+
let dotProduct = 0;
|
|
103
|
+
let normA = 0;
|
|
104
|
+
let normB = 0;
|
|
105
|
+
for (let i = 0; i < a.length; i++) {
|
|
106
|
+
const aVal = a[i] || 0;
|
|
107
|
+
const bVal = b[i] || 0;
|
|
108
|
+
dotProduct += aVal * bVal;
|
|
109
|
+
normA += aVal * aVal;
|
|
110
|
+
normB += bVal * bVal;
|
|
111
|
+
}
|
|
112
|
+
if (normA === 0 || normB === 0)
|
|
113
|
+
return 0;
|
|
114
|
+
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
115
|
+
}
|
|
116
|
+
getStatus() {
|
|
117
|
+
return {
|
|
118
|
+
enabled: CONFIG.deduplicationEnabled,
|
|
119
|
+
threshold: CONFIG.deduplicationSimilarityThreshold,
|
|
120
|
+
isRunning: this.isRunning,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
export const deduplicationService = new DeduplicationService();
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export declare class EmbeddingService {
|
|
2
|
+
private pipe;
|
|
3
|
+
private initPromise;
|
|
4
|
+
isWarmedUp: boolean;
|
|
5
|
+
private cache;
|
|
6
|
+
private cachedModelName;
|
|
7
|
+
static getInstance(): EmbeddingService;
|
|
8
|
+
warmup(progressCallback?: (progress: any) => void): Promise<void>;
|
|
9
|
+
private initializeModel;
|
|
10
|
+
embed(text: string): Promise<Float32Array>;
|
|
11
|
+
embedWithTimeout(text: string): Promise<Float32Array>;
|
|
12
|
+
clearCache(): void;
|
|
13
|
+
}
|
|
14
|
+
export declare const embeddingService: EmbeddingService;
|
|
15
|
+
//# sourceMappingURL=embedding.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/services/embedding.ts"],"names":[],"mappings":"AAoBA,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,WAAW,CAA8B;IAC1C,UAAU,EAAE,OAAO,CAAS;IACnC,OAAO,CAAC,KAAK,CAAwC;IACrD,OAAO,CAAC,eAAe,CAAuB;IAE9C,MAAM,CAAC,WAAW,IAAI,gBAAgB;IAOhC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YAOzD,eAAe;IAiBvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAmD1C,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAI3D,UAAU,IAAI,IAAI;CAGnB;AAED,eAAO,MAAM,gBAAgB,kBAAiC,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { pipeline, env } from "@xenova/transformers";
|
|
2
|
+
import { CONFIG } from "../config.js";
|
|
3
|
+
import { log } from "./logger.js";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
env.allowLocalModels = true;
|
|
6
|
+
env.allowRemoteModels = true;
|
|
7
|
+
env.cacheDir = join(CONFIG.storagePath, ".cache");
|
|
8
|
+
const TIMEOUT_MS = 30000;
|
|
9
|
+
const GLOBAL_EMBEDDING_KEY = Symbol.for("opencode-mem.embedding.instance");
|
|
10
|
+
const MAX_CACHE_SIZE = 100;
|
|
11
|
+
function withTimeout(promise, ms) {
|
|
12
|
+
return Promise.race([
|
|
13
|
+
promise,
|
|
14
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms)),
|
|
15
|
+
]);
|
|
16
|
+
}
|
|
17
|
+
export class EmbeddingService {
|
|
18
|
+
pipe = null;
|
|
19
|
+
initPromise = null;
|
|
20
|
+
isWarmedUp = false;
|
|
21
|
+
cache = new Map();
|
|
22
|
+
cachedModelName = null;
|
|
23
|
+
static getInstance() {
|
|
24
|
+
if (!globalThis[GLOBAL_EMBEDDING_KEY]) {
|
|
25
|
+
globalThis[GLOBAL_EMBEDDING_KEY] = new EmbeddingService();
|
|
26
|
+
}
|
|
27
|
+
return globalThis[GLOBAL_EMBEDDING_KEY];
|
|
28
|
+
}
|
|
29
|
+
async warmup(progressCallback) {
|
|
30
|
+
if (this.isWarmedUp)
|
|
31
|
+
return;
|
|
32
|
+
if (this.initPromise)
|
|
33
|
+
return this.initPromise;
|
|
34
|
+
this.initPromise = this.initializeModel(progressCallback);
|
|
35
|
+
return this.initPromise;
|
|
36
|
+
}
|
|
37
|
+
async initializeModel(progressCallback) {
|
|
38
|
+
try {
|
|
39
|
+
if (CONFIG.embeddingApiUrl && CONFIG.embeddingApiKey) {
|
|
40
|
+
this.isWarmedUp = true;
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
this.pipe = await pipeline("feature-extraction", CONFIG.embeddingModel, {
|
|
44
|
+
progress_callback: progressCallback,
|
|
45
|
+
});
|
|
46
|
+
this.isWarmedUp = true;
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
this.initPromise = null;
|
|
50
|
+
log("Failed to initialize embedding model", { error: String(error) });
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async embed(text) {
|
|
55
|
+
if (this.cachedModelName !== CONFIG.embeddingModel) {
|
|
56
|
+
this.clearCache();
|
|
57
|
+
this.cachedModelName = CONFIG.embeddingModel;
|
|
58
|
+
}
|
|
59
|
+
const cached = this.cache.get(text);
|
|
60
|
+
if (cached)
|
|
61
|
+
return cached;
|
|
62
|
+
if (!this.isWarmedUp && !this.initPromise) {
|
|
63
|
+
await this.warmup();
|
|
64
|
+
}
|
|
65
|
+
if (this.initPromise) {
|
|
66
|
+
await this.initPromise;
|
|
67
|
+
}
|
|
68
|
+
let result;
|
|
69
|
+
if (CONFIG.embeddingApiUrl && CONFIG.embeddingApiKey) {
|
|
70
|
+
const response = await fetch(`${CONFIG.embeddingApiUrl}/embeddings`, {
|
|
71
|
+
method: "POST",
|
|
72
|
+
headers: {
|
|
73
|
+
"Content-Type": "application/json",
|
|
74
|
+
Authorization: `Bearer ${CONFIG.embeddingApiKey}`,
|
|
75
|
+
},
|
|
76
|
+
body: JSON.stringify({
|
|
77
|
+
input: text,
|
|
78
|
+
model: CONFIG.embeddingModel,
|
|
79
|
+
}),
|
|
80
|
+
});
|
|
81
|
+
if (!response.ok) {
|
|
82
|
+
throw new Error(`API embedding failed: ${response.statusText}`);
|
|
83
|
+
}
|
|
84
|
+
const data = await response.json();
|
|
85
|
+
result = new Float32Array(data.data[0].embedding);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
const output = await this.pipe(text, { pooling: "mean", normalize: true });
|
|
89
|
+
result = new Float32Array(output.data);
|
|
90
|
+
}
|
|
91
|
+
if (this.cache.size >= MAX_CACHE_SIZE) {
|
|
92
|
+
const firstKey = this.cache.keys().next().value;
|
|
93
|
+
if (firstKey !== undefined)
|
|
94
|
+
this.cache.delete(firstKey);
|
|
95
|
+
}
|
|
96
|
+
this.cache.set(text, result);
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
async embedWithTimeout(text) {
|
|
100
|
+
return withTimeout(this.embed(text), TIMEOUT_MS);
|
|
101
|
+
}
|
|
102
|
+
clearCache() {
|
|
103
|
+
this.cache.clear();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
export const embeddingService = EmbeddingService.getInstance();
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strips comments from JSONC content while respecting string boundaries.
|
|
3
|
+
* Handles // and /* comments, URLs in strings, and escaped quotes.
|
|
4
|
+
* Also removes trailing commas to support more relaxed JSONC format.
|
|
5
|
+
*/
|
|
6
|
+
export declare function stripJsoncComments(content: string): string;
|
|
7
|
+
//# sourceMappingURL=jsonc.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonc.d.ts","sourceRoot":"","sources":["../../src/services/jsonc.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CA+E1D"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strips comments from JSONC content while respecting string boundaries.
|
|
3
|
+
* Handles // and /* comments, URLs in strings, and escaped quotes.
|
|
4
|
+
* Also removes trailing commas to support more relaxed JSONC format.
|
|
5
|
+
*/
|
|
6
|
+
export function stripJsoncComments(content) {
|
|
7
|
+
let result = "";
|
|
8
|
+
let i = 0;
|
|
9
|
+
let inString = false;
|
|
10
|
+
let inSingleLineComment = false;
|
|
11
|
+
let inMultiLineComment = false;
|
|
12
|
+
while (i < content.length) {
|
|
13
|
+
const char = content[i];
|
|
14
|
+
const nextChar = content[i + 1];
|
|
15
|
+
if (!inSingleLineComment && !inMultiLineComment) {
|
|
16
|
+
if (char === '"') {
|
|
17
|
+
// Count consecutive backslashes before this quote
|
|
18
|
+
let backslashCount = 0;
|
|
19
|
+
let j = i - 1;
|
|
20
|
+
while (j >= 0 && content[j] === "\\") {
|
|
21
|
+
backslashCount++;
|
|
22
|
+
j--;
|
|
23
|
+
}
|
|
24
|
+
// Quote is escaped only if preceded by ODD number of backslashes
|
|
25
|
+
// e.g., \" = escaped, \\" = not escaped (escaped backslash + quote)
|
|
26
|
+
if (backslashCount % 2 === 0) {
|
|
27
|
+
inString = !inString;
|
|
28
|
+
}
|
|
29
|
+
result += char;
|
|
30
|
+
i++;
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (inString) {
|
|
35
|
+
result += char;
|
|
36
|
+
i++;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (!inSingleLineComment && !inMultiLineComment) {
|
|
40
|
+
if (char === "/" && nextChar === "/") {
|
|
41
|
+
inSingleLineComment = true;
|
|
42
|
+
i += 2;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (char === "/" && nextChar === "*") {
|
|
46
|
+
inMultiLineComment = true;
|
|
47
|
+
i += 2;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (inSingleLineComment) {
|
|
52
|
+
if (char === "\n") {
|
|
53
|
+
inSingleLineComment = false;
|
|
54
|
+
result += char;
|
|
55
|
+
}
|
|
56
|
+
i++;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (inMultiLineComment) {
|
|
60
|
+
if (char === "*" && nextChar === "/") {
|
|
61
|
+
inMultiLineComment = false;
|
|
62
|
+
i += 2;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (char === "\n") {
|
|
66
|
+
result += char;
|
|
67
|
+
}
|
|
68
|
+
i++;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
result += char;
|
|
72
|
+
i++;
|
|
73
|
+
}
|
|
74
|
+
// Remove trailing commas before } or ]
|
|
75
|
+
return result.replace(/,\s*([}\]])/g, "$1");
|
|
76
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"language-detector.d.ts","sourceRoot":"","sources":["../../src/services/language-detector.ts"],"names":[],"mappings":"AAGA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYnD;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGpD"}
|