@haisto/opencode-mem 2.14.3-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/README.md +165 -0
  2. package/dist/config.d.ts +62 -0
  3. package/dist/config.d.ts.map +1 -0
  4. package/dist/config.js +457 -0
  5. package/dist/index.d.ts +3 -0
  6. package/dist/index.d.ts.map +1 -0
  7. package/dist/index.js +486 -0
  8. package/dist/plugin.d.ts +9 -0
  9. package/dist/plugin.d.ts.map +1 -0
  10. package/dist/plugin.js +5 -0
  11. package/dist/services/ai/ai-provider-factory.d.ts +8 -0
  12. package/dist/services/ai/ai-provider-factory.d.ts.map +1 -0
  13. package/dist/services/ai/ai-provider-factory.js +28 -0
  14. package/dist/services/ai/opencode-provider.d.ts +36 -0
  15. package/dist/services/ai/opencode-provider.d.ts.map +1 -0
  16. package/dist/services/ai/opencode-provider.js +92 -0
  17. package/dist/services/ai/provider-config.d.ts +17 -0
  18. package/dist/services/ai/provider-config.d.ts.map +1 -0
  19. package/dist/services/ai/provider-config.js +14 -0
  20. package/dist/services/ai/providers/anthropic-messages.d.ts +12 -0
  21. package/dist/services/ai/providers/anthropic-messages.d.ts.map +1 -0
  22. package/dist/services/ai/providers/anthropic-messages.js +184 -0
  23. package/dist/services/ai/providers/base-provider.d.ts +25 -0
  24. package/dist/services/ai/providers/base-provider.d.ts.map +1 -0
  25. package/dist/services/ai/providers/base-provider.js +23 -0
  26. package/dist/services/ai/providers/google-gemini.d.ts +16 -0
  27. package/dist/services/ai/providers/google-gemini.d.ts.map +1 -0
  28. package/dist/services/ai/providers/google-gemini.js +228 -0
  29. package/dist/services/ai/providers/openai-chat-completion.d.ts +14 -0
  30. package/dist/services/ai/providers/openai-chat-completion.d.ts.map +1 -0
  31. package/dist/services/ai/providers/openai-chat-completion.js +318 -0
  32. package/dist/services/ai/providers/openai-responses.d.ts +14 -0
  33. package/dist/services/ai/providers/openai-responses.d.ts.map +1 -0
  34. package/dist/services/ai/providers/openai-responses.js +182 -0
  35. package/dist/services/ai/session/ai-session-manager.d.ts +21 -0
  36. package/dist/services/ai/session/ai-session-manager.d.ts.map +1 -0
  37. package/dist/services/ai/session/ai-session-manager.js +166 -0
  38. package/dist/services/ai/session/session-types.d.ts +43 -0
  39. package/dist/services/ai/session/session-types.d.ts.map +1 -0
  40. package/dist/services/ai/session/session-types.js +1 -0
  41. package/dist/services/ai/tools/tool-schema.d.ts +41 -0
  42. package/dist/services/ai/tools/tool-schema.d.ts.map +1 -0
  43. package/dist/services/ai/tools/tool-schema.js +24 -0
  44. package/dist/services/ai/validators/user-profile-validator.d.ts +13 -0
  45. package/dist/services/ai/validators/user-profile-validator.d.ts.map +1 -0
  46. package/dist/services/ai/validators/user-profile-validator.js +111 -0
  47. package/dist/services/api-handlers.d.ts +164 -0
  48. package/dist/services/api-handlers.d.ts.map +1 -0
  49. package/dist/services/api-handlers.js +927 -0
  50. package/dist/services/auto-capture.d.ts +3 -0
  51. package/dist/services/auto-capture.d.ts.map +1 -0
  52. package/dist/services/auto-capture.js +309 -0
  53. package/dist/services/cleanup-service.d.ts +23 -0
  54. package/dist/services/cleanup-service.d.ts.map +1 -0
  55. package/dist/services/cleanup-service.js +102 -0
  56. package/dist/services/client.d.ts +119 -0
  57. package/dist/services/client.d.ts.map +1 -0
  58. package/dist/services/client.js +257 -0
  59. package/dist/services/context.d.ts +11 -0
  60. package/dist/services/context.d.ts.map +1 -0
  61. package/dist/services/context.js +34 -0
  62. package/dist/services/deduplication-service.d.ts +30 -0
  63. package/dist/services/deduplication-service.d.ts.map +1 -0
  64. package/dist/services/deduplication-service.js +124 -0
  65. package/dist/services/embedding.d.ts +15 -0
  66. package/dist/services/embedding.d.ts.map +1 -0
  67. package/dist/services/embedding.js +127 -0
  68. package/dist/services/jsonc.d.ts +7 -0
  69. package/dist/services/jsonc.d.ts.map +1 -0
  70. package/dist/services/jsonc.js +76 -0
  71. package/dist/services/language-detector.d.ts +3 -0
  72. package/dist/services/language-detector.d.ts.map +1 -0
  73. package/dist/services/language-detector.js +33 -0
  74. package/dist/services/logger.d.ts +11 -0
  75. package/dist/services/logger.d.ts.map +1 -0
  76. package/dist/services/logger.js +97 -0
  77. package/dist/services/migration-service.d.ts +42 -0
  78. package/dist/services/migration-service.d.ts.map +1 -0
  79. package/dist/services/migration-service.js +250 -0
  80. package/dist/services/privacy.d.ts +3 -0
  81. package/dist/services/privacy.d.ts.map +1 -0
  82. package/dist/services/privacy.js +7 -0
  83. package/dist/services/secret-resolver.d.ts +2 -0
  84. package/dist/services/secret-resolver.d.ts.map +1 -0
  85. package/dist/services/secret-resolver.js +55 -0
  86. package/dist/services/sqlite/connection-manager.d.ts +13 -0
  87. package/dist/services/sqlite/connection-manager.d.ts.map +1 -0
  88. package/dist/services/sqlite/connection-manager.js +74 -0
  89. package/dist/services/sqlite/shard-manager.d.ts +23 -0
  90. package/dist/services/sqlite/shard-manager.d.ts.map +1 -0
  91. package/dist/services/sqlite/shard-manager.js +288 -0
  92. package/dist/services/sqlite/sqlite-bootstrap.d.ts +2 -0
  93. package/dist/services/sqlite/sqlite-bootstrap.d.ts.map +1 -0
  94. package/dist/services/sqlite/sqlite-bootstrap.js +8 -0
  95. package/dist/services/sqlite/types.d.ts +42 -0
  96. package/dist/services/sqlite/types.d.ts.map +1 -0
  97. package/dist/services/sqlite/types.js +1 -0
  98. package/dist/services/sqlite/vector-search.d.ts +29 -0
  99. package/dist/services/sqlite/vector-search.d.ts.map +1 -0
  100. package/dist/services/sqlite/vector-search.js +279 -0
  101. package/dist/services/tags.d.ts +24 -0
  102. package/dist/services/tags.d.ts.map +1 -0
  103. package/dist/services/tags.js +145 -0
  104. package/dist/services/user-memory-learning.d.ts +3 -0
  105. package/dist/services/user-memory-learning.d.ts.map +1 -0
  106. package/dist/services/user-memory-learning.js +235 -0
  107. package/dist/services/user-profile/profile-context.d.ts +2 -0
  108. package/dist/services/user-profile/profile-context.d.ts.map +1 -0
  109. package/dist/services/user-profile/profile-context.js +40 -0
  110. package/dist/services/user-profile/profile-utils.d.ts +3 -0
  111. package/dist/services/user-profile/profile-utils.d.ts.map +1 -0
  112. package/dist/services/user-profile/profile-utils.js +45 -0
  113. package/dist/services/user-profile/types.d.ts +46 -0
  114. package/dist/services/user-profile/types.d.ts.map +1 -0
  115. package/dist/services/user-profile/types.js +1 -0
  116. package/dist/services/user-profile/user-profile-manager.d.ts +23 -0
  117. package/dist/services/user-profile/user-profile-manager.d.ts.map +1 -0
  118. package/dist/services/user-profile/user-profile-manager.js +337 -0
  119. package/dist/services/user-prompt/user-prompt-manager.d.ts +41 -0
  120. package/dist/services/user-prompt/user-prompt-manager.d.ts.map +1 -0
  121. package/dist/services/user-prompt/user-prompt-manager.js +192 -0
  122. package/dist/services/vector-backends/backend-factory.d.ts +3 -0
  123. package/dist/services/vector-backends/backend-factory.d.ts.map +1 -0
  124. package/dist/services/vector-backends/backend-factory.js +104 -0
  125. package/dist/services/vector-backends/exact-scan-backend.d.ts +39 -0
  126. package/dist/services/vector-backends/exact-scan-backend.d.ts.map +1 -0
  127. package/dist/services/vector-backends/exact-scan-backend.js +63 -0
  128. package/dist/services/vector-backends/types.d.ts +51 -0
  129. package/dist/services/vector-backends/types.d.ts.map +1 -0
  130. package/dist/services/vector-backends/types.js +1 -0
  131. package/dist/services/vector-backends/usearch-backend.d.ts +47 -0
  132. package/dist/services/vector-backends/usearch-backend.d.ts.map +1 -0
  133. package/dist/services/vector-backends/usearch-backend.js +174 -0
  134. package/dist/services/web-server-worker.d.ts +2 -0
  135. package/dist/services/web-server-worker.d.ts.map +1 -0
  136. package/dist/services/web-server-worker.js +283 -0
  137. package/dist/services/web-server.d.ts +31 -0
  138. package/dist/services/web-server.d.ts.map +1 -0
  139. package/dist/services/web-server.js +356 -0
  140. package/dist/types/index.d.ts +19 -0
  141. package/dist/types/index.d.ts.map +1 -0
  142. package/dist/types/index.js +1 -0
  143. package/dist/web/app.d.ts +2 -0
  144. package/dist/web/app.d.ts.map +1 -0
  145. package/dist/web/app.js +1238 -0
  146. package/dist/web/favicon.ico +0 -0
  147. package/dist/web/i18n.d.ts +2 -0
  148. package/dist/web/i18n.d.ts.map +1 -0
  149. package/dist/web/i18n.js +312 -0
  150. package/dist/web/index.html +293 -0
  151. package/dist/web/styles.css +1786 -0
  152. package/package.json +78 -0
@@ -0,0 +1,257 @@
1
+ import { embeddingService } from "./embedding.js";
2
+ import { shardManager } from "./sqlite/shard-manager.js";
3
+ import { vectorSearch } from "./sqlite/vector-search.js";
4
+ import { connectionManager } from "./sqlite/connection-manager.js";
5
+ import { CONFIG } from "../config.js";
6
+ import { log } from "./logger.js";
7
+ function safeToISOString(timestamp) {
8
+ try {
9
+ if (timestamp === null || timestamp === undefined) {
10
+ return new Date().toISOString();
11
+ }
12
+ const numValue = typeof timestamp === "bigint" ? Number(timestamp) : Number(timestamp);
13
+ if (isNaN(numValue) || numValue < 0) {
14
+ return new Date().toISOString();
15
+ }
16
+ return new Date(numValue).toISOString();
17
+ }
18
+ catch {
19
+ return new Date().toISOString();
20
+ }
21
+ }
22
+ function safeJSONParse(jsonString) {
23
+ if (!jsonString || typeof jsonString !== "string") {
24
+ return undefined;
25
+ }
26
+ try {
27
+ return JSON.parse(jsonString);
28
+ }
29
+ catch {
30
+ return undefined;
31
+ }
32
+ }
33
+ function extractScopeFromContainerTag(containerTag) {
34
+ const parts = containerTag.split("_");
35
+ if (parts.length >= 3) {
36
+ const scope = parts[1];
37
+ const hash = parts.slice(2).join("_");
38
+ return { scope, hash };
39
+ }
40
+ return { scope: "user", hash: containerTag };
41
+ }
42
+ function resolveScopeValue(scope, containerTag) {
43
+ if (scope === "all-projects") {
44
+ return { scope: "project", hash: "" };
45
+ }
46
+ return extractScopeFromContainerTag(containerTag);
47
+ }
48
+ export class LocalMemoryClient {
49
+ initPromise = null;
50
+ isInitialized = false;
51
+ constructor() { }
52
+ async initialize() {
53
+ if (this.isInitialized)
54
+ return;
55
+ if (this.initPromise)
56
+ return this.initPromise;
57
+ this.initPromise = (async () => {
58
+ try {
59
+ this.isInitialized = true;
60
+ }
61
+ catch (error) {
62
+ this.initPromise = null;
63
+ log("SQLite initialization failed", { error: String(error) });
64
+ throw error;
65
+ }
66
+ })();
67
+ return this.initPromise;
68
+ }
69
+ async warmup(progressCallback) {
70
+ await this.initialize();
71
+ await embeddingService.warmup(progressCallback);
72
+ }
73
+ async isReady() {
74
+ return this.isInitialized && embeddingService.isWarmedUp;
75
+ }
76
+ getStatus() {
77
+ return {
78
+ dbConnected: this.isInitialized,
79
+ modelLoaded: embeddingService.isWarmedUp,
80
+ ready: this.isInitialized && embeddingService.isWarmedUp,
81
+ };
82
+ }
83
+ close() {
84
+ connectionManager.closeAll();
85
+ }
86
+ async searchMemories(query, containerTag, scope = "project") {
87
+ try {
88
+ await this.initialize();
89
+ const queryVector = await embeddingService.embedWithTimeout(query);
90
+ const resolved = resolveScopeValue(scope, containerTag);
91
+ const shards = shardManager.getAllShards(resolved.scope, resolved.hash);
92
+ if (shards.length === 0) {
93
+ return { success: true, results: [], total: 0, timing: 0 };
94
+ }
95
+ const results = await vectorSearch.searchAcrossShards(shards, queryVector, scope === "all-projects" ? "" : containerTag, CONFIG.maxMemories, CONFIG.similarityThreshold, query);
96
+ return { success: true, results, total: results.length, timing: 0 };
97
+ }
98
+ catch (error) {
99
+ const errorMessage = error instanceof Error ? error.message : String(error);
100
+ log("searchMemories: error", { error: errorMessage });
101
+ return { success: false, error: errorMessage, results: [], total: 0, timing: 0 };
102
+ }
103
+ }
104
+ async addMemory(content, containerTag, metadata) {
105
+ try {
106
+ await this.initialize();
107
+ const tags = metadata?.tags || [];
108
+ const vector = await embeddingService.embedWithTimeout(content);
109
+ let tagsVector = undefined;
110
+ if (tags.length > 0) {
111
+ tagsVector = await embeddingService.embedWithTimeout(tags.join(", "));
112
+ }
113
+ const { scope, hash } = extractScopeFromContainerTag(containerTag);
114
+ const shard = shardManager.getWriteShard(scope, hash);
115
+ const id = `mem_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
116
+ const now = Date.now();
117
+ const { displayName, userName, userEmail, projectPath, projectName, gitRepoUrl, type, tags: _tags, ...dynamicMetadata } = metadata || {};
118
+ const record = {
119
+ id,
120
+ content,
121
+ vector,
122
+ tagsVector,
123
+ containerTag,
124
+ tags: tags.length > 0 ? tags.join(",") : undefined,
125
+ type,
126
+ createdAt: now,
127
+ updatedAt: now,
128
+ displayName,
129
+ userName,
130
+ userEmail,
131
+ projectPath,
132
+ projectName,
133
+ gitRepoUrl,
134
+ metadata: Object.keys(dynamicMetadata).length > 0 ? JSON.stringify(dynamicMetadata) : undefined,
135
+ };
136
+ const db = connectionManager.getConnection(shard.dbPath);
137
+ await vectorSearch.insertVector(db, record, shard);
138
+ shardManager.incrementVectorCount(shard.id);
139
+ return { success: true, id };
140
+ }
141
+ catch (error) {
142
+ const errorMessage = error instanceof Error ? error.message : String(error);
143
+ log("addMemory: error", { error: errorMessage });
144
+ return { success: false, error: errorMessage };
145
+ }
146
+ }
147
+ async deleteMemory(memoryId) {
148
+ try {
149
+ await this.initialize();
150
+ const userShards = shardManager.getAllShards("user", "");
151
+ const projectShards = shardManager.getAllShards("project", "");
152
+ const allShards = [...userShards, ...projectShards];
153
+ for (const shard of allShards) {
154
+ const db = connectionManager.getConnection(shard.dbPath);
155
+ const memory = vectorSearch.getMemoryById(db, memoryId);
156
+ if (memory) {
157
+ await vectorSearch.deleteVector(db, memoryId, shard);
158
+ shardManager.decrementVectorCount(shard.id);
159
+ return { success: true };
160
+ }
161
+ }
162
+ return { success: false, error: "Memory not found" };
163
+ }
164
+ catch (error) {
165
+ const errorMessage = error instanceof Error ? error.message : String(error);
166
+ log("deleteMemory: error", { memoryId, error: errorMessage });
167
+ return { success: false, error: errorMessage };
168
+ }
169
+ }
170
+ async listMemories(containerTag, limit = 20, scope = "project") {
171
+ try {
172
+ await this.initialize();
173
+ const resolved = resolveScopeValue(scope, containerTag);
174
+ const shards = shardManager.getAllShards(resolved.scope, resolved.hash);
175
+ if (shards.length === 0) {
176
+ return {
177
+ success: true,
178
+ memories: [],
179
+ pagination: { currentPage: 1, totalItems: 0, totalPages: 0 },
180
+ };
181
+ }
182
+ const allMemories = [];
183
+ for (const shard of shards) {
184
+ const db = connectionManager.getConnection(shard.dbPath);
185
+ const memories = vectorSearch.listMemories(db, scope === "all-projects" ? "" : containerTag, limit);
186
+ allMemories.push(...memories);
187
+ }
188
+ allMemories.sort((a, b) => Number(b.created_at) - Number(a.created_at));
189
+ const memories = allMemories.slice(0, limit).map((r) => ({
190
+ id: r.id,
191
+ summary: r.content,
192
+ createdAt: safeToISOString(r.created_at),
193
+ metadata: safeJSONParse(r.metadata),
194
+ displayName: r.display_name,
195
+ userName: r.user_name,
196
+ userEmail: r.user_email,
197
+ projectPath: r.project_path,
198
+ projectName: r.project_name,
199
+ gitRepoUrl: r.git_repo_url,
200
+ }));
201
+ return {
202
+ success: true,
203
+ memories,
204
+ pagination: { currentPage: 1, totalItems: memories.length, totalPages: 1 },
205
+ };
206
+ }
207
+ catch (error) {
208
+ const errorMessage = error instanceof Error ? error.message : String(error);
209
+ log("listMemories: error", { error: errorMessage });
210
+ return {
211
+ success: false,
212
+ error: errorMessage,
213
+ memories: [],
214
+ pagination: { currentPage: 1, totalItems: 0, totalPages: 0 },
215
+ };
216
+ }
217
+ }
218
+ async searchMemoriesBySessionID(sessionID, containerTag, limit = 10) {
219
+ try {
220
+ await this.initialize();
221
+ const { scope, hash } = extractScopeFromContainerTag(containerTag);
222
+ const shards = shardManager.getAllShards(scope, hash);
223
+ if (shards.length === 0) {
224
+ return { success: true, results: [], total: 0, timing: 0 };
225
+ }
226
+ const allMemories = [];
227
+ for (const shard of shards) {
228
+ const db = connectionManager.getConnection(shard.dbPath);
229
+ const memories = vectorSearch.getMemoriesBySessionID(db, sessionID);
230
+ allMemories.push(...memories);
231
+ }
232
+ allMemories.sort((a, b) => b.created_at - a.created_at);
233
+ const results = allMemories.slice(0, limit).map((row) => ({
234
+ id: row.id,
235
+ memory: row.content,
236
+ similarity: 1.0,
237
+ tags: row.tags || [],
238
+ metadata: row.metadata || {},
239
+ containerTag: row.container_tag,
240
+ displayName: row.display_name,
241
+ userName: row.user_name,
242
+ userEmail: row.user_email,
243
+ projectPath: row.project_path,
244
+ projectName: row.project_name,
245
+ gitRepoUrl: row.git_repo_url,
246
+ createdAt: row.created_at,
247
+ }));
248
+ return { success: true, results, total: results.length, timing: 0 };
249
+ }
250
+ catch (error) {
251
+ const errorMessage = error instanceof Error ? error.message : String(error);
252
+ log("searchMemoriesBySessionID: error", { error: errorMessage });
253
+ return { success: false, error: errorMessage, results: [], total: 0, timing: 0 };
254
+ }
255
+ }
256
+ }
257
+ export const memoryClient = new LocalMemoryClient();
@@ -0,0 +1,11 @@
1
+ interface MemoryResultMinimal {
2
+ similarity: number;
3
+ memory?: string;
4
+ chunk?: string;
5
+ }
6
+ interface MemoriesResponseMinimal {
7
+ results?: MemoryResultMinimal[];
8
+ }
9
+ export declare function formatContextForPrompt(userId: string | null, projectMemories: MemoriesResponseMinimal): string;
10
+ export {};
11
+ //# sourceMappingURL=context.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/services/context.ts"],"names":[],"mappings":"AAIA,UAAU,mBAAmB;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,uBAAuB;IAC/B,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACjC;AAED,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,eAAe,EAAE,uBAAuB,GACvC,MAAM,CAkCR"}
@@ -0,0 +1,34 @@
1
+ import { CONFIG } from "../config.js";
2
+ import { getUserProfileContext } from "./user-profile/profile-context.js";
3
+ import { logDebug } from "./logger.js";
4
+ export function formatContextForPrompt(userId, projectMemories) {
5
+ const parts = ["[MEMORY]"];
6
+ if (CONFIG.injectProfile && userId) {
7
+ const profileContext = getUserProfileContext(userId);
8
+ if (profileContext) {
9
+ parts.push("\n" + profileContext);
10
+ }
11
+ }
12
+ const projectResults = projectMemories.results || [];
13
+ if (projectResults.length > 0) {
14
+ parts.push("\nProject Knowledge:");
15
+ projectResults.forEach((mem) => {
16
+ const similarity = Math.round(mem.similarity * 100);
17
+ const content = mem.memory || mem.chunk || "";
18
+ parts.push(`- [${similarity}%] ${content}`);
19
+ });
20
+ }
21
+ if (parts.length === 1) {
22
+ return "";
23
+ }
24
+ const result = parts.join("\n");
25
+ logDebug("Injected context", {
26
+ userId: userId ?? undefined,
27
+ charCount: result.length,
28
+ lineCount: result.split("\n").length,
29
+ hasProfile: CONFIG.injectProfile && userId ? true : false,
30
+ memoryCount: projectMemories.results?.length || 0,
31
+ preview: result.slice(0, 300) + (result.length > 300 ? "..." : ""),
32
+ });
33
+ return result;
34
+ }
@@ -0,0 +1,30 @@
1
+ interface DuplicateGroup {
2
+ representative: {
3
+ id: string;
4
+ content: string;
5
+ containerTag: string;
6
+ createdAt: number;
7
+ };
8
+ duplicates: Array<{
9
+ id: string;
10
+ content: string;
11
+ similarity: number;
12
+ }>;
13
+ }
14
+ interface DeduplicationResult {
15
+ exactDuplicatesDeleted: number;
16
+ nearDuplicateGroups: DuplicateGroup[];
17
+ }
18
+ export declare class DeduplicationService {
19
+ private isRunning;
20
+ detectAndRemoveDuplicates(): Promise<DeduplicationResult>;
21
+ private cosineSimilarity;
22
+ getStatus(): {
23
+ enabled: boolean;
24
+ threshold: number;
25
+ isRunning: boolean;
26
+ };
27
+ }
28
+ export declare const deduplicationService: DeduplicationService;
29
+ export {};
30
+ //# sourceMappingURL=deduplication-service.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"deduplication-service.d.ts","sourceRoot":"","sources":["../../src/services/deduplication-service.ts"],"names":[],"mappings":"AAMA,UAAU,cAAc;IACtB,cAAc,EAAE;QACd,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,EAAE,KAAK,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,UAAU,mBAAmB;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,mBAAmB,EAAE,cAAc,EAAE,CAAC;CACvC;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,SAAS,CAAkB;IAE7B,yBAAyB,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAwG/D,OAAO,CAAC,gBAAgB;IAoBxB,SAAS;;;;;CAOV;AAED,eAAO,MAAM,oBAAoB,sBAA6B,CAAC"}
@@ -0,0 +1,124 @@
1
+ import { shardManager } from "./sqlite/shard-manager.js";
2
+ import { vectorSearch } from "./sqlite/vector-search.js";
3
+ import { connectionManager } from "./sqlite/connection-manager.js";
4
+ import { CONFIG } from "../config.js";
5
+ import { log } from "./logger.js";
6
+ export class DeduplicationService {
7
+ isRunning = false;
8
+ async detectAndRemoveDuplicates() {
9
+ if (this.isRunning) {
10
+ throw new Error("Deduplication already running");
11
+ }
12
+ if (!CONFIG.deduplicationEnabled) {
13
+ throw new Error("Deduplication is disabled in config");
14
+ }
15
+ this.isRunning = true;
16
+ try {
17
+ const userShards = shardManager.getAllShards("user", "");
18
+ const projectShards = shardManager.getAllShards("project", "");
19
+ const allShards = [...userShards, ...projectShards];
20
+ let exactDeleted = 0;
21
+ const nearDuplicateGroups = [];
22
+ for (const shard of allShards) {
23
+ const db = connectionManager.getConnection(shard.dbPath);
24
+ const memories = vectorSearch.getAllMemories(db);
25
+ const contentMap = new Map();
26
+ for (const memory of memories) {
27
+ const key = `${memory.container_tag}:${memory.content}`;
28
+ if (!contentMap.has(key)) {
29
+ contentMap.set(key, []);
30
+ }
31
+ contentMap.get(key).push(memory);
32
+ }
33
+ for (const [, duplicates] of contentMap) {
34
+ if (duplicates.length > 1) {
35
+ duplicates.sort((a, b) => Number(b.created_at) - Number(a.created_at));
36
+ const toDelete = duplicates.slice(1);
37
+ for (const dup of toDelete) {
38
+ try {
39
+ await vectorSearch.deleteVector(db, dup.id, shard);
40
+ shardManager.decrementVectorCount(shard.id);
41
+ exactDeleted++;
42
+ }
43
+ catch (error) {
44
+ log("Deduplication: delete error", {
45
+ memoryId: dup.id,
46
+ error: String(error),
47
+ });
48
+ }
49
+ }
50
+ }
51
+ }
52
+ const uniqueMemories = Array.from(contentMap.values()).map((arr) => arr[0]);
53
+ const processedIds = new Set();
54
+ for (let i = 0; i < uniqueMemories.length; i++) {
55
+ const mem1 = uniqueMemories[i];
56
+ if (!mem1.vector || processedIds.has(mem1.id))
57
+ continue;
58
+ const vector1 = new Float32Array(new Uint8Array(mem1.vector).buffer);
59
+ const similarGroup = {
60
+ representative: {
61
+ id: mem1.id,
62
+ content: mem1.content,
63
+ containerTag: mem1.container_tag,
64
+ createdAt: mem1.created_at,
65
+ },
66
+ duplicates: [],
67
+ };
68
+ for (let j = i + 1; j < uniqueMemories.length; j++) {
69
+ const mem2 = uniqueMemories[j];
70
+ if (!mem2.vector || processedIds.has(mem2.id))
71
+ continue;
72
+ if (mem1.container_tag !== mem2.container_tag)
73
+ continue;
74
+ const vector2 = new Float32Array(new Uint8Array(mem2.vector).buffer);
75
+ const similarity = this.cosineSimilarity(vector1, vector2);
76
+ if (similarity >= CONFIG.deduplicationSimilarityThreshold && similarity < 1.0) {
77
+ similarGroup.duplicates.push({
78
+ id: mem2.id,
79
+ content: mem2.content,
80
+ similarity,
81
+ });
82
+ processedIds.add(mem2.id);
83
+ }
84
+ }
85
+ if (similarGroup.duplicates.length > 0) {
86
+ nearDuplicateGroups.push(similarGroup);
87
+ }
88
+ }
89
+ }
90
+ return {
91
+ exactDuplicatesDeleted: exactDeleted,
92
+ nearDuplicateGroups,
93
+ };
94
+ }
95
+ finally {
96
+ this.isRunning = false;
97
+ }
98
+ }
99
+ cosineSimilarity(a, b) {
100
+ if (a.length !== b.length)
101
+ return 0;
102
+ let dotProduct = 0;
103
+ let normA = 0;
104
+ let normB = 0;
105
+ for (let i = 0; i < a.length; i++) {
106
+ const aVal = a[i] || 0;
107
+ const bVal = b[i] || 0;
108
+ dotProduct += aVal * bVal;
109
+ normA += aVal * aVal;
110
+ normB += bVal * bVal;
111
+ }
112
+ if (normA === 0 || normB === 0)
113
+ return 0;
114
+ return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
115
+ }
116
+ getStatus() {
117
+ return {
118
+ enabled: CONFIG.deduplicationEnabled,
119
+ threshold: CONFIG.deduplicationSimilarityThreshold,
120
+ isRunning: this.isRunning,
121
+ };
122
+ }
123
+ }
124
+ export const deduplicationService = new DeduplicationService();
@@ -0,0 +1,15 @@
1
+ export declare class EmbeddingService {
2
+ private pipe;
3
+ private initPromise;
4
+ isWarmedUp: boolean;
5
+ private cache;
6
+ private cachedModelName;
7
+ static getInstance(): EmbeddingService;
8
+ warmup(progressCallback?: (progress: any) => void): Promise<void>;
9
+ private initializeModel;
10
+ embed(text: string): Promise<Float32Array>;
11
+ embedWithTimeout(text: string): Promise<Float32Array>;
12
+ clearCache(): void;
13
+ }
14
+ export declare const embeddingService: EmbeddingService;
15
+ //# sourceMappingURL=embedding.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/services/embedding.ts"],"names":[],"mappings":"AA6CA,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,WAAW,CAA8B;IAC1C,UAAU,EAAE,OAAO,CAAS;IACnC,OAAO,CAAC,KAAK,CAAwC;IACrD,OAAO,CAAC,eAAe,CAAuB;IAE9C,MAAM,CAAC,WAAW,IAAI,gBAAgB;IAOhC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YAOzD,eAAe;IAkBvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAmD1C,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAI3D,UAAU,IAAI,IAAI;CAGnB;AAED,eAAO,MAAM,gBAAgB,kBAAiC,CAAC"}
@@ -0,0 +1,127 @@
1
+ import { CONFIG } from "../config.js";
2
+ import { log } from "./logger.js";
3
+ import { join } from "node:path";
4
+ const TIMEOUT_MS = 30000;
5
+ const GLOBAL_EMBEDDING_KEY = Symbol.for("opencode-mem.embedding.instance");
6
+ const MAX_CACHE_SIZE = 100;
7
+ let _transformers = null;
8
+ function getTransformersPackageSpecifier() {
9
+ // Keep this non-literal so OpenCode/Bun plugin-loader bundling does not eagerly
10
+ // traverse @xenova/transformers internals during plugin startup. The package
11
+ // is only needed for the local embedding backend, and should stay lazy.
12
+ return ["@xenova", "transformers"].join("/");
13
+ }
14
+ async function ensureTransformersLoaded() {
15
+ if (_transformers !== null)
16
+ return _transformers;
17
+ const mod = (await import(getTransformersPackageSpecifier()));
18
+ mod.env.allowLocalModels = true;
19
+ mod.env.allowRemoteModels = true;
20
+ mod.env.cacheDir = join(CONFIG.storagePath, ".cache");
21
+ // Keep ONNX WASM single-threaded for Bun/Node runtimes without SharedArrayBuffer.
22
+ try {
23
+ mod.env.backends.onnx.wasm.numThreads = 1;
24
+ }
25
+ catch (e) {
26
+ log("Failed to set wasm.numThreads", { error: String(e) });
27
+ }
28
+ _transformers = mod;
29
+ return _transformers;
30
+ }
31
+ function withTimeout(promise, ms) {
32
+ return Promise.race([
33
+ promise,
34
+ new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms)),
35
+ ]);
36
+ }
37
+ export class EmbeddingService {
38
+ pipe = null;
39
+ initPromise = null;
40
+ isWarmedUp = false;
41
+ cache = new Map();
42
+ cachedModelName = null;
43
+ static getInstance() {
44
+ if (!globalThis[GLOBAL_EMBEDDING_KEY]) {
45
+ globalThis[GLOBAL_EMBEDDING_KEY] = new EmbeddingService();
46
+ }
47
+ return globalThis[GLOBAL_EMBEDDING_KEY];
48
+ }
49
+ async warmup(progressCallback) {
50
+ if (this.isWarmedUp)
51
+ return;
52
+ if (this.initPromise)
53
+ return this.initPromise;
54
+ this.initPromise = this.initializeModel(progressCallback);
55
+ return this.initPromise;
56
+ }
57
+ async initializeModel(progressCallback) {
58
+ try {
59
+ if (CONFIG.embeddingApiUrl && CONFIG.embeddingApiKey) {
60
+ this.isWarmedUp = true;
61
+ return;
62
+ }
63
+ const { pipeline } = await ensureTransformersLoaded();
64
+ this.pipe = await pipeline("feature-extraction", CONFIG.embeddingModel, {
65
+ progress_callback: progressCallback,
66
+ });
67
+ this.isWarmedUp = true;
68
+ }
69
+ catch (error) {
70
+ this.initPromise = null;
71
+ log("Failed to initialize embedding model", { error: String(error) });
72
+ throw error;
73
+ }
74
+ }
75
+ async embed(text) {
76
+ if (this.cachedModelName !== CONFIG.embeddingModel) {
77
+ this.clearCache();
78
+ this.cachedModelName = CONFIG.embeddingModel;
79
+ }
80
+ const cached = this.cache.get(text);
81
+ if (cached)
82
+ return cached;
83
+ if (!this.isWarmedUp && !this.initPromise) {
84
+ await this.warmup();
85
+ }
86
+ if (this.initPromise) {
87
+ await this.initPromise;
88
+ }
89
+ let result;
90
+ if (CONFIG.embeddingApiUrl && CONFIG.embeddingApiKey) {
91
+ const response = await fetch(`${CONFIG.embeddingApiUrl}/embeddings`, {
92
+ method: "POST",
93
+ headers: {
94
+ "Content-Type": "application/json",
95
+ Authorization: `Bearer ${CONFIG.embeddingApiKey}`,
96
+ },
97
+ body: JSON.stringify({
98
+ input: text,
99
+ model: CONFIG.embeddingModel,
100
+ }),
101
+ });
102
+ if (!response.ok) {
103
+ throw new Error(`API embedding failed: ${response.statusText}`);
104
+ }
105
+ const data = await response.json();
106
+ result = new Float32Array(data.data[0].embedding);
107
+ }
108
+ else {
109
+ const output = await this.pipe(text, { pooling: "mean", normalize: true });
110
+ result = new Float32Array(output.data);
111
+ }
112
+ if (this.cache.size >= MAX_CACHE_SIZE) {
113
+ const firstKey = this.cache.keys().next().value;
114
+ if (firstKey !== undefined)
115
+ this.cache.delete(firstKey);
116
+ }
117
+ this.cache.set(text, result);
118
+ return result;
119
+ }
120
+ async embedWithTimeout(text) {
121
+ return withTimeout(this.embed(text), TIMEOUT_MS);
122
+ }
123
+ clearCache() {
124
+ this.cache.clear();
125
+ }
126
+ }
127
+ export const embeddingService = EmbeddingService.getInstance();
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Strips comments from JSONC content while respecting string boundaries.
3
+ * Handles // and /* comments, URLs in strings, and escaped quotes.
4
+ * Also removes trailing commas to support more relaxed JSONC format.
5
+ */
6
+ export declare function stripJsoncComments(content: string): string;
7
+ //# sourceMappingURL=jsonc.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jsonc.d.ts","sourceRoot":"","sources":["../../src/services/jsonc.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CA+E1D"}