@xiaoxiamimengfb/my-opencode-mem 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +155 -0
- package/dist/config.d.ts +58 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +411 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +427 -0
- package/dist/plugin.d.ts +5 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +4 -0
- package/dist/services/ai/ai-provider-factory.d.ts +8 -0
- package/dist/services/ai/ai-provider-factory.d.ts.map +1 -0
- package/dist/services/ai/ai-provider-factory.js +28 -0
- package/dist/services/ai/opencode-provider.d.ts +30 -0
- package/dist/services/ai/opencode-provider.d.ts.map +1 -0
- package/dist/services/ai/opencode-provider.js +332 -0
- package/dist/services/ai/provider-config.d.ts +17 -0
- package/dist/services/ai/provider-config.d.ts.map +1 -0
- package/dist/services/ai/provider-config.js +14 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts +12 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts.map +1 -0
- package/dist/services/ai/providers/anthropic-messages.js +184 -0
- package/dist/services/ai/providers/base-provider.d.ts +25 -0
- package/dist/services/ai/providers/base-provider.d.ts.map +1 -0
- package/dist/services/ai/providers/base-provider.js +23 -0
- package/dist/services/ai/providers/google-gemini.d.ts +16 -0
- package/dist/services/ai/providers/google-gemini.d.ts.map +1 -0
- package/dist/services/ai/providers/google-gemini.js +228 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts +13 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts.map +1 -0
- package/dist/services/ai/providers/openai-chat-completion.js +277 -0
- package/dist/services/ai/providers/openai-responses.d.ts +14 -0
- package/dist/services/ai/providers/openai-responses.d.ts.map +1 -0
- package/dist/services/ai/providers/openai-responses.js +182 -0
- package/dist/services/ai/session/ai-session-manager.d.ts +21 -0
- package/dist/services/ai/session/ai-session-manager.d.ts.map +1 -0
- package/dist/services/ai/session/ai-session-manager.js +166 -0
- package/dist/services/ai/session/session-types.d.ts +43 -0
- package/dist/services/ai/session/session-types.d.ts.map +1 -0
- package/dist/services/ai/session/session-types.js +1 -0
- package/dist/services/ai/tools/tool-schema.d.ts +41 -0
- package/dist/services/ai/tools/tool-schema.d.ts.map +1 -0
- package/dist/services/ai/tools/tool-schema.js +24 -0
- package/dist/services/ai/validators/user-profile-validator.d.ts +13 -0
- package/dist/services/ai/validators/user-profile-validator.d.ts.map +1 -0
- package/dist/services/ai/validators/user-profile-validator.js +111 -0
- package/dist/services/api-handlers.d.ts +164 -0
- package/dist/services/api-handlers.d.ts.map +1 -0
- package/dist/services/api-handlers.js +901 -0
- package/dist/services/auto-capture.d.ts +3 -0
- package/dist/services/auto-capture.d.ts.map +1 -0
- package/dist/services/auto-capture.js +306 -0
- package/dist/services/cleanup-service.d.ts +23 -0
- package/dist/services/cleanup-service.d.ts.map +1 -0
- package/dist/services/cleanup-service.js +102 -0
- package/dist/services/client.d.ts +118 -0
- package/dist/services/client.d.ts.map +1 -0
- package/dist/services/client.js +251 -0
- package/dist/services/context.d.ts +11 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +24 -0
- package/dist/services/deduplication-service.d.ts +30 -0
- package/dist/services/deduplication-service.d.ts.map +1 -0
- package/dist/services/deduplication-service.js +124 -0
- package/dist/services/embedding.d.ts +15 -0
- package/dist/services/embedding.d.ts.map +1 -0
- package/dist/services/embedding.js +106 -0
- package/dist/services/jsonc.d.ts +7 -0
- package/dist/services/jsonc.d.ts.map +1 -0
- package/dist/services/jsonc.js +76 -0
- package/dist/services/language-detector.d.ts +3 -0
- package/dist/services/language-detector.d.ts.map +1 -0
- package/dist/services/language-detector.js +16 -0
- package/dist/services/logger.d.ts +2 -0
- package/dist/services/logger.d.ts.map +1 -0
- package/dist/services/logger.js +51 -0
- package/dist/services/migration-service.d.ts +42 -0
- package/dist/services/migration-service.d.ts.map +1 -0
- package/dist/services/migration-service.js +250 -0
- package/dist/services/privacy.d.ts +3 -0
- package/dist/services/privacy.d.ts.map +1 -0
- package/dist/services/privacy.js +7 -0
- package/dist/services/secret-resolver.d.ts +2 -0
- package/dist/services/secret-resolver.d.ts.map +1 -0
- package/dist/services/secret-resolver.js +55 -0
- package/dist/services/sqlite/connection-manager.d.ts +13 -0
- package/dist/services/sqlite/connection-manager.d.ts.map +1 -0
- package/dist/services/sqlite/connection-manager.js +74 -0
- package/dist/services/sqlite/shard-manager.d.ts +23 -0
- package/dist/services/sqlite/shard-manager.d.ts.map +1 -0
- package/dist/services/sqlite/shard-manager.js +288 -0
- package/dist/services/sqlite/sqlite-bootstrap.d.ts +2 -0
- package/dist/services/sqlite/sqlite-bootstrap.d.ts.map +1 -0
- package/dist/services/sqlite/sqlite-bootstrap.js +8 -0
- package/dist/services/sqlite/types.d.ts +42 -0
- package/dist/services/sqlite/types.d.ts.map +1 -0
- package/dist/services/sqlite/types.js +1 -0
- package/dist/services/sqlite/vector-search.d.ts +29 -0
- package/dist/services/sqlite/vector-search.d.ts.map +1 -0
- package/dist/services/sqlite/vector-search.js +268 -0
- package/dist/services/tags.d.ts +24 -0
- package/dist/services/tags.d.ts.map +1 -0
- package/dist/services/tags.js +146 -0
- package/dist/services/user-memory-learning.d.ts +3 -0
- package/dist/services/user-memory-learning.d.ts.map +1 -0
- package/dist/services/user-memory-learning.js +231 -0
- package/dist/services/user-profile/profile-context.d.ts +2 -0
- package/dist/services/user-profile/profile-context.d.ts.map +1 -0
- package/dist/services/user-profile/profile-context.js +40 -0
- package/dist/services/user-profile/profile-utils.d.ts +3 -0
- package/dist/services/user-profile/profile-utils.d.ts.map +1 -0
- package/dist/services/user-profile/profile-utils.js +45 -0
- package/dist/services/user-profile/types.d.ts +46 -0
- package/dist/services/user-profile/types.d.ts.map +1 -0
- package/dist/services/user-profile/types.js +1 -0
- package/dist/services/user-profile/user-profile-manager.d.ts +23 -0
- package/dist/services/user-profile/user-profile-manager.d.ts.map +1 -0
- package/dist/services/user-profile/user-profile-manager.js +292 -0
- package/dist/services/user-prompt/user-prompt-manager.d.ts +41 -0
- package/dist/services/user-prompt/user-prompt-manager.d.ts.map +1 -0
- package/dist/services/user-prompt/user-prompt-manager.js +192 -0
- package/dist/services/vector-backends/backend-factory.d.ts +3 -0
- package/dist/services/vector-backends/backend-factory.d.ts.map +1 -0
- package/dist/services/vector-backends/backend-factory.js +104 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts +39 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/exact-scan-backend.js +63 -0
- package/dist/services/vector-backends/types.d.ts +51 -0
- package/dist/services/vector-backends/types.d.ts.map +1 -0
- package/dist/services/vector-backends/types.js +1 -0
- package/dist/services/vector-backends/usearch-backend.d.ts +47 -0
- package/dist/services/vector-backends/usearch-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/usearch-backend.js +174 -0
- package/dist/services/web-server-worker.d.ts +2 -0
- package/dist/services/web-server-worker.d.ts.map +1 -0
- package/dist/services/web-server-worker.js +283 -0
- package/dist/services/web-server.d.ts +31 -0
- package/dist/services/web-server.d.ts.map +1 -0
- package/dist/services/web-server.js +356 -0
- package/dist/types/index.d.ts +19 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/web/app.d.ts +2 -0
- package/dist/web/app.d.ts.map +1 -0
- package/dist/web/app.js +1194 -0
- package/dist/web/favicon.ico +0 -0
- package/dist/web/i18n.d.ts +2 -0
- package/dist/web/i18n.d.ts.map +1 -0
- package/dist/web/i18n.js +265 -0
- package/dist/web/index.html +284 -0
- package/dist/web/styles.css +1631 -0
- package/package.json +71 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { franc } from "franc-min";
|
|
2
|
+
import { iso6393, iso6393To1 } from "iso-639-3";
|
|
3
|
+
export function detectLanguage(text) {
|
|
4
|
+
if (!text || text.trim().length === 0) {
|
|
5
|
+
return "en";
|
|
6
|
+
}
|
|
7
|
+
const detected = franc(text, { minLength: 10 });
|
|
8
|
+
if (detected === "und") {
|
|
9
|
+
return "en";
|
|
10
|
+
}
|
|
11
|
+
return iso6393To1[detected] || "en";
|
|
12
|
+
}
|
|
13
|
+
export function getLanguageName(code) {
|
|
14
|
+
const lang = iso6393.find((l) => l.iso6391 === code);
|
|
15
|
+
return lang?.name || "English";
|
|
16
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAqDA,wBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,QAQlD"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { appendFileSync, writeFileSync, existsSync, mkdirSync, statSync, renameSync, unlinkSync, } from "fs";
|
|
2
|
+
import { homedir } from "os";
|
|
3
|
+
import { join } from "path";
|
|
4
|
+
function getLogFilePath() {
|
|
5
|
+
return process.env.OPENCODE_MEM_LOG_FILE || join(homedir(), ".opencode-mem", "opencode-mem.log");
|
|
6
|
+
}
|
|
7
|
+
function getLogDirPath() {
|
|
8
|
+
const logFile = getLogFilePath();
|
|
9
|
+
const lastSlash = Math.max(logFile.lastIndexOf("/"), logFile.lastIndexOf("\\"));
|
|
10
|
+
return lastSlash === -1 ? "." : logFile.slice(0, lastSlash);
|
|
11
|
+
}
|
|
12
|
+
const MAX_LOG_SIZE = 5 * 1024 * 1024;
|
|
13
|
+
const GLOBAL_LOGGER_KEY = Symbol.for("opencode-mem.logger.initialized");
|
|
14
|
+
function rotateLog() {
|
|
15
|
+
const logFile = getLogFilePath();
|
|
16
|
+
try {
|
|
17
|
+
if (!existsSync(logFile))
|
|
18
|
+
return;
|
|
19
|
+
const stats = statSync(logFile);
|
|
20
|
+
if (stats.size < MAX_LOG_SIZE)
|
|
21
|
+
return;
|
|
22
|
+
const oldLog = logFile + ".old";
|
|
23
|
+
if (existsSync(oldLog))
|
|
24
|
+
unlinkSync(oldLog);
|
|
25
|
+
renameSync(logFile, oldLog);
|
|
26
|
+
}
|
|
27
|
+
catch { }
|
|
28
|
+
}
|
|
29
|
+
function ensureLoggerInitialized() {
|
|
30
|
+
if (globalThis[GLOBAL_LOGGER_KEY])
|
|
31
|
+
return;
|
|
32
|
+
const logDir = getLogDirPath();
|
|
33
|
+
const logFile = getLogFilePath();
|
|
34
|
+
if (!existsSync(logDir)) {
|
|
35
|
+
mkdirSync(logDir, { recursive: true });
|
|
36
|
+
}
|
|
37
|
+
rotateLog();
|
|
38
|
+
writeFileSync(logFile, `\n--- Session started: ${new Date().toISOString()} ---\n`, {
|
|
39
|
+
flag: "a",
|
|
40
|
+
});
|
|
41
|
+
globalThis[GLOBAL_LOGGER_KEY] = true;
|
|
42
|
+
}
|
|
43
|
+
export function log(message, data) {
|
|
44
|
+
ensureLoggerInitialized();
|
|
45
|
+
const logFile = getLogFilePath();
|
|
46
|
+
const timestamp = new Date().toISOString();
|
|
47
|
+
const line = data
|
|
48
|
+
? `[${timestamp}] ${message}: ${JSON.stringify(data)}\n`
|
|
49
|
+
: `[${timestamp}] ${message}\n`;
|
|
50
|
+
appendFileSync(logFile, line);
|
|
51
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface DimensionMismatch {
|
|
2
|
+
needsMigration: boolean;
|
|
3
|
+
configDimensions: number;
|
|
4
|
+
configModel: string;
|
|
5
|
+
shardMismatches: Array<{
|
|
6
|
+
shardId: number;
|
|
7
|
+
dbPath: string;
|
|
8
|
+
storedDimensions: number;
|
|
9
|
+
storedModel: string;
|
|
10
|
+
vectorCount: number;
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
13
|
+
export interface MigrationProgress {
|
|
14
|
+
phase: "preparing" | "re-embedding" | "cleanup" | "complete";
|
|
15
|
+
processed: number;
|
|
16
|
+
total: number;
|
|
17
|
+
currentShard?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface MigrationResult {
|
|
20
|
+
success: boolean;
|
|
21
|
+
strategy: "fresh-start" | "re-embed";
|
|
22
|
+
deletedShards: number;
|
|
23
|
+
reEmbeddedMemories: number;
|
|
24
|
+
duration: number;
|
|
25
|
+
error?: string;
|
|
26
|
+
}
|
|
27
|
+
export declare class MigrationService {
|
|
28
|
+
private isRunning;
|
|
29
|
+
private progressCallback?;
|
|
30
|
+
detectDimensionMismatch(): Promise<DimensionMismatch>;
|
|
31
|
+
migrateToNewModel(strategy: "fresh-start" | "re-embed", progressCallback?: (progress: MigrationProgress) => void): Promise<MigrationResult>;
|
|
32
|
+
private freshStartMigration;
|
|
33
|
+
private reEmbedMigration;
|
|
34
|
+
private reportProgress;
|
|
35
|
+
getStatus(): {
|
|
36
|
+
isRunning: boolean;
|
|
37
|
+
configModel: string;
|
|
38
|
+
configDimensions: number;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export declare const migrationService: MigrationService;
|
|
42
|
+
//# sourceMappingURL=migration-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migration-service.d.ts","sourceRoot":"","sources":["../../src/services/migration-service.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,KAAK,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,WAAW,GAAG,cAAc,GAAG,SAAS,GAAG,UAAU,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,UAAU,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,gBAAgB,CAAC,CAAwC;IAE3D,uBAAuB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAoDrD,iBAAiB,CACrB,QAAQ,EAAE,aAAa,GAAG,UAAU,EACpC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,GACvD,OAAO,CAAC,eAAe,CAAC;YA2Cb,mBAAmB;YA8CnB,gBAAgB;IA+I9B,OAAO,CAAC,cAAc;IAMtB,SAAS;;;;;CAOV;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import { shardManager } from "./sqlite/shard-manager.js";
|
|
2
|
+
import { connectionManager } from "./sqlite/connection-manager.js";
|
|
3
|
+
import { vectorSearch } from "./sqlite/vector-search.js";
|
|
4
|
+
import { embeddingService } from "./embedding.js";
|
|
5
|
+
import { CONFIG } from "../config.js";
|
|
6
|
+
import { log } from "./logger.js";
|
|
7
|
+
export class MigrationService {
|
|
8
|
+
isRunning = false;
|
|
9
|
+
progressCallback;
|
|
10
|
+
async detectDimensionMismatch() {
|
|
11
|
+
const userShards = shardManager.getAllShards("user", "");
|
|
12
|
+
const projectShards = shardManager.getAllShards("project", "");
|
|
13
|
+
const allShards = [...userShards, ...projectShards];
|
|
14
|
+
const mismatches = [];
|
|
15
|
+
for (const shard of allShards) {
|
|
16
|
+
try {
|
|
17
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
18
|
+
const metadataResult = db
|
|
19
|
+
.prepare(`
|
|
20
|
+
SELECT key, value FROM shard_metadata
|
|
21
|
+
WHERE key IN ('embedding_dimensions', 'embedding_model')
|
|
22
|
+
`)
|
|
23
|
+
.all();
|
|
24
|
+
const metadata = Object.fromEntries(metadataResult.map((row) => [row.key, row.value]));
|
|
25
|
+
const storedDimensions = parseInt(metadata.embedding_dimensions || "0");
|
|
26
|
+
const storedModel = metadata.embedding_model || "unknown";
|
|
27
|
+
if (storedDimensions !== CONFIG.embeddingDimensions) {
|
|
28
|
+
const vectorCount = vectorSearch.countAllVectors(db);
|
|
29
|
+
mismatches.push({
|
|
30
|
+
shardId: shard.id,
|
|
31
|
+
dbPath: shard.dbPath,
|
|
32
|
+
storedDimensions,
|
|
33
|
+
storedModel,
|
|
34
|
+
vectorCount,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
log("Migration: error checking shard", {
|
|
40
|
+
shardId: shard.id,
|
|
41
|
+
error: String(error),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
needsMigration: mismatches.length > 0,
|
|
47
|
+
configDimensions: CONFIG.embeddingDimensions,
|
|
48
|
+
configModel: CONFIG.embeddingModel,
|
|
49
|
+
shardMismatches: mismatches,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
async migrateToNewModel(strategy, progressCallback) {
|
|
53
|
+
if (this.isRunning) {
|
|
54
|
+
throw new Error("Migration already running");
|
|
55
|
+
}
|
|
56
|
+
this.isRunning = true;
|
|
57
|
+
this.progressCallback = progressCallback;
|
|
58
|
+
const startTime = Date.now();
|
|
59
|
+
try {
|
|
60
|
+
const mismatch = await this.detectDimensionMismatch();
|
|
61
|
+
if (!mismatch.needsMigration) {
|
|
62
|
+
return {
|
|
63
|
+
success: true,
|
|
64
|
+
strategy,
|
|
65
|
+
deletedShards: 0,
|
|
66
|
+
reEmbeddedMemories: 0,
|
|
67
|
+
duration: Date.now() - startTime,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
if (strategy === "fresh-start") {
|
|
71
|
+
return await this.freshStartMigration(mismatch, startTime);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
return await this.reEmbedMigration(mismatch, startTime);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
log("Migration: failed", { error: String(error) });
|
|
79
|
+
return {
|
|
80
|
+
success: false,
|
|
81
|
+
strategy,
|
|
82
|
+
deletedShards: 0,
|
|
83
|
+
reEmbeddedMemories: 0,
|
|
84
|
+
duration: Date.now() - startTime,
|
|
85
|
+
error: String(error),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
this.isRunning = false;
|
|
90
|
+
this.progressCallback = undefined;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async freshStartMigration(mismatch, startTime) {
|
|
94
|
+
this.reportProgress({
|
|
95
|
+
phase: "preparing",
|
|
96
|
+
processed: 0,
|
|
97
|
+
total: mismatch.shardMismatches.length,
|
|
98
|
+
});
|
|
99
|
+
let deletedShards = 0;
|
|
100
|
+
for (const [index, shardInfo] of mismatch.shardMismatches.entries()) {
|
|
101
|
+
try {
|
|
102
|
+
this.reportProgress({
|
|
103
|
+
phase: "cleanup",
|
|
104
|
+
processed: index,
|
|
105
|
+
total: mismatch.shardMismatches.length,
|
|
106
|
+
currentShard: String(shardInfo.shardId),
|
|
107
|
+
});
|
|
108
|
+
await shardManager.deleteShard(shardInfo.shardId);
|
|
109
|
+
deletedShards++;
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
log("Migration: error deleting shard", {
|
|
113
|
+
shardId: shardInfo.shardId,
|
|
114
|
+
error: String(error),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
this.reportProgress({
|
|
119
|
+
phase: "complete",
|
|
120
|
+
processed: mismatch.shardMismatches.length,
|
|
121
|
+
total: mismatch.shardMismatches.length,
|
|
122
|
+
});
|
|
123
|
+
return {
|
|
124
|
+
success: true,
|
|
125
|
+
strategy: "fresh-start",
|
|
126
|
+
deletedShards,
|
|
127
|
+
reEmbeddedMemories: 0,
|
|
128
|
+
duration: Date.now() - startTime,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
async reEmbedMigration(mismatch, startTime) {
|
|
132
|
+
await embeddingService.warmup();
|
|
133
|
+
embeddingService.clearCache();
|
|
134
|
+
const totalMemories = mismatch.shardMismatches.reduce((sum, s) => sum + s.vectorCount, 0);
|
|
135
|
+
this.reportProgress({
|
|
136
|
+
phase: "preparing",
|
|
137
|
+
processed: 0,
|
|
138
|
+
total: totalMemories,
|
|
139
|
+
});
|
|
140
|
+
let reEmbeddedCount = 0;
|
|
141
|
+
let processedCount = 0;
|
|
142
|
+
for (const shardInfo of mismatch.shardMismatches) {
|
|
143
|
+
this.reportProgress({
|
|
144
|
+
phase: "re-embedding",
|
|
145
|
+
processed: processedCount,
|
|
146
|
+
total: totalMemories,
|
|
147
|
+
currentShard: String(shardInfo.shardId),
|
|
148
|
+
});
|
|
149
|
+
try {
|
|
150
|
+
const db = connectionManager.getConnection(shardInfo.dbPath);
|
|
151
|
+
const memories = vectorSearch.getAllMemories(db);
|
|
152
|
+
const tempMemories = [];
|
|
153
|
+
for (const memory of memories) {
|
|
154
|
+
tempMemories.push({
|
|
155
|
+
id: memory.id,
|
|
156
|
+
content: memory.content,
|
|
157
|
+
containerTag: memory.container_tag,
|
|
158
|
+
type: memory.type,
|
|
159
|
+
createdAt: memory.created_at,
|
|
160
|
+
updatedAt: memory.updated_at,
|
|
161
|
+
metadata: memory.metadata,
|
|
162
|
+
displayName: memory.display_name,
|
|
163
|
+
userName: memory.user_name,
|
|
164
|
+
userEmail: memory.user_email,
|
|
165
|
+
projectPath: memory.project_path,
|
|
166
|
+
projectName: memory.project_name,
|
|
167
|
+
gitRepoUrl: memory.git_repo_url,
|
|
168
|
+
isPinned: memory.is_pinned || 0,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
await shardManager.deleteShard(shardInfo.shardId);
|
|
172
|
+
for (const memory of tempMemories) {
|
|
173
|
+
try {
|
|
174
|
+
const vector = await embeddingService.embedWithTimeout(memory.content);
|
|
175
|
+
const scope = memory.containerTag.includes("_user_") ? "user" : "project";
|
|
176
|
+
const hash = memory.containerTag.split("_").slice(2).join("_");
|
|
177
|
+
const newShard = shardManager.getWriteShard(scope, hash);
|
|
178
|
+
const newDb = connectionManager.getConnection(newShard.dbPath);
|
|
179
|
+
await vectorSearch.insertVector(newDb, {
|
|
180
|
+
id: memory.id,
|
|
181
|
+
content: memory.content,
|
|
182
|
+
vector,
|
|
183
|
+
containerTag: memory.containerTag,
|
|
184
|
+
type: memory.type || undefined,
|
|
185
|
+
createdAt: memory.createdAt,
|
|
186
|
+
updatedAt: memory.updatedAt,
|
|
187
|
+
metadata: memory.metadata || undefined,
|
|
188
|
+
displayName: memory.displayName || undefined,
|
|
189
|
+
userName: memory.userName || undefined,
|
|
190
|
+
userEmail: memory.userEmail || undefined,
|
|
191
|
+
projectPath: memory.projectPath || undefined,
|
|
192
|
+
projectName: memory.projectName || undefined,
|
|
193
|
+
gitRepoUrl: memory.gitRepoUrl || undefined,
|
|
194
|
+
}, newShard);
|
|
195
|
+
if (memory.isPinned === 1) {
|
|
196
|
+
vectorSearch.pinMemory(newDb, memory.id);
|
|
197
|
+
}
|
|
198
|
+
shardManager.incrementVectorCount(newShard.id);
|
|
199
|
+
reEmbeddedCount++;
|
|
200
|
+
processedCount++;
|
|
201
|
+
this.reportProgress({
|
|
202
|
+
phase: "re-embedding",
|
|
203
|
+
processed: processedCount,
|
|
204
|
+
total: totalMemories,
|
|
205
|
+
currentShard: String(shardInfo.shardId),
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
log("Migration: error re-embedding memory", {
|
|
210
|
+
memoryId: memory.id,
|
|
211
|
+
error: String(error),
|
|
212
|
+
});
|
|
213
|
+
processedCount++;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
log("Migration: error processing shard", {
|
|
219
|
+
shardId: shardInfo.shardId,
|
|
220
|
+
error: String(error),
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
this.reportProgress({
|
|
225
|
+
phase: "complete",
|
|
226
|
+
processed: totalMemories,
|
|
227
|
+
total: totalMemories,
|
|
228
|
+
});
|
|
229
|
+
return {
|
|
230
|
+
success: true,
|
|
231
|
+
strategy: "re-embed",
|
|
232
|
+
deletedShards: mismatch.shardMismatches.length,
|
|
233
|
+
reEmbeddedMemories: reEmbeddedCount,
|
|
234
|
+
duration: Date.now() - startTime,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
reportProgress(progress) {
|
|
238
|
+
if (this.progressCallback) {
|
|
239
|
+
this.progressCallback(progress);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
getStatus() {
|
|
243
|
+
return {
|
|
244
|
+
isRunning: this.isRunning,
|
|
245
|
+
configModel: CONFIG.embeddingModel,
|
|
246
|
+
configDimensions: CONFIG.embeddingDimensions,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
export const migrationService = new MigrationService();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"privacy.d.ts","sourceRoot":"","sources":["../../src/services/privacy.ts"],"names":[],"mappings":"AAAA,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAGvD"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export function stripPrivateContent(content) {
|
|
2
|
+
return content.replace(/<private>[\s\S]*?<\/private>/gi, "[REDACTED]");
|
|
3
|
+
}
|
|
4
|
+
export function isFullyPrivate(content) {
|
|
5
|
+
const stripped = stripPrivateContent(content).trim();
|
|
6
|
+
return stripped === "[REDACTED]" || stripped === "";
|
|
7
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"secret-resolver.d.ts","sourceRoot":"","sources":["../../src/services/secret-resolver.ts"],"names":[],"mappings":"AAiCA,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAkChF"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { homedir, platform } from "node:os";
|
|
4
|
+
function expandPath(path) {
|
|
5
|
+
if (path.startsWith("~/")) {
|
|
6
|
+
return join(homedir(), path.slice(2));
|
|
7
|
+
}
|
|
8
|
+
if (path === "~") {
|
|
9
|
+
return homedir();
|
|
10
|
+
}
|
|
11
|
+
return path;
|
|
12
|
+
}
|
|
13
|
+
function checkFilePermissions(filePath) {
|
|
14
|
+
if (platform() === "win32") {
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
const stats = statSync(filePath);
|
|
19
|
+
const mode = stats.mode & 0o777;
|
|
20
|
+
if (mode > 0o600) {
|
|
21
|
+
console.warn(`Warning: Secret file ${filePath} has permissive permissions (${mode.toString(8)}). Recommend chmod 600.`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
console.warn(`Warning: Could not check file permissions for ${filePath}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export function resolveSecretValue(value) {
|
|
29
|
+
if (!value) {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
if (value.startsWith("file://")) {
|
|
33
|
+
const filePath = expandPath(value.slice(7));
|
|
34
|
+
if (!existsSync(filePath)) {
|
|
35
|
+
throw new Error(`Secret file not found: ${filePath}`);
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
checkFilePermissions(filePath);
|
|
39
|
+
const content = readFileSync(filePath, "utf-8");
|
|
40
|
+
return content.trim();
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw new Error(`Failed to read secret file ${filePath}: ${error}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (value.startsWith("env://")) {
|
|
47
|
+
const envVar = value.slice(6);
|
|
48
|
+
const envValue = process.env[envVar];
|
|
49
|
+
if (!envValue) {
|
|
50
|
+
throw new Error(`Environment variable not found: ${envVar}`);
|
|
51
|
+
}
|
|
52
|
+
return envValue;
|
|
53
|
+
}
|
|
54
|
+
return value;
|
|
55
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
declare const Database: typeof import("bun:sqlite").Database;
|
|
2
|
+
export declare class ConnectionManager {
|
|
3
|
+
private connections;
|
|
4
|
+
private initDatabase;
|
|
5
|
+
private migrateSchema;
|
|
6
|
+
getConnection(dbPath: string): typeof Database.prototype;
|
|
7
|
+
closeConnection(dbPath: string): void;
|
|
8
|
+
closeAll(): void;
|
|
9
|
+
checkpointAll(): void;
|
|
10
|
+
}
|
|
11
|
+
export declare const connectionManager: ConnectionManager;
|
|
12
|
+
export {};
|
|
13
|
+
//# sourceMappingURL=connection-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/connection-manager.ts"],"names":[],"mappings":"AAMA,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAE/B,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,WAAW,CAAqD;IAExE,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,aAAa;IAarB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,QAAQ,CAAC,SAAS;IAiBxD,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IASrC,QAAQ,IAAI,IAAI;IAYhB,aAAa,IAAI,IAAI;CAStB;AAED,eAAO,MAAM,iBAAiB,mBAA0B,CAAC"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { getDatabase } from "./sqlite-bootstrap.js";
|
|
2
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { log } from "../logger.js";
|
|
5
|
+
import { CONFIG } from "../../config.js";
|
|
6
|
+
const Database = getDatabase();
|
|
7
|
+
export class ConnectionManager {
|
|
8
|
+
connections = new Map();
|
|
9
|
+
initDatabase(db) {
|
|
10
|
+
db.run("PRAGMA busy_timeout = 5000");
|
|
11
|
+
db.run("PRAGMA journal_mode = WAL");
|
|
12
|
+
db.run("PRAGMA synchronous = NORMAL");
|
|
13
|
+
db.run("PRAGMA cache_size = -64000");
|
|
14
|
+
db.run("PRAGMA temp_store = MEMORY");
|
|
15
|
+
db.run("PRAGMA foreign_keys = ON");
|
|
16
|
+
this.migrateSchema(db);
|
|
17
|
+
}
|
|
18
|
+
migrateSchema(db) {
|
|
19
|
+
try {
|
|
20
|
+
const columns = db.prepare("PRAGMA table_info(memories)").all();
|
|
21
|
+
const hasTags = columns.some((c) => c.name === "tags");
|
|
22
|
+
if (!hasTags && columns.length > 0) {
|
|
23
|
+
db.run("ALTER TABLE memories ADD COLUMN tags TEXT");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
log("Schema migration error", { error: String(error) });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
getConnection(dbPath) {
|
|
31
|
+
if (this.connections.has(dbPath)) {
|
|
32
|
+
return this.connections.get(dbPath);
|
|
33
|
+
}
|
|
34
|
+
const dir = dirname(dbPath);
|
|
35
|
+
if (!existsSync(dir)) {
|
|
36
|
+
mkdirSync(dir, { recursive: true });
|
|
37
|
+
}
|
|
38
|
+
const db = new Database(dbPath);
|
|
39
|
+
this.initDatabase(db);
|
|
40
|
+
this.connections.set(dbPath, db);
|
|
41
|
+
return db;
|
|
42
|
+
}
|
|
43
|
+
closeConnection(dbPath) {
|
|
44
|
+
const db = this.connections.get(dbPath);
|
|
45
|
+
if (db) {
|
|
46
|
+
db.run("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
47
|
+
db.close();
|
|
48
|
+
this.connections.delete(dbPath);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
closeAll() {
|
|
52
|
+
for (const [path, db] of this.connections) {
|
|
53
|
+
try {
|
|
54
|
+
db.run("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
55
|
+
db.close();
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
log("Error closing database", { path, error: String(error) });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
this.connections.clear();
|
|
62
|
+
}
|
|
63
|
+
checkpointAll() {
|
|
64
|
+
for (const [path, db] of this.connections) {
|
|
65
|
+
try {
|
|
66
|
+
db.run("PRAGMA wal_checkpoint(PASSIVE)");
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
log("Error checkpointing database", { path, error: String(error) });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export const connectionManager = new ConnectionManager();
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { ShardInfo } from "./types.js";
|
|
2
|
+
export declare class ShardManager {
|
|
3
|
+
private metadataDb;
|
|
4
|
+
private metadataPath;
|
|
5
|
+
constructor();
|
|
6
|
+
private initMetadataDb;
|
|
7
|
+
private getShardPath;
|
|
8
|
+
private resolveStoredPath;
|
|
9
|
+
getActiveShard(scope: "user" | "project", scopeHash: string): ShardInfo | null;
|
|
10
|
+
getAllShards(scope: "user" | "project", scopeHash: string): ShardInfo[];
|
|
11
|
+
createShard(scope: "user" | "project", scopeHash: string, shardIndex: number): ShardInfo;
|
|
12
|
+
private initShardDb;
|
|
13
|
+
private isShardValid;
|
|
14
|
+
private ensureShardTables;
|
|
15
|
+
getWriteShard(scope: "user" | "project", scopeHash: string): ShardInfo;
|
|
16
|
+
private markShardReadOnly;
|
|
17
|
+
incrementVectorCount(shardId: number): void;
|
|
18
|
+
decrementVectorCount(shardId: number): void;
|
|
19
|
+
getShardByPath(dbPath: string): ShardInfo | null;
|
|
20
|
+
deleteShard(shardId: number): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
export declare const shardManager: ShardManager;
|
|
23
|
+
//# sourceMappingURL=shard-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/shard-manager.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAO5C,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAe;IACjC,OAAO,CAAC,YAAY,CAAS;;IAQ7B,OAAO,CAAC,cAAc;IAqBtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,iBAAiB;IAKzB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsB9E,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE;IAgCvE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IA2BxF,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,YAAY;IA4BpB,OAAO,CAAC,iBAAiB;IAYzB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS;IA+BtE,OAAO,CAAC,iBAAiB;IAOzB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAkB1C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAkClD;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|