opencode-mem 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +588 -0
- package/dist/config.d.ts +33 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +258 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +618 -0
- package/dist/plugin.d.ts +5 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +15 -0
- package/dist/services/api-handlers.d.ts +102 -0
- package/dist/services/api-handlers.d.ts.map +1 -0
- package/dist/services/api-handlers.js +494 -0
- package/dist/services/auto-capture.d.ts +32 -0
- package/dist/services/auto-capture.d.ts.map +1 -0
- package/dist/services/auto-capture.js +451 -0
- package/dist/services/cleanup-service.d.ts +20 -0
- package/dist/services/cleanup-service.d.ts.map +1 -0
- package/dist/services/cleanup-service.js +88 -0
- package/dist/services/client.d.ts +104 -0
- package/dist/services/client.d.ts.map +1 -0
- package/dist/services/client.js +251 -0
- package/dist/services/compaction.d.ts +92 -0
- package/dist/services/compaction.d.ts.map +1 -0
- package/dist/services/compaction.js +421 -0
- package/dist/services/context.d.ts +17 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +41 -0
- package/dist/services/deduplication-service.d.ts +30 -0
- package/dist/services/deduplication-service.d.ts.map +1 -0
- package/dist/services/deduplication-service.js +131 -0
- package/dist/services/embedding.d.ts +10 -0
- package/dist/services/embedding.d.ts.map +1 -0
- package/dist/services/embedding.js +77 -0
- package/dist/services/jsonc.d.ts +7 -0
- package/dist/services/jsonc.d.ts.map +1 -0
- package/dist/services/jsonc.js +76 -0
- package/dist/services/logger.d.ts +2 -0
- package/dist/services/logger.d.ts.map +1 -0
- package/dist/services/logger.js +16 -0
- package/dist/services/migration-service.d.ts +42 -0
- package/dist/services/migration-service.d.ts.map +1 -0
- package/dist/services/migration-service.js +258 -0
- package/dist/services/privacy.d.ts +4 -0
- package/dist/services/privacy.d.ts.map +1 -0
- package/dist/services/privacy.js +10 -0
- package/dist/services/sqlite/connection-manager.d.ts +10 -0
- package/dist/services/sqlite/connection-manager.d.ts.map +1 -0
- package/dist/services/sqlite/connection-manager.js +45 -0
- package/dist/services/sqlite/shard-manager.d.ts +20 -0
- package/dist/services/sqlite/shard-manager.d.ts.map +1 -0
- package/dist/services/sqlite/shard-manager.js +221 -0
- package/dist/services/sqlite/types.d.ts +39 -0
- package/dist/services/sqlite/types.d.ts.map +1 -0
- package/dist/services/sqlite/types.js +1 -0
- package/dist/services/sqlite/vector-search.d.ts +18 -0
- package/dist/services/sqlite/vector-search.d.ts.map +1 -0
- package/dist/services/sqlite/vector-search.js +129 -0
- package/dist/services/sqlite-client.d.ts +116 -0
- package/dist/services/sqlite-client.d.ts.map +1 -0
- package/dist/services/sqlite-client.js +284 -0
- package/dist/services/tags.d.ts +20 -0
- package/dist/services/tags.d.ts.map +1 -0
- package/dist/services/tags.js +76 -0
- package/dist/services/web-server-lock.d.ts +12 -0
- package/dist/services/web-server-lock.d.ts.map +1 -0
- package/dist/services/web-server-lock.js +157 -0
- package/dist/services/web-server-worker.d.ts +2 -0
- package/dist/services/web-server-worker.d.ts.map +1 -0
- package/dist/services/web-server-worker.js +221 -0
- package/dist/services/web-server.d.ts +22 -0
- package/dist/services/web-server.d.ts.map +1 -0
- package/dist/services/web-server.js +134 -0
- package/dist/types/index.d.ts +48 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/web/app.d.ts +2 -0
- package/dist/web/app.d.ts.map +1 -0
- package/dist/web/app.js +691 -0
- package/dist/web/favicon.ico +0 -0
- package/dist/web/favicon.svg +14 -0
- package/dist/web/index.html +202 -0
- package/dist/web/styles.css +851 -0
- package/package.json +52 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { pipeline, env } from "@xenova/transformers";
|
|
2
|
+
import { CONFIG } from "../config.js";
|
|
3
|
+
import { log } from "./logger.js";
|
|
4
|
+
env.allowLocalModels = true;
|
|
5
|
+
env.allowRemoteModels = true;
|
|
6
|
+
env.cacheDir = CONFIG.storagePath + "/.cache";
|
|
7
|
+
const TIMEOUT_MS = 30000;
|
|
8
|
+
function withTimeout(promise, ms) {
|
|
9
|
+
return Promise.race([
|
|
10
|
+
promise,
|
|
11
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms)),
|
|
12
|
+
]);
|
|
13
|
+
}
|
|
14
|
+
export class EmbeddingService {
|
|
15
|
+
pipe = null;
|
|
16
|
+
initPromise = null;
|
|
17
|
+
isWarmedUp = false;
|
|
18
|
+
async warmup(progressCallback) {
|
|
19
|
+
if (this.isWarmedUp)
|
|
20
|
+
return;
|
|
21
|
+
if (this.initPromise)
|
|
22
|
+
return this.initPromise;
|
|
23
|
+
this.initPromise = (async () => {
|
|
24
|
+
try {
|
|
25
|
+
if (CONFIG.embeddingApiUrl && CONFIG.embeddingApiKey) {
|
|
26
|
+
log("Using OpenAI-compatible API for embeddings");
|
|
27
|
+
this.isWarmedUp = true;
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
log("Downloading embedding model", { model: CONFIG.embeddingModel });
|
|
31
|
+
this.pipe = await pipeline("feature-extraction", CONFIG.embeddingModel, {
|
|
32
|
+
progress_callback: progressCallback,
|
|
33
|
+
});
|
|
34
|
+
this.isWarmedUp = true;
|
|
35
|
+
log("Embedding model ready");
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
this.initPromise = null;
|
|
39
|
+
log("Failed to initialize embedding model", { error: String(error) });
|
|
40
|
+
throw error;
|
|
41
|
+
}
|
|
42
|
+
})();
|
|
43
|
+
return this.initPromise;
|
|
44
|
+
}
|
|
45
|
+
async embed(text) {
|
|
46
|
+
if (!this.isWarmedUp && !this.initPromise) {
|
|
47
|
+
await this.warmup();
|
|
48
|
+
}
|
|
49
|
+
if (this.initPromise) {
|
|
50
|
+
await this.initPromise;
|
|
51
|
+
}
|
|
52
|
+
if (CONFIG.embeddingApiUrl && CONFIG.embeddingApiKey) {
|
|
53
|
+
const response = await fetch(`${CONFIG.embeddingApiUrl}/embeddings`, {
|
|
54
|
+
method: "POST",
|
|
55
|
+
headers: {
|
|
56
|
+
"Content-Type": "application/json",
|
|
57
|
+
Authorization: `Bearer ${CONFIG.embeddingApiKey}`,
|
|
58
|
+
},
|
|
59
|
+
body: JSON.stringify({
|
|
60
|
+
input: text,
|
|
61
|
+
model: CONFIG.embeddingModel,
|
|
62
|
+
}),
|
|
63
|
+
});
|
|
64
|
+
if (!response.ok) {
|
|
65
|
+
throw new Error(`API embedding failed: ${response.statusText}`);
|
|
66
|
+
}
|
|
67
|
+
const data = await response.json();
|
|
68
|
+
return new Float32Array(data.data[0].embedding);
|
|
69
|
+
}
|
|
70
|
+
const output = await this.pipe(text, { pooling: "mean", normalize: true });
|
|
71
|
+
return new Float32Array(output.data);
|
|
72
|
+
}
|
|
73
|
+
async embedWithTimeout(text) {
|
|
74
|
+
return withTimeout(this.embed(text), TIMEOUT_MS);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
export const embeddingService = new EmbeddingService();
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strips comments from JSONC content while respecting string boundaries.
|
|
3
|
+
* Handles // and /* comments, URLs in strings, and escaped quotes.
|
|
4
|
+
* Also removes trailing commas to support more relaxed JSONC format.
|
|
5
|
+
*/
|
|
6
|
+
export declare function stripJsoncComments(content: string): string;
|
|
7
|
+
//# sourceMappingURL=jsonc.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonc.d.ts","sourceRoot":"","sources":["../../src/services/jsonc.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CA+E1D"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strips comments from JSONC content while respecting string boundaries.
|
|
3
|
+
* Handles // and /* comments, URLs in strings, and escaped quotes.
|
|
4
|
+
* Also removes trailing commas to support more relaxed JSONC format.
|
|
5
|
+
*/
|
|
6
|
+
export function stripJsoncComments(content) {
|
|
7
|
+
let result = "";
|
|
8
|
+
let i = 0;
|
|
9
|
+
let inString = false;
|
|
10
|
+
let inSingleLineComment = false;
|
|
11
|
+
let inMultiLineComment = false;
|
|
12
|
+
while (i < content.length) {
|
|
13
|
+
const char = content[i];
|
|
14
|
+
const nextChar = content[i + 1];
|
|
15
|
+
if (!inSingleLineComment && !inMultiLineComment) {
|
|
16
|
+
if (char === '"') {
|
|
17
|
+
// Count consecutive backslashes before this quote
|
|
18
|
+
let backslashCount = 0;
|
|
19
|
+
let j = i - 1;
|
|
20
|
+
while (j >= 0 && content[j] === "\\") {
|
|
21
|
+
backslashCount++;
|
|
22
|
+
j--;
|
|
23
|
+
}
|
|
24
|
+
// Quote is escaped only if preceded by ODD number of backslashes
|
|
25
|
+
// e.g., \" = escaped, \\" = not escaped (escaped backslash + quote)
|
|
26
|
+
if (backslashCount % 2 === 0) {
|
|
27
|
+
inString = !inString;
|
|
28
|
+
}
|
|
29
|
+
result += char;
|
|
30
|
+
i++;
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (inString) {
|
|
35
|
+
result += char;
|
|
36
|
+
i++;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (!inSingleLineComment && !inMultiLineComment) {
|
|
40
|
+
if (char === "/" && nextChar === "/") {
|
|
41
|
+
inSingleLineComment = true;
|
|
42
|
+
i += 2;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (char === "/" && nextChar === "*") {
|
|
46
|
+
inMultiLineComment = true;
|
|
47
|
+
i += 2;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (inSingleLineComment) {
|
|
52
|
+
if (char === "\n") {
|
|
53
|
+
inSingleLineComment = false;
|
|
54
|
+
result += char;
|
|
55
|
+
}
|
|
56
|
+
i++;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (inMultiLineComment) {
|
|
60
|
+
if (char === "*" && nextChar === "/") {
|
|
61
|
+
inMultiLineComment = false;
|
|
62
|
+
i += 2;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (char === "\n") {
|
|
66
|
+
result += char;
|
|
67
|
+
}
|
|
68
|
+
i++;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
result += char;
|
|
72
|
+
i++;
|
|
73
|
+
}
|
|
74
|
+
// Remove trailing commas before } or ]
|
|
75
|
+
return result.replace(/,\s*([}\]])/g, "$1");
|
|
76
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/services/logger.ts"],"names":[],"mappings":"AAaA,wBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,QAMlD"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { appendFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
|
|
2
|
+
import { homedir } from "os";
|
|
3
|
+
import { join, dirname } from "path";
|
|
4
|
+
const LOG_DIR = join(homedir(), ".opencode-mem");
|
|
5
|
+
const LOG_FILE = join(LOG_DIR, "opencode-mem.log");
|
|
6
|
+
if (!existsSync(LOG_DIR)) {
|
|
7
|
+
mkdirSync(LOG_DIR, { recursive: true });
|
|
8
|
+
}
|
|
9
|
+
writeFileSync(LOG_FILE, `\n--- Session started: ${new Date().toISOString()} ---\n`, { flag: "a" });
|
|
10
|
+
export function log(message, data) {
|
|
11
|
+
const timestamp = new Date().toISOString();
|
|
12
|
+
const line = data
|
|
13
|
+
? `[${timestamp}] ${message}: ${JSON.stringify(data)}\n`
|
|
14
|
+
: `[${timestamp}] ${message}\n`;
|
|
15
|
+
appendFileSync(LOG_FILE, line);
|
|
16
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface DimensionMismatch {
|
|
2
|
+
needsMigration: boolean;
|
|
3
|
+
configDimensions: number;
|
|
4
|
+
configModel: string;
|
|
5
|
+
shardMismatches: Array<{
|
|
6
|
+
shardId: number;
|
|
7
|
+
dbPath: string;
|
|
8
|
+
storedDimensions: number;
|
|
9
|
+
storedModel: string;
|
|
10
|
+
vectorCount: number;
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
13
|
+
export interface MigrationProgress {
|
|
14
|
+
phase: "preparing" | "re-embedding" | "cleanup" | "complete";
|
|
15
|
+
processed: number;
|
|
16
|
+
total: number;
|
|
17
|
+
currentShard?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface MigrationResult {
|
|
20
|
+
success: boolean;
|
|
21
|
+
strategy: "fresh-start" | "re-embed";
|
|
22
|
+
deletedShards: number;
|
|
23
|
+
reEmbeddedMemories: number;
|
|
24
|
+
duration: number;
|
|
25
|
+
error?: string;
|
|
26
|
+
}
|
|
27
|
+
export declare class MigrationService {
|
|
28
|
+
private isRunning;
|
|
29
|
+
private progressCallback?;
|
|
30
|
+
detectDimensionMismatch(): Promise<DimensionMismatch>;
|
|
31
|
+
migrateToNewModel(strategy: "fresh-start" | "re-embed", progressCallback?: (progress: MigrationProgress) => void): Promise<MigrationResult>;
|
|
32
|
+
private freshStartMigration;
|
|
33
|
+
private reEmbedMigration;
|
|
34
|
+
private reportProgress;
|
|
35
|
+
getStatus(): {
|
|
36
|
+
isRunning: boolean;
|
|
37
|
+
configModel: string;
|
|
38
|
+
configDimensions: number;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export declare const migrationService: MigrationService;
|
|
42
|
+
//# sourceMappingURL=migration-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migration-service.d.ts","sourceRoot":"","sources":["../../src/services/migration-service.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,KAAK,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,WAAW,GAAG,cAAc,GAAG,SAAS,GAAG,UAAU,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,UAAU,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,gBAAgB,CAAC,CAAwC;IAE3D,uBAAuB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAoDrD,iBAAiB,CACrB,QAAQ,EAAE,aAAa,GAAG,UAAU,EACpC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,GACvD,OAAO,CAAC,eAAe,CAAC;YA6Cb,mBAAmB;YAmDnB,gBAAgB;IA+I9B,OAAO,CAAC,cAAc;IAMtB,SAAS;;;;;CAOV;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import { shardManager } from "./sqlite/shard-manager.js";
|
|
2
|
+
import { connectionManager } from "./sqlite/connection-manager.js";
|
|
3
|
+
import { vectorSearch } from "./sqlite/vector-search.js";
|
|
4
|
+
import { embeddingService } from "./embedding.js";
|
|
5
|
+
import { CONFIG } from "../config.js";
|
|
6
|
+
import { log } from "./logger.js";
|
|
7
|
+
export class MigrationService {
|
|
8
|
+
isRunning = false;
|
|
9
|
+
progressCallback;
|
|
10
|
+
async detectDimensionMismatch() {
|
|
11
|
+
const userShards = shardManager.getAllShards("user", "");
|
|
12
|
+
const projectShards = shardManager.getAllShards("project", "");
|
|
13
|
+
const allShards = [...userShards, ...projectShards];
|
|
14
|
+
const mismatches = [];
|
|
15
|
+
for (const shard of allShards) {
|
|
16
|
+
try {
|
|
17
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
18
|
+
const metadataResult = db
|
|
19
|
+
.prepare(`
|
|
20
|
+
SELECT key, value FROM shard_metadata
|
|
21
|
+
WHERE key IN ('embedding_dimensions', 'embedding_model')
|
|
22
|
+
`)
|
|
23
|
+
.all();
|
|
24
|
+
const metadata = Object.fromEntries(metadataResult.map((row) => [row.key, row.value]));
|
|
25
|
+
const storedDimensions = parseInt(metadata.embedding_dimensions || "0");
|
|
26
|
+
const storedModel = metadata.embedding_model || "unknown";
|
|
27
|
+
if (storedDimensions !== CONFIG.embeddingDimensions) {
|
|
28
|
+
const vectorCount = vectorSearch.countAllVectors(db);
|
|
29
|
+
mismatches.push({
|
|
30
|
+
shardId: shard.id,
|
|
31
|
+
dbPath: shard.dbPath,
|
|
32
|
+
storedDimensions,
|
|
33
|
+
storedModel,
|
|
34
|
+
vectorCount,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
log("Migration: error checking shard", {
|
|
40
|
+
shardId: shard.id,
|
|
41
|
+
error: String(error),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
needsMigration: mismatches.length > 0,
|
|
47
|
+
configDimensions: CONFIG.embeddingDimensions,
|
|
48
|
+
configModel: CONFIG.embeddingModel,
|
|
49
|
+
shardMismatches: mismatches,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
async migrateToNewModel(strategy, progressCallback) {
|
|
53
|
+
if (this.isRunning) {
|
|
54
|
+
throw new Error("Migration already running");
|
|
55
|
+
}
|
|
56
|
+
this.isRunning = true;
|
|
57
|
+
this.progressCallback = progressCallback;
|
|
58
|
+
const startTime = Date.now();
|
|
59
|
+
try {
|
|
60
|
+
log("Migration: starting", { strategy, model: CONFIG.embeddingModel });
|
|
61
|
+
const mismatch = await this.detectDimensionMismatch();
|
|
62
|
+
if (!mismatch.needsMigration) {
|
|
63
|
+
return {
|
|
64
|
+
success: true,
|
|
65
|
+
strategy,
|
|
66
|
+
deletedShards: 0,
|
|
67
|
+
reEmbeddedMemories: 0,
|
|
68
|
+
duration: Date.now() - startTime,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
if (strategy === "fresh-start") {
|
|
72
|
+
return await this.freshStartMigration(mismatch, startTime);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
return await this.reEmbedMigration(mismatch, startTime);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
log("Migration: failed", { error: String(error) });
|
|
80
|
+
return {
|
|
81
|
+
success: false,
|
|
82
|
+
strategy,
|
|
83
|
+
deletedShards: 0,
|
|
84
|
+
reEmbeddedMemories: 0,
|
|
85
|
+
duration: Date.now() - startTime,
|
|
86
|
+
error: String(error),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
this.isRunning = false;
|
|
91
|
+
this.progressCallback = undefined;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
async freshStartMigration(mismatch, startTime) {
|
|
95
|
+
this.reportProgress({
|
|
96
|
+
phase: "preparing",
|
|
97
|
+
processed: 0,
|
|
98
|
+
total: mismatch.shardMismatches.length,
|
|
99
|
+
});
|
|
100
|
+
let deletedShards = 0;
|
|
101
|
+
for (const [index, shardInfo] of mismatch.shardMismatches.entries()) {
|
|
102
|
+
try {
|
|
103
|
+
this.reportProgress({
|
|
104
|
+
phase: "cleanup",
|
|
105
|
+
processed: index,
|
|
106
|
+
total: mismatch.shardMismatches.length,
|
|
107
|
+
currentShard: String(shardInfo.shardId),
|
|
108
|
+
});
|
|
109
|
+
shardManager.deleteShard(shardInfo.shardId);
|
|
110
|
+
deletedShards++;
|
|
111
|
+
log("Migration: deleted shard", {
|
|
112
|
+
shardId: shardInfo.shardId,
|
|
113
|
+
vectorCount: shardInfo.vectorCount,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
log("Migration: error deleting shard", {
|
|
118
|
+
shardId: shardInfo.shardId,
|
|
119
|
+
error: String(error),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
this.reportProgress({
|
|
124
|
+
phase: "complete",
|
|
125
|
+
processed: mismatch.shardMismatches.length,
|
|
126
|
+
total: mismatch.shardMismatches.length,
|
|
127
|
+
});
|
|
128
|
+
return {
|
|
129
|
+
success: true,
|
|
130
|
+
strategy: "fresh-start",
|
|
131
|
+
deletedShards,
|
|
132
|
+
reEmbeddedMemories: 0,
|
|
133
|
+
duration: Date.now() - startTime,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
async reEmbedMigration(mismatch, startTime) {
|
|
137
|
+
await embeddingService.warmup();
|
|
138
|
+
const totalMemories = mismatch.shardMismatches.reduce((sum, s) => sum + s.vectorCount, 0);
|
|
139
|
+
this.reportProgress({
|
|
140
|
+
phase: "preparing",
|
|
141
|
+
processed: 0,
|
|
142
|
+
total: totalMemories,
|
|
143
|
+
});
|
|
144
|
+
let reEmbeddedCount = 0;
|
|
145
|
+
let processedCount = 0;
|
|
146
|
+
for (const shardInfo of mismatch.shardMismatches) {
|
|
147
|
+
this.reportProgress({
|
|
148
|
+
phase: "re-embedding",
|
|
149
|
+
processed: processedCount,
|
|
150
|
+
total: totalMemories,
|
|
151
|
+
currentShard: String(shardInfo.shardId),
|
|
152
|
+
});
|
|
153
|
+
try {
|
|
154
|
+
const db = connectionManager.getConnection(shardInfo.dbPath);
|
|
155
|
+
const memories = vectorSearch.getAllMemories(db);
|
|
156
|
+
const tempMemories = [];
|
|
157
|
+
for (const memory of memories) {
|
|
158
|
+
tempMemories.push({
|
|
159
|
+
id: memory.id,
|
|
160
|
+
content: memory.content,
|
|
161
|
+
containerTag: memory.container_tag,
|
|
162
|
+
type: memory.type,
|
|
163
|
+
createdAt: memory.created_at,
|
|
164
|
+
updatedAt: memory.updated_at,
|
|
165
|
+
metadata: memory.metadata,
|
|
166
|
+
displayName: memory.display_name,
|
|
167
|
+
userName: memory.user_name,
|
|
168
|
+
userEmail: memory.user_email,
|
|
169
|
+
projectPath: memory.project_path,
|
|
170
|
+
projectName: memory.project_name,
|
|
171
|
+
gitRepoUrl: memory.git_repo_url,
|
|
172
|
+
isPinned: memory.is_pinned || 0,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
shardManager.deleteShard(shardInfo.shardId);
|
|
176
|
+
for (const memory of tempMemories) {
|
|
177
|
+
try {
|
|
178
|
+
const vector = await embeddingService.embedWithTimeout(memory.content);
|
|
179
|
+
const scope = memory.containerTag.includes("_user_") ? "user" : "project";
|
|
180
|
+
const hash = memory.containerTag.split("_").slice(2).join("_");
|
|
181
|
+
const newShard = shardManager.getWriteShard(scope, hash);
|
|
182
|
+
const newDb = connectionManager.getConnection(newShard.dbPath);
|
|
183
|
+
vectorSearch.insertVector(newDb, {
|
|
184
|
+
id: memory.id,
|
|
185
|
+
content: memory.content,
|
|
186
|
+
vector,
|
|
187
|
+
containerTag: memory.containerTag,
|
|
188
|
+
type: memory.type || undefined,
|
|
189
|
+
createdAt: memory.createdAt,
|
|
190
|
+
updatedAt: memory.updatedAt,
|
|
191
|
+
metadata: memory.metadata || undefined,
|
|
192
|
+
displayName: memory.displayName || undefined,
|
|
193
|
+
userName: memory.userName || undefined,
|
|
194
|
+
userEmail: memory.userEmail || undefined,
|
|
195
|
+
projectPath: memory.projectPath || undefined,
|
|
196
|
+
projectName: memory.projectName || undefined,
|
|
197
|
+
gitRepoUrl: memory.gitRepoUrl || undefined,
|
|
198
|
+
});
|
|
199
|
+
if (memory.isPinned === 1) {
|
|
200
|
+
vectorSearch.pinMemory(newDb, memory.id);
|
|
201
|
+
}
|
|
202
|
+
shardManager.incrementVectorCount(newShard.id);
|
|
203
|
+
reEmbeddedCount++;
|
|
204
|
+
processedCount++;
|
|
205
|
+
this.reportProgress({
|
|
206
|
+
phase: "re-embedding",
|
|
207
|
+
processed: processedCount,
|
|
208
|
+
total: totalMemories,
|
|
209
|
+
currentShard: String(shardInfo.shardId),
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
log("Migration: error re-embedding memory", {
|
|
214
|
+
memoryId: memory.id,
|
|
215
|
+
error: String(error),
|
|
216
|
+
});
|
|
217
|
+
processedCount++;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
log("Migration: re-embedded shard", {
|
|
221
|
+
shardId: shardInfo.shardId,
|
|
222
|
+
count: tempMemories.length,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
log("Migration: error processing shard", {
|
|
227
|
+
shardId: shardInfo.shardId,
|
|
228
|
+
error: String(error),
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
this.reportProgress({
|
|
233
|
+
phase: "complete",
|
|
234
|
+
processed: totalMemories,
|
|
235
|
+
total: totalMemories,
|
|
236
|
+
});
|
|
237
|
+
return {
|
|
238
|
+
success: true,
|
|
239
|
+
strategy: "re-embed",
|
|
240
|
+
deletedShards: mismatch.shardMismatches.length,
|
|
241
|
+
reEmbeddedMemories: reEmbeddedCount,
|
|
242
|
+
duration: Date.now() - startTime,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
reportProgress(progress) {
|
|
246
|
+
if (this.progressCallback) {
|
|
247
|
+
this.progressCallback(progress);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
getStatus() {
|
|
251
|
+
return {
|
|
252
|
+
isRunning: this.isRunning,
|
|
253
|
+
configModel: CONFIG.embeddingModel,
|
|
254
|
+
configDimensions: CONFIG.embeddingDimensions,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
export const migrationService = new MigrationService();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"privacy.d.ts","sourceRoot":"","sources":["../../src/services/privacy.ts"],"names":[],"mappings":"AAAA,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAE3D;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAGvD"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export function containsPrivateTag(content) {
|
|
2
|
+
return /<private>[\s\S]*?<\/private>/i.test(content);
|
|
3
|
+
}
|
|
4
|
+
export function stripPrivateContent(content) {
|
|
5
|
+
return content.replace(/<private>[\s\S]*?<\/private>/gi, "[REDACTED]");
|
|
6
|
+
}
|
|
7
|
+
export function isFullyPrivate(content) {
|
|
8
|
+
const stripped = stripPrivateContent(content).trim();
|
|
9
|
+
return stripped === "[REDACTED]" || stripped === "";
|
|
10
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
export declare class ConnectionManager {
|
|
3
|
+
private connections;
|
|
4
|
+
private initDatabase;
|
|
5
|
+
getConnection(dbPath: string): Database;
|
|
6
|
+
closeConnection(dbPath: string): void;
|
|
7
|
+
closeAll(): void;
|
|
8
|
+
}
|
|
9
|
+
export declare const connectionManager: ConnectionManager;
|
|
10
|
+
//# sourceMappingURL=connection-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/connection-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAKtC,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,WAAW,CAAoC;IAEvD,OAAO,CAAC,YAAY;IAUpB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ;IAkBvC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IASrC,QAAQ,IAAI,IAAI;CAOjB;AAED,eAAO,MAAM,iBAAiB,mBAA0B,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import * as sqliteVec from "sqlite-vec";
|
|
3
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
4
|
+
import { log } from "../logger.js";
|
|
5
|
+
export class ConnectionManager {
|
|
6
|
+
connections = new Map();
|
|
7
|
+
initDatabase(db) {
|
|
8
|
+
db.run("PRAGMA journal_mode = WAL");
|
|
9
|
+
db.run("PRAGMA synchronous = NORMAL");
|
|
10
|
+
db.run("PRAGMA cache_size = -64000");
|
|
11
|
+
db.run("PRAGMA temp_store = MEMORY");
|
|
12
|
+
db.run("PRAGMA foreign_keys = ON");
|
|
13
|
+
sqliteVec.load(db);
|
|
14
|
+
}
|
|
15
|
+
getConnection(dbPath) {
|
|
16
|
+
if (this.connections.has(dbPath)) {
|
|
17
|
+
return this.connections.get(dbPath);
|
|
18
|
+
}
|
|
19
|
+
const dir = dbPath.substring(0, dbPath.lastIndexOf("/"));
|
|
20
|
+
if (!existsSync(dir)) {
|
|
21
|
+
mkdirSync(dir, { recursive: true });
|
|
22
|
+
}
|
|
23
|
+
const db = new Database(dbPath);
|
|
24
|
+
this.initDatabase(db);
|
|
25
|
+
this.connections.set(dbPath, db);
|
|
26
|
+
log("SQLite connection opened", { path: dbPath });
|
|
27
|
+
return db;
|
|
28
|
+
}
|
|
29
|
+
closeConnection(dbPath) {
|
|
30
|
+
const db = this.connections.get(dbPath);
|
|
31
|
+
if (db) {
|
|
32
|
+
db.close();
|
|
33
|
+
this.connections.delete(dbPath);
|
|
34
|
+
log("SQLite connection closed", { path: dbPath });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
closeAll() {
|
|
38
|
+
for (const [path, db] of this.connections) {
|
|
39
|
+
db.close();
|
|
40
|
+
log("SQLite connection closed", { path });
|
|
41
|
+
}
|
|
42
|
+
this.connections.clear();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
export const connectionManager = new ConnectionManager();
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ShardInfo } from "./types.js";
|
|
2
|
+
export declare class ShardManager {
|
|
3
|
+
private metadataDb;
|
|
4
|
+
private metadataPath;
|
|
5
|
+
constructor();
|
|
6
|
+
private initMetadataDb;
|
|
7
|
+
private getShardPath;
|
|
8
|
+
getActiveShard(scope: "user" | "project", scopeHash: string): ShardInfo | null;
|
|
9
|
+
getAllShards(scope: "user" | "project", scopeHash: string): ShardInfo[];
|
|
10
|
+
createShard(scope: "user" | "project", scopeHash: string, shardIndex: number): ShardInfo;
|
|
11
|
+
private initShardDb;
|
|
12
|
+
getWriteShard(scope: "user" | "project", scopeHash: string): ShardInfo;
|
|
13
|
+
private markShardReadOnly;
|
|
14
|
+
incrementVectorCount(shardId: number): void;
|
|
15
|
+
decrementVectorCount(shardId: number): void;
|
|
16
|
+
getShardByPath(dbPath: string): ShardInfo | null;
|
|
17
|
+
deleteShard(shardId: number): void;
|
|
18
|
+
}
|
|
19
|
+
export declare const shardManager: ShardManager;
|
|
20
|
+
//# sourceMappingURL=shard-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/shard-manager.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAI5C,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAW;IAC7B,OAAO,CAAC,YAAY,CAAS;;IAQ7B,OAAO,CAAC,cAAc;IAqBtB,OAAO,CAAC,YAAY;IAKpB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsB9E,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE;IAgCvE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IA4BxF,OAAO,CAAC,WAAW;IAmDnB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS;IAetE,OAAO,CAAC,iBAAiB;IAQzB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAiBhD,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAsBnC;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|