@lotargo/memory_plugin 1.6.5 → 1.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +576 -443
- package/mcp-server/benchmarks/fetch_real_corpus.js +351 -0
- package/mcp-server/benchmarks/gpu_profile_benchmark.js +170 -0
- package/mcp-server/benchmarks/policy_dominance_test.js +221 -0
- package/mcp-server/benchmarks/quality_evaluator.js +598 -0
- package/mcp-server/benchmarks/raw_corpus_data.js +613 -0
- package/mcp-server/benchmarks/run_benchmarks.js +366 -0
- package/mcp-server/benchmarks/stress_ingestion.js +195 -0
- package/mcp-server/benchmarks/table_code_retrieval.js +453 -0
- package/mcp-server/benchmarks/test_dual_layer.js +141 -0
- package/mcp-server/cli/direct_commands.js +39 -0
- package/mcp-server/cli.js +16 -5
- package/mcp-server/cli_boot.js +4 -1
- package/mcp-server/client_cli.js +73 -0
- package/mcp-server/client_paths.js +44 -0
- package/mcp-server/client_registration.js +38 -0
- package/mcp-server/codex_config.js +86 -8
- package/mcp-server/db/database.js +14 -21
- package/mcp-server/db/migrations.js +66 -77
- package/mcp-server/db/rag_blob_transport.js +143 -0
- package/mcp-server/db/rag_sync.js +284 -0
- package/mcp-server/db/sync_queue.js +219 -307
- package/mcp-server/dev_link.js +142 -0
- package/mcp-server/fact_format.js +44 -12
- package/mcp-server/index.js +17 -7
- package/mcp-server/ingest/exporter.js +44 -38
- package/mcp-server/ingest/pipeline.js +260 -248
- package/mcp-server/persona_migration.js +39 -0
- package/mcp-server/prompt_manager.js +162 -55
- package/mcp-server/rag_scope.js +83 -0
- package/mcp-server/retrieval/retriever.js +99 -64
- package/mcp-server/setup.js +150 -100
- package/mcp-server/storage/blob_store.js +53 -1
- package/mcp-server/tools/core/knowledge_read_core.js +163 -0
- package/mcp-server/tools/core/memory_core.js +24 -4
- package/mcp-server/tools/core/memory_routing.js +10 -0
- package/mcp-server/tools/core/note_core.js +53 -0
- package/mcp-server/tools/core/rag_query_core.js +169 -0
- package/mcp-server/tools/index.js +11 -9
- package/mcp-server/tools/memory_tools.js +4 -1
- package/mcp-server/tools/note_tools.js +35 -0
- package/mcp-server/tools/rag_tools.js +211 -364
- package/mcp-server/uninstall.js +627 -0
- package/opencode-plugin/index.js +80 -12
- package/opencode-plugin/main.js +136 -0
- package/package.json +17 -34
- package/skills/using-memory/SKILL.md +28 -19
package/mcp-server/cli_boot.js
CHANGED
|
@@ -32,6 +32,9 @@ if (major < MIN_MAJOR || (major === MIN_MAJOR && minor < MIN_MINOR)) {
|
|
|
32
32
|
// Version is OK — hand off to the real CLI.
|
|
33
33
|
import("./cli.js").then(m => {
|
|
34
34
|
if (process.argv[1] && process.argv[1].includes("cli_boot.js")) {
|
|
35
|
-
m.runCli().catch((err) =>
|
|
35
|
+
m.runCli().catch((err) => {
|
|
36
|
+
console.error("CLI error:", err);
|
|
37
|
+
process.exitCode = 1;
|
|
38
|
+
});
|
|
36
39
|
}
|
|
37
40
|
});
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { delimiter, extname, join } from "node:path";
|
|
4
|
+
|
|
5
|
+
function envValue(env, name) {
|
|
6
|
+
if (Object.prototype.hasOwnProperty.call(env || {}, name)) return env[name];
|
|
7
|
+
const key = Object.keys(env || {}).find((candidate) => candidate.toLowerCase() === name.toLowerCase());
|
|
8
|
+
return key ? env[key] : undefined;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function resolveClientExecutable(name, {
|
|
12
|
+
env = process.env,
|
|
13
|
+
platform = process.platform,
|
|
14
|
+
} = {}) {
|
|
15
|
+
if (String(env.MEMORY_PLUGIN_DISABLE_NATIVE_CLI || "") === "1") return null;
|
|
16
|
+
const pathValue = envValue(env, "PATH") || "";
|
|
17
|
+
const directories = pathValue.split(delimiter).filter(Boolean);
|
|
18
|
+
const extensions = platform === "win32"
|
|
19
|
+
? (envValue(env, "PATHEXT") || ".EXE;.COM;.CMD;.BAT").split(";").filter(Boolean)
|
|
20
|
+
: [""];
|
|
21
|
+
|
|
22
|
+
for (const directory of directories) {
|
|
23
|
+
const base = join(directory, name);
|
|
24
|
+
const candidates = platform === "win32" && extname(base)
|
|
25
|
+
? [base]
|
|
26
|
+
: extensions.map((extension) => `${base}${extension.toLowerCase()}`)
|
|
27
|
+
.concat(extensions.map((extension) => `${base}${extension.toUpperCase()}`));
|
|
28
|
+
for (const candidate of candidates) {
|
|
29
|
+
if (existsSync(candidate)) return candidate;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function runClientCli(name, args, {
|
|
36
|
+
cwd = process.cwd(),
|
|
37
|
+
env = process.env,
|
|
38
|
+
platform = process.platform,
|
|
39
|
+
timeout = 20_000,
|
|
40
|
+
} = {}) {
|
|
41
|
+
const executable = resolveClientExecutable(name, { env, platform });
|
|
42
|
+
if (!executable) {
|
|
43
|
+
return { available: false, ok: false, status: null, executable: null, stdout: "", stderr: "" };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const isWindowsShim = platform === "win32" && /\.(?:cmd|bat)$/i.test(executable);
|
|
47
|
+
const command = isWindowsShim ? (envValue(env, "ComSpec") || "cmd.exe") : executable;
|
|
48
|
+
const commandArgs = isWindowsShim ? ["/d", "/s", "/c", executable, ...args] : args;
|
|
49
|
+
const result = spawnSync(command, commandArgs, {
|
|
50
|
+
cwd,
|
|
51
|
+
env,
|
|
52
|
+
encoding: "utf-8",
|
|
53
|
+
timeout,
|
|
54
|
+
windowsHide: true,
|
|
55
|
+
shell: false,
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
available: true,
|
|
59
|
+
ok: result.status === 0 && !result.error,
|
|
60
|
+
status: result.status,
|
|
61
|
+
executable,
|
|
62
|
+
stdout: result.stdout || "",
|
|
63
|
+
stderr: result.stderr || "",
|
|
64
|
+
error: result.error || null,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function cliFailureMessage(result) {
|
|
69
|
+
if (!result?.available) return "client CLI is not installed or not available on PATH";
|
|
70
|
+
if (result.error) return result.error.message;
|
|
71
|
+
const detail = String(result.stderr || result.stdout || "").trim().split(/\r?\n/).pop();
|
|
72
|
+
return detail || `client CLI exited with status ${result.status}`;
|
|
73
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
export function resolveClientPaths({
|
|
5
|
+
home = homedir(),
|
|
6
|
+
cwd = process.cwd(),
|
|
7
|
+
env = process.env,
|
|
8
|
+
} = {}) {
|
|
9
|
+
const configHome = env.XDG_CONFIG_HOME || join(home, ".config");
|
|
10
|
+
const cacheHome = env.XDG_CACHE_HOME || join(home, ".cache");
|
|
11
|
+
const opencodeDir = env.OPENCODE_CONFIG_DIR || join(configHome, "opencode");
|
|
12
|
+
const agentConfigDir = join(configHome, "memory-agent");
|
|
13
|
+
const geminiDir = join(home, ".gemini");
|
|
14
|
+
const antigravityConfigDir = join(geminiDir, "config");
|
|
15
|
+
|
|
16
|
+
return {
|
|
17
|
+
home,
|
|
18
|
+
cwd,
|
|
19
|
+
configHome,
|
|
20
|
+
cacheHome,
|
|
21
|
+
opencodeDir,
|
|
22
|
+
opencodeConfigPath: join(opencodeDir, "opencode.json"),
|
|
23
|
+
opencodeCachePackages: join(cacheHome, "opencode", "packages"),
|
|
24
|
+
claudeConfigPath: join(home, ".claude.json"),
|
|
25
|
+
geminiDir,
|
|
26
|
+
geminiSettingsPath: join(geminiDir, "settings.json"),
|
|
27
|
+
geminiPromptPath: join(geminiDir, "GEMINI.md"),
|
|
28
|
+
geminiSkillsDir: join(geminiDir, "skills"),
|
|
29
|
+
// Antigravity intentionally uses its own legacy layout. Keep these paths
|
|
30
|
+
// separate from the real Gemini CLI settings above.
|
|
31
|
+
antigravityConfigDir,
|
|
32
|
+
antigravityMcpConfigPath: join(antigravityConfigDir, "mcp_config.json"),
|
|
33
|
+
geminiConfigDir: antigravityConfigDir,
|
|
34
|
+
geminiMcpConfigPath: join(antigravityConfigDir, "mcp_config.json"),
|
|
35
|
+
localAgentsDir: join(cwd, ".agents"),
|
|
36
|
+
localAgentsMcpConfigPath: join(cwd, ".agents", "mcp_config.json"),
|
|
37
|
+
codexDir: join(home, ".codex"),
|
|
38
|
+
codexConfigPath: join(home, ".codex", "config.toml"),
|
|
39
|
+
agentConfigDir,
|
|
40
|
+
promptFile: join(agentConfigDir, "prompt.md"),
|
|
41
|
+
promptStateFile: join(agentConfigDir, "prompt-state.json"),
|
|
42
|
+
promptBackupDir: join(agentConfigDir, "backups"),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { isMemoryPluginSpec } from "./dev_link.js";
|
|
4
|
+
|
|
5
|
+
export const MEMORY_MCP_ENTRY = Object.freeze({
|
|
6
|
+
command: "npx",
|
|
7
|
+
args: Object.freeze(["-y", "@lotargo/memory_plugin"]),
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
export function isMemoryPluginEntry(entry) {
|
|
11
|
+
const spec = Array.isArray(entry)
|
|
12
|
+
? entry[0]
|
|
13
|
+
: entry && typeof entry === "object"
|
|
14
|
+
? entry.package
|
|
15
|
+
: entry;
|
|
16
|
+
return isMemoryPluginSpec(spec);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function isMemoryMcpServerEntry(entry) {
|
|
20
|
+
if (!entry || typeof entry !== "object") return false;
|
|
21
|
+
const command = String(entry.command || "").replace(/\\/g, "/").toLowerCase();
|
|
22
|
+
const args = Array.isArray(entry.args)
|
|
23
|
+
? entry.args.map((value) => String(value).replace(/\\/g, "/").toLowerCase())
|
|
24
|
+
: [];
|
|
25
|
+
return args.some((arg) => /^@lotargo\/memory_plugin(?:@[^/]+)?$/i.test(arg) || /^opencode-memory-plugin(?:@[^/]+)?$/i.test(arg))
|
|
26
|
+
|| args.some((arg) => arg.endsWith("/mcp-server/boot.js") && arg.includes("memory"))
|
|
27
|
+
|| /(?:^|\/)(?:memory_plugin|memory-agent)(?:\.(?:cmd|exe|ps1|bat))?$/i.test(command);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function readJsonConfig(filePath) {
|
|
31
|
+
if (!existsSync(filePath)) return {};
|
|
32
|
+
const raw = (await readFile(filePath, "utf-8")).replace(/^\uFEFF/, "");
|
|
33
|
+
const value = JSON.parse(raw);
|
|
34
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
35
|
+
throw new Error(`Expected a JSON object in ${filePath}`);
|
|
36
|
+
}
|
|
37
|
+
return value;
|
|
38
|
+
}
|
|
@@ -102,6 +102,44 @@ function sectionRanges(lines) {
|
|
|
102
102
|
}));
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
function validateOwnedMemoryAgentTargets(lines, ranges) {
|
|
106
|
+
const targets = ranges.filter((range) => isMemoryAgentHeader(range.name));
|
|
107
|
+
if (targets.length === 0) return { targets, exactTargets: [], conflict: null };
|
|
108
|
+
|
|
109
|
+
const exactTargets = targets.filter((range) => isMemoryAgentHeader(range.name, { exact: true }));
|
|
110
|
+
const unowned = exactTargets.filter((range) => {
|
|
111
|
+
const text = lines.slice(range.start, range.end).join("\n");
|
|
112
|
+
return !isMemoryPluginOwnedSection(text);
|
|
113
|
+
});
|
|
114
|
+
if (unowned.length > 0 || exactTargets.length === 0) {
|
|
115
|
+
return {
|
|
116
|
+
targets,
|
|
117
|
+
exactTargets,
|
|
118
|
+
conflict: "Existing memory-agent TOML section is not recognized as owned by @lotargo/memory_plugin",
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
let activeOwnedRoot = false;
|
|
123
|
+
for (const range of ranges) {
|
|
124
|
+
if (isMemoryAgentHeader(range.name, { exact: true })) {
|
|
125
|
+
const text = lines.slice(range.start, range.end).join("\n");
|
|
126
|
+
activeOwnedRoot = isMemoryPluginOwnedSection(text);
|
|
127
|
+
} else if (isMemoryAgentHeader(range.name)) {
|
|
128
|
+
if (!activeOwnedRoot) {
|
|
129
|
+
return {
|
|
130
|
+
targets,
|
|
131
|
+
exactTargets,
|
|
132
|
+
conflict: "A memory-agent child TOML section is not attached to an owned plugin section",
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
} else {
|
|
136
|
+
activeOwnedRoot = false;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return { targets, exactTargets, conflict: null };
|
|
141
|
+
}
|
|
142
|
+
|
|
105
143
|
export function getCodexMemoryAgentSections(content) {
|
|
106
144
|
const lines = String(content || "").split(/\r?\n/);
|
|
107
145
|
return sectionRanges(lines)
|
|
@@ -113,25 +151,65 @@ export function getCodexMemoryAgentSections(content) {
|
|
|
113
151
|
}));
|
|
114
152
|
}
|
|
115
153
|
|
|
154
|
+
export function removeCodexMemoryAgentConfig(content) {
|
|
155
|
+
const source = String(content || "");
|
|
156
|
+
const eol = source.includes("\r\n") ? "\r\n" : "\n";
|
|
157
|
+
const lines = source.split(/\r?\n/);
|
|
158
|
+
const ranges = sectionRanges(lines);
|
|
159
|
+
const { targets, conflict } = validateOwnedMemoryAgentTargets(lines, ranges);
|
|
160
|
+
|
|
161
|
+
if (targets.length === 0) {
|
|
162
|
+
return { content: source, changed: false, status: "not_found", removed: 0 };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (conflict) {
|
|
166
|
+
return {
|
|
167
|
+
content: source,
|
|
168
|
+
changed: false,
|
|
169
|
+
status: "conflict",
|
|
170
|
+
reason: conflict,
|
|
171
|
+
removed: 0,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const targetLineIndexes = new Set();
|
|
176
|
+
for (const target of targets) {
|
|
177
|
+
let contentEnd = target.end;
|
|
178
|
+
while (contentEnd > target.start + 1 && lines[contentEnd - 1] === "") contentEnd--;
|
|
179
|
+
for (let i = target.start; i < contentEnd; i++) targetLineIndexes.add(i);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const result = [];
|
|
183
|
+
for (let i = 0; i < lines.length; i++) {
|
|
184
|
+
if (targetLineIndexes.has(i)) continue;
|
|
185
|
+
result.push(lines[i]);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
let updated = result.join(eol);
|
|
189
|
+
if (updated.trim() === "") updated = "";
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
content: updated,
|
|
193
|
+
changed: updated !== source,
|
|
194
|
+
status: updated !== source ? "removed" : "unchanged",
|
|
195
|
+
removed: targets.length,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
116
199
|
export function updateCodexMemoryAgentConfig(content, options) {
|
|
117
200
|
const source = String(content || "");
|
|
118
201
|
const eol = source.includes("\r\n") ? "\r\n" : "\n";
|
|
119
202
|
const lines = source.split(/\r?\n/);
|
|
120
203
|
const desired = buildCodexMemoryAgentSection(options).split("\n");
|
|
121
204
|
const ranges = sectionRanges(lines);
|
|
122
|
-
const targets =
|
|
123
|
-
const exactTargets = targets.filter((range) => isMemoryAgentHeader(range.name, { exact: true }));
|
|
205
|
+
const { targets, conflict } = validateOwnedMemoryAgentTargets(lines, ranges);
|
|
124
206
|
|
|
125
|
-
|
|
126
|
-
const text = lines.slice(range.start, range.end).join("\n");
|
|
127
|
-
return !isMemoryPluginOwnedSection(text);
|
|
128
|
-
});
|
|
129
|
-
if (unowned.length > 0) {
|
|
207
|
+
if (conflict) {
|
|
130
208
|
return {
|
|
131
209
|
content: source,
|
|
132
210
|
changed: false,
|
|
133
211
|
status: "conflict",
|
|
134
|
-
reason:
|
|
212
|
+
reason: conflict,
|
|
135
213
|
};
|
|
136
214
|
}
|
|
137
215
|
|
|
@@ -10,7 +10,7 @@ import { createClient } from "@libsql/client";
|
|
|
10
10
|
let dbInstance = null;
|
|
11
11
|
let dbInitPromise = null;
|
|
12
12
|
let dbLastFailAt = 0;
|
|
13
|
-
const DB_FAIL_COOLDOWN_MS = 5_000;
|
|
13
|
+
const DB_FAIL_COOLDOWN_MS = 5_000;
|
|
14
14
|
|
|
15
15
|
export const STORAGE_DIR = join(MEMORY_DIR, "storage");
|
|
16
16
|
export const BLOBS_DIR = join(STORAGE_DIR, "blobs");
|
|
@@ -34,8 +34,6 @@ class DatabaseWrapper {
|
|
|
34
34
|
|
|
35
35
|
while (attempts < maxAttempts) {
|
|
36
36
|
attempts++;
|
|
37
|
-
// A plain timer is enough here; an AbortController per attempt leaked its
|
|
38
|
-
// "abort" listener because it was never removed.
|
|
39
37
|
let timeoutId = null;
|
|
40
38
|
const timeoutPromise = new Promise((_, reject) => {
|
|
41
39
|
timeoutId = setTimeout(
|
|
@@ -48,7 +46,6 @@ class DatabaseWrapper {
|
|
|
48
46
|
const client = (this.usingFailover && this.failoverClient) ? this.failoverClient : this.cloudClient;
|
|
49
47
|
const result = await Promise.race([fn(client), timeoutPromise]);
|
|
50
48
|
clearTimeout(timeoutId);
|
|
51
|
-
// Successful operation, reset consecutive failures
|
|
52
49
|
this.consecutiveFailures = 0;
|
|
53
50
|
return result;
|
|
54
51
|
} catch (err) {
|
|
@@ -58,12 +55,10 @@ class DatabaseWrapper {
|
|
|
58
55
|
if (this.consecutiveFailures >= 3 && this.failoverClient && !this.usingFailover) {
|
|
59
56
|
console.warn("[WARN] Turso is temporarily unreachable. Switching to LiteFS failover replica...");
|
|
60
57
|
this.usingFailover = true;
|
|
61
|
-
// Retry the operation on the failover client
|
|
62
58
|
return this.runWithRetry(fn);
|
|
63
59
|
}
|
|
64
60
|
throw err;
|
|
65
61
|
}
|
|
66
|
-
// Small delay before retrying (exponential backoff / fixed delay)
|
|
67
62
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
68
63
|
}
|
|
69
64
|
}
|
|
@@ -156,16 +151,12 @@ async function openDatabase(customPath, mode) {
|
|
|
156
151
|
localDb = new DatabaseSync(dbPath);
|
|
157
152
|
localDb.exec("PRAGMA foreign_keys = ON;");
|
|
158
153
|
localDb.exec("PRAGMA journal_mode = WAL;");
|
|
159
|
-
// Default busy_timeout is 0 => an immediate SQLITE_BUSY ("database is
|
|
160
|
-
// locked") whenever background sync, ingestion and MCP calls overlap.
|
|
161
154
|
localDb.exec("PRAGMA busy_timeout = 5000;");
|
|
162
155
|
}
|
|
163
156
|
|
|
164
157
|
let cloudClient = null;
|
|
165
158
|
let failoverClient = null;
|
|
166
159
|
if (mode === "only-cloud" || mode === "hybrid-sync") {
|
|
167
|
-
// Resolve working cloud credentials. An env TURSO_API_TOKEN (which can only
|
|
168
|
-
// call the Platform API) is lazily minted into a per-database JWT here.
|
|
169
160
|
const secrets = await resolveCloudSecrets();
|
|
170
161
|
const tursoUrl = customPath && customPath.startsWith("libsql:") ? customPath : (secrets?.dbUrl || config.tursoUrl);
|
|
171
162
|
const failoverUrl = config.failoverUrl || "";
|
|
@@ -182,8 +173,6 @@ async function openDatabase(customPath, mode) {
|
|
|
182
173
|
authToken: token || undefined,
|
|
183
174
|
});
|
|
184
175
|
}
|
|
185
|
-
// In hybrid-sync mode, ensure remote schema is also fully migrated and up to date.
|
|
186
|
-
// Use a dedicated short-lived client so closing it doesn't kill the shared one.
|
|
187
176
|
if (mode === "hybrid-sync") {
|
|
188
177
|
const remoteClient = createClient({
|
|
189
178
|
url: tursoUrl,
|
|
@@ -199,13 +188,22 @@ async function openDatabase(customPath, mode) {
|
|
|
199
188
|
}
|
|
200
189
|
|
|
201
190
|
const wrappedDb = new DatabaseWrapper(localDb, cloudClient, mode, failoverClient);
|
|
202
|
-
|
|
203
|
-
// Initialize/run migrations
|
|
204
191
|
await runMigrations(wrappedDb);
|
|
205
192
|
|
|
193
|
+
// Upgrade path for RAG content ingested before portable cloud blobs existed.
|
|
194
|
+
// The backfill is content-addressed and uploads only hashes absent in Turso.
|
|
195
|
+
// Missing local files are simply reported/skipped; database availability must
|
|
196
|
+
// never depend on a legacy raw blob still being present on this machine.
|
|
197
|
+
if ((mode === "only-cloud" || mode === "hybrid-sync") && cloudClient) {
|
|
198
|
+
try {
|
|
199
|
+
const { backfillCloudBlobsFromLocal } = await import("./rag_blob_transport.js");
|
|
200
|
+
await backfillCloudBlobsFromLocal(wrappedDb);
|
|
201
|
+
} catch (err) {
|
|
202
|
+
console.warn("[WARN] RAG cloud blob backfill skipped:", err.message);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
206
|
if (!customPath) {
|
|
207
|
-
// Closing the previous instance before replacing it: otherwise a mode switch
|
|
208
|
-
// leaked the old DatabaseSync handle and Turso client.
|
|
209
207
|
if (dbInstance && dbInstance !== wrappedDb) {
|
|
210
208
|
try {
|
|
211
209
|
dbInstance.close();
|
|
@@ -225,15 +223,10 @@ export async function getDatabase(customPath = null, forceMode = null) {
|
|
|
225
223
|
if (dbInstance && dbInstance.mode === mode) {
|
|
226
224
|
return dbInstance;
|
|
227
225
|
}
|
|
228
|
-
// After a failed init, wait before retrying to avoid hammering cloud auth.
|
|
229
|
-
// Only cloud modes are throttled — reopening a local SQLite file is cheap
|
|
230
|
-
// and must never be blocked by an unrelated cloud failure.
|
|
231
226
|
const isCloudMode = mode === "only-cloud" || mode === "hybrid-sync";
|
|
232
227
|
if (isCloudMode && !dbInitPromise && dbLastFailAt && (Date.now() - dbLastFailAt) < DB_FAIL_COOLDOWN_MS) {
|
|
233
228
|
throw new Error("Database initialization failed recently. Retrying in a few seconds...");
|
|
234
229
|
}
|
|
235
|
-
// Deduplicate concurrent default-DB initialization so migrations never run
|
|
236
|
-
// on multiple connections at once (avoids "database is locked" crashes).
|
|
237
230
|
if (!dbInitPromise) {
|
|
238
231
|
dbInitPromise = openDatabase(null, mode).then((result) => {
|
|
239
232
|
dbLastFailAt = 0;
|
|
@@ -3,7 +3,6 @@ const MIGRATIONS = [
|
|
|
3
3
|
version: 1,
|
|
4
4
|
name: "001_initial_rag_schema",
|
|
5
5
|
up: async (db) => {
|
|
6
|
-
// 1. Documents Table
|
|
7
6
|
await db.exec(`
|
|
8
7
|
CREATE TABLE IF NOT EXISTS documents (
|
|
9
8
|
id TEXT PRIMARY KEY,
|
|
@@ -17,8 +16,6 @@ const MIGRATIONS = [
|
|
|
17
16
|
updated_at INTEGER NOT NULL
|
|
18
17
|
);
|
|
19
18
|
`);
|
|
20
|
-
|
|
21
|
-
// 2. Sections Table
|
|
22
19
|
await db.exec(`
|
|
23
20
|
CREATE TABLE IF NOT EXISTS sections (
|
|
24
21
|
id TEXT PRIMARY KEY,
|
|
@@ -29,8 +26,6 @@ const MIGRATIONS = [
|
|
|
29
26
|
token_count INTEGER NOT NULL
|
|
30
27
|
);
|
|
31
28
|
`);
|
|
32
|
-
|
|
33
|
-
// 3. Micro-Chunks Table
|
|
34
29
|
await db.exec(`
|
|
35
30
|
CREATE TABLE IF NOT EXISTS micro_chunks (
|
|
36
31
|
id TEXT PRIMARY KEY,
|
|
@@ -41,8 +36,6 @@ const MIGRATIONS = [
|
|
|
41
36
|
token_count INTEGER NOT NULL
|
|
42
37
|
);
|
|
43
38
|
`);
|
|
44
|
-
|
|
45
|
-
// 4. Full-Text Search (BM25 Index via SQLite FTS5)
|
|
46
39
|
await db.exec(`
|
|
47
40
|
CREATE VIRTUAL TABLE IF NOT EXISTS micro_chunks_fts USING fts5(
|
|
48
41
|
id UNINDEXED,
|
|
@@ -50,8 +43,6 @@ const MIGRATIONS = [
|
|
|
50
43
|
breadcrumbs
|
|
51
44
|
);
|
|
52
45
|
`);
|
|
53
|
-
|
|
54
|
-
// 5. GraphRAG Lite Edges Table
|
|
55
46
|
await db.exec(`
|
|
56
47
|
CREATE TABLE IF NOT EXISTS graph_edges (
|
|
57
48
|
source_id TEXT NOT NULL,
|
|
@@ -66,13 +57,8 @@ const MIGRATIONS = [
|
|
|
66
57
|
version: 2,
|
|
67
58
|
name: "002_agent_knowledge_graph",
|
|
68
59
|
up: async (db) => {
|
|
69
|
-
try {
|
|
70
|
-
|
|
71
|
-
} catch (e) {}
|
|
72
|
-
try {
|
|
73
|
-
await db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`);
|
|
74
|
-
} catch (e) {}
|
|
75
|
-
|
|
60
|
+
try { await db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`); } catch (e) {}
|
|
61
|
+
try { await db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`); } catch (e) {}
|
|
76
62
|
await db.exec(`
|
|
77
63
|
CREATE TABLE IF NOT EXISTS knowledge_links (
|
|
78
64
|
id TEXT PRIMARY KEY,
|
|
@@ -104,10 +90,7 @@ const MIGRATIONS = [
|
|
|
104
90
|
created_at INTEGER
|
|
105
91
|
);
|
|
106
92
|
`);
|
|
107
|
-
|
|
108
|
-
try {
|
|
109
|
-
await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
|
|
110
|
-
} catch (e) {}
|
|
93
|
+
try { await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`); } catch (e) {}
|
|
111
94
|
},
|
|
112
95
|
},
|
|
113
96
|
{
|
|
@@ -116,65 +99,81 @@ const MIGRATIONS = [
|
|
|
116
99
|
up: async (db) => {
|
|
117
100
|
await db.exec(`
|
|
118
101
|
CREATE TABLE IF NOT EXISTS project_identities (
|
|
119
|
-
key
|
|
120
|
-
name
|
|
102
|
+
key TEXT PRIMARY KEY,
|
|
103
|
+
name TEXT NOT NULL,
|
|
121
104
|
primary_remote TEXT,
|
|
122
|
-
created_at
|
|
123
|
-
updated_at
|
|
105
|
+
created_at INTEGER NOT NULL,
|
|
106
|
+
updated_at INTEGER NOT NULL
|
|
124
107
|
);
|
|
125
108
|
`);
|
|
126
|
-
|
|
127
109
|
await db.exec(`
|
|
128
110
|
CREATE TABLE IF NOT EXISTS project_aliases (
|
|
129
|
-
alias
|
|
111
|
+
alias TEXT PRIMARY KEY,
|
|
130
112
|
identity_key TEXT NOT NULL REFERENCES project_identities(key) ON DELETE CASCADE,
|
|
131
|
-
kind
|
|
132
|
-
created_at
|
|
113
|
+
kind TEXT NOT NULL,
|
|
114
|
+
created_at INTEGER NOT NULL
|
|
133
115
|
);
|
|
134
116
|
`);
|
|
135
|
-
|
|
117
|
+
await db.exec(`CREATE INDEX IF NOT EXISTS idx_project_aliases_identity ON project_aliases(identity_key);`);
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
version: 5,
|
|
122
|
+
name: "005_retrieval_policy",
|
|
123
|
+
up: async (db) => {
|
|
124
|
+
try { await db.exec(`ALTER TABLE micro_chunks ADD COLUMN retrieval_policy TEXT DEFAULT 'micro_chunk';`); } catch (e) {}
|
|
125
|
+
try { await db.exec(`ALTER TABLE micro_chunks ADD COLUMN policy_source_id TEXT;`); } catch (e) {}
|
|
126
|
+
await db.exec(`CREATE INDEX IF NOT EXISTS idx_micro_chunks_retrieval_policy ON micro_chunks(retrieval_policy);`);
|
|
127
|
+
},
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
version: 6,
|
|
131
|
+
name: "006_project_scoped_rag",
|
|
132
|
+
up: async (db) => {
|
|
136
133
|
await db.exec(`
|
|
137
|
-
CREATE
|
|
134
|
+
CREATE TABLE IF NOT EXISTS document_scopes (
|
|
135
|
+
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
136
|
+
scope_key TEXT NOT NULL,
|
|
137
|
+
created_at INTEGER NOT NULL,
|
|
138
|
+
PRIMARY KEY (doc_id, scope_key)
|
|
139
|
+
);
|
|
140
|
+
`);
|
|
141
|
+
await db.exec(`CREATE INDEX IF NOT EXISTS idx_document_scopes_scope ON document_scopes(scope_key, doc_id);`);
|
|
142
|
+
await db.exec(`
|
|
143
|
+
INSERT OR IGNORE INTO document_scopes (doc_id, scope_key, created_at)
|
|
144
|
+
SELECT id, 'global', created_at FROM documents;
|
|
138
145
|
`);
|
|
139
146
|
},
|
|
140
147
|
},
|
|
141
|
-
{
|
|
142
|
-
version:
|
|
143
|
-
name: "
|
|
148
|
+
{
|
|
149
|
+
version: 7,
|
|
150
|
+
name: "007_rag_blob_transport",
|
|
144
151
|
up: async (db) => {
|
|
145
|
-
try {
|
|
146
|
-
await db.exec(`ALTER TABLE micro_chunks ADD COLUMN retrieval_policy TEXT DEFAULT 'micro_chunk';`);
|
|
147
|
-
} catch (e) {}
|
|
148
|
-
try {
|
|
149
|
-
await db.exec(`ALTER TABLE micro_chunks ADD COLUMN policy_source_id TEXT;`);
|
|
150
|
-
} catch (e) {}
|
|
151
152
|
await db.exec(`
|
|
152
|
-
CREATE
|
|
153
|
+
CREATE TABLE IF NOT EXISTS rag_blobs (
|
|
154
|
+
hash TEXT PRIMARY KEY,
|
|
155
|
+
gzip_base64 TEXT NOT NULL,
|
|
156
|
+
raw_size INTEGER NOT NULL DEFAULT 0,
|
|
157
|
+
created_at INTEGER NOT NULL
|
|
158
|
+
);
|
|
153
159
|
`);
|
|
154
|
-
},
|
|
155
|
-
},
|
|
156
|
-
{
|
|
157
|
-
version:
|
|
158
|
-
name: "
|
|
159
|
-
up: async (db) => {
|
|
160
|
-
await db.exec(`
|
|
161
|
-
CREATE TABLE IF NOT EXISTS
|
|
162
|
-
doc_id
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
`);
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
await db.exec(`
|
|
172
|
-
INSERT OR IGNORE INTO document_scopes (doc_id, scope_key, created_at)
|
|
173
|
-
SELECT id, 'global', created_at FROM documents;
|
|
174
|
-
`);
|
|
175
|
-
},
|
|
176
|
-
},
|
|
177
|
-
];
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
version: 8,
|
|
164
|
+
name: "008_rag_document_tombstones",
|
|
165
|
+
up: async (db) => {
|
|
166
|
+
await db.exec(`
|
|
167
|
+
CREATE TABLE IF NOT EXISTS rag_document_tombstones (
|
|
168
|
+
doc_id TEXT PRIMARY KEY,
|
|
169
|
+
path TEXT,
|
|
170
|
+
deleted_at INTEGER NOT NULL
|
|
171
|
+
);
|
|
172
|
+
`);
|
|
173
|
+
await db.exec(`CREATE INDEX IF NOT EXISTS idx_rag_tombstones_path ON rag_document_tombstones(path);`);
|
|
174
|
+
},
|
|
175
|
+
},
|
|
176
|
+
];
|
|
178
177
|
|
|
179
178
|
export async function runMigrations(db) {
|
|
180
179
|
let currentVersion = 0;
|
|
@@ -188,12 +187,7 @@ export async function runMigrations(db) {
|
|
|
188
187
|
} catch (e2) {}
|
|
189
188
|
}
|
|
190
189
|
|
|
191
|
-
|
|
192
|
-
try {
|
|
193
|
-
await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY);`);
|
|
194
|
-
} catch (e) {}
|
|
195
|
-
|
|
196
|
-
// Create notebooks table if not exists
|
|
190
|
+
try { await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY);`); } catch (e) {}
|
|
197
191
|
try {
|
|
198
192
|
await db.exec(`
|
|
199
193
|
CREATE TABLE IF NOT EXISTS notebooks (
|
|
@@ -211,19 +205,14 @@ export async function runMigrations(db) {
|
|
|
211
205
|
await migration.up(db);
|
|
212
206
|
await db.prepare("INSERT INTO schema_migrations (version) VALUES (?);").run(migration.version);
|
|
213
207
|
await db.exec("COMMIT;");
|
|
214
|
-
try {
|
|
215
|
-
await db.exec(`PRAGMA user_version = ${migration.version};`);
|
|
216
|
-
} catch (e) {}
|
|
208
|
+
try { await db.exec(`PRAGMA user_version = ${migration.version};`); } catch (e) {}
|
|
217
209
|
} catch (err) {
|
|
218
|
-
try {
|
|
219
|
-
await db.exec("ROLLBACK;");
|
|
220
|
-
} catch (e) {}
|
|
210
|
+
try { await db.exec("ROLLBACK;"); } catch (e) {}
|
|
221
211
|
throw new Error(`Migration ${migration.name} failed: ${err.message}`);
|
|
222
212
|
}
|
|
223
213
|
}
|
|
224
214
|
}
|
|
225
215
|
|
|
226
|
-
// Defensive table & column check for medium_chunks hierarchy
|
|
227
216
|
try {
|
|
228
217
|
await db.exec(`
|
|
229
218
|
CREATE TABLE IF NOT EXISTS medium_chunks (
|