@lotargo/memory_plugin 1.6.5 → 1.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +576 -443
  3. package/mcp-server/benchmarks/fetch_real_corpus.js +351 -0
  4. package/mcp-server/benchmarks/gpu_profile_benchmark.js +170 -0
  5. package/mcp-server/benchmarks/policy_dominance_test.js +221 -0
  6. package/mcp-server/benchmarks/quality_evaluator.js +598 -0
  7. package/mcp-server/benchmarks/raw_corpus_data.js +613 -0
  8. package/mcp-server/benchmarks/run_benchmarks.js +366 -0
  9. package/mcp-server/benchmarks/stress_ingestion.js +195 -0
  10. package/mcp-server/benchmarks/table_code_retrieval.js +453 -0
  11. package/mcp-server/benchmarks/test_dual_layer.js +141 -0
  12. package/mcp-server/cli/direct_commands.js +39 -0
  13. package/mcp-server/cli.js +16 -5
  14. package/mcp-server/cli_boot.js +4 -1
  15. package/mcp-server/client_cli.js +73 -0
  16. package/mcp-server/client_paths.js +44 -0
  17. package/mcp-server/client_registration.js +38 -0
  18. package/mcp-server/codex_config.js +86 -8
  19. package/mcp-server/db/database.js +14 -21
  20. package/mcp-server/db/migrations.js +66 -77
  21. package/mcp-server/db/rag_blob_transport.js +143 -0
  22. package/mcp-server/db/rag_sync.js +284 -0
  23. package/mcp-server/db/sync_queue.js +219 -307
  24. package/mcp-server/dev_link.js +142 -0
  25. package/mcp-server/fact_format.js +44 -12
  26. package/mcp-server/index.js +17 -7
  27. package/mcp-server/ingest/exporter.js +44 -38
  28. package/mcp-server/ingest/pipeline.js +260 -248
  29. package/mcp-server/persona_migration.js +39 -0
  30. package/mcp-server/prompt_manager.js +162 -55
  31. package/mcp-server/rag_scope.js +83 -0
  32. package/mcp-server/retrieval/retriever.js +99 -64
  33. package/mcp-server/setup.js +150 -100
  34. package/mcp-server/storage/blob_store.js +53 -1
  35. package/mcp-server/tools/core/knowledge_read_core.js +163 -0
  36. package/mcp-server/tools/core/memory_core.js +24 -4
  37. package/mcp-server/tools/core/memory_routing.js +10 -0
  38. package/mcp-server/tools/core/note_core.js +53 -0
  39. package/mcp-server/tools/core/rag_query_core.js +169 -0
  40. package/mcp-server/tools/index.js +11 -9
  41. package/mcp-server/tools/memory_tools.js +4 -1
  42. package/mcp-server/tools/note_tools.js +35 -0
  43. package/mcp-server/tools/rag_tools.js +211 -364
  44. package/mcp-server/uninstall.js +627 -0
  45. package/opencode-plugin/index.js +80 -12
  46. package/opencode-plugin/main.js +136 -0
  47. package/package.json +17 -34
  48. package/skills/using-memory/SKILL.md +28 -19
@@ -32,6 +32,9 @@ if (major < MIN_MAJOR || (major === MIN_MAJOR && minor < MIN_MINOR)) {
32
32
  // Version is OK — hand off to the real CLI.
33
33
  import("./cli.js").then(m => {
34
34
  if (process.argv[1] && process.argv[1].includes("cli_boot.js")) {
35
- m.runCli().catch((err) => console.error("CLI error:", err));
35
+ m.runCli().catch((err) => {
36
+ console.error("CLI error:", err);
37
+ process.exitCode = 1;
38
+ });
36
39
  }
37
40
  });
@@ -0,0 +1,73 @@
1
+ import { spawnSync } from "node:child_process";
2
+ import { existsSync } from "node:fs";
3
+ import { delimiter, extname, join } from "node:path";
4
+
5
+ function envValue(env, name) {
6
+ if (Object.prototype.hasOwnProperty.call(env || {}, name)) return env[name];
7
+ const key = Object.keys(env || {}).find((candidate) => candidate.toLowerCase() === name.toLowerCase());
8
+ return key ? env[key] : undefined;
9
+ }
10
+
11
+ export function resolveClientExecutable(name, {
12
+ env = process.env,
13
+ platform = process.platform,
14
+ } = {}) {
15
+ if (String(env.MEMORY_PLUGIN_DISABLE_NATIVE_CLI || "") === "1") return null;
16
+ const pathValue = envValue(env, "PATH") || "";
17
+ const directories = pathValue.split(delimiter).filter(Boolean);
18
+ const extensions = platform === "win32"
19
+ ? (envValue(env, "PATHEXT") || ".EXE;.COM;.CMD;.BAT").split(";").filter(Boolean)
20
+ : [""];
21
+
22
+ for (const directory of directories) {
23
+ const base = join(directory, name);
24
+ const candidates = platform === "win32" && extname(base)
25
+ ? [base]
26
+ : extensions.map((extension) => `${base}${extension.toLowerCase()}`)
27
+ .concat(extensions.map((extension) => `${base}${extension.toUpperCase()}`));
28
+ for (const candidate of candidates) {
29
+ if (existsSync(candidate)) return candidate;
30
+ }
31
+ }
32
+ return null;
33
+ }
34
+
35
+ export function runClientCli(name, args, {
36
+ cwd = process.cwd(),
37
+ env = process.env,
38
+ platform = process.platform,
39
+ timeout = 20_000,
40
+ } = {}) {
41
+ const executable = resolveClientExecutable(name, { env, platform });
42
+ if (!executable) {
43
+ return { available: false, ok: false, status: null, executable: null, stdout: "", stderr: "" };
44
+ }
45
+
46
+ const isWindowsShim = platform === "win32" && /\.(?:cmd|bat)$/i.test(executable);
47
+ const command = isWindowsShim ? (envValue(env, "ComSpec") || "cmd.exe") : executable;
48
+ const commandArgs = isWindowsShim ? ["/d", "/s", "/c", executable, ...args] : args;
49
+ const result = spawnSync(command, commandArgs, {
50
+ cwd,
51
+ env,
52
+ encoding: "utf-8",
53
+ timeout,
54
+ windowsHide: true,
55
+ shell: false,
56
+ });
57
+ return {
58
+ available: true,
59
+ ok: result.status === 0 && !result.error,
60
+ status: result.status,
61
+ executable,
62
+ stdout: result.stdout || "",
63
+ stderr: result.stderr || "",
64
+ error: result.error || null,
65
+ };
66
+ }
67
+
68
+ export function cliFailureMessage(result) {
69
+ if (!result?.available) return "client CLI is not installed or not available on PATH";
70
+ if (result.error) return result.error.message;
71
+ const detail = String(result.stderr || result.stdout || "").trim().split(/\r?\n/).pop();
72
+ return detail || `client CLI exited with status ${result.status}`;
73
+ }
@@ -0,0 +1,44 @@
1
+ import { homedir } from "node:os";
2
+ import { join } from "node:path";
3
+
4
+ export function resolveClientPaths({
5
+ home = homedir(),
6
+ cwd = process.cwd(),
7
+ env = process.env,
8
+ } = {}) {
9
+ const configHome = env.XDG_CONFIG_HOME || join(home, ".config");
10
+ const cacheHome = env.XDG_CACHE_HOME || join(home, ".cache");
11
+ const opencodeDir = env.OPENCODE_CONFIG_DIR || join(configHome, "opencode");
12
+ const agentConfigDir = join(configHome, "memory-agent");
13
+ const geminiDir = join(home, ".gemini");
14
+ const antigravityConfigDir = join(geminiDir, "config");
15
+
16
+ return {
17
+ home,
18
+ cwd,
19
+ configHome,
20
+ cacheHome,
21
+ opencodeDir,
22
+ opencodeConfigPath: join(opencodeDir, "opencode.json"),
23
+ opencodeCachePackages: join(cacheHome, "opencode", "packages"),
24
+ claudeConfigPath: join(home, ".claude.json"),
25
+ geminiDir,
26
+ geminiSettingsPath: join(geminiDir, "settings.json"),
27
+ geminiPromptPath: join(geminiDir, "GEMINI.md"),
28
+ geminiSkillsDir: join(geminiDir, "skills"),
29
+ // Antigravity intentionally uses its own legacy layout. Keep these paths
30
+ // separate from the real Gemini CLI settings above.
31
+ antigravityConfigDir,
32
+ antigravityMcpConfigPath: join(antigravityConfigDir, "mcp_config.json"),
33
+ geminiConfigDir: antigravityConfigDir,
34
+ geminiMcpConfigPath: join(antigravityConfigDir, "mcp_config.json"),
35
+ localAgentsDir: join(cwd, ".agents"),
36
+ localAgentsMcpConfigPath: join(cwd, ".agents", "mcp_config.json"),
37
+ codexDir: join(home, ".codex"),
38
+ codexConfigPath: join(home, ".codex", "config.toml"),
39
+ agentConfigDir,
40
+ promptFile: join(agentConfigDir, "prompt.md"),
41
+ promptStateFile: join(agentConfigDir, "prompt-state.json"),
42
+ promptBackupDir: join(agentConfigDir, "backups"),
43
+ };
44
+ }
@@ -0,0 +1,38 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import { existsSync } from "node:fs";
3
+ import { isMemoryPluginSpec } from "./dev_link.js";
4
+
5
+ export const MEMORY_MCP_ENTRY = Object.freeze({
6
+ command: "npx",
7
+ args: Object.freeze(["-y", "@lotargo/memory_plugin"]),
8
+ });
9
+
10
+ export function isMemoryPluginEntry(entry) {
11
+ const spec = Array.isArray(entry)
12
+ ? entry[0]
13
+ : entry && typeof entry === "object"
14
+ ? entry.package
15
+ : entry;
16
+ return isMemoryPluginSpec(spec);
17
+ }
18
+
19
+ export function isMemoryMcpServerEntry(entry) {
20
+ if (!entry || typeof entry !== "object") return false;
21
+ const command = String(entry.command || "").replace(/\\/g, "/").toLowerCase();
22
+ const args = Array.isArray(entry.args)
23
+ ? entry.args.map((value) => String(value).replace(/\\/g, "/").toLowerCase())
24
+ : [];
25
+ return args.some((arg) => /^@lotargo\/memory_plugin(?:@[^/]+)?$/i.test(arg) || /^opencode-memory-plugin(?:@[^/]+)?$/i.test(arg))
26
+ || args.some((arg) => arg.endsWith("/mcp-server/boot.js") && arg.includes("memory"))
27
+ || /(?:^|\/)(?:memory_plugin|memory-agent)(?:\.(?:cmd|exe|ps1|bat))?$/i.test(command);
28
+ }
29
+
30
+ export async function readJsonConfig(filePath) {
31
+ if (!existsSync(filePath)) return {};
32
+ const raw = (await readFile(filePath, "utf-8")).replace(/^\uFEFF/, "");
33
+ const value = JSON.parse(raw);
34
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
35
+ throw new Error(`Expected a JSON object in ${filePath}`);
36
+ }
37
+ return value;
38
+ }
@@ -102,6 +102,44 @@ function sectionRanges(lines) {
102
102
  }));
103
103
  }
104
104
 
105
+ function validateOwnedMemoryAgentTargets(lines, ranges) {
106
+ const targets = ranges.filter((range) => isMemoryAgentHeader(range.name));
107
+ if (targets.length === 0) return { targets, exactTargets: [], conflict: null };
108
+
109
+ const exactTargets = targets.filter((range) => isMemoryAgentHeader(range.name, { exact: true }));
110
+ const unowned = exactTargets.filter((range) => {
111
+ const text = lines.slice(range.start, range.end).join("\n");
112
+ return !isMemoryPluginOwnedSection(text);
113
+ });
114
+ if (unowned.length > 0 || exactTargets.length === 0) {
115
+ return {
116
+ targets,
117
+ exactTargets,
118
+ conflict: "Existing memory-agent TOML section is not recognized as owned by @lotargo/memory_plugin",
119
+ };
120
+ }
121
+
122
+ let activeOwnedRoot = false;
123
+ for (const range of ranges) {
124
+ if (isMemoryAgentHeader(range.name, { exact: true })) {
125
+ const text = lines.slice(range.start, range.end).join("\n");
126
+ activeOwnedRoot = isMemoryPluginOwnedSection(text);
127
+ } else if (isMemoryAgentHeader(range.name)) {
128
+ if (!activeOwnedRoot) {
129
+ return {
130
+ targets,
131
+ exactTargets,
132
+ conflict: "A memory-agent child TOML section is not attached to an owned plugin section",
133
+ };
134
+ }
135
+ } else {
136
+ activeOwnedRoot = false;
137
+ }
138
+ }
139
+
140
+ return { targets, exactTargets, conflict: null };
141
+ }
142
+
105
143
  export function getCodexMemoryAgentSections(content) {
106
144
  const lines = String(content || "").split(/\r?\n/);
107
145
  return sectionRanges(lines)
@@ -113,25 +151,65 @@ export function getCodexMemoryAgentSections(content) {
113
151
  }));
114
152
  }
115
153
 
154
+ export function removeCodexMemoryAgentConfig(content) {
155
+ const source = String(content || "");
156
+ const eol = source.includes("\r\n") ? "\r\n" : "\n";
157
+ const lines = source.split(/\r?\n/);
158
+ const ranges = sectionRanges(lines);
159
+ const { targets, conflict } = validateOwnedMemoryAgentTargets(lines, ranges);
160
+
161
+ if (targets.length === 0) {
162
+ return { content: source, changed: false, status: "not_found", removed: 0 };
163
+ }
164
+
165
+ if (conflict) {
166
+ return {
167
+ content: source,
168
+ changed: false,
169
+ status: "conflict",
170
+ reason: conflict,
171
+ removed: 0,
172
+ };
173
+ }
174
+
175
+ const targetLineIndexes = new Set();
176
+ for (const target of targets) {
177
+ let contentEnd = target.end;
178
+ while (contentEnd > target.start + 1 && lines[contentEnd - 1] === "") contentEnd--;
179
+ for (let i = target.start; i < contentEnd; i++) targetLineIndexes.add(i);
180
+ }
181
+
182
+ const result = [];
183
+ for (let i = 0; i < lines.length; i++) {
184
+ if (targetLineIndexes.has(i)) continue;
185
+ result.push(lines[i]);
186
+ }
187
+
188
+ let updated = result.join(eol);
189
+ if (updated.trim() === "") updated = "";
190
+
191
+ return {
192
+ content: updated,
193
+ changed: updated !== source,
194
+ status: updated !== source ? "removed" : "unchanged",
195
+ removed: targets.length,
196
+ };
197
+ }
198
+
116
199
  export function updateCodexMemoryAgentConfig(content, options) {
117
200
  const source = String(content || "");
118
201
  const eol = source.includes("\r\n") ? "\r\n" : "\n";
119
202
  const lines = source.split(/\r?\n/);
120
203
  const desired = buildCodexMemoryAgentSection(options).split("\n");
121
204
  const ranges = sectionRanges(lines);
122
- const targets = ranges.filter((range) => isMemoryAgentHeader(range.name));
123
- const exactTargets = targets.filter((range) => isMemoryAgentHeader(range.name, { exact: true }));
205
+ const { targets, conflict } = validateOwnedMemoryAgentTargets(lines, ranges);
124
206
 
125
- const unowned = exactTargets.filter((range) => {
126
- const text = lines.slice(range.start, range.end).join("\n");
127
- return !isMemoryPluginOwnedSection(text);
128
- });
129
- if (unowned.length > 0) {
207
+ if (conflict) {
130
208
  return {
131
209
  content: source,
132
210
  changed: false,
133
211
  status: "conflict",
134
- reason: "Existing [mcp_servers.memory-agent] section is not recognized as owned by @lotargo/memory_plugin",
212
+ reason: conflict,
135
213
  };
136
214
  }
137
215
 
@@ -10,7 +10,7 @@ import { createClient } from "@libsql/client";
10
10
  let dbInstance = null;
11
11
  let dbInitPromise = null;
12
12
  let dbLastFailAt = 0;
13
- const DB_FAIL_COOLDOWN_MS = 5_000; // don't retry cloud init within 5s of a failure
13
+ const DB_FAIL_COOLDOWN_MS = 5_000;
14
14
 
15
15
  export const STORAGE_DIR = join(MEMORY_DIR, "storage");
16
16
  export const BLOBS_DIR = join(STORAGE_DIR, "blobs");
@@ -34,8 +34,6 @@ class DatabaseWrapper {
34
34
 
35
35
  while (attempts < maxAttempts) {
36
36
  attempts++;
37
- // A plain timer is enough here; an AbortController per attempt leaked its
38
- // "abort" listener because it was never removed.
39
37
  let timeoutId = null;
40
38
  const timeoutPromise = new Promise((_, reject) => {
41
39
  timeoutId = setTimeout(
@@ -48,7 +46,6 @@ class DatabaseWrapper {
48
46
  const client = (this.usingFailover && this.failoverClient) ? this.failoverClient : this.cloudClient;
49
47
  const result = await Promise.race([fn(client), timeoutPromise]);
50
48
  clearTimeout(timeoutId);
51
- // Successful operation, reset consecutive failures
52
49
  this.consecutiveFailures = 0;
53
50
  return result;
54
51
  } catch (err) {
@@ -58,12 +55,10 @@ class DatabaseWrapper {
58
55
  if (this.consecutiveFailures >= 3 && this.failoverClient && !this.usingFailover) {
59
56
  console.warn("[WARN] Turso is temporarily unreachable. Switching to LiteFS failover replica...");
60
57
  this.usingFailover = true;
61
- // Retry the operation on the failover client
62
58
  return this.runWithRetry(fn);
63
59
  }
64
60
  throw err;
65
61
  }
66
- // Small delay before retrying (exponential backoff / fixed delay)
67
62
  await new Promise((resolve) => setTimeout(resolve, 200));
68
63
  }
69
64
  }
@@ -156,16 +151,12 @@ async function openDatabase(customPath, mode) {
156
151
  localDb = new DatabaseSync(dbPath);
157
152
  localDb.exec("PRAGMA foreign_keys = ON;");
158
153
  localDb.exec("PRAGMA journal_mode = WAL;");
159
- // Default busy_timeout is 0 => an immediate SQLITE_BUSY ("database is
160
- // locked") whenever background sync, ingestion and MCP calls overlap.
161
154
  localDb.exec("PRAGMA busy_timeout = 5000;");
162
155
  }
163
156
 
164
157
  let cloudClient = null;
165
158
  let failoverClient = null;
166
159
  if (mode === "only-cloud" || mode === "hybrid-sync") {
167
- // Resolve working cloud credentials. An env TURSO_API_TOKEN (which can only
168
- // call the Platform API) is lazily minted into a per-database JWT here.
169
160
  const secrets = await resolveCloudSecrets();
170
161
  const tursoUrl = customPath && customPath.startsWith("libsql:") ? customPath : (secrets?.dbUrl || config.tursoUrl);
171
162
  const failoverUrl = config.failoverUrl || "";
@@ -182,8 +173,6 @@ async function openDatabase(customPath, mode) {
182
173
  authToken: token || undefined,
183
174
  });
184
175
  }
185
- // In hybrid-sync mode, ensure remote schema is also fully migrated and up to date.
186
- // Use a dedicated short-lived client so closing it doesn't kill the shared one.
187
176
  if (mode === "hybrid-sync") {
188
177
  const remoteClient = createClient({
189
178
  url: tursoUrl,
@@ -199,13 +188,22 @@ async function openDatabase(customPath, mode) {
199
188
  }
200
189
 
201
190
  const wrappedDb = new DatabaseWrapper(localDb, cloudClient, mode, failoverClient);
202
-
203
- // Initialize/run migrations
204
191
  await runMigrations(wrappedDb);
205
192
 
193
+ // Upgrade path for RAG content ingested before portable cloud blobs existed.
194
+ // The backfill is content-addressed and uploads only hashes absent in Turso.
195
+ // Missing local files are simply reported/skipped; database availability must
196
+ // never depend on a legacy raw blob still being present on this machine.
197
+ if ((mode === "only-cloud" || mode === "hybrid-sync") && cloudClient) {
198
+ try {
199
+ const { backfillCloudBlobsFromLocal } = await import("./rag_blob_transport.js");
200
+ await backfillCloudBlobsFromLocal(wrappedDb);
201
+ } catch (err) {
202
+ console.warn("[WARN] RAG cloud blob backfill skipped:", err.message);
203
+ }
204
+ }
205
+
206
206
  if (!customPath) {
207
- // Closing the previous instance before replacing it: otherwise a mode switch
208
- // leaked the old DatabaseSync handle and Turso client.
209
207
  if (dbInstance && dbInstance !== wrappedDb) {
210
208
  try {
211
209
  dbInstance.close();
@@ -225,15 +223,10 @@ export async function getDatabase(customPath = null, forceMode = null) {
225
223
  if (dbInstance && dbInstance.mode === mode) {
226
224
  return dbInstance;
227
225
  }
228
- // After a failed init, wait before retrying to avoid hammering cloud auth.
229
- // Only cloud modes are throttled — reopening a local SQLite file is cheap
230
- // and must never be blocked by an unrelated cloud failure.
231
226
  const isCloudMode = mode === "only-cloud" || mode === "hybrid-sync";
232
227
  if (isCloudMode && !dbInitPromise && dbLastFailAt && (Date.now() - dbLastFailAt) < DB_FAIL_COOLDOWN_MS) {
233
228
  throw new Error("Database initialization failed recently. Retrying in a few seconds...");
234
229
  }
235
- // Deduplicate concurrent default-DB initialization so migrations never run
236
- // on multiple connections at once (avoids "database is locked" crashes).
237
230
  if (!dbInitPromise) {
238
231
  dbInitPromise = openDatabase(null, mode).then((result) => {
239
232
  dbLastFailAt = 0;
@@ -3,7 +3,6 @@ const MIGRATIONS = [
3
3
  version: 1,
4
4
  name: "001_initial_rag_schema",
5
5
  up: async (db) => {
6
- // 1. Documents Table
7
6
  await db.exec(`
8
7
  CREATE TABLE IF NOT EXISTS documents (
9
8
  id TEXT PRIMARY KEY,
@@ -17,8 +16,6 @@ const MIGRATIONS = [
17
16
  updated_at INTEGER NOT NULL
18
17
  );
19
18
  `);
20
-
21
- // 2. Sections Table
22
19
  await db.exec(`
23
20
  CREATE TABLE IF NOT EXISTS sections (
24
21
  id TEXT PRIMARY KEY,
@@ -29,8 +26,6 @@ const MIGRATIONS = [
29
26
  token_count INTEGER NOT NULL
30
27
  );
31
28
  `);
32
-
33
- // 3. Micro-Chunks Table
34
29
  await db.exec(`
35
30
  CREATE TABLE IF NOT EXISTS micro_chunks (
36
31
  id TEXT PRIMARY KEY,
@@ -41,8 +36,6 @@ const MIGRATIONS = [
41
36
  token_count INTEGER NOT NULL
42
37
  );
43
38
  `);
44
-
45
- // 4. Full-Text Search (BM25 Index via SQLite FTS5)
46
39
  await db.exec(`
47
40
  CREATE VIRTUAL TABLE IF NOT EXISTS micro_chunks_fts USING fts5(
48
41
  id UNINDEXED,
@@ -50,8 +43,6 @@ const MIGRATIONS = [
50
43
  breadcrumbs
51
44
  );
52
45
  `);
53
-
54
- // 5. GraphRAG Lite Edges Table
55
46
  await db.exec(`
56
47
  CREATE TABLE IF NOT EXISTS graph_edges (
57
48
  source_id TEXT NOT NULL,
@@ -66,13 +57,8 @@ const MIGRATIONS = [
66
57
  version: 2,
67
58
  name: "002_agent_knowledge_graph",
68
59
  up: async (db) => {
69
- try {
70
- await db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`);
71
- } catch (e) {}
72
- try {
73
- await db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`);
74
- } catch (e) {}
75
-
60
+ try { await db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`); } catch (e) {}
61
+ try { await db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`); } catch (e) {}
76
62
  await db.exec(`
77
63
  CREATE TABLE IF NOT EXISTS knowledge_links (
78
64
  id TEXT PRIMARY KEY,
@@ -104,10 +90,7 @@ const MIGRATIONS = [
104
90
  created_at INTEGER
105
91
  );
106
92
  `);
107
-
108
- try {
109
- await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
110
- } catch (e) {}
93
+ try { await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`); } catch (e) {}
111
94
  },
112
95
  },
113
96
  {
@@ -116,65 +99,81 @@ const MIGRATIONS = [
116
99
  up: async (db) => {
117
100
  await db.exec(`
118
101
  CREATE TABLE IF NOT EXISTS project_identities (
119
- key TEXT PRIMARY KEY,
120
- name TEXT NOT NULL,
102
+ key TEXT PRIMARY KEY,
103
+ name TEXT NOT NULL,
121
104
  primary_remote TEXT,
122
- created_at INTEGER NOT NULL,
123
- updated_at INTEGER NOT NULL
105
+ created_at INTEGER NOT NULL,
106
+ updated_at INTEGER NOT NULL
124
107
  );
125
108
  `);
126
-
127
109
  await db.exec(`
128
110
  CREATE TABLE IF NOT EXISTS project_aliases (
129
- alias TEXT PRIMARY KEY,
111
+ alias TEXT PRIMARY KEY,
130
112
  identity_key TEXT NOT NULL REFERENCES project_identities(key) ON DELETE CASCADE,
131
- kind TEXT NOT NULL,
132
- created_at INTEGER NOT NULL
113
+ kind TEXT NOT NULL,
114
+ created_at INTEGER NOT NULL
133
115
  );
134
116
  `);
135
-
117
+ await db.exec(`CREATE INDEX IF NOT EXISTS idx_project_aliases_identity ON project_aliases(identity_key);`);
118
+ },
119
+ },
120
+ {
121
+ version: 5,
122
+ name: "005_retrieval_policy",
123
+ up: async (db) => {
124
+ try { await db.exec(`ALTER TABLE micro_chunks ADD COLUMN retrieval_policy TEXT DEFAULT 'micro_chunk';`); } catch (e) {}
125
+ try { await db.exec(`ALTER TABLE micro_chunks ADD COLUMN policy_source_id TEXT;`); } catch (e) {}
126
+ await db.exec(`CREATE INDEX IF NOT EXISTS idx_micro_chunks_retrieval_policy ON micro_chunks(retrieval_policy);`);
127
+ },
128
+ },
129
+ {
130
+ version: 6,
131
+ name: "006_project_scoped_rag",
132
+ up: async (db) => {
136
133
  await db.exec(`
137
- CREATE INDEX IF NOT EXISTS idx_project_aliases_identity ON project_aliases(identity_key);
134
+ CREATE TABLE IF NOT EXISTS document_scopes (
135
+ doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
136
+ scope_key TEXT NOT NULL,
137
+ created_at INTEGER NOT NULL,
138
+ PRIMARY KEY (doc_id, scope_key)
139
+ );
140
+ `);
141
+ await db.exec(`CREATE INDEX IF NOT EXISTS idx_document_scopes_scope ON document_scopes(scope_key, doc_id);`);
142
+ await db.exec(`
143
+ INSERT OR IGNORE INTO document_scopes (doc_id, scope_key, created_at)
144
+ SELECT id, 'global', created_at FROM documents;
138
145
  `);
139
146
  },
140
147
  },
141
- {
142
- version: 5,
143
- name: "005_retrieval_policy",
148
+ {
149
+ version: 7,
150
+ name: "007_rag_blob_transport",
144
151
  up: async (db) => {
145
- try {
146
- await db.exec(`ALTER TABLE micro_chunks ADD COLUMN retrieval_policy TEXT DEFAULT 'micro_chunk';`);
147
- } catch (e) {}
148
- try {
149
- await db.exec(`ALTER TABLE micro_chunks ADD COLUMN policy_source_id TEXT;`);
150
- } catch (e) {}
151
152
  await db.exec(`
152
- CREATE INDEX IF NOT EXISTS idx_micro_chunks_retrieval_policy ON micro_chunks(retrieval_policy);
153
+ CREATE TABLE IF NOT EXISTS rag_blobs (
154
+ hash TEXT PRIMARY KEY,
155
+ gzip_base64 TEXT NOT NULL,
156
+ raw_size INTEGER NOT NULL DEFAULT 0,
157
+ created_at INTEGER NOT NULL
158
+ );
153
159
  `);
154
- },
155
- },
156
- {
157
- version: 6,
158
- name: "006_project_scoped_rag",
159
- up: async (db) => {
160
- await db.exec(`
161
- CREATE TABLE IF NOT EXISTS document_scopes (
162
- doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
163
- scope_key TEXT NOT NULL,
164
- created_at INTEGER NOT NULL,
165
- PRIMARY KEY (doc_id, scope_key)
166
- );
167
- `);
168
- await db.exec(`
169
- CREATE INDEX IF NOT EXISTS idx_document_scopes_scope ON document_scopes(scope_key, doc_id);
170
- `);
171
- await db.exec(`
172
- INSERT OR IGNORE INTO document_scopes (doc_id, scope_key, created_at)
173
- SELECT id, 'global', created_at FROM documents;
174
- `);
175
- },
176
- },
177
- ];
160
+ },
161
+ },
162
+ {
163
+ version: 8,
164
+ name: "008_rag_document_tombstones",
165
+ up: async (db) => {
166
+ await db.exec(`
167
+ CREATE TABLE IF NOT EXISTS rag_document_tombstones (
168
+ doc_id TEXT PRIMARY KEY,
169
+ path TEXT,
170
+ deleted_at INTEGER NOT NULL
171
+ );
172
+ `);
173
+ await db.exec(`CREATE INDEX IF NOT EXISTS idx_rag_tombstones_path ON rag_document_tombstones(path);`);
174
+ },
175
+ },
176
+ ];
178
177
 
179
178
  export async function runMigrations(db) {
180
179
  let currentVersion = 0;
@@ -188,12 +187,7 @@ export async function runMigrations(db) {
188
187
  } catch (e2) {}
189
188
  }
190
189
 
191
- // Ensure schema_migrations table exists for future
192
- try {
193
- await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY);`);
194
- } catch (e) {}
195
-
196
- // Create notebooks table if not exists
190
+ try { await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY);`); } catch (e) {}
197
191
  try {
198
192
  await db.exec(`
199
193
  CREATE TABLE IF NOT EXISTS notebooks (
@@ -211,19 +205,14 @@ export async function runMigrations(db) {
211
205
  await migration.up(db);
212
206
  await db.prepare("INSERT INTO schema_migrations (version) VALUES (?);").run(migration.version);
213
207
  await db.exec("COMMIT;");
214
- try {
215
- await db.exec(`PRAGMA user_version = ${migration.version};`);
216
- } catch (e) {}
208
+ try { await db.exec(`PRAGMA user_version = ${migration.version};`); } catch (e) {}
217
209
  } catch (err) {
218
- try {
219
- await db.exec("ROLLBACK;");
220
- } catch (e) {}
210
+ try { await db.exec("ROLLBACK;"); } catch (e) {}
221
211
  throw new Error(`Migration ${migration.name} failed: ${err.message}`);
222
212
  }
223
213
  }
224
214
  }
225
215
 
226
- // Defensive table & column check for medium_chunks hierarchy
227
216
  try {
228
217
  await db.exec(`
229
218
  CREATE TABLE IF NOT EXISTS medium_chunks (