@lotargo/memory_plugin 1.6.6 → 1.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +148 -101
  2. package/README.md +436 -304
  3. package/mcp-server/cli/direct_commands.js +39 -0
  4. package/mcp-server/cli.js +16 -5
  5. package/mcp-server/cli_boot.js +4 -1
  6. package/mcp-server/client_cli.js +73 -0
  7. package/mcp-server/client_paths.js +44 -0
  8. package/mcp-server/client_registration.js +38 -0
  9. package/mcp-server/codex_config.js +86 -8
  10. package/mcp-server/db/database.js +14 -21
  11. package/mcp-server/db/migrations.js +66 -77
  12. package/mcp-server/db/rag_blob_transport.js +143 -0
  13. package/mcp-server/db/rag_sync.js +284 -0
  14. package/mcp-server/db/sync_queue.js +219 -307
  15. package/mcp-server/dev_link.js +142 -0
  16. package/mcp-server/fact_format.js +44 -12
  17. package/mcp-server/index.js +17 -7
  18. package/mcp-server/ingest/exporter.js +44 -38
  19. package/mcp-server/ingest/normalizer.js +1 -1
  20. package/mcp-server/ingest/pipeline.js +260 -248
  21. package/mcp-server/persona_migration.js +39 -0
  22. package/mcp-server/prompt_manager.js +162 -55
  23. package/mcp-server/retrieval/retriever.js +99 -64
  24. package/mcp-server/setup.js +150 -100
  25. package/mcp-server/storage/blob_store.js +53 -1
  26. package/mcp-server/tools/core/knowledge_read_core.js +163 -0
  27. package/mcp-server/tools/core/memory_core.js +24 -4
  28. package/mcp-server/tools/core/memory_routing.js +10 -0
  29. package/mcp-server/tools/core/note_core.js +53 -0
  30. package/mcp-server/tools/core/rag_query_core.js +169 -0
  31. package/mcp-server/tools/index.js +11 -9
  32. package/mcp-server/tools/memory_tools.js +4 -1
  33. package/mcp-server/tools/note_tools.js +35 -0
  34. package/mcp-server/tools/rag_tools.js +211 -364
  35. package/mcp-server/uninstall.js +627 -0
  36. package/opencode-plugin/index.js +80 -12
  37. package/opencode-plugin/main.js +136 -0
  38. package/package.json +25 -5
  39. package/skills/using-memory/SKILL.md +28 -19
  40. package/mcp-server/benchmarks/fetch_real_corpus.js +0 -351
  41. package/mcp-server/benchmarks/gpu_profile_benchmark.js +0 -170
  42. package/mcp-server/benchmarks/policy_dominance_test.js +0 -221
  43. package/mcp-server/benchmarks/quality_evaluator.js +0 -598
  44. package/mcp-server/benchmarks/raw_corpus_data.js +0 -613
  45. package/mcp-server/benchmarks/run_benchmarks.js +0 -366
  46. package/mcp-server/benchmarks/stress_ingestion.js +0 -195
  47. package/mcp-server/benchmarks/table_code_retrieval.js +0 -453
  48. package/mcp-server/benchmarks/test_dual_layer.js +0 -141
@@ -1,19 +1,59 @@
1
1
  import { readFile, writeFile, mkdir, cp, readdir } from "fs/promises";
2
2
  import { existsSync } from "fs";
3
- import { join, dirname } from "path";
4
- import { homedir } from "os";
5
- import { fileURLToPath } from "url";
6
-
7
- export async function runSetup() {
8
- const args = process.argv.slice(2);
9
- const hasSpecificFlag = args.some((a) =>
10
- ["--opencode", "--claude", "--codex", "--antigravity", "--gemini"].includes(a.toLowerCase())
11
- );
3
+ import { join, dirname } from "path";
4
+ import { fileURLToPath } from "url";
5
+ import { resolveClientPaths } from "./client_paths.js";
6
+ import { cliFailureMessage, runClientCli } from "./client_cli.js";
7
+ import { MEMORY_MCP_ENTRY, isMemoryMcpServerEntry, isMemoryPluginEntry, readJsonConfig } from "./client_registration.js";
8
+
9
+ async function configureJsonMcpClient({ label, cliName, cliArgs, configPath }) {
10
+ let config = await readJsonConfig(configPath);
11
+ const existing = config.mcpServers?.["memory-agent"];
12
+ if (existing && !isMemoryMcpServerEntry(existing)) {
13
+ throw new Error(`Existing mcpServers.memory-agent in ${configPath} is not owned by this plugin`);
14
+ }
15
+ if (existing) return { method: "existing", configPath };
16
+
17
+ const native = runClientCli(cliName, cliArgs);
18
+ if (native.ok) {
19
+ config = await readJsonConfig(configPath);
20
+ if (isMemoryMcpServerEntry(config.mcpServers?.["memory-agent"])) {
21
+ return { method: "native", configPath };
22
+ }
23
+ }
24
+
25
+ // Compatibility fallback for older/missing clients, and for wrappers that
26
+ // report success without writing the expected user-scope configuration.
27
+ config = await readJsonConfig(configPath);
28
+ const afterNative = config.mcpServers?.["memory-agent"];
29
+ if (afterNative && !isMemoryMcpServerEntry(afterNative)) {
30
+ throw new Error(`Native ${label} setup created an unrecognized memory-agent entry in ${configPath}`);
31
+ }
32
+ if (!config.mcpServers || typeof config.mcpServers !== "object" || Array.isArray(config.mcpServers)) {
33
+ config.mcpServers = {};
34
+ }
35
+ config.mcpServers["memory-agent"] = MEMORY_MCP_ENTRY;
36
+ await mkdir(dirname(configPath), { recursive: true });
37
+ await writeFile(configPath, JSON.stringify(config, null, 2) + "\n", "utf-8");
38
+ return {
39
+ method: "fallback",
40
+ configPath,
41
+ nativeReason: native.ok ? "native command did not create the expected entry" : cliFailureMessage(native),
42
+ };
43
+ }
12
44
 
13
- const doOpenCode = !hasSpecificFlag || args.includes("--opencode");
14
- const doClaude = !hasSpecificFlag || args.includes("--claude");
15
- const doAntigravity = !hasSpecificFlag || args.includes("--antigravity") || args.includes("--gemini");
16
- const doCodex = !hasSpecificFlag || args.includes("--codex");
45
+ export async function runSetup() {
46
+ const args = process.argv.slice(2);
47
+ const lowerArgs = args.map((arg) => String(arg).toLowerCase());
48
+ const hasSpecificFlag = lowerArgs.some((a) =>
49
+ ["--opencode", "--claude", "--codex", "--antigravity", "--gemini"].includes(a.toLowerCase())
50
+ );
51
+
52
+ const doOpenCode = !hasSpecificFlag || lowerArgs.includes("--opencode");
53
+ const doClaude = !hasSpecificFlag || lowerArgs.includes("--claude");
54
+ const doAntigravity = !hasSpecificFlag || lowerArgs.includes("--antigravity");
55
+ const doGemini = !hasSpecificFlag || lowerArgs.includes("--gemini");
56
+ const doCodex = !hasSpecificFlag || lowerArgs.includes("--codex");
17
57
 
18
58
  // Headless cloud setup: --api-key <TURSO_API_TOKEN> and/or --mode <only-local|only-cloud|hybrid-sync>
19
59
  const VALID_MODES = ["only-local", "only-cloud", "hybrid-sync"];
@@ -24,7 +64,8 @@ export async function runSetup() {
24
64
  }
25
65
 
26
66
  console.log("\nSetting up @lotargo/memory_plugin...\n");
27
- const home = homedir();
67
+ const clientPaths = resolveClientPaths();
68
+ const { home } = clientPaths;
28
69
  let configuredCount = 0;
29
70
 
30
71
  // 0. Headless cloud authentication (Google Jules / CI / VPS)
@@ -62,28 +103,13 @@ export async function runSetup() {
62
103
  // 1. OpenCode (~/.config/opencode/opencode.json)
63
104
  if (doOpenCode) {
64
105
  try {
65
- const opencodeDir = process.env.OPENCODE_CONFIG_DIR || join(home, ".config", "opencode");
66
- const opencodeConfigPath = join(opencodeDir, "opencode.json");
106
+ const { opencodeDir, opencodeConfigPath } = clientPaths;
67
107
  await mkdir(opencodeDir, { recursive: true });
68
108
 
69
- let config = {};
70
- if (existsSync(opencodeConfigPath)) {
71
- try {
72
- config = JSON.parse(await readFile(opencodeConfigPath, "utf-8"));
73
- } catch (e) {}
74
- }
75
- if (!Array.isArray(config.plugin)) config.plugin = [];
76
- // Clean up legacy / obsolete / duplicate entries of OUR plugin only
77
- const obsoleteNames = ["opencode-memory-plugin", "memory_plugin", "memory-plugin", "@lotargo/memory_plugin"];
78
- config.plugin = config.plugin.filter((p) => {
79
- if (typeof p !== "string") return true;
80
- if (obsoleteNames.includes(p)) return false;
81
- const normalized = p.replace(/\\/g, "/").toLowerCase();
82
- if (normalized.endsWith("/memory") || normalized.endsWith("/memory_plugin") || normalized.endsWith("/memory-plugin")) {
83
- return false;
84
- }
85
- return true;
86
- });
109
+ const config = await readJsonConfig(opencodeConfigPath);
110
+ if (!Array.isArray(config.plugin)) config.plugin = [];
111
+ // Clean up legacy / obsolete / duplicate entries of OUR plugin only
112
+ config.plugin = config.plugin.filter((entry) => !isMemoryPluginEntry(entry));
87
113
  config.plugin.push("@lotargo/memory_plugin");
88
114
  // Clean up legacy mcp-helper.js standalone file plugin if present
89
115
  const legacyPluginFile = join(opencodeDir, "plugins", "mcp-helper.js");
@@ -92,11 +118,18 @@ export async function runSetup() {
92
118
  }
93
119
 
94
120
  // Purge stale OpenCode package cache for memory plugin so OpenCode downloads latest version
95
- const opencodeCachePackages = join(home, ".cache", "opencode", "packages");
121
+ const opencodeCachePackages = clientPaths.opencodeCachePackages;
96
122
  if (existsSync(opencodeCachePackages)) {
97
123
  try {
98
124
  const { rm } = await import("fs/promises");
99
- const targets = ["@lotargo", "memory_plugin", "memory_plugin@latest", "opencode-memory-plugin", "opencode-memory-plugin@latest"];
125
+ const targets = [
126
+ join("@lotargo", "memory_plugin"),
127
+ join("@lotargo", "memory_plugin@latest"),
128
+ "memory_plugin",
129
+ "memory_plugin@latest",
130
+ "opencode-memory-plugin",
131
+ "opencode-memory-plugin@latest",
132
+ ];
100
133
  for (const t of targets) {
101
134
  const p = join(opencodeCachePackages, t);
102
135
  if (existsSync(p)) await rm(p, { recursive: true, force: true });
@@ -112,52 +145,42 @@ export async function runSetup() {
112
145
  }
113
146
  }
114
147
 
115
- // 2. Claude Code (~/.claude.json)
116
- if (doClaude) {
117
- try {
118
- const claudePath = join(home, ".claude.json");
119
- let config = {};
120
- if (existsSync(claudePath)) {
121
- try {
122
- config = JSON.parse(await readFile(claudePath, "utf-8"));
123
- } catch (e) {}
124
- }
125
- if (!config.mcpServers) config.mcpServers = {};
126
- config.mcpServers["memory-agent"] = {
127
- command: "npx",
128
- args: ["-y", "@lotargo/memory_plugin"],
129
- };
130
- await writeFile(claudePath, JSON.stringify(config, null, 2));
131
- console.log(" [OK] Claude Code: configured MCP server in ~/.claude.json");
132
- configuredCount++;
133
- } catch (err) {
134
- console.log(" [SKIP] Claude Code setup skipped:", err.message);
135
- }
136
- }
137
-
138
- // 3. Antigravity / Gemini CLI (~/.gemini/config/mcp_config.json & .agents/mcp_config.json)
148
+ // 2. Claude Code (native user-scope MCP lifecycle, JSON fallback)
149
+ if (doClaude) {
150
+ try {
151
+ const result = await configureJsonMcpClient({
152
+ label: "Claude Code",
153
+ cliName: "claude",
154
+ cliArgs: ["mcp", "add", "--scope", "user", "memory-agent", "--", "npx", "-y", "@lotargo/memory_plugin"],
155
+ configPath: clientPaths.claudeConfigPath,
156
+ });
157
+ console.log(` [OK] Claude Code: configured MCP server via ${result.method === "native" ? "claude mcp add" : result.method === "existing" ? "existing owned registration" : "ownership-checked JSON fallback"}`);
158
+ if (result.nativeReason) console.log(` [INFO] Claude Code fallback: ${result.nativeReason}`);
159
+ configuredCount++;
160
+ } catch (err) {
161
+ console.log(" [SKIP] Claude Code setup skipped:", err.message);
162
+ }
163
+ }
164
+
165
+ // 3. Antigravity (~/.gemini/config/mcp_config.json & .agents/mcp_config.json)
139
166
  if (doAntigravity) {
140
167
  try {
141
168
  // Global Antigravity config
142
- const geminiConfigDir = join(home, ".gemini", "config");
169
+ const geminiConfigDir = clientPaths.geminiConfigDir;
143
170
  await mkdir(geminiConfigDir, { recursive: true });
144
171
  const geminiConfigFile = join(geminiConfigDir, "mcp_config.json");
145
172
 
146
- let config = {};
147
- if (existsSync(geminiConfigFile)) {
148
- try {
149
- config = JSON.parse(await readFile(geminiConfigFile, "utf-8"));
150
- } catch (e) {}
151
- }
152
- if (!config.mcpServers) config.mcpServers = {};
153
- config.mcpServers["memory-agent"] = {
154
- command: "npx",
155
- args: ["-y", "@lotargo/memory_plugin"],
156
- };
157
- await writeFile(geminiConfigFile, JSON.stringify(config, null, 2));
173
+ const config = await readJsonConfig(geminiConfigFile);
174
+ if (!config.mcpServers) config.mcpServers = {};
175
+ const existingGlobal = config.mcpServers["memory-agent"];
176
+ if (existingGlobal && !isMemoryMcpServerEntry(existingGlobal)) {
177
+ throw new Error(`Existing Antigravity mcpServers.memory-agent in ${geminiConfigFile} is not owned by this plugin`);
178
+ }
179
+ config.mcpServers["memory-agent"] = MEMORY_MCP_ENTRY;
180
+ await writeFile(geminiConfigFile, JSON.stringify(config, null, 2) + "\n", "utf-8");
158
181
 
159
182
  // Local workspace config (.agents/mcp_config.json) only if .agents exists or --local flag is set
160
- const cwd = process.cwd();
183
+ const cwd = clientPaths.cwd;
161
184
  const hasAgentsDir = existsSync(join(cwd, ".agents"));
162
185
  const isLocalRequested = args.includes("--local");
163
186
 
@@ -165,18 +188,14 @@ export async function runSetup() {
165
188
  const localAgentsDir = join(cwd, ".agents");
166
189
  await mkdir(localAgentsDir, { recursive: true });
167
190
  const localMcpFile = join(localAgentsDir, "mcp_config.json");
168
- let localConfig = {};
169
- if (existsSync(localMcpFile)) {
170
- try {
171
- localConfig = JSON.parse(await readFile(localMcpFile, "utf-8"));
172
- } catch (e) {}
173
- }
174
- if (!localConfig.mcpServers) localConfig.mcpServers = {};
175
- localConfig.mcpServers["memory-agent"] = {
176
- command: "npx",
177
- args: ["-y", "@lotargo/memory_plugin"],
178
- };
179
- await writeFile(localMcpFile, JSON.stringify(localConfig, null, 2));
191
+ const localConfig = await readJsonConfig(localMcpFile);
192
+ if (!localConfig.mcpServers) localConfig.mcpServers = {};
193
+ const existingLocal = localConfig.mcpServers["memory-agent"];
194
+ if (existingLocal && !isMemoryMcpServerEntry(existingLocal)) {
195
+ throw new Error(`Existing Antigravity mcpServers.memory-agent in ${localMcpFile} is not owned by this plugin`);
196
+ }
197
+ localConfig.mcpServers["memory-agent"] = MEMORY_MCP_ENTRY;
198
+ await writeFile(localMcpFile, JSON.stringify(localConfig, null, 2) + "\n", "utf-8");
180
199
  console.log(" [OK] Antigravity: configured MCP server in ~/.gemini/config/mcp_config.json and .agents/mcp_config.json");
181
200
  } else {
182
201
  console.log(" [OK] Antigravity: configured MCP server in ~/.gemini/config/mcp_config.json");
@@ -185,17 +204,34 @@ export async function runSetup() {
185
204
  } catch (err) {
186
205
  console.log(" [SKIP] Antigravity setup skipped:", err.message);
187
206
  }
188
- }
189
-
190
- // 4. Codex (~/.codex/config.toml)
207
+ }
208
+
209
+ // 4. Gemini CLI (native user-scope MCP lifecycle, settings.json fallback)
210
+ if (doGemini) {
211
+ try {
212
+ const result = await configureJsonMcpClient({
213
+ label: "Gemini CLI",
214
+ cliName: "gemini",
215
+ cliArgs: ["mcp", "add", "--scope", "user", "memory-agent", "npx", "--", "-y", "@lotargo/memory_plugin"],
216
+ configPath: clientPaths.geminiSettingsPath,
217
+ });
218
+ console.log(` [OK] Gemini CLI: configured MCP server via ${result.method === "native" ? "gemini mcp add" : result.method === "existing" ? "existing owned registration" : "ownership-checked settings.json fallback"}`);
219
+ if (result.nativeReason) console.log(` [INFO] Gemini CLI fallback: ${result.nativeReason}`);
220
+ configuredCount++;
221
+ } catch (err) {
222
+ console.log(" [SKIP] Gemini CLI setup skipped:", err.message);
223
+ }
224
+ }
225
+
226
+ // 5. Codex (~/.codex/config.toml)
191
227
  if (doCodex) {
192
228
  try {
193
229
  const {
194
230
  updateCodexMemoryAgentConfig,
195
231
  validateCodexRuntime,
196
232
  } = await import("./codex_config.js");
197
- const codexDir = join(home, ".codex");
198
- const codexConfig = join(codexDir, "config.toml");
233
+ const codexDir = clientPaths.codexDir;
234
+ const codexConfig = clientPaths.codexConfigPath;
199
235
  const nodePath = process.execPath;
200
236
  const bootPath = fileURLToPath(new URL("./boot.js", import.meta.url));
201
237
  const runtime = validateCodexRuntime({ nodePath, nodeVersion: process.versions.node, bootPath });
@@ -204,32 +240,45 @@ export async function runSetup() {
204
240
  }
205
241
 
206
242
  await mkdir(codexDir, { recursive: true });
207
- const content = existsSync(codexConfig) ? await readFile(codexConfig, "utf-8") : "";
208
- const update = updateCodexMemoryAgentConfig(content, { nodePath, bootPath });
243
+ let content = existsSync(codexConfig) ? await readFile(codexConfig, "utf-8") : "";
244
+ let update = updateCodexMemoryAgentConfig(content, { nodePath, bootPath });
209
245
  if (update.status === "conflict") {
210
246
  throw new Error(update.reason);
211
247
  }
248
+ let nativeReason = null;
249
+ if (update.status === "added") {
250
+ const native = runClientCli("codex", ["mcp", "add", "memory-agent", "--", nodePath, bootPath]);
251
+ if (native.ok) {
252
+ content = existsSync(codexConfig) ? await readFile(codexConfig, "utf-8") : "";
253
+ update = updateCodexMemoryAgentConfig(content, { nodePath, bootPath });
254
+ if (update.status === "conflict") throw new Error(update.reason);
255
+ } else {
256
+ nativeReason = cliFailureMessage(native);
257
+ }
258
+ }
212
259
  if (update.changed) {
213
260
  await writeFile(codexConfig, update.content, "utf-8");
214
261
  console.log(
215
262
  update.status === "added"
216
- ? " [OK] Codex: added direct Node.js memory-agent launcher to ~/.codex/config.toml"
217
- : " [OK] Codex: migrated memory-agent to a direct Node.js launcher in ~/.codex/config.toml"
263
+ ? " [OK] Codex: added direct Node.js memory-agent launcher via ownership-checked TOML fallback"
264
+ : " [OK] Codex: normalized memory-agent registration after native setup"
218
265
  );
219
266
  } else {
220
- console.log(" [INFO] Codex: direct Node.js memory-agent launcher already configured");
267
+ console.log(" [INFO] Codex: direct Node.js memory-agent launcher configured via codex mcp add or already present");
221
268
  }
269
+ if (nativeReason) console.log(` [INFO] Codex fallback: ${nativeReason}`);
222
270
  configuredCount++;
223
271
  } catch (err) {
224
272
  console.log(" [FAIL] Codex setup failed:", err.message);
225
273
  }
226
274
  }
227
275
 
228
- // 5. Global Prompt Instructions (Antigravity, Codex, Claude Code)
276
+ // 6. Global Prompt Instructions
229
277
  try {
230
278
  const { enableGlobalPrompt } = await import("./prompt_manager.js");
231
279
  const promptTargets = [];
232
280
  if (doAntigravity) promptTargets.push("Antigravity");
281
+ if (doGemini) promptTargets.push("Gemini CLI");
233
282
  if (doCodex) promptTargets.push("Codex");
234
283
  if (doClaude) promptTargets.push("Claude Code");
235
284
  const promptResults = await enableGlobalPrompt(promptTargets);
@@ -248,21 +297,22 @@ export async function runSetup() {
248
297
  console.log(" [SKIP] Global prompt setup skipped:", err.message);
249
298
  }
250
299
 
251
- // 6. Global & Local Skill Installation (Antigravity, Codex, Claude Code)
300
+ // 7. Global & Local Skill Installation
252
301
  try {
253
302
  const packageDir = dirname(dirname(fileURLToPath(import.meta.url)));
254
303
  const packageSkillsDir = join(packageDir, "skills");
255
304
  if (existsSync(packageSkillsDir)) {
256
- const opencodeDir = process.env.OPENCODE_CONFIG_DIR || join(home, ".config", "opencode");
305
+ const opencodeDir = clientPaths.opencodeDir;
257
306
  const targets = [];
258
307
  if (doOpenCode) targets.push({ name: "OpenCode", dir: join(opencodeDir, "skills") });
259
308
  if (doAntigravity) targets.push({ name: "Antigravity", dir: join(home, ".gemini", "config", "skills") });
309
+ if (doGemini) targets.push({ name: "Gemini CLI", dir: clientPaths.geminiSkillsDir });
260
310
  if (doCodex) {
261
311
  targets.push({ name: "Codex", dir: join(home, ".codex", "skills") });
262
312
  targets.push({ name: "Codex shared agents", dir: join(home, ".agents", "skills") });
263
313
  }
264
314
  if (doClaude) targets.push({ name: "Claude Code", dir: join(home, ".claude", "skills") });
265
- const cwd = process.cwd();
315
+ const cwd = clientPaths.cwd;
266
316
  if (doAntigravity && existsSync(join(cwd, ".agents"))) {
267
317
  targets.push({ name: "Antigravity (local)", dir: join(cwd, ".agents", "skills") });
268
318
  }
@@ -5,7 +5,6 @@ import { existsSync } from "node:fs";
5
5
  import { join } from "node:path";
6
6
  import { BLOBS_DIR } from "../db/database.js";
7
7
 
8
- // Hard cap on gunzip output to prevent zip-bomb style memory exhaustion.
9
8
  export const MAX_UNPACKED_BYTES = 512 * 1024 * 1024;
10
9
 
11
10
  export function safeGunzip(compressed, maxBytes = MAX_UNPACKED_BYTES) {
@@ -65,6 +64,59 @@ export async function readBlob(hash, baseDir = BLOBS_DIR) {
65
64
  return decompressed.toString("utf-8");
66
65
  }
67
66
 
67
+ /**
68
+ * Return the exact gzip bytes used by the local content-addressed blob store as
69
+ * base64 text for portable SQLite/Turso transport. The raw content is validated
70
+ * against the requested SHA-256 before leaving the machine.
71
+ */
72
+ export async function readBlobTransport(hash, baseDir = BLOBS_DIR) {
73
+ const blobPath = getBlobPath(hash, baseDir);
74
+ if (!existsSync(blobPath)) {
75
+ throw new Error(`Blob not found for hash: ${hash}`);
76
+ }
77
+ const compressed = await readFile(blobPath);
78
+ const decompressed = safeGunzip(compressed);
79
+ const actualHash = hashContent(decompressed);
80
+ if (actualHash !== hash) {
81
+ throw new Error(`Blob integrity check failed: expected ${hash}, received ${actualHash}`);
82
+ }
83
+ return {
84
+ hash,
85
+ gzipBase64: compressed.toString("base64"),
86
+ rawSize: decompressed.length,
87
+ };
88
+ }
89
+
90
+ /**
91
+ * Materialize a transported gzip blob into the local filesystem only after
92
+ * decompression and SHA-256 verification. This prevents a corrupt/cloud payload
93
+ * from poisoning the local content-addressed store.
94
+ */
95
+ export async function saveBlobTransport(hash, gzipBase64, baseDir = BLOBS_DIR) {
96
+ if (!/^[a-f0-9]{64}$/i.test(String(hash || ""))) {
97
+ throw new Error(`Invalid blob hash: ${hash}`);
98
+ }
99
+ if (typeof gzipBase64 !== "string" || !gzipBase64) {
100
+ throw new Error(`Missing transported blob content for hash: ${hash}`);
101
+ }
102
+
103
+ const compressed = Buffer.from(gzipBase64, "base64");
104
+ const decompressed = safeGunzip(compressed);
105
+ const actualHash = hashContent(decompressed);
106
+ if (actualHash !== hash) {
107
+ throw new Error(`Transported blob integrity check failed: expected ${hash}, received ${actualHash}`);
108
+ }
109
+
110
+ const blobPath = getBlobPath(hash, baseDir);
111
+ if (existsSync(blobPath)) {
112
+ return { hash, size: decompressed.length, path: blobPath, deduplicated: true };
113
+ }
114
+ const parentDir = join(blobPath, "..");
115
+ if (!existsSync(parentDir)) await mkdir(parentDir, { recursive: true });
116
+ await writeFile(blobPath, compressed);
117
+ return { hash, size: decompressed.length, path: blobPath, deduplicated: false };
118
+ }
119
+
68
120
  export async function deleteBlob(hash, baseDir = BLOBS_DIR) {
69
121
  const blobPath = getBlobPath(hash, baseDir);
70
122
  if (existsSync(blobPath)) {
@@ -0,0 +1,163 @@
1
+ import { getDatabase } from "../../db/database.js";
2
+ import { resolveManageRagScopeKeys } from "../../rag_scope.js";
3
+ import { readBlob } from "../../storage/blob_store.js";
4
+ import { parseDocumentMetadata } from "../../retrieval/retriever.js";
5
+ import { getConfig } from "../../config/config_manager.js";
6
+
7
+ function normalizeTags(tags) {
8
+ if (Array.isArray(tags)) {
9
+ return [...new Set(tags.map((tag) => String(tag).trim().toLowerCase()).filter(Boolean))].sort();
10
+ }
11
+ if (typeof tags === "string") {
12
+ return [...new Set(tags.split(",").map((tag) => tag.trim().toLowerCase()).filter(Boolean))].sort();
13
+ }
14
+ return [];
15
+ }
16
+
17
+ function normalizeScopes(scopes) {
18
+ if (Array.isArray(scopes)) return [...new Set(scopes.filter(Boolean))];
19
+ if (typeof scopes === "string") return [...new Set(scopes.split(",").map((scope) => scope.trim()).filter(Boolean))];
20
+ return [];
21
+ }
22
+
23
+ export function normalizeKnowledgeDocumentMetadata(doc) {
24
+ const metadata = parseDocumentMetadata(doc?.metadata_json);
25
+ const sourceType = metadata.source_type || (String(doc?.path || "").startsWith("memory://note/") ? "note" : null);
26
+ const noteKind = sourceType === "note" ? (metadata.note_kind || "note") : null;
27
+ const tags = normalizeTags(metadata.tags);
28
+
29
+ return {
30
+ metadata,
31
+ sourceType,
32
+ noteKind,
33
+ tags,
34
+ };
35
+ }
36
+
37
+ function resolveEffectiveDirectory(directory, project, ctx) {
38
+ return directory || project || ctx.directory || null;
39
+ }
40
+
41
+ async function ensureKnowledgeFresh() {
42
+ if (getConfig().mode !== "hybrid-sync") return;
43
+ const { ensureReverseSync } = await import("../../db/sync_queue.js");
44
+ await ensureReverseSync();
45
+ }
46
+
47
+ async function readRawWithCloudFallback(db, blobHash) {
48
+ try {
49
+ return await readBlob(blobHash);
50
+ } catch (localErr) {
51
+ if (getConfig().mode === "only-local") throw localErr;
52
+
53
+ const { materializeBlobFromCloud } = await import("../../db/rag_blob_transport.js");
54
+ const result = await materializeBlobFromCloud(db, blobHash);
55
+ if (!result.materialized && !result.existing) {
56
+ throw new Error(
57
+ `Raw blob ${blobHash} is unavailable locally and could not be restored from cloud (${result.reason || "unknown"})`
58
+ );
59
+ }
60
+ return await readBlob(blobHash);
61
+ }
62
+ }
63
+
64
+ /**
65
+ * List scoped RAG documents/notes with normalized metadata.
66
+ */
67
+ export async function listKnowledgeDocuments(
68
+ { scope = null, directory = null, project = null } = {},
69
+ ctx = {}
70
+ ) {
71
+ await ensureKnowledgeFresh();
72
+ const db = await getDatabase();
73
+ const scopeKeys = await resolveManageRagScopeKeys("list", scope, {
74
+ worktree: ctx.worktree ?? null,
75
+ directory: resolveEffectiveDirectory(directory, project, ctx),
76
+ });
77
+ const placeholders = scopeKeys.map(() => "?").join(",");
78
+
79
+ const docs = await db.prepare(`
80
+ SELECT d.id, d.title, d.path, d.blob_hash, d.metadata_json,
81
+ d.created_at, d.updated_at,
82
+ GROUP_CONCAT(DISTINCT ds.scope_key) AS scopes
83
+ FROM documents d
84
+ JOIN document_scopes ds ON ds.doc_id = d.id
85
+ WHERE ds.scope_key IN (${placeholders})
86
+ GROUP BY d.id, d.title, d.path, d.blob_hash, d.metadata_json, d.created_at, d.updated_at
87
+ ORDER BY d.created_at DESC
88
+ `).all(...scopeKeys);
89
+
90
+ return docs.map((doc) => {
91
+ const { metadata, sourceType, noteKind, tags } = normalizeKnowledgeDocumentMetadata(doc);
92
+ return {
93
+ id: doc.id,
94
+ docId: doc.id,
95
+ title: doc.title,
96
+ path: doc.path,
97
+ blob_hash: doc.blob_hash,
98
+ source_type: sourceType,
99
+ note_kind: noteKind,
100
+ tags,
101
+ metadata,
102
+ scopes: normalizeScopes(doc.scopes),
103
+ created_at: doc.created_at ?? null,
104
+ updated_at: doc.updated_at ?? null,
105
+ };
106
+ });
107
+ }
108
+
109
+ /**
110
+ * Read the authoritative raw content for a scoped RAG document/note.
111
+ *
112
+ * In Turso-backed modes a missing local content-addressed blob is restored from
113
+ * the portable `rag_blobs` table and integrity-checked before use.
114
+ */
115
+ export async function readKnowledgeDocument(
116
+ { docId, scope = null, directory = null, project = null },
117
+ ctx = {}
118
+ ) {
119
+ if (!docId) throw new Error("docId parameter is required for read_document action");
120
+
121
+ await ensureKnowledgeFresh();
122
+ const db = await getDatabase();
123
+ const scopeKeys = await resolveManageRagScopeKeys("read_document", scope, {
124
+ worktree: ctx.worktree ?? null,
125
+ directory: resolveEffectiveDirectory(directory, project, ctx),
126
+ });
127
+ const placeholders = scopeKeys.map(() => "?").join(",");
128
+
129
+ const doc = await db
130
+ .prepare(`
131
+ SELECT d.id, d.title, d.path, d.blob_hash, d.metadata_json,
132
+ d.created_at, d.updated_at
133
+ FROM documents d
134
+ WHERE (d.id = ? OR d.path = ? OR d.title = ?)
135
+ AND EXISTS (
136
+ SELECT 1 FROM document_scopes ds
137
+ WHERE ds.doc_id = d.id
138
+ AND ds.scope_key IN (${placeholders})
139
+ )
140
+ `)
141
+ .get(docId, docId, docId, ...scopeKeys);
142
+
143
+ if (!doc) {
144
+ throw new Error(`Document not found in knowledge base for docId: ${docId}`);
145
+ }
146
+
147
+ const rawContent = await readRawWithCloudFallback(db, doc.blob_hash);
148
+ const { metadata, sourceType, noteKind, tags } = normalizeKnowledgeDocumentMetadata(doc);
149
+
150
+ return {
151
+ id: doc.id,
152
+ docId: doc.id,
153
+ title: doc.title,
154
+ path: doc.path,
155
+ source_type: sourceType,
156
+ note_kind: noteKind,
157
+ tags,
158
+ metadata,
159
+ created_at: doc.created_at ?? null,
160
+ updated_at: doc.updated_at ?? null,
161
+ content: rawContent,
162
+ };
163
+ }