@lotargo/memory_plugin 1.3.2 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { getDatabase } from "../db/database.js";
2
2
  import { randomUUID } from "node:crypto";
3
3
 
4
- export function linkFactToDocument({
4
+ export async function linkFactToDocument({
5
5
  factKey,
6
6
  factText,
7
7
  docId,
@@ -11,11 +11,11 @@ export function linkFactToDocument({
11
11
  relationType = "LINKS_TO",
12
12
  metadata = null,
13
13
  }) {
14
- const db = getDatabase();
14
+ const db = await getDatabase();
15
15
  const id = `link_${randomUUID().substring(0, 12)}`;
16
16
  const now = Date.now();
17
17
 
18
- const doc = db
18
+ const doc = await db
19
19
  .prepare("SELECT id, title, path FROM documents WHERE id = ? OR path = ? OR title = ?")
20
20
  .get(docId, docId, docId);
21
21
 
@@ -28,7 +28,7 @@ export function linkFactToDocument({
28
28
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
29
29
  `);
30
30
 
31
- stmt.run(
31
+ await stmt.run(
32
32
  id,
33
33
  factKey,
34
34
  factText,
@@ -46,7 +46,7 @@ export function linkFactToDocument({
46
46
  VALUES (?, ?, ?, ?, ?)
47
47
  `);
48
48
  const targetSpec = startLine ? `${doc.id}:L${startLine}-${endLine || startLine}` : doc.id;
49
- edgeStmt.run(`fact:${factKey}:${factText.substring(0, 30)}`, targetSpec, relationType, JSON.stringify({ linkId: id }), now);
49
+ await edgeStmt.run(`fact:${factKey}:${factText.substring(0, 30)}`, targetSpec, relationType, JSON.stringify({ linkId: id }), now);
50
50
 
51
51
  return {
52
52
  linkId: id,
@@ -60,8 +60,8 @@ export function linkFactToDocument({
60
60
  };
61
61
  }
62
62
 
63
- export function getLinksForFact(factKey, factText) {
64
- const db = getDatabase();
63
+ export async function getLinksForFact(factKey, factText) {
64
+ const db = await getDatabase();
65
65
  const stmt = db.prepare(`
66
66
  SELECT k.*, d.title as doc_title, d.path as doc_path
67
67
  FROM knowledge_links k
@@ -70,11 +70,11 @@ export function getLinksForFact(factKey, factText) {
70
70
  ORDER BY k.created_at DESC
71
71
  `);
72
72
  const queryPattern = `%${factText.substring(0, 20)}%`;
73
- return stmt.all(factKey, queryPattern, factText);
73
+ return await stmt.all(factKey, queryPattern, factText);
74
74
  }
75
75
 
76
- export function getLinksForDoc(docId) {
77
- const db = getDatabase();
76
+ export async function getLinksForDoc(docId) {
77
+ const db = await getDatabase();
78
78
  const stmt = db.prepare(`
79
79
  SELECT k.*, d.title as doc_title, d.path as doc_path
80
80
  FROM knowledge_links k
@@ -82,11 +82,11 @@ export function getLinksForDoc(docId) {
82
82
  WHERE k.doc_id = ? OR d.path = ? OR d.title = ?
83
83
  ORDER BY k.created_at DESC
84
84
  `);
85
- return stmt.all(docId, docId, docId);
85
+ return await stmt.all(docId, docId, docId);
86
86
  }
87
87
 
88
- export function listAllLinks(factKey = null) {
89
- const db = getDatabase();
88
+ export async function listAllLinks(factKey = null) {
89
+ const db = await getDatabase();
90
90
  let sql = `
91
91
  SELECT k.*, d.title as doc_title, d.path as doc_path
92
92
  FROM knowledge_links k
@@ -98,5 +98,5 @@ export function listAllLinks(factKey = null) {
98
98
  params.push(factKey);
99
99
  }
100
100
  sql += " ORDER BY k.created_at DESC";
101
- return db.prepare(sql).all(...params);
101
+ return await db.prepare(sql).all(...params);
102
102
  }
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env node
1
+ #!/usr/bin/env node
2
2
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
3
3
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
4
4
  import * as z from "zod/v4";
@@ -39,7 +39,7 @@ if (cliArgs.includes("setup") || cliArgs.includes("install") || cliArgs.includes
39
39
  process.exit(0);
40
40
  }
41
41
 
42
- if (cliArgs.includes("cli") || cliArgs.includes("config") || cliArgs.includes("--cli") || cliArgs.includes("-c")) {
42
+ if (cliArgs.includes("cli") || cliArgs.includes("config") || cliArgs.includes("--cli") || cliArgs.includes("-c") || cliArgs.includes("login") || cliArgs.includes("logout")) {
43
43
  const { runCli } = await import("./cli.js");
44
44
  await runCli();
45
45
  process.exit(0);
@@ -315,8 +315,8 @@ server.registerTool(
315
315
  let linksUpdated = 0;
316
316
  try {
317
317
  const { getDatabase } = await import("./db/database.js");
318
- const db = getDatabase();
319
- const res = db
318
+ const db = await getDatabase();
319
+ const res = await db
320
320
  .prepare(
321
321
  "UPDATE knowledge_links SET fact_text = ? WHERE fact_key = ? AND fact_text = ?"
322
322
  )
@@ -353,12 +353,17 @@ server.registerTool(
353
353
  let rag = {};
354
354
  try {
355
355
  const { getDatabase } = await import("./db/database.js");
356
- const db = getDatabase();
357
- rag.documents = db.prepare("SELECT COUNT(*) AS c FROM documents").get().c;
358
- rag.sections = db.prepare("SELECT COUNT(*) AS c FROM sections").get().c;
359
- rag.chunks = db.prepare("SELECT COUNT(*) AS c FROM micro_chunks").get().c;
360
- rag.edges = db.prepare("SELECT COUNT(*) AS c FROM graph_edges").get().c;
361
- rag.links = db.prepare("SELECT COUNT(*) AS c FROM knowledge_links").get().c;
356
+ const db = await getDatabase();
357
+ const docCountRow = await db.prepare("SELECT COUNT(*) AS c FROM documents").get();
358
+ rag.documents = docCountRow ? docCountRow.c : 0;
359
+ const secCountRow = await db.prepare("SELECT COUNT(*) AS c FROM sections").get();
360
+ rag.sections = secCountRow ? secCountRow.c : 0;
361
+ const chunkCountRow = await db.prepare("SELECT COUNT(*) AS c FROM micro_chunks").get();
362
+ rag.chunks = chunkCountRow ? chunkCountRow.c : 0;
363
+ const edgeCountRow = await db.prepare("SELECT COUNT(*) AS c FROM graph_edges").get();
364
+ rag.edges = edgeCountRow ? edgeCountRow.c : 0;
365
+ const linkCountRow = await db.prepare("SELECT COUNT(*) AS c FROM knowledge_links").get();
366
+ rag.links = linkCountRow ? linkCountRow.c : 0;
362
367
  } catch (e) {
363
368
  rag.error = e.message;
364
369
  }
@@ -561,13 +566,17 @@ server.registerTool(
561
566
  },
562
567
  async ({ action, docId, snapshotPath }) => {
563
568
  const { getDatabase } = await import("./db/database.js");
564
- const db = getDatabase();
569
+ const db = await getDatabase();
565
570
 
566
571
  if (action === "stats") {
567
- const docCount = db.prepare("SELECT COUNT(*) as cnt FROM documents").get().cnt;
568
- const secCount = db.prepare("SELECT COUNT(*) as cnt FROM sections").get().cnt;
569
- const chunkCount = db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks").get().cnt;
570
- const edgeCount = db.prepare("SELECT COUNT(*) as cnt FROM graph_edges").get().cnt;
572
+ const docCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM documents").get();
573
+ const docCount = docCountRow ? docCountRow.cnt : 0;
574
+ const secCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM sections").get();
575
+ const secCount = secCountRow ? secCountRow.cnt : 0;
576
+ const chunkCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks").get();
577
+ const chunkCount = chunkCountRow ? chunkCountRow.cnt : 0;
578
+ const edgeCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM graph_edges").get();
579
+ const edgeCount = edgeCountRow ? edgeCountRow.cnt : 0;
571
580
  return {
572
581
  content: [
573
582
  {
@@ -588,7 +597,7 @@ server.registerTool(
588
597
  }
589
598
 
590
599
  if (action === "list") {
591
- const docs = db
600
+ const docs = await db
592
601
  .prepare("SELECT id, title, path, blob_hash, created_at FROM documents ORDER BY created_at DESC")
593
602
  .all();
594
603
  return {
@@ -598,7 +607,7 @@ server.registerTool(
598
607
 
599
608
  if (action === "read_document") {
600
609
  if (!docId) throw new Error("docId parameter is required for read_document action");
601
- const doc = db
610
+ const doc = await db
602
611
  .prepare("SELECT id, title, path, blob_hash, created_at FROM documents WHERE id = ? OR path = ? OR title = ?")
603
612
  .get(docId, docId, docId);
604
613
  if (!doc) {
@@ -12,9 +12,9 @@ export function ensureExportsDir() {
12
12
  return EXPORTS_DIR;
13
13
  }
14
14
 
15
- export function exportDocumentData(docIdOrPath, customDb = null) {
16
- const db = customDb || getDatabase();
17
- const doc = db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
15
+ export async function exportDocumentData(docIdOrPath, customDb = null) {
16
+ const db = customDb || await getDatabase();
17
+ const doc = await db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
18
18
  if (!doc) {
19
19
  throw new Error(`Document not found for ID or path: ${docIdOrPath}`);
20
20
  }
@@ -33,10 +33,10 @@ export function exportDocumentData(docIdOrPath, customDb = null) {
33
33
  metadata = doc.metadata_json;
34
34
  }
35
35
 
36
- const sections = db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
37
- const mediumChunks = db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
38
- const microChunks = db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
39
- const graphEdges = db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
36
+ const sections = await db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
37
+ const mediumChunks = await db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
38
+ const microChunks = await db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
39
+ const graphEdges = await db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
40
40
 
41
41
  return {
42
42
  document: {
@@ -63,15 +63,15 @@ export function exportDocumentData(docIdOrPath, customDb = null) {
63
63
  };
64
64
  }
65
65
 
66
- export function exportDocumentToJsonString(docIdOrPath, customDb = null) {
67
- const data = exportDocumentData(docIdOrPath, customDb);
66
+ export async function exportDocumentToJsonString(docIdOrPath, customDb = null) {
67
+ const data = await exportDocumentData(docIdOrPath, customDb);
68
68
  return JSON.stringify(data, null, 2);
69
69
  }
70
70
 
71
- export function exportDocumentToFile(docIdOrPath, outputPath = null, customDb = null) {
71
+ export async function exportDocumentToFile(docIdOrPath, outputPath = null, customDb = null) {
72
72
  const targetDir = ensureExportsDir();
73
- const db = customDb || getDatabase();
74
- const data = exportDocumentData(docIdOrPath, db);
73
+ const db = customDb || await getDatabase();
74
+ const data = await exportDocumentData(docIdOrPath, db);
75
75
  const jsonStr = JSON.stringify(data, null, 2);
76
76
 
77
77
  const finalPath = outputPath || join(targetDir, `doc_export_${data.document.id}.json`);
@@ -1,4 +1,7 @@
1
1
  import { basename, extname } from "node:path";
2
+ import { PDFParse } from "pdf-parse";
3
+ import mammoth from "mammoth";
4
+ import xlsx from "xlsx";
2
5
 
3
6
  export function cleanHtml(html) {
4
7
  if (!html) return "";
@@ -98,7 +101,60 @@ export function stripMarkdownBadgesAndNoise(text) {
98
101
  return cleaned;
99
102
  }
100
103
 
101
- export function normalizeContent({ content, type = "text", path = null, title = null }) {
104
+ export function parseSpreadsheet(content, fileName, isCsv = false) {
105
+ const options = isCsv && (typeof content === "string") ? { type: "string" } : { type: "buffer" };
106
+ const workbook = xlsx.read(content, options);
107
+ let markdownParts = [];
108
+
109
+ for (const sheetName of workbook.SheetNames) {
110
+ const sheet = workbook.Sheets[sheetName];
111
+ // Convert to JSON 2D array
112
+ const rows = xlsx.utils.sheet_to_json(sheet, { header: 1 });
113
+ if (rows.length === 0) continue;
114
+
115
+ markdownParts.push(`## Sheet: ${sheetName}\n`);
116
+
117
+ // Create Markdown Table representation
118
+ const normalizedRows = rows.map(r => (Array.isArray(r) ? r : []).map(cell => (cell === undefined || cell === null) ? "" : String(cell)));
119
+ const maxCols = Math.max(...normalizedRows.map(r => r.length), 0);
120
+ if (maxCols === 0) continue;
121
+
122
+ // Pad all rows to maxCols
123
+ for (const r of normalizedRows) {
124
+ while (r.length < maxCols) r.push("");
125
+ }
126
+
127
+ // Header
128
+ const headers = normalizedRows[0];
129
+ markdownParts.push(`| ${headers.join(" | ")} |`);
130
+ markdownParts.push(`| ${headers.map(() => "---").join(" | ")} |`);
131
+
132
+ // Data rows
133
+ for (let i = 1; i < normalizedRows.length; i++) {
134
+ markdownParts.push(`| ${normalizedRows[i].join(" | ")} |`);
135
+ }
136
+
137
+ markdownParts.push("\n### Searchable Records\n");
138
+ // Row-by-row key-value representation for chunking/semantic search
139
+ for (let i = 1; i < normalizedRows.length; i++) {
140
+ const row = normalizedRows[i];
141
+ // Skip completely empty rows
142
+ if (row.every(cell => cell.trim() === "")) continue;
143
+
144
+ markdownParts.push(`Record ${i} from sheet ${sheetName}:`);
145
+ for (let j = 0; j < maxCols; j++) {
146
+ const headerName = headers[j]?.trim() || `Column_${j + 1}`;
147
+ const val = row[j]?.trim() || "";
148
+ markdownParts.push(`- ${headerName}: ${val}`);
149
+ }
150
+ markdownParts.push("");
151
+ }
152
+ }
153
+
154
+ return markdownParts.join("\n");
155
+ }
156
+
157
+ export async function normalizeContent({ content, type = "text", path = null, title = null }) {
102
158
  let markdown = "";
103
159
  let docTitle = title;
104
160
  const fileName = path ? basename(path) : "document";
@@ -112,6 +168,18 @@ export function normalizeContent({ content, type = "text", path = null, title =
112
168
  ".js": "javascript",
113
169
  ".ts": "typescript",
114
170
  ".py": "python",
171
+ ".go": "go",
172
+ ".rs": "rust",
173
+ ".cpp": "cpp",
174
+ ".h": "cpp",
175
+ ".hpp": "cpp",
176
+ ".cc": "cpp",
177
+ ".cxx": "cpp",
178
+ ".java": "java",
179
+ ".kt": "kotlin",
180
+ ".cs": "csharp",
181
+ ".php": "php",
182
+ ".rb": "ruby",
115
183
  ".json": "json",
116
184
  ".yaml": "yaml",
117
185
  ".yml": "yaml",
@@ -119,15 +187,44 @@ export function normalizeContent({ content, type = "text", path = null, title =
119
187
  ".html": "html",
120
188
  };
121
189
 
122
- if (codeLangs[ext]) {
123
- markdown = `# ${fileName}\n\n\`\`\`${codeLangs[ext]}\n${content.trim()}\n\`\`\``;
190
+ if (ext === ".pdf") {
191
+ try {
192
+ const pdfBuffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
193
+ const parser = new PDFParse({ data: pdfBuffer });
194
+ const result = await parser.getText();
195
+ markdown = result.text || "";
196
+ docTitle = title || fileName;
197
+ } catch (err) {
198
+ throw new Error(`Failed to parse PDF file '${fileName}': ${err.message}`);
199
+ }
200
+ } else if (ext === ".docx") {
201
+ try {
202
+ const docxBuffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
203
+ const result = await mammoth.convertToMarkdown({ buffer: docxBuffer });
204
+ markdown = result.value || "";
205
+ docTitle = title || fileName;
206
+ } catch (err) {
207
+ throw new Error(`Failed to parse DOCX file '${fileName}': ${err.message}`);
208
+ }
209
+ } else if (ext === ".xlsx" || ext === ".xls" || ext === ".csv") {
210
+ try {
211
+ markdown = parseSpreadsheet(content, fileName, ext === ".csv");
212
+ docTitle = title || fileName;
213
+ } catch (err) {
214
+ throw new Error(`Failed to parse spreadsheet file '${fileName}': ${err.message}`);
215
+ }
216
+ } else if (codeLangs[ext]) {
217
+ const textContent = Buffer.isBuffer(content) ? content.toString("utf8") : String(content);
218
+ markdown = `# ${fileName}\n\n\`\`\`${codeLangs[ext]}\n${textContent.trim()}\n\`\`\``;
124
219
  docTitle = title || fileName;
125
220
  } else {
126
- markdown = content.trim();
221
+ const textContent = Buffer.isBuffer(content) ? content.toString("utf8") : String(content);
222
+ markdown = textContent.trim();
127
223
  docTitle = title || extractTitle(markdown, fileName);
128
224
  }
129
225
  } else {
130
- markdown = typeof content === "string" ? content.trim() : String(content);
226
+ const textContent = Buffer.isBuffer(content) ? content.toString("utf8") : String(content);
227
+ markdown = textContent.trim();
131
228
  docTitle = title || extractTitle(markdown, "Direct Note");
132
229
  }
133
230
 
@@ -1,5 +1,6 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { readFile } from "node:fs/promises";
3
+ import { extname } from "node:path";
3
4
  import { getDatabase, BLOBS_DIR } from "../db/database.js";
4
5
  import { saveBlob, deleteBlob } from "../storage/blob_store.js";
5
6
  import { normalizeContent, fetchUrlContent } from "./normalizer.js";
@@ -17,7 +18,7 @@ export async function ingestDocument({
17
18
  customBlobDir = BLOBS_DIR,
18
19
  generateEmbeddings = true,
19
20
  }) {
20
- const db = customDb || getDatabase();
21
+ const db = customDb || await getDatabase();
21
22
 
22
23
  let effectiveType = type;
23
24
  let effectivePath = path;
@@ -33,12 +34,14 @@ export async function ingestDocument({
33
34
  const filePath = effectivePath || content;
34
35
  const needsRead = !content || content === filePath;
35
36
  if (needsRead && filePath) {
36
- content = await readFile(filePath, "utf-8");
37
+ const ext = extname(filePath).toLowerCase();
38
+ const isBinary = [".pdf", ".docx", ".xlsx", ".xls"].includes(ext);
39
+ content = await readFile(filePath, isBinary ? null : "utf-8");
37
40
  effectivePath = filePath;
38
41
  }
39
42
  }
40
43
 
41
- const { markdown, title: docTitle, metadata } = normalizeContent({ content, type: effectiveType, path: effectivePath, title: effectiveTitle });
44
+ const { markdown, title: docTitle, metadata } = await normalizeContent({ content, type: effectiveType, path: effectivePath, title: effectiveTitle });
42
45
  if (type === "url") metadata.source_type = "url";
43
46
 
44
47
  const blobRes = await saveBlob(markdown, customBlobDir);
@@ -78,21 +81,21 @@ export async function ingestDocument({
78
81
  }
79
82
  }
80
83
 
81
- db.exec("BEGIN IMMEDIATE;");
84
+ await db.exec("BEGIN IMMEDIATE;");
82
85
  try {
83
- const existingDoc = db.prepare("SELECT id FROM documents WHERE path = ?").get(docPath);
86
+ const existingDoc = await db.prepare("SELECT id FROM documents WHERE path = ?").get(docPath);
84
87
  if (existingDoc) {
85
- const microChunks = db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(existingDoc.id);
88
+ const microChunks = await db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(existingDoc.id);
86
89
  for (const mc of microChunks) {
87
90
  try {
88
- db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
91
+ await db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
89
92
  } catch {}
90
93
  }
91
- db.prepare("DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?").run(existingDoc.id, existingDoc.id);
92
- db.prepare("DELETE FROM documents WHERE id = ?").run(existingDoc.id);
94
+ await db.prepare("DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?").run(existingDoc.id, existingDoc.id);
95
+ await db.prepare("DELETE FROM documents WHERE id = ?").run(existingDoc.id);
93
96
  }
94
97
 
95
- db.prepare(`
98
+ await db.prepare(`
96
99
  INSERT INTO documents (id, path, blob_hash, title, checksum, toc_json, metadata_json, created_at, updated_at)
97
100
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
98
101
  `).run(
@@ -112,7 +115,7 @@ export async function ingestDocument({
112
115
  VALUES (?, ?, ?, ?, ?, ?);
113
116
  `);
114
117
  for (const sec of hierarchy.sections) {
115
- insertSectionStmt.run(sec.id, sec.doc_id, sec.heading, sec.breadcrumbs, sec.content, sec.token_count);
118
+ await insertSectionStmt.run(sec.id, sec.doc_id, sec.heading, sec.breadcrumbs, sec.content, sec.token_count);
116
119
  }
117
120
 
118
121
  if (hierarchy.mediumChunks && hierarchy.mediumChunks.length > 0) {
@@ -121,7 +124,7 @@ export async function ingestDocument({
121
124
  VALUES (?, ?, ?, ?, ?, ?, ?);
122
125
  `);
123
126
  for (const med of hierarchy.mediumChunks) {
124
- insertMediumStmt.run(med.id, med.section_id, med.doc_id, med.content, med.block_type, med.token_count, now);
127
+ await insertMediumStmt.run(med.id, med.section_id, med.doc_id, med.content, med.block_type, med.token_count, now);
125
128
  }
126
129
  }
127
130
 
@@ -135,19 +138,30 @@ export async function ingestDocument({
135
138
  `);
136
139
 
137
140
  for (const micro of hierarchy.microChunks) {
138
- insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null);
139
- insertFtsStmt.run(micro.id, micro.content, micro.breadcrumbs);
141
+ await insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null);
142
+ await insertFtsStmt.run(micro.id, micro.content, micro.breadcrumbs);
140
143
  }
141
144
 
142
145
  const edges = buildGraphEdges(docId, hierarchy);
143
- saveGraphEdges(db, edges);
146
+ await saveGraphEdges(db, edges);
144
147
 
145
- db.exec("COMMIT;");
148
+ await db.exec("COMMIT;");
146
149
  } catch (err) {
147
- db.exec("ROLLBACK;");
150
+ await db.exec("ROLLBACK;");
148
151
  throw new Error(`Ingestion transaction failed: ${err.message}`);
149
152
  }
150
153
 
154
+ if (getConfig().mode === "hybrid-sync") {
155
+ try {
156
+ const { exportDocumentData } = await import("./exporter.js");
157
+ const { enqueueSyncTask } = await import("../db/sync_queue.js");
158
+ const exportedData = await exportDocumentData(docId, db);
159
+ await enqueueSyncTask("ingest_document", docId, exportedData);
160
+ } catch (err) {
161
+ console.error("Failed to queue document ingest sync task:", err.message);
162
+ }
163
+ }
164
+
151
165
  return {
152
166
  docId,
153
167
  doc_id: docId,
@@ -164,54 +178,64 @@ export async function ingestDocument({
164
178
  }
165
179
 
166
180
  export async function deleteDocument(docIdOrPath, customDb = null, customBlobDir = BLOBS_DIR) {
167
- const db = customDb || getDatabase();
168
- const doc = db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
181
+ const db = customDb || await getDatabase();
182
+ const doc = await db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
169
183
  if (!doc) {
170
184
  return { deleted: false, reason: "Document not found" };
171
185
  }
172
186
 
173
- const microChunks = db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(doc.id);
187
+ const microChunks = await db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(doc.id);
174
188
 
175
189
  // Collect every id owned by this document so we can purge dangling graph edges
176
190
  // (graph_edges has no FK constraints, so section/chunk/doc references would otherwise leak).
177
191
  const ownedIds = [doc.id];
178
192
  for (const table of ["sections", "medium_chunks", "micro_chunks"]) {
179
- const rows = db.prepare(`SELECT id FROM ${table} WHERE doc_id = ?`).all(doc.id);
193
+ const rows = await db.prepare(`SELECT id FROM ${table} WHERE doc_id = ?`).all(doc.id);
180
194
  for (const r of rows) ownedIds.push(r.id);
181
195
  }
182
196
 
183
- db.exec("BEGIN IMMEDIATE;");
197
+ await db.exec("BEGIN IMMEDIATE;");
184
198
  try {
185
199
  for (const mc of microChunks) {
186
200
  try {
187
- db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
201
+ await db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
188
202
  } catch {}
189
203
  }
190
204
 
191
205
  // Auto-clean Agent knowledge graph links pointing at this document.
192
- db.prepare("DELETE FROM knowledge_links WHERE doc_id = ?").run(doc.id);
206
+ await db.prepare("DELETE FROM knowledge_links WHERE doc_id = ?").run(doc.id);
193
207
 
194
208
  for (const id of ownedIds) {
195
209
  // GLOB: '*' suffix is exact (unlike LIKE, '_' stays literal in ids like doc_xxx).
196
- db.prepare(
210
+ await db.prepare(
197
211
  "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ? OR source_id GLOB ? OR target_id GLOB ?"
198
212
  ).run(id, id, `${id}*`, `${id}*`);
199
213
  }
200
214
 
201
- db.prepare("DELETE FROM documents WHERE id = ?").run(doc.id);
215
+ await db.prepare("DELETE FROM documents WHERE id = ?").run(doc.id);
202
216
 
203
- db.exec("COMMIT;");
217
+ await db.exec("COMMIT;");
204
218
  } catch (err) {
205
- db.exec("ROLLBACK;");
219
+ await db.exec("ROLLBACK;");
206
220
  throw err;
207
221
  }
208
222
 
209
223
  if (doc.blob_hash) {
210
- const refCount = db.prepare("SELECT COUNT(*) as cnt FROM documents WHERE blob_hash = ?").get(doc.blob_hash).cnt;
224
+ const refCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM documents WHERE blob_hash = ?").get(doc.blob_hash);
225
+ const refCount = refCountRow ? refCountRow.cnt : 0;
211
226
  if (refCount === 0) {
212
227
  await deleteBlob(doc.blob_hash, customBlobDir);
213
228
  }
214
229
  }
215
230
 
231
+ if (getConfig().mode === "hybrid-sync") {
232
+ try {
233
+ const { enqueueSyncTask } = await import("../db/sync_queue.js");
234
+ await enqueueSyncTask("delete_document", docIdOrPath);
235
+ } catch (err) {
236
+ console.error("Failed to queue document delete sync task:", err.message);
237
+ }
238
+ }
239
+
216
240
  return { deleted: true, docId: doc.id, title: doc.title, linksCleaned: true };
217
241
  }
@@ -125,6 +125,22 @@ async function maybeMigrateLegacy(key) {
125
125
  }
126
126
 
127
127
  export async function readMemory(key) {
128
+ const { getConfig } = await import("./config/config_manager.js");
129
+ const config = getConfig();
130
+ if (config.mode === "only-cloud") {
131
+ try {
132
+ const { getDatabase } = await import("./db/database.js");
133
+ const db = await getDatabase();
134
+ const row = await db.prepare("SELECT content FROM notebooks WHERE key = ?;").get(key);
135
+ if (row && row.content) {
136
+ return row.content.split("\n").filter((l) => l.startsWith("- ["));
137
+ }
138
+ } catch (err) {
139
+ console.error("Failed to read memory from cloud database:", err.message);
140
+ }
141
+ return [];
142
+ }
143
+
128
144
  const fp = memoryPath(key);
129
145
  if (existsSync(fp)) {
130
146
  const content = await readFile(fp, "utf-8");
@@ -149,10 +165,67 @@ export async function writeMemory(key, entries) {
149
165
  }
150
166
  }
151
167
  const content = lines.join("\n") + "\n" + (entries.length ? entries.join("\n") + "\n" : "");
168
+
169
+ const { getConfig } = await import("./config/config_manager.js");
170
+ const config = getConfig();
171
+ if (config.mode === "only-cloud") {
172
+ try {
173
+ const { getDatabase } = await import("./db/database.js");
174
+ const db = await getDatabase();
175
+ await db.prepare(`
176
+ INSERT INTO notebooks (key, content, updated_at)
177
+ VALUES (?, ?, ?)
178
+ ON CONFLICT(key) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at;
179
+ `).run(key, content, Date.now());
180
+ } catch (err) {
181
+ console.error("Failed to write memory to cloud database:", err.message);
182
+ }
183
+ return;
184
+ }
185
+
152
186
  await writeFile(memoryPath(key), content);
187
+
188
+ if (config.mode === "hybrid-sync") {
189
+ try {
190
+ const { enqueueSyncTask } = await import("./db/sync_queue.js");
191
+ await enqueueSyncTask("write_memory", key, content);
192
+ } catch (err) {
193
+ console.error("Failed to queue memory sync task:", err.message);
194
+ }
195
+ }
153
196
  }
154
197
 
155
198
  export async function listProjectStores() {
199
+ const { getConfig } = await import("./config/config_manager.js");
200
+ const config = getConfig();
201
+ if (config.mode === "only-cloud") {
202
+ try {
203
+ const { getDatabase } = await import("./db/database.js");
204
+ const db = await getDatabase();
205
+ const rows = await db.prepare("SELECT key, content FROM notebooks WHERE key != ?;").all(GLOBAL_KEY);
206
+ const stores = [];
207
+ for (const row of rows) {
208
+ const key = row.key;
209
+ const content = row.content;
210
+ const facts = content.split("\n").filter((l) => l.startsWith("- ["));
211
+ const meta = parseMeta(content);
212
+ stores.push({
213
+ key,
214
+ path: meta.path || (key.includes("/") || key.includes(":") ? key : null),
215
+ basename: basename(meta.path || key) || key,
216
+ file: `${slugify(key)}.md`,
217
+ count: facts.length,
218
+ legacy: !meta.path,
219
+ });
220
+ }
221
+ stores.sort((a, b) => a.basename.localeCompare(b.basename));
222
+ return stores;
223
+ } catch (err) {
224
+ console.error("Failed to list memory stores from cloud database:", err.message);
225
+ }
226
+ return [];
227
+ }
228
+
156
229
  const stores = [];
157
230
  const files = await readdir(MEMORY_DIR).catch(() => []);
158
231
  for (const f of files) {