@lotargo/memory_plugin 1.3.2 → 1.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +334 -324
- package/mcp-server/admin/auth.js +385 -0
- package/mcp-server/admin/snapshot.js +34 -32
- package/mcp-server/cli.js +108 -12
- package/mcp-server/config/auth_store.js +137 -0
- package/mcp-server/config/config_manager.js +5 -0
- package/mcp-server/db/database.js +198 -14
- package/mcp-server/db/migrations.js +62 -33
- package/mcp-server/db/sync_queue.js +211 -0
- package/mcp-server/graph/graph_extractor.js +40 -17
- package/mcp-server/graph/knowledge_linker.js +14 -14
- package/mcp-server/index.js +34 -23
- package/mcp-server/ingest/exporter.js +12 -12
- package/mcp-server/ingest/normalizer.js +102 -5
- package/mcp-server/ingest/pipeline.js +53 -29
- package/mcp-server/memory.js +73 -0
- package/mcp-server/retrieval/retriever.js +24 -15
- package/package.json +6 -2
package/mcp-server/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
3
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
4
4
|
import * as z from "zod/v4";
|
|
@@ -39,7 +39,7 @@ if (cliArgs.includes("setup") || cliArgs.includes("install") || cliArgs.includes
|
|
|
39
39
|
process.exit(0);
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
if (cliArgs.includes("cli") || cliArgs.includes("config") || cliArgs.includes("--cli") || cliArgs.includes("-c")) {
|
|
42
|
+
if (cliArgs.includes("cli") || cliArgs.includes("config") || cliArgs.includes("--cli") || cliArgs.includes("-c") || cliArgs.includes("login") || cliArgs.includes("logout")) {
|
|
43
43
|
const { runCli } = await import("./cli.js");
|
|
44
44
|
await runCli();
|
|
45
45
|
process.exit(0);
|
|
@@ -175,10 +175,10 @@ server.registerTool(
|
|
|
175
175
|
const { getLinksForFact } = await import("./graph/knowledge_linker.js");
|
|
176
176
|
const results = [];
|
|
177
177
|
|
|
178
|
-
const formatFactWithLinks = (factLine, key) => {
|
|
178
|
+
const formatFactWithLinks = async (factLine, key) => {
|
|
179
179
|
let line = displayFact(factLine);
|
|
180
180
|
try {
|
|
181
|
-
const links = getLinksForFact(key, factText(factLine));
|
|
181
|
+
const links = await getLinksForFact(key, factText(factLine));
|
|
182
182
|
if (links && links.length > 0) {
|
|
183
183
|
const docStr = links
|
|
184
184
|
.map((l) => {
|
|
@@ -192,14 +192,16 @@ server.registerTool(
|
|
|
192
192
|
return line;
|
|
193
193
|
};
|
|
194
194
|
|
|
195
|
-
const collect = (entries, key) => {
|
|
195
|
+
const collect = async (entries, key) => {
|
|
196
196
|
const matched = entries.filter(
|
|
197
197
|
(e) => matchesQuery(e, query) && matchesTags(e, tags) && inDateRange(e, since, until)
|
|
198
198
|
);
|
|
199
199
|
if (!matched.length) return;
|
|
200
200
|
if (results.length) results.push("");
|
|
201
201
|
results.push(`--- ${key === GLOBAL_KEY ? "Global" : `Project: ${key === target ? label : key}`} ---`);
|
|
202
|
-
|
|
202
|
+
for (let i = 0; i < matched.length; i++) {
|
|
203
|
+
results.push(`${i + 1}. ${await formatFactWithLinks(matched[i], key)}`);
|
|
204
|
+
}
|
|
203
205
|
results.push(`Store file: ${storeFilePath(key)}`);
|
|
204
206
|
};
|
|
205
207
|
|
|
@@ -225,11 +227,11 @@ server.registerTool(
|
|
|
225
227
|
const label = project ? target : projectName();
|
|
226
228
|
if (scope !== "project") {
|
|
227
229
|
const global = await readMemory(GLOBAL_KEY);
|
|
228
|
-
collect(global, GLOBAL_KEY);
|
|
230
|
+
await collect(global, GLOBAL_KEY);
|
|
229
231
|
}
|
|
230
232
|
if (scope !== "global") {
|
|
231
233
|
const local = await readMemory(target);
|
|
232
|
-
collect(local, target);
|
|
234
|
+
await collect(local, target);
|
|
233
235
|
}
|
|
234
236
|
const filtered = Boolean(query || tags || since || until);
|
|
235
237
|
const text = results.length
|
|
@@ -315,8 +317,8 @@ server.registerTool(
|
|
|
315
317
|
let linksUpdated = 0;
|
|
316
318
|
try {
|
|
317
319
|
const { getDatabase } = await import("./db/database.js");
|
|
318
|
-
const db = getDatabase();
|
|
319
|
-
const res = db
|
|
320
|
+
const db = await getDatabase();
|
|
321
|
+
const res = await db
|
|
320
322
|
.prepare(
|
|
321
323
|
"UPDATE knowledge_links SET fact_text = ? WHERE fact_key = ? AND fact_text = ?"
|
|
322
324
|
)
|
|
@@ -353,12 +355,17 @@ server.registerTool(
|
|
|
353
355
|
let rag = {};
|
|
354
356
|
try {
|
|
355
357
|
const { getDatabase } = await import("./db/database.js");
|
|
356
|
-
const db = getDatabase();
|
|
357
|
-
|
|
358
|
-
rag.
|
|
359
|
-
|
|
360
|
-
rag.
|
|
361
|
-
|
|
358
|
+
const db = await getDatabase();
|
|
359
|
+
const docCountRow = await db.prepare("SELECT COUNT(*) AS c FROM documents").get();
|
|
360
|
+
rag.documents = docCountRow ? docCountRow.c : 0;
|
|
361
|
+
const secCountRow = await db.prepare("SELECT COUNT(*) AS c FROM sections").get();
|
|
362
|
+
rag.sections = secCountRow ? secCountRow.c : 0;
|
|
363
|
+
const chunkCountRow = await db.prepare("SELECT COUNT(*) AS c FROM micro_chunks").get();
|
|
364
|
+
rag.chunks = chunkCountRow ? chunkCountRow.c : 0;
|
|
365
|
+
const edgeCountRow = await db.prepare("SELECT COUNT(*) AS c FROM graph_edges").get();
|
|
366
|
+
rag.edges = edgeCountRow ? edgeCountRow.c : 0;
|
|
367
|
+
const linkCountRow = await db.prepare("SELECT COUNT(*) AS c FROM knowledge_links").get();
|
|
368
|
+
rag.links = linkCountRow ? linkCountRow.c : 0;
|
|
362
369
|
} catch (e) {
|
|
363
370
|
rag.error = e.message;
|
|
364
371
|
}
|
|
@@ -561,13 +568,17 @@ server.registerTool(
|
|
|
561
568
|
},
|
|
562
569
|
async ({ action, docId, snapshotPath }) => {
|
|
563
570
|
const { getDatabase } = await import("./db/database.js");
|
|
564
|
-
const db = getDatabase();
|
|
571
|
+
const db = await getDatabase();
|
|
565
572
|
|
|
566
573
|
if (action === "stats") {
|
|
567
|
-
const
|
|
568
|
-
const
|
|
569
|
-
const
|
|
570
|
-
const
|
|
574
|
+
const docCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM documents").get();
|
|
575
|
+
const docCount = docCountRow ? docCountRow.cnt : 0;
|
|
576
|
+
const secCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM sections").get();
|
|
577
|
+
const secCount = secCountRow ? secCountRow.cnt : 0;
|
|
578
|
+
const chunkCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks").get();
|
|
579
|
+
const chunkCount = chunkCountRow ? chunkCountRow.cnt : 0;
|
|
580
|
+
const edgeCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM graph_edges").get();
|
|
581
|
+
const edgeCount = edgeCountRow ? edgeCountRow.cnt : 0;
|
|
571
582
|
return {
|
|
572
583
|
content: [
|
|
573
584
|
{
|
|
@@ -588,7 +599,7 @@ server.registerTool(
|
|
|
588
599
|
}
|
|
589
600
|
|
|
590
601
|
if (action === "list") {
|
|
591
|
-
const docs = db
|
|
602
|
+
const docs = await db
|
|
592
603
|
.prepare("SELECT id, title, path, blob_hash, created_at FROM documents ORDER BY created_at DESC")
|
|
593
604
|
.all();
|
|
594
605
|
return {
|
|
@@ -598,7 +609,7 @@ server.registerTool(
|
|
|
598
609
|
|
|
599
610
|
if (action === "read_document") {
|
|
600
611
|
if (!docId) throw new Error("docId parameter is required for read_document action");
|
|
601
|
-
const doc = db
|
|
612
|
+
const doc = await db
|
|
602
613
|
.prepare("SELECT id, title, path, blob_hash, created_at FROM documents WHERE id = ? OR path = ? OR title = ?")
|
|
603
614
|
.get(docId, docId, docId);
|
|
604
615
|
if (!doc) {
|
|
@@ -12,9 +12,9 @@ export function ensureExportsDir() {
|
|
|
12
12
|
return EXPORTS_DIR;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
export function exportDocumentData(docIdOrPath, customDb = null) {
|
|
16
|
-
const db = customDb || getDatabase();
|
|
17
|
-
const doc = db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
|
|
15
|
+
export async function exportDocumentData(docIdOrPath, customDb = null) {
|
|
16
|
+
const db = customDb || await getDatabase();
|
|
17
|
+
const doc = await db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
|
|
18
18
|
if (!doc) {
|
|
19
19
|
throw new Error(`Document not found for ID or path: ${docIdOrPath}`);
|
|
20
20
|
}
|
|
@@ -33,10 +33,10 @@ export function exportDocumentData(docIdOrPath, customDb = null) {
|
|
|
33
33
|
metadata = doc.metadata_json;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
const sections = db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
|
|
37
|
-
const mediumChunks = db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
|
|
38
|
-
const microChunks = db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
|
|
39
|
-
const graphEdges = db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
|
|
36
|
+
const sections = await db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
|
|
37
|
+
const mediumChunks = await db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
|
|
38
|
+
const microChunks = await db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
|
|
39
|
+
const graphEdges = await db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
|
|
40
40
|
|
|
41
41
|
return {
|
|
42
42
|
document: {
|
|
@@ -63,15 +63,15 @@ export function exportDocumentData(docIdOrPath, customDb = null) {
|
|
|
63
63
|
};
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
export function exportDocumentToJsonString(docIdOrPath, customDb = null) {
|
|
67
|
-
const data = exportDocumentData(docIdOrPath, customDb);
|
|
66
|
+
export async function exportDocumentToJsonString(docIdOrPath, customDb = null) {
|
|
67
|
+
const data = await exportDocumentData(docIdOrPath, customDb);
|
|
68
68
|
return JSON.stringify(data, null, 2);
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
export function exportDocumentToFile(docIdOrPath, outputPath = null, customDb = null) {
|
|
71
|
+
export async function exportDocumentToFile(docIdOrPath, outputPath = null, customDb = null) {
|
|
72
72
|
const targetDir = ensureExportsDir();
|
|
73
|
-
const db = customDb || getDatabase();
|
|
74
|
-
const data = exportDocumentData(docIdOrPath, db);
|
|
73
|
+
const db = customDb || await getDatabase();
|
|
74
|
+
const data = await exportDocumentData(docIdOrPath, db);
|
|
75
75
|
const jsonStr = JSON.stringify(data, null, 2);
|
|
76
76
|
|
|
77
77
|
const finalPath = outputPath || join(targetDir, `doc_export_${data.document.id}.json`);
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { basename, extname } from "node:path";
|
|
2
|
+
import { PDFParse } from "pdf-parse";
|
|
3
|
+
import mammoth from "mammoth";
|
|
4
|
+
import xlsx from "xlsx";
|
|
2
5
|
|
|
3
6
|
export function cleanHtml(html) {
|
|
4
7
|
if (!html) return "";
|
|
@@ -98,7 +101,60 @@ export function stripMarkdownBadgesAndNoise(text) {
|
|
|
98
101
|
return cleaned;
|
|
99
102
|
}
|
|
100
103
|
|
|
101
|
-
export function
|
|
104
|
+
export function parseSpreadsheet(content, fileName, isCsv = false) {
|
|
105
|
+
const options = isCsv && (typeof content === "string") ? { type: "string" } : { type: "buffer" };
|
|
106
|
+
const workbook = xlsx.read(content, options);
|
|
107
|
+
let markdownParts = [];
|
|
108
|
+
|
|
109
|
+
for (const sheetName of workbook.SheetNames) {
|
|
110
|
+
const sheet = workbook.Sheets[sheetName];
|
|
111
|
+
// Convert to JSON 2D array
|
|
112
|
+
const rows = xlsx.utils.sheet_to_json(sheet, { header: 1 });
|
|
113
|
+
if (rows.length === 0) continue;
|
|
114
|
+
|
|
115
|
+
markdownParts.push(`## Sheet: ${sheetName}\n`);
|
|
116
|
+
|
|
117
|
+
// Create Markdown Table representation
|
|
118
|
+
const normalizedRows = rows.map(r => (Array.isArray(r) ? r : []).map(cell => (cell === undefined || cell === null) ? "" : String(cell)));
|
|
119
|
+
const maxCols = Math.max(...normalizedRows.map(r => r.length), 0);
|
|
120
|
+
if (maxCols === 0) continue;
|
|
121
|
+
|
|
122
|
+
// Pad all rows to maxCols
|
|
123
|
+
for (const r of normalizedRows) {
|
|
124
|
+
while (r.length < maxCols) r.push("");
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Header
|
|
128
|
+
const headers = normalizedRows[0];
|
|
129
|
+
markdownParts.push(`| ${headers.join(" | ")} |`);
|
|
130
|
+
markdownParts.push(`| ${headers.map(() => "---").join(" | ")} |`);
|
|
131
|
+
|
|
132
|
+
// Data rows
|
|
133
|
+
for (let i = 1; i < normalizedRows.length; i++) {
|
|
134
|
+
markdownParts.push(`| ${normalizedRows[i].join(" | ")} |`);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
markdownParts.push("\n### Searchable Records\n");
|
|
138
|
+
// Row-by-row key-value representation for chunking/semantic search
|
|
139
|
+
for (let i = 1; i < normalizedRows.length; i++) {
|
|
140
|
+
const row = normalizedRows[i];
|
|
141
|
+
// Skip completely empty rows
|
|
142
|
+
if (row.every(cell => cell.trim() === "")) continue;
|
|
143
|
+
|
|
144
|
+
markdownParts.push(`Record ${i} from sheet ${sheetName}:`);
|
|
145
|
+
for (let j = 0; j < maxCols; j++) {
|
|
146
|
+
const headerName = headers[j]?.trim() || `Column_${j + 1}`;
|
|
147
|
+
const val = row[j]?.trim() || "";
|
|
148
|
+
markdownParts.push(`- ${headerName}: ${val}`);
|
|
149
|
+
}
|
|
150
|
+
markdownParts.push("");
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return markdownParts.join("\n");
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export async function normalizeContent({ content, type = "text", path = null, title = null }) {
|
|
102
158
|
let markdown = "";
|
|
103
159
|
let docTitle = title;
|
|
104
160
|
const fileName = path ? basename(path) : "document";
|
|
@@ -112,6 +168,18 @@ export function normalizeContent({ content, type = "text", path = null, title =
|
|
|
112
168
|
".js": "javascript",
|
|
113
169
|
".ts": "typescript",
|
|
114
170
|
".py": "python",
|
|
171
|
+
".go": "go",
|
|
172
|
+
".rs": "rust",
|
|
173
|
+
".cpp": "cpp",
|
|
174
|
+
".h": "cpp",
|
|
175
|
+
".hpp": "cpp",
|
|
176
|
+
".cc": "cpp",
|
|
177
|
+
".cxx": "cpp",
|
|
178
|
+
".java": "java",
|
|
179
|
+
".kt": "kotlin",
|
|
180
|
+
".cs": "csharp",
|
|
181
|
+
".php": "php",
|
|
182
|
+
".rb": "ruby",
|
|
115
183
|
".json": "json",
|
|
116
184
|
".yaml": "yaml",
|
|
117
185
|
".yml": "yaml",
|
|
@@ -119,15 +187,44 @@ export function normalizeContent({ content, type = "text", path = null, title =
|
|
|
119
187
|
".html": "html",
|
|
120
188
|
};
|
|
121
189
|
|
|
122
|
-
if (
|
|
123
|
-
|
|
190
|
+
if (ext === ".pdf") {
|
|
191
|
+
try {
|
|
192
|
+
const pdfBuffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
|
|
193
|
+
const parser = new PDFParse({ data: pdfBuffer });
|
|
194
|
+
const result = await parser.getText();
|
|
195
|
+
markdown = result.text || "";
|
|
196
|
+
docTitle = title || fileName;
|
|
197
|
+
} catch (err) {
|
|
198
|
+
throw new Error(`Failed to parse PDF file '${fileName}': ${err.message}`);
|
|
199
|
+
}
|
|
200
|
+
} else if (ext === ".docx") {
|
|
201
|
+
try {
|
|
202
|
+
const docxBuffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
|
|
203
|
+
const result = await mammoth.convertToMarkdown({ buffer: docxBuffer });
|
|
204
|
+
markdown = result.value || "";
|
|
205
|
+
docTitle = title || fileName;
|
|
206
|
+
} catch (err) {
|
|
207
|
+
throw new Error(`Failed to parse DOCX file '${fileName}': ${err.message}`);
|
|
208
|
+
}
|
|
209
|
+
} else if (ext === ".xlsx" || ext === ".xls" || ext === ".csv") {
|
|
210
|
+
try {
|
|
211
|
+
markdown = parseSpreadsheet(content, fileName, ext === ".csv");
|
|
212
|
+
docTitle = title || fileName;
|
|
213
|
+
} catch (err) {
|
|
214
|
+
throw new Error(`Failed to parse spreadsheet file '${fileName}': ${err.message}`);
|
|
215
|
+
}
|
|
216
|
+
} else if (codeLangs[ext]) {
|
|
217
|
+
const textContent = Buffer.isBuffer(content) ? content.toString("utf8") : String(content);
|
|
218
|
+
markdown = `# ${fileName}\n\n\`\`\`${codeLangs[ext]}\n${textContent.trim()}\n\`\`\``;
|
|
124
219
|
docTitle = title || fileName;
|
|
125
220
|
} else {
|
|
126
|
-
|
|
221
|
+
const textContent = Buffer.isBuffer(content) ? content.toString("utf8") : String(content);
|
|
222
|
+
markdown = textContent.trim();
|
|
127
223
|
docTitle = title || extractTitle(markdown, fileName);
|
|
128
224
|
}
|
|
129
225
|
} else {
|
|
130
|
-
|
|
226
|
+
const textContent = Buffer.isBuffer(content) ? content.toString("utf8") : String(content);
|
|
227
|
+
markdown = textContent.trim();
|
|
131
228
|
docTitle = title || extractTitle(markdown, "Direct Note");
|
|
132
229
|
}
|
|
133
230
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { extname } from "node:path";
|
|
3
4
|
import { getDatabase, BLOBS_DIR } from "../db/database.js";
|
|
4
5
|
import { saveBlob, deleteBlob } from "../storage/blob_store.js";
|
|
5
6
|
import { normalizeContent, fetchUrlContent } from "./normalizer.js";
|
|
@@ -17,7 +18,7 @@ export async function ingestDocument({
|
|
|
17
18
|
customBlobDir = BLOBS_DIR,
|
|
18
19
|
generateEmbeddings = true,
|
|
19
20
|
}) {
|
|
20
|
-
const db = customDb || getDatabase();
|
|
21
|
+
const db = customDb || await getDatabase();
|
|
21
22
|
|
|
22
23
|
let effectiveType = type;
|
|
23
24
|
let effectivePath = path;
|
|
@@ -33,12 +34,14 @@ export async function ingestDocument({
|
|
|
33
34
|
const filePath = effectivePath || content;
|
|
34
35
|
const needsRead = !content || content === filePath;
|
|
35
36
|
if (needsRead && filePath) {
|
|
36
|
-
|
|
37
|
+
const ext = extname(filePath).toLowerCase();
|
|
38
|
+
const isBinary = [".pdf", ".docx", ".xlsx", ".xls"].includes(ext);
|
|
39
|
+
content = await readFile(filePath, isBinary ? null : "utf-8");
|
|
37
40
|
effectivePath = filePath;
|
|
38
41
|
}
|
|
39
42
|
}
|
|
40
43
|
|
|
41
|
-
const { markdown, title: docTitle, metadata } = normalizeContent({ content, type: effectiveType, path: effectivePath, title: effectiveTitle });
|
|
44
|
+
const { markdown, title: docTitle, metadata } = await normalizeContent({ content, type: effectiveType, path: effectivePath, title: effectiveTitle });
|
|
42
45
|
if (type === "url") metadata.source_type = "url";
|
|
43
46
|
|
|
44
47
|
const blobRes = await saveBlob(markdown, customBlobDir);
|
|
@@ -78,21 +81,21 @@ export async function ingestDocument({
|
|
|
78
81
|
}
|
|
79
82
|
}
|
|
80
83
|
|
|
81
|
-
db.exec("BEGIN IMMEDIATE;");
|
|
84
|
+
await db.exec("BEGIN IMMEDIATE;");
|
|
82
85
|
try {
|
|
83
|
-
const existingDoc = db.prepare("SELECT id FROM documents WHERE path = ?").get(docPath);
|
|
86
|
+
const existingDoc = await db.prepare("SELECT id FROM documents WHERE path = ?").get(docPath);
|
|
84
87
|
if (existingDoc) {
|
|
85
|
-
const microChunks = db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(existingDoc.id);
|
|
88
|
+
const microChunks = await db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(existingDoc.id);
|
|
86
89
|
for (const mc of microChunks) {
|
|
87
90
|
try {
|
|
88
|
-
db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
|
|
91
|
+
await db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
|
|
89
92
|
} catch {}
|
|
90
93
|
}
|
|
91
|
-
db.prepare("DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?").run(existingDoc.id, existingDoc.id);
|
|
92
|
-
db.prepare("DELETE FROM documents WHERE id = ?").run(existingDoc.id);
|
|
94
|
+
await db.prepare("DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?").run(existingDoc.id, existingDoc.id);
|
|
95
|
+
await db.prepare("DELETE FROM documents WHERE id = ?").run(existingDoc.id);
|
|
93
96
|
}
|
|
94
97
|
|
|
95
|
-
db.prepare(`
|
|
98
|
+
await db.prepare(`
|
|
96
99
|
INSERT INTO documents (id, path, blob_hash, title, checksum, toc_json, metadata_json, created_at, updated_at)
|
|
97
100
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
|
|
98
101
|
`).run(
|
|
@@ -112,7 +115,7 @@ export async function ingestDocument({
|
|
|
112
115
|
VALUES (?, ?, ?, ?, ?, ?);
|
|
113
116
|
`);
|
|
114
117
|
for (const sec of hierarchy.sections) {
|
|
115
|
-
insertSectionStmt.run(sec.id, sec.doc_id, sec.heading, sec.breadcrumbs, sec.content, sec.token_count);
|
|
118
|
+
await insertSectionStmt.run(sec.id, sec.doc_id, sec.heading, sec.breadcrumbs, sec.content, sec.token_count);
|
|
116
119
|
}
|
|
117
120
|
|
|
118
121
|
if (hierarchy.mediumChunks && hierarchy.mediumChunks.length > 0) {
|
|
@@ -121,7 +124,7 @@ export async function ingestDocument({
|
|
|
121
124
|
VALUES (?, ?, ?, ?, ?, ?, ?);
|
|
122
125
|
`);
|
|
123
126
|
for (const med of hierarchy.mediumChunks) {
|
|
124
|
-
insertMediumStmt.run(med.id, med.section_id, med.doc_id, med.content, med.block_type, med.token_count, now);
|
|
127
|
+
await insertMediumStmt.run(med.id, med.section_id, med.doc_id, med.content, med.block_type, med.token_count, now);
|
|
125
128
|
}
|
|
126
129
|
}
|
|
127
130
|
|
|
@@ -135,19 +138,30 @@ export async function ingestDocument({
|
|
|
135
138
|
`);
|
|
136
139
|
|
|
137
140
|
for (const micro of hierarchy.microChunks) {
|
|
138
|
-
insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null);
|
|
139
|
-
insertFtsStmt.run(micro.id, micro.content, micro.breadcrumbs);
|
|
141
|
+
await insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null);
|
|
142
|
+
await insertFtsStmt.run(micro.id, micro.content, micro.breadcrumbs);
|
|
140
143
|
}
|
|
141
144
|
|
|
142
145
|
const edges = buildGraphEdges(docId, hierarchy);
|
|
143
|
-
saveGraphEdges(db, edges);
|
|
146
|
+
await saveGraphEdges(db, edges);
|
|
144
147
|
|
|
145
|
-
db.exec("COMMIT;");
|
|
148
|
+
await db.exec("COMMIT;");
|
|
146
149
|
} catch (err) {
|
|
147
|
-
db.exec("ROLLBACK;");
|
|
150
|
+
await db.exec("ROLLBACK;");
|
|
148
151
|
throw new Error(`Ingestion transaction failed: ${err.message}`);
|
|
149
152
|
}
|
|
150
153
|
|
|
154
|
+
if (getConfig().mode === "hybrid-sync") {
|
|
155
|
+
try {
|
|
156
|
+
const { exportDocumentData } = await import("./exporter.js");
|
|
157
|
+
const { enqueueSyncTask } = await import("../db/sync_queue.js");
|
|
158
|
+
const exportedData = await exportDocumentData(docId, db);
|
|
159
|
+
await enqueueSyncTask("ingest_document", docId, exportedData);
|
|
160
|
+
} catch (err) {
|
|
161
|
+
console.error("Failed to queue document ingest sync task:", err.message);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
151
165
|
return {
|
|
152
166
|
docId,
|
|
153
167
|
doc_id: docId,
|
|
@@ -164,54 +178,64 @@ export async function ingestDocument({
|
|
|
164
178
|
}
|
|
165
179
|
|
|
166
180
|
export async function deleteDocument(docIdOrPath, customDb = null, customBlobDir = BLOBS_DIR) {
|
|
167
|
-
const db = customDb || getDatabase();
|
|
168
|
-
const doc = db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
|
|
181
|
+
const db = customDb || await getDatabase();
|
|
182
|
+
const doc = await db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
|
|
169
183
|
if (!doc) {
|
|
170
184
|
return { deleted: false, reason: "Document not found" };
|
|
171
185
|
}
|
|
172
186
|
|
|
173
|
-
const microChunks = db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(doc.id);
|
|
187
|
+
const microChunks = await db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(doc.id);
|
|
174
188
|
|
|
175
189
|
// Collect every id owned by this document so we can purge dangling graph edges
|
|
176
190
|
// (graph_edges has no FK constraints, so section/chunk/doc references would otherwise leak).
|
|
177
191
|
const ownedIds = [doc.id];
|
|
178
192
|
for (const table of ["sections", "medium_chunks", "micro_chunks"]) {
|
|
179
|
-
const rows = db.prepare(`SELECT id FROM ${table} WHERE doc_id = ?`).all(doc.id);
|
|
193
|
+
const rows = await db.prepare(`SELECT id FROM ${table} WHERE doc_id = ?`).all(doc.id);
|
|
180
194
|
for (const r of rows) ownedIds.push(r.id);
|
|
181
195
|
}
|
|
182
196
|
|
|
183
|
-
db.exec("BEGIN IMMEDIATE;");
|
|
197
|
+
await db.exec("BEGIN IMMEDIATE;");
|
|
184
198
|
try {
|
|
185
199
|
for (const mc of microChunks) {
|
|
186
200
|
try {
|
|
187
|
-
db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
|
|
201
|
+
await db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
|
|
188
202
|
} catch {}
|
|
189
203
|
}
|
|
190
204
|
|
|
191
205
|
// Auto-clean Agent knowledge graph links pointing at this document.
|
|
192
|
-
db.prepare("DELETE FROM knowledge_links WHERE doc_id = ?").run(doc.id);
|
|
206
|
+
await db.prepare("DELETE FROM knowledge_links WHERE doc_id = ?").run(doc.id);
|
|
193
207
|
|
|
194
208
|
for (const id of ownedIds) {
|
|
195
209
|
// GLOB: '*' suffix is exact (unlike LIKE, '_' stays literal in ids like doc_xxx).
|
|
196
|
-
db.prepare(
|
|
210
|
+
await db.prepare(
|
|
197
211
|
"DELETE FROM graph_edges WHERE source_id = ? OR target_id = ? OR source_id GLOB ? OR target_id GLOB ?"
|
|
198
212
|
).run(id, id, `${id}*`, `${id}*`);
|
|
199
213
|
}
|
|
200
214
|
|
|
201
|
-
db.prepare("DELETE FROM documents WHERE id = ?").run(doc.id);
|
|
215
|
+
await db.prepare("DELETE FROM documents WHERE id = ?").run(doc.id);
|
|
202
216
|
|
|
203
|
-
db.exec("COMMIT;");
|
|
217
|
+
await db.exec("COMMIT;");
|
|
204
218
|
} catch (err) {
|
|
205
|
-
db.exec("ROLLBACK;");
|
|
219
|
+
await db.exec("ROLLBACK;");
|
|
206
220
|
throw err;
|
|
207
221
|
}
|
|
208
222
|
|
|
209
223
|
if (doc.blob_hash) {
|
|
210
|
-
const
|
|
224
|
+
const refCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM documents WHERE blob_hash = ?").get(doc.blob_hash);
|
|
225
|
+
const refCount = refCountRow ? refCountRow.cnt : 0;
|
|
211
226
|
if (refCount === 0) {
|
|
212
227
|
await deleteBlob(doc.blob_hash, customBlobDir);
|
|
213
228
|
}
|
|
214
229
|
}
|
|
215
230
|
|
|
231
|
+
if (getConfig().mode === "hybrid-sync") {
|
|
232
|
+
try {
|
|
233
|
+
const { enqueueSyncTask } = await import("../db/sync_queue.js");
|
|
234
|
+
await enqueueSyncTask("delete_document", docIdOrPath);
|
|
235
|
+
} catch (err) {
|
|
236
|
+
console.error("Failed to queue document delete sync task:", err.message);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
216
240
|
return { deleted: true, docId: doc.id, title: doc.title, linksCleaned: true };
|
|
217
241
|
}
|
package/mcp-server/memory.js
CHANGED
|
@@ -125,6 +125,22 @@ async function maybeMigrateLegacy(key) {
|
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
export async function readMemory(key) {
|
|
128
|
+
const { getConfig } = await import("./config/config_manager.js");
|
|
129
|
+
const config = getConfig();
|
|
130
|
+
if (config.mode === "only-cloud") {
|
|
131
|
+
try {
|
|
132
|
+
const { getDatabase } = await import("./db/database.js");
|
|
133
|
+
const db = await getDatabase();
|
|
134
|
+
const row = await db.prepare("SELECT content FROM notebooks WHERE key = ?;").get(key);
|
|
135
|
+
if (row && row.content) {
|
|
136
|
+
return row.content.split("\n").filter((l) => l.startsWith("- ["));
|
|
137
|
+
}
|
|
138
|
+
} catch (err) {
|
|
139
|
+
console.error("Failed to read memory from cloud database:", err.message);
|
|
140
|
+
}
|
|
141
|
+
return [];
|
|
142
|
+
}
|
|
143
|
+
|
|
128
144
|
const fp = memoryPath(key);
|
|
129
145
|
if (existsSync(fp)) {
|
|
130
146
|
const content = await readFile(fp, "utf-8");
|
|
@@ -149,10 +165,67 @@ export async function writeMemory(key, entries) {
|
|
|
149
165
|
}
|
|
150
166
|
}
|
|
151
167
|
const content = lines.join("\n") + "\n" + (entries.length ? entries.join("\n") + "\n" : "");
|
|
168
|
+
|
|
169
|
+
const { getConfig } = await import("./config/config_manager.js");
|
|
170
|
+
const config = getConfig();
|
|
171
|
+
if (config.mode === "only-cloud") {
|
|
172
|
+
try {
|
|
173
|
+
const { getDatabase } = await import("./db/database.js");
|
|
174
|
+
const db = await getDatabase();
|
|
175
|
+
await db.prepare(`
|
|
176
|
+
INSERT INTO notebooks (key, content, updated_at)
|
|
177
|
+
VALUES (?, ?, ?)
|
|
178
|
+
ON CONFLICT(key) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at;
|
|
179
|
+
`).run(key, content, Date.now());
|
|
180
|
+
} catch (err) {
|
|
181
|
+
console.error("Failed to write memory to cloud database:", err.message);
|
|
182
|
+
}
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
152
186
|
await writeFile(memoryPath(key), content);
|
|
187
|
+
|
|
188
|
+
if (config.mode === "hybrid-sync") {
|
|
189
|
+
try {
|
|
190
|
+
const { enqueueSyncTask } = await import("./db/sync_queue.js");
|
|
191
|
+
await enqueueSyncTask("write_memory", key, content);
|
|
192
|
+
} catch (err) {
|
|
193
|
+
console.error("Failed to queue memory sync task:", err.message);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
153
196
|
}
|
|
154
197
|
|
|
155
198
|
export async function listProjectStores() {
|
|
199
|
+
const { getConfig } = await import("./config/config_manager.js");
|
|
200
|
+
const config = getConfig();
|
|
201
|
+
if (config.mode === "only-cloud") {
|
|
202
|
+
try {
|
|
203
|
+
const { getDatabase } = await import("./db/database.js");
|
|
204
|
+
const db = await getDatabase();
|
|
205
|
+
const rows = await db.prepare("SELECT key, content FROM notebooks WHERE key != ?;").all(GLOBAL_KEY);
|
|
206
|
+
const stores = [];
|
|
207
|
+
for (const row of rows) {
|
|
208
|
+
const key = row.key;
|
|
209
|
+
const content = row.content;
|
|
210
|
+
const facts = content.split("\n").filter((l) => l.startsWith("- ["));
|
|
211
|
+
const meta = parseMeta(content);
|
|
212
|
+
stores.push({
|
|
213
|
+
key,
|
|
214
|
+
path: meta.path || (key.includes("/") || key.includes(":") ? key : null),
|
|
215
|
+
basename: basename(meta.path || key) || key,
|
|
216
|
+
file: `${slugify(key)}.md`,
|
|
217
|
+
count: facts.length,
|
|
218
|
+
legacy: !meta.path,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
stores.sort((a, b) => a.basename.localeCompare(b.basename));
|
|
222
|
+
return stores;
|
|
223
|
+
} catch (err) {
|
|
224
|
+
console.error("Failed to list memory stores from cloud database:", err.message);
|
|
225
|
+
}
|
|
226
|
+
return [];
|
|
227
|
+
}
|
|
228
|
+
|
|
156
229
|
const stores = [];
|
|
157
230
|
const files = await readdir(MEMORY_DIR).catch(() => []);
|
|
158
231
|
for (const f of files) {
|