@lotargo/memory_plugin 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +30 -5
- package/mcp-server/boot.js +43 -0
- package/mcp-server/cli_boot.js +37 -0
- package/mcp-server/config/config_manager.js +1 -0
- package/mcp-server/db/migrations.js +15 -0
- package/mcp-server/ingest/chunker.js +179 -12
- package/mcp-server/ingest/pipeline.js +3 -3
- package/mcp-server/preinstall.js +23 -2
- package/mcp-server/prompt_manager.js +226 -225
- package/mcp-server/retrieval/retriever.js +142 -12
- package/mcp-server/tools/rag_tools.js +344 -283
- package/package.json +10 -5
- package/skills/using-memory/SKILL.md +11 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,49 @@ All notable changes to `@lotargo/memory_plugin` are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.6.2] - 2026-08-12
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- **Batch retrieval API** (`batch_query_knowledge_base`): execute multiple search queries in a single MCP call. All query embeddings computed in one ONNX pass, queries run in parallel via `Promise.all`. Ideal for cross-document comparisons and multi-part analysis — significantly reduces API overhead vs N separate `query_knowledge_base` calls.
|
|
13
|
+
- **Policy expansion toggle** (`config.policyExpansion`, default: `true`): table summaries and code signatures are automatically expanded to full content for better recall (~+5-10% recall, slight MRR trade-off). Disable per-call or via config for pure micro_chunk precision.
|
|
14
|
+
- **RAG evaluation test** (`tests/unit/rag_evaluation.test.js`): 10 analytical queries with expected-fact verification (100% pass rate on financial reports). Includes raw-question vs optimized-query comparison demonstrating +33% fact retrieval improvement.
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- `hybridQuery()` accepts `_precomputedVector` internal parameter for batch embedding reuse.
|
|
19
|
+
- `PROMPT_BLOCK` (injected into AGENTS.md/CLAUDE.md) updated with query optimization and batch usage directives.
|
|
20
|
+
- SKILL.md updated with batch query tool and query formulation guidance.
|
|
21
|
+
|
|
22
|
+
## [1.6.1] - 2026-08-10
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- **Cryptic crash on Node.js < 22.5.0** (`No such built-in module: node:sqlite`).
|
|
27
|
+
ESM static imports are hoisted, so the `node:sqlite` import in `database.js`
|
|
28
|
+
crashed before any user code could run. Three layers of protection are now in
|
|
29
|
+
place:
|
|
30
|
+
1. **Boot guard** (`boot.js` / `cli_boot.js`): new lightweight entry points
|
|
31
|
+
that check `process.versions.node` *before* loading the ESM module graph.
|
|
32
|
+
On incompatible versions they print a clear boxed error with upgrade
|
|
33
|
+
instructions (`nvm install 22` / `brew install node@22`) and exit.
|
|
34
|
+
2. **`engine-strict`** (`.npmrc`): `npm install` now **fails** instead of
|
|
35
|
+
merely warning when `engines.node >= 22.5.0` is not satisfied.
|
|
36
|
+
3. **Preinstall warning** (`preinstall.js`): a prominent `stderr` message is
|
|
37
|
+
printed during installation on unsupported Node versions, explaining that
|
|
38
|
+
the server will not start.
|
|
39
|
+
- Process-kill patterns in `preinstall.js` now match the new `boot.js` entry
|
|
40
|
+
point in addition to `index.js`, so global updates correctly terminate running
|
|
41
|
+
server instances.
|
|
42
|
+
|
|
43
|
+
### Changed
|
|
44
|
+
|
|
45
|
+
- All `bin` entry points (`memory_plugin`, `memory-agent`, `memory-cli`) now
|
|
46
|
+
route through `boot.js` / `cli_boot.js` instead of directly to `index.js` /
|
|
47
|
+
`cli.js`.
|
|
48
|
+
- `.npmrc` is no longer git-ignored; it contains only the project-level
|
|
49
|
+
`engine-strict=true` setting (npm never publishes `.npmrc` to the tarball).
|
|
50
|
+
|
|
8
51
|
## [1.6.0] - 2026-08-10
|
|
9
52
|
|
|
10
53
|
This release is the outcome of a full five-part audit (publishing, security, code
|
|
@@ -105,4 +148,5 @@ a critical retrieval regression introduced after `v1.5.3`.
|
|
|
105
148
|
- `BENCHMARKS.md` tables were re-derived from the stored JSON artifacts; the
|
|
106
149
|
bge-m3 section had carried e5-small numbers shifted by a column.
|
|
107
150
|
|
|
151
|
+
[1.6.1]: https://github.com/Lotargo/memory_pugin/releases/tag/v1.6.1
|
|
108
152
|
[1.6.0]: https://github.com/Lotargo/memory_pugin/releases/tag/v1.6.0
|
package/README.md
CHANGED
|
@@ -176,6 +176,7 @@ The MCP server registers **14 MCP tools** accessible across all connected AI env
|
|
|
176
176
|
| :--- | :------------- | :---------- |
|
|
177
177
|
| `ingest_document` | `content`, `type`, `title`, `path`, `generateEmbeddings` | Ingest local files, URLs, or raw text into the 3-tier index (Big/Medium/Small) with ONNX vector embeddings and GraphRAG symbol extraction. |
|
|
178
178
|
| `query_knowledge_base` | `query`, `limit`, `instruction`, `generateEmbeddings` | Perform hybrid search (RSF/RRF BM25 + dense vector similarity) to retrieve candidate document sections with defined code symbols. |
|
|
179
|
+
| `batch_query_knowledge_base` | `queries` (array), `limit`, `instruction`, `generateEmbeddings` | Execute multiple queries in a single batch call. More efficient than separate `query_knowledge_base` calls — all embeddings computed in one ONNX pass, queries run in parallel. Ideal for comparisons and multi-topic analysis. |
|
|
179
180
|
| `manage_knowledge_base` | `action`, `docId`, `snapshotPath` | Inspect DB stats (`stats`), list documents (`list`), read full raw document (`read_document`), delete document (`delete`), or export/import snapshots (`export_snapshot` / `import_snapshot`). |
|
|
180
181
|
| `reindex_knowledge_base` | `model`, `dimension` | Re-embed all stored vectors with the active (or specified) embedding model and vector dimension. Use after switching the embedding model or vector dimension so previously indexed documents remain retrievable. Preserves documents, FTS index, graph edges, and fact links. |
|
|
181
182
|
| `link_knowledge` | `action`, `factText`, `docId`, `scope`, `startLine`, `endLine`, `relationType` | Create, list, or retrieve semantic graph links connecting Notebook facts to Knowledge Base documents, sections, or line ranges. Actions: `link`, `list_links`, `get_doc_links`. |
|
|
@@ -319,6 +320,7 @@ The engine is configured through `<memory-dir>/config.json` (created with defaul
|
|
|
319
320
|
| `onnxThreads` | `0` | ONNX WASM threads: `0` auto-detect, or `1-16` |
|
|
320
321
|
| `executionDevice` | `cpu` | `cpu` or `webgpu` (experimental) |
|
|
321
322
|
| `vectorScanLimit` | `50000` | Max micro-chunks scanned per vector query (`0` = unlimited) |
|
|
323
|
+
| `policyExpansion` | `true` | Expand table_summary/code_signature policy chunks for better recall (slight MRR trade-off). Disable for pure micro_chunk precision. |
|
|
322
324
|
| `injectLimit` | `10` | Max facts injected into the agent's system prompt |
|
|
323
325
|
| `conflictStrategy` | `merge` | Hybrid-sync conflict resolution: `merge`, `cloud-wins`, or `local-wins` |
|
|
324
326
|
| `tursoUrl` | `""` | Primary Turso endpoint URL (set by `login`) |
|
|
@@ -334,18 +336,41 @@ The engine is configured through `<memory-dir>/config.json` (created with defaul
|
|
|
334
336
|
|
|
335
337
|
## Testing & Benchmarking
|
|
336
338
|
|
|
337
|
-
To run the automated test suite and benchmarks locally:
|
|
339
|
+
To run the automated test suite and benchmarks locally, from the repository root:
|
|
338
340
|
|
|
339
341
|
```bash
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
# Run unit and integration tests
|
|
342
|
+
# Unit + integration + cloud suites (12 files) — fast and fully offline
|
|
343
343
|
npm test
|
|
344
344
|
|
|
345
|
-
#
|
|
345
|
+
# End-to-end smoke test with REAL ONNX embeddings — run before a release
|
|
346
|
+
npm run smoke
|
|
347
|
+
|
|
348
|
+
# Search quality & ingestion benchmarks
|
|
346
349
|
npm run benchmark
|
|
347
350
|
```
|
|
348
351
|
|
|
352
|
+
### Two testing modes, and why both exist
|
|
353
|
+
|
|
354
|
+
`npm test` runs every suite with `generateEmbeddings: false`. That keeps it fast
|
|
355
|
+
and offline (no model download, no network), but it means the **dense-vector half
|
|
356
|
+
of the engine is never exercised** — retrieval falls back to BM25-only.
|
|
357
|
+
|
|
358
|
+
`npm run smoke` covers exactly that blind spot: it ingests a document with real
|
|
359
|
+
ONNX vectors and asserts that hybrid retrieval returns a non-zero cosine
|
|
360
|
+
similarity, plus that a Russian query still reaches an English document
|
|
361
|
+
(something BM25 cannot do). It also walks the full user journey — remember →
|
|
362
|
+
recall → ingest → query → link → update → get → forget — and checks the ingest
|
|
363
|
+
path guard.
|
|
364
|
+
|
|
365
|
+
This split is not academic: a regression in `v1.5.3+` disabled vector search
|
|
366
|
+
entirely (`node:sqlite` returns BLOBs as `Uint8Array`, and a `Buffer.isBuffer()`
|
|
367
|
+
guard discarded every stored vector) while all offline suites stayed green. The
|
|
368
|
+
smoke test exists so that class of failure cannot ship unnoticed again.
|
|
369
|
+
|
|
370
|
+
The smoke test reuses the model weights already cached in your data directory, so
|
|
371
|
+
it does not re-download them. Point `MEMORY_MODEL_CACHE` at a cache directory to
|
|
372
|
+
override the lookup; without any cache the weights are fetched once.
|
|
373
|
+
|
|
349
374
|
For complete methodology details and search quality evaluation metrics, see [`docs/BENCHMARKS.md`](./docs/BENCHMARKS.md).
|
|
350
375
|
|
|
351
376
|
### Empirical Search Quality Results
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// ── Boot Guard ──────────────────────────────────────────────────────────────
|
|
4
|
+
// This file is the true entry point for both `memory_plugin` and `memory-agent`
|
|
5
|
+
// binaries. Its sole purpose is to verify the Node.js version BEFORE the ESM
|
|
6
|
+
// module graph is evaluated — because `mcp-server/index.js` transitively
|
|
7
|
+
// imports `node:sqlite` (a built-in available only from Node 22.5.0), and ESM
|
|
8
|
+
// static imports are hoisted, so a version check inside that file would never
|
|
9
|
+
// execute.
|
|
10
|
+
//
|
|
11
|
+
// By keeping this file free of any `node:sqlite` dependency we can print a
|
|
12
|
+
// clear, actionable error message instead of the cryptic
|
|
13
|
+
// "No such built-in module: node:sqlite"
|
|
14
|
+
// that users on Node 18/20/21 would otherwise see.
|
|
15
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
16
|
+
|
|
17
|
+
const MIN_MAJOR = 22;
|
|
18
|
+
const MIN_MINOR = 5;
|
|
19
|
+
|
|
20
|
+
const [major, minor] = process.versions.node.split(".").map(Number);
|
|
21
|
+
|
|
22
|
+
if (major < MIN_MAJOR || (major === MIN_MAJOR && minor < MIN_MINOR)) {
|
|
23
|
+
process.stderr.write(
|
|
24
|
+
`\n` +
|
|
25
|
+
` ╔══════════════════════════════════════════════════════════════════╗\n` +
|
|
26
|
+
` ║ @lotargo/memory_plugin requires Node.js >= 22.5.0 ║\n` +
|
|
27
|
+
` ║ ║\n` +
|
|
28
|
+
` ║ Detected: Node.js ${process.versions.node.padEnd(44)}║\n` +
|
|
29
|
+
` ║ ║\n` +
|
|
30
|
+
` ║ The built-in node:sqlite module used by this plugin was ║\n` +
|
|
31
|
+
` ║ introduced in Node.js 22.5.0. Please upgrade your ║\n` +
|
|
32
|
+
` ║ Node.js installation: ║\n` +
|
|
33
|
+
` ║ ║\n` +
|
|
34
|
+
` ║ nvm install 22 # or: brew install node@22 ║\n` +
|
|
35
|
+
` ║ ║\n` +
|
|
36
|
+
` ╚══════════════════════════════════════════════════════════════════╝\n` +
|
|
37
|
+
`\n`
|
|
38
|
+
);
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Version is OK — hand off to the real entry point.
|
|
43
|
+
import("./index.js");
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// ── CLI Boot Guard ──────────────────────────────────────────────────────────
|
|
4
|
+
// Same version check as boot.js — see that file for the rationale.
|
|
5
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
6
|
+
|
|
7
|
+
const MIN_MAJOR = 22;
|
|
8
|
+
const MIN_MINOR = 5;
|
|
9
|
+
|
|
10
|
+
const [major, minor] = process.versions.node.split(".").map(Number);
|
|
11
|
+
|
|
12
|
+
if (major < MIN_MAJOR || (major === MIN_MAJOR && minor < MIN_MINOR)) {
|
|
13
|
+
process.stderr.write(
|
|
14
|
+
`\n` +
|
|
15
|
+
` ╔══════════════════════════════════════════════════════════════════╗\n` +
|
|
16
|
+
` ║ @lotargo/memory_plugin requires Node.js >= 22.5.0 ║\n` +
|
|
17
|
+
` ║ ║\n` +
|
|
18
|
+
` ║ Detected: Node.js ${process.versions.node.padEnd(44)}║\n` +
|
|
19
|
+
` ║ ║\n` +
|
|
20
|
+
` ║ The built-in node:sqlite module used by this plugin was ║\n` +
|
|
21
|
+
` ║ introduced in Node.js 22.5.0. Please upgrade your ║\n` +
|
|
22
|
+
` ║ Node.js installation: ║\n` +
|
|
23
|
+
` ║ ║\n` +
|
|
24
|
+
` ║ nvm install 22 # or: brew install node@22 ║\n` +
|
|
25
|
+
` ║ ║\n` +
|
|
26
|
+
` ╚══════════════════════════════════════════════════════════════════╝\n` +
|
|
27
|
+
`\n`
|
|
28
|
+
);
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Version is OK — hand off to the real CLI.
|
|
33
|
+
import("./cli.js").then(m => {
|
|
34
|
+
if (process.argv[1] && process.argv[1].includes("cli_boot.js")) {
|
|
35
|
+
m.runCli().catch((err) => console.error("CLI error:", err));
|
|
36
|
+
}
|
|
37
|
+
});
|
|
@@ -25,6 +25,7 @@ export const DEFAULT_CONFIG = {
|
|
|
25
25
|
username: "", // Account username from the Turso OAuth profile
|
|
26
26
|
ingestAllowedPaths: [], // Extra directories ingest_document(type:"file") may read from
|
|
27
27
|
ingestAllowAnyPath: false, // Escape hatch: allow reading ANY path from disk (unsafe)
|
|
28
|
+
policyExpansion: true, // Expand table_summary/code_signature policy chunks (boosts recall, slight MRR trade-off)
|
|
28
29
|
};
|
|
29
30
|
|
|
30
31
|
let cachedConfig = null;
|
|
@@ -138,6 +138,21 @@ const MIGRATIONS = [
|
|
|
138
138
|
`);
|
|
139
139
|
},
|
|
140
140
|
},
|
|
141
|
+
{
|
|
142
|
+
version: 5,
|
|
143
|
+
name: "005_retrieval_policy",
|
|
144
|
+
up: async (db) => {
|
|
145
|
+
try {
|
|
146
|
+
await db.exec(`ALTER TABLE micro_chunks ADD COLUMN retrieval_policy TEXT DEFAULT 'micro_chunk';`);
|
|
147
|
+
} catch (e) {}
|
|
148
|
+
try {
|
|
149
|
+
await db.exec(`ALTER TABLE micro_chunks ADD COLUMN policy_source_id TEXT;`);
|
|
150
|
+
} catch (e) {}
|
|
151
|
+
await db.exec(`
|
|
152
|
+
CREATE INDEX IF NOT EXISTS idx_micro_chunks_retrieval_policy ON micro_chunks(retrieval_policy);
|
|
153
|
+
`);
|
|
154
|
+
},
|
|
155
|
+
},
|
|
141
156
|
];
|
|
142
157
|
|
|
143
158
|
export async function runMigrations(db) {
|
|
@@ -5,6 +5,159 @@ export function estimateTokens(text) {
|
|
|
5
5
|
return Math.ceil(text.length / 4);
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
+
const CODE_SIGNATURE_REGEX = /^\s*(?:export\s+|async\s+)?(?:function|class|def|pub\s+fn|fn|struct|interface|enum)\s+/;
|
|
9
|
+
|
|
10
|
+
function _classifyLine(line) {
|
|
11
|
+
const t = line.trimStart();
|
|
12
|
+
if (t.startsWith("/**")) return "jsdoc_start";
|
|
13
|
+
if (t.startsWith("*/")) return "jsdoc_end";
|
|
14
|
+
if (t.startsWith("*")) return "jsdoc_mid";
|
|
15
|
+
if (t.startsWith("//")) return "line_comment";
|
|
16
|
+
if (t.startsWith("#")) return "hash_comment";
|
|
17
|
+
if (t.startsWith("'''") || t.startsWith('"""')) return "py_docstring";
|
|
18
|
+
return "code";
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function extractCodeSignatures(codeContent) {
|
|
22
|
+
const lines = codeContent.split("\n");
|
|
23
|
+
const signatures = [];
|
|
24
|
+
|
|
25
|
+
const fenceMatch = lines[0] && lines[0].match(/^(\s*)(```|~~~)/);
|
|
26
|
+
const bodyStart = fenceMatch ? 1 : 0
|
|
27
|
+
const bodyEnd = fenceMatch && (lines[lines.length - 1].startsWith("```") || lines[lines.length - 1].startsWith("~~~")) ? lines.length - 1 : lines.length;
|
|
28
|
+
const bodyLines = lines.slice(bodyStart, bodyEnd);
|
|
29
|
+
|
|
30
|
+
let i = 0;
|
|
31
|
+
while (i < bodyLines.length) {
|
|
32
|
+
const line = bodyLines[i];
|
|
33
|
+
const type = _classifyLine(line);
|
|
34
|
+
|
|
35
|
+
if (type === "jsdoc_start") {
|
|
36
|
+
const jsdocBlock = [line];
|
|
37
|
+
let j = i + 1;
|
|
38
|
+
while (j < bodyLines.length) {
|
|
39
|
+
jsdocBlock.push(bodyLines[j]);
|
|
40
|
+
if (_classifyLine(bodyLines[j]) === "jsdoc_end") {
|
|
41
|
+
j++;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
j++;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (j < bodyLines.length && CODE_SIGNATURE_REGEX.test(bodyLines[j])) {
|
|
48
|
+
const sigLines = [bodyLines[j]];
|
|
49
|
+
const pyDocResult = _tryPyDocstring(bodyLines, j + 1);
|
|
50
|
+
if (pyDocResult.docLines.length > 0) {
|
|
51
|
+
sigLines.push(...pyDocResult.docLines);
|
|
52
|
+
}
|
|
53
|
+
const endIdx = pyDocResult.docLines.length > 0 ? pyDocResult.endIdx : j;
|
|
54
|
+
|
|
55
|
+
signatures.push({
|
|
56
|
+
signature: [...jsdocBlock, ...sigLines].join("\n").trim(),
|
|
57
|
+
line_number: j + bodyStart + 1,
|
|
58
|
+
});
|
|
59
|
+
i = endIdx + 1;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
i = j;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (type === "line_comment" || type === "hash_comment") {
|
|
68
|
+
const commentBlock = [line];
|
|
69
|
+
let j = i + 1;
|
|
70
|
+
while (j < bodyLines.length && _classifyLine(bodyLines[j]) === type) {
|
|
71
|
+
commentBlock.push(bodyLines[j]);
|
|
72
|
+
j++;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (j < bodyLines.length && CODE_SIGNATURE_REGEX.test(bodyLines[j])) {
|
|
76
|
+
const sigLines = [bodyLines[j]];
|
|
77
|
+
const pyDocResult = _tryPyDocstring(bodyLines, j + 1);
|
|
78
|
+
if (pyDocResult.docLines.length > 0) {
|
|
79
|
+
sigLines.push(...pyDocResult.docLines);
|
|
80
|
+
}
|
|
81
|
+
const endIdx = pyDocResult.docLines.length > 0 ? pyDocResult.endIdx : j;
|
|
82
|
+
|
|
83
|
+
signatures.push({
|
|
84
|
+
signature: [...commentBlock, ...sigLines].join("\n").trim(),
|
|
85
|
+
line_number: j + bodyStart + 1,
|
|
86
|
+
});
|
|
87
|
+
i = endIdx + 1;
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
i = j;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (CODE_SIGNATURE_REGEX.test(line)) {
|
|
96
|
+
const sigLines = [line];
|
|
97
|
+
const pyDocResult = _tryPyDocstring(bodyLines, i + 1);
|
|
98
|
+
if (pyDocResult.docLines.length > 0) {
|
|
99
|
+
sigLines.push(...pyDocResult.docLines);
|
|
100
|
+
}
|
|
101
|
+
const endIdx = pyDocResult.docLines.length > 0 ? pyDocResult.endIdx : i;
|
|
102
|
+
|
|
103
|
+
signatures.push({
|
|
104
|
+
signature: sigLines.join("\n").trim(),
|
|
105
|
+
line_number: i + bodyStart + 1,
|
|
106
|
+
});
|
|
107
|
+
i = endIdx + 1;
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
i++;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return signatures;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function _tryPyDocstring(bodyLines, startIdx) {
|
|
118
|
+
let k = startIdx;
|
|
119
|
+
while (k < bodyLines.length && bodyLines[k].trim() === "") k++;
|
|
120
|
+
if (k >= bodyLines.length) return { docLines: [], endIdx: startIdx - 1 };
|
|
121
|
+
|
|
122
|
+
const line = bodyLines[k];
|
|
123
|
+
const tripleDouble = /^\s*"""/.test(line);
|
|
124
|
+
const tripleSingle = /^\s*'''/.test(line);
|
|
125
|
+
const marker = tripleDouble ? '"""' : tripleSingle ? "'''" : null;
|
|
126
|
+
if (!marker) return { docLines: [], endIdx: startIdx - 1 };
|
|
127
|
+
|
|
128
|
+
const docLines = [line];
|
|
129
|
+
if (line.includes(marker.repeat(2)) && line.indexOf(marker) !== line.lastIndexOf(marker)) {
|
|
130
|
+
return { docLines, endIdx: k };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
for (let m = k + 1; m < bodyLines.length; m++) {
|
|
134
|
+
docLines.push(bodyLines[m]);
|
|
135
|
+
if (bodyLines[m].includes(marker)) {
|
|
136
|
+
return { docLines, endIdx: m };
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return { docLines, endIdx: k };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function generateTableSummary(tableContent, breadcrumbs = "") {
|
|
143
|
+
const lines = tableContent.split("\n").filter((l) => l.trim().length > 0);
|
|
144
|
+
if (lines.length === 0) return null;
|
|
145
|
+
|
|
146
|
+
const headerLine = lines[0];
|
|
147
|
+
const columns = headerLine
|
|
148
|
+
.split("|")
|
|
149
|
+
.map((c) => c.trim())
|
|
150
|
+
.filter((c) => c.length > 0);
|
|
151
|
+
|
|
152
|
+
const separatorLine = lines[1] || "";
|
|
153
|
+
const hasSeparator = /^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/.test(separatorLine);
|
|
154
|
+
const dataLines = hasSeparator ? lines.slice(2) : lines.slice(1);
|
|
155
|
+
const rowCount = dataLines.length;
|
|
156
|
+
|
|
157
|
+
const contextPart = breadcrumbs ? ` Context: ${breadcrumbs}.` : "";
|
|
158
|
+
return `Table with columns [${columns.join(", ")}] containing ${rowCount} row${rowCount !== 1 ? "s" : ""}.${contextPart}`;
|
|
159
|
+
}
|
|
160
|
+
|
|
8
161
|
// 1. BIG LEVEL: Heading & Section Hierarchy Parser
|
|
9
162
|
export function parseSections(markdown, docTitle = "Document") {
|
|
10
163
|
const lines = markdown.split("\n");
|
|
@@ -72,7 +225,7 @@ export function extractMediumBlocks(section, sectionId, docId) {
|
|
|
72
225
|
let blockIndex = 0;
|
|
73
226
|
|
|
74
227
|
let currentLines = [];
|
|
75
|
-
let currentBlockType = "paragraph";
|
|
228
|
+
let currentBlockType = "paragraph";
|
|
76
229
|
|
|
77
230
|
function pushCurrentBlock() {
|
|
78
231
|
const blockContent = currentLines.join("\n").trim();
|
|
@@ -97,7 +250,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
|
|
|
97
250
|
for (let i = 0; i < lines.length; i++) {
|
|
98
251
|
const line = lines[i];
|
|
99
252
|
|
|
100
|
-
// Check code fence
|
|
101
253
|
const fenceMatch = line.match(/^(\s*)(```|~~~)/);
|
|
102
254
|
if (fenceMatch) {
|
|
103
255
|
if (!inFencedCode) {
|
|
@@ -121,7 +273,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
|
|
|
121
273
|
continue;
|
|
122
274
|
}
|
|
123
275
|
|
|
124
|
-
// Check table line
|
|
125
276
|
const isTableLine = /^\s*\|.*\|\s*$/.test(line);
|
|
126
277
|
if (isTableLine) {
|
|
127
278
|
if (currentBlockType !== "table" && currentLines.length > 0) {
|
|
@@ -134,7 +285,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
|
|
|
134
285
|
pushCurrentBlock();
|
|
135
286
|
}
|
|
136
287
|
|
|
137
|
-
// Check list item line
|
|
138
288
|
const isListLine = /^\s*([*+-]|\d+\.)\s+/.test(line);
|
|
139
289
|
if (isListLine) {
|
|
140
290
|
if (currentBlockType !== "list" && currentBlockType !== "paragraph" && currentLines.length > 0) {
|
|
@@ -145,7 +295,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
|
|
|
145
295
|
continue;
|
|
146
296
|
}
|
|
147
297
|
|
|
148
|
-
// Check empty line
|
|
149
298
|
if (line.trim().length === 0) {
|
|
150
299
|
if (currentLines.length > 0) {
|
|
151
300
|
pushCurrentBlock();
|
|
@@ -169,6 +318,7 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
|
|
|
169
318
|
|
|
170
319
|
function makeChunk(chunkText, extraMeta = {}) {
|
|
171
320
|
if (!chunkText || chunkText.trim().length === 0) return;
|
|
321
|
+
const { retrieval_policy, policy_source_id, ...rest } = extraMeta;
|
|
172
322
|
smallChunks.push({
|
|
173
323
|
id: `${mediumBlock.id}_s${smallIdx++}`,
|
|
174
324
|
medium_id: mediumBlock.id,
|
|
@@ -177,14 +327,24 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
|
|
|
177
327
|
content: chunkText.trim(),
|
|
178
328
|
breadcrumbs: mediumBlock.breadcrumbs,
|
|
179
329
|
token_count: estimateTokens(chunkText),
|
|
180
|
-
|
|
330
|
+
retrieval_policy: retrieval_policy || "micro_chunk",
|
|
331
|
+
policy_source_id: policy_source_id || null,
|
|
332
|
+
...rest,
|
|
181
333
|
});
|
|
182
334
|
}
|
|
183
335
|
|
|
184
336
|
// RULE FOR TABLES
|
|
185
337
|
if (mediumBlock.block_type === "table") {
|
|
338
|
+
const summary = generateTableSummary(content, mediumBlock.breadcrumbs);
|
|
339
|
+
if (summary) {
|
|
340
|
+
makeChunk(summary, {
|
|
341
|
+
retrieval_policy: "table_summary",
|
|
342
|
+
policy_source_id: mediumBlock.id,
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
|
|
186
346
|
if (tokenCount <= 350) {
|
|
187
|
-
makeChunk(content);
|
|
347
|
+
makeChunk(content, { retrieval_policy: "micro_chunk" });
|
|
188
348
|
return smallChunks;
|
|
189
349
|
}
|
|
190
350
|
|
|
@@ -205,15 +365,23 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
|
|
|
205
365
|
for (let i = 0; i < dataLines.length; i += chunkSize) {
|
|
206
366
|
const rowBatch = dataLines.slice(i, i + chunkSize);
|
|
207
367
|
const tableChunkText = `${headerStr}\n${rowBatch.join("\n")}`;
|
|
208
|
-
makeChunk(tableChunkText);
|
|
368
|
+
makeChunk(tableChunkText, { retrieval_policy: "micro_chunk" });
|
|
209
369
|
}
|
|
210
370
|
return smallChunks;
|
|
211
371
|
}
|
|
212
372
|
|
|
213
373
|
// RULE FOR CODE BLOCKS
|
|
214
374
|
if (mediumBlock.block_type === "code") {
|
|
375
|
+
const signatures = extractCodeSignatures(content);
|
|
376
|
+
for (const sig of signatures) {
|
|
377
|
+
makeChunk(sig.signature, {
|
|
378
|
+
retrieval_policy: "code_signature",
|
|
379
|
+
policy_source_id: mediumBlock.id,
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
|
|
215
383
|
if (tokenCount <= 350) {
|
|
216
|
-
makeChunk(content);
|
|
384
|
+
makeChunk(content, { retrieval_policy: "micro_chunk" });
|
|
217
385
|
return smallChunks;
|
|
218
386
|
}
|
|
219
387
|
|
|
@@ -243,7 +411,7 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
|
|
|
243
411
|
|
|
244
412
|
for (const block of astBlocks) {
|
|
245
413
|
const fullChunk = fenceHeader ? `${fenceHeader}\n${block}\n${fenceFooter}` : block;
|
|
246
|
-
makeChunk(fullChunk);
|
|
414
|
+
makeChunk(fullChunk, { retrieval_policy: "micro_chunk" });
|
|
247
415
|
}
|
|
248
416
|
return smallChunks;
|
|
249
417
|
}
|
|
@@ -265,8 +433,7 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
|
|
|
265
433
|
|
|
266
434
|
if (currentTokens + sTokens > TARGET_WINDOW_TOKENS && currentWindow.length > 0) {
|
|
267
435
|
makeChunk(currentWindow.join(" "));
|
|
268
|
-
|
|
269
|
-
// Safe Overlap: Keep the last sentence of the previous window if available
|
|
436
|
+
|
|
270
437
|
const lastSentence = currentWindow[currentWindow.length - 1];
|
|
271
438
|
currentWindow = [lastSentence, sentence];
|
|
272
439
|
currentTokens = estimateTokens(lastSentence) + sTokens;
|
|
@@ -129,8 +129,8 @@ export async function ingestDocument({
|
|
|
129
129
|
}
|
|
130
130
|
|
|
131
131
|
const insertMicroStmt = db.prepare(`
|
|
132
|
-
INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id)
|
|
133
|
-
VALUES (?, ?, ?, ?, ?, ?, ?);
|
|
132
|
+
INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id, retrieval_policy, policy_source_id)
|
|
133
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
|
|
134
134
|
`);
|
|
135
135
|
const insertFtsStmt = db.prepare(`
|
|
136
136
|
INSERT INTO micro_chunks_fts (id, content, breadcrumbs)
|
|
@@ -138,7 +138,7 @@ export async function ingestDocument({
|
|
|
138
138
|
`);
|
|
139
139
|
|
|
140
140
|
for (const micro of hierarchy.microChunks) {
|
|
141
|
-
await insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null);
|
|
141
|
+
await insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null, micro.retrieval_policy || "micro_chunk", micro.policy_source_id || null);
|
|
142
142
|
await insertFtsStmt.run(micro.id, micro.content, micro.breadcrumbs);
|
|
143
143
|
}
|
|
144
144
|
|
package/mcp-server/preinstall.js
CHANGED
|
@@ -5,6 +5,27 @@ if (process.env.CI || process.env.CONTINUOUS_INTEGRATION || process.env.DEBIAN_F
|
|
|
5
5
|
process.exit(0);
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
+
// ── Node version warning ────────────────────────────────────────────────────
|
|
9
|
+
// engines.node >= 22.5.0 is set in package.json but npm only warns by default.
|
|
10
|
+
// Print a loud, actionable message so the user notices before the server crashes.
|
|
11
|
+
{
|
|
12
|
+
const [major, minor] = process.versions.node.split(".").map(Number);
|
|
13
|
+
if (major < 22 || (major === 22 && minor < 5)) {
|
|
14
|
+
console.error(
|
|
15
|
+
`\n` +
|
|
16
|
+
` ⚠️ @lotargo/memory_plugin requires Node.js >= 22.5.0\n` +
|
|
17
|
+
` Detected: Node.js ${process.versions.node}\n` +
|
|
18
|
+
`\n` +
|
|
19
|
+
` The built-in node:sqlite module used by this plugin was\n` +
|
|
20
|
+
` introduced in Node.js 22.5.0. The server WILL NOT START\n` +
|
|
21
|
+
` on your current version.\n` +
|
|
22
|
+
`\n` +
|
|
23
|
+
` Please upgrade: nvm install 22 (or: brew install node@22)\n`
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
8
29
|
// Only run graceful process termination during explicit global npm updates
|
|
9
30
|
if (process.env.npm_config_global === "true" || process.env.MEMORY_PREINSTALL_FORCE === "true") {
|
|
10
31
|
try {
|
|
@@ -13,7 +34,7 @@ if (process.env.npm_config_global === "true" || process.env.MEMORY_PREINSTALL_FO
|
|
|
13
34
|
|
|
14
35
|
if (process.platform === "win32") {
|
|
15
36
|
try {
|
|
16
|
-
const psCmd = `Get-CimInstance Win32_Process | Where-Object { ($_.CommandLine -like '*mcp-server/index.js*' -or $_.CommandLine -like '*mcp-server\\\\index.js*') -and $_.CommandLine -notlike '*install*' -and $_.ProcessId -ne ${currentPid} -and $_.ProcessId -ne ${ppid} } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force }`;
|
|
37
|
+
const psCmd = `Get-CimInstance Win32_Process | Where-Object { ($_.CommandLine -like '*mcp-server/boot.js*' -or $_.CommandLine -like '*mcp-server\\\\boot.js*' -or $_.CommandLine -like '*mcp-server/index.js*' -or $_.CommandLine -like '*mcp-server\\\\index.js*') -and $_.CommandLine -notlike '*install*' -and $_.ProcessId -ne ${currentPid} -and $_.ProcessId -ne ${ppid} } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force }`;
|
|
17
38
|
execSync(`powershell -NoProfile -NonInteractive -Command "${psCmd}"`, { stdio: "ignore" });
|
|
18
39
|
} catch {}
|
|
19
40
|
} else {
|
|
@@ -30,7 +51,7 @@ if (process.env.npm_config_global === "true" || process.env.MEMORY_PREINSTALL_FO
|
|
|
30
51
|
|
|
31
52
|
if (!pid || pid === currentPid || pid === ppid || parentPid === currentPid) continue;
|
|
32
53
|
|
|
33
|
-
const isServer = cmd.includes("mcp-server/
|
|
54
|
+
const isServer = cmd.includes("mcp-server/boot.js") || cmd.includes("mcp-server/index.js");
|
|
34
55
|
const isInstaller = /npm|npx|yarn|pnpm|preinstall|install/i.test(cmd);
|
|
35
56
|
|
|
36
57
|
if (isServer && !isInstaller) {
|