pantheon-opencode 1.5.2 → 1.6.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pantheon/code-mode/README.md +4 -2
- package/.pantheon/code-mode/checkpoint_session.py +138 -102
- package/.pantheon/code-mode/compress-inline.py +102 -57
- package/.pantheon/code-mode/manifest.json +2 -8
- package/CHANGELOG.md +17 -0
- package/README.md +86 -52
- package/README.pt-BR.md +63 -43
- package/commands/pantheon-optimize.md +2 -2
- package/package.json +13 -1
- package/plugin.json +1 -1
- package/pyproject.toml +8 -15
- package/scripts/build-agents-md.mjs +2 -2
- package/scripts/check-plugin-v2-coverage.mjs +213 -0
- package/scripts/ci-validate-yaml.py +7 -5
- package/scripts/code_mode_server.py +116 -24
- package/scripts/doctor.mjs +85 -28
- package/scripts/install/cli-ui.mjs +1 -1
- package/scripts/install/config-migration.mjs +2 -2
- package/scripts/install/health-check.mjs +6 -19
- package/scripts/install/interactive.mjs +6 -6
- package/scripts/install/migrate.mjs +1 -1
- package/scripts/mcp_persistence_server.py +390 -76
- package/scripts/memory_mcp_server.py +370 -196
- package/scripts/postinstall.mjs +0 -7
- package/scripts/probe-context-rehydrate.mjs +28 -11
- package/scripts/prune.mjs +2 -2
- package/scripts/test-opencode-v2-sandbox.sh +1759 -0
- package/scripts/validate-routing.mjs +40 -0
- package/skills-lock.json +9 -10
- package/src/agents/mnemosyne.md +15 -16
- package/src/agents/zeus.md +1 -2
- package/src/mcp/code_mode_server.py +116 -24
- package/src/mcp/mcp_codemap_module.py +559 -129
- package/src/mcp/mcp_persistence_server.py +390 -76
- package/src/mcp/memory_mcp_server.py +371 -248
- package/src/mcp/pantheon_vision_server.py +136 -24
- package/src/mcp/requirements-mcp.txt +5 -3
- package/src/mcp/requirements-vision.txt +2 -0
- package/src/pantheon/model-command.ts +18 -7
- package/src/pantheon/todo-enforcer.ts +1 -1
- package/src/pantheon/vision.ts +1 -1
- package/src/plugin-v2.ts +134 -104
- package/src/plugins/tui/dist/tui.js +1 -1
- package/src/plugins/tui/package-lock.json +20 -20
- package/src/plugins/tui/package.json +12 -12
- package/src/routing.yml +0 -1
- package/src/skills/README.md +31 -36
- package/src/skills/auto-continue/SKILL.md +25 -75
- package/.pantheon/code-mode/eval-llm-judge.py +0 -255
- package/.pantheon/code-mode/eval-monte-carlo.py +0 -368
- package/.pantheon/code-mode/eval-run.py +0 -197
- package/.pantheon/code-mode/eval-static.py +0 -321
- package/.pantheon/code-mode/session-end-save.py +0 -186
- package/.pantheon/code-mode/session-end-save.sh +0 -18
- package/scripts/recover-zenodo-deposition.mjs +0 -146
- package/scripts/test-opencode-v1-v2-sandbox.sh +0 -862
- package/scripts/validate-zenodo-release.mjs +0 -154
- package/scripts/zenodo-artifact.mjs +0 -94
- package/scripts/zenodo-release-state.mjs +0 -53
- package/src/skills/orchestration-workflow/SKILL.md +0 -311
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""Pantheon Memory MCP Server — lightweight, zero heavy deps.
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
Search is FTS5 (BM25) only. The vector pipeline (sqlite-vec + fastembed,
|
|
5
|
+
~50MB of wheels and ~185MB RSS) was removed outright: the prefix-matched OR
|
|
6
|
+
query it fused against returned a candidate pool so polluted that the vector
|
|
7
|
+
half was compensating for a query bug rather than adding recall. See
|
|
8
|
+
``_build_fts_query`` for the query form that replaced it.
|
|
6
9
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
Both embedding backends are optional: if either fails to import the server
|
|
10
|
-
still starts and serves FTS5-only search (``PANTHEON_MEMORY_EMBED=off``
|
|
11
|
-
disables the vector pipeline explicitly).
|
|
10
|
+
Recall is therefore purely lexical. A query that shares no token with a
|
|
11
|
+
stored document will not retrieve it; there is no embedding to fall back on.
|
|
12
12
|
|
|
13
13
|
Tools:
|
|
14
|
-
memory_store — Store
|
|
15
|
-
memory_search —
|
|
14
|
+
memory_store — Store a value (auto-indexed into FTS5 by trigger)
|
|
15
|
+
memory_search — FTS5 BM25 keyword search with optional freshness decay
|
|
16
16
|
memory_recall — Exact recall by key
|
|
17
17
|
memory_forget — Delete by ID or key
|
|
18
18
|
memory_list — Chronological listing
|
|
@@ -27,29 +27,15 @@ from __future__ import annotations
|
|
|
27
27
|
import contextlib
|
|
28
28
|
import functools
|
|
29
29
|
import json
|
|
30
|
-
import os
|
|
31
30
|
import sqlite3
|
|
32
31
|
import time
|
|
33
32
|
from datetime import UTC, datetime
|
|
34
33
|
from pathlib import Path
|
|
35
|
-
from typing import Any
|
|
36
|
-
|
|
37
|
-
# Embedding backends are optional (issue #159): a failed fastembed/sqlite_vec
|
|
38
|
-
# install (network error, incompatible wheel) must not kill the server — it
|
|
39
|
-
# degrades to FTS-only search instead of crashing before the MCP handshake.
|
|
40
|
-
try:
|
|
41
|
-
import sqlite_vec
|
|
42
|
-
except ImportError: # pragma: no cover - degraded mode
|
|
43
|
-
sqlite_vec = None # type: ignore[assignment]
|
|
34
|
+
from typing import Annotated, Any
|
|
44
35
|
|
|
45
36
|
from _pantheon_paths import pantheon_home
|
|
46
|
-
|
|
47
|
-
try:
|
|
48
|
-
from fastembed import TextEmbedding
|
|
49
|
-
except ImportError: # pragma: no cover - degraded mode
|
|
50
|
-
TextEmbedding = None # type: ignore[assignment]
|
|
51
|
-
|
|
52
37
|
from mcp.server.fastmcp import FastMCP
|
|
38
|
+
from pydantic import BeforeValidator, Field
|
|
53
39
|
|
|
54
40
|
try:
|
|
55
41
|
import mcp_codemap_module as _codemap
|
|
@@ -66,17 +52,10 @@ except ImportError: # pragma: no cover - deployed runtime may need path fix
|
|
|
66
52
|
# ── Paths ─────────────────────────────────────────────────────────────────────
|
|
67
53
|
|
|
68
54
|
DB_PATH = pantheon_home() / "memory" / "memory.db"
|
|
69
|
-
EMBED_CACHE = Path.home() / ".cache" / "fastembed"
|
|
70
|
-
|
|
71
|
-
# PANTHEON_MEMORY_EMBED=off disables the embedding/vector pipeline so the
|
|
72
|
-
# server serves FTS-only search (issue #159). Default "on" keeps the current
|
|
73
|
-
# hybrid vector + FTS behavior. The flag is also forced off automatically
|
|
74
|
-
# when fastembed failed to import.
|
|
75
|
-
_EMBED_DISABLED = frozenset({"off", "0", "false", "no"})
|
|
76
|
-
|
|
77
55
|
|
|
78
56
|
_BYTE_UNIT = 1024
|
|
79
57
|
|
|
58
|
+
|
|
80
59
|
def _set_memory_dir(path: str | Path) -> None:
|
|
81
60
|
"""Override the memory db path for testing."""
|
|
82
61
|
global DB_PATH # noqa: PLW0603
|
|
@@ -130,19 +109,15 @@ CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
|
|
|
130
109
|
END;
|
|
131
110
|
"""
|
|
132
111
|
|
|
133
|
-
# Vector store requires the sqlite-vec extension; created only when available.
|
|
134
|
-
VEC_SCHEMA_SQL = """
|
|
135
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(
|
|
136
|
-
id INTEGER PRIMARY KEY,
|
|
137
|
-
embedding float[384]
|
|
138
|
-
);
|
|
139
|
-
"""
|
|
140
|
-
|
|
141
112
|
# ── FastMCP App ───────────────────────────────────────────────────────────────
|
|
142
113
|
|
|
143
114
|
mcp = FastMCP(
|
|
144
115
|
"pantheon-memory",
|
|
145
|
-
instructions=
|
|
116
|
+
instructions=(
|
|
117
|
+
"Lightweight memory with FTS5 (SQLite BM25) keyword search across "
|
|
118
|
+
"namespaces. Lexical retrieval only — there is no embedding or vector "
|
|
119
|
+
"index, so a query must share a token with the stored text to match."
|
|
120
|
+
),
|
|
146
121
|
)
|
|
147
122
|
|
|
148
123
|
|
|
@@ -153,9 +128,9 @@ mcp = FastMCP(
|
|
|
153
128
|
def _get_db() -> sqlite3.Connection:
|
|
154
129
|
"""Get or create the SQLite connection singleton.
|
|
155
130
|
|
|
156
|
-
Creates DB directory, applies WAL/performance pragmas,
|
|
157
|
-
|
|
158
|
-
|
|
131
|
+
Creates DB directory, applies WAL/performance pragmas, and runs schema
|
|
132
|
+
init. The ``code_*`` tools share this connection, so the codemap schema
|
|
133
|
+
is applied on the same handle.
|
|
159
134
|
"""
|
|
160
135
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
161
136
|
conn = sqlite3.connect(str(DB_PATH), check_same_thread=False)
|
|
@@ -163,10 +138,6 @@ def _get_db() -> sqlite3.Connection:
|
|
|
163
138
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
164
139
|
conn.execute("PRAGMA busy_timeout=5000")
|
|
165
140
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
166
|
-
# Register sqlite-vec extension (optional; enables vector search)
|
|
167
|
-
if sqlite_vec is not None:
|
|
168
|
-
sqlite_vec.load(conn)
|
|
169
|
-
conn.executescript(VEC_SCHEMA_SQL)
|
|
170
141
|
conn.executescript(SCHEMA_SQL)
|
|
171
142
|
if _codemap is not None:
|
|
172
143
|
with contextlib.suppress(Exception):
|
|
@@ -179,52 +150,180 @@ def _get_conn() -> sqlite3.Connection:
|
|
|
179
150
|
return _get_db()
|
|
180
151
|
|
|
181
152
|
|
|
182
|
-
# ──
|
|
153
|
+
# ── FTS Query Building ────────────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
# Function words carry no retrieval signal but prefix-match a large slice of
|
|
156
|
+
# the corpus, so OR-ing them in dilutes BM25 ranking with noise instead of
|
|
157
|
+
# narrowing the candidate pool. The list is deliberately limited to English
|
|
158
|
+
# function words and auxiliaries — no domain vocabulary.
|
|
159
|
+
_STOPWORDS = frozenset(
|
|
160
|
+
[
|
|
161
|
+
"a",
|
|
162
|
+
"about",
|
|
163
|
+
"above",
|
|
164
|
+
"after",
|
|
165
|
+
"again",
|
|
166
|
+
"against",
|
|
167
|
+
"all",
|
|
168
|
+
"am",
|
|
169
|
+
"an",
|
|
170
|
+
"and",
|
|
171
|
+
"any",
|
|
172
|
+
"are",
|
|
173
|
+
"as",
|
|
174
|
+
"at",
|
|
175
|
+
"be",
|
|
176
|
+
"because",
|
|
177
|
+
"been",
|
|
178
|
+
"before",
|
|
179
|
+
"being",
|
|
180
|
+
"below",
|
|
181
|
+
"between",
|
|
182
|
+
"both",
|
|
183
|
+
"but",
|
|
184
|
+
"by",
|
|
185
|
+
"can",
|
|
186
|
+
"cannot",
|
|
187
|
+
"could",
|
|
188
|
+
"did",
|
|
189
|
+
"do",
|
|
190
|
+
"does",
|
|
191
|
+
"doing",
|
|
192
|
+
"done",
|
|
193
|
+
"down",
|
|
194
|
+
"during",
|
|
195
|
+
"each",
|
|
196
|
+
"few",
|
|
197
|
+
"for",
|
|
198
|
+
"from",
|
|
199
|
+
"further",
|
|
200
|
+
"had",
|
|
201
|
+
"has",
|
|
202
|
+
"have",
|
|
203
|
+
"having",
|
|
204
|
+
"he",
|
|
205
|
+
"her",
|
|
206
|
+
"here",
|
|
207
|
+
"hers",
|
|
208
|
+
"him",
|
|
209
|
+
"his",
|
|
210
|
+
"how",
|
|
211
|
+
"i",
|
|
212
|
+
"if",
|
|
213
|
+
"in",
|
|
214
|
+
"into",
|
|
215
|
+
"is",
|
|
216
|
+
"it",
|
|
217
|
+
"its",
|
|
218
|
+
"itself",
|
|
219
|
+
"just",
|
|
220
|
+
"me",
|
|
221
|
+
"more",
|
|
222
|
+
"most",
|
|
223
|
+
"my",
|
|
224
|
+
"no",
|
|
225
|
+
"nor",
|
|
226
|
+
"not",
|
|
227
|
+
"now",
|
|
228
|
+
"of",
|
|
229
|
+
"off",
|
|
230
|
+
"on",
|
|
231
|
+
"once",
|
|
232
|
+
"only",
|
|
233
|
+
"or",
|
|
234
|
+
"other",
|
|
235
|
+
"our",
|
|
236
|
+
"ours",
|
|
237
|
+
"out",
|
|
238
|
+
"over",
|
|
239
|
+
"own",
|
|
240
|
+
"same",
|
|
241
|
+
"she",
|
|
242
|
+
"should",
|
|
243
|
+
"so",
|
|
244
|
+
"some",
|
|
245
|
+
"such",
|
|
246
|
+
"than",
|
|
247
|
+
"that",
|
|
248
|
+
"the",
|
|
249
|
+
"their",
|
|
250
|
+
"theirs",
|
|
251
|
+
"them",
|
|
252
|
+
"then",
|
|
253
|
+
"there",
|
|
254
|
+
"these",
|
|
255
|
+
"they",
|
|
256
|
+
"this",
|
|
257
|
+
"those",
|
|
258
|
+
"through",
|
|
259
|
+
"to",
|
|
260
|
+
"too",
|
|
261
|
+
"under",
|
|
262
|
+
"until",
|
|
263
|
+
"up",
|
|
264
|
+
"very",
|
|
265
|
+
"was",
|
|
266
|
+
"we",
|
|
267
|
+
"were",
|
|
268
|
+
"what",
|
|
269
|
+
"when",
|
|
270
|
+
"where",
|
|
271
|
+
"which",
|
|
272
|
+
"while",
|
|
273
|
+
"who",
|
|
274
|
+
"whom",
|
|
275
|
+
"why",
|
|
276
|
+
"will",
|
|
277
|
+
"with",
|
|
278
|
+
"would",
|
|
279
|
+
"you",
|
|
280
|
+
"your",
|
|
281
|
+
"yours",
|
|
282
|
+
]
|
|
283
|
+
)
|
|
183
284
|
|
|
285
|
+
# A prefix query on a token this short is broader than the token itself
|
|
286
|
+
# ("in*" matches ingest/input/install/...), so short terms match exactly.
|
|
287
|
+
_MIN_PREFIX_LEN = 4
|
|
184
288
|
|
|
185
|
-
def _embeddings_enabled() -> bool:
|
|
186
|
-
"""Whether the embedding/vector pipeline is active.
|
|
187
289
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
``fastembed`` failed to import, so the server degrades to FTS-only
|
|
191
|
-
search instead of crashing at startup (issue #159).
|
|
192
|
-
"""
|
|
193
|
-
if TextEmbedding is None:
|
|
194
|
-
return False
|
|
195
|
-
flag = os.environ.get("PANTHEON_MEMORY_EMBED", "on").strip().lower()
|
|
196
|
-
return flag not in _EMBED_DISABLED
|
|
290
|
+
def _build_fts_query(query: str) -> str:
|
|
291
|
+
"""Build an FTS5 ``MATCH`` expression from free-text search input.
|
|
197
292
|
|
|
293
|
+
Replaces the previous ``" OR ".join(f'"{w}"*' for w in query.split())``
|
|
294
|
+
form, which prefix-matched *every* token, stopwords included. Measured on
|
|
295
|
+
the live store (1,553 memories) that form OR-ed in terms like ``the``,
|
|
296
|
+
``is`` and ``how`` whose prefixes match most of the corpus, so a query for
|
|
297
|
+
``purge threshold checkpoint`` pulled 1,001 documents as candidates and
|
|
298
|
+
1,524 of them were noise. Two corrections:
|
|
198
299
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
Uses BAAI/bge-small-en-v1.5 (384-dim, CPU, ONNX) — no PyTorch needed.
|
|
204
|
-
Model is cached in ~/.cache/fastembed/. Returns None when fastembed is
|
|
205
|
-
unavailable.
|
|
206
|
-
"""
|
|
207
|
-
if TextEmbedding is None:
|
|
208
|
-
return None
|
|
209
|
-
os.environ.setdefault("TQDM_DISABLE", "1")
|
|
210
|
-
return TextEmbedding(
|
|
211
|
-
model_name="BAAI/bge-small-en-v1.5",
|
|
212
|
-
cache_dir=str(EMBED_CACHE),
|
|
213
|
-
)
|
|
300
|
+
- Stopwords are dropped rather than OR-ed in.
|
|
301
|
+
- Tokens shorter than ``_MIN_PREFIX_LEN`` characters match exactly instead
|
|
302
|
+
of by prefix, which removes the 3-character prefix blowups (``"com"*``
|
|
303
|
+
alone matched 1,235 of 1,550 documents).
|
|
214
304
|
|
|
305
|
+
Tokens surviving both filters are still prefix-matched, so a query term
|
|
306
|
+
retrieves documents containing a longer word that starts with it. If every
|
|
307
|
+
token is a stopword the unfiltered tokens are matched exactly, so an
|
|
308
|
+
all-stopword query degrades to a literal lookup rather than to no results.
|
|
215
309
|
|
|
216
|
-
|
|
217
|
-
|
|
310
|
+
Args:
|
|
311
|
+
query: Raw, user-supplied search text.
|
|
218
312
|
|
|
219
|
-
Returns
|
|
220
|
-
|
|
313
|
+
Returns:
|
|
314
|
+
An FTS5 expression (terms joined with ``OR``), or ``""`` for an empty
|
|
315
|
+
query.
|
|
221
316
|
"""
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
if
|
|
226
|
-
|
|
227
|
-
|
|
317
|
+
words = [word for word in query.split() if word]
|
|
318
|
+
if not words:
|
|
319
|
+
return ""
|
|
320
|
+
content = [word for word in words if word.lower() not in _STOPWORDS]
|
|
321
|
+
if not content:
|
|
322
|
+
content = words
|
|
323
|
+
return " OR ".join(
|
|
324
|
+
f'"{word}"*' if len(word) >= _MIN_PREFIX_LEN else f'"{word}"'
|
|
325
|
+
for word in content
|
|
326
|
+
)
|
|
228
327
|
|
|
229
328
|
|
|
230
329
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
@@ -251,9 +350,7 @@ def _parse_metadata(row: dict[str, Any]) -> dict[str, Any]:
|
|
|
251
350
|
return row
|
|
252
351
|
|
|
253
352
|
|
|
254
|
-
# ──
|
|
255
|
-
|
|
256
|
-
_RRF_CONST = 60
|
|
353
|
+
# ── Ranking ───────────────────────────────────────────────────────────────────
|
|
257
354
|
|
|
258
355
|
|
|
259
356
|
def _parse_iso_ts(value: str) -> float:
|
|
@@ -271,9 +368,7 @@ def _parse_iso_ts(value: str) -> float:
|
|
|
271
368
|
return dt.timestamp()
|
|
272
369
|
|
|
273
370
|
|
|
274
|
-
def _fetch_created_at_map(
|
|
275
|
-
db: sqlite3.Connection, ids: list[int]
|
|
276
|
-
) -> dict[int, str]:
|
|
371
|
+
def _fetch_created_at_map(db: sqlite3.Connection, ids: list[int]) -> dict[int, str]:
|
|
277
372
|
"""Fetch created_at timestamps for the given memory IDs.
|
|
278
373
|
|
|
279
374
|
Args:
|
|
@@ -289,8 +384,7 @@ def _fetch_created_at_map(
|
|
|
289
384
|
try:
|
|
290
385
|
placeholders = ",".join("?" * len(ids))
|
|
291
386
|
rows = db.execute(
|
|
292
|
-
f"SELECT id, created_at FROM memories "
|
|
293
|
-
f"WHERE id IN ({placeholders})",
|
|
387
|
+
f"SELECT id, created_at FROM memories WHERE id IN ({placeholders})",
|
|
294
388
|
ids,
|
|
295
389
|
).fetchall()
|
|
296
390
|
created_at_map = {r["id"]: r["created_at"] for r in rows}
|
|
@@ -299,34 +393,33 @@ def _fetch_created_at_map(
|
|
|
299
393
|
return created_at_map
|
|
300
394
|
|
|
301
395
|
|
|
302
|
-
def
|
|
303
|
-
|
|
304
|
-
fts_ids: list[int],
|
|
396
|
+
def _score_hits(
|
|
397
|
+
hits: list[tuple[int, float]],
|
|
305
398
|
top_k: int,
|
|
306
399
|
created_at_map: dict[int, str] | None = None,
|
|
307
400
|
decay_days: float | None = None,
|
|
308
401
|
) -> list[tuple[int, float]]:
|
|
309
|
-
"""
|
|
402
|
+
"""Rank FTS5 hits by BM25 relevance, optionally freshness-decayed.
|
|
403
|
+
|
|
404
|
+
This replaces the vector+BM25 Reciprocal Rank Fusion: with a single ranked
|
|
405
|
+
list there is nothing left to fuse, so each hit keeps its own BM25 score
|
|
406
|
+
instead of a rank-derived one.
|
|
310
407
|
|
|
311
408
|
Args:
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
top_k: Maximum number of
|
|
409
|
+
hits: ``(id, bm25)`` pairs from FTS5, most relevant first. ``bm25`` is
|
|
410
|
+
``-rank`` (FTS5 reports ``rank`` negated, lower = better).
|
|
411
|
+
top_k: Maximum number of results to return.
|
|
315
412
|
created_at_map: Optional mapping of doc_id -> created_at ISO string,
|
|
316
413
|
used to compute freshness when ``decay_days`` is set.
|
|
317
|
-
decay_days: Optional freshness half-life in days. When set, each
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
compatible).
|
|
414
|
+
decay_days: Optional freshness half-life in days. When set, each score
|
|
415
|
+
is multiplied by ``2^(-days_since_created/decay_days)`` so older
|
|
416
|
+
entries rank lower. Default None = no decay.
|
|
321
417
|
|
|
322
418
|
Returns:
|
|
323
|
-
List of (id,
|
|
419
|
+
List of ``(id, score)`` tuples sorted by descending score, ties broken
|
|
420
|
+
by ascending id so ranking is deterministic.
|
|
324
421
|
"""
|
|
325
|
-
scores: dict[int, float] = {}
|
|
326
|
-
for rank, doc_id in enumerate(vec_ids, start=1):
|
|
327
|
-
scores[doc_id] = 1.0 / (_RRF_CONST + rank)
|
|
328
|
-
for rank, doc_id in enumerate(fts_ids, start=1):
|
|
329
|
-
scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (_RRF_CONST + rank)
|
|
422
|
+
scores: dict[int, float] = {doc_id: bm25 for doc_id, bm25 in hits}
|
|
330
423
|
|
|
331
424
|
if decay_days is not None and decay_days > 0 and created_at_map:
|
|
332
425
|
now = time.time()
|
|
@@ -346,46 +439,127 @@ def _rrf_fuse(
|
|
|
346
439
|
|
|
347
440
|
# ── Tools ─────────────────────────────────────────────────────────────────────
|
|
348
441
|
|
|
442
|
+
# ── Argument Validation ─────────────────────────────────────────────────────────
|
|
443
|
+
# One call must surface EVERY invalid argument. ``metadata`` is a JSON object
|
|
444
|
+
# encoded as a STRING, and the natural mistake is to pass the object itself,
|
|
445
|
+
# which used to surface as pydantic's bare "Input should be a valid string" —
|
|
446
|
+
# no path, no received type, no example. The validator below names all three,
|
|
447
|
+
# and pydantic still accumulates it alongside every other invalid argument in
|
|
448
|
+
# the same call.
|
|
449
|
+
|
|
450
|
+
METADATA_EXAMPLE: str = """metadata='{"type": "decision", "score": 0.9}'"""
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _json_type_name(value: object) -> str:
|
|
454
|
+
"""Name a value's JSON type for error messages (ints read as ``number``)."""
|
|
455
|
+
if value is None:
|
|
456
|
+
type_name = "null"
|
|
457
|
+
elif isinstance(value, bool):
|
|
458
|
+
type_name = "boolean"
|
|
459
|
+
elif isinstance(value, str):
|
|
460
|
+
type_name = "string"
|
|
461
|
+
elif isinstance(value, int | float):
|
|
462
|
+
type_name = "number"
|
|
463
|
+
elif isinstance(value, list):
|
|
464
|
+
type_name = "array"
|
|
465
|
+
elif isinstance(value, dict):
|
|
466
|
+
type_name = "object"
|
|
467
|
+
else:
|
|
468
|
+
type_name = type(value).__name__
|
|
469
|
+
return type_name
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _metadata_must_be_json_string(value: Any) -> Any:
|
|
473
|
+
"""Accept only a JSON object encoded as a string, with an actionable error.
|
|
474
|
+
|
|
475
|
+
Runs before pydantic's own ``str`` check so the failure names the received
|
|
476
|
+
type and a correct example instead of leaking a raw type error.
|
|
477
|
+
"""
|
|
478
|
+
if isinstance(value, str):
|
|
479
|
+
return value
|
|
480
|
+
raise ValueError(
|
|
481
|
+
"metadata must be a JSON object encoded as a string "
|
|
482
|
+
f"(got {_json_type_name(value)}). json.dumps it first. "
|
|
483
|
+
f"Example: {METADATA_EXAMPLE}"
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _collect_store_violations(value: str, metadata: str) -> tuple[list[str], list[str]]:
|
|
488
|
+
"""Collect every in-body ``memory_store`` violation, plus any examples.
|
|
489
|
+
|
|
490
|
+
Returns ``(violations, examples)`` so the caller reports all of them in one
|
|
491
|
+
message instead of returning on the first problem it happens to notice.
|
|
492
|
+
"""
|
|
493
|
+
violations: list[str] = []
|
|
494
|
+
examples: list[str] = []
|
|
495
|
+
if not value or not value.strip():
|
|
496
|
+
violations.append("value is required and must be a non-empty string")
|
|
497
|
+
if metadata and metadata != "{}":
|
|
498
|
+
try:
|
|
499
|
+
json.loads(metadata)
|
|
500
|
+
except (json.JSONDecodeError, TypeError, ValueError) as exc:
|
|
501
|
+
violations.append(
|
|
502
|
+
"metadata must be a JSON object encoded as a valid JSON string "
|
|
503
|
+
f"(got invalid JSON: {exc})"
|
|
504
|
+
)
|
|
505
|
+
examples.append(METADATA_EXAMPLE)
|
|
506
|
+
return violations, examples
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _format_store_violations(violations: list[str], examples: list[str]) -> str:
|
|
510
|
+
"""Render stored-error text: one line per violation, then one example block."""
|
|
511
|
+
lines = [f"memory_store: {violation}" for violation in violations]
|
|
512
|
+
if len(violations) > 1:
|
|
513
|
+
lines.insert(0, f"memory_store: {len(violations)} invalid arguments")
|
|
514
|
+
deduped = list(dict.fromkeys(examples))
|
|
515
|
+
if deduped:
|
|
516
|
+
lines.append("example: " + " ".join(deduped))
|
|
517
|
+
return "\n".join(lines)
|
|
518
|
+
|
|
349
519
|
|
|
350
520
|
@mcp.tool(
|
|
351
|
-
description="Store a memory entry
|
|
352
|
-
"Returns the entry ID and status."
|
|
521
|
+
description="Store a memory entry. The FTS5 index is updated automatically "
|
|
522
|
+
"by database trigger. Returns the entry ID and status. "
|
|
523
|
+
"metadata is a JSON object encoded as a STRING (not an object) and value "
|
|
524
|
+
"must be non-empty; every invalid argument is reported together in one "
|
|
525
|
+
f"message. Example: {METADATA_EXAMPLE}",
|
|
353
526
|
)
|
|
354
527
|
def memory_store(
|
|
355
528
|
value: str = "",
|
|
356
529
|
namespace: str = "default",
|
|
357
530
|
key: str | None = None,
|
|
358
|
-
metadata:
|
|
531
|
+
metadata: Annotated[
|
|
532
|
+
str,
|
|
533
|
+
BeforeValidator(_metadata_must_be_json_string),
|
|
534
|
+
Field(
|
|
535
|
+
default="{}",
|
|
536
|
+
description="JSON object encoded as a STRING, not an object. "
|
|
537
|
+
'Example: \'{"type": "decision", "score": 0.9}\'.',
|
|
538
|
+
),
|
|
539
|
+
] = "{}",
|
|
359
540
|
) -> dict[str, Any]:
|
|
360
|
-
"""Store a value
|
|
361
|
-
|
|
362
|
-
Generates a 384-dim embedding via fastembed (BAAI/bge-small-en-v1.5)
|
|
363
|
-
and stores it in the vec_memories virtual table alongside the text
|
|
364
|
-
content in the memories table.
|
|
541
|
+
"""Store a value and index it for FTS5 keyword search.
|
|
365
542
|
|
|
366
543
|
Args:
|
|
367
|
-
value: Text content to store.
|
|
544
|
+
value: Text content to store (required, non-empty).
|
|
368
545
|
namespace: Namespace for isolation (default: "default").
|
|
369
546
|
key: Optional unique key within namespace.
|
|
370
|
-
metadata: Optional JSON
|
|
547
|
+
metadata: Optional JSON object encoded as a string.
|
|
371
548
|
|
|
372
549
|
Returns:
|
|
373
550
|
Dict with id, namespace, key, status.
|
|
374
551
|
"""
|
|
375
|
-
|
|
376
|
-
|
|
552
|
+
violations, examples = _collect_store_violations(value, metadata)
|
|
553
|
+
if violations:
|
|
554
|
+
return {"error": _format_store_violations(violations, examples)}
|
|
377
555
|
|
|
378
556
|
value = value.strip()
|
|
379
557
|
db = _get_conn()
|
|
380
558
|
now = _now_iso()
|
|
381
559
|
|
|
382
|
-
|
|
383
|
-
# Validate metadata is valid JSON
|
|
384
|
-
md_obj: dict[str, Any] = (
|
|
385
|
-
json.loads(metadata) if metadata and metadata != "{}" else {}
|
|
386
|
-
)
|
|
387
|
-
metadata_str = json.dumps(md_obj)
|
|
560
|
+
metadata_str = metadata if metadata else "{}"
|
|
388
561
|
|
|
562
|
+
try:
|
|
389
563
|
cur = db.execute(
|
|
390
564
|
"""INSERT INTO memories (namespace, key, value, metadata,
|
|
391
565
|
created_at, updated_at)
|
|
@@ -393,15 +567,6 @@ def memory_store(
|
|
|
393
567
|
[namespace, key, value, metadata_str, now, now],
|
|
394
568
|
)
|
|
395
569
|
entry_id = cur.lastrowid
|
|
396
|
-
|
|
397
|
-
# Generate embedding (None when disabled → FTS-only mode)
|
|
398
|
-
embedding = _embed(value)
|
|
399
|
-
if embedding is not None and sqlite_vec is not None:
|
|
400
|
-
embedding_json = json.dumps(embedding)
|
|
401
|
-
db.execute(
|
|
402
|
-
"INSERT INTO vec_memories(id, embedding) VALUES (?, ?)",
|
|
403
|
-
[entry_id, embedding_json],
|
|
404
|
-
)
|
|
405
570
|
db.commit()
|
|
406
571
|
except sqlite3.IntegrityError as e:
|
|
407
572
|
db.rollback()
|
|
@@ -414,133 +579,101 @@ def memory_store(
|
|
|
414
579
|
|
|
415
580
|
|
|
416
581
|
@mcp.tool(
|
|
417
|
-
description="
|
|
418
|
-
"
|
|
419
|
-
"
|
|
582
|
+
description="Lexical keyword search across memories using SQLite FTS5 "
|
|
583
|
+
"BM25 ranking. Stopwords are ignored and terms of 4+ characters are "
|
|
584
|
+
"prefix-matched, so a query must share a token with the stored text — "
|
|
585
|
+
"there is no semantic/vector retrieval. Optional decay_days applies a "
|
|
420
586
|
"freshness half-life (2^(-days/decay_days)) so recent entries rank "
|
|
421
|
-
"higher; default None keeps the
|
|
587
|
+
"higher; default None keeps the raw BM25 order.",
|
|
422
588
|
)
|
|
423
|
-
def memory_search(
|
|
589
|
+
def memory_search(
|
|
424
590
|
query: str,
|
|
425
591
|
namespace: str | None = None,
|
|
426
592
|
top_k: int = 5,
|
|
427
593
|
decay_days: float | None = None,
|
|
428
594
|
) -> list[dict[str, Any]]:
|
|
429
|
-
"""
|
|
430
|
-
|
|
431
|
-
For short or keyword-heavy queries, FTS5 BM25 dominates.
|
|
432
|
-
For conceptual or long queries, vector cosine dominates.
|
|
433
|
-
RRF combines both into a single ranked list.
|
|
595
|
+
"""Search memories with FTS5 BM25.
|
|
434
596
|
|
|
435
597
|
Args:
|
|
436
598
|
query: Search query text.
|
|
437
599
|
namespace: Optional namespace filter.
|
|
438
600
|
top_k: Maximum results (default 5, max 50).
|
|
439
|
-
decay_days: Optional freshness half-life in days. When set,
|
|
601
|
+
decay_days: Optional freshness half-life in days. When set, BM25
|
|
440
602
|
scores are multiplied by ``2^(-days_since_created/decay_days)``
|
|
441
603
|
so older entries rank lower. Default None = no decay.
|
|
442
604
|
|
|
443
605
|
Returns:
|
|
444
|
-
List of memory entries with score
|
|
606
|
+
List of memory entries with a ``score`` (BM25, higher is better,
|
|
607
|
+
freshness-decayed when ``decay_days`` is set) and parsed metadata.
|
|
445
608
|
"""
|
|
446
609
|
if not query or not query.strip():
|
|
447
610
|
return []
|
|
448
611
|
|
|
449
612
|
top_k = max(1, min(50, int(top_k)))
|
|
450
613
|
db = _get_conn()
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
try:
|
|
455
|
-
query_vec = json.dumps(_embed(query_stripped))
|
|
456
|
-
except Exception:
|
|
457
|
-
query_vec = None
|
|
458
|
-
|
|
459
|
-
vec_ids: list[int] = []
|
|
460
|
-
fts_ids: list[int] = []
|
|
461
|
-
|
|
462
|
-
# 2. Vector search (skipped when embeddings or sqlite-vec are unavailable)
|
|
463
|
-
if query_vec is not None and sqlite_vec is not None:
|
|
464
|
-
nvec = top_k * 2
|
|
465
|
-
try:
|
|
466
|
-
if namespace:
|
|
467
|
-
vec_rows = db.execute(
|
|
468
|
-
"""SELECT v.id FROM vec_memories v
|
|
469
|
-
JOIN memories m ON m.id = v.id
|
|
470
|
-
WHERE m.namespace = ?
|
|
471
|
-
ORDER BY v.embedding MATCH ? LIMIT ?""",
|
|
472
|
-
[namespace, query_vec, nvec],
|
|
473
|
-
).fetchall()
|
|
474
|
-
else:
|
|
475
|
-
vec_rows = db.execute(
|
|
476
|
-
"""SELECT v.id FROM vec_memories v
|
|
477
|
-
ORDER BY v.embedding MATCH ? LIMIT ?""",
|
|
478
|
-
[query_vec, nvec],
|
|
479
|
-
).fetchall()
|
|
480
|
-
vec_ids = [r["id"] for r in vec_rows]
|
|
481
|
-
except Exception:
|
|
482
|
-
vec_ids = []
|
|
614
|
+
fts_query = _build_fts_query(query)
|
|
615
|
+
if not fts_query:
|
|
616
|
+
return []
|
|
483
617
|
|
|
484
|
-
# 3. FTS5 keyword search
|
|
485
618
|
try:
|
|
486
|
-
|
|
487
|
-
|
|
619
|
+
params: list[Any] = [fts_query]
|
|
620
|
+
namespace_clause = ""
|
|
488
621
|
if namespace:
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
622
|
+
namespace_clause = " AND namespace = ?"
|
|
623
|
+
params.append(namespace)
|
|
624
|
+
# Over-fetch 2x: freshness decay can promote a candidate that BM25
|
|
625
|
+
# ranked outside top_k, and ranking happens after the fetch.
|
|
626
|
+
params.append(top_k * 2)
|
|
627
|
+
# ``rank`` is BM25 negated by FTS5 (lower = better), so -rank is the
|
|
628
|
+
# relevance score. It is selected explicitly so the value is available
|
|
629
|
+
# for scoring rather than only for ordering.
|
|
630
|
+
rows = db.execute(
|
|
631
|
+
f"""SELECT rowid, -rank AS bm25 FROM memories_fts
|
|
632
|
+
WHERE memories_fts MATCH ?{namespace_clause}
|
|
633
|
+
ORDER BY rank LIMIT ?""",
|
|
634
|
+
params,
|
|
635
|
+
).fetchall()
|
|
636
|
+
hits = [(r["rowid"], r["bm25"]) for r in rows]
|
|
504
637
|
except Exception:
|
|
505
|
-
|
|
638
|
+
return []
|
|
639
|
+
|
|
640
|
+
if not hits:
|
|
641
|
+
return []
|
|
506
642
|
|
|
507
|
-
|
|
508
|
-
candidate_ids = list(dict.fromkeys([*vec_ids, *fts_ids]))
|
|
643
|
+
candidate_ids = [doc_id for doc_id, _ in hits]
|
|
509
644
|
created_at_map = (
|
|
510
645
|
_fetch_created_at_map(db, candidate_ids)
|
|
511
646
|
if decay_days is not None and decay_days > 0
|
|
512
647
|
else {}
|
|
513
648
|
)
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
if not fused:
|
|
517
|
-
return []
|
|
649
|
+
ranked = _score_hits(hits, top_k, created_at_map, decay_days)
|
|
518
650
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
651
|
+
results: list[dict[str, Any]] = []
|
|
652
|
+
if ranked:
|
|
653
|
+
# 5. Fetch full entries
|
|
654
|
+
id_list = [doc_id for doc_id, _ in ranked]
|
|
655
|
+
score_map = {doc_id: score for doc_id, score in ranked}
|
|
522
656
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
657
|
+
try:
|
|
658
|
+
placeholders = ",".join("?" * len(id_list))
|
|
659
|
+
rows = db.execute(
|
|
660
|
+
f"""SELECT id, namespace, key, value, metadata, created_at
|
|
661
|
+
FROM memories WHERE id IN ({placeholders})""",
|
|
662
|
+
id_list,
|
|
663
|
+
).fetchall()
|
|
664
|
+
except Exception:
|
|
665
|
+
return []
|
|
532
666
|
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
results.append(entry)
|
|
667
|
+
# Preserve ranking order
|
|
668
|
+
row_map = {r["id"]: r for r in rows}
|
|
669
|
+
for doc_id in id_list:
|
|
670
|
+
row = row_map.get(doc_id)
|
|
671
|
+
if row is None:
|
|
672
|
+
continue
|
|
673
|
+
entry = dict(row)
|
|
674
|
+
entry = _parse_metadata(entry)
|
|
675
|
+
entry["score"] = round(score_map.get(doc_id, 0.0), 4)
|
|
676
|
+
results.append(entry)
|
|
544
677
|
|
|
545
678
|
return results
|
|
546
679
|
|
|
@@ -581,18 +714,17 @@ def memory_recall(
|
|
|
581
714
|
|
|
582
715
|
@mcp.tool(
|
|
583
716
|
description="Delete a memory entry by ID or by key. "
|
|
584
|
-
"
|
|
585
|
-
"via CASCADE / triggers.",
|
|
717
|
+
"The FTS5 index entry is removed automatically by trigger.",
|
|
586
718
|
)
|
|
587
719
|
def memory_forget(
|
|
588
720
|
id: int | None = None,
|
|
589
721
|
key: str | None = None,
|
|
590
722
|
namespace: str = "default",
|
|
591
723
|
) -> dict[str, Any]:
|
|
592
|
-
"""Delete a memory entry and its
|
|
724
|
+
"""Delete a memory entry and its FTS5 index entry.
|
|
593
725
|
|
|
594
726
|
Provide either ``id`` (exact rowid) or ``key`` (within namespace)
|
|
595
|
-
to identify the entry.
|
|
727
|
+
to identify the entry. The FTS sync trigger handles index cleanup.
|
|
596
728
|
|
|
597
729
|
Args:
|
|
598
730
|
id: Exact row ID of the entry to delete.
|
|
@@ -617,9 +749,6 @@ def memory_forget(
|
|
|
617
749
|
db = _get_conn()
|
|
618
750
|
try:
|
|
619
751
|
if col == "id":
|
|
620
|
-
# Vec table cascade (absent when sqlite-vec is unavailable)
|
|
621
|
-
if sqlite_vec is not None:
|
|
622
|
-
db.execute("DELETE FROM vec_memories WHERE id = ?", [id])
|
|
623
752
|
cur = db.execute("DELETE FROM memories WHERE id = ?", [id])
|
|
624
753
|
else:
|
|
625
754
|
cur = db.execute(sql, params)
|
|
@@ -688,13 +817,12 @@ def memory_list(
|
|
|
688
817
|
|
|
689
818
|
@mcp.tool(
|
|
690
819
|
description="Memory database statistics: total entries, per-namespace "
|
|
691
|
-
"breakdown,
|
|
820
|
+
"breakdown, and file size on disk.",
|
|
692
821
|
)
|
|
693
822
|
def memory_stats() -> dict[str, Any]:
|
|
694
823
|
"""Return aggregate statistics about the memory database.
|
|
695
824
|
|
|
696
|
-
Includes total count, namespace breakdown,
|
|
697
|
-
vector entry count, and file size on disk.
|
|
825
|
+
Includes total count, namespace breakdown, and file size on disk.
|
|
698
826
|
|
|
699
827
|
Returns:
|
|
700
828
|
Dict with count, namespaces, and storage info.
|
|
@@ -721,13 +849,6 @@ def memory_stats() -> dict[str, Any]:
|
|
|
721
849
|
except Exception:
|
|
722
850
|
stats["namespaces"] = []
|
|
723
851
|
|
|
724
|
-
try:
|
|
725
|
-
stats["vector_entries"] = db.execute(
|
|
726
|
-
"SELECT COUNT(*) AS c FROM vec_memories"
|
|
727
|
-
).fetchone()["c"]
|
|
728
|
-
except Exception:
|
|
729
|
-
stats["vector_entries"] = 0
|
|
730
|
-
|
|
731
852
|
try:
|
|
732
853
|
db_size = DB_PATH.stat().st_size if DB_PATH.exists() else 0
|
|
733
854
|
stats["db_size_bytes"] = db_size
|
|
@@ -772,7 +893,9 @@ def code_index(
|
|
|
772
893
|
|
|
773
894
|
|
|
774
895
|
@mcp.tool(
|
|
775
|
-
description=
|
|
896
|
+
description=(
|
|
897
|
+
"Search code entities via FTS5 (with LIKE fallback) and optional type filter."
|
|
898
|
+
),
|
|
776
899
|
)
|
|
777
900
|
def code_query(
|
|
778
901
|
query: str,
|