@pi-unipi/memory 2.0.13 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -7
- package/bridge/mempalace_bridge.py +556 -0
- package/index.ts +7 -4
- package/mempalace.ts +260 -0
- package/package.json +8 -5
- package/storage.ts +253 -22
- package/tools.ts +0 -0
package/README.md
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# @pi-unipi/memory
|
|
2
2
|
|
|
3
|
-
Persistent memory that survives across sessions. Stores facts, preferences, and decisions
|
|
3
|
+
Persistent memory that survives across sessions. Stores facts, preferences, and decisions with semantic vector search, so the agent remembers what you told it last week.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
**Primary backend: [MemPalace](https://github.com/mempalace/mempalace)** — auto-installed via `uv` on first load, with one-way auto-migration of any existing legacy memories. If MemPalace or `uv` is unavailable, the package transparently falls back to the bundled SQLite + sqlite-vec store, so memory never hard-fails.
|
|
6
|
+
|
|
7
|
+
Two storage tiers: MemPalace (or SQLite) for vector similarity search, markdown files for a durable human-readable copy you can edit by hand. Project-scoped memories stay separate per codebase, global memories are accessible everywhere.
|
|
6
8
|
|
|
7
9
|
## Commands
|
|
8
10
|
|
|
@@ -72,19 +74,61 @@ Examples:
|
|
|
72
74
|
Memory has no configuration file. Storage paths are fixed:
|
|
73
75
|
|
|
74
76
|
```
|
|
75
|
-
~/.unipi/memory/
|
|
77
|
+
~/.unipi/memory/ # UniPi memory root (legacy + markdown tier)
|
|
78
|
+
├── .mempalace-install # Cached MemPalace venv detection
|
|
79
|
+
├── .mempalace-migrated # One-way migration completion flag
|
|
76
80
|
├── global/
|
|
77
|
-
│ ├── memory.db # Global vector DB
|
|
81
|
+
│ ├── memory.db # Global vector DB (SQLite fallback)
|
|
78
82
|
│ └── *.md # Global memory files
|
|
79
83
|
└── <project_name>/
|
|
80
|
-
├── memory.db # Project vector DB
|
|
84
|
+
├── memory.db # Project vector DB (SQLite fallback)
|
|
81
85
|
└── *.md # Project memory files
|
|
86
|
+
|
|
87
|
+
~/.mempalace/palace/ # MemPalace palace (primary backend)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## MemPalace backend
|
|
91
|
+
|
|
92
|
+
On first load, the memory package:
|
|
93
|
+
1. Detects MemPalace; if missing and `uv` is available, runs
|
|
94
|
+
`uv tool install mempalace` once (caches the venv python path in
|
|
95
|
+
`~/.unipi/memory/.mempalace-install`).
|
|
96
|
+
2. Pings the bridge to confirm the palace is usable.
|
|
97
|
+
3. If `~/.unipi/memory/.mempalace-migrated` is absent, performs a one-way
|
|
98
|
+
read-only migration of every legacy memory (SQLite rows + markdown files
|
|
99
|
+
across all projects) into MemPalace drawers, then writes the flag.
|
|
100
|
+
Migration is idempotent (deterministic drawer IDs) and never deletes or
|
|
101
|
+
mutates legacy files.
|
|
102
|
+
|
|
103
|
+
Each memory operation invokes a bundled Python bridge
|
|
104
|
+
(`bridge/mempalace_bridge.py`) once via `spawnSync` (~0.5s per call). The
|
|
105
|
+
first MemPalace use on a machine also downloads the default ONNX embedding
|
|
106
|
+
model (~80MB, cached at `~/.cache/chroma/onnx_models/`).
|
|
107
|
+
|
|
108
|
+
### Forcing re-detection / re-migration
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
rm ~/.unipi/memory/.mempalace-install # re-detect MemPalace next session
|
|
112
|
+
rm ~/.unipi/memory/.mempalace-migrated # re-run one-way migration next session
|
|
82
113
|
```
|
|
83
114
|
|
|
115
|
+
### Backend override
|
|
116
|
+
|
|
117
|
+
Set `UNIPI_MEMPALACE_BACKEND` to force a MemPalace backend
|
|
118
|
+
(`sqlite_exact`, `qdrant`, `pgvector`, default `chroma`).
|
|
119
|
+
|
|
120
|
+
### Embedder identity
|
|
121
|
+
|
|
122
|
+
MemPalace enforces embedder identity. If a palace was created with a
|
|
123
|
+
different embedding model, writes are rejected — the package then falls
|
|
124
|
+
back to SQLite for that session. Use `mempalace palace set-embedder`
|
|
125
|
+
intentionally to realign, then remove the install cache to re-detect.
|
|
126
|
+
|
|
84
127
|
## Dependencies
|
|
85
128
|
|
|
86
|
-
- `
|
|
87
|
-
- `
|
|
129
|
+
- `mempalace` (Python, auto-installed via `uv`) — primary backend
|
|
130
|
+
- `better-sqlite3` — SQLite fallback database
|
|
131
|
+
- `sqlite-vec` — Vector search extension (fallback)
|
|
88
132
|
- `js-yaml` — YAML frontmatter parsing
|
|
89
133
|
- `@pi-unipi/core` — Shared utilities
|
|
90
134
|
|
|
@@ -0,0 +1,556 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""UniPi memory <-> MemPalace bridge.
|
|
3
|
+
|
|
4
|
+
Single-command JSON bridge invoked once per operation by the TypeScript
|
|
5
|
+
memory package. Loads MemPalace, opens the palace collection, performs one
|
|
6
|
+
command, prints one JSON line to stdout, exits.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
python mempalace_bridge.py <palace_path> <command> [args_json]
|
|
10
|
+
|
|
11
|
+
stdout (always exactly one JSON line):
|
|
12
|
+
{"ok": true, "result": <value>}
|
|
13
|
+
{"ok": false, "error": "<message>"}
|
|
14
|
+
|
|
15
|
+
Design notes:
|
|
16
|
+
- MemPalace embeds drawer text internally (default ONNX MiniLM model). UniPi
|
|
17
|
+
never passes embeddings through this bridge; search uses query_texts.
|
|
18
|
+
- Drawer documents are markdown with YAML frontmatter so the human-readable
|
|
19
|
+
tier is preserved and the bridge can recover title/type without metadata.
|
|
20
|
+
- Drawer IDs are deterministic via make_drawer_id_from_chunk(wing, room,
|
|
21
|
+
source_uri, 0), so upserts are idempotent and re-migration does not
|
|
22
|
+
duplicate records.
|
|
23
|
+
- Metadata is kept scalar/string-only for backend portability (ChromaDB,
|
|
24
|
+
sqlite_exact, qdrant, pgvector).
|
|
25
|
+
- This script never deletes or mutates UniPi legacy files; migration is a
|
|
26
|
+
read-only copy into MemPalace.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import hashlib
|
|
32
|
+
import json
|
|
33
|
+
import os
|
|
34
|
+
import re
|
|
35
|
+
import sqlite3
|
|
36
|
+
import sys
|
|
37
|
+
from datetime import datetime, timezone
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Any, Iterable
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
import yaml # type: ignore
|
|
43
|
+
except Exception: # pragma: no cover
|
|
44
|
+
yaml = None
|
|
45
|
+
|
|
46
|
+
MEMORY_TYPES = {"preference", "decision", "pattern", "summary"}
|
|
47
|
+
MIGRATION_AGENT = "unipi-memory-bridge"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
# ID + URI helpers (mirror mempalace.ids when available, else deterministic fallback)
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
def quote_uri_part(value: str) -> str:
|
|
55
|
+
return re.sub(r"[^A-Za-z0-9_.~-]", lambda m: f"%{ord(m.group(0)):02X}", value)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def safe_id_part(value: str) -> str:
|
|
59
|
+
cleaned = re.sub(r"[^A-Za-z0-9]+", "_", value).strip("_").lower()
|
|
60
|
+
return cleaned or "unknown"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def stable_hash(parts: Iterable[Any], length: int = 24) -> str:
|
|
64
|
+
payload = "".join(f"{len(str(part))}:{part}" for part in parts).encode("utf-8")
|
|
65
|
+
return hashlib.sha256(payload).hexdigest()[:length]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def fallback_drawer_id(wing: str, room: str, source_file: str, chunk_index: int = 0) -> str:
|
|
69
|
+
return (
|
|
70
|
+
f"drawer_{safe_id_part(wing)}_{safe_id_part(room)}_"
|
|
71
|
+
f"{stable_hash((source_file, str(chunk_index)))}"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def drawer_id_for(wing: str, room: str, source_uri: str, chunk_index: int = 0) -> str:
|
|
76
|
+
try:
|
|
77
|
+
from mempalace.ids import make_drawer_id_from_chunk # type: ignore
|
|
78
|
+
return make_drawer_id_from_chunk(wing, room, source_uri, chunk_index)
|
|
79
|
+
except Exception:
|
|
80
|
+
return fallback_drawer_id(wing, room, source_uri, chunk_index)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
# Frontmatter (YAML when available, minimal fallback otherwise)
|
|
85
|
+
# ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
def dump_frontmatter(data: dict[str, Any]) -> str:
|
|
88
|
+
if yaml is not None:
|
|
89
|
+
return yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=10_000)
|
|
90
|
+
lines: list[str] = []
|
|
91
|
+
for key, value in data.items():
|
|
92
|
+
if isinstance(value, list):
|
|
93
|
+
lines.append(f"{key}:")
|
|
94
|
+
lines.extend(f" - {item}" for item in value)
|
|
95
|
+
else:
|
|
96
|
+
lines.append(f"{key}: {value}")
|
|
97
|
+
return "\n".join(lines) + "\n"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str] | None:
|
|
101
|
+
if not text.startswith("---\n"):
|
|
102
|
+
return None
|
|
103
|
+
end = text.find("\n---", 4)
|
|
104
|
+
if end == -1:
|
|
105
|
+
return None
|
|
106
|
+
raw_fm = text[4:end]
|
|
107
|
+
body_start = end + len("\n---")
|
|
108
|
+
if text[body_start : body_start + 1] == "\n":
|
|
109
|
+
body_start += 1
|
|
110
|
+
body = text[body_start:]
|
|
111
|
+
if yaml is not None:
|
|
112
|
+
loaded = yaml.safe_load(raw_fm) or {}
|
|
113
|
+
if not isinstance(loaded, dict):
|
|
114
|
+
return None
|
|
115
|
+
return loaded, body
|
|
116
|
+
# minimal fallback parser
|
|
117
|
+
data: dict[str, Any] = {}
|
|
118
|
+
current_list_key: str | None = None
|
|
119
|
+
for raw_line in raw_fm.splitlines():
|
|
120
|
+
line = raw_line.rstrip()
|
|
121
|
+
if not line:
|
|
122
|
+
continue
|
|
123
|
+
if current_list_key and line.startswith(" - "):
|
|
124
|
+
data.setdefault(current_list_key, []).append(line[4:].strip().strip("'\""))
|
|
125
|
+
continue
|
|
126
|
+
current_list_key = None
|
|
127
|
+
if ":" not in line:
|
|
128
|
+
continue
|
|
129
|
+
key, value = line.split(":", 1)
|
|
130
|
+
key = key.strip()
|
|
131
|
+
value = value.strip()
|
|
132
|
+
if value == "":
|
|
133
|
+
data[key] = []
|
|
134
|
+
current_list_key = key
|
|
135
|
+
elif value.startswith("[") and value.endswith("]"):
|
|
136
|
+
data[key] = [v.strip().strip("'\"") for v in value[1:-1].split(",") if v.strip()]
|
|
137
|
+
else:
|
|
138
|
+
data[key] = value.strip("'\"")
|
|
139
|
+
return data, body
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def coerce_tags(value: Any) -> list[str]:
|
|
143
|
+
if value is None:
|
|
144
|
+
return []
|
|
145
|
+
if isinstance(value, list):
|
|
146
|
+
return [str(v) for v in value if str(v).strip()]
|
|
147
|
+
if isinstance(value, str):
|
|
148
|
+
stripped = value.strip()
|
|
149
|
+
if not stripped:
|
|
150
|
+
return []
|
|
151
|
+
try:
|
|
152
|
+
parsed = json.loads(stripped)
|
|
153
|
+
if isinstance(parsed, list):
|
|
154
|
+
return [str(v) for v in parsed if str(v).strip()]
|
|
155
|
+
except Exception:
|
|
156
|
+
pass
|
|
157
|
+
return [part.strip() for part in stripped.split(",") if part.strip()]
|
|
158
|
+
return [str(value)]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def normalize_type(value: Any) -> str:
|
|
162
|
+
lowered = str(value or "summary").strip().lower()
|
|
163
|
+
return lowered if lowered in MEMORY_TYPES else "summary"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def build_document(title: str, content: str, tags: list[str], project: str,
|
|
167
|
+
created: str, updated: str, mtype: str, unipi_id: str) -> str:
|
|
168
|
+
fm = {
|
|
169
|
+
"title": title,
|
|
170
|
+
"tags": tags,
|
|
171
|
+
"project": project,
|
|
172
|
+
"created": created,
|
|
173
|
+
"updated": updated,
|
|
174
|
+
"type": mtype,
|
|
175
|
+
"unipi_id": unipi_id,
|
|
176
|
+
}
|
|
177
|
+
fm = {k: v for k, v in fm.items() if v not in (None, [], "")}
|
|
178
|
+
return f"---\n{dump_frontmatter(fm)}---\n\n{content.strip()}\n"
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def build_metadata(wing: str, room: str, title: str, mtype: str, project: str,
|
|
182
|
+
tags: list[str], unipi_id: str, source_kind: str, now: str,
|
|
183
|
+
content_date: str = "") -> dict[str, Any]:
|
|
184
|
+
return {
|
|
185
|
+
"wing": wing,
|
|
186
|
+
"room": room,
|
|
187
|
+
"source_file": f"unipi://memory/{quote_uri_part(project)}/{quote_uri_part(unipi_id)}",
|
|
188
|
+
"chunk_index": 0,
|
|
189
|
+
"added_by": MIGRATION_AGENT,
|
|
190
|
+
"filed_at": now,
|
|
191
|
+
"content_date": content_date,
|
|
192
|
+
"unipi_project": project,
|
|
193
|
+
"unipi_id": unipi_id,
|
|
194
|
+
"unipi_title": title,
|
|
195
|
+
"unipi_type": mtype,
|
|
196
|
+
"unipi_tags": ",".join(tags),
|
|
197
|
+
"unipi_source_kind": source_kind,
|
|
198
|
+
"normalize_version": 2,
|
|
199
|
+
"id_recipe": "unipi-bridge-v1",
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def record_from_doc(doc: str, meta: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
204
|
+
"""Recover a UniPi-style record from a drawer document + metadata."""
|
|
205
|
+
parsed = parse_frontmatter(doc) if doc else None
|
|
206
|
+
fm: dict[str, Any] = {}
|
|
207
|
+
body = doc or ""
|
|
208
|
+
if parsed:
|
|
209
|
+
fm, body = parsed
|
|
210
|
+
meta = meta or {}
|
|
211
|
+
unipi_id = fm.get("unipi_id") or meta.get("unipi_id") or ""
|
|
212
|
+
if not unipi_id:
|
|
213
|
+
# fall back to drawer id if metadata lost
|
|
214
|
+
return None
|
|
215
|
+
title = str(fm.get("title") or meta.get("unipi_title") or unipi_id)
|
|
216
|
+
return {
|
|
217
|
+
"id": unipi_id,
|
|
218
|
+
"title": title,
|
|
219
|
+
"content": body.strip(),
|
|
220
|
+
"tags": coerce_tags(fm.get("tags") if "tags" in fm else meta.get("unipi_tags")),
|
|
221
|
+
"project": str(fm.get("project") or meta.get("unipi_project") or meta.get("wing") or ""),
|
|
222
|
+
"type": normalize_type(fm.get("type") or meta.get("unipi_type")),
|
|
223
|
+
"created": str(fm.get("created") or ""),
|
|
224
|
+
"updated": str(fm.get("updated") or ""),
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def snippet(text: str, length: int = 200) -> str:
|
|
229
|
+
text = (text or "").strip().replace("\n", " ")
|
|
230
|
+
return text[:length] + ("..." if len(text) > length else "")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ---------------------------------------------------------------------------
|
|
234
|
+
# Legacy UniPi source reading (read-only)
|
|
235
|
+
# ---------------------------------------------------------------------------
|
|
236
|
+
|
|
237
|
+
def parse_markdown_memory(project: str, path: Path) -> dict[str, Any] | None:
|
|
238
|
+
try:
|
|
239
|
+
text = path.read_text(encoding="utf-8")
|
|
240
|
+
except UnicodeDecodeError:
|
|
241
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
242
|
+
except OSError:
|
|
243
|
+
return None
|
|
244
|
+
parsed = parse_frontmatter(text)
|
|
245
|
+
if parsed is None:
|
|
246
|
+
return None
|
|
247
|
+
fm, body = parsed
|
|
248
|
+
mid = safe_id_part(str(fm.get("id") or path.stem))
|
|
249
|
+
return {
|
|
250
|
+
"project": str(fm.get("project") or project),
|
|
251
|
+
"id": mid,
|
|
252
|
+
"title": str(fm.get("title") or path.stem).strip(),
|
|
253
|
+
"content": body.strip(),
|
|
254
|
+
"tags": coerce_tags(fm.get("tags")),
|
|
255
|
+
"type": normalize_type(fm.get("type")),
|
|
256
|
+
"created": str(fm.get("created")) if fm.get("created") else "",
|
|
257
|
+
"updated": str(fm.get("updated")) if fm.get("updated") else "",
|
|
258
|
+
"source_kind": "markdown",
|
|
259
|
+
"source_path": str(path),
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def load_sqlite_memories(project: str, db_path: Path) -> list[dict[str, Any]]:
|
|
264
|
+
if not db_path.exists():
|
|
265
|
+
return []
|
|
266
|
+
uri = f"file:{db_path}?mode=ro"
|
|
267
|
+
try:
|
|
268
|
+
conn = sqlite3.connect(uri, uri=True, timeout=10)
|
|
269
|
+
conn.row_factory = sqlite3.Row
|
|
270
|
+
try:
|
|
271
|
+
rows = list(conn.execute("SELECT * FROM memories"))
|
|
272
|
+
finally:
|
|
273
|
+
conn.close()
|
|
274
|
+
except sqlite3.DatabaseError:
|
|
275
|
+
return []
|
|
276
|
+
out: list[dict[str, Any]] = []
|
|
277
|
+
for row in rows:
|
|
278
|
+
out.append({
|
|
279
|
+
"project": str(row["project"] or project),
|
|
280
|
+
"id": safe_id_part(str(row["id"] or row["title"])),
|
|
281
|
+
"title": str(row["title"] or row["id"]),
|
|
282
|
+
"content": str(row["content"] or "").strip(),
|
|
283
|
+
"tags": coerce_tags(row["tags"]),
|
|
284
|
+
"type": normalize_type(row["type"]),
|
|
285
|
+
"created": str(row["created"]) if row["created"] else "",
|
|
286
|
+
"updated": str(row["updated"]) if row["updated"] else "",
|
|
287
|
+
"source_kind": "sqlite",
|
|
288
|
+
"source_path": str(db_path),
|
|
289
|
+
})
|
|
290
|
+
return out
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def discover_legacy_memories(source_dir: Path, project_filter: list[str] | None = None) -> list[dict[str, Any]]:
|
|
294
|
+
if not source_dir.exists():
|
|
295
|
+
return []
|
|
296
|
+
filters = set(project_filter or [])
|
|
297
|
+
by_key: dict[tuple[str, str], dict[str, Any]] = {}
|
|
298
|
+
for project_dir in sorted(p for p in source_dir.iterdir() if p.is_dir()):
|
|
299
|
+
project = project_dir.name
|
|
300
|
+
if filters and project not in filters:
|
|
301
|
+
continue
|
|
302
|
+
for md_path in sorted(project_dir.glob("*.md")):
|
|
303
|
+
if md_path.name.startswith("."):
|
|
304
|
+
continue
|
|
305
|
+
rec = parse_markdown_memory(project, md_path)
|
|
306
|
+
if rec:
|
|
307
|
+
by_key[(rec["project"], rec["id"])] = rec
|
|
308
|
+
for rec in load_sqlite_memories(project, project_dir / "memory.db"):
|
|
309
|
+
by_key.setdefault((rec["project"], rec["id"]), rec)
|
|
310
|
+
return sorted(by_key.values(), key=lambda r: (r["project"], r["type"], r["title"], r["id"]))
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
# ---------------------------------------------------------------------------
|
|
314
|
+
# Bridge session
|
|
315
|
+
# ---------------------------------------------------------------------------
|
|
316
|
+
|
|
317
|
+
class Bridge:
|
|
318
|
+
def __init__(self, palace_path: str, backend: str | None = None):
|
|
319
|
+
from mempalace.palace import get_collection # type: ignore
|
|
320
|
+
kwargs: dict[str, Any] = {"create": True}
|
|
321
|
+
if backend:
|
|
322
|
+
kwargs["backend"] = backend
|
|
323
|
+
self.palace_path = palace_path
|
|
324
|
+
self.collection = get_collection(palace_path, **kwargs)
|
|
325
|
+
|
|
326
|
+
def _now(self) -> str:
|
|
327
|
+
return datetime.now(timezone.utc).isoformat()
|
|
328
|
+
|
|
329
|
+
def _wing_room(self, project: str, mtype: str) -> tuple[str, str]:
|
|
330
|
+
return project, f"unipi_{normalize_type(mtype)}"
|
|
331
|
+
|
|
332
|
+
def _source_uri(self, project: str, unipi_id: str) -> str:
|
|
333
|
+
return f"unipi://memory/{quote_uri_part(project)}/{quote_uri_part(unipi_id)}"
|
|
334
|
+
|
|
335
|
+
def _upsert_one(self, rec: dict[str, Any], wing: str | None = None, room: str | None = None) -> str:
|
|
336
|
+
wing = wing or rec["project"]
|
|
337
|
+
room = room or f"unipi_{normalize_type(rec['type'])}"
|
|
338
|
+
source_uri = self._source_uri(rec["project"], rec["id"])
|
|
339
|
+
did = drawer_id_for(wing, room, source_uri, 0)
|
|
340
|
+
doc = build_document(
|
|
341
|
+
rec["title"], rec["content"], rec["tags"], rec["project"],
|
|
342
|
+
rec.get("created", ""), rec.get("updated", ""), rec["type"], rec["id"],
|
|
343
|
+
)
|
|
344
|
+
meta = build_metadata(
|
|
345
|
+
wing, room, rec["title"], rec["type"], rec["project"], rec["tags"],
|
|
346
|
+
rec["id"], rec.get("source_kind", "markdown"), self._now(),
|
|
347
|
+
rec.get("updated") or rec.get("created") or "",
|
|
348
|
+
)
|
|
349
|
+
self.collection.upsert(documents=[doc], ids=[did], metadatas=[meta])
|
|
350
|
+
return did
|
|
351
|
+
|
|
352
|
+
# -- commands --
|
|
353
|
+
|
|
354
|
+
def ping(self) -> str:
|
|
355
|
+
return "pong"
|
|
356
|
+
|
|
357
|
+
def count(self, wing: str | None = None) -> int:
|
|
358
|
+
if wing:
|
|
359
|
+
got = self.collection.get(where={"wing": wing})
|
|
360
|
+
else:
|
|
361
|
+
got = self.collection.get()
|
|
362
|
+
return len(got.ids if hasattr(got, "ids") else got.get("ids", []))
|
|
363
|
+
|
|
364
|
+
def store(self, record: dict[str, Any]) -> dict[str, Any]:
|
|
365
|
+
did = self._upsert_one(record)
|
|
366
|
+
return {"id": record["id"], "drawer_id": did}
|
|
367
|
+
|
|
368
|
+
def get(self, id_: str) -> dict[str, Any] | None:
|
|
369
|
+
# IDs in metadata are unipi ids; fetch all and filter (drawer ids differ).
|
|
370
|
+
got = self.collection.get(where={"unipi_id": id_})
|
|
371
|
+
ids = got.ids if hasattr(got, "ids") else got.get("ids", [])
|
|
372
|
+
docs = got.documents if hasattr(got, "documents") else got.get("documents", [])
|
|
373
|
+
metas = got.metadatas if hasattr(got, "metadatas") else got.get("metadatas", [])
|
|
374
|
+
if not ids:
|
|
375
|
+
return None
|
|
376
|
+
return record_from_doc(docs[0], metas[0] if metas else None)
|
|
377
|
+
|
|
378
|
+
def get_by_title(self, wing: str, title: str) -> dict[str, Any] | None:
|
|
379
|
+
got = self.collection.get(where={"wing": wing})
|
|
380
|
+
docs = got.documents if hasattr(got, "documents") else got.get("documents", [])
|
|
381
|
+
metas = got.metadatas if hasattr(got, "metadatas") else got.get("metadatas", [])
|
|
382
|
+
lowered = title.lower()
|
|
383
|
+
for doc, meta in zip(docs, metas):
|
|
384
|
+
rec = record_from_doc(doc, meta)
|
|
385
|
+
if not rec:
|
|
386
|
+
continue
|
|
387
|
+
if rec["title"] == title or rec["title"].lower() == lowered:
|
|
388
|
+
return rec
|
|
389
|
+
return None
|
|
390
|
+
|
|
391
|
+
def list(self, wing: str) -> list[dict[str, Any]]:
|
|
392
|
+
got = self.collection.get(where={"wing": wing})
|
|
393
|
+
docs = got.documents if hasattr(got, "documents") else got.get("documents", [])
|
|
394
|
+
metas = got.metadatas if hasattr(got, "metadatas") else got.get("metadatas", [])
|
|
395
|
+
out: list[dict[str, Any]] = []
|
|
396
|
+
for doc, meta in zip(docs, metas):
|
|
397
|
+
rec = record_from_doc(doc, meta)
|
|
398
|
+
if rec:
|
|
399
|
+
out.append({"id": rec["id"], "title": rec["title"], "type": rec["type"]})
|
|
400
|
+
# sort by updated desc (best effort — metadata has no guaranteed order)
|
|
401
|
+
return out
|
|
402
|
+
|
|
403
|
+
def list_all(self) -> list[dict[str, Any]]:
|
|
404
|
+
got = self.collection.get()
|
|
405
|
+
docs = got.documents if hasattr(got, "documents") else got.get("documents", [])
|
|
406
|
+
metas = got.metadatas if hasattr(got, "metadatas") else got.get("metadatas", [])
|
|
407
|
+
out: list[dict[str, Any]] = []
|
|
408
|
+
for doc, meta in zip(docs, metas):
|
|
409
|
+
rec = record_from_doc(doc, meta)
|
|
410
|
+
if rec:
|
|
411
|
+
out.append({"id": rec["id"], "title": rec["title"], "type": rec["type"],
|
|
412
|
+
"project": rec["project"]})
|
|
413
|
+
return out
|
|
414
|
+
|
|
415
|
+
def search(self, query: str, wing: str | None = None, limit: int = 10) -> list[dict[str, Any]]:
|
|
416
|
+
where = {"wing": wing} if wing else None
|
|
417
|
+
kwargs: dict[str, Any] = {"query_texts": [query], "n_results": limit}
|
|
418
|
+
if where:
|
|
419
|
+
kwargs["where"] = where
|
|
420
|
+
res = self.collection.query(**kwargs)
|
|
421
|
+
ids = res.ids[0] if res.ids else []
|
|
422
|
+
docs = res.documents[0] if res.documents else []
|
|
423
|
+
metas = res.metadatas[0] if res.metadatas else []
|
|
424
|
+
dists = res.distances[0] if res.distances else []
|
|
425
|
+
out: list[dict[str, Any]] = []
|
|
426
|
+
for doc, meta, dist in zip(docs, metas, dists):
|
|
427
|
+
rec = record_from_doc(doc, meta)
|
|
428
|
+
if not rec:
|
|
429
|
+
continue
|
|
430
|
+
score = max(0.0, 1.0 - float(dist))
|
|
431
|
+
out.append({
|
|
432
|
+
"id": rec["id"], "title": rec["title"], "content": rec["content"],
|
|
433
|
+
"tags": rec["tags"], "project": rec["project"], "type": rec["type"],
|
|
434
|
+
"created": rec["created"], "updated": rec["updated"],
|
|
435
|
+
"score": round(score, 4), "snippet": snippet(rec["content"]),
|
|
436
|
+
})
|
|
437
|
+
return out
|
|
438
|
+
|
|
439
|
+
def delete(self, id_: str) -> bool:
|
|
440
|
+
got = self.collection.get(where={"unipi_id": id_})
|
|
441
|
+
ids = got.ids if hasattr(got, "ids") else got.get("ids", [])
|
|
442
|
+
if not ids:
|
|
443
|
+
return False
|
|
444
|
+
self.collection.delete(ids=ids)
|
|
445
|
+
return True
|
|
446
|
+
|
|
447
|
+
def has_title(self, wing: str, title: str) -> bool:
|
|
448
|
+
return self.get_by_title(wing, title) is not None
|
|
449
|
+
|
|
450
|
+
def find_similar(self, wing: str, title: str, threshold: float = 0.6) -> list[dict[str, Any]]:
|
|
451
|
+
got = self.collection.get(where={"wing": wing})
|
|
452
|
+
docs = got.documents if hasattr(got, "documents") else got.get("documents", [])
|
|
453
|
+
metas = got.metadatas if hasattr(got, "metadatas") else got.get("metadatas", [])
|
|
454
|
+
norm = re.sub(r"[^a-z0-9]+", " ", title.lower())
|
|
455
|
+
title_words = set(w for w in norm.split() if len(w) > 2)
|
|
456
|
+
out: list[dict[str, Any]] = []
|
|
457
|
+
for doc, meta in zip(docs, metas):
|
|
458
|
+
rec = record_from_doc(doc, meta)
|
|
459
|
+
if not rec:
|
|
460
|
+
continue
|
|
461
|
+
rnorm = re.sub(r"[^a-z0-9]+", " ", rec["title"].lower())
|
|
462
|
+
rwords = set(w for w in re.split(r"\s+", rnorm) if len(w) > 2)
|
|
463
|
+
union = title_words | rwords
|
|
464
|
+
inter = title_words & rwords
|
|
465
|
+
sim = len(inter) / len(union) if union else 0.0
|
|
466
|
+
if sim >= threshold:
|
|
467
|
+
out.append({"record": rec, "similarity": round(sim, 4)})
|
|
468
|
+
out.sort(key=lambda x: x["similarity"], reverse=True)
|
|
469
|
+
return out
|
|
470
|
+
|
|
471
|
+
def sync_orphaned(self, project_dir: str, wing: str) -> int:
|
|
472
|
+
pdir = Path(project_dir)
|
|
473
|
+
if not pdir.exists():
|
|
474
|
+
return 0
|
|
475
|
+
existing = {r["id"] for r in self.list(wing)}
|
|
476
|
+
synced = 0
|
|
477
|
+
for md_path in sorted(pdir.glob("*.md")):
|
|
478
|
+
if md_path.name.startswith("."):
|
|
479
|
+
continue
|
|
480
|
+
rec = parse_markdown_memory(wing, md_path)
|
|
481
|
+
if not rec:
|
|
482
|
+
continue
|
|
483
|
+
if rec["id"] in existing:
|
|
484
|
+
continue
|
|
485
|
+
self._upsert_one(rec)
|
|
486
|
+
synced += 1
|
|
487
|
+
return synced
|
|
488
|
+
|
|
489
|
+
def migrate(self, source_dir: str, project_filter: list[str] | None = None) -> dict[str, Any]:
|
|
490
|
+
records = discover_legacy_memories(Path(source_dir), project_filter)
|
|
491
|
+
imported = 0
|
|
492
|
+
by_project: dict[str, int] = {}
|
|
493
|
+
for rec in records:
|
|
494
|
+
try:
|
|
495
|
+
self._upsert_one(rec)
|
|
496
|
+
imported += 1
|
|
497
|
+
by_project[rec["project"]] = by_project.get(rec["project"], 0) + 1
|
|
498
|
+
except Exception:
|
|
499
|
+
pass
|
|
500
|
+
return {"imported": imported, "projects": by_project}
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
# ---------------------------------------------------------------------------
|
|
504
|
+
# Entry point
|
|
505
|
+
# ---------------------------------------------------------------------------
|
|
506
|
+
|
|
507
|
+
def main(argv: list[str]) -> int:
|
|
508
|
+
if len(argv) < 3:
|
|
509
|
+
print(json.dumps({"ok": False, "error": "usage: bridge.py <palace> <command> [args_json]"}))
|
|
510
|
+
return 2
|
|
511
|
+
palace = argv[1]
|
|
512
|
+
cmd = argv[2]
|
|
513
|
+
args_raw = argv[3] if len(argv) > 3 else "{}"
|
|
514
|
+
try:
|
|
515
|
+
args = json.loads(args_raw) if args_raw else {}
|
|
516
|
+
except json.JSONDecodeError as exc:
|
|
517
|
+
print(json.dumps({"ok": False, "error": f"invalid args json: {exc}"}))
|
|
518
|
+
return 2
|
|
519
|
+
|
|
520
|
+
backend = os.environ.get("UNIPI_MEMPALACE_BACKEND") or None
|
|
521
|
+
try:
|
|
522
|
+
bridge = Bridge(palace, backend=backend)
|
|
523
|
+
except Exception as exc:
|
|
524
|
+
print(json.dumps({"ok": False, "error": f"mempalace init failed: {type(exc).__name__}: {exc}"}))
|
|
525
|
+
return 1
|
|
526
|
+
|
|
527
|
+
handlers = {
|
|
528
|
+
"ping": lambda: bridge.ping(),
|
|
529
|
+
"count": lambda: bridge.count(args.get("wing")),
|
|
530
|
+
"store": lambda: bridge.store(args["record"]),
|
|
531
|
+
"get": lambda: bridge.get(args["id"]),
|
|
532
|
+
"get_by_title": lambda: bridge.get_by_title(args["wing"], args["title"]),
|
|
533
|
+
"list": lambda: bridge.list(args["wing"]),
|
|
534
|
+
"list_all": lambda: bridge.list_all(),
|
|
535
|
+
"search": lambda: bridge.search(args["query"], args.get("wing"), int(args.get("limit", 10))),
|
|
536
|
+
"delete": lambda: bridge.delete(args["id"]),
|
|
537
|
+
"has_title": lambda: bridge.has_title(args["wing"], args["title"]),
|
|
538
|
+
"find_similar": lambda: bridge.find_similar(args["wing"], args["title"], float(args.get("threshold", 0.6))),
|
|
539
|
+
"sync_orphaned": lambda: bridge.sync_orphaned(args["project_dir"], args["wing"]),
|
|
540
|
+
"migrate": lambda: bridge.migrate(args["source_dir"], args.get("projects")),
|
|
541
|
+
}
|
|
542
|
+
handler = handlers.get(cmd)
|
|
543
|
+
if handler is None:
|
|
544
|
+
print(json.dumps({"ok": False, "error": f"unknown command: {cmd}"}))
|
|
545
|
+
return 2
|
|
546
|
+
try:
|
|
547
|
+
result = handler()
|
|
548
|
+
print(json.dumps({"ok": True, "result": result}, default=str))
|
|
549
|
+
return 0
|
|
550
|
+
except Exception as exc:
|
|
551
|
+
print(json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"}))
|
|
552
|
+
return 1
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
if __name__ == "__main__":
|
|
556
|
+
sys.exit(main(sys.argv))
|
package/index.ts
CHANGED
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
* Auto-consolidates on compaction.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import { dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
10
12
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
11
13
|
import {
|
|
12
14
|
UNIPI_EVENTS,
|
|
@@ -30,7 +32,7 @@ import { registerMemoryCommands } from "./commands.js";
|
|
|
30
32
|
import { isEmbeddingReady, hasModelChanged } from "./settings.js";
|
|
31
33
|
|
|
32
34
|
/** Package version */
|
|
33
|
-
const VERSION = getPackageVersion(
|
|
35
|
+
const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
|
|
34
36
|
|
|
35
37
|
/** Storage instance for current project */
|
|
36
38
|
let projectStorage: MemoryStorage | null = null;
|
|
@@ -170,11 +172,12 @@ export default function (pi: ExtensionAPI) {
|
|
|
170
172
|
} catch (_err) {
|
|
171
173
|
// Count unavailable — status bar shows 0.
|
|
172
174
|
}
|
|
173
|
-
const
|
|
174
|
-
const
|
|
175
|
+
const mempalaceActive = projectStorage?.isMempalace() ?? false;
|
|
176
|
+
const backendIcon = mempalaceActive ? "🧠" : (isEmbeddingReady() ? "⚡" : "📝");
|
|
177
|
+
const warn = hasModelChanged() ? " ⚠" : "";
|
|
175
178
|
ctx.ui.setStatus(
|
|
176
179
|
"unipi-memory",
|
|
177
|
-
`${
|
|
180
|
+
`${backendIcon} mem ${projectCount}p/${projectCountAll}all${warn}`
|
|
178
181
|
);
|
|
179
182
|
}
|
|
180
183
|
});
|