@chatpanel/gateway 0.6.43 → 0.6.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/mcp.js +139 -1
- package/src/memory-store.js +176 -0
- package/src/memory.js +642 -0
- package/src/rrf.js +93 -0
- package/src/server.js +120 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chatpanel/gateway",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.45",
|
|
4
4
|
"description": "Local privacy gateway \u2014 redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/mcp.js
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import { loadConfig } from './config.js';
|
|
18
18
|
import { readBridgeToken } from './bridge.js';
|
|
19
19
|
import { ensureGatewayToken } from './gateway-token.js';
|
|
20
|
+
import { MEMORY_KINDS, MEMORY_KIND_NAMES } from './memory.js';
|
|
20
21
|
|
|
21
22
|
const PROTOCOL_VERSION = '2024-11-05';
|
|
22
23
|
const SERVER = { name: 'chatpanel-history', version: '1.0.0' };
|
|
@@ -33,12 +34,24 @@ const INSTRUCTIONS = [
|
|
|
33
34
|
'meeting, call, demo, note, or past conversation ("outcome of the meeting", "what did we',
|
|
34
35
|
'decide", "action items", "notes from yesterday", a person/day/topic in their history),',
|
|
35
36
|
'ALSO consult ChatPanel — it is the source of truth for that personal history:',
|
|
36
|
-
' •
|
|
37
|
+
' • smart_search — START HERE: give it the question plus 2-4 of your own keyword',
|
|
38
|
+
' phrasings; it runs them all and fuses the rankings, finding what one query misses.',
|
|
39
|
+
' • search_history — one exact keyword query, when you already know the terms. Search by',
|
|
40
|
+
' CONTENT (not the generic meeting title). Supports filters:',
|
|
37
41
|
' type (chat|meeting|note), since/before (dates or relative like "7d", "yesterday"),',
|
|
38
42
|
' and limit/offset paging. Returns compact snippets, not full bodies.',
|
|
39
43
|
' • get_record — the full text of one result id; use maxChars/offset to page a long',
|
|
40
44
|
' transcript instead of pulling it all into context.',
|
|
41
45
|
' • find_related — follow the graph: given a record id, the records most connected to it.',
|
|
46
|
+
'',
|
|
47
|
+
'ChatPanel also holds the user\'s MEMORY — a short list of durable facts about them (their',
|
|
48
|
+
'name, how they want answers written, what they are working on) that applies to EVERY task,',
|
|
49
|
+
'not just ones about their history:',
|
|
50
|
+
' • recall — call it EARLY in a session, and whenever the user states a preference, to see',
|
|
51
|
+
' what is already known. Then simply follow it; do not tell them you looked.',
|
|
52
|
+
' • remember — save a new durable fact when they state one ("call me Alex", "always use',
|
|
53
|
+
' pnpm"). One short sentence. Not for task details or anything true only today.',
|
|
54
|
+
' • forget — remove one when they say it no longer holds.',
|
|
42
55
|
'Prefer these for the user\'s history and combine them with your other tools as you see fit.',
|
|
43
56
|
'Every result states how fresh the local copy is; if something recent is missing it may not',
|
|
44
57
|
'have synced yet — say so rather than concluding it does not exist.',
|
|
@@ -103,6 +116,22 @@ async function bridgeJson(path) {
|
|
|
103
116
|
}
|
|
104
117
|
|
|
105
118
|
const TOOLS = [
|
|
119
|
+
{
|
|
120
|
+
name: 'smart_search',
|
|
121
|
+
description: 'BEST first choice for a question about the user\'s ChatPanel history (meetings, notes, past chats). Ask it a natural-language QUESTION and it expands that into several complementary keyword queries, runs them all, and fuses the rankings — which finds things a single query misses, in one round trip instead of several probes. You know the domain, so pass 2-4 of your own phrasings in `queries` too (e.g. for "what did we decide in the Ben demo": ["Ben demo decisions", "tooling demo action items", "demo outcome next steps"]). Supports the same filters as search_history (type, since, before) and returns snippets with each result\'s id; follow up with get_record for the full text or find_related to expand around a hit.',
|
|
122
|
+
inputSchema: {
|
|
123
|
+
type: 'object',
|
|
124
|
+
properties: {
|
|
125
|
+
question: { type: 'string', description: 'The user\'s question, in natural language.' },
|
|
126
|
+
queries: { type: 'array', items: { type: 'string' }, description: 'Your own 2-4 keyword formulations of it — these lead the search.' },
|
|
127
|
+
type: { type: 'string', enum: ['chat', 'meeting', 'note'], description: 'Only this kind of record.' },
|
|
128
|
+
since: { type: 'string', description: 'Earliest date: 2026-08-01, or a window like "7d"/"yesterday".' },
|
|
129
|
+
before: { type: 'string', description: 'Latest date: a date or window like `since`.' },
|
|
130
|
+
limit: { type: 'number', description: 'Max fused results (default 10).' },
|
|
131
|
+
},
|
|
132
|
+
required: ['question'],
|
|
133
|
+
},
|
|
134
|
+
},
|
|
106
135
|
{
|
|
107
136
|
name: 'search_history',
|
|
108
137
|
description: 'Search the user\'s ChatPanel history — their past chats, meeting/call transcripts, and notes — by keyword relevance. Consult this (in ADDITION to your other tools) whenever the question touches a meeting, call, demo, note, or past conversation: "outcome of the meeting", "what did we decide", "action items", "what did <person> say", "notes from yesterday". Filters: `type` (chat|meeting|note), `since`/`before` (a date like 2026-08-01 or a relative window like "7d"/"yesterday"), and `limit`/`offset` for paging. Returns compact SNIPPETS (the matching excerpt) with each record\'s id/title/type/date — token-friendly; call get_record for the full text and find_related to follow connections. This is a LOCAL WARM COPY that syncs from ChatPanel; very recent items may not be here yet — results report how current the index is, so if something is missing it likely has not synced. Meeting titles are often generic ("Zoom Meeting"), so search by CONTENT, not the title.',
|
|
@@ -155,6 +184,38 @@ const TOOLS = [
|
|
|
155
184
|
},
|
|
156
185
|
},
|
|
157
186
|
},
|
|
187
|
+
{
|
|
188
|
+
name: 'recall',
|
|
189
|
+
description: "What ChatPanel already knows about the USER — their name, how they want answers written, their ongoing work and environment. Short and cheap; call it at the START of a session and whenever the user states a preference, then just FOLLOW what it returns without announcing that you checked. Pass the current task in `text` and it also returns the facts relevant to that task, not only the always-on ones. This is memory, not history: for what was said in a meeting or a past chat use smart_search instead.",
|
|
190
|
+
inputSchema: {
|
|
191
|
+
type: 'object',
|
|
192
|
+
properties: {
|
|
193
|
+
text: { type: 'string', description: 'What the user is asking about right now, so task-relevant facts come back too. Omit for just the always-on ones.' },
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
name: 'remember',
|
|
199
|
+
description: "Save a durable fact about the USER to ChatPanel, carried into every future session on every model and agent — the side panel, other CLIs, everything. Use it the moment they state a standing preference (\"always use pnpm\"), an identity fact (\"call me Alex\") or a constraint that will still be true next week. Do NOT use it for task details, anything obvious from the current work, or notes about a codebase — those belong in the repo. One short self-contained sentence, written in the third person. Saving is a change to the user's own data, so say in a short clause what you saved.",
|
|
200
|
+
inputSchema: {
|
|
201
|
+
type: 'object',
|
|
202
|
+
properties: {
|
|
203
|
+
text: { type: 'string', description: 'The fact, one short sentence in the third person ("Prefers pnpm over npm"). Max 280 characters.' },
|
|
204
|
+
kind: { type: 'string', enum: MEMORY_KIND_NAMES, description: MEMORY_KIND_NAMES.map((k) => `${k}: ${MEMORY_KINDS[k].hint}`).join(' ') },
|
|
205
|
+
tags: { type: 'array', items: { type: 'string' }, description: 'Optional short tags for grouping.' },
|
|
206
|
+
},
|
|
207
|
+
required: ['text'],
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
name: 'forget',
|
|
212
|
+
description: 'Remove a memory from ChatPanel when the user says it no longer holds. Name it however they did ("the Frankfurt thing") or pass the id from recall — it matches on meaning, and tells you exactly what it removed.',
|
|
213
|
+
inputSchema: {
|
|
214
|
+
type: 'object',
|
|
215
|
+
properties: { query: { type: 'string', description: 'The memory to remove: its text, roughly how the user named it, or its id.' } },
|
|
216
|
+
required: ['query'],
|
|
217
|
+
},
|
|
218
|
+
},
|
|
158
219
|
{
|
|
159
220
|
name: 'list_skills',
|
|
160
221
|
description: 'List the reusable skills installed on this machine (via the ChatPanel bridge) — across every agent harness (Claude Code, Codex, Copilot, Gemini, Hermes) and any configured folder. Returns each skill\'s name and one-line description. Call open_skill to load the one that fits the task.',
|
|
@@ -192,6 +253,16 @@ function horizonLine(newest, size) {
|
|
|
192
253
|
return `Index: ${size} records, current through ${iso} (local warm copy — items newer than this may not have synced from ChatPanel yet).`;
|
|
193
254
|
}
|
|
194
255
|
|
|
256
|
+
// Memory WRITES are admin-gated on the gateway (a drive-by localhost page must not be able to
|
|
257
|
+
// install a standing instruction), so they carry the gateway token — readable only by this
|
|
258
|
+
// same-user process. Reads need none: an open warm index is the product.
|
|
259
|
+
function writeAuth() {
|
|
260
|
+
try {
|
|
261
|
+
const token = ensureGatewayToken();
|
|
262
|
+
return token ? { Authorization: `Bearer ${token}` } : {};
|
|
263
|
+
} catch { return {}; }
|
|
264
|
+
}
|
|
265
|
+
|
|
195
266
|
async function gatewayJson(path, init) {
|
|
196
267
|
let res;
|
|
197
268
|
try {
|
|
@@ -230,6 +301,30 @@ function parseWhen(v) {
|
|
|
230
301
|
const fmtRow = (r, i) => `${i + 1}. [${r.id}] ${r.title || '(untitled)'} · ${r.type}${r.date ? ' · ' + new Date(r.date).toISOString().slice(0, 10) : ''}${r.snippet ? `\n ${r.snippet}` : ''}`;
|
|
231
302
|
|
|
232
303
|
async function callTool(name, args = {}) {
|
|
304
|
+
if (name === 'smart_search') {
|
|
305
|
+
const body = {
|
|
306
|
+
question: String(args.question || ''),
|
|
307
|
+
queries: Array.isArray(args.queries) ? args.queries.map(String) : [],
|
|
308
|
+
limit: Number(args.limit) || 10,
|
|
309
|
+
};
|
|
310
|
+
if (args.type) body.type = String(args.type);
|
|
311
|
+
const since = parseWhen(args.since); if (since != null) body.since = since;
|
|
312
|
+
const before = parseWhen(args.before); if (before != null) body.before = before;
|
|
313
|
+
const data = await gatewayJson('/v1/history/smart-search', {
|
|
314
|
+
method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body),
|
|
315
|
+
});
|
|
316
|
+
const rows = data.results || [];
|
|
317
|
+
const horizon = horizonLine(data.newest, data.size);
|
|
318
|
+
const asked = (data.queries || []).map((q) => `"${q}"`).join(', ');
|
|
319
|
+
if (!rows.length) {
|
|
320
|
+
return `No match for "${args.question}".\nSearched ${data.queries?.length || 0} way(s): ${asked}.\n${horizon}\nIf you expected a recent item it may not have synced yet — check ChatPanel directly, or try different keywords (meeting titles are often generic).`;
|
|
321
|
+
}
|
|
322
|
+
return [
|
|
323
|
+
horizon, '',
|
|
324
|
+
`${rows.length} result(s) for "${args.question}" — searched ${data.queries.length} way(s): ${asked}`,
|
|
325
|
+
...rows.map((r, i) => `${fmtRow(r, i)}${r.foundBy?.length > 1 ? `\n (matched ${r.foundBy.length} of the queries)` : ''}`),
|
|
326
|
+
].join('\n') + '\n\nget_record <id> for full text (maxChars/offset to page) · find_related <id> to follow connections.';
|
|
327
|
+
}
|
|
233
328
|
if (name === 'search_history') {
|
|
234
329
|
const body = { query: String(args.query || ''), limit: Number(args.limit) || 10, offset: Math.max(0, Number(args.offset) || 0) };
|
|
235
330
|
if (args.type) body.type = String(args.type);
|
|
@@ -272,6 +367,49 @@ async function callTool(name, args = {}) {
|
|
|
272
367
|
const newest = items[0]?.date || null;
|
|
273
368
|
return [horizonLine(newest, data.total), '', `${items.length} of ${data.total} records:`, ...items.map((it) => `[${it.id}] ${it.title || '(untitled)'} · ${it.type}${it.date ? ' · ' + new Date(it.date).toISOString().slice(0, 10) : ''} · ${it.chars} chars`)].join('\n');
|
|
274
369
|
}
|
|
370
|
+
if (name === 'recall') {
|
|
371
|
+
const data = await gatewayJson('/v1/memory/recall', {
|
|
372
|
+
method: 'POST', headers: { 'content-type': 'application/json' },
|
|
373
|
+
body: JSON.stringify({ text: String(args.text || '') }),
|
|
374
|
+
});
|
|
375
|
+
if (!data.memories?.length) {
|
|
376
|
+
return data.size
|
|
377
|
+
? 'Nothing in ChatPanel memory applies here.'
|
|
378
|
+
: 'ChatPanel memory is empty — nothing is known about this user yet. Use `remember` when they state something durable about themselves or how they want to work.';
|
|
379
|
+
}
|
|
380
|
+
// The SHARED rendering, straight from the gateway, so a CLI agent is told exactly what
|
|
381
|
+
// the side panel's models are told. Two renderings would drift, invisibly.
|
|
382
|
+
return data.block;
|
|
383
|
+
}
|
|
384
|
+
if (name === 'remember') {
|
|
385
|
+
const text = String(args.text || '').trim();
|
|
386
|
+
if (!text) return 'remember needs `text` — one short sentence about the user.';
|
|
387
|
+
const data = await gatewayJson('/v1/memory/remember', {
|
|
388
|
+
method: 'POST', headers: { 'content-type': 'application/json', ...writeAuth() },
|
|
389
|
+
body: JSON.stringify({
|
|
390
|
+
text,
|
|
391
|
+
kind: args.kind ? String(args.kind) : 'fact',
|
|
392
|
+
tags: Array.isArray(args.tags) ? args.tags.map(String) : [],
|
|
393
|
+
// Attribution is the accountability here. A CLI has no confirm dialog to show, so
|
|
394
|
+
// instead every memory an agent writes is stamped with WHICH agent wrote it and
|
|
395
|
+
// shows up that way in the extension's Memory page, where the user can correct or
|
|
396
|
+
// delete it. Silent and anonymous would be the unacceptable combination.
|
|
397
|
+
source: { via: 'mcp', surface: 'mcp', agent: clientName },
|
|
398
|
+
}),
|
|
399
|
+
});
|
|
400
|
+
if (data.action === 'duplicate') return `Already known: "${data.record.text}" — nothing changed.`;
|
|
401
|
+
if (data.action === 'update') return `Updated memory to "${data.record.text}" (was "${data.replaced?.text}"). It applies to every future session.`;
|
|
402
|
+
return `Remembered: "${data.record.text}". It applies to every future ChatPanel session, on every model.`;
|
|
403
|
+
}
|
|
404
|
+
if (name === 'forget') {
|
|
405
|
+
const query = String(args.query || '').trim();
|
|
406
|
+
if (!query) return 'forget needs `query` — the memory to remove.';
|
|
407
|
+
const data = await gatewayJson('/v1/memory/forget', {
|
|
408
|
+
method: 'POST', headers: { 'content-type': 'application/json', ...writeAuth() }, body: JSON.stringify({ query }),
|
|
409
|
+
});
|
|
410
|
+
if (!data.removed?.length) return `No memory matches "${query}". Call recall to see what is stored.`;
|
|
411
|
+
return `Forgot: ${data.removed.map((m) => `"${m.text}"`).join(', ')}.`;
|
|
412
|
+
}
|
|
275
413
|
if (name === 'list_skills') {
|
|
276
414
|
let data;
|
|
277
415
|
try { data = await bridgeJson('/skills'); }
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// MEMORY on the gateway — the durable facts about the user, available to every local agent.
|
|
2
|
+
//
|
|
3
|
+
// The extension has its own copy in chrome.storage; this is the one CLI agents can reach.
|
|
4
|
+
// Claude Code, Codex, OpenCode and anything else that speaks MCP get the user's standing
|
|
5
|
+
// preferences through `chatpanel-gateway mcp`, so "call me Alex, never open with a preamble"
|
|
6
|
+
// holds in the terminal exactly as it does in the side panel. That was the whole point of
|
|
7
|
+
// putting memory in a shared contract rather than in the panel.
|
|
8
|
+
//
|
|
9
|
+
// TWO STORES, ONE TRUTH, because reconcile is idempotent. The extension pushes its memories
|
|
10
|
+
// here and pulls back what the agents wrote (see the extension's warm-sync). A naive
|
|
11
|
+
// two-way sync would duplicate on every pass; this one converges because `reconcile` keys on
|
|
12
|
+
// the FACT (its slot, then its wording), not on a row id — so pushing the same memory twice
|
|
13
|
+
// is a no-op and a corrected one supersedes rather than accumulating. The merge is the same
|
|
14
|
+
// function on both sides, from the same file, which is the only reason that holds.
|
|
15
|
+
//
|
|
16
|
+
// ENCRYPTED AT REST with the same local key as the history store — this is the on-device
|
|
17
|
+
// tier, and a local key is correct for it.
|
|
18
|
+
|
|
19
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, statSync } from 'node:fs';
|
|
20
|
+
import { join, dirname } from 'node:path';
|
|
21
|
+
import os from 'node:os';
|
|
22
|
+
import { randomBytes, createCipheriv, createDecipheriv } from 'node:crypto';
|
|
23
|
+
import {
|
|
24
|
+
normalizeMemory, reconcile, recall, memoryBlock, matchForForget, pruneMemories, markUsed,
|
|
25
|
+
isValidMemory, upcastMemory, DEFAULT_MAX_MEMORIES,
|
|
26
|
+
} from './memory.js';
|
|
27
|
+
|
|
28
|
+
const DIR = join(os.homedir(), '.chatpanel');
|
|
29
|
+
const STORE_PATH = process.env.CHATPANEL_MEMORY_STORE || join(DIR, 'memory-store.enc');
|
|
30
|
+
const KEY_PATH = process.env.CHATPANEL_HISTORY_KEY || join(DIR, 'history-key');
|
|
31
|
+
|
|
32
|
+
const uid = () => `mem_${Date.now().toString(36)}${randomBytes(4).toString('hex')}`;
|
|
33
|
+
|
|
34
|
+
// The same local key file the history store uses. One device key, not two: a second key file
|
|
35
|
+
// is a second thing to lose, and both stores are the same tier with the same threat model.
|
|
36
|
+
function loadOrCreateKey() {
|
|
37
|
+
try {
|
|
38
|
+
if (existsSync(KEY_PATH)) return Buffer.from(readFileSync(KEY_PATH, 'utf8').trim(), 'base64');
|
|
39
|
+
} catch { /* regenerate below */ }
|
|
40
|
+
const key = randomBytes(32);
|
|
41
|
+
mkdirSync(dirname(KEY_PATH), { recursive: true });
|
|
42
|
+
writeFileSync(KEY_PATH, key.toString('base64'), { mode: 0o600 });
|
|
43
|
+
return key;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function encrypt(key, buf) {
|
|
47
|
+
const iv = randomBytes(12);
|
|
48
|
+
const cipher = createCipheriv('aes-256-gcm', key, iv);
|
|
49
|
+
const ct = Buffer.concat([cipher.update(buf), cipher.final()]);
|
|
50
|
+
return { v: 1, iv: iv.toString('base64'), tag: cipher.getAuthTag().toString('base64'), ct: ct.toString('base64') };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function decrypt(key, env) {
|
|
54
|
+
const d = createDecipheriv('aes-256-gcm', key, Buffer.from(env.iv, 'base64'));
|
|
55
|
+
d.setAuthTag(Buffer.from(env.tag, 'base64'));
|
|
56
|
+
return Buffer.concat([d.update(Buffer.from(env.ct, 'base64')), d.final()]);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export class MemoryStore {
|
|
60
|
+
constructor({ storePath = STORE_PATH } = {}) {
|
|
61
|
+
this.storePath = storePath;
|
|
62
|
+
this.memories = [];
|
|
63
|
+
this._key = null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
get size() { return this.memories.length; }
|
|
67
|
+
|
|
68
|
+
get bytes() {
|
|
69
|
+
try { return existsSync(this.storePath) ? statSync(this.storePath).size : 0; } catch { return 0; }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
key() {
|
|
73
|
+
if (!this._key) this._key = loadOrCreateKey();
|
|
74
|
+
return this._key;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Fail-open on a missing or corrupt file — memory is an enhancement, and refusing to start
|
|
78
|
+
// the gateway because one cache file is unreadable trades a small loss for a total one.
|
|
79
|
+
// A record that fails validation is DROPPED, never repaired: it would otherwise be handed
|
|
80
|
+
// to a model as a standing fact about the user.
|
|
81
|
+
load() {
|
|
82
|
+
try {
|
|
83
|
+
if (!existsSync(this.storePath)) return this;
|
|
84
|
+
const raw = JSON.parse(decrypt(this.key(), JSON.parse(readFileSync(this.storePath, 'utf8'))).toString('utf8'));
|
|
85
|
+
this.memories = (Array.isArray(raw) ? raw : [])
|
|
86
|
+
.map((m) => { try { return upcastMemory(m); } catch { return null; } })
|
|
87
|
+
.filter((m) => m && isValidMemory(m));
|
|
88
|
+
} catch {
|
|
89
|
+
this.memories = [];
|
|
90
|
+
}
|
|
91
|
+
return this;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
persistNow() {
|
|
95
|
+
try {
|
|
96
|
+
mkdirSync(dirname(this.storePath), { recursive: true });
|
|
97
|
+
writeFileSync(this.storePath, JSON.stringify(encrypt(this.key(), Buffer.from(JSON.stringify(this.memories), 'utf8'))), { mode: 0o600 });
|
|
98
|
+
} catch { /* a failed cache write must not fail the call that triggered it */ }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
#commit(list) {
|
|
102
|
+
const { kept } = pruneMemories(list, { now: Date.now(), max: DEFAULT_MAX_MEMORIES });
|
|
103
|
+
this.memories = kept.sort((a, b) => (b.updatedAt || 0) - (a.updatedAt || 0));
|
|
104
|
+
this.persistNow();
|
|
105
|
+
return this.memories;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
list() { return this.memories; }
|
|
109
|
+
|
|
110
|
+
/** Save one memory, reconciled against what is held. The only write path. */
|
|
111
|
+
remember(input) {
|
|
112
|
+
const now = Date.now();
|
|
113
|
+
const { action, record, replaces } = reconcile(this.memories, input, { now, newId: uid });
|
|
114
|
+
this.#commit([record, ...(replaces ? this.memories.filter((m) => m.id !== replaces.id) : this.memories)]);
|
|
115
|
+
return { action, record, replaces };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Drop by id, or by however a person named it ("forget the Frankfurt thing"). */
|
|
119
|
+
forget(query) {
|
|
120
|
+
const q = String(query || '').trim();
|
|
121
|
+
const hits = this.memories.some((m) => m.id === q)
|
|
122
|
+
? this.memories.filter((m) => m.id === q)
|
|
123
|
+
: matchForForget(this.memories, q);
|
|
124
|
+
if (!hits.length) return { removed: [] };
|
|
125
|
+
const gone = new Set(hits.map((m) => m.id));
|
|
126
|
+
this.#commit(this.memories.filter((m) => !gone.has(m.id)));
|
|
127
|
+
return { removed: hits };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** The memories a turn should carry, and the rendered block — the SAME ranking the panel uses. */
|
|
131
|
+
recall({ text = '', scopes = ['global'], limit, maxChars } = {}) {
|
|
132
|
+
const chosen = recall(this.memories, {
|
|
133
|
+
text, scopes, now: Date.now(),
|
|
134
|
+
...(limit ? { limit } : {}),
|
|
135
|
+
...(maxChars ? { maxChars } : {}),
|
|
136
|
+
});
|
|
137
|
+
if (chosen.length) {
|
|
138
|
+
this.memories = markUsed(this.memories, chosen.map((m) => m.id), { now: Date.now() });
|
|
139
|
+
this.persistNow();
|
|
140
|
+
}
|
|
141
|
+
return { memories: chosen, block: memoryBlock(chosen) };
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Bulk merge from the extension's sync. Every incoming record goes through `reconcile`, so
|
|
146
|
+
* a repeated push is a no-op rather than a doubling — that idempotence is what makes the
|
|
147
|
+
* two-way sync safe.
|
|
148
|
+
*/
|
|
149
|
+
bulk({ upserts = [], removes = [] } = {}) {
|
|
150
|
+
for (const id of removes) {
|
|
151
|
+
const gone = new Set([String(id)]);
|
|
152
|
+
this.memories = this.memories.filter((m) => !gone.has(m.id));
|
|
153
|
+
}
|
|
154
|
+
let merged = 0;
|
|
155
|
+
for (const raw of upserts) {
|
|
156
|
+
let candidate;
|
|
157
|
+
try { candidate = normalizeMemory(raw, { now: Date.now(), newId: uid }); } catch { continue; }
|
|
158
|
+
const { action, record, replaces } = reconcile(this.memories, candidate, { now: Date.now(), newId: uid });
|
|
159
|
+
if (action === 'duplicate') continue;
|
|
160
|
+
this.#commit([record, ...(replaces ? this.memories.filter((m) => m.id !== replaces.id) : this.memories)]);
|
|
161
|
+
merged += 1;
|
|
162
|
+
}
|
|
163
|
+
if (removes.length && !merged) this.#commit(this.memories);
|
|
164
|
+
return { size: this.memories.length, merged };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
clear() {
|
|
168
|
+
const dropped = this.memories.length;
|
|
169
|
+
this.#commit([]);
|
|
170
|
+
return dropped;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export async function createMemoryStore(opts) {
|
|
175
|
+
return new MemoryStore(opts).load();
|
|
176
|
+
}
|
package/src/memory.js
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
// VENDORED from @chatpanel/events/memory.js — edit there, then copy over.
|
|
2
|
+
// Same pattern as observability.js and rrf.js: one pure module copied in rather than pulling
|
|
3
|
+
// the whole events package. Source of truth: chatpanel-events/memory.js.
|
|
4
|
+
//
|
|
5
|
+
/**
|
|
6
|
+
* MEMORY — the small set of durable facts about the user that every model should already know.
|
|
7
|
+
*
|
|
8
|
+
* ChatPanel remembers conversations, meetings and notes, and can search all of them. What it
|
|
9
|
+
* could not do is KNOW anything. Told "call me Alex, and never open with a preamble", the next
|
|
10
|
+
* turn — let alone the next agent, or Codex over MCP — started from zero, because the only way
|
|
11
|
+
* a fact survived was for someone to search for it, and nobody searches for their own name.
|
|
12
|
+
*
|
|
13
|
+
* Memory is the opposite of history by design. History is large, retrieved, and about events.
|
|
14
|
+
* Memory is SMALL, ambient, and about standing truths — so it can be carried into every turn
|
|
15
|
+
* instead of looked up, and so a person can read the whole of it in one screen and correct it.
|
|
16
|
+
* That size limit is the feature: a memory that grows without bound becomes a second corpus,
|
|
17
|
+
* and a second corpus needs search, and then nothing is ambient any more.
|
|
18
|
+
*
|
|
19
|
+
* Four properties make it work the same everywhere:
|
|
20
|
+
*
|
|
21
|
+
* ONE VOCABULARY. Five kinds, closed set. `identity` and `preference` are AMBIENT — they
|
|
22
|
+
* apply to every turn, so they are carried unconditionally. `project`, `fact` and
|
|
23
|
+
* `reference` are RETRIEVED, because "the staging cluster is in Frankfurt" is only worth
|
|
24
|
+
* tokens on a turn that mentions it. One rule decides which, in `recall`, so the extension,
|
|
25
|
+
* the gateway and a future mobile client cannot disagree about what the model was told.
|
|
26
|
+
*
|
|
27
|
+
* CAPTURE IS DETERMINISTIC. `candidatesFrom` reads a string. No model call, no network, no
|
|
28
|
+
* latency, on every user message — which is the only reason it can run on every user
|
|
29
|
+
* message. It separates what the user COMMANDED ("remember that…") from what they merely
|
|
30
|
+
* REVEALED ("I prefer…"), because those deserve different answers: the first is consent,
|
|
31
|
+
* the second is a guess, and a guess must be offered rather than acted on.
|
|
32
|
+
*
|
|
33
|
+
* WRITES RECONCILE, THEY DO NOT ACCUMULATE. Saying "call me Alex" twice must leave one
|
|
34
|
+
* memory, and saying "actually, call me Sam" must leave one memory with the new value and
|
|
35
|
+
* the old one recoverable. `reconcile` decides create/update/duplicate against what is
|
|
36
|
+
* already stored; a client that just pushes rows produces a list nobody can read by week
|
|
37
|
+
* two.
|
|
38
|
+
*
|
|
39
|
+
* THE PROMPT FORM IS SHARED. `memoryBlock` is the one rendering. If the extension wrote its
|
|
40
|
+
* own the model would be told something subtly different depending on which surface the
|
|
41
|
+
* user typed into, and the bug would be invisible.
|
|
42
|
+
*
|
|
43
|
+
* Class R: pure, dependency-free, no I/O. Persistence is the host's (chrome.storage in the
|
|
44
|
+
* extension, an encrypted file in the gateway) — the same split as `store.js`.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
export const MEMORY_VERSION = 1;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The closed vocabulary. Deliberately five: enough that a memory's kind tells you how to
|
|
51
|
+
* treat it, few enough that a person choosing one does not have to think.
|
|
52
|
+
*
|
|
53
|
+
* `ambient` is the load-bearing bit — it is not a label, it is the retrieval rule.
|
|
54
|
+
*/
|
|
55
|
+
export const MEMORY_KINDS = Object.freeze({
|
|
56
|
+
identity: { label: 'Identity', ambient: true, hint: 'Who the user is — name, role, pronouns, language, timezone.' },
|
|
57
|
+
preference: { label: 'Preference', ambient: true, hint: 'How they want things done — tone, format, defaults, things never to do.' },
|
|
58
|
+
project: { label: 'Project', ambient: false, hint: 'Ongoing work, goals and constraints not derivable from the material itself.' },
|
|
59
|
+
fact: { label: 'Fact', ambient: false, hint: 'A durable fact about their world — systems, teams, environments, conventions.' },
|
|
60
|
+
reference: { label: 'Reference', ambient: false, hint: 'A pointer to something external — a URL, a dashboard, a ticket, a doc.' },
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
export const MEMORY_KIND_NAMES = Object.freeze(Object.keys(MEMORY_KINDS));
|
|
64
|
+
|
|
65
|
+
/** Kinds carried on every turn regardless of what was said. */
|
|
66
|
+
export const AMBIENT_KINDS = Object.freeze(MEMORY_KIND_NAMES.filter((k) => MEMORY_KINDS[k].ambient));
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Bounds. A memory longer than this is a note, and there is already a notes feature; a store
|
|
70
|
+
* larger than this is a corpus, and there is already a history feature. Both limits exist to
|
|
71
|
+
* stop memory turning into the thing next to it.
|
|
72
|
+
*/
|
|
73
|
+
export const MAX_MEMORY_CHARS = 280;
|
|
74
|
+
export const MIN_MEMORY_CHARS = 3;
|
|
75
|
+
export const DEFAULT_MAX_MEMORIES = 200;
|
|
76
|
+
/** Prompt budget for the injected block. ~100 tokens; enough for a readable standing brief. */
|
|
77
|
+
export const DEFAULT_BLOCK_CHARS = 1200;
|
|
78
|
+
|
|
79
|
+
export class MemoryError extends Error {
|
|
80
|
+
constructor(message) { super(message); this.name = 'MemoryError'; }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// --------------------------------------------------------------------------
|
|
84
|
+
// The record
|
|
85
|
+
// --------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Normalize anything memory-shaped into the canonical record. Throws MemoryError on input
|
|
89
|
+
* that cannot be a memory — callers get one validation path rather than each inventing
|
|
90
|
+
* their own defaults.
|
|
91
|
+
*
|
|
92
|
+
* `now` and `newId` are injected for the same reason they are in `event.js`: a pure module
|
|
93
|
+
* that reads the clock cannot be replayed or tested.
|
|
94
|
+
*
|
|
95
|
+
* @returns {{
|
|
96
|
+
* id: string, v: number, text: string, kind: string, scope: string, key: string,
|
|
97
|
+
* tags: string[], pinned: boolean, source: object, confidence: number,
|
|
98
|
+
* createdAt: number, updatedAt: number, usedAt: number, useCount: number,
|
|
99
|
+
* expiresAt: number|null, history: {text: string, at: number}[]
|
|
100
|
+
* }}
|
|
101
|
+
*/
|
|
102
|
+
export function normalizeMemory(input = {}, { now = 0, newId = null } = {}) {
|
|
103
|
+
const text = collapse(input.text);
|
|
104
|
+
if (text.length < MIN_MEMORY_CHARS) throw new MemoryError('a memory needs text');
|
|
105
|
+
if (text.length > MAX_MEMORY_CHARS) {
|
|
106
|
+
throw new MemoryError(`a memory must be at most ${MAX_MEMORY_CHARS} characters — save longer material as a note`);
|
|
107
|
+
}
|
|
108
|
+
const kind = MEMORY_KINDS[input.kind] ? input.kind : 'fact';
|
|
109
|
+
const at = Number(input.createdAt) || Number(now) || 0;
|
|
110
|
+
return {
|
|
111
|
+
id: String(input.id || (newId ? newId() : '') || ''),
|
|
112
|
+
v: MEMORY_VERSION,
|
|
113
|
+
text,
|
|
114
|
+
kind,
|
|
115
|
+
// Free-form so a client can scope to an agent, a workspace or a site without this module
|
|
116
|
+
// enumerating surfaces it cannot know about. 'global' means every turn everywhere.
|
|
117
|
+
scope: String(input.scope || 'global'),
|
|
118
|
+
key: memoryKey(text),
|
|
119
|
+
// Derived, not asked for — see slotOf. An explicit slot still wins.
|
|
120
|
+
slot: collapse(input.slot).toLowerCase() || slotOf(text),
|
|
121
|
+
tags: [...new Set((input.tags || []).map((t) => collapse(t).toLowerCase()).filter(Boolean))].slice(0, 8),
|
|
122
|
+
pinned: !!input.pinned,
|
|
123
|
+
// WHERE IT CAME FROM, always. A memory the user cannot trace is a memory they cannot
|
|
124
|
+
// trust, and the first thing anyone asks of a wrong one is "when did I say that".
|
|
125
|
+
source: {
|
|
126
|
+
via: String(input.source?.via || 'user'), // user | agent | import | mcp
|
|
127
|
+
surface: String(input.source?.surface || ''), // chat | notes | meeting | mcp | settings
|
|
128
|
+
ref: String(input.source?.ref || ''), // conversation/meeting/note id
|
|
129
|
+
agent: String(input.source?.agent || ''), // which model or CLI proposed it
|
|
130
|
+
},
|
|
131
|
+
// How sure the CAPTURE was, not how true the fact is. An explicit command is 1.
|
|
132
|
+
confidence: clamp01(input.confidence == null ? 1 : Number(input.confidence)),
|
|
133
|
+
createdAt: at,
|
|
134
|
+
updatedAt: Number(input.updatedAt) || at,
|
|
135
|
+
usedAt: Number(input.usedAt) || 0,
|
|
136
|
+
useCount: Math.max(0, Math.floor(Number(input.useCount) || 0)),
|
|
137
|
+
expiresAt: input.expiresAt ? Number(input.expiresAt) : null,
|
|
138
|
+
// Bounded supersession ledger — "you used to say X" without an unbounded audit log.
|
|
139
|
+
history: (input.history || []).slice(-5).map((h) => ({ text: collapse(h.text), at: Number(h.at) || 0 })),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** True when `rec` is a well-formed memory. Never throws — for filtering a loaded store. */
|
|
144
|
+
export function isValidMemory(rec) {
|
|
145
|
+
try { normalizeMemory(rec, { now: rec?.createdAt || 1 }); return !!rec?.id; } catch { return false; }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The identity of a FACT rather than of a record — two phrasings of the same standing truth
|
|
150
|
+
* should collide here so `reconcile` can supersede rather than accumulate.
|
|
151
|
+
*
|
|
152
|
+
* Lowercased, stripped of punctuation and of the framing words people vary freely ("I always
|
|
153
|
+
* prefer" / "prefer"), then the remaining words sorted and deduped. Sorting is safe precisely
|
|
154
|
+
* because this is a dedup key and never a display value: "deploy on fridays" and "on fridays,
|
|
155
|
+
* deploy" are the same standing fact, and treating them as two is the failure this prevents.
|
|
156
|
+
*/
|
|
157
|
+
export function memoryKey(text) {
|
|
158
|
+
const words = collapse(text)
|
|
159
|
+
.toLowerCase()
|
|
160
|
+
.replace(/[^\p{L}\p{N}\s]/gu, ' ')
|
|
161
|
+
.split(/\s+/)
|
|
162
|
+
.filter((w) => w && !KEY_STOPWORDS.has(w));
|
|
163
|
+
return [...new Set(words)].sort().join(' ');
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const KEY_STOPWORDS = new Set([
|
|
167
|
+
'a', 'an', 'the', 'i', 'im', 'me', 'my', 'mine', 'we', 'our', 'us', 'you', 'your',
|
|
168
|
+
'is', 'am', 'are', 'was', 'were', 'be', 'been', 'being', 'do', 'does', 'did',
|
|
169
|
+
'to', 'of', 'in', 'on', 'at', 'for', 'with', 'and', 'or', 'that', 'this', 'it',
|
|
170
|
+
'please', 'always', 'usually', 'generally', 'really', 'just', 'very', 'so',
|
|
171
|
+
'remember', 'note', 'noting', 'user', 'prefer', 'prefers', 'preferred', 'like', 'likes',
|
|
172
|
+
]);
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* THE SLOT A MEMORY FILLS, when it fills one — derived from the text itself.
|
|
176
|
+
*
|
|
177
|
+
* Token overlap cannot see that "Goes by Alex" and "Goes by Sam" are the same fact with a new
|
|
178
|
+
* value: they share two words out of four, which is exactly what two unrelated memories look
|
|
179
|
+
* like. So changing your name produced a SECOND memory and the model was told both.
|
|
180
|
+
*
|
|
181
|
+
* A slot is the subject a statement is about. Two memories with the same slot are one memory,
|
|
182
|
+
* whatever their words, so the later one supersedes. Deriving it from the text rather than
|
|
183
|
+
* asking the caller means it works identically for a captured phrase and for a `memory` tool
|
|
184
|
+
* call from some agent that has never heard of slots.
|
|
185
|
+
*
|
|
186
|
+
* Returns '' when a statement is not slot-shaped ("Deploys on Fridays"), which is most of
|
|
187
|
+
* them — those fall back to key and similarity matching.
|
|
188
|
+
*/
|
|
189
|
+
export function slotOf(text) {
|
|
190
|
+
const t = collapse(text).toLowerCase().replace(/^the\s+/, '');
|
|
191
|
+
// Identity phrasings all name the same slot, or the user's name lives in three places.
|
|
192
|
+
if (/^(?:goes by|name is|is called|called)\b/.test(t)) return 'name';
|
|
193
|
+
if (/^pronouns\b/.test(t)) return 'pronouns';
|
|
194
|
+
// "<subject> is/are <value>" — the general form. Bounded to a short subject so a whole
|
|
195
|
+
// sentence containing "is" somewhere does not become a slot that swallows its neighbours.
|
|
196
|
+
const m = /^(?:my\s+|their\s+)?([\p{L}\p{N}][\p{L}\p{N} '-]{0,28}?)\s+(?:is|are|was|were)\b/u.exec(t);
|
|
197
|
+
if (!m) return '';
|
|
198
|
+
const subject = m[1].split(/\s+/).filter((w) => !KEY_STOPWORDS.has(w)).join(' ');
|
|
199
|
+
return subject.length >= 3 ? subject : '';
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Token-set overlap, 0..1 — "prefers terse answers" vs "prefers terse replies". */
|
|
203
|
+
export function similarity(a, b) {
|
|
204
|
+
const A = new Set(memoryKey(a).split(' ').filter(Boolean));
|
|
205
|
+
const B = new Set(memoryKey(b).split(' ').filter(Boolean));
|
|
206
|
+
if (!A.size || !B.size) return 0;
|
|
207
|
+
let hit = 0;
|
|
208
|
+
for (const w of A) if (B.has(w)) hit += 1;
|
|
209
|
+
return hit / (A.size + B.size - hit);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* How much of the SHORTER phrase the longer one contains, 0..1. Different question from
|
|
214
|
+
* `similarity`, and the right one for "forget the Frankfurt thing": a person names a memory by
|
|
215
|
+
* one distinctive word, not by restating it, so a symmetric measure scores that near zero.
|
|
216
|
+
*/
|
|
217
|
+
export function containment(a, b) {
|
|
218
|
+
const A = new Set(memoryKey(a).split(' ').filter(Boolean));
|
|
219
|
+
const B = new Set(memoryKey(b).split(' ').filter(Boolean));
|
|
220
|
+
if (!A.size || !B.size) return 0;
|
|
221
|
+
let hit = 0;
|
|
222
|
+
for (const w of A) if (B.has(w)) hit += 1;
|
|
223
|
+
return hit / Math.min(A.size, B.size);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Above this, two memories are the same standing fact stated differently. */
|
|
227
|
+
export const SAME_FACT = 0.8;
|
|
228
|
+
|
|
229
|
+
// --------------------------------------------------------------------------
|
|
230
|
+
// Capture — reading a message for things worth keeping
|
|
231
|
+
// --------------------------------------------------------------------------
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* COMMANDS. The user is addressing the assistant and telling it to keep something. The capture
|
|
235
|
+
* group is the fact itself, so "remember that I deploy on Fridays" stores "I deploy on
|
|
236
|
+
* Fridays" and not the instruction wrapping it.
|
|
237
|
+
*
|
|
238
|
+
* Each carries the kind it implies, because "call me Alex" is an identity and "from now on,
|
|
239
|
+
* be terse" is a preference, and making the user pick afterwards is a step they should not
|
|
240
|
+
* have to take.
|
|
241
|
+
*/
|
|
242
|
+
const COMMANDS = [
|
|
243
|
+
{ re: /^(?:please\s+)?(?:remember|memorize|memorise)(?:\s+that|\s+this)?[:,]?\s+(.+)$/i, kind: 'fact' },
|
|
244
|
+
{ re: /^(?:please\s+)?(?:keep in mind|bear in mind|don'?t forget|do not forget)(?:\s+that)?[:,]?\s+(.+)$/i, kind: 'fact' },
|
|
245
|
+
{ re: /^(?:please\s+)?(?:make a note|note)(?:\s+that|\s+of)[:,]?\s+(.+)$/i, kind: 'fact' },
|
|
246
|
+
{ re: /^(?:from now on|going forward|in future|in the future|henceforth)[:,]?\s+(.+)$/i, kind: 'preference' },
|
|
247
|
+
{ re: /^(?:call me|i go by|refer to me as)\s+(.+)$/i, kind: 'identity', rebuild: (m) => `Goes by ${trimEnd(m[1])}` },
|
|
248
|
+
{ re: /^my name(?:'s| is)\s+(.+)$/i, kind: 'identity', rebuild: (m) => `Name is ${trimEnd(m[1])}` },
|
|
249
|
+
{ re: /^(?:always|never)\s+(.+)$/i, kind: 'preference', rebuild: (m, raw) => sentence(raw) },
|
|
250
|
+
];
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* REVEALS. Not addressed to the assistant at all — the user simply said something durable
|
|
254
|
+
* about themselves in passing. These are OFFERED, never saved, because the inference is
|
|
255
|
+
* exactly the kind that is right often enough to be useful and wrong often enough to be
|
|
256
|
+
* insulting if acted on silently.
|
|
257
|
+
*/
|
|
258
|
+
const REVEALS = [
|
|
259
|
+
{ re: /^i (?:prefer|like|want|need)\s+(.+)$/i, kind: 'preference', confidence: 0.7 },
|
|
260
|
+
{ re: /^i (?:hate|dislike|don'?t like|do not like|can'?t stand)\s+(.+)$/i, kind: 'preference', confidence: 0.7 },
|
|
261
|
+
{ re: /^i(?:'m| am)(?: a| an| the)?\s+(.+)$/i, kind: 'identity', confidence: 0.6 },
|
|
262
|
+
{ re: /^i (?:work|working) (?:on|at|with)\s+(.+)$/i, kind: 'project', confidence: 0.6 },
|
|
263
|
+
{ re: /^i(?:'m| am) (?:working|building|writing) (?:on\s+)?(.+)$/i, kind: 'project', confidence: 0.6 },
|
|
264
|
+
{ re: /^(?:we|our team|my team) (?:use|uses|are using|run|runs|deploy|deploys)\s+(.+)$/i, kind: 'fact', confidence: 0.6 },
|
|
265
|
+
{ re: /^my (?:\w+\s){0,2}?(?:is|are)\s+(.+)$/i, kind: 'fact', confidence: 0.55 },
|
|
266
|
+
];
|
|
267
|
+
|
|
268
|
+
/** Removal is a command too, and it must be recognised or "forget that" gets stored as a fact. */
|
|
269
|
+
const FORGETS = [
|
|
270
|
+
/^(?:please\s+)?forget(?:\s+that|\s+about)?[:,]?\s+(.+)$/i,
|
|
271
|
+
/^(?:please\s+)?(?:stop remembering|no longer remember|un-?remember)[:,]?\s+(.+)$/i,
|
|
272
|
+
];
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* A trigger word inside a QUESTION is not an instruction — "do you remember what we decided?"
|
|
276
|
+
* asks for recall, and storing it as a fact is the single most obvious way to make this
|
|
277
|
+
* feature look broken. Likewise "I can't remember" is a complaint, not a command.
|
|
278
|
+
*/
|
|
279
|
+
const NOT_A_COMMAND = /(?:^|\s)(?:do|does|did|can|could|would|will|what|when|where|who|why|how|are|is)\s+(?:you|i|we|it)\b/i;
|
|
280
|
+
const NEGATED = /\b(?:can'?t|cannot|don'?t|do not|couldn'?t|never|didn'?t)\s+(?:seem to\s+)?(?:remember|recall|forget)\b/i;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Read one user message for things worth keeping.
|
|
284
|
+
*
|
|
285
|
+
* Returns candidates in the order found. `explicit: true` means the user issued a command and
|
|
286
|
+
* the host may save it without asking; `explicit: false` means offer it and let them tap.
|
|
287
|
+
* That distinction is the whole capture policy, and it lives here rather than in a client so
|
|
288
|
+
* the panel, the gateway and a mobile app cannot each pick a different one.
|
|
289
|
+
*
|
|
290
|
+
* @param text one user message.
|
|
291
|
+
* @param opts.maxCandidates cap per message (default 3) — a wall of chips is not a prompt.
|
|
292
|
+
* @param opts.includeReveals set false for surfaces with nowhere to show an offer.
|
|
293
|
+
* @returns {{op: 'remember'|'forget', text: string, kind: string, confidence: number,
|
|
294
|
+
* explicit: boolean, trigger: string}[]}
|
|
295
|
+
*/
|
|
296
|
+
export function candidatesFrom(text, { maxCandidates = 3, includeReveals = true } = {}) {
|
|
297
|
+
const out = [];
|
|
298
|
+
const raw = String(text || '');
|
|
299
|
+
// Fenced code is material, not speech. A README line that happens to start "I prefer" is
|
|
300
|
+
// not the user telling us anything.
|
|
301
|
+
const speech = raw.replace(/```[\s\S]*?```/g, ' ').replace(/`[^`]*`/g, ' ');
|
|
302
|
+
|
|
303
|
+
for (const line of splitStatements(speech)) {
|
|
304
|
+
if (out.length >= maxCandidates) break;
|
|
305
|
+
if (line.length < MIN_MEMORY_CHARS) continue;
|
|
306
|
+
if (NEGATED.test(line)) continue;
|
|
307
|
+
|
|
308
|
+
let matched = false;
|
|
309
|
+
for (const re of FORGETS) {
|
|
310
|
+
const m = re.exec(line);
|
|
311
|
+
if (!m) continue;
|
|
312
|
+
const body = clip(trimEnd(m[1]));
|
|
313
|
+
if (body.length >= MIN_MEMORY_CHARS) {
|
|
314
|
+
out.push({ op: 'forget', text: body, kind: 'fact', confidence: 1, explicit: true, trigger: 'forget' });
|
|
315
|
+
matched = true;
|
|
316
|
+
}
|
|
317
|
+
break;
|
|
318
|
+
}
|
|
319
|
+
if (matched) continue;
|
|
320
|
+
|
|
321
|
+
// A question mark alone is not disqualifying — "remember that the demo is Friday?" is a
|
|
322
|
+
// command with a tag. It is the interrogative SHAPE that rules a command out.
|
|
323
|
+
if (NOT_A_COMMAND.test(line)) continue;
|
|
324
|
+
|
|
325
|
+
for (const { re, kind, rebuild } of COMMANDS) {
|
|
326
|
+
const m = re.exec(line);
|
|
327
|
+
if (!m) continue;
|
|
328
|
+
const body = clip(rebuild ? rebuild(m, line) : sentence(trimEnd(m[1])));
|
|
329
|
+
if (body.length >= MIN_MEMORY_CHARS) {
|
|
330
|
+
out.push({ op: 'remember', text: body, kind, confidence: 1, explicit: true, trigger: 'command' });
|
|
331
|
+
matched = true;
|
|
332
|
+
}
|
|
333
|
+
break;
|
|
334
|
+
}
|
|
335
|
+
if (matched || !includeReveals) continue;
|
|
336
|
+
|
|
337
|
+
for (const { re, kind, confidence } of REVEALS) {
|
|
338
|
+
if (!re.test(line)) continue;
|
|
339
|
+
const body = clip(sentence(line));
|
|
340
|
+
// Two words minimum: "I am tired" is durable-looking and worthless. The floor is crude
|
|
341
|
+
// on purpose — the cost of a bad offer is one ignored chip, and the cost of a clever
|
|
342
|
+
// filter is a fact the user watched get dropped.
|
|
343
|
+
if (body.split(/\s+/).length >= 3) {
|
|
344
|
+
out.push({ op: 'remember', text: body, kind, confidence, explicit: false, trigger: 'reveal' });
|
|
345
|
+
}
|
|
346
|
+
break;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return out.slice(0, maxCandidates);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// --------------------------------------------------------------------------
|
|
353
|
+
// Reconcile — what a write does to what is already there
|
|
354
|
+
// --------------------------------------------------------------------------
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Decide what saving `incoming` means against the memories already held.
|
|
358
|
+
*
|
|
359
|
+
* duplicate — the same fact, said the same way. Nothing to do but touch it.
|
|
360
|
+
* update — the same fact, restated or changed. Supersede in place, keep the old text.
|
|
361
|
+
* create — genuinely new.
|
|
362
|
+
*
|
|
363
|
+
* There is deliberately no `conflict`. Two contradictory statements cannot be told apart from
|
|
364
|
+
* a correction without understanding the sentence, and a memory system that asks "did you
|
|
365
|
+
* mean to change your mind?" is a memory system people turn off. The later statement wins and
|
|
366
|
+
* the earlier one stays visible in `history`.
|
|
367
|
+
*
|
|
368
|
+
* @returns {{action: 'create'|'update'|'duplicate', record: object, replaces: object|null}}
|
|
369
|
+
*/
|
|
370
|
+
export function reconcile(memories, incoming, { now = 0, newId = null } = {}) {
|
|
371
|
+
const next = normalizeMemory(incoming, { now, newId });
|
|
372
|
+
const pool = (memories || []).filter((m) => m && m.scope === next.scope);
|
|
373
|
+
|
|
374
|
+
// Slot first: it is the only one of the three that can recognise a CHANGED VALUE, which is
|
|
375
|
+
// what a correction is. Then exact key, then near-identical wording.
|
|
376
|
+
const match = (next.slot && pool.find((m) => m.slot === next.slot))
|
|
377
|
+
|| pool.find((m) => m.key === next.key)
|
|
378
|
+
|| pool.find((m) => m.kind === next.kind && similarity(m.text, next.text) >= SAME_FACT);
|
|
379
|
+
|
|
380
|
+
if (!match) return { action: 'create', record: next, replaces: null };
|
|
381
|
+
|
|
382
|
+
if (collapse(match.text).toLowerCase() === next.text.toLowerCase()) {
|
|
383
|
+
// Restating a memory is a signal about it: it is still true, and it is on the user's mind.
|
|
384
|
+
return {
|
|
385
|
+
action: 'duplicate',
|
|
386
|
+
record: { ...match, updatedAt: Number(now) || match.updatedAt, useCount: match.useCount + 1 },
|
|
387
|
+
replaces: match,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
return {
|
|
392
|
+
action: 'update',
|
|
393
|
+
record: {
|
|
394
|
+
...match,
|
|
395
|
+
text: next.text,
|
|
396
|
+
key: next.key,
|
|
397
|
+
slot: next.slot,
|
|
398
|
+
kind: next.kind,
|
|
399
|
+
tags: next.tags.length ? next.tags : match.tags,
|
|
400
|
+
confidence: next.confidence,
|
|
401
|
+
source: next.source,
|
|
402
|
+
updatedAt: Number(now) || match.updatedAt,
|
|
403
|
+
history: [...match.history, { text: match.text, at: match.updatedAt }].slice(-5),
|
|
404
|
+
},
|
|
405
|
+
replaces: match,
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Which stored memories a "forget X" refers to. Matches by id, then by exact key, then by
|
|
411
|
+
* similarity — a person says "forget the Frankfurt thing", not a uuid.
|
|
412
|
+
*/
|
|
413
|
+
export function matchForForget(memories, query, { limit = 5 } = {}) {
|
|
414
|
+
const q = collapse(query);
|
|
415
|
+
if (!q) return [];
|
|
416
|
+
const byId = (memories || []).filter((m) => m.id === q);
|
|
417
|
+
if (byId.length) return byId;
|
|
418
|
+
const key = memoryKey(q);
|
|
419
|
+
return (memories || [])
|
|
420
|
+
.map((m) => ({ m, s: m.key === key ? 1 : Math.max(similarity(m.text, q), containment(q, m.text)) }))
|
|
421
|
+
.filter((x) => x.s >= 0.5)
|
|
422
|
+
.sort((a, b) => b.s - a.s)
|
|
423
|
+
.slice(0, limit)
|
|
424
|
+
.map((x) => x.m);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// --------------------------------------------------------------------------
|
|
428
|
+
// Recall — what the model is told, this turn
|
|
429
|
+
// --------------------------------------------------------------------------
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Choose the memories for one turn, within a character budget.
|
|
433
|
+
*
|
|
434
|
+
* Ambient kinds (identity, preference) come first and unconditionally: they are how the user
|
|
435
|
+
* wants to be spoken to, and a turn that fails to mention their name is still a turn where
|
|
436
|
+
* their name applies. Everything else is scored against the turn's text — a project memory
|
|
437
|
+
* earns its tokens only when the turn is about it.
|
|
438
|
+
*
|
|
439
|
+
* Pinned always wins, of any kind. That is the user's explicit override of this whole ranking.
|
|
440
|
+
*
|
|
441
|
+
* @param memories all stored memories.
|
|
442
|
+
* @param opts.text the turn's text (user message, or the question for a tool call).
|
|
443
|
+
* @param opts.scopes which scopes apply — 'global' plus, say, `agent:claude-code`.
|
|
444
|
+
* @param opts.maxChars budget for the rendered block.
|
|
445
|
+
* @returns the chosen memories, most important first.
|
|
446
|
+
*/
|
|
447
|
+
export function recall(memories, {
|
|
448
|
+
text = '', scopes = ['global'], kinds = null, now = 0,
|
|
449
|
+
limit = 24, maxChars = DEFAULT_BLOCK_CHARS, includeAmbient = true,
|
|
450
|
+
} = {}) {
|
|
451
|
+
const scopeSet = new Set(scopes.length ? scopes : ['global']);
|
|
452
|
+
const terms = new Set(memoryKey(text).split(' ').filter(Boolean));
|
|
453
|
+
|
|
454
|
+
const live = (memories || [])
|
|
455
|
+
.filter((m) => m && m.text)
|
|
456
|
+
.filter((m) => scopeSet.has(m.scope))
|
|
457
|
+
.filter((m) => !kinds || kinds.includes(m.kind))
|
|
458
|
+
.filter((m) => !m.expiresAt || !now || m.expiresAt > now);
|
|
459
|
+
|
|
460
|
+
const scored = live.map((m) => {
|
|
461
|
+
const words = new Set(m.key.split(' ').filter(Boolean));
|
|
462
|
+
let overlap = 0;
|
|
463
|
+
for (const w of words) if (terms.has(w)) overlap += 1;
|
|
464
|
+
const relevance = words.size ? overlap / words.size : 0;
|
|
465
|
+
const ambient = includeAmbient && MEMORY_KINDS[m.kind]?.ambient;
|
|
466
|
+
// Recency and use are TIE-BREAKS, not drivers. A fact does not become truer because it
|
|
467
|
+
// was mentioned recently, but between two equally relevant ones the live one is the
|
|
468
|
+
// better guess.
|
|
469
|
+
const freshness = now && m.updatedAt ? Math.max(0, 1 - (now - m.updatedAt) / YEAR) : 0;
|
|
470
|
+
const used = Math.min(1, m.useCount / 10);
|
|
471
|
+
return {
|
|
472
|
+
m,
|
|
473
|
+
keep: m.pinned || ambient || relevance > 0,
|
|
474
|
+
score: (m.pinned ? 100 : 0) + (ambient ? 10 : 0) + relevance * 8 + freshness + used,
|
|
475
|
+
};
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
const out = [];
|
|
479
|
+
let chars = 0;
|
|
480
|
+
for (const { m, keep, score } of scored.filter((s) => s.keep).sort((a, b) => b.score - a.score)) {
|
|
481
|
+
if (out.length >= limit) break;
|
|
482
|
+
const cost = m.text.length + m.kind.length + 6;
|
|
483
|
+
if (chars + cost > maxChars && out.length) break;
|
|
484
|
+
chars += cost;
|
|
485
|
+
out.push(m);
|
|
486
|
+
void score;
|
|
487
|
+
}
|
|
488
|
+
return out;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const YEAR = 365 * 24 * 60 * 60 * 1000;
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* The ONE prompt rendering. Every surface that puts memory in front of a model uses this, so
|
|
495
|
+
* "what the model was told" is a single, reviewable string rather than per-client prose.
|
|
496
|
+
*
|
|
497
|
+
* Returns '' for an empty set — callers can concatenate unconditionally.
|
|
498
|
+
*/
|
|
499
|
+
export function memoryBlock(memories, { heading = 'What you already know about this user', maxChars = DEFAULT_BLOCK_CHARS } = {}) {
|
|
500
|
+
const list = (memories || []).filter((m) => m && m.text);
|
|
501
|
+
if (!list.length) return '';
|
|
502
|
+
const lines = [];
|
|
503
|
+
let chars = 0;
|
|
504
|
+
for (const m of list) {
|
|
505
|
+
const line = `- (${m.kind}) ${m.text}`;
|
|
506
|
+
if (chars + line.length > maxChars && lines.length) break;
|
|
507
|
+
chars += line.length + 1;
|
|
508
|
+
lines.push(line);
|
|
509
|
+
}
|
|
510
|
+
return [
|
|
511
|
+
`## ${heading}`,
|
|
512
|
+
'Saved by the user in ChatPanel and true across conversations, agents and devices.',
|
|
513
|
+
'Apply them without being asked and without announcing them.',
|
|
514
|
+
...lines,
|
|
515
|
+
'If the user states something durable about themselves, their preferences or their work,'
|
|
516
|
+
+ ' save it with the `memory` tool. If they correct one of these, update it — do not just agree.',
|
|
517
|
+
].join('\n');
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/** Mark memories as used this turn. Recall quality depends on it, so it is not optional. */
|
|
521
|
+
export function markUsed(memories, ids, { now = 0 } = {}) {
|
|
522
|
+
const set = new Set(ids || []);
|
|
523
|
+
return (memories || []).map((m) => (set.has(m.id)
|
|
524
|
+
? { ...m, usedAt: Number(now) || m.usedAt, useCount: m.useCount + 1 }
|
|
525
|
+
: m));
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Drop what has expired, then hold the store to `max` by evicting the least valuable.
|
|
530
|
+
*
|
|
531
|
+
* Eviction order is the inverse of value: never pinned, never ambient, then least used and
|
|
532
|
+
* least recently touched. Returns both halves so a client can TELL the user what went rather
|
|
533
|
+
* than silently shrinking their memory.
|
|
534
|
+
*/
|
|
535
|
+
export function pruneMemories(memories, { now = 0, max = DEFAULT_MAX_MEMORIES } = {}) {
|
|
536
|
+
const all = (memories || []).filter(Boolean);
|
|
537
|
+
const expired = now ? all.filter((m) => m.expiresAt && m.expiresAt <= now) : [];
|
|
538
|
+
let kept = expired.length ? all.filter((m) => !expired.includes(m)) : all;
|
|
539
|
+
const dropped = [...expired];
|
|
540
|
+
|
|
541
|
+
if (kept.length > max) {
|
|
542
|
+
const value = (m) => (m.pinned ? 3 : 0) + (MEMORY_KINDS[m.kind]?.ambient ? 1 : 0);
|
|
543
|
+
const ranked = [...kept].sort((a, b) => value(b) - value(a)
|
|
544
|
+
|| b.useCount - a.useCount
|
|
545
|
+
|| (b.usedAt || b.updatedAt) - (a.usedAt || a.updatedAt));
|
|
546
|
+
dropped.push(...ranked.slice(max));
|
|
547
|
+
kept = ranked.slice(0, max);
|
|
548
|
+
}
|
|
549
|
+
return { kept, dropped };
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// --------------------------------------------------------------------------
|
|
553
|
+
// The tool contract — one definition, every client
|
|
554
|
+
// --------------------------------------------------------------------------
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* The `memory` tool as the model sees it, shared by the extension's turn toolset and the
|
|
558
|
+
* gateway's MCP server. A second copy would drift, and the drift would be a model that
|
|
559
|
+
* behaves differently in Claude Code than in the side panel for no reason a user could see.
|
|
560
|
+
*
|
|
561
|
+
* One tool with an `action`, not four tools: memory is a small feature and four schemas
|
|
562
|
+
* resident on every turn would cost more than the memories themselves.
|
|
563
|
+
*/
|
|
564
|
+
export const MEMORY_TOOL_SPEC = Object.freeze({
|
|
565
|
+
name: 'memory',
|
|
566
|
+
description:
|
|
567
|
+
"Save, update or remove a durable fact about the USER in ChatPanel — carried into every "
|
|
568
|
+
+ 'future conversation, on every model and agent. Use it the moment they state a standing '
|
|
569
|
+
+ 'preference ("always be terse"), an identity fact ("call me Alex"), or a constraint about '
|
|
570
|
+
+ 'their work that will still be true next week. Do NOT use it for one-off task details, '
|
|
571
|
+
+ 'anything already obvious from the conversation, or content that belongs in a note. '
|
|
572
|
+
+ 'Keep each memory one short self-contained sentence. Use `forget` when the user says '
|
|
573
|
+
+ 'something is no longer true, and `list` to see what is already stored before adding.',
|
|
574
|
+
parameters: {
|
|
575
|
+
type: 'object',
|
|
576
|
+
properties: {
|
|
577
|
+
action: { type: 'string', enum: ['remember', 'forget', 'list'], description: 'What to do.' },
|
|
578
|
+
text: { type: 'string', description: `The fact, as one short sentence written in the third person ("Prefers terse answers"). Max ${MAX_MEMORY_CHARS} characters. Required for remember; for forget, the memory to drop (its text or id).` },
|
|
579
|
+
kind: { type: 'string', enum: MEMORY_KIND_NAMES, description: MEMORY_KIND_NAMES.map((k) => `${k}: ${MEMORY_KINDS[k].hint}`).join(' ') },
|
|
580
|
+
tags: { type: 'array', items: { type: 'string' }, description: 'Optional short tags for grouping.' },
|
|
581
|
+
},
|
|
582
|
+
required: ['action'],
|
|
583
|
+
},
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
/** The system text that goes with the tool — why it exists, when NOT to reach for it. */
|
|
587
|
+
export function memoryToolSystem() {
|
|
588
|
+
return [
|
|
589
|
+
'You can give the user a memory that survives this conversation, this model and this device'
|
|
590
|
+
+ ' — call the `memory` tool.',
|
|
591
|
+
'Save when they say something durable about themselves, how they want to be helped, or their'
|
|
592
|
+
+ ' ongoing work. Do not save task details, transient state, or anything they would not want'
|
|
593
|
+
+ ' repeated back to them in a month.',
|
|
594
|
+
'Saving is a change to the user\'s own data: say in one short clause what you saved.',
|
|
595
|
+
].join(' ');
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// --------------------------------------------------------------------------
|
|
599
|
+
// Versioning
|
|
600
|
+
// --------------------------------------------------------------------------
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Upcasters, empty at v1 — present so the first schema change is a one-line addition rather
|
|
604
|
+
* than a migration nobody planned for. Same machinery as `upcast.js`.
|
|
605
|
+
*/
|
|
606
|
+
export const MEMORY_UPCASTERS = Object.freeze({});
|
|
607
|
+
|
|
608
|
+
export function upcastMemory(rec) {
|
|
609
|
+
let out = rec;
|
|
610
|
+
for (let v = Number(out?.v) || 1; v < MEMORY_VERSION; v += 1) {
|
|
611
|
+
const up = MEMORY_UPCASTERS[v];
|
|
612
|
+
if (!up) throw new MemoryError(`no upcaster from memory v${v}`);
|
|
613
|
+
out = up(out);
|
|
614
|
+
}
|
|
615
|
+
return out;
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
// --------------------------------------------------------------------------
|
|
619
|
+
// Text helpers
|
|
620
|
+
// --------------------------------------------------------------------------
|
|
621
|
+
|
|
622
|
+
const collapse = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
|
|
623
|
+
const clamp01 = (n) => (Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : 1);
|
|
624
|
+
const trimEnd = (s) => collapse(s).replace(/[.,;:!]+$/, '');
|
|
625
|
+
const clip = (s) => (s.length > MAX_MEMORY_CHARS ? `${s.slice(0, MAX_MEMORY_CHARS - 1).trimEnd()}…` : s);
|
|
626
|
+
const sentence = (s) => {
|
|
627
|
+
const t = trimEnd(s);
|
|
628
|
+
return t ? t[0].toUpperCase() + t.slice(1) : t;
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Split a message into the statements a trigger could apply to. Newlines and list bullets are
|
|
633
|
+
* hard boundaries; sentence-ending punctuation is a soft one. Deliberately does NOT split on
|
|
634
|
+
* commas — "remember that we ship on Friday, not Thursday" is one fact.
|
|
635
|
+
*/
|
|
636
|
+
function splitStatements(text) {
|
|
637
|
+
return String(text || '')
|
|
638
|
+
.split(/\n+/)
|
|
639
|
+
.flatMap((line) => collapse(line).replace(/^[-*+•]\s*|^\d+[.)]\s*/, '').split(/(?<=[.!?])\s+(?=[A-Z"'])/))
|
|
640
|
+
.map(collapse)
|
|
641
|
+
.filter(Boolean);
|
|
642
|
+
}
|
package/src/rrf.js
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// VENDORED from @chatpanel/events/rrf.js — edit there, then copy over.
|
|
2
|
+
// Same pattern as observability.js: one pure module copied in rather than pulling the whole
|
|
3
|
+
// events package. Source of truth: chatpanel-events/rrf.js.
|
|
4
|
+
//
|
|
5
|
+
// rrf.js — Reciprocal Rank Fusion, and the query planning that feeds it.
|
|
6
|
+
//
|
|
7
|
+
// One question rarely makes one good keyword query. "What was the outcome of the Ben tooling
|
|
8
|
+
// demo?" and "Ben demo decisions action items" retrieve different things, and the answer is
|
|
9
|
+
// usually in the union. RRF merges several ranked lists without needing their scores to be
|
|
10
|
+
// comparable — each list contributes 1/(k+rank) — which is exactly the situation here: BM25
|
|
11
|
+
// scores, vector distances and a hot/warm split are all on different scales.
|
|
12
|
+
//
|
|
13
|
+
// Pure and dependency-free, so the identical fusion runs in the extension (hot+warm), the
|
|
14
|
+
// gateway (multi-query search) and any future client. The extension had this privately; it
|
|
15
|
+
// lives here now so the gateway doesn't grow a second, subtly different copy.
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Fuse ranked id lists. `lists` is an array of arrays of ids, each already in rank order.
|
|
19
|
+
* k dampens the head of each list (60 is the standard default). limit 0 = everything.
|
|
20
|
+
*/
|
|
21
|
+
export function fuseRRF(lists, { k = 60, limit = 0 } = {}) {
|
|
22
|
+
const score = new Map();
|
|
23
|
+
for (const list of lists || []) {
|
|
24
|
+
if (!Array.isArray(list)) continue;
|
|
25
|
+
list.forEach((id, rank) => {
|
|
26
|
+
if (id == null) return;
|
|
27
|
+
score.set(id, (score.get(id) || 0) + 1 / (k + rank));
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const out = [...score.entries()].map(([id, s]) => ({ id, score: s })).sort((a, b) => b.score - a.score);
|
|
31
|
+
return limit > 0 ? out.slice(0, limit) : out;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Words that carry no retrieval signal but do dilute BM25 — dropped to build a keyword-only
|
|
35
|
+
// variant of a natural-language question.
|
|
36
|
+
const STOP = new Set(('a an and are as at be been but by can could did do does for from had has have how i if in into is it its me my of on or our ought shall should '
|
|
37
|
+
+ 'so than that the their them then there these they this those to um was we were what when where which who whom why will with would you your about tell show give find get '
|
|
38
|
+
+ 'please could-you was-there did-we').split(/\s+/));
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Turn one natural-language question into a small set of complementary queries — the cheap,
|
|
42
|
+
* deterministic half of query expansion. No model call, so it costs nothing and cannot fail.
|
|
43
|
+
*
|
|
44
|
+
* A CALLING AGENT can do better (it understands the domain), which is why the tools accept an
|
|
45
|
+
* explicit `queries` list; these variants are the floor, not the ceiling.
|
|
46
|
+
*/
|
|
47
|
+
export function planQueries(question, { extra = [], max = 4 } = {}) {
|
|
48
|
+
const q = String(question || '').trim();
|
|
49
|
+
const out = [];
|
|
50
|
+
const seen = new Set();
|
|
51
|
+
const add = (s) => {
|
|
52
|
+
const t = String(s || '').trim().replace(/\s+/g, ' ');
|
|
53
|
+
const key = t.toLowerCase();
|
|
54
|
+
if (t && !seen.has(key)) { seen.add(key); out.push(t); }
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
add(q); // the question as asked
|
|
58
|
+
for (const e of extra) add(e); // whatever the agent proposed — it knows more
|
|
59
|
+
|
|
60
|
+
const words = q.toLowerCase().match(/[a-z0-9][a-z0-9'’_+-]*/g) || [];
|
|
61
|
+
const keywords = words.filter((w) => !STOP.has(w) && w.length > 2);
|
|
62
|
+
if (keywords.length >= 2) add(keywords.join(' ')); // keyword-only: BM25's best shape
|
|
63
|
+
// The rarest-looking terms (longest words are a decent proxy for specificity) — helps when
|
|
64
|
+
// the full question is too broad to rank anything well.
|
|
65
|
+
if (keywords.length > 3) add([...keywords].sort((a, b) => b.length - a.length).slice(0, 3).join(' '));
|
|
66
|
+
|
|
67
|
+
return out.slice(0, Math.max(1, max));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Run several queries through one `search(query, opts)` function and fuse the results by id.
|
|
72
|
+
* `search` returns arrays of { id, ... }; the fused output keeps the richest record seen for
|
|
73
|
+
* each id (so snippets survive) and reports which queries found it — the "why is this here"
|
|
74
|
+
* a reader needs when a multi-query search surfaces something unexpected.
|
|
75
|
+
*/
|
|
76
|
+
export async function multiSearch(queries, search, { limit = 10, k = 60 } = {}) {
|
|
77
|
+
const lists = [];
|
|
78
|
+
const byId = new Map();
|
|
79
|
+
const foundBy = new Map();
|
|
80
|
+
for (const q of queries || []) {
|
|
81
|
+
let rows = [];
|
|
82
|
+
try { rows = (await search(q)) || []; } catch { rows = []; } // one bad query must not sink the rest
|
|
83
|
+
lists.push(rows.map((r) => r.id));
|
|
84
|
+
for (const r of rows) {
|
|
85
|
+
if (!byId.has(r.id) || (!byId.get(r.id).snippet && r.snippet)) byId.set(r.id, r);
|
|
86
|
+
if (!foundBy.has(r.id)) foundBy.set(r.id, []);
|
|
87
|
+
foundBy.get(r.id).push(q);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return fuseRRF(lists, { k, limit }).map(({ id, score }) => ({
|
|
91
|
+
...byId.get(id), id, score, foundBy: foundBy.get(id) || [],
|
|
92
|
+
}));
|
|
93
|
+
}
|
package/src/server.js
CHANGED
|
@@ -31,6 +31,7 @@ import { shaperFor } from './shape.js';
|
|
|
31
31
|
import { startNer } from './ner.js';
|
|
32
32
|
import { installTimestampedConsole } from './log.js';
|
|
33
33
|
import { saveBackupSecret, clearBackupSecret, loadBackupSecret, hasBackupSecret } from './history-store.js';
|
|
34
|
+
import { createMemoryStore } from './memory-store.js';
|
|
34
35
|
import { createHistoryStore } from './sqlite-store.js';
|
|
35
36
|
import { ingestBackups } from './backup-ingest.js';
|
|
36
37
|
import * as nerEngine from './ner-engine.js';
|
|
@@ -43,17 +44,19 @@ import { publicConfig, applyConfigPatch, applyNerModelSelection, persistConfig,
|
|
|
43
44
|
import { resolveDestination, aggregateModelsAsync } from './router.js';
|
|
44
45
|
import { makeAccessEvent } from './observability.js';
|
|
45
46
|
import { createPersistentAccessLog } from './access-log-store.js';
|
|
47
|
+
import { planQueries, multiSearch } from './rrf.js';
|
|
46
48
|
import * as openai from './openai.js';
|
|
47
49
|
import * as responses from './responses.js';
|
|
48
50
|
import * as anthropic from './anthropic.js';
|
|
49
51
|
|
|
50
|
-
export const VERSION = '0.6.
|
|
52
|
+
export const VERSION = '0.6.45';
|
|
51
53
|
|
|
52
54
|
// WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
|
|
53
55
|
// store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
|
|
54
56
|
// Persistent + memory-mapped, so a restart needs no re-ingest (no cold start).
|
|
55
57
|
// See docs/architecture-data-tiers.
|
|
56
58
|
const historyStore = await createHistoryStore();
|
|
59
|
+
const memoryStore = await createMemoryStore();
|
|
57
60
|
|
|
58
61
|
// OBSERVABILITY — a ring of "which agent read what, when", persisted across restarts (the
|
|
59
62
|
// gateway updates often; an empty panel after each restart reads as "nothing is set up").
|
|
@@ -62,7 +65,7 @@ const historyStore = await createHistoryStore();
|
|
|
62
65
|
// metadata only — client/tool/ms + a REDACTED note (a search query's text is never in it).
|
|
63
66
|
const accessLog = createPersistentAccessLog();
|
|
64
67
|
|
|
65
|
-
const KNOWN_AGENTS = new Set(['codex', 'claude', 'opencode', 'pi', 'kiro', 'antigravity']);
|
|
68
|
+
const KNOWN_AGENTS = new Set(['codex', 'claude', 'opencode', 'pi', 'kiro', 'antigravity', 'hermes']);
|
|
66
69
|
|
|
67
70
|
// Auto-narrow: arm only the top-K most-relevant MCP tools per turn (speed). Mirrors
|
|
68
71
|
// the extension's AUTO mode via the SAME shared ranker. We narrow only tools whose
|
|
@@ -538,6 +541,15 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
538
541
|
if ((pathname === '/v1/history/ingest' || pathname === '/v1/history/clear') && req.method === 'POST' && !isAdminAuthorized(req)) {
|
|
539
542
|
return sendJson(res, 403, { error: { message: 'history write — extension origin or gateway token required', type: 'forbidden' } });
|
|
540
543
|
}
|
|
544
|
+
// Memory WRITES follow the same rule as history ingest, for a stronger reason: a memory is
|
|
545
|
+
// carried into every future turn on every model, so a drive-by localhost page that could
|
|
546
|
+
// POST one would be installing a standing instruction, not injecting a single record.
|
|
547
|
+
// Reads stay open — that IS the product (Codex and Claude Code recall through it).
|
|
548
|
+
if ((pathname === '/v1/memory/remember' || pathname === '/v1/memory/forget'
|
|
549
|
+
|| pathname === '/v1/memory/sync' || pathname === '/v1/memory/clear')
|
|
550
|
+
&& req.method === 'POST' && !isAdminAuthorized(req)) {
|
|
551
|
+
return sendJson(res, 403, { error: { message: 'memory write — extension origin or gateway token required', type: 'forbidden' } });
|
|
552
|
+
}
|
|
541
553
|
// The access log is who-read-what — sensitive, and writable only by the local MCP
|
|
542
554
|
// process (which sends the gateway token). Extension Origin or token for both the
|
|
543
555
|
// read (dashboard) and the report (MCP child); a drive-by page has neither.
|
|
@@ -597,6 +609,80 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
597
609
|
return sendJson(res, 400, { error: { message: `ingest failed: ${e.message}`, type: 'ingest_error' } });
|
|
598
610
|
}
|
|
599
611
|
}
|
|
612
|
+
// --- MEMORY. Small, durable facts about the user, reachable by every local agent.
|
|
613
|
+
// GET /v1/memory/list → { memories }
|
|
614
|
+
// POST /v1/memory/recall { text, scopes } → { memories, block }
|
|
615
|
+
// POST /v1/memory/remember { text, kind, … }→ { action, record }
|
|
616
|
+
// POST /v1/memory/forget { query } → { removed }
|
|
617
|
+
// POST /v1/memory/sync { upserts, removes } → { size, merged, memories }
|
|
618
|
+
if (pathname === '/v1/memory/list' && req.method === 'GET') {
|
|
619
|
+
return sendJson(res, 200, { ok: true, size: memoryStore.size, memories: memoryStore.list() });
|
|
620
|
+
}
|
|
621
|
+
if (pathname === '/v1/memory/recall' && req.method === 'POST') {
|
|
622
|
+
try {
|
|
623
|
+
const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
|
|
624
|
+
const got = memoryStore.recall({
|
|
625
|
+
text: String(body.text || ''),
|
|
626
|
+
scopes: Array.isArray(body.scopes) && body.scopes.length ? body.scopes.map(String) : ['global'],
|
|
627
|
+
limit: Number(body.limit) || 0,
|
|
628
|
+
maxChars: Number(body.maxChars) || 0,
|
|
629
|
+
});
|
|
630
|
+
return sendJson(res, 200, { ok: true, size: memoryStore.size, ...got });
|
|
631
|
+
} catch (e) {
|
|
632
|
+
return sendJson(res, 400, { error: { message: `recall failed: ${e.message}`, type: 'memory_error' } });
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
if (pathname === '/v1/memory/remember' && req.method === 'POST') {
|
|
636
|
+
try {
|
|
637
|
+
const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
|
|
638
|
+
const out = memoryStore.remember({
|
|
639
|
+
text: String(body.text || ''),
|
|
640
|
+
kind: body.kind ? String(body.kind) : 'fact',
|
|
641
|
+
scope: body.scope ? String(body.scope) : 'global',
|
|
642
|
+
tags: Array.isArray(body.tags) ? body.tags.map(String) : [],
|
|
643
|
+
// WHO PROPOSED IT, always recorded. There is no confirm dialog on a CLI, so
|
|
644
|
+
// attribution plus an inspectable list in the extension IS the accountability —
|
|
645
|
+
// see the MCP server's note on why writes are allowed but never anonymous.
|
|
646
|
+
source: {
|
|
647
|
+
via: String(body.source?.via || 'mcp'),
|
|
648
|
+
surface: String(body.source?.surface || 'mcp'),
|
|
649
|
+
agent: String(body.source?.agent || ''),
|
|
650
|
+
ref: String(body.source?.ref || ''),
|
|
651
|
+
},
|
|
652
|
+
});
|
|
653
|
+
return sendJson(res, 200, { ok: true, action: out.action, record: out.record, replaced: out.replaces || null, size: memoryStore.size });
|
|
654
|
+
} catch (e) {
|
|
655
|
+
return sendJson(res, 400, { error: { message: e.message, type: 'memory_error' } });
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
if (pathname === '/v1/memory/forget' && req.method === 'POST') {
|
|
659
|
+
try {
|
|
660
|
+
const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
|
|
661
|
+
const { removed } = memoryStore.forget(String(body.query || ''));
|
|
662
|
+
return sendJson(res, 200, { ok: true, removed, size: memoryStore.size });
|
|
663
|
+
} catch (e) {
|
|
664
|
+
return sendJson(res, 400, { error: { message: e.message, type: 'memory_error' } });
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
if (pathname === '/v1/memory/sync' && req.method === 'POST') {
|
|
668
|
+
try {
|
|
669
|
+
const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
|
|
670
|
+
const out = memoryStore.bulk({
|
|
671
|
+
upserts: Array.isArray(body.upserts) ? body.upserts : [],
|
|
672
|
+
removes: Array.isArray(body.removes) ? body.removes : [],
|
|
673
|
+
});
|
|
674
|
+
// The full set comes BACK, so one round trip is the whole two-way merge: the client
|
|
675
|
+
// pushes what it has and receives what the agents wrote. Convergent because both
|
|
676
|
+
// sides reconcile with the same function.
|
|
677
|
+
return sendJson(res, 200, { ok: true, ...out, memories: memoryStore.list() });
|
|
678
|
+
} catch (e) {
|
|
679
|
+
return sendJson(res, 400, { error: { message: `memory sync failed: ${e.message}`, type: 'memory_error' } });
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
if (pathname === '/v1/memory/clear' && req.method === 'POST') {
|
|
683
|
+
return sendJson(res, 200, { ok: true, dropped: memoryStore.clear(), size: memoryStore.size });
|
|
684
|
+
}
|
|
685
|
+
|
|
600
686
|
if (pathname === '/v1/history/clear' && req.method === 'POST') {
|
|
601
687
|
const dropped = historyStore.clear();
|
|
602
688
|
return sendJson(res, 200, { ok: true, dropped, size: historyStore.size });
|
|
@@ -616,6 +702,38 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
616
702
|
return sendJson(res, 400, { error: { message: `search failed: ${e.message}`, type: 'search_error' } });
|
|
617
703
|
}
|
|
618
704
|
}
|
|
705
|
+
// SMART SEARCH — one round trip that expands the question into several complementary
|
|
706
|
+
// queries, runs them all, and RRF-fuses the results. A natural-language question is a
|
|
707
|
+
// poor BM25 query; asking three ways and fusing beats asking once, and doing it here
|
|
708
|
+
// means the agent pays one call instead of probing repeatedly.
|
|
709
|
+
if (pathname === '/v1/history/smart-search' && req.method === 'POST') {
|
|
710
|
+
try {
|
|
711
|
+
const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
|
|
712
|
+
const question = String(body.question || body.query || '');
|
|
713
|
+
// The agent's own formulations lead (it understands the domain); planQueries adds
|
|
714
|
+
// deterministic variants and dedupes.
|
|
715
|
+
const queries = planQueries(question, {
|
|
716
|
+
extra: Array.isArray(body.queries) ? body.queries.map(String) : [],
|
|
717
|
+
max: Math.min(6, Math.max(1, Number(body.maxQueries) || 4)),
|
|
718
|
+
});
|
|
719
|
+
const filters = {
|
|
720
|
+
type: body.type ? String(body.type) : null,
|
|
721
|
+
since: body.since != null ? Number(body.since) : null,
|
|
722
|
+
before: body.before != null ? Number(body.before) : null,
|
|
723
|
+
};
|
|
724
|
+
const perQuery = Math.min(30, Math.max(5, Number(body.limit) || 10) * 2);
|
|
725
|
+
const results = await multiSearch(
|
|
726
|
+
queries,
|
|
727
|
+
(q) => historyStore.search(q, { limit: perQuery, ...filters }),
|
|
728
|
+
{ limit: Math.min(50, Math.max(1, Number(body.limit) || 10)) },
|
|
729
|
+
);
|
|
730
|
+
return sendJson(res, 200, {
|
|
731
|
+
ok: true, size: historyStore.size, newest: historyStore.newest, queries, results,
|
|
732
|
+
});
|
|
733
|
+
} catch (e) {
|
|
734
|
+
return sendJson(res, 400, { error: { message: `smart search failed: ${e.message}`, type: 'search_error' } });
|
|
735
|
+
}
|
|
736
|
+
}
|
|
619
737
|
// Graph navigation — records most connected to a given one.
|
|
620
738
|
if (pathname === '/v1/history/related' && req.method === 'GET') {
|
|
621
739
|
const id = String(url.searchParams.get('id') || '');
|