@jossuealcala/madre 0.3.3 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +497 -3
- package/CONTRIBUTING.md +3 -1
- package/README.md +68 -186
- package/SECURITY.md +2 -1
- package/bin/madre.mjs +56 -13
- package/docs/INTERNALS.md +16 -0
- package/docs/REFERENCE.md +249 -0
- package/docs/SDK.md +121 -0
- package/docs/room.png +0 -0
- package/docs/sdk/hello-module.mjs +51 -0
- package/package.json +9 -1
- package/public/app.js +3979 -867
- package/public/es.js +2258 -0
- package/public/i18n.js +66 -0
- package/public/index.html +96 -15
- package/public/inquiry.js +220 -0
- package/public/resay.js +77 -0
- package/public/styles.css +622 -65
- package/public/troubleshooting.js +255 -46
- package/src/adapters/claude.mjs +2 -1
- package/src/adapters/codex.mjs +2 -1
- package/src/adapters/gemini.mjs +6 -5
- package/src/adapters/opencode.mjs +2 -1
- package/src/adapters/process.mjs +79 -20
- package/src/asking.mjs +128 -0
- package/src/auth-probe.mjs +58 -1
- package/src/chats.mjs +193 -0
- package/src/checkpoint.mjs +1 -1
- package/src/cold.mjs +56 -0
- package/src/commands.mjs +6 -0
- package/src/conversation-context.mjs +35 -3
- package/src/credentials.mjs +145 -0
- package/src/dataset.mjs +56 -4
- package/src/distiller.mjs +12 -5
- package/src/event-store.mjs +14 -8
- package/src/exam.mjs +240 -0
- package/src/extensions.mjs +3 -2
- package/src/eyecat-watch.mjs +100 -0
- package/src/eyecat.mjs +169 -0
- package/src/i18n.mjs +47 -0
- package/src/image-studio.mjs +2 -0
- package/src/launch.mjs +61 -0
- package/src/maturity.mjs +94 -0
- package/src/mcp/image-server.mjs +36 -3
- package/src/mcp/memory-server.mjs +1 -1
- package/src/memory.mjs +325 -17
- package/src/modules/ahp.mjs +9 -7
- package/src/modules/ash.mjs +36 -0
- package/src/modules/git-pulse.mjs +5 -3
- package/src/modules/helpers.mjs +31 -0
- package/src/modules/image-studio.mjs +10 -4
- package/src/modules/index.mjs +141 -9
- package/src/modules/ollama.mjs +66 -10
- package/src/modules/playwright.mjs +44 -23
- package/src/modules/ripley.mjs +5 -3
- package/src/modules/sdk.mjs +93 -2
- package/src/modules/updates.mjs +81 -0
- package/src/ollama.mjs +5 -2
- package/src/outbound.mjs +297 -0
- package/src/privacy.mjs +54 -7
- package/src/room/context.mjs +4 -4
- package/src/room/economy.mjs +161 -0
- package/src/room/prompt.mjs +118 -46
- package/src/room.mjs +443 -44
- package/src/runtime-detection.mjs +27 -8
- package/src/sentinel-errors.mjs +19 -1
- package/src/server.mjs +709 -71
- package/src/setup.mjs +1 -1
- package/src/updates.mjs +4 -2
- package/src/usage-sentinel.mjs +13 -8
- package/src/verdict.mjs +74 -0
- package/src/ashcode.mjs +0 -64
- package/src/modules/ashcode.mjs +0 -28
package/src/maturity.mjs
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// How grown a room is, and how far from being worth training on.
|
|
2
|
+
//
|
|
3
|
+
// Pure functions. The room hands in what it has; nothing here reads a file or calls a model.
|
|
4
|
+
//
|
|
5
|
+
// This replaces a bar that filled toward three hundred. That number was a rule of thumb from
|
|
6
|
+
// somebody else's paper, not a measurement of this room, and a corpus of three hundred pairs all
|
|
7
|
+
// about the same afternoon teaches less than eighty that are not. What is measured here is what
|
|
8
|
+
// can actually be counted about this archive, and each reading says plainly what would raise it.
|
|
9
|
+
|
|
10
|
+
// How much of the whole each reading is worth. Volume counts, but it is one voice of six: a room
|
|
11
|
+
// can be large and still be narrow, unjudged and lopsided.
|
|
12
|
+
import { t } from './i18n.mjs';
|
|
13
|
+
|
|
14
|
+
export const WEIGHTS = { volume: 0.2, coverage: 0.2, weave: 0.15, judgement: 0.2, balance: 0.15, upkeep: 0.1 };
|
|
15
|
+
export const VOLUME_TARGET = 300; // the usual floor for a small adapter, and nothing more than that
|
|
16
|
+
|
|
17
|
+
const share = (part, whole) => (whole > 0 ? Math.min(1, Math.max(0, part / whole)) : 0);
|
|
18
|
+
const round = (value) => Number(value.toFixed(3));
|
|
19
|
+
|
|
20
|
+
// Six readings, each 0 to 1, each with the one thing that would raise it.
|
|
21
|
+
export function maturity({ readiness = null, notes = [], links = [], stats = null } = {}) {
|
|
22
|
+
const standing = notes.filter((note) => note.kind !== 'aberration' && !note.refutedBy);
|
|
23
|
+
const pairs = Number(readiness?.pairs ?? 0);
|
|
24
|
+
const rated = Number(readiness?.good ?? 0) + Number(readiness?.bad ?? 0);
|
|
25
|
+
const turns = Number(readiness?.turns ?? 0) + Number(readiness?.delegated ?? 0);
|
|
26
|
+
const noteShare = pairs > 0 ? Number(readiness?.notes ?? 0) / pairs : 0;
|
|
27
|
+
|
|
28
|
+
const recalled = standing.filter((note) => Number(note.recalled ?? 0) > 0).length;
|
|
29
|
+
const linked = new Set(links.flatMap((link) => [link.a, link.b]));
|
|
30
|
+
const connected = standing.filter((note) => linked.has(note.id)).length;
|
|
31
|
+
const pending = Number(stats?.pending ?? 0);
|
|
32
|
+
const entries = Number(stats?.entries ?? 0);
|
|
33
|
+
|
|
34
|
+
const signals = [
|
|
35
|
+
{
|
|
36
|
+
id: 'volume', label: t('HOW MUCH THERE IS'),
|
|
37
|
+
value: round(share(pairs, VOLUME_TARGET)),
|
|
38
|
+
detail: t('{pairs} of about {target} exchanges worth training on', { pairs, target: VOLUME_TARGET }),
|
|
39
|
+
next: t('Use the room. Nothing else fills this.'),
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
id: 'coverage', label: t('HOW MUCH OF IT GETS USED'),
|
|
43
|
+
value: round(share(recalled, standing.length)),
|
|
44
|
+
detail: t('{recalled} of {total} memories have been reached for at least once', { recalled, total: standing.length }),
|
|
45
|
+
next: t('Memories nobody has needed may be noise, or may simply not have come up yet. Ask the room about older decisions and see which ones answer.'),
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: 'weave', label: t('HOW WOVEN IT IS'),
|
|
49
|
+
value: round(share(connected, standing.length)),
|
|
50
|
+
detail: t('{connected} of {total} memories share a subject with another', { connected, total: standing.length }),
|
|
51
|
+
next: t('An archive of unrelated notes is a list. Depth comes from returning to the same subjects.'),
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
id: 'judgement', label: t('HOW MUCH OF IT YOU JUDGED'),
|
|
55
|
+
// A tenth rated is enough to steer a small adapter; asking for all of it would never be met.
|
|
56
|
+
value: round(share(rated, Math.max(1, pairs * 0.1))),
|
|
57
|
+
detail: t('{rated} of {pairs} replies rated', { rated, pairs }),
|
|
58
|
+
next: t('Rate replies with the thumbs on a bubble. A corpus nobody judged teaches what the agents said, not what you approved.'),
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
id: 'balance', label: t('HOW MUCH OF IT IS REAL WORK'),
|
|
62
|
+
// Half recall pairs is healthy; a corpus that is mostly recall teaches recitation. A room
|
|
63
|
+
// with no corpus at all is not balanced, it is empty, and saying otherwise would show
|
|
64
|
+
// progress where there is none.
|
|
65
|
+
value: pairs > 0 ? round(1 - Math.min(1, Math.max(0, (noteShare - 0.5) / 0.5))) : 0,
|
|
66
|
+
detail: t('{share}% of the corpus is recall questions, {turns} exchanges are real work', { share: Math.round(noteShare * 100), turns }),
|
|
67
|
+
next: t('Recall pairs are made from notes and cost nothing, so they pile up. Work in the room to balance them.'),
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
id: 'upkeep', label: t('HOW CURRENT IT IS'),
|
|
71
|
+
value: round(entries > 0 ? 1 - share(pending, Math.max(1, entries * 0.15)) : 0),
|
|
72
|
+
detail: t('{pending} exchanges nobody has distilled yet, of {entries}', { pending, entries }),
|
|
73
|
+
next: t('The archivist catches up on its own. A backlog that never clears means it cannot run: check who is allowed to distil.'),
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
|
|
77
|
+
const score = round(signals.reduce((sum, signal) => sum + signal.value * (WEIGHTS[signal.id] ?? 0), 0));
|
|
78
|
+
return { score, stage: stageOf(score), signals, weakest: [...signals].sort((a, b) => a.value - b.value)[0]?.id ?? null };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// What to call it. A name is not a measurement, but a number alone tells nobody whether to act.
|
|
82
|
+
export const STAGES = [
|
|
83
|
+
{ at: 0.85, id: 'mature', label: 'MATURE', says: 'Worth training on. Export and run the recipe.' },
|
|
84
|
+
{ at: 0.6, id: 'working', label: 'WORKING', says: 'Usable, and it will be better for waiting.' },
|
|
85
|
+
{ at: 0.35, id: 'forming', label: 'FORMING', says: 'It has a shape. Too thin to train on.' },
|
|
86
|
+
{ at: 0.12, id: 'sparse', label: 'SPARSE', says: 'A few things remembered, little connecting them.' },
|
|
87
|
+
{ at: 0, id: 'empty', label: 'EMPTY', says: 'Nothing has been distilled yet.' },
|
|
88
|
+
];
|
|
89
|
+
// The table holds the English, and the stage is said in the room's language when it is asked
|
|
90
|
+
// for: this list is built when the file is imported, and the room learns its language after.
|
|
91
|
+
export function stageOf(score) {
|
|
92
|
+
const stage = STAGES.find((one) => score >= one.at) ?? STAGES.at(-1);
|
|
93
|
+
return { ...stage, label: t(stage.label), says: t(stage.says) };
|
|
94
|
+
}
|
package/src/mcp/image-server.mjs
CHANGED
|
@@ -9,14 +9,16 @@
|
|
|
9
9
|
// Environment:
|
|
10
10
|
// PULSE_IMAGE_OUT_DIR required, absolute lease directory
|
|
11
11
|
// PULSE_IMAGE_MODEL default gemini-2.5-flash-image
|
|
12
|
-
// GEMINI_API_KEY optional; otherwise the macOS keychain entry the
|
|
12
|
+
// GEMINI_API_KEY optional; otherwise ~/.gemini/.env, then the macOS keychain entry the
|
|
13
13
|
// Gemini CLI stores (service gemini-cli-api-key)
|
|
14
14
|
// PULSE_IMAGE_FAKE=1 write a 1×1 PNG without calling Google (tests)
|
|
15
15
|
|
|
16
16
|
import { execFile } from 'node:child_process';
|
|
17
|
-
import { mkdir, writeFile, realpath } from 'node:fs/promises';
|
|
17
|
+
import { mkdir, readFile, writeFile, realpath } from 'node:fs/promises';
|
|
18
|
+
import { homedir } from 'node:os';
|
|
18
19
|
import { basename, extname, join, resolve, sep } from 'node:path';
|
|
19
20
|
import { promisify } from 'node:util';
|
|
21
|
+
import { OutboundLog } from '../outbound.mjs';
|
|
20
22
|
|
|
21
23
|
const execFileAsync = promisify(execFile);
|
|
22
24
|
const SERVER_NAME = 'pulse-image';
|
|
@@ -24,9 +26,30 @@ const SERVER_VERSION = '0.1.0';
|
|
|
24
26
|
const DEFAULT_MODEL = 'gemini-2.5-flash-image';
|
|
25
27
|
const FAKE_PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', 'base64');
|
|
26
28
|
|
|
29
|
+
// Where a Gemini key can be, in the order the room should trust them: what this process was
|
|
30
|
+
// given, then the file the CLI reads on every run — which is also the file MADRE itself writes
|
|
31
|
+
// when a key is pasted into CONNECTIONS — and last the macOS keychain, which only exists when
|
|
32
|
+
// the CLI's own sign-in put it there. Leaving the file out meant MADRE saved a key in a place it
|
|
33
|
+
// then refused to look, and on Linux, where there is no keychain, that was the only place it
|
|
34
|
+
// could have been: Image Studio could never find a key there at all.
|
|
35
|
+
async function keyFromGeminiEnvFile(env) {
|
|
36
|
+
const home = env.GEMINI_CLI_HOME ?? homedir();
|
|
37
|
+
let text;
|
|
38
|
+
try { text = await readFile(join(home, '.gemini', '.env'), 'utf8'); } catch { return null; }
|
|
39
|
+
for (const line of text.split('\n')) {
|
|
40
|
+
const match = line.match(/^\s*(?:export\s+)?(GEMINI_API_KEY|GOOGLE_API_KEY)\s*=\s*(.*)$/);
|
|
41
|
+
if (!match) continue;
|
|
42
|
+
const value = match[2].trim().replace(/^(['"])(.*)\1$/, '$2').trim();
|
|
43
|
+
if (value) return value;
|
|
44
|
+
}
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
|
|
27
48
|
export async function resolveGeminiKey(env = process.env) {
|
|
28
49
|
if (env.GEMINI_API_KEY) return env.GEMINI_API_KEY;
|
|
29
50
|
if (env.GOOGLE_API_KEY) return env.GOOGLE_API_KEY;
|
|
51
|
+
const fromFile = await keyFromGeminiEnvFile(env);
|
|
52
|
+
if (fromFile) return fromFile;
|
|
30
53
|
if (process.platform !== 'darwin') return null;
|
|
31
54
|
// The keychain occasionally answers empty under concurrent reads; one retry.
|
|
32
55
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
@@ -62,6 +85,16 @@ export function explainGoogleError(status, body) {
|
|
|
62
85
|
return { code: 'ERROR', message: message || `Google returned HTTP ${status}.` };
|
|
63
86
|
}
|
|
64
87
|
|
|
88
|
+
// The image studio runs in a process of its own, so it writes its own line into the room's
|
|
89
|
+
// outbound log. Without this, the one request MADRE makes that carries somebody's words to
|
|
90
|
+
// Google would be the one request the log could not see.
|
|
91
|
+
let outboundLog = null;
|
|
92
|
+
function outboundFetch(fetchImpl, file) {
|
|
93
|
+
if (!file) return fetchImpl;
|
|
94
|
+
outboundLog ??= new OutboundLog({ file });
|
|
95
|
+
return outboundLog.watch(fetchImpl);
|
|
96
|
+
}
|
|
97
|
+
|
|
65
98
|
export async function generateImage({ prompt, fileName, outDir, model = process.env.PULSE_IMAGE_MODEL || DEFAULT_MODEL, env = process.env, fetchImpl = fetch }) {
|
|
66
99
|
if (!prompt || typeof prompt !== 'string') throw Object.assign(new Error('A text prompt is required.'), { code: 'INVALID' });
|
|
67
100
|
const root = await realpath(outDir).catch(() => null);
|
|
@@ -76,7 +109,7 @@ export async function generateImage({ prompt, fileName, outDir, model = process.
|
|
|
76
109
|
}
|
|
77
110
|
const key = await resolveGeminiKey(env);
|
|
78
111
|
if (!key) throw Object.assign(new Error('No Gemini API key: set GEMINI_API_KEY or sign in with the Gemini CLI (/auth → API key).'), { code: 'NO_KEY' });
|
|
79
|
-
const response = await fetchImpl(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent`, {
|
|
112
|
+
const response = await outboundFetch(fetchImpl, env.PULSE_OUTBOUND_LOG)(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent`, {
|
|
80
113
|
method: 'POST',
|
|
81
114
|
headers: { 'content-type': 'application/json', 'x-goog-api-key': key },
|
|
82
115
|
body: JSON.stringify({ contents: [{ parts: [{ text: prompt }] }], generationConfig: { responseModalities: ['IMAGE'] } }),
|
|
@@ -57,7 +57,7 @@ export const TOOLS = [
|
|
|
57
57
|
name: 'memory_note',
|
|
58
58
|
description: 'Save one durable memory of this room, ONLY when the human explicitly asks you to remember, note or save something (memories are otherwise distilled automatically; never save on your own initiative). One self-contained sentence, at most 240 characters, in the language the room uses, with the ledger sequences it comes from when you know them. Refused in a GHOST turn.',
|
|
59
59
|
inputSchema: { type: 'object', properties: {
|
|
60
|
-
kind: { type: 'string', enum: MEMORY_KINDS, description: 'decision, fact, preference or
|
|
60
|
+
kind: { type: 'string', enum: MEMORY_KINDS, description: 'decision, fact, preference, question, or aberration for a claim this room established is false.' },
|
|
61
61
|
text: { type: 'string', description: 'The memory, one sentence, names and numbers exact.' },
|
|
62
62
|
sources: { type: 'array', items: { type: 'integer' }, description: 'Ledger sequences it comes from, if known.' },
|
|
63
63
|
}, required: ['kind', 'text'] },
|
package/src/memory.mjs
CHANGED
|
@@ -15,8 +15,37 @@ import { dirname } from 'node:path';
|
|
|
15
15
|
import { messageEntry } from './conversation-context.mjs';
|
|
16
16
|
import { cosine, toBlob, fromBlob } from './embeddings.mjs';
|
|
17
17
|
|
|
18
|
+
// How many recall rows a room keeps. Six or so per turn, so this is months of work.
|
|
19
|
+
export const RECALL_HISTORY = 20000;
|
|
20
|
+
|
|
21
|
+
// Spreading activation. Two memories that keep arriving in the same turn are associated, however
|
|
22
|
+
// differently they read: the room's own work says so. The strength is Jaccard over the turns
|
|
23
|
+
// where each was found by the search on its own merits — so a memory the room reaches for
|
|
24
|
+
// constantly does not end up attached to everything, and one that only ever arrived by cascade
|
|
25
|
+
// never votes on what comes next. Without that second rule the network would feed itself into a
|
|
26
|
+
// clique within a few days.
|
|
27
|
+
export const CASCADE_FLOOR = 0.34; // of the turns where either appeared, they appeared together
|
|
28
|
+
export const CASCADE_MIN_TIMES = 2; // once is a coincidence
|
|
29
|
+
export const CASCADE_RESERVE = 2; // slots the search does not get to fill on its own
|
|
30
|
+
|
|
18
31
|
export const MEMORY_SCHEMA_VERSION = 3;
|
|
19
|
-
|
|
32
|
+
// An aberration is the one kind that is not knowledge. It is a claim the room decided is false:
|
|
33
|
+
// a hallucination, an unfounded assertion, a distortion, or a memory that drifted away from what
|
|
34
|
+
// the project actually settled. It is kept because it is worth training against, and it is kept
|
|
35
|
+
// out of every turn because a room that recalls its own hallucinations repeats them.
|
|
36
|
+
export const ABERRATION = 'aberration';
|
|
37
|
+
export const MEMORY_KINDS = ['decision', 'fact', 'preference', 'question', ABERRATION];
|
|
38
|
+
// What the room will hand an agent: knowledge that still stands. Everything else is archive.
|
|
39
|
+
export const STANDING_KINDS = MEMORY_KINDS.filter((kind) => kind !== ABERRATION);
|
|
40
|
+
|
|
41
|
+
// The dedup key. An aberration almost always quotes the claim it refutes word for word, so on a
|
|
42
|
+
// shared key the archive would silently drop the refutation as a duplicate of the thing it is
|
|
43
|
+
// refuting. Aberrations are keyed in their own space: one of each still dedupes, and a false
|
|
44
|
+
// claim can sit beside the note it takes down.
|
|
45
|
+
export function memoryKey(kind, text) {
|
|
46
|
+
const norm = normalizeMemory(text);
|
|
47
|
+
return kind === ABERRATION ? `${ABERRATION}:${norm}` : norm;
|
|
48
|
+
}
|
|
20
49
|
|
|
21
50
|
// Words that carry no meaning for recall, in the two languages the rooms speak.
|
|
22
51
|
const STOPWORDS = new Set(('the and for with that this from what which where when have has are was were will would could should about into your you our their there here they them then than also just like only over under some any all not but can does did done been being make made use used using please into onto ' +
|
|
@@ -112,6 +141,10 @@ export class RoomMemory {
|
|
|
112
141
|
this.#setMeta = this.#db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
|
|
113
142
|
this.#insert = this.#db.prepare('INSERT OR IGNORE INTO entries (sequence, event_id, timestamp, type, role, sender, target, message_id, text) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
114
143
|
if (this.#metaValue('schema') === null) this.#setMeta.run('schema', String(MEMORY_SCHEMA_VERSION));
|
|
144
|
+
// When this room started keeping the trail of which memories travel together. Counters from
|
|
145
|
+
// before that day are real; the company they kept was never written down, and a card must be
|
|
146
|
+
// able to say so instead of showing an empty list as if a memory had always been alone.
|
|
147
|
+
if (this.#metaValue('recalls_since') === null) this.#setMeta.run('recalls_since', new Date().toISOString());
|
|
115
148
|
}
|
|
116
149
|
|
|
117
150
|
#createSchema() {
|
|
@@ -145,7 +178,9 @@ export class RoomMemory {
|
|
|
145
178
|
from_sequence INTEGER NOT NULL,
|
|
146
179
|
through_sequence INTEGER NOT NULL,
|
|
147
180
|
sources TEXT NOT NULL,
|
|
148
|
-
agent TEXT NOT NULL
|
|
181
|
+
agent TEXT NOT NULL,
|
|
182
|
+
recalled INTEGER NOT NULL DEFAULT 0,
|
|
183
|
+
last_recalled TEXT
|
|
149
184
|
);
|
|
150
185
|
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(text, kind, content='memories', content_rowid='id', tokenize='trigram');
|
|
151
186
|
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
|
@@ -154,9 +189,30 @@ export class RoomMemory {
|
|
|
154
189
|
CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
|
|
155
190
|
INSERT INTO memories_fts(memories_fts, rowid, text, kind) VALUES ('delete', old.id, old.text, old.kind);
|
|
156
191
|
END;
|
|
192
|
+
-- Every time the room reaches for a memory, one row. A turn recalls several at once and
|
|
193
|
+
-- they share a batch, so the archive knows not only how often a note was used but which
|
|
194
|
+
-- other notes travelled with it: two memories that keep arriving together are talking.
|
|
195
|
+
CREATE TABLE IF NOT EXISTS recalls (
|
|
196
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
197
|
+
memory_id INTEGER NOT NULL,
|
|
198
|
+
batch TEXT NOT NULL,
|
|
199
|
+
at TEXT NOT NULL,
|
|
200
|
+
agent TEXT,
|
|
201
|
+
turn TEXT,
|
|
202
|
+
via TEXT NOT NULL DEFAULT 'search'
|
|
203
|
+
);
|
|
204
|
+
CREATE INDEX IF NOT EXISTS recalls_memory ON recalls(memory_id, id DESC);
|
|
205
|
+
CREATE INDEX IF NOT EXISTS recalls_batch ON recalls(batch);
|
|
157
206
|
CREATE TABLE IF NOT EXISTS entry_vectors (sequence INTEGER PRIMARY KEY, model TEXT NOT NULL, vec BLOB NOT NULL);
|
|
158
207
|
CREATE TABLE IF NOT EXISTS memory_vectors (id INTEGER PRIMARY KEY, model TEXT NOT NULL, vec BLOB NOT NULL);
|
|
159
208
|
`);
|
|
209
|
+
// Older files have recalls without a reason; everything written before the cascade existed
|
|
210
|
+
// was found by the search, which is what the default says.
|
|
211
|
+
try { this.#db.exec("ALTER TABLE recalls ADD COLUMN via TEXT NOT NULL DEFAULT 'search'"); } catch { /* already there */ }
|
|
212
|
+
// Older files predate the recall counters; adding them is harmless and keeps the notes.
|
|
213
|
+
for (const column of ['recalled INTEGER NOT NULL DEFAULT 0', 'last_recalled TEXT']) {
|
|
214
|
+
try { this.#db.exec(`ALTER TABLE memories ADD COLUMN ${column}`); } catch { /* already there */ }
|
|
215
|
+
}
|
|
160
216
|
// Older files: each entry remembers whether it was distilled (the old watermark seeds it).
|
|
161
217
|
const entryColumns = this.#db.prepare('PRAGMA table_info(entries)').all().map((column) => column.name);
|
|
162
218
|
if (entryColumns.length && !entryColumns.includes('distilled')) {
|
|
@@ -168,6 +224,16 @@ export class RoomMemory {
|
|
|
168
224
|
const columns = this.#db.prepare('PRAGMA table_info(memories)').all().map((column) => column.name);
|
|
169
225
|
if (!columns.includes('origin')) this.#db.exec("ALTER TABLE memories ADD COLUMN origin TEXT NOT NULL DEFAULT 'distilled'");
|
|
170
226
|
if (!columns.includes('message_id')) this.#db.exec('ALTER TABLE memories ADD COLUMN message_id TEXT');
|
|
227
|
+
// Aberrations, and what they do to the notes they refute. A memory's text is never rewritten
|
|
228
|
+
// here: what a refutation changes is its standing, not what it said.
|
|
229
|
+
// contradicts on an aberration, the note it refutes
|
|
230
|
+
// correction on an aberration, what is true instead, when the room knows
|
|
231
|
+
// detector who caught it: a person, the archivist, or a watcher
|
|
232
|
+
// confidence 0..1 from whoever caught it
|
|
233
|
+
// refuted_by on a note, the aberration that took it out of circulation
|
|
234
|
+
for (const column of ['contradicts INTEGER', 'correction TEXT', 'detector TEXT', 'confidence REAL', 'refuted_by INTEGER']) {
|
|
235
|
+
if (!columns.includes(column.split(' ')[0])) this.#db.exec(`ALTER TABLE memories ADD COLUMN ${column}`);
|
|
236
|
+
}
|
|
171
237
|
}
|
|
172
238
|
|
|
173
239
|
/* ---------- embeddings ---------- */
|
|
@@ -324,6 +390,15 @@ export class RoomMemory {
|
|
|
324
390
|
}
|
|
325
391
|
}
|
|
326
392
|
undistilledCount() { return this.#db.prepare('SELECT COUNT(*) AS n FROM entries WHERE distilled = 0').get().n; }
|
|
393
|
+
|
|
394
|
+
// The exchanges a note says it came from, for anyone checking whether they say what it says.
|
|
395
|
+
entriesAt(sequences = []) {
|
|
396
|
+
if (!this.#db) return [];
|
|
397
|
+
const wanted = [...new Set(sequences.filter((n) => Number.isInteger(n)))].slice(0, 60);
|
|
398
|
+
if (!wanted.length) return [];
|
|
399
|
+
const fetch = this.#db.prepare('SELECT sequence, sender, text FROM entries WHERE sequence = ?');
|
|
400
|
+
return wanted.map((sequence) => fetch.get(sequence)).filter(Boolean);
|
|
401
|
+
}
|
|
327
402
|
memoryCount() { return this.#db.prepare('SELECT COUNT(*) AS n FROM memories').get().n; }
|
|
328
403
|
|
|
329
404
|
// The next batch to distil: the NEWEST entries nobody has distilled, cut at a
|
|
@@ -347,7 +422,7 @@ export class RoomMemory {
|
|
|
347
422
|
// Stores distilled memories; a note already held (same text, ignoring case
|
|
348
423
|
// and punctuation) is not stored twice. Returns how many were new.
|
|
349
424
|
addMemories(list, { agent, fromSequence, throughSequence, origin = 'distilled', messageId = null }) {
|
|
350
|
-
const insert = this.#db.prepare('INSERT OR IGNORE INTO memories (created, kind, text, norm, from_sequence, through_sequence, sources, agent, origin, message_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
425
|
+
const insert = this.#db.prepare('INSERT OR IGNORE INTO memories (created, kind, text, norm, from_sequence, through_sequence, sources, agent, origin, message_id, correction, detector, confidence) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
351
426
|
const now = new Date().toISOString();
|
|
352
427
|
let added = 0;
|
|
353
428
|
this.#db.exec('BEGIN');
|
|
@@ -355,10 +430,25 @@ export class RoomMemory {
|
|
|
355
430
|
for (const memory of list) {
|
|
356
431
|
const text = this.#guard(String(memory.text ?? '').trim());
|
|
357
432
|
if (!text) continue;
|
|
358
|
-
|
|
433
|
+
// An unknown kind is dropped, never filed as a fact: with one kind in the list that is
|
|
434
|
+
// not knowledge, a typo either way would put a hallucination where the room trusts it.
|
|
435
|
+
if (!MEMORY_KINDS.includes(memory.kind)) continue;
|
|
436
|
+
const kind = memory.kind;
|
|
359
437
|
const sources = (Array.isArray(memory.sources) ? memory.sources : []).filter((n) => Number.isInteger(n));
|
|
360
|
-
|
|
438
|
+
// Only an aberration carries a correction, and only ever as the archivist heard it: the
|
|
439
|
+
// note that refutes it is wired up later, by whoever can name the id.
|
|
440
|
+
const aberrant = kind === ABERRATION;
|
|
441
|
+
const correction = aberrant ? this.#guard(String(memory.correction ?? '').trim()) || null : null;
|
|
442
|
+
const result = insert.run(now, kind, text, memoryKey(kind, text), fromSequence ?? sources[0] ?? 0, throughSequence ?? sources.at(-1) ?? 0, JSON.stringify(sources), agent, origin, messageId, correction, aberrant ? (memory.detector ?? agent) : null, aberrant ? (Number.isFinite(memory.confidence) ? memory.confidence : null) : null);
|
|
361
443
|
added += Number(result.changes ?? 0);
|
|
444
|
+
if (aberrant && Number(result.changes ?? 0)) {
|
|
445
|
+
const id = Number(result.lastInsertRowid);
|
|
446
|
+
const target = this.#refutedBy(text);
|
|
447
|
+
if (target) {
|
|
448
|
+
this.#db.prepare('UPDATE memories SET contradicts = ? WHERE id = ?').run(target, id);
|
|
449
|
+
this.#db.prepare('UPDATE memories SET refuted_by = ? WHERE id = ? AND refuted_by IS NULL').run(id, target);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
362
452
|
}
|
|
363
453
|
this.#db.exec('COMMIT');
|
|
364
454
|
} catch (error) {
|
|
@@ -368,6 +458,83 @@ export class RoomMemory {
|
|
|
368
458
|
return added;
|
|
369
459
|
}
|
|
370
460
|
|
|
461
|
+
// An aberration filed by the archivist cannot name an id: the archivist reads exchanges, not
|
|
462
|
+
// the archive. So when the room records that a claim is false, the store looks for the note
|
|
463
|
+
// already standing that says the same thing, and wires the two together. Without this the
|
|
464
|
+
// room quarantines the refutation and goes on handing agents the very claim it just recorded
|
|
465
|
+
// as false, which is the one outcome all of this exists to prevent.
|
|
466
|
+
//
|
|
467
|
+
// The bar is high on purpose: almost every distinctive word of one has to be in the other.
|
|
468
|
+
// A merely related note is not the same claim, and taking down the wrong one is worse than
|
|
469
|
+
// taking down none.
|
|
470
|
+
#refutedBy(text) {
|
|
471
|
+
const terms = new Set(normalizeMemory(text).split(' ').filter((word) => word.length > 2));
|
|
472
|
+
if (terms.size < 3) return null;
|
|
473
|
+
let best = null;
|
|
474
|
+
const standing = this.#db.prepare(`SELECT id, text FROM memories WHERE kind != '${ABERRATION}' AND refuted_by IS NULL ORDER BY id DESC LIMIT 200`).all();
|
|
475
|
+
for (const row of standing) {
|
|
476
|
+
const other = new Set(normalizeMemory(row.text).split(' ').filter((word) => word.length > 2));
|
|
477
|
+
if (other.size < 3) continue;
|
|
478
|
+
let shared = 0;
|
|
479
|
+
for (const term of other) if (terms.has(term)) shared += 1;
|
|
480
|
+
const overlap = shared / Math.min(terms.size, other.size);
|
|
481
|
+
if (overlap >= 0.8 && (!best || overlap > best.overlap)) best = { id: row.id, overlap };
|
|
482
|
+
}
|
|
483
|
+
return best?.id ?? null;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// Flagging an aberration is two writes that have to happen together: the claim is filed, and
|
|
487
|
+
// whatever it refutes stops being handed to agents. A refutation does not rewrite what a note
|
|
488
|
+
// said; it takes it out of circulation, which is reversible, where an edit would not be.
|
|
489
|
+
flagAberration({ text, correction = null, contradicts = null, sources = [], agent = 'eyecat', detector = 'eyecat', confidence = null, fromSequence = null, throughSequence = null }) {
|
|
490
|
+
if (!this.#db) return null;
|
|
491
|
+
const claim = this.#guard(String(text ?? '').trim());
|
|
492
|
+
if (!claim) return null;
|
|
493
|
+
const cites = (Array.isArray(sources) ? sources : []).filter((n) => Number.isInteger(n));
|
|
494
|
+
const refuted = Number.isInteger(contradicts) ? this.#db.prepare(`SELECT id, kind FROM memories WHERE id = ? AND kind != '${ABERRATION}'`).get(contradicts) : null;
|
|
495
|
+
this.#db.exec('BEGIN');
|
|
496
|
+
try {
|
|
497
|
+
const result = this.#db.prepare('INSERT OR IGNORE INTO memories (created, kind, text, norm, from_sequence, through_sequence, sources, agent, origin, contradicts, correction, detector, confidence) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
|
|
498
|
+
.run(new Date().toISOString(), ABERRATION, claim, memoryKey(ABERRATION, claim), fromSequence ?? cites[0] ?? 0, throughSequence ?? cites.at(-1) ?? 0, JSON.stringify(cites), agent, 'flagged', refuted?.id ?? null, this.#guard(String(correction ?? '').trim()) || null, detector, Number.isFinite(confidence) ? confidence : null);
|
|
499
|
+
if (!Number(result.changes ?? 0)) { this.#db.exec('ROLLBACK'); return null; } // already known
|
|
500
|
+
const id = Number(result.lastInsertRowid);
|
|
501
|
+
if (refuted) this.#db.prepare('UPDATE memories SET refuted_by = ? WHERE id = ? AND refuted_by IS NULL').run(id, refuted.id);
|
|
502
|
+
this.#db.exec('COMMIT');
|
|
503
|
+
return { id, refuted: refuted?.id ?? null };
|
|
504
|
+
} catch (error) {
|
|
505
|
+
this.#db.exec('ROLLBACK');
|
|
506
|
+
throw error;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// The other direction, for when the room was wrong about being wrong: the aberration goes and
|
|
511
|
+
// whatever it took out of circulation stands again.
|
|
512
|
+
clearAberration(id) {
|
|
513
|
+
if (!this.#db) return null;
|
|
514
|
+
const row = this.#db.prepare(`SELECT id, text, contradicts FROM memories WHERE id = ? AND kind = '${ABERRATION}'`).get(Number(id));
|
|
515
|
+
if (!row) return null;
|
|
516
|
+
this.#db.exec('BEGIN');
|
|
517
|
+
try {
|
|
518
|
+
this.#db.prepare('UPDATE memories SET refuted_by = NULL WHERE refuted_by = ?').run(row.id);
|
|
519
|
+
this.#db.prepare('DELETE FROM memories WHERE id = ?').run(row.id);
|
|
520
|
+
this.#db.prepare('DELETE FROM memory_vectors WHERE id = ?').run(row.id);
|
|
521
|
+
this.#db.exec('COMMIT');
|
|
522
|
+
return { id: row.id, text: row.text, restored: row.contradicts ?? null };
|
|
523
|
+
} catch (error) {
|
|
524
|
+
this.#db.exec('ROLLBACK');
|
|
525
|
+
throw error;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// Everything the room has decided is false, newest first, with what it took down.
|
|
530
|
+
aberrations({ limit = 50 } = {}) {
|
|
531
|
+
if (!this.#db) return [];
|
|
532
|
+
return this.#db.prepare(`SELECT a.id, a.created, a.text, a.correction, a.detector, a.confidence, a.contradicts, a.sources, a.agent, m.text AS contradictsText
|
|
533
|
+
FROM memories a LEFT JOIN memories m ON m.id = a.contradicts
|
|
534
|
+
WHERE a.kind = '${ABERRATION}' ORDER BY a.id DESC LIMIT ?`).all(limit)
|
|
535
|
+
.map((row) => ({ ...row, sources: JSON.parse(row.sources) }));
|
|
536
|
+
}
|
|
537
|
+
|
|
371
538
|
// Exact text of a stretch of the ledger, capped so a tool answer stays readable.
|
|
372
539
|
range({ from = 1, through = Number.MAX_SAFE_INTEGER, limit = 40, maxChars = 12000 } = {}) {
|
|
373
540
|
const rows = this.#db.prepare('SELECT sequence, timestamp, type, role, sender, target, message_id AS messageId, text FROM entries WHERE sequence >= ? AND sequence <= ? ORDER BY sequence LIMIT ?').all(from, through, limit + 1);
|
|
@@ -394,6 +561,10 @@ export class RoomMemory {
|
|
|
394
561
|
this.#db.exec('BEGIN');
|
|
395
562
|
try {
|
|
396
563
|
this.#db.prepare('DELETE FROM memory_vectors WHERE id = ?').run(id);
|
|
564
|
+
this.#db.prepare('DELETE FROM recalls WHERE memory_id = ?').run(id);
|
|
565
|
+
// A note quarantined by this one comes back: with the aberration gone there is nothing
|
|
566
|
+
// holding it out of circulation, and a memory must never be lost to a pointer at nothing.
|
|
567
|
+
this.#db.prepare('UPDATE memories SET refuted_by = NULL WHERE refuted_by = ?').run(id);
|
|
397
568
|
this.#db.prepare('DELETE FROM memories WHERE id = ?').run(id);
|
|
398
569
|
this.#db.exec('COMMIT');
|
|
399
570
|
} catch (error) { this.#db.exec('ROLLBACK'); throw error; }
|
|
@@ -436,10 +607,10 @@ export class RoomMemory {
|
|
|
436
607
|
|
|
437
608
|
memories({ limit = 50, kind = null } = {}) {
|
|
438
609
|
if (kind) {
|
|
439
|
-
return this.#db.prepare('SELECT id, created, kind, text, from_sequence AS fromSequence, through_sequence AS throughSequence, sources, agent, origin, message_id AS messageId FROM memories WHERE kind = ? ORDER BY id DESC LIMIT ?').all(kind, limit)
|
|
610
|
+
return this.#db.prepare('SELECT id, created, kind, text, from_sequence AS fromSequence, through_sequence AS throughSequence, sources, agent, origin, message_id AS messageId, recalled, last_recalled AS lastRecalled, contradicts, correction, detector, confidence, refuted_by AS refutedBy FROM memories WHERE kind = ? ORDER BY id DESC LIMIT ?').all(kind, limit)
|
|
440
611
|
.map((row) => ({ ...row, sources: JSON.parse(row.sources) }));
|
|
441
612
|
}
|
|
442
|
-
return this.#db.prepare('SELECT id, created, kind, text, from_sequence AS fromSequence, through_sequence AS throughSequence, sources, agent, origin, message_id AS messageId FROM memories ORDER BY id DESC LIMIT ?').all(limit)
|
|
613
|
+
return this.#db.prepare('SELECT id, created, kind, text, from_sequence AS fromSequence, through_sequence AS throughSequence, sources, agent, origin, message_id AS messageId, recalled, last_recalled AS lastRecalled, contradicts, correction, detector, confidence, refuted_by AS refutedBy FROM memories ORDER BY id DESC LIMIT ?').all(limit)
|
|
443
614
|
.map((row) => ({ ...row, sources: JSON.parse(row.sources) }));
|
|
444
615
|
}
|
|
445
616
|
|
|
@@ -448,14 +619,16 @@ export class RoomMemory {
|
|
|
448
619
|
// preferences, all from before `beforeSequence` so they add to the window
|
|
449
620
|
// rather than repeat it, within a character budget.
|
|
450
621
|
// `fallback` fills a thin match with the latest decisions and preferences; @madre turns it off to stay honest.
|
|
451
|
-
recallMemories(text, { beforeSequence = Number.MAX_SAFE_INTEGER, limit = 6, maxChars = 1200, queryVector = null, semanticFloor = 0.45, fallback = true } = {}) {
|
|
622
|
+
recallMemories(text, { beforeSequence = Number.MAX_SAFE_INTEGER, limit = 6, maxChars = 1200, queryVector = null, semanticFloor = 0.45, fallback = true, track = true, by = null, cascade = true } = {}) {
|
|
452
623
|
if (!this.#db) return [];
|
|
453
624
|
const total = this.memoryCount();
|
|
454
625
|
if (!total) return [];
|
|
455
626
|
const terms = queryTerms(text);
|
|
456
627
|
const semantic = this.#semanticScores('memory_vectors', 'id', queryVector, { beforeSequence, floor: semanticFloor });
|
|
457
628
|
const scores = new Map();
|
|
458
|
-
|
|
629
|
+
// Nothing that is false and nothing that has been refuted travels into a turn. This is the
|
|
630
|
+
// gate: an archive that hands its own hallucinations back to the room repeats them.
|
|
631
|
+
const lookup = this.#db.prepare(`SELECT m.id FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid WHERE memories_fts MATCH ? AND m.through_sequence < ? AND m.kind != '${ABERRATION}' AND m.refuted_by IS NULL LIMIT 500`);
|
|
459
632
|
for (const term of terms) {
|
|
460
633
|
let rows;
|
|
461
634
|
try { rows = lookup.all(`"${term.replaceAll('"', '""')}"`, beforeSequence); } catch { continue; }
|
|
@@ -465,24 +638,159 @@ export class RoomMemory {
|
|
|
465
638
|
}
|
|
466
639
|
const ids = RoomMemory.fuse(scores, semantic).map(([id]) => id);
|
|
467
640
|
if (fallback && ids.length < 2) {
|
|
468
|
-
const recent = this.#db.prepare("SELECT id FROM memories WHERE through_sequence < ? AND kind IN ('decision', 'preference') ORDER BY id DESC LIMIT ?").all(beforeSequence, limit);
|
|
641
|
+
const recent = this.#db.prepare("SELECT id FROM memories WHERE through_sequence < ? AND kind IN ('decision', 'preference') AND refuted_by IS NULL ORDER BY id DESC LIMIT ?").all(beforeSequence, limit);
|
|
469
642
|
for (const { id } of recent) if (!ids.includes(id)) ids.push(id);
|
|
470
643
|
}
|
|
471
|
-
const fetch = this.#db.prepare('SELECT id, created, kind, text, from_sequence AS fromSequence, through_sequence AS throughSequence, sources, agent, origin, message_id AS messageId FROM memories WHERE id = ?');
|
|
644
|
+
const fetch = this.#db.prepare('SELECT id, created, kind, text, from_sequence AS fromSequence, through_sequence AS throughSequence, sources, agent, origin, message_id AS messageId, contradicts, correction, detector, confidence, refuted_by AS refutedBy FROM memories WHERE id = ?');
|
|
472
645
|
const chosen = [];
|
|
473
646
|
let remaining = Math.max(0, maxChars);
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
const row = fetch.get(id);
|
|
477
|
-
if (!row) continue;
|
|
647
|
+
let cursor = 0;
|
|
648
|
+
const take = (row, via) => {
|
|
478
649
|
const cost = row.text.length + 24;
|
|
479
|
-
if (cost > remaining)
|
|
650
|
+
if (cost > remaining) return false;
|
|
480
651
|
remaining -= cost;
|
|
481
|
-
chosen.push({ ...row, sources: JSON.parse(row.sources) });
|
|
652
|
+
chosen.push({ ...row, sources: JSON.parse(row.sources), via });
|
|
653
|
+
return true;
|
|
654
|
+
};
|
|
655
|
+
// What the search itself found, up to a ceiling. The cascade is given the last slots to fill,
|
|
656
|
+
// and hands back whatever it does not use: association adds to a turn, it never displaces.
|
|
657
|
+
const fill = (ceiling) => {
|
|
658
|
+
while (cursor < ids.length && chosen.length < ceiling) {
|
|
659
|
+
const row = fetch.get(ids[cursor]);
|
|
660
|
+
cursor += 1;
|
|
661
|
+
// The semantic side of the search does not go through the gate above, so it is checked
|
|
662
|
+
// here as well: one path in means one path to keep clean, and there are two.
|
|
663
|
+
if (!row || row.kind === ABERRATION || row.refutedBy !== null) continue;
|
|
664
|
+
take(row, 'search');
|
|
665
|
+
}
|
|
666
|
+
};
|
|
667
|
+
const reserve = cascade ? Math.min(CASCADE_RESERVE, Math.max(0, limit - 1)) : 0;
|
|
668
|
+
fill(limit - reserve);
|
|
669
|
+
// Spreading activation: what the room has kept carrying alongside what it just found. This is
|
|
670
|
+
// the one part of recall that owes nothing to how a memory reads — only to what the room has
|
|
671
|
+
// actually done with it.
|
|
672
|
+
if (reserve && chosen.length) {
|
|
673
|
+
const seeds = chosen.map((note) => note.id);
|
|
674
|
+
for (const mate of this.associates(seeds, { limit: reserve })) {
|
|
675
|
+
if (chosen.length >= limit) break;
|
|
676
|
+
const row = fetch.get(mate.id);
|
|
677
|
+
if (!row || row.kind === ABERRATION || row.refutedBy !== null) continue;
|
|
678
|
+
if (row.throughSequence >= beforeSequence) continue; // it is already in the window
|
|
679
|
+
take(row, 'cascade');
|
|
680
|
+
}
|
|
482
681
|
}
|
|
682
|
+
fill(limit);
|
|
683
|
+
// A note that just travelled into a turn has been used: the archive counts it, so the room
|
|
684
|
+
// can tell which memories it actually leans on, and why each one came.
|
|
685
|
+
if (track && chosen.length) this.#markRecalled(chosen.map((note) => ({ id: note.id, via: note.via })), by);
|
|
483
686
|
return chosen.sort((a, b) => a.fromSequence - b.fromSequence || a.id - b.id);
|
|
484
687
|
}
|
|
485
688
|
|
|
689
|
+
// What one recall leaves behind: the counters the map reads, and one row per note with the
|
|
690
|
+
// batch they shared, so the traffic between two memories can be read back later.
|
|
691
|
+
#markRecalled(entries, by = null) {
|
|
692
|
+
try {
|
|
693
|
+
const now = new Date().toISOString();
|
|
694
|
+
const batch = `${now}/${Math.random().toString(36).slice(2, 10)}`;
|
|
695
|
+
const agent = by?.agent ? String(by.agent).slice(0, 40) : null;
|
|
696
|
+
const turn = by?.turn ? String(by.turn).slice(0, 80) : null;
|
|
697
|
+
const mark = this.#db.prepare('UPDATE memories SET recalled = recalled + 1, last_recalled = ? WHERE id = ?');
|
|
698
|
+
const log = this.#db.prepare('INSERT INTO recalls (memory_id, batch, at, agent, turn, via) VALUES (?, ?, ?, ?, ?, ?)');
|
|
699
|
+
this.#db.exec('BEGIN');
|
|
700
|
+
try {
|
|
701
|
+
for (const entry of entries) {
|
|
702
|
+
const id = typeof entry === 'object' ? entry.id : entry;
|
|
703
|
+
mark.run(now, id);
|
|
704
|
+
log.run(id, batch, now, agent, turn, (typeof entry === 'object' && entry.via) || 'search');
|
|
705
|
+
}
|
|
706
|
+
// The traffic is a record of the recent past, not an archive of its own.
|
|
707
|
+
this.#db.prepare(`DELETE FROM recalls WHERE id <= (SELECT MAX(id) FROM recalls) - ${RECALL_HISTORY}`).run();
|
|
708
|
+
this.#db.exec('COMMIT');
|
|
709
|
+
} catch (error) { this.#db.exec('ROLLBACK'); throw error; }
|
|
710
|
+
} catch (error) { console.error(`MADRE could not count a recall: ${error.message}`); }
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
// Every turn that reached into the archive, oldest first. These are the chances a memory had
|
|
714
|
+
// to be the answer; a note nothing carried across many of them is adrift, not new.
|
|
715
|
+
recallBatches() {
|
|
716
|
+
if (!this.#db) return [];
|
|
717
|
+
return this.#db.prepare('SELECT MIN(at) AS at FROM recalls GROUP BY batch ORDER BY at').all().map((row) => row.at);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// The day this room started keeping that trail. Counters from before it are real; the turns
|
|
721
|
+
// behind them were never written down.
|
|
722
|
+
recallsSince() { return this.#metaValue('recalls_since'); }
|
|
723
|
+
|
|
724
|
+
// Which memories keep travelling with these, and how strongly. Only the turns where each was
|
|
725
|
+
// found by the search itself are counted: a memory that arrived by cascade must never become
|
|
726
|
+
// the evidence for the next cascade, or the network closes into a clique that carries itself.
|
|
727
|
+
associates(ids, { floor = CASCADE_FLOOR, minTimes = CASCADE_MIN_TIMES, limit = CASCADE_RESERVE, exclude = [] } = {}) {
|
|
728
|
+
if (!this.#db) return [];
|
|
729
|
+
const seeds = [...new Set((ids ?? []).filter((id) => Number.isInteger(id)))];
|
|
730
|
+
if (!seeds.length || limit <= 0) return [];
|
|
731
|
+
const blocked = new Set([...seeds, ...(exclude ?? [])]);
|
|
732
|
+
const holes = seeds.map(() => '?').join(',');
|
|
733
|
+
const together = this.#db.prepare(`
|
|
734
|
+
SELECT mine.memory_id AS seed, other.memory_id AS id, COUNT(DISTINCT other.batch) AS times
|
|
735
|
+
FROM recalls mine JOIN recalls other ON other.batch = mine.batch AND other.memory_id != mine.memory_id
|
|
736
|
+
WHERE mine.memory_id IN (${holes}) AND mine.via = 'search' AND other.via = 'search'
|
|
737
|
+
GROUP BY mine.memory_id, other.memory_id
|
|
738
|
+
`).all(...seeds).filter((row) => !blocked.has(row.id) && Number(row.times) >= minTimes);
|
|
739
|
+
if (!together.length) return [];
|
|
740
|
+
// How often each of them was found on its own, for the union underneath the ratio.
|
|
741
|
+
const involved = [...new Set([...seeds, ...together.map((row) => row.id)])];
|
|
742
|
+
const alone = new Map(this.#db.prepare(`
|
|
743
|
+
SELECT memory_id AS id, COUNT(DISTINCT batch) AS times FROM recalls
|
|
744
|
+
WHERE via = 'search' AND memory_id IN (${involved.map(() => '?').join(',')}) GROUP BY memory_id
|
|
745
|
+
`).all(...involved).map((row) => [row.id, Number(row.times)]));
|
|
746
|
+
const best = new Map();
|
|
747
|
+
for (const row of together) {
|
|
748
|
+
const times = Number(row.times);
|
|
749
|
+
const union = (alone.get(row.seed) ?? 0) + (alone.get(row.id) ?? 0) - times;
|
|
750
|
+
const strength = union > 0 ? times / union : 0;
|
|
751
|
+
if (strength < floor) continue;
|
|
752
|
+
const known = best.get(row.id);
|
|
753
|
+
if (!known || strength > known.strength) best.set(row.id, { id: row.id, strength: Number(strength.toFixed(3)), times, with: row.seed });
|
|
754
|
+
}
|
|
755
|
+
return [...best.values()].sort((a, b) => b.strength - a.strength || b.times - a.times).slice(0, limit);
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
// Everything one memory has to say about its own life: how often the room reached for it, who
|
|
759
|
+
// asked, and which other memories keep arriving in the same turn. A memory that has never been
|
|
760
|
+
// recalled answers with zeros, which is itself the reading.
|
|
761
|
+
recallTraffic(id, { limit = 8, together = 8 } = {}) {
|
|
762
|
+
if (!this.#db) return null;
|
|
763
|
+
const note = this.#db.prepare('SELECT id, kind, recalled, last_recalled AS lastRecalled, refuted_by AS refutedBy, contradicts FROM memories WHERE id = ?').get(Number(id));
|
|
764
|
+
if (!note) return null;
|
|
765
|
+
const recent = this.#db.prepare('SELECT batch, at, agent, turn FROM recalls WHERE memory_id = ? ORDER BY id DESC LIMIT ?').all(note.id, limit);
|
|
766
|
+
const fired = this.#db.prepare(`
|
|
767
|
+
SELECT other.memory_id AS id, COUNT(*) AS times, MAX(other.at) AS last
|
|
768
|
+
FROM recalls mine JOIN recalls other ON mine.batch = other.batch AND other.memory_id != mine.memory_id
|
|
769
|
+
WHERE mine.memory_id = ? GROUP BY other.memory_id ORDER BY times DESC, last DESC LIMIT ?
|
|
770
|
+
`).all(note.id, together);
|
|
771
|
+
// Which of that company is association rather than coincidence: the ones this memory would
|
|
772
|
+
// now bring along with it into a turn.
|
|
773
|
+
const pulls = new Map(this.associates([note.id], { limit: 24 }).map((mate) => [mate.id, mate.strength]));
|
|
774
|
+
const askers = this.#db.prepare('SELECT agent, COUNT(*) AS times FROM recalls WHERE memory_id = ? AND agent IS NOT NULL GROUP BY agent ORDER BY times DESC').all(note.id);
|
|
775
|
+
// What a refutation did, from either end: the aberration knows what it took down, and a note
|
|
776
|
+
// taken down knows what took it.
|
|
777
|
+
const refutes = this.#db.prepare('SELECT id, kind, text FROM memories WHERE refuted_by = ?').all(note.id)
|
|
778
|
+
.map((row) => ({ ...row, text: row.text.slice(0, 200) }));
|
|
779
|
+
const refutedBy = note.refutedBy ? this.#db.prepare('SELECT id, kind, text FROM memories WHERE id = ?').get(note.refutedBy) : null;
|
|
780
|
+
return {
|
|
781
|
+
id: note.id,
|
|
782
|
+
kind: note.kind,
|
|
783
|
+
recalled: Number(note.recalled ?? 0),
|
|
784
|
+
lastRecalled: note.lastRecalled ?? null,
|
|
785
|
+
since: this.#metaValue('recalls_since'),
|
|
786
|
+
recent: recent.map((row) => ({ at: row.at, agent: row.agent, turn: row.turn, batch: row.batch })),
|
|
787
|
+
fired: fired.map((row) => ({ id: row.id, times: Number(row.times), last: row.last, strength: pulls.get(row.id) ?? null, cascades: pulls.has(row.id) })),
|
|
788
|
+
askers: askers.map((row) => ({ agent: row.agent, times: Number(row.times) })),
|
|
789
|
+
refutes,
|
|
790
|
+
refutedBy: refutedBy ? { ...refutedBy, text: refutedBy.text.slice(0, 200) } : null,
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
|
|
486
794
|
#metaValue(key) { return this.#meta.get(key)?.value ?? null; }
|
|
487
795
|
|
|
488
796
|
// The last ledger sequence this index has seen, indexed or not.
|