@jossuealcala/madre 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +425 -0
- package/CONTRIBUTING.md +6 -1
- package/README.md +67 -183
- package/SECURITY.md +2 -1
- package/bin/madre.mjs +56 -13
- package/docs/INTERNALS.md +16 -0
- package/docs/REFERENCE.md +249 -0
- package/docs/SDK.md +121 -0
- package/docs/room.png +0 -0
- package/docs/sdk/hello-module.mjs +51 -0
- package/package.json +9 -1
- package/public/app.js +3880 -849
- package/public/es.js +2050 -0
- package/public/i18n.js +66 -0
- package/public/index.html +95 -14
- package/public/inquiry.js +220 -0
- package/public/resay.js +77 -0
- package/public/styles.css +626 -68
- package/public/troubleshooting.js +173 -51
- package/src/adapters/claude.mjs +13 -6
- package/src/adapters/codex.mjs +17 -13
- package/src/adapters/gemini.mjs +27 -16
- package/src/adapters/opencode.mjs +16 -12
- package/src/adapters/process.mjs +17 -5
- package/src/asking.mjs +128 -0
- package/src/auth-probe.mjs +58 -1
- package/src/capabilities.mjs +4 -3
- package/src/chats.mjs +193 -0
- package/src/checkpoint.mjs +1 -1
- package/src/cold.mjs +56 -0
- package/src/commands.mjs +31 -3
- package/src/conversation-context.mjs +35 -3
- package/src/credentials.mjs +145 -0
- package/src/dataset.mjs +56 -4
- package/src/distiller.mjs +12 -5
- package/src/event-store.mjs +14 -8
- package/src/exam.mjs +240 -0
- package/src/extensions.mjs +3 -2
- package/src/eyecat-watch.mjs +100 -0
- package/src/eyecat.mjs +169 -0
- package/src/i18n.mjs +47 -0
- package/src/image-studio.mjs +2 -0
- package/src/launch.mjs +61 -0
- package/src/lease.mjs +5 -3
- package/src/maturity.mjs +94 -0
- package/src/mcp/image-server.mjs +12 -1
- package/src/mcp/memory-server.mjs +1 -1
- package/src/memory.mjs +325 -17
- package/src/modules/ahp.mjs +9 -7
- package/src/modules/ash.mjs +36 -0
- package/src/modules/git-pulse.mjs +6 -4
- package/src/modules/helpers.mjs +31 -0
- package/src/modules/image-studio.mjs +9 -4
- package/src/modules/index.mjs +143 -5
- package/src/modules/ollama.mjs +66 -10
- package/src/modules/playwright.mjs +90 -0
- package/src/modules/ripley.mjs +5 -3
- package/src/modules/sdk.mjs +104 -2
- package/src/modules/updates.mjs +81 -0
- package/src/ollama.mjs +5 -2
- package/src/outbound.mjs +292 -0
- package/src/privacy.mjs +54 -7
- package/src/room/context.mjs +4 -4
- package/src/room/control.mjs +4 -4
- package/src/room/economy.mjs +161 -0
- package/src/room/prompt.mjs +118 -43
- package/src/room.mjs +465 -58
- package/src/runtime-detection.mjs +27 -8
- package/src/server.mjs +724 -68
- package/src/setup.mjs +1 -1
- package/src/updates.mjs +17 -2
- package/src/usage-sentinel.mjs +13 -8
- package/src/verdict.mjs +74 -0
- package/src/ashcode.mjs +0 -64
- package/src/modules/ashcode.mjs +0 -28
package/src/chats.mjs
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// A project has one memory and many conversations.
|
|
2
|
+
//
|
|
3
|
+
// The archive, the crew, the modules and the privacy list belong to the project: they are what
|
|
4
|
+
// MADRE knows about this codebase, and they do not start over because somebody opened a new
|
|
5
|
+
// thread. A conversation is only the record of one line of work — its own ledger, its own
|
|
6
|
+
// transcript, its own window — and everything it says feeds the same archive.
|
|
7
|
+
//
|
|
8
|
+
// The first conversation is the ledger that was always there, kept exactly where it was: no file
|
|
9
|
+
// is moved to add this, so a room that existed before opens as it always did and simply gains a
|
|
10
|
+
// name. New conversations get a folder of their own next to it.
|
|
11
|
+
//
|
|
12
|
+
// <room>/events.jsonl the first conversation
|
|
13
|
+
// <room>/chats/<id>/events.jsonl every one after it
|
|
14
|
+
// <room>/chats.json their names, and which one is open
|
|
15
|
+
//
|
|
16
|
+
// Nothing here starts a room or reads a ledger. It keeps the list.
|
|
17
|
+
|
|
18
|
+
import { readFile, writeFile, mkdir, rm, readdir, open, stat } from 'node:fs/promises';
|
|
19
|
+
import { join } from 'node:path';
|
|
20
|
+
import { randomUUID } from 'node:crypto';
|
|
21
|
+
|
|
22
|
+
export const MAIN_CHAT = 'main';
|
|
23
|
+
export const CHAT_TITLE_MAX = 80;
|
|
24
|
+
const FILE = 'chats.json';
|
|
25
|
+
|
|
26
|
+
export const isChatId = (id) => typeof id === 'string' && (id === MAIN_CHAT || /^[a-z0-9]{8,32}$/.test(id));
|
|
27
|
+
|
|
28
|
+
// Where one conversation's ledger lives. The first one never moved.
|
|
29
|
+
export function chatLedger(roomDir, id) {
|
|
30
|
+
return id === MAIN_CHAT ? join(roomDir, 'events.jsonl') : join(roomDir, 'chats', id, 'events.jsonl');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function readIndex(roomDir) {
|
|
34
|
+
try {
|
|
35
|
+
const raw = JSON.parse(await readFile(join(roomDir, FILE), 'utf8'));
|
|
36
|
+
if (raw && typeof raw === 'object' && raw.chats && typeof raw.chats === 'object') return raw;
|
|
37
|
+
} catch { /* no index yet, or an unreadable one: the room still opens */ }
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function writeIndex(roomDir, index) {
|
|
42
|
+
await mkdir(roomDir, { recursive: true }).catch(() => {});
|
|
43
|
+
await writeFile(join(roomDir, FILE), JSON.stringify(index, null, 2));
|
|
44
|
+
return index;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The list, with the first conversation always in it. A room that predates conversations reads
|
|
48
|
+
// as one conversation, which is what it was.
|
|
49
|
+
export async function chatIndex(roomDir) {
|
|
50
|
+
const index = await readIndex(roomDir);
|
|
51
|
+
const now = new Date().toISOString();
|
|
52
|
+
if (!index) return { active: MAIN_CHAT, chats: { [MAIN_CHAT]: { title: null, createdAt: now, updatedAt: now, messages: 0, preview: null } } };
|
|
53
|
+
if (!index.chats[MAIN_CHAT]) index.chats[MAIN_CHAT] = { title: null, createdAt: now, updatedAt: now, messages: 0, preview: null };
|
|
54
|
+
if (!index.chats[index.active]) index.active = MAIN_CHAT;
|
|
55
|
+
return index;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const shape = (id, entry, active) => ({
|
|
59
|
+
id,
|
|
60
|
+
title: entry.title || (id === MAIN_CHAT ? 'First conversation' : 'Untitled'),
|
|
61
|
+
named: Boolean(entry.title),
|
|
62
|
+
createdAt: entry.createdAt ?? null,
|
|
63
|
+
updatedAt: entry.updatedAt ?? entry.createdAt ?? null,
|
|
64
|
+
messages: Number(entry.messages ?? 0),
|
|
65
|
+
preview: entry.preview ?? null,
|
|
66
|
+
active: id === active,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
// Newest first, because that is the order a person looks for a conversation in.
|
|
70
|
+
export async function listChats(roomDir) {
|
|
71
|
+
const index = await chatIndex(roomDir);
|
|
72
|
+
const chats = Object.entries(index.chats)
|
|
73
|
+
.map(([id, entry]) => shape(id, entry, index.active))
|
|
74
|
+
.sort((a, b) => String(b.updatedAt ?? '').localeCompare(String(a.updatedAt ?? '')));
|
|
75
|
+
return { active: index.active, chats };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export async function createChat(roomDir, { title = null, now = new Date().toISOString() } = {}) {
|
|
79
|
+
const index = await chatIndex(roomDir);
|
|
80
|
+
const id = randomUUID().replace(/-/g, '').slice(0, 16);
|
|
81
|
+
index.chats[id] = { title: title ? String(title).slice(0, CHAT_TITLE_MAX) : null, createdAt: now, updatedAt: now, messages: 0, preview: null };
|
|
82
|
+
index.active = id;
|
|
83
|
+
await mkdir(join(roomDir, 'chats', id), { recursive: true });
|
|
84
|
+
await writeIndex(roomDir, index);
|
|
85
|
+
return shape(id, index.chats[id], index.active);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export async function openChat(roomDir, id) {
|
|
89
|
+
const index = await chatIndex(roomDir);
|
|
90
|
+
if (!index.chats[id]) return null;
|
|
91
|
+
index.active = id;
|
|
92
|
+
await writeIndex(roomDir, index);
|
|
93
|
+
return shape(id, index.chats[id], index.active);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export async function renameChat(roomDir, id, title) {
|
|
97
|
+
const index = await chatIndex(roomDir);
|
|
98
|
+
if (!index.chats[id]) return null;
|
|
99
|
+
const named = String(title ?? '').trim().slice(0, CHAT_TITLE_MAX);
|
|
100
|
+
index.chats[id].title = named || null;
|
|
101
|
+
await writeIndex(roomDir, index);
|
|
102
|
+
return shape(id, index.chats[id], index.active);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Deleting a conversation takes its transcript and nothing else: whatever the archivist distilled
|
|
106
|
+
// from it is the project's memory, not the conversation's, and it stays. The last conversation
|
|
107
|
+
// cannot be deleted — a room always has somewhere to talk.
|
|
108
|
+
export async function deleteChat(roomDir, id) {
|
|
109
|
+
const index = await chatIndex(roomDir);
|
|
110
|
+
if (!index.chats[id]) return null;
|
|
111
|
+
if (Object.keys(index.chats).length < 2) return { error: 'A room always has one conversation. Start another before deleting this one.' };
|
|
112
|
+
delete index.chats[id];
|
|
113
|
+
if (index.active === id) index.active = Object.keys(index.chats)[0];
|
|
114
|
+
if (id === MAIN_CHAT) await rm(join(roomDir, 'events.jsonl'), { force: true });
|
|
115
|
+
else await rm(join(roomDir, 'chats', id), { recursive: true, force: true, maxRetries: 6, retryDelay: 60 });
|
|
116
|
+
await writeIndex(roomDir, index);
|
|
117
|
+
return { deleted: id, active: index.active };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// What the list shows about a conversation, kept as it happens rather than by reading every
|
|
121
|
+
// ledger to draw a sidebar. A conversation with no name of its own takes the first thing the
|
|
122
|
+
// human said in it, which is what they will look for.
|
|
123
|
+
export async function touchChat(roomDir, id, { text = null, now = new Date().toISOString(), counts = true } = {}) {
|
|
124
|
+
const index = await chatIndex(roomDir);
|
|
125
|
+
const entry = index.chats[id];
|
|
126
|
+
if (!entry) return null;
|
|
127
|
+
entry.updatedAt = now;
|
|
128
|
+
if (counts) entry.messages = Number(entry.messages ?? 0) + 1;
|
|
129
|
+
if (text) {
|
|
130
|
+
const line = String(text).replace(/\s+/g, ' ').trim().slice(0, 120);
|
|
131
|
+
if (line) {
|
|
132
|
+
entry.preview = line;
|
|
133
|
+
if (!entry.title) entry.title = line.slice(0, CHAT_TITLE_MAX);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
await writeIndex(roomDir, index);
|
|
137
|
+
return shape(id, entry, index.active);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Conversations whose folder is on disk but which no index knows about: a room copied by hand, or
|
|
141
|
+
// an index lost. They are listed rather than ignored, because a transcript is not disposable.
|
|
142
|
+
export async function adoptStrays(roomDir) {
|
|
143
|
+
let folders = [];
|
|
144
|
+
try { folders = await readdir(join(roomDir, 'chats')); } catch { return { adopted: [] }; }
|
|
145
|
+
const index = await chatIndex(roomDir);
|
|
146
|
+
const adopted = [];
|
|
147
|
+
for (const id of folders) {
|
|
148
|
+
if (!isChatId(id) || index.chats[id]) continue;
|
|
149
|
+
index.chats[id] = { title: null, createdAt: null, updatedAt: null, messages: 0, preview: null };
|
|
150
|
+
adopted.push(id);
|
|
151
|
+
}
|
|
152
|
+
if (adopted.length) await writeIndex(roomDir, index);
|
|
153
|
+
return { adopted };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// The last sequence a ledger reached, read from its tail rather than by loading it. A project
|
|
157
|
+
// numbers its exchanges once across every conversation, so opening one has to know where the
|
|
158
|
+
// others got to; doing that by reading whole ledgers would make opening a conversation cost more
|
|
159
|
+
// the longer the project has been worked in.
|
|
160
|
+
export async function lastSequenceOf(file, { window = 65536 } = {}) {
|
|
161
|
+
let handle;
|
|
162
|
+
try {
|
|
163
|
+
const size = (await stat(file)).size;
|
|
164
|
+
if (!size) return 0;
|
|
165
|
+
handle = await open(file, 'r');
|
|
166
|
+
const length = Math.min(window, size);
|
|
167
|
+
const buffer = Buffer.alloc(length);
|
|
168
|
+
await handle.read(buffer, 0, length, size - length);
|
|
169
|
+
const lines = buffer.toString('utf8').split('\n').filter(Boolean);
|
|
170
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
171
|
+
try {
|
|
172
|
+
const event = JSON.parse(lines[i]);
|
|
173
|
+
if (Number.isInteger(event?.sequence)) return event.sequence;
|
|
174
|
+
} catch { /* a half line at the window's edge, or a torn write: keep looking back */ }
|
|
175
|
+
}
|
|
176
|
+
return 0;
|
|
177
|
+
} catch {
|
|
178
|
+
return 0;
|
|
179
|
+
} finally {
|
|
180
|
+
await handle?.close().catch(() => {});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Where the project's numbering stands, across every conversation but the one being opened.
|
|
185
|
+
export async function projectFloor(roomDir, { except = null } = {}) {
|
|
186
|
+
const { chats } = await listChats(roomDir);
|
|
187
|
+
let floor = 0;
|
|
188
|
+
for (const chat of chats) {
|
|
189
|
+
if (chat.id === except) continue;
|
|
190
|
+
floor = Math.max(floor, await lastSequenceOf(chatLedger(roomDir, chat.id)));
|
|
191
|
+
}
|
|
192
|
+
return floor;
|
|
193
|
+
}
|
package/src/checkpoint.mjs
CHANGED
|
@@ -86,7 +86,7 @@ export async function worktreeTree(root, { exclude = [] } = {}) {
|
|
|
86
86
|
}
|
|
87
87
|
return (await git(root, ['write-tree'], { env })).trim();
|
|
88
88
|
} finally {
|
|
89
|
-
await rm(dir, { recursive: true, force: true });
|
|
89
|
+
await rm(dir, { recursive: true, force: true, maxRetries: 6, retryDelay: 60 });
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
|
package/src/cold.mjs
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// Cold zones: the part of the archive nothing has ever reached for.
|
|
2
|
+
//
|
|
3
|
+
// A memory is cold when three things are true at once, and the three together are what make the
|
|
4
|
+
// claim strong enough to act on:
|
|
5
|
+
//
|
|
6
|
+
// · no turn has ever carried it
|
|
7
|
+
// · it shares a subject with no other memory, so nothing can reach it sideways either
|
|
8
|
+
// · the room has reached into the archive often enough since it was written that it has plainly
|
|
9
|
+
// had its chances
|
|
10
|
+
//
|
|
11
|
+
// That third one is the whole point. Never-recalled is not cold; never-recalled is what every
|
|
12
|
+
// memory is on the day it is written. What makes a memory cold is opportunity that went by: the
|
|
13
|
+
// archive was opened forty times and it was never the answer. Chances are counted in turns that
|
|
14
|
+
// actually reached into the archive, because those are the only chances that existed.
|
|
15
|
+
//
|
|
16
|
+
// Pure functions. The room hands in what it already has.
|
|
17
|
+
|
|
18
|
+
export const COLD_CHANCES = 12; // turns that went into the archive without ever taking it
|
|
19
|
+
|
|
20
|
+
// How many of `times` (ascending) are at or after `from`.
|
|
21
|
+
function after(times, from) {
|
|
22
|
+
let low = 0;
|
|
23
|
+
let high = times.length;
|
|
24
|
+
while (low < high) {
|
|
25
|
+
const middle = (low + high) >> 1;
|
|
26
|
+
if (times[middle] < from) low = middle + 1; else high = middle;
|
|
27
|
+
}
|
|
28
|
+
return times.length - low;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Which memories are adrift, and how many chances each one has had. `since` is the day this room
|
|
32
|
+
// started keeping the trail: a memory older than that is only counted from there, because what
|
|
33
|
+
// happened before it was never written down and must not be held against the note.
|
|
34
|
+
export function coldNotes({ notes = [], links = [], batches = [], since = null, chances = COLD_CHANCES } = {}) {
|
|
35
|
+
const linked = new Set(links.flatMap((link) => [link.a, link.b]));
|
|
36
|
+
const times = batches.map((at) => Date.parse(at)).filter((at) => Number.isFinite(at)).sort((a, b) => a - b);
|
|
37
|
+
const floor = since ? Date.parse(since) : 0;
|
|
38
|
+
const cold = new Map();
|
|
39
|
+
for (const note of notes) {
|
|
40
|
+
if (!note || note.kind === 'aberration' || note.refutedBy) continue; // a refutation is not a cold memory
|
|
41
|
+
if (Number(note.recalled ?? 0) > 0) continue;
|
|
42
|
+
if (linked.has(note.id)) continue;
|
|
43
|
+
const written = Date.parse(note.created);
|
|
44
|
+
const from = Math.max(Number.isFinite(written) ? written : 0, Number.isFinite(floor) ? floor : 0);
|
|
45
|
+
const had = after(times, from);
|
|
46
|
+
if (had >= chances) cold.set(note.id, { chances: had });
|
|
47
|
+
}
|
|
48
|
+
return cold;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// What the room should say about it. Never a number on its own: a count of cold memories means
|
|
52
|
+
// nothing without what it is a count of.
|
|
53
|
+
export function coldReading(cold, notes = []) {
|
|
54
|
+
const standing = notes.filter((note) => note && note.kind !== 'aberration' && !note.refutedBy).length;
|
|
55
|
+
return { count: cold.size, standing, share: standing > 0 ? Number((cold.size / standing).toFixed(3)) : 0 };
|
|
56
|
+
}
|
package/src/commands.mjs
CHANGED
|
@@ -25,8 +25,8 @@ export const COMMANDS = [
|
|
|
25
25
|
name: 'git',
|
|
26
26
|
module: 'git-pulse',
|
|
27
27
|
title: 'Git Pulse',
|
|
28
|
-
usage: '/git [status|log|diff|branches]',
|
|
29
|
-
summary: 'Repository status, recent commits, uncommitted changes and branches
|
|
28
|
+
usage: '/git [status|log|diff|branches|commit "message"|push [confirm]]',
|
|
29
|
+
summary: 'Repository status, recent commits, uncommitted changes and branches from the project itself; commit and push by your own hand, with a confirmation before anything leaves.',
|
|
30
30
|
async available({ projectRoot }) {
|
|
31
31
|
return exists(join(projectRoot, '.git'));
|
|
32
32
|
},
|
|
@@ -48,8 +48,36 @@ export const COMMANDS = [
|
|
|
48
48
|
await add('staged (stat)', ['-c', 'color.ui=never', 'diff', '--cached', '--stat']);
|
|
49
49
|
} else if (what === 'branches') {
|
|
50
50
|
await add('branches', ['-c', 'color.ui=never', 'branch', '-avv']);
|
|
51
|
+
} else if (what === 'commit') {
|
|
52
|
+
// The human commits what the room produced. Everything in the tree, one message, local: reversible with git.
|
|
53
|
+
const message = args.slice(1).join(' ').replace(/^["'“]+|["'”]+$/g, '').trim();
|
|
54
|
+
if (!message) return { ok: false, title: 'Git Pulse · commit', text: 'Give the commit a message: /git commit "what and why".' };
|
|
55
|
+
// Without an identity git refuses the commit with a wall of advice; say the one thing to do.
|
|
56
|
+
const email = await run('git', ['config', '--get', 'user.email'], projectRoot);
|
|
57
|
+
const who = await run('git', ['config', '--get', 'user.name'], projectRoot);
|
|
58
|
+
if (!email.text.trim() || !who.text.trim()) {
|
|
59
|
+
return { ok: false, title: 'Git Pulse · commit', text: 'This computer has no git identity, so the commit would have no author. Set it once in your terminal:\n\n git config --global user.name "Your Name"\n git config --global user.email you@example.com' };
|
|
60
|
+
}
|
|
61
|
+
const staged = await run('git', ['add', '-A', '--', '.'], projectRoot);
|
|
62
|
+
if (!staged.ok) return { ok: false, title: 'Git Pulse · commit', text: staged.text };
|
|
63
|
+
const committed = await run('git', ['-c', 'color.ui=never', 'commit', '-m', message], projectRoot);
|
|
64
|
+
if (!committed.ok) return { ok: false, title: 'Git Pulse · commit', text: committed.text || 'Nothing to commit.' };
|
|
65
|
+
await add('committed', ['-c', 'color.ui=never', 'show', '--stat', '--format=%h %s', 'HEAD']);
|
|
66
|
+
return { ok: true, title: 'Git Pulse · commit', text: sections.join('\n\n') };
|
|
67
|
+
} else if (what === 'push') {
|
|
68
|
+
// Nothing leaves without the word: first the preview of what would go, then /git push confirm.
|
|
69
|
+
const upstream = await run('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], projectRoot);
|
|
70
|
+
if (!upstream.ok) return { ok: false, title: 'Git Pulse · push', text: `This branch has no upstream. Set it once from your terminal: git push -u <remote> <branch>.` };
|
|
71
|
+
const ahead = await run('git', ['-c', 'color.ui=never', 'log', '--oneline', '@{u}..HEAD'], projectRoot);
|
|
72
|
+
const commits = ahead.text ? ahead.text.split('\n').filter(Boolean) : [];
|
|
73
|
+
if (!commits.length) return { ok: true, title: 'Git Pulse · push', text: `Nothing to push: ${upstream.text} already has everything.` };
|
|
74
|
+
if ((args[1] ?? '').toLowerCase() !== 'confirm') {
|
|
75
|
+
return { ok: true, title: 'Git Pulse · push · preview', text: `## would leave for ${upstream.text}\n${commits.join('\n')}\n\nThis leaves the machine and cannot be undone by MADRE. Send it with: /git push confirm` };
|
|
76
|
+
}
|
|
77
|
+
const pushed = await run('git', ['-c', 'color.ui=never', 'push'], projectRoot, 120000);
|
|
78
|
+
return { ok: pushed.ok, title: 'Git Pulse · push', text: `## sent to ${upstream.text}\n${commits.join('\n')}\n\n${pushed.text || '(pushed)'}` };
|
|
51
79
|
} else {
|
|
52
|
-
return { ok: false, title: 'Git Pulse', text: `Unknown subcommand "${what}". Use /git status, /git log [n], /git diff or /git
|
|
80
|
+
return { ok: false, title: 'Git Pulse', text: `Unknown subcommand "${what}". Use /git status, /git log [n], /git diff, /git branches, /git commit "message" or /git push [confirm].` };
|
|
53
81
|
}
|
|
54
82
|
return { ok: true, title: `Git Pulse · ${what}`, text: sections.join('\n\n') };
|
|
55
83
|
},
|
|
@@ -19,14 +19,44 @@ export function messageEntry(event) {
|
|
|
19
19
|
|
|
20
20
|
// `omitSynthetic` drops MADRE's own canned replies (identity, refusals, round-table plans):
|
|
21
21
|
// @madre must never read them back, or a small model starts echoing them.
|
|
22
|
-
|
|
22
|
+
// How much of the budget a window is cut back to when it finally has to let go of its oldest
|
|
23
|
+
// messages. Dropping one message per turn would move the start of the transcript on every turn;
|
|
24
|
+
// dropping a chunk at once and then holding still is what lets a CLI read most of it back from
|
|
25
|
+
// its own cache. Lower means longer stretches of stability and a leaner average window.
|
|
26
|
+
export const CONTEXT_KEEP = 0.62;
|
|
27
|
+
|
|
28
|
+
export function buildConversationContext(events, { excludeMessageId, maxChars = 16000, omitSynthetic = false, anchor = null } = {}) {
|
|
23
29
|
const messages = events
|
|
24
30
|
.map((event) => (omitSynthetic && event?.payload?.synthetic ? null : messageEntry(event)))
|
|
25
31
|
.filter((message) => message && message.messageId !== excludeMessageId);
|
|
32
|
+
const budget = Math.max(0, Number(maxChars) || 0);
|
|
33
|
+
|
|
34
|
+
// The window is anchored, not sliding. A sliding window starts one message later on every
|
|
35
|
+
// turn, so the transcript an agent reads begins with different words every time and none of
|
|
36
|
+
// it can be matched against what it read last turn. Held still, the whole of it but the tail
|
|
37
|
+
// is the same bytes as before, which is the difference between paying for it once and paying
|
|
38
|
+
// for it every turn. It only ever moves when what has been said since no longer fits.
|
|
39
|
+
const weigh = (from) => {
|
|
40
|
+
let used = 0;
|
|
41
|
+
for (let i = from; i < messages.length; i += 1) used += messages[i].sender.length + messages[i].role.length + messages[i].text.length + 4;
|
|
42
|
+
return used;
|
|
43
|
+
};
|
|
44
|
+
let start = 0;
|
|
45
|
+
if (Number.isInteger(anchor)) {
|
|
46
|
+
const held = messages.findIndex((message) => message.sequence >= anchor);
|
|
47
|
+
if (held >= 0 && weigh(held) <= budget) start = held;
|
|
48
|
+
}
|
|
49
|
+
if (!start || weigh(start) > budget) {
|
|
50
|
+
// Let go of a chunk and then hold: back off until the window sits well under its budget, so
|
|
51
|
+
// there is room for many turns of new words before it has to move again.
|
|
52
|
+
start = messages.length;
|
|
53
|
+
while (start > 0 && weigh(start - 1) <= budget * CONTEXT_KEEP) start -= 1;
|
|
54
|
+
}
|
|
55
|
+
|
|
26
56
|
const selected = [];
|
|
27
|
-
let remaining =
|
|
57
|
+
let remaining = budget;
|
|
28
58
|
|
|
29
|
-
for (let index = messages.length - 1; index >=
|
|
59
|
+
for (let index = messages.length - 1; index >= start && remaining > 0; index -= 1) {
|
|
30
60
|
const message = messages[index];
|
|
31
61
|
const label = `${message.sender} (${message.role})`;
|
|
32
62
|
const allowance = Math.max(0, remaining - label.length - 2);
|
|
@@ -42,6 +72,8 @@ export function buildConversationContext(events, { excludeMessageId, maxChars =
|
|
|
42
72
|
|
|
43
73
|
return {
|
|
44
74
|
messages: selected,
|
|
75
|
+
// Where this window begins, for the next turn to hold on to.
|
|
76
|
+
anchor: selected[0]?.sequence ?? null,
|
|
45
77
|
omittedMessages: messages.length - selected.length,
|
|
46
78
|
firstSequence: selected[0]?.sequence ?? null,
|
|
47
79
|
throughSequence: selected.at(-1)?.sequence ?? null,
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// Handing a provider key to a CLI that signs in from its own prompt.
|
|
2
|
+
//
|
|
3
|
+
// MADRE keeps its promise: it never stores a credential of its own. The key the human pastes is
|
|
4
|
+
// written where that CLI looks for it, with the file locked to the owner, and MADRE keeps no
|
|
5
|
+
// copy: not in ~/.pulse/config.json, not in the room's ledger, not in a log line. What the room
|
|
6
|
+
// records is that a key was set, never the key.
|
|
7
|
+
//
|
|
8
|
+
// Every write is verified on the spot by asking the CLI (or MADRE's own probe) whether it is
|
|
9
|
+
// signed in now. If it is not, the previous files are put back, so a format MADRE guessed wrong
|
|
10
|
+
// leaves nothing behind.
|
|
11
|
+
|
|
12
|
+
import { execFile } from 'node:child_process';
|
|
13
|
+
import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
14
|
+
import { homedir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { promisify } from 'node:util';
|
|
17
|
+
|
|
18
|
+
const execFileAsync = promisify(execFile);
|
|
19
|
+
const OWNER_ONLY = 0o600;
|
|
20
|
+
|
|
21
|
+
// A key is one opaque token: no spaces, no newlines, and long enough to be one.
|
|
22
|
+
export function normalizeKey(value) {
|
|
23
|
+
const key = String(value ?? '').trim();
|
|
24
|
+
if (!key) return { ok: false, error: 'Paste the key first.' };
|
|
25
|
+
if (/\s/.test(key)) return { ok: false, error: 'That does not look like a key: it has spaces or line breaks.' };
|
|
26
|
+
if (key.length < 12 || key.length > 400) return { ok: false, error: 'That does not look like a key: check you copied all of it and nothing else.' };
|
|
27
|
+
return { ok: true, key };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// OpenCode brings no model of its own. These are the providers MADRE can wire with a key;
|
|
31
|
+
// any other still works through `opencode auth login` in a terminal.
|
|
32
|
+
export const OPENCODE_PROVIDERS = [
|
|
33
|
+
{ id: 'anthropic', label: 'Anthropic', keyUrl: 'https://console.anthropic.com/settings/keys' },
|
|
34
|
+
{ id: 'openai', label: 'OpenAI', keyUrl: 'https://platform.openai.com/api-keys' },
|
|
35
|
+
{ id: 'openrouter', label: 'OpenRouter', keyUrl: 'https://openrouter.ai/keys' },
|
|
36
|
+
{ id: 'google', label: 'Google AI Studio', keyUrl: 'https://aistudio.google.com/app/apikey' },
|
|
37
|
+
];
|
|
38
|
+
|
|
39
|
+
// What the room offers for each agent that cannot be signed in with a click.
|
|
40
|
+
export function keyPlanFor(id) {
|
|
41
|
+
if (id === 'gemini') {
|
|
42
|
+
return {
|
|
43
|
+
agent: 'gemini',
|
|
44
|
+
label: 'Gemini API key',
|
|
45
|
+
keyUrl: 'https://aistudio.google.com/app/apikey',
|
|
46
|
+
writesTo: '~/.gemini/.env',
|
|
47
|
+
note: 'The key is written where the Gemini CLI looks for it, readable only by you. MADRE keeps no copy.',
|
|
48
|
+
providers: null,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
if (id === 'opencode') {
|
|
52
|
+
return {
|
|
53
|
+
agent: 'opencode',
|
|
54
|
+
label: 'Provider key',
|
|
55
|
+
keyUrl: null,
|
|
56
|
+
writesTo: '~/.local/share/opencode/auth.json',
|
|
57
|
+
note: 'OpenCode brings no model of its own: pick the provider whose key you are pasting. The key is written where OpenCode looks for it, readable only by you. MADRE keeps no copy.',
|
|
58
|
+
providers: OPENCODE_PROVIDERS,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function readIfThere(path) {
|
|
65
|
+
try { return await readFile(path, 'utf8'); } catch { return null; }
|
|
66
|
+
}
|
|
67
|
+
async function restore(path, previous) {
|
|
68
|
+
if (previous === null) await rm(path, { force: true });
|
|
69
|
+
else await writeFile(path, previous, { mode: OWNER_ONLY });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ---- Gemini: a line in ~/.gemini/.env, plus the auth type its settings must select.
|
|
73
|
+
async function applyGeminiKey(key, { home, probe }) {
|
|
74
|
+
const dir = join(home, '.gemini');
|
|
75
|
+
const envFile = join(dir, '.env');
|
|
76
|
+
const settingsFile = join(dir, 'settings.json');
|
|
77
|
+
await mkdir(dir, { recursive: true });
|
|
78
|
+
const beforeEnv = await readIfThere(envFile);
|
|
79
|
+
const beforeSettings = await readIfThere(settingsFile);
|
|
80
|
+
try {
|
|
81
|
+
const kept = (beforeEnv ?? '').split('\n').filter((line) => line.trim() && !/^\s*(GEMINI_API_KEY|GOOGLE_API_KEY)\s*=/.test(line));
|
|
82
|
+
await writeFile(envFile, `${[...kept, `GEMINI_API_KEY=${key}`].join('\n')}\n`, { mode: OWNER_ONLY });
|
|
83
|
+
await chmod(envFile, OWNER_ONLY).catch(() => {});
|
|
84
|
+
let settings = {};
|
|
85
|
+
try { settings = JSON.parse(beforeSettings ?? '{}') ?? {}; } catch { settings = {}; }
|
|
86
|
+
settings.security = { ...(settings.security ?? {}), auth: { ...(settings.security?.auth ?? {}), selectedType: 'gemini-api-key' } };
|
|
87
|
+
await writeFile(settingsFile, `${JSON.stringify(settings, null, 2)}\n`, { mode: OWNER_ONLY });
|
|
88
|
+
const state = await probe();
|
|
89
|
+
if (state?.state !== 'signed-in') {
|
|
90
|
+
await restore(envFile, beforeEnv);
|
|
91
|
+
await restore(settingsFile, beforeSettings);
|
|
92
|
+
return { ok: false, error: `The key was written but Gemini still reads as ${state?.state ?? 'unknown'}${state?.detail ? ` (${state.detail})` : ''}. Nothing was changed.` };
|
|
93
|
+
}
|
|
94
|
+
return { ok: true, detail: state.detail ?? 'via API key' };
|
|
95
|
+
} catch (error) {
|
|
96
|
+
await restore(envFile, beforeEnv);
|
|
97
|
+
await restore(settingsFile, beforeSettings);
|
|
98
|
+
return { ok: false, error: `The key could not be written: ${error.message}` };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// ---- OpenCode: an entry in its auth.json, confirmed by asking `opencode auth list`.
|
|
103
|
+
async function applyOpenCodeKey(key, { home, provider, executable, run }) {
|
|
104
|
+
const known = OPENCODE_PROVIDERS.some((item) => item.id === provider);
|
|
105
|
+
if (!known) return { ok: false, error: 'Pick the provider this key belongs to.' };
|
|
106
|
+
const file = join(home, '.local', 'share', 'opencode', 'auth.json');
|
|
107
|
+
await mkdir(join(home, '.local', 'share', 'opencode'), { recursive: true });
|
|
108
|
+
const before = await readIfThere(file);
|
|
109
|
+
try {
|
|
110
|
+
let auth = {};
|
|
111
|
+
try { auth = JSON.parse(before ?? '{}') ?? {}; } catch { auth = {}; }
|
|
112
|
+
auth[provider] = { type: 'api', key };
|
|
113
|
+
await writeFile(file, `${JSON.stringify(auth, null, 2)}\n`, { mode: OWNER_ONLY });
|
|
114
|
+
await chmod(file, OWNER_ONLY).catch(() => {});
|
|
115
|
+
// OpenCode itself is the judge: if its own listing does not show the provider, the shape
|
|
116
|
+
// MADRE wrote is not the shape it reads, and nothing should be left behind.
|
|
117
|
+
const listed = await run(executable, ['auth', 'list']);
|
|
118
|
+
if (!new RegExp(provider, 'i').test(listed)) {
|
|
119
|
+
await restore(file, before);
|
|
120
|
+
return { ok: false, error: `OpenCode does not list ${provider} after the key was written, so MADRE put the file back. Run \`opencode auth login\` in a terminal for this one.` };
|
|
121
|
+
}
|
|
122
|
+
return { ok: true, detail: `${provider} (api)` };
|
|
123
|
+
} catch (error) {
|
|
124
|
+
await restore(file, before);
|
|
125
|
+
return { ok: false, error: `The key could not be written: ${error.message}` };
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const runOpenCode = async (executable, args) => {
|
|
130
|
+
try {
|
|
131
|
+
const { stdout, stderr } = await execFileAsync(executable, args, { timeout: 15000, env: { ...process.env, NO_COLOR: '1' } });
|
|
132
|
+
return `${stdout}\n${stderr}`;
|
|
133
|
+
} catch (error) {
|
|
134
|
+
return `${error.stdout ?? ''}\n${error.stderr ?? error.message}`;
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
// Writes the key for one agent and answers whether that agent is signed in now.
|
|
139
|
+
export async function applyKey({ agent, key, provider = null, home = homedir(), probe, executable = 'opencode', run = runOpenCode }) {
|
|
140
|
+
const checked = normalizeKey(key);
|
|
141
|
+
if (!checked.ok) return checked;
|
|
142
|
+
if (agent === 'gemini') return applyGeminiKey(checked.key, { home, probe });
|
|
143
|
+
if (agent === 'opencode') return applyOpenCodeKey(checked.key, { home, provider, executable, run });
|
|
144
|
+
return { ok: false, error: `${agent} does not take a key here: it signs in with a click.` };
|
|
145
|
+
}
|
package/src/dataset.mjs
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
9
9
|
import { join } from 'node:path';
|
|
10
10
|
import { redact } from './sentinel-errors.mjs';
|
|
11
|
+
import { ABERRATION } from './memory.mjs';
|
|
11
12
|
|
|
12
13
|
const MIN_ANSWER_CHARS = 40;
|
|
13
14
|
|
|
@@ -82,14 +83,22 @@ export function readiness(events, notes = [], { target = DATASET_TARGET } = {})
|
|
|
82
83
|
const turns = pairs.filter((pair) => pair.kind === 'turn').length;
|
|
83
84
|
const delegated = pairs.length - turns;
|
|
84
85
|
const good = pairs.filter((pair) => pair.rating === 'good').length;
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
// Aberrations are counted apart: they never become one of the pairs the model learns to
|
|
87
|
+
// answer with, so counting them toward readiness would say the room is further along than it is.
|
|
88
|
+
const standing = notes.filter((note) => note.kind !== ABERRATION && !note.refutedBy);
|
|
89
|
+
const aberrations = notes.filter((note) => note.kind === ABERRATION).length;
|
|
90
|
+
const total = pairs.length + standing.length;
|
|
91
|
+
return { pairs: total, turns, delegated, notes: standing.length, aberrations, good, bad, target, ready: total >= target };
|
|
87
92
|
}
|
|
88
93
|
|
|
89
94
|
// Distilled notes become recall pairs: "what does the room remember about …" → the note.
|
|
95
|
+
// Two kinds of note never come through here. An aberration is a claim the room decided is false,
|
|
96
|
+
// and teaching a model to recall it would be teaching it the hallucination. A note that has been
|
|
97
|
+
// refuted is no longer what the room believes. Both are still exported, as preference pairs
|
|
98
|
+
// below, where being wrong is the point.
|
|
90
99
|
export function pairsFromNotes(notes, { project = 'project', home, user, privacy = null } = {}) {
|
|
91
100
|
const clean = guard(privacy);
|
|
92
|
-
return notes.map((note) => ({
|
|
101
|
+
return notes.filter((note) => note.kind !== ABERRATION && !note.refutedBy).map((note) => ({
|
|
93
102
|
kind: 'note',
|
|
94
103
|
agent: 'madre',
|
|
95
104
|
mode: 1,
|
|
@@ -103,6 +112,45 @@ export function pairsFromNotes(notes, { project = 'project', home, user, privacy
|
|
|
103
112
|
}));
|
|
104
113
|
}
|
|
105
114
|
|
|
115
|
+
// What the room got wrong, in the shape that trains against it. Each aberration becomes one
|
|
116
|
+
// preference pair: the same ask, the false claim as what to avoid, the correction as what to say
|
|
117
|
+
// instead. A negative on its own teaches a model very little; a pair tells it which of two
|
|
118
|
+
// answers to prefer, which is what DPO and ORPO are built to read.
|
|
119
|
+
export function preferencesFromAberrations(notes, { project = 'project', home, user, privacy = null } = {}) {
|
|
120
|
+
const clean = guard(privacy);
|
|
121
|
+
const say = (text) => clean(redact(String(text ?? ''), { home, user }));
|
|
122
|
+
const byId = new Map(notes.map((note) => [note.id, note]));
|
|
123
|
+
const pairs = [];
|
|
124
|
+
for (const note of notes) {
|
|
125
|
+
if (note.kind !== ABERRATION) continue;
|
|
126
|
+
// Without something true to put in its place there is nothing to prefer, so it is left out.
|
|
127
|
+
const truth = note.correction || (note.contradicts && byId.get(note.contradicts)?.text) || null;
|
|
128
|
+
if (!truth) continue;
|
|
129
|
+
const claim = say(note.text);
|
|
130
|
+
const better = say(truth);
|
|
131
|
+
if (!claim || !better) continue;
|
|
132
|
+
// A correction that restates the claim teaches nothing, and a model asked to prefer one of
|
|
133
|
+
// two near-identical answers learns the wrong lesson from the noise between them. A weak
|
|
134
|
+
// archivist writes these, so they are caught here rather than shipped.
|
|
135
|
+
const words = (text) => new Set(String(text).toLowerCase().normalize('NFKD').replace(/[\u0300-\u036f]/g, '').replace(/[^\p{L}\p{N}]+/gu, ' ').trim().split(' ').filter((word) => word.length > 2));
|
|
136
|
+
const [left, right] = [words(claim), words(better)];
|
|
137
|
+
if (!left.size || !right.size) continue;
|
|
138
|
+
let shared = 0;
|
|
139
|
+
for (const word of right) if (left.has(word)) shared += 1;
|
|
140
|
+
if (shared / Math.min(left.size, right.size) > 0.8) continue;
|
|
141
|
+
pairs.push({
|
|
142
|
+
kind: 'aberration',
|
|
143
|
+
sequence: note.throughSequence,
|
|
144
|
+
at: note.created,
|
|
145
|
+
detector: note.detector ?? null,
|
|
146
|
+
prompt: `Is this true of the project "${project}"? ${claim}`,
|
|
147
|
+
rejected: claim,
|
|
148
|
+
chosen: better,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
return pairs;
|
|
152
|
+
}
|
|
153
|
+
|
|
106
154
|
// Deterministic split: the sequence decides, so re-exports keep lines on the same side.
|
|
107
155
|
export function split(pairs, { validEvery = 10 } = {}) {
|
|
108
156
|
const train = [];
|
|
@@ -114,14 +162,18 @@ export function split(pairs, { validEvery = 10 } = {}) {
|
|
|
114
162
|
|
|
115
163
|
export async function exportDataset({ events, notes = [], dir, project = 'project', home, user, privacy = null }) {
|
|
116
164
|
const pairs = [...pairsFromEvents(events, { project, home, user, privacy }), ...pairsFromNotes(notes, { project, home, user, privacy })].sort((a, b) => a.sequence - b.sequence);
|
|
165
|
+
const preferences = preferencesFromAberrations(notes, { project, home, user, privacy });
|
|
117
166
|
const { train, valid } = split(pairs);
|
|
118
167
|
await mkdir(dir, { recursive: true });
|
|
119
168
|
const line = (pair) => JSON.stringify({ messages: pair.messages });
|
|
120
169
|
await writeFile(join(dir, 'train.jsonl'), train.map(line).join('\n') + (train.length ? '\n' : ''));
|
|
121
170
|
await writeFile(join(dir, 'valid.jsonl'), valid.map(line).join('\n') + (valid.length ? '\n' : ''));
|
|
171
|
+
// Kept in a file of its own: it is a different shape and a different kind of training, and
|
|
172
|
+
// nothing that reads the chat files should ever pick it up by accident.
|
|
173
|
+
await writeFile(join(dir, 'preferences.jsonl'), preferences.map((pair) => JSON.stringify({ prompt: pair.prompt, chosen: pair.chosen, rejected: pair.rejected })).join('\n') + (preferences.length ? '\n' : ''));
|
|
122
174
|
const byAgent = {};
|
|
123
175
|
for (const pair of pairs) byAgent[pair.agent] = (byAgent[pair.agent] ?? 0) + 1;
|
|
124
|
-
const manifest = { project, exportedAt: new Date().toISOString(), pairs: pairs.length, turns: pairs.filter((p) => p.kind === 'turn').length, delegated: pairs.filter((p) => p.kind === 'delegated').length, good: pairs.filter((p) => p.rating === 'good').length, notes: pairs.filter((p) => p.kind === 'note').length, train: train.length, valid: valid.length, byAgent, files: ['train.jsonl', 'valid.jsonl'], format: 'chat · {"messages":[{role,content}]} · mlx-lm / llama-factory / axolotl' };
|
|
176
|
+
const manifest = { project, exportedAt: new Date().toISOString(), pairs: pairs.length, turns: pairs.filter((p) => p.kind === 'turn').length, delegated: pairs.filter((p) => p.kind === 'delegated').length, good: pairs.filter((p) => p.rating === 'good').length, notes: pairs.filter((p) => p.kind === 'note').length, aberrations: preferences.length, train: train.length, valid: valid.length, byAgent, files: ['train.jsonl', 'valid.jsonl', 'preferences.jsonl'], format: 'chat · {"messages":[{role,content}]} · mlx-lm / llama-factory / axolotl', preferenceFormat: 'preference · {"prompt","chosen","rejected"} · DPO / ORPO' };
|
|
125
177
|
await writeFile(join(dir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
126
178
|
return { dir, ...manifest };
|
|
127
179
|
}
|