@jossuealcala/madre 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +425 -0
- package/CONTRIBUTING.md +6 -1
- package/README.md +67 -183
- package/SECURITY.md +2 -1
- package/bin/madre.mjs +56 -13
- package/docs/INTERNALS.md +16 -0
- package/docs/REFERENCE.md +249 -0
- package/docs/SDK.md +121 -0
- package/docs/room.png +0 -0
- package/docs/sdk/hello-module.mjs +51 -0
- package/package.json +9 -1
- package/public/app.js +3880 -849
- package/public/es.js +2050 -0
- package/public/i18n.js +66 -0
- package/public/index.html +95 -14
- package/public/inquiry.js +220 -0
- package/public/resay.js +77 -0
- package/public/styles.css +626 -68
- package/public/troubleshooting.js +173 -51
- package/src/adapters/claude.mjs +13 -6
- package/src/adapters/codex.mjs +17 -13
- package/src/adapters/gemini.mjs +27 -16
- package/src/adapters/opencode.mjs +16 -12
- package/src/adapters/process.mjs +17 -5
- package/src/asking.mjs +128 -0
- package/src/auth-probe.mjs +58 -1
- package/src/capabilities.mjs +4 -3
- package/src/chats.mjs +193 -0
- package/src/checkpoint.mjs +1 -1
- package/src/cold.mjs +56 -0
- package/src/commands.mjs +31 -3
- package/src/conversation-context.mjs +35 -3
- package/src/credentials.mjs +145 -0
- package/src/dataset.mjs +56 -4
- package/src/distiller.mjs +12 -5
- package/src/event-store.mjs +14 -8
- package/src/exam.mjs +240 -0
- package/src/extensions.mjs +3 -2
- package/src/eyecat-watch.mjs +100 -0
- package/src/eyecat.mjs +169 -0
- package/src/i18n.mjs +47 -0
- package/src/image-studio.mjs +2 -0
- package/src/launch.mjs +61 -0
- package/src/lease.mjs +5 -3
- package/src/maturity.mjs +94 -0
- package/src/mcp/image-server.mjs +12 -1
- package/src/mcp/memory-server.mjs +1 -1
- package/src/memory.mjs +325 -17
- package/src/modules/ahp.mjs +9 -7
- package/src/modules/ash.mjs +36 -0
- package/src/modules/git-pulse.mjs +6 -4
- package/src/modules/helpers.mjs +31 -0
- package/src/modules/image-studio.mjs +9 -4
- package/src/modules/index.mjs +143 -5
- package/src/modules/ollama.mjs +66 -10
- package/src/modules/playwright.mjs +90 -0
- package/src/modules/ripley.mjs +5 -3
- package/src/modules/sdk.mjs +104 -2
- package/src/modules/updates.mjs +81 -0
- package/src/ollama.mjs +5 -2
- package/src/outbound.mjs +292 -0
- package/src/privacy.mjs +54 -7
- package/src/room/context.mjs +4 -4
- package/src/room/control.mjs +4 -4
- package/src/room/economy.mjs +161 -0
- package/src/room/prompt.mjs +118 -43
- package/src/room.mjs +465 -58
- package/src/runtime-detection.mjs +27 -8
- package/src/server.mjs +724 -68
- package/src/setup.mjs +1 -1
- package/src/updates.mjs +17 -2
- package/src/usage-sentinel.mjs +13 -8
- package/src/verdict.mjs +74 -0
- package/src/ashcode.mjs +0 -64
- package/src/modules/ashcode.mjs +0 -28
package/src/distiller.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// keeping for every later turn. Pure functions here; the room schedules and
|
|
4
4
|
// invokes. Nothing in this file talks to a model.
|
|
5
5
|
|
|
6
|
-
import { MEMORY_KINDS } from './memory.mjs';
|
|
6
|
+
import { ABERRATION, MEMORY_KINDS } from './memory.mjs';
|
|
7
7
|
|
|
8
8
|
// Cheapest first, by the models these CLIs run by default.
|
|
9
9
|
// Ollama, when the room has it, is the cheapest of all: local and free.
|
|
@@ -42,11 +42,12 @@ export function distillPrompt({ entries, projectName = 'the project', existing =
|
|
|
42
42
|
`- fact: a verified statement about the project or its environment that is not obvious from the code`,
|
|
43
43
|
`- preference: how the human wants things done or said`,
|
|
44
44
|
`- question: something left open that a later turn should not forget`,
|
|
45
|
+
`- aberration: a claim the exchanges themselves showed to be false. Only when it was refuted out loud: someone stated something about this project and someone else corrected it, or the exchanges proved it wrong. Put the false claim in "text" and what turned out to be true in "correction". Never guess, never file a disagreement of opinion, and never file something merely unverified.`,
|
|
45
46
|
'Skip greetings, restatements, transient status, anything a reader of the code would see anyway, and anything the exchanges do not actually establish.',
|
|
46
47
|
json
|
|
47
|
-
? `Output one JSON object and nothing else: {"memories":[...]} with at most ${MAX_MEMORIES_PER_RUN} items, each {"kind":"
|
|
48
|
+
? `Output one JSON object and nothing else: {"memories":[...]} with at most ${MAX_MEMORIES_PER_RUN} items, each {"kind":"${MEMORY_KINDS.join('|')}","text":"one self-contained sentence in the language the room uses, at most ${MAX_MEMORY_CHARS} characters, naming files, agents and numbers exactly","correction":"only on an aberration: what is true instead","sources":[sequence numbers it comes from]}. If nothing durable was said: {"memories":[]}`
|
|
48
49
|
: `Output only JSON lines, one object per memory, at most ${MAX_MEMORIES_PER_RUN}, nothing else:`,
|
|
49
|
-
json ? null : `{"kind":"
|
|
50
|
+
json ? null : `{"kind":"${MEMORY_KINDS.join('|')}","text":"one self-contained sentence in the language the room uses, at most ${MAX_MEMORY_CHARS} characters, naming files, agents and numbers exactly","correction":"only on an aberration: what is true instead","sources":[sequence numbers it comes from]}`,
|
|
50
51
|
json ? null : 'If nothing durable was said, output exactly: NONE',
|
|
51
52
|
known,
|
|
52
53
|
'<exchanges>',
|
|
@@ -76,13 +77,19 @@ export function parseDistillation(text, { fromSequence, throughSequence } = {})
|
|
|
76
77
|
if (!parsed || typeof parsed !== 'object') continue;
|
|
77
78
|
const memoryText = String(parsed.text ?? '').replace(/\s+/g, ' ').trim();
|
|
78
79
|
if (!memoryText) continue;
|
|
79
|
-
|
|
80
|
+
// An unknown kind used to be filed as a fact. That was harmless while every kind was
|
|
81
|
+
// knowledge; now the list has one that is not, and a typo would turn a hallucination into
|
|
82
|
+
// something the room believes. Anything unrecognised is dropped instead.
|
|
83
|
+
if (!MEMORY_KINDS.includes(parsed.kind)) continue;
|
|
84
|
+
const kind = parsed.kind;
|
|
80
85
|
const inRange = (n) => Number.isInteger(n) && (fromSequence == null || n >= fromSequence) && (throughSequence == null || n <= throughSequence);
|
|
81
86
|
const sources = (Array.isArray(parsed.sources) ? parsed.sources : []).map(Number).filter(inRange);
|
|
82
87
|
const key = memoryText.toLowerCase();
|
|
83
88
|
if (seen.has(key)) continue;
|
|
84
89
|
seen.add(key);
|
|
85
|
-
|
|
90
|
+
const clip = (value) => (value.length > MAX_MEMORY_CHARS ? `${value.slice(0, MAX_MEMORY_CHARS - 1)}…` : value);
|
|
91
|
+
const correction = kind === ABERRATION ? clip(String(parsed.correction ?? '').replace(/\s+/g, ' ').trim()) : '';
|
|
92
|
+
memories.push({ kind, text: clip(memoryText), sources, ...(correction ? { correction } : {}) });
|
|
86
93
|
if (memories.length >= MAX_MEMORIES_PER_RUN) break;
|
|
87
94
|
}
|
|
88
95
|
return memories;
|
package/src/event-store.mjs
CHANGED
|
@@ -8,16 +8,22 @@ export class EventStore {
|
|
|
8
8
|
#nextSequence = 1;
|
|
9
9
|
#writeQueue = Promise.resolve();
|
|
10
10
|
#lockDirectory;
|
|
11
|
+
#floor = 0;
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
// `floor` is the highest sequence the rest of the project has already used. A project keeps one
|
|
14
|
+
// numbering across all of its conversations, so #1411 means one exchange in this project and
|
|
15
|
+
// not one in each thread — the memory index, every citation and NOSTROMO all lean on that.
|
|
16
|
+
// Gaps in a single ledger are fine: sequences are an order, not a count.
|
|
17
|
+
constructor(file, { floor = 0 } = {}) {
|
|
13
18
|
this.#file = file;
|
|
19
|
+
this.#floor = Number.isFinite(floor) ? floor : 0;
|
|
14
20
|
this.#lockDirectory = `${file}.lock`;
|
|
15
21
|
}
|
|
16
22
|
|
|
17
23
|
async initialize() {
|
|
18
24
|
await mkdir(dirname(this.#file), { recursive: true });
|
|
19
25
|
const existing = await this.readAll();
|
|
20
|
-
this.#nextSequence = (existing.at(-1)?.sequence ?? 0) + 1;
|
|
26
|
+
this.#nextSequence = Math.max(existing.at(-1)?.sequence ?? 0, this.#floor) + 1;
|
|
21
27
|
return this;
|
|
22
28
|
}
|
|
23
29
|
|
|
@@ -85,14 +91,14 @@ export class EventStore {
|
|
|
85
91
|
return async () => {
|
|
86
92
|
try {
|
|
87
93
|
const owner = JSON.parse(await readFile(ownerFile, 'utf8'));
|
|
88
|
-
if (owner.token === token) await rm(this.#lockDirectory, { recursive: true, force: true });
|
|
94
|
+
if (owner.token === token) await rm(this.#lockDirectory, { recursive: true, force: true, maxRetries: 6, retryDelay: 60 });
|
|
89
95
|
} catch (error) {
|
|
90
96
|
if (error.code !== 'ENOENT') throw error;
|
|
91
97
|
}
|
|
92
98
|
};
|
|
93
99
|
} catch (error) {
|
|
94
100
|
if (created) {
|
|
95
|
-
await rm(this.#lockDirectory, { recursive: true, force: true });
|
|
101
|
+
await rm(this.#lockDirectory, { recursive: true, force: true, maxRetries: 6, retryDelay: 60 });
|
|
96
102
|
throw error;
|
|
97
103
|
}
|
|
98
104
|
if (error.code !== 'EEXIST') throw error;
|
|
@@ -102,7 +108,7 @@ export class EventStore {
|
|
|
102
108
|
process.kill(owner.pid, 0);
|
|
103
109
|
} catch (ownerError) {
|
|
104
110
|
if (ownerError.code === 'ESRCH') {
|
|
105
|
-
await rm(this.#lockDirectory, { recursive: true, force: true });
|
|
111
|
+
await rm(this.#lockDirectory, { recursive: true, force: true, maxRetries: 6, retryDelay: 60 });
|
|
106
112
|
continue;
|
|
107
113
|
}
|
|
108
114
|
}
|
|
@@ -110,7 +116,7 @@ export class EventStore {
|
|
|
110
116
|
if (ownerError.code !== 'ENOENT' && !(ownerError instanceof SyntaxError)) throw ownerError;
|
|
111
117
|
const lock = await stat(this.#lockDirectory).catch(() => null);
|
|
112
118
|
if (lock && Date.now() - lock.mtimeMs > 10000) {
|
|
113
|
-
await rm(this.#lockDirectory, { recursive: true, force: true });
|
|
119
|
+
await rm(this.#lockDirectory, { recursive: true, force: true, maxRetries: 6, retryDelay: 60 });
|
|
114
120
|
continue;
|
|
115
121
|
}
|
|
116
122
|
}
|
|
@@ -138,7 +144,7 @@ export class EventStore {
|
|
|
138
144
|
const temp = `${this.#file}.rewrite-${process.pid}`;
|
|
139
145
|
await writeFile(temp, next.map((event) => JSON.stringify(event)).join('\n') + (next.length ? '\n' : ''));
|
|
140
146
|
await rename(temp, this.#file);
|
|
141
|
-
this.#nextSequence = (next.at(-1)?.sequence ?? 0) + 1;
|
|
147
|
+
this.#nextSequence = Math.max(next.at(-1)?.sequence ?? 0, this.#floor) + 1;
|
|
142
148
|
return { total: next.length, changed };
|
|
143
149
|
} finally {
|
|
144
150
|
await release();
|
|
@@ -153,7 +159,7 @@ export class EventStore {
|
|
|
153
159
|
const release = await this.#acquireLock();
|
|
154
160
|
try {
|
|
155
161
|
const existing = await this.#readAllUnlocked();
|
|
156
|
-
const sequence = (existing.at(-1)?.sequence ?? 0) + 1;
|
|
162
|
+
const sequence = Math.max(existing.at(-1)?.sequence ?? 0, this.#floor) + 1;
|
|
157
163
|
const event = {
|
|
158
164
|
id: randomUUID(),
|
|
159
165
|
sequence,
|
package/src/exam.mjs
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
// The three tests that answer "is this room ready to be developed with?"
|
|
2
|
+
//
|
|
3
|
+
// The maturity reading counts what the archive is made of. It cannot tell you whether the archive
|
|
4
|
+
// works, because nothing about a pile of notes says whether the right one comes back when it is
|
|
5
|
+
// needed. Only a test says that, and a test is only worth running if it can fail.
|
|
6
|
+
//
|
|
7
|
+
// 1 · COVERAGE — when this project asks its own questions, does the archive already hold the
|
|
8
|
+
// answer? Real human messages from the ledger, recall run at the point each
|
|
9
|
+
// one was asked, scored against the reply that was actually given.
|
|
10
|
+
// 2 · CONSISTENCY — does the archive contradict itself? Contradictions EYECAT is still holding,
|
|
11
|
+
// what has been taken out of circulation, and whether aberrations are being
|
|
12
|
+
// filed more often lately or less.
|
|
13
|
+
// 3 · MATCH — does the local model, with this archive behind it, land where the frontier
|
|
14
|
+
// CLI landed? Real questions from the ledger, asked again locally, compared
|
|
15
|
+
// against the answer that was given at the time.
|
|
16
|
+
//
|
|
17
|
+
// What every one of them refuses to do is grade itself generously. Each reports how it measured
|
|
18
|
+
// (meaning or words), how many cases it had, and the bar it used, because a number without those
|
|
19
|
+
// three is a decoration.
|
|
20
|
+
|
|
21
|
+
import { t } from './i18n.mjs';
|
|
22
|
+
|
|
23
|
+
export const COVERAGE_SAMPLE = 30;
|
|
24
|
+
export const COVERAGE_BAR = 0.55; // cosine between the reply and the closest thing recalled
|
|
25
|
+
export const COVERAGE_WORDS_BAR = 0.25;
|
|
26
|
+
export const MATCH_SAMPLE = 12; // a local model answering is slow; honesty about that beats a big number
|
|
27
|
+
export const MATCH_BAR = 0.6;
|
|
28
|
+
// Every text in one room is about the same handful of subjects, so any two pieces of it read as
|
|
29
|
+
// similar to an embedder. Measured against a bar alone, the first run of coverage scored thirty
|
|
30
|
+
// out of thirty — a test that cannot fail measures nothing. So each case is also measured against
|
|
31
|
+
// a control: the same question scored against another case's material. To count, what the archive
|
|
32
|
+
// actually handed over has to beat what it would have handed over for something else.
|
|
33
|
+
export const CONTROL_MARGIN = 0.05;
|
|
34
|
+
export const MIN_ASK = 40; // shorter than this is "ok", "sí", "dale": nothing to answer
|
|
35
|
+
export const PASS = { coverage: 0.7, match: 0.7 };
|
|
36
|
+
|
|
37
|
+
const rare = (text) => new Set(String(text ?? '').toLowerCase().normalize('NFD').replace(/[̀-ͯ]/g, '').match(/[a-z0-9][a-z0-9_/.-]{3,}/g) ?? []);
|
|
38
|
+
|
|
39
|
+
// How close two pieces of text are when there is nothing to embed them with. Not a semantic
|
|
40
|
+
// score and never reported as one: the share of the answer's own uncommon words that were
|
|
41
|
+
// already in front of the room.
|
|
42
|
+
export function wordScore(reference, candidate) {
|
|
43
|
+
const want = rare(reference);
|
|
44
|
+
const have = rare(candidate);
|
|
45
|
+
if (!want.size || !have.size) return 0;
|
|
46
|
+
let shared = 0;
|
|
47
|
+
for (const word of want) if (have.has(word)) shared += 1;
|
|
48
|
+
return shared / want.size;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function cosine(a, b) {
|
|
52
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
53
|
+
let dot = 0;
|
|
54
|
+
let left = 0;
|
|
55
|
+
let right = 0;
|
|
56
|
+
for (let i = 0; i < a.length; i += 1) { dot += a[i] * b[i]; left += a[i] * a[i]; right += b[i] * b[i]; }
|
|
57
|
+
return left && right ? dot / Math.sqrt(left * right) : 0;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Every human message in the ledger that actually got an answer, oldest first. Ghost turns never
|
|
61
|
+
// happened as far as the archive is concerned, and a one-word message has no answer to find.
|
|
62
|
+
export function exchanges(events = [], { minAsk = MIN_ASK, local = ['madre'] } = {}) {
|
|
63
|
+
const asked = new Map();
|
|
64
|
+
const out = [];
|
|
65
|
+
for (const event of events) {
|
|
66
|
+
if (!event || event.ghost) continue;
|
|
67
|
+
const payload = event.payload ?? {};
|
|
68
|
+
if (event.type !== 'message.created') continue;
|
|
69
|
+
if (payload.role === 'user' && payload.sender === 'you') {
|
|
70
|
+
const text = String(payload.text ?? '').trim();
|
|
71
|
+
if (text.length >= minAsk && !text.startsWith('/')) asked.set(payload.messageId, { sequence: event.sequence, text, target: payload.target });
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (payload.role !== 'assistant') continue;
|
|
75
|
+
const parent = payload.parentMessageId ?? payload.replyTo ?? payload.inReplyTo ?? null;
|
|
76
|
+
const question = parent ? asked.get(parent) : null;
|
|
77
|
+
if (!question) continue;
|
|
78
|
+
const answer = String(payload.text ?? '').trim();
|
|
79
|
+
if (answer.length < minAsk) continue;
|
|
80
|
+
asked.delete(parent);
|
|
81
|
+
out.push({ sequence: question.sequence, answerSequence: event.sequence, asked: question.text, answered: answer, agent: payload.sender, local: local.includes(payload.sender) });
|
|
82
|
+
}
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// The control partner for each case: far enough down the list that it is a different day and a
|
|
87
|
+
// different subject, and deterministic so a run can be repeated.
|
|
88
|
+
export const controlOf = (index, n) => (n > 1 ? (index + Math.floor(n / 2)) % n : 0);
|
|
89
|
+
|
|
90
|
+
// A sample spread across the whole ledger rather than taken off the end: fifty questions from one
|
|
91
|
+
// afternoon test one afternoon.
|
|
92
|
+
export function spread(items, size) {
|
|
93
|
+
if (items.length <= size) return [...items];
|
|
94
|
+
const step = items.length / size;
|
|
95
|
+
return Array.from({ length: size }, (_, i) => items[Math.floor(i * step)]);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/* ---------- 1 · COVERAGE ---------- */
|
|
99
|
+
|
|
100
|
+
// `recall(text, beforeSequence)` hands back what the room would have been given at that moment.
|
|
101
|
+
// `embed(texts)` is optional; without it the scoring says so and uses words.
|
|
102
|
+
export async function coverageExam({ events = [], recall, embed = null, sample = COVERAGE_SAMPLE, bar = null } = {}) {
|
|
103
|
+
const all = exchanges(events);
|
|
104
|
+
const cases = spread(all, sample);
|
|
105
|
+
if (!cases.length) return { id: 'coverage', ran: false, says: t('No exchange in this room is long enough to test with yet.') };
|
|
106
|
+
const found = [];
|
|
107
|
+
for (const one of cases) {
|
|
108
|
+
const held = await recall(one.asked, one.sequence);
|
|
109
|
+
found.push(held.map((item) => String(item ?? '')).filter(Boolean));
|
|
110
|
+
}
|
|
111
|
+
const method = embed ? 'meaning' : 'words';
|
|
112
|
+
const used = bar ?? (embed ? COVERAGE_BAR : COVERAGE_WORDS_BAR);
|
|
113
|
+
const scores = [];
|
|
114
|
+
const controls = [];
|
|
115
|
+
if (embed) {
|
|
116
|
+
// One batch for the answers and one for everything that was recalled, so a remote embedder
|
|
117
|
+
// is asked twice and not sixty times.
|
|
118
|
+
const flat = found.flat();
|
|
119
|
+
const vectors = await embed([...cases.map((one) => one.answered), ...flat]);
|
|
120
|
+
const answers = vectors.slice(0, cases.length);
|
|
121
|
+
const rest = vectors.slice(cases.length);
|
|
122
|
+
const sets = [];
|
|
123
|
+
let at = 0;
|
|
124
|
+
for (const items of found) { sets.push(rest.slice(at, at + items.length)); at += items.length; }
|
|
125
|
+
for (let i = 0; i < cases.length; i += 1) {
|
|
126
|
+
const best = (set) => set.reduce((top, vector) => Math.max(top, cosine(answers[i], vector)), 0);
|
|
127
|
+
scores.push(best(sets[i]));
|
|
128
|
+
controls.push(best(sets[controlOf(i, cases.length)]));
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
for (let i = 0; i < cases.length; i += 1) {
|
|
132
|
+
const best = (set) => set.reduce((top, text) => Math.max(top, wordScore(cases[i].answered, text)), 0);
|
|
133
|
+
scores.push(best(found[i]));
|
|
134
|
+
controls.push(best(found[controlOf(i, cases.length)]));
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// To count, the archive must have handed over this answer's material, and material it would
|
|
138
|
+
// not have handed over for a different question. With a single case there is nothing to
|
|
139
|
+
// compare against, and the result says so rather than pretending otherwise.
|
|
140
|
+
const controlled = cases.length > 1;
|
|
141
|
+
const hit = scores.map((score, i) => score >= used && (!controlled || score > controls[i] + CONTROL_MARGIN));
|
|
142
|
+
const hits = hit.filter(Boolean).length;
|
|
143
|
+
const rate = Number((hits / cases.length).toFixed(3));
|
|
144
|
+
const mean = (list) => Number((list.reduce((sum, value) => sum + value, 0) / list.length).toFixed(3));
|
|
145
|
+
return {
|
|
146
|
+
id: 'coverage', ran: true, at: new Date().toISOString(), n: cases.length, hits, rate, method, bar: used,
|
|
147
|
+
mean: mean(scores), control: controlled ? mean(controls) : null,
|
|
148
|
+
passed: rate >= PASS.coverage,
|
|
149
|
+
says: t('{hits} of {n} questions this room actually asked had their answer already in the archive, matched by {method}{control}.', { hits, n: cases.length, method: t(method), control: controlled ? t(' and against a control') : '' }),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/* ---------- 2 · CONSISTENCY ---------- */
|
|
154
|
+
|
|
155
|
+
// Pure. Contradictions still open, how much has been taken out of circulation, and whether the
|
|
156
|
+
// room is filing aberrations more often lately than it used to.
|
|
157
|
+
export function consistencyExam({ findings = [], notes = [], now = Date.now() } = {}) {
|
|
158
|
+
const standing = notes.filter((note) => note && note.kind !== 'aberration' && !note.refutedBy);
|
|
159
|
+
const refuted = notes.filter((note) => note && note.refutedBy).length;
|
|
160
|
+
const aberrations = notes.filter((note) => note && note.kind === 'aberration');
|
|
161
|
+
const open = findings.filter((finding) => !finding?.settled).length;
|
|
162
|
+
const times = aberrations.map((note) => Date.parse(note.created)).filter((at) => Number.isFinite(at)).sort((a, b) => a - b);
|
|
163
|
+
const written = notes.map((note) => Date.parse(note.created)).filter((at) => Number.isFinite(at)).sort((a, b) => a - b);
|
|
164
|
+
let trend = null;
|
|
165
|
+
if (written.length > 4 && times.length) {
|
|
166
|
+
const middle = written[Math.floor(written.length / 2)];
|
|
167
|
+
const before = times.filter((at) => at < middle).length;
|
|
168
|
+
const after = times.length - before;
|
|
169
|
+
trend = after === before ? 'level' : after < before ? 'falling' : 'rising';
|
|
170
|
+
}
|
|
171
|
+
const passed = open === 0 && trend !== 'rising';
|
|
172
|
+
return {
|
|
173
|
+
id: 'consistency', ran: true, at: new Date(now).toISOString(),
|
|
174
|
+
open, refuted, standing: standing.length, aberrations: aberrations.length, trend,
|
|
175
|
+
passed,
|
|
176
|
+
says: open
|
|
177
|
+
? t('EYECAT is holding {n} contradictions nobody has settled.', { n: open })
|
|
178
|
+
: trend === 'rising'
|
|
179
|
+
? t('Nothing is open, but aberrations are being filed more often lately than they used to be.')
|
|
180
|
+
: t('Nothing contradicts anything: {standing} notes stand, {refuted} were taken out of circulation.', { standing: standing.length, refuted }),
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/* ---------- 3 · MATCH ---------- */
|
|
185
|
+
|
|
186
|
+
// `ask(question)` is the local model with this archive behind it. `embed` is needed: comparing
|
|
187
|
+
// two answers by their words rewards copying the question back.
|
|
188
|
+
export async function matchExam({ events = [], ask, embed, sample = MATCH_SAMPLE, bar = MATCH_BAR, onProgress = () => {}, stop = () => false } = {}) {
|
|
189
|
+
if (!embed) return { id: 'match', ran: false, says: t('This test needs embeddings: two answers cannot be compared by their words alone.') };
|
|
190
|
+
const all = exchanges(events).filter((one) => !one.local); // a model is not tested against itself
|
|
191
|
+
const cases = spread(all, sample);
|
|
192
|
+
if (!cases.length) return { id: 'match', ran: false, says: t('No question in this room was answered by an agent other than the local one yet.') };
|
|
193
|
+
const mine = [];
|
|
194
|
+
for (const [index, one] of cases.entries()) {
|
|
195
|
+
if (stop()) return { id: 'match', ran: false, stopped: true, says: t('Stopped after {index} of {n}.', { index, n: cases.length }) };
|
|
196
|
+
onProgress({ done: index, total: cases.length });
|
|
197
|
+
try { mine.push(String((await ask(one.asked)) ?? '')); } catch { mine.push(''); }
|
|
198
|
+
}
|
|
199
|
+
onProgress({ done: cases.length, total: cases.length });
|
|
200
|
+
const vectors = await embed([...cases.map((one) => one.answered), ...mine]);
|
|
201
|
+
const theirs = vectors.slice(0, cases.length);
|
|
202
|
+
const ours = vectors.slice(cases.length);
|
|
203
|
+
const scores = cases.map((_, i) => (mine[i] ? cosine(theirs[i], ours[i]) : 0));
|
|
204
|
+
// Same control as coverage: an answer that is merely about the same project as every other
|
|
205
|
+
// answer in the room has not landed anywhere.
|
|
206
|
+
const controlled = cases.length > 1;
|
|
207
|
+
const controls = cases.map((_, i) => (mine[i] && controlled ? cosine(ours[i], theirs[controlOf(i, cases.length)]) : 0));
|
|
208
|
+
const matched = scores.filter((score, i) => score >= bar && (!controlled || score > controls[i] + CONTROL_MARGIN)).length;
|
|
209
|
+
const rate = Number((matched / cases.length).toFixed(3));
|
|
210
|
+
const mean = (list) => Number((list.reduce((sum, value) => sum + value, 0) / list.length).toFixed(3));
|
|
211
|
+
return {
|
|
212
|
+
id: 'match', ran: true, at: new Date().toISOString(), n: cases.length, matched, rate, mean: mean(scores), control: controlled ? mean(controls) : null, bar,
|
|
213
|
+
passed: rate >= PASS.match,
|
|
214
|
+
says: t('On {matched} of {n} real questions the local model landed where the agent of the day landed{control}.', { matched, n: cases.length, control: controlled ? t(', and not merely in the same project') : '' }),
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// A reading that was taken months ago is still read today, and the room may have changed its
|
|
219
|
+
// language since. The numbers are what was measured; the sentence is only how they are said, so
|
|
220
|
+
// it is said again, now, from the fields that were stored. A stored result that predates this
|
|
221
|
+
// keeps whatever sentence it was written with.
|
|
222
|
+
export function saysFor(result) {
|
|
223
|
+
if (!result?.ran) return result?.says ?? null;
|
|
224
|
+
if (result.id === 'coverage' && Number.isFinite(result.hits)) {
|
|
225
|
+
return t('{hits} of {n} questions this room actually asked had their answer already in the archive, matched by {method}{control}.',
|
|
226
|
+
{ hits: result.hits, n: result.n, method: t(result.method ?? 'words'), control: result.control === null || result.control === undefined ? '' : t(' and against a control') });
|
|
227
|
+
}
|
|
228
|
+
if (result.id === 'match' && Number.isFinite(result.matched)) {
|
|
229
|
+
return t('On {matched} of {n} real questions the local model landed where the agent of the day landed{control}.',
|
|
230
|
+
{ matched: result.matched, n: result.n, control: result.control === null || result.control === undefined ? '' : t(', and not merely in the same project') });
|
|
231
|
+
}
|
|
232
|
+
if (result.id === 'consistency' && Number.isFinite(result.open)) {
|
|
233
|
+
return result.open
|
|
234
|
+
? t('EYECAT is holding {n} contradictions nobody has settled.', { n: result.open })
|
|
235
|
+
: result.trend === 'rising'
|
|
236
|
+
? t('Nothing is open, but aberrations are being filed more often lately than they used to be.')
|
|
237
|
+
: t('Nothing contradicts anything: {standing} notes stand, {refuted} were taken out of circulation.', { standing: result.standing, refuted: result.refuted });
|
|
238
|
+
}
|
|
239
|
+
return result.says ?? null;
|
|
240
|
+
}
|
package/src/extensions.mjs
CHANGED
|
@@ -8,7 +8,7 @@ export const EXTENSIONS = MODULES;
|
|
|
8
8
|
export const AHP = moduleById('ahp');
|
|
9
9
|
export const IMAGE_STUDIO = moduleById('image-studio');
|
|
10
10
|
export const GIT_PULSE = moduleById('git-pulse');
|
|
11
|
-
export const
|
|
11
|
+
export const ASH = moduleById('ash');
|
|
12
12
|
export const RIPLEY = moduleById('ripley');
|
|
13
13
|
export const OLLAMA = moduleById('ollama');
|
|
14
14
|
export const extensionById = moduleById;
|
|
@@ -26,9 +26,10 @@ export async function listExtensions({ projectRoot, agents = [], config = {}, im
|
|
|
26
26
|
|
|
27
27
|
// Runs one installer inside the project, streaming output lines. `runner`
|
|
28
28
|
// lets tests substitute the real installer with a local script.
|
|
29
|
+
// `npm` and other Node CLIs are .cmd shims on Windows, which cannot be spawned directly.
|
|
29
30
|
export function runInstaller({ command, args, projectRoot, onLine, timeoutMs = 600000, heartbeatMs = 8000, env = process.env }) {
|
|
30
31
|
return new Promise((resolve) => {
|
|
31
|
-
const child = spawn(command, args, { cwd: projectRoot, env: { ...env, NO_COLOR: '1', CI: '1' }, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
32
|
+
const child = spawn(command, args, { cwd: projectRoot, env: { ...env, NO_COLOR: '1', CI: '1' }, stdio: ['ignore', 'pipe', 'pipe'], shell: process.platform === 'win32', windowsHide: true });
|
|
32
33
|
const started = Date.now();
|
|
33
34
|
let lastOutput = started;
|
|
34
35
|
// Installers go quiet for long stretches (npm install, git status); say so.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// EYECAT, as the room runs it. The judgement lives in eyecat.mjs; this schedules it, picks who
|
|
2
|
+
// may answer, and puts what comes back in front of the human.
|
|
3
|
+
//
|
|
4
|
+
// It is not a module and not part of the room. It watches from outside, the way the error
|
|
5
|
+
// sentinel does: it subscribes to the ledger, it never takes a turn, it writes no file and it
|
|
6
|
+
// holds no permission. Nothing an agent says can reach it, and nothing it decides is final.
|
|
7
|
+
// What it produces is a question for a person, never an entry in the archive.
|
|
8
|
+
|
|
9
|
+
import { suspectPairs, unsupportedNotes, judgeFor, verdictPrompt, parseVerdict } from './eyecat.mjs';
|
|
10
|
+
|
|
11
|
+
export const EYECAT_MAX_PER_SWEEP = 3; // a ceiling, so a large archive cannot run up a bill
|
|
12
|
+
export const EYECAT_FLOOR = 0.72; // how close two notes must be to count as the same subject
|
|
13
|
+
export const EYECAT_MIN_CONFIDENCE = 0.5; // below this a verdict is not worth a person's attention
|
|
14
|
+
|
|
15
|
+
export class Eyecat {
|
|
16
|
+
#deps;
|
|
17
|
+
#settled = new Set();
|
|
18
|
+
#open = new Map();
|
|
19
|
+
#running = false;
|
|
20
|
+
|
|
21
|
+
constructor(deps) { this.#deps = deps; }
|
|
22
|
+
|
|
23
|
+
settings() { return { enabled: this.#deps.enabled?.() !== false, floor: EYECAT_FLOOR, maxPerSweep: EYECAT_MAX_PER_SWEEP }; }
|
|
24
|
+
findings() { return [...this.#open.values()].sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0)); }
|
|
25
|
+
|
|
26
|
+
// What a person already decided, replayed from the ledger so a restart does not ask twice.
|
|
27
|
+
seed(events = []) {
|
|
28
|
+
for (const event of events) {
|
|
29
|
+
if (event?.type === 'eyecat.flagged' && event.payload?.key) this.#open.set(event.payload.key, event.payload);
|
|
30
|
+
if (event?.type === 'eyecat.settled' && event.payload?.key) { this.#settled.add(event.payload.key); this.#open.delete(event.payload.key); }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// The room only ever distils between turns, so this is already out of band: no agent is
|
|
35
|
+
// waiting on it and none of them will see the answer.
|
|
36
|
+
async observe(event) {
|
|
37
|
+
if (event?.type !== 'memory.distilled') return null;
|
|
38
|
+
return this.sweep({ reason: 'distilled' }).catch(() => null);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
settle(key, { verdict = 'dismissed' } = {}) {
|
|
42
|
+
if (!this.#open.has(key) && this.#settled.has(key)) return null;
|
|
43
|
+
const finding = this.#open.get(key) ?? { key };
|
|
44
|
+
this.#settled.add(key);
|
|
45
|
+
this.#open.delete(key);
|
|
46
|
+
return { ...finding, settledAs: verdict };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async sweep({ reason = 'asked' } = {}) {
|
|
50
|
+
if (this.#running || this.#deps.enabled?.() === false) return null;
|
|
51
|
+
const research = this.#deps.research?.();
|
|
52
|
+
if (!research?.memories?.length) return null;
|
|
53
|
+
this.#running = true;
|
|
54
|
+
try {
|
|
55
|
+
const skip = new Set([...this.#settled, ...this.#open.keys()]);
|
|
56
|
+
const candidates = [
|
|
57
|
+
...suspectPairs(research.links ?? [], research.memories, { floor: EYECAT_FLOOR, settled: skip }),
|
|
58
|
+
...unsupportedNotes(research.memories, this.#deps.entriesFor?.(research.memories) ?? [], { settled: skip }),
|
|
59
|
+
].slice(0, EYECAT_MAX_PER_SWEEP);
|
|
60
|
+
const raised = [];
|
|
61
|
+
for (const candidate of candidates) {
|
|
62
|
+
const finding = await this.#judge(candidate, reason).catch(() => null);
|
|
63
|
+
if (finding) raised.push(finding);
|
|
64
|
+
}
|
|
65
|
+
return raised.length ? raised : null;
|
|
66
|
+
} finally {
|
|
67
|
+
this.#running = false;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async #judge(candidate, reason) {
|
|
72
|
+
const judge = judgeFor(candidate, this.#deps.bench?.() ?? {});
|
|
73
|
+
// With nobody impartial free, the candidate is left alone rather than handed to an agent
|
|
74
|
+
// with a stake in the answer. It will come round again on the next sweep.
|
|
75
|
+
if (!judge) return null;
|
|
76
|
+
const invoke = this.#deps.bench?.().invokers?.[judge.adapter];
|
|
77
|
+
if (!invoke) return null;
|
|
78
|
+
const cited = candidate.kind === 'unsupported' ? (this.#deps.entriesFor?.([candidate.claim]) ?? []).filter((entry) => candidate.cites.includes(entry.sequence)).map((entry) => entry.text) : [];
|
|
79
|
+
const answer = await invoke({ prompt: verdictPrompt({ ...candidate, citedText: cited }, { json: judge.local === true }), json: judge.local === true });
|
|
80
|
+
const verdict = parseVerdict(String(answer ?? ''), candidate);
|
|
81
|
+
if (!verdict || verdict.verdict !== 'contradiction') return null;
|
|
82
|
+
if (verdict.confidence !== null && verdict.confidence < EYECAT_MIN_CONFIDENCE) return null;
|
|
83
|
+
const accused = candidate.kind === 'unsupported' ? candidate.claim : (verdict.accused === candidate.against?.id ? candidate.against : candidate.claim);
|
|
84
|
+
const finding = {
|
|
85
|
+
key: candidate.key,
|
|
86
|
+
kind: candidate.kind,
|
|
87
|
+
reason,
|
|
88
|
+
judge: judge.id,
|
|
89
|
+
signals: candidate.signals ?? [],
|
|
90
|
+
confidence: verdict.confidence,
|
|
91
|
+
correction: verdict.correction || null,
|
|
92
|
+
claim: { id: accused.id, text: accused.text, kind: accused.kind, agent: accused.agent },
|
|
93
|
+
against: candidate.against && candidate.against.id !== accused.id ? { id: candidate.against.id, text: candidate.against.text } : null,
|
|
94
|
+
at: new Date().toISOString(),
|
|
95
|
+
};
|
|
96
|
+
this.#open.set(finding.key, finding);
|
|
97
|
+
await this.#deps.emit('eyecat.flagged', finding);
|
|
98
|
+
return finding;
|
|
99
|
+
}
|
|
100
|
+
}
|