@jossuealcala/madre 0.7.1 → 1.0.0

package/src/event-store.mjs

@@ -3,8 +3,31 @@ import { dirname, join } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { setTimeout as delay } from 'node:timers/promises';
 
+// A record cut halfway (2026-10-12, LOG-02 of the 1.0 review): a `kill -9`, a power cut or a full
+// disk in the middle of an append leaves half a line. Every read used to throw on it, so the
+// conversation never opened again — and the next event was written glued to the half line, lost
+// with it. Now a line that is not an event is stepped over and counted (`damaged`), never removed:
+// the file keeps it for whoever wants to look. And an append that finds the file not ending in a
+// newline writes one first, so what comes next starts on a line of its own.
+function parseLines(text) {
+  const events = [];
+  let damaged = 0;
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      const event = JSON.parse(line);
+      if (event && typeof event === 'object' && Number.isFinite(event.sequence)) events.push(event);
+      else damaged += 1;
+    } catch {
+      damaged += 1;
+    }
+  }
+  return { events, damaged };
+}
+
 export class EventStore {
   #file;
+  #damaged = 0;
   #nextSequence = 1;
   #writeQueue = Promise.resolve();
   #lockDirectory;
@@ -14,16 +37,29 @@ export class EventStore {
   // numbering across all of its conversations, so #1411 means one exchange in this project and
   // not one in each thread — the memory index, every citation and NOSTROMO all lean on that.
   // Gaps in a single ledger are fine: sequences are an order, not a count.
+  // It may be a function, read at every append, when other conversations of the project are
+  // writing at the same time as this one: a number fixed when the ledger opened would hand out
+  // a sequence another conversation has taken since.
   constructor(file, { floor = 0 } = {}) {
     this.#file = file;
-    this.#floor = Number.isFinite(floor) ? floor : 0;
+    this.#floor = typeof floor === 'function' || Number.isFinite(floor) ? floor : 0;
     this.#lockDirectory = `${file}.lock`;
   }
 
+  #floorNow() {
+    const value = typeof this.#floor === 'function' ? Number(this.#floor()) : this.#floor;
+    return Number.isFinite(value) ? value : 0;
+  }
+
+  // The last sequence this ledger handed out (or the floor it opened above).
+  get lastSequence() { return this.#nextSequence - 1; }
+  // How many lines the last full read stepped over because they were not events.
+  get damaged() { return this.#damaged; }
+
   async initialize() {
     await mkdir(dirname(this.#file), { recursive: true });
     const existing = await this.readAll();
-    this.#nextSequence = Math.max(existing.at(-1)?.sequence ?? 0, this.#floor) + 1;
+    this.#nextSequence = Math.max(existing.at(-1)?.sequence ?? 0, this.#floorNow()) + 1;
     return this;
   }
 
@@ -59,7 +95,7 @@ export class EventStore {
         const lastNewline = text.lastIndexOf('\n');
         if (lastNewline < 0) return { events: [], offset: start };
         text = text.slice(0, lastNewline + 1);
-        const events = text.split('\n').filter(Boolean).map((line) => JSON.parse(line));
+        const { events } = parseLines(text);
         return { events, offset: start + Buffer.byteLength(text, 'utf8') };
       } finally {
         await handle.close();
@@ -71,8 +107,9 @@ export class EventStore {
 
   async #readAllUnlocked() {
     try {
-      const text = await readFile(this.#file, 'utf8');
-      return text.split('\n').filter(Boolean).map((line) => JSON.parse(line));
+      const { events, damaged } = parseLines(await readFile(this.#file, 'utf8'));
+      this.#damaged = damaged;
+      return events;
     } catch (error) {
       if (error.code === 'ENOENT') return [];
       throw error;
@@ -144,7 +181,7 @@ export class EventStore {
         const temp = `${this.#file}.rewrite-${process.pid}`;
         await writeFile(temp, next.map((event) => JSON.stringify(event)).join('\n') + (next.length ? '\n' : ''));
         await rename(temp, this.#file);
-        this.#nextSequence = Math.max(next.at(-1)?.sequence ?? 0, this.#floor) + 1;
+        this.#nextSequence = Math.max(next.at(-1)?.sequence ?? 0, this.#floorNow()) + 1;
         return { total: next.length, changed };
       } finally {
         await release();
@@ -159,7 +196,11 @@ export class EventStore {
       const release = await this.#acquireLock();
       try {
         const existing = await this.#readAllUnlocked();
-        const sequence = Math.max(existing.at(-1)?.sequence ?? 0, this.#floor) + 1;
+        const sequence = Math.max(existing.at(-1)?.sequence ?? 0, this.#floorNow()) + 1;
+        // Taken in the same tick it was read: another conversation's ledger asks for the floor
+        // while this one is still on its way to the disk, and must already see this number as
+        // spent. A write that fails leaves a gap, and gaps are fine.
+        this.#nextSequence = sequence + 1;
         const event = {
           id: randomUUID(),
           sequence,
@@ -167,14 +208,18 @@ export class EventStore {
           timestamp: new Date().toISOString(),
           payload,
         };
-        const handle = await open(this.#file, 'a');
+        const handle = await open(this.#file, 'a+');
         try {
-          await handle.writeFile(`${JSON.stringify(event)}\n`);
+          // A file that does not end in a newline ends in half a line: start on a new one.
+          const { size } = await handle.stat();
+          const last = Buffer.alloc(1);
+          if (size > 0) await handle.read(last, 0, 1, size - 1);
+          const lead = size > 0 && last[0] !== 0x0a ? '\n' : '';
+          await handle.writeFile(`${lead}${JSON.stringify(event)}\n`);
           await handle.sync();
         } finally {
           await handle.close();
         }
-        this.#nextSequence = sequence + 1;
         return event;
       } finally {
         await release();

package/src/exam.mjs

@@ -102,7 +102,7 @@ export function spread(items, size) {
 export async function coverageExam({ events = [], recall, embed = null, sample = COVERAGE_SAMPLE, bar = null } = {}) {
   const all = exchanges(events);
   const cases = spread(all, sample);
-  if (!cases.length) return { id: 'coverage', ran: false, says: t('No exchange in this room is long enough to test with yet.') };
+  if (!cases.length) return { id: 'coverage', ran: false, says: t('No exchange in this project is long enough to test with yet.') };
   const found = [];
   for (const one of cases) {
     const held = await recall(one.asked, one.sequence);
@@ -146,7 +146,7 @@ export async function coverageExam({ events = [], recall, embed = null, sample =
     id: 'coverage', ran: true, at: new Date().toISOString(), n: cases.length, hits, rate, method, bar: used,
     mean: mean(scores), control: controlled ? mean(controls) : null,
     passed: rate >= PASS.coverage,
-    says: t('{hits} of {n} questions this room actually asked had their answer already in the archive, matched by {method}{control}.', { hits, n: cases.length, method: t(method), control: controlled ? t(' and against a control') : '' }),
+    says: t('{hits} of {n} questions this project actually asked had their answer already in the archive, matched by {method}{control}.', { hits, n: cases.length, method: t(method), control: controlled ? t(' and against a control') : '' }),
   };
 }
 
@@ -186,10 +186,10 @@ export function consistencyExam({ findings = [], notes = [], now = Date.now() }
 // `ask(question)` is the local model with this archive behind it. `embed` is needed: comparing
 // two answers by their words rewards copying the question back.
 export async function matchExam({ events = [], ask, embed, sample = MATCH_SAMPLE, bar = MATCH_BAR, onProgress = () => {}, stop = () => false } = {}) {
-  if (!embed) return { id: 'match', ran: false, says: t('This test needs embeddings: two answers cannot be compared by their words alone.') };
+  if (!embed) return { id: 'match', ran: false, says: t('This test needs smart search: two answers cannot be compared by their words alone.') };
   const all = exchanges(events).filter((one) => !one.local);   // a model is not tested against itself
   const cases = spread(all, sample);
-  if (!cases.length) return { id: 'match', ran: false, says: t('No question in this room was answered by an agent other than the local one yet.') };
+  if (!cases.length) return { id: 'match', ran: false, says: t('No question in this project was answered by an agent other than the local one yet.') };
   const mine = [];
   for (const [index, one] of cases.entries()) {
     if (stop()) return { id: 'match', ran: false, stopped: true, says: t('Stopped after {index} of {n}.', { index, n: cases.length }) };
@@ -222,7 +222,7 @@ export async function matchExam({ events = [], ask, embed, sample = MATCH_SAMPLE
 export function saysFor(result) {
   if (!result?.ran) return result?.says ?? null;
   if (result.id === 'coverage' && Number.isFinite(result.hits)) {
-    return t('{hits} of {n} questions this room actually asked had their answer already in the archive, matched by {method}{control}.',
+    return t('{hits} of {n} questions this project actually asked had their answer already in the archive, matched by {method}{control}.',
       { hits: result.hits, n: result.n, method: t(result.method ?? 'words'), control: result.control === null || result.control === undefined ? '' : t(' and against a control') });
   }
   if (result.id === 'match' && Number.isFinite(result.matched)) {

package/src/extensions.mjs

@@ -6,8 +6,6 @@ export { gitToplevel, findOnPath } from './modules/helpers.mjs';
 // server and the tests have always used, now answered by one file per module.
 export const EXTENSIONS = MODULES;
 export const AHP = moduleById('ahp');
-export const IMAGE_STUDIO = moduleById('image-studio');
-export const GIT_PULSE = moduleById('git-pulse');
 export const ASH = moduleById('ash');
 export const RIPLEY = moduleById('ripley');
 export const OLLAMA = moduleById('ollama');

package/src/file-ops.mjs

@@ -12,6 +12,8 @@
 //     the same word the room asks for before CONTROL. Typing a folder name is a small ceremony;
 //     restoring a deleted key is not.
 
+import { realpath } from 'node:fs/promises';
+import { basename, dirname, join, relative, resolve, sep } from 'node:path';
 import { isForbidden } from './checkpoint.mjs';
 
 export const OPERATIONS = ['copy', 'move', 'new-file', 'new-folder'];
@@ -24,6 +26,27 @@ export function needsDesignation(...paths) {
   return paths.filter(Boolean).some((path) => isForbidden(String(path).replace(/^\.\//, '')));
 }
 
+// The same question asked of where a path really lands (2026-10-12, SRV-05 of the 1.0 review): the
+// words of a path are not its place. `src/../.git/config`, a link named `gitlink` that points at
+// `.git`, `.ENV` on a disk that does not tell it from `.env` — each wrote into a guarded zone with
+// no designation, because only the words were read. A path that does not exist yet is judged by
+// the real folder it would land in.
+export async function needsDesignationOnDisk(root, ...paths) {
+  if (needsDesignation(...paths)) return true;
+  const top = await realpath(root).catch(() => null);
+  if (!top) return false;
+  for (const path of paths.filter(Boolean)) {
+    const wanted = resolve(top, String(path).replace(/^\/+/, ''));
+    const real = await realpath(wanted).catch(async () => {
+      const folder = await realpath(dirname(wanted)).catch(() => null);
+      return folder ? join(folder, basename(wanted)) : null;
+    });
+    if (!real || (real !== top && !real.startsWith(top + sep))) continue;   // outside: the fence refuses it elsewhere
+    if (needsDesignation(relative(top, real).split(sep).join('/'))) return true;
+  }
+  return false;
+}
+
 // What the room will say about an operation before doing any of it.
 export function planFileOp({ operation, from, to }) {
   if (!OPERATIONS.includes(operation)) return { ok: false, error: `Unknown operation: ${operation}.` };
@@ -43,3 +66,22 @@ export function planFileOp({ operation, from, to }) {
   if (operation === 'move' && `${target}/`.startsWith(`${source}/`)) return { ok: false, error: 'A folder cannot be moved inside itself.' };
   return { ok: true, operation, from: source, to: target, guarded: needsDesignation(source, target) };
 }
+
+// The name a copy takes when its own is already used where it lands: "notes copy.md", then
+// "notes copy 2.md", the way a file manager does it. `word` is the room's word for "copy", and
+// `taken` answers whether a project-relative path already holds something.
+export async function freeCopyPath(to, { taken, word = 'copy', limit = 100 }) {
+  const path = String(to ?? '').replace(/^\/+/, '').trim();
+  if (!path || !(await taken(path))) return path;
+  const slash = path.lastIndexOf('/');
+  const dir = slash >= 0 ? path.slice(0, slash + 1) : '';
+  const name = path.slice(slash + 1);
+  // A leading dot is part of the name (".env"), not an extension.
+  const dot = name.lastIndexOf('.');
+  const [stem, ext] = dot > 0 ? [name.slice(0, dot), name.slice(dot)] : [name, ''];
+  for (let n = 1; n <= limit; n += 1) {
+    const candidate = `${dir}${stem} ${word}${n > 1 ? ` ${n}` : ''}${ext}`;
+    if (!(await taken(candidate))) return candidate;
+  }
+  return path;
+}

package/src/files.mjs

@@ -1,6 +1,6 @@
 import { createHash, randomUUID } from 'node:crypto';
 import { mkdir, readdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
-import { extname, join, resolve, sep } from 'node:path';
+import { dirname, extname, join, resolve, sep } from 'node:path';
 
 // Read-only file access for the room UI, and human attachments.
 //
@@ -21,6 +21,12 @@ const TYPES = {
   '.mp4': 'video/mp4', '.webm': 'video/webm', '.mp3': 'audio/mpeg', '.wav': 'audio/wav',
 };
 export const contentTypeFor = (name) => TYPES[extname(name).toLowerCase()] ?? 'application/octet-stream';
+// What a file IS, for the record of what a turn made. contentTypeFor is how MADRE SERVES a project
+// file, and a page is served as text on purpose: an HTML file from the project served as HTML would
+// run its scripts in the room's own origin (RIPLEY renders it, sealed, elsewhere). The record said
+// tests.html was text/plain, which is the serving rule leaking into the description.
+const DECLARED = { '.html': 'text/html', '.htm': 'text/html', '.xml': 'application/xml', '.ts': 'text/typescript', '.tsx': 'text/typescript', '.jsx': 'text/javascript' };
+export const declaredTypeFor = (name) => DECLARED[extname(name).toLowerCase()] ?? contentTypeFor(name);
 export const isImage = (name) => contentTypeFor(name).startsWith('image/');
 export const isText = (name) => /^(text\/|application\/(json|x-ndjson))/.test(contentTypeFor(name));
 
@@ -152,7 +158,51 @@ export async function readServable(root, relative, { maxBytes = MAX_FILE_BYTES }
   if (!info || !info.isFile()) return { status: 404, error: 'Not found.' };
   if (info.size > maxBytes) return { status: 413, error: `File is larger than ${Math.round(maxBytes / 1024 / 1024)} MB.` };
   const body = await readFile(path);
-  return { status: 200, body, contentType: contentTypeFor(path), size: info.size, path };
+  return { status: 200, body, contentType: contentTypeFor(path), size: info.size, path, mtimeMs: info.mtimeMs };
+}
+
+// Writing a file the human edited in the room. Only a file that already exists inside the root,
+// only text, and only if it is still the file the editor opened: a file that changed on disk since
+// (an agent wrote it, the human saved it elsewhere) is not overwritten without being asked.
+export const MAX_EDIT_BYTES = 2 * 1024 * 1024;
+export async function writeEdited(root, relative, content, { mtimeMs = null, force = false } = {}) {
+  if (typeof content !== 'string') return { status: 400, error: 'Nothing to save.' };
+  if (Buffer.byteLength(content, 'utf8') > MAX_EDIT_BYTES) return { status: 413, error: `The editor saves files up to ${MAX_EDIT_BYTES / 1024 / 1024} MB.` };
+  const path = await resolveInside(root, relative);
+  if (!path) return { status: 404, error: 'Not found.' };
+  const info = await stat(path).catch(() => null);
+  if (!info || !info.isFile()) return { status: 404, error: 'Not found.' };
+  if (!force && Number.isFinite(Number(mtimeMs)) && Math.abs(info.mtimeMs - Number(mtimeMs)) > 1) {
+    return { status: 409, error: 'The file changed on disk since it was opened.', changed: true, mtimeMs: info.mtimeMs };
+  }
+  await writeFile(path, content, 'utf8');
+  const after = await stat(path);
+  return { status: 200, path, bytes: after.size, mtimeMs: after.mtimeMs };
+}
+
+// A new, empty file in the project, from the Finder. Only a plain relative path: no `..`, no
+// drive (`C:`) or UNC prefix, no folder of git's. Its nearest existing folder must already be
+// inside the project (the same fence as every read and write); missing folders under it are
+// made. An existing file is never touched.
+export async function createEmptyFile(root, relative) {
+  const clean = String(relative ?? '').trim().replace(/^[/\\]+/, '');
+  const parts = clean.split(/[/\\]+/).filter(Boolean);
+  if (!clean || clean.includes('\0') || /[/\\]$/.test(clean) || /^[a-zA-Z]:/.test(clean) || parts.some((part) => part === '..' || part === '.') || parts[0] === '.git') {
+    return { status: 400, error: 'Not a file name this project can create.' };
+  }
+  const canonicalRoot = await realpath(root).catch(() => null);
+  if (!canonicalRoot) return { status: 404, error: 'Not found.' };
+  const target = resolve(canonicalRoot, ...parts);
+  if (!target.startsWith(canonicalRoot + sep)) return { status: 400, error: 'Not a file name this project can create.' };
+  // The nearest folder that exists is checked against the fence, symlinks resolved.
+  let folder = dirname(target);
+  while (folder !== canonicalRoot && !(await stat(folder).catch(() => null))) folder = dirname(folder);
+  const inside = await resolveInside(canonicalRoot, folder === canonicalRoot ? '.' : folder.slice(canonicalRoot.length + 1));
+  if (!inside) return { status: 400, error: 'Not a file name this project can create.' };
+  await mkdir(dirname(target), { recursive: true });
+  try { await writeFile(target, '', { encoding: 'utf8', flag: 'wx' }); }
+  catch (error) { return error.code === 'EEXIST' ? { status: 409, error: 'A file with that name is already there.' } : { status: 400, error: error.message }; }
+  return { status: 201, path: parts.join('/') };
 }
 
 const safeName = (name) => String(name ?? 'file').replace(/[^\w.\- ]+/g, '_').replace(/^\.+/, '').slice(0, 120) || 'file';