@volter/supercode-teams 0.3.151 → 0.3.152

package/mail/import-maildir.mjs

@@ -0,0 +1,350 @@
+// The only reader of the retired layout. It runs before the daemon exposes mail
+// operations, imports idempotently, and records completion only after every input
+// was visited. Failed imports can resume; ordinary reads never consult this tree.
+import { createReadStream } from 'node:fs';
+import { lstat, readFile, readdir } from 'node:fs/promises';
+import { createInterface } from 'node:readline';
+import { basename, join, relative } from 'node:path';
+import { createHash, randomBytes } from 'node:crypto';
+
+const alive = (pid) => {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return false;
+  try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; }
+};
+const headerOf = ({ body, ...header }) => header;
+const validId = (id) => typeof id === 'string' && /^[A-Za-z0-9_-]+$/.test(id);
+const record = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
+const isAddress = (value) => typeof value === 'string' && /^sc:[^:\s]+:[^:\s]+:\S+$/.test(value);
+const validRecordName = (value) => typeof value === 'string' && /^[A-Za-z0-9._-]+$/.test(value);
+
+export async function importMaildir(database, root, { log = () => {}, legacyRoot = root } = {}) {
+  const began = performance.now();
+  const work = { source_reads: 0, source_bytes: 0, committed_batches: 0, max_queued_operations: 0, elapsed_ms: 0 };
+  const complete = database.metadata('maildir_import');
+  if (complete?.complete === true) return complete;
+  let roots;
+  try { roots = await readdir(root, { withFileTypes: true }); }
+  catch (error) {
+    if (error.code !== 'ENOENT') throw error;
+    work.elapsed_ms = Math.round(performance.now() - began);
+    const result = { complete: true, source: root, at_ms: Date.now(), messages: 0, records: 0, skipped: 0, work };
+    database.setMetadata('maildir_import', result);
+    return result;
+  }
+  const result = { complete: false, source: root, at_ms: Date.now(), messages: 0, records: 0, skipped: 0, work };
+  let pending = [];
+  let flushing = false;
+  const flush = () => {
+    if (!pending.length) return;
+    const writes = pending;
+    pending = [];
+    flushing = true;
+    try {
+      database.transaction(() => { for (const write of writes) write(); });
+      work.committed_batches++;
+      work.max_queued_operations = Math.max(work.max_queued_operations, writes.length);
+    }
+    finally { flushing = false; }
+  };
+  // Fingerprints, diagnostics and their imported rows share bounded commits.
+  // A diagnostic discovered while applying a batch belongs to that transaction.
+  const queue = (write) => {
+    if (flushing) { write(); return; }
+    pending.push(write);
+    if (pending.length >= 256) flush();
+  };
+  const errors = database.statement('INSERT INTO mail_import_errors(source, mailbox, detail, critical) VALUES (?, ?, ?, ?) ON CONFLICT(source) DO UPDATE SET mailbox = excluded.mailbox, detail = excluded.detail, critical = excluded.critical');
+  const failed = (path, error, mailbox = null) => {
+    const source = relative(root, path);
+    const parts = source.split(/[\\/]/);
+    const critical = ['agents', 'threads', 'native-questions', 'native-launches', 'relays', 'machines', 'sent'].includes(parts[0])
+      || ['claims', 'subscriptions', 'wake'].includes(parts[1])
+      || parts[0] === 'names.json' || parts[0] === 'relay-endpoint.json'
+      || (parts[0] === 'terminal-delivery' && parts.at(-1) === 'queue.json');
+    queue(() => errors.run(source, mailbox, String(error.message ?? error), critical ? 1 : 0));
+    result.skipped++;
+    log(`mail import: retained unreadable source ${source}: ${error.message ?? error}`);
+  };
+  const sources = database.statement('INSERT INTO mail_import_sources(source, sha256) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET sha256 = excluded.sha256');
+  const readSource = async (path, encoding = 'utf8') => {
+    if (!(await lstat(path)).isFile()) throw new Error('record is not a regular file');
+    const bytes = await readFile(path);
+    work.source_reads++; work.source_bytes += bytes.length;
+    const source = relative(root, path), sha256 = createHash('sha256').update(bytes).digest('hex');
+    queue(() => sources.run(source, sha256));
+    return encoding ? new TextDecoder('utf-8', { fatal: true }).decode(bytes) : bytes;
+  };
+  const json = async (path, mailbox = null, valid = record) => {
+    try {
+      const value = JSON.parse(await readSource(path));
+      if (!valid(value)) throw new Error('record has an unsupported JSON shape');
+      queue(() => database.statement('DELETE FROM mail_import_errors WHERE source = ?').run(relative(root, path)));
+      return value;
+    }
+    catch (error) { failed(path, error, mailbox); return null; }
+  };
+  const entries = async (dir) => {
+    try {
+      if (!(await lstat(dir)).isDirectory()) throw new Error(`mail import refuses a linked or non-directory record group: ${dir}`);
+      return await readdir(dir, { withFileTypes: true });
+    }
+    catch (error) { if (error.code === 'ENOENT') return []; throw error; }
+  };
+  const recordDirectory = async (dir, collection, { nested = false, text = false } = {}) => {
+    for (const entry of await entries(dir)) {
+      const path = join(dir, entry.name);
+      if (entry.isDirectory() && nested) {
+        for (const child of await entries(path)) {
+          if (!child.isFile()) { failed(join(path, child.name), 'not a regular record'); continue; }
+          const value = await json(join(path, child.name));
+          if (value !== null) queue(() => { database.putRecord(collection, `${entry.name}/${child.name}`, value, { create: true }); result.records++; });
+        }
+        continue;
+      }
+      if (!entry.isFile()) { if (!entry.isDirectory()) failed(path, 'not a regular record'); continue; }
+      if (/\.tmp(?:\.|$)|\.lock$/.test(entry.name)) continue;
+      let value;
+      if (text) {
+        try {
+          value = (await readSource(path)).trim();
+          if (!value || !Number.isSafeInteger(Number(value)) || Number(value) < 0) throw new Error('agent cursor is not a nonnegative integer timestamp');
+        }
+        catch (error) { failed(path, error); continue; }
+      } else value = await json(path, null, collection === 'native_questions' && entry.name.endsWith('.delivered')
+        ? Number.isSafeInteger : record);
+      if (value !== null) queue(() => { database.putRecord(collection, entry.name, value, { create: true }); result.records++; });
+    }
+  };
+
+  // Pass 1 imports envelopes before their claims, subscriptions and deliveries,
+  // so the database's foreign keys enforce the same message ownership throughout.
+  const boxes = [];
+  for (const entry of roots) {
+    if (!entry.isDirectory()) continue;
+    const dir = join(root, entry.name);
+    let address;
+    try { address = (await readSource(join(dir, 'address'))).trim(); }
+    catch (error) { if (error.code !== 'ENOENT') failed(join(dir, 'address'), error); continue; }
+    if (!/^sc:[^:\s]+:[^:\s]+:\S+$/.test(address)) { failed(join(dir, 'address'), 'invalid mailbox address'); continue; }
+    queue(() => database.statement('INSERT INTO mailboxes(address, folder) VALUES (?, ?) ON CONFLICT(address) DO NOTHING').run(address, entry.name));
+    boxes.push({ address, dir });
+    for (const part of ['new', 'claimed', 'cur']) {
+      for (const file of await entries(join(dir, part))) {
+        if (!file.isFile() || !file.name.endsWith('.json')) continue;
+        const path = join(dir, part, file.name);
+        const envelope = await json(path, address);
+        if (!envelope) continue;
+        if (!validId(envelope.id) || !Number.isSafeInteger(envelope.created_at_ms) || typeof envelope.body !== 'string') {
+          failed(path, 'invalid envelope identity, creation time or body', address); continue;
+        }
+        const pid = part === 'claimed' ? Number(file.name.split('.')[0]) : null;
+        const state = part === 'claimed' && !alive(pid) ? 'new' : part;
+        const token = state === 'claimed' ? randomBytes(16).toString('hex') : null;
+        const line = Array.from(envelope.body.split('\n', 1)[0]).slice(0, 400).join('');
+        queue(() => {
+          const existing = database.statement('SELECT header, body FROM messages WHERE mailbox = ? AND id = ?').get(address, envelope.id);
+          if (existing) {
+            // Retain an inconsistent duplicate for repair; never overwrite a
+            // previously imported original or choose from its newer mtime.
+            if (existing.header !== JSON.stringify(headerOf(envelope)) || existing.body !== envelope.body) failed(path, 'conflicting duplicate message id', address);
+            return;
+          }
+          database.statement(`INSERT INTO messages(mailbox, id, thread, sender, kind, subject, created_at_ms, state, header, body, first_line, read_at_ms, claim_pid, claim_token)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+          .run(address, envelope.id, envelope.thread ?? envelope.id, envelope.from ?? '', envelope.kind ?? 'peer', envelope.subject ?? null,
+            envelope.created_at_ms, state, JSON.stringify(headerOf(envelope)), envelope.body, line, state === 'cur' ? envelope.created_at_ms : null,
+            state === 'claimed' ? pid : null, token);
+          result.messages++;
+        });
+      }
+    }
+  }
+  flush();
+
+  for (const { address, dir } of boxes) {
+    const present = (id) => Boolean(database.statement('SELECT 1 FROM messages WHERE mailbox = ? AND id = ?').get(address, id));
+    for (const entry of await entries(join(dir, 'claims'))) {
+      if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+      const path = join(dir, 'claims', entry.name), id = basename(entry.name, '.json');
+      const claim = await json(path, address);
+      if (claim && !present(id)) { failed(path, 'claim has no imported message', address); continue; }
+      if (claim) queue(() => database.statement('INSERT INTO claims(mailbox, message, record) VALUES (?, ?, ?) ON CONFLICT(mailbox, message) DO NOTHING').run(address, id, JSON.stringify(claim)));
+    }
+    for (const entry of await entries(join(dir, 'subscriptions'))) {
+      if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+      const path = join(dir, 'subscriptions', entry.name), subscription = await json(path, address, (value) => record(value)
+        && validId(value.message_id) && isAddress(value.subscriber)
+        && (value.created_at_ms == null || Number.isSafeInteger(value.created_at_ms)));
+      if (!subscription?.message_id || !present(subscription.message_id)) { if (subscription) failed(path, 'subscription has no imported message', address); continue; }
+      queue(() => database.statement('INSERT INTO subscriptions(mailbox, message, record) VALUES (?, ?, ?) ON CONFLICT(mailbox, message) DO NOTHING').run(address, subscription.message_id, JSON.stringify(subscription)));
+    }
+    for (const entry of await entries(join(dir, 'wake'))) {
+      if (!entry.isFile()) continue;
+      const path = join(dir, 'wake', entry.name);
+      if (!present(entry.name)) { failed(path, 'wake has no imported message', address); continue; }
+      let state;
+      try {
+        const text = await readSource(path);
+        state = text.trim() ? JSON.parse(text) : {};
+      } catch (error) { failed(path, error, address); continue; }
+      if (!record(state) || !Number.isSafeInteger(state.next_at_ms ?? 0) || !Number.isSafeInteger(state.attempts ?? 0)) {
+        failed(path, 'invalid wake deadline or attempts', address); continue;
+      }
+      queue(() => database.statement('INSERT INTO deliveries(mailbox, message, due_at_ms, attempts, record) VALUES (?, ?, ?, ?, ?) ON CONFLICT(mailbox, message) DO NOTHING')
+        .run(address, entry.name, state.next_at_ms ?? 0, state.attempts ?? 0, JSON.stringify(state)));
+    }
+    for (const entry of await entries(join(dir, 'expired'))) {
+      if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+      const expiry = await json(join(dir, 'expired', entry.name), address, (value) => record(value) && validId(value.id));
+      if (expiry?.id) queue(() => { database.putRecord('expired', `${address}/${expiry.id}`, expiry, { create: true }); result.records++; });
+    }
+  }
+
+  const agents = join(root, 'agents');
+  for (const entry of await entries(agents)) {
+    if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+    const agent = await json(join(agents, entry.name), null, (value) => record(value)
+      && validRecordName(value.name) && value.name === basename(entry.name, '.json') && isAddress(value.main_session)
+      && (value.folder == null || typeof value.folder === 'string')
+      && (value.harness == null || typeof value.harness === 'string')
+      && (value.idle_minutes == null || Number.isFinite(value.idle_minutes)));
+    if (agent) queue(() => { database.putRecord('agents', basename(entry.name, '.json'), agent, { create: true }); result.records++; });
+  }
+  for (const name of ['typed', 'resumed']) await recordDirectory(join(agents, name), `agent_${name}`, { text: true });
+  try {
+    const name = (await readSource(join(agents, 'owners-account-manager'))).trim();
+    if (!validRecordName(name)) throw new Error('owner account-manager selector is not an agent name');
+    queue(() => database.putRecord('agent_settings', 'owners-account-manager', name, { create: true }));
+  }
+  catch (error) { if (error.code !== 'ENOENT') failed(join(agents, 'owners-account-manager'), error); }
+  for (const entry of await entries(join(root, 'threads'))) {
+    if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+    const thread = await json(join(root, 'threads', entry.name), null, (value) => record(value)
+      && validRecordName(value.id) && value.id === basename(entry.name, '.json') && validRecordName(value.agent) && isAddress(value.holder)
+      && Number.isSafeInteger(value.created_at_ms) && Array.isArray(value.participants)
+      && value.participants.every((participant) => record(participant) && isAddress(participant.address) && ['to', 'cc'].includes(participant.role))
+      && (value.markers == null || record(value.markers)));
+    if (thread) queue(() => { database.putRecord('threads', basename(entry.name, '.json'), thread, { create: true }); result.records++; });
+  }
+  await recordDirectory(join(root, 'native-questions'), 'native_questions');
+  await recordDirectory(join(root, 'native-launches'), 'native_launches');
+  await recordDirectory(join(root, 'machines'), 'machine_waits');
+  const endpointFile = join(root, 'relay-endpoint.json');
+  try { await lstat(endpointFile); const endpoint = await json(endpointFile); if (endpoint) queue(() => database.putRecord('relay_endpoint', 'port', endpoint, { create: true })); }
+  catch (error) { if (error.code !== 'ENOENT') throw error; }
+  // Relay state is imported as rows. The runtime's settings are application
+  // configuration, copied to its new runtime home when that runtime is adopted.
+  for (const relay of await entries(join(root, 'relays'))) {
+    if (!relay.isDirectory()) continue;
+    for (const entry of await entries(join(root, 'relays', relay.name))) {
+      if (!entry.isFile() || !/^(?:relay|queue|receipt|sent|settings)\.json$|^turn\.txt$/.test(entry.name)) continue;
+      const path = join(root, 'relays', relay.name, entry.name);
+      let value;
+      if (entry.name === 'turn.txt') {
+        try { value = await readSource(path); } catch (error) { failed(path, error); continue; }
+      } else value = await json(path);
+      const at = (await lstat(path)).mtimeMs;
+      if (entry.name === 'relay.json' && value && typeof value === 'object') value.runtime_directory ??= join(legacyRoot, 'relays', relay.name);
+      if (value !== null) queue(() => { database.putRecord('relays', `${relay.name}/${entry.name}`, value, { create: true, at: Math.trunc(at) }); result.records++; });
+    }
+  }
+  const namesFile = join(root, 'names.json');
+  try {
+    const names = JSON.parse(await readSource(namesFile));
+    if (!record(names)) throw new Error('names ledger is not a JSON object');
+    for (const [name, value] of Object.entries(names)) queue(() => database.putRecord('names', name, value, { create: true }));
+  } catch (error) { if (error.code !== 'ENOENT') failed(namesFile, error); }
+
+  const lines = async (path, accept) => {
+    try {
+      const stat = await lstat(path);
+      if (!stat.isFile()) throw new Error(`mail import requires a regular record: ${path}`);
+      const key = `maildir_lines:${relative(root, path)}`;
+      const progress = database.metadata(key);
+      const stamp = { size: stat.size, mtime: stat.mtimeMs, inode: stat.ino };
+      if (progress && Object.keys(stamp).some((field) => progress[field] !== stamp[field])) {
+        throw new Error(`mail import source changed during an interrupted import: ${path}`);
+      }
+      let number = 0;
+      const hash = createHash('sha256');
+      const utf8 = new TextDecoder('utf-8', { fatal: true });
+      work.source_reads++;
+      const stream = createReadStream(path);
+      stream.on('data', (bytes) => {
+        hash.update(bytes);
+        work.source_bytes += bytes.length;
+        try { utf8.decode(bytes, { stream: true }); }
+        catch (error) { stream.destroy(error); }
+      });
+      const reader = createInterface({ input: stream, crlfDelay: Infinity });
+      stream.on('error', (error) => reader.close());
+      try {
+        for await (const line of reader) {
+          number++;
+          if (number <= (progress?.line ?? 0)) continue;
+          const position = number;
+          let entry = null;
+          if (line.trim()) {
+            try {
+              entry = JSON.parse(line);
+              if (!record(entry)) throw new Error('log entry is not a JSON object');
+            } catch (error) { entry = null; failed(path, error); }
+          }
+          queue(() => {
+            if (entry !== null) { try { accept(entry); } catch (error) { failed(path, error); } }
+            database.setMetadata(key, { ...stamp, line: position });
+          });
+        }
+        if (stream.errored) throw stream.errored;
+        utf8.decode();
+        const source = relative(root, path), sha256 = hash.digest('hex');
+        queue(() => sources.run(source, sha256));
+        flush();
+      } finally { reader.close(); stream.destroy(); }
+    } catch (error) { if (error.code !== 'ENOENT') throw error; }
+  };
+  for (const folder of await entries(join(root, 'terminal-delivery'))) {
+    if (!folder.isDirectory()) continue;
+    const dir = join(root, 'terminal-delivery', folder.name);
+    for (const entry of await entries(dir)) {
+      const path = join(dir, entry.name);
+      if (entry.name === 'queue.json') {
+        const state = await json(path);
+        if (state) queue(() => { database.putRecord('terminal_delivery', `${folder.name}/queue.json`, state, { create: true }); result.records++; });
+      } else if (entry.name === 'receipts.jsonl' || entry.name === 'attempts.jsonl') {
+        await lines(path, (record) => {
+          database.statement('INSERT INTO terminal_log(recipient, kind, record) VALUES (?, ?, ?)').run(folder.name, entry.name === 'receipts.jsonl' ? 'receipt' : 'attempt', JSON.stringify(record));
+          database.statement('DELETE FROM terminal_log WHERE sequence <= (SELECT sequence FROM terminal_log ORDER BY sequence DESC LIMIT 1 OFFSET ?)').run(database.retention.maxTerminalLogRows);
+        });
+      }
+    }
+  }
+  for (const sender of await entries(join(root, 'sent'))) {
+    if (!sender.isDirectory()) continue;
+    const dir = join(root, 'sent', sender.name);
+    await lines(join(dir, 'log.jsonl'), (entry) => {
+      if (!validId(entry.id) || typeof entry.to !== 'string' || !Number.isSafeInteger(entry.at_ms)) throw new Error('invalid sent message record');
+      database.statement('INSERT INTO sent_messages(sender_key, id, recipient, thread, at_ms, record) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(sender_key, id, recipient) DO NOTHING')
+        .run(sender.name, entry.id, entry.to, entry.thread ?? null, entry.at_ms, JSON.stringify(entry));
+    });
+    for (const marker of await entries(dir)) {
+      if (!marker.isFile() || marker.name === 'log.jsonl' || !validId(marker.name)) continue;
+      const value = await readSource(join(dir, marker.name));
+      queue(() => database.putRecord('sent_receipts', `${sender.name}/${marker.name}`, value, { create: true }));
+    }
+  }
+  await lines(join(root, 'carrier.jsonl'), (entry) => database.appendCarrier(entry));
+  flush();
+  result.messages = database.statement('SELECT count(*) AS count FROM messages').get().count;
+  result.records = database.statement('SELECT count(*) AS count FROM mail_records').get().count;
+  result.skipped = database.statement('SELECT count(*) AS count FROM mail_import_errors').get().count;
+  work.elapsed_ms = Math.round(performance.now() - began);
+  const blocked = database.statement('SELECT source FROM mail_import_errors WHERE critical = 1 ORDER BY source').all();
+  if (blocked.length) {
+    throw Object.assign(new Error(`mail migration cannot activate with ${blocked.length} unreadable operational records (${blocked.slice(0, 3).map(({ source }) => source).join(', ')}); original files are retained for repair`), { code: 'mail_import_incomplete' });
+  }
+  result.complete = true;
+  database.setMetadata('maildir_import', result);
+  log(`mail import complete: ${result.messages} envelopes, ${result.records} operational records, ${result.skipped} retained unreadable sources; ${work.source_reads} source reads, ${work.source_bytes} bytes, ${work.committed_batches} batches in ${work.elapsed_ms} ms`);
+  return result;
+}

package/mail/questions.mjs

@@ -1,25 +1,23 @@
 // Native Codex questions as mail (step 4c; mail_question.rs `poll`, deleted): every POLL_MS the daemon asks each Codex
 // pane it launched (its native-launches seed: a socket of its own, its owner alive) for the threads it runs and the
 // questions waiting on them, records each thread's creator (`native-questions/binding-<blake3(address)>.json`) and files
-// each question once (`<q-id>.json`) in the session's mailbox, read, and in its creator's, with a wake, so the carrier
-// hands it over (or carries it to the creator's machine). A reply is answered by mail/send.mjs; the files are the native
-// hook's (`message hook`) as well, so they are kept byte for byte as mail_question.rs writes them. Native request
+// each question once in the session's mailbox, read, and in its creator's, with a wake, so the carrier
+// hands it over (or carries it to the creator's machine). A reply is answered by mail/send.mjs; question and binding
+// records are kept by the same database as the mail. Native request
 // identities decide which question a reply answers, never screen text or terminal keys.
-import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, writeSync } from 'node:fs';
-import { join } from 'node:path';
+import { existsSync } from 'node:fs';
 import { blake3Hex } from './envelope.mjs';
 import { envelope as makeEnvelope, makeAddress } from './compose.mjs';
 
 const POLL_MS = 10_000;
 
-const readJson = (path) => { try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; } };
 const alive = (pid) => { if (!Number.isInteger(pid) || pid <= 0) return false; try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
 
 export class NativeQuestions {
   /** `codexQuestions(request)` the in-process native question client (native-questions.mjs). */
   constructor({ machine, mailRoot, store, codexQuestions, log = () => {} }) {
     Object.assign(this, { machine, mailRoot, store, codexQuestions, log });
-    this.dir = join(mailRoot, 'native-questions');
+    this.stopped = false;
     this.running = false;
   }
 
@@ -30,26 +28,23 @@ export class NativeQuestions {
     return this;
   }
 
-  stop() { clearInterval(this.timer); }
+  stop() { this.stopped = true; clearInterval(this.timer); }
 
   /** One pass over the launched Codex panes; a pass still running is not started twice. Answers how many panes were
    *  asked and how many questions they hold, or null when a pass was already running. */
   async poll() {
-    if (this.running) return null;
+    if (this.stopped || this.running) return null;
     this.running = true;
     const counted = { panes: 0, questions: 0 };
     try {
-      let names = [];
-      try { names = readdirSync(join(this.mailRoot, 'native-launches')); } catch { return counted; }
-      for (const name of names) {
+      for (const { key: name, value: seed } of this.store.records('native_launches')) {
         if (!name.endsWith('.json') || name.endsWith('.session.json')) continue;
-        const seedPath = join(this.mailRoot, 'native-launches', name);
-        const seed = readJson(seedPath);
         // only a socket belonging to a launched pane whose owner lives: a retained socket file is not a live endpoint,
         // and a global shared app-server or a cwd is never guessed at
         if (seed?.kind !== 'codex' || typeof seed.socketPath !== 'string' || !alive(seed.ownerPid) || !existsSync(seed.socketPath)) continue;
         let reply;
         try { reply = await this.codexQuestions({ socketPath: seed.socketPath, discover: true }); } catch { continue; }
+        if (this.stopped) return counted;
         counted.panes++; counted.questions += reply?.questions?.length ?? 0;
         for (const thread of reply?.threads ?? []) {
           try { this.bind(seed, { session_id: thread.id, transcript_path: thread.path ?? null }); }
@@ -64,16 +59,9 @@ export class NativeQuestions {
     } finally { this.running = false; }
   }
 
-  // ---- the files, as mail_question.rs writes them ------------------------------------------------------------------
+  // ---- native question records -----------------------------------------------------------------------------------
 
-  #bindingPath(address) { return join(this.dir, `binding-${blake3Hex(address)}.json`); }
-
-  /** Written whole and synced, never over an existing file (mail_question.rs `create`). */
-  create(path, value) {
-    mkdirSync(this.dir, { recursive: true, mode: 0o700 });
-    const fd = openSync(path, 'wx', 0o600);
-    try { writeSync(fd, JSON.stringify(value)); fsyncSync(fd); } finally { closeSync(fd); }
-  }
+  #bindingKey(address) { return `binding-${blake3Hex(address)}.json`; }
 
   /** A session's binding: its creator recorded at its first launch, its pane, socket and transcript kept current. */
   bind(seed, input, harness = 'codex') {
@@ -81,19 +69,18 @@ export class NativeQuestions {
     if (!session) throw new Error('the thread has no session id');
     const creator = typeof seed.by === 'string' ? seed.by : makeAddress(this.machine, 'operator', 'user');
     const binding = { session, creator, pane: seed.pane ?? '', socket_path: seed.socketPath ?? null, transcript_path: input.transcript_path ?? null };
-    const path = this.#bindingPath(session);
-    try { this.create(path, binding); return binding; }
-    catch (error) { if (error.code !== 'EEXIST') throw error; }
-    const existing = readJson(path);
-    if (!existing) throw new Error(`its binding ${path} could not be read`);
-    if (existing.pane !== binding.pane || existing.socket_path !== binding.socket_path || (binding.transcript_path != null && existing.transcript_path !== binding.transcript_path)) {
-      existing.pane = binding.pane; existing.socket_path = binding.socket_path;
-      if (binding.transcript_path != null) existing.transcript_path = binding.transcript_path;
-      const staged = `${path.slice(0, -'.json'.length)}.${process.pid}.tmp`;
-      this.create(staged, existing);
-      renameSync(staged, path);
-    }
-    return existing;
+    const key = this.#bindingKey(session);
+    return this.store.transaction(() => {
+      if (this.store.putRecord('native_questions', key, binding, { create: true })) return binding;
+      const existing = this.store.record('native_questions', key);
+      if (!existing?.session || typeof existing.creator !== 'string') throw new Error(`its binding ${key} could not be read`);
+      if (existing.pane !== binding.pane || existing.socket_path !== binding.socket_path || (binding.transcript_path != null && existing.transcript_path !== binding.transcript_path)) {
+        existing.pane = binding.pane; existing.socket_path = binding.socket_path;
+        if (binding.transcript_path != null) existing.transcript_path = binding.transcript_path;
+        this.store.putRecord('native_questions', key, existing);
+      }
+      return existing;
+    });
   }
 
   /** A question's id, from its native identity (mail_question.rs `question_id`): each poll and hook finds the same one. */
@@ -111,21 +98,26 @@ export class NativeQuestions {
     }
     body += '\nReply with the answer text for one question, or a JSON object mapping each question id to its answer for several questions.';
     let question = { binding, native, envelope: makeEnvelope({ id, from: binding.session, from_name: binding.session, kind: 'question', reply_via: { mode: 'command' }, body }) };
-    try { this.create(join(this.dir, `${id}.json`), question); }
-    catch (error) { if (error.code !== 'EEXIST') throw error; question = readJson(join(this.dir, `${id}.json`)) ?? question; }
-    this.store.deliverRead(binding.session, question.envelope);
-    // its creator's copy, filed once with a wake: the carrier hands it over, here or on the creator's machine (whose
-    // copy kept here is the session's timeline's mirror, marked read once carried)
-    if (!this.store.findIn(binding.creator, id)) { this.store.deliver(binding.creator, question.envelope); this.store.requestWake(binding.creator, id); }
+    this.store.transaction(() => {
+      if (!this.store.putRecord('native_questions', `${id}.json`, question, { create: true })) question = this.store.record('native_questions', `${id}.json`);
+      if (!question?.envelope) throw new Error(`question ${id} could not be read`);
+      if (this.store.record('native_questions', `${id}.delivered`) != null) return;
+      this.store.deliverRead(binding.session, question.envelope);
+      // Its creator's copy and wake commit with the question, so interruption
+      // cannot publish half of a new question.
+      if (!this.store.findIn(binding.creator, id)) { this.store.deliver(binding.creator, question.envelope); this.store.requestWake(binding.creator, id); }
+    });
   }
 
   /** A reply the native harness used confirmed (mail_question.rs `confirm`): the reply read in the session's mailbox, and
    *  when it was used recorded once (`<q-id>.delivered`, which a listing reads as its delivery). */
   confirm(id, reply) {
-    const question = readJson(join(this.dir, `${id}.json`));
+    const question = this.store.record('native_questions', `${id}.json`);
     if (!question) throw new Error(`question ${id} is not on file`);
-    const found = reply?.envelope?.id ? this.store.findIn(question.binding.session, reply.envelope.id) : null;
-    if (found && found.state === 'unread') { try { this.store.markRead(question.binding.session, found.path); } catch { /* read meanwhile */ } }
-    try { this.create(join(this.dir, `${id}.delivered`), Date.now()); } catch (error) { if (error.code !== 'EEXIST') throw error; }
+    this.store.transaction(() => {
+      const found = reply?.envelope?.id ? this.store.findIn(question.binding.session, reply.envelope.id) : null;
+      if (found && found.state === 'unread') this.store.markRead(question.binding.session, found.path);
+      this.store.putRecord('native_questions', `${id}.delivered`, Date.now(), { create: true });
+    });
   }
 }

package/mail/reads.mjs

@@ -237,8 +237,8 @@ export class MailReads {
       const stored = this.#whole(found[0].stored);
       if (!stored) return { id, found: false };
       if (!transcripts.has(address)) transcripts.set(address, readTranscript(address, this.homes));
-      const delivered = deliveries(address, [stored], await transcripts.get(address), (envelope) => questionDeliveredAt(envelope, this.homes.mailRoot));
-      return { id, found: true, machine: this.machine, message: messageRow(address, stored, delivered, this.homes.mailRoot) };
+      const delivered = deliveries(address, [stored], await transcripts.get(address), (envelope) => questionDeliveredAt(envelope, this.store));
+      return { id, found: true, machine: this.machine, message: messageRow(address, stored, delivered, this.store) };
     }));
   }
 
@@ -304,9 +304,7 @@ export class MailReads {
       const found = this.store.find(id);
       if (found.length !== 1) return { id, found: false };
       const asked = found[0].stored.envelope;
-      const namesIt = (reply) => reply === asked.id || (reply.length >= 10 && asked.id.startsWith(reply));
-      const replies = this.store.inventory(asked.from).stored
-        .filter((stored) => typeof stored.envelope.in_reply_to === 'string' && namesIt(stored.envelope.in_reply_to))
+      const replies = this.store.replies(asked.from, asked.id)
         .map((stored) => this.#whole(stored)).filter(Boolean)
         .map(({ envelope }) => ({ id: envelope.id, from: envelope.from, from_name: envelope.from_name, created_at_ms: envelope.created_at_ms, body: envelope.body }));
       return { id, found: true, message: asked.id, session: found[0].address, asker: asked.from, replies };
@@ -326,17 +324,14 @@ export class MailReads {
     const transcript = all ? await readTranscript(address, this.homes) : null;
     let stored;
     if (all) {
-      stored = this.store.inventory(address).stored
-        .filter((item) => item.state === 'read' || (item.envelope.kind === 'user' && item.part === 'claimed'))
-        .map((item) => this.#whole(item)).filter(Boolean);
-      if (last !== null) stored = stored.slice(Math.max(0, stored.length - last));
+      stored = this.store.history(address, last).map((item) => this.#whole(item)).filter(Boolean);
       stored.push(...claimed);
       stored.sort((a, b) => a.envelope.created_at_ms - b.envelope.created_at_ms);
     } else stored = claimed;
-    const delivered = deliveries(address, stored, transcript, (envelope) => questionDeliveredAt(envelope, this.homes.mailRoot));
+    const delivered = deliveries(address, stored, transcript, (envelope) => questionDeliveredAt(envelope, this.store));
     const now = Date.now();
     for (const item of claimed) delivered.set(item.envelope.id, { state: 'by_this_read', at_ms: now });
-    return { stored, deliveries: delivered, claimed, rendered: stored.map((item) => listed(item.envelope, delivered, this.homes.mailRoot)), rows: stored.map((item) => messageRow(address, item, delivered, this.homes.mailRoot)) };
+    return { stored, deliveries: delivered, claimed, rendered: stored.map((item) => listed(item.envelope, delivered, this.store)), rows: stored.map((item) => messageRow(address, item, delivered, this.store)) };
   }
 
   /** The live session a name or address names here (mail_route.rs `resolve`, without the ledger of earlier names). */
@@ -356,25 +351,19 @@ export class MailReads {
    *  `inbox --session` rows (`inbox_rows`), the newest `last` only when given. */
   async inboxRows(address, last = null) {
     const transcript = await readTranscript(address, this.homes);
-    const received = this.store.inventory(address).stored.filter((item) => !(item.envelope.kind === 'question' && item.envelope.from === address)).map((item) => this.#whole(item)).filter(Boolean);
-    const delivered = deliveries(address, received, transcript, (envelope) => questionDeliveredAt(envelope, this.homes.mailRoot));
-    const sent = [];
-    for (const to of this.store.addresses()) {
-      if (to === address) continue;
-      for (const item of this.store.inventory(to).stored) if (item.envelope.from === address) { const whole = this.#whole(item); if (whole) sent.push([to, whole]); }
-    }
-    let timeline = [...received.map((item) => [address, item, 'in']), ...sent.map(([to, item]) => [to, item, 'out'])];
-    timeline.sort((a, b) => a[1].envelope.created_at_ms - b[1].envelope.created_at_ms);
-    if (last !== null) timeline = timeline.slice(Math.max(0, timeline.length - last));
+    const timeline = this.store.timeline(address, last)
+      .map((row) => [row.address, this.#whole(row.stored), row.direction]).filter(([, item]) => item);
+    const received = timeline.filter(([, , direction]) => direction === 'in').map(([, item]) => item);
+    const delivered = deliveries(address, received, transcript, (envelope) => questionDeliveredAt(envelope, this.store));
     // where a message it sent stands is read from its receiver's transcript, the newest receivers' only
     const receivers = [];
     for (const [to, , direction] of [...timeline].reverse()) if (direction === 'out' && !receivers.includes(to) && receivers.length < MAX_RECEIVERS_READ) receivers.push(to);
     for (const to of receivers) {
       const theirs = timeline.filter(([receiver, , direction]) => direction === 'out' && receiver === to).map(([, item]) => item);
-      for (const [id, delivery] of deliveries(to, theirs, await readTranscript(to, this.homes), (envelope) => questionDeliveredAt(envelope, this.homes.mailRoot))) delivered.set(id, delivery);
+      for (const [id, delivery] of deliveries(to, theirs, await readTranscript(to, this.homes), (envelope) => questionDeliveredAt(envelope, this.store))) delivered.set(id, delivery);
     }
     for (const [, item, direction] of timeline) if (direction === 'out' && !delivered.has(item.envelope.id)) delivered.set(item.envelope.id, { state: 'not_read' });
-    return timeline.map(([box, item, direction]) => ({ ...messageRow(box, item, delivered, this.homes.mailRoot), direction }));
+    return timeline.map(([box, item, direction]) => ({ ...messageRow(box, item, delivered, this.store), direction }));
   }
 
   // ---- the machine record's mail ---------------------------------------------------------------------------------
@@ -385,12 +374,12 @@ export class MailReads {
     const header = (item) => ({ id: item.envelope.id, from: item.envelope.from, from_name: item.envelope.from_name, kind: item.envelope.kind, subject: item.envelope.subject ?? null, thread: item.envelope.thread ?? item.envelope.id, in_reply_to: item.envelope.in_reply_to ?? null, created_at_ms: item.envelope.created_at_ms, unread: item.state === 'unread' });
     const mail = {};
     for (const address of mailboxes ?? this.store.addresses()) {
-      const { stored, skipped } = this.store.inventory(address);
-      const answer = { unread: stored.filter((item) => item.state === 'unread').map(header), recent: [...stored].reverse().slice(0, recent).map(header), count: stored.length };
+      const window = this.store.headerWindow(address, recent);
+      const { skipped } = window;
+      const answer = { unread: window.unread.map(header), recent: window.recent.map(header), count: window.count };
       if (skipped.length) answer.partial = `${skipped.length} envelope file${skipped.length === 1 ? '' : 's'} could not be read (${skipped.slice(0, 5).join(', ')}); its unread mail may be more than listed`;
       mail[address] = answer;
     }
     return { mail, unread_boxes: this.store.unread ? [this.store.unread] : [] };
   }
 }
-