@volter/supercode-teams 0.3.85 → 0.3.87

package/mail/store.mjs

@@ -0,0 +1,419 @@
+// This machine's mail, as the native binary keeps it (crates/harness/src/mailbox.rs): a Maildir per address under the
+// mail folder (`<harness>-<hash>/` with its `address` file and `tmp`, `new`, `claimed`, `cur` folders), one JSON envelope
+// per message named `<created_at_ms, 13 digits>.<id>.json`, moved only by rename (a claim prefixes its reader's pid).
+//
+// The daemon holds every envelope's header in memory (the header index), read once when its file appears or moves, from
+// its watch on the mail folder; a call answers from it, and reads a body from its one file only when that message is
+// asked for. Until `message watch` moves into the daemon, the native binary writes into the same folders, so the writes
+// here are mailbox.rs's own: the same names and the same renames (docs/architecture/overview.md, "The mail operations").
+import { EventEmitter } from 'node:events';
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, unlinkSync, watch, writeFileSync, writeSync } from 'node:fs';
+import { join } from 'node:path';
+import { blake3Hex } from './envelope.mjs';
+
+/** The folders an envelope can be in, and what each means. */
+const STATES = { new: 'unread', claimed: 'unread', cur: 'read' };
+/** Fewest hex digits after the kind (`m-`, `u-`, `a-`) a prefix may give. */
+export const MIN_PREFIX_DIGITS = 4;
+
+const alive = (pid) => {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; }
+};
+const readEnvelope = (path) => { try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; } };
+const order = (a, b) => (a.envelope.created_at_ms - b.envelope.created_at_ms) || (a.envelope.id < b.envelope.id ? -1 : a.envelope.id > b.envelope.id ? 1 : 0);
+
+/** The header an index keeps of an envelope: everything but its body. */
+const headerOf = (envelope) => { const { body, ...header } = envelope; return header; };
+
+export class MailStore extends EventEmitter {
+  constructor({ mailRoot, machine = null, log = () => {} }) {
+    super();
+    this.root = mailRoot; this.machine = machine; this.log = log;
+    this.boxes = new Map(); // folder name → { address, dir, files: Map(`${state folder}/${file}` → { header, state, path }), skipped: Set }
+    this.byAddress = new Map(); // address → folder name
+    this.unread = null; // why the mail folder is not current, when it is not
+  }
+
+  // ---- the index ------------------------------------------------------------------------------------------------
+
+  /** Read every mailbox once (the owner's startup load), then follow the folder's events. */
+  start() {
+    mkdirSync(this.root, { recursive: true });
+    this.#rescan();
+    this.#watch();
+    return this;
+  }
+
+  stop() { try { this.watcher?.close(); } catch { /* closed */ } clearTimeout(this.retry); }
+
+  #watch(delay = 0) {
+    try {
+      this.watcher = watch(this.root, { recursive: true }, (_event, name) => this.#event(name ? String(name) : null));
+      this.watcher.on('error', (error) => { try { this.watcher.close(); } catch { /* closed */ } this.#lost(error, delay); });
+      if (this.unread) { this.unread = null; this.#rescan(); }
+    } catch (error) { this.#lost(error, delay); }
+  }
+
+  #lost(error, delay) {
+    const next = Math.min(300_000, Math.max(2000, (delay || 1000) * 2));
+    this.unread = `the mail folder ${this.root} is not watched (${error.message}); watched again in ${Math.round(next / 1000)} s`;
+    this.log(`mail: ${this.unread}`);
+    this.retry = setTimeout(() => this.#watch(next), next); this.retry.unref?.();
+  }
+
+  /** One event: the file it names read again (or its mailbox, or the whole folder when it names neither). */
+  #event(name) {
+    if (!name) return this.#rescan();
+    const [folder, part, file] = name.split(/[\\/]/);
+    if (!folder) return;
+    if (!part) return this.#readBox(folder);
+    if (!STATES[part] || !file) { if (part === 'address') this.#readBox(folder); return; }
+    if (!this.boxes.has(folder)) return this.#readBox(folder);
+    this.#readFile(folder, part, file);
+  }
+
+  #rescan() {
+    let names = [];
+    try { names = readdirSync(this.root); } catch (error) { this.unread = `the mail folder ${this.root} could not be read: ${error.message}`; return; }
+    const present = new Set(names);
+    for (const folder of [...this.boxes.keys()]) if (!present.has(folder)) this.#dropBox(folder);
+    for (const folder of names) this.#readBox(folder);
+  }
+
+  #dropBox(folder) {
+    const box = this.boxes.get(folder);
+    if (!box) return;
+    this.boxes.delete(folder);
+    if (this.byAddress.get(box.address) === folder) this.byAddress.delete(box.address);
+    this.emit('changed', box.address);
+  }
+
+  /** A mailbox read whole: its address, and every envelope in each of its folders. */
+  #readBox(folder) {
+    let address = null;
+    try { address = readFileSync(join(this.root, folder, 'address'), 'utf8').trim(); } catch { /* not a mailbox */ }
+    if (!address) { this.#dropBox(folder); return; }
+    const box = { address, dir: join(this.root, folder), files: new Map(), skipped: new Set() };
+    this.boxes.set(folder, box);
+    this.byAddress.set(address, folder);
+    for (const part of Object.keys(STATES)) {
+      let files = [];
+      try { files = readdirSync(join(box.dir, part)); } catch { continue; }
+      for (const file of files) this.#take(box, part, file);
+    }
+    this.emit('changed', address);
+  }
+
+  /** One envelope file read again on its event: held while it is there, let go once it is gone. */
+  #readFile(folder, part, file) {
+    const box = this.boxes.get(folder);
+    if (!box) return;
+    box.files.delete(`${part}/${file}`); box.skipped.delete(`${part}/${file}`);
+    this.#take(box, part, file);
+    this.emit('changed', box.address);
+  }
+
+  #take(box, part, file) {
+    if (!file.endsWith('.json')) return;
+    const path = join(box.dir, part, file);
+    let text;
+    try { text = readFileSync(path, 'utf8'); } catch { return; } // gone (a rename took it): its new place has its own event
+    let envelope = null; try { envelope = JSON.parse(text); } catch { /* unreadable */ }
+    // an unreadable file is skipped, never fatal, and said: a mailbox that skipped one is partial
+    if (!envelope || typeof envelope.id !== 'string') { box.skipped.add(`${part}/${file}`); return; }
+    // its body's first line is kept beside the header (never in it): a listing of threads names each root by it with no
+    // body read
+    const firstLine = typeof envelope.body === 'string' ? envelope.body.slice(0, 400).split('\n')[0] : '';
+    box.files.set(`${part}/${file}`, { header: headerOf(envelope), state: STATES[part], path, part, firstLine });
+  }
+
+  // ---- what it holds --------------------------------------------------------------------------------------------
+
+  // What is held is the watch's: a listing and a lookup answer from the index, never from a walk of the mail folder. A
+  // mailbox the index has not seen (its folder's event lost) is read once, by its own folder's name, when it is asked
+  // about by its address; a delivery's decisions (a claim, a send's dedupe) read the one mailbox they name.
+
+  /** Every mailbox's address, as the index holds them. */
+  addresses() { return [...this.byAddress.keys()]; }
+
+  /** One mailbox's envelopes, oldest first, as `{ envelope (header only), state, path }`, and the files it skipped. */
+  inventory(address) {
+    const box = this.boxOf(address) ?? this.#named(address);
+    if (!box) return { stored: [], skipped: [] };
+    const stored = [...box.files.values()].map((entry) => ({ envelope: entry.header, state: entry.state, path: entry.path, part: entry.part }));
+    stored.sort(order);
+    return { stored, skipped: [...box.skipped] };
+  }
+
+  /** An envelope whole (its body read from its file), or null when it is gone. */
+  body(stored) { return readEnvelope(stored.path); }
+
+  /** The messages filed here whose id is `id` or starts with it (a prefix shorter than the kind and
+   *  MIN_PREFIX_DIGITS digits matches nothing): each with the mailbox holding it. */
+  find(id) {
+    const digits = id.includes('-') ? id.slice(id.indexOf('-') + 1).length : 0;
+    if (digits < MIN_PREFIX_DIGITS) return [];
+    const found = [];
+    for (const box of this.boxes.values()) {
+      for (const entry of box.files.values()) {
+        if (!entry.header.id.startsWith(id) || found.some((item) => item.stored.envelope.id === entry.header.id)) continue;
+        found.push({ address: box.address, stored: { envelope: entry.header, state: entry.state, path: entry.path, part: entry.part } });
+      }
+    }
+    return found;
+  }
+
+  /** The first line of each message whose id is in `ids`, from the index in one pass (no file read): id → line. */
+  firstLines(ids) {
+    const wanted = new Set(ids);
+    const lines = new Map();
+    if (!wanted.size) return lines;
+    for (const box of this.boxes.values()) {
+      for (const entry of box.files.values()) {
+        if (wanted.has(entry.header.id) && !lines.has(entry.header.id)) lines.set(entry.header.id, entry.firstLine ?? '');
+      }
+    }
+    return lines;
+  }
+
+  // ---- writes, as mailbox.rs makes them --------------------------------------------------------------------------
+
+  /** The unread mail of `address` (not the user's own turns) moved into `claimed/` under its reader's pid (`pid`, the
+   *  reading process: a claim whose reader ends goes back to `new/`), each claim recorded; another reader that took one
+   *  first keeps it (mailbox.rs `claim_unread`). Oldest first. */
+  claimUnread(address, via, caller, pid = process.pid) {
+    // the mailbox's own folders, read here (one named read): what decides a delivery never comes from the watch-fed
+    // index alone, whose events a platform can drop (an FSEvents drop, an inotify overflow)
+    const box = this.boxOf(address) ?? this.#named(address);
+    if (!box) return [];
+    this.#recoverAbandoned(box);
+    let names = [];
+    try { names = readdirSync(join(box.dir, 'new')); } catch { return []; }
+    const claimed = [];
+    for (const file of names) {
+      if (!file.endsWith('.json')) continue;
+      const path = join(box.dir, 'new', file);
+      const envelope = readEnvelope(path);
+      if (!envelope || typeof envelope.id !== 'string') { box.skipped.add(`new/${file}`); continue; }
+      if (envelope.kind === 'user') continue;
+      const target = join(box.dir, 'claimed', `${pid}.${file}`);
+      try { renameSync(path, target); }
+      catch (error) { if (error.code === 'ENOENT') continue; throw error; }
+      try { this.recordClaim(box, envelope.id, { pid, via, caller: caller ?? null, recipient: true, at_ms: Date.now() }); } catch { /* the claim's record is best kept, never the claim */ }
+      claimed.push({ envelope: headerOf(envelope), state: 'unread', path: target, part: 'claimed' });
+    }
+    claimed.sort(order);
+    return claimed;
+  }
+
+  /** A claimed envelope reached its reader: moved to `cur/` under its own name (mailbox.rs `acknowledge`). */
+  acknowledge(address, claimed) {
+    const box = this.boxes.get(this.byAddress.get(address));
+    if (!box) return;
+    const name = claimed.path.split(/[\\/]/).pop();
+    const original = name.includes('.') ? name.slice(name.indexOf('.') + 1) : name;
+    renameSync(claimed.path, join(box.dir, 'cur', original));
+  }
+
+  /** Who took a message to its reader, kept beside the mailbox (`claims/<id>.json`, mailbox.rs `record_claim`). */
+  recordClaim(box, id, claim) {
+    if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error('invalid message id');
+    mkdirSync(join(box.dir, 'claims'), { recursive: true });
+    const temporary = join(box.dir, 'tmp', `claim.${id}`);
+    writeFileSync(temporary, JSON.stringify(claim));
+    renameSync(temporary, join(box.dir, 'claims', `${id}.json`));
+  }
+
+  /** A claim whose reader has ended goes back to `new/` (mailbox.rs `recover_abandoned_claims`). */
+  #recoverAbandoned(box) {
+    let names = [];
+    try { names = readdirSync(join(box.dir, 'claimed')); } catch { return; }
+    for (const name of names) {
+      const at = name.indexOf('.');
+      if (at < 0) continue;
+      const pid = Number(name.slice(0, at));
+      if (!alive(pid)) { try { renameSync(join(box.dir, 'claimed', name), join(box.dir, 'new', name.slice(at + 1))); } catch { /* another reader recovered it */ } }
+    }
+  }
+
+  // ---- a send's writes (3b-1), as mailbox.rs makes them -------------------------------------------------------------
+
+  /** The mailbox of `address`, made when it is not there yet: its four folders and its `address` file (mailbox.rs
+   *  `open`). Taken into the index at once, so the write that follows is answered from it. */
+  open(address) {
+    const known = this.boxOf(address);
+    if (known) return known;
+    const [, , harness] = String(address).split(':');
+    const folder = `${harness.replace(/[^A-Za-z0-9-]/g, '_')}-${blake3Hex(String(address)).slice(0, 24)}`;
+    const dir = join(this.root, folder);
+    for (const part of ['tmp', 'new', 'claimed', 'cur']) mkdirSync(join(dir, part), { recursive: true });
+    if (!existsSync(join(dir, 'address'))) writeFileSync(join(dir, 'address'), `${address}\n`);
+    this.#readBox(folder);
+    return this.boxes.get(folder);
+  }
+
+  /** The envelope filed in `address`'s mailbox under exactly `id`, read from its folders (mailbox.rs `find`): `{ path,
+   *  part, state }`, or null. Never the index alone: a send's dedupe and its door's "already delivered" decide on it. */
+  findIn(address, id) {
+    const box = this.boxOf(address) ?? this.#named(address);
+    if (!box) return null;
+    const suffix = `.${id}.json`;
+    for (const part of Object.keys(STATES)) {
+      let names = [];
+      try { names = readdirSync(join(box.dir, part)); } catch { continue; }
+      const file = names.find((name) => name.endsWith(suffix));
+      if (file) return { path: join(box.dir, part, file), part, state: STATES[part] };
+    }
+    return null;
+  }
+
+  /** The messages in `address`'s mailbox whose id starts with `prefix`, read from its folders (mailbox.rs `find_prefix`):
+   *  whole envelopes. */
+  findPrefixIn(address, prefix) {
+    const box = this.boxOf(address) ?? this.#named(address);
+    if (!box) return [];
+    const found = [];
+    for (const part of Object.keys(STATES)) {
+      let names = [];
+      try { names = readdirSync(join(box.dir, part)); } catch { continue; }
+      for (const file of names) {
+        if (!file.endsWith('.json')) continue;
+        const fields = file.slice(0, -'.json'.length).split('.');
+        if (!fields[fields.length - 1].startsWith(prefix)) continue;
+        const envelope = readEnvelope(join(box.dir, part, file));
+        if (envelope && !found.some((item) => item.envelope.id === envelope.id)) found.push({ envelope, path: join(box.dir, part, file), part, state: STATES[part] });
+      }
+    }
+    return found;
+  }
+
+  /** Each message id in `address`'s mailbox with the message it answers, read from its folders: a reply chain's links. */
+  links(address) {
+    const box = this.boxOf(address) ?? this.#named(address);
+    const links = new Map();
+    if (!box) return links;
+    for (const part of Object.keys(STATES)) {
+      let names = [];
+      try { names = readdirSync(join(box.dir, part)); } catch { continue; }
+      for (const file of names) {
+        if (!file.endsWith('.json')) continue;
+        const key = `${part}/${file}`;
+        const header = box.files.get(key)?.header ?? (() => { this.#take(box, part, file); return box.files.get(key)?.header; })();
+        if (header) links.set(header.id, header.in_reply_to ?? null);
+      }
+    }
+    return links;
+  }
+
+  /** Mail that came to this machine through a link is for its own sessions only: never filed for another machine's
+   *  address (mailbox.rs `refuse_onward`). */
+  #refuseOnward(address, envelope) {
+    const machine = String(address).split(':')[1];
+    if (this.machine && machine !== this.machine && String(envelope?.sender_identity?.observation ?? '').startsWith('link:')) {
+      throw new Error(`not carried on to ${address}: a message that came to this machine through a link is for its own sessions only`);
+    }
+  }
+
+  /** File `envelope` unread (mailbox.rs `deliver`): `tmp/` written whole with create-new and synced, then renamed into
+   *  `new/`; an id already filed is not written again. Answers its path. */
+  deliver(address, envelope) {
+    this.#refuseOnward(address, envelope);
+    const box = this.open(address);
+    const existing = this.findIn(address, envelope.id);
+    if (existing) return existing.path;
+    const name = `${String(envelope.created_at_ms).padStart(13, '0')}.${envelope.id}.json`;
+    const temporary = join(box.dir, 'tmp', name);
+    const destination = join(box.dir, 'new', name);
+    try {
+      const fd = openSync(temporary, 'wx');
+      try { writeSync(fd, JSON.stringify(envelope)); fsyncSync(fd); } finally { closeSync(fd); }
+      renameSync(temporary, destination);
+    } catch (error) { try { unlinkSync(temporary); } catch { /* not written */ } throw error; }
+    this.#take(box, 'new', name);
+    this.emit('changed', address);
+    return destination;
+  }
+
+  /** File an envelope that reached its reader by another door, already read (mailbox.rs `deliver_read`): into `cur/`;
+   *  an id already filed is not written again. Answers its path. */
+  deliverRead(address, envelope) {
+    this.#refuseOnward(address, envelope);
+    const box = this.open(address);
+    const existing = this.findIn(address, envelope.id);
+    if (existing) return existing.path;
+    const name = `${String(envelope.created_at_ms).padStart(13, '0')}.${envelope.id}.json`;
+    const temporary = join(box.dir, 'tmp', name);
+    writeFileSync(temporary, JSON.stringify(envelope));
+    renameSync(temporary, join(box.dir, 'cur', name));
+    this.#take(box, 'cur', name);
+    this.emit('changed', address);
+    return join(box.dir, 'cur', name);
+  }
+
+  /** A filed message moved to `cur/` under its own name (mailbox.rs `mark_read`). */
+  markRead(address, path) {
+    const box = this.open(address);
+    renameSync(path, join(box.dir, 'cur', path.split(/[\\/]/).pop()));
+  }
+
+  /** A wake kept beside the envelope (mailbox.rs `request_wake`): its watcher hands it to the session's door. */
+  requestWake(address, id) {
+    if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error('invalid wake message id');
+    const box = this.open(address);
+    mkdirSync(join(box.dir, 'wake'), { recursive: true });
+    writeFileSync(join(box.dir, 'wake', id), '');
+  }
+
+  /** A wake's delivery so far (mailbox.rs `wake_state`): an empty or absent file is a fresh wake. */
+  wakeState(address, id) {
+    const box = this.open(address);
+    try { return { attempts: 0, next_at_ms: 0, last_error: null, ...JSON.parse(readFileSync(join(box.dir, 'wake', id), 'utf8')) }; }
+    catch { return { attempts: 0, next_at_ms: 0, last_error: null }; }
+  }
+
+  /** A wake's delivery recorded (mailbox.rs `set_wake_state`): only while its wake is there. */
+  setWakeState(address, id, state) {
+    const box = this.open(address);
+    const path = join(box.dir, 'wake', id);
+    if (!existsSync(path)) return;
+    const temporary = join(box.dir, 'tmp', `wake.${id}`);
+    writeFileSync(temporary, JSON.stringify({ attempts: state.attempts ?? 0, next_at_ms: state.next_at_ms ?? 0, last_error: state.last_error ?? null, handover_at_ms: state.handover_at_ms ?? null, failing_since_ms: state.failing_since_ms ?? null }));
+    renameSync(temporary, path);
+  }
+
+  /** One sender's wish to hear when the receiver next ends a turn (mailbox.rs `subscribe_idle`); the native watch settles it
+   *  until it moves into the daemon (step 4). */
+  subscribeIdle(address, subscription) {
+    const box = this.open(address);
+    const dir = join(box.dir, 'subscriptions');
+    mkdirSync(dir, { recursive: true });
+    const temporary = join(dir, `.${subscription.message_id}.tmp`);
+    writeFileSync(temporary, JSON.stringify({ message_id: subscription.message_id, subscriber: subscription.subscriber, created_at_ms: subscription.created_at_ms ?? Date.now(), seen_working: false, notice: subscription.notice !== false, final_reply: subscription.final_reply === true }));
+    renameSync(temporary, join(dir, `${subscription.message_id}.json`));
+  }
+
+  /** Who took a message to its reader, by its mailbox's address (record_claim). */
+  claimFor(address, id, claim) { this.recordClaim(this.open(address), id, claim); }
+
+  /** The latest claim of a message, when it has one (mailbox.rs `claim_of`). */
+  claimOf(address, id) {
+    const box = this.boxOf(address) ?? this.#named(address);
+    if (!box) return null;
+    try { return JSON.parse(readFileSync(join(box.dir, 'claims', `${id}.json`), 'utf8')); } catch { return null; }
+  }
+
+  /** A mailbox's folder for writes, by its address, when it is one this machine has. */
+  boxOf(address) { return this.boxes.get(this.byAddress.get(address)) ?? null; }
+
+  /** The mailbox of `address` read by its own folder's name (mailbox.rs `directory_name`), when the index has not seen
+   *  it (its folder's event was dropped): taken into the index from that one read, or null when there is none. */
+  #named(address) {
+    const [, , harness] = String(address).split(':');
+    if (!harness) return null;
+    const folder = `${harness.replace(/[^A-Za-z0-9-]/g, '_')}-${blake3Hex(String(address)).slice(0, 24)}`;
+    this.#readBox(folder);
+    const box = this.boxes.get(folder) ?? null;
+    return box?.address === address ? box : null;
+  }
+}