@volter/supercode-teams 0.3.159 → 0.3.161

package/mail/send.mjs

@@ -9,6 +9,7 @@ import { AGENT_HARNESS, RESUME_WAIT_MS } from './agents.mjs';
 import { MAX_RELAYED_BYTES, RELAY_NAME_PREFIX } from './relay.mjs';
 import { FULL_ID_LEN, cameThroughLink, envelope as makeEnvelope, makeAddress, messageIdFor, newMessageId, normalMachineName, parseAddress, senderIdentity, subjectLine, threadOf } from './compose.mjs';
 import { nativeMailContinuation } from './continuation.mjs';
+import { paneWaitReason, terminalReceipt } from './delivery-status.mjs';
 
 export const EXIT_REFUSED = 2;
 export const EXIT_UNKNOWN = 3;
@@ -29,7 +30,7 @@ const CODEX_KNOWN_WAIT_MS = 20_000;
 const RELAY_GRACE_MS = 30_000;
 /** The kinds of delivery that hand nothing over (#deliverWoken's `how`): the copy waits filed. An operator's mailbox,
  *  which its operator reads, and a session's hook, which shows it, are deliveries. */
-const LEFT_FILED = new Set(['queued', 'stored', 'pane_held', 'pane_waits', 'not_running']);
+const LEFT_FILED = new Set(['queued', 'stored', 'pane_held', 'pane_waits', 'native_waits', 'not_running']);
 
 /** Replies in one unbroken chain between two sessions beyond which another is refused. */
 const REPLY_CHAIN_LIMIT = 32;
@@ -41,7 +42,7 @@ const NAME_SEEN_EVERY_MS = 60 * 60 * 1000;
 const outcome = (code, text, receipt = null) => ({ code, text, receipt });
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** The daemon pane a live session runs in, or null (mail_route.rs `daemon_pane`). */
-const paneOf = (session) => { const pane = typeof session.pane === 'string' ? session.pane.split(':')[0] : null; return pane && /^p_[A-Za-z0-9_-]+$/.test(pane) ? pane : null; };
+const paneOf = (session) => { const selected = session.door === 'native' ? session.delivery_pane : session.pane; const pane = typeof selected === 'string' ? selected.split(':')[0] : null; return pane && /^p_[A-Za-z0-9_-]+$/.test(pane) ? pane : null; };
 
 /** A delivery a door refused before anything was sent. */
 class Refused extends Error { constructor(kind, bytes = 0) { super(kind); this.kind = kind; this.bytes = bytes; } }
@@ -274,7 +275,10 @@ export class MailSend {
     const session = sessions.find((row) => row.address === address);
     if (!session) return null;
     if (session.door === 'runtime') return { kind: 'runtime', session };
-    if (session.door === 'native') return { kind: 'native', session };
+    if (session.door === 'native') {
+      const pane = paneOf(session);
+      return pane ? { kind: 'pane', session, pane } : { kind: 'native', session };
+    }
     if (session.door === 'hook') return { kind: 'hook', session, pane: session.pane ?? null, idle: session.status === 'idle' };
     // a session with no hook and no relay (Grok) in a daemon pane: its mail is typed there, into an empty composer
     if (session.door === 'pane' && session.pane) return { kind: 'pane', session, pane: session.pane };
@@ -288,11 +292,16 @@ export class MailSend {
     if (address === caller.address) return outcome(EXIT_REFUSED, 'Not sent: that address is your own session.');
     const door = parseAddress(address).harness === AGENT_HARNESS ? null : this.#door(address, sessions);
     const messageId = id == null && chosen ? chosen : messageIdFor(caller.address, id);
-    const terminal = door?.kind === 'hook' && door.pane != null;
+    const terminal = ['hook', 'pane'].includes(door?.kind) && door.pane != null;
     const previous = this.store.sentMarker(caller.address, messageId);
     if (previous !== null) {
-      if (terminal && !queue) this.store.requestWake(address, messageId);
-      return outcome(0, `Already sent with --id ${id ?? ''}: ${previous} Nothing was sent again.`, this.#deliveryReceipt(address, messageId, terminal));
+      const stored = this.store.findIn(address, messageId);
+      const waiting = !['operator', 'board'].includes(parseAddress(address).harness) && stored?.state === 'unread'
+        && this.store.claimOf(address, messageId)?.recipient !== true;
+      const receipt = this.#deliveryReceipt(address, messageId, terminal || waiting);
+      if ((terminal || waiting) && !queue && !receipt.refused && !receipt.delivered) this.store.requestWake(address, messageId, { keep: true });
+      return outcome(!receipt.delivered ? EXIT_STORED : 0,
+        `Already filed with --id ${id ?? ''}: ${previous} Nothing was sent again.${receipt.refused ? ` Delivery stopped: ${receipt.reason}.` : ''}`, receipt);
     }
     const answered = this.#answeredId(caller.address, in_reply_to);
     if (answered.refused) return answered.refused;
@@ -334,7 +343,7 @@ export class MailSend {
     const [code, what] = {
       steered: [0, 'steered into its running turn'],
       started: [0, 'it was idle, so the message started a turn'],
-      native_busy: [0, 'it will read it at its next tool call'],
+      native_waits: [EXIT_STORED, 'Stored: its native session is not known to be idle, so the message waits unread until it is idle. Do not resend'],
       native_idle: [0, 'it was idle, so the message starts its next turn'],
       hooked: [0, 'queued in its mailbox; terminal delivery waits for native transcript confirmation'],
       hook_woken: [0, 'it was idle, so its pane was told to read its mailbox; its hook shows it the message in that turn'],
@@ -342,8 +351,8 @@ export class MailSend {
       operator: [0, 'filed in its mailbox'],
       already: [0, 'it already had this message; nothing was sent again'],
       typed: [0, 'typed into its pane as its next turn'],
-      pane_held: [0, 'held for its pane: typed once its composer is free'],
-      pane_waits: [EXIT_STORED, 'Stored (not failed): its pane\'s composer was not empty, so the message waits in its mailbox; it sees it with supercode message inbox. Don\'t resend'],
+      pane_held: [EXIT_STORED, 'Stored: held by its terminal queue; native transcript acceptance is not confirmed. Do not resend'],
+      pane_waits: [EXIT_STORED, `Stored: its pane did not accept input (${this.store.wakeState(address, messageId).last_error ?? 'not at an empty composer'}); the message waits unread in its mailbox. Do not resend`],
       stored: [EXIT_STORED, `Stored (not failed): it has no delivery door, so it sees this only if it runs supercode message inbox${parseAddress(address).harness === 'codex' ? ' (a Codex session gets one with: supercode message setup codex)' : ''}. Don't resend, and don't wait for a reply`],
     }[delivered];
     this.#recordSend(caller.address, address, messageId, sentIn, receiver.continuation);
@@ -351,8 +360,8 @@ export class MailSend {
     const text = code === 0
       ? `sent to ${name} (${harness}, ${door.kind}): ${what}. Message id ${shortId(messageId)}.${receiver.alias ?? ''}${idleNote} Don't poll; carry on.`
       : `${name} (${harness}): ${what}. Message id ${shortId(messageId)}.${idleNote}`;
-    if (code === 0 && id != null) { try { this.store.setSentMarker(caller.address, messageId, text); } catch { /* the marker is best kept */ } }
-    return outcome(code, text, code === 0 ? this.#deliveryReceipt(address, messageId, terminal) : null);
+    if ((code === 0 || (terminal && code === EXIT_STORED)) && id != null) { try { this.store.setSentMarker(caller.address, messageId, text); } catch { /* the marker is best kept */ } }
+    return outcome(code, text, code === 0 || terminal ? this.#deliveryReceipt(address, messageId, terminal) : null);
   }
 
   /** An agent's plan for `sent` (its thread store decides), or an outcome when the plan refused it. */
@@ -386,8 +395,16 @@ export class MailSend {
       if (how === 'not_woken') { this.store.deliver(to, sent); delivered = 'queued'; }
       else { this.store.deliverRead(to, handed); finalReply = answers; delivered = how === 'steered' ? 'steered' : 'started'; }
     } else if (door.kind === 'native') {
-      const busy = door.session.status !== 'idle';
+      const busy = door.session.status !== 'idle' || door.session.panes_current === false;
       if (!wake && !busy) throw new Refused('queue');
+      if (busy) {
+        this.store.deliver(to, sent);
+        this.store.requestWake(to, sent.id, { keep: true });
+        const state = this.store.wakeState(to, sent.id);
+        this.store.setWakeState(to, sent.id, { ...state, waiting_for_idle: true, next_at_ms: Date.now() + 300_000 });
+        if (notifyWhenIdle && !cameThroughLink(sent)) this.store.subscribeIdle(to, { message_id: sent.id, subscriber: sent.from, notice: true, final_reply: false });
+        return 'native_waits';
+      }
       // its reply line names the door this session really has: its tools when it runs supercode's MCP server
       const handed = sent.reply_via?.mode === 'command' && door.session.tools ? { ...sent, reply_via: { mode: 'tool' } } : sent;
       const text = render(handed);
@@ -421,7 +438,7 @@ export class MailSend {
       }
       this.relaying.add(key);
       let relayed;
-      try { relayed = await this.relays.send(handed.from, handed.from_name, { sessionId: door.session.address.split(':').slice(3).join(':'), name: door.session.name.split('@')[0], socket: door.session.socket }, text, handed.id); }
+      try { relayed = await this.relays.send(handed.from, handed.from_name, { sessionId: door.session.address.split(':').slice(3).join(':'), name: door.session.name.split('@')[0], socket: door.session.socket }, text, handed.id, { expectsReply: handed.reply_via?.mode !== 'none' }); }
       finally { this.relaying.delete(key); }
       // a send the relay may have made keeps its copy and is unanswered, never taken back and said not sent; only one the
       // relay is known not to have made (refused, never handed in) takes its copy back. One its receiver read from its
@@ -441,7 +458,7 @@ export class MailSend {
         const now = this.store.findIn(to, handed.id);
         if (now?.state === 'unread') { try { this.store.markRead(to, now.path); } catch { /* moved meanwhile */ } }
       }
-      delivered = busy ? 'native_busy' : 'native_idle';
+      delivered = 'native_idle';
     } else if (door.kind === 'hook') {
       this.store.deliver(to, sent);
       if (wake) this.store.requestWake(to, sent.id);
@@ -449,13 +466,21 @@ export class MailSend {
     } else if (door.kind === 'pane') {
       // typed as its next turn only into an empty composer (machine.mjs #paneTurn); otherwise it waits in its mailbox
       let typed = null;
-      if (wake) { try { typed = await this.typeTurn(door.pane, parseAddress(to).harness, render(sent), [sent.id]); } catch { /* waits */ } }
+      if (wake) { try { typed = await this.typeTurn(door.pane, parseAddress(to).harness, render(sent), [sent.id], to); } catch (error) { typed = { reason: error.message }; } }
       if (typed?.delivered) { this.store.deliverRead(to, sent); delivered = 'typed'; }
       // held by the daemon's queue for the pane, typed once its composer frees: not typed yet, so kept unread
-      else if (typed?.queued) { this.store.deliver(to, sent); delivered = 'pane_held'; }
+      else if (typed?.queued) { this.store.deliver(to, sent); if (wake) this.store.requestWake(to, sent.id, { keep: true }); delivered = 'pane_held'; }
       // not typed (its composer held a draft or a turn ran): it waits unread with a wake, so the carrier types it once the
       // composer is empty (mail/carrier.mjs), never left for a read a session with no hook never makes
-      else { this.store.deliver(to, sent); if (wake) this.store.requestWake(to, sent.id); delivered = wake ? 'pane_waits' : 'queued'; }
+      else {
+        this.store.deliver(to, sent);
+        if (wake) {
+          this.store.requestWake(to, sent.id, { keep: true });
+          const state = this.store.wakeState(to, sent.id);
+          this.store.setWakeState(to, sent.id, { ...state, next_at_ms: Date.now() + 2_000, last_error: paneWaitReason(typed) });
+        }
+        delivered = wake ? 'pane_waits' : 'queued';
+      }
     } else if (door.kind === 'stored') { this.store.deliver(to, sent); delivered = 'stored'; }
     else { this.store.deliver(to, sent); delivered = 'operator'; }
     // a hand-over to the recipient's own door is its delivery, recorded as such
@@ -592,7 +617,7 @@ export class MailSend {
       if (error instanceof Refused) throw new Error(`${error.bytes} bytes; the limit is ${MAX_RELAYED_BYTES}`);
       throw error;
     }
-    return { how, text: { steered: 'steered into its running turn', started: 'started a turn', native_busy: 'read at its next tool call', native_idle: 'starts its next turn', hooked: 'in its mailbox, its hook shows it', queued: 'waits in its mailbox', operator: 'filed', already: 'it already had it', stored: 'stored; it has no delivery door', typed: 'typed into its pane', pane_held: 'held for its pane until its composer is free', pane_waits: 'waits in its mailbox (its composer was not empty)' }[how] };
+    return { how, text: { steered: 'steered into its running turn', started: 'started a turn', native_waits: 'waits unread until its native turn is idle', native_idle: 'starts its next turn', hooked: 'in its mailbox, its hook shows it', queued: 'waits in its mailbox', operator: 'filed', already: 'it already had it', stored: 'stored; it has no delivery door', typed: 'typed into its pane', pane_held: 'held for its pane until its composer is free', pane_waits: 'waits in its mailbox (its composer was not empty)' }[how] };
   }
 
   // ---- a filing --------------------------------------------------------------------------------------------------
@@ -665,9 +690,11 @@ export class MailSend {
       id = `m-${blake3Hex(JSON.stringify([sender, receiver, key]))}`.slice(0, 26);
       const previous = this.store.findIn(receiver, id);
       const saved = previous ? this.store.body(previous) : null;
-      if (saved && (saved.body !== text || (saved.subject ?? null) !== subject || saved.kind !== (params.channel ? 'channel' : 'peer') || saved.from_name !== (params.sender_name ?? `${fromName}@${this.machine}`) || (saved.in_reply_to ?? null) !== (params.in_reply_to ?? null) || (saved.voice_for ?? null) !== (params.voice_for ?? null))) {
+      const bodyMatches = previous ? this.store.bodyMatches(receiver, id, text) : null;
+      if (saved && (bodyMatches === false || (saved.subject ?? null) !== subject || saved.kind !== (params.channel ? 'channel' : 'peer') || saved.from_name !== (params.sender_name ?? `${fromName}@${this.machine}`) || (saved.in_reply_to ?? null) !== (params.in_reply_to ?? null) || (saved.voice_for ?? null) !== (params.voice_for ?? null))) {
         return refused('idempotency_conflict', 'this key already names different mail');
       }
+      if (saved && bodyMatches == null) return { ...refused('idempotency_payload_expired', 'this key already names retained mail, but an older release expired its payload before keeping a digest; the original payload cannot be compared. Nothing was filed again.'), message_id: id };
     }
     const inReplyTo = typeof params.in_reply_to === 'string' ? params.in_reply_to : null;
     const sent = makeEnvelope({
@@ -691,9 +718,9 @@ export class MailSend {
       return refused('delivery_failed', error.message);
     }
     return {
-      delivered_to_bus: door.kind !== 'stored' && how !== 'pane_waits', message_id: sent.id, reply_to: sender,
+      delivered_to_bus: door.kind !== 'stored' && !['pane_waits', 'pane_held', 'native_waits'].includes(how), message_id: sent.id, reply_to: sender,
       target: { harness, session_id: sessionId, name: door.kind === 'native' ? door.session.name.split('@')[0] : null },
-      delivery: { door: door.kind, how: { steered: 'steered', started: 'started', native_busy: 'next_tool_call', native_idle: 'started', hooked: 'hook', queued: 'queued', stored: 'stored', operator: 'filed', already: 'already', typed: 'typed', pane_held: 'held', pane_waits: 'stored' }[how] },
+      delivery: { door: door.kind, how: { steered: 'steered', started: 'started', native_waits: 'waiting_for_idle', native_idle: 'started', hooked: 'hook', queued: 'queued', stored: 'stored', operator: 'filed', already: 'already', typed: 'typed', pane_held: 'held', pane_waits: 'stored' }[how] },
     };
   }
 
@@ -807,7 +834,7 @@ export class MailSend {
       const claimed = this.store.claimUserTurn(address, waiting);
       if (!claimed) break;
       let result;
-      try { result = await this.typeTurn(pane, harness, claimed.envelope.body, [claimed.envelope.id]); }
+      try { result = await this.typeTurn(pane, harness, claimed.envelope.body, [claimed.envelope.id], address); }
       catch (error) {
         try { this.store.release(address, claimed); } catch { /* recovered when this process ends */ }
         this.log(`mail: the user's turn ${claimed.envelope.id} for ${address} waits: ${error.message}`);
@@ -859,7 +886,7 @@ export class MailSend {
     const door = this.#door(address, this.live.sessions());
     if (!door) return { outcome: 'later', detail: 'its session is not running' };
     if (door.kind === 'other_machine') return { outcome: 'later', detail: `it is on ${door.machine}` };
-    if (door.kind !== 'native' && door.kind !== 'runtime') {
+    if (door.kind !== 'native' && door.kind !== 'runtime' && door.kind !== 'pane') {
       // a hook's session is pointed at its mailbox by a wake (a native notice filed for it asks one)
       if (door.kind === 'hook' && wake) this.store.requestWake(address, id);
       return { outcome: 'filed', detail: `its door is ${door.kind}` };
@@ -874,6 +901,7 @@ export class MailSend {
     try {
       const how = await this.route(sent, address, door, true, false);
       // handed to the session (or already its): moved to cur/ (one only queued for a runtime not woken would stay unread)
+      if (LEFT_FILED.has(how)) return { outcome: how === 'native_waits' ? 'waiting' : how === 'pane_held' ? 'pending' : 'later', detail: how === 'pane_waits' ? this.store.wakeState(address, id).last_error : how };
       const now = this.store.findIn(address, id);
       if (how !== 'queued' && now?.state === 'unread') this.store.markRead(address, now.path);
       return { outcome: 'delivered', detail: null };
@@ -979,15 +1007,11 @@ export class MailSend {
    *  `delivery_receipt`). */
   #deliveryReceipt(address, id, terminal) {
     if (!terminal) return { delivered: true, message_ids: [id], receipt: 'native-mail' };
-    const parsed = parseAddress(address);
-    const recipient = `${parsed.harness}:${parsed.session}`;
-    const state = this.store.terminalState(recipient);
-    if (state) {
-      const receipt = (Array.isArray(state.receipts) ? state.receipts : []).find((item) => Array.isArray(item?.message_ids) && item.message_ids.includes(id));
-      if (receipt) return receipt;
-      return { delivered: false, queued: true, message_ids: [id], reason: state.active?.error ?? null };
-    }
-    return { delivered: false, queued: true, message_ids: [id], reason: 'awaiting terminal mailbox' };
+    const expired = this.store.record('expired', `${address}/${id}`);
+    const receipt = terminalReceipt(this.store, address, id);
+    if (receipt?.delivered) return receipt;
+    if (expired) return { delivered: false, queued: false, refused: true, message_ids: [id], reason: expired.reason };
+    return receipt ?? { delivered: false, queued: true, message_ids: [id], reason: this.store.wakeState(address, id).last_error ?? 'awaiting terminal mailbox' };
   }
 
   // ---- the names ledger --------------------------------------------------------------------------------------------

package/mail/sqlite-store.mjs

@@ -27,6 +27,7 @@ const stored = (row) => ({
 });
 const COLUMNS = 'mailbox, id, header, state, claim_token';
 const validId = (id) => typeof id === 'string' && /^[A-Za-z0-9_-]+$/.test(id);
+const OFFLINE_CEILING_MS = 30 * 24 * 60 * 60_000;
 
 export class SqliteMailStore extends EventEmitter {
   constructor({ mailRoot, machine = null, log = () => {}, database = null, retention } = {}) {
@@ -51,6 +52,7 @@ export class SqliteMailStore extends EventEmitter {
     this.database ??= new MailDatabase(join(dirname(this.root), 'mail.sqlite'), { retention: this.retention });
     this.imported = await migrateMaildir(this.database, this.root, { log: this.log });
     this.database.retain();
+    this.reconcileDeliverySchedule();
     this.database.on('message', (address) => this.emit('changed', address));
     this.database.on('wake', ({ address, id }) => this.emit('wake', { address, id }));
     this.database.on('subscriptions', (address) => this.emit('subscriptions', address));
@@ -69,6 +71,7 @@ export class SqliteMailStore extends EventEmitter {
     this.owner?.close(); this.owner = null;
   }
   statement(sql) { return this.database.statement(sql); }
+  get changes() { return this.database.changes; }
   transaction(operation) { return this.database.transaction(operation); }
   record(collection, key) { return this.database.record(collection, key); }
   records(collection) { return this.database.records(collection); }
@@ -88,6 +91,29 @@ export class SqliteMailStore extends EventEmitter {
       messages: Object.fromEntries(states.map(({ state, count }) => [state, count])),
       mailbox_count: this.statement('SELECT count(*) AS count FROM mailboxes').get().count,
       pending_deliveries: this.statement('SELECT count(*) AS count FROM deliveries').get().count,
+      delivery_targets: this.statement('SELECT count(*) AS count FROM delivery_targets').get().count,
+      next_delivery: this.nextDeliveryTarget(),
+      change_cursor: this.changes.cursor(),
+      relay_pool: {
+        handles: this.statement("SELECT count(*) AS count FROM mail_records WHERE collection = 'relays' AND key LIKE '%/relay.json' AND json_extract(record, '$.pool') = 1").get().count,
+        unanswered_starts: this.statement("SELECT count(*) AS count FROM mail_records WHERE collection = 'relays' AND key LIKE '%/start.json'").get().count,
+        // These are committed owner handles, not liveness or PID authority.
+        // Let the local operator inspect and close a known runtime through its
+        // owner without reading SQLite or exposing a runtime bearer token.
+        recorded_owners: this.statement(`SELECT key, json_extract(record, '$.runtime_id') AS runtime_id,
+          json_extract(record, '$.connection') AS connection, json_extract(record, '$.pid') AS pid
+          FROM mail_records WHERE collection = 'relays' AND key LIKE '%/relay.json'
+          AND json_extract(record, '$.pool') = 1 ORDER BY key LIMIT 64`).all(),
+        recorded_starts: this.statement(`SELECT key, json_extract(record, '$.name') AS name,
+          json_extract(record, '$.at_ms') AS at_ms, json_extract(record, '$.directory') AS directory
+          FROM mail_records WHERE collection = 'relays' AND key LIKE '%/start.json' ORDER BY key LIMIT 64`).all(),
+        recorded_limit: 64,
+        reply_routes: this.statement("SELECT count(*) AS count FROM mail_records WHERE collection = 'relay_routes'").get().count,
+        ambiguous_routes: this.statement("SELECT count(*) AS count FROM mail_records WHERE collection = 'relay_routes' AND json_extract(record, '$.ambiguous') = 1").get().count,
+        deferred_replies: this.statement("SELECT count(*) AS count FROM mail_records WHERE collection = 'relay_pending'").get().count,
+        undelivered_replies: this.statement("SELECT count(*) AS count FROM messages WHERE json_type(header, '$.pool_reply_undelivered') IS NOT NULL").get().count,
+        pending_writer_notices: this.statement("SELECT count(*) AS count FROM messages WHERE state IN ('new', 'claimed') AND json_type(header, '$.pool_reply_failure_notice') IS NOT NULL").get().count,
+      },
       subscriptions: this.statement('SELECT count(*) AS count FROM subscriptions').get().count,
       carrier_rows: this.statement('SELECT count(*) AS count FROM carrier_log').get().count,
       terminal_log_rows: this.statement('SELECT count(*) AS count FROM terminal_log').get().count,
@@ -130,6 +156,17 @@ export class SqliteMailStore extends EventEmitter {
     return this.statement(`SELECT ${COLUMNS} FROM messages WHERE mailbox = ? AND state IN ('new', 'claimed') ORDER BY created_at_ms, id`).all(address).map(stored);
   }
 
+  hookSummary(address) {
+    const source = `FROM messages m WHERE mailbox = ? AND state IN ('new', 'claimed')
+      AND NOT EXISTS (SELECT 1 FROM deliveries d WHERE d.mailbox = m.mailbox AND d.message = m.id)`;
+    const count = this.statement(`SELECT count(*) AS count ${source}`).get(address).count;
+    const senders = this.statement(`SELECT kind, json_extract(header, '$.from_name') AS name, min(created_at_ms) AS first_at
+      ${source} GROUP BY kind, name ORDER BY first_at, kind, name LIMIT 65`).all(address);
+    const moreSenders = senders.length > 64;
+    if (moreSenders) senders.pop();
+    return { count, senders, moreSenders };
+  }
+
   typedSince(address, since, prefix) {
     return this.statement(`SELECT ${COLUMNS} FROM messages WHERE mailbox = ? AND kind = 'user'
       AND created_at_ms > ? AND id >= ? AND id < ? ORDER BY created_at_ms, id`)
@@ -249,6 +286,15 @@ export class SqliteMailStore extends EventEmitter {
     return { matches, messages: [...members.values()] };
   }
 
+  /** null means an older release expired the body before keeping a digest.
+   * Never compare its retention placeholder as though it were original text. */
+  bodyMatches(address, id, text) {
+    const row = this.statement('SELECT body, body_digest FROM messages WHERE mailbox = ? AND id = ?').get(address, id);
+    if (!row) return null;
+    if (row.body != null) return row.body === text;
+    return row.body_digest ? row.body_digest === createHash('sha256').update(text).digest('hex') : null;
+  }
+
   findIn(address, id) {
     const row = this.statement(`SELECT ${COLUMNS} FROM messages WHERE mailbox = ? AND id = ?`).get(address, id);
     return row ? stored(row) : null;
@@ -298,8 +344,9 @@ export class SqliteMailStore extends EventEmitter {
   }
 
   userTurns(address) {
+    this.#recoverAbandoned(address);
     return this.statement(`SELECT ${COLUMNS} FROM messages WHERE mailbox = ? AND state = 'new' AND kind = 'user'
-      AND COALESCE(json_extract(header, '$.in_reply_to'), '') NOT LIKE 'q-%' ORDER BY created_at_ms, id`)
+      AND COALESCE(json_extract(header, '$.in_reply_to'), '') NOT LIKE 'q-%' ORDER BY created_at_ms, id LIMIT 256`)
       .all(address).map((row) => { const item = stored(row); return { ...item, envelope: this.body(item) }; });
   }
 
@@ -343,11 +390,14 @@ export class SqliteMailStore extends EventEmitter {
       this.open(address);
       const { body, ...header } = envelope;
       const firstLine = Array.from(body.split('\n', 1)[0]).slice(0, 400).join('');
-      const changed = this.statement(`INSERT INTO messages(mailbox, id, thread, sender, kind, subject, created_at_ms, state, header, body, first_line, read_at_ms)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(mailbox, id) DO NOTHING`)
+      const changed = this.statement(`INSERT INTO messages(mailbox, id, thread, sender, kind, subject, created_at_ms, state, header, body, body_digest, first_line, read_at_ms)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(mailbox, id) DO NOTHING`)
         .run(address, envelope.id, envelope.thread ?? envelope.id, envelope.from ?? '', envelope.kind ?? 'peer', envelope.subject ?? null,
-          envelope.created_at_ms, state, JSON.stringify(header), body, firstLine, state === 'cur' ? Date.now() : null).changes;
-      if (changed) this.database.changed('message', address);
+          envelope.created_at_ms, state, JSON.stringify(header), body, createHash('sha256').update(body).digest('hex'), firstLine, state === 'cur' ? Date.now() : null).changes;
+      if (changed) {
+        if (state === 'new' && envelope.kind === 'user' && !String(envelope.in_reply_to ?? '').startsWith('q-')) this.requestWake(address, envelope.id, { keep: true });
+        this.database.changed('message', address);
+      }
       return rowReference(address, envelope.id);
     });
   }
@@ -367,31 +417,127 @@ export class SqliteMailStore extends EventEmitter {
   remove(address, path) {
     const key = reference(path);
     if (!key || key[0] !== address) throw new Error('mail row reference belongs to a different mailbox');
-    const changed = this.statement('DELETE FROM messages WHERE mailbox = ? AND id = ?').run(address, key[1]).changes > 0;
-    if (changed) this.database.changed('message', address);
-    return changed;
+    return this.transaction(() => {
+      const changed = this.statement('DELETE FROM messages WHERE mailbox = ? AND id = ?').run(address, key[1]).changes > 0;
+      if (changed) {
+        this.refreshDeliveryTarget(...this.#deliveryTarget(address));
+        this.database.changed('message', address);
+        this.database.changed('wake', { address, id: key[1] });
+      }
+      return changed;
+    });
   }
 
   requestWake(address, id, { keep = false } = {}) {
     if (!validId(id)) throw new Error('invalid wake message id');
     this.open(address);
-    const sql = `INSERT INTO deliveries(mailbox, message, due_at_ms, attempts, record) VALUES (?, ?, 0, 0, '{}')
+    const sql = `INSERT INTO deliveries(mailbox, message, machine, due_at_ms, attempts, record) VALUES (?, ?, ?, 0, 0, '{}')
       ON CONFLICT(mailbox, message) ${keep ? 'DO NOTHING' : "DO UPDATE SET due_at_ms = 0, attempts = 0, record = '{}'"}`;
-    if (this.statement(sql).run(address, id).changes) this.database.changed('wake', { address, id });
+    this.transaction(() => {
+      if (this.statement(sql).run(address, id, address.split(':')[1]).changes) {
+        this.refreshDeliveryTarget(...this.#deliveryTarget(address));
+        this.database.changed('wake', { address, id });
+      }
+    });
   }
   wakes(address) { return this.statement('SELECT message FROM deliveries WHERE mailbox = ? ORDER BY due_at_ms, message').all(address).map((row) => row.message); }
   allWakes() { return this.statement('SELECT mailbox AS address, message AS id FROM deliveries ORDER BY due_at_ms, mailbox, message').all(); }
   hasWake(address, id) { return validId(id) && Boolean(this.statement('SELECT 1 FROM deliveries WHERE mailbox = ? AND message = ?').get(address, id)); }
-  removeWake(address, id) { return this.statement('DELETE FROM deliveries WHERE mailbox = ? AND message = ?').run(address, id).changes > 0; }
+  removeWake(address, id) {
+    return this.transaction(() => {
+      const changed = this.statement('DELETE FROM deliveries WHERE mailbox = ? AND message = ?').run(address, id).changes > 0;
+      if (changed) {
+        this.refreshDeliveryTarget(...this.#deliveryTarget(address));
+        this.database.changed('wake', { address, id });
+      }
+      return changed;
+    });
+  }
   wakeState(address, id) {
     const row = this.statement('SELECT record FROM deliveries WHERE mailbox = ? AND message = ?').get(address, id);
     return { attempts: 0, next_at_ms: 0, last_error: null, ...(row ? JSON.parse(row.record) : {}) };
   }
   setWakeState(address, id, state) {
     const record = { attempts: state.attempts ?? 0, next_at_ms: state.next_at_ms ?? 0, last_error: state.last_error ?? null,
-      handover_at_ms: state.handover_at_ms ?? null, failing_since_ms: state.failing_since_ms ?? null };
-    if (this.statement('UPDATE deliveries SET record = ?, due_at_ms = ?, attempts = ? WHERE mailbox = ? AND message = ?')
-      .run(JSON.stringify(record), record.next_at_ms, record.attempts, address, id).changes) this.database.changed('wake', { address, id });
+      handover_at_ms: state.handover_at_ms ?? null, failing_since_ms: state.failing_since_ms ?? null, waiting_for_idle: state.waiting_for_idle === true };
+    this.transaction(() => {
+      if (this.statement('UPDATE deliveries SET record = ?, due_at_ms = ?, attempts = ? WHERE mailbox = ? AND message = ?')
+        .run(JSON.stringify(record), record.next_at_ms, record.attempts, address, id).changes) {
+        this.refreshDeliveryTarget(...this.#deliveryTarget(address));
+        this.database.changed('wake', { address, id });
+      }
+    });
+  }
+
+  #deliveryTarget(address) {
+    const machine = address.split(':')[1];
+    return machine === this.machine ? ['mailbox', address] : ['machine', machine];
+  }
+
+  /** One indexed row per local mailbox or remote machine. Offline probing is
+   * a machine deadline, independent of each message's retry deadline. */
+  refreshDeliveryTarget(kind, target) {
+    const row = kind === 'mailbox'
+      ? this.statement('SELECT min(due_at_ms) AS due FROM deliveries WHERE mailbox = ?').get(target)
+      : this.statement('SELECT min(due_at_ms) AS due FROM deliveries WHERE machine = ?').get(target);
+    if (row.due == null) {
+      this.statement('DELETE FROM delivery_targets WHERE kind = ? AND target = ?').run(kind, target);
+      return;
+    }
+    const wait = kind === 'machine' ? this.machineWait(target) : null;
+    const due = wait ? Math.min(wait.next_probe_ms ?? 0, (wait.offline_since_ms ?? Date.now()) + OFFLINE_CEILING_MS) : row.due;
+    this.statement(`INSERT INTO delivery_targets(kind, target, due_at_ms) VALUES (?, ?, ?)
+      ON CONFLICT(kind, target) DO UPDATE SET due_at_ms = excluded.due_at_ms`).run(kind, target, due);
+  }
+
+  /** Startup and the five-minute backstop rebuild derived schedule rows. Normal
+   * writes refresh only the target they changed, in the same transaction. */
+  reconcileDeliverySchedule() {
+    this.transaction(() => {
+      this.statement("UPDATE deliveries SET machine = substr(mailbox, 4, instr(substr(mailbox, 4), ':') - 1) WHERE machine = ''").run();
+      this.statement(`INSERT INTO deliveries(mailbox, message, machine, due_at_ms, attempts, record)
+        SELECT mailbox, id, ?, 0, 0, '{}' FROM messages WHERE kind = 'user' AND state IN ('new', 'claimed')
+        AND substr(mailbox, 4, instr(substr(mailbox, 4), ':') - 1) = ? AND COALESCE(json_extract(header, '$.in_reply_to'), '') NOT LIKE 'q-%'
+        ON CONFLICT(mailbox, message) DO NOTHING`).run(this.machine, this.machine);
+      this.statement('DELETE FROM delivery_targets').run();
+      const targets = this.statement(`SELECT DISTINCT 'mailbox' AS kind, mailbox AS target FROM deliveries WHERE machine = ?
+        UNION SELECT DISTINCT 'machine', machine FROM deliveries WHERE machine <> ?`).all(this.machine, this.machine);
+      for (const { kind, target } of targets) this.refreshDeliveryTarget(kind, target);
+    });
+  }
+
+  /** A live idle transition wakes only the native waits for those addresses. */
+  wakeIdleSessions(addresses) {
+    this.transaction(() => {
+      const changed = this.statement(`UPDATE deliveries SET due_at_ms = 0, record = json_set(record, '$.next_at_ms', 0)
+        WHERE mailbox IN (SELECT value FROM json_each(?)) AND json_extract(record, '$.waiting_for_idle') = 1
+        RETURNING mailbox`).all(JSON.stringify(addresses));
+      for (const address of new Set(changed.map((row) => row.mailbox))) this.refreshDeliveryTarget('mailbox', address);
+      if (changed.length) this.database.changed('wake', {});
+    });
+  }
+
+  nextDeliveryTarget(excluded = []) {
+    return this.statement(`SELECT t.kind, t.target, t.due_at_ms FROM delivery_targets t
+      WHERE NOT EXISTS (SELECT 1 FROM json_each(?) e WHERE json_extract(e.value, '$.kind') = t.kind AND json_extract(e.value, '$.target') = t.target)
+      ORDER BY t.due_at_ms, t.kind, t.target LIMIT 1`).get(JSON.stringify(excluded)) ?? null;
+  }
+
+  delayDeliveryTarget(kind, target, until) {
+    this.statement('UPDATE delivery_targets SET due_at_ms = max(due_at_ms, ?) WHERE kind = ? AND target = ?').run(until, kind, target);
+  }
+
+  dueWakes(address, now, limit = 256) {
+    return this.statement(`SELECT message FROM deliveries WHERE mailbox = ? AND due_at_ms <= ?
+      ORDER BY due_at_ms, message LIMIT ?`).all(address, now, limit).map(({ message }) => message);
+  }
+
+  remoteWakes(machine, now, { probing = false, limit = 256 } = {}) {
+    const sql = `SELECT d.mailbox AS address, d.message AS id, m.created_at_ms AS filed
+      FROM deliveries d JOIN messages m ON m.mailbox = d.mailbox AND m.id = d.message
+      WHERE d.machine = ? ${probing ? '' : 'AND d.due_at_ms <= ?'}
+      ORDER BY ${probing ? '' : 'd.due_at_ms, '}m.created_at_ms, d.mailbox, d.message LIMIT ?`;
+    return this.statement(sql).all(machine, ...(probing ? [] : [now]), limit);
   }
 
   expire(address, id, expiry) {
@@ -404,8 +550,12 @@ export class SqliteMailStore extends EventEmitter {
   #machineKey(machine) { return `${String(machine).replace(/[^A-Za-z0-9._-]/g, '_')}.json`; }
   machineWait(machine) { return this.record('machine_waits', this.#machineKey(machine)); }
   setMachineWait(machine, wait) {
-    if (wait) this.putRecord('machine_waits', this.#machineKey(machine), wait);
-    else this.removeRecord('machine_waits', this.#machineKey(machine));
+    this.transaction(() => {
+      if (wait) this.putRecord('machine_waits', this.#machineKey(machine), wait);
+      else this.removeRecord('machine_waits', this.#machineKey(machine));
+      this.refreshDeliveryTarget('machine', machine);
+      this.database.changed('wake', { machine });
+    });
   }
   carrierLine(line) { this.database.appendCarrier(line); }
   carrierLines(limit = 1000) {

package/mail/sqlite.mjs

@@ -1,11 +1,14 @@
 // The machine daemon's mail database. Only its owner opens a writable handle;
 // clients use the machine socket. This module contains no harness or delivery IO.
 import { EventEmitter } from 'node:events';
+import { createHash } from 'node:crypto';
 import { createRequire } from 'node:module';
 import { chmodSync, closeSync, mkdirSync, openSync } from 'node:fs';
 import { dirname } from 'node:path';
+import { CHANGE_SCHEMA, ChangeStream } from './change-stream.mjs';
+import { RECORD_SCHEMA } from '../record-state.mjs';
 
-export const MAIL_SCHEMA_VERSION = 2;
+export const MAIL_SCHEMA_VERSION = 3;
 export const MAIL_RETENTION = Object.freeze({
   bodyDays: 30,
   maxReadHeaders: 1_000_000,
@@ -34,6 +37,7 @@ CREATE TABLE IF NOT EXISTS messages (
   state TEXT NOT NULL CHECK (state IN ('new', 'claimed', 'cur')),
   header TEXT NOT NULL CHECK (json_valid(header)),
   body TEXT,
+  body_digest TEXT,
   first_line TEXT NOT NULL,
   body_expired_at_ms INTEGER,
   read_at_ms INTEGER,
@@ -62,6 +66,7 @@ CREATE TABLE IF NOT EXISTS claims (
 CREATE TABLE IF NOT EXISTS deliveries (
   mailbox TEXT NOT NULL,
   message TEXT NOT NULL,
+  machine TEXT NOT NULL DEFAULT '',
   due_at_ms INTEGER NOT NULL,
   attempts INTEGER NOT NULL,
   record TEXT NOT NULL CHECK (json_valid(record)),
@@ -69,6 +74,15 @@ CREATE TABLE IF NOT EXISTS deliveries (
   FOREIGN KEY (mailbox, message) REFERENCES messages(mailbox, id) ON DELETE CASCADE
 ) STRICT;
 CREATE INDEX IF NOT EXISTS deliveries_due ON deliveries(due_at_ms, mailbox, message);
+CREATE INDEX IF NOT EXISTS deliveries_machine_due ON deliveries(machine, due_at_ms, mailbox, message);
+CREATE INDEX IF NOT EXISTS deliveries_mailbox_due ON deliveries(mailbox, due_at_ms, message);
+CREATE TABLE IF NOT EXISTS delivery_targets (
+  kind TEXT NOT NULL CHECK (kind IN ('mailbox', 'machine')),
+  target TEXT NOT NULL,
+  due_at_ms INTEGER NOT NULL,
+  PRIMARY KEY (kind, target)
+) STRICT;
+CREATE INDEX IF NOT EXISTS delivery_targets_due ON delivery_targets(due_at_ms, kind, target);
 CREATE TABLE IF NOT EXISTS subscriptions (
   mailbox TEXT NOT NULL,
   message TEXT NOT NULL,
@@ -183,20 +197,30 @@ export class MailDatabase extends EventEmitter {
       if (tables.some(({ name }) => name === 'mail_metadata')) {
         const stored = this.connection.prepare('SELECT value FROM mail_metadata WHERE key = ?').get('schema_version');
         storedVersion = stored ? JSON.parse(stored.value) : null;
-        if (storedVersion !== 1 && storedVersion !== MAIL_SCHEMA_VERSION) throw new Error(`mail schema ${storedVersion} is not supported by this release`);
+        if (![1, 2, MAIL_SCHEMA_VERSION].includes(storedVersion)) throw new Error(`mail schema ${storedVersion} is not supported by this release`);
       } else if (tables.length) throw new Error(`the database at ${path} has no mail schema receipt; its schema has not been changed`);
       const mode = this.connection.prepare('PRAGMA journal_mode = WAL').get();
       if (String(mode.journal_mode).toLowerCase() !== 'wal') throw new Error(`mail database cannot use WAL at ${path}`);
       this.connection.exec('PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 2000;');
+      this.connection.function('supercode_sha256', { deterministic: true }, (body) => typeof body === 'string' ? createHash('sha256').update(body).digest('hex') : null);
       this.transaction(() => {
+        if (storedVersion != null && !this.connection.prepare('PRAGMA table_info(messages)').all().some(({ name }) => name === 'body_digest')) {
+          this.connection.exec('ALTER TABLE messages ADD COLUMN body_digest TEXT;');
+        }
+        if ([1, 2].includes(storedVersion) && !this.connection.prepare('PRAGMA table_info(deliveries)').all().some(({ name }) => name === 'machine')) {
+          this.connection.exec("ALTER TABLE deliveries ADD COLUMN machine TEXT NOT NULL DEFAULT '';");
+        }
         this.connection.exec(SCHEMA);
         // The private first candidate stored only the four legacy send fields.
         // Add the complete record before publishing the upgraded schema receipt.
         if (storedVersion === 1 && !this.connection.prepare('PRAGMA table_info(sent_messages)').all().some(({ name }) => name === 'record')) {
           this.connection.exec("ALTER TABLE sent_messages ADD COLUMN record TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(record) AND json_type(record) = 'object');");
         }
+        this.connection.exec(RECORD_SCHEMA);
+        this.connection.exec(CHANGE_SCHEMA);
         this.setMetadata('schema_version', MAIL_SCHEMA_VERSION);
       });
+      this.changes = new ChangeStream(this);
     } catch (error) {
       this.connection.close();
       throw error;
@@ -205,7 +229,18 @@ export class MailDatabase extends EventEmitter {
 
   statement(sql) {
     if (this.closed) throw new Error('the mail database is closed; no write or native action is resumed');
-    if (!this.statements.has(sql)) this.statements.set(sql, this.connection.prepare(sql));
+    if (!this.statements.has(sql)) {
+      const statement = this.connection.prepare(sql);
+      this.statements.set(sql, new Proxy(statement, { get: (target, key) => {
+        if (key === 'run') return (...args) => {
+          const value = target.run(...args);
+          if (!this.depth && value.changes) this.#notify('commit', null);
+          return value;
+        };
+        const value = target[key];
+        return typeof value === 'function' ? value.bind(target) : value;
+      } }));
+    }
     return this.statements.get(sql);
   }
 
@@ -230,6 +265,7 @@ export class MailDatabase extends EventEmitter {
     if (!depth) {
       const events = this.pending.splice(0);
       for (const event of events) this.#notify(event.topic, event.key);
+      this.#notify('commit', null);
     }
     return value;
   }
@@ -241,6 +277,7 @@ export class MailDatabase extends EventEmitter {
   }
 
   #notify(topic, key) {
+    if (!this.listenerCount(topic)) return;
     // A caller may set the delivery's grace deadline immediately after creating
     // its wake. Notify after that write, as the former filesystem event did.
     setImmediate(() => { if (!this.closed) this.emit(topic, key); });
@@ -299,7 +336,7 @@ export class MailDatabase extends EventEmitter {
   retain(now = Date.now()) {
     const cutoff = now - this.retention.bodyDays * 86_400_000;
     return this.transaction(() => {
-      const expired = this.statement(`UPDATE messages SET body = NULL, first_line = '', body_expired_at_ms = ?
+      const expired = this.statement(`UPDATE messages SET body_digest = coalesce(body_digest, supercode_sha256(body)), body = NULL, first_line = '', body_expired_at_ms = ?
         WHERE state = 'cur' AND created_at_ms < ? AND body IS NOT NULL
         AND NOT EXISTS (SELECT 1 FROM deliveries d WHERE d.mailbox = messages.mailbox AND d.message = messages.id)
         AND NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.mailbox = messages.mailbox AND s.message = messages.id)`)
@@ -345,6 +382,7 @@ export class MailDatabase extends EventEmitter {
 
   close() {
     if (this.closed) return;
+    this.changes?.stop();
     this.closed = true;
     this.removeAllListeners();
     this.statements.clear();