@volter/supercode-teams 0.3.98 → 0.3.99

package/machine.mjs

@@ -30,6 +30,7 @@ import { spawn, spawnSync, execFile } from 'node:child_process';
 import { EventEmitter } from 'node:events';
 import { appendFileSync, readFileSync, writeFileSync, writeSync, statSync, readdirSync, mkdirSync, renameSync, rmSync, existsSync, openSync, readSync, closeSync, accessSync, constants } from 'node:fs';
 import { basename, join, resolve, dirname, isAbsolute } from 'node:path';
+import { open as openAsync, readdir as readdirAsync } from 'node:fs/promises';
 import { askBoardOwner, BOARD_OPERATION_VERBS, boardHome } from '@volter/teams/board-owner';
 import { openMachine, normalMachineName } from '@volter/teams/machine-door';
 import { RecordHolder, RecordProducer, RECENT_PER_MAILBOX, RECORDS_NOT_GRANTED } from './machine-records.mjs';
@@ -153,6 +154,31 @@ const LAUNCH_SETTLE_MS = 500;
 /** How long a send to another machine waits for its mail door's answer (its receiver's door there: a relay's receipt
  *  alone may take 90 s). An answer that does not come is unanswered, never refused or failed. */
 const MAIL_SEND_MS = 120_000;
+
+/** How much of a transcript's end is read for a message that just landed. */
+const LANDING_TAIL_BYTES = 4 * 1024 * 1024;
+
+/**
+ * Whether Claude session `sessionId`'s transcript (`<projects>/<folder>/<id>.jsonl`, in any of `projectsDirs`) holds
+ * `landed` in its last LANDING_TAIL_BYTES: a relay's send whose receipt was lost is read where it lands. Read apart from
+ * the daemon's loop (it runs once per unconfirmed send).
+ */
+async function claudeTranscriptHolds(projectsDirs, sessionId, landed) {
+  for (const projects of projectsDirs) {
+    let folders = [];
+    try { folders = await readdirAsync(projects); } catch { continue; }
+    for (const folder of folders) {
+      let file;
+      try { file = await openAsync(join(projects, folder, `${sessionId}.jsonl`), 'r'); } catch { continue; }
+      try {
+        const { size } = await file.stat(), length = Math.min(size, LANDING_TAIL_BYTES);
+        const { buffer } = await file.read(Buffer.alloc(length), 0, length, size - length);
+        if (buffer.toString('utf8').includes(landed)) return true;
+      } finally { await file.close(); }
+    }
+  }
+  return false;
+}
 /**
  * The doors whose work waits on a session, with the bound each answers within (the node door's own is 30 s): a send or
  * a hand-over waits on its receiver's door (a relay's receipt, up to 90 s; another machine's door, MAIL_SEND_MS), a
@@ -682,6 +708,7 @@ export class TeamsMachine extends MachineCore {
       runtimeOf: (harness, id) => live.runtimeReceipt(harness, id), runtimeNamed: (receiptId) => live.receiptNamed(receiptId),
       runtimesIn: (harness, cwd) => live.runtimesIn(harness, cwd),
       endpointOf: async () => (await this.relayEndpoint)?.url ?? null,
+      arrived: (sessionId, landed) => claudeTranscriptHolds([...live.claudeDirs.keys()].map((dir) => join(dirname(dir), 'projects')), sessionId, landed),
       log: (line) => this.emit('mail-log', line) });
     this.mailSend = new MailSend({ machine, mailRoot, live, store: this.mailStore, agents: this.agents, relays: this.relays,
       // a send waits on its receiver's door there (a relay's receipt takes up to 90 s), never past MAIL_SEND_MS
@@ -1115,7 +1142,7 @@ export class TeamsMachine extends MachineCore {
       this.internalRequests.set(id, { resolve, reject });
       if (!this.#toServe(`${JSON.stringify({ jsonrpc: '2.0', id: joinId(INTERNAL_CONNECTION, id), method, params })}\n`)) {
         this.internalRequests.delete(id);
-        return reject(new Error('harness serve is not running'));
+        return reject(Object.assign(new Error('harness serve is not running'), { notAsked: true }));
       }
       setTimeout(() => { if (this.internalRequests.delete(id)) reject(new Error(`${method} timed out`)); }, timeoutMs).unref();
     });
@@ -1145,7 +1172,8 @@ export class TeamsMachine extends MachineCore {
             if (typeof value === 'string') { if (!this.routes.has(value)) this.routes.set(value, new Set()); this.routes.get(value).add(INTERNAL_CONNECTION); }
           }
         }
-        if (message.error) pending.reject(new Error(message.error.message ?? 'harness serve refused'));
+        // an answer that is an error is marked answered: a call refused, not one whose outcome is unknown
+        if (message.error) pending.reject(Object.assign(new Error(message.error.message ?? 'harness serve refused'), { answered: true, code: message.error.code ?? null }));
         else pending.resolve(message.result);
         return;
       }

package/mail/relay.mjs

@@ -10,7 +10,7 @@
 import { mkdirSync, readFileSync, readdirSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
 import { connect } from 'node:net';
 import { join } from 'node:path';
-import { blake3Hex } from './envelope.mjs';
+import { blake3Hex, shortId } from './envelope.mjs';
 
 /** Model name a relay's requests carry; the relay endpoint answers them without any model. */
 const RELAY_MODEL = 'haiku';
@@ -18,6 +18,26 @@ const RELAY_MODEL = 'haiku';
 export const RELAY_NAME_PREFIX = 'sc-';
 /** How long a send waits for Claude's own receipt. */
 const SEND_TIMEOUT_MS = 90_000;
+
+/** How long an unconfirmed send is given to land in its receiver's transcript before that is read: a SendMessage that
+ *  passed the gate just before the send ended lands within it. */
+const LANDING_MS = 2000;
+
+/** How long a hand-in waits for its relay's last turn to end: a relay's receipt is written mid-turn (at its
+ *  SendMessage), so the next send through it may find that turn still finishing. Past this the turn is stuck. */
+const TURN_END_MS = 15_000;
+
+/** A relay close's bound: over harness serve's own (RUNTIME_CONTROL_DEADLINE, 30 s), past which serve ends the
+ *  runtime's process group itself, so a relay stuck in a turn is closed through serve, never left running. */
+const CLOSE_MS = 35_000;
+
+/**
+ * Whether harness serve's error for a relay's input leaves its outcome unknown: no answer (its bound passed, serve ended
+ * mid-call) or serve's own deadline on the runtime ("gave up after", harness_service.rs `within_control_deadline`).
+ * Any other answer is a refusal of the input (a turn in progress, an unknown or closed connection), and a call never
+ * made (no serve to ask) was not handed in either.
+ */
+const answerLost = (error) => !error.notAsked && (!error.answered || /gave up after \d+s/.test(error.message));
 /** A relay send slower than this says where its time went. */
 const SLOW_SEND_MS = 2000;
 /** Largest message relayed into a Claude session: the relay copies it into a model turn byte for byte. */
@@ -68,8 +88,8 @@ export class Relays {
    * id)` the live-runtime receipt the live-session map holds for a runtime id (its file read whole), or null;
    * `runtimeNamed(receiptId)` one receipt read by its own name.
    */
-  constructor({ mailRoot, machine, supercodeBin, serve, runtimeOf, runtimeNamed = () => null, runtimesIn = () => [], endpointOf = async () => null, log = () => {} }) {
-    Object.assign(this, { mailRoot, machine, supercodeBin, serve, runtimeOf, runtimeNamed, runtimesIn, endpointOf, log });
+  constructor({ mailRoot, machine, supercodeBin, serve, runtimeOf, runtimeNamed = () => null, runtimesIn = () => [], endpointOf = async () => null, arrived = async () => false, log = () => {} }) {
+    Object.assign(this, { mailRoot, machine, supercodeBin, serve, runtimeOf, runtimeNamed, runtimesIn, endpointOf, arrived, log });
     this.lost = new Map(); // relay folder → when a start of its runtime lost its answer (the runtime may run, unnamed)
     this.remoteKept = new Map(); // relay folder of a sender on another machine → the timer that retires it
     this.sending = new Map(); // relay folder → the send in progress through it (one at a time per relay)
@@ -86,7 +106,7 @@ export class Relays {
       this.running.delete(directory);
       // closed under the relay's own lock: a send to it that arrives meanwhile waits for the close, then starts it
       // again, never handing its turn to a runtime that is closing
-      void this.#locked(directory, () => this.serve('harness.v1.runtimes.close', { connection: relay.connection, runtime_id: relay.runtime_id }, 15_000))
+      void this.#locked(directory, () => this.serve('harness.v1.runtimes.close', { connection: relay.connection, runtime_id: relay.runtime_id }, CLOSE_MS))
         .catch((error) => {
           // still running, so still counted: the next cap closes it again (unless a send started it anew meanwhile)
           if (!this.running.has(directory)) this.running.set(directory, relay);
@@ -102,7 +122,7 @@ export class Relays {
       for (const ids of this.runtimesIn('claude-code', directory)) {
         if (ids.includes(named)) continue;
         void this.#locked(directory, async () => {
-          for (const id of ids) { try { await this.serve('harness.v1.runtimes.close', { runtime_id: id }, 15_000); return; } catch { /* the other id */ } }
+          for (const id of ids) { try { await this.serve('harness.v1.runtimes.close', { runtime_id: id }, CLOSE_MS); return; } catch { /* the other id */ } }
           this.log(`relay ${directory.split(/[\\/]/).pop()}: a runtime its lost start left running was not closed`);
         });
       }
@@ -135,7 +155,7 @@ export class Relays {
       if (expected && this.#representedOf(directory) !== expected) return false;
       const record = readJson(join(directory, 'relay.json'));
       if (record?.runtime_id) {
-        try { await this.serve('harness.v1.runtimes.close', { connection: record.connection ?? '', runtime_id: record.runtime_id }, 15_000); }
+        try { await this.serve('harness.v1.runtimes.close', { connection: record.connection ?? '', runtime_id: record.runtime_id }, CLOSE_MS); }
         catch (error) {
           // a runtime still live that could not be closed keeps its folder: it is retired again at the next end or start
           if (this.runtimeOf('claude-code', record.runtime_id)) { this.log(`relay ${directory.split(/[\\/]/).pop()} was not retired: ${error.message}`); return false; }
@@ -244,7 +264,7 @@ export class Relays {
 
   /** The relay's runtime, started in this daemon's harness serve when it is not running (claude_relay.rs
    *  `ensure_relay_runtime`): `{ runtime (its receipt), connection }`. */
-  async #ensure(spec, step = () => {}) {
+  async #ensure(spec, step = () => {}, { stale = false } = {}) {
     const endpoint = await this.endpoint();
     step('relay endpoint');
     if (!endpoint) throw new Error('the relay endpoint is not served by this machine\'s daemon (its log says why); a Claude session can be sent mail once the daemon starts again');
@@ -254,15 +274,16 @@ export class Relays {
     const record = readJson(spec.record);
     if (record?.runtime_id) {
       const runtime = this.runtimeOf('claude-code', record.runtime_id);
-      if (runtime && alive(this.#pid(spec, runtime)) && record.connection && (record.endpoint === endpoint || await answers(Number(String(record.endpoint ?? '').split(':').pop())))) {
+      if (!stale && runtime && alive(this.#pid(spec, runtime)) && record.connection && (record.endpoint === endpoint || await answers(Number(String(record.endpoint ?? '').split(':').pop())))) {
         step('runtime reused');
         return { runtime, connection: record.connection };
       }
       // why it is not reused, said in the timing line: a relay that should have been warm and was not shows here
-      step(`runtime not reused (${!runtime ? 'no live receipt' : !alive(this.#pid(spec, runtime)) ? 'its process ended' : !record.connection ? 'no connection' : 'endpoint moved'})`);
+      step(`runtime not reused (${stale ? 'it held a stale turn' : !runtime ? 'no live receipt' : !alive(this.#pid(spec, runtime)) ? 'its process ended' : !record.connection ? 'no connection' : 'endpoint moved'})`);
       // closed through the owner's door even after its receipt went
-      try { await this.serve('harness.v1.runtimes.close', { connection: record.connection, runtime_id: record.runtime_id }, 15_000); }
+      try { await this.serve('harness.v1.runtimes.close', { connection: record.connection, runtime_id: record.runtime_id }, CLOSE_MS); }
       catch (error) { if (this.runtimeOf('claude-code', record.runtime_id)) throw new Error(`could not retire the old relay: ${error.message}`); }
+      step('old runtime closed (runtimes.close)');
     }
     const key = spec.directory.split(/[\\/]/).pop();
     const started = await this.#start(spec, {
@@ -297,7 +318,7 @@ export class Relays {
     if (!runtime) {
       // a runtime no send can use is closed, never left running untracked
       this.running.delete(spec.directory);
-      try { await this.serve('harness.v1.runtimes.close', { connection, runtime_id: runtimeId }, 15_000); } catch { /* said below */ }
+      try { await this.serve('harness.v1.runtimes.close', { connection, runtime_id: runtimeId }, CLOSE_MS); } catch { /* said below */ }
       throw new Error('the relay started but registered no live runtime; it was closed');
     }
     return { runtime, connection };
@@ -312,11 +333,12 @@ export class Relays {
       const check = () => {
         const value = readJson(spec.receipt);
         if (value) return finish(readReceipt(value));
-        if (pid && !alive(pid)) finish({ delivered: false, detail: 'the Claude relay process exited before confirming the send' });
+        // not known either way: it may have sent before it ended
+        if (pid && !alive(pid)) finish({ delivered: false, unknown: true, detail: 'the Claude relay process exited before confirming the send' });
       };
       try { watcher = watch(spec.directory, () => check()); watcher.on('error', () => {}); } catch { /* checked on the timer */ }
       timer = setInterval(check, 500);
-      deadline = setTimeout(() => finish({ delivered: false, detail: `the Claude relay did not confirm the send within ${SEND_TIMEOUT_MS / 1000} seconds; it may still arrive` }), SEND_TIMEOUT_MS);
+      deadline = setTimeout(() => finish({ delivered: false, unknown: true, detail: `the Claude relay did not confirm the send within ${SEND_TIMEOUT_MS / 1000} seconds; it may still arrive` }), SEND_TIMEOUT_MS);
       check();
     });
   }
@@ -329,14 +351,16 @@ export class Relays {
   async send(sender, senderName, receiver, message, messageId) {
     const spec = this.spec(sender, senderName);
     const asked = performance.now();
-    try { return await this.#locked(spec.directory, () => this.#send(spec, receiver, message, messageId, asked)); }
+    // the message as its receiver's transcript holds it once delivered (envelope.mjs render, its tag escaped there)
+    const landed = `cross-session-message id=\\"${shortId(messageId)}\\" from=\\"${sender}\\"`;
+    try { return await this.#locked(spec.directory, () => this.#send(spec, receiver, message, messageId, asked, landed)); }
     finally {
       if (String(sender).split(':')[1] !== this.machine) this.#keepRemote(spec.directory);
       this.#cap();
     }
   }
 
-  async #send(spec, receiver, message, messageId, asked) {
+  async #send(spec, receiver, message, messageId, asked, landed) {
     try {
       // made under its lock: a retire that held the lock meanwhile may have removed it
       mkdirSync(spec.directory, { recursive: true });
@@ -351,8 +375,30 @@ export class Relays {
       writeFileSync(spec.queue, JSON.stringify(queued));
       let receipt;
       for (let attempt = 0; ; attempt++) {
-        try { await this.serve('harness.v1.runtimes.send_input', { connection, text: sendTurn(queued) }, 30_000); step('turn handed in (send_input)'); receipt = await this.#receipt(spec, this.#pid(spec, runtime)); step('Claude\'s receipt'); }
-        catch (error) { receipt = { delivered: false, detail: `could not reach the Claude relay: ${error.message}` }; }
+        // A hand-in whose answer did not come (its bound passed, serve ended mid-call, serve's own deadline on the
+        // runtime) may still have reached the relay, which then sends: its receipt is waited on as after any hand-in. A
+        // hand-in refused (an answer that is an error) or never asked (no serve) did not reach it, and is said at once,
+        // never held for a receipt that cannot come.
+        let handed = true;
+        const error = await this.#handIn(connection, queued, step);
+        if (!error) step('turn handed in (send_input)');
+        else if (answerLost(error)) step(`turn handed in, its answer lost (send_input: ${error.message})`);
+        else {
+          handed = false;
+          // still in a turn after TURN_END_MS while this send holds its lock: a turn of a send that already ended (its
+          // queue gone, so its gate refuses whatever that turn tries), stuck. The relay is closed through serve and
+          // started again, once
+          if (attempt === 0 && /already in progress/.test(error.message)) {
+            step(`relay still in a turn after ${TURN_END_MS / 1000} s, closed (${error.message})`);
+            ({ runtime, connection } = await this.#ensure(spec, step, { stale: true }));
+            this.#used(spec, connection);
+            rmSync(spec.receipt, { force: true });
+            continue;
+          }
+          receipt = { delivered: false, detail: `the Claude relay did not take the send: ${error.message}` };
+        }
+        if (!handed) break;
+        receipt = await this.#receipt(spec, this.#pid(spec, runtime)); step('Claude\'s receipt');
         const pid = this.#pid(spec, runtime);
         // only a relay proven dead is started again; one alive but slow may still deliver
         if (receipt.delivered || attempt === 1 || pid == null || alive(pid)) break;
@@ -360,6 +406,16 @@ export class Relays {
         rmSync(spec.receipt, { force: true });
       }
       rmSync(spec.queue, { force: true });
+      // an outcome not confirmed is read where it would have landed: the relay can no longer send it (its queue is
+      // gone), so the receiver's transcript says whether it did
+      if (!receipt.delivered && receipt.unknown) {
+        await new Promise((resolve) => setTimeout(resolve, LANDING_MS));
+        let found = false;
+        try { found = await this.arrived(receiver.sessionId, landed); } catch { /* unread: unknown */ }
+        step(found ? 'found in its receiver\'s transcript' : 'not in its receiver\'s transcript');
+        if (found) receipt = { delivered: true };
+        else receipt = { ...receipt, landed: false };
+      }
       if (receipt.delivered) {
         const sent = readJson(spec.sent) ?? {};
         sent[receiver.sessionId] = messageId;
@@ -374,6 +430,25 @@ export class Relays {
     }
   }
 
+  /**
+   * Hand the send's turn to the relay (`runtimes.send_input`): null once taken, else the error. A relay still in a turn
+   * (its last send's: its receipt is written mid-turn, the turn ends a moment later) is asked again every 250 ms for up
+   * to TURN_END_MS; the error is then that refusal.
+   */
+  async #handIn(connection, queued, step) {
+    const asked = performance.now();
+    for (;;) {
+      try { await this.serve('harness.v1.runtimes.send_input', { connection, text: sendTurn(queued) }, 30_000); }
+      catch (error) {
+        if (answerLost(error) || !/already in progress/.test(error.message) || performance.now() - asked > TURN_END_MS) return error;
+        await new Promise((resolve) => setTimeout(resolve, 250));
+        continue;
+      }
+      if (performance.now() - asked > 250) step(`waited for its last turn to end`);
+      return null;
+    }
+  }
+
   /** `runtimes.start` for a relay: a start that fails (its answer lost, or late past its bound) may still have left a
    *  runtime running, so its folder is looked at by the next caps (#cap). */
   async #start(spec, params, timeoutMs) {

package/mail/send.mjs

@@ -254,6 +254,8 @@ export class MailSend {
     catch (error) {
       if (error instanceof Refused && error.kind === 'queue') return outcome(EXIT_REFUSED, `Not sent: ${name} is idle, and a Claude session always starts a turn when a message arrives, so --queue cannot hold it. Send without --queue to wake it.`);
       if (error instanceof Refused) return outcome(EXIT_REFUSED, `Not sent: the message is ${error.bytes} bytes; the limit is ${MAX_RELAYED_BYTES}. Write it to a file and send its path instead.`);
+      // may have arrived: kept filed, and said unanswered with its id (sent again with the same id, nothing is sent twice)
+      if (error.unanswered) { this.#recordSend(caller.address, address, messageId, sentIn); return outcome(EXIT_UNANSWERED, unansweredText(`${name}'s Claude relay`, error.message, messageId)); }
       return outcome(EXIT_FAILED, `Not sent to ${name}: ${error.message}`);
     }
     const [code, what] = {
@@ -321,7 +323,17 @@ export class MailSend {
       const filedHere = this.store.findIn(to, handed.id) == null;
       const filed = this.store.deliverRead(to, handed);
       const relayed = await this.relays.send(handed.from, handed.from_name, { sessionId: door.session.address.split(':').slice(3).join(':'), name: door.session.name.split('@')[0], socket: door.session.socket }, text, handed.id);
-      if (!relayed.delivered) { if (filedHere) { try { unlinkSync(filed); } catch { /* gone */ } } throw new Error(relayed.detail); }
+      // a send the relay may have made keeps its copy and is unanswered, never taken back and said not sent; only one the
+      // relay is known not to have made (refused, never handed in) takes its copy back. One its receiver read from its
+      // mailbox meanwhile (its recipient claim) has reached it. Otherwise the relay can no longer make it (its queue is
+      // gone, so its gate refuses a late send) and it is not in the receiver's transcript: a copy this delivery filed
+      // goes back unread, for the receiver's own mailbox read and the mailbox's wake, which hands it over again
+      const received = this.store.claimOf(to, handed.id)?.recipient === true;
+      if (!relayed.delivered && relayed.unknown && !received) {
+        if (filedHere) { try { this.store.markUnread(to, filed); if (wake) this.store.requestWake(to, handed.id); } catch { /* kept read */ } }
+        throw Object.assign(new Error(relayed.detail), { unanswered: true });
+      }
+      if (!relayed.delivered && !received) { if (filedHere) { try { unlinkSync(filed); } catch { /* gone */ } } throw new Error(relayed.detail); }
       delivered = busy ? 'native_busy' : 'native_idle';
     } else if (door.kind === 'hook') {
       this.store.deliver(to, sent);
@@ -368,7 +380,7 @@ export class MailSend {
         continue;
       }
       try { reached.push(`${label} (${await this.#deliverWoken(sent, recipient.address, sessions, queue, notify_when_idle)})`); }
-      catch (error) { failed.push(`${label} (${error.message})`); }
+      catch (error) { failed.push(error.unanswered ? `${label} (unanswered, whether it arrived is unknown: ${error.message})` : `${label} (${error.message})`); }
     }
     const thread = plan.thread ? ` Thread ${shortId(plan.thread.id)}.` : '';
     let text;
@@ -517,6 +529,7 @@ export class MailSend {
     try { how = await this.route(sent, receiver, door, true, params.notify_when_idle === true); }
     catch (error) {
       if (error instanceof Refused && error.kind === 'too_long') return refused('too_long', `the message is ${error.bytes} bytes; the limit is ${MAX_RELAYED_BYTES}`);
+      if (error.unanswered) return { ...refused('unanswered', unansweredText(`the Claude relay`, error.message, sent.id)), message_id: sent.id };
       return refused('delivery_failed', error.message);
     }
     return {
@@ -686,6 +699,9 @@ export class MailSend {
       return { outcome: 'delivered', detail: null };
     } catch (error) {
       // a message its door can never take (too long for a relay) stays filed for the session to read
+      // one whose hand-over is unanswered was not found where it would have landed: it stays unread and its wake stays,
+      // tried again after its backoff (a wake settled as filed would leave an idle receiver never told)
+      if (error.unanswered) return { outcome: 'later', detail: `unanswered: ${error.message}` };
       return error instanceof Refused ? { outcome: 'filed', detail: error.kind } : { outcome: 'later', detail: error.message };
     } finally {
       const after = this.store.wakeState(address, id);

package/mail/store.mjs

@@ -397,6 +397,12 @@ export class MailStore extends EventEmitter {
     renameSync(path, join(box.dir, 'cur', path.split(/[\\/]/).pop()));
   }
 
+  /** A message filed read moved back to `new/`, unread, under its own name: a hand-over that may not have reached it. */
+  markUnread(address, path) {
+    const box = this.open(address);
+    renameSync(path, join(box.dir, 'new', path.split(/[\\/]/).pop()));
+  }
+
   /** A wake kept beside the envelope (mailbox.rs `request_wake`): its watcher hands it to the session's door. */
   requestWake(address, id) {
     if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error('invalid wake message id');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.3.98",
+  "version": "0.3.99",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",