@volter/supercode-teams 0.3.159 → 0.3.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/connector/index.mjs +2 -2
- package/live-sessions.mjs +12 -3
- package/machine-records.mjs +113 -45
- package/machine.mjs +75 -63
- package/mail/carrier.mjs +202 -157
- package/mail/change-stream.mjs +143 -0
- package/mail/delivery-status.mjs +42 -0
- package/mail/hooks.mjs +90 -15
- package/mail/reads.mjs +2 -1
- package/mail/relay-routing.mjs +41 -0
- package/mail/relay.mjs +120 -204
- package/mail/send.mjs +56 -32
- package/mail/sqlite-store.mjs +166 -16
- package/mail/sqlite.mjs +42 -4
- package/package.json +1 -1
- package/record-state.mjs +143 -0
package/mail/relay.mjs
CHANGED
|
@@ -2,10 +2,11 @@
|
|
|
2
2
|
// its queue, receipts and runtime records in SQLite; Claude reads only its hook
|
|
3
3
|
// configuration in a separate runtime directory. Native runtime start/input/
|
|
4
4
|
// close doors retain their observed-result and unanswered-operation semantics.
|
|
5
|
-
import { mkdirSync,
|
|
5
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
6
6
|
import { connect } from 'node:net';
|
|
7
7
|
import { basename, dirname, join } from 'node:path';
|
|
8
8
|
import { blake3Hex, shortId } from './envelope.mjs';
|
|
9
|
+
import { beginRelayAttempt, settleRelayAttempt } from './relay-routing.mjs';
|
|
9
10
|
|
|
10
11
|
/** Model name a relay's requests carry; the relay endpoint answers them without any model. */
|
|
11
12
|
const RELAY_MODEL = 'haiku';
|
|
@@ -40,25 +41,13 @@ const answerLost = (error) => !error.notAsked && (!error.answered || /gave up af
|
|
|
40
41
|
const SLOW_SEND_MS = 2000;
|
|
41
42
|
/** Largest message relayed into a Claude session: the relay copies it into a model turn byte for byte. */
|
|
42
43
|
export const MAX_RELAYED_BYTES = 100_000;
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
export const RELAYS_RUNNING = 4;
|
|
46
|
-
/** How long a relay speaking for a sender on another machine is kept after its last send (it carries that sender's
|
|
47
|
-
* mail while it is in use; retired after every send, each message from it started a relay). */
|
|
48
|
-
const REMOTE_RELAY_KEPT_MS = 60 * 60_000;
|
|
49
|
-
/** How long a Codex sender's relay is kept unused at a start (a Codex session idle over a day leaves the map). */
|
|
50
|
-
const CODEX_RELAY_KEPT_MS = 7 * 24 * 60 * 60_000;
|
|
51
|
-
/** How long a relay whose runtime start lost its answer is looked at for a runtime that start left running. */
|
|
52
|
-
const LOST_START_KEPT_MS = 5 * 60_000;
|
|
53
|
-
|
|
44
|
+
/** One pool runtime per machine, serialized independently of sender identity. */
|
|
45
|
+
export const RELAYS_RUNNING = 1;
|
|
54
46
|
const alive = (pid) => { if (!Number.isInteger(pid) || pid <= 0) return false; try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
|
|
55
47
|
const shellQuote = (value) => `'${String(value).replaceAll("'", "'\\''")}'`;
|
|
56
48
|
|
|
57
|
-
/**
|
|
58
|
-
export function relayName(
|
|
59
|
-
const base = name.startsWith(RELAY_NAME_PREFIX) ? name.slice(RELAY_NAME_PREFIX.length) : name;
|
|
60
|
-
return `${RELAY_NAME_PREFIX}${base}-on-${machine}`;
|
|
61
|
-
}
|
|
49
|
+
/** Stable registry name of the machine's one relay pool. */
|
|
50
|
+
export function relayName(machine) { return `${RELAY_NAME_PREFIX}mail-pool-on-${machine}`; }
|
|
62
51
|
|
|
63
52
|
/** The host turn asking a relay to make one send (claude_relay.rs `send_turn`). */
|
|
64
53
|
const sendTurn = ({ to, message }) => `Send one message with SendMessage.\nto: ${to}\nmessage: the exact text between the markers, without the markers\n---BEGIN MESSAGE---\n${message}\n---END MESSAGE---`;
|
|
@@ -85,143 +74,78 @@ export class Relays {
|
|
|
85
74
|
* id)` the live-runtime receipt the live-session map holds for a runtime id (its file read whole), or null;
|
|
86
75
|
* `runtimeNamed(receiptId)` one receipt read by its own name.
|
|
87
76
|
*/
|
|
88
|
-
constructor({ mailRoot, store, machine, supercodeBin, serve, runtimeOf, runtimeNamed = () => null, runtimesIn = () => [], endpointOf = async () => null, arrived = async () => false, hooksServed = () => true, log = () => {} }) {
|
|
89
|
-
Object.assign(this, { mailRoot, store, machine, supercodeBin, serve, runtimeOf, runtimeNamed, runtimesIn, endpointOf, arrived, hooksServed, log });
|
|
77
|
+
constructor({ mailRoot, store, machine, supercodeBin, serve, runtimeOf, runtimeNamed = () => null, runtimesIn = () => [], endpointOf = async () => null, arrived = async () => false, hooksServed = () => true, inboundSettled = () => {}, log = () => {} }) {
|
|
78
|
+
Object.assign(this, { mailRoot, store, machine, supercodeBin, serve, runtimeOf, runtimeNamed, runtimesIn, endpointOf, arrived, hooksServed, inboundSettled, log });
|
|
90
79
|
this.runtimeRoot = join(dirname(mailRoot), 'relays');
|
|
91
80
|
this.stopped = false;
|
|
92
81
|
this.receipts = new Set();
|
|
93
|
-
this.lost = new Map(); // relay folder → when a start of its runtime lost its answer (the runtime may run, unnamed)
|
|
94
|
-
this.remoteKept = new Map(); // relay folder of a sender on another machine → the timer that retires it
|
|
95
82
|
this.sending = new Map(); // relay folder → the send in progress through it (one at a time per relay)
|
|
96
|
-
|
|
83
|
+
// Pool identity outlives every runtime handle. Register the stable folder
|
|
84
|
+
// before any startup spool drain, including upgrades whose prior pool
|
|
85
|
+
// already retired without leaving a handle or whose next start is refused.
|
|
86
|
+
const pool = this.spec();
|
|
87
|
+
this.store.putRecord('relays', pool.identity, { pool: true, name: pool.name, represented: pool.represented }, { create: true });
|
|
97
88
|
}
|
|
98
89
|
|
|
99
|
-
/**
|
|
100
|
-
*
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if (
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
this.
|
|
108
|
-
|
|
109
|
-
// again, never handing its turn to a runtime that is closing
|
|
110
|
-
void this.#locked(directory, () => this.serve('harness.v1.runtimes.close', { connection: relay.connection, runtime_id: relay.runtime_id }, CLOSE_MS))
|
|
111
|
-
.catch((error) => {
|
|
112
|
-
// still running, so still counted: the next cap closes it again (unless a send started it anew meanwhile)
|
|
113
|
-
if (!this.running.has(directory)) this.running.set(directory, relay);
|
|
114
|
-
this.log(`relay ${directory.split(/[\\/]/).pop()} beyond the ${RELAYS_RUNNING} most recent was not closed: ${error.message}`);
|
|
115
|
-
});
|
|
90
|
+
/** A proven owner door must retire each predecessor before the pool can start.
|
|
91
|
+
* An absent live-map entry or an unanswered close never proves absence. */
|
|
92
|
+
async #closeRecord(key, record) {
|
|
93
|
+
if (this.stopped) throw new Error('the daemon is stopping; no relay close was submitted');
|
|
94
|
+
if (!record?.runtime_id) throw new Error(`relay ${key} has no proven runtime handle; the pool cannot start`);
|
|
95
|
+
const runtime = this.runtimeOf('claude-code', record.runtime_id);
|
|
96
|
+
const pid = runtime?.metadata?.child_pid ?? record.pid;
|
|
97
|
+
if (!(Number.isInteger(pid) && pid > 0 && !alive(pid))) {
|
|
98
|
+
const answer = await this.serve('harness.v1.runtimes.close', { connection: record.connection ?? '', runtime_id: record.runtime_id }, CLOSE_MS);
|
|
99
|
+
if (answer?.closed !== true || answer.killed === false) throw new Error(`relay ${key}'s owner did not confirm closure; no successor was started`);
|
|
116
100
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
this.log(`relay ${directory.split(/[\\/]/).pop()}: a runtime its lost start left running was not closed`);
|
|
128
|
-
});
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* The relay that speaks for `represented` (known as `name`), retired: its runtime closed and its folder removed, under
|
|
135
|
-
* its own lock (as the native watch's minute sweep did, by event here: the session's end, or a remote
|
|
136
|
-
* sender's quiet hour). A board's or an operator's relay is kept: nothing ends it. Answers whether it was retired.
|
|
137
|
-
*/
|
|
138
|
-
async retire(represented, name) {
|
|
139
|
-
const parsed = String(represented).split(':');
|
|
140
|
-
if (parsed[2] === 'board' || parsed[2] === 'operator') return false;
|
|
141
|
-
// a folder is named by its sender's name, not its address: it is retired only while it still represents this ended
|
|
142
|
-
// address (another live session of that name, or one renamed since, may hold it now)
|
|
143
|
-
return this.#retireFolder(this.spec(represented, name).directory, represented);
|
|
101
|
+
if (this.stopped) throw new Error('the daemon stopped during relay close; its records are retained');
|
|
102
|
+
this.store.transaction(() => {
|
|
103
|
+
const queue = this.store.record('relays', `${key}/queue.json`);
|
|
104
|
+
const receipt = this.store.record('relays', `${key}/receipt.json`);
|
|
105
|
+
settleRelayAttempt(this.store, queue, receipt ? readReceipt(receipt) : { unknown: true });
|
|
106
|
+
if (record.pool === true) this.store.putRecord('relays', `${key}/pool.json`, { pool: true }, { create: true });
|
|
107
|
+
this.store.removeRecord('relays', `${key}/relay.json`);
|
|
108
|
+
this.store.removeRecord('relays', `${key}/queue.json`);
|
|
109
|
+
this.store.removeRecord('relays', `${key}/receipt.json`);
|
|
110
|
+
});
|
|
144
111
|
}
|
|
145
112
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
const
|
|
149
|
-
|
|
113
|
+
async #retireLegacy() {
|
|
114
|
+
const poolKey = basename(this.spec().directory);
|
|
115
|
+
const rows = this.store.records('relays');
|
|
116
|
+
for (const { key, value } of rows) {
|
|
117
|
+
if (!key.endsWith('/relay.json')) continue;
|
|
118
|
+
const folder = key.slice(0, -'/relay.json'.length);
|
|
119
|
+
if (folder === poolKey && value?.pool === true) continue;
|
|
120
|
+
await this.#closeRecord(folder, value);
|
|
121
|
+
}
|
|
122
|
+
// Only a recorded start/handle establishes ownership. A runtime in an old
|
|
123
|
+
// relay directory without that handle cannot be closed by a cwd guess.
|
|
124
|
+
for (const folder of new Set(rows.map(({ key }) => key.split('/')[0]))) {
|
|
125
|
+
if (folder === poolKey) continue;
|
|
126
|
+
if (this.runtimesIn('claude-code', join(this.runtimeRoot, folder)).length) throw new Error(`relay ${folder} still has a live receipt after retirement; the pool cannot start`);
|
|
127
|
+
}
|
|
150
128
|
}
|
|
151
129
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
return this.#locked(directory, async () => {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
const
|
|
160
|
-
if (
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
}
|
|
166
|
-
if (this.stopped) return false;
|
|
167
|
-
this.running.delete(directory); this.lost.delete(directory);
|
|
168
|
-
clearTimeout(this.remoteKept.get(directory)); this.remoteKept.delete(directory);
|
|
169
|
-
this.store.transaction(() => {
|
|
170
|
-
for (const file of ['relay.json', 'queue.json', 'receipt.json', 'sent.json', 'settings.json', 'turn.txt']) this.store.removeRecord('relays', `${basename(directory)}/${file}`);
|
|
130
|
+
/** Sender lifetimes do not retire a shared machine pool. */
|
|
131
|
+
async retire() { return false; }
|
|
132
|
+
|
|
133
|
+
async sweepAtStart() {
|
|
134
|
+
return this.#locked(this.spec().directory, async () => {
|
|
135
|
+
await this.#retireLegacy();
|
|
136
|
+
const spec = this.spec();
|
|
137
|
+
const queue = this.store.record('relays', spec.queue);
|
|
138
|
+
if (!queue) return;
|
|
139
|
+
const receipt = this.store.record('relays', spec.receipt);
|
|
140
|
+
if (receipt) this.store.transaction(() => {
|
|
141
|
+
settleRelayAttempt(this.store, queue, readReceipt(receipt));
|
|
142
|
+
this.store.removeRecord('relays', spec.queue);
|
|
171
143
|
});
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (dirname(directory) === this.runtimeRoot) {
|
|
175
|
-
try { unlinkSync(join(directory, 'settings.json')); } catch (error) { if (error.code !== 'ENOENT') this.log(`relay configuration retained: ${error.message}`); }
|
|
176
|
-
try { rmdirSync(directory); } catch (error) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code)) this.log(`relay directory retained: ${error.message}`); }
|
|
177
|
-
}
|
|
178
|
-
return true;
|
|
144
|
+
else await this.#closeRecord(basename(spec.directory), this.store.record('relays', spec.record));
|
|
145
|
+
this.inboundSettled();
|
|
179
146
|
});
|
|
180
147
|
}
|
|
181
148
|
|
|
182
|
-
/** A relay speaking for a sender on another machine is retired REMOTE_RELAY_KEPT_MS after its last send. */
|
|
183
|
-
#keepRemote(directory, quietSinceMs = Date.now()) {
|
|
184
|
-
if (this.stopped) return;
|
|
185
|
-
clearTimeout(this.remoteKept.get(directory));
|
|
186
|
-
const timer = setTimeout(() => { this.remoteKept.delete(directory); void this.#retireFolder(directory); }, Math.max(0, quietSinceMs + REMOTE_RELAY_KEPT_MS - Date.now()));
|
|
187
|
-
timer.unref?.();
|
|
188
|
-
this.remoteKept.set(directory, timer);
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
/**
|
|
192
|
-
* At a daemon's start, once its live sessions are known: the relays whose sessions ended while no daemon ran are
|
|
193
|
-
* retired, and a remote sender's relay is kept for the rest of its hour (one read of the relays folder, at start only).
|
|
194
|
-
* `running(address)` says whether a session here runs.
|
|
195
|
-
*/
|
|
196
|
-
async sweepAtStart(running) {
|
|
197
|
-
if (this.stopped) return;
|
|
198
|
-
const records = this.store.records('relays');
|
|
199
|
-
const folders = [...new Set(records.map(({ key }) => key.split('/')[0]))];
|
|
200
|
-
const times = new Map(records.map(({ key, updated_at_ms }) => [key, updated_at_ms]));
|
|
201
|
-
let retired = 0;
|
|
202
|
-
for (const folder of folders) {
|
|
203
|
-
if (this.stopped) return;
|
|
204
|
-
const directory = join(this.runtimeRoot, folder);
|
|
205
|
-
const represented = this.#representedOf(directory);
|
|
206
|
-
if (!represented) continue;
|
|
207
|
-
const [, machine, harness] = represented.split(':');
|
|
208
|
-
if (harness === 'board' || harness === 'operator') continue;
|
|
209
|
-
if (machine !== this.machine) {
|
|
210
|
-
const last = times.get(`${folder}/receipt.json`) ?? 0;
|
|
211
|
-
if (Date.now() - last < REMOTE_RELAY_KEPT_MS) { this.#keepRemote(directory, last); continue; }
|
|
212
|
-
} else if (running(represented)) continue;
|
|
213
|
-
else if (harness === 'codex') {
|
|
214
|
-
// a Codex session the map does not hold may be idle, not ended: its relay is kept until unused for a week
|
|
215
|
-
const last = Math.max(times.get(`${folder}/receipt.json`) ?? 0, times.get(`${folder}/relay.json`) ?? 0);
|
|
216
|
-
if (Date.now() - last < CODEX_RELAY_KEPT_MS) continue;
|
|
217
|
-
} else if (harness !== 'claude-code') continue;
|
|
218
|
-
// only a Claude session is provably ended when the map does not hold it (its registry lists every live Claude
|
|
219
|
-
// process); a sender of another harness the map may not see (Grok, Gemini, a Codex session idle a day) is kept
|
|
220
|
-
if (await this.#retireFolder(directory, represented)) retired++;
|
|
221
|
-
}
|
|
222
|
-
if (retired) this.log(`relays: ${retired} of sessions that ended while no daemon ran retired`);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
149
|
/** `work` run under the relay folder's lock: one send, or one close, at a time per relay, in arrival order. */
|
|
226
150
|
async #locked(directory, work) {
|
|
227
151
|
const previous = this.sending.get(directory) ?? Promise.resolve();
|
|
@@ -240,15 +164,15 @@ export class Relays {
|
|
|
240
164
|
}
|
|
241
165
|
}
|
|
242
166
|
|
|
243
|
-
/**
|
|
244
|
-
spec(
|
|
245
|
-
const name = relayName(
|
|
167
|
+
/** Every sender uses this machine's one runtime and one serialized queue. */
|
|
168
|
+
spec() {
|
|
169
|
+
const name = relayName(this.machine);
|
|
246
170
|
const key = blake3Hex(name).slice(0, 24);
|
|
247
171
|
const directory = join(this.runtimeRoot, key);
|
|
248
172
|
return {
|
|
249
|
-
represented
|
|
173
|
+
represented: `sc:${this.machine}:operator:mail-relay`, name, directory,
|
|
250
174
|
queue: `${key}/queue.json`, receipt: `${key}/receipt.json`, settings: join(directory, 'settings.json'),
|
|
251
|
-
settingsKey: `${key}/settings.json`, record: `${key}/relay.json`,
|
|
175
|
+
settingsKey: `${key}/settings.json`, record: `${key}/relay.json`, start: `${key}/start.json`, identity: `${key}/pool.json`,
|
|
252
176
|
};
|
|
253
177
|
}
|
|
254
178
|
|
|
@@ -296,6 +220,8 @@ export class Relays {
|
|
|
296
220
|
const settings = this.#settings(spec);
|
|
297
221
|
this.store.putRecord('relays', spec.settingsKey, settings);
|
|
298
222
|
writeFileSync(spec.settings, JSON.stringify(settings, null, 2), { mode: 0o600 });
|
|
223
|
+
await this.#retireLegacy();
|
|
224
|
+
if (this.store.record('relays', spec.start)) throw new Error('the pool has an unanswered runtime start; its absence is unproven, so no successor was started');
|
|
299
225
|
const record = this.store.record('relays', spec.record);
|
|
300
226
|
if (record?.runtime_id) {
|
|
301
227
|
const runtime = this.runtimeOf('claude-code', record.runtime_id);
|
|
@@ -305,18 +231,8 @@ export class Relays {
|
|
|
305
231
|
}
|
|
306
232
|
// why it is not reused, said in the timing line: a relay that should have been warm and was not shows here
|
|
307
233
|
step(`runtime not reused (${stale ? 'it held a stale turn' : !runtime ? 'no live receipt' : !alive(this.#pid(spec, runtime)) ? 'its process ended' : !record.connection ? 'no connection' : 'endpoint moved'})`);
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
const close = this.serve('harness.v1.runtimes.close', { connection: record.connection, runtime_id: record.runtime_id }, CLOSE_MS);
|
|
311
|
-
const pid = runtime ? this.#pid(spec, runtime) : null;
|
|
312
|
-
if (runtime && !(Number.isInteger(pid) && !alive(pid))) {
|
|
313
|
-
try { await close; }
|
|
314
|
-
catch (error) { if (this.runtimeOf('claude-code', record.runtime_id)) throw new Error(`could not retire the old relay: ${error.message}`); }
|
|
315
|
-
step('old runtime closed (runtimes.close)');
|
|
316
|
-
} else {
|
|
317
|
-
close.catch(() => {});
|
|
318
|
-
step('old runtime let go (runtimes.close, not waited on)');
|
|
319
|
-
}
|
|
234
|
+
await this.#closeRecord(basename(spec.directory), record);
|
|
235
|
+
step('old runtime closed through its owner');
|
|
320
236
|
}
|
|
321
237
|
if (this.stopped) throw new Error('the daemon is stopping; no successor relay was started');
|
|
322
238
|
const key = spec.directory.split(/[\\/]/).pop();
|
|
@@ -330,14 +246,16 @@ export class Relays {
|
|
|
330
246
|
},
|
|
331
247
|
cwd: spec.directory,
|
|
332
248
|
}, 60_000);
|
|
249
|
+
if (this.stopped) throw new Error('the daemon stopped during pool start; its durable intent is retained');
|
|
333
250
|
const runtimeId = started?.handle?.runtime_id;
|
|
334
251
|
if (typeof runtimeId !== 'string') throw new Error(`the machine daemon did not start the relay: ${JSON.stringify(started)}`);
|
|
335
252
|
const connection = typeof started.connection === 'string' ? started.connection : '';
|
|
336
253
|
const pid = Number.isInteger(started?.handle?.endpoint?.pid) ? started.handle.endpoint.pid : null;
|
|
337
254
|
step('runtime started (runtimes.start)');
|
|
338
|
-
this.store.
|
|
339
|
-
|
|
340
|
-
|
|
255
|
+
this.store.transaction(() => {
|
|
256
|
+
this.store.putRecord('relays', spec.record, { pool: true, runtime_id: runtimeId, pid, endpoint, connection, runtime_directory: spec.directory });
|
|
257
|
+
this.store.removeRecord('relays', spec.start);
|
|
258
|
+
});
|
|
341
259
|
// harness serve writes its receipt before it answers the start; the map learns it from its watch a moment later, so
|
|
342
260
|
// one it does not hold yet is read by its own name (the receipt the runtime's terminal launch opens), never waited on
|
|
343
261
|
const known = this.runtimeOf('claude-code', runtimeId);
|
|
@@ -353,12 +271,12 @@ export class Relays {
|
|
|
353
271
|
for (const until = Date.now() + RECEIPT_WAIT_MS; !runtime && Date.now() < until; await new Promise((resolve) => setTimeout(resolve, 100))) {
|
|
354
272
|
runtime = this.runtimeOf('claude-code', runtimeId) ?? (receiptId ? this.runtimeNamed(receiptId) : null);
|
|
355
273
|
}
|
|
274
|
+
if (this.stopped) throw new Error('the daemon stopped while waiting for the pool receipt; its handle is retained');
|
|
356
275
|
step('its receipt read');
|
|
357
276
|
if (!runtime) {
|
|
358
277
|
// a runtime no send can use is closed, never left running untracked
|
|
359
|
-
this.
|
|
360
|
-
|
|
361
|
-
throw new Error('the relay started but registered no live runtime; it was closed');
|
|
278
|
+
await this.#closeRecord(basename(spec.directory), this.store.record('relays', spec.record));
|
|
279
|
+
throw new Error('the pool started but registered no live runtime; its owner confirmed closure');
|
|
362
280
|
}
|
|
363
281
|
return { runtime, connection };
|
|
364
282
|
}
|
|
@@ -391,8 +309,6 @@ export class Relays {
|
|
|
391
309
|
|
|
392
310
|
stop() {
|
|
393
311
|
this.stopped = true;
|
|
394
|
-
for (const timer of this.remoteKept.values()) clearTimeout(timer);
|
|
395
|
-
this.remoteKept.clear();
|
|
396
312
|
for (const cancel of this.receipts) cancel();
|
|
397
313
|
}
|
|
398
314
|
|
|
@@ -401,13 +317,9 @@ export class Relays {
|
|
|
401
317
|
* (claude_relay.rs `send_through_relay`): the receiver is named by its own socket (`uds:<path>`), unique to it, never
|
|
402
318
|
* by its name. One send at a time per relay; a relay whose process is proven dead is started again once.
|
|
403
319
|
*/
|
|
404
|
-
/**
|
|
405
|
-
*
|
|
406
|
-
|
|
407
|
-
queued(sender, senderName) {
|
|
408
|
-
const row = this.store.statement("SELECT updated_at_ms FROM mail_records WHERE collection = 'relays' AND key = ?").get(this.spec(sender, senderName).queue);
|
|
409
|
-
return Boolean(row && Date.now() - row.updated_at_ms < 2 * SEND_TIMEOUT_MS);
|
|
410
|
-
}
|
|
320
|
+
/** A queued native send is not forgotten by age. Startup settles its receipt
|
|
321
|
+
* or obtains predecessor closure before another turn can replace it. */
|
|
322
|
+
queued() { return Boolean(this.store.record('relays', this.spec().queue)); }
|
|
411
323
|
|
|
412
324
|
/** Whether message `messageId` from `sender` landed in Claude session `sessionId`'s transcript, as a relay send reads
|
|
413
325
|
* an unconfirmed outcome (send.mjs handOver asks it of a hand-over the daemon did not live to settle). */
|
|
@@ -415,40 +327,43 @@ export class Relays {
|
|
|
415
327
|
return this.arrived(sessionId, `cross-session-message id=\\"${shortId(messageId)}\\" from=\\"${sender}\\"`);
|
|
416
328
|
}
|
|
417
329
|
|
|
418
|
-
async send(sender, senderName, receiver, message, messageId) {
|
|
330
|
+
async send(sender, senderName, receiver, message, messageId, { expectsReply = true } = {}) {
|
|
419
331
|
const spec = this.spec(sender, senderName);
|
|
420
332
|
const asked = performance.now();
|
|
421
333
|
// the message as its receiver's transcript holds it once delivered (envelope.mjs render, its tag escaped there)
|
|
422
334
|
const landed = `cross-session-message id=\\"${shortId(messageId)}\\" from=\\"${sender}\\"`;
|
|
423
|
-
|
|
424
|
-
finally {
|
|
425
|
-
if (String(sender).split(':')[1] !== this.machine) this.#keepRemote(spec.directory);
|
|
426
|
-
this.#cap();
|
|
427
|
-
}
|
|
335
|
+
return this.#locked(spec.directory, () => this.#send(spec, sender, senderName, receiver, message, messageId, expectsReply, asked, landed));
|
|
428
336
|
}
|
|
429
337
|
|
|
430
|
-
async #send(spec, receiver, message, messageId, asked, landed) {
|
|
338
|
+
async #send(spec, sender, senderName, receiver, message, messageId, expectsReply, asked, landed) {
|
|
339
|
+
let queued = null, submitted = false;
|
|
431
340
|
try {
|
|
432
341
|
// made under its lock: a retire that held the lock meanwhile may have removed it
|
|
433
342
|
if (this.stopped) return { delivered: false, detail: 'the machine daemon is stopping; no relay turn was submitted' };
|
|
434
343
|
mkdirSync(spec.directory, { recursive: true, mode: 0o700 });
|
|
435
|
-
|
|
344
|
+
if (!receiver.socket) throw new Error("the pool requires the receiver's verified native socket; no name-addressed send was submitted");
|
|
345
|
+
queued = { pool: true, expects_reply: expectsReply, to: `uds:${receiver.socket}`, message, sender, sender_name: senderName, message_id: messageId, receiver: receiver.sessionId };
|
|
436
346
|
const steps = [], began = asked;
|
|
437
347
|
let mark = began;
|
|
438
348
|
const step = (name) => { const now = performance.now(); steps.push(`${name} ${Math.round(now - mark)} ms`); mark = now; };
|
|
439
349
|
step('lock');
|
|
440
|
-
|
|
441
|
-
this.#
|
|
442
|
-
this.store.
|
|
443
|
-
|
|
350
|
+
const unsettled = this.store.record('relays', spec.queue) && !this.store.record('relays', spec.receipt);
|
|
351
|
+
let { runtime, connection } = await this.#ensure(spec, step, { stale: Boolean(unsettled) });
|
|
352
|
+
this.store.transaction(() => {
|
|
353
|
+
beginRelayAttempt(this.store, queued);
|
|
354
|
+
this.store.removeRecord('relays', spec.receipt);
|
|
355
|
+
this.store.putRecord('relays', spec.queue, queued);
|
|
356
|
+
});
|
|
444
357
|
let receipt;
|
|
445
358
|
for (let attempt = 0; ; attempt++) {
|
|
446
359
|
// A hand-in whose answer did not come (its bound passed, serve ended mid-call, serve's own deadline on the
|
|
447
360
|
// runtime) may still have reached the relay, which then sends: its receipt is waited on as after any hand-in. A
|
|
448
361
|
// hand-in refused (an answer that is an error) or never asked (no serve) did not reach it, and is said at once,
|
|
449
362
|
// never held for a receipt that cannot come.
|
|
363
|
+
if (!this.store.record('relays', spec.queue)) this.store.transaction(() => { beginRelayAttempt(this.store, queued); this.store.putRecord('relays', spec.queue, queued); });
|
|
450
364
|
let handed = true;
|
|
451
365
|
const error = await this.#handIn(connection, queued, step);
|
|
366
|
+
submitted ||= !error || answerLost(error);
|
|
452
367
|
if (!error) step('turn handed in (send_input)');
|
|
453
368
|
else if (answerLost(error)) step(`turn handed in, its answer lost (send_input: ${error.message})`);
|
|
454
369
|
else {
|
|
@@ -459,7 +374,6 @@ export class Relays {
|
|
|
459
374
|
if (attempt === 0 && /already in progress/.test(error.message)) {
|
|
460
375
|
step(`relay still in a turn after ${TURN_END_MS / 1000} s, closed (${error.message})`);
|
|
461
376
|
({ runtime, connection } = await this.#ensure(spec, step, { stale: true }));
|
|
462
|
-
this.#used(spec, connection);
|
|
463
377
|
this.store.removeRecord('relays', spec.receipt);
|
|
464
378
|
continue;
|
|
465
379
|
}
|
|
@@ -472,32 +386,35 @@ export class Relays {
|
|
|
472
386
|
const pid = this.#pid(spec, runtime);
|
|
473
387
|
// only a relay proven dead is started again; one alive but slow may still deliver
|
|
474
388
|
if (receipt.delivered || attempt === 1 || pid == null || alive(pid)) break;
|
|
475
|
-
try { ({ runtime, connection } = await this.#ensure(spec, step));
|
|
389
|
+
try { ({ runtime, connection } = await this.#ensure(spec, step)); } catch (error) { receipt = { delivered: false, unknown: submitted, detail: error.message }; break; }
|
|
476
390
|
this.store.removeRecord('relays', spec.receipt);
|
|
477
391
|
}
|
|
478
|
-
this.
|
|
479
|
-
//
|
|
480
|
-
//
|
|
392
|
+
if (this.stopped) return { delivered: false, unknown: true, detail: 'the daemon stopped before settling the native send' };
|
|
393
|
+
// An unanswered outcome is read where it would have landed. Keep its
|
|
394
|
+
// queue and pending route together until that observation is committed.
|
|
481
395
|
if (!receipt.delivered && receipt.unknown) {
|
|
482
396
|
await new Promise((resolve) => setTimeout(resolve, LANDING_MS));
|
|
483
397
|
let found = false;
|
|
484
398
|
try { found = await this.arrived(receiver.sessionId, landed); } catch { /* unread: unknown */ }
|
|
399
|
+
if (this.stopped) return { delivered: false, unknown: true, detail: 'the daemon stopped while looking for native delivery' };
|
|
485
400
|
step(found ? 'found in its receiver\'s transcript' : 'not in its receiver\'s transcript');
|
|
486
401
|
if (found) receipt = { delivered: true };
|
|
487
402
|
else receipt = { ...receipt, landed: false };
|
|
488
403
|
}
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
}
|
|
404
|
+
this.store.transaction(() => {
|
|
405
|
+
settleRelayAttempt(this.store, queued, receipt);
|
|
406
|
+
this.store.removeRecord('relays', spec.queue);
|
|
407
|
+
});
|
|
494
408
|
// a slow relay send says where its time went, in the daemon's log (what a person reads of the connector)
|
|
495
409
|
if (performance.now() - began > SLOW_SEND_MS) this.log(`relay send to ${receiver.name || receiver.sessionId} took ${Math.round(performance.now() - began)} ms: ${steps.join(', ')}`);
|
|
496
410
|
return receipt;
|
|
497
411
|
} catch (error) {
|
|
498
|
-
if (!this.stopped) this.store.
|
|
499
|
-
|
|
500
|
-
|
|
412
|
+
if (!this.stopped) this.store.transaction(() => {
|
|
413
|
+
settleRelayAttempt(this.store, queued, { unknown: submitted });
|
|
414
|
+
this.store.removeRecord('relays', spec.queue);
|
|
415
|
+
});
|
|
416
|
+
return { delivered: false, ...(this.stopped || submitted ? { unknown: true } : {}), detail: error.message };
|
|
417
|
+
} finally { if (!this.stopped) this.inboundSettled(); }
|
|
501
418
|
}
|
|
502
419
|
|
|
503
420
|
/**
|
|
@@ -520,18 +437,17 @@ export class Relays {
|
|
|
520
437
|
}
|
|
521
438
|
}
|
|
522
439
|
|
|
523
|
-
/**
|
|
524
|
-
*
|
|
440
|
+
/** Commit intent before asking. An unanswered start remains a durable barrier
|
|
441
|
+
* across restarts; time and a directory scan cannot prove that it never ran. */
|
|
525
442
|
async #start(spec, params, timeoutMs) {
|
|
443
|
+
this.store.putRecord('relays', spec.start, { at_ms: Date.now(), name: spec.name, directory: spec.directory }, { create: true });
|
|
526
444
|
try { return await this.serve('harness.v1.runtimes.start', params, timeoutMs); }
|
|
527
|
-
catch (error) {
|
|
445
|
+
catch (error) {
|
|
446
|
+
if (error.notAsked && !this.stopped) this.store.removeRecord('relays', spec.start);
|
|
447
|
+
throw error;
|
|
448
|
+
}
|
|
528
449
|
}
|
|
529
450
|
|
|
530
|
-
/** A relay this daemon runs, used now. */
|
|
531
|
-
#used(spec, connection) {
|
|
532
|
-
const runtimeId = this.store.record('relays', spec.record)?.runtime_id;
|
|
533
|
-
if (typeof runtimeId === 'string') this.running.set(spec.directory, { connection, runtime_id: runtimeId, usedAt: Date.now() });
|
|
534
|
-
}
|
|
535
451
|
}
|
|
536
452
|
|
|
537
453
|
/** Whether a local port answers a connection within 300 ms. */
|