@manyoars/oars 0.5.2 → 0.5.4

package/dist/sessions/pool-command.js

@@ -11,7 +11,7 @@ import { makeCoverage, readCoverageSetting, writeCoverageSetting } from './cover
 import { checkSession, exclusionPaths, lastWritten, loadTerms, transcriptFilesOf } from './exclusion.js';
 import { localNames, makeRedactor, REDACTION_VERSION, redactTranscript } from './redact.js';
 import { recordSend, recordWire, newTally, tallyBatch } from '../telemetry/wire.js';
-import { discardStale, FIELD_ROWS, fieldsOf, heldAcross, lockedAcross, rescanPulled, machineKeys, markShown, noteMachineKey, outboxFolder, partsOf, pulledAt, removeBatchesExcept, removeBatchOf, staleBatches, unchangedBytes, writeBatch } from './pool-outbox.js';
+import { discardStale, FIELD_ROWS, fieldsOf, heldAcross, lockedAcross, rescanPulled, machineKeys, markShown, noteMachineKey, outboxFolder, partsOf, pulledAt, readPulled, readRemoved, removeBatchesExcept, removeBatchOf, staleBatches, unchangedBytes, writeBatch, writePulled, writeRemoved } from './pool-outbox.js';
 import { deriveConsequences } from './recall-consequence.js';
 import { readRecalls } from './recall-object.js';
 import { HARNESSES, hasEnded, ORG_SIZES, REPO_VISIBILITIES, SESSION_IDLE_HOURS, isBundleVersion, isHarness, parseSkeleton, recommendationsOf, sessionsOf, skeletonOf, unpinnedKid, VOCAB_VERSION, verifyBundle } from './pool-wire.js';
@@ -19,7 +19,7 @@ import { carriedDeltas, heldDeltas, keptDeltas, markDeltasSent, placedOf, sentDe
 import { Masker, printed, TRANSPORT_HEADERS, waitWords } from './pool-print.js';
 import { rawAllowed, rawStale, sendsOf } from './pool-choices.js';
 import { heldRaw, rawParts, removeRawExcept, writeRaw } from './pool-raw.js';
-import { SendLockBusy, withSendLock } from './pool-send-lock.js';
+import { SendInterrupted, SendLockBusy, watchInterrupt, withSendLock } from './pool-send-lock.js';
 import { markSent, retirePulled, sentLog } from './pool-sent.js';
 import { shellReader } from './projection.js';
 import { isVocabularyExpired, REGISTRY_MAX_AGE_DAYS, VOCAB_SNAPSHOT_DATE } from './public-vocabulary.js';
@@ -37,6 +37,7 @@ const LOGIN = 'pool-login';
 const LOGOUT = 'pool-logout';
 export const DEPOSIT = 'pool-deposit';
 const OUTBOX = 'pool-outbox';
+const REMOVE = 'pool-remove';
 const POOL_DEPOSITS = '/v1/deposits';
 const FETCH = 'pool-fetch';
 const EXPORT = 'pool-export';
@@ -74,9 +75,14 @@ export const runtimeOf = (deps) => {
         throw new CliError('The session store is not configured in this run.', ExitCode.UNEXPECTED, 'unexpected');
     return deps.sessions;
 };
-export const signedIn = async (deps, storeDir, server, step) => {
+/** The sign-in held for this server that has not expired, or undefined. */
+const heldSignIn = async (deps, storeDir, server, step) => {
     const found = (await loadSignIns(storeDir, step)).find((entry) => entry.server === server);
-    if (found === undefined || found.expiresAt <= deps.now())
+    return found === undefined || found.expiresAt <= deps.now() ? undefined : found;
+};
+export const signedIn = async (deps, storeDir, server, step) => {
+    const found = await heldSignIn(deps, storeDir, server, step);
+    if (found === undefined)
         throw new CliError(`Not signed in to ${server}. Run oars pool login first.`, ExitCode.AUTH, step);
     return found;
 };
@@ -102,29 +108,95 @@ export const ask = (deps, step, server, path, init, via) => {
     const request = outgoing(deps, server, path, init);
     return via === undefined ? send(deps, step, request, init.show, init.binary === true) : via(request);
 };
-/** Prints the request when asked to, then sends exactly those headers and that body. */
-const send = async (deps, step, request, show, binary = false) => {
+/** The pool took the request and then said nothing for `Guard.stallMs`. */
+class PoolStalled extends Error {
+}
+/** A newcomer's words for the network failures a stalled or restarted pool produces; the raw reason follows the sentence in parentheses. The flag marks a failure that comes after the request began to go out, where the pool may hold the batch: a refused, unresolved, or connect-timed-out connection fails before any of it was sent, while ETIMEDOUT can also come from a retransmit timeout on an open socket after the body was written. */
+const PLAIN_REASONS = [
+    [/ECONNRESET|socket hang up|UND_ERR_SOCKET/, 'the connection was dropped before the pool answered', true],
+    [/ECONNREFUSED/, 'nothing is listening there', false],
+    [/ENOTFOUND|EAI_AGAIN/, 'that name did not resolve', false],
+    [/UND_ERR_CONNECT_TIMEOUT/, 'the connection timed out', false],
+    [/ETIMEDOUT/, 'the connection timed out', true],
+    [/EPIPE/, 'the connection closed while the batch was going out', true],
+];
+const NOT_MARKED = 'Nothing was marked sent.';
+const TRY_AGAIN = 'Run the same command to try again.';
+const RETRY = `${NOT_MARKED} ${TRY_AGAIN}`;
+/** Said after a stop or a stall once the request carrying the batch had gone out: the pool may hold it even though nothing here saw the answer. */
+const MAY_HAVE_RECEIVED = 'The pool may already have received it.';
+/** What `deposit` says after it writes the outbox: the covenant (the person's consent, docs/covenant 2.1) already lets the background sender send what is queued, with or without this command. A machine's batches are never sent by it. */
+const SENT_LATER = 'Nothing was sent yet. Once the covenant is accepted, oars sends what is queued automatically, in the background.';
+/**
+ * What a person reads after a send they stopped (Ctrl-C, a closed terminal, a launcher that died): the batch stays queued, and the background sender (hook-bin.js nightly, started by a later oars command
+ * or the end of a session, about once an hour at most, not on a timer) sends it once the covenant is accepted, whether or not this command ever finishes. The second line is added only when a request carrying the batch had gone out.
+ * The folder switch and the sign-out are the two real ways to stop that: the sender builds the outbox again first, which withdraws a folder that is off, and it sends nothing without a sign-in.
+ */
+export const stoppedText = (received) => [
+    'Stopped. This batch stays queued, and once the covenant is accepted oars sends it automatically in the background.',
+    ...(received ? [MAY_HAVE_RECEIVED] : []),
+    'To keep a folder out, run oars pool raw --folder <folder> --mode off.',
+    'To stop this machine sending, run oars pool logout.',
+].join('\n');
+/** The failure for a send the person or the launcher ended: Ctrl-C (SIGINT) exits 130, the shell convention, and any other way of stopping keeps the unexpected code. The text is the same for all of them. */
+const stopped = (step, cause, received) => new CliError(stoppedText(received), cause.signal === 'SIGINT' ? ExitCode.INTERRUPTED : ExitCode.UNEXPECTED, step);
+/** Why a guarded request ended without an answer, in plain words: a stall, an interrupt, or a network failure whose raw reason is kept after the sentence. The exit code is the unreachable one, as before. */
+const guardedFailure = (step, cause, origin, guard, received) => {
+    if (cause instanceof PoolStalled)
+        return new CliError(`The pool at ${origin} did not answer within ${String(guard.stallMs / 1000)} seconds. ${received ? `${NOT_MARKED} ${MAY_HAVE_RECEIVED} ${TRY_AGAIN}` : RETRY}`, ExitCode.UNREACHABLE, step);
+    if (cause instanceof SendInterrupted)
+        return stopped(step, cause, received);
+    const unreachable = unreachableError(step, cause, origin);
+    const raw = unreachable.message.slice(`Cannot reach the server at ${origin}: `.length);
+    const plain = PLAIN_REASONS.find(([pattern]) => pattern.test(raw));
+    return plain === undefined ? unreachable : new CliError(`Cannot reach the pool at ${origin}: ${plain[1]} (${raw}). ${received && plain[2] ? `${NOT_MARKED} ${MAY_HAVE_RECEIVED} ${TRY_AGAIN}` : RETRY}`, ExitCode.UNREACHABLE, step);
+};
+/** Prints the request when asked to, then sends exactly those headers and that body. A guarded request also ends when its stall time passes or its signal aborts, and fails in plain words. */
+const send = async (deps, step, request, show, binary = false, guard) => {
     if (show !== undefined)
         deps.io.stderr(`${printed(request, show)}
 `);
-    let response;
-    try {
-        response = await fetch(request.url, { method: request.method, headers: request.headers, redirect: 'error', ...(request.body === undefined ? {} : { body: request.body }) });
-    }
-    catch (error) {
-        throw unreachableError(step, error, request.url.origin);
-    }
-    let body;
+    const stop = new AbortController();
+    const relay = () => {
+        stop.abort(guard?.signal?.reason);
+    };
+    guard?.signal?.addEventListener('abort', relay, { once: true });
+    if (guard?.signal?.aborted === true)
+        relay();
+    const timer = guard === undefined ? undefined : setTimeout(() => {
+        stop.abort(new PoolStalled());
+    }, guard.stallMs);
+    // Whether the batch itself had gone out when the run ended: only the deposit request carries it, and only when it went out before the run was stopped.
+    let started = false;
+    const failure = (error) => (guard === undefined ? unreachableError(step, error, request.url.origin) : guardedFailure(step, stop.signal.aborted ? stop.signal.reason : error, request.url.origin, guard, started && request.url.pathname === POOL_DEPOSITS));
     try {
-        body = binary && response.status === 200 ? new Uint8Array(await response.arrayBuffer()) : await response.json();
+        let response;
+        try {
+            started = !stop.signal.aborted;
+            response = await fetch(request.url, { method: request.method, headers: request.headers, redirect: 'error', ...(guard === undefined ? {} : { signal: stop.signal }), ...(request.body === undefined ? {} : { body: request.body }) });
+        }
+        catch (error) {
+            throw failure(error);
+        }
+        let body;
+        try {
+            body = binary && response.status === 200 ? new Uint8Array(await response.arrayBuffer()) : await response.json();
+        }
+        catch (error) {
+            // An answer that began and then stalled is no answer; one that ended badly is read as no body, as before.
+            if (stop.signal.aborted)
+                throw failure(error);
+            body = undefined;
+        }
+        return { status: response.status, body, headers: response.headers };
     }
-    catch {
-        body = undefined;
+    finally {
+        clearTimeout(timer);
+        guard?.signal?.removeEventListener('abort', relay);
     }
-    return { status: response.status, body, headers: response.headers };
 };
-/** The sender `deposit --send` uses when none is given: the request goes out over the network as built. The loopback test passes it to read what reaches the socket. */
-export const networkSender = (deps, step) => (request) => send(deps, step, request, undefined);
+/** The sender `deposit --send` uses when none is given: the request goes out over the network as built. The loopback test passes it to read what reaches the socket. An interactive send passes a `Guard`. */
+export const networkSender = (deps, step, guard) => (request) => send(deps, step, request, undefined, false, guard);
 /** The server's named refusal, as a failure: its `error` is the problem's code in `--json`. */
 export const refusal = (step, { status, body }) => {
     const code = isObject(body) ? textIn(body, 'error') : undefined;
@@ -296,8 +368,18 @@ const regionOf = (text) => {
     const answer = text?.trim().toLowerCase();
     return answer === 'yes' || answer === 'no' ? answer : undefined;
 };
+/**
+ * The CLI's own words around the covenant screen, never inside it: the screen text and the hash of it that the pool records stay exactly what the pool served. They say what a deposit gives and what to run next, and
+ * they promise only access, as covenant 2.1 does ("What you get" and Section 4.3). The third line points at the full text because Part A is a summary and Section 16.2 makes the full covenant the whole agreement.
+ */
+export const OFFER_BEFORE = [
+    'Depositing gives your account access to the pool.',
+    'A counting deposit lifts the weekly limit on pool fetches for 90 days.',
+    'The covenant is the whole deal. Its full text is linked below. Read it before you type.',
+];
+export const OFFER_AFTER = ['Accepted. Sending your batch now.', 'Run oars pool fetch --help for your first pool download.'];
 /** A send stops here when the server is closed or wants a covenant accepted, before the batch is read for the last time. */
-const openCovenant = async (deps, server, token, accept, region, via) => {
+const openCovenant = async (deps, server, token, accept, region, via, signal) => {
     const reply = await ask(deps, DEPOSIT, server, '/v1/covenant', { token }, via);
     if (reply.status !== 200 || !isObject(reply.body))
         throw refusal(DEPOSIT, reply);
@@ -321,13 +403,17 @@ const openCovenant = async (deps, server, token, accept, region, via) => {
     if (!isTerminal(deps)) {
         throw notAccepted(`Covenant ${version} is not accepted for this account. Accepting it takes a person at a terminal who reads it and types its phrase: run this command from one. Nothing was accepted.`);
     }
+    deps.io.stderr(`${OFFER_BEFORE.join('\n')}\n`);
     deps.io.stderr(`\n${screen.text}\n\n> `);
     emitOf(deps)('covenant-shown', { version });
     // One reader for both lines: a second one opened after the first closed would lose a line the first had already taken from a pipe.
-    const lines = createInterface({ input: runtimeOf(deps).stdin, terminal: false });
+    // A stop signal (Ctrl-C, a closed terminal) closes the reader, so a read that is waiting for the person ends with the signal's reason instead of holding the process open.
+    const lines = createInterface({ input: runtimeOf(deps).stdin, terminal: false, ...(signal === undefined ? {} : { signal }) });
     const iterator = lines[Symbol.asyncIterator]();
     const nextLine = async () => {
         const next = await iterator.next();
+        if (signal?.aborted === true)
+            throw signal.reason;
         return next.done === true ? undefined : next.value;
     };
     let answered = region;
@@ -349,6 +435,7 @@ const openCovenant = async (deps, server, token, accept, region, via) => {
     const accepted = await ask(deps, DEPOSIT, server, '/v1/covenant/accept', { token, body }, via);
     if (accepted.status !== 200)
         throw refusal(DEPOSIT, accepted);
+    deps.io.stderr(`${OFFER_AFTER.join('\n')}\n`);
     // A yes to the raw prompt belongs to the covenant it was given under, and a covenant can change what that prompt covers: accepting another version asks the raw question again.
     const reset = { done: false };
     await updatePoolChoices(runtimeOf(deps).storeDir, (choices) => {
@@ -449,10 +536,12 @@ const termsHint = (file) => `No exclusion terms file at ${file}: until it holds
 /** `deposit --send <hash>`: the send lock the detached sender (hook-bin.js nightly) takes too, then a skip for a hash already in `sent.log`, then the batch lock, then the covenant (accepted here or before, or nothing is sent), then the request. The body is read and hashed again just before the request. */
 const sendBatch = async (deps, options) => {
     const { outbox, server, token, hash } = options;
-    const via = options.via ?? networkSender(deps, DEPOSIT);
     if (!/^[0-9a-f]{64}$/.test(hash))
         throw usage('--send takes the SHA-256 that oars pool outbox printed: 64 hexadecimal characters.', DEPOSIT);
     const refuse = (code, title) => new CliError(title, ExitCode.CONFLICT, DEPOSIT, { code, title, status: 409 });
+    // Ctrl-C, a kill, a closed terminal, or a launcher that died ends the wait for the lock and the request in flight, so a send the person gave up on cannot finish later.
+    const interrupt = watchInterrupt();
+    const via = options.via ?? networkSender(deps, DEPOSIT, { stallMs: SEND_STALL_MS, signal: interrupt.signal });
     try {
         return await withSendLock(outbox, async () => {
             // The rule the nightly follows: a hash `sent.log` holds is not posted again. Read under the lock, so two runs cannot both see it unsent.
@@ -470,19 +559,119 @@ const sendBatch = async (deps, options) => {
                 throw refuse(locked.code, lockMessages[locked.code]);
             if (!scan.open && (await heldAcross(outbox)).batches.some((batch) => batch.hash === hash && batch.pulled))
                 throw refuse('terms_missing', 'This batch was held by a cloud session, and your exclusion term file is missing or empty, so it cannot be checked and was not sent.');
-            await openCovenant(deps, server, token, options.acceptCovenant, options.region, via);
+            await openCovenant(deps, server, token, options.acceptCovenant, options.region, via, interrupt.signal);
             const bytes = await unchangedBytes(locked.file, hash);
             if (bytes === undefined)
                 throw refuse('batch_changed', lockMessages.batch_changed);
+            deps.io.stderr(`Sending batch ${hash.slice(0, 12)} to ${server}...\n`);
             await postBatch(deps, { server, token, bytes, hash, via });
             await markSent(outbox, hash, deps.now());
             await retirePulled(outbox, DEPOSIT);
             return { document: { ok: true, server, hash, sent: true }, text: `Sent batch ${hash} to ${server}.` };
+        }, options.lock?.times, options.lock?.fs, {
+            onWait: (waiting) => {
+                deps.io.stderr(`${waitingLine(waiting)}\n`);
+            },
+            signal: interrupt.signal,
         });
     }
     catch (error) {
         if (error instanceof SendLockBusy)
-            throw refuse('send_locked', 'Another oars run is sending from this outbox. Try again in a minute.');
+            throw refuse('send_locked', busyLine(error.detail));
+        if (error instanceof SendInterrupted)
+            throw stopped(DEPOSIT, error, false);
+        throw error;
+    }
+    finally {
+        interrupt.stop();
+    }
+};
+/**
+ * How long the pool gets to answer one request of a send (an interactive `--send` and the background sender) before the send ends. It is a deadline for the whole request, the upload and the answer together,
+ * not an idle timeout: a batch is one session's skeleton, whose largest fixture is 887 KB (docs/pool.md), which a link of 240 kbit/s or better uploads inside 30 seconds. A slower link fails each try with the stall
+ * sentence; resetting the timer once headers arrive would not help it, because the upload is what is slow.
+ */
+export const SEND_STALL_MS = 30_000;
+/** A span in words a person can plan around: seconds under two minutes, else whole minutes. */
+const span = (ms) => {
+    const seconds = Math.max(1, Math.round(ms / 1000));
+    return seconds < 120 ? `${String(seconds)} second${seconds === 1 ? '' : 's'}` : `${String(Math.round(seconds / 60))} minutes`;
+};
+/** What the person reads within a second of a send that has to wait: whose lock it is, how old it is, and how long this run waits. */
+export const waitingLine = ({ holder, ageMs, waitMs }) => holder === undefined
+    ? `Another oars run holds this outbox's send lock (the lock does not name its process; it was last touched ${span(ageMs)} ago). Waiting up to ${span(waitMs)} for it; Ctrl-C stops this run.`
+    : `Another oars run (pid ${String(holder.pid)}) took this outbox's send lock ${span(ageMs)} ago and is still sending. Waiting up to ${span(waitMs)} for it; Ctrl-C stops this run.`;
+/**
+ * What a send that gave up says: nothing was sent, who held the lock, and when a retry can succeed. A live holder on this host touches the lock every quarter of staleMs, even while it waits on a stalled pool, so the clock
+ * never frees its lock: the true condition is its end (a dead holder is taken over at once). Only a holder the lock does not name, or one on another host, whose liveness cannot be read, is decided by the clock: a
+ * retry is certain once the lock has gone untouched for staleMs.
+ */
+export const busyLine = ({ holder, local, ageMs, untouchedMs, waitedMs, staleMs }) => {
+    const who = holder === undefined ? 'Another oars run' : `Another oars run (pid ${String(holder.pid)})`;
+    const held = `${who} has held this outbox's send lock for ${span(ageMs)} and was still sending after the ${span(waitedMs)} this run waited, so nothing was sent.`;
+    if (holder !== undefined && local)
+        return `${held} A retry succeeds as soon as pid ${String(holder.pid)} ends.`;
+    return `${held} Run the same command again once it ends. If it has hung, oars takes the lock over about ${span(Math.max(1, staleMs - untouchedMs))} from now.`;
+};
+/** The session id behind an outbox key, found by deriving the key of each session in the store the way `poolDeposit` does: under every account signed in to this server, or under the machine's environment. `undefined` when the store no longer holds the session. */
+const sessionOfKey = async (deps, storeDir, server, key) => {
+    const machine = machineOf(deps.env);
+    const accounts = machine === undefined ? (await loadSignIns(storeDir, REMOVE)).filter((entry) => entry.server === server).map((entry) => entry.account) : [];
+    const keysOf = (session) => (machine === undefined ? accounts.map((account) => contentHash([account, session])) : [machineKeyOf(machine, session)]);
+    return (await loadStore(storeDir)).sessions.map((observed) => observed.session).find((session) => keysOf(session).includes(key));
+};
+/**
+ * `oars pool remove <hash>`: takes one batch out of this machine's outbox, the batch `oars pool outbox` printed under that hash. It takes the send lock without waiting, because a send in progress may be reading the very file, and refuses while another run holds it.
+ * It reads nothing from the pool and sends nothing, so it needs no sign-in, and it changes only files here. The session of a local batch goes on `removed.json`, which `poolDeposit` (and so the background sender) honors: nothing of it is built, queued, or sent until `--restore <hash>` takes it off. One a cloud session held is admitted again only by `oars pool hold pull`.
+ */
+export const poolRemove = async (deps, hash, options) => {
+    const runtime = runtimeOf(deps);
+    const server = poolServer(deps.env, REMOVE, options.server);
+    if (!/^[0-9a-f]{64}$/.test(hash))
+        throw usage('remove takes the SHA-256 that oars pool outbox printed: 64 hexadecimal characters.', REMOVE);
+    const outbox = outboxFolder(runtime.storeDir, server);
+    const refuse = (code, title) => new CliError(title, ExitCode.CONFLICT, REMOVE, { code, title, status: 409 });
+    try {
+        return await withSendLock(outbox, async () => {
+            if (options.restore === true) {
+                const removed = await readRemoved(outbox, REMOVE);
+                const key = Object.keys(removed).find((candidate) => removed[candidate]?.hash === hash);
+                if (key === undefined)
+                    throw refuse('removal_unknown', 'No removed batch has that hash, so nothing was restored. The hash is the one oars pool remove printed when it removed the batch.');
+                Reflect.deleteProperty(removed, key);
+                await writeRemoved(outbox, removed);
+                return { document: { ok: true, server, hash, restored: true }, text: `Restored the session of batch ${hash} for ${server}. Nothing was sent. A later oars pool deposit builds it again while the session passes the gates.` };
+            }
+            const removed = await readRemoved(outbox, REMOVE);
+            const found = (await heldAcross(outbox)).batches.find((batch) => batch.hash === hash);
+            if (found === undefined)
+                throw refuse('batch_unknown', 'No batch with that hash is in the outbox, so nothing was removed. Run oars pool outbox to list them.');
+            // The session goes on the list first: the build pass would otherwise rebuild it on its next run, the background sender's included, and send what the person withdrew (GDPR Article 7(3)). A list that cannot be read has already failed above, with the batch untouched.
+            const session = found.pulled ? undefined : await sessionOfKey(deps, runtime.storeDir, server, found.key);
+            if (!found.pulled)
+                await writeRemoved(outbox, { ...removed, [found.key]: session === undefined ? { hash } : { hash, session } });
+            await rm(found.file, { force: true });
+            // The session's queued raw copies go in the same lock: the background sender posts every raw copy it holds without looking for its batch, and a batch the person withdrew must take the session's transcript with it (GDPR Article 7(3)). A pulled batch never had one here.
+            const rawGone = await removeRawExcept(outbox, new Set(), found.key);
+            if (found.pulled) {
+                const pulled = await readPulled(outbox, REMOVE);
+                Reflect.deleteProperty(pulled, found.key);
+                await writePulled(outbox, pulled);
+            }
+            const sentAt = (await sentLog(outbox)).get(hash);
+            const text = [
+                `Removed batch ${hash} from the outbox for ${server}. Nothing was sent, and nothing at the pool changed.`,
+                ...(rawGone === 0 ? [] : [`${String(rawGone)} queued raw ${rawGone === 1 ? 'copy' : 'copies'} of the same session went with it.`]),
+                ...(sentAt === undefined ? [] : [`It was already sent at ${sentAt}; removing it here does not take it back from the pool.`]),
+                ...(found.pulled ? ['It came from a cloud session: only oars pool hold pull admits it again.'] : [`Its session stays out of every later oars pool deposit, the background sender's included, until you run oars pool remove ${hash} --restore.`]),
+                ...(found.pulled || session !== undefined ? [] : ['No session in the store matches it, so the exclusion holds for this account and machine settings only.']),
+            ].join('\n');
+            return { document: { ok: true, server, hash, removed: true }, text };
+        }, { ...options.lock?.times, waitMs: 0 }, options.lock?.fs);
+    }
+    catch (error) {
+        if (error instanceof SendLockBusy)
+            throw refuse('send_locked', `Another oars run${error.detail.holder === undefined ? '' : ` (pid ${String(error.detail.holder.pid)})`} holds this outbox's send lock, so nothing was removed. Run the same command again once it ends.`);
         throw error;
     }
 };
@@ -527,7 +716,7 @@ export const poolDeposit = async (deps, options) => {
     const outbox = outboxFolder(runtime.storeDir, server);
     await mkdirOwnerOnly(outbox);
     if (options.send !== undefined)
-        return sendBatch(deps, { outbox, server, token, hash: options.send, acceptCovenant: options.acceptCovenant, region, via: options.via });
+        return sendBatch(deps, { outbox, server, token, hash: options.send, acceptCovenant: options.acceptCovenant, region, via: options.via, lock: options.lock });
     const { terms } = exclusionPaths(runtime.projectsRoot, deps.env);
     const saved = await readState(runtime.storeDir);
     // A yes to the raw prompt counts only for the covenant it was given under, so an interactive build asks the pool which is in force; the nightly already knows and passes it. A pool that cannot be asked, or names none, leaves it unknown and the build goes on as before.
@@ -537,7 +726,7 @@ export const poolDeposit = async (deps, options) => {
     const injectedKey = `injected-block:${(await localSalt(runtime.storeDir)).salt}`;
     const allow = compileAllow(saved.allow);
     const discarded = await discardStale(outbox);
-    const counts = { sessions: 0, batched: 0, excluded: { folder: 0, term: 0, noRaw: 0, noTerms: 0, unreadable: 0 }, off: 0, raw: { copies: 0, bytes: 0, masked: 0 }, uncaptured: 0, notEnded: 0, refused: 0, refusedFields: [], discarded, withdrawn: 0 };
+    const counts = { sessions: 0, batched: 0, excluded: { folder: 0, term: 0, noRaw: 0, noTerms: 0, unreadable: 0 }, off: 0, raw: { copies: 0, bytes: 0, masked: 0 }, uncaptured: 0, removedByPerson: 0, notEnded: 0, refused: 0, refusedFields: [], discarded, withdrawn: 0 };
     // The one gate on a name: the operator's redactor and exclusion terms (t881). Everything else the pool classes itself.
     const termList = await loadTerms(terms);
     const gate = { redact: makeRedactor(await readRedactDomains(runtime.storeDir)), terms: termList };
@@ -560,6 +749,11 @@ export const poolDeposit = async (deps, options) => {
     // The sessions a Stop does not name that still pass the read-only gates: they only fill the windows of a delta, and are never counted, withdrawn, or built.
     const history = [];
     const localStore = await loadStore(runtime.storeDir);
+    // A damaged list stops the pass before anything is built: it must never read as "nothing was removed".
+    const removedList = await readRemoved(outbox, DEPOSIT);
+    const removedKeys = new Set(Object.keys(removedList));
+    // By session id too: the key follows the account and the machine's environment, so a login under another account would otherwise build the same session again.
+    const removedSessions = new Set(Object.values(removedList).flatMap((entry) => (entry.session === undefined ? [] : [entry.session])));
     for (const records of sessionsOf(localStore, roots)) {
         // The Stop of a machine builds the one session that stopped: the others in the store belong to Stops of their own, so their batches stay as they are. They run the three read-only gates below and no more.
         const named = options.session === undefined || records.main.session === options.session;
@@ -567,6 +761,12 @@ export const poolDeposit = async (deps, options) => {
             counts.sessions += 1;
         // Hashed with the account, so a redeposit replaces its own batch and two accounts' copies of one session cannot be joined.
         const key = machine === undefined ? contentHash([account, records.main.session]) : machineKeyOf(machine, records.main.session);
+        // Withdrawn by the person with `oars pool remove`: nothing of the session is built, queued, or read for a window, and its batch and raw copies are already gone.
+        if (removedKeys.has(key) || removedSessions.has(records.main.session)) {
+            if (named)
+                counts.removedByPerson += 1;
+            continue;
+        }
         if (!allow.allows(records.main.project)) {
             if (named) {
                 counts.excluded.folder += 1;
@@ -668,13 +868,14 @@ export const poolDeposit = async (deps, options) => {
     // Counts only: how many sessions each gate kept out, never a folder, a term, or a name.
     emitOf(deps)('deposit', { step: 'build', sessions: counts.sessions, batched: counts.batched, bytes: builtBytes, excludedFolder: folder, excludedTerm: term, excludedNoRaw: noRaw, excludedNoTerms: noTerms, excludedUnreadable: unreadable, notEnded: counts.notEnded, uncaptured: counts.uncaptured, refused: counts.refused, withdrawn: counts.withdrawn, discarded });
     const lines = [
-        `Wrote ${String(counts.batched)} of ${String(counts.sessions)} sessions to the outbox for ${server}. Nothing was sent.`,
+        `Wrote ${String(counts.batched)} of ${String(counts.sessions)} sessions to the outbox for ${server}. ${machine === undefined ? SENT_LATER : 'Nothing was sent.'}`,
         `Kept out by the session gates, never built: ${String(folder)} in a folder not on the allowlist, ${String(term)} for a term, ${String(noRaw)} with no raw to scan, ${String(noTerms)} with no term file, ${String(unreadable)} unreadable.`,
         `Not built: ${String(counts.uncaptured)} captured before steps and frames or with no root, ${String(counts.notEnded)} not ended (a transcript gained a line within ${String(SESSION_IDLE_HOURS)} hours), ${String(counts.refused)} refused by the vocabulary${counts.refusedFields.length === 0 ? '' : ` (${counts.refusedFields.join(', ')})`}.`,
         ...(counts.withdrawn === 0 ? [] : [`Removed ${String(counts.withdrawn)} ${counts.withdrawn === 1 ? 'batch' : 'batches'} of sessions that no longer pass the gates, have not ended, or are no longer in the store.`]),
         ...(discarded === 0 ? [] : [`Discarded ${String(discarded)} ${discarded === 1 ? 'batch' : 'batches'} built under an older vocabulary, field list, or skeleton format.`]),
         `Raw copies queued: ${String(counts.raw.copies)} (${String(counts.raw.bytes)} bytes, ${String(counts.raw.masked)} values masked). ${rawNote(saved.pool.rawConsent)}`.trim(),
         ...(rawStale(saved.pool, covenant) ? [`Your yes to the raw prompt was given under covenant ${saved.pool.rawConsentVersion ?? 'none'}, not ${covenant ?? ''}, so no raw copy is queued. Run oars pool raw --consent yes to answer for the covenant now in force.`] : []),
+        ...(counts.removedByPerson === 0 ? [] : [`${String(counts.removedByPerson)} ${counts.removedByPerson === 1 ? 'session' : 'sessions'} you removed with oars pool remove: nothing of them is built, queued, or sent. oars pool remove <hash> --restore brings one back.`]),
         ...(counts.off === 0 ? [] : [`${String(counts.off)} ${counts.off === 1 ? 'session' : 'sessions'} in a folder switched off: nothing of them is queued.`]),
         ...(termList.length === 0 ? [termsHint(terms)] : []),
         'Run oars pool outbox to print exactly what would be sent.',
@@ -694,7 +895,10 @@ const explained = (bytes) => {
     }
     return rows;
 };
-/** `oars pool outbox`: prints each batch as it would be sent (a batch past one request prints each part's request, in order), the request line and every header included, with the bearer credential as its size and nothing else masked, then its SHA-256, and records that it was shown. `--explain` adds each field's rule. */
+/**
+ * `oars pool outbox`: prints each batch as it would be sent (a batch past one request prints each part's request, in order), the request line and every header included, with the bearer credential as its size and nothing else masked, then its SHA-256, and records that it was shown. `--explain` adds each field's rule.
+ * With `print` each batch is written as soon as it is built and marked shown right after, and the outcome carries no text: an outbox of thousands of sessions is far past V8's string length as one string. Without it (`--json`) the document carries each batch's request, and `--json` writes it element by element.
+ */
 export const poolOutbox = async (deps, options) => {
     const runtime = runtimeOf(deps);
     const server = poolServer(deps.env, OUTBOX, options.server);
@@ -704,25 +908,41 @@ export const poolOutbox = async (deps, options) => {
     const masker = new Masker();
     masker.add(token);
     const sent = await sentLog(outbox);
+    const { print } = options;
     const shown = [];
+    // Blocks are separated by a blank line; each is written alone, so no string holds more than one batch.
+    const state = { wrote: false };
+    const put = (block) => {
+        print?.(`${state.wrote ? '\n\n' : ''}${block}`);
+        state.wrote = true;
+    };
     for (const batch of batches) {
         const fields = options.explain === true ? explained(batch.bytes) : undefined;
         // A batch past one request prints every part's request, in the order they are sent.
         const request = callsOf(token, batch.bytes, batch.hash).map((call) => printed(outgoing(deps, server, POOL_DEPOSITS, call), masker)).join('\n');
-        shown.push({ hash: batch.hash, request, ...(sent.has(batch.hash) ? { sent: sent.get(batch.hash) ?? '' } : {}), ...(fields === undefined ? {} : { fields }) });
+        const at = sent.get(batch.hash);
+        put([request, `sha256: ${batch.hash}`, ...(at === undefined ? [] : [`sent: ${at}`]), ...Object.entries(fields ?? {}).map(([field, row]) => `  ${field}: ${row}`)].join('\n'));
+        shown.push({ hash: batch.hash, ...(print === undefined ? { request } : {}), ...(at === undefined ? {} : { sent: at }), ...(fields === undefined ? {} : { fields }) });
+        await markShown(outbox, batch.hash, deps.now());
     }
     const { copies } = await heldRaw(outbox);
-    const rawBlocks = copies.flatMap((copy) => rawParts(copy.body).map((part, at, all) => [`raw part ${String(at + 1)}/${String(all.length)} sha256: ${copy.hash} bytes: ${String(Buffer.byteLength(part))}${sent.has(copy.hash) ? ` sent: ${sent.get(copy.hash) ?? ''}` : ''}`, ...(options.explain === true ? [part] : [])].join('\n')));
-    for (const { hash } of shown)
-        await markShown(outbox, hash, deps.now());
+    for (const copy of copies) {
+        const parts = rawParts(copy.body);
+        for (const [at, part] of parts.entries())
+            put([`raw part ${String(at + 1)}/${String(parts.length)} sha256: ${copy.hash} bytes: ${String(Buffer.byteLength(part))}${sent.has(copy.hash) ? ` sent: ${sent.get(copy.hash) ?? ''}` : ''}`, ...(options.explain === true ? [part] : [])].join('\n'));
+    }
     const stale = await staleBatches(outbox);
-    const blocks = shown.map(({ hash, request, sent: at, fields }) => [request, `sha256: ${hash}`, ...(at === undefined ? [] : [`sent: ${at}`]), ...Object.entries(fields ?? {}).map(([field, row]) => `  ${field}: ${row}`)].join('\n'));
     const notes = [
         ...(edited === 0 ? [] : [`${String(edited)} ${edited === 1 ? 'batch was' : 'batches were'} edited on disk since ${edited === 1 ? 'it was' : 'they were'} written and ${edited === 1 ? 'is' : 'are'} not printed or sent. Run oars pool deposit to build ${edited === 1 ? 'it' : 'them'} again.`]),
         ...(stale === 0 ? [] : [`${String(stale)} ${stale === 1 ? 'batch was' : 'batches were'} built under an older vocabulary, field list, or skeleton format and will never be sent.`]),
     ];
-    const text = shown.length === 0 && rawBlocks.length === 0 ? ['The outbox is empty. Run oars pool deposit first.', ...notes].join('\n') : [...blocks, ...rawBlocks, ...notes].join('\n\n');
-    return { document: { ok: true, server, batches: shown, raw: copies.map(({ hash, bytes }) => ({ hash, bytes, ...(sent.has(hash) ? { sent: sent.get(hash) ?? '' } : {}) })), edited, stale }, text };
+    if (!state.wrote)
+        put(['The outbox is empty. Run oars pool deposit first.', ...notes].join('\n'));
+    else
+        for (const note of notes)
+            put(note);
+    print?.('\n');
+    return { document: { ok: true, server, batches: shown, raw: copies.map(({ hash, bytes }) => ({ hash, bytes, ...(sent.has(hash) ? { sent: sent.get(hash) ?? '' } : {}) })), edited, stale }, text: '', output: print === undefined ? 'chunked' : 'streamed' };
 };
 /**
  * What the pool says it served a bundle at, printed as the bundle states it: the client holds no floor of its own and applies none (a floor is the pool's deploy setting), so a bundle that states none prints nothing.
@@ -831,6 +1051,74 @@ export const poolFetch = async (deps, options) => {
         text: `Fetched ${harness} ${harnessVersion}: ${String(reads.length)} reads, ${String(cells.length)} cells, and ${String(motifs.length)} motifs, verified against the pinned key and kept at ${file}.${floorSentence(floors)}`,
     };
 };
+const JOIN = 'pool-join';
+const JOIN_AGAIN = 'Run oars pool join again to continue from here.';
+/**
+ * `oars pool join`: the five commands of a first deposit in one run, each by the handler the command itself uses: `login`, `deposit`, `outbox --explain`, `deposit --send <hash>` for each batch not yet sent, then `fetch`. It asks and checks
+ * nothing they do not, and a step that stops ends the run with that step's own error. Run again it skips what stands: a held sign-in, a prepared batch not yet sent, a batch in `sent.log`.
+ * Each step's text prints as it finishes, so the explanation is on screen before the send asks anything: on stdout, or with `json` on stderr, so stdout holds the one document.
+ */
+export const poolJoin = async (deps, options) => {
+    const { storeDir } = runtimeOf(deps);
+    const server = poolServer(deps.env, JOIN, options.server);
+    const say = (line) => {
+        deps.io.stderr(`${line}
+`);
+    };
+    const screen = options.json === true ? deps.io.stderr : deps.io.stdout;
+    const show = (outcome) => {
+        if (outcome.output !== 'streamed')
+            screen(`${outcome.text}
+`);
+        return outcome;
+    };
+    const outbox = outboxFolder(storeDir, server);
+    const unsent = async () => {
+        const sent = await sentLog(outbox);
+        return (await heldAcross(outbox)).batches.flatMap(({ hash }) => (sent.has(hash) ? [] : [hash]));
+    };
+    try {
+        const held = await heldSignIn(deps, storeDir, server, LOGIN);
+        let login = { skipped: true, account: held?.account };
+        if (held === undefined) {
+            say(`Signing in to ${server}.`);
+            login = show(await poolLogin(deps, { server, provider: options.provider, verbose: options.verbose })).document;
+        }
+        else {
+            say(`Already signed in to ${server} as ${held.account}.`);
+        }
+        let prepared = { skipped: true };
+        if ((await unsent()).length > 0) {
+            say('Your outbox already holds a prepared batch; not preparing it again.');
+        }
+        else {
+            say('Preparing your deposit in the outbox.');
+            prepared = show(await poolDeposit(deps, { server })).document;
+        }
+        say('Showing what will be sent, field by field.');
+        const shown = show(await poolOutbox(deps, { server, explain: true, print: screen })).document;
+        const hashes = await unsent();
+        const sends = [];
+        if (hashes.length === 0) {
+            say((await heldAcross(outbox)).batches.length === 0 ? 'No session is ready to deposit yet; sending nothing.' : 'Everything prepared is already sent; sending nothing.');
+        }
+        else {
+            say(`Sending ${plural(hashes.length, 'batch', 'batches')} to ${server}.`);
+            for (const hash of hashes)
+                sends.push(show(await poolDeposit(deps, { server, send: hash, acceptCovenant: options.acceptCovenant, inEuEeaUk: options.inEuEeaUk })).document);
+        }
+        say(`Fetching the bundle for ${options.harness ?? 'your most used harness'} ${options.harnessVersion}.`);
+        const fetched = show(await poolFetch(deps, { server, harness: options.harness, harnessVersion: options.harnessVersion }));
+        return {
+            document: { ok: true, server, login, deposit: prepared, outbox: shown, sends, fetch: fetched.document },
+            text: `Joined ${server}: ${plural(sends.length, 'batch', 'batches')} sent this run, and the ${options.harnessVersion} bundle is kept.`,
+        };
+    }
+    catch (error) {
+        say(JOIN_AGAIN);
+        throw error;
+    }
+};
 const RAW = 'pool-raw';
 /** The raw prompt, in plain words: shown by `oars pool raw` until it is answered, and what the detached sender relies on. Redaction is best effort and the text says so. */
 export const RAW_PROMPT = [

package/dist/sessions/pool-nightly.js

@@ -5,7 +5,7 @@ import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { CliError } from '../errors.js';
 import { machineOf } from './machine.js';
-import { ask, currentTerms, DEPOSIT, isObject, networkSender, poolDeposit, poolServer, postBatch, postRaw, runtimeOf, signedIn, textIn, vocabularyExpired } from './pool-command.js';
+import { ask, currentTerms, DEPOSIT, isObject, networkSender, poolDeposit, poolServer, postBatch, postRaw, runtimeOf, SEND_STALL_MS, signedIn, textIn, vocabularyExpired } from './pool-command.js';
 import { rawStale } from './pool-choices.js';
 import { heldAcross, lockedBatch, outboxFolder, rescanPulled } from './pool-outbox.js';
 import { heldRaw } from './pool-raw.js';
@@ -119,7 +119,8 @@ export const nightlyPoolSend = async (deps, via) => {
     const tally = { outcome: 'failed', batches: 0, raw: 0 };
     let result;
     try {
-        result = await night(deps, via ?? networkSender(deps, DEPOSIT), tally);
+        // A pool that goes silent ends the night after SEND_STALL_MS, so it cannot hold the send lock (and with it every `deposit --send`) for as long as the pool stalls. The next run sends what this one did not.
+        result = await night(deps, via ?? networkSender(deps, DEPOSIT, { stallMs: SEND_STALL_MS }), tally);
     }
     catch {
         result = { ...tally, outcome: 'failed' };

package/dist/sessions/pool-outbox.d.ts

@@ -94,6 +94,19 @@ export interface Pulled {
 export declare const readPulled: (outbox: string, step: string) => Promise<Pulled>;
 /** Staged and renamed, so a crash leaves the whole old file or the whole new one. */
 export declare const writePulled: (outbox: string, pulled: Pulled) => Promise<void>;
+/** One session a person took out with `oars pool remove`: the hash of the batch that was removed, which is how `--restore` names the way back, and the session id when it could be resolved. The outbox key derives from the account or the machine's environment, so only the session id keeps the session out after either changes; an entry with no id (the session was gone from the store, or the file is older) skips by key alone. */
+export interface RemovedEntry {
+    hash: string;
+    session?: string;
+}
+/** The sessions a person took out of the outbox with `oars pool remove`, by outbox key. Local only; nothing in it goes to the pool. */
+export interface Removed {
+    [key: string]: RemovedEntry;
+}
+/** `removed.json` as it stands; none when the file is missing. A file that is not a list of keys and entries fails the command that reads it, so a damaged list never becomes "send everything". A bare string is an entry with only a hash, the form before session ids were kept. */
+export declare const readRemoved: (outbox: string, step: string) => Promise<Removed>;
+/** Staged and renamed, like `pulled.json`: a crash leaves the whole old list or the whole new one. */
+export declare const writeRemoved: (outbox: string, removed: Removed) => Promise<void>;
 /** Whether any string or key in `value` holds one of `terms`, compared lowercased the way the local gate compares a transcript. */
 export declare const holdsTerm: (value: unknown, terms: readonly string[]) => boolean;
 /**

package/dist/sessions/pool-outbox.js

@@ -385,6 +385,52 @@ export const writePulled = async (outbox, pulled) => {
     await writeOwnerOnly(staged, `${JSON.stringify(Object.fromEntries(Object.entries(pulled).sort(([a], [b]) => (a < b ? -1 : 1))), null, 2)}\n`);
     await rename(staged, join(outbox, PULLED));
 };
+const REMOVED = 'removed.json';
+const removedEntryOf = (value) => {
+    if (typeof value === 'string')
+        return { hash: value };
+    if (typeof value !== 'object' || value === null || Array.isArray(value) || !('hash' in value) || typeof value.hash !== 'string')
+        return undefined;
+    if (!('session' in value))
+        return { hash: value.hash };
+    return typeof value.session === 'string' ? { hash: value.hash, session: value.session } : undefined;
+};
+/** `removed.json` as it stands; none when the file is missing. A file that is not a list of keys and entries fails the command that reads it, so a damaged list never becomes "send everything". A bare string is an entry with only a hash, the form before session ids were kept. */
+export const readRemoved = async (outbox, step) => {
+    let text;
+    try {
+        text = await readFile(join(outbox, REMOVED), 'utf8');
+    }
+    catch (error) {
+        if (isMissing(error))
+            return {};
+        throw error;
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        value = undefined;
+    }
+    const entries = typeof value !== 'object' || value === null || Array.isArray(value) ? undefined : Object.entries(value).map(([key, entry]) => [key, removedEntryOf(entry)]);
+    const kept = [];
+    for (const [key, entry] of entries ?? [])
+        if (entry !== undefined)
+            kept.push([key, entry]);
+    if (entries === undefined || kept.length !== entries.length) {
+        throw new CliError(`${join(outbox, REMOVED)} is not a list of removed sessions. Move it aside and run the command again; nothing was changed, built, or sent.`, ExitCode.INVALID, step);
+    }
+    return Object.fromEntries(kept);
+};
+/** Staged and renamed, like `pulled.json`: a crash leaves the whole old list or the whole new one. */
+export const writeRemoved = async (outbox, removed) => {
+    await mkdirOwnerOnly(outbox);
+    const staged = join(outbox, `${REMOVED}.tmp`);
+    await writeOwnerOnly(staged, `${JSON.stringify(Object.fromEntries(Object.entries(removed).sort(([a], [b]) => (a < b ? -1 : 1))), null, 2)}
+`);
+    await rename(staged, join(outbox, REMOVED));
+};
 const POOL_STEP = 'pool-outbox';
 /** Whether any string or key in `value` holds one of `terms`, compared lowercased the way the local gate compares a transcript. */
 export const holdsTerm = (value, terms) => {