@manyoars/oars 0.3.0 → 0.4.1

package/dist/sessions/pool-command.js

@@ -1,22 +1,32 @@
-// `oars pool`: the only part of the sessions code that talks to a remote. The server is always named, by flag or environment, so nothing leaves on a default. Sign-in runs the provider's device flow here and reads no harness token; a session leaves only as a closed-vocabulary skeleton, and only after the exclusion check found it clean. Local capture, recall,
+// `oars pool`: the only part of the sessions code that talks to a remote. The server is the one named by flag or environment, else the Many Oars pool; nothing leaves until a command that sends is run or, after covenant acceptance, the nightly task sends. Sign-in runs the provider's device flow here and reads no harness token; a session leaves only as a skeleton of a closed list of fields, each value redacted first, and only after the exclusion check found it clean. Local capture, recall,
 // and reads never come through here.
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
-import { dirname, join } from 'node:path';
+import { dirname } from 'node:path';
+import { emitOf } from '../telemetry/emit.js';
 import { CliError, exitCodeForProblem, ExitCode, unreachableError } from '../errors.js';
 import { compileAllow } from './allow.js';
-import { checkSession, exclusionPaths, lastWritten } from './exclusion.js';
-import { discardStale, FIELD_ROWS, fieldsOf, heldBatches, isMissing, lockedBatch, markShown, outboxFolder, removeBatchesExcept, removeBatchOf, staleBatches, unchangedBytes, writeBatch } from './pool-outbox.js';
-import { HARNESSES, hasEnded, SESSION_IDLE_HOURS, isBundleVersion, isHarness, parseSkeleton, sessionsOf, skeletonOf, VOCAB_VERSION, verifyBundle } from './pool-wire.js';
+import { makeCoverage, readCoverageSetting, writeCoverageSetting } from './coverage.js';
+import { checkSession, exclusionPaths, lastWritten, loadTerms, transcriptFilesOf } from './exclusion.js';
+import { localNames, makeRedactor, REDACTION_VERSION, redactTranscript } from './redact.js';
+import { recordSend, recordWire, newTally, tallyBatch } from '../telemetry/wire.js';
+import { discardStale, FIELD_ROWS, fieldsOf, heldBatches, lockedBatch, markShown, outboxFolder, partsOf, removeBatchesExcept, removeBatchOf, staleBatches, unchangedBytes, writeBatch } from './pool-outbox.js';
+import { HARNESSES, hasEnded, ORG_SIZES, REPO_VISIBILITIES, SESSION_IDLE_HOURS, isBundleVersion, isHarness, parseSkeleton, sessionsOf, skeletonOf, VOCAB_VERSION, verifyBundle } from './pool-wire.js';
 import { carriedDeltas, markDeltasSent, placedOf, sentDeltas } from './pool-delta.js';
 import { Masker, printed, TRANSPORT_HEADERS, waitWords } from './pool-print.js';
+import { rawAllowed, sendsOf } from './pool-choices.js';
+import { heldRaw, rawParts, removeRawExcept, writeRaw } from './pool-raw.js';
+import { SendLockBusy, withSendLock } from './pool-send-lock.js';
+import { markSent, sentLog } from './pool-sent.js';
 import { shellReader } from './projection.js';
 import { isVocabularyExpired, REGISTRY_MAX_AGE_DAYS, VOCAB_SNAPSHOT_DATE } from './public-vocabulary.js';
-import { contentHash } from './read.js';
-import { harnessOf as sessionHarness } from './schema.js';
-import { loadStore, readSaved, readSkeletons } from './store.js';
+import { contentHash, FIRST_ORDER_FAMILIES } from './read.js';
+import { isObject, keptReplica, loadSignIns, numberIn, replicaFile, signInOf, signInsFile, textIn } from './pool-replica.js';
+import { CAPTURE_VERSION, harnessOf as sessionHarness } from './schema.js';
+import { loadStore, readRedactDomains, readSkeletons, readState, updatePoolChoices } from './store.js';
+import { runGit } from './survival.js';
 import { vocabulary } from './vocab-load.js';
 const LOGIN = 'pool-login';
-const DEPOSIT = 'pool-deposit';
+export const DEPOSIT = 'pool-deposit';
 const OUTBOX = 'pool-outbox';
 const POOL_DEPOSITS = '/v1/deposits';
 const FETCH = 'pool-fetch';
@@ -26,12 +36,13 @@ const DEFAULT_INTERVAL_SEC = 5;
 const DEFAULT_EXPIRY_SEC = 900;
 /** RFC 8628: a `slow_down` adds five seconds to every poll after it. */
 const SLOW_DOWN_MS = 5000;
+export { isObject, loadSignIns, textIn };
 const usage = (message, step) => new CliError(message, ExitCode.USAGE, step);
-/** `--server`, else `OARS_POOL_URL`; never a default. A token and a session's shape cross this connection, so plain http goes only to this machine. */
+/** The Many Oars pool, used when neither `--server` nor `OARS_POOL_URL` names another. */
+export const DEFAULT_POOL_URL = 'https://pool.manyoars.com';
+/** `--server`, else `OARS_POOL_URL`, else the Many Oars pool. A token and a session's shape cross this connection, so plain http goes only to this machine. */
 export const poolServer = (env, step, named) => {
-    const given = named ?? env.OARS_POOL_URL;
-    if (given === undefined || given === '')
-        throw usage('Name the pool server with --server or OARS_POOL_URL; there is no default.', step);
+    const given = named ?? (env.OARS_POOL_URL === undefined || env.OARS_POOL_URL === '' ? DEFAULT_POOL_URL : env.OARS_POOL_URL);
     let url;
     try {
         url = new URL(given);
@@ -45,65 +56,12 @@ export const poolServer = (env, step, named) => {
         throw usage(`The pool server ${given} is an origin only: no credentials, path, or query.`, step);
     return url.origin;
 };
-const runtimeOf = (deps) => {
+export const runtimeOf = (deps) => {
     if (deps.sessions === undefined)
         throw new CliError('The session store is not configured in this run.', ExitCode.UNEXPECTED, 'unexpected');
     return deps.sessions;
 };
-const textIn = (value, key) => {
-    const found = Reflect.get(value, key);
-    return typeof found === 'string' && found !== '' ? found : undefined;
-};
-const numberIn = (value, key) => {
-    const found = Reflect.get(value, key);
-    return typeof found === 'number' && Number.isFinite(found) ? found : undefined;
-};
-const isObject = (value) => typeof value === 'object' && value !== null;
-const keyOf = (value) => {
-    if (!isObject(value))
-        return undefined;
-    const [kty, crv, x] = [textIn(value, 'kty'), textIn(value, 'crv'), textIn(value, 'x')];
-    return kty === 'OKP' && crv === 'Ed25519' && x !== undefined ? { kty, crv, x } : undefined;
-};
-const signInOf = (value) => {
-    if (!isObject(value))
-        return undefined;
-    const [server, account, token, expiresAt] = [textIn(value, 'server'), textIn(value, 'account'), textIn(value, 'token'), numberIn(value, 'expiresAt')];
-    const signingKey = keyOf(Reflect.get(value, 'signingKey'));
-    return server === undefined || account === undefined || token === undefined || expiresAt === undefined || signingKey === undefined ? undefined : { server, account, token, expiresAt, signingKey };
-};
-const signInsFile = (storeDir) => join(storeDir, 'pool-accounts.json');
-/** Every saved sign-in. A file that cannot be read stops the command: treating it as empty would forget a pin and trust whatever key the server names next. */
-const loadSignIns = async (storeDir, step) => {
-    const file = signInsFile(storeDir);
-    let text;
-    try {
-        text = await readFile(file, 'utf8');
-    }
-    catch (error) {
-        if (isMissing(error))
-            return [];
-        throw error;
-    }
-    const damaged = new CliError(`${file} cannot be read. Remove it, then run oars pool login again.`, ExitCode.INVALID, step);
-    let parsed;
-    try {
-        parsed = JSON.parse(text);
-    }
-    catch {
-        throw damaged;
-    }
-    if (!Array.isArray(parsed))
-        throw damaged;
-    const entries = parsed;
-    return entries.map((entry) => {
-        const signIn = signInOf(entry);
-        if (signIn === undefined)
-            throw damaged;
-        return signIn;
-    });
-};
-const signedIn = async (deps, storeDir, server, step) => {
+export const signedIn = async (deps, storeDir, server, step) => {
     const found = (await loadSignIns(storeDir, step)).find((entry) => entry.server === server);
     if (found === undefined || found.expiresAt <= deps.now())
         throw new CliError(`Not signed in to ${server}. Run oars pool login first.`, ExitCode.AUTH, step);
@@ -115,16 +73,19 @@ const outgoing = (deps, server, path, init) => {
     if (init.token !== undefined)
         headers.push(['authorization', `Bearer ${init.token}`]);
     // Node's `fetch` writes `cache-control` and `pragma` itself whenever `if-none-match` is set, so they are named here to be printed.
+    if (init.part !== undefined)
+        headers.push(['oars-part', `${String(init.part.index)}/${String(init.part.count)}`], ['oars-batch', init.part.batch]);
     if (init.ifNoneMatch !== undefined)
         headers.push(['if-none-match', init.ifNoneMatch], ['cache-control', 'no-cache'], ['pragma', 'no-cache']);
+    headers.push(...(init.extra ?? []));
     const body = init.bytes ?? (init.body === undefined ? undefined : JSON.stringify(init.body));
     if (body !== undefined)
-        headers.push(['content-type', 'application/json']);
+        headers.push(['content-type', init.type ?? 'application/json']);
     headers.push(...TRANSPORT_HEADERS);
-    return { method: body === undefined ? 'GET' : 'POST', url: new URL(path, server), headers, ...(body === undefined ? {} : { body }) };
+    return { method: init.method ?? (body === undefined ? 'GET' : 'POST'), url: new URL(path, server), headers, ...(body === undefined ? {} : { body }) };
 };
 /** One call to the pool. A redirect is an error: it could carry a batch or a token somewhere the checks on the named server never saw. */
-const ask = (deps, step, server, path, init, via) => {
+export const ask = (deps, step, server, path, init, via) => {
     const request = outgoing(deps, server, path, init);
     return via === undefined ? send(deps, step, request, init.show) : via(request);
 };
@@ -149,7 +110,7 @@ const send = async (deps, step, request, show) => {
     }
     return { status: response.status, body, headers: response.headers };
 };
-/** The sender a real send would use: the request goes out over the network as built. The command line does not pass it (sending waits on the covenant); the loopback test does, to read what reaches the socket. */
+/** The sender `deposit --send` uses when none is given: the request goes out over the network as built. The loopback test passes it to read what reaches the socket. */
 export const networkSender = (deps, step) => (request) => send(deps, step, request, undefined);
 /** The server's named refusal, as a failure: its `error` is the problem's code in `--json`. */
 const refusal = (step, { status, body }) => {
@@ -282,7 +243,7 @@ const excludedKey = (reason) => (reason === 'no-raw' ? 'noRaw' : reason === 'no-
 /** A deposit answered with this code is sent again: another deposit of the same session landed between the pool's read and its write, and the next one nets against the winner. */
 const DEPOSIT_CONFLICT = 'deposit_conflict';
 const SEND_ATTEMPTS = 3;
-const vocabularyExpired = (deps) => {
+export const vocabularyExpired = (deps) => {
     if (!isVocabularyExpired(new Date(deps.now())))
         return undefined;
     const title = `This oars is out of date: its vocabulary ${VOCAB_VERSION} was taken on ${VOCAB_SNAPSHOT_DATE}, over ${String(REGISTRY_MAX_AGE_DAYS)} days ago. Upgrade with npm install -g @manyoars/oars, then run oars pool deposit again. Your sessions stay on this machine.`;
@@ -293,33 +254,80 @@ const lockMessages = {
     batch_not_shown: 'That batch was never printed. Run oars pool outbox, read it, then send it.',
     batch_changed: 'That batch changed on disk after it was printed, or since it was written, so it was not sent. Run oars pool deposit to build it again, then oars pool outbox to print it.',
 };
-/** `deposit --send <hash>`: the lock, then (only when a sender is given) the covenant and the request. The body is read and hashed again just before the request. */
+/** The calls one batch takes, in order: one, or one per part when it is past a request's bytes. `outbox` prints these and `deposit --send` sends these, so what is printed is what is sent. */
+const callsOf = (token, bytes, hash) => {
+    const parts = partsOf(bytes);
+    return parts.map((piece, at) => ({ token, bytes: piece, ...(parts.length === 1 ? {} : { part: { index: at + 1, count: parts.length, batch: hash } }) }));
+};
+/** One batch to the pool: a batch past one request's bytes goes in parts, in order, each answered 202 until the last completes the deposit with 201. A part sent again is harmless, so the retry covers every part alike. */
+export const postBatch = async (deps, options) => {
+    const { server, token, bytes, hash, via } = options;
+    const calls = callsOf(token, bytes, hash);
+    for (const [at, call] of calls.entries()) {
+        let reply = await ask(deps, DEPOSIT, server, POOL_DEPOSITS, call, via);
+        for (let attempt = 1; attempt < SEND_ATTEMPTS && reply.status === 409 && isObject(reply.body) && textIn(reply.body, 'error') === DEPOSIT_CONFLICT; attempt += 1) {
+            reply = await ask(deps, DEPOSIT, server, POOL_DEPOSITS, call, via);
+        }
+        // 201 on any part means the deposit already finished (this part is a retry), so the rest need not be sent.
+        const ended = reply.status !== 202 || at === calls.length - 1;
+        // One send event per batch, on the reply that ends it, whatever number of parts it took.
+        if (ended)
+            recordSend(deps, { status: reply.status, code: isObject(reply.body) ? textIn(reply.body, 'error') : undefined, bytes: Buffer.byteLength(bytes) });
+        if (reply.status === 201)
+            break;
+        if (ended)
+            throw refusal(DEPOSIT, reply);
+    }
+    emitOf(deps)('deposit', { step: 'send', sent: 1, bytes: Buffer.byteLength(bytes) });
+};
+/**
+ * One redacted raw copy to the pool: its parts by PUT, then the completion, which the pool checks against the SHA-256 in the path. Every request states the redaction and capture versions and that the person said yes to the raw prompt;
+ * the caller sends only for a person who did. A part sent again replaces itself, and one for an object that stands writes nothing, so a copy that failed halfway is sent again whole.
+ */
+export const postRaw = async (deps, options) => {
+    const { server, token, hash, body, bytes, via } = options;
+    const extra = [['oars-redaction-version', REDACTION_VERSION], ['oars-capture-version', CAPTURE_VERSION], ['oars-raw-consent', 'granted']];
+    const parts = rawParts(body);
+    for (const [at, piece] of parts.entries()) {
+        const reply = await ask(deps, DEPOSIT, server, `/v1/raw/${hash}/parts/${String(at + 1)}`, { token, bytes: piece, method: 'PUT', type: 'application/x-ndjson', extra }, via);
+        if (reply.status !== 200)
+            throw refusal(DEPOSIT, reply);
+    }
+    const done = await ask(deps, DEPOSIT, server, `/v1/raw/${hash}/complete`, { token, body: { parts: parts.length, bytes }, extra }, via);
+    if (done.status !== 200 && done.status !== 201)
+        throw refusal(DEPOSIT, done);
+};
+const rawNote = (consent) => consent === 'granted' ? '' : consent === 'declined' ? 'You answered no to the raw prompt; oars pool raw --consent yes changes it.' : 'None is queued until you answer the raw prompt with oars pool raw --consent yes.';
+/** `deposit --send <hash>`: the send lock the nightly task takes too, then a skip for a hash already in `sent.log`, then the batch lock, then the covenant (accepted here or before, or nothing is sent), then the request. The body is read and hashed again just before the request. */
 const sendBatch = async (deps, options) => {
-    const { outbox, server, token, hash, via } = options;
+    const { outbox, server, token, hash } = options;
+    const via = options.via ?? networkSender(deps, DEPOSIT);
     if (!/^[0-9a-f]{64}$/.test(hash))
         throw usage('--send takes the SHA-256 that oars pool outbox printed: 64 hexadecimal characters.', DEPOSIT);
-    const locked = await lockedBatch(outbox, hash);
-    if (!locked.ok) {
-        const title = lockMessages[locked.code];
-        throw new CliError(title, ExitCode.CONFLICT, DEPOSIT, { code: locked.code, title, status: 409 });
-    }
-    if (via === undefined) {
-        const title = 'Sending a batch waits on the pool covenant: this oars does not send one yet. Nothing was sent.';
-        throw new CliError(title, ExitCode.CONFLICT, DEPOSIT, { code: 'sending_disabled', title, status: 409 });
-    }
-    await openCovenant(deps, server, token, options.acceptCovenant, via);
-    const bytes = await unchangedBytes(locked.file, hash);
-    if (bytes === undefined) {
-        const title = lockMessages.batch_changed;
-        throw new CliError(title, ExitCode.CONFLICT, DEPOSIT, { code: 'batch_changed', title, status: 409 });
+    const refuse = (code, title) => new CliError(title, ExitCode.CONFLICT, DEPOSIT, { code, title, status: 409 });
+    try {
+        return await withSendLock(outbox, async () => {
+            // The rule the nightly follows: a hash `sent.log` holds is not posted again. Read under the lock, so two runs cannot both see it unsent.
+            const sentAt = (await sentLog(outbox)).get(hash);
+            if (sentAt !== undefined)
+                return { document: { ok: true, server, hash, sent: false, sentAt }, text: `already sent ${hash} at ${sentAt}` };
+            const locked = await lockedBatch(outbox, hash);
+            if (!locked.ok)
+                throw refuse(locked.code, lockMessages[locked.code]);
+            await openCovenant(deps, server, token, options.acceptCovenant, via);
+            const bytes = await unchangedBytes(locked.file, hash);
+            if (bytes === undefined)
+                throw refuse('batch_changed', lockMessages.batch_changed);
+            await postBatch(deps, { server, token, bytes, hash, via });
+            await markSent(outbox, hash, deps.now());
+            return { document: { ok: true, server, hash, sent: true }, text: `Sent batch ${hash} to ${server}.` };
+        });
     }
-    let reply = await ask(deps, DEPOSIT, server, POOL_DEPOSITS, { token, bytes }, via);
-    for (let attempt = 1; attempt < SEND_ATTEMPTS && reply.status === 409 && isObject(reply.body) && textIn(reply.body, 'error') === DEPOSIT_CONFLICT; attempt += 1) {
-        reply = await ask(deps, DEPOSIT, server, POOL_DEPOSITS, { token, bytes }, via);
+    catch (error) {
+        if (error instanceof SendLockBusy)
+            throw refuse('send_locked', 'Another oars run is sending from this outbox. Try again in a minute.');
+        throw error;
     }
-    if (reply.status !== 201)
-        throw refusal(DEPOSIT, reply);
-    return { document: { ok: true, server, hash, sent: true }, text: `Sent batch ${hash} to ${server}.` };
 };
 /**
  * `oars pool deposit`: every session in the store that passes both session gates is built into a canonical batch in the outbox, and the command stops: nothing is sent. The gates run before any projection does: the folder
@@ -334,17 +342,25 @@ export const poolDeposit = async (deps, options) => {
     const expired = vocabularyExpired(deps);
     if (expired !== undefined)
         throw expired;
-    // Accepting the covenant is a step of sending, which is disabled in this release: say so rather than exit 0 as though it was accepted.
+    // Accepting the covenant is a step of sending a batch: alone it would exit 0 as though something had been sent under it.
     if (options.send === undefined && options.acceptCovenant !== undefined)
-        throw usage('--accept-covenant belongs to --send, and sending is disabled in this release. Nothing was accepted.', DEPOSIT);
+        throw usage('--accept-covenant belongs to --send: it is accepted as a step of sending a batch. Nothing was accepted.', DEPOSIT);
     const outbox = outboxFolder(runtime.storeDir, server);
     if (options.send !== undefined)
         return sendBatch(deps, { outbox, server, token, hash: options.send, acceptCovenant: options.acceptCovenant, via: options.via });
     const { terms } = exclusionPaths(runtime.projectsRoot, deps.env);
-    const allow = compileAllow((await readSaved(runtime.storeDir)).allow);
+    const saved = await readState(runtime.storeDir);
+    const names = localNames();
+    const allow = compileAllow(saved.allow);
     const discarded = await discardStale(outbox);
-    const counts = { sessions: 0, batched: 0, excluded: { folder: 0, term: 0, noRaw: 0, noTerms: 0, unreadable: 0 }, uncaptured: 0, notEnded: 0, refused: 0, refusedFields: [], discarded, withdrawn: 0 };
+    const counts = { sessions: 0, batched: 0, excluded: { folder: 0, term: 0, noRaw: 0, noTerms: 0, unreadable: 0 }, off: 0, raw: { copies: 0, bytes: 0, masked: 0 }, uncaptured: 0, notEnded: 0, refused: 0, refusedFields: [], discarded, withdrawn: 0 };
+    // The one gate on a name: the operator's redactor and exclusion terms (t881). Everything else the pool classes itself.
+    const gate = { redact: makeRedactor(await readRedactDomains(runtime.storeDir)), terms: await loadTerms(terms) };
     const built = new Set();
+    const rawKept = new Set();
+    // The bytes of the batches written in this pass: telemetry counts them, and the command's own document does not carry them.
+    let builtBytes = 0;
+    const wireTally = newTally();
     const refusedFields = new Set();
     const roots = new Map((await readSkeletons(runtime.storeDir, undefined, true)).skeletons.map(({ session, merkleRoot, leaves }) => [session, { merkleRoot, leaves }]));
     // The sessions that pass both gates and have ended, in the order the store holds them: the windows of a delta are counted over these alone, and the batch of each is built after, once its deltas are known.
@@ -352,6 +368,7 @@ export const poolDeposit = async (deps, options) => {
     // A session that fails a gate loses its earlier batch at once, so a later throw in this pass cannot leave one in the outbox.
     const withdraw = async (key) => {
         counts.withdrawn += await removeBatchOf(outbox, key);
+        await removeRawExcept(outbox, new Set(), key);
     };
     for (const records of sessionsOf(await loadStore(runtime.storeDir), roots)) {
         counts.sessions += 1;
@@ -362,6 +379,12 @@ export const poolDeposit = async (deps, options) => {
             await withdraw(key);
             continue;
         }
+        // A folder switched off sends nothing from now on; what it sent before stays where it went.
+        if (sendsOf(saved.pool, records.main.project) === 'off') {
+            counts.off += 1;
+            await withdraw(key);
+            continue;
+        }
         const verdict = await checkSession(runtime, terms, records.main.project, records.main.session);
         if (verdict.reason !== 'clean') {
             counts.excluded[excludedKey(verdict.reason)] += 1;
@@ -379,36 +402,59 @@ export const poolDeposit = async (deps, options) => {
     // A delta is built from the sessions that passed, carried by the batch of the session that filled its after window, and marked sent only once that batch is written: a deposit that stops first builds it again.
     const placed = ready.flatMap(({ records }) => placedOf(records) ?? []);
     const carried = carriedDeltas(account, placed, await sentDeltas(outbox));
+    // The coverage bands (t871): the operator's two settings, read here, and the team band counted from the local git log. Nothing is looked up, and anything unread goes as unknown.
+    const coverageOf = makeCoverage(runtime, await readCoverageSetting(runtime.storeDir), options.git ?? runGit);
     for (const { records, key } of ready) {
         const delta = carried.get(records.main.session);
-        const skeleton = skeletonOf(records, key, shellReader(vocabulary()), contentHash, delta?.entries);
+        const skeleton = skeletonOf(records, key, shellReader(vocabulary()), contentHash, delta?.entries, gate, await coverageOf(records.main));
         if (skeleton === null) {
             counts.uncaptured += 1;
             await withdraw(key);
             continue;
         }
         const parsed = parseSkeleton(skeleton);
-        if (!parsed.ok) {
+        // The server reads a value outside its lists as `other` and keeps a field it does not know, so the parser no longer refuses either: the emitter is the privacy control, and a batch this client built holds neither.
+        const refusedAt = parsed.ok ? parsed.extensions[0]?.at : parsed.field;
+        if (!parsed.ok || refusedAt !== undefined) {
             counts.refused += 1;
-            refusedFields.add(parsed.field);
+            refusedFields.add(refusedAt ?? '');
             await withdraw(key);
             continue;
         }
-        await writeBatch(outbox, key, parsed.skeleton);
+        builtBytes += (await writeBatch(outbox, key, parsed.skeleton)).bytes;
+        // The raw copy comes only after both gates and only with a yes to the raw prompt and a folder not at skeleton only: each transcript of the session, redacted here, before it is queued.
+        if (rawAllowed(saved.pool, records.main.project)) {
+            let place = 0;
+            for (const file of await transcriptFilesOf(runtime, records.main.project, records.main.session)) {
+                const redacted = redactTranscript(await readFile(file.path, 'utf8'), saved.redactDomains, names);
+                counts.raw.bytes += (await writeRaw(outbox, key, place, redacted.text)).bytes;
+                counts.raw.copies += 1;
+                counts.raw.masked += redacted.masked;
+                place += 1;
+            }
+            rawKept.add(key);
+        }
+        tallyBatch(wireTally, parsed.skeleton);
         await markDeltasSent(outbox, delta?.marks ?? []);
         built.add(key);
         counts.batched += 1;
     }
     // The batch of a session or account this store no longer holds goes too.
     counts.withdrawn += await removeBatchesExcept(outbox, built);
+    await removeRawExcept(outbox, rawKept);
     counts.refusedFields = [...refusedFields].sort();
+    recordWire(deps, wireTally);
     const { folder, term, noRaw, noTerms, unreadable } = counts.excluded;
+    // Counts only: how many sessions each gate kept out, never a folder, a term, or a name.
+    emitOf(deps)('deposit', { step: 'build', sessions: counts.sessions, batched: counts.batched, bytes: builtBytes, excludedFolder: folder, excludedTerm: term, excludedNoRaw: noRaw, excludedNoTerms: noTerms, excludedUnreadable: unreadable, notEnded: counts.notEnded, uncaptured: counts.uncaptured, refused: counts.refused, withdrawn: counts.withdrawn, discarded });
     const lines = [
         `Wrote ${String(counts.batched)} of ${String(counts.sessions)} sessions to the outbox for ${server}. Nothing was sent.`,
         `Kept out by the session gates, never built: ${String(folder)} in a folder not on the allowlist, ${String(term)} for a term, ${String(noRaw)} with no raw to scan, ${String(noTerms)} with no term file, ${String(unreadable)} unreadable.`,
         `Not built: ${String(counts.uncaptured)} captured before steps and frames or with no root, ${String(counts.notEnded)} not ended (a transcript gained a line within ${String(SESSION_IDLE_HOURS)} hours), ${String(counts.refused)} refused by the vocabulary${counts.refusedFields.length === 0 ? '' : ` (${counts.refusedFields.join(', ')})`}.`,
         ...(counts.withdrawn === 0 ? [] : [`Removed ${String(counts.withdrawn)} ${counts.withdrawn === 1 ? 'batch' : 'batches'} of sessions that no longer pass the gates, have not ended, or are no longer in the store.`]),
         ...(discarded === 0 ? [] : [`Discarded ${String(discarded)} ${discarded === 1 ? 'batch' : 'batches'} built under an older vocabulary, field list, or skeleton format.`]),
+        `Raw copies queued: ${String(counts.raw.copies)} (${String(counts.raw.bytes)} bytes, ${String(counts.raw.masked)} values masked). ${rawNote(saved.pool.rawConsent)}`.trim(),
+        ...(counts.off === 0 ? [] : [`${String(counts.off)} ${counts.off === 1 ? 'session' : 'sessions'} in a folder switched off: nothing of them is queued.`]),
         'Run oars pool outbox to print exactly what would be sent.',
     ];
     return { document: { ok: true, server, ...counts }, text: lines.join('\n') };
@@ -426,7 +472,7 @@ const explained = (bytes) => {
     }
     return rows;
 };
-/** `oars pool outbox`: prints each batch as it would be sent, the request line and every header included, with the bearer credential as its size and nothing else masked, then its SHA-256, and records that it was shown. `--explain` adds each field's rule. */
+/** `oars pool outbox`: prints each batch as it would be sent (a batch past one request prints each part's request, in order), the request line and every header included, with the bearer credential as its size and nothing else masked, then its SHA-256, and records that it was shown. `--explain` adds each field's rule. */
 export const poolOutbox = async (deps, options) => {
     const runtime = runtimeOf(deps);
     const server = poolServer(deps.env, OUTBOX, options.server);
@@ -435,35 +481,38 @@ export const poolOutbox = async (deps, options) => {
     const { batches, edited } = await heldBatches(outbox);
     const masker = new Masker();
     masker.add(token);
+    const sent = await sentLog(outbox);
     const shown = [];
     for (const batch of batches) {
         const fields = options.explain === true ? explained(batch.bytes) : undefined;
-        shown.push({ hash: batch.hash, request: printed(outgoing(deps, server, POOL_DEPOSITS, { token, bytes: batch.bytes }), masker), ...(fields === undefined ? {} : { fields }) });
+        // A batch past one request prints every part's request, in the order they are sent.
+        const request = callsOf(token, batch.bytes, batch.hash).map((call) => printed(outgoing(deps, server, POOL_DEPOSITS, call), masker)).join('\n');
+        shown.push({ hash: batch.hash, request, ...(sent.has(batch.hash) ? { sent: sent.get(batch.hash) ?? '' } : {}), ...(fields === undefined ? {} : { fields }) });
     }
+    const { copies } = await heldRaw(outbox);
+    const rawBlocks = copies.flatMap((copy) => rawParts(copy.body).map((part, at, all) => [`raw part ${String(at + 1)}/${String(all.length)} sha256: ${copy.hash} bytes: ${String(Buffer.byteLength(part))}${sent.has(copy.hash) ? ` sent: ${sent.get(copy.hash) ?? ''}` : ''}`, ...(options.explain === true ? [part] : [])].join('\n')));
     for (const { hash } of shown)
         await markShown(outbox, hash, deps.now());
     const stale = await staleBatches(outbox);
-    const blocks = shown.map(({ hash, request, fields }) => [request, `sha256: ${hash}`, ...Object.entries(fields ?? {}).map(([field, row]) => `  ${field}: ${row}`)].join('\n'));
+    const blocks = shown.map(({ hash, request, sent: at, fields }) => [request, `sha256: ${hash}`, ...(at === undefined ? [] : [`sent: ${at}`]), ...Object.entries(fields ?? {}).map(([field, row]) => `  ${field}: ${row}`)].join('\n'));
     const notes = [
         ...(edited === 0 ? [] : [`${String(edited)} ${edited === 1 ? 'batch was' : 'batches were'} edited on disk since ${edited === 1 ? 'it was' : 'they were'} written and ${edited === 1 ? 'is' : 'are'} not printed or sent. Run oars pool deposit to build ${edited === 1 ? 'it' : 'them'} again.`]),
         ...(stale === 0 ? [] : [`${String(stale)} ${stale === 1 ? 'batch was' : 'batches were'} built under an older vocabulary, field list, or skeleton format and will never be sent.`]),
     ];
-    const text = shown.length === 0 ? ['The outbox is empty. Run oars pool deposit first.', ...notes].join('\n') : [...blocks, ...notes].join('\n\n');
-    return { document: { ok: true, server, batches: shown, edited, stale }, text };
+    const text = shown.length === 0 && rawBlocks.length === 0 ? ['The outbox is empty. Run oars pool deposit first.', ...notes].join('\n') : [...blocks, ...rawBlocks, ...notes].join('\n\n');
+    return { document: { ok: true, server, batches: shown, raw: copies.map(({ hash, bytes }) => ({ hash, bytes, ...(sent.has(hash) ? { sent: sent.get(hash) ?? '' } : {}) })), edited, stale }, text };
 };
-/** One folder per harness under the server's, so a fetch for one harness never overwrites another's bundle of the same version. */
-const replicaFile = (storeDir, server, harness, harnessVersion) => join(storeDir, 'pool-replica', contentHash(server), harness, `${encodeURIComponent(harnessVersion)}.json`);
-/** What an earlier fetch kept: the bundle, checked again against the pinned key, and the ETag the pool sent with it. Nothing readable and verified is nothing kept, and the next fetch is then an ordinary one. */
-const keptReplica = async (file, key, harness, harnessVersion) => {
-    let parsed;
-    try {
-        parsed = JSON.parse(await readFile(file, 'utf8'));
-    }
-    catch {
-        return undefined;
-    }
-    const verified = await verifyBundle(parsed, key, harness, harnessVersion);
-    return verified.ok ? { etag: isObject(parsed) ? textIn(parsed, 'etag') : undefined, bundle: verified.bundle } : undefined;
+/**
+ * What the pool says it served a bundle at, printed as the bundle states it: the client holds no floor of its own and applies none (a floor is the pool's deploy setting), so a bundle that states none prints nothing.
+ * A family on a floor of its own is named with it, but only a family this client knows: a key the pool adds later parses and travels in `--json`, and no pool-supplied string reaches the terminal.
+ */
+const floorSentence = (floors) => {
+    if (floors === undefined)
+        return '';
+    const own = Object.entries(floors.families ?? {})
+        .filter(([family]) => FIRST_ORDER_FAMILIES.some((known) => known === family))
+        .map(([family, floor]) => `${family} at ${String(floor)}`);
+    return ` The pool served it at a public floor of ${String(floors.public)} clusters and a disclosure floor of ${String(floors.disclosure)}${own.length === 0 ? '' : `, with ${own.join(' and ')}`}.`;
 };
 /** The weekly cap, as a failure of its own code: the bundle on disk is untouched, and the words say when the pool will answer again. */
 const cappedError = (deps, reply, kept) => {
@@ -475,25 +524,34 @@ const cappedError = (deps, reply, kept) => {
 };
 /** What a fetch asks for when no `--harness` is given, and what it falls back to when no one harness leads the local store. */
 const DEFAULT_HARNESS = 'claude-code';
+/** The harnesses a bundle is served for. `otel` reads the framework into the version slot, so no public bundle is keyed by it (lead 219): it is in `HARNESSES` for the wire, not for a fetch. */
+const FETCHABLE = HARNESSES.filter((name) => name !== 'otel');
 /**
  * The harness a fetch asks for. `--harness` names it, and nothing is printed. Without the flag it is the harness the local store holds the most sessions of, and the choice and the way to
- * change it go to stderr before the request: a fetch never picks silently. An empty store or a tie asks for Claude Code's and says so. It reads counts only, and none of them leaves the machine.
+ * change it go to stderr before the request: a fetch never picks silently. An empty store or a tie asks for Claude Code's and says so. Sessions of a harness with no bundle (`otel`) are not counted. It reads counts only, and none of them leaves the machine.
  */
 const fetchHarness = async (deps, storeDir, named) => {
     if (named !== undefined) {
+        if (named === 'otel')
+            throw usage(`otel has no pool bundle. Choose one of ${FETCHABLE.join(', ')}.`, FETCH);
         if (!isHarness(named))
-            throw usage(`${named} is not a harness the pool serves. Choose one of ${HARNESSES.join(', ')}.`, FETCH);
+            throw usage(`${named} is not a harness the pool serves. Choose one of ${FETCHABLE.join(', ')}.`, FETCH);
         return named;
     }
     const counts = new Map();
-    for (const session of (await loadStore(storeDir)).sessions)
-        counts.set(sessionHarness(session), (counts.get(sessionHarness(session)) ?? 0) + 1);
+    const sessions = (await loadStore(storeDir)).sessions;
+    for (const session of sessions) {
+        const harness = sessionHarness(session);
+        if (FETCHABLE.includes(harness))
+            counts.set(harness, (counts.get(harness) ?? 0) + 1);
+    }
     const [top, runnerUp] = [...counts.entries()].sort((a, b) => b[1] - a[1]);
     const lead = top !== undefined && (runnerUp === undefined || top[1] > runnerUp[1]) ? top : undefined;
     const harness = lead === undefined ? DEFAULT_HARNESS : lead[0];
     const total = [...counts.values()].reduce((sum, count) => sum + count, 0);
-    const why = lead === undefined ? `${total === 0 ? 'your local store is empty' : 'no one harness leads your local store'}, so Claude Code is the default` : `the harness your local store holds the most sessions of (${String(lead[1])} of ${String(total)})`;
-    deps.io.stderr(`No --harness given: fetching ${harness}, ${why}. Pass --harness <name> to fetch another (${HARNESSES.join(', ')}).
+    const none = sessions.length === 0 ? 'your local store is empty' : 'your local store holds no session of a harness the pool serves';
+    const why = lead === undefined ? `${total === 0 ? none : 'no one harness leads your local store'}, so Claude Code is the default` : `the harness your local store holds the most sessions of (${String(lead[1])} of ${String(total)})`;
+    deps.io.stderr(`No --harness given: fetching ${harness}, ${why}. Pass --harness <name> to fetch another (${FETCHABLE.join(', ')}).
 `);
     return harness;
 };
@@ -511,32 +569,101 @@ export const poolFetch = async (deps, options) => {
     const credential = new Masker();
     credential.add(token);
     const reply = await ask(deps, FETCH, server, `/bundle/${harness}/${encodeURIComponent(harnessVersion)}/${encodeURIComponent(VOCAB_VERSION)}`, { token, show: credential, ...(kept?.etag === undefined ? {} : { ifNoneMatch: kept.etag }) });
-    if (reply.status === 429)
+    const emit = emitOf(deps);
+    if (reply.status === 429) {
+        emit('fetch', { harness, fetches: 1, capHits: 1 });
         throw cappedError(deps, reply, kept !== undefined);
+    }
     if (reply.status === 304) {
         // A 304 answers an If-None-Match, so one to a fetch that sent none is a pool that has lost track of what it was asked.
-        if (kept?.etag === undefined)
+        if (kept?.etag === undefined) {
+            emit('fetch', { harness, fetches: 1 });
             throw new CliError('The pool answered 304 to a fetch that named no kept bundle.', ExitCode.UNEXPECTED, FETCH);
-        const { cells, motifs, reads, asOf } = kept.bundle;
+        }
+        const { cells, motifs, reads, asOf, floors } = kept.bundle;
+        emit('fetch', { harness, fetches: 1, etagHits: 1, reads: reads.length });
         return {
-            document: { ok: true, server, harness, harnessVersion, file, unchanged: true, cells: cells.length, motifs: motifs.length, reads: reads.length, asOf },
-            text: `${harness} ${harnessVersion} is unchanged: the bundle at ${file} is still current (${String(reads.length)} reads, ${String(cells.length)} cells, and ${String(motifs.length)} motifs).`,
+            document: { ok: true, server, harness, harnessVersion, file, unchanged: true, cells: cells.length, motifs: motifs.length, reads: reads.length, asOf, ...(floors === undefined ? {} : { floors }) },
+            text: `${harness} ${harnessVersion} is unchanged: the bundle at ${file} is still current (${String(reads.length)} reads, ${String(cells.length)} cells, and ${String(motifs.length)} motifs).${floorSentence(floors)}`,
         };
     }
-    if (reply.status !== 200)
+    // A refused fetch and a bundle that fails verification both reached the pool, so each is a fetch; only the success adds reads.
+    if (reply.status !== 200) {
+        emit('fetch', { harness, fetches: 1 });
         throw refusal(FETCH, reply);
+    }
     const verified = await verifyBundle(reply.body, signingKey, harness, harnessVersion);
     if (!verified.ok) {
         const title = `Refused the bundle: ${verified.reason}.`;
+        emit('fetch', { harness, fetches: 1 });
         throw new CliError(title, ExitCode.INVALID, FETCH, { code: 'bundle_unverified', title });
     }
     const etag = reply.headers.get('etag');
     await mkdir(dirname(file), { recursive: true });
     await writeFile(file, `${JSON.stringify({ ...(isObject(reply.body) ? reply.body : {}), ...(etag === null ? {} : { etag }) })}
 `);
-    const { cells, motifs, reads, asOf } = verified.bundle;
+    const { cells, motifs, reads, asOf, floors } = verified.bundle;
+    emit('fetch', { harness, fetches: 1, reads: reads.length });
     return {
-        document: { ok: true, server, harness, harnessVersion, file, unchanged: false, cells: cells.length, motifs: motifs.length, reads: reads.length, asOf },
-        text: `Fetched ${harness} ${harnessVersion}: ${String(reads.length)} reads, ${String(cells.length)} cells, and ${String(motifs.length)} motifs, verified against the pinned key and kept at ${file}.`,
+        document: { ok: true, server, harness, harnessVersion, file, unchanged: false, cells: cells.length, motifs: motifs.length, reads: reads.length, asOf, ...(floors === undefined ? {} : { floors }) },
+        text: `Fetched ${harness} ${harnessVersion}: ${String(reads.length)} reads, ${String(cells.length)} cells, and ${String(motifs.length)} motifs, verified against the pinned key and kept at ${file}.${floorSentence(floors)}`,
+    };
+};
+const RAW = 'pool-raw';
+/** The raw prompt, in plain words: shown by `oars pool raw` until it is answered, and what the nightly send relies on. Redaction is best effort and the text says so. */
+export const RAW_PROMPT = [
+    'Raw transcripts are separate from the covenant. If you say yes, then once the pool says your account accepted its covenant, the nightly task uploads a redacted copy of each full transcript of every folder you allowed, beside its skeleton, except a folder you set to skeleton only or off.',
+    'Redaction runs on this machine, before anything is queued, and it is best effort: it masks secret shapes it knows, emails, your username, home paths, environment values, and hosts, and it can miss a secret that has no recognizable shape or a name written in prose.',
+    'It also does not catch a host with no dot, a public host name outside a link, quoted file contents and command output, a secret split across lines, images and attachments, a JSON env value, your username written another way, another person\'s hyphenated name in a project folder path, or an IPv6 address with a zone or inside a longer word, and a common-word account name is masked everywhere it appears.',
+    'Say no, or say nothing, and only skeletons go to the pool. Change your mind any time: oars pool raw --consent yes|no. A no stops later sends only; it cannot recall what was already sent.',
+].join('\n');
+/** `oars pool raw`: shows the raw prompt and the answer so far; `--consent yes|no` records the answer; `--folder <entry> --mode all|skeleton|off` sets what one folder may send. */
+export const poolRaw = async (deps, options) => {
+    const runtime = runtimeOf(deps);
+    if (options.consent !== undefined && options.consent !== 'yes' && options.consent !== 'no')
+        throw usage('--consent takes yes or no.', RAW);
+    if ((options.folder === undefined) !== (options.mode === undefined))
+        throw usage('--folder and --mode go together: --folder <folder or glob> --mode all|skeleton|off.', RAW);
+    if (options.mode !== undefined && options.mode !== 'all' && options.mode !== 'skeleton' && options.mode !== 'off')
+        throw usage('--mode takes all, skeleton, or off.', RAW);
+    const { folder, mode } = options;
+    const choices = await updatePoolChoices(runtime.storeDir, (current) => {
+        if (options.consent !== undefined)
+            current.rawConsent = options.consent === 'yes' ? 'granted' : 'declined';
+        if (folder !== undefined) {
+            if (mode === 'all')
+                Reflect.deleteProperty(current.folders, folder);
+            else if (mode === 'skeleton' || mode === 'off')
+                current.folders[folder] = mode;
+        }
+    });
+    const answer = choices.rawConsent === null ? 'not answered yet: only skeletons go to the pool' : choices.rawConsent === 'granted' ? 'yes: redacted raw copies leave' : 'no: only skeletons go to the pool';
+    const switched = Object.entries(choices.folders).map(([entry, set]) => `  ${entry}: ${set === 'off' ? 'nothing sent to the pool' : 'skeletons only'}`);
+    const lines = [...(choices.rawConsent === null ? [RAW_PROMPT, ''] : []), `Raw copies: ${answer}.`, ...(switched.length === 0 ? [] : ['Folders switched down:', ...switched])];
+    return { document: { ok: true, rawConsent: choices.rawConsent, folders: choices.folders }, text: lines.join('\n') };
+};
+const COVERAGE = 'pool-coverage';
+/** A named value from a closed list, or a usage error that lists the members. */
+const oneOfList = (list, flag, value) => {
+    const found = list.find((entry) => entry === value);
+    if (found === undefined)
+        throw usage(`${flag} takes one of ${list.join(', ')}; got ${value}. Nothing was saved.`, COVERAGE);
+    return found;
+};
+/**
+ * `oars pool coverage`: the two bands only the operator can state, kept on this machine and read at deposit. They are what the pool can say about whom a deposit covers, not a filter; `unknown` states nothing.
+ * With no flag it prints what is set.
+ */
+export const poolCoverage = async (deps, options) => {
+    const { storeDir } = runtimeOf(deps);
+    const before = await readCoverageSetting(storeDir);
+    const setting = {
+        orgSize: options.orgSize === undefined ? before.orgSize : oneOfList(ORG_SIZES, '--org-size', options.orgSize),
+        repoVisibility: options.repoVisibility === undefined ? before.repoVisibility : oneOfList(REPO_VISIBILITIES, '--repo-visibility', options.repoVisibility),
     };
+    const changed = options.orgSize !== undefined || options.repoVisibility !== undefined;
+    if (changed)
+        await writeCoverageSetting(storeDir, setting);
+    const lines = [`Organization size: ${setting.orgSize}`, `Repository visibility: ${setting.repoVisibility}`, 'Both stay on this machine until a deposit states them as bands; nothing is looked up. The team band is counted from each repository\'s local git log at deposit.'];
+    return { document: { ok: true, saved: changed, ...setting }, text: lines.join('\n') };
 };

package/dist/sessions/pool-nightly-deps.d.ts

@@ -0,0 +1,3 @@
+import type { RunDeps } from '../cli.js';
+import type { SessionsRuntime } from './store.js';
+export declare const nightlyPoolDeps: (env: NodeJS.ProcessEnv, sessions: SessionsRuntime, version: string, now: () => number) => RunDeps;