@manyoars/oars 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +79 -6
- package/dist/club.d.ts +2 -0
- package/dist/club.js +10 -4
- package/dist/errors.d.ts +2 -0
- package/dist/errors.js +2 -0
- package/dist/sessions/archive.js +1 -1
- package/dist/sessions/card-command.d.ts +2 -2
- package/dist/sessions/card-command.js +7 -5
- package/dist/sessions/card-line.d.ts +6 -2
- package/dist/sessions/card-line.js +6 -3
- package/dist/sessions/cell-increments.d.ts +7 -0
- package/dist/sessions/cell-increments.js +25 -10
- package/dist/sessions/cells.d.ts +7 -0
- package/dist/sessions/cells.js +118 -107
- package/dist/sessions/crew-mark.d.ts +6 -0
- package/dist/sessions/crew-mark.js +22 -0
- package/dist/sessions/hook-index.d.ts +2 -1
- package/dist/sessions/hook-index.js +2 -2
- package/dist/sessions/pool-command.d.ts +47 -1
- package/dist/sessions/pool-command.js +694 -90
- package/dist/sessions/pool-delta.d.ts +5 -0
- package/dist/sessions/pool-delta.js +32 -12
- package/dist/sessions/pool-fold.d.ts +22 -0
- package/dist/sessions/pool-fold.js +65 -0
- package/dist/sessions/pool-outbox.d.ts +22 -0
- package/dist/sessions/pool-outbox.js +78 -11
- package/dist/sessions/pool-send-lock.d.ts +2 -1
- package/dist/sessions/pool-send-lock.js +4 -2
- package/dist/sessions/pool.d.ts +10 -5
- package/dist/sessions/pool.js +16 -15
- package/dist/sessions/read-call-shape.js +3 -2
- package/dist/sessions/read-folds.d.ts +21 -0
- package/dist/sessions/read-folds.js +37 -0
- package/dist/sessions/read-recovery.d.ts +1 -1
- package/dist/sessions/read-recovery.js +6 -5
- package/dist/sessions/read-support.d.ts +4 -1
- package/dist/sessions/recall-consequence.d.ts +2 -1
- package/dist/sessions/recall-consequence.js +1 -2
- package/dist/sessions/recall-reads.d.ts +23 -1
- package/dist/sessions/recall-reads.js +91 -63
- package/dist/sessions/reversion.d.ts +41 -2
- package/dist/sessions/reversion.js +17 -10
- package/dist/sessions/store.d.ts +55 -2
- package/dist/sessions/store.js +312 -61
- package/dist/telemetry/measures.d.ts +5 -0
- package/dist/telemetry/measures.js +16 -2
- package/dist/telemetry/schema.js +3 -1
- package/dist/telemetry/words.d.ts +6 -0
- package/dist/telemetry/words.js +4 -0
- package/package.json +1 -1
|
@@ -5,22 +5,24 @@ import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
|
5
5
|
import { dirname, join, resolve } from 'node:path';
|
|
6
6
|
import { createInterface } from 'node:readline';
|
|
7
7
|
import { emitOf } from '../telemetry/emit.js';
|
|
8
|
+
import { JOIN_STEPS } from '../telemetry/words.js';
|
|
8
9
|
import { CliError, exitCodeForProblem, ExitCode, unreachableError } from '../errors.js';
|
|
9
10
|
import { compileAllow } from './allow.js';
|
|
10
11
|
import { makeCoverage, readCoverageSetting, writeCoverageSetting } from './coverage.js';
|
|
11
12
|
import { checkSession, exclusionPaths, lastWritten, loadTerms, transcriptFilesOf } from './exclusion.js';
|
|
12
13
|
import { localNames, makeRedactor, REDACTION_VERSION, redactTranscript } from './redact.js';
|
|
13
14
|
import { recordSend, recordWire, newTally, tallyBatch } from '../telemetry/wire.js';
|
|
14
|
-
import { discardStale, FIELD_ROWS, fieldsOf, heldAcross, lockedAcross, rescanPulled, machineKeys, markShown, noteMachineKey, outboxFolder, partsOf, pulledAt, removeBatchesExcept, removeBatchOf, staleBatches, unchangedBytes, writeBatch } from './pool-outbox.js';
|
|
15
|
+
import { discardStale, FIELD_ROWS, fieldsOf, heldAcross, lockedAcross, namesHeldAcross, shownHashes, wasShown, rescanPulled, machineKeys, markShown, noteMachineKey, outboxFolder, partsOf, pulledAt, readPulled, readRemoved, removeBatchesExcept, removeBatchOf, staleBatches, unchangedBytes, writeBatch, writePulled, writeRemoved } from './pool-outbox.js';
|
|
15
16
|
import { deriveConsequences } from './recall-consequence.js';
|
|
16
17
|
import { readRecalls } from './recall-object.js';
|
|
17
18
|
import { HARNESSES, hasEnded, ORG_SIZES, REPO_VISIBILITIES, SESSION_IDLE_HOURS, isBundleVersion, isHarness, parseSkeleton, recommendationsOf, sessionsOf, skeletonOf, unpinnedKid, VOCAB_VERSION, verifyBundle } from './pool-wire.js';
|
|
18
|
-
import {
|
|
19
|
+
import { keepCrewNumber } from './crew-mark.js';
|
|
20
|
+
import { carriedDeltasByWindow, heldDeltas, keptDeltas, markDeltasSent, placedOf, sentDeltas } from './pool-delta.js';
|
|
19
21
|
import { Masker, printed, TRANSPORT_HEADERS, waitWords } from './pool-print.js';
|
|
20
22
|
import { rawAllowed, rawStale, sendsOf } from './pool-choices.js';
|
|
21
23
|
import { heldRaw, rawParts, removeRawExcept, writeRaw } from './pool-raw.js';
|
|
22
24
|
import { SendInterrupted, SendLockBusy, watchInterrupt, withSendLock } from './pool-send-lock.js';
|
|
23
|
-
import { markSent, retirePulled, sentLog } from './pool-sent.js';
|
|
25
|
+
import { markSent, retirePulled, sentHashes, sentLog } from './pool-sent.js';
|
|
24
26
|
import { shellReader } from './projection.js';
|
|
25
27
|
import { isVocabularyExpired, REGISTRY_MAX_AGE_DAYS, VOCAB_SNAPSHOT_DATE } from './public-vocabulary.js';
|
|
26
28
|
import { contentHash, FIRST_ORDER_FAMILIES } from './read.js';
|
|
@@ -29,7 +31,7 @@ import { CAPTURE_VERSION, harnessOf as sessionHarness } from './schema.js';
|
|
|
29
31
|
import { machineIdOf } from './machine-ids.js';
|
|
30
32
|
import { machineOf } from './machine.js';
|
|
31
33
|
import { scanCache } from './machine-scan-cache.js';
|
|
32
|
-
import { loadStore, localSalt, readRedactDomains, readSkeletons, readState, updatePoolChoices } from './store.js';
|
|
34
|
+
import { loadStore, paths, scanLog, localSalt, readRedactDomains, readSkeletons, readState, updatePoolChoices } from './store.js';
|
|
33
35
|
import { runGit } from './survival.js';
|
|
34
36
|
import { vocabulary } from './vocab-load.js';
|
|
35
37
|
import { mkdirOwnerOnly, writeOwnerOnly } from './owner-only.js';
|
|
@@ -37,6 +39,7 @@ const LOGIN = 'pool-login';
|
|
|
37
39
|
const LOGOUT = 'pool-logout';
|
|
38
40
|
export const DEPOSIT = 'pool-deposit';
|
|
39
41
|
const OUTBOX = 'pool-outbox';
|
|
42
|
+
const REMOVE = 'pool-remove';
|
|
40
43
|
const POOL_DEPOSITS = '/v1/deposits';
|
|
41
44
|
const FETCH = 'pool-fetch';
|
|
42
45
|
const EXPORT = 'pool-export';
|
|
@@ -74,9 +77,14 @@ export const runtimeOf = (deps) => {
|
|
|
74
77
|
throw new CliError('The session store is not configured in this run.', ExitCode.UNEXPECTED, 'unexpected');
|
|
75
78
|
return deps.sessions;
|
|
76
79
|
};
|
|
77
|
-
|
|
80
|
+
/** The sign-in held for this server that has not expired, or undefined. */
|
|
81
|
+
const heldSignIn = async (deps, storeDir, server, step) => {
|
|
78
82
|
const found = (await loadSignIns(storeDir, step)).find((entry) => entry.server === server);
|
|
79
|
-
|
|
83
|
+
return found === undefined || found.expiresAt <= deps.now() ? undefined : found;
|
|
84
|
+
};
|
|
85
|
+
export const signedIn = async (deps, storeDir, server, step) => {
|
|
86
|
+
const found = await heldSignIn(deps, storeDir, server, step);
|
|
87
|
+
if (found === undefined)
|
|
80
88
|
throw new CliError(`Not signed in to ${server}. Run oars pool login first.`, ExitCode.AUTH, step);
|
|
81
89
|
return found;
|
|
82
90
|
};
|
|
@@ -105,19 +113,22 @@ export const ask = (deps, step, server, path, init, via) => {
|
|
|
105
113
|
/** The pool took the request and then said nothing for `Guard.stallMs`. */
|
|
106
114
|
class PoolStalled extends Error {
|
|
107
115
|
}
|
|
108
|
-
/** A newcomer's words for the network failures a stalled or restarted pool produces; the raw reason follows the sentence in parentheses. */
|
|
116
|
+
/** A newcomer's words for the network failures a stalled or restarted pool produces; the raw reason follows the sentence in parentheses. The flag marks a failure that comes after the request began to go out, where the pool may hold the batch: a refused, unresolved, or connect-timed-out connection fails before any of it was sent, while ETIMEDOUT can also come from a retransmit timeout on an open socket after the body was written. */
|
|
109
117
|
const PLAIN_REASONS = [
|
|
110
|
-
[/ECONNRESET|socket hang up|UND_ERR_SOCKET/, 'the connection was dropped before the pool answered'],
|
|
111
|
-
[/ECONNREFUSED/, 'nothing is listening there'],
|
|
112
|
-
[/ENOTFOUND|EAI_AGAIN/, 'that name did not resolve'],
|
|
113
|
-
[/
|
|
114
|
-
[/
|
|
118
|
+
[/ECONNRESET|socket hang up|UND_ERR_SOCKET/, 'the connection was dropped before the pool answered', true],
|
|
119
|
+
[/ECONNREFUSED/, 'nothing is listening there', false],
|
|
120
|
+
[/ENOTFOUND|EAI_AGAIN/, 'that name did not resolve', false],
|
|
121
|
+
[/UND_ERR_CONNECT_TIMEOUT/, 'the connection timed out', false],
|
|
122
|
+
[/ETIMEDOUT/, 'the connection timed out', true],
|
|
123
|
+
[/EPIPE/, 'the connection closed while the batch was going out', true],
|
|
115
124
|
];
|
|
116
125
|
const NOT_MARKED = 'Nothing was marked sent.';
|
|
117
126
|
const TRY_AGAIN = 'Run the same command to try again.';
|
|
118
127
|
const RETRY = `${NOT_MARKED} ${TRY_AGAIN}`;
|
|
119
128
|
/** Said after a stop or a stall once the request carrying the batch had gone out: the pool may hold it even though nothing here saw the answer. */
|
|
120
129
|
const MAY_HAVE_RECEIVED = 'The pool may already have received it.';
|
|
130
|
+
/** How many sessions a build reads and writes at once: the most whose prompts and tools are in memory together. */
|
|
131
|
+
const BUILD_CHUNK = 100;
|
|
121
132
|
/** What `deposit` says after it writes the outbox: the covenant (the person's consent, docs/covenant 2.1) already lets the background sender send what is queued, with or without this command. A machine's batches are never sent by it. */
|
|
122
133
|
const SENT_LATER = 'Nothing was sent yet. Once the covenant is accepted, oars sends what is queued automatically, in the background.';
|
|
123
134
|
/**
|
|
@@ -131,16 +142,18 @@ export const stoppedText = (received) => [
|
|
|
131
142
|
'To keep a folder out, run oars pool raw --folder <folder> --mode off.',
|
|
132
143
|
'To stop this machine sending, run oars pool logout.',
|
|
133
144
|
].join('\n');
|
|
145
|
+
/** The failure for a send the person or the launcher ended: Ctrl-C (SIGINT) exits 130, the shell convention, and any other way of stopping keeps the unexpected code. The text is the same for all of them. */
|
|
146
|
+
const stopped = (step, cause, received) => new CliError(stoppedText(received), cause.signal === 'SIGINT' ? ExitCode.INTERRUPTED : ExitCode.UNEXPECTED, step);
|
|
134
147
|
/** Why a guarded request ended without an answer, in plain words: a stall, an interrupt, or a network failure whose raw reason is kept after the sentence. The exit code is the unreachable one, as before. */
|
|
135
148
|
const guardedFailure = (step, cause, origin, guard, received) => {
|
|
136
149
|
if (cause instanceof PoolStalled)
|
|
137
150
|
return new CliError(`The pool at ${origin} did not answer within ${String(guard.stallMs / 1000)} seconds. ${received ? `${NOT_MARKED} ${MAY_HAVE_RECEIVED} ${TRY_AGAIN}` : RETRY}`, ExitCode.UNREACHABLE, step);
|
|
138
151
|
if (cause instanceof SendInterrupted)
|
|
139
|
-
return
|
|
152
|
+
return stopped(step, cause, received);
|
|
140
153
|
const unreachable = unreachableError(step, cause, origin);
|
|
141
154
|
const raw = unreachable.message.slice(`Cannot reach the server at ${origin}: `.length);
|
|
142
155
|
const plain = PLAIN_REASONS.find(([pattern]) => pattern.test(raw));
|
|
143
|
-
return plain === undefined ? unreachable : new CliError(`Cannot reach the pool at ${origin}: ${plain[1]} (${raw}). ${RETRY}`, ExitCode.UNREACHABLE, step);
|
|
156
|
+
return plain === undefined ? unreachable : new CliError(`Cannot reach the pool at ${origin}: ${plain[1]} (${raw}). ${received && plain[2] ? `${NOT_MARKED} ${MAY_HAVE_RECEIVED} ${TRY_AGAIN}` : RETRY}`, ExitCode.UNREACHABLE, step);
|
|
144
157
|
};
|
|
145
158
|
/** Prints the request when asked to, then sends exactly those headers and that body. A guarded request also ends when its stall time passes or its signal aborts, and fails in plain words. */
|
|
146
159
|
const send = async (deps, step, request, show, binary = false, guard) => {
|
|
@@ -218,15 +231,42 @@ const closedProof = (reply) => {
|
|
|
218
231
|
${proofLines(account, provedAt)}`;
|
|
219
232
|
return new CliError(refused.message, refused.exitCode, LOGIN, { ...refused.problem, detail }, { account, provedAt });
|
|
220
233
|
};
|
|
234
|
+
/** A promise that rejects with the signal's reason once it aborts, so a wait can be raced against it, and the `dispose` that removes its listener once the wait has settled. */
|
|
235
|
+
const abortion = (signal) => {
|
|
236
|
+
let dispose = () => undefined;
|
|
237
|
+
const promise = new Promise((_resolve, reject) => {
|
|
238
|
+
const why = () => (signal.reason instanceof Error ? signal.reason : new Error('aborted'));
|
|
239
|
+
const onAbort = () => { reject(why()); };
|
|
240
|
+
if (signal.aborted)
|
|
241
|
+
reject(why());
|
|
242
|
+
else {
|
|
243
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
244
|
+
dispose = () => { signal.removeEventListener('abort', onAbort); };
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
return { promise, dispose };
|
|
248
|
+
};
|
|
221
249
|
/** Polls the pool until the person approves, and returns the pool's answer with the credential. Time is counted in the waits asked for, so a machine that slept through the code still gets one poll that says it expired. */
|
|
222
|
-
const awaitCredential = async (deps, server, started, show) => {
|
|
250
|
+
const awaitCredential = async (deps, server, started, show, signal) => {
|
|
223
251
|
let { intervalMs } = started;
|
|
224
252
|
let failures = 0;
|
|
225
253
|
let lastFailure;
|
|
226
254
|
for (let waited = 0; waited < started.expiresMs;) {
|
|
227
255
|
// A failed poll (a network error or a 5xx) is the provider or the network having a bad minute, not the person's answer: wait longer each time, up to the cap, and ask again until the code's own expiry.
|
|
228
256
|
const wait = Math.min(intervalMs * 2 ** failures, Math.max(intervalMs, MAX_BACKOFF_MS));
|
|
229
|
-
|
|
257
|
+
if (signal === undefined)
|
|
258
|
+
await deps.sleep(wait);
|
|
259
|
+
else {
|
|
260
|
+
const aborted = abortion(signal);
|
|
261
|
+
try {
|
|
262
|
+
await Promise.race([deps.sleep(wait), aborted.promise]);
|
|
263
|
+
}
|
|
264
|
+
finally {
|
|
265
|
+
aborted.dispose();
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// A sleep that finished first still ends the poll once the signal has fired.
|
|
269
|
+
signal?.throwIfAborted();
|
|
230
270
|
waited += wait;
|
|
231
271
|
// `ask` rejects only with the unreachable failure of `send`.
|
|
232
272
|
const [settled] = await Promise.allSettled([ask(deps, LOGIN, server, '/v1/login/poll', { body: { handle: started.handle }, show })]);
|
|
@@ -290,7 +330,7 @@ export const poolLogin = async (deps, options) => {
|
|
|
290
330
|
await deps.openBrowser(started.verificationUrl).catch(() => {
|
|
291
331
|
deps.io.stderr('Could not open a browser; open the link above yourself.\n');
|
|
292
332
|
});
|
|
293
|
-
const reply = await awaitCredential(deps, server, started, show);
|
|
333
|
+
const reply = await awaitCredential(deps, server, started, show, options.signal);
|
|
294
334
|
const signIn = signInOf(isObject(reply.body) ? { ...reply.body, server } : undefined);
|
|
295
335
|
if (signIn === undefined)
|
|
296
336
|
throw new CliError('The pool answered a sign-in with no account, token, or signing key.', ExitCode.UNEXPECTED, LOGIN);
|
|
@@ -369,8 +409,23 @@ export const OFFER_BEFORE = [
|
|
|
369
409
|
'The covenant is the whole deal. Its full text is linked below. Read it before you type.',
|
|
370
410
|
];
|
|
371
411
|
export const OFFER_AFTER = ['Accepted. Sending your batch now.', 'Run oars pool fetch --help for your first pool download.'];
|
|
412
|
+
const readerOf = (deps, signal) => {
|
|
413
|
+
const lines = createInterface({ input: runtimeOf(deps).stdin, terminal: false, ...(signal === undefined ? {} : { signal }) });
|
|
414
|
+
const iterator = lines[Symbol.asyncIterator]();
|
|
415
|
+
return {
|
|
416
|
+
next: async () => {
|
|
417
|
+
const next = await iterator.next();
|
|
418
|
+
if (signal?.aborted === true)
|
|
419
|
+
throw signal.reason;
|
|
420
|
+
return next.done === true ? undefined : next.value;
|
|
421
|
+
},
|
|
422
|
+
close: () => {
|
|
423
|
+
lines.close();
|
|
424
|
+
},
|
|
425
|
+
};
|
|
426
|
+
};
|
|
372
427
|
/** A send stops here when the server is closed or wants a covenant accepted, before the batch is read for the last time. */
|
|
373
|
-
const openCovenant = async (deps, server, token, accept, region, via, signal) => {
|
|
428
|
+
const openCovenant = async (deps, server, token, accept, region, via, signal, offTerminal = () => Promise.resolve('run this command from one'), refuseAtTerminal = false, shared) => {
|
|
374
429
|
const reply = await ask(deps, DEPOSIT, server, '/v1/covenant', { token }, via);
|
|
375
430
|
if (reply.status !== 200 || !isObject(reply.body))
|
|
376
431
|
throw refusal(DEPOSIT, reply);
|
|
@@ -385,47 +440,48 @@ const openCovenant = async (deps, server, token, accept, region, via, signal) =>
|
|
|
385
440
|
if (accept !== undefined && accept !== version)
|
|
386
441
|
throw usage(`--accept-covenant names covenant ${accept}, but the covenant of this pool is ${version}. Nothing was accepted.`, DEPOSIT);
|
|
387
442
|
if (Reflect.get(reply.body, 'accepted') === true)
|
|
388
|
-
return;
|
|
443
|
+
return false;
|
|
389
444
|
const notAccepted = (title) => new CliError(title, ExitCode.AUTH, DEPOSIT, { code: 'covenant_not_accepted', title, status: 403 });
|
|
390
445
|
const screen = screenIn(reply.body);
|
|
391
446
|
if (screen === undefined)
|
|
392
447
|
throw notAccepted(`The pool has no text for covenant ${version} yet, so there is nothing to show and nothing can be accepted. Nothing was accepted.`);
|
|
393
448
|
// Assent is a person typing the phrase the screen states. A flag or a script on stdin is not one, so off a terminal nothing is asked and nothing is accepted.
|
|
394
|
-
|
|
395
|
-
|
|
449
|
+
// A bulk send never shows the screen: a person reads and accepts through the one-batch command, so a terminal is refused the same way.
|
|
450
|
+
if (refuseAtTerminal || !isTerminal(deps)) {
|
|
451
|
+
throw notAccepted(`Covenant ${version} is not accepted for this account. Accepting it takes a person at a terminal who reads it and types its phrase: ${await offTerminal()}. Nothing was accepted.`);
|
|
396
452
|
}
|
|
397
453
|
deps.io.stderr(`${OFFER_BEFORE.join('\n')}\n`);
|
|
398
454
|
deps.io.stderr(`\n${screen.text}\n\n> `);
|
|
399
455
|
emitOf(deps)('covenant-shown', { version });
|
|
400
|
-
// One reader for both lines: a second one opened after the first closed would lose a line the first had already taken from a pipe.
|
|
401
|
-
|
|
402
|
-
const lines = createInterface({ input: runtimeOf(deps).stdin, terminal: false, ...(signal === undefined ? {} : { signal }) });
|
|
403
|
-
const iterator = lines[Symbol.asyncIterator]();
|
|
404
|
-
const nextLine = async () => {
|
|
405
|
-
const next = await iterator.next();
|
|
406
|
-
if (signal?.aborted === true)
|
|
407
|
-
throw signal.reason;
|
|
408
|
-
return next.done === true ? undefined : next.value;
|
|
409
|
-
};
|
|
456
|
+
// One reader for both lines: a second one opened after the first closed would lose a line the first had already taken from a pipe. A join passes its own, which also served the questions before this one.
|
|
457
|
+
const lines = shared ?? readerOf(deps, signal);
|
|
410
458
|
let answered = region;
|
|
411
459
|
try {
|
|
412
|
-
const typed = await
|
|
460
|
+
const typed = await lines.next();
|
|
413
461
|
if (typed?.trim() !== screen.phrase)
|
|
414
462
|
throw notAccepted(`${typed === undefined ? 'No phrase was typed' : 'That was not the phrase'}. To accept covenant ${version}, type exactly: ${screen.phrase}. Nothing was accepted.`);
|
|
415
463
|
if (region === undefined) {
|
|
416
464
|
deps.io.stderr(`${REGION_QUESTION} > `);
|
|
417
|
-
answered = regionOf(await
|
|
465
|
+
answered = regionOf(await lines.next());
|
|
418
466
|
}
|
|
419
467
|
}
|
|
420
468
|
finally {
|
|
421
|
-
|
|
469
|
+
if (shared === undefined)
|
|
470
|
+
lines.close();
|
|
422
471
|
}
|
|
423
472
|
if (answered === undefined)
|
|
424
473
|
throw notAccepted('Answer yes or no: are you in the EU, the EEA, or the UK? Nothing was accepted.');
|
|
425
474
|
const body = { version, textSha256: createHash('sha256').update(screen.text).digest('hex'), input: 'typed', client: `oars/${deps.version}`, euEeaUk: answered };
|
|
426
475
|
const accepted = await ask(deps, DEPOSIT, server, '/v1/covenant/accept', { token, body }, via);
|
|
427
|
-
if (accepted.status !== 200)
|
|
428
|
-
|
|
476
|
+
if (accepted.status !== 200) {
|
|
477
|
+
const refused = refusal(DEPOSIT, accepted);
|
|
478
|
+
// A join keeps the answer to the region question for its reruns, so a mistyped yes would be refused again every time: name the way to correct it.
|
|
479
|
+
if (shared === undefined || refused.problem.code !== 'region_closed')
|
|
480
|
+
throw refused;
|
|
481
|
+
// A no that was refused on the request's origin is not a mistake to correct.
|
|
482
|
+
const title = answered === 'yes' ? `${refused.message} If you answered yes by mistake, run oars pool join --in-eu-eea-uk no to correct it.` : refused.message;
|
|
483
|
+
throw new CliError(title, refused.exitCode, DEPOSIT, { code: 'region_closed', title, status: 403 });
|
|
484
|
+
}
|
|
429
485
|
deps.io.stderr(`${OFFER_AFTER.join('\n')}\n`);
|
|
430
486
|
// A yes to the raw prompt belongs to the covenant it was given under, and a covenant can change what that prompt covers: accepting another version asks the raw question again.
|
|
431
487
|
const reset = { done: false };
|
|
@@ -438,6 +494,7 @@ const openCovenant = async (deps, server, token, accept, region, via, signal) =>
|
|
|
438
494
|
});
|
|
439
495
|
if (reset.done)
|
|
440
496
|
deps.io.stderr('Your earlier yes to the raw prompt was given under another covenant, so it no longer applies. Run oars pool raw to read the prompt and answer again.\n');
|
|
497
|
+
return true;
|
|
441
498
|
};
|
|
442
499
|
const screenIn = (body) => {
|
|
443
500
|
const screen = Reflect.get(body, 'screen');
|
|
@@ -570,7 +627,138 @@ const sendBatch = async (deps, options) => {
|
|
|
570
627
|
if (error instanceof SendLockBusy)
|
|
571
628
|
throw refuse('send_locked', busyLine(error.detail));
|
|
572
629
|
if (error instanceof SendInterrupted)
|
|
573
|
-
throw
|
|
630
|
+
throw stopped(DEPOSIT, error, false);
|
|
631
|
+
throw error;
|
|
632
|
+
}
|
|
633
|
+
finally {
|
|
634
|
+
interrupt.stop();
|
|
635
|
+
}
|
|
636
|
+
};
|
|
637
|
+
const reasonOf = (signal) => (signal?.reason instanceof Error ? signal.reason : new Error('Interrupted.'));
|
|
638
|
+
export const sleeps = (ms, signal) => new Promise((resolve, reject) => {
|
|
639
|
+
// Ctrl-C during a wait ends it with the interrupt, instead of being ignored until the wait is over.
|
|
640
|
+
if (signal?.aborted === true) {
|
|
641
|
+
reject(reasonOf(signal));
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
const abort = () => {
|
|
645
|
+
clearTimeout(timer);
|
|
646
|
+
reject(reasonOf(signal));
|
|
647
|
+
};
|
|
648
|
+
const timer = setTimeout(() => {
|
|
649
|
+
signal?.removeEventListener('abort', abort);
|
|
650
|
+
resolve();
|
|
651
|
+
}, ms);
|
|
652
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
653
|
+
});
|
|
654
|
+
const retrying = (via, pause, signal) => async (request) => {
|
|
655
|
+
for (let attempt = 1;; attempt += 1) {
|
|
656
|
+
const reply = await via(request);
|
|
657
|
+
if ((reply.status !== 429 && reply.status < 500) || attempt >= SEND_ATTEMPTS)
|
|
658
|
+
return reply;
|
|
659
|
+
const named = Number(reply.headers.get('retry-after'));
|
|
660
|
+
await pause(Number.isFinite(named) && named > 0 ? Math.min(named, 30) * 1000 : 1000 * 2 ** (attempt - 1), signal);
|
|
661
|
+
}
|
|
662
|
+
};
|
|
663
|
+
/** Batches between two progress lines of `--send-all`. */
|
|
664
|
+
const PROGRESS_EVERY = 100;
|
|
665
|
+
/**
|
|
666
|
+
* `deposit --send-all`: the lock and the covenant of `--send`, once, then every batch of the outbox in name order, one file read at a time. A batch whose hash `sent.log` holds is not sent again, which is what makes a rerun send only what is left (the log is
|
|
667
|
+
* written after each batch, so a run that stops loses at most the one in flight, and the pool reads a batch sent again as the same deposit). A batch that `oars pool outbox` never printed, that changed on disk, or that a pulled-batch rule holds back is skipped and counted.
|
|
668
|
+
* The first batch that fails ends the run: the counts print, and the error goes up with its own exit code. Only counts and hashes print, never a session's content.
|
|
669
|
+
*/
|
|
670
|
+
const sendAll = async (deps, options) => {
|
|
671
|
+
const { outbox, server, token } = options;
|
|
672
|
+
const refuse = (code, title) => new CliError(title, ExitCode.CONFLICT, DEPOSIT, { code, title, status: 409 });
|
|
673
|
+
const interrupt = watchInterrupt();
|
|
674
|
+
const net = options.via ?? networkSender(deps, DEPOSIT, { stallMs: SEND_STALL_MS, signal: interrupt.signal });
|
|
675
|
+
const via = retrying(net, options.pause ?? sleeps, interrupt.signal);
|
|
676
|
+
try {
|
|
677
|
+
return await withSendLock(outbox, async () => {
|
|
678
|
+
const scan = await rescanPulled(outbox, await currentTerms(deps));
|
|
679
|
+
const done = await sentHashes(outbox);
|
|
680
|
+
const shown = await shownHashes(outbox);
|
|
681
|
+
// The one-batch command that shows the screen: the first batch that was shown and is not sent.
|
|
682
|
+
const oneBatch = async () => {
|
|
683
|
+
for await (const named of namesHeldAcross(outbox)) {
|
|
684
|
+
if (shown.has(named.hash) && !done.has(named.hash) && !(named.pulled && !scan.open) && (await unchangedBytes(named.file, named.hash)) !== undefined)
|
|
685
|
+
return `run oars pool deposit --send ${named.hash} at a terminal to read it and accept`;
|
|
686
|
+
}
|
|
687
|
+
return 'run oars pool deposit --send <hash> at a terminal to read it and accept';
|
|
688
|
+
};
|
|
689
|
+
await openCovenant(deps, server, token, options.acceptCovenant, options.region, via, interrupt.signal, oneBatch, true);
|
|
690
|
+
const tally = { sent: 0, skipped: 0, alreadySent: 0, failed: 0, left: 0 };
|
|
691
|
+
const handled = new Set();
|
|
692
|
+
const line = () => `${String(tally.sent)} sent, ${String(tally.skipped)} skipped, ${String(tally.alreadySent)} already sent, ${String(tally.failed)} failed, ${String(tally.left)} left`;
|
|
693
|
+
let ended;
|
|
694
|
+
try {
|
|
695
|
+
for await (const named of namesHeldAcross(outbox)) {
|
|
696
|
+
handled.add(named.hash);
|
|
697
|
+
if (done.has(named.hash)) {
|
|
698
|
+
tally.alreadySent += 1;
|
|
699
|
+
}
|
|
700
|
+
else {
|
|
701
|
+
// Read now, hashed again, and sent: the one batch in memory. A file that no longer hashes to its name is the "changed" case.
|
|
702
|
+
const bytes = shown.has(named.hash) && !(named.pulled && !scan.open) ? await unchangedBytes(named.file, named.hash) : undefined;
|
|
703
|
+
if (bytes === undefined) {
|
|
704
|
+
tally.skipped += 1;
|
|
705
|
+
}
|
|
706
|
+
else {
|
|
707
|
+
try {
|
|
708
|
+
await postBatch(deps, { server, token, bytes, hash: named.hash, via });
|
|
709
|
+
}
|
|
710
|
+
catch (error) {
|
|
711
|
+
tally.failed = 1;
|
|
712
|
+
throw error;
|
|
713
|
+
}
|
|
714
|
+
await markSent(outbox, named.hash, deps.now());
|
|
715
|
+
tally.sent += 1;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
const count = tally.sent + tally.skipped + tally.alreadySent;
|
|
719
|
+
if (count % PROGRESS_EVERY === 0)
|
|
720
|
+
deps.io.stderr(`${String(count)} batches handled: ${line()}.
|
|
721
|
+
`);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
catch (error) {
|
|
725
|
+
ended = { error };
|
|
726
|
+
}
|
|
727
|
+
// A pulled batch whose hash is in `sent.log` is done, however the run ended; an error of the loop wins over one of the retiring.
|
|
728
|
+
try {
|
|
729
|
+
await retirePulled(outbox, DEPOSIT);
|
|
730
|
+
}
|
|
731
|
+
catch (error) {
|
|
732
|
+
if (ended === undefined)
|
|
733
|
+
throw error;
|
|
734
|
+
}
|
|
735
|
+
if (ended !== undefined) {
|
|
736
|
+
// Shown, not sent, not reached: counted by name alone, without reading a file. A batch a rerun would skip (never shown) is not left.
|
|
737
|
+
for await (const named of namesHeldAcross(outbox)) {
|
|
738
|
+
if (!handled.has(named.hash) && !done.has(named.hash) && shown.has(named.hash) && !(named.pulled && !scan.open))
|
|
739
|
+
tally.left += 1;
|
|
740
|
+
}
|
|
741
|
+
deps.io.stderr(`Stopped: ${line()}. Run the same command again to send what is left.
|
|
742
|
+
`);
|
|
743
|
+
// The counts reach the --json failure document too, not only stderr.
|
|
744
|
+
if (ended.error instanceof CliError)
|
|
745
|
+
throw new CliError(ended.error.message, ended.error.exitCode, ended.error.step, ended.error.problem, { ...ended.error.details, tally });
|
|
746
|
+
throw ended.error;
|
|
747
|
+
}
|
|
748
|
+
return { document: { ok: true, server, ...tally }, text: `Done: ${line()}.` };
|
|
749
|
+
}, options.lock?.times, options.lock?.fs, {
|
|
750
|
+
onWait: (waiting) => {
|
|
751
|
+
deps.io.stderr(`${waitingLine(waiting)}
|
|
752
|
+
`);
|
|
753
|
+
},
|
|
754
|
+
signal: interrupt.signal,
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
catch (error) {
|
|
758
|
+
if (error instanceof SendLockBusy)
|
|
759
|
+
throw refuse('send_locked', busyLine(error.detail));
|
|
760
|
+
if (error instanceof SendInterrupted)
|
|
761
|
+
throw stopped(DEPOSIT, error, false);
|
|
574
762
|
throw error;
|
|
575
763
|
}
|
|
576
764
|
finally {
|
|
@@ -604,6 +792,68 @@ export const busyLine = ({ holder, local, ageMs, untouchedMs, waitedMs, staleMs
|
|
|
604
792
|
return `${held} A retry succeeds as soon as pid ${String(holder.pid)} ends.`;
|
|
605
793
|
return `${held} Run the same command again once it ends. If it has hung, oars takes the lock over about ${span(Math.max(1, staleMs - untouchedMs))} from now.`;
|
|
606
794
|
};
|
|
795
|
+
/** The session id behind an outbox key, found by deriving the key of each session in the store the way `poolDeposit` does: under every account signed in to this server, or under the machine's environment. `undefined` when the store no longer holds the session. */
|
|
796
|
+
const sessionOfKey = async (deps, storeDir, server, key) => {
|
|
797
|
+
const machine = machineOf(deps.env);
|
|
798
|
+
const accounts = machine === undefined ? (await loadSignIns(storeDir, REMOVE)).filter((entry) => entry.server === server).map((entry) => entry.account) : [];
|
|
799
|
+
const keysOf = (session) => (machine === undefined ? accounts.map((account) => contentHash([account, session])) : [machineKeyOf(machine, session)]);
|
|
800
|
+
return (await loadStore(storeDir)).sessions.map((observed) => observed.session).find((session) => keysOf(session).includes(key));
|
|
801
|
+
};
|
|
802
|
+
/**
|
|
803
|
+
* `oars pool remove <hash>`: takes one batch out of this machine's outbox, the batch `oars pool outbox` printed under that hash. It takes the send lock without waiting, because a send in progress may be reading the very file, and refuses while another run holds it.
|
|
804
|
+
* It reads nothing from the pool and sends nothing, so it needs no sign-in, and it changes only files here. The session of a local batch goes on `removed.json`, which `poolDeposit` (and so the background sender) honors: nothing of it is built, queued, or sent until `--restore <hash>` takes it off. One a cloud session held is admitted again only by `oars pool hold pull`.
|
|
805
|
+
*/
|
|
806
|
+
export const poolRemove = async (deps, hash, options) => {
|
|
807
|
+
const runtime = runtimeOf(deps);
|
|
808
|
+
const server = poolServer(deps.env, REMOVE, options.server);
|
|
809
|
+
if (!/^[0-9a-f]{64}$/.test(hash))
|
|
810
|
+
throw usage('remove takes the SHA-256 that oars pool outbox printed: 64 hexadecimal characters.', REMOVE);
|
|
811
|
+
const outbox = outboxFolder(runtime.storeDir, server);
|
|
812
|
+
const refuse = (code, title) => new CliError(title, ExitCode.CONFLICT, REMOVE, { code, title, status: 409 });
|
|
813
|
+
try {
|
|
814
|
+
return await withSendLock(outbox, async () => {
|
|
815
|
+
if (options.restore === true) {
|
|
816
|
+
const removed = await readRemoved(outbox, REMOVE);
|
|
817
|
+
const key = Object.keys(removed).find((candidate) => removed[candidate]?.hash === hash);
|
|
818
|
+
if (key === undefined)
|
|
819
|
+
throw refuse('removal_unknown', 'No removed batch has that hash, so nothing was restored. The hash is the one oars pool remove printed when it removed the batch.');
|
|
820
|
+
Reflect.deleteProperty(removed, key);
|
|
821
|
+
await writeRemoved(outbox, removed);
|
|
822
|
+
return { document: { ok: true, server, hash, restored: true }, text: `Restored the session of batch ${hash} for ${server}. Nothing was sent. A later oars pool deposit builds it again while the session passes the gates.` };
|
|
823
|
+
}
|
|
824
|
+
const removed = await readRemoved(outbox, REMOVE);
|
|
825
|
+
const found = (await heldAcross(outbox)).batches.find((batch) => batch.hash === hash);
|
|
826
|
+
if (found === undefined)
|
|
827
|
+
throw refuse('batch_unknown', 'No batch with that hash is in the outbox, so nothing was removed. Run oars pool outbox to list them.');
|
|
828
|
+
// The session goes on the list first: the build pass would otherwise rebuild it on its next run, the background sender's included, and send what the person withdrew (GDPR Article 7(3)). A list that cannot be read has already failed above, with the batch untouched.
|
|
829
|
+
const session = found.pulled ? undefined : await sessionOfKey(deps, runtime.storeDir, server, found.key);
|
|
830
|
+
if (!found.pulled)
|
|
831
|
+
await writeRemoved(outbox, { ...removed, [found.key]: session === undefined ? { hash } : { hash, session } });
|
|
832
|
+
await rm(found.file, { force: true });
|
|
833
|
+
// The session's queued raw copies go in the same lock: the background sender posts every raw copy it holds without looking for its batch, and a batch the person withdrew must take the session's transcript with it (GDPR Article 7(3)). A pulled batch never had one here.
|
|
834
|
+
const rawGone = await removeRawExcept(outbox, new Set(), found.key);
|
|
835
|
+
if (found.pulled) {
|
|
836
|
+
const pulled = await readPulled(outbox, REMOVE);
|
|
837
|
+
Reflect.deleteProperty(pulled, found.key);
|
|
838
|
+
await writePulled(outbox, pulled);
|
|
839
|
+
}
|
|
840
|
+
const sentAt = (await sentLog(outbox)).get(hash);
|
|
841
|
+
const text = [
|
|
842
|
+
`Removed batch ${hash} from the outbox for ${server}. Nothing was sent, and nothing at the pool changed.`,
|
|
843
|
+
...(rawGone === 0 ? [] : [`${String(rawGone)} queued raw ${rawGone === 1 ? 'copy' : 'copies'} of the same session went with it.`]),
|
|
844
|
+
...(sentAt === undefined ? [] : [`It was already sent at ${sentAt}; removing it here does not take it back from the pool.`]),
|
|
845
|
+
...(found.pulled ? ['It came from a cloud session: only oars pool hold pull admits it again.'] : [`Its session stays out of every later oars pool deposit, the background sender's included, until you run oars pool remove ${hash} --restore.`]),
|
|
846
|
+
...(found.pulled || session !== undefined ? [] : ['No session in the store matches it, so the exclusion holds for this account and machine settings only.']),
|
|
847
|
+
].join('\n');
|
|
848
|
+
return { document: { ok: true, server, hash, removed: true }, text };
|
|
849
|
+
}, { ...options.lock?.times, waitMs: 0 }, options.lock?.fs);
|
|
850
|
+
}
|
|
851
|
+
catch (error) {
|
|
852
|
+
if (error instanceof SendLockBusy)
|
|
853
|
+
throw refuse('send_locked', `Another oars run${error.detail.holder === undefined ? '' : ` (pid ${String(error.detail.holder.pid)})`} holds this outbox's send lock, so nothing was removed. Run the same command again once it ends.`);
|
|
854
|
+
throw error;
|
|
855
|
+
}
|
|
856
|
+
};
|
|
607
857
|
/** The covenant version the pool names, or `undefined` when it cannot be asked or names none. The route is public, so no credential goes with it. */
|
|
608
858
|
const covenantOrUndefined = async (deps, server) => {
|
|
609
859
|
try {
|
|
@@ -627,14 +877,21 @@ export const poolDeposit = async (deps, options) => {
|
|
|
627
877
|
const server = poolServer(deps.env, DEPOSIT, options.server);
|
|
628
878
|
// A machine (`OARS_HOST_CLASS`, see machine.ts) builds at every Stop and sends nothing from here, so it signs nobody in: the account part of its batch key is its own name, and the holding route's `id` replaces it.
|
|
629
879
|
const machine = machineOf(deps.env);
|
|
630
|
-
if (
|
|
631
|
-
throw usage('
|
|
880
|
+
if (options.send !== undefined && options.sendAll === true)
|
|
881
|
+
throw usage('--send and --send-all together: --send sends the one batch whose hash you name, and --send-all sends every batch that oars pool outbox printed. Use one. Nothing was sent.', DEPOSIT);
|
|
882
|
+
if (machine !== undefined && (options.send !== undefined || options.sendAll === true))
|
|
883
|
+
throw usage('A machine builds batches into its outbox and sends none: --send and --send-all are for a signed-in person.', DEPOSIT);
|
|
632
884
|
const { account, token } = machine === undefined ? await signedIn(deps, runtime.storeDir, server, DEPOSIT) : { account: machineAccount(machine), token: '' };
|
|
633
885
|
// A client past `registry_max_age` builds no batch (spec Section 1.3): its sessions wait in `A` and nothing is lost. This comes before anything is written or sent.
|
|
634
886
|
const expired = vocabularyExpired(deps);
|
|
635
887
|
if (expired !== undefined)
|
|
636
888
|
throw expired;
|
|
637
889
|
// Accepting the covenant is a step of sending a batch: alone it would exit 0 as though something had been sent under it.
|
|
890
|
+
const accepting = 'accept with oars pool deposit --send <hash>, which shows the covenant screen';
|
|
891
|
+
if (options.sendAll === true && options.acceptCovenant !== undefined)
|
|
892
|
+
throw usage(`--accept-covenant belongs to --send: --send-all never accepts a covenant, so ${accepting}. Nothing was accepted.`, DEPOSIT);
|
|
893
|
+
if (options.sendAll === true && options.inEuEeaUk !== undefined)
|
|
894
|
+
throw usage(`${REGION_FLAG} belongs to --send: --send-all never accepts a covenant, so ${accepting}. Nothing was accepted.`, DEPOSIT);
|
|
638
895
|
if (options.send === undefined && options.acceptCovenant !== undefined)
|
|
639
896
|
throw usage('--accept-covenant belongs to --send: it names the covenant version expected when a person accepts before a send. Nothing was accepted.', DEPOSIT);
|
|
640
897
|
if (options.send === undefined && options.inEuEeaUk !== undefined)
|
|
@@ -644,6 +901,8 @@ export const poolDeposit = async (deps, options) => {
|
|
|
644
901
|
throw usage(`${REGION_FLAG} takes yes or no, not "${options.inEuEeaUk}". Nothing was accepted.`, DEPOSIT);
|
|
645
902
|
const outbox = outboxFolder(runtime.storeDir, server);
|
|
646
903
|
await mkdirOwnerOnly(outbox);
|
|
904
|
+
if (options.sendAll === true)
|
|
905
|
+
return sendAll(deps, { outbox, server, token, acceptCovenant: options.acceptCovenant, region, via: options.via, lock: options.lock, pause: options.pause });
|
|
647
906
|
if (options.send !== undefined)
|
|
648
907
|
return sendBatch(deps, { outbox, server, token, hash: options.send, acceptCovenant: options.acceptCovenant, region, via: options.via, lock: options.lock });
|
|
649
908
|
const { terms } = exclusionPaths(runtime.projectsRoot, deps.env);
|
|
@@ -655,7 +914,7 @@ export const poolDeposit = async (deps, options) => {
|
|
|
655
914
|
const injectedKey = `injected-block:${(await localSalt(runtime.storeDir)).salt}`;
|
|
656
915
|
const allow = compileAllow(saved.allow);
|
|
657
916
|
const discarded = await discardStale(outbox);
|
|
658
|
-
const counts = { sessions: 0, batched: 0, excluded: { folder: 0, term: 0, noRaw: 0, noTerms: 0, unreadable: 0 }, off: 0, raw: { copies: 0, bytes: 0, masked: 0 }, uncaptured: 0, notEnded: 0, refused: 0, refusedFields: [], discarded, withdrawn: 0 };
|
|
917
|
+
const counts = { sessions: 0, batched: 0, excluded: { folder: 0, term: 0, noRaw: 0, noTerms: 0, unreadable: 0 }, off: 0, raw: { copies: 0, bytes: 0, masked: 0 }, uncaptured: 0, removedByPerson: 0, notEnded: 0, refused: 0, refusedFields: [], discarded, withdrawn: 0 };
|
|
659
918
|
// The one gate on a name: the operator's redactor and exclusion terms (t881). Everything else the pool classes itself.
|
|
660
919
|
const termList = await loadTerms(terms);
|
|
661
920
|
const gate = { redact: makeRedactor(await readRedactDomains(runtime.storeDir)), terms: termList };
|
|
@@ -669,6 +928,7 @@ export const poolDeposit = async (deps, options) => {
|
|
|
669
928
|
const refusedFields = new Set();
|
|
670
929
|
const roots = new Map((await readSkeletons(runtime.storeDir, undefined, true)).skeletons.map(({ session, merkleRoot, leaves }) => [session, { merkleRoot, leaves }]));
|
|
671
930
|
// The sessions that pass both gates and have ended, in the order the store holds them: the windows of a delta are counted over these alone, and the batch of each is built after, once its deltas are known.
|
|
931
|
+
// Only the session records are held for the gates; a session's prompts and tools are read again when its window or its batch needs them (BUILD_CHUNK at a time), so the history is never in memory at once.
|
|
672
932
|
const ready = [];
|
|
673
933
|
// A session that fails a gate loses its earlier batch at once, so a later throw in this pass cannot leave one in the outbox.
|
|
674
934
|
const withdraw = async (key) => {
|
|
@@ -677,7 +937,27 @@ export const poolDeposit = async (deps, options) => {
|
|
|
677
937
|
};
|
|
678
938
|
// The sessions a Stop does not name that still pass the read-only gates: they only fill the windows of a delta, and are never counted, withdrawn, or built.
|
|
679
939
|
const history = [];
|
|
680
|
-
|
|
940
|
+
// One scan of the log for the whole build: the head read and every chunk read take its revs, so a deposit reads the log's revs once, not once per chunk.
|
|
941
|
+
const { latest, spans, identity } = await scanLog(paths(runtime.storeDir).observations, undefined, true);
|
|
942
|
+
await runtime.afterScan?.();
|
|
943
|
+
const localStore = await loadStore(runtime.storeDir, undefined, { sessionsOnly: true, latest });
|
|
944
|
+
const childrenOf = new Map();
|
|
945
|
+
for (const agent of localStore.subagents) {
|
|
946
|
+
const parent = agent.parent ?? '';
|
|
947
|
+
const list = childrenOf.get(parent) ?? [];
|
|
948
|
+
list.push(agent.session);
|
|
949
|
+
childrenOf.set(parent, list);
|
|
950
|
+
}
|
|
951
|
+
// The full records of some sessions (and their subagents), read from the log with nothing else.
|
|
952
|
+
const readRecords = async (ids) => {
|
|
953
|
+
const store = await loadStore(runtime.storeDir, undefined, { latest, spans, identity, only: new Set([...ids].flatMap((id) => [id, ...(childrenOf.get(id) ?? [])])) });
|
|
954
|
+
return { store, records: new Map(sessionsOf(store, roots).map((records) => [records.main.session, records])) };
|
|
955
|
+
};
|
|
956
|
+
// A damaged list stops the pass before anything is built: it must never read as "nothing was removed".
|
|
957
|
+
const removedList = await readRemoved(outbox, DEPOSIT);
|
|
958
|
+
const removedKeys = new Set(Object.keys(removedList));
|
|
959
|
+
// By session id too: the key follows the account and the machine's environment, so a login under another account would otherwise build the same session again.
|
|
960
|
+
const removedSessions = new Set(Object.values(removedList).flatMap((entry) => (entry.session === undefined ? [] : [entry.session])));
|
|
681
961
|
for (const records of sessionsOf(localStore, roots)) {
|
|
682
962
|
// The Stop of a machine builds the one session that stopped: the others in the store belong to Stops of their own, so their batches stay as they are. They run the three read-only gates below and no more.
|
|
683
963
|
const named = options.session === undefined || records.main.session === options.session;
|
|
@@ -685,6 +965,12 @@ export const poolDeposit = async (deps, options) => {
|
|
|
685
965
|
counts.sessions += 1;
|
|
686
966
|
// Hashed with the account, so a redeposit replaces its own batch and two accounts' copies of one session cannot be joined.
|
|
687
967
|
const key = machine === undefined ? contentHash([account, records.main.session]) : machineKeyOf(machine, records.main.session);
|
|
968
|
+
// Withdrawn by the person with `oars pool remove`: nothing of the session is built, queued, or read for a window, and its batch and raw copies are already gone.
|
|
969
|
+
if (removedKeys.has(key) || removedSessions.has(records.main.session)) {
|
|
970
|
+
if (named)
|
|
971
|
+
counts.removedByPerson += 1;
|
|
972
|
+
continue;
|
|
973
|
+
}
|
|
688
974
|
if (!allow.allows(records.main.project)) {
|
|
689
975
|
if (named) {
|
|
690
976
|
counts.excluded.folder += 1;
|
|
@@ -728,50 +1014,56 @@ export const poolDeposit = async (deps, options) => {
|
|
|
728
1014
|
// A delta is built from the sessions that passed, carried by the batch of the session that filled its after window, and marked sent only once that batch is written: a deposit that stops first builds it again.
|
|
729
1015
|
// A cloud machine has no history of its own to compare a window against (every VM starts clean), so it carries none.
|
|
730
1016
|
const placed = machine?.hostClass === 'cloud' ? [] : [...ready.map(({ records }) => records), ...history].flatMap((records) => placedOf(records) ?? []);
|
|
731
|
-
const carried =
|
|
1017
|
+
const carried = await carriedDeltasByWindow(account, placed, await sentDeltas(outbox), async (ids) => (await readRecords(ids)).records, new Set(ready.map(({ records }) => records.main.session)));
|
|
732
1018
|
// The coverage bands (t871): the operator's two settings, read here, and the team band counted from the local git log. Nothing is looked up, and anything unread goes as unknown.
|
|
733
1019
|
const coverageOf = makeCoverage(runtime, await readCoverageSetting(runtime.storeDir), options.git ?? runGit);
|
|
734
1020
|
// The recommendations each session was shown or held back from, read from the local rows with their windows, so a shown row stays shown at the pool and a session with none sends none (the pool reads that as unknown).
|
|
735
|
-
const
|
|
736
|
-
for (
|
|
737
|
-
const
|
|
738
|
-
|
|
739
|
-
const
|
|
740
|
-
const
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
counts.refused += 1;
|
|
751
|
-
refusedFields.add(refusedAt ?? '');
|
|
752
|
-
await withdraw(key);
|
|
753
|
-
continue;
|
|
754
|
-
}
|
|
755
|
-
// Noted before the batch is written: a person's sweep that runs between the two lines would delete a batch it cannot yet tell is a machine's.
|
|
756
|
-
if (machine !== undefined)
|
|
757
|
-
await noteMachineKey(outbox, key);
|
|
758
|
-
builtBytes += (await writeBatch(outbox, key, parsed.skeleton)).bytes;
|
|
759
|
-
// The raw copy comes only after both gates and only with a yes to the raw prompt and a folder not at skeleton only: each transcript of the session, redacted here, before it is queued.
|
|
760
|
-
if (rawAllowed(saved.pool, records.main.project, covenant)) {
|
|
761
|
-
let place = 0;
|
|
762
|
-
for (const file of await transcriptFilesOf(runtime, records.main.project, records.main.session)) {
|
|
763
|
-
const redacted = redactTranscript(await readFile(file.path, 'utf8'), saved.redactDomains, names, injectedKey);
|
|
764
|
-
counts.raw.bytes += (await writeRaw(outbox, key, place, redacted.text)).bytes;
|
|
765
|
-
counts.raw.copies += 1;
|
|
766
|
-
counts.raw.masked += redacted.masked;
|
|
767
|
-
place += 1;
|
|
1021
|
+
const recorded = await readRecalls(runtime.storeDir);
|
|
1022
|
+
for (let from = 0; from < ready.length; from += BUILD_CHUNK) {
|
|
1023
|
+
const chunk = ready.slice(from, from + BUILD_CHUNK);
|
|
1024
|
+
const { store: chunkStore, records: full } = await readRecords(new Set(chunk.map(({ records }) => records.main.session)));
|
|
1025
|
+
const recalls = deriveConsequences(chunkStore, recorded);
|
|
1026
|
+
for (const { records: head, key } of chunk) {
|
|
1027
|
+
const records = full.get(head.main.session);
|
|
1028
|
+
const delta = carried.get(head.main.session);
|
|
1029
|
+
// A machine builds the open session at every Stop and SessionEnd, so the one it names keeps what its own earlier batch carried and takes only what is new; any other deposit builds a session once.
|
|
1030
|
+
const entries = machine !== undefined && options.session !== undefined ? keptDeltas(await heldDeltas(outbox, key), delta?.entries ?? []) : delta?.entries;
|
|
1031
|
+
const skeleton = records === undefined ? null : skeletonOf(records, key, shellReader(vocabulary()), contentHash, entries, gate, await coverageOf(head.main), recommendationsOf(recalls, head.main.session));
|
|
1032
|
+
if (skeleton === null) {
|
|
1033
|
+
counts.uncaptured += 1;
|
|
1034
|
+
await withdraw(key);
|
|
1035
|
+
continue;
|
|
768
1036
|
}
|
|
769
|
-
|
|
1037
|
+
const parsed = parseSkeleton(skeleton);
|
|
1038
|
+
// The server reads a value outside its lists as `other` and keeps a field it does not know, so the parser no longer refuses either: the emitter is the privacy control, and a batch this client built holds neither.
|
|
1039
|
+
const refusedAt = parsed.ok ? parsed.extensions[0]?.at : parsed.field;
|
|
1040
|
+
if (!parsed.ok || refusedAt !== undefined) {
|
|
1041
|
+
counts.refused += 1;
|
|
1042
|
+
refusedFields.add(refusedAt ?? '');
|
|
1043
|
+
await withdraw(key);
|
|
1044
|
+
continue;
|
|
1045
|
+
}
|
|
1046
|
+
// Noted before the batch is written: a person's sweep that runs between the two lines would delete a batch it cannot yet tell is a machine's.
|
|
1047
|
+
if (machine !== undefined)
|
|
1048
|
+
await noteMachineKey(outbox, key);
|
|
1049
|
+
builtBytes += (await writeBatch(outbox, key, parsed.skeleton)).bytes;
|
|
1050
|
+
// The raw copy comes only after both gates and only with a yes to the raw prompt and a folder not at skeleton only: each transcript of the session, redacted here, before it is queued.
|
|
1051
|
+
if (rawAllowed(saved.pool, head.main.project, covenant)) {
|
|
1052
|
+
let place = 0;
|
|
1053
|
+
for (const file of await transcriptFilesOf(runtime, head.main.project, head.main.session)) {
|
|
1054
|
+
const redacted = redactTranscript(await readFile(file.path, 'utf8'), saved.redactDomains, names, injectedKey);
|
|
1055
|
+
counts.raw.bytes += (await writeRaw(outbox, key, place, redacted.text)).bytes;
|
|
1056
|
+
counts.raw.copies += 1;
|
|
1057
|
+
counts.raw.masked += redacted.masked;
|
|
1058
|
+
place += 1;
|
|
1059
|
+
}
|
|
1060
|
+
rawKept.add(key);
|
|
1061
|
+
}
|
|
1062
|
+
tallyBatch(wireTally, parsed.skeleton);
|
|
1063
|
+
await markDeltasSent(outbox, delta?.marks ?? []);
|
|
1064
|
+
built.add(key);
|
|
1065
|
+
counts.batched += 1;
|
|
770
1066
|
}
|
|
771
|
-
tallyBatch(wireTally, parsed.skeleton);
|
|
772
|
-
await markDeltasSent(outbox, delta?.marks ?? []);
|
|
773
|
-
built.add(key);
|
|
774
|
-
counts.batched += 1;
|
|
775
1067
|
}
|
|
776
1068
|
// The batch of a session or account this store no longer holds goes too. A machine builds one session at a time and nothing here has the others in hand, so it sweeps nothing: nothing is withdrawn but what a gate refused.
|
|
777
1069
|
if (machine === undefined) {
|
|
@@ -793,6 +1085,7 @@ export const poolDeposit = async (deps, options) => {
|
|
|
793
1085
|
...(discarded === 0 ? [] : [`Discarded ${String(discarded)} ${discarded === 1 ? 'batch' : 'batches'} built under an older vocabulary, field list, or skeleton format.`]),
|
|
794
1086
|
`Raw copies queued: ${String(counts.raw.copies)} (${String(counts.raw.bytes)} bytes, ${String(counts.raw.masked)} values masked). ${rawNote(saved.pool.rawConsent)}`.trim(),
|
|
795
1087
|
...(rawStale(saved.pool, covenant) ? [`Your yes to the raw prompt was given under covenant ${saved.pool.rawConsentVersion ?? 'none'}, not ${covenant ?? ''}, so no raw copy is queued. Run oars pool raw --consent yes to answer for the covenant now in force.`] : []),
|
|
1088
|
+
...(counts.removedByPerson === 0 ? [] : [`${String(counts.removedByPerson)} ${counts.removedByPerson === 1 ? 'session' : 'sessions'} you removed with oars pool remove: nothing of them is built, queued, or sent. oars pool remove <hash> --restore brings one back.`]),
|
|
796
1089
|
...(counts.off === 0 ? [] : [`${String(counts.off)} ${counts.off === 1 ? 'session' : 'sessions'} in a folder switched off: nothing of them is queued.`]),
|
|
797
1090
|
...(termList.length === 0 ? [termsHint(terms)] : []),
|
|
798
1091
|
'Run oars pool outbox to print exactly what would be sent.',
|
|
@@ -812,35 +1105,61 @@ const explained = (bytes) => {
|
|
|
812
1105
|
}
|
|
813
1106
|
return rows;
|
|
814
1107
|
};
|
|
815
|
-
/**
|
|
1108
|
+
/**
|
|
1109
|
+
* `oars pool outbox`: prints each batch as it would be sent (a batch past one request prints each part's request, in order), the request line and every header included, with the bearer credential as its size and nothing else masked, then its SHA-256, and records that it was shown. `--explain` adds each field's rule.
|
|
1110
|
+
* With `print` each batch is written as soon as it is built and marked shown right after, and the outcome carries no text: an outbox of thousands of sessions is far past V8's string length as one string. Without it (`--json`) the document carries each batch's request, and `--json` writes it element by element.
|
|
1111
|
+
*/
|
|
816
1112
|
export const poolOutbox = async (deps, options) => {
|
|
817
1113
|
const runtime = runtimeOf(deps);
|
|
818
1114
|
const server = poolServer(deps.env, OUTBOX, options.server);
|
|
819
1115
|
const { token } = await signedIn(deps, runtime.storeDir, server, OUTBOX);
|
|
820
1116
|
const outbox = outboxFolder(runtime.storeDir, server);
|
|
821
|
-
const { batches, edited } = await heldAcross(outbox);
|
|
822
1117
|
const masker = new Masker();
|
|
823
1118
|
masker.add(token);
|
|
824
1119
|
const sent = await sentLog(outbox);
|
|
1120
|
+
const { print } = options;
|
|
825
1121
|
const shown = [];
|
|
826
|
-
|
|
1122
|
+
// Blocks are separated by a blank line; each is written alone, so no string holds more than one batch.
|
|
1123
|
+
const state = { wrote: false };
|
|
1124
|
+
const put = (block) => {
|
|
1125
|
+
print?.(`${state.wrote ? '\n\n' : ''}${block}`);
|
|
1126
|
+
state.wrote = true;
|
|
1127
|
+
};
|
|
1128
|
+
// One batch is read at a time: the outbox of a 6,500-session history is gigabytes, far past what one array of them holds.
|
|
1129
|
+
let edited = 0;
|
|
1130
|
+
for await (const named of namesHeldAcross(outbox)) {
|
|
1131
|
+
const bytes = await unchangedBytes(named.file, named.hash);
|
|
1132
|
+
if (bytes === undefined) {
|
|
1133
|
+
edited += 1;
|
|
1134
|
+
continue;
|
|
1135
|
+
}
|
|
1136
|
+
const batch = { hash: named.hash, bytes };
|
|
827
1137
|
const fields = options.explain === true ? explained(batch.bytes) : undefined;
|
|
828
1138
|
// A batch past one request prints every part's request, in the order they are sent.
|
|
829
1139
|
const request = callsOf(token, batch.bytes, batch.hash).map((call) => printed(outgoing(deps, server, POOL_DEPOSITS, call), masker)).join('\n');
|
|
830
|
-
|
|
1140
|
+
const at = sent.get(batch.hash);
|
|
1141
|
+
put([request, `sha256: ${batch.hash}`, ...(at === undefined ? [] : [`sent: ${at}`]), ...Object.entries(fields ?? {}).map(([field, row]) => ` ${field}: ${row}`)].join('\n'));
|
|
1142
|
+
shown.push({ hash: batch.hash, ...(print === undefined ? { request } : {}), ...(at === undefined ? {} : { sent: at }), ...(fields === undefined ? {} : { fields }) });
|
|
1143
|
+
await markShown(outbox, batch.hash, deps.now());
|
|
831
1144
|
}
|
|
832
1145
|
const { copies } = await heldRaw(outbox);
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
1146
|
+
for (const copy of copies) {
|
|
1147
|
+
const parts = rawParts(copy.body);
|
|
1148
|
+
for (const [at, part] of parts.entries())
|
|
1149
|
+
put([`raw part ${String(at + 1)}/${String(parts.length)} sha256: ${copy.hash} bytes: ${String(Buffer.byteLength(part))}${sent.has(copy.hash) ? ` sent: ${sent.get(copy.hash) ?? ''}` : ''}`, ...(options.explain === true ? [part] : [])].join('\n'));
|
|
1150
|
+
}
|
|
836
1151
|
const stale = await staleBatches(outbox);
|
|
837
|
-
const blocks = shown.map(({ hash, request, sent: at, fields }) => [request, `sha256: ${hash}`, ...(at === undefined ? [] : [`sent: ${at}`]), ...Object.entries(fields ?? {}).map(([field, row]) => ` ${field}: ${row}`)].join('\n'));
|
|
838
1152
|
const notes = [
|
|
839
1153
|
...(edited === 0 ? [] : [`${String(edited)} ${edited === 1 ? 'batch was' : 'batches were'} edited on disk since ${edited === 1 ? 'it was' : 'they were'} written and ${edited === 1 ? 'is' : 'are'} not printed or sent. Run oars pool deposit to build ${edited === 1 ? 'it' : 'them'} again.`]),
|
|
840
1154
|
...(stale === 0 ? [] : [`${String(stale)} ${stale === 1 ? 'batch was' : 'batches were'} built under an older vocabulary, field list, or skeleton format and will never be sent.`]),
|
|
841
1155
|
];
|
|
842
|
-
|
|
843
|
-
|
|
1156
|
+
if (!state.wrote)
|
|
1157
|
+
put(['The outbox is empty. Run oars pool deposit first.', ...notes].join('\n'));
|
|
1158
|
+
else
|
|
1159
|
+
for (const note of notes)
|
|
1160
|
+
put(note);
|
|
1161
|
+
print?.('\n');
|
|
1162
|
+
return { document: { ok: true, server, batches: shown, raw: copies.map(({ hash, bytes }) => ({ hash, bytes, ...(sent.has(hash) ? { sent: sent.get(hash) ?? '' } : {}) })), edited, stale }, text: '', output: print === undefined ? 'chunked' : 'streamed' };
|
|
844
1163
|
};
|
|
845
1164
|
/**
|
|
846
1165
|
* What the pool says it served a bundle at, printed as the bundle states it: the client holds no floor of its own and applies none (a floor is the pool's deploy setting), so a bundle that states none prints nothing.
|
|
@@ -949,6 +1268,291 @@ export const poolFetch = async (deps, options) => {
|
|
|
949
1268
|
text: `Fetched ${harness} ${harnessVersion}: ${String(reads.length)} reads, ${String(cells.length)} cells, and ${String(motifs.length)} motifs, verified against the pinned key and kept at ${file}.${floorSentence(floors)}`,
|
|
950
1269
|
};
|
|
951
1270
|
};
|
|
1271
|
+
const JOIN = 'pool-join';
|
|
1272
|
+
/** The join's invite step: `POST /v1/invites/redeem`. An account that joined on an invite already is a run repeated, which is not an error; any other refusal ends the run with the pool's own words. */
|
|
1273
|
+
const redeemInvite = async (deps, storeDir, server, code) => {
|
|
1274
|
+
const { token } = await signedIn(deps, storeDir, server, JOIN);
|
|
1275
|
+
const credential = new Masker();
|
|
1276
|
+
credential.add(token);
|
|
1277
|
+
const reply = await ask(deps, JOIN, server, '/v1/invites/redeem', { body: { code }, token, show: credential });
|
|
1278
|
+
if (reply.status === 200)
|
|
1279
|
+
return { redeemed: true };
|
|
1280
|
+
if (reply.status === 409 && isObject(reply.body) && textIn(reply.body, 'error') === 'already_vouched')
|
|
1281
|
+
return { skipped: true, reason: 'already_vouched' };
|
|
1282
|
+
throw refusal(JOIN, reply);
|
|
1283
|
+
};
|
|
1284
|
+
const JOIN_AGAIN = 'Run oars pool join again to continue from here.';
|
|
1285
|
+
const INVITE_AGAIN = 'The invite code was not accepted, and running the same line again fails the same way. Run oars pool join again without --invite, or with a new code.';
|
|
1286
|
+
/** The steps `--plan` lists: the seven the join runs, in order, then the raw question, which `oars pool raw` asks and the join never does. */
|
|
1287
|
+
const PLAN_STEPS = [...JOIN_STEPS, 'raw'];
|
|
1288
|
+
/** The two steps only a person can do: typing the covenant phrase, and answering the raw prompt. Nothing here, no flag and no variable, does either for them. Region and terms are a person's too until a flag or the term file answers them. */
|
|
1289
|
+
const PERSON_STEPS = ['covenant', 'raw'];
|
|
1290
|
+
/** Whether the pool reports the covenant accepted for this sign-in. A pool that cannot be asked, or refuses, reads as not accepted: a plan states what it can see. */
|
|
1291
|
+
const covenantAccepted = async (deps, server, token) => {
|
|
1292
|
+
try {
|
|
1293
|
+
const reply = await ask(deps, JOIN, server, '/v1/covenant', { token });
|
|
1294
|
+
return reply.status === 200 && isObject(reply.body) && Reflect.get(reply.body, 'accepted') === true;
|
|
1295
|
+
}
|
|
1296
|
+
catch {
|
|
1297
|
+
return false;
|
|
1298
|
+
}
|
|
1299
|
+
};
|
|
1300
|
+
/** Whether this batch has been printed by `outbox` or sent: the proof `--send` asks for. */
|
|
1301
|
+
const batchSeen = async (outbox, hash, sent) => sent.has(hash) || (await wasShown(outbox, hash));
|
|
1302
|
+
/**
|
|
1303
|
+
* `oars pool join --plan`: each step of the join and whether it stands, from the sign-in, the term file, the outbox, and (when signed in) one read of the covenant. It signs in to nothing, writes nothing, and sends nothing.
|
|
1304
|
+
* `needs_person` marks the steps a person does: the typed covenant phrase and the raw prompt, and region and terms while no flag or file answers them.
|
|
1305
|
+
*/
|
|
1306
|
+
const joinPlan = async (deps, server, flagged) => {
|
|
1307
|
+
const { storeDir } = runtimeOf(deps);
|
|
1308
|
+
const outbox = outboxFolder(storeDir, server);
|
|
1309
|
+
const held = await heldSignIn(deps, storeDir, server, JOIN);
|
|
1310
|
+
const accepted = held !== undefined && (await covenantAccepted(deps, server, held.token));
|
|
1311
|
+
const { batches } = await heldAcross(outbox);
|
|
1312
|
+
const sent = await sentLog(outbox);
|
|
1313
|
+
const seen = await Promise.all(batches.map(({ hash }) => batchSeen(outbox, hash, sent)));
|
|
1314
|
+
const state = await readState(storeDir);
|
|
1315
|
+
const standing = {
|
|
1316
|
+
signin: held !== undefined,
|
|
1317
|
+
region: accepted || state.pool.euEeaUk !== null,
|
|
1318
|
+
terms: (await currentTerms(deps)).length > 0,
|
|
1319
|
+
batch: batches.length > 0,
|
|
1320
|
+
outbox: batches.length > 0 && seen.every(Boolean),
|
|
1321
|
+
covenant: accepted,
|
|
1322
|
+
send: batches.length > 0 && batches.every(({ hash }) => sent.has(hash)),
|
|
1323
|
+
raw: state.pool.rawConsent === 'granted',
|
|
1324
|
+
};
|
|
1325
|
+
const steps = PLAN_STEPS.map((step) => ({ step, state: standing[step] ? 'done' : 'pending', needs_person: PERSON_STEPS.includes(step) || (!standing[step] && (step === 'terms' || (step === 'region' && !flagged))) }));
|
|
1326
|
+
return { document: { ok: true, plan: true, server, steps }, text: steps.map(({ step, state, needs_person: person }) => `${step}: ${state}${person ? ' (a person does this)' : ''}`).join('\n') };
|
|
1327
|
+
};
|
|
1328
|
+
/**
|
|
1329
|
+
* `oars pool join`: the first deposit in one run, each step by the handler its own command uses: sign-in (`login`), the region question, the exclusion term file (scaffolded empty, never filled in), the batch (`deposit`), what it holds (`outbox
|
|
1330
|
+
* --explain`), the covenant (the screen and typed phrase `deposit --send` shows), and the send (`deposit --send <hash>` for each batch not yet sent), then `fetch` when a harness version is named. A step that stops ends the run with its own
|
|
1331
|
+
* error. Run again, each step that stands is skipped: a held sign-in, terms in the file, a prepared batch, an accepted covenant, a batch in `sent.log`. Acceptance is the person's own: this never types the phrase, and off a terminal it stops.
|
|
1332
|
+
* Each step's text prints as it finishes, so the explanation is on screen before the send asks anything: on stdout, or with `json` on stderr, so stdout holds the one document. One `join-step` event per step says which of the closed steps ran and
|
|
1333
|
+
* what it did (done, skipped, or stopped), and nothing else.
|
|
1334
|
+
*/
|
|
1335
|
+
export const poolJoin = async (deps, options) => {
|
|
1336
|
+
const { storeDir, projectsRoot } = runtimeOf(deps);
|
|
1337
|
+
const server = poolServer(deps.env, JOIN, options.server);
|
|
1338
|
+
if (options.plan === true)
|
|
1339
|
+
return joinPlan(deps, server, options.inEuEeaUk !== undefined);
|
|
1340
|
+
const say = (line) => {
|
|
1341
|
+
deps.io.stderr(`${line}
|
|
1342
|
+
`);
|
|
1343
|
+
};
|
|
1344
|
+
const screen = options.json === true ? deps.io.stderr : deps.io.stdout;
|
|
1345
|
+
const show = (outcome) => {
|
|
1346
|
+
if (outcome.output !== 'streamed')
|
|
1347
|
+
screen(`${outcome.text}
|
|
1348
|
+
`);
|
|
1349
|
+
return outcome;
|
|
1350
|
+
};
|
|
1351
|
+
const outbox = outboxFolder(storeDir, server);
|
|
1352
|
+
const unsent = async () => {
|
|
1353
|
+
const sent = await sentLog(outbox);
|
|
1354
|
+
return (await heldAcross(outbox)).batches.flatMap(({ hash }) => (sent.has(hash) ? [] : [hash]));
|
|
1355
|
+
};
|
|
1356
|
+
const emit = emitOf(deps);
|
|
1357
|
+
let settled;
|
|
1358
|
+
const record = (step, state) => {
|
|
1359
|
+
settled = step;
|
|
1360
|
+
emit('join-step', { step, state });
|
|
1361
|
+
};
|
|
1362
|
+
const state = { refused: false };
|
|
1363
|
+
let current = 'signin';
|
|
1364
|
+
let fetching = false;
|
|
1365
|
+
// Ctrl-C reaches every step: the sign-in poll and the line reads take the signal, and each step begins by checking it.
|
|
1366
|
+
const interrupt = watchInterrupt();
|
|
1367
|
+
const begin = (step) => {
|
|
1368
|
+
current = step;
|
|
1369
|
+
interrupt.signal.throwIfAborted();
|
|
1370
|
+
};
|
|
1371
|
+
let reader;
|
|
1372
|
+
const lines = () => (reader ??= readerOf(deps, interrupt.signal));
|
|
1373
|
+
try {
|
|
1374
|
+
const held = await heldSignIn(deps, storeDir, server, LOGIN);
|
|
1375
|
+
let login = { skipped: true, account: held?.account };
|
|
1376
|
+
if (held === undefined) {
|
|
1377
|
+
say(`Signing in to ${server}.`);
|
|
1378
|
+
login = show(await poolLogin(deps, { server, provider: options.provider, verbose: options.verbose, signal: interrupt.signal })).document;
|
|
1379
|
+
record('signin', 'done');
|
|
1380
|
+
}
|
|
1381
|
+
else {
|
|
1382
|
+
say(`Already signed in to ${server} as ${held.account}.`);
|
|
1383
|
+
record('signin', 'skipped');
|
|
1384
|
+
}
|
|
1385
|
+
const { token } = await signedIn(deps, storeDir, server, JOIN);
|
|
1386
|
+
const via = networkSender(deps, DEPOSIT, { stallMs: SEND_STALL_MS, signal: interrupt.signal });
|
|
1387
|
+
let vouched = { skipped: true };
|
|
1388
|
+
if (options.invite !== undefined) {
|
|
1389
|
+
say('Using your invite code.');
|
|
1390
|
+
try {
|
|
1391
|
+
vouched = await redeemInvite(deps, storeDir, server, options.invite);
|
|
1392
|
+
}
|
|
1393
|
+
catch (error) {
|
|
1394
|
+
state.refused = true;
|
|
1395
|
+
throw error;
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
// The region question: asked plainly, once, of a person at a terminal whose covenant is not yet accepted, and sent with the acceptance. A flag answers it; off a terminal the covenant step is what stops.
|
|
1399
|
+
begin('region');
|
|
1400
|
+
let region = regionOf(options.inEuEeaUk);
|
|
1401
|
+
if (options.inEuEeaUk !== undefined && region === undefined)
|
|
1402
|
+
throw usage(`${REGION_FLAG} takes yes or no, not "${options.inEuEeaUk}". Nothing was accepted.`, JOIN);
|
|
1403
|
+
// An answer given on an earlier run stands: the person is not asked twice, whatever stopped the run after the question.
|
|
1404
|
+
if (region === undefined)
|
|
1405
|
+
region = (await readState(storeDir)).pool.euEeaUk ?? undefined;
|
|
1406
|
+
if (region === undefined && isTerminal(deps) && !(await covenantAccepted(deps, server, token))) {
|
|
1407
|
+
say('Before the covenant screen, one question that goes with your acceptance.');
|
|
1408
|
+
deps.io.stderr(`${REGION_QUESTION} > `);
|
|
1409
|
+
region = regionOf(await lines().next());
|
|
1410
|
+
if (region === undefined)
|
|
1411
|
+
throw usage('Answer yes or no: are you in the EU, the EEA, or the UK? Nothing was accepted.', JOIN);
|
|
1412
|
+
const answer = region;
|
|
1413
|
+
await updatePoolChoices(storeDir, (choices) => {
|
|
1414
|
+
choices.euEeaUk = answer;
|
|
1415
|
+
});
|
|
1416
|
+
record('region', 'done');
|
|
1417
|
+
}
|
|
1418
|
+
else {
|
|
1419
|
+
record('region', 'skipped');
|
|
1420
|
+
}
|
|
1421
|
+
// The term file: the person's own list of names to keep out. Without a term every session stays out of the outbox, so an empty file stops the run here, with the file made for them to edit.
|
|
1422
|
+
begin('terms');
|
|
1423
|
+
if ((await currentTerms(deps)).length === 0) {
|
|
1424
|
+
const file = exclusionPaths(projectsRoot, deps.env).terms;
|
|
1425
|
+
await mkdir(dirname(file), { recursive: true });
|
|
1426
|
+
await writeFile(file, '', { flag: 'a' });
|
|
1427
|
+
say(`Your exclusion terms go in ${file}, one per line: a client, a project, or a person whose name must never leave your machine. Every session that mentions one stays out of the outbox. oars adds none for you.`);
|
|
1428
|
+
if (isTerminal(deps)) {
|
|
1429
|
+
say('Press Enter when you have saved the file.');
|
|
1430
|
+
await lines().next();
|
|
1431
|
+
}
|
|
1432
|
+
if ((await currentTerms(deps)).length === 0) {
|
|
1433
|
+
const title = `The exclusion term file ${file} still holds no term, so every session would stay out of the outbox. Add one term per line, save it, and run this again.`;
|
|
1434
|
+
throw new CliError(title, ExitCode.INVALID, JOIN, { code: 'terms_missing', title });
|
|
1435
|
+
}
|
|
1436
|
+
record('terms', 'done');
|
|
1437
|
+
}
|
|
1438
|
+
else {
|
|
1439
|
+
record('terms', 'skipped');
|
|
1440
|
+
}
|
|
1441
|
+
begin('batch');
|
|
1442
|
+
let prepared = { skipped: true };
|
|
1443
|
+
if ((await unsent()).length > 0) {
|
|
1444
|
+
say('Your outbox already holds a prepared batch; not preparing it again.');
|
|
1445
|
+
record('batch', 'skipped');
|
|
1446
|
+
}
|
|
1447
|
+
else {
|
|
1448
|
+
say('Preparing your deposit in the outbox.');
|
|
1449
|
+
prepared = show(await poolDeposit(deps, { server })).document;
|
|
1450
|
+
record('batch', 'done');
|
|
1451
|
+
}
|
|
1452
|
+
begin('outbox');
|
|
1453
|
+
say('Showing what will be sent, field by field.');
|
|
1454
|
+
const shown = show(await poolOutbox(deps, { server, explain: true, print: screen })).document;
|
|
1455
|
+
record('outbox', 'done');
|
|
1456
|
+
const hashes = await unsent();
|
|
1457
|
+
// The covenant, read from the pool and, if not accepted, shown for a person to type the phrase: before the first send, so its own step ends before the sends begin.
|
|
1458
|
+
begin('covenant');
|
|
1459
|
+
const nothingPrepared = (await heldAcross(outbox)).batches.length === 0;
|
|
1460
|
+
if (hashes.length === 0) {
|
|
1461
|
+
record('covenant', nothingPrepared ? 'none' : 'skipped');
|
|
1462
|
+
}
|
|
1463
|
+
else {
|
|
1464
|
+
record('covenant', (await openCovenant(deps, server, token, options.acceptCovenant, region, via, interrupt.signal, undefined, false, reader)) ? 'done' : 'skipped');
|
|
1465
|
+
}
|
|
1466
|
+
begin('send');
|
|
1467
|
+
const sends = [];
|
|
1468
|
+
if (hashes.length === 0) {
|
|
1469
|
+
say(nothingPrepared ? 'No session is ready to deposit yet; sending nothing.' : 'Everything prepared is already sent; sending nothing.');
|
|
1470
|
+
record('send', nothingPrepared ? 'none' : 'skipped');
|
|
1471
|
+
}
|
|
1472
|
+
else {
|
|
1473
|
+
say(`Sending ${plural(hashes.length, 'batch', 'batches')} to ${server}.`);
|
|
1474
|
+
for (const hash of hashes)
|
|
1475
|
+
sends.push(show(await poolDeposit(deps, { server, send: hash, acceptCovenant: options.acceptCovenant, inEuEeaUk: region })).document);
|
|
1476
|
+
record('send', 'done');
|
|
1477
|
+
}
|
|
1478
|
+
await rememberCrew(deps, storeDir, server, token, JOIN);
|
|
1479
|
+
let fetched;
|
|
1480
|
+
if (options.harnessVersion !== undefined) {
|
|
1481
|
+
fetching = true;
|
|
1482
|
+
say(`Fetching the bundle for ${options.harness ?? 'your most used harness'} ${options.harnessVersion}.`);
|
|
1483
|
+
fetched = show(await poolFetch(deps, { server, harness: options.harness, harnessVersion: options.harnessVersion }));
|
|
1484
|
+
}
|
|
1485
|
+
return {
|
|
1486
|
+
document: { ok: true, server, login, invite: vouched, deposit: prepared, outbox: shown, sends, ...(fetched === undefined ? {} : { fetch: fetched.document }) },
|
|
1487
|
+
text: `Joined ${server}: ${plural(sends.length, 'batch', 'batches')} sent this run${options.harnessVersion === undefined ? '' : `, and the ${options.harnessVersion} bundle is kept`}.`,
|
|
1488
|
+
};
|
|
1489
|
+
}
|
|
1490
|
+
catch (error) {
|
|
1491
|
+
// A step that threw is the one that stopped; the fetch after the sends is not one of the seven.
|
|
1492
|
+
// A step already recorded done or skipped is not stopped as well: the failure came after it, in the code between steps.
|
|
1493
|
+
if (!fetching && settled !== current)
|
|
1494
|
+
record(current, 'stopped');
|
|
1495
|
+
say(state.refused ? INVITE_AGAIN : JOIN_AGAIN);
|
|
1496
|
+
// A signal ended the run (the readline read throws its own abort error, the poll and the steps throw the reason): exit as deposit does, 130 for Ctrl-C.
|
|
1497
|
+
if (interrupt.signal.aborted && interrupt.signal.reason instanceof SendInterrupted)
|
|
1498
|
+
throw stopped(JOIN, interrupt.signal.reason, false);
|
|
1499
|
+
throw error;
|
|
1500
|
+
}
|
|
1501
|
+
finally {
|
|
1502
|
+
interrupt.stop();
|
|
1503
|
+
reader?.close();
|
|
1504
|
+
}
|
|
1505
|
+
};
|
|
1506
|
+
const INVITE = 'pool-invite';
|
|
1507
|
+
/** Asks the pool for this account's crew number and keeps it for the card. Best effort: a refusal or a failed call leaves what was held, because nothing the person asked for depends on it. */
|
|
1508
|
+
const rememberCrew = async (deps, storeDir, server, token, step) => {
|
|
1509
|
+
try {
|
|
1510
|
+
const credential = new Masker();
|
|
1511
|
+
credential.add(token);
|
|
1512
|
+
const reply = await ask(deps, step, server, '/v1/me', { token, show: credential });
|
|
1513
|
+
if (reply.status !== 200 || !isObject(reply.body))
|
|
1514
|
+
return undefined;
|
|
1515
|
+
const account = Reflect.get(reply.body, 'account');
|
|
1516
|
+
if (typeof account === 'string')
|
|
1517
|
+
await keepCrewNumber(storeDir, server, account, numberIn(reply.body, 'crewNumber'));
|
|
1518
|
+
return numberIn(reply.body, 'vouchesGiven');
|
|
1519
|
+
}
|
|
1520
|
+
catch {
|
|
1521
|
+
// The card shows the number when it is known and says nothing when it is not.
|
|
1522
|
+
return undefined;
|
|
1523
|
+
}
|
|
1524
|
+
};
|
|
1525
|
+
/** The one line an invitee runs. `<version>` stays a placeholder: the harness version is the invitee's own to name. */
|
|
1526
|
+
export const joinLineOf = (code) => `oars pool join --invite ${code} --harness-version <version>`;
|
|
1527
|
+
/**
|
|
1528
|
+
* `oars pool invite`: makes one invite code (`POST /v1/invites`, a founding or standing account, three open at most) and prints it once with the line to hand on and what the vouch gives. The code is the only secret here, so it is the one thing the pool never shows again.
|
|
1529
|
+
*/
|
|
1530
|
+
export const poolInvite = async (deps, options) => {
|
|
1531
|
+
const { storeDir } = runtimeOf(deps);
|
|
1532
|
+
const server = poolServer(deps.env, INVITE, options.server);
|
|
1533
|
+
const { token } = await signedIn(deps, storeDir, server, INVITE);
|
|
1534
|
+
const credential = new Masker();
|
|
1535
|
+
credential.add(token);
|
|
1536
|
+
const reply = await ask(deps, INVITE, server, '/v1/invites', { body: {}, token, show: credential });
|
|
1537
|
+
if (reply.status !== 201)
|
|
1538
|
+
throw refusal(INVITE, reply);
|
|
1539
|
+
const code = isObject(reply.body) ? textIn(reply.body, 'code') : undefined;
|
|
1540
|
+
const left = isObject(reply.body) ? numberIn(reply.body, 'invitesLeft') : undefined;
|
|
1541
|
+
if (code === undefined)
|
|
1542
|
+
throw new CliError('The pool answered an invite with no code.', ExitCode.UNEXPECTED, INVITE);
|
|
1543
|
+
const given = await rememberCrew(deps, storeDir, server, token, INVITE);
|
|
1544
|
+
const line = joinLineOf(code);
|
|
1545
|
+
return {
|
|
1546
|
+
document: { ok: true, server, code, join: line, ...(left === undefined ? {} : { invitesLeft: left }), ...(given === undefined ? {} : { vouchesGiven: given }) },
|
|
1547
|
+
text: [
|
|
1548
|
+
`Invite code: ${code}`,
|
|
1549
|
+
'This is the only time the code is shown. It works once, for one person.',
|
|
1550
|
+
`Send them this line: ${line}`,
|
|
1551
|
+
`What you get: your vouch is recorded${given === undefined ? '' : ` (you have vouched for ${String(given)} so far)`}. What they get: once they have rowed two counting weeks, the next founding-crew grant lets them join the open board without waiting 90 days, unless you and they are the same operator, which gives neither of you anything.`,
|
|
1552
|
+
...(left === undefined ? [] : [`Open codes you can still make: ${String(left)}.`]),
|
|
1553
|
+
].join('\n'),
|
|
1554
|
+
};
|
|
1555
|
+
};
|
|
952
1556
|
const RAW = 'pool-raw';
|
|
953
1557
|
/** The raw prompt, in plain words: shown by `oars pool raw` until it is answered, and what the detached sender relies on. Redaction is best effort and the text says so. */
|
|
954
1558
|
export const RAW_PROMPT = [
|