@volter/supercode-teams 0.3.191 → 0.3.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/teams-cli.mjs +9 -3
- package/machine.mjs +4 -1
- package/mail/import-maildir.mjs +15 -2
- package/mail/import-progress.mjs +4 -3
- package/mail/migration-readiness.mjs +30 -1
- package/mail/reconcile-maildir.mjs +290 -0
- package/mail/recovery-source.mjs +51 -0
- package/mail/retire-maildir.mjs +45 -4
- package/mail/upgrade-readiness.mjs +36 -2
- package/package.json +1 -1
- package/terminal-mail.mjs +19 -9
package/bin/teams-cli.mjs
CHANGED
|
@@ -49,7 +49,7 @@ import { APP_DOOR_PATH, appDoorFile, appDoorLink, normalOrigin, readAppDoor, rea
|
|
|
49
49
|
|
|
50
50
|
const MACHINE_VERBS = new Set(['machine', 'slow', 'describe', 'records', 'panes', 'input', 'suspend', 'resume', 'files', 'ports', 'port', 'browser', 'stream', 'jobs', 'workspace', 'title', 'attention', 'watch', 'rpc', 'launches', 'log', 'mail', 'board', 'health', 'app-door']);
|
|
51
51
|
// Every flag a machine verb reads; any other is refused rather than silently ignored.
|
|
52
|
-
const KNOWN_FLAGS = new Set(['home', 'since', 'machine', 'context', 'floor', 'supercode', 'tmux-socket', 'verbose', 'lines', 'input', 'name', 'for', 'urgency', 'reason', 'interrupt', 'query', 'out', 'in', 'interval', 'no-enter', 'when-composer-empty', 'key', 'listen', 'list', 'from', 'to', 'path', 'to-path', 'repo', 'branch', 'solo', 'card', 'create', 'presence-only', 'processes', 'panes', 'clear-draft', 'message-ids', 'harness', 'observe', 'attach-context', 'json', 'all', 'origin', 'port', 'remove-origin', 'open', 'foreign']);
|
|
52
|
+
const KNOWN_FLAGS = new Set(['apply', 'prefer-live-maildir', 'restart', 'home', 'since', 'machine', 'context', 'floor', 'supercode', 'tmux-socket', 'verbose', 'lines', 'input', 'name', 'for', 'urgency', 'reason', 'interrupt', 'query', 'out', 'in', 'interval', 'no-enter', 'when-composer-empty', 'key', 'listen', 'list', 'from', 'to', 'path', 'to-path', 'repo', 'branch', 'solo', 'card', 'create', 'presence-only', 'processes', 'panes', 'clear-draft', 'message-ids', 'harness', 'observe', 'attach-context', 'json', 'all', 'origin', 'port', 'remove-origin', 'open', 'foreign']);
|
|
53
53
|
|
|
54
54
|
const argv = process.argv.slice(2);
|
|
55
55
|
const flags = {};
|
|
@@ -66,7 +66,7 @@ for (let i = 0; i < argv.length; i += 1) {
|
|
|
66
66
|
} else positional.push(a);
|
|
67
67
|
}
|
|
68
68
|
const [verb, sub, ...rest] = positional;
|
|
69
|
-
const migrationRead = verb === 'mail' && ['migration-check', 'upgrade-check', 'upgrade-inspect', 'retain-release'].includes(sub);
|
|
69
|
+
const migrationRead = verb === 'mail' && ['recover', 'migration-check', 'upgrade-check', 'upgrade-inspect', 'retain-release'].includes(sub);
|
|
70
70
|
// A Windows scheduled task keeps no output (launchd and systemd name log files), so its unit names one here.
|
|
71
71
|
// The local migration preflight does not open the service log or timing log.
|
|
72
72
|
if (!migrationRead && process.env.SUPERCODE_TEAMS_LOG) {
|
|
@@ -82,7 +82,7 @@ const LONG_RUNNING = new Set(['connect', 'watch', 'port', 'stream', 'attach', 's
|
|
|
82
82
|
const [timedVerb, timedSub] = verb === 'teams' ? [sub, rest[0]] : [verb, sub];
|
|
83
83
|
if (!migrationRead && !LONG_RUNNING.has(timedVerb) && !(timedVerb === 'machine' && timedSub === 'start') && !(timedVerb === 'message' && timedSub === 'watch')) timeCommand(`teams ${[timedVerb, timedSub].filter(Boolean).join(' ')}`, argv);
|
|
84
84
|
|
|
85
|
-
const MACHINE_HELP = 'machine verbs: machine start|status|restart, slow [--since 1h], describe, panes, input, suspend, resume, files, ports, port, stream, jobs, workspace, title, attention, watch, rpc, launches, log, health, browser, app-door on|off|rotate|status|link (add --machine NAME for an enrolled machine); local read: mail migration-check|upgrade-check [--home dir] [--json]';
|
|
85
|
+
const MACHINE_HELP = 'machine verbs: machine start|status|restart, slow [--since 1h], describe, panes, input, suspend, resume, files, ports, port, stream, jobs, workspace, title, attention, watch, rpc, launches, log, health, browser, app-door on|off|rotate|status|link (add --machine NAME for an enrolled machine); local read: mail migration-check|upgrade-check|recover [--home dir] [--json]; offline recovery: mail recover --apply --prefer-live-maildir [--restart]';
|
|
86
86
|
|
|
87
87
|
if (verb === undefined || verb === 'help') {
|
|
88
88
|
process.stdout.write(`supercode teams: ${MACHINE_HELP}\n`);
|
|
@@ -401,6 +401,12 @@ async function main() {
|
|
|
401
401
|
return;
|
|
402
402
|
}
|
|
403
403
|
case 'mail': {
|
|
404
|
+
if (sub === 'recover') {
|
|
405
|
+
if (Object.keys(flags).some(flag => !['home', 'json', 'apply', 'prefer-live-maildir', 'restart'].includes(flag)) || rest.length || flags.home === true) die('mail recover takes --home, --json, --apply, --prefer-live-maildir and --restart');
|
|
406
|
+
const { reconcileMaildir } = await import('../mail/reconcile-maildir.mjs');
|
|
407
|
+
out(await reconcileMaildir(resolve(home), { apply: flags.apply === true, preferLiveMaildir: flags['prefer-live-maildir'] === true, restart: flags.restart === true }));
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
404
410
|
if (sub === 'retain-release') {
|
|
405
411
|
if (!flags.supercode || Object.keys(flags).some(flag => !['home', 'json', 'supercode'].includes(flag)) || rest.length) die('mail retain-release requires --supercode and takes --home and --json');
|
|
406
412
|
const { prepareRetainedRelease } = await import('./workspace/retain-release.mjs');
|
package/machine.mjs
CHANGED
|
@@ -1195,7 +1195,10 @@ export class TeamsMachine extends MachineCore {
|
|
|
1195
1195
|
producer.note('sessions_activity', `the sessions' activity could not be followed: ${error.message}; a session's turn changes on index events only until it is`);
|
|
1196
1196
|
this.emit('records-log', `machine record: the sessions' activity could not be followed (${error.message}); tried again in ${Math.round(activityBackoff / 1000)} s`);
|
|
1197
1197
|
clearTimeout(this.recordsActivityRetry);
|
|
1198
|
-
if (!this.recordsStopped)
|
|
1198
|
+
if (!this.recordsStopped) {
|
|
1199
|
+
this.recordsActivityRetry = setTimeout(() => void this.recordsFollowActivity(producer.record.sessions), activityBackoff);
|
|
1200
|
+
this.recordsActivityRetry.unref?.();
|
|
1201
|
+
}
|
|
1199
1202
|
}
|
|
1200
1203
|
};
|
|
1201
1204
|
// A folder watched for the record: a watch that failed or could not be made holds its retry until it is kept, said
|
package/mail/import-maildir.mjs
CHANGED
|
@@ -48,8 +48,11 @@ async function* logRecords(stream, start) {
|
|
|
48
48
|
else if (fragments.length) yield { bytes: assembled(), offset: position };
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
export async function importMaildir(database, root, { log = () => {}, legacyRoot = root, deferOperational = false } = {}) {
|
|
51
|
+
export async function importMaildir(database, root, { log = () => {}, legacyRoot = root, deferOperational = false, budgetMs = null } = {}) {
|
|
52
52
|
const began = performance.now();
|
|
53
|
+
const budget = () => {
|
|
54
|
+
if (budgetMs != null && performance.now() - began >= budgetMs) throw Object.assign(new Error('mail import yielded its committed progress'), { code: 'mail_import_yield' });
|
|
55
|
+
};
|
|
53
56
|
const work = { source_reads: 0, source_bytes: 0, committed_batches: 0, max_queued_operations: 0, resumed_directories: 0, resumed_entries: 0, resumed_log_bytes: 0, elapsed_ms: 0 };
|
|
54
57
|
const complete = database.metadata('maildir_import');
|
|
55
58
|
if (complete?.complete === true) return complete;
|
|
@@ -116,7 +119,9 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
116
119
|
};
|
|
117
120
|
const progress = new ImportProgress(database, root, queue, work);
|
|
118
121
|
await progress.start();
|
|
119
|
-
const entries =
|
|
122
|
+
const entries = async function* (dir) {
|
|
123
|
+
for await (const entry of progress.entries(dir)) { budget(); yield entry; }
|
|
124
|
+
};
|
|
120
125
|
const recordDirectory = async (dir, collection, { nested = false, text = false } = {}) => {
|
|
121
126
|
for await (const entry of entries(dir)) {
|
|
122
127
|
const path = join(dir, entry.name);
|
|
@@ -145,8 +150,10 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
145
150
|
|
|
146
151
|
// Pass 1 imports envelopes before their claims, subscriptions and deliveries,
|
|
147
152
|
// so the database's foreign keys enforce the same message ownership throughout.
|
|
153
|
+
try {
|
|
148
154
|
const boxes = [];
|
|
149
155
|
for (const entry of roots) {
|
|
156
|
+
budget();
|
|
150
157
|
if (!entry.isDirectory()) continue;
|
|
151
158
|
const dir = join(root, entry.name);
|
|
152
159
|
let address;
|
|
@@ -312,6 +319,7 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
312
319
|
const stream = start < stat.size ? createReadStream(path, { start, end: stat.size - 1 }) : null;
|
|
313
320
|
try {
|
|
314
321
|
if (stream) for await (const row of logRecords(stream, start)) {
|
|
322
|
+
budget();
|
|
315
323
|
number++;
|
|
316
324
|
work.source_bytes += row.offset - readAt;
|
|
317
325
|
readAt = row.offset;
|
|
@@ -393,4 +401,9 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
393
401
|
database.setMetadata('maildir_import', result);
|
|
394
402
|
log(`mail import complete: ${result.messages} envelopes, ${result.records} operational records, ${result.skipped} retained unreadable sources; ${work.source_reads} source reads, ${work.source_bytes} bytes, ${work.committed_batches} batches in ${work.elapsed_ms} ms`);
|
|
395
403
|
return result;
|
|
404
|
+
} catch (error) {
|
|
405
|
+
if (error.code !== 'mail_import_yield') throw error;
|
|
406
|
+
flush();
|
|
407
|
+
return { complete: false, yielded: true, progress: database.metadata('maildir_resume'), elapsed_ms: Math.round(performance.now() - began) };
|
|
408
|
+
}
|
|
396
409
|
}
|
package/mail/import-progress.mjs
CHANGED
|
@@ -32,10 +32,11 @@ export class ImportProgress {
|
|
|
32
32
|
if (kept && !same(binding, kept)) throw new Error('mail import progress belongs to a different snapshot');
|
|
33
33
|
if (!kept) this.database.setMetadata('maildir_resume_root', binding);
|
|
34
34
|
|
|
35
|
-
// A refused input must still be repairable.
|
|
36
|
-
//
|
|
35
|
+
// A refused input must still be repairable. Retained historical diagnostics
|
|
36
|
+
// are not refused input: resetting their directories on every bounded call
|
|
37
|
+
// can keep an orphan claim's large directory from ever finishing.
|
|
37
38
|
const retry = new Set();
|
|
38
|
-
for (const { source } of this.database.statement(
|
|
39
|
+
for (const { source } of this.database.statement("SELECT source FROM mail_import_errors WHERE NOT (source LIKE 'sent/%/log.jsonl' OR detail = 'claim has no imported message')").all()) {
|
|
39
40
|
if (typeof source !== 'string' || isAbsolute(source) || !resolve(this.root, source).startsWith(this.root + sep)) {
|
|
40
41
|
throw new Error('mail import error record is outside its snapshot');
|
|
41
42
|
}
|
|
@@ -76,6 +76,27 @@ function endpointChild(pid, seed, before, after) {
|
|
|
76
76
|
return children.length === 1 ? children[0] : null;
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
async function completedEndpoint(pid, before, after) {
|
|
80
|
+
// A restored Maildir can have lost a bootstrap record while its pane and
|
|
81
|
+
// native server are still alive. The wrapper writes that record before it
|
|
82
|
+
// starts its remote TUI, then never writes it again. Observe both stable
|
|
83
|
+
// children instead of mistaking a missing record for unfinished publication.
|
|
84
|
+
const servers = [...after].filter(([child, row]) => row.parent === pid && same(before.get(child), row)
|
|
85
|
+
&& / app-server --listen unix:\/\/\S+$/.test(row.command));
|
|
86
|
+
if (servers.length !== 1) return null;
|
|
87
|
+
const server = servers[0], socketPath = / app-server --listen unix:\/\/(\S+)$/.exec(server[1].command)[1];
|
|
88
|
+
if (!isAbsolute(socketPath)) return null;
|
|
89
|
+
const endpoint = `unix://${socketPath}`;
|
|
90
|
+
const remote = new RegExp(`(?:^|\\s)--remote(?:\\s+|=)${endpoint.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:\\s|$)`);
|
|
91
|
+
const clients = [...after].filter(([child, row]) => child !== server[0] && row.parent === pid
|
|
92
|
+
&& same(before.get(child), row) && remote.test(row.command));
|
|
93
|
+
if (clients.length !== 1) return null;
|
|
94
|
+
let socket;
|
|
95
|
+
try { socket = await stat(socketPath); } catch (error) { if (error.code !== 'ENOENT') throw error; }
|
|
96
|
+
return socket?.isSocket() ? { endpoint_pid: server[0], endpoint_birth: server[1].birth,
|
|
97
|
+
client_pid: clients[0][0], client_birth: clients[0][1].birth, publication: 'stable_native_children' } : null;
|
|
98
|
+
}
|
|
99
|
+
|
|
79
100
|
/** Checks only the wrapper boundary, without opening SQLite or moving files. */
|
|
80
101
|
export async function inspectLegacyMaildir(root) {
|
|
81
102
|
root = resolve(root);
|
|
@@ -121,6 +142,8 @@ export async function inspectLegacyMaildir(root) {
|
|
|
121
142
|
const binding = wrapperBinding(current);
|
|
122
143
|
if (binding === false) { block(name, pid, 'legacy wrapper arguments cannot be read unambiguously'); continue; }
|
|
123
144
|
if (binding !== path) { result.stale.push({ seed: name, pid, reason: 'current process does not name this legacy binding' }); continue; }
|
|
145
|
+
const completed = await completedEndpoint(pid, before, after);
|
|
146
|
+
if (completed) { result.live.push({ seed: name, pid, birth: current.birth, ...completed }); continue; }
|
|
124
147
|
if (seed.kind !== 'codex' || typeof seed.socketPath !== 'string' || !isAbsolute(seed.socketPath)) {
|
|
125
148
|
block(name, pid, 'legacy endpoint publication is incomplete'); continue;
|
|
126
149
|
}
|
|
@@ -148,8 +171,14 @@ export async function inspectLegacyMaildir(root) {
|
|
|
148
171
|
}
|
|
149
172
|
if (!binding?.startsWith(directory + sep)) continue;
|
|
150
173
|
const item = seeds.get(binding);
|
|
174
|
+
if (same(before.get(pid), after.get(pid)) && !result.live.some(entry => entry.pid === pid)) {
|
|
175
|
+
const completed = await completedEndpoint(pid, before, after);
|
|
176
|
+
if (completed) result.live.push({ seed: item?.name ?? basename(binding), pid, birth: row.birth,
|
|
177
|
+
seed_present: Boolean(item), ...completed });
|
|
178
|
+
}
|
|
151
179
|
if (!same(before.get(pid), after.get(pid)) || item?.seed?.ownerPid !== pid
|
|
152
|
-
|| !result.live.some((entry) => entry.pid === pid && entry.seed === item
|
|
180
|
+
|| !result.live.some((entry) => entry.pid === pid && entry.seed === (item?.name ?? basename(binding)))) {
|
|
181
|
+
if (same(before.get(pid), after.get(pid)) && result.live.some(entry => entry.pid === pid && entry.publication === 'stable_native_children')) continue;
|
|
153
182
|
block(item?.name ?? basename(binding), pid, 'legacy wrapper has not completed endpoint publication');
|
|
154
183
|
}
|
|
155
184
|
}
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
// Explicit offline recovery selects current legacy operational state. Historical
|
|
2
|
+
// envelopes are unioned without overwriting a current envelope or its read state.
|
|
3
|
+
// Old operational revisions and damaged bytes remain in their original stores.
|
|
4
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
6
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
+
import { lstat, mkdir, open, readFile, readdir, rename, writeFile } from 'node:fs/promises';
|
|
8
|
+
import { dirname, join, resolve } from 'node:path';
|
|
9
|
+
import { MailDatabase, MailOwner } from './sqlite.mjs';
|
|
10
|
+
import { importMaildir } from './import-maildir.mjs';
|
|
11
|
+
import { verifyImportedMaildir } from './retire-maildir.mjs';
|
|
12
|
+
import { requireLegacyMaildirReady } from './migration-readiness.mjs';
|
|
13
|
+
import { blake3Hex } from './envelope.mjs';
|
|
14
|
+
import { fileStamp, requireRecoverySource } from './recovery-source.mjs';
|
|
15
|
+
|
|
16
|
+
const stat = async path => { try { return await lstat(path); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } };
|
|
17
|
+
const json = async path => { try { return JSON.parse(await readFile(path, 'utf8')); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } };
|
|
18
|
+
const identity = value => value && ({ device: value.dev, inode: value.ino });
|
|
19
|
+
const same = (a, b) => isDeepStrictEqual(a, b);
|
|
20
|
+
const alive = pid => { if (!Number.isSafeInteger(pid) || pid < 1) return false; try { process.kill(pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; } };
|
|
21
|
+
const save = async (path, value) => {
|
|
22
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
23
|
+
await writeFile(temporary, JSON.stringify(value) + '\n', { mode: 0o600 });
|
|
24
|
+
await rename(temporary, path);
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
// A caller may leave during restoration too. Copy bounded chunks into an owned
|
|
28
|
+
// temporary file; only a complete, synced copy replaces the canonical database.
|
|
29
|
+
async function restoreOriginal(journal, journalPath, suffix, deadline) {
|
|
30
|
+
const original = join(journal.directory, `failed-mail.sqlite${suffix}`);
|
|
31
|
+
const source = await stat(original);
|
|
32
|
+
if (!source) return true;
|
|
33
|
+
if (!source.isFile()) throw new Error('retained recovery database is not a regular file');
|
|
34
|
+
const path = join(journal.home, `mail.sqlite${suffix}`), temporary = `${path}.${journal.id}.restore.tmp`;
|
|
35
|
+
journal.restart_files ??= {};
|
|
36
|
+
let progress = journal.restart_files[suffix];
|
|
37
|
+
if (progress && !same(progress.source, fileStamp(source))) throw new Error('retained recovery database changed during restoration');
|
|
38
|
+
if (progress && (!Number.isSafeInteger(progress.offset) || progress.offset < 0 || progress.offset > source.size)) throw new Error('recovery database restoration has an invalid cursor');
|
|
39
|
+
if (progress?.complete) return true;
|
|
40
|
+
if (!progress || !await stat(temporary)) {
|
|
41
|
+
progress = journal.restart_files[suffix] = { source: fileStamp(source), offset: 0 };
|
|
42
|
+
await save(journalPath, journal);
|
|
43
|
+
}
|
|
44
|
+
const input = await open(original, 'r');
|
|
45
|
+
let output;
|
|
46
|
+
try {
|
|
47
|
+
output = await open(temporary, await stat(temporary) ? 'r+' : 'w+', 0o600);
|
|
48
|
+
await output.truncate(progress.offset);
|
|
49
|
+
const buffer = Buffer.allocUnsafe(1 << 20);
|
|
50
|
+
while (progress.offset < source.size && performance.now() < deadline) {
|
|
51
|
+
const { bytesRead } = await input.read(buffer, 0, Math.min(buffer.length, source.size - progress.offset), progress.offset);
|
|
52
|
+
if (!bytesRead) throw new Error('retained recovery database ended during restoration');
|
|
53
|
+
let written = 0;
|
|
54
|
+
while (written < bytesRead) {
|
|
55
|
+
const { bytesWritten } = await output.write(buffer, written, bytesRead - written, progress.offset + written);
|
|
56
|
+
if (!bytesWritten) throw new Error('recovery database restoration made no progress');
|
|
57
|
+
written += bytesWritten;
|
|
58
|
+
}
|
|
59
|
+
progress.offset += bytesRead;
|
|
60
|
+
}
|
|
61
|
+
await output.sync();
|
|
62
|
+
} finally { await input.close(); await output?.close(); }
|
|
63
|
+
if (!same(fileStamp(await stat(original)), progress.source)) throw new Error('retained recovery database changed during restoration');
|
|
64
|
+
if (progress.offset === source.size) {
|
|
65
|
+
await rename(temporary, path);
|
|
66
|
+
progress.complete = true;
|
|
67
|
+
}
|
|
68
|
+
await save(journalPath, journal);
|
|
69
|
+
return progress.complete === true;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function offline(teamsHome, root) {
|
|
73
|
+
const serving = await json(join(teamsHome, 'machine.json'));
|
|
74
|
+
if (alive(serving?.pid)) throw new Error('mail recovery requires the machine daemon stopped through its owner; it stops no process');
|
|
75
|
+
// The guard admits only wrappers whose endpoint publication has finished.
|
|
76
|
+
// Those panes use the native socket and do not write the Maildir again.
|
|
77
|
+
// A pending publisher still refuses recovery, without closing its pane.
|
|
78
|
+
await requireLegacyMaildirReady(root);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function envelope(database, mailbox, value, state, report) {
|
|
82
|
+
if (!/^sc:[^:\s]+:[^:\s]+:\S+$/.test(mailbox) || !value || typeof value !== 'object'
|
|
83
|
+
|| !/^[A-Za-z0-9_-]+$/.test(value.id) || !Number.isSafeInteger(value.created_at_ms) || typeof value.body !== 'string') {
|
|
84
|
+
throw new Error('historical envelope cannot be decoded without loss; original stores retained');
|
|
85
|
+
}
|
|
86
|
+
const { body, ...header } = value;
|
|
87
|
+
const existing = database.statement('SELECT header, body FROM messages WHERE mailbox = ? AND id = ?').get(mailbox, value.id);
|
|
88
|
+
if (existing) {
|
|
89
|
+
if (!same(JSON.parse(existing.header), header) || existing.body !== body) throw new Error(`conflicting historical envelope ${value.id}; original stores retained`);
|
|
90
|
+
report.matched_messages++;
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
if (!database.statement('SELECT 1 FROM mailboxes WHERE address = ?').get(mailbox)) {
|
|
94
|
+
const harness = mailbox.split(':')[2].replace(/[^A-Za-z0-9-]/g, '_');
|
|
95
|
+
database.statement('INSERT INTO mailboxes(address, folder) VALUES (?, ?)').run(mailbox, `${harness}-${blake3Hex(mailbox).slice(0, 24)}`);
|
|
96
|
+
}
|
|
97
|
+
// A historical PID/token never authorizes a resumed claim. Preserve unread
|
|
98
|
+
// mail as unread, and leave the original claim receipt in the old archive.
|
|
99
|
+
database.statement(`INSERT INTO messages(mailbox, id, thread, sender, kind, subject, created_at_ms, state, header, body, first_line, read_at_ms)
|
|
100
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(mailbox, value.id, value.thread ?? value.id, value.from ?? '', value.kind ?? 'peer', value.subject ?? null,
|
|
101
|
+
value.created_at_ms, state === 'cur' ? 'cur' : 'new', JSON.stringify(header), body, Array.from(body.split('\n', 1)[0]).slice(0, 400).join(''), state === 'cur' ? value.created_at_ms : null);
|
|
102
|
+
report.restored_messages++;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Each invocation makes bounded resumable work, reserving time for the CLI,
|
|
106
|
+
* source checks and durable journal publication. No automatic install
|
|
107
|
+
* selects a store; --prefer-live-maildir is an explicit operator choice. */
|
|
108
|
+
export async function reconcileMaildir(teamsHome, { apply = false, preferLiveMaildir = false, restart = false } = {}) {
|
|
109
|
+
teamsHome = resolve(teamsHome);
|
|
110
|
+
const home = dirname(teamsHome), root = join(home, 'mail'), base = join(home, 'mail-recovery'), journalPath = join(base, 'current.json');
|
|
111
|
+
let journal = await json(journalPath);
|
|
112
|
+
if (restart && !apply) throw new Error('mail recover --restart requires --apply --prefer-live-maildir');
|
|
113
|
+
if (!apply) return { stage: 'mail_recovery', policy: 'prefer-live-maildir', requires_offline: true, journal,
|
|
114
|
+
live_maildir: Boolean(await stat(root)), failed_sqlite: Boolean(await stat(join(home, 'mail.sqlite'))) };
|
|
115
|
+
if (!preferLiveMaildir) throw new Error('mail recover --apply requires --prefer-live-maildir; no store was selected');
|
|
116
|
+
await offline(teamsHome, root);
|
|
117
|
+
const owner = new MailOwner(join(home, 'mail-owner.sqlite'));
|
|
118
|
+
let database, restarted = false;
|
|
119
|
+
try {
|
|
120
|
+
if (journal?.home && journal.home !== home) throw new Error('mail recovery journal belongs to another home');
|
|
121
|
+
if (journal && (!/^[0-9a-f-]{36}$/.test(journal.id) || journal.directory !== join(base, journal.id) || journal.root !== root || journal.candidate !== join(journal.directory, 'recovered.sqlite')
|
|
122
|
+
|| dirname(resolve(journal.archive)) !== join(home, 'mail-retired') || journal.recovered_archive !== join(home, 'mail-retired', `recovered-${journal.id}`))) throw new Error('mail recovery journal contains paths outside its owned home');
|
|
123
|
+
if ((restart || journal?.state === 'restarting') && journal) {
|
|
124
|
+
if (!same(identity(await stat(root)), journal.live_identity)) throw new Error('restart requires the selected original Maildir still at its live path');
|
|
125
|
+
if (journal.state === 'publish') {
|
|
126
|
+
if (await stat(join(journal.directory, 'failed-mail.sqlite'))) {
|
|
127
|
+
// Publication may have moved the old database before caller loss.
|
|
128
|
+
// Restore it through the same resumable path used by a prepared home.
|
|
129
|
+
journal.state = 'restarting';
|
|
130
|
+
await save(journalPath, journal);
|
|
131
|
+
} else if (!await stat(journal.candidate)) {
|
|
132
|
+
throw new Error('interrupted recovery cannot locate its retained original database');
|
|
133
|
+
}
|
|
134
|
+
// Before any move the canonical database is still the old one, and
|
|
135
|
+
// the candidate can simply remain with the abandoned journal.
|
|
136
|
+
}
|
|
137
|
+
if (['complete', 'restarting'].includes(journal.state)) {
|
|
138
|
+
if (journal.state === 'complete') {
|
|
139
|
+
const reader = new MailDatabase(join(home, 'mail.sqlite'));
|
|
140
|
+
let retirement;
|
|
141
|
+
try { retirement = reader.metadata('maildir_retirement'); } finally { reader.close(); }
|
|
142
|
+
if (retirement?.complete || retirement?.explicit_recovery !== journal.id) throw new Error('activated recovery cannot be abandoned');
|
|
143
|
+
journal.state = 'restarting';
|
|
144
|
+
await save(journalPath, journal);
|
|
145
|
+
}
|
|
146
|
+
const restoreDeadline = performance.now() + 2000;
|
|
147
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
148
|
+
const path = join(home, `mail.sqlite${suffix}`), abandoned = join(journal.directory, `abandoned-mail.sqlite${suffix}`);
|
|
149
|
+
if (!await stat(abandoned) && await stat(path)) await rename(path, abandoned);
|
|
150
|
+
if (!await restoreOriginal(journal, journalPath, suffix, restoreDeadline)) return journal;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
journal.state = 'abandoned'; journal.abandoned_at_ms = Date.now();
|
|
154
|
+
await save(journalPath, journal);
|
|
155
|
+
await rename(journalPath, join(journal.directory, 'abandoned.json'));
|
|
156
|
+
journal = null; restarted = true;
|
|
157
|
+
}
|
|
158
|
+
if (journal?.state === 'complete') {
|
|
159
|
+
const reader = new MailDatabase(join(home, 'mail.sqlite'));
|
|
160
|
+
try {
|
|
161
|
+
if (!reader.metadata('maildir_retirement')?.complete) await requireRecoverySource(reader, root);
|
|
162
|
+
} finally { reader.close(); }
|
|
163
|
+
return journal;
|
|
164
|
+
}
|
|
165
|
+
if (!journal) {
|
|
166
|
+
const reader = new DatabaseSync(join(home, 'mail.sqlite'), { readOnly: true });
|
|
167
|
+
let retirement;
|
|
168
|
+
try { retirement = JSON.parse(reader.prepare("SELECT value FROM mail_metadata WHERE key = 'maildir_retirement'").get()?.value ?? 'null'); }
|
|
169
|
+
finally { reader.close(); }
|
|
170
|
+
if (!retirement || retirement.source !== root || !retirement.archive || dirname(resolve(retirement.archive)) !== join(home, 'mail-retired')) throw new Error('mail recovery requires this home’s existing migration receipt');
|
|
171
|
+
const live = await stat(root), archived = await stat(retirement.archive);
|
|
172
|
+
if (!live?.isDirectory() || !archived?.isDirectory()) throw new Error('mail recovery requires both regular Maildir stores; originals retained');
|
|
173
|
+
if (retirement.inode != null && !same(identity(archived), { device: retirement.device, inode: retirement.inode })) throw new Error('retired Maildir differs from its ownership receipt');
|
|
174
|
+
const id = randomUUID(), directory = join(base, id);
|
|
175
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
176
|
+
journal = { version: 1, id, home, directory, state: 'import', root, archive: retirement.archive, live_identity: identity(live), archive_identity: identity(archived),
|
|
177
|
+
recovered_archive: join(home, 'mail-retired', `recovered-${id}`), candidate: join(directory, 'recovered.sqlite'),
|
|
178
|
+
historical: [{ directory: '', last: '' }], matched_messages: 0, restored_messages: 0, operational_revisions: 0, conflicts: [], sql_cursor: ['', ''] };
|
|
179
|
+
await save(journalPath, journal);
|
|
180
|
+
// Restoration and a fresh import each get their own bounded invocation.
|
|
181
|
+
if (restarted) return journal;
|
|
182
|
+
}
|
|
183
|
+
if (!/^[0-9a-f-]{36}$/.test(journal.id) || journal.directory !== join(base, journal.id) || journal.root !== root || journal.candidate !== join(journal.directory, 'recovered.sqlite')
|
|
184
|
+
|| dirname(resolve(journal.archive)) !== join(home, 'mail-retired') || journal.recovered_archive !== join(home, 'mail-retired', `recovered-${journal.id}`)) throw new Error('mail recovery journal contains paths outside its owned home');
|
|
185
|
+
if (!same(identity(await stat(journal.archive)), journal.archive_identity)) throw new Error('historical Maildir changed identity during recovery');
|
|
186
|
+
if (journal.state !== 'publish' && !same(identity(await stat(root)), journal.live_identity)) throw new Error('selected Maildir changed identity during recovery');
|
|
187
|
+
if (journal.state !== 'publish') {
|
|
188
|
+
database = new MailDatabase(journal.candidate);
|
|
189
|
+
database.statement('CREATE TABLE IF NOT EXISTS mail_recovery_revisions(source TEXT PRIMARY KEY, old_sha256 TEXT NOT NULL, live_sha256 TEXT)').run();
|
|
190
|
+
}
|
|
191
|
+
const phase = journal.state;
|
|
192
|
+
const deadline = performance.now() + 4500;
|
|
193
|
+
if (journal.state === 'import') {
|
|
194
|
+
const imported = await importMaildir(database, root, { budgetMs: 4500 });
|
|
195
|
+
journal.import = imported;
|
|
196
|
+
if (imported.complete) {
|
|
197
|
+
const unsafe = database.statement("SELECT source FROM mail_import_errors WHERE NOT (source LIKE 'sent/%/log.jsonl' OR detail = 'claim has no imported message') LIMIT 1").get();
|
|
198
|
+
if (unsafe) throw new Error(`mail recovery cannot lose unreadable input ${unsafe.source}; originals retained`);
|
|
199
|
+
journal.state = 'verify';
|
|
200
|
+
}
|
|
201
|
+
} else if (journal.state === 'verify') {
|
|
202
|
+
const verified = await verifyImportedMaildir(database, root, { after: journal.verified_after ?? '', budgetMs: 4500, rememberStamps: true });
|
|
203
|
+
journal.verified_after = verified.cursor;
|
|
204
|
+
if (verified.complete) journal.state = 'archive';
|
|
205
|
+
} else if (journal.state === 'archive') {
|
|
206
|
+
while (journal.historical.length && performance.now() < deadline) {
|
|
207
|
+
const group = journal.historical.at(-1), directory = join(journal.archive, group.directory);
|
|
208
|
+
const entries = (await readdir(directory, { withFileTypes: true })).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
209
|
+
let exhausted = true;
|
|
210
|
+
for (const entry of entries) {
|
|
211
|
+
if (entry.name <= group.last) continue;
|
|
212
|
+
if (performance.now() >= deadline) { exhausted = false; break; }
|
|
213
|
+
const relative = join(group.directory, entry.name), path = join(journal.archive, relative);
|
|
214
|
+
if (entry.isDirectory()) { group.last = entry.name; journal.historical.push({ directory: relative, last: '' }); exhausted = false; break; }
|
|
215
|
+
if (entry.isFile()) {
|
|
216
|
+
const parts = relative.split(/[/\\]/);
|
|
217
|
+
if (parts.length === 3 && ['new', 'cur', 'claimed'].includes(parts[1]) && entry.name.endsWith('.json')) {
|
|
218
|
+
const address = (await readFile(join(journal.archive, parts[0], 'address'), 'utf8')).trim();
|
|
219
|
+
envelope(database, address, await json(path), parts[1], journal);
|
|
220
|
+
} else {
|
|
221
|
+
const current = await stat(join(root, relative));
|
|
222
|
+
if (current && !current.isFile()) throw new Error(`historical operational input ${relative} changed record type; originals retained`);
|
|
223
|
+
const previousBytes = await readFile(path), currentBytes = current ? await readFile(join(root, relative)) : null;
|
|
224
|
+
if (!currentBytes || !previousBytes.equals(currentBytes)) {
|
|
225
|
+
journal.operational_revisions++;
|
|
226
|
+
const revision = { source: relative, selected: 'live-maildir', old_sha256: createHash('sha256').update(previousBytes).digest('hex'), live_sha256: currentBytes ? createHash('sha256').update(currentBytes).digest('hex') : null };
|
|
227
|
+
database.statement('INSERT OR REPLACE INTO mail_recovery_revisions(source, old_sha256, live_sha256) VALUES (?, ?, ?)').run(relative, revision.old_sha256, revision.live_sha256);
|
|
228
|
+
if (journal.conflicts.length < 20) journal.conflicts.push(revision);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
} else if (!entry.isSocket()) throw new Error(`historical input ${relative} is not a regular record; originals retained`);
|
|
232
|
+
group.last = entry.name;
|
|
233
|
+
}
|
|
234
|
+
if (exhausted) journal.historical.pop();
|
|
235
|
+
}
|
|
236
|
+
if (!journal.historical.length) journal.state = 'sqlite';
|
|
237
|
+
} else if (journal.state === 'sqlite') {
|
|
238
|
+
const legacy = new DatabaseSync(join(home, 'mail.sqlite'), { readOnly: true });
|
|
239
|
+
try {
|
|
240
|
+
const rows = legacy.prepare('SELECT mailbox, id, header, body, state FROM messages WHERE (mailbox, id) > (?, ?) ORDER BY mailbox, id LIMIT 256');
|
|
241
|
+
while (performance.now() < deadline) {
|
|
242
|
+
const batch = rows.all(...journal.sql_cursor);
|
|
243
|
+
if (!batch.length) { journal.state = 'publish'; break; }
|
|
244
|
+
for (const row of batch) {
|
|
245
|
+
if (performance.now() >= deadline) break;
|
|
246
|
+
if (row.body == null) {
|
|
247
|
+
const current = database.statement('SELECT header FROM messages WHERE mailbox = ? AND id = ?').get(row.mailbox, row.id);
|
|
248
|
+
if (!current || !same(JSON.parse(current.header), JSON.parse(row.header))) throw new Error(`bodyless historical envelope ${row.id} is absent or conflicting; originals retained`);
|
|
249
|
+
} else envelope(database, row.mailbox, { ...JSON.parse(row.header), body: row.body }, row.state, journal);
|
|
250
|
+
journal.sql_cursor = [row.mailbox, row.id];
|
|
251
|
+
}
|
|
252
|
+
// Comparison is read-only apart from restoring missing envelopes.
|
|
253
|
+
// Persist its cursor between batches so caller loss cannot discard
|
|
254
|
+
// the whole invocation's progress.
|
|
255
|
+
await save(journalPath, journal);
|
|
256
|
+
}
|
|
257
|
+
} finally { legacy.close(); }
|
|
258
|
+
if (journal.state === 'publish') {
|
|
259
|
+
database.setMetadata('maildir_import', { ...database.metadata('maildir_import'), messages: database.statement('SELECT count(*) AS count FROM messages').get().count });
|
|
260
|
+
database.setMetadata('maildir_retirement', { version: 2, source: root, archive: journal.recovered_archive, ...journal.live_identity,
|
|
261
|
+
complete: false, originals_retained: true, explicit_recovery: journal.id });
|
|
262
|
+
database.setMetadata('maildir_recovery', { policy: 'prefer-live-maildir', prepared: true, historical_maildir: journal.archive, historical_database: join(journal.directory, 'failed-mail.sqlite'), at_ms: Date.now() });
|
|
263
|
+
database.statement('PRAGMA wal_checkpoint(TRUNCATE)').get();
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
database?.close(); database = null;
|
|
267
|
+
await save(journalPath, journal);
|
|
268
|
+
// Publication has its own invocation: its full source-stamp check must
|
|
269
|
+
// not share the preceding comparison's work budget.
|
|
270
|
+
if (journal.state === 'publish' && phase !== 'sqlite') {
|
|
271
|
+
await offline(teamsHome, root);
|
|
272
|
+
const path = await stat(journal.candidate) ? journal.candidate : join(home, 'mail.sqlite');
|
|
273
|
+
const reader = new MailDatabase(path);
|
|
274
|
+
try { await requireRecoverySource(reader, root); } finally { reader.close(); }
|
|
275
|
+
// The journal precedes every move. Re-running after interruption finishes
|
|
276
|
+
// the same moves; no old database or Maildir is deleted or overwritten.
|
|
277
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
278
|
+
const from = join(home, `mail.sqlite${suffix}`), to = join(journal.directory, `failed-mail.sqlite${suffix}`);
|
|
279
|
+
if (!await stat(to) && await stat(from)) await rename(from, to);
|
|
280
|
+
}
|
|
281
|
+
if (await stat(journal.candidate)) await rename(journal.candidate, join(home, 'mail.sqlite'));
|
|
282
|
+
// The old unit and the retained predecessor still have a populated
|
|
283
|
+
// Maildir. Only candidate activation retires it, after its checkpoint.
|
|
284
|
+
if (!same(identity(await stat(root)), journal.live_identity) || !await stat(join(home, 'mail.sqlite'))) throw new Error('recovery publication cannot locate its live Maildir or prepared database');
|
|
285
|
+
journal.state = 'complete'; journal.completed_at_ms = Date.now();
|
|
286
|
+
await save(journalPath, journal);
|
|
287
|
+
}
|
|
288
|
+
return journal;
|
|
289
|
+
} finally { database?.close(); owner.close(); }
|
|
290
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Recovery prepares SQLite while the readable legacy tree stays in place.
|
|
2
|
+
// Admission and retirement recheck the exact files the offline door verified.
|
|
3
|
+
import { lstatSync } from 'node:fs';
|
|
4
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
5
|
+
import { isAbsolute, resolve, sep } from 'node:path';
|
|
6
|
+
|
|
7
|
+
export const fileStamp = value => ({ device: value.dev, inode: value.ino,
|
|
8
|
+
size: value.size, mtime: value.mtimeMs, ctime: value.ctimeMs });
|
|
9
|
+
const inspect = path => { try { return lstatSync(path); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } };
|
|
10
|
+
|
|
11
|
+
function inside(root, relative) {
|
|
12
|
+
const path = resolve(root, relative);
|
|
13
|
+
if (isAbsolute(relative) || (path !== root && !path.startsWith(root + sep))) throw new Error('recovery source receipt is outside its Maildir');
|
|
14
|
+
return path;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function requireDirectories(database, root) {
|
|
18
|
+
for (const row of database.statement("SELECT key, value FROM mail_metadata WHERE key LIKE 'maildir_directory:%'").iterate()) {
|
|
19
|
+
const saved = JSON.parse(row.value), path = inside(root, row.key.slice('maildir_directory:'.length));
|
|
20
|
+
const current = inspect(path);
|
|
21
|
+
if (saved.stamp === null && current === null) continue;
|
|
22
|
+
if (!current?.isDirectory() || !saved.stamp) throw new Error('selected Maildir directory changed; use mail recover --restart while offline');
|
|
23
|
+
const stamp = { device: current.dev, inode: current.ino, mtime: current.mtimeMs, ctime: current.ctimeMs };
|
|
24
|
+
const expected = { ...saved.stamp };
|
|
25
|
+
// Retirement and rollback rename this same root inode. Those renames change
|
|
26
|
+
// its ctime, not its entries; root mtime and every child/file stamp still
|
|
27
|
+
// detect a writer. Nested directories are never renamed by this protocol.
|
|
28
|
+
if (path === root) { delete stamp.ctime; delete expected.ctime; }
|
|
29
|
+
if (!current.isDirectory() || !isDeepStrictEqual(stamp, expected)) throw new Error('selected Maildir changed after verification; use mail recover --restart while offline');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Recheck every verified file and directory in the admission worker. A new, removed
|
|
34
|
+
* or replaced entry during the file pass invalidates the whole observation. */
|
|
35
|
+
export async function requireRecoverySource(database, root) {
|
|
36
|
+
root = resolve(root);
|
|
37
|
+
requireDirectories(database, root);
|
|
38
|
+
let count = 0;
|
|
39
|
+
for (const row of database.statement('SELECT source, stamp FROM mail_recovery_sources').iterate()) {
|
|
40
|
+
// This pass reads metadata only. Scheduling a promise and filesystem job
|
|
41
|
+
// for each of a large home's files put the prepared admission back over
|
|
42
|
+
// its deadline. Direct observations keep every exact stamp check while
|
|
43
|
+
// avoiding those 128,000 asynchronous jobs and their batch barriers.
|
|
44
|
+
const current = inspect(inside(root, row.source));
|
|
45
|
+
if (!current?.isFile() || !isDeepStrictEqual(fileStamp(current), JSON.parse(row.stamp))) throw new Error(`verified source changed: ${row.source}; use mail recover --restart while offline`);
|
|
46
|
+
count++;
|
|
47
|
+
}
|
|
48
|
+
if (count !== database.statement('SELECT count(*) AS count FROM mail_import_sources').get().count) throw new Error('recovery has unverified input; publication refused');
|
|
49
|
+
requireDirectories(database, root);
|
|
50
|
+
return count;
|
|
51
|
+
}
|
package/mail/retire-maildir.mjs
CHANGED
|
@@ -7,13 +7,26 @@ import { createHash, randomUUID } from 'node:crypto';
|
|
|
7
7
|
import { dirname, isAbsolute, join, resolve, sep } from 'node:path';
|
|
8
8
|
import { importMaildir } from './import-maildir.mjs';
|
|
9
9
|
import { requireLegacyMaildirReady } from './migration-readiness.mjs';
|
|
10
|
+
import { fileStamp, requireRecoverySource } from './recovery-source.mjs';
|
|
11
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
10
12
|
|
|
11
13
|
const stat = async (path) => {
|
|
12
14
|
try { return await lstat(path); }
|
|
13
15
|
catch (error) { if (error.code === 'ENOENT') return null; throw error; }
|
|
14
16
|
};
|
|
15
17
|
|
|
16
|
-
export async function verifyImportedMaildir(database, root) {
|
|
18
|
+
export async function verifyImportedMaildir(database, root, { after = '', budgetMs = null, rememberStamps = false } = {}) {
|
|
19
|
+
const began = performance.now();
|
|
20
|
+
if (rememberStamps) database.statement('CREATE TABLE IF NOT EXISTS mail_recovery_sources(source TEXT PRIMARY KEY, stamp TEXT NOT NULL)').run();
|
|
21
|
+
const stamps = [];
|
|
22
|
+
const flushStamps = () => {
|
|
23
|
+
if (!stamps.length) return;
|
|
24
|
+
database.transaction(() => {
|
|
25
|
+
const insert = database.statement('INSERT OR REPLACE INTO mail_recovery_sources(source, stamp) VALUES (?, ?)');
|
|
26
|
+
for (const [source, stamp] of stamps) insert.run(source, stamp);
|
|
27
|
+
});
|
|
28
|
+
stamps.length = 0;
|
|
29
|
+
};
|
|
17
30
|
for (const { key, value } of database.statement("SELECT key, value FROM mail_metadata WHERE key LIKE 'maildir_directory:%'").all()) {
|
|
18
31
|
const saved = JSON.parse(value), path = resolve(root, key.slice('maildir_directory:'.length));
|
|
19
32
|
if (path !== root && !path.startsWith(root + sep)) throw new Error('mail import directory receipt is outside its source');
|
|
@@ -22,11 +35,12 @@ export async function verifyImportedMaildir(database, root) {
|
|
|
22
35
|
if (JSON.stringify(saved.stamp) !== JSON.stringify(stamp)) throw new Error(`mail source directory changed before retirement: ${path}`);
|
|
23
36
|
}
|
|
24
37
|
const sources = database.statement('SELECT source, sha256 FROM mail_import_sources WHERE source > ? ORDER BY source LIMIT 256');
|
|
25
|
-
let cursor =
|
|
38
|
+
let cursor = after;
|
|
26
39
|
for (;;) {
|
|
27
40
|
const batch = sources.all(cursor);
|
|
28
|
-
if (!batch.length) return;
|
|
41
|
+
if (!batch.length) { flushStamps(); return { complete: true, cursor }; }
|
|
29
42
|
for (const row of batch) {
|
|
43
|
+
if (budgetMs != null && performance.now() - began >= budgetMs) { flushStamps(); return { complete: false, cursor }; }
|
|
30
44
|
const path = resolve(root, row.source);
|
|
31
45
|
if (isAbsolute(row.source) || !path.startsWith(root + sep)) throw new Error('an imported source is outside its Maildir');
|
|
32
46
|
let parent = dirname(path);
|
|
@@ -34,10 +48,18 @@ export async function verifyImportedMaildir(database, root) {
|
|
|
34
48
|
if (!(await stat(parent))?.isDirectory()) throw new Error(`mail source directory changed before retirement: ${row.source}`);
|
|
35
49
|
parent = dirname(parent);
|
|
36
50
|
}
|
|
37
|
-
|
|
51
|
+
const before = await stat(path);
|
|
52
|
+
if (!before?.isFile()) throw new Error(`mail source changed before retirement: ${row.source}`);
|
|
38
53
|
const hash = createHash('sha256');
|
|
39
54
|
for await (const bytes of createReadStream(path)) hash.update(bytes);
|
|
40
55
|
if (hash.digest('hex') !== row.sha256) throw new Error(`mail source bytes changed before retirement: ${row.source}`);
|
|
56
|
+
if (rememberStamps) {
|
|
57
|
+
const current = await stat(path);
|
|
58
|
+
if (!current?.isFile() || !isDeepStrictEqual(fileStamp(before), fileStamp(current))) throw new Error(`mail source changed during verification: ${row.source}`);
|
|
59
|
+
stamps.push([row.source, JSON.stringify(fileStamp(current))]);
|
|
60
|
+
if (stamps.length === 256) flushStamps();
|
|
61
|
+
}
|
|
62
|
+
cursor = row.source;
|
|
41
63
|
}
|
|
42
64
|
cursor = batch.at(-1).source;
|
|
43
65
|
}
|
|
@@ -55,6 +77,25 @@ export async function migrateMaildir(database, root, { log = () => {}, committed
|
|
|
55
77
|
database.setMetadata('maildir_retirement', retirement);
|
|
56
78
|
}
|
|
57
79
|
if (retirement.source !== root) throw new Error('the mail migration source differs from its stored receipt');
|
|
80
|
+
if (!retirement.complete && retirement.explicit_recovery && database.metadata('maildir_recovery')?.prepared === true) {
|
|
81
|
+
const archive = resolve(retirement.archive);
|
|
82
|
+
if (dirname(archive) !== resolve(dirname(root), 'mail-retired') || !source?.isDirectory() || await stat(archive)) throw new Error('prepared recovery requires its original live Maildir and unused archive');
|
|
83
|
+
await requireLegacyMaildirReady(root);
|
|
84
|
+
await requireRecoverySource(database, root);
|
|
85
|
+
await rename(root, archive);
|
|
86
|
+
try {
|
|
87
|
+
await requireRecoverySource(database, archive);
|
|
88
|
+
if (await stat(root)) throw new Error('a writer recreated the Maildir during recovery activation; both stores retained');
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
if (!await stat(root)) await rename(archive, root);
|
|
92
|
+
throw error;
|
|
93
|
+
}
|
|
94
|
+
const original = await stat(archive);
|
|
95
|
+
database.setMetadata('maildir_retirement', { ...retirement, inode: original.ino, device: original.dev, complete: true, completed_at_ms: Date.now() });
|
|
96
|
+
log(`prepared recovery retired at activation; original Maildir retained at ${archive}`);
|
|
97
|
+
return database.metadata('maildir_import');
|
|
98
|
+
}
|
|
58
99
|
if (retirement.complete) {
|
|
59
100
|
if (source && !committed) throw new Error(`the retired Maildir was recreated at ${root}; it is retained, and this daemon will not choose between two mail stores`);
|
|
60
101
|
if (source) log(`committed SQLite mailbox remains serving; recreated legacy records at ${root} are retained for explicit recovery`);
|
|
@@ -4,17 +4,32 @@
|
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
5
|
import { lstat } from 'node:fs/promises';
|
|
6
6
|
import { join, dirname, resolve } from 'node:path';
|
|
7
|
-
import { MailDatabase } from './sqlite.mjs';
|
|
7
|
+
import { MailDatabase, MAIL_SCHEMA_VERSION } from './sqlite.mjs';
|
|
8
8
|
import { importMaildir } from './import-maildir.mjs';
|
|
9
9
|
import { requireLegacyMaildirReady } from './migration-readiness.mjs';
|
|
10
10
|
import { verifyImportedMaildir } from './retire-maildir.mjs';
|
|
11
11
|
import { requireSessionIndexRuntime } from '../session-index-owner.mjs';
|
|
12
|
+
import { requireRecoverySource } from './recovery-source.mjs';
|
|
12
13
|
|
|
13
14
|
const exists = async (path) => {
|
|
14
15
|
try { return await lstat(path); }
|
|
15
16
|
catch (error) { if (error.code === 'ENOENT') return null; throw error; }
|
|
16
17
|
};
|
|
17
18
|
|
|
19
|
+
async function preparedAdmission(database, home, root, guard) {
|
|
20
|
+
const retirement = database.metadata('maildir_retirement');
|
|
21
|
+
if (retirement?.complete || !retirement?.explicit_recovery || database.metadata('maildir_recovery')?.prepared !== true
|
|
22
|
+
|| database.metadata('schema_version') !== MAIL_SCHEMA_VERSION) return null;
|
|
23
|
+
if (retirement.source !== root) throw new Error('mail migration source differs from its stored receipt');
|
|
24
|
+
if (!retirement.archive || dirname(resolve(retirement.archive)) !== resolve(home, 'mail-retired')) throw new Error('migration snapshot is outside this home');
|
|
25
|
+
if (!(await exists(root))?.isDirectory() || await exists(retirement.archive)) throw new Error('prepared recovery no longer has its original live Maildir');
|
|
26
|
+
await requireRecoverySource(database, root);
|
|
27
|
+
const imported = database.metadata('maildir_import');
|
|
28
|
+
return { version: 1, stage: 'upgrade_admission', admitted: true, home,
|
|
29
|
+
messages: imported.messages, records: imported.records, retained_diagnostics: imported.skipped,
|
|
30
|
+
legacy_wrappers: guard.live.length, schema: database.metadata('schema_version'), prepared_recovery: retirement.explicit_recovery };
|
|
31
|
+
}
|
|
32
|
+
|
|
18
33
|
export async function inspectUpgradeReadiness(teamsHome, directory) {
|
|
19
34
|
requireSessionIndexRuntime();
|
|
20
35
|
// Resolve all daemon modules before admission, without starting its carrier,
|
|
@@ -29,6 +44,23 @@ export async function inspectUpgradeReadiness(teamsHome, directory) {
|
|
|
29
44
|
if (await exists(live)) {
|
|
30
45
|
const reader = new DatabaseSync(live, { readOnly: true });
|
|
31
46
|
try {
|
|
47
|
+
// This release's offline recovery has already applied its schema and
|
|
48
|
+
// verified every input. Inspect that prepared state in one read-only
|
|
49
|
+
// SQLite transaction; it needs no speculative writable database copy.
|
|
50
|
+
reader.exec('BEGIN');
|
|
51
|
+
if (reader.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'mail_metadata'").get()) {
|
|
52
|
+
const statements = new Map();
|
|
53
|
+
const view = { statement(sql) {
|
|
54
|
+
if (!statements.has(sql)) statements.set(sql, reader.prepare(sql));
|
|
55
|
+
return statements.get(sql);
|
|
56
|
+
}, metadata(key) {
|
|
57
|
+
const row = this.statement('SELECT value FROM mail_metadata WHERE key = ?').get(key);
|
|
58
|
+
return row ? JSON.parse(row.value) : null;
|
|
59
|
+
} };
|
|
60
|
+
const prepared = await preparedAdmission(view, home, root, guard);
|
|
61
|
+
if (prepared) return prepared;
|
|
62
|
+
}
|
|
63
|
+
reader.exec('ROLLBACK');
|
|
32
64
|
// SQLite creates a coherent snapshot, including committed WAL pages;
|
|
33
65
|
// copying the db/WAL files independently cannot establish that.
|
|
34
66
|
reader.prepare('VACUUM INTO ?').run(candidate);
|
|
@@ -37,9 +69,11 @@ export async function inspectUpgradeReadiness(teamsHome, directory) {
|
|
|
37
69
|
database = new MailDatabase(candidate);
|
|
38
70
|
const retirement = database.metadata('maildir_retirement');
|
|
39
71
|
const source = await exists(root);
|
|
40
|
-
let input = root;
|
|
41
72
|
if (retirement?.source && retirement.source !== root) throw new Error('mail migration source differs from its stored receipt');
|
|
42
73
|
if (retirement?.archive && dirname(resolve(retirement.archive)) !== resolve(home, 'mail-retired')) throw new Error('migration snapshot is outside this home');
|
|
74
|
+
const prepared = await preparedAdmission(database, home, root, guard);
|
|
75
|
+
if (prepared) return prepared;
|
|
76
|
+
let input = root;
|
|
43
77
|
if (retirement?.archive && await exists(retirement.archive)) {
|
|
44
78
|
if (source) throw new Error('both the original migration snapshot and a recreated Maildir exist; explicit recovery is required before activation');
|
|
45
79
|
input = retirement.archive;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-teams",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.193",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
|
|
6
6
|
"license": "MIT",
|
package/terminal-mail.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Per-recipient terminal mailbox. A retained paste is never repeated: retries
|
|
2
|
-
// press Enter and look for the exact
|
|
2
|
+
// press Enter and look for the exact message in the native transcript (a user row, or the queued_command
|
|
3
|
+
// attachment a running Claude Code turn absorbs it as).
|
|
3
4
|
import { statSync, openSync, readSync, closeSync } from 'node:fs';
|
|
4
5
|
import { createHash, randomUUID } from 'node:crypto';
|
|
5
6
|
import { spawnSync } from 'node:child_process';
|
|
@@ -52,7 +53,16 @@ function claudeHolds(batch, shown, owned = false) {
|
|
|
52
53
|
return batch.placeholder === placeholder[0];
|
|
53
54
|
}
|
|
54
55
|
|
|
55
|
-
|
|
56
|
+
/** Whether Claude Code recorded `text` as the prompt `wanted`. Claude records a large paste in a native
|
|
57
|
+
* `<pasted_content>` envelope; its whole body is compared, with line-ending and outer-whitespace normalization. */
|
|
58
|
+
function claudeSame(text, wanted) {
|
|
59
|
+
const native = normalized(text), want = normalized(wanted);
|
|
60
|
+
if (native === want) return true;
|
|
61
|
+
const pasted = /^<pasted_content id="([0-9a-f]{4})">\n([\s\S]*)\n<\/pasted_content id="\1">$/u.exec(native);
|
|
62
|
+
return Boolean(pasted) && normalized(pasted[2]) === want;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function confirmed(batch) {
|
|
56
66
|
let fd;
|
|
57
67
|
try { fd = openSync(batch.path, 'r'); } catch (error) { if (error.code === 'ENOENT') return false; throw error; }
|
|
58
68
|
try {
|
|
@@ -68,6 +78,12 @@ function confirmed(batch) {
|
|
|
68
78
|
while ((end = buffered.indexOf('\n')) >= 0) {
|
|
69
79
|
const line = buffered.slice(0, end); buffered = buffered.slice(end + 1);
|
|
70
80
|
let row; try { row = JSON.parse(line); } catch { continue; }
|
|
81
|
+
// A prompt submitted while a turn runs is queued, and reaches the model as a `queued_command` attachment when
|
|
82
|
+
// the turn absorbs it (queue-operation remove, reason absorbed_mid_turn); no `user` row is written for it.
|
|
83
|
+
// That attachment is its delivery. An enqueue alone is not: a queued prompt can still be withdrawn.
|
|
84
|
+
if (batch.kind === 'claude-code' && row.type === 'attachment' && row.attachment?.type === 'queued_command'
|
|
85
|
+
&& row.isSidechain !== true && (!batch.sessionId || row.sessionId === batch.sessionId)
|
|
86
|
+
&& typeof row.attachment.prompt === 'string' && claudeSame(row.attachment.prompt, batch.text)) return { fragments: 1 };
|
|
71
87
|
const content = batch.kind === 'codex' && row.type === 'response_item' && row.payload?.type === 'message' && row.payload.role === 'user'
|
|
72
88
|
? row.payload.content : batch.kind === 'claude-code' && row.type === 'user' && !row.isMeta && (!batch.sessionId || row.sessionId === batch.sessionId)
|
|
73
89
|
? row.message?.content : batch.kind === 'grok' && row.type === 'user' ? row.content : null;
|
|
@@ -75,13 +91,7 @@ function confirmed(batch) {
|
|
|
75
91
|
if (text === null || text.length === 0) continue;
|
|
76
92
|
if (text === batch.text) return { fragments: 1 };
|
|
77
93
|
if (batch.kind === 'claude-code') {
|
|
78
|
-
|
|
79
|
-
if (native === wanted) return { fragments: 1 };
|
|
80
|
-
// Claude records a large paste in this native envelope. Compare its
|
|
81
|
-
// whole body with matching identifiers, using the existing line-ending
|
|
82
|
-
// and outer-whitespace normalization.
|
|
83
|
-
const pasted = /^<pasted_content id="([0-9a-f]{4})">\n([\s\S]*)\n<\/pasted_content id="\1">$/u.exec(native);
|
|
84
|
-
if (pasted && normalized(pasted[2]) === wanted) return { fragments: 1 };
|
|
94
|
+
if (claudeSame(text, batch.text)) return { fragments: 1 };
|
|
85
95
|
continue;
|
|
86
96
|
}
|
|
87
97
|
// A Windows Codex is pasted its text with line ends as `\n` (codexWindowsPastes).
|