@volter/supercode-teams 0.3.184 → 0.3.187
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/app-door.mjs +23 -3
- package/bin/teams-cli.mjs +608 -0
- package/bin/teams-supervised.mjs +5 -0
- package/bin/teams.mjs +49 -582
- package/bin/workspace/published-install.mjs +46 -0
- package/bin/workspace/release-state.mjs +82 -0
- package/bin/workspace/retain-release.mjs +186 -0
- package/bin/workspace/upgrade-supervisor.mjs +234 -0
- package/bin/workspace/upgrade-watch.mjs +5 -2
- package/bin/workspace.mjs +19 -6
- package/machine.mjs +41 -22
- package/mail/import-maildir.mjs +8 -7
- package/mail/recovery-view.mjs +36 -0
- package/mail/retire-maildir.mjs +57 -68
- package/mail/sqlite-store.mjs +15 -5
- package/mail/upgrade-admission.mjs +44 -0
- package/mail/upgrade-readiness.mjs +59 -0
- package/mail/upgrade-state.mjs +78 -0
- package/package.json +1 -1
- package/session-index-owner.mjs +20 -8
- package/session-index-watches.mjs +28 -10
package/machine.mjs
CHANGED
|
@@ -559,7 +559,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
559
559
|
try {
|
|
560
560
|
const record = readJson(this.p.machine, null);
|
|
561
561
|
if (record?.pid !== process.pid) return;
|
|
562
|
-
writeJson(this.p.machine, { ...record, release: this.#release() });
|
|
562
|
+
writeJson(this.p.machine, { ...record, release: this.#release(), mail_ready: false });
|
|
563
563
|
} catch { /* its record says no release */ }
|
|
564
564
|
});
|
|
565
565
|
// the panes' stable porthole hook and the machine's stable supercode brought up to this release's at start, under
|
|
@@ -714,7 +714,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
714
714
|
try {
|
|
715
715
|
requireSessionIndexRuntime();
|
|
716
716
|
this.bornAt ??= Date.now();
|
|
717
|
-
this.mailStore = new SqliteMailStore({ mailRoot: join(supercodeHome(this.env), 'mail'), machine: machineName(this.env), log: (line) => this.emit('records-log', line) });
|
|
717
|
+
this.mailStore = new SqliteMailStore({ mailRoot: join(supercodeHome(this.env), 'mail'), machine: machineName(this.env), committed: this.env.SUPERCODE_MAIL_COMMITTED === '1', startupRecovery: this.env.SUPERCODE_MAIL_STARTUP_RECOVERY === '1', log: (line) => this.emit('records-log', line) });
|
|
718
718
|
this.mailStore.acquire();
|
|
719
719
|
await this.mailStore.start();
|
|
720
720
|
this.sessionIndex = new SessionIndexOwner({ database: this.mailStore.database, env: this.env, log: (line) => this.emit('records-log', line) });
|
|
@@ -738,18 +738,29 @@ export class TeamsMachine extends MachineCore {
|
|
|
738
738
|
|
|
739
739
|
async afterListen() {
|
|
740
740
|
try {
|
|
741
|
+
// First activation must prove its native before any delivery. Later
|
|
742
|
+
// restarts keep serving the committed mail while native indexing starts.
|
|
743
|
+
if (this.env.SUPERCODE_UPGRADE_ACTIVATING === '1') await this.#serveRequest('harness.v1.harnesses.list', {}, 4000);
|
|
741
744
|
this.#startLive();
|
|
742
745
|
await this.#startMail();
|
|
743
|
-
this.mailReady = true;
|
|
744
|
-
} catch (error) {
|
|
745
|
-
await this.stop();
|
|
746
|
-
throw error;
|
|
747
|
-
}
|
|
748
746
|
// Web apps on this machine reach it through the app door when the person has turned it on (app-door.mjs).
|
|
749
747
|
this.appDoor = serveAppDoor({ home: this.home, open: () => { const [page, daemon] = inProcessPair(); void this.acceptLocal(daemon, { appDoor: true }); return page; }, log: (row) => this.log.append({ principal: 'app-door', cap: 'app-door', target: row.origin ?? '*', outcome: row.outcome, detail: row.detail }) });
|
|
750
748
|
this.#startHealth();
|
|
751
749
|
this.#startRecords();
|
|
752
750
|
if(Object.keys(this.inputRepairs).length)this.#watchInputRepairs();
|
|
751
|
+
const readyRecord = readJson(this.p.machine, null);
|
|
752
|
+
if (readyRecord?.pid !== process.pid) throw new Error('daemon startup lost its machine record before mail readiness');
|
|
753
|
+
writeJson(this.p.machine, { ...readyRecord, mail_ready: true, mail_recovery: this.mailStore.deferred === true,
|
|
754
|
+
...(this.env.SUPERCODE_UPGRADE_SUPERVISOR ? { upgrade_supervisor: Number(this.env.SUPERCODE_UPGRADE_SUPERVISOR) } : {}) });
|
|
755
|
+
if (!this.mailStore.deferred && this.env.SUPERCODE_UPGRADE_READY_FILE) writeJson(this.env.SUPERCODE_UPGRADE_READY_FILE, { pid: process.pid,
|
|
756
|
+
supervisor: Number(this.env.SUPERCODE_UPGRADE_SUPERVISOR),
|
|
757
|
+
release: JSON.parse(this.env.SUPERCODE_UPGRADE_RELEASE_RECEIPT), at_ms: Date.now() });
|
|
758
|
+
this.mailReady = true;
|
|
759
|
+
if (!this.mailStore.deferred) this.activateMail();
|
|
760
|
+
} catch (error) {
|
|
761
|
+
await this.stop();
|
|
762
|
+
throw error;
|
|
763
|
+
}
|
|
753
764
|
}
|
|
754
765
|
|
|
755
766
|
// ---- this machine's records (order 3 PR 3, machine-records.mjs) -------------
|
|
@@ -845,8 +856,6 @@ export class TeamsMachine extends MachineCore {
|
|
|
845
856
|
async #startMail() {
|
|
846
857
|
const mailRoot = join(supercodeHome(this.env), 'mail');
|
|
847
858
|
// the Messages API a Claude relay talks to (step 4a: the daemon's, in process; the native watch served it)
|
|
848
|
-
this.relayEndpoint = serveRelayEndpoint(this.mailStore, { log: (line) => this.emit('mail-log', line) })
|
|
849
|
-
.catch((error) => { this.emit('mail-log', `relay endpoint: not served (${error.message}); a Claude session cannot be sent mail until the daemon starts again`); return null; });
|
|
850
859
|
// the session index's rows by address (its descriptors, as the records' index subscription hands them over)
|
|
851
860
|
this.sessionDescriptors = new Map();
|
|
852
861
|
const live = this.live;
|
|
@@ -911,27 +920,23 @@ export class TeamsMachine extends MachineCore {
|
|
|
911
920
|
// a relay's inbound mail its hook kept while this daemon did not answer (the front door's spool): filed now, each
|
|
912
921
|
// minute, and before each relay's inbound call
|
|
913
922
|
this.hookSpool = join(supercodeHome(this.env), 'hook-spool', 'relay-inbound');
|
|
914
|
-
this.mailHooks.drainSpool(this.hookSpool);
|
|
915
|
-
this.mailHooks.drainPoolReplies();
|
|
916
|
-
this.hookSpoolTimer = setInterval(() => this.mailHooks.drainSpool(this.hookSpool), 60_000); this.hookSpoolTimer.unref?.();
|
|
917
923
|
// the mailbox's own delivery (step 4b): each wake attempted by its session's door, or carried to its machine
|
|
918
|
-
|
|
924
|
+
const carrier = new MailCarrier({ machine, mailRoot, store: this.mailStore, live,
|
|
919
925
|
handOver: (address, id) => this.mailSend.handOver(address, id),
|
|
920
926
|
typeTurn, typeUserTurns: (address, pane) => this.mailSend.typeUserTurns(address, pane), paneOf: (session) => this.mail.paneOf(session),
|
|
921
927
|
askMachine: (to, request) => this.#askMailDoor(to, request, MAIL_SEND_MS),
|
|
922
|
-
log: (line) => this.emit('mail-log', line) })
|
|
928
|
+
log: (line) => this.emit('mail-log', line) });
|
|
923
929
|
// idle notices and final replies (step 4c): each subscribed session's activity followed through harness serve
|
|
924
|
-
|
|
930
|
+
const idleNotices = new IdleNotices({ machine, store: this.mailStore, live,
|
|
925
931
|
serveRequest: (method, params, ms) => this.#serveRequest(method, params, ms),
|
|
926
932
|
locatorOf: (address) => this.sessionDescriptors?.get(address)?.locator ?? null,
|
|
927
|
-
log: (line) => this.emit('mail-log', line) })
|
|
933
|
+
log: (line) => this.emit('mail-log', line) });
|
|
928
934
|
// native Codex questions as mail (step 4c): the poll of each launched Codex pane's own app-server, in process
|
|
929
|
-
this.nativeQuestions.start();
|
|
930
935
|
// an agent's upkeep (step 4d): folder sessions become threads, typed lines reach their CC, idle launched threads end
|
|
931
|
-
|
|
936
|
+
const agentUpkeep = new AgentUpkeep({ machine, mailRoot, store: this.mailStore, agents: this.agents, live,
|
|
932
937
|
claudeProjects: () => [...live.claudeDirs.keys()].map((dir) => join(dirname(dir), 'projects')),
|
|
933
938
|
lastMessageAt: (address) => this.sessionDescriptors?.get(address)?.updated_at_ms ?? null,
|
|
934
|
-
program: this.supercodeBin, env: this.env, log: (line) => this.emit('mail-log', line) })
|
|
939
|
+
program: this.supercodeBin, env: this.env, log: (line) => this.emit('mail-log', line) });
|
|
935
940
|
// User turns share the persisted delivery schedule; only their own raw-input door types them.
|
|
936
941
|
// an agent's verbs (3b-2): its threads, a delegation, its record
|
|
937
942
|
this.agentVerbs = new AgentVerbs({ machine, store: this.mailStore, agents: this.agents, live, supercodeBin: this.supercodeBin, env: this.env,
|
|
@@ -941,9 +946,18 @@ export class TeamsMachine extends MachineCore {
|
|
|
941
946
|
// the names sessions ran under (`<mail>/names.json`): a name an agent was given reaches the same session after it
|
|
942
947
|
// runs under another, kept from the map's changes (at most once a second)
|
|
943
948
|
let naming = null;
|
|
944
|
-
|
|
949
|
+
this.activateMail = () => {
|
|
950
|
+
this.relayEndpoint = serveRelayEndpoint(this.mailStore, { log: line => this.emit('mail-log', line) })
|
|
951
|
+
.catch(error => { this.emit('mail-log', `relay endpoint: not served (${error.message})`); return null; });
|
|
952
|
+
this.mailHooks.drainSpool(this.hookSpool);
|
|
953
|
+
this.mailHooks.drainPoolReplies();
|
|
954
|
+
this.hookSpoolTimer = setInterval(() => this.mailHooks.drainSpool(this.hookSpool), 60_000); this.hookSpoolTimer.unref?.();
|
|
955
|
+
this.carrier = carrier.start(); this.idleNotices = idleNotices.start();
|
|
956
|
+
this.nativeQuestions.start(); this.agentUpkeep = agentUpkeep.start();
|
|
957
|
+
void this.relays.sweepAtStart().catch(error => this.emit('mail-log', `relay pool: predecessors retained (${error.message})`));
|
|
958
|
+
};
|
|
945
959
|
live.on('changed', () => {
|
|
946
|
-
if (this.stopping) return;
|
|
960
|
+
if (this.stopping || !this.mailReady || this.mailStore.deferred) return;
|
|
947
961
|
if (naming) return;
|
|
948
962
|
naming = setTimeout(() => { naming = null; try { this.mailSend.rememberNames(live.sessions()); } catch { /* kept at the next change */ } }, 1000);
|
|
949
963
|
naming.unref?.();
|
|
@@ -1901,6 +1915,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
1901
1915
|
// receiver named rather than addressed needs `mail:*`.
|
|
1902
1916
|
const { admitted, principal } = entry;
|
|
1903
1917
|
const request = requestOverride??stream.params?.request;
|
|
1918
|
+
if (this.mailStore.deferred && !['list', 'show', 'status', 'history'].includes(request?.op)) return refuse('mail migration is deferred; this daemon serves reads from retained originals while damaged operational records await repair');
|
|
1904
1919
|
if (request?.op === 'turn') return this.#serveTurn(entry, request, stream, accept, refuse);
|
|
1905
1920
|
if (request?.op === 'history') return this.#serveMailHistory(entry, request, stream, accept, refuse);
|
|
1906
1921
|
const to = typeof request?.to === 'string' ? request.to : '*';
|
|
@@ -2303,7 +2318,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
2303
2318
|
const answer = spawnSync(this.supercodeBin, ['--version'], { env: this.env, encoding: 'utf8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'] });
|
|
2304
2319
|
native = answer.status === 0 ? answer.stdout.trim() || null : null;
|
|
2305
2320
|
} catch { /* unread */ }
|
|
2306
|
-
return { teams, native };
|
|
2321
|
+
return { teams, native, supercode: this.env.SUPERCODE_RELEASE_VERSION ?? null };
|
|
2307
2322
|
}
|
|
2308
2323
|
|
|
2309
2324
|
#portholeHook() {
|
|
@@ -3633,6 +3648,9 @@ export class TeamsMachine extends MachineCore {
|
|
|
3633
3648
|
entry.admitted = fresh;
|
|
3634
3649
|
}
|
|
3635
3650
|
if (!this.mailReady || this.stopping) throw Object.assign(new Error(this.stopping ? 'this machine daemon is stopping' : 'this machine daemon is preparing its mail database; retry once migration completes'), { code: 503 });
|
|
3651
|
+
if (this.mailStore.deferred && ((method.startsWith('harness.v1.mail.') && !['list', 'show', 'statistics', 'names', 'tools', 'changes'].some(name => method === `harness.v1.mail.${name}`)) || method === 'harness.v1.agent.declare')) {
|
|
3652
|
+
throw Object.assign(new Error('mail migration is deferred; public mail reads remain available, delivery and acknowledgement await operational-state repair'), { code: 503 });
|
|
3653
|
+
}
|
|
3636
3654
|
const { admitted, principal } = entry;
|
|
3637
3655
|
const operator = admitted.class === 'operator';
|
|
3638
3656
|
const need = (cap, target) => {
|
|
@@ -3842,6 +3860,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
3842
3860
|
// never needing a person at it. Only a daemon its service manager brings back may go: one run by hand would not return.
|
|
3843
3861
|
if (!operator) { this.log.append({ principal, cap: method, target: '*', outcome: 'refused' }); throw Object.assign(new Error('only an operator restarts the machine daemon'), { code: 403 }); }
|
|
3844
3862
|
if (!this.restartHandler) throw Object.assign(new Error('this daemon runs without a service manager, so nothing would start it again; restart it where it runs'), { code: 'unsupported' });
|
|
3863
|
+
await this.restartAdmission?.();
|
|
3845
3864
|
act(method, '*', 'ok', params.reason ? String(params.reason).slice(0, 200) : null);
|
|
3846
3865
|
// The answer goes out first; the service manager starts the next daemon.
|
|
3847
3866
|
setTimeout(() => this.restartHandler(params.reason ? String(params.reason) : null), 500).unref?.();
|
package/mail/import-maildir.mjs
CHANGED
|
@@ -48,7 +48,7 @@ async function* logRecords(stream, start) {
|
|
|
48
48
|
else if (fragments.length) yield { bytes: assembled(), offset: position };
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
export async function importMaildir(database, root, { log = () => {}, legacyRoot = root } = {}) {
|
|
51
|
+
export async function importMaildir(database, root, { log = () => {}, legacyRoot = root, deferOperational = false } = {}) {
|
|
52
52
|
const began = performance.now();
|
|
53
53
|
const work = { source_reads: 0, source_bytes: 0, committed_batches: 0, max_queued_operations: 0, resumed_directories: 0, resumed_entries: 0, resumed_log_bytes: 0, elapsed_ms: 0 };
|
|
54
54
|
const complete = database.metadata('maildir_import');
|
|
@@ -85,13 +85,13 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
85
85
|
if (pending.length >= 256) flush();
|
|
86
86
|
};
|
|
87
87
|
const errors = database.statement('INSERT INTO mail_import_errors(source, mailbox, detail, critical) VALUES (?, ?, ?, ?) ON CONFLICT(source) DO UPDATE SET mailbox = excluded.mailbox, detail = excluded.detail, critical = excluded.critical');
|
|
88
|
-
const failed = (path, error, mailbox = null) => {
|
|
88
|
+
const failed = (path, error, mailbox = null, { diagnostic = false } = {}) => {
|
|
89
89
|
const source = relative(root, path);
|
|
90
90
|
const parts = source.split(/[\\/]/);
|
|
91
|
-
const critical = ['agents', 'threads', 'native-questions', 'native-launches', 'relays', 'machines'
|
|
91
|
+
const critical = !diagnostic && (['agents', 'threads', 'native-questions', 'native-launches', 'relays', 'machines'].includes(parts[0])
|
|
92
92
|
|| ['claims', 'subscriptions', 'wake'].includes(parts[1])
|
|
93
93
|
|| parts[0] === 'names.json' || parts[0] === 'relay-endpoint.json'
|
|
94
|
-
|| (parts[0] === 'terminal-delivery' && parts.at(-1) === 'queue.json');
|
|
94
|
+
|| (parts[0] === 'terminal-delivery' && parts.at(-1) === 'queue.json'));
|
|
95
95
|
queue(() => errors.run(source, mailbox, String(error.message ?? error), critical ? 1 : 0));
|
|
96
96
|
result.skipped++;
|
|
97
97
|
log(`mail import: retained unreadable source ${source}: ${error.message ?? error}`);
|
|
@@ -194,7 +194,7 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
194
194
|
if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
|
|
195
195
|
const path = join(dir, 'claims', entry.name), id = basename(entry.name, '.json');
|
|
196
196
|
const claim = await json(path, address);
|
|
197
|
-
if (claim && !present(id)) { failed(path, 'claim has no imported message', address); continue; }
|
|
197
|
+
if (claim && !present(id)) { failed(path, 'claim has no imported message', address, { diagnostic: true }); continue; }
|
|
198
198
|
if (claim) queue(() => database.statement('INSERT INTO claims(mailbox, message, record) VALUES (?, ?, ?) ON CONFLICT(mailbox, message) DO NOTHING').run(address, id, JSON.stringify(claim)));
|
|
199
199
|
}
|
|
200
200
|
for await (const entry of entries(join(dir, 'subscriptions'))) {
|
|
@@ -384,11 +384,12 @@ export async function importMaildir(database, root, { log = () => {}, legacyRoot
|
|
|
384
384
|
result.records = database.statement('SELECT count(*) AS count FROM mail_records').get().count;
|
|
385
385
|
result.skipped = database.statement('SELECT count(*) AS count FROM mail_import_errors').get().count;
|
|
386
386
|
work.elapsed_ms = Math.round(performance.now() - began);
|
|
387
|
+
database.statement("UPDATE mail_import_errors SET critical = 0 WHERE source LIKE 'sent/%/log.jsonl' OR detail = 'claim has no imported message'").run();
|
|
387
388
|
const blocked = database.statement('SELECT source FROM mail_import_errors WHERE critical = 1 ORDER BY source').all();
|
|
388
|
-
if (blocked.length) {
|
|
389
|
+
if (blocked.length && !deferOperational) {
|
|
389
390
|
throw Object.assign(new Error(`mail migration cannot activate with ${blocked.length} unreadable operational records (${blocked.slice(0, 3).map(({ source }) => source).join(', ')}); original files are retained for repair`), { code: 'mail_import_incomplete' });
|
|
390
391
|
}
|
|
391
|
-
result.complete =
|
|
392
|
+
result.complete = blocked.length === 0;
|
|
392
393
|
database.setMetadata('maildir_import', result);
|
|
393
394
|
log(`mail import complete: ${result.messages} envelopes, ${result.records} operational records, ${result.skipped} retained unreadable sources; ${work.source_reads} source reads, ${work.source_bytes} bytes, ${work.committed_batches} batches in ${work.elapsed_ms} ms`);
|
|
394
395
|
return result;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// When there is no earlier release to serve, retain the source stores and
|
|
2
|
+
// answer reads from a disposable private view. Never activate damaged routing
|
|
3
|
+
// records or run delivery against an incomplete migration.
|
|
4
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { mkdir, lstat, rm } from 'node:fs/promises';
|
|
6
|
+
import { dirname, join, resolve } from 'node:path';
|
|
7
|
+
import { MailDatabase } from './sqlite.mjs';
|
|
8
|
+
import { importMaildir } from './import-maildir.mjs';
|
|
9
|
+
|
|
10
|
+
export async function recoveryMailView(root, { log = () => {} } = {}) {
|
|
11
|
+
const home = dirname(root), directory = join(home, 'teams', 'service', 'mail-recovery');
|
|
12
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
13
|
+
const path = join(directory, 'mail.sqlite');
|
|
14
|
+
// The caller holds this home's mail owner lease. This is its own disposable
|
|
15
|
+
// view, never the source SQLite database or an archived original.
|
|
16
|
+
for (const suffix of ['', '-wal', '-shm']) await rm(path + suffix, { force: true });
|
|
17
|
+
const live = join(home, 'mail.sqlite');
|
|
18
|
+
try {
|
|
19
|
+
await lstat(live);
|
|
20
|
+
const reader = new DatabaseSync(live, { readOnly: true });
|
|
21
|
+
try { reader.prepare('VACUUM INTO ?').run(path); } finally { reader.close(); }
|
|
22
|
+
} catch (error) { if (error.code !== 'ENOENT') throw error; }
|
|
23
|
+
const database = new MailDatabase(path);
|
|
24
|
+
try {
|
|
25
|
+
const retired = database.metadata('maildir_retirement');
|
|
26
|
+
let input = root;
|
|
27
|
+
try { await lstat(input); }
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error.code !== 'ENOENT') throw error;
|
|
30
|
+
if (retired?.archive && dirname(resolve(retired.archive)) === resolve(home, 'mail-retired')) input = retired.archive;
|
|
31
|
+
}
|
|
32
|
+
await importMaildir(database, input, { legacyRoot: root, deferOperational: true, log });
|
|
33
|
+
log('mail migration is deferred; original stores retained, public mail reads use a private recovery view, delivery is unavailable');
|
|
34
|
+
return database;
|
|
35
|
+
} catch (error) { database.close(); throw error; }
|
|
36
|
+
}
|
package/mail/retire-maildir.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// Import and validate while the legacy store stays at its original path. Only
|
|
2
|
+
// a complete, unchanged import can retire it. Keep originals for recovery;
|
|
3
|
+
// retiring mail is never permission to delete another release's readable data.
|
|
4
4
|
import { createReadStream } from 'node:fs';
|
|
5
|
-
import { lstat, mkdir,
|
|
5
|
+
import { lstat, mkdir, rename } from 'node:fs/promises';
|
|
6
6
|
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
-
import { dirname, isAbsolute, join,
|
|
7
|
+
import { dirname, isAbsolute, join, resolve, sep } from 'node:path';
|
|
8
8
|
import { importMaildir } from './import-maildir.mjs';
|
|
9
9
|
import { requireLegacyMaildirReady } from './migration-readiness.mjs';
|
|
10
10
|
|
|
@@ -12,95 +12,84 @@ const stat = async (path) => {
|
|
|
12
12
|
try { return await lstat(path); }
|
|
13
13
|
catch (error) { if (error.code === 'ENOENT') return null; throw error; }
|
|
14
14
|
};
|
|
15
|
+
|
|
16
|
+
export async function verifyImportedMaildir(database, root) {
|
|
17
|
+
for (const { key, value } of database.statement("SELECT key, value FROM mail_metadata WHERE key LIKE 'maildir_directory:%'").all()) {
|
|
18
|
+
const saved = JSON.parse(value), path = resolve(root, key.slice('maildir_directory:'.length));
|
|
19
|
+
if (path !== root && !path.startsWith(root + sep)) throw new Error('mail import directory receipt is outside its source');
|
|
20
|
+
const current = await stat(path);
|
|
21
|
+
const stamp = current && { device: current.dev, inode: current.ino, mtime: current.mtimeMs, ctime: current.ctimeMs };
|
|
22
|
+
if (JSON.stringify(saved.stamp) !== JSON.stringify(stamp)) throw new Error(`mail source directory changed before retirement: ${path}`);
|
|
23
|
+
}
|
|
24
|
+
const sources = database.statement('SELECT source, sha256 FROM mail_import_sources WHERE source > ? ORDER BY source LIMIT 256');
|
|
25
|
+
let cursor = '';
|
|
26
|
+
for (;;) {
|
|
27
|
+
const batch = sources.all(cursor);
|
|
28
|
+
if (!batch.length) return;
|
|
29
|
+
for (const row of batch) {
|
|
30
|
+
const path = resolve(root, row.source);
|
|
31
|
+
if (isAbsolute(row.source) || !path.startsWith(root + sep)) throw new Error('an imported source is outside its Maildir');
|
|
32
|
+
let parent = dirname(path);
|
|
33
|
+
while (parent !== root) {
|
|
34
|
+
if (!(await stat(parent))?.isDirectory()) throw new Error(`mail source directory changed before retirement: ${row.source}`);
|
|
35
|
+
parent = dirname(parent);
|
|
36
|
+
}
|
|
37
|
+
if (!(await stat(path))?.isFile()) throw new Error(`mail source changed before retirement: ${row.source}`);
|
|
38
|
+
const hash = createHash('sha256');
|
|
39
|
+
for await (const bytes of createReadStream(path)) hash.update(bytes);
|
|
40
|
+
if (hash.digest('hex') !== row.sha256) throw new Error(`mail source bytes changed before retirement: ${row.source}`);
|
|
41
|
+
}
|
|
42
|
+
cursor = batch.at(-1).source;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
15
46
|
/** Called only while this daemon holds the mail owner lease. */
|
|
16
|
-
export async function migrateMaildir(database, root, { log = () => {} } = {}) {
|
|
47
|
+
export async function migrateMaildir(database, root, { log = () => {}, committed = false } = {}) {
|
|
17
48
|
root = resolve(root);
|
|
18
49
|
let retirement = database.metadata('maildir_retirement');
|
|
19
50
|
let source = await stat(root);
|
|
20
51
|
if (!retirement) {
|
|
21
52
|
if (source && !source.isDirectory()) throw new Error(`mail migration requires a regular directory at ${root}`);
|
|
22
53
|
if (source) await requireLegacyMaildirReady(root);
|
|
23
|
-
retirement = { source: root, archive: source ? join(dirname(root), 'mail-retired', randomUUID()) : null, started_at_ms: Date.now(), complete: false };
|
|
54
|
+
retirement = { version: 2, source: root, archive: source ? join(dirname(root), 'mail-retired', randomUUID()) : null, started_at_ms: Date.now(), complete: false };
|
|
24
55
|
database.setMetadata('maildir_retirement', retirement);
|
|
25
56
|
}
|
|
26
57
|
if (retirement.source !== root) throw new Error('the mail migration source differs from its stored receipt');
|
|
27
58
|
if (retirement.complete) {
|
|
28
|
-
if (source) throw new Error(`the retired Maildir was recreated at ${root}; it is retained, and this daemon will not choose between two mail stores`);
|
|
59
|
+
if (source && !committed) throw new Error(`the retired Maildir was recreated at ${root}; it is retained, and this daemon will not choose between two mail stores`);
|
|
60
|
+
if (source) log(`committed SQLite mailbox remains serving; recreated legacy records at ${root} are retained for explicit recovery`);
|
|
29
61
|
return database.metadata('maildir_import');
|
|
30
62
|
}
|
|
31
63
|
if (!retirement.archive) {
|
|
32
64
|
if (source) throw new Error(`a legacy Maildir appeared after this home's SQLite owner started: ${root}; it is retained, and migration is refused`);
|
|
33
65
|
const imported = await importMaildir(database, root, { log });
|
|
34
|
-
database.setMetadata('maildir_retirement', { ...retirement, complete: true, completed_at_ms: Date.now(),
|
|
66
|
+
database.setMetadata('maildir_retirement', { ...retirement, complete: true, completed_at_ms: Date.now(), originals_retained: true });
|
|
35
67
|
return imported;
|
|
36
68
|
}
|
|
37
69
|
const archiveRoot = resolve(dirname(root), 'mail-retired');
|
|
38
70
|
const archive = resolve(retirement.archive);
|
|
39
71
|
if (dirname(archive) !== archiveRoot) throw new Error('the stored migration snapshot is outside this home');
|
|
40
72
|
let snapshot = await stat(archive);
|
|
41
|
-
if (source) {
|
|
42
|
-
|
|
43
|
-
|
|
73
|
+
if (source && snapshot) throw new Error(`the retired Maildir was recreated at ${root}; both stores are retained for explicit recovery`);
|
|
74
|
+
if (!source && !snapshot) throw new Error('the incomplete mail migration source is missing; originals cannot be reconstructed');
|
|
75
|
+
const input = snapshot ? archive : root;
|
|
76
|
+
if (!(snapshot ?? source).isDirectory()) throw new Error('the migration source is not a regular directory');
|
|
77
|
+
if (snapshot && retirement.inode != null && (retirement.inode !== snapshot.ino || retirement.device !== snapshot.dev)) throw new Error('the migration snapshot differs from its ownership receipt');
|
|
78
|
+
// Version-one receipts may already have moved the source before validation.
|
|
79
|
+
// Resume that snapshot, retaining it even when an operational input refuses.
|
|
80
|
+
const imported = await importMaildir(database, input, { log, legacyRoot: root });
|
|
81
|
+
if (snapshot && await stat(root)) throw new Error(`a legacy writer recreated ${root} during migration; both stores are retained`);
|
|
82
|
+
if (!snapshot) {
|
|
44
83
|
await requireLegacyMaildirReady(root);
|
|
84
|
+
await verifyImportedMaildir(database, root);
|
|
85
|
+
const current = await stat(root);
|
|
86
|
+
if (current?.ino !== source.ino || current?.dev !== source.dev) throw new Error('mail root changed before retirement');
|
|
45
87
|
await mkdir(archiveRoot, { recursive: true, mode: 0o700 });
|
|
46
88
|
await rename(root, archive);
|
|
47
89
|
snapshot = await stat(archive);
|
|
48
90
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
}
|
|
53
|
-
if (!snapshot.isDirectory()) throw new Error('the migration snapshot is not a regular directory');
|
|
54
|
-
if (retirement.inode != null && (retirement.inode !== snapshot.ino || retirement.device !== snapshot.dev)) throw new Error('the migration snapshot differs from its ownership receipt');
|
|
55
|
-
retirement = { ...retirement, inode: snapshot.ino, device: snapshot.dev };
|
|
56
|
-
database.setMetadata('maildir_retirement', retirement);
|
|
57
|
-
const imported = await importMaildir(database, archive, { log, legacyRoot: root });
|
|
58
|
-
// No ordinary operation becomes ready if an old entrypoint created a second
|
|
59
|
-
// store during import. It is never silently reimported over the first one.
|
|
60
|
-
if (await stat(root)) throw new Error(`a legacy writer recreated ${root} during migration; originals are retained and mail is not activated`);
|
|
61
|
-
const sources = database.statement(`SELECT source, sha256 FROM mail_import_sources s WHERE purged = 0 AND source > ?
|
|
62
|
-
AND NOT EXISTS (SELECT 1 FROM mail_import_errors e WHERE e.source = s.source) ORDER BY source LIMIT 256`);
|
|
63
|
-
const purged = database.statement('UPDATE mail_import_sources SET purged = 1 WHERE source = ?');
|
|
64
|
-
let removed = 0;
|
|
65
|
-
let cursor = '';
|
|
66
|
-
for (;;) {
|
|
67
|
-
const batch = sources.all(cursor);
|
|
68
|
-
if (!batch.length) break;
|
|
69
|
-
const done = [];
|
|
70
|
-
for (const row of batch) {
|
|
71
|
-
const path = resolve(archive, row.source);
|
|
72
|
-
if (isAbsolute(row.source) || !path.startsWith(archive + sep)) throw new Error('an imported source is outside its migration snapshot');
|
|
73
|
-
let parent = dirname(path), regular = true;
|
|
74
|
-
while (parent !== archive) {
|
|
75
|
-
if (!(await stat(parent))?.isDirectory()) { regular = false; break; }
|
|
76
|
-
parent = dirname(parent);
|
|
77
|
-
}
|
|
78
|
-
if (!regular) { log(`mail migration retained linked source ${row.source}`); continue; }
|
|
79
|
-
const file = await stat(path);
|
|
80
|
-
if (!file) { done.push(row.source); continue; }
|
|
81
|
-
if (!file.isFile()) { log(`mail migration retained changed source ${row.source}`); continue; }
|
|
82
|
-
const hash = createHash('sha256');
|
|
83
|
-
for await (const bytes of createReadStream(path)) hash.update(bytes);
|
|
84
|
-
if (hash.digest('hex') !== row.sha256) { log(`mail migration retained changed bytes ${row.source}`); continue; }
|
|
85
|
-
await unlink(path);
|
|
86
|
-
done.push(row.source); removed++;
|
|
87
|
-
}
|
|
88
|
-
// An interruption between unlink and this commit is safe: the next start
|
|
89
|
-
// observes the absent original and records its retirement again.
|
|
90
|
-
database.transaction(() => { for (const source of done) purged.run(source); });
|
|
91
|
-
cursor = batch.at(-1).source;
|
|
92
|
-
}
|
|
93
|
-
let retained = 0;
|
|
94
|
-
async function tidy(dir) {
|
|
95
|
-
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
96
|
-
const path = join(dir, entry.name);
|
|
97
|
-
if (entry.isDirectory()) await tidy(path);
|
|
98
|
-
else { retained++; log(`mail migration retained diagnostic source ${relative(archive, path)}`); }
|
|
99
|
-
}
|
|
100
|
-
try { await rmdir(dir); } catch (error) { if (!['ENOTEMPTY', 'EEXIST'].includes(error.code)) throw error; }
|
|
101
|
-
}
|
|
102
|
-
await tidy(archive);
|
|
103
|
-
database.setMetadata('maildir_retirement', { ...retirement, complete: true, completed_at_ms: Date.now(), retained_sources: retained });
|
|
104
|
-
log(`mail retired: ${removed} imported source files removed; ${retained} diagnostic sources retained at ${archive}`);
|
|
91
|
+
database.setMetadata('maildir_retirement', { ...retirement, inode: snapshot.ino, device: snapshot.dev,
|
|
92
|
+
complete: true, completed_at_ms: Date.now(), originals_retained: true });
|
|
93
|
+
log(`mail retired after complete import; original files retained at ${archive}`);
|
|
105
94
|
return imported;
|
|
106
95
|
}
|
package/mail/sqlite-store.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { dirname, join, resolve } from 'node:path';
|
|
|
6
6
|
import { blake3Hex } from './envelope.mjs';
|
|
7
7
|
import { MailDatabase, MailOwner, RETAINED_BODY } from './sqlite.mjs';
|
|
8
8
|
import { migrateMaildir } from './retire-maildir.mjs';
|
|
9
|
+
import { recoveryMailView } from './recovery-view.mjs';
|
|
9
10
|
|
|
10
11
|
export const MIN_PREFIX_DIGITS = 4;
|
|
11
12
|
export { RETAINED_BODY };
|
|
@@ -30,13 +31,15 @@ const validId = (id) => typeof id === 'string' && /^[A-Za-z0-9_-]+$/.test(id);
|
|
|
30
31
|
const OFFLINE_CEILING_MS = 30 * 24 * 60 * 60_000;
|
|
31
32
|
|
|
32
33
|
export class SqliteMailStore extends EventEmitter {
|
|
33
|
-
constructor({ mailRoot, machine = null, log = () => {}, database = null, retention } = {}) {
|
|
34
|
+
constructor({ mailRoot, machine = null, log = () => {}, database = null, retention, committed = false, startupRecovery = false } = {}) {
|
|
34
35
|
super();
|
|
35
36
|
this.root = resolve(mailRoot);
|
|
36
37
|
this.machine = machine;
|
|
37
38
|
this.log = log;
|
|
38
39
|
this.database = database;
|
|
39
40
|
this.retention = retention;
|
|
41
|
+
this.committed = committed;
|
|
42
|
+
this.startupRecovery = startupRecovery;
|
|
40
43
|
this.unread = null;
|
|
41
44
|
}
|
|
42
45
|
|
|
@@ -50,14 +53,21 @@ export class SqliteMailStore extends EventEmitter {
|
|
|
50
53
|
async start() {
|
|
51
54
|
if (!this.owner) throw new Error('mail cannot open without this daemon\'s writer lease');
|
|
52
55
|
this.database ??= new MailDatabase(join(dirname(this.root), 'mail.sqlite'), { retention: this.retention });
|
|
53
|
-
this.imported = await migrateMaildir(this.database, this.root, { log: this.log });
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
try { this.imported = await migrateMaildir(this.database, this.root, { log: this.log, committed: this.committed || (this.startupRecovery && this.database.metadata('maildir_retirement')?.complete === true) }); }
|
|
57
|
+
catch (error) {
|
|
58
|
+
if (!this.startupRecovery) throw error;
|
|
59
|
+
this.log(`startup migration refused: ${error.message}`);
|
|
60
|
+
this.database.close();
|
|
61
|
+
this.database = await recoveryMailView(this.root, { log: this.log });
|
|
62
|
+
this.deferred = true;
|
|
63
|
+
}
|
|
64
|
+
if (!this.deferred) this.database.retain();
|
|
65
|
+
if (!this.deferred) this.reconcileDeliverySchedule();
|
|
56
66
|
this.database.on('message', (address) => this.emit('changed', address));
|
|
57
67
|
this.database.on('wake', ({ address, id }) => this.emit('wake', { address, id }));
|
|
58
68
|
this.database.on('subscriptions', (address) => this.emit('subscriptions', address));
|
|
59
69
|
this.retentionTimer = setInterval(() => {
|
|
60
|
-
try { this.database.retain(); }
|
|
70
|
+
try { if (!this.deferred) this.database.retain(); }
|
|
61
71
|
catch (error) { this.log(`mail retention failed: ${error.message}`); }
|
|
62
72
|
}, 3_600_000);
|
|
63
73
|
this.retentionTimer.unref?.();
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// All upgrade entrypoints ask the candidate release, never the old daemon's
|
|
2
|
+
// cached modules. The candidate cannot modify the serving home's state.
|
|
3
|
+
import { spawn } from 'node:child_process';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { resolve, join } from 'node:path';
|
|
6
|
+
import { mkdtemp, rm } from 'node:fs/promises';
|
|
7
|
+
import { tmpdir } from 'node:os';
|
|
8
|
+
|
|
9
|
+
export async function requireUpgradeAdmission({ home, entry = new URL('../bin/teams.mjs', import.meta.url), env = process.env } = {}) {
|
|
10
|
+
const path = entry instanceof URL ? fileURLToPath(entry) : resolve(entry);
|
|
11
|
+
const scratch = await mkdtemp(join(tmpdir(), 'supercode-upgrade-'));
|
|
12
|
+
try { return await new Promise((done, reject) => {
|
|
13
|
+
const child = spawn(process.execPath, [path, 'mail', 'upgrade-inspect', '--home', resolve(home), '--json'],
|
|
14
|
+
{ env: { ...env, SUPERCODE_UPGRADE_SCRATCH: scratch }, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true });
|
|
15
|
+
let stdout = '', stderr = '', refused = false;
|
|
16
|
+
const refuse = (reason) => {
|
|
17
|
+
if (refused) return;
|
|
18
|
+
refused = true;
|
|
19
|
+
// Only the admission child we just started is ended, by its handle.
|
|
20
|
+
child.kill('SIGKILL');
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
const error = Object.assign(new Error(`upgrade admission refused: ${reason}; the serving daemon must remain running`), { code: 'upgrade_not_ready' });
|
|
23
|
+
if (child.exitCode !== null || child.signalCode !== null || !child.pid) reject(error);
|
|
24
|
+
else child.once('close', () => reject(error));
|
|
25
|
+
};
|
|
26
|
+
const timer = setTimeout(() => refuse('candidate did not complete the existing-state check within 8 seconds'), 8000);
|
|
27
|
+
const collect = (which, bytes) => {
|
|
28
|
+
if (which === 'out') stdout += bytes; else stderr += bytes;
|
|
29
|
+
if (stdout.length + stderr.length > 64 * 1024) refuse('candidate returned an oversized admission response');
|
|
30
|
+
};
|
|
31
|
+
child.stdout.on('data', bytes => collect('out', bytes));
|
|
32
|
+
child.stderr.on('data', bytes => collect('err', bytes));
|
|
33
|
+
child.once('error', error => refuse(error.message));
|
|
34
|
+
child.once('close', (code, signal) => {
|
|
35
|
+
clearTimeout(timer);
|
|
36
|
+
if (refused) return;
|
|
37
|
+
if (code !== 0 || signal) return refuse(stderr.trim() || `candidate exited ${code ?? signal}`);
|
|
38
|
+
let receipt;
|
|
39
|
+
try { receipt = JSON.parse(stdout); } catch { return refuse('candidate returned no admission receipt'); }
|
|
40
|
+
if (receipt.stage !== 'upgrade_admission' || receipt.admitted !== true || receipt.home !== resolve(home, '..')) return refuse('candidate returned an invalid admission receipt');
|
|
41
|
+
done(receipt);
|
|
42
|
+
});
|
|
43
|
+
}); } finally { await rm(scratch, { recursive: true, force: true }); }
|
|
44
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Upgrade admission is a candidate command, before the serving daemon exits.
|
|
2
|
+
// It opens live state read-only and applies this release's import/schema work
|
|
3
|
+
// only to a private database. A refusal leaves the current service serving.
|
|
4
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { lstat } from 'node:fs/promises';
|
|
6
|
+
import { join, dirname, resolve } from 'node:path';
|
|
7
|
+
import { MailDatabase } from './sqlite.mjs';
|
|
8
|
+
import { importMaildir } from './import-maildir.mjs';
|
|
9
|
+
import { requireLegacyMaildirReady } from './migration-readiness.mjs';
|
|
10
|
+
import { verifyImportedMaildir } from './retire-maildir.mjs';
|
|
11
|
+
import { requireSessionIndexRuntime } from '../session-index-owner.mjs';
|
|
12
|
+
|
|
13
|
+
const exists = async (path) => {
|
|
14
|
+
try { return await lstat(path); }
|
|
15
|
+
catch (error) { if (error.code === 'ENOENT') return null; throw error; }
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export async function inspectUpgradeReadiness(teamsHome, directory) {
|
|
19
|
+
requireSessionIndexRuntime();
|
|
20
|
+
// Resolve all daemon modules before admission, without starting its carrier,
|
|
21
|
+
// binding a socket or executing any retained delivery against a real session.
|
|
22
|
+
await import('../machine.mjs');
|
|
23
|
+
const home = dirname(teamsHome), root = join(home, 'mail');
|
|
24
|
+
const guard = await requireLegacyMaildirReady(root);
|
|
25
|
+
if (!directory) throw new Error('upgrade inspection requires its private admission directory');
|
|
26
|
+
let database;
|
|
27
|
+
try {
|
|
28
|
+
const live = join(home, 'mail.sqlite'), candidate = join(directory, 'mail.sqlite');
|
|
29
|
+
if (await exists(live)) {
|
|
30
|
+
const reader = new DatabaseSync(live, { readOnly: true });
|
|
31
|
+
try {
|
|
32
|
+
// SQLite creates a coherent snapshot, including committed WAL pages;
|
|
33
|
+
// copying the db/WAL files independently cannot establish that.
|
|
34
|
+
reader.prepare('VACUUM INTO ?').run(candidate);
|
|
35
|
+
} finally { reader.close(); }
|
|
36
|
+
}
|
|
37
|
+
database = new MailDatabase(candidate);
|
|
38
|
+
const retirement = database.metadata('maildir_retirement');
|
|
39
|
+
const source = await exists(root);
|
|
40
|
+
let input = root;
|
|
41
|
+
if (retirement?.source && retirement.source !== root) throw new Error('mail migration source differs from its stored receipt');
|
|
42
|
+
if (retirement?.archive && dirname(resolve(retirement.archive)) !== resolve(home, 'mail-retired')) throw new Error('migration snapshot is outside this home');
|
|
43
|
+
if (retirement?.archive && await exists(retirement.archive)) {
|
|
44
|
+
if (source) throw new Error('both the original migration snapshot and a recreated Maildir exist; explicit recovery is required before activation');
|
|
45
|
+
input = retirement.archive;
|
|
46
|
+
} else if (retirement?.complete && source) {
|
|
47
|
+
throw new Error('a retired Maildir was recreated; explicit recovery is required before activation');
|
|
48
|
+
} else if (retirement?.archive && !source && !retirement.complete) {
|
|
49
|
+
throw new Error('the incomplete mail migration source is missing');
|
|
50
|
+
}
|
|
51
|
+
const imported = await importMaildir(database, input, { legacyRoot: root });
|
|
52
|
+
if (!retirement?.complete && await exists(input)) await verifyImportedMaildir(database, input);
|
|
53
|
+
return { version: 1, stage: 'upgrade_admission', admitted: true, home,
|
|
54
|
+
messages: imported.messages, records: imported.records, retained_diagnostics: imported.skipped,
|
|
55
|
+
legacy_wrappers: guard.live.length, schema: database.metadata('schema_version') };
|
|
56
|
+
} finally {
|
|
57
|
+
database?.close();
|
|
58
|
+
}
|
|
59
|
+
}
|