meta-sovereign 0.49.6 → 0.49.7

package/js/src/cli/index.js

@@ -43,6 +43,7 @@ import {
   createBackup,
   pruneBackups,
   restoreBackup,
+  restoreBackupToDir,
 } from '../storage/backup.js';
 import { writeEncryptedExport } from '../storage/export-encrypted.js';
 import { CURRENT_SCHEMA } from '../storage/migrations/index.js';
@@ -80,7 +81,7 @@ Commands:
   import        --source=<name> --file=<path> --store=<dir>
   export        --file=<path> --store=<dir>
   backup        --store=<dir> --archive=<dir> [--keep=<n>]
-  restore       --file=<path> --store=<dir>
+  restore       --file=<path> --store=<dir> [--replace]
   passcode      --clear [--store=<dir>]   (remove a forgotten local passcode)
   serve         [--port=<n>] [--store=<dir>] [--allow-origin=<origins|none>] [--ui-url=<url>] [--local-ui=false] [--tls=false] [--quiet-logs]
                 verbose logs are written to <store>/logs/meta-sovereign.log by default
@@ -171,6 +172,18 @@ const passcodeCmd = async (args, log) => {
 };
 
 const restoreCmd = async (args, log) => {
+  if (args.replace) {
+    // Swap the whole log in one step; the store must not be in use (R-Z16).
+    const n = await restoreBackupToDir(
+      args.file,
+      args.store ?? '.meta-sovereign',
+      {
+        replace: true,
+      }
+    );
+    log(`replaced the store with ${n} records from ${args.file}`);
+    return 0;
+  }
   const store = await openStore(args.store ?? '.meta-sovereign');
   const n = await restoreBackup(store, args.file);
   log(`restored ${n} links from ${args.file}`);

package/js/src/server/backup-job.js

@@ -1,10 +1,10 @@
 /**
  * The automatic backup, started once the port is ours (R-A4, R-Z7).
  *
- * OFF by default (the streaming backups of R-Z16 switch it on).
- * `MS_BACKUP_INTERVAL_HOURS` (or the `backups.intervalHours` option) sets
- * the interval in hours, `0` or unset means off; `MS_BACKUP_KEEP` (or
- * `backups.keep`) sets how many archives are kept (default 7, `0` keeps
+ * ON by default (R-Z16): a streaming backup every 24 hours, the newest 3
+ * kept. `MS_BACKUP_INTERVAL_HOURS` (or the `backups.intervalHours` option)
+ * sets the interval in hours, `0` turns it off; `MS_BACKUP_KEEP` (or
+ * `backups.keep`) sets how many archives are kept (default 3, `0` keeps
  * all). A value that is not a number >= 0 is ignored for the default and
  * logged once.
  * The archive is encrypted, and holds the secrets, only when the server has
@@ -15,8 +15,8 @@
 import { createBackupScheduler } from '../storage/backup.js';
 
 const HOUR_MS = 3_600_000;
-export const DEFAULT_INTERVAL_HOURS = 0;
-export const DEFAULT_KEEP = 7;
+export const DEFAULT_INTERVAL_HOURS = 24;
+export const DEFAULT_KEEP = 3;
 
 // A number >= 0, or the fallback; `bad` collects the names that were wrong.
 const number = (value, fallback, name, bad) => {

package/js/src/server/routes-backup.js

@@ -18,17 +18,16 @@ import { promises as fs } from 'node:fs';
 import path from 'node:path';
 import { json, readBody } from './util.js';
 import { safeMessage } from './redact.js';
-import { makeBackup, restoreBackup, pruneBackups } from '../storage/backup.js';
+import {
+  isBackupFile,
+  makeBackup,
+  restoreBackup,
+  pruneBackups,
+} from '../storage/backup.js';
 
 const listFiles = async (archiveDir) => {
   const entries = await fs.readdir(archiveDir).catch(() => []);
-  const ours = entries
-    .filter(
-      (f) =>
-        f.startsWith('meta-sovereign-') &&
-        (f.endsWith('.json') || f.endsWith('.json.enc'))
-    )
-    .sort();
+  const ours = entries.filter(isBackupFile).sort();
   const out = [];
   for (const f of ours) {
     const stat = await fs.stat(path.join(archiveDir, f)).catch(() => null);

package/js/src/storage/backup-archive.js

@@ -0,0 +1,571 @@
+/**
+ * The streaming backup archive, format 2 (R-Z16).
+ *
+ * A real store is gigabytes: the old archive (one JSON string holding every
+ * link) cannot even be built past V8's string limit. Format 2 is read and
+ * written a piece at a time, so memory stays flat whatever the store size.
+ *
+ * Plain content (what is gzipped), line by line:
+ *
+ *   {"format":"meta-sovereign-backup","version":2,"createdAt":...,"secrets":...}
+ *   <one Links Notation record per line, the store's log as it was>
+ *   #end {"lines":N,"secretsExcluded":K,"bytes":B,"sha256":"<of every line above>"}
+ *
+ * File on disk: a 28-byte header, then frames of the gzip stream:
+ *
+ *   header = "MSBK2\n" | mode (0 plain, 1 AES-256-GCM) | 0 | salt(16) | nonce prefix(4)
+ *   frame  = final flag (1) | length (4, BE) | data (length) | tag (16)
+ *
+ * Every frame (up to 1 MB) is sealed on its own: AES-256-GCM with the
+ * nonce `prefix | frame index`, or (no passphrase) a SHA-256 check tag. The
+ * additional data is `header | index | final flag`, so a flipped byte, a
+ * reordered or dropped frame, and a file cut short are all detected, and
+ * a frame is checked before any of it is used. The key is scrypt of the
+ * passphrase and the salt, as before.
+ */
+
+import { createReadStream, promises as fs } from 'node:fs';
+import { createGunzip, createGzip } from 'node:zlib';
+import { Readable } from 'node:stream';
+import { StringDecoder } from 'node:string_decoder';
+import {
+  createCipheriv,
+  createDecipheriv,
+  createHash,
+  randomBytes,
+  scrypt,
+} from 'node:crypto';
+import { decodeLine } from 'lino-objects-codec';
+
+export const FRAME_BYTES = 1024 * 1024;
+const MAGIC = Buffer.from('MSBK2\n');
+const HEADER_BYTES = 28;
+const TAG_BYTES = 16;
+const KDF_N = 1 << 14;
+export const FORMAT = 'meta-sovereign-backup';
+const TRAILER = '#end ';
+
+/** An error a person can act on, with a stable `code`. */
+export const archiveError = (code, message) =>
+  Object.assign(new Error(message), { code });
+
+const deriveKey = (passphrase, salt) =>
+  new Promise((resolve, reject) =>
+    scrypt(passphrase, salt, 32, { N: KDF_N }, (error, key) =>
+      error ? reject(error) : resolve(key)
+    )
+  );
+
+const aadOf = (header, index, final) => {
+  const tail = Buffer.alloc(9);
+  tail.writeBigUInt64BE(BigInt(index), 0);
+  tail[8] = final ? 1 : 0;
+  return Buffer.concat([header, tail]);
+};
+
+const nonceOf = (header, index) => {
+  const nonce = Buffer.alloc(12);
+  header.copy(nonce, 0, 24, 28);
+  nonce.writeBigUInt64BE(BigInt(index), 4);
+  return nonce;
+};
+
+// The check tag of a plain frame: it catches damage, not a forger (nobody
+// can promise that without a key).
+const plainTag = (aad, data) =>
+  createHash('sha256').update(aad).update(data).digest().subarray(0, TAG_BYTES);
+
+const sealFrame = ({ header, key }, index, final, data) => {
+  const aad = aadOf(header, index, final);
+  let body = data;
+  let tag;
+  if (key) {
+    const cipher = createCipheriv('aes-256-gcm', key, nonceOf(header, index));
+    cipher.setAAD(aad);
+    body = Buffer.concat([cipher.update(data), cipher.final()]);
+    tag = cipher.getAuthTag();
+  } else {
+    tag = plainTag(aad, data);
+  }
+  const head = Buffer.alloc(5);
+  head[0] = final ? 1 : 0;
+  head.writeUInt32BE(body.length, 1);
+  return Buffer.concat([head, body, tag]);
+};
+
+/** A pipeline stage: gzip chunks in, sealed frames (header first) out. */
+export const sealStage = (passphrase) =>
+  async function* seal(source) {
+    const salt = passphrase ? randomBytes(16) : Buffer.alloc(16);
+    const header = Buffer.concat([
+      MAGIC,
+      Buffer.from([passphrase ? 1 : 0, 0]),
+      salt,
+      randomBytes(4),
+    ]);
+    const key = passphrase ? await deriveKey(passphrase, salt) : null;
+    const ctx = { header, key };
+    yield header;
+    let index = 0;
+    let pending = [];
+    let size = 0;
+    for await (const chunk of source) {
+      pending.push(chunk);
+      size += chunk.length;
+      if (size > FRAME_BYTES) {
+        const all = Buffer.concat(pending);
+        let at = 0;
+        while (all.length - at > FRAME_BYTES) {
+          yield sealFrame(
+            ctx,
+            index,
+            false,
+            all.subarray(at, at + FRAME_BYTES)
+          );
+          index += 1;
+          at += FRAME_BYTES;
+        }
+        pending = [all.subarray(at)];
+        size = all.length - at;
+      }
+    }
+    // The last frame is always sealed as such, even when empty.
+    yield sealFrame(ctx, index, true, Buffer.concat(pending));
+  };
+
+// A reader that hands out exact byte counts from a stream of chunks.
+const createTaker = (source) => {
+  const iterator = source[Symbol.asyncIterator]();
+  let list = [];
+  let have = 0;
+  let done = false;
+  return async (count) => {
+    while (have < count && !done) {
+      const next = await iterator.next();
+      if (next.done) {
+        done = true;
+      } else {
+        list.push(next.value);
+        have += next.value.length;
+      }
+    }
+    if (have < count) {
+      return null;
+    }
+    const all = Buffer.concat(list);
+    list = all.length > count ? [all.subarray(count)] : [];
+    have = all.length - count;
+    return all.subarray(0, count);
+  };
+};
+
+const truncated = () =>
+  archiveError(
+    'BACKUP_TRUNCATED',
+    'the backup file is cut short (incomplete write or copy)'
+  );
+
+const openFrame = ({ header, key }, index, frame, tag) => {
+  const final = frame.final;
+  const aad = aadOf(header, index, final);
+  if (key) {
+    const decipher = createDecipheriv(
+      'aes-256-gcm',
+      key,
+      nonceOf(header, index)
+    );
+    decipher.setAAD(aad);
+    decipher.setAuthTag(tag);
+    return Buffer.concat([decipher.update(frame.data), decipher.final()]);
+  }
+  if (!plainTag(aad, frame.data).equals(tag)) {
+    throw new Error('frame check failed');
+  }
+  return frame.data;
+};
+
+const readHeader = async (take, passphrase) => {
+  const header = await take(HEADER_BYTES);
+  if (!header || !header.subarray(0, MAGIC.length).equals(MAGIC)) {
+    throw archiveError('BACKUP_INVALID', 'not a meta-sovereign backup file');
+  }
+  const encrypted = header[6] === 1;
+  if (encrypted && !passphrase) {
+    throw archiveError(
+      'BACKUP_PASSPHRASE_REQUIRED',
+      'encrypted backup requires a passphrase'
+    );
+  }
+  const key = encrypted
+    ? await deriveKey(passphrase, header.subarray(8, 24))
+    : null;
+  return { header, key, encrypted };
+};
+
+// A frame that does not open: the first of an encrypted file is most
+// likely a wrong passphrase, any other a changed or reordered file.
+const failedFrame = (index, encrypted) =>
+  index === 0 && encrypted
+    ? archiveError(
+        'BACKUP_BAD_PASSPHRASE',
+        'wrong passphrase, or the backup file is damaged'
+      )
+    : archiveError(
+        'BACKUP_TAMPERED',
+        `the backup file was changed or reordered (frame ${index})`
+      );
+
+/**
+ * The gzip bytes of an archive file, frame by frame, each checked before
+ * it is handed on. Throws `BACKUP_*` errors: PASSPHRASE_REQUIRED, INVALID
+ * (not an archive), TRUNCATED, BAD_PASSPHRASE (the first frame does not
+ * open), TAMPERED (a later frame does not).
+ */
+export async function* unframe(source, { passphrase = null } = {}) {
+  const take = createTaker(source);
+  const { header, key, encrypted } = await readHeader(take, passphrase);
+  for (let index = 0; ; index += 1) {
+    const head = await take(5);
+    const length = head?.readUInt32BE(1);
+    if (length > FRAME_BYTES) {
+      throw archiveError('BACKUP_INVALID', 'the backup file is damaged');
+    }
+    const body = head && (await take(length + TAG_BYTES));
+    if (!body) {
+      throw truncated();
+    }
+    const frame = { final: head[0] === 1, data: body.subarray(0, length) };
+    let data;
+    try {
+      data = openFrame({ header, key }, index, frame, body.subarray(length));
+    } catch {
+      throw failedFrame(index, encrypted);
+    }
+    yield data;
+    if (frame.final) {
+      if (await take(1)) {
+        throw archiveError(
+          'BACKUP_INVALID',
+          'data after the end of the backup'
+        );
+      }
+      return;
+    }
+  }
+}
+
+// '\n'-separated lines of a byte stream (a line is never split by a chunk).
+async function* linesOf(source) {
+  const decoder = new StringDecoder('utf8');
+  let rest = '';
+  for await (const chunk of source) {
+    rest += typeof chunk === 'string' ? chunk : decoder.write(chunk);
+    let at = rest.indexOf('\n');
+    while (at !== -1) {
+      yield rest.slice(0, at);
+      rest = rest.slice(at + 1);
+      at = rest.indexOf('\n');
+    }
+  }
+  rest += decoder.end();
+  if (rest.length > 0) {
+    yield rest;
+  }
+}
+
+export const isRecordLine = (line) => line.startsWith('(');
+
+const parseManifest = (line) => {
+  let manifest;
+  try {
+    manifest = JSON.parse(line);
+  } catch {
+    manifest = null;
+  }
+  if (manifest?.format !== FORMAT || manifest.version !== 2) {
+    throw archiveError('BACKUP_INVALID', 'not a meta-sovereign backup file');
+  }
+  return manifest;
+};
+
+const parseTrailer = (line) => {
+  try {
+    return JSON.parse(line.slice(TRAILER.length));
+  } catch {
+    throw archiveError('BACKUP_INVALID', 'the backup trailer is unreadable');
+  }
+};
+
+const checkTrailer = ({ manifest, trailer, lines }, sha256) => {
+  if (!manifest || !trailer) {
+    throw truncated();
+  }
+  if (trailer.sha256 !== sha256 || trailer.lines !== lines) {
+    throw archiveError(
+      'BACKUP_INVALID',
+      'the backup content does not match its trailer'
+    );
+  }
+};
+
+// Sort one plain line into the manifest, the trailer or a record; true for
+// a record (which the caller hears).
+const takeLine = (seen, line) => {
+  if (seen.trailer) {
+    throw archiveError('BACKUP_INVALID', 'data after the backup trailer');
+  }
+  if (!seen.manifest) {
+    seen.manifest = parseManifest(line);
+    return false;
+  }
+  if (line.startsWith(TRAILER)) {
+    seen.trailer = parseTrailer(line);
+    return false;
+  }
+  if (line.length === 0) {
+    return false;
+  }
+  seen.lines += 1;
+  return true;
+};
+
+/**
+ * Read a whole archive, checking everything: every frame, the manifest,
+ * the trailer's line count and SHA-256. `onLine(line)` hears each record
+ * line (awaited, so it can write them somewhere). Resolves to
+ * `{ manifest, trailer, lines, sha256 }`; nothing is trusted before it
+ * does, so a caller that applies lines as they come should scan once first.
+ */
+export const scanArchive = async (file, { passphrase, onLine } = {}) => {
+  const hash = createHash('sha256');
+  const seen = { manifest: null, trailer: null, lines: 0 };
+  let fromCaller = null; // an error of `onLine` is the caller's, not damage
+  const hear = async (line) => {
+    try {
+      await onLine?.(line);
+    } catch (error) {
+      fromCaller = error;
+      throw error;
+    }
+  };
+  const gunzip = createGunzip();
+  const text = Readable.from(unframe(createReadStream(file), { passphrase }));
+  const plain = text.pipe(gunzip);
+  text.on('error', (error) => gunzip.destroy(error));
+  try {
+    for await (const line of linesOf(plain)) {
+      if (takeLine(seen, line)) {
+        await hear(line);
+      }
+      if (!seen.trailer) {
+        hash.update(`${line}\n`);
+      }
+    }
+  } catch (error) {
+    text.destroy();
+    if (error === fromCaller || error.code?.startsWith?.('BACKUP_')) {
+      throw error;
+    }
+    throw archiveError('BACKUP_TRUNCATED', 'the backup file is damaged');
+  }
+  const sha256 = hash.digest('hex');
+  const { manifest, trailer, lines } = seen;
+  checkTrailer(seen, sha256);
+  return { manifest, trailer, lines, sha256 };
+};
+
+/**
+ * The cheap check (no passphrase, no decompression): walk the frame
+ * headers; the file must end exactly on a frame marked final. Catches a
+ * file cut short; a flipped byte needs `scanArchive`.
+ */
+export const quickCheckArchive = async (file) => {
+  const handle = await fs.open(file, 'r');
+  try {
+    const { size } = await handle.stat();
+    const head = Buffer.alloc(HEADER_BYTES);
+    const read = await handle.read(head, 0, HEADER_BYTES, 0);
+    if (read.bytesRead < HEADER_BYTES || !head.subarray(0, 6).equals(MAGIC)) {
+      return false;
+    }
+    let at = HEADER_BYTES;
+    const frame = Buffer.alloc(5);
+    for (;;) {
+      if (at + 5 > size) {
+        return false;
+      }
+      await handle.read(frame, 0, 5, at);
+      at += 5 + frame.readUInt32BE(1) + TAG_BYTES;
+      if (at > size) {
+        return false;
+      }
+      if (frame[0] === 1) {
+        return at === size;
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+};
+
+/** Whether `bytes` start an archive of format 2. */
+export const isArchiveFormat2 = async (file) => {
+  const handle = await fs.open(file, 'r');
+  try {
+    const head = Buffer.alloc(MAGIC.length);
+    await handle.read(head, 0, MAGIC.length, 0);
+    return head.equals(MAGIC);
+  } finally {
+    await handle.close();
+  }
+};
+
+// The last newline in the first `size` bytes of the log: where a snapshot
+// of the log taken now ends. A torn last append (no newline) is left out.
+export const lastNewlineEnd = async (handle, size) => {
+  const block = Buffer.alloc(64 * 1024);
+  let end = size;
+  while (end > 0) {
+    const start = Math.max(0, end - block.length);
+    const { bytesRead } = await handle.read(block, 0, end - start, start);
+    const at = block.subarray(0, bytesRead).lastIndexOf(0x0a);
+    if (at !== -1) {
+      return start + at + 1;
+    }
+    end = start;
+  }
+  return 0;
+};
+
+const idOf = (line) => {
+  try {
+    return decodeLine({ notation: line })?.id;
+  } catch {
+    return undefined;
+  }
+};
+
+export const isSecretLine = (line) =>
+  line.includes('secret:') &&
+  typeof idOf(line) === 'string' &&
+  idOf(line).startsWith('secret:');
+
+// Raw record lines of the store's log up to its length now (appends that
+// land during the backup are after that point and are not read).
+export async function* logLines(store, onError) {
+  await store.flushLog?.();
+  const handle = await fs.open(store.logFile, 'r').catch((error) => {
+    if (error.code === 'ENOENT') {
+      return null;
+    }
+    throw error;
+  });
+  if (!handle) {
+    return;
+  }
+  try {
+    const { size } = await handle.stat();
+    const end = await lastNewlineEnd(handle, size);
+    if (end === 0) {
+      return;
+    }
+    const input = handle.createReadStream({
+      start: 0,
+      end: end - 1,
+      autoClose: false,
+    });
+    for await (const line of linesOf(input)) {
+      if (isRecordLine(line)) {
+        yield line;
+      } else if (line.trim().length > 0) {
+        onError?.(line); // legacy or damaged text: not archived
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+}
+
+// Encoded links of a store that has no log file (memory stores, tests).
+async function* queriedLines(store, encodeLine) {
+  const links = await store.query();
+  for (const link of links) {
+    yield encodeLine({ obj: link });
+  }
+}
+
+/**
+ * The plain content of an archive as an async stream of strings, and the
+ * running `stats` ({ lines, secretsExcluded, skipped, bytes, sha256 }; the
+ * hash is final once the stream ended). Secrets are filtered while
+ * streaming unless `includeSecrets`.
+ */
+export const archiveContent = ({
+  store,
+  createdAt,
+  includeSecrets,
+  encodeLine,
+  onProgress,
+  schema = null,
+}) => {
+  const stats = {
+    lines: 0,
+    secretsExcluded: 0,
+    skipped: 0,
+    bytes: 0,
+    sha256: null,
+  };
+  const hash = createHash('sha256');
+  const emit = (text) => {
+    hash.update(text);
+    stats.bytes += Buffer.byteLength(text);
+    return text;
+  };
+  async function* content() {
+    yield emit(
+      `${JSON.stringify({
+        format: FORMAT,
+        version: 2,
+        createdAt,
+        secrets: includeSecrets ? 'included-encrypted' : 'excluded',
+        // The data schema the records are in (R-Z17).
+        schema: schema === null ? undefined : { version: schema },
+        source: store.logFile ? 'log' : 'links',
+      })}\n`
+    );
+    const lines = store.logFile
+      ? logLines(store, () => (stats.skipped += 1))
+      : queriedLines(store, encodeLine);
+    let batch = '';
+    for await (const line of lines) {
+      if (!includeSecrets && isSecretLine(line)) {
+        stats.secretsExcluded += 1;
+        continue;
+      }
+      stats.lines += 1;
+      batch += `${line}\n`;
+      if (batch.length >= 256 * 1024) {
+        yield emit(batch);
+        batch = '';
+        onProgress?.(stats);
+      }
+    }
+    if (batch) {
+      yield emit(batch);
+    }
+    stats.sha256 = hash.digest('hex');
+    yield `${TRAILER}${JSON.stringify({
+      lines: stats.lines,
+      secretsExcluded: stats.secretsExcluded,
+      bytes: stats.bytes,
+      sha256: stats.sha256,
+    })}\n`;
+  }
+  return { stream: Readable.from(content(), { objectMode: false }), stats };
+};
+
+/** The stages from content to sealed frames: `pipeline(content, ...stages, sink)`. */
+export const archiveStages = (passphrase) => [
+  createGzip({ level: 4 }),
+  sealStage(passphrase),
+];

package/js/src/storage/backup.js

@@ -1,33 +1,57 @@
 /**
- * Backup helper — snapshots a DualStore directory into a single
- * timestamped file in the configured archive folder, and applies
- * a simple keep-last-N retention policy (R-A4).
+ * Backup helper — snapshots a store into a single timestamped file in the
+ * configured archive folder, and applies a simple keep-last-N retention
+ * policy (R-A4).
  *
- * The archive format is intentionally trivial: a JSON manifest that
- * embeds the .lino text and base64'd binary blob. A future PR will
- * swap this for tar.zst once we add zstd-codec; the call sites do not
- * change because they only use `createBackup(store, opts)`.
+ * Archives are written and read as streams (format 2, `backup-archive.js`,
+ * R-Z16): the store's log is gzipped line by line and sealed in 1 MB
+ * authenticated frames (AES-256-GCM with a passphrase), so a multi-gigabyte
+ * store backs up and restores in flat memory. `data.bin` is not archived: it
+ * is a snapshot of the log and is rebuilt from it when the store opens.
+ * Archives of the earlier format (one JSON document, `.json`/`.json.enc`)
+ * still verify and restore.
  */
 
 import { promises as fs } from 'node:fs';
 import path from 'node:path';
 import { setInterval, clearInterval } from 'node:timers';
+import { PassThrough } from 'node:stream';
+import { once } from 'node:events';
 import {
   randomBytes,
   scryptSync,
   createCipheriv,
   createDecipheriv,
 } from 'node:crypto';
+import { decodeLine, encodeLine } from 'lino-objects-codec';
 import { createMemoryStore, putAll } from './universal.js';
-import { ensurePrivateDir, sweepTemp, writeFileDurable } from './durable.js';
-import { enoughSpace, skippedEvent } from './disk-space.js';
 import {
   CURRENT_SCHEMA,
   MIGRATIONS,
   SCHEMA_ID,
   runMigrations,
+  schemaLink,
   versionOf,
 } from './migrations/index.js';
+import {
+  ensurePrivateDir,
+  fsyncDir,
+  tighten,
+  sweepTemp,
+  writeStreamDurable,
+} from './durable.js';
+import { enoughSpace, skippedEvent } from './disk-space.js';
+import { acquireStoreLock } from './store-lock.js';
+import {
+  archiveContent,
+  archiveError,
+  archiveStages,
+  isArchiveFormat2,
+  isSecretLine,
+  logLines,
+  quickCheckArchive,
+  scanArchive,
+} from './backup-archive.js';
 
 const ENC_VERSION = 1;
 const KDF_N = 1 << 14;
@@ -36,10 +60,9 @@ const deriveKey = (passphrase, salt) =>
   scryptSync(passphrase, salt, 32, { N: KDF_N });
 
 /**
- * Encrypt a backup manifest with AES-256-GCM. The output envelope
- * is JSON with base64 fields so it remains diff-friendly and trivially
- * portable — the binary path can use the same envelope when the call
- * sites move to tar.zst (R-A4 hardening).
+ * Encrypt a small text with AES-256-GCM. The output envelope is JSON with
+ * base64 fields. (Used for single secrets and exports; backups of a whole
+ * store are framed, see `backup-archive.js`.)
  */
 export const encryptBackup = (plaintext, passphrase) => {
   const salt = randomBytes(16);
@@ -80,45 +103,93 @@ export const decryptBackup = (envelopeJson, passphrase) => {
 const stamp = (d = new Date()) =>
   d.toISOString().replace(/[:.]/g, '-').replace('T', '_').replace(/Z$/, '');
 
-const isSecretId = (link) =>
-  typeof link?.id === 'string' && link.id.startsWith('secret:');
+/** Whether `name` is one of our archives (format 2 `.msbk`, or the old `.json`). */
+export const isBackupFile = (name) =>
+  /^meta-sovereign-.*\.(json|msbk)(\.enc)?$/.test(name);
 
-const BACKUP_FILE = /^meta-sovereign-.*\.json(\.enc)?$/;
+// A second backup into one folder at the same time fails fast.
+const running = new Set();
+
+const looksWhole = (raw, file, passphrase) => {
+  if (!file.endsWith('.enc')) {
+    return Array.isArray(JSON.parse(raw)?.links);
+  }
+  const env = JSON.parse(raw);
+  const whole = ['salt', 'iv', 'tag', 'ct'].every(
+    (k) => typeof env[k] === 'string' && env[k].length > 0
+  );
+  if (!whole || env.cipher !== 'aes-256-gcm') {
+    return false;
+  }
+  return passphrase
+    ? Array.isArray(JSON.parse(decryptBackup(raw, passphrase))?.links)
+    : true;
+};
 
 /**
- * Whether the archive `file` is complete (R-Z2): a plain one parses and
- * holds a links array; an encrypted one has a whole envelope and, when the
- * passphrase is at hand, decrypts to such a manifest. Never throws.
+ * Whether the archive `file` is complete (R-Z2, R-Z16). A format 2 archive
+ * is checked cheaply by default (its frames must end on the final one; no
+ * passphrase needed); `full: true` reads and checks every frame, the
+ * manifest, the line count and the SHA-256 (needs the passphrase of an
+ * encrypted one). An old-format archive parses and holds a links array.
+ * Never throws.
  */
-export const verifyBackup = async (file, { passphrase = null } = {}) => {
+export const verifyBackup = async (
+  file,
+  { passphrase = null, full = false } = {}
+) => {
   try {
-    const raw = await fs.readFile(file, 'utf8');
-    if (!file.endsWith('.enc')) {
-      return Array.isArray(JSON.parse(raw)?.links);
-    }
-    const env = JSON.parse(raw);
-    const whole = ['salt', 'iv', 'tag', 'ct'].every(
-      (k) => typeof env[k] === 'string' && env[k].length > 0
-    );
-    if (!whole || env.cipher !== 'aes-256-gcm') {
-      return false;
+    if (await isArchiveFormat2(file)) {
+      if (!full) {
+        return await quickCheckArchive(file);
+      }
+      await scanArchive(file, { passphrase });
+      return true;
     }
-    return passphrase
-      ? Array.isArray(JSON.parse(decryptBackup(raw, passphrase))?.links)
-      : true;
+    return looksWhole(await fs.readFile(file, 'utf8'), file, passphrase);
   } catch {
     return false;
   }
 };
 
+// Free space a backup of `store` needs: gzip takes the text to about a
+// third; a store without a log file is estimated from its link count.
+const estimateBytes = async (store, passphrase) => {
+  const text = store.logFile
+    ? ((await fs.stat(store.logFile).catch(() => null))?.size ?? 0)
+    : (await store.query()).length * 1024;
+  return Math.ceil(text * 0.35 * (passphrase ? 1.02 : 1)) + 4096;
+};
+
+// Read the new archive back whole and compare it with what was written.
+const confirmWritten = async (file, passphrase, stats) => {
+  try {
+    const read = await scanArchive(file, { passphrase });
+    if (read.sha256 !== stats.sha256 || read.lines !== stats.lines) {
+      throw new Error('content differs');
+    }
+  } catch (cause) {
+    await fs.unlink(file).catch(() => {});
+    throw Object.assign(new Error('the backup could not be read back'), {
+      code: 'BACKUP_VERIFY_FAILED',
+      cause,
+    });
+  }
+};
+
 /**
  * Write a backup and say what it holds: `{ file, links, secrets,
- * secretsExcluded }`. The archive is written atomically (temp file, fsync,
- * rename, fsync of the folder), 0600, then read back; one that does not
- * verify is removed and the call fails. Secrets (`secret:*` links) go in
- * only when the archive is encrypted; otherwise they are left out and the
- * manifest says so (R-Z2). On a nearly full disk nothing is written and the
- * answer is `{ skipped: true, reason: 'low_disk', needed, free }` (R-Z14).
+ * secretsExcluded, bytes, sha256 }` (`links` counts the log records
+ * archived). The log is streamed up to the length it had when the backup
+ * began, cut back to its last whole line, so writes landing meanwhile
+ * cannot tear it. The archive is written atomically (temp file, fsync,
+ * rename, fsync of the folder), 0600, then read back whole and compared
+ * with what was written; one that does not verify is removed and the call
+ * fails. Secrets (`secret:*` links) go in only when the archive is
+ * encrypted; otherwise they are filtered out while streaming and the
+ * manifest counts them (R-Z2). On a nearly full disk nothing is written
+ * and the answer is `{ skipped: true, reason: 'low_disk', needed, free }`
+ * (R-Z14).
  */
 export const makeBackup = async (
   store,
@@ -127,51 +198,52 @@ export const makeBackup = async (
     now = new Date(),
     passphrase = null,
     onEvent = () => {},
+    onProgress,
     measureFree,
   }
 ) => {
   await ensurePrivateDir(archiveDir);
-  const all = await store.query();
-  const links = passphrase ? all : all.filter((l) => !isSecretId(l));
-  const schemaVersion = versionOf(all.find((l) => l.id === SCHEMA_ID));
-  const secretsExcluded = all.length - links.length;
-  const manifest = {
-    createdAt: now.toISOString(),
-    version: 1,
-    secrets: passphrase ? 'included-encrypted' : 'excluded',
-    secretsExcluded,
-    // The data schema the links are in (R-Z17); a manifest without it is version 0.
-    schema: { version: schemaVersion },
-    links,
-  };
-  const plaintext = JSON.stringify(manifest, null, 2);
-  // The archive is about as large as its text (a third more once encrypted
-  // and base64'd); on a nearly full disk it is skipped, not half written
-  // (R-Z14).
-  const estimate = Math.ceil(plaintext.length * (passphrase ? 1.4 : 1));
-  const space = await enoughSpace(archiveDir, estimate, measureFree);
+  await tighten(archiveDir, 0o700);
+  const space = await enoughSpace(
+    archiveDir,
+    await estimateBytes(store, passphrase),
+    measureFree
+  );
   if (!space.ok) {
     onEvent(skippedEvent('backup_skipped', space));
     return { skipped: true, reason: 'low_disk', ...space, links: 0 };
   }
-  const name = `meta-sovereign-${stamp(now)}.json${passphrase ? '.enc' : ''}`;
-  const file = path.join(archiveDir, name);
-  await writeFileDurable(
-    file,
-    passphrase ? encryptBackup(plaintext, passphrase) : plaintext
+  const folder = path.resolve(archiveDir);
+  if (running.has(folder)) {
+    throw archiveError('BACKUP_BUSY', 'a backup is already running');
+  }
+  running.add(folder);
+  const file = path.join(
+    archiveDir,
+    `meta-sovereign-${stamp(now)}.msbk${passphrase ? '.enc' : ''}`
   );
-  if (!(await verifyBackup(file, { passphrase }))) {
-    await fs.unlink(file).catch(() => {});
-    throw Object.assign(new Error('the backup could not be read back'), {
-      code: 'BACKUP_VERIFY_FAILED',
+  try {
+    const { stream, stats } = archiveContent({
+      store,
+      createdAt: now.toISOString(),
+      includeSecrets: Boolean(passphrase),
+      schema: store.get ? versionOf(await store.get(SCHEMA_ID)) : null,
+      encodeLine,
+      onProgress,
     });
+    await writeStreamDurable(file, stream, ...archiveStages(passphrase));
+    await confirmWritten(file, passphrase, stats);
+    return {
+      file,
+      links: stats.lines,
+      secrets: passphrase ? 'included-encrypted' : 'excluded',
+      secretsExcluded: stats.secretsExcluded,
+      bytes: (await fs.stat(file)).size,
+      sha256: stats.sha256,
+    };
+  } finally {
+    running.delete(folder);
   }
-  return {
-    file,
-    links: links.length,
-    secrets: manifest.secrets,
-    secretsExcluded,
-  };
 };
 
 export const createBackup = async (store, options) => {
@@ -188,12 +260,13 @@ export const createBackup = async (store, options) => {
  * Keep the newest `keep` archives. Nothing is deleted unless the newest
  * archive verifies as complete (a crash or a full disk may have left a
  * truncated one that must not cost the older good ones), and leftover
- * temp files of a crashed write are swept (R-Z2).
+ * temp files of a crashed write are swept (R-Z2). The check is the cheap
+ * one (frame structure; the full read happened when it was written).
  */
 export const pruneBackups = async ({ archiveDir, keep, passphrase = null }) => {
   await sweepTemp(archiveDir);
   const entries = await fs.readdir(archiveDir).catch(() => []);
-  const ours = entries.filter((f) => BACKUP_FILE.test(f)).sort();
+  const ours = entries.filter(isBackupFile).sort();
   const toDelete = ours.slice(0, Math.max(0, ours.length - keep));
   if (keep < 1 || toDelete.length === 0) {
     return [];
@@ -272,12 +345,20 @@ export const createBackupScheduler = ({
   };
 };
 
+const tooNew = (version, current) =>
+  Object.assign(
+    new Error(
+      `this backup was written by a newer version (schema ${version}, this app understands up to ${current}); update the app`
+    ),
+    { code: 'BACKUP_TOO_NEW', found: version, current }
+  );
+
 /**
- * The links of a backup in the current shapes (R-Z17). A backup of an older
- * schema is migrated in memory, before anything is written to the store, so
- * a failing migration leaves the store untouched; one of a newer schema is
- * refused with BACKUP_TOO_NEW. The backup's own `meta:schema` is never
- * restored (the store keeps its own).
+ * The links of an old-format backup in the current shapes (R-Z17). A backup
+ * of an older schema is migrated in memory, before anything is written to
+ * the store, so a failing migration leaves the store untouched; one of a
+ * newer schema is refused with BACKUP_TOO_NEW. The backup's own
+ * `meta:schema` is never restored (the store keeps its own).
  */
 export const linksOfManifest = async (
   manifest,
@@ -287,12 +368,7 @@ export const linksOfManifest = async (
     ? manifest.schema.version
     : 0;
   if (version > current) {
-    throw Object.assign(
-      new Error(
-        `this backup was written by a newer version (schema ${version}, this app understands up to ${current}); update the app`
-      ),
-      { code: 'BACKUP_TOO_NEW', found: version, current }
-    );
+    throw tooNew(version, current);
   }
   const links = manifest.links.filter((l) => l.id !== SCHEMA_ID);
   if (version === current) {
@@ -321,11 +397,90 @@ export const linksOfManifest = async (
   return (await scratch.query()).filter((l) => l.id !== SCHEMA_ID);
 };
 
-export const restoreBackup = async (
-  store,
-  file,
-  { passphrase = null, onEvent = () => {} } = {}
-) => {
+// The schema a format 2 archive was written in; refuses a newer one.
+const archiveSchema = (manifest) => {
+  const version = manifest?.schema?.version;
+  if (!Number.isInteger(version)) {
+    return null;
+  }
+  if (version > CURRENT_SCHEMA) {
+    throw tooNew(version, CURRENT_SCHEMA);
+  }
+  return version;
+};
+
+// Lower the store's schema stamp to the archive's, so the records restored
+// in older shapes are migrated when the store next opens (R-Z17).
+const stampOlder = async (store, version) => {
+  if (version === null || version >= CURRENT_SCHEMA) {
+    return;
+  }
+  const existing = await store.get(SCHEMA_ID);
+  if (versionOf(existing) > version) {
+    await store.put({
+      ...schemaLink({ version, applied: existing?.applied ?? [] }),
+      app: existing?.app ?? null,
+    });
+  }
+};
+
+const TOMBSTONE = '~deleted';
+const isTombstone = (record) =>
+  record?.[TOMBSTONE] === true && Object.keys(record).length === 2;
+
+const BATCH = 500;
+
+const malformed = () =>
+  archiveError('BACKUP_INVALID', 'not a valid backup: a record is malformed');
+
+const recordOfLine = (line) => {
+  let record;
+  try {
+    record = decodeLine({ notation: line });
+  } catch {
+    throw malformed();
+  }
+  if (typeof record?.id !== 'string' || !record.id) {
+    throw malformed();
+  }
+  return record;
+};
+
+// Replay the archive's records into `target`, a batch at a time.
+const applyArchive = async (target, file, passphrase) => {
+  let batch = [];
+  let restored = 0;
+  const flush = async () => {
+    if (batch.length > 0) {
+      await putAll(target, batch);
+      restored += batch.length;
+      batch = [];
+    }
+  };
+  await scanArchive(file, {
+    passphrase,
+    onLine: async (line) => {
+      const record = recordOfLine(line);
+      if (record.id === SCHEMA_ID) {
+        return; // the store keeps its own stamp (see stampOlder)
+      }
+      if (isTombstone(record)) {
+        await flush();
+        await target.delete(record.id);
+        return;
+      }
+      batch.push(record);
+      if (batch.length >= BATCH) {
+        await flush();
+      }
+    },
+  });
+  await flush();
+  return restored;
+};
+
+// The old format: one JSON document, checked whole before anything is written.
+const restoreLegacy = async (store, file, passphrase, onEvent) => {
   let raw = await fs.readFile(file, 'utf8');
   if (file.endsWith('.enc')) {
     if (!passphrase) {
@@ -334,14 +489,132 @@ export const restoreBackup = async (
     raw = decryptBackup(raw, passphrase);
   }
   const manifest = JSON.parse(raw);
-  // Checked whole before anything is written: no half restore (R-Z2).
   const valid =
     Array.isArray(manifest?.links) &&
     manifest.links.every((l) => l && typeof l === 'object' && l.id);
   if (!valid) {
-    throw Object.assign(new Error('not a valid backup: links are malformed'), {
-      code: 'BACKUP_INVALID',
-    });
+    throw archiveError(
+      'BACKUP_INVALID',
+      'not a valid backup: links are malformed'
+    );
   }
   return putAll(store, await linksOfManifest(manifest, { onEvent }));
 };
+
+/**
+ * Put the archive's links into the open `store` and say how many (R-Z16).
+ * The whole archive is read and checked first (every frame, the manifest,
+ * the SHA-256, every record), so a damaged or tampered file changes
+ * nothing; then its records are replayed in order, 500 at a time (a later
+ * record replaces an earlier one, a tombstone deletes). Replaying is
+ * idempotent: an interrupted restore is finished by running it again. The
+ * store is the caller's open one, which holds the folder's lock, so
+ * another process cannot write to the folder meanwhile (R-Z2); to replace
+ * the store of a folder that no process has open, use `restoreBackupToDir`.
+ */
+export const restoreBackup = async (
+  store,
+  file,
+  { passphrase = null, onEvent = () => {} } = {}
+) => {
+  if (!(await isArchiveFormat2(file))) {
+    return restoreLegacy(store, file, passphrase, onEvent);
+  }
+  const { manifest } = await scanArchive(file, {
+    passphrase,
+    onLine: (line) => {
+      recordOfLine(line);
+    },
+  });
+  const version = archiveSchema(manifest);
+  // The secret wrapper would seal the log's already sealed secrets again.
+  const restored = await applyArchive(
+    store.rawStore ?? store,
+    file,
+    passphrase
+  );
+  await stampOlder(store.rawStore ?? store, version);
+  return restored;
+};
+
+// Write `line` to `out`, waiting when its buffer is full.
+const emitLine = async (out, line) => {
+  if (!out.write(`${line}\n`)) {
+    await once(out, 'drain');
+  }
+};
+
+// Feed `out` the archive's records, then (for a plain archive, which holds
+// no secrets) the secrets of the log being replaced; end it when whole.
+const feedRestored = async (out, file, { passphrase, oldLog }) => {
+  try {
+    const { manifest } = await scanArchive(file, {
+      passphrase,
+      onLine: (line) => emitLine(out, line),
+    });
+    if (oldLog && manifest.secrets === 'excluded') {
+      for await (const line of logLines({ logFile: oldLog })) {
+        if (isSecretLine(line)) {
+          await emitLine(out, line);
+        }
+      }
+    }
+    out.end();
+  } catch (error) {
+    out.destroy(error);
+  }
+};
+
+/**
+ * Restore an archive as the store of the folder `dir` (R-Z16): the
+ * archive is checked whole, then written to a temp file, fsynced and
+ * renamed over `data.lino` in one step, so the folder holds the old store
+ * or the whole new one, never half. Refuses (`STORE_LOCKED`) when another
+ * process has the folder open, and (`BACKUP_TARGET_NOT_EMPTY`) when it
+ * already holds a store unless `replace: true`; the replaced log is kept
+ * beside it as `data.lino.before-restore-<time>`. A plain archive has no
+ * secrets, so the folder's own `secret:*` lines are carried over. `data.bin`
+ * is removed: it is rebuilt from the log when the store opens. Resolves to
+ * the number of records restored.
+ */
+export const restoreBackupToDir = async (
+  file,
+  dir,
+  { passphrase = null, replace = false } = {}
+) => {
+  if (!(await isArchiveFormat2(file))) {
+    throw archiveError(
+      'BACKUP_INVALID',
+      'only format 2 backups restore into a folder'
+    );
+  }
+  const checked = await scanArchive(file, { passphrase });
+  archiveSchema(checked.manifest);
+  await ensurePrivateDir(dir);
+  const lock = await acquireStoreLock(dir);
+  try {
+    const log = path.join(dir, 'data.lino');
+    const old = await fs.stat(log).catch(() => null);
+    if (old && old.size > 0 && !replace) {
+      throw archiveError(
+        'BACKUP_TARGET_NOT_EMPTY',
+        'the folder already holds a store (use replace)'
+      );
+    }
+    if (old) {
+      await fs.link(log, `${log}.before-restore-${stamp()}`);
+    }
+    const out = new PassThrough();
+    const feeding = feedRestored(out, file, {
+      passphrase,
+      oldLog: old ? log : null,
+    });
+    await writeStreamDurable(log, out);
+    await feeding;
+    await fs.rm(path.join(dir, 'data.bin'), { force: true });
+    await fsyncDir(dir);
+    return checked.lines;
+  } finally {
+    lock.release();
+  }
+};

package/js/src/storage/dual-store.js

@@ -130,6 +130,7 @@ const readOnlyStore = (text, schema) => ({
   corruptLines: text.corruptLines,
   health: () => ({ degraded: false, queuedBytes: 0, error: null }),
   readOnly: true,
+  logFile: text.filePath, // streamed by a backup (R-Z16)
 });
 
 // A folder and everything below it (the downloaded pictures and files).
@@ -235,6 +236,9 @@ export const openDualStore = async (
       schema,
       migration: schema.migrated,
       corruptLines: text.corruptLines,
+      // What a backup streams, and the flush of just the log (R-Z16).
+      logFile: text.filePath,
+      flushLog: () => text.flush(),
       close: lock.release,
     };
   } catch (error) {

package/js/src/storage/durable.js

@@ -9,8 +9,9 @@
  * user: 0600, directories 0700 (a no-op on Windows).
  */
 
-import { promises as fs } from 'node:fs';
+import { createWriteStream, promises as fs } from 'node:fs';
 import path from 'node:path';
+import { pipeline } from 'node:stream/promises';
 
 export const FILE_MODE = 0o600;
 export const DIR_MODE = 0o700;
@@ -99,6 +100,30 @@ export const writeFileDurable = async (
   await fsyncDir(dir);
 };
 
+/**
+ * Like `writeFileDurable`, but the content is a pipeline of streams
+ * (`source, ...stages`) written as it is produced, so a multi-gigabyte
+ * file never exists in memory (R-Z16). Temp file at 0600, fsync, rename,
+ * fsync of the folder; the temp file goes at once when anything fails.
+ */
+export const writeStreamDurable = async (file, ...streams) => {
+  const dir = path.dirname(file);
+  await ensurePrivateDir(dir);
+  const tmp = `${file}.${process.pid}.tmp`;
+  try {
+    await pipeline(
+      ...streams,
+      createWriteStream(tmp, { flags: 'w', mode: FILE_MODE })
+    );
+    await fsyncFile(tmp);
+    await fs.rename(tmp, file);
+  } catch (error) {
+    await fs.unlink(tmp).catch(() => {});
+    throw error;
+  }
+  await fsyncDir(dir);
+};
+
 /** Remove `*.tmp` leftovers of a crashed write, older than a minute. */
 export const sweepTemp = async (dir, olderThanMs = 60_000) => {
   const removed = [];

package/js/src/storage/secret-store.js

@@ -56,6 +56,11 @@ export const wrapSecretStore = ({ inner, passphrase }) => {
     return inner;
   }
   return {
+    // A backup streams the log as stored (secrets sealed) and a restore
+    // puts them back unsealed-again-free (R-Z16).
+    rawStore: inner,
+    logFile: inner.logFile,
+    flushLog: inner.flushLog,
     async put(link) {
       if (isSecret(link)) {
         const sealed = sealLink(link, passphrase);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "meta-sovereign",
-  "version": "0.49.6",
+  "version": "0.49.7",
   "description": "Personal meta profile sovereign system - fully local, privacy-respecting personal CRM and unified messenger across services",
   "type": "module",
   "main": "js/src/index.js",