@pasko70/pibo 3.4.4 → 3.5.1

package/dist/data/payload-store.js

@@ -1,6 +1,7 @@
+import { pipeline } from "node:stream";
 import { createHash, randomUUID } from "node:crypto";
-import { gunzipSync, gzipSync } from "node:zlib";
-import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { createGunzip, gunzipSync, gzipSync } from "node:zlib";
+import { createReadStream, closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, extname, join, resolve } from "node:path";
 import { piboHomePath } from "../core/pibo-home.js";
 const MAX_SYNC_PAYLOAD_GZIP_BYTES = 512 * 1024;
@@ -23,30 +24,55 @@ export class PiboPayloadMetadataConflictError extends Error {
 export class PayloadStore {
     db;
     rootDir;
-    constructor(db, rootDir = piboHomePath("payloads")) {
+    constructor(db, rootDir = piboHomePath("payloads"), readOnly = false) {
         this.db = db;
         this.rootDir = rootDir === ":memory:" ? rootDir : resolve(rootDir);
-        if (this.rootDir !== ":memory:")
+        if (!readOnly && this.rootDir !== ":memory:")
             mkdirSync(this.rootDir, { recursive: true });
     }
     writePayload(input) {
+        return this.commitPreparedPayload(this.preparePayload(input));
+    }
+    /** Performs hashing, compression and atomic file publication before a DB transaction begins. */
+    preparePayload(input) {
         const contentType = input.contentType ?? defaultContentType(input.value);
         const createdAt = input.createdAt ?? new Date().toISOString();
         const bytes = payloadToBytes(input.value, contentType);
         const sha256 = createHash("sha256").update(bytes).digest("hex");
         const existing = this.findByIdentity(sha256, contentType, input.retentionClass);
         if (existing) {
-            this.db.prepare("UPDATE payloads SET ref_count = ref_count + 1 WHERE id = ?").run(existing.id);
-            return this.getPayload(existing.id) ?? existing;
+            return { ...existing, refCount: 1, status: "staged" };
         }
         const shouldCompress = bytes.byteLength <= MAX_SYNC_PAYLOAD_GZIP_BYTES;
         const encoding = shouldCompress ? "gzip" : "identity";
         const bytesToStore = shouldCompress ? gzipSync(bytes) : bytes;
-        const compressedByteSize = shouldCompress ? bytesToStore.byteLength : null;
+        const compressedByteSize = shouldCompress ? bytesToStore.byteLength : undefined;
         const relativePath = buildRelativePayloadPath(sha256, contentType, input.retentionClass, encoding);
         const absolutePath = this.rootDir === ":memory:" ? relativePath : join(this.rootDir, relativePath);
         writePayloadFile(absolutePath, bytesToStore);
-        const id = input.id ?? `payload_${randomUUID()}`;
+        return {
+            id: input.id ?? `payload_${randomUUID()}`,
+            sha256,
+            storageKind: "file",
+            storagePath: relativePath,
+            contentType,
+            encoding,
+            byteSize: bytes.byteLength,
+            compressedByteSize,
+            previewText: previewTextFromValue(input.value),
+            retentionClass: input.retentionClass,
+            refCount: 1,
+            status: "staged",
+            createdAt,
+        };
+    }
+    /** Links a previously published payload file with only bounded metadata SQL. */
+    commitPreparedPayload(prepared) {
+        const existing = this.findByIdentity(prepared.sha256, prepared.contentType, prepared.retentionClass);
+        if (existing) {
+            this.db.prepare("UPDATE payloads SET ref_count = ref_count + 1 WHERE id = ?").run(existing.id);
+            return this.getPayload(existing.id) ?? existing;
+        }
         this.db.prepare(`
 			INSERT INTO payloads (
 				id,
@@ -64,10 +90,10 @@ export class PayloadStore {
 				created_at,
 				last_verified_at
 			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-		`).run(id, sha256, "file", relativePath, contentType, encoding, bytes.byteLength, compressedByteSize, previewTextFromValue(input.value) ?? null, input.retentionClass, 1, "committed", createdAt, createdAt);
-        const stored = this.getPayload(id);
+		`).run(prepared.id, prepared.sha256, "file", prepared.storagePath ?? null, prepared.contentType, prepared.encoding, prepared.byteSize, prepared.compressedByteSize ?? null, prepared.previewText ?? null, prepared.retentionClass, 1, "committed", prepared.createdAt, prepared.createdAt);
+        const stored = this.getPayload(prepared.id);
         if (!stored)
-            throw new Error(`Failed to persist payload \"${id}\"`);
+            throw new Error(`Failed to persist payload \"${prepared.id}\"`);
         return stored;
     }
     getPayload(id) {
@@ -88,6 +114,61 @@ export class PayloadStore {
             return bytes;
         throw new Error(`Unsupported payload encoding \"${payload.encoding}\"`);
     }
+    openPayloadStream(id) {
+        const payload = this.getPayload(id);
+        if (!payload?.storagePath)
+            throw new Error("Payload not found");
+        if (payload.encoding !== "identity" && payload.encoding !== "gzip")
+            throw new Error("Unsupported payload encoding");
+        const source = createReadStream(this.rootDir === ":memory:" ? payload.storagePath : join(this.rootDir, payload.storagePath));
+        if (payload.encoding === "identity")
+            return source;
+        const decoded = createGunzip();
+        pipeline(source, decoded, () => { }); // Pipeline closes both ends on cancellation or read failure.
+        return decoded;
+    }
+    /** Read a bounded uncompressed range without materializing the entire payload. */
+    async readPayloadRange(id, offset, limit) {
+        if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1)
+            throw new RangeError("Invalid payload range");
+        const payload = this.getPayload(id);
+        if (!payload?.storagePath)
+            throw new Error("Payload not found");
+        if (offset >= payload.byteSize)
+            return Buffer.alloc(0);
+        const path = this.rootDir === ":memory:" ? payload.storagePath : join(this.rootDir, payload.storagePath);
+        const end = Math.min(payload.byteSize, offset + limit);
+        const source = createReadStream(path, payload.encoding === "identity" ? { start: offset, end: end - 1 } : {});
+        if (payload.encoding !== "identity" && payload.encoding !== "gzip") {
+            source.destroy();
+            throw new Error("Unsupported payload encoding");
+        }
+        const stream = payload.encoding === "gzip" ? source.pipe(createGunzip()) : source;
+        const forwardError = (error) => stream.destroy(error);
+        if (stream !== source)
+            source.on("error", forwardError);
+        let position = payload.encoding === "identity" ? offset : 0;
+        const parts = [];
+        try {
+            for await (const value of stream) {
+                const bytes = Buffer.from(value);
+                const from = Math.max(0, offset - position), to = Math.min(bytes.length, end - position);
+                if (to > from)
+                    parts.push(Buffer.from(bytes.subarray(from, to)));
+                position += bytes.length;
+                if (position >= end)
+                    break;
+            }
+        }
+        finally {
+            stream.destroy();
+            source.destroy();
+        }
+        const result = Buffer.concat(parts);
+        if (result.length !== end - offset)
+            throw new Error("Incomplete payload range");
+        return result;
+    }
     readPayloadText(id) {
         return Buffer.from(this.readPayloadBytes(id)).toString("utf8");
     }

package/dist/data/pibo-store.js

@@ -23,10 +23,12 @@ export class PiboDataStore {
     telemetry;
     constructor(path = piboHomePath("pibo.sqlite"), options = {}) {
         this.path = path === ":memory:" ? path : resolve(path);
-        const insidePiboHome = ensurePrivatePiboHomeForPath(this.path);
-        if (this.path !== ":memory:")
+        const insidePiboHome = !options.readOnly && ensurePrivatePiboHomeForPath(this.path);
+        if (!options.readOnly && this.path !== ":memory:")
             mkdirSync(dirname(this.path), { recursive: true });
-        this.db = new DatabaseSync(this.path);
+        this.db = new DatabaseSync(this.path, { readOnly: options.readOnly === true });
+        // Connection-only configuration also protects the initial schema read.
+        this.db.exec("PRAGMA busy_timeout = 5000");
         try {
             assertSupportedPiboDataSchemaVersion(this.db);
         }
@@ -34,14 +36,14 @@ export class PiboDataStore {
             this.db.close();
             throw error;
         }
-        if (insidePiboHome)
+        if (!options.readOnly && insidePiboHome)
             protectPrivateFileSync(this.path);
-        this.db.exec("PRAGMA busy_timeout = 5000");
         this.db.exec("PRAGMA foreign_keys = ON");
-        if (this.path !== ":memory:")
+        if (!options.readOnly && this.path !== ":memory:")
             this.db.exec("PRAGMA journal_mode = WAL");
-        applyPiboDataSchema(this.db);
-        this.payloads = new PayloadStore(this.db, options.payloadRootDir ?? piboHomePath("payloads"));
+        if (!options.readOnly)
+            applyPiboDataSchema(this.db);
+        this.payloads = new PayloadStore(this.db, options.payloadRootDir ?? piboHomePath("payloads"), options.readOnly);
         this.eventLog = new PiboEventLogStore(this.db);
         this.messages = new MessageStore(this.db);
         this.observations = new ObservationStore(this.db);

package/dist/data/schema.js

@@ -1,5 +1,9 @@
-export const PIBO_DATA_SCHEMA_VERSION = 9;
+import { CHAT_READ_PROJECTION_SCHEMA, CHAT_NAVIGATION_REVISION_SCHEMA } from "./chat-read-projections.js";
+import { TELEMETRY_MAINTENANCE_SCHEMA } from "./telemetry-maintenance.js";
+import { MESSAGE_COMMAND_SCHEMA } from "./message-command-store.js";
+export const PIBO_DATA_SCHEMA_VERSION = 14;
 const NATIVE_HISTORY_FALLBACK_SCHEMA_VERSION = 5;
+const TERMINAL_MESSAGE_UNREAD_SCHEMA_VERSION = 14;
 const retiredScopeColumn = ["owner", "scope"].join("_");
 const payloadTableDefinition = `
 	id TEXT PRIMARY KEY,
@@ -198,6 +202,14 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
     if (rebuildPayloadTable)
         rebuildPayloadTableForMetadataIdentity(db);
     hooks.afterStep?.("payload-metadata-identity");
+    if (previousVersion < TERMINAL_MESSAGE_UNREAD_SCHEMA_VERSION) {
+        db.exec(`
+			DROP TRIGGER IF EXISTS chat_unread_event_insert;
+			DROP TRIGGER IF EXISTS chat_unread_event_update;
+			DROP INDEX IF EXISTS idx_event_log_unread_stream;
+			DROP INDEX IF EXISTS idx_event_log_unread_session_stream;
+		`);
+    }
     db.exec(`
 		CREATE TABLE IF NOT EXISTS sessions (
 			id TEXT PRIMARY KEY,
@@ -660,8 +672,10 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 			ON event_log(session_id, stream_id);
 		CREATE INDEX IF NOT EXISTS idx_event_log_unread_session_stream
 			ON event_log(session_id, stream_id)
-			WHERE (retention_class = 'chat_message' AND type IN ('user.message.accepted', 'assistant_message'))
-				OR type = 'session_error';
+			WHERE type = 'message_finished';
+		CREATE INDEX IF NOT EXISTS idx_event_log_unread_error_session_stream
+			ON event_log(session_id, stream_id)
+			WHERE type = 'session_error';
 		CREATE INDEX IF NOT EXISTS idx_event_log_session_sequence_stream
 			ON event_log(session_id, session_sequence DESC, stream_id DESC);
 		CREATE INDEX IF NOT EXISTS idx_event_log_session_type_sequence_stream
@@ -710,6 +724,12 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 			ON telemetry_turns(retention_class, updated_at);
 		CREATE INDEX IF NOT EXISTS idx_telemetry_phases_turn_started
 			ON telemetry_phases(turn_id, started_at ASC);
+		CREATE INDEX IF NOT EXISTS idx_telemetry_phases_open_name
+			ON telemetry_phases(turn_id, name, COALESCE(last_progress_at, started_at) DESC, created_at DESC) WHERE status='open';
+		CREATE INDEX IF NOT EXISTS idx_telemetry_phases_open_turn
+			ON telemetry_phases(turn_id, started_at ASC, created_at ASC) WHERE status='open';
+		CREATE INDEX IF NOT EXISTS idx_telemetry_phases_turn_name
+			ON telemetry_phases(turn_id, name);
 		CREATE INDEX IF NOT EXISTS idx_telemetry_phases_session_status
 			ON telemetry_phases(pibo_session_id, status, last_progress_at);
 		CREATE INDEX IF NOT EXISTS idx_telemetry_phases_provider_request
@@ -759,6 +779,19 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 		CREATE INDEX IF NOT EXISTS idx_telemetry_tool_calls_retention_updated
 			ON telemetry_tool_calls(retention_class, updated_at);
 	`);
+    db.exec(CHAT_READ_PROJECTION_SCHEMA);
+    if (previousVersion < TERMINAL_MESSAGE_UNREAD_SCHEMA_VERSION) {
+        db.exec(`
+			DELETE FROM chat_unread_counts;
+			DELETE FROM chat_unread_index;
+			UPDATE chat_read_backfill
+			SET event_cursor = 0,
+				event_target = (SELECT COALESCE(MAX(stream_id), 0) FROM event_log)
+			WHERE id = 1;
+		`);
+    }
+    db.exec(CHAT_NAVIGATION_REVISION_SCHEMA);
+    db.exec(TELEMETRY_MAINTENANCE_SCHEMA);
     hooks.afterStep?.("schema");
     db.exec(`
 		INSERT OR IGNORE INTO session_runtime_bindings (
@@ -798,6 +831,7 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 			END;
 	`);
     hooks.afterStep?.("render-high-water");
+    db.exec("CREATE INDEX IF NOT EXISTS idx_event_log_sequence_repair_candidates ON event_log(session_id) WHERE session_id IS NOT NULL AND (session_sequence IS NULL OR session_sequence <= 0)");
     // Always inspect for interrupted pre-atomic v7 repairs. A previous process
     // may have written negative temporary values before setting user_version.
     db.exec(`
@@ -807,7 +841,7 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 		);
 		INSERT INTO pibo_v7_sequence_repair_sessions (session_id)
 		SELECT DISTINCT session_id
-		FROM event_log
+		FROM event_log INDEXED BY idx_event_log_sequence_repair_candidates
 		WHERE session_id IS NOT NULL
 			AND (session_sequence IS NULL OR session_sequence <= 0);
 	`);
@@ -858,6 +892,7 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
         }
     }
     hooks.afterStep?.("runtime-binding-metadata");
+    db.exec(MESSAGE_COMMAND_SCHEMA);
     db.exec(`PRAGMA user_version = ${PIBO_DATA_SCHEMA_VERSION}`);
     hooks.afterStep?.("user-version");
 }

package/dist/data/session-store.js

@@ -31,7 +31,7 @@ export class SessionStore {
             now,
         ];
         const assignments = [
-            "pi_session_id = excluded.pi_session_id",
+            ...(input.preserveRuntimeBinding ? [] : ["pi_session_id = excluded.pi_session_id"]),
             "room_id = excluded.room_id",
             "root_session_id = excluded.root_session_id",
             "parent_id = excluded.parent_id",
@@ -53,6 +53,8 @@ export class SessionStore {
 			VALUES (${baseColumns.map(() => "?").join(", ")})
 			ON CONFLICT(id) DO UPDATE SET ${assignments.join(", ")}
 		`).run(...values);
+        if (input.preserveRuntimeBinding)
+            return;
         const binding = input.session.runtimeBinding
             ?? createLegacyPiRuntimeSessionBinding(input.session.id, input.session.piSessionId, input.session.createdAt);
         this.db.prepare(`

package/dist/data/storage-backup.js

@@ -0,0 +1,278 @@
+import { DatabaseSync, backup } from "node:sqlite";
+import { createHash } from "node:crypto";
+import { createReadStream, createWriteStream, statSync } from "node:fs";
+import { mkdir, open, readFile, rename, stat, copyFile } from "node:fs/promises";
+import { basename, dirname, resolve, relative, join, isAbsolute } from "node:path";
+import { createGunzip } from "node:zlib";
+import { pipeline } from "node:stream/promises";
+const MANIFEST = "manifest.json", CATALOG = "payloads.jsonl", DATABASE = "snapshot.sqlite";
+function ownedPath(root, path) { const target = resolve(root, path), rel = relative(resolve(root), target); if (isAbsolute(path) || rel === ".." || rel.startsWith("../") || rel.startsWith("..\\"))
+    throw Error("Backup path escapes its directory"); return target; }
+async function syncFile(path) { const file = await open(path, "r"); try {
+    await file.sync();
+}
+finally {
+    await file.close();
+} }
+async function publishManifest(root, manifest) { const temp = join(root, MANIFEST + ".tmp"), file = await open(temp, "w", 0o600); try {
+    await file.writeFile(JSON.stringify(manifest) + "\n");
+    await file.sync();
+}
+finally {
+    await file.close();
+} await rename(temp, join(root, MANIFEST)); await syncFile(root); }
+export async function readStorageBackupManifest(root) { const path = join(root, MANIFEST); if ((await stat(path)).size > 16384)
+    throw Error("Backup manifest exceeds budget"); const m = JSON.parse(await readFile(path, "utf8")); if (m.format !== "pibo-storage-backup-v1" || !["snapshot", "payloads", "complete"].includes(m.status) || !Number.isSafeInteger(m.maxBytes) || !Number.isSafeInteger(m.maxPayloads))
+    throw Error("Unsupported backup manifest"); return m; }
+async function digest(path, maximum, encoding, signal) { const input = createReadStream(path, { highWaterMark: 65536, signal }); const stream = encoding === "gzip" ? input.pipe(createGunzip()) : input; let bytes = 0; const hash = createHash("sha256"); try {
+    for await (const chunk of stream) {
+        signal?.throwIfAborted();
+        bytes += chunk.length;
+        if (bytes > maximum)
+            throw Error("Backup byte quota exceeded");
+        hash.update(chunk);
+    }
+    return { sha256: hash.digest("hex"), bytes };
+}
+finally {
+    input.destroy();
+    stream.destroy();
+} }
+/** Explicit offline operator operation. The SQLite online-backup API includes committed WAL data. */
+export async function createStorageBackup(input) {
+    const milliseconds = input.maxMilliseconds ?? 60000;
+    if (!Number.isSafeInteger(milliseconds) || milliseconds < 1 || milliseconds > 3600000)
+        throw Error("Invalid backup time quota");
+    const timeout = AbortSignal.timeout(milliseconds);
+    input = { ...input, signal: input.signal ? AbortSignal.any([input.signal, timeout]) : timeout };
+    const root = resolve(input.destination), source = resolve(input.source), payloadRoot = resolve(input.payloadRoot), deadline = Date.now() + (input.maxMilliseconds ?? 60000);
+    const check = () => { input.signal?.throwIfAborted(); if (Date.now() > deadline)
+        throw Error("Backup time budget exhausted; resume explicitly"); };
+    let manifest;
+    let snapshotVerified = false;
+    if (input.resume) {
+        manifest = await readStorageBackupManifest(root);
+        if (manifest.source !== source || manifest.payloadRoot !== payloadRoot)
+            throw Error("Backup resume source does not match manifest");
+    }
+    else {
+        const maxBytes = input.maxBytes ?? 1024 * 1024 * 1024, maxPayloads = input.maxPayloads ?? 100000;
+        if (!Number.isSafeInteger(maxBytes) || maxBytes < 4096 || !Number.isSafeInteger(maxPayloads) || maxPayloads < 1 || maxPayloads > 1000000)
+            throw Error("Invalid backup quota");
+        await mkdir(root, { mode: 0o700 });
+        manifest = { format: "pibo-storage-backup-v1", status: "snapshot", source, payloadRoot, createdAt: new Date().toISOString(), payloadCount: 0, payloadBytes: 0, cursor: "", maxBytes, maxPayloads };
+        await publishManifest(root, manifest);
+    }
+    if (manifest.status === "complete") {
+        await verifyStorageBackup(root, input.signal);
+        return manifest;
+    }
+    if (manifest.status === "snapshot") {
+        check();
+        const walBytes = () => { try {
+            return statSync(source + "-wal").size;
+        }
+        catch {
+            return 0;
+        } };
+        const initialWalBytes = walBytes(), maxWalGrowth = input.maxWalGrowthBytes ?? 64 * 1024 * 1024;
+        if (!Number.isSafeInteger(maxWalGrowth) || maxWalGrowth < 0)
+            throw Error("Invalid WAL growth quota");
+        const db = new DatabaseSync(source, { readOnly: true });
+        const temporary = join(root, DATABASE + ".partial");
+        try {
+            db.exec("PRAGMA busy_timeout=10; BEGIN");
+            const pages = db.prepare("PRAGMA page_count").get().page_count, pageSize = db.prepare("PRAGMA page_size").get().page_size;
+            if (pages * pageSize > manifest.maxBytes)
+                throw Error("Database exceeds backup byte quota");
+            await backup(db, temporary, { rate: 128, progress: ({ totalPages, remainingPages }) => { check(); if (walBytes() - initialWalBytes > maxWalGrowth)
+                    throw Error("Backup WAL growth quota exceeded; source snapshot released"); if (totalPages * pageSize > manifest.maxBytes)
+                    throw Error("Database exceeds backup byte quota"); input.onProgress?.({ stage: "snapshot", copied: (totalPages - remainingPages) * pageSize }); } });
+        }
+        finally {
+            if (db.isTransaction)
+                db.exec("ROLLBACK");
+            db.close();
+        }
+        await syncFile(temporary);
+        await rename(temporary, join(root, DATABASE));
+        const hash = await digest(join(root, DATABASE), manifest.maxBytes, undefined, input.signal);
+        snapshotVerified = true;
+        manifest.databaseSha256 = hash.sha256;
+        manifest.databaseBytes = hash.bytes;
+        manifest.status = "payloads";
+        await publishManifest(root, manifest);
+    }
+    const database = join(root, DATABASE);
+    if (!snapshotVerified && (await digest(database, manifest.maxBytes, undefined, input.signal)).sha256 !== manifest.databaseSha256)
+        throw Error("Backup snapshot hash changed");
+    const db = new DatabaseSync(database, { readOnly: true });
+    try {
+        if (db.prepare("PRAGMA quick_check").get().quick_check !== "ok")
+            throw Error("Backup SQLite integrity check failed");
+        // Rebuild the small cursor from the fsynced catalog, including a file published before a crash.
+        let count = 0, bytes = 0, cursor = "";
+        const catalog = join(root, CATALOG);
+        const catalogFile = await open(catalog, "a+", 0o600);
+        try {
+            const size = (await catalogFile.stat()).size;
+            if (size) {
+                const tail = Buffer.alloc(Math.min(8192, size));
+                await catalogFile.read(tail, 0, tail.length, size - tail.length);
+                if (tail.at(-1) !== 10) {
+                    const newline = tail.lastIndexOf(10);
+                    if (newline < 0 && size > 8192)
+                        throw Error("Invalid partial backup catalog");
+                    await catalogFile.truncate(newline < 0 ? 0 : size - tail.length + newline + 1);
+                    await catalogFile.sync();
+                }
+            }
+        }
+        finally {
+            await catalogFile.close();
+        }
+        for await (const row of catalogRows(catalog, manifest.maxPayloads)) {
+            check();
+            if (row.id <= cursor)
+                throw Error("Backup catalog order is invalid");
+            const hash = await digest(ownedPath(join(root, "payloads"), row.path), manifest.maxBytes, undefined, input.signal);
+            if (hash.sha256 !== row.storedSha256 || hash.bytes !== row.bytes)
+                throw Error("Backup payload changed");
+            count++;
+            bytes += row.bytes;
+            cursor = row.id;
+        }
+        manifest.payloadCount = count;
+        manifest.payloadBytes = bytes;
+        manifest.cursor = cursor;
+        const hasPayloads = Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='payloads'").get());
+        while (hasPayloads) {
+            check();
+            const rows = db.prepare("SELECT id,storage_path,sha256,encoding,byte_size,status FROM payloads WHERE id>? ORDER BY id LIMIT 128").all(manifest.cursor);
+            if (!rows.length)
+                break;
+            for (const row of rows) {
+                check();
+                if (!row.storage_path || row.status !== "committed")
+                    throw Error("Snapshot contains unavailable payload metadata");
+                if (manifest.payloadCount >= manifest.maxPayloads)
+                    throw Error("Backup payload count quota exceeded");
+                const sourcePath = ownedPath(payloadRoot, row.storage_path), target = ownedPath(join(root, "payloads"), row.storage_path), size = (await stat(sourcePath)).size;
+                if ((manifest.databaseBytes ?? 0) + manifest.payloadBytes + size > manifest.maxBytes)
+                    throw Error("Backup byte quota exceeded");
+                await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+                const temporary = target + ".partial";
+                await pipeline(createReadStream(sourcePath, { highWaterMark: 65536 }), createWriteStream(temporary, { mode: 0o600 }), { signal: input.signal });
+                const raw = await digest(temporary, manifest.maxBytes, undefined, input.signal), content = await digest(temporary, row.byte_size, row.encoding, input.signal);
+                if (content.sha256 !== row.sha256 || content.bytes !== row.byte_size)
+                    throw Error("Source payload failed content verification");
+                await syncFile(temporary);
+                await rename(temporary, target);
+                await syncFile(dirname(target));
+                const entry = { id: row.id, path: row.storage_path, sha256: row.sha256, storedSha256: raw.sha256, bytes: raw.bytes, encoding: row.encoding, contentBytes: row.byte_size };
+                const file = await open(catalog, "a", 0o600);
+                try {
+                    await file.writeFile(JSON.stringify(entry) + "\n");
+                    await file.sync();
+                }
+                finally {
+                    await file.close();
+                }
+                manifest.payloadCount++;
+                manifest.payloadBytes += raw.bytes;
+                manifest.cursor = row.id;
+                input.onProgress?.({ stage: "payloads", copied: manifest.payloadCount });
+            }
+            await publishManifest(root, manifest);
+        }
+        manifest.catalogSha256 = (await digest(catalog, Math.max(1024, manifest.maxPayloads * 4096), undefined, input.signal)).sha256;
+        manifest.status = "complete";
+        await publishManifest(root, manifest);
+        return manifest;
+    }
+    finally {
+        db.close();
+    }
+}
+async function* catalogRows(path, maximum) {
+    const stream = createReadStream(path, { highWaterMark: 65536, encoding: "utf8" });
+    let count = 0, pending = "";
+    const parse = (line) => { if (line.length > 4096 || ++count > maximum)
+        throw Error("Backup catalog exceeds quota"); const row = JSON.parse(line); if (typeof row.id !== "string" || typeof row.path !== "string" || !Number.isSafeInteger(row.bytes) || row.bytes < 0)
+        throw Error("Invalid backup catalog row"); return row; };
+    try {
+        for await (const chunk of stream) {
+            pending += chunk;
+            let newline;
+            while ((newline = pending.indexOf("\n")) >= 0) {
+                const line = pending.slice(0, newline);
+                pending = pending.slice(newline + 1);
+                yield parse(line);
+            }
+            if (pending.length > 4096)
+                throw Error("Backup catalog line exceeds quota");
+        }
+        if (pending)
+            throw Error("Incomplete backup catalog line");
+    }
+    finally {
+        stream.destroy();
+    }
+}
+export async function verifyStorageBackup(root, signal) {
+    const m = await readStorageBackupManifest(root);
+    if (m.status !== "complete")
+        throw Error("Backup is incomplete");
+    if ((await digest(join(root, DATABASE), m.maxBytes, undefined, signal)).sha256 !== m.databaseSha256 || (await digest(join(root, CATALOG), Math.max(1024, m.maxPayloads * 4096), undefined, signal)).sha256 !== m.catalogSha256)
+        throw Error("Backup manifest hash mismatch");
+    const db = new DatabaseSync(join(root, DATABASE), { readOnly: true });
+    let payloads = 0, bytes = m.databaseBytes ?? 0, previousId = "";
+    try {
+        if (db.prepare("PRAGMA quick_check").get().quick_check !== "ok")
+            throw Error("Backup database failed verification");
+        const hasPayloads = Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='payloads'").get());
+        for await (const row of catalogRows(join(root, CATALOG), m.maxPayloads)) {
+            signal?.throwIfAborted();
+            if (row.id <= previousId)
+                throw Error("Backup catalog order is invalid");
+            previousId = row.id;
+            const metadata = hasPayloads ? db.prepare("SELECT sha256,storage_path,byte_size FROM payloads WHERE id=?").get(row.id) : undefined;
+            if (!metadata || metadata.sha256 !== row.sha256 || metadata.storage_path !== row.path || metadata.byte_size !== row.contentBytes)
+                throw Error("Catalog does not match snapshot");
+            const path = ownedPath(join(root, "payloads"), row.path);
+            const raw = await digest(path, m.maxBytes, undefined, signal), content = await digest(path, row.contentBytes, row.encoding, signal);
+            if (raw.sha256 !== row.storedSha256 || raw.bytes !== row.bytes || content.sha256 !== row.sha256 || content.bytes !== row.contentBytes)
+                throw Error("Backup payload hash mismatch");
+            payloads++;
+            bytes += raw.bytes;
+            if (bytes > m.maxBytes)
+                throw Error("Backup exceeds byte quota");
+        }
+        const expected = hasPayloads ? Number(db.prepare("SELECT COUNT(*) AS count FROM payloads").get().count) : 0;
+        if (payloads !== expected || payloads !== m.payloadCount)
+            throw Error("Backup payload coverage mismatch");
+        return { payloads, bytes };
+    }
+    finally {
+        db.close();
+    }
+}
+export async function restoreStorageBackup(root, destination, signal) {
+    await verifyStorageBackup(root, signal);
+    const m = await readStorageBackupManifest(root), name = basename(m.source);
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(name))
+        throw Error("Unsafe restored database name");
+    await mkdir(destination, { mode: 0o700 });
+    const payloadRoot = join(destination, "payloads"), database = join(destination, name);
+    for await (const row of catalogRows(join(root, CATALOG), m.maxPayloads)) {
+        signal?.throwIfAborted();
+        const target = ownedPath(payloadRoot, row.path);
+        await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+        await copyFile(ownedPath(join(root, "payloads"), row.path), target);
+        await syncFile(target);
+    }
+    await copyFile(join(root, DATABASE), database);
+    await syncFile(database);
+    await syncFile(destination);
+    return { database, payloadRoot };
+}