@gentbajko/slopify 2.2.0 → 2.3.0

package/dist/slices/storage/backup-export.js

@@ -0,0 +1,321 @@
+import { createHash } from "node:crypto";
+import { constants, lstatSync, readdirSync } from "node:fs";
+import { open } from "node:fs/promises";
+import { join } from "node:path";
+import { transact } from "../../kernel/db/tx.js";
+import { backupFormat, backupSchemaVersion, checksumsMember, fontMember, libraryMember, manifestMember, projectFileMember, projectMember, projectTables, safeRelativePath, stagedMember, usageMember, } from "./backup-format.js";
+import { projectDir, stagingPath } from "./layout.js";
+import { exportableSettings, installedUploadedFontFiles } from "./portable.js";
+import { tarEnd, tarHeader, tarMemberBytes, tarPadding } from "./tar.js";
+// A project that is being made has half-written files and rows that say "running"; copied
+// as they are, the other install would show a run nothing is doing. So a backup waits for
+// every project to be finished, paused or canceled, and says which ones it is waiting for.
+export class BackupBusyError extends Error {
+    projects;
+    constructor(projects) {
+        super(busySentence(projects));
+        this.projects = projects;
+    }
+}
+export function busySentence(projects) {
+    const named = projects
+        .slice(0, 3)
+        .map((project) => `"${project.title}"`)
+        .join(", ");
+    const more = projects.length > 3 ? ` and ${projects.length - 3} more` : "";
+    return `Slopify can't export while projects are being made (${named}${more}): their files are still being written. Wait for them to finish, or pause them on their project page, then press Export everything again.`;
+}
+export function busyProjects(deps) {
+    return deps.db
+        .prepare(`SELECT p.id AS id, p.title AS title,
+         EXISTS(SELECT 1 FROM stages s WHERE s.project_id=p.id AND s.state='running')
+         OR EXISTS(SELECT 1 FROM revision_work w WHERE w.project_id=p.id AND w.state='running')
+         OR EXISTS(SELECT 1 FROM project_queue q WHERE q.project_id=p.id AND q.state='queued')
+         AS busy
+       FROM projects p ORDER BY p.created_at, p.id`)
+        .all()
+        .flatMap((row) => {
+        const id = String(row.id);
+        return row.busy === 1 || deps.hasInflight?.(id) === true
+            ? [{ id, title: String(row.title) }]
+            : [];
+    });
+}
+// Reads everything the backup carries in one read transaction, so the rows agree with each
+// other, and sizes every file so the archive's length is known before its first byte.
+export function planBackup(deps) {
+    const busy = busyProjects(deps);
+    if (busy.length > 0)
+        throw new BackupBusyError(busy);
+    const snapshot = transact(deps.db, () => ({
+        databaseVersion: databaseVersion(deps.db),
+        library: librarySnapshot(deps),
+        usage: {
+            tables: {
+                telemetry_events: rowsOf(deps.db, "SELECT * FROM telemetry_events WHERE type<>'install' ORDER BY rowid"),
+            },
+        },
+        projects: deps.db
+            .prepare("SELECT id,title FROM projects ORDER BY created_at,id")
+            .all()
+            .map((row) => projectSnapshot(deps, String(row.id), String(row.title))),
+    }));
+    const createdAt = deps.clock.now().toISOString();
+    const files = [];
+    for (const project of snapshot.projects)
+        for (const file of project.part.files)
+            files.push({
+                name: projectFileMember(project.part.id, file.path),
+                kind: "file",
+                path: join(projectDir(deps.paths, project.part.id), file.path),
+                size: file.bytes,
+            });
+    for (const font of snapshot.library.part.fonts)
+        files.push({
+            name: fontMember(font.name),
+            kind: "file",
+            path: join(deps.paths.dataDir, "fonts", font.name),
+            size: font.bytes,
+        });
+    for (const staged of snapshot.library.part.staged)
+        files.push({
+            name: stagedMember(staged.id),
+            kind: "file",
+            path: stagingPath(deps.paths, staged.id),
+            size: staged.bytes,
+        });
+    const manifest = {
+        format: backupFormat,
+        schemaVersion: backupSchemaVersion,
+        appVersion: deps.appVersion.slice(0, 40),
+        databaseVersion: snapshot.databaseVersion,
+        backupId: deps.ids.next(),
+        createdAt,
+        projects: snapshot.projects.map((project) => ({
+            id: project.part.id,
+            title: project.title,
+            bytes: project.part.files.reduce((sum, file) => sum + file.bytes, 0),
+        })),
+        files: files.length,
+        bytes: files.reduce((sum, file) => sum + (file.kind === "file" ? file.size : 0), 0),
+    };
+    const members = [
+        json(manifestMember, manifest),
+        json(libraryMember, snapshot.library.part),
+        json(usageMember, snapshot.usage),
+        ...snapshot.projects.map((project) => json(projectMember(project.part.id), project.part)),
+        ...files,
+    ];
+    const archiveBytes = members.reduce((sum, member) => sum +
+        tarMemberBytes({
+            name: member.name,
+            size: member.kind === "bytes" ? member.content.byteLength : member.size,
+        }), 0) +
+        tarMemberBytes({
+            name: checksumsMember,
+            size: checksumsBytes(files, () => "0".repeat(64)).byteLength,
+        }) +
+        tarEnd.byteLength;
+    return {
+        manifest,
+        members,
+        archiveBytes,
+        fileName: `slopify-backup-${createdAt.slice(0, 10)}.tar`,
+    };
+}
+const readChunk = 1024 * 1024;
+// The archive, a chunk at a time. Files are read as they are sent, so a two-hour video
+// costs one megabyte of memory, not its size. A file that changed size since the plan was
+// made ends the stream with an error, and the browser marks the download failed rather
+// than saving a backup that would not import.
+export async function* streamBackup(plan) {
+    const digests = new Map();
+    const files = [];
+    for (const member of plan.members) {
+        const size = member.kind === "bytes" ? member.content.byteLength : member.size;
+        yield tarHeader({ name: member.name, size });
+        if (member.kind === "bytes") {
+            yield member.content;
+        }
+        else {
+            files.push(member);
+            const hash = createHash("sha256");
+            const handle = await open(member.path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+            try {
+                const stat = await handle.stat();
+                if (!stat.isFile() || stat.size !== member.size)
+                    throw changed(member.name);
+                let sent = 0;
+                while (sent < member.size) {
+                    const buffer = new Uint8Array(Math.min(readChunk, member.size - sent));
+                    const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, sent);
+                    if (bytesRead === 0)
+                        throw changed(member.name);
+                    const chunk = buffer.subarray(0, bytesRead);
+                    hash.update(chunk);
+                    sent += bytesRead;
+                    yield chunk;
+                }
+            }
+            finally {
+                await handle.close();
+            }
+            digests.set(member.name, hash.digest("hex"));
+        }
+        const padding = tarPadding(size);
+        if (padding > 0)
+            yield new Uint8Array(padding);
+    }
+    const checksums = checksumsBytes(files, (name) => digests.get(name) ?? "");
+    yield tarHeader({ name: checksumsMember, size: checksums.byteLength });
+    yield checksums;
+    const padding = tarPadding(checksums.byteLength);
+    if (padding > 0)
+        yield new Uint8Array(padding);
+    yield tarEnd;
+}
+function changed(name) {
+    return new Error(`${name} changed while the backup was being written.`);
+}
+function checksumsBytes(files, digest) {
+    return new TextEncoder().encode(JSON.stringify({
+        files: files.map((file) => ({ member: file.name, sha256: digest(file.name) })),
+    }));
+}
+function json(name, value) {
+    return { name, kind: "bytes", content: new TextEncoder().encode(JSON.stringify(value)) };
+}
+function databaseVersion(db) {
+    const row = db.prepare("SELECT max(version) AS version FROM schema_migrations").get();
+    return Number(row?.version ?? 0);
+}
+// Every row of one project in one table. The same selection reads the scratch database on
+// import, so what is exported and what is copied in are the same rows by construction.
+export function projectRows(db, table, projectId) {
+    switch (table) {
+        case "projects":
+            return rowsOf(db, "SELECT * FROM projects WHERE id=?", projectId);
+        case "attempts":
+        case "stage_pieces":
+            return rowsOf(db, `SELECT * FROM ${table} WHERE stage_id IN (SELECT id FROM stages WHERE project_id=?) ORDER BY rowid`, projectId);
+        case "revision_work_pieces":
+            return rowsOf(db, "SELECT * FROM revision_work_pieces WHERE work_id IN (SELECT id FROM revision_work WHERE project_id=?) ORDER BY rowid", projectId);
+        default:
+            return rowsOf(db, `SELECT * FROM ${table} WHERE project_id=? ORDER BY rowid`, projectId);
+    }
+}
+function projectSnapshot(deps, id, title) {
+    const tables = {};
+    for (const table of projectTables) {
+        const rows = projectRows(deps.db, table, id);
+        if (rows.length > 0)
+            tables[table] = rows;
+    }
+    return { title, part: { id, tables, files: projectFiles(projectDir(deps.paths, id)) } };
+}
+// Every plain file under the project's folder, symlinks and all else left out. Not only the
+// rows' files: audio chunks a retry would reuse and the files a revision history keeps are
+// named inside JSON the backup has no business parsing, and reconcile already removes what
+// nothing names.
+function projectFiles(root) {
+    const out = [];
+    const walk = (relative) => {
+        let entries;
+        try {
+            entries = readdirSync(join(root, relative), { withFileTypes: true });
+        }
+        catch (error) {
+            if (error instanceof Error && "code" in error && error.code === "ENOENT")
+                return;
+            throw error;
+        }
+        for (const entry of entries.toSorted((a, b) => a.name.localeCompare(b.name))) {
+            const path = relative === "" ? entry.name : `${relative}/${entry.name}`;
+            if (entry.isDirectory())
+                walk(path);
+            else if (entry.isFile() && safeRelativePath(path))
+                out.push({ path, bytes: lstatSync(join(root, path)).size });
+        }
+    };
+    walk("");
+    return out;
+}
+function librarySnapshot(deps) {
+    const { db, paths } = deps;
+    const tables = {};
+    const put = (table, rows) => {
+        if (rows.length > 0)
+            tables[table] = rows;
+    };
+    const stored = {};
+    for (const row of db.prepare("SELECT key,value FROM settings ORDER BY key").all())
+        stored[String(row.key)] = String(row.value);
+    put("settings", Object.entries(exportableSettings(stored)).map(([key, value]) => ({ key, value })));
+    put("prompts", rowsOf(db, "SELECT * FROM prompts ORDER BY rowid"));
+    put("entries", rowsOf(db, "SELECT * FROM entries ORDER BY rowid"));
+    put("voices", rowsOf(db, "SELECT * FROM voices ORDER BY rowid"));
+    put("document_themes", rowsOf(db, "SELECT * FROM document_themes ORDER BY rowid"));
+    put("project_templates", rowsOf(db, "SELECT * FROM project_templates ORDER BY rowid"));
+    put("project_template_revisions", rowsOf(db, "SELECT * FROM project_template_revisions ORDER BY template_id,version"));
+    put("schedules", rowsOf(db, "SELECT * FROM schedules ORDER BY rowid"));
+    put("schedule_runs", rowsOf(db, "SELECT * FROM schedule_runs ORDER BY rowid"));
+    // Play drafts are work the user typed and has not started yet, so they travel. A draft
+    // that already became projects is not: its projects carry it. Its uploads travel with it
+    // when they are complete on disk; one that is not comes back asking to be reattached,
+    // the same thing a restart does with it.
+    const drafts = rowsOf(db, "SELECT * FROM play_drafts WHERE state='active' ORDER BY rowid");
+    put("play_drafts", drafts);
+    const draftIds = new Set(drafts.map((row) => String(row.id)));
+    const attachments = rowsOf(db, "SELECT * FROM play_draft_attachments ORDER BY rowid").filter((row) => draftIds.has(String(row.draft_id)));
+    const staged = [];
+    const stagedRows = [];
+    const reattach = "Upload is missing or incomplete. Reattach the file.";
+    put("play_draft_attachments", attachments.map((row) => {
+        const stagedId = row.staged_file_id;
+        if (typeof stagedId !== "string")
+            return row;
+        const file = db.prepare("SELECT * FROM staged_files WHERE id=?").get(stagedId);
+        const complete = file !== undefined &&
+            file.state === "staged" &&
+            file.path === stagedId &&
+            fileBytes(stagingPath(paths, stagedId)) === file.bytes;
+        if (!complete)
+            return { ...row, staged_file_id: null, status: "reattach", error: reattach };
+        if (!staged.some((one) => one.id === stagedId)) {
+            staged.push({ id: stagedId, bytes: Number(file.bytes) });
+            stagedRows.push(toRow(file));
+        }
+        return row;
+    }));
+    put("staged_files", stagedRows);
+    put("project_template_instantiations", rowsOf(db, "SELECT * FROM project_template_instantiations ORDER BY rowid").filter((row) => draftIds.has(String(row.draft_id))));
+    const fonts = installedUploadedFontFiles(paths).map((font) => ({
+        name: font.name,
+        bytes: font.bytes,
+    }));
+    return { part: { tables, fonts, staged } };
+}
+function fileBytes(path) {
+    try {
+        const stat = lstatSync(path);
+        return stat.isFile() ? stat.size : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function rowsOf(db, sql, ...params) {
+    return db
+        .prepare(sql)
+        .all(...params)
+        .map(toRow);
+}
+function toRow(row) {
+    const out = {};
+    for (const [key, value] of Object.entries(row)) {
+        if (value !== null && typeof value !== "string" && typeof value !== "number")
+            throw new Error(`Column ${key} holds a value a backup cannot carry.`);
+        out[key] = value;
+    }
+    return out;
+}

package/dist/slices/storage/backup-format.js

@@ -0,0 +1,155 @@
+import { z } from "zod";
+import { fontMaxBytes } from "../fonts/model.js";
+// The full backup (Settings → Backup & storage → Export everything): a tar whose members
+// come in this order, so an import can check every JSON part before it writes a single file:
+//
+//   manifest.json                what made it, when, and which projects it carries
+//   data/library.json            settings, library, templates, schedules, drafts
+//   data/usage.json              the Usage screen's event log
+//   data/projects/<id>.json      one per project: every row it needs to open and keep going
+//   files/...                    the bytes: project folders, uploaded fonts, draft uploads
+//   checksums.json               a SHA-256 per file, written as the files streamed out
+//
+// Rows travel as the database holds them, table by table, with the schema version they fit;
+// an import loads them into a scratch database at that version and lets the same migrations
+// an upgrade runs carry them forward. Provider keys, the telemetry machine id, logs, the
+// model cache, updates and staging leftovers are never read.
+export const backupFormat = "slopify-backup";
+export const backupSchemaVersion = 2;
+export const manifestMember = "manifest.json";
+export const libraryMember = "data/library.json";
+export const usageMember = "data/usage.json";
+export const checksumsMember = "checksums.json";
+export const projectMember = (id) => `data/projects/${id}.json`;
+export const projectFileMember = (id, path) => `files/projects/${id}/${path}`;
+export const fontMember = (name) => `files/fonts/${name}`;
+export const stagedMember = (id) => `files/staging/${id}`;
+// Every table a project's rows live in, parents before children. The queue is left out on
+// purpose: a backup never carries a project that is waiting to run.
+export const projectTables = [
+    "projects",
+    "project_controls",
+    "stages",
+    "attempts",
+    "stage_pieces",
+    "outputs",
+    "project_revisions",
+    "project_heads",
+    "project_assets",
+    "revision_outputs",
+    "revision_pieces",
+    "revision_mutations",
+    "rebuild_previews",
+    "rebuild_admissions",
+    "revision_work",
+    "revision_work_pieces",
+    "revision_work_reservations",
+    "project_control_receipts",
+    "revision_provided_reviews",
+    "review_checkpoints",
+    "review_checkpoint_approvals",
+    "project_recovery_requests",
+];
+export const libraryTables = [
+    "settings",
+    "prompts",
+    "entries",
+    "voices",
+    "document_themes",
+    "project_templates",
+    "project_template_revisions",
+    "schedules",
+    "schedule_runs",
+    "staged_files",
+    "play_drafts",
+    "play_draft_attachments",
+    "project_template_instantiations",
+];
+export const usageTables = ["telemetry_events"];
+// Ceilings a well-formed backup of a real install stays far below. They bound what a hostile
+// or damaged file can make an import hold in memory before anything is written.
+export const backupMaxJsonBytes = 256 * 1024 * 1024;
+export const backupMaxManifestBytes = 4 * 1024 * 1024;
+const maxRows = 2_000_000;
+const maxProjects = 50_000;
+const maxFilesPerProject = 200_000;
+const idPattern = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
+export const backupId = z.string().regex(idPattern);
+// A relative path inside a project folder: plain segments, no way up or out.
+export function safeRelativePath(path) {
+    if (path.length === 0 || path.length > 512 || path.includes("\\") || path.includes("\0"))
+        return false;
+    return path
+        .split("/")
+        .every((part) => part.length > 0 && part !== "." && part !== ".." && part.length <= 255);
+}
+const relativePath = z.string().refine(safeRelativePath, "A file path leaves its folder.");
+const cell = z.union([z.string(), z.number().finite(), z.null()]);
+const row = z.record(z.string().regex(/^[a-z][a-z0-9_]{0,63}$/), cell);
+const rows = z.array(row).max(maxRows);
+function tablesOf(names) {
+    return z
+        .object(Object.fromEntries(names.map((name) => [name, rows.optional()])))
+        .strict();
+}
+// Only the first three fields are read before the version check, so a backup from a newer
+// Slopify is refused with "update first" rather than a list of fields this build can't read.
+export const manifestHead = z
+    .object({
+    format: z.literal(backupFormat),
+    schemaVersion: z.number().int().positive(),
+    appVersion: z.string().max(40),
+})
+    .loose();
+export const manifestSchema = z
+    .object({
+    format: z.literal(backupFormat),
+    schemaVersion: z.literal(backupSchemaVersion),
+    appVersion: z.string().max(40),
+    databaseVersion: z.number().int().positive(),
+    backupId,
+    createdAt: z.string().max(40),
+    projects: z
+        .array(z
+        .object({ id: backupId, title: z.string().max(200), bytes: z.number().int().min(0) })
+        .strict())
+        .max(maxProjects),
+    files: z.number().int().min(0),
+    bytes: z.number().int().min(0),
+})
+    .strict();
+const fileEntry = z.object({ path: relativePath, bytes: z.number().int().min(0) }).strict();
+export const libraryPartSchema = z
+    .object({
+    tables: tablesOf(libraryTables),
+    fonts: z
+        .array(z
+        .object({
+        name: z.string().regex(/^uploaded-[a-f0-9]{64}\.(ttf|otf)$/),
+        bytes: z.number().int().min(12).max(fontMaxBytes),
+    })
+        .strict())
+        .max(64),
+    staged: z
+        .array(z.object({ id: backupId, bytes: z.number().int().positive() }).strict())
+        .max(10_000),
+})
+    .strict();
+export const usagePartSchema = z.object({ tables: tablesOf(usageTables) }).strict();
+export const projectPartSchema = z
+    .object({
+    id: backupId,
+    tables: tablesOf(projectTables),
+    files: z.array(fileEntry).max(maxFilesPerProject),
+})
+    .strict()
+    .refine((part) => part.tables.projects?.length === 1 && part.tables.projects[0]?.id === part.id, "A project part must hold exactly its own project row.");
+export const checksumsSchema = z
+    .object({
+    files: z
+        .array(z
+        .object({ member: z.string().max(1024), sha256: z.string().regex(/^[a-f0-9]{64}$/) })
+        .strict())
+        .max(maxProjects * 10 + 1_000_000),
+})
+    .strict();