@gentbajko/slopify 2.5.0 → 3.0.0

package/dist/slices/storage/files-location.js

@@ -0,0 +1,531 @@
+import { createHash } from "node:crypto";
+import { createReadStream, lstatSync, readdirSync } from "node:fs";
+import { access, constants, copyFile, lstat, mkdir, readdir, rename, statfs, utimes, } from "node:fs/promises";
+import { dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import { repoint } from "../../kernel/paths.js";
+import { readSetting, writeSetting } from "../settings/repo.js";
+import { backupsFolderName } from "./layout.js";
+// Where the files a user looks at live: project folders, scheduled backups and anything else
+// Slopify saves for them. New installs keep them in <Documents>/Slopify; installs from before
+// 3.0 keep them where they were, inside the data dir, until the user moves them from Settings →
+// Backup & storage. The database, settings, models and logs always stay in the data dir.
+export const filesLocationKey = "files.location";
+// A move that was started and not finished: its target, so the next try picks up the files it
+// already copied instead of refusing a folder that isn't empty any more.
+export const filesMoveKey = "files.move";
+export const filesRootName = "Slopify";
+const locationSchema = z.discriminatedUnion("kind", [
+    // Everything inside the data dir, the way every install before 3.0 kept it. Stored as a kind,
+    // not a path, so a data dir moved by hand still finds its projects.
+    z.object({ kind: z.literal("data-dir") }).strict(),
+    z.object({ kind: z.literal("root"), root: z.string().min(1) }).strict(),
+]);
+const moveSchema = z.object({ target: z.string().min(1), from: locationSchema }).strict();
+export function filesLayoutOf(location, dataDir) {
+    if (location.kind === "root")
+        return {
+            projects: join(location.root, "Projects"),
+            backups: join(location.root, backupsFolderName),
+            exports: join(location.root, "Exports"),
+        };
+    const projects = join(resolve(dataDir), "projects");
+    return { projects, backups: join(projects, backupsFolderName), exports: null };
+}
+export function readFilesLocation(db) {
+    const stored = readSetting(db, filesLocationKey);
+    if (stored === undefined)
+        return undefined;
+    try {
+        return locationSchema.parse(JSON.parse(stored));
+    }
+    catch {
+        throw new Error(`Slopify's saved files folder setting is damaged, so it doesn't know where your projects are. Start Slopify once with its files in the data dir by deleting the "${filesLocationKey}" row from the settings table, or restore the database from a backup.`);
+    }
+}
+// Decides, once per install, where its files live, and remembers it. An install that already
+// had a database keeps its files where they are; a new one gets <Documents>/Slopify, or
+// "Slopify 2", "Slopify 3"… when that one already holds another install's projects, because
+// storage cleanup deletes project folders its own database doesn't know.
+export async function settleFilesLocation(input) {
+    const stored = readFilesLocation(input.db);
+    if (stored !== undefined)
+        return stored;
+    let location = { kind: "data-dir" };
+    if (input.fresh && input.documents !== undefined) {
+        const documents = await input.documents();
+        for (let n = 1; n <= 20; n++) {
+            const root = join(documents, n === 1 ? filesRootName : `${filesRootName} ${n}`);
+            if (await emptyOrMissing(join(root, "Projects"))) {
+                location = { kind: "root", root };
+                break;
+            }
+        }
+    }
+    writeSetting(input.db, filesLocationKey, JSON.stringify(location));
+    return location;
+}
+// Creates the visible folders of a root, so the user finds Projects and Backups the first time
+// they look. Exports is only made when something is saved there.
+export async function ensureFilesFolders(layout) {
+    await mkdir(layout.projects, { recursive: true });
+    await mkdir(layout.backups, { recursive: true });
+}
+async function emptyOrMissing(path) {
+    const entries = await readdir(path).catch((error) => {
+        if (hasCode(error, "ENOENT"))
+            return [];
+        throw error;
+    });
+    return entries.length === 0;
+}
+export const dockerMoveCommand = "npx @gentbajko/slopify@latest update --docker --projects-dir documents";
+export function createFilesService(deps) {
+    let job;
+    let finished;
+    // From the first check to the start of the copy, so two presses never start two moves.
+    let starting = false;
+    const location = () => readFilesLocation(deps.db) ?? { kind: "data-dir" };
+    const pending = () => {
+        const stored = readSetting(deps.db, filesMoveKey);
+        if (stored === undefined)
+            return undefined;
+        try {
+            return moveSchema.parse(JSON.parse(stored));
+        }
+        catch {
+            return undefined;
+        }
+    };
+    const documentsRoot = async () => join(resolve(await deps.documents()), filesRootName);
+    // Checks the target and starts the copy in the background; the view reports its progress.
+    const begin = async (requested) => {
+        let target;
+        try {
+            target = requested === "documents" ? await documentsRoot() : requested.trim();
+        }
+        catch {
+            return {
+                ok: false,
+                status: 409,
+                detail: "Slopify couldn't find your Documents folder. Use Choose another folder in Settings → Backup & storage and type the full path of the folder you want instead.",
+            };
+        }
+        const from = location();
+        const stored = pending();
+        const resuming = stored !== undefined && samePath(stored.target, target);
+        const problem = await targetProblem(deps, from, target, resuming);
+        if (problem !== undefined)
+            return { ok: false, status: 400, detail: problem };
+        target = resolve(target);
+        const busy = deps.busy();
+        if (busy.length > 0)
+            return { ok: false, status: 409, detail: busyDetail(busy) };
+        const release = deps.beginMutation === undefined ? () => { } : deps.beginMutation();
+        if (release === undefined)
+            return {
+                ok: false,
+                status: 409,
+                detail: "Slopify is installing an update right now. Wait for it to restart, then press the button again in Settings → Backup & storage.",
+            };
+        const pairs = sourcePairs(deps.paths, from, target);
+        const free = deps.freeBytes ?? freeBytesAt;
+        let needed;
+        try {
+            needed = pairs.reduce((sum, pair) => sum + pendingBytes(pair), 0);
+        }
+        catch (error) {
+            release();
+            throw error;
+        }
+        const available = await free(nearestExisting(target)).catch(() => Number.POSITIVE_INFINITY);
+        if (available < needed + 64 * 1024 * 1024) {
+            release();
+            return {
+                ok: false,
+                status: 400,
+                detail: `There isn't enough free space for your files in ${target}: they need ${formatBytes(needed)} and the disk has ${formatBytes(available)} free. Free up space there, or choose a folder on another disk with Choose another folder.`,
+            };
+        }
+        writeSetting(deps.db, filesMoveKey, JSON.stringify({ target, from }));
+        finished = undefined;
+        const progress = {
+            target,
+            phase: "copying",
+            files: 0,
+            totalFiles: 0,
+            bytes: 0,
+            totalBytes: 0,
+            error: null,
+            oldFolder: null,
+        };
+        const current = { progress, done: Promise.resolve() };
+        job = current;
+        const update = (change) => {
+            current.progress = { ...current.progress, ...change };
+        };
+        current.done = runMove(deps, from, target, pairs, update)
+            .then((oldFolder) => {
+            finished = { ...current.progress, phase: "done", oldFolder };
+        })
+            .catch((error) => {
+            finished = {
+                ...current.progress,
+                phase: "failed",
+                error: error instanceof Error ? error.message : String(error),
+            };
+        })
+            .finally(() => {
+            job = undefined;
+            release();
+        });
+        return { ok: true };
+    };
+    return {
+        view: async () => {
+            if (deps.docker !== undefined)
+                return {
+                    docker: true,
+                    folder: deps.docker.hostProjects,
+                    projects: deps.docker.hostProjects,
+                    backups: deps.docker.hostBackups,
+                    exports: null,
+                    inDataDir: false,
+                    documentsRoot: null,
+                    inDocuments: false,
+                    move: null,
+                    dockerCommand: dockerMoveCommand,
+                };
+            const here = location();
+            const root = await documentsRoot().catch(() => null);
+            const stored = pending();
+            const move = job?.progress ??
+                finished ??
+                (stored === undefined
+                    ? null
+                    : {
+                        target: stored.target,
+                        phase: "interrupted",
+                        files: 0,
+                        totalFiles: 0,
+                        bytes: 0,
+                        totalBytes: 0,
+                        error: null,
+                        oldFolder: null,
+                    });
+            return {
+                docker: false,
+                folder: here.kind === "root" ? here.root : deps.paths.projects,
+                projects: deps.paths.projects,
+                backups: deps.paths.backups,
+                exports: deps.paths.exports,
+                inDataDir: here.kind === "data-dir",
+                documentsRoot: root,
+                inDocuments: here.kind === "root" && root !== null && samePath(here.root, root),
+                move,
+                dockerCommand: null,
+            };
+        },
+        move: async (requested) => {
+            if (deps.docker !== undefined)
+                return {
+                    ok: false,
+                    status: 409,
+                    detail: `Slopify runs in Docker here, and a container can't move the folder it was started with. On the computer running Docker, run ${dockerMoveCommand} (or --projects-dir <folder> for another folder); it waits for running work, copies your projects and checks the copy.`,
+                };
+            if (job !== undefined || starting)
+                return {
+                    ok: false,
+                    status: 409,
+                    detail: `Your files are already being moved${job === undefined ? "" : ` to ${job.progress.target}`}. Wait for it to finish; Settings → Backup & storage shows its progress.`,
+                };
+            starting = true;
+            try {
+                return await begin(requested);
+            }
+            finally {
+                starting = false;
+            }
+        },
+        open: async () => {
+            if (deps.docker !== undefined) {
+                const host = deps.docker.hostProjects;
+                if (host === null)
+                    throw new Error("No host folder");
+                const opened = (await deps.docker.openOnHost?.(host, AbortSignal.timeout(10_000)).catch(() => false)) ===
+                    true;
+                return { opened, path: host };
+            }
+            const here = location();
+            const folder = here.kind === "root" ? here.root : deps.paths.projects;
+            if (deps.openFolder === undefined)
+                return { opened: false, path: folder };
+            await deps.openFolder(folder);
+            return { opened: true, path: folder };
+        },
+        idle: async () => {
+            await job?.done;
+        },
+    };
+}
+// What moves where. An install that keeps its files in the data dir has Backups inside its
+// projects folder; the new root has it beside Projects.
+function sourcePairs(paths, from, target) {
+    const next = filesLayoutOf({ kind: "root", root: target }, "");
+    const pairs = [
+        {
+            from: paths.projects,
+            to: next.projects,
+            skip: from.kind === "data-dir" ? backupsFolderName : null,
+        },
+        { from: paths.backups, to: next.backups, skip: null },
+    ];
+    if (paths.exports !== null && next.exports !== null)
+        pairs.push({ from: paths.exports, to: next.exports, skip: null });
+    return pairs;
+}
+// Every regular file under a pair's source, relative to it. Links are left out: Slopify never
+// makes one, and following one could copy a folder the user never meant to move.
+function sourceFiles(pair) {
+    const files = [];
+    const walk = (dir) => {
+        let entries;
+        try {
+            entries = readdirSync(dir, { withFileTypes: true });
+        }
+        catch (error) {
+            if (hasCode(error, "ENOENT"))
+                return;
+            throw error;
+        }
+        for (const entry of entries) {
+            const full = join(dir, entry.name);
+            if (dir === pair.from && entry.name === pair.skip)
+                continue;
+            if (entry.isDirectory())
+                walk(full);
+            else if (entry.isFile()) {
+                const stat = lstatSync(full);
+                files.push({ path: relative(pair.from, full), size: stat.size, mtimeMs: stat.mtimeMs });
+            }
+        }
+    };
+    walk(pair.from);
+    return files;
+}
+function pendingBytes(pair) {
+    let bytes = 0;
+    for (const file of sourceFiles(pair)) {
+        const there = lstatSync(join(pair.to, file.path), { throwIfNoEntry: false });
+        if (!(there?.isFile() && there.size === file.size && there.mtimeMs === file.mtimeMs))
+            bytes += file.size;
+    }
+    return bytes;
+}
+// Copy, check every file's size and SHA-256 against the original, then switch. A file already
+// in the target with the same size and time is one an interrupted move copied, and is only
+// checked. The old folder is never changed.
+async function runMove(deps, from, target, pairs, update) {
+    const listed = pairs.map((pair) => ({ pair, files: sourceFiles(pair) }));
+    const totalFiles = listed.reduce((sum, item) => sum + item.files.length, 0);
+    const totalBytes = listed.reduce((sum, item) => sum + item.files.reduce((s, f) => s + f.size, 0), 0);
+    update({ totalFiles, totalBytes });
+    const copyOne = async (pair, file) => {
+        const source = join(pair.from, file.path);
+        const destination = join(pair.to, file.path);
+        const partial = `${destination}.slopify-partial`;
+        await mkdir(dirname(destination), { recursive: true });
+        await copyFile(source, partial);
+        await utimes(partial, new Date(), new Date(file.mtimeMs));
+        await rename(partial, destination);
+    };
+    for (const pair of pairs)
+        await mkdir(pair.to, { recursive: true });
+    let files = 0;
+    let bytes = 0;
+    for (const { pair, files: list } of listed)
+        for (const file of list) {
+            const there = await lstat(join(pair.to, file.path)).catch(() => undefined);
+            if (!(there?.isFile() && there.size === file.size && there.mtimeMs === file.mtimeMs))
+                await copyOne(pair, file);
+            files += 1;
+            bytes += file.size;
+            update({ files, bytes });
+        }
+    update({ phase: "verifying", files: 0, bytes: 0 });
+    files = 0;
+    bytes = 0;
+    for (const { pair, files: list } of listed)
+        for (const file of list) {
+            const source = join(pair.from, file.path);
+            const destination = join(pair.to, file.path);
+            if (!(await sameContent(source, destination))) {
+                await copyOne(pair, file);
+                if (!(await sameContent(source, destination)))
+                    throw new Error(`The copy of ${source} in ${destination} doesn't match the original, so Slopify kept using the old folder. Check the disk holding ${target} has no errors, then press Continue moving in Settings → Backup & storage.`);
+            }
+            files += 1;
+            bytes += file.size;
+            update({ files, bytes });
+        }
+    // The last look and the switch happen in one go, so nothing can start writing in between.
+    const busy = deps.busy();
+    if (busy.length > 0)
+        throw new Error(`A project started while your files were being copied, so Slopify kept using the old folder. Everything copied so far is kept: once ${busy[0]?.title === undefined ? "it" : `"${busy[0].title}"`} has finished, press Continue moving in Settings → Backup & storage.`);
+    for (const { pair, files: list } of listed) {
+        const now = sourceFiles(pair);
+        const before = new Map(list.map((file) => [file.path, file]));
+        for (const file of now) {
+            const was = before.get(file.path);
+            if (was === undefined || was.size !== file.size || was.mtimeMs !== file.mtimeMs)
+                throw new Error(`Files in ${pair.from} changed while they were being copied (${file.path}), so Slopify kept using the old folder. Press Continue moving in Settings → Backup & storage; it copies only what changed.`);
+        }
+    }
+    const to = { kind: "root", root: target };
+    writeSetting(deps.db, filesLocationKey, JSON.stringify(to));
+    deps.db.prepare("DELETE FROM settings WHERE key=?").run(filesMoveKey);
+    const old = from.kind === "root" ? from.root : deps.paths.projects;
+    repoint(deps.paths, filesLayoutOf(to, deps.paths.dataDir));
+    return old;
+}
+async function sameContent(a, b) {
+    const [sa, sb] = await Promise.all([lstat(a), lstat(b).catch(() => undefined)]);
+    if (sb === undefined || !sb.isFile() || sa.size !== sb.size)
+        return false;
+    const [ha, hb] = await Promise.all([sha256(a), sha256(b)]);
+    return ha === hb;
+}
+async function sha256(path) {
+    const hash = createHash("sha256");
+    for await (const chunk of createReadStream(path))
+        hash.update(chunk);
+    return hash.digest("hex");
+}
+// ----- choosing a folder ------------------------------------------------------------------
+const unixSystem = [
+    "/bin",
+    "/boot",
+    "/dev",
+    "/etc",
+    "/lib",
+    "/lib32",
+    "/lib64",
+    "/opt",
+    "/proc",
+    "/root",
+    "/run",
+    "/sbin",
+    "/snap",
+    "/srv",
+    "/sys",
+    "/usr",
+    "/var",
+    "/System",
+    "/Library",
+    "/Applications",
+    "/private",
+    "/cores",
+];
+const unixShared = ["/", "/home", "/Users", "/mnt", "/media", "/Volumes", "/tmp"];
+// Why a folder can't hold Slopify's files, as the sentence Choose another folder shows;
+// undefined when it can.
+export async function targetProblem(deps, from, requested, resuming) {
+    const platform = deps.platform ?? process.platform;
+    if (requested === "" || !isAbsolute(requested))
+        return `Enter the folder's full path, starting from the top of the disk (for example ${platform === "win32" ? "D:\\Slopify" : "/home/you/Videos/Slopify"}).`;
+    if (/[\p{Cc}]/u.test(requested))
+        return "The folder's name has a line break or another control character in it. Choose a folder whose name has none.";
+    const target = resolve(requested);
+    if (systemFolder(target, platform, deps.home))
+        return `${target} is a system or shared folder. Choose a folder of your own, for example one inside your Documents or Videos folder.`;
+    if (within(deps.paths.dataDir, target))
+        return `${target} is inside Slopify's data folder (${deps.paths.dataDir}), which holds its database and settings. Choose a folder outside it.`;
+    if (from.kind === "root" && samePath(from.root, target))
+        return `Your files are already in ${target}.`;
+    const next = filesLayoutOf({ kind: "root", root: target }, "");
+    for (const [mine, theirs] of [
+        [deps.paths.projects, next.projects],
+        [deps.paths.projects, next.backups],
+        [deps.paths.backups, next.projects],
+        [deps.paths.backups, next.backups],
+        [deps.paths.projects, target],
+    ])
+        if (within(mine, theirs) || (theirs !== target && within(theirs, mine)))
+            return `${target} overlaps the folder your files are in now (${mine}), so copying would copy the folder into itself. Choose a separate folder.`;
+    if (!resuming)
+        for (const folder of [next.projects, next.backups])
+            if (!(await emptyOrMissing(folder).catch(() => false)))
+                return `${folder} already has files in it. Slopify cleans up its Projects folder on its own, so it only moves into an empty one: choose another folder, or move those files away first.`;
+    const existing = nearestExisting(target);
+    const stat = await lstat(existing).catch(() => undefined);
+    if (stat === undefined || !stat.isDirectory())
+        return `${existing} is a file, not a folder, so ${target} can't be made there. Choose another folder.`;
+    try {
+        await access(existing, constants.W_OK);
+    }
+    catch {
+        return `Slopify isn't allowed to write in ${existing}. Choose a folder your user can write to, or change that folder's permissions.`;
+    }
+    return undefined;
+}
+function systemFolder(path, platform, home) {
+    if (platform === "win32") {
+        const p = path.replace(/[\\/]+$/, "").toLowerCase();
+        if (parse(path)
+            .root.replace(/[\\/]+$/, "")
+            .toLowerCase() === p)
+            return true;
+        const env = process.env;
+        return [env.SystemRoot, env.ProgramFiles, env["ProgramFiles(x86)"], env.ProgramData]
+            .filter((dir) => dir !== undefined && dir !== "")
+            .some((dir) => within(dir, path));
+    }
+    if (unixShared.includes(path) || (home !== undefined && samePath(home, path)))
+        return true;
+    return unixSystem.some((dir) => within(dir, path));
+}
+function nearestExisting(path) {
+    let current = resolve(path);
+    for (;;) {
+        if (lstatSync(current, { throwIfNoEntry: false }) !== undefined)
+            return current;
+        const parent = dirname(current);
+        if (parent === current)
+            return current;
+        current = parent;
+    }
+}
+async function freeBytesAt(path) {
+    const stats = await statfs(path);
+    return Number(stats.bavail) * Number(stats.bsize);
+}
+function busyDetail(projects) {
+    const named = projects
+        .slice(0, 3)
+        .map((project) => `"${project.title}"`)
+        .join(", ");
+    const more = projects.length > 3 ? ` and ${projects.length - 3} more` : "";
+    return `Slopify can't move your files while projects are being made (${named}${more}): their files are still being written. Wait for them to finish, or pause them on their project page, then press the button again.`;
+}
+function within(root, path) {
+    const r = relative(root, path);
+    return r === "" || (r !== ".." && !r.startsWith(`..${sep}`) && !isAbsolute(r));
+}
+function samePath(a, b) {
+    return relative(resolve(a), resolve(b)) === "";
+}
+function hasCode(error, code) {
+    return typeof error === "object" && error !== null && "code" in error && error.code === code;
+}
+function formatBytes(bytes) {
+    if (!Number.isFinite(bytes))
+        return "unknown";
+    const units = ["B", "KB", "MB", "GB", "TB"];
+    let value = bytes;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit += 1;
+    }
+    return `${value >= 10 || unit === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}`;
+}

package/dist/slices/storage/layout.js

@@ -5,13 +5,13 @@ export function projectDir(paths, projectId) {
 export function outputPath(paths, projectId, relativePath) {
     return contained(projectDir(paths, projectId), relativePath);
 }
-// Scheduled backups (slices/backups) land here unless the user picks another folder. It sits
-// inside the projects root because that is the one folder a Docker install shares with the
-// host, so the archives end up on the user's disk and not only in the container's volume.
+// Scheduled backups (slices/backups) land in paths.backups unless the user picks another
+// folder. Installs from before 3.0 keep it inside the projects root, since that is the one
+// folder an older Docker install shares with the host; newer ones have it beside Projects.
 // Project folders are ULIDs, so the name never collides with one; reconcile leaves it alone.
 export const backupsFolderName = "Backups";
 export function defaultBackupsDir(paths) {
-    return contained(paths.projects, backupsFolderName);
+    return paths.backups;
 }
 export function stagingPath(paths, stagedFileId) {
     return contained(paths.staging, stagedFileId);
@@ -76,6 +76,8 @@ export function outputFileName(role, index, extension, stageKind) {
             return `shorts/short-${String(index).padStart(2, "0")}${extension}`;
         case "animated_image":
             return `animated/image-${String(index).padStart(3, "0")}${extension}`;
+        case "figure_card":
+            return `cards/card-${String(index).padStart(3, "0")}${extension}`;
         case "audio_body":
             return `audio-body${extension}`;
         case "audio_export":
@@ -86,6 +88,14 @@ export function outputFileName(role, index, extension, stageKind) {
             return `audio-outro${extension}`;
         case "thumbnail":
             return `thumbnail${extension}`;
+        case "script_md":
+            return "script.md";
+        case "audio_mp3":
+            return "narration.mp3";
+        case "audio_m4b":
+            return "audiobook.m4b";
+        case "audio_levelled":
+            return `audio-levelled${extension}`;
         case "reference":
             return `reference${extension}`;
         case "image":

package/dist/slices/storage/model.js

@@ -33,8 +33,19 @@ export const outputRoles = [
     "short_video",
     // An image of the slideshow brought to life by an image-to-video model (Animate images).
     "animated_image",
+    // "Show tables and figures on screen": a table, figure, equation or code block of the
+    // article drawn as a card, in the video's format (and upright for Shorts).
+    "figure_card",
     // The Images stage's establishing image: made or uploaded first, drawn from by the other
     // images, never shown in the video.
     "reference",
+    // Multiple voices: the speaker split of an audiobook's text, and the listening files with
+    // chapter markers made beside the video.
+    "script_md",
+    "audio_mp3",
+    "audio_m4b",
+    // Level the volume: a narration segment joined again from its pieces, each brought to one
+    // loudness first (`meta.segment` says which). The exports play it in place of the plain join.
+    "audio_levelled",
 ];
 export const stagedFileStates = ["copying", "staged"];