@gentbajko/slopify 2.4.0 → 3.0.0

package/dist/main.js

@@ -11,13 +11,15 @@ import { createHostCliClient } from "./adapters/host-cli/index.js";
 import { nodeRunCli } from "./adapters/llm/run-cli.js";
 import { curateRegistry } from "./catalog/registry.js";
 import { createCatalogueStore } from "./catalog/store.js";
-import { dockerActivationCommitted, dockerFolderConfiguration, } from "./edge/docker-projects/activation.js";
-import { createHub } from "./edge/events/hub.js";
+import { createAutostart } from "./edge/autostart/index.js";
+import { dockerActivationCommitted, dockerFilesLayout, dockerFolderConfiguration, } from "./edge/docker-install/activation.js";
+import { createHub, observedHub } from "./edge/events/hub.js";
 import { currentProjectEvent } from "./edge/events/visibility.js";
 import { createApp } from "./edge/http/app.js";
 import { createMutationLifecycle, drainMutationsWithDeadline } from "./edge/http/mutations.js";
 import { limitRequestTimes } from "./edge/http/timeouts.js";
 import { openFolder } from "./edge/open-folder.js";
+import { readHostLogin } from "./host-cli/status.js";
 import { createAudioPreviewStore } from "./kernel/audio-preview.js";
 import { systemClock } from "./kernel/clock.js";
 import { openDb } from "./kernel/db/index.js";
@@ -27,34 +29,57 @@ import { causedBy } from "./kernel/errors.js";
 import { ulidIds } from "./kernel/ids.js";
 import { acquireInstanceLock } from "./kernel/lock.js";
 import { openLog } from "./kernel/log.js";
-import { ensureDirs, layout, subtitleModelDir } from "./kernel/paths.js";
+import { ensureDataDirs, ensureDirs, layout, repoint, subtitleModelDir } from "./kernel/paths.js";
 import { stageKinds } from "./kernel/pipeline.js";
 import { sqliteAttempts } from "./kernel/runner/attempt-repo.js";
+import { auditionVoice } from "./kernel/runner/audition.js";
 import { createCheckpointAuthority, } from "./kernel/runner/checkpoint-authority.js";
 import { createRunner } from "./kernel/runner/index.js";
 import { stageProviders } from "./kernel/runner/providers.js";
 import { createProviderQueue } from "./kernel/runner/queue.js";
+import { standaloneImage, standaloneLlm } from "./kernel/runner/standalone.js";
 import { readVersion } from "./kernel/version.js";
 import { modelSources } from "./model-catalog.js";
-import { projectPaused } from "./slices/admission/repo.js";
+import { projectById, projectPaused, projectTrashed } from "./slices/admission/repo.js";
+import { createBackupService } from "./slices/backups/service.js";
 import { pumpQueue, queueWaiting } from "./slices/batch/index.js";
+import { settleInterruptedCastImages } from "./slices/channels/cast-images.js";
 import { approveCheckpoint } from "./slices/checkpoints/index.js";
 import { checkpointDecisionForWork, recoverCheckpointWork, settleReleasedCheckpoints, } from "./slices/checkpoints/recovery.js";
+import { createEpisodeMemoryWatcher, } from "./slices/episodes/summarize.js";
 import { resolveFont } from "./slices/fonts/index.js";
+import { decodePeaks } from "./slices/narration/peaks.js";
+import { createRunNotifier } from "./slices/notifications/notifier.js";
+import { createNotificationSender } from "./slices/notifications/send.js";
+import { readNotificationUrl } from "./slices/notifications/settings.js";
+import { seedSamples } from "./slices/onboarding/sample.js";
 import { templateById } from "./slices/project-templates/repo.js";
+import { recoverProject } from "./slices/rebuild/recovery.js";
 import { claimWork, finishWork, maySubmit } from "./slices/rebuild/repo.js";
+import { waitToRetry, wakeRetries } from "./slices/rebuild/retry.js";
+import { createReviewRedos, reviewHold } from "./slices/rebuild/review-redo.js";
 import { materializeAdmittedWork } from "./slices/rebuild/runtime-materialize.js";
 import { runRevisionInvocation } from "./slices/rebuild/runtime-run.js";
 import { executionStages, executionStandings, invocationReady, projectStandings, recordWorkProgress, } from "./slices/rebuild/runtime-store.js";
+import { currentRevisionId } from "./slices/revisions/repo.js";
+import { createLimitGate, resumeAfterRestart } from "./slices/run-cost/limits.js";
+import { createStandaloneMeter, createUsageMeter } from "./slices/run-cost/meter.js";
 import { settleTerminalScheduleRuns } from "./slices/schedules/repo.js";
 import { createScheduleRunner } from "./slices/schedules/scheduler.js";
 import { nodeCliProbe } from "./slices/settings/cli-status.js";
-import { isLocalCliProvider } from "./slices/settings/model.js";
+import { isLocalCliProvider, localCliConcurrency } from "./slices/settings/model.js";
 import { providerStatuses } from "./slices/settings/readiness.js";
+import { busyProjects } from "./slices/storage/backup-export.js";
+import { documentsDir, nodeDocumentsHost } from "./slices/storage/documents.js";
+import { createFilesService, ensureFilesFolders, filesLayoutOf, settleFilesLocation, } from "./slices/storage/files-location.js";
 import { reconcileStorage } from "./slices/storage/reconcile.js";
+import { previewPictures } from "./slices/style-preview/images.js";
+import { ffmpegStylePreview } from "./slices/style-preview/render.js";
+import { createStylePreviews, stylePreviewDir } from "./slices/style-preview/service.js";
 import { collectorEndpoint, httpPostEvents } from "./slices/telemetry/collector-client.js";
 import { createFlusher } from "./slices/telemetry/flush.js";
 import { record } from "./slices/telemetry/record.js";
+import { createTrashPurge } from "./slices/trash/service.js";
 import { probeDurationMs } from "./slices/video/ffmpeg.js";
 import { watchActivation } from "./updater/candidate.js";
 import { launchUpdate } from "./updater/install.js";
@@ -121,11 +146,12 @@ export async function boot(config, options = {}) {
     if (dockerState !== undefined &&
         (process.env.SLOPIFY_CONTAINER !== "1" ||
             dockerState !== "/opt/slopify-install/activation.json"))
-        throw new Error("This container was started with Slopify's Docker settings in the wrong place (SLOPIFY_DOCKER_INSTALL_STATE). Start it with the launcher instead: npx @gentbajko/slopify --docker");
+        throw new Error("This container was started with Slopify's Docker settings in the wrong place (SLOPIFY_DOCKER_INSTALL_STATE must be /opt/slopify-install/activation.json). Start it with Slopify's compose.yaml, or install it with npx @gentbajko/slopify --docker");
     const candidateToken = process.env.SLOPIFY_UPDATE_TOKEN ?? "";
     const pendingActivation = isUpdateToken(candidateToken) && process.env.SLOPIFY_UPDATE_PENDING === "1";
-    ensureDirs(paths, { mode: 0o700 });
+    ensureDataDirs(paths, { mode: 0o700 });
     const lock = acquireInstanceLock(paths.lock);
+    const freshInstall = !existsSync(paths.db);
     let db;
     try {
         const ffmpeg = await prepareFfmpeg({
@@ -136,10 +162,29 @@ export async function boot(config, options = {}) {
         });
         db = openDb(paths.db);
         migrate(db, clock);
+        // Where the projects, backups and exports live. A container's are the installer's mounts;
+        // a native install decides once and remembers (slices/storage/files-location.ts).
+        const container = process.env.SLOPIFY_CONTAINER === "1";
+        const files = container
+            ? dockerFilesLayout(process.env, paths.dataDir)
+            : filesLayoutOf(await settleFilesLocation({
+                db,
+                dataDir: paths.dataDir,
+                fresh: freshInstall,
+                documents: options.filesInDocuments === true
+                    ? () => documentsDir(nodeDocumentsHost())
+                    : undefined,
+            }), paths.dataDir);
+        repoint(paths, files);
+        if (files.exports === null)
+            ensureDirs(paths, { mode: 0o700 });
+        else
+            await ensureFilesFolders(files);
         const runtimeDb = db;
         const interrupted = markInterruptedStages(db, clock);
         recoverCheckpointWork(db);
         const settledSchedules = settleTerminalScheduleRuns(db, clock.now().toISOString());
+        settleInterruptedCastImages({ db });
         // The updater can restore the database after a failed candidate boot, but it cannot
         // restore files deleted by reconciliation. Leave the filesystem untouched until the
         // candidate's committed activation pointer has been verified.
@@ -151,10 +196,28 @@ export async function boot(config, options = {}) {
                 : `interrupted stages ${interrupted}, settled schedules ${settledSchedules}, orphan files ${reconciled.orphanFiles}, staged files ${reconciled.stagedFiles}`,
         });
         const eventDb = db;
-        const hub = createHub({
+        const sendNotification = createNotificationSender(globalThis.fetch);
+        const notifier = createRunNotifier({
+            url: () => readNotificationUrl(eventDb),
+            subject: (projectId) => {
+                const project = projectById(eventDb, projectId);
+                return project === undefined
+                    ? undefined
+                    : { title: project.title, makesVideo: project.config.sources.video !== "off" };
+            },
+            send: sendNotification,
+            log,
+        });
+        // Episode memory summarises a project that has just finished; wired once the providers
+        // exist, below.
+        let episodes;
+        const hub = observedHub(createHub({
             ids,
             log,
             acceptEvent: (event) => currentProjectEvent(eventDb, event),
+        }), (event) => {
+            notifier.observe(event);
+            episodes?.observe(event);
         });
         const version = readVersion();
         const telemetry = { db, ids, clock, log, appVersion: version };
@@ -176,8 +239,26 @@ export async function boot(config, options = {}) {
             probe: nodeCliProbe,
             hostCli,
         }), catalogue);
+        // Calls that belong to no project (topic generation, episode summaries, cast pictures)
+        // still go through the attempt wrapper, and are metered against their schedule or channel.
+        const standalone = {
+            registry,
+            clock,
+            log,
+            meter: createStandaloneMeter({ db, ids, clock, catalogue: () => catalogue.read() }),
+        };
+        episodes = createEpisodeMemoryWatcher({
+            db,
+            paths,
+            clock,
+            log,
+            uuid: randomUUID,
+            llm: (call) => standaloneLlm(standalone, call),
+        });
         const audioPreviews = createAudioPreviewStore();
+        const reviewRedos = createReviewRedos();
         const runner = wireRunner({
+            onFinished: (work) => reviewRedos.kick(work.projectId),
             db,
             paths,
             clock,
@@ -241,7 +322,7 @@ export async function boot(config, options = {}) {
                 ? Promise.resolve(version)
                 : publishedVersion(globalThis.fetch),
             unsupported: () => process.env.SLOPIFY_DISABLE_UPDATES === "1"
-                ? "This container is updated by pulling a new image and recreating it."
+                ? "Slopify in Docker is updated from the terminal: npx @gentbajko/slopify@latest update. It waits for running work, keeps a recovery copy of your data and puts the previous version back if the new one doesn't start."
                 : !existsSync(oldEntry) || !existsSync(workerEntry)
                     ? "Run Slopify from its installed package to use in-app updates."
                     : npm === undefined
@@ -253,6 +334,18 @@ export async function boot(config, options = {}) {
                     .prepare("SELECT id FROM projects")
                     .all()
                     .some((row) => typeof row.id === "string" && runner.hasInflight?.(row.id) === true),
+            busyWith: () => {
+                const running = updateDb
+                    .prepare("SELECT p.title FROM stages s JOIN projects p ON p.id=s.project_id WHERE s.state='running' LIMIT 1")
+                    .get();
+                if (typeof running?.title === "string")
+                    return running.title;
+                const inflight = updateDb
+                    .prepare("SELECT id,title FROM projects")
+                    .all()
+                    .find((row) => typeof row.id === "string" && runner.hasInflight?.(row.id) === true);
+                return typeof inflight?.title === "string" ? inflight.title : undefined;
+            },
             report: (message) => log.write("warn", "update", { detail: message }),
             install: async (next, restarting) => {
                 if (npm === undefined)
@@ -286,6 +379,21 @@ export async function boot(config, options = {}) {
             modelsFor: modelSources(registry).modelsFor,
             emit: (projectId, event) => hub.emit(projectId, event),
         };
+        // Redos a review asked for before the last shutdown start now.
+        reviewRedos.bind(rebuild);
+        reviewRedos.kick();
+        // A stage that was waiting for a CLI's plan limits when the app stopped carries on waiting.
+        void resumeAfterRestart(db, async (projectId) => {
+            const baseRevisionId = currentRevisionId(runtimeDb, projectId);
+            if (baseRevisionId === undefined)
+                return false;
+            const result = await recoverProject(rebuild, projectId, {
+                baseRevisionId,
+                idempotencyKey: randomUUID(),
+                action: { kind: "resume" },
+            });
+            return result.ok;
+        }, log);
         const draftDeps = {
             db,
             paths,
@@ -305,11 +413,20 @@ export async function boot(config, options = {}) {
                 flusher.soon();
             },
         };
-        const scheduleDeps = {
+        const scheduleRunnerDeps = {
             ...draftDeps,
             template: (id, templateVersion) => templateById(runtimeDb, id, templateVersion),
+            topicLlm: (call) => standaloneLlm(standalone, call),
+            topicsWaiting: (event) => {
+                hub.emitGlobal(event);
+                notifier.observeTopics(event);
+            },
+        };
+        const scheduleRunner = createScheduleRunner(scheduleRunnerDeps);
+        const scheduleDeps = {
+            ...scheduleRunnerDeps,
+            requestTopics: scheduleRunner.requestTopics,
         };
-        const scheduleRunner = createScheduleRunner(scheduleDeps);
         scheduleRunner.recover(clock.now());
         const scheduleTicks = createScheduleTickLifecycle({
             beginMutation: updater.beginMutation,
@@ -319,14 +436,67 @@ export async function boot(config, options = {}) {
             }),
         });
         const mutations = createMutationLifecycle();
+        const folderConfiguration = await dockerFolderConfiguration(process.env, paths.projects);
+        const filesService = createFilesService({
+            db,
+            paths,
+            documents: () => documentsDir(nodeDocumentsHost()),
+            busy: () => busyProjects({ db: runtimeDb, hasInflight: runner.hasInflight }),
+            openFolder,
+            beginMutation: updater.beginMutation,
+            ...(container
+                ? {
+                    docker: {
+                        hostProjects: folderConfiguration.hostProjects,
+                        hostBackups: folderConfiguration.hostBackups,
+                        ...(hostCli === undefined ? {} : { openOnHost: hostCli.openFolder }),
+                    },
+                }
+                : {}),
+        });
+        const backups = createBackupService({
+            db,
+            paths,
+            clock,
+            ids,
+            log,
+            appVersion: version,
+            hasInflight: runner.hasInflight,
+            location: folderConfiguration,
+            bootedAt: clock.now(),
+            beginMutation: updater.beginMutation,
+        });
+        // Before the server answers, so the first page already lists the samples. A failure costs
+        // only the samples: the log says why and Settings → Restore samples tries again.
+        if (options.seedSample === true && !pendingActivation)
+            await seedSamples({ db, paths, clock, ids, log, appVersion: version }).catch((error) => {
+                log.write("warn", "sample.seed", { detail: causedBy(error) });
+            });
+        const autostart = await createAutostart({
+            db,
+            dataDir: paths.dataDir,
+            logs: paths.logs,
+            port: config.port,
+            host: config.host,
+            version,
+            env: process.env,
+        });
+        if (options.refreshAutostart === true)
+            void autostart.refresh().catch((error) => {
+                log.write("warn", "autostart.refresh", { detail: causedBy(error) });
+            });
         const app = createApp({
+            autostart,
             rebuild,
             drafts: draftDeps,
             schedules: scheduleDeps,
+            backups,
+            files: filesService,
             ...(rebuild.measureAudio === undefined ? {} : { measureAudio: rebuild.measureAudio }),
+            decodePeaks: (path, signal) => decodePeaks(ffmpeg, path, signal),
             openFolder,
             folderLocation: {
-                ...(await dockerFolderConfiguration(process.env, paths.projects)),
+                ...folderConfiguration,
                 ...(hostCli === undefined ? {} : { openOnHost: hostCli.openFolder }),
             },
             ...(dockerState === undefined ? {} : { installationPending: () => updater.locked() }),
@@ -336,8 +506,24 @@ export async function boot(config, options = {}) {
             runner,
             updater,
             mutations,
+            sendNotification,
             audioPreviews,
+            stylePreviews: createStylePreviews({
+                dir: stylePreviewDir(paths.dataDir),
+                render: ffmpegStylePreview({ ffmpeg, paths, log, dir: stylePreviewDir(paths.dataDir) }),
+                pictures: previewPictures({ db, paths }),
+                log,
+            }),
             ...modelSources(registry),
+            audition: (call, signal) => auditionVoice({ registry, clock, log }, call, signal),
+            voiceLanguages: async (provider, voiceId, signal) => {
+                try {
+                    return await registry.tts(provider).voiceLanguages?.(voiceId, signal);
+                }
+                catch {
+                    return undefined;
+                }
+            },
             catalogue,
             clock,
             ids,
@@ -347,6 +533,13 @@ export async function boot(config, options = {}) {
             flushSoon: flusher.soon,
             probe: nodeCliProbe,
             hostCliStatus: hostCli?.status,
+            generateCastImage: ({ channelId, ...call }) => standaloneImage(standalone, {
+                ...call,
+                owner: { kind: "channel", id: channelId },
+                purpose: "cast-image",
+            }),
+            fetch: globalThis.fetch,
+            ...(hostCli === undefined ? { cliLogin: readHostLogin } : {}),
         });
         const server = await listen(app, config, log);
         const queueTimer = setInterval(() => {
@@ -366,7 +559,70 @@ export async function boot(config, options = {}) {
         const scheduleTimer = setInterval(() => {
             void scheduleTicks.tick();
         }, 15_000);
+        // A step waiting out a rate limit or a timeout runs again once its wait is over; the wait
+        // is in the database, so this also picks up the ones a restart left waiting.
+        const retryTimer = setInterval(() => {
+            const release = updater.beginMutation();
+            if (!release)
+                return;
+            try {
+                wakeRetries(updateDb, clock, runner);
+            }
+            catch (error) {
+                log.write("error", "stage.retry", { detail: causedBy(error) });
+            }
+            finally {
+                release();
+            }
+        }, 5_000);
         void scheduleTicks.tick();
+        // Once a minute is plenty for a daily slot; the decision itself (slices/backups/schedule)
+        // holds the first run back for a couple of minutes after a start.
+        // The model catalogue keeps itself current: once at start, then whenever a day has passed
+        // since the last check (looked at hourly, so a laptop that slept catches up).
+        const syncModels = () => {
+            if (!options.refreshModels || !catalogue.sync)
+                return;
+            catalogue.sync().then((status) => {
+                if (status.warning !== null)
+                    log.write("warn", "model-catalog.sync", { detail: status.warning });
+            }, (error) => log.write("warn", "model-catalog.sync", { detail: causedBy(error) }));
+        };
+        syncModels();
+        const modelTimer = setInterval(() => {
+            if (catalogue.syncDue?.(Date.now()) === true)
+                syncModels();
+        }, 60 * 60_000);
+        const backupTimer = setInterval(() => {
+            backups.tick().catch((error) => {
+                log.write("error", "backups.tick", { detail: causedBy(error) });
+            });
+        }, 60_000);
+        // Settings → Trash keeps deleted items 30 days; once a day (looked at hourly, and once at
+        // start) whatever is older goes for good, a project's folder with it.
+        const trashPurge = createTrashPurge({
+            db: updateDb,
+            paths,
+            clock,
+            log,
+            hasInflight: runner.hasInflight,
+        });
+        const purgeTrash = () => {
+            const release = updater.beginMutation();
+            if (!release)
+                return;
+            try {
+                trashPurge.tick();
+            }
+            catch (error) {
+                log.write("error", "trash.purge", { detail: causedBy(error) });
+            }
+            finally {
+                release();
+            }
+        };
+        purgeTrash();
+        const trashTimer = setInterval(purgeTrash, 60 * 60_000);
         listeningPort = portOf(server) ?? config.port;
         // Whatever last run left queued goes out at start. Nothing waits for
         // it, and an unreachable collector costs one refused socket.
@@ -388,10 +644,19 @@ export async function boot(config, options = {}) {
         let stopActivation = () => { };
         shutdown = () => {
             stopping ??= (async () => {
+                notifier.close();
+                episodes?.close();
                 clearInterval(queueTimer);
                 clearInterval(scheduleTimer);
+                clearInterval(retryTimer);
+                clearInterval(backupTimer);
+                clearInterval(trashTimer);
+                clearInterval(modelTimer);
                 const mutationDrain = mutations.stop();
                 const scheduleDrain = scheduleTicks.stop();
+                const topicDrain = scheduleRunner.stop();
+                // A backup being written is stopped and its partial file removed, not waited for.
+                const backupDrain = backups.stop();
                 const serverClose = beginServerClose(server);
                 stopActivation();
                 audioPreviews.close();
@@ -401,7 +666,10 @@ export async function boot(config, options = {}) {
                     // database until their response settles; anything racing shutdown receives 503.
                     await drainMutationsWithDeadline(mutationDrain, serverClose.terminate, mutationDrainTimeoutMs);
                     await scheduleDrain;
+                    await topicDrain;
+                    await backupDrain;
                     await runner.abortAll();
+                    await episodes?.settled();
                 }
                 finally {
                     serverClose.terminate();
@@ -425,7 +693,7 @@ export async function boot(config, options = {}) {
         };
         if (pendingActivation)
             stopActivation = watchActivation(updater, candidateToken, shutdown, (message) => log.write("warn", "update", { detail: message }));
-        return { paths, url: urlOf(config.host, listeningPort), stop: shutdown };
+        return { paths, url: urlOf(config.host, listeningPort), stop: shutdown, autostart };
     }
     catch (error) {
         db?.close();
@@ -433,7 +701,7 @@ export async function boot(config, options = {}) {
         throw error;
     }
 }
-export function wireRunner({ audioPreviews, db, paths, clock, ids, log, hub, telemetry, flusher, registry, catalogue, ffmpeg, }) {
+export function wireRunner({ onFinished, audioPreviews, db, paths, clock, ids, log, hub, telemetry, flusher, registry, catalogue, ffmpeg, alignSubtitles: aligner = alignSubtitles, }) {
     // A stage counts what it did and the queue is flushed after each new event. `record`
     // swallows its own failures, so this can neither fail a stage nor widen what leaves the
     // machine - the payload allow-list is checked inside it.
@@ -441,7 +709,17 @@ export function wireRunner({ audioPreviews, db, paths, clock, ids, log, hub, tel
         record(telemetry, type, counters);
         flusher.soon();
     };
-    const execution = { db, paths, ids, clock, log, ffmpeg, alignSubtitles, audioPreviews, count };
+    const execution = {
+        db,
+        paths,
+        ids,
+        clock,
+        log,
+        ffmpeg,
+        alignSubtitles: aligner,
+        audioPreviews,
+        count,
+    };
     // A stage slice is handed the wrapped calls, never the registry: every provider call
     // it makes is already inside the retry policy (kernel/runner/providers.ts).
     const providers = {
@@ -449,10 +727,23 @@ export function wireRunner({ audioPreviews, db, paths, clock, ids, log, hub, tel
         attempts: sqliteAttempts(db, ids),
         clock,
         log,
-        queue: createProviderQueue((provider) => isLocalCliProvider(provider) ? 3 : (catalogue.read().providers[provider]?.maxConcurrent ?? 1)),
+        meter: createUsageMeter({ db, ids, clock, catalogue: () => catalogue.read() }),
+        limits: createLimitGate({
+            db,
+            clock,
+            log,
+            changed: (projectId) => hub.emit(projectId, { type: "project.updated", projectId }),
+        }),
+        queue: createProviderQueue((provider) => isLocalCliProvider(provider)
+            ? localCliConcurrency(provider)
+            : (catalogue.read().providers[provider]?.maxConcurrent ?? 1)),
     };
     const checkpoints = createCheckpointAuthority({
-        decide: (work) => checkpointDecisionForWork(execution, work),
+        // A review waiting to send its item back holds the item's dependents like a checkpoint.
+        decide: (work) => {
+            const review = reviewHold(execution, work);
+            return review.kind === "held" ? review : checkpointDecisionForWork(execution, work);
+        },
         approve: (projectId, checkpointId, identity) => approveCheckpoint(db, {
             ...identity,
             projectId,
@@ -473,20 +764,30 @@ export function wireRunner({ audioPreviews, db, paths, clock, ids, log, hub, tel
             },
             standingsOf: (projectId) => executionStandings(execution, projectId),
             ready: (work) => invocationReady(execution, work),
-            paused: (projectId) => projectPaused(db, projectId) || queueWaiting(db, projectId),
+            // A project in the trash (Settings → Trash) holds like a paused one until it is restored.
+            paused: (projectId) => projectPaused(db, projectId) ||
+                queueWaiting(db, projectId) ||
+                projectTrashed(db, projectId),
             claim: (work) => {
                 const claimed = claimWork(db, work);
                 projectStandings(execution, work.projectId);
                 return claimed;
             },
             maySubmit: (work, pieceId) => maySubmit(db, work, pieceId),
-            finish: (work, state, reason) => {
+            waitToRetry: (work, fault, reason) => transact(db, () => {
+                const at = waitToRetry(execution, work, fault, reason, Math.random);
+                if (at !== undefined)
+                    projectStandings(execution, work.projectId);
+                return at;
+            }),
+            finish: (work, state, reason, kind) => {
                 transact(db, () => {
-                    finishWork(db, work, state, reason);
+                    finishWork(db, work, state, reason, kind ?? null);
                     materializeAdmittedWork(execution, work.projectId);
                     projectStandings(execution, work.projectId);
                     settleReleasedCheckpoints(execution, work.projectId);
                 });
+                onFinished?.(work);
             },
         },
         runs: Object.fromEntries(stageKinds.map((kind) => [

package/dist/patch-notes/0.4.2-to-1.0.md

@@ -0,0 +1,93 @@
+# Slopify 0.4.2 to 1.0
+
+Versions 0.4.3 to 1.0.0, released 9 to 13 September 2026 (0.4.2 shipped on 3 September).
+
+This is the stretch where Slopify went from a pipeline that ran once, start to finish, to something you can live with: pause it, change your mind halfway, edit a finished project without paying for everything twice, and leave it running on a schedule. It also picked up free local captions, the Gemini CLI, Inworld voices and an updater. 1.0.0 is the first release where a finished project is never a dead end.
+
+## Highlights
+
+- Captions, made on your machine for free: English word timing with a 95 MB speech model, no caption service and no key.
+- Edit any finished project. Slopify keeps every version and remakes only what your change affects.
+- Pause and resume a run, and switch its providers while it is paused.
+- A cost review before anything starts, with the exact inputs, prices and a readiness check.
+- Review checkpoints hold Audio, Images or Video until you approve them, while the rest keeps going.
+- Templates for setups you reuse, and daily, weekly or one-off schedules that run them.
+- Gemini CLI joins Claude Code and Codex as a text provider, using the login you already have.
+- Inworld TTS-2 and TTS-2 Flash voices.
+- Update from inside the app with one click.
+
+## What's new
+
+### Getting started
+- An interactive tutorial walks from API keys to a downloaded video in 18 steps (19 from 0.5.1), highlighting the real controls on each screen. Open it from Tutorial in the navigation or Start tutorial on an empty Projects page.
+- The tutorial remembers where you were across reloads and never starts a run for you.
+- Two ways to install on slopify.stream: `npx @gentbajko/slopify@latest` with nothing to install, or `npm install -g @gentbajko/slopify` and then `slopify`.
+
+### Runs
+- Pause and Resume. Pausing stops active work, keeps every finished piece and survives an app restart.
+- Article is the only required stage. Research, Audio, Images, Thumbnail and Video can each be Off.
+- With Images off and Audio on, the run exports a 48 kHz stereo WAV. With Audio off, it renders a silent MP4 at 5 seconds per image.
+- Images from a saved prompt and prewritten thumbnails start straight away, beside research and writing.
+- Uploaded narration is used as it is.
+- Play is four sections, with a preview of the output shape and caption style next to the controls.
+- Play drafts save to this computer as you type, uploads included, and come back after a restart.
+- Review shows the exact inputs, prices and catalogue date before you start. Nothing runs until Start.
+- Review also checks that every provider, model and voice the run needs is ready.
+- Batches: several title or keyword variants from one setup, run one after another.
+- Provider-aware queues cap how many requests go to each provider at once.
+
+### Editing finished projects
+- Change the settings, article, narration, images or captions of any project, finished or not.
+- Every save is a new version in History. Restore any earlier one.
+- Before a rebuild, Slopify lists the outputs your change affects and what they cost. Work that is unaffected is reused, not paid for again.
+- Each project keeps its own copy of the prompts it was made with, so editing a Library prompt never changes an old project.
+- Edit a single image, a caption line, or the article text by hand.
+
+### Review checkpoints
+- Hold Audio, Images or Video/export for approval. Independent work carries on while one part waits.
+- Approve held work from the project's progress view. Approvals survive restarts and later edits.
+
+### Templates and schedules
+- Save a project's setup as a named template and apply it in Play. Templates keep versions, so an old draft is never changed under you.
+- Schedules run a template once, daily or weekly, in your time zone, with up to 49 keyword variants and an optional spend limit.
+- Each schedule has a run history and Pause, Resume, Cancel and Delete. Missed runs follow a policy you choose, and a run never overlaps the one before it.
+
+### Captions and fonts
+- Free English captions for new and existing narrated projects, timed word by word to the actual speech.
+- SRT, VTT and ASS files, and captions burned into the video.
+- Five caption positions, a font size and a live preview in the real output shape.
+- Bundled Barlow, your installed fonts, or upload a TTF or OTF up to 32 MiB.
+- Changing only the caption style reuses the narration and the WAV instead of making them again.
+- When the narration skips a short passage, captions resync on the following words and add a note beside the caption download saying what was missed.
+
+### Narration
+- Inworld TTS-2 and TTS-2 Flash. TTS-2 takes up to 100,000 characters per job.
+- Split narration by characters as well as by paragraph or word count: 3,000 characters by default, always on a sentence end.
+- Hear narration while it is still being made, and watch the article write itself.
+
+### Providers and models
+- Gemini CLI as a text and research provider, signed in with the login you already have.
+- Set the path to each CLI in Settings when Slopify can't find it on your PATH.
+- Model lists load from the provider or the installed CLI, with a refresh button and space to type a model ID.
+- A curated model catalogue with prices, context sizes and thinking settings, refreshed from GitHub.
+
+### Project page
+- A focused workspace: stage list on the left, the output of the chosen stage beside it, and a direct download of the final file.
+- Overall progress next to each stage's own progress.
+- Open folder next to every download.
+
+### Updates and upkeep
+- An update button checks every 15 minutes and installs on one click. It waits for active work, checks the new version starts, and rolls the database back if it doesn't.
+- Settings exports and imports a backup of your templates, prompts, voices and settings. Provider keys are never included.
+- Disk usage per project, cleanup of files no project uses, and a diagnostics download with no secrets in it.
+
+## Fixes
+
+- A missing ffmpeg is downloaded into the app's data folder and checked before any provider work starts.
+- Claude Code no longer applies your personal coding setup to article writing.
+- Google's image errors are read correctly; an image quota of zero fails once with a clear message instead of retrying.
+- Slideshow zoom is faster and travels from 100% to 122.5%.
+- The update indicator no longer sticks on Updating after a check finds nothing new.
+- A stalled or interrupted update recovers its status on the next start.
+- Many fixes to editing and rebuilds: regenerated articles open current, uploaded audio can be replaced by generated narration, and interrupted rebuilds recover their unfinished outputs.
+- Many fixes to Play drafts: unfinished uploads, image counts, empty chunk sizes and discarded drafts are all kept or recovered correctly.