@gentbajko/slopify 2.3.0 → 2.4.0

package/dist/slices/video/chapters.js

@@ -0,0 +1,89 @@
+import { splitEndMatter } from "../article/split.js";
+// ceiling: a heading is looked for within 200 words either side of where its share of the
+// article puts it; past that, a rewritten narration has drifted too far to trust a match.
+const searchWords = 200;
+// The heading's first words that have to match, in order.
+const matchWords = 4;
+export function headingChapters(markdown, words) {
+    if (words.length === 0)
+        return [];
+    const headings = topHeadings(splitEndMatter(markdown).body);
+    const articleWords = tokens(splitEndMatter(markdown).body).length;
+    if (headings.length === 0 || articleWords === 0)
+        return [];
+    const spoken = words.map((word) => normalise(word.text));
+    const chapters = [];
+    let after = -1;
+    for (const heading of headings) {
+        const estimate = Math.round((heading.at / articleWords) * words.length);
+        const wanted = tokens(heading.title).slice(0, matchWords);
+        const found = findNear(spoken, wanted, estimate, after + 1);
+        const index = found ?? Math.max(after + 1, Math.min(estimate, words.length - 1));
+        const word = words[index];
+        if (word === undefined || index <= after)
+            continue;
+        chapters.push({ title: heading.title, start: word.start });
+        after = index;
+    }
+    return chapters;
+}
+// The article's chapter headings: the highest level used, except a lone title heading at the
+// very top, which names the article rather than a part of it.
+export function topHeadings(markdown) {
+    const found = [];
+    let fenced = false;
+    let before = 0;
+    for (const line of markdown.split(/\r?\n/)) {
+        if (/^\s*(```|~~~)/.test(line))
+            fenced = !fenced;
+        const matched = fenced ? null : /^\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);
+        if (matched?.[1] !== undefined && matched[2] !== undefined) {
+            const title = plainHeading(matched[2]);
+            if (title !== "")
+                found.push({ level: matched[1].length, title, at: before });
+        }
+        before += tokens(line).length;
+    }
+    if (found.length === 0)
+        return [];
+    const top = Math.min(...found.map((heading) => heading.level));
+    const atTop = found.filter((heading) => heading.level === top);
+    const first = found[0];
+    if (atTop.length === 1 && first !== undefined && first.level === top && found.length > 1) {
+        const rest = found.slice(1);
+        const next = Math.min(...rest.map((heading) => heading.level));
+        return rest.filter((heading) => heading.level === next);
+    }
+    return atTop;
+}
+function plainHeading(text) {
+    return text
+        .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1")
+        .replace(/[*_`~]/g, "")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function tokens(text) {
+    return text
+        .split(/\s+/)
+        .map(normalise)
+        .filter((token) => token !== "");
+}
+function normalise(word) {
+    return word.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
+}
+// The start of the run of `wanted` in `spoken` nearest to `estimate`, at or after `from`.
+function findNear(spoken, wanted, estimate, from) {
+    if (wanted.length === 0)
+        return undefined;
+    const low = Math.max(from, estimate - searchWords);
+    const high = Math.min(spoken.length - wanted.length, estimate + searchWords);
+    let best;
+    for (let at = low; at <= high; at += 1) {
+        if (!wanted.every((token, offset) => spoken[at + offset] === token))
+            continue;
+        if (best === undefined || Math.abs(at - estimate) < Math.abs(best - estimate))
+            best = at;
+    }
+    return best;
+}

package/dist/slices/video/cuts.js

@@ -0,0 +1,141 @@
+import { transcriptSentences } from "../youtube/transcript.js";
+import { cutFloorShare } from "./edit-settings.js";
+// "Follow the narration": every cut lands in the pause after a sentence, as near to Seconds
+// per image as the sentences allow, and a chapter always starts on a new shot. Pure, so the
+// same timing plans the same cuts every time and a re-render is identical.
+//
+// The images carry no link to a passage of the article: each one is rendered from an image
+// prompt template filled with the run's keyword values, never from a paragraph. So they are
+// spread over the narration in slideshow order, taking turns as they do with Every N
+// seconds; only where the cuts fall changes.
+// A cut sits in the pause between two sentences: halfway through it, but never more than this
+// long after the last word, so a long breath does not hold the old picture over the new
+// sentence's first word.
+const pauseCutSeconds = 0.25;
+// A chapter start is moved onto the nearest sentence pause within this many seconds; a
+// chapter the model or a heading places mid-sentence would otherwise cut mid-word.
+const chapterSnapSeconds = 3;
+// Where the narration pauses between sentences, in seconds into the video. The word times
+// already count the silence at the start, the intro and the gaps (`runtime-subtitles.ts`).
+export function sentenceCutPoints(words) {
+    const sentences = transcriptSentences(words);
+    const points = [];
+    for (const [index, sentence] of sentences.entries()) {
+        const next = sentences[index + 1];
+        if (next === undefined)
+            break;
+        const pause = Math.max(0, next.start - sentence.end);
+        points.push(sentence.end + Math.min(pause / 2, pauseCutSeconds));
+    }
+    return points;
+}
+// The frames of each shot, in timeline order, adding up to `totalFrames`. Between two
+// chapter starts the shots are laid greedily: each next cut is the sentence pause nearest to
+// Seconds per image after the last one, no nearer than the floor (40% of it) and no further
+// than twice it; with no pause in that window the cut falls at Seconds per image, as it does
+// with Every N seconds. What is left before a chapter start that could not hold a full shot
+// and a floor-length one after it becomes the last shot, so no shot runs past 1.4 times
+// Seconds per image unless the sentences leave no pause.
+export function narrationShotFrames(input) {
+    const total = input.totalFrames / input.fps;
+    if (!(total > 0) || !(input.imageSeconds > 0))
+        return [Math.max(1, input.totalFrames)];
+    const floor = input.imageSeconds * (input.floorShare ?? cutFloorShare);
+    const points = [...input.cutPoints]
+        .filter((point) => Number.isFinite(point) && point > 0 && point < total)
+        .toSorted((left, right) => left - right);
+    // A chapter that opens with the narration's first words would cut the silence before them
+    // into a shot of its own; the first shot opens that chapter anyway.
+    const bounds = chapterCuts(input.chapterStarts, points, total).filter((cut) => cut >= floor);
+    const cuts = [];
+    let start = 0;
+    for (const end of [...bounds, total]) {
+        let position = start;
+        while (end - position > input.imageSeconds + floor) {
+            const target = position + input.imageSeconds;
+            const low = position + floor;
+            const high = Math.min(position + 2 * input.imageSeconds, end - floor);
+            const cut = nearest(points.filter((point) => point >= low && point <= high), target);
+            position = cut ?? target;
+            cuts.push(position);
+        }
+        if (end < total)
+            cuts.push(end);
+        start = end;
+    }
+    return framesBetween(cuts, input.totalFrames, input.fps);
+}
+// The chapter starts as cuts: inside the video, each moved to the nearest pause within 3 s,
+// and at least one frame from the start, the end and each other.
+export function chapterCuts(starts, points, total) {
+    const snapped = starts
+        .filter((start) => Number.isFinite(start))
+        .map((start) => {
+        const near = nearest(points.filter((point) => Math.abs(point - start) <= chapterSnapSeconds), start);
+        return near ?? start;
+    })
+        .filter((start) => start > 0 && start < total)
+        .toSorted((left, right) => left - right);
+    return snapped.filter((start, index) => index === 0 || start > (snapped[index - 1] ?? 0));
+}
+function nearest(values, target) {
+    let best;
+    for (const value of values)
+        if (best === undefined || Math.abs(value - target) < Math.abs(best - target))
+            best = value;
+    return best;
+}
+// Cut times to whole frames. Two cuts that round to the same frame are one cut, so every shot
+// is at least one frame, and the last shot runs to the end.
+function framesBetween(cuts, totalFrames, fps) {
+    const marks = [];
+    for (const cut of cuts) {
+        const frame = Math.round(cut * fps);
+        if (frame > (marks.at(-1) ?? 0) && frame < totalFrames)
+            marks.push(frame);
+    }
+    const shots = [];
+    let from = 0;
+    for (const mark of [...marks, totalFrames]) {
+        shots.push(mark - from);
+        from = mark;
+    }
+    return shots.filter((frames) => frames > 0);
+}
+// The shots (0-based) the video and each chapter open on, as the render will cut them: what
+// "Chapter openers" animates. The cuts before a chapter do not depend on how long the video
+// runs after it, so the timeline is taken to end just past the last chapter.
+export function chapterOpeningShots(input) {
+    const { fps, imageSeconds } = input;
+    const last = Math.max(0, ...input.chapterStarts.filter((start) => Number.isFinite(start)));
+    const totalFrames = Math.round((last + 1) * fps);
+    if (!input.narration) {
+        // Every N seconds does not move a chapter: the image on screen when it starts opens it.
+        const each = Math.max(1, Math.round(imageSeconds * fps));
+        return [
+            ...new Set([
+                0,
+                ...input.chapterStarts
+                    .filter((start) => Number.isFinite(start) && start > 0)
+                    .map((start) => Math.floor(Math.round(start * fps) / each)),
+            ]),
+        ];
+    }
+    const starts = chapterStartFrames(input.chapterStarts, input.cutPoints, totalFrames, fps);
+    const shots = narrationShotFrames({ ...input, totalFrames });
+    const opening = [0];
+    let frame = 0;
+    for (const [index, frames] of shots.entries()) {
+        if (starts.includes(frame) && index > 0)
+            opening.push(index);
+        frame += frames;
+    }
+    return opening;
+}
+// The frame each chapter's shot starts on, for the chapter cards and the chapter openers:
+// the same snapping the cuts use, so a card appears with its shot.
+export function chapterStartFrames(starts, points, totalFrames, fps) {
+    return chapterCuts(starts, points, totalFrames / fps)
+        .map((start) => Math.round(start * fps))
+        .filter((frame) => frame > 0 && frame < totalFrames);
+}

package/dist/slices/video/edit-list.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { atmospheres, colorGrades, lookLevels, } from "./edit-settings.js";
 // The edit list is the whole video as data: what plays, for how many frames, how it moves,
 // and the audio under it. Planning (`plan.ts`) writes one and the renderer (`ffmpeg.ts`,
 // `slideshow.ts`) reads nothing else, so a new kind of shot or movement is a new case
@@ -6,11 +7,10 @@ import { z } from "zod";
 // It is plain JSON, recorded in render.json, and carries a version so a later Slopify can
 // tell an older list from its own.
 //
-// Extension points, none of them built yet: `Shot.source` gains a `video` kind for clips,
-// `Shot.transition` says how a shot enters from the one before, and effects (a colour
-// grade, a text card) become a list on the shot. Each is a new optional field or union
-// case, so a version-1 list stays readable; a change that alters what an existing field
-// means bumps the version instead.
+// Each extension is a new optional field or union case, so a version-1 list stays readable:
+// `Shot.source` has a `video` kind for clips, `Shot.transition` says how a shot enters from
+// the one before, and the list's `look` and `cards` are the effects every shot is rendered
+// with. A change that alters what an existing field means bumps the version instead.
 export const editListVersion = 1;
 // The same list with every file path passed through `map`: render.json records paths
 // relative to the project folder.
@@ -25,6 +25,9 @@ export function withPaths(edit, map) {
             ...shot,
             source: { ...shot.source, path: map(shot.source.path) },
         })),
+        ...(edit.cardFont === undefined
+            ? {}
+            : { cardFont: { ...edit.cardFont, path: map(edit.cardFont.path) } }),
     };
 }
 const share = z.number().min(0).max(1);
@@ -36,6 +39,26 @@ const motionSchema = z.discriminatedUnion("kind", [
     z.object({ kind: z.literal("still") }).strict(),
 ]);
 const count = z.number().int().positive();
+const sourceSchema = z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("image"), path: z.string().min(1) }).strict(),
+    z
+        .object({ kind: z.literal("video"), path: z.string().min(1), seconds: z.number().positive() })
+        .strict(),
+]);
+const transitionSchema = z
+    .object({ kind: z.enum(["crossfade", "fadeblack", "slide", "wipe"]), frames: count })
+    .strict();
+const lookSchema = z
+    .object({
+    vignette: z.enum(lookLevels),
+    grain: z.enum(lookLevels),
+    grade: z.enum(colorGrades),
+    atmosphere: z.enum(atmospheres),
+})
+    .strict();
+const cardSchema = z
+    .object({ title: z.string().min(1), startFrame: z.number().int().nonnegative(), frames: count })
+    .strict();
 const editListSchema = z
     .object({
     version: z.literal(editListVersion),
@@ -52,12 +75,19 @@ const editListSchema = z
     shots: z
         .array(z
         .object({
-        source: z.object({ kind: z.literal("image"), path: z.string().min(1) }).strict(),
+        source: sourceSchema,
         frames: count,
         motion: motionSchema,
+        transition: transitionSchema.optional(),
     })
         .strict())
         .min(1),
+    look: lookSchema.optional(),
+    cards: z.array(cardSchema).optional(),
+    cardFont: z
+        .object({ path: z.string().min(1), name: z.string().min(1) })
+        .strict()
+        .optional(),
 })
     .strict();
 // Reads a recorded edit list back. Strict, so a list written by a newer Slopify, with a

package/dist/slices/video/edit-settings.js

@@ -0,0 +1,191 @@
+// How the Video stage edits the slideshow beyond seconds per image, zoom and motion: where the
+// cuts fall, how one shot hands over to the next, the Look (vignette, grain, colour, an
+// atmosphere overlay and chapter cards), and which images come to life as short clips.
+// Browser-safe: Play, Edit project, the review summary and the change list read these names,
+// labels and limits too.
+//
+// A project saved before this existed has no settings at all, and absent reads as
+// `legacyVideoEdit`: cuts every N seconds, hard cuts, no Look, nothing animated. That is what
+// every video was, and the render recipe leaves every one of those values out of its
+// fingerprint, so no finished video turns outdated. A new project starts from
+// `defaultVideoEdit`, which follows the narration.
+export const cutModes = ["interval", "narration"];
+export const transitionKinds = ["cut", "crossfade", "fadeblack", "slide", "wipe"];
+export const lookLevels = ["off", "subtle", "strong"];
+export const colorGrades = ["none", "warm", "cold", "desaturated", "sepia"];
+// Drawn procedurally by ffmpeg's own filters (`look.ts`); no third-party footage.
+export const atmospheres = ["none", "embers", "dust", "fog"];
+export const animateModes = ["off", "chapters", "every"];
+export const legacyVideoEdit = {
+    cuts: "interval",
+    transition: "cut",
+    transitionSeconds: 0.6,
+    vignette: "off",
+    grain: "off",
+    grade: "none",
+    atmosphere: "none",
+    chapterCards: false,
+    animate: "off",
+    animateEvery: 3,
+    animateModel: "",
+};
+// What a new project starts with: the owner's pick is to follow the narration.
+export const defaultVideoEdit = { ...legacyVideoEdit, cuts: "narration" };
+// ceiling: under 0.2 s a transition is a flash rather than a move; over 2 s it eats a short
+// shot. The steps are the ones Play and Edit project offer.
+export const transitionSecondsMin = 0.2;
+export const transitionSecondsMax = 2;
+export const transitionSecondsSteps = [0.2, 0.4, 0.6, 0.8, 1, 1.2, 1.5, 2];
+export const animateEveryMin = 2;
+export const animateEveryMax = 10;
+// Image-to-video models make 5-second clips at their cheapest; a longer shot slows or loops
+// the clip (`video/clip-timing.ts`).
+export const animatedClipSeconds = 5;
+// What an image is animated with: keep the picture, move a little. Image-to-video models
+// invent the most when told the least.
+export const animatePrompt = "Subtle cinematic motion: a slow, steady camera push-in with gentle ambient movement in the scene. Keep the composition, colours, lighting and style of the image exactly. No new objects, no text, no cuts.";
+// A cut never lands closer than this share of Seconds per image to the one before, unless a
+// chapter starts there.
+export const cutFloorShare = 0.4;
+// How long a chapter card stays on screen, fades included.
+export const chapterCardSeconds = 2.5;
+export const cutModeLabels = {
+    interval: "Every N seconds",
+    narration: "Follow the narration",
+};
+export const transitionLabels = {
+    cut: "Cut",
+    crossfade: "Crossfade",
+    fadeblack: "Fade through black",
+    slide: "Slide",
+    wipe: "Wipe",
+};
+export const lookLevelLabels = {
+    off: "Off",
+    subtle: "Subtle",
+    strong: "Strong",
+};
+export const colorGradeLabels = {
+    none: "None",
+    warm: "Warm fantasy",
+    cold: "Cold",
+    desaturated: "Desaturated",
+    sepia: "Sepia",
+};
+export const atmosphereLabels = {
+    none: "None",
+    embers: "Embers",
+    dust: "Dust",
+    fog: "Fog",
+};
+export const animateModeLabels = {
+    off: "Off",
+    chapters: "Chapter openers",
+    every: "Every Nth image",
+};
+// The settings a project renders with: its own, or today's behaviour when it has none.
+export function videoEditOf(config) {
+    return config.videoEdit ?? legacyVideoEdit;
+}
+// Following the narration needs narration to follow and a video to cut; otherwise the video
+// falls back to Every N seconds, which is also what its fingerprint says.
+export function usesNarrationCuts(config) {
+    return (videoEditOf(config).cuts === "narration" &&
+        config.sources.audio !== "off" &&
+        config.sources.video === "generate");
+}
+// Chapter cards are placed from the narration's word timing, so they need narration too.
+export function usesChapterCards(config) {
+    return (videoEditOf(config).chapterCards &&
+        config.sources.audio !== "off" &&
+        config.sources.video === "generate");
+}
+// "Chapter openers" needs chapters, which come from the narration's word timing.
+export function usesAnimation(config) {
+    const edit = videoEditOf(config);
+    return (edit.animate !== "off" &&
+        config.sources.video === "generate" &&
+        (edit.animate === "every" || config.sources.audio !== "off"));
+}
+// Whether the render needs the word timing: to cut on sentences, to place chapter cards, or to
+// know which images open a chapter.
+export function editNeedsTiming(config) {
+    return (usesNarrationCuts(config) ||
+        usesChapterCards(config) ||
+        (usesAnimation(config) && videoEditOf(config).animate === "chapters"));
+}
+// The words a problem with one of the settings is said in; shared by admission, Play's draft
+// conversion and Edit project.
+export function videoEditProblems(config) {
+    const edit = config.videoEdit;
+    if (edit === undefined || config.sources.video !== "generate")
+        return [];
+    const problems = [];
+    if (edit.transition !== "cut" &&
+        !(Number.isFinite(edit.transitionSeconds) &&
+            edit.transitionSeconds >= transitionSecondsMin &&
+            edit.transitionSeconds <= transitionSecondsMax))
+        problems.push({
+            field: "transitionSeconds",
+            message: `Choose a transition length between ${String(transitionSecondsMin)} and ${String(transitionSecondsMax)} seconds.`,
+        });
+    if (edit.chapterCards && config.sources.audio === "off")
+        problems.push({
+            field: "chapterCards",
+            message: "Chapter cards are placed from the narration. Turn narration on, or turn chapter cards off.",
+        });
+    if (edit.animate === "chapters" && config.sources.audio === "off")
+        problems.push({
+            field: "animate",
+            message: "Chapter openers are found from the narration. Turn narration on, or choose Every Nth image.",
+        });
+    if (edit.animate === "every" &&
+        !(Number.isInteger(edit.animateEvery) &&
+            edit.animateEvery >= animateEveryMin &&
+            edit.animateEvery <= animateEveryMax))
+        problems.push({
+            field: "animateEvery",
+            message: `Choose every 2nd to every ${String(animateEveryMax)}th image.`,
+        });
+    if (edit.animate !== "off" && edit.animateModel.trim() === "")
+        problems.push({
+            field: "animateModel",
+            message: "Choose an image-to-video model to animate images with, or turn Animate images off.",
+        });
+    return problems;
+}
+// The settings in words, one row each, as the change list and the review summary show them.
+export function videoEditRows(edit) {
+    const nth = (n) => (n === 2 ? "2nd" : n === 3 ? "3rd" : `${String(n)}th`);
+    return [
+        ["Cuts", cutModeLabels[edit.cuts]],
+        [
+            "Transition",
+            edit.transition === "cut"
+                ? transitionLabels.cut
+                : `${transitionLabels[edit.transition]} · ${String(edit.transitionSeconds)} s`,
+        ],
+        ["Vignette", lookLevelLabels[edit.vignette]],
+        ["Film grain", lookLevelLabels[edit.grain]],
+        ["Colour grade", colorGradeLabels[edit.grade]],
+        ["Atmosphere", atmosphereLabels[edit.atmosphere]],
+        ["Chapter cards", edit.chapterCards ? "On" : "Off"],
+        [
+            "Animate images",
+            edit.animate === "off"
+                ? animateModeLabels.off
+                : `${edit.animate === "every" ? `Every ${nth(edit.animateEvery)} image` : animateModeLabels.chapters}${edit.animateModel.trim() === "" ? "" : ` · ${edit.animateModel}`}`,
+        ],
+    ];
+}
+// Which images of the slideshow order (0-based) are animated: every Nth from the first, or
+// the ones that open a chapter. `chapterImages` is the image index each chapter opens on.
+export function animatedImageIndexes(edit, imageCount, chapterImages = []) {
+    if (edit.animate === "off" || imageCount <= 0)
+        return [];
+    if (edit.animate === "every") {
+        const every = Math.max(1, Math.round(edit.animateEvery));
+        return Array.from({ length: imageCount }, (_value, at) => at).filter((at) => at % every === 0);
+    }
+    return [...new Set(chapterImages.filter((at) => at >= 0 && at < imageCount))].toSorted((left, right) => left - right);
+}