mulmocast 2.1.0 → 2.1.2

package/lib/actions/movie.d.ts

@@ -1,7 +1,7 @@
-import { MulmoStudioContext, MulmoBeat, MulmoTransition, MulmoCanvasDimension, MulmoFillOption } from "../types/index.js";
+import { MulmoStudioContext, MulmoBeat, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
 import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 type VideoId = string | undefined;
-export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number) => {
+export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number, filters?: MulmoVideoFilter[]) => {
     videoId: string;
     videoPart: string;
 };

package/lib/actions/movie.js

@@ -1,13 +1,14 @@
 import { GraphAILogger, assert } from "graphai";
-import { mulmoFillOptionSchema } from "../types/index.js";
+import { mulmoFillOptionSchema, } from "../types/index.js";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage, isFile } from "../utils/file.js";
 import { createVideoFileError, createVideoSourceError } from "../utils/error_cause.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput, } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
+import { convertVideoFilterToFFmpeg } from "../utils/video_filter.js";
 // const isMac = process.platform === "darwin";
 const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
-export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed) => {
+export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed, filters) => {
     const videoId = `v${inputIndex}`;
     const videoFilters = [];
     // Handle different media types
@@ -41,6 +42,12 @@ export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOpti
         `pad=${canvasInfo.width}:${canvasInfo.height}:(ow-iw)/2:(oh-ih)/2:color=black`);
     }
     videoFilters.push("setsar=1", "format=yuv420p");
+    // Apply custom video filters if specified
+    if (filters && filters.length > 0) {
+        filters.forEach((filter) => {
+            videoFilters.push(convertVideoFilterToFFmpeg(filter));
+        });
+    }
     return {
         videoId,
         videoPart: `[${inputIndex}:v]` + videoFilters.join(",") + `[${videoId}]`,
@@ -178,10 +185,13 @@ const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transiti
                 // Cannot apply wipe without first frame
                 return prevVideoId;
             }
-            // Use xfade offset instead of trimming to avoid framerate issues
-            // The static frames are created with proper duration, use offset to start transition at the right time
-            const prevBeatDuration = context.studio.beats[beatIndex - 1].duration ?? 0;
-            const xfadeOffset = prevBeatDuration - duration;
+            // Calculate the actual duration of the static frames
+            // The _last frame was generated with the previous beat's actual duration
+            const prevStudioBeat = context.studio.beats[beatIndex - 1];
+            const prevBeatActualDuration = Math.max(prevStudioBeat.duration + getExtraPadding(context, beatIndex - 1), prevStudioBeat.movieDuration ?? 0);
+            // xfade offset must be non-negative and within the first video's duration
+            // Start the transition at the end of the first video minus the transition duration
+            const xfadeOffset = Math.max(0, prevBeatActualDuration - duration);
             // Apply xfade with explicit pixel format
             const xfadeOutputId = `${transitionVideoId}_xfade`;
             ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
@@ -345,7 +355,8 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
             studioBeat.movieFile ||
             MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === "movie");
         const speed = beat.movieParams?.speed ?? 1.0;
-        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed);
+        const filters = beat.movieParams?.filters;
+        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed, filters);
         ffmpegContext.filterComplex.push(videoPart);
         // for transition
         const needFirst = needsFirstFrame[index]; // This beat has slidein

package/lib/methods/mulmo_presentation_style.d.ts

@@ -25,6 +25,152 @@ export declare const MulmoPresentationStyleMethods: {
                 type: "fade" | "slideout_left" | "slideout_right" | "slideout_up" | "slideout_down" | "slidein_left" | "slidein_right" | "slidein_up" | "slidein_down" | "wipeleft" | "wiperight" | "wipeup" | "wipedown" | "wipetl" | "wipetr" | "wipebl" | "wipebr";
                 duration: number;
             } | undefined;
+            filters?: ({
+                type: "mono";
+            } | {
+                type: "sepia";
+            } | {
+                type: "brightness_contrast";
+                brightness: number;
+                contrast: number;
+            } | {
+                type: "hue";
+                hue: number;
+                saturation: number;
+                brightness: number;
+            } | {
+                type: "colorbalance";
+                rs: number;
+                gs: number;
+                bs: number;
+                rm: number;
+                gm: number;
+                bm: number;
+                rh: number;
+                gh: number;
+                bh: number;
+            } | {
+                type: "vibrance";
+                intensity: number;
+            } | {
+                type: "negate";
+                negate_alpha: boolean;
+            } | {
+                type: "colorhold";
+                color: string;
+                similarity: number;
+                blend: number;
+            } | {
+                type: "colorkey";
+                color: string;
+                similarity: number;
+                blend: number;
+            } | {
+                type: "blur";
+                radius: number;
+                power: number;
+            } | {
+                type: "gblur";
+                sigma: number;
+            } | {
+                type: "avgblur";
+                sizeX: number;
+                sizeY: number;
+            } | {
+                type: "unsharp";
+                luma_msize_x: number;
+                luma_msize_y: number;
+                luma_amount: number;
+                chroma_msize_x: number;
+                chroma_msize_y: number;
+                chroma_amount: number;
+            } | {
+                type: "edgedetect";
+                low: number;
+                high: number;
+                mode: "wires" | "colormix" | "canny";
+            } | {
+                type: "sobel";
+                planes: number;
+                scale: number;
+                delta: number;
+            } | {
+                type: "emboss";
+            } | {
+                type: "glitch";
+                intensity: number;
+                style: "blend" | "noise";
+            } | {
+                type: "grain";
+                intensity: number;
+            } | {
+                type: "hflip";
+            } | {
+                type: "vflip";
+            } | {
+                type: "rotate";
+                angle: number;
+                fillcolor: string;
+            } | {
+                type: "transpose";
+                dir: "cclock" | "clock" | "cclock_flip" | "clock_flip";
+            } | {
+                type: "vignette";
+                angle: number;
+                mode: "forward" | "backward";
+                x0?: number | undefined;
+                y0?: number | undefined;
+            } | {
+                type: "fade";
+                mode: "in" | "out";
+                start_frame: number;
+                nb_frames: number;
+                alpha: boolean;
+                color: string;
+            } | {
+                type: "pixelize";
+                width: number;
+                height: number;
+                mode: "avg" | "min" | "max";
+            } | {
+                type: "pseudocolor";
+                preset: "magma" | "inferno" | "plasma" | "viridis" | "turbo" | "cividis" | "range1" | "range2" | "shadows" | "highlights" | "solar" | "nominal" | "preferred" | "total";
+            } | {
+                type: "tmix";
+                frames: number;
+                weights?: string | undefined;
+            } | {
+                type: "lagfun";
+                decay: number;
+                planes: number;
+            } | {
+                type: "threshold";
+                planes: number;
+            } | {
+                type: "elbg";
+                codebook_length: number;
+            } | {
+                type: "lensdistortion";
+                k1: number;
+                k2: number;
+            } | {
+                type: "chromashift";
+                cbh: number;
+                cbv: number;
+                crh: number;
+                crv: number;
+                edge: "smear" | "wrap";
+            } | {
+                type: "deflicker";
+                size: number;
+                mode: "gm" | "am" | "hm" | "qm" | "cm" | "pm" | "median";
+            } | {
+                type: "dctdnoiz";
+                sigma: number;
+            } | {
+                type: "custom";
+                filter: string;
+            })[] | undefined;
             speed?: number | undefined;
         };
         keyName: string;