mulmocast 1.2.14 → 1.2.16

package/lib/actions/captions.js

@@ -2,7 +2,7 @@ import { mulmoCaptionParamsSchema } from "../types/index.js";
 import { GraphAI, GraphAILogger } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { getHTMLFile, getCaptionImagePath, getOutputStudioFilePath } from "../utils/file.js";
-import { localizedText } from "../utils/utils.js";
+import { localizedText, processLineBreaks } from "../utils/utils.js";
 import { renderHTMLToImage, interpolate } from "../utils/markdown.js";
 import { MulmoStudioContextMethods, MulmoPresentationStyleMethods } from "../methods/index.js";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
@@ -36,7 +36,7 @@ const graph_data = {
                                 }
                                 const text = localizedText(beat, context.multiLingual?.[index], captionParams.lang, context.studio.script.lang);
                                 const htmlData = interpolate(template, {
-                                    caption: text,
+                                    caption: processLineBreaks(text),
                                     width: `${canvasSize.width}`,
                                     height: `${canvasSize.height}`,
                                     styles: captionParams.styles.join(";\n"),

package/lib/actions/image_agents.js

@@ -42,8 +42,9 @@ export const imagePreprocessAgent = async (namedInputs) => {
         returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? lipSyncAgentInfo.defaultModel;
         returnValue.lipSyncFile = moviePaths.lipSyncFile;
         if (context.studio.script.audioParams?.suppressSpeech) {
-            returnValue.startAt = studioBeat?.startAt ?? 0;
-            returnValue.duration = studioBeat?.duration ?? 0;
+            // studio beat may ot have startAt and duration yet, in case of API call from the app.
+            returnValue.startAt = context.studio.script.beats.filter((_, i) => i < index).reduce((acc, curr) => acc + (curr.duration ?? 0), 0);
+            returnValue.duration = beat.duration ?? 0;
             returnValue.lipSyncTrimAudio = true;
             returnValue.bgmFile = MulmoMediaSourceMethods.resolve(context.studio.script.audioParams.bgm, context);
             const folderName = MulmoStudioContextMethods.getFileName(context);

package/lib/actions/images.js

@@ -231,6 +231,7 @@ const beat_graph_data = {
                 return { buffer };
             },
             inputs: {
+                onComplete: [":imageGenerator", ":imagePlugin"],
                 audioFile: ":preprocessor.audioFile",
                 bgmFile: ":preprocessor.bgmFile",
                 startAt: ":preprocessor.startAt",
@@ -240,8 +241,8 @@ const beat_graph_data = {
                     file: ":preprocessor.audioFile",
                     index: ":__mapIndex",
                     id: ":beat.id",
-                    sessionType: "audioTrimmer",
                     mulmoContext: ":context",
+                    // sessionType: undefined, // no need to notify state change
                 },
             },
             defaultValue: {},

package/lib/methods/mulmo_studio_context.d.ts

@@ -14,7 +14,7 @@ export declare const MulmoStudioContextMethods: {
     getFileName(context: MulmoStudioContext): string;
     getCaption(context: MulmoStudioContext): string | undefined;
     setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
-    setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, id: string | undefined, value: boolean): void;
+    setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType | undefined, index: number, id: string | undefined, value: boolean): void;
     needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
     getIntroPadding(context: MulmoStudioContext): number;
 };

package/lib/methods/mulmo_studio_context.js

@@ -53,6 +53,9 @@ export const MulmoStudioContextMethods = {
         notifyStateChange(context, sessionType);
     },
     setBeatSessionState(context, sessionType, index, id, value) {
+        if (!sessionType) {
+            return;
+        }
         const key = beatId(id, index);
         if (value) {
             if (!context.sessionState.inBeatSession[sessionType]) {

package/lib/utils/utils.d.ts

@@ -14,6 +14,7 @@ export declare const llmPair: (_llm?: LLM, _model?: string) => {
 export declare const chunkArray: <T>(array: T[], size?: number) => T[][];
 export declare const isHttp: (fileOrUrl: string) => boolean;
 export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoStudioMultiLingualData, targetLang?: string, defaultLang?: string) => string;
+export declare function processLineBreaks(text: string): string;
 export declare function userAssert(condition: boolean, message: string): asserts condition;
 export declare const settings2GraphAIConfig: (settings?: Record<string, string>, env?: Record<string, string | undefined>) => ConfigDataDictionary<DefaultConfigData>;
 export declare const getExtention: (contentType: string | null, url: string) => string;

package/lib/utils/utils.js

@@ -30,6 +30,9 @@ export const localizedText = (beat, multiLingualData, targetLang, defaultLang) =
     }
     return beat.text;
 };
+export function processLineBreaks(text) {
+    return text.replace(/\n/g, "<br>");
+}
 export function userAssert(condition, message) {
     if (!condition) {
         throw new Error(message);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.2.14",
+  "version": "1.2.16",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",

package/scripts/test/test_captions.json

@@ -4,7 +4,7 @@
   },
   "lang": "en",
   "captionParams": {
-    "lang": "ja",
+    "lang": "en",
     "styles": ["color: yellow"]
   },
   "beats": [
@@ -19,7 +19,7 @@
       }
     },
     {
-      "text": "Hello World. May the force be with you. This is a long caption. This is a really long caption.",
+      "text": "Hello World. May the force be with you.\nThis is a long caption.\nThis is a really long caption.",
       "image": {
         "type": "textSlide",
         "slide": {