@nodetool-ai/runtime 0.8.2 → 0.8.3

package/dist/providers/dreamina-provider.js

@@ -76,6 +76,33 @@ export function setDreaminaPageRunner(runner) {
 function runner() {
     return (pageRunner ??= createExtensionRunner());
 }
+/**
+ * A request to the tab that did not answer in time. Its outcome is unknown:
+ * for the generate request, Dreamina may already run (and bill) the job.
+ */
+class DreaminaStallError extends Error {
+}
+/** Whether `error` means a request stalled, so its outcome is unknown. */
+function stalled(error) {
+    return error instanceof DreaminaStallError;
+}
+/**
+ * Evaluate `expression` in the tab, but stop waiting after `timeoutMs`. The
+ * provider keeps its own limit, so a stalled tab or bridge cannot hang a poll.
+ */
+async function evaluateBounded(expression, what, timeoutMs = TAB_CALL_TIMEOUT_MS) {
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new DreaminaStallError(`Dreamina tab did not answer ${what} within ${timeoutMs / 1000}s`)), timeoutMs);
+        timer.unref?.();
+    });
+    try {
+        return await Promise.race([runner().evaluate(expression), timeout]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
 const API_BASE = "https://mweb-api-sg.capcut.com";
 const APP_ID = "513641";
 const APP_VERSION = "8.4.0";
@@ -83,9 +110,18 @@ const DA_VERSION = "3.3.28";
 const WEB_VERSION = "7.5.0";
 const POLL_INTERVAL_MS = 2500;
 const MAX_POLL_FAILURES = 5;
+/** How long a submit cut off by a lost tab has to show up in the history. */
+const UNCONFIRMED_SUBMIT_GRACE_MS = 30_000;
+const GENERATE_PATH = "/mweb/v1/aigc_draft/generate";
 const DEFAULT_TIMEOUT_MS = 5 * 60_000;
 const VIDEO_TIMEOUT_MS = 10 * 60_000;
 const CONFIG_TTL_MS = 5 * 60_000;
+/** One API request in the tab gives up after this, so a stalled request cannot block a poll. */
+const API_TIMEOUT_MS = 30_000;
+/** The whole tab round trip, past the request's own limit, before the provider stops waiting. */
+const TAB_CALL_TIMEOUT_MS = API_TIMEOUT_MS + 15_000;
+/** One image download attempt gives up after this. */
+const DOWNLOAD_TIMEOUT_MS = 60_000;
 const VIDEO_FPS = 24;
 /** Dreamina's `image_ratio` enum, in the order the web app numbers it. */
 const RATIO_TYPES = {
@@ -125,7 +161,33 @@ function signedHeaders(path) {
         appid: APP_ID
     };
 }
+/**
+ * Whether `error` means the Dreamina tab went away: closed, reloaded or
+ * navigated while a command ran. The runner drops its attach on any error, so
+ * the next call attaches afresh.
+ */
+function tabLost(error) {
+    return error instanceof Error && /detached|target[ _]closed|no tab with/i.test(error.message);
+}
+/**
+ * Run `step` and, if the tab is lost, once more on a fresh attach. Only for
+ * steps that cost nothing and can repeat: never the generate request.
+ */
+async function retryOnTabLoss(what, step) {
+    try {
+        return await step();
+    }
+    catch (error) {
+        if (!tabLost(error))
+            throw error;
+        log.warn("Dreamina tab lost, retrying on a fresh attach", { what, error: error.message });
+        return step();
+    }
+}
 async function callApi(path, body) {
+    return path === GENERATE_PATH ? callApiOnce(path, body) : retryOnTabLoss(path, () => callApiOnce(path, body));
+}
+async function callApiOnce(path, body) {
     const query = new URLSearchParams({ aid: APP_ID, device_platform: "web", region: region(), da_version: DA_VERSION, web_version: WEB_VERSION, aigc_features: "app_lip_sync" });
     const request = { url: `${API_BASE}${path}?${query}`, headers: signedHeaders(path), body: JSON.stringify(body) };
     const expression = `(() => { const a = ${JSON.stringify(request)};
@@ -136,10 +198,23 @@ async function callApi(path, body) {
       for (const k of Object.keys(a.headers)) x.setRequestHeader(k, a.headers[k]);
       x.onload = () => { try { resolve(JSON.parse(x.responseText)); } catch { reject(new Error("Dreamina returned a non-JSON response (HTTP " + x.status + ")")); } };
       x.onerror = () => reject(new Error("Dreamina request failed"));
+      x.timeout = ${API_TIMEOUT_MS};
+      x.ontimeout = () => reject(new Error("Dreamina request timed out after ${API_TIMEOUT_MS / 1000}s"));
       x.send(a.body);
     });
   })()`;
-    const response = (await runner().evaluate(expression));
+    const started = Date.now();
+    let response;
+    try {
+        response = (await evaluateBounded(expression, path));
+    }
+    catch (error) {
+        // A request that ran out of time has an unknown outcome, like a lost tab.
+        if (error instanceof Error && /timed out after/.test(error.message))
+            throw new DreaminaStallError(error.message);
+        throw error;
+    }
+    log.debug("Dreamina API call", { path, ret: response.ret, ms: Date.now() - started });
     // `submit_audit_job` answers with an empty `ret`, which carries no error.
     if (response.ret !== "0" && response.ret !== "") {
         const hint = response.ret === "1015" || response.ret === "1014" ? " (is the Dreamina tab logged in?)" : "";
@@ -316,7 +391,10 @@ function imagexHeaders(token, method, query, payload, service = "imagex") {
     };
 }
 /** Upload an image through ImageX the way the web app does and return its store URI. */
-async function uploadImage(bytes) {
+function uploadImage(bytes) {
+    return retryOnTabLoss("image upload", () => uploadImageOnce(bytes));
+}
+async function uploadImageOnce(bytes) {
     const token = await callApi("/mweb/v1/get_upload_token", { scene: 2 });
     const account = (await runner().evaluate(`fetch("https://dreamina.capcut.com/passport/web/account/info/?aid=${APP_ID}&account_sdk_source=web&language=en", { credentials: "include" }).then((r) => r.json())`));
     const userId = account.data?.user_id_str;
@@ -360,7 +438,10 @@ async function uploadImage(bytes) {
     return { uri: result.ImageUri, width: result.ImageWidth, height: result.ImageHeight };
 }
 /** Upload a video or audio file through VOD the way the web app does and return its vid and size. VOD files audio as `video`. */
-async function uploadVideo(bytes) {
+function uploadVideo(bytes) {
+    return retryOnTabLoss("video upload", () => uploadVideoOnce(bytes));
+}
+async function uploadVideoOnce(bytes) {
     const token = await callApi("/mweb/v1/get_upload_token", { scene: 1 });
     const account = (await runner().evaluate(`fetch("https://dreamina.capcut.com/passport/web/account/info/?aid=${APP_ID}&account_sdk_source=web&language=en", { credentials: "include" }).then((r) => r.json())`));
     const userId = account.data?.user_id_str;
@@ -421,23 +502,22 @@ async function auditReferences(model, images, videos = [], audios = []) {
 }
 /**
  * Split a prompt on `[Image N]`, `[Video N]` and `[Audio N]` markers into the
- * text, image, video and audio segments the site's editor produces. A
- * material's index counts images first, then videos, then audios. Without
- * markers every image, video and audio leads and the prompt follows.
+ * segments the site's editor produces: every image, video and audio leads,
+ * then the prompt follows with each marker as an inline mention of its
+ * material. A material's index counts images first, then videos, then audios.
  */
 function referenceMeta(prompt, imageCount, videoCount = 0, audioCount = 0) {
     const counts = { image: imageCount, video: videoCount, audio: audioCount };
     const offsets = { image: 0, video: imageCount, audio: imageCount + videoCount };
-    const media = (kind, index) => ({ meta_type: kind, text: "", material_ref: { material_idx: offsets[kind] + index } });
-    if (!/\[(Image|Video|Audio) (\d+)\]/.test(prompt)) {
-        return [
-            ...Array.from({ length: imageCount }, (_, i) => media("image", i)),
-            ...Array.from({ length: videoCount }, (_, i) => media("video", i)),
-            ...Array.from({ length: audioCount }, (_, i) => media("audio", i)),
-            ...(prompt ? [{ meta_type: "text", text: prompt }] : [])
-        ];
-    }
-    const parts = [];
+    const media = (kind, index) => ({ type: "", id: randomUUID(), meta_type: kind, text: "", material_ref: { type: "", id: randomUUID(), material_idx: offsets[kind] + index } });
+    const text = (value) => ({ type: "", id: randomUUID(), meta_type: "text", text: value });
+    // Captured from the web app: the materials always lead, in order, even when
+    // the prompt mentions them again inline.
+    const parts = [
+        ...Array.from({ length: imageCount }, (_, i) => media("image", i)),
+        ...Array.from({ length: videoCount }, (_, i) => media("video", i)),
+        ...Array.from({ length: audioCount }, (_, i) => media("audio", i))
+    ];
     let last = 0;
     for (const match of prompt.matchAll(/\[(Image|Video|Audio) (\d+)\]/g)) {
         const kind = match[1].toLowerCase();
@@ -445,22 +525,27 @@ function referenceMeta(prompt, imageCount, videoCount = 0, audioCount = 0) {
         if (index < 0 || index >= counts[kind])
             throw new Error(`The prompt refers to [${match[1]} ${match[2]}] but only ${counts[kind]} reference ${kind}(s) were given`);
         if (match.index > last)
-            parts.push({ meta_type: "text", text: prompt.slice(last, match.index) });
+            parts.push(text(prompt.slice(last, match.index)));
         parts.push(media(kind, index));
         last = match.index + match[0].length;
     }
     if (last < prompt.length)
-        parts.push({ meta_type: "text", text: prompt.slice(last) });
+        parts.push(text(prompt.slice(last)));
     return parts;
 }
 function videoDraftFor(model, prompt, seed, ratio, resolution, durationMs, submitId, references = [], frames, videos = [], audios = []) {
     const id = () => randomUUID();
+    // Seedance 2.5's 480p preview model renders in Dreamina's draft mode. The web
+    // app marks the input as a draft on the current schema version, and the API
+    // answers "invalid parameter" (ret 1000) without it.
+    const draftMode = model.endsWith("_draft");
     const frameImage = (image) => ({ type: "image", source_from: "upload", platform_type: 1, name: "", image_uri: image.uri, aigc_image: {}, width: image.width, height: image.height, format: "", uri: image.uri });
     const videoInput = {
         type: "",
         id: id(),
-        min_version: references.length + videos.length + audios.length > 0 ? "3.3.9" : "3.0.5",
+        min_version: draftMode ? DA_VERSION : references.length + videos.length + audios.length > 0 ? "3.3.9" : "3.0.5",
         prompt: references.length + videos.length + audios.length > 0 ? "" : prompt,
+        ...(draftMode && { is_draft_mode: true }),
         video_mode: 2,
         fps: VIDEO_FPS,
         duration_ms: durationMs,
@@ -477,12 +562,18 @@ function videoDraftFor(model, prompt, seed, ratio, resolution, durationMs, submi
     if (references.length + videos.length + audios.length > 0) {
         videoInput.unified_edit_input = {
             material_list: [...references.map((image) => ({
+                    type: "",
+                    id: id(),
                     material_type: "image",
                     image_info: { type: "image", source_from: "upload", platform_type: 1, name: "", image_uri: image.uri, aigc_image: {}, width: image.width, height: image.height, format: "", title: "", uri: image.uri }
                 })), ...videos.map((video) => ({
+                    type: "",
+                    id: id(),
                     material_type: "video",
                     video_info: { type: "video", source_from: "upload", name: "", vid: video.vid, fps: 0, width: video.width, height: video.height, duration: video.durationMs }
                 })), ...audios.map((audio) => ({
+                    type: "",
+                    id: id(),
                     material_type: "audio",
                     audio_info: { type: "audio", source_from: "upload", vid: audio.vid, duration: audio.durationMs, name: "" }
                 }))],
@@ -516,13 +607,37 @@ function videoDraftFor(model, prompt, seed, ratio, resolution, durationMs, submi
         },
         process_type: 1
     };
-    return JSON.stringify({ type: "draft", id: id(), min_version: "3.0.5", min_features: [], is_from_tsn: true, version: DA_VERSION, main_component_id: component.id, component_list: [component] });
+    // Captured from the web app: reference input declares the unified-edit
+    // feature on the 3.3.9 schema, and every Seedance 2.5 model the 2.5 result
+    // action.
+    const unifiedEdit = references.length + videos.length + audios.length > 0;
+    const minFeatures = [
+        ...(unifiedEdit ? ["AIGC_Video_UnifiedEdit"] : []),
+        ...(model.includes("seedance_45") ? ["AIGC_Video_Seedance25ResultAction"] : [])
+    ];
+    const minVersion = draftMode ? DA_VERSION : unifiedEdit ? "3.3.9" : "3.0.5";
+    return JSON.stringify({ type: "draft", id: id(), min_version: minVersion, min_features: minFeatures, is_from_tsn: true, version: DA_VERSION, main_component_id: component.id, component_list: [component] });
 }
 /** Submit a generation and poll until it finishes, then return the finished record. */
 async function submitAndAwait(body, submitId, timeoutMs, signal) {
-    await callApi("/mweb/v1/aigc_draft/generate", body);
-    const deadline = Date.now() + timeoutMs;
+    // A lost tab leaves the submit's outcome unknown: Dreamina may already run
+    // (and bill) the job. Sending it again could pay twice, so the job is looked
+    // up by its submit id instead, and only a job that never appears fails.
+    let unconfirmed = null;
+    try {
+        await callApi(GENERATE_PATH, body);
+    }
+    catch (error) {
+        if (!tabLost(error) && !stalled(error))
+            throw error;
+        unconfirmed = error;
+        log.warn("Dreamina submit has no answer, looking the job up by id", { submitId, error: unconfirmed.message });
+    }
+    const started = Date.now();
+    const deadline = started + timeoutMs;
+    log.info("Dreamina generation submitted", { submitId, timeoutMs, confirmed: !unconfirmed });
     let pollFailures = 0;
+    let lastStatus;
     for (;;) {
         await sleep(POLL_INTERVAL_MS, signal);
         let history;
@@ -532,28 +647,84 @@ async function submitAndAwait(body, submitId, timeoutMs, signal) {
         catch (error) {
             // Reading the history is idempotent, and Dreamina answers it with a
             // transient error while a fresh task registers. The submit is never retried.
+            log.warn("Dreamina poll failed", { submitId, pollFailures: pollFailures + 1, error: error instanceof Error ? error.message : String(error) });
             if (++pollFailures > MAX_POLL_FAILURES)
                 throw error;
+            if (Date.now() > deadline) {
+                throw new Error(`Dreamina generation ${submitId} timed out after ${timeoutMs / 1000}s (last task status ${lastStatus ?? "none"}, last poll failed: ${error instanceof Error ? error.message : String(error)})`);
+            }
             continue;
         }
         pollFailures = 0;
         const record = history[submitId];
+        if (record) {
+            unconfirmed = null;
+        }
+        else if (unconfirmed && Date.now() - started > UNCONFIRMED_SUBMIT_GRACE_MS) {
+            throw new Error(`Dreamina generation ${submitId} did not reach Dreamina before the tab was lost (${unconfirmed.message}). Keep the Dreamina tab open and try again`);
+        }
         const status = record?.task?.status;
+        if (status !== lastStatus) {
+            log.info("Dreamina task status", { submitId, status, hasRecord: record !== undefined, elapsedMs: Date.now() - started });
+            lastStatus = status;
+        }
         if (status === TASK_DONE)
             return record;
         if (status === TASK_FAILED) {
             throw new Error(`Dreamina generation failed: ${record?.fail_msg || "unknown reason"} (fail_code ${record?.fail_code ?? "unknown"})`);
         }
-        if (Date.now() > deadline)
-            throw new Error(`Dreamina generation timed out after ${timeoutMs / 1000}s`);
+        if (Date.now() > deadline) {
+            log.warn("Dreamina generation timed out", { submitId, lastStatus, record: JSON.stringify(record ?? null).slice(0, 2000) });
+            throw new Error(`Dreamina generation ${submitId} timed out after ${timeoutMs / 1000}s (last task status ${lastStatus ?? "none"})`);
+        }
     }
 }
-async function download(url, signal) {
-    const response = await fetchExternalMedia(url, signal ? { signal } : undefined);
+async function downloadOnce(url, signal) {
+    const limit = AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS);
+    const combined = signal ? AbortSignal.any([signal, limit]) : limit;
+    const response = await fetchExternalMedia(url, { signal: combined });
     if (!response.ok)
         throw new Error(`Dreamina download failed: HTTP ${response.status}`);
     return new Uint8Array(await response.arrayBuffer());
 }
+/** Fetch `url` inside the Dreamina tab, which can always reach Dreamina's CDN. */
+async function downloadViaTab(url) {
+    const expression = `(async () => {
+    const response = await fetch(${JSON.stringify(url)}, { credentials: "omit", signal: AbortSignal.timeout(${DOWNLOAD_TIMEOUT_MS}) });
+    if (!response.ok) throw new Error("Dreamina download failed in the tab: HTTP " + response.status);
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    let binary = "";
+    for (let i = 0; i < bytes.length; i += 0x8000) binary += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));
+    return btoa(binary);
+  })()`;
+    const base64 = (await evaluateBounded(expression, "the download", DOWNLOAD_TIMEOUT_MS + 15_000));
+    return new Uint8Array(Buffer.from(base64, "base64"));
+}
+/**
+ * Download a finished result. The job is paid for by now, so a slow or broken
+ * connection gets a second attempt, then the tab's own connection.
+ */
+async function download(url, signal) {
+    let lastError;
+    for (let attempt = 1; attempt <= 2; attempt++) {
+        try {
+            return await downloadOnce(url, signal);
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            lastError = error;
+            log.warn("Dreamina download failed", { attempt, error: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    try {
+        return await downloadViaTab(url);
+    }
+    catch (error) {
+        log.warn("Dreamina download through the tab failed", { error: error instanceof Error ? error.message : String(error) });
+        throw new Error(`Dreamina download failed: ${lastError instanceof Error ? lastError.message : String(lastError)}. The result is still in your Dreamina history`);
+    }
+}
 function videoEnum(model, key) {
     const option = model.options?.find((o) => o.key === key)?.enum_val;
     const values = (option?.string_value ?? option?.int_value ?? []);
@@ -563,6 +734,34 @@ function videoEnum(model, key) {
 function takesReferences(model) {
     return Boolean(model.options?.some((o) => o.key === "unified_edit")) || videoEnum(model, "input_media_type").values.includes("unified_edit");
 }
+/**
+ * Turn a start frame, an end frame and named references into omni-reference
+ * input: every image becomes a reference, and the prompt opens with one
+ * mention per image (`[Image 2] is Mara.`), which the draft turns into inline
+ * image mentions. The caller's own references keep their numbers, so markers
+ * already in the prompt still point at them. Named references past the
+ * model's limit drop.
+ */
+function omniInputs(prompt, inputs) {
+    const references = [...inputs.references];
+    const mentions = [];
+    const mention = (image, says) => {
+        references.push(image);
+        mentions.push(says(`[Image ${references.length}]`));
+    };
+    if (inputs.first)
+        mention(inputs.first, (m) => `${m} is the first frame.`);
+    if (inputs.last)
+        mention(inputs.last, (m) => `${m} is the last frame.`);
+    for (const named of inputs.named ?? []) {
+        if (references.length >= MAX_REFERENCE_IMAGES) {
+            log.warn("Dreamina reference limit reached, dropping a named reference", { name: named.name, limit: MAX_REFERENCE_IMAGES });
+            continue;
+        }
+        mention(named.image, (m) => (named.name ? `${m} is ${named.name}.` : `${m} is a reference.`));
+    }
+    return { prompt: [mentions.join(" "), prompt].filter((part) => part.trim().length > 0).join("\n\n"), references };
+}
 function videoInputTypes(model) {
     return videoEnum(model, "input_media_type").values.map(String);
 }
@@ -688,15 +887,17 @@ export class DreaminaProvider extends BaseProvider {
         return [...DREAMINA_VIDEO_MODELS];
     }
     async textToVideo(params) {
-        return this.generateVideo(params, { references: [] });
+        return this.generateVideo(params, { references: [], named: params.references ?? [] });
     }
     /**
-     * Image-to-video through Dreamina's "first and last frames" mode. The image
-     * is the first frame. `endImage` sets the last frame on models that list
-     * `end_frame`. The prompt describes the motion and may stay empty.
+     * Image-to-video. Models that take references (Seedance 2.x) always render
+     * in "omni reference" mode: the image, `endImage` and `references` all go in
+     * as references, and the prompt names each one, starting with
+     * `[Image 1] is the first frame.` Older models use "first and last frames"
+     * mode, where `endImage` needs `end_frame` and references are ignored.
      */
     async imageToVideo(image, params) {
-        return this.generateVideo({ ...params, prompt: params.prompt ?? "" }, { references: [], first: image, last: params.endImage ?? undefined });
+        return this.generateVideo({ ...params, prompt: params.prompt ?? "" }, { references: [], first: image, last: params.endImage ?? undefined, named: params.references ?? [] });
     }
     /**
      * Reference-to-video through Dreamina's "omni reference" mode: the images
@@ -716,7 +917,7 @@ export class DreaminaProvider extends BaseProvider {
             throw new Error(`Dreamina takes at most ${MAX_REFERENCE_IMAGES} reference images, not ${inputs.images.length}`);
         if (inputs.videos.length > MAX_REFERENCE_VIDEOS)
             throw new Error(`Dreamina takes at most ${MAX_REFERENCE_VIDEOS} reference videos, not ${inputs.videos.length}`);
-        return this.generateVideo(params, { references: inputs.images, videos: inputs.videos, audios });
+        return this.generateVideo(params, { references: inputs.images, videos: inputs.videos, audios, named: params.references ?? undefined });
     }
     async getAvailableMusicModels() {
         return this.music.models();
@@ -725,10 +926,20 @@ export class DreaminaProvider extends BaseProvider {
     async textToMusic(params) {
         return this.music.generate(params);
     }
-    async generateVideo(params, inputs) {
+    async generateVideo(params, requested) {
         const config = (await this.videoConfigs()).find((m) => m.model_req_key === params.model.id);
         if (!config)
             throw new Error(`Unknown Dreamina video model: ${params.model.id}`);
+        // Models that take references always render in omni-reference mode, so a
+        // start frame sits beside the entity images instead of excluding them.
+        const omni = takesReferences(config) && Boolean(requested.first || requested.named?.length)
+            ? omniInputs(params.prompt, requested)
+            : null;
+        if (!omni && requested.named?.length) {
+            log.debug("Dreamina model takes no references, ignoring them", { model: config.model_req_key, count: requested.named.length });
+        }
+        const inputs = omni ? { ...requested, references: omni.references, first: undefined, last: undefined } : requested;
+        const prompt = omni?.prompt ?? params.prompt;
         const resolutions = videoEnum(config, "resolution");
         const resolution = params.resolution
             ? resolutions.values.map(String).find((r) => r.toLowerCase() === params.resolution.toLowerCase())
@@ -777,7 +988,7 @@ export class DreaminaProvider extends BaseProvider {
             extend: { root_model: config.model_req_key },
             submit_id: submitId,
             metrics_extra: JSON.stringify({ isDefaultSeed: 1, originSubmitId: submitId, isRegenerate: false }),
-            draft_content: videoDraftFor(config.model_req_key, params.prompt, seed, ratio, resolution, seconds * 1000, submitId, references, bounds, videos, audios),
+            draft_content: videoDraftFor(config.model_req_key, prompt, seed, ratio, resolution, seconds * 1000, submitId, references, bounds, videos, audios),
             http_common_info: { aid: Number(APP_ID) }
         }, submitId, (params.timeoutSeconds ?? VIDEO_TIMEOUT_MS / 1000) * 1000, params.signal);
         const url = record.item_list?.[0]?.video?.transcoded_video?.origin?.video_url;