@nodetool-ai/agents 0.8.2 → 0.8.3

package/dist/capabilities/storyboards.js

@@ -20,9 +20,9 @@ import { z } from "zod";
 import { SpanStatusCode } from "@opentelemetry/api";
 import { mp4DurationSeconds, withSpan } from "@nodetool-ai/runtime";
 import { storyboardMotionDesign, storyboardShotGraphics } from "@nodetool-ai/protocol/api-schemas/storyboards.js";
-import { assertProductionGenerationAllowed, isFullResourceId, isShortResourceId, productionRequirement } from "@nodetool-ai/protocol";
+import { assertProductionGenerationAllowed, compileOneTake, isFullResourceId, isShortResourceId, productionRequirement } from "@nodetool-ai/protocol";
 import { stampScriptStoryboardId } from "./script-link.js";
-import { listStoryboardsSpec, createStoryboardSpec, getStoryboardSpec, renderStoryboardStillsSpec, renderStoryboardClipsSpec, reviseStoryboardClipSpec, assembleStoryboardTimelineSpec, editStoryboardSpec, directStoryboardSpec, extractScriptFromStoryboardSpec, deleteStoryboardSpec } from "./storyboards.specs.js";
+import { listStoryboardsSpec, createStoryboardSpec, getStoryboardSpec, renderStoryboardStillsSpec, renderStoryboardClipsSpec, reviseStoryboardClipSpec, assembleStoryboardTimelineSpec, deriveShotFromClipSpec, editStoryboardSpec, directStoryboardSpec, extractScriptFromStoryboardSpec, deleteStoryboardSpec, getStoryboardOneTakeSpec, updateStoryboardOneTakeSpec } from "./storyboards.specs.js";
 import { measureStoryboardSources } from "./storyboard-measured-sources.js";
 import { clampConcurrency } from "./concurrency.js";
 import { isNonBlankString, isNumber, isRecord, isString } from "../utils/type-guards.js";
@@ -439,6 +439,8 @@ const getStoryboard = {
                 action: shot.action,
                 camera: shot.camera,
                 motion: shot.motion,
+                end_state: shot.end_state,
+                sound: shot.sound,
                 graphics: shot.graphics,
                 production: shot.production,
                 duration_seconds: shot.duration_seconds,
@@ -1291,6 +1293,8 @@ const SHOT_EDIT_FIELDS = new Set([
     "slug",
     "camera",
     "motion",
+    "end_state",
+    "sound",
     "graphics",
     "production",
     "dialogue",
@@ -1436,6 +1440,17 @@ function applyShotFields(shot, args, shots = []) {
         next.camera = args["camera"];
     if (args["motion"] !== undefined)
         next.motion = String(args["motion"]);
+    for (const field of ["end_state", "sound"]) {
+        const value = args[field];
+        if (value === undefined)
+            continue;
+        if (value === null || String(value).trim() === "") {
+            delete next[field];
+        }
+        else {
+            next[field] = String(value);
+        }
+    }
     if (args["graphics"] !== undefined) {
         if (args["graphics"] === null) {
             delete next.graphics;
@@ -2131,6 +2146,212 @@ const editStoryboard = {
     }
 };
 // ---------------------------------------------------------------------------
+// derive_shot_from_clip
+// ---------------------------------------------------------------------------
+const DEFAULT_DERIVED_FRAMES = 3;
+const formatSeconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
+/**
+ * The headless twin of the editor's Derived shot section
+ * (`useDeriveShotFromClip`): frames → assets → a shot after the source shot
+ * → a clip following it on a new track above the source clip. The board is
+ * written before the timeline, so a timeline conflict leaves the shot on the
+ * board and says so.
+ */
+const deriveShotFromClip = {
+    spec: deriveShotFromClipSpec,
+    impl: async (run, params) => {
+        const timelineId = params["timeline_id"];
+        if (!isNonBlankString(timelineId)) {
+            return {
+                error: "timeline_id is required (use list_timelines to find one)."
+            };
+        }
+        if (!isNonBlankString(params["clip_id"])) {
+            return { error: "clip_id is required (a clip id or name)." };
+        }
+        const { MAX_DERIVED_FRAMES } = await import("@nodetool-ai/timeline");
+        const frameCount = params["frame_count"] === undefined
+            ? DEFAULT_DERIVED_FRAMES
+            : params["frame_count"];
+        if (!isNumber(frameCount) ||
+            !Number.isInteger(frameCount) ||
+            frameCount < 1 ||
+            frameCount > MAX_DERIVED_FRAMES) {
+            return {
+                error: `frame_count must be a whole number from 1 to ${MAX_DERIVED_FRAMES}.`
+            };
+        }
+        const actionParam = params["action"];
+        if (actionParam !== undefined && !isString(actionParam)) {
+            return { error: "action must be a string." };
+        }
+        const { TimelineSequence, Storyboard } = await import("@nodetool-ai/models");
+        const sequence = await TimelineSequence.findById(timelineId);
+        if (!sequence || sequence.user_id !== run.context.userId) {
+            return { error: `Timeline ${timelineId} was not found.` };
+        }
+        const { findDocumentClip, captureDerivedFrames, placeDerivedShotClip } = await import("./derive-shot.js");
+        const source = findDocumentClip(sequence.toDocument().clips, params["clip_id"]);
+        if (!source) {
+            return {
+                error: `No clip matching "${params["clip_id"]}" in timeline ${timelineId}.`
+            };
+        }
+        const boardId = source.storyboardBoardId;
+        const sourceShotId = source.storyboardShotId;
+        if (!boardId || !sourceShotId) {
+            return {
+                error: `Clip "${source.name}" is not linked to a storyboard shot.`
+            };
+        }
+        if (source.mediaType !== "video" || !source.currentAssetId) {
+            return {
+                error: "Reference frames come from a video clip with footage."
+            };
+        }
+        const board = await loadBoard(run, boardId);
+        if (isError(board))
+            return board;
+        const sourceShot = board.doc.shots.find((shot) => shot.id === sourceShotId);
+        if (!sourceShot) {
+            return {
+                error: `The shot clip "${source.name}" follows is no longer on its board.`
+            };
+        }
+        const label = sourceShot.slug ?? `Shot ${sourceShot.index + 1}`;
+        let frames;
+        try {
+            frames = await captureDerivedFrames(run.context, source, frameCount);
+        }
+        catch (error) {
+            return { error: error instanceof Error ? error.message : String(error) };
+        }
+        const bindings = [];
+        for (const [index, frame] of frames.entries()) {
+            const created = await run.context.createAsset({
+                name: `${label}-frame-${index + 1}.png`,
+                contentType: "image/png",
+                content: frame.png
+            });
+            const assetId = isRecord(created) && isString(created["id"]) ? created["id"] : null;
+            if (!assetId) {
+                return { error: "A reference frame was saved without an asset id." };
+            }
+            bindings.push({
+                kind: "style",
+                asset_id: assetId,
+                label: `${label} at ${formatSeconds(frame.sourceTimeMs)}`
+            });
+        }
+        const action = actionParam?.trim() || sourceShot.action;
+        const shotArgs = {
+            after_shot_id: sourceShot.id,
+            action,
+            slug: `${label} (derived)`,
+            duration_seconds: source.durationMs / 1000,
+            duration_source: "manual",
+            render_mode: "reference",
+            production: {
+                schema_version: 1,
+                speech_mode: "none",
+                requested_take_count: 1,
+                reference_bindings: bindings
+            }
+        };
+        if (sourceShot.camera !== undefined)
+            shotArgs["camera"] = sourceShot.camera;
+        if (sourceShot.motion !== undefined)
+            shotArgs["motion"] = sourceShot.motion;
+        if (sourceShot.entity_ids !== undefined) {
+            shotArgs["entity_ids"] = sourceShot.entity_ids;
+        }
+        if (sourceShot.location_id !== undefined) {
+            shotArgs["location_id"] = sourceShot.location_id;
+        }
+        let shot;
+        for (let attempt = 0; attempt < CAS_ATTEMPTS && !shot; attempt++) {
+            const current = await loadBoard(run, boardId);
+            if (isError(current))
+                return current;
+            const { row, doc } = current;
+            let shotId;
+            try {
+                const added = applyBoardOp(doc, { op: "add_shot", args: shotArgs });
+                shotId = added.id;
+            }
+            catch (error) {
+                return {
+                    error: error instanceof Error ? error.message : String(error)
+                };
+            }
+            const index = doc.shots.findIndex((candidate) => candidate.id === shotId);
+            if (sourceShot.clip_model !== undefined) {
+                doc.shots[index] = {
+                    ...doc.shots[index],
+                    clip_model: sourceShot.clip_model
+                };
+            }
+            if (doc.screenplay) {
+                doc.screenplay.shots = doc.shots;
+            }
+            const saved = await Storyboard.updateFieldsIfUnchanged(row.id, row.updated_at, { document: JSON.stringify(doc) }, { ops: [{ tool: "add_shot", input: { ...shotArgs, id: shotId } }] });
+            if (saved)
+                shot = doc.shots[index];
+        }
+        if (!shot) {
+            return {
+                error: `Storyboard ${boardId} is being modified concurrently; nothing was saved. Retry the call.`
+            };
+        }
+        for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
+            const current = await TimelineSequence.findById(timelineId);
+            if (!current)
+                break;
+            const document = current.toDocument();
+            const latest = document.clips.find((clip) => clip.id === source.id);
+            if (!latest)
+                break;
+            const placed = placeDerivedShotClip(document, latest, {
+                boardId,
+                shotId: shot.id,
+                name: shot.slug ?? "Derived shot"
+            });
+            const saved = await TimelineSequence.updateDocumentIfUnchanged(current.id, current.updated_at, { ...document, tracks: placed.tracks, clips: placed.clips }, {
+                ops: [
+                    {
+                        tool: "ui_timeline_add_track",
+                        input: { id: placed.trackId, type: "video" }
+                    },
+                    {
+                        tool: "ui_timeline_add_media_clip",
+                        input: { id: placed.clipId, track_id: placed.trackId }
+                    }
+                ]
+            });
+            if (saved) {
+                return {
+                    storyboard_id: boardId,
+                    shot_id: shot.id,
+                    shot_index: shot.index,
+                    slug: shot.slug,
+                    reference_asset_ids: bindings.map((binding) => binding.asset_id),
+                    timeline_id: timelineId,
+                    track_id: placed.trackId,
+                    clip_id: placed.clipId,
+                    next: "Render the shot with render_storyboard_clips, targets " +
+                        `["${shot.id}"]. The clip plays the take once it lands.`
+                };
+            }
+        }
+        return {
+            error: `Shot ${shot.id} was added to storyboard ${boardId}, but timeline ` +
+                `${timelineId} changed or its source clip was removed, so no clip ` +
+                "was placed. Add a clip for the shot with edit_timeline, or remove " +
+                "the shot with edit_storyboard."
+        };
+    }
+};
+// ---------------------------------------------------------------------------
 // direct_storyboard
 // ---------------------------------------------------------------------------
 const DEFAULT_DIRECTED_SHOTS = 6;
@@ -2443,6 +2664,244 @@ const deleteStoryboard = {
             : { error: `Storyboard ${id} was not found, or it is not yours.` };
     }
 };
+// ---------------------------------------------------------------------------
+// get_storyboard_one_take / update_storyboard_one_take
+// ---------------------------------------------------------------------------
+/** Above this the providers that take one continuous clip refuse or truncate. */
+const ONE_TAKE_MAX_SECONDS = 30;
+/** The most reference images a one-take provider accepts. */
+const ONE_TAKE_MAX_IMAGES = 9;
+const oneTakeUpdateInput = z.object({
+    prompt: z.string().optional(),
+    duration_seconds: z.number().positive().nullable().optional(),
+    aspect_ratio: z.string().trim().min(1).nullable().optional(),
+    resolution: z.string().trim().min(1).nullable().optional(),
+    model: z
+        .object({
+        provider: z.string().trim().min(1),
+        id: z.string().trim().min(1),
+        name: z.string().optional()
+    })
+        .nullable()
+        .optional(),
+    shots: z
+        .array(z.object({
+        shot_id: z.string().trim().min(1),
+        end_state: z.string().nullable().optional(),
+        sound: z.string().nullable().optional()
+    }))
+        .optional()
+});
+/** The one-take render settings a call may set; null clears one. */
+const ONE_TAKE_SETTINGS = [
+    "duration_seconds",
+    "aspect_ratio",
+    "resolution",
+    "model"
+];
+/** The board's `videoModel` as a model ref, or null when it names none. */
+function boardVideoModel(value) {
+    if (!value || typeof value !== "object")
+        return null;
+    const record = value;
+    const id = record["id"];
+    const provider = record["provider"];
+    if (typeof id !== "string" || id === "" || typeof provider !== "string" || provider === "") {
+        return null;
+    }
+    const model = {
+        id,
+        provider: provider
+    };
+    const name = record["name"];
+    if (typeof name === "string") {
+        model.name = name;
+    }
+    return model;
+}
+/** The shape both one-take capabilities return. */
+function oneTakeSummary(row, doc) {
+    const compiled = compileOneTake({
+        shots: doc.shots,
+        oneTake: doc.one_take
+    });
+    const direction = doc.one_take;
+    const settings = {
+        duration_seconds: direction?.duration_seconds ?? null,
+        aspect_ratio: direction?.aspect_ratio ?? null,
+        resolution: direction?.resolution ?? null,
+        model: direction?.model ?? null
+    };
+    const effective = {
+        duration_seconds: compiled.duration_seconds,
+        shot_total_seconds: compiled.shot_total_seconds,
+        aspect_ratio: settings.aspect_ratio ?? doc.aspectRatio ?? null,
+        resolution: settings.resolution,
+        model: settings.model ?? boardVideoModel(doc.videoModel)
+    };
+    const warnings = [];
+    if (!effective.model) {
+        warnings.push("No video model is chosen: set `model` here or the board's video model.");
+    }
+    if (compiled.duration_seconds > ONE_TAKE_MAX_SECONDS) {
+        warnings.push(`The take runs ${compiled.duration_seconds}s, over the ${ONE_TAKE_MAX_SECONDS}s one-take limit.`);
+    }
+    if (compiled.references.length > ONE_TAKE_MAX_IMAGES) {
+        warnings.push(`${compiled.references.length} reference images; one-take models accept at most ${ONE_TAKE_MAX_IMAGES}.`);
+    }
+    const byId = new Map(doc.shots.map((shot) => [shot.id, shot]));
+    const withoutStill = compiled.steps
+        .map((step) => byId.get(step.shot_id))
+        .filter((shot) => !!shot && !shot.keyframe?.asset_id);
+    if (withoutStill.length > 0) {
+        warnings.push(`Shots without a still send no image: ${withoutStill
+            .map((shot) => shot.slug?.trim() || shot.id)
+            .join(", ")}.`);
+    }
+    return {
+        storyboard_id: row.id,
+        revision: row.revision,
+        updated_at: row.updated_at,
+        stored: doc.one_take !== undefined,
+        prompt: doc.one_take?.prompt ?? "",
+        compiled: compiled.compiled,
+        full_prompt: compiled.prompt,
+        references: compiled.references.map((reference, index) => ({
+            image_number: index + 1,
+            marker: `[Image ${index + 1}]`,
+            ...reference
+        })),
+        steps: compiled.steps.map((step) => {
+            const shot = byId.get(step.shot_id);
+            return {
+                shot_id: step.shot_id,
+                index: shot?.index,
+                slug: shot?.slug,
+                start_seconds: step.start_seconds,
+                end_seconds: step.end_seconds,
+                end_state: shot?.end_state ?? null,
+                sound: shot?.sound ?? null
+            };
+        }),
+        duration_seconds: compiled.duration_seconds,
+        settings,
+        effective,
+        warnings
+    };
+}
+const getStoryboardOneTake = {
+    spec: getStoryboardOneTakeSpec,
+    impl: async (run, params) => {
+        const board = await loadBoard(run, params["storyboard_id"]);
+        if (isError(board))
+            return board;
+        return oneTakeSummary(board.row, board.doc);
+    }
+};
+const updateStoryboardOneTake = {
+    spec: updateStoryboardOneTakeSpec,
+    impl: async (run, params) => {
+        const parsed = oneTakeUpdateInput.safeParse({
+            prompt: params["prompt"],
+            duration_seconds: params["duration_seconds"],
+            aspect_ratio: params["aspect_ratio"],
+            resolution: params["resolution"],
+            model: params["model"],
+            shots: params["shots"]
+        });
+        if (!parsed.success) {
+            return {
+                error: parsed.error.issues
+                    .map((issue) => `${issue.path.join(".") || "input"}: ${issue.message}`)
+                    .join("; ")
+            };
+        }
+        const { prompt, shots: shotPatches } = parsed.data;
+        const settingsGiven = ONE_TAKE_SETTINGS.filter((key) => parsed.data[key] !== undefined);
+        if (prompt === undefined && settingsGiven.length === 0 && !shotPatches) {
+            return {
+                error: "Pass prompt, duration_seconds, aspect_ratio, resolution, model or shots to update."
+            };
+        }
+        const { Storyboard } = await import("@nodetool-ai/models");
+        for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
+            const board = await loadBoard(run, params["storyboard_id"]);
+            if (isError(board))
+                return board;
+            const { row } = board;
+            const doc = board.doc;
+            if (params["expected_revision"] !== undefined &&
+                params["expected_revision"] !== row.revision) {
+                return {
+                    error: "Storyboard revision conflict. Refresh the board before editing."
+                };
+            }
+            const ops = [];
+            if (prompt !== undefined || settingsGiven.length > 0) {
+                const next = {
+                    ...doc.one_take,
+                    prompt: prompt ?? doc.one_take?.prompt ?? ""
+                };
+                const { duration_seconds, aspect_ratio, resolution, model } = parsed.data;
+                if (duration_seconds !== undefined)
+                    next.duration_seconds = duration_seconds;
+                if (aspect_ratio !== undefined)
+                    next.aspect_ratio = aspect_ratio;
+                if (resolution !== undefined)
+                    next.resolution = resolution;
+                if (model !== undefined) {
+                    next.model = model && {
+                        ...model,
+                        provider: model.provider
+                    };
+                }
+                // A null clears the setting: store it absent, not null.
+                for (const key of ONE_TAKE_SETTINGS) {
+                    if (next[key] === null)
+                        delete next[key];
+                }
+                doc.one_take = next;
+                // Field-level, the way set_board is: an open editor takes the
+                // direction whole.
+                ops.push({ tool: "set_board", input: { one_take: doc.one_take } });
+            }
+            for (const patch of shotPatches ?? []) {
+                const shot = findShot(doc.shots, patch.shot_id);
+                if (!shot) {
+                    return {
+                        error: `No shot matches "${patch.shot_id}". Call get_storyboard for shot ids.`
+                    };
+                }
+                const next = { ...shot };
+                for (const field of ["end_state", "sound"]) {
+                    const value = patch[field];
+                    if (value === undefined)
+                        continue;
+                    if (value === null || value.trim() === "") {
+                        delete next[field];
+                    }
+                    else {
+                        next[field] = value;
+                    }
+                }
+                doc.shots = doc.shots.map((s) => (s.id === shot.id ? next : s));
+                ops.push({
+                    tool: "update_shot",
+                    input: { id: shot.id, target: shot.id }
+                });
+            }
+            if (doc.screenplay)
+                doc.screenplay.shots = doc.shots;
+            const saved = await Storyboard.updateFieldsIfUnchanged(row.id, row.updated_at, { document: JSON.stringify(doc) }, { ops });
+            if (!saved)
+                continue;
+            return oneTakeSummary(saved, doc);
+        }
+        return {
+            error: `Storyboard ${String(params["storyboard_id"])} is being modified concurrently; nothing was saved. Retry the call.`
+        };
+    }
+};
 export const STORYBOARD_CAPABILITIES = [
     finishStoryboard,
     layoutStoryboard,
@@ -2454,14 +2913,17 @@ export const STORYBOARD_CAPABILITIES = [
     renderStoryboardClips,
     reviseStoryboardClip,
     assembleStoryboardTimeline,
+    deriveShotFromClip,
     editStoryboard,
     directStoryboard,
     extractScriptFromStoryboard,
-    deleteStoryboard
+    deleteStoryboard,
+    getStoryboardOneTake,
+    updateStoryboardOneTake
 ];
 export const module = {
     module: "storyboards",
     exports: STORYBOARD_CAPABILITIES
 };
-export { finishStoryboard, layoutStoryboard, previewStoryboardDesign, listStoryboards, createStoryboard, getStoryboard, renderStoryboardStills, renderStoryboardClips, reviseStoryboardClip, assembleStoryboardTimeline, editStoryboard, directStoryboard, extractScriptFromStoryboard, deleteStoryboard };
+export { finishStoryboard, layoutStoryboard, previewStoryboardDesign, listStoryboards, createStoryboard, getStoryboard, renderStoryboardStills, renderStoryboardClips, reviseStoryboardClip, assembleStoryboardTimeline, deriveShotFromClip, editStoryboard, directStoryboard, extractScriptFromStoryboard, deleteStoryboard, getStoryboardOneTake, updateStoryboardOneTake };
 //# sourceMappingURL=storyboards.js.map