@nodetool-ai/agents 0.8.2 → 0.8.4-a24.1

package/dist/capabilities/timelines.js

@@ -17,17 +17,14 @@
  *
  * Design: docs/tool-class-retirement-design.md § "Migration".
  */
-import { z } from "zod";
-import { applyTimelineTrackOp } from "@nodetool-ai/timeline/ops";
-import { STAGGER_UNITS, ANIMATED_PROPERTIES, DEFAULT_BEAT_TOLERANCE_MS } from "@nodetool-ai/timeline";
-import { buildTimelineToolContracts } from "@nodetool-ai/protocol/api-schemas/timeline-tool-contracts.js";
-import { resolveMoveTrackArgs, resolveDeleteTrackArgs } from "@nodetool-ai/protocol/api-schemas/timeline-tool-params.js";
-import { parseWithTypeCoercion } from "@nodetool-ai/runtime";
+import { parseOps, applyOps, TOOL_PREFIX, READ_ONLY_OPS, timelineModel3DBaker } from "./timeline-operations.js";
+export { parseOps, applyOps } from "./timeline-operations.js";
 import { listTimelinesSpec, createTimelineSpec, getTimelineSpec, listTimelineVersionsSpec, getTimelineVersionSpec, createTimelineVersionSpec, restoreTimelineVersionSpec, deleteTimelineVersionSpec, editTimelineSpec, validateTimelineSpec, setTimelineDocumentSpec, previewTimelineFrameSpec, compareTimelineFramesSpec, renderTimelineSpec, bakeAudioAnimationSpec, isolateSubjectSpec, ISOLATE_SUBJECT_MODELS, ISOLATE_SUBJECT_RESOLUTIONS, DEFAULT_ISOLATE_SUBJECT_MODEL, DEFAULT_ISOLATE_SUBJECT_RESOLUTION, DEFAULT_BAKE_ATTACK_MS, DEFAULT_BAKE_RELEASE_MS, DEFAULT_BAKE_TOLERANCE, DEFAULT_BAKE_MAX_POINTS, DEFAULT_BAKE_MAX_SECONDS, MAX_BAKE_POINTS, DEFAULT_RENDER_TIMEOUT_MS, MAX_RENDER_TIMEOUT_MS, DEFAULT_PREVIEW_COUNT, DEFAULT_COMPARE_WIDTH, MAX_PREVIEW_TIMES, MAX_PREVIEW_RANGE_COUNT, DEFAULT_VERSION_LIMIT, MAX_VERSION_LIMIT, trackObjectSpec, TRACK_OBJECT_DIRECTIONS, getTimelineCodeSpec, setTimelineCodeSpec, editTimelineCodeSpec, rebakeTimelineCodeSpec, detachTimelineCodeSpec, deleteTimelineSpec } from "./timelines.specs.js";
 import { AUDIO_BAKED_ANIMATION_KIND, normalizeAuthoredDocument, hashSceneSubtree, mergeTimelineSource } from "@nodetool-ai/timeline";
 import { bakeTimelineCode, bakedDocumentMatchesSaved } from "../timeline-code-bake.js";
 import { applyJsScriptEditOps, parseJsScriptEditOps } from "./js-scripts.js";
 import { clipSourceWindowMs } from "./timeline-audio-bake.js";
+import { canonicalizeTimelineAssetIds, userAssetFinder } from "./timeline-asset-ids.js";
 import { TIMELINE_NATIVE_MEDIA_CAPABILITIES } from "./timeline-native-media.js";
 import { isFiniteNumber, isRecord, isString } from "../utils/type-guards.js";
 import { resolveProjectId } from "./project-scope.js";
@@ -372,400 +369,6 @@ const deleteTimelineVersion = {
 // ---------------------------------------------------------------------------
 /** Attempts to land the document write before reporting a conflict. */
 const CAS_ATTEMPTS = 2;
-/** Operations one call may apply, so a runaway script cannot rewrite a cut. */
-const MAX_OPS = 60;
-const TOOL_PREFIX = "ui_timeline_";
-/** Browser generation without a real provider adapter remains excluded. */
-const EXCLUDED_OPS = new Set([`${TOOL_PREFIX}generate_clip`]);
-const READ_ONLY_OPS = new Set([`${TOOL_PREFIX}get_clip_frames`]);
-/** `add_track`, `ui_add_track`, and `ui_timeline_add_track` all name one tool. */
-function normalizeOpName(name) {
-    const trimmed = name.trim();
-    if (trimmed.startsWith(TOOL_PREFIX))
-        return trimmed;
-    const bare = trimmed.startsWith("ui_")
-        ? trimmed.slice("ui_".length)
-        : trimmed;
-    return `${TOOL_PREFIX}${bare}`;
-}
-/** An op is `{op, ...args}` — the arguments sit alongside the verb. */
-export function parseOps(raw) {
-    if (!Array.isArray(raw) || raw.length === 0) {
-        return {
-            error: 'ops must be a non-empty array, e.g. [{"op": "add_track", "type": "audio"}].'
-        };
-    }
-    if (raw.length > MAX_OPS) {
-        return {
-            error: `ops holds ${raw.length} entries; at most ${MAX_OPS} per call.`
-        };
-    }
-    const parsed = [];
-    for (const [index, entry] of raw.entries()) {
-        if (!isRecord(entry)) {
-            return { error: `ops[${index}] must be an object.` };
-        }
-        const { op, ...input } = entry;
-        if (!isString(op) || op.trim() === "") {
-            return { error: `ops[${index}] has no \`op\` name.` };
-        }
-        parsed.push({ op: normalizeOpName(op), input });
-    }
-    return parsed;
-}
-/**
- * Read one of the caller's assets for `add_media_clip`.
- *
- * The ref a model has in hand is whatever `list_assets` printed — a bare id or
- * an `asset://<id>.<ext>` URI — so both resolve here.
- */
-async function resolveTimelineAsset(run, ref) {
-    const id = ref.startsWith("asset://")
-        ? ref.slice("asset://".length).replace(/\.[A-Za-z0-9]{1,8}$/, "")
-        : ref;
-    if (!id)
-        return null;
-    const userId = run.context.userId;
-    if (!userId)
-        return null;
-    const { Asset } = await import("@nodetool-ai/models");
-    // `find` is already user-scoped: someone else's asset reads as missing.
-    const asset = await Asset.find(userId, id);
-    if (!asset)
-        return null;
-    const thumbnails = isRecord(asset.metadata)
-        ? asset.metadata["thumbnails"]
-        : undefined;
-    const thumbnailAssetId = Array.isArray(thumbnails) && isString(thumbnails[0])
-        ? thumbnails[0]
-        : undefined;
-    const resolved = {
-        id: asset.id,
-        name: asset.name,
-        contentType: asset.content_type
-    };
-    let duration = asset.duration;
-    if ((!isFiniteNumber(duration) || duration <= 0) &&
-        (asset.content_type.startsWith("audio/") || asset.content_type.startsWith("video/"))) {
-        try {
-            const { loadMediaRefBytes, probeVideoDurationSeconds } = await import("@nodetool-ai/runtime");
-            const bytes = await loadMediaRefBytes({ asset_id: asset.id }, run.context);
-            if (bytes) {
-                duration = await probeVideoDurationSeconds(bytes, new AbortController().signal);
-                if (isFiniteNumber(duration) && duration > 0) {
-                    asset.duration = duration;
-                    await asset.save();
-                }
-            }
-        }
-        catch {
-            // An explicit clip duration remains usable when the source cannot be probed.
-        }
-    }
-    if (isFiniteNumber(duration) && duration > 0) {
-        resolved.durationMs = Math.round(duration * 1000);
-    }
-    if (thumbnailAssetId)
-        resolved.thumbnailAssetId = thumbnailAssetId;
-    return resolved;
-}
-/**
- * Run a custom animation's body once and return its curves.
- *
- * The bake is the only place that code runs — the curves are stored and every
- * render site samples them — so it goes through `bakeCustomAnimation`, the
- * same hermetic path `POST /api/timelines/animations/bake` uses. Imported
- * lazily: the sandbox is a heavy dependency for a capability most calls never
- * reach.
- */
-async function bakeTimelineAnimation(run, request) {
-    const { bakeCustomAnimation } = await import("../custom-animation-bake.js");
-    const params = {
-        code: request.code,
-        role: request.role,
-        durationMs: request.durationMs,
-        clipDurationMs: request.clipDurationMs,
-        canvas: request.canvas
-    };
-    if (request.params !== undefined)
-        params.params = request.params;
-    if (request.staggerCount !== undefined) {
-        params.staggerCount = request.staggerCount;
-    }
-    const baked = await bakeCustomAnimation(run.context, params);
-    const result = { ok: baked.ok };
-    if (baked.curves !== undefined)
-        result.curves = baked.curves;
-    if (baked.mask !== undefined)
-        result.mask = baked.mask;
-    if (baked.error !== undefined)
-        result.error = baked.error;
-    return result;
-}
-/**
- * The bridge hooks that write outside the document: a Blender render of a 3D
- * clip, and the new sequence `retarget_format` creates.
- */
-function writingHooks(run, sequence) {
-    return {
-        bakeModel3DClip: async (request) => {
-            const { bakeModel3DClipOnServer } = await import("./timeline-bake.js");
-            return bakeModel3DClipOnServer(run.context, request);
-        },
-        retargetFormat: async (adapted) => {
-            const { TimelineSequence } = await import("@nodetool-ai/models");
-            const derivedDocument = {
-                tracks: adapted.tracks,
-                clips: adapted.clips,
-                markers: adapted.markers
-            };
-            if (adapted.transcript !== undefined) {
-                derivedDocument.transcript = adapted.transcript;
-            }
-            if (adapted.scriptEnabled !== undefined) {
-                derivedDocument.scriptEnabled = adapted.scriptEnabled;
-            }
-            if (adapted.tempo !== undefined)
-                derivedDocument.tempo = adapted.tempo;
-            if (adapted.camera2d !== undefined)
-                derivedDocument.camera2d = adapted.camera2d;
-            if (adapted.setup !== undefined)
-                derivedDocument.setup = adapted.setup;
-            if (adapted.templateId !== undefined) {
-                derivedDocument.templateId = adapted.templateId;
-            }
-            if (adapted.mediaTracks !== undefined) {
-                derivedDocument.mediaTracks = adapted.mediaTracks;
-            }
-            const name = `${sequence.name} (${adapted.width}×${adapted.height})`;
-            const created = await TimelineSequence.create({
-                user_id: run.context.userId,
-                project_id: sequence.project_id,
-                workflow_id: sequence.workflow_id,
-                name,
-                fps: adapted.fps,
-                width: adapted.width,
-                height: adapted.height,
-                duration_ms: adapted.durationMs,
-                document: JSON.stringify(derivedDocument)
-            });
-            return { sequenceId: created.id, name };
-        }
-    };
-}
-/** Track-only batches need no simulated editor or external hooks. */
-function applyTrackOps(sequence, document, ops) {
-    const contracts = buildTimelineToolContracts({
-        staggerUnits: STAGGER_UNITS,
-        animatedProperties: ANIMATED_PROPERTIES,
-        beatToleranceMs: DEFAULT_BEAT_TOLERANCE_MS
-    });
-    let state = {
-        fps: sequence.fps,
-        width: sequence.width,
-        height: sequence.height,
-        tracks: document.tracks,
-        clips: document.clips,
-        markers: document.markers,
-        mediaTracks: document.mediaTracks,
-        tempo: document.tempo,
-        playheadMs: 0,
-        selectedClipIds: []
-    };
-    const usedIds = new Set([
-        ...[
-            ...document.tracks,
-            ...document.clips,
-            ...document.markers,
-            ...(document.mediaTracks ?? [])
-        ].map((unit) => unit.id),
-        ...document.clips.flatMap((clip) => (clip.animations ?? []).map((animation) => animation.id))
-    ]);
-    let nextId = 0;
-    const newId = () => {
-        let id;
-        do {
-            id = `track_${++nextId}`;
-        } while (usedIds.has(id));
-        usedIds.add(id);
-        return id;
-    };
-    const records = [];
-    for (const entry of ops) {
-        try {
-            let op;
-            if (entry.op === "ui_timeline_add_track") {
-                const args = parseWithTypeCoercion(z.object(contracts.ui_timeline_add_track.shape), entry.input);
-                op = { op: "add_track", ...args };
-            }
-            else if (entry.op === "ui_timeline_move_track") {
-                const args = parseWithTypeCoercion(z.object(contracts.ui_timeline_move_track.shape), entry.input);
-                op = { op: "move_track", ...resolveMoveTrackArgs(args) };
-            }
-            else {
-                const args = parseWithTypeCoercion(z.object(contracts.ui_timeline_delete_track.shape), entry.input);
-                op = { op: "delete_track", ...resolveDeleteTrackArgs(args) };
-            }
-            const outcome = applyTimelineTrackOp(state, op, { newId });
-            if (outcome.error)
-                throw new Error(outcome.error);
-            state = outcome.state;
-            records.push({ op: entry.op, ok: true, result: outcome.result });
-        }
-        catch (error) {
-            records.push({
-                op: entry.op,
-                ok: false,
-                error: error instanceof Error ? error.message : String(error)
-            });
-        }
-    }
-    return {
-        records,
-        state: {
-            fps: state.fps,
-            width: state.width,
-            height: state.height,
-            durationMs: state.clips.reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0),
-            playheadMs: state.playheadMs,
-            tracks: state.tracks.map(({ id, name, type, index }) => ({
-                id,
-                name,
-                type,
-                index
-            })),
-            clips: state.clips.map((clip) => ({
-                id: clip.id,
-                name: clip.name,
-                trackId: clip.trackId,
-                mediaType: clip.mediaType,
-                startMs: clip.startMs,
-                durationMs: clip.durationMs,
-                prompt: clip.prompt,
-                animations: (clip.animations ?? []).map(({ role, preset }) => ({
-                    role,
-                    preset
-                }))
-            })),
-            documentTracks: state.tracks,
-            documentClips: state.clips,
-            markers: state.markers,
-            mediaTracks: state.mediaTracks ?? [],
-            derivedSequences: [],
-            tempo: state.tempo,
-            camera2d: document.camera2d ?? null,
-            setup: document.setup ?? null,
-            transitionCandidates: [],
-            appliedTransitionCandidates: [],
-            toolLog: ops.map((entry) => entry.op),
-            previewTimesMs: [],
-            previewedLayerKinds: []
-        }
-    };
-}
-/**
- * Run `ops` against a bridge seeded from `document`.
- *
- * A failing op is recorded and the script continues: stopping at the first
- * error hides every problem behind it, and the caller wants the whole picture.
- *
- * `hermetic` leaves out the hooks that write outside the document: the 3D
- * render and the new sequence a retarget creates. A code bake sets it. The
- * bridge then uses its in-memory fallbacks for those ops.
- */
-export async function applyOps(run, sequence, document, ops, options = {}) {
-    if (ops.every(({ op }) => op === "ui_timeline_add_track" || op === "ui_timeline_move_track" || op === "ui_timeline_delete_track")) {
-        return applyTrackOps(sequence, document, ops);
-    }
-    const { createTimelineToolBridge } = await import("./timeline-bridge.js");
-    let operationIndex = 0;
-    const init = {
-        sequenceId: sequence.id,
-        sequenceName: sequence.name,
-        projectId: sequence.project_id,
-        sequence: {
-            fps: sequence.fps,
-            width: sequence.width,
-            height: sequence.height,
-            tracks: document.tracks,
-            clips: document.clips,
-            markers: document.markers,
-            transcript: document.transcript,
-            scriptEnabled: document.scriptEnabled,
-            templateId: document.templateId,
-            tempo: document.tempo,
-            camera2d: document.camera2d,
-            setup: document.setup,
-            mediaTracks: document.mediaTracks
-        },
-        resolveAsset: (ref) => resolveTimelineAsset(run, ref),
-        bakeAnimation: (request) => bakeTimelineAnimation(run, request),
-        loadComposition: {
-            get: async (id) => {
-                const { loadComposition } = await import("./compositions.js");
-                const found = await loadComposition(run, id);
-                return found ? found.composition : null;
-            },
-            listIds: async () => {
-                const { loadShippedCompositions, loadUserCompositions } = await import("./compositions.js");
-                const mine = await loadUserCompositions(run);
-                return [
-                    ...loadShippedCompositions().map((comp) => comp.id),
-                    ...(Array.isArray(mine) ? mine.map((comp) => comp.id) : [])
-                ];
-            }
-        }
-    };
-    if (!options.hermetic) {
-        Object.assign(init, writingHooks(run, sequence));
-        const { timelineMediaEditGenerator } = await import("./timeline-media-edit.js");
-        const generate = options.generateMediaEdit ?? timelineMediaEditGenerator(run);
-        init.generateMediaEdit = (request) => generate(request, operationIndex);
-        init.getClipFrames = async (clip, frameOptions) => {
-            const current = bridge.finalState();
-            const { inspectTimelineClipFrames } = await import("./timeline-clip-frames.js");
-            return inspectTimelineClipFrames(run.context, { ...sequence.toDocument(),
-                tracks: current.documentTracks, clips: current.documentClips, markers: current.markers,
-                mediaTracks: current.mediaTracks, tempo: current.tempo, camera2d: current.camera2d,
-                id: sequence.id, projectId: sequence.project_id ?? "default", name: sequence.name,
-                fps: sequence.fps, width: sequence.width, height: sequence.height,
-                durationMs: sequence.duration_ms, createdAt: sequence.created_at, updatedAt: sequence.updated_at
-            }, clip, frameOptions);
-        };
-    }
-    const bridge = createTimelineToolBridge(init);
-    const byName = new Map(bridge.tools
-        .filter((tool) => !EXCLUDED_OPS.has(tool.name) && !(options.hermetic && (tool.name === `${TOOL_PREFIX}generatively_edit_clip` || READ_ONLY_OPS.has(tool.name))))
-        .map((tool) => [tool.name, tool]));
-    const records = [];
-    for (const [index, { op, input }] of ops.entries()) {
-        operationIndex = index;
-        const tool = byName.get(op);
-        if (!tool) {
-            const known = [...byName.keys()]
-                .map((name) => name.slice(TOOL_PREFIX.length))
-                .sort()
-                .join(", ");
-            records.push({
-                op,
-                ok: false,
-                error: `No timeline operation named "${op.slice(TOOL_PREFIX.length)}". Available: ${known}.`
-            });
-            continue;
-        }
-        try {
-            records.push({ op, ok: true, result: await tool.execute(input) });
-        }
-        catch (e) {
-            records.push({
-                op,
-                ok: false,
-                error: e instanceof Error ? e.message : String(e)
-            });
-        }
-    }
-    return { records, state: bridge.finalState() };
-}
 /**
  * The keys an op names its unit under, and which list each one means.
  *
@@ -886,6 +489,9 @@ const editTimeline = {
         const { TimelineSequence, TimelineSequenceVersion } = await import("@nodetool-ai/models");
         const { timelineMediaEditGenerator } = await import("./timeline-media-edit.js");
         const generateMediaEdit = timelineMediaEditGenerator(run);
+        // Shared across retries, like the media edit generator: a retry re-applies
+        // every op, and a 3D bake already rendered must not render again.
+        const bakeModel3DClip = timelineModel3DBaker(run);
         for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
             const sequence = await TimelineSequence.findById(timelineId);
             // A sequence owned by someone else reads as missing — the rule the tRPC
@@ -894,7 +500,7 @@ const editTimeline = {
                 return { error: `Timeline ${timelineId} was not found.` };
             }
             const document = sequence.toDocument();
-            const { records, state } = await applyOps(run, sequence, document, ops, { generateMediaEdit });
+            const { records, state } = await applyOps(run, sequence, document, ops, { generateMediaEdit, bakeModel3DClip });
             // The transcript rides along untouched — no timeline operation edits it,
             // so the stored copy stays authoritative. Markers come back from the
             // bridge, which seeded them and which the marker ops write to.
@@ -1134,7 +740,11 @@ const validateTimeline = {
  * whole-document write is that what was sent is what the sequence becomes.
  */
 function documentToStore(raw) {
-    return normalizeAuthoredDocument(raw);
+    const { name, id: _id, projectId: _projectId, workflowId: _workflowId, durationMs: _durationMs, createdAt: _createdAt, updatedAt: _updatedAt, ...document } = raw;
+    return {
+        ...normalizeAuthoredDocument(document),
+        name: isString(name) && name.trim() ? name.trim() : undefined
+    };
 }
 /** The end of the last clip — what the sequence's stored duration means. */
 function documentDurationMs(document) {
@@ -1201,7 +811,19 @@ const setTimelineDocument = {
                 };
             }
         }
-        const { document, settings } = documentToStore(raw);
+        // `get_timeline` returns the sequence row's fields beside the document.
+        // A caller that sends that object back renames the sequence with `name`.
+        // The other row fields are owned by the row, so they are dropped rather
+        // than reported as stripped document fields.
+        const stored = documentToStore(raw);
+        const { settings, name } = stored;
+        // Sandbox code holds the `asset://<short id>.<ext>` form the asset tools
+        // print; the stored clip holds the asset row's full id.
+        const canonical = await canonicalizeTimelineAssetIds(stored.document, userAssetFinder(sequence.user_id));
+        if ("error" in canonical) {
+            return { error: canonical.error, written: false, timeline_id: sequence.id };
+        }
+        const { document } = canonical;
         // Document-level fps/width/height are sequence settings, so opts win and a
         // document that carries them configures the sequence instead of having
         // them stripped with a `field_stripped` warning.
@@ -1237,16 +859,19 @@ const setTimelineDocument = {
             saveType: "manual",
             name: snapshotName
         });
-        const durationMs = documentDurationMs(document);
+        const fields = {
+            document: JSON.stringify(document),
+            fps,
+            width,
+            height,
+            duration_ms: documentDurationMs(document)
+        };
+        if (name !== undefined) {
+            fields.name = name;
+        }
         let saved;
         try {
-            saved = await TimelineSequence.updateFieldsIfUnchanged(sequence.id, sequence.updated_at, {
-                document: JSON.stringify(document),
-                fps,
-                width,
-                height,
-                duration_ms: durationMs
-            });
+            saved = await TimelineSequence.updateFieldsIfUnchanged(sequence.id, sequence.updated_at, fields);
         }
         catch (error) {
             // The model rejects a document without tracks/clips/markers arrays. The
@@ -1273,6 +898,7 @@ const setTimelineDocument = {
             ok: true,
             written: true,
             timeline_id: saved.id,
+            name: saved.name,
             updated_at: saved.updated_at,
             undo_version: undo.version,
             fps: saved.fps,
@@ -2294,7 +1920,7 @@ const isolateSubject = {
         let expectedUpdatedAt = sequence.updated_at;
         const persist = async (next) => {
             const clips = document.clips.map((c) => (c.id === next.id ? next : c));
-            const saved = await TimelineSequence.updateDocumentIfUnchanged(timelineId, expectedUpdatedAt, { ...document, clips }, {
+            const saved = await TimelineSequence.updateDocumentIfUnchanged(sequence.id, expectedUpdatedAt, { ...document, clips }, {
                 ops: [
                     {
                         tool: `${TOOL_PREFIX}set_generated_matte`,
@@ -2416,7 +2042,7 @@ const trackObject = {
         let expectedUpdatedAt = sequence.updated_at;
         let mediaTracks = document.mediaTracks ?? [];
         const persist = async (next) => {
-            const saved = await TimelineSequence.updateDocumentIfUnchanged(timelineId, expectedUpdatedAt, { ...document, mediaTracks: next }, { ops: [{ tool: "track_object", input: { clip_id: clip.id, track_id: trackId } }] });
+            const saved = await TimelineSequence.updateDocumentIfUnchanged(sequence.id, expectedUpdatedAt, { ...document, mediaTracks: next }, { ops: [{ tool: "track_object", input: { clip_id: clip.id, track_id: trackId } }] });
             if (!saved)
                 return false;
             mediaTracks = next;
@@ -2495,7 +2121,12 @@ function normalizeForceOption(value) {
  * state being replaced so the write is undoable, then a CAS write so a
  * concurrent edit is reported rather than clobbered.
  */
-export async function writeTimelineCodeDocument(sequence, document, snapshotName) {
+export async function writeTimelineCodeDocument(sequence, authored, snapshotName) {
+    const canonical = await canonicalizeTimelineAssetIds(authored, userAssetFinder(sequence.user_id));
+    if ("error" in canonical) {
+        return { written: false, error: canonical.error, errors: [], warnings: [] };
+    }
+    const { document } = canonical;
     const { validateTimelineSequence } = await import("@nodetool-ai/execution/timeline-debug");
     const before = validateTimelineSequence(document, { fps: sequence.fps, width: sequence.width, height: sequence.height });
     const errors = before.errors.map((issue) => issue.message);