@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/capabilities/timeline-isolate-subject.js

@@ -0,0 +1,293 @@
+/**
+ * `isolate_subject`: cut the subject out of a video clip and carry the result
+ * as that clip's generated matte (D2).
+ *
+ * Three things are separated here on purpose, because only one of them costs
+ * money and only one of them can be tested without a provider key:
+ *
+ * 1. {@link isolateSubjectOnClip} — the state machine over one clip. It marks
+ *    the matte in flight, asks the runner for a mask, stores it, applies the
+ *    result, and persists at every step. No models, no HTTP, no node registry:
+ *    everything it cannot do itself arrives on {@link IsolateSubjectDeps}.
+ * 2. {@link falIsolateSubjectRunner} — the provider call, wrapped in the
+ *    generation seam so the run gets a row, a cost receipt and cancellation.
+ * 3. {@link contextMaskStore} — the download, under the media-egress policy,
+ *    and the asset it becomes.
+ *
+ * **The whole source asset is sent, never the clip's window.** The mask video
+ * is read back at the clip's own source time, so its first frame has to be the
+ * source's first frame; a submission trimmed to `clipSourceWindowMs` would
+ * need an offset the document does not carry. That is also why a later trim or
+ * extend of the clip stays inside `sourceRange` and never reads as stale.
+ *
+ * **A failure never loses a working matte.** A clip that already had a ready
+ * result keeps it, selected, with its status back at `ready`; only a clip that
+ * had none is left with a `failed` marker. The user's own knobs — invert,
+ * strength, feather — survive a regenerate too, which is
+ * `applyGeneratedMatteResult`'s rule rather than this module's.
+ */
+import { applyGeneratedMatteResult, clipSourceWindowMs, isGeneratedMatteStale } from "@nodetool-ai/timeline";
+import { isRecord, isString } from "../utils/type-guards.js";
+import { ISOLATE_SUBJECT_ENDPOINT, ISOLATE_SUBJECT_NODE_TYPE, ISOLATE_SUBJECT_PROVIDER } from "./timelines.specs.js";
+const CONCURRENT_EDIT = "The timeline is being modified concurrently; nothing was saved. Retry the call.";
+/** The provenance a finished generation stamps on the matte. */
+function matteSettings(settings) {
+    return {
+        model: settings.model,
+        operating_resolution: settings.operating_resolution,
+        refine_foreground: settings.refine_foreground,
+        provider: ISOLATE_SUBJECT_PROVIDER,
+        endpoint: ISOLATE_SUBJECT_ENDPOINT
+    };
+}
+/**
+ * The clip with its matte marked in flight, keeping the current asset and the
+ * version list intact — the shot goes on showing the cutout it has while the
+ * next one is being cut.
+ *
+ * A clip with no matte yet gets a placeholder whose `assetId` is empty. The
+ * scene model draws a matte only at `status: "ready"`, so an empty id is never
+ * read; what it buys is a field the UI can show a spinner against.
+ */
+export function markGeneratedMatteGenerating(clip, sourceAssetId, sourceRange, settings) {
+    const previous = clip.generatedMatte;
+    const generating = previous
+        ? { ...previous, status: "generating" }
+        : {
+            assetId: "",
+            sourceAssetId,
+            sourceRange: { ...sourceRange },
+            settings: matteSettings(settings),
+            status: "generating"
+        };
+    return { ...clip, generatedMatte: generating };
+}
+/**
+ * The clip after a generation failed or was cancelled.
+ *
+ * A previous ready result is put back exactly as it was — the same asset, the
+ * same versions, the same knobs — because a failed regenerate must not cost
+ * the user the matte they were already working with. Only a first attempt
+ * leaves a `failed` marker behind.
+ */
+export function generatedMatteAfterFailure(clip, previous) {
+    if (previous)
+        return { ...clip, generatedMatte: { ...previous } };
+    const matte = clip.generatedMatte;
+    if (!matte)
+        return clip;
+    return { ...clip, generatedMatte: { ...matte, status: "failed" } };
+}
+/**
+ * Cut the subject out of one clip, or hand back the cutout it already has.
+ *
+ * Refuses — without spending — a clip that is not video and a clip with no
+ * asset. Reuses a ready, non-stale matte unless `regenerate` is set: a matte
+ * whose source asset was replaced under it describes a picture the clip no
+ * longer shows, so reusing that one would be worse than paying again.
+ */
+export async function isolateSubjectOnClip(deps, input) {
+    const { clip } = input;
+    if (clip.mediaType !== "video") {
+        return {
+            error: `"${clip.name}" is a ${clip.mediaType} clip. isolate_subject cuts a ` +
+                "subject out of video footage."
+        };
+    }
+    const sourceAssetId = clip.currentAssetId;
+    if (!isString(sourceAssetId) || sourceAssetId === "") {
+        return {
+            error: `"${clip.name}" has no asset yet — generate or import its media ` +
+                "before cutting a matte from it."
+        };
+    }
+    const sourceRange = {
+        fromMs: 0,
+        toMs: input.sourceDurationMs ?? clipSourceWindowMs(clip).toMs
+    };
+    const previous = clip.generatedMatte;
+    if (!input.regenerate &&
+        previous?.status === "ready" &&
+        !isGeneratedMatteStale(clip)) {
+        return {
+            clip,
+            status: "ready",
+            assetId: previous.assetId,
+            sourceRange: previous.sourceRange,
+            reused: true
+        };
+    }
+    const generating = markGeneratedMatteGenerating(clip, sourceAssetId, sourceRange, input.settings);
+    if (!(await deps.persist(generating))) {
+        return { error: CONCURRENT_EDIT };
+    }
+    try {
+        const run = await deps.runner({ sourceAssetId, settings: input.settings });
+        const stored = await deps.storeMask(run.maskVideoUrl);
+        const result = {
+            assetId: stored.assetId,
+            sourceAssetId,
+            sourceRange,
+            settings: matteSettings(input.settings)
+        };
+        if (run.generationId !== undefined)
+            result.jobId = run.generationId;
+        // Applied to the clip as it stood before the placeholder, so the version
+        // list records the result it replaces and not the in-flight marker.
+        const settled = applyGeneratedMatteResult(clip, result);
+        if (!(await deps.persist(settled))) {
+            return { error: CONCURRENT_EDIT };
+        }
+        const outcome = {
+            clip: settled,
+            status: "ready",
+            assetId: stored.assetId,
+            sourceRange,
+            reused: false
+        };
+        if (run.generationId !== undefined)
+            outcome.generationId = run.generationId;
+        if (run.costUsd !== undefined)
+            outcome.costUsd = run.costUsd;
+        return outcome;
+    }
+    catch (error) {
+        const reverted = generatedMatteAfterFailure(generating, previous);
+        // Best effort: the generation already failed, and a save that also fails
+        // leaves the in-flight marker, which the next call overwrites.
+        await deps.persist(reverted);
+        return {
+            clip: reverted,
+            status: "failed",
+            sourceRange,
+            reused: false,
+            error: error instanceof Error ? error.message : String(error)
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// The real deps
+// ---------------------------------------------------------------------------
+/** The mask URL, wherever the node's result put it. */
+function maskVideoUrlOf(record) {
+    const direct = record["mask_video"] ?? record["mask_video_url"];
+    if (isString(direct) && direct !== "")
+        return direct;
+    if (isRecord(direct) && isString(direct["url"]) && direct["url"] !== "") {
+        return direct["url"];
+    }
+    return null;
+}
+/** A receipt's charge, when the provider or the catalog stated one in USD. */
+function costUsdOf(receipt) {
+    const cost = receipt.cost;
+    if (!cost)
+        return undefined;
+    const currency = cost.currency ?? "USD";
+    return currency.toUpperCase() === "USD" ? cost.amount : undefined;
+}
+/**
+ * Run the endpoint through its node, inside the generation seam.
+ *
+ * The seam is what makes this a *generation* rather than a call: the row is
+ * open before the provider is reached, `cancel_generation` can abort it, the
+ * FAL node's own `reportFalCost` lands on the receipt, and a request id the
+ * reconciler can settle against the invoice is recorded either way. The node
+ * holds the API key, so the seam runs `withoutProvider`.
+ *
+ * The mask is the point of the call, so a result without one is an error
+ * rather than a silent no-matte success.
+ */
+export function falIsolateSubjectRunner(context, registry, generationId) {
+    return async (request) => {
+        if (!registry.has(ISOLATE_SUBJECT_NODE_TYPE)) {
+            throw new Error(`${ISOLATE_SUBJECT_NODE_TYPE} is not registered in this install, so ` +
+                "there is nothing to cut the subject out with.");
+        }
+        const properties = {
+            video: {
+                type: "video",
+                uri: `asset://${request.sourceAssetId}`,
+                asset_id: request.sourceAssetId
+            },
+            model: request.settings.model,
+            operating_resolution: request.settings.operating_resolution,
+            refine_foreground: request.settings.refine_foreground,
+            output_mask: true
+        };
+        const result = await context.runGenerationWith({
+            id: generationId,
+            provider: ISOLATE_SUBJECT_PROVIDER,
+            capability: "video_to_video",
+            model: ISOLATE_SUBJECT_ENDPOINT,
+            params: {
+                asset_id: request.sourceAssetId,
+                model: request.settings.model,
+                operating_resolution: request.settings.operating_resolution,
+                refine_foreground: request.settings.refine_foreground,
+                output_mask: true
+            },
+            origin: { surface: "capability" }
+        }, async () => {
+            const executor = registry.resolve({
+                id: `isolate-subject-${generationId}`,
+                type: ISOLATE_SUBJECT_NODE_TYPE,
+                properties
+            });
+            const emitted = [];
+            for await (const chunk of executor.genProcess(properties, context)) {
+                emitted.push(chunk);
+            }
+            return emitted;
+        }, { withoutProvider: true });
+        let maskVideoUrl = null;
+        for (const record of result.output) {
+            maskVideoUrl = maskVideoUrlOf(record) ?? maskVideoUrl;
+        }
+        if (maskVideoUrl === null) {
+            throw new Error(`${ISOLATE_SUBJECT_ENDPOINT} returned no mask video, so there is ` +
+                "nothing to use as a matte.");
+        }
+        const runResult = {
+            maskVideoUrl,
+            generationId: result.id
+        };
+        const cost = result.receipt ? costUsdOf(result.receipt) : undefined;
+        if (cost !== undefined)
+            runResult.costUsd = cost;
+        return runResult;
+    };
+}
+/** What a stored mask is named, before the asset id replaces it. */
+const MASK_ASSET_PREFIX = "isolated-subject-matte";
+/**
+ * Download the provider's mask and store it as an asset.
+ *
+ * `fetchExternalMedia` is the one door a provider-chosen URL leaves through —
+ * https to a public host, every redirect hop re-checked — because the URL in a
+ * provider's response body is not this host's to trust.
+ */
+export function contextMaskStore(context) {
+    return async (maskVideoUrl) => {
+        const { fetchExternalMedia } = await import("@nodetool-ai/runtime");
+        const response = await fetchExternalMedia(maskVideoUrl);
+        if (!response.ok) {
+            throw new Error(`The mask video could not be downloaded: ${response.status} ${response.statusText}.`);
+        }
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        if (bytes.byteLength === 0) {
+            throw new Error("The mask video downloaded as zero bytes.");
+        }
+        const { persistOutput } = await import("../tools/asset-persist.js");
+        const saved = await persistOutput(context, bytes, {
+            namePrefix: MASK_ASSET_PREFIX,
+            mime: "video/mp4"
+        });
+        if (!isString(saved.asset_id) || saved.asset_id === "") {
+            throw new Error("The mask video was downloaded but could not be stored as an asset, " +
+                "so no matte can reference it.");
+        }
+        return { assetId: saved.asset_id };
+    };
+}
+//# sourceMappingURL=timeline-isolate-subject.js.map

package/dist/capabilities/timeline-isolate-subject.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"timeline-isolate-subject.js","sourceRoot":"","sources":["../../src/capabilities/timeline-isolate-subject.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,OAAO,EACL,yBAAyB,EACzB,kBAAkB,EAClB,qBAAqB,EAItB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EACL,wBAAwB,EACxB,yBAAyB,EACzB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAqF9B,MAAM,eAAe,GACnB,iFAAiF,CAAC;AAEpF,gEAAgE;AAChE,SAAS,aAAa,CACpB,QAAgC;IAEhC,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;QACnD,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,QAAQ,EAAE,wBAAwB;QAClC,QAAQ,EAAE,wBAAwB;KACnC,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,4BAA4B,CAC1C,IAAkB,EAClB,aAAqB,EACrB,WAA6C,EAC7C,QAAgC;IAEhC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC;IACrC,MAAM,UAAU,GAAuB,QAAQ;QAC7C,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE;QACvC,CAAC,CAAC;YACE,OAAO,EAAE,EAAE;YACX,aAAa;YACb,WAAW,EAAE,EAAE,GAAG,WAAW,EAAE;YAC/B,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;YACjC,MAAM,EAAE,YAAY;SACrB,CAAC;IACN,OAAO,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;AACjD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CACxC,IAAkB,EAClB,QAAwC;IAExC,IAAI,QAAQ;QAAE,OAAO,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,EAAE,GAAG,QAAQ,EAAE,EAAE,CAAC;IAClE,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC;IAClC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;AACrE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAwB,EACxB,KAA0B;IAE1B,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IACvB,IAAI,IAAI,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EACH,IAAI,IAAI,CAAC,IAAI,UAAU,IAAI,CAAC,SAAS,gCAAgC;gBACrE,+BAA+B;SAClC,CAAC;IACJ,CAAC;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC;IAC1C,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;QACrD,OAAO;YACL,KAAK,EACH,IAAI,IAAI,CAAC,IAAI,oDAAoD;gBACjE,iCAAiC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG;QAClB,MAAM,EAAE,CAAC;QACT,IAAI,EAAE,KAAK,CAAC,gBAAgB,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI;KAC9D,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC;IAErC,IACE,CAAC,KAAK,CAAC,UAAU;QACjB,QAAQ,EAAE,MAAM,KAAK,OAAO;QAC5B,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAC5B,CAAC;QACD,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,4BAA4B,CAC7C,IAAI,EACJ,aAAa,EACb,WAAW,EACX,KAAK,CAAC,QAAQ,CACf,CAAC;IACF,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;IACpC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACtD,MAAM,MAAM,GAAyB;YACnC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,aAAa;YACb,WAAW;YACX,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC;SACxC,CAAC;QACF,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;YAAE,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC;QACpE,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,OAAO,GAAG,yBAAyB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;QACpC,CAAC;QACD,MAAM,OAAO,GAA0B;YACrC,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,WAAW;YACX,MAAM,EAAE,KAAK;SACd,CAAC;QACF,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;YAAE,OAAO,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC;QAC5E,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAC7D,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,0BAA0B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAClE,yEAAyE;QACzE,+DAA+D;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,QAAQ;YAChB,WAAW;YACX,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,uDAAuD;AACvD,SAAS,cAAc,CAAC,MAA+B;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAChE,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,EAAE;QAAE,OAAO,MAAM,CAAC;IACrD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QACxE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,SAAS,SAAS,CAAC,OAElB;IACC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IACxC,OAAO,QAAQ,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACpE,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAA0B,EAC1B,QAAsB,EACtB,YAAoB;IAEpB,OAAO,KAAK,EAAE,OAAO,EAAE,EAAE;QACvB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,yBAAyB,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CACb,GAAG,yBAAyB,yCAAyC;gBACnE,+CAA+C,CAClD,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAA4B;YAC1C,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO;gBACb,GAAG,EAAE,WAAW,OAAO,CAAC,aAAa,EAAE;gBACvC,QAAQ,EAAE,OAAO,CAAC,aAAa;aAChC;YACD,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK;YAC7B,oBAAoB,EAAE,OAAO,CAAC,QAAQ,CAAC,oBAAoB;YAC3D,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,iBAAiB;YACrD,WAAW,EAAE,IAAI;SAClB,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAC5C;YACE,EAAE,EAAE,YAAY;YAChB,QAAQ,EAAE,wBAAwB;YAClC,UAAU,EAAE,gBAAgB;YAC5B,KAAK,EAAE,wBAAwB;YAC/B,MAAM,EAAE;gBACN,QAAQ,EAAE,OAAO,CAAC,aAAa;gBAC/B,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK;gBAC7B,oBAAoB,EAAE,OAAO,CAAC,QAAQ,CAAC,oBAAoB;gBAC3D,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,iBAAiB;gBACrD,WAAW,EAAE,IAAI;aAClB;YACD,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE;SAClC,EACD,KAAK,IAAI,EAAE;YACT,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAChC,EAAE,EAAE,mBAAmB,YAAY,EAAE;gBACrC,IAAI,EAAE,yBAAyB;gBAC/B,UAAU;aACX,CAAC,CAAC;YACH,MAAM,OAAO,GAA8B,EAAE,CAAC;YAC9C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,UAAW,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;gBACpE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,EACD,EAAE,eAAe,EAAE,IAAI,EAAE,CAC1B,CAAC;QAEF,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACnC,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC;QACxD,CAAC;QACD,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACb,GAAG,wBAAwB,uCAAuC;gBAChE,4BAA4B,CAC/B,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GAA4B;YACzC,YAAY;YACZ,YAAY,EAAE,MAAM,CAAC,EAAE;SACxB,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpE,IAAI,IAAI,KAAK,SAAS;YAAE,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;QACjD,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC;AACJ,CAAC;AAED,oEAAoE;AACpE,MAAM,iBAAiB,GAAG,wBAAwB,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA0B;IACzD,OAAO,KAAK,EAAE,YAAY,EAAE,EAAE;QAC5B,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,CACrF,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3D,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE;YAChD,UAAU,EAAE,iBAAiB;YAC7B,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CACb,qEAAqE;gBACnE,+BAA+B,CAClC,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;IACrC,CAAC,CAAC;AACJ,CAAC"}

package/dist/capabilities/timeline-media-edit.d.ts

@@ -0,0 +1,8 @@
+import type { MediaEditRequest } from "@nodetool-ai/timeline";
+import type { CapabilityRun } from "./types.js";
+/** Cache one submission across document CAS retries, including a rejected job. */
+export declare function timelineMediaEditGenerator(run: CapabilityRun): (request: MediaEditRequest, operationIndex?: number) => Promise<{
+    generationId: string;
+    assetId: string;
+}>;
+//# sourceMappingURL=timeline-media-edit.d.ts.map

package/dist/capabilities/timeline-media-edit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timeline-media-edit.d.ts","sourceRoot":"","sources":["../../src/capabilities/timeline-media-edit.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,kFAAkF;AAClF,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,aAAa,aAQpC,gBAAgB;kBAHH,MAAM;aAAW,MAAM;GA8G5D"}

package/dist/capabilities/timeline-media-edit.js

@@ -0,0 +1,94 @@
+import { randomUUID } from "node:crypto";
+import { loadMediaRefBytes } from "@nodetool-ai/runtime";
+import { trimVideoWindow } from "@nodetool-ai/runtime/trim-video-window";
+import { isNonBlankString } from "../utils/type-guards.js";
+/** Cache one submission across document CAS retries, including a rejected job. */
+export function timelineMediaEditGenerator(run) {
+    const submitted = new Map();
+    return async (request, operationIndex = 0) => {
+        const key = JSON.stringify([
+            operationIndex,
+            request.sourceContext.clipId,
+            request.provider,
+            request.model,
+            request.instruction
+        ]);
+        const source = JSON.stringify(request.sourceContext);
+        const existing = submitted.get(key);
+        if (existing) {
+            if (existing.source !== source) {
+                throw new Error("The source clip changed while generation was running. The generated asset remains saved in the asset library.");
+            }
+            return existing.result;
+        }
+        const generation = generate(request);
+        submitted.set(key, { source, result: generation });
+        return generation;
+    };
+    async function generate(request) {
+        if (request.provider === "headless" ||
+            request.model === "headless-video-to-video") {
+            throw new Error("Pass provider and model together for a known video-to-video model when editing headlessly.");
+        }
+        const { TimelineSequence, findFinishResourceIds, assertStoryboardClipGenerationAllowed } = await import("@nodetool-ai/models");
+        const sourceContext = request.sourceContext;
+        const userId = run.context.userId;
+        if (!userId) {
+            throw new Error("No user is bound to this session.");
+        }
+        const ids = await findFinishResourceIds("timeline", sourceContext.sequenceId, userId, run.projectId);
+        if (ids.length !== 1) {
+            throw new Error("Timeline generation context was not found uniquely in the caller's project.");
+        }
+        const sequence = await TimelineSequence.findById(ids[0]);
+        if (!sequence) {
+            throw new Error("Timeline generation context was not found.");
+        }
+        const clips = sequence.toDocument().clips.filter((clip) => clip.id === sourceContext.clipId || (/^[a-f0-9]{12}$/.test(sourceContext.clipId) && clip.id.startsWith(sourceContext.clipId)));
+        if (clips.length !== 1 || clips[0].currentAssetId !== sourceContext.sourceAssetId) {
+            throw new Error("Timeline generation source changed or was not found. Save and retry.");
+        }
+        await assertStoryboardClipGenerationAllowed(userId, sequence.project_id, clips[0], "video_to_video");
+        const provider = await run.context.getProvider(request.provider);
+        const models = await provider.getAvailableVideoModels();
+        if (!provider.getCapabilities().includes("video_to_video") ||
+            !models.some((model) => model.id === request.model &&
+                model.supportedTasks?.includes("video_to_video"))) {
+            throw new Error(`Model ${request.provider}:${request.model} does not support video_to_video. Choose a video-edit model with find_model.`);
+        }
+        const source = request.sourceContext;
+        const bytes = await loadMediaRefBytes({ asset_id: source.sourceAssetId }, run.context);
+        if (!bytes?.length) {
+            throw new Error(`The source video ${source.sourceAssetId} could not be loaded.`);
+        }
+        const video = await trimVideoWindow(bytes, source.sourceStartMs, source.sourceEndMs, run.context.signal);
+        await assertStoryboardClipGenerationAllowed(userId, sequence.project_id, clips[0], "video_to_video");
+        const generation = await run.context.runGeneration({
+            id: randomUUID().replaceAll("-", ""),
+            provider: request.provider,
+            model: request.model,
+            capability: "video_to_video",
+            params: {
+                video,
+                prompt: request.instruction,
+                duration_seconds: source.timelineDurationMs / 1000,
+                media_edit: request,
+                source_context: source
+            },
+            origin: { surface: "capability" },
+            persist: { name: `video-edit-${source.clipId}`, mime: "video/mp4" },
+            destination: {
+                document_id: source.sequenceId,
+                target_type: "timeline_clip",
+                target_id: source.clipId,
+                selected: false
+            }
+        });
+        const assetId = generation.assets.find((asset) => isNonBlankString(asset.asset_id))?.asset_id;
+        if (!assetId) {
+            throw new Error(`Generation ${generation.id} completed without a persisted video asset.`);
+        }
+        return { generationId: generation.id, assetId };
+    }
+}
+//# sourceMappingURL=timeline-media-edit.js.map

package/dist/capabilities/timeline-media-edit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"timeline-media-edit.js","sourceRoot":"","sources":["../../src/capabilities/timeline-media-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,wCAAwC,CAAC;AAEzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAG3D,kFAAkF;AAClF,MAAM,UAAU,0BAA0B,CAAC,GAAkB;IAC3D,MAAM,SAAS,GAAG,IAAI,GAAG,EAMtB,CAAC;IACJ,OAAO,KAAK,EAAE,OAAyB,EAAE,cAAc,GAAG,CAAC,EAAE,EAAE;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC;YACzB,cAAc;YACd,OAAO,CAAC,aAAa,CAAC,MAAM;YAC5B,OAAO,CAAC,QAAQ;YAChB,OAAO,CAAC,KAAK;YACb,OAAO,CAAC,WAAW;SACpB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CACb,+GAA+G,CAChH,CAAC;YACJ,CAAC;YACD,OAAO,QAAQ,CAAC,MAAM,CAAC;QACzB,CAAC;QACD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;IAEF,KAAK,UAAU,QAAQ,CAAC,OAAyB;QAC/C,IACE,OAAO,CAAC,QAAQ,KAAK,UAAU;YAC/B,OAAO,CAAC,KAAK,KAAK,yBAAyB,EAC3C,CAAC;YACD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,qCAAqC,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAC/H,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAAC,CAAC;QACtE,MAAM,GAAG,GAAG,MAAM,qBAAqB,CAAC,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QACrG,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAAC,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;QAAC,CAAC;QACzH,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;YAAC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAAC,CAAC;QACjF,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,aAAa,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1L,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,aAAa,CAAC,aAAa,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,qCAAqC,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrG,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,uBAAuB,EAAE,CAAC;QACxD,IACE,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YACtD,CAAC,MAAM,CAAC,IAAI,CACV,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,EAAE,KAAK,OAAO,CAAC,KAAK;gBAC1B,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CACnD,EACD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,SAAS,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,8EAA8E,CACzH,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,iBAAiB,CACnC,EAAE,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,EAClC,GAAG,CAAC,OAAO,CACZ,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,oBAAoB,MAAM,CAAC,aAAa,uBAAuB,CAChE,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,eAAe,CACjC,KAAK,EACL,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CAAC,MAAM,CACnB,CAAC;QACF,MAAM,qCAAqC,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrG,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,EAAE,EAAE,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC;YACpC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,gBAAgB;YAC5B,MAAM,EAAE;gBACN,KAAK;gBACL,MAAM,EAAE,OAAO,CAAC,WAAW;gBAC3B,gBAAgB,EAAE,MAAM,CAAC,kBAAkB,GAAG,IAAI;gBAClD,UAAU,EAAE,OAAO;gBACnB,cAAc,EAAE,MAAM;aACvB;YACD,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE;YACjC,OAAO,EAAE,EAAE,IAAI,EAAE,cAAc,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;YACnE,WAAW,EAAE;gBACX,WAAW,EAAE,MAAM,CAAC,UAAU;gBAC9B,WAAW,EAAE,eAAe;gBAC5B,SAAS,EAAE,MAAM,CAAC,MAAM;gBACxB,QAAQ,EAAE,KAAK;aAChB;SACF,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAC/C,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CACjC,EAAE,QAAQ,CAAC;QACZ,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,cAAc,UAAU,CAAC,EAAE,6CAA6C,CACzE,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC;IAClD,CAAC;AACH,CAAC"}

package/dist/capabilities/timeline-native-media.d.ts

@@ -0,0 +1,4 @@
+import type { CapabilityExport } from "./types.js";
+/** Agent-facing candidate-only entry points for the native media edit tasks. */
+export declare const TIMELINE_NATIVE_MEDIA_CAPABILITIES: readonly CapabilityExport[];
+//# sourceMappingURL=timeline-native-media.d.ts.map

package/dist/capabilities/timeline-native-media.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timeline-native-media.d.ts","sourceRoot":"","sources":["../../src/capabilities/timeline-native-media.ts"],"names":[],"mappings":"AA0CA,OAAO,KAAK,EAAE,gBAAgB,EAAiB,MAAM,YAAY,CAAC;AAmsClE,gFAAgF;AAChF,eAAO,MAAM,kCAAkC,EAAE,SAAS,gBAAgB,EAMzE,CAAC"}