@mevdragon/vidfarm-devcli 0.19.0 → 0.19.1

package/dist/src/services/hyperframes.js

@@ -7,6 +7,7 @@ import path from "node:path";
 import { pipeline } from "node:stream/promises";
 import { CloudFormationClient, DescribeStacksCommand } from "@aws-sdk/client-cloudformation";
 import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { DescribeExecutionCommand, ExecutionDoesNotExist, SFNClient } from "@aws-sdk/client-sfn";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import ffmpegStatic from "ffmpeg-static";
 import ffprobeStatic from "ffprobe-static";
@@ -21,6 +22,24 @@ import { applyMarkupUsd } from "./billing-pricing.js";
 // coupling to the (parallel) clip-curation work — a plain leaf import; the
 // retrieval logic itself (search over these criteria) is merged in later.
 import { buildClipEmbeddingText, sanitizeTagValues, taxonomyForPrompt } from "./clip-curation/index.js";
+/** Thrown by the primitive render wrapper when a distributed render is still in
+ *  flight. The job runner catches it and parks the job as `waiting_for_render`
+ *  so Step Functions waits + re-invokes — no single Lambda holds the render open
+ *  for its 15-minute lifetime. */
+export class RenderInProgressError extends Error {
+    executionArn;
+    renderId;
+    waitSeconds;
+    progressFraction;
+    constructor(input) {
+        super(`HyperFrames render ${input.renderId} still in progress`);
+        this.name = "RenderInProgressError";
+        this.executionArn = input.executionArn;
+        this.renderId = input.renderId;
+        this.waitSeconds = input.waitSeconds;
+        this.progressFraction = input.progressFraction;
+    }
+}
 let stackCache = null;
 const SMART_DECOMPOSE_TEMP_PREFIX = "vidfarm-smart-decompose-";
 const SMART_DECOMPOSE_TEMP_MAX_AGE_MS = 24 * 60 * 60 * 1000;
@@ -81,57 +100,12 @@ export class HyperframesService {
         if (shouldRenderLocally()) {
             return this.renderLocal({ ...input, compositionHtml });
         }
-        const metadata = extractCompositionMetadata(compositionHtml, input);
-        const stack = await resolveHyperframesStack();
-        devLog("hyperframes.render.staging", {
-            composition_id: metadata.compositionId,
-            width: metadata.width,
-            height: metadata.height,
-            fps: metadata.fps,
-            stack_name: stack.stackName,
-            region: stack.region
-        });
-        const projectDir = await stageProject({
-            compositionHtml,
-            compositionId: metadata.compositionId,
-            projectFiles: input.projectFiles,
-            copyPaths: input.copyPaths
-        });
-        const timestamp = Date.now().toString(36);
-        const renderId = input.executionName || `vf-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
-        const outputKey = input.outputKey || `renders/vidfarm/${metadata.compositionId}/${renderId}/output.mp4`;
-        devLog("hyperframes.render.dispatch", {
-            composition_id: metadata.compositionId,
-            render_id: renderId,
-            output_key: outputKey,
-            chunk_size: input.chunkSize ?? 240,
-            max_parallel_chunks: input.maxParallelChunks ?? 8,
-            wait: input.wait !== false
-        });
-        const { renderToLambda, getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
-        const handle = await renderToLambda({
-            projectDir,
-            bucketName: stack.bucketName,
-            stateMachineArn: stack.stateMachineArn,
-            region: stack.region,
-            executionName: renderId,
-            outputKey,
-            config: {
-                fps: normalizeRenderFps(metadata.fps),
-                width: metadata.width,
-                height: metadata.height,
-                format: input.format || "mp4",
-                codec: "h264",
-                quality: "standard",
-                chunkSize: input.chunkSize ?? 240,
-                maxParallelChunks: input.maxParallelChunks ?? 8,
-                runtimeCap: "lambda"
-            }
-        });
+        const prep = await this.prepareCloudRender(input, compositionHtml);
+        const handle = await this.dispatchCloudRender(prep, input);
         if (input.wait === false) {
             devLog("hyperframes.render.dispatched_async", {
-                composition_id: metadata.compositionId,
-                render_id: renderId,
+                composition_id: prep.metadata.compositionId,
+                render_id: prep.renderId,
                 execution_arn: handle.executionArn
             });
             return {
@@ -142,11 +116,12 @@ export class HyperframesService {
                 metadata: {
                     mode: "hyperframes_lambda",
                     status: "RUNNING",
-                    stack,
-                    composition: metadata
+                    stack: prep.stack,
+                    composition: prep.metadata
                 }
             };
         }
+        const { getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
         const startedAtMs = Date.now();
         const timeoutMs = input.timeoutMs ?? 12 * 60 * 1000;
         const pollIntervalMs = input.pollIntervalMs ?? 5000;
@@ -155,14 +130,14 @@ export class HyperframesService {
         for (;;) {
             const progress = await getRenderProgress({
                 executionArn: handle.executionArn,
-                region: stack.region,
-                defaultMemorySizeMb: stack.lambdaMemoryMb
+                region: prep.stack.region,
+                defaultMemorySizeMb: prep.stack.lambdaMemoryMb
             });
             pollCount += 1;
             if (progress.status !== lastLoggedStatus) {
                 devLog("hyperframes.render.progress", {
-                    composition_id: metadata.compositionId,
-                    render_id: renderId,
+                    composition_id: prep.metadata.compositionId,
+                    render_id: prep.renderId,
                     status: progress.status,
                     poll_count: pollCount,
                     elapsed_ms: Date.now() - startedAtMs,
@@ -172,42 +147,22 @@ export class HyperframesService {
                 lastLoggedStatus = progress.status;
             }
             if (progress.status === "SUCCEEDED") {
-                const outputS3Uri = progress.outputFile?.s3Uri || handle.outputS3Uri;
                 devLog("hyperframes.render.succeeded", {
-                    composition_id: metadata.compositionId,
-                    render_id: renderId,
+                    composition_id: prep.metadata.compositionId,
+                    render_id: prep.renderId,
                     duration_ms: Date.now() - startedAtMs,
-                    output_s3_uri: outputS3Uri,
+                    output_s3_uri: progress.outputFile?.s3Uri || handle.outputS3Uri,
                     frames_rendered: progress.framesRendered,
                     total_frames: progress.totalFrames,
                     lambdas_invoked: progress.lambdasInvoked,
                     cost_usd: progress.costs?.accruedSoFarUsd
                 });
-                return {
-                    ...handle,
-                    outputS3Uri,
-                    outputUrl: await signedS3Url(outputS3Uri, stack.region),
-                    endedAt: progress.endedAt,
-                    metadata: {
-                        mode: "hyperframes_lambda",
-                        status: progress.status,
-                        stack,
-                        composition: metadata,
-                        progress,
-                        costs: progress.costs,
-                        estimatedCostUsd: progress.costs.accruedSoFarUsd,
-                        chargeUsd: applyMarkupUsd(progress.costs.accruedSoFarUsd),
-                        costDisplay: progress.costs.displayCost,
-                        lambdasInvoked: progress.lambdasInvoked,
-                        framesRendered: progress.framesRendered,
-                        totalFrames: progress.totalFrames
-                    }
-                };
+                return this.buildCloudRenderResult(handle, progress, prep.metadata, prep.stack);
             }
             if (progress.fatalErrorEncountered || ["FAILED", "TIMED_OUT", "ABORTED"].includes(progress.status)) {
                 devLog("hyperframes.render.failed", {
-                    composition_id: metadata.compositionId,
-                    render_id: renderId,
+                    composition_id: prep.metadata.compositionId,
+                    render_id: prep.renderId,
                     status: progress.status,
                     duration_ms: Date.now() - startedAtMs,
                     errors: Array.isArray(progress.errors) ? progress.errors.slice(0, 3) : progress.errors
@@ -216,8 +171,8 @@ export class HyperframesService {
             }
             if (Date.now() - startedAtMs > timeoutMs) {
                 devLog("hyperframes.render.timeout", {
-                    composition_id: metadata.compositionId,
-                    render_id: renderId,
+                    composition_id: prep.metadata.compositionId,
+                    render_id: prep.renderId,
                     timeout_ms: timeoutMs,
                     last_status: progress.status
                 }, "error");
@@ -226,6 +181,182 @@ export class HyperframesService {
             await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
         }
     }
+    /**
+     * One non-blocking step of a distributed render, keyed by a stable
+     * `executionName`. The first call stages + dispatches the render to the
+     * HyperFrames render state machine and returns `running` immediately; later
+     * calls (from a re-invoked job runner) DescribeExecution the SAME execution
+     * and either keep waiting, finalize on SUCCEEDED, or report failure. This is
+     * what keeps a 40-minute render off any single Lambda's 15-minute clock.
+     */
+    async renderStep(input) {
+        const compositionHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
+        if (!compositionHtml.includes("data-composition-id=")) {
+            throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
+        }
+        // Local/dev has no Step Functions — render in-process, blocking, and finalize
+        // in one step so the devcli and local backend behave exactly as before.
+        if (shouldRenderLocally()) {
+            try {
+                const result = await this.renderLocal({ ...input, compositionHtml });
+                return { state: "succeeded", ...result };
+            }
+            catch (error) {
+                return {
+                    state: "failed",
+                    renderId: input.executionName,
+                    executionArn: null,
+                    message: error instanceof Error ? error.message : String(error)
+                };
+            }
+        }
+        const stack = await resolveHyperframesStack();
+        const executionArn = executionArnFromName(stack.stateMachineArn, input.executionName);
+        const existingStatus = await describeRenderExecutionStatus(executionArn, stack.region);
+        if (existingStatus === null) {
+            // Never dispatched — stage + kick off the render, then park.
+            const prep = await this.prepareCloudRender(input, compositionHtml, input.executionName);
+            const handle = await this.dispatchCloudRender(prep, input);
+            devLog("hyperframes.render.dispatched_async", {
+                composition_id: prep.metadata.compositionId,
+                render_id: prep.renderId,
+                execution_arn: handle.executionArn
+            });
+            return { state: "running", executionArn: handle.executionArn, renderId: handle.renderId, progressFraction: 0.1 };
+        }
+        if (existingStatus === "RUNNING" || existingStatus === "PENDING_REDRIVE") {
+            // Cheap path: DescribeExecution already told us it's still going. Avoid the
+            // heavier GetExecutionHistory pagination that getRenderProgress performs.
+            return { state: "running", executionArn, renderId: input.executionName, progressFraction: 0.5 };
+        }
+        // Terminal (SUCCEEDED / FAILED / TIMED_OUT / ABORTED) — read the full snapshot
+        // for output location, frames, and cost metadata.
+        const { getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
+        const progress = await getRenderProgress({
+            executionArn,
+            region: stack.region,
+            defaultMemorySizeMb: stack.lambdaMemoryMb
+        });
+        if (progress.status === "SUCCEEDED") {
+            const outputS3Uri = progress.outputFile?.s3Uri || null;
+            if (!outputS3Uri) {
+                return {
+                    state: "failed",
+                    renderId: input.executionName,
+                    executionArn,
+                    message: `HyperFrames render ${input.executionName} reported SUCCEEDED without an output file.`
+                };
+            }
+            const metadata = extractCompositionMetadata(compositionHtml, input);
+            const handleShape = {
+                renderId: input.executionName,
+                executionArn,
+                bucketName: stack.bucketName,
+                stateMachineArn: stack.stateMachineArn,
+                outputS3Uri,
+                // Not carried across re-invocations; unused by downstream consumers,
+                // which read outputS3Uri / outputUrl / renderId / metadata only.
+                projectS3Uri: "",
+                startedAt: progress.startedAt
+            };
+            const result = await this.buildCloudRenderResult(handleShape, progress, metadata, stack);
+            devLog("hyperframes.render.succeeded", {
+                composition_id: metadata.compositionId,
+                render_id: input.executionName,
+                output_s3_uri: outputS3Uri,
+                frames_rendered: progress.framesRendered,
+                total_frames: progress.totalFrames,
+                lambdas_invoked: progress.lambdasInvoked,
+                cost_usd: progress.costs?.accruedSoFarUsd
+            });
+            return { state: "succeeded", ...result };
+        }
+        devLog("hyperframes.render.failed", {
+            render_id: input.executionName,
+            status: progress.status,
+            errors: Array.isArray(progress.errors) ? progress.errors.slice(0, 3) : progress.errors
+        }, "error");
+        return {
+            state: "failed",
+            renderId: input.executionName,
+            executionArn,
+            message: `HyperFrames render ${input.executionName} ${progress.status}: ${scrubVendorDocLinks(JSON.stringify(progress.errors))}`
+        };
+    }
+    async prepareCloudRender(input, compositionHtml, executionName) {
+        const metadata = extractCompositionMetadata(compositionHtml, input);
+        const stack = await resolveHyperframesStack();
+        devLog("hyperframes.render.staging", {
+            composition_id: metadata.compositionId,
+            width: metadata.width,
+            height: metadata.height,
+            fps: metadata.fps,
+            stack_name: stack.stackName,
+            region: stack.region
+        });
+        const projectDir = await stageProject({
+            compositionHtml,
+            compositionId: metadata.compositionId,
+            projectFiles: input.projectFiles,
+            copyPaths: input.copyPaths
+        });
+        const timestamp = Date.now().toString(36);
+        const renderId = executionName || input.executionName || `vf-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
+        const outputKey = input.outputKey || `renders/vidfarm/${metadata.compositionId}/${renderId}/output.mp4`;
+        return { metadata, stack, projectDir, renderId, outputKey };
+    }
+    async dispatchCloudRender(prep, input) {
+        devLog("hyperframes.render.dispatch", {
+            composition_id: prep.metadata.compositionId,
+            render_id: prep.renderId,
+            output_key: prep.outputKey,
+            chunk_size: input.chunkSize ?? 240,
+            max_parallel_chunks: input.maxParallelChunks ?? 8
+        });
+        const { renderToLambda } = await import("@hyperframes/aws-lambda/sdk");
+        return renderToLambda({
+            projectDir: prep.projectDir,
+            bucketName: prep.stack.bucketName,
+            stateMachineArn: prep.stack.stateMachineArn,
+            region: prep.stack.region,
+            executionName: prep.renderId,
+            outputKey: prep.outputKey,
+            config: {
+                fps: normalizeRenderFps(prep.metadata.fps),
+                width: prep.metadata.width,
+                height: prep.metadata.height,
+                format: input.format || "mp4",
+                codec: "h264",
+                quality: "standard",
+                chunkSize: input.chunkSize ?? 240,
+                maxParallelChunks: input.maxParallelChunks ?? 8,
+                runtimeCap: "lambda"
+            }
+        });
+    }
+    async buildCloudRenderResult(handle, progress, metadata, stack) {
+        const outputS3Uri = progress.outputFile?.s3Uri || handle.outputS3Uri;
+        return {
+            ...handle,
+            outputS3Uri,
+            outputUrl: outputS3Uri ? await signedS3Url(outputS3Uri, stack.region) : null,
+            endedAt: progress.endedAt,
+            metadata: {
+                mode: "hyperframes_lambda",
+                status: progress.status,
+                stack,
+                composition: metadata,
+                progress,
+                costs: progress.costs,
+                estimatedCostUsd: progress.costs.accruedSoFarUsd,
+                chargeUsd: applyMarkupUsd(progress.costs.accruedSoFarUsd),
+                costDisplay: progress.costs.displayCost,
+                lambdasInvoked: progress.lambdasInvoked,
+                framesRendered: progress.framesRendered,
+                totalFrames: progress.totalFrames
+            }
+        };
+    }
     // In-process render: stage the project, then drive HyperFrames' single-machine
     // orchestrator (headless Chromium → per-frame capture → ffmpeg encode) to write
     // the MP4 straight to local disk storage. No AWS, no Lambda fan-out. The result
@@ -2905,6 +3036,44 @@ export function normalizePublishHtml(html) {
     }
     return `<!doctype html>\n${document.documentElement.outerHTML}`;
 }
+/**
+ * Reconstruct a Step Functions execution ARN from the render state machine ARN
+ * plus a deterministic execution name, so a re-invoked job runner can poll the
+ * SAME render without persisting the ARN handed back at dispatch time.
+ *   arn:...:stateMachine:NAME  ->  arn:...:execution:NAME:EXEC_NAME
+ */
+function executionArnFromName(stateMachineArn, executionName) {
+    const marker = ":stateMachine:";
+    const idx = stateMachineArn.indexOf(marker);
+    if (idx === -1) {
+        throw new Error(`Unexpected HyperFrames render state machine ARN: ${stateMachineArn}`);
+    }
+    const prefix = stateMachineArn.slice(0, idx);
+    const machineName = stateMachineArn.slice(idx + marker.length);
+    return `${prefix}:execution:${machineName}:${executionName}`;
+}
+/**
+ * DescribeExecution for a render execution. Returns the Step Functions status,
+ * or `null` when the execution does not exist yet (never dispatched) — the
+ * signal renderStep() uses to decide between dispatch and resume.
+ */
+async function describeRenderExecutionStatus(executionArn, region) {
+    const client = new SFNClient({ region });
+    try {
+        const response = await client.send(new DescribeExecutionCommand({ executionArn }));
+        return response.status ?? "RUNNING";
+    }
+    catch (error) {
+        if (error instanceof ExecutionDoesNotExist) {
+            return null;
+        }
+        const name = error instanceof Error ? error.name : "";
+        if (name === "ExecutionDoesNotExist") {
+            return null;
+        }
+        throw error;
+    }
+}
 async function resolveHyperframesStack() {
     const region = config.HYPERFRAMES_RENDER_REGION || config.AWS_REGION || "us-east-1";
     const lambdaMemoryMb = positiveInteger(config.HYPERFRAMES_RENDER_MEMORY_MB, 3072);

package/dist/src/services/serverless-jobs.js

@@ -17,7 +17,7 @@ const dynamodb = config.RECORDS_DRIVER === "local"
         }
     });
 const sfn = new SFNClient({});
-const ACTIVE_ROOT_JOB_STATUSES = ["queued", "running", "waiting_for_provider", "waiting_for_child", "waiting_for_human"];
+const ACTIVE_ROOT_JOB_STATUSES = ["queued", "running", "waiting_for_provider", "waiting_for_render", "waiting_for_child", "waiting_for_human"];
 const ACTIVE_JOB_STATUSES = new Set(ACTIVE_ROOT_JOB_STATUSES);
 export class ServerlessJobsService {
     async createRootJob(input) {
@@ -282,6 +282,7 @@ export class ServerlessJobsService {
                 "#status = :queuedStatus",
                 "#status = :runningStatus",
                 "#status = :waitingForProviderStatus",
+                "#status = :waitingForRenderStatus",
                 "#status = :waitingForChildStatus",
                 "#status = :waitingForHumanStatus"
             ].join(" OR ")
@@ -293,6 +294,7 @@ export class ServerlessJobsService {
             expressionAttributeValues[":queuedStatus"] = "queued";
             expressionAttributeValues[":runningStatus"] = "running";
             expressionAttributeValues[":waitingForProviderStatus"] = "waiting_for_provider";
+            expressionAttributeValues[":waitingForRenderStatus"] = "waiting_for_render";
             expressionAttributeValues[":waitingForChildStatus"] = "waiting_for_child";
             expressionAttributeValues[":waitingForHumanStatus"] = "waiting_for_human";
         }

package/dist/src/template-editor-pages.js

@@ -1925,7 +1925,7 @@ export function renderCustomInpaintPage(input) {
           }
 
           function isPendingJobStatus(status) {
-            return ["queued", "running", "waiting_for_provider", "waiting_for_child", "waiting_for_human"].includes(String(status || ""));
+            return ["queued", "running", "waiting_for_provider", "waiting_for_render", "waiting_for_child", "waiting_for_human"].includes(String(status || ""));
           }
 
           function pendingHistoryItemKey(jobId) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.19.0",
+  "version": "0.19.1",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {