@mevdragon/vidfarm-devcli 0.20.1 → 0.20.3

package/dist/src/services/hyperframes.js

@@ -469,7 +469,7 @@ export class HyperframesService {
         });
         const timestamp = Date.now().toString(36);
         const renderId = `vfimg-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
-        const framesDir = path.join(config.VIDFARM_DATA_DIR, "hyperframes-projects", `${renderId}-frames`);
+        const framesDir = path.join(config.VIDFARM_DATA_DIR, "vidfarm-projects", `${renderId}-frames`);
         await mkdir(framesDir, { recursive: true });
         devLog("hyperframes.render_still.start", {
             composition_id: metadata.compositionId,
@@ -1493,6 +1493,28 @@ export async function annotateScenesFromSource(input) {
         await rm(tempDir, { recursive: true, force: true });
     }
 }
+// The frames handed to the vision passes are SILENT stills — but a short-form
+// video's emotional punch is often carried by its AUDIO (the trending sound, a
+// music-bed drop, vocal intonation, a punchline's delivery). This digest folds
+// the transcript pass into a compact block so the viral-DNA and editor-harness
+// prompts can reason about the sound, not just the pixels. Music-only clips
+// (no speech) still get a note so the model infers a trending-sound bed rather
+// than guessing there is silence.
+function buildAudioDigestForPrompt(transcript) {
+    if (!transcript) {
+        return `\nAUDIO TRACK: none detected, or transcription unavailable. Judge emotional punch from the visuals, on-screen text, and pacing alone — and do not fabricate spoken lines, music, or a trending sound you cannot infer.\n`;
+    }
+    const text = (transcript.text || "").trim();
+    const clipped = text.length > 1200 ? `${text.slice(0, 1200)} …` : text;
+    const segs = transcript.segments.slice(0, 24)
+        .map((s) => `  [${s.start.toFixed(1)}-${s.end.toFixed(1)}s] ${s.text.trim()}`)
+        .filter((line) => line.trim().length > 0)
+        .join("\n");
+    return `\nAUDIO TRANSCRIPT — the video's spoken/sung audio. The SOUND is a primary driver of emotional punch: read the words, the intonation and cadence they imply, WHERE the delivery lands (the pause before a punchline, an exclamation, a beat-synced line), and infer any music / trending-sound bed even when there are no lyrics.
+- language: ${transcript.language ?? "unknown"}
+- full text: ${clipped || "(no speech transcribed — likely a music-only or trending-sound bed; treat the audio as a music/sound-driven format and reason about the beat/drop, not dialogue)"}${segs ? `\n- timed lines:\n${segs}` : ""}
+`;
+}
 function buildSmartDecomposePrompt(input) {
     const sceneCutHint = input.sceneCuts && input.sceneCuts.length > 0
         ? `\nDETECTED SCENE CUTS (authoritative — ffmpeg measured hard visual cuts at these timestamps, in seconds):\n${input.sceneCuts.map((t) => `  cut @ ${t.toFixed(2)}s`).join("\n")}\n\nThese are ground truth for where the video visually changes. Align scene boundaries to these cuts: each detected cut should begin a new scene (a scene "start" should equal a detected cut time, within ~0.15s), and do NOT place a scene boundary where there is no detected cut unless the frames clearly show a change the detector missed. If two adjacent cuts are very close and belong to the same beat, you may merge them into one scene, but never exceed 6 scenes.\n`
@@ -1506,12 +1528,13 @@ function buildSmartDecomposePrompt(input) {
     const frameListing = input.frameTimestamps.length > 0
         ? `\nFRAMES SUPPLIED (in order — each image is captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => `  frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n\nUse the frame timestamps to determine caption "start" and "duration". Cross-reference consecutive frames: a caption whose text is identical across frames N and M was on-screen from at least the timestamp of frame N through the timestamp of frame M (extend "duration" accordingly). A caption present in only one frame is short-lived — set duration to roughly (next_frame_timestamp - this_frame_timestamp) or 0.8s, whichever is smaller.\n`
         : "";
+    const audioDigest = buildAudioDigestForPrompt(input.transcript ?? null);
     return `You are decomposing a short-form vertical video (TikTok/Reel) into editable HyperFrames timeline layers.
 ${userGuidance}${trendTaglineHint}
 Video metadata:
 - duration: ${input.durationSeconds.toFixed(3)} seconds
 - dimensions: ${input.width} x ${input.height} (9:16 vertical, canvas is 100% x 100%)
-${frameListing}${sceneCutHint}
+${frameListing}${sceneCutHint}${audioDigest}
 
 Return strictly valid JSON matching this schema:
 {
@@ -1521,6 +1544,16 @@ Return strictly valid JSON matching this schema:
     "hook": string,
     "retention": string,
     "payoff": string,
+    "emotional_punch": {
+      "core_emotion": string,
+      "tone": string,
+      "arc": string,
+      "peak_moment": string,
+      "mechanism": string,
+      "humor": string,
+      "delivery": string,
+      "preserve": [string, ...]
+    },
     "preserve": [string, ...],
     "avoid": [string, ...],
     "promotions": [string, ...],
@@ -1559,6 +1592,16 @@ VIRAL_DNA RULES:
 - "hook": the 1-sentence tactic used in the first 1-2 seconds to stop the scroll (e.g., "Pinned comment screenshot dares the viewer to keep watching").
 - "retention": how the middle of the video keeps attention (visual cuts, pattern loops, escalating stakes).
 - "payoff": what the viewer earns at the end (reveal, punchline, transformation).
+- "emotional_punch": the FEELING this format delivers and exactly what makes it land — the part remixes most often flatten. hook/retention/payoff are the mechanical structure; this is the vibe, the joke, and the intonation that make a viewer actually FEEL something. Fill EVERY sub-field:
+  - "core_emotion": the dominant feeling a viewer walks away with (e.g. "delighted surprise", "righteous validation", "warm nostalgia", "second-hand cringe", "smug satisfaction").
+  - "tone": the register / vibe in 1-3 words (comedic, deadpan, earnest, dramatic, wholesome, chaotic, satirical, hype, ...).
+  - "arc": how the feeling MOVES across the runtime (e.g. "curiosity → mounting tension → relief-laugh", "flat setup → sudden absurd turn").
+  - "peak_moment": the single beat that lands the emotional hit — the punchline, the reveal, the gut-punch, the beat-drop — and roughly when.
+  - "mechanism": WHY it lands — the technique doing the work (comedic timing, irony, subversion of expectation, escalation, deadpan intonation, relatability, a trending-sound drop synced to the visual). Weigh the AUDIO here: on TikTok/Reels the trending sound or music bed is frequently the single biggest driver of the emotional hit, so if a recognizable trending sound, a beat drop, or a music swell is doing the work, name it as the mechanism.
+  - "humor": if it's funny, state what the JOKE actually is and why it works, concretely enough that a remixer can rebuild the same joke with new subject matter; write "none" if the format is not comedic.
+  - "delivery": the performance cues to preserve so the vibe survives — speech intonation / cadence (from the AUDIO TRANSCRIPT), the pause held before the punchline, a facial beat or reaction, and especially the SOUND: the music / trending-audio bed, where the beat drop or lyric hit lands relative to the cut, a music swell, an sfx sting, the deadpan-then-cut. This is what keeps intonation, comedic timing, and the audio-driven vibe from being lost in a remix.
+  - "preserve": 2-4 emotional beats a remix MUST keep to reproduce the same feeling (e.g. "keep the beat drop landing exactly on the reveal cut", "hold the silent beat before the reveal", "keep the flat unbothered delivery", "the payoff must undercut the buildup, not confirm it").
+  Use BOTH the frames and the AUDIO TRANSCRIPT block above. Never leave emotional_punch empty — infer the feeling from the frames, captions, and pacing even when there is no audio, and when the audio is music-only, treat the sound itself as the emotional engine.
 - "preserve": 3-5 concrete elements a remixer MUST keep to preserve the format (e.g., "Pinned comment stays on-screen the whole video", "Wedding dance payoff shot").
 - "avoid": 3-5 things that would break the format if changed (e.g., "Do not remove the on-screen text overlay", "Do not slow the cuts below 1.5s").
 - "promotions": 3-5 categories of product / service / offer this exact format could be reused to promote, WITHOUT changing the beat structure or hook mechanic. Each item is a concrete pitchable use-case, not a genre. Examples: "SaaS free-trial signup (before/after workflow)", "Fitness challenge (7-day transformation)", "Local restaurant grand-opening (line-out-the-door proof)", "Ebook / course launch (result-first payoff)", "Real-estate open-house (walk-through reveal)". Anchor each promotion to the same viral tactic captured in "hook" — do not invent unrelated verticals.
@@ -1737,9 +1780,31 @@ function resolveFontSize(input) {
     }
     return normalizeFontSize(input.fontSize, input.canvasHeight);
 }
-const DEFAULT_VIRAL_DNA = {
-    trend_tagline: "", hook: "", retention: "", payoff: "", preserve: [], avoid: [], promotions: [], keywords: []
+const DEFAULT_EMOTIONAL_PUNCH = {
+    core_emotion: "", tone: "", arc: "", peak_moment: "", mechanism: "", humor: "", delivery: "", preserve: []
+};
+export const DEFAULT_VIRAL_DNA = {
+    trend_tagline: "", hook: "", retention: "", payoff: "", emotional_punch: { ...DEFAULT_EMOTIONAL_PUNCH }, preserve: [], avoid: [], promotions: [], keywords: []
 };
+function normalizeEmotionalPunch(value) {
+    if (!value || typeof value !== "object")
+        return { ...DEFAULT_EMOTIONAL_PUNCH };
+    const record = value;
+    const asString = (v) => typeof v === "string" ? v.trim() : "";
+    const asStringArray = (v) => Array.isArray(v)
+        ? v.map((item) => (typeof item === "string" ? item.trim() : "")).filter((item) => item.length > 0)
+        : [];
+    return {
+        core_emotion: asString(record.core_emotion ?? record.coreEmotion),
+        tone: asString(record.tone),
+        arc: asString(record.arc ?? record.emotional_arc),
+        peak_moment: asString(record.peak_moment ?? record.peakMoment),
+        mechanism: asString(record.mechanism),
+        humor: asString(record.humor),
+        delivery: asString(record.delivery),
+        preserve: asStringArray(record.preserve).slice(0, 4)
+    };
+}
 function normalizeViralDna(value) {
     if (!value || typeof value !== "object")
         return DEFAULT_VIRAL_DNA;
@@ -1753,6 +1818,7 @@ function normalizeViralDna(value) {
         hook: asString(record.hook),
         retention: asString(record.retention),
         payoff: asString(record.payoff),
+        emotional_punch: normalizeEmotionalPunch(record.emotional_punch ?? record.emotionalPunch),
         preserve: asStringArray(record.preserve),
         avoid: asStringArray(record.avoid),
         promotions: asStringArray(record.promotions),
@@ -1775,6 +1841,7 @@ export const DEFAULT_EDITOR_HARNESS = {
     broll: { reliance: "none", sourcing: "", shot_kinds: [], cadence: "" },
     transitions: { default: "cut", intro: "", outro: "", usage: "" },
     audio: { voiceover: "", music: "", sfx: "", captions_from: "" },
+    emotional: { target_feeling: "", tone: "", comedic_timing: "", intonation: "", vibe_anchors: [] },
     scenes: [],
     important_scenes: [],
     editing_bias: [],
@@ -1797,12 +1864,13 @@ function buildEditorHarnessPrompt(input) {
         : "";
     const roleVocab = SCENE_ROLE_VOCAB.join(" | ");
     const formatVocab = CONTENT_FORMATS.join(" | ");
+    const audioDigest = buildAudioDigestForPrompt(input.transcript ?? null);
     return `You are Vidfarm's EDITING DIRECTOR. You are studying a short-form video to write an EDITOR HARNESS — a compact, technical brief that lets another AI editor recreate this video's STYLE (not its exact content) in one clean pass on a timeline editor (scenes, captions, audio, transitions, keyframes).
 
 This is DIFFERENT from "viral DNA" (why the video works / what to preserve). The harness is HOW to edit: pace, typography, b-roll usage, transitions, audio bed, and the role each scene plays. Be concrete and directive — every field should be something an editor can act on, not vibes.
 
 Video duration: ${input.durationSeconds.toFixed(2)} seconds.
-${frameListing}${cutListing}${promptHint}
+${frameListing}${cutListing}${audioDigest}${promptHint}
 Return strictly valid JSON matching this schema:
 {
   "format": string,                 // one of: ${formatVocab}
@@ -1839,6 +1907,13 @@ Return strictly valid JSON matching this schema:
     "sfx": string,                  // e.g. "whoosh on cuts" | "none"
     "captions_from": string         // "voiceover" | "on-screen text" | "none"
   },
+  "emotional": {
+    "target_feeling": string,       // the emotion the edit must reproduce (the vibe/joke that makes it land)
+    "tone": string,                 // register to hold: comedic | deadpan | earnest | dramatic | wholesome | hype | ...
+    "comedic_timing": string,       // where the beat/pause/cut sells the joke; "none" if not comedic
+    "intonation": string,           // vocal delivery/cadence to preserve on VO or on-cam speech; "none" if silent
+    "vibe_anchors": [string, ...]   // 2-4 concrete edit moves that carry the emotion
+  },
   "scenes": [{
     "role": string,                 // ${roleVocab}
     "timestamp": string,            // human range, e.g. "0:00-0:03"
@@ -1858,6 +1933,7 @@ RULES:
 - Ground pacing numbers on the detected cuts + duration when cuts are supplied; otherwise estimate from the frames. Round avg_scene_seconds and cuts_per_10s to at most 2 decimals.
 - "scenes": 3-8 entries covering the whole runtime in order, each tied to a real moment you see. Reserve importance "critical" for the 1-3 beats that actually carry the style/retention (usually the hook + payoff).
 - Prefer the caption_style / transition / shot_kind vocabulary above so the editor can map straight to presets; a free-form label is fine when nothing fits.
+- "emotional": this is HOW to edit so the FEELING survives — the vibe, the joke, the intonation, and the SOUND are the first things a remix flattens, so make these directive. Use the AUDIO TRANSCRIPT block: on TikTok/Reels the trending sound / music bed is often the biggest carrier of the feeling. "target_feeling" = the emotion the recreated edit must produce; "tone" = the register to hold; "comedic_timing" = the exact editing move that sells the joke (the beat held before a cut, the deadpan-then-punch, the sfx sting or beat-drop on the button word) or "none"; "intonation" = the vocal delivery/cadence to preserve on any VO or on-cam speech (or "none" if silent) — flag it if the speech must be re-voiced in the SAME cadence; "vibe_anchors" = 2-4 concrete edit choices that carry the emotion, and include the audio ones (e.g. "keep the trending sound and land the beat drop on the reveal cut", "swell the music on the payoff", "hold a full beat of silence before the reveal cut", "cut hard on the punch word"). Infer the feeling from frames + pacing even without audio, and when the audio is music-only treat the sound as the emotional engine; never leave it empty.
 - Keep every string tight and directive. Never invent audio you cannot infer — say "none"/"unknown" instead.
 - Output ONLY the JSON object.`;
 }
@@ -1880,6 +1956,7 @@ function normalizeEditorHarness(raw) {
     const broll = asPlainRecord(rec.broll);
     const transitions = asPlainRecord(rec.transitions);
     const audio = asPlainRecord(rec.audio);
+    const emotional = asPlainRecord(rec.emotional);
     const scenesRaw = Array.isArray(rec.scenes) ? rec.scenes : [];
     const scenes = scenesRaw
         .map((s) => {
@@ -1931,6 +2008,13 @@ function normalizeEditorHarness(raw) {
             sfx: readAnnotationString(audio.sfx),
             captions_from: readAnnotationString(audio.captions_from)
         },
+        emotional: {
+            target_feeling: readAnnotationString(emotional.target_feeling),
+            tone: readAnnotationString(emotional.tone),
+            comedic_timing: readAnnotationString(emotional.comedic_timing),
+            intonation: readAnnotationString(emotional.intonation),
+            vibe_anchors: readAnnotationStringList(emotional.vibe_anchors, 4)
+        },
         scenes,
         important_scenes: readAnnotationStringList(rec.important_scenes, 8),
         editing_bias: readAnnotationStringList(rec.editing_bias, 8),
@@ -1945,7 +2029,8 @@ async function runEditorHarnessPass(input) {
         durationSeconds: input.durationSeconds,
         frameTimestamps: input.frames.map((f) => f.timestamp),
         sceneCuts: input.sceneCuts,
-        userPrompt: input.userPrompt ?? null
+        userPrompt: input.userPrompt ?? null,
+        transcript: input.transcript ?? null
     });
     const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
     return normalizeEditorHarness(attempt.result);
@@ -2313,13 +2398,22 @@ export async function smartDecomposeVideo(input) {
         });
         // Sparse frames for the vision pass (scenes, captions, style, viral_dna)
         // and dense frames for the OCR pass (deterministic caption timing).
-        // Extract both up front so the OCR API calls can run in parallel with
-        // the main vision call.
-        const [frames, ocrRawFrames, audioPath, sceneCuts] = await Promise.all([
+        // Phase 1 — deterministic ffmpeg/whisper work up front. Transcription is
+        // pulled INTO this phase (chained off the audio extract) so the transcript
+        // is ready BEFORE the vision + harness passes run: emotional punch on
+        // short-form video is frequently carried by the AUDIO (a trending sound, a
+        // beat drop, vocal intonation), and the silent frames alone can't capture
+        // it. The transcript overlaps with frame-sampling/OCR/cut-detection here
+        // instead of with the vision call, so this stays roughly latency-neutral.
+        const [frames, ocrRawFrames, transcript, sceneCuts] = await Promise.all([
             sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir),
             extractDenseOcrFrames(sourcePath, meta.durationSeconds, tempDir),
-            extractAudioAsWav(sourcePath, tempDir).catch((err) => {
-                console.error("smart decompose audio extract failed", err instanceof Error ? err.message : err);
+            extractAudioAsWav(sourcePath, tempDir)
+                .then((audioPath) => audioPath
+                ? transcribeAudioWithFallback({ audioPath, durationSeconds: meta.durationSeconds, keys: input.keys })
+                : null)
+                .catch((err) => {
+                console.error("smart decompose audio extract/transcription failed", err instanceof Error ? err.message : err);
                 return null;
             }),
             detectSceneCuts(sourcePath, meta.durationSeconds, tempDir).catch((err) => {
@@ -2330,7 +2424,7 @@ export async function smartDecomposeVideo(input) {
         devLog("hyperframes.smart_decompose.frames_sampled", {
             frame_count: frames.length,
             ocr_frame_count: ocrRawFrames.length,
-            has_audio_track: Boolean(audioPath),
+            has_audio_track: Boolean(transcript),
             scene_cut_count: sceneCuts.length
         });
         const prompt = buildSmartDecomposePrompt({
@@ -2340,14 +2434,16 @@ export async function smartDecomposeVideo(input) {
             userPrompt: input.userPrompt ?? null,
             trendTaglineHint: input.trendTaglineHint ?? null,
             frameTimestamps: frames.map((f) => f.timestamp),
-            sceneCuts
+            sceneCuts,
+            transcript
         });
-        // Vision, OCR, transcription, and the product-placement scout run
-        // concurrently. OCR, transcription, and placement failures are all
-        // non-fatal. The placement call is DELIBERATELY a separate request from the
-        // caption/scene vision call so it can never dilute timeline-layer quality —
-        // it just reuses the same already-sampled sparse frames (no extra ffmpeg).
-        const [attempt, ocrFrames, transcript, placementSurfaces, editorHarness] = await Promise.all([
+        // Phase 2 — vision, OCR, product-placement, and the editor harness run
+        // concurrently, all now transcript-aware. OCR and placement failures are
+        // non-fatal. The placement + harness calls are DELIBERATELY separate
+        // requests from the caption/scene vision call so they can never dilute
+        // timeline-layer quality — they reuse the same sparse frames (no extra
+        // ffmpeg) plus the transcript from phase 1.
+        const [attempt, ocrFrames, placementSurfaces, editorHarness] = await Promise.all([
             callVisionWithFallback({
                 prompt,
                 frames,
@@ -2357,12 +2453,6 @@ export async function smartDecomposeVideo(input) {
                 console.error("smart decompose OCR pass failed", err instanceof Error ? err.message : err);
                 return [];
             }),
-            audioPath
-                ? transcribeAudioWithFallback({ audioPath, durationSeconds: meta.durationSeconds, keys: input.keys }).catch((err) => {
-                    console.error("smart decompose transcription failed", err instanceof Error ? err.message : err);
-                    return null;
-                })
-                : Promise.resolve(null),
             runPlacementSurfacePass({
                 frames,
                 keys: input.keys,
@@ -2373,13 +2463,14 @@ export async function smartDecomposeVideo(input) {
                 return [];
             }),
             // Editor harness — technical editing direction. Isolated call in the same
-            // burst (reuses these frames + the detected cuts); non-fatal.
+            // burst (reuses these frames + the detected cuts + the transcript); non-fatal.
             runEditorHarnessPass({
                 frames,
                 keys: input.keys,
                 durationSeconds: meta.durationSeconds,
                 sceneCuts,
-                userPrompt: input.userPrompt ?? null
+                userPrompt: input.userPrompt ?? null,
+                transcript
             }).catch((err) => {
                 console.error("smart decompose editor-harness pass failed", err instanceof Error ? err.message : err);
                 return DEFAULT_EDITOR_HARNESS;
@@ -2588,6 +2679,49 @@ export function buildSmartDecomposedHtml(input) {
 </body>
 </html>`;
 }
+export function buildSmartDecomposedData(input) {
+    const { sourceUrl, result } = input;
+    const durationSeconds = Number(result.durationSeconds.toFixed(3));
+    return {
+        summary: result.summary,
+        scenes: result.scenes.map((scene) => ({
+            slug: scene.slug,
+            start: scene.start,
+            duration: scene.duration,
+            label: scene.label,
+            description: scene.description,
+            viral_note: scene.viral_note
+        })),
+        captions: result.captions.map((caption) => ({
+            slug: caption.slug,
+            text: caption.text,
+            start: caption.start,
+            duration: caption.duration,
+            x: caption.x,
+            y: caption.y,
+            width: caption.width,
+            height: caption.height,
+            color: caption.color,
+            background: caption.background,
+            background_style: caption.background_style,
+            font_family: caption.font_family,
+            font_size: caption.font_size,
+            font_weight: caption.font_weight,
+            viral_note: caption.viral_note
+        })),
+        viralDna: result.viralDna,
+        width: result.width,
+        height: result.height,
+        durationSeconds,
+        source_assets: [{
+                type: "video",
+                url: sourceUrl,
+                width: result.width,
+                height: result.height,
+                duration: durationSeconds
+            }]
+    };
+}
 // The public placeholder tile every blank scene starts on — a gray diagonal-
 // striped 720×1280 image with a dashed text box (uploaded once by
 // scripts/upload-scene-placeholder.ts). Hardcoded to the prod bucket URL (a
@@ -3113,7 +3247,7 @@ async function resolveHyperframesStack() {
     return value;
 }
 async function stageProject(input) {
-    const root = path.join(config.VIDFARM_DATA_DIR, "hyperframes-projects");
+    const root = path.join(config.VIDFARM_DATA_DIR, "vidfarm-projects");
     const projectDir = path.join(root, `${Date.now().toString(36)}-${safeIdentifier(input.compositionId)}-${randomUUID().slice(0, 8)}`);
     await rm(projectDir, { recursive: true, force: true });
     await mkdir(projectDir, { recursive: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mevdragon/vidfarm-devcli",
-  "version": "0.20.1",
+  "version": "0.20.3",
   "description": "Local bridge for the Vidfarm Trackpad Editor. `vidfarm serve <template_id>` boots the FULL editor on localhost (disk-backed records/storage, free in-process render); edit composition.html on disk (Claude Code, Codex, etc.) and the browser live-morphs it.",
   "type": "module",
   "bin": {