@lalalic/markcut 3.1.0 → 3.2.0

package/tests/evals/metrics.py

@@ -0,0 +1,70 @@
+"""DeepEval metrics for the markcut storyboard eval suite.
+
+Judge model is the token-capped OpenRouter wrapper so the whole suite
+runs on the workspace's free OpenRouter key.
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+from deepeval.metrics import GEval, TaskCompletionMetric
+from deepeval.test_case import LLMTestCaseParams
+
+from openrouter_model import OpenRouterLLM
+
+
+def _judge():
+    return OpenRouterLLM(temperature=0.0)
+
+
+# Trace-level: did the agent finish the storyboard task at all?
+TASK_COMPLETION_METRICS = [
+    TaskCompletionMetric(model=_judge(), threshold=0.7),
+]
+
+# Is the storyboard valid markcut markdown following the skill's rules?
+STORYBOARD_FORMAT_METRIC = GEval(
+    name="Storyboard Format",
+    criteria=(
+        "Evaluate whether the actual output is a valid markcut storyboard "
+        "in markdown descriptive format. It must: use '## scene' headings "
+        "for scenes; use '- image prompt:\"...\"' or similar bullet syntax "
+        "for visuals; include narration via script \"...\"; set "
+        "isBackground:true on the primary visual of any scene that has a "
+        "script or audio (otherwise the scene plays black); and NOT set "
+        "manual durations on scripted scenes since the resolver derives "
+        "duration from the audio script."
+    ),
+    evaluation_params=[
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ],
+    model=_judge(),
+    threshold=0.7,
+)
+
+# Does the storyboard tell a compelling viral story?
+STORY_NARRATIVE_METRIC = GEval(
+    name="Story Narrative",
+    criteria=(
+        "Evaluate whether the storyboard tells a compelling short video "
+        "story. It should contain a hook (why watch), conflict or "
+        "challenge, resolution, emotional beats, a call to action, and "
+        "ideally an open ending where appropriate. Score low if it is a "
+        "flat list of scenes with no narrative structure."
+    ),
+    evaluation_params=[
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ],
+    model=_judge(),
+    threshold=0.6,
+)
+
+SINGLE_TURN_TRACE_METRICS = [
+    TASK_COMPLETION_METRICS[0],
+    STORYBOARD_FORMAT_METRIC,
+    STORY_NARRATIVE_METRIC,
+]

package/tests/evals/openrouter_model.py

@@ -0,0 +1,143 @@
+"""OpenRouter-backed DeepEval LLM with a capped max_tokens budget.
+
+The workspace OPENROUTER_API_KEY has a daily credit cap that rejects
+requests asking for 65536 max_tokens, so we wrap the OpenAI-compatible
+client ourselves and ask for far fewer output tokens.
+"""
+
+import os
+from typing import Optional, List
+
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.synthesizer.schema import (
+    Response as _ResponseSchema,
+    SyntheticData,
+    SyntheticDataList,
+)
+from deepeval.metrics.utils import trimAndLoadJson
+
+MAX_OUTPUT_TOKENS = 16000
+
+# Free-tier models are flaky; rotate through candidates on rate limits.
+FALLBACK_MODELS = [
+    "google/gemma-4-31b-it:free",
+    "nvidia/nemotron-3-super-120b-a12b:free",
+    "inclusionai/ling-3.0-flash-sante:free",
+    "minimax/minimax-m2.7:free",
+]
+
+
+class OpenRouterLLM(DeepEvalBaseLLM):
+    def __init__(
+        self,
+        model: str = "google/gemma-4-31b-it:free",
+        temperature: float = 0.0,
+        **kwargs,
+    ):
+        self.model_id = model
+        self.temperature = temperature
+        super().__init__(model, **kwargs)
+
+    def load_model(self):
+        from openai import OpenAI
+
+        return OpenAI(
+            api_key=os.environ["OPENROUTER_API_KEY"],
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    def generate(
+        self,
+        prompt: str,
+        schema=None,
+        max_output_tokens: int = MAX_OUTPUT_TOKENS,
+    ) -> str:
+        client = self.load_model()
+        if schema is not None:
+            # Sending the raw JSON schema makes weak models echo it back;
+            # describe the shape in prose instead.
+            if schema is SyntheticDataList:
+                shape_desc = (
+                    'a JSON object {"data": [{"input": "<brief text>"}]} '
+                    "with as many items as requested"
+                )
+            elif schema is _ResponseSchema:
+                shape_desc = 'a JSON object {"response": "<your answer>"}'
+            else:
+                shape_desc = (
+                    "a single JSON object with exactly these keys: "
+                    f"{list(schema.model_fields.keys())}"
+                )
+            prompt = (
+                "You are a JSON generator. Respond with ONLY one valid JSON "
+                "value - no prose, no markdown fences, no commentary, and "
+                "never repeat the schema itself. The JSON must be exactly "
+                f"{shape_desc}.\n\nTask:\n{prompt}"
+            )
+
+        import time
+
+        text = None
+        last_err = None
+        for model_id in [self.model_id] + [
+            m for m in FALLBACK_MODELS if m != self.model_id
+        ]:
+            for attempt in range(3):
+                try:
+                    response = client.chat.completions.create(
+                        model=model_id,
+                        messages=[{"role": "user", "content": prompt}],
+                        temperature=self.temperature,
+                        max_tokens=min(max_output_tokens, MAX_OUTPUT_TOKENS),
+                    )
+                    text = response.choices[0].message.content or ""
+                    break
+                except Exception as err:  # rate limits, upstream 429s
+                    last_err = err
+                    time.sleep(5 * (attempt + 1))
+            if text:
+                break
+        if not text:
+            raise last_err
+        if schema is None:
+            return text
+        # Parse the JSON the prompt asked for into the requested schema
+        try:
+            data = trimAndLoadJson(text, self)
+        except ValueError:
+            with open("/tmp/deepeval_last_output.txt", "w") as f:
+                f.write(text)
+            raise
+        if schema is SyntheticDataList:
+            return SyntheticDataList(
+                data=[SyntheticData(**item) for item in data["data"]]
+            )
+        if schema is _ResponseSchema and isinstance(data, str):
+            return _ResponseSchema(response=data)
+        return schema(**data)
+
+    async def a_generate(
+        self,
+        prompt: str,
+        schema=None,
+        max_output_tokens: int = MAX_OUTPUT_TOKENS,
+    ) -> str:
+        return self.generate(prompt, schema, max_output_tokens)
+
+    def get_model_name(self) -> str:
+        return self.model_id
+
+    def supports_json_mode(self) -> bool:
+        return False
+
+    def supports_structured_outputs(self) -> bool:
+        return False
+
+    def supports_log_probs(self) -> Optional[bool]:
+        return False
+
+    def supports_multimodal(self) -> bool:
+        return False
+
+    def batch_generate(self, prompts: List[str]) -> List[str]:
+        return [self.generate(p) for p in prompts]

package/tests/evals/storyboard_app.py

@@ -0,0 +1,55 @@
+"""Traced markcut storyboard authoring app.
+
+Runs the pi agent CLI with the markcut skill against a video brief and
+returns the produced storyboard markdown. The @observe decorator makes
+the run a DeepEval agent trace so traced single-turn evals can score it.
+"""
+
+import os
+import subprocess
+import tempfile
+
+from deepeval.tracing import observe
+
+MARKCUT_SKILL = os.path.abspath(
+    os.path.join(
+        os.path.dirname(__file__), "..", "..", "skills", "markcut", "SKILL.md"
+    )
+)
+
+
+def _read_skill() -> str:
+    with open(MARKCUT_SKILL) as f:
+        return f.read()
+
+
+@observe(type="agent", name="markcut-storyboard-author")
+def run_traced_storyboard(brief: str) -> str:
+    """Author a markcut storyboard markdown for the given brief."""
+    with tempfile.TemporaryDirectory() as workdir:
+        prompt = (
+            "You are a video storyboard author using the markcut skill.\n"
+            f"Video brief: {brief}\n\n"
+            "Write the storyboard as a markcut markdown file named "
+            "storyboard.md in the current directory. Follow the skill's "
+            "rules strictly. When done, print ONLY the final storyboard "
+            "markdown to stdout with no extra commentary."
+        )
+        result = subprocess.run(
+            [
+                "pi",
+                "-p",
+                "--no-session",
+                "--skill", MARKCUT_SKILL,
+                prompt,
+            ],
+            capture_output=True,
+            text=True,
+            timeout=600,
+            cwd=workdir,
+        )
+        storyboard_path = os.path.join(workdir, "storyboard.md")
+        if os.path.exists(storyboard_path):
+            with open(storyboard_path) as f:
+                return f.read()
+        return result.stdout

package/tests/evals/test_storyboard.py

@@ -0,0 +1,32 @@
+"""Traced single-turn evals for the markcut storyboard authoring agent.
+
+Each golden's input is a video brief; the app runs the pi agent CLI with
+the markcut skill to author a storyboard markdown, traced with @observe.
+
+Run:
+  cd tests/evals && ../../.venv-evals/bin/deepeval test run test_storyboard.py
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+import pytest
+from deepeval import assert_test
+from deepeval.dataset import EvaluationDataset, Golden
+
+from metrics import SINGLE_TURN_TRACE_METRICS
+from storyboard_app import run_traced_storyboard
+
+
+dataset = EvaluationDataset()
+dataset.add_goldens_from_json_file(
+    file_path=os.path.join(os.path.dirname(__file__), "dataset.json")
+)
+
+
+@pytest.mark.parametrize("golden", dataset.goldens)
+def test_storyboard_authoring(golden: Golden):
+    run_traced_storyboard(golden.input)
+    assert_test(golden=golden, metrics=SINGLE_TURN_TRACE_METRICS)

package/tests/fixtures/map-overlay.json

@@ -0,0 +1,56 @@
+{
+  "root": {
+    "type": "root",
+    "width": 640,
+    "height": 480,
+    "fps": 30,
+    "layout": "series",
+    "children": [
+      {
+        "type": "map",
+        "id": "map-overlay-test",
+        "view": "route",
+        "travelMode": "DRIVING",
+        "mapType": "roadmap",
+        "routeColor": "#4285F4",
+        "routeWeight": 5,
+        "waypoints": [
+          {
+            "lat": 37.8199,
+            "lng": -122.4783,
+            "label": "Golden Gate"
+          },
+          {
+            "lat": 37.6213,
+            "lng": -122.379,
+            "label": "SFO"
+          }
+        ],
+        "start": 0,
+        "end": 12,
+        "duration": 12,
+        "children": [
+          {
+            "type": "effect",
+            "animation": "zoomIn",
+            "animationIterationCount": 1,
+            "start": 0,
+            "end": 4,
+            "at": "Golden Gate",
+            "children": [
+              {
+                "type": "image",
+                "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAACXBIWXMAAAAAAAAAAQCEeRdzAAAAYUlEQVR4nO3PwQkAIBDAMAX3X/kcwkcQmgnaPWvWz44OeNWA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaAdgFCjAL8Lb/n7gAAAABJRU5ErkJggg==",
+                "fit": "contain",
+                "start": 0,
+                "end": 4,
+                "at": "Golden Gate"
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "seed": 592821268
+  }
+}

package/tests/fixtures/md/map-all-views.md

@@ -12,6 +12,13 @@ layout:parallel
 - script "The route winds from the Golden Gate to the airport, with photos at each stop."
 - map view:route duration:6 travelMode:DRIVING mapType:roadmap routeColor:"#4285F4" routeWeight:5 routeMarker:"🚗" waypoints:[37.8199,-122.4783,"Golden Gate","https://picsum.photos/seed/gg-bridge/96/96"; 37.7749,-122.4194,"Civic Center","https://picsum.photos/seed/civic-center/96/96"; 37.6213,-122.3790,"SFO","https://picsum.photos/seed/sfo-airport/96/96"]
 
+## Stops
+layout:parallel
+- script "The pin pauses at each landmark while a photo grows out of it."
+- map view:route duration:16 travelMode:DRIVING mapType:roadmap routeColor:"#4285F4" routeWeight:5 routeMarker:"🚗" waypoints:[37.8199,-122.4783,"Golden Gate"; 37.7749,-122.4194,"Civic Center"; 37.6213,-122.3790,"SFO"]
+  - image src:https://picsum.photos/seed/gg-photo/200/200 at:"Golden Gate" duration:3 effects:[zoomIn]
+  - image src:https://picsum.photos/seed/civic-photo/200/200 at:"Civic Center" duration:3 effects:[zoomIn]
+
 ## Cinematic
 layout:parallel
 - script "The camera tilts and chases the road like a drone."
@@ -25,4 +32,4 @@ layout:parallel
 ## Street-View-Walk
 layout:parallel
 - script "A quick walk down the block."
-- map view:streetview duration:6 streetView:{route:[{lat:37.7793,lng:-122.4193}, {lat:37.7785,lng:-122.4185}, {lat:37.7777,lng:-122.4178}, {lat:37.7769,lng:-122.4170}], radius:50, pov:{heading:tween(0, 40, easeInOut), pitch:-5}}
+- map view:streetview duration:6 streetView:{route:[{lat:37.7785,lng:-122.4185}, {lat:37.7777,lng:-122.4178}], radius:50, pov:{heading:tween(0, 40, easeInOut), pitch:-5}}

package/tests/fixtures/md/map-children.md

@@ -0,0 +1,11 @@
+# video
+seed:42
+width:640 height:480 fps:30 layout:series transition:fade transitionTime:0.5
+
+## Route-Tour
+layout:parallel
+- script "We tour from the Golden Gate to the airport, stopping at each landmark."
+- map view:route travelMode:DRIVING mapType:roadmap routeColor:"#4285F4" routeWeight:5
+  waypoints:[37.8199,-122.4783,"Golden Gate"; 37.7749,-122.4194,"Civic Center"; 37.6213,-122.3790,"SFO"]
+  - image src:https://picsum.photos/seed/gg-photo/200/200 at:"Golden Gate" duration:3 effects:[zoomIn]
+  - image src:https://picsum.photos/seed/civic-photo/200/200 at:"Civic Center" duration:3 effects:[zoomIn]

package/tests/fixtures/md/map-multimode.md

@@ -0,0 +1,9 @@
+# video
+seed:1
+width:1080 height:1920 fps:30 layout:series transition:fade transitionTime:0.5
+
+## Multi-Leg-Trip
+layout:parallel
+- script "We fly to the coast, take a boat across the bay, walk the promenade, then drive to the airport."
+- map view:route duration:12 travelMode:DRIVING mapType:roadmap routeColor:"#4285F4" routeWeight:5
+  waypoints:[37.8199,-122.4783,"SFO",FLIGHT; 33.94,-118.41,"LAX",BOAT; 33.75,-118.28,"Long Beach",WALKING; 33.77,-118.19,"Promenade",DRIVING; 33.94,-118.41,"LAX"]

package/tests/fixtures/streetview-walk.json

@@ -0,0 +1,36 @@
+{
+  "root": {
+    "type": "root",
+    "width": 640,
+    "height": 480,
+    "fps": 30,
+    "layout": "series",
+    "children": [
+      {
+        "type": "map",
+        "id": "sv-walk",
+        "view": "streetview",
+        "start": 0,
+        "end": 6,
+        "duration": 6,
+        "streetView": {
+          "route": [
+            {
+              "lat": 37.7785,
+              "lng": -122.4185
+            },
+            {
+              "lat": 37.7777,
+              "lng": -122.4178
+            }
+          ],
+          "radius": 50,
+          "pov": {
+            "heading": 15,
+            "pitch": -5
+          }
+        }
+      }
+    ]
+  }
+}

package/tests/md-descriptive.test.ts

@@ -955,6 +955,81 @@ layout:parallel
   });
 });
 
+// ── Map Multi-Leg Travel Modes ─────────────────────────────────────────────
+
+describe("map multi-leg travel modes (per-waypoint)", () => {
+  const md = `
+# video
+width:640 height:480 fps:30 layout:series
+
+## Trip
+layout:parallel
+- map view:route duration:12 travelMode:DRIVING
+  waypoints:[37.8199,-122.4783,"SFO",FLIGHT; 33.94,-118.41,"LAX",BOAT; 33.75,-118.28,"Long Beach",WALKING; 33.77,-118.19,"Promenade",DRIVING; 33.94,-118.41,"LAX"]
+`;
+
+  it("parses a per-waypoint travel mode as the 5th field", () => {
+    const parsed = parseMarkdownDescriptive(md);
+    const map = (parsed.children as any[])[0].children.find((c: any) => c.type === "map");
+    expect(map.waypoints.map((w: any) => w.mode)).toEqual([
+      "FLIGHT", "BOAT", "WALKING", "DRIVING", undefined,
+    ]);
+    expect(map.waypoints[0].media).toBeUndefined();
+    expect(map.waypoints[0].label).toBe("SFO");
+  });
+
+  it("compiles waypoint modes into the stream tree", () => {
+    const compiled = compileDescriptiveRoot(parseMarkdownDescriptive(md));
+    const map = (compiled.children as any[])[0].children.find((c: any) => c.type === "map");
+    expect(map.waypoints.map((w: any) => w.mode)).toEqual([
+      "FLIGHT", "BOAT", "WALKING", "DRIVING", undefined,
+    ]);
+  });
+});
+
+// ── Map Overlay Children (at:"Waypoint") ──────────────────────────────────
+
+describe("map overlay children (at:\"Waypoint\")", () => {
+  const md = `
+# video
+width:640 height:480 fps:30 layout:series
+
+## Tour
+layout:parallel
+- map view:route duration:12 travelMode:DRIVING
+  waypoints:[37.81,-122.48,"Golden Gate"; 37.77,-122.42,"Civic Center"]
+  - image src:gg.jpg at:"Golden Gate" start:3 duration:3 effects:[zoomIn]
+  - image src:civic.jpg at:"Civic Center" start:7 duration:3 effects:[zoomIn]
+`;
+
+  it("parses nested children with at onto the map node", () => {
+    const parsed = parseMarkdownDescriptive(md);
+    const map = (parsed.children as any[])[0].children.find((c: any) => c.type === "map");
+    expect(map.children).toHaveLength(2);
+    expect(map.children[0].type).toBe("image");
+    expect(map.children[0].at).toBe("Golden Gate");
+    expect(map.children[1].at).toBe("Civic Center");
+  });
+
+  it("compiles children into the map stream and propagates at through effects", () => {
+    const compiled = compileDescriptiveRoot(parseMarkdownDescriptive(md));
+    const map = (compiled.children as any[])[0].children.find((c: any) => c.type === "map");
+    expect(map.children).toHaveLength(2);
+    // effects:[zoomIn] wraps each child in an effect node that carries `at`
+    expect(map.children[0].type).toBe("effect");
+    expect(map.children[0].at).toBe("Golden Gate");
+    expect(map.children[0].start).toBe(3);
+    expect(map.children[1].at).toBe("Civic Center");
+  });
+
+  it("keeps the map duration from its own end, not max(children)", () => {
+    const compiled = compileDescriptiveRoot(parseMarkdownDescriptive(md));
+    const map = (compiled.children as any[])[0].children.find((c: any) => c.type === "map");
+    expect(map.durationInSeconds).toBe(12);
+    expect(map.end).toBe(12);
+  });
+});
+
 // ── Helper ────────────────────────────────────────────────────────────────
 
 function findComponents(node: any): any[] {

package/tests/render.test.ts

@@ -21,6 +21,7 @@ import {
   FIXTURES_DIR,
 } from "./utils";
 import { existsSync, rmSync, mkdirSync } from "node:fs";
+import { execSync } from "node:child_process";
 import { resolve } from "node:path";
 
 // Increase timeout for integration tests (rendering + STT can take a while)
@@ -258,6 +259,97 @@ describe("Map Rendering", () => {
     // Check file size as proxy for visual content.
     expect(getFrameFileSize(frame)).toBeGreaterThan(1000);
   });
+
+  it("renders anchored overlay children at waypoints (dwell vs drive)", async () => {
+    const output = renderFixture(fixturePath("map-overlay.json"), {
+      outputName: "map-overlay.mp4",
+      timeout: RENDER_TIMEOUT,
+    });
+
+    const info = getVideoInfo(output);
+    expect(info.width).toBe(640);
+
+    // The map has a solid-magenta photo anchored at "Golden Gate" (0–4s,
+    // zoomIn), then the pin drives to SFO. The photo must be visible during
+    // the dwell and gone during the drive — guards regressions where anchored
+    // overlays silently never render (getProjection() null / lazy effect path).
+    const dwell = outPath("frames/map-overlay-dwell.png");
+    const drive = outPath("frames/map-overlay-drive.png");
+    extractFrame(output, 2.5, dwell);
+    extractFrame(output, 6, drive);
+
+    const magentaFraction = (f: string): number => {
+      const p = execSync(
+        `ffmpeg -loglevel error -i "${f}" -vf scale=160:120 -f rawvideo -pix_fmt rgb24 pipe:1`,
+      );
+      let count = 0;
+      for (let i = 0; i < p.length; i += 3) {
+        if (p[i]! > 200 && p[i + 1]! < 80 && p[i + 2]! > 200) count++;
+      }
+      return count / (p.length / 3);
+    };
+
+    expect(magentaFraction(dwell)).toBeGreaterThan(0.02); // photo visible during dwell
+    expect(magentaFraction(drive)).toBeLessThan(0.001); // gone during drive
+  });
+
+  it("renders street view panoramas non-dark", async () => {
+    const output = renderFixture(fixturePath("map-dynamic.json"), {
+      outputName: "map-dynamic-streetview.mp4",
+      timeout: RENDER_TIMEOUT,
+    });
+
+    // map-dynamic.json is a series: dolly 0–3s, cinematic 3–11s, then the
+    // static streetview-pan scene plays ~11–17s. Its panorama must not be
+    // black — guards regressions where static Street View captures dark frames
+    // (pano_changed race + too-short fallback).
+    const frame = outPath("frames/map-dynamic-streetview.png");
+    extractFrame(output, 13, frame);
+
+    const darkFraction = (f: string): number => {
+      const p = execSync(
+        `ffmpeg -loglevel error -i "${f}" -vf scale=120:90 -f rawvideo -pix_fmt rgb24 pipe:1`,
+      );
+      let dark = 0;
+      for (let i = 0; i < p.length; i += 3) {
+        const v = (p[i]! + p[i + 1]! + p[i + 2]!) / 3;
+        if (v < 20) dark++;
+      }
+      return dark / (p.length / 3);
+    };
+
+    expect(darkFraction(frame)).toBeLessThan(0.5);
+  });
+
+  it("renders street view walk (route) non-dark at both waypoints", async () => {
+    const output = renderFixture(fixturePath("streetview-walk.json"), {
+      outputName: "streetview-walk.mp4",
+      timeout: RENDER_TIMEOUT,
+    });
+
+    // The snap-walk holds at route[0] (101 Grove St) for the first half and
+    // route[1] (95 Hayes St) for the second. Both have imagery; neither frame
+    // should be black. Guards regressions where the walk's per-frame pano
+    // requests get rate-limited (429) and render dark frames.
+    const darkFraction = (f: string): number => {
+      const p = execSync(
+        `ffmpeg -loglevel error -i "${f}" -vf scale=120:90 -f rawvideo -pix_fmt rgb24 pipe:1`,
+      );
+      let dark = 0;
+      for (let i = 0; i < p.length; i += 3) {
+        const v = (p[i]! + p[i + 1]! + p[i + 2]!) / 3;
+        if (v < 20) dark++;
+      }
+      return dark / (p.length / 3);
+    };
+
+    const wp0 = outPath("frames/streetview-walk-wp0.png");
+    const wp1 = outPath("frames/streetview-walk-wp1.png");
+    extractFrame(output, 2, wp0);
+    extractFrame(output, 4.5, wp1);
+    expect(darkFraction(wp0)).toBeLessThan(0.5);
+    expect(darkFraction(wp1)).toBeLessThan(0.5);
+  });
 });
 
 // ───────────────────────────────────────────────────────────────────────────