@lalalic/markcut 3.1.1 → 3.2.1

package/tests/evals/gen_dataset.py

@@ -0,0 +1,63 @@
+"""Generate the markcut storyboard goldens with a token-capped OpenRouter model.
+
+Run:
+  OPENROUTER_API_KEY=... .venv-evals/bin/python tests/evals/gen_dataset.py
+"""
+
+import os
+
+from deepeval.synthesizer import Synthesizer
+from deepeval.synthesizer.config import StylingConfig
+
+from openrouter_model import OpenRouterLLM
+
+SCENARIO = (
+    "Video creators and AI agents using the markcut markdown-to-video skill "
+    "to author video storyboards from a brief. markcut uses a markdown "
+    "descriptive format: '## scene' headings, '- image prompt:...' bullets, "
+    "script \"...\" narration, isBackground:true for visuals under scripted "
+    "scenes, map stream type with view/tween camera moves for route vlogs, "
+    "built-in components, TTS/TTI/STT media pipeline, and golden rules such "
+    "as never set duration on scripted scenes, keep manual assets in the "
+    "assets/ folder, and review md then compiled json then rendered video."
+)
+TASK = (
+    "Act as a storyboard author: given a video brief, produce a markcut "
+    "storyboard in markdown descriptive format that follows the skill's "
+    "scene/isBackground/duration rules and narrative structure (hook, "
+    "conflict, resolution, emotion, call to action)."
+)
+INPUT_FORMAT = (
+    "A short video brief, e.g. 'a 30s travel vlog of a coastal route with "
+    "narration', 'a product launch teaser with stats', 'a two-speaker "
+    "dialogue explainer', 'a recipe short with background music'."
+)
+EXPECTED_OUTPUT_FORMAT = (
+    "A complete markcut storyboard markdown using ## scene headings, "
+    "- image/script bullets, correct isBackground usage, and no manual "
+    "durations on scripted scenes."
+)
+
+
+def main():
+    model = OpenRouterLLM()
+    synthesizer = Synthesizer(
+        model=model,
+        styling_config=StylingConfig(
+            scenario=SCENARIO,
+            task=TASK,
+            input_format=INPUT_FORMAT,
+            expected_output_format=EXPECTED_OUTPUT_FORMAT,
+        ),
+    )
+    goldens = synthesizer.generate_goldens_from_scratch(num_goldens=12)
+    synthesizer.save_as(
+        file_type="json",
+        directory="tests/evals",
+        file_name="dataset",
+    )
+    print(f"Saved {len(goldens)} goldens to tests/evals/dataset.json")
+
+
+if __name__ == "__main__":
+    main()

package/tests/evals/metrics.py

@@ -0,0 +1,70 @@
+"""DeepEval metrics for the markcut storyboard eval suite.
+
+Judge model is the token-capped OpenRouter wrapper so the whole suite
+runs on the workspace's free OpenRouter key.
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+from deepeval.metrics import GEval, TaskCompletionMetric
+from deepeval.test_case import LLMTestCaseParams
+
+from openrouter_model import OpenRouterLLM
+
+
+def _judge():
+    return OpenRouterLLM(temperature=0.0)
+
+
+# Trace-level: did the agent finish the storyboard task at all?
+TASK_COMPLETION_METRICS = [
+    TaskCompletionMetric(model=_judge(), threshold=0.7),
+]
+
+# Is the storyboard valid markcut markdown following the skill's rules?
+STORYBOARD_FORMAT_METRIC = GEval(
+    name="Storyboard Format",
+    criteria=(
+        "Evaluate whether the actual output is a valid markcut storyboard "
+        "in markdown descriptive format. It must: use '## scene' headings "
+        "for scenes; use '- image prompt:\"...\"' or similar bullet syntax "
+        "for visuals; include narration via script \"...\"; set "
+        "isBackground:true on the primary visual of any scene that has a "
+        "script or audio (otherwise the scene plays black); and NOT set "
+        "manual durations on scripted scenes since the resolver derives "
+        "duration from the audio script."
+    ),
+    evaluation_params=[
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ],
+    model=_judge(),
+    threshold=0.7,
+)
+
+# Does the storyboard tell a compelling viral story?
+STORY_NARRATIVE_METRIC = GEval(
+    name="Story Narrative",
+    criteria=(
+        "Evaluate whether the storyboard tells a compelling short video "
+        "story. It should contain a hook (why watch), conflict or "
+        "challenge, resolution, emotional beats, a call to action, and "
+        "ideally an open ending where appropriate. Score low if it is a "
+        "flat list of scenes with no narrative structure."
+    ),
+    evaluation_params=[
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ],
+    model=_judge(),
+    threshold=0.6,
+)
+
+SINGLE_TURN_TRACE_METRICS = [
+    TASK_COMPLETION_METRICS[0],
+    STORYBOARD_FORMAT_METRIC,
+    STORY_NARRATIVE_METRIC,
+]

package/tests/evals/openrouter_model.py

@@ -0,0 +1,143 @@
+"""OpenRouter-backed DeepEval LLM with a capped max_tokens budget.
+
+The workspace OPENROUTER_API_KEY has a daily credit cap that rejects
+requests asking for 65536 max_tokens, so we wrap the OpenAI-compatible
+client ourselves and ask for far fewer output tokens.
+"""
+
+import os
+from typing import Optional, List
+
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.synthesizer.schema import (
+    Response as _ResponseSchema,
+    SyntheticData,
+    SyntheticDataList,
+)
+from deepeval.metrics.utils import trimAndLoadJson
+
+MAX_OUTPUT_TOKENS = 16000
+
+# Free-tier models are flaky; rotate through candidates on rate limits.
+FALLBACK_MODELS = [
+    "google/gemma-4-31b-it:free",
+    "nvidia/nemotron-3-super-120b-a12b:free",
+    "inclusionai/ling-3.0-flash-sante:free",
+    "minimax/minimax-m2.7:free",
+]
+
+
+class OpenRouterLLM(DeepEvalBaseLLM):
+    def __init__(
+        self,
+        model: str = "google/gemma-4-31b-it:free",
+        temperature: float = 0.0,
+        **kwargs,
+    ):
+        self.model_id = model
+        self.temperature = temperature
+        super().__init__(model, **kwargs)
+
+    def load_model(self):
+        from openai import OpenAI
+
+        return OpenAI(
+            api_key=os.environ["OPENROUTER_API_KEY"],
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    def generate(
+        self,
+        prompt: str,
+        schema=None,
+        max_output_tokens: int = MAX_OUTPUT_TOKENS,
+    ) -> str:
+        client = self.load_model()
+        if schema is not None:
+            # Sending the raw JSON schema makes weak models echo it back;
+            # describe the shape in prose instead.
+            if schema is SyntheticDataList:
+                shape_desc = (
+                    'a JSON object {"data": [{"input": "<brief text>"}]} '
+                    "with as many items as requested"
+                )
+            elif schema is _ResponseSchema:
+                shape_desc = 'a JSON object {"response": "<your answer>"}'
+            else:
+                shape_desc = (
+                    "a single JSON object with exactly these keys: "
+                    f"{list(schema.model_fields.keys())}"
+                )
+            prompt = (
+                "You are a JSON generator. Respond with ONLY one valid JSON "
+                "value - no prose, no markdown fences, no commentary, and "
+                "never repeat the schema itself. The JSON must be exactly "
+                f"{shape_desc}.\n\nTask:\n{prompt}"
+            )
+
+        import time
+
+        text = None
+        last_err = None
+        for model_id in [self.model_id] + [
+            m for m in FALLBACK_MODELS if m != self.model_id
+        ]:
+            for attempt in range(3):
+                try:
+                    response = client.chat.completions.create(
+                        model=model_id,
+                        messages=[{"role": "user", "content": prompt}],
+                        temperature=self.temperature,
+                        max_tokens=min(max_output_tokens, MAX_OUTPUT_TOKENS),
+                    )
+                    text = response.choices[0].message.content or ""
+                    break
+                except Exception as err:  # rate limits, upstream 429s
+                    last_err = err
+                    time.sleep(5 * (attempt + 1))
+            if text:
+                break
+        if not text:
+            raise last_err
+        if schema is None:
+            return text
+        # Parse the JSON the prompt asked for into the requested schema
+        try:
+            data = trimAndLoadJson(text, self)
+        except ValueError:
+            with open("/tmp/deepeval_last_output.txt", "w") as f:
+                f.write(text)
+            raise
+        if schema is SyntheticDataList:
+            return SyntheticDataList(
+                data=[SyntheticData(**item) for item in data["data"]]
+            )
+        if schema is _ResponseSchema and isinstance(data, str):
+            return _ResponseSchema(response=data)
+        return schema(**data)
+
+    async def a_generate(
+        self,
+        prompt: str,
+        schema=None,
+        max_output_tokens: int = MAX_OUTPUT_TOKENS,
+    ) -> str:
+        return self.generate(prompt, schema, max_output_tokens)
+
+    def get_model_name(self) -> str:
+        return self.model_id
+
+    def supports_json_mode(self) -> bool:
+        return False
+
+    def supports_structured_outputs(self) -> bool:
+        return False
+
+    def supports_log_probs(self) -> Optional[bool]:
+        return False
+
+    def supports_multimodal(self) -> bool:
+        return False
+
+    def batch_generate(self, prompts: List[str]) -> List[str]:
+        return [self.generate(p) for p in prompts]

package/tests/evals/storyboard_app.py

@@ -0,0 +1,55 @@
+"""Traced markcut storyboard authoring app.
+
+Runs the pi agent CLI with the markcut skill against a video brief and
+returns the produced storyboard markdown. The @observe decorator makes
+the run a DeepEval agent trace so traced single-turn evals can score it.
+"""
+
+import os
+import subprocess
+import tempfile
+
+from deepeval.tracing import observe
+
+MARKCUT_SKILL = os.path.abspath(
+    os.path.join(
+        os.path.dirname(__file__), "..", "..", "skills", "markcut", "SKILL.md"
+    )
+)
+
+
+def _read_skill() -> str:
+    with open(MARKCUT_SKILL) as f:
+        return f.read()
+
+
+@observe(type="agent", name="markcut-storyboard-author")
+def run_traced_storyboard(brief: str) -> str:
+    """Author a markcut storyboard markdown for the given brief."""
+    with tempfile.TemporaryDirectory() as workdir:
+        prompt = (
+            "You are a video storyboard author using the markcut skill.\n"
+            f"Video brief: {brief}\n\n"
+            "Write the storyboard as a markcut markdown file named "
+            "storyboard.md in the current directory. Follow the skill's "
+            "rules strictly. When done, print ONLY the final storyboard "
+            "markdown to stdout with no extra commentary."
+        )
+        result = subprocess.run(
+            [
+                "pi",
+                "-p",
+                "--no-session",
+                "--skill", MARKCUT_SKILL,
+                prompt,
+            ],
+            capture_output=True,
+            text=True,
+            timeout=600,
+            cwd=workdir,
+        )
+        storyboard_path = os.path.join(workdir, "storyboard.md")
+        if os.path.exists(storyboard_path):
+            with open(storyboard_path) as f:
+                return f.read()
+        return result.stdout

package/tests/evals/test_storyboard.py

@@ -0,0 +1,32 @@
+"""Traced single-turn evals for the markcut storyboard authoring agent.
+
+Each golden's input is a video brief; the app runs the pi agent CLI with
+the markcut skill to author a storyboard markdown, traced with @observe.
+
+Run:
+  cd tests/evals && ../../.venv-evals/bin/deepeval test run test_storyboard.py
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+import pytest
+from deepeval import assert_test
+from deepeval.dataset import EvaluationDataset, Golden
+
+from metrics import SINGLE_TURN_TRACE_METRICS
+from storyboard_app import run_traced_storyboard
+
+
+dataset = EvaluationDataset()
+dataset.add_goldens_from_json_file(
+    file_path=os.path.join(os.path.dirname(__file__), "dataset.json")
+)
+
+
+@pytest.mark.parametrize("golden", dataset.goldens)
+def test_storyboard_authoring(golden: Golden):
+    run_traced_storyboard(golden.input)
+    assert_test(golden=golden, metrics=SINGLE_TURN_TRACE_METRICS)

package/.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-    "copilot-infinite.discord.enabled": true
-}