@lalalic/markcut 3.2.4 → 3.2.6

package/.env.example

@@ -25,3 +25,13 @@ GOOGLE_MAPS_API_KEY=your_key_here
 # MARKCUT_MAX_VIDEO_DURATION=60
 # MARKCUT_MAX_VIDEO_DIMENSION=360
 # MARKCUT_VTT_SAMPLE_INTERVAL=5
+# ── Default Vision backend policy ────────────────────────────────────────────
+# Browser ChatGPT Temporary Chat is attempted first when this command exists.
+# MARKCUT_CHATGPT_BROWSER_INFER_CLI=/path/to/chatgpt-browser-infer
+# Codex fallback profiles are tried left-to-right; `default` means no -p flag.
+# MARKCUT_CODEX_VISION_PROFILES=zai,default
+# MARKCUT_CHATGPT_VISION_TIMEOUT_MS=600000
+# MARKCUT_CODEX_VISION_TIMEOUT_MS=600000
+# Local MLX is final fallback only.
+# MARKCUT_LOCAL_VISION_MODEL=mlx-community/MiniCPM-V-4.6-bf16
+# MARKCUT_LOCAL_VISION_TIMEOUT_MS=600000

package/README.md

@@ -341,4 +341,32 @@ flowchart LR
 | `--dry-run` | Show what would be processed without running AI |
 | `--show-prompts` | Print the prompts template file and exit |
 
+### Default Vision backend priority
+
+Markcut Vision is remote-first by default so ordinary perception does not require a heavy local VLM on the host Mac. Explicit `--itt` / `--vtt` flags and `MARKCUT_ITT_CLI` / `MARKCUT_VTT_CLI` environment variables still override these defaults.
+
+Default ITT tries, in order:
+
+1. **Browser ChatGPT Temporary Chat** through `chatgpt-browser-infer` (or `MARKCUT_CHATGPT_BROWSER_INFER_CLI`). The Neo runtime owns browser automation, isolation, attachment readiness, retries, and cleanup.
+2. **Codex CLI remote vision**, trying profiles from `MARKCUT_CODEX_VISION_PROFILES` (default `zai,default`). On this Mac that means the lightweight Z.ai profile first, then the normal Codex/OpenAI profile.
+3. **Local MLX MiniCPM** as the final fallback only. Override its model with `MARKCUT_LOCAL_VISION_MODEL`.
+
+Default VTT is intentionally empty. Markcut therefore performs its existing deterministic chronological frame extraction (5–10 representative frames, controlled by `--vtt-sample-interval`) and sends those frames through the same ITT priority chain. Set `--vtt` or `MARKCUT_VTT_CLI` when a dedicated video backend is explicitly desired.
+
+To expose the Neo Browser ChatGPT runtime on this Mac:
+
+```bash
+ln -sf "$HOME/Workspace/neo/skills/chatgpt-browser-worker/bin/chatgpt-browser-infer" "$HOME/.local/bin/chatgpt-browser-infer"
+```
+
+Useful policy overrides:
+
+```bash
+export MARKCUT_CODEX_VISION_PROFILES='zai,default'
+export MARKCUT_CHATGPT_BROWSER_INFER_CLI='/path/to/chatgpt-browser-infer'
+# Explicit custom backends still win:
+export MARKCUT_ITT_CLI='my-image-backend --prompt "{prompt}" {input}'
+export MARKCUT_VTT_CLI='my-video-backend --prompt "{prompt}" {input}'
+```
+
 ## Architecture

package/docs/markdown-strict-descriptive.md

@@ -0,0 +1,16 @@
+
+## HTML comments: ignored metadata
+
+Standard Markdown HTML comments are intentionally ignored by Markcut's descriptive parser and never enter the descriptive tree or rendered output:
+
+```md
+<!-- arbitrary metadata -->
+```
+
+This makes comments suitable for **out-of-band metadata owned by another tool**, for example:
+
+```md
+<!-- execution {"id":"profiles-demo","type":"demo","scene_id":"profiles","output":"assets/profiles-demo.mp4"} -->
+```
+
+Markcut does **not** parse, validate, or interpret the comment contents. It only guarantees that HTML comment nodes are ignored. A separate tool may read its own comment convention directly from the Markdown source.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lalalic/markcut",
-  "version": "3.2.4",
+  "version": "3.2.6",
   "description": "Markdown-to-video engine. Describe scenes in markdown, get a rendered video.",
   "bin": {
     "markcut": "bin/markcut"

package/src/config.mjs

@@ -1,3 +1,5 @@
+import { fileURLToPath } from "node:url";
+
 /**
  * Default CLI templates and constants for the markcut pipeline.
  *
@@ -96,7 +98,8 @@ export const DEFAULT_EDIT_CLI = args.cliOverrides.editCli || process.env.MARKCUT
 export const DEFAULT_TTI_CLI = args.cliOverrides.tti || process.env.MARKCUT_TTI_CLI || 'uvx --from mflux mflux-generate-flux2 --model flux2-klein-4b --steps 2 --prompt "{input}" --output "{output}" --seed {seed}';
 /** Text-to-video CLI. Override via --ttv flag or MARKCUT_TTV_CLI env var. */
 export const DEFAULT_TTV_CLI = args.cliOverrides.ttv || process.env.MARKCUT_TTV_CLI || '';
+const DEFAULT_VISION_INFER = fileURLToPath(new URL("./vision/default-infer.mjs", import.meta.url));
 /** Image-to-text CLI. Override via --itt flag or MARKCUT_ITT_CLI env var. */
-export const DEFAULT_ITT_CLI = args.cliOverrides.itt || process.env.MARKCUT_ITT_CLI || 'uvx --from mlx-vlm mlx_vlm.generate --model mlx-community/MiniCPM-V-4.6-bf16 --max-tokens 2048 --prompt "{prompt}" --image {input} --temperature 0.0 --thinking-mode disabled';;
-/** Video-to-text CLI. Override via --vtt flag or MARKCUT_VTT_CLI env var. */
-export const DEFAULT_VTT_CLI = args.cliOverrides.vtt || process.env.MARKCUT_VTT_CLI || 'uvx --from mlx-vlm mlx_vlm.generate --model mlx-community/MiniCPM-V-4.6-bf16 --max-tokens 2048 --prompt "{prompt}" --video {input} --temperature 0.0 --processor-kwargs \'{"max_num_frames": 32, "stack_frames": 1, "max_slice_nums": 1, "use_image_id": false}\'';
+export const DEFAULT_ITT_CLI = args.cliOverrides.itt || process.env.MARKCUT_ITT_CLI || `node ${JSON.stringify(DEFAULT_VISION_INFER)} --prompt "{prompt}" {input}`;
+/** Video-to-text CLI. Override via --vtt flag or MARKCUT_VTT_CLI env var. Empty by default so Vision uses chronological frame extraction through ITT. */
+export const DEFAULT_VTT_CLI = args.cliOverrides.vtt || process.env.MARKCUT_VTT_CLI || '';

package/src/descriptive/markdown.test.ts

@@ -864,3 +864,41 @@ layout:parallel
     });
   });
 });
+
+// HTML comments are an intentional out-of-band metadata channel. remark-parse
+// emits them as `html` nodes; Markcut ignores those nodes completely.
+describe("HTML comments", () => {
+  it("ignores generic HTML comments", () => {
+    const doc = `# video
+<!-- arbitrary metadata -->
+## Scene
+- image src:a.jpg duration:1`;
+    const parsed = parseMarkdownDescriptive(doc);
+    expect(parsed.children).toHaveLength(1);
+    const scene = parsed.children[0] as any;
+    expect(scene.name).toBe("Scene");
+    expect(scene.children).toHaveLength(1);
+    expect(scene.children[0].type).toBe("image");
+  });
+
+  it("ignores execution metadata comments between scene metadata and media", () => {
+    const doc = `# video
+width:1920 height:1080 fps:30 layout:series
+
+## profiles
+title:"Reveal profiles" instruction:"Each child has a distinct profile"
+<!-- execution {"id":"profiles-demo","type":"demo","scene_id":"profiles","output":"assets/profiles-demo.mp4"} -->
+- video src:"assets/profiles-demo.mp4" duration:6`;
+    const parsed = parseMarkdownDescriptive(doc);
+    const scene = parsed.children[0] as any;
+    expect(scene.name).toBe("profiles");
+    expect(scene.title).toBe("Reveal profiles");
+    expect(scene.instruction).toBe("Each child has a distinct profile");
+    expect(scene.children).toHaveLength(1);
+    expect(scene.children[0]).toMatchObject({
+      type: "video",
+      src: "assets/profiles-demo.mp4",
+      duration: 6,
+    });
+  });
+});

package/src/vision/cli.mjs

@@ -395,7 +395,8 @@ function extractVideoFrames(videoPath, intervalSeconds = DEFAULT_VTT_SAMPLE_INTE
   catch (e) { try { rmSync(tmpDir, { recursive: true, force: true }); } catch {} throw new Error(`Frame extraction failed: ${e.message}`); }
   const frames = readdirSync(tmpDir).filter((f) => f.startsWith(baseName) && f.endsWith(".jpg")).sort().map((f) => join(tmpDir, f));
   if (frames.length === 0) { try { rmSync(tmpDir, { recursive: true, force: true }); } catch {} throw new Error("No frames extracted from video"); }
-  return { frames, cleanup: () => { try { rmSync(tmpDir, { recursive: true, force: true }); } catch {} } };
+  const timing = frames.map((frame, i) => ({ frame, seconds: ((i + 0.5) * Math.max(duration || 5, 1)) / frames.length }));
+  return { frames, timing, cleanup: () => { try { rmSync(tmpDir, { recursive: true, force: true }); } catch {} } };
 }
 
 function runVTT(videoPath, promptText, vttCli, ittCli, sampleInterval = DEFAULT_VTT_SAMPLE_INTERVAL) {
@@ -416,7 +417,11 @@ function runVTT(videoPath, promptText, vttCli, ittCli, sampleInterval = DEFAULT_
   let extracted;
   try { extracted = extractVideoFrames(videoPath, sampleInterval); }
   catch (e) { emitWarn(`  ${e.message}`); return ""; }
-  const result = runITT(extracted.frames, promptText, ittCli);
+  const timingContext = extracted.timing
+    .map((item, i) => `Frame ${i + 1}: approximately ${item.seconds.toFixed(1)}s`)
+    .join("\n");
+  const framePrompt = `${promptText}\n\nThe attached frames are ordered chronologically. Approximate timing:\n${timingContext}`;
+  const result = runITT(extracted.frames, framePrompt, ittCli);
   extracted.cleanup();
   return result;
 }

package/src/vision/default-infer.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { accessSync, constants, existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const LOCAL_MODEL = process.env.MARKCUT_LOCAL_VISION_MODEL || "mlx-community/MiniCPM-V-4.6-bf16";
+
+function parseArgs(argv) {
+  let prompt = "";
+  const inputs = [];
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--prompt") prompt = argv[++i] || "";
+    else if (arg === "--input") inputs.push(argv[++i]);
+    else if (arg && !arg.startsWith("--")) inputs.push(arg.startsWith("@") ? arg.slice(1) : arg);
+  }
+  return { prompt, inputs: inputs.filter(Boolean) };
+}
+
+function which(name) {
+  if (!name) return "";
+  const candidates = name.includes("/") ? [name] : (process.env.PATH || "").split(":").filter(Boolean).map(dir => join(dir, name));
+  for (const candidate of candidates) {
+    try { accessSync(candidate, constants.X_OK); return candidate; } catch {}
+  }
+  return "";
+}
+
+function run(command, args, options = {}) {
+  const r = spawnSync(command, args, { encoding: "utf8", timeout: options.timeout || 600_000, env: process.env });
+  return { ok: r.status === 0, stdout: (r.stdout || "").trim(), stderr: (r.stderr || "").trim(), status: r.status };
+}
+
+function tryBrowser(prompt, inputs) {
+  const explicit = process.env.MARKCUT_CHATGPT_BROWSER_INFER_CLI || "";
+  const command = explicit || which("chatgpt-browser-infer");
+  if (!command) return { ok: false, reason: "browser-infer-unavailable" };
+  const args = ["--prompt", prompt];
+  for (const input of inputs) args.push("--file", input);
+  const r = run(command, args, { timeout: Number(process.env.MARKCUT_CHATGPT_VISION_TIMEOUT_MS) || 600_000 });
+  return r.ok && r.stdout ? { ok: true, output: r.stdout } : { ok: false, reason: r.stderr || `browser-infer-exit-${r.status}` };
+}
+
+function tryCodex(prompt, inputs) {
+  const codex = which("codex");
+  if (!codex) return { ok: false, reason: "codex-unavailable" };
+  const profiles = (process.env.MARKCUT_CODEX_VISION_PROFILES || "zai,default").split(",").map(s => s.trim()).filter(Boolean);
+  const errors = [];
+  for (const profile of profiles) {
+    const dir = mkdtempSync(join(tmpdir(), "markcut-codex-vision-"));
+    const output = join(dir, "result.txt");
+    const args = ["exec", "--ephemeral", "--skip-git-repo-check"];
+    if (profile !== "default") args.push("-p", profile);
+    if (inputs.length) args.push("-i", ...inputs);
+    args.push("-o", output, prompt);
+    const r = run(codex, args, { timeout: Number(process.env.MARKCUT_CODEX_VISION_TIMEOUT_MS) || 600_000 });
+    let text = "";
+    if (existsSync(output)) text = readFileSync(output, "utf8").trim();
+    rmSync(dir, { recursive: true, force: true });
+    if (r.ok && text) return { ok: true, output: text };
+    errors.push(`${profile}:${r.stderr || `exit-${r.status}`}`);
+  }
+  return { ok: false, reason: errors.join("; ") || "codex-failed" };
+}
+
+function tryLocal(prompt, inputs) {
+  const uvx = which("uvx");
+  if (!uvx) return { ok: false, reason: "uvx-unavailable" };
+  const args = ["--from", "mlx-vlm", "mlx_vlm.generate", "--model", LOCAL_MODEL, "--max-tokens", "2048", "--prompt", prompt, "--image", ...inputs, "--temperature", "0.0", "--thinking-mode", "disabled"];
+  const r = run(uvx, args, { timeout: Number(process.env.MARKCUT_LOCAL_VISION_TIMEOUT_MS) || 600_000 });
+  return r.ok && r.stdout ? { ok: true, output: r.stdout } : { ok: false, reason: r.stderr || `local-vlm-exit-${r.status}` };
+}
+
+export function infer({ prompt, inputs }) {
+  if (!prompt) throw new Error("--prompt is required");
+  if (!inputs?.length) throw new Error("at least one image input is required");
+
+  const attempts = [
+    ["browser-chatgpt", () => tryBrowser(prompt, inputs)],
+    ["codex", () => tryCodex(prompt, inputs)],
+    ["local-vlm", () => tryLocal(prompt, inputs)],
+  ];
+  const failures = [];
+  for (const [name, fn] of attempts) {
+    const result = fn();
+    if (result.ok) return result.output;
+    failures.push(`${name}: ${result.reason}`);
+  }
+  throw new Error(`all vision backends failed (${failures.join(" | ")})`);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  try {
+    const { prompt, inputs } = parseArgs(process.argv.slice(2));
+    process.stdout.write(infer({ prompt, inputs }) + "\n");
+  } catch (error) {
+    process.stderr.write(`${error?.message || error}\n`);
+    process.exit(1);
+  }
+}

package/tests/vision-default-infer.test.ts

@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+const ROOT = resolve(fileURLToPath(import.meta.url), "..", "..");
+const HELPER = join(ROOT, "src", "vision", "default-infer.mjs");
+const tempDirs: string[] = [];
+
+function tempBin() {
+  const dir = mkdtempSync(join(tmpdir(), "markcut-vision-test-"));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function script(dir: string, name: string, body: string) {
+  const path = join(dir, name);
+  writeFileSync(path, `#!/bin/sh\n${body}\n`);
+  chmodSync(path, 0o755);
+  return path;
+}
+
+function runHelper(bin: string) {
+  return spawnSync(process.execPath, [HELPER, "--prompt", "describe", "/tmp/a.jpg"], {
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      PATH: `${bin}:/bin:/usr/bin`,
+      MARKCUT_CHATGPT_BROWSER_INFER_CLI: "",
+      MARKCUT_CODEX_VISION_PROFILES: "zai,default",
+    },
+  });
+}
+
+afterEach(() => {
+  for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+describe("default vision inference priority", () => {
+  it("uses Browser ChatGPT first", () => {
+    const bin = tempBin();
+    script(bin, "chatgpt-browser-infer", 'echo browser-result');
+    script(bin, "codex", 'echo codex-should-not-run >&2; exit 9');
+    script(bin, "uvx", 'echo local-should-not-run >&2; exit 9');
+    const r = runHelper(bin);
+    expect(r.status).toBe(0);
+    expect(r.stdout.trim()).toBe("browser-result");
+  });
+
+  it("falls back to Codex after Browser ChatGPT failure", () => {
+    const bin = tempBin();
+    script(bin, "chatgpt-browser-infer", 'exit 2');
+    script(bin, "codex", 'out=""; while [ "$#" -gt 0 ]; do if [ "$1" = "-o" ]; then out="$2"; shift 2; else shift; fi; done; echo codex-result > "$out"');
+    script(bin, "uvx", 'echo local-should-not-run >&2; exit 9');
+    const r = runHelper(bin);
+    expect(r.status).toBe(0);
+    expect(r.stdout.trim()).toBe("codex-result");
+  });
+
+  it("uses local VLM only after Browser ChatGPT and Codex fail", () => {
+    const bin = tempBin();
+    script(bin, "chatgpt-browser-infer", 'exit 2');
+    script(bin, "codex", 'exit 3');
+    script(bin, "uvx", 'echo local-result');
+    const r = runHelper(bin);
+    expect(r.status).toBe(0);
+    expect(r.stdout.trim()).toBe("local-result");
+  });
+
+  it("config defaults ITT to the policy helper and VTT to frame-based ITT", () => {
+    const code = 'import("./src/config.mjs").then(m => console.log(JSON.stringify({itt:m.DEFAULT_ITT_CLI,vtt:m.DEFAULT_VTT_CLI})))';
+    const env = { ...process.env } as Record<string, string>;
+    delete env.MARKCUT_ITT_CLI;
+    delete env.MARKCUT_VTT_CLI;
+    const r = spawnSync(process.execPath, ["--input-type=module", "-e", code], { cwd: ROOT, encoding: "utf8", env });
+    expect(r.status).toBe(0);
+    const value = JSON.parse(r.stdout.trim());
+    expect(value.itt).toContain("default-infer.mjs");
+    expect(value.vtt).toBe("");
+  });
+});