@lalalic/markcut 3.2.3 → 3.2.4

package/.env.example

@@ -13,9 +13,6 @@ GOOGLE_MAPS_API_KEY=your_key_here
 # Shared by vision and render pipelines:
 # MARKCUT_ITT_CLI=    # Image-to-text — placeholders: {input} {prompt}
 # MARKCUT_VTT_CLI=    # Video-to-text — placeholders: {input} {prompt}
-# MARKCUT_CHATGPT_BROWSER_INFER_CLI=chatgpt-browser-infer  # authenticated synchronous browser inference
-# MARKCUT_CHATGPT_VISION_TIMEOUT_MS=600000
-# MARKCUT_CHATGPT_DIRECT_VIDEO=0       # set 1 to try experimental direct MP4 before deterministic frames
 # MARKCUT_STT_CLI=    # Speech-to-text — placeholders: {input} {output}
 # MARKCUT_TTS_CLI=    # Text-to-speech — placeholders: {input} {output}
 # MARKCUT_AGENT_CLI=  # General-purpose agent — placeholders: {prompt}

package/README.md

@@ -342,22 +342,3 @@ flowchart LR
 | `--show-prompts` | Print the prompts template file and exit |
 
 ## Architecture
-
-
-### Browser ChatGPT Vision backend
-
-Markcut can use the authenticated Neo Browser ChatGPT inference command as its configurable ITT/VTT backend without changing the default local VLMs. Each inference uses a fresh worker-owned Temporary Chat tab, uploads the media, waits for a verified user turn and complete assistant result in the same tab, then closes that owned tab. It does not depend on reopening a ChatGPT conversation.
-
-Expose the Neo inference command on `PATH` as `chatgpt-browser-infer`, or point Markcut at it explicitly:
-
-```bash
-export MARKCUT_CHATGPT_BROWSER_INFER_CLI='/path/to/neo/skills/chatgpt-browser-worker/bin/chatgpt-browser-infer'
-export MARKCUT_ITT_CLI='markcut vision-chatgpt --mode image --prompt "{prompt}" --input {input}'
-export MARKCUT_VTT_CLI='markcut vision-chatgpt --mode video --prompt "{prompt}" --input {input}'
-```
-
-For a source checkout, replace `markcut` above with `node src/render/cli.mjs`.
-
-Image inputs are uploaded directly. Video uses deterministic chronological frame sampling by default, builds a contact sheet with timing context, and analyzes that image through the same inference surface. This is the production path because direct MP4 upload through the current ChatGPT web client was materially slower and did not complete reliably in E2E testing. Set `MARKCUT_CHATGPT_DIRECT_VIDEO=1` only to experiment with direct MP4 first; failure still falls back to frames. Prompts requesting JSON enable strict whole-response JSON validation; truncated or prose-wrapped JSON is never accepted as success (a single whole-response JSON code fence is normalized), and incomplete/failed inference attempts are retried in a fresh owned tab.
-
-`MARKCUT_CHATGPT_VISION_TIMEOUT_MS` controls the overall Markcut-side timeout. The inference command itself owns attachment readiness, submission verification, complete-result detection, retries, and owned-tab cleanup.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lalalic/markcut",
-  "version": "3.2.3",
+  "version": "3.2.4",
   "description": "Markdown-to-video engine. Describe scenes in markdown, get a rendered video.",
   "bin": {
     "markcut": "bin/markcut"

package/src/render/cli.mjs

@@ -81,11 +81,6 @@ Commands:
     --label                            Add interactive labeling step before AI pipeline
     --instruct "text"                   Background context about people/places (injected into prompts)
 
-  vision-chatgpt                       Browser ChatGPT ITT/VTT facade
-    --mode image|video|auto            Media mode (default: auto)
-    --prompt "text"                    Vision instructions
-    --input <path...>                  Local media input(s)
-
   spots --waypoints "lat,lng;..."      Discover POIs along a route (Directions + Places API)
     --travelMode DRIVING               DRIVING | WALKING | BICYCLING (default DRIVING)
     --limit 8                          Max spots after ranking
@@ -228,11 +223,6 @@ edit=${DEFAULT_EDIT_CLI}`);
     process.exit(0);
   }
 
-  if (args.command === "vision-chatgpt") {
-    const { main: visionChatGptMain } = await import("../vision/chatgpt-browser-cli.mjs");
-    process.exit(visionChatGptMain(process.argv.slice(3)));
-  }
-
   if (args.command === "spots") {
     await import("../spots/cli.mjs");  // self-executing top-level script
     process.exit(0);

package/src/vision/chatgpt-browser-cli.mjs

@@ -1,141 +0,0 @@
-#!/usr/bin/env node
-import { execFileSync, execSync } from "node:child_process";
-import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { basename, extname, join, resolve } from "node:path";
-
-const IMAGE_EXTS = new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff", ".heic", ".avif"]);
-const VIDEO_EXTS = new Set([".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv"]);
-
-function shellQuote(value) { return `'${String(value).replace(/'/g, `'\\''`)}'`; }
-
-export function parseArgs(argv) {
-  const out = { mode: "auto", inputs: [], prompt: "", timeoutMs: Number(process.env.MARKCUT_CHATGPT_VISION_TIMEOUT_MS) || 600_000, maxFrames: 8 };
-  for (let i = 0; i < argv.length; i++) {
-    const arg = argv[i];
-    if (arg === "--mode") out.mode = argv[++i] || "auto";
-    else if (arg === "--prompt") out.prompt = argv[++i] || "";
-    else if (arg === "--input") {
-      while (argv[i + 1] && !argv[i + 1].startsWith("--")) out.inputs.push(argv[++i]);
-    } else if (arg === "--timeout-ms") out.timeoutMs = Number(argv[++i]);
-    else if (arg === "--max-frames") out.maxFrames = Number(argv[++i]);
-    else if (!arg.startsWith("--")) out.inputs.push(arg);
-    else throw new Error(`Unknown argument: ${arg}`);
-  }
-  if (!out.prompt) throw new Error("--prompt is required");
-  if (out.inputs.length === 0) throw new Error("--input is required");
-  if (!Number.isFinite(out.timeoutMs) || out.timeoutMs <= 0) throw new Error("--timeout-ms must be > 0");
-  if (!Number.isFinite(out.maxFrames) || out.maxFrames < 1 || out.maxFrames > 16) throw new Error("--max-frames must be 1..16");
-  out.inputs = out.inputs.map((p) => resolve(p.replace(/^@/, "")));
-  for (const input of out.inputs) if (!existsSync(input)) throw new Error(`Input not found: ${input}`);
-  if (out.mode === "auto") out.mode = out.inputs.some((p) => VIDEO_EXTS.has(extname(p).toLowerCase())) ? "video" : "image";
-  if (!['image', 'video'].includes(out.mode)) throw new Error("--mode must be image, video, or auto");
-  return out;
-}
-
-function ffprobeDuration(videoPath) {
-  const raw = execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", videoPath], { encoding: "utf8" }).trim();
-  const value = Number(raw);
-  if (!Number.isFinite(value) || value <= 0) throw new Error(`Unable to read video duration: ${videoPath}`);
-  return value;
-}
-
-export function prepareMedia(mode, inputs, workDir, maxFrames = 8) {
-  if (mode === "image") return { files: inputs, context: inputs.map((p) => `Image: ${basename(p)}`).join("\n") };
-  if (inputs.length !== 1) throw new Error("video mode accepts exactly one input video");
-  const videoPath = inputs[0];
-  if (!VIDEO_EXTS.has(extname(videoPath).toLowerCase())) throw new Error(`Unsupported video input: ${videoPath}`);
-  const duration = ffprobeDuration(videoPath);
-  // Preserve chronology even for short clips. A one-frame fallback cannot
-  // distinguish ordering, so request at least two samples whenever maxFrames
-  // permits it; longer videos still scale at roughly one sample per 5 seconds.
-  const count = Math.max(1, Math.min(maxFrames, Math.max(2, Math.ceil(duration / 5))));
-  const framesDir = join(workDir, "frames");
-  mkdirSync(framesDir, { recursive: true });
-  const pattern = join(framesDir, "frame-%03d.jpg");
-  const fps = count / duration;
-  execFileSync("ffmpeg", ["-y", "-i", videoPath, "-vf", `fps=${fps},scale='min(640,iw)':-2`, "-frames:v", String(count), "-q:v", "3", pattern], { stdio: "ignore" });
-  const frames = readdirSync(framesDir).filter((f) => f.endsWith(".jpg")).sort().map((f) => join(framesDir, f));
-  if (frames.length === 0) throw new Error("No representative frames extracted from video");
-  const cols = Math.min(4, frames.length);
-  const contact = join(workDir, "contact-sheet.jpg");
-  if (frames.length === 1) {
-    execFileSync("ffmpeg", ["-y", "-i", frames[0], "-vf", "scale=320:-2", "-frames:v", "1", contact], { stdio: "ignore" });
-  } else {
-    execSync(`ffmpeg -y ${frames.map((p) => `-i ${shellQuote(p)}`).join(" ")} -filter_complex ${shellQuote(`xstack=inputs=${frames.length}:layout=${frames.map((_, i) => `${i % cols}*w0_${Math.floor(i / cols)}*h0`).join('|')},scale=${cols * 320}:-2`)} -frames:v 1 ${shellQuote(contact)}`, { stdio: "ignore" });
-  }
-  const timing = frames.map((p, i) => `${basename(p)} ≈ ${(i * duration / frames.length).toFixed(1)}s`).join(", ");
-  return { files: [contact], context: `Video: ${basename(videoPath)}\nDuration: ${duration.toFixed(2)}s\nRepresentative frames are chronological, left-to-right then top-to-bottom.\nTiming: ${timing}` };
-}
-
-function runInference(launcher, prompt, files, timeoutMs, env) {
-  const expectJson = /\bjson\b/i.test(prompt);
-  const attemptSeconds = Math.max(30, Math.floor(timeoutMs / 1000 / 3));
-  const parts = [launcher, "--prompt", shellQuote(prompt), "--result-timeout", String(attemptSeconds), "--attempts", "3"];
-  if (expectJson) parts.push("--expect-json");
-  for (const file of files) parts.push("--file", shellQuote(file));
-  return execSync(parts.join(" "), { env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: timeoutMs }).trim();
-}
-
-function mediaPrompt(prompt, context) {
-  return `${prompt}\n\nMedia context:\n${context}\n\nAnalyze only the attached media. Preserve chronology for video.`;
-}
-
-export function runVision({ mode, inputs, prompt, timeoutMs, maxFrames }, env = process.env) {
-  const launcher = env.MARKCUT_CHATGPT_BROWSER_INFER_CLI || "chatgpt-browser-infer";
-  const workDir = mkdtempSync(join(tmpdir(), "markcut-chatgpt-vision-"));
-  const deadline = Date.now() + timeoutMs;
-  const remaining = () => {
-    const value = deadline - Date.now();
-    if (value <= 0) throw new Error("ChatGPT browser vision overall timeout expired");
-    return value;
-  };
-  try {
-    if (mode === "image") {
-      const media = prepareMedia("image", inputs, workDir, maxFrames);
-      return runInference(launcher, mediaPrompt(prompt, media.context), media.files, remaining(), env);
-    }
-    if (inputs.length !== 1) throw new Error("video mode accepts exactly one input video");
-    const videoPath = inputs[0];
-    let directError = null;
-    if (env.MARKCUT_CHATGPT_DIRECT_VIDEO === "1") {
-      const directContext = `Video: ${basename(videoPath)}\nAnalyze the video directly and preserve chronology.`;
-      try {
-        // Direct MP4 analysis is experimental because the web client may accept
-        // the upload without exposing usable temporal media to the model. Never
-        // let it consume the entire caller budget needed for frame fallback.
-        const directBudget = Math.max(1, Math.min(remaining(), Math.floor(timeoutMs / 2)));
-        return runInference(launcher, mediaPrompt(prompt, directContext), [videoPath], directBudget, env);
-      } catch (error) {
-        directError = error;
-      }
-    }
-    const fallback = prepareMedia("video", inputs, workDir, maxFrames);
-    try {
-      const fallbackContext = directError
-        ? `${fallback.context}\nDirect video upload failed; using deterministic frame analysis.`
-        : `${fallback.context}\nUsing deterministic frame analysis.`;
-      return runInference(launcher, mediaPrompt(prompt, fallbackContext), fallback.files, remaining(), env);
-    } catch (fallbackError) {
-      const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
-      if (!directError) throw new Error(`ChatGPT video frame inference failed: ${fallbackMessage}`);
-      const directMessage = directError instanceof Error ? directError.message : String(directError);
-      throw new Error(`ChatGPT video inference failed directly and via frame fallback. direct=${directMessage}; fallback=${fallbackMessage}`);
-    }
-  } finally {
-    rmSync(workDir, { recursive: true, force: true });
-  }
-}
-
-export function main(argv = process.argv.slice(2), env = process.env) {
-  try {
-    const result = runVision(parseArgs(argv), env);
-    process.stdout.write(`${result}\n`);
-    return 0;
-  } catch (error) {
-    process.stderr.write(`markcut chatgpt vision: ${error instanceof Error ? error.message : String(error)}\n`);
-    return 1;
-  }
-}
-
-if (process.argv[1] && process.argv[1].endsWith("chatgpt-browser-cli.mjs")) process.exitCode = main();

package/tests/vision-chatgpt-browser.test.ts

@@ -1,99 +0,0 @@
-import { afterEach, describe, expect, it } from "vitest";
-import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { execFileSync } from "node:child_process";
-import { parseArgs, prepareMedia, runVision } from "../src/vision/chatgpt-browser-cli.mjs";
-
-const roots: string[] = [];
-afterEach(() => { while (roots.length) rmSync(roots.pop()!, { recursive: true, force: true }); });
-function temp() { const p = mkdtempSync(join(tmpdir(), "markcut-browser-test-")); roots.push(p); return p; }
-
-function makeMock(root: string, { failVideo = false } = {}) {
-  const path = join(root, "mock-infer.mjs");
-  writeFileSync(path, `
-const args=process.argv.slice(2);
-const files=[];
-let prompt='';
-for(let i=0;i<args.length;i++){
-  if(args[i]==='--prompt') prompt=args[++i];
-  else if(args[i]==='--file') files.push(args[++i]);
-}
-if(!prompt.includes('Media context:')) process.exit(3);
-if(!files.length) process.exit(4);
-if(${failVideo ? "true" : "false"} && files.some(f=>f.endsWith('.mp4'))) process.exit(7);
-process.stdout.write(JSON.stringify({prompt,files})+'\\n');
-`);
-  return `node ${JSON.stringify(path)}`;
-}
-
-it("parses Markcut template-style image arguments", () => {
-  const root = temp(); const a = join(root, "a.jpg"); const b = join(root, "b.jpg"); writeFileSync(a, "a"); writeFileSync(b, "b");
-  const args = parseArgs(["--mode", "image", "--prompt", "describe", "--input", `@${a}`, b]);
-  expect(args.mode).toBe("image"); expect(args.inputs).toEqual([a, b]);
-});
-
-it("returns synchronous browser inference stdout", () => {
-  const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image"); const launcher = makeMock(root);
-  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher } as NodeJS.ProcessEnv;
-  const result = JSON.parse(runVision({ mode:"image", inputs:[image], prompt:"describe", timeoutMs:2000, maxFrames:8 }, env));
-  expect(result.files).toEqual([image]);
-  expect(result.prompt).toContain("Image: a.jpg");
-});
-
-it("extracts deterministic chronological video representation", () => {
-  const root = temp(); const video = join(root, "clip.mp4"); const work = join(root, "work");
-  execFileSync("ffmpeg", ["-y", "-f", "lavfi", "-i", "testsrc=s=64x64:d=2:r=4", "-c:v", "libx264", video], { stdio:"ignore" });
-  const prepared = prepareMedia("video", [video], work, 4);
-  expect(prepared.files).toHaveLength(1); expect(existsSync(prepared.files[0])).toBe(true);
-  expect(prepared.context).toContain("chronological"); expect(prepared.context).toContain("Duration:");
-  expect(prepared.context).toContain("frame-001.jpg"); expect(prepared.context).toContain("frame-002.jpg");
-});
-
-it("uses deterministic frame analysis for video by default", () => {
-  const root = temp(); const video = join(root, "clip.mp4");
-  execFileSync("ffmpeg", ["-y", "-f", "lavfi", "-i", "testsrc=s=64x64:d=2:r=4", "-c:v", "libx264", video], { stdio:"ignore" });
-  const launcher = makeMock(root);
-  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher } as NodeJS.ProcessEnv;
-  delete env.MARKCUT_CHATGPT_DIRECT_VIDEO;
-  const result = JSON.parse(runVision({ mode:"video", inputs:[video], prompt:"describe chronology", timeoutMs:4000, maxFrames:4 }, env));
-  expect(result.files).toHaveLength(1);
-  expect(result.files[0]).toContain("contact-sheet.jpg");
-  expect(result.prompt).toContain("Using deterministic frame analysis");
-});
-
-it("can opt into direct video and falls back to deterministic frames", () => {
-  const root = temp(); const video = join(root, "clip.mp4");
-  execFileSync("ffmpeg", ["-y", "-f", "lavfi", "-i", "testsrc=s=64x64:d=2:r=4", "-c:v", "libx264", video], { stdio:"ignore" });
-  const launcher = makeMock(root, { failVideo: true });
-  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher, MARKCUT_CHATGPT_DIRECT_VIDEO: "1" } as NodeJS.ProcessEnv;
-  const result = JSON.parse(runVision({ mode:"video", inputs:[video], prompt:"describe chronology", timeoutMs:4000, maxFrames:4 }, env));
-  expect(result.files).toHaveLength(1);
-  expect(result.files[0]).toContain("contact-sheet.jpg");
-  expect(result.prompt).toContain("Direct video upload failed");
-});
-
-it("requests strict JSON validation when prompt requires JSON", () => {
-  const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image");
-  const path = join(root, "expect-json.mjs");
-  writeFileSync(path, `
-if(!process.argv.includes('--expect-json')) process.exit(9);
-process.stdout.write('{"ok":true}\\n');
-`);
-  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: `node ${JSON.stringify(path)}` } as NodeJS.ProcessEnv;
-  expect(runVision({ mode:"image", inputs:[image], prompt:"Return JSON only", timeoutMs:2000, maxFrames:8 }, env)).toBe('{"ok":true}');
-});
-
-it("returns nonzero from CLI when inference fails", () => {
-  const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image");
-  expect(() => execFileSync("node", ["src/vision/chatgpt-browser-cli.mjs", "--mode", "image", "--prompt", "x", "--input", image], { cwd: process.cwd(), env: { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: "exit 7" }, stdio:"pipe" })).toThrow();
-});
-
-describe("concurrency contract", () => {
-  it("keeps per-invocation media state isolated", async () => {
-    const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image"); const launcher = makeMock(root);
-    const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher } as NodeJS.ProcessEnv;
-    const results = await Promise.all(Array.from({ length: 4 }, (_, i) => Promise.resolve().then(() => runVision({ mode: "image", inputs: [image], prompt: `p${i}`, timeoutMs: 2000, maxFrames: 8 }, env))));
-    expect(results.map((x) => JSON.parse(x).prompt)).toHaveLength(4);
-  });
-});