@lalalic/markcut 3.2.2 → 3.2.3

package/.env.example

@@ -13,6 +13,9 @@ GOOGLE_MAPS_API_KEY=your_key_here
 # Shared by vision and render pipelines:
 # MARKCUT_ITT_CLI=    # Image-to-text — placeholders: {input} {prompt}
 # MARKCUT_VTT_CLI=    # Video-to-text — placeholders: {input} {prompt}
+# MARKCUT_CHATGPT_BROWSER_INFER_CLI=chatgpt-browser-infer  # authenticated synchronous browser inference
+# MARKCUT_CHATGPT_VISION_TIMEOUT_MS=600000
+# MARKCUT_CHATGPT_DIRECT_VIDEO=0       # set 1 to try experimental direct MP4 before deterministic frames
 # MARKCUT_STT_CLI=    # Speech-to-text — placeholders: {input} {output}
 # MARKCUT_TTS_CLI=    # Text-to-speech — placeholders: {input} {output}
 # MARKCUT_AGENT_CLI=  # General-purpose agent — placeholders: {prompt}

package/README.md

@@ -346,23 +346,18 @@ flowchart LR
 
 ### Browser ChatGPT Vision backend
 
-Markcut can use the Neo `chatgpt-browser-worker` agent as the configurable ITT/VTT backend without changing the default local VLMs. Each invocation creates one unique durable file output plus unique Neo job/task correlation, then hands the complete one-shot request to the existing browser-worker agent runtime.
+Markcut can use the authenticated Neo Browser ChatGPT inference command as its configurable ITT/VTT backend without changing the default local VLMs. Each inference uses a fresh worker-owned Temporary Chat tab, uploads the media, waits for a verified user turn and complete assistant result in the same tab, then closes that owned tab. It does not depend on reopening a ChatGPT conversation.
 
-For an installed package, select it with the existing CLI-template surface:
+Expose the Neo inference command on `PATH` as `chatgpt-browser-infer`, or point Markcut at it explicitly:
 
 ```bash
+export MARKCUT_CHATGPT_BROWSER_INFER_CLI='/path/to/neo/skills/chatgpt-browser-worker/bin/chatgpt-browser-infer'
 export MARKCUT_ITT_CLI='markcut vision-chatgpt --mode image --prompt "{prompt}" --input {input}'
 export MARKCUT_VTT_CLI='markcut vision-chatgpt --mode video --prompt "{prompt}" --input {input}'
 ```
 
 For a source checkout, replace `markcut` above with `node src/render/cli.mjs`.
 
-The facade intentionally does not call `chatgpt-browser-worker/scripts/*` or automate Chrome itself. Set `MARKCUT_CHATGPT_BROWSER_WORKER_AGENT_CLI` to the local command that launches the Neo **browser-worker agent/runtime**. The older `MARKCUT_CHATGPT_BROWSER_WORKER_CLI` name remains as a compatibility fallback.
+Image inputs are uploaded directly. Video uses deterministic chronological frame sampling by default, builds a contact sheet with timing context, and analyzes that image through the same inference surface. This is the production path because direct MP4 upload through the current ChatGPT web client was materially slower and did not complete reliably in E2E testing. Set `MARKCUT_CHATGPT_DIRECT_VIDEO=1` only to experiment with direct MP4 first; failure still falls back to frames. Prompts requesting JSON enable strict whole-response JSON validation; truncated or prose-wrapped JSON is never accepted as success (a single whole-response JSON code fence is normalized), and incomplete/failed inference attempts are retried in a fresh owned tab.
 
-- `MARKCUT_CHATGPT_PROMPT_FILE` — complete one-shot prompt, including the exact `Output: file` contract and worker-owned event IDs.
-- `MARKCUT_CHATGPT_MEDIA_FILES_JSON` — JSON array of attachments. Images are passed directly; video is deterministically reduced to a chronological contact sheet of representative frames plus timing context.
-- `MARKCUT_CHATGPT_OUTPUT_FILE` — authoritative result path that the delegated ChatGPT task must write.
-- `NEO_JOB_ID` / `NEO_TASK_ID` — unique correlation for the delegated worker's canonical `task.started` and terminal task event.
-- `MARKCUT_CHATGPT_TAB_CLOSE_POLICY=after-terminal` — requests that only the isolated worker-owned tab remain open until the exact task terminal event.
-
-The launcher may stay active through `after-terminal`; launcher/process exit is not treated as task completion. Markcut's only result is the declared durable output file, and it exits nonzero on launcher failure, timeout, preprocessing failure, or an empty/missing result. Carrier-only `task.process.launched`, `task.process.exited`, and `task.process.async_exited` events never substitute for worker-owned task lifecycle.
+`MARKCUT_CHATGPT_VISION_TIMEOUT_MS` controls the overall Markcut-side timeout. The inference command itself owns attachment readiness, submission verification, complete-result detection, retries, and owned-tab cleanup.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lalalic/markcut",
-  "version": "3.2.2",
+  "version": "3.2.3",
   "description": "Markdown-to-video engine. Describe scenes in markdown, get a rendered video.",
   "bin": {
     "markcut": "bin/markcut"

package/src/vision/chatgpt-browser-cli.mjs

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
 import { execFileSync, execSync } from "node:child_process";
-import { randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { basename, extname, join, resolve } from "node:path";
 
@@ -47,7 +46,10 @@ export function prepareMedia(mode, inputs, workDir, maxFrames = 8) {
   const videoPath = inputs[0];
   if (!VIDEO_EXTS.has(extname(videoPath).toLowerCase())) throw new Error(`Unsupported video input: ${videoPath}`);
   const duration = ffprobeDuration(videoPath);
-  const count = Math.max(1, Math.min(maxFrames, Math.ceil(duration / 5)));
+  // Preserve chronology even for short clips. A one-frame fallback cannot
+  // distinguish ordering, so request at least two samples whenever maxFrames
+  // permits it; longer videos still scale at roughly one sample per 5 seconds.
+  const count = Math.max(1, Math.min(maxFrames, Math.max(2, Math.ceil(duration / 5))));
   const framesDir = join(workDir, "frames");
   mkdirSync(framesDir, { recursive: true });
   const pattern = join(framesDir, "frame-%03d.jpg");
@@ -66,43 +68,60 @@ export function prepareMedia(mode, inputs, workDir, maxFrames = 8) {
   return { files: [contact], context: `Video: ${basename(videoPath)}\nDuration: ${duration.toFixed(2)}s\nRepresentative frames are chronological, left-to-right then top-to-bottom.\nTiming: ${timing}` };
 }
 
-function waitForResult(outputPath, timeoutMs) {
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    if (existsSync(outputPath)) {
-      const text = readFileSync(outputPath, "utf8").trim();
-      if (text) return text;
-    }
-    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 250);
-  }
-  throw new Error(`Timed out waiting for Browser ChatGPT result: ${outputPath}`);
+function runInference(launcher, prompt, files, timeoutMs, env) {
+  const expectJson = /\bjson\b/i.test(prompt);
+  const attemptSeconds = Math.max(30, Math.floor(timeoutMs / 1000 / 3));
+  const parts = [launcher, "--prompt", shellQuote(prompt), "--result-timeout", String(attemptSeconds), "--attempts", "3"];
+  if (expectJson) parts.push("--expect-json");
+  for (const file of files) parts.push("--file", shellQuote(file));
+  return execSync(parts.join(" "), { env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: timeoutMs }).trim();
+}
+
+function mediaPrompt(prompt, context) {
+  return `${prompt}\n\nMedia context:\n${context}\n\nAnalyze only the attached media. Preserve chronology for video.`;
 }
 
 export function runVision({ mode, inputs, prompt, timeoutMs, maxFrames }, env = process.env) {
-  const launcher = env.MARKCUT_CHATGPT_BROWSER_WORKER_AGENT_CLI || env.MARKCUT_CHATGPT_BROWSER_WORKER_CLI;
-  if (!launcher) {
-    throw new Error("MARKCUT_CHATGPT_BROWSER_WORKER_AGENT_CLI is required; it must launch the Neo browser-worker agent runtime");
-  }
+  const launcher = env.MARKCUT_CHATGPT_BROWSER_INFER_CLI || "chatgpt-browser-infer";
   const workDir = mkdtempSync(join(tmpdir(), "markcut-chatgpt-vision-"));
-  const outputPath = join(workDir, "result.txt");
+  const deadline = Date.now() + timeoutMs;
+  const remaining = () => {
+    const value = deadline - Date.now();
+    if (value <= 0) throw new Error("ChatGPT browser vision overall timeout expired");
+    return value;
+  };
   try {
-    const media = prepareMedia(mode, inputs, workDir, maxFrames);
-    const jobId = `markcut-vision-${randomUUID()}`;
-    const taskId = `browser-vision-${randomUUID()}`;
-    const fullPrompt = `${prompt}\n\nMedia context:\n${media.context}\n\nAnalyze only the attached media. Preserve chronology for video. Write only the final answer to the declared file output.\n\nExecution event contract:\n- Job: ${jobId}\n- Task: ${taskId}\n- Publish exactly one worker-owned task.started before analysis.\n- Publish exactly one worker-owned task.completed only after the output file is durable; publish task.failed instead if execution cannot complete.\n- task.process.* events are carrier lifecycle only and never substitute for task lifecycle.\n\nOutput:\nfile\n${outputPath}`;
-    const promptPath = join(workDir, "prompt.txt");
-    writeFileSync(promptPath, fullPrompt, "utf8");
-    const childEnv = {
-      ...env,
-      MARKCUT_CHATGPT_PROMPT_FILE: promptPath,
-      MARKCUT_CHATGPT_MEDIA_FILES_JSON: JSON.stringify(media.files),
-      MARKCUT_CHATGPT_OUTPUT_FILE: outputPath,
-      MARKCUT_CHATGPT_TAB_CLOSE_POLICY: "after-terminal",
-      NEO_JOB_ID: jobId,
-      NEO_TASK_ID: taskId,
-    };
-    execSync(launcher, { env: childEnv, stdio: ["ignore", "pipe", "pipe"], timeout: timeoutMs });
-    return waitForResult(outputPath, timeoutMs);
+    if (mode === "image") {
+      const media = prepareMedia("image", inputs, workDir, maxFrames);
+      return runInference(launcher, mediaPrompt(prompt, media.context), media.files, remaining(), env);
+    }
+    if (inputs.length !== 1) throw new Error("video mode accepts exactly one input video");
+    const videoPath = inputs[0];
+    let directError = null;
+    if (env.MARKCUT_CHATGPT_DIRECT_VIDEO === "1") {
+      const directContext = `Video: ${basename(videoPath)}\nAnalyze the video directly and preserve chronology.`;
+      try {
+        // Direct MP4 analysis is experimental because the web client may accept
+        // the upload without exposing usable temporal media to the model. Never
+        // let it consume the entire caller budget needed for frame fallback.
+        const directBudget = Math.max(1, Math.min(remaining(), Math.floor(timeoutMs / 2)));
+        return runInference(launcher, mediaPrompt(prompt, directContext), [videoPath], directBudget, env);
+      } catch (error) {
+        directError = error;
+      }
+    }
+    const fallback = prepareMedia("video", inputs, workDir, maxFrames);
+    try {
+      const fallbackContext = directError
+        ? `${fallback.context}\nDirect video upload failed; using deterministic frame analysis.`
+        : `${fallback.context}\nUsing deterministic frame analysis.`;
+      return runInference(launcher, mediaPrompt(prompt, fallbackContext), fallback.files, remaining(), env);
+    } catch (fallbackError) {
+      const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
+      if (!directError) throw new Error(`ChatGPT video frame inference failed: ${fallbackMessage}`);
+      const directMessage = directError instanceof Error ? directError.message : String(directError);
+      throw new Error(`ChatGPT video inference failed directly and via frame fallback. direct=${directMessage}; fallback=${fallbackMessage}`);
+    }
   } finally {
     rmSync(workDir, { recursive: true, force: true });
   }

package/tests/vision-chatgpt-browser.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, it } from "vitest";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { execFileSync } from "node:child_process";
@@ -8,9 +8,22 @@ import { parseArgs, prepareMedia, runVision } from "../src/vision/chatgpt-browse
 const roots: string[] = [];
 afterEach(() => { while (roots.length) rmSync(roots.pop()!, { recursive: true, force: true }); });
 function temp() { const p = mkdtempSync(join(tmpdir(), "markcut-browser-test-")); roots.push(p); return p; }
-function makeMock(root: string) {
-  const path = join(root, "mock-worker.mjs");
-  writeFileSync(path, `import {readFileSync,writeFileSync} from 'node:fs';\nconst p=readFileSync(process.env.MARKCUT_CHATGPT_PROMPT_FILE,'utf8');\nif(!p.includes('Output:\\nfile\\n'+process.env.MARKCUT_CHATGPT_OUTPUT_FILE)) process.exit(3);\nif(!process.env.NEO_JOB_ID?.startsWith('markcut-vision-')||!process.env.NEO_TASK_ID?.startsWith('browser-vision-')) process.exit(5);\nif(process.env.MARKCUT_CHATGPT_TAB_CLOSE_POLICY!=='after-terminal') process.exit(6);\nif(!p.includes('Publish exactly one worker-owned task.started')||!p.includes(process.env.NEO_JOB_ID)||!p.includes(process.env.NEO_TASK_ID)) process.exit(7);\nconst files=JSON.parse(process.env.MARKCUT_CHATGPT_MEDIA_FILES_JSON);\nif(!files.length||files.some(f=>!readFileSync(f))) process.exit(4);\nwriteFileSync(process.env.MARKCUT_CHATGPT_OUTPUT_FILE,'mock answer');\n`);
+
+function makeMock(root: string, { failVideo = false } = {}) {
+  const path = join(root, "mock-infer.mjs");
+  writeFileSync(path, `
+const args=process.argv.slice(2);
+const files=[];
+let prompt='';
+for(let i=0;i<args.length;i++){
+  if(args[i]==='--prompt') prompt=args[++i];
+  else if(args[i]==='--file') files.push(args[++i]);
+}
+if(!prompt.includes('Media context:')) process.exit(3);
+if(!files.length) process.exit(4);
+if(${failVideo ? "true" : "false"} && files.some(f=>f.endsWith('.mp4'))) process.exit(7);
+process.stdout.write(JSON.stringify({prompt,files})+'\\n');
+`);
   return `node ${JSON.stringify(path)}`;
 }
 
@@ -20,11 +33,12 @@ it("parses Markcut template-style image arguments", () => {
   expect(args.mode).toBe("image"); expect(args.inputs).toEqual([a, b]);
 });
 
-it("uses a unique file output per invocation and propagates result", () => {
+it("returns synchronous browser inference stdout", () => {
   const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image"); const launcher = makeMock(root);
-  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_WORKER_AGENT_CLI: launcher } as NodeJS.ProcessEnv;
-  expect(runVision({ mode:"image", inputs:[image], prompt:"describe", timeoutMs:2000, maxFrames:8 }, env)).toBe("mock answer");
-  expect(runVision({ mode:"image", inputs:[image], prompt:"describe again", timeoutMs:2000, maxFrames:8 }, env)).toBe("mock answer");
+  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher } as NodeJS.ProcessEnv;
+  const result = JSON.parse(runVision({ mode:"image", inputs:[image], prompt:"describe", timeoutMs:2000, maxFrames:8 }, env));
+  expect(result.files).toEqual([image]);
+  expect(result.prompt).toContain("Image: a.jpg");
 });
 
 it("extracts deterministic chronological video representation", () => {
@@ -33,18 +47,53 @@ it("extracts deterministic chronological video representation", () => {
   const prepared = prepareMedia("video", [video], work, 4);
   expect(prepared.files).toHaveLength(1); expect(existsSync(prepared.files[0])).toBe(true);
   expect(prepared.context).toContain("chronological"); expect(prepared.context).toContain("Duration:");
+  expect(prepared.context).toContain("frame-001.jpg"); expect(prepared.context).toContain("frame-002.jpg");
+});
+
+it("uses deterministic frame analysis for video by default", () => {
+  const root = temp(); const video = join(root, "clip.mp4");
+  execFileSync("ffmpeg", ["-y", "-f", "lavfi", "-i", "testsrc=s=64x64:d=2:r=4", "-c:v", "libx264", video], { stdio:"ignore" });
+  const launcher = makeMock(root);
+  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher } as NodeJS.ProcessEnv;
+  delete env.MARKCUT_CHATGPT_DIRECT_VIDEO;
+  const result = JSON.parse(runVision({ mode:"video", inputs:[video], prompt:"describe chronology", timeoutMs:4000, maxFrames:4 }, env));
+  expect(result.files).toHaveLength(1);
+  expect(result.files[0]).toContain("contact-sheet.jpg");
+  expect(result.prompt).toContain("Using deterministic frame analysis");
+});
+
+it("can opt into direct video and falls back to deterministic frames", () => {
+  const root = temp(); const video = join(root, "clip.mp4");
+  execFileSync("ffmpeg", ["-y", "-f", "lavfi", "-i", "testsrc=s=64x64:d=2:r=4", "-c:v", "libx264", video], { stdio:"ignore" });
+  const launcher = makeMock(root, { failVideo: true });
+  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher, MARKCUT_CHATGPT_DIRECT_VIDEO: "1" } as NodeJS.ProcessEnv;
+  const result = JSON.parse(runVision({ mode:"video", inputs:[video], prompt:"describe chronology", timeoutMs:4000, maxFrames:4 }, env));
+  expect(result.files).toHaveLength(1);
+  expect(result.files[0]).toContain("contact-sheet.jpg");
+  expect(result.prompt).toContain("Direct video upload failed");
+});
+
+it("requests strict JSON validation when prompt requires JSON", () => {
+  const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image");
+  const path = join(root, "expect-json.mjs");
+  writeFileSync(path, `
+if(!process.argv.includes('--expect-json')) process.exit(9);
+process.stdout.write('{"ok":true}\\n');
+`);
+  const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: `node ${JSON.stringify(path)}` } as NodeJS.ProcessEnv;
+  expect(runVision({ mode:"image", inputs:[image], prompt:"Return JSON only", timeoutMs:2000, maxFrames:8 }, env)).toBe('{"ok":true}');
 });
 
-it("returns nonzero from CLI when launcher fails", () => {
+it("returns nonzero from CLI when inference fails", () => {
   const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image");
-  expect(() => execFileSync("node", ["src/vision/chatgpt-browser-cli.mjs", "--mode", "image", "--prompt", "x", "--input", image], { cwd: process.cwd(), env: { ...process.env, MARKCUT_CHATGPT_BROWSER_WORKER_CLI: "exit 7" }, stdio:"pipe" })).toThrow();
+  expect(() => execFileSync("node", ["src/vision/chatgpt-browser-cli.mjs", "--mode", "image", "--prompt", "x", "--input", image], { cwd: process.cwd(), env: { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: "exit 7" }, stdio:"pipe" })).toThrow();
 });
 
-describe("concurrency", () => {
-  it("does not collide across simultaneous invocations", async () => {
+describe("concurrency contract", () => {
+  it("keeps per-invocation media state isolated", async () => {
     const root = temp(); const image = join(root, "a.jpg"); writeFileSync(image, "image"); const launcher = makeMock(root);
-    const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_WORKER_AGENT_CLI: launcher } as NodeJS.ProcessEnv;
-    const results = await Promise.all(Array.from({length:4}, (_, i) => Promise.resolve().then(() => runVision({ mode:"image", inputs:[image], prompt:`p${i}`, timeoutMs:2000, maxFrames:8 }, env))));
-    expect(results).toEqual(["mock answer", "mock answer", "mock answer", "mock answer"]);
+    const env = { ...process.env, MARKCUT_CHATGPT_BROWSER_INFER_CLI: launcher } as NodeJS.ProcessEnv;
+    const results = await Promise.all(Array.from({ length: 4 }, (_, i) => Promise.resolve().then(() => runVision({ mode: "image", inputs: [image], prompt: `p${i}`, timeoutMs: 2000, maxFrames: 8 }, env))));
+    expect(results.map((x) => JSON.parse(x).prompt)).toHaveLength(4);
   });
 });