@coseung2/opencodex 2.8.0-cs.1

package/src/images/fulfill-video.ts

@@ -0,0 +1,163 @@
+import { pathToFileURL } from "node:url";
+import type { VideoBridgePlan, VideoCallResult } from "./types";
+import { submitVideoJob, pollVideoJob } from "./xai-video-client";
+import { downloadVideoToArtifact, createVideoBudget, type VideoBudget } from "./artifacts";
+
+/** Parsed arguments from the model's video_gen tool call. */
+export interface ParsedVideoArgs {
+  ok: true;
+  prompt: string;
+  duration?: number;
+  resolution?: string;
+  aspectRatio?: string;
+}
+
+const INITIAL_POLL_INTERVAL_MS = 5_000;
+const MAX_POLL_INTERVAL_MS = 15_000;
+const POLL_BACKOFF = 1.5;
+const DEFAULT_VIDEO_TIMEOUT_MS = 300_000; // 5 min
+
+export type SleepFn = (ms: number, signal?: AbortSignal) => Promise<void>;
+export type PollVideoJobFn = (
+  requestId: string,
+  auth: { baseUrl: string; token: string },
+  signal?: AbortSignal,
+) => ReturnType<typeof pollVideoJob>;
+
+export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.reject(new Error("aborted"));
+  return new Promise((resolve, reject) => {
+    const onAbort = (): void => { clearTimeout(timer); reject(new Error("aborted")); };
+    const timer = setTimeout(() => { signal?.removeEventListener("abort", onAbort); resolve(); }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+/**
+ * Parse the JSON arguments string from a video_gen tool call. Mirrors fulfillImageCall's
+ * defensive parsing: invalid JSON or missing prompt returns an error result.
+ */
+export function parseVideoCallArgs(raw: string):
+  | ParsedVideoArgs
+  | { ok: false; error: string } {
+  let args: unknown;
+  try {
+    args = JSON.parse(raw || "{}");
+  } catch {
+    return { ok: false, error: "invalid arguments JSON" };
+  }
+  if (typeof args !== "object" || args === null) {
+    return { ok: false, error: "invalid arguments JSON" };
+  }
+  const obj = args as Record<string, unknown>;
+
+  const prompt =
+    typeof obj.prompt === "string" ? obj.prompt : typeof obj.input === "string" ? obj.input : "";
+  if (!prompt) {
+    return { ok: false, error: "missing prompt" };
+  }
+
+  const result: ParsedVideoArgs = { ok: true, prompt };
+
+  // duration: clamp to 1-15
+  if (typeof obj.duration === "number") {
+    result.duration = Math.max(1, Math.min(15, Math.floor(obj.duration)));
+  }
+
+  if (typeof obj.resolution === "string" && (obj.resolution === "480p" || obj.resolution === "720p")) {
+    result.resolution = obj.resolution;
+  }
+  if (typeof obj.aspect_ratio === "string") {
+    const VALID_RATIOS = ["16:9", "9:16", "1:1", "4:3", "3:4", "3:2", "2:3"];
+    if (VALID_RATIOS.includes(obj.aspect_ratio)) result.aspectRatio = obj.aspect_ratio;
+  }
+
+  return result;
+}
+
+/**
+ * Poll a video generation job, yielding heartbeats with elapsed-time messages so the
+ * caller can forward them to the SSE stream. Returns the final result when the job
+ * completes or fails/times out.
+ */
+export async function* pollVideoWithHeartbeats(
+  requestId: string,
+  auth: { baseUrl: string; token: string },
+  signal: AbortSignal,
+  timeoutMs: number = DEFAULT_VIDEO_TIMEOUT_MS,
+  /** Test seams — avoid mock.module / global setTimeout (both poison parallel Bun CI). */
+  sleepFn: SleepFn = sleep,
+  pollFn: PollVideoJobFn = pollVideoJob,
+): AsyncGenerator<
+  { type: "heartbeat"; message: string },
+  { ok: true; videoUrl: string } | { ok: false; error: string }
+> {
+  const start = Date.now();
+  let interval = INITIAL_POLL_INTERVAL_MS;
+  const timeoutStr = `video generation timed out after ${Math.floor(timeoutMs / 1000)}s`;
+
+  for (;;) {
+    if (Date.now() - start >= timeoutMs) {
+      return { ok: false, error: timeoutStr };
+    }
+
+    try {
+      const poll = await pollFn(requestId, auth, signal);
+      if (poll.status === "done") {
+        if (poll.videoUrl) {
+          return { ok: true, videoUrl: poll.videoUrl };
+        }
+        return { ok: false, error: "video generation completed but no video URL was returned" };
+      }
+      if (poll.status === "failed") {
+        return { ok: false, error: "video generation failed" };
+      }
+      if (poll.status === "expired") {
+        return { ok: false, error: "video generation expired" };
+      }
+      // still processing — continue with backoff
+    } catch (e) {
+      // Distinguish deadline-expiry from client-cancel.
+      // AbortSignal.timeout sets signal.reason to a TimeoutError.
+      const isDeadline = signal.aborted && e instanceof Error && e.name === "TimeoutError";
+      if (isDeadline) return { ok: false, error: timeoutStr };
+      if (signal.aborted) return { ok: false, error: "client closed request during video generation" };
+      const status = (e as Error & { status?: number }).status;
+      if (status && status >= 400 && status < 500 && status !== 429) {
+        return { ok: false, error: `video poll failed permanently: HTTP ${status}` };
+      }
+      console.warn(`[videos] poll error (will retry): ${e instanceof Error ? e.message : String(e)}`);
+    }
+
+    const elapsed = Math.floor((Date.now() - start) / 1000);
+    yield { type: "heartbeat", message: `Generating video... ${elapsed}s` };
+
+    try {
+      await sleepFn(interval, signal);
+    } catch {
+      const isDeadline = signal.aborted && signal.reason instanceof Error && signal.reason.name === "TimeoutError";
+      if (isDeadline) return { ok: false, error: timeoutStr };
+      return { ok: false, error: "client closed request during video generation" };
+    }
+
+    interval = Math.min(MAX_POLL_INTERVAL_MS, Math.floor(interval * POLL_BACKOFF));
+  }
+}
+
+/**
+ * Build a success VideoCallResult after the video has been downloaded to disk.
+ */
+export function buildVideoResult(path: string, prompt: string, model: string): VideoCallResult {
+  return {
+    ok: true,
+    model,
+    prompt,
+    path,
+    files: [path],
+    count: 1,
+    markdown: `[video](${pathToFileURL(path).href})`,
+  };
+}
+
+export type { VideoBudget };
+export { createVideoBudget };

package/src/images/fulfill.ts

@@ -0,0 +1,149 @@
+import { existsSync } from "node:fs";
+import { pathToFileURL } from "node:url";
+import type { ImageBridgePlan, ImageCallResult } from "./types";
+import { callXaiImages } from "./xai-client";
+import {
+  materializeInlineImage,
+  downloadImageToArtifact,
+  pruneArtifacts,
+  type ImageBudget,
+} from "./artifacts";
+import { MAX_PENDING_IMAGE_FULFILLMENTS } from "../lib/translator-budget";
+
+const fulfillmentEncoder = new TextEncoder();
+let pendingFulfillments = 0;
+let fulfillmentCurrentBytes = 0;
+let fulfillmentHighWaterBytes = 0;
+
+function addFulfillmentBytes(bytes: number): void {
+  fulfillmentCurrentBytes += bytes;
+  fulfillmentHighWaterBytes = Math.max(fulfillmentHighWaterBytes, fulfillmentCurrentBytes);
+}
+
+function releaseFulfillmentBytes(bytes: number): void {
+  fulfillmentCurrentBytes = Math.max(0, fulfillmentCurrentBytes - bytes);
+}
+
+export function imageFulfillmentTailSnapshot(): { currentBytes: number; highWaterBytes: number; active: number } {
+  return { currentBytes: fulfillmentCurrentBytes, highWaterBytes: fulfillmentHighWaterBytes, active: pendingFulfillments };
+}
+
+/** Serialize write→prune→retain across concurrent fulfillments sharing artifacts/. */
+let retentionTail: Promise<void> = Promise.resolve();
+
+async function retainAfterBatch(paths: string[], keepCount?: number): Promise<string[]> {
+  const retainedBytes = paths.reduce((total, path) => total + fulfillmentEncoder.encode(path).byteLength, 0);
+  addFulfillmentBytes(retainedBytes);
+  const run = retentionTail.then(() => {
+    pruneArtifacts(keepCount);
+    return paths.filter((p) => existsSync(p));
+  });
+  retentionTail = run.then(
+    () => undefined,
+    () => undefined,
+  );
+  try {
+    return await run;
+  } finally {
+    releaseFulfillmentBytes(retainedBytes);
+  }
+}
+
+/**
+ * Fulfill ONE image-generation tool call end-to-end: parse args, call xAI, materialize the returned
+ * images to disk, and return a structured result. NEVER throws — all errors become `{ ok: false }`
+ * so the caller can inject the error as a tool result and let the model respond gracefully.
+ */
+export async function fulfillImageCall(
+  call: { id: string; name: string; arguments: string },
+  plan: ImageBridgePlan,
+  budget: ImageBudget,
+  signal?: AbortSignal,
+): Promise<ImageCallResult> {
+  if (pendingFulfillments >= MAX_PENDING_IMAGE_FULFILLMENTS) {
+    return { ok: false, model: plan.model, prompt: "", files: [], count: 0, error: "image_fulfillment_busy" };
+  }
+  pendingFulfillments += 1;
+  const callBytes = fulfillmentEncoder.encode(call.id).byteLength
+    + fulfillmentEncoder.encode(call.name).byteLength
+    + fulfillmentEncoder.encode(call.arguments).byteLength;
+  addFulfillmentBytes(callBytes);
+  try {
+  let args: unknown;
+  try {
+    args = JSON.parse(call.arguments || "{}");
+  } catch {
+    return { ok: false, model: plan.model, prompt: "", files: [], count: 0, error: "invalid arguments JSON" };
+  }
+  if (typeof args !== "object" || args === null) {
+    return { ok: false, model: plan.model, prompt: "", files: [], count: 0, error: "invalid arguments JSON" };
+  }
+  const obj = args as Record<string, unknown>;
+
+  const prompt =
+    typeof obj.prompt === "string" ? obj.prompt : typeof obj.input === "string" ? obj.input : "";
+  if (!prompt) {
+    return { ok: false, model: plan.model, prompt: "", files: [], count: 0, error: "missing prompt" };
+  }
+
+  const n = typeof obj.n === "number" ? Math.max(1, Math.min(4, Math.floor(obj.n))) : 1;
+  const imageUrl =
+    typeof obj.image_url === "string" ? obj.image_url : typeof obj.image === "string" ? obj.image : undefined;
+  const size = typeof obj.size === "string" ? obj.size : plan.defaultSize;
+  const quality = typeof obj.quality === "string" ? obj.quality : plan.defaultQuality;
+
+  let result;
+  try {
+    result = await callXaiImages(
+      { prompt, model: plan.model, n, imageUrl, size, quality },
+      plan.auth,
+      signal,
+      plan.timeoutMs,
+    );
+  } catch (e) {
+    const error = e instanceof Error ? e.message : String(e);
+    return { ok: false, model: plan.model, prompt, files: [], count: 0, error };
+  }
+
+  const files: string[] = [];
+  for (const img of result.images ?? []) {
+    try {
+      if (img.b64_json) {
+        files.push(await materializeInlineImage(img.b64_json, budget));
+      } else if (img.url) {
+        files.push(await downloadImageToArtifact(img.url, budget, signal));
+      }
+    } catch {
+      // Keep warnings URL-free — error messages may embed provider CDN URLs.
+      // Partial success is OK — silently skip this image and continue.
+      console.warn("[images] failed to materialize image");
+    }
+  }
+
+  // Prune only after the full batch is on disk so a tight keepCount cannot delete
+  // an earlier image from this same call before we return its path. Concurrent
+  // fulfillments share one retention chain so they cannot prune each other's
+  // just-written paths mid-filter.
+  const retained = await retainAfterBatch(files, plan.artifactsKeepCount);
+
+  if (retained.length === 0) {
+    return { ok: false, model: plan.model, prompt, files: [], count: 0, error: "image generation returned no usable images" };
+  }
+
+  const primary = retained[0]!;
+  return {
+    ok: true,
+    model: plan.model,
+    prompt,
+    path: primary,
+    files: retained,
+    count: retained.length,
+    // Keep path/files as native FS paths; Markdown needs a file: URI so Windows
+    // backslashes are not treated as escapes by renderers.
+    markdown: `![image](${pathToFileURL(primary).href})`,
+  };
+  } finally {
+    releaseFulfillmentBytes(callBytes);
+    pendingFulfillments -= 1;
+  }
+}

package/src/images/index.ts

@@ -0,0 +1,4 @@
+export { planImageBridge, planVideoBridge, findXaiProvider, resolveXaiImageApiKey } from "./plan";
+export { runWithImageBridge, clampImageMaxRounds, DEFAULT_MAX_ROUNDS, MAX_ROUNDS_HARD_LIMIT } from "./loop";
+export type { ImageBridgePlan, ImageCallResult, VideoBridgePlan, VideoCallResult } from "./types";
+export { buildImageTool, buildVideoTool, extractHostedImageGeneration, IMAGE_GEN_TOOL_NAME, VIDEO_GEN_TOOL_NAME, isImageGenName, isVideoGenName } from "./synthetic-tool";