@juspay/neurolink 12.29.2 → 12.30.1

package/dist/processors/media/VideoProcessor.js

@@ -50,13 +50,15 @@ import { join } from "path";
 import { Readable } from "stream";
 import { pipeline } from "stream/promises";
 import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
-import { formatMediaDuration } from "../../utils/mediaDuration.js";
+import { formatKeyframeTimestamp, formatMediaDuration, } from "../../utils/mediaDuration.js";
 import { SIZE_LIMITS_MB } from "../config/index.js";
 import { extensionsForModality, mimeTypesForModality, } from "../config/fileTypeRegistry.js";
 import { FileErrorCode } from "../errors/index.js";
 import { tracers, ATTR, withSpan } from "../../telemetry/index.js";
 import { logger } from "../../utils/logger.js";
 import { tryImport } from "../../utils/tryImport.js";
+import { withTimeout } from "../../utils/errorHandling.js";
+import { runFfmpeg } from "../../adapters/video/ffmpegAdapter.js";
 /**
  * Narrow a loaded `fluent-ffmpeg` export to the shape this file actually uses:
  * a callable carrying the `ffprobe` and `setFfmpegPath` statics.
@@ -202,6 +204,21 @@ const VIDEO_CONFIG = {
     FFMPEG_TIMEOUT_MS: 120_000,
     /** Timeout for ffprobe metadata extraction in milliseconds */
     FFPROBE_TIMEOUT_MS: 10_000,
+    /**
+     * Timeout for demuxing the audio track out of a video (#433).
+     *
+     * Generous next to frame extraction: this reads the whole file rather than
+     * seeking to a handful of offsets, and a meeting recording is long.
+     */
+    AUDIO_EXTRACT_TIMEOUT_MS: 180_000,
+    /** Timeout for one Whisper transcription request in milliseconds */
+    TRANSCRIPTION_TIMEOUT_MS: 120_000,
+    /**
+     * Whisper's upload ceiling in MB. The extracted track is mono 16 kHz MP3,
+     * so this is roughly six hours of speech — a clip that breaches it is
+     * unusual enough to be worth saying so rather than silently truncating.
+     */
+    WHISPER_MAX_SIZE_MB: 25,
 };
 /**
  * Supported video MIME types — derived from the canonical registry so this
@@ -285,6 +302,8 @@ export class VideoProcessor extends BaseFileProcessor {
             filename: this.getFilename(fileInfo),
             textContent: "",
             keyframes: [],
+            keyframeTimestampsSec: [],
+            hasTranscript: false,
             metadata: {
                 duration: 0,
                 durationFormatted: formatMediaDuration(0),
@@ -456,6 +475,14 @@ export class VideoProcessor extends BaseFileProcessor {
                     logger.warn(`[NEUROLINK] Video keyframe extraction failed for ${filename}, continuing without keyframes`);
                 }
                 span.setAttribute(ATTR.VIDEO_KEYFRAMES_EXTRACTED, keyframes.length);
+                // Debug-only: the real per-frame sample times, so a caller running
+                // with --debug can see what the model was actually told without
+                // re-deriving it from the request body.
+                if (logger.shouldLog("debug") && keyframes.length > 0) {
+                    logger.debug(`[NEUROLINK] Video keyframe timestamps: ${keyframes
+                        .map((frame) => frame.timestampSec.toFixed(3))
+                        .join(",")}`);
+                }
                 // Step 6: Extract subtitles
                 let subtitleText;
                 if (metadata.subtitleTracks > 0) {
@@ -466,14 +493,16 @@ export class VideoProcessor extends BaseFileProcessor {
                         // Non-fatal: continue without subtitles if extraction fails
                     }
                 }
-                // Step 7: Build textContent for LLM
-                const textContent = this.buildTextContent(metadata, keyframes.length, subtitleText, this.getFilename(fileInfo));
+                // Step 7: Transcribe spoken audio, if asked
+                const transcription = await this.extractAndTranscribeAudio(tempVideoPath, tempDir, metadata, filename, options);
+                // Step 8: Build textContent for LLM
+                const textContent = this.buildTextContent(metadata, keyframes.map((frame) => frame.timestampSec), subtitleText, this.getFilename(fileInfo), transcription.transcript);
                 span.setAttribute(ATTR.VIDEO_TEXT_CONTENT_LENGTH, textContent.length);
                 span.setAttribute(ATTR.FILE_OUTPUT_LENGTH, textContent.length);
                 span.setAttribute(ATTR.FILE_SUCCESS, true);
                 logger.info(`[NEUROLINK] Video processed: ${filename} → ${textContent.length} bytes text + ${keyframes.length} keyframes ` +
                     `(${metadata.durationFormatted}, ${metadata.width}x${metadata.height}, ${metadata.codec})`);
-                // Step 8: Return structured result
+                // Step 9: Return structured result
                 return {
                     success: true,
                     data: {
@@ -482,9 +511,15 @@ export class VideoProcessor extends BaseFileProcessor {
                         size: fileInfo.size,
                         filename: this.getFilename(fileInfo),
                         textContent,
-                        keyframes,
+                        keyframes: keyframes.map((frame) => frame.buffer),
+                        keyframeTimestampsSec: keyframes.map((frame) => frame.timestampSec),
                         metadata,
                         subtitleText,
+                        transcript: transcription.transcript,
+                        hasTranscript: !!transcription.transcript,
+                        ...(transcription.skippedReason
+                            ? { transcriptionSkippedReason: transcription.skippedReason }
+                            : {}),
                         hasKeyframes: keyframes.length > 0,
                         frameCount: keyframes.length,
                     },
@@ -504,7 +539,7 @@ export class VideoProcessor extends BaseFileProcessor {
                 };
             }
             finally {
-                // Step 8: Clean up temp files
+                // Step 10: Clean up temp files
                 if (tempCreated) {
                     await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {
                         // Ignore cleanup errors - temp files will be cleaned by OS eventually
@@ -662,6 +697,154 @@ export class VideoProcessor extends BaseFileProcessor {
         }
         return Math.min(100, Math.max(1, Math.round(quality)));
     }
+    /**
+     * Extract the clip's audio track and transcribe the speech in it (#433).
+     *
+     * ## Why this exists at all, now that Gemini hears video directly
+     *
+     * It does not help Gemini — a native provider receives the clip and its
+     * audio together. It is for everyone else. A provider on the frame path
+     * gets stills and nothing else, so a recorded standup, a support call or a
+     * narrated demo arrives with its entire spoken content missing, and
+     * "summarise this meeting" is answered from four screenshots.
+     *
+     * ## Why it is not a stub
+     *
+     * The issue asked for one, on the grounds that no transcription backend
+     * existed yet. One does: `AudioProcessor` has shipped Whisper transcription
+     * for standalone audio files for some time. A method that logged "not yet
+     * implemented" and returned undefined would be dead code sitting next to a
+     * working implementation of the same thing, and `--transcribe-audio` would
+     * still do nothing.
+     *
+     * The Whisper call is reproduced here rather than shared with
+     * `AudioProcessor`: extracting it into a common module is the better
+     * long-term shape, but that file is being edited concurrently, and a
+     * merge conflict in the audio pipeline is a worse outcome than fifty
+     * duplicated lines. The duplication is worth removing once both land.
+     *
+     * ## Failure behaviour
+     *
+     * Best-effort throughout, and never throws: a clip with no audio track, a
+     * machine without ffmpeg, a missing key, an oversized track or a failed
+     * request all return a *reason* and leave the rest of the pipeline intact.
+     * Each reason is distinct, because from the outside every one of them
+     * looks the same — no transcript — while the remedies differ completely.
+     *
+     * @param videoPath - Temp path to the video, already written by the caller
+     * @param tempDir - Caller-owned temp directory; removed in its `finally`
+     * @param metadata - Probed metadata, consulted for an audio track
+     * @param filename - Display name, for logs
+     * @param options - Caller settings; transcription runs only when asked
+     */
+    async extractAndTranscribeAudio(videoPath, tempDir, metadata, filename, options) {
+        if (!options?.transcribeAudio) {
+            // Not a skip worth reporting: nobody asked. Reporting it would put a
+            // "no transcript because..." line in every video result.
+            return {};
+        }
+        const skipped = (reason) => {
+            logger.warn(`[VideoProcessor] No transcript for ${filename}: ${reason}. ` +
+                `The model will receive the keyframes and metadata only.`);
+            return { skippedReason: reason };
+        };
+        // Probing reports no audio codec both for a genuinely silent clip and
+        // for one nothing could open. Either way there is nothing to send, and
+        // the ffmpeg pass below would only fail more slowly.
+        if (!metadata.audioCodec) {
+            return skipped("the video has no audio track");
+        }
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            return skipped("OPENAI_API_KEY is not set, and Whisper is the only transcription backend wired up");
+        }
+        const audioPath = join(tempDir, "audio.mp3");
+        try {
+            await runFfmpeg([
+                "-y",
+                "-v",
+                "error",
+                "-i",
+                videoPath,
+                // Drop the video stream outright. Without -vn ffmpeg tries to carry
+                // it into an MP3 container as cover art and fails on most inputs.
+                "-vn",
+                // Mono at 16 kHz is what Whisper resamples to anyway, and it keeps
+                // an hour-long recording comfortably under the upload ceiling.
+                "-ac",
+                "1",
+                "-ar",
+                "16000",
+                "-c:a",
+                "libmp3lame",
+                "-q:a",
+                "4",
+                audioPath,
+            ], { timeoutMs: VIDEO_CONFIG.AUDIO_EXTRACT_TIMEOUT_MS });
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message.split("\n")[0] : String(error);
+            return skipped(`the audio track could not be extracted — ${detail}`);
+        }
+        let audioBuffer;
+        try {
+            audioBuffer = await fs.readFile(audioPath);
+        }
+        catch {
+            return skipped("ffmpeg reported success but wrote no audio file");
+        }
+        if (audioBuffer.length === 0) {
+            return skipped("the extracted audio track was empty");
+        }
+        const sizeMB = audioBuffer.length / (1024 * 1024);
+        if (sizeMB > VIDEO_CONFIG.WHISPER_MAX_SIZE_MB) {
+            return skipped(`the extracted audio is ${sizeMB.toFixed(1)}MB, over Whisper's ` +
+                `${VIDEO_CONFIG.WHISPER_MAX_SIZE_MB}MB limit — split the recording`);
+        }
+        const baseUrl = (process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/+$/, "");
+        const form = new FormData();
+        form.append("file", new Blob([new Uint8Array(audioBuffer)], { type: "audio/mpeg" }), "audio.mp3");
+        form.append("model", "whisper-1");
+        form.append("response_format", "verbose_json");
+        // `withTimeout` only races a promise against a timer — it cannot cancel
+        // the request. Without the abort, a timed-out upload keeps its socket and
+        // its in-flight body alive after the caller has already moved on.
+        const abort = new AbortController();
+        const timer = setTimeout(() => abort.abort(), VIDEO_CONFIG.TRANSCRIPTION_TIMEOUT_MS);
+        try {
+            const response = await withTimeout(fetch(`${baseUrl}/audio/transcriptions`, {
+                method: "POST",
+                headers: { Authorization: `Bearer ${apiKey}` },
+                body: form,
+                signal: abort.signal,
+            }), VIDEO_CONFIG.TRANSCRIPTION_TIMEOUT_MS);
+            if (!response.ok) {
+                const detail = await response.text().catch(() => "");
+                return skipped(`the transcription request failed — HTTP ${response.status}` +
+                    `${detail ? `: ${detail.slice(0, 200)}` : ""}`);
+            }
+            const payload = await response.json();
+            const text = typeof payload === "object" &&
+                payload !== null &&
+                typeof payload.text === "string"
+                ? payload.text.trim()
+                : "";
+            if (text.length === 0) {
+                // A successful call returning nothing is a real outcome — silence,
+                // music, no speech — and not the same as a failure.
+                return skipped("Whisper returned an empty transcript for this audio");
+            }
+            logger.debug(`[VideoProcessor] Transcribed ${filename} via openai-whisper (${text.length} chars)`);
+            return { transcript: text };
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            return skipped(`the transcription request failed — ${detail}`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
     /**
      * Extract keyframes from a video at calculated intervals.
      *
@@ -686,7 +869,7 @@ export class VideoProcessor extends BaseFileProcessor {
      * @param tempDir - Temp directory for frame output
      * @param durationSec - Video duration in seconds
      * @param options - Caller frame budget / encoder settings
-     * @returns Array of encoded frame buffers (JPEG unless png was requested)
+     * @returns Each kept frame with the second it was sampled at, ascending
      */
     async extractKeyframes(videoPath, tempDir, durationSec, options) {
         if (durationSec <= 0) {
@@ -726,7 +909,14 @@ export class VideoProcessor extends BaseFileProcessor {
         // Extract frames using ffmpeg
         const framesDir = join(tempDir, "frames");
         await fs.mkdir(framesDir, { recursive: true });
-        await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps, intervalSec);
+        // The requested `timestamps` are an idealized, evenly-spaced schedule.
+        // ffmpeg's `-vf select` only guarantees a MINIMUM gap since the last
+        // frame it actually selected — on a source whose real frame spacing
+        // exceeds that gap, the selected frames land later than the schedule
+        // and the gap compounds across every subsequent frame. `showinfo`
+        // reports each selected frame's true `pts_time`, which is what gets
+        // paired with the encoded file below instead of the idealized value.
+        const realTimestamps = await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps, intervalSec);
         // Read extracted frames and resize with sharp
         const keyframes = [];
         for (let i = 0; i < timestamps.length; i++) {
@@ -746,7 +936,14 @@ export class VideoProcessor extends BaseFileProcessor {
                 const resized = await (options?.format === "png"
                     ? pipeline.png({ quality })
                     : pipeline.jpeg({ quality })).toBuffer();
-                keyframes.push(resized);
+                // Paired as the frame is kept, not derived afterwards from the
+                // interval: the catch below drops a frame whose encode failed, and a
+                // reconstructed schedule would then mislabel every frame after it.
+                // Prefer the real PTS ffmpeg reported for this output file; only
+                // fall back to the idealized schedule value if `showinfo` didn't
+                // report one (e.g. an older/unexpected ffmpeg build).
+                const timestampSec = realTimestamps[i] ?? timestamps[i];
+                keyframes.push({ buffer: resized, timestampSec });
             }
             catch {
                 // Skip individual frame on resize/encode failure
@@ -757,12 +954,22 @@ export class VideoProcessor extends BaseFileProcessor {
     /**
      * Run ffmpeg to extract frames at specified timestamps.
      *
-     * Uses the `-vf select` filter to pick frames at exact timestamps,
-     * which is more efficient than seeking for each frame individually.
+     * Uses the `-vf select` filter to pick frames at least `intervalSec` apart,
+     * which is more efficient than seeking for each frame individually. This is
+     * a minimum-gap filter, not a seek to an exact time: on a source whose real
+     * frame spacing exceeds the requested interval, the frame actually selected
+     * can land well after the ideal schedule. A `showinfo` stage between
+     * `select` and `scale` reports each selected frame's true `pts_time` to
+     * stderr so the caller can label frames with when they were really
+     * sampled, not the idealized request.
      *
      * @param videoPath - Path to the video file
      * @param outputDir - Directory to write frame files
      * @param timestamps - Array of timestamps in seconds
+     * @param intervalSec - Minimum gap enforced between selected frames
+     * @returns The real `pts_time` (seconds) ffmpeg reported for each selected
+     *   frame, in output-file order; may be shorter than `timestamps` if fewer
+     *   real frames were available to select
      */
     async runFfmpegFrameExtraction(videoPath, outputDir, timestamps, intervalSec) {
         const ff = await loadFluentFfmpeg();
@@ -770,13 +977,15 @@ export class VideoProcessor extends BaseFileProcessor {
             // Improved select expression to pick exactly one frame per interval
             // instead of multiple frames within a 0.5s window.
             const selectExpr = `isnan(prev_selected_t)+gte(t-prev_selected_t,${intervalSec}-0.001)`;
+            const PTS_TIME_RE = /pts_time:\s*(-?\d+(?:\.\d+)?)/;
+            const realTimestamps = [];
             const timeoutId = setTimeout(() => {
                 reject(new Error(`ffmpeg frame extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
             }, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
             ff(videoPath)
                 .outputOptions([
                 "-vf",
-                `select='${selectExpr}',scale='min(${VIDEO_CONFIG.FRAME_MAX_DIMENSION}\\,iw):-2'`,
+                `select='${selectExpr}',showinfo,scale='min(${VIDEO_CONFIG.FRAME_MAX_DIMENSION}\\,iw):-2'`,
                 "-vsync",
                 "vfr",
                 "-q:v",
@@ -785,9 +994,17 @@ export class VideoProcessor extends BaseFileProcessor {
                 String(timestamps.length),
             ])
                 .output(join(outputDir, "frame_%04d.jpg"))
+                .on("stderr", (line) => {
+                // showinfo writes one log line per selected frame, in the same
+                // ascending order ffmpeg encodes frame_0001, frame_0002, ...
+                const match = PTS_TIME_RE.exec(line);
+                if (match) {
+                    realTimestamps.push(Number.parseFloat(match[1]));
+                }
+            })
                 .on("end", () => {
                 clearTimeout(timeoutId);
-                resolve();
+                resolve(realTimestamps);
             })
                 .on("error", (err) => {
                 clearTimeout(timeoutId);
@@ -906,12 +1123,13 @@ export class VideoProcessor extends BaseFileProcessor {
      * - Subtitle text (if available)
      *
      * @param metadata - Extracted video metadata
-     * @param frameCount - Number of keyframes extracted
+     * @param keyframeTimestampsSec - The real second each kept keyframe was sampled at
      * @param subtitleText - Extracted subtitle text (if any)
      * @param filename - Original filename
+     * @param transcript - Machine transcription of the spoken audio (if requested and available)
      * @returns Formatted text content for the LLM
      */
-    buildTextContent(metadata, frameCount, subtitleText, filename) {
+    buildTextContent(metadata, keyframeTimestampsSec, subtitleText, filename, transcript) {
         const lines = [];
         lines.push(`[Video File: ${filename}]`);
         lines.push("");
@@ -936,15 +1154,41 @@ export class VideoProcessor extends BaseFileProcessor {
         }
         lines.push(`- File Size: ${(metadata.fileSize / (1024 * 1024)).toFixed(1)} MB`);
         lines.push("");
-        if (frameCount > 0) {
-            const intervalSec = this.getFrameInterval(metadata.duration);
-            lines.push(`## Keyframes (${frameCount} frames extracted every ~${intervalSec}s)`);
-            lines.push("The following images are keyframes extracted from the video at regular intervals.");
+        if (keyframeTimestampsSec.length > 0) {
+            // The real sample times, not the duration tier's nominal interval. Those
+            // two disagree whenever the caller passed `videoOptions.frames`, because
+            // an explicit budget spreads frames evenly across the clip instead of
+            // following the tier — so a 3s clip asked for 16 frames used to be
+            // described as "every ~1s" while the frames were 0.19s apart. Listing
+            // what was actually sampled cannot drift from what was sent, and it is
+            // also what lets the model answer a question about a specific moment.
+            lines.push(`## Keyframes (${keyframeTimestampsSec.length} extracted)`);
+            lines.push("The images attached below are keyframes from this video, in order. " +
+                "Each was sampled at the timestamp listed here:");
+            for (const [index, timestampSec] of keyframeTimestampsSec.entries()) {
+                lines.push(`- Frame ${index + 1}: ${formatKeyframeTimestamp(timestampSec)}`);
+            }
         }
         else {
             lines.push("## Keyframes");
             lines.push("No keyframes could be extracted from this video (it may be audio-only or use an unsupported codec).");
         }
+        if (transcript) {
+            lines.push("");
+            // Labelled as transcribed speech, not as captions: the two can both be
+            // present and disagree, and a model told which is which can say so
+            // rather than averaging them into one confident answer.
+            lines.push("## Spoken Audio (transcribed)");
+            lines.push("The following is a machine transcription of the speech in this video:");
+            const maxTranscriptChars = 20_000;
+            if (transcript.length > maxTranscriptChars) {
+                lines.push(transcript.substring(0, maxTranscriptChars) +
+                    `\n... [truncated, ${transcript.length - maxTranscriptChars} more characters]`);
+            }
+            else {
+                lines.push(transcript);
+            }
+        }
         if (subtitleText) {
             lines.push("");
             lines.push("## Subtitles / Captions");

package/dist/types/file.d.ts

@@ -1,6 +1,7 @@
 /**
  * File detection and processing types for unified file handling
  */
+import type { ImageWithAltText } from "./multimodal.js";
 /**
  * Supported file types for multimodal input
  */
@@ -276,8 +277,15 @@ export type FileProcessingResult = {
     type: FileType;
     content: string | Buffer;
     mimeType: string;
-    /** Additional images extracted from the file (e.g., video keyframes, audio cover art) */
-    images?: Array<Buffer | string>;
+    /**
+     * Additional images extracted from the file (e.g., video keyframes, audio
+     * cover art).
+     *
+     * An entry may carry alt text, which is how a video keyframe states the
+     * timestamp it was sampled at. Bare bytes still work — the field was
+     * widened, not changed.
+     */
+    images?: Array<Buffer | string | ImageWithAltText>;
     metadata: {
         confidence: number;
         size?: number;
@@ -549,6 +557,16 @@ export type VideoProcessorOptions = {
     quality?: number;
     /** Frame encoding. Defaults to jpeg. */
     format?: "jpeg" | "png";
+    /**
+     * Transcribe the clip's spoken audio (#433). Off by default: it costs an
+     * ffmpeg pass plus a Whisper call, and most attached video is silent
+     * screen capture.
+     *
+     * Best-effort — a clip with no audio track, no `OPENAI_API_KEY`, or a
+     * failed call still processes, and the reason lands in
+     * `ProcessedVideo.transcriptionSkippedReason`.
+     */
+    transcribeAudio?: boolean;
 };
 /**
  * Office processor options for Word, PowerPoint, and Excel documents

package/dist/types/generate.d.ts

@@ -18,7 +18,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
 import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
-import type { CSVProcessorOptions, FileWithMetadata, MultimodalAudioEntry, MultimodalVideoEntry, VisionImageOutputFormat } from "./file.js";
+import type { CSVProcessorOptions, VideoProcessorOptions, FileWithMetadata, MultimodalAudioEntry, MultimodalVideoEntry, VisionImageOutputFormat } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { Schema, Tool, ToolChoice } from "./tools.js";
 import type { StepResult, LanguageModel } from "./providers.js";
@@ -180,16 +180,15 @@ export type GenerateOptions = {
          */
         outputFormat?: VisionImageOutputFormat;
     };
-    videoOptions?: {
-        /** Frames to extract. Unset lets VideoProcessor pick from the clip's duration; clamped to 100. */
-        frames?: number;
-        /** Frame encoder quality, clamped to 1-100. Default 80. */
-        quality?: number;
-        /** Frame encoding. Default jpeg. */
-        format?: "jpeg" | "png";
-        /** Not implemented yet (#433) — warns rather than silently doing nothing. */
-        transcribeAudio?: boolean;
-    };
+    /**
+     * Video processing options — keyframe budget, encoder settings, and
+     * whether to transcribe the clip's spoken audio.
+     *
+     * The canonical shape, rather than a third structurally-identical copy of
+     * it: `StreamOptions` and `TextGenerationOptions` declare the same field,
+     * and the inline copies had already drifted apart in what they documented.
+     */
+    videoOptions?: VideoProcessorOptions;
     /**
      * Text-to-Speech (TTS) configuration
      *
@@ -1428,6 +1427,16 @@ export type TextGenerationOptions = {
     expectedOutcome?: string;
     evaluationCriteria?: string[];
     csvOptions?: CSVProcessorOptions;
+    /**
+     * Video processing options (#478, #433).
+     *
+     * Reconstructed option objects have to carry this the same way they carry
+     * `csvOptions` and `pdfOptions`: the message builder hands it to the
+     * detector, which hands it to `VideoProcessor`, and anything that rebuilds
+     * the options along the way and omits it restores the defaults without
+     * saying so.
+     */
+    videoOptions?: VideoProcessorOptions;
     /** PDF processing options (#258). */
     pdfOptions?: {
         /** Password for an encrypted PDF (image-conversion fallback path). */
@@ -1453,21 +1462,6 @@ export type TextGenerationOptions = {
          */
         outputFormat?: VisionImageOutputFormat;
     };
-    /**
-     * Video processing options (mirrors `GenerateOptions.videoOptions`). Never
-     * declared here before, so `buildGenerateTextOptions` had nowhere to
-     * forward the caller's setting even once the allowlist itself named it.
-     */
-    videoOptions?: {
-        /** Frames to extract. Unset lets VideoProcessor pick from the clip's duration; clamped to 100. */
-        frames?: number;
-        /** Frame encoder quality, clamped to 1-100. Default 80. */
-        quality?: number;
-        /** Frame encoding. Default jpeg. */
-        format?: "jpeg" | "png";
-        /** Not implemented yet (#433) — warns rather than silently doing nothing. */
-        transcribeAudio?: boolean;
-    };
     enableSummarization?: boolean;
     /**
      * Skip injecting tool schemas into the system prompt.

package/dist/types/processor.d.ts

@@ -711,6 +711,20 @@ export type ProcessedAudio = ProcessedFileBase & {
     transcriptionSkippedReason?: string;
     coverArt?: Buffer;
 };
+/**
+ * One extracted keyframe and the moment it came from.
+ *
+ * Internal to extraction: `ProcessedVideo` splits this back into the parallel
+ * `keyframes` / `keyframeTimestampsSec` arrays its existing consumers expect.
+ * Keeping the pair together while frames are being read and encoded is what
+ * makes a dropped frame impossible to mislabel — the alternative is
+ * reconstructing timestamps from an interval after the fact, which is wrong
+ * for every frame following a failed encode.
+ */
+export type VideoKeyframe = {
+    readonly buffer: Buffer;
+    readonly timestampSec: number;
+};
 /**
  * Processed video result.
  * Extends ProcessedFileBase with video-specific fields including metadata,
@@ -720,6 +734,19 @@ export type ProcessedAudio = ProcessedFileBase & {
 export type ProcessedVideo = ProcessedFileBase & {
     textContent: string;
     keyframes: Buffer[];
+    /**
+     * When each entry of `keyframes` was sampled, in seconds from the start of
+     * the clip. Same length, same order, ascending.
+     *
+     * A parallel array rather than a richer `keyframes` element type: that
+     * field is read by `fileReferenceRegistry` and by the detector as a plain
+     * `Buffer[]`, and reshaping it would break them for a value they do not
+     * need. Extraction can also drop an individual frame when its encode
+     * fails, so the timestamps are recorded as frames are kept — never
+     * reconstructed from an interval, which would silently mislabel every
+     * frame after a dropped one.
+     */
+    keyframeTimestampsSec: number[];
     metadata: {
         duration: number;
         durationFormatted: string;
@@ -735,6 +762,22 @@ export type ProcessedVideo = ProcessedFileBase & {
         fileSize: number;
     };
     subtitleText?: string;
+    /**
+     * Speech transcribed from the clip's audio track, when
+     * `VideoProcessorOptions.transcribeAudio` asked for it and it worked.
+     *
+     * Distinct from `subtitleText`, which is an embedded subtitle stream the
+     * file already carried. A recording can have one, both or neither.
+     */
+    transcript?: string;
+    hasTranscript: boolean;
+    /**
+     * Why no transcript was produced. Follows `ProcessedAudio`'s precedent:
+     * "nobody asked", "there is no audio track", "no API key", "the call
+     * failed" and "nobody was speaking" are five different situations that
+     * otherwise all present as an absent transcript.
+     */
+    transcriptionSkippedReason?: string;
     hasKeyframes: boolean;
     frameCount: number;
 };

package/dist/types/proxyClient.d.ts

@@ -29,8 +29,10 @@ export type CliProxyClientConfigurator = {
     /**
      * Point the CLI at the proxy. `proxyBaseUrl` is the bare proxy origin
      * (e.g. "http://127.0.0.1:55669"); the configurator appends whatever path
-     * suffix its CLI needs. Returns false when nothing was written, so callers
-     * never print a success message for work that did not happen.
+     * suffix its CLI needs. Returns false when the CLI is not left pointing at
+     * the proxy, so callers never print a success message for work that did
+     * not happen. A configurator whose file already matches may return true
+     * without rewriting it.
      */
     apply: (proxyBaseUrl: string, options?: CliProxyClientApplyOptions) => Promise<boolean>;
     /**
@@ -55,7 +57,10 @@ export type CliProxyClientConfigurator = {
 export type CliProxyClientApplyResult = {
     id: string;
     displayName: string;
-    /** True only when the configurator actually wrote configuration. */
+    /**
+     * True only when the CLI's configuration points at the proxy: written now,
+     * or already written and left as it was.
+     */
     applied: boolean;
     /**
      * Set when the write landed but is not yet in effect — see
@@ -110,6 +115,26 @@ export type CliGrokSnapshot = {
     originalExisted: boolean;
     writtenBaseUrl: string;
 };
+/**
+ * One header-delimited run of a client's TOML config. `path` is the parsed
+ * header key (`["model", "gemini-2.5-pro"]` for `[model."gemini-2.5-pro"]`),
+ * or null for the keys before the first header. `lines` keep their own
+ * terminators, so text the writer does not own round-trips byte for byte.
+ */
+export type CliTomlSection = {
+    path: readonly string[] | null;
+    arrayTable: boolean;
+    lines: readonly string[];
+};
+/**
+ * What a line of TOML leaves open for the next one: a multi-line string, or
+ * brackets of an array or inline table. A `[` that starts a line is a table
+ * header only when neither is open.
+ */
+export type CliTomlScanState = {
+    multiline: '"""' | "'''" | null;
+    depth: number;
+};
 /**
  * Snapshot of the user's pre-existing Gemini CLI `~/.gemini/.env`.
  *