@juspay/neurolink 12.29.1 → 12.30.0

package/dist/cli/factories/commandFactory.js

@@ -171,7 +171,8 @@ export class CLICommandFactory {
         "transcribe-audio": {
             type: "boolean",
             default: false,
-            description: "Extract and transcribe audio from video",
+            description: "Transcribe the video's spoken audio (needs ffmpeg and OPENAI_API_KEY). " +
+                "Unnecessary for Gemini, which hears the clip directly",
         },
         file: {
             type: "string",

package/dist/core/baseProvider.js

@@ -936,6 +936,7 @@ export class BaseProvider {
                 context: options.context,
                 csvOptions: options.csvOptions,
                 pdfOptions: options.pdfOptions,
+                videoOptions: options.videoOptions,
                 // Forward abort, tool filtering, and timeout options to prevent
                 // silent bypass when falling back from real streaming to fake streaming
                 abortSignal: options.abortSignal,

package/dist/core/modules/MessageBuilder.js

@@ -105,6 +105,10 @@ export class MessageBuilder {
                     csvOptions: options.csvOptions,
                     pdfOptions: options.pdfOptions,
                     imageOptions: options.imageOptions,
+                    // Same reason as csvOptions: the keyframe budget and the
+                    // transcription flag are read downstream by the detector, and
+                    // a reconstruction that omits them silently restores defaults.
+                    videoOptions: options.videoOptions,
                     provider: options.provider,
                     model: options.model,
                     temperature: options.temperature,
@@ -236,6 +240,10 @@ export class MessageBuilder {
                     csvOptions: options.csvOptions,
                     pdfOptions: options.pdfOptions,
                     imageOptions: options.imageOptions,
+                    // Same reason as csvOptions: the keyframe budget and the
+                    // transcription flag are read downstream by the detector, and
+                    // a reconstruction that omits them silently restores defaults.
+                    videoOptions: options.videoOptions,
                     provider: options.provider,
                     model: options.model,
                     temperature: options.temperature,

package/dist/processors/media/VideoProcessor.d.ts

@@ -142,6 +142,47 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
      * falling back to the default when absent or non-numeric (#478).
      */
     private static resolveFrameQuality;
+    /**
+     * Extract the clip's audio track and transcribe the speech in it (#433).
+     *
+     * ## Why this exists at all, now that Gemini hears video directly
+     *
+     * It does not help Gemini — a native provider receives the clip and its
+     * audio together. It is for everyone else. A provider on the frame path
+     * gets stills and nothing else, so a recorded standup, a support call or a
+     * narrated demo arrives with its entire spoken content missing, and
+     * "summarise this meeting" is answered from four screenshots.
+     *
+     * ## Why it is not a stub
+     *
+     * The issue asked for one, on the grounds that no transcription backend
+     * existed yet. One does: `AudioProcessor` has shipped Whisper transcription
+     * for standalone audio files for some time. A method that logged "not yet
+     * implemented" and returned undefined would be dead code sitting next to a
+     * working implementation of the same thing, and `--transcribe-audio` would
+     * still do nothing.
+     *
+     * The Whisper call is reproduced here rather than shared with
+     * `AudioProcessor`: extracting it into a common module is the better
+     * long-term shape, but that file is being edited concurrently, and a
+     * merge conflict in the audio pipeline is a worse outcome than fifty
+     * duplicated lines. The duplication is worth removing once both land.
+     *
+     * ## Failure behaviour
+     *
+     * Best-effort throughout, and never throws: a clip with no audio track, a
+     * machine without ffmpeg, a missing key, an oversized track or a failed
+     * request all return a *reason* and leave the rest of the pipeline intact.
+     * Each reason is distinct, because from the outside every one of them
+     * looks the same — no transcript — while the remedies differ completely.
+     *
+     * @param videoPath - Temp path to the video, already written by the caller
+     * @param tempDir - Caller-owned temp directory; removed in its `finally`
+     * @param metadata - Probed metadata, consulted for an audio track
+     * @param filename - Display name, for logs
+     * @param options - Caller settings; transcription runs only when asked
+     */
+    private extractAndTranscribeAudio;
     /**
      * Extract keyframes from a video at calculated intervals.
      *
@@ -166,18 +207,28 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
      * @param tempDir - Temp directory for frame output
      * @param durationSec - Video duration in seconds
      * @param options - Caller frame budget / encoder settings
-     * @returns Array of encoded frame buffers (JPEG unless png was requested)
+     * @returns Each kept frame with the second it was sampled at, ascending
      */
     private extractKeyframes;
     /**
      * Run ffmpeg to extract frames at specified timestamps.
      *
-     * Uses the `-vf select` filter to pick frames at exact timestamps,
-     * which is more efficient than seeking for each frame individually.
+     * Uses the `-vf select` filter to pick frames at least `intervalSec` apart,
+     * which is more efficient than seeking for each frame individually. This is
+     * a minimum-gap filter, not a seek to an exact time: on a source whose real
+     * frame spacing exceeds the requested interval, the frame actually selected
+     * can land well after the ideal schedule. A `showinfo` stage between
+     * `select` and `scale` reports each selected frame's true `pts_time` to
+     * stderr so the caller can label frames with when they were really
+     * sampled, not the idealized request.
      *
      * @param videoPath - Path to the video file
      * @param outputDir - Directory to write frame files
      * @param timestamps - Array of timestamps in seconds
+     * @param intervalSec - Minimum gap enforced between selected frames
+     * @returns The real `pts_time` (seconds) ffmpeg reported for each selected
+     *   frame, in output-file order; may be shorter than `timestamps` if fewer
+     *   real frames were available to select
      */
     private runFfmpegFrameExtraction;
     /**
@@ -217,9 +268,10 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
      * - Subtitle text (if available)
      *
      * @param metadata - Extracted video metadata
-     * @param frameCount - Number of keyframes extracted
+     * @param keyframeTimestampsSec - The real second each kept keyframe was sampled at
      * @param subtitleText - Extracted subtitle text (if any)
      * @param filename - Original filename
+     * @param transcript - Machine transcription of the spoken audio (if requested and available)
      * @returns Formatted text content for the LLM
      */
     private buildTextContent;

package/dist/processors/media/VideoProcessor.js

@@ -50,13 +50,15 @@ import { join } from "path";
 import { Readable } from "stream";
 import { pipeline } from "stream/promises";
 import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
-import { formatMediaDuration } from "../../utils/mediaDuration.js";
+import { formatKeyframeTimestamp, formatMediaDuration, } from "../../utils/mediaDuration.js";
 import { SIZE_LIMITS_MB } from "../config/index.js";
 import { extensionsForModality, mimeTypesForModality, } from "../config/fileTypeRegistry.js";
 import { FileErrorCode } from "../errors/index.js";
 import { tracers, ATTR, withSpan } from "../../telemetry/index.js";
 import { logger } from "../../utils/logger.js";
 import { tryImport } from "../../utils/tryImport.js";
+import { withTimeout } from "../../utils/errorHandling.js";
+import { runFfmpeg } from "../../adapters/video/ffmpegAdapter.js";
 /**
  * Narrow a loaded `fluent-ffmpeg` export to the shape this file actually uses:
  * a callable carrying the `ffprobe` and `setFfmpegPath` statics.
@@ -202,6 +204,21 @@ const VIDEO_CONFIG = {
     FFMPEG_TIMEOUT_MS: 120_000,
     /** Timeout for ffprobe metadata extraction in milliseconds */
     FFPROBE_TIMEOUT_MS: 10_000,
+    /**
+     * Timeout for demuxing the audio track out of a video (#433).
+     *
+     * Generous next to frame extraction: this reads the whole file rather than
+     * seeking to a handful of offsets, and a meeting recording is long.
+     */
+    AUDIO_EXTRACT_TIMEOUT_MS: 180_000,
+    /** Timeout for one Whisper transcription request in milliseconds */
+    TRANSCRIPTION_TIMEOUT_MS: 120_000,
+    /**
+     * Whisper's upload ceiling in MB. The extracted track is mono 16 kHz MP3,
+     * so this is roughly six hours of speech — a clip that breaches it is
+     * unusual enough to be worth saying so rather than silently truncating.
+     */
+    WHISPER_MAX_SIZE_MB: 25,
 };
 /**
  * Supported video MIME types — derived from the canonical registry so this
@@ -285,6 +302,8 @@ export class VideoProcessor extends BaseFileProcessor {
             filename: this.getFilename(fileInfo),
             textContent: "",
             keyframes: [],
+            keyframeTimestampsSec: [],
+            hasTranscript: false,
             metadata: {
                 duration: 0,
                 durationFormatted: formatMediaDuration(0),
@@ -456,6 +475,14 @@ export class VideoProcessor extends BaseFileProcessor {
                     logger.warn(`[NEUROLINK] Video keyframe extraction failed for ${filename}, continuing without keyframes`);
                 }
                 span.setAttribute(ATTR.VIDEO_KEYFRAMES_EXTRACTED, keyframes.length);
+                // Debug-only: the real per-frame sample times, so a caller running
+                // with --debug can see what the model was actually told without
+                // re-deriving it from the request body.
+                if (logger.shouldLog("debug") && keyframes.length > 0) {
+                    logger.debug(`[NEUROLINK] Video keyframe timestamps: ${keyframes
+                        .map((frame) => frame.timestampSec.toFixed(3))
+                        .join(",")}`);
+                }
                 // Step 6: Extract subtitles
                 let subtitleText;
                 if (metadata.subtitleTracks > 0) {
@@ -466,14 +493,16 @@ export class VideoProcessor extends BaseFileProcessor {
                         // Non-fatal: continue without subtitles if extraction fails
                     }
                 }
-                // Step 7: Build textContent for LLM
-                const textContent = this.buildTextContent(metadata, keyframes.length, subtitleText, this.getFilename(fileInfo));
+                // Step 7: Transcribe spoken audio, if asked
+                const transcription = await this.extractAndTranscribeAudio(tempVideoPath, tempDir, metadata, filename, options);
+                // Step 8: Build textContent for LLM
+                const textContent = this.buildTextContent(metadata, keyframes.map((frame) => frame.timestampSec), subtitleText, this.getFilename(fileInfo), transcription.transcript);
                 span.setAttribute(ATTR.VIDEO_TEXT_CONTENT_LENGTH, textContent.length);
                 span.setAttribute(ATTR.FILE_OUTPUT_LENGTH, textContent.length);
                 span.setAttribute(ATTR.FILE_SUCCESS, true);
                 logger.info(`[NEUROLINK] Video processed: ${filename} → ${textContent.length} bytes text + ${keyframes.length} keyframes ` +
                     `(${metadata.durationFormatted}, ${metadata.width}x${metadata.height}, ${metadata.codec})`);
-                // Step 8: Return structured result
+                // Step 9: Return structured result
                 return {
                     success: true,
                     data: {
@@ -482,9 +511,15 @@ export class VideoProcessor extends BaseFileProcessor {
                         size: fileInfo.size,
                         filename: this.getFilename(fileInfo),
                         textContent,
-                        keyframes,
+                        keyframes: keyframes.map((frame) => frame.buffer),
+                        keyframeTimestampsSec: keyframes.map((frame) => frame.timestampSec),
                         metadata,
                         subtitleText,
+                        transcript: transcription.transcript,
+                        hasTranscript: !!transcription.transcript,
+                        ...(transcription.skippedReason
+                            ? { transcriptionSkippedReason: transcription.skippedReason }
+                            : {}),
                         hasKeyframes: keyframes.length > 0,
                         frameCount: keyframes.length,
                     },
@@ -504,7 +539,7 @@ export class VideoProcessor extends BaseFileProcessor {
                 };
             }
             finally {
-                // Step 8: Clean up temp files
+                // Step 10: Clean up temp files
                 if (tempCreated) {
                     await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {
                         // Ignore cleanup errors - temp files will be cleaned by OS eventually
@@ -662,6 +697,154 @@ export class VideoProcessor extends BaseFileProcessor {
         }
         return Math.min(100, Math.max(1, Math.round(quality)));
     }
+    /**
+     * Extract the clip's audio track and transcribe the speech in it (#433).
+     *
+     * ## Why this exists at all, now that Gemini hears video directly
+     *
+     * It does not help Gemini — a native provider receives the clip and its
+     * audio together. It is for everyone else. A provider on the frame path
+     * gets stills and nothing else, so a recorded standup, a support call or a
+     * narrated demo arrives with its entire spoken content missing, and
+     * "summarise this meeting" is answered from four screenshots.
+     *
+     * ## Why it is not a stub
+     *
+     * The issue asked for one, on the grounds that no transcription backend
+     * existed yet. One does: `AudioProcessor` has shipped Whisper transcription
+     * for standalone audio files for some time. A method that logged "not yet
+     * implemented" and returned undefined would be dead code sitting next to a
+     * working implementation of the same thing, and `--transcribe-audio` would
+     * still do nothing.
+     *
+     * The Whisper call is reproduced here rather than shared with
+     * `AudioProcessor`: extracting it into a common module is the better
+     * long-term shape, but that file is being edited concurrently, and a
+     * merge conflict in the audio pipeline is a worse outcome than fifty
+     * duplicated lines. The duplication is worth removing once both land.
+     *
+     * ## Failure behaviour
+     *
+     * Best-effort throughout, and never throws: a clip with no audio track, a
+     * machine without ffmpeg, a missing key, an oversized track or a failed
+     * request all return a *reason* and leave the rest of the pipeline intact.
+     * Each reason is distinct, because from the outside every one of them
+     * looks the same — no transcript — while the remedies differ completely.
+     *
+     * @param videoPath - Temp path to the video, already written by the caller
+     * @param tempDir - Caller-owned temp directory; removed in its `finally`
+     * @param metadata - Probed metadata, consulted for an audio track
+     * @param filename - Display name, for logs
+     * @param options - Caller settings; transcription runs only when asked
+     */
+    async extractAndTranscribeAudio(videoPath, tempDir, metadata, filename, options) {
+        if (!options?.transcribeAudio) {
+            // Not a skip worth reporting: nobody asked. Reporting it would put a
+            // "no transcript because..." line in every video result.
+            return {};
+        }
+        const skipped = (reason) => {
+            logger.warn(`[VideoProcessor] No transcript for ${filename}: ${reason}. ` +
+                `The model will receive the keyframes and metadata only.`);
+            return { skippedReason: reason };
+        };
+        // Probing reports no audio codec both for a genuinely silent clip and
+        // for one nothing could open. Either way there is nothing to send, and
+        // the ffmpeg pass below would only fail more slowly.
+        if (!metadata.audioCodec) {
+            return skipped("the video has no audio track");
+        }
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            return skipped("OPENAI_API_KEY is not set, and Whisper is the only transcription backend wired up");
+        }
+        const audioPath = join(tempDir, "audio.mp3");
+        try {
+            await runFfmpeg([
+                "-y",
+                "-v",
+                "error",
+                "-i",
+                videoPath,
+                // Drop the video stream outright. Without -vn ffmpeg tries to carry
+                // it into an MP3 container as cover art and fails on most inputs.
+                "-vn",
+                // Mono at 16 kHz is what Whisper resamples to anyway, and it keeps
+                // an hour-long recording comfortably under the upload ceiling.
+                "-ac",
+                "1",
+                "-ar",
+                "16000",
+                "-c:a",
+                "libmp3lame",
+                "-q:a",
+                "4",
+                audioPath,
+            ], { timeoutMs: VIDEO_CONFIG.AUDIO_EXTRACT_TIMEOUT_MS });
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message.split("\n")[0] : String(error);
+            return skipped(`the audio track could not be extracted — ${detail}`);
+        }
+        let audioBuffer;
+        try {
+            audioBuffer = await fs.readFile(audioPath);
+        }
+        catch {
+            return skipped("ffmpeg reported success but wrote no audio file");
+        }
+        if (audioBuffer.length === 0) {
+            return skipped("the extracted audio track was empty");
+        }
+        const sizeMB = audioBuffer.length / (1024 * 1024);
+        if (sizeMB > VIDEO_CONFIG.WHISPER_MAX_SIZE_MB) {
+            return skipped(`the extracted audio is ${sizeMB.toFixed(1)}MB, over Whisper's ` +
+                `${VIDEO_CONFIG.WHISPER_MAX_SIZE_MB}MB limit — split the recording`);
+        }
+        const baseUrl = (process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/+$/, "");
+        const form = new FormData();
+        form.append("file", new Blob([new Uint8Array(audioBuffer)], { type: "audio/mpeg" }), "audio.mp3");
+        form.append("model", "whisper-1");
+        form.append("response_format", "verbose_json");
+        // `withTimeout` only races a promise against a timer — it cannot cancel
+        // the request. Without the abort, a timed-out upload keeps its socket and
+        // its in-flight body alive after the caller has already moved on.
+        const abort = new AbortController();
+        const timer = setTimeout(() => abort.abort(), VIDEO_CONFIG.TRANSCRIPTION_TIMEOUT_MS);
+        try {
+            const response = await withTimeout(fetch(`${baseUrl}/audio/transcriptions`, {
+                method: "POST",
+                headers: { Authorization: `Bearer ${apiKey}` },
+                body: form,
+                signal: abort.signal,
+            }), VIDEO_CONFIG.TRANSCRIPTION_TIMEOUT_MS);
+            if (!response.ok) {
+                const detail = await response.text().catch(() => "");
+                return skipped(`the transcription request failed — HTTP ${response.status}` +
+                    `${detail ? `: ${detail.slice(0, 200)}` : ""}`);
+            }
+            const payload = await response.json();
+            const text = typeof payload === "object" &&
+                payload !== null &&
+                typeof payload.text === "string"
+                ? payload.text.trim()
+                : "";
+            if (text.length === 0) {
+                // A successful call returning nothing is a real outcome — silence,
+                // music, no speech — and not the same as a failure.
+                return skipped("Whisper returned an empty transcript for this audio");
+            }
+            logger.debug(`[VideoProcessor] Transcribed ${filename} via openai-whisper (${text.length} chars)`);
+            return { transcript: text };
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            return skipped(`the transcription request failed — ${detail}`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
     /**
      * Extract keyframes from a video at calculated intervals.
      *
@@ -686,7 +869,7 @@ export class VideoProcessor extends BaseFileProcessor {
      * @param tempDir - Temp directory for frame output
      * @param durationSec - Video duration in seconds
      * @param options - Caller frame budget / encoder settings
-     * @returns Array of encoded frame buffers (JPEG unless png was requested)
+     * @returns Each kept frame with the second it was sampled at, ascending
      */
     async extractKeyframes(videoPath, tempDir, durationSec, options) {
         if (durationSec <= 0) {
@@ -726,7 +909,14 @@ export class VideoProcessor extends BaseFileProcessor {
         // Extract frames using ffmpeg
         const framesDir = join(tempDir, "frames");
         await fs.mkdir(framesDir, { recursive: true });
-        await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps, intervalSec);
+        // The requested `timestamps` are an idealized, evenly-spaced schedule.
+        // ffmpeg's `-vf select` only guarantees a MINIMUM gap since the last
+        // frame it actually selected — on a source whose real frame spacing
+        // exceeds that gap, the selected frames land later than the schedule
+        // and the gap compounds across every subsequent frame. `showinfo`
+        // reports each selected frame's true `pts_time`, which is what gets
+        // paired with the encoded file below instead of the idealized value.
+        const realTimestamps = await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps, intervalSec);
         // Read extracted frames and resize with sharp
         const keyframes = [];
         for (let i = 0; i < timestamps.length; i++) {
@@ -746,7 +936,14 @@ export class VideoProcessor extends BaseFileProcessor {
                 const resized = await (options?.format === "png"
                     ? pipeline.png({ quality })
                     : pipeline.jpeg({ quality })).toBuffer();
-                keyframes.push(resized);
+                // Paired as the frame is kept, not derived afterwards from the
+                // interval: the catch below drops a frame whose encode failed, and a
+                // reconstructed schedule would then mislabel every frame after it.
+                // Prefer the real PTS ffmpeg reported for this output file; only
+                // fall back to the idealized schedule value if `showinfo` didn't
+                // report one (e.g. an older/unexpected ffmpeg build).
+                const timestampSec = realTimestamps[i] ?? timestamps[i];
+                keyframes.push({ buffer: resized, timestampSec });
             }
             catch {
                 // Skip individual frame on resize/encode failure
@@ -757,12 +954,22 @@ export class VideoProcessor extends BaseFileProcessor {
     /**
      * Run ffmpeg to extract frames at specified timestamps.
      *
-     * Uses the `-vf select` filter to pick frames at exact timestamps,
-     * which is more efficient than seeking for each frame individually.
+     * Uses the `-vf select` filter to pick frames at least `intervalSec` apart,
+     * which is more efficient than seeking for each frame individually. This is
+     * a minimum-gap filter, not a seek to an exact time: on a source whose real
+     * frame spacing exceeds the requested interval, the frame actually selected
+     * can land well after the ideal schedule. A `showinfo` stage between
+     * `select` and `scale` reports each selected frame's true `pts_time` to
+     * stderr so the caller can label frames with when they were really
+     * sampled, not the idealized request.
      *
      * @param videoPath - Path to the video file
      * @param outputDir - Directory to write frame files
      * @param timestamps - Array of timestamps in seconds
+     * @param intervalSec - Minimum gap enforced between selected frames
+     * @returns The real `pts_time` (seconds) ffmpeg reported for each selected
+     *   frame, in output-file order; may be shorter than `timestamps` if fewer
+     *   real frames were available to select
      */
     async runFfmpegFrameExtraction(videoPath, outputDir, timestamps, intervalSec) {
         const ff = await loadFluentFfmpeg();
@@ -770,13 +977,15 @@ export class VideoProcessor extends BaseFileProcessor {
             // Improved select expression to pick exactly one frame per interval
             // instead of multiple frames within a 0.5s window.
             const selectExpr = `isnan(prev_selected_t)+gte(t-prev_selected_t,${intervalSec}-0.001)`;
+            const PTS_TIME_RE = /pts_time:\s*(-?\d+(?:\.\d+)?)/;
+            const realTimestamps = [];
             const timeoutId = setTimeout(() => {
                 reject(new Error(`ffmpeg frame extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
             }, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
             ff(videoPath)
                 .outputOptions([
                 "-vf",
-                `select='${selectExpr}',scale='min(${VIDEO_CONFIG.FRAME_MAX_DIMENSION}\\,iw):-2'`,
+                `select='${selectExpr}',showinfo,scale='min(${VIDEO_CONFIG.FRAME_MAX_DIMENSION}\\,iw):-2'`,
                 "-vsync",
                 "vfr",
                 "-q:v",
@@ -785,9 +994,17 @@ export class VideoProcessor extends BaseFileProcessor {
                 String(timestamps.length),
             ])
                 .output(join(outputDir, "frame_%04d.jpg"))
+                .on("stderr", (line) => {
+                // showinfo writes one log line per selected frame, in the same
+                // ascending order ffmpeg encodes frame_0001, frame_0002, ...
+                const match = PTS_TIME_RE.exec(line);
+                if (match) {
+                    realTimestamps.push(Number.parseFloat(match[1]));
+                }
+            })
                 .on("end", () => {
                 clearTimeout(timeoutId);
-                resolve();
+                resolve(realTimestamps);
             })
                 .on("error", (err) => {
                 clearTimeout(timeoutId);
@@ -906,12 +1123,13 @@ export class VideoProcessor extends BaseFileProcessor {
      * - Subtitle text (if available)
      *
      * @param metadata - Extracted video metadata
-     * @param frameCount - Number of keyframes extracted
+     * @param keyframeTimestampsSec - The real second each kept keyframe was sampled at
      * @param subtitleText - Extracted subtitle text (if any)
      * @param filename - Original filename
+     * @param transcript - Machine transcription of the spoken audio (if requested and available)
      * @returns Formatted text content for the LLM
      */
-    buildTextContent(metadata, frameCount, subtitleText, filename) {
+    buildTextContent(metadata, keyframeTimestampsSec, subtitleText, filename, transcript) {
         const lines = [];
         lines.push(`[Video File: ${filename}]`);
         lines.push("");
@@ -936,15 +1154,41 @@ export class VideoProcessor extends BaseFileProcessor {
         }
         lines.push(`- File Size: ${(metadata.fileSize / (1024 * 1024)).toFixed(1)} MB`);
         lines.push("");
-        if (frameCount > 0) {
-            const intervalSec = this.getFrameInterval(metadata.duration);
-            lines.push(`## Keyframes (${frameCount} frames extracted every ~${intervalSec}s)`);
-            lines.push("The following images are keyframes extracted from the video at regular intervals.");
+        if (keyframeTimestampsSec.length > 0) {
+            // The real sample times, not the duration tier's nominal interval. Those
+            // two disagree whenever the caller passed `videoOptions.frames`, because
+            // an explicit budget spreads frames evenly across the clip instead of
+            // following the tier — so a 3s clip asked for 16 frames used to be
+            // described as "every ~1s" while the frames were 0.19s apart. Listing
+            // what was actually sampled cannot drift from what was sent, and it is
+            // also what lets the model answer a question about a specific moment.
+            lines.push(`## Keyframes (${keyframeTimestampsSec.length} extracted)`);
+            lines.push("The images attached below are keyframes from this video, in order. " +
+                "Each was sampled at the timestamp listed here:");
+            for (const [index, timestampSec] of keyframeTimestampsSec.entries()) {
+                lines.push(`- Frame ${index + 1}: ${formatKeyframeTimestamp(timestampSec)}`);
+            }
         }
         else {
             lines.push("## Keyframes");
             lines.push("No keyframes could be extracted from this video (it may be audio-only or use an unsupported codec).");
         }
+        if (transcript) {
+            lines.push("");
+            // Labelled as transcribed speech, not as captions: the two can both be
+            // present and disagree, and a model told which is which can say so
+            // rather than averaging them into one confident answer.
+            lines.push("## Spoken Audio (transcribed)");
+            lines.push("The following is a machine transcription of the speech in this video:");
+            const maxTranscriptChars = 20_000;
+            if (transcript.length > maxTranscriptChars) {
+                lines.push(transcript.substring(0, maxTranscriptChars) +
+                    `\n... [truncated, ${transcript.length - maxTranscriptChars} more characters]`);
+            }
+            else {
+                lines.push(transcript);
+            }
+        }
         if (subtitleText) {
             lines.push("");
             lines.push("## Subtitles / Captions");

package/dist/providers/catalog/deepseek.json

@@ -74,7 +74,8 @@
         }
     ],
     "quirks": {
-        "responseFormatDowngrade": "json-schema-to-json-object"
+        "responseFormatDowngrade": "json-schema-to-json-object",
+        "replayReasoningContent": true
     },
     "setup": {
         "url": "https://platform.deepseek.com/api_keys",

package/dist/providers/catalog/loader.js

@@ -158,6 +158,9 @@ export function buildCatalogEntries() {
         if (entry.quirks?.responseFormatDowngrade) {
             base.responseFormatDowngrade = entry.quirks.responseFormatDowngrade;
         }
+        if (entry.quirks?.replayReasoningContent) {
+            base.replayReasoningContent = true;
+        }
         return base;
     });
 }

package/dist/providers/catalog/provider-catalog.schema.json

@@ -280,6 +280,10 @@
           "type": "string",
           "enum": ["json-schema-to-json-object"],
           "description": "Vendor rejects response_format: { type: \"json_schema\" } outright but accepts { type: \"json_object\" }. ConfiguredOpenAICompatProvider downgrades the request so generate({ schema }) keeps working."
+        },
+        "replayReasoningContent": {
+          "type": "boolean",
+          "description": "Vendor wants each assistant turn's reasoning_content sent back on every later request of the conversation (DeepSeek documents a 400 without it once tools are in play). Off for every other provider, since strict OpenAI-compatible backends reject the unknown field."
         }
       }
     },

package/dist/providers/catalog/schema.d.ts

@@ -85,6 +85,7 @@ export declare const providerCatalogJsonSchema: z.ZodObject<{
         messageContentFormat: z.ZodOptional<z.ZodLiteral<"string">>;
         registryDefaultIgnoresModelEnvVar: z.ZodOptional<z.ZodBoolean>;
         responseFormatDowngrade: z.ZodOptional<z.ZodLiteral<"json-schema-to-json-object">>;
+        replayReasoningContent: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strict>>;
     timeouts: z.ZodOptional<z.ZodObject<{
         generateMs: z.ZodOptional<z.ZodNumber>;

package/dist/providers/catalog/schema.js

@@ -140,6 +140,7 @@ const catalogQuirksSchema = z.strictObject({
     messageContentFormat: z.literal("string").optional(),
     registryDefaultIgnoresModelEnvVar: z.boolean().optional(),
     responseFormatDowngrade: z.literal("json-schema-to-json-object").optional(),
+    replayReasoningContent: z.boolean().optional(),
 });
 const catalogTimeoutsSchema = z.strictObject({
     generateMs: z.number().optional(),

package/dist/providers/configuredOpenAICompat.d.ts

@@ -57,5 +57,6 @@ export declare class ConfiguredOpenAICompatProvider extends OpenAIChatCompletion
     supportsTools(): boolean;
     protected adjustRequestBody(body: OpenAICompatChatRequest, modelId: string): OpenAICompatChatRequest;
     protected adjustResponseFormat(rf: OpenAICompatResponseFormat | undefined, modelId: string): OpenAICompatResponseFormat | undefined;
+    protected replayReasoningContent(): boolean;
     protected formatProviderError(error: unknown): Error;
 }

package/dist/providers/configuredOpenAICompat.js

@@ -130,6 +130,9 @@ export class ConfiguredOpenAICompatProvider extends OpenAIChatCompletionsProvide
         }
         return super.adjustResponseFormat(rf, modelId);
     }
+    replayReasoningContent() {
+        return this.entry.replayReasoningContent === true;
+    }
     formatProviderError(error) {
         // classifyProviderError hard-codes TimeoutError -> NetworkError ahead
         // of any rule table and does not allow a per-provider override. An

package/dist/providers/openaiChatCompletionsBase.d.ts

@@ -102,6 +102,14 @@ export declare abstract class OpenAIChatCompletionsProvider extends BaseProvider
      * deliberately. LiteLLM overrides this to true.
      */
     protected useStreamingWireForGenerate(): boolean;
+    /**
+     * When true, an assistant turn's reasoning goes back to the vendor as
+     * `reasoning_content` on every later request of the conversation. DeepSeek
+     * documents this as required once tools are in play and answers 400
+     * without it. Default false: strict OpenAI-compatible backends reject a
+     * field they don't know. Read per request, never during construction.
+     */
+    protected replayReasoningContent(): boolean;
     /**
      * Hook to adjust the fully-built wire request body before it is sent, on
      * both the streaming and non-streaming paths. Default identity. Override for