@juspay/neurolink 12.29.2 → 12.30.1

package/dist/types/stream.d.ts

@@ -16,7 +16,7 @@ import type { AIModelProviderConfig, NeurolinkCredentials } from "./providers.js
 import type { TTSChunk, TTSOptions, TTSResult } from "./tts.js";
 import type { STTOptions, STTResult } from "./stt.js";
 import type { StandardRecord, ValidationSchema } from "./aliases.js";
-import type { CSVProcessorOptions, FileWithMetadata, VisionImageOutputFormat } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata, VideoProcessorOptions, VisionImageOutputFormat } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { LanguageModel, StepResult } from "./providers.js";
 import type { Tool, ToolChoice } from "./tools.js";
@@ -356,6 +356,16 @@ export type StreamOptions = {
         };
     };
     csvOptions?: CSVProcessorOptions;
+    /**
+     * Video processing options (#478, #433).
+     *
+     * Reconstructed option objects have to carry this the same way they carry
+     * `csvOptions` and `pdfOptions`: the message builder hands it to the
+     * detector, which hands it to `VideoProcessor`, and anything that rebuilds
+     * the options along the way and omits it restores the defaults without
+     * saying so.
+     */
+    videoOptions?: VideoProcessorOptions;
     /** PDF processing options (#258). */
     pdfOptions?: {
         /** Password for an encrypted PDF (image-conversion fallback path). */
@@ -386,16 +396,6 @@ export type StreamOptions = {
          */
         outputFormat?: VisionImageOutputFormat;
     };
-    videoOptions?: {
-        /** Frames to extract. Unset lets VideoProcessor pick from the clip's duration; clamped to 100. */
-        frames?: number;
-        /** Frame encoder quality, clamped to 1-100. Default 80. */
-        quality?: number;
-        /** Frame encoding. Default jpeg. */
-        format?: "jpeg" | "png";
-        /** Not implemented yet (#433) — warns rather than silently doing nothing. */
-        transcribeAudio?: boolean;
-    };
     /**
      * Text-to-Speech (TTS) configuration for streaming
      *

package/dist/utils/fileDetector.d.ts

@@ -114,6 +114,21 @@ export declare class FileDetector {
     /**
      * SDK-8: Format an informative placeholder when a file processor fails.
      * Instead of bare "[Video file: name]" strings, include size, format, and
+     * Describe one keyframe well enough for a model to place it in time.
+     *
+     * The clip's total length rides along because the eighth frame of a
+     * recording means nothing without knowing how long the recording is.
+     *
+     * Explicit units rather than a clock — see `formatKeyframeTimestamp`. An
+     * earlier draft used "0:03" here and the model reported the frame as
+     * appearing at "3:00".
+     *
+     * A timestamp can be missing if extraction and its schedule ever disagree
+     * in length. That should not happen, but an unlabelled frame is better
+     * than one labelled "NaN".
+     */
+    private static describeKeyframe;
+    /**
      * the reason for failure so the LLM can acknowledge the attachment.
      */
     private static formatInformativePlaceholder;

package/dist/utils/fileDetector.js

@@ -37,6 +37,7 @@ import { withTimeout } from "./errorHandling.js";
 import { normalizeUrlForCache, redactUrlForError, sanitizeErrorCause, } from "./logSanitize.js";
 import { mimeHintToExtension, mimeHintToFileType, normalizeMimeHint, } from "./mimeTypeHints.js";
 import { PDFProcessor } from "./pdfProcessor.js";
+import { formatKeyframeTimestamp } from "./mediaDuration.js";
 /**
  * Short-TTL cache of URL → Content-Type (#323). A URL is commonly detected more
  * than once (repeated multimodal prompts reuse the same asset URL); caching the
@@ -797,6 +798,26 @@ export class FileDetector {
     /**
      * SDK-8: Format an informative placeholder when a file processor fails.
      * Instead of bare "[Video file: name]" strings, include size, format, and
+     * Describe one keyframe well enough for a model to place it in time.
+     *
+     * The clip's total length rides along because the eighth frame of a
+     * recording means nothing without knowing how long the recording is.
+     *
+     * Explicit units rather than a clock — see `formatKeyframeTimestamp`. An
+     * earlier draft used "0:03" here and the model reported the frame as
+     * appearing at "3:00".
+     *
+     * A timestamp can be missing if extraction and its schedule ever disagree
+     * in length. That should not happen, but an unlabelled frame is better
+     * than one labelled "NaN".
+     */
+    static describeKeyframe(timestampSec, durationFormatted) {
+        if (typeof timestampSec !== "number" || !Number.isFinite(timestampSec)) {
+            return undefined;
+        }
+        return `Video keyframe at ${formatKeyframeTimestamp(timestampSec)} into a ${durationFormatted} video`;
+    }
+    /**
      * the reason for failure so the LLM can acknowledge the attachment.
      */
     static formatInformativePlaceholder(typeName, filename, content, detection, error) {
@@ -1120,25 +1141,37 @@ export class FileDetector {
             // the processor; previously these stopped at the CLI layer.
             videoOptions);
             if (videoResult.success && videoResult.data) {
+                // Bound to a local because the keyframe map below is a closure, and
+                // TypeScript drops the `videoResult.data` narrowing across one.
+                const video = videoResult.data;
                 return {
                     type: "video",
-                    content: videoResult.data.textContent ||
+                    content: video.textContent ||
                         FileDetector.formatInformativePlaceholder("Video", videoFilename, content, detection),
                     mimeType: detection.mimeType,
-                    images: videoResult.data.keyframes && videoResult.data.keyframes.length > 0
-                        ? videoResult.data.keyframes
+                    // Each keyframe carries the moment it was sampled at. Alt text is
+                    // the only per-image channel that survives to the model — the
+                    // message builder folds it into the prompt as
+                    // "[Image N: <alt>]" — so without it the frames arrive as an
+                    // unlabelled pile and "what happens at 0:30?" has no answer even
+                    // though the frame at 0:30 is right there.
+                    images: video.keyframes && video.keyframes.length > 0
+                        ? video.keyframes.map((frame, index) => ({
+                            data: frame,
+                            altText: FileDetector.describeKeyframe(video.keyframeTimestampsSec[index], video.metadata.durationFormatted),
+                        }))
                         : undefined,
                     metadata: {
                         ...detection.metadata,
-                        frameCount: videoResult.data.frameCount,
-                        hasKeyframes: videoResult.data.hasKeyframes,
+                        frameCount: video.frameCount,
+                        hasKeyframes: video.hasKeyframes,
                         // Carried so native delivery can apply its duration ceiling
                         // without opening the container a second time. The processor
                         // reports 0 when nothing could probe it; that is "unknown", not a
                         // zero-length clip, so it is left off rather than passed through
                         // as a number a ceiling check would happily accept.
-                        ...(videoResult.data.metadata.duration > 0
-                            ? { durationSec: videoResult.data.metadata.duration }
+                        ...(video.metadata.duration > 0
+                            ? { durationSec: video.metadata.duration }
                             : {}),
                     },
                 };

package/dist/utils/mediaDuration.d.ts

@@ -21,3 +21,22 @@
  * fabricated number would be worse than an obvious zero.
  */
 export declare function formatMediaDuration(seconds: number): string;
+/**
+ * Format the moment a video keyframe was sampled at — "3.00s",
+ * "1234.00s (20m 34s)".
+ *
+ * Deliberately not the clock form, for the reason in this module's header:
+ * the string is read by a language model, and "0:03" is ambiguous. That is
+ * not theoretical here — an early draft labelled frames "0:03" and the model
+ * reported the green frame as appearing at "3:00".
+ *
+ * Deliberately not `formatMediaDuration` alone either. That rounds to whole
+ * seconds, and an explicit `videoOptions.frames` spreads frames evenly rather
+ * than on the duration tier: sixteen frames across a 3.6s clip are 0.225s
+ * apart, and every one of them would render as "0s". The precise value is
+ * therefore always present, with the readable form added only past a minute,
+ * where "1234.00s" stops meaning anything to a reader.
+ *
+ * Non-finite and negative inputs clamp to zero rather than emitting "NaNs".
+ */
+export declare function formatKeyframeTimestamp(seconds: number): string;

package/dist/utils/mediaDuration.js

@@ -42,3 +42,26 @@ export function formatMediaDuration(seconds) {
     }
     return parts.join(" ");
 }
+/**
+ * Format the moment a video keyframe was sampled at — "3.00s",
+ * "1234.00s (20m 34s)".
+ *
+ * Deliberately not the clock form, for the reason in this module's header:
+ * the string is read by a language model, and "0:03" is ambiguous. That is
+ * not theoretical here — an early draft labelled frames "0:03" and the model
+ * reported the green frame as appearing at "3:00".
+ *
+ * Deliberately not `formatMediaDuration` alone either. That rounds to whole
+ * seconds, and an explicit `videoOptions.frames` spreads frames evenly rather
+ * than on the duration tier: sixteen frames across a 3.6s clip are 0.225s
+ * apart, and every one of them would render as "0s". The precise value is
+ * therefore always present, with the readable form added only past a minute,
+ * where "1234.00s" stops meaning anything to a reader.
+ *
+ * Non-finite and negative inputs clamp to zero rather than emitting "NaNs".
+ */
+export function formatKeyframeTimestamp(seconds) {
+    const safe = Number.isFinite(seconds) && seconds > 0 ? seconds : 0;
+    const precise = `${safe.toFixed(2)}s`;
+    return safe >= 60 ? `${precise} (${formatMediaDuration(safe)})` : precise;
+}

package/dist/utils/messageBuilder.js

@@ -832,20 +832,6 @@ export function mergeMediaFileAliases(input) {
     input.audioFiles = undefined;
     input.videoFiles = undefined;
 }
-/**
- * #478: `transcribeAudio` (CLI `--transcribe-audio`) is accepted by the options
- * surface but no video-audio transcription exists yet — VideoProcessor extracts
- * keyframes and embedded subtitle tracks only, and the transcription step is
- * still open as #433. Say so once per request rather than letting the caller
- * believe a transcript was produced and silently omitted.
- */
-function warnIfVideoTranscriptionRequested(videoOptions) {
-    if (videoOptions?.transcribeAudio) {
-        logger.warn("[NEUROLINK] Video audio transcription was requested but is not implemented yet " +
-            "(tracked as #433). Keyframes and any embedded subtitle tracks are still extracted; " +
-            "spoken audio will not be transcribed.");
-    }
-}
 /**
  * Process the unified files array with auto-detection.
  * Handles lazy file registration, full processing, and preview injection.
@@ -916,6 +902,10 @@ async function detectFileForUnifiedArray(file, options, provider, genericFileMax
                 frames: options.videoOptions.frames,
                 quality: options.videoOptions.quality,
                 format: options.videoOptions.format,
+                // #433: `--transcribe-audio` stopped here until now — the flag was
+                // accepted, warned about, and never handed to the processor that
+                // would have to act on it.
+                transcribeAudio: options.videoOptions.transcribeAudio,
             }
             : undefined,
         provider: provider,
@@ -952,7 +942,6 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
     }
     const totalFiles = pending.length;
     const files = pending;
-    warnIfVideoTranscriptionRequested(options.videoOptions);
     return withSpan({
         name: "neurolink.file.process_all",
         tracer: tracers.file,

package/dist/utils/multimodalOptionsBuilder.d.ts

@@ -15,6 +15,7 @@ import type { StreamOptions } from "../types/index.js";
  *   - input.audioFiles: Audio files (Buffer | string paths)
  *   - input.videoFiles: Video files (Buffer | string paths)
  *   - csvOptions: CSV parsing options
+ *   - videoOptions: Video frame/quality/format/transcription options
  *   - systemPrompt: System-level instructions
  *   - conversationMessages: Chat history
  *   - temperature: Model temperature (0-1)
@@ -64,6 +65,7 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
     imageOptions: {
         outputFormat?: import("../index.js").VisionImageOutputFormat;
     } | undefined;
+    videoOptions: import("../index.js").VideoProcessorOptions | undefined;
     systemPrompt: string | undefined;
     conversationHistory: import("../index.js").ChatMessage[] | undefined;
     provider: string;

package/dist/utils/multimodalOptionsBuilder.js

@@ -14,6 +14,7 @@
  *   - input.audioFiles: Audio files (Buffer | string paths)
  *   - input.videoFiles: Video files (Buffer | string paths)
  *   - csvOptions: CSV parsing options
+ *   - videoOptions: Video frame/quality/format/transcription options
  *   - systemPrompt: System-level instructions
  *   - conversationMessages: Chat history
  *   - temperature: Model temperature (0-1)
@@ -60,6 +61,12 @@ export function buildMultimodalOptions(options, providerName, modelName) {
         csvOptions: options.csvOptions,
         pdfOptions: options.pdfOptions,
         imageOptions: options.imageOptions,
+        // Same whitelist hazard as above, one level up: `videoFiles` reaching
+        // the processor is not enough on its own — without `videoOptions` too,
+        // the frame/quality/format/transcribeAudio knobs a caller set on
+        // `StreamOptions` never reach VideoProcessor on the Bedrock branch, and
+        // it silently falls back to the duration-tier default.
+        videoOptions: options.videoOptions,
         systemPrompt: options.systemPrompt,
         conversationHistory: options.conversationMessages,
         provider: providerName,

package/docs-site/static/search-index.json

@@ -3899,10 +3899,10 @@
 {"objectID":"2c4c2bb10b6fea62d177994e6df1345d05606b65f8122b3c969cf214df3363f1","title":"CLI usage","url":"/docs/features/multimodal#cli-usage","content":"`bash","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CLI usage","lvl3":""}},
 {"objectID":"915a9e9701a9e6b02890e744550ed66754549288fa044c90bd2dd873e6b48c2a","title":"Auto-detect","url":"/docs/features/multimodal#auto-detect","content":"neurolink generate \"Describe this video\" --file demo.mp4 --provider google-ai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Auto-detect","lvl3":""}},
 {"objectID":"12ce623ba884b34886674b24daff6ce9e58465c68dc25f4f66c01229c6855513","title":"Explicit flag","url":"/docs/features/multimodal#explicit-flag","content":"neurolink generate \"Summarise this\" --video standup.mp4 --provider vertex","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Explicit flag","lvl3":""}},
-{"objectID":"08774b3c1d8fd456cb7d64ad98281449b9271854310a9a713048ae2112572d5c","title":"Frame-extraction knobs","url":"/docs/features/multimodal#frame-extraction-knobs","content":"neurolink generate \"Describe each scene\" --file demo.mp4 \\\n  --provider openai --video-frames 16 --video-quality 90 --video-format jpeg\n--transcribe-audiovideoOptions.transcribeAudiotranscribeAudio`.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Frame-extraction knobs","lvl3":""}},
+{"objectID":"08774b3c1d8fd456cb7d64ad98281449b9271854310a9a713048ae2112572d5c","title":"Frame-extraction knobs","url":"/docs/features/multimodal#frame-extraction-knobs","content":"neurolink generate \"Describe each scene\" --file demo.mp4 \\\n  --provider openai --video-frames 16 --video-quality 90 --video-format jpeg\n--transcribe-audiovideoOptions.transcribeAudiowhisper-1OPENAIAPIKEYOPENAIBASEURLSpoken Audio (transcribed)OPENAIAPIKEYNo transcript for : --transcribe-audiotranscribeAudio`.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Frame-extraction knobs","lvl3":""}},
 {"objectID":"d0faa964a225271bc41d6c279e3e925829ac8e8828518387c39ba60d97cd363f","title":"Asking which mechanism applies","url":"/docs/features/multimodal#asking-which-mechanism-applies","content":"The capability table is exported, so a caller can ask before sending — useful\nfor choosing a provider, or for budgeting.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Asking which mechanism applies","lvl3":""}},
 {"objectID":"10a8f5bf0ea5129e242eb09d5db285a4d91aac091c171050471ebd7eb528cb72","title":"Best practices","url":"/docs/features/multimodal#best-practices","content":"Ask an audio question of a Gemini provider only. Nothing on the frame\n  path can hear the clip, so \"what was said\" has no answer there.\nKeep clips under 14 MB when you want native delivery. Re-encoding a\n  screen recording at a lower bitrate usually gets a long clip under the\n  ceiling without losing anything a model would read.\nRaise  rather than  for detail over time. Frames cost\n  roughly 250 tokens each regardless of quality; more of them is what buys\n  coverage of a clip where things change.\nLower  for local runtimes. Ollama and llama.cpp hold every\n  frame in the same memory as the model; four is the practical default.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Best practices","lvl3":""}},
-{"objectID":"7bd6ebb59bff22b19f26c8c358eb7012f8a73d4f90eed575ca136f632c6f9bc6","title":"Troubleshooting","url":"/docs/features/multimodal#troubleshooting","content":"| Symptom                                             | Cause                                                                   | Fix                                                                     |\n| --------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |\n| Model describes the file but not its content        | No keyframes extracted — ffmpeg missing — and the provider takes frames | Install ffmpeg, or use a Gemini provider, which needs none              |\n|  in the logs | The clip failed the inline gate; the reason is on the same line         | Shorten or re-encode it, or accept the frames                           |\n| Model cannot hear speech on a Gemini provider       | The clip exceeded the inline ceiling and fell back to frames            | Get the source under 14 MB                                              |\n|  produced no transcript         | Not implemented yet (#433); the flag is a no-op that only warns         | None yet — track #1757 |\n| Only the first seconds of a long clip are described | The frame budget was hit before the end                                 | Raise ; the interval then spreads evenly across the whole clip  |","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Troubleshooting","lvl3":""}},
+{"objectID":"7bd6ebb59bff22b19f26c8c358eb7012f8a73d4f90eed575ca136f632c6f9bc6","title":"Troubleshooting","url":"/docs/features/multimodal#troubleshooting","content":"| Symptom                                             | Cause                                                                                                                         | Fix                                                                    |\n| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |\n| Model describes the file but not its content        | No keyframes extracted — ffmpeg missing — and the provider takes frames                                                       | Install ffmpeg, or use a Gemini provider, which needs none             |\n|  in the logs | The clip failed the inline gate; the reason is on the same line                                                               | Shorten or re-encode it, or accept the frames                          |\n| Model cannot hear speech on a Gemini provider       | The clip exceeded the inline ceiling and fell back to frames                                                                  | Get the source under 14 MB, or pass                |\n|  produced no transcript         | The  warning names the reason: no audio track,  unset, or audio over Whisper's 25 MB limit | Set , or split the recording                           |\n| Only the first seconds of a long clip are described | The frame budget was hit before the end                                                                                       | Raise ; the interval then spreads evenly across the whole clip |","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Troubleshooting","lvl3":""}},
 {"objectID":"79318378a7df7cc816465eb447fa4bd2ba9e6344f17b0a9cf6661377bb03e7e2","title":"Combining Multiple Input Types","url":"/docs/features/multimodal#combining-multiple-input-types","content":"NeuroLink excels at combining different media types in a single request.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Combining Multiple Input Types","lvl3":""}},
 {"objectID":"412f09080a5801a35250df393e2e0694602d5312e2b3d1b017fe469bf9dc0474","title":"Batch with Multimodal (CLI)","url":"/docs/features/multimodal#batch-with-multimodal-cli","content":"The  command supports , , , and . The file(s) are attached identically to every prompt in the batch (a one-line notice is printed to stderr, unconditionally — it is not suppressed by , so it never corrupts  output written to stdout):\n\n(auto-detect) is not available in , because it collides with the  positional (the prompts-list path). Use the explicit  /  /  /  flags instead.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Batch with Multimodal (CLI)","lvl3":""}},
 {"objectID":"66d907236327d676dbe6e05193c2793f7c5990dce2ad113cf078eb1ff4c73216","title":"File validation & troubleshooting (CLI)","url":"/docs/features/multimodal#file-validation-troubleshooting-cli","content":"Before any provider call, the CLI validates local file inputs across , , and :\nA path that points at a directory, doesn't exist, or can't be read (e.g. a permissions error) is rejected up front with a clear error and troubleshooting hints — no cryptic / deep in processing. This also applies to the   prompts-list positional itself.\nA large file (images > 10 MB, CSV/PDF > 50 MB) prints a non-blocking warning to stderr. Like the batch attachment notice, this is unconditional — visible without  and regardless of  — and never mixes into stdout, so  output stays valid JSON even when large-file warnings fire.\n\nThis runs even under  and without API keys configured.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"File validation & troubleshooting (CLI)","lvl3":""}},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.29.2",
+  "version": "12.30.1",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -248,8 +248,9 @@
     "pre-push": "pnpm run check:deps && pnpm run build && pnpm run test:provider-structure && pnpm run test:model-manifests",
     "check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit",
     "test:file-formats": "pnpm exec tsx test/continuous-test-suite-file-formats.ts",
-    "test:multimodal": "pnpm run test:file-formats && pnpm run test:multimodal:sdk && pnpm run test:video-native",
+    "test:multimodal": "pnpm run test:file-formats && pnpm run test:multimodal:sdk && pnpm run test:video-native && pnpm run test:video-frames",
     "test:multimodal:sdk": "pnpm exec tsx test/continuous-test-suite-multimodal-sdk.ts",
+    "test:video-frames": "pnpm exec tsx test/continuous-test-suite-video-frames.ts",
     "test:video-native": "pnpm exec tsx test/continuous-test-suite-video-native.ts",
     "test:mcp-result-cache": "pnpm exec tsx test/continuous-test-suite-mcp-result-cache.ts",
     "test:mcp-breaker-resolved-errors": "pnpm exec tsx test/continuous-test-suite-mcp-breaker-resolved-errors.ts",