@juspay/neurolink 12.26.3 → 12.27.1

package/docs-site/static/search-index.json

@@ -3874,6 +3874,8 @@
 {"objectID":"73d7935973c0a686585af87403eac8b065162f05b5a7c956894d5cc17807712d","title":"PDF Documents","url":"/docs/features/multimodal#pdf-documents","content":"| Provider              | Supported | Max Size | Max Pages | Processing Mode  | Notes                                   |\n| --------------------- | --------- | -------- | --------- | ---------------- | --------------------------------------- |\n| Google Vertex AI  | ✅        | 5 MB     | 100       | Native PDF       | Best for document analysis              |\n| Anthropic         | ✅        | 5 MB     | 100       | Native PDF       | Claude excels at document understanding |\n| AWS Bedrock       | ✅        | 5 MB     | 100       | Native PDF       | Via Claude models                       |\n| Google AI Studio  | ✅        | 2000 MB  | 100       | Native PDF       | Handles very large files                |\n| OpenAI            | ✅        | 10 MB    | 100       | Files API        | , ,            |\n| Azure OpenAI      | ✅        | 10 MB    | 100       | Files API        | Uses OpenAI Files API                   |\n| LiteLLM           | ✅        | 10 MB    | 100       | Proxy            | Depends on upstream model               |\n| OpenAI Compatible | ✅        | 10 MB    | 100       | Varies           | Server-dependent                        |\n| Mistral           | ✅        | 10 MB    | 100       | Native PDF       | Native support                          |\n| Hugging Face      | ✅        | 10 MB    | 100       | Model-dependent  | Varies by model                         |\n| Ollama            | ❌        | -        | -         | -                | Not supported                           |\n| OpenRouter        | ⚠️        | Varies   | Varies    | Depends on model | Route-dependent                         |\n| AWS SageMaker     | ❌        | -        | -         | -                | Not supported                           |","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"PDF Documents","lvl3":""}},
 {"objectID":"96bf471d5383c56e1f0144483b12e8a6f6c221ab852ed86b42f9ea057e68a91d","title":"CSV/Spreadsheet Data","url":"/docs/features/multimodal#csvspreadsheet-data","content":"| Provider          | Supported | Max Rows | Format Options      | Notes                                 |\n| ----------------- | --------- | -------- | ------------------- | ------------------------------------- |\n| All Providers | ✅        | 10,000   | raw, json, markdown | Universal support - processed as text |\n\nCSV support works with all providers because files are converted to text before sending to the AI model. The file is parsed and formatted (raw CSV, JSON, or Markdown table) before inclusion in the prompt.\n\nFormat Recommendations:\nRaw format - Best for large files (minimal token usage)\nJSON format - Best for structured data processing\nMarkdown format - Best for small datasets (\\<100 rows), readable tables","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CSV/Spreadsheet Data","lvl3":""}},
 {"objectID":"b52c712e324d5d40df527e8a5877d9ea0ae63d2ae80bd022cfbaffc9c3c91a82","title":"Audio Input","url":"/docs/features/multimodal#audio-input","content":"| Provider             | Native Audio | Transcription | Real-time | Max Duration | Notes                               |\n| -------------------- | ------------ | ------------- | --------- | ------------ | ----------------------------------- |\n| Google AI Studio | ✅           | ✅            | ✅        | 1 hour       | Best for real-time voice            |\n| Google Vertex AI | ✅           | ✅            | ✅        | 1 hour       | Native Gemini audio support         |\n| OpenAI           | ❌           | ✅ Whisper    | ❌        | 25 MB        | Excellent transcription accuracy    |\n| Azure OpenAI     | ❌           | ✅ Whisper    | ❌        | 25 MB        | Via Whisper integration             |\n| Anthropic        | ❌           | Via fallback  | ❌        | -            | Uses transcription approach         |\n| AWS Bedrock      | ❌           | Via fallback  | ❌        | -            | Uses transcription approach         |\n| Others           | ❌           | Via fallback  | ❌        | -            | Audio transcribed before processing |\n\nFor comprehensive audio documentation, see the Audio Input Guide.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Audio Input","lvl3":""}},
+{"objectID":"6c75d56315cf5bcec368645e07a84c0b2b067c8052da1028e9deb65d16aab9ee","title":"Video","url":"/docs/features/multimodal#video","content":"Two mechanisms, and which one a provider gets changes what it can answer.\n\nNative (inline) — the video file itself travels in the request. The model\nsees continuous motion and hears the audio track. Only Google's Gemini front\nends accept this today.\n\nFrame extraction — ffmpeg pulls keyframes at intervals and they are\nattached as ordinary images, alongside a metadata summary in the prompt text.\nThe model sees a handful of stills and hears nothing.\n\n| Provider                             | Mechanism        | Inline ceiling | Audio track | Default frame budget |\n| ------------------------------------ | ---------------- | -------------- | ----------- | -------------------- |\n| Google AI Studio                 | Native (inline)  | 14 MB          | ✅ heard    | 16 (fallback only)   |\n| Google Vertex AI (Gemini models) | Native (inline)  | 14 MB          | ✅ heard    | 16 (fallback only)   |\n| OpenAI / Azure OpenAI            | Frame extraction | –              | ❌          | 8                    |\n| Anthropic / AWS Bedrock          | Frame extraction | –              | ❌          | 8                    |\n| Mistral                          | Frame extraction | –              | ❌          | 8                    |\n| LiteLLM / OpenRouter             | Frame extraction | –              | ❌          | 8                    |\n| Ollama / llama.cpp               | Frame extraction | –              | ❌          | 4                    |\n\nThe 14 MB ceiling is one shared source-byte budget for the whole request,\nnot a separate per-clip limit. Before a clip ships natively, its bytes are\nadded to the source bytes of every inline part already in the request, and\nthe clip is delivered only if the total stays within 14 MB. Those parts\ninclude images (keyframes among them), PDFs, audio and any clip accepted\nearlier. With nothing else attached, that reduces to the clip's own size.\nThe budget is in source bytes because inline data travels as base64 in a\nJSON body, which costs four bytes per th","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Video","lvl3":""}},
+{"objectID":"623b369433add73c32c5d31532a82b33ea81350ec9e96ead3a7dc6262db5c09f","title":"Supported containers","url":"/docs/features/multimodal#supported-containers","content":", , , , , , , ,  and\nthe other formats in the detector's video registry are all accepted for frame\nextraction. Inline delivery is narrower — it is limited to the containers\nGemini reads directly (mp4, mpeg, mov, avi, flv, webm, wmv, 3gpp) — and a\ncontainer outside that set falls back to keyframes rather than being\nre-encoded. Transcoding a long recording mid-request costs minutes of CPU for\na payload that would usually breach the inline ceiling anyway.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Supported containers","lvl3":""}},
 {"objectID":"0a6aa016f95cc342e4dd2a5dd050bd8ded76685337214426811889d4fdd671b7","title":"Quick Start","url":"/docs/features/multimodal#quick-start","content":"CLI:\n\n`bash","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Quick Start","lvl3":""}},
 {"objectID":"de245f012caaeb0388d4424c7d969aed288374590e5c1f58a0016a1ae1996e17","title":"Single image","url":"/docs/features/multimodal#single-image","content":"npx @juspay/neurolink generate \"Describe this interface\" \\\n  --image ./designs/dashboard.png --provider google-ai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Single image","lvl3":""}},
 {"objectID":"05b1002029479df0ee1386f3f8434822765351def1e4c18cf8c6349b373418fa","title":"Remote URL","url":"/docs/features/multimodal#remote-url","content":"npx @juspay/neurolink generate \"Analyze this diagram\" \\\n  --image https://example.com/architecture.png --provider openai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Remote URL","lvl3":""}},
@@ -3892,6 +3894,14 @@
 {"objectID":"0ffdeaf2a4d433739fb1f96c6487b2e35e254c4459ead6ecbfd3dcb3a98de8b3","title":"Explicit CSV with options","url":"/docs/features/multimodal#explicit-csv-with-options","content":"npx @juspay/neurolink generate \"Summarize data\" \\\n  --csv ./data.csv \\\n  --csv-max-rows 500 \\\n  --csv-format raw\ntypescript\n// Auto-detect (recommended)\nawait neurolink.generate({\n  input: {\n    text: \"Analyze this sales data\",\n    files: [\"./sales.csv\"], // Auto-detected as CSV\n  },\n});\n\n// Explicit CSV with options\nawait neurolink.generate({\n  input: {\n    text: \"Compare quarterly data\",\n    csvFiles: [\"./q1.csv\", \"./q2.csv\"],\n  },\n  csvOptions: {\n    maxRows: 1000,\n    formatStyle: \"json\", // or \"raw\", \"markdown\"\n  },\n});\n`","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Explicit CSV with options","lvl3":""}},
 {"objectID":"93535721732536c013e0204b266f2a131be12a7745f6c88a6e61cdeb76704728","title":"CSV Format Options","url":"/docs/features/multimodal#csv-format-options","content":"Three format styles:\nRaw format (default)\nBest for large files\nMinimal token usage\nPreserves original CSV structure\nJSON format\nStructured data processing\nEasier for AI to parse\nHigher token usage\nMarkdown format\nReadable tables\nGood for small datasets (\\<100 rows)\nModerate token usage","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CSV Format Options","lvl3":""}},
 {"objectID":"427b303d134e7e002fe1fbb0ae86243f33506e795e3029cdfb7994c2014c0db4","title":"CSV Best Practices","url":"/docs/features/multimodal#csv-best-practices","content":"Use raw format for large files to minimize token usage\nUse JSON format for structured processing when AI needs to manipulate data\nLimit to 1000 rows by default (configurable up to 10,000)\nCombine CSV with visualization images for comprehensive analysis\nWorks with ALL providers (not just vision-capable models)","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CSV Best Practices","lvl3":""}},
+{"objectID":"5f2f649cdd7639b3fc3fb30871df07bf3331ad446ac4d23340aaab8976ccf1a7","title":"Frame-extraction options","url":"/docs/features/multimodal#frame-extraction-options","content":"These control the keyframes and are ignored by a provider on the native path.\n\nWithout an explicit , the interval is chosen from the clip's duration\n— roughly every second for clips under 10s, widening to every three minutes\nfor recordings over half an hour, always capped at 100 frames.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Frame-extraction options","lvl3":""}},
+{"objectID":"2c4c2bb10b6fea62d177994e6df1345d05606b65f8122b3c969cf214df3363f1","title":"CLI usage","url":"/docs/features/multimodal#cli-usage","content":"`bash","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CLI usage","lvl3":""}},
+{"objectID":"915a9e9701a9e6b02890e744550ed66754549288fa044c90bd2dd873e6b48c2a","title":"Auto-detect","url":"/docs/features/multimodal#auto-detect","content":"neurolink generate \"Describe this video\" --file demo.mp4 --provider google-ai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Auto-detect","lvl3":""}},
+{"objectID":"12ce623ba884b34886674b24daff6ce9e58465c68dc25f4f66c01229c6855513","title":"Explicit flag","url":"/docs/features/multimodal#explicit-flag","content":"neurolink generate \"Summarise this\" --video standup.mp4 --provider vertex","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Explicit flag","lvl3":""}},
+{"objectID":"08774b3c1d8fd456cb7d64ad98281449b9271854310a9a713048ae2112572d5c","title":"Frame-extraction knobs","url":"/docs/features/multimodal#frame-extraction-knobs","content":"neurolink generate \"Describe each scene\" --file demo.mp4 \\\n  --provider openai --video-frames 16 --video-quality 90 --video-format jpeg\n--transcribe-audiovideoOptions.transcribeAudiotranscribeAudio`.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Frame-extraction knobs","lvl3":""}},
+{"objectID":"d0faa964a225271bc41d6c279e3e925829ac8e8828518387c39ba60d97cd363f","title":"Asking which mechanism applies","url":"/docs/features/multimodal#asking-which-mechanism-applies","content":"The capability table is exported, so a caller can ask before sending — useful\nfor choosing a provider, or for budgeting.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Asking which mechanism applies","lvl3":""}},
+{"objectID":"10a8f5bf0ea5129e242eb09d5db285a4d91aac091c171050471ebd7eb528cb72","title":"Best practices","url":"/docs/features/multimodal#best-practices","content":"Ask an audio question of a Gemini provider only. Nothing on the frame\n  path can hear the clip, so \"what was said\" has no answer there.\nKeep clips under 14 MB when you want native delivery. Re-encoding a\n  screen recording at a lower bitrate usually gets a long clip under the\n  ceiling without losing anything a model would read.\nRaise  rather than  for detail over time. Frames cost\n  roughly 250 tokens each regardless of quality; more of them is what buys\n  coverage of a clip where things change.\nLower  for local runtimes. Ollama and llama.cpp hold every\n  frame in the same memory as the model; four is the practical default.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Best practices","lvl3":""}},
+{"objectID":"7bd6ebb59bff22b19f26c8c358eb7012f8a73d4f90eed575ca136f632c6f9bc6","title":"Troubleshooting","url":"/docs/features/multimodal#troubleshooting","content":"| Symptom                                             | Cause                                                                   | Fix                                                                     |\n| --------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |\n| Model describes the file but not its content        | No keyframes extracted — ffmpeg missing — and the provider takes frames | Install ffmpeg, or use a Gemini provider, which needs none              |\n|  in the logs | The clip failed the inline gate; the reason is on the same line         | Shorten or re-encode it, or accept the frames                           |\n| Model cannot hear speech on a Gemini provider       | The clip exceeded the inline ceiling and fell back to frames            | Get the source under 14 MB                                              |\n|  produced no transcript         | Not implemented yet (#433); the flag is a no-op that only warns         | None yet — track #1757 |\n| Only the first seconds of a long clip are described | The frame budget was hit before the end                                 | Raise ; the interval then spreads evenly across the whole clip  |","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Troubleshooting","lvl3":""}},
 {"objectID":"79318378a7df7cc816465eb447fa4bd2ba9e6344f17b0a9cf6661377bb03e7e2","title":"Combining Multiple Input Types","url":"/docs/features/multimodal#combining-multiple-input-types","content":"NeuroLink excels at combining different media types in a single request.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Combining Multiple Input Types","lvl3":""}},
 {"objectID":"412f09080a5801a35250df393e2e0694602d5312e2b3d1b017fe469bf9dc0474","title":"Batch with Multimodal (CLI)","url":"/docs/features/multimodal#batch-with-multimodal-cli","content":"The  command supports , , , and . The file(s) are attached identically to every prompt in the batch (a one-line notice is printed to stderr, unconditionally — it is not suppressed by , so it never corrupts  output written to stdout):\n\n(auto-detect) is not available in , because it collides with the  positional (the prompts-list path). Use the explicit  /  /  /  flags instead.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Batch with Multimodal (CLI)","lvl3":""}},
 {"objectID":"66d907236327d676dbe6e05193c2793f7c5990dce2ad113cf078eb1ff4c73216","title":"File validation & troubleshooting (CLI)","url":"/docs/features/multimodal#file-validation-troubleshooting-cli","content":"Before any provider call, the CLI validates local file inputs across , , and :\nA path that points at a directory, doesn't exist, or can't be read (e.g. a permissions error) is rejected up front with a clear error and troubleshooting hints — no cryptic / deep in processing. This also applies to the   prompts-list positional itself.\nA large file (images > 10 MB, CSV/PDF > 50 MB) prints a non-blocking warning to stderr. Like the batch attachment notice, this is unconditional — visible without  and regardless of  — and never mixes into stdout, so  output stays valid JSON even when large-file warnings fire.\n\nThis runs even under  and without API keys configured.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"File validation & troubleshooting (CLI)","lvl3":""}},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.26.3",
+  "version": "12.27.1",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -240,8 +240,9 @@
     "pre-push": "pnpm run check:deps && pnpm run build && pnpm run test:provider-structure && pnpm run test:model-manifests",
     "check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit",
     "test:file-formats": "pnpm exec tsx test/continuous-test-suite-file-formats.ts",
-    "test:multimodal": "pnpm run test:file-formats && pnpm run test:multimodal:sdk",
+    "test:multimodal": "pnpm run test:file-formats && pnpm run test:multimodal:sdk && pnpm run test:video-native",
     "test:multimodal:sdk": "pnpm exec tsx test/continuous-test-suite-multimodal-sdk.ts",
+    "test:video-native": "pnpm exec tsx test/continuous-test-suite-video-native.ts",
     "test:mcp-result-cache": "pnpm exec tsx test/continuous-test-suite-mcp-result-cache.ts",
     "test:mcp-breaker-resolved-errors": "pnpm exec tsx test/continuous-test-suite-mcp-breaker-resolved-errors.ts",
     "test:archive:security": "pnpm exec tsx test/continuous-test-suite-archive-security.ts",