@stabgan/openrouter-mcp-multimodal 2.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +198 -145
  2. package/dist/errors.d.ts +42 -0
  3. package/dist/errors.js +46 -0
  4. package/dist/index.js +1 -1
  5. package/dist/logger.d.ts +22 -0
  6. package/dist/logger.js +47 -0
  7. package/dist/model-cache.d.ts +10 -0
  8. package/dist/model-cache.js +31 -1
  9. package/dist/openrouter-api.d.ts +54 -0
  10. package/dist/openrouter-api.js +136 -12
  11. package/dist/tool-handlers/analyze-audio.d.ts +5 -9
  12. package/dist/tool-handlers/analyze-audio.js +41 -8
  13. package/dist/tool-handlers/analyze-image.d.ts +5 -9
  14. package/dist/tool-handlers/analyze-image.js +38 -8
  15. package/dist/tool-handlers/analyze-video.d.ts +19 -0
  16. package/dist/tool-handlers/analyze-video.js +93 -0
  17. package/dist/tool-handlers/audio-utils.js +7 -9
  18. package/dist/tool-handlers/chat-completion.d.ts +6 -10
  19. package/dist/tool-handlers/chat-completion.js +27 -7
  20. package/dist/tool-handlers/completion-utils.d.ts +27 -0
  21. package/dist/tool-handlers/completion-utils.js +69 -0
  22. package/dist/tool-handlers/fetch-utils.d.ts +32 -0
  23. package/dist/tool-handlers/fetch-utils.js +216 -12
  24. package/dist/tool-handlers/generate-audio.d.ts +32 -12
  25. package/dist/tool-handlers/generate-audio.js +77 -46
  26. package/dist/tool-handlers/generate-image.d.ts +48 -10
  27. package/dist/tool-handlers/generate-image.js +148 -33
  28. package/dist/tool-handlers/generate-video.d.ts +78 -0
  29. package/dist/tool-handlers/generate-video.js +353 -0
  30. package/dist/tool-handlers/get-model-info.d.ts +2 -9
  31. package/dist/tool-handlers/get-model-info.js +15 -5
  32. package/dist/tool-handlers/image-utils.d.ts +17 -1
  33. package/dist/tool-handlers/image-utils.js +66 -13
  34. package/dist/tool-handlers/openrouter-errors.d.ts +18 -0
  35. package/dist/tool-handlers/openrouter-errors.js +99 -0
  36. package/dist/tool-handlers/path-safety.d.ts +11 -0
  37. package/dist/tool-handlers/path-safety.js +88 -0
  38. package/dist/tool-handlers/search-models.d.ts +2 -9
  39. package/dist/tool-handlers/search-models.js +10 -6
  40. package/dist/tool-handlers/validate-model.d.ts +2 -9
  41. package/dist/tool-handlers/validate-model.js +15 -4
  42. package/dist/tool-handlers/video-utils.d.ts +29 -0
  43. package/dist/tool-handlers/video-utils.js +174 -0
  44. package/dist/tool-handlers.js +229 -21
  45. package/package.json +3 -3
  46. package/dist/__tests__/audio-utils.test.d.ts +0 -1
  47. package/dist/__tests__/audio-utils.test.js +0 -120
  48. package/dist/__tests__/fetch-utils.test.d.ts +0 -1
  49. package/dist/__tests__/fetch-utils.test.js +0 -76
  50. package/dist/__tests__/generate-audio.test.d.ts +0 -1
  51. package/dist/__tests__/generate-audio.test.js +0 -90
  52. package/dist/__tests__/image-utils.test.d.ts +0 -1
  53. package/dist/__tests__/image-utils.test.js +0 -75
  54. package/dist/__tests__/integration.test.d.ts +0 -1
  55. package/dist/__tests__/integration.test.js +0 -219
  56. package/dist/__tests__/model-cache.test.d.ts +0 -1
  57. package/dist/__tests__/model-cache.test.js +0 -96
@@ -3,6 +3,7 @@ export interface OpenRouterModelRecord {
3
3
  name?: string;
4
4
  architecture?: {
5
5
  input_modalities?: string[];
6
+ output_modalities?: string[];
6
7
  };
7
8
  context_length?: number;
8
9
  [key: string]: unknown;
@@ -11,9 +12,16 @@ export declare class ModelCache {
11
12
  private static instance;
12
13
  private models;
13
14
  private fetchedAt;
15
+ private inflight;
14
16
  static getInstance(): ModelCache;
15
17
  isValid(): boolean;
16
18
  setModels(models: OpenRouterModelRecord[]): void;
19
+ /**
20
+ * Populate the cache using `fetcher` if stale, coalescing concurrent callers
21
+ * so only one request hits the upstream API per stale window. Callers that
22
+ * arrive while a populate is in flight await the same promise.
23
+ */
24
+ ensureFresh(fetcher: () => Promise<OpenRouterModelRecord[]>): Promise<void>;
17
25
  getAll(): OpenRouterModelRecord[];
18
26
  get(id: string): OpenRouterModelRecord | null;
19
27
  has(id: string): boolean;
@@ -22,6 +30,8 @@ export declare class ModelCache {
22
30
  provider?: string;
23
31
  capabilities?: {
24
32
  vision?: boolean;
33
+ audio?: boolean;
34
+ video?: boolean;
25
35
  };
26
36
  limit?: number;
27
37
  }): OpenRouterModelRecord[];
@@ -5,10 +5,12 @@ function getCacheTtlMs() {
5
5
  const n = parseInt(raw, 10);
6
6
  return Number.isFinite(n) && n > 0 ? n : 3600000;
7
7
  }
8
+ const MAX_SEARCH_LIMIT = 50;
8
9
  export class ModelCache {
9
10
  static instance;
10
11
  models = {};
11
12
  fetchedAt = 0;
13
+ inflight = null;
12
14
  static getInstance() {
13
15
  return (ModelCache.instance ??= new ModelCache());
14
16
  }
@@ -19,6 +21,27 @@ export class ModelCache {
19
21
  this.models = Object.fromEntries(models.map((m) => [m.id, m]));
20
22
  this.fetchedAt = Date.now();
21
23
  }
24
+ /**
25
+ * Populate the cache using `fetcher` if stale, coalescing concurrent callers
26
+ * so only one request hits the upstream API per stale window. Callers that
27
+ * arrive while a populate is in flight await the same promise.
28
+ */
29
+ async ensureFresh(fetcher) {
30
+ if (this.isValid())
31
+ return;
32
+ if (this.inflight) {
33
+ await this.inflight;
34
+ return;
35
+ }
36
+ this.inflight = (async () => fetcher())();
37
+ try {
38
+ const models = await this.inflight;
39
+ this.setModels(models);
40
+ }
41
+ finally {
42
+ this.inflight = null;
43
+ }
44
+ }
22
45
  getAll() {
23
46
  return Object.values(this.models);
24
47
  }
@@ -41,6 +64,13 @@ export class ModelCache {
41
64
  if (params.capabilities?.vision) {
42
65
  results = results.filter((m) => m.architecture?.input_modalities?.includes('image'));
43
66
  }
44
- return results.slice(0, params.limit ?? 10);
67
+ if (params.capabilities?.audio) {
68
+ results = results.filter((m) => m.architecture?.input_modalities?.includes('audio'));
69
+ }
70
+ if (params.capabilities?.video) {
71
+ results = results.filter((m) => m.architecture?.input_modalities?.includes('video'));
72
+ }
73
+ const limit = Math.min(Math.max(1, params.limit ?? 10), MAX_SEARCH_LIMIT);
74
+ return results.slice(0, limit);
45
75
  }
46
76
  }
@@ -1,6 +1,60 @@
1
1
  import type { OpenRouterModelRecord } from './model-cache.js';
2
+ declare function parseRetryAfter(headerValue: string | null): number | null;
3
+ declare function backoffWithJitter(attempt: number, retryAfterMs: number | null): number;
4
+ /**
5
+ * fetch() wrapper with retries on 429 / 5xx / network error.
6
+ *
7
+ * A fresh `AbortSignal.timeout(timeoutMs)` is created per attempt so retries
8
+ * each get a full timeout budget. Backoff honors `Retry-After` (seconds or
9
+ * HTTP-date) and applies jitter to avoid thundering-herd synchronization.
10
+ */
11
+ declare function fetchWithRetry(url: string, init: Omit<RequestInit, 'signal'>, { retries, timeoutMs }?: {
12
+ retries?: number;
13
+ timeoutMs?: number;
14
+ }): Promise<Response>;
2
15
  export declare class OpenRouterAPIClient {
3
16
  private apiKey;
4
17
  constructor(apiKey: string);
18
+ private authHeaders;
5
19
  getModels(): Promise<OpenRouterModelRecord[]>;
20
+ /** Submit a video-generation job. Returns the `{ id, polling_url, status }` envelope. */
21
+ submitVideoJob(body: Record<string, unknown>): Promise<VideoJobEnvelope>;
22
+ /** Poll a submitted video-generation job by id. */
23
+ pollVideoJob(id: string): Promise<VideoJobStatus>;
24
+ /**
25
+ * Download the generated video binary. Returns `{ buffer, contentType }`.
26
+ * This intentionally does NOT go through our SSRF-guarded `fetchHttpResource`
27
+ * because the URL is always OpenRouter itself (trusted origin) — and it can
28
+ * return arbitrarily large bodies that the caller bounds via
29
+ * `OPENROUTER_VIDEO_MAX_DOWNLOAD_BYTES`.
30
+ */
31
+ downloadVideoContent(id: string, index?: number, maxBytes?: number): Promise<{
32
+ buffer: Buffer;
33
+ contentType: string | null;
34
+ }>;
6
35
  }
36
+ export interface VideoJobEnvelope {
37
+ id: string;
38
+ status?: VideoJobStatusName;
39
+ polling_url?: string;
40
+ [key: string]: unknown;
41
+ }
42
+ export type VideoJobStatusName = 'pending' | 'queued' | 'processing' | 'completed' | 'failed';
43
+ export interface VideoJobStatus {
44
+ id: string;
45
+ status: VideoJobStatusName | string;
46
+ unsigned_urls?: string[];
47
+ error?: {
48
+ message?: string;
49
+ code?: string;
50
+ } | string;
51
+ usage?: Record<string, unknown>;
52
+ progress?: number;
53
+ [key: string]: unknown;
54
+ }
55
+ export declare const _internals: {
56
+ parseRetryAfter: typeof parseRetryAfter;
57
+ backoffWithJitter: typeof backoffWithJitter;
58
+ fetchWithRetry: typeof fetchWithRetry;
59
+ };
60
+ export {};
@@ -1,25 +1,63 @@
1
1
  const BASE_URL = 'https://openrouter.ai/api/v1';
2
+ const DEFAULT_TIMEOUT_MS = 30_000;
3
+ const VIDEO_TIMEOUT_MS = 60_000;
4
+ const MAX_BACKOFF_MS = 10_000;
2
5
  async function sleep(ms) {
3
6
  return new Promise((r) => setTimeout(r, ms));
4
7
  }
5
- async function fetchWithRetry(url, init, retries = 2) {
8
+ function parseRetryAfter(headerValue) {
9
+ if (!headerValue)
10
+ return null;
11
+ const asInt = parseInt(headerValue, 10);
12
+ if (Number.isFinite(asInt) && asInt >= 0)
13
+ return asInt * 1000;
14
+ const asDate = Date.parse(headerValue);
15
+ if (Number.isFinite(asDate)) {
16
+ const delta = asDate - Date.now();
17
+ return delta > 0 ? delta : 0;
18
+ }
19
+ return null;
20
+ }
21
+ function backoffWithJitter(attempt, retryAfterMs) {
22
+ const base = 400 * (attempt + 1);
23
+ const target = Math.min(Math.max(base, retryAfterMs ?? 0), MAX_BACKOFF_MS);
24
+ const jitter = 0.5 + Math.random(); // 0.5x .. 1.5x
25
+ return Math.round(target * jitter);
26
+ }
27
+ /**
28
+ * fetch() wrapper with retries on 429 / 5xx / network error.
29
+ *
30
+ * A fresh `AbortSignal.timeout(timeoutMs)` is created per attempt so retries
31
+ * each get a full timeout budget. Backoff honors `Retry-After` (seconds or
32
+ * HTTP-date) and applies jitter to avoid thundering-herd synchronization.
33
+ */
34
+ async function fetchWithRetry(url, init, { retries = 2, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
6
35
  let lastErr;
7
36
  for (let attempt = 0; attempt <= retries; attempt++) {
8
37
  try {
9
- const res = await fetch(url, init);
38
+ const res = await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
10
39
  if (res.status === 429 || res.status >= 500) {
11
- if (attempt < retries)
12
- await sleep(400 * (attempt + 1));
13
- else
14
- return res;
15
- continue;
40
+ if (attempt < retries) {
41
+ const retryAfter = parseRetryAfter(res.headers.get('retry-after'));
42
+ // Release the connection before retrying so undici/pool doesn't
43
+ // keep it open while we sleep.
44
+ try {
45
+ await res.body?.cancel();
46
+ }
47
+ catch {
48
+ /* ignore */
49
+ }
50
+ await sleep(backoffWithJitter(attempt, retryAfter));
51
+ continue;
52
+ }
53
+ return res;
16
54
  }
17
55
  return res;
18
56
  }
19
57
  catch (e) {
20
58
  lastErr = e;
21
59
  if (attempt < retries)
22
- await sleep(400 * (attempt + 1));
60
+ await sleep(backoffWithJitter(attempt, null));
23
61
  }
24
62
  }
25
63
  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
@@ -29,14 +67,100 @@ export class OpenRouterAPIClient {
29
67
  constructor(apiKey) {
30
68
  this.apiKey = apiKey;
31
69
  }
70
+ authHeaders(extra) {
71
+ return {
72
+ Authorization: `Bearer ${this.apiKey}`,
73
+ 'HTTP-Referer': 'https://github.com/stabgan/openrouter-mcp-multimodal',
74
+ 'X-Title': 'openrouter-mcp-multimodal',
75
+ ...extra,
76
+ };
77
+ }
32
78
  async getModels() {
33
- const res = await fetchWithRetry(`${BASE_URL}/models`, {
34
- headers: { Authorization: `Bearer ${this.apiKey}` },
35
- signal: AbortSignal.timeout(30000),
36
- }, 2);
79
+ const res = await fetchWithRetry(`${BASE_URL}/models`, { headers: this.authHeaders() }, { retries: 2, timeoutMs: DEFAULT_TIMEOUT_MS });
37
80
  if (!res.ok)
38
81
  throw new Error(`Failed to fetch models: HTTP ${res.status}`);
39
82
  const data = (await res.json());
40
83
  return data.data ?? [];
41
84
  }
85
+ /** Submit a video-generation job. Returns the `{ id, polling_url, status }` envelope. */
86
+ async submitVideoJob(body) {
87
+ const res = await fetchWithRetry(`${BASE_URL}/videos`, {
88
+ method: 'POST',
89
+ headers: this.authHeaders({ 'Content-Type': 'application/json' }),
90
+ body: JSON.stringify(body),
91
+ }, { retries: 2, timeoutMs: VIDEO_TIMEOUT_MS });
92
+ if (!res.ok) {
93
+ const detail = await safeReadText(res);
94
+ throw new Error(`POST /videos failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
95
+ }
96
+ return (await res.json());
97
+ }
98
+ /** Poll a submitted video-generation job by id. */
99
+ async pollVideoJob(id) {
100
+ const res = await fetchWithRetry(`${BASE_URL}/videos/${encodeURIComponent(id)}`, { headers: this.authHeaders() }, { retries: 2, timeoutMs: DEFAULT_TIMEOUT_MS });
101
+ if (!res.ok) {
102
+ const detail = await safeReadText(res);
103
+ throw new Error(`GET /videos/${id} failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
104
+ }
105
+ return (await res.json());
106
+ }
107
+ /**
108
+ * Download the generated video binary. Returns `{ buffer, contentType }`.
109
+ * This intentionally does NOT go through our SSRF-guarded `fetchHttpResource`
110
+ * because the URL is always OpenRouter itself (trusted origin) — and it can
111
+ * return arbitrarily large bodies that the caller bounds via
112
+ * `OPENROUTER_VIDEO_MAX_DOWNLOAD_BYTES`.
113
+ */
114
+ async downloadVideoContent(id, index = 0, maxBytes = 256 * 1024 * 1024) {
115
+ const url = `${BASE_URL}/videos/${encodeURIComponent(id)}/content?index=${index}`;
116
+ const res = await fetchWithRetry(url, { headers: this.authHeaders() }, { retries: 1, timeoutMs: VIDEO_TIMEOUT_MS * 2 });
117
+ if (!res.ok) {
118
+ const detail = await safeReadText(res);
119
+ throw new Error(`GET /videos/${id}/content failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
120
+ }
121
+ const declared = res.headers.get('content-length');
122
+ if (declared) {
123
+ const n = parseInt(declared, 10);
124
+ if (Number.isFinite(n) && n > maxBytes) {
125
+ throw new Error(`Generated video too large: ${n} bytes > ${maxBytes}`);
126
+ }
127
+ }
128
+ const reader = res.body?.getReader();
129
+ if (!reader) {
130
+ const buf = Buffer.from(await res.arrayBuffer());
131
+ if (buf.length > maxBytes)
132
+ throw new Error('Generated video too large');
133
+ return { buffer: buf, contentType: res.headers.get('content-type') };
134
+ }
135
+ const chunks = [];
136
+ let total = 0;
137
+ for (;;) {
138
+ const { done, value } = await reader.read();
139
+ if (done)
140
+ break;
141
+ total += value.byteLength;
142
+ if (total > maxBytes) {
143
+ try {
144
+ await reader.cancel();
145
+ }
146
+ catch {
147
+ /* ignore */
148
+ }
149
+ throw new Error('Generated video too large');
150
+ }
151
+ chunks.push(Buffer.from(value));
152
+ }
153
+ return { buffer: Buffer.concat(chunks), contentType: res.headers.get('content-type') };
154
+ }
155
+ }
156
+ async function safeReadText(res) {
157
+ try {
158
+ const t = await res.text();
159
+ return t.length > 500 ? t.slice(0, 500) + '…' : t;
160
+ }
161
+ catch {
162
+ return '';
163
+ }
42
164
  }
165
+ // Exported for tests.
166
+ export const _internals = { parseRetryAfter, backoffWithJitter, fetchWithRetry };
@@ -8,16 +8,12 @@ export declare function handleAnalyzeAudio(request: {
8
8
  params: {
9
9
  arguments: AnalyzeAudioToolRequest;
10
10
  };
11
- }, openai: OpenAI, defaultModel?: string): Promise<{
11
+ }, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
12
12
  content: {
13
- type: string;
13
+ type: "text";
14
14
  text: string;
15
15
  }[];
16
- isError: boolean;
17
- } | {
18
- content: {
19
- type: string;
20
- text: string;
21
- }[];
22
- isError?: undefined;
16
+ _meta: {
17
+ finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
18
+ };
23
19
  }>;
@@ -1,13 +1,32 @@
1
1
  import { prepareAudioData } from './audio-utils.js';
2
+ import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
3
+ import { classifyUpstreamError } from './openrouter-errors.js';
4
+ import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
2
5
  const DEFAULT_MODEL = 'google/gemini-2.5-flash';
3
6
  export async function handleAnalyzeAudio(request, openai, defaultModel) {
4
- const { audio_path, question, model } = request.params.arguments;
7
+ const { audio_path, question, model } = request.params.arguments ?? { audio_path: '' };
5
8
  if (!audio_path) {
6
- return { content: [{ type: 'text', text: 'audio_path is required.' }], isError: true };
9
+ return toolError(ErrorCode.INVALID_INPUT, 'audio_path is required.');
7
10
  }
11
+ let audioData;
8
12
  try {
9
- const audioData = await prepareAudioData(audio_path);
10
- const completion = await openai.chat.completions.create({
13
+ audioData = await prepareAudioData(audio_path);
14
+ }
15
+ catch (err) {
16
+ const msg = err instanceof Error ? err.message : String(err);
17
+ if (msg.includes('Blocked host'))
18
+ return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
19
+ if (msg.toLowerCase().includes('too large')) {
20
+ return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
21
+ }
22
+ if (msg.toLowerCase().includes('unsupported')) {
23
+ return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err);
24
+ }
25
+ return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
26
+ }
27
+ let completion;
28
+ try {
29
+ completion = await openai.chat.completions.create({
11
30
  model: model || defaultModel || DEFAULT_MODEL,
12
31
  messages: [
13
32
  {
@@ -25,10 +44,24 @@ export async function handleAnalyzeAudio(request, openai, defaultModel) {
25
44
  },
26
45
  ],
27
46
  });
28
- return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
29
47
  }
30
- catch (error) {
31
- const msg = error instanceof Error ? error.message : String(error);
32
- return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
48
+ catch (err) {
49
+ return classifyUpstreamError(err);
50
+ }
51
+ const extracted = extractCompletionText(completion);
52
+ const cutoff = detectReasoningCutoff(extracted);
53
+ if (cutoff)
54
+ return cutoff;
55
+ if (!extracted.text) {
56
+ return toolError(ErrorCode.INTERNAL, 'Audio model returned no textual content.', {
57
+ finish_reason: extracted.finishReason,
58
+ });
33
59
  }
60
+ return {
61
+ content: [{ type: 'text', text: extracted.text }],
62
+ _meta: {
63
+ finish_reason: extracted.finishReason,
64
+ ...(toUsageMeta(extracted.usage) ?? {}),
65
+ },
66
+ };
34
67
  }
@@ -8,16 +8,12 @@ export declare function handleAnalyzeImage(request: {
8
8
  params: {
9
9
  arguments: AnalyzeImageToolRequest;
10
10
  };
11
- }, openai: OpenAI, defaultModel?: string): Promise<{
11
+ }, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
12
12
  content: {
13
- type: string;
13
+ type: "text";
14
14
  text: string;
15
15
  }[];
16
- isError: boolean;
17
- } | {
18
- content: {
19
- type: string;
20
- text: string;
21
- }[];
22
- isError?: undefined;
16
+ _meta: {
17
+ finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
18
+ };
23
19
  }>;
@@ -1,13 +1,29 @@
1
1
  import { prepareImageUrl } from './image-utils.js';
2
+ import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
3
+ import { classifyUpstreamError } from './openrouter-errors.js';
4
+ import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
2
5
  const DEFAULT_MODEL = 'nvidia/nemotron-nano-12b-v2-vl:free';
3
6
  export async function handleAnalyzeImage(request, openai, defaultModel) {
4
- const { image_path, question, model } = request.params.arguments;
7
+ const { image_path, question, model } = request.params.arguments ?? { image_path: '' };
5
8
  if (!image_path) {
6
- return { content: [{ type: 'text', text: 'image_path is required.' }], isError: true };
9
+ return toolError(ErrorCode.INVALID_INPUT, 'image_path is required.');
7
10
  }
11
+ let imageUrl;
8
12
  try {
9
- const imageUrl = await prepareImageUrl(image_path);
10
- const completion = await openai.chat.completions.create({
13
+ imageUrl = await prepareImageUrl(image_path);
14
+ }
15
+ catch (err) {
16
+ const msg = err instanceof Error ? err.message : String(err);
17
+ if (msg.includes('Blocked host'))
18
+ return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
19
+ if (msg.toLowerCase().includes('too large')) {
20
+ return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
21
+ }
22
+ return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
23
+ }
24
+ let completion;
25
+ try {
26
+ completion = await openai.chat.completions.create({
11
27
  model: model || defaultModel || DEFAULT_MODEL,
12
28
  messages: [
13
29
  {
@@ -19,10 +35,24 @@ export async function handleAnalyzeImage(request, openai, defaultModel) {
19
35
  },
20
36
  ],
21
37
  });
22
- return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
23
38
  }
24
- catch (error) {
25
- const msg = error instanceof Error ? error.message : String(error);
26
- return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
39
+ catch (err) {
40
+ return classifyUpstreamError(err);
41
+ }
42
+ const extracted = extractCompletionText(completion);
43
+ const cutoff = detectReasoningCutoff(extracted);
44
+ if (cutoff)
45
+ return cutoff;
46
+ if (!extracted.text) {
47
+ return toolError(ErrorCode.INTERNAL, 'Vision model returned no textual content.', {
48
+ finish_reason: extracted.finishReason,
49
+ });
27
50
  }
51
+ return {
52
+ content: [{ type: 'text', text: extracted.text }],
53
+ _meta: {
54
+ finish_reason: extracted.finishReason,
55
+ ...(toUsageMeta(extracted.usage) ?? {}),
56
+ },
57
+ };
28
58
  }
@@ -0,0 +1,19 @@
1
+ import OpenAI from 'openai';
2
+ export interface AnalyzeVideoToolRequest {
3
+ video_path: string;
4
+ question?: string;
5
+ model?: string;
6
+ }
7
+ export declare function handleAnalyzeVideo(request: {
8
+ params: {
9
+ arguments: AnalyzeVideoToolRequest;
10
+ };
11
+ }, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
12
+ content: {
13
+ type: "text";
14
+ text: string;
15
+ }[];
16
+ _meta: {
17
+ finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
18
+ };
19
+ }>;
@@ -0,0 +1,93 @@
1
+ import { prepareVideoData } from './video-utils.js';
2
+ import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
3
+ import { logger } from '../logger.js';
4
+ import { classifyUpstreamError } from './openrouter-errors.js';
5
+ import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
6
+ /**
7
+ * Default model — `google/gemini-2.5-flash` has the widest video-input
8
+ * support on OpenRouter at time of writing. Override via env
9
+ * `OPENROUTER_DEFAULT_VIDEO_MODEL` or per-call `model`.
10
+ */
11
+ const FALLBACK_DEFAULT_MODEL = 'google/gemini-2.5-flash';
12
+ export async function handleAnalyzeVideo(request, openai, defaultModel) {
13
+ const { video_path, question, model } = request.params.arguments ?? {
14
+ video_path: '',
15
+ };
16
+ if (!video_path) {
17
+ return toolError(ErrorCode.INVALID_INPUT, 'video_path is required.');
18
+ }
19
+ const pickedModel = model ||
20
+ process.env.OPENROUTER_DEFAULT_VIDEO_MODEL ||
21
+ defaultModel ||
22
+ FALLBACK_DEFAULT_MODEL;
23
+ let videoData;
24
+ try {
25
+ videoData = await prepareVideoData(video_path);
26
+ }
27
+ catch (err) {
28
+ const msg = err instanceof Error ? err.message : String(err);
29
+ if (msg.includes('Blocked host')) {
30
+ return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
31
+ }
32
+ if (msg.includes('too large')) {
33
+ return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
34
+ }
35
+ if (msg.includes('Unsupported') || msg.includes('not a video')) {
36
+ return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err);
37
+ }
38
+ return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
39
+ }
40
+ let completion;
41
+ try {
42
+ logger.debug('analyze_video.submit', {
43
+ model: pickedModel,
44
+ format: videoData.format,
45
+ size_bytes: videoData.sizeBytes,
46
+ });
47
+ completion = await openai.chat.completions.create({
48
+ model: pickedModel,
49
+ messages: [
50
+ {
51
+ role: 'user',
52
+ content: [
53
+ {
54
+ type: 'text',
55
+ text: question || 'Describe what happens in this video, step by step.',
56
+ },
57
+ {
58
+ // The `video_url` content type is an OpenRouter extension; the
59
+ // OpenAI SDK's typings don't know about it yet. See:
60
+ // https://openrouter.ai/docs/guides/overview/multimodal/videos
61
+ type: 'video_url',
62
+ video_url: {
63
+ url: `data:${videoData.mediaType};base64,${videoData.data}`,
64
+ },
65
+ },
66
+ ],
67
+ },
68
+ ],
69
+ });
70
+ }
71
+ catch (err) {
72
+ logger.warn('analyze_video.error', {
73
+ err: err instanceof Error ? err.message : String(err),
74
+ });
75
+ return classifyUpstreamError(err);
76
+ }
77
+ const extracted = extractCompletionText(completion);
78
+ const cutoff = detectReasoningCutoff(extracted);
79
+ if (cutoff)
80
+ return cutoff;
81
+ if (!extracted.text) {
82
+ return toolError(ErrorCode.INTERNAL, 'Video model returned no textual content.', {
83
+ finish_reason: extracted.finishReason,
84
+ });
85
+ }
86
+ return {
87
+ content: [{ type: 'text', text: extracted.text }],
88
+ _meta: {
89
+ finish_reason: extracted.finishReason,
90
+ ...(toUsageMeta(extracted.usage) ?? {}),
91
+ },
92
+ };
93
+ }
@@ -4,7 +4,7 @@
4
4
  */
5
5
  import path from 'path';
6
6
  import { promises as fs } from 'fs';
7
- import { readEnvInt, fetchHttpResource } from './fetch-utils.js';
7
+ import { readEnvInt, fetchHttpResource, parseBase64DataUrl } from './fetch-utils.js';
8
8
  // Re-export for tests
9
9
  export { isBlockedIPv4, assertUrlSafeForFetch } from './fetch-utils.js';
10
10
  const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
@@ -89,19 +89,17 @@ function formatFromContentType(ct) {
89
89
  export async function prepareAudioData(source) {
90
90
  // --- data URL ---
91
91
  if (source.startsWith('data:')) {
92
- const match = source.match(/^data:([^;]+);base64,(.+)$/);
93
- if (!match)
92
+ const parsed = parseBase64DataUrl(source);
93
+ if (!parsed)
94
94
  throw new Error('Invalid data URL format');
95
- const mime = match[1];
96
- const b64 = match[2];
97
- const format = mimeSubtypeToFormat(mime.split('/')[1] ?? '');
95
+ const format = mimeSubtypeToFormat(parsed.mediaType.split('/')[1] ?? '');
98
96
  if (!format) {
99
- throw new Error(`Unsupported audio format from MIME: ${mime}. Supported: ${SUPPORTED_AUDIO_FORMATS.join(', ')}`);
97
+ throw new Error(`Unsupported audio format from MIME: ${parsed.mediaType}. Supported: ${SUPPORTED_AUDIO_FORMATS.join(', ')}`);
100
98
  }
101
- const approxBytes = Math.ceil((b64.length * 3) / 4);
99
+ const approxBytes = Math.ceil((parsed.base64.length * 3) / 4);
102
100
  if (approxBytes > getMaxDataUrlBytes())
103
101
  throw new Error('Data URL too large');
104
- return { data: b64, format };
102
+ return { data: parsed.base64, format };
105
103
  }
106
104
  // --- HTTP(S) URL ---
107
105
  if (source.startsWith('http://') || source.startsWith('https://')) {