@stabgan/openrouter-mcp-multimodal 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -43
- package/dist/errors.d.ts +42 -0
- package/dist/errors.js +46 -0
- package/dist/index.js +1 -1
- package/dist/logger.d.ts +22 -0
- package/dist/logger.js +47 -0
- package/dist/model-cache.d.ts +10 -0
- package/dist/model-cache.js +31 -1
- package/dist/openrouter-api.d.ts +54 -0
- package/dist/openrouter-api.js +128 -12
- package/dist/tool-handlers/analyze-audio.d.ts +5 -9
- package/dist/tool-handlers/analyze-audio.js +41 -8
- package/dist/tool-handlers/analyze-image.d.ts +5 -9
- package/dist/tool-handlers/analyze-image.js +38 -8
- package/dist/tool-handlers/analyze-video.d.ts +19 -0
- package/dist/tool-handlers/analyze-video.js +93 -0
- package/dist/tool-handlers/audio-utils.js +7 -9
- package/dist/tool-handlers/chat-completion.d.ts +6 -10
- package/dist/tool-handlers/chat-completion.js +27 -7
- package/dist/tool-handlers/completion-utils.d.ts +27 -0
- package/dist/tool-handlers/completion-utils.js +69 -0
- package/dist/tool-handlers/fetch-utils.d.ts +21 -0
- package/dist/tool-handlers/fetch-utils.js +166 -11
- package/dist/tool-handlers/generate-audio.d.ts +32 -12
- package/dist/tool-handlers/generate-audio.js +77 -46
- package/dist/tool-handlers/generate-image.d.ts +26 -10
- package/dist/tool-handlers/generate-image.js +79 -27
- package/dist/tool-handlers/generate-video.d.ts +78 -0
- package/dist/tool-handlers/generate-video.js +353 -0
- package/dist/tool-handlers/get-model-info.js +8 -2
- package/dist/tool-handlers/image-utils.d.ts +17 -1
- package/dist/tool-handlers/image-utils.js +66 -13
- package/dist/tool-handlers/openrouter-errors.d.ts +18 -0
- package/dist/tool-handlers/openrouter-errors.js +99 -0
- package/dist/tool-handlers/path-safety.d.ts +11 -0
- package/dist/tool-handlers/path-safety.js +88 -0
- package/dist/tool-handlers/search-models.js +1 -3
- package/dist/tool-handlers/validate-model.js +8 -2
- package/dist/tool-handlers/video-utils.d.ts +29 -0
- package/dist/tool-handlers/video-utils.js +174 -0
- package/dist/tool-handlers.js +199 -21
- package/package.json +3 -3
- package/dist/__tests__/audio-utils.test.d.ts +0 -1
- package/dist/__tests__/audio-utils.test.js +0 -120
- package/dist/__tests__/fetch-utils.test.d.ts +0 -1
- package/dist/__tests__/fetch-utils.test.js +0 -76
- package/dist/__tests__/generate-audio.test.d.ts +0 -1
- package/dist/__tests__/generate-audio.test.js +0 -90
- package/dist/__tests__/image-utils.test.d.ts +0 -1
- package/dist/__tests__/image-utils.test.js +0 -75
- package/dist/__tests__/integration.test.d.ts +0 -1
- package/dist/__tests__/integration.test.js +0 -219
- package/dist/__tests__/model-cache.test.d.ts +0 -1
- package/dist/__tests__/model-cache.test.js +0 -96
package/dist/openrouter-api.js
CHANGED
|
@@ -1,25 +1,55 @@
|
|
|
1
1
|
const BASE_URL = 'https://openrouter.ai/api/v1';
|
|
2
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
3
|
+
const VIDEO_TIMEOUT_MS = 60_000;
|
|
4
|
+
const MAX_BACKOFF_MS = 10_000;
|
|
2
5
|
async function sleep(ms) {
|
|
3
6
|
return new Promise((r) => setTimeout(r, ms));
|
|
4
7
|
}
|
|
5
|
-
|
|
8
|
+
function parseRetryAfter(headerValue) {
|
|
9
|
+
if (!headerValue)
|
|
10
|
+
return null;
|
|
11
|
+
const asInt = parseInt(headerValue, 10);
|
|
12
|
+
if (Number.isFinite(asInt) && asInt >= 0)
|
|
13
|
+
return asInt * 1000;
|
|
14
|
+
const asDate = Date.parse(headerValue);
|
|
15
|
+
if (Number.isFinite(asDate)) {
|
|
16
|
+
const delta = asDate - Date.now();
|
|
17
|
+
return delta > 0 ? delta : 0;
|
|
18
|
+
}
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
function backoffWithJitter(attempt, retryAfterMs) {
|
|
22
|
+
const base = 400 * (attempt + 1);
|
|
23
|
+
const target = Math.min(Math.max(base, retryAfterMs ?? 0), MAX_BACKOFF_MS);
|
|
24
|
+
const jitter = 0.5 + Math.random(); // 0.5x .. 1.5x
|
|
25
|
+
return Math.round(target * jitter);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* fetch() wrapper with retries on 429 / 5xx / network error.
|
|
29
|
+
*
|
|
30
|
+
* A fresh `AbortSignal.timeout(timeoutMs)` is created per attempt so retries
|
|
31
|
+
* each get a full timeout budget. Backoff honors `Retry-After` (seconds or
|
|
32
|
+
* HTTP-date) and applies jitter to avoid thundering-herd synchronization.
|
|
33
|
+
*/
|
|
34
|
+
async function fetchWithRetry(url, init, { retries = 2, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
6
35
|
let lastErr;
|
|
7
36
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
8
37
|
try {
|
|
9
|
-
const res = await fetch(url, init);
|
|
38
|
+
const res = await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
10
39
|
if (res.status === 429 || res.status >= 500) {
|
|
11
|
-
if (attempt < retries)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
40
|
+
if (attempt < retries) {
|
|
41
|
+
const retryAfter = parseRetryAfter(res.headers.get('retry-after'));
|
|
42
|
+
await sleep(backoffWithJitter(attempt, retryAfter));
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
return res;
|
|
16
46
|
}
|
|
17
47
|
return res;
|
|
18
48
|
}
|
|
19
49
|
catch (e) {
|
|
20
50
|
lastErr = e;
|
|
21
51
|
if (attempt < retries)
|
|
22
|
-
await sleep(
|
|
52
|
+
await sleep(backoffWithJitter(attempt, null));
|
|
23
53
|
}
|
|
24
54
|
}
|
|
25
55
|
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
|
@@ -29,14 +59,100 @@ export class OpenRouterAPIClient {
|
|
|
29
59
|
constructor(apiKey) {
|
|
30
60
|
this.apiKey = apiKey;
|
|
31
61
|
}
|
|
62
|
+
authHeaders(extra) {
|
|
63
|
+
return {
|
|
64
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
65
|
+
'HTTP-Referer': 'https://github.com/stabgan/openrouter-mcp-multimodal',
|
|
66
|
+
'X-Title': 'openrouter-mcp-multimodal',
|
|
67
|
+
...extra,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
32
70
|
async getModels() {
|
|
33
|
-
const res = await fetchWithRetry(`${BASE_URL}/models`, {
|
|
34
|
-
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
35
|
-
signal: AbortSignal.timeout(30000),
|
|
36
|
-
}, 2);
|
|
71
|
+
const res = await fetchWithRetry(`${BASE_URL}/models`, { headers: this.authHeaders() }, { retries: 2, timeoutMs: DEFAULT_TIMEOUT_MS });
|
|
37
72
|
if (!res.ok)
|
|
38
73
|
throw new Error(`Failed to fetch models: HTTP ${res.status}`);
|
|
39
74
|
const data = (await res.json());
|
|
40
75
|
return data.data ?? [];
|
|
41
76
|
}
|
|
77
|
+
/** Submit a video-generation job. Returns the `{ id, polling_url, status }` envelope. */
|
|
78
|
+
async submitVideoJob(body) {
|
|
79
|
+
const res = await fetchWithRetry(`${BASE_URL}/videos`, {
|
|
80
|
+
method: 'POST',
|
|
81
|
+
headers: this.authHeaders({ 'Content-Type': 'application/json' }),
|
|
82
|
+
body: JSON.stringify(body),
|
|
83
|
+
}, { retries: 2, timeoutMs: VIDEO_TIMEOUT_MS });
|
|
84
|
+
if (!res.ok) {
|
|
85
|
+
const detail = await safeReadText(res);
|
|
86
|
+
throw new Error(`POST /videos failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
|
|
87
|
+
}
|
|
88
|
+
return (await res.json());
|
|
89
|
+
}
|
|
90
|
+
/** Poll a submitted video-generation job by id. */
|
|
91
|
+
async pollVideoJob(id) {
|
|
92
|
+
const res = await fetchWithRetry(`${BASE_URL}/videos/${encodeURIComponent(id)}`, { headers: this.authHeaders() }, { retries: 2, timeoutMs: DEFAULT_TIMEOUT_MS });
|
|
93
|
+
if (!res.ok) {
|
|
94
|
+
const detail = await safeReadText(res);
|
|
95
|
+
throw new Error(`GET /videos/${id} failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
|
|
96
|
+
}
|
|
97
|
+
return (await res.json());
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Download the generated video binary. Returns `{ buffer, contentType }`.
|
|
101
|
+
* This intentionally does NOT go through our SSRF-guarded `fetchHttpResource`
|
|
102
|
+
* because the URL is always OpenRouter itself (trusted origin) — and it can
|
|
103
|
+
* return arbitrarily large bodies that the caller bounds via
|
|
104
|
+
* `OPENROUTER_VIDEO_MAX_DOWNLOAD_BYTES`.
|
|
105
|
+
*/
|
|
106
|
+
async downloadVideoContent(id, index = 0, maxBytes = 256 * 1024 * 1024) {
|
|
107
|
+
const url = `${BASE_URL}/videos/${encodeURIComponent(id)}/content?index=${index}`;
|
|
108
|
+
const res = await fetchWithRetry(url, { headers: this.authHeaders() }, { retries: 1, timeoutMs: VIDEO_TIMEOUT_MS * 2 });
|
|
109
|
+
if (!res.ok) {
|
|
110
|
+
const detail = await safeReadText(res);
|
|
111
|
+
throw new Error(`GET /videos/${id}/content failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
|
|
112
|
+
}
|
|
113
|
+
const declared = res.headers.get('content-length');
|
|
114
|
+
if (declared) {
|
|
115
|
+
const n = parseInt(declared, 10);
|
|
116
|
+
if (Number.isFinite(n) && n > maxBytes) {
|
|
117
|
+
throw new Error(`Generated video too large: ${n} bytes > ${maxBytes}`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const reader = res.body?.getReader();
|
|
121
|
+
if (!reader) {
|
|
122
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
123
|
+
if (buf.length > maxBytes)
|
|
124
|
+
throw new Error('Generated video too large');
|
|
125
|
+
return { buffer: buf, contentType: res.headers.get('content-type') };
|
|
126
|
+
}
|
|
127
|
+
const chunks = [];
|
|
128
|
+
let total = 0;
|
|
129
|
+
for (;;) {
|
|
130
|
+
const { done, value } = await reader.read();
|
|
131
|
+
if (done)
|
|
132
|
+
break;
|
|
133
|
+
total += value.byteLength;
|
|
134
|
+
if (total > maxBytes) {
|
|
135
|
+
try {
|
|
136
|
+
await reader.cancel();
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
/* ignore */
|
|
140
|
+
}
|
|
141
|
+
throw new Error('Generated video too large');
|
|
142
|
+
}
|
|
143
|
+
chunks.push(Buffer.from(value));
|
|
144
|
+
}
|
|
145
|
+
return { buffer: Buffer.concat(chunks), contentType: res.headers.get('content-type') };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
async function safeReadText(res) {
|
|
149
|
+
try {
|
|
150
|
+
const t = await res.text();
|
|
151
|
+
return t.length > 500 ? t.slice(0, 500) + '…' : t;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return '';
|
|
155
|
+
}
|
|
42
156
|
}
|
|
157
|
+
// Exported for tests.
|
|
158
|
+
export const _internals = { parseRetryAfter, backoffWithJitter, fetchWithRetry };
|
|
@@ -8,16 +8,12 @@ export declare function handleAnalyzeAudio(request: {
|
|
|
8
8
|
params: {
|
|
9
9
|
arguments: AnalyzeAudioToolRequest;
|
|
10
10
|
};
|
|
11
|
-
}, openai: OpenAI, defaultModel?: string): Promise<{
|
|
11
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
12
12
|
content: {
|
|
13
|
-
type:
|
|
13
|
+
type: "text";
|
|
14
14
|
text: string;
|
|
15
15
|
}[];
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
type: string;
|
|
20
|
-
text: string;
|
|
21
|
-
}[];
|
|
22
|
-
isError?: undefined;
|
|
16
|
+
_meta: {
|
|
17
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
18
|
+
};
|
|
23
19
|
}>;
|
|
@@ -1,13 +1,32 @@
|
|
|
1
1
|
import { prepareAudioData } from './audio-utils.js';
|
|
2
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
3
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
4
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
2
5
|
const DEFAULT_MODEL = 'google/gemini-2.5-flash';
|
|
3
6
|
export async function handleAnalyzeAudio(request, openai, defaultModel) {
|
|
4
|
-
const { audio_path, question, model } = request.params.arguments;
|
|
7
|
+
const { audio_path, question, model } = request.params.arguments ?? { audio_path: '' };
|
|
5
8
|
if (!audio_path) {
|
|
6
|
-
return
|
|
9
|
+
return toolError(ErrorCode.INVALID_INPUT, 'audio_path is required.');
|
|
7
10
|
}
|
|
11
|
+
let audioData;
|
|
8
12
|
try {
|
|
9
|
-
|
|
10
|
-
|
|
13
|
+
audioData = await prepareAudioData(audio_path);
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
17
|
+
if (msg.includes('Blocked host'))
|
|
18
|
+
return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
|
|
19
|
+
if (msg.toLowerCase().includes('too large')) {
|
|
20
|
+
return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
|
|
21
|
+
}
|
|
22
|
+
if (msg.toLowerCase().includes('unsupported')) {
|
|
23
|
+
return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err);
|
|
24
|
+
}
|
|
25
|
+
return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
|
|
26
|
+
}
|
|
27
|
+
let completion;
|
|
28
|
+
try {
|
|
29
|
+
completion = await openai.chat.completions.create({
|
|
11
30
|
model: model || defaultModel || DEFAULT_MODEL,
|
|
12
31
|
messages: [
|
|
13
32
|
{
|
|
@@ -25,10 +44,24 @@ export async function handleAnalyzeAudio(request, openai, defaultModel) {
|
|
|
25
44
|
},
|
|
26
45
|
],
|
|
27
46
|
});
|
|
28
|
-
return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
|
|
29
47
|
}
|
|
30
|
-
catch (
|
|
31
|
-
|
|
32
|
-
|
|
48
|
+
catch (err) {
|
|
49
|
+
return classifyUpstreamError(err);
|
|
50
|
+
}
|
|
51
|
+
const extracted = extractCompletionText(completion);
|
|
52
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
53
|
+
if (cutoff)
|
|
54
|
+
return cutoff;
|
|
55
|
+
if (!extracted.text) {
|
|
56
|
+
return toolError(ErrorCode.INTERNAL, 'Audio model returned no textual content.', {
|
|
57
|
+
finish_reason: extracted.finishReason,
|
|
58
|
+
});
|
|
33
59
|
}
|
|
60
|
+
return {
|
|
61
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
62
|
+
_meta: {
|
|
63
|
+
finish_reason: extracted.finishReason,
|
|
64
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
65
|
+
},
|
|
66
|
+
};
|
|
34
67
|
}
|
|
@@ -8,16 +8,12 @@ export declare function handleAnalyzeImage(request: {
|
|
|
8
8
|
params: {
|
|
9
9
|
arguments: AnalyzeImageToolRequest;
|
|
10
10
|
};
|
|
11
|
-
}, openai: OpenAI, defaultModel?: string): Promise<{
|
|
11
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
12
12
|
content: {
|
|
13
|
-
type:
|
|
13
|
+
type: "text";
|
|
14
14
|
text: string;
|
|
15
15
|
}[];
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
type: string;
|
|
20
|
-
text: string;
|
|
21
|
-
}[];
|
|
22
|
-
isError?: undefined;
|
|
16
|
+
_meta: {
|
|
17
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
18
|
+
};
|
|
23
19
|
}>;
|
|
@@ -1,13 +1,29 @@
|
|
|
1
1
|
import { prepareImageUrl } from './image-utils.js';
|
|
2
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
3
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
4
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
2
5
|
const DEFAULT_MODEL = 'nvidia/nemotron-nano-12b-v2-vl:free';
|
|
3
6
|
export async function handleAnalyzeImage(request, openai, defaultModel) {
|
|
4
|
-
const { image_path, question, model } = request.params.arguments;
|
|
7
|
+
const { image_path, question, model } = request.params.arguments ?? { image_path: '' };
|
|
5
8
|
if (!image_path) {
|
|
6
|
-
return
|
|
9
|
+
return toolError(ErrorCode.INVALID_INPUT, 'image_path is required.');
|
|
7
10
|
}
|
|
11
|
+
let imageUrl;
|
|
8
12
|
try {
|
|
9
|
-
|
|
10
|
-
|
|
13
|
+
imageUrl = await prepareImageUrl(image_path);
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
17
|
+
if (msg.includes('Blocked host'))
|
|
18
|
+
return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
|
|
19
|
+
if (msg.toLowerCase().includes('too large')) {
|
|
20
|
+
return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
|
|
21
|
+
}
|
|
22
|
+
return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
|
|
23
|
+
}
|
|
24
|
+
let completion;
|
|
25
|
+
try {
|
|
26
|
+
completion = await openai.chat.completions.create({
|
|
11
27
|
model: model || defaultModel || DEFAULT_MODEL,
|
|
12
28
|
messages: [
|
|
13
29
|
{
|
|
@@ -19,10 +35,24 @@ export async function handleAnalyzeImage(request, openai, defaultModel) {
|
|
|
19
35
|
},
|
|
20
36
|
],
|
|
21
37
|
});
|
|
22
|
-
return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
|
|
23
38
|
}
|
|
24
|
-
catch (
|
|
25
|
-
|
|
26
|
-
|
|
39
|
+
catch (err) {
|
|
40
|
+
return classifyUpstreamError(err);
|
|
41
|
+
}
|
|
42
|
+
const extracted = extractCompletionText(completion);
|
|
43
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
44
|
+
if (cutoff)
|
|
45
|
+
return cutoff;
|
|
46
|
+
if (!extracted.text) {
|
|
47
|
+
return toolError(ErrorCode.INTERNAL, 'Vision model returned no textual content.', {
|
|
48
|
+
finish_reason: extracted.finishReason,
|
|
49
|
+
});
|
|
27
50
|
}
|
|
51
|
+
return {
|
|
52
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
53
|
+
_meta: {
|
|
54
|
+
finish_reason: extracted.finishReason,
|
|
55
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
56
|
+
},
|
|
57
|
+
};
|
|
28
58
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
export interface AnalyzeVideoToolRequest {
|
|
3
|
+
video_path: string;
|
|
4
|
+
question?: string;
|
|
5
|
+
model?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function handleAnalyzeVideo(request: {
|
|
8
|
+
params: {
|
|
9
|
+
arguments: AnalyzeVideoToolRequest;
|
|
10
|
+
};
|
|
11
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
12
|
+
content: {
|
|
13
|
+
type: "text";
|
|
14
|
+
text: string;
|
|
15
|
+
}[];
|
|
16
|
+
_meta: {
|
|
17
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
18
|
+
};
|
|
19
|
+
}>;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { prepareVideoData } from './video-utils.js';
|
|
2
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
3
|
+
import { logger } from '../logger.js';
|
|
4
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
5
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
6
|
+
/**
|
|
7
|
+
* Default model — `google/gemini-2.5-flash` has the widest video-input
|
|
8
|
+
* support on OpenRouter at time of writing. Override via env
|
|
9
|
+
* `OPENROUTER_DEFAULT_VIDEO_MODEL` or per-call `model`.
|
|
10
|
+
*/
|
|
11
|
+
const FALLBACK_DEFAULT_MODEL = 'google/gemini-2.5-flash';
|
|
12
|
+
export async function handleAnalyzeVideo(request, openai, defaultModel) {
|
|
13
|
+
const { video_path, question, model } = request.params.arguments ?? {
|
|
14
|
+
video_path: '',
|
|
15
|
+
};
|
|
16
|
+
if (!video_path) {
|
|
17
|
+
return toolError(ErrorCode.INVALID_INPUT, 'video_path is required.');
|
|
18
|
+
}
|
|
19
|
+
const pickedModel = model ||
|
|
20
|
+
process.env.OPENROUTER_DEFAULT_VIDEO_MODEL ||
|
|
21
|
+
defaultModel ||
|
|
22
|
+
FALLBACK_DEFAULT_MODEL;
|
|
23
|
+
let videoData;
|
|
24
|
+
try {
|
|
25
|
+
videoData = await prepareVideoData(video_path);
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
29
|
+
if (msg.includes('Blocked host')) {
|
|
30
|
+
return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
|
|
31
|
+
}
|
|
32
|
+
if (msg.includes('too large')) {
|
|
33
|
+
return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
|
|
34
|
+
}
|
|
35
|
+
if (msg.includes('Unsupported') || msg.includes('not a video')) {
|
|
36
|
+
return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err);
|
|
37
|
+
}
|
|
38
|
+
return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
|
|
39
|
+
}
|
|
40
|
+
let completion;
|
|
41
|
+
try {
|
|
42
|
+
logger.debug('analyze_video.submit', {
|
|
43
|
+
model: pickedModel,
|
|
44
|
+
format: videoData.format,
|
|
45
|
+
size_bytes: videoData.sizeBytes,
|
|
46
|
+
});
|
|
47
|
+
completion = await openai.chat.completions.create({
|
|
48
|
+
model: pickedModel,
|
|
49
|
+
messages: [
|
|
50
|
+
{
|
|
51
|
+
role: 'user',
|
|
52
|
+
content: [
|
|
53
|
+
{
|
|
54
|
+
type: 'text',
|
|
55
|
+
text: question || 'Describe what happens in this video, step by step.',
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
// The `video_url` content type is an OpenRouter extension; the
|
|
59
|
+
// OpenAI SDK's typings don't know about it yet. See:
|
|
60
|
+
// https://openrouter.ai/docs/guides/overview/multimodal/videos
|
|
61
|
+
type: 'video_url',
|
|
62
|
+
video_url: {
|
|
63
|
+
url: `data:${videoData.mediaType};base64,${videoData.data}`,
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
],
|
|
67
|
+
},
|
|
68
|
+
],
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
logger.warn('analyze_video.error', {
|
|
73
|
+
err: err instanceof Error ? err.message : String(err),
|
|
74
|
+
});
|
|
75
|
+
return classifyUpstreamError(err);
|
|
76
|
+
}
|
|
77
|
+
const extracted = extractCompletionText(completion);
|
|
78
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
79
|
+
if (cutoff)
|
|
80
|
+
return cutoff;
|
|
81
|
+
if (!extracted.text) {
|
|
82
|
+
return toolError(ErrorCode.INTERNAL, 'Video model returned no textual content.', {
|
|
83
|
+
finish_reason: extracted.finishReason,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
88
|
+
_meta: {
|
|
89
|
+
finish_reason: extracted.finishReason,
|
|
90
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import path from 'path';
|
|
6
6
|
import { promises as fs } from 'fs';
|
|
7
|
-
import { readEnvInt, fetchHttpResource } from './fetch-utils.js';
|
|
7
|
+
import { readEnvInt, fetchHttpResource, parseBase64DataUrl } from './fetch-utils.js';
|
|
8
8
|
// Re-export for tests
|
|
9
9
|
export { isBlockedIPv4, assertUrlSafeForFetch } from './fetch-utils.js';
|
|
10
10
|
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
|
@@ -89,19 +89,17 @@ function formatFromContentType(ct) {
|
|
|
89
89
|
export async function prepareAudioData(source) {
|
|
90
90
|
// --- data URL ---
|
|
91
91
|
if (source.startsWith('data:')) {
|
|
92
|
-
const
|
|
93
|
-
if (!
|
|
92
|
+
const parsed = parseBase64DataUrl(source);
|
|
93
|
+
if (!parsed)
|
|
94
94
|
throw new Error('Invalid data URL format');
|
|
95
|
-
const
|
|
96
|
-
const b64 = match[2];
|
|
97
|
-
const format = mimeSubtypeToFormat(mime.split('/')[1] ?? '');
|
|
95
|
+
const format = mimeSubtypeToFormat(parsed.mediaType.split('/')[1] ?? '');
|
|
98
96
|
if (!format) {
|
|
99
|
-
throw new Error(`Unsupported audio format from MIME: ${
|
|
97
|
+
throw new Error(`Unsupported audio format from MIME: ${parsed.mediaType}. Supported: ${SUPPORTED_AUDIO_FORMATS.join(', ')}`);
|
|
100
98
|
}
|
|
101
|
-
const approxBytes = Math.ceil((
|
|
99
|
+
const approxBytes = Math.ceil((parsed.base64.length * 3) / 4);
|
|
102
100
|
if (approxBytes > getMaxDataUrlBytes())
|
|
103
101
|
throw new Error('Data URL too large');
|
|
104
|
-
return { data:
|
|
102
|
+
return { data: parsed.base64, format };
|
|
105
103
|
}
|
|
106
104
|
// --- HTTP(S) URL ---
|
|
107
105
|
if (source.startsWith('http://') || source.startsWith('https://')) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import OpenAI from 'openai';
|
|
2
|
-
import { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js';
|
|
2
|
+
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js';
|
|
3
3
|
export interface ChatCompletionToolRequest {
|
|
4
4
|
model?: string;
|
|
5
5
|
messages: ChatCompletionMessageParam[];
|
|
@@ -10,16 +10,12 @@ export declare function handleChatCompletion(request: {
|
|
|
10
10
|
params: {
|
|
11
11
|
arguments: ChatCompletionToolRequest;
|
|
12
12
|
};
|
|
13
|
-
}, openai: OpenAI, defaultModel?: string): Promise<{
|
|
13
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
14
14
|
content: {
|
|
15
|
-
type:
|
|
15
|
+
type: "text";
|
|
16
16
|
text: string;
|
|
17
17
|
}[];
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
type: string;
|
|
22
|
-
text: string;
|
|
23
|
-
}[];
|
|
24
|
-
isError?: undefined;
|
|
18
|
+
_meta: {
|
|
19
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
20
|
+
};
|
|
25
21
|
}>;
|
|
@@ -1,19 +1,39 @@
|
|
|
1
|
+
import { ErrorCode, toolError } from '../errors.js';
|
|
2
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
3
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
1
4
|
export async function handleChatCompletion(request, openai, defaultModel) {
|
|
2
|
-
const { messages, model, temperature, max_tokens } = request.params.arguments
|
|
5
|
+
const { messages, model, temperature, max_tokens } = request.params.arguments ?? {
|
|
6
|
+
messages: [],
|
|
7
|
+
};
|
|
3
8
|
if (!messages?.length) {
|
|
4
|
-
return
|
|
9
|
+
return toolError(ErrorCode.INVALID_INPUT, 'Messages array cannot be empty.');
|
|
5
10
|
}
|
|
11
|
+
let completion;
|
|
6
12
|
try {
|
|
7
|
-
|
|
13
|
+
completion = await openai.chat.completions.create({
|
|
8
14
|
model: model || defaultModel || 'nvidia/nemotron-nano-12b-v2-vl:free',
|
|
9
15
|
messages,
|
|
10
16
|
temperature: temperature ?? 1,
|
|
11
17
|
...(max_tokens && { max_tokens }),
|
|
12
18
|
});
|
|
13
|
-
return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
|
|
14
19
|
}
|
|
15
|
-
catch (
|
|
16
|
-
|
|
17
|
-
return { content: [{ type: 'text', text: `API error: ${msg}` }], isError: true };
|
|
20
|
+
catch (err) {
|
|
21
|
+
return classifyUpstreamError(err);
|
|
18
22
|
}
|
|
23
|
+
const extracted = extractCompletionText(completion);
|
|
24
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
25
|
+
if (cutoff)
|
|
26
|
+
return cutoff;
|
|
27
|
+
if (!extracted.text) {
|
|
28
|
+
return toolError(ErrorCode.INTERNAL, 'Model returned no textual content.', {
|
|
29
|
+
finish_reason: extracted.finishReason,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
34
|
+
_meta: {
|
|
35
|
+
finish_reason: extracted.finishReason,
|
|
36
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
37
|
+
},
|
|
38
|
+
};
|
|
19
39
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for tools that call `openai.chat.completions.create` and
|
|
3
|
+
* return the assistant's message as text. Handles:
|
|
4
|
+
* - plain string content (the common case)
|
|
5
|
+
* - multimodal array content (concatenate text parts)
|
|
6
|
+
* - reasoning-only responses (`content: null` + `reasoning`/`reasoning_details`)
|
|
7
|
+
* - `finish_reason === 'length'` — warn the caller so they know to raise
|
|
8
|
+
* `max_tokens` instead of silently getting nothing back.
|
|
9
|
+
*/
|
|
10
|
+
import type { ChatCompletion } from 'openai/resources/chat/completions.js';
|
|
11
|
+
import { type ToolErrorResult } from '../errors.js';
|
|
12
|
+
export interface ExtractedText {
|
|
13
|
+
text: string;
|
|
14
|
+
/** True when `text` came from the reasoning trace (not a final answer). */
|
|
15
|
+
reasonedOnly: boolean;
|
|
16
|
+
finishReason: ChatCompletion.Choice['finish_reason'] | undefined;
|
|
17
|
+
usage?: ChatCompletion['usage'];
|
|
18
|
+
}
|
|
19
|
+
export declare function extractCompletionText(completion: ChatCompletion): ExtractedText;
|
|
20
|
+
/**
|
|
21
|
+
* If the extracted response is reasoning-only and was cut off by
|
|
22
|
+
* `max_tokens`, return a structured INVALID_INPUT suggesting the caller
|
|
23
|
+
* raise the budget. Otherwise return `null` (let the caller format the
|
|
24
|
+
* success response).
|
|
25
|
+
*/
|
|
26
|
+
export declare function detectReasoningCutoff(extracted: ExtractedText): ToolErrorResult | null;
|
|
27
|
+
export declare function toUsageMeta(usage: ChatCompletion['usage'] | undefined): Record<string, unknown> | undefined;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { ErrorCode, toolError } from '../errors.js';
|
|
2
|
+
export function extractCompletionText(completion) {
|
|
3
|
+
const choice = completion.choices?.[0];
|
|
4
|
+
const msg = choice?.message;
|
|
5
|
+
const finishReason = choice?.finish_reason;
|
|
6
|
+
const usage = completion.usage ?? undefined;
|
|
7
|
+
if (!msg)
|
|
8
|
+
return { text: '', reasonedOnly: false, finishReason, usage };
|
|
9
|
+
const { content, reasoning, reasoning_details } = msg;
|
|
10
|
+
if (typeof content === 'string' && content.length > 0) {
|
|
11
|
+
return { text: content, reasonedOnly: false, finishReason, usage };
|
|
12
|
+
}
|
|
13
|
+
if (Array.isArray(content)) {
|
|
14
|
+
const parts = content
|
|
15
|
+
.filter((p) => p.type === 'text' && typeof p.text === 'string')
|
|
16
|
+
.map((p) => p.text ?? '');
|
|
17
|
+
const joined = parts.join('');
|
|
18
|
+
if (joined.length > 0) {
|
|
19
|
+
return { text: joined, reasonedOnly: false, finishReason, usage };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
if (typeof reasoning === 'string' && reasoning.length > 0) {
|
|
23
|
+
return { text: reasoning, reasonedOnly: true, finishReason, usage };
|
|
24
|
+
}
|
|
25
|
+
if (Array.isArray(reasoning_details) && reasoning_details.length > 0) {
|
|
26
|
+
const joined = reasoning_details
|
|
27
|
+
.filter((d) => typeof d.text === 'string')
|
|
28
|
+
.map((d) => d.text)
|
|
29
|
+
.join('\n');
|
|
30
|
+
if (joined.length > 0) {
|
|
31
|
+
return { text: joined, reasonedOnly: true, finishReason, usage };
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return { text: '', reasonedOnly: false, finishReason, usage };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* If the extracted response is reasoning-only and was cut off by
|
|
38
|
+
* `max_tokens`, return a structured INVALID_INPUT suggesting the caller
|
|
39
|
+
* raise the budget. Otherwise return `null` (let the caller format the
|
|
40
|
+
* success response).
|
|
41
|
+
*/
|
|
42
|
+
export function detectReasoningCutoff(extracted) {
|
|
43
|
+
if (extracted.reasonedOnly && extracted.finishReason === 'length') {
|
|
44
|
+
return toolError(ErrorCode.INVALID_INPUT, 'Model exhausted max_tokens during internal reasoning without emitting a final answer. ' +
|
|
45
|
+
'Raise max_tokens or choose a non-reasoning model.', {
|
|
46
|
+
finish_reason: extracted.finishReason,
|
|
47
|
+
reasoning_preview: extracted.text.slice(0, 200),
|
|
48
|
+
usage: extracted.usage
|
|
49
|
+
? {
|
|
50
|
+
prompt_tokens: extracted.usage.prompt_tokens,
|
|
51
|
+
completion_tokens: extracted.usage.completion_tokens,
|
|
52
|
+
total_tokens: extracted.usage.total_tokens,
|
|
53
|
+
}
|
|
54
|
+
: undefined,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
export function toUsageMeta(usage) {
|
|
60
|
+
if (!usage)
|
|
61
|
+
return undefined;
|
|
62
|
+
return {
|
|
63
|
+
usage: {
|
|
64
|
+
prompt_tokens: usage.prompt_tokens,
|
|
65
|
+
completion_tokens: usage.completion_tokens,
|
|
66
|
+
total_tokens: usage.total_tokens,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|