@stabgan/openrouter-mcp-multimodal 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -145
- package/dist/errors.d.ts +42 -0
- package/dist/errors.js +46 -0
- package/dist/index.js +1 -1
- package/dist/logger.d.ts +22 -0
- package/dist/logger.js +47 -0
- package/dist/model-cache.d.ts +10 -0
- package/dist/model-cache.js +31 -1
- package/dist/openrouter-api.d.ts +54 -0
- package/dist/openrouter-api.js +136 -12
- package/dist/tool-handlers/analyze-audio.d.ts +5 -9
- package/dist/tool-handlers/analyze-audio.js +41 -8
- package/dist/tool-handlers/analyze-image.d.ts +5 -9
- package/dist/tool-handlers/analyze-image.js +38 -8
- package/dist/tool-handlers/analyze-video.d.ts +19 -0
- package/dist/tool-handlers/analyze-video.js +93 -0
- package/dist/tool-handlers/audio-utils.js +7 -9
- package/dist/tool-handlers/chat-completion.d.ts +6 -10
- package/dist/tool-handlers/chat-completion.js +27 -7
- package/dist/tool-handlers/completion-utils.d.ts +27 -0
- package/dist/tool-handlers/completion-utils.js +69 -0
- package/dist/tool-handlers/fetch-utils.d.ts +32 -0
- package/dist/tool-handlers/fetch-utils.js +216 -12
- package/dist/tool-handlers/generate-audio.d.ts +32 -12
- package/dist/tool-handlers/generate-audio.js +77 -46
- package/dist/tool-handlers/generate-image.d.ts +48 -10
- package/dist/tool-handlers/generate-image.js +148 -33
- package/dist/tool-handlers/generate-video.d.ts +78 -0
- package/dist/tool-handlers/generate-video.js +353 -0
- package/dist/tool-handlers/get-model-info.d.ts +2 -9
- package/dist/tool-handlers/get-model-info.js +15 -5
- package/dist/tool-handlers/image-utils.d.ts +17 -1
- package/dist/tool-handlers/image-utils.js +66 -13
- package/dist/tool-handlers/openrouter-errors.d.ts +18 -0
- package/dist/tool-handlers/openrouter-errors.js +99 -0
- package/dist/tool-handlers/path-safety.d.ts +11 -0
- package/dist/tool-handlers/path-safety.js +88 -0
- package/dist/tool-handlers/search-models.d.ts +2 -9
- package/dist/tool-handlers/search-models.js +10 -6
- package/dist/tool-handlers/validate-model.d.ts +2 -9
- package/dist/tool-handlers/validate-model.js +15 -4
- package/dist/tool-handlers/video-utils.d.ts +29 -0
- package/dist/tool-handlers/video-utils.js +174 -0
- package/dist/tool-handlers.js +229 -21
- package/package.json +3 -3
- package/dist/__tests__/audio-utils.test.d.ts +0 -1
- package/dist/__tests__/audio-utils.test.js +0 -120
- package/dist/__tests__/fetch-utils.test.d.ts +0 -1
- package/dist/__tests__/fetch-utils.test.js +0 -76
- package/dist/__tests__/generate-audio.test.d.ts +0 -1
- package/dist/__tests__/generate-audio.test.js +0 -90
- package/dist/__tests__/image-utils.test.d.ts +0 -1
- package/dist/__tests__/image-utils.test.js +0 -75
- package/dist/__tests__/integration.test.d.ts +0 -1
- package/dist/__tests__/integration.test.js +0 -219
- package/dist/__tests__/model-cache.test.d.ts +0 -1
- package/dist/__tests__/model-cache.test.js +0 -96
package/dist/model-cache.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ export interface OpenRouterModelRecord {
|
|
|
3
3
|
name?: string;
|
|
4
4
|
architecture?: {
|
|
5
5
|
input_modalities?: string[];
|
|
6
|
+
output_modalities?: string[];
|
|
6
7
|
};
|
|
7
8
|
context_length?: number;
|
|
8
9
|
[key: string]: unknown;
|
|
@@ -11,9 +12,16 @@ export declare class ModelCache {
|
|
|
11
12
|
private static instance;
|
|
12
13
|
private models;
|
|
13
14
|
private fetchedAt;
|
|
15
|
+
private inflight;
|
|
14
16
|
static getInstance(): ModelCache;
|
|
15
17
|
isValid(): boolean;
|
|
16
18
|
setModels(models: OpenRouterModelRecord[]): void;
|
|
19
|
+
/**
|
|
20
|
+
* Populate the cache using `fetcher` if stale, coalescing concurrent callers
|
|
21
|
+
* so only one request hits the upstream API per stale window. Callers that
|
|
22
|
+
* arrive while a populate is in flight await the same promise.
|
|
23
|
+
*/
|
|
24
|
+
ensureFresh(fetcher: () => Promise<OpenRouterModelRecord[]>): Promise<void>;
|
|
17
25
|
getAll(): OpenRouterModelRecord[];
|
|
18
26
|
get(id: string): OpenRouterModelRecord | null;
|
|
19
27
|
has(id: string): boolean;
|
|
@@ -22,6 +30,8 @@ export declare class ModelCache {
|
|
|
22
30
|
provider?: string;
|
|
23
31
|
capabilities?: {
|
|
24
32
|
vision?: boolean;
|
|
33
|
+
audio?: boolean;
|
|
34
|
+
video?: boolean;
|
|
25
35
|
};
|
|
26
36
|
limit?: number;
|
|
27
37
|
}): OpenRouterModelRecord[];
|
package/dist/model-cache.js
CHANGED
|
@@ -5,10 +5,12 @@ function getCacheTtlMs() {
|
|
|
5
5
|
const n = parseInt(raw, 10);
|
|
6
6
|
return Number.isFinite(n) && n > 0 ? n : 3600000;
|
|
7
7
|
}
|
|
8
|
+
const MAX_SEARCH_LIMIT = 50;
|
|
8
9
|
export class ModelCache {
|
|
9
10
|
static instance;
|
|
10
11
|
models = {};
|
|
11
12
|
fetchedAt = 0;
|
|
13
|
+
inflight = null;
|
|
12
14
|
static getInstance() {
|
|
13
15
|
return (ModelCache.instance ??= new ModelCache());
|
|
14
16
|
}
|
|
@@ -19,6 +21,27 @@ export class ModelCache {
|
|
|
19
21
|
this.models = Object.fromEntries(models.map((m) => [m.id, m]));
|
|
20
22
|
this.fetchedAt = Date.now();
|
|
21
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Populate the cache using `fetcher` if stale, coalescing concurrent callers
|
|
26
|
+
* so only one request hits the upstream API per stale window. Callers that
|
|
27
|
+
* arrive while a populate is in flight await the same promise.
|
|
28
|
+
*/
|
|
29
|
+
async ensureFresh(fetcher) {
|
|
30
|
+
if (this.isValid())
|
|
31
|
+
return;
|
|
32
|
+
if (this.inflight) {
|
|
33
|
+
await this.inflight;
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
this.inflight = (async () => fetcher())();
|
|
37
|
+
try {
|
|
38
|
+
const models = await this.inflight;
|
|
39
|
+
this.setModels(models);
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
this.inflight = null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
22
45
|
getAll() {
|
|
23
46
|
return Object.values(this.models);
|
|
24
47
|
}
|
|
@@ -41,6 +64,13 @@ export class ModelCache {
|
|
|
41
64
|
if (params.capabilities?.vision) {
|
|
42
65
|
results = results.filter((m) => m.architecture?.input_modalities?.includes('image'));
|
|
43
66
|
}
|
|
44
|
-
|
|
67
|
+
if (params.capabilities?.audio) {
|
|
68
|
+
results = results.filter((m) => m.architecture?.input_modalities?.includes('audio'));
|
|
69
|
+
}
|
|
70
|
+
if (params.capabilities?.video) {
|
|
71
|
+
results = results.filter((m) => m.architecture?.input_modalities?.includes('video'));
|
|
72
|
+
}
|
|
73
|
+
const limit = Math.min(Math.max(1, params.limit ?? 10), MAX_SEARCH_LIMIT);
|
|
74
|
+
return results.slice(0, limit);
|
|
45
75
|
}
|
|
46
76
|
}
|
package/dist/openrouter-api.d.ts
CHANGED
|
@@ -1,6 +1,60 @@
|
|
|
1
1
|
import type { OpenRouterModelRecord } from './model-cache.js';
|
|
2
|
+
declare function parseRetryAfter(headerValue: string | null): number | null;
|
|
3
|
+
declare function backoffWithJitter(attempt: number, retryAfterMs: number | null): number;
|
|
4
|
+
/**
|
|
5
|
+
* fetch() wrapper with retries on 429 / 5xx / network error.
|
|
6
|
+
*
|
|
7
|
+
* A fresh `AbortSignal.timeout(timeoutMs)` is created per attempt so retries
|
|
8
|
+
* each get a full timeout budget. Backoff honors `Retry-After` (seconds or
|
|
9
|
+
* HTTP-date) and applies jitter to avoid thundering-herd synchronization.
|
|
10
|
+
*/
|
|
11
|
+
declare function fetchWithRetry(url: string, init: Omit<RequestInit, 'signal'>, { retries, timeoutMs }?: {
|
|
12
|
+
retries?: number;
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
}): Promise<Response>;
|
|
2
15
|
export declare class OpenRouterAPIClient {
|
|
3
16
|
private apiKey;
|
|
4
17
|
constructor(apiKey: string);
|
|
18
|
+
private authHeaders;
|
|
5
19
|
getModels(): Promise<OpenRouterModelRecord[]>;
|
|
20
|
+
/** Submit a video-generation job. Returns the `{ id, polling_url, status }` envelope. */
|
|
21
|
+
submitVideoJob(body: Record<string, unknown>): Promise<VideoJobEnvelope>;
|
|
22
|
+
/** Poll a submitted video-generation job by id. */
|
|
23
|
+
pollVideoJob(id: string): Promise<VideoJobStatus>;
|
|
24
|
+
/**
|
|
25
|
+
* Download the generated video binary. Returns `{ buffer, contentType }`.
|
|
26
|
+
* This intentionally does NOT go through our SSRF-guarded `fetchHttpResource`
|
|
27
|
+
* because the URL is always OpenRouter itself (trusted origin) — and it can
|
|
28
|
+
* return arbitrarily large bodies that the caller bounds via
|
|
29
|
+
* `OPENROUTER_VIDEO_MAX_DOWNLOAD_BYTES`.
|
|
30
|
+
*/
|
|
31
|
+
downloadVideoContent(id: string, index?: number, maxBytes?: number): Promise<{
|
|
32
|
+
buffer: Buffer;
|
|
33
|
+
contentType: string | null;
|
|
34
|
+
}>;
|
|
6
35
|
}
|
|
36
|
+
export interface VideoJobEnvelope {
|
|
37
|
+
id: string;
|
|
38
|
+
status?: VideoJobStatusName;
|
|
39
|
+
polling_url?: string;
|
|
40
|
+
[key: string]: unknown;
|
|
41
|
+
}
|
|
42
|
+
export type VideoJobStatusName = 'pending' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
43
|
+
export interface VideoJobStatus {
|
|
44
|
+
id: string;
|
|
45
|
+
status: VideoJobStatusName | string;
|
|
46
|
+
unsigned_urls?: string[];
|
|
47
|
+
error?: {
|
|
48
|
+
message?: string;
|
|
49
|
+
code?: string;
|
|
50
|
+
} | string;
|
|
51
|
+
usage?: Record<string, unknown>;
|
|
52
|
+
progress?: number;
|
|
53
|
+
[key: string]: unknown;
|
|
54
|
+
}
|
|
55
|
+
export declare const _internals: {
|
|
56
|
+
parseRetryAfter: typeof parseRetryAfter;
|
|
57
|
+
backoffWithJitter: typeof backoffWithJitter;
|
|
58
|
+
fetchWithRetry: typeof fetchWithRetry;
|
|
59
|
+
};
|
|
60
|
+
export {};
|
package/dist/openrouter-api.js
CHANGED
|
@@ -1,25 +1,63 @@
|
|
|
1
1
|
const BASE_URL = 'https://openrouter.ai/api/v1';
|
|
2
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
3
|
+
const VIDEO_TIMEOUT_MS = 60_000;
|
|
4
|
+
const MAX_BACKOFF_MS = 10_000;
|
|
2
5
|
async function sleep(ms) {
|
|
3
6
|
return new Promise((r) => setTimeout(r, ms));
|
|
4
7
|
}
|
|
5
|
-
|
|
8
|
+
function parseRetryAfter(headerValue) {
|
|
9
|
+
if (!headerValue)
|
|
10
|
+
return null;
|
|
11
|
+
const asInt = parseInt(headerValue, 10);
|
|
12
|
+
if (Number.isFinite(asInt) && asInt >= 0)
|
|
13
|
+
return asInt * 1000;
|
|
14
|
+
const asDate = Date.parse(headerValue);
|
|
15
|
+
if (Number.isFinite(asDate)) {
|
|
16
|
+
const delta = asDate - Date.now();
|
|
17
|
+
return delta > 0 ? delta : 0;
|
|
18
|
+
}
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
function backoffWithJitter(attempt, retryAfterMs) {
|
|
22
|
+
const base = 400 * (attempt + 1);
|
|
23
|
+
const target = Math.min(Math.max(base, retryAfterMs ?? 0), MAX_BACKOFF_MS);
|
|
24
|
+
const jitter = 0.5 + Math.random(); // 0.5x .. 1.5x
|
|
25
|
+
return Math.round(target * jitter);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* fetch() wrapper with retries on 429 / 5xx / network error.
|
|
29
|
+
*
|
|
30
|
+
* A fresh `AbortSignal.timeout(timeoutMs)` is created per attempt so retries
|
|
31
|
+
* each get a full timeout budget. Backoff honors `Retry-After` (seconds or
|
|
32
|
+
* HTTP-date) and applies jitter to avoid thundering-herd synchronization.
|
|
33
|
+
*/
|
|
34
|
+
async function fetchWithRetry(url, init, { retries = 2, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
6
35
|
let lastErr;
|
|
7
36
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
8
37
|
try {
|
|
9
|
-
const res = await fetch(url, init);
|
|
38
|
+
const res = await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
10
39
|
if (res.status === 429 || res.status >= 500) {
|
|
11
|
-
if (attempt < retries)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
40
|
+
if (attempt < retries) {
|
|
41
|
+
const retryAfter = parseRetryAfter(res.headers.get('retry-after'));
|
|
42
|
+
// Release the connection before retrying so undici/pool doesn't
|
|
43
|
+
// keep it open while we sleep.
|
|
44
|
+
try {
|
|
45
|
+
await res.body?.cancel();
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
/* ignore */
|
|
49
|
+
}
|
|
50
|
+
await sleep(backoffWithJitter(attempt, retryAfter));
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
return res;
|
|
16
54
|
}
|
|
17
55
|
return res;
|
|
18
56
|
}
|
|
19
57
|
catch (e) {
|
|
20
58
|
lastErr = e;
|
|
21
59
|
if (attempt < retries)
|
|
22
|
-
await sleep(
|
|
60
|
+
await sleep(backoffWithJitter(attempt, null));
|
|
23
61
|
}
|
|
24
62
|
}
|
|
25
63
|
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
|
@@ -29,14 +67,100 @@ export class OpenRouterAPIClient {
|
|
|
29
67
|
constructor(apiKey) {
|
|
30
68
|
this.apiKey = apiKey;
|
|
31
69
|
}
|
|
70
|
+
authHeaders(extra) {
|
|
71
|
+
return {
|
|
72
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
73
|
+
'HTTP-Referer': 'https://github.com/stabgan/openrouter-mcp-multimodal',
|
|
74
|
+
'X-Title': 'openrouter-mcp-multimodal',
|
|
75
|
+
...extra,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
32
78
|
async getModels() {
|
|
33
|
-
const res = await fetchWithRetry(`${BASE_URL}/models`, {
|
|
34
|
-
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
35
|
-
signal: AbortSignal.timeout(30000),
|
|
36
|
-
}, 2);
|
|
79
|
+
const res = await fetchWithRetry(`${BASE_URL}/models`, { headers: this.authHeaders() }, { retries: 2, timeoutMs: DEFAULT_TIMEOUT_MS });
|
|
37
80
|
if (!res.ok)
|
|
38
81
|
throw new Error(`Failed to fetch models: HTTP ${res.status}`);
|
|
39
82
|
const data = (await res.json());
|
|
40
83
|
return data.data ?? [];
|
|
41
84
|
}
|
|
85
|
+
/** Submit a video-generation job. Returns the `{ id, polling_url, status }` envelope. */
|
|
86
|
+
async submitVideoJob(body) {
|
|
87
|
+
const res = await fetchWithRetry(`${BASE_URL}/videos`, {
|
|
88
|
+
method: 'POST',
|
|
89
|
+
headers: this.authHeaders({ 'Content-Type': 'application/json' }),
|
|
90
|
+
body: JSON.stringify(body),
|
|
91
|
+
}, { retries: 2, timeoutMs: VIDEO_TIMEOUT_MS });
|
|
92
|
+
if (!res.ok) {
|
|
93
|
+
const detail = await safeReadText(res);
|
|
94
|
+
throw new Error(`POST /videos failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
|
|
95
|
+
}
|
|
96
|
+
return (await res.json());
|
|
97
|
+
}
|
|
98
|
+
/** Poll a submitted video-generation job by id. */
|
|
99
|
+
async pollVideoJob(id) {
|
|
100
|
+
const res = await fetchWithRetry(`${BASE_URL}/videos/${encodeURIComponent(id)}`, { headers: this.authHeaders() }, { retries: 2, timeoutMs: DEFAULT_TIMEOUT_MS });
|
|
101
|
+
if (!res.ok) {
|
|
102
|
+
const detail = await safeReadText(res);
|
|
103
|
+
throw new Error(`GET /videos/${id} failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
|
|
104
|
+
}
|
|
105
|
+
return (await res.json());
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Download the generated video binary. Returns `{ buffer, contentType }`.
|
|
109
|
+
* This intentionally does NOT go through our SSRF-guarded `fetchHttpResource`
|
|
110
|
+
* because the URL is always OpenRouter itself (trusted origin) — and it can
|
|
111
|
+
* return arbitrarily large bodies that the caller bounds via
|
|
112
|
+
* `OPENROUTER_VIDEO_MAX_DOWNLOAD_BYTES`.
|
|
113
|
+
*/
|
|
114
|
+
async downloadVideoContent(id, index = 0, maxBytes = 256 * 1024 * 1024) {
|
|
115
|
+
const url = `${BASE_URL}/videos/${encodeURIComponent(id)}/content?index=${index}`;
|
|
116
|
+
const res = await fetchWithRetry(url, { headers: this.authHeaders() }, { retries: 1, timeoutMs: VIDEO_TIMEOUT_MS * 2 });
|
|
117
|
+
if (!res.ok) {
|
|
118
|
+
const detail = await safeReadText(res);
|
|
119
|
+
throw new Error(`GET /videos/${id}/content failed: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`);
|
|
120
|
+
}
|
|
121
|
+
const declared = res.headers.get('content-length');
|
|
122
|
+
if (declared) {
|
|
123
|
+
const n = parseInt(declared, 10);
|
|
124
|
+
if (Number.isFinite(n) && n > maxBytes) {
|
|
125
|
+
throw new Error(`Generated video too large: ${n} bytes > ${maxBytes}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const reader = res.body?.getReader();
|
|
129
|
+
if (!reader) {
|
|
130
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
131
|
+
if (buf.length > maxBytes)
|
|
132
|
+
throw new Error('Generated video too large');
|
|
133
|
+
return { buffer: buf, contentType: res.headers.get('content-type') };
|
|
134
|
+
}
|
|
135
|
+
const chunks = [];
|
|
136
|
+
let total = 0;
|
|
137
|
+
for (;;) {
|
|
138
|
+
const { done, value } = await reader.read();
|
|
139
|
+
if (done)
|
|
140
|
+
break;
|
|
141
|
+
total += value.byteLength;
|
|
142
|
+
if (total > maxBytes) {
|
|
143
|
+
try {
|
|
144
|
+
await reader.cancel();
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
/* ignore */
|
|
148
|
+
}
|
|
149
|
+
throw new Error('Generated video too large');
|
|
150
|
+
}
|
|
151
|
+
chunks.push(Buffer.from(value));
|
|
152
|
+
}
|
|
153
|
+
return { buffer: Buffer.concat(chunks), contentType: res.headers.get('content-type') };
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
async function safeReadText(res) {
|
|
157
|
+
try {
|
|
158
|
+
const t = await res.text();
|
|
159
|
+
return t.length > 500 ? t.slice(0, 500) + '…' : t;
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
return '';
|
|
163
|
+
}
|
|
42
164
|
}
|
|
165
|
+
// Exported for tests.
|
|
166
|
+
export const _internals = { parseRetryAfter, backoffWithJitter, fetchWithRetry };
|
|
@@ -8,16 +8,12 @@ export declare function handleAnalyzeAudio(request: {
|
|
|
8
8
|
params: {
|
|
9
9
|
arguments: AnalyzeAudioToolRequest;
|
|
10
10
|
};
|
|
11
|
-
}, openai: OpenAI, defaultModel?: string): Promise<{
|
|
11
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
12
12
|
content: {
|
|
13
|
-
type:
|
|
13
|
+
type: "text";
|
|
14
14
|
text: string;
|
|
15
15
|
}[];
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
type: string;
|
|
20
|
-
text: string;
|
|
21
|
-
}[];
|
|
22
|
-
isError?: undefined;
|
|
16
|
+
_meta: {
|
|
17
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
18
|
+
};
|
|
23
19
|
}>;
|
|
@@ -1,13 +1,32 @@
|
|
|
1
1
|
import { prepareAudioData } from './audio-utils.js';
|
|
2
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
3
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
4
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
2
5
|
const DEFAULT_MODEL = 'google/gemini-2.5-flash';
|
|
3
6
|
export async function handleAnalyzeAudio(request, openai, defaultModel) {
|
|
4
|
-
const { audio_path, question, model } = request.params.arguments;
|
|
7
|
+
const { audio_path, question, model } = request.params.arguments ?? { audio_path: '' };
|
|
5
8
|
if (!audio_path) {
|
|
6
|
-
return
|
|
9
|
+
return toolError(ErrorCode.INVALID_INPUT, 'audio_path is required.');
|
|
7
10
|
}
|
|
11
|
+
let audioData;
|
|
8
12
|
try {
|
|
9
|
-
|
|
10
|
-
|
|
13
|
+
audioData = await prepareAudioData(audio_path);
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
17
|
+
if (msg.includes('Blocked host'))
|
|
18
|
+
return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
|
|
19
|
+
if (msg.toLowerCase().includes('too large')) {
|
|
20
|
+
return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
|
|
21
|
+
}
|
|
22
|
+
if (msg.toLowerCase().includes('unsupported')) {
|
|
23
|
+
return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err);
|
|
24
|
+
}
|
|
25
|
+
return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
|
|
26
|
+
}
|
|
27
|
+
let completion;
|
|
28
|
+
try {
|
|
29
|
+
completion = await openai.chat.completions.create({
|
|
11
30
|
model: model || defaultModel || DEFAULT_MODEL,
|
|
12
31
|
messages: [
|
|
13
32
|
{
|
|
@@ -25,10 +44,24 @@ export async function handleAnalyzeAudio(request, openai, defaultModel) {
|
|
|
25
44
|
},
|
|
26
45
|
],
|
|
27
46
|
});
|
|
28
|
-
return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
|
|
29
47
|
}
|
|
30
|
-
catch (
|
|
31
|
-
|
|
32
|
-
|
|
48
|
+
catch (err) {
|
|
49
|
+
return classifyUpstreamError(err);
|
|
50
|
+
}
|
|
51
|
+
const extracted = extractCompletionText(completion);
|
|
52
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
53
|
+
if (cutoff)
|
|
54
|
+
return cutoff;
|
|
55
|
+
if (!extracted.text) {
|
|
56
|
+
return toolError(ErrorCode.INTERNAL, 'Audio model returned no textual content.', {
|
|
57
|
+
finish_reason: extracted.finishReason,
|
|
58
|
+
});
|
|
33
59
|
}
|
|
60
|
+
return {
|
|
61
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
62
|
+
_meta: {
|
|
63
|
+
finish_reason: extracted.finishReason,
|
|
64
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
65
|
+
},
|
|
66
|
+
};
|
|
34
67
|
}
|
|
@@ -8,16 +8,12 @@ export declare function handleAnalyzeImage(request: {
|
|
|
8
8
|
params: {
|
|
9
9
|
arguments: AnalyzeImageToolRequest;
|
|
10
10
|
};
|
|
11
|
-
}, openai: OpenAI, defaultModel?: string): Promise<{
|
|
11
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
12
12
|
content: {
|
|
13
|
-
type:
|
|
13
|
+
type: "text";
|
|
14
14
|
text: string;
|
|
15
15
|
}[];
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
type: string;
|
|
20
|
-
text: string;
|
|
21
|
-
}[];
|
|
22
|
-
isError?: undefined;
|
|
16
|
+
_meta: {
|
|
17
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
18
|
+
};
|
|
23
19
|
}>;
|
|
@@ -1,13 +1,29 @@
|
|
|
1
1
|
import { prepareImageUrl } from './image-utils.js';
|
|
2
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
3
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
4
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
2
5
|
const DEFAULT_MODEL = 'nvidia/nemotron-nano-12b-v2-vl:free';
|
|
3
6
|
export async function handleAnalyzeImage(request, openai, defaultModel) {
|
|
4
|
-
const { image_path, question, model } = request.params.arguments;
|
|
7
|
+
const { image_path, question, model } = request.params.arguments ?? { image_path: '' };
|
|
5
8
|
if (!image_path) {
|
|
6
|
-
return
|
|
9
|
+
return toolError(ErrorCode.INVALID_INPUT, 'image_path is required.');
|
|
7
10
|
}
|
|
11
|
+
let imageUrl;
|
|
8
12
|
try {
|
|
9
|
-
|
|
10
|
-
|
|
13
|
+
imageUrl = await prepareImageUrl(image_path);
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
17
|
+
if (msg.includes('Blocked host'))
|
|
18
|
+
return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
|
|
19
|
+
if (msg.toLowerCase().includes('too large')) {
|
|
20
|
+
return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
|
|
21
|
+
}
|
|
22
|
+
return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
|
|
23
|
+
}
|
|
24
|
+
let completion;
|
|
25
|
+
try {
|
|
26
|
+
completion = await openai.chat.completions.create({
|
|
11
27
|
model: model || defaultModel || DEFAULT_MODEL,
|
|
12
28
|
messages: [
|
|
13
29
|
{
|
|
@@ -19,10 +35,24 @@ export async function handleAnalyzeImage(request, openai, defaultModel) {
|
|
|
19
35
|
},
|
|
20
36
|
],
|
|
21
37
|
});
|
|
22
|
-
return { content: [{ type: 'text', text: completion.choices[0].message.content || '' }] };
|
|
23
38
|
}
|
|
24
|
-
catch (
|
|
25
|
-
|
|
26
|
-
|
|
39
|
+
catch (err) {
|
|
40
|
+
return classifyUpstreamError(err);
|
|
41
|
+
}
|
|
42
|
+
const extracted = extractCompletionText(completion);
|
|
43
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
44
|
+
if (cutoff)
|
|
45
|
+
return cutoff;
|
|
46
|
+
if (!extracted.text) {
|
|
47
|
+
return toolError(ErrorCode.INTERNAL, 'Vision model returned no textual content.', {
|
|
48
|
+
finish_reason: extracted.finishReason,
|
|
49
|
+
});
|
|
27
50
|
}
|
|
51
|
+
return {
|
|
52
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
53
|
+
_meta: {
|
|
54
|
+
finish_reason: extracted.finishReason,
|
|
55
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
56
|
+
},
|
|
57
|
+
};
|
|
28
58
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
export interface AnalyzeVideoToolRequest {
|
|
3
|
+
video_path: string;
|
|
4
|
+
question?: string;
|
|
5
|
+
model?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function handleAnalyzeVideo(request: {
|
|
8
|
+
params: {
|
|
9
|
+
arguments: AnalyzeVideoToolRequest;
|
|
10
|
+
};
|
|
11
|
+
}, openai: OpenAI, defaultModel?: string): Promise<import("../errors.js").ToolErrorResult | {
|
|
12
|
+
content: {
|
|
13
|
+
type: "text";
|
|
14
|
+
text: string;
|
|
15
|
+
}[];
|
|
16
|
+
_meta: {
|
|
17
|
+
finish_reason: "length" | "stop" | "tool_calls" | "content_filter" | "function_call" | undefined;
|
|
18
|
+
};
|
|
19
|
+
}>;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { prepareVideoData } from './video-utils.js';
|
|
2
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
3
|
+
import { logger } from '../logger.js';
|
|
4
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
5
|
+
import { extractCompletionText, detectReasoningCutoff, toUsageMeta, } from './completion-utils.js';
|
|
6
|
+
/**
|
|
7
|
+
* Default model — `google/gemini-2.5-flash` has the widest video-input
|
|
8
|
+
* support on OpenRouter at time of writing. Override via env
|
|
9
|
+
* `OPENROUTER_DEFAULT_VIDEO_MODEL` or per-call `model`.
|
|
10
|
+
*/
|
|
11
|
+
const FALLBACK_DEFAULT_MODEL = 'google/gemini-2.5-flash';
|
|
12
|
+
export async function handleAnalyzeVideo(request, openai, defaultModel) {
|
|
13
|
+
const { video_path, question, model } = request.params.arguments ?? {
|
|
14
|
+
video_path: '',
|
|
15
|
+
};
|
|
16
|
+
if (!video_path) {
|
|
17
|
+
return toolError(ErrorCode.INVALID_INPUT, 'video_path is required.');
|
|
18
|
+
}
|
|
19
|
+
const pickedModel = model ||
|
|
20
|
+
process.env.OPENROUTER_DEFAULT_VIDEO_MODEL ||
|
|
21
|
+
defaultModel ||
|
|
22
|
+
FALLBACK_DEFAULT_MODEL;
|
|
23
|
+
let videoData;
|
|
24
|
+
try {
|
|
25
|
+
videoData = await prepareVideoData(video_path);
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
29
|
+
if (msg.includes('Blocked host')) {
|
|
30
|
+
return toolErrorFrom(ErrorCode.UPSTREAM_REFUSED, err);
|
|
31
|
+
}
|
|
32
|
+
if (msg.includes('too large')) {
|
|
33
|
+
return toolErrorFrom(ErrorCode.RESOURCE_TOO_LARGE, err);
|
|
34
|
+
}
|
|
35
|
+
if (msg.includes('Unsupported') || msg.includes('not a video')) {
|
|
36
|
+
return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err);
|
|
37
|
+
}
|
|
38
|
+
return toolErrorFrom(ErrorCode.INVALID_INPUT, err);
|
|
39
|
+
}
|
|
40
|
+
let completion;
|
|
41
|
+
try {
|
|
42
|
+
logger.debug('analyze_video.submit', {
|
|
43
|
+
model: pickedModel,
|
|
44
|
+
format: videoData.format,
|
|
45
|
+
size_bytes: videoData.sizeBytes,
|
|
46
|
+
});
|
|
47
|
+
completion = await openai.chat.completions.create({
|
|
48
|
+
model: pickedModel,
|
|
49
|
+
messages: [
|
|
50
|
+
{
|
|
51
|
+
role: 'user',
|
|
52
|
+
content: [
|
|
53
|
+
{
|
|
54
|
+
type: 'text',
|
|
55
|
+
text: question || 'Describe what happens in this video, step by step.',
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
// The `video_url` content type is an OpenRouter extension; the
|
|
59
|
+
// OpenAI SDK's typings don't know about it yet. See:
|
|
60
|
+
// https://openrouter.ai/docs/guides/overview/multimodal/videos
|
|
61
|
+
type: 'video_url',
|
|
62
|
+
video_url: {
|
|
63
|
+
url: `data:${videoData.mediaType};base64,${videoData.data}`,
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
],
|
|
67
|
+
},
|
|
68
|
+
],
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
logger.warn('analyze_video.error', {
|
|
73
|
+
err: err instanceof Error ? err.message : String(err),
|
|
74
|
+
});
|
|
75
|
+
return classifyUpstreamError(err);
|
|
76
|
+
}
|
|
77
|
+
const extracted = extractCompletionText(completion);
|
|
78
|
+
const cutoff = detectReasoningCutoff(extracted);
|
|
79
|
+
if (cutoff)
|
|
80
|
+
return cutoff;
|
|
81
|
+
if (!extracted.text) {
|
|
82
|
+
return toolError(ErrorCode.INTERNAL, 'Video model returned no textual content.', {
|
|
83
|
+
finish_reason: extracted.finishReason,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
content: [{ type: 'text', text: extracted.text }],
|
|
88
|
+
_meta: {
|
|
89
|
+
finish_reason: extracted.finishReason,
|
|
90
|
+
...(toUsageMeta(extracted.usage) ?? {}),
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import path from 'path';
|
|
6
6
|
import { promises as fs } from 'fs';
|
|
7
|
-
import { readEnvInt, fetchHttpResource } from './fetch-utils.js';
|
|
7
|
+
import { readEnvInt, fetchHttpResource, parseBase64DataUrl } from './fetch-utils.js';
|
|
8
8
|
// Re-export for tests
|
|
9
9
|
export { isBlockedIPv4, assertUrlSafeForFetch } from './fetch-utils.js';
|
|
10
10
|
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
|
@@ -89,19 +89,17 @@ function formatFromContentType(ct) {
|
|
|
89
89
|
export async function prepareAudioData(source) {
|
|
90
90
|
// --- data URL ---
|
|
91
91
|
if (source.startsWith('data:')) {
|
|
92
|
-
const
|
|
93
|
-
if (!
|
|
92
|
+
const parsed = parseBase64DataUrl(source);
|
|
93
|
+
if (!parsed)
|
|
94
94
|
throw new Error('Invalid data URL format');
|
|
95
|
-
const
|
|
96
|
-
const b64 = match[2];
|
|
97
|
-
const format = mimeSubtypeToFormat(mime.split('/')[1] ?? '');
|
|
95
|
+
const format = mimeSubtypeToFormat(parsed.mediaType.split('/')[1] ?? '');
|
|
98
96
|
if (!format) {
|
|
99
|
-
throw new Error(`Unsupported audio format from MIME: ${
|
|
97
|
+
throw new Error(`Unsupported audio format from MIME: ${parsed.mediaType}. Supported: ${SUPPORTED_AUDIO_FORMATS.join(', ')}`);
|
|
100
98
|
}
|
|
101
|
-
const approxBytes = Math.ceil((
|
|
99
|
+
const approxBytes = Math.ceil((parsed.base64.length * 3) / 4);
|
|
102
100
|
if (approxBytes > getMaxDataUrlBytes())
|
|
103
101
|
throw new Error('Data URL too large');
|
|
104
|
-
return { data:
|
|
102
|
+
return { data: parsed.base64, format };
|
|
105
103
|
}
|
|
106
104
|
// --- HTTP(S) URL ---
|
|
107
105
|
if (source.startsWith('http://') || source.startsWith('https://')) {
|