@stabgan/openrouter-mcp-multimodal 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -43
- package/dist/errors.d.ts +42 -0
- package/dist/errors.js +46 -0
- package/dist/index.js +1 -1
- package/dist/logger.d.ts +22 -0
- package/dist/logger.js +47 -0
- package/dist/model-cache.d.ts +10 -0
- package/dist/model-cache.js +31 -1
- package/dist/openrouter-api.d.ts +54 -0
- package/dist/openrouter-api.js +128 -12
- package/dist/tool-handlers/analyze-audio.d.ts +5 -9
- package/dist/tool-handlers/analyze-audio.js +41 -8
- package/dist/tool-handlers/analyze-image.d.ts +5 -9
- package/dist/tool-handlers/analyze-image.js +38 -8
- package/dist/tool-handlers/analyze-video.d.ts +19 -0
- package/dist/tool-handlers/analyze-video.js +93 -0
- package/dist/tool-handlers/audio-utils.js +7 -9
- package/dist/tool-handlers/chat-completion.d.ts +6 -10
- package/dist/tool-handlers/chat-completion.js +27 -7
- package/dist/tool-handlers/completion-utils.d.ts +27 -0
- package/dist/tool-handlers/completion-utils.js +69 -0
- package/dist/tool-handlers/fetch-utils.d.ts +21 -0
- package/dist/tool-handlers/fetch-utils.js +166 -11
- package/dist/tool-handlers/generate-audio.d.ts +32 -12
- package/dist/tool-handlers/generate-audio.js +77 -46
- package/dist/tool-handlers/generate-image.d.ts +26 -10
- package/dist/tool-handlers/generate-image.js +79 -27
- package/dist/tool-handlers/generate-video.d.ts +78 -0
- package/dist/tool-handlers/generate-video.js +353 -0
- package/dist/tool-handlers/get-model-info.js +8 -2
- package/dist/tool-handlers/image-utils.d.ts +17 -1
- package/dist/tool-handlers/image-utils.js +66 -13
- package/dist/tool-handlers/openrouter-errors.d.ts +18 -0
- package/dist/tool-handlers/openrouter-errors.js +99 -0
- package/dist/tool-handlers/path-safety.d.ts +11 -0
- package/dist/tool-handlers/path-safety.js +88 -0
- package/dist/tool-handlers/search-models.js +1 -3
- package/dist/tool-handlers/validate-model.js +8 -2
- package/dist/tool-handlers/video-utils.d.ts +29 -0
- package/dist/tool-handlers/video-utils.js +174 -0
- package/dist/tool-handlers.js +199 -21
- package/package.json +3 -3
- package/dist/__tests__/audio-utils.test.d.ts +0 -1
- package/dist/__tests__/audio-utils.test.js +0 -120
- package/dist/__tests__/fetch-utils.test.d.ts +0 -1
- package/dist/__tests__/fetch-utils.test.js +0 -76
- package/dist/__tests__/generate-audio.test.d.ts +0 -1
- package/dist/__tests__/generate-audio.test.js +0 -90
- package/dist/__tests__/image-utils.test.d.ts +0 -1
- package/dist/__tests__/image-utils.test.js +0 -75
- package/dist/__tests__/integration.test.d.ts +0 -1
- package/dist/__tests__/integration.test.js +0 -219
- package/dist/__tests__/model-cache.test.d.ts +0 -1
- package/dist/__tests__/model-cache.test.js +0 -96
|
@@ -1,8 +1,29 @@
|
|
|
1
1
|
export declare function readEnvInt(name: string, fallback: number, min?: number): number;
|
|
2
2
|
/** Blocks RFC1918, loopback, link-local, CGNAT, metadata. */
|
|
3
3
|
export declare function isBlockedIPv4(ip: string): boolean;
|
|
4
|
+
/**
|
|
5
|
+
* Comprehensive IPv6 SSRF block list. Covers loopback, unspecified,
|
|
6
|
+
* IPv4-mapped, IPv4-compatible, link-local, site-local, ULA, multicast,
|
|
7
|
+
* discard, documentation, Teredo, 6to4 (re-validates the embedded IPv4
|
|
8
|
+
* against `isBlockedIPv4`), and ORCHID. Returns `true` for any input that
|
|
9
|
+
* is a valid IPv6 literal in a reserved or private range.
|
|
10
|
+
*
|
|
11
|
+
* For non-IPv6 input returns `false` (the caller is expected to also run
|
|
12
|
+
* `isBlockedIPv4` for IPv4 input).
|
|
13
|
+
*/
|
|
14
|
+
export declare function isBlockedIPv6(ip: string): boolean;
|
|
4
15
|
/** Resolve hostname and ensure the resolved address is not private/link-local. */
|
|
5
16
|
export declare function assertUrlSafeForFetch(urlString: string): Promise<URL>;
|
|
17
|
+
/**
|
|
18
|
+
* Parse an RFC 2397 data URL into `{ mediaType, base64 }`. Accepts MIME
|
|
19
|
+
* parameters (`data:audio/wav;charset=binary;base64,...`) and the bare
|
|
20
|
+
* `data:;base64,...` form. Returns `null` for anything that is not a
|
|
21
|
+
* base64-encoded data URL.
|
|
22
|
+
*/
|
|
23
|
+
export declare function parseBase64DataUrl(source: string): {
|
|
24
|
+
mediaType: string;
|
|
25
|
+
base64: string;
|
|
26
|
+
} | null;
|
|
6
27
|
export interface FetchOptions {
|
|
7
28
|
timeoutMs: number;
|
|
8
29
|
maxBytes: number;
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Used by both image-utils and audio-utils to avoid duplication.
|
|
4
4
|
*/
|
|
5
5
|
import dns from 'node:dns/promises';
|
|
6
|
+
import net from 'node:net';
|
|
6
7
|
export function readEnvInt(name, fallback, min = 1) {
|
|
7
8
|
const raw = process.env[name];
|
|
8
9
|
if (raw === undefined || raw === '')
|
|
@@ -36,18 +37,136 @@ export function isBlockedIPv4(ip) {
|
|
|
36
37
|
return true;
|
|
37
38
|
return false;
|
|
38
39
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Expand an IPv6 literal to eight 16-bit groups as lowercase hex without
|
|
42
|
+
* separators. Accepts compressed forms (::), zone ids (%eth0), and IPv4-mapped
|
|
43
|
+
* / IPv4-compatible tails. Returns null if the input is not a valid IPv6
|
|
44
|
+
* literal.
|
|
45
|
+
*/
|
|
46
|
+
function expandIPv6(ip) {
|
|
47
|
+
// Strip optional brackets (URL host form) and zone id before validation.
|
|
48
|
+
const noZone = ip.includes('%') ? ip.split('%')[0] : ip;
|
|
49
|
+
const noBrackets = noZone.replace(/^\[|\]$/g, '');
|
|
50
|
+
if (!net.isIPv6(noBrackets))
|
|
51
|
+
return null;
|
|
52
|
+
let addr = noBrackets.toLowerCase();
|
|
53
|
+
// Pull out any IPv4 tail (`::ffff:a.b.c.d`, `::a.b.c.d`, `x:y::a.b.c.d`)
|
|
54
|
+
// and substitute two 16-bit zero groups in its place. This way the rest
|
|
55
|
+
// of the parser only needs to handle pure-hex 8-group form.
|
|
56
|
+
let v4Tail = null;
|
|
57
|
+
const dotIndex = addr.indexOf('.');
|
|
58
|
+
if (dotIndex >= 0) {
|
|
59
|
+
const lastColon = addr.lastIndexOf(':', dotIndex);
|
|
60
|
+
if (lastColon < 0)
|
|
61
|
+
return null;
|
|
62
|
+
const tail = addr.slice(lastColon + 1);
|
|
63
|
+
const parts = tail.split('.').map((p) => parseInt(p, 10));
|
|
64
|
+
if (parts.length !== 4 || parts.some((p) => !Number.isInteger(p) || p < 0 || p > 255)) {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
v4Tail = [((parts[0] << 8) | parts[1]) & 0xffff, ((parts[2] << 8) | parts[3]) & 0xffff];
|
|
68
|
+
// Substitute "g6:g7" in hex. E.g. "::ffff:127.0.0.1" -> "::ffff:7f00:0001"
|
|
69
|
+
const hex6 = v4Tail[0].toString(16);
|
|
70
|
+
const hex7 = v4Tail[1].toString(16);
|
|
71
|
+
addr = addr.slice(0, lastColon) + ':' + hex6 + ':' + hex7;
|
|
72
|
+
}
|
|
73
|
+
// Split on "::" at most once; fill the gap with zero groups.
|
|
74
|
+
const halves = addr.split('::');
|
|
75
|
+
if (halves.length > 2)
|
|
76
|
+
return null;
|
|
77
|
+
const left = halves[0] ? halves[0].split(':') : [];
|
|
78
|
+
const right = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
|
|
79
|
+
const missing = 8 - left.length - right.length;
|
|
80
|
+
if (halves.length === 2) {
|
|
81
|
+
if (missing < 0)
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
if (missing !== 0)
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
const zeros = Array(Math.max(0, missing)).fill('0');
|
|
89
|
+
const hexGroups = [...left, ...zeros, ...right];
|
|
90
|
+
if (hexGroups.length !== 8)
|
|
91
|
+
return null;
|
|
92
|
+
const out = [];
|
|
93
|
+
for (const g of hexGroups) {
|
|
94
|
+
if (g.length === 0 || g.length > 4 || !/^[0-9a-f]+$/.test(g))
|
|
95
|
+
return null;
|
|
96
|
+
out.push(parseInt(g, 16));
|
|
97
|
+
}
|
|
98
|
+
return out.length === 8 ? out : null;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Comprehensive IPv6 SSRF block list. Covers loopback, unspecified,
|
|
102
|
+
* IPv4-mapped, IPv4-compatible, link-local, site-local, ULA, multicast,
|
|
103
|
+
* discard, documentation, Teredo, 6to4 (re-validates the embedded IPv4
|
|
104
|
+
* against `isBlockedIPv4`), and ORCHID. Returns `true` for any input that
|
|
105
|
+
* is a valid IPv6 literal in a reserved or private range.
|
|
106
|
+
*
|
|
107
|
+
* For non-IPv6 input returns `false` (the caller is expected to also run
|
|
108
|
+
* `isBlockedIPv4` for IPv4 input).
|
|
109
|
+
*/
|
|
110
|
+
export function isBlockedIPv6(ip) {
|
|
111
|
+
const groups = expandIPv6(ip);
|
|
112
|
+
if (!groups)
|
|
113
|
+
return false;
|
|
114
|
+
const [g0, g1, g2, g3, g4, g5, g6, g7] = groups;
|
|
115
|
+
// :: (unspecified)
|
|
116
|
+
if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0 && g6 === 0 && g7 === 0) {
|
|
43
117
|
return true;
|
|
44
|
-
|
|
118
|
+
}
|
|
119
|
+
// ::1 (loopback)
|
|
120
|
+
if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0 && g6 === 0 && g7 === 1) {
|
|
45
121
|
return true;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
122
|
+
}
|
|
123
|
+
// ::ffff:0:0/96 — IPv4-mapped. Re-check the embedded IPv4.
|
|
124
|
+
if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0xffff) {
|
|
125
|
+
const v4 = ((g6 << 16) >>> 0) | g7;
|
|
126
|
+
const dotted = `${(v4 >>> 24) & 0xff}.${(v4 >>> 16) & 0xff}.${(v4 >>> 8) & 0xff}.${v4 & 0xff}`;
|
|
127
|
+
return isBlockedIPv4(dotted);
|
|
128
|
+
}
|
|
129
|
+
// ::/96 IPv4-compatible (deprecated but still routable in places).
|
|
130
|
+
if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0) {
|
|
131
|
+
// Only treat as IPv4-compat if g6/g7 actually look like an IPv4 (both
|
|
132
|
+
// are nonzero or this is the all-zeros case handled above).
|
|
133
|
+
if (g6 !== 0 || g7 !== 0) {
|
|
134
|
+
const v4 = ((g6 << 16) >>> 0) | g7;
|
|
135
|
+
const dotted = `${(v4 >>> 24) & 0xff}.${(v4 >>> 16) & 0xff}.${(v4 >>> 8) & 0xff}.${v4 & 0xff}`;
|
|
136
|
+
return isBlockedIPv4(dotted);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// fc00::/7 — ULA
|
|
140
|
+
if ((g0 & 0xfe00) === 0xfc00)
|
|
141
|
+
return true;
|
|
142
|
+
// fe80::/10 — link-local
|
|
143
|
+
if ((g0 & 0xffc0) === 0xfe80)
|
|
144
|
+
return true;
|
|
145
|
+
// fec0::/10 — deprecated site-local
|
|
146
|
+
if ((g0 & 0xffc0) === 0xfec0)
|
|
147
|
+
return true;
|
|
148
|
+
// ff00::/8 — multicast (all forms)
|
|
149
|
+
if ((g0 & 0xff00) === 0xff00)
|
|
150
|
+
return true;
|
|
151
|
+
// 100::/64 — discard prefix (RFC 6666)
|
|
152
|
+
if (g0 === 0x0100 && g1 === 0 && g2 === 0 && g3 === 0)
|
|
153
|
+
return true;
|
|
154
|
+
// 2001:db8::/32 — documentation
|
|
155
|
+
if (g0 === 0x2001 && g1 === 0x0db8)
|
|
156
|
+
return true;
|
|
157
|
+
// 2001::/32 — Teredo
|
|
158
|
+
if (g0 === 0x2001 && g1 === 0x0000)
|
|
159
|
+
return true;
|
|
160
|
+
// 2001:10::/28, 2001:20::/28 — ORCHID / deprecated
|
|
161
|
+
if (g0 === 0x2001 && (g1 & 0xfff0) === 0x0010)
|
|
162
|
+
return true;
|
|
163
|
+
if (g0 === 0x2001 && (g1 & 0xfff0) === 0x0020)
|
|
164
|
+
return true;
|
|
165
|
+
// 2002::/16 — 6to4; re-check embedded IPv4 for private/reserved use.
|
|
166
|
+
if (g0 === 0x2002) {
|
|
167
|
+
const v4 = ((g1 << 16) >>> 0) | g2;
|
|
168
|
+
const dotted = `${(v4 >>> 24) & 0xff}.${(v4 >>> 16) & 0xff}.${(v4 >>> 8) & 0xff}.${v4 & 0xff}`;
|
|
169
|
+
return isBlockedIPv4(dotted);
|
|
51
170
|
}
|
|
52
171
|
return false;
|
|
53
172
|
}
|
|
@@ -107,6 +226,13 @@ export async function assertUrlSafeForFetch(urlString) {
|
|
|
107
226
|
return url;
|
|
108
227
|
}
|
|
109
228
|
async function readResponseBodyWithLimit(res, maxBytes) {
|
|
229
|
+
const declared = res.headers.get('content-length');
|
|
230
|
+
if (declared) {
|
|
231
|
+
const n = parseInt(declared, 10);
|
|
232
|
+
if (Number.isFinite(n) && n > maxBytes) {
|
|
233
|
+
throw new Error('Response too large');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
110
236
|
const reader = res.body?.getReader();
|
|
111
237
|
if (!reader) {
|
|
112
238
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
@@ -121,12 +247,41 @@ async function readResponseBodyWithLimit(res, maxBytes) {
|
|
|
121
247
|
if (done)
|
|
122
248
|
break;
|
|
123
249
|
total += value.byteLength;
|
|
124
|
-
if (total > maxBytes)
|
|
250
|
+
if (total > maxBytes) {
|
|
251
|
+
// Cancel the underlying body so the server connection can be released.
|
|
252
|
+
try {
|
|
253
|
+
await reader.cancel();
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
/* ignore */
|
|
257
|
+
}
|
|
125
258
|
throw new Error('Response too large');
|
|
259
|
+
}
|
|
126
260
|
chunks.push(Buffer.from(value));
|
|
127
261
|
}
|
|
128
262
|
return Buffer.concat(chunks);
|
|
129
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* Parse an RFC 2397 data URL into `{ mediaType, base64 }`. Accepts MIME
|
|
266
|
+
* parameters (`data:audio/wav;charset=binary;base64,...`) and the bare
|
|
267
|
+
* `data:;base64,...` form. Returns `null` for anything that is not a
|
|
268
|
+
* base64-encoded data URL.
|
|
269
|
+
*/
|
|
270
|
+
export function parseBase64DataUrl(source) {
|
|
271
|
+
if (!source.startsWith('data:'))
|
|
272
|
+
return null;
|
|
273
|
+
const comma = source.indexOf(',');
|
|
274
|
+
if (comma < 0)
|
|
275
|
+
return null;
|
|
276
|
+
const prefix = source.slice(5, comma); // between "data:" and ","
|
|
277
|
+
const payload = source.slice(comma + 1);
|
|
278
|
+
const parts = prefix.split(';').map((p) => p.trim());
|
|
279
|
+
const hasBase64 = parts[parts.length - 1]?.toLowerCase() === 'base64';
|
|
280
|
+
if (!hasBase64)
|
|
281
|
+
return null;
|
|
282
|
+
const mediaType = (parts[0] && parts[0].includes('/') ? parts[0] : 'application/octet-stream').toLowerCase();
|
|
283
|
+
return { mediaType, base64: payload };
|
|
284
|
+
}
|
|
130
285
|
/**
|
|
131
286
|
* Fetch a remote HTTP(S) resource with SSRF protection, size limits,
|
|
132
287
|
* redirect cap, and timeout. Returns body Buffer + Content-Type header.
|
|
@@ -6,40 +6,60 @@ export interface GenerateAudioToolRequest {
|
|
|
6
6
|
format?: string;
|
|
7
7
|
save_path?: string;
|
|
8
8
|
}
|
|
9
|
-
/** Create a 44-byte WAV header for raw PCM16 data. */
|
|
9
|
+
/** Create a 44-byte WAV header for raw PCM16 data at `sampleRate` Hz. */
|
|
10
10
|
export declare function createWavHeader(dataLength: number, sampleRate?: number): Buffer;
|
|
11
11
|
/**
|
|
12
|
-
* Detect audio container format from magic bytes.
|
|
13
|
-
*
|
|
12
|
+
* Detect audio container format from magic bytes. Uses `Buffer.subarray()`
|
|
13
|
+
* (not deprecated `slice()`). MP3 detection is intentionally strict:
|
|
14
|
+
* - Accept ID3v2 tags (`'ID3'`) as unambiguous MP3.
|
|
15
|
+
* - Accept raw frame sync only when every MPEG header field falls in a
|
|
16
|
+
* non-reserved range: version != 0b01, layer != 0b00, bitrate != 0b1111,
|
|
17
|
+
* sample rate index != 0b11. This removes the false positives that a
|
|
18
|
+
* sync-word-only check produces on random binary.
|
|
14
19
|
*/
|
|
15
20
|
export declare function detectAudioFormat(data: Buffer): {
|
|
16
21
|
ext: string;
|
|
17
22
|
mimeType: string;
|
|
18
23
|
};
|
|
19
|
-
export declare function wrapPcmInWav(pcmData: Buffer): Buffer;
|
|
24
|
+
export declare function wrapPcmInWav(pcmData: Buffer, sampleRate?: number): Buffer;
|
|
20
25
|
/** Strip existing extension (if any) and append a new one. */
|
|
21
26
|
export declare function replaceExtension(filePath: string, newExt: string): string;
|
|
22
27
|
export declare function handleGenerateAudio(request: {
|
|
23
28
|
params: {
|
|
24
29
|
arguments: GenerateAudioToolRequest;
|
|
25
30
|
};
|
|
26
|
-
}, openai: OpenAI): Promise<{
|
|
27
|
-
content: {
|
|
28
|
-
type:
|
|
31
|
+
}, openai: OpenAI): Promise<import("../errors.js").ToolErrorResult | {
|
|
32
|
+
content: ({
|
|
33
|
+
type: "text";
|
|
29
34
|
text: string;
|
|
30
|
-
|
|
31
|
-
|
|
35
|
+
mimeType?: undefined;
|
|
36
|
+
data?: undefined;
|
|
37
|
+
} | {
|
|
38
|
+
type: "audio";
|
|
39
|
+
mimeType: string;
|
|
40
|
+
data: string;
|
|
41
|
+
text?: undefined;
|
|
42
|
+
})[];
|
|
43
|
+
_meta: {
|
|
44
|
+
save_path: string;
|
|
45
|
+
mime: string;
|
|
46
|
+
size_bytes: number;
|
|
47
|
+
};
|
|
32
48
|
} | {
|
|
33
49
|
content: ({
|
|
34
|
-
type:
|
|
50
|
+
type: "text";
|
|
35
51
|
text: string;
|
|
36
52
|
mimeType?: undefined;
|
|
37
53
|
data?: undefined;
|
|
38
54
|
} | {
|
|
39
|
-
type:
|
|
55
|
+
type: "audio";
|
|
40
56
|
mimeType: string;
|
|
41
57
|
data: string;
|
|
42
58
|
text?: undefined;
|
|
43
59
|
})[];
|
|
44
|
-
|
|
60
|
+
_meta: {
|
|
61
|
+
mime: string;
|
|
62
|
+
size_bytes: number;
|
|
63
|
+
save_path?: undefined;
|
|
64
|
+
};
|
|
45
65
|
}>;
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import { promises as fs } from 'fs';
|
|
2
|
-
import {
|
|
2
|
+
import { extname } from 'path';
|
|
3
|
+
import { resolveSafeOutputPath, UnsafeOutputPathError } from './path-safety.js';
|
|
4
|
+
import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
|
|
5
|
+
import { classifyUpstreamError } from './openrouter-errors.js';
|
|
3
6
|
const DEFAULT_MODEL = 'openai/gpt-audio';
|
|
4
7
|
const DEFAULT_VOICE = 'alloy';
|
|
5
8
|
const DEFAULT_FORMAT = 'pcm16';
|
|
6
9
|
const VALID_FORMATS = ['wav', 'mp3', 'flac', 'opus', 'pcm16'];
|
|
7
|
-
const
|
|
10
|
+
const DEFAULT_PCM_SAMPLE_RATE = 24000;
|
|
8
11
|
const PCM_BITS_PER_SAMPLE = 16;
|
|
9
12
|
const PCM_NUM_CHANNELS = 1;
|
|
10
|
-
/** Create a 44-byte WAV header for raw PCM16 data. */
|
|
11
|
-
export function createWavHeader(dataLength, sampleRate =
|
|
13
|
+
/** Create a 44-byte WAV header for raw PCM16 data at `sampleRate` Hz. */
|
|
14
|
+
export function createWavHeader(dataLength, sampleRate = DEFAULT_PCM_SAMPLE_RATE) {
|
|
12
15
|
const header = Buffer.alloc(44);
|
|
13
16
|
const byteRate = sampleRate * PCM_NUM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8);
|
|
14
17
|
const blockAlign = PCM_NUM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8);
|
|
@@ -28,20 +31,31 @@ export function createWavHeader(dataLength, sampleRate = PCM_SAMPLE_RATE) {
|
|
|
28
31
|
return header;
|
|
29
32
|
}
|
|
30
33
|
/**
|
|
31
|
-
* Detect audio container format from magic bytes.
|
|
32
|
-
*
|
|
34
|
+
* Detect audio container format from magic bytes. Uses `Buffer.subarray()`
|
|
35
|
+
* (not deprecated `slice()`). MP3 detection is intentionally strict:
|
|
36
|
+
* - Accept ID3v2 tags (`'ID3'`) as unambiguous MP3.
|
|
37
|
+
* - Accept raw frame sync only when every MPEG header field falls in a
|
|
38
|
+
* non-reserved range: version != 0b01, layer != 0b00, bitrate != 0b1111,
|
|
39
|
+
* sample rate index != 0b11. This removes the false positives that a
|
|
40
|
+
* sync-word-only check produces on random binary.
|
|
33
41
|
*/
|
|
34
42
|
export function detectAudioFormat(data) {
|
|
35
|
-
if (data.length >= 3) {
|
|
36
|
-
|
|
43
|
+
if (data.length >= 3 && data[0] === 0x49 && data[1] === 0x44 && data[2] === 0x33) {
|
|
44
|
+
return { ext: 'mp3', mimeType: 'audio/mpeg' };
|
|
45
|
+
}
|
|
46
|
+
if (data.length >= 4 && data[0] === 0xff && (data[1] & 0xe0) === 0xe0) {
|
|
47
|
+
const b1 = data[1];
|
|
48
|
+
const b2 = data[2];
|
|
49
|
+
const versionBits = (b1 >> 3) & 0x03; // 01 = reserved
|
|
50
|
+
const layerBits = (b1 >> 1) & 0x03; // 00 = reserved
|
|
51
|
+
const bitrateIndex = (b2 >> 4) & 0x0f; // 1111 = bad
|
|
52
|
+
const sampleRateIndex = (b2 >> 2) & 0x03; // 11 = reserved
|
|
53
|
+
if (versionBits !== 0x01 &&
|
|
54
|
+
layerBits !== 0x00 &&
|
|
55
|
+
bitrateIndex !== 0x0f &&
|
|
56
|
+
sampleRateIndex !== 0x03) {
|
|
37
57
|
return { ext: 'mp3', mimeType: 'audio/mpeg' };
|
|
38
58
|
}
|
|
39
|
-
if (data[0] === 0xff && (data[1] & 0xe0) === 0xe0) {
|
|
40
|
-
const versionBits = (data[1] >> 3) & 0x03;
|
|
41
|
-
if (versionBits !== 0x01) {
|
|
42
|
-
return { ext: 'mp3', mimeType: 'audio/mpeg' };
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
59
|
}
|
|
46
60
|
if (data.length >= 12) {
|
|
47
61
|
const riff = data.subarray(0, 4).toString('ascii');
|
|
@@ -59,8 +73,8 @@ export function detectAudioFormat(data) {
|
|
|
59
73
|
}
|
|
60
74
|
return { ext: 'pcm', mimeType: 'audio/pcm' };
|
|
61
75
|
}
|
|
62
|
-
export function wrapPcmInWav(pcmData) {
|
|
63
|
-
return Buffer.concat([createWavHeader(pcmData.length), pcmData]);
|
|
76
|
+
export function wrapPcmInWav(pcmData, sampleRate = DEFAULT_PCM_SAMPLE_RATE) {
|
|
77
|
+
return Buffer.concat([createWavHeader(pcmData.length, sampleRate), pcmData]);
|
|
64
78
|
}
|
|
65
79
|
/** Strip existing extension (if any) and append a new one. */
|
|
66
80
|
export function replaceExtension(filePath, newExt) {
|
|
@@ -69,39 +83,62 @@ export function replaceExtension(filePath, newExt) {
|
|
|
69
83
|
return `${base}.${newExt}`;
|
|
70
84
|
}
|
|
71
85
|
export async function handleGenerateAudio(request, openai) {
|
|
72
|
-
const { prompt, model, voice, format, save_path } = request.params.arguments
|
|
86
|
+
const { prompt, model, voice, format, save_path } = request.params.arguments ?? {
|
|
87
|
+
prompt: '',
|
|
88
|
+
};
|
|
73
89
|
if (!prompt?.trim()) {
|
|
74
|
-
return
|
|
90
|
+
return toolError(ErrorCode.INVALID_INPUT, 'prompt is required.');
|
|
91
|
+
}
|
|
92
|
+
// Fail-fast on unsafe paths BEFORE spending tokens.
|
|
93
|
+
let safeBase = null;
|
|
94
|
+
if (save_path) {
|
|
95
|
+
try {
|
|
96
|
+
safeBase = await resolveSafeOutputPath(save_path);
|
|
97
|
+
}
|
|
98
|
+
catch (e) {
|
|
99
|
+
if (e instanceof UnsafeOutputPathError)
|
|
100
|
+
return toolErrorFrom(ErrorCode.UNSAFE_PATH, e);
|
|
101
|
+
return toolErrorFrom(ErrorCode.INTERNAL, e);
|
|
102
|
+
}
|
|
75
103
|
}
|
|
76
104
|
const selectedFormat = VALID_FORMATS.includes(format ?? '')
|
|
77
105
|
? format
|
|
78
106
|
: DEFAULT_FORMAT;
|
|
79
107
|
const selectedVoice = voice?.trim() || DEFAULT_VOICE;
|
|
108
|
+
let stream;
|
|
80
109
|
try {
|
|
81
|
-
|
|
110
|
+
stream = (await openai.chat.completions.create({
|
|
82
111
|
model: model || DEFAULT_MODEL,
|
|
83
112
|
messages: [{ role: 'user', content: prompt }],
|
|
84
113
|
modalities: ['text', 'audio'],
|
|
85
114
|
audio: { voice: selectedVoice, format: selectedFormat },
|
|
86
115
|
stream: true,
|
|
87
116
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
88
|
-
});
|
|
117
|
+
}));
|
|
118
|
+
}
|
|
119
|
+
catch (err) {
|
|
120
|
+
return classifyUpstreamError(err, 'generate_audio');
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
89
123
|
const audioChunks = [];
|
|
90
124
|
const transcriptChunks = [];
|
|
91
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
92
125
|
for await (const chunk of stream) {
|
|
93
|
-
const delta = chunk.choices?.[0]
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if (
|
|
98
|
-
|
|
126
|
+
const delta = chunk.choices?.[0]
|
|
127
|
+
?.delta;
|
|
128
|
+
if (delta && typeof delta === 'object' && delta.audio) {
|
|
129
|
+
const a = delta.audio;
|
|
130
|
+
if (typeof a.data === 'string')
|
|
131
|
+
audioChunks.push(a.data);
|
|
132
|
+
if (typeof a.transcript === 'string')
|
|
133
|
+
transcriptChunks.push(a.transcript);
|
|
99
134
|
}
|
|
100
135
|
}
|
|
101
136
|
const fullAudioBase64 = audioChunks.join('');
|
|
102
137
|
const transcript = transcriptChunks.join('');
|
|
103
138
|
if (!fullAudioBase64) {
|
|
104
|
-
return
|
|
139
|
+
return toolError(ErrorCode.INTERNAL, transcript
|
|
140
|
+
? `No audio returned (model emitted transcript only): ${transcript.slice(0, 300)}`
|
|
141
|
+
: 'No audio returned.', { reason: 'no_audio_in_stream' });
|
|
105
142
|
}
|
|
106
143
|
let audioBuffer = Buffer.from(fullAudioBase64, 'base64');
|
|
107
144
|
const detected = detectAudioFormat(audioBuffer);
|
|
@@ -112,13 +149,11 @@ export async function handleGenerateAudio(request, openai) {
|
|
|
112
149
|
detected.mimeType = 'audio/wav';
|
|
113
150
|
}
|
|
114
151
|
const returnBase64 = audioBuffer.toString('base64');
|
|
115
|
-
if (
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
const fileExt = extname(save_path).toLowerCase().slice(1);
|
|
119
|
-
const actualSavePath = fileExt === detected.ext ? save_path : replaceExtension(save_path, detected.ext);
|
|
152
|
+
if (safeBase) {
|
|
153
|
+
const fileExt = extname(safeBase).toLowerCase().slice(1);
|
|
154
|
+
const actualSavePath = fileExt === detected.ext ? safeBase : replaceExtension(safeBase, detected.ext);
|
|
120
155
|
await fs.writeFile(actualSavePath, audioBuffer);
|
|
121
|
-
const formatNote = actualSavePath !==
|
|
156
|
+
const formatNote = actualSavePath !== safeBase
|
|
122
157
|
? ` (detected ${detected.ext.toUpperCase()}, saved as ${actualSavePath})`
|
|
123
158
|
: '';
|
|
124
159
|
const result = transcript
|
|
@@ -129,6 +164,11 @@ export async function handleGenerateAudio(request, openai) {
|
|
|
129
164
|
{ type: 'text', text: result },
|
|
130
165
|
{ type: 'audio', mimeType: detected.mimeType, data: returnBase64 },
|
|
131
166
|
],
|
|
167
|
+
_meta: {
|
|
168
|
+
save_path: actualSavePath,
|
|
169
|
+
mime: detected.mimeType,
|
|
170
|
+
size_bytes: audioBuffer.length,
|
|
171
|
+
},
|
|
132
172
|
};
|
|
133
173
|
}
|
|
134
174
|
return {
|
|
@@ -136,19 +176,10 @@ export async function handleGenerateAudio(request, openai) {
|
|
|
136
176
|
{ type: 'text', text: transcript || 'Audio generated successfully.' },
|
|
137
177
|
{ type: 'audio', mimeType: detected.mimeType, data: returnBase64 },
|
|
138
178
|
],
|
|
179
|
+
_meta: { mime: detected.mimeType, size_bytes: audioBuffer.length },
|
|
139
180
|
};
|
|
140
181
|
}
|
|
141
|
-
catch (
|
|
142
|
-
|
|
143
|
-
if (error instanceof Error) {
|
|
144
|
-
msg = error.message;
|
|
145
|
-
const oaiErr = error;
|
|
146
|
-
if (oaiErr.error?.message)
|
|
147
|
-
msg = `${msg} - ${oaiErr.error.message}`;
|
|
148
|
-
}
|
|
149
|
-
else {
|
|
150
|
-
msg = String(error);
|
|
151
|
-
}
|
|
152
|
-
return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
|
|
182
|
+
catch (err) {
|
|
183
|
+
return classifyUpstreamError(err, 'generate_audio (stream)');
|
|
153
184
|
}
|
|
154
185
|
}
|
|
@@ -8,23 +8,39 @@ export declare function handleGenerateImage(request: {
|
|
|
8
8
|
params: {
|
|
9
9
|
arguments: GenerateImageToolRequest;
|
|
10
10
|
};
|
|
11
|
-
}, openai: OpenAI): Promise<{
|
|
12
|
-
content: {
|
|
13
|
-
type: string;
|
|
14
|
-
text: string;
|
|
15
|
-
}[];
|
|
16
|
-
isError: boolean;
|
|
17
|
-
} | {
|
|
11
|
+
}, openai: OpenAI): Promise<import("../errors.js").ToolErrorResult | {
|
|
18
12
|
content: ({
|
|
19
|
-
type:
|
|
13
|
+
type: "text";
|
|
20
14
|
text: string;
|
|
21
15
|
mimeType?: undefined;
|
|
22
16
|
data?: undefined;
|
|
23
17
|
} | {
|
|
24
|
-
type:
|
|
18
|
+
type: "image";
|
|
25
19
|
mimeType: string;
|
|
26
20
|
data: string;
|
|
27
21
|
text?: undefined;
|
|
28
22
|
})[];
|
|
29
|
-
|
|
23
|
+
_meta: {
|
|
24
|
+
usage?: {
|
|
25
|
+
prompt_tokens: number;
|
|
26
|
+
completion_tokens: number;
|
|
27
|
+
total_tokens: number;
|
|
28
|
+
} | undefined;
|
|
29
|
+
save_path: string;
|
|
30
|
+
mime: string;
|
|
31
|
+
};
|
|
32
|
+
} | {
|
|
33
|
+
content: {
|
|
34
|
+
type: "image";
|
|
35
|
+
mimeType: string;
|
|
36
|
+
data: string;
|
|
37
|
+
}[];
|
|
38
|
+
_meta: {
|
|
39
|
+
usage?: {
|
|
40
|
+
prompt_tokens: number;
|
|
41
|
+
completion_tokens: number;
|
|
42
|
+
total_tokens: number;
|
|
43
|
+
} | undefined;
|
|
44
|
+
mime: string;
|
|
45
|
+
};
|
|
30
46
|
}>;
|