@moxxy/plugin-stt-whisper 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/audio.d.ts +52 -0
- package/dist/audio.d.ts.map +1 -0
- package/dist/audio.js +129 -0
- package/dist/audio.js.map +1 -0
- package/dist/index.d.ts +38 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +47 -0
- package/dist/index.js.map +1 -0
- package/dist/whisper.d.ts +40 -0
- package/dist/whisper.d.ts.map +1 -0
- package/dist/whisper.js +161 -0
- package/dist/whisper.js.map +1 -0
- package/package.json +62 -0
- package/src/audio.test.ts +101 -0
- package/src/audio.ts +150 -0
- package/src/index.ts +73 -0
- package/src/whisper.test.ts +281 -0
- package/src/whisper.ts +212 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { MOXXY_PCM16_24KHZ_MIME, normalizeWhisperUpload, pcm16MonoToWav } from './audio.js';
|
|
3
|
+
|
|
4
|
+
function ascii(bytes: Uint8Array, offset: number, len: number): string {
|
|
5
|
+
return String.fromCharCode(...bytes.slice(offset, offset + len));
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
describe('pcm16MonoToWav', () => {
|
|
9
|
+
it('writes a 44-byte canonical mono/16-bit WAV header for 24kHz', () => {
|
|
10
|
+
const pcm = new Uint8Array(8); // 8 bytes of samples
|
|
11
|
+
const wav = pcm16MonoToWav(pcm, 24_000);
|
|
12
|
+
expect(wav.byteLength).toBe(44 + 8);
|
|
13
|
+
|
|
14
|
+
const view = new DataView(wav.buffer, wav.byteOffset, wav.byteLength);
|
|
15
|
+
expect(ascii(wav, 0, 4)).toBe('RIFF');
|
|
16
|
+
expect(view.getUint32(4, true)).toBe(36 + 8); // chunk size = 36 + data
|
|
17
|
+
expect(ascii(wav, 8, 4)).toBe('WAVE');
|
|
18
|
+
expect(ascii(wav, 12, 4)).toBe('fmt ');
|
|
19
|
+
expect(view.getUint32(16, true)).toBe(16); // PCM fmt chunk size
|
|
20
|
+
expect(view.getUint16(20, true)).toBe(1); // audio format = PCM
|
|
21
|
+
expect(view.getUint16(22, true)).toBe(1); // channels = mono
|
|
22
|
+
expect(view.getUint32(24, true)).toBe(24_000); // sample rate
|
|
23
|
+
expect(view.getUint32(28, true)).toBe(24_000 * 2); // byte rate = rate * blockAlign
|
|
24
|
+
expect(view.getUint16(32, true)).toBe(2); // block align = channels * bytesPerSample
|
|
25
|
+
expect(view.getUint16(34, true)).toBe(16); // bits per sample
|
|
26
|
+
expect(ascii(wav, 36, 4)).toBe('data');
|
|
27
|
+
expect(view.getUint32(40, true)).toBe(8); // data chunk size
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('copies the PCM samples verbatim after the header', () => {
|
|
31
|
+
const pcm = new Uint8Array([1, 2, 3, 4]);
|
|
32
|
+
const wav = pcm16MonoToWav(pcm, 24_000);
|
|
33
|
+
expect([...wav.slice(44)]).toEqual([1, 2, 3, 4]);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe('normalizeWhisperUpload', () => {
|
|
38
|
+
it('wraps raw PCM16 bytes in a WAV and reports audio/wav', () => {
|
|
39
|
+
const pcm = new Uint8Array(4);
|
|
40
|
+
const out = normalizeWhisperUpload(pcm, MOXXY_PCM16_24KHZ_MIME);
|
|
41
|
+
expect(out.mimeType).toBe('audio/wav');
|
|
42
|
+
expect(out.filename).toBe('audio.wav');
|
|
43
|
+
expect(out.bytes.byteLength).toBe(44 + 4);
|
|
44
|
+
expect(ascii(out.bytes, 0, 4)).toBe('RIFF');
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it('brands the PCM16 WAV filename with a prefix when supplied', () => {
|
|
48
|
+
const out = normalizeWhisperUpload(new Uint8Array(2), MOXXY_PCM16_24KHZ_MIME, 'moxxy');
|
|
49
|
+
expect(out.filename).toBe('moxxy.wav');
|
|
50
|
+
expect(out.mimeType).toBe('audio/wav');
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('passes non-PCM bytes through and infers the filename from the MIME table', () => {
|
|
54
|
+
const bytes = new Uint8Array([9, 9]);
|
|
55
|
+
const out = normalizeWhisperUpload(bytes, 'audio/ogg');
|
|
56
|
+
expect(out.mimeType).toBe('audio/ogg');
|
|
57
|
+
expect(out.filename).toBe('audio.ogg');
|
|
58
|
+
expect(out.bytes).toBe(bytes); // no copy on passthrough
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('falls back to audio.bin for an unknown MIME', () => {
|
|
62
|
+
const out = normalizeWhisperUpload(new Uint8Array(1), 'audio/weird');
|
|
63
|
+
expect(out.filename).toBe('audio.bin');
|
|
64
|
+
expect(out.mimeType).toBe('audio/weird');
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('keeps the inferred extension when branding the passthrough filename', () => {
|
|
68
|
+
const out = normalizeWhisperUpload(new Uint8Array(1), 'audio/mpeg', 'moxxy');
|
|
69
|
+
// audio/mpeg maps to audio.mp3 → branded as moxxy.mp3.
|
|
70
|
+
expect(out.filename).toBe('moxxy.mp3');
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('defaults a missing MIME to audio/wav', () => {
|
|
74
|
+
const out = normalizeWhisperUpload(new Uint8Array(1), undefined);
|
|
75
|
+
expect(out.mimeType).toBe('audio/wav');
|
|
76
|
+
expect(out.filename).toBe('audio.wav');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('accepts an ArrayBuffer input', () => {
|
|
80
|
+
const buf = new Uint8Array([5, 6, 7]).buffer;
|
|
81
|
+
const out = normalizeWhisperUpload(buf, 'audio/ogg');
|
|
82
|
+
expect([...out.bytes]).toEqual([5, 6, 7]);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('degrades to the default when a non-string mimeType slips past the type', () => {
|
|
86
|
+
// The Telegram path forwards `media.mime_type` from raw update JSON, which a
|
|
87
|
+
// crafted message could make a number/object/array. `.toLowerCase()` would
|
|
88
|
+
// throw on these; the runtime guard must fall back to audio/wav instead.
|
|
89
|
+
for (const bad of [123, {}, [], true, Symbol('x')] as unknown[]) {
|
|
90
|
+
const out = normalizeWhisperUpload(new Uint8Array([1]), bad as string | undefined);
|
|
91
|
+
expect(out.mimeType).toBe('audio/wav');
|
|
92
|
+
expect(out.filename).toBe('audio.wav');
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it('treats null mimeType like a missing one', () => {
|
|
97
|
+
const out = normalizeWhisperUpload(new Uint8Array([1]), null as unknown as undefined);
|
|
98
|
+
expect(out.mimeType).toBe('audio/wav');
|
|
99
|
+
expect(out.filename).toBe('audio.wav');
|
|
100
|
+
});
|
|
101
|
+
});
|
package/src/audio.ts
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Audio helpers shared across STT plugins. Whisper-family backends — the
|
|
3
|
+
* generic OpenAI Whisper endpoint, the Codex OAuth `/transcribe` proxy,
|
|
4
|
+
* any future Whisper-compatible vendor — all want the same preprocessing
|
|
5
|
+
* (filename inference from MIME, raw PCM16 → WAV wrapping). Centralizing
|
|
6
|
+
* them here keeps the wrapper plugins thin.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { MOXXY_PCM16_24KHZ_MIME } from '@moxxy/sdk';
|
|
10
|
+
|
|
11
|
+
/** A custom MIME tag used by the TUI voice recorder to flag raw PCM16
|
|
12
|
+
* mono @ 24kHz bytes (ffmpeg's `-f s16le` output) so the transcriber
|
|
13
|
+
* knows to wrap them in a WAV header before upload. Public so other
|
|
14
|
+
* recorders / channels can mark their captures identically. Sourced from
|
|
15
|
+
* the SDK's zero-dep cross-package source of truth; re-exported here so the
|
|
16
|
+
* existing `@moxxy/plugin-stt-whisper` surface stays stable. */
|
|
17
|
+
export { MOXXY_PCM16_24KHZ_MIME };
|
|
18
|
+
|
|
19
|
+
/** Default filename per MIME type, used to set the upload `filename` so
|
|
20
|
+
* the vendor's content sniffer routes to the right decoder. Defaults to
|
|
21
|
+
* `audio.bin` for anything unmapped.
|
|
22
|
+
*
|
|
23
|
+
* The MIME string is caller-supplied and can be fully untrusted (the
|
|
24
|
+
* Telegram voice path forwards `media.mime_type` verbatim). A plain object
|
|
25
|
+
* literal would let a crafted MIME equal to an inherited member name
|
|
26
|
+
* ('constructor', 'toString', '__proto__', …) resolve to a function/object
|
|
27
|
+
* instead of `undefined`, defeating the `?? 'audio.bin'` fallback. Using a
|
|
28
|
+
* null-prototype map closes that hole; lookups can only hit own keys. */
|
|
29
|
+
export const WHISPER_FILENAME_BY_MIME: Readonly<Record<string, string>> =
|
|
30
|
+
/* null-prototype: see comment above */ Object.assign(Object.create(null) as Record<string, string>, {
|
|
31
|
+
'audio/ogg': 'audio.ogg',
|
|
32
|
+
'audio/opus': 'audio.opus',
|
|
33
|
+
'audio/mpeg': 'audio.mp3',
|
|
34
|
+
'audio/mp3': 'audio.mp3',
|
|
35
|
+
'audio/wav': 'audio.wav',
|
|
36
|
+
'audio/x-wav': 'audio.wav',
|
|
37
|
+
'audio/webm': 'audio.webm',
|
|
38
|
+
'audio/m4a': 'audio.m4a',
|
|
39
|
+
'audio/mp4': 'audio.mp4',
|
|
40
|
+
'audio/flac': 'audio.flac',
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
export interface NormalizedAudioUpload {
|
|
44
|
+
readonly bytes: Uint8Array;
|
|
45
|
+
readonly mimeType: string;
|
|
46
|
+
readonly filename: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Normalize a raw audio buffer for upload to a Whisper-family endpoint.
|
|
51
|
+
*
|
|
52
|
+
* - If the input is `MOXXY_PCM16_24KHZ_MIME`, wraps the raw samples in a
|
|
53
|
+
* WAV header (samples are mono, 24kHz, 16-bit little-endian) and
|
|
54
|
+
* reports the upload as `audio/wav`.
|
|
55
|
+
* - Otherwise passes the bytes through and picks a filename from the
|
|
56
|
+
* MIME table (or `audio.bin` for unknown types).
|
|
57
|
+
*
|
|
58
|
+
* The optional `filenamePrefix` lets callers brand the upload filename
|
|
59
|
+
* (e.g. Codex uses `moxxy.wav`); when omitted the default `audio.<ext>`
|
|
60
|
+
* shape is used.
|
|
61
|
+
*/
|
|
62
|
+
export function normalizeWhisperUpload(
|
|
63
|
+
audio: Uint8Array | ArrayBuffer,
|
|
64
|
+
mimeType: string | undefined,
|
|
65
|
+
filenamePrefix?: string,
|
|
66
|
+
): NormalizedAudioUpload {
|
|
67
|
+
const bytes = audio instanceof Uint8Array ? audio : new Uint8Array(audio);
|
|
68
|
+
// MIME types are case-insensitive (RFC 2045) and may carry a `; codecs=…`
|
|
69
|
+
// parameter; some callers (Telegram) forward the header verbatim without
|
|
70
|
+
// normalizing. Canonicalize once so casing/params don't miss the table.
|
|
71
|
+
//
|
|
72
|
+
// `mimeType` is *typed* `string | undefined`, but the value crosses a trust
|
|
73
|
+
// boundary unvalidated: the Telegram path forwards `media.mime_type` from raw
|
|
74
|
+
// update JSON, which a crafted message could make a number/object/array.
|
|
75
|
+
// Guard the runtime type so `.toLowerCase()` can't throw on hostile input —
|
|
76
|
+
// anything non-string degrades to the default rather than crashing the path.
|
|
77
|
+
const rawMime = typeof mimeType === 'string' ? mimeType : '';
|
|
78
|
+
const mt = (rawMime || 'audio/wav').toLowerCase().split(';')[0]!.trim();
|
|
79
|
+
if (mt === MOXXY_PCM16_24KHZ_MIME) {
|
|
80
|
+
return {
|
|
81
|
+
bytes: pcm16MonoToWav(bytes, 24_000),
|
|
82
|
+
mimeType: 'audio/wav',
|
|
83
|
+
filename: rename(filenamePrefix, 'wav') ?? 'audio.wav',
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
// Null-prototype map + string guard: an untrusted MIME can't surface an
|
|
87
|
+
// inherited member (function/object), so the upload filename is always a
|
|
88
|
+
// real string and `extOf` never receives a non-string.
|
|
89
|
+
const mapped = WHISPER_FILENAME_BY_MIME[mt];
|
|
90
|
+
const defaultName = typeof mapped === 'string' ? mapped : 'audio.bin';
|
|
91
|
+
return {
|
|
92
|
+
bytes,
|
|
93
|
+
mimeType: mt,
|
|
94
|
+
filename: rename(filenamePrefix, extOf(defaultName)) ?? defaultName,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function rename(prefix: string | undefined, ext: string): string | undefined {
|
|
99
|
+
return prefix ? `${prefix}.${ext}` : undefined;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function extOf(filename: string): string {
|
|
103
|
+
const dot = filename.lastIndexOf('.');
|
|
104
|
+
return dot >= 0 ? filename.slice(dot + 1) : 'bin';
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Wrap raw PCM16 mono samples in a WAV container. Used by recorders that
|
|
109
|
+
* stream `s16le` (ffmpeg's default raw output) so the bytes can be sent
|
|
110
|
+
* to endpoints that only accept container formats.
|
|
111
|
+
*/
|
|
112
|
+
export function pcm16MonoToWav(pcm: Uint8Array | ArrayBuffer, sampleRate = 24_000): Uint8Array {
|
|
113
|
+
const raw = pcm instanceof Uint8Array ? pcm : new Uint8Array(pcm);
|
|
114
|
+
// 16-bit samples are 2 bytes; a trailing odd byte is half a sample and
|
|
115
|
+
// breaks RIFF even-alignment, so drop it. `subarray` is a view, no copy.
|
|
116
|
+
const data = raw.byteLength & 1 ? raw.subarray(0, raw.byteLength & ~1) : raw;
|
|
117
|
+
// The RIFF/data chunk sizes are unsigned 32-bit; >~4GiB would silently
|
|
118
|
+
// wrap to a tiny size and emit a structurally corrupt WAV. Reject loudly.
|
|
119
|
+
if (data.byteLength > 0xffffffff - 36) {
|
|
120
|
+
throw new RangeError(
|
|
121
|
+
`PCM payload too large for a WAV container (${data.byteLength} bytes; max ${0xffffffff - 36}).`,
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
const headerBytes = 44;
|
|
125
|
+
const wav = new Uint8Array(headerBytes + data.byteLength);
|
|
126
|
+
const view = new DataView(wav.buffer);
|
|
127
|
+
|
|
128
|
+
writeAscii(wav, 0, 'RIFF');
|
|
129
|
+
view.setUint32(4, 36 + data.byteLength, true);
|
|
130
|
+
writeAscii(wav, 8, 'WAVE');
|
|
131
|
+
writeAscii(wav, 12, 'fmt ');
|
|
132
|
+
view.setUint32(16, 16, true);
|
|
133
|
+
view.setUint16(20, 1, true);
|
|
134
|
+
view.setUint16(22, 1, true);
|
|
135
|
+
view.setUint32(24, sampleRate, true);
|
|
136
|
+
view.setUint32(28, sampleRate * 2, true);
|
|
137
|
+
view.setUint16(32, 2, true);
|
|
138
|
+
view.setUint16(34, 16, true);
|
|
139
|
+
writeAscii(wav, 36, 'data');
|
|
140
|
+
view.setUint32(40, data.byteLength, true);
|
|
141
|
+
wav.set(data, headerBytes);
|
|
142
|
+
|
|
143
|
+
return wav;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function writeAscii(target: Uint8Array, offset: number, value: string): void {
|
|
147
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
148
|
+
target[offset + i] = value.charCodeAt(i);
|
|
149
|
+
}
|
|
150
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { definePlugin, defineTranscriber, type Plugin } from '@moxxy/sdk';
|
|
2
|
+
import { WhisperTranscriber, type WhisperModel, type WhisperTranscriberOptions } from './whisper.js';
|
|
3
|
+
|
|
4
|
+
export {
|
|
5
|
+
WhisperTranscriber,
|
|
6
|
+
createWhisperTranscriber,
|
|
7
|
+
type WhisperModel,
|
|
8
|
+
type WhisperTranscriberOptions,
|
|
9
|
+
} from './whisper.js';
|
|
10
|
+
export {
|
|
11
|
+
MOXXY_PCM16_24KHZ_MIME,
|
|
12
|
+
WHISPER_FILENAME_BY_MIME,
|
|
13
|
+
normalizeWhisperUpload,
|
|
14
|
+
pcm16MonoToWav,
|
|
15
|
+
type NormalizedAudioUpload,
|
|
16
|
+
} from './audio.js';
|
|
17
|
+
|
|
18
|
+
export interface BuildWhisperPluginOptions {
|
|
19
|
+
/**
|
|
20
|
+
* Default model used when the host calls `session.transcribers.setActive(name)`
|
|
21
|
+
* without an explicit config. Defaults to `whisper-1`.
|
|
22
|
+
*/
|
|
23
|
+
readonly model?: WhisperModel;
|
|
24
|
+
/**
|
|
25
|
+
* Optional default config baked into the transcriber def. Callers can
|
|
26
|
+
* still override per-`setActive(name, config)`.
|
|
27
|
+
*/
|
|
28
|
+
readonly defaults?: Omit<WhisperTranscriberOptions, 'client'>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Build the @moxxy/plugin-stt-whisper plugin. Each instance registers
|
|
33
|
+
* exactly one transcriber, named `openai-<model>`.
|
|
34
|
+
*
|
|
35
|
+
* To run several Whisper-family models at once, build the plugin once per
|
|
36
|
+
* model — the transcriber names won't collide (`openai-whisper-1` vs
|
|
37
|
+
* `openai-gpt-4o-transcribe`), but each registration carries the SAME
|
|
38
|
+
* plugin name, so the host must register them under DISTINCT plugin names
|
|
39
|
+
* (the name-keyed plugin registry would otherwise clobber the first).
|
|
40
|
+
*
|
|
41
|
+
* Activation:
|
|
42
|
+
* session.transcribers.setActive('openai-whisper-1', { apiKey: ... });
|
|
43
|
+
*
|
|
44
|
+
* The plugin is intentionally side-effect free — registering it does
|
|
45
|
+
* NOT activate the transcriber. The host (CLI / config loader) chooses
|
|
46
|
+
* activation explicitly so users on local providers aren't surprised by
|
|
47
|
+
* outbound calls to OpenAI.
|
|
48
|
+
*/
|
|
49
|
+
export function buildWhisperPlugin(opts: BuildWhisperPluginOptions = {}): Plugin {
|
|
50
|
+
const model = opts.model ?? 'whisper-1';
|
|
51
|
+
const name = `openai-${model}`;
|
|
52
|
+
return definePlugin({
|
|
53
|
+
name: '@moxxy/plugin-stt-whisper',
|
|
54
|
+
version: '0.0.0',
|
|
55
|
+
transcribers: [
|
|
56
|
+
defineTranscriber({
|
|
57
|
+
name,
|
|
58
|
+
displayName: `OpenAI ${model}`,
|
|
59
|
+
createClient: (config) => {
|
|
60
|
+
const merged: WhisperTranscriberOptions = {
|
|
61
|
+
...(opts.defaults ?? {}),
|
|
62
|
+
...(config as WhisperTranscriberOptions),
|
|
63
|
+
model,
|
|
64
|
+
};
|
|
65
|
+
return new WhisperTranscriber(merged);
|
|
66
|
+
},
|
|
67
|
+
}),
|
|
68
|
+
],
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Discovery entry: `createPluginLoader` requires a default Plugin export.
|
|
73
|
+
export default buildWhisperPlugin();
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import type OpenAI from 'openai';
|
|
3
|
+
import { APIError, APIConnectionError, APIUserAbortError } from 'openai';
|
|
4
|
+
import { WhisperTranscriber } from './whisper.js';
|
|
5
|
+
import { buildWhisperPlugin } from './index.js';
|
|
6
|
+
import { MOXXY_PCM16_24KHZ_MIME, normalizeWhisperUpload, pcm16MonoToWav } from './audio.js';
|
|
7
|
+
|
|
8
|
+
const fakeOpenAI = (impl: (req: unknown) => unknown): OpenAI =>
|
|
9
|
+
({
|
|
10
|
+
audio: {
|
|
11
|
+
transcriptions: {
|
|
12
|
+
create: vi.fn(async (req: unknown) => impl(req)),
|
|
13
|
+
},
|
|
14
|
+
},
|
|
15
|
+
}) as unknown as OpenAI;
|
|
16
|
+
|
|
17
|
+
/** A client whose `create` always rejects with the supplied error — exercises
|
|
18
|
+
* the `run()` failure-translation surface without touching the network. */
|
|
19
|
+
const throwingOpenAI = (err: unknown): OpenAI =>
|
|
20
|
+
({
|
|
21
|
+
audio: {
|
|
22
|
+
transcriptions: {
|
|
23
|
+
create: vi.fn(async () => {
|
|
24
|
+
throw err;
|
|
25
|
+
}),
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
// `run()` reads client.baseURL for error context.
|
|
29
|
+
baseURL: 'https://api.openai.com/v1',
|
|
30
|
+
}) as unknown as OpenAI;
|
|
31
|
+
|
|
32
|
+
describe('WhisperTranscriber', () => {
|
|
33
|
+
it('returns text, language, duration, and segments from verbose_json', async () => {
|
|
34
|
+
const client = fakeOpenAI(() => ({
|
|
35
|
+
text: 'hello world',
|
|
36
|
+
language: 'en',
|
|
37
|
+
duration: 1.5,
|
|
38
|
+
segments: [
|
|
39
|
+
{ start: 0, end: 1.5, text: 'hello world' },
|
|
40
|
+
],
|
|
41
|
+
}));
|
|
42
|
+
const t = new WhisperTranscriber({ client });
|
|
43
|
+
const result = await t.transcribe(new Uint8Array([1, 2, 3]), { mimeType: 'audio/ogg' });
|
|
44
|
+
expect(result.text).toBe('hello world');
|
|
45
|
+
expect(result.language).toBe('en');
|
|
46
|
+
expect(result.durationSec).toBe(1.5);
|
|
47
|
+
expect(result.segments).toEqual([{ start: 0, end: 1.5, text: 'hello world' }]);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('passes language hint + prompt to the OpenAI client', async () => {
|
|
51
|
+
const create = vi.fn(async () => ({ text: 'cześć', language: 'pl', segments: [] }));
|
|
52
|
+
const client = { audio: { transcriptions: { create } } } as unknown as OpenAI;
|
|
53
|
+
const t = new WhisperTranscriber({ client, language: 'pl' });
|
|
54
|
+
await t.transcribe(new Uint8Array(), { mimeType: 'audio/ogg', prompt: 'jargon-list' });
|
|
55
|
+
const req = create.mock.calls[0]![0] as { language: string; prompt: string; response_format: string };
|
|
56
|
+
expect(req.language).toBe('pl');
|
|
57
|
+
expect(req.prompt).toBe('jargon-list');
|
|
58
|
+
expect(req.response_format).toBe('verbose_json');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('uses the per-call language over the default', async () => {
|
|
62
|
+
const create = vi.fn(async () => ({ text: '' }));
|
|
63
|
+
const client = { audio: { transcriptions: { create } } } as unknown as OpenAI;
|
|
64
|
+
const t = new WhisperTranscriber({ client, language: 'pl' });
|
|
65
|
+
await t.transcribe(new Uint8Array(), { language: 'en' });
|
|
66
|
+
const req = create.mock.calls[0]![0] as { language: string };
|
|
67
|
+
expect(req.language).toBe('en');
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it('falls back to plain text on gpt-4o-transcribe (no verbose_json branch)', async () => {
|
|
71
|
+
const create = vi.fn(async () => ({ text: 'plain' }));
|
|
72
|
+
const client = { audio: { transcriptions: { create } } } as unknown as OpenAI;
|
|
73
|
+
const t = new WhisperTranscriber({ client, model: 'gpt-4o-transcribe' });
|
|
74
|
+
const out = await t.transcribe(new Uint8Array(), { mimeType: 'audio/wav' });
|
|
75
|
+
expect(out.text).toBe('plain');
|
|
76
|
+
const req = create.mock.calls[0]![0] as { response_format?: string };
|
|
77
|
+
expect(req.response_format).toBeUndefined();
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('plugin registers a transcriber whose createClient yields the right name', () => {
|
|
81
|
+
const plugin = buildWhisperPlugin({});
|
|
82
|
+
expect(plugin.transcribers).toHaveLength(1);
|
|
83
|
+
const def = plugin.transcribers![0]!;
|
|
84
|
+
expect(def.name).toBe('openai-whisper-1');
|
|
85
|
+
expect(def.displayName).toBe('OpenAI whisper-1');
|
|
86
|
+
const inst = def.createClient({ apiKey: 'sk-test' });
|
|
87
|
+
expect(inst.name).toBe('openai-whisper-1');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('WAV-wraps the project raw-PCM16 MIME before upload', async () => {
|
|
91
|
+
const create = vi.fn(async () => ({ text: '' }));
|
|
92
|
+
const client = { audio: { transcriptions: { create } } } as unknown as OpenAI;
|
|
93
|
+
const t = new WhisperTranscriber({ client });
|
|
94
|
+
// 4 raw s16le samples of silence.
|
|
95
|
+
const raw = new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0]);
|
|
96
|
+
await t.transcribe(raw, { mimeType: MOXXY_PCM16_24KHZ_MIME });
|
|
97
|
+
|
|
98
|
+
const req = create.mock.calls[0]![0] as { file: File };
|
|
99
|
+
expect(req.file.type).toBe('audio/wav');
|
|
100
|
+
const header = new Uint8Array(await req.file.arrayBuffer());
|
|
101
|
+
const ascii = (s: number, e: number): string =>
|
|
102
|
+
String.fromCharCode(...header.slice(s, e));
|
|
103
|
+
expect(ascii(0, 4)).toBe('RIFF');
|
|
104
|
+
expect(ascii(8, 12)).toBe('WAVE');
|
|
105
|
+
// Header (44 bytes) + the 8 raw sample bytes.
|
|
106
|
+
expect(header.byteLength).toBe(44 + raw.byteLength);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('plugin honors a non-default model name', () => {
|
|
110
|
+
const plugin = buildWhisperPlugin({ model: 'gpt-4o-mini-transcribe' });
|
|
111
|
+
expect(plugin.transcribers![0]!.name).toBe('openai-gpt-4o-mini-transcribe');
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('throws PROVIDER_UNKNOWN_RESPONSE when verbose_json lacks a string text', async () => {
|
|
115
|
+
const client = fakeOpenAI(() => ({ language: 'en', segments: [] }));
|
|
116
|
+
const t = new WhisperTranscriber({ client });
|
|
117
|
+
await expect(
|
|
118
|
+
t.transcribe(new Uint8Array([1, 2, 3]), { mimeType: 'audio/ogg' }),
|
|
119
|
+
).rejects.toMatchObject({ code: 'PROVIDER_UNKNOWN_RESPONSE' });
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('throws PROVIDER_UNKNOWN_RESPONSE when the gpt-4o response lacks text', async () => {
|
|
123
|
+
const client = fakeOpenAI(() => ({ foo: 'bar' }));
|
|
124
|
+
const t = new WhisperTranscriber({ client, model: 'gpt-4o-transcribe' });
|
|
125
|
+
await expect(
|
|
126
|
+
t.transcribe(new Uint8Array([1, 2, 3]), { mimeType: 'audio/wav' }),
|
|
127
|
+
).rejects.toMatchObject({ code: 'PROVIDER_UNKNOWN_RESPONSE' });
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it('ignores a non-string language/duration in verbose_json', async () => {
|
|
131
|
+
const client = fakeOpenAI(() => ({ text: 'ok', language: 123, duration: 'nope' }));
|
|
132
|
+
const t = new WhisperTranscriber({ client });
|
|
133
|
+
const result = await t.transcribe(new Uint8Array([1]), { mimeType: 'audio/ogg' });
|
|
134
|
+
expect(result.text).toBe('ok');
|
|
135
|
+
expect(result.language).toBeUndefined();
|
|
136
|
+
expect(result.durationSec).toBeUndefined();
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('drops malformed segments from a hostile verbose_json response without crashing', async () => {
|
|
140
|
+
// A vendor (or man-in-the-middle) response whose segments array is full of
|
|
141
|
+
// junk: null, a primitive, missing fields, wrong types, and non-finite
|
|
142
|
+
// bounds. A blind `.start` read on `null` would throw; an unchecked map
|
|
143
|
+
// would leak `{ text: 123 }`. Each bad entry must be dropped; only the one
|
|
144
|
+
// well-formed segment survives.
|
|
145
|
+
const client = fakeOpenAI(() => ({
|
|
146
|
+
text: 'ok',
|
|
147
|
+
segments: [
|
|
148
|
+
null,
|
|
149
|
+
42,
|
|
150
|
+
'not-an-object',
|
|
151
|
+
{ start: 0, end: 1 }, // missing text
|
|
152
|
+
{ start: 0, text: 'no end' }, // missing end
|
|
153
|
+
{ start: 'x', end: 1, text: 'bad start' }, // non-number start
|
|
154
|
+
{ start: 0, end: 1, text: 99 }, // non-string text
|
|
155
|
+
{ start: Number.NaN, end: 1, text: 'nan start' }, // non-finite
|
|
156
|
+
{ start: 0, end: Infinity, text: 'inf end' }, // non-finite
|
|
157
|
+
{ start: 1, end: 2, text: 'good' }, // the only valid one
|
|
158
|
+
],
|
|
159
|
+
}));
|
|
160
|
+
const t = new WhisperTranscriber({ client });
|
|
161
|
+
const result = await t.transcribe(new Uint8Array([1]), { mimeType: 'audio/ogg' });
|
|
162
|
+
expect(result.text).toBe('ok');
|
|
163
|
+
expect(result.segments).toEqual([{ start: 1, end: 2, text: 'good' }]);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it('returns an empty segments array when every segment is malformed', async () => {
|
|
167
|
+
const client = fakeOpenAI(() => ({ text: 'ok', segments: [null, undefined, {}] }));
|
|
168
|
+
const t = new WhisperTranscriber({ client });
|
|
169
|
+
const result = await t.transcribe(new Uint8Array([1]), { mimeType: 'audio/ogg' });
|
|
170
|
+
expect(result.segments).toEqual([]);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('does not crash when mimeType is a non-string at runtime (untrusted Telegram path)', async () => {
|
|
174
|
+
const create = vi.fn(async () => ({ text: 'ok' }));
|
|
175
|
+
const client = { audio: { transcriptions: { create } } } as unknown as OpenAI;
|
|
176
|
+
const t = new WhisperTranscriber({ client });
|
|
177
|
+
// A crafted Telegram update could deliver a non-string mime_type; the type
|
|
178
|
+
// says `string` but the runtime value is hostile. Must degrade, not throw.
|
|
179
|
+
const result = await t.transcribe(new Uint8Array([1]), {
|
|
180
|
+
mimeType: 123 as unknown as string,
|
|
181
|
+
});
|
|
182
|
+
expect(result.text).toBe('ok');
|
|
183
|
+
const req = create.mock.calls[0]![0] as { file: File };
|
|
184
|
+
// Defaults to the audio/wav fallback filename rather than crashing.
|
|
185
|
+
expect(req.file.name).toBe('audio.wav');
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
describe('WhisperTranscriber.run() error translation', () => {
|
|
190
|
+
const audio = new Uint8Array([1, 2, 3]);
|
|
191
|
+
|
|
192
|
+
it('re-throws a user abort unchanged (not masked as a provider error)', async () => {
|
|
193
|
+
const abort = new APIUserAbortError();
|
|
194
|
+
const t = new WhisperTranscriber({ client: throwingOpenAI(abort) });
|
|
195
|
+
await expect(t.transcribe(audio, { mimeType: 'audio/ogg' })).rejects.toBe(abort);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('maps a refused connection to NETWORK_UNREACHABLE', async () => {
|
|
199
|
+
const cause = Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' });
|
|
200
|
+
const conn = new APIConnectionError({ cause });
|
|
201
|
+
const t = new WhisperTranscriber({ client: throwingOpenAI(conn) });
|
|
202
|
+
await expect(t.transcribe(audio, { mimeType: 'audio/ogg' })).rejects.toMatchObject({
|
|
203
|
+
code: 'NETWORK_UNREACHABLE',
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('maps HTTP 401 to AUTH_INVALID', async () => {
|
|
208
|
+
const err = new APIError(401, { error: { message: 'bad key' } }, 'Unauthorized', {});
|
|
209
|
+
const t = new WhisperTranscriber({ client: throwingOpenAI(err) });
|
|
210
|
+
await expect(t.transcribe(audio, { mimeType: 'audio/ogg' })).rejects.toMatchObject({
|
|
211
|
+
code: 'AUTH_INVALID',
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
it('maps HTTP 500 to PROVIDER_SERVER_ERROR', async () => {
|
|
216
|
+
const err = new APIError(500, undefined, 'boom', {});
|
|
217
|
+
const t = new WhisperTranscriber({ client: throwingOpenAI(err) });
|
|
218
|
+
await expect(t.transcribe(audio, { mimeType: 'audio/ogg' })).rejects.toMatchObject({
|
|
219
|
+
code: 'PROVIDER_SERVER_ERROR',
|
|
220
|
+
});
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it('falls back to PROVIDER_BAD_REQUEST for an unmapped status', async () => {
|
|
224
|
+
const err = new APIError(418, undefined, 'teapot', {});
|
|
225
|
+
const t = new WhisperTranscriber({ client: throwingOpenAI(err) });
|
|
226
|
+
await expect(t.transcribe(audio, { mimeType: 'audio/ogg' })).rejects.toMatchObject({
|
|
227
|
+
code: 'PROVIDER_BAD_REQUEST',
|
|
228
|
+
context: { status: 418 },
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it('re-throws an unclassifiable non-network error rather than swallowing it', async () => {
|
|
233
|
+
const odd = new Error('totally unexpected');
|
|
234
|
+
const t = new WhisperTranscriber({ client: throwingOpenAI(odd) });
|
|
235
|
+
await expect(t.transcribe(audio, { mimeType: 'audio/ogg' })).rejects.toBe(odd);
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
describe('normalizeWhisperUpload hardening', () => {
|
|
240
|
+
it('does not resolve inherited prototype members for a crafted MIME', () => {
|
|
241
|
+
// 'constructor' / 'toString' / '__proto__' must NOT surface a function/object.
|
|
242
|
+
for (const mt of ['constructor', 'toString', 'valueOf', 'hasOwnProperty', '__proto__']) {
|
|
243
|
+
const out = normalizeWhisperUpload(new Uint8Array([1]), mt);
|
|
244
|
+
expect(out.filename).toBe('audio.bin');
|
|
245
|
+
expect(typeof out.filename).toBe('string');
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
it('does not throw inside the prefix path for a crafted MIME', () => {
|
|
250
|
+
// extOf() would have thrown if the lookup returned a function.
|
|
251
|
+
const out = normalizeWhisperUpload(new Uint8Array([1]), 'constructor', 'moxxy');
|
|
252
|
+
expect(out.filename).toBe('moxxy.bin');
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it('matches MIME case-insensitively and strips codec params', () => {
|
|
256
|
+
expect(normalizeWhisperUpload(new Uint8Array([1]), 'AUDIO/OGG').filename).toBe('audio.ogg');
|
|
257
|
+
expect(
|
|
258
|
+
normalizeWhisperUpload(new Uint8Array([1]), 'audio/webm; codecs=opus').filename,
|
|
259
|
+
).toBe('audio.webm');
|
|
260
|
+
expect(normalizeWhisperUpload(new Uint8Array([1]), ' Audio/Wav ').filename).toBe('audio.wav');
|
|
261
|
+
});
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
describe('pcm16MonoToWav hardening', () => {
|
|
265
|
+
it('drops a trailing odd byte so the data chunk stays even-aligned', () => {
|
|
266
|
+
const wav = pcm16MonoToWav(new Uint8Array([1, 2, 3]));
|
|
267
|
+
const view = new DataView(wav.buffer);
|
|
268
|
+
// 3 input bytes → 2 retained (one whole 16-bit sample).
|
|
269
|
+
expect(view.getUint32(40, true)).toBe(2);
|
|
270
|
+
expect(wav.byteLength).toBe(44 + 2);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
it('rejects a payload too large for the 32-bit WAV size fields', () => {
|
|
274
|
+
// Fake a (real) Uint8Array reporting an oversized, even byteLength without
|
|
275
|
+
// allocating 4GiB. Even length skips the odd-byte subarray branch, so the
|
|
276
|
+
// size guard is what must fire. 0xfffffffe > 0xffffffff - 36.
|
|
277
|
+
const huge = new Uint8Array(2);
|
|
278
|
+
Object.defineProperty(huge, 'byteLength', { value: 0xfffffffe });
|
|
279
|
+
expect(() => pcm16MonoToWav(huge)).toThrow(RangeError);
|
|
280
|
+
});
|
|
281
|
+
});
|