@ai-ecoverse/pi-audio 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/captions.js +150 -0
- package/dist/captions.js.map +7 -0
- package/dist/elevenlabs/catalog.js +136 -0
- package/dist/elevenlabs/catalog.js.map +7 -0
- package/dist/elevenlabs/index.js +5 -0
- package/dist/elevenlabs/index.js.map +7 -0
- package/dist/elevenlabs/listen.js +268 -0
- package/dist/elevenlabs/listen.js.map +7 -0
- package/dist/elevenlabs/speak.js +251 -0
- package/dist/elevenlabs/speak.js.map +7 -0
- package/dist/elevenlabs/transcribe.js +132 -0
- package/dist/elevenlabs/transcribe.js.map +7 -0
- package/dist/elevenlabs/wire.js +215 -0
- package/dist/elevenlabs/wire.js.map +7 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +2 -2
- package/dist/turns.js +51 -13
- package/dist/turns.js.map +2 -2
- package/dist/types/captions.d.ts +31 -0
- package/dist/types/elevenlabs/catalog.d.ts +1 -0
- package/dist/types/elevenlabs/index.d.ts +1 -0
- package/dist/types/elevenlabs/listen.d.ts +4 -0
- package/dist/types/elevenlabs/speak.d.ts +4 -0
- package/dist/types/elevenlabs/transcribe.d.ts +4 -0
- package/dist/types/elevenlabs/wire.d.ts +26 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/turns.d.ts +1 -1
- package/package.json +5 -1
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/elevenlabs/listen.ts"],
|
|
4
|
+
"sourcesContent": ["import type { AudioChunk, AudioFormat } from '../audio.ts';\nimport { chunkSeconds } from '../audio.ts';\nimport { encodeAudioBase64 } from '../convert.ts';\nimport type { TranscriptionModel } from '../models.ts';\nimport { asVoiceError, isRecord, numberField, stringField } from '../openai/wire.ts';\nimport {\n type ListenOptions,\n type TranscriptionEvent,\n TranscriptionEventStream,\n type TranscriptSegment,\n} from '../transcription.ts';\nimport { priceAudio } from '../usage.ts';\nimport {\n awaitOpen,\n connectSocket,\n messageText,\n sameFormat,\n socketUrl,\n sttAudioFormat,\n xiHeaders,\n} from './wire.ts';\n\nconst failures = new Set([\n 'error',\n 'auth_error',\n 'quota_exceeded',\n 'commit_throttled',\n 'unaccepted_terms',\n 'rate_limited',\n 'queue_overflow',\n 'resource_exhausted',\n 'session_time_limit_exceeded',\n 'input_error',\n 'invalid_request',\n 'chunk_size_exceeded',\n 'insufficient_audio_activity',\n 'transcriber_error',\n]);\n\ninterface ListenState {\n turns: string[];\n segments: TranscriptSegment[];\n segment: number;\n started: boolean;\n bytes: number;\n opened: boolean;\n ready: boolean;\n clientDone: boolean;\n failed: boolean;\n completed: boolean;\n lastCommit: string | undefined;\n}\n\nfunction terms(options: ListenOptions | undefined): string[] {\n const vocabulary = options?.vocabulary;\n if (!vocabulary) return [];\n return vocabulary;\n}\n\nfunction commitStrategy(model: TranscriptionModel<string>): string {\n if (model.turnDetection === 'semantic') return 'turn_prediction';\n if (model.turnDetection === 'vad') return 'vad';\n return 'manual';\n}\n\nfunction sttUrl(\n model: TranscriptionModel<string>,\n format: AudioFormat,\n options: ListenOptions | undefined\n): string {\n const url = socketUrl(model.baseUrl, 'speech-to-text/realtime');\n const params = new URLSearchParams();\n params.set('model_id', model.id);\n params.set('audio_format', sttAudioFormat(format));\n params.set('commit_strategy', commitStrategy(model));\n params.set('include_timestamps', 'true');\n if (options?.language) params.set('language_code', options.language);\n for (const term of terms(options)) params.append('keyterms', term);\n url.search = params.toString();\n return url.toString();\n}\n\nfunction chosenFormat(\n model: TranscriptionModel<string>,\n chunk: AudioChunk | undefined\n): AudioFormat {\n const format = chunk === undefined ? model.inputFormats[0] : chunk.format;\n if (!format) throw new Error('No input format');\n sttAudioFormat(format);\n const known = model.inputFormats.some((item) => sameFormat(item, format));\n if (!known) throw new Error(`Unsupported audio format ${format.encoding}`);\n return format;\n}\n\nfunction textOr(event: Record<string, unknown>): string {\n const text = stringField(event, 'text');\n if (text === undefined) return '';\n return text;\n}\n\nfunction secondsOf(word: Record<string, unknown>, key: string): number {\n const value = numberField(word, key);\n if (value === undefined) return 0;\n return value;\n}\n\nfunction wordFrom(word: Record<string, unknown>): TranscriptSegment | undefined {\n if (stringField(word, 'type') === 'spacing') return undefined;\n const speaker = stringField(word, 'speaker_id');\n const segment: TranscriptSegment = {\n text: textOr(word),\n start: secondsOf(word, 'start'),\n end: secondsOf(word, 'end'),\n };\n if (speaker !== undefined) segment.speaker = speaker;\n return segment;\n}\n\nfunction wordsOf(event: Record<string, unknown>, state: ListenState) {\n if (!Array.isArray(event.words)) return;\n for (const word of event.words) {\n if (!isRecord(word)) continue;\n const segment = wordFrom(word);\n if (segment) state.segments.push(segment);\n }\n}\n\nfunction emitCommit(spoken: string, state: ListenState, push: (event: TranscriptionEvent) => void) {\n if (!state.started) {\n state.started = true;\n push({ type: 'speech_start', at: 0 });\n }\n const segmentId = String(state.segment);\n state.segment += 1;\n state.turns.push(spoken);\n push({ type: 'final', segmentId, text: spoken });\n state.started = false;\n push({ type: 'turn_end', text: spoken, at: 0 });\n}\n\nfunction applyListen(\n event: Record<string, unknown>,\n state: ListenState,\n push: (event: TranscriptionEvent) => void\n) {\n const type = stringField(event, 'message_type');\n if (type === 'session_started') {\n state.ready = true;\n return;\n }\n if (type === 'partial_transcript') {\n if (!state.started) {\n state.started = true;\n push({ type: 'speech_start', at: 0 });\n }\n push({ type: 'partial', segmentId: String(state.segment), text: textOr(event) });\n return;\n }\n if (type === 'committed_transcript') {\n const spoken = textOr(event);\n state.lastCommit = spoken;\n emitCommit(spoken, state, push);\n return;\n }\n if (type === 'committed_transcript_with_timestamps') {\n const spoken = textOr(event);\n wordsOf(event, state);\n if (state.lastCommit !== spoken) emitCommit(spoken, state, push);\n state.lastCommit = undefined;\n return;\n }\n if (type !== undefined && failures.has(type)) {\n const message = stringField(event, 'error');\n state.failed = true;\n push({\n type: 'error',\n reason: 'error',\n error: message === undefined ? 'ElevenLabs transcription failed' : message,\n });\n }\n}\n\nfunction finish(\n model: TranscriptionModel<string>,\n format: AudioFormat,\n state: ListenState,\n push: (event: TranscriptionEvent) => void\n) {\n if (state.failed || state.completed) return;\n state.completed = true;\n const heard = chunkSeconds({ data: new Uint8Array(state.bytes), format, timestamp: 0 });\n push({\n type: 'done',\n transcript: {\n text: state.turns.join(' '),\n ...(state.segments.length > 0 ? { segments: state.segments } : {}),\n usage: priceAudio(model.audioCost, { inputSeconds: heard }),\n },\n });\n}\n\nexport function listenElevenLabs(\n model: TranscriptionModel<string>,\n audio: AsyncIterable<AudioChunk>,\n options?: ListenOptions\n): TranscriptionEventStream {\n const stream = new TranscriptionEventStream();\n void pumpListen(model, audio, options, stream);\n return stream;\n}\n\nasync function pumpListen(\n model: TranscriptionModel<string>,\n audio: AsyncIterable<AudioChunk>,\n options: ListenOptions | undefined,\n stream: TranscriptionEventStream\n) {\n const state: ListenState = {\n turns: [],\n segments: [],\n segment: 0,\n started: false,\n bytes: 0,\n opened: false,\n ready: false,\n clientDone: false,\n failed: false,\n completed: false,\n lastCommit: undefined,\n };\n let settled = false;\n const push = (event: TranscriptionEvent) => {\n if (settled) return;\n if (event.type === 'done' || event.type === 'error') settled = true;\n stream.push(event);\n };\n let socket: WebSocket | undefined;\n const readyWait = Promise.withResolvers<void>();\n const doneWait = Promise.withResolvers<void>();\n const wake = (kind: string | undefined) => {\n if (state.ready || state.failed) readyWait.resolve();\n if (state.failed) doneWait.resolve();\n if (!state.clientDone) return;\n if (kind === 'committed_transcript' || kind === 'committed_transcript_with_timestamps') {\n doneWait.resolve();\n }\n if (kind === 'close') doneWait.resolve();\n };\n try {\n if (options?.signal?.aborted) throw new DOMException('aborted', 'AbortError');\n const iterator = audio[Symbol.asyncIterator]();\n const first = await iterator.next();\n const format = chosenFormat(model, first.done ? undefined : first.value);\n socket = connectSocket(sttUrl(model, format, options), xiHeaders(model, options));\n let incoming = Promise.resolve();\n socket.addEventListener('message', (event) => {\n incoming = incoming.then(() => onMessage(event.data, state, push, wake));\n });\n socket.addEventListener('close', () => {\n if (state.opened && !state.failed && !state.completed && !state.clientDone) {\n state.failed = true;\n push({ type: 'error', reason: 'error', error: 'ElevenLabs transcription socket closed' });\n }\n wake('close');\n });\n await awaitOpen(socket, 'ElevenLabs transcription', (message) => {\n state.failed = true;\n push({ type: 'error', reason: 'error', error: message });\n wake('fail');\n });\n if (state.failed) return;\n state.opened = true;\n if (!state.ready) await readyWait.promise;\n if (state.failed) return;\n if (!first.done) sendChunk(socket, first.value, format, state, false);\n while (!state.failed) {\n if (options?.signal?.aborted) throw new DOMException('aborted', 'AbortError');\n const next = await iterator.next();\n if (next.done) break;\n sendChunk(socket, next.value, format, state, false);\n }\n if (state.failed) return;\n sendChunk(socket, undefined, format, state, true);\n state.clientDone = true;\n await doneWait.promise;\n finish(model, format, state, push);\n } catch (error) {\n const voice = asVoiceError(error);\n state.failed = true;\n push({ type: 'error', reason: voice.reason, error: voice.error });\n } finally {\n socket?.close();\n }\n}\n\nfunction sendChunk(\n socket: WebSocket,\n chunk: AudioChunk | undefined,\n format: AudioFormat,\n state: ListenState,\n commit: boolean\n) {\n if (chunk && !sameFormat(chunk.format, format)) {\n throw new Error(`Unsupported audio format ${chunk.format.encoding}`);\n }\n const data = chunk ? chunk.data : new Uint8Array();\n state.bytes += data.byteLength;\n socket.send(\n JSON.stringify({\n message_type: 'input_audio_chunk',\n audio_base_64: encodeAudioBase64(data),\n commit,\n sample_rate: format.sampleRate,\n })\n );\n}\n\nasync function onMessage(\n data: unknown,\n state: ListenState,\n push: (event: TranscriptionEvent) => void,\n wake: (kind: string | undefined) => void\n) {\n try {\n const parsed = JSON.parse(await messageText(data)) as unknown;\n if (!isRecord(parsed)) throw new Error('ElevenLabs transcription event was not JSON');\n applyListen(parsed, state, push);\n wake(stringField(parsed, 'message_type'));\n } catch (error) {\n const voice = asVoiceError(error);\n state.failed = true;\n push({ type: 'error', reason: voice.reason, error: voice.error });\n wake('fail');\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oBAAoB;AAC7B,SAAS,yBAAyB;AAElC,SAAS,cAAc,UAAU,aAAa,mBAAmB;AACjE;AAAA,EAGE;AAAA,OAEK;AACP,SAAS,kBAAkB;AAC3B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,MAAM,WAAW,oBAAI,IAAI;AAAA,EACvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAgBD,SAAS,MAAM,SAA8C;AAC3D,QAAM,aAAa,SAAS;AAC5B,MAAI,CAAC,WAAY,QAAO,CAAC;AACzB,SAAO;AACT;AAEA,SAAS,eAAe,OAA2C;AACjE,MAAI,MAAM,kBAAkB,WAAY,QAAO;AAC/C,MAAI,MAAM,kBAAkB,MAAO,QAAO;AAC1C,SAAO;AACT;AAEA,SAAS,OACP,OACA,QACA,SACQ;AACR,QAAM,MAAM,UAAU,MAAM,SAAS,yBAAyB;AAC9D,QAAM,SAAS,IAAI,gBAAgB;AACnC,SAAO,IAAI,YAAY,MAAM,EAAE;AAC/B,SAAO,IAAI,gBAAgB,eAAe,MAAM,CAAC;AACjD,SAAO,IAAI,mBAAmB,eAAe,KAAK,CAAC;AACnD,SAAO,IAAI,sBAAsB,MAAM;AACvC,MAAI,SAAS,SAAU,QAAO,IAAI,iBAAiB,QAAQ,QAAQ;AACnE,aAAW,QAAQ,MAAM,OAAO,EAAG,QAAO,OAAO,YAAY,IAAI;AACjE,MAAI,SAAS,OAAO,SAAS;AAC7B,SAAO,IAAI,SAAS;AACtB;AAEA,SAAS,aACP,OACA,OACa;AACb,QAAM,SAAS,UAAU,SAAY,MAAM,aAAa,CAAC,IAAI,MAAM;AACnE,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,iBAAiB;AAC9C,iBAAe,MAAM;AACrB,QAAM,QAAQ,MAAM,aAAa,KAAK,CAAC,SAAS,WAAW,MAAM,MAAM,CAAC;AACxE,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,4BAA4B,OAAO,QAAQ,EAAE;AACzE,SAAO;AACT;AAEA,SAAS,OAAO,OAAwC;AACtD,QAAM,OAAO,YAAY,OAAO,MAAM;AACtC,MAAI,SAAS,OAAW,QAAO;AAC/B,SAAO;AACT;AAEA,SAAS,UAAU,MAA+B,KAAqB;AACrE,QAAM,QAAQ,YAAY,MAAM,GAAG;AACnC,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO;AACT;AAEA,SAAS,SAAS,MAA8D;AAC9E,MAAI,YAAY,MAAM,MAAM,MAAM,UAAW,QAAO;AACpD,QAAM,UAAU,YAAY,MAAM,YAAY;AAC9C,QAAM,UAA6B;AAAA,IACjC,MAAM,OAAO,IAAI;AAAA,IACjB,OAAO,UAAU,MAAM,OAAO;AAAA,IAC9B,KAAK,UAAU,MAAM,KAAK;AAAA,EAC5B;AACA,MAAI,YAAY,OAAW,SAAQ,UAAU;AAC7C,SAAO;AACT;AAEA,SAAS,QAAQ,OAAgC,OAAoB;AACnE,MAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,EAAG;AACjC,aAAW,QAAQ,MAAM,OAAO;AAC9B,QAAI,CAAC,SAAS,IAAI,EAAG;AACrB,UAAM,UAAU,SAAS,IAAI;AAC7B,QAAI,QAAS,OAAM,SAAS,KAAK,OAAO;AAAA,EAC1C;AACF;AAEA,SAAS,WAAW,QAAgB,OAAoB,MAA2C;AACjG,MAAI,CAAC,MAAM,SAAS;AAClB,UAAM,UAAU;AAChB,SAAK,EAAE,MAAM,gBAAgB,IAAI,EAAE,CAAC;AAAA,EACtC;AACA,QAAM,YAAY,OAAO,MAAM,OAAO;AACtC,QAAM,WAAW;AACjB,QAAM,MAAM,KAAK,MAAM;AACvB,OAAK,EAAE,MAAM,SAAS,WAAW,MAAM,OAAO,CAAC;AAC/C,QAAM,UAAU;AAChB,OAAK,EAAE,MAAM,YAAY,MAAM,QAAQ,IAAI,EAAE,CAAC;AAChD;AAEA,SAAS,YACP,OACA,OACA,MACA;AACA,QAAM,OAAO,YAAY,OAAO,cAAc;AAC9C,MAAI,SAAS,mBAAmB;AAC9B,UAAM,QAAQ;AACd;AAAA,EACF;AACA,MAAI,SAAS,sBAAsB;AACjC,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAChB,WAAK,EAAE,MAAM,gBAAgB,IAAI,EAAE,CAAC;AAAA,IACtC;AACA,SAAK,EAAE,MAAM,WAAW,WAAW,OAAO,MAAM,OAAO,GAAG,MAAM,OAAO,KAAK,EAAE,CAAC;AAC/E;AAAA,EACF;AACA,MAAI,SAAS,wBAAwB;AACnC,UAAM,SAAS,OAAO,KAAK;AAC3B,UAAM,aAAa;AACnB,eAAW,QAAQ,OAAO,IAAI;AAC9B;AAAA,EACF;AACA,MAAI,SAAS,wCAAwC;AACnD,UAAM,SAAS,OAAO,KAAK;AAC3B,YAAQ,OAAO,KAAK;AACpB,QAAI,MAAM,eAAe,OAAQ,YAAW,QAAQ,OAAO,IAAI;AAC/D,UAAM,aAAa;AACnB;AAAA,EACF;AACA,MAAI,SAAS,UAAa,SAAS,IAAI,IAAI,GAAG;AAC5C,UAAM,UAAU,YAAY,OAAO,OAAO;AAC1C,UAAM,SAAS;AACf,SAAK;AAAA,MACH,MAAM;AAAA,MACN,QAAQ;AAAA,MACR,OAAO,YAAY,SAAY,oCAAoC;AAAA,IACrE,CAAC;AAAA,EACH;AACF;AAEA,SAAS,OACP,OACA,QACA,OACA,MACA;AACA,MAAI,MAAM,UAAU,MAAM,UAAW;AACrC,QAAM,YAAY;AAClB,QAAM,QAAQ,aAAa,EAAE,MAAM,IAAI,WAAW,MAAM,KAAK,GAAG,QAAQ,WAAW,EAAE,CAAC;AACtF,OAAK;AAAA,IACH,MAAM;AAAA,IACN,YAAY;AAAA,MACV,MAAM,MAAM,MAAM,KAAK,GAAG;AAAA,MAC1B,GAAI,MAAM,SAAS,SAAS,IAAI,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,MAChE,OAAO,WAAW,MAAM,WAAW,EAAE,cAAc,MAAM,CAAC;AAAA,IAC5D;AAAA,EACF,CAAC;AACH;AAEO,SAAS,iBACd,OACA,OACA,SAC0B;AAC1B,QAAM,SAAS,IAAI,yBAAyB;AAC5C,OAAK,WAAW,OAAO,OAAO,SAAS,MAAM;AAC7C,SAAO;AACT;AAEA,eAAe,WACb,OACA,OACA,SACA,QACA;AACA,QAAM,QAAqB;AAAA,IACzB,OAAO,CAAC;AAAA,IACR,UAAU,CAAC;AAAA,IACX,SAAS;AAAA,IACT,SAAS;AAAA,IACT,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,YAAY;AAAA,IACZ,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,YAAY;AAAA,EACd;AACA,MAAI,UAAU;AACd,QAAM,OAAO,CAAC,UAA8B;AAC1C,QAAI,QAAS;AACb,QAAI,MAAM,SAAS,UAAU,MAAM,SAAS,QAAS,WAAU;AAC/D,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,MAAI;AACJ,QAAM,YAAY,QAAQ,cAAoB;AAC9C,QAAM,WAAW,QAAQ,cAAoB;AAC7C,QAAM,OAAO,CAAC,SAA6B;AACzC,QAAI,MAAM,SAAS,MAAM,OAAQ,WAAU,QAAQ;AACnD,QAAI,MAAM,OAAQ,UAAS,QAAQ;AACnC,QAAI,CAAC,MAAM,WAAY;AACvB,QAAI,SAAS,0BAA0B,SAAS,wCAAwC;AACtF,eAAS,QAAQ;AAAA,IACnB;AACA,QAAI,SAAS,QAAS,UAAS,QAAQ;AAAA,EACzC;AACA,MAAI;AACF,QAAI,SAAS,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AAC5E,UAAM,WAAW,MAAM,OAAO,aAAa,EAAE;AAC7C,UAAM,QAAQ,MAAM,SAAS,KAAK;AAClC,UAAM,SAAS,aAAa,OAAO,MAAM,OAAO,SAAY,MAAM,KAAK;AACvE,aAAS,cAAc,OAAO,OAAO,QAAQ,OAAO,GAAG,UAAU,OAAO,OAAO,CAAC;AAChF,QAAI,WAAW,QAAQ,QAAQ;AAC/B,WAAO,iBAAiB,WAAW,CAAC,UAAU;AAC5C,iBAAW,SAAS,KAAK,MAAM,UAAU,MAAM,MAAM,OAAO,MAAM,IAAI,CAAC;AAAA,IACzE,CAAC;AACD,WAAO,iBAAiB,SAAS,MAAM;AACrC,UAAI,MAAM,UAAU,CAAC,MAAM,UAAU,CAAC,MAAM,aAAa,CAAC,MAAM,YAAY;AAC1E,cAAM,SAAS;AACf,aAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,yCAAyC,CAAC;AAAA,MAC1F;AACA,WAAK,OAAO;AAAA,IACd,CAAC;AACD,UAAM,UAAU,QAAQ,4BAA4B,CAAC,YAAY;AAC/D,YAAM,SAAS;AACf,WAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,CAAC;AACvD,WAAK,MAAM;AAAA,IACb,CAAC;AACD,QAAI,MAAM,OAAQ;AAClB,UAAM,SAAS;AACf,QAAI,CAAC,MAAM,MAAO,OAAM,UAAU;AAClC,QAAI,MAAM,OAAQ;AAClB,QAAI,CAAC,MAAM,KAAM,WAAU,QAAQ,MAAM,OAAO,QAAQ,OAAO,KAAK;AACpE,WAAO,CAAC,MAAM,QAAQ;AACpB,UAAI,SAAS,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AAC5E,YAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAI,KAAK,KAAM;AACf,gBAAU,QAAQ,KAAK,OAAO,QAAQ,OAAO,KAAK;AAAA,IACpD;AACA,QAAI,MAAM,OAAQ;AAClB,cAAU,QAAQ,QAAW,QAAQ,OAAO,IAAI;AAChD,UAAM,aAAa;AACnB,UAAM,SAAS;AACf,WAAO,OAAO,QAAQ,OAAO,IAAI;AAAA,EACnC,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,KAAK;AAChC,UAAM,SAAS;AACf,SAAK,EAAE,MAAM,SAAS,QAAQ,MAAM,QAAQ,OAAO,MAAM,MAAM,CAAC;AAAA,EAClE,UAAE;AACA,YAAQ,MAAM;AAAA,EAChB;AACF;AAEA,SAAS,UACP,QACA,OACA,QACA,OACA,QACA;AACA,MAAI,SAAS,CAAC,WAAW,MAAM,QAAQ,MAAM,GAAG;AAC9C,UAAM,IAAI,MAAM,4BAA4B,MAAM,OAAO,QAAQ,EAAE;AAAA,EACrE;AACA,QAAM,OAAO,QAAQ,MAAM,OAAO,IAAI,WAAW;AACjD,QAAM,SAAS,KAAK;AACpB,SAAO;AAAA,IACL,KAAK,UAAU;AAAA,MACb,cAAc;AAAA,MACd,eAAe,kBAAkB,IAAI;AAAA,MACrC;AAAA,MACA,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AACF;AAEA,eAAe,UACb,MACA,OACA,MACA,MACA;AACA,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,MAAM,YAAY,IAAI,CAAC;AACjD,QAAI,CAAC,SAAS,MAAM,EAAG,OAAM,IAAI,MAAM,6CAA6C;AACpF,gBAAY,QAAQ,OAAO,IAAI;AAC/B,SAAK,YAAY,QAAQ,cAAc,CAAC;AAAA,EAC1C,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,KAAK;AAChC,UAAM,SAAS;AACf,SAAK,EAAE,MAAM,SAAS,QAAQ,MAAM,QAAQ,OAAO,MAAM,MAAM,CAAC;AAChE,SAAK,MAAM;AAAA,EACb;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { chunkSeconds } from "../audio.js";
|
|
2
|
+
import { decodeAudioBase64 } from "../convert.js";
|
|
3
|
+
import { VoiceRequestError } from "../errors.js";
|
|
4
|
+
import {
|
|
5
|
+
applyPayload,
|
|
6
|
+
asVoiceError,
|
|
7
|
+
endpoint,
|
|
8
|
+
isRecord,
|
|
9
|
+
readJson,
|
|
10
|
+
requestSignal,
|
|
11
|
+
stringField
|
|
12
|
+
} from "../openai/wire.js";
|
|
13
|
+
import { SpeechEventStream } from "../speech.js";
|
|
14
|
+
import { priceAudio } from "../usage.js";
|
|
15
|
+
import {
|
|
16
|
+
alignmentFrom,
|
|
17
|
+
awaitOpen,
|
|
18
|
+
connectSocket,
|
|
19
|
+
messageText,
|
|
20
|
+
outputFormatName,
|
|
21
|
+
sameFormat,
|
|
22
|
+
socketUrl,
|
|
23
|
+
xiHeaders
|
|
24
|
+
} from "./wire.js";
|
|
25
|
+
function languageCode(language) {
|
|
26
|
+
if (language === void 0) return void 0;
|
|
27
|
+
if (language.trim() === "") return void 0;
|
|
28
|
+
return language;
|
|
29
|
+
}
|
|
30
|
+
function resolveFormat(model, requested) {
|
|
31
|
+
const fallback = model.outputFormats[0];
|
|
32
|
+
if (!fallback) throw new Error("No output format");
|
|
33
|
+
if (!requested) return fallback;
|
|
34
|
+
const match = model.outputFormats.find((format) => sameFormat(format, requested));
|
|
35
|
+
if (!match) throw new Error(`Unsupported speech format ${requested.encoding}`);
|
|
36
|
+
return match;
|
|
37
|
+
}
|
|
38
|
+
function speechBody(model, text, language) {
|
|
39
|
+
const body = { text, model_id: model.id };
|
|
40
|
+
const code = languageCode(language);
|
|
41
|
+
if (code !== void 0 && model.id !== "eleven_multilingual_v2") body.language_code = code;
|
|
42
|
+
return body;
|
|
43
|
+
}
|
|
44
|
+
function speechUrl(model, voice, format) {
|
|
45
|
+
const url = socketUrl(model.baseUrl, `text-to-speech/${encodeURIComponent(voice)}/stream-input`);
|
|
46
|
+
const params = new URLSearchParams();
|
|
47
|
+
params.set("model_id", model.id);
|
|
48
|
+
params.set("output_format", outputFormatName(format));
|
|
49
|
+
params.set("sync_alignment", "true");
|
|
50
|
+
url.search = params.toString();
|
|
51
|
+
return url.toString();
|
|
52
|
+
}
|
|
53
|
+
async function readText(text, signal) {
|
|
54
|
+
if (typeof text === "string") return text;
|
|
55
|
+
let combined = "";
|
|
56
|
+
for await (const part of text) {
|
|
57
|
+
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
|
58
|
+
combined += part;
|
|
59
|
+
}
|
|
60
|
+
return combined;
|
|
61
|
+
}
|
|
62
|
+
async function xiFetch(model, options, url, body) {
|
|
63
|
+
const headers = xiHeaders(model, options);
|
|
64
|
+
headers["Content-Type"] = "application/json";
|
|
65
|
+
const fetchImpl = options?.fetch ?? globalThis.fetch;
|
|
66
|
+
const response = await fetchImpl(url, {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers,
|
|
69
|
+
body,
|
|
70
|
+
signal: requestSignal(options)
|
|
71
|
+
});
|
|
72
|
+
await options?.onResponse?.({ status: response.status, headers: {} }, model);
|
|
73
|
+
if (!response.ok) {
|
|
74
|
+
const text = await response.text();
|
|
75
|
+
throw new Error(text === "" ? `HTTP ${response.status}` : text);
|
|
76
|
+
}
|
|
77
|
+
return response;
|
|
78
|
+
}
|
|
79
|
+
function speakElevenLabs(model, input, options) {
|
|
80
|
+
const stream = new SpeechEventStream();
|
|
81
|
+
void pumpSpeech(model, input, options, stream);
|
|
82
|
+
return stream;
|
|
83
|
+
}
|
|
84
|
+
async function pumpSpeech(model, input, options, stream) {
|
|
85
|
+
try {
|
|
86
|
+
if (options?.signal?.aborted) throw new DOMException("aborted", "AbortError");
|
|
87
|
+
const format = resolveFormat(model, input.format);
|
|
88
|
+
if (model.textStreaming && typeof input.text !== "string") {
|
|
89
|
+
await streamSpeech(model, input.text, input.voice, format, options, stream);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
const text = await readText(input.text, options?.signal);
|
|
93
|
+
await postSpeech(model, text, input.voice, input.language, format, options, stream);
|
|
94
|
+
} catch (error) {
|
|
95
|
+
const voice = asVoiceError(error);
|
|
96
|
+
stream.push({ type: "error", reason: voice.reason, error: voice.error });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async function postSpeech(model, text, voice, language, format, options, stream) {
|
|
100
|
+
const payload = speechBody(model, text, language);
|
|
101
|
+
const body = await applyPayload(payload, model, options);
|
|
102
|
+
if (!isRecord(body)) throw new Error("Speech payload was not JSON");
|
|
103
|
+
const query = `output_format=${outputFormatName(format)}`;
|
|
104
|
+
const path = `text-to-speech/${encodeURIComponent(voice)}/with-timestamps?${query}`;
|
|
105
|
+
const response = await xiFetch(
|
|
106
|
+
model,
|
|
107
|
+
options,
|
|
108
|
+
endpoint(model.baseUrl, path),
|
|
109
|
+
JSON.stringify(body)
|
|
110
|
+
);
|
|
111
|
+
const parsed = await readJson(response);
|
|
112
|
+
if (!isRecord(parsed)) throw new Error("Speech response had no audio");
|
|
113
|
+
const audio = stringField(parsed, "audio_base64");
|
|
114
|
+
if (audio === void 0) throw new Error("Speech response had no audio");
|
|
115
|
+
const data = decodeAudioBase64(audio);
|
|
116
|
+
const seconds = chunkSeconds({ data, format, timestamp: 0 });
|
|
117
|
+
const meters = { characters: text.length };
|
|
118
|
+
if (seconds > 0) meters.outputSeconds = seconds;
|
|
119
|
+
const alignment = alignmentFrom(parsed.alignment);
|
|
120
|
+
stream.push({ type: "start", format });
|
|
121
|
+
stream.push({
|
|
122
|
+
type: "audio",
|
|
123
|
+
chunk: { data, format, timestamp: 0 },
|
|
124
|
+
span: { itemId: model.id, start: 0, end: text.length },
|
|
125
|
+
...alignment ? { alignment } : {}
|
|
126
|
+
});
|
|
127
|
+
stream.push({ type: "done", usage: priceAudio(model.audioCost, meters) });
|
|
128
|
+
}
|
|
129
|
+
async function streamSpeech(model, text, voice, format, options, stream) {
|
|
130
|
+
const state = {
|
|
131
|
+
opened: false,
|
|
132
|
+
started: false,
|
|
133
|
+
done: false,
|
|
134
|
+
failed: false,
|
|
135
|
+
outputSeconds: 0,
|
|
136
|
+
characters: 0
|
|
137
|
+
};
|
|
138
|
+
let settled = false;
|
|
139
|
+
const push = (event) => {
|
|
140
|
+
if (settled) return;
|
|
141
|
+
if (event.type === "done" || event.type === "error") settled = true;
|
|
142
|
+
stream.push(event);
|
|
143
|
+
};
|
|
144
|
+
const doneWait = Promise.withResolvers();
|
|
145
|
+
const wake = () => {
|
|
146
|
+
doneWait.resolve();
|
|
147
|
+
};
|
|
148
|
+
const socket = connectSocket(speechUrl(model, voice, format), xiHeaders(model, options));
|
|
149
|
+
let incoming = Promise.resolve();
|
|
150
|
+
socket.addEventListener("message", (event) => {
|
|
151
|
+
incoming = incoming.then(() => onMessage(event.data, model.id, format, state, push, wake));
|
|
152
|
+
});
|
|
153
|
+
socket.addEventListener("close", () => {
|
|
154
|
+
if (state.opened && !state.done && !state.failed) {
|
|
155
|
+
state.failed = true;
|
|
156
|
+
push({ type: "error", reason: "error", error: "ElevenLabs speech socket closed" });
|
|
157
|
+
}
|
|
158
|
+
wake();
|
|
159
|
+
});
|
|
160
|
+
try {
|
|
161
|
+
await awaitOpen(socket, "ElevenLabs speech", (message) => {
|
|
162
|
+
state.failed = true;
|
|
163
|
+
push({ type: "error", reason: "error", error: message });
|
|
164
|
+
wake();
|
|
165
|
+
});
|
|
166
|
+
if (state.failed) return;
|
|
167
|
+
state.opened = true;
|
|
168
|
+
state.characters = await writeText(socket, text, options?.signal);
|
|
169
|
+
if (state.failed) return;
|
|
170
|
+
socket.send(JSON.stringify({ text: " ", flush: true }));
|
|
171
|
+
socket.send(JSON.stringify({ text: "" }));
|
|
172
|
+
if (!state.done) await doneWait.promise;
|
|
173
|
+
if (state.failed) return;
|
|
174
|
+
const meters = { characters: state.characters };
|
|
175
|
+
if (state.outputSeconds > 0) meters.outputSeconds = state.outputSeconds;
|
|
176
|
+
push({ type: "done", usage: priceAudio(model.audioCost, meters) });
|
|
177
|
+
} catch (error) {
|
|
178
|
+
const voice2 = asVoiceError(error);
|
|
179
|
+
state.failed = true;
|
|
180
|
+
push({ type: "error", reason: voice2.reason, error: voice2.error });
|
|
181
|
+
} finally {
|
|
182
|
+
socket.close();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
async function writeText(socket, text, signal) {
|
|
186
|
+
let combined = "";
|
|
187
|
+
socket.send(
|
|
188
|
+
JSON.stringify({
|
|
189
|
+
text: " ",
|
|
190
|
+
generation_config: { chunk_length_schedule: [50, 120, 160, 290] }
|
|
191
|
+
})
|
|
192
|
+
);
|
|
193
|
+
for await (const part of text) {
|
|
194
|
+
if (signal?.aborted) throw new VoiceRequestError("aborted", "aborted");
|
|
195
|
+
combined += part;
|
|
196
|
+
if (part === "") continue;
|
|
197
|
+
const delta = part.endsWith(" ") ? part : `${part} `;
|
|
198
|
+
socket.send(JSON.stringify({ text: delta }));
|
|
199
|
+
}
|
|
200
|
+
return combined.length;
|
|
201
|
+
}
|
|
202
|
+
async function onMessage(data, itemId, format, state, push, wake) {
|
|
203
|
+
try {
|
|
204
|
+
const parsed = JSON.parse(await messageText(data));
|
|
205
|
+
if (!isRecord(parsed)) throw new Error("ElevenLabs speech event was not JSON");
|
|
206
|
+
applySpeech(parsed, itemId, format, state, push);
|
|
207
|
+
if (state.done || state.failed) wake();
|
|
208
|
+
} catch (error) {
|
|
209
|
+
const voice = asVoiceError(error);
|
|
210
|
+
state.failed = true;
|
|
211
|
+
push({ type: "error", reason: voice.reason, error: voice.error });
|
|
212
|
+
wake();
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function audioEvent(itemId, format, data, end, alignment) {
|
|
216
|
+
return {
|
|
217
|
+
type: "audio",
|
|
218
|
+
chunk: { data, format, timestamp: 0 },
|
|
219
|
+
span: { itemId, start: 0, end },
|
|
220
|
+
...alignment ? { alignment } : {}
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
function applySpeech(event, itemId, format, state, push) {
|
|
224
|
+
if ("error" in event) {
|
|
225
|
+
const message = stringField(event, "error");
|
|
226
|
+
state.failed = true;
|
|
227
|
+
push({
|
|
228
|
+
type: "error",
|
|
229
|
+
reason: "error",
|
|
230
|
+
error: message === void 0 ? "ElevenLabs speech failed" : message
|
|
231
|
+
});
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
if (event.isFinal === true) {
|
|
235
|
+
state.done = true;
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
const audio = stringField(event, "audio");
|
|
239
|
+
if (audio === void 0 || audio === "") return;
|
|
240
|
+
const data = decodeAudioBase64(audio);
|
|
241
|
+
if (!state.started) {
|
|
242
|
+
state.started = true;
|
|
243
|
+
push({ type: "start", format });
|
|
244
|
+
}
|
|
245
|
+
state.outputSeconds += chunkSeconds({ data, format, timestamp: 0 });
|
|
246
|
+
push(audioEvent(itemId, format, data, state.characters, alignmentFrom(event.alignment)));
|
|
247
|
+
}
|
|
248
|
+
export {
|
|
249
|
+
speakElevenLabs
|
|
250
|
+
};
|
|
251
|
+
//# sourceMappingURL=speak.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/elevenlabs/speak.ts"],
|
|
4
|
+
"sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioFormat, CharAlignment } from '../audio.ts';\nimport { chunkSeconds } from '../audio.ts';\nimport { decodeAudioBase64 } from '../convert.ts';\nimport { VoiceRequestError } from '../errors.ts';\nimport type { SpeechModel } from '../models.ts';\nimport {\n applyPayload,\n asVoiceError,\n endpoint,\n isRecord,\n readJson,\n requestSignal,\n stringField,\n} from '../openai/wire.ts';\nimport { type SpeechEvent, SpeechEventStream, type SpeechInput } from '../speech.ts';\nimport { type AudioCounters, priceAudio } from '../usage.ts';\nimport {\n alignmentFrom,\n awaitOpen,\n connectSocket,\n messageText,\n outputFormatName,\n sameFormat,\n socketUrl,\n xiHeaders,\n} from './wire.ts';\n\ninterface SpeechState {\n opened: boolean;\n started: boolean;\n done: boolean;\n failed: boolean;\n outputSeconds: number;\n characters: number;\n}\n\nfunction languageCode(language: string | undefined): string | undefined {\n if (language === undefined) return undefined;\n if (language.trim() === '') return undefined;\n return language;\n}\n\nfunction resolveFormat(\n model: SpeechModel<string>,\n requested: AudioFormat | undefined\n): AudioFormat {\n const fallback = model.outputFormats[0];\n if (!fallback) throw new Error('No output format');\n if (!requested) return fallback;\n const match = model.outputFormats.find((format) => sameFormat(format, requested));\n if (!match) throw new Error(`Unsupported speech format ${requested.encoding}`);\n return match;\n}\n\nfunction speechBody(\n model: SpeechModel<string>,\n text: string,\n language: string | undefined\n): Record<string, unknown> {\n const body: Record<string, unknown> = { text, model_id: model.id };\n const code = languageCode(language);\n if (code !== undefined && model.id !== 'eleven_multilingual_v2') body.language_code = code;\n return body;\n}\n\nfunction speechUrl(model: SpeechModel<string>, voice: string, format: AudioFormat): string {\n const url = socketUrl(model.baseUrl, `text-to-speech/${encodeURIComponent(voice)}/stream-input`);\n const params = new URLSearchParams();\n params.set('model_id', model.id);\n params.set('output_format', outputFormatName(format));\n params.set('sync_alignment', 'true');\n url.search = params.toString();\n return url.toString();\n}\n\nasync function readText(\n text: string | AsyncIterable<string>,\n signal: AbortSignal | undefined\n): Promise<string> {\n if (typeof text === 'string') return text;\n let combined = '';\n for await (const part of text) {\n if (signal?.aborted) throw new DOMException('aborted', 'AbortError');\n combined += part;\n }\n return combined;\n}\n\nasync function xiFetch(\n model: SpeechModel<string>,\n options: ProviderRequestOptions | undefined,\n url: string,\n body: string\n): Promise<Response> {\n const headers = xiHeaders(model, options);\n headers['Content-Type'] = 'application/json';\n const fetchImpl = options?.fetch ?? globalThis.fetch;\n const response = await fetchImpl(url, {\n method: 'POST',\n headers,\n body,\n signal: requestSignal(options),\n });\n await options?.onResponse?.({ status: response.status, headers: {} }, model as never);\n if (!response.ok) {\n const text = await response.text();\n throw new Error(text === '' ? `HTTP ${response.status}` : text);\n }\n return response;\n}\n\nexport function speakElevenLabs(\n model: SpeechModel<string>,\n input: SpeechInput,\n options?: ProviderRequestOptions\n): SpeechEventStream {\n const stream = new SpeechEventStream();\n void pumpSpeech(model, input, options, stream);\n return stream;\n}\n\nasync function pumpSpeech(\n model: SpeechModel<string>,\n input: SpeechInput,\n options: ProviderRequestOptions | undefined,\n stream: SpeechEventStream\n) {\n try {\n if (options?.signal?.aborted) throw new DOMException('aborted', 'AbortError');\n const format = resolveFormat(model, input.format);\n if (model.textStreaming && typeof input.text !== 'string') {\n await streamSpeech(model, input.text, input.voice, format, options, stream);\n return;\n }\n const text = await readText(input.text, options?.signal);\n await postSpeech(model, text, input.voice, input.language, format, options, stream);\n } catch (error) {\n const voice = asVoiceError(error);\n stream.push({ type: 'error', reason: voice.reason, error: voice.error });\n }\n}\n\nasync function postSpeech(\n model: SpeechModel<string>,\n text: string,\n voice: string,\n language: string | undefined,\n format: AudioFormat,\n options: ProviderRequestOptions | undefined,\n stream: SpeechEventStream\n) {\n const payload = speechBody(model, text, language);\n const body = await applyPayload(payload, model, options);\n if (!isRecord(body)) throw new Error('Speech payload was not JSON');\n const query = `output_format=${outputFormatName(format)}`;\n const path = `text-to-speech/${encodeURIComponent(voice)}/with-timestamps?${query}`;\n const response = await xiFetch(\n model,\n options,\n endpoint(model.baseUrl, path),\n JSON.stringify(body)\n );\n const parsed = await readJson(response);\n if (!isRecord(parsed)) throw new Error('Speech response had no audio');\n const audio = stringField(parsed, 'audio_base64');\n if (audio === undefined) throw new Error('Speech response had no audio');\n const data = decodeAudioBase64(audio);\n const seconds = chunkSeconds({ data, format, timestamp: 0 });\n const meters: AudioCounters = { characters: text.length };\n if (seconds > 0) meters.outputSeconds = seconds;\n const alignment = alignmentFrom(parsed.alignment);\n stream.push({ type: 'start', format });\n stream.push({\n type: 'audio',\n chunk: { data, format, timestamp: 0 },\n span: { itemId: model.id, start: 0, end: text.length },\n ...(alignment ? { alignment } : {}),\n });\n stream.push({ type: 'done', usage: priceAudio(model.audioCost, meters) });\n}\n\nasync function streamSpeech(\n model: SpeechModel<string>,\n text: AsyncIterable<string>,\n voice: string,\n format: AudioFormat,\n options: ProviderRequestOptions | undefined,\n stream: SpeechEventStream\n) {\n const state: SpeechState = {\n opened: false,\n started: false,\n done: false,\n failed: false,\n outputSeconds: 0,\n characters: 0,\n };\n let settled = false;\n const push = (event: SpeechEvent) => {\n if (settled) return;\n if (event.type === 'done' || event.type === 'error') settled = true;\n stream.push(event);\n };\n const doneWait = Promise.withResolvers<void>();\n const wake = () => {\n doneWait.resolve();\n };\n const socket = connectSocket(speechUrl(model, voice, format), xiHeaders(model, options));\n let incoming = Promise.resolve();\n socket.addEventListener('message', (event) => {\n incoming = incoming.then(() => onMessage(event.data, model.id, format, state, push, wake));\n });\n socket.addEventListener('close', () => {\n if (state.opened && !state.done && !state.failed) {\n state.failed = true;\n push({ type: 'error', reason: 'error', error: 'ElevenLabs speech socket closed' });\n }\n wake();\n });\n try {\n await awaitOpen(socket, 'ElevenLabs speech', (message) => {\n state.failed = true;\n push({ type: 'error', reason: 'error', error: message });\n wake();\n });\n if (state.failed) return;\n state.opened = true;\n state.characters = await writeText(socket, text, options?.signal);\n if (state.failed) return;\n socket.send(JSON.stringify({ text: ' ', flush: true }));\n socket.send(JSON.stringify({ text: '' }));\n if (!state.done) await doneWait.promise;\n if (state.failed) return;\n const meters: AudioCounters = { characters: state.characters };\n if (state.outputSeconds > 0) meters.outputSeconds = state.outputSeconds;\n push({ type: 'done', usage: priceAudio(model.audioCost, meters) });\n } catch (error) {\n const voice = asVoiceError(error);\n state.failed = true;\n push({ type: 'error', reason: voice.reason, error: voice.error });\n } finally {\n socket.close();\n }\n}\n\nasync function writeText(\n socket: WebSocket,\n text: AsyncIterable<string>,\n signal: AbortSignal | undefined\n): Promise<number> {\n let combined = '';\n socket.send(\n JSON.stringify({\n text: ' ',\n generation_config: { chunk_length_schedule: [50, 120, 160, 290] },\n })\n );\n for await (const part of text) {\n if (signal?.aborted) throw new VoiceRequestError('aborted', 'aborted');\n combined += part;\n if (part === '') continue;\n const delta = part.endsWith(' ') ? part : `${part} `;\n socket.send(JSON.stringify({ text: delta }));\n }\n return combined.length;\n}\n\nasync function onMessage(\n data: unknown,\n itemId: string,\n format: AudioFormat,\n state: SpeechState,\n push: (event: SpeechEvent) => void,\n wake: () => void\n) {\n try {\n const parsed = JSON.parse(await messageText(data)) as unknown;\n if (!isRecord(parsed)) throw new Error('ElevenLabs speech event was not JSON');\n applySpeech(parsed, itemId, format, state, push);\n if (state.done || state.failed) wake();\n } catch (error) {\n const voice = asVoiceError(error);\n state.failed = true;\n push({ type: 'error', reason: voice.reason, error: voice.error });\n wake();\n }\n}\n\nfunction audioEvent(\n itemId: string,\n format: AudioFormat,\n data: Uint8Array,\n end: number,\n alignment: CharAlignment | undefined\n): SpeechEvent {\n return {\n type: 'audio',\n chunk: { data, format, timestamp: 0 },\n span: { itemId, start: 0, end },\n ...(alignment ? { alignment } : {}),\n };\n}\n\nfunction applySpeech(\n event: Record<string, unknown>,\n itemId: string,\n format: AudioFormat,\n state: SpeechState,\n push: (event: SpeechEvent) => void\n) {\n if ('error' in event) {\n const message = stringField(event, 'error');\n state.failed = true;\n push({\n type: 'error',\n reason: 'error',\n error: message === undefined ? 'ElevenLabs speech failed' : message,\n });\n return;\n }\n if (event.isFinal === true) {\n state.done = true;\n return;\n }\n const audio = stringField(event, 'audio');\n if (audio === undefined || audio === '') return;\n const data = decodeAudioBase64(audio);\n if (!state.started) {\n state.started = true;\n push({ type: 'start', format });\n }\n state.outputSeconds += chunkSeconds({ data, format, timestamp: 0 });\n push(audioEvent(itemId, format, data, state.characters, alignmentFrom(event.alignment)));\n}\n"],
|
|
5
|
+
"mappings": "AAEA,SAAS,oBAAoB;AAC7B,SAAS,yBAAyB;AAClC,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAA2B,yBAA2C;AACtE,SAA6B,kBAAkB;AAC/C;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAWP,SAAS,aAAa,UAAkD;AACtE,MAAI,aAAa,OAAW,QAAO;AACnC,MAAI,SAAS,KAAK,MAAM,GAAI,QAAO;AACnC,SAAO;AACT;AAEA,SAAS,cACP,OACA,WACa;AACb,QAAM,WAAW,MAAM,cAAc,CAAC;AACtC,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,kBAAkB;AACjD,MAAI,CAAC,UAAW,QAAO;AACvB,QAAM,QAAQ,MAAM,cAAc,KAAK,CAAC,WAAW,WAAW,QAAQ,SAAS,CAAC;AAChF,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,6BAA6B,UAAU,QAAQ,EAAE;AAC7E,SAAO;AACT;AAEA,SAAS,WACP,OACA,MACA,UACyB;AACzB,QAAM,OAAgC,EAAE,MAAM,UAAU,MAAM,GAAG;AACjE,QAAM,OAAO,aAAa,QAAQ;AAClC,MAAI,SAAS,UAAa,MAAM,OAAO,yBAA0B,MAAK,gBAAgB;AACtF,SAAO;AACT;AAEA,SAAS,UAAU,OAA4B,OAAe,QAA6B;AACzF,QAAM,MAAM,UAAU,MAAM,SAAS,kBAAkB,mBAAmB,KAAK,CAAC,eAAe;AAC/F,QAAM,SAAS,IAAI,gBAAgB;AACnC,SAAO,IAAI,YAAY,MAAM,EAAE;AAC/B,SAAO,IAAI,iBAAiB,iBAAiB,MAAM,CAAC;AACpD,SAAO,IAAI,kBAAkB,MAAM;AACnC,MAAI,SAAS,OAAO,SAAS;AAC7B,SAAO,IAAI,SAAS;AACtB;AAEA,eAAe,SACb,MACA,QACiB;AACjB,MAAI,OAAO,SAAS,SAAU,QAAO;AACrC,MAAI,WAAW;AACf,mBAAiB,QAAQ,MAAM;AAC7B,QAAI,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AACnE,gBAAY;AAAA,EACd;AACA,SAAO;AACT;AAEA,eAAe,QACb,OACA,SACA,KACA,MACmB;AACnB,QAAM,UAAU,UAAU,OAAO,OAAO;AACxC,UAAQ,cAAc,IAAI;AAC1B,QAAM,YAAY,SAAS,SAAS,WAAW;AAC/C,QAAM,WAAW,MAAM,UAAU,KAAK;AAAA,IACpC,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA,QAAQ,cAAc,OAAO;AAAA,EAC/B,CAAC;AACD,QAAM,SAAS,aAAa,EAAE,QAAQ,SAAS,QAAQ,SAAS,CAAC,EAAE,GAAG,KAAc;AACpF,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI,MAAM,SAAS,KAAK,QAAQ,SAAS,MAAM,KAAK,IAAI;AAAA,EAChE;AACA,SAAO;AACT;AAEO,SAAS,gBACd,OACA,OACA,SACmB;AACnB,QAAM,SAAS,IAAI,kBAAkB;AACrC,OAAK,WAAW,OAAO,OAAO,SAAS,MAAM;AAC7C,SAAO;AACT;AAEA,eAAe,WACb,OACA,OACA,SACA,QACA;AACA,MAAI;AACF,QAAI,SAAS,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AAC5E,UAAM,SAAS,cAAc,OAAO,MAAM,MAAM;AAChD,QAAI,MAAM,iBAAiB,OAAO,MAAM,SAAS,UAAU;AACzD,YAAM,aAAa,OAAO,MAAM,MAAM,MAAM,OAAO,QAAQ,SAAS,MAAM;AAC1E;AAAA,IACF;AACA,UAAM,OAAO,MAAM,SAAS,MAAM,MAAM,SAAS,MAAM;AACvD,UAAM,WAAW,OAAO,MAAM,MAAM,OAAO,MAAM,UAAU,QAAQ,SAAS,MAAM;AAAA,EACpF,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,KAAK;AAChC,WAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,MAAM,QAAQ,OAAO,MAAM,MAAM,CAAC;AAAA,EACzE;AACF;AAEA,eAAe,WACb,OACA,MACA,OACA,UACA,QACA,SACA,QACA;AACA,QAAM,UAAU,WAAW,OAAO,MAAM,QAAQ;AAChD,QAAM,OAAO,MAAM,aAAa,SAAS,OAAO,OAAO;AACvD,MAAI,CAAC,SAAS,IAAI,EAAG,OAAM,IAAI,MAAM,6BAA6B;AAClE,QAAM,QAAQ,iBAAiB,iBAAiB,MAAM,CAAC;AACvD,QAAM,OAAO,kBAAkB,mBAAmB,KAAK,CAAC,oBAAoB,KAAK;AACjF,QAAM,WAAW,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,IACA,SAAS,MAAM,SAAS,IAAI;AAAA,IAC5B,KAAK,UAAU,IAAI;AAAA,EACrB;AACA,QAAM,SAAS,MAAM,SAAS,QAAQ;AACtC,MAAI,CAAC,SAAS,MAAM,EAAG,OAAM,IAAI,MAAM,8BAA8B;AACrE,QAAM,QAAQ,YAAY,QAAQ,cAAc;AAChD,MAAI,UAAU,OAAW,OAAM,IAAI,MAAM,8BAA8B;AACvE,QAAM,OAAO,kBAAkB,KAAK;AACpC,QAAM,UAAU,aAAa,EAAE,MAAM,QAAQ,WAAW,EAAE,CAAC;AAC3D,QAAM,SAAwB,EAAE,YAAY,KAAK,OAAO;AACxD,MAAI,UAAU,EAAG,QAAO,gBAAgB;AACxC,QAAM,YAAY,cAAc,OAAO,SAAS;AAChD,SAAO,KAAK,EAAE,MAAM,SAAS,OAAO,CAAC;AACrC,SAAO,KAAK;AAAA,IACV,MAAM;AAAA,IACN,OAAO,EAAE,MAAM,QAAQ,WAAW,EAAE;AAAA,IACpC,MAAM,EAAE,QAAQ,MAAM,IAAI,OAAO,GAAG,KAAK,KAAK,OAAO;AAAA,IACrD,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;AAAA,EACnC,CAAC;AACD,SAAO,KAAK,EAAE,MAAM,QAAQ,OAAO,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;AAC1E;AAEA,eAAe,aACb,OACA,MACA,OACA,QACA,SACA,QACA;AACA,QAAM,QAAqB;AAAA,IACzB,QAAQ;AAAA,IACR,SAAS;AAAA,IACT,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,YAAY;AAAA,EACd;AACA,MAAI,UAAU;AACd,QAAM,OAAO,CAAC,UAAuB;AACnC,QAAI,QAAS;AACb,QAAI,MAAM,SAAS,UAAU,MAAM,SAAS,QAAS,WAAU;AAC/D,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,QAAM,WAAW,QAAQ,cAAoB;AAC7C,QAAM,OAAO,MAAM;AACjB,aAAS,QAAQ;AAAA,EACnB;AACA,QAAM,SAAS,cAAc,UAAU,OAAO,OAAO,MAAM,GAAG,UAAU,OAAO,OAAO,CAAC;AACvF,MAAI,WAAW,QAAQ,QAAQ;AAC/B,SAAO,iBAAiB,WAAW,CAAC,UAAU;AAC5C,eAAW,SAAS,KAAK,MAAM,UAAU,MAAM,MAAM,MAAM,IAAI,QAAQ,OAAO,MAAM,IAAI,CAAC;AAAA,EAC3F,CAAC;AACD,SAAO,iBAAiB,SAAS,MAAM;AACrC,QAAI,MAAM,UAAU,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ;AAChD,YAAM,SAAS;AACf,WAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,kCAAkC,CAAC;AAAA,IACnF;AACA,SAAK;AAAA,EACP,CAAC;AACD,MAAI;AACF,UAAM,UAAU,QAAQ,qBAAqB,CAAC,YAAY;AACxD,YAAM,SAAS;AACf,WAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,CAAC;AACvD,WAAK;AAAA,IACP,CAAC;AACD,QAAI,MAAM,OAAQ;AAClB,UAAM,SAAS;AACf,UAAM,aAAa,MAAM,UAAU,QAAQ,MAAM,SAAS,MAAM;AAChE,QAAI,MAAM,OAAQ;AAClB,WAAO,KAAK,KAAK,UAAU,EAAE,MAAM,KAAK,OAAO,KAAK,CAAC,CAAC;AACtD,WAAO,KAAK,KAAK,UAAU,EAAE,MAAM,GAAG,CAAC,CAAC;AACxC,QAAI,CAAC,MAAM,KAAM,OAAM,SAAS;AAChC,QAAI,MAAM,OAAQ;AAClB,UAAM,SAAwB,EAAE,YAAY,MAAM,WAAW;AAC7D,QAAI,MAAM,gBAAgB,EAAG,QAAO,gBAAgB,MAAM;AAC1D,SAAK,EAAE,MAAM,QAAQ,OAAO,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;AAAA,EACnE,SAAS,OAAO;AACd,UAAMA,SAAQ,aAAa,KAAK;AAChC,UAAM,SAAS;AACf,SAAK,EAAE,MAAM,SAAS,QAAQA,OAAM,QAAQ,OAAOA,OAAM,MAAM,CAAC;AAAA,EAClE,UAAE;AACA,WAAO,MAAM;AAAA,EACf;AACF;AAEA,eAAe,UACb,QACA,MACA,QACiB;AACjB,MAAI,WAAW;AACf,SAAO;AAAA,IACL,KAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,mBAAmB,EAAE,uBAAuB,CAAC,IAAI,KAAK,KAAK,GAAG,EAAE;AAAA,IAClE,CAAC;AAAA,EACH;AACA,mBAAiB,QAAQ,MAAM;AAC7B,QAAI,QAAQ,QAAS,OAAM,IAAI,kBAAkB,WAAW,SAAS;AACrE,gBAAY;AACZ,QAAI,SAAS,GAAI;AACjB,UAAM,QAAQ,KAAK,SAAS,GAAG,IAAI,OAAO,GAAG,IAAI;AACjD,WAAO,KAAK,KAAK,UAAU,EAAE,MAAM,MAAM,CAAC,CAAC;AAAA,EAC7C;AACA,SAAO,SAAS;AAClB;AAEA,eAAe,UACb,MACA,QACA,QACA,OACA,MACA,MACA;AACA,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,MAAM,YAAY,IAAI,CAAC;AACjD,QAAI,CAAC,SAAS,MAAM,EAAG,OAAM,IAAI,MAAM,sCAAsC;AAC7E,gBAAY,QAAQ,QAAQ,QAAQ,OAAO,IAAI;AAC/C,QAAI,MAAM,QAAQ,MAAM,OAAQ,MAAK;AAAA,EACvC,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,KAAK;AAChC,UAAM,SAAS;AACf,SAAK,EAAE,MAAM,SAAS,QAAQ,MAAM,QAAQ,OAAO,MAAM,MAAM,CAAC;AAChE,SAAK;AAAA,EACP;AACF;AAEA,SAAS,WACP,QACA,QACA,MACA,KACA,WACa;AACb,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,EAAE,MAAM,QAAQ,WAAW,EAAE;AAAA,IACpC,MAAM,EAAE,QAAQ,OAAO,GAAG,IAAI;AAAA,IAC9B,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;AAAA,EACnC;AACF;AAEA,SAAS,YACP,OACA,QACA,QACA,OACA,MACA;AACA,MAAI,WAAW,OAAO;AACpB,UAAM,UAAU,YAAY,OAAO,OAAO;AAC1C,UAAM,SAAS;AACf,SAAK;AAAA,MACH,MAAM;AAAA,MACN,QAAQ;AAAA,MACR,OAAO,YAAY,SAAY,6BAA6B;AAAA,IAC9D,CAAC;AACD;AAAA,EACF;AACA,MAAI,MAAM,YAAY,MAAM;AAC1B,UAAM,OAAO;AACb;AAAA,EACF;AACA,QAAM,QAAQ,YAAY,OAAO,OAAO;AACxC,MAAI,UAAU,UAAa,UAAU,GAAI;AACzC,QAAM,OAAO,kBAAkB,KAAK;AACpC,MAAI,CAAC,MAAM,SAAS;AAClB,UAAM,UAAU;AAChB,SAAK,EAAE,MAAM,SAAS,OAAO,CAAC;AAAA,EAChC;AACA,QAAM,iBAAiB,aAAa,EAAE,MAAM,QAAQ,WAAW,EAAE,CAAC;AAClE,OAAK,WAAW,QAAQ,QAAQ,MAAM,MAAM,YAAY,cAAc,MAAM,SAAS,CAAC,CAAC;AACzF;",
|
|
6
|
+
"names": ["voice"]
|
|
7
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { VoiceRequestError } from "../errors.js";
|
|
2
|
+
import {
|
|
3
|
+
applyPayload,
|
|
4
|
+
asVoiceError,
|
|
5
|
+
endpoint,
|
|
6
|
+
isRecord,
|
|
7
|
+
numberField,
|
|
8
|
+
readAudio,
|
|
9
|
+
readJson,
|
|
10
|
+
requestSignal,
|
|
11
|
+
stringField
|
|
12
|
+
} from "../openai/wire.js";
|
|
13
|
+
import { priceAudio } from "../usage.js";
|
|
14
|
+
import { audioFilename, encodeForm, formType, pcmS16, xiHeaders } from "./wire.js";
|
|
15
|
+
function textOf(record, key) {
|
|
16
|
+
const text = stringField(record, key);
|
|
17
|
+
if (text === void 0) return "";
|
|
18
|
+
return text;
|
|
19
|
+
}
|
|
20
|
+
function secondsOf(record, key) {
|
|
21
|
+
const value = numberField(record, key);
|
|
22
|
+
if (value === void 0) return 0;
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
function wordFrom(word) {
|
|
26
|
+
const kind = stringField(word, "type");
|
|
27
|
+
if (kind === "spacing") return void 0;
|
|
28
|
+
const speaker = stringField(word, "speaker_id");
|
|
29
|
+
const segment = {
|
|
30
|
+
text: textOf(word, "text"),
|
|
31
|
+
start: secondsOf(word, "start"),
|
|
32
|
+
end: secondsOf(word, "end")
|
|
33
|
+
};
|
|
34
|
+
if (speaker !== void 0) segment.speaker = speaker;
|
|
35
|
+
return segment;
|
|
36
|
+
}
|
|
37
|
+
function wordsFrom(value) {
|
|
38
|
+
if (!Array.isArray(value)) return [];
|
|
39
|
+
const segments = [];
|
|
40
|
+
for (const word of value) {
|
|
41
|
+
if (!isRecord(word)) continue;
|
|
42
|
+
const segment = wordFrom(word);
|
|
43
|
+
if (segment) segments.push(segment);
|
|
44
|
+
}
|
|
45
|
+
return segments;
|
|
46
|
+
}
|
|
47
|
+
function transcriptFrom(body, model) {
|
|
48
|
+
if (!isRecord(body)) throw new Error("Transcription response had no text");
|
|
49
|
+
const direct = stringField(body, "text");
|
|
50
|
+
const channels = Array.isArray(body.transcripts) ? body.transcripts.filter(isRecord) : [];
|
|
51
|
+
const joined = channels.map((channel) => textOf(channel, "text")).join(" ");
|
|
52
|
+
const text = direct === void 0 ? joined : direct;
|
|
53
|
+
if (direct === void 0 && channels.length === 0)
|
|
54
|
+
throw new Error("Transcription response had no text");
|
|
55
|
+
const language = stringField(body, "language_code");
|
|
56
|
+
const words = direct === void 0 ? channels.flatMap((channel) => wordsFrom(channel.words)) : wordsFrom(body.words);
|
|
57
|
+
const duration = numberField(body, "audio_duration_secs");
|
|
58
|
+
return {
|
|
59
|
+
text,
|
|
60
|
+
...language !== void 0 ? { language } : {},
|
|
61
|
+
...words.length > 0 ? { segments: words } : {},
|
|
62
|
+
usage: priceAudio(model.audioCost, { inputSeconds: duration === void 0 ? 0 : duration })
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function fileOf(body) {
|
|
66
|
+
const file = body.file;
|
|
67
|
+
if (!isRecord(file) || !(file.data instanceof Uint8Array) || typeof file.mimeType !== "string") {
|
|
68
|
+
throw new Error("Transcription payload was not a file");
|
|
69
|
+
}
|
|
70
|
+
const named = typeof file.filename === "string" ? file.filename : audioFilename(file.mimeType);
|
|
71
|
+
return { data: file.data, mimeType: file.mimeType, filename: named };
|
|
72
|
+
}
|
|
73
|
+
function fieldStrings(body) {
|
|
74
|
+
const fields = {};
|
|
75
|
+
for (const [key, value] of Object.entries(body)) {
|
|
76
|
+
if (key === "file") continue;
|
|
77
|
+
if (typeof value !== "string") continue;
|
|
78
|
+
fields[key] = value;
|
|
79
|
+
}
|
|
80
|
+
return fields;
|
|
81
|
+
}
|
|
82
|
+
async function xiFetch(model, options, url, body, contentType) {
|
|
83
|
+
const headers = xiHeaders(model, options);
|
|
84
|
+
headers["Content-Type"] = contentType;
|
|
85
|
+
const fetchImpl = options?.fetch ?? globalThis.fetch;
|
|
86
|
+
const response = await fetchImpl(url, {
|
|
87
|
+
method: "POST",
|
|
88
|
+
headers,
|
|
89
|
+
body,
|
|
90
|
+
signal: requestSignal(options)
|
|
91
|
+
});
|
|
92
|
+
await options?.onResponse?.({ status: response.status, headers: {} }, model);
|
|
93
|
+
if (!response.ok) {
|
|
94
|
+
const text = await response.text();
|
|
95
|
+
throw new Error(text === "" ? `HTTP ${response.status}` : text);
|
|
96
|
+
}
|
|
97
|
+
return response;
|
|
98
|
+
}
|
|
99
|
+
async function transcribeElevenLabs(model, input, options) {
|
|
100
|
+
try {
|
|
101
|
+
if (options?.signal?.aborted) throw new DOMException("aborted", "AbortError");
|
|
102
|
+
const fetchImpl = options?.fetch ?? globalThis.fetch;
|
|
103
|
+
const loaded = await readAudio(input, fetchImpl, requestSignal(options));
|
|
104
|
+
const filename = audioFilename(loaded.mimeType);
|
|
105
|
+
const payload = {
|
|
106
|
+
model_id: model.id,
|
|
107
|
+
timestamps_granularity: "word",
|
|
108
|
+
file: { data: loaded.data, mimeType: loaded.mimeType, filename }
|
|
109
|
+
};
|
|
110
|
+
if (model.diarization) payload.diarize = "true";
|
|
111
|
+
if (pcmS16(loaded.mimeType)) payload.file_format = "pcm_s16le_16";
|
|
112
|
+
const body = await applyPayload(payload, model, options);
|
|
113
|
+
if (!isRecord(body)) throw new Error("Transcription payload was not a file");
|
|
114
|
+
const file = fileOf(body);
|
|
115
|
+
const encoded = encodeForm(file, fieldStrings(body));
|
|
116
|
+
const response = await xiFetch(
|
|
117
|
+
model,
|
|
118
|
+
options,
|
|
119
|
+
endpoint(model.baseUrl, "speech-to-text"),
|
|
120
|
+
encoded,
|
|
121
|
+
formType()
|
|
122
|
+
);
|
|
123
|
+
return transcriptFrom(await readJson(response), model);
|
|
124
|
+
} catch (error) {
|
|
125
|
+
const voice = asVoiceError(error);
|
|
126
|
+
throw new VoiceRequestError(voice.reason, voice.error);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
export {
|
|
130
|
+
transcribeElevenLabs
|
|
131
|
+
};
|
|
132
|
+
//# sourceMappingURL=transcribe.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/elevenlabs/transcribe.ts"],
|
|
4
|
+
"sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport { VoiceRequestError } from '../errors.ts';\nimport type { TranscriptionModel } from '../models.ts';\nimport {\n applyPayload,\n asVoiceError,\n endpoint,\n isRecord,\n numberField,\n readAudio,\n readJson,\n requestSignal,\n stringField,\n} from '../openai/wire.ts';\nimport type { AudioInput, Transcript, TranscriptSegment } from '../transcription.ts';\nimport { priceAudio } from '../usage.ts';\nimport { audioFilename, type ElevenFile, encodeForm, formType, pcmS16, xiHeaders } from './wire.ts';\n\nfunction textOf(record: Record<string, unknown>, key: string): string {\n const text = stringField(record, key);\n if (text === undefined) return '';\n return text;\n}\n\nfunction secondsOf(record: Record<string, unknown>, key: string): number {\n const value = numberField(record, key);\n if (value === undefined) return 0;\n return value;\n}\n\nfunction wordFrom(word: Record<string, unknown>): TranscriptSegment | undefined {\n const kind = stringField(word, 'type');\n if (kind === 'spacing') return undefined;\n const speaker = stringField(word, 'speaker_id');\n const segment: TranscriptSegment = {\n text: textOf(word, 'text'),\n start: secondsOf(word, 'start'),\n end: secondsOf(word, 'end'),\n };\n if (speaker !== undefined) segment.speaker = speaker;\n return segment;\n}\n\nfunction wordsFrom(value: unknown): TranscriptSegment[] {\n if (!Array.isArray(value)) return [];\n const segments: TranscriptSegment[] = [];\n for (const word of value) {\n if (!isRecord(word)) continue;\n const segment = wordFrom(word);\n if (segment) segments.push(segment);\n }\n return segments;\n}\n\nfunction transcriptFrom(body: unknown, model: TranscriptionModel<string>): Transcript {\n if (!isRecord(body)) throw new Error('Transcription response had no text');\n const direct = stringField(body, 'text');\n const channels = Array.isArray(body.transcripts) ? body.transcripts.filter(isRecord) : [];\n const joined = channels.map((channel) => textOf(channel, 'text')).join(' ');\n const text = direct === undefined ? joined : direct;\n if (direct === undefined && channels.length === 0)\n throw new Error('Transcription response had no text');\n const language = stringField(body, 'language_code');\n const words =\n direct === undefined\n ? channels.flatMap((channel) => wordsFrom(channel.words))\n : wordsFrom(body.words);\n const duration = numberField(body, 'audio_duration_secs');\n return {\n text,\n ...(language !== undefined ? { language } : {}),\n ...(words.length > 0 ? { segments: words } : {}),\n usage: priceAudio(model.audioCost, { inputSeconds: duration === undefined ? 0 : duration }),\n };\n}\n\nfunction fileOf(body: Record<string, unknown>): ElevenFile {\n const file = body.file;\n if (!isRecord(file) || !(file.data instanceof Uint8Array) || typeof file.mimeType !== 'string') {\n throw new Error('Transcription payload was not a file');\n }\n const named = typeof file.filename === 'string' ? file.filename : audioFilename(file.mimeType);\n return { data: file.data, mimeType: file.mimeType, filename: named };\n}\n\nfunction fieldStrings(body: Record<string, unknown>): Record<string, string> {\n const fields: Record<string, string> = {};\n for (const [key, value] of Object.entries(body)) {\n if (key === 'file') continue;\n if (typeof value !== 'string') continue;\n fields[key] = value;\n }\n return fields;\n}\n\nasync function xiFetch(\n model: TranscriptionModel<string>,\n options: ProviderRequestOptions | undefined,\n url: string,\n body: BodyInit,\n contentType: string\n): Promise<Response> {\n const headers = xiHeaders(model, options);\n headers['Content-Type'] = contentType;\n const fetchImpl = options?.fetch ?? globalThis.fetch;\n const response = await fetchImpl(url, {\n method: 'POST',\n headers,\n body,\n signal: requestSignal(options),\n });\n await options?.onResponse?.({ status: response.status, headers: {} }, model as never);\n if (!response.ok) {\n const text = await response.text();\n throw new Error(text === '' ? `HTTP ${response.status}` : text);\n }\n return response;\n}\n\nexport async function transcribeElevenLabs(\n model: TranscriptionModel<string>,\n input: AudioInput,\n options?: ProviderRequestOptions\n): Promise<Transcript> {\n try {\n if (options?.signal?.aborted) throw new DOMException('aborted', 'AbortError');\n const fetchImpl = options?.fetch ?? globalThis.fetch;\n const loaded = await readAudio(input, fetchImpl, requestSignal(options));\n const filename = audioFilename(loaded.mimeType);\n const payload: Record<string, unknown> = {\n model_id: model.id,\n timestamps_granularity: 'word',\n file: { data: loaded.data, mimeType: loaded.mimeType, filename },\n };\n if (model.diarization) payload.diarize = 'true';\n if (pcmS16(loaded.mimeType)) payload.file_format = 'pcm_s16le_16';\n const body = await applyPayload(payload, model, options);\n if (!isRecord(body)) throw new Error('Transcription payload was not a file');\n const file = fileOf(body);\n const encoded = encodeForm(file, fieldStrings(body));\n const response = await xiFetch(\n model,\n options,\n endpoint(model.baseUrl, 'speech-to-text'),\n encoded,\n formType()\n );\n return transcriptFrom(await readJson(response), model);\n } catch (error) {\n const voice = asVoiceError(error);\n throw new VoiceRequestError(voice.reason, voice.error);\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,kBAAkB;AAC3B,SAAS,eAAgC,YAAY,UAAU,QAAQ,iBAAiB;AAExF,SAAS,OAAO,QAAiC,KAAqB;AACpE,QAAM,OAAO,YAAY,QAAQ,GAAG;AACpC,MAAI,SAAS,OAAW,QAAO;AAC/B,SAAO;AACT;AAEA,SAAS,UAAU,QAAiC,KAAqB;AACvE,QAAM,QAAQ,YAAY,QAAQ,GAAG;AACrC,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO;AACT;AAEA,SAAS,SAAS,MAA8D;AAC9E,QAAM,OAAO,YAAY,MAAM,MAAM;AACrC,MAAI,SAAS,UAAW,QAAO;AAC/B,QAAM,UAAU,YAAY,MAAM,YAAY;AAC9C,QAAM,UAA6B;AAAA,IACjC,MAAM,OAAO,MAAM,MAAM;AAAA,IACzB,OAAO,UAAU,MAAM,OAAO;AAAA,IAC9B,KAAK,UAAU,MAAM,KAAK;AAAA,EAC5B;AACA,MAAI,YAAY,OAAW,SAAQ,UAAU;AAC7C,SAAO;AACT;AAEA,SAAS,UAAU,OAAqC;AACtD,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,WAAgC,CAAC;AACvC,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,SAAS,IAAI,EAAG;AACrB,UAAM,UAAU,SAAS,IAAI;AAC7B,QAAI,QAAS,UAAS,KAAK,OAAO;AAAA,EACpC;AACA,SAAO;AACT;AAEA,SAAS,eAAe,MAAe,OAA+C;AACpF,MAAI,CAAC,SAAS,IAAI,EAAG,OAAM,IAAI,MAAM,oCAAoC;AACzE,QAAM,SAAS,YAAY,MAAM,MAAM;AACvC,QAAM,WAAW,MAAM,QAAQ,KAAK,WAAW,IAAI,KAAK,YAAY,OAAO,QAAQ,IAAI,CAAC;AACxF,QAAM,SAAS,SAAS,IAAI,CAAC,YAAY,OAAO,SAAS,MAAM,CAAC,EAAE,KAAK,GAAG;AAC1E,QAAM,OAAO,WAAW,SAAY,SAAS;AAC7C,MAAI,WAAW,UAAa,SAAS,WAAW;AAC9C,UAAM,IAAI,MAAM,oCAAoC;AACtD,QAAM,WAAW,YAAY,MAAM,eAAe;AAClD,QAAM,QACJ,WAAW,SACP,SAAS,QAAQ,CAAC,YAAY,UAAU,QAAQ,KAAK,CAAC,IACtD,UAAU,KAAK,KAAK;AAC1B,QAAM,WAAW,YAAY,MAAM,qBAAqB;AACxD,SAAO;AAAA,IACL;AAAA,IACA,GAAI,aAAa,SAAY,EAAE,SAAS,IAAI,CAAC;AAAA,IAC7C,GAAI,MAAM,SAAS,IAAI,EAAE,UAAU,MAAM,IAAI,CAAC;AAAA,IAC9C,OAAO,WAAW,MAAM,WAAW,EAAE,cAAc,aAAa,SAAY,IAAI,SAAS,CAAC;AAAA,EAC5F;AACF;AAEA,SAAS,OAAO,MAA2C;AACzD,QAAM,OAAO,KAAK;AAClB,MAAI,CAAC,SAAS,IAAI,KAAK,EAAE,KAAK,gBAAgB,eAAe,OAAO,KAAK,aAAa,UAAU;AAC9F,UAAM,IAAI,MAAM,sCAAsC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,KAAK,aAAa,WAAW,KAAK,WAAW,cAAc,KAAK,QAAQ;AAC7F,SAAO,EAAE,MAAM,KAAK,MAAM,UAAU,KAAK,UAAU,UAAU,MAAM;AACrE;AAEA,SAAS,aAAa,MAAuD;AAC3E,QAAM,SAAiC,CAAC;AACxC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,IAAI,GAAG;AAC/C,QAAI,QAAQ,OAAQ;AACpB,QAAI,OAAO,UAAU,SAAU;AAC/B,WAAO,GAAG,IAAI;AAAA,EAChB;AACA,SAAO;AACT;AAEA,eAAe,QACb,OACA,SACA,KACA,MACA,aACmB;AACnB,QAAM,UAAU,UAAU,OAAO,OAAO;AACxC,UAAQ,cAAc,IAAI;AAC1B,QAAM,YAAY,SAAS,SAAS,WAAW;AAC/C,QAAM,WAAW,MAAM,UAAU,KAAK;AAAA,IACpC,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA,QAAQ,cAAc,OAAO;AAAA,EAC/B,CAAC;AACD,QAAM,SAAS,aAAa,EAAE,QAAQ,SAAS,QAAQ,SAAS,CAAC,EAAE,GAAG,KAAc;AACpF,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI,MAAM,SAAS,KAAK,QAAQ,SAAS,MAAM,KAAK,IAAI;AAAA,EAChE;AACA,SAAO;AACT;AAEA,eAAsB,qBACpB,OACA,OACA,SACqB;AACrB,MAAI;AACF,QAAI,SAAS,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AAC5E,UAAM,YAAY,SAAS,SAAS,WAAW;AAC/C,UAAM,SAAS,MAAM,UAAU,OAAO,WAAW,cAAc,OAAO,CAAC;AACvE,UAAM,WAAW,cAAc,OAAO,QAAQ;AAC9C,UAAM,UAAmC;AAAA,MACvC,UAAU,MAAM;AAAA,MAChB,wBAAwB;AAAA,MACxB,MAAM,EAAE,MAAM,OAAO,MAAM,UAAU,OAAO,UAAU,SAAS;AAAA,IACjE;AACA,QAAI,MAAM,YAAa,SAAQ,UAAU;AACzC,QAAI,OAAO,OAAO,QAAQ,EAAG,SAAQ,cAAc;AACnD,UAAM,OAAO,MAAM,aAAa,SAAS,OAAO,OAAO;AACvD,QAAI,CAAC,SAAS,IAAI,EAAG,OAAM,IAAI,MAAM,sCAAsC;AAC3E,UAAM,OAAO,OAAO,IAAI;AACxB,UAAM,UAAU,WAAW,MAAM,aAAa,IAAI,CAAC;AACnD,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA,SAAS,MAAM,SAAS,gBAAgB;AAAA,MACxC;AAAA,MACA,SAAS;AAAA,IACX;AACA,WAAO,eAAe,MAAM,SAAS,QAAQ,GAAG,KAAK;AAAA,EACvD,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,KAAK;AAChC,UAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,EACvD;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|