@effect-uai/inworld 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/InworldRealtimeSynthesizer.d.mts +30 -0
- package/dist/InworldRealtimeSynthesizer.d.mts.map +1 -0
- package/dist/InworldRealtimeSynthesizer.mjs +43 -0
- package/dist/InworldRealtimeSynthesizer.mjs.map +1 -0
- package/dist/InworldRealtimeTranscriber.d.mts +30 -0
- package/dist/InworldRealtimeTranscriber.d.mts.map +1 -0
- package/dist/InworldRealtimeTranscriber.mjs +37 -0
- package/dist/InworldRealtimeTranscriber.mjs.map +1 -0
- package/dist/InworldSynthesizer.d.mts +71 -0
- package/dist/InworldSynthesizer.d.mts.map +1 -0
- package/dist/InworldSynthesizer.mjs +131 -0
- package/dist/InworldSynthesizer.mjs.map +1 -0
- package/dist/InworldTranscriber.d.mts +48 -0
- package/dist/InworldTranscriber.d.mts.map +1 -0
- package/dist/InworldTranscriber.mjs +116 -0
- package/dist/InworldTranscriber.mjs.map +1 -0
- package/dist/chunk-CfYAbeIz.mjs +13 -0
- package/dist/codec-C-2ssVAz.d.mts +35 -0
- package/dist/codec-C-2ssVAz.d.mts.map +1 -0
- package/dist/codec.d.mts +2 -0
- package/dist/codec.mjs +100 -0
- package/dist/codec.mjs.map +1 -0
- package/dist/index.d.mts +7 -0
- package/dist/index.mjs +7 -0
- package/dist/models-CPLZDcgt.d.mts +47 -0
- package/dist/models-CPLZDcgt.d.mts.map +1 -0
- package/dist/models.d.mts +2 -0
- package/dist/models.mjs +1 -0
- package/dist/realtimeStt.d.mts +38 -0
- package/dist/realtimeStt.d.mts.map +1 -0
- package/dist/realtimeStt.mjs +145 -0
- package/dist/realtimeStt.mjs.map +1 -0
- package/dist/realtimeTts.d.mts +29 -0
- package/dist/realtimeTts.d.mts.map +1 -0
- package/dist/realtimeTts.mjs +94 -0
- package/dist/realtimeTts.mjs.map +1 -0
- package/dist/wsAuth.d.mts +8 -0
- package/dist/wsAuth.d.mts.map +1 -0
- package/dist/wsAuth.mjs +8 -0
- package/dist/wsAuth.mjs.map +1 -0
- package/package.json +80 -0
- package/src/InworldRealtimeSynthesizer.ts +76 -0
- package/src/InworldRealtimeTranscriber.ts +64 -0
- package/src/InworldSynthesizer.ts +235 -0
- package/src/InworldTranscriber.ts +225 -0
- package/src/codec.ts +141 -0
- package/src/index.ts +6 -0
- package/src/models.ts +69 -0
- package/src/realtimeStt.ts +236 -0
- package/src/realtimeTts.ts +167 -0
- package/src/wsAuth.ts +21 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync transcription via Inworld's REST endpoint.
|
|
3
|
+
*
|
|
4
|
+
* - `transcribe` → `POST /stt/v1/transcribe` with audio as base64 in
|
|
5
|
+
* `audioData.content`. URL input is not accepted by Inworld — the
|
|
6
|
+
* adapter rejects `_tag: "url"` audio sources with `InvalidRequest`.
|
|
7
|
+
*
|
|
8
|
+
* Streaming transcription lives at the `InworldRealtimeTranscriber`
|
|
9
|
+
* subpath, which adds the WS path and the `SttStreaming` capability marker.
|
|
10
|
+
*/
|
|
11
|
+
import { Context, Effect, Encoding, Layer, Match, Redacted, Schema, Stream } from "effect"
|
|
12
|
+
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
13
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
14
|
+
import type { AudioSource } from "@effect-uai/core/Audio"
|
|
15
|
+
import type { TranscriptEvent, TranscriptResult, WordTimestamp } from "@effect-uai/core/Transcript"
|
|
16
|
+
import {
|
|
17
|
+
type CommonStreamTranscribeRequest,
|
|
18
|
+
type CommonTranscribeRequest,
|
|
19
|
+
Transcriber,
|
|
20
|
+
type TranscriberService,
|
|
21
|
+
} from "@effect-uai/core/Transcriber"
|
|
22
|
+
import { authHeader, httpStatusError, transportFailure } from "./codec.js"
|
|
23
|
+
import type { InworldSttModel } from "./models.js"
|
|
24
|
+
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Public types
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Inworld-typed transcribe request. `numberOfChannels` defaults to 1.
|
|
31
|
+
* `audioEncoding` is autodetected from the audio MIME by default; pass
|
|
32
|
+
* an explicit value to override (useful for raw PCM bytes).
|
|
33
|
+
*/
|
|
34
|
+
export type InworldTranscribeRequest = Omit<CommonTranscribeRequest, "model"> & {
|
|
35
|
+
readonly model: InworldSttModel
|
|
36
|
+
readonly audioEncoding?: "AUTO_DETECT" | "LINEAR16" | "MP3" | "OGG_OPUS" | "FLAC"
|
|
37
|
+
readonly sampleRateHertz?: number
|
|
38
|
+
readonly numberOfChannels?: number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type InworldTranscriberService = {
|
|
42
|
+
readonly transcribe: (
|
|
43
|
+
r: InworldTranscribeRequest,
|
|
44
|
+
) => Effect.Effect<TranscriptResult, AiError.AiError>
|
|
45
|
+
readonly streamTranscriptionFrom: TranscriberService["streamTranscriptionFrom"]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export class InworldTranscriber extends Context.Service<
|
|
49
|
+
InworldTranscriber,
|
|
50
|
+
InworldTranscriberService
|
|
51
|
+
>()("@betalyra/effect-uai/providers/inworld/InworldTranscriber") {}
|
|
52
|
+
|
|
53
|
+
export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
|
|
54
|
+
|
|
55
|
+
// ---------------------------------------------------------------------------
|
|
56
|
+
// AudioSource → base64. Inworld's sync endpoint takes inline base64 only.
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
const urlNotSupported: AiError.AiError = new AiError.InvalidRequest({
|
|
60
|
+
provider: "inworld",
|
|
61
|
+
param: "audio",
|
|
62
|
+
raw: 'Inworld accepts only inline base64. Fetch the URL yourself and pass `{ _tag: "bytes", bytes, mimeType }`.',
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
const audioToBase64: (audio: AudioSource) => Effect.Effect<string, AiError.AiError> =
|
|
66
|
+
Match.type<AudioSource>().pipe(
|
|
67
|
+
Match.tag("bytes", (a) => Effect.succeed(Encoding.encodeBase64(a.bytes))),
|
|
68
|
+
Match.tag("base64", (a) => Effect.succeed(a.base64)),
|
|
69
|
+
Match.tag("url", () => Effect.fail(urlNotSupported)),
|
|
70
|
+
Match.exhaustive,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
// Request body
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
const promptToTerms = (
|
|
78
|
+
prompt: string | { readonly terms: ReadonlyArray<string> } | undefined,
|
|
79
|
+
): ReadonlyArray<string> | undefined =>
|
|
80
|
+
prompt === undefined ? undefined : typeof prompt === "string" ? [prompt] : prompt.terms
|
|
81
|
+
|
|
82
|
+
const buildBody = (request: InworldTranscribeRequest) =>
|
|
83
|
+
Effect.gen(function* () {
|
|
84
|
+
const content = yield* audioToBase64(request.audio)
|
|
85
|
+
const prompts = promptToTerms(request.prompt)
|
|
86
|
+
return {
|
|
87
|
+
transcribeConfig: {
|
|
88
|
+
modelId: request.model,
|
|
89
|
+
audioEncoding: request.audioEncoding ?? "AUTO_DETECT",
|
|
90
|
+
...(request.sampleRateHertz !== undefined && {
|
|
91
|
+
sampleRateHertz: request.sampleRateHertz,
|
|
92
|
+
}),
|
|
93
|
+
numberOfChannels: request.numberOfChannels ?? 1,
|
|
94
|
+
...(request.language !== undefined && { language: request.language }),
|
|
95
|
+
...(prompts !== undefined && { prompts }),
|
|
96
|
+
...(request.wordTimestamps === true && { includeWordTimestamps: true }),
|
|
97
|
+
},
|
|
98
|
+
audioData: { content },
|
|
99
|
+
}
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
// Wire schema
|
|
104
|
+
// ---------------------------------------------------------------------------
|
|
105
|
+
|
|
106
|
+
const WireWord = Schema.Struct({
|
|
107
|
+
word: Schema.String,
|
|
108
|
+
startTimeMs: Schema.optional(Schema.Number),
|
|
109
|
+
endTimeMs: Schema.optional(Schema.Number),
|
|
110
|
+
confidence: Schema.optional(Schema.Number),
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
const WireResponse = Schema.Struct({
|
|
114
|
+
transcription: Schema.Struct({
|
|
115
|
+
transcript: Schema.String,
|
|
116
|
+
isFinal: Schema.optional(Schema.Boolean),
|
|
117
|
+
wordTimestamps: Schema.optional(Schema.NullOr(Schema.Array(WireWord))),
|
|
118
|
+
}),
|
|
119
|
+
})
|
|
120
|
+
const decodeResponse = Schema.decodeUnknownEffect(WireResponse)
|
|
121
|
+
|
|
122
|
+
const wireWordToCommon = (w: typeof WireWord.Type): WordTimestamp | undefined =>
|
|
123
|
+
w.startTimeMs === undefined || w.endTimeMs === undefined
|
|
124
|
+
? undefined
|
|
125
|
+
: {
|
|
126
|
+
text: w.word,
|
|
127
|
+
startSeconds: w.startTimeMs / 1000,
|
|
128
|
+
endSeconds: w.endTimeMs / 1000,
|
|
129
|
+
...(w.confidence !== undefined && { confidence: w.confidence }),
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// ---------------------------------------------------------------------------
|
|
133
|
+
// HTTP plumbing
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
|
|
136
|
+
const baseUrl = (cfg: Config) => cfg.baseUrl ?? "https://api.inworld.ai"
|
|
137
|
+
|
|
138
|
+
/** Exported for reuse by `InworldRealtimeTranscriber`. */
|
|
139
|
+
export const transcribeImpl = (cfg: Config) => (request: InworldTranscribeRequest) =>
|
|
140
|
+
Effect.gen(function* () {
|
|
141
|
+
const client = yield* HttpClient.HttpClient
|
|
142
|
+
const body = yield* buildBody(request)
|
|
143
|
+
const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/stt/v1/transcribe`).pipe(
|
|
144
|
+
HttpClientRequest.setHeader("Authorization", authHeader(cfg.apiKey)),
|
|
145
|
+
HttpClientRequest.bodyJsonUnsafe(body),
|
|
146
|
+
)
|
|
147
|
+
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
148
|
+
if (response.status >= 400) {
|
|
149
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
150
|
+
return yield* Effect.fail(httpStatusError(response.status, text))
|
|
151
|
+
}
|
|
152
|
+
const json = yield* response.json.pipe(Effect.mapError(transportFailure))
|
|
153
|
+
const wire = yield* decodeResponse(json).pipe(
|
|
154
|
+
Effect.mapError(
|
|
155
|
+
(cause) =>
|
|
156
|
+
new AiError.GenerationFailed({
|
|
157
|
+
provider: "inworld",
|
|
158
|
+
raw: { message: "STT response missing `transcription.transcript`", cause, json },
|
|
159
|
+
}),
|
|
160
|
+
),
|
|
161
|
+
)
|
|
162
|
+
const words = wire.transcription.wordTimestamps
|
|
163
|
+
?.map(wireWordToCommon)
|
|
164
|
+
.filter((w): w is WordTimestamp => w !== undefined)
|
|
165
|
+
return {
|
|
166
|
+
text: wire.transcription.transcript,
|
|
167
|
+
...(words !== undefined && words.length > 0 && { words }),
|
|
168
|
+
raw: json,
|
|
169
|
+
} satisfies TranscriptResult
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
// ---------------------------------------------------------------------------
|
|
173
|
+
// streamTranscriptionFrom Unsupported on this Layer
|
|
174
|
+
// ---------------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
const streamUnsupported = <E, R>(
|
|
177
|
+
_audioIn: Stream.Stream<Uint8Array, E, R>,
|
|
178
|
+
_request: CommonStreamTranscribeRequest,
|
|
179
|
+
): Stream.Stream<TranscriptEvent, AiError.AiError | E, R> => {
|
|
180
|
+
const fail: Stream.Stream<TranscriptEvent, AiError.AiError | E, R> = Stream.fail(
|
|
181
|
+
new AiError.Unsupported({
|
|
182
|
+
provider: "inworld",
|
|
183
|
+
capability: "streamTranscriptionFrom",
|
|
184
|
+
reason:
|
|
185
|
+
"This Layer is sync-only. Import `@effect-uai/inworld/InworldRealtimeTranscriber` to use the WS streamTranscriptionFrom path.",
|
|
186
|
+
}),
|
|
187
|
+
)
|
|
188
|
+
return fail
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// ---------------------------------------------------------------------------
|
|
192
|
+
// Constructors
|
|
193
|
+
// ---------------------------------------------------------------------------
|
|
194
|
+
|
|
195
|
+
export const make = (
|
|
196
|
+
cfg: Config,
|
|
197
|
+
): Effect.Effect<InworldTranscriberService, never, HttpClient.HttpClient> =>
|
|
198
|
+
Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
|
|
199
|
+
transcribe: (request) =>
|
|
200
|
+
transcribeImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
|
|
201
|
+
streamTranscriptionFrom: streamUnsupported,
|
|
202
|
+
}))
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Sync-only Layer. Registers `InworldTranscriber` + the generic
|
|
206
|
+
* `Transcriber`. Does **not** register `SttStreaming` — for live
|
|
207
|
+
* transcription use `@effect-uai/inworld/InworldRealtimeTranscriber`.
|
|
208
|
+
*/
|
|
209
|
+
export const layer = (
|
|
210
|
+
cfg: Config,
|
|
211
|
+
): Layer.Layer<InworldTranscriber | Transcriber, never, HttpClient.HttpClient> =>
|
|
212
|
+
Layer.merge(
|
|
213
|
+
Layer.effect(InworldTranscriber, make(cfg)),
|
|
214
|
+
Layer.effect(
|
|
215
|
+
Transcriber,
|
|
216
|
+
Effect.map(
|
|
217
|
+
make(cfg),
|
|
218
|
+
(s): TranscriberService => ({
|
|
219
|
+
transcribe: (req: CommonTranscribeRequest) =>
|
|
220
|
+
s.transcribe(req as InworldTranscribeRequest),
|
|
221
|
+
streamTranscriptionFrom: s.streamTranscriptionFrom,
|
|
222
|
+
}),
|
|
223
|
+
),
|
|
224
|
+
),
|
|
225
|
+
)
|
package/src/codec.ts
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { Effect, Encoding, Match, Redacted, Result } from "effect"
|
|
2
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
3
|
+
import type { AudioFormat } from "@effect-uai/core/Audio"
|
|
4
|
+
import type { InworldAudioEncoding } from "./models.js"
|
|
5
|
+
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
// Auth — `Authorization: Basic $INWORLD_API_KEY`. The portal key is already
|
|
8
|
+
// base64-encoded, so we inject it verbatim. Do NOT re-encode `key:secret`.
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
|
|
11
|
+
export const authHeader = (apiKey: Redacted.Redacted): string => `Basic ${Redacted.value(apiKey)}`
|
|
12
|
+
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
// AudioFormat → Inworld `audioConfig`
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
|
|
17
|
+
export type AudioConfig = {
|
|
18
|
+
readonly audioEncoding: InworldAudioEncoding
|
|
19
|
+
readonly sampleRateHertz?: number
|
|
20
|
+
readonly bitRate?: number
|
|
21
|
+
readonly speakingRate?: number
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const unsupportedFormat = (format: AudioFormat): AiError.AiError =>
|
|
25
|
+
new AiError.Unsupported({
|
|
26
|
+
provider: "inworld",
|
|
27
|
+
capability: "outputFormat",
|
|
28
|
+
reason: `Cannot encode ${JSON.stringify(format)} as Inworld audioConfig. Supported: mp3, wav (LINEAR16), ogg+opus, flac, raw pcm_s16le/mulaw/alaw.`,
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Map our cross-provider `AudioFormat` to Inworld's `audioConfig.audioEncoding`.
|
|
33
|
+
*
|
|
34
|
+
* Inworld quirk: sync `LINEAR16` / `WAV` responses include a WAV header in
|
|
35
|
+
* the returned `audioContent`; streaming (NDJSON + WS) responses for the
|
|
36
|
+
* same encoding do NOT include the header. Callers consuming raw PCM via
|
|
37
|
+
* streaming should request `raw + pcm_s16le` (→ `PCM`) for predictable
|
|
38
|
+
* concatenation, and reserve `wav` for sync.
|
|
39
|
+
*/
|
|
40
|
+
export const audioEncodingFor: (
|
|
41
|
+
format: AudioFormat,
|
|
42
|
+
) => Effect.Effect<InworldAudioEncoding, AiError.AiError> = (format) =>
|
|
43
|
+
Match.value(format).pipe(
|
|
44
|
+
Match.when({ container: "mp3" }, () => Effect.succeed<InworldAudioEncoding>("MP3")),
|
|
45
|
+
Match.when({ container: "wav" }, () => Effect.succeed<InworldAudioEncoding>("LINEAR16")),
|
|
46
|
+
Match.when({ container: "ogg", encoding: "opus" }, () =>
|
|
47
|
+
Effect.succeed<InworldAudioEncoding>("OGG_OPUS"),
|
|
48
|
+
),
|
|
49
|
+
Match.when({ container: "flac" }, () => Effect.succeed<InworldAudioEncoding>("FLAC")),
|
|
50
|
+
Match.when({ container: "raw", encoding: "pcm_s16le" }, () =>
|
|
51
|
+
Effect.succeed<InworldAudioEncoding>("PCM"),
|
|
52
|
+
),
|
|
53
|
+
Match.when({ container: "raw", encoding: "pcm_mulaw" }, () =>
|
|
54
|
+
Effect.succeed<InworldAudioEncoding>("MULAW"),
|
|
55
|
+
),
|
|
56
|
+
Match.when({ container: "raw", encoding: "pcm_alaw" }, () =>
|
|
57
|
+
Effect.succeed<InworldAudioEncoding>("ALAW"),
|
|
58
|
+
),
|
|
59
|
+
Match.orElse((f) => Effect.fail(unsupportedFormat(f))),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
export const audioConfigFor = (
|
|
63
|
+
format: AudioFormat,
|
|
64
|
+
speakingRate?: number,
|
|
65
|
+
): Effect.Effect<AudioConfig, AiError.AiError> =>
|
|
66
|
+
Effect.map(audioEncodingFor(format), (audioEncoding) => ({
|
|
67
|
+
audioEncoding,
|
|
68
|
+
sampleRateHertz: format.sampleRate,
|
|
69
|
+
...(format.bitRate !== undefined && { bitRate: format.bitRate }),
|
|
70
|
+
...(speakingRate !== undefined && { speakingRate }),
|
|
71
|
+
}))
|
|
72
|
+
|
|
73
|
+
/** Default Inworld output: 24 kHz mono mp3. Matches Inworld's docs default. */
|
|
74
|
+
export const defaultFormat: AudioFormat = {
|
|
75
|
+
container: "mp3",
|
|
76
|
+
encoding: "mp3",
|
|
77
|
+
sampleRate: 24000,
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
81
|
+
// base64 → bytes — decoding `audioContent` from sync, NDJSON, and WS responses
|
|
82
|
+
// ---------------------------------------------------------------------------
|
|
83
|
+
|
|
84
|
+
export const decodeAudioContent = (b64: string): Effect.Effect<Uint8Array, AiError.AiError> =>
|
|
85
|
+
Result.match(Encoding.decodeBase64(b64), {
|
|
86
|
+
onSuccess: Effect.succeed,
|
|
87
|
+
onFailure: (cause) =>
|
|
88
|
+
Effect.fail(
|
|
89
|
+
new AiError.GenerationFailed({
|
|
90
|
+
provider: "inworld",
|
|
91
|
+
raw: { message: "audioContent base64 decode failed", cause },
|
|
92
|
+
}),
|
|
93
|
+
),
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
// HTTP status → AiError
|
|
98
|
+
// ---------------------------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
export const httpStatusError: (status: number, body: string) => AiError.AiError = (status, body) =>
|
|
101
|
+
Match.value(status).pipe(
|
|
102
|
+
Match.when(
|
|
103
|
+
429,
|
|
104
|
+
(): AiError.AiError => new AiError.RateLimited({ provider: "inworld", raw: body }),
|
|
105
|
+
),
|
|
106
|
+
Match.whenOr(
|
|
107
|
+
408,
|
|
108
|
+
504,
|
|
109
|
+
(): AiError.AiError => new AiError.Timeout({ provider: "inworld", raw: body }),
|
|
110
|
+
),
|
|
111
|
+
Match.when(
|
|
112
|
+
401,
|
|
113
|
+
(): AiError.AiError =>
|
|
114
|
+
new AiError.AuthFailed({ provider: "inworld", subtype: "auth", raw: body }),
|
|
115
|
+
),
|
|
116
|
+
Match.when(
|
|
117
|
+
403,
|
|
118
|
+
(): AiError.AiError =>
|
|
119
|
+
new AiError.AuthFailed({ provider: "inworld", subtype: "permission", raw: body }),
|
|
120
|
+
),
|
|
121
|
+
Match.when(
|
|
122
|
+
402,
|
|
123
|
+
(): AiError.AiError =>
|
|
124
|
+
new AiError.AuthFailed({ provider: "inworld", subtype: "billing", raw: body }),
|
|
125
|
+
),
|
|
126
|
+
Match.when(
|
|
127
|
+
413,
|
|
128
|
+
(): AiError.AiError => new AiError.ContextLengthExceeded({ provider: "inworld", raw: body }),
|
|
129
|
+
),
|
|
130
|
+
Match.when(
|
|
131
|
+
(n) => n >= 500,
|
|
132
|
+
(n): AiError.AiError =>
|
|
133
|
+
new AiError.Unavailable({ provider: "inworld", status: n, raw: body }),
|
|
134
|
+
),
|
|
135
|
+
Match.orElse(
|
|
136
|
+
(): AiError.AiError => new AiError.InvalidRequest({ provider: "inworld", raw: body }),
|
|
137
|
+
),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
export const transportFailure = (cause: unknown): AiError.AiError =>
|
|
141
|
+
new AiError.Unavailable({ provider: "inworld", raw: cause })
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export * as InworldSynthesizer from "./InworldSynthesizer.js"
|
|
2
|
+
export * as InworldRealtimeSynthesizer from "./InworldRealtimeSynthesizer.js"
|
|
3
|
+
export * as InworldTranscriber from "./InworldTranscriber.js"
|
|
4
|
+
export * as InworldRealtimeTranscriber from "./InworldRealtimeTranscriber.js"
|
|
5
|
+
export * as codec from "./codec.js"
|
|
6
|
+
export * from "./models.js"
|
package/src/models.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inworld TTS models (current per [docs.inworld.ai/tts/tts](https://docs.inworld.ai/tts/tts)).
|
|
3
|
+
*
|
|
4
|
+
* - `inworld-tts-2` — flagship, 100+ languages, supports `deliveryMode` style
|
|
5
|
+
* steering (`STABLE` / `BALANCED` / `CREATIVE`). ~200ms P50.
|
|
6
|
+
* - `inworld-tts-1.5-max` — 15 languages, ~200ms P50.
|
|
7
|
+
* - `inworld-tts-1.5-mini` — 15 languages, ~120ms P50, lowest latency.
|
|
8
|
+
*
|
|
9
|
+
* `inworld-tts-1` / `inworld-tts-1-max` are not in current docs — treat as
|
|
10
|
+
* superseded. The `(string & {})` tail keeps autocomplete on literals while
|
|
11
|
+
* accepting any string so newly-released models work without an SDK update.
|
|
12
|
+
*/
|
|
13
|
+
export type InworldTtsModel =
|
|
14
|
+
| "inworld-tts-2"
|
|
15
|
+
| "inworld-tts-1.5-max"
|
|
16
|
+
| "inworld-tts-1.5-mini"
|
|
17
|
+
// eslint-disable-next-line @typescript-eslint/ban-types
|
|
18
|
+
| (string & {})
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Inworld voice ID. Inworld voices are human-readable names ("Sarah",
|
|
22
|
+
* "Edward", …) but there's no public list-voices REST endpoint — voices
|
|
23
|
+
* are browsed via the Inworld Portal. Typing this as `string` rather than
|
|
24
|
+
* a hard-coded union avoids the union rotting between SDK releases.
|
|
25
|
+
*/
|
|
26
|
+
export type InworldVoiceId = string
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Inworld STT models. `inworld/inworld-stt-1` is the first-party model
|
|
30
|
+
* (marked Experimental in docs at time of writing). The rest are
|
|
31
|
+
* router-style passthroughs — Inworld proxies to the named provider
|
|
32
|
+
* behind their auth & billing.
|
|
33
|
+
*
|
|
34
|
+
* Streaming WS support varies: `inworld/inworld-stt-1` and AssemblyAI /
|
|
35
|
+
* Soniox variants support both sync + WS; Groq Whisper is sync-only.
|
|
36
|
+
*/
|
|
37
|
+
export type InworldSttModel =
|
|
38
|
+
| "inworld/inworld-stt-1"
|
|
39
|
+
| "groq/whisper-large-v3"
|
|
40
|
+
| "assemblyai/universal-streaming-english"
|
|
41
|
+
| "assemblyai/universal-streaming-multilingual"
|
|
42
|
+
| "assemblyai/u3-rt-pro"
|
|
43
|
+
| "assemblyai/whisper-rt"
|
|
44
|
+
| "soniox/stt-rt-v4"
|
|
45
|
+
// eslint-disable-next-line @typescript-eslint/ban-types
|
|
46
|
+
| (string & {})
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* `deliveryMode` style steering — only honored by `inworld-tts-2`. Older
|
|
50
|
+
* models ignore the field silently.
|
|
51
|
+
*/
|
|
52
|
+
export type InworldDeliveryMode = "STABLE" | "BALANCED" | "CREATIVE"
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Inworld's `audioEncoding` enum (TTS audioConfig).
|
|
56
|
+
*
|
|
57
|
+
* Note: sync `LINEAR16` / `WAV` responses include a WAV header in the
|
|
58
|
+
* `audioContent` bytes. Streaming (NDJSON + WS) chunks do **not**. The
|
|
59
|
+
* codec layer surfaces this difference in `AudioFormat.container`.
|
|
60
|
+
*/
|
|
61
|
+
export type InworldAudioEncoding =
|
|
62
|
+
| "LINEAR16"
|
|
63
|
+
| "MP3"
|
|
64
|
+
| "OGG_OPUS"
|
|
65
|
+
| "ALAW"
|
|
66
|
+
| "MULAW"
|
|
67
|
+
| "FLAC"
|
|
68
|
+
| "PCM"
|
|
69
|
+
| "WAV"
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inworld Realtime STT — `wss://api.inworld.ai/stt/v1/transcribe:streamBidirectional`.
|
|
3
|
+
*
|
|
4
|
+
* Auth: `Authorization: Basic <API_KEY>` on the WS upgrade header (matches
|
|
5
|
+
* Inworld's own JS sample). Setting headers needs the `ws` peer dep
|
|
6
|
+
* (Node/Bun); see `./wsAuth.ts`.
|
|
7
|
+
*
|
|
8
|
+
* Wire shape (per [inworld-ai/inworld-api-examples](https://github.com/inworld-ai/inworld-api-examples)):
|
|
9
|
+
* client → server:
|
|
10
|
+
* `{ "transcribeConfig": { modelId, audioEncoding, sampleRateHertz, language, ... } }` (first frame)
|
|
11
|
+
* `{ "audioChunk": { "content": "<base64 audio>" } }` (×N)
|
|
12
|
+
* `{ "closeStream": {} }` (end-of-input)
|
|
13
|
+
* server → client:
|
|
14
|
+
* `{ "result": { "transcription": { "transcript": "...", "isFinal": true|false } } }`
|
|
15
|
+
* `{ "result": { "speechStarted": { "startTimeMs": ... } } }`
|
|
16
|
+
* `{ "result": { "speechStopped": { "silenceDurationMs": ... } } }`
|
|
17
|
+
*
|
|
18
|
+
* Inworld's STT WS sends audio as base64 inside JSON (NOT binary frames),
|
|
19
|
+
* matching the rest of the Inworld API style.
|
|
20
|
+
*/
|
|
21
|
+
import { Cause, Effect, Encoding, Match, Queue, Redacted, Schema, Stream } from "effect"
|
|
22
|
+
import * as Socket from "effect/unstable/socket/Socket"
|
|
23
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
24
|
+
import type { AudioFormat } from "@effect-uai/core/Audio"
|
|
25
|
+
import * as JSONL from "@effect-uai/core/JSONL"
|
|
26
|
+
import type { TranscriptEvent, WordTimestamp } from "@effect-uai/core/Transcript"
|
|
27
|
+
import type { CommonStreamTranscribeRequest } from "@effect-uai/core/Transcriber"
|
|
28
|
+
import { authedWsConstructor } from "./wsAuth.js"
|
|
29
|
+
|
|
30
|
+
export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
|
|
31
|
+
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
// AudioFormat → `audioEncoding` slug for Inworld STT
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
|
|
36
|
+
type WireEncoding = "LINEAR16" | "MP3" | "OGG_OPUS" | "FLAC"
|
|
37
|
+
|
|
38
|
+
const unsupportedFormat = (format: AudioFormat) =>
|
|
39
|
+
new AiError.Unsupported({
|
|
40
|
+
provider: "inworld",
|
|
41
|
+
capability: "inputFormat",
|
|
42
|
+
reason: `Inworld realtime STT accepts pcm_s16le (LINEAR16), mp3, ogg/opus, or flac. Got ${JSON.stringify(format)}.`,
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
const inputFormatToWire: (format: AudioFormat) => Effect.Effect<WireEncoding, AiError.AiError> =
|
|
46
|
+
Match.type<AudioFormat>().pipe(
|
|
47
|
+
Match.when({ container: "raw", encoding: "pcm_s16le" }, () =>
|
|
48
|
+
Effect.succeed<WireEncoding>("LINEAR16"),
|
|
49
|
+
),
|
|
50
|
+
Match.when({ container: "wav", encoding: "pcm_s16le" }, () =>
|
|
51
|
+
Effect.succeed<WireEncoding>("LINEAR16"),
|
|
52
|
+
),
|
|
53
|
+
Match.when({ container: "mp3" }, () => Effect.succeed<WireEncoding>("MP3")),
|
|
54
|
+
Match.when({ container: "ogg", encoding: "opus" }, () =>
|
|
55
|
+
Effect.succeed<WireEncoding>("OGG_OPUS"),
|
|
56
|
+
),
|
|
57
|
+
Match.when({ container: "flac" }, () => Effect.succeed<WireEncoding>("FLAC")),
|
|
58
|
+
Match.orElse((f) => Effect.fail(unsupportedFormat(f))),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
// URL + frame builders
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
const buildWsUrl = (cfg: Config) => {
|
|
66
|
+
const wsBase = (cfg.baseUrl ?? "https://api.inworld.ai").replace(/^http/, "ws")
|
|
67
|
+
return `${wsBase}/stt/v1/transcribe:streamBidirectional`
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const promptToTerms = (
|
|
71
|
+
prompt: string | { readonly terms: ReadonlyArray<string> } | undefined,
|
|
72
|
+
): ReadonlyArray<string> | undefined =>
|
|
73
|
+
prompt === undefined ? undefined : typeof prompt === "string" ? [prompt] : prompt.terms
|
|
74
|
+
|
|
75
|
+
const configFrame = (encoding: WireEncoding, request: CommonStreamTranscribeRequest) => {
|
|
76
|
+
const prompts = promptToTerms(request.prompt)
|
|
77
|
+
return JSON.stringify({
|
|
78
|
+
transcribeConfig: {
|
|
79
|
+
modelId: request.model,
|
|
80
|
+
audioEncoding: encoding,
|
|
81
|
+
sampleRateHertz: request.inputFormat.sampleRate,
|
|
82
|
+
numberOfChannels: request.inputFormat.channels ?? 1,
|
|
83
|
+
// Inworld's sample includes `language` even though docs mark it optional.
|
|
84
|
+
// Default to en-US to match the sample's behavior; caller can override.
|
|
85
|
+
language: request.language ?? "en-US",
|
|
86
|
+
...(prompts !== undefined && { prompts }),
|
|
87
|
+
...(request.wordTimestamps === true && { includeWordTimestamps: true }),
|
|
88
|
+
},
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const audioChunkFrame = (bytes: Uint8Array) =>
|
|
93
|
+
JSON.stringify({ audioChunk: { content: Encoding.encodeBase64(bytes) } })
|
|
94
|
+
|
|
95
|
+
const endTurnFrame = JSON.stringify({ endTurn: {} })
|
|
96
|
+
const closeStreamFrame = JSON.stringify({ closeStream: {} })
|
|
97
|
+
|
|
98
|
+
// ---------------------------------------------------------------------------
|
|
99
|
+
// Wire schemas (server → client)
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
const WireWord = Schema.Struct({
|
|
103
|
+
word: Schema.String,
|
|
104
|
+
startTimeMs: Schema.optional(Schema.Number),
|
|
105
|
+
endTimeMs: Schema.optional(Schema.Number),
|
|
106
|
+
confidence: Schema.optional(Schema.Number),
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
const ResultBody = Schema.Struct({
|
|
110
|
+
transcription: Schema.optional(
|
|
111
|
+
Schema.Struct({
|
|
112
|
+
transcript: Schema.String,
|
|
113
|
+
isFinal: Schema.optional(Schema.Boolean),
|
|
114
|
+
wordTimestamps: Schema.optional(Schema.NullOr(Schema.Array(WireWord))),
|
|
115
|
+
}),
|
|
116
|
+
),
|
|
117
|
+
speechStarted: Schema.optional(
|
|
118
|
+
Schema.Struct({
|
|
119
|
+
startTimeMs: Schema.optional(Schema.Number),
|
|
120
|
+
confidence: Schema.optional(Schema.Number),
|
|
121
|
+
}),
|
|
122
|
+
),
|
|
123
|
+
speechStopped: Schema.optional(
|
|
124
|
+
Schema.Struct({
|
|
125
|
+
silenceDurationMs: Schema.optional(Schema.Number),
|
|
126
|
+
}),
|
|
127
|
+
),
|
|
128
|
+
status: Schema.optional(Schema.Unknown),
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
const ServerFrame = Schema.Struct({
|
|
132
|
+
result: Schema.optional(ResultBody),
|
|
133
|
+
error: Schema.optional(Schema.Unknown),
|
|
134
|
+
})
|
|
135
|
+
const decodeServerFrame = Schema.decodeUnknownEffect(ServerFrame)
|
|
136
|
+
|
|
137
|
+
const wireWordToCommon = (w: typeof WireWord.Type): WordTimestamp | undefined =>
|
|
138
|
+
w.startTimeMs === undefined || w.endTimeMs === undefined
|
|
139
|
+
? undefined
|
|
140
|
+
: {
|
|
141
|
+
text: w.word,
|
|
142
|
+
startSeconds: w.startTimeMs / 1000,
|
|
143
|
+
endSeconds: w.endTimeMs / 1000,
|
|
144
|
+
...(w.confidence !== undefined && { confidence: w.confidence }),
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export const wireToEvent = (frame: typeof ServerFrame.Type): TranscriptEvent | undefined => {
|
|
148
|
+
if (frame.error !== undefined) {
|
|
149
|
+
return {
|
|
150
|
+
_tag: "error",
|
|
151
|
+
message: typeof frame.error === "string" ? frame.error : JSON.stringify(frame.error),
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const result = frame.result
|
|
155
|
+
if (result === undefined) return undefined
|
|
156
|
+
if (result.transcription !== undefined) {
|
|
157
|
+
const t = result.transcription
|
|
158
|
+
const words = t.wordTimestamps
|
|
159
|
+
?.map(wireWordToCommon)
|
|
160
|
+
.filter((w): w is WordTimestamp => w !== undefined)
|
|
161
|
+
return t.isFinal === true
|
|
162
|
+
? {
|
|
163
|
+
_tag: "final",
|
|
164
|
+
text: t.transcript,
|
|
165
|
+
...(words !== undefined && words.length > 0 && { words }),
|
|
166
|
+
}
|
|
167
|
+
: {
|
|
168
|
+
_tag: "partial",
|
|
169
|
+
text: t.transcript,
|
|
170
|
+
...(words !== undefined && words.length > 0 && { words }),
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (result.speechStarted !== undefined) {
|
|
174
|
+
return {
|
|
175
|
+
_tag: "speech-started",
|
|
176
|
+
atSeconds: (result.speechStarted.startTimeMs ?? 0) / 1000,
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (result.speechStopped !== undefined) {
|
|
180
|
+
return { _tag: "utterance-ended", atSeconds: 0 }
|
|
181
|
+
}
|
|
182
|
+
return undefined
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const handleServerMessage = (queue: Queue.Queue<TranscriptEvent, Cause.Done>) => (raw: string) =>
|
|
186
|
+
Effect.gen(function* () {
|
|
187
|
+
const json = yield* JSONL.parseSafe(raw)
|
|
188
|
+
if (json === undefined) return
|
|
189
|
+
const decoded = yield* decodeServerFrame(json).pipe(Effect.option)
|
|
190
|
+
if (decoded._tag === "None") return
|
|
191
|
+
const event = wireToEvent(decoded.value)
|
|
192
|
+
if (event !== undefined) yield* Queue.offer(queue, event)
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
// ---------------------------------------------------------------------------
|
|
196
|
+
// Stream<Uint8Array> → Stream<TranscriptEvent>
|
|
197
|
+
// ---------------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
export const streamTranscription =
|
|
200
|
+
(cfg: Config) =>
|
|
201
|
+
<E, R>(
|
|
202
|
+
audioIn: Stream.Stream<Uint8Array, E, R>,
|
|
203
|
+
request: CommonStreamTranscribeRequest,
|
|
204
|
+
): Stream.Stream<TranscriptEvent, AiError.AiError | E, R> =>
|
|
205
|
+
Stream.unwrap(
|
|
206
|
+
Effect.gen(function* () {
|
|
207
|
+
const encoding = yield* inputFormatToWire(request.inputFormat)
|
|
208
|
+
const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg), {
|
|
209
|
+
// Effect's Socket treats all close codes as errors by default —
|
|
210
|
+
// whitelist standard clean-close codes (1000 / 1001 / 1005).
|
|
211
|
+
closeCodeIsError: (code) => code !== 1000 && code !== 1001 && code !== 1005,
|
|
212
|
+
}).pipe(Effect.provideService(Socket.WebSocketConstructor, authedWsConstructor(cfg.apiKey)))
|
|
213
|
+
const queue = yield* Queue.bounded<TranscriptEvent, Cause.Done>(64)
|
|
214
|
+
const write = yield* socket.writer
|
|
215
|
+
|
|
216
|
+
// Writer: config → drain audio → endTurn → closeStream. Inworld
|
|
217
|
+
// emits `result.transcription` partials in real-time via VAD;
|
|
218
|
+
// `endTurn` is sent on input-stream end to flush any tail audio
|
|
219
|
+
// into a final transcript, then `closeStream` for graceful close.
|
|
220
|
+
yield* Effect.gen(function* () {
|
|
221
|
+
yield* write(configFrame(encoding, request))
|
|
222
|
+
yield* Stream.runForEach(audioIn, (bytes) => write(audioChunkFrame(bytes)))
|
|
223
|
+
yield* write(endTurnFrame)
|
|
224
|
+
yield* write(closeStreamFrame)
|
|
225
|
+
}).pipe(Effect.ignore, Effect.forkScoped)
|
|
226
|
+
|
|
227
|
+
// `Queue.end` flushes pending events then fails the next take with
|
|
228
|
+
// `Done` — clean stream end. `Queue.shutdown` would CLEAR queued
|
|
229
|
+
// items and interrupt pending takes (wrong for graceful teardown).
|
|
230
|
+
yield* socket
|
|
231
|
+
.runString(handleServerMessage(queue))
|
|
232
|
+
.pipe(Effect.ensuring(Queue.end(queue)), Effect.forkScoped)
|
|
233
|
+
|
|
234
|
+
return Stream.fromQueue(queue)
|
|
235
|
+
}),
|
|
236
|
+
)
|