@effect-uai/google 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Gemini.d.mts +105 -0
- package/dist/Gemini.d.mts.map +1 -0
- package/dist/Gemini.mjs +170 -0
- package/dist/Gemini.mjs.map +1 -0
- package/dist/GeminiEmbedding.d.mts +64 -0
- package/dist/GeminiEmbedding.d.mts.map +1 -0
- package/dist/GeminiEmbedding.mjs +167 -0
- package/dist/GeminiEmbedding.mjs.map +1 -0
- package/dist/GeminiSynthesizer.d.mts +48 -0
- package/dist/GeminiSynthesizer.d.mts.map +1 -0
- package/dist/GeminiSynthesizer.mjs +95 -0
- package/dist/GeminiSynthesizer.mjs.map +1 -0
- package/dist/GeminiSynthesizer.test.d.mts +1 -0
- package/dist/GeminiSynthesizer.test.mjs +84 -0
- package/dist/GeminiSynthesizer.test.mjs.map +1 -0
- package/dist/GeminiTranscriber.d.mts +42 -0
- package/dist/GeminiTranscriber.d.mts.map +1 -0
- package/dist/GeminiTranscriber.mjs +80 -0
- package/dist/GeminiTranscriber.mjs.map +1 -0
- package/dist/GeminiTranscriber.test.d.mts +1 -0
- package/dist/GeminiTranscriber.test.mjs +125 -0
- package/dist/GeminiTranscriber.test.mjs.map +1 -0
- package/dist/LyriaGenerator.d.mts +57 -0
- package/dist/LyriaGenerator.d.mts.map +1 -0
- package/dist/LyriaGenerator.mjs +202 -0
- package/dist/LyriaGenerator.mjs.map +1 -0
- package/dist/LyriaGenerator.test.d.mts +1 -0
- package/dist/LyriaGenerator.test.mjs +138 -0
- package/dist/LyriaGenerator.test.mjs.map +1 -0
- package/dist/{chunk-CfYAbeIz.mjs → chunk-uyGKjUfl.mjs} +2 -1
- package/dist/codec.d.mts +107 -0
- package/dist/codec.d.mts.map +1 -0
- package/dist/codec.mjs +151 -0
- package/dist/codec.mjs.map +1 -0
- package/dist/dist-DMsg8YLb.mjs +13782 -0
- package/dist/dist-DMsg8YLb.mjs.map +1 -0
- package/dist/geminiSpeechCodec.d.mts +37 -0
- package/dist/geminiSpeechCodec.d.mts.map +1 -0
- package/dist/geminiSpeechCodec.mjs +110 -0
- package/dist/geminiSpeechCodec.mjs.map +1 -0
- package/dist/geminiSpeechCodec.test.d.mts +1 -0
- package/dist/geminiSpeechCodec.test.mjs +78 -0
- package/dist/geminiSpeechCodec.test.mjs.map +1 -0
- package/dist/index.d.mts +8 -214
- package/dist/index.mjs +8 -289
- package/dist/magic-string.es-CaWgF-3p.mjs +1013 -0
- package/dist/magic-string.es-CaWgF-3p.mjs.map +1 -0
- package/dist/models.d.mts +61 -0
- package/dist/models.d.mts.map +1 -0
- package/dist/models.mjs +1 -0
- package/package.json +32 -12
- package/src/Gemini.ts +3 -3
- package/src/GeminiEmbedding.ts +351 -0
- package/src/GeminiSynthesizer.test.ts +93 -0
- package/src/GeminiSynthesizer.ts +214 -0
- package/src/GeminiTranscriber.test.ts +136 -0
- package/src/GeminiTranscriber.ts +204 -0
- package/src/LyriaGenerator.test.ts +162 -0
- package/src/LyriaGenerator.ts +394 -0
- package/src/codec.ts +42 -35
- package/src/geminiSpeechCodec.test.ts +56 -0
- package/src/geminiSpeechCodec.ts +137 -0
- package/src/index.ts +6 -2
- package/src/models.ts +104 -0
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
import { Context, Effect, Encoding, Layer, Match, Redacted, Schema } from "effect"
|
|
2
|
+
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
3
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
4
|
+
import type { Embedding, EmbedContentPart, EmbedInput, Usage } from "@effect-uai/core/Embedding"
|
|
5
|
+
import {
|
|
6
|
+
type CommonEmbedManyRequest,
|
|
7
|
+
type CommonEmbedRequest,
|
|
8
|
+
EmbeddingModel,
|
|
9
|
+
type EmbeddingModelService,
|
|
10
|
+
type EmbedManyResponse,
|
|
11
|
+
type EmbedResponse,
|
|
12
|
+
} from "@effect-uai/core/EmbeddingModel"
|
|
13
|
+
import type { ImageSource } from "@effect-uai/core/Image"
|
|
14
|
+
import type { GoogleEmbeddingModel } from "./models.js"
|
|
15
|
+
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
// Public types
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Gemini's task-type enum, exposed under our friendly names. Mapped to the
|
|
22
|
+
* wire `taskType` values inside the codec. Honoured by
|
|
23
|
+
* `gemini-embedding-001`; ignored by `gemini-embedding-2` (which expects
|
|
24
|
+
* task instructions in the prompt text instead).
|
|
25
|
+
*/
|
|
26
|
+
export type GoogleEmbeddingTask =
|
|
27
|
+
| "query"
|
|
28
|
+
| "document"
|
|
29
|
+
| "similarity"
|
|
30
|
+
| "classification"
|
|
31
|
+
| "clustering"
|
|
32
|
+
| "qa"
|
|
33
|
+
| "fact_verification"
|
|
34
|
+
| "code_query"
|
|
35
|
+
|
|
36
|
+
export type GeminiEmbedRequest = Omit<CommonEmbedRequest, "model" | "task" | "encoding"> & {
|
|
37
|
+
/** Narrows `CommonEmbedRequest.model` to the typed Google union. */
|
|
38
|
+
readonly model: GoogleEmbeddingModel
|
|
39
|
+
/**
|
|
40
|
+
* Widens the cross-provider `"query" | "document"` to the full Google
|
|
41
|
+
* task enum. Optional. Honoured by `gemini-embedding-001`; ignored by
|
|
42
|
+
* `gemini-embedding-2`.
|
|
43
|
+
*/
|
|
44
|
+
readonly task?: GoogleEmbeddingTask
|
|
45
|
+
/**
|
|
46
|
+
* Optional document title. Only meaningful for `RETRIEVAL_DOCUMENT`
|
|
47
|
+
* tasks on `gemini-embedding-001`.
|
|
48
|
+
*/
|
|
49
|
+
readonly title?: string
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export type GeminiEmbedManyRequest = Omit<GeminiEmbedRequest, "input"> & {
|
|
53
|
+
readonly inputs: ReadonlyArray<EmbedInput>
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export type GeminiEmbeddingService = {
|
|
57
|
+
readonly embed: (request: GeminiEmbedRequest) => Effect.Effect<EmbedResponse, AiError.AiError>
|
|
58
|
+
readonly embedMany: (
|
|
59
|
+
request: GeminiEmbedManyRequest,
|
|
60
|
+
) => Effect.Effect<EmbedManyResponse, AiError.AiError>
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Provider-typed service tag. Yield this when you want Gemini-specific
|
|
65
|
+
* options (full `task` enum, `title`); yield the generic `EmbeddingModel`
|
|
66
|
+
* tag for provider-portable code. Both are registered by `layer`.
|
|
67
|
+
*/
|
|
68
|
+
export class GeminiEmbedding extends Context.Service<GeminiEmbedding, GeminiEmbeddingService>()(
|
|
69
|
+
"@betalyra/effect-uai/providers/google/GeminiEmbedding",
|
|
70
|
+
) {}
|
|
71
|
+
|
|
72
|
+
export type Config = {
|
|
73
|
+
readonly apiKey: Redacted.Redacted
|
|
74
|
+
readonly baseUrl?: string
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// ---------------------------------------------------------------------------
|
|
78
|
+
// Codec - request body
|
|
79
|
+
// ---------------------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
const taskToWire: Record<GoogleEmbeddingTask, string> = {
|
|
82
|
+
query: "RETRIEVAL_QUERY",
|
|
83
|
+
document: "RETRIEVAL_DOCUMENT",
|
|
84
|
+
similarity: "SEMANTIC_SIMILARITY",
|
|
85
|
+
classification: "CLASSIFICATION",
|
|
86
|
+
clustering: "CLUSTERING",
|
|
87
|
+
qa: "QUESTION_ANSWERING",
|
|
88
|
+
fact_verification: "FACT_VERIFICATION",
|
|
89
|
+
code_query: "CODE_RETRIEVAL_QUERY",
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
type WireTextPart = {
|
|
93
|
+
readonly text: string
|
|
94
|
+
}
|
|
95
|
+
type WireInlineDataPart = {
|
|
96
|
+
readonly inlineData: { readonly mimeType: string; readonly data: string }
|
|
97
|
+
}
|
|
98
|
+
type WirePart = WireTextPart | WireInlineDataPart
|
|
99
|
+
type WireContent = {
|
|
100
|
+
readonly parts: ReadonlyArray<WirePart>
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const urlRejected = (param: string): AiError.AiError =>
|
|
104
|
+
new AiError.InvalidRequest({
|
|
105
|
+
provider: "gemini",
|
|
106
|
+
param,
|
|
107
|
+
raw: "URL-form image inputs aren't supported; pass base64 or bytes",
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Pre-uploading via Google's Files API isn't free, so we reject `url`
|
|
112
|
+
* sources up front rather than silently dropping them. Users with HTTPS
|
|
113
|
+
* URLs should fetch + pass `bytes` (or `base64`).
|
|
114
|
+
*/
|
|
115
|
+
const imageSourceToPart = (param: string) =>
|
|
116
|
+
Match.type<ImageSource>().pipe(
|
|
117
|
+
Match.tag(
|
|
118
|
+
"base64",
|
|
119
|
+
(s): Effect.Effect<WirePart, AiError.AiError> =>
|
|
120
|
+
Effect.succeed({ inlineData: { mimeType: s.mimeType, data: s.base64 } }),
|
|
121
|
+
),
|
|
122
|
+
Match.tag(
|
|
123
|
+
"bytes",
|
|
124
|
+
(s): Effect.Effect<WirePart, AiError.AiError> =>
|
|
125
|
+
Effect.succeed({
|
|
126
|
+
inlineData: { mimeType: s.mimeType, data: Encoding.encodeBase64(s.bytes) },
|
|
127
|
+
}),
|
|
128
|
+
),
|
|
129
|
+
Match.tag(
|
|
130
|
+
"url",
|
|
131
|
+
(): Effect.Effect<WirePart, AiError.AiError> => Effect.fail(urlRejected(param)),
|
|
132
|
+
),
|
|
133
|
+
Match.exhaustive,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
const contentPartToPart = (
|
|
137
|
+
param: string,
|
|
138
|
+
part: EmbedContentPart,
|
|
139
|
+
): Effect.Effect<WirePart, AiError.AiError> =>
|
|
140
|
+
"text" in part ? Effect.succeed({ text: part.text }) : imageSourceToPart(param)(part.image)
|
|
141
|
+
|
|
142
|
+
const inputToContent = (input: EmbedInput): Effect.Effect<WireContent, AiError.AiError> => {
|
|
143
|
+
if (typeof input === "string") return Effect.succeed({ parts: [{ text: input }] })
|
|
144
|
+
if ("text" in input) return Effect.succeed({ parts: [{ text: input.text }] })
|
|
145
|
+
if ("image" in input) {
|
|
146
|
+
return imageSourceToPart("input.image")(input.image).pipe(
|
|
147
|
+
Effect.map((part) => ({ parts: [part] })),
|
|
148
|
+
)
|
|
149
|
+
}
|
|
150
|
+
return Effect.forEach(input.content, (p) => contentPartToPart("input.content[].image", p)).pipe(
|
|
151
|
+
Effect.map((parts) => ({ parts })),
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
type WireSingleBody = {
|
|
156
|
+
readonly content: WireContent
|
|
157
|
+
readonly taskType?: string
|
|
158
|
+
readonly outputDimensionality?: number
|
|
159
|
+
readonly title?: string
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const wireOptions = (
|
|
163
|
+
request: Pick<GeminiEmbedRequest, "task" | "dimensions" | "title">,
|
|
164
|
+
): Omit<WireSingleBody, "content"> => ({
|
|
165
|
+
...(request.task !== undefined && { taskType: taskToWire[request.task] }),
|
|
166
|
+
...(request.dimensions !== undefined && { outputDimensionality: request.dimensions }),
|
|
167
|
+
...(request.title !== undefined && { title: request.title }),
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
const buildSingleBody = (
|
|
171
|
+
request: GeminiEmbedRequest,
|
|
172
|
+
): Effect.Effect<WireSingleBody, AiError.AiError> =>
|
|
173
|
+
inputToContent(request.input).pipe(
|
|
174
|
+
Effect.map((content) => ({ content, ...wireOptions(request) })),
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
type WireBatchEntry = WireSingleBody & {
|
|
178
|
+
readonly model: string
|
|
179
|
+
}
|
|
180
|
+
type WireBatchBody = {
|
|
181
|
+
readonly requests: ReadonlyArray<WireBatchEntry>
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const buildBatchBody = (
|
|
185
|
+
request: GeminiEmbedManyRequest,
|
|
186
|
+
): Effect.Effect<WireBatchBody, AiError.AiError> => {
|
|
187
|
+
const modelPath = `models/${request.model}`
|
|
188
|
+
const options = wireOptions(request)
|
|
189
|
+
return Effect.forEach(request.inputs, (input) =>
|
|
190
|
+
inputToContent(input).pipe(
|
|
191
|
+
Effect.map((content): WireBatchEntry => ({ model: modelPath, content, ...options })),
|
|
192
|
+
),
|
|
193
|
+
).pipe(Effect.map((requests) => ({ requests })))
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// ---------------------------------------------------------------------------
|
|
197
|
+
// Codec - response
|
|
198
|
+
// ---------------------------------------------------------------------------
|
|
199
|
+
|
|
200
|
+
const WireEmbedding = Schema.Struct({
|
|
201
|
+
values: Schema.Array(Schema.Number),
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
const WireSingleResponse = Schema.Struct({
|
|
205
|
+
embedding: WireEmbedding,
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
const WireBatchResponse = Schema.Struct({
|
|
209
|
+
embeddings: Schema.Array(WireEmbedding),
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
const valuesToEmbedding = (values: ReadonlyArray<number>): Embedding => ({
|
|
213
|
+
_tag: "float32",
|
|
214
|
+
vector: Float32Array.from(values),
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
// Gemini's embed endpoints don't return token-count metadata. Estimating
|
|
218
|
+
// client-side would be provider-specific cruft; honest `undefined` is better.
|
|
219
|
+
const emptyUsage: Usage = {}
|
|
220
|
+
|
|
221
|
+
// ---------------------------------------------------------------------------
|
|
222
|
+
// HTTP errors
|
|
223
|
+
// ---------------------------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
const httpStatusError = (status: number, body: string): AiError.AiError => {
|
|
226
|
+
const provider = "gemini"
|
|
227
|
+
const raw = body
|
|
228
|
+
if (status === 429) return new AiError.RateLimited({ provider, raw })
|
|
229
|
+
if (status === 408 || status === 504) return new AiError.Timeout({ provider, raw })
|
|
230
|
+
if (status === 401) return new AiError.AuthFailed({ provider, subtype: "auth", raw })
|
|
231
|
+
if (status === 403) return new AiError.AuthFailed({ provider, subtype: "permission", raw })
|
|
232
|
+
if (status === 402) return new AiError.AuthFailed({ provider, subtype: "billing", raw })
|
|
233
|
+
if (status === 413) return new AiError.ContextLengthExceeded({ provider, raw })
|
|
234
|
+
if (status >= 500) return new AiError.Unavailable({ provider, status, raw })
|
|
235
|
+
return new AiError.InvalidRequest({ provider, raw })
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const transportFailure = (cause: unknown): AiError.AiError =>
|
|
239
|
+
new AiError.Unavailable({ provider: "gemini", raw: cause })
|
|
240
|
+
|
|
241
|
+
// ---------------------------------------------------------------------------
|
|
242
|
+
// HTTP plumbing - send a JSON body, error-map status, return parsed JSON.
|
|
243
|
+
// ---------------------------------------------------------------------------
|
|
244
|
+
|
|
245
|
+
const postJson = (
|
|
246
|
+
cfg: Config,
|
|
247
|
+
url: string,
|
|
248
|
+
body: unknown,
|
|
249
|
+
): Effect.Effect<unknown, AiError.AiError, HttpClient.HttpClient> =>
|
|
250
|
+
Effect.gen(function* () {
|
|
251
|
+
const client = yield* HttpClient.HttpClient
|
|
252
|
+
const httpRequest = HttpClientRequest.post(url).pipe(
|
|
253
|
+
HttpClientRequest.setHeader("x-goog-api-key", Redacted.value(cfg.apiKey)),
|
|
254
|
+
HttpClientRequest.bodyJsonUnsafe(body),
|
|
255
|
+
)
|
|
256
|
+
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
257
|
+
if (response.status >= 400) {
|
|
258
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
259
|
+
return yield* Effect.fail(httpStatusError(response.status, text))
|
|
260
|
+
}
|
|
261
|
+
return yield* response.json.pipe(Effect.mapError(transportFailure))
|
|
262
|
+
})
|
|
263
|
+
|
|
264
|
+
const decodeSingle = (json: unknown) =>
|
|
265
|
+
Schema.decodeUnknownEffect(WireSingleResponse)(json).pipe(Effect.mapError(transportFailure))
|
|
266
|
+
|
|
267
|
+
const decodeBatch = (json: unknown) =>
|
|
268
|
+
Schema.decodeUnknownEffect(WireBatchResponse)(json).pipe(Effect.mapError(transportFailure))
|
|
269
|
+
|
|
270
|
+
const baseUrl = (cfg: Config): string =>
|
|
271
|
+
cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta"
|
|
272
|
+
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
// Service implementation
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
|
|
277
|
+
const embedImpl =
|
|
278
|
+
(cfg: Config) =>
|
|
279
|
+
(
|
|
280
|
+
request: GeminiEmbedRequest,
|
|
281
|
+
): Effect.Effect<EmbedResponse, AiError.AiError, HttpClient.HttpClient> =>
|
|
282
|
+
buildSingleBody(request).pipe(
|
|
283
|
+
Effect.flatMap((body) =>
|
|
284
|
+
postJson(cfg, `${baseUrl(cfg)}/models/${request.model}:embedContent`, body),
|
|
285
|
+
),
|
|
286
|
+
Effect.flatMap(decodeSingle),
|
|
287
|
+
Effect.map(
|
|
288
|
+
(decoded): EmbedResponse => ({
|
|
289
|
+
embedding: valuesToEmbedding(decoded.embedding.values),
|
|
290
|
+
usage: emptyUsage,
|
|
291
|
+
}),
|
|
292
|
+
),
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
const embedManyImpl =
|
|
296
|
+
(cfg: Config) =>
|
|
297
|
+
(
|
|
298
|
+
request: GeminiEmbedManyRequest,
|
|
299
|
+
): Effect.Effect<EmbedManyResponse, AiError.AiError, HttpClient.HttpClient> =>
|
|
300
|
+
buildBatchBody(request).pipe(
|
|
301
|
+
Effect.flatMap((body) =>
|
|
302
|
+
postJson(cfg, `${baseUrl(cfg)}/models/${request.model}:batchEmbedContents`, body),
|
|
303
|
+
),
|
|
304
|
+
Effect.flatMap(decodeBatch),
|
|
305
|
+
Effect.map(
|
|
306
|
+
(decoded): EmbedManyResponse => ({
|
|
307
|
+
embeddings: decoded.embeddings.map((e) => valuesToEmbedding(e.values)),
|
|
308
|
+
usage: emptyUsage,
|
|
309
|
+
}),
|
|
310
|
+
),
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
// ---------------------------------------------------------------------------
|
|
314
|
+
// Constructors
|
|
315
|
+
// ---------------------------------------------------------------------------
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Build a `GeminiEmbeddingService` value. For Layer-based setup, prefer
|
|
319
|
+
* `layer`.
|
|
320
|
+
*/
|
|
321
|
+
export const make = (
|
|
322
|
+
cfg: Config,
|
|
323
|
+
): Effect.Effect<GeminiEmbeddingService, never, HttpClient.HttpClient> =>
|
|
324
|
+
Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
|
|
325
|
+
embed: (request) =>
|
|
326
|
+
embedImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
|
|
327
|
+
embedMany: (request) =>
|
|
328
|
+
embedManyImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
|
|
329
|
+
}))
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Layer that registers both the provider-specific `GeminiEmbedding` tag
|
|
333
|
+
* and the generic `EmbeddingModel` tag, sharing one underlying
|
|
334
|
+
* implementation.
|
|
335
|
+
*/
|
|
336
|
+
export const layer = (
|
|
337
|
+
cfg: Config,
|
|
338
|
+
): Layer.Layer<GeminiEmbedding | EmbeddingModel, never, HttpClient.HttpClient> => {
|
|
339
|
+
const typed = Layer.effect(GeminiEmbedding, make(cfg))
|
|
340
|
+
const generic = Layer.effect(
|
|
341
|
+
EmbeddingModel,
|
|
342
|
+
Effect.map(
|
|
343
|
+
make(cfg),
|
|
344
|
+
(s): EmbeddingModelService => ({
|
|
345
|
+
embed: (req: CommonEmbedRequest) => s.embed(req as GeminiEmbedRequest),
|
|
346
|
+
embedMany: (req: CommonEmbedManyRequest) => s.embedMany(req as GeminiEmbedManyRequest),
|
|
347
|
+
}),
|
|
348
|
+
),
|
|
349
|
+
)
|
|
350
|
+
return Layer.merge(typed, generic)
|
|
351
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { Effect, Layer, Redacted, Stream } from "effect"
|
|
2
|
+
import { FetchHttpClient } from "effect/unstable/http"
|
|
3
|
+
import { describe, expect, expectTypeOf, it } from "vitest"
|
|
4
|
+
import type * as AiError from "@effect-uai/core/AiError"
|
|
5
|
+
import type { AudioBlob } from "@effect-uai/core/Audio"
|
|
6
|
+
import * as SpeechSynthesizer from "@effect-uai/core/SpeechSynthesizer"
|
|
7
|
+
import * as GeminiSynthesizer from "./GeminiSynthesizer.js"
|
|
8
|
+
|
|
9
|
+
const cfg: GeminiSynthesizer.Config = { apiKey: Redacted.make("test-key") }
|
|
10
|
+
// FetchHttpClient is required for `make`, but these tests only exercise
|
|
11
|
+
// the codec and the compile-time / runtime Unsupported branches — no
|
|
12
|
+
// real HTTP call is made.
|
|
13
|
+
const live = Layer.provide(GeminiSynthesizer.layer(cfg), FetchHttpClient.layer)
|
|
14
|
+
|
|
15
|
+
describe("GeminiSynthesizer.realizeOutput", () => {
|
|
16
|
+
it("maps raw to native PCM 24 kHz mono with an identity wrapper", async () => {
|
|
17
|
+
const [format, wrap] = await Effect.runPromise(GeminiSynthesizer.realizeOutput("raw"))
|
|
18
|
+
expect(format).toEqual({
|
|
19
|
+
container: "raw",
|
|
20
|
+
encoding: "pcm_s16le",
|
|
21
|
+
sampleRate: 24000,
|
|
22
|
+
channels: 1,
|
|
23
|
+
})
|
|
24
|
+
const pcm = new Uint8Array([1, 2, 3, 4])
|
|
25
|
+
expect(wrap(pcm)).toBe(pcm)
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
it("maps wav to a RIFF/WAVE wrapper that prepends a 44-byte header", async () => {
|
|
29
|
+
const [format, wrap] = await Effect.runPromise(GeminiSynthesizer.realizeOutput("wav"))
|
|
30
|
+
expect(format.container).toBe("wav")
|
|
31
|
+
const out = wrap(new Uint8Array([0, 1, 2, 3]))
|
|
32
|
+
expect(out.length).toBe(44 + 4)
|
|
33
|
+
const ascii = (start: number, len: number) =>
|
|
34
|
+
String.fromCharCode(...Array.from(out.slice(start, start + len)))
|
|
35
|
+
expect(ascii(0, 4)).toBe("RIFF")
|
|
36
|
+
expect(ascii(8, 4)).toBe("WAVE")
|
|
37
|
+
expect(ascii(12, 4)).toBe("fmt ")
|
|
38
|
+
expect(ascii(36, 4)).toBe("data")
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
it("fails Unsupported for mp3 / opus / aac / flac / ogg / webm", async () => {
|
|
42
|
+
const exit = await Effect.runPromiseExit(GeminiSynthesizer.realizeOutput("mp3"))
|
|
43
|
+
expect(exit._tag).toBe("Failure")
|
|
44
|
+
if (exit._tag === "Failure") {
|
|
45
|
+
expect(JSON.stringify(exit.cause)).toContain("Unsupported")
|
|
46
|
+
expect(JSON.stringify(exit.cause)).toContain("outputFormat")
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
describe("GeminiSynthesizer capability guards (runtime)", () => {
|
|
52
|
+
it("streamSynthesisFrom returns an Unsupported stream", async () => {
|
|
53
|
+
const program = GeminiSynthesizer.GeminiSynthesizer.use((s) =>
|
|
54
|
+
Stream.runDrain(
|
|
55
|
+
s.streamSynthesisFrom(Stream.fromIterable(["hi"]), {
|
|
56
|
+
model: "gemini-2.5-flash-preview-tts",
|
|
57
|
+
voiceId: "Kore",
|
|
58
|
+
}),
|
|
59
|
+
),
|
|
60
|
+
)
|
|
61
|
+
const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
|
|
62
|
+
expect(exit._tag).toBe("Failure")
|
|
63
|
+
if (exit._tag === "Failure") {
|
|
64
|
+
expect(JSON.stringify(exit.cause)).toContain("Unsupported")
|
|
65
|
+
expect(JSON.stringify(exit.cause)).toContain("streamSynthesisFrom")
|
|
66
|
+
}
|
|
67
|
+
})
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
describe("GeminiSynthesizer Layer (compile-time)", () => {
|
|
71
|
+
it("leaves `TtsIncrementalText` unsatisfied when using `streamSynthesisFrom` against this Layer", () => {
|
|
72
|
+
const tokens: Stream.Stream<string> = Stream.fromIterable(["a"])
|
|
73
|
+
const audio = tokens.pipe(
|
|
74
|
+
SpeechSynthesizer.streamSynthesisFrom({
|
|
75
|
+
model: "gemini-2.5-flash-preview-tts",
|
|
76
|
+
voiceId: "Kore",
|
|
77
|
+
}),
|
|
78
|
+
)
|
|
79
|
+
const provided = Stream.runDrain(audio).pipe(Effect.provide(live))
|
|
80
|
+
expectTypeOf(provided).toEqualTypeOf<
|
|
81
|
+
Effect.Effect<void, AiError.AiError, SpeechSynthesizer.TtsIncrementalText>
|
|
82
|
+
>()
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
it("sync `synthesize` requires no marker", () => {
|
|
86
|
+
const synth = SpeechSynthesizer.synthesize({
|
|
87
|
+
text: "hi",
|
|
88
|
+
model: "gemini-2.5-flash-preview-tts",
|
|
89
|
+
voiceId: "Kore",
|
|
90
|
+
}).pipe(Effect.provide(live))
|
|
91
|
+
expectTypeOf(synth).toEqualTypeOf<Effect.Effect<AudioBlob, AiError.AiError, never>>()
|
|
92
|
+
})
|
|
93
|
+
})
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { Context, Effect, Layer, Match, Redacted, Schema, Stream } from "effect"
|
|
2
|
+
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
3
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
4
|
+
import type { AudioBlob, AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
|
|
5
|
+
import {
|
|
6
|
+
type CommonStreamSynthesizeRequest,
|
|
7
|
+
type CommonSynthesizeRequest,
|
|
8
|
+
SpeechSynthesizer,
|
|
9
|
+
type SpeechSynthesizerService,
|
|
10
|
+
} from "@effect-uai/core/SpeechSynthesizer"
|
|
11
|
+
import {
|
|
12
|
+
decodeBase64Audio,
|
|
13
|
+
httpStatusError,
|
|
14
|
+
transportFailure,
|
|
15
|
+
wrapPcmAsWav,
|
|
16
|
+
} from "./geminiSpeechCodec.js"
|
|
17
|
+
import type { GeminiTtsModel, GeminiVoiceName } from "./models.js"
|
|
18
|
+
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Public types
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
export type GeminiSynthesizeRequest = Omit<CommonSynthesizeRequest, "model" | "voiceId"> & {
|
|
24
|
+
readonly model: GeminiTtsModel
|
|
25
|
+
readonly voiceId: GeminiVoiceName
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type GeminiSynthesizerService = {
|
|
29
|
+
readonly synthesize: (r: GeminiSynthesizeRequest) => Effect.Effect<AudioBlob, AiError.AiError>
|
|
30
|
+
readonly streamSynthesis: (
|
|
31
|
+
r: GeminiSynthesizeRequest,
|
|
32
|
+
) => Stream.Stream<AudioChunk, AiError.AiError>
|
|
33
|
+
readonly streamSynthesisFrom: SpeechSynthesizerService["streamSynthesisFrom"]
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export class GeminiSynthesizer extends Context.Service<
|
|
37
|
+
GeminiSynthesizer,
|
|
38
|
+
GeminiSynthesizerService
|
|
39
|
+
>()("@betalyra/effect-uai/providers/google/GeminiSynthesizer") {}
|
|
40
|
+
|
|
41
|
+
export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
|
|
42
|
+
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
// Codec — Gemini TTS always emits 24 kHz mono signed-16 PCM. Caller picks
|
|
45
|
+
// raw bytes or WAV-wrapped.
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
const PCM_RATE = 24000
|
|
49
|
+
const PCM_CHANNELS = 1
|
|
50
|
+
|
|
51
|
+
export const realizeOutput = Match.type<AudioFormat["container"]>().pipe(
|
|
52
|
+
Match.when("raw", () =>
|
|
53
|
+
Effect.succeed([
|
|
54
|
+
{ container: "raw", encoding: "pcm_s16le", sampleRate: PCM_RATE, channels: PCM_CHANNELS },
|
|
55
|
+
(b: Uint8Array) => b,
|
|
56
|
+
] as const),
|
|
57
|
+
),
|
|
58
|
+
Match.when("wav", () =>
|
|
59
|
+
Effect.succeed([
|
|
60
|
+
{ container: "wav", encoding: "pcm_s16le", sampleRate: PCM_RATE, channels: PCM_CHANNELS },
|
|
61
|
+
(b: Uint8Array) => wrapPcmAsWav(b, PCM_RATE, PCM_CHANNELS),
|
|
62
|
+
] as const),
|
|
63
|
+
),
|
|
64
|
+
Match.whenOr("mp3", "opus", "aac", "flac", "ogg", "webm", (c) =>
|
|
65
|
+
Effect.fail(
|
|
66
|
+
new AiError.Unsupported({
|
|
67
|
+
provider: "gemini",
|
|
68
|
+
capability: "outputFormat",
|
|
69
|
+
reason: `Gemini TTS emits raw 16-bit PCM at 24 kHz. Request "raw" or "wav"; ${c} is unavailable.`,
|
|
70
|
+
}),
|
|
71
|
+
),
|
|
72
|
+
),
|
|
73
|
+
Match.exhaustive,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
// ---------------------------------------------------------------------------
|
|
77
|
+
// Wire
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
const Wire = Schema.Struct({
|
|
81
|
+
candidates: Schema.optional(
|
|
82
|
+
Schema.Array(
|
|
83
|
+
Schema.Struct({
|
|
84
|
+
content: Schema.optional(
|
|
85
|
+
Schema.Struct({
|
|
86
|
+
parts: Schema.optional(
|
|
87
|
+
Schema.Array(
|
|
88
|
+
Schema.Struct({
|
|
89
|
+
inlineData: Schema.optional(
|
|
90
|
+
Schema.Struct({
|
|
91
|
+
mimeType: Schema.optional(Schema.String),
|
|
92
|
+
data: Schema.String,
|
|
93
|
+
}),
|
|
94
|
+
),
|
|
95
|
+
}),
|
|
96
|
+
),
|
|
97
|
+
),
|
|
98
|
+
}),
|
|
99
|
+
),
|
|
100
|
+
}),
|
|
101
|
+
),
|
|
102
|
+
),
|
|
103
|
+
})
|
|
104
|
+
const decodeWire = Schema.decodeUnknownEffect(Wire)
|
|
105
|
+
|
|
106
|
+
const findInline = (wire: typeof Wire.Type) =>
|
|
107
|
+
(wire.candidates ?? [])
|
|
108
|
+
.flatMap((c) => c.content?.parts ?? [])
|
|
109
|
+
.find((p) => p.inlineData !== undefined)?.inlineData
|
|
110
|
+
|
|
111
|
+
const ttsBody = (r: GeminiSynthesizeRequest) => ({
|
|
112
|
+
contents: [{ parts: [{ text: r.text }] }],
|
|
113
|
+
generationConfig: {
|
|
114
|
+
responseModalities: ["AUDIO"],
|
|
115
|
+
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: r.voiceId } } },
|
|
116
|
+
},
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
// Service implementation
|
|
121
|
+
// ---------------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
const baseUrl = (cfg: Config) => cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta"
|
|
124
|
+
|
|
125
|
+
const synthesizeImpl = (cfg: Config) => (request: GeminiSynthesizeRequest) =>
|
|
126
|
+
Effect.gen(function* () {
|
|
127
|
+
const client = yield* HttpClient.HttpClient
|
|
128
|
+
const [format, wrap] = yield* realizeOutput(request.outputFormat?.container ?? "raw")
|
|
129
|
+
const httpRequest = HttpClientRequest.post(
|
|
130
|
+
`${baseUrl(cfg)}/models/${request.model}:generateContent`,
|
|
131
|
+
).pipe(
|
|
132
|
+
HttpClientRequest.setHeader("x-goog-api-key", Redacted.value(cfg.apiKey)),
|
|
133
|
+
HttpClientRequest.bodyJsonUnsafe(ttsBody(request)),
|
|
134
|
+
)
|
|
135
|
+
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
136
|
+
if (response.status >= 400) {
|
|
137
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
138
|
+
return yield* Effect.fail(httpStatusError(response.status, text))
|
|
139
|
+
}
|
|
140
|
+
const json = yield* response.json.pipe(Effect.mapError(transportFailure))
|
|
141
|
+
const inline = yield* decodeWire(json).pipe(
|
|
142
|
+
Effect.map(findInline),
|
|
143
|
+
Effect.mapError((cause) => new AiError.GenerationFailed({ provider: "gemini", raw: cause })),
|
|
144
|
+
Effect.flatMap((found) =>
|
|
145
|
+
found === undefined
|
|
146
|
+
? Effect.fail(
|
|
147
|
+
new AiError.GenerationFailed({
|
|
148
|
+
provider: "gemini",
|
|
149
|
+
raw: {
|
|
150
|
+
message: "Gemini TTS response had no inlineData audio part.",
|
|
151
|
+
hint: "Likely a prompt-safety rejection or the model didn't honour responseModalities.",
|
|
152
|
+
response: json,
|
|
153
|
+
},
|
|
154
|
+
}),
|
|
155
|
+
)
|
|
156
|
+
: Effect.succeed(found),
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
const pcm = yield* decodeBase64Audio(inline.data)
|
|
160
|
+
return { format, bytes: wrap(pcm) } satisfies AudioBlob
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
const unsupportedStreamFrom: SpeechSynthesizerService["streamSynthesisFrom"] = () =>
|
|
164
|
+
Stream.fail(
|
|
165
|
+
new AiError.Unsupported({
|
|
166
|
+
provider: "gemini",
|
|
167
|
+
capability: "streamSynthesisFrom",
|
|
168
|
+
reason:
|
|
169
|
+
"Gemini TTS via generateContent is sync-only. Use Cloud Text-to-Speech (Chirp 3 HD) for incremental text-in.",
|
|
170
|
+
}),
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
// ---------------------------------------------------------------------------
|
|
174
|
+
// Constructors
|
|
175
|
+
// ---------------------------------------------------------------------------
|
|
176
|
+
|
|
177
|
+
export const make = (cfg: Config) =>
|
|
178
|
+
Effect.map(
|
|
179
|
+
HttpClient.HttpClient.asEffect(),
|
|
180
|
+
(client): GeminiSynthesizerService => ({
|
|
181
|
+
synthesize: (r) =>
|
|
182
|
+
synthesizeImpl(cfg)(r).pipe(Effect.provideService(HttpClient.HttpClient, client)),
|
|
183
|
+
streamSynthesis: (r) =>
|
|
184
|
+
Stream.fromEffect(synthesizeImpl(cfg)(r)).pipe(
|
|
185
|
+
Stream.map((b): AudioChunk => ({ bytes: b.bytes })),
|
|
186
|
+
Stream.provideService(HttpClient.HttpClient, client),
|
|
187
|
+
),
|
|
188
|
+
streamSynthesisFrom: unsupportedStreamFrom,
|
|
189
|
+
}),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Layer registers both `GeminiSynthesizer` and the generic
|
|
194
|
+
* `SpeechSynthesizer`. Does NOT register `TtsIncrementalText` — Gemini
|
|
195
|
+
* TTS is sync-only at the wire level, so `streamSynthesisFrom` against
|
|
196
|
+
* this Layer alone is a compile-time error.
|
|
197
|
+
*/
|
|
198
|
+
export const layer = (cfg: Config) =>
|
|
199
|
+
Layer.merge(
|
|
200
|
+
Layer.effect(GeminiSynthesizer, make(cfg)),
|
|
201
|
+
Layer.effect(
|
|
202
|
+
SpeechSynthesizer,
|
|
203
|
+
Effect.map(
|
|
204
|
+
make(cfg),
|
|
205
|
+
(s): SpeechSynthesizerService => ({
|
|
206
|
+
synthesize: (req: CommonSynthesizeRequest) =>
|
|
207
|
+
s.synthesize(req as GeminiSynthesizeRequest),
|
|
208
|
+
streamSynthesis: (req: CommonSynthesizeRequest) =>
|
|
209
|
+
s.streamSynthesis(req as GeminiSynthesizeRequest),
|
|
210
|
+
streamSynthesisFrom: s.streamSynthesisFrom,
|
|
211
|
+
}),
|
|
212
|
+
),
|
|
213
|
+
),
|
|
214
|
+
)
|