@effect-uai/google 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Gemini.d.mts +105 -0
- package/dist/Gemini.d.mts.map +1 -0
- package/dist/Gemini.mjs +170 -0
- package/dist/Gemini.mjs.map +1 -0
- package/dist/GeminiEmbedding.d.mts +64 -0
- package/dist/GeminiEmbedding.d.mts.map +1 -0
- package/dist/GeminiEmbedding.mjs +167 -0
- package/dist/GeminiEmbedding.mjs.map +1 -0
- package/dist/GeminiSynthesizer.d.mts +48 -0
- package/dist/GeminiSynthesizer.d.mts.map +1 -0
- package/dist/GeminiSynthesizer.mjs +95 -0
- package/dist/GeminiSynthesizer.mjs.map +1 -0
- package/dist/GeminiSynthesizer.test.d.mts +1 -0
- package/dist/GeminiSynthesizer.test.mjs +84 -0
- package/dist/GeminiSynthesizer.test.mjs.map +1 -0
- package/dist/GeminiTranscriber.d.mts +42 -0
- package/dist/GeminiTranscriber.d.mts.map +1 -0
- package/dist/GeminiTranscriber.mjs +80 -0
- package/dist/GeminiTranscriber.mjs.map +1 -0
- package/dist/GeminiTranscriber.test.d.mts +1 -0
- package/dist/GeminiTranscriber.test.mjs +125 -0
- package/dist/GeminiTranscriber.test.mjs.map +1 -0
- package/dist/LyriaGenerator.d.mts +57 -0
- package/dist/LyriaGenerator.d.mts.map +1 -0
- package/dist/LyriaGenerator.mjs +202 -0
- package/dist/LyriaGenerator.mjs.map +1 -0
- package/dist/LyriaGenerator.test.d.mts +1 -0
- package/dist/LyriaGenerator.test.mjs +138 -0
- package/dist/LyriaGenerator.test.mjs.map +1 -0
- package/dist/{chunk-CfYAbeIz.mjs → chunk-uyGKjUfl.mjs} +2 -1
- package/dist/codec.d.mts +107 -0
- package/dist/codec.d.mts.map +1 -0
- package/dist/codec.mjs +151 -0
- package/dist/codec.mjs.map +1 -0
- package/dist/dist-DMsg8YLb.mjs +13782 -0
- package/dist/dist-DMsg8YLb.mjs.map +1 -0
- package/dist/geminiSpeechCodec.d.mts +37 -0
- package/dist/geminiSpeechCodec.d.mts.map +1 -0
- package/dist/geminiSpeechCodec.mjs +110 -0
- package/dist/geminiSpeechCodec.mjs.map +1 -0
- package/dist/geminiSpeechCodec.test.d.mts +1 -0
- package/dist/geminiSpeechCodec.test.mjs +78 -0
- package/dist/geminiSpeechCodec.test.mjs.map +1 -0
- package/dist/index.d.mts +8 -214
- package/dist/index.mjs +8 -289
- package/dist/magic-string.es-CaWgF-3p.mjs +1013 -0
- package/dist/magic-string.es-CaWgF-3p.mjs.map +1 -0
- package/dist/models.d.mts +61 -0
- package/dist/models.d.mts.map +1 -0
- package/dist/models.mjs +1 -0
- package/package.json +32 -12
- package/src/Gemini.ts +3 -3
- package/src/GeminiEmbedding.ts +351 -0
- package/src/GeminiSynthesizer.test.ts +93 -0
- package/src/GeminiSynthesizer.ts +214 -0
- package/src/GeminiTranscriber.test.ts +136 -0
- package/src/GeminiTranscriber.ts +204 -0
- package/src/LyriaGenerator.test.ts +162 -0
- package/src/LyriaGenerator.ts +394 -0
- package/src/codec.ts +42 -35
- package/src/geminiSpeechCodec.test.ts +56 -0
- package/src/geminiSpeechCodec.ts +137 -0
- package/src/index.ts +6 -2
- package/src/models.ts +104 -0
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Array as Arr,
|
|
3
|
+
Context,
|
|
4
|
+
Effect,
|
|
5
|
+
Encoding,
|
|
6
|
+
Layer,
|
|
7
|
+
Match,
|
|
8
|
+
Option,
|
|
9
|
+
Redacted,
|
|
10
|
+
Result,
|
|
11
|
+
Schema,
|
|
12
|
+
Stream,
|
|
13
|
+
} from "effect"
|
|
14
|
+
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
15
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
16
|
+
import type { AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
|
|
17
|
+
import type {
|
|
18
|
+
CommonGenerateMusicRequest,
|
|
19
|
+
CommonStreamGenerateMusicRequest,
|
|
20
|
+
MusicResult,
|
|
21
|
+
MusicSessionInput,
|
|
22
|
+
WeightedPrompt,
|
|
23
|
+
} from "@effect-uai/core/Music"
|
|
24
|
+
import { MusicGenerator, type MusicGeneratorService } from "@effect-uai/core/MusicGenerator"
|
|
25
|
+
import type { LyriaModel } from "./models.js"
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// Public types
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Lyria-typed generate request. Narrows `model` to `LyriaModel`. The
|
|
33
|
+
* Lyria sync API (`generateContent`) accepts only `audio/mp3` and
|
|
34
|
+
* `audio/wav` output; `bpm`, `scale`, `instrumental` etc. on the
|
|
35
|
+
* `CommonGenerateMusicRequest` are flattened into the prompt text
|
|
36
|
+
* because the public API does not expose them as structured fields.
|
|
37
|
+
*/
|
|
38
|
+
export type LyriaGenerateRequest = Omit<CommonGenerateMusicRequest, "model"> & {
|
|
39
|
+
readonly model: LyriaModel
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type LyriaGeneratorService = {
|
|
43
|
+
readonly generate: (request: LyriaGenerateRequest) => Effect.Effect<MusicResult, AiError.AiError>
|
|
44
|
+
readonly streamGeneration: (
|
|
45
|
+
request: LyriaGenerateRequest,
|
|
46
|
+
) => Stream.Stream<AudioChunk, AiError.AiError>
|
|
47
|
+
readonly streamGenerationFrom: <E, R>(
|
|
48
|
+
input: Stream.Stream<MusicSessionInput, E, R>,
|
|
49
|
+
request: CommonStreamGenerateMusicRequest,
|
|
50
|
+
) => Stream.Stream<AudioChunk, AiError.AiError | E, R>
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export class LyriaGenerator extends Context.Service<LyriaGenerator, LyriaGeneratorService>()(
|
|
54
|
+
"@betalyra/effect-uai/providers/google/LyriaGenerator",
|
|
55
|
+
) {}
|
|
56
|
+
|
|
57
|
+
export type Config = {
|
|
58
|
+
readonly apiKey: Redacted.Redacted
|
|
59
|
+
readonly baseUrl?: string
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
// Codec — request → prompt text + response_format
|
|
64
|
+
// ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
export type LyriaResponseMimeType = "audio/mp3" | "audio/wav"
|
|
67
|
+
|
|
68
|
+
export const containerToMimeType: (
|
|
69
|
+
container: AudioFormat["container"],
|
|
70
|
+
) => Effect.Effect<LyriaResponseMimeType, AiError.AiError> = Match.type<
|
|
71
|
+
AudioFormat["container"]
|
|
72
|
+
>().pipe(
|
|
73
|
+
Match.when("mp3", () => Effect.succeed<LyriaResponseMimeType>("audio/mp3")),
|
|
74
|
+
Match.when("wav", () => Effect.succeed<LyriaResponseMimeType>("audio/wav")),
|
|
75
|
+
Match.whenOr("ogg", "opus", "flac", "aac", "webm", "raw", (c) =>
|
|
76
|
+
Effect.fail(
|
|
77
|
+
new AiError.Unsupported({
|
|
78
|
+
provider: "lyria",
|
|
79
|
+
capability: "outputFormat",
|
|
80
|
+
reason: `Lyria 3 only produces audio/mp3 or audio/wav; ${c} is not available. Use container: "mp3" or "wav".`,
|
|
81
|
+
}),
|
|
82
|
+
),
|
|
83
|
+
),
|
|
84
|
+
Match.exhaustive,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Is this a Lyria 3 Clip model variant? Clip is fixed at 30 s MP3 — it
|
|
89
|
+
* does not accept a `responseFormat` on `generationConfig` (the live
|
|
90
|
+
* API rejects unknown fields, despite what the docs imply). Pro
|
|
91
|
+
* accepts the documented `responseFormat` knob.
|
|
92
|
+
*/
|
|
93
|
+
const isClipModel = (model: string): boolean => model.includes("clip")
|
|
94
|
+
|
|
95
|
+
export const realizedFormat: (mime: LyriaResponseMimeType) => AudioFormat =
|
|
96
|
+
Match.type<LyriaResponseMimeType>().pipe(
|
|
97
|
+
Match.when(
|
|
98
|
+
"audio/mp3",
|
|
99
|
+
(): AudioFormat => ({ container: "mp3", encoding: "mp3", sampleRate: 44100, channels: 2 }),
|
|
100
|
+
),
|
|
101
|
+
Match.when(
|
|
102
|
+
"audio/wav",
|
|
103
|
+
(): AudioFormat => ({
|
|
104
|
+
container: "wav",
|
|
105
|
+
encoding: "pcm_s16le",
|
|
106
|
+
sampleRate: 44100,
|
|
107
|
+
channels: 2,
|
|
108
|
+
}),
|
|
109
|
+
),
|
|
110
|
+
Match.exhaustive,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Collapse `prompts | WeightedPrompt[]` into a single prompt string and
|
|
115
|
+
* splice in non-structural hints (`lyrics`, `bpm`, `scale`,
|
|
116
|
+
* `instrumental`, `durationSeconds`). The Lyria 3 sync API only takes
|
|
117
|
+
* one text part — every condition lives inline.
|
|
118
|
+
*/
|
|
119
|
+
export const buildPrompt = (request: CommonGenerateMusicRequest): string => {
|
|
120
|
+
const promptText = Match.value(request.prompts).pipe(
|
|
121
|
+
Match.when(Match.string, (s) => s),
|
|
122
|
+
Match.orElse((ws: ReadonlyArray<WeightedPrompt>) =>
|
|
123
|
+
Arr.map(ws, (w) =>
|
|
124
|
+
w.weight !== undefined && w.weight !== 1 ? `${w.text} (weight ${w.weight})` : w.text,
|
|
125
|
+
).join(". "),
|
|
126
|
+
),
|
|
127
|
+
)
|
|
128
|
+
const hints = Arr.getSomes([
|
|
129
|
+
request.instrumental === true ? Option.some("Instrumental only — no vocals.") : Option.none(),
|
|
130
|
+
request.bpm !== undefined ? Option.some(`BPM: ${request.bpm}.`) : Option.none(),
|
|
131
|
+
request.scale !== undefined ? Option.some(`Key/scale: ${request.scale}.`) : Option.none(),
|
|
132
|
+
request.durationSeconds !== undefined
|
|
133
|
+
? Option.some(`Target duration: ${request.durationSeconds}s.`)
|
|
134
|
+
: Option.none(),
|
|
135
|
+
])
|
|
136
|
+
const lyricsBlock =
|
|
137
|
+
request.lyrics !== undefined && request.instrumental !== true
|
|
138
|
+
? `\n\nLyrics:\n${request.lyrics}`
|
|
139
|
+
: ""
|
|
140
|
+
return [promptText, ...hints].join(" ") + lyricsBlock
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
// Wire response schema — just enough to find audio + optional lyrics text
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
const InlineData = Schema.Struct({
|
|
148
|
+
mimeType: Schema.optional(Schema.String),
|
|
149
|
+
data: Schema.String,
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
const Part = Schema.Struct({
|
|
153
|
+
text: Schema.optional(Schema.String),
|
|
154
|
+
inlineData: Schema.optional(InlineData),
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
const Content = Schema.Struct({
|
|
158
|
+
parts: Schema.optional(Schema.Array(Part)),
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
const Candidate = Schema.Struct({
|
|
162
|
+
content: Schema.optional(Content),
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
const Wire = Schema.Struct({
|
|
166
|
+
candidates: Schema.optional(Schema.Array(Candidate)),
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
const decodeWire = Schema.decodeUnknownEffect(Wire)
|
|
170
|
+
|
|
171
|
+
// ---------------------------------------------------------------------------
|
|
172
|
+
// HTTP plumbing
|
|
173
|
+
// ---------------------------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
const baseUrl = (cfg: Config): string =>
|
|
176
|
+
cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta"
|
|
177
|
+
|
|
178
|
+
const transportFailure = (cause: unknown): AiError.AiError =>
|
|
179
|
+
new AiError.Unavailable({ provider: "lyria", raw: cause })
|
|
180
|
+
|
|
181
|
+
const httpStatusError: (status: number, body: string) => AiError.AiError = (status, body) =>
|
|
182
|
+
Match.value(status).pipe(
|
|
183
|
+
Match.when(
|
|
184
|
+
429,
|
|
185
|
+
(): AiError.AiError => new AiError.RateLimited({ provider: "lyria", raw: body }),
|
|
186
|
+
),
|
|
187
|
+
Match.whenOr(
|
|
188
|
+
408,
|
|
189
|
+
504,
|
|
190
|
+
(): AiError.AiError => new AiError.Timeout({ provider: "lyria", raw: body }),
|
|
191
|
+
),
|
|
192
|
+
Match.when(
|
|
193
|
+
401,
|
|
194
|
+
(): AiError.AiError =>
|
|
195
|
+
new AiError.AuthFailed({ provider: "lyria", subtype: "auth", raw: body }),
|
|
196
|
+
),
|
|
197
|
+
Match.when(
|
|
198
|
+
403,
|
|
199
|
+
(): AiError.AiError =>
|
|
200
|
+
new AiError.AuthFailed({ provider: "lyria", subtype: "permission", raw: body }),
|
|
201
|
+
),
|
|
202
|
+
Match.when(
|
|
203
|
+
402,
|
|
204
|
+
(): AiError.AiError =>
|
|
205
|
+
new AiError.AuthFailed({ provider: "lyria", subtype: "billing", raw: body }),
|
|
206
|
+
),
|
|
207
|
+
Match.when(
|
|
208
|
+
(n) => n >= 500,
|
|
209
|
+
(n): AiError.AiError => new AiError.Unavailable({ provider: "lyria", status: n, raw: body }),
|
|
210
|
+
),
|
|
211
|
+
Match.orElse(
|
|
212
|
+
(): AiError.AiError => new AiError.InvalidRequest({ provider: "lyria", raw: body }),
|
|
213
|
+
),
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
const decodeBase64ToBytes = (b64: string): Effect.Effect<Uint8Array, AiError.AiError> =>
|
|
217
|
+
Result.match(Encoding.decodeBase64(b64), {
|
|
218
|
+
onSuccess: Effect.succeed,
|
|
219
|
+
onFailure: (cause) =>
|
|
220
|
+
Effect.fail(new AiError.InvalidRequest({ provider: "lyria", param: "audio", raw: cause })),
|
|
221
|
+
})
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Find the first inline-data audio part and the concatenated text parts
|
|
225
|
+
* from a Wire response.
|
|
226
|
+
*/
|
|
227
|
+
const extractParts = (
|
|
228
|
+
wire: typeof Wire.Type,
|
|
229
|
+
): { readonly inline?: typeof InlineData.Type; readonly text?: string } => {
|
|
230
|
+
const allParts = (wire.candidates ?? []).flatMap((c) => c.content?.parts ?? [])
|
|
231
|
+
const inline = allParts.find((p) => p.inlineData !== undefined)?.inlineData
|
|
232
|
+
const textParts = allParts.flatMap((p) => (p.text !== undefined ? [p.text] : []))
|
|
233
|
+
const text = textParts.length === 0 ? undefined : textParts.join("\n")
|
|
234
|
+
return {
|
|
235
|
+
...(inline !== undefined && { inline }),
|
|
236
|
+
...(text !== undefined && { text }),
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ---------------------------------------------------------------------------
|
|
241
|
+
// Service implementation
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
|
|
244
|
+
const generateImpl =
|
|
245
|
+
(cfg: Config) =>
|
|
246
|
+
(
|
|
247
|
+
request: LyriaGenerateRequest,
|
|
248
|
+
): Effect.Effect<MusicResult, AiError.AiError, HttpClient.HttpClient> =>
|
|
249
|
+
Effect.gen(function* () {
|
|
250
|
+
const client = yield* HttpClient.HttpClient
|
|
251
|
+
const requestedContainer = (request.outputFormat?.container ??
|
|
252
|
+
"mp3") satisfies AudioFormat["container"]
|
|
253
|
+
const mimeType = yield* containerToMimeType(requestedContainer)
|
|
254
|
+
// Clip is fixed 30 s mp3 — reject wav up front rather than sending
|
|
255
|
+
// a `responseFormat` that the live API ignores or rejects.
|
|
256
|
+
if (isClipModel(request.model) && mimeType === "audio/wav") {
|
|
257
|
+
return yield* Effect.fail(
|
|
258
|
+
new AiError.Unsupported({
|
|
259
|
+
provider: "lyria",
|
|
260
|
+
capability: "outputFormat",
|
|
261
|
+
reason: `lyria-3-clip-preview is fixed at audio/mp3. Use lyria-3-pro-preview for audio/wav output.`,
|
|
262
|
+
}),
|
|
263
|
+
)
|
|
264
|
+
}
|
|
265
|
+
const promptText = buildPrompt(request)
|
|
266
|
+
const body = isClipModel(request.model)
|
|
267
|
+
? { contents: [{ parts: [{ text: promptText }] }] }
|
|
268
|
+
: {
|
|
269
|
+
contents: [{ parts: [{ text: promptText }] }],
|
|
270
|
+
generationConfig: {
|
|
271
|
+
responseModalities: ["AUDIO", "TEXT"],
|
|
272
|
+
responseFormat: { audio: { mimeType } },
|
|
273
|
+
},
|
|
274
|
+
}
|
|
275
|
+
const httpRequest = HttpClientRequest.post(
|
|
276
|
+
`${baseUrl(cfg)}/models/${request.model}:generateContent`,
|
|
277
|
+
).pipe(
|
|
278
|
+
HttpClientRequest.setHeader("x-goog-api-key", Redacted.value(cfg.apiKey)),
|
|
279
|
+
HttpClientRequest.bodyJsonUnsafe(body),
|
|
280
|
+
)
|
|
281
|
+
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
282
|
+
if (response.status >= 400) {
|
|
283
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
284
|
+
return yield* Effect.fail(httpStatusError(response.status, text))
|
|
285
|
+
}
|
|
286
|
+
const json = yield* response.json.pipe(Effect.mapError(transportFailure))
|
|
287
|
+
const wire = yield* decodeWire(json).pipe(
|
|
288
|
+
Effect.mapError(
|
|
289
|
+
(cause): AiError.AiError =>
|
|
290
|
+
new AiError.GenerationFailed({ provider: "lyria", raw: cause }),
|
|
291
|
+
),
|
|
292
|
+
)
|
|
293
|
+
const { inline, text } = extractParts(wire)
|
|
294
|
+
if (inline === undefined) {
|
|
295
|
+
// No audio came back. Most common cause: Lyria filtered the prompt
|
|
296
|
+
// (named-artist references, copyrighted lyrics, etc.). Surface the
|
|
297
|
+
// full response body — text parts, finish reasons, prompt-feedback
|
|
298
|
+
// blocks — so the caller can see why.
|
|
299
|
+
return yield* Effect.fail(
|
|
300
|
+
new AiError.GenerationFailed({
|
|
301
|
+
provider: "lyria",
|
|
302
|
+
raw: {
|
|
303
|
+
message: "Lyria response had no inlineData audio part.",
|
|
304
|
+
hint: "Likely a prompt-filter rejection. Lyria filters references to real artists / copyrighted song lyrics. Rephrase with descriptive style language instead of artist names.",
|
|
305
|
+
textParts: text,
|
|
306
|
+
response: json,
|
|
307
|
+
},
|
|
308
|
+
}),
|
|
309
|
+
)
|
|
310
|
+
}
|
|
311
|
+
const bytes = yield* decodeBase64ToBytes(inline.data)
|
|
312
|
+
return {
|
|
313
|
+
format: realizedFormat(mimeType),
|
|
314
|
+
bytes,
|
|
315
|
+
...(text !== undefined && { lyrics: text }),
|
|
316
|
+
watermark: { kind: "synthid" },
|
|
317
|
+
} satisfies MusicResult
|
|
318
|
+
})
|
|
319
|
+
|
|
320
|
+
const streamGenerationImpl =
|
|
321
|
+
(cfg: Config) =>
|
|
322
|
+
(
|
|
323
|
+
request: LyriaGenerateRequest,
|
|
324
|
+
): Stream.Stream<AudioChunk, AiError.AiError, HttpClient.HttpClient> =>
|
|
325
|
+
Stream.unwrap(
|
|
326
|
+
Effect.map(generateImpl(cfg)(request), (result) =>
|
|
327
|
+
Stream.succeed<AudioChunk>({ bytes: result.bytes }),
|
|
328
|
+
),
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Lyria 3 sync has no bidirectional session at the wire level — that's
|
|
333
|
+
* Lyria RealTime (Phase 1b, separate Layer). The provider Layer here
|
|
334
|
+
* also does NOT register `MusicInteractiveSession`, so callers using
|
|
335
|
+
* `MusicGenerator.streamGenerationFrom` against only this Layer get a
|
|
336
|
+
* compile-time error before this runtime fallback ever fires.
|
|
337
|
+
*/
|
|
338
|
+
const streamGenerationFromUnsupported = <E, R>(
|
|
339
|
+
_input: Stream.Stream<MusicSessionInput, E, R>,
|
|
340
|
+
_request: CommonStreamGenerateMusicRequest,
|
|
341
|
+
): Stream.Stream<AudioChunk, AiError.AiError | E, R> => {
|
|
342
|
+
const fail: Stream.Stream<AudioChunk, AiError.AiError | E, R> = Stream.fail(
|
|
343
|
+
new AiError.Unsupported({
|
|
344
|
+
provider: "lyria",
|
|
345
|
+
capability: "streamGenerationFrom",
|
|
346
|
+
reason:
|
|
347
|
+
"Lyria 3 sync does not support bidirectional sessions. Use the Lyria RealTime Layer (WebSocket BidiGenerateMusic) for interactive mid-session prompts.",
|
|
348
|
+
}),
|
|
349
|
+
)
|
|
350
|
+
return fail
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// ---------------------------------------------------------------------------
|
|
354
|
+
// Constructors
|
|
355
|
+
// ---------------------------------------------------------------------------
|
|
356
|
+
|
|
357
|
+
export const make = (
|
|
358
|
+
cfg: Config,
|
|
359
|
+
): Effect.Effect<LyriaGeneratorService, never, HttpClient.HttpClient> =>
|
|
360
|
+
Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
|
|
361
|
+
generate: (request) =>
|
|
362
|
+
generateImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
|
|
363
|
+
streamGeneration: (request) =>
|
|
364
|
+
streamGenerationImpl(cfg)(request).pipe(Stream.provideService(HttpClient.HttpClient, client)),
|
|
365
|
+
streamGenerationFrom: streamGenerationFromUnsupported,
|
|
366
|
+
}))
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Layer that registers both the provider-specific `LyriaGenerator` tag
|
|
370
|
+
* and the generic `MusicGenerator` tag.
|
|
371
|
+
*
|
|
372
|
+
* Does NOT register the `MusicInteractiveSession` capability marker —
|
|
373
|
+
* Lyria 3 sync has no bidirectional session at the wire level. Code
|
|
374
|
+
* calling `MusicGenerator.streamGenerationFrom` will fail to typecheck
|
|
375
|
+
* against this Layer alone, which is the intended UX.
|
|
376
|
+
*/
|
|
377
|
+
export const layer = (
|
|
378
|
+
cfg: Config,
|
|
379
|
+
): Layer.Layer<LyriaGenerator | MusicGenerator, never, HttpClient.HttpClient> =>
|
|
380
|
+
Layer.merge(
|
|
381
|
+
Layer.effect(LyriaGenerator, make(cfg)),
|
|
382
|
+
Layer.effect(
|
|
383
|
+
MusicGenerator,
|
|
384
|
+
Effect.map(
|
|
385
|
+
make(cfg),
|
|
386
|
+
(s): MusicGeneratorService => ({
|
|
387
|
+
generate: (req: CommonGenerateMusicRequest) => s.generate(req as LyriaGenerateRequest),
|
|
388
|
+
streamGeneration: (req: CommonStreamGenerateMusicRequest) =>
|
|
389
|
+
s.streamGeneration(req as LyriaGenerateRequest),
|
|
390
|
+
streamGenerationFrom: s.streamGenerationFrom,
|
|
391
|
+
}),
|
|
392
|
+
),
|
|
393
|
+
),
|
|
394
|
+
)
|
package/src/codec.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { Array as Arr, Match, Option, Result, Schema, pipe } from "effect"
|
|
2
|
-
import type { ContentBlock, Item, Message } from "@effect-uai/core/Items"
|
|
3
|
-
import { matchType } from "@effect-uai/core/Match"
|
|
1
|
+
import { Array as Arr, Encoding, Match, Option, Result, Schema, pipe } from "effect"
|
|
2
|
+
import type { ContentBlock, InputImage, Item, Message } from "@effect-uai/core/Items"
|
|
4
3
|
import type { Turn } from "@effect-uai/core/Turn"
|
|
5
4
|
|
|
6
5
|
// ---------------------------------------------------------------------------
|
|
@@ -53,20 +52,20 @@ type RequestPart =
|
|
|
53
52
|
| { readonly text: string }
|
|
54
53
|
| { readonly inlineData: { readonly mimeType: string; readonly data: string } }
|
|
55
54
|
|
|
56
|
-
|
|
55
|
+
type RequestContent = {
|
|
57
56
|
readonly role: "user" | "model"
|
|
58
57
|
readonly parts: ReadonlyArray<RequestPart>
|
|
59
58
|
}
|
|
60
59
|
|
|
61
|
-
|
|
60
|
+
type RequestSystemInstruction = {
|
|
62
61
|
readonly parts: ReadonlyArray<{ readonly text: string }>
|
|
63
62
|
}
|
|
64
63
|
|
|
65
|
-
export
|
|
64
|
+
export type ThinkingConfig = {
|
|
66
65
|
readonly thinkingBudget: number
|
|
67
66
|
}
|
|
68
67
|
|
|
69
|
-
export
|
|
68
|
+
export type GenerationConfig = {
|
|
70
69
|
readonly temperature?: number
|
|
71
70
|
readonly maxOutputTokens?: number
|
|
72
71
|
readonly topP?: number
|
|
@@ -77,47 +76,55 @@ export interface GenerationConfig {
|
|
|
77
76
|
readonly responseJsonSchema?: Record<string, unknown>
|
|
78
77
|
}
|
|
79
78
|
|
|
80
|
-
export
|
|
79
|
+
export type RequestBody = {
|
|
81
80
|
readonly contents: ReadonlyArray<RequestContent>
|
|
82
81
|
readonly systemInstruction?: RequestSystemInstruction
|
|
83
82
|
readonly generationConfig?: GenerationConfig
|
|
84
83
|
}
|
|
85
84
|
|
|
86
85
|
const blockText = Match.type<ContentBlock>().pipe(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
Match.discriminatorsExhaustive("type")({
|
|
87
|
+
input_text: (b) => b.text,
|
|
88
|
+
input_image: () => "",
|
|
89
|
+
output_text: (b) => b.text,
|
|
90
|
+
refusal: (b) => b.text,
|
|
91
|
+
}),
|
|
92
92
|
)
|
|
93
93
|
|
|
94
94
|
const messageText = (message: Message): string => message.content.map(blockText).join("")
|
|
95
95
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
b.source._tag === "base64"
|
|
108
|
-
? [{ inlineData: { mimeType: b.source.media_type, data: b.source.data } }]
|
|
109
|
-
: [],
|
|
96
|
+
/**
|
|
97
|
+
* Gemini's `inlineData` form expects a base64 payload. URL-form images
|
|
98
|
+
* would need to go through Gemini's Files API (upload then `fileData`
|
|
99
|
+
* with the returned URI); pre-uploading isn't free, so we skip those for
|
|
100
|
+
* now and document as a follow-up.
|
|
101
|
+
*/
|
|
102
|
+
const imageSourceToParts = Match.type<InputImage["source"]>().pipe(
|
|
103
|
+
Match.tag("url", (): ReadonlyArray<RequestPart> => []),
|
|
104
|
+
Match.tag(
|
|
105
|
+
"base64",
|
|
106
|
+
(s): ReadonlyArray<RequestPart> => [{ inlineData: { mimeType: s.mimeType, data: s.base64 } }],
|
|
110
107
|
),
|
|
111
|
-
|
|
112
|
-
"
|
|
113
|
-
(
|
|
108
|
+
Match.tag(
|
|
109
|
+
"bytes",
|
|
110
|
+
(s): ReadonlyArray<RequestPart> => [
|
|
111
|
+
{ inlineData: { mimeType: s.mimeType, data: Encoding.encodeBase64(s.bytes) } },
|
|
112
|
+
],
|
|
114
113
|
),
|
|
115
|
-
// Refusals are assistant-side content; they don't round-trip into Gemini's
|
|
116
|
-
// request body as parts. Skip.
|
|
117
|
-
matchType("refusal", (): ReadonlyArray<RequestPart> => []),
|
|
118
114
|
Match.exhaustive,
|
|
119
115
|
)
|
|
120
116
|
|
|
117
|
+
const blockToParts = Match.type<ContentBlock>().pipe(
|
|
118
|
+
Match.discriminatorsExhaustive("type")({
|
|
119
|
+
input_text: (b): ReadonlyArray<RequestPart> => (b.text.length === 0 ? [] : [{ text: b.text }]),
|
|
120
|
+
input_image: (b): ReadonlyArray<RequestPart> => imageSourceToParts(b.source),
|
|
121
|
+
output_text: (b): ReadonlyArray<RequestPart> => (b.text.length === 0 ? [] : [{ text: b.text }]),
|
|
122
|
+
// Refusals are assistant-side content; they don't round-trip into Gemini's
|
|
123
|
+
// request body as parts. Skip.
|
|
124
|
+
refusal: (): ReadonlyArray<RequestPart> => [],
|
|
125
|
+
}),
|
|
126
|
+
)
|
|
127
|
+
|
|
121
128
|
const messageToContent = (message: Message): Result.Result<RequestContent, void> => {
|
|
122
129
|
const parts = pipe(message.content, Arr.flatMap(blockToParts))
|
|
123
130
|
if (parts.length === 0) return Result.failVoid
|
|
@@ -173,7 +180,7 @@ const finishReasonToStop = (reason: Option.Option<string>): Turn["stop_reason"]
|
|
|
173
180
|
onSome: (r) => (r === "MAX_TOKENS" ? ("max_tokens" as const) : ("stop" as const)),
|
|
174
181
|
})
|
|
175
182
|
|
|
176
|
-
export
|
|
183
|
+
export type Accumulator = {
|
|
177
184
|
readonly text: string
|
|
178
185
|
readonly reasoning: string
|
|
179
186
|
readonly finishReason: Option.Option<string>
|
|
@@ -202,7 +209,7 @@ export type ChunkPart =
|
|
|
202
209
|
| { readonly kind: "text"; readonly text: string }
|
|
203
210
|
| { readonly kind: "reasoning"; readonly text: string }
|
|
204
211
|
|
|
205
|
-
export
|
|
212
|
+
export type ChunkResult = {
|
|
206
213
|
readonly accumulator: Accumulator
|
|
207
214
|
readonly parts: ReadonlyArray<ChunkPart>
|
|
208
215
|
readonly finished: boolean
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { Effect, Encoding } from "effect"
|
|
2
|
+
import { describe, expect, it } from "vitest"
|
|
3
|
+
import type { AudioSource } from "@effect-uai/core/Audio"
|
|
4
|
+
import { audioSourceToInlineData, wrapPcmAsWav } from "./geminiSpeechCodec.js"
|
|
5
|
+
|
|
6
|
+
describe("audioSourceToInlineData", () => {
|
|
7
|
+
it("encodes a `bytes` source to base64 with its MIME type", async () => {
|
|
8
|
+
const original = new Uint8Array([0xde, 0xad, 0xbe, 0xef])
|
|
9
|
+
const source: AudioSource = { _tag: "bytes", bytes: original, mimeType: "audio/wav" }
|
|
10
|
+
const inline = await Effect.runPromise(audioSourceToInlineData(source))
|
|
11
|
+
expect(inline.mimeType).toBe("audio/wav")
|
|
12
|
+
expect(inline.data).toBe(Encoding.encodeBase64(original))
|
|
13
|
+
})
|
|
14
|
+
|
|
15
|
+
it("passes a `base64` source through unchanged", async () => {
|
|
16
|
+
const b64 = Encoding.encodeBase64(new Uint8Array([1, 2, 3]))
|
|
17
|
+
const source: AudioSource = { _tag: "base64", base64: b64, mimeType: "audio/mp3" }
|
|
18
|
+
const inline = await Effect.runPromise(audioSourceToInlineData(source))
|
|
19
|
+
expect(inline.mimeType).toBe("audio/mp3")
|
|
20
|
+
expect(inline.data).toBe(b64)
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
it("fails InvalidRequest for a URL source", async () => {
|
|
24
|
+
const source: AudioSource = { _tag: "url", url: "https://x/y.mp3", mimeType: "audio/mp3" }
|
|
25
|
+
const exit = await Effect.runPromiseExit(audioSourceToInlineData(source))
|
|
26
|
+
expect(exit._tag).toBe("Failure")
|
|
27
|
+
if (exit._tag === "Failure") {
|
|
28
|
+
expect(JSON.stringify(exit.cause)).toContain("InvalidRequest")
|
|
29
|
+
expect(JSON.stringify(exit.cause)).toContain("Files API")
|
|
30
|
+
}
|
|
31
|
+
})
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
describe("wrapPcmAsWav", () => {
|
|
35
|
+
it("produces a 44-byte RIFF/WAVE header for 16-bit mono PCM at 24 kHz", () => {
|
|
36
|
+
const pcm = new Uint8Array([0x00, 0x01, 0x00, 0x02])
|
|
37
|
+
const wav = wrapPcmAsWav(pcm, 24000, 1, 16)
|
|
38
|
+
expect(wav.length).toBe(44 + pcm.length)
|
|
39
|
+
const ascii = (start: number, len: number) =>
|
|
40
|
+
String.fromCharCode(...Array.from(wav.slice(start, start + len)))
|
|
41
|
+
expect(ascii(0, 4)).toBe("RIFF")
|
|
42
|
+
expect(ascii(8, 4)).toBe("WAVE")
|
|
43
|
+
expect(ascii(12, 4)).toBe("fmt ")
|
|
44
|
+
expect(ascii(36, 4)).toBe("data")
|
|
45
|
+
// RIFF chunk size = total - 8
|
|
46
|
+
const view = new DataView(wav.buffer)
|
|
47
|
+
expect(view.getUint32(4, true)).toBe(wav.length - 8)
|
|
48
|
+
expect(view.getUint16(20, true)).toBe(1) // PCM
|
|
49
|
+
expect(view.getUint16(22, true)).toBe(1) // mono
|
|
50
|
+
expect(view.getUint32(24, true)).toBe(24000) // sample rate
|
|
51
|
+
expect(view.getUint16(34, true)).toBe(16) // bits per sample
|
|
52
|
+
expect(view.getUint32(40, true)).toBe(pcm.length) // data size
|
|
53
|
+
// Payload preserved
|
|
54
|
+
expect(Array.from(wav.slice(44))).toEqual(Array.from(pcm))
|
|
55
|
+
})
|
|
56
|
+
})
|