@effect-uai/google 0.5.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Gemini.d.mts.map +1 -1
- package/dist/Gemini.mjs +7 -2
- package/dist/Gemini.mjs.map +1 -1
- package/dist/GeminiEmbedding.d.mts.map +1 -1
- package/dist/GeminiEmbedding.mjs +3 -3
- package/dist/GeminiEmbedding.mjs.map +1 -1
- package/dist/GeminiSynthesizer.d.mts +2 -0
- package/dist/GeminiSynthesizer.d.mts.map +1 -1
- package/dist/GeminiSynthesizer.mjs +38 -6
- package/dist/GeminiSynthesizer.mjs.map +1 -1
- package/dist/GeminiSynthesizer.test.mjs +14 -48
- package/dist/GeminiSynthesizer.test.mjs.map +1 -1
- package/dist/LyriaGenerator.d.mts +18 -18
- package/dist/LyriaGenerator.d.mts.map +1 -1
- package/dist/LyriaGenerator.mjs +59 -42
- package/dist/LyriaGenerator.mjs.map +1 -1
- package/dist/LyriaGenerator.test.mjs +26 -72
- package/dist/LyriaGenerator.test.mjs.map +1 -1
- package/dist/codec.d.mts +13 -5
- package/dist/codec.d.mts.map +1 -1
- package/dist/codec.mjs +21 -4
- package/dist/codec.mjs.map +1 -1
- package/dist/index.d.mts +2 -3
- package/dist/index.mjs +1 -2
- package/dist/models.d.mts +1 -10
- package/dist/models.d.mts.map +1 -1
- package/package.json +2 -6
- package/src/Gemini.ts +15 -1
- package/src/GeminiEmbedding.ts +14 -9
- package/src/GeminiSynthesizer.test.ts +13 -67
- package/src/GeminiSynthesizer.ts +56 -5
- package/src/LyriaGenerator.test.ts +34 -78
- package/src/LyriaGenerator.ts +92 -73
- package/src/codec.ts +55 -18
- package/src/index.ts +0 -1
- package/src/models.ts +0 -17
- package/dist/GeminiTranscriber.d.mts +0 -42
- package/dist/GeminiTranscriber.d.mts.map +0 -1
- package/dist/GeminiTranscriber.mjs +0 -80
- package/dist/GeminiTranscriber.mjs.map +0 -1
- package/dist/GeminiTranscriber.test.d.mts +0 -1
- package/dist/GeminiTranscriber.test.mjs +0 -125
- package/dist/GeminiTranscriber.test.mjs.map +0 -1
- package/src/GeminiTranscriber.test.ts +0 -136
- package/src/GeminiTranscriber.ts +0 -204
|
@@ -1,81 +1,17 @@
|
|
|
1
|
-
import { Effect, Layer, Redacted, Stream } from "effect"
|
|
1
|
+
import { Duration, Effect, Layer, Redacted, Stream } from "effect"
|
|
2
2
|
import { FetchHttpClient } from "effect/unstable/http"
|
|
3
3
|
import { describe, expect, expectTypeOf, it } from "vitest"
|
|
4
4
|
import type * as AiError from "@effect-uai/core/AiError"
|
|
5
|
-
import
|
|
5
|
+
import type { GenerateResult } from "@effect-uai/core/Music"
|
|
6
6
|
import { promptsInput } from "@effect-uai/core/Music"
|
|
7
|
+
import * as MusicGenerator from "@effect-uai/core/MusicGenerator"
|
|
7
8
|
import * as LyriaGenerator from "./LyriaGenerator.js"
|
|
8
9
|
|
|
9
10
|
const cfg: LyriaGenerator.Config = { apiKey: Redacted.make("test-key") }
|
|
10
11
|
// FetchHttpClient is required for `make`, but these tests only exercise the
|
|
11
|
-
// compile-time gating and runtime Unsupported branches
|
|
12
|
+
// compile-time gating and runtime Unsupported branches, no real HTTP is made.
|
|
12
13
|
const live = Layer.provide(LyriaGenerator.layer(cfg), FetchHttpClient.layer)
|
|
13
14
|
|
|
14
|
-
describe("buildPrompt", () => {
|
|
15
|
-
it("returns a plain string when prompts is a string", () => {
|
|
16
|
-
const out = LyriaGenerator.buildPrompt({
|
|
17
|
-
model: "lyria-3-clip-preview",
|
|
18
|
-
prompts: "upbeat indie pop",
|
|
19
|
-
})
|
|
20
|
-
expect(out).toBe("upbeat indie pop")
|
|
21
|
-
})
|
|
22
|
-
|
|
23
|
-
it("joins weighted prompts with weight annotations", () => {
|
|
24
|
-
const out = LyriaGenerator.buildPrompt({
|
|
25
|
-
model: "lyria-3-clip-preview",
|
|
26
|
-
prompts: [
|
|
27
|
-
{ text: "minimal techno", weight: 1.0 },
|
|
28
|
-
{ text: "1980s synthwave", weight: 0.3 },
|
|
29
|
-
],
|
|
30
|
-
})
|
|
31
|
-
expect(out).toContain("minimal techno")
|
|
32
|
-
expect(out).toContain("1980s synthwave (weight 0.3)")
|
|
33
|
-
})
|
|
34
|
-
|
|
35
|
-
it("omits the weight annotation for weight === 1", () => {
|
|
36
|
-
const out = LyriaGenerator.buildPrompt({
|
|
37
|
-
model: "lyria-3-clip-preview",
|
|
38
|
-
prompts: [{ text: "ambient", weight: 1 }],
|
|
39
|
-
})
|
|
40
|
-
expect(out).toBe("ambient")
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
it("splices in bpm, scale, duration, and instrumental hints", () => {
|
|
44
|
-
const out = LyriaGenerator.buildPrompt({
|
|
45
|
-
model: "lyria-3-clip-preview",
|
|
46
|
-
prompts: "house",
|
|
47
|
-
bpm: 124,
|
|
48
|
-
scale: "C_MAJOR",
|
|
49
|
-
durationSeconds: 30,
|
|
50
|
-
instrumental: true,
|
|
51
|
-
})
|
|
52
|
-
expect(out).toContain("Instrumental only")
|
|
53
|
-
expect(out).toContain("BPM: 124")
|
|
54
|
-
expect(out).toContain("Key/scale: C_MAJOR")
|
|
55
|
-
expect(out).toContain("Target duration: 30s")
|
|
56
|
-
})
|
|
57
|
-
|
|
58
|
-
it("appends a lyrics block when lyrics is set and instrumental is not true", () => {
|
|
59
|
-
const out = LyriaGenerator.buildPrompt({
|
|
60
|
-
model: "lyria-3-clip-preview",
|
|
61
|
-
prompts: "indie rock",
|
|
62
|
-
lyrics: "[Verse]\nhello world",
|
|
63
|
-
})
|
|
64
|
-
expect(out).toContain("Lyrics:")
|
|
65
|
-
expect(out).toContain("[Verse]")
|
|
66
|
-
})
|
|
67
|
-
|
|
68
|
-
it("drops the lyrics block when instrumental is true", () => {
|
|
69
|
-
const out = LyriaGenerator.buildPrompt({
|
|
70
|
-
model: "lyria-3-clip-preview",
|
|
71
|
-
prompts: "indie rock",
|
|
72
|
-
lyrics: "[Verse]\nhello world",
|
|
73
|
-
instrumental: true,
|
|
74
|
-
})
|
|
75
|
-
expect(out).not.toContain("Lyrics:")
|
|
76
|
-
})
|
|
77
|
-
})
|
|
78
|
-
|
|
79
15
|
describe("containerToMimeType", () => {
|
|
80
16
|
it("maps mp3 and wav", async () => {
|
|
81
17
|
expect(await Effect.runPromise(LyriaGenerator.containerToMimeType("mp3"))).toBe("audio/mp3")
|
|
@@ -115,7 +51,7 @@ describe("LyriaGenerator capability guards (runtime)", () => {
|
|
|
115
51
|
Stream.runDrain(
|
|
116
52
|
s.streamGenerationFrom(Stream.fromIterable([promptsInput([{ text: "x" }])]), {
|
|
117
53
|
model: "lyria-3-clip-preview",
|
|
118
|
-
|
|
54
|
+
prompt: "",
|
|
119
55
|
}),
|
|
120
56
|
),
|
|
121
57
|
)
|
|
@@ -131,32 +67,52 @@ describe("LyriaGenerator capability guards (runtime)", () => {
|
|
|
131
67
|
describe("LyriaGenerator Layer (compile-time)", () => {
|
|
132
68
|
it("leaves `MusicInteractiveSession` unsatisfied when using `streamGenerationFrom` against this Layer", () => {
|
|
133
69
|
const inputs = Stream.fromIterable([promptsInput([{ text: "techno" }])])
|
|
134
|
-
const
|
|
70
|
+
const events = inputs.pipe(
|
|
135
71
|
MusicGenerator.streamGenerationFrom({
|
|
136
72
|
model: "lyria-3-clip-preview",
|
|
137
|
-
|
|
73
|
+
prompt: "",
|
|
138
74
|
}),
|
|
139
75
|
)
|
|
140
|
-
const provided = Stream.runDrain(
|
|
76
|
+
const provided = Stream.runDrain(events).pipe(Effect.provide(live))
|
|
141
77
|
expectTypeOf(provided).toEqualTypeOf<
|
|
142
78
|
Effect.Effect<void, AiError.AiError, MusicGenerator.MusicInteractiveSession>
|
|
143
79
|
>()
|
|
144
80
|
})
|
|
145
81
|
|
|
146
|
-
it("sync `generate`
|
|
82
|
+
it("sync `generate` returns GenerateResult and requires no marker", () => {
|
|
147
83
|
const gen = MusicGenerator.generate({
|
|
148
84
|
model: "lyria-3-clip-preview",
|
|
149
|
-
|
|
85
|
+
prompt: "ambient",
|
|
150
86
|
}).pipe(Effect.provide(live))
|
|
151
|
-
expectTypeOf(gen).toEqualTypeOf<
|
|
152
|
-
|
|
153
|
-
|
|
87
|
+
expectTypeOf(gen).toEqualTypeOf<Effect.Effect<GenerateResult, AiError.AiError, never>>()
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
it("chunked `streamGeneration` requires no marker", () => {
|
|
154
91
|
const _stream = Stream.runCollect(
|
|
155
92
|
MusicGenerator.streamGeneration({
|
|
156
93
|
model: "lyria-3-clip-preview",
|
|
157
|
-
|
|
94
|
+
prompt: "ambient",
|
|
158
95
|
}),
|
|
159
96
|
).pipe(Effect.provide(live))
|
|
160
97
|
expect(_stream).toBeDefined()
|
|
161
98
|
})
|
|
99
|
+
|
|
100
|
+
it("Lyria-typed request accepts no `instrumental` field", () => {
|
|
101
|
+
const _req: LyriaGenerator.LyriaGenerateRequest = {
|
|
102
|
+
model: "lyria-3-clip-preview",
|
|
103
|
+
prompt: "ambient",
|
|
104
|
+
// @ts-expect-error: instrumental was removed from LyriaGenerateRequest in v0.7
|
|
105
|
+
instrumental: true,
|
|
106
|
+
}
|
|
107
|
+
expect(_req).toBeDefined()
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
it("Common request supports `duration: Duration.Duration`", () => {
|
|
111
|
+
const _req: MusicGenerator.CommonGenerateMusicRequest = {
|
|
112
|
+
model: "lyria-3-clip-preview",
|
|
113
|
+
prompt: "ambient",
|
|
114
|
+
duration: Duration.seconds(30),
|
|
115
|
+
}
|
|
116
|
+
expect(_req).toBeDefined()
|
|
117
|
+
})
|
|
162
118
|
})
|
package/src/LyriaGenerator.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
|
-
Array as Arr,
|
|
3
2
|
Context,
|
|
3
|
+
Duration,
|
|
4
4
|
Effect,
|
|
5
5
|
Encoding,
|
|
6
6
|
Layer,
|
|
7
7
|
Match,
|
|
8
|
-
Option,
|
|
9
8
|
Redacted,
|
|
10
9
|
Result,
|
|
11
10
|
Schema,
|
|
@@ -14,13 +13,16 @@ import {
|
|
|
14
13
|
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
15
14
|
import * as AiError from "@effect-uai/core/AiError"
|
|
16
15
|
import type { AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
|
|
16
|
+
import { warnDroppedWhen } from "@effect-uai/core/Capabilities"
|
|
17
17
|
import type {
|
|
18
18
|
CommonGenerateMusicRequest,
|
|
19
19
|
CommonStreamGenerateMusicRequest,
|
|
20
|
+
GenerateResult,
|
|
20
21
|
MusicResult,
|
|
21
22
|
MusicSessionInput,
|
|
22
|
-
|
|
23
|
+
MusicStreamEvent,
|
|
23
24
|
} from "@effect-uai/core/Music"
|
|
25
|
+
import { singleVariant } from "@effect-uai/core/Music"
|
|
24
26
|
import { MusicGenerator, type MusicGeneratorService } from "@effect-uai/core/MusicGenerator"
|
|
25
27
|
import type { LyriaModel } from "./models.js"
|
|
26
28
|
|
|
@@ -29,25 +31,34 @@ import type { LyriaModel } from "./models.js"
|
|
|
29
31
|
// ---------------------------------------------------------------------------
|
|
30
32
|
|
|
31
33
|
/**
|
|
32
|
-
* Lyria-typed generate request. Narrows `model` to `LyriaModel`.
|
|
33
|
-
*
|
|
34
|
-
* `
|
|
35
|
-
* `
|
|
36
|
-
*
|
|
34
|
+
* Lyria-typed generate request. Narrows `model` to `LyriaModel`.
|
|
35
|
+
*
|
|
36
|
+
* Lyria 3 sync `generateContent` has no structured wire fields for
|
|
37
|
+
* `lyrics` / `duration` / `seed` / `instrumental` / `bpm` / `scale` /
|
|
38
|
+
* etc. Those fields on the Common request are logged as
|
|
39
|
+
* dropped-bucket-2 hints; the adapter never rewrites or augments the
|
|
40
|
+
* caller's prompt on their behalf. If you want lyrics or vocal
|
|
41
|
+
* suppression, include them in your `prompt` text yourself.
|
|
42
|
+
*
|
|
43
|
+
* Output format: only `audio/mp3` and `audio/wav` are accepted; other
|
|
44
|
+
* containers fail `Unsupported`. The Clip model variant is fixed at
|
|
45
|
+
* `audio/mp3`.
|
|
37
46
|
*/
|
|
38
47
|
export type LyriaGenerateRequest = Omit<CommonGenerateMusicRequest, "model"> & {
|
|
39
48
|
readonly model: LyriaModel
|
|
40
49
|
}
|
|
41
50
|
|
|
42
51
|
export type LyriaGeneratorService = {
|
|
43
|
-
readonly generate: (
|
|
52
|
+
readonly generate: (
|
|
53
|
+
request: LyriaGenerateRequest,
|
|
54
|
+
) => Effect.Effect<GenerateResult, AiError.AiError>
|
|
44
55
|
readonly streamGeneration: (
|
|
45
56
|
request: LyriaGenerateRequest,
|
|
46
57
|
) => Stream.Stream<AudioChunk, AiError.AiError>
|
|
47
58
|
readonly streamGenerationFrom: <E, R>(
|
|
48
59
|
input: Stream.Stream<MusicSessionInput, E, R>,
|
|
49
60
|
request: CommonStreamGenerateMusicRequest,
|
|
50
|
-
) => Stream.Stream<
|
|
61
|
+
) => Stream.Stream<MusicStreamEvent, AiError.AiError | E, R>
|
|
51
62
|
}
|
|
52
63
|
|
|
53
64
|
export class LyriaGenerator extends Context.Service<LyriaGenerator, LyriaGeneratorService>()(
|
|
@@ -60,7 +71,7 @@ export type Config = {
|
|
|
60
71
|
}
|
|
61
72
|
|
|
62
73
|
// ---------------------------------------------------------------------------
|
|
63
|
-
// Codec
|
|
74
|
+
// Codec: AudioFormat → Lyria mimeType
|
|
64
75
|
// ---------------------------------------------------------------------------
|
|
65
76
|
|
|
66
77
|
export type LyriaResponseMimeType = "audio/mp3" | "audio/wav"
|
|
@@ -85,10 +96,10 @@ export const containerToMimeType: (
|
|
|
85
96
|
)
|
|
86
97
|
|
|
87
98
|
/**
|
|
88
|
-
* Is this a Lyria 3 Clip model variant? Clip is fixed at 30 s MP3
|
|
99
|
+
* Is this a Lyria 3 Clip model variant? Clip is fixed at 30 s MP3, it
|
|
89
100
|
* does not accept a `responseFormat` on `generationConfig` (the live
|
|
90
|
-
* API rejects unknown fields
|
|
91
|
-
*
|
|
101
|
+
* API rejects unknown fields). Pro accepts the documented
|
|
102
|
+
* `responseFormat` knob.
|
|
92
103
|
*/
|
|
93
104
|
const isClipModel = (model: string): boolean => model.includes("clip")
|
|
94
105
|
|
|
@@ -110,38 +121,51 @@ export const realizedFormat: (mime: LyriaResponseMimeType) => AudioFormat =
|
|
|
110
121
|
Match.exhaustive,
|
|
111
122
|
)
|
|
112
123
|
|
|
124
|
+
// ---------------------------------------------------------------------------
|
|
125
|
+
// Bucket-2 warn-and-drop for Common fields Lyria 3 can't honor
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
|
|
113
128
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
129
|
+
* Warn (don't fail) when the caller passed Common fields Lyria 3 has
|
|
130
|
+
* no wire field for. Bucket-2 per capabilities policy: provider has
|
|
131
|
+
* no structured interpretation; the output is still valid music, just
|
|
132
|
+
* less aligned with the caller's hint. We never embed these into the
|
|
133
|
+
* prompt on the caller's behalf, prompt construction is the
|
|
134
|
+
* developer's job.
|
|
118
135
|
*/
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
136
|
+
const warnDroppedHints = (request: LyriaGenerateRequest): Effect.Effect<void> =>
|
|
137
|
+
Effect.all(
|
|
138
|
+
[
|
|
139
|
+
warnDroppedWhen(request.lyrics, {
|
|
140
|
+
provider: "lyria",
|
|
141
|
+
capability: "lyrics",
|
|
142
|
+
field: "lyrics",
|
|
143
|
+
reason:
|
|
144
|
+
"Lyria 3 sync has no `lyrics` wire field. If you want vocals to follow specific lyrics, embed them in your prompt yourself (e.g. with `[Verse]` / `[Chorus]` tags).",
|
|
145
|
+
}),
|
|
146
|
+
warnDroppedWhen(
|
|
147
|
+
request.duration !== undefined ? Duration.toMillis(request.duration) : undefined,
|
|
148
|
+
{
|
|
149
|
+
provider: "lyria",
|
|
150
|
+
capability: "duration",
|
|
151
|
+
field: "duration",
|
|
152
|
+
reason:
|
|
153
|
+
"Lyria 3 clip is fixed at 30 s; pro derives duration from prompt content. There is no wire field for duration. If you want to steer length, mention it in your prompt.",
|
|
154
|
+
},
|
|
155
|
+
),
|
|
156
|
+
warnDroppedWhen(request.seed, {
|
|
157
|
+
provider: "lyria",
|
|
158
|
+
capability: "seed",
|
|
159
|
+
field: "seed",
|
|
160
|
+
reason:
|
|
161
|
+
"Lyria 3 (Gemini surface) does not expose a seed parameter. Output is non-deterministic across identical prompts. Use Lyria 2 (Vertex) for seeded generation.",
|
|
162
|
+
}),
|
|
163
|
+
],
|
|
164
|
+
{ discard: true },
|
|
127
165
|
)
|
|
128
|
-
const hints = Arr.getSomes([
|
|
129
|
-
request.instrumental === true ? Option.some("Instrumental only — no vocals.") : Option.none(),
|
|
130
|
-
request.bpm !== undefined ? Option.some(`BPM: ${request.bpm}.`) : Option.none(),
|
|
131
|
-
request.scale !== undefined ? Option.some(`Key/scale: ${request.scale}.`) : Option.none(),
|
|
132
|
-
request.durationSeconds !== undefined
|
|
133
|
-
? Option.some(`Target duration: ${request.durationSeconds}s.`)
|
|
134
|
-
: Option.none(),
|
|
135
|
-
])
|
|
136
|
-
const lyricsBlock =
|
|
137
|
-
request.lyrics !== undefined && request.instrumental !== true
|
|
138
|
-
? `\n\nLyrics:\n${request.lyrics}`
|
|
139
|
-
: ""
|
|
140
|
-
return [promptText, ...hints].join(" ") + lyricsBlock
|
|
141
|
-
}
|
|
142
166
|
|
|
143
167
|
// ---------------------------------------------------------------------------
|
|
144
|
-
// Wire response schema
|
|
168
|
+
// Wire response schema
|
|
145
169
|
// ---------------------------------------------------------------------------
|
|
146
170
|
|
|
147
171
|
const InlineData = Schema.Struct({
|
|
@@ -245,26 +269,23 @@ const generateImpl =
|
|
|
245
269
|
(cfg: Config) =>
|
|
246
270
|
(
|
|
247
271
|
request: LyriaGenerateRequest,
|
|
248
|
-
): Effect.Effect<
|
|
272
|
+
): Effect.Effect<GenerateResult, AiError.AiError, HttpClient.HttpClient> =>
|
|
249
273
|
Effect.gen(function* () {
|
|
250
274
|
const client = yield* HttpClient.HttpClient
|
|
251
275
|
const requestedContainer = (request.outputFormat?.container ??
|
|
252
276
|
"mp3") satisfies AudioFormat["container"]
|
|
253
|
-
const
|
|
254
|
-
//
|
|
255
|
-
//
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
})
|
|
262
|
-
}
|
|
263
|
-
const promptText = buildPrompt(request)
|
|
277
|
+
const requestedMime = yield* containerToMimeType(requestedContainer)
|
|
278
|
+
// lyria-3-clip-preview has no `responseFormat` wire field and always
|
|
279
|
+
// emits mp3, so there is no format to send and no 400 to surface. Rather
|
|
280
|
+
// than keep a per-model table that rejects `wav`, report the format the
|
|
281
|
+
// model actually produces (mp3) so the result is never mislabeled.
|
|
282
|
+
// lyria-3-pro honours the requested container.
|
|
283
|
+
const mimeType = isClipModel(request.model) ? ("audio/mp3" as const) : requestedMime
|
|
284
|
+
yield* warnDroppedHints(request)
|
|
264
285
|
const body = isClipModel(request.model)
|
|
265
|
-
? { contents: [{ parts: [{ text:
|
|
286
|
+
? { contents: [{ parts: [{ text: request.prompt }] }] }
|
|
266
287
|
: {
|
|
267
|
-
contents: [{ parts: [{ text:
|
|
288
|
+
contents: [{ parts: [{ text: request.prompt }] }],
|
|
268
289
|
generationConfig: {
|
|
269
290
|
responseModalities: ["AUDIO", "TEXT"],
|
|
270
291
|
responseFormat: { audio: { mimeType } },
|
|
@@ -290,10 +311,6 @@ const generateImpl =
|
|
|
290
311
|
)
|
|
291
312
|
const { inline, text } = extractParts(wire)
|
|
292
313
|
if (inline === undefined) {
|
|
293
|
-
// No audio came back. Most common cause: Lyria filtered the prompt
|
|
294
|
-
// (named-artist references, copyrighted lyrics, etc.). Surface the
|
|
295
|
-
// full response body — text parts, finish reasons, prompt-feedback
|
|
296
|
-
// blocks — so the caller can see why.
|
|
297
314
|
return yield* new AiError.GenerationFailed({
|
|
298
315
|
provider: "lyria",
|
|
299
316
|
raw: {
|
|
@@ -305,12 +322,13 @@ const generateImpl =
|
|
|
305
322
|
})
|
|
306
323
|
}
|
|
307
324
|
const bytes = yield* decodeBase64ToBytes(inline.data)
|
|
308
|
-
|
|
309
|
-
format: realizedFormat(mimeType),
|
|
310
|
-
|
|
325
|
+
const result: MusicResult = {
|
|
326
|
+
audio: { format: realizedFormat(mimeType), bytes },
|
|
327
|
+
provider: "lyria",
|
|
311
328
|
...(text !== undefined && { lyrics: text }),
|
|
312
|
-
watermark:
|
|
313
|
-
}
|
|
329
|
+
watermark: "synthid",
|
|
330
|
+
}
|
|
331
|
+
return singleVariant(result)
|
|
314
332
|
})
|
|
315
333
|
|
|
316
334
|
const streamGenerationImpl =
|
|
@@ -320,22 +338,22 @@ const streamGenerationImpl =
|
|
|
320
338
|
): Stream.Stream<AudioChunk, AiError.AiError, HttpClient.HttpClient> =>
|
|
321
339
|
Stream.unwrap(
|
|
322
340
|
Effect.map(generateImpl(cfg)(request), (result) =>
|
|
323
|
-
Stream.succeed<AudioChunk>({ bytes: result.bytes }),
|
|
341
|
+
Stream.succeed<AudioChunk>({ bytes: result.primary.audio.bytes }),
|
|
324
342
|
),
|
|
325
343
|
)
|
|
326
344
|
|
|
327
345
|
/**
|
|
328
|
-
* Lyria 3 sync has no bidirectional session at the wire level
|
|
329
|
-
* Lyria RealTime (
|
|
330
|
-
*
|
|
346
|
+
* Lyria 3 sync has no bidirectional session at the wire level, that's
|
|
347
|
+
* Lyria RealTime (separate Layer). The provider Layer here also does
|
|
348
|
+
* NOT register `MusicInteractiveSession`, so callers using
|
|
331
349
|
* `MusicGenerator.streamGenerationFrom` against only this Layer get a
|
|
332
350
|
* compile-time error before this runtime fallback ever fires.
|
|
333
351
|
*/
|
|
334
352
|
const streamGenerationFromUnsupported = <E, R>(
|
|
335
353
|
_input: Stream.Stream<MusicSessionInput, E, R>,
|
|
336
354
|
_request: CommonStreamGenerateMusicRequest,
|
|
337
|
-
): Stream.Stream<
|
|
338
|
-
const fail: Stream.Stream<
|
|
355
|
+
): Stream.Stream<MusicStreamEvent, AiError.AiError | E, R> => {
|
|
356
|
+
const fail: Stream.Stream<MusicStreamEvent, AiError.AiError | E, R> = Stream.fail(
|
|
339
357
|
new AiError.Unsupported({
|
|
340
358
|
provider: "lyria",
|
|
341
359
|
capability: "streamGenerationFrom",
|
|
@@ -365,7 +383,7 @@ export const make = (
|
|
|
365
383
|
* Layer that registers both the provider-specific `LyriaGenerator` tag
|
|
366
384
|
* and the generic `MusicGenerator` tag.
|
|
367
385
|
*
|
|
368
|
-
* Does NOT register the `MusicInteractiveSession` capability marker
|
|
386
|
+
* Does NOT register the `MusicInteractiveSession` capability marker,
|
|
369
387
|
* Lyria 3 sync has no bidirectional session at the wire level. Code
|
|
370
388
|
* calling `MusicGenerator.streamGenerationFrom` will fail to typecheck
|
|
371
389
|
* against this Layer alone, which is the intended UX.
|
|
@@ -380,9 +398,10 @@ export const layer = (
|
|
|
380
398
|
Effect.map(
|
|
381
399
|
make(cfg),
|
|
382
400
|
(s): MusicGeneratorService => ({
|
|
383
|
-
generate: (req: CommonGenerateMusicRequest) =>
|
|
401
|
+
generate: (req: CommonGenerateMusicRequest) =>
|
|
402
|
+
s.generate({ ...req, model: req.model as LyriaModel }),
|
|
384
403
|
streamGeneration: (req: CommonStreamGenerateMusicRequest) =>
|
|
385
|
-
s.streamGeneration(req as
|
|
404
|
+
s.streamGeneration({ ...req, model: req.model as LyriaModel }),
|
|
386
405
|
streamGenerationFrom: s.streamGenerationFrom,
|
|
387
406
|
}),
|
|
388
407
|
),
|
package/src/codec.ts
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import { Array as Arr, Encoding, Match, Option, Result, Schema, pipe } from "effect"
|
|
2
|
-
import type {
|
|
2
|
+
import type {
|
|
3
|
+
ContentBlock,
|
|
4
|
+
ToolCall,
|
|
5
|
+
InputImage,
|
|
6
|
+
HistoryItem,
|
|
7
|
+
Message,
|
|
8
|
+
} from "@effect-uai/core/Items"
|
|
3
9
|
import type { ToolDescriptor } from "@effect-uai/core/Tool"
|
|
4
10
|
import type { Turn } from "@effect-uai/core/Turn"
|
|
11
|
+
import type { CommonRequest } from "@effect-uai/core/LanguageModel"
|
|
5
12
|
|
|
6
13
|
// ---------------------------------------------------------------------------
|
|
7
14
|
// Wire schemas - the subset of Gemini's streamGenerateContent payload we
|
|
@@ -183,7 +190,7 @@ const systemMessageText = (message: Message): Result.Result<string, void> => {
|
|
|
183
190
|
return text.length === 0 ? Result.failVoid : Result.succeed(text)
|
|
184
191
|
}
|
|
185
192
|
|
|
186
|
-
const allMessages = (history: ReadonlyArray<
|
|
193
|
+
const allMessages = (history: ReadonlyArray<HistoryItem>): ReadonlyArray<Message> =>
|
|
187
194
|
pipe(
|
|
188
195
|
history,
|
|
189
196
|
Arr.filterMap((item) => (item.type === "message" ? Result.succeed(item) : Result.failVoid)),
|
|
@@ -192,7 +199,7 @@ const allMessages = (history: ReadonlyArray<Item>): ReadonlyArray<Message> =>
|
|
|
192
199
|
// ---------------------------------------------------------------------------
|
|
193
200
|
// Function-call round-trip
|
|
194
201
|
//
|
|
195
|
-
// `
|
|
202
|
+
// `ToolCall.arguments` and `ToolCallOutput.output` are JSON-encoded
|
|
196
203
|
// strings; Gemini's `functionCall.args` and `functionResponse.response`
|
|
197
204
|
// expect parsed JSON *objects*. Decode via Schema:
|
|
198
205
|
// - object payload → use as-is
|
|
@@ -218,20 +225,21 @@ const parsedArgs = parsedJsonObject(() => ({}))
|
|
|
218
225
|
const parsedResponse = parsedJsonObject((raw) => ({ output: raw }))
|
|
219
226
|
|
|
220
227
|
/**
|
|
221
|
-
* `
|
|
228
|
+
* `ToolCallOutput` only carries `call_id`; Gemini's `functionResponse`
|
|
222
229
|
* requires the declared function `name`. Resolve the name by scanning prior
|
|
223
230
|
* `function_call` items in the history for a matching `call_id`. If we
|
|
224
231
|
* cannot resolve, fall back to `call_id` as the name - imperfect but
|
|
225
232
|
* preserves stream shape so the model sees *some* response.
|
|
226
233
|
*/
|
|
227
|
-
const isFunctionCallItem = (item:
|
|
234
|
+
const isFunctionCallItem = (item: HistoryItem): item is ToolCall => item.type === "function_call"
|
|
228
235
|
|
|
229
|
-
const nameForCallId = (
|
|
236
|
+
const nameForCallId = (
|
|
237
|
+
history: ReadonlyArray<HistoryItem>,
|
|
238
|
+
call_id: string,
|
|
239
|
+
): Option.Option<string> =>
|
|
230
240
|
pipe(
|
|
231
241
|
history,
|
|
232
|
-
Arr.findFirst(
|
|
233
|
-
(item): item is FunctionCall => isFunctionCallItem(item) && item.call_id === call_id,
|
|
234
|
-
),
|
|
242
|
+
Arr.findFirst((item): item is ToolCall => isFunctionCallItem(item) && item.call_id === call_id),
|
|
235
243
|
Option.map((f) => f.name),
|
|
236
244
|
)
|
|
237
245
|
|
|
@@ -244,7 +252,7 @@ const ProviderDataWithGeminiId = Schema.Struct({
|
|
|
244
252
|
})
|
|
245
253
|
const decodeProviderId = Schema.decodeUnknownResult(ProviderDataWithGeminiId)
|
|
246
254
|
|
|
247
|
-
const providerIdFor = (item:
|
|
255
|
+
const providerIdFor = (item: ToolCall): Option.Option<string> =>
|
|
248
256
|
pipe(
|
|
249
257
|
decodeProviderId(item.providerData),
|
|
250
258
|
Result.match({
|
|
@@ -254,8 +262,8 @@ const providerIdFor = (item: FunctionCall): Option.Option<string> =>
|
|
|
254
262
|
)
|
|
255
263
|
|
|
256
264
|
const itemToContent =
|
|
257
|
-
(history: ReadonlyArray<
|
|
258
|
-
(item:
|
|
265
|
+
(history: ReadonlyArray<HistoryItem>) =>
|
|
266
|
+
(item: HistoryItem): Result.Result<RequestContent, void> =>
|
|
259
267
|
Match.value(item).pipe(
|
|
260
268
|
Match.discriminatorsExhaustive("type")({
|
|
261
269
|
message: messageToContent,
|
|
@@ -334,10 +342,39 @@ const toolDescriptorsToTools = (tools: ReadonlyArray<ToolDescriptor>): ReadonlyA
|
|
|
334
342
|
},
|
|
335
343
|
]
|
|
336
344
|
|
|
345
|
+
/**
|
|
346
|
+
* Map our cross-provider `toolChoice` onto Gemini's `functionCallingConfig`.
|
|
347
|
+
* Gemini honours the same three modes plus a function allow-list, so the
|
|
348
|
+
* choice is mapped, not dropped. `undefined` keeps the API default (`AUTO`).
|
|
349
|
+
*/
|
|
350
|
+
const toolChoiceToFunctionCallingConfig = (
|
|
351
|
+
toolChoice: CommonRequest["toolChoice"],
|
|
352
|
+
): RequestToolConfig["functionCallingConfig"] =>
|
|
353
|
+
Match.value(toolChoice ?? "auto").pipe(
|
|
354
|
+
Match.when("auto", () => ({ mode: "AUTO" as const })),
|
|
355
|
+
Match.when("required", () => ({ mode: "ANY" as const })),
|
|
356
|
+
Match.when("none", () => ({ mode: "NONE" as const })),
|
|
357
|
+
Match.orElse((tc) => ({ mode: "ANY" as const, allowedFunctionNames: [tc.name] })),
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* URL-form images need Gemini's Files API (upload, then `fileData`); we
|
|
362
|
+
* don't pre-upload, so such a part would otherwise vanish from the request.
|
|
363
|
+
* The caller fails the request with `Unsupported` instead of silently
|
|
364
|
+
* dropping the image.
|
|
365
|
+
*/
|
|
366
|
+
export const hasUrlImageSource = (history: ReadonlyArray<HistoryItem>): boolean =>
|
|
367
|
+
history.some(
|
|
368
|
+
(item) =>
|
|
369
|
+
item.type === "message" &&
|
|
370
|
+
item.content.some((b) => b.type === "input_image" && b.source._tag === "url"),
|
|
371
|
+
)
|
|
372
|
+
|
|
337
373
|
export const buildRequestBody = (
|
|
338
|
-
history: ReadonlyArray<
|
|
374
|
+
history: ReadonlyArray<HistoryItem>,
|
|
339
375
|
generationConfig: Option.Option<GenerationConfig>,
|
|
340
376
|
tools: ReadonlyArray<ToolDescriptor> = [],
|
|
377
|
+
toolChoice: CommonRequest["toolChoice"] = undefined,
|
|
341
378
|
): RequestBody => {
|
|
342
379
|
const systemTexts = pipe(allMessages(history), Arr.filterMap(systemMessageText))
|
|
343
380
|
const contents = pipe(history, Arr.filterMap(itemToContent(history)))
|
|
@@ -353,7 +390,7 @@ export const buildRequestBody = (
|
|
|
353
390
|
}),
|
|
354
391
|
...(requestTools.length > 0 && {
|
|
355
392
|
tools: requestTools,
|
|
356
|
-
toolConfig: { functionCallingConfig:
|
|
393
|
+
toolConfig: { functionCallingConfig: toolChoiceToFunctionCallingConfig(toolChoice) },
|
|
357
394
|
}),
|
|
358
395
|
}
|
|
359
396
|
}
|
|
@@ -377,7 +414,7 @@ export type AccumulatedFunctionCall = {
|
|
|
377
414
|
readonly name: string
|
|
378
415
|
/** Wire id from Gemini 3, when present - echoed back on `functionResponse`. */
|
|
379
416
|
readonly providerId: Option.Option<string>
|
|
380
|
-
/** Args as JSON-encoded string, mirroring `Items.
|
|
417
|
+
/** Args as JSON-encoded string, mirroring `Items.ToolCall.arguments`. */
|
|
381
418
|
readonly arguments: string
|
|
382
419
|
}
|
|
383
420
|
|
|
@@ -530,10 +567,10 @@ export const ingestChunk = (acc: Accumulator, chunk: WireChunk): ChunkResult =>
|
|
|
530
567
|
}
|
|
531
568
|
}
|
|
532
569
|
|
|
533
|
-
const reasoningItems = (acc: Accumulator): ReadonlyArray<
|
|
570
|
+
const reasoningItems = (acc: Accumulator): ReadonlyArray<HistoryItem> =>
|
|
534
571
|
acc.reasoning.length > 0 ? [{ type: "reasoning", summary: acc.reasoning }] : []
|
|
535
572
|
|
|
536
|
-
const assistantMessageItems = (acc: Accumulator): ReadonlyArray<
|
|
573
|
+
const assistantMessageItems = (acc: Accumulator): ReadonlyArray<HistoryItem> =>
|
|
537
574
|
acc.text.length === 0
|
|
538
575
|
? []
|
|
539
576
|
: [
|
|
@@ -544,7 +581,7 @@ const assistantMessageItems = (acc: Accumulator): ReadonlyArray<Item> =>
|
|
|
544
581
|
},
|
|
545
582
|
]
|
|
546
583
|
|
|
547
|
-
const functionCallItems = (acc: Accumulator): ReadonlyArray<
|
|
584
|
+
const functionCallItems = (acc: Accumulator): ReadonlyArray<HistoryItem> =>
|
|
548
585
|
pipe(
|
|
549
586
|
acc.functionCalls,
|
|
550
587
|
Arr.map((c) => ({
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
export * as Gemini from "./Gemini.js"
|
|
2
2
|
export * as GeminiEmbedding from "./GeminiEmbedding.js"
|
|
3
3
|
export * as GeminiSynthesizer from "./GeminiSynthesizer.js"
|
|
4
|
-
export * as GeminiTranscriber from "./GeminiTranscriber.js"
|
|
5
4
|
export * as LyriaGenerator from "./LyriaGenerator.js"
|
|
6
5
|
export * as codec from "./codec.js"
|
|
7
6
|
export * from "./models.js"
|
package/src/models.ts
CHANGED
|
@@ -44,23 +44,6 @@ export type GeminiTtsModel =
|
|
|
44
44
|
// eslint-disable-next-line @typescript-eslint/ban-types
|
|
45
45
|
| (string & {})
|
|
46
46
|
|
|
47
|
-
/**
|
|
48
|
-
* Known Gemini models that accept audio input via `inlineData` /
|
|
49
|
-
* `fileData` on `generateContent`. Used by `GeminiTranscriber` —
|
|
50
|
-
* transcription is prompt-driven on top of these models, not a
|
|
51
|
-
* dedicated endpoint.
|
|
52
|
-
*
|
|
53
|
-
* Reference: https://ai.google.dev/gemini-api/docs/audio
|
|
54
|
-
*/
|
|
55
|
-
export type GeminiSttModel =
|
|
56
|
-
| "gemini-3-flash-preview"
|
|
57
|
-
| "gemini-3.1-pro-preview"
|
|
58
|
-
| "gemini-3.1-flash-lite-preview"
|
|
59
|
-
| "gemini-2.5-flash"
|
|
60
|
-
| "gemini-2.5-pro"
|
|
61
|
-
// eslint-disable-next-line @typescript-eslint/ban-types
|
|
62
|
-
| (string & {})
|
|
63
|
-
|
|
64
47
|
/**
|
|
65
48
|
* 30 prebuilt voice names for Gemini TTS. No custom-voice / cloning path
|
|
66
49
|
* is exposed on this surface, so the tail is stock-only (no
|