@ai-sdk/google 4.0.53 → 4.0.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/index.d.ts +62 -3
- package/dist/index.js +555 -28
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/google-provider.ts +27 -0
- package/src/index.ts +5 -0
- package/src/transcription/google-transcription-model-options.ts +50 -0
- package/src/transcription/google-transcription-model.ts +714 -0
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "@ai-sdk/provider-utils";
|
|
8
8
|
|
|
9
9
|
// src/version.ts
|
|
10
|
-
var VERSION = true ? "4.0.
|
|
10
|
+
var VERSION = true ? "4.0.54" : "0.0.0-test";
|
|
11
11
|
|
|
12
12
|
// src/google-embedding-model.ts
|
|
13
13
|
import {
|
|
@@ -8003,21 +8003,538 @@ var GoogleRealtimeModel = class {
|
|
|
8003
8003
|
}
|
|
8004
8004
|
};
|
|
8005
8005
|
|
|
8006
|
-
// src/
|
|
8006
|
+
// src/transcription/google-transcription-model.ts
|
|
8007
8007
|
import {
|
|
8008
8008
|
InvalidArgumentError as InvalidArgumentError2
|
|
8009
8009
|
} from "@ai-sdk/provider";
|
|
8010
8010
|
import {
|
|
8011
|
-
connectToWebSocket,
|
|
8012
8011
|
combineHeaders as combineHeaders9,
|
|
8013
|
-
|
|
8012
|
+
connectToWebSocket,
|
|
8014
8013
|
convertToBase64 as convertToBase644,
|
|
8014
|
+
createJsonResponseHandler as createJsonResponseHandler9,
|
|
8015
8015
|
parseProviderOptions as parseProviderOptions8,
|
|
8016
|
+
postJsonToApi as postJsonToApi7,
|
|
8017
|
+
resolve as resolve7,
|
|
8016
8018
|
safeParseJSON as safeParseJSON2,
|
|
8017
8019
|
serializeModelOptions as serializeModelOptions6,
|
|
8020
|
+
waitForWebSocketBufferDrain,
|
|
8018
8021
|
WORKFLOW_DESERIALIZE as WORKFLOW_DESERIALIZE7,
|
|
8019
|
-
WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE7
|
|
8020
|
-
|
|
8022
|
+
WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE7
|
|
8023
|
+
} from "@ai-sdk/provider-utils";
|
|
8024
|
+
import { z as z23 } from "zod/v4";
|
|
8025
|
+
|
|
8026
|
+
// src/transcription/google-transcription-model-options.ts
|
|
8027
|
+
import { z as z22 } from "zod/v4";
|
|
8028
|
+
var googleTranscriptionModelOptions = z22.object({
|
|
8029
|
+
/**
|
|
8030
|
+
* BCP-47 language codes providing hints about the languages present in the
|
|
8031
|
+
* audio. If omitted or empty, defaults to automatic language detection.
|
|
8032
|
+
*/
|
|
8033
|
+
languageCodes: z22.array(z22.string()).optional(),
|
|
8034
|
+
/**
|
|
8035
|
+
* Custom vocabulary phrases, which bias the speech recognition model
|
|
8036
|
+
* toward recognizing specific terms.
|
|
8037
|
+
*/
|
|
8038
|
+
customVocabulary: z22.array(z22.string()).optional(),
|
|
8039
|
+
/**
|
|
8040
|
+
* Enables word-level timestamp generation.
|
|
8041
|
+
*/
|
|
8042
|
+
wordTimestamp: z22.boolean().optional(),
|
|
8043
|
+
/**
|
|
8044
|
+
* Enables speaker diarization.
|
|
8045
|
+
*/
|
|
8046
|
+
diarization: z22.boolean().optional(),
|
|
8047
|
+
/**
|
|
8048
|
+
* Transcription output formatting mode.
|
|
8049
|
+
*
|
|
8050
|
+
* - `VERBATIM` (default): exact literal transcript preserving filler
|
|
8051
|
+
* words, repetitions, and false starts.
|
|
8052
|
+
* - `SMART`: cleans up and structures the transcript in real time —
|
|
8053
|
+
* disfluency removal, inline self-corrections, structured formatting
|
|
8054
|
+
* (lists, numbers, dates, paragraph breaks), and grammar/casing polish.
|
|
8055
|
+
*/
|
|
8056
|
+
mode: z22.enum(["SMART", "VERBATIM"]).optional()
|
|
8057
|
+
});
|
|
8058
|
+
|
|
8059
|
+
// src/transcription/google-transcription-model.ts
|
|
8060
|
+
var liveWebSocketPath = "google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
8061
|
+
var defaultFinishGraceMs = 3e3;
|
|
8062
|
+
function getLiveWebSocketURL(baseURL, apiKey) {
|
|
8063
|
+
const url = getRealtimeWebSocketURL(baseURL, liveWebSocketPath);
|
|
8064
|
+
url.searchParams.set("key", apiKey);
|
|
8065
|
+
return url;
|
|
8066
|
+
}
|
|
8067
|
+
function isLiveTranscriptionModelId(modelId) {
|
|
8068
|
+
return modelId.includes("-live");
|
|
8069
|
+
}
|
|
8070
|
+
var GoogleTranscriptionModel = class _GoogleTranscriptionModel {
|
|
8071
|
+
constructor(modelId, config) {
|
|
8072
|
+
this.modelId = modelId;
|
|
8073
|
+
this.config = config;
|
|
8074
|
+
this.specificationVersion = "v4";
|
|
8075
|
+
}
|
|
8076
|
+
static [WORKFLOW_SERIALIZE7](model) {
|
|
8077
|
+
return serializeModelOptions6({
|
|
8078
|
+
modelId: model.modelId,
|
|
8079
|
+
config: model.config
|
|
8080
|
+
});
|
|
8081
|
+
}
|
|
8082
|
+
static [WORKFLOW_DESERIALIZE7](options) {
|
|
8083
|
+
return new _GoogleTranscriptionModel(options.modelId, options.config);
|
|
8084
|
+
}
|
|
8085
|
+
get provider() {
|
|
8086
|
+
return this.config.provider;
|
|
8087
|
+
}
|
|
8088
|
+
async parseOptions(providerOptions) {
|
|
8089
|
+
return parseProviderOptions8({
|
|
8090
|
+
provider: "google",
|
|
8091
|
+
providerOptions,
|
|
8092
|
+
schema: googleTranscriptionModelOptions
|
|
8093
|
+
});
|
|
8094
|
+
}
|
|
8095
|
+
async doGenerate(options) {
|
|
8096
|
+
var _a, _b, _c, _d, _e, _f;
|
|
8097
|
+
if (isLiveTranscriptionModelId(this.modelId)) {
|
|
8098
|
+
throw new InvalidArgumentError2({
|
|
8099
|
+
argument: "modelId",
|
|
8100
|
+
message: `Model '${this.modelId}' only supports streaming transcription. Use experimental_streamTranscribe, or a unary model such as 'gemini-3.5-transcribe'.`
|
|
8101
|
+
});
|
|
8102
|
+
}
|
|
8103
|
+
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
8104
|
+
const warnings = [];
|
|
8105
|
+
const googleOptions = await this.parseOptions(options.providerOptions);
|
|
8106
|
+
const transcriptionConfig = buildTranscriptionConfig(googleOptions);
|
|
8107
|
+
const requestBody = {
|
|
8108
|
+
model: this.modelId,
|
|
8109
|
+
input: [
|
|
8110
|
+
{
|
|
8111
|
+
type: "audio",
|
|
8112
|
+
data: convertToBase644(options.audio),
|
|
8113
|
+
mime_type: options.mediaType
|
|
8114
|
+
}
|
|
8115
|
+
],
|
|
8116
|
+
...transcriptionConfig != null ? { generation_config: { transcription_config: transcriptionConfig } } : {}
|
|
8117
|
+
};
|
|
8118
|
+
const {
|
|
8119
|
+
value: response,
|
|
8120
|
+
responseHeaders,
|
|
8121
|
+
rawValue: rawResponse
|
|
8122
|
+
} = await postJsonToApi7({
|
|
8123
|
+
url: `${this.config.baseURL}/interactions`,
|
|
8124
|
+
headers: combineHeaders9(
|
|
8125
|
+
this.config.headers ? await resolve7(this.config.headers) : void 0,
|
|
8126
|
+
options.headers
|
|
8127
|
+
),
|
|
8128
|
+
body: requestBody,
|
|
8129
|
+
failedResponseHandler: googleFailedResponseHandler,
|
|
8130
|
+
successfulResponseHandler: createJsonResponseHandler9(
|
|
8131
|
+
googleInteractionsTranscriptionResponseSchema
|
|
8132
|
+
),
|
|
8133
|
+
abortSignal: options.abortSignal,
|
|
8134
|
+
fetch: this.config.fetch
|
|
8135
|
+
});
|
|
8136
|
+
let text = "";
|
|
8137
|
+
const segments = [];
|
|
8138
|
+
for (const step of (_d = response.steps) != null ? _d : []) {
|
|
8139
|
+
for (const content of (_e = step.content) != null ? _e : []) {
|
|
8140
|
+
if (content.type !== "text" || content.text == null) continue;
|
|
8141
|
+
text += content.text;
|
|
8142
|
+
for (const annotation of (_f = content.annotations) != null ? _f : []) {
|
|
8143
|
+
if (annotation.type !== "word_info") continue;
|
|
8144
|
+
const startSecond = parseOffsetSeconds(annotation.start_offset);
|
|
8145
|
+
const endSecond = parseOffsetSeconds(annotation.end_offset);
|
|
8146
|
+
if (annotation.text == null || startSecond == null || endSecond == null) {
|
|
8147
|
+
continue;
|
|
8148
|
+
}
|
|
8149
|
+
segments.push({ text: annotation.text, startSecond, endSecond });
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
8152
|
+
}
|
|
8153
|
+
return {
|
|
8154
|
+
text,
|
|
8155
|
+
segments,
|
|
8156
|
+
language: void 0,
|
|
8157
|
+
durationInSeconds: void 0,
|
|
8158
|
+
warnings,
|
|
8159
|
+
response: {
|
|
8160
|
+
timestamp: currentDate,
|
|
8161
|
+
modelId: this.modelId,
|
|
8162
|
+
headers: responseHeaders,
|
|
8163
|
+
body: rawResponse
|
|
8164
|
+
},
|
|
8165
|
+
...response.usage != null ? {
|
|
8166
|
+
providerMetadata: {
|
|
8167
|
+
google: { usage: response.usage }
|
|
8168
|
+
}
|
|
8169
|
+
} : {}
|
|
8170
|
+
};
|
|
8171
|
+
}
|
|
8172
|
+
async doStream(options) {
|
|
8173
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
8174
|
+
if (!isLiveTranscriptionModelId(this.modelId)) {
|
|
8175
|
+
throw new InvalidArgumentError2({
|
|
8176
|
+
argument: "modelId",
|
|
8177
|
+
message: `Model '${this.modelId}' does not support streaming transcription. Use a live model such as 'gemini-3.5-transcribe-live'.`
|
|
8178
|
+
});
|
|
8179
|
+
}
|
|
8180
|
+
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
8181
|
+
const warnings = [];
|
|
8182
|
+
const googleOptions = await this.parseOptions(options.providerOptions);
|
|
8183
|
+
validateLiveInputAudioFormat(options.inputAudioFormat);
|
|
8184
|
+
const headers = combineHeaders9(
|
|
8185
|
+
this.config.headers ? await resolve7(this.config.headers) : void 0,
|
|
8186
|
+
options.headers
|
|
8187
|
+
);
|
|
8188
|
+
let apiKey;
|
|
8189
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
8190
|
+
if (key.toLowerCase() === "x-goog-api-key" && value != null) {
|
|
8191
|
+
apiKey = value;
|
|
8192
|
+
}
|
|
8193
|
+
}
|
|
8194
|
+
if (apiKey == null) {
|
|
8195
|
+
throw new Error(
|
|
8196
|
+
"Google Generative AI API key is required for streaming transcription."
|
|
8197
|
+
);
|
|
8198
|
+
}
|
|
8199
|
+
const webSocketHeaders = Object.fromEntries(
|
|
8200
|
+
Object.entries(headers).filter(
|
|
8201
|
+
([key]) => key.toLowerCase() !== "x-goog-api-key"
|
|
8202
|
+
)
|
|
8203
|
+
);
|
|
8204
|
+
const setup = {
|
|
8205
|
+
model: getModelPath(this.modelId),
|
|
8206
|
+
inputAudioTranscription: (_d = buildAudioTranscriptionConfig(googleOptions)) != null ? _d : {}
|
|
8207
|
+
};
|
|
8208
|
+
return {
|
|
8209
|
+
request: { body: setup },
|
|
8210
|
+
response: {
|
|
8211
|
+
timestamp: currentDate,
|
|
8212
|
+
modelId: this.modelId
|
|
8213
|
+
},
|
|
8214
|
+
stream: createGoogleLiveTranscriptionStream({
|
|
8215
|
+
webSocket: this.config.webSocket,
|
|
8216
|
+
url: getLiveWebSocketURL(this.config.baseURL, apiKey),
|
|
8217
|
+
headers: webSocketHeaders,
|
|
8218
|
+
setup,
|
|
8219
|
+
inputAudioRate: (_e = options.inputAudioFormat.rate) != null ? _e : 16e3,
|
|
8220
|
+
finishGraceMs: (_g = (_f = this.config._internal) == null ? void 0 : _f.finishGraceMs) != null ? _g : defaultFinishGraceMs,
|
|
8221
|
+
warnings,
|
|
8222
|
+
audio: options.audio,
|
|
8223
|
+
abortSignal: options.abortSignal,
|
|
8224
|
+
includeRawChunks: options.includeRawChunks
|
|
8225
|
+
})
|
|
8226
|
+
};
|
|
8227
|
+
}
|
|
8228
|
+
};
|
|
8229
|
+
function createGoogleLiveTranscriptionStream({
|
|
8230
|
+
webSocket,
|
|
8231
|
+
url,
|
|
8232
|
+
headers,
|
|
8233
|
+
setup,
|
|
8234
|
+
inputAudioRate,
|
|
8235
|
+
finishGraceMs,
|
|
8236
|
+
warnings,
|
|
8237
|
+
audio,
|
|
8238
|
+
abortSignal,
|
|
8239
|
+
includeRawChunks
|
|
8240
|
+
}) {
|
|
8241
|
+
let finished = false;
|
|
8242
|
+
let cleanup = () => {
|
|
8243
|
+
};
|
|
8244
|
+
return new ReadableStream({
|
|
8245
|
+
start: (controller) => {
|
|
8246
|
+
let audioReader;
|
|
8247
|
+
let connection;
|
|
8248
|
+
let resolveSetupComplete;
|
|
8249
|
+
const setupComplete = new Promise((resolve8) => {
|
|
8250
|
+
resolveSetupComplete = resolve8;
|
|
8251
|
+
});
|
|
8252
|
+
let segmentCounter = 0;
|
|
8253
|
+
let segmentBuffer = "";
|
|
8254
|
+
let fullText = "";
|
|
8255
|
+
let latestInterim = "";
|
|
8256
|
+
let language;
|
|
8257
|
+
let audioEnded = false;
|
|
8258
|
+
let usageMetadata;
|
|
8259
|
+
let finishTimer;
|
|
8260
|
+
const segmentId = () => `google-segment-${segmentCounter}`;
|
|
8261
|
+
const cancelPendingFinish = () => {
|
|
8262
|
+
if (finishTimer != null) {
|
|
8263
|
+
clearTimeout(finishTimer);
|
|
8264
|
+
finishTimer = void 0;
|
|
8265
|
+
}
|
|
8266
|
+
};
|
|
8267
|
+
const schedulePendingFinish = () => {
|
|
8268
|
+
if (finished || !audioEnded) return;
|
|
8269
|
+
cancelPendingFinish();
|
|
8270
|
+
finishTimer = setTimeout(() => {
|
|
8271
|
+
finishTimer = void 0;
|
|
8272
|
+
finish();
|
|
8273
|
+
}, finishGraceMs);
|
|
8274
|
+
};
|
|
8275
|
+
cleanup = (closeCode) => {
|
|
8276
|
+
cancelPendingFinish();
|
|
8277
|
+
if (audioReader != null) {
|
|
8278
|
+
void audioReader.cancel().catch(() => {
|
|
8279
|
+
});
|
|
8280
|
+
} else {
|
|
8281
|
+
void audio.cancel().catch(() => {
|
|
8282
|
+
});
|
|
8283
|
+
}
|
|
8284
|
+
connection == null ? void 0 : connection.close(closeCode);
|
|
8285
|
+
};
|
|
8286
|
+
const finishWithError = (error) => {
|
|
8287
|
+
if (finished) return;
|
|
8288
|
+
finished = true;
|
|
8289
|
+
cleanup();
|
|
8290
|
+
controller.error(error);
|
|
8291
|
+
};
|
|
8292
|
+
const completeSegment = () => {
|
|
8293
|
+
if (segmentBuffer === "") {
|
|
8294
|
+
if (latestInterim === "") return;
|
|
8295
|
+
segmentBuffer = latestInterim;
|
|
8296
|
+
}
|
|
8297
|
+
latestInterim = "";
|
|
8298
|
+
controller.enqueue({
|
|
8299
|
+
type: "transcript-final",
|
|
8300
|
+
id: segmentId(),
|
|
8301
|
+
text: segmentBuffer
|
|
8302
|
+
});
|
|
8303
|
+
fullText += fullText === "" ? segmentBuffer : ` ${segmentBuffer}`;
|
|
8304
|
+
segmentBuffer = "";
|
|
8305
|
+
segmentCounter++;
|
|
8306
|
+
};
|
|
8307
|
+
const finish = () => {
|
|
8308
|
+
if (finished) return;
|
|
8309
|
+
completeSegment();
|
|
8310
|
+
finished = true;
|
|
8311
|
+
controller.enqueue({
|
|
8312
|
+
type: "finish",
|
|
8313
|
+
text: fullText,
|
|
8314
|
+
segments: [],
|
|
8315
|
+
language,
|
|
8316
|
+
durationInSeconds: void 0,
|
|
8317
|
+
...usageMetadata != null ? { providerMetadata: { google: { usageMetadata } } } : {}
|
|
8318
|
+
});
|
|
8319
|
+
controller.close();
|
|
8320
|
+
cleanup(1e3);
|
|
8321
|
+
};
|
|
8322
|
+
const sendAudio = async (socket) => {
|
|
8323
|
+
audioReader = audio.getReader();
|
|
8324
|
+
try {
|
|
8325
|
+
while (true) {
|
|
8326
|
+
const { done, value } = await audioReader.read();
|
|
8327
|
+
if (done || finished) break;
|
|
8328
|
+
socket.send(
|
|
8329
|
+
JSON.stringify({
|
|
8330
|
+
realtimeInput: {
|
|
8331
|
+
audio: {
|
|
8332
|
+
data: convertToBase644(value),
|
|
8333
|
+
mimeType: `audio/pcm;rate=${inputAudioRate}`
|
|
8334
|
+
}
|
|
8335
|
+
}
|
|
8336
|
+
})
|
|
8337
|
+
);
|
|
8338
|
+
await waitForWebSocketBufferDrain(socket);
|
|
8339
|
+
}
|
|
8340
|
+
} finally {
|
|
8341
|
+
audioReader.releaseLock();
|
|
8342
|
+
audioReader = void 0;
|
|
8343
|
+
}
|
|
8344
|
+
if (!finished) {
|
|
8345
|
+
socket.send(
|
|
8346
|
+
JSON.stringify({ realtimeInput: { audioStreamEnd: true } })
|
|
8347
|
+
);
|
|
8348
|
+
audioEnded = true;
|
|
8349
|
+
schedulePendingFinish();
|
|
8350
|
+
}
|
|
8351
|
+
};
|
|
8352
|
+
connection = connectToWebSocket({
|
|
8353
|
+
url,
|
|
8354
|
+
headers,
|
|
8355
|
+
webSocket,
|
|
8356
|
+
abortSignal,
|
|
8357
|
+
onAbort: finishWithError,
|
|
8358
|
+
onProcessingError: finishWithError,
|
|
8359
|
+
onOpen: (socket) => {
|
|
8360
|
+
controller.enqueue({ type: "stream-start", warnings });
|
|
8361
|
+
socket.send(JSON.stringify({ setup }));
|
|
8362
|
+
void setupComplete.then(() => finished ? void 0 : sendAudio(socket)).catch(finishWithError);
|
|
8363
|
+
},
|
|
8364
|
+
onMessageText: async (text) => {
|
|
8365
|
+
var _a, _b;
|
|
8366
|
+
if (finished) return;
|
|
8367
|
+
const parsed = await safeParseJSON2({ text });
|
|
8368
|
+
if (!parsed.success) return;
|
|
8369
|
+
const message = parsed.value;
|
|
8370
|
+
if (includeRawChunks) {
|
|
8371
|
+
controller.enqueue({ type: "raw", rawValue: message });
|
|
8372
|
+
}
|
|
8373
|
+
if (message.setupComplete != null) {
|
|
8374
|
+
resolveSetupComplete();
|
|
8375
|
+
}
|
|
8376
|
+
if (message.usageMetadata != null) {
|
|
8377
|
+
usageMetadata = message.usageMetadata;
|
|
8378
|
+
}
|
|
8379
|
+
if (message.error != null) {
|
|
8380
|
+
finishWithError(
|
|
8381
|
+
new Error((_a = message.error.message) != null ? _a : "Google Live API error")
|
|
8382
|
+
);
|
|
8383
|
+
return;
|
|
8384
|
+
}
|
|
8385
|
+
const serverContent = message.serverContent;
|
|
8386
|
+
const interim = serverContent == null ? void 0 : serverContent.interimInputTranscription;
|
|
8387
|
+
if (interim == null ? void 0 : interim.text) {
|
|
8388
|
+
schedulePendingFinish();
|
|
8389
|
+
latestInterim = interim.text;
|
|
8390
|
+
controller.enqueue({
|
|
8391
|
+
type: "transcript-partial",
|
|
8392
|
+
id: segmentId(),
|
|
8393
|
+
text: interim.text
|
|
8394
|
+
});
|
|
8395
|
+
}
|
|
8396
|
+
const transcription = (_b = serverContent == null ? void 0 : serverContent.inputTranscription) != null ? _b : message.inputTranscription;
|
|
8397
|
+
if (transcription != null) {
|
|
8398
|
+
if (transcription.languageCode != null) {
|
|
8399
|
+
language = transcription.languageCode;
|
|
8400
|
+
}
|
|
8401
|
+
if (transcription.text) {
|
|
8402
|
+
schedulePendingFinish();
|
|
8403
|
+
latestInterim = "";
|
|
8404
|
+
segmentBuffer += transcription.text;
|
|
8405
|
+
controller.enqueue({
|
|
8406
|
+
type: "transcript-delta",
|
|
8407
|
+
id: segmentId(),
|
|
8408
|
+
delta: transcription.text
|
|
8409
|
+
});
|
|
8410
|
+
}
|
|
8411
|
+
if (transcription.finished === true) {
|
|
8412
|
+
completeSegment();
|
|
8413
|
+
}
|
|
8414
|
+
}
|
|
8415
|
+
if (serverContent == null ? void 0 : serverContent.turnComplete) {
|
|
8416
|
+
completeSegment();
|
|
8417
|
+
}
|
|
8418
|
+
const interactionStatus = serverContent == null ? void 0 : serverContent.interactionStatus;
|
|
8419
|
+
if (audioEnded && (interactionStatus === "IDLE" || interactionStatus === "REQUIRES_ACTION" || (serverContent == null ? void 0 : serverContent.turnComplete) === true && interactionStatus == null)) {
|
|
8420
|
+
finish();
|
|
8421
|
+
}
|
|
8422
|
+
},
|
|
8423
|
+
onSocketError: () => {
|
|
8424
|
+
finishWithError(new Error("Google Live transcription error"));
|
|
8425
|
+
},
|
|
8426
|
+
onClose: ({ code, reason }) => {
|
|
8427
|
+
if (finished) return;
|
|
8428
|
+
if (audioEnded) {
|
|
8429
|
+
finish();
|
|
8430
|
+
return;
|
|
8431
|
+
}
|
|
8432
|
+
finishWithError(
|
|
8433
|
+
new Error(
|
|
8434
|
+
`Google Live transcription WebSocket closed unexpectedly before finishing (code ${code != null ? code : "unknown"}${reason ? `, reason: ${reason}` : ""}).`
|
|
8435
|
+
)
|
|
8436
|
+
);
|
|
8437
|
+
}
|
|
8438
|
+
});
|
|
8439
|
+
},
|
|
8440
|
+
cancel: () => {
|
|
8441
|
+
if (finished) return;
|
|
8442
|
+
finished = true;
|
|
8443
|
+
cleanup();
|
|
8444
|
+
}
|
|
8445
|
+
});
|
|
8446
|
+
}
|
|
8447
|
+
function buildAudioTranscriptionConfig(options) {
|
|
8448
|
+
if (options == null) return void 0;
|
|
8449
|
+
const config = {};
|
|
8450
|
+
if (options.languageCodes != null) {
|
|
8451
|
+
config.languageCodes = options.languageCodes;
|
|
8452
|
+
}
|
|
8453
|
+
if (options.customVocabulary != null) {
|
|
8454
|
+
config.customVocabulary = options.customVocabulary;
|
|
8455
|
+
}
|
|
8456
|
+
if (options.wordTimestamp != null) {
|
|
8457
|
+
config.wordTimestamp = options.wordTimestamp;
|
|
8458
|
+
}
|
|
8459
|
+
if (options.diarization != null) {
|
|
8460
|
+
config.diarization = options.diarization;
|
|
8461
|
+
}
|
|
8462
|
+
if (options.mode != null) {
|
|
8463
|
+
config.mode = options.mode;
|
|
8464
|
+
}
|
|
8465
|
+
return Object.keys(config).length > 0 ? config : void 0;
|
|
8466
|
+
}
|
|
8467
|
+
function buildTranscriptionConfig(options) {
|
|
8468
|
+
var _a;
|
|
8469
|
+
if (options == null) return void 0;
|
|
8470
|
+
const config = {};
|
|
8471
|
+
if (options.languageCodes != null) {
|
|
8472
|
+
config.language_codes = options.languageCodes;
|
|
8473
|
+
}
|
|
8474
|
+
if (options.customVocabulary != null) {
|
|
8475
|
+
config.custom_vocabulary = options.customVocabulary;
|
|
8476
|
+
}
|
|
8477
|
+
if (options.mode != null || options.diarization === true || options.wordTimestamp === true) {
|
|
8478
|
+
config.mode = {
|
|
8479
|
+
type: ((_a = options.mode) != null ? _a : "VERBATIM").toLowerCase(),
|
|
8480
|
+
...options.diarization === true ? { diarization_mode: "speaker" } : {},
|
|
8481
|
+
...options.wordTimestamp === true ? { timestamp_granularities: ["word"] } : {}
|
|
8482
|
+
};
|
|
8483
|
+
}
|
|
8484
|
+
return Object.keys(config).length > 0 ? config : void 0;
|
|
8485
|
+
}
|
|
8486
|
+
function parseOffsetSeconds(offset) {
|
|
8487
|
+
if (offset == null) return void 0;
|
|
8488
|
+
const parsed = Number.parseFloat(offset);
|
|
8489
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
8490
|
+
}
|
|
8491
|
+
function validateLiveInputAudioFormat(inputAudioFormat) {
|
|
8492
|
+
if (inputAudioFormat.type !== "audio/pcm" || inputAudioFormat.rate != null && inputAudioFormat.rate !== 16e3) {
|
|
8493
|
+
throw new InvalidArgumentError2({
|
|
8494
|
+
argument: "inputAudioFormat",
|
|
8495
|
+
message: "The Gemini Live transcription API only supports 16kHz 16-bit PCM input audio."
|
|
8496
|
+
});
|
|
8497
|
+
}
|
|
8498
|
+
}
|
|
8499
|
+
var googleInteractionsWordAnnotationSchema = z23.object({
|
|
8500
|
+
type: z23.string().nullish(),
|
|
8501
|
+
text: z23.string().nullish(),
|
|
8502
|
+
speaker: z23.string().nullish(),
|
|
8503
|
+
start_offset: z23.string().nullish(),
|
|
8504
|
+
end_offset: z23.string().nullish()
|
|
8505
|
+
});
|
|
8506
|
+
var googleInteractionsTranscriptionResponseSchema = z23.object({
|
|
8507
|
+
status: z23.string().nullish(),
|
|
8508
|
+
steps: z23.array(
|
|
8509
|
+
z23.object({
|
|
8510
|
+
type: z23.string().nullish(),
|
|
8511
|
+
content: z23.array(
|
|
8512
|
+
z23.object({
|
|
8513
|
+
type: z23.string().nullish(),
|
|
8514
|
+
text: z23.string().nullish(),
|
|
8515
|
+
annotations: z23.array(googleInteractionsWordAnnotationSchema).nullish()
|
|
8516
|
+
})
|
|
8517
|
+
).nullish()
|
|
8518
|
+
})
|
|
8519
|
+
).nullish(),
|
|
8520
|
+
usage: z23.record(z23.string(), z23.unknown()).nullish()
|
|
8521
|
+
});
|
|
8522
|
+
|
|
8523
|
+
// src/speech-translation/google-speech-translation-model.ts
|
|
8524
|
+
import {
|
|
8525
|
+
InvalidArgumentError as InvalidArgumentError3
|
|
8526
|
+
} from "@ai-sdk/provider";
|
|
8527
|
+
import {
|
|
8528
|
+
connectToWebSocket as connectToWebSocket2,
|
|
8529
|
+
combineHeaders as combineHeaders10,
|
|
8530
|
+
convertBase64ToUint8Array as convertBase64ToUint8Array2,
|
|
8531
|
+
convertToBase64 as convertToBase645,
|
|
8532
|
+
parseProviderOptions as parseProviderOptions9,
|
|
8533
|
+
safeParseJSON as safeParseJSON3,
|
|
8534
|
+
serializeModelOptions as serializeModelOptions7,
|
|
8535
|
+
WORKFLOW_DESERIALIZE as WORKFLOW_DESERIALIZE8,
|
|
8536
|
+
WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE8,
|
|
8537
|
+
waitForWebSocketBufferDrain as waitForWebSocketBufferDrain2
|
|
8021
8538
|
} from "@ai-sdk/provider-utils";
|
|
8022
8539
|
|
|
8023
8540
|
// src/speech-translation/google-speech-translation-model-options.ts
|
|
@@ -8025,26 +8542,26 @@ import {
|
|
|
8025
8542
|
lazySchema as lazySchema20,
|
|
8026
8543
|
zodSchema as zodSchema20
|
|
8027
8544
|
} from "@ai-sdk/provider-utils";
|
|
8028
|
-
import { z as
|
|
8545
|
+
import { z as z24 } from "zod/v4";
|
|
8029
8546
|
var googleSpeechTranslationModelOptions = lazySchema20(
|
|
8030
8547
|
() => zodSchema20(
|
|
8031
|
-
|
|
8548
|
+
z24.object({
|
|
8032
8549
|
/**
|
|
8033
8550
|
* Whether input audio already in the target language should be echoed
|
|
8034
8551
|
* instead of producing silence.
|
|
8035
8552
|
*/
|
|
8036
|
-
echoTargetLanguage:
|
|
8553
|
+
echoTargetLanguage: z24.boolean().optional()
|
|
8037
8554
|
})
|
|
8038
8555
|
)
|
|
8039
8556
|
);
|
|
8040
8557
|
|
|
8041
8558
|
// src/speech-translation/google-speech-translation-model.ts
|
|
8042
|
-
var
|
|
8043
|
-
var
|
|
8559
|
+
var liveWebSocketPath2 = "google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
8560
|
+
var defaultFinishGraceMs2 = 1e3;
|
|
8044
8561
|
var googleLiveOutputAudioRate = 24e3;
|
|
8045
8562
|
var pcm16SilenceAmplitudeThreshold = 128;
|
|
8046
|
-
function
|
|
8047
|
-
const url = getRealtimeWebSocketURL(baseURL,
|
|
8563
|
+
function getLiveWebSocketURL2(baseURL, apiKey) {
|
|
8564
|
+
const url = getRealtimeWebSocketURL(baseURL, liveWebSocketPath2);
|
|
8048
8565
|
url.searchParams.set("key", apiKey);
|
|
8049
8566
|
return url;
|
|
8050
8567
|
}
|
|
@@ -8054,13 +8571,13 @@ var GoogleSpeechTranslationModel = class _GoogleSpeechTranslationModel {
|
|
|
8054
8571
|
this.modelId = modelId;
|
|
8055
8572
|
this.config = config;
|
|
8056
8573
|
}
|
|
8057
|
-
static [
|
|
8058
|
-
return
|
|
8574
|
+
static [WORKFLOW_SERIALIZE8](model) {
|
|
8575
|
+
return serializeModelOptions7({
|
|
8059
8576
|
modelId: model.modelId,
|
|
8060
8577
|
config: model.config
|
|
8061
8578
|
});
|
|
8062
8579
|
}
|
|
8063
|
-
static [
|
|
8580
|
+
static [WORKFLOW_DESERIALIZE8](options) {
|
|
8064
8581
|
return new _GoogleSpeechTranslationModel(options.modelId, options.config);
|
|
8065
8582
|
}
|
|
8066
8583
|
get provider() {
|
|
@@ -8069,13 +8586,13 @@ var GoogleSpeechTranslationModel = class _GoogleSpeechTranslationModel {
|
|
|
8069
8586
|
async doStream(options) {
|
|
8070
8587
|
var _a, _b, _c, _d, _e, _f;
|
|
8071
8588
|
if (options.targetLanguage == null) {
|
|
8072
|
-
throw new
|
|
8589
|
+
throw new InvalidArgumentError3({
|
|
8073
8590
|
argument: "targetLanguage",
|
|
8074
8591
|
message: `targetLanguage is required for translation model '${this.modelId}'.`
|
|
8075
8592
|
});
|
|
8076
8593
|
}
|
|
8077
8594
|
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
8078
|
-
const googleOptions = await
|
|
8595
|
+
const googleOptions = await parseProviderOptions9({
|
|
8079
8596
|
provider: "google",
|
|
8080
8597
|
providerOptions: options.providerOptions,
|
|
8081
8598
|
schema: googleSpeechTranslationModelOptions
|
|
@@ -8096,7 +8613,7 @@ var GoogleSpeechTranslationModel = class _GoogleSpeechTranslationModel {
|
|
|
8096
8613
|
details: "The Gemini Live API always outputs 24kHz 16-bit PCM audio and does not accept an output audio format."
|
|
8097
8614
|
});
|
|
8098
8615
|
}
|
|
8099
|
-
const headers =
|
|
8616
|
+
const headers = combineHeaders10(this.config.headers(), options.headers);
|
|
8100
8617
|
let apiKey;
|
|
8101
8618
|
for (const [key, value] of Object.entries(headers)) {
|
|
8102
8619
|
if (key.toLowerCase() === "x-goog-api-key" && value != null) {
|
|
@@ -8126,11 +8643,11 @@ var GoogleSpeechTranslationModel = class _GoogleSpeechTranslationModel {
|
|
|
8126
8643
|
},
|
|
8127
8644
|
stream: createGoogleLiveSpeechTranslationStream({
|
|
8128
8645
|
webSocket: this.config.webSocket,
|
|
8129
|
-
url:
|
|
8646
|
+
url: getLiveWebSocketURL2(this.config.baseURL, apiKey),
|
|
8130
8647
|
headers: webSocketHeaders,
|
|
8131
8648
|
setup,
|
|
8132
8649
|
inputAudioRate: (_d = options.inputAudioFormat.rate) != null ? _d : 16e3,
|
|
8133
|
-
finishGraceMs: (_f = (_e = this.config._internal) == null ? void 0 : _e.finishGraceMs) != null ? _f :
|
|
8650
|
+
finishGraceMs: (_f = (_e = this.config._internal) == null ? void 0 : _e.finishGraceMs) != null ? _f : defaultFinishGraceMs2,
|
|
8134
8651
|
warnings,
|
|
8135
8652
|
audio: options.audio,
|
|
8136
8653
|
abortSignal: options.abortSignal,
|
|
@@ -8159,8 +8676,8 @@ function createGoogleLiveSpeechTranslationStream({
|
|
|
8159
8676
|
let audioReader;
|
|
8160
8677
|
let connection;
|
|
8161
8678
|
let resolveSetupComplete;
|
|
8162
|
-
const setupComplete = new Promise((
|
|
8163
|
-
resolveSetupComplete =
|
|
8679
|
+
const setupComplete = new Promise((resolve8) => {
|
|
8680
|
+
resolveSetupComplete = resolve8;
|
|
8164
8681
|
});
|
|
8165
8682
|
let turnCounter = 0;
|
|
8166
8683
|
let sourceText = "";
|
|
@@ -8255,13 +8772,13 @@ function createGoogleLiveSpeechTranslationStream({
|
|
|
8255
8772
|
JSON.stringify({
|
|
8256
8773
|
realtimeInput: {
|
|
8257
8774
|
audio: {
|
|
8258
|
-
data:
|
|
8775
|
+
data: convertToBase645(value),
|
|
8259
8776
|
mimeType: `audio/pcm;rate=${inputAudioRate}`
|
|
8260
8777
|
}
|
|
8261
8778
|
}
|
|
8262
8779
|
})
|
|
8263
8780
|
);
|
|
8264
|
-
await
|
|
8781
|
+
await waitForWebSocketBufferDrain2(socket);
|
|
8265
8782
|
}
|
|
8266
8783
|
} finally {
|
|
8267
8784
|
audioReader.releaseLock();
|
|
@@ -8277,7 +8794,7 @@ function createGoogleLiveSpeechTranslationStream({
|
|
|
8277
8794
|
}
|
|
8278
8795
|
}
|
|
8279
8796
|
};
|
|
8280
|
-
connection =
|
|
8797
|
+
connection = connectToWebSocket2({
|
|
8281
8798
|
url,
|
|
8282
8799
|
headers,
|
|
8283
8800
|
webSocket,
|
|
@@ -8292,7 +8809,7 @@ function createGoogleLiveSpeechTranslationStream({
|
|
|
8292
8809
|
onMessageText: async (text) => {
|
|
8293
8810
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
8294
8811
|
if (finished) return;
|
|
8295
|
-
const parsed = await
|
|
8812
|
+
const parsed = await safeParseJSON3({ text });
|
|
8296
8813
|
if (!parsed.success) return;
|
|
8297
8814
|
const message = parsed.value;
|
|
8298
8815
|
if (includeRawChunks) {
|
|
@@ -8449,7 +8966,7 @@ function buildGoogleLiveSpeechTranslationSetup({
|
|
|
8449
8966
|
}
|
|
8450
8967
|
function validateGoogleSpeechTranslationInputAudioFormat(inputAudioFormat) {
|
|
8451
8968
|
if (inputAudioFormat.type !== "audio/pcm" || inputAudioFormat.rate != null && inputAudioFormat.rate !== 16e3) {
|
|
8452
|
-
throw new
|
|
8969
|
+
throw new InvalidArgumentError3({
|
|
8453
8970
|
argument: "inputAudioFormat",
|
|
8454
8971
|
message: "The Gemini Live translation API only supports 16kHz 16-bit PCM input audio."
|
|
8455
8972
|
});
|
|
@@ -8574,6 +9091,13 @@ function createGoogle(options = {}) {
|
|
|
8574
9091
|
headers: getHeaders,
|
|
8575
9092
|
fetch: options.fetch
|
|
8576
9093
|
});
|
|
9094
|
+
const createTranscriptionModel = (modelId) => new GoogleTranscriptionModel(modelId, {
|
|
9095
|
+
provider: `${providerName}.transcription`,
|
|
9096
|
+
baseURL,
|
|
9097
|
+
headers: getHeaders,
|
|
9098
|
+
fetch: options.fetch,
|
|
9099
|
+
webSocket: options.webSocket
|
|
9100
|
+
});
|
|
8577
9101
|
const experimentalRealtimeFactory = Object.assign(
|
|
8578
9102
|
(modelId) => createRealtimeModel(modelId),
|
|
8579
9103
|
{
|
|
@@ -8628,6 +9152,8 @@ function createGoogle(options = {}) {
|
|
|
8628
9152
|
provider.files = createFiles;
|
|
8629
9153
|
provider.speech = createSpeechModel;
|
|
8630
9154
|
provider.speechModel = createSpeechModel;
|
|
9155
|
+
provider.transcription = createTranscriptionModel;
|
|
9156
|
+
provider.transcriptionModel = createTranscriptionModel;
|
|
8631
9157
|
provider.translation = createSpeechTranslationModel;
|
|
8632
9158
|
provider.speechTranslationModel = createSpeechTranslationModel;
|
|
8633
9159
|
provider.interactions = createInteractionsModel;
|
|
@@ -8639,6 +9165,7 @@ export {
|
|
|
8639
9165
|
GoogleRealtimeModel as Experimental_GoogleRealtimeModel,
|
|
8640
9166
|
GoogleSpeechTranslationModel as Experimental_GoogleSpeechTranslationModel,
|
|
8641
9167
|
GoogleSpeechTranslationModel as Experimental_GoogleTranslationModel,
|
|
9168
|
+
GoogleTranscriptionModel,
|
|
8642
9169
|
VERSION,
|
|
8643
9170
|
createGoogle,
|
|
8644
9171
|
createGoogle as createGoogleGenerativeAI,
|