@ai-sdk/google 4.0.25 → 4.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.d.ts +50 -3
- package/dist/index.js +516 -30
- package/dist/index.js.map +1 -1
- package/dist/internal/index.js +34 -14
- package/dist/internal/index.js.map +1 -1
- package/docs/15-google.mdx +51 -0
- package/package.json +2 -2
- package/src/get-realtime-base-url.ts +31 -0
- package/src/google-files.ts +0 -1
- package/src/google-provider.ts +32 -0
- package/src/index.ts +6 -0
- package/src/interactions/google-interactions-language-model.ts +21 -0
- package/src/interactions/google-interactions-prompt.ts +1 -0
- package/src/realtime/google-realtime-model.ts +5 -17
- package/src/translation/google-translation-model-options.ts +26 -0
- package/src/translation/google-translation-model.ts +612 -0
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "@ai-sdk/provider-utils";
|
|
8
8
|
|
|
9
9
|
// src/version.ts
|
|
10
|
-
var VERSION = true ? "4.0.
|
|
10
|
+
var VERSION = true ? "4.0.27" : "0.0.0-test";
|
|
11
11
|
|
|
12
12
|
// src/google-embedding-model.ts
|
|
13
13
|
import {
|
|
@@ -3330,7 +3330,6 @@ var GoogleFiles = class {
|
|
|
3330
3330
|
const uploadResponse = await fetchFn(uploadUrl, {
|
|
3331
3331
|
method: "POST",
|
|
3332
3332
|
headers: {
|
|
3333
|
-
"Content-Length": String(fileBytes.length),
|
|
3334
3333
|
"X-Goog-Upload-Offset": "0",
|
|
3335
3334
|
"X-Goog-Upload-Command": "upload, finalize"
|
|
3336
3335
|
},
|
|
@@ -6488,7 +6487,7 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6488
6487
|
};
|
|
6489
6488
|
}
|
|
6490
6489
|
async getArgs(options) {
|
|
6491
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z;
|
|
6490
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
|
|
6492
6491
|
const warnings = [];
|
|
6493
6492
|
const googleOptions = await parseProviderOptions7({
|
|
6494
6493
|
provider: "google",
|
|
@@ -6496,6 +6495,20 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6496
6495
|
schema: googleInteractionsLanguageModelOptions
|
|
6497
6496
|
});
|
|
6498
6497
|
const isAgent = this.agent != null;
|
|
6498
|
+
if (!isAgent) {
|
|
6499
|
+
if (options.frequencyPenalty != null) {
|
|
6500
|
+
warnings.push({
|
|
6501
|
+
type: "unsupported",
|
|
6502
|
+
feature: "frequencyPenalty"
|
|
6503
|
+
});
|
|
6504
|
+
}
|
|
6505
|
+
if (options.presencePenalty != null) {
|
|
6506
|
+
warnings.push({
|
|
6507
|
+
type: "unsupported",
|
|
6508
|
+
feature: "presencePenalty"
|
|
6509
|
+
});
|
|
6510
|
+
}
|
|
6511
|
+
}
|
|
6499
6512
|
const hasTools = options.tools != null && options.tools.length > 0;
|
|
6500
6513
|
let toolsForBody;
|
|
6501
6514
|
let toolChoiceForBody;
|
|
@@ -6579,6 +6592,11 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6579
6592
|
const droppedFields = [];
|
|
6580
6593
|
if (options.temperature != null) droppedFields.push("temperature");
|
|
6581
6594
|
if (options.topP != null) droppedFields.push("topP");
|
|
6595
|
+
if (options.topK != null) droppedFields.push("topK");
|
|
6596
|
+
if (options.frequencyPenalty != null)
|
|
6597
|
+
droppedFields.push("frequencyPenalty");
|
|
6598
|
+
if (options.presencePenalty != null)
|
|
6599
|
+
droppedFields.push("presencePenalty");
|
|
6582
6600
|
if (options.seed != null) droppedFields.push("seed");
|
|
6583
6601
|
if (options.stopSequences != null && options.stopSequences.length > 0) {
|
|
6584
6602
|
droppedFields.push("stopSequences");
|
|
@@ -6602,11 +6620,12 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6602
6620
|
generationConfig = pruneUndefined({
|
|
6603
6621
|
temperature: (_l = options.temperature) != null ? _l : void 0,
|
|
6604
6622
|
top_p: (_m = options.topP) != null ? _m : void 0,
|
|
6605
|
-
|
|
6623
|
+
top_k: (_n = options.topK) != null ? _n : void 0,
|
|
6624
|
+
seed: (_o = options.seed) != null ? _o : void 0,
|
|
6606
6625
|
stop_sequences: options.stopSequences != null && options.stopSequences.length > 0 ? options.stopSequences : void 0,
|
|
6607
|
-
max_output_tokens: (
|
|
6608
|
-
thinking_level: (
|
|
6609
|
-
thinking_summaries: (
|
|
6626
|
+
max_output_tokens: (_p = options.maxOutputTokens) != null ? _p : void 0,
|
|
6627
|
+
thinking_level: (_q = googleOptions == null ? void 0 : googleOptions.thinkingLevel) != null ? _q : void 0,
|
|
6628
|
+
thinking_summaries: (_r = googleOptions == null ? void 0 : googleOptions.thinkingSummaries) != null ? _r : void 0,
|
|
6610
6629
|
tool_choice: toolChoiceForBody
|
|
6611
6630
|
});
|
|
6612
6631
|
if ((googleOptions == null ? void 0 : googleOptions.imageConfig) != null) {
|
|
@@ -6633,9 +6652,9 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6633
6652
|
if (agentConfigOptions.type === "deep-research") {
|
|
6634
6653
|
agentConfig = pruneUndefined({
|
|
6635
6654
|
type: "deep-research",
|
|
6636
|
-
thinking_summaries: (
|
|
6637
|
-
visualization: (
|
|
6638
|
-
collaborative_planning: (
|
|
6655
|
+
thinking_summaries: (_s = agentConfigOptions.thinkingSummaries) != null ? _s : void 0,
|
|
6656
|
+
visualization: (_t = agentConfigOptions.visualization) != null ? _t : void 0,
|
|
6657
|
+
collaborative_planning: (_u = agentConfigOptions.collaborativePlanning) != null ? _u : void 0
|
|
6639
6658
|
});
|
|
6640
6659
|
} else if (agentConfigOptions.type === "dynamic") {
|
|
6641
6660
|
agentConfig = { type: "dynamic" };
|
|
@@ -6652,7 +6671,7 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6652
6671
|
environment = googleOptions.environment;
|
|
6653
6672
|
} else {
|
|
6654
6673
|
const environmentOptions = googleOptions.environment;
|
|
6655
|
-
const sources = (
|
|
6674
|
+
const sources = (_v = environmentOptions.sources) == null ? void 0 : _v.map((source) => {
|
|
6656
6675
|
var _a2;
|
|
6657
6676
|
if (source.type === "inline") {
|
|
6658
6677
|
return {
|
|
@@ -6697,20 +6716,20 @@ var GoogleInteractionsLanguageModel = class _GoogleInteractionsLanguageModel {
|
|
|
6697
6716
|
tools: toolsForBody,
|
|
6698
6717
|
response_format: responseFormatEntries.length > 0 ? responseFormatEntries : void 0,
|
|
6699
6718
|
response_modalities: (googleOptions == null ? void 0 : googleOptions.responseModalities) != null ? googleOptions.responseModalities : void 0,
|
|
6700
|
-
previous_interaction_id: (
|
|
6701
|
-
service_tier: (
|
|
6702
|
-
store: (
|
|
6719
|
+
previous_interaction_id: (_w = googleOptions == null ? void 0 : googleOptions.previousInteractionId) != null ? _w : void 0,
|
|
6720
|
+
service_tier: (_x = googleOptions == null ? void 0 : googleOptions.serviceTier) != null ? _x : void 0,
|
|
6721
|
+
store: (_y = googleOptions == null ? void 0 : googleOptions.store) != null ? _y : void 0,
|
|
6703
6722
|
generation_config: generationConfig != null && Object.keys(generationConfig).length > 0 ? generationConfig : void 0,
|
|
6704
6723
|
agent_config: agentConfig,
|
|
6705
6724
|
environment,
|
|
6706
|
-
background: (
|
|
6725
|
+
background: (_z = googleOptions == null ? void 0 : googleOptions.background) != null ? _z : void 0
|
|
6707
6726
|
});
|
|
6708
6727
|
return {
|
|
6709
6728
|
args,
|
|
6710
6729
|
warnings,
|
|
6711
6730
|
isAgent,
|
|
6712
6731
|
isBackground: (googleOptions == null ? void 0 : googleOptions.background) === true,
|
|
6713
|
-
pollingTimeoutMs: (
|
|
6732
|
+
pollingTimeoutMs: (_A = googleOptions == null ? void 0 : googleOptions.pollingTimeoutMs) != null ? _A : void 0
|
|
6714
6733
|
};
|
|
6715
6734
|
}
|
|
6716
6735
|
async doGenerate(options) {
|
|
@@ -6930,6 +6949,24 @@ function pruneUndefined(obj) {
|
|
|
6930
6949
|
return result;
|
|
6931
6950
|
}
|
|
6932
6951
|
|
|
6952
|
+
// src/get-realtime-base-url.ts
|
|
6953
|
+
function getRealtimeBaseURL(baseURL) {
|
|
6954
|
+
const url = new URL(baseURL);
|
|
6955
|
+
const pathSegments = url.pathname.split("/");
|
|
6956
|
+
const version = pathSegments.at(-1);
|
|
6957
|
+
if (version === "v1beta" || version === "v1alpha") {
|
|
6958
|
+
pathSegments.pop();
|
|
6959
|
+
url.pathname = pathSegments.join("/") || "/";
|
|
6960
|
+
}
|
|
6961
|
+
return url;
|
|
6962
|
+
}
|
|
6963
|
+
function getRealtimeWebSocketURL(baseURL, webSocketPath) {
|
|
6964
|
+
const url = getRealtimeBaseURL(baseURL);
|
|
6965
|
+
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
|
6966
|
+
url.pathname = `${url.pathname.replace(/\/$/, "")}/ws/${webSocketPath}`;
|
|
6967
|
+
return url;
|
|
6968
|
+
}
|
|
6969
|
+
|
|
6933
6970
|
// src/realtime/google-realtime-event-mapper.ts
|
|
6934
6971
|
import { safeParseJSON } from "@ai-sdk/provider-utils";
|
|
6935
6972
|
function isRecord(value) {
|
|
@@ -7257,26 +7294,13 @@ function buildGoogleSessionConfig(config, modelId) {
|
|
|
7257
7294
|
|
|
7258
7295
|
// src/realtime/google-realtime-model.ts
|
|
7259
7296
|
var realtimeWebSocketPath = "google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
|
|
7260
|
-
function getRealtimeBaseURL(baseURL) {
|
|
7261
|
-
const url = new URL(baseURL);
|
|
7262
|
-
const pathSegments = url.pathname.split("/");
|
|
7263
|
-
const version = pathSegments.at(-1);
|
|
7264
|
-
if (version === "v1beta" || version === "v1alpha") {
|
|
7265
|
-
pathSegments.pop();
|
|
7266
|
-
url.pathname = pathSegments.join("/") || "/";
|
|
7267
|
-
}
|
|
7268
|
-
return url;
|
|
7269
|
-
}
|
|
7270
7297
|
function getAuthTokensURL(baseURL) {
|
|
7271
7298
|
const url = getRealtimeBaseURL(baseURL);
|
|
7272
7299
|
url.pathname = `${url.pathname.replace(/\/$/, "")}/v1alpha/auth_tokens`;
|
|
7273
7300
|
return url.toString();
|
|
7274
7301
|
}
|
|
7275
7302
|
function getWebSocketURL(baseURL) {
|
|
7276
|
-
|
|
7277
|
-
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
|
7278
|
-
url.pathname = `${url.pathname.replace(/\/$/, "")}/ws/${realtimeWebSocketPath}`;
|
|
7279
|
-
return url.toString();
|
|
7303
|
+
return getRealtimeWebSocketURL(baseURL, realtimeWebSocketPath).toString();
|
|
7280
7304
|
}
|
|
7281
7305
|
var GoogleRealtimeModel = class {
|
|
7282
7306
|
constructor(modelId, config) {
|
|
@@ -7351,6 +7375,459 @@ var GoogleRealtimeModel = class {
|
|
|
7351
7375
|
}
|
|
7352
7376
|
};
|
|
7353
7377
|
|
|
7378
|
+
// src/translation/google-translation-model.ts
|
|
7379
|
+
import {
|
|
7380
|
+
InvalidArgumentError
|
|
7381
|
+
} from "@ai-sdk/provider";
|
|
7382
|
+
import {
|
|
7383
|
+
connectToWebSocket,
|
|
7384
|
+
combineHeaders as combineHeaders9,
|
|
7385
|
+
convertBase64ToUint8Array as convertBase64ToUint8Array2,
|
|
7386
|
+
convertToBase64 as convertToBase644,
|
|
7387
|
+
parseProviderOptions as parseProviderOptions8,
|
|
7388
|
+
safeParseJSON as safeParseJSON2,
|
|
7389
|
+
serializeModelOptions as serializeModelOptions6,
|
|
7390
|
+
WORKFLOW_DESERIALIZE as WORKFLOW_DESERIALIZE6,
|
|
7391
|
+
WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE6,
|
|
7392
|
+
waitForWebSocketBufferDrain
|
|
7393
|
+
} from "@ai-sdk/provider-utils";
|
|
7394
|
+
|
|
7395
|
+
// src/translation/google-translation-model-options.ts
|
|
7396
|
+
import {
|
|
7397
|
+
lazySchema as lazySchema20,
|
|
7398
|
+
zodSchema as zodSchema20
|
|
7399
|
+
} from "@ai-sdk/provider-utils";
|
|
7400
|
+
import { z as z22 } from "zod/v4";
|
|
7401
|
+
var googleTranslationModelOptions = lazySchema20(
|
|
7402
|
+
() => zodSchema20(
|
|
7403
|
+
z22.object({
|
|
7404
|
+
/**
|
|
7405
|
+
* Whether input audio already in the target language should be echoed
|
|
7406
|
+
* instead of producing silence.
|
|
7407
|
+
*/
|
|
7408
|
+
echoTargetLanguage: z22.boolean().optional()
|
|
7409
|
+
})
|
|
7410
|
+
)
|
|
7411
|
+
);
|
|
7412
|
+
|
|
7413
|
+
// src/translation/google-translation-model.ts
|
|
7414
|
+
var liveWebSocketPath = "google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
7415
|
+
var defaultFinishGraceMs = 1e3;
|
|
7416
|
+
var googleLiveOutputAudioRate = 24e3;
|
|
7417
|
+
var pcm16SilenceAmplitudeThreshold = 128;
|
|
7418
|
+
function getLiveWebSocketURL(baseURL, apiKey) {
|
|
7419
|
+
const url = getRealtimeWebSocketURL(baseURL, liveWebSocketPath);
|
|
7420
|
+
url.searchParams.set("key", apiKey);
|
|
7421
|
+
return url;
|
|
7422
|
+
}
|
|
7423
|
+
var GoogleTranslationModel = class _GoogleTranslationModel {
|
|
7424
|
+
constructor(modelId, config) {
|
|
7425
|
+
this.specificationVersion = "v4";
|
|
7426
|
+
this.modelId = modelId;
|
|
7427
|
+
this.config = config;
|
|
7428
|
+
}
|
|
7429
|
+
static [WORKFLOW_SERIALIZE6](model) {
|
|
7430
|
+
return serializeModelOptions6({
|
|
7431
|
+
modelId: model.modelId,
|
|
7432
|
+
config: model.config
|
|
7433
|
+
});
|
|
7434
|
+
}
|
|
7435
|
+
static [WORKFLOW_DESERIALIZE6](options) {
|
|
7436
|
+
return new _GoogleTranslationModel(options.modelId, options.config);
|
|
7437
|
+
}
|
|
7438
|
+
get provider() {
|
|
7439
|
+
return this.config.provider;
|
|
7440
|
+
}
|
|
7441
|
+
async doStream(options) {
|
|
7442
|
+
var _a, _b, _c, _d, _e, _f;
|
|
7443
|
+
if (options.targetLanguage == null) {
|
|
7444
|
+
throw new InvalidArgumentError({
|
|
7445
|
+
argument: "targetLanguage",
|
|
7446
|
+
message: `targetLanguage is required for translation model '${this.modelId}'.`
|
|
7447
|
+
});
|
|
7448
|
+
}
|
|
7449
|
+
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
7450
|
+
const googleOptions = await parseProviderOptions8({
|
|
7451
|
+
provider: "google",
|
|
7452
|
+
providerOptions: options.providerOptions,
|
|
7453
|
+
schema: googleTranslationModelOptions
|
|
7454
|
+
});
|
|
7455
|
+
const warnings = [];
|
|
7456
|
+
validateGoogleTranslationInputAudioFormat(options.inputAudioFormat);
|
|
7457
|
+
if (options.sourceLanguage != null) {
|
|
7458
|
+
warnings.push({
|
|
7459
|
+
type: "unsupported",
|
|
7460
|
+
feature: "sourceLanguage",
|
|
7461
|
+
details: "The Gemini Live translation API auto-detects the source language and does not accept a source language."
|
|
7462
|
+
});
|
|
7463
|
+
}
|
|
7464
|
+
if (options.outputAudioFormat != null) {
|
|
7465
|
+
warnings.push({
|
|
7466
|
+
type: "unsupported",
|
|
7467
|
+
feature: "outputAudioFormat",
|
|
7468
|
+
details: "The Gemini Live API always outputs 24kHz 16-bit PCM audio and does not accept an output audio format."
|
|
7469
|
+
});
|
|
7470
|
+
}
|
|
7471
|
+
const headers = combineHeaders9(this.config.headers(), options.headers);
|
|
7472
|
+
let apiKey;
|
|
7473
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
7474
|
+
if (key.toLowerCase() === "x-goog-api-key" && value != null) {
|
|
7475
|
+
apiKey = value;
|
|
7476
|
+
}
|
|
7477
|
+
}
|
|
7478
|
+
if (apiKey == null) {
|
|
7479
|
+
throw new Error(
|
|
7480
|
+
"Google Generative AI API key is required for streaming translation."
|
|
7481
|
+
);
|
|
7482
|
+
}
|
|
7483
|
+
const webSocketHeaders = Object.fromEntries(
|
|
7484
|
+
Object.entries(headers).filter(
|
|
7485
|
+
([key]) => key.toLowerCase() !== "x-goog-api-key"
|
|
7486
|
+
)
|
|
7487
|
+
);
|
|
7488
|
+
const setup = buildGoogleLiveTranslationSetup({
|
|
7489
|
+
modelId: this.modelId,
|
|
7490
|
+
targetLanguage: options.targetLanguage,
|
|
7491
|
+
providerOptions: googleOptions
|
|
7492
|
+
});
|
|
7493
|
+
return {
|
|
7494
|
+
request: { body: setup },
|
|
7495
|
+
response: {
|
|
7496
|
+
timestamp: currentDate,
|
|
7497
|
+
modelId: this.modelId
|
|
7498
|
+
},
|
|
7499
|
+
stream: createGoogleLiveTranslationStream({
|
|
7500
|
+
webSocket: this.config.webSocket,
|
|
7501
|
+
url: getLiveWebSocketURL(this.config.baseURL, apiKey),
|
|
7502
|
+
headers: webSocketHeaders,
|
|
7503
|
+
setup,
|
|
7504
|
+
inputAudioRate: (_d = options.inputAudioFormat.rate) != null ? _d : 16e3,
|
|
7505
|
+
finishGraceMs: (_f = (_e = this.config._internal) == null ? void 0 : _e.finishGraceMs) != null ? _f : defaultFinishGraceMs,
|
|
7506
|
+
warnings,
|
|
7507
|
+
audio: options.audio,
|
|
7508
|
+
abortSignal: options.abortSignal,
|
|
7509
|
+
includeRawChunks: options.includeRawChunks
|
|
7510
|
+
})
|
|
7511
|
+
};
|
|
7512
|
+
}
|
|
7513
|
+
};
|
|
7514
|
+
function createGoogleLiveTranslationStream({
|
|
7515
|
+
webSocket,
|
|
7516
|
+
url,
|
|
7517
|
+
headers,
|
|
7518
|
+
setup,
|
|
7519
|
+
inputAudioRate,
|
|
7520
|
+
finishGraceMs,
|
|
7521
|
+
warnings,
|
|
7522
|
+
audio,
|
|
7523
|
+
abortSignal,
|
|
7524
|
+
includeRawChunks
|
|
7525
|
+
}) {
|
|
7526
|
+
let finished = false;
|
|
7527
|
+
let cleanup = () => {
|
|
7528
|
+
};
|
|
7529
|
+
return new ReadableStream({
|
|
7530
|
+
start: (controller) => {
|
|
7531
|
+
let audioReader;
|
|
7532
|
+
let connection;
|
|
7533
|
+
let resolveSetupComplete;
|
|
7534
|
+
const setupComplete = new Promise((resolve7) => {
|
|
7535
|
+
resolveSetupComplete = resolve7;
|
|
7536
|
+
});
|
|
7537
|
+
let turnCounter = 0;
|
|
7538
|
+
let sourceText = "";
|
|
7539
|
+
let sourceTurnBuffer = "";
|
|
7540
|
+
let translationText = "";
|
|
7541
|
+
let translationTurnBuffer = "";
|
|
7542
|
+
let audioEnded = false;
|
|
7543
|
+
let usage;
|
|
7544
|
+
let openTurn = false;
|
|
7545
|
+
let sawTurnComplete = false;
|
|
7546
|
+
let trailingSilenceMs = 0;
|
|
7547
|
+
let finishTimer;
|
|
7548
|
+
const itemId = () => `google-item-${turnCounter}`;
|
|
7549
|
+
const cancelPendingFinish = () => {
|
|
7550
|
+
if (finishTimer != null) {
|
|
7551
|
+
clearTimeout(finishTimer);
|
|
7552
|
+
finishTimer = void 0;
|
|
7553
|
+
}
|
|
7554
|
+
};
|
|
7555
|
+
const schedulePendingFinish = () => {
|
|
7556
|
+
if (finished || finishTimer != null) return;
|
|
7557
|
+
finishTimer = setTimeout(() => {
|
|
7558
|
+
finishTimer = void 0;
|
|
7559
|
+
finish();
|
|
7560
|
+
}, finishGraceMs);
|
|
7561
|
+
};
|
|
7562
|
+
const onTurnActivity = () => {
|
|
7563
|
+
openTurn = true;
|
|
7564
|
+
trailingSilenceMs = 0;
|
|
7565
|
+
cancelPendingFinish();
|
|
7566
|
+
};
|
|
7567
|
+
cleanup = (closeCode) => {
|
|
7568
|
+
cancelPendingFinish();
|
|
7569
|
+
if (audioReader != null) {
|
|
7570
|
+
void audioReader.cancel().catch(() => {
|
|
7571
|
+
});
|
|
7572
|
+
} else {
|
|
7573
|
+
void audio.cancel().catch(() => {
|
|
7574
|
+
});
|
|
7575
|
+
}
|
|
7576
|
+
connection == null ? void 0 : connection.close(closeCode);
|
|
7577
|
+
};
|
|
7578
|
+
const finishWithError = (error) => {
|
|
7579
|
+
if (finished) return;
|
|
7580
|
+
finished = true;
|
|
7581
|
+
cleanup();
|
|
7582
|
+
controller.error(error);
|
|
7583
|
+
};
|
|
7584
|
+
const finish = () => {
|
|
7585
|
+
if (finished) return;
|
|
7586
|
+
if (sourceTurnBuffer !== "" || translationTurnBuffer !== "") {
|
|
7587
|
+
completeTurn();
|
|
7588
|
+
}
|
|
7589
|
+
finished = true;
|
|
7590
|
+
controller.enqueue({
|
|
7591
|
+
type: "finish",
|
|
7592
|
+
sourceText,
|
|
7593
|
+
outputText: translationText,
|
|
7594
|
+
usage
|
|
7595
|
+
});
|
|
7596
|
+
controller.close();
|
|
7597
|
+
cleanup(1e3);
|
|
7598
|
+
};
|
|
7599
|
+
const completeTurn = () => {
|
|
7600
|
+
if (sourceTurnBuffer !== "") {
|
|
7601
|
+
controller.enqueue({
|
|
7602
|
+
type: "source-transcript-final",
|
|
7603
|
+
id: itemId(),
|
|
7604
|
+
text: sourceTurnBuffer
|
|
7605
|
+
});
|
|
7606
|
+
sourceText += sourceTurnBuffer;
|
|
7607
|
+
sourceTurnBuffer = "";
|
|
7608
|
+
}
|
|
7609
|
+
if (translationTurnBuffer !== "") {
|
|
7610
|
+
controller.enqueue({
|
|
7611
|
+
type: "output-text-final",
|
|
7612
|
+
id: itemId(),
|
|
7613
|
+
text: translationTurnBuffer
|
|
7614
|
+
});
|
|
7615
|
+
translationText += translationTurnBuffer;
|
|
7616
|
+
translationTurnBuffer = "";
|
|
7617
|
+
}
|
|
7618
|
+
turnCounter++;
|
|
7619
|
+
};
|
|
7620
|
+
const sendAudio = async (socket) => {
|
|
7621
|
+
audioReader = audio.getReader();
|
|
7622
|
+
try {
|
|
7623
|
+
while (true) {
|
|
7624
|
+
const { done, value } = await audioReader.read();
|
|
7625
|
+
if (done || finished) break;
|
|
7626
|
+
socket.send(
|
|
7627
|
+
JSON.stringify({
|
|
7628
|
+
realtimeInput: {
|
|
7629
|
+
audio: {
|
|
7630
|
+
data: convertToBase644(value),
|
|
7631
|
+
mimeType: `audio/pcm;rate=${inputAudioRate}`
|
|
7632
|
+
}
|
|
7633
|
+
}
|
|
7634
|
+
})
|
|
7635
|
+
);
|
|
7636
|
+
await waitForWebSocketBufferDrain(socket);
|
|
7637
|
+
}
|
|
7638
|
+
} finally {
|
|
7639
|
+
audioReader.releaseLock();
|
|
7640
|
+
audioReader = void 0;
|
|
7641
|
+
}
|
|
7642
|
+
if (!finished) {
|
|
7643
|
+
socket.send(
|
|
7644
|
+
JSON.stringify({ realtimeInput: { audioStreamEnd: true } })
|
|
7645
|
+
);
|
|
7646
|
+
audioEnded = true;
|
|
7647
|
+
if (sawTurnComplete && !openTurn) {
|
|
7648
|
+
schedulePendingFinish();
|
|
7649
|
+
}
|
|
7650
|
+
}
|
|
7651
|
+
};
|
|
7652
|
+
connection = connectToWebSocket({
|
|
7653
|
+
url,
|
|
7654
|
+
headers,
|
|
7655
|
+
webSocket,
|
|
7656
|
+
abortSignal,
|
|
7657
|
+
onAbort: finishWithError,
|
|
7658
|
+
onProcessingError: finishWithError,
|
|
7659
|
+
onOpen: (socket) => {
|
|
7660
|
+
controller.enqueue({ type: "stream-start", warnings });
|
|
7661
|
+
socket.send(JSON.stringify({ setup }));
|
|
7662
|
+
void setupComplete.then(() => finished ? void 0 : sendAudio(socket)).catch(finishWithError);
|
|
7663
|
+
},
|
|
7664
|
+
onMessageText: async (text) => {
|
|
7665
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
7666
|
+
if (finished) return;
|
|
7667
|
+
const parsed = await safeParseJSON2({ text });
|
|
7668
|
+
if (!parsed.success) return;
|
|
7669
|
+
const message = parsed.value;
|
|
7670
|
+
if (includeRawChunks) {
|
|
7671
|
+
controller.enqueue({ type: "raw", rawValue: message });
|
|
7672
|
+
}
|
|
7673
|
+
if (message.setupComplete != null) {
|
|
7674
|
+
resolveSetupComplete();
|
|
7675
|
+
}
|
|
7676
|
+
if (message.usageMetadata != null) {
|
|
7677
|
+
usage = accumulateGoogleLiveUsage(usage, message.usageMetadata);
|
|
7678
|
+
}
|
|
7679
|
+
if (message.error != null) {
|
|
7680
|
+
finishWithError(
|
|
7681
|
+
new Error((_a = message.error.message) != null ? _a : "Google Live API error")
|
|
7682
|
+
);
|
|
7683
|
+
return;
|
|
7684
|
+
}
|
|
7685
|
+
const inputTranscriptionText = (_e = (_c = (_b = message.serverContent) == null ? void 0 : _b.inputTranscription) == null ? void 0 : _c.text) != null ? _e : (_d = message.inputTranscription) == null ? void 0 : _d.text;
|
|
7686
|
+
if (inputTranscriptionText) {
|
|
7687
|
+
onTurnActivity();
|
|
7688
|
+
sourceTurnBuffer += inputTranscriptionText;
|
|
7689
|
+
controller.enqueue({
|
|
7690
|
+
type: "source-transcript-delta",
|
|
7691
|
+
id: itemId(),
|
|
7692
|
+
delta: inputTranscriptionText
|
|
7693
|
+
});
|
|
7694
|
+
}
|
|
7695
|
+
const serverContent = message.serverContent;
|
|
7696
|
+
if (serverContent == null) {
|
|
7697
|
+
return;
|
|
7698
|
+
}
|
|
7699
|
+
for (const part of (_g = (_f = serverContent.modelTurn) == null ? void 0 : _f.parts) != null ? _g : []) {
|
|
7700
|
+
if ((_h = part.inlineData) == null ? void 0 : _h.data) {
|
|
7701
|
+
controller.enqueue({
|
|
7702
|
+
type: "audio",
|
|
7703
|
+
id: itemId(),
|
|
7704
|
+
audio: part.inlineData.data
|
|
7705
|
+
});
|
|
7706
|
+
const silenceDurationMs = getPcm16SilenceDurationMs(
|
|
7707
|
+
part.inlineData.data
|
|
7708
|
+
);
|
|
7709
|
+
if (audioEnded && silenceDurationMs != null) {
|
|
7710
|
+
trailingSilenceMs += silenceDurationMs;
|
|
7711
|
+
if (trailingSilenceMs >= finishGraceMs) {
|
|
7712
|
+
finish();
|
|
7713
|
+
return;
|
|
7714
|
+
}
|
|
7715
|
+
} else {
|
|
7716
|
+
onTurnActivity();
|
|
7717
|
+
}
|
|
7718
|
+
}
|
|
7719
|
+
}
|
|
7720
|
+
if ((_i = serverContent.outputTranscription) == null ? void 0 : _i.text) {
|
|
7721
|
+
onTurnActivity();
|
|
7722
|
+
translationTurnBuffer += serverContent.outputTranscription.text;
|
|
7723
|
+
controller.enqueue({
|
|
7724
|
+
type: "output-text-delta",
|
|
7725
|
+
id: itemId(),
|
|
7726
|
+
delta: serverContent.outputTranscription.text
|
|
7727
|
+
});
|
|
7728
|
+
}
|
|
7729
|
+
if (serverContent.turnComplete) {
|
|
7730
|
+
completeTurn();
|
|
7731
|
+
openTurn = false;
|
|
7732
|
+
sawTurnComplete = true;
|
|
7733
|
+
if (audioEnded) {
|
|
7734
|
+
schedulePendingFinish();
|
|
7735
|
+
}
|
|
7736
|
+
}
|
|
7737
|
+
},
|
|
7738
|
+
onSocketError: () => {
|
|
7739
|
+
finishWithError(new Error("Google Live translation error"));
|
|
7740
|
+
},
|
|
7741
|
+
onClose: ({ code, reason }) => {
|
|
7742
|
+
if (finished) return;
|
|
7743
|
+
if (finishTimer != null) {
|
|
7744
|
+
finish();
|
|
7745
|
+
return;
|
|
7746
|
+
}
|
|
7747
|
+
finishWithError(
|
|
7748
|
+
new Error(
|
|
7749
|
+
`Google Live translation WebSocket closed unexpectedly before finishing (code ${code != null ? code : "unknown"}${reason ? `, reason: ${reason}` : ""}).`
|
|
7750
|
+
)
|
|
7751
|
+
);
|
|
7752
|
+
}
|
|
7753
|
+
});
|
|
7754
|
+
},
|
|
7755
|
+
cancel: () => {
|
|
7756
|
+
if (finished) return;
|
|
7757
|
+
finished = true;
|
|
7758
|
+
cleanup();
|
|
7759
|
+
}
|
|
7760
|
+
});
|
|
7761
|
+
}
|
|
7762
|
+
function accumulateGoogleLiveUsage(usage, usageMetadata) {
|
|
7763
|
+
var _a, _b;
|
|
7764
|
+
let inputAudioTokens = usage == null ? void 0 : usage.inputAudioTokens;
|
|
7765
|
+
let outputAudioTokens = usage == null ? void 0 : usage.outputAudioTokens;
|
|
7766
|
+
for (const detail of (_a = usageMetadata.promptTokensDetails) != null ? _a : []) {
|
|
7767
|
+
if (detail.modality === "AUDIO" && detail.tokenCount != null) {
|
|
7768
|
+
inputAudioTokens = (inputAudioTokens != null ? inputAudioTokens : 0) + detail.tokenCount;
|
|
7769
|
+
}
|
|
7770
|
+
}
|
|
7771
|
+
for (const detail of (_b = usageMetadata.responseTokensDetails) != null ? _b : []) {
|
|
7772
|
+
if (detail.modality === "AUDIO" && detail.tokenCount != null) {
|
|
7773
|
+
outputAudioTokens = (outputAudioTokens != null ? outputAudioTokens : 0) + detail.tokenCount;
|
|
7774
|
+
}
|
|
7775
|
+
}
|
|
7776
|
+
if (inputAudioTokens == null && outputAudioTokens == null) {
|
|
7777
|
+
return usage;
|
|
7778
|
+
}
|
|
7779
|
+
return {
|
|
7780
|
+
...usage,
|
|
7781
|
+
...inputAudioTokens != null ? { inputAudioTokens } : {},
|
|
7782
|
+
...outputAudioTokens != null ? { outputAudioTokens } : {}
|
|
7783
|
+
};
|
|
7784
|
+
}
|
|
7785
|
+
function getPcm16SilenceDurationMs(audio) {
|
|
7786
|
+
let bytes;
|
|
7787
|
+
try {
|
|
7788
|
+
bytes = convertBase64ToUint8Array2(audio);
|
|
7789
|
+
} catch (e) {
|
|
7790
|
+
return void 0;
|
|
7791
|
+
}
|
|
7792
|
+
if (bytes.byteLength < 2) {
|
|
7793
|
+
return void 0;
|
|
7794
|
+
}
|
|
7795
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
7796
|
+
const sampleCount = Math.floor(bytes.byteLength / 2);
|
|
7797
|
+
for (let i = 0; i < sampleCount; i++) {
|
|
7798
|
+
if (Math.abs(view.getInt16(i * 2, true)) > pcm16SilenceAmplitudeThreshold) {
|
|
7799
|
+
return void 0;
|
|
7800
|
+
}
|
|
7801
|
+
}
|
|
7802
|
+
return sampleCount / googleLiveOutputAudioRate * 1e3;
|
|
7803
|
+
}
|
|
7804
|
+
function buildGoogleLiveTranslationSetup({
|
|
7805
|
+
modelId,
|
|
7806
|
+
targetLanguage,
|
|
7807
|
+
providerOptions
|
|
7808
|
+
}) {
|
|
7809
|
+
return {
|
|
7810
|
+
model: getModelPath(modelId),
|
|
7811
|
+
generationConfig: {
|
|
7812
|
+
responseModalities: ["AUDIO"],
|
|
7813
|
+
translationConfig: {
|
|
7814
|
+
targetLanguageCode: targetLanguage,
|
|
7815
|
+
...(providerOptions == null ? void 0 : providerOptions.echoTargetLanguage) != null ? { echoTargetLanguage: providerOptions.echoTargetLanguage } : {}
|
|
7816
|
+
}
|
|
7817
|
+
},
|
|
7818
|
+
inputAudioTranscription: {},
|
|
7819
|
+
outputAudioTranscription: {}
|
|
7820
|
+
};
|
|
7821
|
+
}
|
|
7822
|
+
function validateGoogleTranslationInputAudioFormat(inputAudioFormat) {
|
|
7823
|
+
if (inputAudioFormat.type !== "audio/pcm" || inputAudioFormat.rate != null && inputAudioFormat.rate !== 16e3) {
|
|
7824
|
+
throw new InvalidArgumentError({
|
|
7825
|
+
argument: "inputAudioFormat",
|
|
7826
|
+
message: "The Gemini Live translation API only supports 16kHz 16-bit PCM input audio."
|
|
7827
|
+
});
|
|
7828
|
+
}
|
|
7829
|
+
}
|
|
7830
|
+
|
|
7354
7831
|
// src/google-provider.ts
|
|
7355
7832
|
var supportedExternalUrlMediaTypes = [
|
|
7356
7833
|
"text/html",
|
|
@@ -7457,6 +7934,12 @@ function createGoogle(options = {}) {
|
|
|
7457
7934
|
headers: getHeaders,
|
|
7458
7935
|
fetch: options.fetch
|
|
7459
7936
|
});
|
|
7937
|
+
const createTranslationModel = (modelId) => new GoogleTranslationModel(modelId, {
|
|
7938
|
+
provider: `${providerName}.translation`,
|
|
7939
|
+
baseURL,
|
|
7940
|
+
headers: getHeaders,
|
|
7941
|
+
webSocket: options.webSocket
|
|
7942
|
+
});
|
|
7460
7943
|
const createSpeechModel = (modelId) => new GoogleSpeechModel(modelId, {
|
|
7461
7944
|
provider: `${providerName}.speech`,
|
|
7462
7945
|
baseURL,
|
|
@@ -7517,6 +8000,8 @@ function createGoogle(options = {}) {
|
|
|
7517
8000
|
provider.files = createFiles;
|
|
7518
8001
|
provider.speech = createSpeechModel;
|
|
7519
8002
|
provider.speechModel = createSpeechModel;
|
|
8003
|
+
provider.translation = createTranslationModel;
|
|
8004
|
+
provider.speechTranslationModel = createTranslationModel;
|
|
7520
8005
|
provider.interactions = createInteractionsModel;
|
|
7521
8006
|
provider.tools = googleTools;
|
|
7522
8007
|
return provider;
|
|
@@ -7524,6 +8009,7 @@ function createGoogle(options = {}) {
|
|
|
7524
8009
|
var google = createGoogle();
|
|
7525
8010
|
export {
|
|
7526
8011
|
GoogleRealtimeModel as Experimental_GoogleRealtimeModel,
|
|
8012
|
+
GoogleTranslationModel as Experimental_GoogleTranslationModel,
|
|
7527
8013
|
VERSION,
|
|
7528
8014
|
createGoogle,
|
|
7529
8015
|
createGoogle as createGoogleGenerativeAI,
|