@ai-sdk/google-vertex 5.0.65 → 5.0.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/anthropic/edge/index.js +1 -1
- package/dist/edge/index.d.ts +7 -1
- package/dist/edge/index.js +571 -45
- package/dist/edge/index.js.map +1 -1
- package/dist/index.d.ts +67 -3
- package/dist/index.js +572 -45
- package/dist/index.js.map +1 -1
- package/dist/maas/edge/index.js +1 -1
- package/dist/xai/edge/index.js +1 -1
- package/package.json +2 -2
- package/src/gemini-transcription/google-vertex-gemini-transcription-model-options.ts +51 -0
- package/src/gemini-transcription/google-vertex-gemini-transcription-model.ts +709 -0
- package/src/google-vertex-provider-base.ts +25 -0
- package/src/index.ts +5 -0
package/dist/edge/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/edge/google-vertex-provider-edge.ts
|
|
2
|
-
import { loadOptionalSetting as loadOptionalSetting3, resolve as
|
|
2
|
+
import { loadOptionalSetting as loadOptionalSetting3, resolve as resolve7 } from "@ai-sdk/provider-utils";
|
|
3
3
|
|
|
4
4
|
// src/google-vertex-provider-base.ts
|
|
5
5
|
import {
|
|
@@ -12,13 +12,13 @@ import {
|
|
|
12
12
|
loadOptionalSetting,
|
|
13
13
|
loadSetting,
|
|
14
14
|
normalizeHeaders,
|
|
15
|
-
resolve as
|
|
15
|
+
resolve as resolve6,
|
|
16
16
|
withoutTrailingSlash,
|
|
17
17
|
withUserAgentSuffix
|
|
18
18
|
} from "@ai-sdk/provider-utils";
|
|
19
19
|
|
|
20
20
|
// src/version.ts
|
|
21
|
-
var VERSION = true ? "5.0.
|
|
21
|
+
var VERSION = true ? "5.0.67" : "0.0.0-test";
|
|
22
22
|
|
|
23
23
|
// src/google-vertex-embedding-model.ts
|
|
24
24
|
import {
|
|
@@ -738,36 +738,551 @@ var googleVertexTranscriptionResponseSchema = z6.object({
|
|
|
738
738
|
}).nullish()
|
|
739
739
|
});
|
|
740
740
|
|
|
741
|
-
// src/google-vertex-
|
|
741
|
+
// src/gemini-transcription/google-vertex-gemini-transcription-model.ts
|
|
742
742
|
import {
|
|
743
|
-
|
|
743
|
+
InvalidArgumentError
|
|
744
744
|
} from "@ai-sdk/provider";
|
|
745
745
|
import {
|
|
746
746
|
combineHeaders as combineHeaders4,
|
|
747
|
-
|
|
747
|
+
connectToWebSocket,
|
|
748
|
+
convertToBase64 as convertToBase642,
|
|
748
749
|
createJsonResponseHandler as createJsonResponseHandler4,
|
|
749
750
|
parseProviderOptions as parseProviderOptions3,
|
|
750
751
|
postJsonToApi as postJsonToApi4,
|
|
751
|
-
resolve as resolve4
|
|
752
|
+
resolve as resolve4,
|
|
753
|
+
safeParseJSON,
|
|
754
|
+
serializeModelOptions as serializeModelOptions5,
|
|
755
|
+
waitForWebSocketBufferDrain,
|
|
756
|
+
WORKFLOW_DESERIALIZE as WORKFLOW_DESERIALIZE5,
|
|
757
|
+
WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE5
|
|
752
758
|
} from "@ai-sdk/provider-utils";
|
|
753
759
|
import { z as z8 } from "zod/v4";
|
|
754
760
|
|
|
761
|
+
// src/gemini-transcription/google-vertex-gemini-transcription-model-options.ts
|
|
762
|
+
import { z as z7 } from "zod/v4";
|
|
763
|
+
var googleVertexGeminiTranscriptionModelOptions = z7.object({
|
|
764
|
+
/**
|
|
765
|
+
* BCP-47 language codes providing hints about the languages present in the
|
|
766
|
+
* audio. If omitted or empty, defaults to automatic language detection.
|
|
767
|
+
*/
|
|
768
|
+
languageCodes: z7.array(z7.string()).optional(),
|
|
769
|
+
/**
|
|
770
|
+
* Custom vocabulary phrases, which bias the speech recognition model
|
|
771
|
+
* toward recognizing specific terms.
|
|
772
|
+
*/
|
|
773
|
+
customVocabulary: z7.array(z7.string()).optional(),
|
|
774
|
+
/**
|
|
775
|
+
* Enables word-level timestamp generation.
|
|
776
|
+
*/
|
|
777
|
+
wordTimestamp: z7.boolean().optional(),
|
|
778
|
+
/**
|
|
779
|
+
* Enables speaker diarization.
|
|
780
|
+
*/
|
|
781
|
+
diarization: z7.boolean().optional(),
|
|
782
|
+
/**
|
|
783
|
+
* Transcription output formatting mode.
|
|
784
|
+
*
|
|
785
|
+
* - `VERBATIM` (default): exact literal transcript preserving filler
|
|
786
|
+
* words, repetitions, and false starts.
|
|
787
|
+
* - `SMART`: cleans up and structures the transcript in real time —
|
|
788
|
+
* disfluency removal, inline self-corrections, structured formatting
|
|
789
|
+
* (lists, numbers, dates, paragraph breaks), and grammar/casing polish.
|
|
790
|
+
*/
|
|
791
|
+
mode: z7.enum(["SMART", "VERBATIM"]).optional()
|
|
792
|
+
});
|
|
793
|
+
|
|
794
|
+
// src/gemini-transcription/google-vertex-gemini-transcription-model.ts
|
|
795
|
+
var liveWebSocketPath = "google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent";
|
|
796
|
+
var defaultFinishGraceMs = 3e3;
|
|
797
|
+
function isLiveTranscriptionModelId(modelId) {
|
|
798
|
+
return modelId.includes("-live");
|
|
799
|
+
}
|
|
800
|
+
function vertexHost(location) {
|
|
801
|
+
if (location === "global") return "aiplatform.googleapis.com";
|
|
802
|
+
if (location === "eu" || location === "us") {
|
|
803
|
+
return `aiplatform.${location}.rep.googleapis.com`;
|
|
804
|
+
}
|
|
805
|
+
return `${location}-aiplatform.googleapis.com`;
|
|
806
|
+
}
|
|
807
|
+
var GoogleVertexGeminiTranscriptionModel = class _GoogleVertexGeminiTranscriptionModel {
|
|
808
|
+
constructor(modelId, config) {
|
|
809
|
+
this.modelId = modelId;
|
|
810
|
+
this.config = config;
|
|
811
|
+
this.specificationVersion = "v4";
|
|
812
|
+
}
|
|
813
|
+
static [WORKFLOW_SERIALIZE5](model) {
|
|
814
|
+
return serializeModelOptions5({
|
|
815
|
+
modelId: model.modelId,
|
|
816
|
+
config: model.config
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
static [WORKFLOW_DESERIALIZE5](options) {
|
|
820
|
+
return new _GoogleVertexGeminiTranscriptionModel(
|
|
821
|
+
options.modelId,
|
|
822
|
+
options.config
|
|
823
|
+
);
|
|
824
|
+
}
|
|
825
|
+
get provider() {
|
|
826
|
+
return this.config.provider;
|
|
827
|
+
}
|
|
828
|
+
async parseOptions(providerOptions) {
|
|
829
|
+
for (const provider of ["googleVertex", "vertex", "google"]) {
|
|
830
|
+
const parsed = await parseProviderOptions3({
|
|
831
|
+
provider,
|
|
832
|
+
providerOptions,
|
|
833
|
+
schema: googleVertexGeminiTranscriptionModelOptions
|
|
834
|
+
});
|
|
835
|
+
if (parsed != null) return parsed;
|
|
836
|
+
}
|
|
837
|
+
return void 0;
|
|
838
|
+
}
|
|
839
|
+
async doGenerate(options) {
|
|
840
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
841
|
+
if (isLiveTranscriptionModelId(this.modelId)) {
|
|
842
|
+
throw new InvalidArgumentError({
|
|
843
|
+
argument: "modelId",
|
|
844
|
+
message: `Model '${this.modelId}' only supports streaming transcription. Use experimental_streamTranscribe, or a unary model such as 'gemini-3.5-transcribe'.`
|
|
845
|
+
});
|
|
846
|
+
}
|
|
847
|
+
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
848
|
+
const warnings = [];
|
|
849
|
+
const googleOptions = await this.parseOptions(options.providerOptions);
|
|
850
|
+
const audioTranscriptionConfig = buildAudioTranscriptionConfig(googleOptions);
|
|
851
|
+
const requestBody = {
|
|
852
|
+
contents: [
|
|
853
|
+
{
|
|
854
|
+
role: "user",
|
|
855
|
+
parts: [
|
|
856
|
+
{
|
|
857
|
+
inlineData: {
|
|
858
|
+
mimeType: options.mediaType,
|
|
859
|
+
data: convertToBase642(options.audio)
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
]
|
|
863
|
+
}
|
|
864
|
+
],
|
|
865
|
+
...audioTranscriptionConfig != null ? { generationConfig: { audioTranscriptionConfig } } : {}
|
|
866
|
+
};
|
|
867
|
+
const {
|
|
868
|
+
value: response,
|
|
869
|
+
responseHeaders,
|
|
870
|
+
rawValue: rawResponse
|
|
871
|
+
} = await postJsonToApi4({
|
|
872
|
+
url: `${this.config.baseURL}/models/${this.modelId}:generateContent`,
|
|
873
|
+
headers: combineHeaders4(
|
|
874
|
+
this.config.headers ? await resolve4(this.config.headers) : void 0,
|
|
875
|
+
options.headers
|
|
876
|
+
),
|
|
877
|
+
body: requestBody,
|
|
878
|
+
failedResponseHandler: googleVertexFailedResponseHandler,
|
|
879
|
+
successfulResponseHandler: createJsonResponseHandler4(
|
|
880
|
+
googleVertexGeminiTranscriptionResponseSchema
|
|
881
|
+
),
|
|
882
|
+
abortSignal: options.abortSignal,
|
|
883
|
+
fetch: this.config.fetch
|
|
884
|
+
});
|
|
885
|
+
const parts = (_g = (_f = (_e = (_d = response.candidates) == null ? void 0 : _d[0]) == null ? void 0 : _e.content) == null ? void 0 : _f.parts) != null ? _g : [];
|
|
886
|
+
const plainText = parts.map((part) => {
|
|
887
|
+
var _a2;
|
|
888
|
+
return (_a2 = part.text) != null ? _a2 : "";
|
|
889
|
+
}).join("");
|
|
890
|
+
const transcriptionText = parts.map((part) => {
|
|
891
|
+
var _a2, _b2;
|
|
892
|
+
return (_b2 = (_a2 = part.audioTranscription) == null ? void 0 : _a2.text) != null ? _b2 : "";
|
|
893
|
+
}).join("");
|
|
894
|
+
const text = plainText !== "" ? plainText : transcriptionText;
|
|
895
|
+
let language;
|
|
896
|
+
const segments = [];
|
|
897
|
+
for (const part of parts) {
|
|
898
|
+
const transcription = part.audioTranscription;
|
|
899
|
+
if (transcription == null) continue;
|
|
900
|
+
language != null ? language : language = (_h = transcription.languageCode) != null ? _h : void 0;
|
|
901
|
+
for (const word of (_i = transcription.words) != null ? _i : []) {
|
|
902
|
+
const startSecond = parseOffsetSeconds(word.startOffset);
|
|
903
|
+
const endSecond = parseOffsetSeconds(word.endOffset);
|
|
904
|
+
if (word.word == null || startSecond == null || endSecond == null) {
|
|
905
|
+
continue;
|
|
906
|
+
}
|
|
907
|
+
segments.push({ text: word.word, startSecond, endSecond });
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
return {
|
|
911
|
+
text,
|
|
912
|
+
segments,
|
|
913
|
+
language,
|
|
914
|
+
durationInSeconds: void 0,
|
|
915
|
+
warnings,
|
|
916
|
+
response: {
|
|
917
|
+
timestamp: currentDate,
|
|
918
|
+
modelId: this.modelId,
|
|
919
|
+
headers: responseHeaders,
|
|
920
|
+
body: rawResponse
|
|
921
|
+
},
|
|
922
|
+
...response.usageMetadata != null ? {
|
|
923
|
+
providerMetadata: {
|
|
924
|
+
google: { usageMetadata: response.usageMetadata }
|
|
925
|
+
}
|
|
926
|
+
} : {}
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
async doStream(options) {
|
|
930
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
931
|
+
if (!isLiveTranscriptionModelId(this.modelId)) {
|
|
932
|
+
throw new InvalidArgumentError({
|
|
933
|
+
argument: "modelId",
|
|
934
|
+
message: `Model '${this.modelId}' does not support streaming transcription. Use a live model such as 'gemini-3.5-transcribe-live'.`
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
938
|
+
const warnings = [];
|
|
939
|
+
const googleOptions = await this.parseOptions(options.providerOptions);
|
|
940
|
+
validateLiveInputAudioFormat(options.inputAudioFormat);
|
|
941
|
+
const headers = combineHeaders4(
|
|
942
|
+
this.config.headers ? await resolve4(this.config.headers) : void 0,
|
|
943
|
+
options.headers
|
|
944
|
+
);
|
|
945
|
+
const { project, location } = this.config;
|
|
946
|
+
const modelResource = `projects/${project}/locations/${location}/publishers/google/models/${this.modelId}`;
|
|
947
|
+
const url = new URL(
|
|
948
|
+
`wss://${vertexHost(location)}/ws/${liveWebSocketPath}`
|
|
949
|
+
);
|
|
950
|
+
const setup = {
|
|
951
|
+
model: modelResource,
|
|
952
|
+
inputAudioTranscription: (_d = buildAudioTranscriptionConfig(googleOptions)) != null ? _d : {}
|
|
953
|
+
};
|
|
954
|
+
return {
|
|
955
|
+
request: { body: setup },
|
|
956
|
+
response: {
|
|
957
|
+
timestamp: currentDate,
|
|
958
|
+
modelId: this.modelId
|
|
959
|
+
},
|
|
960
|
+
stream: createVertexLiveTranscriptionStream({
|
|
961
|
+
webSocket: this.config.webSocket,
|
|
962
|
+
url,
|
|
963
|
+
headers,
|
|
964
|
+
setup,
|
|
965
|
+
inputAudioRate: (_e = options.inputAudioFormat.rate) != null ? _e : 16e3,
|
|
966
|
+
finishGraceMs: (_g = (_f = this.config._internal) == null ? void 0 : _f.finishGraceMs) != null ? _g : defaultFinishGraceMs,
|
|
967
|
+
warnings,
|
|
968
|
+
audio: options.audio,
|
|
969
|
+
abortSignal: options.abortSignal,
|
|
970
|
+
includeRawChunks: options.includeRawChunks
|
|
971
|
+
})
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
};
|
|
975
|
+
function createVertexLiveTranscriptionStream({
|
|
976
|
+
webSocket,
|
|
977
|
+
url,
|
|
978
|
+
headers,
|
|
979
|
+
setup,
|
|
980
|
+
inputAudioRate,
|
|
981
|
+
finishGraceMs,
|
|
982
|
+
warnings,
|
|
983
|
+
audio,
|
|
984
|
+
abortSignal,
|
|
985
|
+
includeRawChunks
|
|
986
|
+
}) {
|
|
987
|
+
let finished = false;
|
|
988
|
+
let cleanup = () => {
|
|
989
|
+
};
|
|
990
|
+
return new ReadableStream({
|
|
991
|
+
start: (controller) => {
|
|
992
|
+
let audioReader;
|
|
993
|
+
let connection;
|
|
994
|
+
let resolveSetupComplete;
|
|
995
|
+
const setupComplete = new Promise((resolvePromise) => {
|
|
996
|
+
resolveSetupComplete = resolvePromise;
|
|
997
|
+
});
|
|
998
|
+
let segmentCounter = 0;
|
|
999
|
+
let segmentBuffer = "";
|
|
1000
|
+
let fullText = "";
|
|
1001
|
+
let latestInterim = "";
|
|
1002
|
+
let language;
|
|
1003
|
+
let audioEnded = false;
|
|
1004
|
+
let usageMetadata;
|
|
1005
|
+
let finishTimer;
|
|
1006
|
+
const segmentId = () => `google-segment-${segmentCounter}`;
|
|
1007
|
+
const cancelPendingFinish = () => {
|
|
1008
|
+
if (finishTimer != null) {
|
|
1009
|
+
clearTimeout(finishTimer);
|
|
1010
|
+
finishTimer = void 0;
|
|
1011
|
+
}
|
|
1012
|
+
};
|
|
1013
|
+
const schedulePendingFinish = () => {
|
|
1014
|
+
if (finished || !audioEnded) return;
|
|
1015
|
+
cancelPendingFinish();
|
|
1016
|
+
finishTimer = setTimeout(() => {
|
|
1017
|
+
finishTimer = void 0;
|
|
1018
|
+
finish();
|
|
1019
|
+
}, finishGraceMs);
|
|
1020
|
+
};
|
|
1021
|
+
cleanup = (closeCode) => {
|
|
1022
|
+
cancelPendingFinish();
|
|
1023
|
+
if (audioReader != null) {
|
|
1024
|
+
void audioReader.cancel().catch(() => {
|
|
1025
|
+
});
|
|
1026
|
+
} else {
|
|
1027
|
+
void audio.cancel().catch(() => {
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
connection == null ? void 0 : connection.close(closeCode);
|
|
1031
|
+
};
|
|
1032
|
+
const finishWithError = (error) => {
|
|
1033
|
+
if (finished) return;
|
|
1034
|
+
finished = true;
|
|
1035
|
+
cleanup();
|
|
1036
|
+
controller.error(error);
|
|
1037
|
+
};
|
|
1038
|
+
const completeSegment = () => {
|
|
1039
|
+
if (segmentBuffer === "") {
|
|
1040
|
+
if (latestInterim === "") return;
|
|
1041
|
+
segmentBuffer = latestInterim;
|
|
1042
|
+
}
|
|
1043
|
+
latestInterim = "";
|
|
1044
|
+
controller.enqueue({
|
|
1045
|
+
type: "transcript-final",
|
|
1046
|
+
id: segmentId(),
|
|
1047
|
+
text: segmentBuffer
|
|
1048
|
+
});
|
|
1049
|
+
fullText += fullText === "" ? segmentBuffer : ` ${segmentBuffer}`;
|
|
1050
|
+
segmentBuffer = "";
|
|
1051
|
+
segmentCounter++;
|
|
1052
|
+
};
|
|
1053
|
+
const finish = () => {
|
|
1054
|
+
if (finished) return;
|
|
1055
|
+
completeSegment();
|
|
1056
|
+
finished = true;
|
|
1057
|
+
controller.enqueue({
|
|
1058
|
+
type: "finish",
|
|
1059
|
+
text: fullText,
|
|
1060
|
+
segments: [],
|
|
1061
|
+
language,
|
|
1062
|
+
durationInSeconds: void 0,
|
|
1063
|
+
...usageMetadata != null ? { providerMetadata: { google: { usageMetadata } } } : {}
|
|
1064
|
+
});
|
|
1065
|
+
controller.close();
|
|
1066
|
+
cleanup(1e3);
|
|
1067
|
+
};
|
|
1068
|
+
const sendAudio = async (socket) => {
|
|
1069
|
+
audioReader = audio.getReader();
|
|
1070
|
+
try {
|
|
1071
|
+
while (true) {
|
|
1072
|
+
const { done, value } = await audioReader.read();
|
|
1073
|
+
if (done || finished) break;
|
|
1074
|
+
socket.send(
|
|
1075
|
+
JSON.stringify({
|
|
1076
|
+
realtimeInput: {
|
|
1077
|
+
audio: {
|
|
1078
|
+
data: convertToBase642(value),
|
|
1079
|
+
mimeType: `audio/pcm;rate=${inputAudioRate}`
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
})
|
|
1083
|
+
);
|
|
1084
|
+
await waitForWebSocketBufferDrain(socket);
|
|
1085
|
+
}
|
|
1086
|
+
} finally {
|
|
1087
|
+
audioReader.releaseLock();
|
|
1088
|
+
audioReader = void 0;
|
|
1089
|
+
}
|
|
1090
|
+
if (!finished) {
|
|
1091
|
+
socket.send(
|
|
1092
|
+
JSON.stringify({ realtimeInput: { audioStreamEnd: true } })
|
|
1093
|
+
);
|
|
1094
|
+
audioEnded = true;
|
|
1095
|
+
schedulePendingFinish();
|
|
1096
|
+
}
|
|
1097
|
+
};
|
|
1098
|
+
connection = connectToWebSocket({
|
|
1099
|
+
url,
|
|
1100
|
+
headers,
|
|
1101
|
+
webSocket,
|
|
1102
|
+
abortSignal,
|
|
1103
|
+
onAbort: finishWithError,
|
|
1104
|
+
onProcessingError: finishWithError,
|
|
1105
|
+
onOpen: (socket) => {
|
|
1106
|
+
controller.enqueue({ type: "stream-start", warnings });
|
|
1107
|
+
socket.send(JSON.stringify({ setup }));
|
|
1108
|
+
void setupComplete.then(() => finished ? void 0 : sendAudio(socket)).catch(finishWithError);
|
|
1109
|
+
},
|
|
1110
|
+
onMessageText: async (text) => {
|
|
1111
|
+
var _a, _b;
|
|
1112
|
+
if (finished) return;
|
|
1113
|
+
const parsed = await safeParseJSON({ text });
|
|
1114
|
+
if (!parsed.success) return;
|
|
1115
|
+
const message = parsed.value;
|
|
1116
|
+
if (includeRawChunks) {
|
|
1117
|
+
controller.enqueue({ type: "raw", rawValue: message });
|
|
1118
|
+
}
|
|
1119
|
+
if (message.setupComplete != null) {
|
|
1120
|
+
resolveSetupComplete();
|
|
1121
|
+
}
|
|
1122
|
+
if (message.usageMetadata != null) {
|
|
1123
|
+
usageMetadata = message.usageMetadata;
|
|
1124
|
+
}
|
|
1125
|
+
if (message.error != null) {
|
|
1126
|
+
finishWithError(
|
|
1127
|
+
new Error((_a = message.error.message) != null ? _a : "Vertex Live API error")
|
|
1128
|
+
);
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
const serverContent = message.serverContent;
|
|
1132
|
+
const interim = serverContent == null ? void 0 : serverContent.interimInputTranscription;
|
|
1133
|
+
if (interim == null ? void 0 : interim.text) {
|
|
1134
|
+
schedulePendingFinish();
|
|
1135
|
+
latestInterim = interim.text;
|
|
1136
|
+
controller.enqueue({
|
|
1137
|
+
type: "transcript-partial",
|
|
1138
|
+
id: segmentId(),
|
|
1139
|
+
text: interim.text
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
const transcription = (_b = serverContent == null ? void 0 : serverContent.inputTranscription) != null ? _b : message.inputTranscription;
|
|
1143
|
+
if (transcription != null) {
|
|
1144
|
+
if (transcription.languageCode != null) {
|
|
1145
|
+
language = transcription.languageCode;
|
|
1146
|
+
}
|
|
1147
|
+
if (transcription.text) {
|
|
1148
|
+
schedulePendingFinish();
|
|
1149
|
+
latestInterim = "";
|
|
1150
|
+
segmentBuffer += transcription.text;
|
|
1151
|
+
controller.enqueue({
|
|
1152
|
+
type: "transcript-delta",
|
|
1153
|
+
id: segmentId(),
|
|
1154
|
+
delta: transcription.text
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
if (transcription.finished === true) {
|
|
1158
|
+
completeSegment();
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
if (serverContent == null ? void 0 : serverContent.turnComplete) {
|
|
1162
|
+
completeSegment();
|
|
1163
|
+
}
|
|
1164
|
+
const interactionStatus = serverContent == null ? void 0 : serverContent.interactionStatus;
|
|
1165
|
+
if (audioEnded && (interactionStatus === "IDLE" || interactionStatus === "REQUIRES_ACTION" || (serverContent == null ? void 0 : serverContent.turnComplete) === true && interactionStatus == null)) {
|
|
1166
|
+
finish();
|
|
1167
|
+
}
|
|
1168
|
+
},
|
|
1169
|
+
onSocketError: () => {
|
|
1170
|
+
finishWithError(
|
|
1171
|
+
new Error(
|
|
1172
|
+
"Vertex Live transcription error." + (webSocket == null ? " Note: the native WebSocket implementation cannot send the Authorization header required by Vertex. Pass a header-capable WebSocket implementation (e.g. the 'ws' package) via createVertex({ webSocket })." : "")
|
|
1173
|
+
)
|
|
1174
|
+
);
|
|
1175
|
+
},
|
|
1176
|
+
onClose: ({ code, reason }) => {
|
|
1177
|
+
if (finished) return;
|
|
1178
|
+
if (audioEnded) {
|
|
1179
|
+
finish();
|
|
1180
|
+
return;
|
|
1181
|
+
}
|
|
1182
|
+
finishWithError(
|
|
1183
|
+
new Error(
|
|
1184
|
+
`Vertex Live transcription WebSocket closed unexpectedly before finishing (code ${code != null ? code : "unknown"}${reason ? `, reason: ${reason}` : ""}).`
|
|
1185
|
+
)
|
|
1186
|
+
);
|
|
1187
|
+
}
|
|
1188
|
+
});
|
|
1189
|
+
},
|
|
1190
|
+
cancel: () => {
|
|
1191
|
+
if (finished) return;
|
|
1192
|
+
finished = true;
|
|
1193
|
+
cleanup();
|
|
1194
|
+
}
|
|
1195
|
+
});
|
|
1196
|
+
}
|
|
1197
|
+
function buildAudioTranscriptionConfig(options) {
|
|
1198
|
+
if (options == null) return void 0;
|
|
1199
|
+
const config = {};
|
|
1200
|
+
if (options.languageCodes != null) {
|
|
1201
|
+
config.languageCodes = options.languageCodes;
|
|
1202
|
+
}
|
|
1203
|
+
if (options.customVocabulary != null) {
|
|
1204
|
+
config.customVocabulary = options.customVocabulary;
|
|
1205
|
+
}
|
|
1206
|
+
if (options.wordTimestamp != null) {
|
|
1207
|
+
config.wordTimestamp = options.wordTimestamp;
|
|
1208
|
+
}
|
|
1209
|
+
if (options.diarization != null) {
|
|
1210
|
+
config.diarization = options.diarization;
|
|
1211
|
+
}
|
|
1212
|
+
if (options.mode != null) {
|
|
1213
|
+
config.mode = options.mode;
|
|
1214
|
+
}
|
|
1215
|
+
return Object.keys(config).length > 0 ? config : void 0;
|
|
1216
|
+
}
|
|
1217
|
+
function validateLiveInputAudioFormat(inputAudioFormat) {
|
|
1218
|
+
if (inputAudioFormat.type !== "audio/pcm" || inputAudioFormat.rate != null && inputAudioFormat.rate !== 16e3) {
|
|
1219
|
+
throw new InvalidArgumentError({
|
|
1220
|
+
argument: "inputAudioFormat",
|
|
1221
|
+
message: "The Gemini Live transcription API only supports 16kHz 16-bit PCM input audio."
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
function parseOffsetSeconds(offset) {
|
|
1226
|
+
if (offset == null) return void 0;
|
|
1227
|
+
const parsed = Number.parseFloat(offset);
|
|
1228
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
1229
|
+
}
|
|
1230
|
+
var googleVertexGeminiTranscriptionWordSchema = z8.object({
|
|
1231
|
+
word: z8.string().nullish(),
|
|
1232
|
+
startOffset: z8.string().nullish(),
|
|
1233
|
+
endOffset: z8.string().nullish()
|
|
1234
|
+
});
|
|
1235
|
+
var googleVertexGeminiTranscriptionResponseSchema = z8.object({
|
|
1236
|
+
candidates: z8.array(
|
|
1237
|
+
z8.object({
|
|
1238
|
+
content: z8.object({
|
|
1239
|
+
parts: z8.array(
|
|
1240
|
+
z8.object({
|
|
1241
|
+
text: z8.string().nullish(),
|
|
1242
|
+
audioTranscription: z8.object({
|
|
1243
|
+
text: z8.string().nullish(),
|
|
1244
|
+
languageCode: z8.string().nullish(),
|
|
1245
|
+
speakerLabel: z8.string().nullish(),
|
|
1246
|
+
words: z8.array(googleVertexGeminiTranscriptionWordSchema).nullish()
|
|
1247
|
+
}).nullish()
|
|
1248
|
+
})
|
|
1249
|
+
).nullish()
|
|
1250
|
+
}).nullish()
|
|
1251
|
+
})
|
|
1252
|
+
).nullish(),
|
|
1253
|
+
usageMetadata: z8.record(z8.string(), z8.unknown()).nullish()
|
|
1254
|
+
});
|
|
1255
|
+
|
|
1256
|
+
// src/google-vertex-video-model.ts
|
|
1257
|
+
import {
|
|
1258
|
+
AISDKError
|
|
1259
|
+
} from "@ai-sdk/provider";
|
|
1260
|
+
import {
|
|
1261
|
+
combineHeaders as combineHeaders5,
|
|
1262
|
+
convertUint8ArrayToBase64 as convertUint8ArrayToBase642,
|
|
1263
|
+
createJsonResponseHandler as createJsonResponseHandler5,
|
|
1264
|
+
parseProviderOptions as parseProviderOptions4,
|
|
1265
|
+
postJsonToApi as postJsonToApi5,
|
|
1266
|
+
resolve as resolve5
|
|
1267
|
+
} from "@ai-sdk/provider-utils";
|
|
1268
|
+
import { z as z10 } from "zod/v4";
|
|
1269
|
+
|
|
755
1270
|
// src/google-vertex-video-model-options.ts
|
|
756
1271
|
import { lazySchema, zodSchema } from "@ai-sdk/provider-utils";
|
|
757
|
-
import { z as
|
|
1272
|
+
import { z as z9 } from "zod/v4";
|
|
758
1273
|
var googleVertexVideoModelOptionsSchema = lazySchema(
|
|
759
1274
|
() => zodSchema(
|
|
760
|
-
|
|
761
|
-
pollIntervalMs:
|
|
762
|
-
pollTimeoutMs:
|
|
763
|
-
personGeneration:
|
|
764
|
-
negativePrompt:
|
|
765
|
-
generateAudio:
|
|
766
|
-
gcsOutputDirectory:
|
|
767
|
-
referenceImages:
|
|
768
|
-
|
|
769
|
-
bytesBase64Encoded:
|
|
770
|
-
gcsUri:
|
|
1275
|
+
z9.looseObject({
|
|
1276
|
+
pollIntervalMs: z9.number().positive().nullish(),
|
|
1277
|
+
pollTimeoutMs: z9.number().positive().nullish(),
|
|
1278
|
+
personGeneration: z9.enum(["dont_allow", "allow_adult", "allow_all"]).nullish(),
|
|
1279
|
+
negativePrompt: z9.string().nullish(),
|
|
1280
|
+
generateAudio: z9.boolean().nullish(),
|
|
1281
|
+
gcsOutputDirectory: z9.string().nullish(),
|
|
1282
|
+
referenceImages: z9.array(
|
|
1283
|
+
z9.object({
|
|
1284
|
+
bytesBase64Encoded: z9.string().nullish(),
|
|
1285
|
+
gcsUri: z9.string().nullish()
|
|
771
1286
|
})
|
|
772
1287
|
).nullish()
|
|
773
1288
|
})
|
|
@@ -833,11 +1348,11 @@ var GoogleVertexVideoModel = class {
|
|
|
833
1348
|
async buildRequest(options) {
|
|
834
1349
|
var _a, _b;
|
|
835
1350
|
const warnings = [];
|
|
836
|
-
const googleVertexOptions = (_a = await
|
|
1351
|
+
const googleVertexOptions = (_a = await parseProviderOptions4({
|
|
837
1352
|
provider: "googleVertex",
|
|
838
1353
|
providerOptions: options.providerOptions,
|
|
839
1354
|
schema: googleVertexVideoModelOptionsSchema
|
|
840
|
-
})) != null ? _a : await
|
|
1355
|
+
})) != null ? _a : await parseProviderOptions4({
|
|
841
1356
|
provider: "vertex",
|
|
842
1357
|
providerOptions: options.providerOptions,
|
|
843
1358
|
schema: googleVertexVideoModelOptionsSchema
|
|
@@ -988,17 +1503,17 @@ var GoogleVertexVideoModel = class {
|
|
|
988
1503
|
var _a, _b, _c;
|
|
989
1504
|
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
990
1505
|
const { instances, parameters, warnings } = await this.buildRequest(options);
|
|
991
|
-
const { value: operation, responseHeaders } = await
|
|
1506
|
+
const { value: operation, responseHeaders } = await postJsonToApi5({
|
|
992
1507
|
url: `${this.config.baseURL}/models/${this.modelId}:predictLongRunning`,
|
|
993
|
-
headers:
|
|
994
|
-
await
|
|
1508
|
+
headers: combineHeaders5(
|
|
1509
|
+
await resolve5(this.config.headers),
|
|
995
1510
|
options.headers
|
|
996
1511
|
),
|
|
997
1512
|
body: {
|
|
998
1513
|
instances,
|
|
999
1514
|
parameters
|
|
1000
1515
|
},
|
|
1001
|
-
successfulResponseHandler:
|
|
1516
|
+
successfulResponseHandler: createJsonResponseHandler5(
|
|
1002
1517
|
googleVertexOperationSchema
|
|
1003
1518
|
),
|
|
1004
1519
|
failedResponseHandler: googleVertexFailedResponseHandler,
|
|
@@ -1026,16 +1541,16 @@ var GoogleVertexVideoModel = class {
|
|
|
1026
1541
|
var _a, _b, _c;
|
|
1027
1542
|
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
1028
1543
|
const { operationName } = options.operation;
|
|
1029
|
-
const { value: statusOperation, responseHeaders } = await
|
|
1544
|
+
const { value: statusOperation, responseHeaders } = await postJsonToApi5({
|
|
1030
1545
|
url: `${this.config.baseURL}/models/${this.modelId}:fetchPredictOperation`,
|
|
1031
|
-
headers:
|
|
1032
|
-
await
|
|
1546
|
+
headers: combineHeaders5(
|
|
1547
|
+
await resolve5(this.config.headers),
|
|
1033
1548
|
options.headers
|
|
1034
1549
|
),
|
|
1035
1550
|
body: {
|
|
1036
1551
|
operationName
|
|
1037
1552
|
},
|
|
1038
|
-
successfulResponseHandler:
|
|
1553
|
+
successfulResponseHandler: createJsonResponseHandler5(
|
|
1039
1554
|
googleVertexOperationSchema
|
|
1040
1555
|
),
|
|
1041
1556
|
failedResponseHandler: googleVertexFailedResponseHandler,
|
|
@@ -1071,23 +1586,23 @@ var GoogleVertexVideoModel = class {
|
|
|
1071
1586
|
});
|
|
1072
1587
|
}
|
|
1073
1588
|
};
|
|
1074
|
-
var googleVertexOperationSchema =
|
|
1075
|
-
name:
|
|
1076
|
-
done:
|
|
1077
|
-
error:
|
|
1078
|
-
code:
|
|
1079
|
-
message:
|
|
1080
|
-
status:
|
|
1589
|
+
var googleVertexOperationSchema = z10.object({
|
|
1590
|
+
name: z10.string().nullish(),
|
|
1591
|
+
done: z10.boolean().nullish(),
|
|
1592
|
+
error: z10.object({
|
|
1593
|
+
code: z10.number().nullish(),
|
|
1594
|
+
message: z10.string(),
|
|
1595
|
+
status: z10.string().nullish()
|
|
1081
1596
|
}).nullish(),
|
|
1082
|
-
response:
|
|
1083
|
-
videos:
|
|
1084
|
-
|
|
1085
|
-
bytesBase64Encoded:
|
|
1086
|
-
gcsUri:
|
|
1087
|
-
mimeType:
|
|
1597
|
+
response: z10.object({
|
|
1598
|
+
videos: z10.array(
|
|
1599
|
+
z10.object({
|
|
1600
|
+
bytesBase64Encoded: z10.string().nullish(),
|
|
1601
|
+
gcsUri: z10.string().nullish(),
|
|
1602
|
+
mimeType: z10.string().nullish()
|
|
1088
1603
|
})
|
|
1089
1604
|
).nullish(),
|
|
1090
|
-
raiMediaFilteredCount:
|
|
1605
|
+
raiMediaFilteredCount: z10.number().nullish()
|
|
1091
1606
|
}).nullish()
|
|
1092
1607
|
});
|
|
1093
1608
|
|
|
@@ -1147,7 +1662,7 @@ function createGoogleVertex(options = {}) {
|
|
|
1147
1662
|
const createConfig = (name, { endpoint = false } = {}) => {
|
|
1148
1663
|
const getHeaders = async () => {
|
|
1149
1664
|
var _a;
|
|
1150
|
-
const originalHeaders = await
|
|
1665
|
+
const originalHeaders = await resolve6((_a = options.headers) != null ? _a : {});
|
|
1151
1666
|
return withUserAgentSuffix(
|
|
1152
1667
|
originalHeaders,
|
|
1153
1668
|
`ai-sdk/google-vertex/${VERSION}`
|
|
@@ -1238,6 +1753,17 @@ function createGoogleVertex(options = {}) {
|
|
|
1238
1753
|
);
|
|
1239
1754
|
}
|
|
1240
1755
|
const config = createConfig("transcription");
|
|
1756
|
+
if (modelId.startsWith("gemini")) {
|
|
1757
|
+
return new GoogleVertexGeminiTranscriptionModel(modelId, {
|
|
1758
|
+
provider: config.provider,
|
|
1759
|
+
baseURL: loadBaseURL(),
|
|
1760
|
+
headers: config.headers,
|
|
1761
|
+
fetch: config.fetch,
|
|
1762
|
+
webSocket: options.webSocket,
|
|
1763
|
+
project: loadGoogleVertexProject(),
|
|
1764
|
+
location: loadGoogleVertexLocation()
|
|
1765
|
+
});
|
|
1766
|
+
}
|
|
1241
1767
|
return new GoogleVertexTranscriptionModel(modelId, {
|
|
1242
1768
|
provider: config.provider,
|
|
1243
1769
|
headers: config.headers,
|
|
@@ -1397,7 +1923,7 @@ function createGoogleVertex2(options = {}) {
|
|
|
1397
1923
|
Authorization: `Bearer ${await generateAuthToken(
|
|
1398
1924
|
options.googleCredentials
|
|
1399
1925
|
)}`,
|
|
1400
|
-
...await
|
|
1926
|
+
...await resolve7(options.headers)
|
|
1401
1927
|
})
|
|
1402
1928
|
});
|
|
1403
1929
|
}
|