@ai-sdk/provider 4.0.2 → 4.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/index.d.ts +316 -39
- package/package.json +1 -1
- package/src/embedding-model/v4/embedding-model-v4.ts +1 -1
- package/src/image-model/v4/image-model-v4.ts +1 -1
- package/src/index.ts +1 -0
- package/src/realtime-model/v4/realtime-model-v4-session-config.ts +3 -22
- package/src/reranking-model/v4/reranking-model-v4.ts +1 -1
- package/src/shared/v4/index.ts +1 -0
- package/src/shared/v4/shared-v4-audio-format.ts +14 -0
- package/src/speech-model/v4/speech-model-v4.ts +1 -1
- package/src/speech-translation-model/index.ts +1 -0
- package/src/speech-translation-model/v4/index.ts +5 -0
- package/src/speech-translation-model/v4/speech-translation-model-v4-stream-options.ts +65 -0
- package/src/speech-translation-model/v4/speech-translation-model-v4-stream-part.ts +141 -0
- package/src/speech-translation-model/v4/speech-translation-model-v4-stream-result.ts +47 -0
- package/src/speech-translation-model/v4/speech-translation-model-v4-usage.ts +32 -0
- package/src/speech-translation-model/v4/speech-translation-model-v4.ts +40 -0
- package/src/transcription-model/v4/transcription-model-v4-stream-options.ts +2 -11
- package/src/transcription-model/v4/transcription-model-v4.ts +1 -1
- package/src/video-model/v3/video-model-v3-call-options.ts +5 -1
- package/src/video-model/v4/video-model-v4-call-options.ts +5 -1
- package/src/video-model/v4/video-model-v4-file.ts +7 -0
- package/src/video-model/v4/video-model-v4.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @ai-sdk/provider
|
|
2
2
|
|
|
3
|
+
## 4.0.4
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 1e2f324: feat: add experimental speech translation model specification (`Experimental_SpeechTranslationModelV4`) and `experimental_streamTranslate` for streaming speech-to-speech translation
|
|
8
|
+
|
|
9
|
+
## 4.0.3
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 0f93c57: feat (video): support video (not just image) reference inputs in `inputReferences` for reference-to-video generation
|
|
14
|
+
|
|
3
15
|
## 4.0.2
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
import { JSONSchema7 } from 'json-schema';
|
|
2
2
|
export { JSONSchema7, JSONSchema7Definition } from 'json-schema';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* Audio format configuration shared by provider V4 model specifications.
|
|
6
|
+
*/
|
|
7
|
+
type SharedV4AudioFormat = {
|
|
8
|
+
/**
|
|
9
|
+
* Audio format type, e.g. `audio/pcm`, `audio/pcmu`, or `audio/pcma`.
|
|
10
|
+
*/
|
|
11
|
+
type: string;
|
|
12
|
+
/**
|
|
13
|
+
* Sample rate in Hz. Only applicable for formats that require a rate.
|
|
14
|
+
*/
|
|
15
|
+
rate?: number;
|
|
16
|
+
};
|
|
17
|
+
|
|
4
18
|
/**
|
|
5
19
|
* A mapping of provider names to provider-specific file identifiers.
|
|
6
20
|
*
|
|
@@ -386,7 +400,7 @@ type EmbeddingModelV4Result = {
|
|
|
386
400
|
|
|
387
401
|
/**
|
|
388
402
|
* Specification for an embedding model that implements the embedding model
|
|
389
|
-
* interface version
|
|
403
|
+
* interface version 4.
|
|
390
404
|
*
|
|
391
405
|
* It is specific to text embeddings.
|
|
392
406
|
*/
|
|
@@ -1151,7 +1165,7 @@ type GetMaxImagesPerCallFunction$2 = (options: {
|
|
|
1151
1165
|
modelId: string;
|
|
1152
1166
|
}) => PromiseLike<number | undefined> | number | undefined;
|
|
1153
1167
|
/**
|
|
1154
|
-
* Image generation model specification version
|
|
1168
|
+
* Image generation model specification version 4.
|
|
1155
1169
|
*/
|
|
1156
1170
|
type ImageModelV4 = {
|
|
1157
1171
|
/**
|
|
@@ -4933,7 +4947,7 @@ type RerankingModelV4Result = {
|
|
|
4933
4947
|
};
|
|
4934
4948
|
|
|
4935
4949
|
/**
|
|
4936
|
-
* Specification for a reranking model that implements the reranking model interface version
|
|
4950
|
+
* Specification for a reranking model that implements the reranking model interface version 4.
|
|
4937
4951
|
*/
|
|
4938
4952
|
type RerankingModelV4 = {
|
|
4939
4953
|
/**
|
|
@@ -5061,7 +5075,7 @@ type SpeechModelV4Result = {
|
|
|
5061
5075
|
};
|
|
5062
5076
|
|
|
5063
5077
|
/**
|
|
5064
|
-
* Speech model specification version
|
|
5078
|
+
* Speech model specification version 4.
|
|
5065
5079
|
*/
|
|
5066
5080
|
type SpeechModelV4 = {
|
|
5067
5081
|
/**
|
|
@@ -5216,16 +5230,7 @@ type TranscriptionModelV4StreamOptions = {
|
|
|
5216
5230
|
/**
|
|
5217
5231
|
* The input audio format for the raw audio chunks.
|
|
5218
5232
|
*/
|
|
5219
|
-
inputAudioFormat:
|
|
5220
|
-
/**
|
|
5221
|
-
* Audio format type, e.g. `audio/pcm`, `audio/pcmu`, or `audio/pcma`.
|
|
5222
|
-
*/
|
|
5223
|
-
type: string;
|
|
5224
|
-
/**
|
|
5225
|
-
* Sample rate in Hz. Only applicable for formats that require a rate.
|
|
5226
|
-
*/
|
|
5227
|
-
rate?: number;
|
|
5228
|
-
};
|
|
5233
|
+
inputAudioFormat: SharedV4AudioFormat;
|
|
5229
5234
|
/**
|
|
5230
5235
|
* Additional provider-specific options that are passed through to the provider.
|
|
5231
5236
|
*
|
|
@@ -5362,7 +5367,7 @@ type TranscriptionModelV4StreamResult = {
|
|
|
5362
5367
|
};
|
|
5363
5368
|
|
|
5364
5369
|
/**
|
|
5365
|
-
* Transcription model specification version
|
|
5370
|
+
* Transcription model specification version 4.
|
|
5366
5371
|
*/
|
|
5367
5372
|
type TranscriptionModelV4 = {
|
|
5368
5373
|
/**
|
|
@@ -6402,16 +6407,7 @@ type RealtimeModelV4SessionConfig = {
|
|
|
6402
6407
|
/**
|
|
6403
6408
|
* Audio format configuration for input audio.
|
|
6404
6409
|
*/
|
|
6405
|
-
inputAudioFormat?:
|
|
6406
|
-
/**
|
|
6407
|
-
* Audio format type (e.g. "audio/pcm", "audio/pcmu", "audio/pcma").
|
|
6408
|
-
*/
|
|
6409
|
-
type: string;
|
|
6410
|
-
/**
|
|
6411
|
-
* Sample rate in Hz. Only applicable for PCM format.
|
|
6412
|
-
*/
|
|
6413
|
-
rate?: number;
|
|
6414
|
-
};
|
|
6410
|
+
inputAudioFormat?: SharedV4AudioFormat;
|
|
6415
6411
|
/**
|
|
6416
6412
|
* Input audio transcription configuration.
|
|
6417
6413
|
*
|
|
@@ -6458,16 +6454,7 @@ type RealtimeModelV4SessionConfig = {
|
|
|
6458
6454
|
/**
|
|
6459
6455
|
* Audio format configuration for output audio.
|
|
6460
6456
|
*/
|
|
6461
|
-
outputAudioFormat?:
|
|
6462
|
-
/**
|
|
6463
|
-
* Audio format type (e.g. "audio/pcm", "audio/pcmu", "audio/pcma").
|
|
6464
|
-
*/
|
|
6465
|
-
type: string;
|
|
6466
|
-
/**
|
|
6467
|
-
* Sample rate in Hz. Only applicable for PCM format.
|
|
6468
|
-
*/
|
|
6469
|
-
rate?: number;
|
|
6470
|
-
};
|
|
6457
|
+
outputAudioFormat?: SharedV4AudioFormat;
|
|
6471
6458
|
/**
|
|
6472
6459
|
* Voice activity detection configuration.
|
|
6473
6460
|
* Set to null or type 'disabled' to turn off VAD (push-to-talk mode).
|
|
@@ -6864,6 +6851,282 @@ interface RealtimeFactoryV4 {
|
|
|
6864
6851
|
getToken(options: RealtimeFactoryV4GetTokenOptions): Promise<RealtimeFactoryV4GetTokenResult>;
|
|
6865
6852
|
}
|
|
6866
6853
|
|
|
6854
|
+
type SpeechTranslationModelV4ProviderOptions = Record<string, JSONObject>;
|
|
6855
|
+
/**
|
|
6856
|
+
* Options for a speech translation model stream call.
|
|
6857
|
+
*/
|
|
6858
|
+
type SpeechTranslationModelV4StreamOptions = {
|
|
6859
|
+
/**
|
|
6860
|
+
* Source audio chunks to transform.
|
|
6861
|
+
*
|
|
6862
|
+
* `Uint8Array` chunks contain raw audio bytes. `string` chunks contain
|
|
6863
|
+
* base64-encoded raw audio bytes.
|
|
6864
|
+
*/
|
|
6865
|
+
audio: ReadableStream<Uint8Array | string>;
|
|
6866
|
+
/**
|
|
6867
|
+
* The input audio format for the raw audio chunks.
|
|
6868
|
+
*/
|
|
6869
|
+
inputAudioFormat: SharedV4AudioFormat;
|
|
6870
|
+
/**
|
|
6871
|
+
* The language to produce output audio and text in, as a BCP-47-style
|
|
6872
|
+
* language tag (e.g. `en`, `es`, `fr-CA`). Supported values are
|
|
6873
|
+
* provider-specific and validated by the provider.
|
|
6874
|
+
*/
|
|
6875
|
+
targetLanguage: string;
|
|
6876
|
+
/**
|
|
6877
|
+
* The language of the source audio, as a BCP-47-style language tag.
|
|
6878
|
+
* When absent, providers should auto-detect the source language.
|
|
6879
|
+
*/
|
|
6880
|
+
sourceLanguage?: string;
|
|
6881
|
+
/**
|
|
6882
|
+
* The desired audio format for output audio chunks.
|
|
6883
|
+
* When absent, the provider default output format is used.
|
|
6884
|
+
*/
|
|
6885
|
+
outputAudioFormat?: SharedV4AudioFormat;
|
|
6886
|
+
/**
|
|
6887
|
+
* Additional provider-specific options that are passed through to the provider.
|
|
6888
|
+
*
|
|
6889
|
+
* The outer record is keyed by the provider name, and the inner record is keyed
|
|
6890
|
+
* by provider-specific option names.
|
|
6891
|
+
*/
|
|
6892
|
+
providerOptions?: SpeechTranslationModelV4ProviderOptions;
|
|
6893
|
+
/**
|
|
6894
|
+
* Abort signal for cancelling the operation.
|
|
6895
|
+
*/
|
|
6896
|
+
abortSignal?: AbortSignal;
|
|
6897
|
+
/**
|
|
6898
|
+
* Additional HTTP headers to be sent with the request.
|
|
6899
|
+
* Only applicable for HTTP/WebSocket-based providers that support headers.
|
|
6900
|
+
*/
|
|
6901
|
+
headers?: Record<string, string | undefined>;
|
|
6902
|
+
/**
|
|
6903
|
+
* When true, providers should include raw provider chunks in the stream.
|
|
6904
|
+
*/
|
|
6905
|
+
includeRawChunks?: boolean;
|
|
6906
|
+
};
|
|
6907
|
+
|
|
6908
|
+
/**
|
|
6909
|
+
* Usage information for a speech translation call.
|
|
6910
|
+
*
|
|
6911
|
+
* All fields are optional because providers report usage with different
|
|
6912
|
+
* granularity (seconds of audio, audio tokens, and/or text tokens).
|
|
6913
|
+
*/
|
|
6914
|
+
type SpeechTranslationModelV4Usage = {
|
|
6915
|
+
/**
|
|
6916
|
+
* Seconds of input audio that were processed, if reported.
|
|
6917
|
+
*/
|
|
6918
|
+
inputAudioSeconds?: number;
|
|
6919
|
+
/**
|
|
6920
|
+
* Number of input audio tokens, if reported.
|
|
6921
|
+
*/
|
|
6922
|
+
inputAudioTokens?: number;
|
|
6923
|
+
/**
|
|
6924
|
+
* Number of output audio tokens, if reported.
|
|
6925
|
+
*/
|
|
6926
|
+
outputAudioTokens?: number;
|
|
6927
|
+
/**
|
|
6928
|
+
* Number of input text tokens, if reported.
|
|
6929
|
+
*/
|
|
6930
|
+
inputTextTokens?: number;
|
|
6931
|
+
/**
|
|
6932
|
+
* Number of output text tokens, if reported.
|
|
6933
|
+
*/
|
|
6934
|
+
outputTextTokens?: number;
|
|
6935
|
+
};
|
|
6936
|
+
|
|
6937
|
+
type SpeechTranslationModelV4StreamPart = {
|
|
6938
|
+
/**
|
|
6939
|
+
* Stream start event with warnings for the call, e.g. unsupported settings.
|
|
6940
|
+
*/
|
|
6941
|
+
type: 'stream-start';
|
|
6942
|
+
warnings: Array<SharedV4Warning>;
|
|
6943
|
+
} | {
|
|
6944
|
+
/**
|
|
6945
|
+
* Output audio chunk.
|
|
6946
|
+
*
|
|
6947
|
+
* `Uint8Array` chunks contain raw audio bytes. `string` chunks contain
|
|
6948
|
+
* base64-encoded raw audio bytes.
|
|
6949
|
+
*/
|
|
6950
|
+
type: 'audio';
|
|
6951
|
+
id?: string;
|
|
6952
|
+
audio: Uint8Array | string;
|
|
6953
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
6954
|
+
} | {
|
|
6955
|
+
/**
|
|
6956
|
+
* Append-only output text delta.
|
|
6957
|
+
*
|
|
6958
|
+
* Output text is append-only: providers stream `output-text-delta`
|
|
6959
|
+
* parts and finalize per-utterance with `output-text-final`. There is
|
|
6960
|
+
* no partial/revision part for output text by design for now.
|
|
6961
|
+
*/
|
|
6962
|
+
type: 'output-text-delta';
|
|
6963
|
+
id?: string;
|
|
6964
|
+
delta: string;
|
|
6965
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
6966
|
+
} | {
|
|
6967
|
+
/**
|
|
6968
|
+
* Final output text for a provider-defined segment or utterance.
|
|
6969
|
+
*
|
|
6970
|
+
* Output text is append-only: providers stream `output-text-delta`
|
|
6971
|
+
* parts and finalize per-utterance with `output-text-final`. There is
|
|
6972
|
+
* no partial/revision part for output text by design for now.
|
|
6973
|
+
*/
|
|
6974
|
+
type: 'output-text-final';
|
|
6975
|
+
id?: string;
|
|
6976
|
+
text: string;
|
|
6977
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
6978
|
+
} | {
|
|
6979
|
+
/**
|
|
6980
|
+
* Append-only source transcript delta.
|
|
6981
|
+
*/
|
|
6982
|
+
type: 'source-transcript-delta';
|
|
6983
|
+
id?: string;
|
|
6984
|
+
delta: string;
|
|
6985
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
6986
|
+
} | {
|
|
6987
|
+
/**
|
|
6988
|
+
* Non-final source transcript text. The text may be revised by later parts.
|
|
6989
|
+
*/
|
|
6990
|
+
type: 'source-transcript-partial';
|
|
6991
|
+
id?: string;
|
|
6992
|
+
text: string;
|
|
6993
|
+
startSecond?: number;
|
|
6994
|
+
endSecond?: number;
|
|
6995
|
+
channelIndex?: number;
|
|
6996
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
6997
|
+
} | {
|
|
6998
|
+
/**
|
|
6999
|
+
* Final source transcript text for a provider-defined segment or utterance.
|
|
7000
|
+
*/
|
|
7001
|
+
type: 'source-transcript-final';
|
|
7002
|
+
id?: string;
|
|
7003
|
+
text: string;
|
|
7004
|
+
startSecond?: number;
|
|
7005
|
+
endSecond?: number;
|
|
7006
|
+
channelIndex?: number;
|
|
7007
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
7008
|
+
} | {
|
|
7009
|
+
/**
|
|
7010
|
+
* Metadata for the response, emitted once available.
|
|
7011
|
+
*/
|
|
7012
|
+
type: 'response-metadata';
|
|
7013
|
+
timestamp?: Date;
|
|
7014
|
+
modelId?: string;
|
|
7015
|
+
headers?: SharedV4Headers;
|
|
7016
|
+
body?: unknown;
|
|
7017
|
+
} | {
|
|
7018
|
+
/**
|
|
7019
|
+
* Metadata that is available after the stream is finished.
|
|
7020
|
+
*/
|
|
7021
|
+
type: 'finish';
|
|
7022
|
+
/**
|
|
7023
|
+
* The final source-language transcript of the input audio.
|
|
7024
|
+
*/
|
|
7025
|
+
sourceText: string;
|
|
7026
|
+
/**
|
|
7027
|
+
* The final output text. May be an empty string for providers that
|
|
7028
|
+
* produce only audio output.
|
|
7029
|
+
*/
|
|
7030
|
+
outputText: string;
|
|
7031
|
+
/**
|
|
7032
|
+
* The duration of the source audio in seconds, if available.
|
|
7033
|
+
*/
|
|
7034
|
+
durationInSeconds?: number;
|
|
7035
|
+
/**
|
|
7036
|
+
* Usage information for the call, if reported by the provider.
|
|
7037
|
+
*/
|
|
7038
|
+
usage?: SpeechTranslationModelV4Usage;
|
|
7039
|
+
/**
|
|
7040
|
+
* Additional provider-specific metadata.
|
|
7041
|
+
*/
|
|
7042
|
+
providerMetadata?: Record<string, JSONObject>;
|
|
7043
|
+
} | {
|
|
7044
|
+
/**
|
|
7045
|
+
* Raw provider chunks if enabled.
|
|
7046
|
+
*/
|
|
7047
|
+
type: 'raw';
|
|
7048
|
+
rawValue: unknown;
|
|
7049
|
+
} | {
|
|
7050
|
+
/**
|
|
7051
|
+
* Error parts are streamed, allowing for multiple errors.
|
|
7052
|
+
*/
|
|
7053
|
+
type: 'error';
|
|
7054
|
+
error: unknown;
|
|
7055
|
+
};
|
|
7056
|
+
|
|
7057
|
+
/**
|
|
7058
|
+
* The result of a speech translation model doStream call.
|
|
7059
|
+
*/
|
|
7060
|
+
type SpeechTranslationModelV4StreamResult = {
|
|
7061
|
+
/**
|
|
7062
|
+
* The stream.
|
|
7063
|
+
*/
|
|
7064
|
+
stream: ReadableStream<SpeechTranslationModelV4StreamPart>;
|
|
7065
|
+
/**
|
|
7066
|
+
* Optional request information for telemetry and debugging purposes.
|
|
7067
|
+
*/
|
|
7068
|
+
request?: {
|
|
7069
|
+
/**
|
|
7070
|
+
* Request body or setup payload that was sent to the provider API.
|
|
7071
|
+
*/
|
|
7072
|
+
body?: unknown;
|
|
7073
|
+
};
|
|
7074
|
+
/**
|
|
7075
|
+
* Optional response data.
|
|
7076
|
+
*/
|
|
7077
|
+
response?: {
|
|
7078
|
+
/**
|
|
7079
|
+
* Timestamp for the start of the streamed response.
|
|
7080
|
+
*/
|
|
7081
|
+
timestamp?: Date;
|
|
7082
|
+
/**
|
|
7083
|
+
* The ID of the response model that was used to generate the response.
|
|
7084
|
+
*/
|
|
7085
|
+
modelId?: string;
|
|
7086
|
+
/**
|
|
7087
|
+
* Response headers.
|
|
7088
|
+
*/
|
|
7089
|
+
headers?: SharedV4Headers;
|
|
7090
|
+
/**
|
|
7091
|
+
* Response body.
|
|
7092
|
+
*/
|
|
7093
|
+
body?: unknown;
|
|
7094
|
+
};
|
|
7095
|
+
};
|
|
7096
|
+
|
|
7097
|
+
/**
|
|
7098
|
+
* Speech translation model specification version 4.
|
|
7099
|
+
*
|
|
7100
|
+
* Speech translation is a streaming-only modality: models translate live
|
|
7101
|
+
* source audio into target-language audio and text.
|
|
7102
|
+
*
|
|
7103
|
+
* Experimental: the speech translation model contract may change in patch
|
|
7104
|
+
* releases while the functions built on it are experimental. All types of
|
|
7105
|
+
* this modality are exported with `Experimental_` prefixes for this reason.
|
|
7106
|
+
*/
|
|
7107
|
+
type SpeechTranslationModelV4 = {
|
|
7108
|
+
/**
|
|
7109
|
+
* The speech translation model must specify which speech translation model
|
|
7110
|
+
* interface version it implements. This will allow us to evolve the
|
|
7111
|
+
* speech translation model interface and retain backwards compatibility.
|
|
7112
|
+
* The different implementation versions can be handled as a discriminated
|
|
7113
|
+
* union on our side.
|
|
7114
|
+
*/
|
|
7115
|
+
readonly specificationVersion: 'v4';
|
|
7116
|
+
/**
|
|
7117
|
+
* Name of the provider for logging purposes.
|
|
7118
|
+
*/
|
|
7119
|
+
readonly provider: string;
|
|
7120
|
+
/**
|
|
7121
|
+
* Provider-specific model ID for logging purposes.
|
|
7122
|
+
*/
|
|
7123
|
+
readonly modelId: string;
|
|
7124
|
+
/**
|
|
7125
|
+
* Streams a speech translation for live audio.
|
|
7126
|
+
*/
|
|
7127
|
+
doStream(options: SpeechTranslationModelV4StreamOptions): PromiseLike<SpeechTranslationModelV4StreamResult>;
|
|
7128
|
+
};
|
|
7129
|
+
|
|
6867
7130
|
/**
|
|
6868
7131
|
* A video or image file that can be used for video editing or image-to-video generation.
|
|
6869
7132
|
* Supports both image inputs (for image-to-video) and video inputs (for editing).
|
|
@@ -6890,6 +7153,12 @@ type VideoModelV4File = {
|
|
|
6890
7153
|
* The URL of the video or image file.
|
|
6891
7154
|
*/
|
|
6892
7155
|
url: string;
|
|
7156
|
+
/**
|
|
7157
|
+
* The media type of the referenced file, when known.
|
|
7158
|
+
* Video types: 'video/mp4', 'video/webm', 'video/quicktime'
|
|
7159
|
+
* Image types: 'image/png', 'image/jpeg', 'image/webp'
|
|
7160
|
+
*/
|
|
7161
|
+
mediaType?: string;
|
|
6893
7162
|
/**
|
|
6894
7163
|
* Optional provider-specific metadata for the file part.
|
|
6895
7164
|
*/
|
|
@@ -6969,7 +7238,11 @@ type VideoModelV4CallOptions = {
|
|
|
6969
7238
|
*/
|
|
6970
7239
|
frameImages: Array<VideoModelV4FrameImage> | undefined;
|
|
6971
7240
|
/**
|
|
6972
|
-
* Reference
|
|
7241
|
+
* Reference inputs for reference-to-video generation.
|
|
7242
|
+
*
|
|
7243
|
+
* Each entry is an image or video file. Providers route each reference by
|
|
7244
|
+
* its media type (image vs. video) and warn when a reference kind is
|
|
7245
|
+
* unsupported.
|
|
6973
7246
|
*/
|
|
6974
7247
|
inputReferences: Array<VideoModelV4File> | undefined;
|
|
6975
7248
|
/**
|
|
@@ -7085,7 +7358,7 @@ type GetMaxVideosPerCallFunction$1 = (options: {
|
|
|
7085
7358
|
modelId: string;
|
|
7086
7359
|
}) => PromiseLike<number | undefined> | number | undefined;
|
|
7087
7360
|
/**
|
|
7088
|
-
* Video generation model specification version
|
|
7361
|
+
* Video generation model specification version 4.
|
|
7089
7362
|
*/
|
|
7090
7363
|
type VideoModelV4 = {
|
|
7091
7364
|
/**
|
|
@@ -7225,7 +7498,11 @@ type VideoModelV3CallOptions = {
|
|
|
7225
7498
|
*/
|
|
7226
7499
|
frameImages: Array<VideoModelV3FrameImage> | undefined;
|
|
7227
7500
|
/**
|
|
7228
|
-
* Reference
|
|
7501
|
+
* Reference inputs for reference-to-video generation.
|
|
7502
|
+
*
|
|
7503
|
+
* Each entry is an image or video file. Providers route each reference by
|
|
7504
|
+
* its media type (image vs. video) and warn when a reference kind is
|
|
7505
|
+
* unsupported.
|
|
7229
7506
|
*/
|
|
7230
7507
|
inputReferences: Array<VideoModelV3File> | undefined;
|
|
7231
7508
|
/**
|
|
@@ -7371,4 +7648,4 @@ type VideoModelV3 = {
|
|
|
7371
7648
|
}>;
|
|
7372
7649
|
};
|
|
7373
7650
|
|
|
7374
|
-
export { AISDKError, APICallError, type EmbeddingModelV2, type EmbeddingModelV2Embedding, type EmbeddingModelV3, type EmbeddingModelV3CallOptions, type EmbeddingModelV3Embedding, type EmbeddingModelV3Middleware, type EmbeddingModelV3Result, type EmbeddingModelV4, type EmbeddingModelV4CallOptions, type EmbeddingModelV4Embedding, type EmbeddingModelV4Middleware, type EmbeddingModelV4Result, EmptyResponseBodyError, type RealtimeFactoryV4 as Experimental_RealtimeFactoryV4, type RealtimeFactoryV4GetTokenOptions as Experimental_RealtimeFactoryV4GetTokenOptions, type RealtimeFactoryV4GetTokenResult as Experimental_RealtimeFactoryV4GetTokenResult, type RealtimeModelV4 as Experimental_RealtimeModelV4, type RealtimeModelV4AudioMessage as Experimental_RealtimeModelV4AudioMessage, type RealtimeModelV4ClientEvent as Experimental_RealtimeModelV4ClientEvent, type RealtimeModelV4ClientSecretOptions as Experimental_RealtimeModelV4ClientSecretOptions, type RealtimeModelV4ClientSecretResult as Experimental_RealtimeModelV4ClientSecretResult, type RealtimeModelV4ConversationItem as Experimental_RealtimeModelV4ConversationItem, type RealtimeModelV4FunctionCallOutput as Experimental_RealtimeModelV4FunctionCallOutput, type RealtimeModelV4ServerEvent as Experimental_RealtimeModelV4ServerEvent, type RealtimeModelV4SessionConfig as Experimental_RealtimeModelV4SessionConfig, type RealtimeModelV4TextMessage as Experimental_RealtimeModelV4TextMessage, type RealtimeModelV4ToolDefinition as Experimental_RealtimeModelV4ToolDefinition, type TranscriptionModelV4StreamOptions as Experimental_TranscriptionModelV4StreamOptions, type TranscriptionModelV4StreamPart as Experimental_TranscriptionModelV4StreamPart, type TranscriptionModelV4StreamResult as Experimental_TranscriptionModelV4StreamResult, type VideoModelV3 as Experimental_VideoModelV3, type VideoModelV3CallOptions as Experimental_VideoModelV3CallOptions, type VideoModelV3File as Experimental_VideoModelV3File, type VideoModelV3FrameImage as Experimental_VideoModelV3FrameImage, type VideoModelV3FrameType as Experimental_VideoModelV3FrameType, type VideoModelV3VideoData as Experimental_VideoModelV3VideoData, type VideoModelV4 as Experimental_VideoModelV4, type VideoModelV4CallOptions as Experimental_VideoModelV4CallOptions, type VideoModelV4File as Experimental_VideoModelV4File, type VideoModelV4FrameImage as Experimental_VideoModelV4FrameImage, type VideoModelV4FrameType as Experimental_VideoModelV4FrameType, type VideoModelV4Result as Experimental_VideoModelV4Result, type VideoModelV4VideoData as Experimental_VideoModelV4VideoData, type FilesV4, type FilesV4UploadFileCallOptions, type FilesV4UploadFileResult, type ImageModelV2, type ImageModelV2CallOptions, type ImageModelV2CallWarning, type ImageModelV2ProviderMetadata, type ImageModelV3, type ImageModelV3CallOptions, type ImageModelV3File, type ImageModelV3Middleware, type ImageModelV3ProviderMetadata, type ImageModelV3Usage, type ImageModelV4, type ImageModelV4CallOptions, type ImageModelV4File, type ImageModelV4Middleware, type ImageModelV4ProviderMetadata, type ImageModelV4Result, type ImageModelV4Usage, InvalidArgumentError, InvalidPromptError, InvalidResponseDataError, type JSONArray, type JSONObject, JSONParseError, type JSONValue, type LanguageModelV2, type LanguageModelV2CallOptions, type LanguageModelV2CallWarning, type LanguageModelV2Content, type LanguageModelV2DataContent, type LanguageModelV2File, type LanguageModelV2FilePart, type LanguageModelV2FinishReason, type LanguageModelV2FunctionTool, type LanguageModelV2Message, type LanguageModelV2Middleware, type LanguageModelV2Prompt, type LanguageModelV2ProviderDefinedTool, type LanguageModelV2Reasoning, type LanguageModelV2ReasoningPart, type LanguageModelV2ResponseMetadata, type LanguageModelV2Source, type LanguageModelV2StreamPart, type LanguageModelV2Text, type LanguageModelV2TextPart, type LanguageModelV2ToolCall, type LanguageModelV2ToolCallPart, type LanguageModelV2ToolChoice, type LanguageModelV2ToolResultOutput, type LanguageModelV2ToolResultPart, type LanguageModelV2Usage, type LanguageModelV3, type LanguageModelV3CallOptions, type LanguageModelV3Content, type LanguageModelV3DataContent, type LanguageModelV3File, type LanguageModelV3FilePart, type LanguageModelV3FinishReason, type LanguageModelV3FunctionTool, type LanguageModelV3GenerateResult, type LanguageModelV3Message, type LanguageModelV3Middleware, type LanguageModelV3Prompt, type LanguageModelV3ProviderTool, type LanguageModelV3Reasoning, type LanguageModelV3ReasoningPart, type LanguageModelV3ResponseMetadata, type LanguageModelV3Source, type LanguageModelV3StreamPart, type LanguageModelV3StreamResult, type LanguageModelV3Text, type LanguageModelV3TextPart, type LanguageModelV3ToolApprovalRequest, type LanguageModelV3ToolApprovalResponsePart, type LanguageModelV3ToolCall, type LanguageModelV3ToolCallPart, type LanguageModelV3ToolChoice, type LanguageModelV3ToolResult, type LanguageModelV3ToolResultOutput, type LanguageModelV3ToolResultPart, type LanguageModelV3Usage, type LanguageModelV4, type LanguageModelV4CallOptions, type LanguageModelV4Content, type LanguageModelV4CustomContent, type LanguageModelV4CustomPart, type LanguageModelV4File, type LanguageModelV4FilePart, type LanguageModelV4FinishReason, type LanguageModelV4FunctionTool, type LanguageModelV4GenerateResult, type LanguageModelV4Message, type LanguageModelV4Middleware, type LanguageModelV4Prompt, type LanguageModelV4ProviderTool, type LanguageModelV4Reasoning, type LanguageModelV4ReasoningFile, type LanguageModelV4ReasoningFilePart, type LanguageModelV4ReasoningPart, type LanguageModelV4ResponseMetadata, type LanguageModelV4Source, type LanguageModelV4StreamPart, type LanguageModelV4StreamResult, type LanguageModelV4Text, type LanguageModelV4TextPart, type LanguageModelV4ToolApprovalRequest, type LanguageModelV4ToolApprovalResponsePart, type LanguageModelV4ToolCall, type LanguageModelV4ToolCallPart, type LanguageModelV4ToolChoice, type LanguageModelV4ToolResult, type LanguageModelV4ToolResultOutput, type LanguageModelV4ToolResultPart, type LanguageModelV4Usage, LoadAPIKeyError, LoadSettingError, NoContentGeneratedError, NoSuchModelError, NoSuchProviderReferenceError, type ProviderV2, type ProviderV3, type ProviderV4, type RerankingModelV3, type RerankingModelV3CallOptions, type RerankingModelV4, type RerankingModelV4CallOptions, type RerankingModelV4Result, type SharedV2Headers, type SharedV2ProviderMetadata, type SharedV2ProviderOptions, type SharedV3Headers, type SharedV3ProviderMetadata, type SharedV3ProviderOptions, type SharedV3Warning, type SharedV4FileData, type SharedV4FileDataData, type SharedV4FileDataReference, type SharedV4FileDataText, type SharedV4FileDataUrl, type SharedV4Headers, type SharedV4ProviderMetadata, type SharedV4ProviderOptions, type SharedV4ProviderReference, type SharedV4Warning, type SkillsV4, type SkillsV4File, type SkillsV4UploadSkillCallOptions, type SkillsV4UploadSkillResult, type SpeechModelV2, type SpeechModelV2CallOptions, type SpeechModelV2CallWarning, type SpeechModelV3, type SpeechModelV3CallOptions, type SpeechModelV4, type SpeechModelV4CallOptions, type SpeechModelV4Result, TooManyEmbeddingValuesForCallError, type TranscriptionModelV2, type TranscriptionModelV2CallOptions, type TranscriptionModelV2CallWarning, type TranscriptionModelV3, type TranscriptionModelV3CallOptions, type TranscriptionModelV4, type TranscriptionModelV4CallOptions, type TranscriptionModelV4Result, type TypeValidationContext, TypeValidationError, UnsupportedFunctionalityError, getErrorMessage, isJSONArray, isJSONObject, isJSONValue };
|
|
7651
|
+
export { AISDKError, APICallError, type EmbeddingModelV2, type EmbeddingModelV2Embedding, type EmbeddingModelV3, type EmbeddingModelV3CallOptions, type EmbeddingModelV3Embedding, type EmbeddingModelV3Middleware, type EmbeddingModelV3Result, type EmbeddingModelV4, type EmbeddingModelV4CallOptions, type EmbeddingModelV4Embedding, type EmbeddingModelV4Middleware, type EmbeddingModelV4Result, EmptyResponseBodyError, type RealtimeFactoryV4 as Experimental_RealtimeFactoryV4, type RealtimeFactoryV4GetTokenOptions as Experimental_RealtimeFactoryV4GetTokenOptions, type RealtimeFactoryV4GetTokenResult as Experimental_RealtimeFactoryV4GetTokenResult, type RealtimeModelV4 as Experimental_RealtimeModelV4, type RealtimeModelV4AudioMessage as Experimental_RealtimeModelV4AudioMessage, type RealtimeModelV4ClientEvent as Experimental_RealtimeModelV4ClientEvent, type RealtimeModelV4ClientSecretOptions as Experimental_RealtimeModelV4ClientSecretOptions, type RealtimeModelV4ClientSecretResult as Experimental_RealtimeModelV4ClientSecretResult, type RealtimeModelV4ConversationItem as Experimental_RealtimeModelV4ConversationItem, type RealtimeModelV4FunctionCallOutput as Experimental_RealtimeModelV4FunctionCallOutput, type RealtimeModelV4ServerEvent as Experimental_RealtimeModelV4ServerEvent, type RealtimeModelV4SessionConfig as Experimental_RealtimeModelV4SessionConfig, type RealtimeModelV4TextMessage as Experimental_RealtimeModelV4TextMessage, type RealtimeModelV4ToolDefinition as Experimental_RealtimeModelV4ToolDefinition, type SpeechTranslationModelV4 as Experimental_SpeechTranslationModelV4, type SpeechTranslationModelV4StreamOptions as Experimental_SpeechTranslationModelV4StreamOptions, type SpeechTranslationModelV4StreamPart as Experimental_SpeechTranslationModelV4StreamPart, type SpeechTranslationModelV4StreamResult as Experimental_SpeechTranslationModelV4StreamResult, type SpeechTranslationModelV4Usage as Experimental_SpeechTranslationModelV4Usage, type TranscriptionModelV4StreamOptions as Experimental_TranscriptionModelV4StreamOptions, type TranscriptionModelV4StreamPart as Experimental_TranscriptionModelV4StreamPart, type TranscriptionModelV4StreamResult as Experimental_TranscriptionModelV4StreamResult, type VideoModelV3 as Experimental_VideoModelV3, type VideoModelV3CallOptions as Experimental_VideoModelV3CallOptions, type VideoModelV3File as Experimental_VideoModelV3File, type VideoModelV3FrameImage as Experimental_VideoModelV3FrameImage, type VideoModelV3FrameType as Experimental_VideoModelV3FrameType, type VideoModelV3VideoData as Experimental_VideoModelV3VideoData, type VideoModelV4 as Experimental_VideoModelV4, type VideoModelV4CallOptions as Experimental_VideoModelV4CallOptions, type VideoModelV4File as Experimental_VideoModelV4File, type VideoModelV4FrameImage as Experimental_VideoModelV4FrameImage, type VideoModelV4FrameType as Experimental_VideoModelV4FrameType, type VideoModelV4Result as Experimental_VideoModelV4Result, type VideoModelV4VideoData as Experimental_VideoModelV4VideoData, type FilesV4, type FilesV4UploadFileCallOptions, type FilesV4UploadFileResult, type ImageModelV2, type ImageModelV2CallOptions, type ImageModelV2CallWarning, type ImageModelV2ProviderMetadata, type ImageModelV3, type ImageModelV3CallOptions, type ImageModelV3File, type ImageModelV3Middleware, type ImageModelV3ProviderMetadata, type ImageModelV3Usage, type ImageModelV4, type ImageModelV4CallOptions, type ImageModelV4File, type ImageModelV4Middleware, type ImageModelV4ProviderMetadata, type ImageModelV4Result, type ImageModelV4Usage, InvalidArgumentError, InvalidPromptError, InvalidResponseDataError, type JSONArray, type JSONObject, JSONParseError, type JSONValue, type LanguageModelV2, type LanguageModelV2CallOptions, type LanguageModelV2CallWarning, type LanguageModelV2Content, type LanguageModelV2DataContent, type LanguageModelV2File, type LanguageModelV2FilePart, type LanguageModelV2FinishReason, type LanguageModelV2FunctionTool, type LanguageModelV2Message, type LanguageModelV2Middleware, type LanguageModelV2Prompt, type LanguageModelV2ProviderDefinedTool, type LanguageModelV2Reasoning, type LanguageModelV2ReasoningPart, type LanguageModelV2ResponseMetadata, type LanguageModelV2Source, type LanguageModelV2StreamPart, type LanguageModelV2Text, type LanguageModelV2TextPart, type LanguageModelV2ToolCall, type LanguageModelV2ToolCallPart, type LanguageModelV2ToolChoice, type LanguageModelV2ToolResultOutput, type LanguageModelV2ToolResultPart, type LanguageModelV2Usage, type LanguageModelV3, type LanguageModelV3CallOptions, type LanguageModelV3Content, type LanguageModelV3DataContent, type LanguageModelV3File, type LanguageModelV3FilePart, type LanguageModelV3FinishReason, type LanguageModelV3FunctionTool, type LanguageModelV3GenerateResult, type LanguageModelV3Message, type LanguageModelV3Middleware, type LanguageModelV3Prompt, type LanguageModelV3ProviderTool, type LanguageModelV3Reasoning, type LanguageModelV3ReasoningPart, type LanguageModelV3ResponseMetadata, type LanguageModelV3Source, type LanguageModelV3StreamPart, type LanguageModelV3StreamResult, type LanguageModelV3Text, type LanguageModelV3TextPart, type LanguageModelV3ToolApprovalRequest, type LanguageModelV3ToolApprovalResponsePart, type LanguageModelV3ToolCall, type LanguageModelV3ToolCallPart, type LanguageModelV3ToolChoice, type LanguageModelV3ToolResult, type LanguageModelV3ToolResultOutput, type LanguageModelV3ToolResultPart, type LanguageModelV3Usage, type LanguageModelV4, type LanguageModelV4CallOptions, type LanguageModelV4Content, type LanguageModelV4CustomContent, type LanguageModelV4CustomPart, type LanguageModelV4File, type LanguageModelV4FilePart, type LanguageModelV4FinishReason, type LanguageModelV4FunctionTool, type LanguageModelV4GenerateResult, type LanguageModelV4Message, type LanguageModelV4Middleware, type LanguageModelV4Prompt, type LanguageModelV4ProviderTool, type LanguageModelV4Reasoning, type LanguageModelV4ReasoningFile, type LanguageModelV4ReasoningFilePart, type LanguageModelV4ReasoningPart, type LanguageModelV4ResponseMetadata, type LanguageModelV4Source, type LanguageModelV4StreamPart, type LanguageModelV4StreamResult, type LanguageModelV4Text, type LanguageModelV4TextPart, type LanguageModelV4ToolApprovalRequest, type LanguageModelV4ToolApprovalResponsePart, type LanguageModelV4ToolCall, type LanguageModelV4ToolCallPart, type LanguageModelV4ToolChoice, type LanguageModelV4ToolResult, type LanguageModelV4ToolResultOutput, type LanguageModelV4ToolResultPart, type LanguageModelV4Usage, LoadAPIKeyError, LoadSettingError, NoContentGeneratedError, NoSuchModelError, NoSuchProviderReferenceError, type ProviderV2, type ProviderV3, type ProviderV4, type RerankingModelV3, type RerankingModelV3CallOptions, type RerankingModelV4, type RerankingModelV4CallOptions, type RerankingModelV4Result, type SharedV2Headers, type SharedV2ProviderMetadata, type SharedV2ProviderOptions, type SharedV3Headers, type SharedV3ProviderMetadata, type SharedV3ProviderOptions, type SharedV3Warning, type SharedV4AudioFormat, type SharedV4FileData, type SharedV4FileDataData, type SharedV4FileDataReference, type SharedV4FileDataText, type SharedV4FileDataUrl, type SharedV4Headers, type SharedV4ProviderMetadata, type SharedV4ProviderOptions, type SharedV4ProviderReference, type SharedV4Warning, type SkillsV4, type SkillsV4File, type SkillsV4UploadSkillCallOptions, type SkillsV4UploadSkillResult, type SpeechModelV2, type SpeechModelV2CallOptions, type SpeechModelV2CallWarning, type SpeechModelV3, type SpeechModelV3CallOptions, type SpeechModelV4, type SpeechModelV4CallOptions, type SpeechModelV4Result, TooManyEmbeddingValuesForCallError, type TranscriptionModelV2, type TranscriptionModelV2CallOptions, type TranscriptionModelV2CallWarning, type TranscriptionModelV3, type TranscriptionModelV3CallOptions, type TranscriptionModelV4, type TranscriptionModelV4CallOptions, type TranscriptionModelV4Result, type TypeValidationContext, TypeValidationError, UnsupportedFunctionalityError, getErrorMessage, isJSONArray, isJSONObject, isJSONValue };
|
package/package.json
CHANGED
|
@@ -6,7 +6,7 @@ type GetMaxImagesPerCallFunction = (options: {
|
|
|
6
6
|
}) => PromiseLike<number | undefined> | number | undefined;
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
|
-
* Image generation model specification version
|
|
9
|
+
* Image generation model specification version 4.
|
|
10
10
|
*/
|
|
11
11
|
export type ImageModelV4 = {
|
|
12
12
|
/**
|
package/src/index.ts
CHANGED
|
@@ -13,6 +13,7 @@ export * from './reranking-model/index';
|
|
|
13
13
|
export * from './shared/index';
|
|
14
14
|
export * from './skills/index';
|
|
15
15
|
export * from './speech-model/index';
|
|
16
|
+
export * from './speech-translation-model/index';
|
|
16
17
|
export * from './transcription-model/index';
|
|
17
18
|
export * from './video-model/index';
|
|
18
19
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { SharedV4AudioFormat } from '../../shared';
|
|
1
2
|
import type { RealtimeModelV4ToolDefinition } from './realtime-model-v4-tool-definition';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -23,17 +24,7 @@ export type RealtimeModelV4SessionConfig = {
|
|
|
23
24
|
/**
|
|
24
25
|
* Audio format configuration for input audio.
|
|
25
26
|
*/
|
|
26
|
-
inputAudioFormat?:
|
|
27
|
-
/**
|
|
28
|
-
* Audio format type (e.g. "audio/pcm", "audio/pcmu", "audio/pcma").
|
|
29
|
-
*/
|
|
30
|
-
type: string;
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Sample rate in Hz. Only applicable for PCM format.
|
|
34
|
-
*/
|
|
35
|
-
rate?: number;
|
|
36
|
-
};
|
|
27
|
+
inputAudioFormat?: SharedV4AudioFormat;
|
|
37
28
|
|
|
38
29
|
/**
|
|
39
30
|
* Input audio transcription configuration.
|
|
@@ -87,17 +78,7 @@ export type RealtimeModelV4SessionConfig = {
|
|
|
87
78
|
/**
|
|
88
79
|
* Audio format configuration for output audio.
|
|
89
80
|
*/
|
|
90
|
-
outputAudioFormat?:
|
|
91
|
-
/**
|
|
92
|
-
* Audio format type (e.g. "audio/pcm", "audio/pcmu", "audio/pcma").
|
|
93
|
-
*/
|
|
94
|
-
type: string;
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* Sample rate in Hz. Only applicable for PCM format.
|
|
98
|
-
*/
|
|
99
|
-
rate?: number;
|
|
100
|
-
};
|
|
81
|
+
outputAudioFormat?: SharedV4AudioFormat;
|
|
101
82
|
|
|
102
83
|
/**
|
|
103
84
|
* Voice activity detection configuration.
|
|
@@ -2,7 +2,7 @@ import type { RerankingModelV4CallOptions } from './reranking-model-v4-call-opti
|
|
|
2
2
|
import type { RerankingModelV4Result } from './reranking-model-v4-result';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Specification for a reranking model that implements the reranking model interface version
|
|
5
|
+
* Specification for a reranking model that implements the reranking model interface version 4.
|
|
6
6
|
*/
|
|
7
7
|
export type RerankingModelV4 = {
|
|
8
8
|
/**
|
package/src/shared/v4/index.ts
CHANGED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Audio format configuration shared by provider V4 model specifications.
|
|
3
|
+
*/
|
|
4
|
+
export type SharedV4AudioFormat = {
|
|
5
|
+
/**
|
|
6
|
+
* Audio format type, e.g. `audio/pcm`, `audio/pcmu`, or `audio/pcma`.
|
|
7
|
+
*/
|
|
8
|
+
type: string;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Sample rate in Hz. Only applicable for formats that require a rate.
|
|
12
|
+
*/
|
|
13
|
+
rate?: number;
|
|
14
|
+
};
|
|
@@ -2,7 +2,7 @@ import type { SpeechModelV4CallOptions } from './speech-model-v4-call-options';
|
|
|
2
2
|
import type { SpeechModelV4Result } from './speech-model-v4-result';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* Speech model specification version
|
|
5
|
+
* Speech model specification version 4.
|
|
6
6
|
*/
|
|
7
7
|
export type SpeechModelV4 = {
|
|
8
8
|
/**
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './v4/index';
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export type { SpeechTranslationModelV4 as Experimental_SpeechTranslationModelV4 } from './speech-translation-model-v4';
|
|
2
|
+
export type { SpeechTranslationModelV4StreamOptions as Experimental_SpeechTranslationModelV4StreamOptions } from './speech-translation-model-v4-stream-options';
|
|
3
|
+
export type { SpeechTranslationModelV4StreamPart as Experimental_SpeechTranslationModelV4StreamPart } from './speech-translation-model-v4-stream-part';
|
|
4
|
+
export type { SpeechTranslationModelV4StreamResult as Experimental_SpeechTranslationModelV4StreamResult } from './speech-translation-model-v4-stream-result';
|
|
5
|
+
export type { SpeechTranslationModelV4Usage as Experimental_SpeechTranslationModelV4Usage } from './speech-translation-model-v4-usage';
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { JSONObject } from '../../json-value/json-value';
|
|
2
|
+
import type { SharedV4AudioFormat } from '../../shared';
|
|
3
|
+
|
|
4
|
+
type SpeechTranslationModelV4ProviderOptions = Record<string, JSONObject>;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Options for a speech translation model stream call.
|
|
8
|
+
*/
|
|
9
|
+
export type SpeechTranslationModelV4StreamOptions = {
|
|
10
|
+
/**
|
|
11
|
+
* Source audio chunks to transform.
|
|
12
|
+
*
|
|
13
|
+
* `Uint8Array` chunks contain raw audio bytes. `string` chunks contain
|
|
14
|
+
* base64-encoded raw audio bytes.
|
|
15
|
+
*/
|
|
16
|
+
audio: ReadableStream<Uint8Array | string>;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The input audio format for the raw audio chunks.
|
|
20
|
+
*/
|
|
21
|
+
inputAudioFormat: SharedV4AudioFormat;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The language to produce output audio and text in, as a BCP-47-style
|
|
25
|
+
* language tag (e.g. `en`, `es`, `fr-CA`). Supported values are
|
|
26
|
+
* provider-specific and validated by the provider.
|
|
27
|
+
*/
|
|
28
|
+
targetLanguage: string;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The language of the source audio, as a BCP-47-style language tag.
|
|
32
|
+
* When absent, providers should auto-detect the source language.
|
|
33
|
+
*/
|
|
34
|
+
sourceLanguage?: string;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The desired audio format for output audio chunks.
|
|
38
|
+
* When absent, the provider default output format is used.
|
|
39
|
+
*/
|
|
40
|
+
outputAudioFormat?: SharedV4AudioFormat;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Additional provider-specific options that are passed through to the provider.
|
|
44
|
+
*
|
|
45
|
+
* The outer record is keyed by the provider name, and the inner record is keyed
|
|
46
|
+
* by provider-specific option names.
|
|
47
|
+
*/
|
|
48
|
+
providerOptions?: SpeechTranslationModelV4ProviderOptions;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Abort signal for cancelling the operation.
|
|
52
|
+
*/
|
|
53
|
+
abortSignal?: AbortSignal;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Additional HTTP headers to be sent with the request.
|
|
57
|
+
* Only applicable for HTTP/WebSocket-based providers that support headers.
|
|
58
|
+
*/
|
|
59
|
+
headers?: Record<string, string | undefined>;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* When true, providers should include raw provider chunks in the stream.
|
|
63
|
+
*/
|
|
64
|
+
includeRawChunks?: boolean;
|
|
65
|
+
};
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import type { JSONObject } from '../../json-value/json-value';
|
|
2
|
+
import type { SharedV4Headers } from '../../shared';
|
|
3
|
+
import type { SharedV4ProviderMetadata } from '../../shared/v4/shared-v4-provider-metadata';
|
|
4
|
+
import type { SharedV4Warning } from '../../shared/v4/shared-v4-warning';
|
|
5
|
+
import type { SpeechTranslationModelV4Usage } from './speech-translation-model-v4-usage';
|
|
6
|
+
|
|
7
|
+
export type SpeechTranslationModelV4StreamPart =
|
|
8
|
+
| {
|
|
9
|
+
/**
|
|
10
|
+
* Stream start event with warnings for the call, e.g. unsupported settings.
|
|
11
|
+
*/
|
|
12
|
+
type: 'stream-start';
|
|
13
|
+
warnings: Array<SharedV4Warning>;
|
|
14
|
+
}
|
|
15
|
+
| {
|
|
16
|
+
/**
|
|
17
|
+
* Output audio chunk.
|
|
18
|
+
*
|
|
19
|
+
* `Uint8Array` chunks contain raw audio bytes. `string` chunks contain
|
|
20
|
+
* base64-encoded raw audio bytes.
|
|
21
|
+
*/
|
|
22
|
+
type: 'audio';
|
|
23
|
+
id?: string;
|
|
24
|
+
audio: Uint8Array | string;
|
|
25
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
26
|
+
}
|
|
27
|
+
| {
|
|
28
|
+
/**
|
|
29
|
+
* Append-only output text delta.
|
|
30
|
+
*
|
|
31
|
+
* Output text is append-only: providers stream `output-text-delta`
|
|
32
|
+
* parts and finalize per-utterance with `output-text-final`. There is
|
|
33
|
+
* no partial/revision part for output text by design for now.
|
|
34
|
+
*/
|
|
35
|
+
type: 'output-text-delta';
|
|
36
|
+
id?: string;
|
|
37
|
+
delta: string;
|
|
38
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
39
|
+
}
|
|
40
|
+
| {
|
|
41
|
+
/**
|
|
42
|
+
* Final output text for a provider-defined segment or utterance.
|
|
43
|
+
*
|
|
44
|
+
* Output text is append-only: providers stream `output-text-delta`
|
|
45
|
+
* parts and finalize per-utterance with `output-text-final`. There is
|
|
46
|
+
* no partial/revision part for output text by design for now.
|
|
47
|
+
*/
|
|
48
|
+
type: 'output-text-final';
|
|
49
|
+
id?: string;
|
|
50
|
+
text: string;
|
|
51
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
52
|
+
}
|
|
53
|
+
| {
|
|
54
|
+
/**
|
|
55
|
+
* Append-only source transcript delta.
|
|
56
|
+
*/
|
|
57
|
+
type: 'source-transcript-delta';
|
|
58
|
+
id?: string;
|
|
59
|
+
delta: string;
|
|
60
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
61
|
+
}
|
|
62
|
+
| {
|
|
63
|
+
/**
|
|
64
|
+
* Non-final source transcript text. The text may be revised by later parts.
|
|
65
|
+
*/
|
|
66
|
+
type: 'source-transcript-partial';
|
|
67
|
+
id?: string;
|
|
68
|
+
text: string;
|
|
69
|
+
startSecond?: number;
|
|
70
|
+
endSecond?: number;
|
|
71
|
+
channelIndex?: number;
|
|
72
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
73
|
+
}
|
|
74
|
+
| {
|
|
75
|
+
/**
|
|
76
|
+
* Final source transcript text for a provider-defined segment or utterance.
|
|
77
|
+
*/
|
|
78
|
+
type: 'source-transcript-final';
|
|
79
|
+
id?: string;
|
|
80
|
+
text: string;
|
|
81
|
+
startSecond?: number;
|
|
82
|
+
endSecond?: number;
|
|
83
|
+
channelIndex?: number;
|
|
84
|
+
providerMetadata?: SharedV4ProviderMetadata;
|
|
85
|
+
}
|
|
86
|
+
| {
|
|
87
|
+
/**
|
|
88
|
+
* Metadata for the response, emitted once available.
|
|
89
|
+
*/
|
|
90
|
+
type: 'response-metadata';
|
|
91
|
+
timestamp?: Date;
|
|
92
|
+
modelId?: string;
|
|
93
|
+
headers?: SharedV4Headers;
|
|
94
|
+
body?: unknown;
|
|
95
|
+
}
|
|
96
|
+
| {
|
|
97
|
+
/**
|
|
98
|
+
* Metadata that is available after the stream is finished.
|
|
99
|
+
*/
|
|
100
|
+
type: 'finish';
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* The final source-language transcript of the input audio.
|
|
104
|
+
*/
|
|
105
|
+
sourceText: string;
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The final output text. May be an empty string for providers that
|
|
109
|
+
* produce only audio output.
|
|
110
|
+
*/
|
|
111
|
+
outputText: string;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The duration of the source audio in seconds, if available.
|
|
115
|
+
*/
|
|
116
|
+
durationInSeconds?: number;
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Usage information for the call, if reported by the provider.
|
|
120
|
+
*/
|
|
121
|
+
usage?: SpeechTranslationModelV4Usage;
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Additional provider-specific metadata.
|
|
125
|
+
*/
|
|
126
|
+
providerMetadata?: Record<string, JSONObject>;
|
|
127
|
+
}
|
|
128
|
+
| {
|
|
129
|
+
/**
|
|
130
|
+
* Raw provider chunks if enabled.
|
|
131
|
+
*/
|
|
132
|
+
type: 'raw';
|
|
133
|
+
rawValue: unknown;
|
|
134
|
+
}
|
|
135
|
+
| {
|
|
136
|
+
/**
|
|
137
|
+
* Error parts are streamed, allowing for multiple errors.
|
|
138
|
+
*/
|
|
139
|
+
type: 'error';
|
|
140
|
+
error: unknown;
|
|
141
|
+
};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { SharedV4Headers } from '../../shared';
|
|
2
|
+
import type { SpeechTranslationModelV4StreamPart } from './speech-translation-model-v4-stream-part';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The result of a speech translation model doStream call.
|
|
6
|
+
*/
|
|
7
|
+
export type SpeechTranslationModelV4StreamResult = {
|
|
8
|
+
/**
|
|
9
|
+
* The stream.
|
|
10
|
+
*/
|
|
11
|
+
stream: ReadableStream<SpeechTranslationModelV4StreamPart>;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Optional request information for telemetry and debugging purposes.
|
|
15
|
+
*/
|
|
16
|
+
request?: {
|
|
17
|
+
/**
|
|
18
|
+
* Request body or setup payload that was sent to the provider API.
|
|
19
|
+
*/
|
|
20
|
+
body?: unknown;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Optional response data.
|
|
25
|
+
*/
|
|
26
|
+
response?: {
|
|
27
|
+
/**
|
|
28
|
+
* Timestamp for the start of the streamed response.
|
|
29
|
+
*/
|
|
30
|
+
timestamp?: Date;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The ID of the response model that was used to generate the response.
|
|
34
|
+
*/
|
|
35
|
+
modelId?: string;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Response headers.
|
|
39
|
+
*/
|
|
40
|
+
headers?: SharedV4Headers;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Response body.
|
|
44
|
+
*/
|
|
45
|
+
body?: unknown;
|
|
46
|
+
};
|
|
47
|
+
};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage information for a speech translation call.
|
|
3
|
+
*
|
|
4
|
+
* All fields are optional because providers report usage with different
|
|
5
|
+
* granularity (seconds of audio, audio tokens, and/or text tokens).
|
|
6
|
+
*/
|
|
7
|
+
export type SpeechTranslationModelV4Usage = {
|
|
8
|
+
/**
|
|
9
|
+
* Seconds of input audio that were processed, if reported.
|
|
10
|
+
*/
|
|
11
|
+
inputAudioSeconds?: number;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Number of input audio tokens, if reported.
|
|
15
|
+
*/
|
|
16
|
+
inputAudioTokens?: number;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Number of output audio tokens, if reported.
|
|
20
|
+
*/
|
|
21
|
+
outputAudioTokens?: number;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Number of input text tokens, if reported.
|
|
25
|
+
*/
|
|
26
|
+
inputTextTokens?: number;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Number of output text tokens, if reported.
|
|
30
|
+
*/
|
|
31
|
+
outputTextTokens?: number;
|
|
32
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { SpeechTranslationModelV4StreamOptions } from './speech-translation-model-v4-stream-options';
|
|
2
|
+
import type { SpeechTranslationModelV4StreamResult } from './speech-translation-model-v4-stream-result';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Speech translation model specification version 4.
|
|
6
|
+
*
|
|
7
|
+
* Speech translation is a streaming-only modality: models translate live
|
|
8
|
+
* source audio into target-language audio and text.
|
|
9
|
+
*
|
|
10
|
+
* Experimental: the speech translation model contract may change in patch
|
|
11
|
+
* releases while the functions built on it are experimental. All types of
|
|
12
|
+
* this modality are exported with `Experimental_` prefixes for this reason.
|
|
13
|
+
*/
|
|
14
|
+
export type SpeechTranslationModelV4 = {
|
|
15
|
+
/**
|
|
16
|
+
* The speech translation model must specify which speech translation model
|
|
17
|
+
* interface version it implements. This will allow us to evolve the
|
|
18
|
+
* speech translation model interface and retain backwards compatibility.
|
|
19
|
+
* The different implementation versions can be handled as a discriminated
|
|
20
|
+
* union on our side.
|
|
21
|
+
*/
|
|
22
|
+
readonly specificationVersion: 'v4';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Name of the provider for logging purposes.
|
|
26
|
+
*/
|
|
27
|
+
readonly provider: string;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Provider-specific model ID for logging purposes.
|
|
31
|
+
*/
|
|
32
|
+
readonly modelId: string;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Streams a speech translation for live audio.
|
|
36
|
+
*/
|
|
37
|
+
doStream(
|
|
38
|
+
options: SpeechTranslationModelV4StreamOptions,
|
|
39
|
+
): PromiseLike<SpeechTranslationModelV4StreamResult>;
|
|
40
|
+
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { JSONObject } from '../../json-value/json-value';
|
|
2
|
+
import type { SharedV4AudioFormat } from '../../shared';
|
|
2
3
|
|
|
3
4
|
type TranscriptionModelV4ProviderOptions = Record<string, JSONObject>;
|
|
4
5
|
|
|
@@ -14,17 +15,7 @@ export type TranscriptionModelV4StreamOptions = {
|
|
|
14
15
|
/**
|
|
15
16
|
* The input audio format for the raw audio chunks.
|
|
16
17
|
*/
|
|
17
|
-
inputAudioFormat:
|
|
18
|
-
/**
|
|
19
|
-
* Audio format type, e.g. `audio/pcm`, `audio/pcmu`, or `audio/pcma`.
|
|
20
|
-
*/
|
|
21
|
-
type: string;
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* Sample rate in Hz. Only applicable for formats that require a rate.
|
|
25
|
-
*/
|
|
26
|
-
rate?: number;
|
|
27
|
-
};
|
|
18
|
+
inputAudioFormat: SharedV4AudioFormat;
|
|
28
19
|
|
|
29
20
|
/**
|
|
30
21
|
* Additional provider-specific options that are passed through to the provider.
|
|
@@ -4,7 +4,7 @@ import type { TranscriptionModelV4StreamOptions } from './transcription-model-v4
|
|
|
4
4
|
import type { TranscriptionModelV4StreamResult } from './transcription-model-v4-stream-result';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Transcription model specification version
|
|
7
|
+
* Transcription model specification version 4.
|
|
8
8
|
*/
|
|
9
9
|
export type TranscriptionModelV4 = {
|
|
10
10
|
/**
|
|
@@ -63,7 +63,11 @@ export type VideoModelV3CallOptions = {
|
|
|
63
63
|
frameImages: Array<VideoModelV3FrameImage> | undefined;
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
|
-
* Reference
|
|
66
|
+
* Reference inputs for reference-to-video generation.
|
|
67
|
+
*
|
|
68
|
+
* Each entry is an image or video file. Providers route each reference by
|
|
69
|
+
* its media type (image vs. video) and warn when a reference kind is
|
|
70
|
+
* unsupported.
|
|
67
71
|
*/
|
|
68
72
|
inputReferences: Array<VideoModelV3File> | undefined;
|
|
69
73
|
|
|
@@ -63,7 +63,11 @@ export type VideoModelV4CallOptions = {
|
|
|
63
63
|
frameImages: Array<VideoModelV4FrameImage> | undefined;
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
|
-
* Reference
|
|
66
|
+
* Reference inputs for reference-to-video generation.
|
|
67
|
+
*
|
|
68
|
+
* Each entry is an image or video file. Providers route each reference by
|
|
69
|
+
* its media type (image vs. video) and warn when a reference kind is
|
|
70
|
+
* unsupported.
|
|
67
71
|
*/
|
|
68
72
|
inputReferences: Array<VideoModelV4File> | undefined;
|
|
69
73
|
|
|
@@ -33,6 +33,13 @@ export type VideoModelV4File =
|
|
|
33
33
|
*/
|
|
34
34
|
url: string;
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* The media type of the referenced file, when known.
|
|
38
|
+
* Video types: 'video/mp4', 'video/webm', 'video/quicktime'
|
|
39
|
+
* Image types: 'image/png', 'image/jpeg', 'image/webp'
|
|
40
|
+
*/
|
|
41
|
+
mediaType?: string;
|
|
42
|
+
|
|
36
43
|
/**
|
|
37
44
|
* Optional provider-specific metadata for the file part.
|
|
38
45
|
*/
|
|
@@ -6,7 +6,7 @@ type GetMaxVideosPerCallFunction = (options: {
|
|
|
6
6
|
}) => PromiseLike<number | undefined> | number | undefined;
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
|
-
* Video generation model specification version
|
|
9
|
+
* Video generation model specification version 4.
|
|
10
10
|
*/
|
|
11
11
|
export type VideoModelV4 = {
|
|
12
12
|
/**
|