@depup/firebase__ai 2.9.0-depup.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/changes.json +10 -0
- package/dist/ai-public.d.ts +3472 -0
- package/dist/ai.d.ts +3712 -0
- package/dist/esm/index.esm.js +4765 -0
- package/dist/esm/index.esm.js.map +1 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/src/api.d.ts +121 -0
- package/dist/esm/src/backend.d.ts +98 -0
- package/dist/esm/src/constants.d.ts +29 -0
- package/dist/esm/src/errors.d.ts +35 -0
- package/dist/esm/src/factory-browser.d.ts +19 -0
- package/dist/esm/src/factory-node.d.ts +19 -0
- package/dist/esm/src/googleai-mappers.d.ts +73 -0
- package/dist/esm/src/helpers.d.ts +30 -0
- package/dist/esm/src/index.d.ts +13 -0
- package/dist/esm/src/index.node.d.ts +7 -0
- package/dist/esm/src/logger.d.ts +18 -0
- package/dist/esm/src/methods/chat-session-helpers.d.ts +18 -0
- package/dist/esm/src/methods/chat-session.d.ts +77 -0
- package/dist/esm/src/methods/chrome-adapter.d.ts +124 -0
- package/dist/esm/src/methods/count-tokens.d.ts +21 -0
- package/dist/esm/src/methods/generate-content.d.ts +25 -0
- package/dist/esm/src/methods/live-session-helpers.d.ts +154 -0
- package/dist/esm/src/methods/live-session.d.ts +154 -0
- package/dist/esm/src/models/ai-model.d.ts +72 -0
- package/dist/esm/src/models/generative-model.d.ts +56 -0
- package/dist/esm/src/models/imagen-model.d.ts +102 -0
- package/dist/esm/src/models/index.d.ts +20 -0
- package/dist/esm/src/models/live-generative-model.d.ts +55 -0
- package/dist/esm/src/models/template-generative-model.d.ts +64 -0
- package/dist/esm/src/models/template-imagen-model.d.ts +51 -0
- package/dist/esm/src/models/utils.d.ts +26 -0
- package/dist/esm/src/public-types.d.ts +97 -0
- package/dist/esm/src/requests/hybrid-helpers.d.ts +33 -0
- package/dist/esm/src/requests/imagen-image-format.d.ts +61 -0
- package/dist/esm/src/requests/request-helpers.d.ts +28 -0
- package/dist/esm/src/requests/request.d.ts +69 -0
- package/dist/esm/src/requests/response-helpers.d.ts +57 -0
- package/dist/esm/src/requests/schema-builder.d.ts +170 -0
- package/dist/esm/src/requests/stream-reader.d.ts +39 -0
- package/dist/esm/src/service.d.ts +35 -0
- package/dist/esm/src/types/chrome-adapter.d.ts +61 -0
- package/dist/esm/src/types/content.d.ts +266 -0
- package/dist/esm/src/types/enums.d.ts +419 -0
- package/dist/esm/src/types/error.d.ts +89 -0
- package/dist/esm/src/types/googleai.d.ts +57 -0
- package/dist/esm/src/types/imagen/index.d.ts +18 -0
- package/dist/esm/src/types/imagen/internal.d.ts +134 -0
- package/dist/esm/src/types/imagen/requests.d.ts +245 -0
- package/dist/esm/src/types/imagen/responses.d.ts +79 -0
- package/dist/esm/src/types/index.d.ts +26 -0
- package/dist/esm/src/types/internal.d.ts +35 -0
- package/dist/esm/src/types/language-model.d.ts +107 -0
- package/dist/esm/src/types/live-responses.d.ts +79 -0
- package/dist/esm/src/types/requests.d.ts +543 -0
- package/dist/esm/src/types/responses.d.ts +607 -0
- package/dist/esm/src/types/schema.d.ts +139 -0
- package/dist/esm/src/websocket.d.ts +67 -0
- package/dist/index.cjs.js +4820 -0
- package/dist/index.cjs.js.map +1 -0
- package/dist/index.node.cjs.js +4512 -0
- package/dist/index.node.cjs.js.map +1 -0
- package/dist/index.node.mjs +4457 -0
- package/dist/index.node.mjs.map +1 -0
- package/dist/src/api.d.ts +121 -0
- package/dist/src/backend.d.ts +98 -0
- package/dist/src/constants.d.ts +29 -0
- package/dist/src/errors.d.ts +35 -0
- package/dist/src/factory-browser.d.ts +19 -0
- package/dist/src/factory-node.d.ts +19 -0
- package/dist/src/googleai-mappers.d.ts +73 -0
- package/dist/src/helpers.d.ts +30 -0
- package/dist/src/index.d.ts +13 -0
- package/dist/src/index.node.d.ts +7 -0
- package/dist/src/logger.d.ts +18 -0
- package/dist/src/methods/chat-session-helpers.d.ts +18 -0
- package/dist/src/methods/chat-session.d.ts +77 -0
- package/dist/src/methods/chrome-adapter.d.ts +124 -0
- package/dist/src/methods/count-tokens.d.ts +21 -0
- package/dist/src/methods/generate-content.d.ts +25 -0
- package/dist/src/methods/live-session-helpers.d.ts +154 -0
- package/dist/src/methods/live-session.d.ts +154 -0
- package/dist/src/models/ai-model.d.ts +72 -0
- package/dist/src/models/generative-model.d.ts +56 -0
- package/dist/src/models/imagen-model.d.ts +102 -0
- package/dist/src/models/index.d.ts +20 -0
- package/dist/src/models/live-generative-model.d.ts +55 -0
- package/dist/src/models/template-generative-model.d.ts +64 -0
- package/dist/src/models/template-imagen-model.d.ts +51 -0
- package/dist/src/models/utils.d.ts +26 -0
- package/dist/src/public-types.d.ts +97 -0
- package/dist/src/requests/hybrid-helpers.d.ts +33 -0
- package/dist/src/requests/imagen-image-format.d.ts +61 -0
- package/dist/src/requests/request-helpers.d.ts +28 -0
- package/dist/src/requests/request.d.ts +69 -0
- package/dist/src/requests/response-helpers.d.ts +57 -0
- package/dist/src/requests/schema-builder.d.ts +170 -0
- package/dist/src/requests/stream-reader.d.ts +39 -0
- package/dist/src/service.d.ts +35 -0
- package/dist/src/tsdoc-metadata.json +11 -0
- package/dist/src/types/chrome-adapter.d.ts +61 -0
- package/dist/src/types/content.d.ts +266 -0
- package/dist/src/types/enums.d.ts +419 -0
- package/dist/src/types/error.d.ts +89 -0
- package/dist/src/types/googleai.d.ts +57 -0
- package/dist/src/types/imagen/index.d.ts +18 -0
- package/dist/src/types/imagen/internal.d.ts +134 -0
- package/dist/src/types/imagen/requests.d.ts +245 -0
- package/dist/src/types/imagen/responses.d.ts +79 -0
- package/dist/src/types/index.d.ts +26 -0
- package/dist/src/types/internal.d.ts +35 -0
- package/dist/src/types/language-model.d.ts +107 -0
- package/dist/src/types/live-responses.d.ts +79 -0
- package/dist/src/types/requests.d.ts +543 -0
- package/dist/src/types/responses.d.ts +607 -0
- package/dist/src/types/schema.d.ts +139 -0
- package/dist/src/websocket.d.ts +67 -0
- package/package.json +106 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { CountTokensRequest, GenerateContentRequest, InferenceMode, OnDeviceParams } from '../types';
|
|
18
|
+
import { ChromeAdapter } from '../types/chrome-adapter';
|
|
19
|
+
import { LanguageModel } from '../types/language-model';
|
|
20
|
+
/**
|
|
21
|
+
* Defines an inference "backend" that uses Chrome's on-device model,
|
|
22
|
+
* and encapsulates logic for detecting when on-device inference is
|
|
23
|
+
* possible.
|
|
24
|
+
*/
|
|
25
|
+
export declare class ChromeAdapterImpl implements ChromeAdapter {
|
|
26
|
+
languageModelProvider: LanguageModel;
|
|
27
|
+
mode: InferenceMode;
|
|
28
|
+
static SUPPORTED_MIME_TYPES: string[];
|
|
29
|
+
private isDownloading;
|
|
30
|
+
private downloadPromise;
|
|
31
|
+
private oldSession;
|
|
32
|
+
onDeviceParams: OnDeviceParams;
|
|
33
|
+
constructor(languageModelProvider: LanguageModel, mode: InferenceMode, onDeviceParams?: OnDeviceParams);
|
|
34
|
+
/**
|
|
35
|
+
* Checks if a given request can be made on-device.
|
|
36
|
+
*
|
|
37
|
+
* Encapsulates a few concerns:
|
|
38
|
+
* the mode
|
|
39
|
+
* API existence
|
|
40
|
+
* prompt formatting
|
|
41
|
+
* model availability, including triggering download if necessary
|
|
42
|
+
*
|
|
43
|
+
*
|
|
44
|
+
* Pros: callers needn't be concerned with details of on-device availability.</p>
|
|
45
|
+
* Cons: this method spans a few concerns and splits request validation from usage.
|
|
46
|
+
* If instance variables weren't already part of the API, we could consider a better
|
|
47
|
+
* separation of concerns.
|
|
48
|
+
*/
|
|
49
|
+
isAvailable(request: GenerateContentRequest): Promise<boolean>;
|
|
50
|
+
/**
|
|
51
|
+
* Generates content on device.
|
|
52
|
+
*
|
|
53
|
+
* @remarks
|
|
54
|
+
* This is comparable to {@link GenerativeModel.generateContent} for generating content in
|
|
55
|
+
* Cloud.
|
|
56
|
+
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
57
|
+
* @returns {@link Response}, so we can reuse common response formatting.
|
|
58
|
+
*/
|
|
59
|
+
generateContent(request: GenerateContentRequest): Promise<Response>;
|
|
60
|
+
/**
|
|
61
|
+
* Generates content stream on device.
|
|
62
|
+
*
|
|
63
|
+
* @remarks
|
|
64
|
+
* This is comparable to {@link GenerativeModel.generateContentStream} for generating content in
|
|
65
|
+
* Cloud.
|
|
66
|
+
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
67
|
+
* @returns {@link Response}, so we can reuse common response formatting.
|
|
68
|
+
*/
|
|
69
|
+
generateContentStream(request: GenerateContentRequest): Promise<Response>;
|
|
70
|
+
countTokens(_request: CountTokensRequest): Promise<Response>;
|
|
71
|
+
/**
|
|
72
|
+
* Asserts inference for the given request can be performed by an on-device model.
|
|
73
|
+
*/
|
|
74
|
+
private static isOnDeviceRequest;
|
|
75
|
+
/**
|
|
76
|
+
* Encapsulates logic to get availability and download a model if one is downloadable.
|
|
77
|
+
*/
|
|
78
|
+
private downloadIfAvailable;
|
|
79
|
+
/**
|
|
80
|
+
* Triggers out-of-band download of an on-device model.
|
|
81
|
+
*
|
|
82
|
+
* Chrome only downloads models as needed. Chrome knows a model is needed when code calls
|
|
83
|
+
* LanguageModel.create.
|
|
84
|
+
*
|
|
85
|
+
* Since Chrome manages the download, the SDK can only avoid redundant download requests by
|
|
86
|
+
* tracking if a download has previously been requested.
|
|
87
|
+
*/
|
|
88
|
+
private download;
|
|
89
|
+
/**
|
|
90
|
+
* Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object.
|
|
91
|
+
*/
|
|
92
|
+
private static toLanguageModelMessage;
|
|
93
|
+
/**
|
|
94
|
+
* Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object.
|
|
95
|
+
*/
|
|
96
|
+
private static toLanguageModelMessageContent;
|
|
97
|
+
/**
|
|
98
|
+
* Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string.
|
|
99
|
+
*/
|
|
100
|
+
private static toLanguageModelMessageRole;
|
|
101
|
+
/**
|
|
102
|
+
* Abstracts Chrome session creation.
|
|
103
|
+
*
|
|
104
|
+
* Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all
|
|
105
|
+
* inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all
|
|
106
|
+
* inference.
|
|
107
|
+
*
|
|
108
|
+
* Chrome will remove a model from memory if it's no longer in use, so this method ensures a
|
|
109
|
+
* new session is created before an old session is destroyed.
|
|
110
|
+
*/
|
|
111
|
+
private createSession;
|
|
112
|
+
/**
|
|
113
|
+
* Formats string returned by Chrome as a {@link Response} returned by Firebase AI.
|
|
114
|
+
*/
|
|
115
|
+
private static toResponse;
|
|
116
|
+
/**
|
|
117
|
+
* Formats string stream returned by Chrome as SSE returned by Firebase AI.
|
|
118
|
+
*/
|
|
119
|
+
private static toStreamResponse;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Creates a ChromeAdapterImpl on demand.
|
|
123
|
+
*/
|
|
124
|
+
export declare function chromeAdapterFactory(mode: InferenceMode, window?: Window, params?: OnDeviceParams): ChromeAdapterImpl | undefined;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2024 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { CountTokensRequest, CountTokensResponse, SingleRequestOptions, RequestOptions } from '../types';
|
|
18
|
+
import { ApiSettings } from '../types/internal';
|
|
19
|
+
import { ChromeAdapter } from '../types/chrome-adapter';
|
|
20
|
+
export declare function countTokensOnCloud(apiSettings: ApiSettings, model: string, params: CountTokensRequest, singleRequestOptions?: SingleRequestOptions): Promise<CountTokensResponse>;
|
|
21
|
+
export declare function countTokens(apiSettings: ApiSettings, model: string, params: CountTokensRequest, chromeAdapter?: ChromeAdapter, requestOptions?: RequestOptions): Promise<CountTokensResponse>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2024 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { GenerateContentRequest, GenerateContentResponse, GenerateContentResult, GenerateContentStreamResult, SingleRequestOptions } from '../types';
|
|
18
|
+
import { ApiSettings } from '../types/internal';
|
|
19
|
+
import { ChromeAdapter } from '../types/chrome-adapter';
|
|
20
|
+
export declare function generateContentStream(apiSettings: ApiSettings, model: string, params: GenerateContentRequest, chromeAdapter?: ChromeAdapter, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult & {
|
|
21
|
+
firstValue?: GenerateContentResponse;
|
|
22
|
+
}>;
|
|
23
|
+
export declare function templateGenerateContent(apiSettings: ApiSettings, templateId: string, templateParams: object, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentResult>;
|
|
24
|
+
export declare function templateGenerateContentStream(apiSettings: ApiSettings, templateId: string, templateParams: object, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult>;
|
|
25
|
+
export declare function generateContent(apiSettings: ApiSettings, model: string, params: GenerateContentRequest, chromeAdapter?: ChromeAdapter, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentResult>;
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { FunctionCall, FunctionResponse } from '../types';
|
|
18
|
+
import { LiveSession } from './live-session';
|
|
19
|
+
/**
|
|
20
|
+
* A controller for managing an active audio conversation.
|
|
21
|
+
*
|
|
22
|
+
* @beta
|
|
23
|
+
*/
|
|
24
|
+
export interface AudioConversationController {
|
|
25
|
+
/**
|
|
26
|
+
* Stops the audio conversation, closes the microphone connection, and
|
|
27
|
+
* cleans up resources. Returns a promise that resolves when cleanup is complete.
|
|
28
|
+
*/
|
|
29
|
+
stop: () => Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Options for {@link startAudioConversation}.
|
|
33
|
+
*
|
|
34
|
+
* @beta
|
|
35
|
+
*/
|
|
36
|
+
export interface StartAudioConversationOptions {
|
|
37
|
+
/**
|
|
38
|
+
* An async handler that is called when the model requests a function to be executed.
|
|
39
|
+
* The handler should perform the function call and return the result as a `Part`,
|
|
40
|
+
* which will then be sent back to the model.
|
|
41
|
+
*/
|
|
42
|
+
functionCallingHandler?: (functionCalls: FunctionCall[]) => Promise<FunctionResponse>;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Dependencies needed by the {@link AudioConversationRunner}.
|
|
46
|
+
*
|
|
47
|
+
* @internal
|
|
48
|
+
*/
|
|
49
|
+
interface RunnerDependencies {
|
|
50
|
+
audioContext: AudioContext;
|
|
51
|
+
mediaStream: MediaStream;
|
|
52
|
+
sourceNode: MediaStreamAudioSourceNode;
|
|
53
|
+
workletNode: AudioWorkletNode;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Encapsulates the core logic of an audio conversation.
|
|
57
|
+
*
|
|
58
|
+
* @internal
|
|
59
|
+
*/
|
|
60
|
+
export declare class AudioConversationRunner {
|
|
61
|
+
private readonly liveSession;
|
|
62
|
+
private readonly options;
|
|
63
|
+
private readonly deps;
|
|
64
|
+
/** A flag to indicate if the conversation has been stopped. */
|
|
65
|
+
private isStopped;
|
|
66
|
+
/** A deferred that contains a promise that is resolved when stop() is called, to unblock the receive loop. */
|
|
67
|
+
private readonly stopDeferred;
|
|
68
|
+
/** A promise that tracks the lifecycle of the main `runReceiveLoop`. */
|
|
69
|
+
private readonly receiveLoopPromise;
|
|
70
|
+
/** A FIFO queue of 24kHz, 16-bit PCM audio chunks received from the server. */
|
|
71
|
+
private readonly playbackQueue;
|
|
72
|
+
/** Tracks scheduled audio sources. Used to cancel scheduled audio when the model is interrupted. */
|
|
73
|
+
private scheduledSources;
|
|
74
|
+
/** A high-precision timeline pointer for scheduling gapless audio playback. */
|
|
75
|
+
private nextStartTime;
|
|
76
|
+
/** A mutex to prevent the playback processing loop from running multiple times concurrently. */
|
|
77
|
+
private isPlaybackLoopRunning;
|
|
78
|
+
constructor(liveSession: LiveSession, options: StartAudioConversationOptions, deps: RunnerDependencies);
|
|
79
|
+
/**
|
|
80
|
+
* Stops the conversation and unblocks the main receive loop.
|
|
81
|
+
*/
|
|
82
|
+
stop(): Promise<void>;
|
|
83
|
+
/**
|
|
84
|
+
* Cleans up all audio resources (nodes, stream tracks, context) and marks the
|
|
85
|
+
* session as no longer in a conversation.
|
|
86
|
+
*/
|
|
87
|
+
private cleanup;
|
|
88
|
+
/**
|
|
89
|
+
* Adds audio data to the queue and ensures the playback loop is running.
|
|
90
|
+
*/
|
|
91
|
+
private enqueueAndPlay;
|
|
92
|
+
/**
|
|
93
|
+
* Stops all current and pending audio playback and clears the queue. This is
|
|
94
|
+
* called when the server indicates the model's speech was interrupted with
|
|
95
|
+
* `LiveServerContent.modelTurn.interrupted`.
|
|
96
|
+
*/
|
|
97
|
+
private interruptPlayback;
|
|
98
|
+
/**
|
|
99
|
+
* Processes the playback queue in a loop, scheduling each chunk in a gapless sequence.
|
|
100
|
+
*/
|
|
101
|
+
private processPlaybackQueue;
|
|
102
|
+
/**
|
|
103
|
+
* The main loop that listens for and processes messages from the server.
|
|
104
|
+
*/
|
|
105
|
+
private runReceiveLoop;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Starts a real-time, bidirectional audio conversation with the model. This helper function manages
|
|
109
|
+
* the complexities of microphone access, audio recording, playback, and interruptions.
|
|
110
|
+
*
|
|
111
|
+
* @remarks Important: This function must be called in response to a user gesture
|
|
112
|
+
* (for example, a button click) to comply with {@link https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Best_practices#autoplay_policy | browser autoplay policies}.
|
|
113
|
+
*
|
|
114
|
+
* @example
|
|
115
|
+
* ```javascript
|
|
116
|
+
* const liveSession = await model.connect();
|
|
117
|
+
* let conversationController;
|
|
118
|
+
*
|
|
119
|
+
* // This function must be called from within a click handler.
|
|
120
|
+
* async function startConversation() {
|
|
121
|
+
* try {
|
|
122
|
+
* conversationController = await startAudioConversation(liveSession);
|
|
123
|
+
* } catch (e) {
|
|
124
|
+
* // Handle AI-specific errors
|
|
125
|
+
* if (e instanceof AIError) {
|
|
126
|
+
* console.error("AI Error:", e.message);
|
|
127
|
+
* }
|
|
128
|
+
* // Handle microphone permission and hardware errors
|
|
129
|
+
* else if (e instanceof DOMException) {
|
|
130
|
+
* console.error("Microphone Error:", e.message);
|
|
131
|
+
* }
|
|
132
|
+
* // Handle other unexpected errors
|
|
133
|
+
* else {
|
|
134
|
+
* console.error("An unexpected error occurred:", e);
|
|
135
|
+
* }
|
|
136
|
+
* }
|
|
137
|
+
* }
|
|
138
|
+
*
|
|
139
|
+
* // Later, to stop the conversation:
|
|
140
|
+
* // if (conversationController) {
|
|
141
|
+
* // await conversationController.stop();
|
|
142
|
+
* // }
|
|
143
|
+
* ```
|
|
144
|
+
*
|
|
145
|
+
* @param liveSession - An active {@link LiveSession} instance.
|
|
146
|
+
* @param options - Configuration options for the audio conversation.
|
|
147
|
+
* @returns A `Promise` that resolves with an {@link AudioConversationController}.
|
|
148
|
+
* @throws `AIError` if the environment does not support required Web APIs (`UNSUPPORTED`), if a conversation is already active (`REQUEST_ERROR`), the session is closed (`SESSION_CLOSED`), or if an unexpected initialization error occurs (`ERROR`).
|
|
149
|
+
* @throws `DOMException` Thrown by `navigator.mediaDevices.getUserMedia()` if issues occur with microphone access, such as permissions being denied (`NotAllowedError`) or no compatible hardware being found (`NotFoundError`). See the {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia#exceptions | MDN documentation} for a full list of exceptions.
|
|
150
|
+
*
|
|
151
|
+
* @beta
|
|
152
|
+
*/
|
|
153
|
+
export declare function startAudioConversation(liveSession: LiveSession, options?: StartAudioConversationOptions): Promise<AudioConversationController>;
|
|
154
|
+
export {};
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { FunctionResponse, GenerativeContentBlob, LiveServerContent, LiveServerGoingAwayNotice, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
|
|
18
|
+
import { WebSocketHandler } from '../websocket';
|
|
19
|
+
/**
|
|
20
|
+
* Represents an active, real-time, bidirectional conversation with the model.
|
|
21
|
+
*
|
|
22
|
+
* This class should only be instantiated by calling {@link LiveGenerativeModel.connect}.
|
|
23
|
+
*
|
|
24
|
+
* @beta
|
|
25
|
+
*/
|
|
26
|
+
export declare class LiveSession {
|
|
27
|
+
private webSocketHandler;
|
|
28
|
+
private serverMessages;
|
|
29
|
+
/**
|
|
30
|
+
* Indicates whether this Live session is closed.
|
|
31
|
+
*
|
|
32
|
+
* @beta
|
|
33
|
+
*/
|
|
34
|
+
isClosed: boolean;
|
|
35
|
+
/**
|
|
36
|
+
* Indicates whether this Live session is being controlled by an `AudioConversationController`.
|
|
37
|
+
*
|
|
38
|
+
* @beta
|
|
39
|
+
*/
|
|
40
|
+
inConversation: boolean;
|
|
41
|
+
/**
|
|
42
|
+
* @internal
|
|
43
|
+
*/
|
|
44
|
+
constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
|
|
45
|
+
/**
|
|
46
|
+
* Sends content to the server.
|
|
47
|
+
*
|
|
48
|
+
* @param request - The message to send to the model.
|
|
49
|
+
* @param turnComplete - Indicates if the turn is complete. Defaults to false.
|
|
50
|
+
* @throws If this session has been closed.
|
|
51
|
+
*
|
|
52
|
+
* @beta
|
|
53
|
+
*/
|
|
54
|
+
send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* Sends text to the server in realtime.
|
|
57
|
+
*
|
|
58
|
+
* @example
|
|
59
|
+
* ```javascript
|
|
60
|
+
* liveSession.sendTextRealtime("Hello, how are you?");
|
|
61
|
+
* ```
|
|
62
|
+
*
|
|
63
|
+
* @param text - The text data to send.
|
|
64
|
+
* @throws If this session has been closed.
|
|
65
|
+
*
|
|
66
|
+
* @beta
|
|
67
|
+
*/
|
|
68
|
+
sendTextRealtime(text: string): Promise<void>;
|
|
69
|
+
/**
|
|
70
|
+
* Sends audio data to the server in realtime.
|
|
71
|
+
*
|
|
72
|
+
* @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
|
|
73
|
+
* little-endian.
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* ```javascript
|
|
77
|
+
* // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
|
|
78
|
+
* const blob = { mimeType: "audio/pcm", data: pcmData };
|
|
79
|
+
* liveSession.sendAudioRealtime(blob);
|
|
80
|
+
* ```
|
|
81
|
+
*
|
|
82
|
+
* @param blob - The base64-encoded PCM data to send to the server in realtime.
|
|
83
|
+
* @throws If this session has been closed.
|
|
84
|
+
*
|
|
85
|
+
* @beta
|
|
86
|
+
*/
|
|
87
|
+
sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
|
|
88
|
+
/**
|
|
89
|
+
* Sends video data to the server in realtime.
|
|
90
|
+
*
|
|
91
|
+
* @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
|
|
92
|
+
* is recommended to set `mimeType` to `image/jpeg`.
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```javascript
|
|
96
|
+
* // const videoFrame = ... base64-encoded JPEG data
|
|
97
|
+
* const blob = { mimeType: "image/jpeg", data: videoFrame };
|
|
98
|
+
* liveSession.sendVideoRealtime(blob);
|
|
99
|
+
* ```
|
|
100
|
+
* @param blob - The base64-encoded video data to send to the server in realtime.
|
|
101
|
+
* @throws If this session has been closed.
|
|
102
|
+
*
|
|
103
|
+
* @beta
|
|
104
|
+
*/
|
|
105
|
+
sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
|
|
106
|
+
/**
|
|
107
|
+
* Sends function responses to the server.
|
|
108
|
+
*
|
|
109
|
+
* @param functionResponses - The function responses to send.
|
|
110
|
+
* @throws If this session has been closed.
|
|
111
|
+
*
|
|
112
|
+
* @beta
|
|
113
|
+
*/
|
|
114
|
+
sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Yields messages received from the server.
|
|
117
|
+
* This can only be used by one consumer at a time.
|
|
118
|
+
*
|
|
119
|
+
* @returns An `AsyncGenerator` that yields server messages as they arrive.
|
|
120
|
+
* @throws If the session is already closed, or if we receive a response that we don't support.
|
|
121
|
+
*
|
|
122
|
+
* @beta
|
|
123
|
+
*/
|
|
124
|
+
receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation | LiveServerGoingAwayNotice>;
|
|
125
|
+
/**
|
|
126
|
+
* Closes this session.
|
|
127
|
+
* All methods on this session will throw an error once this resolves.
|
|
128
|
+
*
|
|
129
|
+
* @beta
|
|
130
|
+
*/
|
|
131
|
+
close(): Promise<void>;
|
|
132
|
+
/**
|
|
133
|
+
* Sends realtime input to the server.
|
|
134
|
+
*
|
|
135
|
+
* @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
|
|
136
|
+
*
|
|
137
|
+
* @param mediaChunks - The media chunks to send.
|
|
138
|
+
* @throws If this session has been closed.
|
|
139
|
+
*
|
|
140
|
+
* @beta
|
|
141
|
+
*/
|
|
142
|
+
sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
|
|
143
|
+
/**
|
|
144
|
+
* @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
|
|
145
|
+
*
|
|
146
|
+
* Sends a stream of {@link GenerativeContentBlob}.
|
|
147
|
+
*
|
|
148
|
+
* @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
|
|
149
|
+
* @throws If this session has been closed.
|
|
150
|
+
*
|
|
151
|
+
* @beta
|
|
152
|
+
*/
|
|
153
|
+
sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
|
|
154
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { AI, BackendType } from '../public-types';
|
|
18
|
+
import { ApiSettings } from '../types/internal';
|
|
19
|
+
/**
|
|
20
|
+
* Base class for Firebase AI model APIs.
|
|
21
|
+
*
|
|
22
|
+
* Instances of this class are associated with a specific Firebase AI {@link Backend}
|
|
23
|
+
* and provide methods for interacting with the configured generative model.
|
|
24
|
+
*
|
|
25
|
+
* @public
|
|
26
|
+
*/
|
|
27
|
+
export declare abstract class AIModel {
|
|
28
|
+
/**
|
|
29
|
+
* The fully qualified model resource name to use for generating images
|
|
30
|
+
* (for example, `publishers/google/models/imagen-3.0-generate-002`).
|
|
31
|
+
*/
|
|
32
|
+
readonly model: string;
|
|
33
|
+
/**
|
|
34
|
+
* @internal
|
|
35
|
+
*/
|
|
36
|
+
_apiSettings: ApiSettings;
|
|
37
|
+
/**
|
|
38
|
+
* Constructs a new instance of the {@link AIModel} class.
|
|
39
|
+
*
|
|
40
|
+
* This constructor should only be called from subclasses that provide
|
|
41
|
+
* a model API.
|
|
42
|
+
*
|
|
43
|
+
* @param ai - an {@link AI} instance.
|
|
44
|
+
* @param modelName - The name of the model being used. It can be in one of the following formats:
|
|
45
|
+
* - `my-model` (short name, will resolve to `publishers/google/models/my-model`)
|
|
46
|
+
* - `models/my-model` (will resolve to `publishers/google/models/my-model`)
|
|
47
|
+
* - `publishers/my-publisher/models/my-model` (fully qualified model name)
|
|
48
|
+
*
|
|
49
|
+
* @throws If the `apiKey` or `projectId` fields are missing in your
|
|
50
|
+
* Firebase config.
|
|
51
|
+
*
|
|
52
|
+
* @internal
|
|
53
|
+
*/
|
|
54
|
+
protected constructor(ai: AI, modelName: string);
|
|
55
|
+
/**
|
|
56
|
+
* Normalizes the given model name to a fully qualified model resource name.
|
|
57
|
+
*
|
|
58
|
+
* @param modelName - The model name to normalize.
|
|
59
|
+
* @returns The fully qualified model resource name.
|
|
60
|
+
*
|
|
61
|
+
* @internal
|
|
62
|
+
*/
|
|
63
|
+
static normalizeModelName(modelName: string, backendType: BackendType): string;
|
|
64
|
+
/**
|
|
65
|
+
* @internal
|
|
66
|
+
*/
|
|
67
|
+
private static normalizeGoogleAIModelName;
|
|
68
|
+
/**
|
|
69
|
+
* @internal
|
|
70
|
+
*/
|
|
71
|
+
private static normalizeVertexAIModelName;
|
|
72
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2024 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
import { Content, CountTokensRequest, CountTokensResponse, GenerateContentRequest, GenerateContentResult, GenerateContentStreamResult, GenerationConfig, ModelParams, Part, SafetySetting, RequestOptions, StartChatParams, Tool, ToolConfig, SingleRequestOptions } from '../types';
|
|
18
|
+
import { ChatSession } from '../methods/chat-session';
|
|
19
|
+
import { AI } from '../public-types';
|
|
20
|
+
import { AIModel } from './ai-model';
|
|
21
|
+
import { ChromeAdapter } from '../types/chrome-adapter';
|
|
22
|
+
/**
|
|
23
|
+
* Class for generative model APIs.
|
|
24
|
+
* @public
|
|
25
|
+
*/
|
|
26
|
+
export declare class GenerativeModel extends AIModel {
|
|
27
|
+
private chromeAdapter?;
|
|
28
|
+
generationConfig: GenerationConfig;
|
|
29
|
+
safetySettings: SafetySetting[];
|
|
30
|
+
requestOptions?: RequestOptions;
|
|
31
|
+
tools?: Tool[];
|
|
32
|
+
toolConfig?: ToolConfig;
|
|
33
|
+
systemInstruction?: Content;
|
|
34
|
+
constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined);
|
|
35
|
+
/**
|
|
36
|
+
* Makes a single non-streaming call to the model
|
|
37
|
+
* and returns an object containing a single {@link GenerateContentResponse}.
|
|
38
|
+
*/
|
|
39
|
+
generateContent(request: GenerateContentRequest | string | Array<string | Part>, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentResult>;
|
|
40
|
+
/**
|
|
41
|
+
* Makes a single streaming call to the model
|
|
42
|
+
* and returns an object containing an iterable stream that iterates
|
|
43
|
+
* over all chunks in the streaming response as well as
|
|
44
|
+
* a promise that returns the final aggregated response.
|
|
45
|
+
*/
|
|
46
|
+
generateContentStream(request: GenerateContentRequest | string | Array<string | Part>, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult>;
|
|
47
|
+
/**
|
|
48
|
+
* Gets a new {@link ChatSession} instance which can be used for
|
|
49
|
+
* multi-turn chats.
|
|
50
|
+
*/
|
|
51
|
+
startChat(startChatParams?: StartChatParams): ChatSession;
|
|
52
|
+
/**
|
|
53
|
+
* Counts the tokens in the provided request.
|
|
54
|
+
*/
|
|
55
|
+
countTokens(request: CountTokensRequest | string | Array<string | Part>, singleRequestOptions?: SingleRequestOptions): Promise<CountTokensResponse>;
|
|
56
|
+
}
|