@firebase/ai 2.2.0 → 2.2.1-canary.06ab5c4f9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +19 -138
- package/dist/ai.d.ts +19 -175
- package/dist/esm/index.esm.js +604 -515
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/factory-browser.d.ts +19 -0
- package/dist/esm/src/index.d.ts +0 -3
- package/dist/esm/src/requests/hybrid-helpers.d.ts +28 -0
- package/dist/esm/src/types/enums.d.ts +19 -0
- package/dist/index.cjs.js +603 -515
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +90 -18
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +90 -18
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/factory-browser.d.ts +19 -0
- package/dist/src/index.d.ts +0 -3
- package/dist/src/requests/hybrid-helpers.d.ts +28 -0
- package/dist/src/types/enums.d.ts +19 -0
- package/package.json +10 -9
package/dist/index.cjs.js
CHANGED
|
@@ -8,7 +8,7 @@ var util = require('@firebase/util');
|
|
|
8
8
|
var logger$1 = require('@firebase/logger');
|
|
9
9
|
|
|
10
10
|
var name = "@firebase/ai";
|
|
11
|
-
var version = "2.2.
|
|
11
|
+
var version = "2.2.1-canary.06ab5c4f9";
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* @license
|
|
@@ -38,6 +38,62 @@ const DEFAULT_FETCH_TIMEOUT_MS = 180 * 1000;
|
|
|
38
38
|
*/
|
|
39
39
|
const DEFAULT_HYBRID_IN_CLOUD_MODEL = 'gemini-2.0-flash-lite';
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* @license
|
|
43
|
+
* Copyright 2024 Google LLC
|
|
44
|
+
*
|
|
45
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
46
|
+
* you may not use this file except in compliance with the License.
|
|
47
|
+
* You may obtain a copy of the License at
|
|
48
|
+
*
|
|
49
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
50
|
+
*
|
|
51
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
52
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
53
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
54
|
+
* See the License for the specific language governing permissions and
|
|
55
|
+
* limitations under the License.
|
|
56
|
+
*/
|
|
57
|
+
/**
|
|
58
|
+
* Error class for the Firebase AI SDK.
|
|
59
|
+
*
|
|
60
|
+
* @public
|
|
61
|
+
*/
|
|
62
|
+
class AIError extends util.FirebaseError {
|
|
63
|
+
/**
|
|
64
|
+
* Constructs a new instance of the `AIError` class.
|
|
65
|
+
*
|
|
66
|
+
* @param code - The error code from {@link (AIErrorCode:type)}.
|
|
67
|
+
* @param message - A human-readable message describing the error.
|
|
68
|
+
* @param customErrorData - Optional error data.
|
|
69
|
+
*/
|
|
70
|
+
constructor(code, message, customErrorData) {
|
|
71
|
+
// Match error format used by FirebaseError from ErrorFactory
|
|
72
|
+
const service = AI_TYPE;
|
|
73
|
+
const fullCode = `${service}/${code}`;
|
|
74
|
+
const fullMessage = `${service}: ${message} (${fullCode})`;
|
|
75
|
+
super(code, fullMessage);
|
|
76
|
+
this.code = code;
|
|
77
|
+
this.customErrorData = customErrorData;
|
|
78
|
+
// FirebaseError initializes a stack trace, but it assumes the error is created from the error
|
|
79
|
+
// factory. Since we break this assumption, we set the stack trace to be originating from this
|
|
80
|
+
// constructor.
|
|
81
|
+
// This is only supported in V8.
|
|
82
|
+
if (Error.captureStackTrace) {
|
|
83
|
+
// Allows us to initialize the stack trace without including the constructor itself at the
|
|
84
|
+
// top level of the stack trace.
|
|
85
|
+
Error.captureStackTrace(this, AIError);
|
|
86
|
+
}
|
|
87
|
+
// Allows instanceof AIError in ES5/ES6
|
|
88
|
+
// https://github.com/Microsoft/TypeScript-wiki/blob/master/Breaking-Changes.md#extending-built-ins-like-error-array-and-map-may-no-longer-work
|
|
89
|
+
// TODO(dlarocque): Replace this with `new.target`: https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget
|
|
90
|
+
// which we can now use since we no longer target ES5.
|
|
91
|
+
Object.setPrototypeOf(this, AIError.prototype);
|
|
92
|
+
// Since Error is an interface, we don't inherit toString and so we define it ourselves.
|
|
93
|
+
this.toString = () => fullMessage;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
41
97
|
/**
|
|
42
98
|
* @license
|
|
43
99
|
* Copyright 2024 Google LLC
|
|
@@ -303,12 +359,30 @@ const ResponseModality = {
|
|
|
303
359
|
/**
|
|
304
360
|
* <b>(EXPERIMENTAL)</b>
|
|
305
361
|
* Determines whether inference happens on-device or in-cloud.
|
|
362
|
+
*
|
|
363
|
+
* @remarks
|
|
364
|
+
* <b>PREFER_ON_DEVICE:</b> Attempt to make inference calls using an
|
|
365
|
+
* on-device model. If on-device inference is not available, the SDK
|
|
366
|
+
* will fall back to using a cloud-hosted model.
|
|
367
|
+
* <br/>
|
|
368
|
+
* <b>ONLY_ON_DEVICE:</b> Only attempt to make inference calls using an
|
|
369
|
+
* on-device model. The SDK will not fall back to a cloud-hosted model.
|
|
370
|
+
* If on-device inference is not available, inference methods will throw.
|
|
371
|
+
* <br/>
|
|
372
|
+
* <b>ONLY_IN_CLOUD:</b> Only attempt to make inference calls using a
|
|
373
|
+
* cloud-hosted model. The SDK will not fall back to an on-device model.
|
|
374
|
+
* <br/>
|
|
375
|
+
* <b>PREFER_IN_CLOUD:</b> Attempt to make inference calls to a
|
|
376
|
+
* cloud-hosted model. If not available, the SDK will fall back to an
|
|
377
|
+
* on-device model.
|
|
378
|
+
*
|
|
306
379
|
* @public
|
|
307
380
|
*/
|
|
308
381
|
const InferenceMode = {
|
|
309
382
|
'PREFER_ON_DEVICE': 'prefer_on_device',
|
|
310
383
|
'ONLY_ON_DEVICE': 'only_on_device',
|
|
311
|
-
'ONLY_IN_CLOUD': 'only_in_cloud'
|
|
384
|
+
'ONLY_IN_CLOUD': 'only_in_cloud',
|
|
385
|
+
'PREFER_IN_CLOUD': 'prefer_in_cloud'
|
|
312
386
|
};
|
|
313
387
|
|
|
314
388
|
/**
|
|
@@ -657,105 +731,6 @@ class VertexAIBackend extends Backend {
|
|
|
657
731
|
}
|
|
658
732
|
}
|
|
659
733
|
|
|
660
|
-
/**
|
|
661
|
-
* @license
|
|
662
|
-
* Copyright 2024 Google LLC
|
|
663
|
-
*
|
|
664
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
665
|
-
* you may not use this file except in compliance with the License.
|
|
666
|
-
* You may obtain a copy of the License at
|
|
667
|
-
*
|
|
668
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
669
|
-
*
|
|
670
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
671
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
672
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
673
|
-
* See the License for the specific language governing permissions and
|
|
674
|
-
* limitations under the License.
|
|
675
|
-
*/
|
|
676
|
-
class AIService {
|
|
677
|
-
constructor(app, backend, authProvider, appCheckProvider, chromeAdapterFactory) {
|
|
678
|
-
this.app = app;
|
|
679
|
-
this.backend = backend;
|
|
680
|
-
this.chromeAdapterFactory = chromeAdapterFactory;
|
|
681
|
-
const appCheck = appCheckProvider?.getImmediate({ optional: true });
|
|
682
|
-
const auth = authProvider?.getImmediate({ optional: true });
|
|
683
|
-
this.auth = auth || null;
|
|
684
|
-
this.appCheck = appCheck || null;
|
|
685
|
-
if (backend instanceof VertexAIBackend) {
|
|
686
|
-
this.location = backend.location;
|
|
687
|
-
}
|
|
688
|
-
else {
|
|
689
|
-
this.location = '';
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
_delete() {
|
|
693
|
-
return Promise.resolve();
|
|
694
|
-
}
|
|
695
|
-
set options(optionsToSet) {
|
|
696
|
-
this._options = optionsToSet;
|
|
697
|
-
}
|
|
698
|
-
get options() {
|
|
699
|
-
return this._options;
|
|
700
|
-
}
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
/**
|
|
704
|
-
* @license
|
|
705
|
-
* Copyright 2024 Google LLC
|
|
706
|
-
*
|
|
707
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
708
|
-
* you may not use this file except in compliance with the License.
|
|
709
|
-
* You may obtain a copy of the License at
|
|
710
|
-
*
|
|
711
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
712
|
-
*
|
|
713
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
714
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
715
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
716
|
-
* See the License for the specific language governing permissions and
|
|
717
|
-
* limitations under the License.
|
|
718
|
-
*/
|
|
719
|
-
/**
|
|
720
|
-
* Error class for the Firebase AI SDK.
|
|
721
|
-
*
|
|
722
|
-
* @public
|
|
723
|
-
*/
|
|
724
|
-
class AIError extends util.FirebaseError {
|
|
725
|
-
/**
|
|
726
|
-
* Constructs a new instance of the `AIError` class.
|
|
727
|
-
*
|
|
728
|
-
* @param code - The error code from {@link (AIErrorCode:type)}.
|
|
729
|
-
* @param message - A human-readable message describing the error.
|
|
730
|
-
* @param customErrorData - Optional error data.
|
|
731
|
-
*/
|
|
732
|
-
constructor(code, message, customErrorData) {
|
|
733
|
-
// Match error format used by FirebaseError from ErrorFactory
|
|
734
|
-
const service = AI_TYPE;
|
|
735
|
-
const fullCode = `${service}/${code}`;
|
|
736
|
-
const fullMessage = `${service}: ${message} (${fullCode})`;
|
|
737
|
-
super(code, fullMessage);
|
|
738
|
-
this.code = code;
|
|
739
|
-
this.customErrorData = customErrorData;
|
|
740
|
-
// FirebaseError initializes a stack trace, but it assumes the error is created from the error
|
|
741
|
-
// factory. Since we break this assumption, we set the stack trace to be originating from this
|
|
742
|
-
// constructor.
|
|
743
|
-
// This is only supported in V8.
|
|
744
|
-
if (Error.captureStackTrace) {
|
|
745
|
-
// Allows us to initialize the stack trace without including the constructor itself at the
|
|
746
|
-
// top level of the stack trace.
|
|
747
|
-
Error.captureStackTrace(this, AIError);
|
|
748
|
-
}
|
|
749
|
-
// Allows instanceof AIError in ES5/ES6
|
|
750
|
-
// https://github.com/Microsoft/TypeScript-wiki/blob/master/Breaking-Changes.md#extending-built-ins-like-error-array-and-map-may-no-longer-work
|
|
751
|
-
// TODO(dlarocque): Replace this with `new.target`: https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget
|
|
752
|
-
// which we can now use since we no longer target ES5.
|
|
753
|
-
Object.setPrototypeOf(this, AIError.prototype);
|
|
754
|
-
// Since Error is an interface, we don't inherit toString and so we define it ourselves.
|
|
755
|
-
this.toString = () => fullMessage;
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
|
|
759
734
|
/**
|
|
760
735
|
* @license
|
|
761
736
|
* Copyright 2025 Google LLC
|
|
@@ -816,7 +791,7 @@ function decodeInstanceIdentifier(instanceIdentifier) {
|
|
|
816
791
|
|
|
817
792
|
/**
|
|
818
793
|
* @license
|
|
819
|
-
* Copyright
|
|
794
|
+
* Copyright 2024 Google LLC
|
|
820
795
|
*
|
|
821
796
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
822
797
|
* you may not use this file except in compliance with the License.
|
|
@@ -830,113 +805,300 @@ function decodeInstanceIdentifier(instanceIdentifier) {
|
|
|
830
805
|
* See the License for the specific language governing permissions and
|
|
831
806
|
* limitations under the License.
|
|
832
807
|
*/
|
|
808
|
+
const logger = new logger$1.Logger('@firebase/vertexai');
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* @internal
|
|
812
|
+
*/
|
|
813
|
+
var Availability;
|
|
814
|
+
(function (Availability) {
|
|
815
|
+
Availability["UNAVAILABLE"] = "unavailable";
|
|
816
|
+
Availability["DOWNLOADABLE"] = "downloadable";
|
|
817
|
+
Availability["DOWNLOADING"] = "downloading";
|
|
818
|
+
Availability["AVAILABLE"] = "available";
|
|
819
|
+
})(Availability || (Availability = {}));
|
|
820
|
+
|
|
833
821
|
/**
|
|
834
|
-
*
|
|
822
|
+
* @license
|
|
823
|
+
* Copyright 2025 Google LLC
|
|
835
824
|
*
|
|
836
|
-
*
|
|
837
|
-
*
|
|
825
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
826
|
+
* you may not use this file except in compliance with the License.
|
|
827
|
+
* You may obtain a copy of the License at
|
|
838
828
|
*
|
|
839
|
-
*
|
|
829
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
830
|
+
*
|
|
831
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
832
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
833
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
834
|
+
* See the License for the specific language governing permissions and
|
|
835
|
+
* limitations under the License.
|
|
840
836
|
*/
|
|
841
|
-
|
|
837
|
+
/**
|
|
838
|
+
* Defines an inference "backend" that uses Chrome's on-device model,
|
|
839
|
+
* and encapsulates logic for detecting when on-device inference is
|
|
840
|
+
* possible.
|
|
841
|
+
*/
|
|
842
|
+
class ChromeAdapterImpl {
|
|
843
|
+
constructor(languageModelProvider, mode, onDeviceParams = {
|
|
844
|
+
createOptions: {
|
|
845
|
+
// Defaults to support image inputs for convenience.
|
|
846
|
+
expectedInputs: [{ type: 'image' }]
|
|
847
|
+
}
|
|
848
|
+
}) {
|
|
849
|
+
this.languageModelProvider = languageModelProvider;
|
|
850
|
+
this.mode = mode;
|
|
851
|
+
this.onDeviceParams = onDeviceParams;
|
|
852
|
+
this.isDownloading = false;
|
|
853
|
+
}
|
|
842
854
|
/**
|
|
843
|
-
*
|
|
844
|
-
*
|
|
845
|
-
* This constructor should only be called from subclasses that provide
|
|
846
|
-
* a model API.
|
|
855
|
+
* Checks if a given request can be made on-device.
|
|
847
856
|
*
|
|
848
|
-
*
|
|
849
|
-
*
|
|
850
|
-
*
|
|
851
|
-
*
|
|
852
|
-
*
|
|
857
|
+
* Encapsulates a few concerns:
|
|
858
|
+
* the mode
|
|
859
|
+
* API existence
|
|
860
|
+
* prompt formatting
|
|
861
|
+
* model availability, including triggering download if necessary
|
|
853
862
|
*
|
|
854
|
-
* @throws If the `apiKey` or `projectId` fields are missing in your
|
|
855
|
-
* Firebase config.
|
|
856
863
|
*
|
|
857
|
-
*
|
|
864
|
+
* Pros: callers needn't be concerned with details of on-device availability.</p>
|
|
865
|
+
* Cons: this method spans a few concerns and splits request validation from usage.
|
|
866
|
+
* If instance variables weren't already part of the API, we could consider a better
|
|
867
|
+
* separation of concerns.
|
|
858
868
|
*/
|
|
859
|
-
|
|
860
|
-
if (!
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
else if (!ai.app?.options?.projectId) {
|
|
864
|
-
throw new AIError(AIErrorCode.NO_PROJECT_ID, `The "projectId" field is empty in the local Firebase config. Firebase AI requires this field to contain a valid project ID.`);
|
|
869
|
+
async isAvailable(request) {
|
|
870
|
+
if (!this.mode) {
|
|
871
|
+
logger.debug(`On-device inference unavailable because mode is undefined.`);
|
|
872
|
+
return false;
|
|
865
873
|
}
|
|
866
|
-
|
|
867
|
-
|
|
874
|
+
if (this.mode === InferenceMode.ONLY_IN_CLOUD) {
|
|
875
|
+
logger.debug(`On-device inference unavailable because mode is "only_in_cloud".`);
|
|
876
|
+
return false;
|
|
868
877
|
}
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
location: ai.location,
|
|
876
|
-
backend: ai.backend
|
|
877
|
-
};
|
|
878
|
-
if (app._isFirebaseServerApp(ai.app) && ai.app.settings.appCheckToken) {
|
|
879
|
-
const token = ai.app.settings.appCheckToken;
|
|
880
|
-
this._apiSettings.getAppCheckToken = () => {
|
|
881
|
-
return Promise.resolve({ token });
|
|
882
|
-
};
|
|
878
|
+
// Triggers out-of-band download so model will eventually become available.
|
|
879
|
+
const availability = await this.downloadIfAvailable();
|
|
880
|
+
if (this.mode === InferenceMode.ONLY_ON_DEVICE) {
|
|
881
|
+
// If it will never be available due to API inavailability, throw.
|
|
882
|
+
if (availability === Availability.UNAVAILABLE) {
|
|
883
|
+
throw new AIError(AIErrorCode.API_NOT_ENABLED, 'Local LanguageModel API not available in this environment.');
|
|
883
884
|
}
|
|
884
|
-
else if (
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
}
|
|
885
|
+
else if (availability === Availability.DOWNLOADABLE ||
|
|
886
|
+
availability === Availability.DOWNLOADING) {
|
|
887
|
+
// TODO(chholland): Better user experience during download - progress?
|
|
888
|
+
logger.debug(`Waiting for download of LanguageModel to complete.`);
|
|
889
|
+
await this.downloadPromise;
|
|
890
|
+
return true;
|
|
891
891
|
}
|
|
892
|
-
|
|
893
|
-
|
|
892
|
+
return true;
|
|
893
|
+
}
|
|
894
|
+
// Applies prefer_on_device logic.
|
|
895
|
+
if (availability !== Availability.AVAILABLE) {
|
|
896
|
+
logger.debug(`On-device inference unavailable because availability is "${availability}".`);
|
|
897
|
+
return false;
|
|
898
|
+
}
|
|
899
|
+
if (!ChromeAdapterImpl.isOnDeviceRequest(request)) {
|
|
900
|
+
logger.debug(`On-device inference unavailable because request is incompatible.`);
|
|
901
|
+
return false;
|
|
902
|
+
}
|
|
903
|
+
return true;
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Generates content on device.
|
|
907
|
+
*
|
|
908
|
+
* @remarks
|
|
909
|
+
* This is comparable to {@link GenerativeModel.generateContent} for generating content in
|
|
910
|
+
* Cloud.
|
|
911
|
+
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
912
|
+
* @returns {@link Response}, so we can reuse common response formatting.
|
|
913
|
+
*/
|
|
914
|
+
async generateContent(request) {
|
|
915
|
+
const session = await this.createSession();
|
|
916
|
+
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
917
|
+
const text = await session.prompt(contents, this.onDeviceParams.promptOptions);
|
|
918
|
+
return ChromeAdapterImpl.toResponse(text);
|
|
919
|
+
}
|
|
920
|
+
/**
|
|
921
|
+
* Generates content stream on device.
|
|
922
|
+
*
|
|
923
|
+
* @remarks
|
|
924
|
+
* This is comparable to {@link GenerativeModel.generateContentStream} for generating content in
|
|
925
|
+
* Cloud.
|
|
926
|
+
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
927
|
+
* @returns {@link Response}, so we can reuse common response formatting.
|
|
928
|
+
*/
|
|
929
|
+
async generateContentStream(request) {
|
|
930
|
+
const session = await this.createSession();
|
|
931
|
+
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
932
|
+
const stream = session.promptStreaming(contents, this.onDeviceParams.promptOptions);
|
|
933
|
+
return ChromeAdapterImpl.toStreamResponse(stream);
|
|
934
|
+
}
|
|
935
|
+
async countTokens(_request) {
|
|
936
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'Count Tokens is not yet available for on-device model.');
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Asserts inference for the given request can be performed by an on-device model.
|
|
940
|
+
*/
|
|
941
|
+
static isOnDeviceRequest(request) {
|
|
942
|
+
// Returns false if the prompt is empty.
|
|
943
|
+
if (request.contents.length === 0) {
|
|
944
|
+
logger.debug('Empty prompt rejected for on-device inference.');
|
|
945
|
+
return false;
|
|
946
|
+
}
|
|
947
|
+
for (const content of request.contents) {
|
|
948
|
+
if (content.role === 'function') {
|
|
949
|
+
logger.debug(`"Function" role rejected for on-device inference.`);
|
|
950
|
+
return false;
|
|
951
|
+
}
|
|
952
|
+
// Returns false if request contains an image with an unsupported mime type.
|
|
953
|
+
for (const part of content.parts) {
|
|
954
|
+
if (part.inlineData &&
|
|
955
|
+
ChromeAdapterImpl.SUPPORTED_MIME_TYPES.indexOf(part.inlineData.mimeType) === -1) {
|
|
956
|
+
logger.debug(`Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.`);
|
|
957
|
+
return false;
|
|
958
|
+
}
|
|
894
959
|
}
|
|
895
|
-
this.model = AIModel.normalizeModelName(modelName, this._apiSettings.backend.backendType);
|
|
896
960
|
}
|
|
961
|
+
return true;
|
|
897
962
|
}
|
|
898
963
|
/**
|
|
899
|
-
*
|
|
964
|
+
* Encapsulates logic to get availability and download a model if one is downloadable.
|
|
965
|
+
*/
|
|
966
|
+
async downloadIfAvailable() {
|
|
967
|
+
const availability = await this.languageModelProvider?.availability(this.onDeviceParams.createOptions);
|
|
968
|
+
if (availability === Availability.DOWNLOADABLE) {
|
|
969
|
+
this.download();
|
|
970
|
+
}
|
|
971
|
+
return availability;
|
|
972
|
+
}
|
|
973
|
+
/**
|
|
974
|
+
* Triggers out-of-band download of an on-device model.
|
|
900
975
|
*
|
|
901
|
-
*
|
|
902
|
-
*
|
|
976
|
+
* Chrome only downloads models as needed. Chrome knows a model is needed when code calls
|
|
977
|
+
* LanguageModel.create.
|
|
903
978
|
*
|
|
904
|
-
*
|
|
979
|
+
* Since Chrome manages the download, the SDK can only avoid redundant download requests by
|
|
980
|
+
* tracking if a download has previously been requested.
|
|
905
981
|
*/
|
|
906
|
-
|
|
907
|
-
if (
|
|
908
|
-
return
|
|
982
|
+
download() {
|
|
983
|
+
if (this.isDownloading) {
|
|
984
|
+
return;
|
|
909
985
|
}
|
|
910
|
-
|
|
911
|
-
|
|
986
|
+
this.isDownloading = true;
|
|
987
|
+
this.downloadPromise = this.languageModelProvider
|
|
988
|
+
?.create(this.onDeviceParams.createOptions)
|
|
989
|
+
.finally(() => {
|
|
990
|
+
this.isDownloading = false;
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object.
|
|
995
|
+
*/
|
|
996
|
+
static async toLanguageModelMessage(content) {
|
|
997
|
+
const languageModelMessageContents = await Promise.all(content.parts.map(ChromeAdapterImpl.toLanguageModelMessageContent));
|
|
998
|
+
return {
|
|
999
|
+
role: ChromeAdapterImpl.toLanguageModelMessageRole(content.role),
|
|
1000
|
+
content: languageModelMessageContents
|
|
1001
|
+
};
|
|
1002
|
+
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object.
|
|
1005
|
+
*/
|
|
1006
|
+
static async toLanguageModelMessageContent(part) {
|
|
1007
|
+
if (part.text) {
|
|
1008
|
+
return {
|
|
1009
|
+
type: 'text',
|
|
1010
|
+
value: part.text
|
|
1011
|
+
};
|
|
912
1012
|
}
|
|
1013
|
+
else if (part.inlineData) {
|
|
1014
|
+
const formattedImageContent = await fetch(`data:${part.inlineData.mimeType};base64,${part.inlineData.data}`);
|
|
1015
|
+
const imageBlob = await formattedImageContent.blob();
|
|
1016
|
+
const imageBitmap = await createImageBitmap(imageBlob);
|
|
1017
|
+
return {
|
|
1018
|
+
type: 'image',
|
|
1019
|
+
value: imageBitmap
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, `Processing of this Part type is not currently supported.`);
|
|
913
1023
|
}
|
|
914
1024
|
/**
|
|
915
|
-
* @
|
|
1025
|
+
* Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string.
|
|
916
1026
|
*/
|
|
917
|
-
static
|
|
918
|
-
|
|
1027
|
+
static toLanguageModelMessageRole(role) {
|
|
1028
|
+
// Assumes 'function' rule has been filtered by isOnDeviceRequest
|
|
1029
|
+
return role === 'model' ? 'assistant' : 'user';
|
|
919
1030
|
}
|
|
920
1031
|
/**
|
|
921
|
-
*
|
|
1032
|
+
* Abstracts Chrome session creation.
|
|
1033
|
+
*
|
|
1034
|
+
* Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all
|
|
1035
|
+
* inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all
|
|
1036
|
+
* inference.
|
|
1037
|
+
*
|
|
1038
|
+
* Chrome will remove a model from memory if it's no longer in use, so this method ensures a
|
|
1039
|
+
* new session is created before an old session is destroyed.
|
|
922
1040
|
*/
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
if (modelName.startsWith('models/')) {
|
|
927
|
-
// Add 'publishers/google' if the user is only passing in 'models/model-name'.
|
|
928
|
-
model = `publishers/google/${modelName}`;
|
|
929
|
-
}
|
|
930
|
-
else {
|
|
931
|
-
// Any other custom format (e.g. tuned models) must be passed in correctly.
|
|
932
|
-
model = modelName;
|
|
933
|
-
}
|
|
1041
|
+
async createSession() {
|
|
1042
|
+
if (!this.languageModelProvider) {
|
|
1043
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'Chrome AI requested for unsupported browser version.');
|
|
934
1044
|
}
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1045
|
+
const newSession = await this.languageModelProvider.create(this.onDeviceParams.createOptions);
|
|
1046
|
+
if (this.oldSession) {
|
|
1047
|
+
this.oldSession.destroy();
|
|
938
1048
|
}
|
|
939
|
-
|
|
1049
|
+
// Holds session reference, so model isn't unloaded from memory.
|
|
1050
|
+
this.oldSession = newSession;
|
|
1051
|
+
return newSession;
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Formats string returned by Chrome as a {@link Response} returned by Firebase AI.
|
|
1055
|
+
*/
|
|
1056
|
+
static toResponse(text) {
|
|
1057
|
+
return {
|
|
1058
|
+
json: async () => ({
|
|
1059
|
+
candidates: [
|
|
1060
|
+
{
|
|
1061
|
+
content: {
|
|
1062
|
+
parts: [{ text }]
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
]
|
|
1066
|
+
})
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
/**
|
|
1070
|
+
* Formats string stream returned by Chrome as SSE returned by Firebase AI.
|
|
1071
|
+
*/
|
|
1072
|
+
static toStreamResponse(stream) {
|
|
1073
|
+
const encoder = new TextEncoder();
|
|
1074
|
+
return {
|
|
1075
|
+
body: stream.pipeThrough(new TransformStream({
|
|
1076
|
+
transform(chunk, controller) {
|
|
1077
|
+
const json = JSON.stringify({
|
|
1078
|
+
candidates: [
|
|
1079
|
+
{
|
|
1080
|
+
content: {
|
|
1081
|
+
role: 'model',
|
|
1082
|
+
parts: [{ text: chunk }]
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
]
|
|
1086
|
+
});
|
|
1087
|
+
controller.enqueue(encoder.encode(`data: ${json}\n\n`));
|
|
1088
|
+
}
|
|
1089
|
+
}))
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
// Visible for testing
|
|
1094
|
+
ChromeAdapterImpl.SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png'];
|
|
1095
|
+
/**
|
|
1096
|
+
* Creates a ChromeAdapterImpl on demand.
|
|
1097
|
+
*/
|
|
1098
|
+
function chromeAdapterFactory(mode, window, params) {
|
|
1099
|
+
// Do not initialize a ChromeAdapter if we are not in hybrid mode.
|
|
1100
|
+
if (typeof window !== 'undefined' && mode) {
|
|
1101
|
+
return new ChromeAdapterImpl(window.LanguageModel, mode, params);
|
|
940
1102
|
}
|
|
941
1103
|
}
|
|
942
1104
|
|
|
@@ -956,18 +1118,197 @@ class AIModel {
|
|
|
956
1118
|
* See the License for the specific language governing permissions and
|
|
957
1119
|
* limitations under the License.
|
|
958
1120
|
*/
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
1121
|
+
class AIService {
|
|
1122
|
+
constructor(app, backend, authProvider, appCheckProvider, chromeAdapterFactory) {
|
|
1123
|
+
this.app = app;
|
|
1124
|
+
this.backend = backend;
|
|
1125
|
+
this.chromeAdapterFactory = chromeAdapterFactory;
|
|
1126
|
+
const appCheck = appCheckProvider?.getImmediate({ optional: true });
|
|
1127
|
+
const auth = authProvider?.getImmediate({ optional: true });
|
|
1128
|
+
this.auth = auth || null;
|
|
1129
|
+
this.appCheck = appCheck || null;
|
|
1130
|
+
if (backend instanceof VertexAIBackend) {
|
|
1131
|
+
this.location = backend.location;
|
|
1132
|
+
}
|
|
1133
|
+
else {
|
|
1134
|
+
this.location = '';
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
_delete() {
|
|
1138
|
+
return Promise.resolve();
|
|
1139
|
+
}
|
|
1140
|
+
set options(optionsToSet) {
|
|
1141
|
+
this._options = optionsToSet;
|
|
1142
|
+
}
|
|
1143
|
+
get options() {
|
|
1144
|
+
return this._options;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
/**
|
|
1149
|
+
* @license
|
|
1150
|
+
* Copyright 2025 Google LLC
|
|
1151
|
+
*
|
|
1152
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1153
|
+
* you may not use this file except in compliance with the License.
|
|
1154
|
+
* You may obtain a copy of the License at
|
|
1155
|
+
*
|
|
1156
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1157
|
+
*
|
|
1158
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
1159
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1160
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1161
|
+
* See the License for the specific language governing permissions and
|
|
1162
|
+
* limitations under the License.
|
|
1163
|
+
*/
|
|
1164
|
+
function factory(container, { instanceIdentifier }) {
|
|
1165
|
+
if (!instanceIdentifier) {
|
|
1166
|
+
throw new AIError(AIErrorCode.ERROR, 'AIService instance identifier is undefined.');
|
|
1167
|
+
}
|
|
1168
|
+
const backend = decodeInstanceIdentifier(instanceIdentifier);
|
|
1169
|
+
// getImmediate for FirebaseApp will always succeed
|
|
1170
|
+
const app = container.getProvider('app').getImmediate();
|
|
1171
|
+
const auth = container.getProvider('auth-internal');
|
|
1172
|
+
const appCheckProvider = container.getProvider('app-check-internal');
|
|
1173
|
+
return new AIService(app, backend, auth, appCheckProvider, chromeAdapterFactory);
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* @license
|
|
1178
|
+
* Copyright 2025 Google LLC
|
|
1179
|
+
*
|
|
1180
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1181
|
+
* you may not use this file except in compliance with the License.
|
|
1182
|
+
* You may obtain a copy of the License at
|
|
1183
|
+
*
|
|
1184
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1185
|
+
*
|
|
1186
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
1187
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1188
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1189
|
+
* See the License for the specific language governing permissions and
|
|
1190
|
+
* limitations under the License.
|
|
1191
|
+
*/
|
|
1192
|
+
/**
|
|
1193
|
+
* Base class for Firebase AI model APIs.
|
|
1194
|
+
*
|
|
1195
|
+
* Instances of this class are associated with a specific Firebase AI {@link Backend}
|
|
1196
|
+
* and provide methods for interacting with the configured generative model.
|
|
1197
|
+
*
|
|
1198
|
+
* @public
|
|
1199
|
+
*/
|
|
1200
|
+
class AIModel {
|
|
1201
|
+
/**
|
|
1202
|
+
* Constructs a new instance of the {@link AIModel} class.
|
|
1203
|
+
*
|
|
1204
|
+
* This constructor should only be called from subclasses that provide
|
|
1205
|
+
* a model API.
|
|
1206
|
+
*
|
|
1207
|
+
* @param ai - an {@link AI} instance.
|
|
1208
|
+
* @param modelName - The name of the model being used. It can be in one of the following formats:
|
|
1209
|
+
* - `my-model` (short name, will resolve to `publishers/google/models/my-model`)
|
|
1210
|
+
* - `models/my-model` (will resolve to `publishers/google/models/my-model`)
|
|
1211
|
+
* - `publishers/my-publisher/models/my-model` (fully qualified model name)
|
|
1212
|
+
*
|
|
1213
|
+
* @throws If the `apiKey` or `projectId` fields are missing in your
|
|
1214
|
+
* Firebase config.
|
|
1215
|
+
*
|
|
1216
|
+
* @internal
|
|
1217
|
+
*/
|
|
1218
|
+
constructor(ai, modelName) {
|
|
1219
|
+
if (!ai.app?.options?.apiKey) {
|
|
1220
|
+
throw new AIError(AIErrorCode.NO_API_KEY, `The "apiKey" field is empty in the local Firebase config. Firebase AI requires this field to contain a valid API key.`);
|
|
1221
|
+
}
|
|
1222
|
+
else if (!ai.app?.options?.projectId) {
|
|
1223
|
+
throw new AIError(AIErrorCode.NO_PROJECT_ID, `The "projectId" field is empty in the local Firebase config. Firebase AI requires this field to contain a valid project ID.`);
|
|
1224
|
+
}
|
|
1225
|
+
else if (!ai.app?.options?.appId) {
|
|
1226
|
+
throw new AIError(AIErrorCode.NO_APP_ID, `The "appId" field is empty in the local Firebase config. Firebase AI requires this field to contain a valid app ID.`);
|
|
1227
|
+
}
|
|
1228
|
+
else {
|
|
1229
|
+
this._apiSettings = {
|
|
1230
|
+
apiKey: ai.app.options.apiKey,
|
|
1231
|
+
project: ai.app.options.projectId,
|
|
1232
|
+
appId: ai.app.options.appId,
|
|
1233
|
+
automaticDataCollectionEnabled: ai.app.automaticDataCollectionEnabled,
|
|
1234
|
+
location: ai.location,
|
|
1235
|
+
backend: ai.backend
|
|
1236
|
+
};
|
|
1237
|
+
if (app._isFirebaseServerApp(ai.app) && ai.app.settings.appCheckToken) {
|
|
1238
|
+
const token = ai.app.settings.appCheckToken;
|
|
1239
|
+
this._apiSettings.getAppCheckToken = () => {
|
|
1240
|
+
return Promise.resolve({ token });
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
1243
|
+
else if (ai.appCheck) {
|
|
1244
|
+
if (ai.options?.useLimitedUseAppCheckTokens) {
|
|
1245
|
+
this._apiSettings.getAppCheckToken = () => ai.appCheck.getLimitedUseToken();
|
|
1246
|
+
}
|
|
1247
|
+
else {
|
|
1248
|
+
this._apiSettings.getAppCheckToken = () => ai.appCheck.getToken();
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
if (ai.auth) {
|
|
1252
|
+
this._apiSettings.getAuthToken = () => ai.auth.getToken();
|
|
1253
|
+
}
|
|
1254
|
+
this.model = AIModel.normalizeModelName(modelName, this._apiSettings.backend.backendType);
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
/**
|
|
1258
|
+
* Normalizes the given model name to a fully qualified model resource name.
|
|
1259
|
+
*
|
|
1260
|
+
* @param modelName - The model name to normalize.
|
|
1261
|
+
* @returns The fully qualified model resource name.
|
|
1262
|
+
*
|
|
1263
|
+
* @internal
|
|
1264
|
+
*/
|
|
1265
|
+
static normalizeModelName(modelName, backendType) {
|
|
1266
|
+
if (backendType === BackendType.GOOGLE_AI) {
|
|
1267
|
+
return AIModel.normalizeGoogleAIModelName(modelName);
|
|
1268
|
+
}
|
|
1269
|
+
else {
|
|
1270
|
+
return AIModel.normalizeVertexAIModelName(modelName);
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
/**
|
|
1274
|
+
* @internal
|
|
1275
|
+
*/
|
|
1276
|
+
static normalizeGoogleAIModelName(modelName) {
|
|
1277
|
+
return `models/${modelName}`;
|
|
1278
|
+
}
|
|
1279
|
+
/**
|
|
1280
|
+
* @internal
|
|
1281
|
+
*/
|
|
1282
|
+
static normalizeVertexAIModelName(modelName) {
|
|
1283
|
+
let model;
|
|
1284
|
+
if (modelName.includes('/')) {
|
|
1285
|
+
if (modelName.startsWith('models/')) {
|
|
1286
|
+
// Add 'publishers/google' if the user is only passing in 'models/model-name'.
|
|
1287
|
+
model = `publishers/google/${modelName}`;
|
|
1288
|
+
}
|
|
1289
|
+
else {
|
|
1290
|
+
// Any other custom format (e.g. tuned models) must be passed in correctly.
|
|
1291
|
+
model = modelName;
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
else {
|
|
1295
|
+
// If path is not included, assume it's a non-tuned model.
|
|
1296
|
+
model = `publishers/google/models/${modelName}`;
|
|
1297
|
+
}
|
|
1298
|
+
return model;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
/**
|
|
1303
|
+
* @license
|
|
1304
|
+
* Copyright 2024 Google LLC
|
|
1305
|
+
*
|
|
1306
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1307
|
+
* you may not use this file except in compliance with the License.
|
|
1308
|
+
* You may obtain a copy of the License at
|
|
1309
|
+
*
|
|
1310
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1311
|
+
*
|
|
971
1312
|
* Unless required by applicable law or agreed to in writing, software
|
|
972
1313
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
973
1314
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
@@ -1738,6 +2079,72 @@ function aggregateResponses(responses) {
|
|
|
1738
2079
|
return aggregatedResponse;
|
|
1739
2080
|
}
|
|
1740
2081
|
|
|
2082
|
+
/**
|
|
2083
|
+
* @license
|
|
2084
|
+
* Copyright 2025 Google LLC
|
|
2085
|
+
*
|
|
2086
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
2087
|
+
* you may not use this file except in compliance with the License.
|
|
2088
|
+
* You may obtain a copy of the License at
|
|
2089
|
+
*
|
|
2090
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
2091
|
+
*
|
|
2092
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
2093
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
2094
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
2095
|
+
* See the License for the specific language governing permissions and
|
|
2096
|
+
* limitations under the License.
|
|
2097
|
+
*/
|
|
2098
|
+
const errorsCausingFallback = [
|
|
2099
|
+
// most network errors
|
|
2100
|
+
AIErrorCode.FETCH_ERROR,
|
|
2101
|
+
// fallback code for all other errors in makeRequest
|
|
2102
|
+
AIErrorCode.ERROR,
|
|
2103
|
+
// error due to API not being enabled in project
|
|
2104
|
+
AIErrorCode.API_NOT_ENABLED
|
|
2105
|
+
];
|
|
2106
|
+
/**
|
|
2107
|
+
* Dispatches a request to the appropriate backend (on-device or in-cloud)
|
|
2108
|
+
* based on the inference mode.
|
|
2109
|
+
*
|
|
2110
|
+
* @param request - The request to be sent.
|
|
2111
|
+
* @param chromeAdapter - The on-device model adapter.
|
|
2112
|
+
* @param onDeviceCall - The function to call for on-device inference.
|
|
2113
|
+
* @param inCloudCall - The function to call for in-cloud inference.
|
|
2114
|
+
* @returns The response from the backend.
|
|
2115
|
+
*/
|
|
2116
|
+
async function callCloudOrDevice(request, chromeAdapter, onDeviceCall, inCloudCall) {
|
|
2117
|
+
if (!chromeAdapter) {
|
|
2118
|
+
return inCloudCall();
|
|
2119
|
+
}
|
|
2120
|
+
switch (chromeAdapter.mode) {
|
|
2121
|
+
case InferenceMode.ONLY_ON_DEVICE:
|
|
2122
|
+
if (await chromeAdapter.isAvailable(request)) {
|
|
2123
|
+
return onDeviceCall();
|
|
2124
|
+
}
|
|
2125
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'Inference mode is ONLY_ON_DEVICE, but an on-device model is not available.');
|
|
2126
|
+
case InferenceMode.ONLY_IN_CLOUD:
|
|
2127
|
+
return inCloudCall();
|
|
2128
|
+
case InferenceMode.PREFER_IN_CLOUD:
|
|
2129
|
+
try {
|
|
2130
|
+
return await inCloudCall();
|
|
2131
|
+
}
|
|
2132
|
+
catch (e) {
|
|
2133
|
+
if (e instanceof AIError && errorsCausingFallback.includes(e.code)) {
|
|
2134
|
+
return onDeviceCall();
|
|
2135
|
+
}
|
|
2136
|
+
throw e;
|
|
2137
|
+
}
|
|
2138
|
+
case InferenceMode.PREFER_ON_DEVICE:
|
|
2139
|
+
if (await chromeAdapter.isAvailable(request)) {
|
|
2140
|
+
return onDeviceCall();
|
|
2141
|
+
}
|
|
2142
|
+
return inCloudCall();
|
|
2143
|
+
default:
|
|
2144
|
+
throw new AIError(AIErrorCode.ERROR, `Unexpected infererence mode: ${chromeAdapter.mode}`);
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
|
|
1741
2148
|
/**
|
|
1742
2149
|
* @license
|
|
1743
2150
|
* Copyright 2024 Google LLC
|
|
@@ -1762,13 +2169,7 @@ async function generateContentStreamOnCloud(apiSettings, model, params, requestO
|
|
|
1762
2169
|
/* stream */ true, JSON.stringify(params), requestOptions);
|
|
1763
2170
|
}
|
|
1764
2171
|
async function generateContentStream(apiSettings, model, params, chromeAdapter, requestOptions) {
|
|
1765
|
-
|
|
1766
|
-
if (chromeAdapter && (await chromeAdapter.isAvailable(params))) {
|
|
1767
|
-
response = await chromeAdapter.generateContentStream(params);
|
|
1768
|
-
}
|
|
1769
|
-
else {
|
|
1770
|
-
response = await generateContentStreamOnCloud(apiSettings, model, params, requestOptions);
|
|
1771
|
-
}
|
|
2172
|
+
const response = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContentStream(params), () => generateContentStreamOnCloud(apiSettings, model, params, requestOptions));
|
|
1772
2173
|
return processStream(response, apiSettings); // TODO: Map streaming responses
|
|
1773
2174
|
}
|
|
1774
2175
|
async function generateContentOnCloud(apiSettings, model, params, requestOptions) {
|
|
@@ -1779,13 +2180,7 @@ async function generateContentOnCloud(apiSettings, model, params, requestOptions
|
|
|
1779
2180
|
/* stream */ false, JSON.stringify(params), requestOptions);
|
|
1780
2181
|
}
|
|
1781
2182
|
async function generateContent(apiSettings, model, params, chromeAdapter, requestOptions) {
|
|
1782
|
-
|
|
1783
|
-
if (chromeAdapter && (await chromeAdapter.isAvailable(params))) {
|
|
1784
|
-
response = await chromeAdapter.generateContent(params);
|
|
1785
|
-
}
|
|
1786
|
-
else {
|
|
1787
|
-
response = await generateContentOnCloud(apiSettings, model, params, requestOptions);
|
|
1788
|
-
}
|
|
2183
|
+
const response = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContent(params), () => generateContentOnCloud(apiSettings, model, params, requestOptions));
|
|
1789
2184
|
const generateContentResponse = await processGenerateContentResponse(response, apiSettings);
|
|
1790
2185
|
const enhancedResponse = createEnhancedContentResponse(generateContentResponse);
|
|
1791
2186
|
return {
|
|
@@ -2196,8 +2591,8 @@ async function countTokensOnCloud(apiSettings, model, params, requestOptions) {
|
|
|
2196
2591
|
return response.json();
|
|
2197
2592
|
}
|
|
2198
2593
|
async function countTokens(apiSettings, model, params, chromeAdapter, requestOptions) {
|
|
2199
|
-
if (chromeAdapter
|
|
2200
|
-
|
|
2594
|
+
if (chromeAdapter?.mode === InferenceMode.ONLY_ON_DEVICE) {
|
|
2595
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'countTokens() is not supported for on-device models.');
|
|
2201
2596
|
}
|
|
2202
2597
|
return countTokensOnCloud(apiSettings, model, params, requestOptions);
|
|
2203
2598
|
}
|
|
@@ -3611,316 +4006,10 @@ function getLiveGenerativeModel(ai, modelParams) {
|
|
|
3611
4006
|
}
|
|
3612
4007
|
|
|
3613
4008
|
/**
|
|
3614
|
-
*
|
|
3615
|
-
*/
|
|
3616
|
-
var Availability;
|
|
3617
|
-
(function (Availability) {
|
|
3618
|
-
Availability["UNAVAILABLE"] = "unavailable";
|
|
3619
|
-
Availability["DOWNLOADABLE"] = "downloadable";
|
|
3620
|
-
Availability["DOWNLOADING"] = "downloading";
|
|
3621
|
-
Availability["AVAILABLE"] = "available";
|
|
3622
|
-
})(Availability || (Availability = {}));
|
|
3623
|
-
|
|
3624
|
-
/**
|
|
3625
|
-
* @license
|
|
3626
|
-
* Copyright 2025 Google LLC
|
|
3627
|
-
*
|
|
3628
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3629
|
-
* you may not use this file except in compliance with the License.
|
|
3630
|
-
* You may obtain a copy of the License at
|
|
3631
|
-
*
|
|
3632
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3633
|
-
*
|
|
3634
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
3635
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3636
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3637
|
-
* See the License for the specific language governing permissions and
|
|
3638
|
-
* limitations under the License.
|
|
3639
|
-
*/
|
|
3640
|
-
/**
|
|
3641
|
-
* Defines an inference "backend" that uses Chrome's on-device model,
|
|
3642
|
-
* and encapsulates logic for detecting when on-device inference is
|
|
3643
|
-
* possible.
|
|
3644
|
-
*/
|
|
3645
|
-
class ChromeAdapterImpl {
|
|
3646
|
-
constructor(languageModelProvider, mode, onDeviceParams = {
|
|
3647
|
-
createOptions: {
|
|
3648
|
-
// Defaults to support image inputs for convenience.
|
|
3649
|
-
expectedInputs: [{ type: 'image' }]
|
|
3650
|
-
}
|
|
3651
|
-
}) {
|
|
3652
|
-
this.languageModelProvider = languageModelProvider;
|
|
3653
|
-
this.mode = mode;
|
|
3654
|
-
this.onDeviceParams = onDeviceParams;
|
|
3655
|
-
this.isDownloading = false;
|
|
3656
|
-
}
|
|
3657
|
-
/**
|
|
3658
|
-
* Checks if a given request can be made on-device.
|
|
3659
|
-
*
|
|
3660
|
-
* Encapsulates a few concerns:
|
|
3661
|
-
* the mode
|
|
3662
|
-
* API existence
|
|
3663
|
-
* prompt formatting
|
|
3664
|
-
* model availability, including triggering download if necessary
|
|
3665
|
-
*
|
|
3666
|
-
*
|
|
3667
|
-
* Pros: callers needn't be concerned with details of on-device availability.</p>
|
|
3668
|
-
* Cons: this method spans a few concerns and splits request validation from usage.
|
|
3669
|
-
* If instance variables weren't already part of the API, we could consider a better
|
|
3670
|
-
* separation of concerns.
|
|
3671
|
-
*/
|
|
3672
|
-
async isAvailable(request) {
|
|
3673
|
-
if (!this.mode) {
|
|
3674
|
-
logger.debug(`On-device inference unavailable because mode is undefined.`);
|
|
3675
|
-
return false;
|
|
3676
|
-
}
|
|
3677
|
-
if (this.mode === InferenceMode.ONLY_IN_CLOUD) {
|
|
3678
|
-
logger.debug(`On-device inference unavailable because mode is "only_in_cloud".`);
|
|
3679
|
-
return false;
|
|
3680
|
-
}
|
|
3681
|
-
// Triggers out-of-band download so model will eventually become available.
|
|
3682
|
-
const availability = await this.downloadIfAvailable();
|
|
3683
|
-
if (this.mode === InferenceMode.ONLY_ON_DEVICE) {
|
|
3684
|
-
// If it will never be available due to API inavailability, throw.
|
|
3685
|
-
if (availability === Availability.UNAVAILABLE) {
|
|
3686
|
-
throw new AIError(AIErrorCode.API_NOT_ENABLED, 'Local LanguageModel API not available in this environment.');
|
|
3687
|
-
}
|
|
3688
|
-
else if (availability === Availability.DOWNLOADABLE ||
|
|
3689
|
-
availability === Availability.DOWNLOADING) {
|
|
3690
|
-
// TODO(chholland): Better user experience during download - progress?
|
|
3691
|
-
logger.debug(`Waiting for download of LanguageModel to complete.`);
|
|
3692
|
-
await this.downloadPromise;
|
|
3693
|
-
return true;
|
|
3694
|
-
}
|
|
3695
|
-
return true;
|
|
3696
|
-
}
|
|
3697
|
-
// Applies prefer_on_device logic.
|
|
3698
|
-
if (availability !== Availability.AVAILABLE) {
|
|
3699
|
-
logger.debug(`On-device inference unavailable because availability is "${availability}".`);
|
|
3700
|
-
return false;
|
|
3701
|
-
}
|
|
3702
|
-
if (!ChromeAdapterImpl.isOnDeviceRequest(request)) {
|
|
3703
|
-
logger.debug(`On-device inference unavailable because request is incompatible.`);
|
|
3704
|
-
return false;
|
|
3705
|
-
}
|
|
3706
|
-
return true;
|
|
3707
|
-
}
|
|
3708
|
-
/**
|
|
3709
|
-
* Generates content on device.
|
|
3710
|
-
*
|
|
3711
|
-
* @remarks
|
|
3712
|
-
* This is comparable to {@link GenerativeModel.generateContent} for generating content in
|
|
3713
|
-
* Cloud.
|
|
3714
|
-
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
3715
|
-
* @returns {@link Response}, so we can reuse common response formatting.
|
|
3716
|
-
*/
|
|
3717
|
-
async generateContent(request) {
|
|
3718
|
-
const session = await this.createSession();
|
|
3719
|
-
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
3720
|
-
const text = await session.prompt(contents, this.onDeviceParams.promptOptions);
|
|
3721
|
-
return ChromeAdapterImpl.toResponse(text);
|
|
3722
|
-
}
|
|
3723
|
-
/**
|
|
3724
|
-
* Generates content stream on device.
|
|
3725
|
-
*
|
|
3726
|
-
* @remarks
|
|
3727
|
-
* This is comparable to {@link GenerativeModel.generateContentStream} for generating content in
|
|
3728
|
-
* Cloud.
|
|
3729
|
-
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
3730
|
-
* @returns {@link Response}, so we can reuse common response formatting.
|
|
3731
|
-
*/
|
|
3732
|
-
async generateContentStream(request) {
|
|
3733
|
-
const session = await this.createSession();
|
|
3734
|
-
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
3735
|
-
const stream = session.promptStreaming(contents, this.onDeviceParams.promptOptions);
|
|
3736
|
-
return ChromeAdapterImpl.toStreamResponse(stream);
|
|
3737
|
-
}
|
|
3738
|
-
async countTokens(_request) {
|
|
3739
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'Count Tokens is not yet available for on-device model.');
|
|
3740
|
-
}
|
|
3741
|
-
/**
|
|
3742
|
-
* Asserts inference for the given request can be performed by an on-device model.
|
|
3743
|
-
*/
|
|
3744
|
-
static isOnDeviceRequest(request) {
|
|
3745
|
-
// Returns false if the prompt is empty.
|
|
3746
|
-
if (request.contents.length === 0) {
|
|
3747
|
-
logger.debug('Empty prompt rejected for on-device inference.');
|
|
3748
|
-
return false;
|
|
3749
|
-
}
|
|
3750
|
-
for (const content of request.contents) {
|
|
3751
|
-
if (content.role === 'function') {
|
|
3752
|
-
logger.debug(`"Function" role rejected for on-device inference.`);
|
|
3753
|
-
return false;
|
|
3754
|
-
}
|
|
3755
|
-
// Returns false if request contains an image with an unsupported mime type.
|
|
3756
|
-
for (const part of content.parts) {
|
|
3757
|
-
if (part.inlineData &&
|
|
3758
|
-
ChromeAdapterImpl.SUPPORTED_MIME_TYPES.indexOf(part.inlineData.mimeType) === -1) {
|
|
3759
|
-
logger.debug(`Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.`);
|
|
3760
|
-
return false;
|
|
3761
|
-
}
|
|
3762
|
-
}
|
|
3763
|
-
}
|
|
3764
|
-
return true;
|
|
3765
|
-
}
|
|
3766
|
-
/**
|
|
3767
|
-
* Encapsulates logic to get availability and download a model if one is downloadable.
|
|
3768
|
-
*/
|
|
3769
|
-
async downloadIfAvailable() {
|
|
3770
|
-
const availability = await this.languageModelProvider?.availability(this.onDeviceParams.createOptions);
|
|
3771
|
-
if (availability === Availability.DOWNLOADABLE) {
|
|
3772
|
-
this.download();
|
|
3773
|
-
}
|
|
3774
|
-
return availability;
|
|
3775
|
-
}
|
|
3776
|
-
/**
|
|
3777
|
-
* Triggers out-of-band download of an on-device model.
|
|
3778
|
-
*
|
|
3779
|
-
* Chrome only downloads models as needed. Chrome knows a model is needed when code calls
|
|
3780
|
-
* LanguageModel.create.
|
|
3781
|
-
*
|
|
3782
|
-
* Since Chrome manages the download, the SDK can only avoid redundant download requests by
|
|
3783
|
-
* tracking if a download has previously been requested.
|
|
3784
|
-
*/
|
|
3785
|
-
download() {
|
|
3786
|
-
if (this.isDownloading) {
|
|
3787
|
-
return;
|
|
3788
|
-
}
|
|
3789
|
-
this.isDownloading = true;
|
|
3790
|
-
this.downloadPromise = this.languageModelProvider
|
|
3791
|
-
?.create(this.onDeviceParams.createOptions)
|
|
3792
|
-
.finally(() => {
|
|
3793
|
-
this.isDownloading = false;
|
|
3794
|
-
});
|
|
3795
|
-
}
|
|
3796
|
-
/**
|
|
3797
|
-
* Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object.
|
|
3798
|
-
*/
|
|
3799
|
-
static async toLanguageModelMessage(content) {
|
|
3800
|
-
const languageModelMessageContents = await Promise.all(content.parts.map(ChromeAdapterImpl.toLanguageModelMessageContent));
|
|
3801
|
-
return {
|
|
3802
|
-
role: ChromeAdapterImpl.toLanguageModelMessageRole(content.role),
|
|
3803
|
-
content: languageModelMessageContents
|
|
3804
|
-
};
|
|
3805
|
-
}
|
|
3806
|
-
/**
|
|
3807
|
-
* Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object.
|
|
3808
|
-
*/
|
|
3809
|
-
static async toLanguageModelMessageContent(part) {
|
|
3810
|
-
if (part.text) {
|
|
3811
|
-
return {
|
|
3812
|
-
type: 'text',
|
|
3813
|
-
value: part.text
|
|
3814
|
-
};
|
|
3815
|
-
}
|
|
3816
|
-
else if (part.inlineData) {
|
|
3817
|
-
const formattedImageContent = await fetch(`data:${part.inlineData.mimeType};base64,${part.inlineData.data}`);
|
|
3818
|
-
const imageBlob = await formattedImageContent.blob();
|
|
3819
|
-
const imageBitmap = await createImageBitmap(imageBlob);
|
|
3820
|
-
return {
|
|
3821
|
-
type: 'image',
|
|
3822
|
-
value: imageBitmap
|
|
3823
|
-
};
|
|
3824
|
-
}
|
|
3825
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, `Processing of this Part type is not currently supported.`);
|
|
3826
|
-
}
|
|
3827
|
-
/**
|
|
3828
|
-
* Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string.
|
|
3829
|
-
*/
|
|
3830
|
-
static toLanguageModelMessageRole(role) {
|
|
3831
|
-
// Assumes 'function' rule has been filtered by isOnDeviceRequest
|
|
3832
|
-
return role === 'model' ? 'assistant' : 'user';
|
|
3833
|
-
}
|
|
3834
|
-
/**
|
|
3835
|
-
* Abstracts Chrome session creation.
|
|
3836
|
-
*
|
|
3837
|
-
* Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all
|
|
3838
|
-
* inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all
|
|
3839
|
-
* inference.
|
|
3840
|
-
*
|
|
3841
|
-
* Chrome will remove a model from memory if it's no longer in use, so this method ensures a
|
|
3842
|
-
* new session is created before an old session is destroyed.
|
|
3843
|
-
*/
|
|
3844
|
-
async createSession() {
|
|
3845
|
-
if (!this.languageModelProvider) {
|
|
3846
|
-
throw new AIError(AIErrorCode.UNSUPPORTED, 'Chrome AI requested for unsupported browser version.');
|
|
3847
|
-
}
|
|
3848
|
-
const newSession = await this.languageModelProvider.create(this.onDeviceParams.createOptions);
|
|
3849
|
-
if (this.oldSession) {
|
|
3850
|
-
this.oldSession.destroy();
|
|
3851
|
-
}
|
|
3852
|
-
// Holds session reference, so model isn't unloaded from memory.
|
|
3853
|
-
this.oldSession = newSession;
|
|
3854
|
-
return newSession;
|
|
3855
|
-
}
|
|
3856
|
-
/**
|
|
3857
|
-
* Formats string returned by Chrome as a {@link Response} returned by Firebase AI.
|
|
3858
|
-
*/
|
|
3859
|
-
static toResponse(text) {
|
|
3860
|
-
return {
|
|
3861
|
-
json: async () => ({
|
|
3862
|
-
candidates: [
|
|
3863
|
-
{
|
|
3864
|
-
content: {
|
|
3865
|
-
parts: [{ text }]
|
|
3866
|
-
}
|
|
3867
|
-
}
|
|
3868
|
-
]
|
|
3869
|
-
})
|
|
3870
|
-
};
|
|
3871
|
-
}
|
|
3872
|
-
/**
|
|
3873
|
-
* Formats string stream returned by Chrome as SSE returned by Firebase AI.
|
|
3874
|
-
*/
|
|
3875
|
-
static toStreamResponse(stream) {
|
|
3876
|
-
const encoder = new TextEncoder();
|
|
3877
|
-
return {
|
|
3878
|
-
body: stream.pipeThrough(new TransformStream({
|
|
3879
|
-
transform(chunk, controller) {
|
|
3880
|
-
const json = JSON.stringify({
|
|
3881
|
-
candidates: [
|
|
3882
|
-
{
|
|
3883
|
-
content: {
|
|
3884
|
-
role: 'model',
|
|
3885
|
-
parts: [{ text: chunk }]
|
|
3886
|
-
}
|
|
3887
|
-
}
|
|
3888
|
-
]
|
|
3889
|
-
});
|
|
3890
|
-
controller.enqueue(encoder.encode(`data: ${json}\n\n`));
|
|
3891
|
-
}
|
|
3892
|
-
}))
|
|
3893
|
-
};
|
|
3894
|
-
}
|
|
3895
|
-
}
|
|
3896
|
-
// Visible for testing
|
|
3897
|
-
ChromeAdapterImpl.SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png'];
|
|
3898
|
-
/**
|
|
3899
|
-
* Creates a ChromeAdapterImpl on demand.
|
|
3900
|
-
*/
|
|
3901
|
-
function chromeAdapterFactory(mode, window, params) {
|
|
3902
|
-
// Do not initialize a ChromeAdapter if we are not in hybrid mode.
|
|
3903
|
-
if (typeof window !== 'undefined' && mode) {
|
|
3904
|
-
return new ChromeAdapterImpl(window.LanguageModel, mode, params);
|
|
3905
|
-
}
|
|
3906
|
-
}
|
|
3907
|
-
|
|
3908
|
-
/**
|
|
3909
|
-
* The Firebase AI Web SDK.
|
|
4009
|
+
* The Firebase AI Web SDK.
|
|
3910
4010
|
*
|
|
3911
4011
|
* @packageDocumentation
|
|
3912
4012
|
*/
|
|
3913
|
-
function factory(container, { instanceIdentifier }) {
|
|
3914
|
-
if (!instanceIdentifier) {
|
|
3915
|
-
throw new AIError(AIErrorCode.ERROR, 'AIService instance identifier is undefined.');
|
|
3916
|
-
}
|
|
3917
|
-
const backend = decodeInstanceIdentifier(instanceIdentifier);
|
|
3918
|
-
// getImmediate for FirebaseApp will always succeed
|
|
3919
|
-
const app = container.getProvider('app').getImmediate();
|
|
3920
|
-
const auth = container.getProvider('auth-internal');
|
|
3921
|
-
const appCheckProvider = container.getProvider('app-check-internal');
|
|
3922
|
-
return new AIService(app, backend, auth, appCheckProvider, chromeAdapterFactory);
|
|
3923
|
-
}
|
|
3924
4013
|
function registerAI() {
|
|
3925
4014
|
app._registerComponent(new component.Component(AI_TYPE, factory, "PUBLIC" /* ComponentType.PUBLIC */).setMultipleInstances(true));
|
|
3926
4015
|
app.registerVersion(name, version);
|
|
@@ -3967,7 +4056,6 @@ exports.Schema = Schema;
|
|
|
3967
4056
|
exports.SchemaType = SchemaType;
|
|
3968
4057
|
exports.StringSchema = StringSchema;
|
|
3969
4058
|
exports.VertexAIBackend = VertexAIBackend;
|
|
3970
|
-
exports.factory = factory;
|
|
3971
4059
|
exports.getAI = getAI;
|
|
3972
4060
|
exports.getGenerativeModel = getGenerativeModel;
|
|
3973
4061
|
exports.getImagenModel = getImagenModel;
|