@firebase/ai 2.1.0-canary.9b63cd60e → 2.1.0-canary.cbef6c6e5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +115 -1
- package/dist/ai.d.ts +153 -1
- package/dist/esm/index.esm.js +374 -362
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/index.d.ts +2 -1
- package/dist/esm/src/methods/chrome-adapter.d.ts +7 -3
- package/dist/esm/src/service.d.ts +4 -2
- package/dist/esm/src/types/imagen/internal.d.ts +10 -0
- package/dist/index.cjs.js +374 -362
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +12 -295
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +12 -295
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/index.d.ts +2 -1
- package/dist/src/methods/chrome-adapter.d.ts +7 -3
- package/dist/src/service.d.ts +4 -2
- package/dist/src/types/imagen/internal.d.ts +10 -0
- package/package.json +10 -8
package/dist/esm/index.esm.js
CHANGED
|
@@ -4,7 +4,7 @@ import { FirebaseError, getModularInstance } from '@firebase/util';
|
|
|
4
4
|
import { Logger } from '@firebase/logger';
|
|
5
5
|
|
|
6
6
|
var name = "@firebase/ai";
|
|
7
|
-
var version = "2.1.0-canary.
|
|
7
|
+
var version = "2.1.0-canary.cbef6c6e5";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* @license
|
|
@@ -636,9 +636,10 @@ class VertexAIBackend extends Backend {
|
|
|
636
636
|
* limitations under the License.
|
|
637
637
|
*/
|
|
638
638
|
class AIService {
|
|
639
|
-
constructor(app, backend, authProvider, appCheckProvider) {
|
|
639
|
+
constructor(app, backend, authProvider, appCheckProvider, chromeAdapterFactory) {
|
|
640
640
|
this.app = app;
|
|
641
641
|
this.backend = backend;
|
|
642
|
+
this.chromeAdapterFactory = chromeAdapterFactory;
|
|
642
643
|
const appCheck = appCheckProvider?.getImmediate({ optional: true });
|
|
643
644
|
const auth = authProvider?.getImmediate({ optional: true });
|
|
644
645
|
this.auth = auth || null;
|
|
@@ -1324,8 +1325,9 @@ async function handlePredictResponse(response) {
|
|
|
1324
1325
|
gcsURI: prediction.gcsUri
|
|
1325
1326
|
});
|
|
1326
1327
|
}
|
|
1328
|
+
else if (prediction.safetyAttributes) ;
|
|
1327
1329
|
else {
|
|
1328
|
-
throw new AIError(AIErrorCode.RESPONSE_ERROR, `
|
|
1330
|
+
throw new AIError(AIErrorCode.RESPONSE_ERROR, `Unexpected element in 'predictions' array in response: '${JSON.stringify(prediction)}'`);
|
|
1329
1331
|
}
|
|
1330
1332
|
}
|
|
1331
1333
|
return { images, filteredReason };
|
|
@@ -1866,7 +1868,8 @@ function createPredictRequestBody(prompt, { gcsURI, imageFormat, addWatermark, n
|
|
|
1866
1868
|
addWatermark,
|
|
1867
1869
|
safetyFilterLevel,
|
|
1868
1870
|
personGeneration: personFilterLevel,
|
|
1869
|
-
includeRaiReason: true
|
|
1871
|
+
includeRaiReason: true,
|
|
1872
|
+
includeSafetyAttributes: true
|
|
1870
1873
|
}
|
|
1871
1874
|
};
|
|
1872
1875
|
return body;
|
|
@@ -2345,20 +2348,9 @@ class ImagenModel extends AIModel {
|
|
|
2345
2348
|
}
|
|
2346
2349
|
}
|
|
2347
2350
|
|
|
2348
|
-
/**
|
|
2349
|
-
* @internal
|
|
2350
|
-
*/
|
|
2351
|
-
var Availability;
|
|
2352
|
-
(function (Availability) {
|
|
2353
|
-
Availability["UNAVAILABLE"] = "unavailable";
|
|
2354
|
-
Availability["DOWNLOADABLE"] = "downloadable";
|
|
2355
|
-
Availability["DOWNLOADING"] = "downloading";
|
|
2356
|
-
Availability["AVAILABLE"] = "available";
|
|
2357
|
-
})(Availability || (Availability = {}));
|
|
2358
|
-
|
|
2359
2351
|
/**
|
|
2360
2352
|
* @license
|
|
2361
|
-
* Copyright
|
|
2353
|
+
* Copyright 2024 Google LLC
|
|
2362
2354
|
*
|
|
2363
2355
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
2364
2356
|
* you may not use this file except in compliance with the License.
|
|
@@ -2373,362 +2365,87 @@ var Availability;
|
|
|
2373
2365
|
* limitations under the License.
|
|
2374
2366
|
*/
|
|
2375
2367
|
/**
|
|
2376
|
-
*
|
|
2377
|
-
*
|
|
2378
|
-
*
|
|
2368
|
+
* Parent class encompassing all Schema types, with static methods that
|
|
2369
|
+
* allow building specific Schema types. This class can be converted with
|
|
2370
|
+
* `JSON.stringify()` into a JSON string accepted by Vertex AI REST endpoints.
|
|
2371
|
+
* (This string conversion is automatically done when calling SDK methods.)
|
|
2372
|
+
* @public
|
|
2379
2373
|
*/
|
|
2380
|
-
class
|
|
2381
|
-
constructor(
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2374
|
+
class Schema {
|
|
2375
|
+
constructor(schemaParams) {
|
|
2376
|
+
// TODO(dlarocque): Enforce this with union types
|
|
2377
|
+
if (!schemaParams.type && !schemaParams.anyOf) {
|
|
2378
|
+
throw new AIError(AIErrorCode.INVALID_SCHEMA, "A schema must have either a 'type' or an 'anyOf' array of sub-schemas.");
|
|
2385
2379
|
}
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2380
|
+
// eslint-disable-next-line guard-for-in
|
|
2381
|
+
for (const paramKey in schemaParams) {
|
|
2382
|
+
this[paramKey] = schemaParams[paramKey];
|
|
2383
|
+
}
|
|
2384
|
+
// Ensure these are explicitly set to avoid TS errors.
|
|
2385
|
+
this.type = schemaParams.type;
|
|
2386
|
+
this.format = schemaParams.hasOwnProperty('format')
|
|
2387
|
+
? schemaParams.format
|
|
2388
|
+
: undefined;
|
|
2389
|
+
this.nullable = schemaParams.hasOwnProperty('nullable')
|
|
2390
|
+
? !!schemaParams.nullable
|
|
2391
|
+
: false;
|
|
2391
2392
|
}
|
|
2392
2393
|
/**
|
|
2393
|
-
*
|
|
2394
|
-
*
|
|
2395
|
-
*
|
|
2396
|
-
* the mode
|
|
2397
|
-
* API existence
|
|
2398
|
-
* prompt formatting
|
|
2399
|
-
* model availability, including triggering download if necessary
|
|
2400
|
-
*
|
|
2401
|
-
*
|
|
2402
|
-
* Pros: callers needn't be concerned with details of on-device availability.</p>
|
|
2403
|
-
* Cons: this method spans a few concerns and splits request validation from usage.
|
|
2404
|
-
* If instance variables weren't already part of the API, we could consider a better
|
|
2405
|
-
* separation of concerns.
|
|
2394
|
+
* Defines how this Schema should be serialized as JSON.
|
|
2395
|
+
* See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior
|
|
2396
|
+
* @internal
|
|
2406
2397
|
*/
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
// Triggers out-of-band download so model will eventually become available.
|
|
2417
|
-
const availability = await this.downloadIfAvailable();
|
|
2418
|
-
if (this.mode === InferenceMode.ONLY_ON_DEVICE) {
|
|
2419
|
-
// If it will never be available due to API inavailability, throw.
|
|
2420
|
-
if (availability === Availability.UNAVAILABLE) {
|
|
2421
|
-
throw new AIError(AIErrorCode.API_NOT_ENABLED, 'Local LanguageModel API not available in this environment.');
|
|
2422
|
-
}
|
|
2423
|
-
else if (availability === Availability.DOWNLOADABLE ||
|
|
2424
|
-
availability === Availability.DOWNLOADING) {
|
|
2425
|
-
// TODO(chholland): Better user experience during download - progress?
|
|
2426
|
-
logger.debug(`Waiting for download of LanguageModel to complete.`);
|
|
2427
|
-
await this.downloadPromise;
|
|
2428
|
-
return true;
|
|
2398
|
+
toJSON() {
|
|
2399
|
+
const obj = {
|
|
2400
|
+
type: this.type
|
|
2401
|
+
};
|
|
2402
|
+
for (const prop in this) {
|
|
2403
|
+
if (this.hasOwnProperty(prop) && this[prop] !== undefined) {
|
|
2404
|
+
if (prop !== 'required' || this.type === SchemaType.OBJECT) {
|
|
2405
|
+
obj[prop] = this[prop];
|
|
2406
|
+
}
|
|
2429
2407
|
}
|
|
2430
|
-
return true;
|
|
2431
2408
|
}
|
|
2432
|
-
|
|
2433
|
-
if (availability !== Availability.AVAILABLE) {
|
|
2434
|
-
logger.debug(`On-device inference unavailable because availability is "${availability}".`);
|
|
2435
|
-
return false;
|
|
2436
|
-
}
|
|
2437
|
-
if (!ChromeAdapterImpl.isOnDeviceRequest(request)) {
|
|
2438
|
-
logger.debug(`On-device inference unavailable because request is incompatible.`);
|
|
2439
|
-
return false;
|
|
2440
|
-
}
|
|
2441
|
-
return true;
|
|
2409
|
+
return obj;
|
|
2442
2410
|
}
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
*
|
|
2446
|
-
* @remarks
|
|
2447
|
-
* This is comparable to {@link GenerativeModel.generateContent} for generating content in
|
|
2448
|
-
* Cloud.
|
|
2449
|
-
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
2450
|
-
* @returns {@link Response}, so we can reuse common response formatting.
|
|
2451
|
-
*/
|
|
2452
|
-
async generateContent(request) {
|
|
2453
|
-
const session = await this.createSession();
|
|
2454
|
-
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
2455
|
-
const text = await session.prompt(contents, this.onDeviceParams.promptOptions);
|
|
2456
|
-
return ChromeAdapterImpl.toResponse(text);
|
|
2411
|
+
static array(arrayParams) {
|
|
2412
|
+
return new ArraySchema(arrayParams, arrayParams.items);
|
|
2457
2413
|
}
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
*
|
|
2461
|
-
* @remarks
|
|
2462
|
-
* This is comparable to {@link GenerativeModel.generateContentStream} for generating content in
|
|
2463
|
-
* Cloud.
|
|
2464
|
-
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
2465
|
-
* @returns {@link Response}, so we can reuse common response formatting.
|
|
2466
|
-
*/
|
|
2467
|
-
async generateContentStream(request) {
|
|
2468
|
-
const session = await this.createSession();
|
|
2469
|
-
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
2470
|
-
const stream = session.promptStreaming(contents, this.onDeviceParams.promptOptions);
|
|
2471
|
-
return ChromeAdapterImpl.toStreamResponse(stream);
|
|
2414
|
+
static object(objectParams) {
|
|
2415
|
+
return new ObjectSchema(objectParams, objectParams.properties, objectParams.optionalProperties);
|
|
2472
2416
|
}
|
|
2473
|
-
|
|
2474
|
-
|
|
2417
|
+
// eslint-disable-next-line id-blacklist
|
|
2418
|
+
static string(stringParams) {
|
|
2419
|
+
return new StringSchema(stringParams);
|
|
2475
2420
|
}
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
*/
|
|
2479
|
-
static isOnDeviceRequest(request) {
|
|
2480
|
-
// Returns false if the prompt is empty.
|
|
2481
|
-
if (request.contents.length === 0) {
|
|
2482
|
-
logger.debug('Empty prompt rejected for on-device inference.');
|
|
2483
|
-
return false;
|
|
2484
|
-
}
|
|
2485
|
-
for (const content of request.contents) {
|
|
2486
|
-
if (content.role === 'function') {
|
|
2487
|
-
logger.debug(`"Function" role rejected for on-device inference.`);
|
|
2488
|
-
return false;
|
|
2489
|
-
}
|
|
2490
|
-
// Returns false if request contains an image with an unsupported mime type.
|
|
2491
|
-
for (const part of content.parts) {
|
|
2492
|
-
if (part.inlineData &&
|
|
2493
|
-
ChromeAdapterImpl.SUPPORTED_MIME_TYPES.indexOf(part.inlineData.mimeType) === -1) {
|
|
2494
|
-
logger.debug(`Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.`);
|
|
2495
|
-
return false;
|
|
2496
|
-
}
|
|
2497
|
-
}
|
|
2498
|
-
}
|
|
2499
|
-
return true;
|
|
2421
|
+
static enumString(stringParams) {
|
|
2422
|
+
return new StringSchema(stringParams, stringParams.enum);
|
|
2500
2423
|
}
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
*/
|
|
2504
|
-
async downloadIfAvailable() {
|
|
2505
|
-
const availability = await this.languageModelProvider?.availability(this.onDeviceParams.createOptions);
|
|
2506
|
-
if (availability === Availability.DOWNLOADABLE) {
|
|
2507
|
-
this.download();
|
|
2508
|
-
}
|
|
2509
|
-
return availability;
|
|
2424
|
+
static integer(integerParams) {
|
|
2425
|
+
return new IntegerSchema(integerParams);
|
|
2510
2426
|
}
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
* Chrome only downloads models as needed. Chrome knows a model is needed when code calls
|
|
2515
|
-
* LanguageModel.create.
|
|
2516
|
-
*
|
|
2517
|
-
* Since Chrome manages the download, the SDK can only avoid redundant download requests by
|
|
2518
|
-
* tracking if a download has previously been requested.
|
|
2519
|
-
*/
|
|
2520
|
-
download() {
|
|
2521
|
-
if (this.isDownloading) {
|
|
2522
|
-
return;
|
|
2523
|
-
}
|
|
2524
|
-
this.isDownloading = true;
|
|
2525
|
-
this.downloadPromise = this.languageModelProvider
|
|
2526
|
-
?.create(this.onDeviceParams.createOptions)
|
|
2527
|
-
.finally(() => {
|
|
2528
|
-
this.isDownloading = false;
|
|
2529
|
-
});
|
|
2427
|
+
// eslint-disable-next-line id-blacklist
|
|
2428
|
+
static number(numberParams) {
|
|
2429
|
+
return new NumberSchema(numberParams);
|
|
2530
2430
|
}
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
static async toLanguageModelMessage(content) {
|
|
2535
|
-
const languageModelMessageContents = await Promise.all(content.parts.map(ChromeAdapterImpl.toLanguageModelMessageContent));
|
|
2536
|
-
return {
|
|
2537
|
-
role: ChromeAdapterImpl.toLanguageModelMessageRole(content.role),
|
|
2538
|
-
content: languageModelMessageContents
|
|
2539
|
-
};
|
|
2431
|
+
// eslint-disable-next-line id-blacklist
|
|
2432
|
+
static boolean(booleanParams) {
|
|
2433
|
+
return new BooleanSchema(booleanParams);
|
|
2540
2434
|
}
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
*/
|
|
2544
|
-
static async toLanguageModelMessageContent(part) {
|
|
2545
|
-
if (part.text) {
|
|
2546
|
-
return {
|
|
2547
|
-
type: 'text',
|
|
2548
|
-
value: part.text
|
|
2549
|
-
};
|
|
2550
|
-
}
|
|
2551
|
-
else if (part.inlineData) {
|
|
2552
|
-
const formattedImageContent = await fetch(`data:${part.inlineData.mimeType};base64,${part.inlineData.data}`);
|
|
2553
|
-
const imageBlob = await formattedImageContent.blob();
|
|
2554
|
-
const imageBitmap = await createImageBitmap(imageBlob);
|
|
2555
|
-
return {
|
|
2556
|
-
type: 'image',
|
|
2557
|
-
value: imageBitmap
|
|
2558
|
-
};
|
|
2559
|
-
}
|
|
2560
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, `Processing of this Part type is not currently supported.`);
|
|
2435
|
+
static anyOf(anyOfParams) {
|
|
2436
|
+
return new AnyOfSchema(anyOfParams);
|
|
2561
2437
|
}
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
* inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all
|
|
2574
|
-
* inference.
|
|
2575
|
-
*
|
|
2576
|
-
* Chrome will remove a model from memory if it's no longer in use, so this method ensures a
|
|
2577
|
-
* new session is created before an old session is destroyed.
|
|
2578
|
-
*/
|
|
2579
|
-
async createSession() {
|
|
2580
|
-
if (!this.languageModelProvider) {
|
|
2581
|
-
throw new AIError(AIErrorCode.UNSUPPORTED, 'Chrome AI requested for unsupported browser version.');
|
|
2582
|
-
}
|
|
2583
|
-
const newSession = await this.languageModelProvider.create(this.onDeviceParams.createOptions);
|
|
2584
|
-
if (this.oldSession) {
|
|
2585
|
-
this.oldSession.destroy();
|
|
2586
|
-
}
|
|
2587
|
-
// Holds session reference, so model isn't unloaded from memory.
|
|
2588
|
-
this.oldSession = newSession;
|
|
2589
|
-
return newSession;
|
|
2590
|
-
}
|
|
2591
|
-
/**
|
|
2592
|
-
* Formats string returned by Chrome as a {@link Response} returned by Firebase AI.
|
|
2593
|
-
*/
|
|
2594
|
-
static toResponse(text) {
|
|
2595
|
-
return {
|
|
2596
|
-
json: async () => ({
|
|
2597
|
-
candidates: [
|
|
2598
|
-
{
|
|
2599
|
-
content: {
|
|
2600
|
-
parts: [{ text }]
|
|
2601
|
-
}
|
|
2602
|
-
}
|
|
2603
|
-
]
|
|
2604
|
-
})
|
|
2605
|
-
};
|
|
2606
|
-
}
|
|
2607
|
-
/**
|
|
2608
|
-
* Formats string stream returned by Chrome as SSE returned by Firebase AI.
|
|
2609
|
-
*/
|
|
2610
|
-
static toStreamResponse(stream) {
|
|
2611
|
-
const encoder = new TextEncoder();
|
|
2612
|
-
return {
|
|
2613
|
-
body: stream.pipeThrough(new TransformStream({
|
|
2614
|
-
transform(chunk, controller) {
|
|
2615
|
-
const json = JSON.stringify({
|
|
2616
|
-
candidates: [
|
|
2617
|
-
{
|
|
2618
|
-
content: {
|
|
2619
|
-
role: 'model',
|
|
2620
|
-
parts: [{ text: chunk }]
|
|
2621
|
-
}
|
|
2622
|
-
}
|
|
2623
|
-
]
|
|
2624
|
-
});
|
|
2625
|
-
controller.enqueue(encoder.encode(`data: ${json}\n\n`));
|
|
2626
|
-
}
|
|
2627
|
-
}))
|
|
2628
|
-
};
|
|
2629
|
-
}
|
|
2630
|
-
}
|
|
2631
|
-
// Visible for testing
|
|
2632
|
-
ChromeAdapterImpl.SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png'];
|
|
2633
|
-
|
|
2634
|
-
/**
|
|
2635
|
-
* @license
|
|
2636
|
-
* Copyright 2024 Google LLC
|
|
2637
|
-
*
|
|
2638
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
2639
|
-
* you may not use this file except in compliance with the License.
|
|
2640
|
-
* You may obtain a copy of the License at
|
|
2641
|
-
*
|
|
2642
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
2643
|
-
*
|
|
2644
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
2645
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
2646
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
2647
|
-
* See the License for the specific language governing permissions and
|
|
2648
|
-
* limitations under the License.
|
|
2649
|
-
*/
|
|
2650
|
-
/**
|
|
2651
|
-
* Parent class encompassing all Schema types, with static methods that
|
|
2652
|
-
* allow building specific Schema types. This class can be converted with
|
|
2653
|
-
* `JSON.stringify()` into a JSON string accepted by Vertex AI REST endpoints.
|
|
2654
|
-
* (This string conversion is automatically done when calling SDK methods.)
|
|
2655
|
-
* @public
|
|
2656
|
-
*/
|
|
2657
|
-
class Schema {
|
|
2658
|
-
constructor(schemaParams) {
|
|
2659
|
-
// TODO(dlarocque): Enforce this with union types
|
|
2660
|
-
if (!schemaParams.type && !schemaParams.anyOf) {
|
|
2661
|
-
throw new AIError(AIErrorCode.INVALID_SCHEMA, "A schema must have either a 'type' or an 'anyOf' array of sub-schemas.");
|
|
2662
|
-
}
|
|
2663
|
-
// eslint-disable-next-line guard-for-in
|
|
2664
|
-
for (const paramKey in schemaParams) {
|
|
2665
|
-
this[paramKey] = schemaParams[paramKey];
|
|
2666
|
-
}
|
|
2667
|
-
// Ensure these are explicitly set to avoid TS errors.
|
|
2668
|
-
this.type = schemaParams.type;
|
|
2669
|
-
this.format = schemaParams.hasOwnProperty('format')
|
|
2670
|
-
? schemaParams.format
|
|
2671
|
-
: undefined;
|
|
2672
|
-
this.nullable = schemaParams.hasOwnProperty('nullable')
|
|
2673
|
-
? !!schemaParams.nullable
|
|
2674
|
-
: false;
|
|
2675
|
-
}
|
|
2676
|
-
/**
|
|
2677
|
-
* Defines how this Schema should be serialized as JSON.
|
|
2678
|
-
* See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior
|
|
2679
|
-
* @internal
|
|
2680
|
-
*/
|
|
2681
|
-
toJSON() {
|
|
2682
|
-
const obj = {
|
|
2683
|
-
type: this.type
|
|
2684
|
-
};
|
|
2685
|
-
for (const prop in this) {
|
|
2686
|
-
if (this.hasOwnProperty(prop) && this[prop] !== undefined) {
|
|
2687
|
-
if (prop !== 'required' || this.type === SchemaType.OBJECT) {
|
|
2688
|
-
obj[prop] = this[prop];
|
|
2689
|
-
}
|
|
2690
|
-
}
|
|
2691
|
-
}
|
|
2692
|
-
return obj;
|
|
2693
|
-
}
|
|
2694
|
-
static array(arrayParams) {
|
|
2695
|
-
return new ArraySchema(arrayParams, arrayParams.items);
|
|
2696
|
-
}
|
|
2697
|
-
static object(objectParams) {
|
|
2698
|
-
return new ObjectSchema(objectParams, objectParams.properties, objectParams.optionalProperties);
|
|
2699
|
-
}
|
|
2700
|
-
// eslint-disable-next-line id-blacklist
|
|
2701
|
-
static string(stringParams) {
|
|
2702
|
-
return new StringSchema(stringParams);
|
|
2703
|
-
}
|
|
2704
|
-
static enumString(stringParams) {
|
|
2705
|
-
return new StringSchema(stringParams, stringParams.enum);
|
|
2706
|
-
}
|
|
2707
|
-
static integer(integerParams) {
|
|
2708
|
-
return new IntegerSchema(integerParams);
|
|
2709
|
-
}
|
|
2710
|
-
// eslint-disable-next-line id-blacklist
|
|
2711
|
-
static number(numberParams) {
|
|
2712
|
-
return new NumberSchema(numberParams);
|
|
2713
|
-
}
|
|
2714
|
-
// eslint-disable-next-line id-blacklist
|
|
2715
|
-
static boolean(booleanParams) {
|
|
2716
|
-
return new BooleanSchema(booleanParams);
|
|
2717
|
-
}
|
|
2718
|
-
static anyOf(anyOfParams) {
|
|
2719
|
-
return new AnyOfSchema(anyOfParams);
|
|
2720
|
-
}
|
|
2721
|
-
}
|
|
2722
|
-
/**
|
|
2723
|
-
* Schema class for "integer" types.
|
|
2724
|
-
* @public
|
|
2725
|
-
*/
|
|
2726
|
-
class IntegerSchema extends Schema {
|
|
2727
|
-
constructor(schemaParams) {
|
|
2728
|
-
super({
|
|
2729
|
-
type: SchemaType.INTEGER,
|
|
2730
|
-
...schemaParams
|
|
2731
|
-
});
|
|
2438
|
+
}
|
|
2439
|
+
/**
|
|
2440
|
+
* Schema class for "integer" types.
|
|
2441
|
+
* @public
|
|
2442
|
+
*/
|
|
2443
|
+
class IntegerSchema extends Schema {
|
|
2444
|
+
constructor(schemaParams) {
|
|
2445
|
+
super({
|
|
2446
|
+
type: SchemaType.INTEGER,
|
|
2447
|
+
...schemaParams
|
|
2448
|
+
});
|
|
2732
2449
|
}
|
|
2733
2450
|
}
|
|
2734
2451
|
/**
|
|
@@ -3018,11 +2735,11 @@ function getGenerativeModel(ai, modelParams, requestOptions) {
|
|
|
3018
2735
|
if (!inCloudParams.model) {
|
|
3019
2736
|
throw new AIError(AIErrorCode.NO_MODEL, `Must provide a model name. Example: getGenerativeModel({ model: 'my-model-name' })`);
|
|
3020
2737
|
}
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
2738
|
+
/**
|
|
2739
|
+
* An AIService registered by index.node.ts will not have a
|
|
2740
|
+
* chromeAdapterFactory() method.
|
|
2741
|
+
*/
|
|
2742
|
+
const chromeAdapter = ai.chromeAdapterFactory?.(hybridParams.mode, typeof window === 'undefined' ? undefined : window, hybridParams.onDeviceParams);
|
|
3026
2743
|
return new GenerativeModel(ai, inCloudParams, requestOptions, chromeAdapter);
|
|
3027
2744
|
}
|
|
3028
2745
|
/**
|
|
@@ -3046,6 +2763,301 @@ function getImagenModel(ai, modelParams, requestOptions) {
|
|
|
3046
2763
|
return new ImagenModel(ai, modelParams, requestOptions);
|
|
3047
2764
|
}
|
|
3048
2765
|
|
|
2766
|
+
/**
|
|
2767
|
+
* @internal
|
|
2768
|
+
*/
|
|
2769
|
+
var Availability;
|
|
2770
|
+
(function (Availability) {
|
|
2771
|
+
Availability["UNAVAILABLE"] = "unavailable";
|
|
2772
|
+
Availability["DOWNLOADABLE"] = "downloadable";
|
|
2773
|
+
Availability["DOWNLOADING"] = "downloading";
|
|
2774
|
+
Availability["AVAILABLE"] = "available";
|
|
2775
|
+
})(Availability || (Availability = {}));
|
|
2776
|
+
|
|
2777
|
+
/**
|
|
2778
|
+
* @license
|
|
2779
|
+
* Copyright 2025 Google LLC
|
|
2780
|
+
*
|
|
2781
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
2782
|
+
* you may not use this file except in compliance with the License.
|
|
2783
|
+
* You may obtain a copy of the License at
|
|
2784
|
+
*
|
|
2785
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
2786
|
+
*
|
|
2787
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
2788
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
2789
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
2790
|
+
* See the License for the specific language governing permissions and
|
|
2791
|
+
* limitations under the License.
|
|
2792
|
+
*/
|
|
2793
|
+
/**
|
|
2794
|
+
* Defines an inference "backend" that uses Chrome's on-device model,
|
|
2795
|
+
* and encapsulates logic for detecting when on-device inference is
|
|
2796
|
+
* possible.
|
|
2797
|
+
*/
|
|
2798
|
+
class ChromeAdapterImpl {
|
|
2799
|
+
constructor(languageModelProvider, mode, onDeviceParams = {
|
|
2800
|
+
createOptions: {
|
|
2801
|
+
// Defaults to support image inputs for convenience.
|
|
2802
|
+
expectedInputs: [{ type: 'image' }]
|
|
2803
|
+
}
|
|
2804
|
+
}) {
|
|
2805
|
+
this.languageModelProvider = languageModelProvider;
|
|
2806
|
+
this.mode = mode;
|
|
2807
|
+
this.onDeviceParams = onDeviceParams;
|
|
2808
|
+
this.isDownloading = false;
|
|
2809
|
+
}
|
|
2810
|
+
/**
|
|
2811
|
+
* Checks if a given request can be made on-device.
|
|
2812
|
+
*
|
|
2813
|
+
* Encapsulates a few concerns:
|
|
2814
|
+
* the mode
|
|
2815
|
+
* API existence
|
|
2816
|
+
* prompt formatting
|
|
2817
|
+
* model availability, including triggering download if necessary
|
|
2818
|
+
*
|
|
2819
|
+
*
|
|
2820
|
+
* Pros: callers needn't be concerned with details of on-device availability.</p>
|
|
2821
|
+
* Cons: this method spans a few concerns and splits request validation from usage.
|
|
2822
|
+
* If instance variables weren't already part of the API, we could consider a better
|
|
2823
|
+
* separation of concerns.
|
|
2824
|
+
*/
|
|
2825
|
+
async isAvailable(request) {
|
|
2826
|
+
if (!this.mode) {
|
|
2827
|
+
logger.debug(`On-device inference unavailable because mode is undefined.`);
|
|
2828
|
+
return false;
|
|
2829
|
+
}
|
|
2830
|
+
if (this.mode === InferenceMode.ONLY_IN_CLOUD) {
|
|
2831
|
+
logger.debug(`On-device inference unavailable because mode is "only_in_cloud".`);
|
|
2832
|
+
return false;
|
|
2833
|
+
}
|
|
2834
|
+
// Triggers out-of-band download so model will eventually become available.
|
|
2835
|
+
const availability = await this.downloadIfAvailable();
|
|
2836
|
+
if (this.mode === InferenceMode.ONLY_ON_DEVICE) {
|
|
2837
|
+
// If it will never be available due to API inavailability, throw.
|
|
2838
|
+
if (availability === Availability.UNAVAILABLE) {
|
|
2839
|
+
throw new AIError(AIErrorCode.API_NOT_ENABLED, 'Local LanguageModel API not available in this environment.');
|
|
2840
|
+
}
|
|
2841
|
+
else if (availability === Availability.DOWNLOADABLE ||
|
|
2842
|
+
availability === Availability.DOWNLOADING) {
|
|
2843
|
+
// TODO(chholland): Better user experience during download - progress?
|
|
2844
|
+
logger.debug(`Waiting for download of LanguageModel to complete.`);
|
|
2845
|
+
await this.downloadPromise;
|
|
2846
|
+
return true;
|
|
2847
|
+
}
|
|
2848
|
+
return true;
|
|
2849
|
+
}
|
|
2850
|
+
// Applies prefer_on_device logic.
|
|
2851
|
+
if (availability !== Availability.AVAILABLE) {
|
|
2852
|
+
logger.debug(`On-device inference unavailable because availability is "${availability}".`);
|
|
2853
|
+
return false;
|
|
2854
|
+
}
|
|
2855
|
+
if (!ChromeAdapterImpl.isOnDeviceRequest(request)) {
|
|
2856
|
+
logger.debug(`On-device inference unavailable because request is incompatible.`);
|
|
2857
|
+
return false;
|
|
2858
|
+
}
|
|
2859
|
+
return true;
|
|
2860
|
+
}
|
|
2861
|
+
/**
|
|
2862
|
+
* Generates content on device.
|
|
2863
|
+
*
|
|
2864
|
+
* @remarks
|
|
2865
|
+
* This is comparable to {@link GenerativeModel.generateContent} for generating content in
|
|
2866
|
+
* Cloud.
|
|
2867
|
+
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
2868
|
+
* @returns {@link Response}, so we can reuse common response formatting.
|
|
2869
|
+
*/
|
|
2870
|
+
async generateContent(request) {
|
|
2871
|
+
const session = await this.createSession();
|
|
2872
|
+
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
2873
|
+
const text = await session.prompt(contents, this.onDeviceParams.promptOptions);
|
|
2874
|
+
return ChromeAdapterImpl.toResponse(text);
|
|
2875
|
+
}
|
|
2876
|
+
/**
|
|
2877
|
+
* Generates content stream on device.
|
|
2878
|
+
*
|
|
2879
|
+
* @remarks
|
|
2880
|
+
* This is comparable to {@link GenerativeModel.generateContentStream} for generating content in
|
|
2881
|
+
* Cloud.
|
|
2882
|
+
* @param request - a standard Firebase AI {@link GenerateContentRequest}
|
|
2883
|
+
* @returns {@link Response}, so we can reuse common response formatting.
|
|
2884
|
+
*/
|
|
2885
|
+
async generateContentStream(request) {
|
|
2886
|
+
const session = await this.createSession();
|
|
2887
|
+
const contents = await Promise.all(request.contents.map(ChromeAdapterImpl.toLanguageModelMessage));
|
|
2888
|
+
const stream = session.promptStreaming(contents, this.onDeviceParams.promptOptions);
|
|
2889
|
+
return ChromeAdapterImpl.toStreamResponse(stream);
|
|
2890
|
+
}
|
|
2891
|
+
async countTokens(_request) {
|
|
2892
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'Count Tokens is not yet available for on-device model.');
|
|
2893
|
+
}
|
|
2894
|
+
/**
|
|
2895
|
+
* Asserts inference for the given request can be performed by an on-device model.
|
|
2896
|
+
*/
|
|
2897
|
+
static isOnDeviceRequest(request) {
|
|
2898
|
+
// Returns false if the prompt is empty.
|
|
2899
|
+
if (request.contents.length === 0) {
|
|
2900
|
+
logger.debug('Empty prompt rejected for on-device inference.');
|
|
2901
|
+
return false;
|
|
2902
|
+
}
|
|
2903
|
+
for (const content of request.contents) {
|
|
2904
|
+
if (content.role === 'function') {
|
|
2905
|
+
logger.debug(`"Function" role rejected for on-device inference.`);
|
|
2906
|
+
return false;
|
|
2907
|
+
}
|
|
2908
|
+
// Returns false if request contains an image with an unsupported mime type.
|
|
2909
|
+
for (const part of content.parts) {
|
|
2910
|
+
if (part.inlineData &&
|
|
2911
|
+
ChromeAdapterImpl.SUPPORTED_MIME_TYPES.indexOf(part.inlineData.mimeType) === -1) {
|
|
2912
|
+
logger.debug(`Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.`);
|
|
2913
|
+
return false;
|
|
2914
|
+
}
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
return true;
|
|
2918
|
+
}
|
|
2919
|
+
/**
|
|
2920
|
+
* Encapsulates logic to get availability and download a model if one is downloadable.
|
|
2921
|
+
*/
|
|
2922
|
+
async downloadIfAvailable() {
|
|
2923
|
+
const availability = await this.languageModelProvider?.availability(this.onDeviceParams.createOptions);
|
|
2924
|
+
if (availability === Availability.DOWNLOADABLE) {
|
|
2925
|
+
this.download();
|
|
2926
|
+
}
|
|
2927
|
+
return availability;
|
|
2928
|
+
}
|
|
2929
|
+
/**
|
|
2930
|
+
* Triggers out-of-band download of an on-device model.
|
|
2931
|
+
*
|
|
2932
|
+
* Chrome only downloads models as needed. Chrome knows a model is needed when code calls
|
|
2933
|
+
* LanguageModel.create.
|
|
2934
|
+
*
|
|
2935
|
+
* Since Chrome manages the download, the SDK can only avoid redundant download requests by
|
|
2936
|
+
* tracking if a download has previously been requested.
|
|
2937
|
+
*/
|
|
2938
|
+
download() {
|
|
2939
|
+
if (this.isDownloading) {
|
|
2940
|
+
return;
|
|
2941
|
+
}
|
|
2942
|
+
this.isDownloading = true;
|
|
2943
|
+
this.downloadPromise = this.languageModelProvider
|
|
2944
|
+
?.create(this.onDeviceParams.createOptions)
|
|
2945
|
+
.finally(() => {
|
|
2946
|
+
this.isDownloading = false;
|
|
2947
|
+
});
|
|
2948
|
+
}
|
|
2949
|
+
/**
|
|
2950
|
+
* Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object.
|
|
2951
|
+
*/
|
|
2952
|
+
static async toLanguageModelMessage(content) {
|
|
2953
|
+
const languageModelMessageContents = await Promise.all(content.parts.map(ChromeAdapterImpl.toLanguageModelMessageContent));
|
|
2954
|
+
return {
|
|
2955
|
+
role: ChromeAdapterImpl.toLanguageModelMessageRole(content.role),
|
|
2956
|
+
content: languageModelMessageContents
|
|
2957
|
+
};
|
|
2958
|
+
}
|
|
2959
|
+
/**
|
|
2960
|
+
* Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object.
|
|
2961
|
+
*/
|
|
2962
|
+
static async toLanguageModelMessageContent(part) {
|
|
2963
|
+
if (part.text) {
|
|
2964
|
+
return {
|
|
2965
|
+
type: 'text',
|
|
2966
|
+
value: part.text
|
|
2967
|
+
};
|
|
2968
|
+
}
|
|
2969
|
+
else if (part.inlineData) {
|
|
2970
|
+
const formattedImageContent = await fetch(`data:${part.inlineData.mimeType};base64,${part.inlineData.data}`);
|
|
2971
|
+
const imageBlob = await formattedImageContent.blob();
|
|
2972
|
+
const imageBitmap = await createImageBitmap(imageBlob);
|
|
2973
|
+
return {
|
|
2974
|
+
type: 'image',
|
|
2975
|
+
value: imageBitmap
|
|
2976
|
+
};
|
|
2977
|
+
}
|
|
2978
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, `Processing of this Part type is not currently supported.`);
|
|
2979
|
+
}
|
|
2980
|
+
/**
|
|
2981
|
+
* Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string.
|
|
2982
|
+
*/
|
|
2983
|
+
static toLanguageModelMessageRole(role) {
|
|
2984
|
+
// Assumes 'function' rule has been filtered by isOnDeviceRequest
|
|
2985
|
+
return role === 'model' ? 'assistant' : 'user';
|
|
2986
|
+
}
|
|
2987
|
+
/**
|
|
2988
|
+
* Abstracts Chrome session creation.
|
|
2989
|
+
*
|
|
2990
|
+
* Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all
|
|
2991
|
+
* inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all
|
|
2992
|
+
* inference.
|
|
2993
|
+
*
|
|
2994
|
+
* Chrome will remove a model from memory if it's no longer in use, so this method ensures a
|
|
2995
|
+
* new session is created before an old session is destroyed.
|
|
2996
|
+
*/
|
|
2997
|
+
async createSession() {
|
|
2998
|
+
if (!this.languageModelProvider) {
|
|
2999
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'Chrome AI requested for unsupported browser version.');
|
|
3000
|
+
}
|
|
3001
|
+
const newSession = await this.languageModelProvider.create(this.onDeviceParams.createOptions);
|
|
3002
|
+
if (this.oldSession) {
|
|
3003
|
+
this.oldSession.destroy();
|
|
3004
|
+
}
|
|
3005
|
+
// Holds session reference, so model isn't unloaded from memory.
|
|
3006
|
+
this.oldSession = newSession;
|
|
3007
|
+
return newSession;
|
|
3008
|
+
}
|
|
3009
|
+
/**
|
|
3010
|
+
* Formats string returned by Chrome as a {@link Response} returned by Firebase AI.
|
|
3011
|
+
*/
|
|
3012
|
+
static toResponse(text) {
|
|
3013
|
+
return {
|
|
3014
|
+
json: async () => ({
|
|
3015
|
+
candidates: [
|
|
3016
|
+
{
|
|
3017
|
+
content: {
|
|
3018
|
+
parts: [{ text }]
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
]
|
|
3022
|
+
})
|
|
3023
|
+
};
|
|
3024
|
+
}
|
|
3025
|
+
/**
|
|
3026
|
+
* Formats string stream returned by Chrome as SSE returned by Firebase AI.
|
|
3027
|
+
*/
|
|
3028
|
+
static toStreamResponse(stream) {
|
|
3029
|
+
const encoder = new TextEncoder();
|
|
3030
|
+
return {
|
|
3031
|
+
body: stream.pipeThrough(new TransformStream({
|
|
3032
|
+
transform(chunk, controller) {
|
|
3033
|
+
const json = JSON.stringify({
|
|
3034
|
+
candidates: [
|
|
3035
|
+
{
|
|
3036
|
+
content: {
|
|
3037
|
+
role: 'model',
|
|
3038
|
+
parts: [{ text: chunk }]
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
]
|
|
3042
|
+
});
|
|
3043
|
+
controller.enqueue(encoder.encode(`data: ${json}\n\n`));
|
|
3044
|
+
}
|
|
3045
|
+
}))
|
|
3046
|
+
};
|
|
3047
|
+
}
|
|
3048
|
+
}
|
|
3049
|
+
// Visible for testing
|
|
3050
|
+
ChromeAdapterImpl.SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png'];
|
|
3051
|
+
/**
|
|
3052
|
+
* Creates a ChromeAdapterImpl on demand.
|
|
3053
|
+
*/
|
|
3054
|
+
function chromeAdapterFactory(mode, window, params) {
|
|
3055
|
+
// Do not initialize a ChromeAdapter if we are not in hybrid mode.
|
|
3056
|
+
if (typeof window !== 'undefined' && mode) {
|
|
3057
|
+
return new ChromeAdapterImpl(window.LanguageModel, mode, params);
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
|
|
3049
3061
|
/**
|
|
3050
3062
|
* The Firebase AI Web SDK.
|
|
3051
3063
|
*
|
|
@@ -3060,7 +3072,7 @@ function factory(container, { instanceIdentifier }) {
|
|
|
3060
3072
|
const app = container.getProvider('app').getImmediate();
|
|
3061
3073
|
const auth = container.getProvider('auth-internal');
|
|
3062
3074
|
const appCheckProvider = container.getProvider('app-check-internal');
|
|
3063
|
-
return new AIService(app, backend, auth, appCheckProvider);
|
|
3075
|
+
return new AIService(app, backend, auth, appCheckProvider, chromeAdapterFactory);
|
|
3064
3076
|
}
|
|
3065
3077
|
function registerAI() {
|
|
3066
3078
|
_registerComponent(new Component(AI_TYPE, factory, "PUBLIC" /* ComponentType.PUBLIC */).setMultipleInstances(true));
|