@ai-sdk/google-vertex 4.0.23 → 4.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/anthropic/edge/index.js +1 -1
- package/dist/anthropic/edge/index.mjs +1 -1
- package/dist/edge/index.js +1 -1
- package/dist/edge/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/docs/16-google-vertex.mdx +1407 -0
- package/package.json +10 -5
- package/src/__snapshots__/google-vertex-embedding-model.test.ts.snap +39 -0
- package/src/anthropic/edge/google-vertex-anthropic-provider-edge.test.ts +87 -0
- package/src/anthropic/edge/google-vertex-anthropic-provider-edge.ts +41 -0
- package/src/anthropic/edge/index.ts +8 -0
- package/src/anthropic/google-vertex-anthropic-messages-options.ts +15 -0
- package/src/anthropic/google-vertex-anthropic-provider-node.test.ts +73 -0
- package/src/anthropic/google-vertex-anthropic-provider-node.ts +40 -0
- package/src/anthropic/google-vertex-anthropic-provider.test.ts +208 -0
- package/src/anthropic/google-vertex-anthropic-provider.ts +210 -0
- package/src/anthropic/index.ts +8 -0
- package/src/edge/google-vertex-auth-edge.test.ts +308 -0
- package/src/edge/google-vertex-auth-edge.ts +161 -0
- package/src/edge/google-vertex-provider-edge.test.ts +105 -0
- package/src/edge/google-vertex-provider-edge.ts +50 -0
- package/src/edge/index.ts +5 -0
- package/src/google-vertex-auth-google-auth-library.test.ts +59 -0
- package/src/google-vertex-auth-google-auth-library.ts +27 -0
- package/src/google-vertex-config.ts +8 -0
- package/src/google-vertex-embedding-model.test.ts +315 -0
- package/src/google-vertex-embedding-model.ts +135 -0
- package/src/google-vertex-embedding-options.ts +63 -0
- package/src/google-vertex-error.ts +19 -0
- package/src/google-vertex-image-model.test.ts +926 -0
- package/src/google-vertex-image-model.ts +288 -0
- package/src/google-vertex-image-settings.ts +8 -0
- package/src/google-vertex-options.ts +32 -0
- package/src/google-vertex-provider-node.test.ts +88 -0
- package/src/google-vertex-provider-node.ts +49 -0
- package/src/google-vertex-provider.test.ts +318 -0
- package/src/google-vertex-provider.ts +217 -0
- package/src/google-vertex-tools.ts +11 -0
- package/src/index.ts +7 -0
- package/src/version.ts +6 -0
|
@@ -0,0 +1,926 @@
|
|
|
1
|
+
import { createTestServer } from '@ai-sdk/test-server/with-vitest';
|
|
2
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
3
|
+
import {
|
|
4
|
+
GoogleVertexImageModel,
|
|
5
|
+
GoogleVertexImageProviderOptions,
|
|
6
|
+
} from './google-vertex-image-model';
|
|
7
|
+
import { createVertex } from './google-vertex-provider';
|
|
8
|
+
|
|
9
|
+
vi.mock('./version', () => ({
|
|
10
|
+
VERSION: '0.0.0-test',
|
|
11
|
+
}));
|
|
12
|
+
|
|
13
|
+
const prompt = 'A cute baby sea otter';
|
|
14
|
+
|
|
15
|
+
const model = new GoogleVertexImageModel('imagen-3.0-generate-002', {
|
|
16
|
+
provider: 'google-vertex',
|
|
17
|
+
baseURL: 'https://api.example.com',
|
|
18
|
+
headers: { 'api-key': 'test-key' },
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const server = createTestServer({
|
|
22
|
+
'https://api.example.com/models/imagen-3.0-generate-002:predict': {},
|
|
23
|
+
'https://api.example.com/models/imagen-4.0-generate-preview-06-06:predict':
|
|
24
|
+
{},
|
|
25
|
+
'https://api.example.com/models/imagen-4.0-fast-generate-preview-06-06:predict':
|
|
26
|
+
{},
|
|
27
|
+
'https://api.example.com/models/imagen-4.0-ultra-generate-preview-06-06:predict':
|
|
28
|
+
{},
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
describe('GoogleVertexImageModel', () => {
|
|
32
|
+
describe('doGenerate', () => {
|
|
33
|
+
function prepareJsonResponse({
|
|
34
|
+
headers,
|
|
35
|
+
}: {
|
|
36
|
+
headers?: Record<string, string>;
|
|
37
|
+
} = {}) {
|
|
38
|
+
server.urls[
|
|
39
|
+
'https://api.example.com/models/imagen-3.0-generate-002:predict'
|
|
40
|
+
].response = {
|
|
41
|
+
type: 'json-value',
|
|
42
|
+
headers,
|
|
43
|
+
body: {
|
|
44
|
+
predictions: [
|
|
45
|
+
{
|
|
46
|
+
mimeType: 'image/png',
|
|
47
|
+
prompt: 'revised prompt 1',
|
|
48
|
+
bytesBase64Encoded: 'base64-image-1',
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
mimeType: 'image/png',
|
|
52
|
+
prompt: 'revised prompt 2',
|
|
53
|
+
bytesBase64Encoded: 'base64-image-2',
|
|
54
|
+
someFutureField: 'some future value',
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// changed test to go through the provider `createVertex`
|
|
62
|
+
it('should pass headers', async () => {
|
|
63
|
+
prepareJsonResponse();
|
|
64
|
+
|
|
65
|
+
const provider = createVertex({
|
|
66
|
+
project: 'test-project',
|
|
67
|
+
location: 'us-central1',
|
|
68
|
+
baseURL: 'https://api.example.com',
|
|
69
|
+
headers: { 'Custom-Provider-Header': 'provider-header-value' },
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
await provider.imageModel('imagen-3.0-generate-002').doGenerate({
|
|
73
|
+
prompt,
|
|
74
|
+
files: undefined,
|
|
75
|
+
mask: undefined,
|
|
76
|
+
n: 2,
|
|
77
|
+
size: undefined,
|
|
78
|
+
aspectRatio: undefined,
|
|
79
|
+
seed: undefined,
|
|
80
|
+
providerOptions: {},
|
|
81
|
+
headers: { 'Custom-Request-Header': 'request-header-value' },
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
expect(server.calls[0].requestHeaders).toStrictEqual({
|
|
85
|
+
'content-type': 'application/json',
|
|
86
|
+
'custom-provider-header': 'provider-header-value',
|
|
87
|
+
'custom-request-header': 'request-header-value',
|
|
88
|
+
});
|
|
89
|
+
expect(server.calls[0].requestUserAgent).toContain(
|
|
90
|
+
`ai-sdk/google-vertex/0.0.0-test`,
|
|
91
|
+
);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('should use default maxImagesPerCall when not specified', () => {
|
|
95
|
+
const defaultModel = new GoogleVertexImageModel(
|
|
96
|
+
'imagen-3.0-generate-002',
|
|
97
|
+
{
|
|
98
|
+
provider: 'google-vertex',
|
|
99
|
+
baseURL: 'https://api.example.com',
|
|
100
|
+
headers: { 'api-key': 'test-key' },
|
|
101
|
+
},
|
|
102
|
+
);
|
|
103
|
+
|
|
104
|
+
expect(defaultModel.maxImagesPerCall).toBe(4);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('should extract the generated images', async () => {
|
|
108
|
+
prepareJsonResponse();
|
|
109
|
+
|
|
110
|
+
const result = await model.doGenerate({
|
|
111
|
+
prompt,
|
|
112
|
+
files: undefined,
|
|
113
|
+
mask: undefined,
|
|
114
|
+
n: 2,
|
|
115
|
+
size: undefined,
|
|
116
|
+
aspectRatio: undefined,
|
|
117
|
+
seed: undefined,
|
|
118
|
+
providerOptions: {},
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
expect(result.images).toStrictEqual(['base64-image-1', 'base64-image-2']);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('sends aspect ratio in the request', async () => {
|
|
125
|
+
prepareJsonResponse();
|
|
126
|
+
|
|
127
|
+
await model.doGenerate({
|
|
128
|
+
prompt: 'test prompt',
|
|
129
|
+
files: undefined,
|
|
130
|
+
mask: undefined,
|
|
131
|
+
n: 1,
|
|
132
|
+
size: undefined,
|
|
133
|
+
aspectRatio: '16:9',
|
|
134
|
+
seed: undefined,
|
|
135
|
+
providerOptions: {},
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
139
|
+
{
|
|
140
|
+
"instances": [
|
|
141
|
+
{
|
|
142
|
+
"prompt": "test prompt",
|
|
143
|
+
},
|
|
144
|
+
],
|
|
145
|
+
"parameters": {
|
|
146
|
+
"aspectRatio": "16:9",
|
|
147
|
+
"sampleCount": 1,
|
|
148
|
+
},
|
|
149
|
+
}
|
|
150
|
+
`);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('should pass aspect ratio directly when specified', async () => {
|
|
154
|
+
prepareJsonResponse();
|
|
155
|
+
|
|
156
|
+
await model.doGenerate({
|
|
157
|
+
prompt: 'test prompt',
|
|
158
|
+
files: undefined,
|
|
159
|
+
mask: undefined,
|
|
160
|
+
n: 1,
|
|
161
|
+
size: undefined,
|
|
162
|
+
aspectRatio: '16:9',
|
|
163
|
+
seed: undefined,
|
|
164
|
+
providerOptions: {},
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
168
|
+
{
|
|
169
|
+
"instances": [
|
|
170
|
+
{
|
|
171
|
+
"prompt": "test prompt",
|
|
172
|
+
},
|
|
173
|
+
],
|
|
174
|
+
"parameters": {
|
|
175
|
+
"aspectRatio": "16:9",
|
|
176
|
+
"sampleCount": 1,
|
|
177
|
+
},
|
|
178
|
+
}
|
|
179
|
+
`);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('should pass seed directly when specified', async () => {
|
|
183
|
+
prepareJsonResponse();
|
|
184
|
+
|
|
185
|
+
await model.doGenerate({
|
|
186
|
+
prompt: 'test prompt',
|
|
187
|
+
files: undefined,
|
|
188
|
+
mask: undefined,
|
|
189
|
+
n: 1,
|
|
190
|
+
size: undefined,
|
|
191
|
+
aspectRatio: undefined,
|
|
192
|
+
seed: 42,
|
|
193
|
+
providerOptions: {},
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
197
|
+
{
|
|
198
|
+
"instances": [
|
|
199
|
+
{
|
|
200
|
+
"prompt": "test prompt",
|
|
201
|
+
},
|
|
202
|
+
],
|
|
203
|
+
"parameters": {
|
|
204
|
+
"sampleCount": 1,
|
|
205
|
+
"seed": 42,
|
|
206
|
+
},
|
|
207
|
+
}
|
|
208
|
+
`);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it('should combine aspectRatio, seed and provider options', async () => {
|
|
212
|
+
prepareJsonResponse();
|
|
213
|
+
|
|
214
|
+
await model.doGenerate({
|
|
215
|
+
prompt: 'test prompt',
|
|
216
|
+
files: undefined,
|
|
217
|
+
mask: undefined,
|
|
218
|
+
n: 1,
|
|
219
|
+
size: undefined,
|
|
220
|
+
aspectRatio: '1:1',
|
|
221
|
+
seed: 42,
|
|
222
|
+
providerOptions: {
|
|
223
|
+
vertex: {
|
|
224
|
+
addWatermark: false,
|
|
225
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
230
|
+
{
|
|
231
|
+
"instances": [
|
|
232
|
+
{
|
|
233
|
+
"prompt": "test prompt",
|
|
234
|
+
},
|
|
235
|
+
],
|
|
236
|
+
"parameters": {
|
|
237
|
+
"addWatermark": false,
|
|
238
|
+
"aspectRatio": "1:1",
|
|
239
|
+
"sampleCount": 1,
|
|
240
|
+
"seed": 42,
|
|
241
|
+
},
|
|
242
|
+
}
|
|
243
|
+
`);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
it('should return warnings for unsupported settings', async () => {
|
|
247
|
+
prepareJsonResponse();
|
|
248
|
+
|
|
249
|
+
const result = await model.doGenerate({
|
|
250
|
+
prompt,
|
|
251
|
+
files: undefined,
|
|
252
|
+
mask: undefined,
|
|
253
|
+
n: 1,
|
|
254
|
+
size: '1024x1024',
|
|
255
|
+
aspectRatio: '1:1',
|
|
256
|
+
seed: 123,
|
|
257
|
+
providerOptions: {},
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
expect(result.warnings).toMatchInlineSnapshot(`
|
|
261
|
+
[
|
|
262
|
+
{
|
|
263
|
+
"details": "This model does not support the \`size\` option. Use \`aspectRatio\` instead.",
|
|
264
|
+
"feature": "size",
|
|
265
|
+
"type": "unsupported",
|
|
266
|
+
},
|
|
267
|
+
]
|
|
268
|
+
`);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
it('should include response data with timestamp, modelId and headers', async () => {
|
|
272
|
+
prepareJsonResponse({
|
|
273
|
+
headers: {
|
|
274
|
+
'request-id': 'test-request-id',
|
|
275
|
+
'x-goog-quota-remaining': '123',
|
|
276
|
+
},
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
const testDate = new Date('2024-03-15T12:00:00Z');
|
|
280
|
+
|
|
281
|
+
const customModel = new GoogleVertexImageModel(
|
|
282
|
+
'imagen-3.0-generate-002',
|
|
283
|
+
{
|
|
284
|
+
provider: 'google-vertex',
|
|
285
|
+
baseURL: 'https://api.example.com',
|
|
286
|
+
headers: { 'api-key': 'test-key' },
|
|
287
|
+
_internal: {
|
|
288
|
+
currentDate: () => testDate,
|
|
289
|
+
},
|
|
290
|
+
},
|
|
291
|
+
);
|
|
292
|
+
|
|
293
|
+
const result = await customModel.doGenerate({
|
|
294
|
+
prompt,
|
|
295
|
+
files: undefined,
|
|
296
|
+
mask: undefined,
|
|
297
|
+
n: 1,
|
|
298
|
+
size: undefined,
|
|
299
|
+
aspectRatio: undefined,
|
|
300
|
+
seed: undefined,
|
|
301
|
+
providerOptions: {},
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
expect(result.response).toStrictEqual({
|
|
305
|
+
timestamp: testDate,
|
|
306
|
+
modelId: 'imagen-3.0-generate-002',
|
|
307
|
+
headers: {
|
|
308
|
+
'content-length': '237',
|
|
309
|
+
'content-type': 'application/json',
|
|
310
|
+
'request-id': 'test-request-id',
|
|
311
|
+
'x-goog-quota-remaining': '123',
|
|
312
|
+
},
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
it('should use real date when no custom date provider is specified', async () => {
|
|
317
|
+
prepareJsonResponse();
|
|
318
|
+
const beforeDate = new Date();
|
|
319
|
+
|
|
320
|
+
const result = await model.doGenerate({
|
|
321
|
+
prompt,
|
|
322
|
+
files: undefined,
|
|
323
|
+
mask: undefined,
|
|
324
|
+
n: 2,
|
|
325
|
+
size: undefined,
|
|
326
|
+
aspectRatio: undefined,
|
|
327
|
+
seed: undefined,
|
|
328
|
+
providerOptions: {},
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
const afterDate = new Date();
|
|
332
|
+
|
|
333
|
+
expect(result.response.timestamp.getTime()).toBeGreaterThanOrEqual(
|
|
334
|
+
beforeDate.getTime(),
|
|
335
|
+
);
|
|
336
|
+
expect(result.response.timestamp.getTime()).toBeLessThanOrEqual(
|
|
337
|
+
afterDate.getTime(),
|
|
338
|
+
);
|
|
339
|
+
expect(result.response.modelId).toBe('imagen-3.0-generate-002');
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
it('should only pass valid provider options', async () => {
|
|
343
|
+
prepareJsonResponse();
|
|
344
|
+
|
|
345
|
+
await model.doGenerate({
|
|
346
|
+
prompt,
|
|
347
|
+
files: undefined,
|
|
348
|
+
mask: undefined,
|
|
349
|
+
n: 2,
|
|
350
|
+
size: undefined,
|
|
351
|
+
aspectRatio: '16:9',
|
|
352
|
+
seed: undefined,
|
|
353
|
+
providerOptions: {
|
|
354
|
+
vertex: {
|
|
355
|
+
addWatermark: false,
|
|
356
|
+
negativePrompt: 'negative prompt',
|
|
357
|
+
personGeneration: 'allow_all',
|
|
358
|
+
sampleImageSize: '2K',
|
|
359
|
+
// @ts-expect-error Testing invalid option
|
|
360
|
+
foo: 'bar',
|
|
361
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
362
|
+
},
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
366
|
+
{
|
|
367
|
+
"instances": [
|
|
368
|
+
{
|
|
369
|
+
"prompt": "A cute baby sea otter",
|
|
370
|
+
},
|
|
371
|
+
],
|
|
372
|
+
"parameters": {
|
|
373
|
+
"addWatermark": false,
|
|
374
|
+
"aspectRatio": "16:9",
|
|
375
|
+
"negativePrompt": "negative prompt",
|
|
376
|
+
"personGeneration": "allow_all",
|
|
377
|
+
"sampleCount": 2,
|
|
378
|
+
"sampleImageSize": "2K",
|
|
379
|
+
},
|
|
380
|
+
}
|
|
381
|
+
`);
|
|
382
|
+
});
|
|
383
|
+
|
|
384
|
+
it('should return image meta data', async () => {
|
|
385
|
+
prepareJsonResponse();
|
|
386
|
+
|
|
387
|
+
const result = await model.doGenerate({
|
|
388
|
+
prompt,
|
|
389
|
+
files: undefined,
|
|
390
|
+
mask: undefined,
|
|
391
|
+
n: 2,
|
|
392
|
+
size: undefined,
|
|
393
|
+
aspectRatio: undefined,
|
|
394
|
+
seed: undefined,
|
|
395
|
+
providerOptions: {},
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
expect(result.providerMetadata?.vertex).toStrictEqual({
|
|
399
|
+
images: [
|
|
400
|
+
{
|
|
401
|
+
revisedPrompt: 'revised prompt 1',
|
|
402
|
+
},
|
|
403
|
+
{
|
|
404
|
+
revisedPrompt: 'revised prompt 2',
|
|
405
|
+
},
|
|
406
|
+
],
|
|
407
|
+
});
|
|
408
|
+
});
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
describe('Image Editing', () => {
|
|
412
|
+
function prepareJsonResponse({
|
|
413
|
+
headers,
|
|
414
|
+
}: {
|
|
415
|
+
headers?: Record<string, string>;
|
|
416
|
+
} = {}) {
|
|
417
|
+
server.urls[
|
|
418
|
+
'https://api.example.com/models/imagen-3.0-generate-002:predict'
|
|
419
|
+
].response = {
|
|
420
|
+
type: 'json-value',
|
|
421
|
+
headers,
|
|
422
|
+
body: {
|
|
423
|
+
predictions: [
|
|
424
|
+
{
|
|
425
|
+
mimeType: 'image/png',
|
|
426
|
+
bytesBase64Encoded: 'edited-base64-image',
|
|
427
|
+
},
|
|
428
|
+
],
|
|
429
|
+
},
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
it('should send edit request with files and mask', async () => {
|
|
434
|
+
prepareJsonResponse();
|
|
435
|
+
|
|
436
|
+
// Create test image data (base64 encoded)
|
|
437
|
+
const imageData = 'base64-source-image';
|
|
438
|
+
const maskData = 'base64-mask-image';
|
|
439
|
+
|
|
440
|
+
await model.doGenerate({
|
|
441
|
+
prompt: 'A sunlit indoor lounge with a flamingo',
|
|
442
|
+
files: [
|
|
443
|
+
{
|
|
444
|
+
type: 'file',
|
|
445
|
+
data: imageData,
|
|
446
|
+
mediaType: 'image/png',
|
|
447
|
+
},
|
|
448
|
+
],
|
|
449
|
+
mask: {
|
|
450
|
+
type: 'file',
|
|
451
|
+
data: maskData,
|
|
452
|
+
mediaType: 'image/png',
|
|
453
|
+
},
|
|
454
|
+
n: 1,
|
|
455
|
+
size: undefined,
|
|
456
|
+
aspectRatio: undefined,
|
|
457
|
+
seed: undefined,
|
|
458
|
+
providerOptions: {},
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
462
|
+
{
|
|
463
|
+
"instances": [
|
|
464
|
+
{
|
|
465
|
+
"prompt": "A sunlit indoor lounge with a flamingo",
|
|
466
|
+
"referenceImages": [
|
|
467
|
+
{
|
|
468
|
+
"referenceId": 1,
|
|
469
|
+
"referenceImage": {
|
|
470
|
+
"bytesBase64Encoded": "base64-source-image",
|
|
471
|
+
},
|
|
472
|
+
"referenceType": "REFERENCE_TYPE_RAW",
|
|
473
|
+
},
|
|
474
|
+
{
|
|
475
|
+
"maskImageConfig": {
|
|
476
|
+
"maskMode": "MASK_MODE_USER_PROVIDED",
|
|
477
|
+
},
|
|
478
|
+
"referenceId": 2,
|
|
479
|
+
"referenceImage": {
|
|
480
|
+
"bytesBase64Encoded": "base64-mask-image",
|
|
481
|
+
},
|
|
482
|
+
"referenceType": "REFERENCE_TYPE_MASK",
|
|
483
|
+
},
|
|
484
|
+
],
|
|
485
|
+
},
|
|
486
|
+
],
|
|
487
|
+
"parameters": {
|
|
488
|
+
"editMode": "EDIT_MODE_INPAINT_INSERTION",
|
|
489
|
+
"sampleCount": 1,
|
|
490
|
+
},
|
|
491
|
+
}
|
|
492
|
+
`);
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
it('should send edit request with Uint8Array data', async () => {
|
|
496
|
+
prepareJsonResponse();
|
|
497
|
+
|
|
498
|
+
// Create test Uint8Array data (represents 'hello' in bytes)
|
|
499
|
+
const imageUint8Array = new Uint8Array([104, 101, 108, 108, 111]);
|
|
500
|
+
const maskUint8Array = new Uint8Array([119, 111, 114, 108, 100]);
|
|
501
|
+
|
|
502
|
+
await model.doGenerate({
|
|
503
|
+
prompt: 'Edit this image',
|
|
504
|
+
files: [
|
|
505
|
+
{
|
|
506
|
+
type: 'file',
|
|
507
|
+
data: imageUint8Array,
|
|
508
|
+
mediaType: 'image/png',
|
|
509
|
+
},
|
|
510
|
+
],
|
|
511
|
+
mask: {
|
|
512
|
+
type: 'file',
|
|
513
|
+
data: maskUint8Array,
|
|
514
|
+
mediaType: 'image/png',
|
|
515
|
+
},
|
|
516
|
+
n: 1,
|
|
517
|
+
size: undefined,
|
|
518
|
+
aspectRatio: undefined,
|
|
519
|
+
seed: undefined,
|
|
520
|
+
providerOptions: {},
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
const requestBody = await server.calls[0].requestBodyJson;
|
|
524
|
+
// Check that the data was converted to base64
|
|
525
|
+
expect(
|
|
526
|
+
requestBody.instances[0].referenceImages[0].referenceImage
|
|
527
|
+
.bytesBase64Encoded,
|
|
528
|
+
).toBe('aGVsbG8='); // 'hello' in base64
|
|
529
|
+
expect(
|
|
530
|
+
requestBody.instances[0].referenceImages[1].referenceImage
|
|
531
|
+
.bytesBase64Encoded,
|
|
532
|
+
).toBe('d29ybGQ='); // 'world' in base64
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it('should send edit request with custom edit options', async () => {
|
|
536
|
+
prepareJsonResponse();
|
|
537
|
+
|
|
538
|
+
await model.doGenerate({
|
|
539
|
+
prompt: 'Remove the object',
|
|
540
|
+
files: [
|
|
541
|
+
{
|
|
542
|
+
type: 'file',
|
|
543
|
+
data: 'base64-source-image',
|
|
544
|
+
mediaType: 'image/png',
|
|
545
|
+
},
|
|
546
|
+
],
|
|
547
|
+
mask: {
|
|
548
|
+
type: 'file',
|
|
549
|
+
data: 'base64-mask-image',
|
|
550
|
+
mediaType: 'image/png',
|
|
551
|
+
},
|
|
552
|
+
n: 1,
|
|
553
|
+
size: undefined,
|
|
554
|
+
aspectRatio: undefined,
|
|
555
|
+
seed: undefined,
|
|
556
|
+
providerOptions: {
|
|
557
|
+
vertex: {
|
|
558
|
+
edit: {
|
|
559
|
+
mode: 'EDIT_MODE_INPAINT_REMOVAL',
|
|
560
|
+
baseSteps: 50,
|
|
561
|
+
maskMode: 'MASK_MODE_USER_PROVIDED',
|
|
562
|
+
maskDilation: 0.01,
|
|
563
|
+
},
|
|
564
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
565
|
+
},
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
569
|
+
{
|
|
570
|
+
"instances": [
|
|
571
|
+
{
|
|
572
|
+
"prompt": "Remove the object",
|
|
573
|
+
"referenceImages": [
|
|
574
|
+
{
|
|
575
|
+
"referenceId": 1,
|
|
576
|
+
"referenceImage": {
|
|
577
|
+
"bytesBase64Encoded": "base64-source-image",
|
|
578
|
+
},
|
|
579
|
+
"referenceType": "REFERENCE_TYPE_RAW",
|
|
580
|
+
},
|
|
581
|
+
{
|
|
582
|
+
"maskImageConfig": {
|
|
583
|
+
"dilation": 0.01,
|
|
584
|
+
"maskMode": "MASK_MODE_USER_PROVIDED",
|
|
585
|
+
},
|
|
586
|
+
"referenceId": 2,
|
|
587
|
+
"referenceImage": {
|
|
588
|
+
"bytesBase64Encoded": "base64-mask-image",
|
|
589
|
+
},
|
|
590
|
+
"referenceType": "REFERENCE_TYPE_MASK",
|
|
591
|
+
},
|
|
592
|
+
],
|
|
593
|
+
},
|
|
594
|
+
],
|
|
595
|
+
"parameters": {
|
|
596
|
+
"editConfig": {
|
|
597
|
+
"baseSteps": 50,
|
|
598
|
+
},
|
|
599
|
+
"editMode": "EDIT_MODE_INPAINT_REMOVAL",
|
|
600
|
+
"sampleCount": 1,
|
|
601
|
+
},
|
|
602
|
+
}
|
|
603
|
+
`);
|
|
604
|
+
});
|
|
605
|
+
|
|
606
|
+
it('should extract the edited images', async () => {
|
|
607
|
+
prepareJsonResponse();
|
|
608
|
+
|
|
609
|
+
const result = await model.doGenerate({
|
|
610
|
+
prompt: 'Edit this image',
|
|
611
|
+
files: [
|
|
612
|
+
{
|
|
613
|
+
type: 'file',
|
|
614
|
+
data: 'base64-source-image',
|
|
615
|
+
mediaType: 'image/png',
|
|
616
|
+
},
|
|
617
|
+
],
|
|
618
|
+
mask: {
|
|
619
|
+
type: 'file',
|
|
620
|
+
data: 'base64-mask-image',
|
|
621
|
+
mediaType: 'image/png',
|
|
622
|
+
},
|
|
623
|
+
n: 1,
|
|
624
|
+
size: undefined,
|
|
625
|
+
aspectRatio: undefined,
|
|
626
|
+
seed: undefined,
|
|
627
|
+
providerOptions: {},
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
expect(result.images).toStrictEqual(['edited-base64-image']);
|
|
631
|
+
});
|
|
632
|
+
|
|
633
|
+
it('should send edit request without mask (for operations that do not require mask)', async () => {
|
|
634
|
+
prepareJsonResponse();
|
|
635
|
+
|
|
636
|
+
await model.doGenerate({
|
|
637
|
+
prompt: 'Upscale this image',
|
|
638
|
+
files: [
|
|
639
|
+
{
|
|
640
|
+
type: 'file',
|
|
641
|
+
data: 'base64-source-image',
|
|
642
|
+
mediaType: 'image/png',
|
|
643
|
+
},
|
|
644
|
+
],
|
|
645
|
+
mask: undefined,
|
|
646
|
+
n: 1,
|
|
647
|
+
size: undefined,
|
|
648
|
+
aspectRatio: undefined,
|
|
649
|
+
seed: undefined,
|
|
650
|
+
providerOptions: {
|
|
651
|
+
vertex: {
|
|
652
|
+
edit: { mode: 'EDIT_MODE_CONTROLLED_EDITING' },
|
|
653
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
654
|
+
},
|
|
655
|
+
});
|
|
656
|
+
|
|
657
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
658
|
+
{
|
|
659
|
+
"instances": [
|
|
660
|
+
{
|
|
661
|
+
"prompt": "Upscale this image",
|
|
662
|
+
"referenceImages": [
|
|
663
|
+
{
|
|
664
|
+
"referenceId": 1,
|
|
665
|
+
"referenceImage": {
|
|
666
|
+
"bytesBase64Encoded": "base64-source-image",
|
|
667
|
+
},
|
|
668
|
+
"referenceType": "REFERENCE_TYPE_RAW",
|
|
669
|
+
},
|
|
670
|
+
],
|
|
671
|
+
},
|
|
672
|
+
],
|
|
673
|
+
"parameters": {
|
|
674
|
+
"editMode": "EDIT_MODE_CONTROLLED_EDITING",
|
|
675
|
+
"sampleCount": 1,
|
|
676
|
+
},
|
|
677
|
+
}
|
|
678
|
+
`);
|
|
679
|
+
});
|
|
680
|
+
});
|
|
681
|
+
|
|
682
|
+
describe('Imagen 4 Models', () => {
|
|
683
|
+
describe('imagen-4.0-generate-preview-06-06', () => {
|
|
684
|
+
const imagen4Model = new GoogleVertexImageModel(
|
|
685
|
+
'imagen-4.0-generate-preview-06-06',
|
|
686
|
+
{
|
|
687
|
+
provider: 'google-vertex',
|
|
688
|
+
baseURL: 'https://api.example.com',
|
|
689
|
+
headers: { 'api-key': 'test-key' },
|
|
690
|
+
},
|
|
691
|
+
);
|
|
692
|
+
|
|
693
|
+
function prepareImagen4Response() {
|
|
694
|
+
server.urls[
|
|
695
|
+
'https://api.example.com/models/imagen-4.0-generate-preview-06-06:predict'
|
|
696
|
+
].response = {
|
|
697
|
+
type: 'json-value',
|
|
698
|
+
body: {
|
|
699
|
+
predictions: [
|
|
700
|
+
{
|
|
701
|
+
mimeType: 'image/png',
|
|
702
|
+
prompt: 'revised imagen 4 prompt',
|
|
703
|
+
bytesBase64Encoded: 'base64-imagen4-image',
|
|
704
|
+
},
|
|
705
|
+
],
|
|
706
|
+
},
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
it('should generate images with Imagen 4', async () => {
|
|
711
|
+
prepareImagen4Response();
|
|
712
|
+
|
|
713
|
+
const result = await imagen4Model.doGenerate({
|
|
714
|
+
prompt: 'A beautiful sunset over mountains',
|
|
715
|
+
files: undefined,
|
|
716
|
+
mask: undefined,
|
|
717
|
+
n: 1,
|
|
718
|
+
size: undefined,
|
|
719
|
+
aspectRatio: '16:9',
|
|
720
|
+
seed: 42,
|
|
721
|
+
providerOptions: {
|
|
722
|
+
vertex: {
|
|
723
|
+
addWatermark: false,
|
|
724
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
725
|
+
},
|
|
726
|
+
});
|
|
727
|
+
|
|
728
|
+
expect(result.images).toStrictEqual(['base64-imagen4-image']);
|
|
729
|
+
expect(result.providerMetadata?.vertex).toStrictEqual({
|
|
730
|
+
images: [
|
|
731
|
+
{
|
|
732
|
+
revisedPrompt: 'revised imagen 4 prompt',
|
|
733
|
+
},
|
|
734
|
+
],
|
|
735
|
+
});
|
|
736
|
+
});
|
|
737
|
+
|
|
738
|
+
it('should send correct request parameters for Imagen 4', async () => {
|
|
739
|
+
prepareImagen4Response();
|
|
740
|
+
|
|
741
|
+
await imagen4Model.doGenerate({
|
|
742
|
+
prompt: 'test imagen 4 prompt',
|
|
743
|
+
files: undefined,
|
|
744
|
+
mask: undefined,
|
|
745
|
+
n: 2,
|
|
746
|
+
size: undefined,
|
|
747
|
+
aspectRatio: '1:1',
|
|
748
|
+
seed: 123,
|
|
749
|
+
providerOptions: {
|
|
750
|
+
vertex: {
|
|
751
|
+
personGeneration: 'allow_adult',
|
|
752
|
+
safetySetting: 'block_medium_and_above',
|
|
753
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
754
|
+
},
|
|
755
|
+
});
|
|
756
|
+
|
|
757
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
758
|
+
{
|
|
759
|
+
"instances": [
|
|
760
|
+
{
|
|
761
|
+
"prompt": "test imagen 4 prompt",
|
|
762
|
+
},
|
|
763
|
+
],
|
|
764
|
+
"parameters": {
|
|
765
|
+
"aspectRatio": "1:1",
|
|
766
|
+
"personGeneration": "allow_adult",
|
|
767
|
+
"safetySetting": "block_medium_and_above",
|
|
768
|
+
"sampleCount": 2,
|
|
769
|
+
"seed": 123,
|
|
770
|
+
},
|
|
771
|
+
}
|
|
772
|
+
`);
|
|
773
|
+
});
|
|
774
|
+
});
|
|
775
|
+
|
|
776
|
+
describe('imagen-4.0-fast-generate-preview-06-06', () => {
|
|
777
|
+
const imagen4FastModel = new GoogleVertexImageModel(
|
|
778
|
+
'imagen-4.0-fast-generate-preview-06-06',
|
|
779
|
+
{
|
|
780
|
+
provider: 'google-vertex',
|
|
781
|
+
baseURL: 'https://api.example.com',
|
|
782
|
+
headers: { 'api-key': 'test-key' },
|
|
783
|
+
},
|
|
784
|
+
);
|
|
785
|
+
|
|
786
|
+
function prepareImagen4FastResponse() {
|
|
787
|
+
server.urls[
|
|
788
|
+
'https://api.example.com/models/imagen-4.0-fast-generate-preview-06-06:predict'
|
|
789
|
+
].response = {
|
|
790
|
+
type: 'json-value',
|
|
791
|
+
body: {
|
|
792
|
+
predictions: [
|
|
793
|
+
{
|
|
794
|
+
mimeType: 'image/png',
|
|
795
|
+
prompt: 'revised imagen 4 fast prompt',
|
|
796
|
+
bytesBase64Encoded: 'base64-imagen4-fast-image',
|
|
797
|
+
},
|
|
798
|
+
],
|
|
799
|
+
},
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
it('should generate images with Imagen 4 Fast', async () => {
|
|
804
|
+
prepareImagen4FastResponse();
|
|
805
|
+
|
|
806
|
+
const result = await imagen4FastModel.doGenerate({
|
|
807
|
+
prompt: 'A quick sketch of a cat',
|
|
808
|
+
files: undefined,
|
|
809
|
+
mask: undefined,
|
|
810
|
+
n: 1,
|
|
811
|
+
size: undefined,
|
|
812
|
+
aspectRatio: '3:4',
|
|
813
|
+
seed: undefined,
|
|
814
|
+
providerOptions: {},
|
|
815
|
+
});
|
|
816
|
+
|
|
817
|
+
expect(result.images).toStrictEqual(['base64-imagen4-fast-image']);
|
|
818
|
+
expect(result.providerMetadata?.vertex).toStrictEqual({
|
|
819
|
+
images: [
|
|
820
|
+
{
|
|
821
|
+
revisedPrompt: 'revised imagen 4 fast prompt',
|
|
822
|
+
},
|
|
823
|
+
],
|
|
824
|
+
});
|
|
825
|
+
});
|
|
826
|
+
});
|
|
827
|
+
|
|
828
|
+
describe('imagen-4.0-ultra-generate-preview-06-06', () => {
|
|
829
|
+
const imagen4UltraModel = new GoogleVertexImageModel(
|
|
830
|
+
'imagen-4.0-ultra-generate-preview-06-06',
|
|
831
|
+
{
|
|
832
|
+
provider: 'google-vertex',
|
|
833
|
+
baseURL: 'https://api.example.com',
|
|
834
|
+
headers: { 'api-key': 'test-key' },
|
|
835
|
+
},
|
|
836
|
+
);
|
|
837
|
+
|
|
838
|
+
function prepareImagen4UltraResponse() {
|
|
839
|
+
server.urls[
|
|
840
|
+
'https://api.example.com/models/imagen-4.0-ultra-generate-preview-06-06:predict'
|
|
841
|
+
].response = {
|
|
842
|
+
type: 'json-value',
|
|
843
|
+
body: {
|
|
844
|
+
predictions: [
|
|
845
|
+
{
|
|
846
|
+
mimeType: 'image/png',
|
|
847
|
+
prompt: 'revised imagen 4 ultra prompt',
|
|
848
|
+
bytesBase64Encoded: 'base64-imagen4-ultra-image',
|
|
849
|
+
},
|
|
850
|
+
],
|
|
851
|
+
},
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
it('should generate images with Imagen 4 Ultra', async () => {
|
|
856
|
+
prepareImagen4UltraResponse();
|
|
857
|
+
|
|
858
|
+
const result = await imagen4UltraModel.doGenerate({
|
|
859
|
+
prompt: 'A highly detailed photorealistic portrait',
|
|
860
|
+
files: undefined,
|
|
861
|
+
mask: undefined,
|
|
862
|
+
n: 1,
|
|
863
|
+
size: undefined,
|
|
864
|
+
aspectRatio: '4:3',
|
|
865
|
+
seed: 999,
|
|
866
|
+
providerOptions: {
|
|
867
|
+
vertex: {
|
|
868
|
+
negativePrompt: 'blurry, low quality',
|
|
869
|
+
addWatermark: true,
|
|
870
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
871
|
+
},
|
|
872
|
+
});
|
|
873
|
+
|
|
874
|
+
expect(result.images).toStrictEqual(['base64-imagen4-ultra-image']);
|
|
875
|
+
expect(result.providerMetadata?.vertex).toStrictEqual({
|
|
876
|
+
images: [
|
|
877
|
+
{
|
|
878
|
+
revisedPrompt: 'revised imagen 4 ultra prompt',
|
|
879
|
+
},
|
|
880
|
+
],
|
|
881
|
+
});
|
|
882
|
+
});
|
|
883
|
+
|
|
884
|
+
it('should handle all provider options with Imagen 4 Ultra', async () => {
|
|
885
|
+
prepareImagen4UltraResponse();
|
|
886
|
+
|
|
887
|
+
await imagen4UltraModel.doGenerate({
|
|
888
|
+
prompt: 'comprehensive test prompt',
|
|
889
|
+
files: undefined,
|
|
890
|
+
mask: undefined,
|
|
891
|
+
n: 1,
|
|
892
|
+
size: undefined,
|
|
893
|
+
aspectRatio: undefined,
|
|
894
|
+
seed: undefined,
|
|
895
|
+
providerOptions: {
|
|
896
|
+
vertex: {
|
|
897
|
+
negativePrompt: 'avoid this content',
|
|
898
|
+
personGeneration: 'dont_allow',
|
|
899
|
+
safetySetting: 'block_only_high',
|
|
900
|
+
addWatermark: true,
|
|
901
|
+
storageUri: 'gs://my-bucket/images/',
|
|
902
|
+
} satisfies GoogleVertexImageProviderOptions,
|
|
903
|
+
},
|
|
904
|
+
});
|
|
905
|
+
|
|
906
|
+
expect(await server.calls[0].requestBodyJson).toMatchInlineSnapshot(`
|
|
907
|
+
{
|
|
908
|
+
"instances": [
|
|
909
|
+
{
|
|
910
|
+
"prompt": "comprehensive test prompt",
|
|
911
|
+
},
|
|
912
|
+
],
|
|
913
|
+
"parameters": {
|
|
914
|
+
"addWatermark": true,
|
|
915
|
+
"negativePrompt": "avoid this content",
|
|
916
|
+
"personGeneration": "dont_allow",
|
|
917
|
+
"safetySetting": "block_only_high",
|
|
918
|
+
"sampleCount": 1,
|
|
919
|
+
"storageUri": "gs://my-bucket/images/",
|
|
920
|
+
},
|
|
921
|
+
}
|
|
922
|
+
`);
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
});
|
|
926
|
+
});
|