@fgv/ts-extras 5.1.0-18 → 5.1.0-19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packlets/ai-assist/apiClient.js +247 -24
- package/dist/packlets/ai-assist/index.js +1 -1
- package/dist/packlets/ai-assist/registry.js +49 -4
- package/dist/packlets/crypto-utils/nodeCryptoProvider.js +96 -0
- package/dist/ts-extras.d.ts +222 -8
- package/lib/packlets/ai-assist/apiClient.d.ts +11 -3
- package/lib/packlets/ai-assist/apiClient.js +245 -22
- package/lib/packlets/ai-assist/index.d.ts +2 -2
- package/lib/packlets/ai-assist/index.js +3 -1
- package/lib/packlets/ai-assist/model.d.ts +66 -5
- package/lib/packlets/ai-assist/registry.d.ts +25 -1
- package/lib/packlets/ai-assist/registry.js +51 -4
- package/lib/packlets/crypto-utils/model.d.ts +92 -0
- package/lib/packlets/crypto-utils/nodeCryptoProvider.d.ts +20 -1
- package/lib/packlets/crypto-utils/nodeCryptoProvider.js +96 -0
- package/package.json +7 -7
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
* @packageDocumentation
|
|
31
31
|
*/
|
|
32
32
|
import { isJsonObject } from '@fgv/ts-json-base';
|
|
33
|
-
import { fail, succeed, Validators } from '@fgv/ts-utils';
|
|
33
|
+
import { fail, mapResults, succeed, Validators } from '@fgv/ts-utils';
|
|
34
34
|
import { resolveModel } from './model';
|
|
35
35
|
import { buildAnthropicMessages, buildGeminiContents, buildMessages, buildOpenAiChatUserContent, buildOpenAiResponsesUserContent } from './chatRequestBuilders';
|
|
36
|
-
import { DEFAULT_MODEL_CAPABILITY_CONFIG } from './registry';
|
|
36
|
+
import { DEFAULT_MODEL_CAPABILITY_CONFIG, resolveImageCapability, supportsImageGeneration } from './registry';
|
|
37
37
|
import { toAnthropicTools, toGeminiTools, toResponsesApiTools } from './toolFormats';
|
|
38
38
|
// ============================================================================
|
|
39
39
|
// Shared helpers
|
|
@@ -84,6 +84,102 @@ async function fetchJson(url, headers, body, logger, signal) {
|
|
|
84
84
|
}
|
|
85
85
|
return succeed(json);
|
|
86
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Makes a multipart/form-data POST request and returns the parsed JSON, or a
|
|
89
|
+
* failure. The Content-Type header (with boundary) is set automatically by
|
|
90
|
+
* `fetch` from the `FormData` body — callers must NOT pass it explicitly.
|
|
91
|
+
* @internal
|
|
92
|
+
*/
|
|
93
|
+
async function fetchMultipart(url, headers, body, logger, signal) {
|
|
94
|
+
/* c8 ignore next 1 - optional logger */
|
|
95
|
+
logger === null || logger === void 0 ? void 0 : logger.detail(`AI API request: POST ${url} (multipart)`);
|
|
96
|
+
let response;
|
|
97
|
+
try {
|
|
98
|
+
response = await fetch(url, {
|
|
99
|
+
method: 'POST',
|
|
100
|
+
headers,
|
|
101
|
+
body,
|
|
102
|
+
signal
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
107
|
+
/* c8 ignore next 1 - optional logger */
|
|
108
|
+
logger === null || logger === void 0 ? void 0 : logger.error(`AI API request failed: ${detail}`);
|
|
109
|
+
return fail(`AI API request failed: ${detail}`);
|
|
110
|
+
}
|
|
111
|
+
if (!response.ok) {
|
|
112
|
+
const errorText = await response.text().catch(() => 'unknown error');
|
|
113
|
+
/* c8 ignore next 1 - optional logger */
|
|
114
|
+
logger === null || logger === void 0 ? void 0 : logger.error(`AI API returned ${response.status}: ${errorText}`);
|
|
115
|
+
return fail(`AI API returned ${response.status}: ${errorText}`);
|
|
116
|
+
}
|
|
117
|
+
/* c8 ignore next 1 - optional logger */
|
|
118
|
+
logger === null || logger === void 0 ? void 0 : logger.detail(`AI API response: ${response.status}`);
|
|
119
|
+
let json;
|
|
120
|
+
try {
|
|
121
|
+
json = await response.json();
|
|
122
|
+
}
|
|
123
|
+
catch (_a) {
|
|
124
|
+
/* c8 ignore next 1 - optional logger */
|
|
125
|
+
logger === null || logger === void 0 ? void 0 : logger.error('AI API returned invalid JSON response');
|
|
126
|
+
return fail('AI API returned invalid JSON response');
|
|
127
|
+
}
|
|
128
|
+
if (!isJsonObject(json)) {
|
|
129
|
+
/* c8 ignore next 1 - optional logger */
|
|
130
|
+
logger === null || logger === void 0 ? void 0 : logger.error('AI API returned non-object JSON response');
|
|
131
|
+
return fail('AI API returned non-object JSON response');
|
|
132
|
+
}
|
|
133
|
+
return succeed(json);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Decodes a base64-encoded image attachment into a `Blob` suitable for use as
|
|
137
|
+
* a multipart file field. On Node hands the `Buffer` straight to `Blob`
|
|
138
|
+
* (Buffer extends Uint8Array) to skip an intermediate copy; falls back to
|
|
139
|
+
* `atob` in browsers. Inputs come from `FileReader` or prior provider
|
|
140
|
+
* responses, which are trusted to be valid. Note that Node's
|
|
141
|
+
* `Buffer.from(..., 'base64')` silently strips invalid characters rather
|
|
142
|
+
* than throwing, so failures are only observable in the browser path.
|
|
143
|
+
* @internal
|
|
144
|
+
*/
|
|
145
|
+
function attachmentToBlob(attachment) {
|
|
146
|
+
if (typeof Buffer !== 'undefined') {
|
|
147
|
+
return succeed(new Blob([Buffer.from(attachment.base64, 'base64')], { type: attachment.mimeType }));
|
|
148
|
+
}
|
|
149
|
+
/* c8 ignore start - Browser-only fallback cannot be tested in Node.js environment */
|
|
150
|
+
try {
|
|
151
|
+
const binary = atob(attachment.base64);
|
|
152
|
+
const bytes = new Uint8Array(binary.length);
|
|
153
|
+
for (let i = 0; i < binary.length; i++) {
|
|
154
|
+
bytes[i] = binary.charCodeAt(i);
|
|
155
|
+
}
|
|
156
|
+
return succeed(new Blob([bytes], { type: attachment.mimeType }));
|
|
157
|
+
}
|
|
158
|
+
catch (e) {
|
|
159
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
160
|
+
return fail(`Invalid base64: ${message}`);
|
|
161
|
+
}
|
|
162
|
+
/* c8 ignore stop */
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Maps a MIME type to a sensible file extension for multipart filenames.
|
|
166
|
+
* @internal
|
|
167
|
+
*/
|
|
168
|
+
function extensionForMimeType(mimeType) {
|
|
169
|
+
switch (mimeType) {
|
|
170
|
+
case 'image/png':
|
|
171
|
+
return 'png';
|
|
172
|
+
case 'image/jpeg':
|
|
173
|
+
case 'image/jpg':
|
|
174
|
+
return 'jpg';
|
|
175
|
+
case 'image/webp':
|
|
176
|
+
return 'webp';
|
|
177
|
+
case 'image/gif':
|
|
178
|
+
return 'gif';
|
|
179
|
+
default:
|
|
180
|
+
return 'bin';
|
|
181
|
+
}
|
|
182
|
+
}
|
|
87
183
|
/**
|
|
88
184
|
* Makes an HTTP GET request and returns the parsed JSON, or a failure.
|
|
89
185
|
* @internal
|
|
@@ -457,6 +553,24 @@ const imagenPrediction = Validators.object({
|
|
|
457
553
|
const imagenResponse = Validators.object({
|
|
458
554
|
predictions: Validators.arrayOf(imagenPrediction).withConstraint((arr) => arr.length > 0)
|
|
459
555
|
});
|
|
556
|
+
const geminiImageInlineData = Validators.object({
|
|
557
|
+
mimeType: Validators.string,
|
|
558
|
+
data: Validators.string
|
|
559
|
+
});
|
|
560
|
+
const geminiImageOutPart = Validators.object({
|
|
561
|
+
text: Validators.string.optional(),
|
|
562
|
+
inlineData: geminiImageInlineData.optional()
|
|
563
|
+
});
|
|
564
|
+
const geminiImageOutContent = Validators.object({
|
|
565
|
+
parts: Validators.arrayOf(geminiImageOutPart).withConstraint((arr) => arr.length > 0)
|
|
566
|
+
});
|
|
567
|
+
const geminiImageOutCandidate = Validators.object({
|
|
568
|
+
content: geminiImageOutContent,
|
|
569
|
+
finishReason: Validators.string.optional()
|
|
570
|
+
});
|
|
571
|
+
const geminiImageOutResponse = Validators.object({
|
|
572
|
+
candidates: Validators.arrayOf(geminiImageOutCandidate).withConstraint((arr) => arr.length > 0)
|
|
573
|
+
});
|
|
460
574
|
// ---- Proxied image generation response ----
|
|
461
575
|
const proxiedGeneratedImage = Validators.object({
|
|
462
576
|
mimeType: Validators.string,
|
|
@@ -482,16 +596,42 @@ const proxiedListModelsResponse = Validators.object({
|
|
|
482
596
|
* formats — the request shape is the same; the only difference is whether the
|
|
483
597
|
* `size` field is honored (OpenAI: yes, xAI: ignored at the provider).
|
|
484
598
|
*
|
|
599
|
+
* When `request.referenceImages` is non-empty, routes to `/images/edits`
|
|
600
|
+
* (multipart) instead of `/images/generations` (JSON). Per-model edit support
|
|
601
|
+
* is not validated here (e.g. dall-e-3 does not support edits) — the
|
|
602
|
+
* provider's 400 surfaces through the failure path.
|
|
603
|
+
*
|
|
485
604
|
* @internal
|
|
486
605
|
*/
|
|
487
606
|
async function callOpenAiImageGeneration(config, request, defaultMimeType, logger, signal) {
|
|
488
|
-
var _a, _b;
|
|
489
|
-
const
|
|
607
|
+
var _a, _b, _c;
|
|
608
|
+
const opts = (_a = request.options) !== null && _a !== void 0 ? _a : {};
|
|
609
|
+
const refs = (_b = request.referenceImages) !== null && _b !== void 0 ? _b : [];
|
|
610
|
+
const headers = {
|
|
611
|
+
Authorization: `Bearer ${config.apiKey}`
|
|
612
|
+
};
|
|
613
|
+
const n = (_c = opts.count) !== null && _c !== void 0 ? _c : 1;
|
|
614
|
+
const fetched = refs.length > 0
|
|
615
|
+
? await callOpenAiImagesEdits(config, request, headers, n, refs, logger, signal)
|
|
616
|
+
: await callOpenAiImagesGenerations(config, request, headers, n, logger, signal);
|
|
617
|
+
return fetched.onSuccess((json) => openAiImageResponse
|
|
618
|
+
.validate(json)
|
|
619
|
+
.withErrorFormat((msg) => `OpenAI images API response: ${msg}`)
|
|
620
|
+
.onSuccess((response) => succeed({
|
|
621
|
+
images: response.data.map((item) => (Object.assign({ mimeType: defaultMimeType, base64: item.b64_json }, (item.revised_prompt !== undefined ? { revisedPrompt: item.revised_prompt } : {}))))
|
|
622
|
+
})));
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Builds and posts the JSON `/images/generations` request (no refs).
|
|
626
|
+
* @internal
|
|
627
|
+
*/
|
|
628
|
+
function callOpenAiImagesGenerations(config, request, headers, n, logger, signal) {
|
|
629
|
+
var _a;
|
|
490
630
|
const opts = (_a = request.options) !== null && _a !== void 0 ? _a : {};
|
|
491
631
|
const body = {
|
|
492
632
|
model: config.model,
|
|
493
633
|
prompt: request.prompt,
|
|
494
|
-
n
|
|
634
|
+
n,
|
|
495
635
|
response_format: 'b64_json'
|
|
496
636
|
};
|
|
497
637
|
if (opts.size !== undefined) {
|
|
@@ -503,22 +643,86 @@ async function callOpenAiImageGeneration(config, request, defaultMimeType, logge
|
|
|
503
643
|
if (opts.seed !== undefined) {
|
|
504
644
|
body.seed = opts.seed;
|
|
505
645
|
}
|
|
646
|
+
/* c8 ignore next 1 - optional logger */
|
|
647
|
+
logger === null || logger === void 0 ? void 0 : logger.info(`Image generation: model=${config.model}, n=${n}`);
|
|
648
|
+
return fetchJson(`${config.baseUrl}/images/generations`, headers, body, logger, signal);
|
|
649
|
+
}
|
|
650
|
+
/**
|
|
651
|
+
* Builds and posts the multipart `/images/edits` request (with refs).
|
|
652
|
+
* @internal
|
|
653
|
+
*/
|
|
654
|
+
async function callOpenAiImagesEdits(config, request, headers, n, refs, logger, signal) {
|
|
655
|
+
var _a;
|
|
656
|
+
const blobsResult = mapResults(refs.map((ref, i) => attachmentToBlob(ref).withErrorFormat((msg) => `reference image ${i}: ${msg}`)));
|
|
657
|
+
/* c8 ignore next 3 - decode failure unreachable via Node's Buffer.from (silently strips invalid input) */
|
|
658
|
+
if (blobsResult.isFailure()) {
|
|
659
|
+
return fail(blobsResult.message);
|
|
660
|
+
}
|
|
661
|
+
const opts = (_a = request.options) !== null && _a !== void 0 ? _a : {};
|
|
662
|
+
const form = new FormData();
|
|
663
|
+
form.append('model', config.model);
|
|
664
|
+
form.append('prompt', request.prompt);
|
|
665
|
+
form.append('n', String(n));
|
|
666
|
+
form.append('response_format', 'b64_json');
|
|
667
|
+
if (opts.size !== undefined) {
|
|
668
|
+
form.append('size', opts.size);
|
|
669
|
+
}
|
|
670
|
+
if (opts.quality !== undefined) {
|
|
671
|
+
form.append('quality', opts.quality);
|
|
672
|
+
}
|
|
673
|
+
if (opts.seed !== undefined) {
|
|
674
|
+
form.append('seed', String(opts.seed));
|
|
675
|
+
}
|
|
676
|
+
blobsResult.value.forEach((blob, i) => {
|
|
677
|
+
form.append('image[]', blob, `ref-${i}.${extensionForMimeType(refs[i].mimeType)}`);
|
|
678
|
+
});
|
|
679
|
+
/* c8 ignore next 1 - optional logger */
|
|
680
|
+
logger === null || logger === void 0 ? void 0 : logger.info(`Image edit: model=${config.model}, n=${n}, refs=${refs.length}`);
|
|
681
|
+
return fetchMultipart(`${config.baseUrl}/images/edits`, headers, form, logger, signal);
|
|
682
|
+
}
|
|
683
|
+
/**
|
|
684
|
+
* Calls Gemini's chat-style `:generateContent` endpoint for image output
|
|
685
|
+
* (Gemini 2.5 Flash Image / "Nano Banana"). Accepts reference images, which
|
|
686
|
+
* are passed as `inlineData` parts alongside the text prompt.
|
|
687
|
+
*
|
|
688
|
+
* @internal
|
|
689
|
+
*/
|
|
690
|
+
async function callGeminiImageOutGeneration(config, request, logger, signal) {
|
|
691
|
+
var _a;
|
|
692
|
+
const url = `${config.baseUrl}/models/${config.model}:generateContent`;
|
|
693
|
+
const refs = (_a = request.referenceImages) !== null && _a !== void 0 ? _a : [];
|
|
694
|
+
const parts = [{ text: request.prompt }];
|
|
695
|
+
for (const ref of refs) {
|
|
696
|
+
parts.push({ inlineData: { mimeType: ref.mimeType, data: ref.base64 } });
|
|
697
|
+
}
|
|
698
|
+
const body = {
|
|
699
|
+
contents: [{ role: 'user', parts }]
|
|
700
|
+
};
|
|
506
701
|
const headers = {
|
|
507
|
-
|
|
702
|
+
'x-goog-api-key': config.apiKey
|
|
508
703
|
};
|
|
509
704
|
/* c8 ignore next 1 - optional logger */
|
|
510
|
-
logger === null || logger === void 0 ? void 0 : logger.info(`
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
}
|
|
515
|
-
return openAiImageResponse
|
|
516
|
-
.validate(jsonResult.value)
|
|
517
|
-
.withErrorFormat((msg) => `OpenAI images API response: ${msg}`)
|
|
705
|
+
logger === null || logger === void 0 ? void 0 : logger.info(`Gemini image-out: model=${config.model}, refs=${refs.length}`);
|
|
706
|
+
return (await fetchJson(url, headers, body, logger, signal)).onSuccess((json) => geminiImageOutResponse
|
|
707
|
+
.validate(json)
|
|
708
|
+
.withErrorFormat((msg) => `Gemini image API response: ${msg}`)
|
|
518
709
|
.onSuccess((response) => {
|
|
519
|
-
const images =
|
|
710
|
+
const images = [];
|
|
711
|
+
for (const candidate of response.candidates) {
|
|
712
|
+
for (const part of candidate.content.parts) {
|
|
713
|
+
if (part.inlineData) {
|
|
714
|
+
images.push({
|
|
715
|
+
mimeType: part.inlineData.mimeType,
|
|
716
|
+
base64: part.inlineData.data
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
if (images.length === 0) {
|
|
722
|
+
return fail('Gemini image API response: no image parts in response');
|
|
723
|
+
}
|
|
520
724
|
return succeed({ images });
|
|
521
|
-
});
|
|
725
|
+
}));
|
|
522
726
|
}
|
|
523
727
|
/**
|
|
524
728
|
* Calls the Gemini Imagen `:predict` endpoint.
|
|
@@ -573,45 +777,61 @@ async function callImagenGeneration(config, request, logger, signal) {
|
|
|
573
777
|
/**
|
|
574
778
|
* Calls the appropriate image-generation API for a given provider.
|
|
575
779
|
*
|
|
576
|
-
*
|
|
780
|
+
* Resolves a {@link IAiImageModelCapability} from
|
|
781
|
+
* {@link IAiProviderDescriptor.imageGeneration} for the requested model and
|
|
782
|
+
* routes by its `format`:
|
|
577
783
|
* - `'openai-images'` for OpenAI (DALL-E, gpt-image-1)
|
|
578
784
|
* - `'xai-images'` for xAI Grok image models
|
|
579
|
-
* - `'gemini-imagen'` for Google Imagen
|
|
785
|
+
* - `'gemini-imagen'` for Google Imagen `:predict`
|
|
786
|
+
* - `'gemini-image-out'` for Gemini chat-style image output (Nano Banana)
|
|
580
787
|
*
|
|
581
788
|
* Image-model selection reuses the existing `'image'` {@link ModelSpecKey}.
|
|
789
|
+
* When `request.referenceImages` is non-empty, the call is rejected up front
|
|
790
|
+
* unless the resolved capability declares `acceptsImageReferenceInput`.
|
|
582
791
|
*
|
|
583
792
|
* @param params - Request parameters including descriptor, API key, and prompt
|
|
584
793
|
* @returns The generated images, or a failure
|
|
585
794
|
* @public
|
|
586
795
|
*/
|
|
587
796
|
export async function callProviderImageGeneration(params) {
|
|
797
|
+
var _a, _b;
|
|
588
798
|
const { descriptor, apiKey, params: request, modelOverride, logger, signal } = params;
|
|
589
|
-
if (descriptor
|
|
799
|
+
if (!supportsImageGeneration(descriptor)) {
|
|
590
800
|
return fail(`provider "${descriptor.id}" does not support image generation`);
|
|
591
801
|
}
|
|
592
802
|
if (!descriptor.baseUrl) {
|
|
593
803
|
return fail(`provider "${descriptor.id}" has no API endpoint configured`);
|
|
594
804
|
}
|
|
805
|
+
const model = resolveModel(modelOverride !== null && modelOverride !== void 0 ? modelOverride : descriptor.defaultModel, 'image');
|
|
806
|
+
const capability = resolveImageCapability(descriptor, model);
|
|
807
|
+
if (capability === undefined) {
|
|
808
|
+
return fail(`provider "${descriptor.id}" does not support image generation for model "${model}"`);
|
|
809
|
+
}
|
|
810
|
+
if (((_b = (_a = request.referenceImages) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0) > 0 && !capability.acceptsImageReferenceInput) {
|
|
811
|
+
return fail(`model "${model}" does not support reference images`);
|
|
812
|
+
}
|
|
595
813
|
const config = {
|
|
596
814
|
baseUrl: descriptor.baseUrl,
|
|
597
815
|
apiKey,
|
|
598
|
-
model
|
|
816
|
+
model
|
|
599
817
|
};
|
|
600
818
|
/* c8 ignore next 6 - optional logger diagnostic output */
|
|
601
819
|
if (logger) {
|
|
602
|
-
logger.info(`AI image generation: provider=${descriptor.id}, format=${
|
|
820
|
+
logger.info(`AI image generation: provider=${descriptor.id}, format=${capability.format}, ` +
|
|
603
821
|
`model=${config.model}`);
|
|
604
822
|
}
|
|
605
|
-
switch (
|
|
823
|
+
switch (capability.format) {
|
|
606
824
|
case 'openai-images':
|
|
607
825
|
return callOpenAiImageGeneration(config, request, 'image/png', logger, signal);
|
|
608
826
|
case 'xai-images':
|
|
609
827
|
return callOpenAiImageGeneration(config, request, 'image/jpeg', logger, signal);
|
|
610
828
|
case 'gemini-imagen':
|
|
611
829
|
return callImagenGeneration(config, request, logger, signal);
|
|
830
|
+
case 'gemini-image-out':
|
|
831
|
+
return callGeminiImageOutGeneration(config, request, logger, signal);
|
|
612
832
|
/* c8 ignore next 4 - defensive coding: exhaustive switch guaranteed by TypeScript */
|
|
613
833
|
default: {
|
|
614
|
-
const _exhaustive =
|
|
834
|
+
const _exhaustive = capability.format;
|
|
615
835
|
return fail(`unsupported image API format: ${String(_exhaustive)}`);
|
|
616
836
|
}
|
|
617
837
|
}
|
|
@@ -961,7 +1181,10 @@ export async function callProxiedCompletion(proxyUrl, params) {
|
|
|
961
1181
|
* - Error response body: `{error: string}` (surfaced as `proxy: ${error}`)
|
|
962
1182
|
*
|
|
963
1183
|
* The proxy server is responsible for descriptor lookup, model resolution,
|
|
964
|
-
* provider dispatch, and response normalization.
|
|
1184
|
+
* provider dispatch, and response normalization. When `params.referenceImages`
|
|
1185
|
+
* is present, the proxy is also responsible for repackaging it into the
|
|
1186
|
+
* upstream wire format (e.g. multipart/form-data for OpenAI `/images/edits`,
|
|
1187
|
+
* `inlineData` parts for Gemini `:generateContent`).
|
|
965
1188
|
*
|
|
966
1189
|
* @param proxyUrl - Base URL of the proxy server (e.g. `http://localhost:3001`)
|
|
967
1190
|
* @param params - Same parameters as {@link callProviderImageGeneration}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* @packageDocumentation
|
|
4
4
|
*/
|
|
5
5
|
export { AiPrompt, DEFAULT_AI_ASSIST, allModelSpecKeys, MODEL_SPEC_BASE_KEY, resolveModel, toDataUrl } from './model';
|
|
6
|
-
export { allProviderIds, getProviderDescriptors, getProviderDescriptor, DEFAULT_MODEL_CAPABILITY_CONFIG } from './registry';
|
|
6
|
+
export { allProviderIds, getProviderDescriptors, getProviderDescriptor, resolveImageCapability, supportsImageGeneration, DEFAULT_MODEL_CAPABILITY_CONFIG } from './registry';
|
|
7
7
|
export { callProviderCompletion, callProxiedCompletion, callProviderImageGeneration, callProxiedImageGeneration, callProviderListModels, callProxiedListModels } from './apiClient';
|
|
8
8
|
export { callProviderCompletionStream, callProxiedCompletionStream } from './streamingClient';
|
|
9
9
|
export { aiProviderId, aiServerToolType, aiWebSearchToolConfig, aiServerToolConfig, aiToolEnablement, aiAssistProviderConfig, aiAssistSettings, modelSpecKey, modelSpec } from './converters';
|
|
@@ -63,12 +63,17 @@ const BUILTIN_PROVIDERS = [
|
|
|
63
63
|
needsSecret: true,
|
|
64
64
|
apiFormat: 'gemini',
|
|
65
65
|
baseUrl: 'https://generativelanguage.googleapis.com/v1beta',
|
|
66
|
-
defaultModel: { base: 'gemini-2.5-flash', image: '
|
|
66
|
+
defaultModel: { base: 'gemini-2.5-flash', image: 'gemini-2.5-flash-image' },
|
|
67
67
|
supportedTools: ['web_search'],
|
|
68
68
|
corsRestricted: false,
|
|
69
69
|
streamingCorsRestricted: false,
|
|
70
70
|
acceptsImageInput: true,
|
|
71
|
-
|
|
71
|
+
imageGeneration: [
|
|
72
|
+
// imagen-* models are predict-only and do not accept reference images;
|
|
73
|
+
// everything else uses chat-style :generateContent with refs.
|
|
74
|
+
{ modelPrefix: 'imagen-', format: 'gemini-imagen' },
|
|
75
|
+
{ modelPrefix: '', format: 'gemini-image-out', acceptsImageReferenceInput: true }
|
|
76
|
+
]
|
|
72
77
|
},
|
|
73
78
|
{
|
|
74
79
|
id: 'groq',
|
|
@@ -108,7 +113,14 @@ const BUILTIN_PROVIDERS = [
|
|
|
108
113
|
corsRestricted: false,
|
|
109
114
|
streamingCorsRestricted: false,
|
|
110
115
|
acceptsImageInput: true,
|
|
111
|
-
|
|
116
|
+
imageGeneration: [
|
|
117
|
+
// gpt-image-1 supports /images/edits with reference images. dall-e-3
|
|
118
|
+
// (the default image model) does not, so the catch-all rule omits
|
|
119
|
+
// acceptsImageReferenceInput; callers selecting dall-e-3 with refs hit
|
|
120
|
+
// the up-front rejection rather than a provider 400.
|
|
121
|
+
{ modelPrefix: 'gpt-image-', format: 'openai-images', acceptsImageReferenceInput: true },
|
|
122
|
+
{ modelPrefix: '', format: 'openai-images' }
|
|
123
|
+
]
|
|
112
124
|
},
|
|
113
125
|
{
|
|
114
126
|
id: 'xai-grok',
|
|
@@ -126,7 +138,7 @@ const BUILTIN_PROVIDERS = [
|
|
|
126
138
|
corsRestricted: true,
|
|
127
139
|
streamingCorsRestricted: true,
|
|
128
140
|
acceptsImageInput: true,
|
|
129
|
-
|
|
141
|
+
imageGeneration: [{ modelPrefix: '', format: 'xai-images' }]
|
|
130
142
|
}
|
|
131
143
|
];
|
|
132
144
|
/**
|
|
@@ -163,6 +175,38 @@ export function getProviderDescriptor(id) {
|
|
|
163
175
|
}
|
|
164
176
|
return succeed(descriptor);
|
|
165
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* Whether a provider declares any image-generation capability at all.
|
|
180
|
+
*
|
|
181
|
+
* @param descriptor - The provider descriptor
|
|
182
|
+
* @returns `true` when {@link IAiProviderDescriptor.imageGeneration} has at
|
|
183
|
+
* least one entry; `false` otherwise.
|
|
184
|
+
* @public
|
|
185
|
+
*/
|
|
186
|
+
export function supportsImageGeneration(descriptor) {
|
|
187
|
+
var _a, _b;
|
|
188
|
+
return ((_b = (_a = descriptor.imageGeneration) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0) > 0;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Resolve the image-generation capability that applies to a given model id
|
|
192
|
+
* for a provider. Returns the entry from
|
|
193
|
+
* {@link IAiProviderDescriptor.imageGeneration} whose `modelPrefix` is the
|
|
194
|
+
* longest prefix of `modelId`. Ties are broken by first-encountered, so rule
|
|
195
|
+
* order does not matter for correctness — only for tie-breaking among rules
|
|
196
|
+
* with identical-length prefixes (an unusual case).
|
|
197
|
+
*
|
|
198
|
+
* @param descriptor - The provider descriptor
|
|
199
|
+
* @param modelId - The resolved image model id
|
|
200
|
+
* @returns The matching capability, or `undefined` when no rule matches or
|
|
201
|
+
* the provider declares no image-generation capabilities.
|
|
202
|
+
* @public
|
|
203
|
+
*/
|
|
204
|
+
export function resolveImageCapability(descriptor, modelId) {
|
|
205
|
+
var _a;
|
|
206
|
+
return ((_a = descriptor.imageGeneration) !== null && _a !== void 0 ? _a : [])
|
|
207
|
+
.filter((cap) => modelId.startsWith(cap.modelPrefix))
|
|
208
|
+
.reduce((best, cap) => (best && best.modelPrefix.length >= cap.modelPrefix.length ? best : cap), undefined);
|
|
209
|
+
}
|
|
166
210
|
// ============================================================================
|
|
167
211
|
// Default model capability config
|
|
168
212
|
// ============================================================================
|
|
@@ -191,6 +235,7 @@ export const DEFAULT_MODEL_CAPABILITY_CONFIG = {
|
|
|
191
235
|
],
|
|
192
236
|
'google-gemini': [
|
|
193
237
|
{ idPattern: /^imagen/, capabilities: ['image-generation'] },
|
|
238
|
+
{ idPattern: /^gemini-.*-image/, capabilities: ['image-generation'] },
|
|
194
239
|
{ idPattern: /^gemini-/, capabilities: ['chat', 'tools', 'vision'] }
|
|
195
240
|
],
|
|
196
241
|
anthropic: [{ idPattern: /^claude-/, capabilities: ['chat', 'tools', 'vision'] }],
|
|
@@ -205,6 +205,102 @@ export class NodeCryptoProvider {
|
|
|
205
205
|
const result = await captureAsyncResult(() => crypto.webcrypto.subtle.importKey('jwk', jwk, params.importPublicKey, true, params.publicKeyUsages));
|
|
206
206
|
return result.withErrorFormat((e) => `Failed to import ${algorithm} public key from JWK: ${e}`);
|
|
207
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Wraps `plaintext` for the holder of `recipientPublicKey` using
|
|
210
|
+
* ECIES (ECDH P-256 + HKDF-SHA256 + AES-GCM-256). See
|
|
211
|
+
* {@link CryptoUtils.ICryptoProvider.wrapBytes | ICryptoProvider.wrapBytes}.
|
|
212
|
+
* @param plaintext - The bytes to wrap.
|
|
213
|
+
* @param recipientPublicKey - The recipient's ECDH P-256 public `CryptoKey`.
|
|
214
|
+
* @param options - HKDF salt and info; see {@link CryptoUtils.IWrapBytesOptions | IWrapBytesOptions}.
|
|
215
|
+
* @returns `Success` with the wrapped payload, or `Failure` with an error.
|
|
216
|
+
*/
|
|
217
|
+
async wrapBytes(plaintext, recipientPublicKey, options) {
|
|
218
|
+
const recipientCheck = checkEcdhP256(recipientPublicKey, 'public', 'recipient public key');
|
|
219
|
+
if (recipientCheck.isFailure()) {
|
|
220
|
+
return fail(`wrapBytes failed: ${recipientCheck.message}`);
|
|
221
|
+
}
|
|
222
|
+
const subtle = crypto.webcrypto.subtle;
|
|
223
|
+
const result = await captureAsyncResult(async () => {
|
|
224
|
+
const ephemeral = (await subtle.generateKey({ name: 'ECDH', namedCurve: 'P-256' }, true, [
|
|
225
|
+
'deriveKey'
|
|
226
|
+
]));
|
|
227
|
+
const hkdfBase = await subtle.deriveKey({ name: 'ECDH', public: recipientPublicKey }, ephemeral.privateKey, { name: 'HKDF' }, false, ['deriveKey']);
|
|
228
|
+
const wrapKey = await subtle.deriveKey({ name: 'HKDF', salt: options.salt, info: options.info, hash: 'SHA-256' }, hkdfBase, { name: 'AES-GCM', length: 256 }, false, ['encrypt']);
|
|
229
|
+
const nonce = crypto.randomBytes(Constants.GCM_IV_SIZE);
|
|
230
|
+
const ctBuf = await subtle.encrypt({ name: 'AES-GCM', iv: nonce }, wrapKey, plaintext);
|
|
231
|
+
const ephemeralPublicKey = await subtle.exportKey('jwk', ephemeral.publicKey);
|
|
232
|
+
return {
|
|
233
|
+
ephemeralPublicKey,
|
|
234
|
+
nonce: this.toBase64(nonce),
|
|
235
|
+
ciphertext: this.toBase64(new Uint8Array(ctBuf))
|
|
236
|
+
};
|
|
237
|
+
});
|
|
238
|
+
return result.withErrorFormat((e) => `wrapBytes failed: ${e}`);
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Unwraps a payload produced by `wrapBytes` using the recipient's private
|
|
242
|
+
* key. See {@link CryptoUtils.ICryptoProvider.unwrapBytes | ICryptoProvider.unwrapBytes}.
|
|
243
|
+
* @param wrapped - The wrapped payload.
|
|
244
|
+
* @param recipientPrivateKey - The recipient's ECDH P-256 private `CryptoKey`.
|
|
245
|
+
* @param options - HKDF salt and info matching the wrap call.
|
|
246
|
+
* @returns `Success` with the original `plaintext`, or `Failure` with an error.
|
|
247
|
+
*/
|
|
248
|
+
async unwrapBytes(wrapped, recipientPrivateKey, options) {
|
|
249
|
+
const recipientCheck = checkEcdhP256(recipientPrivateKey, 'private', 'recipient private key');
|
|
250
|
+
if (recipientCheck.isFailure()) {
|
|
251
|
+
return fail(`unwrapBytes failed: ${recipientCheck.message}`);
|
|
252
|
+
}
|
|
253
|
+
const nonceResult = this.fromBase64(wrapped.nonce);
|
|
254
|
+
if (nonceResult.isFailure()) {
|
|
255
|
+
return fail(`unwrapBytes failed: nonce: ${nonceResult.message}`);
|
|
256
|
+
}
|
|
257
|
+
if (nonceResult.value.length !== Constants.GCM_IV_SIZE) {
|
|
258
|
+
return fail(`unwrapBytes failed: nonce must be ${Constants.GCM_IV_SIZE} bytes (got ${nonceResult.value.length})`);
|
|
259
|
+
}
|
|
260
|
+
const ciphertextResult = this.fromBase64(wrapped.ciphertext);
|
|
261
|
+
if (ciphertextResult.isFailure()) {
|
|
262
|
+
return fail(`unwrapBytes failed: ciphertext: ${ciphertextResult.message}`);
|
|
263
|
+
}
|
|
264
|
+
if (ciphertextResult.value.length < Constants.GCM_AUTH_TAG_SIZE) {
|
|
265
|
+
return fail(`unwrapBytes failed: ciphertext must be at least ${Constants.GCM_AUTH_TAG_SIZE} bytes (got ${ciphertextResult.value.length})`);
|
|
266
|
+
}
|
|
267
|
+
const subtle = crypto.webcrypto.subtle;
|
|
268
|
+
const result = await captureAsyncResult(async () => {
|
|
269
|
+
const ephemeralPub = await subtle.importKey('jwk', wrapped.ephemeralPublicKey, { name: 'ECDH', namedCurve: 'P-256' }, false, []);
|
|
270
|
+
const hkdfBase = await subtle.deriveKey({ name: 'ECDH', public: ephemeralPub }, recipientPrivateKey, { name: 'HKDF' }, false, ['deriveKey']);
|
|
271
|
+
const wrapKey = await subtle.deriveKey({ name: 'HKDF', salt: options.salt, info: options.info, hash: 'SHA-256' }, hkdfBase, { name: 'AES-GCM', length: 256 }, false, ['decrypt']);
|
|
272
|
+
const ptBuf = await subtle.decrypt({ name: 'AES-GCM', iv: nonceResult.value }, wrapKey, ciphertextResult.value);
|
|
273
|
+
return new Uint8Array(ptBuf);
|
|
274
|
+
});
|
|
275
|
+
return result.withErrorFormat((e) => `unwrapBytes failed: ${e}`);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Verifies that `key` is an ECDH P-256 `CryptoKey` of the expected `keyType`
|
|
280
|
+
* (public or private). Used by the wrap/unwrap methods to surface a clean
|
|
281
|
+
* `Failure` instead of letting the WebCrypto deriveKey call throw a less
|
|
282
|
+
* informative error later in the pipeline. Key usages are intentionally not
|
|
283
|
+
* checked here: WebCrypto already produces a specific error if `deriveKey` is
|
|
284
|
+
* not in `usages`, and `deriveBits` is an equally valid alternative usage that
|
|
285
|
+
* an explicit check would have to track.
|
|
286
|
+
* @param key - The CryptoKey to validate.
|
|
287
|
+
* @param keyType - The required `key.type` ('public' for wrap, 'private' for unwrap).
|
|
288
|
+
* @param label - Human-readable role label included in the failure message.
|
|
289
|
+
* @returns `Success` with the key (unchanged) when the algorithm, curve, and
|
|
290
|
+
* type all match; otherwise `Failure` with `<label> must be ECDH P-256 (...)`.
|
|
291
|
+
*/
|
|
292
|
+
function checkEcdhP256(key, keyType, label) {
|
|
293
|
+
if (key.algorithm.name !== 'ECDH') {
|
|
294
|
+
return fail(`${label} must be ECDH P-256 (got algorithm '${key.algorithm.name}')`);
|
|
295
|
+
}
|
|
296
|
+
const namedCurve = key.algorithm.namedCurve;
|
|
297
|
+
if (namedCurve !== 'P-256') {
|
|
298
|
+
return fail(`${label} must be ECDH P-256 (got curve '${namedCurve}')`);
|
|
299
|
+
}
|
|
300
|
+
if (key.type !== keyType) {
|
|
301
|
+
return fail(`${label} must be a ${keyType} CryptoKey (got '${key.type}')`);
|
|
302
|
+
}
|
|
303
|
+
return succeed(key);
|
|
208
304
|
}
|
|
209
305
|
/**
|
|
210
306
|
* Singleton instance of {@link CryptoUtils.NodeCryptoProvider}.
|