@sixb/azure-ai-foundry 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +187 -2
- package/dist/decision.d.ts +18 -0
- package/dist/decision.d.ts.map +1 -0
- package/dist/discovery.d.ts +3 -0
- package/dist/discovery.d.ts.map +1 -1
- package/dist/embedding.d.ts +1 -1
- package/dist/embedding.d.ts.map +1 -1
- package/dist/image.d.ts +20 -0
- package/dist/image.d.ts.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1176 -70
- package/dist/index.js.map +14 -8
- package/dist/kinds.d.ts +13 -0
- package/dist/kinds.d.ts.map +1 -0
- package/dist/provider.d.ts +15 -0
- package/dist/provider.d.ts.map +1 -1
- package/dist/reranking.d.ts +18 -0
- package/dist/reranking.d.ts.map +1 -0
- package/dist/speech.d.ts +20 -0
- package/dist/speech.d.ts.map +1 -0
- package/dist/transcription.d.ts +20 -0
- package/dist/transcription.d.ts.map +1 -0
- package/dist/transport.d.ts +10 -2
- package/dist/transport.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/decision.ts +211 -0
- package/src/discovery.ts +18 -0
- package/src/embedding.ts +15 -36
- package/src/image.ts +457 -0
- package/src/index.ts +8 -0
- package/src/kinds.ts +100 -0
- package/src/provider.ts +73 -12
- package/src/reranking.ts +264 -0
- package/src/speech.ts +390 -0
- package/src/transcription.ts +322 -0
- package/src/transport.ts +90 -11
package/src/image.ts
ADDED
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
import {
|
|
2
|
+
defineModelRateCard,
|
|
3
|
+
estimateModelReservation,
|
|
4
|
+
type ImageCostEstimator,
|
|
5
|
+
type ImageData,
|
|
6
|
+
type ImageModel,
|
|
7
|
+
type ImageModelDefinition,
|
|
8
|
+
type ImageModelRequest,
|
|
9
|
+
ImageModelResponseError,
|
|
10
|
+
type ImageModelResponseMetadata,
|
|
11
|
+
type ImageModelResult,
|
|
12
|
+
type JsonObject,
|
|
13
|
+
type ModelRateCard,
|
|
14
|
+
type ModelUsage,
|
|
15
|
+
rateModelCall,
|
|
16
|
+
UnsupportedModelFeatureError,
|
|
17
|
+
} from "@sixb/core/models"
|
|
18
|
+
import {
|
|
19
|
+
type AzureAIFoundryDeployment,
|
|
20
|
+
deploymentMetadata,
|
|
21
|
+
type FoundryDeployments,
|
|
22
|
+
type ResolvedDeployment,
|
|
23
|
+
} from "./discovery"
|
|
24
|
+
import { assertDeploymentKind } from "./kinds"
|
|
25
|
+
import type { AzureAIFoundryModelMetadata } from "./provider"
|
|
26
|
+
import { azurePublisher } from "./publisher"
|
|
27
|
+
import { abortable, type FoundryTransport, readJson, requestId } from "./transport"
|
|
28
|
+
import { counter, object, PREFIX, positiveInteger } from "./util"
|
|
29
|
+
|
|
30
|
+
export interface AzureAIFoundryImageOptions {
|
|
31
|
+
/** When to pick this model, for people and the agent. */
|
|
32
|
+
readonly description?: string
|
|
33
|
+
/** Deadline for one generation, response included. Default: 180,000 ms. */
|
|
34
|
+
readonly timeoutMs?: number
|
|
35
|
+
readonly rateCard?: ModelRateCard
|
|
36
|
+
readonly costEstimator?: ImageCostEstimator
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface AzureAIFoundryImageModel extends ImageModel {
|
|
40
|
+
readonly metadata: AzureAIFoundryModelMetadata
|
|
41
|
+
resolve(options?: { readonly offline?: boolean }): Promise<AzureAIFoundryImageModel>
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Ten large PNGs in base64; a reply over it fails, keeping its request ID. */
|
|
45
|
+
const MAX_RESPONSE_BYTES = 512 * 1024 * 1024
|
|
46
|
+
|
|
47
|
+
/** AbortSignal.timeout's largest delay. */
|
|
48
|
+
const MAX_TIMEOUT_MS = 2 ** 31 - 1
|
|
49
|
+
|
|
50
|
+
const INPUT_MEDIA_TYPES = Object.freeze(["image/png", "image/jpeg", "image/webp"])
|
|
51
|
+
const EXTENSIONS: Readonly<Record<string, string>> = {
|
|
52
|
+
"image/png": "png",
|
|
53
|
+
"image/jpeg": "jpg",
|
|
54
|
+
"image/webp": "webp",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* What each family takes through Azure's OpenAI images API. GPT Image returns up to 10 images
|
|
59
|
+
* and edits up to 16 references; FLUX returns one (Azure rejects `n` above 1), FLUX.1 Kontext
|
|
60
|
+
* edits one reference, and FLUX1.1 [pro] edits none.
|
|
61
|
+
*/
|
|
62
|
+
interface Family {
|
|
63
|
+
readonly gpt: boolean
|
|
64
|
+
readonly definition: Omit<ImageModelDefinition, "kind" | "providerId" | "modelId">
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const GPT_SIZES = Object.freeze(["auto", "1024x1024", "1536x1024", "1024x1536"])
|
|
68
|
+
const GPT_QUALITIES = Object.freeze(["auto", "low", "medium", "high"])
|
|
69
|
+
const GPT_INPUT = Object.freeze({
|
|
70
|
+
maxCount: 16,
|
|
71
|
+
maxBytes: 50 * 1024 * 1024,
|
|
72
|
+
mediaTypes: INPUT_MEDIA_TYPES,
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
function family(deployment: AzureAIFoundryDeployment): Family {
|
|
76
|
+
const name = deployment.modelName.toLowerCase()
|
|
77
|
+
if (name.startsWith("gpt-image-"))
|
|
78
|
+
return {
|
|
79
|
+
gpt: true,
|
|
80
|
+
definition: {
|
|
81
|
+
maxImagesPerCall: 10,
|
|
82
|
+
// gpt-image-2 and later take free-form sizes; Azure validates them.
|
|
83
|
+
...(/^gpt-image-1(\.5|-mini)?$/.test(name) ? { sizes: GPT_SIZES } : {}),
|
|
84
|
+
qualities: GPT_QUALITIES,
|
|
85
|
+
inputImages: GPT_INPUT,
|
|
86
|
+
},
|
|
87
|
+
}
|
|
88
|
+
if (name === "flux.1-kontext-pro")
|
|
89
|
+
return {
|
|
90
|
+
gpt: false,
|
|
91
|
+
definition: {
|
|
92
|
+
maxImagesPerCall: 1,
|
|
93
|
+
inputImages: { maxCount: 1, maxBytes: 20 * 1024 * 1024, mediaTypes: INPUT_MEDIA_TYPES },
|
|
94
|
+
},
|
|
95
|
+
}
|
|
96
|
+
if (name === "flux-1.1-pro") return { gpt: false, definition: { maxImagesPerCall: 1 } }
|
|
97
|
+
// FLUX.2 answers only Black Forest Labs' own route, with a request and per-megapixel price of
|
|
98
|
+
// its own; DALL-E 3 is retired.
|
|
99
|
+
throw new UnsupportedModelFeatureError(
|
|
100
|
+
`${PREFIX} ${deployment.modelName} is not supported: use a GPT Image, FLUX.1 Kontext [pro] or FLUX1.1 [pro] deployment.`
|
|
101
|
+
)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Azure Retail Prices (verified 2026-11-13), by deployment SKU. GPT Image bills USD per million
|
|
106
|
+
* text input, text output and image output tokens; image input tokens bill at another rate that a
|
|
107
|
+
* rate card cannot hold, so an edit stays unpriced. FLUX bills per image.
|
|
108
|
+
*/
|
|
109
|
+
interface PublishedImagePrice {
|
|
110
|
+
readonly card: ModelRateCard
|
|
111
|
+
/** Whether Azure publishes a price for text output tokens. */
|
|
112
|
+
readonly textOutput: boolean
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* With a text output price, image output tokens are billed apart, which needs the reply's split.
|
|
117
|
+
* Without one, every output token is billed at the image rate: gpt-image-1 reports no split.
|
|
118
|
+
*/
|
|
119
|
+
const gpt = (input: string, imageOutput: string, textOutput?: string): PublishedImagePrice => ({
|
|
120
|
+
card:
|
|
121
|
+
textOutput === undefined
|
|
122
|
+
? { currency: "USD", input, output: imageOutput }
|
|
123
|
+
: { currency: "USD", input, output: textOutput, imageOutput },
|
|
124
|
+
textOutput: textOutput !== undefined,
|
|
125
|
+
})
|
|
126
|
+
const perImage = (images: string): PublishedImagePrice => ({
|
|
127
|
+
card: { currency: "USD", images },
|
|
128
|
+
textOutput: false,
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
const PUBLISHED: Readonly<
|
|
132
|
+
Record<string, { readonly version: string; readonly skus: Record<string, PublishedImagePrice> }>
|
|
133
|
+
> = {
|
|
134
|
+
// gpt-image-1-{inp-txt,output-img}-{glbl,dzone}
|
|
135
|
+
"gpt-image-1": {
|
|
136
|
+
version: "2025-04-15",
|
|
137
|
+
skus: { GlobalStandard: gpt("5", "40"), DataZoneStandard: gpt("5.5", "44") },
|
|
138
|
+
},
|
|
139
|
+
// gpt img 1 mini {inp txt,out img} {glbl,dzone}
|
|
140
|
+
"gpt-image-1-mini": {
|
|
141
|
+
version: "2025-10-06",
|
|
142
|
+
skus: { GlobalStandard: gpt("2", "8"), DataZoneStandard: gpt("2.2", "8.8") },
|
|
143
|
+
},
|
|
144
|
+
// gpt img 1.5 {in txt,out img,out txt} {gl,DZ}
|
|
145
|
+
"gpt-image-1.5": {
|
|
146
|
+
version: "2025-12-16",
|
|
147
|
+
skus: {
|
|
148
|
+
GlobalStandard: gpt("5", "32", "10"),
|
|
149
|
+
DataZoneStandard: gpt("5.5", "35.2", "11"),
|
|
150
|
+
},
|
|
151
|
+
},
|
|
152
|
+
// Image 2 {txt inp,img opt} {Gl,DZ}
|
|
153
|
+
"gpt-image-2": {
|
|
154
|
+
version: "2026-04-21",
|
|
155
|
+
skus: { GlobalStandard: gpt("5", "30"), DataZoneStandard: gpt("5.5", "33") },
|
|
156
|
+
},
|
|
157
|
+
// Kontext Pro {glbl,Dzone,regnl} Images
|
|
158
|
+
"flux.1-kontext-pro": {
|
|
159
|
+
version: "1",
|
|
160
|
+
skus: {
|
|
161
|
+
GlobalStandard: perImage("0.04"),
|
|
162
|
+
DataZoneStandard: perImage("0.044"),
|
|
163
|
+
Standard: perImage("0.044"),
|
|
164
|
+
},
|
|
165
|
+
},
|
|
166
|
+
// Flux 1.1 Pro {glbl,Dzone,regnl} Images
|
|
167
|
+
"flux-1.1-pro": {
|
|
168
|
+
version: "1",
|
|
169
|
+
skus: {
|
|
170
|
+
GlobalStandard: perImage("0.04"),
|
|
171
|
+
DataZoneStandard: perImage("0.044"),
|
|
172
|
+
Standard: perImage("0.044"),
|
|
173
|
+
},
|
|
174
|
+
},
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
interface ImageSettings {
|
|
178
|
+
readonly description?: string
|
|
179
|
+
readonly timeoutMs: number
|
|
180
|
+
readonly rateCard?: ModelRateCard
|
|
181
|
+
readonly costEstimator?: ImageCostEstimator
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
interface ImageResolution extends ResolvedDeployment {
|
|
185
|
+
readonly family: Family
|
|
186
|
+
readonly costEstimator?: ImageCostEstimator
|
|
187
|
+
readonly rateCard?: ModelRateCard
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export function createFoundryImage(
|
|
191
|
+
providerId: string,
|
|
192
|
+
modelId: string,
|
|
193
|
+
options: AzureAIFoundryImageOptions,
|
|
194
|
+
transport: FoundryTransport,
|
|
195
|
+
deployments: FoundryDeployments
|
|
196
|
+
): AzureAIFoundryImageModel {
|
|
197
|
+
if (!modelId.trim() || modelId !== modelId.trim())
|
|
198
|
+
throw new TypeError(`${PREFIX} deploymentName must be nonempty without surrounding whitespace.`)
|
|
199
|
+
positiveInteger(options.timeoutMs, "timeoutMs")
|
|
200
|
+
if (options.timeoutMs !== undefined && options.timeoutMs > MAX_TIMEOUT_MS)
|
|
201
|
+
throw new TypeError(`${PREFIX} timeoutMs must be at most ${MAX_TIMEOUT_MS}.`)
|
|
202
|
+
if (
|
|
203
|
+
options.description !== undefined &&
|
|
204
|
+
(typeof options.description !== "string" || !options.description.trim())
|
|
205
|
+
)
|
|
206
|
+
throw new TypeError(`${PREFIX} description must be a nonblank string.`)
|
|
207
|
+
if (options.rateCard && options.costEstimator)
|
|
208
|
+
throw new TypeError(`${PREFIX} Configure either rateCard or costEstimator, not both.`)
|
|
209
|
+
// Validate and snapshot now: later edits to the caller's object change nothing.
|
|
210
|
+
const settings: ImageSettings = Object.freeze({
|
|
211
|
+
timeoutMs: options.timeoutMs ?? 180_000,
|
|
212
|
+
...(options.description ? { description: options.description } : {}),
|
|
213
|
+
...(options.rateCard ? { rateCard: defineModelRateCard(options.rateCard) } : {}),
|
|
214
|
+
...(options.costEstimator ? { costEstimator: options.costEstimator } : {}),
|
|
215
|
+
})
|
|
216
|
+
return imageBinding(providerId, modelId, settings, transport, deployments)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Discovery pins the deployed model's family and price; inference uses the images API. */
|
|
220
|
+
function imageBinding(
|
|
221
|
+
providerId: string,
|
|
222
|
+
modelId: string,
|
|
223
|
+
settings: ImageSettings,
|
|
224
|
+
transport: FoundryTransport,
|
|
225
|
+
deployments: FoundryDeployments,
|
|
226
|
+
resolution?: ImageResolution
|
|
227
|
+
): AzureAIFoundryImageModel {
|
|
228
|
+
const deployment = resolution?.deployment
|
|
229
|
+
const definition: ImageModelDefinition = Object.freeze({
|
|
230
|
+
kind: "image",
|
|
231
|
+
providerId,
|
|
232
|
+
modelId,
|
|
233
|
+
...(deployment ? { publisher: azurePublisher(deployment.modelPublisher) } : {}),
|
|
234
|
+
via: "Azure AI Foundry",
|
|
235
|
+
...(settings.description ? { description: settings.description } : {}),
|
|
236
|
+
// Sixb checks a request against the binding before resolving it: until then, the most any
|
|
237
|
+
// supported family takes. Resolution narrows it to the deployment's family.
|
|
238
|
+
...(resolution ? resolution.family.definition : { inputImages: GPT_INPUT }),
|
|
239
|
+
})
|
|
240
|
+
|
|
241
|
+
let pending: Promise<AzureAIFoundryImageModel> | undefined
|
|
242
|
+
const binding: AzureAIFoundryImageModel = Object.freeze({
|
|
243
|
+
providerId,
|
|
244
|
+
modelId,
|
|
245
|
+
definition,
|
|
246
|
+
metadata: deploymentMetadata(resolution),
|
|
247
|
+
...(resolution?.costEstimator ? { costEstimator: resolution.costEstimator } : {}),
|
|
248
|
+
...(resolution?.rateCard ? { rateCard: resolution.rateCard } : {}),
|
|
249
|
+
|
|
250
|
+
async resolve(input?: { readonly offline?: boolean }) {
|
|
251
|
+
if (resolution) return binding
|
|
252
|
+
const resolved = await deployments.resolve(modelId, input?.offline === true)
|
|
253
|
+
assertDeploymentKind(resolved.deployment, modelId, "image")
|
|
254
|
+
const found = family(resolved.deployment)
|
|
255
|
+
const published = settings.costEstimator ? undefined : publishedPrice(resolved.deployment)
|
|
256
|
+
const rateCard = settings.costEstimator ? undefined : (settings.rateCard ?? published?.card)
|
|
257
|
+
return imageBinding(providerId, modelId, settings, transport, deployments, {
|
|
258
|
+
...resolved,
|
|
259
|
+
family: found,
|
|
260
|
+
...(rateCard ? { rateCard } : {}),
|
|
261
|
+
...(settings.costEstimator || rateCard
|
|
262
|
+
? {
|
|
263
|
+
costEstimator:
|
|
264
|
+
settings.costEstimator ??
|
|
265
|
+
imageEstimator(rateCard!, settings.rateCard ? true : published!.textOutput),
|
|
266
|
+
}
|
|
267
|
+
: {}),
|
|
268
|
+
})
|
|
269
|
+
},
|
|
270
|
+
|
|
271
|
+
async generate(request: ImageModelRequest): Promise<ImageModelResult> {
|
|
272
|
+
const signal = AbortSignal.any([
|
|
273
|
+
...(request.signal ? [request.signal] : []),
|
|
274
|
+
AbortSignal.timeout(settings.timeoutMs),
|
|
275
|
+
])
|
|
276
|
+
signal.throwIfAborted()
|
|
277
|
+
if (typeof request.prompt !== "string" || !request.prompt.trim())
|
|
278
|
+
throw new TypeError(`${PREFIX} Image prompt must be a nonblank string.`)
|
|
279
|
+
if (!Number.isSafeInteger(request.n) || request.n < 1)
|
|
280
|
+
throw new TypeError(`${PREFIX} Image count must be a positive integer.`)
|
|
281
|
+
if (!resolution) {
|
|
282
|
+
// A direct call pins the deployment once, as the runtime does before admission.
|
|
283
|
+
pending ??= binding.resolve().catch((error) => {
|
|
284
|
+
pending = undefined
|
|
285
|
+
throw error
|
|
286
|
+
})
|
|
287
|
+
const resolved = await abortable(() => pending!, signal)
|
|
288
|
+
return resolved.generate({ ...request, signal })
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const { gpt } = resolution.family
|
|
292
|
+
const { maxImagesPerCall, inputImages } = resolution.family.definition
|
|
293
|
+
if (maxImagesPerCall !== undefined && request.n > maxImagesPerCall)
|
|
294
|
+
throw new TypeError(`${PREFIX} ${modelId} returns at most ${maxImagesPerCall} per call.`)
|
|
295
|
+
const images = request.images ?? []
|
|
296
|
+
if (images.length > (inputImages?.maxCount ?? 0))
|
|
297
|
+
throw new TypeError(
|
|
298
|
+
`${PREFIX} ${modelId} takes at most ${inputImages?.maxCount ?? 0} reference images.`
|
|
299
|
+
)
|
|
300
|
+
for (const image of images)
|
|
301
|
+
if (!EXTENSIONS[image.mediaType] || image.data.byteLength > inputImages!.maxBytes)
|
|
302
|
+
throw new TypeError(
|
|
303
|
+
`${PREFIX} Reference images must be PNG, JPEG or WebP within maxBytes.`
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
// The API has no seed, style or aspect ratio; FLUX has no quality.
|
|
307
|
+
const warnings = [
|
|
308
|
+
...(request.seed === undefined ? [] : ["seed: not supported"]),
|
|
309
|
+
...(request.style === undefined ? [] : ["style: not supported"]),
|
|
310
|
+
...(request.aspectRatio === undefined ? [] : ["aspectRatio: not supported; use size"]),
|
|
311
|
+
...(request.quality === undefined || gpt ? [] : ["quality: not supported"]),
|
|
312
|
+
]
|
|
313
|
+
// One image is the default; FLUX.1 Kontext's edit form rejects even `n=1`.
|
|
314
|
+
const fields = {
|
|
315
|
+
prompt: request.prompt,
|
|
316
|
+
...(request.n === 1 ? {} : { n: request.n }),
|
|
317
|
+
...(request.size === undefined ? {} : { size: request.size }),
|
|
318
|
+
...(request.quality === undefined || !gpt ? {} : { quality: request.quality }),
|
|
319
|
+
}
|
|
320
|
+
const edit = images.length > 0
|
|
321
|
+
const response = await transport.post(
|
|
322
|
+
edit ? editForm(fields, images, gpt) : JSON.stringify(fields),
|
|
323
|
+
signal,
|
|
324
|
+
providerId,
|
|
325
|
+
modelId,
|
|
326
|
+
edit ? "image-edit" : "image"
|
|
327
|
+
)
|
|
328
|
+
const id = transport.redactText(response, requestId(response))
|
|
329
|
+
const providerIds = id ? { providerIds: { requestId: id } } : {}
|
|
330
|
+
let body: unknown
|
|
331
|
+
try {
|
|
332
|
+
body = await readJson(response, signal, MAX_RESPONSE_BYTES)
|
|
333
|
+
} catch {
|
|
334
|
+
if (signal.aborted) throw signal.reason
|
|
335
|
+
// Azure billed the images: the error keeps the request ID to trace them.
|
|
336
|
+
throw new ImageModelResponseError(
|
|
337
|
+
`${PREFIX} Image generation returned invalid or oversized JSON.`,
|
|
338
|
+
providerId,
|
|
339
|
+
modelId,
|
|
340
|
+
providerIds
|
|
341
|
+
)
|
|
342
|
+
}
|
|
343
|
+
const payload = object(body)
|
|
344
|
+
const decoded = decodeImages(payload?.data)
|
|
345
|
+
const metadata: ImageModelResponseMetadata = {
|
|
346
|
+
// Azure bills every image it returned, readable or not.
|
|
347
|
+
usage: imageUsage(payload, Array.isArray(payload?.data) ? payload.data.length : undefined),
|
|
348
|
+
...providerIds,
|
|
349
|
+
}
|
|
350
|
+
if (!decoded || decoded.length === 0 || decoded.length > request.n)
|
|
351
|
+
throw new ImageModelResponseError(
|
|
352
|
+
`${PREFIX} Image generation returned no readable images.`,
|
|
353
|
+
providerId,
|
|
354
|
+
modelId,
|
|
355
|
+
metadata
|
|
356
|
+
)
|
|
357
|
+
return {
|
|
358
|
+
...metadata,
|
|
359
|
+
output: { images: decoded },
|
|
360
|
+
...(warnings.length ? { warnings } : {}),
|
|
361
|
+
}
|
|
362
|
+
},
|
|
363
|
+
})
|
|
364
|
+
return binding
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function editForm(
|
|
368
|
+
fields: Readonly<Record<string, string | number>>,
|
|
369
|
+
images: readonly ImageData[],
|
|
370
|
+
gpt: boolean
|
|
371
|
+
): FormData {
|
|
372
|
+
const form = new FormData()
|
|
373
|
+
for (const [key, value] of Object.entries(fields)) form.set(key, String(value))
|
|
374
|
+
// GPT Image takes a list of references; FLUX.1 Kontext a single `image`.
|
|
375
|
+
images.forEach((image, index) => {
|
|
376
|
+
const file = new Blob([new Uint8Array(image.data)], { type: image.mediaType })
|
|
377
|
+
form.append(gpt ? "image[]" : "image", file, `image-${index}.${EXTENSIONS[image.mediaType]}`)
|
|
378
|
+
})
|
|
379
|
+
return form
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Images arrive as base64; one that is missing or not a known image format fails the call. */
|
|
383
|
+
function decodeImages(value: unknown): ImageData[] | undefined {
|
|
384
|
+
if (!Array.isArray(value)) return undefined
|
|
385
|
+
const images: ImageData[] = []
|
|
386
|
+
for (const entry of value) {
|
|
387
|
+
const encoded = object(entry)?.b64_json
|
|
388
|
+
if (typeof encoded !== "string" || !encoded) return undefined
|
|
389
|
+
const data = new Uint8Array(Buffer.from(encoded, "base64"))
|
|
390
|
+
const mediaType = imageMediaType(data)
|
|
391
|
+
if (!mediaType) return undefined
|
|
392
|
+
images.push({ data, mediaType })
|
|
393
|
+
}
|
|
394
|
+
return images
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function imageMediaType(bytes: Uint8Array): string | undefined {
|
|
398
|
+
const head = Buffer.from(bytes.subarray(0, 12))
|
|
399
|
+
if (head.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])))
|
|
400
|
+
return "image/png"
|
|
401
|
+
if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) return "image/jpeg"
|
|
402
|
+
if (head.toString("latin1", 0, 4) === "RIFF" && head.toString("latin1", 8, 12) === "WEBP")
|
|
403
|
+
return "image/webp"
|
|
404
|
+
return undefined
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* GPT Image reports tokens, image and text output apart; FLUX reports nothing, and is billed by
|
|
409
|
+
* the images returned.
|
|
410
|
+
*/
|
|
411
|
+
function imageUsage(payload: JsonObject | undefined, images: number | undefined): ModelUsage {
|
|
412
|
+
const raw = object(payload?.usage)
|
|
413
|
+
const output = object(raw?.output_tokens_details)
|
|
414
|
+
const inputTokens = counter(raw?.input_tokens)
|
|
415
|
+
const outputTokens = counter(raw?.output_tokens)
|
|
416
|
+
const imageOutputTokens = counter(output?.image_tokens)
|
|
417
|
+
const textOutputTokens = counter(output?.text_tokens)
|
|
418
|
+
return {
|
|
419
|
+
...(images === undefined ? {} : { outputImages: images }),
|
|
420
|
+
...(inputTokens === undefined ? {} : { inputTokens }),
|
|
421
|
+
...(outputTokens === undefined ? {} : { outputTokens }),
|
|
422
|
+
...(imageOutputTokens === undefined ? {} : { imageOutputTokens }),
|
|
423
|
+
...(textOutputTokens === undefined ? {} : { textOutputTokens }),
|
|
424
|
+
...(raw ? { raw } : {}),
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function publishedPrice(deployment: AzureAIFoundryDeployment): PublishedImagePrice | undefined {
|
|
429
|
+
const name = deployment.modelName.toLowerCase()
|
|
430
|
+
const entry = Object.hasOwn(PUBLISHED, name) ? PUBLISHED[name] : undefined
|
|
431
|
+
if (!entry || entry.version !== deployment.modelVersion) return undefined
|
|
432
|
+
const price = Object.hasOwn(entry.skus, deployment.sku.name)
|
|
433
|
+
? entry.skus[deployment.sku.name]
|
|
434
|
+
: undefined
|
|
435
|
+
return price && { card: defineModelRateCard(price.card), textOutput: price.textOutput }
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function imageEstimator(rateCard: ModelRateCard, textOutput: boolean): ImageCostEstimator {
|
|
439
|
+
return {
|
|
440
|
+
// A per-image price bounds a call; token-priced output has no bound before the call.
|
|
441
|
+
estimateReservation: (input) =>
|
|
442
|
+
rateCard.images === undefined
|
|
443
|
+
? undefined
|
|
444
|
+
: estimateModelReservation({ outputImages: input.n, rateCard }),
|
|
445
|
+
estimate: ({ usage, dimensions }) => {
|
|
446
|
+
const details = object(usage.raw?.input_tokens_details)
|
|
447
|
+
// Image input tokens, from reference images, bill at a rate the card cannot hold; text
|
|
448
|
+
// output without a published price is not billed at the image rate.
|
|
449
|
+
if (
|
|
450
|
+
(details?.image_tokens !== undefined && details.image_tokens !== 0) ||
|
|
451
|
+
(!textOutput && (usage.textOutputTokens ?? 0) > 0)
|
|
452
|
+
)
|
|
453
|
+
return { status: "unpriceable", reason: "missing-rate-card" }
|
|
454
|
+
return rateModelCall({ usage, rateCard, dimensions })
|
|
455
|
+
},
|
|
456
|
+
}
|
|
457
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export type { AzureAIFoundryCatalogOptions } from "./catalog"
|
|
2
|
+
export type { AzureAIFoundryDecisionModel, AzureAIFoundryDecisionOptions } from "./decision"
|
|
2
3
|
export type {
|
|
3
4
|
AzureAIFoundryCatalog,
|
|
4
5
|
AzureAIFoundryDeployment,
|
|
@@ -9,6 +10,7 @@ export type {
|
|
|
9
10
|
AzureAIFoundryEmbeddingOptions,
|
|
10
11
|
AzureAIFoundryEmbeddingTransportOptions,
|
|
11
12
|
} from "./embedding"
|
|
13
|
+
export type { AzureAIFoundryImageModel, AzureAIFoundryImageOptions } from "./image"
|
|
12
14
|
export type {
|
|
13
15
|
AzureAIFoundryChatOptions,
|
|
14
16
|
AzureAIFoundryMessagesOptions,
|
|
@@ -19,4 +21,10 @@ export type {
|
|
|
19
21
|
AzureAIFoundryProvider,
|
|
20
22
|
} from "./provider"
|
|
21
23
|
export { createAzureAIFoundry } from "./provider"
|
|
24
|
+
export type { AzureAIFoundryRerankingModel, AzureAIFoundryRerankingOptions } from "./reranking"
|
|
25
|
+
export type { AzureAIFoundrySpeechModel, AzureAIFoundrySpeechOptions } from "./speech"
|
|
26
|
+
export type {
|
|
27
|
+
AzureAIFoundryTranscriptionModel,
|
|
28
|
+
AzureAIFoundryTranscriptionOptions,
|
|
29
|
+
} from "./transcription"
|
|
22
30
|
export type { FoundryProtocol } from "./transport"
|
package/src/kinds.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { UnsupportedModelFeatureError } from "@sixb/core/models"
|
|
2
|
+
import type { AzureAIFoundryDeployment } from "./discovery"
|
|
3
|
+
import { PREFIX } from "./util"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What a deployment runs, read from its model name. Project discovery reports no kind: an audio
|
|
7
|
+
* or rerank deployment lists no capabilities, or `chat_completion`, like a language model.
|
|
8
|
+
*/
|
|
9
|
+
export type FoundryKind =
|
|
10
|
+
| "language"
|
|
11
|
+
| "embedding"
|
|
12
|
+
| "decision"
|
|
13
|
+
| "transcription"
|
|
14
|
+
| "reranking"
|
|
15
|
+
| "speech"
|
|
16
|
+
| "image"
|
|
17
|
+
|
|
18
|
+
export const KIND_BINDINGS: Readonly<Record<FoundryKind, string>> = {
|
|
19
|
+
language: "foundry() or foundry.chat()",
|
|
20
|
+
embedding: "foundry.embedding()",
|
|
21
|
+
decision: "foundry.decision()",
|
|
22
|
+
transcription: "foundry.transcription()",
|
|
23
|
+
reranking: "foundry.reranking()",
|
|
24
|
+
speech: "foundry.speech()",
|
|
25
|
+
image: "foundry.image()",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function deploymentKind(deployment: AzureAIFoundryDeployment): FoundryKind {
|
|
29
|
+
if (isEmbeddingDeployment(deployment)) return "embedding"
|
|
30
|
+
if (isDecisionDeployment(deployment)) return "decision"
|
|
31
|
+
if (isTranscriptionDeployment(deployment)) return "transcription"
|
|
32
|
+
// Discovery reports `chat_completion: "true"` for Rerank v4.0; it is no chat model.
|
|
33
|
+
if (/^cohere-rerank-/i.test(deployment.modelName)) return "reranking"
|
|
34
|
+
// tts and tts-hd are speech models too, which foundry.speech() rejects with the reason.
|
|
35
|
+
if (
|
|
36
|
+
deployment.capabilities.audioSpeech === "true" ||
|
|
37
|
+
/^(gpt-4o-mini-tts|tts|tts-hd)$/i.test(deployment.modelName)
|
|
38
|
+
)
|
|
39
|
+
return "speech"
|
|
40
|
+
// FLUX.2 and DALL-E are image models too, which foundry.image() rejects with the reason.
|
|
41
|
+
if (
|
|
42
|
+
deployment.capabilities.imageGenerations === "true" ||
|
|
43
|
+
/^(gpt-image-|flux[-.]|dall-e-)/i.test(deployment.modelName)
|
|
44
|
+
)
|
|
45
|
+
return "image"
|
|
46
|
+
return "language"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Reject a deployment of another kind, naming the binding that serves it. */
|
|
50
|
+
export function assertDeploymentKind(
|
|
51
|
+
deployment: AzureAIFoundryDeployment,
|
|
52
|
+
deploymentName: string,
|
|
53
|
+
kind: Exclude<FoundryKind, "language">
|
|
54
|
+
): void {
|
|
55
|
+
const actual = deploymentKind(deployment)
|
|
56
|
+
if (actual !== kind)
|
|
57
|
+
throw new UnsupportedModelFeatureError(
|
|
58
|
+
`${PREFIX} Deployment '${deploymentName}' runs ${deployment.modelName}, not ${article(kind)} ${kind} model. Use ${KIND_BINDINGS[actual]}.`
|
|
59
|
+
)
|
|
60
|
+
// A connection's deployment lives on another resource, which its record does not name.
|
|
61
|
+
if (deployment.connectionName)
|
|
62
|
+
throw new UnsupportedModelFeatureError(
|
|
63
|
+
`${PREFIX} ${capitalize(article(kind))} ${kind} deployment from a project connection cannot be routed; use the owning resource's project URL and API key.`
|
|
64
|
+
)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** "an image", "a speech": the article a kind's name takes. */
|
|
68
|
+
export function article(kind: FoundryKind): string {
|
|
69
|
+
return /^[aeiou]/.test(kind) ? "an" : "a"
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function capitalize(word: string): string {
|
|
73
|
+
return word.charAt(0).toUpperCase() + word.slice(1)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function isEmbeddingDeployment(deployment: AzureAIFoundryDeployment): boolean {
|
|
77
|
+
return (
|
|
78
|
+
deployment.capabilities.embeddings === "true" ||
|
|
79
|
+
deployment.capabilities.embedding === "true" ||
|
|
80
|
+
/^(text-embedding-|cohere-embed)/i.test(deployment.modelName)
|
|
81
|
+
)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The Foundry catalog's decision models; project discovery reports no `decision` flag. */
|
|
85
|
+
function isDecisionDeployment(deployment: AzureAIFoundryDeployment): boolean {
|
|
86
|
+
return (
|
|
87
|
+
deployment.capabilities.decision === "true" ||
|
|
88
|
+
/^microsoft-decision-/i.test(deployment.modelName)
|
|
89
|
+
)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** File transcription models; realtime-only ones such as gpt-live-transcribe are not. */
|
|
93
|
+
function isTranscriptionDeployment(deployment: AzureAIFoundryDeployment): boolean {
|
|
94
|
+
return (
|
|
95
|
+
deployment.capabilities.audioTranscriptions === "true" ||
|
|
96
|
+
/^(whisper|gpt-4o-transcribe|gpt-4o-mini-transcribe|gpt-transcribe)(-|$)/i.test(
|
|
97
|
+
deployment.modelName
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
}
|