@sudobility/shapeshyft_types 1.0.22 → 1.0.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/dist/index.cjs +368 -105
  2. package/dist/index.d.ts +44 -0
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +368 -105
  5. package/dist/index.js.map +1 -1
  6. package/package.json +1 -1
package/dist/index.cjs CHANGED
@@ -8,6 +8,10 @@ exports.COST_PER_MILLION_TOKENS = exports.MODEL_CAPABILITIES = exports.PROVIDER_
8
8
  exports.getModelCapabilities = getModelCapabilities;
9
9
  exports.hasInputCapability = hasInputCapability;
10
10
  exports.hasOutputCapability = hasOutputCapability;
11
+ exports.getMediaFormats = getMediaFormats;
12
+ exports.supportsMediaFormat = supportsMediaFormat;
13
+ exports.getDefaultMediaFormat = getDefaultMediaFormat;
14
+ exports.supportsMediaUrl = supportsMediaUrl;
11
15
  exports.getModelPricing = getModelPricing;
12
16
  exports.estimateCost = estimateCost;
13
17
  exports.estimateMultimodalCost = estimateMultimodalCost;
@@ -120,84 +124,288 @@ exports.PROVIDER_ALLOWS_CUSTOM_MODEL = {
120
124
  exports.MODEL_CAPABILITIES = {
121
125
  // ===========================================================================
122
126
  // OpenAI (January 2026)
127
+ // OpenAI supports URL and base64 for images
123
128
  // ===========================================================================
124
129
  // GPT-4.1 family
125
- 'gpt-4.1': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
126
- 'gpt-4.1-mini': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
127
- 'gpt-4.1-nano': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
130
+ 'gpt-4.1': {
131
+ visionInput: true, audioInput: false, videoInput: false,
132
+ imageOutput: false, audioOutput: false, videoOutput: false,
133
+ mediaFormats: { imageFormats: ['url', 'base64'] },
134
+ },
135
+ 'gpt-4.1-mini': {
136
+ visionInput: true, audioInput: false, videoInput: false,
137
+ imageOutput: false, audioOutput: false, videoOutput: false,
138
+ mediaFormats: { imageFormats: ['url', 'base64'] },
139
+ },
140
+ 'gpt-4.1-nano': {
141
+ visionInput: false, audioInput: false, videoInput: false,
142
+ imageOutput: false, audioOutput: false, videoOutput: false,
143
+ },
128
144
  // GPT-4o (omni - multimodal)
129
- 'gpt-4o': { visionInput: true, audioInput: true, videoInput: false, imageOutput: false, audioOutput: true, videoOutput: false },
130
- 'gpt-4o-mini': { visionInput: true, audioInput: true, videoInput: false, imageOutput: false, audioOutput: true, videoOutput: false },
145
+ 'gpt-4o': {
146
+ visionInput: true, audioInput: true, videoInput: false,
147
+ imageOutput: false, audioOutput: true, videoOutput: false,
148
+ mediaFormats: { imageFormats: ['url', 'base64'], audioFormats: ['base64', 'file'] },
149
+ },
150
+ 'gpt-4o-mini': {
151
+ visionInput: true, audioInput: true, videoInput: false,
152
+ imageOutput: false, audioOutput: true, videoOutput: false,
153
+ mediaFormats: { imageFormats: ['url', 'base64'], audioFormats: ['base64', 'file'] },
154
+ },
131
155
  // Reasoning models (o-series)
132
- 'o3': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
133
- 'o3-pro': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
134
- 'o4-mini': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
156
+ 'o3': {
157
+ visionInput: true, audioInput: false, videoInput: false,
158
+ imageOutput: false, audioOutput: false, videoOutput: false,
159
+ mediaFormats: { imageFormats: ['url', 'base64'] },
160
+ },
161
+ 'o3-pro': {
162
+ visionInput: true, audioInput: false, videoInput: false,
163
+ imageOutput: false, audioOutput: false, videoOutput: false,
164
+ mediaFormats: { imageFormats: ['url', 'base64'] },
165
+ },
166
+ 'o4-mini': {
167
+ visionInput: true, audioInput: false, videoInput: false,
168
+ imageOutput: false, audioOutput: false, videoOutput: false,
169
+ mediaFormats: { imageFormats: ['url', 'base64'] },
170
+ },
135
171
  // Legacy
136
- 'gpt-4-turbo': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
137
- 'o1': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
172
+ 'gpt-4-turbo': {
173
+ visionInput: true, audioInput: false, videoInput: false,
174
+ imageOutput: false, audioOutput: false, videoOutput: false,
175
+ mediaFormats: { imageFormats: ['url', 'base64'] },
176
+ },
177
+ 'o1': {
178
+ visionInput: true, audioInput: false, videoInput: false,
179
+ imageOutput: false, audioOutput: false, videoOutput: false,
180
+ mediaFormats: { imageFormats: ['url', 'base64'] },
181
+ },
138
182
  // ===========================================================================
139
183
  // Anthropic (January 2026)
184
+ // Anthropic supports URL and base64 for images
140
185
  // ===========================================================================
141
186
  // Claude 4.5
142
- 'claude-opus-4-5-20251124': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
143
- 'claude-sonnet-4-5-20251124': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
187
+ 'claude-opus-4-5-20251124': {
188
+ visionInput: true, audioInput: false, videoInput: false,
189
+ imageOutput: false, audioOutput: false, videoOutput: false,
190
+ mediaFormats: { imageFormats: ['url', 'base64'] },
191
+ },
192
+ 'claude-sonnet-4-5-20251124': {
193
+ visionInput: true, audioInput: false, videoInput: false,
194
+ imageOutput: false, audioOutput: false, videoOutput: false,
195
+ mediaFormats: { imageFormats: ['url', 'base64'] },
196
+ },
144
197
  // Claude 4.1
145
- 'claude-opus-4-1-20250805': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
198
+ 'claude-opus-4-1-20250805': {
199
+ visionInput: true, audioInput: false, videoInput: false,
200
+ imageOutput: false, audioOutput: false, videoOutput: false,
201
+ mediaFormats: { imageFormats: ['url', 'base64'] },
202
+ },
146
203
  // Claude 4
147
- 'claude-sonnet-4-20250514': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
148
- 'claude-opus-4-20250514': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
204
+ 'claude-sonnet-4-20250514': {
205
+ visionInput: true, audioInput: false, videoInput: false,
206
+ imageOutput: false, audioOutput: false, videoOutput: false,
207
+ mediaFormats: { imageFormats: ['url', 'base64'] },
208
+ },
209
+ 'claude-opus-4-20250514': {
210
+ visionInput: true, audioInput: false, videoInput: false,
211
+ imageOutput: false, audioOutput: false, videoOutput: false,
212
+ mediaFormats: { imageFormats: ['url', 'base64'] },
213
+ },
149
214
  // Claude 3.5
150
- 'claude-3-5-haiku-20241022': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
215
+ 'claude-3-5-haiku-20241022': {
216
+ visionInput: true, audioInput: false, videoInput: false,
217
+ imageOutput: false, audioOutput: false, videoOutput: false,
218
+ mediaFormats: { imageFormats: ['url', 'base64'] },
219
+ },
151
220
  // ===========================================================================
152
221
  // Google Gemini (January 2026)
222
+ // Gemini supports URL, base64, and GCS URIs for all media types
153
223
  // ===========================================================================
154
224
  // Gemini 3 (latest)
155
- 'gemini-3-pro-preview': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: false, videoOutput: false },
156
- 'gemini-3-flash-preview': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: false, videoOutput: false },
157
- 'gemini-3-pro-image-preview': { visionInput: true, audioInput: false, videoInput: false, imageOutput: true, audioOutput: false, videoOutput: false },
225
+ 'gemini-3-pro-preview': {
226
+ visionInput: true, audioInput: true, videoInput: true,
227
+ imageOutput: false, audioOutput: false, videoOutput: false,
228
+ mediaFormats: {
229
+ imageFormats: ['url', 'base64', 'gcs'],
230
+ audioFormats: ['url', 'base64', 'gcs'],
231
+ videoFormats: ['url', 'gcs'],
232
+ },
233
+ },
234
+ 'gemini-3-flash-preview': {
235
+ visionInput: true, audioInput: true, videoInput: true,
236
+ imageOutput: false, audioOutput: false, videoOutput: false,
237
+ mediaFormats: {
238
+ imageFormats: ['url', 'base64', 'gcs'],
239
+ audioFormats: ['url', 'base64', 'gcs'],
240
+ videoFormats: ['url', 'gcs'],
241
+ },
242
+ },
243
+ 'gemini-3-pro-image-preview': {
244
+ visionInput: true, audioInput: false, videoInput: false,
245
+ imageOutput: true, audioOutput: false, videoOutput: false,
246
+ mediaFormats: { imageFormats: ['url', 'base64', 'gcs'] },
247
+ },
158
248
  // Gemini 2.5
159
- 'gemini-2.5-pro': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: false, videoOutput: false },
160
- 'gemini-2.5-flash': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: false, videoOutput: false },
161
- 'gemini-2.5-flash-lite': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: false, videoOutput: false },
162
- 'gemini-2.5-flash-image': { visionInput: true, audioInput: false, videoInput: false, imageOutput: true, audioOutput: false, videoOutput: false },
163
- 'gemini-2.5-flash-native-audio-preview': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: true, videoOutput: false },
249
+ 'gemini-2.5-pro': {
250
+ visionInput: true, audioInput: true, videoInput: true,
251
+ imageOutput: false, audioOutput: false, videoOutput: false,
252
+ mediaFormats: {
253
+ imageFormats: ['url', 'base64', 'gcs'],
254
+ audioFormats: ['url', 'base64', 'gcs'],
255
+ videoFormats: ['url', 'gcs'],
256
+ },
257
+ },
258
+ 'gemini-2.5-flash': {
259
+ visionInput: true, audioInput: true, videoInput: true,
260
+ imageOutput: false, audioOutput: false, videoOutput: false,
261
+ mediaFormats: {
262
+ imageFormats: ['url', 'base64', 'gcs'],
263
+ audioFormats: ['url', 'base64', 'gcs'],
264
+ videoFormats: ['url', 'gcs'],
265
+ },
266
+ },
267
+ 'gemini-2.5-flash-lite': {
268
+ visionInput: true, audioInput: true, videoInput: true,
269
+ imageOutput: false, audioOutput: false, videoOutput: false,
270
+ mediaFormats: {
271
+ imageFormats: ['url', 'base64', 'gcs'],
272
+ audioFormats: ['url', 'base64', 'gcs'],
273
+ videoFormats: ['url', 'gcs'],
274
+ },
275
+ },
276
+ 'gemini-2.5-flash-image': {
277
+ visionInput: true, audioInput: false, videoInput: false,
278
+ imageOutput: true, audioOutput: false, videoOutput: false,
279
+ mediaFormats: { imageFormats: ['url', 'base64', 'gcs'] },
280
+ },
281
+ 'gemini-2.5-flash-native-audio-preview': {
282
+ visionInput: true, audioInput: true, videoInput: true,
283
+ imageOutput: false, audioOutput: true, videoOutput: false,
284
+ mediaFormats: {
285
+ imageFormats: ['url', 'base64', 'gcs'],
286
+ audioFormats: ['url', 'base64', 'gcs'],
287
+ videoFormats: ['url', 'gcs'],
288
+ },
289
+ },
164
290
  // Gemini 2.0
165
- 'gemini-2.0-flash': { visionInput: true, audioInput: true, videoInput: true, imageOutput: true, audioOutput: true, videoOutput: false },
166
- 'gemini-2.0-flash-lite': { visionInput: true, audioInput: true, videoInput: true, imageOutput: false, audioOutput: false, videoOutput: false },
291
+ 'gemini-2.0-flash': {
292
+ visionInput: true, audioInput: true, videoInput: true,
293
+ imageOutput: true, audioOutput: true, videoOutput: false,
294
+ mediaFormats: {
295
+ imageFormats: ['url', 'base64', 'gcs'],
296
+ audioFormats: ['url', 'base64', 'gcs'],
297
+ videoFormats: ['url', 'gcs'],
298
+ },
299
+ },
300
+ 'gemini-2.0-flash-lite': {
301
+ visionInput: true, audioInput: true, videoInput: true,
302
+ imageOutput: false, audioOutput: false, videoOutput: false,
303
+ mediaFormats: {
304
+ imageFormats: ['url', 'base64', 'gcs'],
305
+ audioFormats: ['url', 'base64', 'gcs'],
306
+ videoFormats: ['url', 'gcs'],
307
+ },
308
+ },
167
309
  // ===========================================================================
168
310
  // Mistral AI (January 2026)
311
+ // Mistral supports URL and base64 for images
169
312
  // ===========================================================================
170
313
  // Large models (Mistral Large 3 has vision)
171
- 'mistral-large-2512': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
172
- 'mistral-large-latest': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
314
+ 'mistral-large-2512': {
315
+ visionInput: true, audioInput: false, videoInput: false,
316
+ imageOutput: false, audioOutput: false, videoOutput: false,
317
+ mediaFormats: { imageFormats: ['url', 'base64'] },
318
+ },
319
+ 'mistral-large-latest': {
320
+ visionInput: true, audioInput: false, videoInput: false,
321
+ imageOutput: false, audioOutput: false, videoOutput: false,
322
+ mediaFormats: { imageFormats: ['url', 'base64'] },
323
+ },
173
324
  // Medium models (vision capable)
174
- 'mistral-medium-3.1': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
175
- 'mistral-medium-latest': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
325
+ 'mistral-medium-3.1': {
326
+ visionInput: true, audioInput: false, videoInput: false,
327
+ imageOutput: false, audioOutput: false, videoOutput: false,
328
+ mediaFormats: { imageFormats: ['url', 'base64'] },
329
+ },
330
+ 'mistral-medium-latest': {
331
+ visionInput: true, audioInput: false, videoInput: false,
332
+ imageOutput: false, audioOutput: false, videoOutput: false,
333
+ mediaFormats: { imageFormats: ['url', 'base64'] },
334
+ },
176
335
  // Small models (vision capable)
177
- 'mistral-small-3.2': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
178
- 'mistral-small-latest': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
336
+ 'mistral-small-3.2': {
337
+ visionInput: true, audioInput: false, videoInput: false,
338
+ imageOutput: false, audioOutput: false, videoOutput: false,
339
+ mediaFormats: { imageFormats: ['url', 'base64'] },
340
+ },
341
+ 'mistral-small-latest': {
342
+ visionInput: false, audioInput: false, videoInput: false,
343
+ imageOutput: false, audioOutput: false, videoOutput: false,
344
+ },
179
345
  // Ministral (vision capable)
180
- 'ministral-3b-2512': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
181
- 'ministral-8b-2512': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
182
- 'ministral-14b-2512': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
346
+ 'ministral-3b-2512': {
347
+ visionInput: true, audioInput: false, videoInput: false,
348
+ imageOutput: false, audioOutput: false, videoOutput: false,
349
+ mediaFormats: { imageFormats: ['url', 'base64'] },
350
+ },
351
+ 'ministral-8b-2512': {
352
+ visionInput: true, audioInput: false, videoInput: false,
353
+ imageOutput: false, audioOutput: false, videoOutput: false,
354
+ mediaFormats: { imageFormats: ['url', 'base64'] },
355
+ },
356
+ 'ministral-14b-2512': {
357
+ visionInput: true, audioInput: false, videoInput: false,
358
+ imageOutput: false, audioOutput: false, videoOutput: false,
359
+ mediaFormats: { imageFormats: ['url', 'base64'] },
360
+ },
183
361
  // Code models
184
- 'codestral-2501': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
185
- 'codestral-latest': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
362
+ 'codestral-2501': {
363
+ visionInput: false, audioInput: false, videoInput: false,
364
+ imageOutput: false, audioOutput: false, videoOutput: false,
365
+ },
366
+ 'codestral-latest': {
367
+ visionInput: false, audioInput: false, videoInput: false,
368
+ imageOutput: false, audioOutput: false, videoOutput: false,
369
+ },
186
370
  // Vision models (Pixtral)
187
- 'pixtral-large-2411': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
188
- 'pixtral-large-latest': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
371
+ 'pixtral-large-2411': {
372
+ visionInput: true, audioInput: false, videoInput: false,
373
+ imageOutput: false, audioOutput: false, videoOutput: false,
374
+ mediaFormats: { imageFormats: ['url', 'base64'] },
375
+ },
376
+ 'pixtral-large-latest': {
377
+ visionInput: true, audioInput: false, videoInput: false,
378
+ imageOutput: false, audioOutput: false, videoOutput: false,
379
+ mediaFormats: { imageFormats: ['url', 'base64'] },
380
+ },
189
381
  // Audio models (Voxtral)
190
- 'voxtral-small': { visionInput: false, audioInput: true, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
191
- 'voxtral-mini': { visionInput: false, audioInput: true, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
382
+ 'voxtral-small': {
383
+ visionInput: false, audioInput: true, videoInput: false,
384
+ imageOutput: false, audioOutput: false, videoOutput: false,
385
+ mediaFormats: { audioFormats: ['base64', 'file'] },
386
+ },
387
+ 'voxtral-mini': {
388
+ visionInput: false, audioInput: true, videoInput: false,
389
+ imageOutput: false, audioOutput: false, videoOutput: false,
390
+ mediaFormats: { audioFormats: ['base64', 'file'] },
391
+ },
192
392
  // Document AI
193
- 'mistral-ocr-2512': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
393
+ 'mistral-ocr-2512': {
394
+ visionInput: true, audioInput: false, videoInput: false,
395
+ imageOutput: false, audioOutput: false, videoOutput: false,
396
+ mediaFormats: { imageFormats: ['url', 'base64'] },
397
+ },
194
398
  // ===========================================================================
195
399
  // Cohere (January 2026)
196
400
  // ===========================================================================
197
401
  // Command A family
198
402
  'command-a-03-2025': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
199
403
  'command-a-reasoning': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
200
- 'command-a-vision': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
404
+ 'command-a-vision': {
405
+ visionInput: true, audioInput: false, videoInput: false,
406
+ imageOutput: false, audioOutput: false, videoOutput: false,
407
+ mediaFormats: { imageFormats: ['base64'] },
408
+ },
201
409
  // Command R family
202
410
  'command-r-plus-08-2024': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
203
411
  'command-r-08-2024': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
@@ -212,19 +420,51 @@ exports.MODEL_CAPABILITIES = {
212
420
  'openai/gpt-oss-20b': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
213
421
  'groq/compound': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
214
422
  'groq/compound-mini': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
215
- 'meta-llama/llama-guard-4-12b': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
216
- 'whisper-large-v3': { visionInput: false, audioInput: true, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
217
- 'whisper-large-v3-turbo': { visionInput: false, audioInput: true, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
423
+ 'meta-llama/llama-guard-4-12b': {
424
+ visionInput: true, audioInput: false, videoInput: false,
425
+ imageOutput: false, audioOutput: false, videoOutput: false,
426
+ mediaFormats: { imageFormats: ['base64'] },
427
+ },
428
+ 'whisper-large-v3': {
429
+ visionInput: false, audioInput: true, videoInput: false,
430
+ imageOutput: false, audioOutput: false, videoOutput: false,
431
+ mediaFormats: { audioFormats: ['file'] },
432
+ },
433
+ 'whisper-large-v3-turbo': {
434
+ visionInput: false, audioInput: true, videoInput: false,
435
+ imageOutput: false, audioOutput: false, videoOutput: false,
436
+ mediaFormats: { audioFormats: ['file'] },
437
+ },
218
438
  // ===========================================================================
219
439
  // xAI Grok (January 2026)
220
440
  // ===========================================================================
221
- 'grok-4': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
222
- 'grok-4.1-fast': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
223
- 'grok-3': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
441
+ 'grok-4': {
442
+ visionInput: true, audioInput: false, videoInput: false,
443
+ imageOutput: false, audioOutput: false, videoOutput: false,
444
+ mediaFormats: { imageFormats: ['url', 'base64'] },
445
+ },
446
+ 'grok-4.1-fast': {
447
+ visionInput: true, audioInput: false, videoInput: false,
448
+ imageOutput: false, audioOutput: false, videoOutput: false,
449
+ mediaFormats: { imageFormats: ['url', 'base64'] },
450
+ },
451
+ 'grok-3': {
452
+ visionInput: true, audioInput: false, videoInput: false,
453
+ imageOutput: false, audioOutput: false, videoOutput: false,
454
+ mediaFormats: { imageFormats: ['url', 'base64'] },
455
+ },
224
456
  'grok-3-mini': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
225
- 'grok-3-vision': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
457
+ 'grok-3-vision': {
458
+ visionInput: true, audioInput: false, videoInput: false,
459
+ imageOutput: false, audioOutput: false, videoOutput: false,
460
+ mediaFormats: { imageFormats: ['url', 'base64'] },
461
+ },
226
462
  'grok-2': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
227
- 'grok-2-vision': { visionInput: true, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
463
+ 'grok-2-vision': {
464
+ visionInput: true, audioInput: false, videoInput: false,
465
+ imageOutput: false, audioOutput: false, videoOutput: false,
466
+ mediaFormats: { imageFormats: ['url', 'base64'] },
467
+ },
228
468
  // ===========================================================================
229
469
  // DeepSeek (January 2026 - V3.2)
230
470
  // ===========================================================================
@@ -240,78 +480,52 @@ exports.MODEL_CAPABILITIES = {
240
480
  'sonar-deep-research': { visionInput: false, audioInput: false, videoInput: false, imageOutput: false, audioOutput: false, videoOutput: false },
241
481
  // ===========================================================================
242
482
  // LM Studio / Local Models (common vision models)
483
+ // Local models typically only support base64 as they don't have internet access
243
484
  // ===========================================================================
244
485
  'llava-v1.6-mistral-7b': {
245
- visionInput: true,
246
- audioInput: false,
247
- videoInput: false,
248
- imageOutput: false,
249
- audioOutput: false,
250
- videoOutput: false,
486
+ visionInput: true, audioInput: false, videoInput: false,
487
+ imageOutput: false, audioOutput: false, videoOutput: false,
488
+ mediaFormats: { imageFormats: ['base64'] },
251
489
  },
252
490
  'llava-v1.6-vicuna-13b': {
253
- visionInput: true,
254
- audioInput: false,
255
- videoInput: false,
256
- imageOutput: false,
257
- audioOutput: false,
258
- videoOutput: false,
491
+ visionInput: true, audioInput: false, videoInput: false,
492
+ imageOutput: false, audioOutput: false, videoOutput: false,
493
+ mediaFormats: { imageFormats: ['base64'] },
259
494
  },
260
495
  'qwen2-vl-7b-instruct': {
261
- visionInput: true,
262
- audioInput: false,
263
- videoInput: false,
264
- imageOutput: false,
265
- audioOutput: false,
266
- videoOutput: false,
496
+ visionInput: true, audioInput: false, videoInput: false,
497
+ imageOutput: false, audioOutput: false, videoOutput: false,
498
+ mediaFormats: { imageFormats: ['base64'] },
267
499
  },
268
500
  'phi-3-vision-128k-instruct': {
269
- visionInput: true,
270
- audioInput: false,
271
- videoInput: false,
272
- imageOutput: false,
273
- audioOutput: false,
274
- videoOutput: false,
501
+ visionInput: true, audioInput: false, videoInput: false,
502
+ imageOutput: false, audioOutput: false, videoOutput: false,
503
+ mediaFormats: { imageFormats: ['base64'] },
275
504
  },
276
505
  'minicpm-v-2_6': {
277
- visionInput: true,
278
- audioInput: false,
279
- videoInput: false,
280
- imageOutput: false,
281
- audioOutput: false,
282
- videoOutput: false,
506
+ visionInput: true, audioInput: false, videoInput: false,
507
+ imageOutput: false, audioOutput: false, videoOutput: false,
508
+ mediaFormats: { imageFormats: ['base64'] },
283
509
  },
284
510
  'moondream2': {
285
- visionInput: true,
286
- audioInput: false,
287
- videoInput: false,
288
- imageOutput: false,
289
- audioOutput: false,
290
- videoOutput: false,
511
+ visionInput: true, audioInput: false, videoInput: false,
512
+ imageOutput: false, audioOutput: false, videoOutput: false,
513
+ mediaFormats: { imageFormats: ['base64'] },
291
514
  },
292
515
  'internvl2-8b': {
293
- visionInput: true,
294
- audioInput: false,
295
- videoInput: false,
296
- imageOutput: false,
297
- audioOutput: false,
298
- videoOutput: false,
516
+ visionInput: true, audioInput: false, videoInput: false,
517
+ imageOutput: false, audioOutput: false, videoOutput: false,
518
+ mediaFormats: { imageFormats: ['base64'] },
299
519
  },
300
520
  'llama-3.2-11b-vision': {
301
- visionInput: true,
302
- audioInput: false,
303
- videoInput: false,
304
- imageOutput: false,
305
- audioOutput: false,
306
- videoOutput: false,
521
+ visionInput: true, audioInput: false, videoInput: false,
522
+ imageOutput: false, audioOutput: false, videoOutput: false,
523
+ mediaFormats: { imageFormats: ['base64'] },
307
524
  },
308
525
  'pixtral-12b': {
309
- visionInput: true,
310
- audioInput: false,
311
- videoInput: false,
312
- imageOutput: false,
313
- audioOutput: false,
314
- videoOutput: false,
526
+ visionInput: true, audioInput: false, videoInput: false,
527
+ imageOutput: false, audioOutput: false, videoOutput: false,
528
+ mediaFormats: { imageFormats: ['base64'] },
315
529
  },
316
530
  };
317
531
  /**
@@ -349,6 +563,55 @@ function hasOutputCapability(model, capability) {
349
563
  case 'video': return caps.videoOutput;
350
564
  }
351
565
  }
566
+ /**
567
+ * Get supported media input formats for a specific media type.
568
+ * Returns the list of supported formats, or undefined if model is unknown.
569
+ * Returns empty array if model is known but doesn't support that media type.
570
+ */
571
+ function getMediaFormats(model, mediaType) {
572
+ const caps = exports.MODEL_CAPABILITIES[model];
573
+ if (!caps)
574
+ return undefined;
575
+ if (!caps.mediaFormats)
576
+ return [];
577
+ switch (mediaType) {
578
+ case 'image': return caps.mediaFormats.imageFormats ?? [];
579
+ case 'audio': return caps.mediaFormats.audioFormats ?? [];
580
+ case 'video': return caps.mediaFormats.videoFormats ?? [];
581
+ }
582
+ }
583
+ /**
584
+ * Check if a model supports a specific media input format.
585
+ * Returns undefined if model is unknown, true/false if known.
586
+ */
587
+ function supportsMediaFormat(model, mediaType, format) {
588
+ const formats = getMediaFormats(model, mediaType);
589
+ if (formats === undefined)
590
+ return undefined;
591
+ return formats.includes(format);
592
+ }
593
+ /**
594
+ * Get the preferred/default media input format for a model.
595
+ * Returns 'url' if supported, otherwise the first supported format.
596
+ * Returns undefined if model is unknown or doesn't support the media type.
597
+ */
598
+ function getDefaultMediaFormat(model, mediaType) {
599
+ const formats = getMediaFormats(model, mediaType);
600
+ if (!formats || formats.length === 0)
601
+ return undefined;
602
+ // Prefer URL if available (more efficient for providers)
603
+ if (formats.includes('url'))
604
+ return 'url';
605
+ // Otherwise return first available format
606
+ return formats[0];
607
+ }
608
+ /**
609
+ * Check if a model supports URL-based media input for a specific media type.
610
+ * This is a convenience function since URL support affects UI behavior.
611
+ */
612
+ function supportsMediaUrl(model, mediaType) {
613
+ return supportsMediaFormat(model, mediaType, 'url');
614
+ }
352
615
  /**
353
616
  * Cost estimation per 1M tokens (in cents)
354
617
  * Prices sourced from official provider pricing pages as of January 2026
package/dist/index.d.ts CHANGED
@@ -36,6 +36,27 @@ export declare const PROVIDER_MODELS: Record<LlmProvider, readonly string[]>;
36
36
  export declare const DEFAULT_PROVIDER_MODEL: Record<LlmProvider, string>;
37
37
  /** Providers that allow custom model input (user can type any model name) */
38
38
  export declare const PROVIDER_ALLOWS_CUSTOM_MODEL: Record<LlmProvider, boolean>;
39
+ /**
40
+ * Supported media input formats for different providers/models.
41
+ * - url: HTTP/HTTPS URLs to media files
42
+ * - base64: Base64-encoded binary data (data URLs)
43
+ * - gcs: Google Cloud Storage URIs (gs://bucket/path)
44
+ * - s3: AWS S3 URIs (s3://bucket/key)
45
+ * - file: Direct file upload (multipart form data)
46
+ */
47
+ export type MediaInputFormat = 'url' | 'base64' | 'gcs' | 's3' | 'file';
48
+ /**
49
+ * Media format support configuration for a model.
50
+ * Specifies which input formats are supported for each media type.
51
+ */
52
+ export interface MediaFormatSupport {
53
+ /** Supported formats for image input */
54
+ imageFormats?: MediaInputFormat[];
55
+ /** Supported formats for audio input */
56
+ audioFormats?: MediaInputFormat[];
57
+ /** Supported formats for video input */
58
+ videoFormats?: MediaInputFormat[];
59
+ }
39
60
  /**
40
61
  * Capabilities for a model's input and output modalities.
41
62
  * undefined = unknown (user's responsibility to verify)
@@ -49,6 +70,7 @@ export interface ModelCapabilities {
49
70
  imageOutput?: boolean;
50
71
  audioOutput?: boolean;
51
72
  videoOutput?: boolean;
73
+ mediaFormats?: MediaFormatSupport;
52
74
  }
53
75
  /**
54
76
  * Model capabilities map (January 2026).
@@ -71,6 +93,28 @@ export declare function hasInputCapability(model: string, capability: 'vision' |
71
93
  * Returns undefined if unknown, true/false if known.
72
94
  */
73
95
  export declare function hasOutputCapability(model: string, capability: 'image' | 'audio' | 'video'): boolean | undefined;
96
+ /**
97
+ * Get supported media input formats for a specific media type.
98
+ * Returns the list of supported formats, or undefined if model is unknown.
99
+ * Returns empty array if model is known but doesn't support that media type.
100
+ */
101
+ export declare function getMediaFormats(model: string, mediaType: 'image' | 'audio' | 'video'): MediaInputFormat[] | undefined;
102
+ /**
103
+ * Check if a model supports a specific media input format.
104
+ * Returns undefined if model is unknown, true/false if known.
105
+ */
106
+ export declare function supportsMediaFormat(model: string, mediaType: 'image' | 'audio' | 'video', format: MediaInputFormat): boolean | undefined;
107
+ /**
108
+ * Get the preferred/default media input format for a model.
109
+ * Returns 'url' if supported, otherwise the first supported format.
110
+ * Returns undefined if model is unknown or doesn't support the media type.
111
+ */
112
+ export declare function getDefaultMediaFormat(model: string, mediaType: 'image' | 'audio' | 'video'): MediaInputFormat | undefined;
113
+ /**
114
+ * Check if a model supports URL-based media input for a specific media type.
115
+ * This is a convenience function since URL support affects UI behavior.
116
+ */
117
+ export declare function supportsMediaUrl(model: string, mediaType: 'image' | 'audio' | 'video'): boolean | undefined;
74
118
  /**
75
119
  * Model pricing structure with multimodal support.
76
120
  * All costs are in cents.