@ljoukov/llm 5.0.4 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -38
- package/dist/index.cjs +836 -350
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -9
- package/dist/index.d.ts +12 -9
- package/dist/index.js +841 -353
- package/dist/index.js.map +1 -1
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -6,6 +6,8 @@ import path5 from "path";
|
|
|
6
6
|
import {
|
|
7
7
|
FinishReason,
|
|
8
8
|
FunctionCallingConfigMode,
|
|
9
|
+
MediaResolution,
|
|
10
|
+
PartMediaResolutionLevel,
|
|
9
11
|
ThinkingLevel,
|
|
10
12
|
createPartFromBase64,
|
|
11
13
|
createPartFromFunctionResponse,
|
|
@@ -204,11 +206,6 @@ function getGeminiImagePricing(modelId) {
|
|
|
204
206
|
}
|
|
205
207
|
|
|
206
208
|
// src/openai/pricing.ts
|
|
207
|
-
var OPENAI_GPT_52_PRICING = {
|
|
208
|
-
inputRate: 1.75 / 1e6,
|
|
209
|
-
cachedRate: 0.175 / 1e6,
|
|
210
|
-
outputRate: 14 / 1e6
|
|
211
|
-
};
|
|
212
209
|
var OPENAI_GPT_54_PRICING = {
|
|
213
210
|
inputRate: 2.5 / 1e6,
|
|
214
211
|
cachedRate: 0.25 / 1e6,
|
|
@@ -219,37 +216,31 @@ var OPENAI_GPT_54_PRIORITY_PRICING = {
|
|
|
219
216
|
cachedRate: 0.5 / 1e6,
|
|
220
217
|
outputRate: 30 / 1e6
|
|
221
218
|
};
|
|
222
|
-
var
|
|
223
|
-
inputRate: 1.25 / 1e6,
|
|
224
|
-
cachedRate: 0.125 / 1e6,
|
|
225
|
-
outputRate: 10 / 1e6
|
|
226
|
-
};
|
|
227
|
-
var OPENAI_GPT_5_MINI_PRICING = {
|
|
219
|
+
var OPENAI_GPT_54_MINI_PRICING = {
|
|
228
220
|
inputRate: 0.25 / 1e6,
|
|
229
221
|
cachedRate: 0.025 / 1e6,
|
|
230
222
|
outputRate: 2 / 1e6
|
|
231
223
|
};
|
|
224
|
+
var OPENAI_GPT_54_NANO_PRICING = {
|
|
225
|
+
inputRate: 0.05 / 1e6,
|
|
226
|
+
cachedRate: 5e-3 / 1e6,
|
|
227
|
+
outputRate: 0.4 / 1e6
|
|
228
|
+
};
|
|
232
229
|
function getOpenAiPricing(modelId) {
|
|
233
230
|
if (modelId.includes("gpt-5.4-fast")) {
|
|
234
231
|
return OPENAI_GPT_54_PRIORITY_PRICING;
|
|
235
232
|
}
|
|
236
|
-
if (modelId.includes("gpt-5.4")) {
|
|
237
|
-
return
|
|
238
|
-
}
|
|
239
|
-
if (modelId.includes("gpt-5.3-codex-spark")) {
|
|
240
|
-
return OPENAI_GPT_5_MINI_PRICING;
|
|
241
|
-
}
|
|
242
|
-
if (modelId.includes("gpt-5.3-codex")) {
|
|
243
|
-
return OPENAI_GPT_53_CODEX_PRICING;
|
|
233
|
+
if (modelId.includes("gpt-5.4-mini")) {
|
|
234
|
+
return OPENAI_GPT_54_MINI_PRICING;
|
|
244
235
|
}
|
|
245
|
-
if (modelId.includes("gpt-5.
|
|
246
|
-
return
|
|
236
|
+
if (modelId.includes("gpt-5.4-nano")) {
|
|
237
|
+
return OPENAI_GPT_54_NANO_PRICING;
|
|
247
238
|
}
|
|
248
|
-
if (modelId.includes("gpt-5-
|
|
249
|
-
return
|
|
239
|
+
if (modelId.includes("gpt-5.3-codex-spark")) {
|
|
240
|
+
return OPENAI_GPT_54_MINI_PRICING;
|
|
250
241
|
}
|
|
251
|
-
if (modelId.includes("gpt-5.
|
|
252
|
-
return
|
|
242
|
+
if (modelId.includes("gpt-5.4")) {
|
|
243
|
+
return OPENAI_GPT_54_PRICING;
|
|
253
244
|
}
|
|
254
245
|
return void 0;
|
|
255
246
|
}
|
|
@@ -2718,22 +2709,15 @@ async function runOpenAiCall(fn, modelId, runOptions) {
|
|
|
2718
2709
|
}
|
|
2719
2710
|
|
|
2720
2711
|
// src/openai/models.ts
|
|
2721
|
-
var OPENAI_MODEL_IDS = [
|
|
2722
|
-
"gpt-5.4",
|
|
2723
|
-
"gpt-5.3-codex",
|
|
2724
|
-
"gpt-5.2",
|
|
2725
|
-
"gpt-5.1-codex-mini"
|
|
2726
|
-
];
|
|
2712
|
+
var OPENAI_MODEL_IDS = ["gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano"];
|
|
2727
2713
|
function isOpenAiModelId(value) {
|
|
2728
2714
|
return OPENAI_MODEL_IDS.includes(value);
|
|
2729
2715
|
}
|
|
2730
2716
|
var CHATGPT_MODEL_IDS = [
|
|
2731
2717
|
"chatgpt-gpt-5.4",
|
|
2732
2718
|
"chatgpt-gpt-5.4-fast",
|
|
2733
|
-
"chatgpt-gpt-5.
|
|
2734
|
-
"chatgpt-gpt-5.3-codex-spark"
|
|
2735
|
-
"chatgpt-gpt-5.2",
|
|
2736
|
-
"chatgpt-gpt-5.1-codex-mini"
|
|
2719
|
+
"chatgpt-gpt-5.4-mini",
|
|
2720
|
+
"chatgpt-gpt-5.3-codex-spark"
|
|
2737
2721
|
];
|
|
2738
2722
|
function isChatGptModelId(value) {
|
|
2739
2723
|
return CHATGPT_MODEL_IDS.includes(value);
|
|
@@ -3256,19 +3240,16 @@ function getCurrentAgentLoggingSession() {
|
|
|
3256
3240
|
|
|
3257
3241
|
// src/files.ts
|
|
3258
3242
|
import { AsyncLocalStorage as AsyncLocalStorage2 } from "async_hooks";
|
|
3259
|
-
import { Buffer as Buffer4
|
|
3243
|
+
import { Buffer as Buffer4 } from "buffer";
|
|
3260
3244
|
import { createHash } from "crypto";
|
|
3261
|
-
import { createReadStream
|
|
3262
|
-
import { copyFile, mkdir as mkdir2,
|
|
3245
|
+
import { createReadStream } from "fs";
|
|
3246
|
+
import { copyFile, mkdir as mkdir2, readFile, stat, unlink, writeFile as writeFile2 } from "fs/promises";
|
|
3263
3247
|
import os3 from "os";
|
|
3264
3248
|
import path4 from "path";
|
|
3265
|
-
import { Readable } from "stream";
|
|
3266
3249
|
import { pipeline } from "stream/promises";
|
|
3267
3250
|
import { Storage } from "@google-cloud/storage";
|
|
3268
3251
|
import mime from "mime";
|
|
3269
3252
|
var DEFAULT_FILE_TTL_SECONDS = 48 * 60 * 60;
|
|
3270
|
-
var OPENAI_FILE_CREATE_MAX_BYTES = 512 * 1024 * 1024;
|
|
3271
|
-
var OPENAI_UPLOAD_PART_MAX_BYTES = 64 * 1024 * 1024;
|
|
3272
3253
|
var GEMINI_FILE_POLL_INTERVAL_MS = 1e3;
|
|
3273
3254
|
var GEMINI_FILE_POLL_TIMEOUT_MS = 6e4;
|
|
3274
3255
|
var FILES_TEMP_ROOT = path4.join(os3.tmpdir(), "ljoukov-llm-files");
|
|
@@ -3277,7 +3258,7 @@ var FILES_CACHE_CONTENT_ROOT = path4.join(FILES_CACHE_ROOT, "content");
|
|
|
3277
3258
|
var FILES_CACHE_METADATA_ROOT = path4.join(FILES_CACHE_ROOT, "metadata");
|
|
3278
3259
|
var filesState = getRuntimeSingleton(/* @__PURE__ */ Symbol.for("@ljoukov/llm.filesState"), () => ({
|
|
3279
3260
|
metadataById: /* @__PURE__ */ new Map(),
|
|
3280
|
-
|
|
3261
|
+
canonicalUploadCacheByKey: /* @__PURE__ */ new Map(),
|
|
3281
3262
|
materializedById: /* @__PURE__ */ new Map(),
|
|
3282
3263
|
geminiMirrorById: /* @__PURE__ */ new Map(),
|
|
3283
3264
|
vertexMirrorById: /* @__PURE__ */ new Map(),
|
|
@@ -3358,7 +3339,7 @@ function formatUploadLogLine(event) {
|
|
|
3358
3339
|
}
|
|
3359
3340
|
function recordUploadEvent(event) {
|
|
3360
3341
|
const scope = fileUploadScopeStorage.getStore();
|
|
3361
|
-
const resolvedSource = event.source ?? scope?.source ?? (event.backend === "
|
|
3342
|
+
const resolvedSource = event.source ?? scope?.source ?? (event.backend === "gcs" ? "files_api" : "provider_mirror");
|
|
3362
3343
|
const timestampedEvent = {
|
|
3363
3344
|
...event,
|
|
3364
3345
|
source: resolvedSource,
|
|
@@ -3405,16 +3386,117 @@ async function computeFileSha256Hex(filePath) {
|
|
|
3405
3386
|
}
|
|
3406
3387
|
return hash.digest("hex");
|
|
3407
3388
|
}
|
|
3408
|
-
function
|
|
3389
|
+
function buildCanonicalFileId(filename, mimeType, sha256Hex) {
|
|
3390
|
+
return `file_${createHash("sha256").update(filename).update("\0").update(mimeType).update("\0").update(sha256Hex).digest("hex")}`;
|
|
3391
|
+
}
|
|
3392
|
+
function resolveCanonicalFilesBucket() {
|
|
3393
|
+
const raw = process.env.LLM_FILES_GCS_BUCKET ?? process.env.VERTEX_GCS_BUCKET ?? process.env.LLM_VERTEX_GCS_BUCKET;
|
|
3394
|
+
const trimmed = raw?.trim();
|
|
3395
|
+
if (!trimmed) {
|
|
3396
|
+
throw new Error(
|
|
3397
|
+
"LLM_FILES_GCS_BUCKET (or VERTEX_GCS_BUCKET) must be set to use the canonical files API."
|
|
3398
|
+
);
|
|
3399
|
+
}
|
|
3400
|
+
return trimmed.replace(/^gs:\/\//u, "").replace(/\/+$/u, "");
|
|
3401
|
+
}
|
|
3402
|
+
function resolveCanonicalFilesPrefix() {
|
|
3403
|
+
const raw = process.env.LLM_FILES_GCS_PREFIX;
|
|
3404
|
+
const trimmed = raw?.trim().replace(/^\/+/u, "").replace(/\/+$/u, "");
|
|
3405
|
+
return trimmed ? `${trimmed}/` : "canonical-files/";
|
|
3406
|
+
}
|
|
3407
|
+
function isLatexLikeFile(filename, mimeType) {
|
|
3408
|
+
const extension = path4.extname(filename).trim().toLowerCase();
|
|
3409
|
+
const normalisedMimeType = mimeType.trim().toLowerCase();
|
|
3410
|
+
return extension === ".tex" || extension === ".ltx" || extension === ".latex" || normalisedMimeType === "application/x-tex" || normalisedMimeType === "text/x-tex";
|
|
3411
|
+
}
|
|
3412
|
+
function resolveCanonicalStorageContentType(filename, mimeType) {
|
|
3413
|
+
if (isLatexLikeFile(filename, mimeType)) {
|
|
3414
|
+
return "text/plain";
|
|
3415
|
+
}
|
|
3416
|
+
return mimeType;
|
|
3417
|
+
}
|
|
3418
|
+
function resolveCanonicalObjectExtension(filename, mimeType) {
|
|
3419
|
+
if (isLatexLikeFile(filename, mimeType)) {
|
|
3420
|
+
return "txt";
|
|
3421
|
+
}
|
|
3422
|
+
const fromFilename = path4.extname(filename).replace(/^\./u, "").trim().toLowerCase();
|
|
3423
|
+
if (fromFilename) {
|
|
3424
|
+
return fromFilename;
|
|
3425
|
+
}
|
|
3426
|
+
const fromMimeType = mime.getExtension(mimeType)?.trim().toLowerCase();
|
|
3427
|
+
if (fromMimeType) {
|
|
3428
|
+
return fromMimeType;
|
|
3429
|
+
}
|
|
3430
|
+
return "bin";
|
|
3431
|
+
}
|
|
3432
|
+
function buildCanonicalObjectName(fileId, filename, mimeType) {
|
|
3433
|
+
const extension = resolveCanonicalObjectExtension(filename, mimeType);
|
|
3434
|
+
return `${resolveCanonicalFilesPrefix()}${fileId}.${extension}`;
|
|
3435
|
+
}
|
|
3436
|
+
function toSafeStorageFilename(filename) {
|
|
3437
|
+
const normalized = normaliseFilename(filename).replace(/[^\w.-]+/gu, "-");
|
|
3438
|
+
return normalized.length > 0 ? normalized : "attachment.bin";
|
|
3439
|
+
}
|
|
3440
|
+
function parseUnixSeconds(value, fallback) {
|
|
3441
|
+
if (value) {
|
|
3442
|
+
const numeric = Number.parseInt(value, 10);
|
|
3443
|
+
if (Number.isFinite(numeric) && numeric > 0) {
|
|
3444
|
+
return numeric;
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
if (fallback) {
|
|
3448
|
+
const millis = Date.parse(fallback);
|
|
3449
|
+
if (Number.isFinite(millis)) {
|
|
3450
|
+
return Math.floor(millis / 1e3);
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
return Math.floor(Date.now() / 1e3);
|
|
3454
|
+
}
|
|
3455
|
+
function parseOptionalUnixSeconds(value) {
|
|
3456
|
+
if (!value) {
|
|
3457
|
+
return void 0;
|
|
3458
|
+
}
|
|
3459
|
+
const millis = Date.parse(value);
|
|
3460
|
+
if (Number.isFinite(millis)) {
|
|
3461
|
+
return Math.floor(millis / 1e3);
|
|
3462
|
+
}
|
|
3463
|
+
const numeric = Number.parseInt(value, 10);
|
|
3464
|
+
return Number.isFinite(numeric) && numeric > 0 ? numeric : void 0;
|
|
3465
|
+
}
|
|
3466
|
+
function toStoredFileFromCanonicalMetadata(options) {
|
|
3467
|
+
const metadata = options.objectMetadata.metadata;
|
|
3468
|
+
const filenameRaw = typeof metadata?.filename === "string" && metadata.filename.trim().length > 0 ? metadata.filename.trim() : path4.basename(options.objectName);
|
|
3469
|
+
const filename = normaliseFilename(filenameRaw);
|
|
3470
|
+
const bytesRaw = options.objectMetadata.size;
|
|
3471
|
+
const bytes = typeof bytesRaw === "string" ? Number.parseInt(bytesRaw, 10) : typeof bytesRaw === "number" ? bytesRaw : 0;
|
|
3472
|
+
const purpose = metadata?.purpose === "user_data" ? "user_data" : "user_data";
|
|
3473
|
+
const createdAt = parseUnixSeconds(
|
|
3474
|
+
typeof metadata?.createdAtUnix === "string" ? metadata.createdAtUnix : void 0,
|
|
3475
|
+
typeof options.objectMetadata.timeCreated === "string" ? options.objectMetadata.timeCreated : void 0
|
|
3476
|
+
);
|
|
3477
|
+
const expiresAt = parseOptionalUnixSeconds(
|
|
3478
|
+
typeof metadata?.expiresAt === "string" ? metadata.expiresAt : void 0
|
|
3479
|
+
);
|
|
3480
|
+
const mimeType = typeof metadata?.mimeType === "string" && metadata.mimeType.trim().length > 0 ? metadata.mimeType.trim() : typeof options.objectMetadata.contentType === "string" && options.objectMetadata.contentType.trim().length > 0 ? options.objectMetadata.contentType.trim() : resolveMimeType(filename, void 0);
|
|
3481
|
+
const sha256Hex = typeof metadata?.sha256 === "string" && metadata.sha256.trim().length > 0 ? metadata.sha256.trim() : void 0;
|
|
3409
3482
|
return {
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3483
|
+
file: {
|
|
3484
|
+
id: options.fileId,
|
|
3485
|
+
bytes: Number.isFinite(bytes) ? bytes : 0,
|
|
3486
|
+
created_at: createdAt,
|
|
3487
|
+
filename,
|
|
3488
|
+
object: "file",
|
|
3489
|
+
purpose,
|
|
3490
|
+
status: "processed",
|
|
3491
|
+
...expiresAt ? { expires_at: expiresAt } : {}
|
|
3492
|
+
},
|
|
3493
|
+
filename,
|
|
3494
|
+
bytes: Number.isFinite(bytes) ? bytes : 0,
|
|
3495
|
+
mimeType,
|
|
3496
|
+
sha256Hex,
|
|
3497
|
+
localPath: options.localPath,
|
|
3498
|
+
bucketName: options.bucketName,
|
|
3499
|
+
objectName: options.objectName
|
|
3418
3500
|
};
|
|
3419
3501
|
}
|
|
3420
3502
|
function buildCacheKey(filename, mimeType, sha256Hex) {
|
|
@@ -3435,7 +3517,7 @@ function isFresh(file) {
|
|
|
3435
3517
|
function recordMetadata(metadata) {
|
|
3436
3518
|
filesState.metadataById.set(metadata.file.id, metadata);
|
|
3437
3519
|
if (metadata.sha256Hex) {
|
|
3438
|
-
filesState.
|
|
3520
|
+
filesState.canonicalUploadCacheByKey.set(
|
|
3439
3521
|
buildCacheKey(
|
|
3440
3522
|
metadata.filename,
|
|
3441
3523
|
metadata.mimeType ?? "application/octet-stream",
|
|
@@ -3484,7 +3566,9 @@ async function persistMetadataToDisk(metadata) {
|
|
|
3484
3566
|
bytes: metadata.bytes,
|
|
3485
3567
|
mimeType: metadata.mimeType,
|
|
3486
3568
|
sha256Hex: metadata.sha256Hex,
|
|
3487
|
-
localPath: metadata.localPath
|
|
3569
|
+
localPath: metadata.localPath,
|
|
3570
|
+
bucketName: metadata.bucketName,
|
|
3571
|
+
objectName: metadata.objectName
|
|
3488
3572
|
};
|
|
3489
3573
|
await writeFile2(
|
|
3490
3574
|
buildCachedMetadataPath(metadata.file.id),
|
|
@@ -3516,175 +3600,271 @@ async function loadPersistedMetadata(fileId) {
|
|
|
3516
3600
|
bytes: payload.bytes,
|
|
3517
3601
|
mimeType: payload.mimeType,
|
|
3518
3602
|
sha256Hex: payload.sha256Hex,
|
|
3519
|
-
localPath: payload.localPath
|
|
3603
|
+
localPath: payload.localPath,
|
|
3604
|
+
bucketName: payload.bucketName,
|
|
3605
|
+
objectName: payload.objectName
|
|
3520
3606
|
});
|
|
3521
3607
|
} catch {
|
|
3522
3608
|
return void 0;
|
|
3523
3609
|
}
|
|
3524
3610
|
}
|
|
3525
|
-
async function
|
|
3526
|
-
const
|
|
3527
|
-
const
|
|
3528
|
-
|
|
3529
|
-
|
|
3611
|
+
async function writeCanonicalFileFromPath(options) {
|
|
3612
|
+
const file = getStorageClient().bucket(options.bucketName).file(options.objectName);
|
|
3613
|
+
const storageContentType = resolveCanonicalStorageContentType(
|
|
3614
|
+
options.metadata.filename ?? "attachment.bin",
|
|
3615
|
+
options.mimeType
|
|
3616
|
+
);
|
|
3617
|
+
try {
|
|
3618
|
+
await pipeline(
|
|
3619
|
+
createReadStream(options.filePath),
|
|
3620
|
+
file.createWriteStream({
|
|
3621
|
+
resumable: options.bytes >= 10 * 1024 * 1024,
|
|
3622
|
+
preconditionOpts: { ifGenerationMatch: 0 },
|
|
3623
|
+
metadata: {
|
|
3624
|
+
contentType: storageContentType,
|
|
3625
|
+
contentDisposition: `inline; filename="${toSafeStorageFilename(options.metadata.filename ?? "attachment.bin")}"`,
|
|
3626
|
+
metadata: options.metadata
|
|
3627
|
+
}
|
|
3628
|
+
})
|
|
3629
|
+
);
|
|
3630
|
+
return true;
|
|
3631
|
+
} catch (error) {
|
|
3632
|
+
const code = error.code;
|
|
3633
|
+
if (code === 412 || code === "412") {
|
|
3634
|
+
return false;
|
|
3635
|
+
}
|
|
3636
|
+
throw error;
|
|
3530
3637
|
}
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
3544
|
-
|
|
3638
|
+
}
|
|
3639
|
+
async function writeCanonicalFileFromBytes(options) {
|
|
3640
|
+
const file = getStorageClient().bucket(options.bucketName).file(options.objectName);
|
|
3641
|
+
const storageContentType = resolveCanonicalStorageContentType(
|
|
3642
|
+
options.metadata.filename ?? "attachment.bin",
|
|
3643
|
+
options.mimeType
|
|
3644
|
+
);
|
|
3645
|
+
try {
|
|
3646
|
+
await file.save(options.bytes, {
|
|
3647
|
+
resumable: options.bytes.byteLength >= 10 * 1024 * 1024,
|
|
3648
|
+
preconditionOpts: { ifGenerationMatch: 0 },
|
|
3649
|
+
metadata: {
|
|
3650
|
+
contentType: storageContentType,
|
|
3651
|
+
contentDisposition: `inline; filename="${toSafeStorageFilename(options.metadata.filename ?? "attachment.bin")}"`,
|
|
3652
|
+
metadata: options.metadata
|
|
3545
3653
|
}
|
|
3546
3654
|
});
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
const
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
mime_type: params.mimeType,
|
|
3553
|
-
purpose: params.purpose
|
|
3554
|
-
});
|
|
3555
|
-
const partIds = [];
|
|
3556
|
-
for (let offset = 0; offset < params.bytes.byteLength; offset += OPENAI_UPLOAD_PART_MAX_BYTES) {
|
|
3557
|
-
const chunk = params.bytes.subarray(
|
|
3558
|
-
offset,
|
|
3559
|
-
Math.min(offset + OPENAI_UPLOAD_PART_MAX_BYTES, params.bytes.byteLength)
|
|
3560
|
-
);
|
|
3561
|
-
const uploadPart = await client.uploads.parts.create(upload.id, {
|
|
3562
|
-
data: new NodeFile([new Uint8Array(chunk)], `${params.sha256Hex}.part`, {
|
|
3563
|
-
type: params.mimeType
|
|
3564
|
-
})
|
|
3565
|
-
});
|
|
3566
|
-
partIds.push(uploadPart.id);
|
|
3567
|
-
}
|
|
3568
|
-
const completed = await client.uploads.complete(upload.id, { part_ids: partIds });
|
|
3569
|
-
const fileId = completed.file?.id;
|
|
3570
|
-
if (!fileId) {
|
|
3571
|
-
throw new Error("OpenAI upload completed without a file id.");
|
|
3655
|
+
return true;
|
|
3656
|
+
} catch (error) {
|
|
3657
|
+
const code = error.code;
|
|
3658
|
+
if (code === 412 || code === "412") {
|
|
3659
|
+
return false;
|
|
3572
3660
|
}
|
|
3573
|
-
|
|
3661
|
+
throw error;
|
|
3574
3662
|
}
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
filename
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3663
|
+
}
|
|
3664
|
+
async function refreshCanonicalObjectMetadata(options) {
|
|
3665
|
+
const storageContentType = resolveCanonicalStorageContentType(
|
|
3666
|
+
options.metadata.filename ?? "attachment.bin",
|
|
3667
|
+
options.mimeType
|
|
3668
|
+
);
|
|
3669
|
+
await getStorageClient().bucket(options.bucketName).file(options.objectName).setMetadata({
|
|
3670
|
+
contentType: storageContentType,
|
|
3671
|
+
contentDisposition: `inline; filename="${toSafeStorageFilename(options.metadata.filename ?? "attachment.bin")}"`,
|
|
3672
|
+
metadata: options.metadata
|
|
3582
3673
|
});
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3674
|
+
}
|
|
3675
|
+
async function createCanonicalMetadata(options) {
|
|
3676
|
+
const createdAt = Math.floor(Date.now() / 1e3);
|
|
3677
|
+
const expiresAt = createdAt + options.expiresAfterSeconds;
|
|
3678
|
+
const storedFile = {
|
|
3679
|
+
id: options.fileId,
|
|
3680
|
+
bytes: options.bytes,
|
|
3681
|
+
created_at: createdAt,
|
|
3682
|
+
filename: options.filename,
|
|
3683
|
+
object: "file",
|
|
3684
|
+
purpose: options.purpose,
|
|
3685
|
+
status: "processed",
|
|
3686
|
+
expires_at: expiresAt
|
|
3687
|
+
};
|
|
3688
|
+
const metadata = recordMetadata({
|
|
3689
|
+
file: storedFile,
|
|
3690
|
+
filename: options.filename,
|
|
3691
|
+
bytes: options.bytes,
|
|
3692
|
+
mimeType: options.mimeType,
|
|
3693
|
+
sha256Hex: options.sha256Hex,
|
|
3694
|
+
localPath: options.localPath,
|
|
3695
|
+
bucketName: options.bucketName,
|
|
3696
|
+
objectName: options.objectName
|
|
3591
3697
|
});
|
|
3698
|
+
await persistMetadataToDisk(metadata);
|
|
3592
3699
|
return metadata;
|
|
3593
3700
|
}
|
|
3594
|
-
async function
|
|
3701
|
+
async function uploadCanonicalFileFromBytes(params) {
|
|
3595
3702
|
const cacheKey = buildCacheKey(params.filename, params.mimeType, params.sha256Hex);
|
|
3596
|
-
const cached = filesState.
|
|
3703
|
+
const cached = filesState.canonicalUploadCacheByKey.get(cacheKey);
|
|
3597
3704
|
if (cached && isFresh(cached.file)) {
|
|
3598
3705
|
return cached;
|
|
3599
3706
|
}
|
|
3600
|
-
const
|
|
3707
|
+
const fileId = buildCanonicalFileId(params.filename, params.mimeType, params.sha256Hex);
|
|
3708
|
+
const bucketName = resolveCanonicalFilesBucket();
|
|
3709
|
+
const objectName = buildCanonicalObjectName(fileId, params.filename, params.mimeType);
|
|
3710
|
+
const metadataFields = {
|
|
3711
|
+
fileId,
|
|
3712
|
+
filename: params.filename,
|
|
3713
|
+
mimeType: params.mimeType,
|
|
3714
|
+
purpose: params.purpose,
|
|
3715
|
+
sha256: params.sha256Hex,
|
|
3716
|
+
createdAtUnix: Math.floor(Date.now() / 1e3).toString(),
|
|
3717
|
+
expiresAt: new Date(Date.now() + params.expiresAfterSeconds * 1e3).toISOString()
|
|
3718
|
+
};
|
|
3601
3719
|
const startedAtMs = Date.now();
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3720
|
+
const uploaded = await writeCanonicalFileFromBytes({
|
|
3721
|
+
bytes: params.bytes,
|
|
3722
|
+
bucketName,
|
|
3723
|
+
objectName,
|
|
3724
|
+
mimeType: params.mimeType,
|
|
3725
|
+
metadata: metadataFields
|
|
3726
|
+
});
|
|
3727
|
+
if (!uploaded) {
|
|
3728
|
+
await refreshCanonicalObjectMetadata({
|
|
3729
|
+
bucketName,
|
|
3730
|
+
objectName,
|
|
3731
|
+
mimeType: params.mimeType,
|
|
3732
|
+
metadata: metadataFields
|
|
3614
3733
|
});
|
|
3615
|
-
}
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3734
|
+
}
|
|
3735
|
+
const localPath = await cacheBufferLocally(params.bytes, params.sha256Hex);
|
|
3736
|
+
const canonical = await createCanonicalMetadata({
|
|
3737
|
+
fileId,
|
|
3738
|
+
filename: params.filename,
|
|
3739
|
+
mimeType: params.mimeType,
|
|
3740
|
+
purpose: params.purpose,
|
|
3741
|
+
expiresAfterSeconds: params.expiresAfterSeconds,
|
|
3742
|
+
sha256Hex: params.sha256Hex,
|
|
3743
|
+
bytes: params.bytes.byteLength,
|
|
3744
|
+
bucketName,
|
|
3745
|
+
objectName,
|
|
3746
|
+
localPath
|
|
3747
|
+
});
|
|
3748
|
+
if (uploaded) {
|
|
3749
|
+
recordUploadEvent({
|
|
3750
|
+
backend: "gcs",
|
|
3751
|
+
mode: "gcs",
|
|
3619
3752
|
filename: params.filename,
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
highWaterMark: OPENAI_UPLOAD_PART_MAX_BYTES
|
|
3753
|
+
bytes: params.bytes.byteLength,
|
|
3754
|
+
durationMs: Math.max(0, Date.now() - startedAtMs),
|
|
3755
|
+
mimeType: params.mimeType,
|
|
3756
|
+
fileId,
|
|
3757
|
+
fileUri: `gs://${bucketName}/${objectName}`
|
|
3626
3758
|
});
|
|
3627
|
-
let partIndex = 0;
|
|
3628
|
-
for await (const chunk of stream) {
|
|
3629
|
-
const buffer = Buffer4.isBuffer(chunk) ? chunk : Buffer4.from(chunk);
|
|
3630
|
-
const uploadPart = await client.uploads.parts.create(upload.id, {
|
|
3631
|
-
data: new NodeFile(
|
|
3632
|
-
[new Uint8Array(buffer)],
|
|
3633
|
-
`${params.sha256Hex}.${partIndex.toString()}.part`,
|
|
3634
|
-
{
|
|
3635
|
-
type: params.mimeType
|
|
3636
|
-
}
|
|
3637
|
-
)
|
|
3638
|
-
});
|
|
3639
|
-
partIds.push(uploadPart.id);
|
|
3640
|
-
partIndex += 1;
|
|
3641
|
-
}
|
|
3642
|
-
const completed = await client.uploads.complete(upload.id, { part_ids: partIds });
|
|
3643
|
-
const fileId = completed.file?.id;
|
|
3644
|
-
if (!fileId) {
|
|
3645
|
-
throw new Error("OpenAI upload completed without a file id.");
|
|
3646
|
-
}
|
|
3647
|
-
uploaded = await client.files.retrieve(fileId);
|
|
3648
3759
|
}
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3760
|
+
return canonical;
|
|
3761
|
+
}
|
|
3762
|
+
async function uploadCanonicalFileFromPath(params) {
|
|
3763
|
+
const cacheKey = buildCacheKey(params.filename, params.mimeType, params.sha256Hex);
|
|
3764
|
+
const cached = filesState.canonicalUploadCacheByKey.get(cacheKey);
|
|
3765
|
+
if (cached && isFresh(cached.file)) {
|
|
3766
|
+
return cached;
|
|
3767
|
+
}
|
|
3768
|
+
const fileId = buildCanonicalFileId(params.filename, params.mimeType, params.sha256Hex);
|
|
3769
|
+
const bucketName = resolveCanonicalFilesBucket();
|
|
3770
|
+
const objectName = buildCanonicalObjectName(fileId, params.filename, params.mimeType);
|
|
3771
|
+
const metadataFields = {
|
|
3772
|
+
fileId,
|
|
3773
|
+
filename: params.filename,
|
|
3774
|
+
mimeType: params.mimeType,
|
|
3775
|
+
purpose: params.purpose,
|
|
3776
|
+
sha256: params.sha256Hex,
|
|
3777
|
+
createdAtUnix: Math.floor(Date.now() / 1e3).toString(),
|
|
3778
|
+
expiresAt: new Date(Date.now() + params.expiresAfterSeconds * 1e3).toISOString()
|
|
3779
|
+
};
|
|
3780
|
+
const startedAtMs = Date.now();
|
|
3781
|
+
const uploaded = await writeCanonicalFileFromPath({
|
|
3782
|
+
filePath: params.filePath,
|
|
3783
|
+
bucketName,
|
|
3784
|
+
objectName,
|
|
3785
|
+
bytes: params.bytes,
|
|
3654
3786
|
mimeType: params.mimeType,
|
|
3655
|
-
|
|
3787
|
+
metadata: metadataFields
|
|
3656
3788
|
});
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3789
|
+
if (!uploaded) {
|
|
3790
|
+
await refreshCanonicalObjectMetadata({
|
|
3791
|
+
bucketName,
|
|
3792
|
+
objectName,
|
|
3793
|
+
mimeType: params.mimeType,
|
|
3794
|
+
metadata: metadataFields
|
|
3795
|
+
});
|
|
3796
|
+
}
|
|
3797
|
+
const localPath = await cacheFileLocally(params.filePath, params.sha256Hex);
|
|
3798
|
+
const canonical = await createCanonicalMetadata({
|
|
3799
|
+
fileId,
|
|
3800
|
+
filename: params.filename,
|
|
3663
3801
|
mimeType: params.mimeType,
|
|
3664
|
-
|
|
3802
|
+
purpose: params.purpose,
|
|
3803
|
+
expiresAfterSeconds: params.expiresAfterSeconds,
|
|
3804
|
+
sha256Hex: params.sha256Hex,
|
|
3805
|
+
bytes: params.bytes,
|
|
3806
|
+
bucketName,
|
|
3807
|
+
objectName,
|
|
3808
|
+
localPath
|
|
3665
3809
|
});
|
|
3666
|
-
|
|
3810
|
+
if (uploaded) {
|
|
3811
|
+
recordUploadEvent({
|
|
3812
|
+
backend: "gcs",
|
|
3813
|
+
mode: "gcs",
|
|
3814
|
+
filename: params.filename,
|
|
3815
|
+
bytes: params.bytes,
|
|
3816
|
+
durationMs: Math.max(0, Date.now() - startedAtMs),
|
|
3817
|
+
mimeType: params.mimeType,
|
|
3818
|
+
fileId,
|
|
3819
|
+
fileUri: `gs://${bucketName}/${objectName}`
|
|
3820
|
+
});
|
|
3821
|
+
}
|
|
3822
|
+
return canonical;
|
|
3823
|
+
}
|
|
3824
|
+
async function resolveCanonicalStorageLocation(fileId) {
|
|
3825
|
+
const cached = filesState.metadataById.get(fileId) ?? await loadPersistedMetadata(fileId);
|
|
3826
|
+
if (cached?.bucketName && cached.objectName) {
|
|
3827
|
+
return {
|
|
3828
|
+
bucketName: cached.bucketName,
|
|
3829
|
+
objectName: cached.objectName
|
|
3830
|
+
};
|
|
3831
|
+
}
|
|
3832
|
+
const bucketName = resolveCanonicalFilesBucket();
|
|
3833
|
+
const [files2] = await getStorageClient().bucket(bucketName).getFiles({
|
|
3834
|
+
prefix: `${resolveCanonicalFilesPrefix()}${fileId}.`,
|
|
3835
|
+
maxResults: 1,
|
|
3836
|
+
autoPaginate: false
|
|
3837
|
+
});
|
|
3838
|
+
const file = files2[0];
|
|
3839
|
+
if (!file) {
|
|
3840
|
+
throw new Error(`Canonical file ${fileId} was not found in GCS.`);
|
|
3841
|
+
}
|
|
3842
|
+
return {
|
|
3843
|
+
bucketName,
|
|
3844
|
+
objectName: file.name
|
|
3845
|
+
};
|
|
3667
3846
|
}
|
|
3668
|
-
async function
|
|
3847
|
+
async function retrieveCanonicalFile(fileId) {
|
|
3669
3848
|
const cached = filesState.metadataById.get(fileId);
|
|
3670
|
-
if (cached && isFresh(cached.file)) {
|
|
3849
|
+
if (cached && isFresh(cached.file) && cached.bucketName && cached.objectName) {
|
|
3671
3850
|
return cached;
|
|
3672
3851
|
}
|
|
3673
3852
|
const persisted = await loadPersistedMetadata(fileId);
|
|
3674
|
-
if (persisted && isFresh(persisted.file)) {
|
|
3853
|
+
if (persisted && isFresh(persisted.file) && persisted.bucketName && persisted.objectName) {
|
|
3675
3854
|
return persisted;
|
|
3676
3855
|
}
|
|
3677
|
-
const
|
|
3678
|
-
const
|
|
3679
|
-
const
|
|
3680
|
-
const metadata = recordMetadata(
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3687
|
-
|
|
3856
|
+
const existingLocalPath = cached?.localPath ?? persisted?.localPath;
|
|
3857
|
+
const { bucketName, objectName } = await resolveCanonicalStorageLocation(fileId);
|
|
3858
|
+
const [objectMetadata] = await getStorageClient().bucket(bucketName).file(objectName).getMetadata();
|
|
3859
|
+
const metadata = recordMetadata(
|
|
3860
|
+
toStoredFileFromCanonicalMetadata({
|
|
3861
|
+
fileId,
|
|
3862
|
+
bucketName,
|
|
3863
|
+
objectName,
|
|
3864
|
+
objectMetadata,
|
|
3865
|
+
localPath: existingLocalPath
|
|
3866
|
+
})
|
|
3867
|
+
);
|
|
3688
3868
|
await persistMetadataToDisk(metadata);
|
|
3689
3869
|
return metadata;
|
|
3690
3870
|
}
|
|
@@ -3712,7 +3892,7 @@ function resolveVertexMirrorBucket() {
|
|
|
3712
3892
|
const trimmed = raw?.trim();
|
|
3713
3893
|
if (!trimmed) {
|
|
3714
3894
|
throw new Error(
|
|
3715
|
-
"VERTEX_GCS_BUCKET must be set to use
|
|
3895
|
+
"VERTEX_GCS_BUCKET must be set to use canonical file ids with Vertex Gemini models."
|
|
3716
3896
|
);
|
|
3717
3897
|
}
|
|
3718
3898
|
return trimmed.replace(/^gs:\/\//u, "").replace(/\/+$/u, "");
|
|
@@ -3742,61 +3922,41 @@ function getGeminiMirrorClient() {
|
|
|
3742
3922
|
}
|
|
3743
3923
|
return filesState.geminiClientPromise;
|
|
3744
3924
|
}
|
|
3745
|
-
async function
|
|
3925
|
+
async function materializeCanonicalFile(fileId) {
|
|
3746
3926
|
const cachedPromise = filesState.materializedById.get(fileId);
|
|
3747
3927
|
if (cachedPromise) {
|
|
3748
3928
|
return await cachedPromise;
|
|
3749
3929
|
}
|
|
3750
3930
|
const promise = (async () => {
|
|
3751
|
-
const metadata = await
|
|
3752
|
-
if (metadata.localPath && metadata.sha256Hex && metadata.mimeType) {
|
|
3931
|
+
const metadata = await retrieveCanonicalFile(fileId);
|
|
3932
|
+
if (metadata.localPath && metadata.sha256Hex && metadata.mimeType && metadata.bucketName && metadata.objectName) {
|
|
3753
3933
|
return {
|
|
3754
3934
|
file: metadata.file,
|
|
3755
3935
|
filename: metadata.filename,
|
|
3756
3936
|
bytes: metadata.bytes,
|
|
3757
3937
|
mimeType: metadata.mimeType,
|
|
3758
3938
|
sha256Hex: metadata.sha256Hex,
|
|
3759
|
-
localPath: metadata.localPath
|
|
3939
|
+
localPath: metadata.localPath,
|
|
3940
|
+
bucketName: metadata.bucketName,
|
|
3941
|
+
objectName: metadata.objectName
|
|
3760
3942
|
};
|
|
3761
3943
|
}
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
path4.join(FILES_TEMP_ROOT, `${fileId.replace(/[^a-z0-9_-]/giu, "")}-`)
|
|
3765
|
-
);
|
|
3766
|
-
const localPath = path4.join(tempDir, normaliseFilename(metadata.filename, `${fileId}.bin`));
|
|
3767
|
-
const response = await getOpenAiClient().files.content(fileId);
|
|
3768
|
-
if (!response.ok) {
|
|
3769
|
-
throw new Error(
|
|
3770
|
-
`Failed to download OpenAI file ${fileId}: ${response.status} ${response.statusText}`
|
|
3771
|
-
);
|
|
3944
|
+
if (!metadata.bucketName || !metadata.objectName) {
|
|
3945
|
+
throw new Error(`Canonical file ${fileId} is missing GCS location metadata.`);
|
|
3772
3946
|
}
|
|
3773
|
-
const
|
|
3774
|
-
const mimeType = resolveMimeType(metadata.filename,
|
|
3775
|
-
const
|
|
3776
|
-
|
|
3777
|
-
if (response.body) {
|
|
3778
|
-
const source = Readable.fromWeb(response.body);
|
|
3779
|
-
const writable = createWriteStream(localPath, { flags: "wx" });
|
|
3780
|
-
source.on("data", (chunk) => {
|
|
3781
|
-
const buffer = Buffer4.isBuffer(chunk) ? chunk : Buffer4.from(chunk);
|
|
3782
|
-
hash.update(buffer);
|
|
3783
|
-
bytes += buffer.byteLength;
|
|
3784
|
-
});
|
|
3785
|
-
await pipeline(source, writable);
|
|
3786
|
-
} else {
|
|
3787
|
-
const buffer = Buffer4.from(await response.arrayBuffer());
|
|
3788
|
-
hash.update(buffer);
|
|
3789
|
-
bytes = buffer.byteLength;
|
|
3790
|
-
await writeFile2(localPath, buffer);
|
|
3791
|
-
}
|
|
3792
|
-
const sha256Hex = hash.digest("hex");
|
|
3947
|
+
const [downloadedBytes] = await getStorageClient().bucket(metadata.bucketName).file(metadata.objectName).download();
|
|
3948
|
+
const mimeType = metadata.mimeType ?? resolveMimeType(metadata.filename, void 0);
|
|
3949
|
+
const sha256Hex = metadata.sha256Hex ?? computeSha256Hex(downloadedBytes);
|
|
3950
|
+
const localPath = await cacheBufferLocally(downloadedBytes, sha256Hex);
|
|
3793
3951
|
const updated = recordMetadata({
|
|
3794
3952
|
file: metadata.file,
|
|
3795
3953
|
filename: metadata.filename,
|
|
3796
|
-
bytes:
|
|
3954
|
+
bytes: downloadedBytes.byteLength || metadata.bytes,
|
|
3797
3955
|
mimeType,
|
|
3798
3956
|
sha256Hex,
|
|
3799
|
-
localPath
|
|
3957
|
+
localPath,
|
|
3958
|
+
bucketName: metadata.bucketName,
|
|
3959
|
+
objectName: metadata.objectName
|
|
3800
3960
|
});
|
|
3801
3961
|
await persistMetadataToDisk(updated);
|
|
3802
3962
|
return {
|
|
@@ -3805,7 +3965,9 @@ async function materializeOpenAiFile(fileId) {
|
|
|
3805
3965
|
bytes: updated.bytes,
|
|
3806
3966
|
mimeType: updated.mimeType ?? mimeType,
|
|
3807
3967
|
sha256Hex,
|
|
3808
|
-
localPath
|
|
3968
|
+
localPath,
|
|
3969
|
+
bucketName: metadata.bucketName,
|
|
3970
|
+
objectName: metadata.objectName
|
|
3809
3971
|
};
|
|
3810
3972
|
})();
|
|
3811
3973
|
filesState.materializedById.set(fileId, promise);
|
|
@@ -3821,14 +3983,14 @@ async function ensureGeminiFileMirror(fileId) {
|
|
|
3821
3983
|
if (cached) {
|
|
3822
3984
|
return cached;
|
|
3823
3985
|
}
|
|
3824
|
-
const materialized = await
|
|
3986
|
+
const materialized = await materializeCanonicalFile(fileId);
|
|
3825
3987
|
const client = await getGeminiMirrorClient();
|
|
3826
3988
|
const name = buildGeminiMirrorName(materialized.sha256Hex);
|
|
3827
3989
|
try {
|
|
3828
3990
|
const existing = await client.files.get({ name });
|
|
3829
3991
|
if (existing.name && existing.uri && existing.mimeType) {
|
|
3830
3992
|
const mirror2 = {
|
|
3831
|
-
|
|
3993
|
+
canonicalFileId: fileId,
|
|
3832
3994
|
name: existing.name,
|
|
3833
3995
|
uri: existing.uri,
|
|
3834
3996
|
mimeType: existing.mimeType,
|
|
@@ -3856,7 +4018,7 @@ async function ensureGeminiFileMirror(fileId) {
|
|
|
3856
4018
|
throw new Error("Gemini file upload completed without a usable URI.");
|
|
3857
4019
|
}
|
|
3858
4020
|
const mirror = {
|
|
3859
|
-
|
|
4021
|
+
canonicalFileId: fileId,
|
|
3860
4022
|
name: resolved.name,
|
|
3861
4023
|
uri: resolved.uri,
|
|
3862
4024
|
mimeType: resolved.mimeType,
|
|
@@ -3881,7 +4043,7 @@ async function ensureVertexFileMirror(fileId) {
|
|
|
3881
4043
|
if (cached) {
|
|
3882
4044
|
return cached;
|
|
3883
4045
|
}
|
|
3884
|
-
const materialized = await
|
|
4046
|
+
const materialized = await materializeCanonicalFile(fileId);
|
|
3885
4047
|
const bucketName = resolveVertexMirrorBucket();
|
|
3886
4048
|
const prefix = resolveVertexMirrorPrefix();
|
|
3887
4049
|
const extension = mime.getExtension(materialized.mimeType) ?? path4.extname(materialized.filename).replace(/^\./u, "") ?? "bin";
|
|
@@ -3922,7 +4084,7 @@ async function ensureVertexFileMirror(fileId) {
|
|
|
3922
4084
|
}
|
|
3923
4085
|
}
|
|
3924
4086
|
const mirror = {
|
|
3925
|
-
|
|
4087
|
+
canonicalFileId: fileId,
|
|
3926
4088
|
bucket: bucketName,
|
|
3927
4089
|
objectName,
|
|
3928
4090
|
fileUri: `gs://${bucketName}/${objectName}`,
|
|
@@ -3953,7 +4115,7 @@ async function filesCreate(params) {
|
|
|
3953
4115
|
const filename2 = normaliseFilename(params.filename, path4.basename(filePath));
|
|
3954
4116
|
const mimeType2 = resolveMimeType(filename2, params.mimeType);
|
|
3955
4117
|
const sha256Hex2 = await computeFileSha256Hex(filePath);
|
|
3956
|
-
const uploaded2 = await
|
|
4118
|
+
const uploaded2 = await uploadCanonicalFileFromPath({
|
|
3957
4119
|
filePath,
|
|
3958
4120
|
filename: filename2,
|
|
3959
4121
|
mimeType: mimeType2,
|
|
@@ -3962,19 +4124,13 @@ async function filesCreate(params) {
|
|
|
3962
4124
|
sha256Hex: sha256Hex2,
|
|
3963
4125
|
bytes: info.size
|
|
3964
4126
|
});
|
|
3965
|
-
|
|
3966
|
-
const cached2 = recordMetadata({
|
|
3967
|
-
...uploaded2,
|
|
3968
|
-
localPath: localPath2
|
|
3969
|
-
});
|
|
3970
|
-
await persistMetadataToDisk(cached2);
|
|
3971
|
-
return cached2.file;
|
|
4127
|
+
return uploaded2.file;
|
|
3972
4128
|
}
|
|
3973
4129
|
const filename = normaliseFilename(params.filename);
|
|
3974
4130
|
const bytes = toBuffer(params.data);
|
|
3975
4131
|
const mimeType = resolveMimeType(filename, params.mimeType, "text/plain");
|
|
3976
4132
|
const sha256Hex = computeSha256Hex(bytes);
|
|
3977
|
-
const uploaded = await
|
|
4133
|
+
const uploaded = await uploadCanonicalFileFromBytes({
|
|
3978
4134
|
bytes,
|
|
3979
4135
|
filename,
|
|
3980
4136
|
mimeType,
|
|
@@ -3982,16 +4138,10 @@ async function filesCreate(params) {
|
|
|
3982
4138
|
expiresAfterSeconds,
|
|
3983
4139
|
sha256Hex
|
|
3984
4140
|
});
|
|
3985
|
-
|
|
3986
|
-
const cached = recordMetadata({
|
|
3987
|
-
...uploaded,
|
|
3988
|
-
localPath
|
|
3989
|
-
});
|
|
3990
|
-
await persistMetadataToDisk(cached);
|
|
3991
|
-
return cached.file;
|
|
4141
|
+
return uploaded.file;
|
|
3992
4142
|
}
|
|
3993
4143
|
async function filesRetrieve(fileId) {
|
|
3994
|
-
return (await
|
|
4144
|
+
return (await retrieveCanonicalFile(fileId)).file;
|
|
3995
4145
|
}
|
|
3996
4146
|
async function filesDelete(fileId) {
|
|
3997
4147
|
const cachedGemini = filesState.geminiMirrorById.get(fileId);
|
|
@@ -4018,34 +4168,73 @@ async function filesDelete(fileId) {
|
|
|
4018
4168
|
} catch {
|
|
4019
4169
|
}
|
|
4020
4170
|
}
|
|
4021
|
-
|
|
4171
|
+
try {
|
|
4172
|
+
const { bucketName, objectName } = await resolveCanonicalStorageLocation(fileId);
|
|
4173
|
+
await getStorageClient().bucket(bucketName).file(objectName).delete({ ignoreNotFound: true });
|
|
4174
|
+
} catch {
|
|
4175
|
+
}
|
|
4022
4176
|
filesState.metadataById.delete(fileId);
|
|
4177
|
+
filesState.canonicalUploadCacheByKey.forEach((value, key) => {
|
|
4178
|
+
if (value.file.id === fileId) {
|
|
4179
|
+
filesState.canonicalUploadCacheByKey.delete(key);
|
|
4180
|
+
}
|
|
4181
|
+
});
|
|
4023
4182
|
filesState.materializedById.delete(fileId);
|
|
4024
4183
|
try {
|
|
4025
4184
|
await unlink(buildCachedMetadataPath(fileId));
|
|
4026
4185
|
} catch {
|
|
4027
4186
|
}
|
|
4028
4187
|
return {
|
|
4029
|
-
id:
|
|
4030
|
-
deleted:
|
|
4188
|
+
id: fileId,
|
|
4189
|
+
deleted: true,
|
|
4031
4190
|
object: "file"
|
|
4032
4191
|
};
|
|
4033
4192
|
}
|
|
4034
4193
|
async function filesContent(fileId) {
|
|
4035
|
-
|
|
4194
|
+
const metadata = await retrieveCanonicalFile(fileId);
|
|
4195
|
+
if (!metadata.bucketName || !metadata.objectName) {
|
|
4196
|
+
throw new Error(`Canonical file ${fileId} is missing GCS location metadata.`);
|
|
4197
|
+
}
|
|
4198
|
+
const [bytes] = await getStorageClient().bucket(metadata.bucketName).file(metadata.objectName).download();
|
|
4199
|
+
const headers = new Headers();
|
|
4200
|
+
headers.set("content-type", metadata.mimeType ?? resolveMimeType(metadata.filename, void 0));
|
|
4201
|
+
headers.set("content-length", bytes.byteLength.toString());
|
|
4202
|
+
headers.set(
|
|
4203
|
+
"content-disposition",
|
|
4204
|
+
`inline; filename="${toSafeStorageFilename(metadata.filename)}"`
|
|
4205
|
+
);
|
|
4206
|
+
return new Response(bytes, {
|
|
4207
|
+
status: 200,
|
|
4208
|
+
headers
|
|
4209
|
+
});
|
|
4036
4210
|
}
|
|
4037
4211
|
async function getCanonicalFileMetadata(fileId) {
|
|
4038
|
-
const metadata = await
|
|
4212
|
+
const metadata = await retrieveCanonicalFile(fileId);
|
|
4039
4213
|
const mimeType = metadata.mimeType ?? resolveMimeType(metadata.filename, void 0);
|
|
4040
4214
|
const updated = metadata.mimeType === mimeType ? metadata : recordMetadata({
|
|
4041
4215
|
...metadata,
|
|
4042
4216
|
mimeType
|
|
4043
4217
|
});
|
|
4218
|
+
if (!updated.bucketName || !updated.objectName) {
|
|
4219
|
+
throw new Error(`Canonical file ${fileId} is missing GCS location metadata.`);
|
|
4220
|
+
}
|
|
4044
4221
|
return {
|
|
4045
4222
|
...updated,
|
|
4046
|
-
mimeType
|
|
4223
|
+
mimeType,
|
|
4224
|
+
bucketName: updated.bucketName,
|
|
4225
|
+
objectName: updated.objectName
|
|
4047
4226
|
};
|
|
4048
4227
|
}
|
|
4228
|
+
async function getCanonicalFileSignedUrl(options) {
|
|
4229
|
+
const metadata = await getCanonicalFileMetadata(options.fileId);
|
|
4230
|
+
const [signedUrl] = await getStorageClient().bucket(metadata.bucketName).file(metadata.objectName).getSignedUrl({
|
|
4231
|
+
version: "v4",
|
|
4232
|
+
action: "read",
|
|
4233
|
+
expires: Date.now() + (options.expiresAfterSeconds ?? 15 * 60) * 1e3,
|
|
4234
|
+
responseType: resolveCanonicalStorageContentType(metadata.filename, metadata.mimeType)
|
|
4235
|
+
});
|
|
4236
|
+
return signedUrl;
|
|
4237
|
+
}
|
|
4049
4238
|
var files = {
|
|
4050
4239
|
create: filesCreate,
|
|
4051
4240
|
retrieve: filesRetrieve,
|
|
@@ -4407,6 +4596,7 @@ function isJsonSchemaObject(schema) {
|
|
|
4407
4596
|
return false;
|
|
4408
4597
|
}
|
|
4409
4598
|
var CANONICAL_GEMINI_FILE_URI_PREFIX = "openai://file/";
|
|
4599
|
+
var CANONICAL_LLM_FILE_ID_PATTERN = /^file_[a-f0-9]{64}$/u;
|
|
4410
4600
|
function buildCanonicalGeminiFileUri(fileId) {
|
|
4411
4601
|
return `${CANONICAL_GEMINI_FILE_URI_PREFIX}${fileId}`;
|
|
4412
4602
|
}
|
|
@@ -4417,6 +4607,75 @@ function parseCanonicalGeminiFileId(fileUri) {
|
|
|
4417
4607
|
const fileId = fileUri.slice(CANONICAL_GEMINI_FILE_URI_PREFIX.length).trim();
|
|
4418
4608
|
return fileId.length > 0 ? fileId : void 0;
|
|
4419
4609
|
}
|
|
4610
|
+
function isCanonicalLlmFileId(fileId) {
|
|
4611
|
+
return typeof fileId === "string" && CANONICAL_LLM_FILE_ID_PATTERN.test(fileId.trim());
|
|
4612
|
+
}
|
|
4613
|
+
function isLlmMediaResolution(value) {
|
|
4614
|
+
return value === "auto" || value === "low" || value === "medium" || value === "high" || value === "original";
|
|
4615
|
+
}
|
|
4616
|
+
function resolveEffectiveMediaResolution(detail, fallback) {
|
|
4617
|
+
return detail ?? fallback;
|
|
4618
|
+
}
|
|
4619
|
+
function supportsOpenAiOriginalImageDetail(model) {
|
|
4620
|
+
if (!model) {
|
|
4621
|
+
return false;
|
|
4622
|
+
}
|
|
4623
|
+
const providerModel = isChatGptModelId(model) ? resolveChatGptProviderModel(model) : model;
|
|
4624
|
+
const match = /^gpt-(\d+)(?:\.(\d+))?/u.exec(providerModel);
|
|
4625
|
+
if (!match) {
|
|
4626
|
+
return false;
|
|
4627
|
+
}
|
|
4628
|
+
const major = Number(match[1]);
|
|
4629
|
+
const minor = Number(match[2] ?? "0");
|
|
4630
|
+
if (!Number.isFinite(major) || !Number.isFinite(minor)) {
|
|
4631
|
+
return false;
|
|
4632
|
+
}
|
|
4633
|
+
return major > 5 || major === 5 && minor >= 4;
|
|
4634
|
+
}
|
|
4635
|
+
function toOpenAiImageDetail(mediaResolution, model) {
|
|
4636
|
+
switch (mediaResolution) {
|
|
4637
|
+
case "low":
|
|
4638
|
+
return "low";
|
|
4639
|
+
case "medium":
|
|
4640
|
+
return "high";
|
|
4641
|
+
case "high":
|
|
4642
|
+
return "high";
|
|
4643
|
+
case "original":
|
|
4644
|
+
return supportsOpenAiOriginalImageDetail(model) ? "original" : "high";
|
|
4645
|
+
case "auto":
|
|
4646
|
+
default:
|
|
4647
|
+
return "auto";
|
|
4648
|
+
}
|
|
4649
|
+
}
|
|
4650
|
+
function toGeminiMediaResolution(mediaResolution) {
|
|
4651
|
+
switch (mediaResolution) {
|
|
4652
|
+
case "low":
|
|
4653
|
+
return MediaResolution.MEDIA_RESOLUTION_LOW;
|
|
4654
|
+
case "medium":
|
|
4655
|
+
return MediaResolution.MEDIA_RESOLUTION_MEDIUM;
|
|
4656
|
+
case "high":
|
|
4657
|
+
case "original":
|
|
4658
|
+
return MediaResolution.MEDIA_RESOLUTION_HIGH;
|
|
4659
|
+
case "auto":
|
|
4660
|
+
default:
|
|
4661
|
+
return void 0;
|
|
4662
|
+
}
|
|
4663
|
+
}
|
|
4664
|
+
function toGeminiPartMediaResolution(mediaResolution) {
|
|
4665
|
+
switch (mediaResolution) {
|
|
4666
|
+
case "low":
|
|
4667
|
+
return PartMediaResolutionLevel.MEDIA_RESOLUTION_LOW;
|
|
4668
|
+
case "medium":
|
|
4669
|
+
return PartMediaResolutionLevel.MEDIA_RESOLUTION_MEDIUM;
|
|
4670
|
+
case "high":
|
|
4671
|
+
return PartMediaResolutionLevel.MEDIA_RESOLUTION_HIGH;
|
|
4672
|
+
case "original":
|
|
4673
|
+
return PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH;
|
|
4674
|
+
case "auto":
|
|
4675
|
+
default:
|
|
4676
|
+
return void 0;
|
|
4677
|
+
}
|
|
4678
|
+
}
|
|
4420
4679
|
function cloneContentPart(part) {
|
|
4421
4680
|
switch (part.type) {
|
|
4422
4681
|
case "text":
|
|
@@ -4545,7 +4804,8 @@ function convertGeminiContentToLlmContent(content) {
|
|
|
4545
4804
|
parts: convertGooglePartsToLlmParts(content.parts ?? [])
|
|
4546
4805
|
};
|
|
4547
4806
|
}
|
|
4548
|
-
function toGeminiPart(part) {
|
|
4807
|
+
function toGeminiPart(part, options) {
|
|
4808
|
+
const defaultMediaResolution = options?.defaultMediaResolution;
|
|
4549
4809
|
switch (part.type) {
|
|
4550
4810
|
case "text":
|
|
4551
4811
|
return {
|
|
@@ -4553,6 +4813,18 @@ function toGeminiPart(part) {
|
|
|
4553
4813
|
thought: part.thought === true ? true : void 0
|
|
4554
4814
|
};
|
|
4555
4815
|
case "inlineData": {
|
|
4816
|
+
if (isInlineImageMime(part.mimeType)) {
|
|
4817
|
+
const mimeType = part.mimeType ?? "application/octet-stream";
|
|
4818
|
+
const geminiPart = createPartFromBase64(
|
|
4819
|
+
part.data,
|
|
4820
|
+
mimeType,
|
|
4821
|
+
toGeminiPartMediaResolution(defaultMediaResolution)
|
|
4822
|
+
);
|
|
4823
|
+
if (part.filename && geminiPart.inlineData) {
|
|
4824
|
+
geminiPart.inlineData.displayName = part.filename;
|
|
4825
|
+
}
|
|
4826
|
+
return geminiPart;
|
|
4827
|
+
}
|
|
4556
4828
|
const inlineData = {
|
|
4557
4829
|
data: part.data,
|
|
4558
4830
|
mimeType: part.mimeType
|
|
@@ -4565,31 +4837,35 @@ function toGeminiPart(part) {
|
|
|
4565
4837
|
};
|
|
4566
4838
|
}
|
|
4567
4839
|
case "input_image": {
|
|
4840
|
+
const mediaResolution = resolveEffectiveMediaResolution(part.detail, defaultMediaResolution);
|
|
4841
|
+
const geminiPartMediaResolution = toGeminiPartMediaResolution(mediaResolution);
|
|
4568
4842
|
if (part.file_id) {
|
|
4569
|
-
return
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
};
|
|
4843
|
+
return createPartFromUri(
|
|
4844
|
+
buildCanonicalGeminiFileUri(part.file_id),
|
|
4845
|
+
inferToolOutputMimeTypeFromFilename(part.filename) ?? "application/octet-stream",
|
|
4846
|
+
geminiPartMediaResolution
|
|
4847
|
+
);
|
|
4575
4848
|
}
|
|
4576
4849
|
if (typeof part.image_url !== "string" || part.image_url.trim().length === 0) {
|
|
4577
4850
|
throw new Error("input_image requires image_url or file_id.");
|
|
4578
4851
|
}
|
|
4579
4852
|
const parsed = parseDataUrlPayload(part.image_url);
|
|
4580
4853
|
if (parsed) {
|
|
4581
|
-
const geminiPart = createPartFromBase64(
|
|
4854
|
+
const geminiPart = createPartFromBase64(
|
|
4855
|
+
parsed.dataBase64,
|
|
4856
|
+
parsed.mimeType,
|
|
4857
|
+
geminiPartMediaResolution
|
|
4858
|
+
);
|
|
4582
4859
|
if (part.filename && geminiPart.inlineData) {
|
|
4583
4860
|
geminiPart.inlineData.displayName = part.filename;
|
|
4584
4861
|
}
|
|
4585
4862
|
return geminiPart;
|
|
4586
4863
|
}
|
|
4587
|
-
return
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
};
|
|
4864
|
+
return createPartFromUri(
|
|
4865
|
+
part.image_url,
|
|
4866
|
+
inferToolOutputMimeTypeFromFilename(part.filename) ?? "application/octet-stream",
|
|
4867
|
+
geminiPartMediaResolution
|
|
4868
|
+
);
|
|
4593
4869
|
}
|
|
4594
4870
|
case "input_file": {
|
|
4595
4871
|
if (part.file_id) {
|
|
@@ -4632,11 +4908,11 @@ function toGeminiPart(part) {
|
|
|
4632
4908
|
throw new Error("Unsupported LLM content part");
|
|
4633
4909
|
}
|
|
4634
4910
|
}
|
|
4635
|
-
function convertLlmContentToGeminiContent(content) {
|
|
4911
|
+
function convertLlmContentToGeminiContent(content, options) {
|
|
4636
4912
|
const role = content.role === "assistant" ? "model" : "user";
|
|
4637
4913
|
return {
|
|
4638
4914
|
role,
|
|
4639
|
-
parts: content.parts.map(toGeminiPart)
|
|
4915
|
+
parts: content.parts.map((part) => toGeminiPart(part, options))
|
|
4640
4916
|
};
|
|
4641
4917
|
}
|
|
4642
4918
|
function resolveProvider(model) {
|
|
@@ -4817,11 +5093,25 @@ async function storeCanonicalPromptFile(options) {
|
|
|
4817
5093
|
mimeType: options.mimeType
|
|
4818
5094
|
};
|
|
4819
5095
|
}
|
|
4820
|
-
async function prepareOpenAiPromptContentItem(item) {
|
|
5096
|
+
async function prepareOpenAiPromptContentItem(item, options) {
|
|
4821
5097
|
if (!isOpenAiNativeContentItem(item)) {
|
|
4822
5098
|
return item;
|
|
4823
5099
|
}
|
|
4824
|
-
if (item.type === "input_image"
|
|
5100
|
+
if (item.type === "input_image") {
|
|
5101
|
+
if (isCanonicalLlmFileId(item.file_id)) {
|
|
5102
|
+
const signedUrl2 = await getCanonicalFileSignedUrl({ fileId: item.file_id });
|
|
5103
|
+
return {
|
|
5104
|
+
type: "input_image",
|
|
5105
|
+
image_url: signedUrl2,
|
|
5106
|
+
detail: toOpenAiImageDetail(
|
|
5107
|
+
isLlmMediaResolution(item.detail) ? item.detail : void 0,
|
|
5108
|
+
options?.model
|
|
5109
|
+
)
|
|
5110
|
+
};
|
|
5111
|
+
}
|
|
5112
|
+
if (options?.offloadInlineData !== true || typeof item.image_url !== "string" || !item.image_url.trim().toLowerCase().startsWith("data:")) {
|
|
5113
|
+
return item;
|
|
5114
|
+
}
|
|
4825
5115
|
const parsed = parseDataUrlPayload(item.image_url);
|
|
4826
5116
|
if (!parsed) {
|
|
4827
5117
|
return item;
|
|
@@ -4834,13 +5124,27 @@ async function prepareOpenAiPromptContentItem(item) {
|
|
|
4834
5124
|
guessInlineDataFilename(parsed.mimeType)
|
|
4835
5125
|
)
|
|
4836
5126
|
});
|
|
5127
|
+
const signedUrl = await getCanonicalFileSignedUrl({ fileId: uploaded.fileId });
|
|
4837
5128
|
return {
|
|
4838
5129
|
type: "input_image",
|
|
4839
|
-
|
|
4840
|
-
|
|
5130
|
+
image_url: signedUrl,
|
|
5131
|
+
detail: toOpenAiImageDetail(
|
|
5132
|
+
isLlmMediaResolution(item.detail) ? item.detail : void 0,
|
|
5133
|
+
options?.model
|
|
5134
|
+
)
|
|
5135
|
+
};
|
|
5136
|
+
}
|
|
5137
|
+
if (item.type !== "input_file") {
|
|
5138
|
+
return item;
|
|
5139
|
+
}
|
|
5140
|
+
if (isCanonicalLlmFileId(item.file_id)) {
|
|
5141
|
+
const signedUrl = await getCanonicalFileSignedUrl({ fileId: item.file_id });
|
|
5142
|
+
return {
|
|
5143
|
+
type: "input_file",
|
|
5144
|
+
file_url: signedUrl
|
|
4841
5145
|
};
|
|
4842
5146
|
}
|
|
4843
|
-
if (
|
|
5147
|
+
if (options?.offloadInlineData !== true) {
|
|
4844
5148
|
return item;
|
|
4845
5149
|
}
|
|
4846
5150
|
if (typeof item.file_data === "string" && item.file_data.trim().length > 0) {
|
|
@@ -4854,7 +5158,11 @@ async function prepareOpenAiPromptContentItem(item) {
|
|
|
4854
5158
|
mimeType,
|
|
4855
5159
|
filename
|
|
4856
5160
|
});
|
|
4857
|
-
|
|
5161
|
+
const signedUrl = await getCanonicalFileSignedUrl({ fileId: uploaded.fileId });
|
|
5162
|
+
return {
|
|
5163
|
+
type: "input_file",
|
|
5164
|
+
file_url: signedUrl
|
|
5165
|
+
};
|
|
4858
5166
|
}
|
|
4859
5167
|
if (typeof item.file_url === "string" && item.file_url.trim().toLowerCase().startsWith("data:")) {
|
|
4860
5168
|
const parsed = parseDataUrlPayload(item.file_url);
|
|
@@ -4869,11 +5177,15 @@ async function prepareOpenAiPromptContentItem(item) {
|
|
|
4869
5177
|
guessInlineDataFilename(parsed.mimeType)
|
|
4870
5178
|
)
|
|
4871
5179
|
});
|
|
4872
|
-
|
|
5180
|
+
const signedUrl = await getCanonicalFileSignedUrl({ fileId: uploaded.fileId });
|
|
5181
|
+
return {
|
|
5182
|
+
type: "input_file",
|
|
5183
|
+
file_url: signedUrl
|
|
5184
|
+
};
|
|
4873
5185
|
}
|
|
4874
5186
|
return item;
|
|
4875
5187
|
}
|
|
4876
|
-
async function prepareOpenAiPromptInput(input) {
|
|
5188
|
+
async function prepareOpenAiPromptInput(input, options) {
|
|
4877
5189
|
const prepareItem = async (item) => {
|
|
4878
5190
|
if (!item || typeof item !== "object") {
|
|
4879
5191
|
return item;
|
|
@@ -4883,7 +5195,7 @@ async function prepareOpenAiPromptInput(input) {
|
|
|
4883
5195
|
return {
|
|
4884
5196
|
...record,
|
|
4885
5197
|
content: await Promise.all(
|
|
4886
|
-
record.content.map((part) => prepareOpenAiPromptContentItem(part))
|
|
5198
|
+
record.content.map((part) => prepareOpenAiPromptContentItem(part, options))
|
|
4887
5199
|
)
|
|
4888
5200
|
};
|
|
4889
5201
|
}
|
|
@@ -4891,19 +5203,48 @@ async function prepareOpenAiPromptInput(input) {
|
|
|
4891
5203
|
return {
|
|
4892
5204
|
...record,
|
|
4893
5205
|
output: await Promise.all(
|
|
4894
|
-
record.output.map((part) => prepareOpenAiPromptContentItem(part))
|
|
5206
|
+
record.output.map((part) => prepareOpenAiPromptContentItem(part, options))
|
|
4895
5207
|
)
|
|
4896
5208
|
};
|
|
4897
5209
|
}
|
|
4898
|
-
return await prepareOpenAiPromptContentItem(item);
|
|
5210
|
+
return await prepareOpenAiPromptContentItem(item, options);
|
|
4899
5211
|
};
|
|
4900
5212
|
return await Promise.all(input.map((item) => prepareItem(item)));
|
|
4901
5213
|
}
|
|
4902
|
-
|
|
4903
|
-
|
|
5214
|
+
function hasCanonicalOpenAiFileReferences(input) {
|
|
5215
|
+
let found = false;
|
|
5216
|
+
const visitItems = (items) => {
|
|
5217
|
+
for (const item of items) {
|
|
5218
|
+
if (found || !item || typeof item !== "object") {
|
|
5219
|
+
continue;
|
|
5220
|
+
}
|
|
5221
|
+
if (Array.isArray(item.content)) {
|
|
5222
|
+
visitItems(item.content);
|
|
5223
|
+
}
|
|
5224
|
+
if (Array.isArray(item.output)) {
|
|
5225
|
+
visitItems(item.output);
|
|
5226
|
+
}
|
|
5227
|
+
if (!isOpenAiNativeContentItem(item)) {
|
|
5228
|
+
continue;
|
|
5229
|
+
}
|
|
5230
|
+
if ((item.type === "input_image" || item.type === "input_file") && isCanonicalLlmFileId(item.file_id)) {
|
|
5231
|
+
found = true;
|
|
5232
|
+
return;
|
|
5233
|
+
}
|
|
5234
|
+
}
|
|
5235
|
+
};
|
|
5236
|
+
visitItems(input);
|
|
5237
|
+
return found;
|
|
5238
|
+
}
|
|
5239
|
+
async function maybePrepareOpenAiPromptInput(input, options) {
|
|
5240
|
+
const offloadInlineData = estimateOpenAiInlinePromptBytes(input) > INLINE_ATTACHMENT_PROMPT_THRESHOLD_BYTES;
|
|
5241
|
+
if (!offloadInlineData && !hasCanonicalOpenAiFileReferences(input)) {
|
|
4904
5242
|
return Array.from(input);
|
|
4905
5243
|
}
|
|
4906
|
-
return await prepareOpenAiPromptInput(input
|
|
5244
|
+
return await prepareOpenAiPromptInput(input, {
|
|
5245
|
+
...options,
|
|
5246
|
+
offloadInlineData
|
|
5247
|
+
});
|
|
4907
5248
|
}
|
|
4908
5249
|
function estimateGeminiInlinePromptBytes(contents) {
|
|
4909
5250
|
let total = 0;
|
|
@@ -4934,22 +5275,25 @@ async function prepareGeminiPromptContents(contents) {
|
|
|
4934
5275
|
for (const part of content.parts ?? []) {
|
|
4935
5276
|
const canonicalFileId = parseCanonicalGeminiFileId(part.fileData?.fileUri);
|
|
4936
5277
|
if (canonicalFileId) {
|
|
5278
|
+
const mediaResolution = part.mediaResolution?.level;
|
|
4937
5279
|
await getCanonicalFileMetadata(canonicalFileId);
|
|
4938
5280
|
if (backend === "api") {
|
|
4939
5281
|
const mirrored = await ensureGeminiFileMirror(canonicalFileId);
|
|
4940
|
-
parts.push(createPartFromUri(mirrored.uri, mirrored.mimeType));
|
|
5282
|
+
parts.push(createPartFromUri(mirrored.uri, mirrored.mimeType, mediaResolution));
|
|
4941
5283
|
} else {
|
|
4942
5284
|
const mirrored = await ensureVertexFileMirror(canonicalFileId);
|
|
4943
5285
|
parts.push({
|
|
4944
5286
|
fileData: {
|
|
4945
5287
|
fileUri: mirrored.fileUri,
|
|
4946
5288
|
mimeType: mirrored.mimeType
|
|
4947
|
-
}
|
|
5289
|
+
},
|
|
5290
|
+
...mediaResolution ? { mediaResolution: { level: mediaResolution } } : {}
|
|
4948
5291
|
});
|
|
4949
5292
|
}
|
|
4950
5293
|
continue;
|
|
4951
5294
|
}
|
|
4952
5295
|
if (part.inlineData?.data) {
|
|
5296
|
+
const mediaResolution = part.mediaResolution?.level;
|
|
4953
5297
|
const mimeType = part.inlineData.mimeType ?? "application/octet-stream";
|
|
4954
5298
|
const filename = normaliseAttachmentFilename(
|
|
4955
5299
|
getInlineAttachmentFilename(part.inlineData) ?? part.inlineData.displayName ?? guessInlineDataFilename(mimeType),
|
|
@@ -4962,14 +5306,15 @@ async function prepareGeminiPromptContents(contents) {
|
|
|
4962
5306
|
});
|
|
4963
5307
|
if (backend === "api") {
|
|
4964
5308
|
const mirrored = await ensureGeminiFileMirror(stored.fileId);
|
|
4965
|
-
parts.push(createPartFromUri(mirrored.uri, mirrored.mimeType));
|
|
5309
|
+
parts.push(createPartFromUri(mirrored.uri, mirrored.mimeType, mediaResolution));
|
|
4966
5310
|
} else {
|
|
4967
5311
|
const mirrored = await ensureVertexFileMirror(stored.fileId);
|
|
4968
5312
|
parts.push({
|
|
4969
5313
|
fileData: {
|
|
4970
5314
|
fileUri: mirrored.fileUri,
|
|
4971
5315
|
mimeType: mirrored.mimeType
|
|
4972
|
-
}
|
|
5316
|
+
},
|
|
5317
|
+
...mediaResolution ? { mediaResolution: { level: mediaResolution } } : {}
|
|
4973
5318
|
});
|
|
4974
5319
|
}
|
|
4975
5320
|
continue;
|
|
@@ -5432,7 +5777,7 @@ function resolveTextContents(input) {
|
|
|
5432
5777
|
}
|
|
5433
5778
|
return contents;
|
|
5434
5779
|
}
|
|
5435
|
-
function toOpenAiInput(contents) {
|
|
5780
|
+
function toOpenAiInput(contents, options) {
|
|
5436
5781
|
const OPENAI_ROLE_FROM_LLM = {
|
|
5437
5782
|
user: "user",
|
|
5438
5783
|
assistant: "assistant",
|
|
@@ -5440,6 +5785,8 @@ function toOpenAiInput(contents) {
|
|
|
5440
5785
|
developer: "developer",
|
|
5441
5786
|
tool: "assistant"
|
|
5442
5787
|
};
|
|
5788
|
+
const defaultMediaResolution = options?.defaultMediaResolution;
|
|
5789
|
+
const model = options?.model;
|
|
5443
5790
|
return contents.map((content) => {
|
|
5444
5791
|
const parts = [];
|
|
5445
5792
|
for (const part of content.parts) {
|
|
@@ -5454,7 +5801,7 @@ function toOpenAiInput(contents) {
|
|
|
5454
5801
|
const imagePart = {
|
|
5455
5802
|
type: "input_image",
|
|
5456
5803
|
image_url: dataUrl,
|
|
5457
|
-
detail:
|
|
5804
|
+
detail: toOpenAiImageDetail(defaultMediaResolution, model)
|
|
5458
5805
|
};
|
|
5459
5806
|
setInlineAttachmentFilename(
|
|
5460
5807
|
imagePart,
|
|
@@ -5471,11 +5818,15 @@ function toOpenAiInput(contents) {
|
|
|
5471
5818
|
break;
|
|
5472
5819
|
}
|
|
5473
5820
|
case "input_image": {
|
|
5821
|
+
const mediaResolution = resolveEffectiveMediaResolution(
|
|
5822
|
+
part.detail,
|
|
5823
|
+
defaultMediaResolution
|
|
5824
|
+
);
|
|
5474
5825
|
const imagePart = {
|
|
5475
5826
|
type: "input_image",
|
|
5476
5827
|
...part.file_id ? { file_id: part.file_id } : {},
|
|
5477
5828
|
...part.image_url ? { image_url: part.image_url } : {},
|
|
5478
|
-
detail:
|
|
5829
|
+
detail: toOpenAiImageDetail(mediaResolution, model)
|
|
5479
5830
|
};
|
|
5480
5831
|
if (part.filename) {
|
|
5481
5832
|
setInlineAttachmentFilename(imagePart, part.filename);
|
|
@@ -5508,9 +5859,11 @@ function toOpenAiInput(contents) {
|
|
|
5508
5859
|
};
|
|
5509
5860
|
});
|
|
5510
5861
|
}
|
|
5511
|
-
function toChatGptInput(contents) {
|
|
5862
|
+
function toChatGptInput(contents, options) {
|
|
5512
5863
|
const instructionsParts = [];
|
|
5513
5864
|
const input = [];
|
|
5865
|
+
const defaultMediaResolution = options?.defaultMediaResolution;
|
|
5866
|
+
const model = options?.model;
|
|
5514
5867
|
for (const content of contents) {
|
|
5515
5868
|
if (content.role === "system" || content.role === "developer") {
|
|
5516
5869
|
for (const part of content.parts) {
|
|
@@ -5546,7 +5899,7 @@ function toChatGptInput(contents) {
|
|
|
5546
5899
|
parts.push({
|
|
5547
5900
|
type: "input_image",
|
|
5548
5901
|
image_url: dataUrl,
|
|
5549
|
-
detail:
|
|
5902
|
+
detail: toOpenAiImageDetail(defaultMediaResolution, model)
|
|
5550
5903
|
});
|
|
5551
5904
|
} else {
|
|
5552
5905
|
parts.push({
|
|
@@ -5560,14 +5913,19 @@ function toChatGptInput(contents) {
|
|
|
5560
5913
|
}
|
|
5561
5914
|
break;
|
|
5562
5915
|
}
|
|
5563
|
-
case "input_image":
|
|
5916
|
+
case "input_image": {
|
|
5917
|
+
const mediaResolution = resolveEffectiveMediaResolution(
|
|
5918
|
+
part.detail,
|
|
5919
|
+
defaultMediaResolution
|
|
5920
|
+
);
|
|
5564
5921
|
parts.push({
|
|
5565
5922
|
type: "input_image",
|
|
5566
5923
|
...part.file_id ? { file_id: part.file_id } : {},
|
|
5567
5924
|
...part.image_url ? { image_url: part.image_url } : {},
|
|
5568
|
-
detail:
|
|
5925
|
+
detail: toOpenAiImageDetail(mediaResolution, model)
|
|
5569
5926
|
});
|
|
5570
5927
|
break;
|
|
5928
|
+
}
|
|
5571
5929
|
case "input_file":
|
|
5572
5930
|
parts.push({
|
|
5573
5931
|
type: "input_file",
|
|
@@ -5960,6 +6318,9 @@ function isLlmToolOutputContentItem(value) {
|
|
|
5960
6318
|
return false;
|
|
5961
6319
|
}
|
|
5962
6320
|
}
|
|
6321
|
+
if (value.detail !== void 0 && value.detail !== null && !isLlmMediaResolution(value.detail)) {
|
|
6322
|
+
return false;
|
|
6323
|
+
}
|
|
5963
6324
|
return value.image_url !== void 0 || value.file_id !== void 0;
|
|
5964
6325
|
}
|
|
5965
6326
|
if (itemType === "input_file") {
|
|
@@ -5974,17 +6335,30 @@ function isLlmToolOutputContentItem(value) {
|
|
|
5974
6335
|
}
|
|
5975
6336
|
return false;
|
|
5976
6337
|
}
|
|
5977
|
-
function toOpenAiToolOutput(value) {
|
|
6338
|
+
function toOpenAiToolOutput(value, options) {
|
|
6339
|
+
const normalizeImageItem = (item) => {
|
|
6340
|
+
if (item.type !== "input_image") {
|
|
6341
|
+
return item;
|
|
6342
|
+
}
|
|
6343
|
+
const mediaResolution = resolveEffectiveMediaResolution(
|
|
6344
|
+
item.detail,
|
|
6345
|
+
options?.defaultMediaResolution
|
|
6346
|
+
);
|
|
6347
|
+
return {
|
|
6348
|
+
...item,
|
|
6349
|
+
detail: toOpenAiImageDetail(mediaResolution, options?.model)
|
|
6350
|
+
};
|
|
6351
|
+
};
|
|
5978
6352
|
if (isLlmToolOutputContentItem(value)) {
|
|
5979
|
-
return [value];
|
|
6353
|
+
return [normalizeImageItem(value)];
|
|
5980
6354
|
}
|
|
5981
6355
|
if (Array.isArray(value) && value.every((item) => isLlmToolOutputContentItem(item))) {
|
|
5982
|
-
return value;
|
|
6356
|
+
return value.map((item) => normalizeImageItem(item));
|
|
5983
6357
|
}
|
|
5984
6358
|
return mergeToolOutput(value);
|
|
5985
6359
|
}
|
|
5986
|
-
function toChatGptToolOutput(value) {
|
|
5987
|
-
const toolOutput = toOpenAiToolOutput(value);
|
|
6360
|
+
function toChatGptToolOutput(value, options) {
|
|
6361
|
+
const toolOutput = toOpenAiToolOutput(value, options);
|
|
5988
6362
|
if (typeof toolOutput === "string") {
|
|
5989
6363
|
return toolOutput;
|
|
5990
6364
|
}
|
|
@@ -5996,7 +6370,12 @@ function toChatGptToolOutput(value) {
|
|
|
5996
6370
|
type: "input_image",
|
|
5997
6371
|
...item.file_id ? { file_id: item.file_id } : {},
|
|
5998
6372
|
...item.image_url ? { image_url: item.image_url } : {},
|
|
5999
|
-
...item.detail ? {
|
|
6373
|
+
...item.detail ? {
|
|
6374
|
+
detail: toOpenAiImageDetail(
|
|
6375
|
+
resolveEffectiveMediaResolution(item.detail, options?.defaultMediaResolution),
|
|
6376
|
+
options?.model
|
|
6377
|
+
)
|
|
6378
|
+
} : {}
|
|
6000
6379
|
};
|
|
6001
6380
|
});
|
|
6002
6381
|
}
|
|
@@ -6167,9 +6546,6 @@ async function maybeSpillToolOutputItem(item, toolName, options) {
|
|
|
6167
6546
|
return item;
|
|
6168
6547
|
}
|
|
6169
6548
|
async function maybeSpillToolOutput(value, toolName, options) {
|
|
6170
|
-
if (options?.provider === "chatgpt") {
|
|
6171
|
-
return value;
|
|
6172
|
-
}
|
|
6173
6549
|
if (typeof value === "string") {
|
|
6174
6550
|
if (options?.force !== true && Buffer5.byteLength(value, "utf8") <= TOOL_OUTPUT_SPILL_THRESHOLD_BYTES) {
|
|
6175
6551
|
return value;
|
|
@@ -6255,34 +6631,41 @@ async function maybeSpillCombinedToolCallOutputs(callResults, options) {
|
|
|
6255
6631
|
})
|
|
6256
6632
|
);
|
|
6257
6633
|
}
|
|
6258
|
-
function buildGeminiToolOutputMediaPart(item) {
|
|
6634
|
+
function buildGeminiToolOutputMediaPart(item, options) {
|
|
6259
6635
|
if (item.type === "input_image") {
|
|
6636
|
+
const mediaResolution = resolveEffectiveMediaResolution(
|
|
6637
|
+
item.detail,
|
|
6638
|
+
options?.defaultMediaResolution
|
|
6639
|
+
);
|
|
6640
|
+
const geminiPartMediaResolution = toGeminiPartMediaResolution(mediaResolution);
|
|
6260
6641
|
if (typeof item.file_id === "string" && item.file_id.trim().length > 0) {
|
|
6261
|
-
return
|
|
6262
|
-
|
|
6263
|
-
|
|
6264
|
-
|
|
6265
|
-
|
|
6266
|
-
};
|
|
6642
|
+
return createPartFromUri(
|
|
6643
|
+
buildCanonicalGeminiFileUri(item.file_id),
|
|
6644
|
+
inferToolOutputMimeTypeFromFilename(item.filename) ?? "application/octet-stream",
|
|
6645
|
+
geminiPartMediaResolution
|
|
6646
|
+
);
|
|
6267
6647
|
}
|
|
6268
6648
|
if (typeof item.image_url !== "string" || item.image_url.trim().length === 0) {
|
|
6269
6649
|
return null;
|
|
6270
6650
|
}
|
|
6271
6651
|
const parsed = parseDataUrlPayload(item.image_url);
|
|
6272
6652
|
if (parsed) {
|
|
6273
|
-
const part = createPartFromBase64(
|
|
6653
|
+
const part = createPartFromBase64(
|
|
6654
|
+
parsed.dataBase64,
|
|
6655
|
+
parsed.mimeType,
|
|
6656
|
+
geminiPartMediaResolution
|
|
6657
|
+
);
|
|
6274
6658
|
const displayName = item.filename?.trim();
|
|
6275
6659
|
if (displayName && part.inlineData) {
|
|
6276
6660
|
part.inlineData.displayName = displayName;
|
|
6277
6661
|
}
|
|
6278
6662
|
return part;
|
|
6279
6663
|
}
|
|
6280
|
-
return
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
};
|
|
6664
|
+
return createPartFromUri(
|
|
6665
|
+
item.image_url,
|
|
6666
|
+
inferToolOutputMimeTypeFromFilename(item.filename) ?? "application/octet-stream",
|
|
6667
|
+
geminiPartMediaResolution
|
|
6668
|
+
);
|
|
6286
6669
|
}
|
|
6287
6670
|
if (item.type === "input_file") {
|
|
6288
6671
|
if (typeof item.file_id === "string" && item.file_id.trim().length > 0) {
|
|
@@ -6360,7 +6743,9 @@ function buildGeminiFunctionResponseParts(options) {
|
|
|
6360
6743
|
}
|
|
6361
6744
|
const responseOutput = outputItems.map((item) => toGeminiToolOutputPlaceholder(item));
|
|
6362
6745
|
const responseParts = outputItems.flatMap((item) => {
|
|
6363
|
-
const mediaPart = buildGeminiToolOutputMediaPart(item
|
|
6746
|
+
const mediaPart = buildGeminiToolOutputMediaPart(item, {
|
|
6747
|
+
defaultMediaResolution: options.defaultMediaResolution
|
|
6748
|
+
});
|
|
6364
6749
|
return mediaPart ? [mediaPart] : [];
|
|
6365
6750
|
});
|
|
6366
6751
|
const responsePayload = { output: responseOutput };
|
|
@@ -7127,6 +7512,7 @@ function startLlmCallLoggerFromContents(options) {
|
|
|
7127
7512
|
...options.request.imageAspectRatio ? { imageAspectRatio: options.request.imageAspectRatio } : {},
|
|
7128
7513
|
...options.request.imageSize ? { imageSize: options.request.imageSize } : {},
|
|
7129
7514
|
...options.request.thinkingLevel ? { thinkingLevel: options.request.thinkingLevel } : {},
|
|
7515
|
+
...options.request.mediaResolution ? { mediaResolution: options.request.mediaResolution } : {},
|
|
7130
7516
|
...options.request.openAiTextFormat ? { openAiTextFormat: sanitiseLogValue(options.request.openAiTextFormat) } : {},
|
|
7131
7517
|
...getCurrentToolCallContext() ? { toolContext: getCurrentToolCallContext() } : {}
|
|
7132
7518
|
},
|
|
@@ -7237,7 +7623,13 @@ async function runTextCall(params) {
|
|
|
7237
7623
|
const { result } = await collectFileUploadMetrics(async () => {
|
|
7238
7624
|
try {
|
|
7239
7625
|
if (provider === "openai") {
|
|
7240
|
-
const openAiInput = await maybePrepareOpenAiPromptInput(
|
|
7626
|
+
const openAiInput = await maybePrepareOpenAiPromptInput(
|
|
7627
|
+
toOpenAiInput(contents, {
|
|
7628
|
+
defaultMediaResolution: request.mediaResolution,
|
|
7629
|
+
model: request.model
|
|
7630
|
+
}),
|
|
7631
|
+
{ model: request.model, provider: "openai" }
|
|
7632
|
+
);
|
|
7241
7633
|
const openAiTools = toOpenAiTools(request.tools);
|
|
7242
7634
|
const reasoningEffort = resolveOpenAiReasoningEffort(
|
|
7243
7635
|
modelForProvider,
|
|
@@ -7311,7 +7703,14 @@ async function runTextCall(params) {
|
|
|
7311
7703
|
}
|
|
7312
7704
|
}, modelForProvider);
|
|
7313
7705
|
} else if (provider === "chatgpt") {
|
|
7314
|
-
const chatGptInput = toChatGptInput(contents
|
|
7706
|
+
const chatGptInput = toChatGptInput(contents, {
|
|
7707
|
+
defaultMediaResolution: request.mediaResolution,
|
|
7708
|
+
model: request.model
|
|
7709
|
+
});
|
|
7710
|
+
const preparedChatGptInput = await maybePrepareOpenAiPromptInput(chatGptInput.input, {
|
|
7711
|
+
model: request.model,
|
|
7712
|
+
provider: "chatgpt"
|
|
7713
|
+
});
|
|
7315
7714
|
const reasoningEffort = resolveOpenAiReasoningEffort(request.model, request.thinkingLevel);
|
|
7316
7715
|
const openAiTools = toOpenAiTools(request.tools);
|
|
7317
7716
|
const requestPayload = {
|
|
@@ -7320,7 +7719,7 @@ async function runTextCall(params) {
|
|
|
7320
7719
|
stream: true,
|
|
7321
7720
|
...providerInfo.serviceTier ? { service_tier: providerInfo.serviceTier } : {},
|
|
7322
7721
|
instructions: chatGptInput.instructions ?? "You are a helpful assistant.",
|
|
7323
|
-
input:
|
|
7722
|
+
input: preparedChatGptInput,
|
|
7324
7723
|
include: ["reasoning.encrypted_content"],
|
|
7325
7724
|
reasoning: {
|
|
7326
7725
|
effort: toOpenAiReasoningEffort(reasoningEffort),
|
|
@@ -7408,12 +7807,18 @@ async function runTextCall(params) {
|
|
|
7408
7807
|
}, modelForProvider);
|
|
7409
7808
|
} else {
|
|
7410
7809
|
const geminiContents = await maybePrepareGeminiPromptContents(
|
|
7411
|
-
contents.map(
|
|
7810
|
+
contents.map(
|
|
7811
|
+
(content2) => convertLlmContentToGeminiContent(content2, {
|
|
7812
|
+
defaultMediaResolution: request.mediaResolution
|
|
7813
|
+
})
|
|
7814
|
+
)
|
|
7412
7815
|
);
|
|
7413
7816
|
const thinkingConfig = resolveGeminiThinkingConfig(modelForProvider, request.thinkingLevel);
|
|
7817
|
+
const mediaResolution = toGeminiMediaResolution(request.mediaResolution);
|
|
7414
7818
|
const config = {
|
|
7415
7819
|
maxOutputTokens: 32e3,
|
|
7416
7820
|
...thinkingConfig ? { thinkingConfig } : {},
|
|
7821
|
+
...mediaResolution ? { mediaResolution } : {},
|
|
7417
7822
|
...request.responseMimeType ? { responseMimeType: request.responseMimeType } : {},
|
|
7418
7823
|
...request.responseJsonSchema ? { responseJsonSchema: request.responseJsonSchema } : {},
|
|
7419
7824
|
...request.responseModalities ? { responseModalities: Array.from(request.responseModalities) } : {},
|
|
@@ -8091,7 +8496,10 @@ async function runToolLoop(request) {
|
|
|
8091
8496
|
summary: "detailed"
|
|
8092
8497
|
};
|
|
8093
8498
|
let previousResponseId;
|
|
8094
|
-
let input = toOpenAiInput(contents
|
|
8499
|
+
let input = toOpenAiInput(contents, {
|
|
8500
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8501
|
+
model: request.model
|
|
8502
|
+
});
|
|
8095
8503
|
for (let stepIndex = 0; stepIndex < maxSteps; stepIndex += 1) {
|
|
8096
8504
|
const turn = stepIndex + 1;
|
|
8097
8505
|
const stepStartedAtMs = Date.now();
|
|
@@ -8118,7 +8526,10 @@ async function runToolLoop(request) {
|
|
|
8118
8526
|
let reasoningSummary = "";
|
|
8119
8527
|
let stepToolCallText;
|
|
8120
8528
|
let stepToolCallPayload;
|
|
8121
|
-
const preparedInput = await maybePrepareOpenAiPromptInput(input
|
|
8529
|
+
const preparedInput = await maybePrepareOpenAiPromptInput(input, {
|
|
8530
|
+
model: request.model,
|
|
8531
|
+
provider: "openai"
|
|
8532
|
+
});
|
|
8122
8533
|
const stepRequestPayload = {
|
|
8123
8534
|
model: providerInfo.model,
|
|
8124
8535
|
input: preparedInput,
|
|
@@ -8249,7 +8660,10 @@ async function runToolLoop(request) {
|
|
|
8249
8660
|
const stepToolCalls = [];
|
|
8250
8661
|
if (responseToolCalls.length === 0) {
|
|
8251
8662
|
const steeringInput2 = steeringInternal?.drainPendingContents() ?? [];
|
|
8252
|
-
const steeringItems2 = steeringInput2.length > 0 ? toOpenAiInput(steeringInput2
|
|
8663
|
+
const steeringItems2 = steeringInput2.length > 0 ? toOpenAiInput(steeringInput2, {
|
|
8664
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8665
|
+
model: request.model
|
|
8666
|
+
}) : [];
|
|
8253
8667
|
finalText = responseText;
|
|
8254
8668
|
finalThoughts = reasoningSummary;
|
|
8255
8669
|
const stepCompletedAtMs2 = Date.now();
|
|
@@ -8380,13 +8794,19 @@ async function runToolLoop(request) {
|
|
|
8380
8794
|
toolOutputs.push({
|
|
8381
8795
|
type: "custom_tool_call_output",
|
|
8382
8796
|
call_id: entry.call.call_id,
|
|
8383
|
-
output: toOpenAiToolOutput(outputPayload
|
|
8797
|
+
output: toOpenAiToolOutput(outputPayload, {
|
|
8798
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8799
|
+
model: request.model
|
|
8800
|
+
})
|
|
8384
8801
|
});
|
|
8385
8802
|
} else {
|
|
8386
8803
|
toolOutputs.push({
|
|
8387
8804
|
type: "function_call_output",
|
|
8388
8805
|
call_id: entry.call.call_id,
|
|
8389
|
-
output: toOpenAiToolOutput(outputPayload
|
|
8806
|
+
output: toOpenAiToolOutput(outputPayload, {
|
|
8807
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8808
|
+
model: request.model
|
|
8809
|
+
})
|
|
8390
8810
|
});
|
|
8391
8811
|
}
|
|
8392
8812
|
}
|
|
@@ -8411,7 +8831,10 @@ async function runToolLoop(request) {
|
|
|
8411
8831
|
timing
|
|
8412
8832
|
});
|
|
8413
8833
|
const steeringInput = steeringInternal?.drainPendingContents() ?? [];
|
|
8414
|
-
const steeringItems = steeringInput.length > 0 ? toOpenAiInput(steeringInput
|
|
8834
|
+
const steeringItems = steeringInput.length > 0 ? toOpenAiInput(steeringInput, {
|
|
8835
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8836
|
+
model: request.model
|
|
8837
|
+
}) : [];
|
|
8415
8838
|
stepCallLogger?.complete({
|
|
8416
8839
|
responseText,
|
|
8417
8840
|
toolCallText: stepToolCallText,
|
|
@@ -8456,7 +8879,10 @@ async function runToolLoop(request) {
|
|
|
8456
8879
|
const openAiNativeTools = toOpenAiTools(request.modelTools);
|
|
8457
8880
|
const openAiTools = openAiNativeTools ? [...openAiNativeTools, ...openAiAgentTools] : [...openAiAgentTools];
|
|
8458
8881
|
const reasoningEffort = resolveOpenAiReasoningEffort(request.model, request.thinkingLevel);
|
|
8459
|
-
const toolLoopInput = toChatGptInput(contents
|
|
8882
|
+
const toolLoopInput = toChatGptInput(contents, {
|
|
8883
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8884
|
+
model: request.model
|
|
8885
|
+
});
|
|
8460
8886
|
const conversationId = `tool-loop-${randomBytes(8).toString("hex")}`;
|
|
8461
8887
|
const promptCacheKey = conversationId;
|
|
8462
8888
|
let input = [...toolLoopInput.input];
|
|
@@ -8472,6 +8898,10 @@ async function runToolLoop(request) {
|
|
|
8472
8898
|
let reasoningSummaryText = "";
|
|
8473
8899
|
let stepToolCallText;
|
|
8474
8900
|
let stepToolCallPayload;
|
|
8901
|
+
const preparedInput = await maybePrepareOpenAiPromptInput(input, {
|
|
8902
|
+
model: request.model,
|
|
8903
|
+
provider: "chatgpt"
|
|
8904
|
+
});
|
|
8475
8905
|
const markFirstModelEvent = () => {
|
|
8476
8906
|
if (firstModelEventAtMs === void 0) {
|
|
8477
8907
|
firstModelEventAtMs = Date.now();
|
|
@@ -8483,7 +8913,7 @@ async function runToolLoop(request) {
|
|
|
8483
8913
|
stream: true,
|
|
8484
8914
|
...providerInfo.serviceTier ? { service_tier: providerInfo.serviceTier } : {},
|
|
8485
8915
|
instructions: toolLoopInput.instructions ?? "You are a helpful assistant.",
|
|
8486
|
-
input,
|
|
8916
|
+
input: preparedInput,
|
|
8487
8917
|
prompt_cache_key: promptCacheKey,
|
|
8488
8918
|
include: ["reasoning.encrypted_content"],
|
|
8489
8919
|
tools: openAiTools,
|
|
@@ -8560,7 +8990,10 @@ async function runToolLoop(request) {
|
|
|
8560
8990
|
stepToolCallText = serialiseLogArtifactText(stepToolCallPayload);
|
|
8561
8991
|
if (responseToolCalls.length === 0) {
|
|
8562
8992
|
const steeringInput2 = steeringInternal?.drainPendingContents() ?? [];
|
|
8563
|
-
const steeringItems2 = steeringInput2.length > 0 ? toChatGptInput(steeringInput2
|
|
8993
|
+
const steeringItems2 = steeringInput2.length > 0 ? toChatGptInput(steeringInput2, {
|
|
8994
|
+
defaultMediaResolution: request.mediaResolution,
|
|
8995
|
+
model: request.model
|
|
8996
|
+
}).input : [];
|
|
8564
8997
|
finalText = responseText;
|
|
8565
8998
|
finalThoughts = reasoningSummaryText;
|
|
8566
8999
|
const stepCompletedAtMs2 = Date.now();
|
|
@@ -8692,7 +9125,10 @@ async function runToolLoop(request) {
|
|
|
8692
9125
|
toolOutputs.push({
|
|
8693
9126
|
type: "custom_tool_call_output",
|
|
8694
9127
|
call_id: entry.ids.callId,
|
|
8695
|
-
output: toChatGptToolOutput(outputPayload
|
|
9128
|
+
output: toChatGptToolOutput(outputPayload, {
|
|
9129
|
+
defaultMediaResolution: request.mediaResolution,
|
|
9130
|
+
model: request.model
|
|
9131
|
+
})
|
|
8696
9132
|
});
|
|
8697
9133
|
} else {
|
|
8698
9134
|
toolOutputs.push({
|
|
@@ -8706,7 +9142,10 @@ async function runToolLoop(request) {
|
|
|
8706
9142
|
toolOutputs.push({
|
|
8707
9143
|
type: "function_call_output",
|
|
8708
9144
|
call_id: entry.ids.callId,
|
|
8709
|
-
output: toChatGptToolOutput(outputPayload
|
|
9145
|
+
output: toChatGptToolOutput(outputPayload, {
|
|
9146
|
+
defaultMediaResolution: request.mediaResolution,
|
|
9147
|
+
model: request.model
|
|
9148
|
+
})
|
|
8710
9149
|
});
|
|
8711
9150
|
}
|
|
8712
9151
|
}
|
|
@@ -8730,7 +9169,10 @@ async function runToolLoop(request) {
|
|
|
8730
9169
|
timing
|
|
8731
9170
|
});
|
|
8732
9171
|
const steeringInput = steeringInternal?.drainPendingContents() ?? [];
|
|
8733
|
-
const steeringItems = steeringInput.length > 0 ? toChatGptInput(steeringInput
|
|
9172
|
+
const steeringItems = steeringInput.length > 0 ? toChatGptInput(steeringInput, {
|
|
9173
|
+
defaultMediaResolution: request.mediaResolution,
|
|
9174
|
+
model: request.model
|
|
9175
|
+
}).input : [];
|
|
8734
9176
|
stepCallLogger?.complete({
|
|
8735
9177
|
responseText,
|
|
8736
9178
|
toolCallText: stepToolCallText,
|
|
@@ -9061,7 +9503,11 @@ async function runToolLoop(request) {
|
|
|
9061
9503
|
const geminiFunctionTools = buildGeminiFunctionDeclarations(request.tools);
|
|
9062
9504
|
const geminiNativeTools = toGeminiTools(request.modelTools);
|
|
9063
9505
|
const geminiTools = geminiNativeTools ? geminiNativeTools.concat(geminiFunctionTools) : geminiFunctionTools;
|
|
9064
|
-
const geminiContents = contents.map(
|
|
9506
|
+
const geminiContents = contents.map(
|
|
9507
|
+
(content) => convertLlmContentToGeminiContent(content, {
|
|
9508
|
+
defaultMediaResolution: request.mediaResolution
|
|
9509
|
+
})
|
|
9510
|
+
);
|
|
9065
9511
|
for (let stepIndex = 0; stepIndex < maxSteps; stepIndex += 1) {
|
|
9066
9512
|
const turn = stepIndex + 1;
|
|
9067
9513
|
const stepStartedAtMs = Date.now();
|
|
@@ -9079,6 +9525,7 @@ async function runToolLoop(request) {
|
|
|
9079
9525
|
}
|
|
9080
9526
|
};
|
|
9081
9527
|
const thinkingConfig = resolveGeminiThinkingConfig(request.model, request.thinkingLevel);
|
|
9528
|
+
const mediaResolution = toGeminiMediaResolution(request.mediaResolution);
|
|
9082
9529
|
const config = {
|
|
9083
9530
|
maxOutputTokens: 32e3,
|
|
9084
9531
|
tools: geminiTools,
|
|
@@ -9087,7 +9534,8 @@ async function runToolLoop(request) {
|
|
|
9087
9534
|
mode: FunctionCallingConfigMode.VALIDATED
|
|
9088
9535
|
}
|
|
9089
9536
|
},
|
|
9090
|
-
...thinkingConfig ? { thinkingConfig } : {}
|
|
9537
|
+
...thinkingConfig ? { thinkingConfig } : {},
|
|
9538
|
+
...mediaResolution ? { mediaResolution } : {}
|
|
9091
9539
|
};
|
|
9092
9540
|
const onEvent = request.onEvent;
|
|
9093
9541
|
const preparedGeminiContents = await maybePrepareGeminiPromptContents(geminiContents);
|
|
@@ -9243,7 +9691,13 @@ async function runToolLoop(request) {
|
|
|
9243
9691
|
} else if (response.responseText.length > 0) {
|
|
9244
9692
|
geminiContents.push({ role: "model", parts: [{ text: response.responseText }] });
|
|
9245
9693
|
}
|
|
9246
|
-
geminiContents.push(
|
|
9694
|
+
geminiContents.push(
|
|
9695
|
+
...steeringInput2.map(
|
|
9696
|
+
(content) => convertLlmContentToGeminiContent(content, {
|
|
9697
|
+
defaultMediaResolution: request.mediaResolution
|
|
9698
|
+
})
|
|
9699
|
+
)
|
|
9700
|
+
);
|
|
9247
9701
|
continue;
|
|
9248
9702
|
}
|
|
9249
9703
|
const toolCalls = [];
|
|
@@ -9335,7 +9789,8 @@ async function runToolLoop(request) {
|
|
|
9335
9789
|
...buildGeminiFunctionResponseParts({
|
|
9336
9790
|
toolName: entry.toolName,
|
|
9337
9791
|
callId: entry.call.id,
|
|
9338
|
-
outputPayload
|
|
9792
|
+
outputPayload,
|
|
9793
|
+
defaultMediaResolution: request.mediaResolution
|
|
9339
9794
|
})
|
|
9340
9795
|
);
|
|
9341
9796
|
}
|
|
@@ -9380,7 +9835,13 @@ async function runToolLoop(request) {
|
|
|
9380
9835
|
geminiContents.push({ role: "user", parts: responseParts });
|
|
9381
9836
|
const steeringInput = steeringInternal?.drainPendingContents() ?? [];
|
|
9382
9837
|
if (steeringInput.length > 0) {
|
|
9383
|
-
geminiContents.push(
|
|
9838
|
+
geminiContents.push(
|
|
9839
|
+
...steeringInput.map(
|
|
9840
|
+
(content) => convertLlmContentToGeminiContent(content, {
|
|
9841
|
+
defaultMediaResolution: request.mediaResolution
|
|
9842
|
+
})
|
|
9843
|
+
)
|
|
9844
|
+
);
|
|
9384
9845
|
}
|
|
9385
9846
|
} catch (error) {
|
|
9386
9847
|
stepCallLogger?.fail(error, {
|
|
@@ -9636,7 +10097,7 @@ async function generateImages(request) {
|
|
|
9636
10097
|
}
|
|
9637
10098
|
return image;
|
|
9638
10099
|
})(),
|
|
9639
|
-
model: "gpt-5.
|
|
10100
|
+
model: "gpt-5.4-mini"
|
|
9640
10101
|
})
|
|
9641
10102
|
)
|
|
9642
10103
|
);
|
|
@@ -9942,7 +10403,6 @@ function resolveSubagentToolConfig(selection, currentDepth) {
|
|
|
9942
10403
|
maxWaitTimeoutMs,
|
|
9943
10404
|
promptPattern,
|
|
9944
10405
|
...instructions ? { instructions } : {},
|
|
9945
|
-
...config.model ? { model: config.model } : {},
|
|
9946
10406
|
...maxSteps ? { maxSteps } : {},
|
|
9947
10407
|
inheritTools: config.inheritTools !== false,
|
|
9948
10408
|
inheritFilesystemTool: config.inheritFilesystemTool !== false
|
|
@@ -9994,7 +10454,6 @@ function createSubagentToolController(options) {
|
|
|
9994
10454
|
`Subagent depth limit reached (${options.config.maxDepth}). Cannot spawn at depth ${childDepth}.`
|
|
9995
10455
|
);
|
|
9996
10456
|
}
|
|
9997
|
-
const model = options.config.model ?? options.parentModel;
|
|
9998
10457
|
const id = `agent_${randomBytes2(6).toString("hex")}`;
|
|
9999
10458
|
const now = Date.now();
|
|
10000
10459
|
const { roleName, roleInstructions } = resolveAgentType(input.agent_type);
|
|
@@ -10014,7 +10473,7 @@ function createSubagentToolController(options) {
|
|
|
10014
10473
|
const agent = {
|
|
10015
10474
|
id,
|
|
10016
10475
|
depth: childDepth,
|
|
10017
|
-
model,
|
|
10476
|
+
model: options.parentModel,
|
|
10018
10477
|
...nickname ? { nickname } : {},
|
|
10019
10478
|
agentRole: roleName,
|
|
10020
10479
|
status: "idle",
|
|
@@ -11769,7 +12228,8 @@ async function viewImageCodex(input, options) {
|
|
|
11769
12228
|
return [
|
|
11770
12229
|
{
|
|
11771
12230
|
type: "input_image",
|
|
11772
|
-
image_url: `data:${mimeType};base64,${bytes.toString("base64")}
|
|
12231
|
+
image_url: `data:${mimeType};base64,${bytes.toString("base64")}`,
|
|
12232
|
+
...options.mediaResolution ? { detail: options.mediaResolution } : {}
|
|
11773
12233
|
}
|
|
11774
12234
|
];
|
|
11775
12235
|
}
|
|
@@ -12449,7 +12909,11 @@ async function runAgentLoopInternal(request, context) {
|
|
|
12449
12909
|
const toolLoopRequestWithSteering = toolLoopRequest.steering === steeringChannel ? toolLoopRequest : { ...toolLoopRequest, steering: steeringChannel };
|
|
12450
12910
|
const filesystemSelection = filesystemTool ?? filesystem_tool;
|
|
12451
12911
|
const subagentSelection = subagentTool ?? subagent_tool ?? subagents;
|
|
12452
|
-
const filesystemTools = resolveFilesystemTools(
|
|
12912
|
+
const filesystemTools = resolveFilesystemTools(
|
|
12913
|
+
request.model,
|
|
12914
|
+
filesystemSelection,
|
|
12915
|
+
request.mediaResolution
|
|
12916
|
+
);
|
|
12453
12917
|
const resolvedSubagentConfig = resolveSubagentToolConfig(subagentSelection, context.depth);
|
|
12454
12918
|
const subagentController = createSubagentController({
|
|
12455
12919
|
runId,
|
|
@@ -12601,24 +13065,47 @@ async function runAgentLoopInternal(request, context) {
|
|
|
12601
13065
|
await subagentController?.closeAll();
|
|
12602
13066
|
}
|
|
12603
13067
|
}
|
|
12604
|
-
function resolveFilesystemTools(model, selection) {
|
|
13068
|
+
function resolveFilesystemTools(model, selection, defaultMediaResolution) {
|
|
13069
|
+
const withDefaultMediaResolution = (options) => {
|
|
13070
|
+
if (defaultMediaResolution === void 0) {
|
|
13071
|
+
return options;
|
|
13072
|
+
}
|
|
13073
|
+
return {
|
|
13074
|
+
mediaResolution: defaultMediaResolution,
|
|
13075
|
+
...options ?? {}
|
|
13076
|
+
};
|
|
13077
|
+
};
|
|
12605
13078
|
if (selection === void 0 || selection === false) {
|
|
12606
13079
|
return {};
|
|
12607
13080
|
}
|
|
12608
13081
|
if (selection === true) {
|
|
12609
|
-
return createFilesystemToolSetForModel(model,
|
|
13082
|
+
return createFilesystemToolSetForModel(model, withDefaultMediaResolution(void 0) ?? {});
|
|
12610
13083
|
}
|
|
12611
13084
|
if (typeof selection === "string") {
|
|
12612
|
-
return createFilesystemToolSetForModel(model, selection);
|
|
13085
|
+
return createFilesystemToolSetForModel(model, selection, withDefaultMediaResolution(void 0));
|
|
12613
13086
|
}
|
|
12614
13087
|
if (selection.enabled === false) {
|
|
12615
13088
|
return {};
|
|
12616
13089
|
}
|
|
12617
13090
|
if (selection.options && selection.profile !== void 0) {
|
|
12618
|
-
return createFilesystemToolSetForModel(
|
|
13091
|
+
return createFilesystemToolSetForModel(
|
|
13092
|
+
model,
|
|
13093
|
+
selection.profile,
|
|
13094
|
+
withDefaultMediaResolution(selection.options)
|
|
13095
|
+
);
|
|
12619
13096
|
}
|
|
12620
13097
|
if (selection.options) {
|
|
12621
|
-
return createFilesystemToolSetForModel(
|
|
13098
|
+
return createFilesystemToolSetForModel(
|
|
13099
|
+
model,
|
|
13100
|
+
withDefaultMediaResolution(selection.options) ?? {}
|
|
13101
|
+
);
|
|
13102
|
+
}
|
|
13103
|
+
if (defaultMediaResolution !== void 0) {
|
|
13104
|
+
return createFilesystemToolSetForModel(
|
|
13105
|
+
model,
|
|
13106
|
+
selection.profile ?? "auto",
|
|
13107
|
+
withDefaultMediaResolution(void 0)
|
|
13108
|
+
);
|
|
12622
13109
|
}
|
|
12623
13110
|
return createFilesystemToolSetForModel(model, selection.profile ?? "auto");
|
|
12624
13111
|
}
|
|
@@ -12641,7 +13128,7 @@ function createSubagentController(params) {
|
|
|
12641
13128
|
return createSubagentToolController({
|
|
12642
13129
|
config: params.resolvedSubagentConfig,
|
|
12643
13130
|
parentDepth: params.depth,
|
|
12644
|
-
parentModel: params.
|
|
13131
|
+
parentModel: params.model,
|
|
12645
13132
|
forkContextMessages: normalizeForkContextMessages(params.toolLoopRequest.input),
|
|
12646
13133
|
onBackgroundMessage: (message) => {
|
|
12647
13134
|
params.steering?.append({ role: "user", content: message });
|
|
@@ -12661,6 +13148,7 @@ function createSubagentController(params) {
|
|
|
12661
13148
|
modelTools: params.toolLoopRequest.modelTools,
|
|
12662
13149
|
maxSteps: subagentRequest.maxSteps,
|
|
12663
13150
|
thinkingLevel: params.toolLoopRequest.thinkingLevel,
|
|
13151
|
+
mediaResolution: params.toolLoopRequest.mediaResolution,
|
|
12664
13152
|
signal: subagentRequest.signal
|
|
12665
13153
|
},
|
|
12666
13154
|
{
|