github-router 0.3.249 → 0.3.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/background.js +8 -1
- package/dist/browser-ext/manifest.json +1 -1
- package/dist/engine-dOYoMQis.js +2 -0
- package/dist/main.js +26 -6
- package/dist/main.js.map +1 -1
- package/dist/{peer-mcp-personas-DJzLpfDJ.js → peer-mcp-personas-BvommiSI.js} +845 -98
- package/dist/peer-mcp-personas-BvommiSI.js.map +1 -0
- package/package.json +1 -1
- package/dist/engine-BuuHUb4t.js +0 -2
- package/dist/peer-mcp-personas-DJzLpfDJ.js.map +0 -1
|
@@ -11,7 +11,7 @@ import * as path$1 from "node:path";
|
|
|
11
11
|
import path, { dirname, join } from "node:path";
|
|
12
12
|
import process$1 from "node:process";
|
|
13
13
|
import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
|
|
14
|
-
import { chmodSync, closeSync, cpSync, existsSync, mkdirSync, openSync, promises, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
14
|
+
import { chmodSync, closeSync, constants, cpSync, existsSync, mkdirSync, openSync, promises, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
15
15
|
import { fileURLToPath } from "node:url";
|
|
16
16
|
import { Agent, ProxyAgent } from "undici";
|
|
17
17
|
import { z } from "zod";
|
|
@@ -14245,6 +14245,9 @@ function isSensitivePath(absPath, workspaceAbs) {
|
|
|
14245
14245
|
*/
|
|
14246
14246
|
function rejectWindowsHostilePath(raw) {
|
|
14247
14247
|
if (/^[\\/]{2}/.test(raw)) return "rejected: UNC or device path";
|
|
14248
|
+
if (IS_WINDOWS) {
|
|
14249
|
+
if ((/^[A-Za-z]:/.test(raw) ? raw.slice(2) : raw).includes(":")) return "rejected: alternate data stream";
|
|
14250
|
+
}
|
|
14248
14251
|
if (/^[A-Za-z]:(?![\\/])/.test(raw)) return "rejected: drive-relative path";
|
|
14249
14252
|
return null;
|
|
14250
14253
|
}
|
|
@@ -18750,6 +18753,103 @@ function preflightUrlPolicy(toolName, args) {
|
|
|
18750
18753
|
return checkUrlPolicy(args.url);
|
|
18751
18754
|
}
|
|
18752
18755
|
//#endregion
|
|
18756
|
+
//#region src/lib/attachments.ts
|
|
18757
|
+
/**
|
|
18758
|
+
* Build a text + image tool result.
|
|
18759
|
+
*
|
|
18760
|
+
* TEXT FIRST, IMAGE SECOND — and this ordering is load-bearing for a reason
|
|
18761
|
+
* worth recording honestly: it is test-debt accommodation, not a design
|
|
18762
|
+
* argument. 75 assertions across the suite read `result.content[0].text`, and
|
|
18763
|
+
* putting the image first would break every one of them for no gain. Nothing
|
|
18764
|
+
* in the MCP spec or in any known consumer is sensitive to block order, so the
|
|
18765
|
+
* cheap accommodation wins. If that ever stops being true, codemod the
|
|
18766
|
+
* assertions rather than contorting the wire shape.
|
|
18767
|
+
*/
|
|
18768
|
+
function mcpTextAndImage(text, image) {
|
|
18769
|
+
return { content: [{
|
|
18770
|
+
type: "text",
|
|
18771
|
+
text
|
|
18772
|
+
}, {
|
|
18773
|
+
type: "image",
|
|
18774
|
+
data: image.data,
|
|
18775
|
+
mimeType: image.mimeType
|
|
18776
|
+
}] };
|
|
18777
|
+
}
|
|
18778
|
+
/**
|
|
18779
|
+
* Media types that appear in some Copilot model's
|
|
18780
|
+
* `capabilities.limits.vision.supported_media_types`. This is the UNION across
|
|
18781
|
+
* models, not a per-model allowlist — an individual model accepts a subset
|
|
18782
|
+
* (gemini takes heic/heif but not gif; the gpt and claude lanes take gif but
|
|
18783
|
+
* not heic). Per-model narrowing is the outbound validator's job; this set only
|
|
18784
|
+
* answers "is this an image shape any Copilot model could take at all".
|
|
18785
|
+
*/
|
|
18786
|
+
const SUPPORTED_IMAGE_MIME_TYPES = /* @__PURE__ */ new Set([
|
|
18787
|
+
"image/jpeg",
|
|
18788
|
+
"image/png",
|
|
18789
|
+
"image/webp",
|
|
18790
|
+
"image/gif",
|
|
18791
|
+
"image/heic",
|
|
18792
|
+
"image/heif"
|
|
18793
|
+
]);
|
|
18794
|
+
function startsWith(bytes, offset, sig) {
|
|
18795
|
+
if (bytes.length < offset + sig.length) return false;
|
|
18796
|
+
for (const [i, element] of sig.entries()) if (bytes[offset + i] !== element) return false;
|
|
18797
|
+
return true;
|
|
18798
|
+
}
|
|
18799
|
+
function startsWithAscii(bytes, offset, ascii) {
|
|
18800
|
+
return startsWith(bytes, offset, [...ascii].map((c) => c.codePointAt(0) ?? 0));
|
|
18801
|
+
}
|
|
18802
|
+
const PNG_SIGNATURE = [
|
|
18803
|
+
137,
|
|
18804
|
+
80,
|
|
18805
|
+
78,
|
|
18806
|
+
71,
|
|
18807
|
+
13,
|
|
18808
|
+
10,
|
|
18809
|
+
26,
|
|
18810
|
+
10
|
|
18811
|
+
];
|
|
18812
|
+
/**
|
|
18813
|
+
* Identify an image by its leading bytes, returning a media type from
|
|
18814
|
+
* `SUPPORTED_IMAGE_MIME_TYPES` or `undefined` when the bytes are not a
|
|
18815
|
+
* supported image.
|
|
18816
|
+
*
|
|
18817
|
+
* `undefined` is a deliberate two-in-one answer: "not an image" and "an image
|
|
18818
|
+
* kind nothing upstream accepts" are the same outcome for every caller here,
|
|
18819
|
+
* because both mean the bytes must not be sent as an image block.
|
|
18820
|
+
*
|
|
18821
|
+
* Truncated and malformed inputs return `undefined` rather than throwing — a
|
|
18822
|
+
* caller reading an arbitrary file off disk must not be able to crash a tool
|
|
18823
|
+
* dispatch with a 3-byte file.
|
|
18824
|
+
*/
|
|
18825
|
+
function detectImageMimeType(bytes) {
|
|
18826
|
+
if (startsWith(bytes, 0, [
|
|
18827
|
+
255,
|
|
18828
|
+
216,
|
|
18829
|
+
255
|
|
18830
|
+
])) return bytes[3] === 247 ? void 0 : "image/jpeg";
|
|
18831
|
+
if (startsWith(bytes, 0, PNG_SIGNATURE)) return bytes.length >= 16 && new DataView(bytes.buffer, bytes.byteOffset).getUint32(PNG_SIGNATURE.length) === 13 && startsWithAscii(bytes, 12, "IHDR") ? "image/png" : void 0;
|
|
18832
|
+
if (startsWithAscii(bytes, 0, "GIF8")) return "image/gif";
|
|
18833
|
+
if (startsWithAscii(bytes, 0, "RIFF") && startsWithAscii(bytes, 8, "WEBP")) return "image/webp";
|
|
18834
|
+
if (startsWithAscii(bytes, 4, "ftyp")) {
|
|
18835
|
+
const brand = new TextDecoder().decode(bytes.subarray(8, 12));
|
|
18836
|
+
if (brand === "heic" || brand === "heix" || brand === "hevc" || brand === "hevx") return "image/heic";
|
|
18837
|
+
if (brand === "mif1" || brand === "msf1" || brand === "heim" || brand === "heis") return "image/heif";
|
|
18838
|
+
}
|
|
18839
|
+
}
|
|
18840
|
+
/** Decode base64 for size-validation purposes; see the notes inside. */
|
|
18841
|
+
function decodeBase64Strict(input) {
|
|
18842
|
+
const compact = input.replaceAll(/[\s]/g, "");
|
|
18843
|
+
if (compact.length === 0) return null;
|
|
18844
|
+
if (!/^[A-Za-z0-9+/]*={0,2}$/.test(compact)) return null;
|
|
18845
|
+
const remainder = compact.length % 4;
|
|
18846
|
+
if (remainder === 1) return null;
|
|
18847
|
+
const padded = remainder === 0 ? compact : compact + "=".repeat(4 - remainder);
|
|
18848
|
+
const buf = Buffer.from(padded, "base64");
|
|
18849
|
+
if (buf.toString("base64") !== padded) return null;
|
|
18850
|
+
return new Uint8Array(buf);
|
|
18851
|
+
}
|
|
18852
|
+
//#endregion
|
|
18753
18853
|
//#region src/lib/browser-mcp/dispatch.ts
|
|
18754
18854
|
/**
|
|
18755
18855
|
* Tools whose dispatch counts as a mutating user action for pacing
|
|
@@ -18969,6 +19069,52 @@ async function bridgeCall(endpoint, tool, args, timeoutMs, signal) {
|
|
|
18969
19069
|
});
|
|
18970
19070
|
});
|
|
18971
19071
|
}
|
|
19072
|
+
/**
|
|
19073
|
+
* Turn a bridge envelope that carries captured pixels into a real MCP image
|
|
19074
|
+
* block, or return `null` when it doesn't (the overwhelmingly common case —
|
|
19075
|
+
* every non-screenshot tool).
|
|
19076
|
+
*
|
|
19077
|
+
* The extension returns `{contentType, dataBase64}` for `browser_screenshot`.
|
|
19078
|
+
* That used to be `JSON.stringify`'d into a text block, which meant the calling
|
|
19079
|
+
* model received base64 characters it could not interpret, at roughly 130x the
|
|
19080
|
+
* token cost of the same image sent natively (base64 tokenizes at ~1.46
|
|
19081
|
+
* chars/token under o200k, so a 200 KB PNG is ~187k tokens — enough to be
|
|
19082
|
+
* rejected outright by a 200k-context model).
|
|
19083
|
+
*
|
|
19084
|
+
* Two deliberate choices:
|
|
19085
|
+
*
|
|
19086
|
+
* - The media type is SNIFFED from the bytes, not taken from the envelope's
|
|
19087
|
+
* `contentType`. A declared type is an assertion; the payload is what
|
|
19088
|
+
* upstream will actually try to decode. The declared value is preserved in
|
|
19089
|
+
* the text block so a mismatch stays debuggable.
|
|
19090
|
+
* - The text block keeps every envelope field EXCEPT `dataBase64`, and gains
|
|
19091
|
+
* a decoded `bytes` count. Callers that parsed the JSON metadata keep
|
|
19092
|
+
* working; the payload simply stops being duplicated in a form nothing
|
|
19093
|
+
* could read.
|
|
19094
|
+
*
|
|
19095
|
+
* Bytes that don't decode strictly, or that aren't a supported image, fall
|
|
19096
|
+
* through to `null` and take the plain-text path — a malformed capture must
|
|
19097
|
+
* not become an image block that upstream then rejects.
|
|
19098
|
+
*/
|
|
19099
|
+
function imageEnvelope(data) {
|
|
19100
|
+
if (data === null || typeof data !== "object" || Array.isArray(data)) return null;
|
|
19101
|
+
const record = data;
|
|
19102
|
+
const encoded = record.dataBase64;
|
|
19103
|
+
if (typeof encoded !== "string" || encoded.length === 0) return null;
|
|
19104
|
+
const bytes = decodeBase64Strict(encoded);
|
|
19105
|
+
if (!bytes) return null;
|
|
19106
|
+
const mimeType = detectImageMimeType(bytes);
|
|
19107
|
+
if (!mimeType) return null;
|
|
19108
|
+
const meta = {
|
|
19109
|
+
bytes: bytes.length,
|
|
19110
|
+
mimeType
|
|
19111
|
+
};
|
|
19112
|
+
for (const [key, value] of Object.entries(record)) if (key !== "dataBase64") meta[key] = value;
|
|
19113
|
+
return mcpTextAndImage(JSON.stringify(meta, null, 2), {
|
|
19114
|
+
data: encoded,
|
|
19115
|
+
mimeType
|
|
19116
|
+
});
|
|
19117
|
+
}
|
|
18972
19118
|
function blockedUrlEnvelope(reason) {
|
|
18973
19119
|
return {
|
|
18974
19120
|
content: [{
|
|
@@ -19034,7 +19180,6 @@ async function dispatchBrowserTool(tool, args, signal, opts = {}) {
|
|
|
19034
19180
|
token: ready.token
|
|
19035
19181
|
}, tool, args, callerTimeout, signal);
|
|
19036
19182
|
if (resp.ok) {
|
|
19037
|
-
const text = typeof resp.data === "string" ? resp.data : JSON.stringify(resp.data, null, 2);
|
|
19038
19183
|
logAudit$1({
|
|
19039
19184
|
tool,
|
|
19040
19185
|
argsBytes: argsByteSize(args),
|
|
@@ -19042,9 +19187,9 @@ async function dispatchBrowserTool(tool, args, signal, opts = {}) {
|
|
|
19042
19187
|
profile: typeof args.profile === "string" ? args.profile : "isolated",
|
|
19043
19188
|
result: "ok"
|
|
19044
19189
|
});
|
|
19045
|
-
return { content: [{
|
|
19190
|
+
return imageEnvelope(resp.data) ?? { content: [{
|
|
19046
19191
|
type: "text",
|
|
19047
|
-
text
|
|
19192
|
+
text: typeof resp.data === "string" ? resp.data : JSON.stringify(resp.data, null, 2)
|
|
19048
19193
|
}] };
|
|
19049
19194
|
}
|
|
19050
19195
|
logAudit$1({
|
|
@@ -19674,6 +19819,169 @@ function currentInFlight() {
|
|
|
19674
19819
|
return inFlight$2;
|
|
19675
19820
|
}
|
|
19676
19821
|
//#endregion
|
|
19822
|
+
//#region src/lib/vision-preflight.ts
|
|
19823
|
+
/**
|
|
19824
|
+
* Outbound vision preflight.
|
|
19825
|
+
*
|
|
19826
|
+
* Converts what would otherwise be an opaque upstream `400` into a local,
|
|
19827
|
+
* actionable rejection — the same trade `predictedWindowOverflow` in
|
|
19828
|
+
* `src/routes/mcp/handler.ts` already makes for prompt tokens, and the reason
|
|
19829
|
+
* this file exists at all: `capabilities.limits.vision` advertises three fields
|
|
19830
|
+
* (`max_prompt_images`, `max_prompt_image_size`, `supported_media_types`) and
|
|
19831
|
+
* until now the proxy read exactly one of them, in the `models` pretty-printer.
|
|
19832
|
+
*
|
|
19833
|
+
* WHY ONE CHOKEPOINT, NOT PER-ADAPTER CHECKS
|
|
19834
|
+
*
|
|
19835
|
+
* Images reach the wire through more paths than is obvious: a top-level user
|
|
19836
|
+
* block, a block nested inside a `tool_result`, the synthetic follow-up user
|
|
19837
|
+
* message the shim emits because a tool-output item cannot carry images, images
|
|
19838
|
+
* already present in replayed conversation history, and peer-critic
|
|
19839
|
+
* attachments. Validating in each adapter means each adapter has to
|
|
19840
|
+
* rediscover every one of those shapes, and the ones it forgets fail silently.
|
|
19841
|
+
* So validation runs ONCE, on the fully assembled payload, immediately before
|
|
19842
|
+
* transport serialization — `createResponses` and `createChatCompletions` — and
|
|
19843
|
+
* is therefore total by construction.
|
|
19844
|
+
*
|
|
19845
|
+
* POLICY ON MISSING METADATA
|
|
19846
|
+
*
|
|
19847
|
+
* Deliberately NOT fail-open. "We don't know the limit" is not "there is no
|
|
19848
|
+
* limit"; treating it as the latter reproduces the exact opaque-400 this
|
|
19849
|
+
* module exists to remove. The three cases are distinguished:
|
|
19850
|
+
*
|
|
19851
|
+
* - model absent from the catalog entirely → ALLOW. We have no basis for a
|
|
19852
|
+
* judgement, upstream remains authoritative, and blocking would break
|
|
19853
|
+
* custom catalogs and offline tests. This mirrors `modelSupportsEndpoint`.
|
|
19854
|
+
* - model present, `supports.vision` not true → REJECT, naming the model.
|
|
19855
|
+
* - model present, vision supported, limits absent → CONSERVATIVE FLOOR
|
|
19856
|
+
* (`FLOOR_MAX_IMAGES` / `FLOOR_MAX_IMAGE_BYTES`), and the message says so,
|
|
19857
|
+
* so the caller can tell a real limit from an assumed one.
|
|
19858
|
+
*
|
|
19859
|
+
* Error strings are returned to callers and end up in front of a model, so they
|
|
19860
|
+
* name the model, the numbers, and the fix — never a stack, a local path, or a
|
|
19861
|
+
* raw catalog object.
|
|
19862
|
+
*/
|
|
19863
|
+
/**
|
|
19864
|
+
* Floor applied when a model advertises vision but publishes no limits. Every
|
|
19865
|
+
* vision-capable model in the live catalog reports at least 1 image and a
|
|
19866
|
+
* 3 MiB ceiling, so this is the observed minimum rather than a guess.
|
|
19867
|
+
*/
|
|
19868
|
+
const FLOOR_MAX_IMAGES = 1;
|
|
19869
|
+
const FLOOR_MAX_IMAGE_BYTES = 3145728;
|
|
19870
|
+
/**
|
|
19871
|
+
* Split a `data:<mime>;base64,<payload>` URI. Returns `null` for any other URL
|
|
19872
|
+
* shape (including a plain remote `https://` reference, which the caller
|
|
19873
|
+
* handles separately because its bytes are not ours to inspect).
|
|
19874
|
+
*/
|
|
19875
|
+
function parseDataUrl(url) {
|
|
19876
|
+
const match = /^data:([^;,]+);base64,(.*)$/s.exec(url);
|
|
19877
|
+
if (!match) return null;
|
|
19878
|
+
return {
|
|
19879
|
+
mimeType: match[1],
|
|
19880
|
+
base64: match[2]
|
|
19881
|
+
};
|
|
19882
|
+
}
|
|
19883
|
+
function reject(message) {
|
|
19884
|
+
return {
|
|
19885
|
+
ok: false,
|
|
19886
|
+
message
|
|
19887
|
+
};
|
|
19888
|
+
}
|
|
19889
|
+
/**
|
|
19890
|
+
* Validate every image on an outbound request against the resolved model's
|
|
19891
|
+
* advertised vision capability.
|
|
19892
|
+
*
|
|
19893
|
+
* Remote-URL images are counted toward the cardinality limit but not size- or
|
|
19894
|
+
* type-checked: the bytes live on someone else's server, so any local claim
|
|
19895
|
+
* about them would be fiction. Upstream stays authoritative for those.
|
|
19896
|
+
*/
|
|
19897
|
+
function checkOutboundImages(modelId, images) {
|
|
19898
|
+
if (images.length === 0) return { ok: true };
|
|
19899
|
+
const model = state.models?.data.find((m) => m.id === modelId);
|
|
19900
|
+
if (!model) return { ok: true };
|
|
19901
|
+
if ((model.capabilities?.supports)?.vision !== true) return reject(`Model ${modelId} does not support image input. Send text only, or select a vision-capable model.`);
|
|
19902
|
+
const limits = model.capabilities?.limits?.vision;
|
|
19903
|
+
const maxImages = limits?.max_prompt_images ?? FLOOR_MAX_IMAGES;
|
|
19904
|
+
const maxBytes = limits?.max_prompt_image_size ?? FLOOR_MAX_IMAGE_BYTES;
|
|
19905
|
+
const assumed = limits?.max_prompt_images === void 0 ? " (assumed; the model publishes no image limits)" : "";
|
|
19906
|
+
if (images.length > maxImages) return reject(`Model ${modelId} accepts at most ${maxImages} image(s) per request${assumed}, but this request carries ${images.length}. Send fewer images, or use a model with a higher image limit.`);
|
|
19907
|
+
const allowedTypes = limits?.supported_media_types;
|
|
19908
|
+
for (const [index, image] of images.entries()) {
|
|
19909
|
+
if (image.url !== void 0 && image.base64 === void 0) continue;
|
|
19910
|
+
const position = `image ${index + 1} of ${images.length}`;
|
|
19911
|
+
if (image.declaredMimeType === void 0 || image.declaredMimeType.length === 0) return reject(`${position} has no media type. Anthropic's \`source.media_type\` (or a \`data:<mime>;base64,\` prefix) is required — it used to be defaulted, which silently asserted a type the model may not accept.`);
|
|
19912
|
+
if (image.base64 === void 0) continue;
|
|
19913
|
+
const bytes = decodeBase64Strict(image.base64);
|
|
19914
|
+
if (!bytes) return reject(`${position} is not valid base64 and cannot be sent.`);
|
|
19915
|
+
if (bytes.length > maxBytes) return reject(`${position} is ${bytes.length} bytes, over model ${modelId}'s ${maxBytes}-byte limit${assumed}. Re-capture at a smaller scale or lower quality.`);
|
|
19916
|
+
const actual = detectImageMimeType(bytes);
|
|
19917
|
+
if (!actual) return reject(`${position} is declared ${image.declaredMimeType} but its bytes are not a supported image (jpeg, png, webp, gif, heic, heif).`);
|
|
19918
|
+
if (actual !== image.declaredMimeType) return reject(`${position} is declared ${image.declaredMimeType} but its bytes are ${actual}. Send the correct media type.`);
|
|
19919
|
+
if (allowedTypes && allowedTypes.length > 0 && !allowedTypes.includes(actual)) return reject(`${position} is ${actual}, which model ${modelId} does not accept. Supported: ${allowedTypes.join(", ")}.`);
|
|
19920
|
+
}
|
|
19921
|
+
return { ok: true };
|
|
19922
|
+
}
|
|
19923
|
+
/** Collect the images on an assembled Copilot `/responses` payload. */
|
|
19924
|
+
function imagesInResponsesPayload(input) {
|
|
19925
|
+
const images = [];
|
|
19926
|
+
if (!Array.isArray(input)) return images;
|
|
19927
|
+
for (const item of input) {
|
|
19928
|
+
if (item === null || typeof item !== "object") continue;
|
|
19929
|
+
const content = item.content;
|
|
19930
|
+
if (!Array.isArray(content)) continue;
|
|
19931
|
+
for (const part of content) {
|
|
19932
|
+
if (part === null || typeof part !== "object") continue;
|
|
19933
|
+
const p = part;
|
|
19934
|
+
if (p.type !== "input_image" || typeof p.image_url !== "string") continue;
|
|
19935
|
+
images.push(fromUrl(p.image_url));
|
|
19936
|
+
}
|
|
19937
|
+
}
|
|
19938
|
+
return images;
|
|
19939
|
+
}
|
|
19940
|
+
/** Collect the images on an assembled Copilot `/chat/completions` payload. */
|
|
19941
|
+
function imagesInChatPayload(messages) {
|
|
19942
|
+
const images = [];
|
|
19943
|
+
if (!Array.isArray(messages)) return images;
|
|
19944
|
+
for (const message of messages) {
|
|
19945
|
+
if (message === null || typeof message !== "object") continue;
|
|
19946
|
+
const content = message.content;
|
|
19947
|
+
if (!Array.isArray(content)) continue;
|
|
19948
|
+
for (const part of content) {
|
|
19949
|
+
if (part === null || typeof part !== "object") continue;
|
|
19950
|
+
const p = part;
|
|
19951
|
+
if (p.type !== "image_url") continue;
|
|
19952
|
+
const url = p.image_url?.url;
|
|
19953
|
+
if (typeof url !== "string") continue;
|
|
19954
|
+
images.push(fromUrl(url));
|
|
19955
|
+
}
|
|
19956
|
+
}
|
|
19957
|
+
return images;
|
|
19958
|
+
}
|
|
19959
|
+
function fromUrl(url) {
|
|
19960
|
+
const parsed = parseDataUrl(url);
|
|
19961
|
+
return parsed ? {
|
|
19962
|
+
base64: parsed.base64,
|
|
19963
|
+
declaredMimeType: parsed.mimeType
|
|
19964
|
+
} : { url };
|
|
19965
|
+
}
|
|
19966
|
+
/**
|
|
19967
|
+
* Run the preflight and throw a client-visible 400 when it fails.
|
|
19968
|
+
*
|
|
19969
|
+
* Shaped as an `HTTPError` carrying a synthetic `Response` so it flows through
|
|
19970
|
+
* the existing `forwardError` path and reaches the client as a normal
|
|
19971
|
+
* Anthropic-format `invalid_request_error` — the same envelope an upstream
|
|
19972
|
+
* rejection would produce, except it names the actual problem and no upstream
|
|
19973
|
+
* request was made. Tests assert that second property: a preflight failure must
|
|
19974
|
+
* cost zero upstream calls.
|
|
19975
|
+
*/
|
|
19976
|
+
function assertOutboundImagesOk(modelId, images) {
|
|
19977
|
+
const verdict = checkOutboundImages(modelId, images);
|
|
19978
|
+
if (verdict.ok) return;
|
|
19979
|
+
throw new HTTPError(verdict.message, new Response(JSON.stringify({ error: { message: verdict.message } }), {
|
|
19980
|
+
status: 400,
|
|
19981
|
+
headers: { "content-type": "application/json" }
|
|
19982
|
+
}));
|
|
19983
|
+
}
|
|
19984
|
+
//#endregion
|
|
19677
19985
|
//#region src/lib/diagnose-response.ts
|
|
19678
19986
|
const PREVIEW_LIMIT = 200;
|
|
19679
19987
|
async function parseJsonOrDiagnose(response, routePath) {
|
|
@@ -19795,6 +20103,7 @@ async function readResponseBodyCapped(response, routePath, capBytes = MAX_RESPON
|
|
|
19795
20103
|
const createChatCompletions = async (payload, modelHeaders, callerSignal, retryTransient = false) => {
|
|
19796
20104
|
if (!state.copilotToken) throw new Error("Copilot token not found");
|
|
19797
20105
|
const enableVision = payload.messages.some((x) => typeof x.content !== "string" && x.content?.some((x) => x.type === "image_url"));
|
|
20106
|
+
if (enableVision) assertOutboundImagesOk(payload.model, imagesInChatPayload(payload.messages));
|
|
19798
20107
|
const isAgentCall = payload.messages.some((msg) => ["assistant", "tool"].includes(msg.role));
|
|
19799
20108
|
const url = `${copilotBaseUrl(state)}/chat/completions`;
|
|
19800
20109
|
const doFetch = () => {
|
|
@@ -19857,6 +20166,7 @@ const createChatCompletions = async (payload, modelHeaders, callerSignal, retryT
|
|
|
19857
20166
|
const createResponses = async (payload, modelHeaders, callerSignal, retryTransient = false) => {
|
|
19858
20167
|
if (!state.copilotToken) throw new Error("Copilot token not found");
|
|
19859
20168
|
const enableVision = detectVision(payload.input);
|
|
20169
|
+
if (enableVision) assertOutboundImagesOk(payload.model, imagesInResponsesPayload(payload.input));
|
|
19860
20170
|
const isAgentCall = detectAgentCall(payload.input);
|
|
19861
20171
|
const url = `${copilotBaseUrl(state)}/responses`;
|
|
19862
20172
|
const doFetch = () => {
|
|
@@ -20878,7 +21188,7 @@ const BROWSER_TOOLS = Object.freeze([
|
|
|
20878
21188
|
},
|
|
20879
21189
|
{
|
|
20880
21190
|
toolNameHttp: "browser_screenshot",
|
|
20881
|
-
description: "Captures a screenshot of the visible area of a tab
|
|
21191
|
+
description: "Captures a screenshot of the visible area of a tab and returns it as an image you can actually look at, plus a small text envelope of capture metadata. It takes a tab id, an optional format (PNG default, JPEG for smaller bytes), and an optional JPEG quality. The tab must be active in its window, so this tool auto-activates the tab if needed and that changes which tab is focused. If a capture is rejected for exceeding a model's image-size limit, the most reliable lever is a smaller browser window; PNG is usually SMALLER than JPEG for ordinary UI and text pages, so switching format is not a dependable way to shrink one. JPEG plus a low quality helps mainly for photographic or dense-colour content. Use screenshot for visual layout, canvas, SVG, maps, or image-only regions; prefer browser_observe when page text and actionable state are enough.",
|
|
20882
21192
|
inputSchema: {
|
|
20883
21193
|
type: "object",
|
|
20884
21194
|
required: ["tabId"],
|
|
@@ -20892,6 +21202,12 @@ const BROWSER_TOOLS = Object.freeze([
|
|
|
20892
21202
|
type: "string",
|
|
20893
21203
|
enum: ["png", "jpeg"],
|
|
20894
21204
|
description: "Image format for the returned screenshot. Default 'png'; use 'jpeg' when smaller image bytes are preferable."
|
|
21205
|
+
},
|
|
21206
|
+
quality: {
|
|
21207
|
+
type: "number",
|
|
21208
|
+
minimum: 1,
|
|
21209
|
+
maximum: 100,
|
|
21210
|
+
description: "JPEG quality 1-100 (ignored for PNG). Only meaningful with format='jpeg'. Note JPEG is often LARGER than PNG for flat UI screenshots — measured on a plain documentation page, PNG 35553 bytes vs JPEG q30 40599 bytes at the same viewport — so reach for it for photographic content, not as a general size lever."
|
|
20895
21211
|
}
|
|
20896
21212
|
}
|
|
20897
21213
|
},
|
|
@@ -21622,21 +21938,12 @@ async function runAtomicIntentStep(tabId, intent, value, signal) {
|
|
|
21622
21938
|
error: "no text match; screenshot for visual fallback failed",
|
|
21623
21939
|
picked
|
|
21624
21940
|
}, true);
|
|
21625
|
-
const
|
|
21626
|
-
|
|
21627
|
-
try {
|
|
21628
|
-
shot = shotText ? JSON.parse(shotText) : {};
|
|
21629
|
-
} catch {
|
|
21630
|
-
return toolEnvelope({
|
|
21631
|
-
ok: false,
|
|
21632
|
-
error: "no text match; screenshot envelope unparseable"
|
|
21633
|
-
}, true);
|
|
21634
|
-
}
|
|
21635
|
-
if (!shot.contentType || !shot.dataBase64) return toolEnvelope({
|
|
21941
|
+
const shot = shotEnv.content.find((b) => b.type === "image");
|
|
21942
|
+
if (!shot) return toolEnvelope({
|
|
21636
21943
|
ok: false,
|
|
21637
|
-
error: "no text match; screenshot
|
|
21944
|
+
error: "no text match; screenshot returned no image"
|
|
21638
21945
|
}, true);
|
|
21639
|
-
const visual = await pickElementVisual(shot.
|
|
21946
|
+
const visual = await pickElementVisual(shot.data, shot.mimeType, intent, surfaces, signal);
|
|
21640
21947
|
if (visual.confidence < .5) return toolEnvelope({
|
|
21641
21948
|
ok: false,
|
|
21642
21949
|
error: "no element matched intent (text + visual)",
|
|
@@ -24232,10 +24539,14 @@ async function runChatAttempt(stream, payload, opts, signal, state) {
|
|
|
24232
24539
|
let nextContentIndex = 0;
|
|
24233
24540
|
let activeTextIndex = null;
|
|
24234
24541
|
const toolPiIndexByOAI = /* @__PURE__ */ new Map();
|
|
24542
|
+
let sawDone = false;
|
|
24235
24543
|
for await (const evt of sseStream) {
|
|
24236
24544
|
const data = evt?.data;
|
|
24237
24545
|
if (data == null) continue;
|
|
24238
|
-
if (data === "[DONE]")
|
|
24546
|
+
if (data === "[DONE]") {
|
|
24547
|
+
sawDone = true;
|
|
24548
|
+
break;
|
|
24549
|
+
}
|
|
24239
24550
|
let chunk;
|
|
24240
24551
|
try {
|
|
24241
24552
|
chunk = JSON.parse(data);
|
|
@@ -24328,12 +24639,26 @@ async function runChatAttempt(stream, payload, opts, signal, state) {
|
|
|
24328
24639
|
if (choice.finish_reason) accum.finishReason = choice.finish_reason;
|
|
24329
24640
|
}
|
|
24330
24641
|
if (!state.active) return;
|
|
24642
|
+
if (!sawDone && accum.finishReason == null) {
|
|
24643
|
+
if (activeTextIndex != null) stream.push({
|
|
24644
|
+
type: "text_end",
|
|
24645
|
+
contentIndex: activeTextIndex,
|
|
24646
|
+
content: joinTextChunks(accum, activeTextIndex),
|
|
24647
|
+
partial: buildPartial(resolved, accum)
|
|
24648
|
+
});
|
|
24649
|
+
pushTerminalError(stream, resolved, /* @__PURE__ */ new Error("chat stream ended without a [DONE] sentinel (truncated)"));
|
|
24650
|
+
return;
|
|
24651
|
+
}
|
|
24331
24652
|
if (activeTextIndex != null) stream.push({
|
|
24332
24653
|
type: "text_end",
|
|
24333
24654
|
contentIndex: activeTextIndex,
|
|
24334
24655
|
content: joinTextChunks(accum, activeTextIndex),
|
|
24335
24656
|
partial: buildPartial(resolved, accum)
|
|
24336
24657
|
});
|
|
24658
|
+
if (accum.finishReason === "content_filter") {
|
|
24659
|
+
pushTerminalError(stream, resolved, /* @__PURE__ */ new Error("upstream content filter blocked the response"));
|
|
24660
|
+
return;
|
|
24661
|
+
}
|
|
24337
24662
|
for (const block of accum.blocks) {
|
|
24338
24663
|
if (block.kind !== "tool") continue;
|
|
24339
24664
|
const entry = accum.toolByIndex.get(block.contentIndex);
|
|
@@ -24359,10 +24684,7 @@ function buildPayload(context, resolved) {
|
|
|
24359
24684
|
role: "system",
|
|
24360
24685
|
content: context.systemPrompt
|
|
24361
24686
|
});
|
|
24362
|
-
|
|
24363
|
-
const oai = translateMessage(m);
|
|
24364
|
-
if (oai) messages.push(oai);
|
|
24365
|
-
}
|
|
24687
|
+
messages.push(...translateMessages(context.messages));
|
|
24366
24688
|
const tools = translateTools(context.tools);
|
|
24367
24689
|
const payload = {
|
|
24368
24690
|
model: resolved.modelId,
|
|
@@ -24376,11 +24698,55 @@ function buildPayload(context, resolved) {
|
|
|
24376
24698
|
if (resolved.thinking !== "off") payload.reasoning_effort = resolved.thinking;
|
|
24377
24699
|
return payload;
|
|
24378
24700
|
}
|
|
24379
|
-
|
|
24380
|
-
|
|
24381
|
-
|
|
24382
|
-
|
|
24383
|
-
|
|
24701
|
+
/**
|
|
24702
|
+
* Pi messages → wire messages, with tool-result images hoisted to the END of
|
|
24703
|
+
* each contiguous run of tool results.
|
|
24704
|
+
*
|
|
24705
|
+
* A tool-output wire item cannot carry an image, so images have to ride in a
|
|
24706
|
+
* separate user message. The obvious implementation — fan each `toolResult` out
|
|
24707
|
+
* to `[tool, user]` independently — is WRONG for parallel tool calls: it yields
|
|
24708
|
+
* `tool A, user A, tool B, user B`, and every provider requires the tool
|
|
24709
|
+
* messages answering one assistant turn to be CONTIGUOUS. The interjected user
|
|
24710
|
+
* message orphans the following tool message and the request is rejected.
|
|
24711
|
+
*
|
|
24712
|
+
* So images accumulate across the run and flush once, after the last tool
|
|
24713
|
+
* message of that run: `tool A, tool B, user(imgA, imgB)`.
|
|
24714
|
+
*
|
|
24715
|
+
* This runs at REQUEST-ASSEMBLY time over `context.messages` and never writes
|
|
24716
|
+
* back to worker state, so the synthetic message is ephemeral — an image is not
|
|
24717
|
+
* appended to the transcript and re-sent on every later turn.
|
|
24718
|
+
*/
|
|
24719
|
+
function translateMessages(messages) {
|
|
24720
|
+
const out = [];
|
|
24721
|
+
let pending = [];
|
|
24722
|
+
const flushImages = () => {
|
|
24723
|
+
if (pending.length === 0) return;
|
|
24724
|
+
out.push({
|
|
24725
|
+
role: "user",
|
|
24726
|
+
content: pending.map((img) => ({
|
|
24727
|
+
type: "image_url",
|
|
24728
|
+
image_url: { url: `data:${img.mimeType};base64,${img.data}` }
|
|
24729
|
+
}))
|
|
24730
|
+
});
|
|
24731
|
+
pending = [];
|
|
24732
|
+
};
|
|
24733
|
+
for (const m of messages) {
|
|
24734
|
+
if (m.role === "toolResult") {
|
|
24735
|
+
const images = imagePartsOf(m.content);
|
|
24736
|
+
out.push({
|
|
24737
|
+
role: "tool",
|
|
24738
|
+
tool_call_id: m.toolCallId,
|
|
24739
|
+
content: toolResultText(m, images.length > 0)
|
|
24740
|
+
});
|
|
24741
|
+
pending.push(...images);
|
|
24742
|
+
continue;
|
|
24743
|
+
}
|
|
24744
|
+
flushImages();
|
|
24745
|
+
if (m.role === "user") out.push(translateUser(m));
|
|
24746
|
+
else if (m.role === "assistant") out.push(translateAssistant(m));
|
|
24747
|
+
}
|
|
24748
|
+
flushImages();
|
|
24749
|
+
return out;
|
|
24384
24750
|
}
|
|
24385
24751
|
function translateUser(m) {
|
|
24386
24752
|
if (typeof m.content === "string") return {
|
|
@@ -24423,13 +24789,6 @@ function translateAssistant(m) {
|
|
|
24423
24789
|
if (toolCalls.length > 0) out.tool_calls = toolCalls;
|
|
24424
24790
|
return out;
|
|
24425
24791
|
}
|
|
24426
|
-
function translateToolResult(m) {
|
|
24427
|
-
return {
|
|
24428
|
-
role: "tool",
|
|
24429
|
-
tool_call_id: m.toolCallId,
|
|
24430
|
-
content: joinTextParts(m.content)
|
|
24431
|
-
};
|
|
24432
|
-
}
|
|
24433
24792
|
function translateTools(tools) {
|
|
24434
24793
|
if (!tools || tools.length === 0) return void 0;
|
|
24435
24794
|
return tools.map((t) => ({
|
|
@@ -24441,6 +24800,37 @@ function translateTools(tools) {
|
|
|
24441
24800
|
}
|
|
24442
24801
|
}));
|
|
24443
24802
|
}
|
|
24803
|
+
/** The image parts of a Pi tool result, in wire order. */
|
|
24804
|
+
function imagePartsOf(parts) {
|
|
24805
|
+
const out = [];
|
|
24806
|
+
for (const p of parts) {
|
|
24807
|
+
if (p.type !== "image") continue;
|
|
24808
|
+
if (typeof p.data !== "string" || p.data.length === 0) continue;
|
|
24809
|
+
out.push({
|
|
24810
|
+
data: p.data,
|
|
24811
|
+
mimeType: p.mimeType ?? "image/png"
|
|
24812
|
+
});
|
|
24813
|
+
}
|
|
24814
|
+
return out;
|
|
24815
|
+
}
|
|
24816
|
+
/**
|
|
24817
|
+
* The text of a tool result, with two pieces of information restored that were
|
|
24818
|
+
* previously thrown away:
|
|
24819
|
+
*
|
|
24820
|
+
* - `isError`. Pi records it on the message, but the wire tool-output item has
|
|
24821
|
+
* no error flag, so without a marker the model could not tell a failed call
|
|
24822
|
+
* from a successful one. The Anthropic shim has always prefixed
|
|
24823
|
+
* `[tool error]`; this matches it.
|
|
24824
|
+
* - a pointer to the follow-up image message, when the tool returned only
|
|
24825
|
+
* images. An empty tool output is confusing on its own; naming what follows
|
|
24826
|
+
* is not.
|
|
24827
|
+
*/
|
|
24828
|
+
function toolResultText(m, hasImages) {
|
|
24829
|
+
let text = joinTextParts(m.content);
|
|
24830
|
+
if (hasImages && text.length === 0) text = "[image result below]";
|
|
24831
|
+
if (m.isError === true) text = text.length > 0 ? `[tool error] ${text}` : "[tool error]";
|
|
24832
|
+
return text;
|
|
24833
|
+
}
|
|
24444
24834
|
function joinTextParts(parts) {
|
|
24445
24835
|
let s = "";
|
|
24446
24836
|
for (const p of parts) if (p.type === "text" && typeof p.text === "string") s += p.text;
|
|
@@ -24513,6 +24903,7 @@ async function runResponsesAttempt(stream, payload, opts, signal, state) {
|
|
|
24513
24903
|
});
|
|
24514
24904
|
activeTextIndex = null;
|
|
24515
24905
|
};
|
|
24906
|
+
let sawTerminal = false;
|
|
24516
24907
|
for await (const evt of sseStream) {
|
|
24517
24908
|
const data = evt?.data;
|
|
24518
24909
|
if (data == null) continue;
|
|
@@ -24658,6 +25049,7 @@ async function runResponsesAttempt(stream, payload, opts, signal, state) {
|
|
|
24658
25049
|
}
|
|
24659
25050
|
case "response.completed":
|
|
24660
25051
|
case "response.incomplete":
|
|
25052
|
+
sawTerminal = true;
|
|
24661
25053
|
accum.usage = mapResponsesUsage(ev.response?.usage);
|
|
24662
25054
|
if (ev.type === "response.incomplete" && ev.response?.incomplete_details?.reason === "max_output_tokens") accum.finishReason = "length";
|
|
24663
25055
|
if (opts.onChunk && accum.usage) try {
|
|
@@ -24679,6 +25071,11 @@ async function runResponsesAttempt(stream, payload, opts, signal, state) {
|
|
|
24679
25071
|
}
|
|
24680
25072
|
}
|
|
24681
25073
|
if (!state.active) return;
|
|
25074
|
+
if (!sawTerminal) {
|
|
25075
|
+
closeActiveText();
|
|
25076
|
+
pushTerminalError(stream, resolved, /* @__PURE__ */ new Error("responses stream ended without a terminal event (truncated)"));
|
|
25077
|
+
return;
|
|
25078
|
+
}
|
|
24682
25079
|
closeActiveText();
|
|
24683
25080
|
for (const block of accum.blocks) {
|
|
24684
25081
|
if (block.kind !== "tool") continue;
|
|
@@ -24702,11 +25099,7 @@ async function runResponsesAttempt(stream, payload, opts, signal, state) {
|
|
|
24702
25099
|
});
|
|
24703
25100
|
}
|
|
24704
25101
|
function buildResponsesPayload(context, resolved) {
|
|
24705
|
-
const messages =
|
|
24706
|
-
for (const m of context.messages) {
|
|
24707
|
-
const neutral = piMessageToNeutral(m);
|
|
24708
|
-
if (neutral) messages.push(neutral);
|
|
24709
|
-
}
|
|
25102
|
+
const messages = piMessagesToNeutral(context.messages);
|
|
24710
25103
|
return assembleResponsesPayload({
|
|
24711
25104
|
model: resolved.modelId,
|
|
24712
25105
|
instructions: context.systemPrompt || void 0,
|
|
@@ -24716,50 +25109,82 @@ function buildResponsesPayload(context, resolved) {
|
|
|
24716
25109
|
stream: true
|
|
24717
25110
|
});
|
|
24718
25111
|
}
|
|
24719
|
-
|
|
24720
|
-
|
|
24721
|
-
|
|
25112
|
+
/**
|
|
25113
|
+
* Pi messages → neutral messages, with tool-result images hoisted to the end of
|
|
25114
|
+
* each contiguous run of tool results.
|
|
25115
|
+
*
|
|
25116
|
+
* Exact twin of `translateMessages` on the chat path — see that doc comment for
|
|
25117
|
+
* why the grouping is load-bearing rather than cosmetic. A per-message fan-out
|
|
25118
|
+
* interleaves a user message between parallel `function_call_output` items,
|
|
25119
|
+
* which at best is a shape no client emits and at worst is rejected outright.
|
|
25120
|
+
*/
|
|
25121
|
+
function piMessagesToNeutral(messages) {
|
|
25122
|
+
const out = [];
|
|
25123
|
+
let pending = [];
|
|
25124
|
+
const flushImages = () => {
|
|
25125
|
+
if (pending.length === 0) return;
|
|
25126
|
+
out.push({
|
|
24722
25127
|
role: "user",
|
|
24723
|
-
content:
|
|
24724
|
-
|
|
24725
|
-
|
|
24726
|
-
|
|
24727
|
-
|
|
24728
|
-
text: c.text
|
|
25128
|
+
content: pending.map((img) => ({
|
|
25129
|
+
type: "image",
|
|
25130
|
+
mimeType: img.mimeType,
|
|
25131
|
+
data: img.data
|
|
25132
|
+
}))
|
|
24729
25133
|
});
|
|
24730
|
-
|
|
24731
|
-
|
|
24732
|
-
|
|
24733
|
-
|
|
24734
|
-
|
|
24735
|
-
|
|
25134
|
+
pending = [];
|
|
25135
|
+
};
|
|
25136
|
+
for (const m of messages) {
|
|
25137
|
+
if (m.role === "toolResult") {
|
|
25138
|
+
const images = imagePartsOf(m.content);
|
|
25139
|
+
out.push({
|
|
25140
|
+
role: "toolResult",
|
|
25141
|
+
toolCallId: m.toolCallId,
|
|
25142
|
+
output: toolResultText(m, images.length > 0)
|
|
25143
|
+
});
|
|
25144
|
+
pending.push(...images);
|
|
25145
|
+
continue;
|
|
25146
|
+
}
|
|
25147
|
+
flushImages();
|
|
25148
|
+
if (m.role === "user") if (typeof m.content === "string") out.push({
|
|
24736
25149
|
role: "user",
|
|
24737
|
-
content:
|
|
24738
|
-
};
|
|
24739
|
-
}
|
|
24740
|
-
if (m.role === "assistant") {
|
|
24741
|
-
const parts = [];
|
|
24742
|
-
for (const c of m.content) if (c.type === "text") parts.push({
|
|
24743
|
-
type: "text",
|
|
24744
|
-
text: c.text
|
|
24745
|
-
});
|
|
24746
|
-
else if (c.type === "toolCall") parts.push({
|
|
24747
|
-
type: "toolCall",
|
|
24748
|
-
id: c.id,
|
|
24749
|
-
name: c.name,
|
|
24750
|
-
arguments: c.arguments
|
|
25150
|
+
content: m.content
|
|
24751
25151
|
});
|
|
24752
|
-
|
|
24753
|
-
|
|
24754
|
-
content
|
|
24755
|
-
|
|
25152
|
+
else {
|
|
25153
|
+
const parts = [];
|
|
25154
|
+
for (const c of m.content) if (c.type === "text") parts.push({
|
|
25155
|
+
type: "text",
|
|
25156
|
+
text: c.text
|
|
25157
|
+
});
|
|
25158
|
+
else if (c.type === "image") parts.push({
|
|
25159
|
+
type: "image",
|
|
25160
|
+
mimeType: c.mimeType,
|
|
25161
|
+
data: c.data
|
|
25162
|
+
});
|
|
25163
|
+
out.push({
|
|
25164
|
+
role: "user",
|
|
25165
|
+
content: parts
|
|
25166
|
+
});
|
|
25167
|
+
}
|
|
25168
|
+
else if (m.role === "assistant") {
|
|
25169
|
+
const parts = [];
|
|
25170
|
+
for (const c of m.content) if (c.type === "text") parts.push({
|
|
25171
|
+
type: "text",
|
|
25172
|
+
text: c.text
|
|
25173
|
+
});
|
|
25174
|
+
else if (c.type === "toolCall") parts.push({
|
|
25175
|
+
type: "toolCall",
|
|
25176
|
+
id: c.id,
|
|
25177
|
+
name: c.name,
|
|
25178
|
+
arguments: c.arguments
|
|
25179
|
+
});
|
|
25180
|
+
out.push({
|
|
25181
|
+
role: "assistant",
|
|
25182
|
+
content: parts
|
|
25183
|
+
});
|
|
25184
|
+
}
|
|
24756
25185
|
}
|
|
24757
|
-
|
|
24758
|
-
|
|
24759
|
-
toolCallId: m.toolCallId,
|
|
24760
|
-
output: joinTextParts(m.content)
|
|
24761
|
-
};
|
|
24762
|
-
return null;
|
|
25186
|
+
flushImages();
|
|
25187
|
+
return out;
|
|
24763
25188
|
}
|
|
24764
25189
|
function piToolsToNeutral(tools) {
|
|
24765
25190
|
if (!tools || tools.length === 0) return void 0;
|
|
@@ -25094,7 +25519,37 @@ function argsRecord(params) {
|
|
|
25094
25519
|
* `tools.ts` uses for `peer_review`.
|
|
25095
25520
|
*/
|
|
25096
25521
|
function joinEnvelopeText(env) {
|
|
25097
|
-
return (env.content ?? []).map((c) => c.text).join("\n");
|
|
25522
|
+
return (env.content ?? []).filter((c) => c.type === "text").map((c) => c.text).join("\n");
|
|
25523
|
+
}
|
|
25524
|
+
/**
|
|
25525
|
+
* Wrap a text payload PLUS a captured image in Pi's tool-result shape.
|
|
25526
|
+
*
|
|
25527
|
+
* The browse worker's own system prompt tells it to "use screenshot to SEE the
|
|
25528
|
+
* page", but every result was flattened to text — so the pixels never reached
|
|
25529
|
+
* it and it received base64 characters instead. Pi's tool-result content
|
|
25530
|
+
* supports image blocks natively, and the engine's `afterToolCall` cap
|
|
25531
|
+
* (`tool-output-cap.ts`) already preserves them and exempts them from the text
|
|
25532
|
+
* byte cap. Nothing was ever producing one.
|
|
25533
|
+
*
|
|
25534
|
+
* Text first, image second — same ordering rule as the MCP surface
|
|
25535
|
+
* (`mcpTextAndImage`), so a consumer reading the first block still finds text.
|
|
25536
|
+
*/
|
|
25537
|
+
function imageResult$1(text, image) {
|
|
25538
|
+
return {
|
|
25539
|
+
content: [{
|
|
25540
|
+
type: "text",
|
|
25541
|
+
text
|
|
25542
|
+
}, {
|
|
25543
|
+
type: "image",
|
|
25544
|
+
data: image.data,
|
|
25545
|
+
mimeType: image.mimeType
|
|
25546
|
+
}],
|
|
25547
|
+
details: {}
|
|
25548
|
+
};
|
|
25549
|
+
}
|
|
25550
|
+
/** The first image block on a dispatch envelope, if the tool captured one. */
|
|
25551
|
+
function envelopeImage(env) {
|
|
25552
|
+
return (env.content ?? []).find((c) => c.type === "image");
|
|
25098
25553
|
}
|
|
25099
25554
|
/**
|
|
25100
25555
|
* How a tool interacts with a session's owned tabs:
|
|
@@ -25390,7 +25845,8 @@ function makeBrowserTool(meta, parameters, dispatch, sessionId) {
|
|
|
25390
25845
|
if (typeof tabId === "number") recordSessionTab(sessionId, tabId);
|
|
25391
25846
|
} else if (policy === "closes") for (const tabId of toNumberArray(args.tabIds)) releaseSessionTab(sessionId, tabId);
|
|
25392
25847
|
}
|
|
25393
|
-
|
|
25848
|
+
const image = envelopeImage(env);
|
|
25849
|
+
return image ? imageResult$1(text, image) : textResult$1(text);
|
|
25394
25850
|
}
|
|
25395
25851
|
};
|
|
25396
25852
|
if (meta.executionMode) tool.executionMode = meta.executionMode;
|
|
@@ -25660,6 +26116,62 @@ function truncateModelText(text, capBytes) {
|
|
|
25660
26116
|
return head + notice + tail;
|
|
25661
26117
|
}
|
|
25662
26118
|
/**
|
|
26119
|
+
* Per-result image budget.
|
|
26120
|
+
*
|
|
26121
|
+
* Images are cheap in TOKENS (a vision image costs ~1.5k regardless of byte
|
|
26122
|
+
* size) but not in BYTES: three 3 MiB captures is ~12 MiB of base64 on the wire
|
|
26123
|
+
* and in memory, and they accumulate across turns. The text cap does not bound
|
|
26124
|
+
* them — and should not, since counting base64 against a text budget would
|
|
26125
|
+
* evict real text to make room for an image the model reads for free.
|
|
26126
|
+
*
|
|
26127
|
+
* `MAX_IMAGES_PER_RESULT` is the ceiling of the most permissive model in the
|
|
26128
|
+
* live catalog (gemini's 10). `MAX_IMAGE_BYTES_PER_RESULT` is the published
|
|
26129
|
+
* 3 MiB per-image ceiling times a small factor, so a legitimate multi-image
|
|
26130
|
+
* result passes and a runaway one does not.
|
|
26131
|
+
*/
|
|
26132
|
+
const MAX_IMAGES_PER_RESULT = 10;
|
|
26133
|
+
const MAX_IMAGE_BYTES_PER_RESULT = 12582912;
|
|
26134
|
+
/** Decoded byte size of a base64 payload, without allocating the buffer. */
|
|
26135
|
+
function base64Bytes(data) {
|
|
26136
|
+
if (typeof data !== "string" || data.length === 0) return 0;
|
|
26137
|
+
const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0;
|
|
26138
|
+
return Math.floor(data.length * 3 / 4) - padding;
|
|
26139
|
+
}
|
|
26140
|
+
/**
|
|
26141
|
+
* Trim an image list to the budget, returning the survivors plus a note naming
|
|
26142
|
+
* what was dropped. Dropping silently is not an option: a model told to compare
|
|
26143
|
+
* five screenshots, shown three, and given no indication, will reason
|
|
26144
|
+
* confidently about a set it never saw.
|
|
26145
|
+
*/
|
|
26146
|
+
function capImages(images) {
|
|
26147
|
+
const kept = [];
|
|
26148
|
+
let bytes = 0;
|
|
26149
|
+
let dropped = 0;
|
|
26150
|
+
for (const img of images) {
|
|
26151
|
+
if (kept.length >= MAX_IMAGES_PER_RESULT) {
|
|
26152
|
+
dropped++;
|
|
26153
|
+
continue;
|
|
26154
|
+
}
|
|
26155
|
+
const size = base64Bytes(img.data);
|
|
26156
|
+
if (bytes + size > MAX_IMAGE_BYTES_PER_RESULT) {
|
|
26157
|
+
dropped++;
|
|
26158
|
+
continue;
|
|
26159
|
+
}
|
|
26160
|
+
bytes += size;
|
|
26161
|
+
kept.push(img);
|
|
26162
|
+
}
|
|
26163
|
+
if (dropped === 0) return {
|
|
26164
|
+
kept,
|
|
26165
|
+
note: void 0
|
|
26166
|
+
};
|
|
26167
|
+
return {
|
|
26168
|
+
kept,
|
|
26169
|
+
note: NEWLINE + NEWLINE + `[...${dropped} of ${images.length} image(s) dropped: over the per-result budget of ${MAX_IMAGES_PER_RESULT} images / ${Math.round(MAX_IMAGE_BYTES_PER_RESULT / 1048576)} MiB. Request fewer images at a time, or capture at a lower quality....]`
|
|
26170
|
+
};
|
|
26171
|
+
}
|
|
26172
|
+
/** Literal newline, kept out of the template above for readability. */
|
|
26173
|
+
const NEWLINE = "\n";
|
|
26174
|
+
/**
|
|
25663
26175
|
* Cap a tool result's TEXT content to `capBytes`, preserving any non-text
|
|
25664
26176
|
* (image) blocks. Returns the replacement content array, or `undefined` when
|
|
25665
26177
|
* the result is already under the cap (caller leaves it untouched).
|
|
@@ -25680,20 +26192,180 @@ function capToolResultText(content, capBytes) {
|
|
|
25680
26192
|
let textBytes = 0;
|
|
25681
26193
|
const texts = [];
|
|
25682
26194
|
const images = [];
|
|
26195
|
+
const other = [];
|
|
25683
26196
|
for (const block of content) {
|
|
25684
26197
|
if (!block || typeof block !== "object") continue;
|
|
25685
26198
|
const b = block;
|
|
25686
26199
|
if (b.type === "text" && typeof b.text === "string") {
|
|
25687
26200
|
texts.push(b.text);
|
|
25688
26201
|
textBytes += Buffer.byteLength(b.text, "utf8");
|
|
25689
|
-
} else images.push(block);
|
|
26202
|
+
} else if (b.type === "image") images.push(block);
|
|
26203
|
+
else other.push(block);
|
|
25690
26204
|
}
|
|
25691
|
-
|
|
25692
|
-
|
|
25693
|
-
|
|
25694
|
-
|
|
25695
|
-
|
|
25696
|
-
|
|
26205
|
+
const { kept, note } = capImages(images);
|
|
26206
|
+
if (textBytes <= capBytes && note === void 0) return void 0;
|
|
26207
|
+
const joined = texts.join(NEWLINE);
|
|
26208
|
+
const capped = textBytes > capBytes ? truncateModelText(joined, capBytes) : joined;
|
|
26209
|
+
return [
|
|
26210
|
+
...kept,
|
|
26211
|
+
...other,
|
|
26212
|
+
{
|
|
26213
|
+
type: "text",
|
|
26214
|
+
text: capped + (note ?? "")
|
|
26215
|
+
}
|
|
26216
|
+
];
|
|
26217
|
+
}
|
|
26218
|
+
//#endregion
|
|
26219
|
+
//#region src/lib/peer-attachments.ts
|
|
26220
|
+
/**
|
|
26221
|
+
* Server-side image loading for peer-critic attachments (`imagePaths`).
|
|
26222
|
+
*
|
|
26223
|
+
* WHY PATHS AND NOT BASE64
|
|
26224
|
+
*
|
|
26225
|
+
* The obvious schema would take base64 directly. It would also mean megabytes
|
|
26226
|
+
* of payload crossing the MCP boundary and sitting in the CALLER's context — the
|
|
26227
|
+
* exact cost the browser-screenshot fix exists to remove — and it would trip the
|
|
26228
|
+
* `predictedTooLong` pre-flight, which sizes the brief before a slot is
|
|
26229
|
+
* acquired. Reading the bytes here instead keeps the caller's context clean and
|
|
26230
|
+
* the wire small: the caller sends a path, the proxy sends the pixels.
|
|
26231
|
+
*
|
|
26232
|
+
* The proxy and the caller are the same machine in this product (the MCP server
|
|
26233
|
+
* is loopback-only and the CLI is spawned by it), so a local path is meaningful
|
|
26234
|
+
* on both sides.
|
|
26235
|
+
*
|
|
26236
|
+
* THREAT MODEL
|
|
26237
|
+
*
|
|
26238
|
+
* This is a second file-reading path, so it must not become a way around the
|
|
26239
|
+
* first one's rules. Every path goes through `confineToWorkspaceResult` — the
|
|
26240
|
+
* SAME chokepoint the worker's `read`/`glob`/`grep` tools use — which enforces
|
|
26241
|
+
* workspace confinement, `realpathSync.native()` (so a symlink or junction is
|
|
26242
|
+
* resolved to its true target before the prefix check), and syntactic rejection
|
|
26243
|
+
* of UNC, device, and drive-relative Windows paths. `isSensitivePath` then
|
|
26244
|
+
* applies the credential-shaped denylist (`.env*`, `*.pem`, `id_rsa*`, `.ssh/`,
|
|
26245
|
+
* `.git/` interior, `.netrc`, …).
|
|
26246
|
+
*
|
|
26247
|
+
* On top of that, content identification adds a second barrier: a file is only
|
|
26248
|
+
* ever sent if its leading BYTES are a supported image, so a `.env` renamed to
|
|
26249
|
+
* `shot.png` is refused even if it somehow passed confinement.
|
|
26250
|
+
*
|
|
26251
|
+
* That barrier is header-only, and worth stating honestly: a file beginning
|
|
26252
|
+
* with a valid PNG signature and carrying arbitrary data afterwards would pass.
|
|
26253
|
+
* Constructing one requires write access, and a caller with write access
|
|
26254
|
+
* already has `bash` and could exfiltrate directly — so this stops the
|
|
26255
|
+
* realistic case (a model pointing at a credential file by mistake or via
|
|
26256
|
+
* prompt injection), not a determined adversary. Confinement and the denylist
|
|
26257
|
+
* are the primary controls; this is defence in depth.
|
|
26258
|
+
*/
|
|
26259
|
+
/**
|
|
26260
|
+
* Pre-encode size ceiling. Every vision-capable model in the live catalog
|
|
26261
|
+
* publishes `max_prompt_image_size: 3145728`, so anything larger cannot be sent
|
|
26262
|
+
* to any of them. Checking BEFORE `readFile` means an oversized file is never
|
|
26263
|
+
* loaded into memory, let alone base64-expanded by a third.
|
|
26264
|
+
*
|
|
26265
|
+
* The per-model check still runs later at the transport boundary
|
|
26266
|
+
* (`assertOutboundImagesOk`); this is the cheap guard, not the authority.
|
|
26267
|
+
*/
|
|
26268
|
+
const MAX_ATTACHMENT_BYTES = 3145728;
|
|
26269
|
+
const MAX_TOTAL_ATTACHMENT_BYTES = 12582912;
|
|
26270
|
+
/**
|
|
26271
|
+
* Open flags. `O_NOFOLLOW` is POSIX-only and simply absent on Windows, where
|
|
26272
|
+
* `fsConstants.O_NOFOLLOW` is `undefined` — OR-ing that in would produce `NaN`
|
|
26273
|
+
* and break every open, so it is added only when the platform defines it.
|
|
26274
|
+
*/
|
|
26275
|
+
const READ_FLAGS = typeof constants.O_NOFOLLOW === "number" ? constants.O_RDONLY | constants.O_NOFOLLOW : constants.O_RDONLY;
|
|
26276
|
+
/**
|
|
26277
|
+
* Resolve, validate, and base64-encode each path.
|
|
26278
|
+
*
|
|
26279
|
+
* Fails on the FIRST bad path rather than silently skipping it: a caller that
|
|
26280
|
+
* attached four screenshots and got three has been quietly misled about what the
|
|
26281
|
+
* reviewer actually saw.
|
|
26282
|
+
*
|
|
26283
|
+
* Error strings reach a model, so they say what was wrong and what to do, and
|
|
26284
|
+
* never echo the resolved absolute path (the confinement helper deliberately
|
|
26285
|
+
* keeps its own messages path-free for the same reason).
|
|
26286
|
+
*/
|
|
26287
|
+
async function loadPeerImages(paths, workspace) {
|
|
26288
|
+
if (paths.length > 10) return {
|
|
26289
|
+
ok: false,
|
|
26290
|
+
error: `imagePaths: ${paths.length} paths exceeds the 10-image ceiling (the most any Copilot model accepts). Send fewer.`
|
|
26291
|
+
};
|
|
26292
|
+
let workspaceAbs;
|
|
26293
|
+
try {
|
|
26294
|
+
workspaceAbs = realpathSync.native(workspace);
|
|
26295
|
+
} catch {
|
|
26296
|
+
workspaceAbs = workspace;
|
|
26297
|
+
}
|
|
26298
|
+
const images = [];
|
|
26299
|
+
let totalBytes = 0;
|
|
26300
|
+
for (const [index, raw] of paths.entries()) {
|
|
26301
|
+
const position = `imagePaths[${index}]`;
|
|
26302
|
+
if (typeof raw !== "string" || raw.length === 0) return {
|
|
26303
|
+
ok: false,
|
|
26304
|
+
error: `${position}: must be a non-empty string.`
|
|
26305
|
+
};
|
|
26306
|
+
const confined = confineToWorkspaceResult(raw, workspaceAbs);
|
|
26307
|
+
if (!confined.ok) return {
|
|
26308
|
+
ok: false,
|
|
26309
|
+
error: `${position}: ${confined.error}`
|
|
26310
|
+
};
|
|
26311
|
+
if (isSensitivePath(confined.abs, workspaceAbs)) return {
|
|
26312
|
+
ok: false,
|
|
26313
|
+
error: `${position}: rejected: sensitive path`
|
|
26314
|
+
};
|
|
26315
|
+
let handle;
|
|
26316
|
+
let buf;
|
|
26317
|
+
try {
|
|
26318
|
+
handle = await open(confined.abs, READ_FLAGS);
|
|
26319
|
+
const info = await handle.stat();
|
|
26320
|
+
if (!info.isFile()) return {
|
|
26321
|
+
ok: false,
|
|
26322
|
+
error: `${position}: rejected: not a regular file`
|
|
26323
|
+
};
|
|
26324
|
+
if (info.size > 3145728) return {
|
|
26325
|
+
ok: false,
|
|
26326
|
+
error: `${position}: ${info.size} bytes exceeds the ${MAX_ATTACHMENT_BYTES}-byte limit every vision model publishes. Re-capture at a smaller scale or lower quality.`
|
|
26327
|
+
};
|
|
26328
|
+
totalBytes += info.size;
|
|
26329
|
+
if (totalBytes > 12582912) return {
|
|
26330
|
+
ok: false,
|
|
26331
|
+
error: `${position}: the attachments total more than ${MAX_TOTAL_ATTACHMENT_BYTES} bytes. Send fewer or smaller images.`
|
|
26332
|
+
};
|
|
26333
|
+
const capacity = Math.min(info.size, MAX_ATTACHMENT_BYTES);
|
|
26334
|
+
const scratch = Buffer.alloc(Math.min(capacity + 1, 3145729));
|
|
26335
|
+
let filled = 0;
|
|
26336
|
+
for (;;) {
|
|
26337
|
+
const { bytesRead } = await handle.read(scratch, filled, scratch.length - filled, filled);
|
|
26338
|
+
if (bytesRead === 0) break;
|
|
26339
|
+
filled += bytesRead;
|
|
26340
|
+
if (filled >= scratch.length) break;
|
|
26341
|
+
}
|
|
26342
|
+
if (filled > capacity) return {
|
|
26343
|
+
ok: false,
|
|
26344
|
+
error: `${position}: the file grew while it was being read. Retry once it has stopped changing.`
|
|
26345
|
+
};
|
|
26346
|
+
buf = scratch.subarray(0, filled);
|
|
26347
|
+
} catch {
|
|
26348
|
+
return {
|
|
26349
|
+
ok: false,
|
|
26350
|
+
error: `${position}: file not found or unreadable`
|
|
26351
|
+
};
|
|
26352
|
+
} finally {
|
|
26353
|
+
await handle?.close().catch(() => {});
|
|
26354
|
+
}
|
|
26355
|
+
const mimeType = detectImageMimeType(buf);
|
|
26356
|
+
if (!mimeType) return {
|
|
26357
|
+
ok: false,
|
|
26358
|
+
error: `${position}: not a supported image. Content is identified by its bytes, not its extension. Supported: ${[...SUPPORTED_IMAGE_MIME_TYPES].join(", ")}.`
|
|
26359
|
+
};
|
|
26360
|
+
images.push({
|
|
26361
|
+
data: buf.toString("base64"),
|
|
26362
|
+
mimeType
|
|
26363
|
+
});
|
|
26364
|
+
}
|
|
26365
|
+
return {
|
|
26366
|
+
ok: true,
|
|
26367
|
+
images
|
|
26368
|
+
};
|
|
25697
26369
|
}
|
|
25698
26370
|
//#endregion
|
|
25699
26371
|
//#region src/lib/tokenizer.ts
|
|
@@ -25924,9 +26596,15 @@ const getTokenCount = async (payload, model) => {
|
|
|
25924
26596
|
* - anthropic-version (VS Code's Anthropic SDK sends this)
|
|
25925
26597
|
* - X-Interaction-Id (VS Code sends a session-scoped UUID)
|
|
25926
26598
|
*
|
|
25927
|
-
* We intentionally omit copilot-vision-request
|
|
26599
|
+
* We intentionally omit copilot-vision-request. VS Code only sends it when
|
|
25928
26600
|
* images are present, and the native /v1/messages endpoint handles vision
|
|
25929
|
-
* without
|
|
26601
|
+
* without it — VERIFIED live (2026-08-03) rather than assumed: the same
|
|
26602
|
+
* base64 image sent to claude-opus-5 with the header omitted and with it set
|
|
26603
|
+
* both returned 200 AND the model named the image's colour in each case, so
|
|
26604
|
+
* the pixels genuinely reach it either way. Probe `passthrough_image_claude`
|
|
26605
|
+
* in scripts/probe-copilot-compat.sh keeps that verified; if Copilot ever
|
|
26606
|
+
* starts gating vision on the header, that probe fails rather than images
|
|
26607
|
+
* silently degrading on the lead model's own path.
|
|
25930
26608
|
*
|
|
25931
26609
|
* extraHeaders allows callers to forward client-supplied beta headers
|
|
25932
26610
|
* (anthropic-beta) so Copilot enables extended features.
|
|
@@ -26543,6 +27221,11 @@ function toolEntries(scope) {
|
|
|
26543
27221
|
type: "string",
|
|
26544
27222
|
description: "Optional additional context (extra file content, prior decisions). Concatenated to the brief before sending."
|
|
26545
27223
|
},
|
|
27224
|
+
imagePaths: {
|
|
27225
|
+
type: "array",
|
|
27226
|
+
items: { type: "string" },
|
|
27227
|
+
description: "Optional paths to image files (screenshots, diagrams, charts) INSIDE the workspace for the persona to LOOK AT. The proxy reads and encodes them, so no image data passes through your context. Content is identified by bytes, not extension; jpeg/png/webp/gif/heic/heif, 3 MiB each. Note paths are confined to the proxy's working directory. Note the per-model ceiling: the gemini-backed personas accept 10 images, the gpt- and opus-backed ones accept 1."
|
|
27228
|
+
},
|
|
26546
27229
|
effort: {
|
|
26547
27230
|
type: "string",
|
|
26548
27231
|
enum: [...p.allowedEfforts],
|
|
@@ -26802,7 +27485,10 @@ async function dispatchModelCall(args) {
|
|
|
26802
27485
|
content: [{
|
|
26803
27486
|
type: "input_text",
|
|
26804
27487
|
text: args.userText
|
|
26805
|
-
}]
|
|
27488
|
+
}, ...(args.images ?? []).map((img) => ({
|
|
27489
|
+
type: "input_image",
|
|
27490
|
+
image_url: `data:${img.mimeType};base64,${img.data}`
|
|
27491
|
+
}))]
|
|
26806
27492
|
}],
|
|
26807
27493
|
stream: false,
|
|
26808
27494
|
reasoning: { effort: args.effort }
|
|
@@ -26820,7 +27506,20 @@ async function dispatchModelCall(args) {
|
|
|
26820
27506
|
system: args.instructions,
|
|
26821
27507
|
thinking: { type: "adaptive" },
|
|
26822
27508
|
output_config: { effort: args.effort },
|
|
26823
|
-
messages: [{
|
|
27509
|
+
messages: [args.images && args.images.length > 0 ? {
|
|
27510
|
+
role: "user",
|
|
27511
|
+
content: [{
|
|
27512
|
+
type: "text",
|
|
27513
|
+
text: args.userText
|
|
27514
|
+
}, ...args.images.map((img) => ({
|
|
27515
|
+
type: "image",
|
|
27516
|
+
source: {
|
|
27517
|
+
type: "base64",
|
|
27518
|
+
media_type: img.mimeType,
|
|
27519
|
+
data: img.data
|
|
27520
|
+
}
|
|
27521
|
+
}))]
|
|
27522
|
+
} : {
|
|
26824
27523
|
role: "user",
|
|
26825
27524
|
content: args.userText
|
|
26826
27525
|
}]
|
|
@@ -26835,7 +27534,16 @@ async function dispatchModelCall(args) {
|
|
|
26835
27534
|
messages: [{
|
|
26836
27535
|
role: "system",
|
|
26837
27536
|
content: args.instructions
|
|
26838
|
-
}, {
|
|
27537
|
+
}, args.images && args.images.length > 0 ? {
|
|
27538
|
+
role: "user",
|
|
27539
|
+
content: [{
|
|
27540
|
+
type: "text",
|
|
27541
|
+
text: args.userText
|
|
27542
|
+
}, ...args.images.map((img) => ({
|
|
27543
|
+
type: "image_url",
|
|
27544
|
+
image_url: { url: `data:${img.mimeType};base64,${img.data}` }
|
|
27545
|
+
}))]
|
|
27546
|
+
} : {
|
|
26839
27547
|
role: "user",
|
|
26840
27548
|
content: args.userText
|
|
26841
27549
|
}],
|
|
@@ -26847,7 +27555,7 @@ async function dispatchModelCall(args) {
|
|
|
26847
27555
|
label: resolvedModel
|
|
26848
27556
|
}));
|
|
26849
27557
|
}
|
|
26850
|
-
async function callPersona(persona, prompt, context, effort, signal) {
|
|
27558
|
+
async function callPersona(persona, prompt, context, effort, signal, images) {
|
|
26851
27559
|
const userText = buildUserText(prompt, context);
|
|
26852
27560
|
const text = await dispatchModelCall({
|
|
26853
27561
|
model: persona.model,
|
|
@@ -26855,6 +27563,7 @@ async function callPersona(persona, prompt, context, effort, signal) {
|
|
|
26855
27563
|
instructions: persona.baseInstructions,
|
|
26856
27564
|
userText,
|
|
26857
27565
|
effort,
|
|
27566
|
+
images,
|
|
26858
27567
|
signal
|
|
26859
27568
|
});
|
|
26860
27569
|
if (!text) return toolError(`persona ${persona.agentName}: empty assistant output`);
|
|
@@ -26903,6 +27612,7 @@ async function handleToolsCall(body, scope, sessionWorkspace) {
|
|
|
26903
27612
|
let personaPrompt;
|
|
26904
27613
|
let personaContext;
|
|
26905
27614
|
let personaEffort;
|
|
27615
|
+
let personaImages;
|
|
26906
27616
|
if (persona) {
|
|
26907
27617
|
if (args.effort !== void 0 && !isEffort(args.effort)) return rpcError(body.id, RPC_INVALID_PARAMS, `tools/call: arguments.effort must be one of ${EFFORT_LEVELS.join("|")}; got ${JSON.stringify(args.effort)}`);
|
|
26908
27618
|
const requestedEffort = args.effort;
|
|
@@ -26910,6 +27620,12 @@ async function handleToolsCall(body, scope, sessionWorkspace) {
|
|
|
26910
27620
|
if (!prompt) return rpcError(body.id, RPC_INVALID_PARAMS, `tools/call: arguments.prompt is required`);
|
|
26911
27621
|
personaPrompt = prompt;
|
|
26912
27622
|
personaContext = typeof args.context === "string" ? args.context : void 0;
|
|
27623
|
+
if (args.imagePaths !== void 0) {
|
|
27624
|
+
if (!Array.isArray(args.imagePaths) || args.imagePaths.some((v) => typeof v !== "string")) return rpcError(body.id, RPC_INVALID_PARAMS, "tools/call: arguments.imagePaths must be an array of strings");
|
|
27625
|
+
const loaded = await loadPeerImages(args.imagePaths, process.cwd());
|
|
27626
|
+
if (!loaded.ok) return rpcError(body.id, RPC_INVALID_PARAMS, `tools/call: ${loaded.error}`);
|
|
27627
|
+
personaImages = loaded.images;
|
|
27628
|
+
}
|
|
26913
27629
|
if (requestedEffort !== void 0 && !persona.allowedEfforts.includes(requestedEffort)) return rpcError(body.id, RPC_INVALID_PARAMS, `tools/call: persona "${persona.toolNameHttp}" does not accept effort="${requestedEffort}". Allowed: ${persona.allowedEfforts.join("|")}.`);
|
|
26914
27630
|
personaEffort = requestedEffort ?? persona.defaultEffort;
|
|
26915
27631
|
}
|
|
@@ -26945,7 +27661,7 @@ async function handleToolsCall(body, scope, sessionWorkspace) {
|
|
|
26945
27661
|
const telemetryModel = persona ? persona.model : "(non-persona)";
|
|
26946
27662
|
try {
|
|
26947
27663
|
if (nonPersonaTool) applySessionWorkspace(args, sessionWorkspace, nonPersonaTool);
|
|
26948
|
-
const result = persona ? await callPersona(persona, personaPrompt, personaContext, personaEffort, aborter?.signal) : await nonPersonaTool.handler(args, aborter?.signal);
|
|
27664
|
+
const result = persona ? await callPersona(persona, personaPrompt, personaContext, personaEffort, aborter?.signal, personaImages) : await nonPersonaTool.handler(args, aborter?.signal);
|
|
26949
27665
|
logTelemetry({
|
|
26950
27666
|
name: telemetryName,
|
|
26951
27667
|
model: telemetryModel,
|
|
@@ -29230,18 +29946,49 @@ const READ_PARAMS = Type$1.Object({
|
|
|
29230
29946
|
description: "Max lines to return."
|
|
29231
29947
|
}))
|
|
29232
29948
|
});
|
|
29949
|
+
/**
|
|
29950
|
+
* A file is an image if its BYTES say so. Extensions are a caller assertion and
|
|
29951
|
+
* are not consulted: a `.png` holding HTML must not be shipped as an image, and
|
|
29952
|
+
* a screenshot saved without a suffix should still work.
|
|
29953
|
+
*/
|
|
29954
|
+
function imageResult(data, mimeType) {
|
|
29955
|
+
return {
|
|
29956
|
+
content: [{
|
|
29957
|
+
type: "text",
|
|
29958
|
+
text: `[${mimeType} image, ${data.length} base64 chars]`
|
|
29959
|
+
}, {
|
|
29960
|
+
type: "image",
|
|
29961
|
+
data,
|
|
29962
|
+
mimeType
|
|
29963
|
+
}],
|
|
29964
|
+
details: {}
|
|
29965
|
+
};
|
|
29966
|
+
}
|
|
29967
|
+
/**
|
|
29968
|
+
* Non-image binary detection. A NUL byte in the first few KiB is the classic
|
|
29969
|
+
* heuristic (it is what `grep`, `git`, and `file` all effectively use) and it is
|
|
29970
|
+
* enough here: the goal is not to classify the file, only to avoid handing the
|
|
29971
|
+
* model a wall of U+FFFD replacement characters and calling it text.
|
|
29972
|
+
*/
|
|
29973
|
+
function looksBinary(buf) {
|
|
29974
|
+
return buf.subarray(0, Math.min(buf.length, 8192)).includes(0);
|
|
29975
|
+
}
|
|
29233
29976
|
function readTool(workspace) {
|
|
29234
29977
|
return {
|
|
29235
29978
|
name: "read",
|
|
29236
29979
|
label: "Read file",
|
|
29237
|
-
description: "Read a file from the worker's workspace. Returns UTF-8 text. Files larger than 10 MiB are refused; use offset/limit to page.",
|
|
29980
|
+
description: "Read a file from the worker's workspace. Returns UTF-8 text, or the image itself for a jpeg/png/webp/gif/heic/heif file (detected by content, not extension) so you can actually SEE it. Other binary files are refused rather than returned as mojibake. Files larger than 10 MiB are refused; use offset/limit to page.",
|
|
29238
29981
|
parameters: READ_PARAMS,
|
|
29239
29982
|
async execute(_toolCallId, params, signal) {
|
|
29240
29983
|
const abs = resolvePathOrThrow(params.path, workspace);
|
|
29241
29984
|
const st = await stat(abs);
|
|
29242
29985
|
if (!st.isFile()) throw new Error("rejected: not a regular file");
|
|
29243
29986
|
if (st.size > READ_MAX_BYTES) throw new Error(`rejected: file >${READ_MAX_BYTES} bytes (10 MiB) — got ${st.size}`);
|
|
29244
|
-
const
|
|
29987
|
+
const buf = await readFile(abs, { signal });
|
|
29988
|
+
const imageMime = detectImageMimeType(buf);
|
|
29989
|
+
if (imageMime) return imageResult(buf.toString("base64"), imageMime);
|
|
29990
|
+
if (looksBinary(buf)) throw new Error(`rejected: binary file (${st.size} bytes), not a supported image. Supported image types are jpeg, png, webp, gif, heic, heif. Use \`bash\` with a suitable CLI to inspect or convert it.`);
|
|
29991
|
+
const text = buf.toString("utf8");
|
|
29245
29992
|
if (params.offset === void 0 && params.limit === void 0) return textResult(text);
|
|
29246
29993
|
const lines = text.split(/\r?\n/);
|
|
29247
29994
|
const start = params.offset ?? 0;
|
|
@@ -35176,4 +35923,4 @@ function enumerateInjectedMcpToolNames(groupKeys, opts = {}) {
|
|
|
35176
35923
|
//#endregion
|
|
35177
35924
|
export { searchWeb as $, DEFAULT_CLAUDE_MODEL_FALLBACKS as $t, trustRepo as A, state as An, getTokenCount as At, TEST_DEFAULT_MODEL as B, hasSupportedBrowserInstalled as Bt, fileLastPromptStore as C, fetchWithTransientRetry as Cn, scoutModel as Ct, repoRoot as D, copilotBaseUrl as Dn, shimDefaultsToXhigh as Dt, repoFingerprint as E, GITHUB_API_BASE_URL as En, workerToolsEnabled as Et, EXPLORE_DEFAULT_MODEL as F, createChatCompletions as Ft, buildEnv as G, CONDENSED_OPERATING_SEQUENCE as Gt, resolveModeDefaults as H, extractTarGzMember as Ht, EXPLORE_DEFAULT_THINKING as I, MAX_RESPONSE_BODY_BYTES as It, toolbeltEnabled as J, ArtifactClient as Jt, availableToolCommands as K, DEFINITION_OF_GREATNESS as Kt, IMPLEMENT_DEFAULT_MODEL as L, readResponseBodyCapped as Lt, resolveSealedGate as M, resolveMcpToolTimeoutMs as Mt, BROWSE_DEFAULT_MODEL as N, pickEndpoint as Nt, stopGateEnabledForRepo as O, copilotHeaders as On, countTokens as Ot, DEFAULT_MODEL as P, createResponses as Pt, assetFor as Q, toolbeltPathOverride as Qt, PLAN_DEFAULT_MODEL as R, parseJsonOrDiagnose as Rt, fileFindingsStore as S, getGitHubUser as Sn, reviewerModel as St, isSubagentContext as T, forwardError as Tn, standInToolEnabled as Tt, resolveWorkerRunOpts as U, extractZipMember as Ut, appendPlanReminder as V, provisionAndIndexColbert as Vt, runWorkerAgent as W, warmTreeSitterPool as Wt, vscodeRipgrepPath as X, buildWorkspaceHeaderJson as Xt, toolbeltSkipSet as Y, buildWorkspaceHeaderHelperCommand as Yt, TOOLBELT_TOOLS$1 as Z, collapsePathKeys as Zt, stopGateDisabled as _, isNullish as _n, browserCompoundToolsEnabled as _t, buildPeerAwarenessSnippet as a, generateRandomPort as an, buildAnthropicErrorEvent as at, stopReviewEnabled as b, sleep$3 as bn, geminiAvailable as bt, personasFor as c, withInstallLock as cn, logStreamError as ct, buildStopHookCommand as d, setupGitHubToken as dn, handleMcpDelete as dt, DEFAULT_CODEX_MODEL as en, ADVISOR_INTERNAL_TOOL_NAME as et, captureLaunchBaseline as f, tryRefreshAndRetry as fn, handleMcpPost as ft, launchBaselineKey as g, filterBetaHeader as gn, browseAgentEnabled as gt, injectStopHookIntoSettingsFile as h, cacheVSCodeVersion as hn, brainstormModel as ht, buildAgentPrompt as i, UPSTREAM_INACTIVITY_TIMEOUT_MS as in, isAdvisorRequested as it, liveExec as j, assembleResponsesPayload as jt, stopReviewStateDir as k, githubHeaders as kn, createMessages as kt, buildArtifactOpenHookCommand as l, setupCopilotToken as ln, readIteratorWithTimeout as lt, fileBlockBudget as m, cacheModels as mn, artifactToolsEnabled as mt, MCP_GROUPS as n, DEFAULT_PORT as nn, buildAdvisorStream as nt, buildPeerAwarenessSummary as o, pickClaudeDefault as on, buildOpenAIErrorEvent as ot, decideStopHook as p, cacheCopilotVersion as pn, agentToolsEnabled as pt, buildToolbeltAwareness as q, shouldUseInsecureTls as qt, assertMcpToolSurfaceConsistent as r, UPSTREAM_FETCH_TIMEOUT_MS as rn, injectAdvisorTool as rt, enumerateInjectedMcpToolNames as s, getPackageVersion as sn, isControllerClosedError as st, GROUP_META as t, DEFAULT_CODEX_MODEL_FALLBACKS as tn, ADVISOR_TOOL_INSTRUCTIONS as tt, buildSessionBindHookCommand as u, setupGitHubAgentToken as un, relayAnthropicStream as ut, stopGateId as v, resolveCodexModel as vn, browserToolsEnabled as vt, fileReviewDebounce as w, HTTPError as wn, scribeModel as wt, fileBaselineStore as x, getModels as xn, nativeSubagentModel as xt, stopGatePlanMode as y, resolveModel as yn, fleetToolsEnabled as yt, REVIEW_DEFAULT_MODEL as z, provisionBrowserAssets as zt };
|
|
35178
35925
|
|
|
35179
|
-
//# sourceMappingURL=peer-mcp-personas-
|
|
35926
|
+
//# sourceMappingURL=peer-mcp-personas-BvommiSI.js.map
|