@stigmer/react 3.8.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/attachment/AttachmentChipList.d.ts.map +1 -1
- package/attachment/AttachmentChipList.js +30 -1
- package/attachment/AttachmentChipList.js.map +1 -1
- package/attachment/attachment-utils.d.ts +11 -0
- package/attachment/attachment-utils.d.ts.map +1 -1
- package/attachment/attachment-utils.js +23 -0
- package/attachment/attachment-utils.js.map +1 -1
- package/attachment/clipboard.d.ts +51 -0
- package/attachment/clipboard.d.ts.map +1 -0
- package/attachment/clipboard.js +89 -0
- package/attachment/clipboard.js.map +1 -0
- package/attachment/index.d.ts +6 -1
- package/attachment/index.d.ts.map +1 -1
- package/attachment/index.js +4 -1
- package/attachment/index.js.map +1 -1
- package/attachment/prepare-image.d.ts +43 -0
- package/attachment/prepare-image.d.ts.map +1 -0
- package/attachment/prepare-image.js +162 -0
- package/attachment/prepare-image.js.map +1 -0
- package/attachment/useAttachments.d.ts.map +1 -1
- package/attachment/useAttachments.js +23 -3
- package/attachment/useAttachments.js.map +1 -1
- package/attachment/vision-fit.d.ts +50 -0
- package/attachment/vision-fit.d.ts.map +1 -0
- package/attachment/vision-fit.js +76 -0
- package/attachment/vision-fit.js.map +1 -0
- package/composer/SessionComposer.d.ts +3 -1
- package/composer/SessionComposer.d.ts.map +1 -1
- package/composer/SessionComposer.js +50 -4
- package/composer/SessionComposer.js.map +1 -1
- package/index.d.ts +2 -2
- package/index.d.ts.map +1 -1
- package/index.js +3 -2
- package/index.js.map +1 -1
- package/package.json +4 -4
- package/src/attachment/AttachmentChipList.tsx +45 -1
- package/src/attachment/__tests__/AttachmentChipList.test.tsx +103 -0
- package/src/attachment/__tests__/attachment-utils.test.ts +54 -0
- package/src/attachment/__tests__/clipboard.test.ts +110 -0
- package/src/attachment/__tests__/prepare-image.browser.test.ts +166 -0
- package/src/attachment/__tests__/prepare-image.test.ts +32 -0
- package/src/attachment/__tests__/vision-fit.test.ts +93 -0
- package/src/attachment/attachment-utils.ts +27 -0
- package/src/attachment/clipboard.ts +102 -0
- package/src/attachment/index.ts +13 -0
- package/src/attachment/prepare-image.ts +179 -0
- package/src/attachment/useAttachments.ts +26 -2
- package/src/attachment/vision-fit.ts +90 -0
- package/src/composer/SessionComposer.tsx +73 -4
- package/src/composer/__tests__/SessionComposer-paste.test.tsx +263 -0
- package/src/composer/__tests__/SessionComposer-uploadGate.test.tsx +239 -0
- package/src/index.ts +11 -1
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
MAX_VISION_LONG_EDGE_PX,
|
|
4
|
+
MAX_VISION_PIXELS,
|
|
5
|
+
exceedsVisionResolution,
|
|
6
|
+
fitToVisionResolution,
|
|
7
|
+
} from "../vision-fit.js";
|
|
8
|
+
|
|
9
|
+
describe("exceedsVisionResolution", () => {
|
|
10
|
+
it("is false at and below both limits", () => {
|
|
11
|
+
expect(exceedsVisionResolution(200, 200)).toBe(false);
|
|
12
|
+
expect(exceedsVisionResolution(1000, 1000)).toBe(false);
|
|
13
|
+
// Exactly at the pixel ceiling.
|
|
14
|
+
expect(exceedsVisionResolution(1000, 1150)).toBe(false);
|
|
15
|
+
// Exactly at the edge ceiling, pixels under budget.
|
|
16
|
+
expect(exceedsVisionResolution(MAX_VISION_LONG_EDGE_PX, 700)).toBe(false);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it("is true when either limit is exceeded", () => {
|
|
20
|
+
expect(exceedsVisionResolution(1920, 1080)).toBe(true); // pixels
|
|
21
|
+
expect(exceedsVisionResolution(MAX_VISION_LONG_EDGE_PX + 1, 10)).toBe(true); // edge
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
describe("fitToVisionResolution", () => {
|
|
26
|
+
it("returns within-limit dimensions unchanged (never upscales)", () => {
|
|
27
|
+
expect(fitToVisionResolution(200, 200)).toEqual({ width: 200, height: 200 });
|
|
28
|
+
expect(fitToVisionResolution(1000, 1000)).toEqual({ width: 1000, height: 1000 });
|
|
29
|
+
expect(fitToVisionResolution(1000, 1150)).toEqual({ width: 1000, height: 1150 });
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("returns degenerate dimensions unchanged instead of crashing", () => {
|
|
33
|
+
expect(fitToVisionResolution(0, 100)).toEqual({ width: 0, height: 100 });
|
|
34
|
+
expect(fitToVisionResolution(-5, 100)).toEqual({ width: -5, height: 100 });
|
|
35
|
+
expect(fitToVisionResolution(Number.NaN, 100)).toEqual({
|
|
36
|
+
width: Number.NaN,
|
|
37
|
+
height: 100,
|
|
38
|
+
});
|
|
39
|
+
expect(fitToVisionResolution(Number.POSITIVE_INFINITY, 100)).toEqual({
|
|
40
|
+
width: Number.POSITIVE_INFINITY,
|
|
41
|
+
height: 100,
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// Invariant checks across representative shapes: exact pixel values are
|
|
46
|
+
// an implementation detail of the fitting math, but every output must
|
|
47
|
+
// satisfy the published limits, preserve aspect, and never upscale.
|
|
48
|
+
const shapes: Array<[string, number, number]> = [
|
|
49
|
+
["1080p screenshot", 1920, 1080],
|
|
50
|
+
["4K screenshot", 3840, 2160],
|
|
51
|
+
["Retina laptop screenshot", 2880, 1800],
|
|
52
|
+
["portrait phone screenshot", 1170, 2532],
|
|
53
|
+
["3:2 camera photo", 6000, 4000],
|
|
54
|
+
["square", 4000, 4000],
|
|
55
|
+
["wide panorama (edge limit binds)", 10000, 300],
|
|
56
|
+
["tall receipt scan (edge limit binds)", 300, 10000],
|
|
57
|
+
["just over the pixel ceiling", 1073, 1073],
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
for (const [label, width, height] of shapes) {
|
|
61
|
+
it(`fits ${label} (${width}x${height}) within all limits`, () => {
|
|
62
|
+
const fitted = fitToVisionResolution(width, height);
|
|
63
|
+
|
|
64
|
+
// Both published limits hold.
|
|
65
|
+
expect(fitted.width * fitted.height).toBeLessThanOrEqual(MAX_VISION_PIXELS);
|
|
66
|
+
expect(Math.max(fitted.width, fitted.height)).toBeLessThanOrEqual(
|
|
67
|
+
MAX_VISION_LONG_EDGE_PX,
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
// Never upscales, always yields drawable integer dimensions.
|
|
71
|
+
expect(fitted.width).toBeLessThanOrEqual(width);
|
|
72
|
+
expect(fitted.height).toBeLessThanOrEqual(height);
|
|
73
|
+
expect(fitted.width).toBeGreaterThanOrEqual(1);
|
|
74
|
+
expect(fitted.height).toBeGreaterThanOrEqual(1);
|
|
75
|
+
expect(Number.isInteger(fitted.width)).toBe(true);
|
|
76
|
+
expect(Number.isInteger(fitted.height)).toBe(true);
|
|
77
|
+
|
|
78
|
+
// Aspect ratio preserved within integer-rounding tolerance.
|
|
79
|
+
const originalAspect = width / height;
|
|
80
|
+
const fittedAspect = fitted.width / fitted.height;
|
|
81
|
+
expect(Math.abs(fittedAspect - originalAspect) / originalAspect).toBeLessThan(
|
|
82
|
+
0.02,
|
|
83
|
+
);
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
it("uses most of the pixel budget rather than over-shrinking", () => {
|
|
88
|
+
// The fit should land close under the ceiling, not waste resolution:
|
|
89
|
+
// a 4K screenshot must keep at least 90% of the allowed pixels.
|
|
90
|
+
const fitted = fitToVisionResolution(3840, 2160);
|
|
91
|
+
expect(fitted.width * fitted.height).toBeGreaterThan(MAX_VISION_PIXELS * 0.9);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
@@ -112,3 +112,30 @@ export function validateAttachmentSize(file: File): string | null {
|
|
|
112
112
|
}
|
|
113
113
|
return null;
|
|
114
114
|
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Returns `name` unchanged when it is not in `taken`, otherwise the first
|
|
118
|
+
* free `stem-2.ext`, `stem-3.ext`, … variant.
|
|
119
|
+
*
|
|
120
|
+
* Duplicate filenames within one turn are not a cosmetic problem:
|
|
121
|
+
* attachments materialize at `.stigmer/inputs/{filename}`, where the
|
|
122
|
+
* deep-agent harness fails the whole execution on a mount-path collision
|
|
123
|
+
* and the Cursor harness silently overwrites the earlier file. A visible
|
|
124
|
+
* rename on the attachment chip is strictly better than either outcome.
|
|
125
|
+
*/
|
|
126
|
+
export function uniquifyFilename(
|
|
127
|
+
name: string,
|
|
128
|
+
taken: ReadonlySet<string>,
|
|
129
|
+
): string {
|
|
130
|
+
if (!taken.has(name)) return name;
|
|
131
|
+
|
|
132
|
+
const dotIndex = name.lastIndexOf(".");
|
|
133
|
+
// A leading dot (".env") is a hidden-file prefix, not an extension.
|
|
134
|
+
const stem = dotIndex > 0 ? name.slice(0, dotIndex) : name;
|
|
135
|
+
const ext = dotIndex > 0 ? name.slice(dotIndex) : "";
|
|
136
|
+
|
|
137
|
+
for (let n = 2; ; n++) {
|
|
138
|
+
const candidate = `${stem}-${n}${ext}`;
|
|
139
|
+
if (!taken.has(candidate)) return candidate;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Clipboard file extraction for composer surfaces — the paste half of
|
|
3
|
+
* "paste a screenshot and the agent sees it" (stigmer/stigmer#284).
|
|
4
|
+
*
|
|
5
|
+
* Extraction is deliberately SYNCHRONOUS. Clipboard file handles are only
|
|
6
|
+
* reliable while the paste event is being dispatched, and the caller must
|
|
7
|
+
* decide `preventDefault()` in the same tick (a copied image usually carries
|
|
8
|
+
* an HTML/text flavor that would otherwise paste as junk markup). An async
|
|
9
|
+
* "one call" API here would pass in tests and lose pastes in real browsers.
|
|
10
|
+
* Async work on the extracted files (e.g. {@link prepareImageForVision})
|
|
11
|
+
* happens after extraction, on plain `File` objects that stay valid.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Structural source for {@link extractClipboardFiles}: both the native
|
|
16
|
+
* `ClipboardEvent` and React's synthetic clipboard event satisfy it.
|
|
17
|
+
*/
|
|
18
|
+
export interface ClipboardFilesSource {
|
|
19
|
+
readonly clipboardData: { readonly files: FileList } | null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Names browsers assign to clipboard images that never had a real filename
|
|
24
|
+
* (a screenshot, an image copied off a web page). Chrome, Firefox, and
|
|
25
|
+
* Safari all use `image.<ext>`. A file pasted from the OS file manager
|
|
26
|
+
* keeps its real name and never matches.
|
|
27
|
+
*/
|
|
28
|
+
const GENERIC_CLIPBOARD_IMAGE_NAME = /^image\.(png|jpe?g|gif|webp|tiff?|bmp|avif)$/i;
|
|
29
|
+
|
|
30
|
+
/** Extension for a synthesized name, keyed by the clipboard MIME type. */
|
|
31
|
+
const IMAGE_MIME_EXTENSIONS: Record<string, string> = {
|
|
32
|
+
"image/png": "png",
|
|
33
|
+
"image/jpeg": "jpg",
|
|
34
|
+
"image/gif": "gif",
|
|
35
|
+
"image/webp": "webp",
|
|
36
|
+
"image/tiff": "tiff",
|
|
37
|
+
"image/bmp": "bmp",
|
|
38
|
+
"image/avif": "avif",
|
|
39
|
+
"image/svg+xml": "svg",
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Monotonic per-page-session counter. Combined with the time component this
|
|
44
|
+
* keeps synthesized names unique even across page reloads within one agent
|
|
45
|
+
* session — required because attachments materialize at
|
|
46
|
+
* `.stigmer/inputs/{filename}` in a session-scoped directory, where a
|
|
47
|
+
* repeated name from a later turn silently replaces the earlier turn's file.
|
|
48
|
+
*/
|
|
49
|
+
let pasteSequence = 0;
|
|
50
|
+
|
|
51
|
+
function synthesizePastedImageName(mimeType: string, now: Date): string {
|
|
52
|
+
const pad = (n: number) => String(n).padStart(2, "0");
|
|
53
|
+
const time = `${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
|
54
|
+
const ext = IMAGE_MIME_EXTENSIONS[mimeType.toLowerCase()] ?? "png";
|
|
55
|
+
pasteSequence += 1;
|
|
56
|
+
return `pasted-image-${time}-${pasteSequence}.${ext}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Extracts files from a paste event, giving clipboard images that carry the
|
|
61
|
+
* browser's generic `image.png` name a unique, human-readable one
|
|
62
|
+
* (`pasted-image-<HHMMSS>-<n>.<ext>`).
|
|
63
|
+
*
|
|
64
|
+
* Unique names are load-bearing, not cosmetic: attachments mount at
|
|
65
|
+
* `.stigmer/inputs/{filename}`, and two same-named attachments either fail
|
|
66
|
+
* the execution (deep-agent harness) or silently overwrite each other
|
|
67
|
+
* (Cursor harness). Files with real names (pasted from a file manager) are
|
|
68
|
+
* returned unchanged.
|
|
69
|
+
*
|
|
70
|
+
* Returns an empty array for a text-only paste — callers use that to let
|
|
71
|
+
* the default text insertion proceed untouched.
|
|
72
|
+
*
|
|
73
|
+
* Must be called synchronously from the paste event handler. Call
|
|
74
|
+
* `event.preventDefault()` in the same tick when the result is non-empty;
|
|
75
|
+
* the returned `File` objects remain valid afterwards.
|
|
76
|
+
*
|
|
77
|
+
* @example
|
|
78
|
+
* ```tsx
|
|
79
|
+
* function handlePaste(e: React.ClipboardEvent<HTMLTextAreaElement>) {
|
|
80
|
+
* const files = extractClipboardFiles(e);
|
|
81
|
+
* if (files.length === 0) return; // plain text paste — leave it alone
|
|
82
|
+
* e.preventDefault();
|
|
83
|
+
* attachments.addFiles(files);
|
|
84
|
+
* }
|
|
85
|
+
* ```
|
|
86
|
+
*/
|
|
87
|
+
export function extractClipboardFiles(event: ClipboardFilesSource): File[] {
|
|
88
|
+
const fileList = event.clipboardData?.files;
|
|
89
|
+
if (!fileList || fileList.length === 0) return [];
|
|
90
|
+
|
|
91
|
+
return Array.from(fileList).map((file) => {
|
|
92
|
+
const isGenericImage =
|
|
93
|
+
file.type.toLowerCase().startsWith("image/") &&
|
|
94
|
+
(file.name === "" || GENERIC_CLIPBOARD_IMAGE_NAME.test(file.name));
|
|
95
|
+
if (!isGenericImage) return file;
|
|
96
|
+
|
|
97
|
+
return new File([file], synthesizePastedImageName(file.type, new Date()), {
|
|
98
|
+
type: file.type,
|
|
99
|
+
lastModified: file.lastModified,
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
package/src/attachment/index.ts
CHANGED
|
@@ -13,5 +13,18 @@ export {
|
|
|
13
13
|
MAX_ATTACHMENT_BYTES,
|
|
14
14
|
detectContentType,
|
|
15
15
|
formatFileSize,
|
|
16
|
+
uniquifyFilename,
|
|
16
17
|
validateAttachmentSize,
|
|
17
18
|
} from "./attachment-utils.js";
|
|
19
|
+
|
|
20
|
+
export { extractClipboardFiles } from "./clipboard.js";
|
|
21
|
+
export type { ClipboardFilesSource } from "./clipboard.js";
|
|
22
|
+
|
|
23
|
+
export { prepareImageForVision } from "./prepare-image.js";
|
|
24
|
+
export {
|
|
25
|
+
MAX_VISION_LONG_EDGE_PX,
|
|
26
|
+
MAX_VISION_PIXELS,
|
|
27
|
+
exceedsVisionResolution,
|
|
28
|
+
fitToVisionResolution,
|
|
29
|
+
} from "./vision-fit.js";
|
|
30
|
+
export type { VisionFitSize } from "./vision-fit.js";
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser-side image preparation for agent vision — the canvas half of
|
|
3
|
+
* "paste a screenshot and the agent sees it" (stigmer/stigmer#284).
|
|
4
|
+
*
|
|
5
|
+
* A screenshot from a large display is routinely a 4-10 MB PNG. The runner
|
|
6
|
+
* delivers images to the model inline only under its per-image byte cap
|
|
7
|
+
* (3 MiB raw; see the runner's `shared/attachment-vision.ts`), so an
|
|
8
|
+
* unprepared paste degrades to "I can't see this file". Bounding the image
|
|
9
|
+
* to the resolution providers actually process (see vision-fit.ts) is
|
|
10
|
+
* quality-neutral and brings real screenshots far under that cap — while
|
|
11
|
+
* also making the upload roughly 20× faster.
|
|
12
|
+
*
|
|
13
|
+
* Applied to PASTED images only, by the composer's paste handler. Picked
|
|
14
|
+
* and dragged files are never re-encoded: a chosen file may be the subject
|
|
15
|
+
* of the task ("read the EXIF", "embed this logo"), and silently altering
|
|
16
|
+
* it would be a regression. A pasted image has no file identity to preserve.
|
|
17
|
+
*
|
|
18
|
+
* Failure policy: this module never throws and never returns a broken file.
|
|
19
|
+
* Every failure path — no canvas API (old browsers, privacy extensions,
|
|
20
|
+
* non-browser test environments), undecodable bytes, encoder failure —
|
|
21
|
+
* returns the ORIGINAL file, which still works end to end: it uploads,
|
|
22
|
+
* mounts in the workspace, and merely degrades at the runner with the
|
|
23
|
+
* standard "not viewable inline" disclosure.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import {
|
|
27
|
+
exceedsVisionResolution,
|
|
28
|
+
fitToVisionResolution,
|
|
29
|
+
} from "./vision-fit.js";
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Formats the harnesses can display inline everywhere: the Cursor harness
|
|
33
|
+
* re-sniffs magic bytes and recognizes ONLY PNG and JPEG (the deep-agent
|
|
34
|
+
* harness also takes WebP/GIF). Anything else that must ride as pixels is
|
|
35
|
+
* re-encoded to PNG so a Safari TIFF or Chrome WebP paste is not invisible
|
|
36
|
+
* on one harness.
|
|
37
|
+
*/
|
|
38
|
+
const UNIVERSAL_VISION_TYPES = new Set(["image/png", "image/jpeg"]);
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* PNG density (encoded bytes per pixel) above which content is treated as
|
|
42
|
+
* photographic and re-encoded as JPEG instead.
|
|
43
|
+
*
|
|
44
|
+
* PNG keeps UI text crisp and compresses flat screenshot content to
|
|
45
|
+
* 0.1-0.5 B/px, but photographic content in PNG runs 1.5-3 B/px and can
|
|
46
|
+
* exceed the runner's 3 MiB inline cap even at the fitted resolution
|
|
47
|
+
* (measured: ~3.4 MB for worst-case noise at 1.15 MP). Density is the
|
|
48
|
+
* content signal — scale-invariant, so small UI images never flip to JPEG
|
|
49
|
+
* just because their absolute sizes are tiny (a pure size ratio fails
|
|
50
|
+
* exactly that way) — and it bounds every kept PNG by construction:
|
|
51
|
+
* 1.15 MP × 1.0 B/px ≈ 1.1 MB, far inside the runner's cap, without this
|
|
52
|
+
* module ever referencing the runner's byte constant.
|
|
53
|
+
*/
|
|
54
|
+
const PHOTOGRAPHIC_PNG_BYTES_PER_PIXEL = 1.0;
|
|
55
|
+
const JPEG_QUALITY = 0.9;
|
|
56
|
+
|
|
57
|
+
function replaceExtension(name: string, ext: string): string {
|
|
58
|
+
const dotIndex = name.lastIndexOf(".");
|
|
59
|
+
const stem = dotIndex > 0 ? name.slice(0, dotIndex) : name;
|
|
60
|
+
return `${stem}.${ext}`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function encodeCanvas(
|
|
64
|
+
canvas: HTMLCanvasElement,
|
|
65
|
+
type: string,
|
|
66
|
+
quality?: number,
|
|
67
|
+
): Promise<Blob | null> {
|
|
68
|
+
return new Promise((resolve) => {
|
|
69
|
+
try {
|
|
70
|
+
canvas.toBlob(resolve, type, quality);
|
|
71
|
+
} catch {
|
|
72
|
+
resolve(null);
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Bounds a pasted image to the resolution vision providers actually
|
|
79
|
+
* process, re-encoding to a format both harnesses can display inline.
|
|
80
|
+
*
|
|
81
|
+
* - PNG and JPEG within the resolution limits pass through byte-identical
|
|
82
|
+
* (also guards edit-and-resubmit flows against generation loss).
|
|
83
|
+
* - GIF within the limits passes through untouched — re-encoding would
|
|
84
|
+
* silently flatten an animation the agent also receives as a file.
|
|
85
|
+
* - Oversized images resize to {@link fitToVisionResolution}; other formats
|
|
86
|
+
* (WebP, TIFF, BMP…) re-encode even when small, for harness visibility.
|
|
87
|
+
* - Output format: JPEG stays JPEG; everything else prefers PNG (crisp UI
|
|
88
|
+
* text), switching to JPEG only for photographic content — detected by
|
|
89
|
+
* PNG byte density, where PNG can exceed the runner's inline byte cap.
|
|
90
|
+
*
|
|
91
|
+
* Never throws; every failure path returns the original file (see module
|
|
92
|
+
* doc). Non-image files are returned unchanged.
|
|
93
|
+
*/
|
|
94
|
+
export async function prepareImageForVision(file: File): Promise<File> {
|
|
95
|
+
const sourceType = file.type.toLowerCase();
|
|
96
|
+
if (!sourceType.startsWith("image/")) return file;
|
|
97
|
+
if (typeof createImageBitmap !== "function" || typeof document === "undefined") {
|
|
98
|
+
return file;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
let bitmap: ImageBitmap;
|
|
102
|
+
try {
|
|
103
|
+
// "from-image" applies EXIF orientation, so a pasted phone photo lands
|
|
104
|
+
// upright instead of sideways (most browsers default to this, older
|
|
105
|
+
// Safari does not).
|
|
106
|
+
bitmap = await createImageBitmap(file, { imageOrientation: "from-image" });
|
|
107
|
+
} catch {
|
|
108
|
+
return file;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
try {
|
|
112
|
+
const needsResize = exceedsVisionResolution(bitmap.width, bitmap.height);
|
|
113
|
+
const isGif = sourceType === "image/gif";
|
|
114
|
+
|
|
115
|
+
// Within limits and already displayable everywhere (or an animated GIF,
|
|
116
|
+
// which only the resize case may flatten): leave every byte alone.
|
|
117
|
+
if (!needsResize && (UNIVERSAL_VISION_TYPES.has(sourceType) || isGif)) {
|
|
118
|
+
return file;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const target = needsResize
|
|
122
|
+
? fitToVisionResolution(bitmap.width, bitmap.height)
|
|
123
|
+
: { width: bitmap.width, height: bitmap.height };
|
|
124
|
+
|
|
125
|
+
const canvas = document.createElement("canvas");
|
|
126
|
+
canvas.width = target.width;
|
|
127
|
+
canvas.height = target.height;
|
|
128
|
+
const ctx = canvas.getContext("2d");
|
|
129
|
+
if (!ctx) return file;
|
|
130
|
+
ctx.drawImage(bitmap, 0, 0, target.width, target.height);
|
|
131
|
+
|
|
132
|
+
let blob: Blob | null;
|
|
133
|
+
let outType: string;
|
|
134
|
+
|
|
135
|
+
if (sourceType === "image/jpeg") {
|
|
136
|
+
outType = "image/jpeg";
|
|
137
|
+
blob = await encodeCanvas(canvas, outType, JPEG_QUALITY);
|
|
138
|
+
} else {
|
|
139
|
+
const png = await encodeCanvas(canvas, "image/png");
|
|
140
|
+
const pngDensity = png ? png.size / (target.width * target.height) : Infinity;
|
|
141
|
+
|
|
142
|
+
if (png && pngDensity <= PHOTOGRAPHIC_PNG_BYTES_PER_PIXEL) {
|
|
143
|
+
outType = "image/png";
|
|
144
|
+
blob = png;
|
|
145
|
+
} else {
|
|
146
|
+
// Photographic content (or a failed PNG encode): take JPEG. JPEG
|
|
147
|
+
// cannot represent alpha, so composite onto white first —
|
|
148
|
+
// destination-over paints beneath the already-drawn image, and the
|
|
149
|
+
// alpha-preserving PNG encode above is already done.
|
|
150
|
+
ctx.globalCompositeOperation = "destination-over";
|
|
151
|
+
ctx.fillStyle = "#ffffff";
|
|
152
|
+
ctx.fillRect(0, 0, target.width, target.height);
|
|
153
|
+
const jpeg = await encodeCanvas(canvas, "image/jpeg", JPEG_QUALITY);
|
|
154
|
+
|
|
155
|
+
blob = jpeg ?? png;
|
|
156
|
+
outType = jpeg ? "image/jpeg" : "image/png";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (!blob) return file;
|
|
161
|
+
|
|
162
|
+
// A pure resize that somehow grew the file is a regression, not a win:
|
|
163
|
+
// the runner caps bytes, not pixels, so the smaller original is the
|
|
164
|
+
// better payload. (Cross-format re-encodes are exempt — growing a WebP
|
|
165
|
+
// into a PNG is the price of being visible on the Cursor harness.)
|
|
166
|
+
if (outType === sourceType && blob.size >= file.size) return file;
|
|
167
|
+
|
|
168
|
+
const extension = outType === "image/jpeg" ? "jpg" : "png";
|
|
169
|
+
const name =
|
|
170
|
+
outType === sourceType ? file.name : replaceExtension(file.name, extension);
|
|
171
|
+
|
|
172
|
+
return new File([blob], name, {
|
|
173
|
+
type: outType,
|
|
174
|
+
lastModified: file.lastModified,
|
|
175
|
+
});
|
|
176
|
+
} finally {
|
|
177
|
+
bitmap.close();
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -9,6 +9,7 @@ import { toError } from "../internal/toError.js";
|
|
|
9
9
|
import {
|
|
10
10
|
detectContentType,
|
|
11
11
|
formatFileSize,
|
|
12
|
+
uniquifyFilename,
|
|
12
13
|
validateAttachmentSize,
|
|
13
14
|
} from "./attachment-utils.js";
|
|
14
15
|
|
|
@@ -166,6 +167,14 @@ export function useAttachments(
|
|
|
166
167
|
const [entries, setEntries] = useState<AttachmentEntry[]>([]);
|
|
167
168
|
const abortControllers = useRef<Map<string, AbortController>>(new Map());
|
|
168
169
|
|
|
170
|
+
// Latest-entries mirror so `addFiles` can uniquify filenames against
|
|
171
|
+
// current entries without depending on `entries` (which would give the
|
|
172
|
+
// callback a new identity every upload-phase change). Re-assigned each
|
|
173
|
+
// render to reconcile removals, and synchronously inside `addFiles` so
|
|
174
|
+
// two adds in one tick still see each other's names.
|
|
175
|
+
const entriesRef = useRef<readonly AttachmentEntry[]>(entries);
|
|
176
|
+
entriesRef.current = entries;
|
|
177
|
+
|
|
169
178
|
const uploadFile = useCallback(
|
|
170
179
|
async (id: string, file: File, contentType: string) => {
|
|
171
180
|
const controller = new AbortController();
|
|
@@ -215,14 +224,28 @@ export function useAttachments(
|
|
|
215
224
|
(files: FileList | File[]) => {
|
|
216
225
|
const fileArray = Array.from(files);
|
|
217
226
|
const validEntries: AttachmentEntry[] = [];
|
|
227
|
+
const takenNames = new Set(entriesRef.current.map((e) => e.file.name));
|
|
218
228
|
|
|
219
|
-
for (const
|
|
220
|
-
const sizeError = validateAttachmentSize(
|
|
229
|
+
for (const rawFile of fileArray) {
|
|
230
|
+
const sizeError = validateAttachmentSize(rawFile);
|
|
221
231
|
if (sizeError) {
|
|
222
232
|
options?.onValidationError?.(sizeError);
|
|
223
233
|
continue;
|
|
224
234
|
}
|
|
225
235
|
|
|
236
|
+
// Duplicate names within a turn break the execution downstream
|
|
237
|
+
// (see uniquifyFilename) — rename before the bytes ever upload,
|
|
238
|
+
// so the chip, the upload, and the mounted file all agree.
|
|
239
|
+
const uniqueName = uniquifyFilename(rawFile.name, takenNames);
|
|
240
|
+
takenNames.add(uniqueName);
|
|
241
|
+
const file =
|
|
242
|
+
uniqueName === rawFile.name
|
|
243
|
+
? rawFile
|
|
244
|
+
: new File([rawFile], uniqueName, {
|
|
245
|
+
type: rawFile.type,
|
|
246
|
+
lastModified: rawFile.lastModified,
|
|
247
|
+
});
|
|
248
|
+
|
|
226
249
|
const id = generateId();
|
|
227
250
|
const contentType = detectContentType(file);
|
|
228
251
|
|
|
@@ -239,6 +262,7 @@ export function useAttachments(
|
|
|
239
262
|
}
|
|
240
263
|
|
|
241
264
|
if (validEntries.length > 0) {
|
|
265
|
+
entriesRef.current = [...entriesRef.current, ...validEntries];
|
|
242
266
|
setEntries((prev) => [...prev, ...validEntries]);
|
|
243
267
|
}
|
|
244
268
|
},
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolution policy for images attached to agent executions — the pure math
|
|
3
|
+
* behind {@link prepareImageForVision}.
|
|
4
|
+
*
|
|
5
|
+
* Vision providers cap what they will actually look at: Anthropic's standard
|
|
6
|
+
* tier downscales anything beyond a 1568 px long edge or ~1.15 megapixels
|
|
7
|
+
* before the model sees it (OpenAI's high-detail tiling lands in the same
|
|
8
|
+
* range). Shrinking to that ceiling in the browser is therefore
|
|
9
|
+
* quality-neutral — the provider would do it anyway — while cutting a 4-10 MB
|
|
10
|
+
* screenshot paste to a few hundred KB before it ever hits the wire.
|
|
11
|
+
*
|
|
12
|
+
* Deliberately NOT named "budget": in this codebase *vision budget* means the
|
|
13
|
+
* runner's per-turn byte/count budget (`VisionBudget` in the runner's
|
|
14
|
+
* `shared/attachment-vision.ts`, 3 MiB per image raw). That byte cap stays
|
|
15
|
+
* the runner's concern alone; this module bounds pixels, the provider's
|
|
16
|
+
* concern. Two limits, one owner each, no constants to drift.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Maximum long-edge length, mirroring Anthropic's standard-tier limit.
|
|
21
|
+
* @see https://platform.claude.com/docs/en/build-with-claude/vision
|
|
22
|
+
*/
|
|
23
|
+
export const MAX_VISION_LONG_EDGE_PX = 1568;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Maximum total pixels, mirroring Anthropic's standard-tier visual-token
|
|
27
|
+
* ceiling (1568 tokens ≈ 1.15 megapixels). For nearly all photos and
|
|
28
|
+
* screenshots this — not the edge limit — is the binding constraint: a
|
|
29
|
+
* 1920×1080 screenshot fits the edge limit but still resizes to 1456×819.
|
|
30
|
+
*/
|
|
31
|
+
export const MAX_VISION_PIXELS = 1_150_000;
|
|
32
|
+
|
|
33
|
+
/** Integer pixel dimensions produced by {@link fitToVisionResolution}. */
|
|
34
|
+
export interface VisionFitSize {
|
|
35
|
+
readonly width: number;
|
|
36
|
+
readonly height: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* True when an image of these dimensions would be downscaled by the
|
|
41
|
+
* provider — i.e. when resizing it ourselves loses nothing.
|
|
42
|
+
*/
|
|
43
|
+
export function exceedsVisionResolution(width: number, height: number): boolean {
|
|
44
|
+
return (
|
|
45
|
+
width * height > MAX_VISION_PIXELS ||
|
|
46
|
+
Math.max(width, height) > MAX_VISION_LONG_EDGE_PX
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The largest size that fits both vision limits while preserving aspect
|
|
52
|
+
* ratio. Never upscales: dimensions already within the limits come back
|
|
53
|
+
* unchanged. Mirrors Anthropic's published "max API fit" computation.
|
|
54
|
+
*
|
|
55
|
+
* Degenerate inputs (zero, negative, or non-finite dimensions) come back
|
|
56
|
+
* unchanged — the caller's decode has already failed or will fail, and this
|
|
57
|
+
* module never turns a bad input into a crash.
|
|
58
|
+
*/
|
|
59
|
+
export function fitToVisionResolution(width: number, height: number): VisionFitSize {
|
|
60
|
+
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
|
61
|
+
return { width, height };
|
|
62
|
+
}
|
|
63
|
+
if (!exceedsVisionResolution(width, height)) {
|
|
64
|
+
return { width, height };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const aspect = width / height;
|
|
68
|
+
|
|
69
|
+
// Largest size under the pixel ceiling at this aspect ratio.
|
|
70
|
+
let fitHeight = Math.sqrt(MAX_VISION_PIXELS / aspect);
|
|
71
|
+
let fitWidth = fitHeight * aspect;
|
|
72
|
+
|
|
73
|
+
// The edge limit takes over only for extreme aspect ratios (panoramas,
|
|
74
|
+
// tall phone screenshots), where the pixel-fitted size still has a long
|
|
75
|
+
// edge beyond the cap.
|
|
76
|
+
if (Math.max(fitWidth, fitHeight) > MAX_VISION_LONG_EDGE_PX) {
|
|
77
|
+
if (fitWidth >= fitHeight) {
|
|
78
|
+
fitWidth = MAX_VISION_LONG_EDGE_PX;
|
|
79
|
+
fitHeight = fitWidth / aspect;
|
|
80
|
+
} else {
|
|
81
|
+
fitHeight = MAX_VISION_LONG_EDGE_PX;
|
|
82
|
+
fitWidth = fitHeight * aspect;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
width: Math.max(1, Math.floor(Math.min(fitWidth, width))),
|
|
88
|
+
height: Math.max(1, Math.floor(Math.min(fitHeight, height))),
|
|
89
|
+
};
|
|
90
|
+
}
|