@bitkyc08/opencodex 2.7.17 → 2.7.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gui/dist/assets/index-BUBsQALh.css +1 -0
- package/gui/dist/assets/index-DEbBFENM.js +40 -0
- package/gui/dist/index.html +2 -2
- package/package.json +1 -1
- package/src/adapters/anthropic-image-guard.ts +63 -7
- package/src/adapters/anthropic-image-normalize.ts +383 -0
- package/src/adapters/anthropic.ts +7 -2
- package/src/adapters/base.ts +6 -0
- package/src/adapters/cursor/exec-policy.ts +9 -1
- package/src/adapters/cursor/live-transport.ts +19 -11
- package/src/adapters/cursor/protobuf-request.ts +7 -6
- package/src/adapters/kiro-images.ts +94 -0
- package/src/adapters/kiro.ts +6 -2
- package/src/adapters/openai-chat.ts +49 -0
- package/src/adapters/openai-responses.ts +128 -2
- package/src/cli/claude.ts +3 -0
- package/src/codex/catalog.ts +11 -0
- package/src/providers/registry.ts +27 -2
- package/src/server/claude-messages.ts +11 -0
- package/src/server/image-retry.ts +42 -0
- package/src/server/management-api.ts +27 -0
- package/src/server/responses.ts +74 -22
- package/src/server/system-env.ts +7 -3
- package/src/types.ts +22 -4
- package/src/web-search/index.ts +8 -5
- package/gui/dist/assets/index-Cq8maiJf.css +0 -1
- package/gui/dist/assets/index-m4o3xsSn.js +0 -40
package/gui/dist/index.html
CHANGED
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
} catch (e) {}
|
|
17
17
|
})();
|
|
18
18
|
</script>
|
|
19
|
-
<script type="module" crossorigin src="/assets/index-
|
|
20
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
19
|
+
<script type="module" crossorigin src="/assets/index-DEbBFENM.js"></script>
|
|
20
|
+
<link rel="stylesheet" crossorigin href="/assets/index-BUBsQALh.css">
|
|
21
21
|
</head>
|
|
22
22
|
<body>
|
|
23
23
|
<div id="root"></div>
|
package/package.json
CHANGED
|
@@ -7,10 +7,13 @@
|
|
|
7
7
|
* - Hard cap: 100 images per request.
|
|
8
8
|
*
|
|
9
9
|
* Codex threads accumulate screenshots in history, so long sessions cross 20 images
|
|
10
|
-
* easily and any single retina capture (>2000px wide) kills every later turn.
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
10
|
+
* easily and any single retina capture (>2000px wide) kills every later turn. The
|
|
11
|
+
* PRIMARY layer is now anthropic-image-normalize.ts (Bun.Image resize/re-encode with an
|
|
12
|
+
* age-tier pyramid — devlog/260714_image_normalization_pipeline/020), which runs before
|
|
13
|
+
* this guard; these rules remain the deterministic BACKSTOP for whatever normalization
|
|
14
|
+
* could not shrink (undecodable passthroughs, all-terminal overflow). When this guard
|
|
15
|
+
* must drop, it textifies the OLDEST image blocks — newest screenshots are the ones the
|
|
16
|
+
* model needs.
|
|
14
17
|
*/
|
|
15
18
|
|
|
16
19
|
export const MANY_IMAGE_THRESHOLD = 20;
|
|
@@ -18,8 +21,33 @@ export const MANY_IMAGE_MAX_DIMENSION = 2000;
|
|
|
18
21
|
export const ABSOLUTE_MAX_DIMENSION = 8000;
|
|
19
22
|
export const MAX_IMAGES_PER_REQUEST = 100;
|
|
20
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Anthropic rejects any single image over 5MiB ("image exceeds 5 MB maximum", HTTP 400)
|
|
26
|
+
* regardless of dimensions or count. The unit is the BASE64 STRING LENGTH, not decoded
|
|
27
|
+
* bytes — verified in Claude Code's apiLimits.ts against Anthropic's internal API source
|
|
28
|
+
* (devlog/260714_image_normalization_pipeline/001_prior_art.md §1). A decoded-bytes
|
|
29
|
+
* comparison would let images with base64 length in (5.24MiB, 6.99MiB] through to a 400.
|
|
30
|
+
*/
|
|
31
|
+
export const MAX_IMAGE_BASE64_LENGTH = 5 * 1024 * 1024;
|
|
32
|
+
|
|
33
|
+
/** @deprecated Renamed — the cap is measured in base64 chars. Use MAX_IMAGE_BASE64_LENGTH. */
|
|
34
|
+
export const MAX_IMAGE_FILE_BYTES = MAX_IMAGE_BASE64_LENGTH;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Anthropic rejects raw HTTP bodies over ~32MB with 413 request_too_large, and base64
|
|
38
|
+
* image data dominates image-heavy histories (base64 is single-byte ASCII, so base64
|
|
39
|
+
* chars ≈ serialized body bytes for the image share). The guard runs inside buildRequest
|
|
40
|
+
* BEFORE system/tools attach, so it cannot measure the final body; instead we bound the
|
|
41
|
+
* image share to 20MiB, leaving ≥11MB headroom even against a decimal 32,000,000-byte
|
|
42
|
+
* cap — realistic non-image share (context-capped text history + tool schemas) stays
|
|
43
|
+
* well under that. Residual: a request dominated by non-image content can still 413.
|
|
44
|
+
*/
|
|
45
|
+
export const TOTAL_IMAGE_BASE64_BUDGET = 20 * 1024 * 1024;
|
|
46
|
+
|
|
21
47
|
const OMITTED_TEXT = "[image omitted: Anthropic request exceeded the 20-image limit for large images; older screenshots were dropped]";
|
|
22
48
|
const OVERSIZED_TEXT = "[image omitted: exceeds Anthropic's 8000px per-side limit]";
|
|
49
|
+
const PER_IMAGE_TOO_LARGE_TEXT = "[image omitted: exceeds Anthropic's 5MB per-image limit]";
|
|
50
|
+
const BYTE_BUDGET_TEXT = "[image omitted: total image payload exceeded Anthropic's 32MB request limit; older screenshots were dropped]";
|
|
23
51
|
|
|
24
52
|
interface ImageDimensions { width: number; height: number }
|
|
25
53
|
|
|
@@ -105,7 +133,7 @@ export function sniffImageDimensions(base64: string): ImageDimensions | null {
|
|
|
105
133
|
return pngDimensions(bytes) ?? jpegDimensions(bytes) ?? gifDimensions(bytes) ?? webpDimensions(bytes) ?? null;
|
|
106
134
|
}
|
|
107
135
|
|
|
108
|
-
interface ImageBlockRef {
|
|
136
|
+
export interface ImageBlockRef {
|
|
109
137
|
/** The array holding the block (message content or tool_result content). */
|
|
110
138
|
container: unknown[];
|
|
111
139
|
index: number;
|
|
@@ -117,7 +145,7 @@ function isImageBlock(block: unknown): block is { type: "image"; source: Record<
|
|
|
117
145
|
}
|
|
118
146
|
|
|
119
147
|
/** Collect refs to every image block in wire order (oldest first), descending into tool_result content. */
|
|
120
|
-
function collectImageRefs(messages: unknown[]): ImageBlockRef[] {
|
|
148
|
+
export function collectImageRefs(messages: unknown[]): ImageBlockRef[] {
|
|
121
149
|
const refs: ImageBlockRef[] = [];
|
|
122
150
|
const scanArray = (arr: unknown[]): void => {
|
|
123
151
|
for (let i = 0; i < arr.length; i++) {
|
|
@@ -150,7 +178,9 @@ function textify(ref: ImageBlockRef, text: string): void {
|
|
|
150
178
|
* Enforce Anthropic image limits on already-built wire messages (mutates in place).
|
|
151
179
|
* Policy: unconditionally textify >8000px images; when the request would be a
|
|
152
180
|
* many-image request (>20) with at least one image over 2000px, textify oldest
|
|
153
|
-
* images until <=20 so the 8000px allowance applies; always cap at 100 images
|
|
181
|
+
* images until <=20 so the 8000px allowance applies; always cap at 100 images;
|
|
182
|
+
* textify images over the 5MB per-image cap; and drop oldest base64 images until
|
|
183
|
+
* the total base64 payload fits the request-size budget.
|
|
154
184
|
*/
|
|
155
185
|
export function enforceAnthropicImageLimits(messages: unknown[]): void {
|
|
156
186
|
const refs = collectImageRefs(messages);
|
|
@@ -168,6 +198,16 @@ export function enforceAnthropicImageLimits(messages: unknown[]): void {
|
|
|
168
198
|
}
|
|
169
199
|
}
|
|
170
200
|
|
|
201
|
+
// Rule 1b: images over the 5MiB per-image cap (base64 chars) are invalid in any request.
|
|
202
|
+
for (let i = 0; i < refs.length; i++) {
|
|
203
|
+
if (!live.has(i)) continue;
|
|
204
|
+
const b64 = refs[i].base64;
|
|
205
|
+
if (b64 && b64.length > MAX_IMAGE_BASE64_LENGTH) {
|
|
206
|
+
textify(refs[i], PER_IMAGE_TOO_LARGE_TEXT);
|
|
207
|
+
live.delete(i);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
171
211
|
// Rule 2: many-image requests cap each image at 2000px. Keep the request at <=20
|
|
172
212
|
// images (dropping oldest first) whenever a surviving image exceeds that cap OR has
|
|
173
213
|
// unknown dimensions (URL sources and unsniffable formats): one unverifiable offender
|
|
@@ -192,4 +232,20 @@ export function enforceAnthropicImageLimits(messages: unknown[]): void {
|
|
|
192
232
|
live.delete(i);
|
|
193
233
|
}
|
|
194
234
|
}
|
|
235
|
+
|
|
236
|
+
// Rule 4: bound the total base64 payload (see TOTAL_IMAGE_BASE64_BUDGET rationale).
|
|
237
|
+
// Oldest base64 images are dropped first — newest screenshots are the ones the model
|
|
238
|
+
// needs. URL-source images carry no base64 weight and are never evicted here.
|
|
239
|
+
let base64Sum = 0;
|
|
240
|
+
for (const i of live) base64Sum += refs[i].base64?.length ?? 0;
|
|
241
|
+
if (base64Sum > TOTAL_IMAGE_BASE64_BUDGET) {
|
|
242
|
+
for (const i of [...live]) {
|
|
243
|
+
if (base64Sum <= TOTAL_IMAGE_BASE64_BUDGET) break;
|
|
244
|
+
const b64 = refs[i].base64;
|
|
245
|
+
if (!b64) continue;
|
|
246
|
+
textify(refs[i], BYTE_BUDGET_TEXT);
|
|
247
|
+
live.delete(i);
|
|
248
|
+
base64Sum -= b64.length;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
195
251
|
}
|
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic image normalization: resize/re-encode images to fit Anthropic's request
|
|
3
|
+
* limits instead of dropping them (devlog/260714_image_normalization_pipeline/020).
|
|
4
|
+
*
|
|
5
|
+
* Age-tier pyramid: newest images keep near-full fidelity, older images become
|
|
6
|
+
* progressively smaller JPEG thumbnails, so a whole session's screenshots stay visible
|
|
7
|
+
* under the request byte budget. An aggregate demotion loop re-encodes the OLDEST
|
|
8
|
+
* not-yet-terminal image one ladder position at a time until the total fits; only when
|
|
9
|
+
* every image is terminal-floored does the guard's Rule 4 (textify) fire as backstop.
|
|
10
|
+
*
|
|
11
|
+
* Runs inside the anthropic adapter's buildRequest BEFORE enforceAnthropicImageLimits,
|
|
12
|
+
* on freshly-built wire messages (in-place mutation is safe: messagesToAnthropicFormat
|
|
13
|
+
* creates new arrays/blocks). Encoding uses Bun.Image (bun >= 1.3.14, probe-verified:
|
|
14
|
+
* decodes JPEG/PNG/WebP/GIF/BMP/TIFF/HEIC/AVIF; corrupt input throws).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
collectImageRefs,
|
|
19
|
+
sniffImageDimensions,
|
|
20
|
+
TOTAL_IMAGE_BASE64_BUDGET,
|
|
21
|
+
type ImageBlockRef,
|
|
22
|
+
} from "./anthropic-image-guard";
|
|
23
|
+
|
|
24
|
+
/** One ladder position: dimension cap, JPEG quality attempts, per-image base64 cap. */
|
|
25
|
+
export interface TierSpec {
|
|
26
|
+
maxEdge: number;
|
|
27
|
+
qualities: number[];
|
|
28
|
+
/** Hard per-image base64-length cap at this position; Infinity = terminal (measured size accepted). */
|
|
29
|
+
hardCap: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const KiB = 1024;
|
|
33
|
+
const MiB = 1024 * 1024;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Ladder positions 0-5. 0-2 are the age-assigned tiers; 3-5 are demotion floor steps.
|
|
37
|
+
* Terminal (last) accepts its measured output so the aggregate loop always terminates
|
|
38
|
+
* (audit round 2, blocker 1).
|
|
39
|
+
*/
|
|
40
|
+
export const TIER_SPECS: TierSpec[] = [
|
|
41
|
+
{ maxEdge: 2000, qualities: [80, 60, 40, 30], hardCap: 2 * MiB },
|
|
42
|
+
{ maxEdge: 1024, qualities: [70, 50], hardCap: 512 * KiB },
|
|
43
|
+
{ maxEdge: 700, qualities: [60, 40], hardCap: 192 * KiB },
|
|
44
|
+
{ maxEdge: 500, qualities: [40], hardCap: 100 * KiB },
|
|
45
|
+
{ maxEdge: 400, qualities: [30], hardCap: 100 * KiB },
|
|
46
|
+
{ maxEdge: 320, qualities: [25], hardCap: Infinity },
|
|
47
|
+
];
|
|
48
|
+
const TERMINAL_POS = TIER_SPECS.length - 1;
|
|
49
|
+
|
|
50
|
+
/** Newest 6 images ride tier 0, the next 14 tier 1, the rest tier 2 (020 tier table). */
|
|
51
|
+
const TIER0_COUNT = 6;
|
|
52
|
+
const TIER1_COUNT = 14;
|
|
53
|
+
|
|
54
|
+
/** Decode-bomb guards: refuse to decode absurd inputs (020 guards; "extreme values excluded"). */
|
|
55
|
+
export const MAX_INPUT_BASE64_LENGTH = 64 * MiB;
|
|
56
|
+
export const MAX_INPUT_PIXELS = 100_000_000;
|
|
57
|
+
|
|
58
|
+
const UNDECODABLE_TEXT = "[image omitted: undecodable or corrupt image data]";
|
|
59
|
+
const BOMB_TEXT = "[image omitted: image too large to process safely]";
|
|
60
|
+
const OVERFLOW_DROP_TEXT = "[image omitted: total image payload exceeded the provider request budget; older images were dropped]";
|
|
61
|
+
|
|
62
|
+
/** Formats Anthropic accepts as-is; anything else must be transcoded or dropped. */
|
|
63
|
+
const PASSTHROUGH_MEDIA = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
|
|
64
|
+
|
|
65
|
+
export interface NormalizeOptions {
|
|
66
|
+
/** Shift every image's starting ladder position down (413 retry tightening; 030). */
|
|
67
|
+
tierBias?: number;
|
|
68
|
+
/** Test seam: replaces the Bun.Image encode path (audit round 1, blocker 6). */
|
|
69
|
+
encode?: EncodeFn;
|
|
70
|
+
/** Test seam: replaces the pass-through decode validation (C-gate round 1, blocker 1). */
|
|
71
|
+
validate?: ValidateFn;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export type EncodeFn = (
|
|
75
|
+
input: Uint8Array,
|
|
76
|
+
spec: TierSpec,
|
|
77
|
+
quality: number,
|
|
78
|
+
) => Promise<{ data: string; mediaType: string }>;
|
|
79
|
+
|
|
80
|
+
/** Proves the payload fully decodes; must throw for corrupt/truncated data. */
|
|
81
|
+
export type ValidateFn = (input: Uint8Array) => Promise<void>;
|
|
82
|
+
|
|
83
|
+
type ProcessResult =
|
|
84
|
+
| { kind: "pass"; b64Length: number }
|
|
85
|
+
| { kind: "encoded"; data: string; mediaType: string }
|
|
86
|
+
| { kind: "failed" };
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Byte-weighted LRU over normalized outputs (audit round 1, blocker 2): aggregate cap,
|
|
90
|
+
* not entry count. Entries are immutable snapshots — demotions write NEW tier-suffixed
|
|
91
|
+
* keys, never mutate stored values.
|
|
92
|
+
*/
|
|
93
|
+
const CACHE_BYTE_CAP = 64 * MiB;
|
|
94
|
+
// "pass" = validated pass-through; "miss" = this position's ladder cannot meet its hard
|
|
95
|
+
// cap for these bytes (skip straight to the next position — C-gate round 2, blocker 1).
|
|
96
|
+
const cache = new Map<string, { data: string; mediaType: string } | "pass" | "miss">();
|
|
97
|
+
let cacheBytes = 0;
|
|
98
|
+
let encodeCalls = 0;
|
|
99
|
+
|
|
100
|
+
function cachePut(key: string, value: { data: string; mediaType: string } | "pass" | "miss"): void {
|
|
101
|
+
const size = typeof value === "string" ? 0 : value.data.length;
|
|
102
|
+
const existing = cache.get(key);
|
|
103
|
+
if (existing !== undefined) {
|
|
104
|
+
cacheBytes -= typeof existing === "string" ? 0 : existing.data.length;
|
|
105
|
+
cache.delete(key); // re-insert refreshes recency and prevents double-count on concurrent misses
|
|
106
|
+
}
|
|
107
|
+
while (cacheBytes + size > CACHE_BYTE_CAP && cache.size > 0) {
|
|
108
|
+
const oldest = cache.keys().next().value as string;
|
|
109
|
+
const evicted = cache.get(oldest);
|
|
110
|
+
cacheBytes -= typeof evicted === "string" || evicted === undefined ? 0 : evicted.data.length;
|
|
111
|
+
cache.delete(oldest);
|
|
112
|
+
}
|
|
113
|
+
cache.set(key, value);
|
|
114
|
+
cacheBytes += size;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Read a cache entry, refreshing its recency (true LRU, C-gate round 1 blocker 5). */
|
|
118
|
+
function cacheGet(key: string): { data: string; mediaType: string } | "pass" | "miss" | undefined {
|
|
119
|
+
const value = cache.get(key);
|
|
120
|
+
if (value !== undefined) {
|
|
121
|
+
cache.delete(key);
|
|
122
|
+
cache.set(key, value);
|
|
123
|
+
}
|
|
124
|
+
return value;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Test hooks: encoder-invocation counter + cache reset (no production caller). */
|
|
128
|
+
export function getNormalizeStatsForTests(): { encodeCalls: number; cacheEntries: number; cacheBytes: number } {
|
|
129
|
+
return { encodeCalls, cacheEntries: cache.size, cacheBytes };
|
|
130
|
+
}
|
|
131
|
+
export function resetNormalizeStateForTests(): void {
|
|
132
|
+
cache.clear();
|
|
133
|
+
cacheBytes = 0;
|
|
134
|
+
encodeCalls = 0;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Default encoder: Bun.Image resize-to-fit + JPEG at the given quality. */
|
|
138
|
+
const bunImageEncode: EncodeFn = async (input, spec, quality) => {
|
|
139
|
+
const image = new Bun.Image(input);
|
|
140
|
+
const meta = await image.metadata();
|
|
141
|
+
const w = typeof meta.width === "number" ? meta.width : 0;
|
|
142
|
+
const h = typeof meta.height === "number" ? meta.height : 0;
|
|
143
|
+
let pipeline = new Bun.Image(input);
|
|
144
|
+
if (w > spec.maxEdge || h > spec.maxEdge) {
|
|
145
|
+
const scale = spec.maxEdge / Math.max(w, h);
|
|
146
|
+
pipeline = pipeline.resize(Math.max(1, Math.round(w * scale)), Math.max(1, Math.round(h * scale)));
|
|
147
|
+
}
|
|
148
|
+
const out = await pipeline.jpeg({ quality }).toBuffer();
|
|
149
|
+
return { data: Buffer.from(out).toString("base64"), mediaType: "image/jpeg" };
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Default pass-through validation: force a full decode (resize forces pixel decoding, a
|
|
154
|
+
* header-only metadata read does not). A sniffable-but-truncated payload must throw here
|
|
155
|
+
* instead of riding pass-through to an Anthropic 400 (C-gate round 1, blocker 1).
|
|
156
|
+
*/
|
|
157
|
+
const bunImageValidate: ValidateFn = async input => {
|
|
158
|
+
await new Bun.Image(input).resize(1, 1).jpeg({ quality: 1 }).toBuffer();
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
function mediaTypeOf(ref: ImageBlockRef): string {
|
|
162
|
+
const block = ref.container[ref.index] as { source?: { media_type?: unknown } } | undefined;
|
|
163
|
+
const mt = block?.source?.media_type;
|
|
164
|
+
return typeof mt === "string" ? mt.toLowerCase() : "";
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function textify(ref: ImageBlockRef, text: string): void {
|
|
168
|
+
ref.container[ref.index] = { type: "text", text };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function replaceImage(ref: ImageBlockRef, data: string, mediaType: string): void {
|
|
172
|
+
ref.container[ref.index] = { type: "image", source: { type: "base64", media_type: mediaType, data } };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function initialPosition(newestFirstIndex: number, bias: number): number {
|
|
176
|
+
const base = newestFirstIndex < TIER0_COUNT ? 0 : newestFirstIndex < TIER0_COUNT + TIER1_COUNT ? 1 : 2;
|
|
177
|
+
return Math.min(base + Math.max(0, bias), TERMINAL_POS);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Process one image at a ladder position: pass through when it already fits the
|
|
182
|
+
* position's caps (Anthropic-native format, dims within maxEdge, size within hardCap —
|
|
183
|
+
* this also exempts possibly-animated GIF/WebP from a lossy re-encode; pass-through is
|
|
184
|
+
* additionally VALIDATED with a full decode once, cached), otherwise walk positions
|
|
185
|
+
* downward encoding until a hard cap is met; terminal accepts measured size.
|
|
186
|
+
* `mediaType` must be the ORIGINAL source media type (cache keys include it — C-gate
|
|
187
|
+
* round 1, blocker 4 — and pass-through eligibility depends on it).
|
|
188
|
+
*/
|
|
189
|
+
async function processAt(
|
|
190
|
+
b64: string,
|
|
191
|
+
startPos: number,
|
|
192
|
+
mediaType: string,
|
|
193
|
+
encode: EncodeFn,
|
|
194
|
+
validate: ValidateFn,
|
|
195
|
+
): Promise<ProcessResult & { pos: number }> {
|
|
196
|
+
const dims = sniffImageDimensions(b64);
|
|
197
|
+
const hash = Bun.hash(b64).toString(36);
|
|
198
|
+
let input: Uint8Array;
|
|
199
|
+
try {
|
|
200
|
+
input = Uint8Array.from(Buffer.from(b64, "base64"));
|
|
201
|
+
} catch {
|
|
202
|
+
return { kind: "failed", pos: startPos };
|
|
203
|
+
}
|
|
204
|
+
for (let pos = startPos; pos <= TERMINAL_POS; pos++) {
|
|
205
|
+
const spec = TIER_SPECS[pos];
|
|
206
|
+
const key = `${hash}:${mediaType}:${pos}`;
|
|
207
|
+
const cached = cacheGet(key);
|
|
208
|
+
if (cached === "pass") return { kind: "pass", b64Length: b64.length, pos };
|
|
209
|
+
if (cached === "miss") continue; // known cap miss: skip to the next position
|
|
210
|
+
if (cached) return { kind: "encoded", data: cached.data, mediaType: cached.mediaType, pos };
|
|
211
|
+
|
|
212
|
+
const fitsDims = dims !== null && dims.width <= spec.maxEdge && dims.height <= spec.maxEdge;
|
|
213
|
+
if (PASSTHROUGH_MEDIA.has(mediaType) && fitsDims && b64.length <= spec.hardCap) {
|
|
214
|
+
try {
|
|
215
|
+
await validate(input); // sniffable-but-truncated data must not ride pass-through
|
|
216
|
+
} catch {
|
|
217
|
+
return { kind: "failed", pos };
|
|
218
|
+
}
|
|
219
|
+
cachePut(key, "pass");
|
|
220
|
+
return { kind: "pass", b64Length: b64.length, pos };
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
let last: { data: string; mediaType: string } | null = null;
|
|
224
|
+
try {
|
|
225
|
+
for (const quality of spec.qualities) {
|
|
226
|
+
encodeCalls++;
|
|
227
|
+
last = await encode(input, spec, quality);
|
|
228
|
+
if (last.data.length <= spec.hardCap) {
|
|
229
|
+
cachePut(key, last);
|
|
230
|
+
return { kind: "encoded", data: last.data, mediaType: last.mediaType, pos };
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
} catch {
|
|
234
|
+
// Decode/encode failure: corrupt or unsupported payload (audit round 2, blocker 2).
|
|
235
|
+
return { kind: "failed", pos };
|
|
236
|
+
}
|
|
237
|
+
if (pos === TERMINAL_POS && last) {
|
|
238
|
+
cachePut(key, last);
|
|
239
|
+
return { kind: "encoded", data: last.data, mediaType: last.mediaType, pos };
|
|
240
|
+
}
|
|
241
|
+
// Hard cap missed at this position — remember the miss, continue down the ladder.
|
|
242
|
+
cachePut(key, "miss");
|
|
243
|
+
}
|
|
244
|
+
return { kind: "failed", pos: TERMINAL_POS };
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Wire-neutral image handle (devlog/260714_image_normalization_pipeline/050): the core
|
|
249
|
+
* algorithm below normalizes THROUGH this interface so non-Anthropic wire shapes (kiro
|
|
250
|
+
* CodeWhisperer) reuse the exact same tier/cache/demotion machinery. `mediaType` is the
|
|
251
|
+
* canonical lowercased MIME ("image/<format>") — cache identity and pass-through
|
|
252
|
+
* decisions depend on it; wire-specific conversions live inside `replace`.
|
|
253
|
+
*/
|
|
254
|
+
export interface NormalizeTarget {
|
|
255
|
+
base64: string | null;
|
|
256
|
+
mediaType: string;
|
|
257
|
+
replace(data: string, mediaType: string): void;
|
|
258
|
+
drop(note: string): void;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export interface NormalizeTargetsOptions extends NormalizeOptions {
|
|
262
|
+
/** Total base64 budget across all targets. Default: TOTAL_IMAGE_BASE64_BUDGET. */
|
|
263
|
+
budget?: number;
|
|
264
|
+
/**
|
|
265
|
+
* What to do when every image is terminal-floored and the sum still exceeds budget:
|
|
266
|
+
* "none" (anthropic — the guard's Rule 4 backstop textifies downstream) or "drop"
|
|
267
|
+
* (kiro — no downstream guard exists, so drop OLDEST targets here until it fits).
|
|
268
|
+
*/
|
|
269
|
+
overflowAction?: "none" | "drop";
|
|
270
|
+
/** Only the newest N images are processed (older ones skipped). Default: unlimited. */
|
|
271
|
+
processLimit?: number;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Core normalization over wire-neutral targets (mutates via target callbacks).
|
|
276
|
+
* Null-base64 targets (URL/file sources) pass through untouched.
|
|
277
|
+
*/
|
|
278
|
+
export async function normalizeImageTargets(targets: NormalizeTarget[], options: NormalizeTargetsOptions = {}): Promise<void> {
|
|
279
|
+
if (targets.length === 0) return;
|
|
280
|
+
const encode = options.encode ?? bunImageEncode;
|
|
281
|
+
const validate = options.validate ?? bunImageValidate;
|
|
282
|
+
const bias = options.tierBias ?? 0;
|
|
283
|
+
const budget = options.budget ?? TOTAL_IMAGE_BASE64_BUDGET;
|
|
284
|
+
const overflowAction = options.overflowAction ?? "none";
|
|
285
|
+
const processLimit = options.processLimit ?? Number.POSITIVE_INFINITY;
|
|
286
|
+
const n = targets.length;
|
|
287
|
+
|
|
288
|
+
// sourceB64/sourceMedia are the ORIGINAL input (encode source + cache identity);
|
|
289
|
+
// size always reflects the bytes currently ON the wire for this target (the core is
|
|
290
|
+
// the only mutator, so tracked size cannot drift from reality).
|
|
291
|
+
interface Entry { target: NormalizeTarget; sourceB64: string; sourceMedia: string; pos: number; size: number; done: boolean }
|
|
292
|
+
const entries: (Entry | null)[] = new Array(n).fill(null);
|
|
293
|
+
|
|
294
|
+
for (let i = 0; i < n; i++) {
|
|
295
|
+
const target = targets[i];
|
|
296
|
+
const b64 = target.base64;
|
|
297
|
+
if (!b64) continue; // URL source: no base64 weight, never touched here.
|
|
298
|
+
const newestFirstIndex = n - 1 - i;
|
|
299
|
+
// Images beyond the processing limit are left untouched (anthropic passes 100:
|
|
300
|
+
// its guard textifies the surplus anyway, so decode/encode work there is waste).
|
|
301
|
+
if (newestFirstIndex >= processLimit) continue;
|
|
302
|
+
if (b64.length > MAX_INPUT_BASE64_LENGTH) {
|
|
303
|
+
target.drop(BOMB_TEXT);
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
const dims = sniffImageDimensions(b64);
|
|
307
|
+
if (dims && dims.width * dims.height > MAX_INPUT_PIXELS) {
|
|
308
|
+
target.drop(BOMB_TEXT);
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
const sourceMedia = target.mediaType.toLowerCase();
|
|
312
|
+
const pos = initialPosition(newestFirstIndex, bias);
|
|
313
|
+
const result = await processAt(b64, pos, sourceMedia, encode, validate);
|
|
314
|
+
if (result.kind === "failed") {
|
|
315
|
+
target.drop(UNDECODABLE_TEXT);
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
let size = b64.length;
|
|
319
|
+
if (result.kind === "encoded") {
|
|
320
|
+
target.replace(result.data, result.mediaType);
|
|
321
|
+
size = result.data.length;
|
|
322
|
+
}
|
|
323
|
+
entries[i] = { target, sourceB64: b64, sourceMedia, pos: result.pos, size, done: result.pos >= TERMINAL_POS };
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Aggregate demotion loop (audit rounds 1+3): while the measured total exceeds the
|
|
327
|
+
// budget, demote the OLDEST not-yet-terminal image one position and re-encode.
|
|
328
|
+
let sum = 0;
|
|
329
|
+
for (const e of entries) if (e) sum += e.size;
|
|
330
|
+
while (sum > budget) {
|
|
331
|
+
const entry = entries.find((e): e is Entry => e !== null && !e.done);
|
|
332
|
+
if (!entry) break; // all terminal — overflowAction below decides
|
|
333
|
+
const result = await processAt(entry.sourceB64, entry.pos + 1, entry.sourceMedia, encode, validate);
|
|
334
|
+
if (result.kind === "failed") {
|
|
335
|
+
entry.target.drop(UNDECODABLE_TEXT);
|
|
336
|
+
sum -= entry.size;
|
|
337
|
+
entries[entries.indexOf(entry)] = null;
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
let newSize = entry.size;
|
|
341
|
+
if (result.kind === "encoded") {
|
|
342
|
+
entry.target.replace(result.data, result.mediaType);
|
|
343
|
+
newSize = result.data.length;
|
|
344
|
+
} else {
|
|
345
|
+
newSize = result.b64Length; // pass leaves current bytes (only reachable for never-encoded entries)
|
|
346
|
+
}
|
|
347
|
+
sum += newSize - entry.size;
|
|
348
|
+
entry.size = newSize;
|
|
349
|
+
entry.pos = result.pos;
|
|
350
|
+
entry.done = result.pos >= TERMINAL_POS;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Terminal overflow (050 audit round 1, blocker 3): with no downstream guard, drop
|
|
354
|
+
// OLDEST targets until the sum fits.
|
|
355
|
+
if (overflowAction === "drop") {
|
|
356
|
+
for (let i = 0; i < entries.length && sum > budget; i++) {
|
|
357
|
+
const e = entries[i];
|
|
358
|
+
if (!e) continue;
|
|
359
|
+
e.target.drop(OVERFLOW_DROP_TEXT);
|
|
360
|
+
sum -= e.size;
|
|
361
|
+
entries[i] = null;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Normalize every base64 image in already-built Anthropic wire messages (mutates in
|
|
368
|
+
* place). URL-source images pass through untouched. See module header for the contract.
|
|
369
|
+
*/
|
|
370
|
+
export async function normalizeAnthropicImages(messages: unknown[], options: NormalizeOptions = {}): Promise<void> {
|
|
371
|
+
const refs = collectImageRefs(messages);
|
|
372
|
+
if (refs.length === 0) return;
|
|
373
|
+
const targets: NormalizeTarget[] = refs.map(ref => ({
|
|
374
|
+
base64: ref.base64,
|
|
375
|
+
mediaType: mediaTypeOf(ref),
|
|
376
|
+
replace: (data: string, mediaType: string) => replaceImage(ref, data, mediaType),
|
|
377
|
+
drop: (note: string) => textify(ref, note),
|
|
378
|
+
}));
|
|
379
|
+
// Anthropic hard-caps 100 images/request and its guard textifies the surplus, so
|
|
380
|
+
// processing beyond the newest 100 is pure waste; terminal overflow stays with the
|
|
381
|
+
// guard's Rule 4 backstop (overflowAction "none").
|
|
382
|
+
await normalizeImageTargets(targets, { ...options, processLimit: 100, overflowAction: "none" });
|
|
383
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ProviderAdapter } from "./base";
|
|
1
|
+
import type { IncomingMeta, ProviderAdapter } from "./base";
|
|
2
2
|
import { debugDroppedFrame } from "../lib/debug";
|
|
3
3
|
import type {
|
|
4
4
|
AdapterEvent,
|
|
@@ -17,6 +17,7 @@ import { isAllowedToolChoice, namespacedToolName, resolveToolChoiceWireName, too
|
|
|
17
17
|
import { ANTHROPIC_OAUTH_BETA, CLAUDE_CODE_SYSTEM_INSTRUCTION, applyClaudeToolPrefix, stripClaudeToolPrefix } from "../oauth/anthropic";
|
|
18
18
|
import { parseDataUrl } from "./image";
|
|
19
19
|
import { enforceAnthropicImageLimits } from "./anthropic-image-guard";
|
|
20
|
+
import { normalizeAnthropicImages } from "./anthropic-image-normalize";
|
|
20
21
|
import { neutralizeIdentity } from "./identity";
|
|
21
22
|
import { CLAUDE_CODE_HEADERS, claudeCodeSessionId } from "./client-fingerprint";
|
|
22
23
|
import { buildNonOpenAIToolCatalogNudgeForTools } from "./tool-catalog-nudge";
|
|
@@ -584,7 +585,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti
|
|
|
584
585
|
return {
|
|
585
586
|
name: "anthropic",
|
|
586
587
|
|
|
587
|
-
buildRequest(parsed: OcxParsedRequest) {
|
|
588
|
+
async buildRequest(parsed: OcxParsedRequest, incoming?: IncomingMeta) {
|
|
588
589
|
if (typeof provider.apiKey !== "string" || provider.apiKey.trim() === "") {
|
|
589
590
|
if (isOAuth) {
|
|
590
591
|
throw new Error("anthropic oauth token missing — run ocx login anthropic");
|
|
@@ -593,6 +594,10 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti
|
|
|
593
594
|
}
|
|
594
595
|
|
|
595
596
|
const { system, messages } = messagesToAnthropicFormat(parsed, toolNames);
|
|
597
|
+
// Primary image layer: resize/re-encode to fit Anthropic limits without dropping
|
|
598
|
+
// (anthropic-image-normalize.ts); the guard below remains the deterministic backstop.
|
|
599
|
+
// imageTierBias > 0 = upstream-413 tightened retry (030): start every image one tier lower.
|
|
600
|
+
await normalizeAnthropicImages(messages, { tierBias: incoming?.imageTierBias ?? 0 });
|
|
596
601
|
// Anthropic rejects many-image requests (>20 images) carrying any image over
|
|
597
602
|
// 2000px per side; see anthropic-image-guard.ts for the full limit policy.
|
|
598
603
|
enforceAnthropicImageLimits(messages);
|
package/src/adapters/base.ts
CHANGED
|
@@ -4,6 +4,12 @@ import type { AdapterEvent, OcxParsedRequest } from "../types";
|
|
|
4
4
|
export interface IncomingMeta {
|
|
5
5
|
headers: Headers;
|
|
6
6
|
abortSignal?: AbortSignal;
|
|
7
|
+
/**
|
|
8
|
+
* Image-normalization ladder bias for upstream-413 tightened retries: every image
|
|
9
|
+
* starts one tier lower (devlog/260714_image_normalization_pipeline/030). Only the
|
|
10
|
+
* anthropic adapter consumes it; others ignore it.
|
|
11
|
+
*/
|
|
12
|
+
imageTierBias?: number;
|
|
7
13
|
}
|
|
8
14
|
|
|
9
15
|
export interface ProviderAdapter {
|
|
@@ -6,7 +6,15 @@ export type CursorNativeExecMode = "off" | "codex-sandbox" | "on";
|
|
|
6
6
|
/** Codex permissions template marker, e.g. "`sandbox_mode` is `danger-full-access`". */
|
|
7
7
|
export const CURSOR_SANDBOX_FULL_ACCESS_RE = /sandbox_mode[^\n]{0,80}danger-full-access/i;
|
|
8
8
|
|
|
9
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* Config-owner-selected policy; explicit mode wins, legacy boolean maps to "on".
|
|
11
|
+
* The UNSET default is "codex-sandbox": native local exec is APPROVED for requests that
|
|
12
|
+
* declare the Codex danger-full-access sandbox (the normal full-access Codex flow — "approve
|
|
13
|
+
* most") and DENIED for requests that do not. Set `nativeLocalExec: "off"` to deny all, or
|
|
14
|
+
* "on" to always allow. Legacy `unsafeAllowNativeLocalExec: true` still maps to "on".
|
|
15
|
+
* Security note: codex-sandbox trusts a caller-controlled full-access marker the proxy cannot
|
|
16
|
+
* verify, and the auth-free loopback bind admits any local process — see the src/types.ts doc.
|
|
17
|
+
*/
|
|
10
18
|
export function resolveCursorNativeExecMode(provider: OcxProviderConfig): CursorNativeExecMode {
|
|
11
19
|
const mode = provider.nativeLocalExec;
|
|
12
20
|
if (mode === "off" || mode === "codex-sandbox" || mode === "on") return mode;
|
|
@@ -15,14 +15,14 @@ import {
|
|
|
15
15
|
CreatePlanResultSchema,
|
|
16
16
|
CreatePlanSuccessSchema,
|
|
17
17
|
ExaFetchRequestResponseSchema,
|
|
18
|
-
|
|
18
|
+
ExaFetchRequestResponse_ApprovedSchema,
|
|
19
19
|
ExaSearchRequestResponseSchema,
|
|
20
|
-
|
|
20
|
+
ExaSearchRequestResponse_ApprovedSchema,
|
|
21
21
|
InteractionResponseSchema,
|
|
22
22
|
SwitchModeRequestResponseSchema,
|
|
23
23
|
SwitchModeRequestResponse_RejectedSchema,
|
|
24
24
|
WebSearchRequestResponseSchema,
|
|
25
|
-
|
|
25
|
+
WebSearchRequestResponse_ApprovedSchema,
|
|
26
26
|
type AgentServerMessage,
|
|
27
27
|
type ExecServerMessage,
|
|
28
28
|
type InteractionQuery,
|
|
@@ -181,8 +181,16 @@ export function planMcpArgsHandling(
|
|
|
181
181
|
* Codex as visible output so the user still sees it.
|
|
182
182
|
* - askQuestion: reject with a reason — the agent must proceed autonomously; there is no human to
|
|
183
183
|
* answer mid-turn. (Future: bridge to a Codex user-input request.)
|
|
184
|
-
* -
|
|
185
|
-
*
|
|
184
|
+
* - webSearch / exaSearch / exaFetch: APPROVE (empty approval). These are approve/reject
|
|
185
|
+
* permission gates, not client-run requests — the response schema has no result field, so
|
|
186
|
+
* approval delegates the search to Cursor's SERVER, which runs it and injects results into the
|
|
187
|
+
* model server-side (the answer then streams back as textDelta; the display-plane
|
|
188
|
+
* web_search_tool_call/exa_*_tool_call result frames are native, non-mcp, and safely dropped by
|
|
189
|
+
* the event mapper). Rejecting them (the old default) killed the model's web capability on the
|
|
190
|
+
* Cursor path. Tradeoff: approval consumes the user's Cursor web-search/Exa quota. The synthetic
|
|
191
|
+
* web_search sidecar (src/web-search) is an orthogonal proxy-side path used only when the client
|
|
192
|
+
* sends a hosted web_search tool; it does not cover Cursor-native web search.
|
|
193
|
+
* - switchMode: reject (deterministic default; no non-interactive mode switch).
|
|
186
194
|
* - setupVmEnvironment: the result schema has no error case — reply success so the agent is not
|
|
187
195
|
* left waiting; the command itself was never run locally.
|
|
188
196
|
* Pure (no I/O) for unit testing; `handleServerMessage` writes the frame and emits liveness.
|
|
@@ -240,10 +248,10 @@ export function planInteractionQueryReply(query: InteractionQuery): { response:
|
|
|
240
248
|
response: respond({
|
|
241
249
|
case: "webSearchRequestResponse",
|
|
242
250
|
value: create(WebSearchRequestResponseSchema, {
|
|
243
|
-
result: { case: "
|
|
251
|
+
result: { case: "approved", value: create(WebSearchRequestResponse_ApprovedSchema, {}) },
|
|
244
252
|
}),
|
|
245
253
|
}),
|
|
246
|
-
replyCase: "webSearchRequestResponse:
|
|
254
|
+
replyCase: "webSearchRequestResponse:approved",
|
|
247
255
|
};
|
|
248
256
|
}
|
|
249
257
|
if (q.case === "exaSearchRequestQuery") {
|
|
@@ -251,10 +259,10 @@ export function planInteractionQueryReply(query: InteractionQuery): { response:
|
|
|
251
259
|
response: respond({
|
|
252
260
|
case: "exaSearchRequestResponse",
|
|
253
261
|
value: create(ExaSearchRequestResponseSchema, {
|
|
254
|
-
result: { case: "
|
|
262
|
+
result: { case: "approved", value: create(ExaSearchRequestResponse_ApprovedSchema, {}) },
|
|
255
263
|
}),
|
|
256
264
|
}),
|
|
257
|
-
replyCase: "exaSearchRequestResponse:
|
|
265
|
+
replyCase: "exaSearchRequestResponse:approved",
|
|
258
266
|
};
|
|
259
267
|
}
|
|
260
268
|
if (q.case === "exaFetchRequestQuery") {
|
|
@@ -262,10 +270,10 @@ export function planInteractionQueryReply(query: InteractionQuery): { response:
|
|
|
262
270
|
response: respond({
|
|
263
271
|
case: "exaFetchRequestResponse",
|
|
264
272
|
value: create(ExaFetchRequestResponseSchema, {
|
|
265
|
-
result: { case: "
|
|
273
|
+
result: { case: "approved", value: create(ExaFetchRequestResponse_ApprovedSchema, {}) },
|
|
266
274
|
}),
|
|
267
275
|
}),
|
|
268
|
-
replyCase: "exaFetchRequestResponse:
|
|
276
|
+
replyCase: "exaFetchRequestResponse:approved",
|
|
269
277
|
};
|
|
270
278
|
}
|
|
271
279
|
if (q.case === "setupVmEnvironmentArgs") {
|