humanish 0.146.0 → 0.147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/actors/computer-use/loop/spend.js +5 -4
- package/dist/actors/computer-use/loop/trace.js +3 -0
- package/dist/actors/computer-use/loop/types.d.ts +5 -0
- package/dist/actors/computer-use/loop/usage.js +10 -2
- package/dist/actors/computer-use/openai-conversation.d.ts +7 -0
- package/dist/actors/computer-use/openai-conversation.js +19 -2
- package/dist/actors/computer-use/openai-provider.d.ts +2 -0
- package/dist/actors/computer-use/openai-provider.js +2 -0
- package/dist/actors/computer-use/openai-wire.d.ts +4 -0
- package/dist/actors/computer-use/openai-wire.js +9 -1
- package/dist/actors/contract.d.ts +5 -0
- package/dist/analysis/evidence.js +8 -4
- package/dist/analysis/image-tokens.js +9 -3
- package/dist/evidence/image.d.ts +2 -0
- package/dist/evidence/image.js +24 -3
- package/dist/evidence/webp-encode-worker.d.ts +1 -0
- package/dist/evidence/webp-encode-worker.js +31 -0
- package/dist/evidence/webp-transcoder.d.ts +15 -0
- package/dist/evidence/webp-transcoder.js +98 -0
- package/dist/evidence/webp.d.ts +31 -0
- package/dist/evidence/webp.js +94 -0
- package/dist/feedback/derivative-text.d.ts +7 -2
- package/dist/feedback/derivative-text.js +20 -1
- package/dist/feedback/export-bundle.js +79 -25
- package/dist/remote/launch.js +9 -4
- package/dist/routes/computer-use/participant-execution.d.ts +0 -7
- package/dist/routes/computer-use/participant-execution.js +6 -28
- package/dist/routes/computer-use/participant-frames.d.ts +32 -0
- package/dist/routes/computer-use/participant-frames.js +93 -0
- package/dist/routes/computer-use/participant-model.js +3 -0
- package/dist/routes/computer-use/participant-runs.js +4 -0
- package/dist/routes/computer-use/plan.js +5 -1
- package/dist/routes/computer-use/result.js +6 -0
- package/dist/routes/computer-use/types.d.ts +4 -0
- package/dist/routes/shared-world/plan.js +4 -1
- package/dist/run/pricing.d.ts +7 -0
- package/dist/run/pricing.js +58 -15
- package/dist/study/config.js +4 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -0
- package/dist/study/parse/actors.js +5 -0
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +2 -0
- package/dist/study/types.d.ts +7 -0
- package/dist/study/validation.d.ts +5 -0
- package/dist/study/validation.js +23 -0
- package/dist/verify/artifacts.js +2 -1
- package/docs/contracts/run-bundle.md +29 -11
- package/docs/contracts/schemas.md +19 -11
- package/docs/contracts/study-analysis.md +2 -2
- package/package.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,16 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.147.0: Screenshots take half the space (2026-10-25)
|
|
12
|
+
|
|
13
|
+
humanish 0.147.0 keeps a computer-use participant's screenshots as lossless WebP, 0.47 to 0.55 of
|
|
14
|
+
the PNG size with the same pixels, and `verify`, study analysis and export read them; earlier
|
|
15
|
+
humanish versions refuse a bundle with WebP screenshots. `actor.serviceTier: fast` runs OpenAI
|
|
16
|
+
computer-use participants in Fast mode. On the Taskly benchmark a Fast mode step took 14% less
|
|
17
|
+
time at 2.15 times the participant cost, and each request is priced at the tier that served it.
|
|
18
|
+
|
|
19
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.147.0)
|
|
20
|
+
|
|
11
21
|
## 0.146.0: Studies start and finish sooner (2026-10-23)
|
|
12
22
|
|
|
13
23
|
humanish 0.146.0 cuts the wait around a hosted study. A Chrome participant starts once its page
|
|
@@ -58,11 +58,12 @@ export function bookLostRequest(session, turnNumber, lost) {
|
|
|
58
58
|
}
|
|
59
59
|
const input = session.usage.lastReportedInput();
|
|
60
60
|
session.usage.reserve(input, maxOutputTokens);
|
|
61
|
+
const serviceTier = session.provider.modelSettings?.serviceTier;
|
|
62
|
+
const booked = { input, output: maxOutputTokens, cachedInput: 0, cacheWriteInput: input };
|
|
61
63
|
const usd = session.settings.estimateTurnCostUsd?.({
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
cacheWriteInput: input,
|
|
64
|
+
...booked,
|
|
65
|
+
// A request in Fast mode is booked at the fast rate, which a per-request record selects.
|
|
66
|
+
...(serviceTier === undefined ? {} : { turns: [{ ...booked, serviceTier }] }),
|
|
66
67
|
});
|
|
67
68
|
const charge = `${input} input tokens at the highest input rate and ${maxOutputTokens} output tokens${typeof usd === "number" ? ` ($${usd})` : ""}`;
|
|
68
69
|
const over = spendStop(session);
|
|
@@ -127,6 +127,9 @@ export function loopResult(session, stop, debrief) {
|
|
|
127
127
|
...(provider.modelSettings.maxOutputTokens === undefined
|
|
128
128
|
? {}
|
|
129
129
|
: { maxOutputTokens: provider.modelSettings.maxOutputTokens }),
|
|
130
|
+
...(provider.modelSettings.serviceTier === undefined
|
|
131
|
+
? {}
|
|
132
|
+
: { serviceTier: provider.modelSettings.serviceTier }),
|
|
130
133
|
},
|
|
131
134
|
}),
|
|
132
135
|
waitSettings: {
|
|
@@ -191,12 +191,15 @@ export interface CuaTurn {
|
|
|
191
191
|
output?: number;
|
|
192
192
|
cachedInput?: number;
|
|
193
193
|
cacheWriteInput?: number;
|
|
194
|
+
/** The processing tier that served this request, when the session asked for one. */
|
|
195
|
+
serviceTier?: string;
|
|
194
196
|
/** Per model inference inside this provider interaction. */
|
|
195
197
|
turns?: Array<{
|
|
196
198
|
input?: number;
|
|
197
199
|
output?: number;
|
|
198
200
|
cachedInput?: number;
|
|
199
201
|
cacheWriteInput?: number;
|
|
202
|
+
serviceTier?: string;
|
|
200
203
|
}>;
|
|
201
204
|
};
|
|
202
205
|
/** True when the model reported a natural endpoint (no further action). */
|
|
@@ -234,6 +237,8 @@ export interface CuaProvider {
|
|
|
234
237
|
readonly modelSettings?: {
|
|
235
238
|
readonly reasoningEffort: ReasoningEffort;
|
|
236
239
|
readonly maxOutputTokens?: number;
|
|
240
|
+
/** The processing tier every request asks for; absent asks for none. */
|
|
241
|
+
readonly serviceTier?: "fast";
|
|
237
242
|
};
|
|
238
243
|
readonly capabilities: ActorCapabilities;
|
|
239
244
|
/**
|
|
@@ -14,12 +14,19 @@ function normalizedUsageTurns(usage) {
|
|
|
14
14
|
return undefined;
|
|
15
15
|
if ((raw.cachedInput ?? 0) + (raw.cacheWriteInput ?? 0) > (raw.input ?? 0))
|
|
16
16
|
return undefined;
|
|
17
|
-
turns.push(
|
|
17
|
+
turns.push({
|
|
18
|
+
...Object.fromEntries(fields.flatMap((field) => (raw[field] === undefined ? [] : [[field, raw[field]]]))),
|
|
19
|
+
...servedTier(raw.serviceTier),
|
|
20
|
+
});
|
|
18
21
|
}
|
|
19
22
|
return fields.every((field) => turns.reduce((sum, turn) => sum + (turn[field] ?? 0), 0) === (usage[field] ?? 0))
|
|
20
23
|
? turns
|
|
21
24
|
: undefined;
|
|
22
25
|
}
|
|
26
|
+
/** The served-tier field of a usage record, kept only when the provider named one. */
|
|
27
|
+
function servedTier(tier) {
|
|
28
|
+
return typeof tier === "string" && tier !== "" ? { serviceTier: tier } : {};
|
|
29
|
+
}
|
|
23
30
|
export function isCompleteTurnUsage(usage) {
|
|
24
31
|
return (usage !== undefined &&
|
|
25
32
|
validTokenCount(usage.input) &&
|
|
@@ -130,7 +137,7 @@ export class UsageLedger {
|
|
|
130
137
|
this.cacheWriteInput += usage.cacheWriteInput ?? 0;
|
|
131
138
|
this.output += usage.output ?? 0;
|
|
132
139
|
if (raw.turns === undefined)
|
|
133
|
-
this.turns.push(usage);
|
|
140
|
+
this.turns.push({ ...usage, ...servedTier(raw.serviceTier) });
|
|
134
141
|
else if (turns !== undefined)
|
|
135
142
|
this.turns.push(...turns);
|
|
136
143
|
}
|
|
@@ -218,6 +225,7 @@ export class UsageLedger {
|
|
|
218
225
|
output,
|
|
219
226
|
cachedInput: 0,
|
|
220
227
|
cacheWriteInput: input,
|
|
228
|
+
...servedTier(this.provider.modelSettings?.serviceTier),
|
|
221
229
|
}));
|
|
222
230
|
const sum = (field) => booked.reduce((total, turn) => total + turn[field], 0);
|
|
223
231
|
const input = (settled.input ?? 0) + sum("input");
|
|
@@ -8,6 +8,8 @@ export interface OpenAiConversationSettings {
|
|
|
8
8
|
readonly reasoningEffort: ReasoningEffort;
|
|
9
9
|
readonly maxOutputTokens?: number;
|
|
10
10
|
readonly safetyIdentifier?: string;
|
|
11
|
+
/** OpenAI Fast mode for every request of the session. */
|
|
12
|
+
readonly serviceTier?: "fast";
|
|
11
13
|
/** The summaries to ask for until the account or model rejects them; undefined asks for none. */
|
|
12
14
|
readonly reasoningSummary: OpenAiReasoningSummary | undefined;
|
|
13
15
|
/** Carry the conversation on the client from the first request (explicit_context). */
|
|
@@ -98,6 +100,11 @@ export declare class OpenAiConversation {
|
|
|
98
100
|
*/
|
|
99
101
|
private waitOutcome;
|
|
100
102
|
private context;
|
|
103
|
+
/**
|
|
104
|
+
* A reply, parsed. When the session asked for a tier, the reply's usage records the tier that
|
|
105
|
+
* served it, which is what the request bills at: Fast mode can fall back to `default`.
|
|
106
|
+
*/
|
|
107
|
+
private parse;
|
|
101
108
|
/**
|
|
102
109
|
* A participant request: the opening request, or a continuation carrying `sent`. A threaded
|
|
103
110
|
* continuation names the previous response; an explicit-context one carries the conversation
|
|
@@ -75,7 +75,7 @@ export class OpenAiConversation {
|
|
|
75
75
|
if (closing === "impressions")
|
|
76
76
|
this.evidence.requested(this.mode, this.carried, sent, closing);
|
|
77
77
|
const completed = asRecord(reply).status === "completed";
|
|
78
|
-
const parsed =
|
|
78
|
+
const parsed = this.parse(reply);
|
|
79
79
|
const account = completed ? closingAccount(parsed.turn.message, closing, strict) : undefined;
|
|
80
80
|
// A refusal would most likely stand under the strict schema too, so it is not asked again.
|
|
81
81
|
if (completed && account === undefined && !strict && !refused(reply)) {
|
|
@@ -164,6 +164,23 @@ export class OpenAiConversation {
|
|
|
164
164
|
...(settings.safetyIdentifier === undefined
|
|
165
165
|
? {}
|
|
166
166
|
: { safetyIdentifier: settings.safetyIdentifier }),
|
|
167
|
+
...(settings.serviceTier === undefined ? {} : { serviceTier: settings.serviceTier }),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* A reply, parsed. When the session asked for a tier, the reply's usage records the tier that
|
|
172
|
+
* served it, which is what the request bills at: Fast mode can fall back to `default`.
|
|
173
|
+
*/
|
|
174
|
+
parse(reply) {
|
|
175
|
+
const parsed = parseOpenAiResponse(reply);
|
|
176
|
+
const { usage } = parsed.turn;
|
|
177
|
+
if (this.settings.serviceTier === undefined || usage === undefined)
|
|
178
|
+
return parsed;
|
|
179
|
+
if (parsed.serviceTier === undefined)
|
|
180
|
+
return parsed;
|
|
181
|
+
return {
|
|
182
|
+
...parsed,
|
|
183
|
+
turn: { ...parsed.turn, usage: { ...usage, serviceTier: parsed.serviceTier } },
|
|
167
184
|
};
|
|
168
185
|
}
|
|
169
186
|
/**
|
|
@@ -204,7 +221,7 @@ export class OpenAiConversation {
|
|
|
204
221
|
// Recorded before the reply is accepted: accepting can cut the carried history, and the record
|
|
205
222
|
// counts what this request carried.
|
|
206
223
|
this.evidence.requested(this.mode, this.carried, sent);
|
|
207
|
-
const parsed =
|
|
224
|
+
const parsed = this.parse(reply);
|
|
208
225
|
if (parsed.turn.interruption === "output_limit")
|
|
209
226
|
return parsed.turn;
|
|
210
227
|
this.moveOn(parsed, sent);
|
|
@@ -68,6 +68,8 @@ export interface OpenAiResponsesProviderOptions {
|
|
|
68
68
|
reasoningEffort?: ReasoningEffort;
|
|
69
69
|
/** Optional positive integer output limit per response, including reasoning. Not a spend cap. */
|
|
70
70
|
maxOutputTokens?: number;
|
|
71
|
+
/** OpenAI Fast mode on every request; absent sends no `service_tier`. */
|
|
72
|
+
serviceTier?: "fast";
|
|
71
73
|
/**
|
|
72
74
|
* Reasoning-summary capture. Defaults to "auto" (the provider picks the best
|
|
73
75
|
* summarizer the model supports); "off" never asks. If the account/model rejects the
|
|
@@ -418,6 +418,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
418
418
|
...(options.safetyIdentifier === undefined
|
|
419
419
|
? {}
|
|
420
420
|
: { safetyIdentifier: options.safetyIdentifier }),
|
|
421
|
+
...(options.serviceTier === undefined ? {} : { serviceTier: options.serviceTier }),
|
|
421
422
|
reasoningSummary: options.reasoningSummary === "off" ? undefined : (options.reasoningSummary ?? "auto"),
|
|
422
423
|
zeroDataRetention: options.zeroDataRetention === true,
|
|
423
424
|
now: options.now ?? Date.now,
|
|
@@ -471,6 +472,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
471
472
|
modelSettings: {
|
|
472
473
|
reasoningEffort,
|
|
473
474
|
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
475
|
+
...(options.serviceTier === undefined ? {} : { serviceTier: options.serviceTier }),
|
|
474
476
|
},
|
|
475
477
|
capabilities: OPENAI_RESPONSES_CU_CAPABILITIES,
|
|
476
478
|
// This is a vision provider: nextTurn sends the screenshot as the computer_call_output, so
|
|
@@ -15,6 +15,8 @@ export interface ParsedOpenAiResponse {
|
|
|
15
15
|
/** The function calls, in reply order, each owed a function_call_output. */
|
|
16
16
|
functionCalls: FunctionCall[];
|
|
17
17
|
outputItems: unknown[];
|
|
18
|
+
/** The processing tier the reply says served it (`service_tier`), when it says one. */
|
|
19
|
+
serviceTier?: string;
|
|
18
20
|
}
|
|
19
21
|
/** A function call a reply made: its id, and its name, which is WAIT_TOOL_NAME unless it erred. */
|
|
20
22
|
export interface FunctionCall {
|
|
@@ -52,6 +54,8 @@ export interface OpenAiCuContext {
|
|
|
52
54
|
/** When set, request provider-sanctioned reasoning summaries. Absent = do not ask. */
|
|
53
55
|
reasoningSummary?: OpenAiReasoningSummary;
|
|
54
56
|
safetyIdentifier?: string;
|
|
57
|
+
/** OpenAI Fast mode (`service_tier: "fast"`). Absent sends no tier: the project's default. */
|
|
58
|
+
serviceTier?: "fast";
|
|
55
59
|
}
|
|
56
60
|
/**
|
|
57
61
|
* Build the first-turn request body: the instructions and the first screen, so the first decision
|
|
@@ -271,7 +271,14 @@ export function parseOpenAiResponse(raw) {
|
|
|
271
271
|
...(message.length > 0 ? { message } : {}),
|
|
272
272
|
...(usage === undefined ? {} : { usage }),
|
|
273
273
|
};
|
|
274
|
-
|
|
274
|
+
const serviceTier = optionalString(root.service_tier);
|
|
275
|
+
return {
|
|
276
|
+
turn,
|
|
277
|
+
callIds,
|
|
278
|
+
functionCalls,
|
|
279
|
+
outputItems: output,
|
|
280
|
+
...(serviceTier === undefined ? {} : { serviceTier }),
|
|
281
|
+
};
|
|
275
282
|
}
|
|
276
283
|
/**
|
|
277
284
|
* A function the participant names its wait length with; the computer tool's `wait` names none.
|
|
@@ -324,6 +331,7 @@ function sharedRequestFields(ctx) {
|
|
|
324
331
|
...(ctx.reasoningSummary === undefined ? {} : { summary: ctx.reasoningSummary }),
|
|
325
332
|
},
|
|
326
333
|
...(ctx.safetyIdentifier === undefined ? {} : { safety_identifier: ctx.safetyIdentifier }),
|
|
334
|
+
...(ctx.serviceTier === undefined ? {} : { service_tier: ctx.serviceTier }),
|
|
327
335
|
...(ctx.stateless === true ? { store: false, include: ["reasoning.encrypted_content"] } : {}),
|
|
328
336
|
};
|
|
329
337
|
}
|
|
@@ -187,6 +187,10 @@ export interface ActorTokenUsage {
|
|
|
187
187
|
cachedInput?: number;
|
|
188
188
|
cacheWriteInput?: number;
|
|
189
189
|
output?: number;
|
|
190
|
+
/** The processing tier the provider reported serving this request at (OpenAI `service_tier`:
|
|
191
|
+
* `default`, `priority`, `fast`). Recorded when the participant asked for a tier; Fast mode
|
|
192
|
+
* can fall back to `default`, which bills the standard rate. */
|
|
193
|
+
serviceTier?: string;
|
|
190
194
|
}>;
|
|
191
195
|
total?: number;
|
|
192
196
|
costUsd?: number;
|
|
@@ -364,6 +368,7 @@ export interface ActorTrace {
|
|
|
364
368
|
modelSettings?: {
|
|
365
369
|
reasoningEffort: string;
|
|
366
370
|
maxOutputTokens?: number;
|
|
371
|
+
serviceTier?: string;
|
|
367
372
|
};
|
|
368
373
|
/**
|
|
369
374
|
* Additive + optional: the wait lengths the computer-use loop applied, defaults included. Absent
|
|
@@ -348,13 +348,13 @@ function packEvidence(selected, packings, textByEntry, evidence, images) {
|
|
|
348
348
|
eventId: source.eventId,
|
|
349
349
|
path: source.capturePath,
|
|
350
350
|
sha256: sha256(bytes),
|
|
351
|
-
mimeType:
|
|
351
|
+
mimeType: captureMimeType(source.capturePath),
|
|
352
352
|
}
|
|
353
353
|
: null;
|
|
354
|
-
if (
|
|
354
|
+
if (capture)
|
|
355
355
|
images.push({
|
|
356
356
|
evidenceId: id,
|
|
357
|
-
dataUrl: `data
|
|
357
|
+
dataUrl: `data:${capture.mimeType};base64,${bytes.toString("base64")}`,
|
|
358
358
|
});
|
|
359
359
|
evidence.push({
|
|
360
360
|
id,
|
|
@@ -448,7 +448,7 @@ export async function validateAnalysisEvidence(prepared, artifact, bundleBytes)
|
|
|
448
448
|
if (source.capturePath === null ||
|
|
449
449
|
entry.capture.path !== source.capturePath ||
|
|
450
450
|
entry.capture.eventId !== source.eventId ||
|
|
451
|
-
entry.capture.mimeType !==
|
|
451
|
+
entry.capture.mimeType !== captureMimeType(source.capturePath))
|
|
452
452
|
throw new Error("ANALYSIS_CAPTURE_REFERENCE_INVALID");
|
|
453
453
|
let hash = checkedCaptures.get(source.capturePath);
|
|
454
454
|
if (hash === undefined) {
|
|
@@ -474,3 +474,7 @@ export async function validateAnalysisEvidence(prepared, artifact, bundleBytes)
|
|
|
474
474
|
if (!current?.equals(bundleBytes))
|
|
475
475
|
throw new Error("ANALYSIS_SOURCE_CHANGED");
|
|
476
476
|
}
|
|
477
|
+
/** A capture's image type, from the extension screenshotEvidenceError checked it against. */
|
|
478
|
+
function captureMimeType(capturePath) {
|
|
479
|
+
return capturePath.toLowerCase().endsWith(".webp") ? "image/webp" : "image/png";
|
|
480
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { WEBP_HEADER_BYTES, webpHeaderDimensions } from "../evidence/webp.js";
|
|
1
2
|
/** An input image the analysis providers accept; the groups are the image type and its base64 bytes. */
|
|
2
3
|
export const INPUT_IMAGE_DATA_URL = /^data:image\/(png|jpeg|webp);base64,([A-Za-z0-9+/]+={0,2})$/;
|
|
3
4
|
/**
|
|
@@ -36,8 +37,13 @@ const GEMINI_HIGH_RESOLUTION_IMAGE_TOKENS = Object.freeze({
|
|
|
36
37
|
/** The most one image can cost on `model`, whatever its size. */
|
|
37
38
|
export const imageTokenCeiling = (model) => GEMINI_HIGH_RESOLUTION_IMAGE_TOKENS[model] ?? HIGH_DETAIL_IMAGE_TOKEN_CEILING;
|
|
38
39
|
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
39
|
-
/**
|
|
40
|
-
|
|
40
|
+
/** Base64 characters that hold WEBP_HEADER_BYTES, rounded up to whole 3-byte groups. */
|
|
41
|
+
const WEBP_HEADER_CHARS = Math.ceil(WEBP_HEADER_BYTES / 3) * 4;
|
|
42
|
+
/** Width and height from a PNG data URL's IHDR chunk or a lossless WebP's VP8L header. */
|
|
43
|
+
function imageDimensions(dataUrl) {
|
|
44
|
+
const webp = new RegExp(`^data:image/webp;base64,([A-Za-z0-9+/]{${WEBP_HEADER_CHARS}})`).exec(dataUrl);
|
|
45
|
+
if (webp)
|
|
46
|
+
return webpHeaderDimensions(Buffer.from(webp[1], "base64"));
|
|
41
47
|
const match = /^data:image\/png;base64,([A-Za-z0-9+/]{32})/.exec(dataUrl);
|
|
42
48
|
if (!match)
|
|
43
49
|
return undefined;
|
|
@@ -53,7 +59,7 @@ export function highDetailImageTokens(model, dataUrl) {
|
|
|
53
59
|
if (fixed !== undefined)
|
|
54
60
|
return fixed;
|
|
55
61
|
const sizing = HIGH_DETAIL_SIZING[model];
|
|
56
|
-
const size = sizing === undefined ? undefined :
|
|
62
|
+
const size = sizing === undefined ? undefined : imageDimensions(dataUrl);
|
|
57
63
|
if (sizing === undefined || size === undefined)
|
|
58
64
|
return HIGH_DETAIL_IMAGE_TOKEN_CEILING;
|
|
59
65
|
const patches = Math.ceil(size.width / 32) * Math.ceil(size.height / 32);
|
package/dist/evidence/image.d.ts
CHANGED
|
@@ -22,3 +22,5 @@ export declare function pngDecodeRefusal(bytes: Buffer): string | null;
|
|
|
22
22
|
export declare function isRedactedFrameShape(bytes: Buffer): boolean;
|
|
23
23
|
export declare function screenshotEvidenceError(relativePath: string, bytes: Buffer): Promise<string | null>;
|
|
24
24
|
export declare function assertScreenshotEvidence(relativePath: string, bytes: Buffer): Promise<void>;
|
|
25
|
+
/** Why an image of these dimensions is refused before any decode, or null. */
|
|
26
|
+
export declare function imageDimensionsError(format: string, width: number, height: number): string | null;
|
package/dist/evidence/image.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { crc32, createInflate } from "node:zlib";
|
|
2
2
|
import { PNG } from "pngjs";
|
|
3
|
+
import { webpStructure } from "./webp.js";
|
|
3
4
|
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
4
5
|
const PNG_IHDR_LENGTH = 13;
|
|
5
6
|
// The pixel cap pngDecodeRefusal applies before any decode, for verify and the redactor alike.
|
|
@@ -130,8 +131,10 @@ export function isRedactedFrameShape(bytes) {
|
|
|
130
131
|
}
|
|
131
132
|
export async function screenshotEvidenceError(relativePath, bytes) {
|
|
132
133
|
const extension = relativePath.toLowerCase().split(".").pop() ?? "";
|
|
134
|
+
if (extension === "webp")
|
|
135
|
+
return webpEvidenceError(bytes);
|
|
133
136
|
if (extension !== "png") {
|
|
134
|
-
return `unsupported screenshot extension .${extension || "unknown"};
|
|
137
|
+
return `unsupported screenshot extension .${extension || "unknown"}; screenshot evidence is PNG or lossless WebP`;
|
|
135
138
|
}
|
|
136
139
|
const refusal = pngDecodeRefusal(bytes);
|
|
137
140
|
if (refusal)
|
|
@@ -150,6 +153,20 @@ export async function screenshotEvidenceError(relativePath, bytes) {
|
|
|
150
153
|
? "could not decode PNG evidence"
|
|
151
154
|
: null;
|
|
152
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Why a WebP frame is not screenshot evidence, read from its structure (webpStructure) without
|
|
158
|
+
* decoding it. A WebP frame is always raw evidence, which verify never grades share-ready, so a
|
|
159
|
+
* decode would add 17 to 20 ms a frame on the main thread and establish nothing more about bytes
|
|
160
|
+
* no one reviews.
|
|
161
|
+
*/
|
|
162
|
+
function webpEvidenceError(bytes) {
|
|
163
|
+
if (bytes.length > SCREENSHOT_MAX_BYTES)
|
|
164
|
+
return `WebP byte size exceeds ${SCREENSHOT_MAX_BYTES} byte limit`;
|
|
165
|
+
const structure = webpStructure(bytes);
|
|
166
|
+
if (structure.error !== null)
|
|
167
|
+
return structure.error;
|
|
168
|
+
return imageDimensionsError("WebP", structure.dimensions.width, structure.dimensions.height);
|
|
169
|
+
}
|
|
153
170
|
/** The bit depths PNG allows for each color type. */
|
|
154
171
|
const PNG_BIT_DEPTHS = {
|
|
155
172
|
0: [1, 2, 4, 8, 16],
|
|
@@ -254,11 +271,15 @@ export async function assertScreenshotEvidence(relativePath, bytes) {
|
|
|
254
271
|
}
|
|
255
272
|
}
|
|
256
273
|
function pngDimensionsError(width, height) {
|
|
274
|
+
return imageDimensionsError("PNG", width, height);
|
|
275
|
+
}
|
|
276
|
+
/** Why an image of these dimensions is refused before any decode, or null. */
|
|
277
|
+
export function imageDimensionsError(format, width, height) {
|
|
257
278
|
if (width === 0 || height === 0) {
|
|
258
|
-
return
|
|
279
|
+
return `${format} dimensions must be greater than zero`;
|
|
259
280
|
}
|
|
260
281
|
if (width * height > SCREENSHOT_MAX_SOURCE_PIXELS) {
|
|
261
|
-
return
|
|
282
|
+
return `${format} pixel count exceeds ${SCREENSHOT_MAX_SOURCE_PIXELS} pixel limit`;
|
|
262
283
|
}
|
|
263
284
|
return null;
|
|
264
285
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// The worker thread src/evidence/webp-transcoder.ts runs: PNG frames in, lossless WebP out, one at
|
|
2
|
+
// a time. It imports packages only, so Node can run this file from source with its types stripped
|
|
3
|
+
// as well as from dist.
|
|
4
|
+
import { readFile } from "node:fs/promises";
|
|
5
|
+
import { createRequire } from "node:module";
|
|
6
|
+
import { parentPort } from "node:worker_threads";
|
|
7
|
+
import encode, { init } from "@jsquash/webp/encode.js";
|
|
8
|
+
import { PNG } from "pngjs";
|
|
9
|
+
/**
|
|
10
|
+
* Lossless, and `exact` keeps the color under fully transparent pixels, so decoded pixels equal the
|
|
11
|
+
* PNG's. Method 1 at quality 50 measured 0.47 of the PNG bytes at about 330 ms for a 1440x950
|
|
12
|
+
* frame; higher efforts saved 2% more at twice the time.
|
|
13
|
+
*/
|
|
14
|
+
const OPTIONS = { lossless: 1, exact: 1, method: 1, quality: 50 };
|
|
15
|
+
const port = parentPort;
|
|
16
|
+
if (port === null)
|
|
17
|
+
throw new Error("webp-encode-worker runs only as a worker thread.");
|
|
18
|
+
const ready = readFile(createRequire(import.meta.url).resolve("@jsquash/webp/codec/enc/webp_enc_simd.wasm"))
|
|
19
|
+
.then((wasm) => WebAssembly.compile(wasm))
|
|
20
|
+
.then(async (module) => void (await init(module)));
|
|
21
|
+
ready.then(() => port.postMessage({ ready: true }), (error) => port.postMessage({ ready: false, error: String(error) }));
|
|
22
|
+
port.on("message", (job) => {
|
|
23
|
+
void ready
|
|
24
|
+
.then(async () => {
|
|
25
|
+
const image = PNG.sync.read(Buffer.from(job.png.buffer, job.png.byteOffset, job.png.length));
|
|
26
|
+
const data = new Uint8ClampedArray(image.data.buffer, image.data.byteOffset, image.data.length);
|
|
27
|
+
const webp = await encode({ data, width: image.width, height: image.height, colorSpace: "srgb" }, OPTIONS);
|
|
28
|
+
port.postMessage({ id: job.id, webp: new Uint8Array(webp) }, [webp]);
|
|
29
|
+
})
|
|
30
|
+
.catch((error) => port.postMessage({ id: job.id, error: String(error) }));
|
|
31
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface WebpTranscoder {
|
|
2
|
+
/**
|
|
3
|
+
* The frame's WebP bytes, or undefined when the queue is full or the encoder failed. `abandon`
|
|
4
|
+
* drops a frame no one waits for any more, so it no longer keeps the process alive.
|
|
5
|
+
*/
|
|
6
|
+
transcode(png: Buffer): {
|
|
7
|
+
webp: Promise<Buffer>;
|
|
8
|
+
abandon: () => void;
|
|
9
|
+
} | undefined;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The process's transcoder, started on first use. From source the worker runs as TypeScript, which
|
|
13
|
+
* Node loads with its types stripped.
|
|
14
|
+
*/
|
|
15
|
+
export declare function webpTranscoder(): WebpTranscoder;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// Turns computer-use frames into lossless WebP in a worker thread, off the turn's path: a frame is
|
|
2
|
+
// written and referenced as PNG first, and only a finished WebP replaces it (see
|
|
3
|
+
// src/routes/computer-use/participant-frames.ts). The process shares one worker, so encoding uses
|
|
4
|
+
// at most one core however many participants run.
|
|
5
|
+
import { Worker } from "node:worker_threads";
|
|
6
|
+
/**
|
|
7
|
+
* The PNG bytes that may wait for the worker at once. A frame past it stays PNG, so a busy runner
|
|
8
|
+
* keeps today's format for that frame instead of holding more frames in memory.
|
|
9
|
+
*/
|
|
10
|
+
const QUEUE_BYTE_BUDGET = 64 * 1024 * 1024;
|
|
11
|
+
class WorkerTranscoder {
|
|
12
|
+
worker;
|
|
13
|
+
/**
|
|
14
|
+
* Every frame posted to the worker until it answers. An abandoned frame still holds its bytes
|
|
15
|
+
* there, so it counts against the budget, but it no longer keeps the process alive.
|
|
16
|
+
*/
|
|
17
|
+
posted = new Map();
|
|
18
|
+
queuedBytes = 0;
|
|
19
|
+
wanted = 0;
|
|
20
|
+
nextId = 0;
|
|
21
|
+
failed = false;
|
|
22
|
+
constructor(entry) {
|
|
23
|
+
this.worker = new Worker(entry);
|
|
24
|
+
this.worker.on("message", (message) => {
|
|
25
|
+
if ("ready" in message) {
|
|
26
|
+
if (!message.ready)
|
|
27
|
+
this.fail(new Error(message.error));
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const job = this.posted.get(message.id);
|
|
31
|
+
if (job === undefined)
|
|
32
|
+
return;
|
|
33
|
+
this.posted.delete(message.id);
|
|
34
|
+
this.queuedBytes -= job.size;
|
|
35
|
+
if (job.abandoned)
|
|
36
|
+
return;
|
|
37
|
+
this.release();
|
|
38
|
+
if ("error" in message)
|
|
39
|
+
job.reject(new Error(message.error));
|
|
40
|
+
else
|
|
41
|
+
job.resolve(Buffer.from(message.webp.buffer, message.webp.byteOffset, message.webp.length));
|
|
42
|
+
});
|
|
43
|
+
this.worker.on("error", (error) => this.fail(error));
|
|
44
|
+
this.worker.on("exit", (code) => this.fail(new Error(`The WebP worker exited with code ${code}.`)));
|
|
45
|
+
// An idle worker never keeps the process alive; a waiting frame does. Adding a message listener
|
|
46
|
+
// refs the worker, so this comes after the listeners.
|
|
47
|
+
this.worker.unref();
|
|
48
|
+
}
|
|
49
|
+
transcode(png) {
|
|
50
|
+
if (this.failed || this.queuedBytes + png.length > QUEUE_BYTE_BUDGET)
|
|
51
|
+
return undefined;
|
|
52
|
+
const id = this.nextId++;
|
|
53
|
+
if (this.wanted++ === 0)
|
|
54
|
+
this.worker.ref();
|
|
55
|
+
this.queuedBytes += png.length;
|
|
56
|
+
const webp = new Promise((resolve, reject) => {
|
|
57
|
+
this.posted.set(id, { size: png.length, abandoned: false, resolve, reject });
|
|
58
|
+
this.worker.postMessage({ id, png: new Uint8Array(png) });
|
|
59
|
+
});
|
|
60
|
+
return {
|
|
61
|
+
webp,
|
|
62
|
+
abandon: () => {
|
|
63
|
+
const job = this.posted.get(id);
|
|
64
|
+
if (job === undefined || job.abandoned)
|
|
65
|
+
return;
|
|
66
|
+
job.abandoned = true;
|
|
67
|
+
this.release();
|
|
68
|
+
job.reject(new Error("The frame's WebP was abandoned."));
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/** One frame fewer that someone waits for; with none left the worker stops holding the process. */
|
|
73
|
+
release() {
|
|
74
|
+
if (--this.wanted === 0)
|
|
75
|
+
this.worker.unref();
|
|
76
|
+
}
|
|
77
|
+
fail(error) {
|
|
78
|
+
if (this.failed)
|
|
79
|
+
return;
|
|
80
|
+
this.failed = true;
|
|
81
|
+
for (const job of this.posted.values())
|
|
82
|
+
if (!job.abandoned)
|
|
83
|
+
job.reject(error);
|
|
84
|
+
this.posted.clear();
|
|
85
|
+
this.queuedBytes = 0;
|
|
86
|
+
this.wanted = 0;
|
|
87
|
+
void this.worker.terminate();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
let shared;
|
|
91
|
+
/**
|
|
92
|
+
* The process's transcoder, started on first use. From source the worker runs as TypeScript, which
|
|
93
|
+
* Node loads with its types stripped.
|
|
94
|
+
*/
|
|
95
|
+
export function webpTranscoder() {
|
|
96
|
+
shared ??= new WorkerTranscoder(new URL(import.meta.url.endsWith(".ts") ? "./webp-encode-worker.ts" : "./webp-encode-worker.js", import.meta.url));
|
|
97
|
+
return shared;
|
|
98
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export interface WebpDimensions {
|
|
2
|
+
width: number;
|
|
3
|
+
height: number;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Why `bytes` is not a simple-format lossless WebP, or null with its dimensions when it is: RIFF
|
|
7
|
+
* and WEBP, then one VP8L chunk that ends the file (with its pad byte, zero, when its size is
|
|
8
|
+
* odd), whose header has the VP8L signature and version 0.
|
|
9
|
+
*/
|
|
10
|
+
export declare function webpStructure(bytes: Buffer): {
|
|
11
|
+
error: string;
|
|
12
|
+
} | {
|
|
13
|
+
error: null;
|
|
14
|
+
dimensions: WebpDimensions;
|
|
15
|
+
};
|
|
16
|
+
/** The bytes webpHeaderDimensions reads: the RIFF header, the VP8L chunk header, the VP8L header. */
|
|
17
|
+
export declare const WEBP_HEADER_BYTES: number;
|
|
18
|
+
/**
|
|
19
|
+
* Width and height from the first WEBP_HEADER_BYTES of a simple-format lossless WebP: a VP8L
|
|
20
|
+
* signature and version 0. Undefined for anything else.
|
|
21
|
+
*/
|
|
22
|
+
export declare function webpHeaderDimensions(header: Buffer): WebpDimensions | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* The RGBA pixels of a WebP frame that webpStructure accepts, whose dimensions the caller has
|
|
25
|
+
* checked against its pixel cap; the decoder allocates from them.
|
|
26
|
+
*/
|
|
27
|
+
export declare function decodeWebp(bytes: Buffer): Promise<{
|
|
28
|
+
width: number;
|
|
29
|
+
height: number;
|
|
30
|
+
data: Buffer;
|
|
31
|
+
}>;
|