humanish 0.145.0 → 0.147.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/actors/computer-use/loop/spend.js +5 -4
  3. package/dist/actors/computer-use/loop/trace.js +3 -0
  4. package/dist/actors/computer-use/loop/types.d.ts +5 -0
  5. package/dist/actors/computer-use/loop/usage.js +10 -2
  6. package/dist/actors/computer-use/openai-conversation.d.ts +7 -0
  7. package/dist/actors/computer-use/openai-conversation.js +19 -2
  8. package/dist/actors/computer-use/openai-provider.d.ts +2 -0
  9. package/dist/actors/computer-use/openai-provider.js +2 -0
  10. package/dist/actors/computer-use/openai-wire.d.ts +4 -0
  11. package/dist/actors/computer-use/openai-wire.js +9 -1
  12. package/dist/actors/contract.d.ts +5 -0
  13. package/dist/analysis/admission.d.ts +8 -3
  14. package/dist/analysis/admission.js +7 -3
  15. package/dist/analysis/automatic-config.js +2 -1
  16. package/dist/analysis/automatic.js +2 -2
  17. package/dist/analysis/evidence.js +8 -4
  18. package/dist/analysis/execute.d.ts +7 -10
  19. package/dist/analysis/execute.js +79 -49
  20. package/dist/analysis/image-tokens.js +9 -3
  21. package/dist/analysis/output-limit.d.ts +12 -0
  22. package/dist/analysis/output-limit.js +34 -0
  23. package/dist/analysis/output-retry.d.ts +20 -0
  24. package/dist/analysis/output-retry.js +63 -0
  25. package/dist/analysis/provider.d.ts +1 -1
  26. package/dist/analysis/provider.js +3 -3
  27. package/dist/analysis/responses.js +1 -1
  28. package/dist/analysis/service.js +2 -2
  29. package/dist/cli/commands/analyze.js +1 -1
  30. package/dist/cli/io.js +2 -0
  31. package/dist/cli/run-findings.js +10 -2
  32. package/dist/evidence/image.d.ts +2 -0
  33. package/dist/evidence/image.js +24 -3
  34. package/dist/evidence/webp-encode-worker.d.ts +1 -0
  35. package/dist/evidence/webp-encode-worker.js +31 -0
  36. package/dist/evidence/webp-transcoder.d.ts +15 -0
  37. package/dist/evidence/webp-transcoder.js +98 -0
  38. package/dist/evidence/webp.d.ts +31 -0
  39. package/dist/evidence/webp.js +94 -0
  40. package/dist/feedback/derivative-text.d.ts +7 -2
  41. package/dist/feedback/derivative-text.js +20 -1
  42. package/dist/feedback/export-bundle.js +79 -25
  43. package/dist/observer-app.html +1 -1
  44. package/dist/remote/launch.js +93 -48
  45. package/dist/remote/runner.d.ts +47 -14
  46. package/dist/remote/runner.js +199 -64
  47. package/dist/routes/computer-use/e2b-desktop/desktop.js +47 -7
  48. package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +4 -2
  49. package/dist/routes/computer-use/e2b-desktop/prepare.js +4 -2
  50. package/dist/routes/computer-use/e2b-desktop/start.d.ts +14 -6
  51. package/dist/routes/computer-use/e2b-desktop/start.js +72 -34
  52. package/dist/routes/computer-use/participant-execution.d.ts +0 -7
  53. package/dist/routes/computer-use/participant-execution.js +6 -28
  54. package/dist/routes/computer-use/participant-frames.d.ts +32 -0
  55. package/dist/routes/computer-use/participant-frames.js +93 -0
  56. package/dist/routes/computer-use/participant-model.js +3 -0
  57. package/dist/routes/computer-use/participant-runs.js +4 -0
  58. package/dist/routes/computer-use/plan.js +5 -1
  59. package/dist/routes/computer-use/result.js +6 -0
  60. package/dist/routes/computer-use/types.d.ts +4 -0
  61. package/dist/routes/shared-world/plan.js +4 -1
  62. package/dist/run/pricing.d.ts +7 -0
  63. package/dist/run/pricing.js +58 -15
  64. package/dist/study/config.js +4 -1
  65. package/dist/study/keys.d.ts +1 -0
  66. package/dist/study/keys.js +1 -0
  67. package/dist/study/parse/actors.js +5 -0
  68. package/dist/study/plan-participants.d.ts +2 -0
  69. package/dist/study/plan-participants.js +2 -0
  70. package/dist/study/types.d.ts +7 -0
  71. package/dist/study/validation.d.ts +5 -0
  72. package/dist/study/validation.js +23 -0
  73. package/dist/substrates/e2b/cdp-probe.d.ts +16 -1
  74. package/dist/substrates/e2b/cdp-probe.js +67 -5
  75. package/dist/substrates/e2b/desktop-browser.js +4 -2
  76. package/dist/substrates/e2b/desktop-cdp.d.ts +9 -2
  77. package/dist/substrates/e2b/desktop-cdp.js +26 -4
  78. package/dist/verify/artifacts.js +2 -1
  79. package/docs/contracts/run-bundle.md +29 -11
  80. package/docs/contracts/schemas.md +20 -12
  81. package/docs/contracts/study-analysis.md +64 -11
  82. package/docs/product/automatic-analysis.md +9 -6
  83. package/package.json +6 -5
@@ -1,3 +1,4 @@
1
+ import { WEBP_HEADER_BYTES, webpHeaderDimensions } from "../evidence/webp.js";
1
2
  /** An input image the analysis providers accept; the groups are the image type and its base64 bytes. */
2
3
  export const INPUT_IMAGE_DATA_URL = /^data:image\/(png|jpeg|webp);base64,([A-Za-z0-9+/]+={0,2})$/;
3
4
  /**
@@ -36,8 +37,13 @@ const GEMINI_HIGH_RESOLUTION_IMAGE_TOKENS = Object.freeze({
36
37
  /** The most one image can cost on `model`, whatever its size. */
37
38
  export const imageTokenCeiling = (model) => GEMINI_HIGH_RESOLUTION_IMAGE_TOKENS[model] ?? HIGH_DETAIL_IMAGE_TOKEN_CEILING;
38
39
  const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
39
- /** Width and height from a PNG data URL's IHDR chunk; undefined for anything else. */
40
- function pngDimensions(dataUrl) {
40
+ /** Base64 characters that hold WEBP_HEADER_BYTES, rounded up to whole 3-byte groups. */
41
+ const WEBP_HEADER_CHARS = Math.ceil(WEBP_HEADER_BYTES / 3) * 4;
42
+ /** Width and height from a PNG data URL's IHDR chunk or a lossless WebP's VP8L header. */
43
+ function imageDimensions(dataUrl) {
44
+ const webp = new RegExp(`^data:image/webp;base64,([A-Za-z0-9+/]{${WEBP_HEADER_CHARS}})`).exec(dataUrl);
45
+ if (webp)
46
+ return webpHeaderDimensions(Buffer.from(webp[1], "base64"));
41
47
  const match = /^data:image\/png;base64,([A-Za-z0-9+/]{32})/.exec(dataUrl);
42
48
  if (!match)
43
49
  return undefined;
@@ -53,7 +59,7 @@ export function highDetailImageTokens(model, dataUrl) {
53
59
  if (fixed !== undefined)
54
60
  return fixed;
55
61
  const sizing = HIGH_DETAIL_SIZING[model];
56
- const size = sizing === undefined ? undefined : pngDimensions(dataUrl);
62
+ const size = sizing === undefined ? undefined : imageDimensions(dataUrl);
57
63
  if (sizing === undefined || size === undefined)
58
64
  return HIGH_DETAIL_IMAGE_TOKEN_CEILING;
59
65
  const patches = Math.ceil(size.width / 32) * Math.ceil(size.height / 32);
@@ -0,0 +1,12 @@
1
+ import type { AnalysisConfig } from "./types.js";
2
+ /** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
3
+ * returns at most 32,768 output tokens from one generateContent request. */
4
+ export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
5
+ export declare const MAX_ANALYSIS_OUTPUT_TOKENS = 32768;
6
+ /**
7
+ * The output limit an analysis of `participants` asks for when its study declares none, or
8
+ * undefined for Codex, which takes no limit. A high-effort or Gemini analysis, and one analysed in
9
+ * cohorts, gets the most the analysts return: at those efforts one finished report of three
10
+ * participants used up to 26,439 tokens.
11
+ */
12
+ export declare function defaultOutputTokens(config: AnalysisConfig, participants: number): number | undefined;
@@ -0,0 +1,34 @@
1
+ // The output limit an API analysis asks for when its study declares none. A request that runs
2
+ // away writes until it reaches the limit, so the limit is sized to what a finished report of the
3
+ // packet needs, with room to spare.
4
+ import { analysisCohortCount } from "./analysis-limits.js";
5
+ import { analysisEffort } from "./models.js";
6
+ /** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
7
+ * returns at most 32,768 output tokens from one generateContent request. */
8
+ export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
9
+ export const MAX_ANALYSIS_OUTPUT_TOKENS = 32_768;
10
+ /**
11
+ * The limit of a low- or medium-effort OpenAI request: this many tokens, and this many more for
12
+ * each participant. On the 2026-10-22 and 2026-10-23 benchmark recordings the longest finished
13
+ * report at those efforts used 8,354 tokens for one participant and 11,313 for three (gpt-6-luna at
14
+ * medium; gpt-6.1-sol at medium used at most 5,659 and 9,000), so the limit is 1.7 and 1.6 times
15
+ * those. docs/contracts/study-analysis.md has the table.
16
+ */
17
+ const LIMIT_BASE_TOKENS = 12_288;
18
+ const LIMIT_TOKENS_PER_PARTICIPANT = 2_048;
19
+ const SIZED_EFFORTS = new Set(["none", "low", "medium"]);
20
+ /**
21
+ * The output limit an analysis of `participants` asks for when its study declares none, or
22
+ * undefined for Codex, which takes no limit. A high-effort or Gemini analysis, and one analysed in
23
+ * cohorts, gets the most the analysts return: at those efforts one finished report of three
24
+ * participants used up to 26,439 tokens.
25
+ */
26
+ export function defaultOutputTokens(config, participants) {
27
+ if (config.provider === "codex")
28
+ return undefined;
29
+ if (config.provider === "gemini" ||
30
+ !SIZED_EFFORTS.has(analysisEffort(config)) ||
31
+ analysisCohortCount(participants) > 1)
32
+ return MAX_ANALYSIS_OUTPUT_TOKENS;
33
+ return Math.min(MAX_ANALYSIS_OUTPUT_TOKENS, LIMIT_BASE_TOKENS + LIMIT_TOKENS_PER_PARTICIPANT * participants);
34
+ }
@@ -0,0 +1,20 @@
1
+ import type { RequestCost } from "./admission.js";
2
+ import type { AnalysisProviderResult } from "./provider.js";
3
+ import type { AnalysisConfig, AnalysisUsage } from "./types.js";
4
+ export interface OutputRetries {
5
+ /**
6
+ * Whether to send request `index` again after `response`, given the attempt's usage with that
7
+ * response counted. Adds a warning that says what ran out and what humanish did.
8
+ */
9
+ retry(index: number, response: AnalysisProviderResult, usage: AnalysisUsage): boolean;
10
+ /** Request `index` has its final answer, so its cost is no longer held against the cap. */
11
+ settle(index: number): void;
12
+ /** The summed worst case of every request sent again, which admission did not price. */
13
+ worstCaseUsd(): number;
14
+ }
15
+ /**
16
+ * The retries of one attempt. `costs` are each request's costs from admission (the cohorts', then
17
+ * the merge request's), `describe` names request `index` at the start of a sentence, and warnings
18
+ * receive what happened.
19
+ */
20
+ export declare function outputRetries(config: AnalysisConfig, costs: readonly RequestCost[], describe: (index: number) => string, warnings: string[] | undefined): OutputRetries;
@@ -0,0 +1,63 @@
1
+ // One more request when an API analyst runs out of output before it finishes its report. The same
2
+ // request is sent again, at most once, and only while the analysis's cost cap still admits it
3
+ // after counting what every request so far cost. Codex account analysis has no dollar cap to
4
+ // check, so it is never sent again.
5
+ import { estimateActorCost } from "../run/pricing.js";
6
+ const usd = (value) => `$${value.toFixed(2)}`;
7
+ /**
8
+ * The retries of one attempt. `costs` are each request's costs from admission (the cohorts', then
9
+ * the merge request's), `describe` names request `index` at the start of a sentence, and warnings
10
+ * receive what happened.
11
+ */
12
+ export function outputRetries(config, costs, describe, warnings) {
13
+ // What each request without a final answer may still cost: admission's figure for it.
14
+ const held = new Map(costs.map((cost, index) => [index, cost.admittedCostUsd]));
15
+ const retried = new Set();
16
+ let worstCase = 0;
17
+ const warn = (text) => {
18
+ warnings?.push(text);
19
+ };
20
+ return {
21
+ retry(index, response, usage) {
22
+ if (response.errorCode !== "output_incomplete" || config.maxCostUsd === null)
23
+ return false;
24
+ const which = describe(index);
25
+ if (retried.has(index)) {
26
+ warn(`${which} ran out of output again, so this attempt has no report.`);
27
+ return false;
28
+ }
29
+ const cap = config.maxCostUsd;
30
+ const tokens = response.usage;
31
+ const cost = tokens === null
32
+ ? null
33
+ : estimateActorCost({ ...tokens, turns: [tokens] }, config.model).estimatedCostUsd;
34
+ if (tokens === null ||
35
+ cost === null ||
36
+ !usage.usageComplete ||
37
+ usage.estimatedCostUsd === null) {
38
+ warn(`${which} ran out of output before it finished its report and did not report its usage, so humanish cannot tell whether a second request fits the $${cap} cap. It was not sent again.`);
39
+ return false;
40
+ }
41
+ const ranOut = `${which} ran out of output before it finished its report: ${tokens.output} of ${config.maxOutputTokens} output tokens, ${tokens.input} input tokens, ${usd(cost)}.`;
42
+ const again = costs[index];
43
+ let others = 0;
44
+ for (const [other, amount] of held)
45
+ if (other !== index)
46
+ others += amount;
47
+ const total = usage.estimatedCostUsd + others + again.admittedCostUsd;
48
+ if (total > cap) {
49
+ const running = others > 0 ? `, the ${usd(others)} admission holds for requests still running,` : "";
50
+ warn(`${ranOut} humanish did not send it again: the ${usd(usage.estimatedCostUsd)} spent so far${running} plus ${usd(again.admittedCostUsd)} for one more request at admission's estimate is over the $${cap} cap. To retry, run the analysis again; a higher cost cap leaves room for a second request.`);
51
+ return false;
52
+ }
53
+ retried.add(index);
54
+ worstCase += again.worstCaseCostUsd;
55
+ warn(`${ranOut} humanish sent the same request once more; the attempt's usage counts both requests.`);
56
+ return true;
57
+ },
58
+ settle(index) {
59
+ held.delete(index);
60
+ },
61
+ worstCaseUsd: () => worstCase,
62
+ };
63
+ }
@@ -73,7 +73,7 @@ export interface AnalysisWire {
73
73
  body(request: AnalysisProviderRequest): unknown;
74
74
  parse(raw: unknown): AnalysisProviderResult;
75
75
  }
76
- /** One request to the vendor's fixed endpoint, never retried and never sent elsewhere. */
76
+ /** One request to the vendor's fixed endpoint, sent once and never elsewhere. */
77
77
  export declare function createAnalysisProvider(options: {
78
78
  apiKey: string;
79
79
  /** Omitted is OpenAI, the analyst humanish has always called. */
@@ -1,8 +1,8 @@
1
1
  // The analysis provider seam and the HTTP transport its API adapters share. Each API vendor owns
2
2
  // only its wire format (gemini-wire.ts, openai-wire.ts): the endpoint, the request body, and how a
3
3
  // response becomes a result. The transport owns what must hold for every vendor: one fixed endpoint
4
- // per vendor, request and response byte caps, the caller's wall-clock deadline, no retry, and a
5
- // closed error vocabulary that never carries provider error prose. The Codex account analyst is
4
+ // per vendor, request and response byte caps, the caller's wall-clock deadline, one send per call
5
+ // (runAnalysis decides whether to call again, in output-retry.ts), and a closed error vocabulary that never carries provider error prose. The Codex account analyst is
6
6
  // the third adapter at the same seam (restricted-codex.ts).
7
7
  import { Agent, fetch as undiciFetch } from "undici";
8
8
  import { isRecord } from "../run/type-guards.js";
@@ -49,7 +49,7 @@ const WIRES = {
49
49
  openai: openaiWire,
50
50
  gemini: geminiWire,
51
51
  };
52
- /** One request to the vendor's fixed endpoint, never retried and never sent elsewhere. */
52
+ /** One request to the vendor's fixed endpoint, sent once and never elsewhere. */
53
53
  export function createAnalysisProvider(options) {
54
54
  const fetchFn = options.fetchFn ?? undiciFetch;
55
55
  const wire = WIRES[options.vendor ?? "openai"];
@@ -202,7 +202,7 @@ export function httpStatusWarning(config, response) {
202
202
  return undefined;
203
203
  const api = config.provider === "gemini" ? "Gemini" : "OpenAI";
204
204
  const meaning = HTTP_STATUS_MEANINGS[response.httpStatus];
205
- return `The ${api} API answered HTTP ${response.httpStatus}${meaning === undefined ? "" : ` (${meaning})`}. humanish does not retry an analysis request; run the command again to retry.`;
205
+ return `The ${api} API answered HTTP ${response.httpStatus}${meaning === undefined ? "" : ` (${meaning})`}. humanish sends a request again only when it ran out of output; run the command again to retry.`;
206
206
  }
207
207
  /** The warnings a Codex response carries about its CLI release and protocol. */
208
208
  export function codexWarnings(config, response) {
@@ -17,7 +17,7 @@ import { withAnalysisLock } from "./analysis-lock.js";
17
17
  import { captureEvidence, EVIDENCE_LIMITS } from "./evidence.js";
18
18
  import { AnalysisSourceRefusedError, runFileTooLarge, throwAsAnalysisRefusal } from "./source.js";
19
19
  import { pathMissing, readBoundedFile, readBoundedFileResult } from "../run/evidence-files.js";
20
- import { estimateAnalysisAdmission, preferLargerAnalysisOutput, runAnalysis, ANALYSIS_PROMPT_VERSION, } from "./execute.js";
20
+ import { estimateAnalysisAdmission, defaultAnalysisOutput, runAnalysis, ANALYSIS_PROMPT_VERSION, } from "./execute.js";
21
21
  import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalyses, writeAnalysis, } from "./store.js";
22
22
  import { beginAnalysisExecution, writeAnalysisExecutionReceipt } from "./store-executions.js";
23
23
  import { keepRejectedAnalysisOutput } from "./diagnostics.js";
@@ -261,7 +261,7 @@ async function executeAnalysis(attempt) {
261
261
  if (input.evidence.length === 0)
262
262
  return fail(input.runId, dryRun, "ANALYSIS_NO_PARTICIPANTS");
263
263
  if (options.preferLargerOutput)
264
- config = preferLargerAnalysisOutput(input, config);
264
+ config = defaultAnalysisOutput(input, config);
265
265
  const admission = estimateAnalysisAdmission(input, config);
266
266
  const base = {
267
267
  schema: ANALYZE_RESULT_SCHEMA,
@@ -38,7 +38,7 @@ export function registerAnalyzeCommand(parent, io) {
38
38
  .option("--reasoning-effort <level>", `OpenAI reasoning.effort or Gemini thinking level, from none, minimal, low, medium, high, xhigh and max; each model takes only the levels its provider documents for it, and others are refused before any request. When omitted: ${DEFAULT_ANALYSIS_EFFORT} for the default OpenAI analyst, high with --model or --provider gemini. Unsupported for Codex.`)
39
39
  .option("--question <text>", "Additional reviewer question; does not change participant instructions.")
40
40
  .option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
41
- .option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for admission-based sizing. Unsupported for Codex.")
41
+ .option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for a limit sized to the participants and effort. Unsupported for Codex.")
42
42
  .option("--dry-run", "Capture and validate local input and estimate admission; no request or analysis artifact.")
43
43
  .option("--rerun", "Create a new immutable version even when the same input and configuration were analyzed.")
44
44
  .option("--json", JSON_OPTION_DESCRIPTION)
package/dist/cli/io.js CHANGED
@@ -208,6 +208,8 @@ export function writeResult(command, io, result, formatHuman, analysisLines) {
208
208
  io.writeOut(`analysis: refused before it started. ${refusal.text} ${refusal.command}\n`);
209
209
  else if (analysis)
210
210
  io.writeOut(`analysis: ${analysisOutcomeText(analysis)}\n`);
211
+ for (const warning of analysis?.result?.warnings ?? [])
212
+ io.writeOut(`analysis: ${warning}\n`);
211
213
  const rejected = analysis?.result?.rejectedOutputPath;
212
214
  if (rejected)
213
215
  io.writeOut(`analysis: ${analysis.result?.error?.code ?? "rejected"}; the rejected output is kept locally at ${rejected}\n`);
@@ -33,10 +33,12 @@ async function readRunReport(cwd, result, deps) {
33
33
  if (outcome === undefined || result.runId === undefined || result.runId === "not-created")
34
34
  return undefined;
35
35
  const outcomeLine = `analysis: ${analysisOutcomeText(outcome)}`;
36
+ // What the attempt warned about, such as a request sent again after it ran out of output.
37
+ const warningLines = (outcome.result?.warnings ?? []).map((warning) => `analysis: ${warning}`);
36
38
  const plain = {
37
39
  message: `The automatic analysis was ${analysisOutcomeText(outcome)}.`,
38
40
  nextCommand: null,
39
- lines: [outcomeLine],
41
+ lines: [outcomeLine, ...warningLines],
40
42
  };
41
43
  // A dry run left no evidence, and its own output names the live run.
42
44
  if (outcome.reason === "AUTOMATIC_ANALYSIS_DRY_RUN")
@@ -61,7 +63,12 @@ async function readRunReport(cwd, result, deps) {
61
63
  return {
62
64
  message: view.message,
63
65
  nextCommand: review,
64
- lines: [outcomeLine, ...rejectedLine, ...formatFindingsSummary(view, review)],
66
+ lines: [
67
+ outcomeLine,
68
+ ...warningLines,
69
+ ...rejectedLine,
70
+ ...formatFindingsSummary(view, review),
71
+ ],
65
72
  findings: findingsSummary(view, review, result.observer?.observerPath),
66
73
  };
67
74
  // The line reads "analysis: skipped because ...", as it does without the view.
@@ -71,6 +78,7 @@ async function readRunReport(cwd, result, deps) {
71
78
  nextCommand: view.next,
72
79
  lines: [
73
80
  `analysis: ${said}`,
81
+ ...warningLines,
74
82
  ...rejectedLine,
75
83
  ...(view.next === null ? [] : [`next: ${view.next}`]),
76
84
  ],
@@ -22,3 +22,5 @@ export declare function pngDecodeRefusal(bytes: Buffer): string | null;
22
22
  export declare function isRedactedFrameShape(bytes: Buffer): boolean;
23
23
  export declare function screenshotEvidenceError(relativePath: string, bytes: Buffer): Promise<string | null>;
24
24
  export declare function assertScreenshotEvidence(relativePath: string, bytes: Buffer): Promise<void>;
25
+ /** Why an image of these dimensions is refused before any decode, or null. */
26
+ export declare function imageDimensionsError(format: string, width: number, height: number): string | null;
@@ -1,5 +1,6 @@
1
1
  import { crc32, createInflate } from "node:zlib";
2
2
  import { PNG } from "pngjs";
3
+ import { webpStructure } from "./webp.js";
3
4
  const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
4
5
  const PNG_IHDR_LENGTH = 13;
5
6
  // The pixel cap pngDecodeRefusal applies before any decode, for verify and the redactor alike.
@@ -130,8 +131,10 @@ export function isRedactedFrameShape(bytes) {
130
131
  }
131
132
  export async function screenshotEvidenceError(relativePath, bytes) {
132
133
  const extension = relativePath.toLowerCase().split(".").pop() ?? "";
134
+ if (extension === "webp")
135
+ return webpEvidenceError(bytes);
133
136
  if (extension !== "png") {
134
- return `unsupported screenshot extension .${extension || "unknown"}; only decoded PNG evidence is supported`;
137
+ return `unsupported screenshot extension .${extension || "unknown"}; screenshot evidence is PNG or lossless WebP`;
135
138
  }
136
139
  const refusal = pngDecodeRefusal(bytes);
137
140
  if (refusal)
@@ -150,6 +153,20 @@ export async function screenshotEvidenceError(relativePath, bytes) {
150
153
  ? "could not decode PNG evidence"
151
154
  : null;
152
155
  }
156
+ /**
157
+ * Why a WebP frame is not screenshot evidence, read from its structure (webpStructure) without
158
+ * decoding it. A WebP frame is always raw evidence, which verify never grades share-ready, so a
159
+ * decode would add 17 to 20 ms a frame on the main thread and establish nothing more about bytes
160
+ * no one reviews.
161
+ */
162
+ function webpEvidenceError(bytes) {
163
+ if (bytes.length > SCREENSHOT_MAX_BYTES)
164
+ return `WebP byte size exceeds ${SCREENSHOT_MAX_BYTES} byte limit`;
165
+ const structure = webpStructure(bytes);
166
+ if (structure.error !== null)
167
+ return structure.error;
168
+ return imageDimensionsError("WebP", structure.dimensions.width, structure.dimensions.height);
169
+ }
153
170
  /** The bit depths PNG allows for each color type. */
154
171
  const PNG_BIT_DEPTHS = {
155
172
  0: [1, 2, 4, 8, 16],
@@ -254,11 +271,15 @@ export async function assertScreenshotEvidence(relativePath, bytes) {
254
271
  }
255
272
  }
256
273
  function pngDimensionsError(width, height) {
274
+ return imageDimensionsError("PNG", width, height);
275
+ }
276
+ /** Why an image of these dimensions is refused before any decode, or null. */
277
+ export function imageDimensionsError(format, width, height) {
257
278
  if (width === 0 || height === 0) {
258
- return "PNG dimensions must be greater than zero";
279
+ return `${format} dimensions must be greater than zero`;
259
280
  }
260
281
  if (width * height > SCREENSHOT_MAX_SOURCE_PIXELS) {
261
- return `PNG pixel count exceeds ${SCREENSHOT_MAX_SOURCE_PIXELS} pixel limit`;
282
+ return `${format} pixel count exceeds ${SCREENSHOT_MAX_SOURCE_PIXELS} pixel limit`;
262
283
  }
263
284
  return null;
264
285
  }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,31 @@
1
+ // The worker thread src/evidence/webp-transcoder.ts runs: PNG frames in, lossless WebP out, one at
2
+ // a time. It imports packages only, so Node can run this file from source with its types stripped
3
+ // as well as from dist.
4
+ import { readFile } from "node:fs/promises";
5
+ import { createRequire } from "node:module";
6
+ import { parentPort } from "node:worker_threads";
7
+ import encode, { init } from "@jsquash/webp/encode.js";
8
+ import { PNG } from "pngjs";
9
+ /**
10
+ * Lossless, and `exact` keeps the color under fully transparent pixels, so decoded pixels equal the
11
+ * PNG's. Method 1 at quality 50 measured 0.47 of the PNG bytes at about 330 ms for a 1440x950
12
+ * frame; higher efforts saved 2% more at twice the time.
13
+ */
14
+ const OPTIONS = { lossless: 1, exact: 1, method: 1, quality: 50 };
15
+ const port = parentPort;
16
+ if (port === null)
17
+ throw new Error("webp-encode-worker runs only as a worker thread.");
18
+ const ready = readFile(createRequire(import.meta.url).resolve("@jsquash/webp/codec/enc/webp_enc_simd.wasm"))
19
+ .then((wasm) => WebAssembly.compile(wasm))
20
+ .then(async (module) => void (await init(module)));
21
+ ready.then(() => port.postMessage({ ready: true }), (error) => port.postMessage({ ready: false, error: String(error) }));
22
+ port.on("message", (job) => {
23
+ void ready
24
+ .then(async () => {
25
+ const image = PNG.sync.read(Buffer.from(job.png.buffer, job.png.byteOffset, job.png.length));
26
+ const data = new Uint8ClampedArray(image.data.buffer, image.data.byteOffset, image.data.length);
27
+ const webp = await encode({ data, width: image.width, height: image.height, colorSpace: "srgb" }, OPTIONS);
28
+ port.postMessage({ id: job.id, webp: new Uint8Array(webp) }, [webp]);
29
+ })
30
+ .catch((error) => port.postMessage({ id: job.id, error: String(error) }));
31
+ });
@@ -0,0 +1,15 @@
1
+ export interface WebpTranscoder {
2
+ /**
3
+ * The frame's WebP bytes, or undefined when the queue is full or the encoder failed. `abandon`
4
+ * drops a frame no one waits for any more, so it no longer keeps the process alive.
5
+ */
6
+ transcode(png: Buffer): {
7
+ webp: Promise<Buffer>;
8
+ abandon: () => void;
9
+ } | undefined;
10
+ }
11
+ /**
12
+ * The process's transcoder, started on first use. From source the worker runs as TypeScript, which
13
+ * Node loads with its types stripped.
14
+ */
15
+ export declare function webpTranscoder(): WebpTranscoder;
@@ -0,0 +1,98 @@
1
+ // Turns computer-use frames into lossless WebP in a worker thread, off the turn's path: a frame is
2
+ // written and referenced as PNG first, and only a finished WebP replaces it (see
3
+ // src/routes/computer-use/participant-frames.ts). The process shares one worker, so encoding uses
4
+ // at most one core however many participants run.
5
+ import { Worker } from "node:worker_threads";
6
+ /**
7
+ * The PNG bytes that may wait for the worker at once. A frame past it stays PNG, so a busy runner
8
+ * keeps today's format for that frame instead of holding more frames in memory.
9
+ */
10
+ const QUEUE_BYTE_BUDGET = 64 * 1024 * 1024;
11
+ class WorkerTranscoder {
12
+ worker;
13
+ /**
14
+ * Every frame posted to the worker until it answers. An abandoned frame still holds its bytes
15
+ * there, so it counts against the budget, but it no longer keeps the process alive.
16
+ */
17
+ posted = new Map();
18
+ queuedBytes = 0;
19
+ wanted = 0;
20
+ nextId = 0;
21
+ failed = false;
22
+ constructor(entry) {
23
+ this.worker = new Worker(entry);
24
+ this.worker.on("message", (message) => {
25
+ if ("ready" in message) {
26
+ if (!message.ready)
27
+ this.fail(new Error(message.error));
28
+ return;
29
+ }
30
+ const job = this.posted.get(message.id);
31
+ if (job === undefined)
32
+ return;
33
+ this.posted.delete(message.id);
34
+ this.queuedBytes -= job.size;
35
+ if (job.abandoned)
36
+ return;
37
+ this.release();
38
+ if ("error" in message)
39
+ job.reject(new Error(message.error));
40
+ else
41
+ job.resolve(Buffer.from(message.webp.buffer, message.webp.byteOffset, message.webp.length));
42
+ });
43
+ this.worker.on("error", (error) => this.fail(error));
44
+ this.worker.on("exit", (code) => this.fail(new Error(`The WebP worker exited with code ${code}.`)));
45
+ // An idle worker never keeps the process alive; a waiting frame does. Adding a message listener
46
+ // refs the worker, so this comes after the listeners.
47
+ this.worker.unref();
48
+ }
49
+ transcode(png) {
50
+ if (this.failed || this.queuedBytes + png.length > QUEUE_BYTE_BUDGET)
51
+ return undefined;
52
+ const id = this.nextId++;
53
+ if (this.wanted++ === 0)
54
+ this.worker.ref();
55
+ this.queuedBytes += png.length;
56
+ const webp = new Promise((resolve, reject) => {
57
+ this.posted.set(id, { size: png.length, abandoned: false, resolve, reject });
58
+ this.worker.postMessage({ id, png: new Uint8Array(png) });
59
+ });
60
+ return {
61
+ webp,
62
+ abandon: () => {
63
+ const job = this.posted.get(id);
64
+ if (job === undefined || job.abandoned)
65
+ return;
66
+ job.abandoned = true;
67
+ this.release();
68
+ job.reject(new Error("The frame's WebP was abandoned."));
69
+ },
70
+ };
71
+ }
72
+ /** One frame fewer that someone waits for; with none left the worker stops holding the process. */
73
+ release() {
74
+ if (--this.wanted === 0)
75
+ this.worker.unref();
76
+ }
77
+ fail(error) {
78
+ if (this.failed)
79
+ return;
80
+ this.failed = true;
81
+ for (const job of this.posted.values())
82
+ if (!job.abandoned)
83
+ job.reject(error);
84
+ this.posted.clear();
85
+ this.queuedBytes = 0;
86
+ this.wanted = 0;
87
+ void this.worker.terminate();
88
+ }
89
+ }
90
+ let shared;
91
+ /**
92
+ * The process's transcoder, started on first use. From source the worker runs as TypeScript, which
93
+ * Node loads with its types stripped.
94
+ */
95
+ export function webpTranscoder() {
96
+ shared ??= new WorkerTranscoder(new URL(import.meta.url.endsWith(".ts") ? "./webp-encode-worker.ts" : "./webp-encode-worker.js", import.meta.url));
97
+ return shared;
98
+ }
@@ -0,0 +1,31 @@
1
+ export interface WebpDimensions {
2
+ width: number;
3
+ height: number;
4
+ }
5
+ /**
6
+ * Why `bytes` is not a simple-format lossless WebP, or null with its dimensions when it is: RIFF
7
+ * and WEBP, then one VP8L chunk that ends the file (with its pad byte, zero, when its size is
8
+ * odd), whose header has the VP8L signature and version 0.
9
+ */
10
+ export declare function webpStructure(bytes: Buffer): {
11
+ error: string;
12
+ } | {
13
+ error: null;
14
+ dimensions: WebpDimensions;
15
+ };
16
+ /** The bytes webpHeaderDimensions reads: the RIFF header, the VP8L chunk header, the VP8L header. */
17
+ export declare const WEBP_HEADER_BYTES: number;
18
+ /**
19
+ * Width and height from the first WEBP_HEADER_BYTES of a simple-format lossless WebP: a VP8L
20
+ * signature and version 0. Undefined for anything else.
21
+ */
22
+ export declare function webpHeaderDimensions(header: Buffer): WebpDimensions | undefined;
23
+ /**
24
+ * The RGBA pixels of a WebP frame that webpStructure accepts, whose dimensions the caller has
25
+ * checked against its pixel cap; the decoder allocates from them.
26
+ */
27
+ export declare function decodeWebp(bytes: Buffer): Promise<{
28
+ width: number;
29
+ height: number;
30
+ data: Buffer;
31
+ }>;
@@ -0,0 +1,94 @@
1
+ // Lossless WebP screenshots: the file shape verify accepts, and a bounded decode for the readers
2
+ // that need pixels. Only the simple lossless form is accepted: a RIFF container holding one VP8L
3
+ // chunk. The extended form (VP8X) is where EXIF, XMP, ICC and animation chunks live, so a frame
4
+ // in the simple form carries image data only. The VP8L bitstream itself is not proven consumed,
5
+ // so a WebP frame is never redacted evidence: a blurred frame is the PNG redactScreenshot writes.
6
+ import { readFile } from "node:fs/promises";
7
+ import { createRequire } from "node:module";
8
+ import decodeWebpImage, { init as initWebpDecoder } from "@jsquash/webp/decode.js";
9
+ const RIFF_HEADER_BYTES = 12;
10
+ const CHUNK_HEADER_BYTES = 8;
11
+ const VP8L_SIGNATURE = 0x2f;
12
+ /** The VP8L header: its signature byte, then 14-bit width and height, an alpha hint, a version. */
13
+ const VP8L_HEADER_BYTES = 5;
14
+ /**
15
+ * Why `bytes` is not a simple-format lossless WebP, or null with its dimensions when it is: RIFF
16
+ * and WEBP, then one VP8L chunk that ends the file (with its pad byte, zero, when its size is
17
+ * odd), whose header has the VP8L signature and version 0.
18
+ */
19
+ export function webpStructure(bytes) {
20
+ const failed = (error) => ({ error });
21
+ if (bytes.length < RIFF_HEADER_BYTES + CHUNK_HEADER_BYTES + VP8L_HEADER_BYTES ||
22
+ bytes.toString("latin1", 0, 4) !== "RIFF" ||
23
+ bytes.toString("latin1", 8, 12) !== "WEBP")
24
+ return failed("expected a RIFF WEBP file");
25
+ if (bytes.readUInt32LE(4) !== bytes.length - 8)
26
+ return failed("the RIFF size does not match the file size");
27
+ const chunk = bytes.toString("latin1", 12, 16);
28
+ if (chunk !== "VP8L")
29
+ return failed(`WebP evidence must be one lossless VP8L chunk, not ${/^[A-Za-z0-9 ]{4}$/.test(chunk) ? chunk.trim() : "a malformed chunk"}`);
30
+ const size = bytes.readUInt32LE(16);
31
+ const start = RIFF_HEADER_BYTES + CHUNK_HEADER_BYTES;
32
+ const end = start + size + (size % 2);
33
+ if (end !== bytes.length || (size % 2 === 1 && bytes[end - 1] !== 0))
34
+ return failed("the VP8L chunk must end the file");
35
+ if (size < VP8L_HEADER_BYTES)
36
+ return failed("the VP8L chunk has no VP8L header");
37
+ const dimensions = webpHeaderDimensions(bytes);
38
+ if (dimensions === undefined)
39
+ return failed("the VP8L header is not one this reader knows");
40
+ return { error: null, dimensions };
41
+ }
42
+ /** The bytes webpHeaderDimensions reads: the RIFF header, the VP8L chunk header, the VP8L header. */
43
+ export const WEBP_HEADER_BYTES = RIFF_HEADER_BYTES + CHUNK_HEADER_BYTES + VP8L_HEADER_BYTES;
44
+ /**
45
+ * Width and height from the first WEBP_HEADER_BYTES of a simple-format lossless WebP: a VP8L
46
+ * signature and version 0. Undefined for anything else.
47
+ */
48
+ export function webpHeaderDimensions(header) {
49
+ const start = RIFF_HEADER_BYTES + CHUNK_HEADER_BYTES;
50
+ if (header.length < WEBP_HEADER_BYTES ||
51
+ header.toString("latin1", 0, 4) !== "RIFF" ||
52
+ header.toString("latin1", 8, 16) !== "WEBPVP8L" ||
53
+ header[start] !== VP8L_SIGNATURE)
54
+ return undefined;
55
+ const bits = header.readUInt32LE(start + 1);
56
+ if (bits >>> 29 !== 0)
57
+ return undefined;
58
+ return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 };
59
+ }
60
+ let decoderReady;
61
+ /** Loads the decoder from the package's own wasm file: the codec's default fetches it by URL. */
62
+ function readyDecoder() {
63
+ // The codec builds its result with ImageData, which Node does not define.
64
+ const scope = globalThis;
65
+ scope.ImageData ??= class {
66
+ data;
67
+ width;
68
+ height;
69
+ constructor(data, width, height) {
70
+ this.data = data;
71
+ this.width = width;
72
+ this.height = height;
73
+ }
74
+ };
75
+ decoderReady ??= readFile(createRequire(import.meta.url).resolve("@jsquash/webp/codec/dec/webp_dec.wasm"))
76
+ .then((wasm) => WebAssembly.compile(wasm))
77
+ .then((module) => initWebpDecoder(module));
78
+ return decoderReady;
79
+ }
80
+ /**
81
+ * The RGBA pixels of a WebP frame that webpStructure accepts, whose dimensions the caller has
82
+ * checked against its pixel cap; the decoder allocates from them.
83
+ */
84
+ export async function decodeWebp(bytes) {
85
+ await readyDecoder();
86
+ const copy = new Uint8Array(bytes.length);
87
+ copy.set(bytes);
88
+ const image = await decodeWebpImage(copy.buffer);
89
+ return {
90
+ width: image.width,
91
+ height: image.height,
92
+ data: Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength),
93
+ };
94
+ }
@@ -5,8 +5,13 @@ interface TextFile {
5
5
  bytes: Buffer;
6
6
  }
7
7
  export declare function jsonBytes(value: unknown): Buffer;
8
- export declare function transformText(file: TextFile, imagePaths: ReadonlySet<string>): Buffer;
8
+ /**
9
+ * Each kept image's source path and the path the copy holds it at; they differ for a WebP frame,
10
+ * which the copy holds as a blurred PNG.
11
+ */
12
+ type ImagePaths = ReadonlyMap<string, string>;
13
+ export declare function transformText(file: TextFile, imagePaths: ImagePaths): Buffer;
9
14
  export declare function assertRewritableNumbers(json: string): void;
10
15
  /** The loaded source bundle as the copy records it: screenshots declared and labelled blurred. */
11
- export declare function derivedBundle(bundle: RunBundle, imagePaths: ReadonlySet<string>): RunBundle;
16
+ export declare function derivedBundle(bundle: RunBundle, imagePaths: ImagePaths): RunBundle;
12
17
  export {};