humanish 0.145.0 → 0.147.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/actors/computer-use/loop/spend.js +5 -4
  3. package/dist/actors/computer-use/loop/trace.js +3 -0
  4. package/dist/actors/computer-use/loop/types.d.ts +5 -0
  5. package/dist/actors/computer-use/loop/usage.js +10 -2
  6. package/dist/actors/computer-use/openai-conversation.d.ts +7 -0
  7. package/dist/actors/computer-use/openai-conversation.js +19 -2
  8. package/dist/actors/computer-use/openai-provider.d.ts +2 -0
  9. package/dist/actors/computer-use/openai-provider.js +2 -0
  10. package/dist/actors/computer-use/openai-wire.d.ts +4 -0
  11. package/dist/actors/computer-use/openai-wire.js +9 -1
  12. package/dist/actors/contract.d.ts +5 -0
  13. package/dist/analysis/admission.d.ts +8 -3
  14. package/dist/analysis/admission.js +7 -3
  15. package/dist/analysis/automatic-config.js +2 -1
  16. package/dist/analysis/automatic.js +2 -2
  17. package/dist/analysis/evidence.js +8 -4
  18. package/dist/analysis/execute.d.ts +7 -10
  19. package/dist/analysis/execute.js +79 -49
  20. package/dist/analysis/image-tokens.js +9 -3
  21. package/dist/analysis/output-limit.d.ts +12 -0
  22. package/dist/analysis/output-limit.js +34 -0
  23. package/dist/analysis/output-retry.d.ts +20 -0
  24. package/dist/analysis/output-retry.js +63 -0
  25. package/dist/analysis/provider.d.ts +1 -1
  26. package/dist/analysis/provider.js +3 -3
  27. package/dist/analysis/responses.js +1 -1
  28. package/dist/analysis/service.js +2 -2
  29. package/dist/cli/commands/analyze.js +1 -1
  30. package/dist/cli/io.js +2 -0
  31. package/dist/cli/run-findings.js +10 -2
  32. package/dist/evidence/image.d.ts +2 -0
  33. package/dist/evidence/image.js +24 -3
  34. package/dist/evidence/webp-encode-worker.d.ts +1 -0
  35. package/dist/evidence/webp-encode-worker.js +31 -0
  36. package/dist/evidence/webp-transcoder.d.ts +15 -0
  37. package/dist/evidence/webp-transcoder.js +98 -0
  38. package/dist/evidence/webp.d.ts +31 -0
  39. package/dist/evidence/webp.js +94 -0
  40. package/dist/feedback/derivative-text.d.ts +7 -2
  41. package/dist/feedback/derivative-text.js +20 -1
  42. package/dist/feedback/export-bundle.js +79 -25
  43. package/dist/observer-app.html +1 -1
  44. package/dist/remote/launch.js +93 -48
  45. package/dist/remote/runner.d.ts +47 -14
  46. package/dist/remote/runner.js +199 -64
  47. package/dist/routes/computer-use/e2b-desktop/desktop.js +47 -7
  48. package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +4 -2
  49. package/dist/routes/computer-use/e2b-desktop/prepare.js +4 -2
  50. package/dist/routes/computer-use/e2b-desktop/start.d.ts +14 -6
  51. package/dist/routes/computer-use/e2b-desktop/start.js +72 -34
  52. package/dist/routes/computer-use/participant-execution.d.ts +0 -7
  53. package/dist/routes/computer-use/participant-execution.js +6 -28
  54. package/dist/routes/computer-use/participant-frames.d.ts +32 -0
  55. package/dist/routes/computer-use/participant-frames.js +93 -0
  56. package/dist/routes/computer-use/participant-model.js +3 -0
  57. package/dist/routes/computer-use/participant-runs.js +4 -0
  58. package/dist/routes/computer-use/plan.js +5 -1
  59. package/dist/routes/computer-use/result.js +6 -0
  60. package/dist/routes/computer-use/types.d.ts +4 -0
  61. package/dist/routes/shared-world/plan.js +4 -1
  62. package/dist/run/pricing.d.ts +7 -0
  63. package/dist/run/pricing.js +58 -15
  64. package/dist/study/config.js +4 -1
  65. package/dist/study/keys.d.ts +1 -0
  66. package/dist/study/keys.js +1 -0
  67. package/dist/study/parse/actors.js +5 -0
  68. package/dist/study/plan-participants.d.ts +2 -0
  69. package/dist/study/plan-participants.js +2 -0
  70. package/dist/study/types.d.ts +7 -0
  71. package/dist/study/validation.d.ts +5 -0
  72. package/dist/study/validation.js +23 -0
  73. package/dist/substrates/e2b/cdp-probe.d.ts +16 -1
  74. package/dist/substrates/e2b/cdp-probe.js +67 -5
  75. package/dist/substrates/e2b/desktop-browser.js +4 -2
  76. package/dist/substrates/e2b/desktop-cdp.d.ts +9 -2
  77. package/dist/substrates/e2b/desktop-cdp.js +26 -4
  78. package/dist/verify/artifacts.js +2 -1
  79. package/docs/contracts/run-bundle.md +29 -11
  80. package/docs/contracts/schemas.md +20 -12
  81. package/docs/contracts/study-analysis.md +64 -11
  82. package/docs/product/automatic-analysis.md +9 -6
  83. package/package.json +6 -5
@@ -54,6 +54,26 @@ function geminiRate(inPer1M, cachedPer1M, outPer1M, change = {}) {
54
54
  source: "ai.google.dev/gemini-api/docs/pricing (paid tier, standard)",
55
55
  };
56
56
  }
57
+ // Fast mode: OpenAI's `service_tier: "fast"` (formerly priority processing), billed per token at
58
+ // its own rates with the cached-input discount and the long-context tier. gpt-5.6-sol's fast rates
59
+ // are twice every standard rate ("For GPT-5.6 Sol, Fast mode costs twice the corresponding Standard
60
+ // rate", developers.openai.com/api/docs/guides/fast-mode, read 2026-10-24); the 6-series rates are
61
+ // the Fast tab of developers.openai.com/api/docs/pricing?latest-pricing=fast, read 2026-10-24.
62
+ const FAST_MODE_GUIDE = "developers.openai.com/api/docs/guides/fast-mode (gpt-5.6-sol, 2x standard)";
63
+ const FAST_MODE_SHEET = "developers.openai.com/api/docs/pricing?latest-pricing=fast (fast tier)";
64
+ /** A rate with Fast mode rates per 1M tokens: input, cached input, cache write and output. */
65
+ function withFast(rate, inPer1M, cachedPer1M, writePer1M, outPer1M, source) {
66
+ return {
67
+ ...rate,
68
+ fast: {
69
+ inputUsdPerToken: inPer1M * 1e-6,
70
+ cachedInputUsdPerToken: cachedPer1M * 1e-6,
71
+ cacheWriteUsdPerToken: writePer1M * 1e-6,
72
+ outputUsdPerToken: outPer1M * 1e-6,
73
+ source,
74
+ },
75
+ };
76
+ }
57
77
  // gpt-5.6-sol promotional rates (live sheet 2026-09-03: $4 / $0.40 cached / $5 write / $20 out,
58
78
  // "available at least through November 21, 2026"). Re-verify against the sheet after Nov 21.
59
79
  const GPT56_SOL_PROMO_AS_OF = "2026-09-03";
@@ -84,10 +104,10 @@ export const MODEL_RATES = {
84
104
  // the longContext block prices the >272K re-tier when per-request turns are recorded).
85
105
  // sol = flagship (the shipped CUA default), terra = cost-balanced, luna = high-volume,
86
106
  // cyber = the Daybreak frontier tier.
87
- "gpt-5.6-sol": rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF),
107
+ "gpt-5.6-sol": withFast(rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF), 8, 0.8, 10, 40, FAST_MODE_GUIDE),
88
108
  // "gpt-5.6" is OpenAI's own alias for gpt-5.6-sol (models index); priced identically so a
89
109
  // study configured with the alias never reads as unpriced.
90
- "gpt-5.6": rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF),
110
+ "gpt-5.6": withFast(rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF), 8, 0.8, 10, 40, FAST_MODE_GUIDE),
91
111
  "gpt-5.6-terra": rateWithLongContextTier(2, 0.2, 2.5, 12),
92
112
  "gpt-5.6-luna": rateWithLongContextTier(0.2, 0.02, 0.25, 1.2),
93
113
  "gpt-5.6-cyber": rateWithLongContextTier(12.5, 1.25, 15.625, 75),
@@ -100,11 +120,11 @@ export const MODEL_RATES = {
100
120
  // as the 5.6 family on the sheet: writes at 1.25x, >272K re-tiers at 2x input-side / 1.5x
101
121
  // output ($20 / $2 / $25 / $75 long-context columns). These rates cover explicit study
102
122
  // model choices and the study-analysis default; the computer-use default is separate.
103
- "gpt-6-astra": rateWithLongContextTier(10, 1, 12.5, 50, GPT6_ASTRA_AS_OF, GPT6_ASTRA_SOURCE),
123
+ "gpt-6-astra": withFast(rateWithLongContextTier(10, 1, 12.5, 50, GPT6_ASTRA_AS_OF, GPT6_ASTRA_SOURCE), 20, 2, 25, 100, FAST_MODE_SHEET),
104
124
  // gpt-6.1-sol and gpt-6-luna: the same write and long-context mechanics on the sheet and each
105
125
  // model page. 6.1 Sol bills cache reads at 5% of input ($0.10), where the others bill 10%.
106
- "gpt-6.1-sol": rateWithLongContextTier(2, 0.1, 2.5, 10, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE),
107
- "gpt-6-luna": rateWithLongContextTier(0.1, 0.01, 0.125, 0.5, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE),
126
+ "gpt-6.1-sol": withFast(rateWithLongContextTier(2, 0.1, 2.5, 10, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE), 4, 0.2, 5, 20, FAST_MODE_SHEET),
127
+ "gpt-6-luna": withFast(rateWithLongContextTier(0.1, 0.01, 0.125, 0.5, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE), 0.2, 0.02, 0.25, 1, FAST_MODE_SHEET),
108
128
  // Gemini analysis models, paid standard tier. Thinking tokens bill as output. No long-context
109
129
  // tier is published for them. Cached input is the implicit context-cache rate. On the free tier
110
130
  // these requests bill nothing, so the estimate is the paid-tier cost.
@@ -283,24 +303,41 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
283
303
  // price every token at the full rate while the session totals sit ignored, overstating spend
284
304
  // 3.5-5x and tripping caps early. Inconsistent evidence falls back to the totals path, which
285
305
  // honors the declared splits on the base tier.
286
- const tiered = rate.longContext !== undefined &&
306
+ const tiered = (rate.longContext !== undefined || turns.some((turn) => isFastServiceTier(turn.serviceTier))) &&
287
307
  turns.length > 0 &&
288
308
  sumOf("input") === inTok &&
289
309
  sumOf("output") === outTok &&
290
310
  sumOf("cachedInput") === (tokenUsage.cachedInput ?? 0) &&
291
311
  sumOf("cacheWriteInput") === (tokenUsage.cacheWriteInput ?? 0);
312
+ // A request served in Fast mode bills at the fast rates. Totals that the ledger does not
313
+ // decompose price at the fast rate when any request was served fast: the over-estimate direction.
314
+ const fastTotals = !tiered && turns.some((turn) => isFastServiceTier(turn.serviceTier));
315
+ // A request served fast on a model with no Fast mode rate has no rate in this table.
316
+ if (rate.fast === undefined && turns.some((turn) => isFastServiceTier(turn.serviceTier))) {
317
+ return {
318
+ schema: ACTOR_ESTIMATED_COST_SCHEMA,
319
+ estimatedCostUsd: null,
320
+ reason: "no_rate_for_model",
321
+ ratesAsOf: null,
322
+ ...(modelId ? { modelId } : {}),
323
+ };
324
+ }
292
325
  let inputUsd = 0;
293
326
  let outputUsd = 0;
294
327
  let cachedTotal = 0;
295
328
  let writeTotal = 0;
296
329
  let longContextTurns = 0;
330
+ let fastTurns = 0;
297
331
  // Price one request's usage at one tier. Cached input is billed at a fraction of the full rate,
298
332
  // and on a session that threads provider state it is the majority of input, and pricing it at the
299
333
  // full rate overstates real spend by up to ~10x. Cache writes bill at their own rate (1.25x on
300
334
  // OpenAI 5.6+) as the total rate for those tokens; a sheet without a write rate prices writes as
301
335
  // plain input (pre-5.6: writes are free-of-extra-fee, i.e. plain input). An unreported piece
302
336
  // stays absent: no reported split means no discount and no surcharge assumed.
303
- const priceRequest = (usage, tierable) => {
337
+ const priceRequest = (usage, tierable, fast) => {
338
+ const sheet = fast && rate.fast !== undefined ? { ...rate, ...rate.fast } : rate;
339
+ if (fast)
340
+ fastTurns += 1;
304
341
  const reqIn = usage.input ?? 0;
305
342
  const reqOut = usage.output ?? 0;
306
343
  // Tiering applies only to a real per-request record: session totals crossing the threshold
@@ -310,7 +347,7 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
310
347
  const outMul = long ? rate.longContext.outputMultiplier : 1;
311
348
  if (long)
312
349
  longContextTurns += 1;
313
- const cached = rate.cachedInputUsdPerToken === undefined
350
+ const cached = sheet.cachedInputUsdPerToken === undefined
314
351
  ? 0
315
352
  : Math.min(reqIn, Math.max(0, usage.cachedInput ?? 0));
316
353
  const writes = Math.min(reqIn - cached, Math.max(0, usage.cacheWriteInput ?? 0));
@@ -318,14 +355,14 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
318
355
  cachedTotal += cached;
319
356
  writeTotal += writes;
320
357
  inputUsd +=
321
- full * rate.inputUsdPerToken * inMul +
322
- cached * (rate.cachedInputUsdPerToken ?? 0) * inMul +
323
- writes * (rate.cacheWriteUsdPerToken ?? rate.inputUsdPerToken) * inMul;
324
- outputUsd += reqOut * rate.outputUsdPerToken * outMul;
358
+ full * sheet.inputUsdPerToken * inMul +
359
+ cached * (sheet.cachedInputUsdPerToken ?? 0) * inMul +
360
+ writes * (sheet.cacheWriteUsdPerToken ?? sheet.inputUsdPerToken) * inMul;
361
+ outputUsd += reqOut * sheet.outputUsdPerToken * outMul;
325
362
  };
326
363
  if (tiered) {
327
364
  for (const turn of turns)
328
- priceRequest(turn, true);
365
+ priceRequest(turn, true, isFastServiceTier(turn.serviceTier));
329
366
  }
330
367
  else {
331
368
  priceRequest({
@@ -335,7 +372,7 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
335
372
  ...(tokenUsage.cacheWriteInput === undefined
336
373
  ? {}
337
374
  : { cacheWriteInput: tokenUsage.cacheWriteInput }),
338
- }, false);
375
+ }, false, fastTotals);
339
376
  }
340
377
  inputUsd = round6(inputUsd);
341
378
  outputUsd = round6(outputUsd);
@@ -343,7 +380,7 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
343
380
  schema: ACTOR_ESTIMATED_COST_SCHEMA,
344
381
  estimatedCostUsd: round6(inputUsd + outputUsd),
345
382
  ratesAsOf: rate.asOf,
346
- source: rate.source,
383
+ source: fastTurns > 0 && rate.fast?.source !== undefined ? rate.fast.source : rate.source,
347
384
  ...(modelId ? { modelId } : {}),
348
385
  ...(rate.placeholder ? { placeholder: true } : {}),
349
386
  breakdown: {
@@ -354,9 +391,15 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
354
391
  ...(cachedTotal > 0 ? { cachedInputTokens: cachedTotal } : {}),
355
392
  ...(writeTotal > 0 ? { cacheWriteInputTokens: writeTotal } : {}),
356
393
  ...(longContextTurns > 0 ? { longContextTurns } : {}),
394
+ ...(fastTurns > 0 ? { fastTurns } : {}),
357
395
  },
358
396
  };
359
397
  }
398
+ /** Whether a provider-reported tier bills at Fast mode rates. GPT-5.6 reports `priority` for a
399
+ * request that asked for `fast`. */
400
+ function isFastServiceTier(tier) {
401
+ return tier === "fast" || tier === "priority";
402
+ }
360
403
  /**
361
404
  * Estimate the E2B desktop-minute cost from a host-side create->teardown span (minutes). The rate
362
405
  * is injectable. Returns a declared-absent estimate (null + "no_duration") when no duration was
@@ -39,7 +39,7 @@ import { invalid, optionalStr, str } from "./parse/values.js";
39
39
  import { isComputerUseComposition, routeOf } from "./routing.js";
40
40
  import { declaredParticipantIds } from "./plan-participants.js";
41
41
  import { ID_PATTERN, STUDY_SCHEMA, } from "./types.js";
42
- import { automaticAnalysisRouteReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, smtpValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "./validation.js";
42
+ import { automaticAnalysisRouteReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, serviceTierValidationReason, receivingEmailValidationReason, smtpValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "./validation.js";
43
43
  import { forwardDeclaredWarnings, inertFieldLabels, inertFieldPaths } from "./warnings.js";
44
44
  import { plural } from "../run/text.js";
45
45
  /**
@@ -234,6 +234,9 @@ function checkStudyConfig(config) {
234
234
  const outputLimitReason = outputTokenLimitValidationReason(config);
235
235
  if (outputLimitReason)
236
236
  return invalid(outputLimitReason);
237
+ const serviceTierReason = serviceTierValidationReason(config);
238
+ if (serviceTierReason)
239
+ return invalid(serviceTierReason);
237
240
  const waitLimitReason = waitLimitValidationReason(config);
238
241
  if (waitLimitReason)
239
242
  return invalid(waitLimitReason);
@@ -126,6 +126,7 @@ export declare const ACTOR: {
126
126
  };
127
127
  localAgent: true;
128
128
  reasoningEffort: true;
129
+ serviceTier: true;
129
130
  stopWhen: {
130
131
  any: {
131
132
  id: true;
@@ -106,6 +106,7 @@ export const ACTOR = {
106
106
  clock: { timeZone: true },
107
107
  localAgent: true,
108
108
  reasoningEffort: true,
109
+ serviceTier: true,
109
110
  stopWhen: STOP_WHEN,
110
111
  dwell: DWELL,
111
112
  tasks: { id: true, goal: true, success: STOP_WHEN },
@@ -112,6 +112,11 @@ export function parseActorFields(entry, path) {
112
112
  }
113
113
  actor.reasoningEffort = entry.reasoningEffort;
114
114
  }
115
+ if (entry.serviceTier !== undefined) {
116
+ if (entry.serviceTier !== "fast")
117
+ return invalid(`${path}.serviceTier must be "fast" (OpenAI Fast mode), or left out for the standard tier.`);
118
+ actor.serviceTier = entry.serviceTier;
119
+ }
115
120
  const stopWhenResult = parseStopWhen(entry.stopWhen, `${path}.stopWhen`);
116
121
  if (!stopWhenResult.ok)
117
122
  return stopWhenResult;
@@ -37,6 +37,8 @@ export interface DesktopParticipant extends Participant {
37
37
  readonly dwell?: DwellWindow;
38
38
  readonly reasoningEffort?: ReasoningEffort;
39
39
  readonly maxOutputTokens?: number;
40
+ /** actor.serviceTier: OpenAI Fast mode on every request. */
41
+ readonly serviceTier?: "fast";
40
42
  /** actor.maxWaitMs: the longest one wait or one turn of only waits lasts. */
41
43
  readonly maxWaitMs?: number;
42
44
  /**
@@ -13,6 +13,7 @@ function desktopParticipant(config, entry, index, kind, focus) {
13
13
  const dwell = entry?.dwell ?? actor?.dwell;
14
14
  const reasoningEffort = entry?.reasoningEffort ?? actor?.reasoningEffort;
15
15
  const maxOutputTokens = actor?.maxOutputTokens;
16
+ const serviceTier = actor?.serviceTier;
16
17
  const maxWaitMs = actor?.maxWaitMs;
17
18
  const idleWaitMs = entry?.idleWaitMs ?? actor?.idleWaitMs;
18
19
  const timeZone = entry?.clock?.timeZone ?? actor?.clock?.timeZone;
@@ -35,6 +36,7 @@ function desktopParticipant(config, entry, index, kind, focus) {
35
36
  ...(dwell === undefined ? {} : { dwell }),
36
37
  ...(reasoningEffort === undefined ? {} : { reasoningEffort }),
37
38
  ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
39
+ ...(serviceTier === undefined ? {} : { serviceTier }),
38
40
  ...(maxWaitMs === undefined ? {} : { maxWaitMs }),
39
41
  ...(idleWaitMs === undefined ? {} : { idleWaitMs }),
40
42
  ...(timeZone === undefined ? {} : { timeZone }),
@@ -453,6 +453,13 @@ export interface StudyActor {
453
453
  * docs/principles/actor-fidelity.md.
454
454
  */
455
455
  reasoningEffort?: ReasoningEffort;
456
+ /**
457
+ * `fast` sends every participant request in OpenAI Fast mode (`service_tier: "fast"`), which
458
+ * bills at its own per-token rates. Absent sends no tier. `openai-computer-use` only, and only for
459
+ * a model whose Fast mode rate src/run/pricing.ts carries, so the spend estimate and caps price it.
460
+ * Each request's trace usage records the tier that served it.
461
+ */
462
+ serviceTier?: "fast";
456
463
  /**
457
464
  * Deterministic completion guard used as the default for CUA participants. A `participants` entry's stopWhen
458
465
  * overrides this value.
@@ -44,6 +44,11 @@ export declare function taskProtocolValidationReason(config: StudyConfig, suppor
44
44
  export declare function cloneTargetValidationReason(config: StudyConfig): string | null;
45
45
  /** Refuse a claimed output bound when the route cannot pass it to the first-party provider. */
46
46
  export declare function outputTokenLimitValidationReason(config: StudyConfig): string | null;
47
+ /**
48
+ * Refuse a Fast mode request humanish cannot send or price: the first-party OpenAI computer-use
49
+ * route sends it, and the spend estimate needs the model's Fast mode rate.
50
+ */
51
+ export declare function serviceTierValidationReason(config: StudyConfig): string | null;
47
52
  /**
48
53
  * Refuse a wait field out of range, or an idle wait longer than the longest wait: the actor's, and
49
54
  * each participant's own idleWaitMs under actor.maxWaitMs. On a route whose participants take no
@@ -1,4 +1,6 @@
1
1
  import { isMaxOutputTokens } from "../actors/output-token-limit.js";
2
+ import { DEFAULT_OPENAI_CU_MODEL } from "../actors/computer-use/openai-provider.js";
3
+ import { MODEL_RATES, participantRate } from "../run/pricing.js";
2
4
  import { MAX_RECEIVING_INBOXES } from "../comms/receiving-types.js";
3
5
  import { waitFieldsReason } from "../actors/computer-use/wait.js";
4
6
  import { PARTICIPANT_ID_MAX_CHARS, PARTICIPANT_ID_PATTERN } from "./parse/actors.js";
@@ -288,6 +290,27 @@ export function outputTokenLimitValidationReason(config) {
288
290
  }
289
291
  return null;
290
292
  }
293
+ /**
294
+ * Refuse a Fast mode request humanish cannot send or price: the first-party OpenAI computer-use
295
+ * route sends it, and the spend estimate needs the model's Fast mode rate.
296
+ */
297
+ export function serviceTierValidationReason(config) {
298
+ const actor = config.actor;
299
+ if (actor?.serviceTier === undefined)
300
+ return null;
301
+ if (actor.type !== "openai-computer-use" ||
302
+ !isComputerUseComposition(config) ||
303
+ config.subject.source === "local-app")
304
+ return "actor.serviceTier is supported only by first-party OpenAI computer-use routes; terminal, local-agent, scripted and custom in-process routes do not send it.";
305
+ const model = actor.model ?? DEFAULT_OPENAI_CU_MODEL;
306
+ if (participantRate(model)?.fast === undefined) {
307
+ const priced = Object.keys(MODEL_RATES)
308
+ .filter((id) => participantRate(id)?.fast !== undefined)
309
+ .sort();
310
+ return `actor.serviceTier: fast needs a Fast mode rate for model "${model}", and humanish has none, so it could not price the run. Models with a Fast mode rate: ${priced.join(", ")}. Use one of them, or remove serviceTier.`;
311
+ }
312
+ return null;
313
+ }
291
314
  /**
292
315
  * Refuse a wait field out of range, or an idle wait longer than the longest wait: the actor's, and
293
316
  * each participant's own idleWaitMs under actor.maxWaitMs. On a route whose participants take no
@@ -30,8 +30,11 @@ export interface ChromeCdpProbeArgs {
30
30
  * pausing it; it applies `emulation` (how a participant keeps the overrides for its whole life,
31
31
  * with a reload on each page) and `network` capture, either or both. "fidelity": read
32
32
  * back what the page reports about itself (UA, DPR, viewport, touch), the proof for the bundle.
33
+ * "load": poll the page until it has loaded and painted, or until `deadlineMs`.
33
34
  */
34
- mode: "state" | "geometry" | "port" | "emulate" | "hold" | "fidelity";
35
+ mode: "state" | "geometry" | "port" | "emulate" | "hold" | "fidelity" | "load";
36
+ /** For "load": how long to poll before reporting the page as not loaded. */
37
+ deadlineMs?: number;
35
38
  /** For "emulate" and "hold": what to apply. */
36
39
  emulation?: ChromeMobileEmulationRequest;
37
40
  /** For "hold": capture each held page's requests to a sandbox file. */
@@ -41,6 +44,16 @@ export interface ChromeCdpProbeArgs {
41
44
  export interface ChromeMobileEmulationRequest extends MobileEmulationRequest {
42
45
  platform?: string;
43
46
  }
47
+ /**
48
+ * What the "load" mode saw. `loaded` is true once the page reports document.readyState complete,
49
+ * is visible and has a first contentful paint. `seen` describes the last read in words that name
50
+ * no URL.
51
+ */
52
+ export interface ChromePageLoad {
53
+ loaded: boolean;
54
+ waitedMs: number;
55
+ seen: string;
56
+ }
44
57
  /** What the page reports after emulation: the proof, never copied from the request. */
45
58
  interface ChromeFidelityRead {
46
59
  userAgent: string;
@@ -80,6 +93,8 @@ export interface ChromeCdpProbeResult {
80
93
  } | {
81
94
  unavailable: string;
82
95
  };
96
+ /** "load": whether the page loaded and painted before the deadline, and what the last read saw. */
97
+ load?: ChromePageLoad;
83
98
  /**
84
99
  * Seconds since the epoch on the sandbox clock when the probe printed, on every answer including
85
100
  * an unavailable one. It is the clock CDP `wallTime` uses, so a turn mark compares with request
@@ -213,6 +213,10 @@ GEOMETRY_EXPRESSION = (
213
213
  "({ browserWindow: { x: window.screenX, y: window.screenY, width: window.outerWidth, height: window.outerHeight }, "
214
214
  "viewport: { width: window.innerWidth, height: window.innerHeight, deviceScaleFactor: window.devicePixelRatio } })"
215
215
  )
216
+ LOAD_EXPRESSION = (
217
+ "({ readyState: document.readyState, visible: document.visibilityState === 'visible', "
218
+ "painted: performance.getEntriesByType('paint').some(function (entry) { return entry.name === 'first-contentful-paint'; }) })"
219
+ )
216
220
  FIDELITY_EXPRESSION = (
217
221
  "({ userAgent: navigator.userAgent, devicePixelRatio: window.devicePixelRatio, "
218
222
  "innerWidth: window.innerWidth, innerHeight: window.innerHeight, "
@@ -405,6 +409,50 @@ def follow_targets(ws, launch_id, request, log):
405
409
  if params.get("waitingForDebugger"):
406
410
  ws.send("Runtime.runIfWaitingForDebugger", {}, new_session)
407
411
 
412
+ def page_ws_url(port, page):
413
+ # Chrome omits webSocketDebuggerUrl from /json while another DevTools client is attached to the
414
+ # page, or for a moment after one detaches; the page socket URL is still /devtools/page/<id>.
415
+ return page.get("webSocketDebuggerUrl") or (
416
+ "ws://127.0.0.1:%d/devtools/page/%s" % (port, page.get("id")) if page.get("id") else None
417
+ )
418
+
419
+ def wait_for_load(port, args):
420
+ """Poll the page until document.readyState is complete, the page is visible and Chrome has
421
+ recorded its first contentful paint: the page has loaded and something of it is drawn. Stops at
422
+ deadlineMs and reports what the last read saw."""
423
+ started = time.monotonic()
424
+ deadline_ms = int(args.get("deadlineMs") or 0)
425
+ seen = "no http page yet"
426
+ page = None
427
+ loaded = False
428
+ while True:
429
+ try:
430
+ page = select_page(list_pages(port), args)
431
+ seen = "no http page yet" if page is None else seen
432
+ except Exception as error:
433
+ page, seen = None, "the DevTools endpoint did not answer (%s)" % type(error).__name__
434
+ ws_url = page_ws_url(port, page) if page is not None else None
435
+ state = evaluate(ws_url, LOAD_EXPRESSION) if ws_url else None
436
+ if isinstance(state, dict):
437
+ ready = str(state.get("readyState") or "unknown")
438
+ seen = "document.readyState %s, %s, %s" % (
439
+ ready,
440
+ "visible" if state.get("visible") else "hidden",
441
+ "first contentful paint recorded" if state.get("painted") else "no first contentful paint yet",
442
+ )
443
+ loaded = ready == "complete" and bool(state.get("visible")) and bool(state.get("painted"))
444
+ if loaded:
445
+ break
446
+ elif ws_url:
447
+ seen = "the page did not answer Runtime.evaluate"
448
+ if (time.monotonic() - started) * 1000 >= deadline_ms:
449
+ break
450
+ time.sleep(0.25)
451
+ answer = {"load": {"loaded": loaded, "waitedMs": int((time.monotonic() - started) * 1000), "seen": seen}}
452
+ if page is not None:
453
+ answer["targetId"] = str(page.get("id") or "")
454
+ emit(answer)
455
+
408
456
  def emit(answer):
409
457
  """Print one answer line stamped with this sandbox's clock. CDP wallTime uses the same clock, so
410
458
  a turn mark taken from it compares with request times without host skew, and the reading needs
@@ -419,6 +467,9 @@ def main():
419
467
  if mode == "port":
420
468
  emit({"cdpPort": port})
421
469
  return
470
+ if mode == "load":
471
+ wait_for_load(port, args)
472
+ return
422
473
  try:
423
474
  pages = list_pages(port)
424
475
  except Exception as error:
@@ -441,11 +492,7 @@ def main():
441
492
  if page is None:
442
493
  emit({"unavailable": "no http page among %d CDP targets on 127.0.0.1:%d" % (len(pages), port)})
443
494
  return
444
- # Chrome omits webSocketDebuggerUrl from /json while another DevTools client is attached to the
445
- # page, or for a moment after one detaches; the page socket URL is still /devtools/page/<id>.
446
- ws_url = page.get("webSocketDebuggerUrl") or (
447
- "ws://127.0.0.1:%d/devtools/page/%s" % (port, page.get("id")) if page.get("id") else None
448
- )
495
+ ws_url = page_ws_url(port, page)
449
496
  if mode == "emulate":
450
497
  applied, failure = emulate(ws_url, args.get("emulation") or {}) if ws_url else (None, "the page has no socket")
451
498
  if failure is not None:
@@ -502,6 +549,8 @@ export function chromeCdpProbeCommand(args) {
502
549
  payload.emulation = args.emulation;
503
550
  if (args.network !== undefined)
504
551
  payload.network = args.network;
552
+ if (args.deadlineMs !== undefined)
553
+ payload.deadlineMs = args.deadlineMs;
505
554
  return `python3 -c ${shellQuote(CHROME_CDP_PROBE_PY)} ${shellQuote(JSON.stringify(payload))}`;
506
555
  }
507
556
  /**
@@ -529,6 +578,7 @@ export function parseChromeCdpProbeOutput(stdout) {
529
578
  const numberOr = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
530
579
  const now = numberOr(record.now);
531
580
  const network = networkOf(record.network);
581
+ const load = loadOf(record.load);
532
582
  if (typeof record.unavailable === "string") {
533
583
  return {
534
584
  unavailable: record.unavailable,
@@ -602,9 +652,21 @@ export function parseChromeCdpProbeOutput(stdout) {
602
652
  ...(browserWindow === undefined ? {} : { browserWindow }),
603
653
  ...(viewport === undefined ? {} : { viewport }),
604
654
  ...(network === undefined ? {} : { network }),
655
+ ...(load === undefined ? {} : { load }),
605
656
  ...(now === undefined ? {} : { now }),
606
657
  };
607
658
  }
659
+ function loadOf(value) {
660
+ if (!value || typeof value !== "object")
661
+ return undefined;
662
+ const source = value;
663
+ if (typeof source.loaded !== "boolean" ||
664
+ typeof source.waitedMs !== "number" ||
665
+ !Number.isFinite(source.waitedMs) ||
666
+ typeof source.seen !== "string")
667
+ return undefined;
668
+ return { loaded: source.loaded, waitedMs: source.waitedMs, seen: source.seen };
669
+ }
608
670
  function networkOf(value) {
609
671
  if (!value || typeof value !== "object")
610
672
  return undefined;
@@ -7,8 +7,10 @@ import { runDetachedStep } from "../detached.js";
7
7
  import { shellQuote } from "../shell.js";
8
8
  import { externalLinkHandlerSetup } from "./desktop-handoff.js";
9
9
  import { e2bShell } from "./shell.js";
10
- // Settle after opening the browser or the terminal, before the first screenshot: long enough for
11
- // a cold browser and page load to paint (2 s captured a blank desktop; the render needs 6-9 s).
10
+ // The wait before the first screenshot. After a terminal opens, or a browser without a DevTools
11
+ // read, it is a fixed sleep: long enough for a cold browser and page load to paint (2 s captured a
12
+ // blank desktop; the render needs 6-9 s). A Chromium participant waits for its page to load and
13
+ // paint (waitForChromePageLoad) and gives up after this long.
12
14
  export const DESKTOP_SETTLE_MS = 8_000;
13
15
  /** The DevTools port every Chromium participant launches with; see `chrome_debug_flags` below. */
14
16
  export const CHROME_DEVTOOLS_PORT = 9222;
@@ -1,5 +1,5 @@
1
1
  import type { ChromeNetworkCaptureRequest } from "./cdp-network-capture.js";
2
- import { type ChromeCdpProbeResult, type ChromeMobileEmulationRequest } from "./cdp-probe.js";
2
+ import { type ChromeCdpProbeResult, type ChromeMobileEmulationRequest, type ChromePageLoad } from "./cdp-probe.js";
3
3
  import type { RunDesktopGeometry } from "../../run/streams.js";
4
4
  import type { E2BDesktopSandbox } from "./sdk.js";
5
5
  /**
@@ -85,6 +85,13 @@ export declare function applyMobileEmulation(desktop: E2BDesktopSandbox, request
85
85
  /** The holder's capture status, when `network` was passed and the holder said. */
86
86
  network?: ChromeCdpProbeResult["network"];
87
87
  }>;
88
+ /**
89
+ * Wait until the participant's launch page has loaded and painted, read over DevTools in the
90
+ * sandbox, for at most `deadlineMs`. Resolves with what the wait saw, or undefined when it could
91
+ * not run (no python3, a failed command, output that is not the probe's), so the caller can fall
92
+ * back to a fixed wait.
93
+ */
94
+ export declare function waitForChromePageLoad(desktop: E2BDesktopSandbox, requestTimeoutMs: number, endpoint: ChromeCdpEndpoint, deadlineMs: number): Promise<ChromePageLoad | undefined>;
88
95
  /**
89
96
  * Start a holder that records the launch page's requests, and every later tab's, to
90
97
  * `network.eventsPath`, for a participant without mobile emulation; applyMobileEmulation passes
@@ -94,7 +101,7 @@ export declare function applyMobileEmulation(desktop: E2BDesktopSandbox, request
94
101
  export declare function startNetworkCapture(desktop: E2BDesktopSandbox, requestTimeoutMs: number, endpoint: ChromeCdpEndpoint, network: ChromeNetworkCaptureRequest): Promise<string>;
95
102
  /**
96
103
  * What the capture holder announced: when capture started on the launch page, on the sandbox
97
- * clock, or why it did not. Reads the holder's log up to 6 times, half a second apart by
104
+ * clock, or why it did not. Reads the holder's log up to 22 times, half a second apart by
98
105
  * `desktop.wait`, and reports a holder that printed nothing as unavailable.
99
106
  */
100
107
  export declare function readNetworkCaptureStart(desktop: E2BDesktopSandbox, requestTimeoutMs: number, holderName: string): Promise<NonNullable<ChromeCdpProbeResult["network"]>>;
@@ -213,10 +213,32 @@ export async function applyMobileEmulation(desktop, requestTimeoutMs, endpoint,
213
213
  ...capture,
214
214
  };
215
215
  }
216
+ /**
217
+ * Wait until the participant's launch page has loaded and painted, read over DevTools in the
218
+ * sandbox, for at most `deadlineMs`. Resolves with what the wait saw, or undefined when it could
219
+ * not run (no python3, a failed command, output that is not the probe's), so the caller can fall
220
+ * back to a fixed wait.
221
+ */
222
+ export async function waitForChromePageLoad(desktop, requestTimeoutMs, endpoint, deadlineMs) {
223
+ const result = await e2bShell(desktop)
224
+ .run(chromeCdpProbeCommand({ ...endpoint, prefer: "pinned", mode: "load", deadlineMs }), {
225
+ requestTimeoutMs,
226
+ // One poll can take 3.5 s past the deadline: a 2 s listing and a 1.5 s evaluate.
227
+ timeoutMs: deadlineMs + 10_000,
228
+ })
229
+ .catch(() => undefined);
230
+ if (result === undefined || result.exitCode !== 0)
231
+ return undefined;
232
+ return parseChromeCdpProbeOutput(result.stdout).load;
233
+ }
216
234
  /** The holder stops writing its events file at 32 MiB, so a read stops there too. */
217
235
  const NETWORK_EVENTS_MAX_BYTES = 32 * 1024 * 1024;
218
- /** How many times the capture holder's log is read for its announce, and how far apart. */
219
- const NETWORK_ANNOUNCE_READS = 6;
236
+ /**
237
+ * How many times the capture holder's log is read for its announce, and how far apart. The reads
238
+ * start after the page wait, which can end within a second of the holder's start, so they cover
239
+ * the 11 s a holder had when a fixed 8 s settle came before 3 s of reads.
240
+ */
241
+ const NETWORK_ANNOUNCE_READS = 22;
220
242
  const NETWORK_ANNOUNCE_EVERY_MS = 500;
221
243
  /**
222
244
  * Start a holder that records the launch page's requests, and every later tab's, to
@@ -235,7 +257,7 @@ export async function startNetworkCapture(desktop, requestTimeoutMs, endpoint, n
235
257
  }
236
258
  /**
237
259
  * What the capture holder announced: when capture started on the launch page, on the sandbox
238
- * clock, or why it did not. Reads the holder's log up to 6 times, half a second apart by
260
+ * clock, or why it did not. Reads the holder's log up to 22 times, half a second apart by
239
261
  * `desktop.wait`, and reports a holder that printed nothing as unavailable.
240
262
  */
241
263
  export async function readNetworkCaptureStart(desktop, requestTimeoutMs, holderName) {
@@ -253,7 +275,7 @@ export async function readNetworkCaptureStart(desktop, requestTimeoutMs, holderN
253
275
  });
254
276
  }
255
277
  return {
256
- unavailable: `the in-sandbox holder printed nothing within ${(NETWORK_ANNOUNCE_READS * NETWORK_ANNOUNCE_EVERY_MS) / 1000} s of the desktop settling`,
278
+ unavailable: `the in-sandbox holder printed nothing within ${(NETWORK_ANNOUNCE_READS * NETWORK_ANNOUNCE_EVERY_MS) / 1000} s of the page wait`,
257
279
  };
258
280
  }
259
281
  /**
@@ -192,7 +192,8 @@ export function streamScreenshotPaths(bundle) {
192
192
  return paths;
193
193
  }
194
194
  function isHarnessScreenshotPath(normalized) {
195
- return normalized.startsWith("screenshots/") && normalized.toLowerCase().endsWith(".png");
195
+ const lower = normalized.toLowerCase();
196
+ return (normalized.startsWith("screenshots/") && (lower.endsWith(".png") || lower.endsWith(".webp")));
196
197
  }
197
198
  function isRunRootEvidenceReference(value) {
198
199
  return (typeof value === "string" &&
@@ -1092,11 +1092,23 @@ an unclaimed frame on a raw, silent or live-only trace, and `ocr_scrubbed`, whic
1092
1092
  the trace contract reserves but no writer produces. An aggregate raw declaration
1093
1093
  wins over every frame claim.
1094
1094
 
1095
- A screenshot is a non-interlaced PNG whose first chunk is a 13-byte IHDR, whose
1096
- IEND is empty, and which holds image data only. A chunk outside IHDR, PLTE, IDAT,
1097
- IEND, tRNS, cHRM, gAMA, sBIT, sRGB, pHYs and bKGD (text chunks, ICC profiles,
1098
- Exif, timestamps, private chunks) fails verification. The harness writers drop
1099
- those chunks before writing a frame.
1095
+ A screenshot is a PNG or, at a path ending `.webp`, a lossless WebP. A PNG
1096
+ screenshot is non-interlaced, its first chunk is a 13-byte IHDR, its IEND is
1097
+ empty, and it holds image data only. A chunk outside IHDR, PLTE, IDAT, IEND,
1098
+ tRNS, cHRM, gAMA, sBIT, sRGB, pHYs and bKGD (text chunks, ICC profiles, Exif,
1099
+ timestamps, private chunks) fails verification. The harness writers drop those
1100
+ chunks before writing a frame. A WebP screenshot is the simple lossless form: a
1101
+ RIFF container holding one VP8L chunk that ends the file, with version 0 and
1102
+ dimensions under the PNG pixel cap. The extended form (VP8X), which carries Exif,
1103
+ XMP, ICC profiles and animation, fails verification. Verify reads a WebP's
1104
+ structure without decoding its pixels, so a WebP frame is always full-fidelity
1105
+ evidence: only a PNG in the redactor's shape counts as blurred. Earlier humanish
1106
+ versions do not accept WebP screenshots. A computer-use participant writes each
1107
+ frame as PNG during its session. When the session ends, its trace references
1108
+ the lossless WebP of each frame encoded by then in place of its PNG, and the run
1109
+ removes those PNGs once its final bundle is written. A study that blurs its
1110
+ screenshots keeps PNG frames, and an interrupted run can hold both files of a
1111
+ frame, with run.json's live trace naming the PNG.
1100
1112
 
1101
1113
  The public-safety scan classifies a run file by its bytes, not its name. It scans
1102
1114
  a file for secret and path patterns only when the bytes are strict UTF-8 with no
@@ -1175,7 +1187,8 @@ Its `derivation.json` uses schema `humanish.redacted-derivation.v1` and records:
1175
1187
  rows, sorted by path using JavaScript string order;
1176
1188
  - `files[]`: source-relative `path`, `sourceSha256`, action (`copied`, `updated`,
1177
1189
  `blurred`, or `omitted`), and `outputSha256` for retained files or a reason for
1178
- omission;
1190
+ omission; `outputPath` when the copy holds the file at another path, as it does a
1191
+ WebP frame;
1179
1192
  - `generated[]`: paths and SHA256s of rebuilt Observer projections.
1180
1193
 
1181
1194
  The receipt describes the transformation; it does not attest that a participant's
@@ -1195,11 +1208,16 @@ An id that appears only in free text, in a run with no receipts, is not known to
1195
1208
  export. `sandbox-receipts.ndjson` is omitted, and the source keeps its raw ids,
1196
1209
  which reclaim reads.
1197
1210
 
1198
- PNG files that actor traces reference as stream screenshots are re-encoded as
1199
- blurred thumbnails. A PNG that nothing in `run.json` cites is omitted and
1200
- inventoried. Export refuses a run whose feedback candidates, adapter artifacts or
1201
- stream artifact entries cite a PNG that is not a stream screenshot, since verify
1202
- cannot read that image and dropping it would change the evidence.
1211
+ PNG and WebP files that actor traces reference as stream screenshots are
1212
+ re-encoded as blurred PNG thumbnails. A WebP frame's thumbnail is written at the
1213
+ same path ending `.png`, and the copy's actor traces, stream artifacts, feedback
1214
+ evidence and adapter artifacts name that path. Export refuses a run that already
1215
+ holds a file at that path, and a run whose other copied text names the WebP frame,
1216
+ since the copy could not resolve that reference. An image that nothing in
1217
+ `run.json` cites is omitted and inventoried. Export refuses a run whose feedback
1218
+ candidates, adapter artifacts or stream artifact entries cite an image that is not
1219
+ a stream screenshot, since verify cannot read that image and dropping it would
1220
+ change the evidence.
1203
1221
  Known actor screenshot declarations describe export-time blur while retaining the
1204
1222
  original redaction notes. Observer is rebuilt; old feedback outputs, local process
1205
1223
  status and the operational sandbox journal are omitted and inventoried. A derivative