humanish 0.111.0 → 0.111.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/CONTEXT.md +1 -1
  3. package/dist/actors/codex/restricted-participant-policy.d.ts +2 -0
  4. package/dist/actors/codex/restricted-participant-policy.js +30 -2
  5. package/dist/actors/codex/restricted-participant.js +1 -1
  6. package/dist/actors/computer-use/executor-diagnostic.d.ts +15 -0
  7. package/dist/actors/computer-use/executor-diagnostic.js +105 -0
  8. package/dist/actors/computer-use/executor-error.d.ts +18 -3
  9. package/dist/actors/computer-use/executor-error.js +30 -4
  10. package/dist/actors/computer-use/loop/actions.d.ts +6 -3
  11. package/dist/actors/computer-use/loop/actions.js +27 -7
  12. package/dist/actors/computer-use/loop/ending.js +9 -1
  13. package/dist/actors/computer-use/loop/types.d.ts +14 -0
  14. package/dist/actors/computer-use/loop.d.ts +1 -1
  15. package/dist/actors/computer-use/loop.js +1 -1
  16. package/dist/actors/computer-use/openai-context.d.ts +8 -2
  17. package/dist/actors/computer-use/openai-context.js +91 -26
  18. package/dist/actors/computer-use/openai-provider.js +55 -12
  19. package/dist/actors/computer-use/openai-requests.d.ts +2 -1
  20. package/dist/actors/computer-use/openai-requests.js +4 -1
  21. package/dist/actors/contract.d.ts +7 -1
  22. package/dist/browser-control/client.js +5 -5
  23. package/dist/browser-control/protocol.d.ts +30 -1
  24. package/dist/browser-control/protocol.js +19 -2
  25. package/dist/browser-control/transport.d.ts +3 -2
  26. package/dist/browser-control/transport.js +12 -7
  27. package/dist/cli/commands/study-route-computer-use.js +2 -2
  28. package/dist/cli/commands/study-route-shared-world.js +2 -2
  29. package/dist/cli/commands/study.js +1 -2
  30. package/dist/cli/doctor.js +11 -0
  31. package/dist/evidence/encoded-text.d.ts +3 -3
  32. package/dist/evidence/encoded-text.js +10 -5
  33. package/dist/evidence/redaction.d.ts +2 -2
  34. package/dist/evidence/redaction.js +18 -7
  35. package/dist/guest/browser-tools.js +5 -1
  36. package/dist/guest/chromium-text.js +42 -18
  37. package/dist/guest/desktop-executor.js +5 -1
  38. package/dist/keys/key-resolution.d.ts +3 -0
  39. package/dist/keys/key-resolution.js +15 -2
  40. package/dist/observer-app.html +6 -6
  41. package/dist/routes/computer-use/bundle-parts.d.ts +4 -0
  42. package/dist/routes/computer-use/bundle-parts.js +7 -2
  43. package/dist/routes/computer-use/fanout-bundle.js +4 -2
  44. package/dist/routes/computer-use/local-vm.js +1 -2
  45. package/dist/routes/computer-use/participant-model.js +6 -1
  46. package/dist/routes/computer-use/plan.js +3 -4
  47. package/dist/routes/computer-use/route.js +1 -2
  48. package/dist/routes/computer-use/setup.js +2 -2
  49. package/dist/routes/preview.js +1 -0
  50. package/dist/routes/scripted/plan.js +4 -5
  51. package/dist/routes/scripted/route.js +2 -2
  52. package/dist/routes/shared-world/plan.js +4 -5
  53. package/dist/routes/shared-world/route.js +2 -2
  54. package/dist/routes/shared-world/setup.js +1 -0
  55. package/dist/routes/terminal/dry-run.js +1 -0
  56. package/dist/routes/terminal/live-finish.js +24 -5
  57. package/dist/routes/terminal/live-sandbox.js +5 -1
  58. package/dist/routes/terminal/plan.js +3 -4
  59. package/dist/routes/terminal/recorder.d.ts +6 -0
  60. package/dist/routes/terminal/recorder.js +61 -6
  61. package/dist/routes/terminal/route.js +1 -2
  62. package/dist/routes/terminal/session.js +2 -1
  63. package/dist/routes/terminal/token-usage.d.ts +3 -0
  64. package/dist/routes/terminal/token-usage.js +5 -0
  65. package/dist/routes/terminal/types.d.ts +2 -1
  66. package/dist/routes/terminal/types.js +2 -1
  67. package/dist/run/dry-run.js +1 -0
  68. package/dist/run/results.d.ts +2 -0
  69. package/dist/run/run.d.ts +2 -0
  70. package/dist/run/run.js +16 -2
  71. package/dist/run/terminal-contract.d.ts +10 -0
  72. package/dist/run/terminal-contract.js +27 -9
  73. package/dist/study/composition-rules.js +17 -18
  74. package/dist/study/doctor.js +4 -5
  75. package/dist/study/migrate/v2.d.ts +5 -0
  76. package/dist/study/migrate/v2.js +23 -0
  77. package/dist/study/parse/front.js +2 -21
  78. package/dist/study/parse/url-credentials.d.ts +3 -3
  79. package/dist/study/parse/url-credentials.js +155 -13
  80. package/dist/study/persona-resolve.js +2 -2
  81. package/dist/study/plan-base.js +3 -4
  82. package/dist/study/plan-participants.js +3 -3
  83. package/dist/study/plan-types.d.ts +2 -0
  84. package/dist/study/plan.js +11 -5
  85. package/dist/study/routing.js +11 -11
  86. package/dist/study/run-study-options.js +2 -2
  87. package/dist/study/study-fields.d.ts +1 -13
  88. package/dist/study/study-fields.js +0 -24
  89. package/dist/study/summary.js +7 -7
  90. package/dist/study/types.d.ts +1 -0
  91. package/dist/study/validation.js +11 -11
  92. package/dist/study/warnings.d.ts +8 -0
  93. package/dist/study/warnings.js +22 -7
  94. package/dist/substrates/local/runtime-config.js +6 -6
  95. package/docs/architecture/browser-control.md +12 -2
  96. package/docs/architecture/guest-desktop.md +7 -1
  97. package/docs/architecture/terminal-product-route.md +3 -2
  98. package/docs/contracts/schemas.md +16 -5
  99. package/package.json +1 -1
@@ -7,20 +7,46 @@ import { asArray, asRecord, inputMessage, openAiActionToCua } from "./openai-wir
7
7
  // The Responses API pairs items: every computer_call needs its computer_call_output, and a
8
8
  // reasoning item travels with the item it preceded. The conversation is therefore cut only at
9
9
  // exchange boundaries: one reply's output items together with the items that answered them.
10
- // When the estimate passes the budget, the opening screenshot goes first, then the oldest
11
- // exchanges become lines of a progress note that keeps their reasoning summaries, messages and
12
- // actions as text. The note is an assistant message, so text the model wrote (which can quote a
13
- // web page) keeps the trust it had when the model wrote it. The newest MIN_KEPT_EXCHANGES
14
- // exchanges are never collapsed, so a request passes the budget when they alone exceed it; the
15
- // trace's per-request estimate shows when that happens.
16
- /** The estimated input a carried conversation may reach before its oldest turns are summarized. */
10
+ // When the estimate passes the budget, the conversation is cut down to half the budget in one
11
+ // step: the opening screenshot goes first, then the oldest exchanges become lines of a progress
12
+ // note that keeps their reasoning summaries, messages and actions, and the text that answered
13
+ // them, such as a note that an action was not run. The note is an assistant message, so text the
14
+ // model wrote (which can quote a web page) keeps the trust it had when the model wrote it. The newest MIN_KEPT_EXCHANGES exchanges are never collapsed, so a
15
+ // request passes the budget when they alone exceed it; the trace's per-request estimate shows when
16
+ // that happens.
17
+ //
18
+ // A cut rewrites the start of the prompt, so the provider's prompt cache misses on the request
19
+ // after it. Between cuts nothing already sent changes: each request starts with the whole of the
20
+ // previous request, which the cache can serve. A cut of one exchange per turn would rewrite
21
+ // the note on every request past the budget, and each of those requests would be billed in full.
22
+ /** The estimated input a carried conversation may reach before it is cut. */
17
23
  export const CONTEXT_TOKEN_BUDGET = 64_000;
24
+ /**
25
+ * The share of the budget a cut brings the carried conversation down to. The rest is the growth
26
+ * the cache serves until the next cut: about 19 turns of 1280x800 screenshots, or 12 at
27
+ * 1920x1080.
28
+ */
29
+ const CUT_TARGET_SHARE = 0.5;
18
30
  /** Exchanges always carried whole, so the model sees its latest actions with their screens. */
19
31
  const MIN_KEPT_EXCHANGES = 2;
20
- /** The progress note's length cap; past it, the oldest lines are counted instead of shown. */
21
- const NOTE_CHAR_LIMIT = 16_000;
22
- /** One note line's cap, so a reply with many actions cannot push the note past its own cap. */
23
- const LINE_CHAR_LIMIT = 1_200;
32
+ /**
33
+ * The progress note's length cap, about 8,000 tokens. Past it, the oldest lines after the first
34
+ * NOTE_HEAD_LINES are counted instead of shown. The note sits in the cached prefix between cuts,
35
+ * so its size costs little after the request that writes it.
36
+ */
37
+ const NOTE_CHAR_LIMIT = 32_000;
38
+ /**
39
+ * Note lines kept whatever the note's length. A session's first turns are where it usually learns
40
+ * what it must carry to the end, such as an account, an address or a code, and the newest lines
41
+ * hold its latest progress, so the note gives up the turns between them.
42
+ */
43
+ const NOTE_HEAD_LINES = 4;
44
+ /**
45
+ * One note line's cap, so a reply with many actions cannot push the note past its own cap. It
46
+ * leaves room for the clipped thought, message and hint text plus about 1,000 characters of
47
+ * actions, such as several typed form fields.
48
+ */
49
+ const LINE_CHAR_LIMIT = 2_000;
24
50
  /**
25
51
  * Image input tokens per 32-pixel patch. The gpt-5.x and gpt-6 models bill 1.2 per patch, and at
26
52
  * the default (auto) detail gpt-5.6 and gpt-6 keep the screenshot's own size. Models that resize
@@ -36,7 +62,11 @@ export class CarriedConversation {
36
62
  /** The latest accepted reply's output items, which the next request answers. */
37
63
  pending;
38
64
  notes = [];
39
- notesDropped = 0;
65
+ /**
66
+ * The turns whose note lines were dropped to keep the note under its cap. Lines are dropped in
67
+ * turn order from just after the head, so they are one run of consecutive turns.
68
+ */
69
+ elided;
40
70
  collapsedTurns = 0;
41
71
  constructor(budget = CONTEXT_TOKEN_BUDGET) {
42
72
  this.budget = budget;
@@ -76,7 +106,10 @@ export class CarriedConversation {
76
106
  };
77
107
  }
78
108
  trim() {
79
- while (estimateTokens(this.carried()) > this.budget) {
109
+ if (estimateTokens(this.carried()) <= this.budget)
110
+ return;
111
+ const target = Math.floor(this.budget * CUT_TARGET_SHARE);
112
+ while (estimateTokens(this.carried()) > target) {
80
113
  if (this.opening !== undefined && imagesOf(this.opening).length > 0) {
81
114
  this.opening = withoutImages(this.opening);
82
115
  continue;
@@ -84,30 +117,56 @@ export class CarriedConversation {
84
117
  if (this.exchanges.length <= MIN_KEPT_EXCHANGES)
85
118
  return;
86
119
  const oldest = this.exchanges.shift();
87
- this.notes.push(describeExchange(oldest));
120
+ this.notes.push({ turn: oldest.turn, text: describeExchange(oldest) });
88
121
  this.collapsedTurns += 1;
89
- while (this.notes.join("\n").length > NOTE_CHAR_LIMIT && this.notes.length > 1) {
90
- this.notes.shift();
91
- this.notesDropped += 1;
92
- }
122
+ this.capNote();
123
+ }
124
+ }
125
+ capNote() {
126
+ while (this.noteText().length > NOTE_CHAR_LIMIT && this.notes.length > NOTE_HEAD_LINES + 1) {
127
+ const [dropped] = this.notes.splice(NOTE_HEAD_LINES, 1);
128
+ this.elided = { first: this.elided?.first ?? dropped.turn, last: dropped.turn };
129
+ }
130
+ }
131
+ noteText() {
132
+ const lines = this.notes.map((line) => line.text);
133
+ if (this.elided !== undefined) {
134
+ const { first, last } = this.elided;
135
+ const which = first === last ? `Turn ${first} is` : `Turns ${first} to ${last} are`;
136
+ lines.splice(NOTE_HEAD_LINES, 0, `(${which} not listed.)`);
93
137
  }
138
+ return lines.join("\n");
94
139
  }
95
140
  noteItem() {
96
141
  if (this.notes.length === 0)
97
142
  return undefined;
98
143
  const lead = `My notes on my earlier turns in this session, oldest first. Their screenshots are no ` +
99
- `longer shown; this is what I thought, said and did.` +
100
- (this.notesDropped === 0 ? "" : ` (${this.notesDropped} earlier turns are not listed.)`);
144
+ `longer shown; this is what I thought, said and did, and what I was told after.`;
101
145
  return inputMessage("assistant", [
102
- { type: "output_text", text: `${lead}\n${this.notes.join("\n")}` },
146
+ { type: "output_text", text: `${lead}\n${this.noteText()}` },
103
147
  ]);
104
148
  }
105
149
  }
106
- /** One exchange as a progress-note line: its reasoning summary, message and actions. */
150
+ /**
151
+ * One exchange as a progress-note line: its reasoning summary, message and actions, and the text
152
+ * that came back with their screenshot, such as a note that an action was not run.
153
+ */
107
154
  function describeExchange(exchange) {
108
155
  const thought = [];
109
156
  const said = [];
110
157
  const did = [];
158
+ const told = [];
159
+ // The answers are the call outputs, whose content is a screenshot, and hint messages.
160
+ for (const raw of exchange.answers) {
161
+ const item = asRecord(raw);
162
+ if (item.type === "computer_call_output")
163
+ continue;
164
+ for (const entry of asArray(item.content)) {
165
+ const text = asRecord(entry).text;
166
+ if (typeof text === "string" && text.length > 0)
167
+ told.push(text);
168
+ }
169
+ }
111
170
  for (const raw of exchange.output) {
112
171
  const item = asRecord(raw);
113
172
  if (item.type === "reasoning")
@@ -131,13 +190,19 @@ function describeExchange(exchange) {
131
190
  }
132
191
  }
133
192
  }
134
- const parts = [
193
+ const lead = `Turn ${exchange.turn}: `;
194
+ const before = [
135
195
  thought.length > 0 ? `thought: ${clip(thought.join(" "), 400)}` : undefined,
136
196
  said.length > 0 ? `said: ${clip(said.join(" "), 300)}` : undefined,
137
- did.length > 0 ? `did: ${did.join(", ")}` : undefined,
138
197
  ].filter((part) => part !== undefined);
139
- const line = `Turn ${exchange.turn}: ${parts.length > 0 ? parts.join("; ") : "no recorded output"}`;
140
- return `${clip(line, LINE_CHAR_LIMIT - 1)}.`;
198
+ const after = told.length > 0 ? [`was told: ${clip(told.join(" "), 300)}`] : [];
199
+ // The hint can say an action did not run, so the actions give up whatever room it needs.
200
+ const fixed = [...before, ...after].reduce((length, part) => length + part.length + 2, 0);
201
+ const room = LINE_CHAR_LIMIT - 1 - lead.length - fixed - "did: ".length;
202
+ const actions = did.length > 0 ? [`did: ${clip(did.join(", "), room)}`] : [];
203
+ const parts = [...before, ...actions, ...after];
204
+ const line = clip(`${lead}${parts.length > 0 ? parts.join("; ") : "no recorded output"}`, LINE_CHAR_LIMIT - 1);
205
+ return /[.!?]$/.test(line) ? line : `${line}.`;
141
206
  }
142
207
  function describeAction(action) {
143
208
  switch (action.kind) {
@@ -175,17 +175,51 @@ export function namedProviderErrorCode(bodyText) {
175
175
  // by switching to explicit-context mode) from any other non-ok status. It never
176
176
  // carries the apiKey or the response body.
177
177
  class ZdrError extends Error {
178
- constructor() {
179
- super("OpenAI Responses rejected server-side state (zero data retention)");
178
+ rejection;
179
+ constructor(rejection) {
180
+ super(rejection === "stored_item"
181
+ ? "OpenAI Responses could not find an item the request referenced: the server keeps none for this organization (zero data retention)"
182
+ : "OpenAI Responses rejected server-side state (zero data retention)");
183
+ this.rejection = rejection;
180
184
  this.name = "ZdrError";
181
185
  }
182
186
  }
183
- // A 400 whose body mentions any of these means the account/org cannot use
184
- // server-side response state, so we must fall back to explicit-context mode.
185
- function isZdrRejection(bodyText) {
186
- return (bodyText.includes("Zero Data Retention") ||
187
- bodyText.includes("zero data retention") ||
188
- bodyText.includes("previous_response_id"));
187
+ /** The message of a 404 for an item the server never kept, as the wire sends it. */
188
+ const STORED_ITEM_NOT_FOUND = /Item with id '?[A-Za-z0-9_-]+'? not found/;
189
+ /**
190
+ * Error codes that name a failure of their own. Their messages can echo request text, such as a
191
+ * model name or the prompt, so the words below must not turn them into a retention answer.
192
+ */
193
+ const OTHER_FAILURE_CODES = new Set([
194
+ "invalid_prompt",
195
+ "model_not_found",
196
+ "context_length_exceeded",
197
+ "rate_limit_exceeded",
198
+ "insufficient_quota",
199
+ "invalid_image",
200
+ "invalid_image_format",
201
+ "invalid_base64_image",
202
+ "invalid_image_url",
203
+ "image_too_large",
204
+ "image_parse_error",
205
+ "image_content_policy_violation",
206
+ ]);
207
+ // A 400 or 404 whose body says the organization cannot use server-side response state, so the
208
+ // provider must carry the conversation itself (explicit-context mode). A 404 counts only with the
209
+ // stored-item message. Captured bodies are in tests/fixtures/openai-store-less/.
210
+ function zdrRejection(status, bodyText) {
211
+ const code = namedProviderErrorCode(bodyText);
212
+ if (code !== undefined && OTHER_FAILURE_CODES.has(code))
213
+ return undefined;
214
+ if (status === 404)
215
+ return STORED_ITEM_NOT_FOUND.test(bodyText) ? "stored_item" : undefined;
216
+ if (/zero[ -]data[ -]retention/i.test(bodyText))
217
+ return "zero_data_retention";
218
+ if (bodyText.includes("previous_response_id"))
219
+ return "previous_response";
220
+ if (STORED_ITEM_NOT_FOUND.test(bodyText))
221
+ return "stored_item";
222
+ return undefined;
189
223
  }
190
224
  // A typed error so nextTurn can latch reasoning summaries off and retry the turn
191
225
  // (an org not verified for summaries, or a model without a summarizer, 400s the
@@ -307,11 +341,20 @@ async function postResponse(transport, body, signal, retries, interaction, spend
307
341
  return parsed;
308
342
  }
309
343
  lastStatus = res.status;
344
+ if (res.status === 404) {
345
+ // A store-less organization answers a reference to an item it never kept with 404.
346
+ const bodyText = await res.text().catch(() => "");
347
+ const rejection = zdrRejection(404, bodyText);
348
+ if (rejection !== undefined)
349
+ throw new ZdrError(rejection);
350
+ const code = namedProviderErrorCode(bodyText);
351
+ throw new Error(`OpenAI Responses 404${code === undefined ? "" : ` ${code}`}`);
352
+ }
310
353
  if (res.status === 400) {
311
354
  const bodyText = await res.text();
312
- if (isZdrRejection(bodyText)) {
313
- throw new ZdrError();
314
- }
355
+ const rejection = zdrRejection(400, bodyText);
356
+ if (rejection !== undefined)
357
+ throw new ZdrError(rejection);
315
358
  if (isSummaryRejection(bodyText)) {
316
359
  throw new SummaryRejectionError();
317
360
  }
@@ -415,7 +458,7 @@ export function createOpenAiResponsesProvider(options) {
415
458
  continue;
416
459
  }
417
460
  if (error instanceof ZdrError && state.mode !== "explicit_context") {
418
- record.switched();
461
+ record.switched(error.rejection);
419
462
  state.mode = "explicit_context";
420
463
  continue;
421
464
  }
@@ -48,13 +48,14 @@ export declare function acceptReply(state: OpenAiConversationState, parsed: Pars
48
48
  export declare class ConversationRecord {
49
49
  private readonly now;
50
50
  private explicitReason;
51
+ private rejection;
51
52
  private switchedAt;
52
53
  private switchedAtRequest;
53
54
  private summarizedTurns;
54
55
  private readonly requests;
55
56
  constructor(configured: boolean, now: () => number);
56
57
  /** The organization rejected server-side state, so the request being sent is explicit_context. */
57
- switched(): void;
58
+ switched(rejection: NonNullable<ActorConversation["rejection"]>): void;
58
59
  /** A participant request just went out with `sent` after the carried conversation. */
59
60
  requested(state: OpenAiConversationState, sent: readonly unknown[]): void;
60
61
  snapshot(state: OpenAiConversationState): ActorConversation;
@@ -94,6 +94,7 @@ export function acceptReply(state, parsed, closing, sent) {
94
94
  export class ConversationRecord {
95
95
  now;
96
96
  explicitReason;
97
+ rejection;
97
98
  switchedAt;
98
99
  switchedAtRequest;
99
100
  summarizedTurns = 0;
@@ -104,8 +105,9 @@ export class ConversationRecord {
104
105
  this.explicitReason = "configured";
105
106
  }
106
107
  /** The organization rejected server-side state, so the request being sent is explicit_context. */
107
- switched() {
108
+ switched(rejection) {
108
109
  this.explicitReason = "zdr_rejection";
110
+ this.rejection = rejection;
109
111
  this.switchedAt = new Date(this.now()).toISOString();
110
112
  // Counted like requests[], which includes replies set aside at their output limit.
111
113
  this.switchedAtRequest = this.requests.length + 1;
@@ -130,6 +132,7 @@ export class ConversationRecord {
130
132
  return {
131
133
  mode: state.mode === "explicit_context" ? "explicit_context" : "threaded",
132
134
  ...(this.explicitReason === undefined ? {} : { explicitReason: this.explicitReason }),
135
+ ...(this.rejection === undefined ? {} : { rejection: this.rejection }),
133
136
  ...(this.switchedAt === undefined ? {} : { switchedAt: this.switchedAt }),
134
137
  ...(this.switchedAtRequest === undefined
135
138
  ? {}
@@ -206,13 +206,19 @@ export interface ActorRuntimeProvenance {
206
206
  * How a provider carried the conversation between its requests. `threaded`: the provider's server
207
207
  * kept it (OpenAI previous_response_id). `explicit_context`: the server kept none (a
208
208
  * zero-data-retention org, or zeroDataRetention set), so each request carried it from the client
209
- * within a token budget, with the oldest turns summarized as text once it passed the budget.
209
+ * within a token budget, with the oldest turns summarized as text at each cut past the budget.
210
210
  */
211
211
  export interface ActorConversation {
212
212
  /** The mode of the session's last request. */
213
213
  mode: "threaded" | "explicit_context";
214
214
  /** Why the session used explicit_context: configured, or the org rejected server-side state. */
215
215
  explicitReason?: "configured" | "zdr_rejection";
216
+ /**
217
+ * With `zdr_rejection`, the answer that made the switch: the error named zero data retention, it
218
+ * refused or could not find the previous response, or it could not find a stored item that the
219
+ * request referenced.
220
+ */
221
+ rejection?: "zero_data_retention" | "previous_response" | "stored_item";
216
222
  /** When a threaded session switched to explicit_context (ISO-8601), and on which request. */
217
223
  switchedAt?: string;
218
224
  switchedAtRequest?: number;
@@ -23,7 +23,7 @@ export function createBrowserControlClient(options) {
23
23
  // The transport closes synchronously when its stream is already unusable, so its close handler
24
24
  // takes the state that exists before the transport does.
25
25
  const session = Object.assign(state, {
26
- transport: new BrowserControlTransport(options.transport, (value) => acceptReply(session, value), (code) => closeClient(state, code)),
26
+ transport: new BrowserControlTransport(options.transport, (value) => acceptReply(session, value), (code, diagnostic) => closeClient(state, code, diagnostic)),
27
27
  });
28
28
  const executor = {
29
29
  stallRecovery: "fail_closed",
@@ -41,13 +41,13 @@ export function createBrowserControlClient(options) {
41
41
  },
42
42
  };
43
43
  }
44
- function closeClient(state, code) {
44
+ function closeClient(state, code, diagnostic) {
45
45
  state.closed = true;
46
46
  const operation = state.pending;
47
47
  state.pending = undefined;
48
48
  if (operation) {
49
49
  operation.dispose();
50
- operation.reject(new ComputerUseExecutorError(code, operation.written ? "outcome_uncertain" : "not_dispatched"));
50
+ operation.reject(new ComputerUseExecutorError(code, operation.written ? "outcome_uncertain" : "not_dispatched", { diagnostic }));
51
51
  }
52
52
  }
53
53
  /** Settles the pending exchange once both its reply has arrived and its request write finished. */
@@ -63,12 +63,12 @@ function finishExchange(session) {
63
63
  else if (operation.reply.ok)
64
64
  operation.resolve(operation.reply);
65
65
  else {
66
- const { code, disposition } = operation.reply.error;
66
+ const { code, disposition, reason, diagnostic } = operation.reply.error;
67
67
  if (operation.request.operation !== "EXECUTE" ||
68
68
  code !== "action_rejected" ||
69
69
  disposition !== "not_dispatched")
70
70
  session.transport.close(code);
71
- operation.reject(new ComputerUseExecutorError(code, disposition));
71
+ operation.reject(new ComputerUseExecutorError(code, disposition, { reason, diagnostic }));
72
72
  }
73
73
  }
74
74
  /** Matches a reply frame to the pending request; any mismatch closes the channel. */
@@ -1,7 +1,8 @@
1
1
  import { z } from "zod";
2
2
  import type { CuaAction, CuaObservation } from "../actors/computer-use/loop.js";
3
3
  import { type HeardSpeech } from "../actors/computer-use/speech.js";
4
- import { type CuaExecutorErrorCode } from "../actors/computer-use/executor-error.js";
4
+ import { type CuaExecutorErrorCode, type CuaRejectionReason } from "../actors/computer-use/executor-error.js";
5
+ import { type CuaExecutorDiagnostic } from "../actors/computer-use/executor-diagnostic.js";
5
6
  export declare const BROWSER_CONTROL_VERSION = 1;
6
7
  export declare const BROWSER_CONTROL_LIMITS: Readonly<{
7
8
  frameBytes: number;
@@ -471,6 +472,32 @@ declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
471
472
  not_dispatched: "not_dispatched";
472
473
  outcome_uncertain: "outcome_uncertain";
473
474
  }>;
475
+ reason: z.ZodOptional<z.ZodEnum<{
476
+ extra_tab: "extra_tab";
477
+ }>>;
478
+ diagnostic: z.ZodOptional<z.ZodObject<{
479
+ step: z.ZodEnum<{
480
+ cdp_detach: "cdp_detach";
481
+ cdp_session: "cdp_session";
482
+ channel: "channel";
483
+ channel_write: "channel_write";
484
+ focus_probe: "focus_probe";
485
+ frame_tree: "frame_tree";
486
+ insert_text: "insert_text";
487
+ isolated_world: "isolated_world";
488
+ owner_window_check: "owner_window_check";
489
+ text_port: "text_port";
490
+ }>;
491
+ category: z.ZodEnum<{
492
+ channel_closed: "channel_closed";
493
+ detached: "detached";
494
+ navigation: "navigation";
495
+ protocol_error: "protocol_error";
496
+ target_closed: "target_closed";
497
+ timeout: "timeout";
498
+ unknown: "unknown";
499
+ }>;
500
+ }, z.core.$strict>>;
474
501
  }, z.core.$strict>;
475
502
  }, z.core.$strict>], "ok">;
476
503
  export type BrowserControlRequest = z.infer<typeof requestSchema>;
@@ -488,5 +515,7 @@ export declare function decodeBrowserControlObservation(value: unknown): CuaObse
488
515
  export declare function safeBrowserControlFailure(error: unknown, dispatched: boolean): {
489
516
  code: CuaExecutorErrorCode;
490
517
  disposition: "not_dispatched" | "outcome_uncertain";
518
+ reason?: CuaRejectionReason;
519
+ diagnostic?: CuaExecutorDiagnostic;
491
520
  };
492
521
  export {};
@@ -1,7 +1,8 @@
1
1
  import { PNG } from "pngjs";
2
2
  import { z } from "zod";
3
3
  import { CUA_SPEECH_LIMITS } from "../actors/computer-use/speech.js";
4
- import { ComputerUseExecutorError, isComputerUseExecutorError, } from "../actors/computer-use/executor-error.js";
4
+ import { ComputerUseExecutorError, CUA_REJECTION_REASONS, isComputerUseExecutorError, } from "../actors/computer-use/executor-error.js";
5
+ import { CUA_DIAGNOSTIC_CATEGORIES, CUA_DIAGNOSTIC_STEPS, } from "../actors/computer-use/executor-diagnostic.js";
5
6
  import { desktopRecordingMetadataSchema } from "../evidence/desktop-recording-types.js";
6
7
  export const BROWSER_CONTROL_VERSION = 1;
7
8
  export const BROWSER_CONTROL_LIMITS = Object.freeze({
@@ -194,6 +195,11 @@ const errorCode = z.enum([
194
195
  "action_rejected",
195
196
  "execution_failed",
196
197
  ]);
198
+ // Two fixed words; a reply never carries browser error text.
199
+ const diagnosticSchema = z.strictObject({
200
+ step: z.enum(CUA_DIAGNOSTIC_STEPS),
201
+ category: z.enum(CUA_DIAGNOSTIC_CATEGORIES),
202
+ });
197
203
  const replyBase = {
198
204
  ...common,
199
205
  type: z.literal("reply"),
@@ -213,6 +219,8 @@ const replySchema = z.discriminatedUnion("ok", [
213
219
  error: z.strictObject({
214
220
  code: errorCode,
215
221
  disposition: z.enum(["not_dispatched", "outcome_uncertain"]),
222
+ reason: z.enum(CUA_REJECTION_REASONS).optional(),
223
+ diagnostic: diagnosticSchema.optional(),
216
224
  }),
217
225
  }),
218
226
  ]);
@@ -243,6 +251,9 @@ export function parseBrowserControlReply(value) {
243
251
  throw new ComputerUseExecutorError("invalid_response", "outcome_uncertain");
244
252
  const reply = parsed.data;
245
253
  if ((reply.operation === "EXECUTE") !== (reply.actionId !== undefined) ||
254
+ (!reply.ok &&
255
+ reply.error.reason !== undefined &&
256
+ (reply.error.code !== "action_rejected" || reply.error.disposition !== "not_dispatched")) ||
246
257
  (reply.ok &&
247
258
  ((reply.operation === "OBSERVE") !== (reply.observation !== undefined) ||
248
259
  (reply.operation === "FINISH_RECORDING") !== (reply.recording !== undefined))))
@@ -351,8 +362,14 @@ export function decodeBrowserControlObservation(value) {
351
362
  };
352
363
  }
353
364
  export function safeBrowserControlFailure(error, dispatched) {
365
+ // Only a declared error's fixed diagnostic crosses; an arbitrary exception's text never does.
354
366
  if (isComputerUseExecutorError(error))
355
- return { code: error.code, disposition: error.disposition };
367
+ return {
368
+ code: error.code,
369
+ disposition: error.disposition,
370
+ ...(error.reason === undefined ? {} : { reason: error.reason }),
371
+ ...(error.diagnostic === undefined ? {} : { diagnostic: { ...error.diagnostic } }),
372
+ };
356
373
  return {
357
374
  code: "action_rejected",
358
375
  disposition: dispatched ? "outcome_uncertain" : "not_dispatched",
@@ -1,5 +1,6 @@
1
1
  import type { Duplex } from "node:stream";
2
2
  import { type CuaExecutorErrorCode } from "../actors/computer-use/executor-error.js";
3
+ import { type CuaExecutorDiagnostic } from "../actors/computer-use/executor-diagnostic.js";
3
4
  /** Owned byte channel only. This module never discovers endpoints or reconnects. */
4
5
  export declare class BrowserControlTransport {
5
6
  private readonly stream;
@@ -15,7 +16,7 @@ export declare class BrowserControlTransport {
15
16
  private pendingWrite;
16
17
  private frameTimer;
17
18
  private readonly decoder;
18
- constructor(stream: Duplex, onFrame: (value: unknown) => void, onClose: (code: CuaExecutorErrorCode) => void);
19
+ constructor(stream: Duplex, onFrame: (value: unknown) => void, onClose: (code: CuaExecutorErrorCode, diagnostic: CuaExecutorDiagnostic | undefined) => void);
19
20
  get closed(): boolean;
20
21
  private data;
21
22
  /** Relinquish the live byte stream synchronously after a complete frame. */
@@ -23,5 +24,5 @@ export declare class BrowserControlTransport {
23
24
  private error;
24
25
  private end;
25
26
  send(value: unknown, beforeWrite?: () => void): Promise<void>;
26
- close(code?: CuaExecutorErrorCode): void;
27
+ close(code?: CuaExecutorErrorCode, diagnostic?: CuaExecutorDiagnostic): void;
27
28
  }
@@ -1,6 +1,7 @@
1
1
  import { TextDecoder } from "node:util";
2
2
  import { BROWSER_CONTROL_LIMITS } from "./protocol.js";
3
3
  import { ComputerUseExecutorError, } from "../actors/computer-use/executor-error.js";
4
+ import { classifyExecutorFailure, } from "../actors/computer-use/executor-diagnostic.js";
4
5
  /** Owned byte channel only. This module never discovers endpoints or reconnects. */
5
6
  export class BrowserControlTransport {
6
7
  stream;
@@ -104,11 +105,15 @@ export class BrowserControlTransport {
104
105
  this.stream.off("error", this.error);
105
106
  return this.stream;
106
107
  }
107
- error = () => {
108
- this.close("transport_failed");
108
+ error = (error) => {
109
+ this.close("transport_failed", classifyExecutorFailure("channel", error));
109
110
  };
110
111
  end = () => {
111
- this.close(this.payload || this.headerUsed ? "invalid_response" : "transport_failed");
112
+ const partial = this.payload !== undefined || this.headerUsed > 0;
113
+ this.close(partial ? "invalid_response" : "transport_failed", {
114
+ step: "channel",
115
+ category: "channel_closed",
116
+ });
112
117
  };
113
118
  send(value, beforeWrite = () => { }) {
114
119
  if (this.ended || this.handedOff)
@@ -149,7 +154,7 @@ export class BrowserControlTransport {
149
154
  }
150
155
  this.stream.write(frame, (error) => {
151
156
  if (error) {
152
- this.close("transport_failed");
157
+ this.close("transport_failed", classifyExecutorFailure("channel_write", error));
153
158
  return;
154
159
  }
155
160
  done();
@@ -160,7 +165,7 @@ export class BrowserControlTransport {
160
165
  }
161
166
  });
162
167
  }
163
- close(code = "executor_closed") {
168
+ close(code = "executor_closed", diagnostic) {
164
169
  if (this.ended || this.handedOff)
165
170
  return;
166
171
  this.ended = true;
@@ -178,12 +183,12 @@ export class BrowserControlTransport {
178
183
  this.stream.off("close", releaseListeners);
179
184
  };
180
185
  this.stream.once("close", releaseListeners);
181
- this.pendingWrite?.(new ComputerUseExecutorError(code, "outcome_uncertain"));
186
+ this.pendingWrite?.(new ComputerUseExecutorError(code, "outcome_uncertain", { diagnostic }));
182
187
  this.stream.destroy();
183
188
  // A stream whose close event already happened will not emit it again. `closed`
184
189
  // can also precede queued error/close events, so defer removal past native ticks.
185
190
  if (this.stream.closed)
186
191
  setImmediate(releaseListeners);
187
- this.onClose(code);
192
+ this.onClose(code, diagnostic);
188
193
  }
189
194
  }
@@ -13,7 +13,7 @@ import { writeRunFindings } from "../findings.js";
13
13
  import { formatCuaStudyHuman } from "./study-format.js";
14
14
  import { resolveRouteShouldOpen } from "./study-route-open.js";
15
15
  import { studyResultIdentity } from "../../run/study-result.js";
16
- import { actorOf, declaredParticipantCount } from "../../study/study-fields.js";
16
+ import { declaredParticipantCount } from "../../study/study-fields.js";
17
17
  /**
18
18
  * The computer-use route's CLI setup: its settings, the watch and exposure plan, the runStudyWith options
19
19
  * with the live Observer hook, and how it presents the outcome or a run error. Undefined when
@@ -55,7 +55,7 @@ function refuseCua(args, dryRun, code, message) {
55
55
  ...studyResultIdentity("computer-use", args.config.id),
56
56
  ok: false,
57
57
  cwd: args.options.cwd,
58
- actor: actorOf(args.config)?.type ?? "",
58
+ actor: args.config.actor?.type ?? "",
59
59
  appUrl: "",
60
60
  dryRun,
61
61
  runId: args.options.runId ?? "not-created",
@@ -10,7 +10,7 @@ import { writeRunFindings } from "../findings.js";
10
10
  import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
11
11
  import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
12
12
  import { studyResultIdentity } from "../../run/study-result.js";
13
- import { actorOf, participantList } from "../../study/study-fields.js";
13
+ import { participantList } from "../../study/study-fields.js";
14
14
  /**
15
15
  * The shared-world route's CLI setup: dry run, open and follow semantics, the live Observer server
16
16
  * the run attaches, and how it presents the outcome. Undefined when setup has already written its
@@ -34,7 +34,7 @@ export function sharedWorldRouteRun(args) {
34
34
  ...studyResultIdentity("shared-world", args.config.id),
35
35
  ok: false,
36
36
  cwd: args.options.cwd,
37
- actor: actorOf(args.config)?.type ?? "",
37
+ actor: args.config.actor?.type ?? "",
38
38
  topology: "shared-world",
39
39
  topologyMode: "concurrent",
40
40
  roleCount: participantList(args.config)?.length ?? 0,
@@ -4,7 +4,6 @@ import { Command, Option } from "commander";
4
4
  import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
5
5
  import { runStudyPreflight, } from "../../study/preflight.js";
6
6
  import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, parsePositiveInteger, writeResult, humanError, } from "../io.js";
7
- import { actorOf } from "../../study/study-fields.js";
8
7
  export function registerStudyCommands(parent, io) {
9
8
  const study = parent
10
9
  .command("study")
@@ -115,7 +114,7 @@ function formatStudyShowHuman(result) {
115
114
  `id: ${config.id}`,
116
115
  `subject: ${config.subject.source}`,
117
116
  ...(config.execution?.target ? [`execution: ${config.execution.target}`] : []),
118
- `actor: ${actorOf(config)?.type ?? ""}`,
117
+ `actor: ${config.actor?.type ?? ""}`,
119
118
  ...(config.title ? [`title: ${config.title}`] : []),
120
119
  ...(config.description ? [`description: ${config.description}`] : []),
121
120
  ...(result.path ? [`path: ${result.path}`] : []),
@@ -336,6 +336,17 @@ function keyChecks(probes, receivingKey, setup, keyUsers) {
336
336
  const hint = probe.name === receivingKey
337
337
  ? `provide ${probe.name} through process env or --dotenv`
338
338
  : probe.hint;
339
+ if (probe.placeholder) {
340
+ // A participant in a terminal study's sandbox reads this row. It names the placeholder so the
341
+ // participant is not told it holds a key.
342
+ const required = setup?.keys.includes(probe.name) === true;
343
+ return {
344
+ name: `key ${probe.name}`,
345
+ ok: !required,
346
+ ...(required ? {} : { status: "note" }),
347
+ message: `not a key: ${probe.name} holds the placeholder humanish sets in a terminal study's sandbox, so no usable ${probe.name} is available here; ${hint}`,
348
+ };
349
+ }
339
350
  if (!setup) {
340
351
  const users = keyUsers?.get(probe.name) ?? [];
341
352
  const usedBy = users.length > 0 ? `used by ${studyList(users)}` : undefined;