humanish 0.111.0 → 0.111.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/CONTEXT.md +1 -1
- package/dist/actors/codex/restricted-participant-policy.d.ts +2 -0
- package/dist/actors/codex/restricted-participant-policy.js +30 -2
- package/dist/actors/codex/restricted-participant.js +1 -1
- package/dist/actors/computer-use/executor-diagnostic.d.ts +15 -0
- package/dist/actors/computer-use/executor-diagnostic.js +105 -0
- package/dist/actors/computer-use/executor-error.d.ts +18 -3
- package/dist/actors/computer-use/executor-error.js +30 -4
- package/dist/actors/computer-use/loop/actions.d.ts +6 -3
- package/dist/actors/computer-use/loop/actions.js +27 -7
- package/dist/actors/computer-use/loop/ending.js +9 -1
- package/dist/actors/computer-use/loop/types.d.ts +14 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/openai-context.d.ts +8 -2
- package/dist/actors/computer-use/openai-context.js +91 -26
- package/dist/actors/computer-use/openai-provider.js +55 -12
- package/dist/actors/computer-use/openai-requests.d.ts +2 -1
- package/dist/actors/computer-use/openai-requests.js +4 -1
- package/dist/actors/contract.d.ts +7 -1
- package/dist/browser-control/client.js +5 -5
- package/dist/browser-control/protocol.d.ts +30 -1
- package/dist/browser-control/protocol.js +19 -2
- package/dist/browser-control/transport.d.ts +3 -2
- package/dist/browser-control/transport.js +12 -7
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +2 -2
- package/dist/cli/commands/study.js +1 -2
- package/dist/cli/doctor.js +11 -0
- package/dist/evidence/encoded-text.d.ts +3 -3
- package/dist/evidence/encoded-text.js +10 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +18 -7
- package/dist/guest/browser-tools.js +5 -1
- package/dist/guest/chromium-text.js +42 -18
- package/dist/guest/desktop-executor.js +5 -1
- package/dist/keys/key-resolution.d.ts +3 -0
- package/dist/keys/key-resolution.js +15 -2
- package/dist/observer-app.html +6 -6
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -0
- package/dist/routes/computer-use/bundle-parts.js +7 -2
- package/dist/routes/computer-use/fanout-bundle.js +4 -2
- package/dist/routes/computer-use/local-vm.js +1 -2
- package/dist/routes/computer-use/participant-model.js +6 -1
- package/dist/routes/computer-use/plan.js +3 -4
- package/dist/routes/computer-use/route.js +1 -2
- package/dist/routes/computer-use/setup.js +2 -2
- package/dist/routes/preview.js +1 -0
- package/dist/routes/scripted/plan.js +4 -5
- package/dist/routes/scripted/route.js +2 -2
- package/dist/routes/shared-world/plan.js +4 -5
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/routes/shared-world/setup.js +1 -0
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +24 -5
- package/dist/routes/terminal/live-sandbox.js +5 -1
- package/dist/routes/terminal/plan.js +3 -4
- package/dist/routes/terminal/recorder.d.ts +6 -0
- package/dist/routes/terminal/recorder.js +61 -6
- package/dist/routes/terminal/route.js +1 -2
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/token-usage.d.ts +3 -0
- package/dist/routes/terminal/token-usage.js +5 -0
- package/dist/routes/terminal/types.d.ts +2 -1
- package/dist/routes/terminal/types.js +2 -1
- package/dist/run/dry-run.js +1 -0
- package/dist/run/results.d.ts +2 -0
- package/dist/run/run.d.ts +2 -0
- package/dist/run/run.js +16 -2
- package/dist/run/terminal-contract.d.ts +10 -0
- package/dist/run/terminal-contract.js +27 -9
- package/dist/study/composition-rules.js +17 -18
- package/dist/study/doctor.js +4 -5
- package/dist/study/migrate/v2.d.ts +5 -0
- package/dist/study/migrate/v2.js +23 -0
- package/dist/study/parse/front.js +2 -21
- package/dist/study/parse/url-credentials.d.ts +3 -3
- package/dist/study/parse/url-credentials.js +155 -13
- package/dist/study/persona-resolve.js +2 -2
- package/dist/study/plan-base.js +3 -4
- package/dist/study/plan-participants.js +3 -3
- package/dist/study/plan-types.d.ts +2 -0
- package/dist/study/plan.js +11 -5
- package/dist/study/routing.js +11 -11
- package/dist/study/run-study-options.js +2 -2
- package/dist/study/study-fields.d.ts +1 -13
- package/dist/study/study-fields.js +0 -24
- package/dist/study/summary.js +7 -7
- package/dist/study/types.d.ts +1 -0
- package/dist/study/validation.js +11 -11
- package/dist/study/warnings.d.ts +8 -0
- package/dist/study/warnings.js +22 -7
- package/dist/substrates/local/runtime-config.js +6 -6
- package/docs/architecture/browser-control.md +12 -2
- package/docs/architecture/guest-desktop.md +7 -1
- package/docs/architecture/terminal-product-route.md +3 -2
- package/docs/contracts/schemas.md +16 -5
- package/package.json +1 -1
|
@@ -7,20 +7,46 @@ import { asArray, asRecord, inputMessage, openAiActionToCua } from "./openai-wir
|
|
|
7
7
|
// The Responses API pairs items: every computer_call needs its computer_call_output, and a
|
|
8
8
|
// reasoning item travels with the item it preceded. The conversation is therefore cut only at
|
|
9
9
|
// exchange boundaries: one reply's output items together with the items that answered them.
|
|
10
|
-
// When the estimate passes the budget, the
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
// trace's per-request estimate shows when
|
|
16
|
-
|
|
10
|
+
// When the estimate passes the budget, the conversation is cut down to half the budget in one
|
|
11
|
+
// step: the opening screenshot goes first, then the oldest exchanges become lines of a progress
|
|
12
|
+
// note that keeps their reasoning summaries, messages and actions, and the text that answered
|
|
13
|
+
// them, such as a note that an action was not run. The note is an assistant message, so text the
|
|
14
|
+
// model wrote (which can quote a web page) keeps the trust it had when the model wrote it. The newest MIN_KEPT_EXCHANGES exchanges are never collapsed, so a
|
|
15
|
+
// request passes the budget when they alone exceed it; the trace's per-request estimate shows when
|
|
16
|
+
// that happens.
|
|
17
|
+
//
|
|
18
|
+
// A cut rewrites the start of the prompt, so the provider's prompt cache misses on the request
|
|
19
|
+
// after it. Between cuts nothing already sent changes: each request starts with the whole of the
|
|
20
|
+
// previous request, which the cache can serve. A cut of one exchange per turn would rewrite
|
|
21
|
+
// the note on every request past the budget, and each of those requests would be billed in full.
|
|
22
|
+
/** The estimated input a carried conversation may reach before it is cut. */
|
|
17
23
|
export const CONTEXT_TOKEN_BUDGET = 64_000;
|
|
24
|
+
/**
|
|
25
|
+
* The share of the budget a cut brings the carried conversation down to. The rest is the growth
|
|
26
|
+
* the cache serves until the next cut: about 19 turns of 1280x800 screenshots, or 12 at
|
|
27
|
+
* 1920x1080.
|
|
28
|
+
*/
|
|
29
|
+
const CUT_TARGET_SHARE = 0.5;
|
|
18
30
|
/** Exchanges always carried whole, so the model sees its latest actions with their screens. */
|
|
19
31
|
const MIN_KEPT_EXCHANGES = 2;
|
|
20
|
-
/**
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
32
|
+
/**
|
|
33
|
+
* The progress note's length cap, about 8,000 tokens. Past it, the oldest lines after the first
|
|
34
|
+
* NOTE_HEAD_LINES are counted instead of shown. The note sits in the cached prefix between cuts,
|
|
35
|
+
* so its size costs little after the request that writes it.
|
|
36
|
+
*/
|
|
37
|
+
const NOTE_CHAR_LIMIT = 32_000;
|
|
38
|
+
/**
|
|
39
|
+
* Note lines kept whatever the note's length. A session's first turns are where it usually learns
|
|
40
|
+
* what it must carry to the end, such as an account, an address or a code, and the newest lines
|
|
41
|
+
* hold its latest progress, so the note gives up the turns between them.
|
|
42
|
+
*/
|
|
43
|
+
const NOTE_HEAD_LINES = 4;
|
|
44
|
+
/**
|
|
45
|
+
* One note line's cap, so a reply with many actions cannot push the note past its own cap. It
|
|
46
|
+
* leaves room for the clipped thought, message and hint text plus about 1,000 characters of
|
|
47
|
+
* actions, such as several typed form fields.
|
|
48
|
+
*/
|
|
49
|
+
const LINE_CHAR_LIMIT = 2_000;
|
|
24
50
|
/**
|
|
25
51
|
* Image input tokens per 32-pixel patch. The gpt-5.x and gpt-6 models bill 1.2 per patch, and at
|
|
26
52
|
* the default (auto) detail gpt-5.6 and gpt-6 keep the screenshot's own size. Models that resize
|
|
@@ -36,7 +62,11 @@ export class CarriedConversation {
|
|
|
36
62
|
/** The latest accepted reply's output items, which the next request answers. */
|
|
37
63
|
pending;
|
|
38
64
|
notes = [];
|
|
39
|
-
|
|
65
|
+
/**
|
|
66
|
+
* The turns whose note lines were dropped to keep the note under its cap. Lines are dropped in
|
|
67
|
+
* turn order from just after the head, so they are one run of consecutive turns.
|
|
68
|
+
*/
|
|
69
|
+
elided;
|
|
40
70
|
collapsedTurns = 0;
|
|
41
71
|
constructor(budget = CONTEXT_TOKEN_BUDGET) {
|
|
42
72
|
this.budget = budget;
|
|
@@ -76,7 +106,10 @@ export class CarriedConversation {
|
|
|
76
106
|
};
|
|
77
107
|
}
|
|
78
108
|
trim() {
|
|
79
|
-
|
|
109
|
+
if (estimateTokens(this.carried()) <= this.budget)
|
|
110
|
+
return;
|
|
111
|
+
const target = Math.floor(this.budget * CUT_TARGET_SHARE);
|
|
112
|
+
while (estimateTokens(this.carried()) > target) {
|
|
80
113
|
if (this.opening !== undefined && imagesOf(this.opening).length > 0) {
|
|
81
114
|
this.opening = withoutImages(this.opening);
|
|
82
115
|
continue;
|
|
@@ -84,30 +117,56 @@ export class CarriedConversation {
|
|
|
84
117
|
if (this.exchanges.length <= MIN_KEPT_EXCHANGES)
|
|
85
118
|
return;
|
|
86
119
|
const oldest = this.exchanges.shift();
|
|
87
|
-
this.notes.push(describeExchange(oldest));
|
|
120
|
+
this.notes.push({ turn: oldest.turn, text: describeExchange(oldest) });
|
|
88
121
|
this.collapsedTurns += 1;
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
122
|
+
this.capNote();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
capNote() {
|
|
126
|
+
while (this.noteText().length > NOTE_CHAR_LIMIT && this.notes.length > NOTE_HEAD_LINES + 1) {
|
|
127
|
+
const [dropped] = this.notes.splice(NOTE_HEAD_LINES, 1);
|
|
128
|
+
this.elided = { first: this.elided?.first ?? dropped.turn, last: dropped.turn };
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
noteText() {
|
|
132
|
+
const lines = this.notes.map((line) => line.text);
|
|
133
|
+
if (this.elided !== undefined) {
|
|
134
|
+
const { first, last } = this.elided;
|
|
135
|
+
const which = first === last ? `Turn ${first} is` : `Turns ${first} to ${last} are`;
|
|
136
|
+
lines.splice(NOTE_HEAD_LINES, 0, `(${which} not listed.)`);
|
|
93
137
|
}
|
|
138
|
+
return lines.join("\n");
|
|
94
139
|
}
|
|
95
140
|
noteItem() {
|
|
96
141
|
if (this.notes.length === 0)
|
|
97
142
|
return undefined;
|
|
98
143
|
const lead = `My notes on my earlier turns in this session, oldest first. Their screenshots are no ` +
|
|
99
|
-
`longer shown; this is what I thought, said and did
|
|
100
|
-
(this.notesDropped === 0 ? "" : ` (${this.notesDropped} earlier turns are not listed.)`);
|
|
144
|
+
`longer shown; this is what I thought, said and did, and what I was told after.`;
|
|
101
145
|
return inputMessage("assistant", [
|
|
102
|
-
{ type: "output_text", text: `${lead}\n${this.
|
|
146
|
+
{ type: "output_text", text: `${lead}\n${this.noteText()}` },
|
|
103
147
|
]);
|
|
104
148
|
}
|
|
105
149
|
}
|
|
106
|
-
/**
|
|
150
|
+
/**
|
|
151
|
+
* One exchange as a progress-note line: its reasoning summary, message and actions, and the text
|
|
152
|
+
* that came back with their screenshot, such as a note that an action was not run.
|
|
153
|
+
*/
|
|
107
154
|
function describeExchange(exchange) {
|
|
108
155
|
const thought = [];
|
|
109
156
|
const said = [];
|
|
110
157
|
const did = [];
|
|
158
|
+
const told = [];
|
|
159
|
+
// The answers are the call outputs, whose content is a screenshot, and hint messages.
|
|
160
|
+
for (const raw of exchange.answers) {
|
|
161
|
+
const item = asRecord(raw);
|
|
162
|
+
if (item.type === "computer_call_output")
|
|
163
|
+
continue;
|
|
164
|
+
for (const entry of asArray(item.content)) {
|
|
165
|
+
const text = asRecord(entry).text;
|
|
166
|
+
if (typeof text === "string" && text.length > 0)
|
|
167
|
+
told.push(text);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
111
170
|
for (const raw of exchange.output) {
|
|
112
171
|
const item = asRecord(raw);
|
|
113
172
|
if (item.type === "reasoning")
|
|
@@ -131,13 +190,19 @@ function describeExchange(exchange) {
|
|
|
131
190
|
}
|
|
132
191
|
}
|
|
133
192
|
}
|
|
134
|
-
const
|
|
193
|
+
const lead = `Turn ${exchange.turn}: `;
|
|
194
|
+
const before = [
|
|
135
195
|
thought.length > 0 ? `thought: ${clip(thought.join(" "), 400)}` : undefined,
|
|
136
196
|
said.length > 0 ? `said: ${clip(said.join(" "), 300)}` : undefined,
|
|
137
|
-
did.length > 0 ? `did: ${did.join(", ")}` : undefined,
|
|
138
197
|
].filter((part) => part !== undefined);
|
|
139
|
-
const
|
|
140
|
-
|
|
198
|
+
const after = told.length > 0 ? [`was told: ${clip(told.join(" "), 300)}`] : [];
|
|
199
|
+
// The hint can say an action did not run, so the actions give up whatever room it needs.
|
|
200
|
+
const fixed = [...before, ...after].reduce((length, part) => length + part.length + 2, 0);
|
|
201
|
+
const room = LINE_CHAR_LIMIT - 1 - lead.length - fixed - "did: ".length;
|
|
202
|
+
const actions = did.length > 0 ? [`did: ${clip(did.join(", "), room)}`] : [];
|
|
203
|
+
const parts = [...before, ...actions, ...after];
|
|
204
|
+
const line = clip(`${lead}${parts.length > 0 ? parts.join("; ") : "no recorded output"}`, LINE_CHAR_LIMIT - 1);
|
|
205
|
+
return /[.!?]$/.test(line) ? line : `${line}.`;
|
|
141
206
|
}
|
|
142
207
|
function describeAction(action) {
|
|
143
208
|
switch (action.kind) {
|
|
@@ -175,17 +175,51 @@ export function namedProviderErrorCode(bodyText) {
|
|
|
175
175
|
// by switching to explicit-context mode) from any other non-ok status. It never
|
|
176
176
|
// carries the apiKey or the response body.
|
|
177
177
|
class ZdrError extends Error {
|
|
178
|
-
|
|
179
|
-
|
|
178
|
+
rejection;
|
|
179
|
+
constructor(rejection) {
|
|
180
|
+
super(rejection === "stored_item"
|
|
181
|
+
? "OpenAI Responses could not find an item the request referenced: the server keeps none for this organization (zero data retention)"
|
|
182
|
+
: "OpenAI Responses rejected server-side state (zero data retention)");
|
|
183
|
+
this.rejection = rejection;
|
|
180
184
|
this.name = "ZdrError";
|
|
181
185
|
}
|
|
182
186
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
187
|
+
/** The message of a 404 for an item the server never kept, as the wire sends it. */
|
|
188
|
+
const STORED_ITEM_NOT_FOUND = /Item with id '?[A-Za-z0-9_-]+'? not found/;
|
|
189
|
+
/**
|
|
190
|
+
* Error codes that name a failure of their own. Their messages can echo request text, such as a
|
|
191
|
+
* model name or the prompt, so the words below must not turn them into a retention answer.
|
|
192
|
+
*/
|
|
193
|
+
const OTHER_FAILURE_CODES = new Set([
|
|
194
|
+
"invalid_prompt",
|
|
195
|
+
"model_not_found",
|
|
196
|
+
"context_length_exceeded",
|
|
197
|
+
"rate_limit_exceeded",
|
|
198
|
+
"insufficient_quota",
|
|
199
|
+
"invalid_image",
|
|
200
|
+
"invalid_image_format",
|
|
201
|
+
"invalid_base64_image",
|
|
202
|
+
"invalid_image_url",
|
|
203
|
+
"image_too_large",
|
|
204
|
+
"image_parse_error",
|
|
205
|
+
"image_content_policy_violation",
|
|
206
|
+
]);
|
|
207
|
+
// A 400 or 404 whose body says the organization cannot use server-side response state, so the
|
|
208
|
+
// provider must carry the conversation itself (explicit-context mode). A 404 counts only with the
|
|
209
|
+
// stored-item message. Captured bodies are in tests/fixtures/openai-store-less/.
|
|
210
|
+
function zdrRejection(status, bodyText) {
|
|
211
|
+
const code = namedProviderErrorCode(bodyText);
|
|
212
|
+
if (code !== undefined && OTHER_FAILURE_CODES.has(code))
|
|
213
|
+
return undefined;
|
|
214
|
+
if (status === 404)
|
|
215
|
+
return STORED_ITEM_NOT_FOUND.test(bodyText) ? "stored_item" : undefined;
|
|
216
|
+
if (/zero[ -]data[ -]retention/i.test(bodyText))
|
|
217
|
+
return "zero_data_retention";
|
|
218
|
+
if (bodyText.includes("previous_response_id"))
|
|
219
|
+
return "previous_response";
|
|
220
|
+
if (STORED_ITEM_NOT_FOUND.test(bodyText))
|
|
221
|
+
return "stored_item";
|
|
222
|
+
return undefined;
|
|
189
223
|
}
|
|
190
224
|
// A typed error so nextTurn can latch reasoning summaries off and retry the turn
|
|
191
225
|
// (an org not verified for summaries, or a model without a summarizer, 400s the
|
|
@@ -307,11 +341,20 @@ async function postResponse(transport, body, signal, retries, interaction, spend
|
|
|
307
341
|
return parsed;
|
|
308
342
|
}
|
|
309
343
|
lastStatus = res.status;
|
|
344
|
+
if (res.status === 404) {
|
|
345
|
+
// A store-less organization answers a reference to an item it never kept with 404.
|
|
346
|
+
const bodyText = await res.text().catch(() => "");
|
|
347
|
+
const rejection = zdrRejection(404, bodyText);
|
|
348
|
+
if (rejection !== undefined)
|
|
349
|
+
throw new ZdrError(rejection);
|
|
350
|
+
const code = namedProviderErrorCode(bodyText);
|
|
351
|
+
throw new Error(`OpenAI Responses 404${code === undefined ? "" : ` ${code}`}`);
|
|
352
|
+
}
|
|
310
353
|
if (res.status === 400) {
|
|
311
354
|
const bodyText = await res.text();
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
355
|
+
const rejection = zdrRejection(400, bodyText);
|
|
356
|
+
if (rejection !== undefined)
|
|
357
|
+
throw new ZdrError(rejection);
|
|
315
358
|
if (isSummaryRejection(bodyText)) {
|
|
316
359
|
throw new SummaryRejectionError();
|
|
317
360
|
}
|
|
@@ -415,7 +458,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
415
458
|
continue;
|
|
416
459
|
}
|
|
417
460
|
if (error instanceof ZdrError && state.mode !== "explicit_context") {
|
|
418
|
-
record.switched();
|
|
461
|
+
record.switched(error.rejection);
|
|
419
462
|
state.mode = "explicit_context";
|
|
420
463
|
continue;
|
|
421
464
|
}
|
|
@@ -48,13 +48,14 @@ export declare function acceptReply(state: OpenAiConversationState, parsed: Pars
|
|
|
48
48
|
export declare class ConversationRecord {
|
|
49
49
|
private readonly now;
|
|
50
50
|
private explicitReason;
|
|
51
|
+
private rejection;
|
|
51
52
|
private switchedAt;
|
|
52
53
|
private switchedAtRequest;
|
|
53
54
|
private summarizedTurns;
|
|
54
55
|
private readonly requests;
|
|
55
56
|
constructor(configured: boolean, now: () => number);
|
|
56
57
|
/** The organization rejected server-side state, so the request being sent is explicit_context. */
|
|
57
|
-
switched(): void;
|
|
58
|
+
switched(rejection: NonNullable<ActorConversation["rejection"]>): void;
|
|
58
59
|
/** A participant request just went out with `sent` after the carried conversation. */
|
|
59
60
|
requested(state: OpenAiConversationState, sent: readonly unknown[]): void;
|
|
60
61
|
snapshot(state: OpenAiConversationState): ActorConversation;
|
|
@@ -94,6 +94,7 @@ export function acceptReply(state, parsed, closing, sent) {
|
|
|
94
94
|
export class ConversationRecord {
|
|
95
95
|
now;
|
|
96
96
|
explicitReason;
|
|
97
|
+
rejection;
|
|
97
98
|
switchedAt;
|
|
98
99
|
switchedAtRequest;
|
|
99
100
|
summarizedTurns = 0;
|
|
@@ -104,8 +105,9 @@ export class ConversationRecord {
|
|
|
104
105
|
this.explicitReason = "configured";
|
|
105
106
|
}
|
|
106
107
|
/** The organization rejected server-side state, so the request being sent is explicit_context. */
|
|
107
|
-
switched() {
|
|
108
|
+
switched(rejection) {
|
|
108
109
|
this.explicitReason = "zdr_rejection";
|
|
110
|
+
this.rejection = rejection;
|
|
109
111
|
this.switchedAt = new Date(this.now()).toISOString();
|
|
110
112
|
// Counted like requests[], which includes replies set aside at their output limit.
|
|
111
113
|
this.switchedAtRequest = this.requests.length + 1;
|
|
@@ -130,6 +132,7 @@ export class ConversationRecord {
|
|
|
130
132
|
return {
|
|
131
133
|
mode: state.mode === "explicit_context" ? "explicit_context" : "threaded",
|
|
132
134
|
...(this.explicitReason === undefined ? {} : { explicitReason: this.explicitReason }),
|
|
135
|
+
...(this.rejection === undefined ? {} : { rejection: this.rejection }),
|
|
133
136
|
...(this.switchedAt === undefined ? {} : { switchedAt: this.switchedAt }),
|
|
134
137
|
...(this.switchedAtRequest === undefined
|
|
135
138
|
? {}
|
|
@@ -206,13 +206,19 @@ export interface ActorRuntimeProvenance {
|
|
|
206
206
|
* How a provider carried the conversation between its requests. `threaded`: the provider's server
|
|
207
207
|
* kept it (OpenAI previous_response_id). `explicit_context`: the server kept none (a
|
|
208
208
|
* zero-data-retention org, or zeroDataRetention set), so each request carried it from the client
|
|
209
|
-
* within a token budget, with the oldest turns summarized as text
|
|
209
|
+
* within a token budget, with the oldest turns summarized as text at each cut past the budget.
|
|
210
210
|
*/
|
|
211
211
|
export interface ActorConversation {
|
|
212
212
|
/** The mode of the session's last request. */
|
|
213
213
|
mode: "threaded" | "explicit_context";
|
|
214
214
|
/** Why the session used explicit_context: configured, or the org rejected server-side state. */
|
|
215
215
|
explicitReason?: "configured" | "zdr_rejection";
|
|
216
|
+
/**
|
|
217
|
+
* With `zdr_rejection`, the answer that made the switch: the error named zero data retention, it
|
|
218
|
+
* refused or could not find the previous response, or it could not find a stored item that the
|
|
219
|
+
* request referenced.
|
|
220
|
+
*/
|
|
221
|
+
rejection?: "zero_data_retention" | "previous_response" | "stored_item";
|
|
216
222
|
/** When a threaded session switched to explicit_context (ISO-8601), and on which request. */
|
|
217
223
|
switchedAt?: string;
|
|
218
224
|
switchedAtRequest?: number;
|
|
@@ -23,7 +23,7 @@ export function createBrowserControlClient(options) {
|
|
|
23
23
|
// The transport closes synchronously when its stream is already unusable, so its close handler
|
|
24
24
|
// takes the state that exists before the transport does.
|
|
25
25
|
const session = Object.assign(state, {
|
|
26
|
-
transport: new BrowserControlTransport(options.transport, (value) => acceptReply(session, value), (code) => closeClient(state, code)),
|
|
26
|
+
transport: new BrowserControlTransport(options.transport, (value) => acceptReply(session, value), (code, diagnostic) => closeClient(state, code, diagnostic)),
|
|
27
27
|
});
|
|
28
28
|
const executor = {
|
|
29
29
|
stallRecovery: "fail_closed",
|
|
@@ -41,13 +41,13 @@ export function createBrowserControlClient(options) {
|
|
|
41
41
|
},
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
|
-
function closeClient(state, code) {
|
|
44
|
+
function closeClient(state, code, diagnostic) {
|
|
45
45
|
state.closed = true;
|
|
46
46
|
const operation = state.pending;
|
|
47
47
|
state.pending = undefined;
|
|
48
48
|
if (operation) {
|
|
49
49
|
operation.dispose();
|
|
50
|
-
operation.reject(new ComputerUseExecutorError(code, operation.written ? "outcome_uncertain" : "not_dispatched"));
|
|
50
|
+
operation.reject(new ComputerUseExecutorError(code, operation.written ? "outcome_uncertain" : "not_dispatched", { diagnostic }));
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
/** Settles the pending exchange once both its reply has arrived and its request write finished. */
|
|
@@ -63,12 +63,12 @@ function finishExchange(session) {
|
|
|
63
63
|
else if (operation.reply.ok)
|
|
64
64
|
operation.resolve(operation.reply);
|
|
65
65
|
else {
|
|
66
|
-
const { code, disposition } = operation.reply.error;
|
|
66
|
+
const { code, disposition, reason, diagnostic } = operation.reply.error;
|
|
67
67
|
if (operation.request.operation !== "EXECUTE" ||
|
|
68
68
|
code !== "action_rejected" ||
|
|
69
69
|
disposition !== "not_dispatched")
|
|
70
70
|
session.transport.close(code);
|
|
71
|
-
operation.reject(new ComputerUseExecutorError(code, disposition));
|
|
71
|
+
operation.reject(new ComputerUseExecutorError(code, disposition, { reason, diagnostic }));
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
/** Matches a reply frame to the pending request; any mismatch closes the channel. */
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import type { CuaAction, CuaObservation } from "../actors/computer-use/loop.js";
|
|
3
3
|
import { type HeardSpeech } from "../actors/computer-use/speech.js";
|
|
4
|
-
import { type CuaExecutorErrorCode } from "../actors/computer-use/executor-error.js";
|
|
4
|
+
import { type CuaExecutorErrorCode, type CuaRejectionReason } from "../actors/computer-use/executor-error.js";
|
|
5
|
+
import { type CuaExecutorDiagnostic } from "../actors/computer-use/executor-diagnostic.js";
|
|
5
6
|
export declare const BROWSER_CONTROL_VERSION = 1;
|
|
6
7
|
export declare const BROWSER_CONTROL_LIMITS: Readonly<{
|
|
7
8
|
frameBytes: number;
|
|
@@ -471,6 +472,32 @@ declare const replySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
471
472
|
not_dispatched: "not_dispatched";
|
|
472
473
|
outcome_uncertain: "outcome_uncertain";
|
|
473
474
|
}>;
|
|
475
|
+
reason: z.ZodOptional<z.ZodEnum<{
|
|
476
|
+
extra_tab: "extra_tab";
|
|
477
|
+
}>>;
|
|
478
|
+
diagnostic: z.ZodOptional<z.ZodObject<{
|
|
479
|
+
step: z.ZodEnum<{
|
|
480
|
+
cdp_detach: "cdp_detach";
|
|
481
|
+
cdp_session: "cdp_session";
|
|
482
|
+
channel: "channel";
|
|
483
|
+
channel_write: "channel_write";
|
|
484
|
+
focus_probe: "focus_probe";
|
|
485
|
+
frame_tree: "frame_tree";
|
|
486
|
+
insert_text: "insert_text";
|
|
487
|
+
isolated_world: "isolated_world";
|
|
488
|
+
owner_window_check: "owner_window_check";
|
|
489
|
+
text_port: "text_port";
|
|
490
|
+
}>;
|
|
491
|
+
category: z.ZodEnum<{
|
|
492
|
+
channel_closed: "channel_closed";
|
|
493
|
+
detached: "detached";
|
|
494
|
+
navigation: "navigation";
|
|
495
|
+
protocol_error: "protocol_error";
|
|
496
|
+
target_closed: "target_closed";
|
|
497
|
+
timeout: "timeout";
|
|
498
|
+
unknown: "unknown";
|
|
499
|
+
}>;
|
|
500
|
+
}, z.core.$strict>>;
|
|
474
501
|
}, z.core.$strict>;
|
|
475
502
|
}, z.core.$strict>], "ok">;
|
|
476
503
|
export type BrowserControlRequest = z.infer<typeof requestSchema>;
|
|
@@ -488,5 +515,7 @@ export declare function decodeBrowserControlObservation(value: unknown): CuaObse
|
|
|
488
515
|
export declare function safeBrowserControlFailure(error: unknown, dispatched: boolean): {
|
|
489
516
|
code: CuaExecutorErrorCode;
|
|
490
517
|
disposition: "not_dispatched" | "outcome_uncertain";
|
|
518
|
+
reason?: CuaRejectionReason;
|
|
519
|
+
diagnostic?: CuaExecutorDiagnostic;
|
|
491
520
|
};
|
|
492
521
|
export {};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { PNG } from "pngjs";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { CUA_SPEECH_LIMITS } from "../actors/computer-use/speech.js";
|
|
4
|
-
import { ComputerUseExecutorError, isComputerUseExecutorError, } from "../actors/computer-use/executor-error.js";
|
|
4
|
+
import { ComputerUseExecutorError, CUA_REJECTION_REASONS, isComputerUseExecutorError, } from "../actors/computer-use/executor-error.js";
|
|
5
|
+
import { CUA_DIAGNOSTIC_CATEGORIES, CUA_DIAGNOSTIC_STEPS, } from "../actors/computer-use/executor-diagnostic.js";
|
|
5
6
|
import { desktopRecordingMetadataSchema } from "../evidence/desktop-recording-types.js";
|
|
6
7
|
export const BROWSER_CONTROL_VERSION = 1;
|
|
7
8
|
export const BROWSER_CONTROL_LIMITS = Object.freeze({
|
|
@@ -194,6 +195,11 @@ const errorCode = z.enum([
|
|
|
194
195
|
"action_rejected",
|
|
195
196
|
"execution_failed",
|
|
196
197
|
]);
|
|
198
|
+
// Two fixed words; a reply never carries browser error text.
|
|
199
|
+
const diagnosticSchema = z.strictObject({
|
|
200
|
+
step: z.enum(CUA_DIAGNOSTIC_STEPS),
|
|
201
|
+
category: z.enum(CUA_DIAGNOSTIC_CATEGORIES),
|
|
202
|
+
});
|
|
197
203
|
const replyBase = {
|
|
198
204
|
...common,
|
|
199
205
|
type: z.literal("reply"),
|
|
@@ -213,6 +219,8 @@ const replySchema = z.discriminatedUnion("ok", [
|
|
|
213
219
|
error: z.strictObject({
|
|
214
220
|
code: errorCode,
|
|
215
221
|
disposition: z.enum(["not_dispatched", "outcome_uncertain"]),
|
|
222
|
+
reason: z.enum(CUA_REJECTION_REASONS).optional(),
|
|
223
|
+
diagnostic: diagnosticSchema.optional(),
|
|
216
224
|
}),
|
|
217
225
|
}),
|
|
218
226
|
]);
|
|
@@ -243,6 +251,9 @@ export function parseBrowserControlReply(value) {
|
|
|
243
251
|
throw new ComputerUseExecutorError("invalid_response", "outcome_uncertain");
|
|
244
252
|
const reply = parsed.data;
|
|
245
253
|
if ((reply.operation === "EXECUTE") !== (reply.actionId !== undefined) ||
|
|
254
|
+
(!reply.ok &&
|
|
255
|
+
reply.error.reason !== undefined &&
|
|
256
|
+
(reply.error.code !== "action_rejected" || reply.error.disposition !== "not_dispatched")) ||
|
|
246
257
|
(reply.ok &&
|
|
247
258
|
((reply.operation === "OBSERVE") !== (reply.observation !== undefined) ||
|
|
248
259
|
(reply.operation === "FINISH_RECORDING") !== (reply.recording !== undefined))))
|
|
@@ -351,8 +362,14 @@ export function decodeBrowserControlObservation(value) {
|
|
|
351
362
|
};
|
|
352
363
|
}
|
|
353
364
|
export function safeBrowserControlFailure(error, dispatched) {
|
|
365
|
+
// Only a declared error's fixed diagnostic crosses; an arbitrary exception's text never does.
|
|
354
366
|
if (isComputerUseExecutorError(error))
|
|
355
|
-
return {
|
|
367
|
+
return {
|
|
368
|
+
code: error.code,
|
|
369
|
+
disposition: error.disposition,
|
|
370
|
+
...(error.reason === undefined ? {} : { reason: error.reason }),
|
|
371
|
+
...(error.diagnostic === undefined ? {} : { diagnostic: { ...error.diagnostic } }),
|
|
372
|
+
};
|
|
356
373
|
return {
|
|
357
374
|
code: "action_rejected",
|
|
358
375
|
disposition: dispatched ? "outcome_uncertain" : "not_dispatched",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Duplex } from "node:stream";
|
|
2
2
|
import { type CuaExecutorErrorCode } from "../actors/computer-use/executor-error.js";
|
|
3
|
+
import { type CuaExecutorDiagnostic } from "../actors/computer-use/executor-diagnostic.js";
|
|
3
4
|
/** Owned byte channel only. This module never discovers endpoints or reconnects. */
|
|
4
5
|
export declare class BrowserControlTransport {
|
|
5
6
|
private readonly stream;
|
|
@@ -15,7 +16,7 @@ export declare class BrowserControlTransport {
|
|
|
15
16
|
private pendingWrite;
|
|
16
17
|
private frameTimer;
|
|
17
18
|
private readonly decoder;
|
|
18
|
-
constructor(stream: Duplex, onFrame: (value: unknown) => void, onClose: (code: CuaExecutorErrorCode) => void);
|
|
19
|
+
constructor(stream: Duplex, onFrame: (value: unknown) => void, onClose: (code: CuaExecutorErrorCode, diagnostic: CuaExecutorDiagnostic | undefined) => void);
|
|
19
20
|
get closed(): boolean;
|
|
20
21
|
private data;
|
|
21
22
|
/** Relinquish the live byte stream synchronously after a complete frame. */
|
|
@@ -23,5 +24,5 @@ export declare class BrowserControlTransport {
|
|
|
23
24
|
private error;
|
|
24
25
|
private end;
|
|
25
26
|
send(value: unknown, beforeWrite?: () => void): Promise<void>;
|
|
26
|
-
close(code?: CuaExecutorErrorCode): void;
|
|
27
|
+
close(code?: CuaExecutorErrorCode, diagnostic?: CuaExecutorDiagnostic): void;
|
|
27
28
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { TextDecoder } from "node:util";
|
|
2
2
|
import { BROWSER_CONTROL_LIMITS } from "./protocol.js";
|
|
3
3
|
import { ComputerUseExecutorError, } from "../actors/computer-use/executor-error.js";
|
|
4
|
+
import { classifyExecutorFailure, } from "../actors/computer-use/executor-diagnostic.js";
|
|
4
5
|
/** Owned byte channel only. This module never discovers endpoints or reconnects. */
|
|
5
6
|
export class BrowserControlTransport {
|
|
6
7
|
stream;
|
|
@@ -104,11 +105,15 @@ export class BrowserControlTransport {
|
|
|
104
105
|
this.stream.off("error", this.error);
|
|
105
106
|
return this.stream;
|
|
106
107
|
}
|
|
107
|
-
error = () => {
|
|
108
|
-
this.close("transport_failed");
|
|
108
|
+
error = (error) => {
|
|
109
|
+
this.close("transport_failed", classifyExecutorFailure("channel", error));
|
|
109
110
|
};
|
|
110
111
|
end = () => {
|
|
111
|
-
this.
|
|
112
|
+
const partial = this.payload !== undefined || this.headerUsed > 0;
|
|
113
|
+
this.close(partial ? "invalid_response" : "transport_failed", {
|
|
114
|
+
step: "channel",
|
|
115
|
+
category: "channel_closed",
|
|
116
|
+
});
|
|
112
117
|
};
|
|
113
118
|
send(value, beforeWrite = () => { }) {
|
|
114
119
|
if (this.ended || this.handedOff)
|
|
@@ -149,7 +154,7 @@ export class BrowserControlTransport {
|
|
|
149
154
|
}
|
|
150
155
|
this.stream.write(frame, (error) => {
|
|
151
156
|
if (error) {
|
|
152
|
-
this.close("transport_failed");
|
|
157
|
+
this.close("transport_failed", classifyExecutorFailure("channel_write", error));
|
|
153
158
|
return;
|
|
154
159
|
}
|
|
155
160
|
done();
|
|
@@ -160,7 +165,7 @@ export class BrowserControlTransport {
|
|
|
160
165
|
}
|
|
161
166
|
});
|
|
162
167
|
}
|
|
163
|
-
close(code = "executor_closed") {
|
|
168
|
+
close(code = "executor_closed", diagnostic) {
|
|
164
169
|
if (this.ended || this.handedOff)
|
|
165
170
|
return;
|
|
166
171
|
this.ended = true;
|
|
@@ -178,12 +183,12 @@ export class BrowserControlTransport {
|
|
|
178
183
|
this.stream.off("close", releaseListeners);
|
|
179
184
|
};
|
|
180
185
|
this.stream.once("close", releaseListeners);
|
|
181
|
-
this.pendingWrite?.(new ComputerUseExecutorError(code, "outcome_uncertain"));
|
|
186
|
+
this.pendingWrite?.(new ComputerUseExecutorError(code, "outcome_uncertain", { diagnostic }));
|
|
182
187
|
this.stream.destroy();
|
|
183
188
|
// A stream whose close event already happened will not emit it again. `closed`
|
|
184
189
|
// can also precede queued error/close events, so defer removal past native ticks.
|
|
185
190
|
if (this.stream.closed)
|
|
186
191
|
setImmediate(releaseListeners);
|
|
187
|
-
this.onClose(code);
|
|
192
|
+
this.onClose(code, diagnostic);
|
|
188
193
|
}
|
|
189
194
|
}
|
|
@@ -13,7 +13,7 @@ import { writeRunFindings } from "../findings.js";
|
|
|
13
13
|
import { formatCuaStudyHuman } from "./study-format.js";
|
|
14
14
|
import { resolveRouteShouldOpen } from "./study-route-open.js";
|
|
15
15
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
16
|
-
import {
|
|
16
|
+
import { declaredParticipantCount } from "../../study/study-fields.js";
|
|
17
17
|
/**
|
|
18
18
|
* The computer-use route's CLI setup: its settings, the watch and exposure plan, the runStudyWith options
|
|
19
19
|
* with the live Observer hook, and how it presents the outcome or a run error. Undefined when
|
|
@@ -55,7 +55,7 @@ function refuseCua(args, dryRun, code, message) {
|
|
|
55
55
|
...studyResultIdentity("computer-use", args.config.id),
|
|
56
56
|
ok: false,
|
|
57
57
|
cwd: args.options.cwd,
|
|
58
|
-
actor:
|
|
58
|
+
actor: args.config.actor?.type ?? "",
|
|
59
59
|
appUrl: "",
|
|
60
60
|
dryRun,
|
|
61
61
|
runId: args.options.runId ?? "not-created",
|
|
@@ -10,7 +10,7 @@ import { writeRunFindings } from "../findings.js";
|
|
|
10
10
|
import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
|
|
11
11
|
import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
12
12
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
13
|
-
import {
|
|
13
|
+
import { participantList } from "../../study/study-fields.js";
|
|
14
14
|
/**
|
|
15
15
|
* The shared-world route's CLI setup: dry run, open and follow semantics, the live Observer server
|
|
16
16
|
* the run attaches, and how it presents the outcome. Undefined when setup has already written its
|
|
@@ -34,7 +34,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
34
34
|
...studyResultIdentity("shared-world", args.config.id),
|
|
35
35
|
ok: false,
|
|
36
36
|
cwd: args.options.cwd,
|
|
37
|
-
actor:
|
|
37
|
+
actor: args.config.actor?.type ?? "",
|
|
38
38
|
topology: "shared-world",
|
|
39
39
|
topologyMode: "concurrent",
|
|
40
40
|
roleCount: participantList(args.config)?.length ?? 0,
|
|
@@ -4,7 +4,6 @@ import { Command, Option } from "commander";
|
|
|
4
4
|
import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
|
|
5
5
|
import { runStudyPreflight, } from "../../study/preflight.js";
|
|
6
6
|
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, parsePositiveInteger, writeResult, humanError, } from "../io.js";
|
|
7
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
8
7
|
export function registerStudyCommands(parent, io) {
|
|
9
8
|
const study = parent
|
|
10
9
|
.command("study")
|
|
@@ -115,7 +114,7 @@ function formatStudyShowHuman(result) {
|
|
|
115
114
|
`id: ${config.id}`,
|
|
116
115
|
`subject: ${config.subject.source}`,
|
|
117
116
|
...(config.execution?.target ? [`execution: ${config.execution.target}`] : []),
|
|
118
|
-
`actor: ${
|
|
117
|
+
`actor: ${config.actor?.type ?? ""}`,
|
|
119
118
|
...(config.title ? [`title: ${config.title}`] : []),
|
|
120
119
|
...(config.description ? [`description: ${config.description}`] : []),
|
|
121
120
|
...(result.path ? [`path: ${result.path}`] : []),
|
package/dist/cli/doctor.js
CHANGED
|
@@ -336,6 +336,17 @@ function keyChecks(probes, receivingKey, setup, keyUsers) {
|
|
|
336
336
|
const hint = probe.name === receivingKey
|
|
337
337
|
? `provide ${probe.name} through process env or --dotenv`
|
|
338
338
|
: probe.hint;
|
|
339
|
+
if (probe.placeholder) {
|
|
340
|
+
// A participant in a terminal study's sandbox reads this row. It names the placeholder so the
|
|
341
|
+
// participant is not told it holds a key.
|
|
342
|
+
const required = setup?.keys.includes(probe.name) === true;
|
|
343
|
+
return {
|
|
344
|
+
name: `key ${probe.name}`,
|
|
345
|
+
ok: !required,
|
|
346
|
+
...(required ? {} : { status: "note" }),
|
|
347
|
+
message: `not a key: ${probe.name} holds the placeholder humanish sets in a terminal study's sandbox, so no usable ${probe.name} is available here; ${hint}`,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
339
350
|
if (!setup) {
|
|
340
351
|
const users = keyUsers?.get(probe.name) ?? [];
|
|
341
352
|
const usedBy = users.length > 0 ? `used by ${studyList(users)}` : undefined;
|