humanish 0.115.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +7 -6
- package/TELEMETRY.md +6 -0
- package/dist/actors/codex/app-server-ui.js +11 -4
- package/dist/actors/codex/restricted-participant-policy.d.ts +0 -2
- package/dist/actors/codex/restricted-participant-policy.js +4 -32
- package/dist/actors/codex/restricted-participant.d.ts +5 -0
- package/dist/actors/codex/restricted-participant.js +3 -2
- package/dist/actors/computer-use/actor.d.ts +4 -0
- package/dist/actors/computer-use/actor.js +2 -0
- package/dist/actors/computer-use/loop/actions.d.ts +4 -4
- package/dist/actors/computer-use/loop/actions.js +25 -18
- package/dist/actors/computer-use/loop/debrief.js +61 -27
- package/dist/actors/computer-use/loop/session.d.ts +3 -0
- package/dist/actors/computer-use/loop/session.js +9 -1
- package/dist/actors/computer-use/loop/trace.js +1 -0
- package/dist/actors/computer-use/loop/types.d.ts +16 -14
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/wait.d.ts +74 -0
- package/dist/actors/computer-use/wait.js +100 -0
- package/dist/actors/contract.d.ts +14 -0
- package/dist/actors/scripted-browser/steps.js +6 -5
- package/dist/analysis/analysis-limits.d.ts +35 -0
- package/dist/analysis/analysis-limits.js +41 -0
- package/dist/analysis/validation.js +14 -21
- package/dist/browser-control/protocol.d.ts +86 -6
- package/dist/browser-control/protocol.js +13 -6
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/notes.js +2 -1
- package/dist/cli/commands/run-signals.js +5 -1
- package/dist/cli/commands/runs.js +2 -1
- package/dist/cli/commands/setup.js +3 -3
- package/dist/cli/commands/study-format.d.ts +5 -2
- package/dist/cli/commands/study-format.js +29 -9
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +1 -1
- package/dist/cli/commands/tui.js +59 -17
- package/dist/cli/doctor.js +9 -2
- package/dist/cli/findings-text.d.ts +5 -0
- package/dist/cli/findings-text.js +169 -0
- package/dist/cli/findings.d.ts +0 -4
- package/dist/cli/findings.js +5 -167
- package/dist/cli/io.js +6 -1
- package/dist/cli/update-check.js +4 -1
- package/dist/comms/connections.js +4 -3
- package/dist/comms/setup.js +18 -12
- package/dist/evidence/redaction.js +9 -1
- package/dist/evidence/secret-scrub.d.ts +15 -5
- package/dist/evidence/secret-scrub.js +67 -9
- package/dist/feedback/draft.js +1 -1
- package/dist/feedback/export-bundle.js +18 -14
- package/dist/feedback/feedback.js +25 -7
- package/dist/guest/desktop-executor.js +3 -1
- package/dist/guest/desktop-keys.js +59 -5
- package/dist/keys/key-resolution.d.ts +9 -0
- package/dist/keys/key-resolution.js +60 -48
- package/dist/keys/key-status.d.ts +2 -0
- package/dist/keys/key-status.js +9 -16
- package/dist/observer/pinned-files.js +1 -1
- package/dist/observer/render.js +3 -2
- package/dist/observer-app.html +5 -5
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -4
- package/dist/routes/computer-use/bundle-parts.js +2 -2
- package/dist/routes/computer-use/bundle.d.ts +4 -5
- package/dist/routes/computer-use/bundle.js +1 -3
- package/dist/routes/computer-use/e2b-desktop/prepare.js +40 -31
- package/dist/routes/computer-use/fanout-records.js +10 -11
- package/dist/routes/computer-use/live-phase.d.ts +1 -1
- package/dist/routes/computer-use/live-phase.js +0 -1
- package/dist/routes/computer-use/local-vm.js +5 -2
- package/dist/routes/computer-use/participant-execution.d.ts +3 -2
- package/dist/routes/computer-use/participant-model.js +9 -0
- package/dist/routes/computer-use/participant-runs.d.ts +6 -2
- package/dist/routes/computer-use/participant-runs.js +9 -5
- package/dist/routes/computer-use/plan.d.ts +3 -0
- package/dist/routes/computer-use/plan.js +18 -13
- package/dist/routes/computer-use/rerun-selection.js +2 -2
- package/dist/routes/computer-use/result.js +3 -6
- package/dist/routes/computer-use/setup.d.ts +4 -6
- package/dist/routes/computer-use/setup.js +2 -6
- package/dist/routes/computer-use/single-bundle.d.ts +4 -5
- package/dist/routes/computer-use/single-bundle.js +22 -25
- package/dist/routes/computer-use/subject-projection.d.ts +4 -8
- package/dist/routes/computer-use/subject-projection.js +9 -31
- package/dist/routes/computer-use/types.d.ts +5 -42
- package/dist/routes/scripted/scenario.js +28 -12
- package/dist/routes/scripted/session-result.js +6 -3
- package/dist/routes/shared-world/bundle-records.js +2 -8
- package/dist/routes/shared-world/participant-specs.d.ts +0 -2
- package/dist/routes/shared-world/participant-specs.js +1 -20
- package/dist/routes/shared-world/plan.d.ts +3 -0
- package/dist/routes/shared-world/plan.js +25 -4
- package/dist/routes/shared-world/setup.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +9 -9
- package/dist/routes/terminal/ledger.js +63 -40
- package/dist/routes/terminal/live-finish.js +7 -10
- package/dist/routes/terminal/live-sandbox.d.ts +1 -1
- package/dist/routes/terminal/live-sandbox.js +14 -11
- package/dist/routes/terminal/plan.d.ts +3 -0
- package/dist/routes/terminal/plan.js +8 -5
- package/dist/routes/terminal/sandbox.d.ts +4 -3
- package/dist/routes/terminal/sandbox.js +5 -5
- package/dist/routes/terminal/token-usage.d.ts +1 -2
- package/dist/routes/terminal/token-usage.js +4 -6
- package/dist/routes/terminal/types.d.ts +9 -1
- package/dist/run/bundle-shape.js +6 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/contained-output.d.ts +53 -5
- package/dist/run/contained-output.js +141 -42
- package/dist/run/detail.d.ts +2 -1
- package/dist/run/detail.js +12 -7
- package/dist/run/escape-sequences.d.ts +11 -0
- package/dist/run/escape-sequences.js +93 -0
- package/dist/run/evidence-files.d.ts +8 -8
- package/dist/run/evidence-files.js +19 -23
- package/dist/run/locate.d.ts +32 -3
- package/dist/run/locate.js +45 -40
- package/dist/run/notes.d.ts +0 -8
- package/dist/run/notes.js +5 -51
- package/dist/run/outcomes.d.ts +24 -0
- package/dist/run/outcomes.js +19 -0
- package/dist/run/paths.d.ts +2 -0
- package/dist/run/paths.js +5 -1
- package/dist/run/preflight-receipts.js +13 -8
- package/dist/run/project.js +7 -4
- package/dist/run/reclaim-outcomes.d.ts +8 -6
- package/dist/run/reclaim-outcomes.js +10 -6
- package/dist/run/reclaim.d.ts +5 -0
- package/dist/run/reclaim.js +43 -16
- package/dist/run/run-clock.d.ts +50 -0
- package/dist/run/run-clock.js +66 -0
- package/dist/run/run.js +5 -2
- package/dist/run/sandbox-ids.d.ts +7 -2
- package/dist/run/sandbox-ids.js +27 -11
- package/dist/run/secrets.d.ts +2 -1
- package/dist/run/secrets.js +18 -22
- package/dist/run/stored-runs.js +14 -7
- package/dist/run/transient-comms-secrets.js +6 -17
- package/dist/run-study.js +22 -13
- package/dist/study/adapter-scorer-loader.js +6 -4
- package/dist/study/config.js +4 -1
- package/dist/study/discover.js +9 -6
- package/dist/study/init-paths.js +7 -5
- package/dist/study/keys.d.ts +2 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +2 -1
- package/dist/study/parse/actors.js +8 -0
- package/dist/study/persona-resolve.d.ts +2 -2
- package/dist/study/persona-resolve.js +9 -5
- package/dist/study/plan-participants.d.ts +4 -0
- package/dist/study/plan-participants.js +4 -0
- package/dist/study/plan-types.d.ts +2 -2
- package/dist/study/plan.d.ts +5 -3
- package/dist/study/plan.js +18 -9
- package/dist/study/preflight-probes.js +11 -3
- package/dist/study/preflight.js +9 -5
- package/dist/study/summary.d.ts +8 -2
- package/dist/study/summary.js +20 -22
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.d.ts +5 -0
- package/dist/study/validation.js +8 -0
- package/dist/study/warnings.d.ts +19 -12
- package/dist/study/warnings.js +50 -17
- package/dist/subject/serve.d.ts +3 -0
- package/dist/subject/serve.js +4 -1
- package/dist/substrates/detached.js +14 -7
- package/dist/substrates/e2b/connection.d.ts +12 -0
- package/dist/substrates/e2b/connection.js +35 -0
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -5
- package/dist/substrates/e2b/desktop-executor.js +45 -12
- package/dist/substrates/e2b/lifetime.d.ts +24 -4
- package/dist/substrates/e2b/lifetime.js +41 -4
- package/dist/substrates/e2b/sandbox.d.ts +4 -3
- package/dist/substrates/e2b/sandbox.js +13 -18
- package/dist/substrates/e2b/sdk.d.ts +20 -9
- package/dist/substrates/local/capacity.d.ts +18 -2
- package/dist/substrates/local/capacity.js +21 -2
- package/dist/substrates/local/runtime-host.js +21 -14
- package/dist/substrates/local/runtime.js +3 -11
- package/dist/tui/contract.d.ts +30 -5
- package/dist/tui/contract.js +4 -1
- package/dist/tui-app.js +150 -147
- package/dist/verify/artifacts.d.ts +9 -2
- package/dist/verify/artifacts.js +37 -13
- package/dist/verify/liveness.js +19 -5
- package/dist/verify/verify.js +54 -27
- package/docs/architecture/browser-control.md +10 -4
- package/docs/architecture/guest-desktop.md +7 -2
- package/docs/architecture/state-driven-executor.md +4 -2
- package/docs/contracts/run-bundle.md +18 -10
- package/docs/contracts/schemas.md +46 -29
- package/docs/contracts/study-analysis.md +10 -1
- package/docs/product/automatic-analysis.md +7 -1
- package/package.json +2 -2
|
@@ -55,15 +55,6 @@ export type CuaAction = {
|
|
|
55
55
|
} | {
|
|
56
56
|
kind: "screenshot";
|
|
57
57
|
};
|
|
58
|
-
/** A wait action whose duration the provider shortened before the loop dispatched it. */
|
|
59
|
-
export interface ShortenedWait {
|
|
60
|
-
/** The action's index in its turn. */
|
|
61
|
-
index: number;
|
|
62
|
-
/** What the participant asked for. */
|
|
63
|
-
requestedMs: number;
|
|
64
|
-
/** What the action waits: the `ms` it carries. */
|
|
65
|
-
ms: number;
|
|
66
|
-
}
|
|
67
58
|
/** A captured desktop state: the (optional) frame plus a coarse signature for progress. */
|
|
68
59
|
export interface CuaObservation {
|
|
69
60
|
/**
|
|
@@ -171,11 +162,6 @@ export interface CuaTurn {
|
|
|
171
162
|
message?: string;
|
|
172
163
|
/** Actions to perform this turn. Empty means the model is done. */
|
|
173
164
|
actions: CuaAction[];
|
|
174
|
-
/**
|
|
175
|
-
* Waits the provider shortened to the longest wait its desktop accepts, by index into
|
|
176
|
-
* `actions`. The loop records each one and tells the participant on its next request.
|
|
177
|
-
*/
|
|
178
|
-
shortenedWaits?: ReadonlyArray<ShortenedWait>;
|
|
179
165
|
/** Safety checks the provider flagged this turn. Non-empty pauses the run. */
|
|
180
166
|
pendingSafetyChecks: CuaSafetyCheck[];
|
|
181
167
|
/** Token accounting for this turn, if available. */
|
|
@@ -304,6 +290,22 @@ export interface CuaLoopOptions {
|
|
|
304
290
|
idleSteps?: number;
|
|
305
291
|
/** No-progress streak that trips the backstop. Default 20. */
|
|
306
292
|
noProgressSteps?: number;
|
|
293
|
+
/**
|
|
294
|
+
* The longest one `wait` action lasts, in ms: the study's actor.maxWaitMs. A longer wait is
|
|
295
|
+
* shortened to it, the trace records a `wait shortened` notice, and the participant is told on its
|
|
296
|
+
* next request. A wait longer than one desktop call carries is sent in steps. A whole number from
|
|
297
|
+
* 1000 to 600000; any other value throws a RangeError before the session starts. Default
|
|
298
|
+
* defaultMaxWaitMs: 120000, or 30000 when the executor is speechEnabled.
|
|
299
|
+
*/
|
|
300
|
+
maxWaitMs?: number;
|
|
301
|
+
/**
|
|
302
|
+
* How long a `wait` that names no duration lasts, in ms, when its turn only waits or takes
|
|
303
|
+
* screenshots: the study's actor.idleWaitMs. After an action in the same turn such a wait lasts
|
|
304
|
+
* 500 ms. A whole number from 1000 to 600000 and no longer than an explicit maxWaitMs; any other
|
|
305
|
+
* value throws a RangeError before the session starts. Default 5000. The session never applies
|
|
306
|
+
* more than its maxWaitMs, and the trace records the applied lengths as `waitSettings`.
|
|
307
|
+
*/
|
|
308
|
+
idleWaitMs?: number;
|
|
307
309
|
/**
|
|
308
310
|
* If the model flags safety checks, decide which to acknowledge. Returning the
|
|
309
311
|
* list proceeds (the acks are echoed back on the next turn's request); returning
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CuaLoopOptions, CuaLoopResult, LoopRunOptions } from "./loop/types.js";
|
|
2
|
-
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest,
|
|
2
|
+
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest, } from "./loop/types.js";
|
|
3
3
|
export { describeCuaAction } from "./loop/actions.js";
|
|
4
4
|
export { stableProgressKey } from "./loop/backstop.js";
|
|
5
5
|
export { IMPRESSIONS_ASK } from "./loop/impressions.js";
|
|
@@ -133,7 +133,7 @@ async function runTurns(session, conversation) {
|
|
|
133
133
|
? { ...ended, debriefTrigger: { kind: "participant_end", observation } }
|
|
134
134
|
: ended;
|
|
135
135
|
}
|
|
136
|
-
const batch = await runActionBatch(session, turn.actions
|
|
136
|
+
const batch = await runActionBatch(session, turn.actions);
|
|
137
137
|
conversation.previousExecution = batch.execution;
|
|
138
138
|
const checkpoint = await observer.checkpoint(turnNumber);
|
|
139
139
|
if ("stop" in checkpoint)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { ActorWaitSettings } from "../contract.js";
|
|
2
|
+
export declare const CUA_WAIT_LIMITS: Readonly<{
|
|
3
|
+
/**
|
|
4
|
+
* The longest wait one desktop call is given. A browser-control request must be answered within
|
|
5
|
+
* its 35 s deadline, and a hosted desktop runs a wait as a sandbox command that times out after
|
|
6
|
+
* 60 s, so the loop sends a longer wait as consecutive steps of at most this.
|
|
7
|
+
*/
|
|
8
|
+
stepMs: 30000;
|
|
9
|
+
/**
|
|
10
|
+
* The longest one wait action lasts when a study does not set actor.maxWaitMs and the desktop
|
|
11
|
+
* has no speech (defaultMaxWaitMs). Two minutes covers waiting in a lobby or for an email to
|
|
12
|
+
* arrive, and the participant still sees a fresh screenshot at least that often.
|
|
13
|
+
*/
|
|
14
|
+
defaultMaxMs: 120000;
|
|
15
|
+
/**
|
|
16
|
+
* How long a wait that names no duration lasts in a turn that only waits or takes screenshots,
|
|
17
|
+
* when a study does not set actor.idleWaitMs. OpenAI's computer-use `wait` names no duration. A
|
|
18
|
+
* participant told to wait about a minute sent ten in one turn, which lasts 50 s at this length;
|
|
19
|
+
* one wait a turn makes a one-minute wait about six turns.
|
|
20
|
+
*/
|
|
21
|
+
defaultIdleMs: 5000;
|
|
22
|
+
/**
|
|
23
|
+
* How long a wait that names no duration lasts after the participant acts in the same turn, so
|
|
24
|
+
* the screen can settle before the next screenshot. Nearly every OpenAI computer-use wait follows
|
|
25
|
+
* a click or keypress this way, so a longer one would slow every run.
|
|
26
|
+
*/
|
|
27
|
+
settleMs: 500;
|
|
28
|
+
/** The smallest actor.maxWaitMs or actor.idleWaitMs a study may set. */
|
|
29
|
+
leastMs: 1000;
|
|
30
|
+
/** The largest actor.maxWaitMs or actor.idleWaitMs a study may set. */
|
|
31
|
+
mostMs: 600000;
|
|
32
|
+
}>;
|
|
33
|
+
/** The wait lengths one session applies; the trace records them as `waitSettings`. */
|
|
34
|
+
export type WaitSettings = Readonly<ActorWaitSettings>;
|
|
35
|
+
/** One wait as the desktop runs it. */
|
|
36
|
+
export interface PlannedWait {
|
|
37
|
+
/** How long the wait lasts. */
|
|
38
|
+
readonly ms: number;
|
|
39
|
+
/** The desktop calls it takes, in order, each at most CUA_WAIT_LIMITS.stepMs. */
|
|
40
|
+
readonly steps: readonly number[];
|
|
41
|
+
/** What the participant asked for, when that was longer than maxWaitMs. */
|
|
42
|
+
readonly shortenedFromMs?: number;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The longest one wait action lasts when a study does not set actor.maxWaitMs. On a desktop with
|
|
46
|
+
* speech, heard speech reaches the participant only with a screenshot, and the desktop session ends
|
|
47
|
+
* when more utterances arrive between two screenshots than it holds. A wait takes no screenshot, so
|
|
48
|
+
* there the default keeps one wait to one desktop call.
|
|
49
|
+
*/
|
|
50
|
+
export declare function defaultMaxWaitMs(speechEnabled: boolean): number;
|
|
51
|
+
/**
|
|
52
|
+
* Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
|
|
53
|
+
* in a study file, empty for loop options.
|
|
54
|
+
*/
|
|
55
|
+
export declare function waitFieldsReason(fields: {
|
|
56
|
+
readonly maxWaitMs?: unknown;
|
|
57
|
+
readonly idleWaitMs?: unknown;
|
|
58
|
+
}, prefix: string): string | null;
|
|
59
|
+
/**
|
|
60
|
+
* The wait lengths a session applies: the study's values, or their defaults. The idle wait never
|
|
61
|
+
* outlasts the longest wait, which on a desktop with speech defaults to 30 s. Throws a RangeError
|
|
62
|
+
* for fields waitFieldsReason refuses.
|
|
63
|
+
*/
|
|
64
|
+
export declare function resolveWaitSettings(options: {
|
|
65
|
+
readonly maxWaitMs?: number | undefined;
|
|
66
|
+
readonly idleWaitMs?: number | undefined;
|
|
67
|
+
readonly speechEnabled: boolean;
|
|
68
|
+
}): WaitSettings;
|
|
69
|
+
/**
|
|
70
|
+
* How long one wait lasts and the desktop calls it takes. A wait that names no duration lasts
|
|
71
|
+
* idleWaitMs when its turn only waits or takes screenshots (`idleTurn`), and settleWaitMs after an
|
|
72
|
+
* action. A longer wait than maxWaitMs is shortened to it. 70 s is sent as 30 s, 30 s and 10 s.
|
|
73
|
+
*/
|
|
74
|
+
export declare function planWait(settings: WaitSettings, requestedMs: number | undefined, idleTurn: boolean): PlannedWait;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// How long one participant wait action lasts, for every provider and desktop. A study sets the
|
|
2
|
+
// longest (actor.maxWaitMs) and how long a wait that names no duration lasts while the participant
|
|
3
|
+
// only waits (actor.idleWaitMs). The loop asks planWait for each wait and sends it to the desktop
|
|
4
|
+
// in steps one desktop call carries.
|
|
5
|
+
export const CUA_WAIT_LIMITS = Object.freeze({
|
|
6
|
+
/**
|
|
7
|
+
* The longest wait one desktop call is given. A browser-control request must be answered within
|
|
8
|
+
* its 35 s deadline, and a hosted desktop runs a wait as a sandbox command that times out after
|
|
9
|
+
* 60 s, so the loop sends a longer wait as consecutive steps of at most this.
|
|
10
|
+
*/
|
|
11
|
+
stepMs: 30_000,
|
|
12
|
+
/**
|
|
13
|
+
* The longest one wait action lasts when a study does not set actor.maxWaitMs and the desktop
|
|
14
|
+
* has no speech (defaultMaxWaitMs). Two minutes covers waiting in a lobby or for an email to
|
|
15
|
+
* arrive, and the participant still sees a fresh screenshot at least that often.
|
|
16
|
+
*/
|
|
17
|
+
defaultMaxMs: 120_000,
|
|
18
|
+
/**
|
|
19
|
+
* How long a wait that names no duration lasts in a turn that only waits or takes screenshots,
|
|
20
|
+
* when a study does not set actor.idleWaitMs. OpenAI's computer-use `wait` names no duration. A
|
|
21
|
+
* participant told to wait about a minute sent ten in one turn, which lasts 50 s at this length;
|
|
22
|
+
* one wait a turn makes a one-minute wait about six turns.
|
|
23
|
+
*/
|
|
24
|
+
defaultIdleMs: 5_000,
|
|
25
|
+
/**
|
|
26
|
+
* How long a wait that names no duration lasts after the participant acts in the same turn, so
|
|
27
|
+
* the screen can settle before the next screenshot. Nearly every OpenAI computer-use wait follows
|
|
28
|
+
* a click or keypress this way, so a longer one would slow every run.
|
|
29
|
+
*/
|
|
30
|
+
settleMs: 500,
|
|
31
|
+
/** The smallest actor.maxWaitMs or actor.idleWaitMs a study may set. */
|
|
32
|
+
leastMs: 1_000,
|
|
33
|
+
/** The largest actor.maxWaitMs or actor.idleWaitMs a study may set. */
|
|
34
|
+
mostMs: 600_000,
|
|
35
|
+
});
|
|
36
|
+
/** A whole number of milliseconds within the range actor.maxWaitMs and actor.idleWaitMs accept. */
|
|
37
|
+
function isWaitMs(value) {
|
|
38
|
+
return (typeof value === "number" &&
|
|
39
|
+
Number.isSafeInteger(value) &&
|
|
40
|
+
value >= CUA_WAIT_LIMITS.leastMs &&
|
|
41
|
+
value <= CUA_WAIT_LIMITS.mostMs);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The longest one wait action lasts when a study does not set actor.maxWaitMs. On a desktop with
|
|
45
|
+
* speech, heard speech reaches the participant only with a screenshot, and the desktop session ends
|
|
46
|
+
* when more utterances arrive between two screenshots than it holds. A wait takes no screenshot, so
|
|
47
|
+
* there the default keeps one wait to one desktop call.
|
|
48
|
+
*/
|
|
49
|
+
export function defaultMaxWaitMs(speechEnabled) {
|
|
50
|
+
return speechEnabled ? CUA_WAIT_LIMITS.stepMs : CUA_WAIT_LIMITS.defaultMaxMs;
|
|
51
|
+
}
|
|
52
|
+
const FIELDS = [
|
|
53
|
+
["maxWaitMs", "the longest one wait action lasts"],
|
|
54
|
+
["idleWaitMs", "how long a wait with no duration lasts while the participant only waits"],
|
|
55
|
+
];
|
|
56
|
+
/**
|
|
57
|
+
* Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
|
|
58
|
+
* in a study file, empty for loop options.
|
|
59
|
+
*/
|
|
60
|
+
export function waitFieldsReason(fields, prefix) {
|
|
61
|
+
for (const [name, meaning] of FIELDS) {
|
|
62
|
+
if (fields[name] !== undefined && !isWaitMs(fields[name]))
|
|
63
|
+
return `${prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${CUA_WAIT_LIMITS.mostMs}.`;
|
|
64
|
+
}
|
|
65
|
+
const { maxWaitMs, idleWaitMs } = fields;
|
|
66
|
+
if (typeof maxWaitMs === "number" && typeof idleWaitMs === "number" && idleWaitMs > maxWaitMs)
|
|
67
|
+
return `${prefix}idleWaitMs (${idleWaitMs}) is longer than ${prefix}maxWaitMs (${maxWaitMs}), the longest one wait action lasts. Lower ${prefix}idleWaitMs or raise ${prefix}maxWaitMs.`;
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The wait lengths a session applies: the study's values, or their defaults. The idle wait never
|
|
72
|
+
* outlasts the longest wait, which on a desktop with speech defaults to 30 s. Throws a RangeError
|
|
73
|
+
* for fields waitFieldsReason refuses.
|
|
74
|
+
*/
|
|
75
|
+
export function resolveWaitSettings(options) {
|
|
76
|
+
const reason = waitFieldsReason(options, "");
|
|
77
|
+
if (reason !== null)
|
|
78
|
+
throw new RangeError(reason);
|
|
79
|
+
const maxWaitMs = options.maxWaitMs ?? defaultMaxWaitMs(options.speechEnabled);
|
|
80
|
+
return {
|
|
81
|
+
maxWaitMs,
|
|
82
|
+
idleWaitMs: Math.min(options.idleWaitMs ?? CUA_WAIT_LIMITS.defaultIdleMs, maxWaitMs),
|
|
83
|
+
settleWaitMs: CUA_WAIT_LIMITS.settleMs,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* How long one wait lasts and the desktop calls it takes. A wait that names no duration lasts
|
|
88
|
+
* idleWaitMs when its turn only waits or takes screenshots (`idleTurn`), and settleWaitMs after an
|
|
89
|
+
* action. A longer wait than maxWaitMs is shortened to it. 70 s is sent as 30 s, 30 s and 10 s.
|
|
90
|
+
*/
|
|
91
|
+
export function planWait(settings, requestedMs, idleTurn) {
|
|
92
|
+
const askedMs = requestedMs ?? (idleTurn ? settings.idleWaitMs : settings.settleWaitMs);
|
|
93
|
+
const ms = Math.min(askedMs, settings.maxWaitMs);
|
|
94
|
+
const { stepMs } = CUA_WAIT_LIMITS;
|
|
95
|
+
const steps = [];
|
|
96
|
+
for (let left = ms; left > stepMs; left -= stepMs)
|
|
97
|
+
steps.push(stepMs);
|
|
98
|
+
steps.push(ms - steps.length * stepMs);
|
|
99
|
+
return { ms, steps, ...(askedMs > ms ? { shortenedFromMs: askedMs } : {}) };
|
|
100
|
+
}
|
|
@@ -256,6 +256,15 @@ export interface ActorConversation {
|
|
|
256
256
|
estimatedInputTokens?: number;
|
|
257
257
|
}>;
|
|
258
258
|
}
|
|
259
|
+
/** How long a computer-use participant's waits lasted, in milliseconds. */
|
|
260
|
+
export interface ActorWaitSettings {
|
|
261
|
+
/** The longest one wait action lasts (actor.maxWaitMs). A longer wait is shortened to it. */
|
|
262
|
+
maxWaitMs: number;
|
|
263
|
+
/** A wait with no duration in a turn that only waits or takes screenshots (actor.idleWaitMs). */
|
|
264
|
+
idleWaitMs: number;
|
|
265
|
+
/** A wait with no duration after the participant acts in the same turn. */
|
|
266
|
+
settleWaitMs: number;
|
|
267
|
+
}
|
|
259
268
|
export interface ActorTrace {
|
|
260
269
|
schema: typeof ACTOR_TRACE_SCHEMA;
|
|
261
270
|
provider: string;
|
|
@@ -303,6 +312,11 @@ export interface ActorTrace {
|
|
|
303
312
|
reasoningEffort: string;
|
|
304
313
|
maxOutputTokens?: number;
|
|
305
314
|
};
|
|
315
|
+
/**
|
|
316
|
+
* Additive + optional: the wait lengths the computer-use loop applied, defaults included. Absent
|
|
317
|
+
* on other producers and on bundles written before it was recorded.
|
|
318
|
+
*/
|
|
319
|
+
waitSettings?: ActorWaitSettings;
|
|
306
320
|
counts: Record<string, number>;
|
|
307
321
|
/**
|
|
308
322
|
* Additive + optional affordance record (humanish.affordance-use.v1): which kind of route
|
|
@@ -6,7 +6,7 @@ import os from "node:os";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
8
8
|
import { digestText, redactToSecretLabel } from "../../evidence/redaction.js";
|
|
9
|
-
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputFile, prepareSelectedOutputDirectory, readContainedRegularFile, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
9
|
+
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputFile, prepareSelectedOutputDirectory, readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
10
10
|
import { LOOPBACK_EVIDENCE_URL_POLICY, } from "./types.js";
|
|
11
11
|
export async function executeBrowserPersonaStep(args) {
|
|
12
12
|
const started = Date.now();
|
|
@@ -402,11 +402,12 @@ async function captureScriptedPageScreenshot(page) {
|
|
|
402
402
|
if (Buffer.isBuffer(returned) || returned instanceof Uint8Array) {
|
|
403
403
|
return stripPngMetadataChunks(browserScreenshotBytes(returned));
|
|
404
404
|
}
|
|
405
|
-
const
|
|
406
|
-
if (
|
|
405
|
+
const staged = await readContainedRegularFile(stagingRoot, "capture.png", RUN_ARTIFACT_MAX_BYTES);
|
|
406
|
+
if (staged.status === "missing")
|
|
407
407
|
throw new Error("Browser screenshot did not return bytes or write a single-link staging file.");
|
|
408
|
-
|
|
409
|
-
|
|
408
|
+
if (staged.status === "refused")
|
|
409
|
+
throw new Error(`Browser screenshot did not return bytes, and ${refusalText("its staging file capture.png", staged)}.`);
|
|
410
|
+
return stripPngMetadataChunks(staged.bytes);
|
|
410
411
|
}
|
|
411
412
|
finally {
|
|
412
413
|
await assertPreparedSelectedOutputDirectory(stagingRoot)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How much an analysis result may hold, for the fields the Observer checks as well. Characters are
|
|
3
|
+
* counted as `String.length` counts them.
|
|
4
|
+
*/
|
|
5
|
+
export declare const ANALYSIS_LIMITS: Readonly<{
|
|
6
|
+
/** A label: a finding's title and headline, a design finding's headline and screen, or a name. */
|
|
7
|
+
labelChars: 240;
|
|
8
|
+
experienceChars: 1200;
|
|
9
|
+
noticeChars: 1500;
|
|
10
|
+
whyItMattersChars: 1000;
|
|
11
|
+
suggestionChars: 1000;
|
|
12
|
+
/** Evidence one observation or design finding may cite. */
|
|
13
|
+
evidenceRefs: 100;
|
|
14
|
+
findings: 100;
|
|
15
|
+
designFindings: 40;
|
|
16
|
+
concernReviews: 60;
|
|
17
|
+
}>;
|
|
18
|
+
/** Whether text holds a control character other than tab, line feed and carriage return. */
|
|
19
|
+
export declare function hasControlCharacter(text: string): boolean;
|
|
20
|
+
/** The parts of an analysis result the revision boundaries read. */
|
|
21
|
+
interface RevisionFields {
|
|
22
|
+
readonly concernReviews?: unknown;
|
|
23
|
+
readonly designFindings?: unknown;
|
|
24
|
+
readonly findings: ReadonlyArray<{
|
|
25
|
+
readonly headline?: unknown;
|
|
26
|
+
readonly experience?: unknown;
|
|
27
|
+
}>;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Whether a result carries every field its prompt revision requires: concern reviews from
|
|
31
|
+
* study-evidence-5, and headlines, experiences and design findings from study-evidence-7. A prompt
|
|
32
|
+
* version without a study-evidence revision number requires none of them.
|
|
33
|
+
*/
|
|
34
|
+
export declare function hasRevisionFields(promptVersion: string, result: RevisionFields): boolean;
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// The analysis result's limits and prompt revision boundaries, kept apart from the zod schemas in
|
|
2
|
+
// validation.ts so the Observer can read them: it bundles this module, and zod stays out of its
|
|
3
|
+
// page.
|
|
4
|
+
/**
|
|
5
|
+
* How much an analysis result may hold, for the fields the Observer checks as well. Characters are
|
|
6
|
+
* counted as `String.length` counts them.
|
|
7
|
+
*/
|
|
8
|
+
export const ANALYSIS_LIMITS = Object.freeze({
|
|
9
|
+
/** A label: a finding's title and headline, a design finding's headline and screen, or a name. */
|
|
10
|
+
labelChars: 240,
|
|
11
|
+
experienceChars: 1200,
|
|
12
|
+
noticeChars: 1500,
|
|
13
|
+
whyItMattersChars: 1000,
|
|
14
|
+
suggestionChars: 1000,
|
|
15
|
+
/** Evidence one observation or design finding may cite. */
|
|
16
|
+
evidenceRefs: 100,
|
|
17
|
+
findings: 100,
|
|
18
|
+
designFindings: 40,
|
|
19
|
+
concernReviews: 60,
|
|
20
|
+
});
|
|
21
|
+
/** Whether text holds a control character other than tab, line feed and carriage return. */
|
|
22
|
+
export function hasControlCharacter(text) {
|
|
23
|
+
return /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(text);
|
|
24
|
+
}
|
|
25
|
+
// The prompt revision from which a result must carry each field. A boundary stays where it is when
|
|
26
|
+
// the prompt version advances, so a later prompt cannot bring back a legacy omission.
|
|
27
|
+
const CONCERN_REVIEWS_FROM = 5;
|
|
28
|
+
const HEADLINES_AND_DESIGN_FINDINGS_FROM = 7;
|
|
29
|
+
/**
|
|
30
|
+
* Whether a result carries every field its prompt revision requires: concern reviews from
|
|
31
|
+
* study-evidence-5, and headlines, experiences and design findings from study-evidence-7. A prompt
|
|
32
|
+
* version without a study-evidence revision number requires none of them.
|
|
33
|
+
*/
|
|
34
|
+
export function hasRevisionFields(promptVersion, result) {
|
|
35
|
+
const revision = Number(/^study-evidence-(\d+)$/.exec(promptVersion)?.[1] ?? 0);
|
|
36
|
+
if (revision >= CONCERN_REVIEWS_FROM && result.concernReviews === undefined)
|
|
37
|
+
return false;
|
|
38
|
+
return (revision < HEADLINES_AND_DESIGN_FINDINGS_FROM ||
|
|
39
|
+
(result.designFindings !== undefined &&
|
|
40
|
+
result.findings.every((finding) => finding.headline !== undefined && finding.experience !== undefined)));
|
|
41
|
+
}
|
|
@@ -2,12 +2,13 @@ import { validStoredCodexAnalysisConfig } from "./codex-config.js";
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
|
+
import { ANALYSIS_LIMITS, hasControlCharacter, hasRevisionFields } from "./analysis-limits.js";
|
|
5
6
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
6
7
|
const text = (max) => z
|
|
7
8
|
.string()
|
|
8
9
|
.max(max)
|
|
9
|
-
.refine((value) =>
|
|
10
|
-
const label = text(
|
|
10
|
+
.refine((value) => !hasControlCharacter(value), "Invalid text.");
|
|
11
|
+
const label = text(ANALYSIS_LIMITS.labelChars).min(1);
|
|
11
12
|
const id = z
|
|
12
13
|
.string()
|
|
13
14
|
.min(1)
|
|
@@ -17,7 +18,7 @@ const sourceId = text(256).min(1);
|
|
|
17
18
|
const digest = z.string().regex(SHA256_HEX_PATTERN);
|
|
18
19
|
const timestamp = z.string().datetime({ offset: true });
|
|
19
20
|
const sourceIds = z.array(sourceId).max(128);
|
|
20
|
-
const refs = z.array(id).min(1).max(
|
|
21
|
+
const refs = z.array(id).min(1).max(ANALYSIS_LIMITS.evidenceRefs);
|
|
21
22
|
const limitations = z.array(text(2000).min(1)).max(100);
|
|
22
23
|
const observationSchema = z
|
|
23
24
|
.object({
|
|
@@ -32,9 +33,9 @@ const designFindingSchema = z
|
|
|
32
33
|
id,
|
|
33
34
|
headline: label,
|
|
34
35
|
screen: label,
|
|
35
|
-
notice: text(
|
|
36
|
-
whyItMatters: text(
|
|
37
|
-
suggestion: text(
|
|
36
|
+
notice: text(ANALYSIS_LIMITS.noticeChars).min(1),
|
|
37
|
+
whyItMatters: text(ANALYSIS_LIMITS.whyItMattersChars).min(1),
|
|
38
|
+
suggestion: text(ANALYSIS_LIMITS.suggestionChars).min(1),
|
|
38
39
|
severity: z.enum(["minor", "moderate", "major"]),
|
|
39
40
|
confidence: z.enum(["low", "medium", "high"]),
|
|
40
41
|
seenByStreamIds: sourceIds.min(1),
|
|
@@ -66,7 +67,7 @@ const analysisResultSchema = z
|
|
|
66
67
|
id,
|
|
67
68
|
title: label,
|
|
68
69
|
headline: label.optional(),
|
|
69
|
-
experience: text(
|
|
70
|
+
experience: text(ANALYSIS_LIMITS.experienceChars).min(1).optional(),
|
|
70
71
|
summary: text(4000).min(1),
|
|
71
72
|
impact: z.enum(["blocked_task", "friction", "recovery", "uncertain"]),
|
|
72
73
|
affectedStreamIds: sourceIds.min(1),
|
|
@@ -79,8 +80,8 @@ const analysisResultSchema = z
|
|
|
79
80
|
priorityReason: text(2000).min(1),
|
|
80
81
|
})
|
|
81
82
|
.strict())
|
|
82
|
-
.max(
|
|
83
|
-
designFindings: z.array(designFindingSchema).max(
|
|
83
|
+
.max(ANALYSIS_LIMITS.findings),
|
|
84
|
+
designFindings: z.array(designFindingSchema).max(ANALYSIS_LIMITS.designFindings).optional(),
|
|
84
85
|
concernReviews: z
|
|
85
86
|
.array(observationSchema
|
|
86
87
|
.extend({
|
|
@@ -89,7 +90,7 @@ const analysisResultSchema = z
|
|
|
89
90
|
reason: text(2000).min(1),
|
|
90
91
|
})
|
|
91
92
|
.strict())
|
|
92
|
-
.max(
|
|
93
|
+
.max(ANALYSIS_LIMITS.concernReviews)
|
|
93
94
|
.optional(),
|
|
94
95
|
limitations,
|
|
95
96
|
})
|
|
@@ -100,8 +101,8 @@ export const analysisResponseSchema = analysisResultSchema
|
|
|
100
101
|
.extend({
|
|
101
102
|
findings: z
|
|
102
103
|
.array(analysisResultSchema.shape.findings.element.required({ headline: true, experience: true }))
|
|
103
|
-
.max(
|
|
104
|
-
designFindings: z.array(designFindingSchema).max(
|
|
104
|
+
.max(ANALYSIS_LIMITS.findings),
|
|
105
|
+
designFindings: z.array(designFindingSchema).max(ANALYSIS_LIMITS.designFindings),
|
|
105
106
|
})
|
|
106
107
|
.required({ concernReviews: true });
|
|
107
108
|
export const analysisResultJsonSchema = z.toJSONSchema(analysisResponseSchema);
|
|
@@ -469,15 +470,7 @@ export function validateAnalysisArtifact(value) {
|
|
|
469
470
|
(artifact.status === "complete" && !coverage.complete))
|
|
470
471
|
throw new Error("ANALYSIS_STATUS_INVALID");
|
|
471
472
|
if (artifact.result !== null) {
|
|
472
|
-
|
|
473
|
-
// findings with revision 7. Keep each boundary stable when the prompt version advances; a later
|
|
474
|
-
// prompt must not regain legacy omissions.
|
|
475
|
-
const revision = Number(/^study-evidence-(\d+)$/.exec(artifact.promptVersion)?.[1] ?? 0);
|
|
476
|
-
const result = artifact.result;
|
|
477
|
-
if ((revision >= 5 && result.concernReviews === undefined) ||
|
|
478
|
-
(revision >= 7 &&
|
|
479
|
-
(result.designFindings === undefined ||
|
|
480
|
-
result.findings.some((finding) => finding.headline === undefined || finding.experience === undefined))))
|
|
473
|
+
if (!hasRevisionFields(artifact.promptVersion, artifact.result))
|
|
481
474
|
throw new Error("ANALYSIS_RESULT_SCHEMA_INVALID");
|
|
482
475
|
validateAnalysisResult({ ...artifact, images: [] }, artifact.result);
|
|
483
476
|
}
|
|
@@ -26,7 +26,7 @@ export interface BrowserControlIdentity {
|
|
|
26
26
|
challenge: string;
|
|
27
27
|
runtimeRevision: string;
|
|
28
28
|
}
|
|
29
|
-
export declare const
|
|
29
|
+
export declare const participantActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<readonly [z.ZodObject<{
|
|
30
30
|
x: z.ZodNumber;
|
|
31
31
|
y: z.ZodNumber;
|
|
32
32
|
kind: z.ZodLiteral<"click">;
|
|
@@ -66,8 +66,88 @@ export declare const browserOnlyControlActionSchema: z.ZodDiscriminatedUnion<rea
|
|
|
66
66
|
ms: z.ZodOptional<z.ZodNumber>;
|
|
67
67
|
}, z.core.$strict>, z.ZodObject<{
|
|
68
68
|
kind: z.ZodLiteral<"screenshot">;
|
|
69
|
-
}, z.core.$strict>], "kind"
|
|
70
|
-
|
|
69
|
+
}, z.core.$strict>], "kind">, z.ZodTransform<{
|
|
70
|
+
kind: "click";
|
|
71
|
+
x: number;
|
|
72
|
+
y: number;
|
|
73
|
+
button?: "left" | "right" | "middle";
|
|
74
|
+
heldKeys?: string[];
|
|
75
|
+
} | {
|
|
76
|
+
kind: "double_click";
|
|
77
|
+
x: number;
|
|
78
|
+
y: number;
|
|
79
|
+
heldKeys?: string[];
|
|
80
|
+
} | {
|
|
81
|
+
kind: "move";
|
|
82
|
+
x: number;
|
|
83
|
+
y: number;
|
|
84
|
+
heldKeys?: string[];
|
|
85
|
+
} | {
|
|
86
|
+
kind: "scroll";
|
|
87
|
+
x: number;
|
|
88
|
+
y: number;
|
|
89
|
+
dx: number;
|
|
90
|
+
dy: number;
|
|
91
|
+
heldKeys?: string[];
|
|
92
|
+
} | {
|
|
93
|
+
kind: "type";
|
|
94
|
+
text: string;
|
|
95
|
+
} | {
|
|
96
|
+
kind: "keypress";
|
|
97
|
+
keys: string[];
|
|
98
|
+
} | {
|
|
99
|
+
kind: "drag";
|
|
100
|
+
path: Array<{
|
|
101
|
+
x: number;
|
|
102
|
+
y: number;
|
|
103
|
+
}>;
|
|
104
|
+
heldKeys?: string[];
|
|
105
|
+
} | {
|
|
106
|
+
kind: "wait";
|
|
107
|
+
ms?: number;
|
|
108
|
+
} | {
|
|
109
|
+
kind: "speak";
|
|
110
|
+
text: string;
|
|
111
|
+
} | {
|
|
112
|
+
kind: "screenshot";
|
|
113
|
+
}, {
|
|
114
|
+
kind: "type";
|
|
115
|
+
text: string;
|
|
116
|
+
} | {
|
|
117
|
+
kind: "keypress";
|
|
118
|
+
keys: string[];
|
|
119
|
+
} | {
|
|
120
|
+
kind: "screenshot";
|
|
121
|
+
} | {
|
|
122
|
+
x: number;
|
|
123
|
+
y: number;
|
|
124
|
+
kind: "click";
|
|
125
|
+
button?: "left" | "middle" | "right" | undefined;
|
|
126
|
+
} | {
|
|
127
|
+
x: number;
|
|
128
|
+
y: number;
|
|
129
|
+
kind: "double_click";
|
|
130
|
+
} | {
|
|
131
|
+
x: number;
|
|
132
|
+
y: number;
|
|
133
|
+
kind: "move";
|
|
134
|
+
} | {
|
|
135
|
+
x: number;
|
|
136
|
+
y: number;
|
|
137
|
+
kind: "scroll";
|
|
138
|
+
dx: number;
|
|
139
|
+
dy: number;
|
|
140
|
+
} | {
|
|
141
|
+
kind: "drag";
|
|
142
|
+
path: {
|
|
143
|
+
x: number;
|
|
144
|
+
y: number;
|
|
145
|
+
}[];
|
|
146
|
+
} | {
|
|
147
|
+
kind: "wait";
|
|
148
|
+
ms?: number | undefined;
|
|
149
|
+
}>>;
|
|
150
|
+
export declare const participantSpeechActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
71
151
|
x: z.ZodNumber;
|
|
72
152
|
y: z.ZodNumber;
|
|
73
153
|
kind: z.ZodLiteral<"click">;
|
|
@@ -160,9 +240,6 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
|
|
|
160
240
|
} | {
|
|
161
241
|
kind: "keypress";
|
|
162
242
|
keys: string[];
|
|
163
|
-
} | {
|
|
164
|
-
kind: "wait";
|
|
165
|
-
ms?: number | undefined;
|
|
166
243
|
} | {
|
|
167
244
|
kind: "screenshot";
|
|
168
245
|
} | {
|
|
@@ -193,6 +270,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
|
|
|
193
270
|
x: number;
|
|
194
271
|
y: number;
|
|
195
272
|
}[];
|
|
273
|
+
} | {
|
|
274
|
+
kind: "wait";
|
|
275
|
+
ms?: number | undefined;
|
|
196
276
|
}>>;
|
|
197
277
|
declare const observationSchema: z.ZodObject<{
|
|
198
278
|
png: z.ZodString;
|
|
@@ -64,13 +64,18 @@ const dragAction = z.strictObject({
|
|
|
64
64
|
});
|
|
65
65
|
const typeAction = z.strictObject({ kind: z.literal("type"), text });
|
|
66
66
|
const keypressAction = z.strictObject({ kind: z.literal("keypress"), keys: chord });
|
|
67
|
+
const waitMs = z.number().finite().min(0);
|
|
68
|
+
// One request carries one wait, so the wire bounds it by the request deadline.
|
|
67
69
|
const waitAction = z.strictObject({
|
|
68
70
|
kind: z.literal("wait"),
|
|
69
|
-
ms:
|
|
71
|
+
ms: waitMs.max(BROWSER_CONTROL_LIMITS.waitMs).optional(),
|
|
70
72
|
});
|
|
73
|
+
// A participant may ask for any wait: the loop shortens it to the study's longest and sends it in
|
|
74
|
+
// steps the wire accepts.
|
|
75
|
+
const participantWaitAction = z.strictObject({ kind: z.literal("wait"), ms: waitMs.optional() });
|
|
71
76
|
const screenshotAction = z.strictObject({ kind: z.literal("screenshot") });
|
|
72
77
|
const speakAction = z.strictObject({ kind: z.literal("speak"), text: speechText });
|
|
73
|
-
const
|
|
78
|
+
const participantActionSchemas = [
|
|
74
79
|
clickAction,
|
|
75
80
|
doubleClickAction,
|
|
76
81
|
moveAction,
|
|
@@ -78,13 +83,15 @@ const browserActionSchemas = [
|
|
|
78
83
|
typeAction,
|
|
79
84
|
keypressAction,
|
|
80
85
|
dragAction,
|
|
81
|
-
|
|
86
|
+
participantWaitAction,
|
|
82
87
|
screenshotAction,
|
|
83
88
|
];
|
|
84
89
|
// The Codex participant's tool schema: no held keys, so its tool surface stays as published.
|
|
85
|
-
export const
|
|
86
|
-
|
|
87
|
-
.
|
|
90
|
+
export const participantActionSchema = z
|
|
91
|
+
.discriminatedUnion("kind", participantActionSchemas)
|
|
92
|
+
.transform(toCuaAction);
|
|
93
|
+
export const participantSpeechActionSchema = z
|
|
94
|
+
.discriminatedUnion("kind", [...participantActionSchemas, speakAction])
|
|
88
95
|
.transform(toCuaAction);
|
|
89
96
|
// What crosses the host-guest wire: the participant actions plus held keys on pointer actions,
|
|
90
97
|
// which the OpenAI provider maps from its computer tool.
|
|
@@ -6,7 +6,8 @@ import { listAnalyses } from "../../analysis/store.js";
|
|
|
6
6
|
import { listAnalysisExecutions } from "../../analysis/store-executions.js";
|
|
7
7
|
import { resolveRunPath } from "../../run/locate.js";
|
|
8
8
|
import { CWD_OPTION_DESCRIPTION, discoverCliKeys, JSON_OPTION_DESCRIPTION, RUN_OPTION_DESCRIPTION, writeResult, humanError, wantsJson, } from "../io.js";
|
|
9
|
-
import { analysisFindings,
|
|
9
|
+
import { analysisFindings, missingRunFindings, readRunAnalysis, } from "../findings.js";
|
|
10
|
+
import { formatFindings } from "../findings-text.js";
|
|
10
11
|
import { resolvePhysicalCwd } from "../../run/paths.js";
|
|
11
12
|
/** Commander may collect a shared flag on the parent; only explicit values override leaf defaults. */
|
|
12
13
|
function analysisSelection(options, command) {
|
|
@@ -2,9 +2,10 @@ import { renderObserver } from "../../observer/render.js";
|
|
|
2
2
|
import { loadRunBundlePrepared, resolveRunPath } from "../../run/locate.js";
|
|
3
3
|
import { readRunNotes } from "../../run/note-files.js";
|
|
4
4
|
import { byRunTime } from "../../run/note-shape.js";
|
|
5
|
-
import { addRunNote
|
|
5
|
+
import { addRunNote } from "../../run/notes.js";
|
|
6
6
|
import { runParticipantCaptions, streamParticipantIdOf } from "../../run/participant-records.js";
|
|
7
7
|
import { resolvePhysicalCwd, runIdOf } from "../../run/paths.js";
|
|
8
|
+
import { formatRunTime } from "../../run/run-clock.js";
|
|
8
9
|
import { runNotFoundMessage } from "../../run/run-not-found.js";
|
|
9
10
|
import { cli } from "../invocation.js";
|
|
10
11
|
import { CWD_OPTION_DESCRIPTION, humanError, JSON_OPTION_DESCRIPTION, writeResult, } from "../io.js";
|