humanish 0.139.1 → 0.141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +5 -4
- package/dist/actors/codex/restricted-participant.js +1 -1
- package/dist/actors/computer-use/external-handoff.d.ts +22 -0
- package/dist/actors/computer-use/external-handoff.js +83 -0
- package/dist/actors/computer-use/loop/observation.d.ts +7 -0
- package/dist/actors/computer-use/loop/observation.js +17 -1
- package/dist/actors/computer-use/loop/types.d.ts +6 -0
- package/dist/analysis/analysis-limits.d.ts +6 -0
- package/dist/analysis/analysis-limits.js +6 -0
- package/dist/analysis/analysis-lock.d.ts +8 -0
- package/dist/analysis/analysis-lock.js +129 -0
- package/dist/analysis/automatic-completion.js +31 -0
- package/dist/analysis/automatic-config.d.ts +15 -6
- package/dist/analysis/automatic-config.js +25 -9
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/job.d.ts +1 -0
- package/dist/analysis/job.js +6 -2
- package/dist/analysis/recording-ended.d.ts +16 -0
- package/dist/analysis/recording-ended.js +29 -0
- package/dist/analysis/service.d.ts +1 -3
- package/dist/analysis/service.js +23 -50
- package/dist/analysis/source.d.ts +1 -1
- package/dist/browser-control/protocol.d.ts +8 -0
- package/dist/browser-control/protocol.js +21 -0
- package/dist/cli/commands/analysis-signals.d.ts +10 -4
- package/dist/cli/commands/analysis-signals.js +28 -6
- package/dist/cli/commands/runs.js +8 -2
- package/dist/cli/commands/study-route-computer-use.js +4 -5
- package/dist/cli/commands/study-route-scripted.js +5 -6
- package/dist/cli/commands/study-route-shared-world.js +4 -5
- package/dist/cli/commands/study-route-terminal.js +5 -6
- package/dist/cli/commands/study-run.js +1 -1
- package/dist/cli/doctor.d.ts +2 -0
- package/dist/cli/doctor.js +3 -0
- package/dist/cli/findings.d.ts +40 -14
- package/dist/cli/findings.js +117 -35
- package/dist/cli/io.d.ts +5 -1
- package/dist/cli/io.js +12 -2
- package/dist/cli/run-findings.d.ts +19 -0
- package/dist/cli/run-findings.js +106 -0
- package/dist/evidence/browser-hygiene.d.ts +12 -1
- package/dist/evidence/browser-hygiene.js +45 -2
- package/dist/feedback/export-bundle.js +1 -1
- package/dist/guest/bootstrap.d.ts +4 -2
- package/dist/guest/bootstrap.js +15 -7
- package/dist/guest/display-config.d.ts +31 -0
- package/dist/guest/display-config.js +35 -0
- package/dist/guest/external-links.d.ts +14 -0
- package/dist/guest/external-links.js +48 -0
- package/dist/guest/runtime-desktop.d.ts +9 -3
- package/dist/guest/runtime-desktop.js +64 -27
- package/dist/guest/runtime.d.ts +2 -1
- package/dist/guest/runtime.js +1 -1
- package/dist/guest-runtime-main.js +2 -1
- package/dist/observer/data.d.ts +4 -0
- package/dist/observer/run-routes.js +12 -3
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/e2b-desktop/desktop.js +27 -17
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +10 -17
- package/dist/routes/computer-use/e2b-desktop/teardown.js +1 -1
- package/dist/routes/computer-use/in-process-desktop.js +2 -2
- package/dist/routes/computer-use/live-phase.js +1 -1
- package/dist/routes/computer-use/local-vm.js +34 -1
- package/dist/routes/computer-use/plan.js +1 -1
- package/dist/routes/computer-use/setup.js +2 -2
- package/dist/routes/scripted/plan.js +1 -1
- package/dist/routes/shared-world/plan.js +1 -1
- package/dist/routes/shared-world/route.js +1 -1
- package/dist/routes/terminal/plan.js +1 -1
- package/dist/run/display.d.ts +6 -1
- package/dist/run/display.js +11 -0
- package/dist/run/host-suspension.js +4 -1
- package/dist/run/preflight-receipts.d.ts +3 -12
- package/dist/run/preflight-receipts.js +7 -67
- package/dist/run/process-owner.d.ts +28 -0
- package/dist/run/process-owner.js +83 -0
- package/dist/run/projection.js +1 -1
- package/dist/run/reclaim.d.ts +6 -0
- package/dist/run/reclaim.js +32 -2
- package/dist/run/route-shell.js +2 -2
- package/dist/run/run-index.d.ts +3 -1
- package/dist/run/run-index.js +23 -11
- package/dist/run/run.d.ts +16 -5
- package/dist/run/run.js +25 -10
- package/dist/run/status.d.ts +57 -2
- package/dist/run/status.js +87 -1
- package/dist/run/stored-runs.d.ts +3 -0
- package/dist/run/stored-runs.js +9 -4
- package/dist/study/doctor.js +11 -6
- package/dist/study/init-templates.js +9 -3
- package/dist/study/mobile-emulation.d.ts +27 -0
- package/dist/study/mobile-emulation.js +60 -0
- package/dist/study/plan.js +1 -1
- package/dist/study/preflight-result.js +1 -1
- package/dist/study/run-study-events.d.ts +7 -1
- package/dist/study/summary.js +1 -1
- package/dist/study/warnings.js +5 -2
- package/dist/substrates/e2b/cdp-probe.d.ts +2 -6
- package/dist/substrates/e2b/desktop-browser.js +3 -1
- package/dist/substrates/e2b/desktop-cdp.d.ts +0 -2
- package/dist/substrates/e2b/desktop-cdp.js +0 -2
- package/dist/substrates/e2b/desktop-executor.d.ts +7 -0
- package/dist/substrates/e2b/desktop-executor.js +3 -0
- package/dist/substrates/e2b/desktop-handoff.d.ts +22 -0
- package/dist/substrates/e2b/desktop-handoff.js +86 -0
- package/dist/substrates/local/firecracker-desktop.d.ts +5 -0
- package/dist/substrates/local/firecracker-desktop.js +7 -1
- package/dist/substrates/local/runtime-config.d.ts +12 -0
- package/dist/substrates/local/runtime-config.js +55 -13
- package/dist/substrates/local/runtime-release.js +8 -8
- package/dist/substrates/local/runtime.js +1 -0
- package/dist/tui-app.js +115 -115
- package/docs/architecture/observer.md +7 -3
- package/docs/contracts/schemas.md +24 -3
- package/docs/contracts/study-analysis.md +15 -9
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +75 -18
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +153 -31
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { DevicePreset } from "./device-presets.js";
|
|
2
|
+
import type { StudyDesktopFidelity } from "./types.js";
|
|
3
|
+
/** The CDP Emulation overrides one participant's Chromium pages get. */
|
|
4
|
+
export interface MobileEmulationRequest {
|
|
5
|
+
width: number;
|
|
6
|
+
height: number;
|
|
7
|
+
deviceScaleFactor: number;
|
|
8
|
+
touch: boolean;
|
|
9
|
+
userAgent: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The emulation a participant gets, or undefined. Only a participant on a mobile preset is
|
|
13
|
+
* emulated: a run-wide `mobileEmulation` must not hand a desktop or tablet participant beside a
|
|
14
|
+
* phone participant an iPhone user agent. Hosted and local desktops both read this.
|
|
15
|
+
*/
|
|
16
|
+
export declare function participantMobileEmulation(fidelity: StudyDesktopFidelity | undefined, preset: DevicePreset): MobileEmulationRequest | undefined;
|
|
17
|
+
/** Chromium flags that carry the user agent and touch support to every page and worker. */
|
|
18
|
+
export declare function mobileEmulationLaunchFlags(request: MobileEmulationRequest | undefined): string[];
|
|
19
|
+
/**
|
|
20
|
+
* The CDP commands that apply a request to one page, in order. The local guest sends these over
|
|
21
|
+
* its own page session; the hosted desktop's in-sandbox probe sends the same Emulation domain.
|
|
22
|
+
* Touch adds mouse-to-touch conversion, since participants drive a desktop pointer.
|
|
23
|
+
*/
|
|
24
|
+
export declare function mobileEmulationCommands(request: MobileEmulationRequest): {
|
|
25
|
+
method: string;
|
|
26
|
+
params: Record<string, unknown>;
|
|
27
|
+
}[];
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/** The user agent a mobile-emulated participant presents unless the study sets its own. */
|
|
2
|
+
const DEFAULT_MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1";
|
|
3
|
+
/**
|
|
4
|
+
* The emulation a participant gets, or undefined. Only a participant on a mobile preset is
|
|
5
|
+
* emulated: a run-wide `mobileEmulation` must not hand a desktop or tablet participant beside a
|
|
6
|
+
* phone participant an iPhone user agent. Hosted and local desktops both read this.
|
|
7
|
+
*/
|
|
8
|
+
export function participantMobileEmulation(fidelity, preset) {
|
|
9
|
+
if (!fidelity?.mobileEmulation || !preset.isMobile)
|
|
10
|
+
return undefined;
|
|
11
|
+
return {
|
|
12
|
+
width: preset.width,
|
|
13
|
+
height: preset.height,
|
|
14
|
+
deviceScaleFactor: fidelity.deviceScaleFactor ?? preset.deviceScaleFactor,
|
|
15
|
+
touch: fidelity.touch ?? true,
|
|
16
|
+
userAgent: fidelity.userAgent ?? DEFAULT_MOBILE_USER_AGENT,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/** Chromium flags that carry the user agent and touch support to every page and worker. */
|
|
20
|
+
export function mobileEmulationLaunchFlags(request) {
|
|
21
|
+
if (request === undefined)
|
|
22
|
+
return [];
|
|
23
|
+
return [
|
|
24
|
+
`--user-agent=${request.userAgent}`,
|
|
25
|
+
...(request.touch ? ["--touch-events=enabled"] : []),
|
|
26
|
+
];
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The CDP commands that apply a request to one page, in order. The local guest sends these over
|
|
30
|
+
* its own page session; the hosted desktop's in-sandbox probe sends the same Emulation domain.
|
|
31
|
+
* Touch adds mouse-to-touch conversion, since participants drive a desktop pointer.
|
|
32
|
+
*/
|
|
33
|
+
export function mobileEmulationCommands(request) {
|
|
34
|
+
return [
|
|
35
|
+
{
|
|
36
|
+
method: "Emulation.setDeviceMetricsOverride",
|
|
37
|
+
params: {
|
|
38
|
+
width: request.width,
|
|
39
|
+
height: request.height,
|
|
40
|
+
deviceScaleFactor: request.deviceScaleFactor,
|
|
41
|
+
mobile: true,
|
|
42
|
+
screenWidth: request.width,
|
|
43
|
+
screenHeight: request.height,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
...(request.touch
|
|
47
|
+
? [
|
|
48
|
+
{
|
|
49
|
+
method: "Emulation.setTouchEmulationEnabled",
|
|
50
|
+
params: { enabled: true, maxTouchPoints: 5 },
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
method: "Emulation.setEmitTouchEventsForMouse",
|
|
54
|
+
params: { enabled: true, configuration: "mobile" },
|
|
55
|
+
},
|
|
56
|
+
]
|
|
57
|
+
: []),
|
|
58
|
+
{ method: "Emulation.setUserAgentOverride", params: { userAgent: request.userAgent } },
|
|
59
|
+
];
|
|
60
|
+
}
|
package/dist/study/plan.js
CHANGED
|
@@ -41,7 +41,7 @@ function planPreview(config, options, input) {
|
|
|
41
41
|
});
|
|
42
42
|
if (String(config.comms?.email?.kind) === "real")
|
|
43
43
|
return refuse("HUMANISH_STUDY_COMMS_UNSUPPORTED", "Real email receiving is unsupported on the preview route. Use a supported hosted computer-use study.");
|
|
44
|
-
const analysis = resolveAutomaticAnalysis(config.review?.analysis);
|
|
44
|
+
const analysis = resolveAutomaticAnalysis(config.review?.analysis, config.actor);
|
|
45
45
|
if (!analysis.ok)
|
|
46
46
|
return refuse("HUMANISH_STUDY_ANALYSIS_INVALID", analysis.message);
|
|
47
47
|
const unsupported = automaticAnalysisRouteReason(config);
|
|
@@ -7,7 +7,7 @@ import { computerUseParticipantCount } from "./routing.js";
|
|
|
7
7
|
export const STUDY_CHECK_SCHEMA = "humanish.study-check.v1";
|
|
8
8
|
export function finalize(ctx, args) {
|
|
9
9
|
const checks = [...ctx.checks, ...(args?.check ? [args.check] : []), ...(args?.checks ?? [])];
|
|
10
|
-
const analysis = automaticAnalysisBudget(ctx.config.review?.analysis, ctx.route, computerUseParticipantCount(ctx.config));
|
|
10
|
+
const analysis = automaticAnalysisBudget(ctx.config.review?.analysis, ctx.route, computerUseParticipantCount(ctx.config), ctx.config.actor);
|
|
11
11
|
return {
|
|
12
12
|
schema: STUDY_CHECK_SCHEMA,
|
|
13
13
|
...(analysis ? { analysis } : {}),
|
|
@@ -19,7 +19,8 @@ export type SetupTarget = {
|
|
|
19
19
|
readonly participant: ParticipantRef;
|
|
20
20
|
};
|
|
21
21
|
/**
|
|
22
|
-
* What a run reports while it runs. `
|
|
22
|
+
* What a run reports while it runs. `run-started` comes from every route that records a run, once
|
|
23
|
+
* its run id exists. `plan` comes from computer use only; the other routes run from
|
|
23
24
|
* the study plan but do not emit it. `subject-phase` comes from computer use (participant target)
|
|
24
25
|
* and shared world (subject target).
|
|
25
26
|
*/
|
|
@@ -40,6 +41,11 @@ export type StudyEvent = {
|
|
|
40
41
|
at: string;
|
|
41
42
|
ok?: boolean;
|
|
42
43
|
durationMs?: number;
|
|
44
|
+
}
|
|
45
|
+
/** The run's directory and status.json exist: `runId` names it from here on. */
|
|
46
|
+
| {
|
|
47
|
+
type: "run-started";
|
|
48
|
+
runId: string;
|
|
43
49
|
} | {
|
|
44
50
|
type: "analysis-started";
|
|
45
51
|
} | {
|
package/dist/study/summary.js
CHANGED
|
@@ -123,7 +123,7 @@ export async function readStudySummary(cwd, study, options = {}) {
|
|
|
123
123
|
missingKeys = missing;
|
|
124
124
|
}
|
|
125
125
|
// Computed once: a test-then-use pair reads as though the two calls could differ.
|
|
126
|
-
const analysis = automaticAnalysisBudget(inspected.config.review?.analysis, route, computerUseParticipantCount(inspected.config));
|
|
126
|
+
const analysis = automaticAnalysisBudget(inspected.config.review?.analysis, route, computerUseParticipantCount(inspected.config), inspected.config.actor);
|
|
127
127
|
const subject = subjectOf(config);
|
|
128
128
|
const participants = participantsOf(inspected.config);
|
|
129
129
|
const caps = summaryCapsOf(inspected.config);
|
package/dist/study/warnings.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { cli } from "../cli/invocation.js";
|
|
2
2
|
import { isComputerUseComposition, isScriptedBrowserComposition, isSharedWorldComposition, isTerminalProductComposition, } from "./routing.js";
|
|
3
3
|
import { DESKTOP_FILES_ROUTE_REASON } from "./desktop-files.js";
|
|
4
|
+
import { isLocalBrowserStudy } from "../substrates/local/runtime-config.js";
|
|
4
5
|
import { computerUseParticipants, declaredParticipantIds, sharedWorldParticipants, } from "./plan-participants.js";
|
|
5
6
|
import { addressedRecipients } from "./parse/comms.js";
|
|
6
7
|
import { participantList, participantInstruction, declaredParticipantCount, } from "./study-fields.js";
|
|
@@ -198,8 +199,10 @@ const CONFIG_ROWS = [
|
|
|
198
199
|
},
|
|
199
200
|
{
|
|
200
201
|
field: "execution.desktop.fidelity",
|
|
201
|
-
reason: "mobile emulation is applied only to computer-use participants in
|
|
202
|
-
applies: (config, routes) => !routes.hostedCuaBrowser &&
|
|
202
|
+
reason: "mobile emulation is applied only to computer-use participants in Chromium on a hosted desktop or execution.target: local",
|
|
203
|
+
applies: (config, routes) => !routes.hostedCuaBrowser &&
|
|
204
|
+
!isLocalBrowserStudy(config) &&
|
|
205
|
+
config.execution?.desktop?.fidelity !== undefined,
|
|
203
206
|
},
|
|
204
207
|
{
|
|
205
208
|
field: "execution.desktop.sandboxTimeoutMs",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type ChromeNetworkCaptureRequest } from "./cdp-network-capture.js";
|
|
2
|
+
import type { MobileEmulationRequest } from "../../study/mobile-emulation.js";
|
|
2
3
|
/** Which target to attribute when the endpoint lists several pages. */
|
|
3
4
|
export type ChromeCdpPagePreference = "pinned" | "active";
|
|
4
5
|
export interface ChromeCdpProbeArgs {
|
|
@@ -37,12 +38,7 @@ export interface ChromeCdpProbeArgs {
|
|
|
37
38
|
network?: ChromeNetworkCaptureRequest;
|
|
38
39
|
}
|
|
39
40
|
/** Mobile emulation request: the CDP Emulation domain applied to one page target. */
|
|
40
|
-
export interface ChromeMobileEmulationRequest {
|
|
41
|
-
width: number;
|
|
42
|
-
height: number;
|
|
43
|
-
deviceScaleFactor: number;
|
|
44
|
-
touch: boolean;
|
|
45
|
-
userAgent: string;
|
|
41
|
+
export interface ChromeMobileEmulationRequest extends MobileEmulationRequest {
|
|
46
42
|
platform?: string;
|
|
47
43
|
}
|
|
48
44
|
/** What the page reports after emulation: the proof, never copied from the request. */
|
|
@@ -5,6 +5,7 @@ import { failureTail } from "../../evidence/redaction.js";
|
|
|
5
5
|
import { isHttpUrl } from "../../study/parse/subject.js";
|
|
6
6
|
import { runDetachedStep } from "../detached.js";
|
|
7
7
|
import { shellQuote } from "../shell.js";
|
|
8
|
+
import { externalLinkHandlerSetup } from "./desktop-handoff.js";
|
|
8
9
|
import { e2bShell } from "./shell.js";
|
|
9
10
|
// Settle after opening the browser or the terminal, before the first screenshot: long enough for
|
|
10
11
|
// a cold browser and page load to paint (2 s captured a blank desktop; the render needs 6-9 s).
|
|
@@ -210,7 +211,8 @@ extraChromiumFlags = [], environment) {
|
|
|
210
211
|
`target_url=${shellQuote(targetUrl)}`,
|
|
211
212
|
`browser_preference=${shellQuote(requestedBrowser)}`,
|
|
212
213
|
"profile_dir=",
|
|
213
|
-
`chrome_preferences_json=${shellQuote(chromiumEvidenceProfilePreferencesJson())}`,
|
|
214
|
+
`chrome_preferences_json=${shellQuote(chromiumEvidenceProfilePreferencesJson(targetUrl))}`,
|
|
215
|
+
externalLinkHandlerSetup(),
|
|
214
216
|
"prepare_profile() {",
|
|
215
217
|
' profile_dir="$(mktemp -d /tmp/humanish-chrome-profile.XXXXXX)"',
|
|
216
218
|
' mkdir -p "$profile_dir/Default"',
|
|
@@ -66,8 +66,6 @@ export declare function makeChromeBrowserStateObserver(desktop: E2BDesktopSandbo
|
|
|
66
66
|
text?: string;
|
|
67
67
|
scrollY?: number;
|
|
68
68
|
}>;
|
|
69
|
-
/** The user agent a mobile-emulated participant presents unless the study sets its own. */
|
|
70
|
-
export declare const DEFAULT_MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1";
|
|
71
69
|
/**
|
|
72
70
|
* Apply mobile emulation to the participant's launch page and read back what the page reports.
|
|
73
71
|
* Fails closed: a request that cannot be applied throws, because a desktop run labelled mobile is
|
|
@@ -97,8 +97,6 @@ export function makeChromeBrowserStateObserver(desktop, requestTimeoutMs, endpoi
|
|
|
97
97
|
};
|
|
98
98
|
};
|
|
99
99
|
}
|
|
100
|
-
/** The user agent a mobile-emulated participant presents unless the study sets its own. */
|
|
101
|
-
export const DEFAULT_MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1";
|
|
102
100
|
/**
|
|
103
101
|
* Apply mobile emulation to the participant's launch page and read back what the page reports.
|
|
104
102
|
* Fails closed: a request that cannot be applied throws, because a desktop run labelled mobile is
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { CuaExecutor, CuaObservation } from "../../actors/computer-use/loop.js";
|
|
2
|
+
import type { ExternalHandoff } from "../../actors/computer-use/external-handoff.js";
|
|
2
3
|
/**
|
|
3
4
|
* The minimal slice of the @e2b/desktop Sandbox the executor depends on: a structural subset of
|
|
4
5
|
* the SDK class (checked against 2.4.0), so a Sandbox satisfies it with no adapter. Methods are typed to return `Promise<void> | void` (and the
|
|
@@ -65,6 +66,12 @@ export interface E2BDesktopExecutorOptions {
|
|
|
65
66
|
* never persists raw URL/title/text; it only uses them in memory to decide whether to stop.
|
|
66
67
|
*/
|
|
67
68
|
observeBrowserState?: () => Promise<Pick<CuaObservation, "url" | "title" | "text" | "scrollY">>;
|
|
69
|
+
/**
|
|
70
|
+
* Optional read of the links the browser handed to another app since the last read
|
|
71
|
+
* (desktop-handoff.ts). It runs beside the browser probe and the screenshot; a read that fails
|
|
72
|
+
* leaves its links for the next one.
|
|
73
|
+
*/
|
|
74
|
+
readExternalHandoffs?: () => Promise<readonly ExternalHandoff[]>;
|
|
68
75
|
}
|
|
69
76
|
/** The stage of a `type` action that failed. */
|
|
70
77
|
export type CuaTypePhase = "desktop-write" | "text-tempfile" | "text-command";
|
|
@@ -151,10 +151,13 @@ export function createE2BDesktopExecutor(desktop, options = {}) {
|
|
|
151
151
|
// Probe browser state before the screenshot so a deterministic stopWhen match
|
|
152
152
|
// and its persisted frame describe the same settled surface as closely as the
|
|
153
153
|
// desktop substrate allows.
|
|
154
|
+
const handoffs = options.readExternalHandoffs?.().catch(() => []);
|
|
154
155
|
const browserState = await options.observeBrowserState?.().catch(() => undefined);
|
|
155
156
|
const raw = await desktop.screenshot();
|
|
156
157
|
const screenshot = toBuffer(raw);
|
|
158
|
+
const externalHandoffs = (await handoffs) ?? [];
|
|
157
159
|
return {
|
|
160
|
+
...(externalHandoffs.length === 0 ? {} : { externalHandoffs }),
|
|
158
161
|
screenshot,
|
|
159
162
|
stateSignature: perceptualSignature(screenshot),
|
|
160
163
|
...(browserState?.url === undefined ? {} : { url: browserState.url }),
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type ExternalHandoff } from "../../actors/computer-use/external-handoff.js";
|
|
2
|
+
import { type Shell } from "../shell.js";
|
|
3
|
+
/**
|
|
4
|
+
* One stand-in, installed in `bin`, appending to `log` (a shell word). `$0` names the command it
|
|
5
|
+
* stands in for. The local guest image ships the same script
|
|
6
|
+
* (runtime/browser-guest/control/root/opt/humanish/control/bin/xdg-open).
|
|
7
|
+
*/
|
|
8
|
+
export declare function externalLinkStandIn(bin: string, log: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* Shell lines for the browser launch script, run before the browser starts: they install the
|
|
11
|
+
* stand-ins and put them first on the search path for every command the script starts after them.
|
|
12
|
+
*/
|
|
13
|
+
export declare function externalLinkHandlerSetup(dir?: string): string;
|
|
14
|
+
/**
|
|
15
|
+
* Returns a read of the links handed off since the previous read, oldest first and at most
|
|
16
|
+
* EXTERNAL_HANDOFF_LIMITS.perObservation at a time. Reads run one at a time, so a retried
|
|
17
|
+
* observation cannot return a link twice. A line still being written is left for the next read.
|
|
18
|
+
*/
|
|
19
|
+
export declare function externalHandoffReader(shell: Pick<Shell, "run">, options: {
|
|
20
|
+
requestTimeoutMs: number;
|
|
21
|
+
dir?: string;
|
|
22
|
+
}): () => Promise<ExternalHandoff[]>;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// Links the participant's browser hands to another app (mailto:, tel:, sms:) on a hosted desktop.
|
|
2
|
+
//
|
|
3
|
+
// Chrome on Linux opens such a link by running `xdg-email` (mailto) or `xdg-open` (the rest) by
|
|
4
|
+
// name, unless an XDG desktop portal answers on the session bus; the stock desktop runs none.
|
|
5
|
+
// The stock desktop has no mail, phone or messaging app, so `xdg-open` ended in XFCE's "Choose
|
|
6
|
+
// Preferred Application" dialog, which participants reported as a fault of the app under test.
|
|
7
|
+
// The browser launch puts stand-ins for both commands first on the browser's search path. A stand-in
|
|
8
|
+
// appends the link to a log and exits; a web or file link goes on to the desktop's own command.
|
|
9
|
+
// The executor reads the log on each observation, and the participant loop records each link
|
|
10
|
+
// and tells the participant what happened. Chrome's own "Open xdg-open?" prompt for tel: and
|
|
11
|
+
// the other non-mailto schemes is answered by the profile preference in
|
|
12
|
+
// src/evidence/browser-hygiene.ts; Chrome never prompts for mailto.
|
|
13
|
+
import { EXTERNAL_HANDOFF_LIMITS, externalHandoffOf, } from "../../actors/computer-use/external-handoff.js";
|
|
14
|
+
import { shellQuote } from "../shell.js";
|
|
15
|
+
/** Where the stand-ins and their log live on the desktop. */
|
|
16
|
+
const EXTERNAL_LINK_DIR = "/tmp/humanish-external-links";
|
|
17
|
+
const LINK_LENGTH = 4096;
|
|
18
|
+
/**
|
|
19
|
+
* One stand-in, installed in `bin`, appending to `log` (a shell word). `$0` names the command it
|
|
20
|
+
* stands in for. The local guest image ships the same script
|
|
21
|
+
* (runtime/browser-guest/control/root/opt/humanish/control/bin/xdg-open).
|
|
22
|
+
*/
|
|
23
|
+
export function externalLinkStandIn(bin, log) {
|
|
24
|
+
return [
|
|
25
|
+
"#!/bin/sh",
|
|
26
|
+
"# Installed by humanish for a study participant's browser: this desktop has no app for this",
|
|
27
|
+
"# link, so the link is written down for the study instead of opening a chooser dialog.",
|
|
28
|
+
`bin=${shellQuote(bin)}`,
|
|
29
|
+
"url=",
|
|
30
|
+
'for arg in "$@"; do case "$arg" in -*) ;; *) url=$arg; break ;; esac; done',
|
|
31
|
+
'case "$url" in',
|
|
32
|
+
" [Hh][Tt][Tt][Pp]:*|[Hh][Tt][Tt][Pp][Ss]:*|[Ff][Ii][Ll][Ee]:*) ;;",
|
|
33
|
+
" [A-Za-z]*:*)",
|
|
34
|
+
` printf '%s\\n' "$(printf '%s' "$url" | tr -d '\\r\\n' | cut -c 1-${LINK_LENGTH})" >> ${log}`,
|
|
35
|
+
" exit 0 ;;",
|
|
36
|
+
"esac",
|
|
37
|
+
// Anything else goes to the desktop's own command, found on the search path without this
|
|
38
|
+
// directory.
|
|
39
|
+
"set -f",
|
|
40
|
+
"real=",
|
|
41
|
+
"IFS=:",
|
|
42
|
+
'for entry in $PATH; do [ "$entry" = "$bin" ] || real=${real:+$real:}$entry; done',
|
|
43
|
+
'PATH=$real exec "${0##*/}" "$@"',
|
|
44
|
+
"",
|
|
45
|
+
].join("\n");
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Shell lines for the browser launch script, run before the browser starts: they install the
|
|
49
|
+
* stand-ins and put them first on the search path for every command the script starts after them.
|
|
50
|
+
*/
|
|
51
|
+
export function externalLinkHandlerSetup(dir = EXTERNAL_LINK_DIR) {
|
|
52
|
+
const bin = `${dir}/bin`;
|
|
53
|
+
return [
|
|
54
|
+
`mkdir -p ${shellQuote(bin)}`,
|
|
55
|
+
`touch ${shellQuote(`${dir}/links.log`)}`,
|
|
56
|
+
`printf '%s' ${shellQuote(externalLinkStandIn(bin, shellQuote(`${dir}/links.log`)))} > ${shellQuote(`${bin}/xdg-open`)}`,
|
|
57
|
+
`cp ${shellQuote(`${bin}/xdg-open`)} ${shellQuote(`${bin}/xdg-email`)}`,
|
|
58
|
+
`chmod 755 ${shellQuote(`${bin}/xdg-open`)} ${shellQuote(`${bin}/xdg-email`)}`,
|
|
59
|
+
`export PATH=${shellQuote(bin)}:"$PATH"`,
|
|
60
|
+
].join("\n");
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Returns a read of the links handed off since the previous read, oldest first and at most
|
|
64
|
+
* EXTERNAL_HANDOFF_LIMITS.perObservation at a time. Reads run one at a time, so a retried
|
|
65
|
+
* observation cannot return a link twice. A line still being written is left for the next read.
|
|
66
|
+
*/
|
|
67
|
+
export function externalHandoffReader(shell, options) {
|
|
68
|
+
const log = `${options.dir ?? EXTERNAL_LINK_DIR}/links.log`;
|
|
69
|
+
let consumed = 0;
|
|
70
|
+
let previous = Promise.resolve();
|
|
71
|
+
const read = async () => {
|
|
72
|
+
const result = await shell.run(`tail -n +${consumed + 1} ${shellQuote(log)} 2>/dev/null | head -n ${EXTERNAL_HANDOFF_LIMITS.perObservation}`, { requestTimeoutMs: options.requestTimeoutMs, timeoutMs: 10_000 });
|
|
73
|
+
const lines = result.stdout.split("\n");
|
|
74
|
+
lines.pop();
|
|
75
|
+
consumed += lines.length;
|
|
76
|
+
return lines.flatMap((line) => {
|
|
77
|
+
const handoff = externalHandoffOf(line);
|
|
78
|
+
return handoff === undefined ? [] : [handoff];
|
|
79
|
+
});
|
|
80
|
+
};
|
|
81
|
+
return () => {
|
|
82
|
+
const next = previous.then(read, read);
|
|
83
|
+
previous = next.catch(() => undefined);
|
|
84
|
+
return next;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
@@ -2,11 +2,14 @@ import type { Writable } from "node:stream";
|
|
|
2
2
|
import { type DesktopRecordingConfig, type DesktopRecordingMetadata } from "../../evidence/desktop-recording-types.js";
|
|
3
3
|
import { type DesktopSession } from "../desktop-session.js";
|
|
4
4
|
import { type GuestMediaConfig } from "../../guest/media-config.js";
|
|
5
|
+
import { type GuestDisplayConfig } from "../../guest/display-config.js";
|
|
5
6
|
/** The installed runtime image; `humanish runtime setup` chooses it, never a participant. */
|
|
6
7
|
export interface LocalFirecrackerAssets {
|
|
7
8
|
image: string;
|
|
8
9
|
runtimeRevision: string;
|
|
9
10
|
media?: boolean;
|
|
11
|
+
/** The guest starts the screen and emulation a bootstrap names; older images are 960×720 only. */
|
|
12
|
+
displaySizes?: boolean;
|
|
10
13
|
}
|
|
11
14
|
export interface LocalFirecrackerDesktop extends DesktopSession {
|
|
12
15
|
finishRecording(destination: Writable): Promise<DesktopRecordingMetadata>;
|
|
@@ -26,6 +29,8 @@ interface LocalFirecrackerOptions {
|
|
|
26
29
|
signal?: AbortSignal;
|
|
27
30
|
media?: GuestMediaConfig;
|
|
28
31
|
recording?: DesktopRecordingConfig;
|
|
32
|
+
/** The participant's screen and mobile emulation; absent is the guest's 960×720 default. */
|
|
33
|
+
display?: GuestDisplayConfig;
|
|
29
34
|
inboxUrl?: string;
|
|
30
35
|
}
|
|
31
36
|
/** One VM with opaque TCP forwards to the selected app and optional scoped inbox. */
|
|
@@ -9,6 +9,7 @@ import { ownDesktopAllocation, } from "../desktop-session.js";
|
|
|
9
9
|
import { runtimeDocker, runtimeExec, usesLima } from "./runtime-host.js";
|
|
10
10
|
import { openLimaTunnel } from "./runtime-ssh.js";
|
|
11
11
|
import { guestMediaConfigSchema } from "../../guest/media-config.js";
|
|
12
|
+
import { guestDisplayConfigSchema } from "../../guest/display-config.js";
|
|
12
13
|
import { cli } from "../../cli/invocation.js";
|
|
13
14
|
import { DESKTOP_RESERVATION } from "./capacity.js";
|
|
14
15
|
const docker = async (args) => (await runtimeDocker(args, {}, 60_000)).stdout.trim();
|
|
@@ -55,6 +56,11 @@ function checkDesktopOptions(options) {
|
|
|
55
56
|
if (options.assets.media !== true)
|
|
56
57
|
throw new Error(`This runtime does not include the recorder. Run ${cli("runtime setup --media")}.`);
|
|
57
58
|
}
|
|
59
|
+
if (options.display !== undefined) {
|
|
60
|
+
const { width, height } = guestDisplayConfigSchema.parse(options.display);
|
|
61
|
+
if (options.assets.displaySizes !== true)
|
|
62
|
+
throw new Error(`This local runtime starts every participant at 960×720, so it cannot start the ${width}×${height} screen this participant needs. Omit device and resolution, use a hosted desktop, or set HUMANISH_LOCAL_RUNTIME_IMAGE to a runtime built from current source.`);
|
|
63
|
+
}
|
|
58
64
|
let url;
|
|
59
65
|
try {
|
|
60
66
|
url = new URL(validateGuestInitialUrl(options.appUrl));
|
|
@@ -261,7 +267,7 @@ async function bootstrapGuest(stream, options, url, signal) {
|
|
|
261
267
|
runtimeRevision: options.assets.runtimeRevision,
|
|
262
268
|
};
|
|
263
269
|
try {
|
|
264
|
-
return await connectGuestBootstrap(stream, identity, signal, url.href, options.media, options.recording);
|
|
270
|
+
return await connectGuestBootstrap(stream, identity, signal, url.href, options.media, options.recording, options.display);
|
|
265
271
|
}
|
|
266
272
|
catch (error) {
|
|
267
273
|
signal.throwIfAborted();
|
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
import type { StudyConfig } from "../../study/types.js";
|
|
2
|
+
import { type DevicePreset } from "../../study/device-presets.js";
|
|
3
|
+
import { type GuestDisplayConfig } from "../../guest/display-config.js";
|
|
2
4
|
export declare const LOCAL_BROWSER_LIFETIME_MS: number;
|
|
3
5
|
export declare function isLocalBrowserStudy(config: StudyConfig): boolean;
|
|
4
6
|
/** Defaults for the explicitly selected local substrate; hosted configurations are untouched. */
|
|
5
7
|
export declare function localBrowserDefaults(config: StudyConfig): StudyConfig;
|
|
6
8
|
export declare function localBrowserUnsupportedReason(config: StudyConfig): string | undefined;
|
|
9
|
+
/**
|
|
10
|
+
* The screen a local participant's guest starts, with mobile emulation where hosted desktops apply
|
|
11
|
+
* it. Undefined is the guest's 960×720 default, which every installed runtime supports.
|
|
12
|
+
*/
|
|
13
|
+
export declare function localGuestDisplay(config: StudyConfig, device: {
|
|
14
|
+
preset: DevicePreset;
|
|
15
|
+
resolution: readonly [number, number];
|
|
16
|
+
}): GuestDisplayConfig | undefined;
|
|
17
|
+
/** Whether any participant of a local study needs a screen other than the guest's default. */
|
|
18
|
+
export declare function localStudySetsScreens(config: StudyConfig): boolean;
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { participantList } from "../../study/study-fields.js";
|
|
2
|
+
import { DEVICE_PRESET_NAMES, resolveParticipantDevice, } from "../../study/device-presets.js";
|
|
3
|
+
import { participantMobileEmulation } from "../../study/mobile-emulation.js";
|
|
4
|
+
import { DEFAULT_GUEST_SCREEN, GUEST_SCREEN_LIMITS, } from "../../guest/display-config.js";
|
|
2
5
|
// The runtime image enforces a 30-minute lifetime; reserve setup/teardown room.
|
|
3
6
|
export const LOCAL_BROWSER_LIFETIME_MS = 30 * 60_000;
|
|
4
7
|
const MAX_SESSION_MS = 20 * 60_000;
|
|
@@ -24,12 +27,18 @@ export function localBrowserDefaults(config) {
|
|
|
24
27
|
timeoutMs: config.execution?.timeoutMs ?? MAX_SESSION_MS,
|
|
25
28
|
desktop: {
|
|
26
29
|
...config.execution?.desktop,
|
|
27
|
-
|
|
30
|
+
// A named device governs the screen when the study sets one; resolution would override it.
|
|
31
|
+
...(config.execution?.desktop?.device !== undefined ||
|
|
32
|
+
participantList(config)?.some((entry) => entry.device !== undefined)
|
|
33
|
+
? {}
|
|
34
|
+
: {
|
|
35
|
+
resolution: config.execution?.desktop?.resolution ?? [
|
|
36
|
+
DEFAULT_GUEST_SCREEN.width,
|
|
37
|
+
DEFAULT_GUEST_SCREEN.height,
|
|
38
|
+
],
|
|
39
|
+
}),
|
|
28
40
|
},
|
|
29
41
|
},
|
|
30
|
-
...(config.review?.analysis === undefined && actor.type === "local-agent"
|
|
31
|
-
? { review: { ...config.review, analysis: { provider: "codex" } } }
|
|
32
|
-
: {}),
|
|
33
42
|
};
|
|
34
43
|
}
|
|
35
44
|
export function localBrowserUnsupportedReason(config) {
|
|
@@ -42,20 +51,19 @@ export function localBrowserUnsupportedReason(config) {
|
|
|
42
51
|
(actor.type === "local-agent" && actor.localAgent !== undefined && actor.localAgent !== "codex")) {
|
|
43
52
|
return "Local browser studies support only Codex local-agent or openai-computer-use participants.";
|
|
44
53
|
}
|
|
45
|
-
if (desktop?.
|
|
46
|
-
desktop
|
|
47
|
-
desktop
|
|
48
|
-
|
|
49
|
-
participantList(config)?.some((entry) => entry.device !== undefined) ||
|
|
50
|
-
desktop.template !== undefined ||
|
|
51
|
-
desktop.sandboxTimeoutMs !== undefined) {
|
|
52
|
-
return "The local browser runtime currently uses a 960×720 Chromium desktop. Omit hosted templates, device presets and sandboxTimeoutMs.";
|
|
54
|
+
if ((desktop?.browser !== undefined && !["default", "chromium"].includes(desktop.browser)) ||
|
|
55
|
+
desktop?.template !== undefined ||
|
|
56
|
+
desktop?.sandboxTimeoutMs !== undefined) {
|
|
57
|
+
return "The local browser runtime runs Chromium and has no hosted templates or sandboxTimeoutMs. Remove execution.desktop.browser, template and sandboxTimeoutMs, or use a hosted desktop.";
|
|
53
58
|
}
|
|
59
|
+
const screenReason = localScreenReason(config);
|
|
60
|
+
if (screenReason)
|
|
61
|
+
return screenReason;
|
|
54
62
|
if (config.comms !== undefined &&
|
|
55
63
|
(config.comms.email?.kind !== "fake" || !config.comms.email.external)) {
|
|
56
64
|
return "Local browser inboxes require captured email with comms.email.external.catchBaseUrl. Run humanish comms catch and point your app's email sends at it. Real receiving requires a supported hosted route.";
|
|
57
65
|
}
|
|
58
|
-
if (desktop
|
|
66
|
+
if (desktop?.media?.camera !== undefined && desktop.media.camera.source !== "synthetic") {
|
|
59
67
|
return "Local cameras currently use source: synthetic. Camera files remain supported on hosted desktops.";
|
|
60
68
|
}
|
|
61
69
|
if (actor.type === "local-agent" &&
|
|
@@ -87,3 +95,37 @@ export function localBrowserUnsupportedReason(config) {
|
|
|
87
95
|
}
|
|
88
96
|
return undefined;
|
|
89
97
|
}
|
|
98
|
+
/** Why a participant's screen does not fit the local guest, or undefined. */
|
|
99
|
+
function localScreenReason(config) {
|
|
100
|
+
const limits = GUEST_SCREEN_LIMITS;
|
|
101
|
+
for (const entry of participantList(config) ?? [{}]) {
|
|
102
|
+
const [width, height] = resolveParticipantDevice(config, entry.device).resolution;
|
|
103
|
+
if (width < limits.minWidth ||
|
|
104
|
+
width > limits.maxWidth ||
|
|
105
|
+
height < limits.minHeight ||
|
|
106
|
+
height > limits.maxHeight)
|
|
107
|
+
return `Local browser desktops support screens up to ${limits.maxWidth}×${limits.maxHeight} and at least ${limits.minHeight} pixels high; this study asks for ${width}×${height}. Use a device preset (${DEVICE_PRESET_NAMES.join(", ")}), a resolution within those bounds, or a hosted desktop.`;
|
|
108
|
+
}
|
|
109
|
+
const userAgent = config.execution?.desktop?.fidelity?.userAgent;
|
|
110
|
+
if (userAgent !== undefined && !/^[\x20-\x7e]{1,512}$/.test(userAgent))
|
|
111
|
+
return "Local browser desktops take an execution.desktop.fidelity.userAgent of at most 512 printable ASCII characters. Shorten it or remove non-ASCII characters.";
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The screen a local participant's guest starts, with mobile emulation where hosted desktops apply
|
|
116
|
+
* it. Undefined is the guest's 960×720 default, which every installed runtime supports.
|
|
117
|
+
*/
|
|
118
|
+
export function localGuestDisplay(config, device) {
|
|
119
|
+
const mobileEmulation = participantMobileEmulation(config.execution?.desktop?.fidelity, device.preset);
|
|
120
|
+
const [width, height] = device.resolution;
|
|
121
|
+
if (mobileEmulation === undefined &&
|
|
122
|
+
width === DEFAULT_GUEST_SCREEN.width &&
|
|
123
|
+
height === DEFAULT_GUEST_SCREEN.height)
|
|
124
|
+
return undefined;
|
|
125
|
+
return { width, height, ...(mobileEmulation === undefined ? {} : { mobileEmulation }) };
|
|
126
|
+
}
|
|
127
|
+
/** Whether any participant of a local study needs a screen other than the guest's default. */
|
|
128
|
+
export function localStudySetsScreens(config) {
|
|
129
|
+
const study = localBrowserDefaults(config);
|
|
130
|
+
return (participantList(study) ?? [{}]).some((entry) => localGuestDisplay(study, resolveParticipantDevice(study, entry.device)) !== undefined);
|
|
131
|
+
}
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
/** Updated with the verified, public runtime artifact before release. */
|
|
2
2
|
const LOCAL_RUNTIME_RELEASE = {
|
|
3
|
-
url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.
|
|
4
|
-
sha256: "
|
|
5
|
-
bytes:
|
|
6
|
-
image: "sha256:
|
|
3
|
+
url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.10.21.3/runtime-linux-amd64.tar.gz",
|
|
4
|
+
sha256: "6fc19792e9b8f414165a73470526cc414c98d0ad368210c5e95daf29469b8dbd",
|
|
5
|
+
bytes: 597049520,
|
|
6
|
+
image: "sha256:4672388737bd7d136910fca6f8b6d99e09fbe96c13e1b100491c05444e7d267a",
|
|
7
7
|
};
|
|
8
8
|
/** Architecture-specific artifacts; each retains matching sources and notices. */
|
|
9
9
|
export const LOCAL_RUNTIME_RELEASES = {
|
|
10
10
|
amd64: LOCAL_RUNTIME_RELEASE,
|
|
11
11
|
arm64: {
|
|
12
|
-
url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.
|
|
13
|
-
sha256: "
|
|
14
|
-
bytes:
|
|
15
|
-
image: "sha256:
|
|
12
|
+
url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.10.21.4/runtime-linux-arm64.tar.gz",
|
|
13
|
+
sha256: "d590de49a86eabe1cd09d7556aa75b2ebd62bc2c88fe37122e2af315c478c901",
|
|
14
|
+
bytes: 583551803,
|
|
15
|
+
image: "sha256:4606122520504f7f587f52b872165e6ab900be0ba3eca933b2f6dd3b9dcde723",
|
|
16
16
|
},
|
|
17
17
|
};
|
|
18
18
|
/** Media assets are separate so ordinary browser studies keep the smaller download. */
|
|
@@ -164,6 +164,7 @@ async function engineStatus(options, architecture, lima) {
|
|
|
164
164
|
image: image.Id,
|
|
165
165
|
runtimeRevision: revision,
|
|
166
166
|
...(labels["to.humanish.runtime.media"] === "1" ? { media: true } : {}),
|
|
167
|
+
...(labels["to.humanish.runtime.display-sizes"] === "1" ? { displaySizes: true } : {}),
|
|
167
168
|
},
|
|
168
169
|
};
|
|
169
170
|
}
|