humanish 0.111.2 → 0.111.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +32 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/cli/findings.d.ts +15 -1
- package/dist/cli/findings.js +43 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +14 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +71 -35
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +8 -5
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +35 -0
- package/dist/run/secrets.js +158 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +89 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/docs/contracts/study-analysis.md +6 -2
- package/package.json +1 -1
|
@@ -6,6 +6,7 @@ import type { RunBundle } from "../../run/bundle.js";
|
|
|
6
6
|
import type { RunScope } from "../../run/run.js";
|
|
7
7
|
import { type ExecutionFailure } from "../../run/judge.js";
|
|
8
8
|
import type { ScriptedSubject } from "./subject.js";
|
|
9
|
+
import type { E2BSubjectSandbox } from "../../substrates/e2b/subject-sandbox.js";
|
|
9
10
|
import { type ScriptedBrowserStudyResult } from "./types.js";
|
|
10
11
|
/** What the finish reads from the run. */
|
|
11
12
|
export interface ScriptedFinishInputs {
|
|
@@ -44,6 +45,6 @@ export declare function scriptedExecutionFailures(args: {
|
|
|
44
45
|
sessionError: string | undefined;
|
|
45
46
|
expected: number;
|
|
46
47
|
sessionResults: readonly Pick<ScriptedBrowserSessionResult, "completionReason" | "reason">[];
|
|
47
|
-
subject: Pick<
|
|
48
|
+
subject: Pick<E2BSubjectSandbox, "sandboxId" | "killed" | "releaseWarning"> | undefined;
|
|
48
49
|
}): ExecutionFailure[];
|
|
49
50
|
export declare function finishScriptedRun(inputs: ScriptedFinishInputs): Promise<ScriptedBrowserStudyResult>;
|
|
@@ -53,11 +53,11 @@ export async function finishScriptedRun(inputs) {
|
|
|
53
53
|
const { actor, dryRun } = plan;
|
|
54
54
|
const { runId, paths: runPaths } = run;
|
|
55
55
|
const subjectCommit = scriptedSubject?.commit;
|
|
56
|
-
const subjectSandboxId = scriptedSubject?.sandboxId;
|
|
57
|
-
const subjectKilled = scriptedSubject?.killed ?? false;
|
|
56
|
+
const subjectSandboxId = scriptedSubject?.sandbox.sandboxId;
|
|
57
|
+
const subjectKilled = scriptedSubject?.sandbox.killed ?? false;
|
|
58
58
|
const hostDigest = scriptedSubject?.hostDigest;
|
|
59
59
|
const stateStepRecords = scriptedSubject?.stateStepRecords ?? [];
|
|
60
|
-
const subjectDesktop = scriptedSubject?.desktopUsage();
|
|
60
|
+
const subjectDesktop = scriptedSubject?.sandbox.desktopUsage();
|
|
61
61
|
const screenshotsBySurface = new Map();
|
|
62
62
|
for (const result of sessionResults) {
|
|
63
63
|
screenshotsBySurface.set(result.capture.surface.id, await existingScreenshots(runPaths, result));
|
|
@@ -114,7 +114,7 @@ export async function finishScriptedRun(inputs) {
|
|
|
114
114
|
sessionError,
|
|
115
115
|
expected: surfaces.length,
|
|
116
116
|
sessionResults,
|
|
117
|
-
subject: scriptedSubject,
|
|
117
|
+
subject: scriptedSubject?.sandbox,
|
|
118
118
|
}), policy);
|
|
119
119
|
const finished = await run.finish(bundle, {
|
|
120
120
|
ok: resultOk({ judgment, execution, scorerFailures: [], policy }),
|
|
@@ -14,11 +14,9 @@
|
|
|
14
14
|
// subject, so its build provenance is unpinned and the evidence binds to the scenario digest. A
|
|
15
15
|
// clone run records commit, env names, state provenance and a host digest, never the raw getHost
|
|
16
16
|
// URL or a secret value.
|
|
17
|
-
import { randomBytes } from "node:crypto";
|
|
18
17
|
import { realpath } from "node:fs/promises";
|
|
19
18
|
import path from "node:path";
|
|
20
19
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
21
|
-
import { buildRunSource } from "../../run/bundle.js";
|
|
22
20
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
23
21
|
import {} from "../../run/run.js";
|
|
24
22
|
import { admitRoute, completeRefusalAnalysis } from "../../run/route-shell.js";
|
|
@@ -30,7 +28,7 @@ import { prepareScriptedRun } from "./setup.js";
|
|
|
30
28
|
import { ScriptedSubject } from "./subject.js";
|
|
31
29
|
import { runScriptedSessions, writeSurfaceTraces } from "./surface-sessions.js";
|
|
32
30
|
import {} from "./types.js";
|
|
33
|
-
import {
|
|
31
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
34
32
|
/**
|
|
35
33
|
* A refused scripted study's result, at the refusal's stage: a before-scope refusal has its own field
|
|
36
34
|
* order and no analysis record; the others come after the output directory checks.
|
|
@@ -39,34 +37,25 @@ export async function scriptedStudyRefusal(options, refusal) {
|
|
|
39
37
|
const { config, dryRun } = options;
|
|
40
38
|
const cwd = path.resolve(options.cwd);
|
|
41
39
|
const actorType = config.actor?.type ?? "";
|
|
40
|
+
const error = { code: refusal.code, message: refusal.message };
|
|
42
41
|
if (refusal.beforeScope)
|
|
43
|
-
return {
|
|
44
|
-
...studyResultIdentity("scripted", config.id),
|
|
45
|
-
ok: false,
|
|
46
|
-
cwd,
|
|
42
|
+
return refusedResult("scripted", { studyId: config.id, cwd, error }, {
|
|
47
43
|
actor: actorType,
|
|
48
44
|
dryRun,
|
|
49
45
|
runId: options.runId ?? "not-created",
|
|
50
46
|
appUrl: config.subject.appUrl ?? "",
|
|
51
47
|
sessions: [],
|
|
52
|
-
|
|
53
|
-
error: { code: refusal.code, message: refusal.message },
|
|
54
|
-
};
|
|
48
|
+
});
|
|
55
49
|
// The other refusals come after the output directory checks and carry the analysis record.
|
|
56
50
|
const physicalCwd = await realpath(cwd);
|
|
57
51
|
await prepareSelectedOutputDirectory(path.dirname(physicalCwd), physicalCwd);
|
|
58
|
-
const refused = {
|
|
59
|
-
...studyResultIdentity("scripted", config.id),
|
|
60
|
-
ok: false,
|
|
61
|
-
cwd,
|
|
52
|
+
const refused = refusedResult("scripted", { studyId: config.id, cwd, error }, {
|
|
62
53
|
actor: refusal.actor ?? actorType,
|
|
63
54
|
appUrl: refusal.appUrl ?? config.subject.appUrl ?? "",
|
|
64
55
|
dryRun,
|
|
65
56
|
runId: options.runId ?? "not-created",
|
|
66
57
|
sessions: [],
|
|
67
|
-
|
|
68
|
-
error: { code: refusal.code, message: refusal.message },
|
|
69
|
-
};
|
|
58
|
+
});
|
|
70
59
|
return completeRefusalAnalysis(refused, config, options);
|
|
71
60
|
}
|
|
72
61
|
/**
|
|
@@ -88,30 +77,20 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
88
77
|
return prepared.result;
|
|
89
78
|
const { dryRun } = plan;
|
|
90
79
|
const { setup } = prepared;
|
|
91
|
-
const started = await scope.startRun({
|
|
80
|
+
const started = await scope.startRun(plan, input, {
|
|
92
81
|
cwd: setup.physicalCwd,
|
|
93
|
-
|
|
94
|
-
mintRunId: makeScriptedRunId,
|
|
95
|
-
mode: dryRun ? "dry-run" : "live",
|
|
96
|
-
study: plan.study,
|
|
97
|
-
warnings: plan.warnings,
|
|
82
|
+
prefix: "scripted",
|
|
98
83
|
// Only a clone subject is served from a sandbox; an app-url run drives a local browser.
|
|
99
84
|
sandboxes: setup.clone ? undefined : "none",
|
|
100
85
|
renderReview: renderScriptedReviewMarkdown,
|
|
101
|
-
|
|
86
|
+
secrets: setup.secrets,
|
|
102
87
|
});
|
|
103
88
|
if (!started.ok) {
|
|
104
89
|
return setup.failed(started.code, started.message);
|
|
105
90
|
}
|
|
106
91
|
const { run } = started;
|
|
107
|
-
const {
|
|
92
|
+
const { paths: runPaths, source } = run;
|
|
108
93
|
const artifactRoot = runPaths.physicalRunRoot;
|
|
109
|
-
const source = await buildRunSource({
|
|
110
|
-
capturedAt: createdAt,
|
|
111
|
-
cwd: setup.physicalCwd,
|
|
112
|
-
humanishSource: "present",
|
|
113
|
-
packageName: "humanish",
|
|
114
|
-
});
|
|
115
94
|
const scriptedSubject = setup.clone
|
|
116
95
|
? new ScriptedSubject({
|
|
117
96
|
plan,
|
|
@@ -124,7 +103,7 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
124
103
|
timeoutMs: setup.timeoutMs,
|
|
125
104
|
subjectEnvNames: setup.subjectEnvNames,
|
|
126
105
|
hasGithubToken: setup.hasGithubToken,
|
|
127
|
-
scrubKnownValues:
|
|
106
|
+
scrubKnownValues: run.secrets.scrub,
|
|
128
107
|
now: setup.deps.now ?? Date.now,
|
|
129
108
|
warnings: setup.warnings,
|
|
130
109
|
})
|
|
@@ -142,6 +121,7 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
142
121
|
persona: setup.persona,
|
|
143
122
|
timeoutMs: setup.timeoutMs,
|
|
144
123
|
artifactRoot,
|
|
124
|
+
scrubKnownValues: run.secrets.scrub,
|
|
145
125
|
}, {
|
|
146
126
|
surfaces: setup.surfaces,
|
|
147
127
|
deps: setup.deps,
|
|
@@ -156,10 +136,10 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
156
136
|
}
|
|
157
137
|
// The session itself maps launch failures to harness_error; reaching here means the
|
|
158
138
|
// harness around it failed. Redacted at this boundary before persisting anywhere.
|
|
159
|
-
sessionError = redactText(
|
|
139
|
+
sessionError = redactText(run.secrets.scrub(toErrorMessage(error)));
|
|
160
140
|
}
|
|
161
141
|
finally {
|
|
162
|
-
await scriptedSubject?.
|
|
142
|
+
await scriptedSubject?.sandbox.release();
|
|
163
143
|
}
|
|
164
144
|
await writeSurfaceTraces(runPaths, sessionResults);
|
|
165
145
|
}
|
|
@@ -182,7 +162,3 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
182
162
|
scriptedSubject,
|
|
183
163
|
});
|
|
184
164
|
}
|
|
185
|
-
function makeScriptedRunId() {
|
|
186
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
187
|
-
return `scripted-${stamp}-${randomBytes(4).toString("hex")}`;
|
|
188
|
-
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ActorPersonaRef } from "../../actors/contract.js";
|
|
2
2
|
import type { BrowserPersonaJourney, ScriptedBrowserEvidenceUrlPolicy } from "../../actors/scripted-browser/types.js";
|
|
3
3
|
import type { ScriptedPlan } from "../../study/plan-types.js";
|
|
4
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
4
5
|
import { type ScriptedBrowserStudyResult, type ScriptedRunInput } from "./types.js";
|
|
5
6
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
6
7
|
/** What setup hands the run: the plan's subject and scenario, keys and the browser to launch. */
|
|
@@ -20,7 +21,7 @@ export interface ScriptedRunSetup {
|
|
|
20
21
|
e2bApiKey: string;
|
|
21
22
|
hasGithubToken: boolean;
|
|
22
23
|
redactRepoLabel: boolean;
|
|
23
|
-
|
|
24
|
+
secrets: RunSecrets;
|
|
24
25
|
/** The loopback app URL, or the clone's serve URL until its getHost URL replaces it. */
|
|
25
26
|
appUrl: string;
|
|
26
27
|
scenario: {
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
// What a scripted run needs before it starts: the checks the plan cannot make (the scenario file,
|
|
2
2
|
// the E2B key and subject env of a live clone, a browser to launch) and the values the run reads
|
|
3
|
-
// (the evidence URL policy, the
|
|
3
|
+
// (the evidence URL policy, the run's known secrets, the persona and the session budget).
|
|
4
4
|
import { realpath } from "node:fs/promises";
|
|
5
|
-
import {
|
|
5
|
+
import { firstLiveRefusal, keysCheck, subjectEnvCheck } from "../../study/requirements.js";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-command.js";
|
|
8
|
-
import {
|
|
8
|
+
import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
|
|
9
9
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
10
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
10
11
|
import { evidenceAppUrlOf } from "./plan.js";
|
|
11
12
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
12
13
|
import {} from "./types.js";
|
|
13
|
-
import {
|
|
14
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
14
15
|
// Journey wall-clock budget per surface: 5 minutes. A scripted surface has zero model cost and
|
|
15
16
|
// sandbox-seconds are pennies; a short default only truncated slow-loading subjects.
|
|
16
17
|
const DEFAULT_SESSION_TIMEOUT_MS = 300_000;
|
|
@@ -26,32 +27,28 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
26
27
|
const deps = input.deps ?? {};
|
|
27
28
|
const warnings = [];
|
|
28
29
|
const clone = plan.subject.kind === "clone" ? plan.subject : undefined;
|
|
29
|
-
const
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
const subjectEnvNames = [...(clone?.env ?? [])];
|
|
31
|
+
const env = input.env ?? process.env;
|
|
32
|
+
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
33
|
+
const secrets = new RunSecrets([
|
|
34
|
+
e2bApiKey,
|
|
35
|
+
...(clone ? [clone.repo] : []),
|
|
36
|
+
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
37
|
+
]);
|
|
38
|
+
// An app-url run records the URL it was given, in the bundle and every summary.
|
|
39
|
+
const evidenceAppUrl = secrets.scrub(evidenceAppUrlOf(plan.subject));
|
|
40
|
+
const failed = (code, message) => refusedResult("scripted", { studyId: plan.studyId, cwd, warnings, error: { code, message } }, {
|
|
34
41
|
actor: plan.actor,
|
|
35
42
|
appUrl: evidenceAppUrl,
|
|
36
43
|
dryRun,
|
|
37
44
|
runId: input.runId ?? "not-created",
|
|
38
45
|
sessions: [],
|
|
39
|
-
warnings,
|
|
40
|
-
error: { code, message },
|
|
41
46
|
});
|
|
42
47
|
const urlPolicy = clone
|
|
43
48
|
? { kind: "provisioned-subject", evidenceOrigin: evidenceAppUrl }
|
|
44
49
|
: { kind: "loopback" };
|
|
45
|
-
const subjectEnvNames = [...(clone?.env ?? [])];
|
|
46
|
-
const env = input.env ?? process.env;
|
|
47
|
-
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
48
50
|
const hasGithubToken = subjectEnvNames.includes("GITHUB_TOKEN");
|
|
49
51
|
const redactRepoLabel = plan.residual.policies?.redactRepos ?? hasGithubToken;
|
|
50
|
-
const scrubSourceValues = [
|
|
51
|
-
...(clone ? [clone.repo] : []),
|
|
52
|
-
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
53
|
-
].filter(Boolean);
|
|
54
|
-
const scrubKnownValues = (text) => scrubSourceValues.reduce((acc, value) => acc.split(value).join("[redacted]"), text);
|
|
55
52
|
// A clone's URL is replaced by its getHost URL once it is served.
|
|
56
53
|
let appUrl = plan.subject.kind === "clone" ? plan.subject.serve.url : plan.subject.appUrl;
|
|
57
54
|
const scenario = await resolveScriptedScenario(projectRoot, plan.scenarioRef);
|
|
@@ -61,22 +58,30 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
61
58
|
result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
|
|
62
59
|
};
|
|
63
60
|
}
|
|
64
|
-
|
|
61
|
+
// The goal, title and step labels are recorded; a step's path, selector, value and expected text
|
|
62
|
+
// drive the browser and stay as written.
|
|
63
|
+
const journey = {
|
|
64
|
+
...scenario.journey,
|
|
65
|
+
scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
|
|
66
|
+
? "scenario"
|
|
67
|
+
: scenario.journey.scenarioId,
|
|
68
|
+
goal: secrets.scrub(scenario.journey.goal),
|
|
69
|
+
scenarioTitle: secrets.scrub(scenario.journey.scenarioTitle),
|
|
70
|
+
steps: recordedStepIds(scenario.journey.steps, secrets),
|
|
71
|
+
};
|
|
65
72
|
// The plan lists E2B_API_KEY and the subject env only for a live clone.
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
};
|
|
79
|
-
}
|
|
73
|
+
const { requirements } = plan;
|
|
74
|
+
const refusal = await firstLiveRefusal([
|
|
75
|
+
() => keysCheck({
|
|
76
|
+
requirements,
|
|
77
|
+
env,
|
|
78
|
+
code: "HUMANISH_SCRIPTED_KEYS_MISSING",
|
|
79
|
+
need: (names) => `Live clone scripted-browser studies require ${names} (dry-run remains $0 and does not provision a subject).`,
|
|
80
|
+
}),
|
|
81
|
+
() => subjectEnvCheck({ requirements, env, code: "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" }),
|
|
82
|
+
]);
|
|
83
|
+
if (refusal)
|
|
84
|
+
return { ok: false, result: failed(refusal.code, refusal.message) };
|
|
80
85
|
const surfaces = plan.surfaces;
|
|
81
86
|
const timeoutMs = plan.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
|
|
82
87
|
const persona = {
|
|
@@ -114,7 +119,7 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
114
119
|
e2bApiKey,
|
|
115
120
|
hasGithubToken,
|
|
116
121
|
redactRepoLabel,
|
|
117
|
-
|
|
122
|
+
secrets,
|
|
118
123
|
appUrl,
|
|
119
124
|
scenario,
|
|
120
125
|
journey,
|
|
@@ -125,3 +130,34 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
125
130
|
},
|
|
126
131
|
};
|
|
127
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
|
|
135
|
+
* other characters as `-`, which the scrub would not find.
|
|
136
|
+
*/
|
|
137
|
+
function idHoldsKnownValue(id, secrets) {
|
|
138
|
+
return (secrets.scrub(id) !== id ||
|
|
139
|
+
secrets.values().some((value) => {
|
|
140
|
+
const token = publicSafeToken(value, "");
|
|
141
|
+
return token.length > 0 && id.includes(token);
|
|
142
|
+
}));
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The steps with their labels scrubbed and their ids safe to record. A step id names the step's
|
|
146
|
+
* screenshot file and is recorded beside it, and the parser derives an omitted id from the label,
|
|
147
|
+
* lower-cased and cut to 80 characters, so the scrub may not find a value in it. A step whose
|
|
148
|
+
* label or id holds a known value is renamed by its position instead.
|
|
149
|
+
*/
|
|
150
|
+
function recordedStepIds(steps, secrets) {
|
|
151
|
+
const taken = new Set(steps.map((step) => step.id));
|
|
152
|
+
return steps.map((step, index) => {
|
|
153
|
+
const label = secrets.scrub(step.label);
|
|
154
|
+
if (label === step.label && !idHoldsKnownValue(step.id, secrets))
|
|
155
|
+
return { ...step, label };
|
|
156
|
+
const position = `step-${String(index + 1).padStart(2, "0")}`;
|
|
157
|
+
let id = position;
|
|
158
|
+
for (let suffix = 2; taken.has(id); suffix += 1)
|
|
159
|
+
id = `${position}-${suffix}`;
|
|
160
|
+
taken.add(id);
|
|
161
|
+
return { ...step, id, label };
|
|
162
|
+
});
|
|
163
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ScriptedPlan } from "../../study/plan-types.js";
|
|
2
2
|
import { type PreparedRunArtifactPaths } from "../../run/paths.js";
|
|
3
3
|
import type { RunSubjectStateStepRecord } from "../../run/bundle.js";
|
|
4
|
-
import {
|
|
4
|
+
import { E2BSubjectSandbox } from "../../substrates/e2b/subject-sandbox.js";
|
|
5
5
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
6
6
|
import type { RunStudyHomes } from "../../study/run-study-homes.js";
|
|
7
7
|
type ScriptedCloneSubject = Extract<ScriptedPlan["subject"], {
|
|
@@ -22,33 +22,18 @@ export interface ScriptedSubjectInputs {
|
|
|
22
22
|
hasGithubToken: boolean;
|
|
23
23
|
scrubKnownValues: (text: string) => string;
|
|
24
24
|
now: () => number;
|
|
25
|
-
/** The run's warnings.
|
|
25
|
+
/** The run's warnings. The subject sandbox's release appends to it. */
|
|
26
26
|
warnings: string[];
|
|
27
27
|
}
|
|
28
28
|
/** The subject sandbox and what provisioning it recorded. */
|
|
29
29
|
export declare class ScriptedSubject {
|
|
30
|
-
|
|
31
|
-
killed: boolean;
|
|
32
|
-
/** The scrubbed release warning when the subject's release is unconfirmed. */
|
|
33
|
-
releaseWarning: string | undefined;
|
|
30
|
+
readonly sandbox: E2BSubjectSandbox;
|
|
34
31
|
commit: string | undefined;
|
|
35
32
|
hostDigest: string | undefined;
|
|
36
33
|
readonly stateStepRecords: RunSubjectStateStepRecord[];
|
|
37
34
|
private readonly inputs;
|
|
38
|
-
private allocation;
|
|
39
|
-
private createdAtMs;
|
|
40
|
-
private tornDownAtMs;
|
|
41
|
-
private resources;
|
|
42
35
|
constructor(inputs: ScriptedSubjectInputs);
|
|
43
36
|
/** Acquires, provisions and serves the subject. Returns the tokenless getHost URL to drive. */
|
|
44
37
|
provision(): Promise<string>;
|
|
45
|
-
/** Releases the subject by exact id, when one was acquired. An unconfirmed release is reported. */
|
|
46
|
-
teardown(): Promise<void>;
|
|
47
|
-
/** The subject desktop's billed span and size, when one was acquired. */
|
|
48
|
-
desktopUsage(): {
|
|
49
|
-
durationMs: number | undefined;
|
|
50
|
-
observation: DesktopResourceObservation | undefined;
|
|
51
|
-
killed: boolean;
|
|
52
|
-
} | undefined;
|
|
53
38
|
}
|
|
54
39
|
export {};
|
|
@@ -5,11 +5,9 @@ import { commandDigestOf } from "../../subject/state.js";
|
|
|
5
5
|
import { validatePreparedRunArtifactPaths, } from "../../run/paths.js";
|
|
6
6
|
import { provisionCloneSubject } from "../../subject/clone.js";
|
|
7
7
|
import { loadE2BDesktopModule } from "../../substrates/e2b/sdk.js";
|
|
8
|
-
import {
|
|
9
|
-
import { acquireE2BDesktopSandbox, readE2BRelease } from "../../substrates/e2b/sandbox.js";
|
|
8
|
+
import { E2BSubjectSandbox } from "../../substrates/e2b/subject-sandbox.js";
|
|
10
9
|
import { e2bShell } from "../../substrates/e2b/shell.js";
|
|
11
|
-
import { e2bRequestTimeoutMs
|
|
12
|
-
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
10
|
+
import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
|
|
13
11
|
function servePort(serveUrl) {
|
|
14
12
|
const url = new URL(serveUrl);
|
|
15
13
|
if (url.port)
|
|
@@ -35,65 +33,44 @@ function hostOriginDigest(url) {
|
|
|
35
33
|
}
|
|
36
34
|
/** The subject sandbox and what provisioning it recorded. */
|
|
37
35
|
export class ScriptedSubject {
|
|
38
|
-
|
|
39
|
-
killed = false;
|
|
40
|
-
/** The scrubbed release warning when the subject's release is unconfirmed. */
|
|
41
|
-
releaseWarning;
|
|
36
|
+
sandbox;
|
|
42
37
|
commit;
|
|
43
38
|
hostDigest;
|
|
44
39
|
stateStepRecords = [];
|
|
45
40
|
inputs;
|
|
46
|
-
allocation;
|
|
47
|
-
createdAtMs;
|
|
48
|
-
tornDownAtMs;
|
|
49
|
-
resources;
|
|
50
41
|
constructor(inputs) {
|
|
51
42
|
this.inputs = inputs;
|
|
43
|
+
this.sandbox = new E2BSubjectSandbox({
|
|
44
|
+
warnings: inputs.warnings,
|
|
45
|
+
scrub: inputs.scrubKnownValues,
|
|
46
|
+
now: inputs.now,
|
|
47
|
+
});
|
|
52
48
|
}
|
|
53
49
|
/** Acquires, provisions and serves the subject. Returns the tokenless getHost URL to drive. */
|
|
54
50
|
async provision() {
|
|
55
51
|
const { plan, clone, deps, prepareDesktop, env, e2bApiKey, runPaths, timeoutMs } = this.inputs;
|
|
56
|
-
const { subjectEnvNames, hasGithubToken, scrubKnownValues
|
|
52
|
+
const { subjectEnvNames, hasGithubToken, scrubKnownValues } = this.inputs;
|
|
57
53
|
const requestTimeoutMs = e2bRequestTimeoutMs(env);
|
|
58
54
|
const timers = deps.detachedTimers ?? {};
|
|
59
|
-
const subjectSandboxTimeoutMs = timeoutMs +
|
|
60
|
-
SUBJECT_PROVISION_BUDGET_MS +
|
|
61
|
-
(clone.state?.seed ?? []).reduce((sum, step) => sum + (step.timeoutMs ?? DEFAULT_STATE_STEP_TIMEOUT_MS), 0) +
|
|
62
|
-
SANDBOX_TIMEOUT_BUFFER_MS;
|
|
63
55
|
const subjectModule = await (deps.desktopModule ?? loadE2BDesktopModule)();
|
|
64
56
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
65
|
-
|
|
66
|
-
// it by exact id when this process dies mid-run; the finally block below only runs while
|
|
67
|
-
// the process is alive.
|
|
68
|
-
const subject = await acquireE2BDesktopSandbox({
|
|
57
|
+
const subjectDesktop = await this.sandbox.acquire({
|
|
69
58
|
module: subjectModule,
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
},
|
|
81
|
-
...(subjectEnvNames.length > 0
|
|
82
|
-
? {
|
|
83
|
-
envs: Object.fromEntries(subjectEnvNames.map((name) => [name, env[name]])),
|
|
84
|
-
}
|
|
85
|
-
: {}),
|
|
86
|
-
dpi: 96,
|
|
87
|
-
lifecycle: { onTimeout: "kill" },
|
|
59
|
+
apiKey: e2bApiKey,
|
|
60
|
+
requestTimeoutMs,
|
|
61
|
+
sessionTimeoutMs: timeoutMs,
|
|
62
|
+
seed: clone.state?.seed ?? [],
|
|
63
|
+
metadata: {
|
|
64
|
+
mode: "scripted-browser-lab",
|
|
65
|
+
tool: "humanish",
|
|
66
|
+
labId: plan.studyId,
|
|
67
|
+
kind: "subject",
|
|
68
|
+
actor: plan.actor,
|
|
88
69
|
},
|
|
70
|
+
envs: Object.fromEntries(subjectEnvNames.map((name) => [name, env[name]])),
|
|
89
71
|
template: plan.residual.execution?.desktop?.template,
|
|
90
|
-
|
|
72
|
+
root: runPaths,
|
|
91
73
|
});
|
|
92
|
-
const subjectDesktop = subject.sandbox;
|
|
93
|
-
this.allocation = subject.allocation;
|
|
94
|
-
this.sandboxId = subject.allocation.resourceId;
|
|
95
|
-
this.createdAtMs = now();
|
|
96
|
-
this.resources = await observeDesktopResources(subjectDesktop);
|
|
97
74
|
if (prepareDesktop) {
|
|
98
75
|
await prepareDesktop(subjectDesktop, { kind: "subject" });
|
|
99
76
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
@@ -125,37 +102,4 @@ export class ScriptedSubject {
|
|
|
125
102
|
this.hostDigest = hostOriginDigest(hostUrl);
|
|
126
103
|
return hostUrl;
|
|
127
104
|
}
|
|
128
|
-
/** Releases the subject by exact id, when one was acquired. An unconfirmed release is reported. */
|
|
129
|
-
async teardown() {
|
|
130
|
-
const { warnings, scrubKnownValues, now } = this.inputs;
|
|
131
|
-
if (this.allocation === undefined)
|
|
132
|
-
return;
|
|
133
|
-
const released = await this.allocation.close();
|
|
134
|
-
const reading = readE2BRelease(released, {
|
|
135
|
-
label: "Subject sandbox",
|
|
136
|
-
scrub: scrubKnownValues,
|
|
137
|
-
costSpan: true,
|
|
138
|
-
});
|
|
139
|
-
this.killed = reading.released;
|
|
140
|
-
if (reading.warning)
|
|
141
|
-
warnings.push(reading.warning);
|
|
142
|
-
if (!reading.released)
|
|
143
|
-
this.releaseWarning = reading.warning ?? "Subject sandbox release is unconfirmed.";
|
|
144
|
-
// Without a kill method, or in E2B debug mode, no kill reached E2B, so there is no teardown
|
|
145
|
-
// time to record.
|
|
146
|
-
if (released.status !== "unconfirmed" || released.reason !== "release_unavailable")
|
|
147
|
-
this.tornDownAtMs = now();
|
|
148
|
-
}
|
|
149
|
-
/** The subject desktop's billed span and size, when one was acquired. */
|
|
150
|
-
desktopUsage() {
|
|
151
|
-
if (this.sandboxId === undefined)
|
|
152
|
-
return undefined;
|
|
153
|
-
return {
|
|
154
|
-
durationMs: this.createdAtMs === undefined || this.tornDownAtMs === undefined
|
|
155
|
-
? undefined
|
|
156
|
-
: Math.max(0, this.tornDownAtMs - this.createdAtMs),
|
|
157
|
-
observation: this.resources,
|
|
158
|
-
killed: this.killed,
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
105
|
}
|
|
@@ -3,7 +3,7 @@ import type { ScriptedPlan } from "../../study/plan-types.js";
|
|
|
3
3
|
import { type PreparedRunArtifactPaths } from "../../run/paths.js";
|
|
4
4
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
5
5
|
/** One session per surface, in parallel. */
|
|
6
|
-
export declare function runScriptedSessions(session: Required<Pick<ScriptedBrowserSessionOptions, "appUrl" | "evidenceAppUrl" | "urlPolicy" | "journey" | "persona" | "timeoutMs" | "artifactRoot">>, run: {
|
|
6
|
+
export declare function runScriptedSessions(session: Required<Pick<ScriptedBrowserSessionOptions, "appUrl" | "evidenceAppUrl" | "urlPolicy" | "journey" | "persona" | "timeoutMs" | "artifactRoot" | "scrubKnownValues">>, run: {
|
|
7
7
|
surfaces: ScriptedPlan["surfaces"];
|
|
8
8
|
deps: StudyDeps;
|
|
9
9
|
browserCommand: string | undefined;
|
|
@@ -7,6 +7,7 @@ import { validateScriptedSessionResult } from "./session-result.js";
|
|
|
7
7
|
/** One session per surface, in parallel. */
|
|
8
8
|
export function runScriptedSessions(session, run) {
|
|
9
9
|
const { appUrl, evidenceAppUrl, urlPolicy, journey, persona, timeoutMs, artifactRoot } = session;
|
|
10
|
+
const { scrubKnownValues } = session;
|
|
10
11
|
const { surfaces, deps, browserCommand, runPaths, onSessionStart } = run;
|
|
11
12
|
const runSession = deps.runScriptedSession;
|
|
12
13
|
return Promise.all(surfaces.map((surface) => {
|
|
@@ -20,6 +21,7 @@ export function runScriptedSessions(session, run) {
|
|
|
20
21
|
persona,
|
|
21
22
|
timeoutMs,
|
|
22
23
|
artifactRoot,
|
|
24
|
+
scrubKnownValues,
|
|
23
25
|
...(browserCommand === undefined ? {} : { browserCommand }),
|
|
24
26
|
...(deps.launchBrowser === undefined ? {} : { launchBrowser: deps.launchBrowser }),
|
|
25
27
|
...(deps.now === undefined ? {} : { now: deps.now }),
|
|
@@ -5,7 +5,7 @@ import type { ObserverResult } from "../../observer/render.js";
|
|
|
5
5
|
import type { RunSubjectProvenance } from "../../run/bundle.js";
|
|
6
6
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
7
7
|
import type { RunStudyHomes } from "../../study/run-study-homes.js";
|
|
8
|
-
import { type StudyResultIdentity } from "../../run/study-result.js";
|
|
8
|
+
import { type StudyResultErrorCode, type StudyResultIdentity } from "../../run/study-result.js";
|
|
9
9
|
/** What a scripted run takes besides its plan. The plan carries the config, dry run and study. */
|
|
10
10
|
export interface ScriptedRunInput {
|
|
11
11
|
/** Cancels post-run analysis only. */
|
|
@@ -58,7 +58,7 @@ export interface ScriptedBrowserStudyResult extends AutomaticAnalysisResult, Stu
|
|
|
58
58
|
observer?: ObserverResult;
|
|
59
59
|
warnings: string[];
|
|
60
60
|
error?: {
|
|
61
|
-
code:
|
|
61
|
+
code: StudyResultErrorCode | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE";
|
|
62
62
|
message: string;
|
|
63
63
|
};
|
|
64
64
|
}
|
|
@@ -8,6 +8,7 @@ import { SHARED_WORLD_SCHEMA, concurrencyFacts, sharedWorldParticipantKeys, } fr
|
|
|
8
8
|
import { commandDigestOf } from "../../subject/state.js";
|
|
9
9
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
10
10
|
import { participantFactsOf } from "../computer-use/participant-facts.js";
|
|
11
|
+
import { participantFeedbackCandidates } from "../computer-use/participant-feedback.js";
|
|
11
12
|
import { judgeSharedWorld, participantPassed, sharedWorldShortfall, verdictText, } from "../../run/judge.js";
|
|
12
13
|
import { reviewOutcome } from "../../run/display.js";
|
|
13
14
|
import { combineCheckpointDigest } from "./checkpoints.js";
|
|
@@ -369,7 +370,7 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
369
370
|
},
|
|
370
371
|
artifacts: bundleArtifacts(),
|
|
371
372
|
review,
|
|
372
|
-
feedbackCandidates:
|
|
373
|
+
feedbackCandidates: sharedWorldFeedbackCandidates(args),
|
|
373
374
|
// Custom desktop image provenance (subject + every actor sandbox launched on it); omitted on the default.
|
|
374
375
|
...(plan.residual.execution?.desktop?.template === undefined
|
|
375
376
|
? {}
|
|
@@ -380,6 +381,40 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
380
381
|
...(cost === undefined ? {} : { cost }),
|
|
381
382
|
};
|
|
382
383
|
}
|
|
384
|
+
/**
|
|
385
|
+
* What the participants reported, built by the same builder as a computer-use fan-out: one
|
|
386
|
+
* candidate per participant who reported friction or abandoned the goal, named by its participant
|
|
387
|
+
* id, with that participant's own instructions as `expected`. Dry-run and in-progress bundles
|
|
388
|
+
* carry none: there is no participant yet to quote. The comms thread belongs to the one shared
|
|
389
|
+
* app, so each candidate cites it.
|
|
390
|
+
*/
|
|
391
|
+
function sharedWorldFeedbackCandidates(args) {
|
|
392
|
+
if (args.dryRun || args.inProgress === true)
|
|
393
|
+
return [];
|
|
394
|
+
return participantFeedbackCandidates({
|
|
395
|
+
runId: args.run.runId,
|
|
396
|
+
scenarioId: `concurrent-shared-world-${args.plan.studyId}`,
|
|
397
|
+
adapterId: args.plan.studyId,
|
|
398
|
+
// planSharedWorldStudy refuses any target other than e2b-desktop on both planes.
|
|
399
|
+
substrate: "e2b-desktop",
|
|
400
|
+
participants: args.actorSpecs.map((spec, index) => {
|
|
401
|
+
const outcome = args.actorResults[index]?.outcome;
|
|
402
|
+
return {
|
|
403
|
+
participantId: spec.planned.id,
|
|
404
|
+
streamId: spec.streamId,
|
|
405
|
+
personaId: spec.persona.id,
|
|
406
|
+
goal: redactText(spec.evidenceInstructions ?? spec.instructions),
|
|
407
|
+
...(outcome?.session === undefined
|
|
408
|
+
? {}
|
|
409
|
+
: { session: outcome.session, traceArtifactPath: spec.traceArtifactPath }),
|
|
410
|
+
screenshots: outcome?.screenshots ?? [],
|
|
411
|
+
...(args.commsArtifactPath === undefined
|
|
412
|
+
? {}
|
|
413
|
+
: { commsArtifactPath: args.commsArtifactPath }),
|
|
414
|
+
};
|
|
415
|
+
}),
|
|
416
|
+
});
|
|
417
|
+
}
|
|
383
418
|
/**
|
|
384
419
|
* Priced like `per-lane-worlds`: each participant's model tokens and desktop, plus the provisioned plane's
|
|
385
420
|
* desktop. Absent when nothing ran, so dry runs and in-progress snapshots stay byte-stable.
|
|
@@ -3,6 +3,7 @@ import type { SharedWorldPlan } from "../../study/plan-types.js";
|
|
|
3
3
|
import type { CommsReceivingRun } from "../../comms/receiving.js";
|
|
4
4
|
import { type DeployedCommsCatch } from "../../comms/sandbox-catch.js";
|
|
5
5
|
import type { StudyCommsEmail, StudyConfig } from "../../study/types.js";
|
|
6
|
+
import type { RunSecrets } from "../../run/secrets.js";
|
|
6
7
|
import type { Shell } from "../../substrates/shell.js";
|
|
7
8
|
import type { ConcurrentSharedWorldPlaneClass, ExternalCommsWiring, PlaneContext } from "./types.js";
|
|
8
9
|
/** The in-sandbox email catch a provisioned plane deploys, when the study declares one. */
|
|
@@ -58,7 +59,7 @@ export declare function prepareEmailReceiving(args: {
|
|
|
58
59
|
env: Record<string, string | undefined>;
|
|
59
60
|
participants: string[];
|
|
60
61
|
runPaths: PlaneContext["runPaths"];
|
|
61
|
-
|
|
62
|
+
secrets: RunSecrets;
|
|
62
63
|
dryRun: boolean;
|
|
63
64
|
}): Promise<{
|
|
64
65
|
ok: true;
|