humanish 0.111.2 → 0.111.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +13 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +21 -28
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +2 -2
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +1 -1
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +22 -0
- package/dist/run/secrets.js +33 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +88 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/package.json +1 -1
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
// The computer-use run's setup: resolve the project and route, plan the participants, make the
|
|
2
2
|
// live checks that need no sandbox, pack a local tree, then start the run and build the
|
|
3
3
|
// participant deps and bundle base that the participants and the finish share.
|
|
4
|
-
import { randomBytes } from "node:crypto";
|
|
5
4
|
import { realpath } from "node:fs/promises";
|
|
6
5
|
import path from "node:path";
|
|
7
6
|
import {} from "../../run/run.js";
|
|
8
7
|
import { externalInboxUrl } from "../../comms/sandbox-catch.js";
|
|
9
|
-
import { redactText,
|
|
10
|
-
import {
|
|
8
|
+
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
9
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
10
|
+
import {} from "../../run/bundle.js";
|
|
11
11
|
import { assertPreparedSelectedOutputDirectory, prepareContainedOutputDirectory, prepareSelectedOutputDirectory, } from "../../run/contained-output.js";
|
|
12
12
|
import {} from "../../subject/local-tree-archive.js";
|
|
13
13
|
import { renderCuaReviewMarkdown } from "./bundle.js";
|
|
14
14
|
import { emitPreflightPlan, compileParticipantPersonas, loadCuaParticipants, sanitizeParticipantRuns, } from "./participant-runs.js";
|
|
15
|
-
import {
|
|
16
|
-
import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
|
|
15
|
+
import { desktopParticipantDeps } from "./participant-deps.js";
|
|
17
16
|
import { liveCuaRejection } from "./preflight.js";
|
|
18
17
|
import { cuaDescriptorOf, declaredAppUrl, plannedAppUrl } from "./plan.js";
|
|
19
18
|
import { trackRuntimeStreams } from "./live-flush.js";
|
|
@@ -24,7 +23,7 @@ import { packRunLocalTree } from "./local-tree-pack.js";
|
|
|
24
23
|
import { projectParticipantSubjects, subjectProvenanceArg } from "./subject-projection.js";
|
|
25
24
|
import { participantSubjectEnv, } from "./types.js";
|
|
26
25
|
import { studyPersonaIds } from "../../study/persona-resolve.js";
|
|
27
|
-
import {
|
|
26
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
28
27
|
/** The physical project, bound before any caller hook runs. */
|
|
29
28
|
async function bindProject(cwd) {
|
|
30
29
|
// Capture the physical project before reading or invoking any caller hook. A supported
|
|
@@ -43,18 +42,17 @@ export async function refuseCuaStudy(options, refusal) {
|
|
|
43
42
|
// The committed personas are read before the refusal returns, so a persona-file error wins.
|
|
44
43
|
if (refusal.stage === "after-personas")
|
|
45
44
|
await compileParticipantPersonas(projectRoot, studyPersonaIds(config));
|
|
46
|
-
return {
|
|
47
|
-
|
|
48
|
-
ok: false,
|
|
45
|
+
return refusedResult("computer-use", {
|
|
46
|
+
studyId: config.id,
|
|
49
47
|
cwd: projectRoot.physicalPath,
|
|
48
|
+
error: { code: refusal.code, message: refusal.message },
|
|
49
|
+
}, {
|
|
50
50
|
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
51
51
|
appUrl: declaredAppUrl(config),
|
|
52
52
|
dryRun,
|
|
53
53
|
runId: options.runId ?? "not-created",
|
|
54
54
|
lanes: [],
|
|
55
|
-
|
|
56
|
-
error: { code: refusal.code, message: refusal.message },
|
|
57
|
-
};
|
|
55
|
+
});
|
|
58
56
|
}
|
|
59
57
|
/**
|
|
60
58
|
* Everything before the run starts: the physical project, the route, the participant plan and
|
|
@@ -73,17 +71,12 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
73
71
|
const inProcess = desktop === "in-process";
|
|
74
72
|
const appUrl = plannedAppUrl(subject);
|
|
75
73
|
const subjectEnvNames = [...participantSubjectEnv(subject)];
|
|
76
|
-
const fail = (code, message, actorLabel) => ({
|
|
77
|
-
...studyResultIdentity("computer-use", plan.studyId),
|
|
78
|
-
ok: false,
|
|
79
|
-
cwd,
|
|
74
|
+
const fail = (code, message, actorLabel) => refusedResult("computer-use", { studyId: plan.studyId, cwd, error: { code, message } }, {
|
|
80
75
|
actor: actorLabel ?? plan.actor,
|
|
81
76
|
appUrl,
|
|
82
77
|
dryRun,
|
|
83
78
|
runId: input.runId ?? "not-created",
|
|
84
79
|
lanes: [],
|
|
85
|
-
warnings: [],
|
|
86
|
-
error: { code, message },
|
|
87
80
|
});
|
|
88
81
|
const refuse = (...args) => ({
|
|
89
82
|
ok: false,
|
|
@@ -131,13 +124,13 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
131
124
|
const openaiApiKey = env.OPENAI_API_KEY?.trim() ?? "";
|
|
132
125
|
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
133
126
|
// Literal scrubber for every known provisioned value (no secret "shape" to pattern-match).
|
|
134
|
-
|
|
127
|
+
// Dry and in-process runs scrub both keys too: a mission or persona can still quote them.
|
|
128
|
+
const secrets = new RunSecrets([
|
|
135
129
|
openaiApiKey,
|
|
136
130
|
e2bApiKey,
|
|
137
131
|
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
138
|
-
]
|
|
139
|
-
|
|
140
|
-
sanitizeParticipantRuns(participantRuns, scrubKnownValues);
|
|
132
|
+
]);
|
|
133
|
+
sanitizeParticipantRuns(participantRuns, secrets.scrub);
|
|
141
134
|
const redactRepoLabel = plan.residual.policies?.redactRepos ?? subjectEnvNames.includes("GITHUB_TOKEN");
|
|
142
135
|
const publicRepo = subject.kind === "clone" ? (redactRepoLabel ? "repo-01" : subject.repo) : undefined;
|
|
143
136
|
// Key-gating is route-aware: the in-process route uses the caller's own model + executor, and
|
|
@@ -167,7 +160,7 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
167
160
|
localTreeArchiveBuffer = packed.buffer;
|
|
168
161
|
}
|
|
169
162
|
catch (error) {
|
|
170
|
-
return refuse("HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `local-tree packing failed: ${redactText(
|
|
163
|
+
return refuse("HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `local-tree packing failed: ${redactText(secrets.scrub(toErrorMessage(error)))}`, descriptor.id);
|
|
171
164
|
}
|
|
172
165
|
}
|
|
173
166
|
return {
|
|
@@ -192,8 +185,7 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
192
185
|
participantCount,
|
|
193
186
|
openaiApiKey,
|
|
194
187
|
e2bApiKey,
|
|
195
|
-
|
|
196
|
-
scrubKnownValues,
|
|
188
|
+
secrets,
|
|
197
189
|
publicRepo,
|
|
198
190
|
localTreeArchive,
|
|
199
191
|
localTreeArchiveBuffer,
|
|
@@ -202,33 +194,23 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
202
194
|
}
|
|
203
195
|
/** Starts the run and builds what the participants and the finish read: the deps and the bundle base. */
|
|
204
196
|
export async function startCuaRun(plan, input, admitted, scope) {
|
|
205
|
-
const {
|
|
197
|
+
const { cwd, descriptor, participantRuns, participantPlan, publicRepo } = admitted;
|
|
206
198
|
const { appUrl } = admitted;
|
|
207
199
|
// The run's status record exists from here on, so anything watching the runs directory can
|
|
208
200
|
// tell which study this is and that it is alive. The fail-closed returns below leave it finished
|
|
209
201
|
// with no outcome when the scope closes; a crash leaves it stale, which reads as interrupted.
|
|
210
|
-
const started = await scope.startRun({
|
|
202
|
+
const started = await scope.startRun(plan, input, {
|
|
211
203
|
cwd,
|
|
212
|
-
|
|
213
|
-
mintRunId: makeCuaRunId,
|
|
214
|
-
mode: dryRun ? "dry-run" : "live",
|
|
215
|
-
study: plan.study,
|
|
216
|
-
warnings: plan.warnings,
|
|
204
|
+
prefix: "cua",
|
|
217
205
|
renderReview: renderCuaReviewMarkdown,
|
|
218
|
-
|
|
206
|
+
secrets: admitted.secrets,
|
|
219
207
|
});
|
|
220
208
|
if (!started.ok)
|
|
221
209
|
return admitted.refuse(started.code, started.message, descriptor.id);
|
|
222
210
|
const { run } = started;
|
|
223
|
-
const {
|
|
211
|
+
const { paths: runPaths, source } = run;
|
|
224
212
|
const redactScreenshots = plan.residual.policies?.redactScreenshots === true;
|
|
225
213
|
await prepareContainedOutputDirectory(runPaths, "screenshots");
|
|
226
|
-
const source = await buildRunSource({
|
|
227
|
-
capturedAt: createdAt,
|
|
228
|
-
cwd,
|
|
229
|
-
humanishSource: "present",
|
|
230
|
-
packageName: "humanish",
|
|
231
|
-
});
|
|
232
214
|
// Live-trace flush seam: runStudyParticipants fills it when a live run has an in-progress bundle
|
|
233
215
|
// to grow; participants call it through deps.onTrace. It exists before deps so deps can reference it as
|
|
234
216
|
// a stable indirection.
|
|
@@ -279,12 +261,10 @@ export async function startCuaRun(plan, input, admitted, scope) {
|
|
|
279
261
|
streams: admitted.streams,
|
|
280
262
|
participantRuns,
|
|
281
263
|
participantPlan,
|
|
282
|
-
scrubKnownValues: admitted.scrubKnownValues,
|
|
283
264
|
bundleBase,
|
|
284
265
|
},
|
|
285
266
|
participants: {
|
|
286
267
|
env: admitted.env,
|
|
287
|
-
knownSecretValues: admitted.knownSecretValues,
|
|
288
268
|
deps,
|
|
289
269
|
liveTrace,
|
|
290
270
|
externalComms: externalCommsConfig === undefined || externalCommsEmail === undefined
|
|
@@ -300,60 +280,38 @@ export async function startCuaRun(plan, input, admitted, scope) {
|
|
|
300
280
|
finish: { rerunLineage: admitted.rerunLineage, publicRepo, subjectArgs },
|
|
301
281
|
};
|
|
302
282
|
}
|
|
303
|
-
/** The participant deps every participant reads
|
|
283
|
+
/** The participant deps every participant reads, with computer use's own overrides. */
|
|
304
284
|
function cuaParticipantDeps(plan, input, admitted, run) {
|
|
305
|
-
const { config,
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
const { externalCommsConfig, externalCommsEmail } = admitted;
|
|
309
|
-
const { appUrl } = admitted;
|
|
310
|
-
const { runPaths, redactScreenshots, liveTrace } = run;
|
|
311
|
-
const createDesktop = input.localVm?.desktop;
|
|
312
|
-
const timeoutMs = plan.sessionBudgetMs;
|
|
313
|
-
const requestTimeoutMs = e2bRequestTimeoutMs(env);
|
|
314
|
-
return {
|
|
315
|
-
...(createDesktop === undefined ? {} : { createDesktop }),
|
|
316
|
-
onTrace: (participantId, items, usage, metadata) => liveTrace.flush?.(participantId, items, usage, metadata),
|
|
317
|
-
...callerDriving(input, config),
|
|
318
|
-
residual: plan.residual,
|
|
319
|
-
studyId: plan.studyId,
|
|
320
|
-
caps: plan.caps,
|
|
321
|
-
appUrl,
|
|
285
|
+
const { config, seams, env, externalCommsConfig, externalCommsEmail } = admitted;
|
|
286
|
+
const deps = desktopParticipantDeps({
|
|
287
|
+
plan,
|
|
322
288
|
brain: plan.runner.brain,
|
|
323
289
|
subject: plan.runner.subject,
|
|
324
|
-
...(localTreeArchiveBuffer === undefined ? {} : { localTreeArchiveBuffer }),
|
|
325
290
|
env,
|
|
326
|
-
openaiApiKey,
|
|
327
|
-
e2bApiKey,
|
|
328
|
-
|
|
291
|
+
openaiApiKey: admitted.openaiApiKey,
|
|
292
|
+
e2bApiKey: admitted.e2bApiKey,
|
|
293
|
+
timeoutMs: plan.sessionBudgetMs,
|
|
329
294
|
sandboxMs: plan.sandboxMs,
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
artifactRoot: runPaths,
|
|
295
|
+
participantCount: admitted.participantCount,
|
|
296
|
+
artifactRoot: run.runPaths,
|
|
333
297
|
studyCwd: input.cwd,
|
|
334
|
-
redactScreenshots,
|
|
335
|
-
scrubKnownValues,
|
|
336
|
-
runSession,
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
...(dryRun || plan.caps.maxTotalUsd === undefined
|
|
340
|
-
? {}
|
|
341
|
-
: { runBudget: makeCuaRunBudget(plan.caps.maxTotalUsd) }),
|
|
342
|
-
...(externalCommsConfig === undefined || externalCommsEmail === undefined
|
|
343
|
-
? {}
|
|
344
|
-
: {
|
|
345
|
-
externalComms: {
|
|
346
|
-
email: externalCommsEmail,
|
|
347
|
-
inboxUrl: externalInboxUrl(externalCommsConfig),
|
|
348
|
-
},
|
|
349
|
-
}),
|
|
350
|
-
now: seams.now ?? Date.now,
|
|
351
|
-
...(seams.desktopModule === undefined ? {} : { desktopModule: seams.desktopModule }),
|
|
352
|
-
...(input.prepareDesktop === undefined ? {} : { prepareDesktop: input.prepareDesktop }),
|
|
353
|
-
...(seams.detachedTimers === undefined ? {} : { detachedTimers: seams.detachedTimers }),
|
|
354
|
-
onStream: streams.onStream,
|
|
298
|
+
redactScreenshots: run.redactScreenshots,
|
|
299
|
+
scrubKnownValues: admitted.secrets.scrub,
|
|
300
|
+
runSession: admitted.runSession,
|
|
301
|
+
liveFlush: () => run.liveTrace.flush,
|
|
302
|
+
onStream: admitted.streams.onStream,
|
|
355
303
|
reportSubjectPhase: subjectPhaseReporter(input),
|
|
356
|
-
|
|
304
|
+
seams,
|
|
305
|
+
prepareDesktop: input.prepareDesktop,
|
|
306
|
+
}, {
|
|
307
|
+
createDesktop: input.localVm?.desktop,
|
|
308
|
+
...callerDriving(input, config),
|
|
309
|
+
localTreeArchiveBuffer: admitted.localTreeArchiveBuffer,
|
|
310
|
+
externalComms: externalCommsConfig === undefined || externalCommsEmail === undefined
|
|
311
|
+
? undefined
|
|
312
|
+
: { email: externalCommsEmail, inboxUrl: externalInboxUrl(externalCommsConfig) },
|
|
313
|
+
});
|
|
314
|
+
return { ...deps, appUrl: admitted.appUrl };
|
|
357
315
|
}
|
|
358
316
|
/** The caller's createProvider and inProcess executor, with the run's config bound. */
|
|
359
317
|
function callerDriving(input, config) {
|
|
@@ -377,7 +335,3 @@ function subjectPhaseReporter(input) {
|
|
|
377
335
|
input.emit?.(phaseEvent(event, { kind: "participant", participant }));
|
|
378
336
|
};
|
|
379
337
|
}
|
|
380
|
-
function makeCuaRunId() {
|
|
381
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
382
|
-
return `cua-${stamp}-${randomBytes(4).toString("hex")}`;
|
|
383
|
-
}
|
|
@@ -3,7 +3,8 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/streams.js";
|
|
5
5
|
import { aggregateTaskFunnels, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
6
|
-
import { describeSubjectState,
|
|
6
|
+
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
7
|
+
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
7
8
|
import { participantStatus as participantStatusFor } from "../../run/judge.js";
|
|
8
9
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
9
10
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
@@ -359,13 +360,13 @@ export function buildSingleParticipantBundle(args) {
|
|
|
359
360
|
runId: args.run.runId,
|
|
360
361
|
scenarioId: `cua-${args.studyId}`,
|
|
361
362
|
adapterId: args.studyId,
|
|
362
|
-
goal: redactText(args.mission),
|
|
363
363
|
substrate: runnerSubstrate(args),
|
|
364
364
|
participants: [
|
|
365
365
|
{
|
|
366
366
|
participantId: args.participantId ?? "lane-01",
|
|
367
367
|
streamId: SINGLE.streamId,
|
|
368
368
|
personaId: args.persona.id,
|
|
369
|
+
goal: redactText(args.mission),
|
|
369
370
|
...(args.session === undefined ? {} : { session: args.session }),
|
|
370
371
|
...(args.traceArtifactPath === undefined
|
|
371
372
|
? {}
|
|
@@ -26,7 +26,7 @@ import { type RunDesktopGeometry } from "../../run/streams.js";
|
|
|
26
26
|
import { type PreparedOutputRoot } from "../../run/contained-output.js";
|
|
27
27
|
import type { ComputerUseParticipant, SharedWorldParticipant } from "../../study/plan-participants.js";
|
|
28
28
|
import type { ResolvedParticipant } from "../../run/participant.js";
|
|
29
|
-
import { type StudyResultIdentity } from "../../run/study-result.js";
|
|
29
|
+
import { type StudyResultErrorCode, type StudyResultIdentity } from "../../run/study-result.js";
|
|
30
30
|
export declare const CUA_FANOUT_STRATEGY: "per-lane-worlds";
|
|
31
31
|
export declare const CUA_MAX_CONCURRENCY_ENV = "HUMANISH_CUA_MAX_CONCURRENCY";
|
|
32
32
|
export declare const DEFAULT_APP_URL_SESSION_TIMEOUT_MS: number;
|
|
@@ -182,7 +182,7 @@ export interface CuaParticipantSummary {
|
|
|
182
182
|
concurrency: number;
|
|
183
183
|
waves: number;
|
|
184
184
|
}
|
|
185
|
-
export type CuaActorStudyErrorCode =
|
|
185
|
+
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
186
186
|
/** Subject provenance projection: what the actor actually drove. */
|
|
187
187
|
export interface CuaSubjectProjection {
|
|
188
188
|
source: "app-url" | "clone" | "local-tree";
|
package/dist/routes/preview.js
CHANGED
|
@@ -1,17 +1,11 @@
|
|
|
1
1
|
// The preview route: a study whose subject is this repo runs the synthetic dry run. planStudy decides
|
|
2
2
|
// the sim count and refuses what a synthetic run would ignore; runPreviewPlan does the rest.
|
|
3
|
-
import { studyResultIdentity } from "../run/study-result.js";
|
|
3
|
+
import { refusedResult, studyResultIdentity, } from "../run/study-result.js";
|
|
4
4
|
import { runDryRun } from "../run/dry-run.js";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
/** A refused preview's result. */
|
|
7
7
|
export function previewStudyRefusal(cwd, studyId, refusal) {
|
|
8
|
-
return {
|
|
9
|
-
...studyResultIdentity("preview", studyId),
|
|
10
|
-
ok: false,
|
|
11
|
-
cwd: path.resolve(cwd),
|
|
12
|
-
warnings: [],
|
|
13
|
-
error: { code: refusal.code, message: refusal.message },
|
|
14
|
-
};
|
|
8
|
+
return refusedResult("preview", { studyId, cwd: path.resolve(cwd), error: { code: refusal.code, message: refusal.message } }, {});
|
|
15
9
|
}
|
|
16
10
|
/** runStudyWith's step for a preview plan: it has no local checks and takes no scorer, so it returns its run. */
|
|
17
11
|
export function admitPreviewPlan(plan, input) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ActorPersonaRef } from "../../actors/contract.js";
|
|
2
2
|
import type { ScriptedBrowserSessionResult } from "../../actors/scripted-browser/actor.js";
|
|
3
3
|
import type { BrowserPersonaJourney, BrowserSurface } from "../../actors/scripted-browser/types.js";
|
|
4
|
-
import type {
|
|
4
|
+
import type { SubjectDesktopUsage } from "../../substrates/e2b/subject-sandbox.js";
|
|
5
5
|
import { type ReviewSummary, type RunBundle, type RunSubjectProvenance, type BundleRun } from "../../run/bundle.js";
|
|
6
6
|
/** What the scripted study's bundle is built from. */
|
|
7
7
|
interface ScriptedBundleArgs {
|
|
@@ -24,11 +24,7 @@ interface ScriptedBundleArgs {
|
|
|
24
24
|
source: RunBundle["source"];
|
|
25
25
|
subject?: RunSubjectProvenance;
|
|
26
26
|
/** The provisioned clone's desktop; absent when no sandbox was created. */
|
|
27
|
-
subjectDesktop?:
|
|
28
|
-
durationMs: number | undefined;
|
|
29
|
-
observation: DesktopResourceObservation | undefined;
|
|
30
|
-
killed: boolean;
|
|
31
|
-
};
|
|
27
|
+
subjectDesktop?: SubjectDesktopUsage;
|
|
32
28
|
surfaces: readonly BrowserSurface[];
|
|
33
29
|
/** The run's verdict (judgeScripted). */
|
|
34
30
|
verdict: ReviewSummary["verdict"];
|
|
@@ -6,6 +6,7 @@ import type { RunBundle } from "../../run/bundle.js";
|
|
|
6
6
|
import type { RunScope } from "../../run/run.js";
|
|
7
7
|
import { type ExecutionFailure } from "../../run/judge.js";
|
|
8
8
|
import type { ScriptedSubject } from "./subject.js";
|
|
9
|
+
import type { E2BSubjectSandbox } from "../../substrates/e2b/subject-sandbox.js";
|
|
9
10
|
import { type ScriptedBrowserStudyResult } from "./types.js";
|
|
10
11
|
/** What the finish reads from the run. */
|
|
11
12
|
export interface ScriptedFinishInputs {
|
|
@@ -44,6 +45,6 @@ export declare function scriptedExecutionFailures(args: {
|
|
|
44
45
|
sessionError: string | undefined;
|
|
45
46
|
expected: number;
|
|
46
47
|
sessionResults: readonly Pick<ScriptedBrowserSessionResult, "completionReason" | "reason">[];
|
|
47
|
-
subject: Pick<
|
|
48
|
+
subject: Pick<E2BSubjectSandbox, "sandboxId" | "killed" | "releaseWarning"> | undefined;
|
|
48
49
|
}): ExecutionFailure[];
|
|
49
50
|
export declare function finishScriptedRun(inputs: ScriptedFinishInputs): Promise<ScriptedBrowserStudyResult>;
|
|
@@ -53,11 +53,11 @@ export async function finishScriptedRun(inputs) {
|
|
|
53
53
|
const { actor, dryRun } = plan;
|
|
54
54
|
const { runId, paths: runPaths } = run;
|
|
55
55
|
const subjectCommit = scriptedSubject?.commit;
|
|
56
|
-
const subjectSandboxId = scriptedSubject?.sandboxId;
|
|
57
|
-
const subjectKilled = scriptedSubject?.killed ?? false;
|
|
56
|
+
const subjectSandboxId = scriptedSubject?.sandbox.sandboxId;
|
|
57
|
+
const subjectKilled = scriptedSubject?.sandbox.killed ?? false;
|
|
58
58
|
const hostDigest = scriptedSubject?.hostDigest;
|
|
59
59
|
const stateStepRecords = scriptedSubject?.stateStepRecords ?? [];
|
|
60
|
-
const subjectDesktop = scriptedSubject?.desktopUsage();
|
|
60
|
+
const subjectDesktop = scriptedSubject?.sandbox.desktopUsage();
|
|
61
61
|
const screenshotsBySurface = new Map();
|
|
62
62
|
for (const result of sessionResults) {
|
|
63
63
|
screenshotsBySurface.set(result.capture.surface.id, await existingScreenshots(runPaths, result));
|
|
@@ -114,7 +114,7 @@ export async function finishScriptedRun(inputs) {
|
|
|
114
114
|
sessionError,
|
|
115
115
|
expected: surfaces.length,
|
|
116
116
|
sessionResults,
|
|
117
|
-
subject: scriptedSubject,
|
|
117
|
+
subject: scriptedSubject?.sandbox,
|
|
118
118
|
}), policy);
|
|
119
119
|
const finished = await run.finish(bundle, {
|
|
120
120
|
ok: resultOk({ judgment, execution, scorerFailures: [], policy }),
|
|
@@ -14,11 +14,9 @@
|
|
|
14
14
|
// subject, so its build provenance is unpinned and the evidence binds to the scenario digest. A
|
|
15
15
|
// clone run records commit, env names, state provenance and a host digest, never the raw getHost
|
|
16
16
|
// URL or a secret value.
|
|
17
|
-
import { randomBytes } from "node:crypto";
|
|
18
17
|
import { realpath } from "node:fs/promises";
|
|
19
18
|
import path from "node:path";
|
|
20
19
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
21
|
-
import { buildRunSource } from "../../run/bundle.js";
|
|
22
20
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
23
21
|
import {} from "../../run/run.js";
|
|
24
22
|
import { admitRoute, completeRefusalAnalysis } from "../../run/route-shell.js";
|
|
@@ -30,7 +28,7 @@ import { prepareScriptedRun } from "./setup.js";
|
|
|
30
28
|
import { ScriptedSubject } from "./subject.js";
|
|
31
29
|
import { runScriptedSessions, writeSurfaceTraces } from "./surface-sessions.js";
|
|
32
30
|
import {} from "./types.js";
|
|
33
|
-
import {
|
|
31
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
34
32
|
/**
|
|
35
33
|
* A refused scripted study's result, at the refusal's stage: a before-scope refusal has its own field
|
|
36
34
|
* order and no analysis record; the others come after the output directory checks.
|
|
@@ -39,34 +37,25 @@ export async function scriptedStudyRefusal(options, refusal) {
|
|
|
39
37
|
const { config, dryRun } = options;
|
|
40
38
|
const cwd = path.resolve(options.cwd);
|
|
41
39
|
const actorType = config.actor?.type ?? "";
|
|
40
|
+
const error = { code: refusal.code, message: refusal.message };
|
|
42
41
|
if (refusal.beforeScope)
|
|
43
|
-
return {
|
|
44
|
-
...studyResultIdentity("scripted", config.id),
|
|
45
|
-
ok: false,
|
|
46
|
-
cwd,
|
|
42
|
+
return refusedResult("scripted", { studyId: config.id, cwd, error }, {
|
|
47
43
|
actor: actorType,
|
|
48
44
|
dryRun,
|
|
49
45
|
runId: options.runId ?? "not-created",
|
|
50
46
|
appUrl: config.subject.appUrl ?? "",
|
|
51
47
|
sessions: [],
|
|
52
|
-
|
|
53
|
-
error: { code: refusal.code, message: refusal.message },
|
|
54
|
-
};
|
|
48
|
+
});
|
|
55
49
|
// The other refusals come after the output directory checks and carry the analysis record.
|
|
56
50
|
const physicalCwd = await realpath(cwd);
|
|
57
51
|
await prepareSelectedOutputDirectory(path.dirname(physicalCwd), physicalCwd);
|
|
58
|
-
const refused = {
|
|
59
|
-
...studyResultIdentity("scripted", config.id),
|
|
60
|
-
ok: false,
|
|
61
|
-
cwd,
|
|
52
|
+
const refused = refusedResult("scripted", { studyId: config.id, cwd, error }, {
|
|
62
53
|
actor: refusal.actor ?? actorType,
|
|
63
54
|
appUrl: refusal.appUrl ?? config.subject.appUrl ?? "",
|
|
64
55
|
dryRun,
|
|
65
56
|
runId: options.runId ?? "not-created",
|
|
66
57
|
sessions: [],
|
|
67
|
-
|
|
68
|
-
error: { code: refusal.code, message: refusal.message },
|
|
69
|
-
};
|
|
58
|
+
});
|
|
70
59
|
return completeRefusalAnalysis(refused, config, options);
|
|
71
60
|
}
|
|
72
61
|
/**
|
|
@@ -88,30 +77,20 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
88
77
|
return prepared.result;
|
|
89
78
|
const { dryRun } = plan;
|
|
90
79
|
const { setup } = prepared;
|
|
91
|
-
const started = await scope.startRun({
|
|
80
|
+
const started = await scope.startRun(plan, input, {
|
|
92
81
|
cwd: setup.physicalCwd,
|
|
93
|
-
|
|
94
|
-
mintRunId: makeScriptedRunId,
|
|
95
|
-
mode: dryRun ? "dry-run" : "live",
|
|
96
|
-
study: plan.study,
|
|
97
|
-
warnings: plan.warnings,
|
|
82
|
+
prefix: "scripted",
|
|
98
83
|
// Only a clone subject is served from a sandbox; an app-url run drives a local browser.
|
|
99
84
|
sandboxes: setup.clone ? undefined : "none",
|
|
100
85
|
renderReview: renderScriptedReviewMarkdown,
|
|
101
|
-
|
|
86
|
+
secrets: setup.secrets,
|
|
102
87
|
});
|
|
103
88
|
if (!started.ok) {
|
|
104
89
|
return setup.failed(started.code, started.message);
|
|
105
90
|
}
|
|
106
91
|
const { run } = started;
|
|
107
|
-
const {
|
|
92
|
+
const { paths: runPaths, source } = run;
|
|
108
93
|
const artifactRoot = runPaths.physicalRunRoot;
|
|
109
|
-
const source = await buildRunSource({
|
|
110
|
-
capturedAt: createdAt,
|
|
111
|
-
cwd: setup.physicalCwd,
|
|
112
|
-
humanishSource: "present",
|
|
113
|
-
packageName: "humanish",
|
|
114
|
-
});
|
|
115
94
|
const scriptedSubject = setup.clone
|
|
116
95
|
? new ScriptedSubject({
|
|
117
96
|
plan,
|
|
@@ -124,7 +103,7 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
124
103
|
timeoutMs: setup.timeoutMs,
|
|
125
104
|
subjectEnvNames: setup.subjectEnvNames,
|
|
126
105
|
hasGithubToken: setup.hasGithubToken,
|
|
127
|
-
scrubKnownValues:
|
|
106
|
+
scrubKnownValues: run.secrets.scrub,
|
|
128
107
|
now: setup.deps.now ?? Date.now,
|
|
129
108
|
warnings: setup.warnings,
|
|
130
109
|
})
|
|
@@ -156,10 +135,10 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
156
135
|
}
|
|
157
136
|
// The session itself maps launch failures to harness_error; reaching here means the
|
|
158
137
|
// harness around it failed. Redacted at this boundary before persisting anywhere.
|
|
159
|
-
sessionError = redactText(
|
|
138
|
+
sessionError = redactText(run.secrets.scrub(toErrorMessage(error)));
|
|
160
139
|
}
|
|
161
140
|
finally {
|
|
162
|
-
await scriptedSubject?.
|
|
141
|
+
await scriptedSubject?.sandbox.release();
|
|
163
142
|
}
|
|
164
143
|
await writeSurfaceTraces(runPaths, sessionResults);
|
|
165
144
|
}
|
|
@@ -182,7 +161,3 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
182
161
|
scriptedSubject,
|
|
183
162
|
});
|
|
184
163
|
}
|
|
185
|
-
function makeScriptedRunId() {
|
|
186
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
187
|
-
return `scripted-${stamp}-${randomBytes(4).toString("hex")}`;
|
|
188
|
-
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ActorPersonaRef } from "../../actors/contract.js";
|
|
2
2
|
import type { BrowserPersonaJourney, ScriptedBrowserEvidenceUrlPolicy } from "../../actors/scripted-browser/types.js";
|
|
3
3
|
import type { ScriptedPlan } from "../../study/plan-types.js";
|
|
4
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
4
5
|
import { type ScriptedBrowserStudyResult, type ScriptedRunInput } from "./types.js";
|
|
5
6
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
6
7
|
/** What setup hands the run: the plan's subject and scenario, keys and the browser to launch. */
|
|
@@ -20,7 +21,7 @@ export interface ScriptedRunSetup {
|
|
|
20
21
|
e2bApiKey: string;
|
|
21
22
|
hasGithubToken: boolean;
|
|
22
23
|
redactRepoLabel: boolean;
|
|
23
|
-
|
|
24
|
+
secrets: RunSecrets;
|
|
24
25
|
/** The loopback app URL, or the clone's serve URL until its getHost URL replaces it. */
|
|
25
26
|
appUrl: string;
|
|
26
27
|
scenario: {
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
// What a scripted run needs before it starts: the checks the plan cannot make (the scenario file,
|
|
2
2
|
// the E2B key and subject env of a live clone, a browser to launch) and the values the run reads
|
|
3
|
-
// (the evidence URL policy, the
|
|
3
|
+
// (the evidence URL policy, the run's known secrets, the persona and the session budget).
|
|
4
4
|
import { realpath } from "node:fs/promises";
|
|
5
|
-
import {
|
|
5
|
+
import { firstLiveRefusal, keysCheck, subjectEnvCheck } from "../../study/requirements.js";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-command.js";
|
|
8
|
-
import { describeMissingKeys } from "../../keys/key-resolution.js";
|
|
9
8
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
9
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
10
10
|
import { evidenceAppUrlOf } from "./plan.js";
|
|
11
11
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
12
12
|
import {} from "./types.js";
|
|
13
|
-
import {
|
|
13
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
14
14
|
// Journey wall-clock budget per surface: 5 minutes. A scripted surface has zero model cost and
|
|
15
15
|
// sandbox-seconds are pennies; a short default only truncated slow-loading subjects.
|
|
16
16
|
const DEFAULT_SESSION_TIMEOUT_MS = 300_000;
|
|
@@ -27,17 +27,12 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
27
27
|
const warnings = [];
|
|
28
28
|
const clone = plan.subject.kind === "clone" ? plan.subject : undefined;
|
|
29
29
|
const evidenceAppUrl = evidenceAppUrlOf(plan.subject);
|
|
30
|
-
const failed = (code, message) => ({
|
|
31
|
-
...studyResultIdentity("scripted", plan.studyId),
|
|
32
|
-
ok: false,
|
|
33
|
-
cwd,
|
|
30
|
+
const failed = (code, message) => refusedResult("scripted", { studyId: plan.studyId, cwd, warnings, error: { code, message } }, {
|
|
34
31
|
actor: plan.actor,
|
|
35
32
|
appUrl: evidenceAppUrl,
|
|
36
33
|
dryRun,
|
|
37
34
|
runId: input.runId ?? "not-created",
|
|
38
35
|
sessions: [],
|
|
39
|
-
warnings,
|
|
40
|
-
error: { code, message },
|
|
41
36
|
});
|
|
42
37
|
const urlPolicy = clone
|
|
43
38
|
? { kind: "provisioned-subject", evidenceOrigin: evidenceAppUrl }
|
|
@@ -47,11 +42,11 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
47
42
|
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
48
43
|
const hasGithubToken = subjectEnvNames.includes("GITHUB_TOKEN");
|
|
49
44
|
const redactRepoLabel = plan.residual.policies?.redactRepos ?? hasGithubToken;
|
|
50
|
-
const
|
|
45
|
+
const secrets = new RunSecrets([
|
|
46
|
+
e2bApiKey,
|
|
51
47
|
...(clone ? [clone.repo] : []),
|
|
52
48
|
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
53
|
-
]
|
|
54
|
-
const scrubKnownValues = (text) => scrubSourceValues.reduce((acc, value) => acc.split(value).join("[redacted]"), text);
|
|
49
|
+
]);
|
|
55
50
|
// A clone's URL is replaced by its getHost URL once it is served.
|
|
56
51
|
let appUrl = plan.subject.kind === "clone" ? plan.subject.serve.url : plan.subject.appUrl;
|
|
57
52
|
const scenario = await resolveScriptedScenario(projectRoot, plan.scenarioRef);
|
|
@@ -63,20 +58,18 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
63
58
|
}
|
|
64
59
|
const journey = scenario.journey;
|
|
65
60
|
// The plan lists E2B_API_KEY and the subject env only for a live clone.
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
};
|
|
79
|
-
}
|
|
61
|
+
const { requirements } = plan;
|
|
62
|
+
const refusal = await firstLiveRefusal([
|
|
63
|
+
() => keysCheck({
|
|
64
|
+
requirements,
|
|
65
|
+
env,
|
|
66
|
+
code: "HUMANISH_SCRIPTED_KEYS_MISSING",
|
|
67
|
+
need: (names) => `Live clone scripted-browser studies require ${names} (dry-run remains $0 and does not provision a subject).`,
|
|
68
|
+
}),
|
|
69
|
+
() => subjectEnvCheck({ requirements, env, code: "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" }),
|
|
70
|
+
]);
|
|
71
|
+
if (refusal)
|
|
72
|
+
return { ok: false, result: failed(refusal.code, refusal.message) };
|
|
80
73
|
const surfaces = plan.surfaces;
|
|
81
74
|
const timeoutMs = plan.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
|
|
82
75
|
const persona = {
|
|
@@ -114,7 +107,7 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
114
107
|
e2bApiKey,
|
|
115
108
|
hasGithubToken,
|
|
116
109
|
redactRepoLabel,
|
|
117
|
-
|
|
110
|
+
secrets,
|
|
118
111
|
appUrl,
|
|
119
112
|
scenario,
|
|
120
113
|
journey,
|