humanish 0.111.2 → 0.111.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +32 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/cli/findings.d.ts +15 -1
- package/dist/cli/findings.js +43 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +14 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +71 -35
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +8 -5
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +35 -0
- package/dist/run/secrets.js +158 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +89 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/docs/contracts/study-analysis.md +6 -2
- package/package.json +1 -1
|
@@ -116,7 +116,7 @@ export async function drainExternalComms(ctx, comms) {
|
|
|
116
116
|
email: comms.email,
|
|
117
117
|
env: ctx.env,
|
|
118
118
|
runPaths: ctx.runPaths,
|
|
119
|
-
knownSecretValues: ctx.
|
|
119
|
+
knownSecretValues: ctx.run.secrets.values(),
|
|
120
120
|
});
|
|
121
121
|
ctx.warnings.push(...warnings);
|
|
122
122
|
return path;
|
|
@@ -136,7 +136,7 @@ export function receivingSourceOf(residual, env) {
|
|
|
136
136
|
* when setup fails.
|
|
137
137
|
*/
|
|
138
138
|
export async function prepareEmailReceiving(args) {
|
|
139
|
-
const { source,
|
|
139
|
+
const { source, secrets } = args;
|
|
140
140
|
if (args.dryRun || source.comms?.email?.kind !== "real")
|
|
141
141
|
return { ok: true, receiving: undefined };
|
|
142
142
|
try {
|
|
@@ -147,11 +147,7 @@ export async function prepareEmailReceiving(args) {
|
|
|
147
147
|
env: args.env,
|
|
148
148
|
participants: args.participants,
|
|
149
149
|
runPaths: args.runPaths,
|
|
150
|
-
registerSecrets: (values) =>
|
|
151
|
-
for (const value of values)
|
|
152
|
-
if (value.length >= 4 && !knownSecretValues.includes(value))
|
|
153
|
-
knownSecretValues.push(value);
|
|
154
|
-
},
|
|
150
|
+
registerSecrets: (values) => secrets.add(values),
|
|
155
151
|
});
|
|
156
152
|
return { ok: true, receiving };
|
|
157
153
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { DesktopParticipantRun, ParticipantRunOutcome } from "../computer-use/types.js";
|
|
2
|
-
import {
|
|
2
|
+
import type { DesktopParticipantDeps } from "../computer-use/participant-deps.js";
|
|
3
3
|
import type { ActorRunResult, ExternalCommsWiring } from "./types.js";
|
|
4
4
|
/** What the host and follower participants need besides the handoff. */
|
|
5
5
|
export interface HandoffParticipantDeps {
|
|
6
|
-
runDeps:
|
|
6
|
+
runDeps: DesktopParticipantDeps;
|
|
7
7
|
publicAppUrl: string;
|
|
8
8
|
inbox: ExternalCommsWiring | undefined;
|
|
9
9
|
now: () => number;
|
|
@@ -14,7 +14,7 @@ import { withInboxMission } from "../computer-use/participant-prompt.js";
|
|
|
14
14
|
import { runCuaParticipant } from "../computer-use/participant-execution.js";
|
|
15
15
|
import { extractLobbyCode, extractLobbyCodeFromNarration } from "./lobby-code.js";
|
|
16
16
|
import { hostOriginDigest } from "./provenance.js";
|
|
17
|
-
import { makeBlockedFollowerOutcome, withLobbyCodeMission
|
|
17
|
+
import { makeBlockedFollowerOutcome, withLobbyCodeMission } from "./participant-specs.js";
|
|
18
18
|
// The floor for the host-first handoff barrier deadline (ms). The host must surface a
|
|
19
19
|
// shared-session (/lobby/CODE) URL within the deadline or the run fails closed and no follower
|
|
20
20
|
// opens. The effective deadline scales with the per-participant run budget (execution.timeoutMs): a
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { SharedWorldPlan } from "../../study/plan-types.js";
|
|
2
2
|
import type { Participant, SharedWorldParticipant } from "../../study/plan-participants.js";
|
|
3
3
|
import type { RunBundle } from "../../run/bundle.js";
|
|
4
|
-
import type {
|
|
4
|
+
import type { DesktopParticipantRun, ParticipantRunOutcome } from "../computer-use/types.js";
|
|
5
|
+
import { type DesktopParticipantDeps } from "../computer-use/participant-deps.js";
|
|
5
6
|
import type { LiveParticipants, PlaneContext } from "./types.js";
|
|
6
7
|
export declare function defaultSessionTimeoutMs(plan: SharedWorldPlan): number;
|
|
7
8
|
/** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
|
|
@@ -16,8 +17,6 @@ export declare function withLobbyCodeMission(spec: DesktopParticipantRun, code:
|
|
|
16
17
|
/** A follower blocked by an expired deadline or an ended host. It never opened a browser;
|
|
17
18
|
* keep the actual reason rather than turning every upstream failure into a timeout. */
|
|
18
19
|
export declare function makeBlockedFollowerOutcome(spec: DesktopParticipantRun, reason: string, timedOut: boolean): ParticipantRunOutcome;
|
|
19
|
-
/** The computer-use runner deps every participant shares, on either plane. */
|
|
20
|
-
export type ParticipantRunDeps = Omit<CuaParticipantDeps, "signalProvisioned" | "appUrl" | "onObservedUrl">;
|
|
21
20
|
/**
|
|
22
21
|
* A live run publishes an in-progress bundle before its participants start, whether or not an
|
|
23
22
|
* Observer is attached, and the participants' live traces rewrite it as they go, as on the
|
|
@@ -30,9 +29,9 @@ export declare function startParticipantFlush(ctx: PlaneContext, live: LiveParti
|
|
|
30
29
|
* out of every actor sandbox: the subject env names' values, the GitHub token, and clone
|
|
31
30
|
* or local-tree provisioning. The committed subject.envValues still reach it, non-secret by parse.
|
|
32
31
|
* `scrubKnownValues` is the plane's scrub: the external-public plane also scrubs the latched lobby
|
|
33
|
-
* code.
|
|
32
|
+
* code. A plane calls this once per run, since the deps carry the run's one budget.
|
|
34
33
|
*/
|
|
35
|
-
export declare function participantRunDeps(ctx: PlaneContext, live: LiveParticipants, scrubKnownValues: (text: string) => string):
|
|
34
|
+
export declare function participantRunDeps(ctx: PlaneContext, live: LiveParticipants, scrubKnownValues: (text: string) => string): DesktopParticipantDeps;
|
|
36
35
|
/**
|
|
37
36
|
* Each participant's actor spec, with committed personas compiled in so each prompt carries real
|
|
38
37
|
* behavioral directives. Evidence copies of the assignment, instructions and persona are
|
|
@@ -7,6 +7,7 @@ import { resolveParticipant } from "../../run/participant.js";
|
|
|
7
7
|
import { attachObserverRuntimeStreamUrls } from "../../observer/render.js";
|
|
8
8
|
import { composeParticipantInstructions } from "../computer-use/participant-prompt.js";
|
|
9
9
|
import { startLiveTraceFlush } from "../computer-use/live-flush.js";
|
|
10
|
+
import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
10
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
11
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
12
13
|
import { redactText } from "../../evidence/redaction.js";
|
|
@@ -143,26 +144,25 @@ function participantStreams(onStream, live) {
|
|
|
143
144
|
* out of every actor sandbox: the subject env names' values, the GitHub token, and clone
|
|
144
145
|
* or local-tree provisioning. The committed subject.envValues still reach it, non-secret by parse.
|
|
145
146
|
* `scrubKnownValues` is the plane's scrub: the external-public plane also scrubs the latched lobby
|
|
146
|
-
* code.
|
|
147
|
+
* code. A plane calls this once per run, since the deps carry the run's one budget.
|
|
147
148
|
*/
|
|
148
149
|
export function participantRunDeps(ctx, live, scrubKnownValues) {
|
|
149
|
-
const { config,
|
|
150
|
-
return {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
150
|
+
const { config, plan } = ctx;
|
|
151
|
+
return desktopParticipantDeps({
|
|
152
|
+
plan,
|
|
153
|
+
// A local-agent brain runs each participant on the operator's signed-in agent, as on
|
|
154
|
+
// computer use.
|
|
155
|
+
brain: plan.brain,
|
|
155
156
|
subject: {
|
|
156
157
|
kind: "shared-app",
|
|
157
158
|
...(config.subject.serve?.url === undefined ? {} : { serveUrl: config.subject.serve.url }),
|
|
158
159
|
},
|
|
159
|
-
env,
|
|
160
|
+
env: ctx.env,
|
|
160
161
|
openaiApiKey: ctx.openaiApiKey,
|
|
161
162
|
e2bApiKey: ctx.e2bApiKey,
|
|
162
|
-
requestTimeoutMs: ctx.requestTimeoutMs,
|
|
163
|
-
sandboxMs: ctx.timeoutMs + SANDBOX_TIMEOUT_BUFFER_MS,
|
|
164
163
|
timeoutMs: ctx.timeoutMs,
|
|
165
|
-
|
|
164
|
+
sandboxMs: ctx.timeoutMs + SANDBOX_TIMEOUT_BUFFER_MS,
|
|
165
|
+
participantCount: plan.plane.participants.length,
|
|
166
166
|
artifactRoot: ctx.runPaths,
|
|
167
167
|
studyCwd: ctx.cwd,
|
|
168
168
|
redactScreenshots: ctx.redactScreenshots,
|
|
@@ -172,27 +172,21 @@ export function participantRunDeps(ctx, live, scrubKnownValues) {
|
|
|
172
172
|
ctx.run.participantStarted();
|
|
173
173
|
return ctx.runSession(options);
|
|
174
174
|
},
|
|
175
|
-
|
|
176
|
-
// use.
|
|
177
|
-
brain: ctx.plan.brain,
|
|
178
|
-
...(receiving ? { receiving } : {}),
|
|
179
|
-
now: ctx.now,
|
|
180
|
-
// The participant's desktop seams, and the caller's prepareDesktop with the participant as its
|
|
181
|
-
// target.
|
|
182
|
-
...(ctx.deps.desktopModule === undefined ? {} : { desktopModule: ctx.deps.desktopModule }),
|
|
183
|
-
...(ctx.deps.detachedTimers === undefined ? {} : { detachedTimers: ctx.deps.detachedTimers }),
|
|
184
|
-
...(ctx.input.prepareDesktop === undefined ? {} : { prepareDesktop: ctx.input.prepareDesktop }),
|
|
175
|
+
liveFlush: () => live.flush?.flush,
|
|
185
176
|
onStream: participantStreams(ctx.input.onStream, live),
|
|
186
|
-
// A participant visits the shared app and provisions no subject, so it reports no phase of
|
|
187
|
-
// own.
|
|
177
|
+
// A participant visits the shared app and provisions no subject, so it reports no phase of
|
|
178
|
+
// its own.
|
|
188
179
|
reportSubjectPhase: defaultSubjectPhaseSink,
|
|
189
|
-
|
|
180
|
+
seams: ctx.deps,
|
|
181
|
+
prepareDesktop: ctx.input.prepareDesktop,
|
|
182
|
+
}, {
|
|
183
|
+
receiving: ctx.receiving,
|
|
190
184
|
// Concurrent participants are independent evidence: a requested-vs-verified screen
|
|
191
185
|
// mismatch is recorded as separate facts + a warning instead of failing the participant's
|
|
192
186
|
// device claim closed, so one participant's window-manager drift cannot abort the whole
|
|
193
187
|
// live multi-actor world (the single-participant/fan-out routes keep fail-closed).
|
|
194
188
|
screenMismatchPolicy: "record-evidence",
|
|
195
|
-
};
|
|
189
|
+
});
|
|
196
190
|
}
|
|
197
191
|
/**
|
|
198
192
|
* Each participant's actor spec, with committed personas compiled in so each prompt carries real
|
|
@@ -11,7 +11,7 @@ import { sharedWorldParticipants } from "../../study/plan-participants.js";
|
|
|
11
11
|
import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
13
|
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
14
|
-
import {
|
|
14
|
+
import { unpricedCapCheck } from "../../study/requirements.js";
|
|
15
15
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
16
16
|
export function sharedWorldDescriptorOf(actorType) {
|
|
17
17
|
const descriptor = actorRegistry[actorType];
|
|
@@ -60,12 +60,16 @@ export function planSharedWorldStudy(config, input) {
|
|
|
60
60
|
return refuse(invalid, invalidReason, actor);
|
|
61
61
|
if (config.actor?.maxOutputTokens !== undefined && input.hasRunSession === true)
|
|
62
62
|
return refuse(invalid, "maxOutputTokens cannot be enforced by a custom runSession.", actor);
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
63
|
+
// A live cap is priced here, at plan time, so its refusal wins over the route's key checks.
|
|
64
|
+
const unpriced = input.dryRun
|
|
65
|
+
? undefined
|
|
66
|
+
: unpricedCapCheck({
|
|
67
|
+
caps: config.caps ?? {},
|
|
68
|
+
model: (config.actor?.model ?? DEFAULT_OPENAI_CU_MODEL).trim().toLowerCase(),
|
|
69
|
+
code: "HUMANISH_SHARED_WORLD_UNPRICED_CAP",
|
|
70
|
+
});
|
|
71
|
+
if (unpriced)
|
|
72
|
+
return refuse(unpriced.code, unpriced.message, actor);
|
|
69
73
|
const receivingReason = receivingEmailValidationReason(config);
|
|
70
74
|
if (receivingReason)
|
|
71
75
|
return refuse(invalid, receivingReason, actor);
|
|
@@ -3,8 +3,9 @@ import type { RunSubjectStateStepRecord } from "../../run/bundle.js";
|
|
|
3
3
|
import type { SharedWorldStateSnapshot } from "../../run/shared-world-evidence.js";
|
|
4
4
|
import type { LocalTreeArchive } from "../../subject/local-tree-archive.js";
|
|
5
5
|
import type { DetachedTimers } from "../../substrates/detached.js";
|
|
6
|
+
import { type SubjectDesktopUsage } from "../../substrates/e2b/subject-sandbox.js";
|
|
6
7
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
7
|
-
import { type ActorRunResult, type
|
|
8
|
+
import { type ActorRunResult, type LiveParticipants, type PlaneContext } from "./types.js";
|
|
8
9
|
/** What the provisioned plane needs besides the shared plane context. */
|
|
9
10
|
export interface ProvisionedPlaneSetup {
|
|
10
11
|
serve: StudySubjectServe;
|
|
@@ -12,9 +12,9 @@ import { provisionLocalTreeSubject } from "../../subject/local-tree.js";
|
|
|
12
12
|
import { defaultSharedWorldPhaseSink } from "../../subject/steps.js";
|
|
13
13
|
import { phaseEvent } from "../../study/run-study-events.js";
|
|
14
14
|
import { loadE2BDesktopModule } from "../../substrates/e2b/sdk.js";
|
|
15
|
-
import {
|
|
16
|
-
import { acquireE2BDesktopSandbox, readE2BRelease } from "../../substrates/e2b/sandbox.js";
|
|
15
|
+
import { E2BSubjectSandbox, } from "../../substrates/e2b/subject-sandbox.js";
|
|
17
16
|
import { e2bShell } from "../../substrates/e2b/shell.js";
|
|
17
|
+
import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
18
18
|
import { defaultPackLocalTree } from "../computer-use/local-tree-pack.js";
|
|
19
19
|
import { inboxRecipientFor, participantHasInboxRecipient, } from "../computer-use/participant-desktop.js";
|
|
20
20
|
import { resolveSubjectState } from "../computer-use/subject-projection.js";
|
|
@@ -25,8 +25,6 @@ import { buildConcurrentSharedWorldBundle, judgeSharedWorldRun } from "./bundle.
|
|
|
25
25
|
import { runCheckpointSnapshot } from "./checkpoints.js";
|
|
26
26
|
import { drainSubjectComms } from "./comms.js";
|
|
27
27
|
import { buildSubjectProvenance, hostOriginDigest, isTokenlessHost, servePort, } from "./provenance.js";
|
|
28
|
-
import { SANDBOX_TIMEOUT_BUFFER_MS, SUBJECT_PROVISION_BUDGET_MS, } from "../../substrates/e2b/lifetime.js";
|
|
29
|
-
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
30
28
|
import { resolveActorEntryUrl, participantRunDeps, startParticipantFlush, } from "./participant-specs.js";
|
|
31
29
|
import { CONCURRENT_SHARED_WORLD_PROVIDER_METADATA, } from "./types.js";
|
|
32
30
|
import { addressedRecipients } from "../../study/parse/comms.js";
|
|
@@ -46,17 +44,10 @@ class ObserverGateError extends Error {
|
|
|
46
44
|
class SubjectPlane {
|
|
47
45
|
commsInboxUrl;
|
|
48
46
|
subjectCommit;
|
|
49
|
-
|
|
50
|
-
subjectKilled = false;
|
|
51
|
-
subjectReleaseWarning;
|
|
47
|
+
subject;
|
|
52
48
|
getHostUrl;
|
|
53
|
-
// The subject desktop's host-side span and size, priced in the run's cost estimate.
|
|
54
|
-
subjectCreatedAtMs;
|
|
55
|
-
subjectTornDownAtMs;
|
|
56
|
-
subjectResources;
|
|
57
49
|
ctx;
|
|
58
50
|
setup;
|
|
59
|
-
subjectAllocation;
|
|
60
51
|
subjectDesktop;
|
|
61
52
|
subjectShell;
|
|
62
53
|
// The in-sandbox email catch on the one subject sandbox; drained at teardown. Undefined
|
|
@@ -77,6 +68,11 @@ class SubjectPlane {
|
|
|
77
68
|
constructor(ctx, setup) {
|
|
78
69
|
this.ctx = ctx;
|
|
79
70
|
this.setup = setup;
|
|
71
|
+
this.subject = new E2BSubjectSandbox({
|
|
72
|
+
warnings: ctx.warnings,
|
|
73
|
+
scrub: ctx.scrubKnownValues,
|
|
74
|
+
now: ctx.now,
|
|
75
|
+
});
|
|
80
76
|
this.disposeSignal = new Promise((resolve) => {
|
|
81
77
|
this.releaseDispose = resolve;
|
|
82
78
|
});
|
|
@@ -106,46 +102,35 @@ class SubjectPlane {
|
|
|
106
102
|
// The one subject sandbox: headless service host (no GUI participant). The subject env is
|
|
107
103
|
// provisioned here; the actor sandboxes get none of it. A custom desktop template (image) is
|
|
108
104
|
// honored on both the subject sandbox (here) and every actor sandbox (via runCuaParticipant, which
|
|
109
|
-
// reads the same config); absent keeps the byte-stable Sandbox.create(opts) default.
|
|
110
|
-
|
|
111
|
-
const subject = await acquireE2BDesktopSandbox({
|
|
105
|
+
// reads the same config); absent keeps the byte-stable Sandbox.create(opts) default.
|
|
106
|
+
this.subjectDesktop = await this.subject.acquire({
|
|
112
107
|
module: subjectModule,
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
...(subjectEnvNames.length > 0 || Object.keys(commsEnv).length > 0
|
|
129
|
-
? {
|
|
130
|
-
envs: {
|
|
131
|
-
...Object.fromEntries(subjectEnvNames.map((name) => [name, env[name]])),
|
|
132
|
-
...commsEnv,
|
|
133
|
-
},
|
|
134
|
-
}
|
|
135
|
-
: {}),
|
|
136
|
-
dpi: 96,
|
|
137
|
-
lifecycle: { onTimeout: "kill" },
|
|
108
|
+
apiKey: this.ctx.e2bApiKey,
|
|
109
|
+
requestTimeoutMs,
|
|
110
|
+
sessionTimeoutMs: timeoutMs,
|
|
111
|
+
seed: planeStateOf(plan)?.seed ?? [],
|
|
112
|
+
metadata: {
|
|
113
|
+
...CONCURRENT_SHARED_WORLD_PROVIDER_METADATA,
|
|
114
|
+
labId: plan.studyId,
|
|
115
|
+
topology: "shared-world",
|
|
116
|
+
topologyMode: "concurrent",
|
|
117
|
+
kind: "subject",
|
|
118
|
+
participantCount: String(plan.plane.participants.length),
|
|
119
|
+
},
|
|
120
|
+
envs: {
|
|
121
|
+
...Object.fromEntries(subjectEnvNames.map((name) => [name, env[name]])),
|
|
122
|
+
...commsEnv,
|
|
138
123
|
},
|
|
139
124
|
template: plan.residual.execution?.desktop?.template,
|
|
140
|
-
|
|
125
|
+
root: this.ctx.runPaths,
|
|
141
126
|
});
|
|
142
|
-
this.subjectDesktop = subject.sandbox;
|
|
143
127
|
this.subjectShell = e2bShell(this.subjectDesktop);
|
|
144
|
-
this.
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
128
|
+
if (this.ctx.input.prepareDesktop) {
|
|
129
|
+
await this.ctx.input.prepareDesktop(this.subjectDesktop, { kind: "subject" });
|
|
130
|
+
// The caller's hook runs with this process's file access, so the run directory is checked
|
|
131
|
+
// again before more evidence goes into it.
|
|
132
|
+
await validatePreparedRunArtifactPaths(this.ctx.runPaths);
|
|
133
|
+
}
|
|
149
134
|
}
|
|
150
135
|
// Start the in-sandbox email catch before the subject serve, so the app's send-API base URL
|
|
151
136
|
// (injected into its env at create) resolves the moment it boots. Fail closed if the catch can't
|
|
@@ -320,7 +305,6 @@ class SubjectPlane {
|
|
|
320
305
|
* Returns the comms thread's path when the drain wrote one.
|
|
321
306
|
*/
|
|
322
307
|
async teardown() {
|
|
323
|
-
const { warnings, scrubKnownValues } = this.ctx;
|
|
324
308
|
this.proberDisposed = true;
|
|
325
309
|
this.releaseDispose();
|
|
326
310
|
if (this.proberLoop) {
|
|
@@ -339,36 +323,9 @@ class SubjectPlane {
|
|
|
339
323
|
if (commsEmail && this.deployedComms?.ready && this.subjectShell) {
|
|
340
324
|
commsArtifactPath = await drainSubjectComms(this.ctx, commsEmail, this.subjectShell, this.deployedComms);
|
|
341
325
|
}
|
|
342
|
-
|
|
343
|
-
const released = await this.subjectAllocation.close();
|
|
344
|
-
const reading = readE2BRelease(released, {
|
|
345
|
-
label: "Subject sandbox",
|
|
346
|
-
scrub: scrubKnownValues,
|
|
347
|
-
costSpan: true,
|
|
348
|
-
});
|
|
349
|
-
this.subjectKilled = reading.released;
|
|
350
|
-
if (reading.warning)
|
|
351
|
-
warnings.push(reading.warning);
|
|
352
|
-
if (!reading.released)
|
|
353
|
-
this.subjectReleaseWarning = reading.warning ?? "Subject sandbox release is unconfirmed.";
|
|
354
|
-
// Without a kill method, or in E2B debug mode, no kill reached E2B, so there is no teardown
|
|
355
|
-
// time to record.
|
|
356
|
-
if (released.status !== "unconfirmed" || released.reason !== "release_unavailable")
|
|
357
|
-
this.subjectTornDownAtMs = this.ctx.now();
|
|
358
|
-
}
|
|
326
|
+
await this.subject.release();
|
|
359
327
|
return commsArtifactPath;
|
|
360
328
|
}
|
|
361
|
-
desktopUsage() {
|
|
362
|
-
if (this.subjectSandboxId === undefined)
|
|
363
|
-
return undefined;
|
|
364
|
-
return {
|
|
365
|
-
durationMs: this.subjectCreatedAtMs === undefined || this.subjectTornDownAtMs === undefined
|
|
366
|
-
? undefined
|
|
367
|
-
: Math.max(0, this.subjectTornDownAtMs - this.subjectCreatedAtMs),
|
|
368
|
-
observation: this.subjectResources,
|
|
369
|
-
killed: this.subjectKilled,
|
|
370
|
-
};
|
|
371
|
-
}
|
|
372
329
|
}
|
|
373
330
|
/** Writes the in-progress bundle, runs the caller's Observer gate and starts the participants'
|
|
374
331
|
* flush. */
|
|
@@ -476,10 +433,10 @@ export async function runProvisionedPlane(ctx, live, setup) {
|
|
|
476
433
|
actorResults,
|
|
477
434
|
runError,
|
|
478
435
|
subjectCommit: plane.subjectCommit,
|
|
479
|
-
subjectSandboxId: plane.
|
|
480
|
-
subjectKilled: plane.
|
|
481
|
-
subjectReleaseWarning: plane.
|
|
482
|
-
subjectDesktop: plane.desktopUsage(),
|
|
436
|
+
subjectSandboxId: plane.subject.sandboxId,
|
|
437
|
+
subjectKilled: plane.subject.killed,
|
|
438
|
+
subjectReleaseWarning: plane.subject.releaseWarning,
|
|
439
|
+
subjectDesktop: plane.subject.desktopUsage(),
|
|
483
440
|
getHostUrl: plane.getHostUrl,
|
|
484
441
|
commsArtifactPath,
|
|
485
442
|
};
|
|
@@ -3,14 +3,16 @@
|
|
|
3
3
|
import { redactText } from "../../evidence/redaction.js";
|
|
4
4
|
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
5
5
|
import { attachObserverRuntimeStreamUrls } from "../../observer/render.js";
|
|
6
|
-
import { foldScorerFailures, judgeExecution, sandboxCleanupFailure, OUTCOME_POLICIES,
|
|
7
|
-
import {
|
|
6
|
+
import { foldScorerFailures, judgeExecution, sandboxCleanupFailure, OUTCOME_POLICIES, resultOk, sharedWorldShortfall, } from "../../run/judge.js";
|
|
7
|
+
import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
8
|
+
import { participantExecutionFailures } from "../computer-use/participant-facts.js";
|
|
9
|
+
import { participantCapWarning } from "../computer-use/participant-model.js";
|
|
8
10
|
import { resolveSubjectState } from "../computer-use/subject-projection.js";
|
|
9
11
|
import { actorRunPassed, buildConcurrentSharedWorldBundle, judgeSharedWorldRun, maxSimultaneousWindows, } from "./bundle.js";
|
|
10
12
|
import { planeStateOf } from "./plan.js";
|
|
11
13
|
import { buildSubjectProvenance, hostOriginDigest } from "./provenance.js";
|
|
12
14
|
import {} from "./types.js";
|
|
13
|
-
import { studyResultIdentity } from "../../run/study-result.js";
|
|
15
|
+
import { refusedResult, studyResultIdentity } from "../../run/study-result.js";
|
|
14
16
|
import { plural } from "../../run/text.js";
|
|
15
17
|
/** The results of a run whose plane did not run (a dry run) or has not reported yet. */
|
|
16
18
|
export function emptyPlaneResults() {
|
|
@@ -143,10 +145,7 @@ function concurrentStudyError(args) {
|
|
|
143
145
|
* and the run id the caller asked for, if any. `actor` is the label when a refusal names none.
|
|
144
146
|
*/
|
|
145
147
|
export function concurrentStudyFailure(envelope) {
|
|
146
|
-
return (code, message, actorLabel) => ({
|
|
147
|
-
...studyResultIdentity("shared-world", envelope.studyId),
|
|
148
|
-
ok: false,
|
|
149
|
-
cwd: envelope.cwd,
|
|
148
|
+
return (code, message, actorLabel) => refusedResult("shared-world", { studyId: envelope.studyId, cwd: envelope.cwd, error: { code, message } }, {
|
|
150
149
|
actor: actorLabel ?? envelope.actor,
|
|
151
150
|
topology: "shared-world",
|
|
152
151
|
topologyMode: "concurrent",
|
|
@@ -155,43 +154,18 @@ export function concurrentStudyFailure(envelope) {
|
|
|
155
154
|
dryRun: envelope.dryRun,
|
|
156
155
|
runId: envelope.runId ?? "not-created",
|
|
157
156
|
roles: [],
|
|
158
|
-
warnings: [],
|
|
159
|
-
error: { code, message },
|
|
160
157
|
});
|
|
161
158
|
}
|
|
162
159
|
/**
|
|
163
|
-
* The run's execution failures before the Observer renders: a run error (the handoff, the plane)
|
|
164
|
-
*
|
|
165
|
-
* or that reported a disallowed item after its last request, and each sandbox whose release is
|
|
160
|
+
* The run's execution failures before the Observer renders: a run error (the handoff, the plane)
|
|
161
|
+
* first, then the participants' failures, then the subject sandbox when its release is
|
|
166
162
|
* unconfirmed. FinishedRun.renderObserver adds an Observer that did not render.
|
|
167
163
|
*/
|
|
168
164
|
function sharedWorldExecutionFailures(args) {
|
|
169
165
|
const { runId, runError, actorResults, subject } = args;
|
|
170
166
|
return [
|
|
171
167
|
...(runError === undefined ? [] : [{ kind: "run", message: runError }]),
|
|
172
|
-
...actorResults
|
|
173
|
-
.filter((result) => participantHarnessFailed(participantFactsOf(result.outcome)))
|
|
174
|
-
.map((result) => ({
|
|
175
|
-
kind: "harness",
|
|
176
|
-
message: `${result.spec.planned.id}: ${result.outcome.sessionError ?? result.outcome.session?.reason ?? "harness error"}`,
|
|
177
|
-
})),
|
|
178
|
-
...actorResults.flatMap((result) => result.outcome.providerCleanupError === undefined
|
|
179
|
-
? []
|
|
180
|
-
: [
|
|
181
|
-
{
|
|
182
|
-
kind: "provider-cleanup",
|
|
183
|
-
message: `${result.spec.planned.id}: ${result.outcome.providerCleanupError}`,
|
|
184
|
-
},
|
|
185
|
-
]),
|
|
186
|
-
...actorResults.flatMap((result) => result.outcome.providerPolicyError === undefined
|
|
187
|
-
? []
|
|
188
|
-
: [
|
|
189
|
-
{
|
|
190
|
-
kind: "provider-policy",
|
|
191
|
-
message: `${result.spec.planned.id}: ${result.outcome.providerPolicyError}`,
|
|
192
|
-
},
|
|
193
|
-
]),
|
|
194
|
-
...unreleasedSandboxFailures(actorResults.map((result) => result.outcome), runId),
|
|
168
|
+
...participantExecutionFailures(actorResults.map((result) => result.outcome), runId),
|
|
195
169
|
...(subject.subjectSandboxId === undefined || subject.subjectKilled
|
|
196
170
|
? []
|
|
197
171
|
: [sandboxCleanupFailure("subject", subject.subjectReleaseWarning, runId)]),
|
|
@@ -230,6 +204,9 @@ export async function finishConcurrentRun(ctx, live, results, plane) {
|
|
|
230
204
|
for (const result of actorResults) {
|
|
231
205
|
warnings.push(...result.outcome.warnings);
|
|
232
206
|
}
|
|
207
|
+
const capWarning = participantCapWarning(plan.caps, participantCount, "each participant's model spend");
|
|
208
|
+
if (capWarning !== undefined)
|
|
209
|
+
warnings.push(capWarning);
|
|
233
210
|
const bundleArgs = {
|
|
234
211
|
run,
|
|
235
212
|
plan,
|
|
@@ -293,6 +270,11 @@ export async function finishConcurrentRun(ctx, live, results, plane) {
|
|
|
293
270
|
policy,
|
|
294
271
|
});
|
|
295
272
|
const observer = await finished.renderObserver();
|
|
273
|
+
// A caller's renderer runs with this process's file access, so after one that reports success
|
|
274
|
+
// the run directory is checked again before the result points anyone at it. A failed render is
|
|
275
|
+
// already an evidence failure, and its error code stays the run's diagnosis.
|
|
276
|
+
if (observer.ok)
|
|
277
|
+
await validatePreparedRunArtifactPaths(finished.paths);
|
|
296
278
|
if (observer.ok && live.observer) {
|
|
297
279
|
attachObserverRuntimeStreamUrls(observer, live.streamUrls);
|
|
298
280
|
}
|
|
@@ -33,15 +33,12 @@
|
|
|
33
33
|
// requires the author attestation subject.exposure: synthetic. This is author-trust + a provenance
|
|
34
34
|
// gate; humanish cannot tell synthetic from real data, so it makes no claim about real data.
|
|
35
35
|
import path from "node:path";
|
|
36
|
-
import {
|
|
37
|
-
import { scrubLiterals } from "../../evidence/redaction.js";
|
|
38
|
-
import { describeMissingKeys } from "../../keys/key-resolution.js";
|
|
36
|
+
import { firstLiveRefusal, keysCheck, localAgentCheck, subjectEnvCheck, } from "../../study/requirements.js";
|
|
39
37
|
import {} from "../../run/run.js";
|
|
38
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
40
39
|
import { admitRoute, completeRefusalAnalysis, } from "../../run/route-shell.js";
|
|
41
|
-
import { makeCuaRunBudget } from "../computer-use/participant-model.js";
|
|
42
40
|
import { runExternalPublicPlane } from "./external-public.js";
|
|
43
41
|
import { sharedWorldDescriptorOf } from "./plan.js";
|
|
44
|
-
import { localAgentRefusal } from "../../actors/local-agent/readiness.js";
|
|
45
42
|
import { runProvisionedPlane } from "./provisioned.js";
|
|
46
43
|
import { concurrentStudyFailure, finishConcurrentRun } from "./result.js";
|
|
47
44
|
import { prepareConcurrentRun } from "./setup.js";
|
|
@@ -99,29 +96,35 @@ async function admitSharedWorldRun(plan, input) {
|
|
|
99
96
|
if (plan.dryRun)
|
|
100
97
|
return admitted;
|
|
101
98
|
const env = input.env ?? process.env;
|
|
102
|
-
const
|
|
99
|
+
const { requirements } = plan;
|
|
100
|
+
// The plan prices a cap before these, and its refusal wins over theirs.
|
|
101
|
+
const refusal = await firstLiveRefusal([
|
|
102
|
+
// The plan lists OPENAI_API_KEY for an openai brain's participants and the external-public
|
|
103
|
+
// plane's lobby-code reader; a local-agent brain's participants run on the operator's
|
|
104
|
+
// signed-in agent instead. Only on the provisioned plane does a local agent remove the key,
|
|
105
|
+
// so only there does the refusal suggest one.
|
|
106
|
+
() => keysCheck({
|
|
107
|
+
requirements,
|
|
108
|
+
env,
|
|
109
|
+
code: "HUMANISH_SHARED_WORLD_KEYS_MISSING",
|
|
110
|
+
need: (names) => `Live concurrent shared-world studies need ${names} in the environment (values are never persisted).`,
|
|
111
|
+
suggestLocalAgent: plan.plane.kind === "provisioned",
|
|
112
|
+
}),
|
|
113
|
+
() => localAgentCheck({
|
|
114
|
+
brain: plan.brain,
|
|
115
|
+
env,
|
|
116
|
+
caps: plan.caps,
|
|
117
|
+
codes: LOCAL_AGENT_REFUSAL_CODES,
|
|
118
|
+
}),
|
|
119
|
+
() => subjectEnvCheck({ requirements, env, code: "HUMANISH_SHARED_WORLD_SUBJECT_ENV_MISSING" }),
|
|
120
|
+
]);
|
|
121
|
+
if (refusal === undefined)
|
|
122
|
+
return admitted;
|
|
123
|
+
const fail = sharedWorldFailure(plan, input);
|
|
124
|
+
return {
|
|
103
125
|
ok: false,
|
|
104
|
-
result:
|
|
105
|
-
}
|
|
106
|
-
// The plan lists OPENAI_API_KEY for an openai brain's participants and the external-public
|
|
107
|
-
// plane's lobby-code reader; a local-agent brain's participants run on the operator's signed-in
|
|
108
|
-
// agent instead.
|
|
109
|
-
const missing = missingKeys(plan.requirements, env);
|
|
110
|
-
if (missing.length > 0) {
|
|
111
|
-
return fail("HUMANISH_SHARED_WORLD_KEYS_MISSING", `Live concurrent shared-world studies need ${missing.join(" and ")} in the environment (values are never persisted). ${describeMissingKeys(missing, env)}`);
|
|
112
|
-
}
|
|
113
|
-
if (plan.brain.kind === "local-agent") {
|
|
114
|
-
// A missing or signed-out agent found after the participants' desktops are paid for is the same
|
|
115
|
-
// news at the worst moment.
|
|
116
|
-
const refusal = await localAgentRefusal({ agent: plan.brain.agent, env, caps: plan.caps });
|
|
117
|
-
if (refusal)
|
|
118
|
-
return fail(LOCAL_AGENT_REFUSAL_CODES[refusal.kind], refusal.message);
|
|
119
|
-
}
|
|
120
|
-
const unsetSubjectEnv = missingSubjectEnv(plan.requirements, env);
|
|
121
|
-
if (unsetSubjectEnv.length > 0) {
|
|
122
|
-
return fail("HUMANISH_SHARED_WORLD_SUBJECT_ENV_MISSING", `subject.env declares ${unsetSubjectEnv.join(", ")} but the environment does not provide ${unsetSubjectEnv.length === 1 ? "it" : "them"} (pass via --dotenv; values are never persisted).`);
|
|
123
|
-
}
|
|
124
|
-
return admitted;
|
|
126
|
+
result: fail(refusal.code, refusal.message, sharedWorldDescriptorOf(plan.actor).id),
|
|
127
|
+
};
|
|
125
128
|
}
|
|
126
129
|
/** The route's envelope for a run that stops before its bundle. */
|
|
127
130
|
function sharedWorldFailure(plan, input) {
|
|
@@ -136,15 +139,12 @@ function sharedWorldFailure(plan, input) {
|
|
|
136
139
|
});
|
|
137
140
|
}
|
|
138
141
|
async function runPlanInScope(plan, input, config, scope) {
|
|
139
|
-
const { dryRun } = plan;
|
|
140
142
|
const requestedCwd = path.resolve(input.cwd);
|
|
141
143
|
const deps = input.deps ?? {};
|
|
142
144
|
const env = input.env ?? process.env;
|
|
143
145
|
const fail = sharedWorldFailure(plan, input);
|
|
144
146
|
const descriptor = sharedWorldDescriptorOf(plan.actor);
|
|
145
147
|
const planeClass = plan.plane.kind === "external-public" ? "external-public" : "provisioned-getHost";
|
|
146
|
-
const { maxTotalUsd } = plan.caps;
|
|
147
|
-
const runBudget = !dryRun && maxTotalUsd !== undefined ? makeCuaRunBudget(maxTotalUsd) : undefined;
|
|
148
148
|
// The provisioned plane's subject; the external-public plane has none.
|
|
149
149
|
const subject = plan.plane.kind === "provisioned" ? plan.plane.subject : undefined;
|
|
150
150
|
const serve = subject?.serve;
|
|
@@ -155,13 +155,12 @@ async function runPlanInScope(plan, input, config, scope) {
|
|
|
155
155
|
const runSession = deps.runSession ?? descriptor.runSession;
|
|
156
156
|
const openaiApiKey = env.OPENAI_API_KEY?.trim() ?? "";
|
|
157
157
|
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
158
|
-
const
|
|
158
|
+
const secrets = new RunSecrets([
|
|
159
159
|
openaiApiKey,
|
|
160
160
|
e2bApiKey,
|
|
161
161
|
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
162
162
|
...checkpoints.flatMap((probe) => probe.redact ?? []),
|
|
163
|
-
]
|
|
164
|
-
const scrubKnownValues = scrubLiterals(knownSecretValues);
|
|
163
|
+
]);
|
|
165
164
|
const redactRepoLabel = plan.residual.policies?.redactRepos ?? subjectEnvNames.includes("GITHUB_TOKEN");
|
|
166
165
|
const publicRepo = redactRepoLabel ? "repo-01" : subjectRepo;
|
|
167
166
|
const hasGithubToken = subjectEnvNames.includes("GITHUB_TOKEN");
|
|
@@ -174,7 +173,6 @@ async function runPlanInScope(plan, input, config, scope) {
|
|
|
174
173
|
env,
|
|
175
174
|
descriptor,
|
|
176
175
|
planeClass,
|
|
177
|
-
runBudget,
|
|
178
176
|
runSession,
|
|
179
177
|
serve,
|
|
180
178
|
localTreeRoute,
|
|
@@ -183,8 +181,7 @@ async function runPlanInScope(plan, input, config, scope) {
|
|
|
183
181
|
checkpoints,
|
|
184
182
|
openaiApiKey,
|
|
185
183
|
e2bApiKey,
|
|
186
|
-
|
|
187
|
-
scrubKnownValues,
|
|
184
|
+
secrets,
|
|
188
185
|
publicRepo,
|
|
189
186
|
hasGithubToken,
|
|
190
187
|
fail,
|