humanish 0.111.2 → 0.111.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +32 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/cli/findings.d.ts +15 -1
- package/dist/cli/findings.js +43 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +14 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +71 -35
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +8 -5
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +35 -0
- package/dist/run/secrets.js +158 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +89 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/docs/contracts/study-analysis.md +6 -2
- package/package.json +1 -1
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// The parts both computer-use bundle shapes share: the subject-state story and provenance line,
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// single-bundle.ts and fanout-bundle.ts.
|
|
5
|
-
|
|
2
|
+
// provider-resource records, and the public-safe app URL label, the URL digest and the
|
|
3
|
+
// subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
|
|
4
|
+
// single-bundle.ts and fanout-bundle.ts. Feedback candidates from participant reports are in
|
|
5
|
+
// participant-feedback.ts, which the shared-world bundle reads too.
|
|
6
6
|
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
7
|
-
import { digestText
|
|
7
|
+
import { digestText } from "../../evidence/redaction.js";
|
|
8
8
|
import {} from "../../run/bundle.js";
|
|
9
9
|
import { participantResourceIds } from "../../run/participant-records.js";
|
|
10
|
-
import { resolveSelfReportedFriction } from "./self-report.js";
|
|
11
10
|
import {} from "./types.js";
|
|
12
11
|
import { plural } from "../../run/text.js";
|
|
13
12
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
@@ -25,92 +24,6 @@ export function describeSubjectState(state, dryRun) {
|
|
|
25
24
|
return "external-public (operator-declared, operator-owned public deployment; neither provisioned nor seeded)";
|
|
26
25
|
}
|
|
27
26
|
}
|
|
28
|
-
/**
|
|
29
|
-
* Feedback candidates derived from what live participants actually reported.
|
|
30
|
-
*
|
|
31
|
-
* A live run's feedback draft comes from these candidates, never from a dry-run template. The
|
|
32
|
-
* candidate worth filing is the one the study produced: a participant
|
|
33
|
-
* who reported friction on the way (the most valuable thing a run captures), or one who stopped
|
|
34
|
-
* trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
|
|
35
|
-
* any falls back to a live summary in the draft layer instead of a template.
|
|
36
|
-
*
|
|
37
|
-
* Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
|
|
38
|
-
* through redactNarration in the loop) and passes redactText again here as defense-in-depth.
|
|
39
|
-
*/
|
|
40
|
-
export function participantFeedbackCandidates(args) {
|
|
41
|
-
const candidates = [];
|
|
42
|
-
for (const participant of args.participants) {
|
|
43
|
-
const session = participant.session;
|
|
44
|
-
if (session === undefined)
|
|
45
|
-
continue;
|
|
46
|
-
const friction = resolveSelfReportedFriction(session);
|
|
47
|
-
const abandoned = session.status === "abandoned";
|
|
48
|
-
if (friction === undefined && !abandoned)
|
|
49
|
-
continue;
|
|
50
|
-
const summary = friction !== undefined
|
|
51
|
-
? `Participant ${participant.personaId} (${participant.participantId}) reported friction on the way through the study goal`
|
|
52
|
-
: `Participant ${participant.personaId} (${participant.participantId}) stopped before completing the study goal`;
|
|
53
|
-
const lastScreenshot = participant.screenshots[participant.screenshots.length - 1];
|
|
54
|
-
candidates.push({
|
|
55
|
-
schema: "humanish.feedback-candidate.v1",
|
|
56
|
-
id: `participant-report-${participant.participantId}`,
|
|
57
|
-
run_id: args.runId,
|
|
58
|
-
stream_id: participant.streamId,
|
|
59
|
-
adapter_id: args.adapterId,
|
|
60
|
-
scenario_id: args.scenarioId,
|
|
61
|
-
persona_id: participant.personaId,
|
|
62
|
-
actor: "computer-use",
|
|
63
|
-
substrate: args.substrate,
|
|
64
|
-
// The participant is reporting on the product: friction and abandonment are target-app
|
|
65
|
-
// findings by the three-roles rule. A harness failure never reaches this builder; it is
|
|
66
|
-
// not a participant report.
|
|
67
|
-
failure_owner: "target-app",
|
|
68
|
-
summary,
|
|
69
|
-
expected: args.goal,
|
|
70
|
-
actual: redactText(friction ?? session.reason),
|
|
71
|
-
evidence: [
|
|
72
|
-
...(participant.traceArtifactPath === undefined
|
|
73
|
-
? []
|
|
74
|
-
: [
|
|
75
|
-
{
|
|
76
|
-
path: participant.traceArtifactPath,
|
|
77
|
-
kind: "trace",
|
|
78
|
-
note: "Full actor trace: turns, actions, and the participant's own report.",
|
|
79
|
-
},
|
|
80
|
-
]),
|
|
81
|
-
...(lastScreenshot === undefined
|
|
82
|
-
? []
|
|
83
|
-
: [
|
|
84
|
-
{
|
|
85
|
-
path: lastScreenshot,
|
|
86
|
-
kind: "screenshot",
|
|
87
|
-
note: "Final screenshot at the moment the session ended.",
|
|
88
|
-
},
|
|
89
|
-
]),
|
|
90
|
-
...(participant.commsArtifactPath === undefined
|
|
91
|
-
? []
|
|
92
|
-
: [
|
|
93
|
-
{
|
|
94
|
-
path: participant.commsArtifactPath,
|
|
95
|
-
kind: "log",
|
|
96
|
-
note: "Digest-only comms thread captured in-sandbox.",
|
|
97
|
-
},
|
|
98
|
-
]),
|
|
99
|
-
],
|
|
100
|
-
redaction: {
|
|
101
|
-
status: "passed",
|
|
102
|
-
notes: "Quoted participant text passed the loop's known-value scrub and pattern redaction before persisting, and redactText again here.",
|
|
103
|
-
},
|
|
104
|
-
idempotency_key: `humanish:${args.runId}:${participant.participantId}:participant-report`,
|
|
105
|
-
proposed_next_state: "study-quality-review",
|
|
106
|
-
acceptance_proof: [
|
|
107
|
-
feedbackProofCommands(args.runId).verify,
|
|
108
|
-
feedbackProofCommands(args.runId).observe,
|
|
109
|
-
],
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
return candidates;
|
|
113
|
-
}
|
|
114
27
|
/** Human-readable provenance line for the single-participant subject.provenance event:
|
|
115
28
|
* claims "cloned/packed and served" only when it actually happened. */
|
|
116
29
|
export function subjectProvenanceMessage(provenance, publicAppUrl, dryRun, hasSession) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// The first half of an E2B desktop participant's preparation: acquire the sandbox, verify its screen, and
|
|
2
2
|
// provision the subject. Each step fills the participant's state; a step that cannot proceed throws.
|
|
3
3
|
import { redactText } from "../../../evidence/redaction.js";
|
|
4
|
+
import { validatePreparedRunArtifactPaths } from "../../../run/paths.js";
|
|
4
5
|
import { provisionCloneSubject } from "../../../subject/clone.js";
|
|
5
6
|
import { provisionDesktopCli } from "../../../subject/desktop-cli.js";
|
|
6
7
|
import { provisionLocalTreeSubject } from "../../../subject/local-tree.js";
|
|
@@ -94,10 +95,16 @@ export async function acquireParticipantDesktop(ctx, state) {
|
|
|
94
95
|
if ("reason" in desktopResources) {
|
|
95
96
|
warnings.push(`Desktop resource size unavailable (${desktopResources.reason}); compute cost remains unpriced.`);
|
|
96
97
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
if (deps.prepareDesktop) {
|
|
99
|
+
await deps.prepareDesktop(desktop, {
|
|
100
|
+
kind: "participant",
|
|
101
|
+
participant: { id: spec.planned.id, index: spec.planned.index, count: deps.participantCount },
|
|
102
|
+
});
|
|
103
|
+
// The caller's hook runs with this process's file access, so the run directory is checked
|
|
104
|
+
// again before more evidence goes into it.
|
|
105
|
+
if ("physicalRunRoot" in deps.artifactRoot)
|
|
106
|
+
await validatePreparedRunArtifactPaths(deps.artifactRoot);
|
|
107
|
+
}
|
|
101
108
|
if (deps.receiving) {
|
|
102
109
|
state.receivingInboxUrl = await attachReceivingInbox(shell, spec, { ...deps, receiving: deps.receiving }, ctx.targetUrl);
|
|
103
110
|
state.commsArtifactPath = "comms/receiving.json";
|
|
@@ -4,7 +4,8 @@ import { actorEnding } from "../../actors/stop-cause.js";
|
|
|
4
4
|
import { redactText } from "../../evidence/redaction.js";
|
|
5
5
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
6
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
|
-
import {
|
|
7
|
+
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
8
9
|
import { participantFactsOf } from "./participant-facts.js";
|
|
9
10
|
import { participantPassed, participantStatus } from "../../run/judge.js";
|
|
10
11
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
@@ -141,7 +142,6 @@ function fanoutFeedbackCandidates(args) {
|
|
|
141
142
|
runId: args.run.runId,
|
|
142
143
|
scenarioId: `cua-${plan.studyId}`,
|
|
143
144
|
adapterId: plan.studyId,
|
|
144
|
-
goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
145
145
|
substrate: plan.residual.execution?.target === "local" ? "local-desktop" : "e2b-desktop",
|
|
146
146
|
participants: specs.map((spec, index) => {
|
|
147
147
|
const outcome = outcomes?.[index];
|
|
@@ -149,6 +149,7 @@ function fanoutFeedbackCandidates(args) {
|
|
|
149
149
|
participantId: spec.planned.id,
|
|
150
150
|
streamId: spec.streamId,
|
|
151
151
|
personaId: spec.persona.id,
|
|
152
|
+
goal: redactText(spec.evidenceInstructions ?? spec.instructions),
|
|
152
153
|
...(outcome?.session === undefined ? {} : { session: outcome.session }),
|
|
153
154
|
...(outcome?.session === undefined
|
|
154
155
|
? {}
|
|
@@ -15,7 +15,7 @@ import { participantSubjectEnv } from "./types.js";
|
|
|
15
15
|
export async function runStudyParticipants(setup, participants) {
|
|
16
16
|
const { plan, input, cwd, streams, descriptor, run } = setup;
|
|
17
17
|
const { participantRuns, participantPlan, bundleBase } = setup;
|
|
18
|
-
const { env,
|
|
18
|
+
const { env, deps, liveTrace, externalComms, inProgress, fail } = participants;
|
|
19
19
|
const { dryRun } = plan;
|
|
20
20
|
const inProcess = plan.runner.desktop === "in-process";
|
|
21
21
|
const subjectEnvNames = [...participantSubjectEnv(plan.runner.subject)];
|
|
@@ -76,11 +76,7 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
76
76
|
env,
|
|
77
77
|
participants: participantRuns.map((spec) => spec.planned.id),
|
|
78
78
|
runPaths,
|
|
79
|
-
registerSecrets: (values) =>
|
|
80
|
-
for (const value of values)
|
|
81
|
-
if (value.length >= 4 && !knownSecretValues.includes(value))
|
|
82
|
-
knownSecretValues.push(value);
|
|
83
|
-
},
|
|
79
|
+
registerSecrets: (values) => run.secrets.add(values),
|
|
84
80
|
});
|
|
85
81
|
if (receiving)
|
|
86
82
|
deps.receiving = receiving;
|
|
@@ -129,7 +125,7 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
129
125
|
runPaths,
|
|
130
126
|
participantRuns: participantRuns,
|
|
131
127
|
outcomes,
|
|
132
|
-
knownSecretValues,
|
|
128
|
+
knownSecretValues: run.secrets.values(),
|
|
133
129
|
})
|
|
134
130
|
: [];
|
|
135
131
|
return {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { ComputerUsePlan, SharedWorldPlan } from "../../study/plan-types.js";
|
|
2
|
+
import type { StudyDeps } from "../../study/study-deps.js";
|
|
3
|
+
import type { LiveTraceFlush } from "./live-flush.js";
|
|
4
|
+
import type { CuaParticipantDeps } from "./types.js";
|
|
5
|
+
/** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
|
|
6
|
+
export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalProvisioned" | "appUrl" | "onObservedUrl">;
|
|
7
|
+
/** What every participant of a desktop run reads. Each route supplies all of it. */
|
|
8
|
+
interface DesktopRunFacts {
|
|
9
|
+
plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
|
|
10
|
+
/** The plan's brain. A local agent runs each participant on the operator's signed-in CLI. */
|
|
11
|
+
brain: CuaParticipantDeps["brain"];
|
|
12
|
+
/** What the participant's desktop does with the subject. */
|
|
13
|
+
subject: CuaParticipantDeps["subject"];
|
|
14
|
+
/** The run's env. The E2B request timeout is read from it. */
|
|
15
|
+
env: CuaParticipantDeps["env"];
|
|
16
|
+
openaiApiKey: string;
|
|
17
|
+
e2bApiKey: string;
|
|
18
|
+
/** Each participant's session budget. */
|
|
19
|
+
timeoutMs: number;
|
|
20
|
+
/** Each participant's sandbox lifetime. */
|
|
21
|
+
sandboxMs: number;
|
|
22
|
+
participantCount: number;
|
|
23
|
+
artifactRoot: CuaParticipantDeps["artifactRoot"];
|
|
24
|
+
studyCwd: string;
|
|
25
|
+
redactScreenshots: boolean;
|
|
26
|
+
scrubKnownValues: (text: string) => string;
|
|
27
|
+
runSession: CuaParticipantDeps["runSession"];
|
|
28
|
+
/** The run's live-trace flush, once a live run has started it. */
|
|
29
|
+
liveFlush: () => LiveTraceFlush["flush"] | undefined;
|
|
30
|
+
onStream: CuaParticipantDeps["onStream"];
|
|
31
|
+
reportSubjectPhase: CuaParticipantDeps["reportSubjectPhase"];
|
|
32
|
+
/** The run's test seams: the clock, the E2B SDK loader and the detached timers. */
|
|
33
|
+
seams: StudyDeps;
|
|
34
|
+
/** The caller's prepareDesktop, which each participant's desktop runs. */
|
|
35
|
+
prepareDesktop: CuaParticipantDeps["prepareDesktop"];
|
|
36
|
+
}
|
|
37
|
+
/** The fields one desktop route sets and the other does not. Undefined leaves a field out. */
|
|
38
|
+
type DesktopRouteOverrides = {
|
|
39
|
+
[K in "createDesktop" | "createProvider" | "inProcessExecutor" | "localTreeArchiveBuffer" | "externalComms" | "receiving" | "screenMismatchPolicy"]?: CuaParticipantDeps[K] | undefined;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* One run's participant deps. Call it once per run: the run budget it creates is the one ledger
|
|
43
|
+
* every participant notes its spend on, and a dry run, which never spends, gets none.
|
|
44
|
+
*/
|
|
45
|
+
export declare function desktopParticipantDeps(run: DesktopRunFacts, overrides: DesktopRouteOverrides): DesktopParticipantDeps;
|
|
46
|
+
export {};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// The deps every desktop participant reads, built here for computer use and shared world. A field
|
|
2
|
+
// both routes hand their participants is added once, in desktopParticipantDeps; each route passes
|
|
3
|
+
// what only it knows as overrides.
|
|
4
|
+
import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
|
|
5
|
+
import { makeCuaRunBudget } from "./participant-model.js";
|
|
6
|
+
/**
|
|
7
|
+
* One run's participant deps. Call it once per run: the run budget it creates is the one ledger
|
|
8
|
+
* every participant notes its spend on, and a dry run, which never spends, gets none.
|
|
9
|
+
*/
|
|
10
|
+
export function desktopParticipantDeps(run, overrides) {
|
|
11
|
+
const { plan, seams } = run;
|
|
12
|
+
const { maxTotalUsd } = plan.caps;
|
|
13
|
+
const optional = {
|
|
14
|
+
...overrides,
|
|
15
|
+
prepareDesktop: run.prepareDesktop,
|
|
16
|
+
desktopModule: seams.desktopModule,
|
|
17
|
+
detachedTimers: seams.detachedTimers,
|
|
18
|
+
runBudget: plan.dryRun || maxTotalUsd === undefined ? undefined : makeCuaRunBudget(maxTotalUsd),
|
|
19
|
+
};
|
|
20
|
+
return {
|
|
21
|
+
...definedFields(optional),
|
|
22
|
+
onTrace: (participantId, items, usage, metadata) => run.liveFlush()?.(participantId, items, usage, metadata),
|
|
23
|
+
residual: plan.residual,
|
|
24
|
+
studyId: plan.studyId,
|
|
25
|
+
caps: plan.caps,
|
|
26
|
+
brain: run.brain,
|
|
27
|
+
subject: run.subject,
|
|
28
|
+
env: run.env,
|
|
29
|
+
openaiApiKey: run.openaiApiKey,
|
|
30
|
+
e2bApiKey: run.e2bApiKey,
|
|
31
|
+
requestTimeoutMs: e2bRequestTimeoutMs(run.env),
|
|
32
|
+
sandboxMs: run.sandboxMs,
|
|
33
|
+
timeoutMs: run.timeoutMs,
|
|
34
|
+
participantCount: run.participantCount,
|
|
35
|
+
artifactRoot: run.artifactRoot,
|
|
36
|
+
studyCwd: run.studyCwd,
|
|
37
|
+
redactScreenshots: run.redactScreenshots,
|
|
38
|
+
scrubKnownValues: run.scrubKnownValues,
|
|
39
|
+
runSession: run.runSession,
|
|
40
|
+
now: seams.now ?? Date.now,
|
|
41
|
+
onStream: run.onStream,
|
|
42
|
+
reportSubjectPhase: run.reportSubjectPhase,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** The fields of `fields` that are set. The deps never carry a key whose value is undefined. */
|
|
46
|
+
function definedFields(fields) {
|
|
47
|
+
return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined));
|
|
48
|
+
}
|
|
@@ -5,8 +5,9 @@ export declare function participantFactsOf(outcome: ParticipantRunOutcome): Part
|
|
|
5
5
|
/** Whether a participant passed. A dry-run participant passes as a contract. */
|
|
6
6
|
export declare function participantOutcomeOk(outcome: ParticipantRunOutcome | undefined, dryRun: boolean): boolean;
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* The execution failures a run's participants caused, by kind: each session that failed in the
|
|
9
|
+
* harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
|
|
10
|
+
* item after its last request, then each desktop not confirmed released. Participants keep their
|
|
11
|
+
* order within each kind.
|
|
11
12
|
*/
|
|
12
|
-
export declare function
|
|
13
|
+
export declare function participantExecutionFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
|
|
2
2
|
// them. The bundle builders, the result, the participant runner and shared-world all read these,
|
|
3
3
|
// so they sit in a file that imports only the judge and the types.
|
|
4
|
-
import { participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
4
|
+
import { participantHarnessFailed, participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
5
5
|
/** A participant outcome's facts for the judge. */
|
|
6
6
|
export function participantFactsOf(outcome) {
|
|
7
7
|
return {
|
|
@@ -20,12 +20,47 @@ export function participantOutcomeOk(outcome, dryRun) {
|
|
|
20
20
|
return true;
|
|
21
21
|
return outcome !== undefined && participantPassed(participantFactsOf(outcome));
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* The execution failures a run's participants caused, by kind: each session that failed in the
|
|
25
|
+
* harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
|
|
26
|
+
* item after its last request, then each desktop not confirmed released. Participants keep their
|
|
27
|
+
* order within each kind.
|
|
28
|
+
*/
|
|
29
|
+
export function participantExecutionFailures(outcomes, runId) {
|
|
30
|
+
const all = outcomes ?? [];
|
|
31
|
+
const named = (outcome, text) => `${outcome.spec.planned.id}: ${text}`;
|
|
32
|
+
return [
|
|
33
|
+
...all
|
|
34
|
+
.filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
|
|
35
|
+
.map((outcome) => ({
|
|
36
|
+
kind: "harness",
|
|
37
|
+
message: named(outcome, outcome.sessionError ?? outcome.session?.reason ?? "harness error"),
|
|
38
|
+
})),
|
|
39
|
+
...all.flatMap((outcome) => outcome.providerCleanupError === undefined
|
|
40
|
+
? []
|
|
41
|
+
: [
|
|
42
|
+
{
|
|
43
|
+
kind: "provider-cleanup",
|
|
44
|
+
message: named(outcome, outcome.providerCleanupError),
|
|
45
|
+
},
|
|
46
|
+
]),
|
|
47
|
+
...all.flatMap((outcome) => outcome.providerPolicyError === undefined
|
|
48
|
+
? []
|
|
49
|
+
: [
|
|
50
|
+
{
|
|
51
|
+
kind: "provider-policy",
|
|
52
|
+
message: named(outcome, outcome.providerPolicyError),
|
|
53
|
+
},
|
|
54
|
+
]),
|
|
55
|
+
...unreleasedSandboxFailures(all, runId),
|
|
56
|
+
];
|
|
57
|
+
}
|
|
23
58
|
/**
|
|
24
59
|
* One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
|
|
25
60
|
* was not killed, or a desktop whose release says why (a local VM names its container). A
|
|
26
61
|
* participant that never acquired a desktop (skipped, in-process) adds none.
|
|
27
62
|
*/
|
|
28
|
-
|
|
63
|
+
function unreleasedSandboxFailures(outcomes, runId) {
|
|
29
64
|
return (outcomes ?? [])
|
|
30
65
|
.filter((outcome) => outcome.sandboxRelease !== undefined ||
|
|
31
66
|
(outcome.sandboxId !== undefined && !outcome.killed))
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
|
|
2
|
+
import type { RunFeedbackCandidate } from "../../run/bundle.js";
|
|
3
|
+
/**
|
|
4
|
+
* Feedback candidates derived from what live participants actually reported.
|
|
5
|
+
*
|
|
6
|
+
* A live run's feedback draft comes from these candidates, never from a dry-run template. The
|
|
7
|
+
* candidate worth filing is the one the study produced: a participant
|
|
8
|
+
* who reported friction on the way (the most valuable thing a run captures), or one who stopped
|
|
9
|
+
* trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
|
|
10
|
+
* any falls back to a live summary in the draft layer instead of a template.
|
|
11
|
+
*
|
|
12
|
+
* Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
|
|
13
|
+
* through redactNarration in the loop) and passes redactText again here as defense-in-depth.
|
|
14
|
+
*/
|
|
15
|
+
export declare function participantFeedbackCandidates(args: {
|
|
16
|
+
runId: string;
|
|
17
|
+
scenarioId: string;
|
|
18
|
+
adapterId: string;
|
|
19
|
+
substrate: RunFeedbackCandidate["substrate"];
|
|
20
|
+
participants: Array<{
|
|
21
|
+
/** The participant's plan id; it names the candidate and its idempotency key. */
|
|
22
|
+
participantId: string;
|
|
23
|
+
streamId: string;
|
|
24
|
+
personaId: string;
|
|
25
|
+
/** The participant's own already-redacted instructions, which the candidate's `expected`
|
|
26
|
+
* quotes. Participants of one study get different instructions, each naming its persona. */
|
|
27
|
+
goal: string;
|
|
28
|
+
session?: CuaLoopResult;
|
|
29
|
+
traceArtifactPath?: string;
|
|
30
|
+
screenshots: string[];
|
|
31
|
+
commsArtifactPath?: string;
|
|
32
|
+
}>;
|
|
33
|
+
}): RunFeedbackCandidate[];
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Feedback candidates from what live participants reported. The computer-use bundles (single and
|
|
2
|
+
// fan-out) and the shared-world bundle all build them here, so a participant's report becomes the
|
|
3
|
+
// same candidate whichever route ran it.
|
|
4
|
+
import { redactText } from "../../evidence/redaction.js";
|
|
5
|
+
import { feedbackProofCommands } from "../../feedback/proof.js";
|
|
6
|
+
import { resolveSelfReportedFriction } from "./self-report.js";
|
|
7
|
+
/**
|
|
8
|
+
* Feedback candidates derived from what live participants actually reported.
|
|
9
|
+
*
|
|
10
|
+
* A live run's feedback draft comes from these candidates, never from a dry-run template. The
|
|
11
|
+
* candidate worth filing is the one the study produced: a participant
|
|
12
|
+
* who reported friction on the way (the most valuable thing a run captures), or one who stopped
|
|
13
|
+
* trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
|
|
14
|
+
* any falls back to a live summary in the draft layer instead of a template.
|
|
15
|
+
*
|
|
16
|
+
* Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
|
|
17
|
+
* through redactNarration in the loop) and passes redactText again here as defense-in-depth.
|
|
18
|
+
*/
|
|
19
|
+
export function participantFeedbackCandidates(args) {
|
|
20
|
+
const candidates = [];
|
|
21
|
+
for (const participant of args.participants) {
|
|
22
|
+
const session = participant.session;
|
|
23
|
+
if (session === undefined)
|
|
24
|
+
continue;
|
|
25
|
+
const friction = resolveSelfReportedFriction(session);
|
|
26
|
+
const abandoned = session.status === "abandoned";
|
|
27
|
+
if (friction === undefined && !abandoned)
|
|
28
|
+
continue;
|
|
29
|
+
const summary = friction !== undefined
|
|
30
|
+
? `Participant ${participant.personaId} (${participant.participantId}) reported friction on the way through the study goal`
|
|
31
|
+
: `Participant ${participant.personaId} (${participant.participantId}) stopped before completing the study goal`;
|
|
32
|
+
const lastScreenshot = participant.screenshots[participant.screenshots.length - 1];
|
|
33
|
+
candidates.push({
|
|
34
|
+
schema: "humanish.feedback-candidate.v1",
|
|
35
|
+
id: `participant-report-${participant.participantId}`,
|
|
36
|
+
run_id: args.runId,
|
|
37
|
+
stream_id: participant.streamId,
|
|
38
|
+
adapter_id: args.adapterId,
|
|
39
|
+
scenario_id: args.scenarioId,
|
|
40
|
+
persona_id: participant.personaId,
|
|
41
|
+
actor: "computer-use",
|
|
42
|
+
substrate: args.substrate,
|
|
43
|
+
// The participant is reporting on the product: friction and abandonment are target-app
|
|
44
|
+
// findings by the three-roles rule. A harness failure never reaches this builder; it is
|
|
45
|
+
// not a participant report.
|
|
46
|
+
failure_owner: "target-app",
|
|
47
|
+
summary,
|
|
48
|
+
expected: participant.goal,
|
|
49
|
+
actual: redactText(friction ?? session.reason),
|
|
50
|
+
evidence: [
|
|
51
|
+
...(participant.traceArtifactPath === undefined
|
|
52
|
+
? []
|
|
53
|
+
: [
|
|
54
|
+
{
|
|
55
|
+
path: participant.traceArtifactPath,
|
|
56
|
+
kind: "trace",
|
|
57
|
+
note: "Full actor trace: turns, actions, and the participant's own report.",
|
|
58
|
+
},
|
|
59
|
+
]),
|
|
60
|
+
...(lastScreenshot === undefined
|
|
61
|
+
? []
|
|
62
|
+
: [
|
|
63
|
+
{
|
|
64
|
+
path: lastScreenshot,
|
|
65
|
+
kind: "screenshot",
|
|
66
|
+
note: "Final screenshot at the moment the session ended.",
|
|
67
|
+
},
|
|
68
|
+
]),
|
|
69
|
+
...(participant.commsArtifactPath === undefined
|
|
70
|
+
? []
|
|
71
|
+
: [
|
|
72
|
+
{
|
|
73
|
+
path: participant.commsArtifactPath,
|
|
74
|
+
kind: "log",
|
|
75
|
+
note: "Digest-only comms thread captured in-sandbox.",
|
|
76
|
+
},
|
|
77
|
+
]),
|
|
78
|
+
],
|
|
79
|
+
redaction: {
|
|
80
|
+
status: "passed",
|
|
81
|
+
notes: "Quoted participant text passed the loop's known-value scrub and pattern redaction before persisting, and redactText again here.",
|
|
82
|
+
},
|
|
83
|
+
idempotency_key: `humanish:${args.runId}:${participant.participantId}:participant-report`,
|
|
84
|
+
proposed_next_state: "study-quality-review",
|
|
85
|
+
acceptance_proof: [
|
|
86
|
+
feedbackProofCommands(args.runId).verify,
|
|
87
|
+
feedbackProofCommands(args.runId).observe,
|
|
88
|
+
],
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
return candidates;
|
|
92
|
+
}
|
|
@@ -41,4 +41,15 @@ export declare function judgeParticipantSession(session: CuaLoopResult | undefin
|
|
|
41
41
|
selfReportedBlocker: boolean;
|
|
42
42
|
reportedFriction: boolean;
|
|
43
43
|
};
|
|
44
|
+
/**
|
|
45
|
+
* caps.maxUsd is enforced inside each participant's loop independently, so N participants can
|
|
46
|
+
* spend up to N × maxUsd before any of them stops, while the run cost summary reports the larger
|
|
47
|
+
* aggregate. The warning names that ceiling, unless the study declared a shared maxTotalUsd budget.
|
|
48
|
+
* `capped` names what the cap counts: shared world says model spend, because the external-public
|
|
49
|
+
* plane's lobby-code reads are not counted against it.
|
|
50
|
+
*/
|
|
51
|
+
export declare function participantCapWarning(caps: {
|
|
52
|
+
readonly maxUsd?: number;
|
|
53
|
+
readonly maxTotalUsd?: number;
|
|
54
|
+
}, participantCount: number, capped?: string): string | undefined;
|
|
44
55
|
export declare function makeCuaRunBudget(maxTotalUsd: number): CuaRunBudget;
|
|
@@ -257,6 +257,21 @@ export function judgeParticipantSession(session, deps, warnings) {
|
|
|
257
257
|
}
|
|
258
258
|
return { noEngagement, selfReportedBlocker, reportedFriction };
|
|
259
259
|
}
|
|
260
|
+
/**
|
|
261
|
+
* caps.maxUsd is enforced inside each participant's loop independently, so N participants can
|
|
262
|
+
* spend up to N × maxUsd before any of them stops, while the run cost summary reports the larger
|
|
263
|
+
* aggregate. The warning names that ceiling, unless the study declared a shared maxTotalUsd budget.
|
|
264
|
+
* `capped` names what the cap counts: shared world says model spend, because the external-public
|
|
265
|
+
* plane's lobby-code reads are not counted against it.
|
|
266
|
+
*/
|
|
267
|
+
export function participantCapWarning(caps, participantCount, capped = "each participant") {
|
|
268
|
+
const capUsd = caps.maxUsd;
|
|
269
|
+
if (capUsd === undefined || participantCount <= 1)
|
|
270
|
+
return undefined;
|
|
271
|
+
if (caps.maxTotalUsd !== undefined)
|
|
272
|
+
return undefined;
|
|
273
|
+
return `caps.maxUsd ($${capUsd}) caps ${capped}, so ${participantCount} participants may spend up to ${participantCount} × $${capUsd} (about $${round6(capUsd * participantCount)}) before any of them stops. Set caps.maxTotalUsd for one budget across the study.`;
|
|
274
|
+
}
|
|
260
275
|
export function makeCuaRunBudget(maxTotalUsd) {
|
|
261
276
|
const participantEstimates = new Map();
|
|
262
277
|
return {
|