humanish 0.111.2 → 0.111.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/ARCHITECTURE.md +1 -1
  2. package/CHANGELOG.md +32 -0
  3. package/dist/actors/scripted-browser/actor.d.ts +5 -0
  4. package/dist/actors/scripted-browser/actor.js +40 -19
  5. package/dist/actors/scripted-browser/journey.d.ts +2 -0
  6. package/dist/actors/scripted-browser/journey.js +2 -1
  7. package/dist/actors/scripted-browser/steps.d.ts +2 -0
  8. package/dist/actors/scripted-browser/steps.js +17 -0
  9. package/dist/cli/commands/study-route-computer-use.js +3 -8
  10. package/dist/cli/commands/study-route-preview.js +4 -6
  11. package/dist/cli/commands/study-route-shared-world.js +6 -7
  12. package/dist/cli/findings.d.ts +15 -1
  13. package/dist/cli/findings.js +43 -5
  14. package/dist/evidence/redaction.d.ts +2 -2
  15. package/dist/evidence/redaction.js +3 -3
  16. package/dist/evidence/secret-scrub.d.ts +7 -0
  17. package/dist/evidence/secret-scrub.js +21 -12
  18. package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
  19. package/dist/routes/computer-use/bundle-parts.js +5 -92
  20. package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
  21. package/dist/routes/computer-use/fanout-bundle.js +3 -2
  22. package/dist/routes/computer-use/live-phase.js +3 -7
  23. package/dist/routes/computer-use/participant-deps.d.ts +46 -0
  24. package/dist/routes/computer-use/participant-deps.js +48 -0
  25. package/dist/routes/computer-use/participant-facts.d.ts +5 -4
  26. package/dist/routes/computer-use/participant-facts.js +37 -2
  27. package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
  28. package/dist/routes/computer-use/participant-feedback.js +92 -0
  29. package/dist/routes/computer-use/participant-model.d.ts +11 -0
  30. package/dist/routes/computer-use/participant-model.js +15 -0
  31. package/dist/routes/computer-use/preflight.js +33 -65
  32. package/dist/routes/computer-use/result.js +13 -52
  33. package/dist/routes/computer-use/route.js +6 -7
  34. package/dist/routes/computer-use/setup.d.ts +2 -6
  35. package/dist/routes/computer-use/setup.js +47 -93
  36. package/dist/routes/computer-use/single-bundle.js +3 -2
  37. package/dist/routes/computer-use/types.d.ts +2 -2
  38. package/dist/routes/preview.js +2 -8
  39. package/dist/routes/scripted/bundle.d.ts +2 -6
  40. package/dist/routes/scripted/result.d.ts +2 -1
  41. package/dist/routes/scripted/result.js +4 -4
  42. package/dist/routes/scripted/route.js +14 -38
  43. package/dist/routes/scripted/setup.d.ts +2 -1
  44. package/dist/routes/scripted/setup.js +71 -35
  45. package/dist/routes/scripted/subject.d.ts +3 -18
  46. package/dist/routes/scripted/subject.js +22 -78
  47. package/dist/routes/scripted/surface-sessions.d.ts +1 -1
  48. package/dist/routes/scripted/surface-sessions.js +2 -0
  49. package/dist/routes/scripted/types.d.ts +2 -2
  50. package/dist/routes/shared-world/bundle.js +36 -1
  51. package/dist/routes/shared-world/comms.d.ts +2 -1
  52. package/dist/routes/shared-world/comms.js +3 -7
  53. package/dist/routes/shared-world/handoff.d.ts +2 -2
  54. package/dist/routes/shared-world/handoff.js +1 -1
  55. package/dist/routes/shared-world/participant-specs.d.ts +4 -5
  56. package/dist/routes/shared-world/participant-specs.js +19 -25
  57. package/dist/routes/shared-world/plan.js +11 -7
  58. package/dist/routes/shared-world/provisioned.d.ts +2 -1
  59. package/dist/routes/shared-world/provisioned.js +37 -80
  60. package/dist/routes/shared-world/result.js +17 -35
  61. package/dist/routes/shared-world/route.js +33 -36
  62. package/dist/routes/shared-world/setup.d.ts +2 -3
  63. package/dist/routes/shared-world/setup.js +11 -29
  64. package/dist/routes/shared-world/types.d.ts +6 -15
  65. package/dist/routes/terminal/credentials.js +14 -1
  66. package/dist/routes/terminal/dry-run.js +14 -24
  67. package/dist/routes/terminal/live-finish.d.ts +0 -1
  68. package/dist/routes/terminal/live-finish.js +8 -5
  69. package/dist/routes/terminal/route.js +9 -15
  70. package/dist/routes/terminal/session.d.ts +2 -3
  71. package/dist/routes/terminal/session.js +28 -40
  72. package/dist/routes/terminal/trace.d.ts +4 -2
  73. package/dist/routes/terminal/trace.js +30 -35
  74. package/dist/routes/terminal/types.d.ts +2 -2
  75. package/dist/run/run.d.ts +55 -1
  76. package/dist/run/run.js +38 -2
  77. package/dist/run/secrets.d.ts +35 -0
  78. package/dist/run/secrets.js +158 -0
  79. package/dist/run/study-result.d.ts +34 -0
  80. package/dist/run/study-result.js +16 -0
  81. package/dist/study/requirements.d.ts +60 -5
  82. package/dist/study/requirements.js +89 -2
  83. package/dist/study/run-study-events.d.ts +4 -3
  84. package/dist/study/run-study-events.js +9 -14
  85. package/dist/study/run-study-options.d.ts +4 -1
  86. package/dist/study/run-study-options.js +22 -32
  87. package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
  88. package/dist/substrates/e2b/subject-sandbox.js +94 -0
  89. package/docs/contracts/core.md +5 -4
  90. package/docs/contracts/feedback.md +12 -5
  91. package/docs/contracts/study-analysis.md +6 -2
  92. package/package.json +1 -1
@@ -1,13 +1,12 @@
1
1
  // The parts both computer-use bundle shapes share: the subject-state story and provenance line,
2
- // feedback candidates from what live participants reported, provider-resource records, and the
3
- // public-safe app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
4
- // single-bundle.ts and fanout-bundle.ts.
5
- import { feedbackProofCommands } from "../../feedback/proof.js";
2
+ // provider-resource records, and the public-safe app URL label, the URL digest and the
3
+ // subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
4
+ // single-bundle.ts and fanout-bundle.ts. Feedback candidates from participant reports are in
5
+ // participant-feedback.ts, which the shared-world bundle reads too.
6
6
  import { scanEncodedText } from "../../evidence/encoded-text.js";
7
- import { digestText, redactText } from "../../evidence/redaction.js";
7
+ import { digestText } from "../../evidence/redaction.js";
8
8
  import {} from "../../run/bundle.js";
9
9
  import { participantResourceIds } from "../../run/participant-records.js";
10
- import { resolveSelfReportedFriction } from "./self-report.js";
11
10
  import {} from "./types.js";
12
11
  import { plural } from "../../run/text.js";
13
12
  /** The human-readable state story appended to the provenance event (and review.md via it). */
@@ -25,92 +24,6 @@ export function describeSubjectState(state, dryRun) {
25
24
  return "external-public (operator-declared, operator-owned public deployment; neither provisioned nor seeded)";
26
25
  }
27
26
  }
28
- /**
29
- * Feedback candidates derived from what live participants actually reported.
30
- *
31
- * A live run's feedback draft comes from these candidates, never from a dry-run template. The
32
- * candidate worth filing is the one the study produced: a participant
33
- * who reported friction on the way (the most valuable thing a run captures), or one who stopped
34
- * trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
35
- * any falls back to a live summary in the draft layer instead of a template.
36
- *
37
- * Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
38
- * through redactNarration in the loop) and passes redactText again here as defense-in-depth.
39
- */
40
- export function participantFeedbackCandidates(args) {
41
- const candidates = [];
42
- for (const participant of args.participants) {
43
- const session = participant.session;
44
- if (session === undefined)
45
- continue;
46
- const friction = resolveSelfReportedFriction(session);
47
- const abandoned = session.status === "abandoned";
48
- if (friction === undefined && !abandoned)
49
- continue;
50
- const summary = friction !== undefined
51
- ? `Participant ${participant.personaId} (${participant.participantId}) reported friction on the way through the study goal`
52
- : `Participant ${participant.personaId} (${participant.participantId}) stopped before completing the study goal`;
53
- const lastScreenshot = participant.screenshots[participant.screenshots.length - 1];
54
- candidates.push({
55
- schema: "humanish.feedback-candidate.v1",
56
- id: `participant-report-${participant.participantId}`,
57
- run_id: args.runId,
58
- stream_id: participant.streamId,
59
- adapter_id: args.adapterId,
60
- scenario_id: args.scenarioId,
61
- persona_id: participant.personaId,
62
- actor: "computer-use",
63
- substrate: args.substrate,
64
- // The participant is reporting on the product: friction and abandonment are target-app
65
- // findings by the three-roles rule. A harness failure never reaches this builder; it is
66
- // not a participant report.
67
- failure_owner: "target-app",
68
- summary,
69
- expected: args.goal,
70
- actual: redactText(friction ?? session.reason),
71
- evidence: [
72
- ...(participant.traceArtifactPath === undefined
73
- ? []
74
- : [
75
- {
76
- path: participant.traceArtifactPath,
77
- kind: "trace",
78
- note: "Full actor trace: turns, actions, and the participant's own report.",
79
- },
80
- ]),
81
- ...(lastScreenshot === undefined
82
- ? []
83
- : [
84
- {
85
- path: lastScreenshot,
86
- kind: "screenshot",
87
- note: "Final screenshot at the moment the session ended.",
88
- },
89
- ]),
90
- ...(participant.commsArtifactPath === undefined
91
- ? []
92
- : [
93
- {
94
- path: participant.commsArtifactPath,
95
- kind: "log",
96
- note: "Digest-only comms thread captured in-sandbox.",
97
- },
98
- ]),
99
- ],
100
- redaction: {
101
- status: "passed",
102
- notes: "Quoted participant text passed the loop's known-value scrub and pattern redaction before persisting, and redactText again here.",
103
- },
104
- idempotency_key: `humanish:${args.runId}:${participant.participantId}:participant-report`,
105
- proposed_next_state: "study-quality-review",
106
- acceptance_proof: [
107
- feedbackProofCommands(args.runId).verify,
108
- feedbackProofCommands(args.runId).observe,
109
- ],
110
- });
111
- }
112
- return candidates;
113
- }
114
27
  /** Human-readable provenance line for the single-participant subject.provenance event:
115
28
  * claims "cloned/packed and served" only when it actually happened. */
116
29
  export function subjectProvenanceMessage(provenance, publicAppUrl, dryRun, hasSession) {
@@ -1,6 +1,7 @@
1
1
  // The first half of an E2B desktop participant's preparation: acquire the sandbox, verify its screen, and
2
2
  // provision the subject. Each step fills the participant's state; a step that cannot proceed throws.
3
3
  import { redactText } from "../../../evidence/redaction.js";
4
+ import { validatePreparedRunArtifactPaths } from "../../../run/paths.js";
4
5
  import { provisionCloneSubject } from "../../../subject/clone.js";
5
6
  import { provisionDesktopCli } from "../../../subject/desktop-cli.js";
6
7
  import { provisionLocalTreeSubject } from "../../../subject/local-tree.js";
@@ -94,10 +95,16 @@ export async function acquireParticipantDesktop(ctx, state) {
94
95
  if ("reason" in desktopResources) {
95
96
  warnings.push(`Desktop resource size unavailable (${desktopResources.reason}); compute cost remains unpriced.`);
96
97
  }
97
- await deps.prepareDesktop?.(desktop, {
98
- kind: "participant",
99
- participant: { id: spec.planned.id, index: spec.planned.index, count: deps.participantCount },
100
- });
98
+ if (deps.prepareDesktop) {
99
+ await deps.prepareDesktop(desktop, {
100
+ kind: "participant",
101
+ participant: { id: spec.planned.id, index: spec.planned.index, count: deps.participantCount },
102
+ });
103
+ // The caller's hook runs with this process's file access, so the run directory is checked
104
+ // again before more evidence goes into it.
105
+ if ("physicalRunRoot" in deps.artifactRoot)
106
+ await validatePreparedRunArtifactPaths(deps.artifactRoot);
107
+ }
101
108
  if (deps.receiving) {
102
109
  state.receivingInboxUrl = await attachReceivingInbox(shell, spec, { ...deps, receiving: deps.receiving }, ctx.targetUrl);
103
110
  state.commsArtifactPath = "comms/receiving.json";
@@ -4,7 +4,8 @@ import { actorEnding } from "../../actors/stop-cause.js";
4
4
  import { redactText } from "../../evidence/redaction.js";
5
5
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
6
6
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
7
- import { participantFeedbackCandidates, providerResourcesForOutcome, publicSafeAppUrlLabel, } from "./bundle-parts.js";
7
+ import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
8
+ import { participantFeedbackCandidates } from "./participant-feedback.js";
8
9
  import { participantFactsOf } from "./participant-facts.js";
9
10
  import { participantPassed, participantStatus } from "../../run/judge.js";
10
11
  import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
@@ -141,7 +142,6 @@ function fanoutFeedbackCandidates(args) {
141
142
  runId: args.run.runId,
142
143
  scenarioId: `cua-${plan.studyId}`,
143
144
  adapterId: plan.studyId,
144
- goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
145
145
  substrate: plan.residual.execution?.target === "local" ? "local-desktop" : "e2b-desktop",
146
146
  participants: specs.map((spec, index) => {
147
147
  const outcome = outcomes?.[index];
@@ -149,6 +149,7 @@ function fanoutFeedbackCandidates(args) {
149
149
  participantId: spec.planned.id,
150
150
  streamId: spec.streamId,
151
151
  personaId: spec.persona.id,
152
+ goal: redactText(spec.evidenceInstructions ?? spec.instructions),
152
153
  ...(outcome?.session === undefined ? {} : { session: outcome.session }),
153
154
  ...(outcome?.session === undefined
154
155
  ? {}
@@ -15,7 +15,7 @@ import { participantSubjectEnv } from "./types.js";
15
15
  export async function runStudyParticipants(setup, participants) {
16
16
  const { plan, input, cwd, streams, descriptor, run } = setup;
17
17
  const { participantRuns, participantPlan, bundleBase } = setup;
18
- const { env, knownSecretValues, deps, liveTrace, externalComms, inProgress, fail } = participants;
18
+ const { env, deps, liveTrace, externalComms, inProgress, fail } = participants;
19
19
  const { dryRun } = plan;
20
20
  const inProcess = plan.runner.desktop === "in-process";
21
21
  const subjectEnvNames = [...participantSubjectEnv(plan.runner.subject)];
@@ -76,11 +76,7 @@ export async function runStudyParticipants(setup, participants) {
76
76
  env,
77
77
  participants: participantRuns.map((spec) => spec.planned.id),
78
78
  runPaths,
79
- registerSecrets: (values) => {
80
- for (const value of values)
81
- if (value.length >= 4 && !knownSecretValues.includes(value))
82
- knownSecretValues.push(value);
83
- },
79
+ registerSecrets: (values) => run.secrets.add(values),
84
80
  });
85
81
  if (receiving)
86
82
  deps.receiving = receiving;
@@ -129,7 +125,7 @@ export async function runStudyParticipants(setup, participants) {
129
125
  runPaths,
130
126
  participantRuns: participantRuns,
131
127
  outcomes,
132
- knownSecretValues,
128
+ knownSecretValues: run.secrets.values(),
133
129
  })
134
130
  : [];
135
131
  return {
@@ -0,0 +1,46 @@
1
+ import type { ComputerUsePlan, SharedWorldPlan } from "../../study/plan-types.js";
2
+ import type { StudyDeps } from "../../study/study-deps.js";
3
+ import type { LiveTraceFlush } from "./live-flush.js";
4
+ import type { CuaParticipantDeps } from "./types.js";
5
+ /** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
6
+ export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalProvisioned" | "appUrl" | "onObservedUrl">;
7
+ /** What every participant of a desktop run reads. Each route supplies all of it. */
8
+ interface DesktopRunFacts {
9
+ plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
10
+ /** The plan's brain. A local agent runs each participant on the operator's signed-in CLI. */
11
+ brain: CuaParticipantDeps["brain"];
12
+ /** What the participant's desktop does with the subject. */
13
+ subject: CuaParticipantDeps["subject"];
14
+ /** The run's env. The E2B request timeout is read from it. */
15
+ env: CuaParticipantDeps["env"];
16
+ openaiApiKey: string;
17
+ e2bApiKey: string;
18
+ /** Each participant's session budget. */
19
+ timeoutMs: number;
20
+ /** Each participant's sandbox lifetime. */
21
+ sandboxMs: number;
22
+ participantCount: number;
23
+ artifactRoot: CuaParticipantDeps["artifactRoot"];
24
+ studyCwd: string;
25
+ redactScreenshots: boolean;
26
+ scrubKnownValues: (text: string) => string;
27
+ runSession: CuaParticipantDeps["runSession"];
28
+ /** The run's live-trace flush, once a live run has started it. */
29
+ liveFlush: () => LiveTraceFlush["flush"] | undefined;
30
+ onStream: CuaParticipantDeps["onStream"];
31
+ reportSubjectPhase: CuaParticipantDeps["reportSubjectPhase"];
32
+ /** The run's test seams: the clock, the E2B SDK loader and the detached timers. */
33
+ seams: StudyDeps;
34
+ /** The caller's prepareDesktop, which each participant's desktop runs. */
35
+ prepareDesktop: CuaParticipantDeps["prepareDesktop"];
36
+ }
37
+ /** The fields one desktop route sets and the other does not. Undefined leaves a field out. */
38
+ type DesktopRouteOverrides = {
39
+ [K in "createDesktop" | "createProvider" | "inProcessExecutor" | "localTreeArchiveBuffer" | "externalComms" | "receiving" | "screenMismatchPolicy"]?: CuaParticipantDeps[K] | undefined;
40
+ };
41
+ /**
42
+ * One run's participant deps. Call it once per run: the run budget it creates is the one ledger
43
+ * every participant notes its spend on, and a dry run, which never spends, gets none.
44
+ */
45
+ export declare function desktopParticipantDeps(run: DesktopRunFacts, overrides: DesktopRouteOverrides): DesktopParticipantDeps;
46
+ export {};
@@ -0,0 +1,48 @@
1
+ // The deps every desktop participant reads, built here for computer use and shared world. A field
2
+ // both routes hand their participants is added once, in desktopParticipantDeps; each route passes
3
+ // what only it knows as overrides.
4
+ import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
5
+ import { makeCuaRunBudget } from "./participant-model.js";
6
+ /**
7
+ * One run's participant deps. Call it once per run: the run budget it creates is the one ledger
8
+ * every participant notes its spend on, and a dry run, which never spends, gets none.
9
+ */
10
+ export function desktopParticipantDeps(run, overrides) {
11
+ const { plan, seams } = run;
12
+ const { maxTotalUsd } = plan.caps;
13
+ const optional = {
14
+ ...overrides,
15
+ prepareDesktop: run.prepareDesktop,
16
+ desktopModule: seams.desktopModule,
17
+ detachedTimers: seams.detachedTimers,
18
+ runBudget: plan.dryRun || maxTotalUsd === undefined ? undefined : makeCuaRunBudget(maxTotalUsd),
19
+ };
20
+ return {
21
+ ...definedFields(optional),
22
+ onTrace: (participantId, items, usage, metadata) => run.liveFlush()?.(participantId, items, usage, metadata),
23
+ residual: plan.residual,
24
+ studyId: plan.studyId,
25
+ caps: plan.caps,
26
+ brain: run.brain,
27
+ subject: run.subject,
28
+ env: run.env,
29
+ openaiApiKey: run.openaiApiKey,
30
+ e2bApiKey: run.e2bApiKey,
31
+ requestTimeoutMs: e2bRequestTimeoutMs(run.env),
32
+ sandboxMs: run.sandboxMs,
33
+ timeoutMs: run.timeoutMs,
34
+ participantCount: run.participantCount,
35
+ artifactRoot: run.artifactRoot,
36
+ studyCwd: run.studyCwd,
37
+ redactScreenshots: run.redactScreenshots,
38
+ scrubKnownValues: run.scrubKnownValues,
39
+ runSession: run.runSession,
40
+ now: seams.now ?? Date.now,
41
+ onStream: run.onStream,
42
+ reportSubjectPhase: run.reportSubjectPhase,
43
+ };
44
+ }
45
+ /** The fields of `fields` that are set. The deps never carry a key whose value is undefined. */
46
+ function definedFields(fields) {
47
+ return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined));
48
+ }
@@ -5,8 +5,9 @@ export declare function participantFactsOf(outcome: ParticipantRunOutcome): Part
5
5
  /** Whether a participant passed. A dry-run participant passes as a contract. */
6
6
  export declare function participantOutcomeOk(outcome: ParticipantRunOutcome | undefined, dryRun: boolean): boolean;
7
7
  /**
8
- * One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
9
- * was not killed, or a desktop whose release says why (a local VM names its container). A
10
- * participant that never acquired a desktop (skipped, in-process) adds none.
8
+ * The execution failures a run's participants caused, by kind: each session that failed in the
9
+ * harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
10
+ * item after its last request, then each desktop not confirmed released. Participants keep their
11
+ * order within each kind.
11
12
  */
12
- export declare function unreleasedSandboxFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
13
+ export declare function participantExecutionFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
@@ -1,7 +1,7 @@
1
1
  // A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
2
2
  // them. The bundle builders, the result, the participant runner and shared-world all read these,
3
3
  // so they sit in a file that imports only the judge and the types.
4
- import { participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
4
+ import { participantHarnessFailed, participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
5
5
  /** A participant outcome's facts for the judge. */
6
6
  export function participantFactsOf(outcome) {
7
7
  return {
@@ -20,12 +20,47 @@ export function participantOutcomeOk(outcome, dryRun) {
20
20
  return true;
21
21
  return outcome !== undefined && participantPassed(participantFactsOf(outcome));
22
22
  }
23
+ /**
24
+ * The execution failures a run's participants caused, by kind: each session that failed in the
25
+ * harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
26
+ * item after its last request, then each desktop not confirmed released. Participants keep their
27
+ * order within each kind.
28
+ */
29
+ export function participantExecutionFailures(outcomes, runId) {
30
+ const all = outcomes ?? [];
31
+ const named = (outcome, text) => `${outcome.spec.planned.id}: ${text}`;
32
+ return [
33
+ ...all
34
+ .filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
35
+ .map((outcome) => ({
36
+ kind: "harness",
37
+ message: named(outcome, outcome.sessionError ?? outcome.session?.reason ?? "harness error"),
38
+ })),
39
+ ...all.flatMap((outcome) => outcome.providerCleanupError === undefined
40
+ ? []
41
+ : [
42
+ {
43
+ kind: "provider-cleanup",
44
+ message: named(outcome, outcome.providerCleanupError),
45
+ },
46
+ ]),
47
+ ...all.flatMap((outcome) => outcome.providerPolicyError === undefined
48
+ ? []
49
+ : [
50
+ {
51
+ kind: "provider-policy",
52
+ message: named(outcome, outcome.providerPolicyError),
53
+ },
54
+ ]),
55
+ ...unreleasedSandboxFailures(all, runId),
56
+ ];
57
+ }
23
58
  /**
24
59
  * One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
25
60
  * was not killed, or a desktop whose release says why (a local VM names its container). A
26
61
  * participant that never acquired a desktop (skipped, in-process) adds none.
27
62
  */
28
- export function unreleasedSandboxFailures(outcomes, runId) {
63
+ function unreleasedSandboxFailures(outcomes, runId) {
29
64
  return (outcomes ?? [])
30
65
  .filter((outcome) => outcome.sandboxRelease !== undefined ||
31
66
  (outcome.sandboxId !== undefined && !outcome.killed))
@@ -0,0 +1,33 @@
1
+ import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
2
+ import type { RunFeedbackCandidate } from "../../run/bundle.js";
3
+ /**
4
+ * Feedback candidates derived from what live participants actually reported.
5
+ *
6
+ * A live run's feedback draft comes from these candidates, never from a dry-run template. The
7
+ * candidate worth filing is the one the study produced: a participant
8
+ * who reported friction on the way (the most valuable thing a run captures), or one who stopped
9
+ * trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
10
+ * any falls back to a live summary in the draft layer instead of a template.
11
+ *
12
+ * Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
13
+ * through redactNarration in the loop) and passes redactText again here as defense-in-depth.
14
+ */
15
+ export declare function participantFeedbackCandidates(args: {
16
+ runId: string;
17
+ scenarioId: string;
18
+ adapterId: string;
19
+ substrate: RunFeedbackCandidate["substrate"];
20
+ participants: Array<{
21
+ /** The participant's plan id; it names the candidate and its idempotency key. */
22
+ participantId: string;
23
+ streamId: string;
24
+ personaId: string;
25
+ /** The participant's own already-redacted instructions, which the candidate's `expected`
26
+ * quotes. Participants of one study get different instructions, each naming its persona. */
27
+ goal: string;
28
+ session?: CuaLoopResult;
29
+ traceArtifactPath?: string;
30
+ screenshots: string[];
31
+ commsArtifactPath?: string;
32
+ }>;
33
+ }): RunFeedbackCandidate[];
@@ -0,0 +1,92 @@
1
+ // Feedback candidates from what live participants reported. The computer-use bundles (single and
2
+ // fan-out) and the shared-world bundle all build them here, so a participant's report becomes the
3
+ // same candidate whichever route ran it.
4
+ import { redactText } from "../../evidence/redaction.js";
5
+ import { feedbackProofCommands } from "../../feedback/proof.js";
6
+ import { resolveSelfReportedFriction } from "./self-report.js";
7
+ /**
8
+ * Feedback candidates derived from what live participants actually reported.
9
+ *
10
+ * A live run's feedback draft comes from these candidates, never from a dry-run template. The
11
+ * candidate worth filing is the one the study produced: a participant
12
+ * who reported friction on the way (the most valuable thing a run captures), or one who stopped
13
+ * trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
14
+ * any falls back to a live summary in the draft layer instead of a template.
15
+ *
16
+ * Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
17
+ * through redactNarration in the loop) and passes redactText again here as defense-in-depth.
18
+ */
19
+ export function participantFeedbackCandidates(args) {
20
+ const candidates = [];
21
+ for (const participant of args.participants) {
22
+ const session = participant.session;
23
+ if (session === undefined)
24
+ continue;
25
+ const friction = resolveSelfReportedFriction(session);
26
+ const abandoned = session.status === "abandoned";
27
+ if (friction === undefined && !abandoned)
28
+ continue;
29
+ const summary = friction !== undefined
30
+ ? `Participant ${participant.personaId} (${participant.participantId}) reported friction on the way through the study goal`
31
+ : `Participant ${participant.personaId} (${participant.participantId}) stopped before completing the study goal`;
32
+ const lastScreenshot = participant.screenshots[participant.screenshots.length - 1];
33
+ candidates.push({
34
+ schema: "humanish.feedback-candidate.v1",
35
+ id: `participant-report-${participant.participantId}`,
36
+ run_id: args.runId,
37
+ stream_id: participant.streamId,
38
+ adapter_id: args.adapterId,
39
+ scenario_id: args.scenarioId,
40
+ persona_id: participant.personaId,
41
+ actor: "computer-use",
42
+ substrate: args.substrate,
43
+ // The participant is reporting on the product: friction and abandonment are target-app
44
+ // findings by the three-roles rule. A harness failure never reaches this builder; it is
45
+ // not a participant report.
46
+ failure_owner: "target-app",
47
+ summary,
48
+ expected: participant.goal,
49
+ actual: redactText(friction ?? session.reason),
50
+ evidence: [
51
+ ...(participant.traceArtifactPath === undefined
52
+ ? []
53
+ : [
54
+ {
55
+ path: participant.traceArtifactPath,
56
+ kind: "trace",
57
+ note: "Full actor trace: turns, actions, and the participant's own report.",
58
+ },
59
+ ]),
60
+ ...(lastScreenshot === undefined
61
+ ? []
62
+ : [
63
+ {
64
+ path: lastScreenshot,
65
+ kind: "screenshot",
66
+ note: "Final screenshot at the moment the session ended.",
67
+ },
68
+ ]),
69
+ ...(participant.commsArtifactPath === undefined
70
+ ? []
71
+ : [
72
+ {
73
+ path: participant.commsArtifactPath,
74
+ kind: "log",
75
+ note: "Digest-only comms thread captured in-sandbox.",
76
+ },
77
+ ]),
78
+ ],
79
+ redaction: {
80
+ status: "passed",
81
+ notes: "Quoted participant text passed the loop's known-value scrub and pattern redaction before persisting, and redactText again here.",
82
+ },
83
+ idempotency_key: `humanish:${args.runId}:${participant.participantId}:participant-report`,
84
+ proposed_next_state: "study-quality-review",
85
+ acceptance_proof: [
86
+ feedbackProofCommands(args.runId).verify,
87
+ feedbackProofCommands(args.runId).observe,
88
+ ],
89
+ });
90
+ }
91
+ return candidates;
92
+ }
@@ -41,4 +41,15 @@ export declare function judgeParticipantSession(session: CuaLoopResult | undefin
41
41
  selfReportedBlocker: boolean;
42
42
  reportedFriction: boolean;
43
43
  };
44
+ /**
45
+ * caps.maxUsd is enforced inside each participant's loop independently, so N participants can
46
+ * spend up to N × maxUsd before any of them stops, while the run cost summary reports the larger
47
+ * aggregate. The warning names that ceiling, unless the study declared a shared maxTotalUsd budget.
48
+ * `capped` names what the cap counts: shared world says model spend, because the external-public
49
+ * plane's lobby-code reads are not counted against it.
50
+ */
51
+ export declare function participantCapWarning(caps: {
52
+ readonly maxUsd?: number;
53
+ readonly maxTotalUsd?: number;
54
+ }, participantCount: number, capped?: string): string | undefined;
44
55
  export declare function makeCuaRunBudget(maxTotalUsd: number): CuaRunBudget;
@@ -257,6 +257,21 @@ export function judgeParticipantSession(session, deps, warnings) {
257
257
  }
258
258
  return { noEngagement, selfReportedBlocker, reportedFriction };
259
259
  }
260
+ /**
261
+ * caps.maxUsd is enforced inside each participant's loop independently, so N participants can
262
+ * spend up to N × maxUsd before any of them stops, while the run cost summary reports the larger
263
+ * aggregate. The warning names that ceiling, unless the study declared a shared maxTotalUsd budget.
264
+ * `capped` names what the cap counts: shared world says model spend, because the external-public
265
+ * plane's lobby-code reads are not counted against it.
266
+ */
267
+ export function participantCapWarning(caps, participantCount, capped = "each participant") {
268
+ const capUsd = caps.maxUsd;
269
+ if (capUsd === undefined || participantCount <= 1)
270
+ return undefined;
271
+ if (caps.maxTotalUsd !== undefined)
272
+ return undefined;
273
+ return `caps.maxUsd ($${capUsd}) caps ${capped}, so ${participantCount} participants may spend up to ${participantCount} × $${capUsd} (about $${round6(capUsd * participantCount)}) before any of them stops. Set caps.maxTotalUsd for one budget across the study.`;
274
+ }
260
275
  export function makeCuaRunBudget(maxTotalUsd) {
261
276
  const participantEstimates = new Map();
262
277
  return {