humanish 0.111.2 → 0.111.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/ARCHITECTURE.md +1 -1
  2. package/CHANGELOG.md +17 -0
  3. package/dist/cli/commands/study-route-computer-use.js +3 -8
  4. package/dist/cli/commands/study-route-preview.js +4 -6
  5. package/dist/cli/commands/study-route-shared-world.js +6 -7
  6. package/dist/evidence/redaction.d.ts +2 -2
  7. package/dist/evidence/redaction.js +3 -3
  8. package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
  9. package/dist/routes/computer-use/bundle-parts.js +5 -92
  10. package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
  11. package/dist/routes/computer-use/fanout-bundle.js +3 -2
  12. package/dist/routes/computer-use/live-phase.js +3 -7
  13. package/dist/routes/computer-use/participant-deps.d.ts +46 -0
  14. package/dist/routes/computer-use/participant-deps.js +48 -0
  15. package/dist/routes/computer-use/participant-facts.d.ts +5 -4
  16. package/dist/routes/computer-use/participant-facts.js +37 -2
  17. package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
  18. package/dist/routes/computer-use/participant-feedback.js +92 -0
  19. package/dist/routes/computer-use/participant-model.d.ts +11 -0
  20. package/dist/routes/computer-use/participant-model.js +15 -0
  21. package/dist/routes/computer-use/preflight.js +33 -65
  22. package/dist/routes/computer-use/result.js +13 -52
  23. package/dist/routes/computer-use/route.js +6 -7
  24. package/dist/routes/computer-use/setup.d.ts +2 -6
  25. package/dist/routes/computer-use/setup.js +47 -93
  26. package/dist/routes/computer-use/single-bundle.js +3 -2
  27. package/dist/routes/computer-use/types.d.ts +2 -2
  28. package/dist/routes/preview.js +2 -8
  29. package/dist/routes/scripted/bundle.d.ts +2 -6
  30. package/dist/routes/scripted/result.d.ts +2 -1
  31. package/dist/routes/scripted/result.js +4 -4
  32. package/dist/routes/scripted/route.js +13 -38
  33. package/dist/routes/scripted/setup.d.ts +2 -1
  34. package/dist/routes/scripted/setup.js +21 -28
  35. package/dist/routes/scripted/subject.d.ts +3 -18
  36. package/dist/routes/scripted/subject.js +22 -78
  37. package/dist/routes/scripted/types.d.ts +2 -2
  38. package/dist/routes/shared-world/bundle.js +36 -1
  39. package/dist/routes/shared-world/comms.d.ts +2 -1
  40. package/dist/routes/shared-world/comms.js +3 -7
  41. package/dist/routes/shared-world/handoff.d.ts +2 -2
  42. package/dist/routes/shared-world/handoff.js +1 -1
  43. package/dist/routes/shared-world/participant-specs.d.ts +4 -5
  44. package/dist/routes/shared-world/participant-specs.js +19 -25
  45. package/dist/routes/shared-world/plan.js +11 -7
  46. package/dist/routes/shared-world/provisioned.d.ts +2 -1
  47. package/dist/routes/shared-world/provisioned.js +37 -80
  48. package/dist/routes/shared-world/result.js +17 -35
  49. package/dist/routes/shared-world/route.js +33 -36
  50. package/dist/routes/shared-world/setup.d.ts +2 -3
  51. package/dist/routes/shared-world/setup.js +11 -29
  52. package/dist/routes/shared-world/types.d.ts +6 -15
  53. package/dist/routes/terminal/dry-run.js +14 -24
  54. package/dist/routes/terminal/live-finish.d.ts +0 -1
  55. package/dist/routes/terminal/live-finish.js +2 -2
  56. package/dist/routes/terminal/route.js +9 -15
  57. package/dist/routes/terminal/session.d.ts +2 -3
  58. package/dist/routes/terminal/session.js +28 -40
  59. package/dist/routes/terminal/trace.d.ts +1 -1
  60. package/dist/routes/terminal/types.d.ts +2 -2
  61. package/dist/run/run.d.ts +55 -1
  62. package/dist/run/run.js +38 -2
  63. package/dist/run/secrets.d.ts +22 -0
  64. package/dist/run/secrets.js +33 -0
  65. package/dist/run/study-result.d.ts +34 -0
  66. package/dist/run/study-result.js +16 -0
  67. package/dist/study/requirements.d.ts +60 -5
  68. package/dist/study/requirements.js +88 -2
  69. package/dist/study/run-study-events.d.ts +4 -3
  70. package/dist/study/run-study-events.js +9 -14
  71. package/dist/study/run-study-options.d.ts +4 -1
  72. package/dist/study/run-study-options.js +22 -32
  73. package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
  74. package/dist/substrates/e2b/subject-sandbox.js +94 -0
  75. package/docs/contracts/core.md +5 -4
  76. package/docs/contracts/feedback.md +12 -5
  77. package/package.json +1 -1
package/ARCHITECTURE.md CHANGED
@@ -129,7 +129,7 @@ comments name an invariant by its rule.
129
129
  | The run bundle is the source of truth, and the Observer derives from it | `verifyRun` grades only `run.json` and its artifacts (`src/verify/verify.ts`). It reads `status.json` and the sandbox and reclaim receipts only for the `RUN_NOT_FINISHED` warning (`runNotFinished` in `src/verify/liveness.ts`), which changes no check and no grade. `buildObserverData` takes the bundle (`src/observer/data.ts`) | `tests/observer/data-contract.test.ts`; `tests/verify/run-not-finished.test.ts` checks that the grades are the same when `status.json` contradicts the bundle or is gone; `docs/decisions/0001-run-bundle-is-the-source-of-truth.md` |
130
130
  | Unsupported execution is refused before side effects | `parseStudy` (`src/study/config.ts`), `taskProtocolValidationReason` (`src/study/validation.ts`), `planComputerUseStudy` (`src/routes/computer-use/plan.ts`) | `tests/admission/library.test.ts` checks each library refusal for a run directory, a desktop module load, a caller executor or provider, and a subprocess; `tests/admission/cli.test.ts` checks each CLI refusal for a run directory, a started process and a scorer's host code; `tests/study/task-route-preflight.test.ts`; `pnpm cli:preflight:test` runs `scripts/task-route-preflight-proof.mjs` with `tests/fixtures/task-route-preflight/deny-side-effects.mjs` preloaded |
131
131
  | A sandbox id is recorded before any work runs in it | `acquireE2BDesktopSandbox` and `acquireE2BShellSandbox` (`src/substrates/e2b/sandbox.ts`) tag the sandbox with its run's owner tags, register the create with `beginSandboxCreate` (`src/run/sandbox-creates.ts`) and write the receipt before they return the handle, and every route that creates a sandbox calls them; `reclaimRunSandboxes` (`src/run/reclaim.ts`) switches on the receipt's provider, kills by exact id and by the run's exact tags | `tests/substrates/e2b/sandbox.test.ts`, which also checks that `Sandbox.create` has one call site in `src/`, `tests/routes/terminal/acquisition-boundary.test.ts`, `tests/run/reclaim.test.ts`; `tests/routes/scripted/route.test.ts` checks create, receipt, then first work |
132
- | Bundles carry no secrets | `redactText` and `scrubLiterals` (`src/evidence/redaction.ts`); `scanRunPublicSafetyArtifacts` (`src/verify/artifacts.ts`); `buildShareSafety` (`src/verify/verify.ts`) grades `share_ready`, `local_only` or `blocked` | `tests/evidence/redaction-hooks.test.ts`, `tests/run/transient-comms-secrets.test.ts`, `tests/verify/evidence-refs.test.ts` |
132
+ | Bundles carry no secrets | `redactText` and `scrubLiterals` (`src/evidence/redaction.ts`); `RunSecrets` (`src/run/secrets.ts`) holds the literal values each run scrubs; `scanRunPublicSafetyArtifacts` (`src/verify/artifacts.ts`); `buildShareSafety` (`src/verify/verify.ts`) grades `share_ready`, `local_only` or `blocked` | `tests/evidence/redaction-hooks.test.ts`, `tests/routes/known-secrets.test.ts`, `tests/run/transient-comms-secrets.test.ts`, `tests/verify/evidence-refs.test.ts` |
133
133
  | Goldens pin route output | `runDirSnapshot` (`tests/helpers/run-golden.ts`) snapshots a whole run folder | `tests/golden/routes/`, `tests/golden/observer-data/`, `tests/golden/labs/`; `tests/study/plan-lab.test.ts` checks that every `StudyRoute` has a dry-run golden |
134
134
  | A run is closed on every exit, only a published run is analyzed, and `latest.json` moves only after its run is published | `runScope` and `FinishedRun` (`src/run/run.ts`); `completeAutomaticAnalysis` requires an issued `FinishedRun` for the result's run; `Run.finish` writes `latest.json` last through `writePreparedRunLatestPointer` (`src/run/contained-output.ts`) | `tests/run/run-lifecycle.test.ts`, which also checks that `src/run/run.ts` is the pointer's one writer, `tests/analysis/automatic-analysis.test.ts` |
135
135
 
package/CHANGELOG.md CHANGED
@@ -8,6 +8,23 @@ The Unreleased section holds the full notes for the next version until it is tag
8
8
 
9
9
  ## Unreleased
10
10
 
11
+ ## 0.111.3: Shared-world feedback candidates, scripted E2B key scrubbed, subject sandbox warnings (2026-10-06)
12
+
13
+ humanish 0.111.3 gives shared-world runs feedback candidates and repairs seven places where one
14
+ route's warnings, refusals, secret scrubbing or run directory checks differed from the other
15
+ routes'. A shared-world participant who reports friction or gives up becomes a candidate that
16
+ `humanish feedback` drafts, and each candidate from a run with several participants shows that
17
+ participant's own instructions as `expected`. A scripted run scrubs the value of `E2B_API_KEY` from
18
+ the errors and subject setup output it records, and writes `[REDACTED_SECRET]` where it wrote
19
+ `[redacted]`. A shared-world study with `caps.maxUsd` and no `caps.maxTotalUsd` warns that the cap
20
+ applies to each participant. The subject sandbox of a scripted clone study or a provisioned
21
+ shared-world study warns when its create was retried or E2B reports no size. A scripted clone study
22
+ missing a `subject.env` value, and a provisioned shared-world study missing `OPENAI_API_KEY`,
23
+ refuse with the wording computer use gives. Computer-use and shared-world runs check the run
24
+ directory again after a `prepareDesktop` hook and an Observer render.
25
+
26
+ [Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.111.3)
27
+
11
28
  ## 0.111.2: A taller Observer frame, even grid cards, a notice when the server stops (2026-10-06)
12
29
 
13
30
  humanish 0.111.2 changes the Observer and nothing else. The focus view's heading line holds the
@@ -12,7 +12,7 @@ import { exposureRequestFromOptions, followDecision, followObserver, planObserve
12
12
  import { writeRunFindings } from "../findings.js";
13
13
  import { formatCuaStudyHuman } from "./study-format.js";
14
14
  import { resolveRouteShouldOpen } from "./study-route-open.js";
15
- import { studyResultIdentity } from "../../run/study-result.js";
15
+ import { refusedResult } from "../../run/study-result.js";
16
16
  import { declaredParticipantCount } from "../../study/study-fields.js";
17
17
  /**
18
18
  * The computer-use route's CLI setup: its settings, the watch and exposure plan, the runStudyWith options
@@ -51,17 +51,12 @@ export function computerUseRouteRun(args) {
51
51
  };
52
52
  }
53
53
  function refuseCua(args, dryRun, code, message) {
54
- const result = {
55
- ...studyResultIdentity("computer-use", args.config.id),
56
- ok: false,
57
- cwd: args.options.cwd,
54
+ const result = refusedResult("computer-use", { studyId: args.config.id, cwd: args.options.cwd, error: { code, message } }, {
58
55
  actor: args.config.actor?.type ?? "",
59
56
  appUrl: "",
60
57
  dryRun,
61
58
  runId: args.options.runId ?? "not-created",
62
- warnings: [],
63
- error: { code, message },
64
- };
59
+ });
65
60
  writeResult(args.command, args.io, result, formatCuaStudyHuman);
66
61
  args.io.setExitCode(2);
67
62
  }
@@ -1,5 +1,5 @@
1
1
  import { Command } from "commander";
2
- import { studyResultIdentity } from "../../run/study-result.js";
2
+ import { refusedResult } from "../../run/study-result.js";
3
3
  import { formatRunHuman, parseStudyCount, writeResult, } from "../io.js";
4
4
  import { planObserver, showObserver, staticObserverOpen } from "../observer-follow.js";
5
5
  import { declaredParticipantCount } from "../../study/study-fields.js";
@@ -10,16 +10,14 @@ import { declaredParticipantCount } from "../../study/study-fields.js";
10
10
  export function previewRouteRun(args) {
11
11
  const participantCount = parseStudyCount(args.options.count, declaredParticipantCount(args.config) ?? 4);
12
12
  if (participantCount === null) {
13
- const result = {
14
- ...studyResultIdentity("preview", args.config.id),
15
- ok: false,
13
+ const result = refusedResult("preview", {
14
+ studyId: args.config.id,
16
15
  cwd: args.options.cwd,
17
- warnings: [],
18
16
  error: {
19
17
  code: "HUMANISH_INVALID_PARTICIPANT_COUNT",
20
18
  message: "--count must be a positive integer.",
21
19
  },
22
- };
20
+ }, {});
23
21
  writeResult(args.command, args.io, result, formatRunHuman);
24
22
  args.io.setExitCode(2);
25
23
  return undefined;
@@ -9,7 +9,7 @@ import { followDecision, followObserver, showObserver, staticObserverOpen, withO
9
9
  import { writeRunFindings } from "../findings.js";
10
10
  import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
11
11
  import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
12
- import { studyResultIdentity } from "../../run/study-result.js";
12
+ import { refusedResult } from "../../run/study-result.js";
13
13
  import { participantList } from "../../study/study-fields.js";
14
14
  /**
15
15
  * The shared-world route's CLI setup: dry run, open and follow semantics, the live Observer server
@@ -30,10 +30,11 @@ export function sharedWorldRouteRun(args) {
30
30
  followDecision({ wantsMachine, detach: args.options.detach, serve: args.options.serve }).follow;
31
31
  const port = parseObserverPort(args.options.port ?? "0");
32
32
  const failConcurrent = (message, runId) => {
33
- const result = {
34
- ...studyResultIdentity("shared-world", args.config.id),
35
- ok: false,
33
+ const result = refusedResult("shared-world", {
34
+ studyId: args.config.id,
36
35
  cwd: args.options.cwd,
36
+ error: { code: "HUMANISH_SHARED_WORLD_FAILED", message },
37
+ }, {
37
38
  actor: args.config.actor?.type ?? "",
38
39
  topology: "shared-world",
39
40
  topologyMode: "concurrent",
@@ -42,9 +43,7 @@ export function sharedWorldRouteRun(args) {
42
43
  dryRun,
43
44
  runId: runId ?? args.options.runId ?? "not-created",
44
45
  roles: [],
45
- warnings: [],
46
- error: { code: "HUMANISH_SHARED_WORLD_FAILED", message },
47
- };
46
+ });
48
47
  writeResult(args.command, args.io, result, formatConcurrentSharedWorldStudyHuman);
49
48
  args.io.setExitCode(2);
50
49
  };
@@ -112,11 +112,11 @@ export interface RedactionHooks {
112
112
  /** The default hooks wired into every actor: redaction.ts is the source of truth. */
113
113
  export declare const defaultRedactionHooks: RedactionHooks;
114
114
  /**
115
- * Replace each literal value with [REDACTED_SECRET]. For values that have no secret shape to
115
+ * Replace each literal value with `marker`. For values that have no secret shape to
116
116
  * pattern-match, such as provisioned keys and subject env values. `values` is read on each call,
117
117
  * so a value registered later is scrubbed too.
118
118
  */
119
- export declare function scrubLiterals(values: readonly string[]): (text: string) => string;
119
+ export declare function scrubLiterals(values: readonly string[], marker?: string): (text: string) => string;
120
120
  /**
121
121
  * Coerce an unknown thrown value to its message string. Prefer this over the
122
122
  * inline `error instanceof Error ? error.message : String(error)` so error
@@ -522,12 +522,12 @@ function blurPass(src, width, height, radius, horizontal) {
522
522
  return out;
523
523
  }
524
524
  /**
525
- * Replace each literal value with [REDACTED_SECRET]. For values that have no secret shape to
525
+ * Replace each literal value with `marker`. For values that have no secret shape to
526
526
  * pattern-match, such as provisioned keys and subject env values. `values` is read on each call,
527
527
  * so a value registered later is scrubbed too.
528
528
  */
529
- export function scrubLiterals(values) {
530
- return (text) => values.reduce((current, value) => current.split(value).join("[REDACTED_SECRET]"), text);
529
+ export function scrubLiterals(values, marker = "[REDACTED_SECRET]") {
530
+ return (text) => values.reduce((current, value) => current.split(value).join(marker), text);
531
531
  }
532
532
  /**
533
533
  * Coerce an unknown thrown value to its message string. Prefer this over the
@@ -1,39 +1,8 @@
1
- import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
2
- import { type RunFeedbackCandidate, type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
1
+ import { type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
3
2
  import { type ParticipantIds } from "../../run/participant-records.js";
4
3
  import { type CuaSubjectProvenanceArg, type ParticipantRunOutcome } from "./types.js";
5
4
  /** The human-readable state story appended to the provenance event (and review.md via it). */
6
5
  export declare function describeSubjectState(state: RunSubjectProvenance["state"], dryRun: boolean): string;
7
- /**
8
- * Feedback candidates derived from what live participants actually reported.
9
- *
10
- * A live run's feedback draft comes from these candidates, never from a dry-run template. The
11
- * candidate worth filing is the one the study produced: a participant
12
- * who reported friction on the way (the most valuable thing a run captures), or one who stopped
13
- * trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
14
- * any falls back to a live summary in the draft layer instead of a template.
15
- *
16
- * Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
17
- * through redactNarration in the loop) and passes redactText again here as defense-in-depth.
18
- */
19
- export declare function participantFeedbackCandidates(args: {
20
- runId: string;
21
- scenarioId: string;
22
- adapterId: string;
23
- /** The already-redacted study goal (what bundle.scenario.goal carries). */
24
- goal: string;
25
- substrate: RunFeedbackCandidate["substrate"];
26
- participants: Array<{
27
- /** The participant's plan id; it names the candidate and its idempotency key. */
28
- participantId: string;
29
- streamId: string;
30
- personaId: string;
31
- session?: CuaLoopResult;
32
- traceArtifactPath?: string;
33
- screenshots: string[];
34
- commsArtifactPath?: string;
35
- }>;
36
- }): RunFeedbackCandidate[];
37
6
  /** Human-readable provenance line for the single-participant subject.provenance event:
38
7
  * claims "cloned/packed and served" only when it actually happened. */
39
8
  export declare function subjectProvenanceMessage(provenance: CuaSubjectProvenanceArg, publicAppUrl: string, dryRun: boolean, hasSession: boolean): string;
@@ -1,13 +1,12 @@
1
1
  // The parts both computer-use bundle shapes share: the subject-state story and provenance line,
2
- // feedback candidates from what live participants reported, provider-resource records, and the
3
- // public-safe app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
4
- // single-bundle.ts and fanout-bundle.ts.
5
- import { feedbackProofCommands } from "../../feedback/proof.js";
2
+ // provider-resource records, and the public-safe app URL label, the URL digest and the
3
+ // subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
4
+ // single-bundle.ts and fanout-bundle.ts. Feedback candidates from participant reports are in
5
+ // participant-feedback.ts, which the shared-world bundle reads too.
6
6
  import { scanEncodedText } from "../../evidence/encoded-text.js";
7
- import { digestText, redactText } from "../../evidence/redaction.js";
7
+ import { digestText } from "../../evidence/redaction.js";
8
8
  import {} from "../../run/bundle.js";
9
9
  import { participantResourceIds } from "../../run/participant-records.js";
10
- import { resolveSelfReportedFriction } from "./self-report.js";
11
10
  import {} from "./types.js";
12
11
  import { plural } from "../../run/text.js";
13
12
  /** The human-readable state story appended to the provenance event (and review.md via it). */
@@ -25,92 +24,6 @@ export function describeSubjectState(state, dryRun) {
25
24
  return "external-public (operator-declared, operator-owned public deployment; neither provisioned nor seeded)";
26
25
  }
27
26
  }
28
- /**
29
- * Feedback candidates derived from what live participants actually reported.
30
- *
31
- * A live run's feedback draft comes from these candidates, never from a dry-run template. The
32
- * candidate worth filing is the one the study produced: a participant
33
- * who reported friction on the way (the most valuable thing a run captures), or one who stopped
34
- * trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
35
- * any falls back to a live summary in the draft layer instead of a template.
36
- *
37
- * Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
38
- * through redactNarration in the loop) and passes redactText again here as defense-in-depth.
39
- */
40
- export function participantFeedbackCandidates(args) {
41
- const candidates = [];
42
- for (const participant of args.participants) {
43
- const session = participant.session;
44
- if (session === undefined)
45
- continue;
46
- const friction = resolveSelfReportedFriction(session);
47
- const abandoned = session.status === "abandoned";
48
- if (friction === undefined && !abandoned)
49
- continue;
50
- const summary = friction !== undefined
51
- ? `Participant ${participant.personaId} (${participant.participantId}) reported friction on the way through the study goal`
52
- : `Participant ${participant.personaId} (${participant.participantId}) stopped before completing the study goal`;
53
- const lastScreenshot = participant.screenshots[participant.screenshots.length - 1];
54
- candidates.push({
55
- schema: "humanish.feedback-candidate.v1",
56
- id: `participant-report-${participant.participantId}`,
57
- run_id: args.runId,
58
- stream_id: participant.streamId,
59
- adapter_id: args.adapterId,
60
- scenario_id: args.scenarioId,
61
- persona_id: participant.personaId,
62
- actor: "computer-use",
63
- substrate: args.substrate,
64
- // The participant is reporting on the product: friction and abandonment are target-app
65
- // findings by the three-roles rule. A harness failure never reaches this builder; it is
66
- // not a participant report.
67
- failure_owner: "target-app",
68
- summary,
69
- expected: args.goal,
70
- actual: redactText(friction ?? session.reason),
71
- evidence: [
72
- ...(participant.traceArtifactPath === undefined
73
- ? []
74
- : [
75
- {
76
- path: participant.traceArtifactPath,
77
- kind: "trace",
78
- note: "Full actor trace: turns, actions, and the participant's own report.",
79
- },
80
- ]),
81
- ...(lastScreenshot === undefined
82
- ? []
83
- : [
84
- {
85
- path: lastScreenshot,
86
- kind: "screenshot",
87
- note: "Final screenshot at the moment the session ended.",
88
- },
89
- ]),
90
- ...(participant.commsArtifactPath === undefined
91
- ? []
92
- : [
93
- {
94
- path: participant.commsArtifactPath,
95
- kind: "log",
96
- note: "Digest-only comms thread captured in-sandbox.",
97
- },
98
- ]),
99
- ],
100
- redaction: {
101
- status: "passed",
102
- notes: "Quoted participant text passed the loop's known-value scrub and pattern redaction before persisting, and redactText again here.",
103
- },
104
- idempotency_key: `humanish:${args.runId}:${participant.participantId}:participant-report`,
105
- proposed_next_state: "study-quality-review",
106
- acceptance_proof: [
107
- feedbackProofCommands(args.runId).verify,
108
- feedbackProofCommands(args.runId).observe,
109
- ],
110
- });
111
- }
112
- return candidates;
113
- }
114
27
  /** Human-readable provenance line for the single-participant subject.provenance event:
115
28
  * claims "cloned/packed and served" only when it actually happened. */
116
29
  export function subjectProvenanceMessage(provenance, publicAppUrl, dryRun, hasSession) {
@@ -1,6 +1,7 @@
1
1
  // The first half of an E2B desktop participant's preparation: acquire the sandbox, verify its screen, and
2
2
  // provision the subject. Each step fills the participant's state; a step that cannot proceed throws.
3
3
  import { redactText } from "../../../evidence/redaction.js";
4
+ import { validatePreparedRunArtifactPaths } from "../../../run/paths.js";
4
5
  import { provisionCloneSubject } from "../../../subject/clone.js";
5
6
  import { provisionDesktopCli } from "../../../subject/desktop-cli.js";
6
7
  import { provisionLocalTreeSubject } from "../../../subject/local-tree.js";
@@ -94,10 +95,16 @@ export async function acquireParticipantDesktop(ctx, state) {
94
95
  if ("reason" in desktopResources) {
95
96
  warnings.push(`Desktop resource size unavailable (${desktopResources.reason}); compute cost remains unpriced.`);
96
97
  }
97
- await deps.prepareDesktop?.(desktop, {
98
- kind: "participant",
99
- participant: { id: spec.planned.id, index: spec.planned.index, count: deps.participantCount },
100
- });
98
+ if (deps.prepareDesktop) {
99
+ await deps.prepareDesktop(desktop, {
100
+ kind: "participant",
101
+ participant: { id: spec.planned.id, index: spec.planned.index, count: deps.participantCount },
102
+ });
103
+ // The caller's hook runs with this process's file access, so the run directory is checked
104
+ // again before more evidence goes into it.
105
+ if ("physicalRunRoot" in deps.artifactRoot)
106
+ await validatePreparedRunArtifactPaths(deps.artifactRoot);
107
+ }
101
108
  if (deps.receiving) {
102
109
  state.receivingInboxUrl = await attachReceivingInbox(shell, spec, { ...deps, receiving: deps.receiving }, ctx.targetUrl);
103
110
  state.commsArtifactPath = "comms/receiving.json";
@@ -4,7 +4,8 @@ import { actorEnding } from "../../actors/stop-cause.js";
4
4
  import { redactText } from "../../evidence/redaction.js";
5
5
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
6
6
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
7
- import { participantFeedbackCandidates, providerResourcesForOutcome, publicSafeAppUrlLabel, } from "./bundle-parts.js";
7
+ import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
8
+ import { participantFeedbackCandidates } from "./participant-feedback.js";
8
9
  import { participantFactsOf } from "./participant-facts.js";
9
10
  import { participantPassed, participantStatus } from "../../run/judge.js";
10
11
  import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
@@ -141,7 +142,6 @@ function fanoutFeedbackCandidates(args) {
141
142
  runId: args.run.runId,
142
143
  scenarioId: `cua-${plan.studyId}`,
143
144
  adapterId: plan.studyId,
144
- goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
145
145
  substrate: plan.residual.execution?.target === "local" ? "local-desktop" : "e2b-desktop",
146
146
  participants: specs.map((spec, index) => {
147
147
  const outcome = outcomes?.[index];
@@ -149,6 +149,7 @@ function fanoutFeedbackCandidates(args) {
149
149
  participantId: spec.planned.id,
150
150
  streamId: spec.streamId,
151
151
  personaId: spec.persona.id,
152
+ goal: redactText(spec.evidenceInstructions ?? spec.instructions),
152
153
  ...(outcome?.session === undefined ? {} : { session: outcome.session }),
153
154
  ...(outcome?.session === undefined
154
155
  ? {}
@@ -15,7 +15,7 @@ import { participantSubjectEnv } from "./types.js";
15
15
  export async function runStudyParticipants(setup, participants) {
16
16
  const { plan, input, cwd, streams, descriptor, run } = setup;
17
17
  const { participantRuns, participantPlan, bundleBase } = setup;
18
- const { env, knownSecretValues, deps, liveTrace, externalComms, inProgress, fail } = participants;
18
+ const { env, deps, liveTrace, externalComms, inProgress, fail } = participants;
19
19
  const { dryRun } = plan;
20
20
  const inProcess = plan.runner.desktop === "in-process";
21
21
  const subjectEnvNames = [...participantSubjectEnv(plan.runner.subject)];
@@ -76,11 +76,7 @@ export async function runStudyParticipants(setup, participants) {
76
76
  env,
77
77
  participants: participantRuns.map((spec) => spec.planned.id),
78
78
  runPaths,
79
- registerSecrets: (values) => {
80
- for (const value of values)
81
- if (value.length >= 4 && !knownSecretValues.includes(value))
82
- knownSecretValues.push(value);
83
- },
79
+ registerSecrets: (values) => run.secrets.add(values),
84
80
  });
85
81
  if (receiving)
86
82
  deps.receiving = receiving;
@@ -129,7 +125,7 @@ export async function runStudyParticipants(setup, participants) {
129
125
  runPaths,
130
126
  participantRuns: participantRuns,
131
127
  outcomes,
132
- knownSecretValues,
128
+ knownSecretValues: run.secrets.values(),
133
129
  })
134
130
  : [];
135
131
  return {
@@ -0,0 +1,46 @@
1
+ import type { ComputerUsePlan, SharedWorldPlan } from "../../study/plan-types.js";
2
+ import type { StudyDeps } from "../../study/study-deps.js";
3
+ import type { LiveTraceFlush } from "./live-flush.js";
4
+ import type { CuaParticipantDeps } from "./types.js";
5
+ /** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
6
+ export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalProvisioned" | "appUrl" | "onObservedUrl">;
7
+ /** What every participant of a desktop run reads. Each route supplies all of it. */
8
+ interface DesktopRunFacts {
9
+ plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
10
+ /** The plan's brain. A local agent runs each participant on the operator's signed-in CLI. */
11
+ brain: CuaParticipantDeps["brain"];
12
+ /** What the participant's desktop does with the subject. */
13
+ subject: CuaParticipantDeps["subject"];
14
+ /** The run's env. The E2B request timeout is read from it. */
15
+ env: CuaParticipantDeps["env"];
16
+ openaiApiKey: string;
17
+ e2bApiKey: string;
18
+ /** Each participant's session budget. */
19
+ timeoutMs: number;
20
+ /** Each participant's sandbox lifetime. */
21
+ sandboxMs: number;
22
+ participantCount: number;
23
+ artifactRoot: CuaParticipantDeps["artifactRoot"];
24
+ studyCwd: string;
25
+ redactScreenshots: boolean;
26
+ scrubKnownValues: (text: string) => string;
27
+ runSession: CuaParticipantDeps["runSession"];
28
+ /** The run's live-trace flush, once a live run has started it. */
29
+ liveFlush: () => LiveTraceFlush["flush"] | undefined;
30
+ onStream: CuaParticipantDeps["onStream"];
31
+ reportSubjectPhase: CuaParticipantDeps["reportSubjectPhase"];
32
+ /** The run's test seams: the clock, the E2B SDK loader and the detached timers. */
33
+ seams: StudyDeps;
34
+ /** The caller's prepareDesktop, which each participant's desktop runs. */
35
+ prepareDesktop: CuaParticipantDeps["prepareDesktop"];
36
+ }
37
+ /** The fields one desktop route sets and the other does not. Undefined leaves a field out. */
38
+ type DesktopRouteOverrides = {
39
+ [K in "createDesktop" | "createProvider" | "inProcessExecutor" | "localTreeArchiveBuffer" | "externalComms" | "receiving" | "screenMismatchPolicy"]?: CuaParticipantDeps[K] | undefined;
40
+ };
41
+ /**
42
+ * One run's participant deps. Call it once per run: the run budget it creates is the one ledger
43
+ * every participant notes its spend on, and a dry run, which never spends, gets none.
44
+ */
45
+ export declare function desktopParticipantDeps(run: DesktopRunFacts, overrides: DesktopRouteOverrides): DesktopParticipantDeps;
46
+ export {};
@@ -0,0 +1,48 @@
1
+ // The deps every desktop participant reads, built here for computer use and shared world. A field
2
+ // both routes hand their participants is added once, in desktopParticipantDeps; each route passes
3
+ // what only it knows as overrides.
4
+ import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
5
+ import { makeCuaRunBudget } from "./participant-model.js";
6
+ /**
7
+ * One run's participant deps. Call it once per run: the run budget it creates is the one ledger
8
+ * every participant notes its spend on, and a dry run, which never spends, gets none.
9
+ */
10
+ export function desktopParticipantDeps(run, overrides) {
11
+ const { plan, seams } = run;
12
+ const { maxTotalUsd } = plan.caps;
13
+ const optional = {
14
+ ...overrides,
15
+ prepareDesktop: run.prepareDesktop,
16
+ desktopModule: seams.desktopModule,
17
+ detachedTimers: seams.detachedTimers,
18
+ runBudget: plan.dryRun || maxTotalUsd === undefined ? undefined : makeCuaRunBudget(maxTotalUsd),
19
+ };
20
+ return {
21
+ ...definedFields(optional),
22
+ onTrace: (participantId, items, usage, metadata) => run.liveFlush()?.(participantId, items, usage, metadata),
23
+ residual: plan.residual,
24
+ studyId: plan.studyId,
25
+ caps: plan.caps,
26
+ brain: run.brain,
27
+ subject: run.subject,
28
+ env: run.env,
29
+ openaiApiKey: run.openaiApiKey,
30
+ e2bApiKey: run.e2bApiKey,
31
+ requestTimeoutMs: e2bRequestTimeoutMs(run.env),
32
+ sandboxMs: run.sandboxMs,
33
+ timeoutMs: run.timeoutMs,
34
+ participantCount: run.participantCount,
35
+ artifactRoot: run.artifactRoot,
36
+ studyCwd: run.studyCwd,
37
+ redactScreenshots: run.redactScreenshots,
38
+ scrubKnownValues: run.scrubKnownValues,
39
+ runSession: run.runSession,
40
+ now: seams.now ?? Date.now,
41
+ onStream: run.onStream,
42
+ reportSubjectPhase: run.reportSubjectPhase,
43
+ };
44
+ }
45
+ /** The fields of `fields` that are set. The deps never carry a key whose value is undefined. */
46
+ function definedFields(fields) {
47
+ return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined));
48
+ }
@@ -5,8 +5,9 @@ export declare function participantFactsOf(outcome: ParticipantRunOutcome): Part
5
5
  /** Whether a participant passed. A dry-run participant passes as a contract. */
6
6
  export declare function participantOutcomeOk(outcome: ParticipantRunOutcome | undefined, dryRun: boolean): boolean;
7
7
  /**
8
- * One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
9
- * was not killed, or a desktop whose release says why (a local VM names its container). A
10
- * participant that never acquired a desktop (skipped, in-process) adds none.
8
+ * The execution failures a run's participants caused, by kind: each session that failed in the
9
+ * harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
10
+ * item after its last request, then each desktop not confirmed released. Participants keep their
11
+ * order within each kind.
11
12
  */
12
- export declare function unreleasedSandboxFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
13
+ export declare function participantExecutionFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
@@ -1,7 +1,7 @@
1
1
  // A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
2
2
  // them. The bundle builders, the result, the participant runner and shared-world all read these,
3
3
  // so they sit in a file that imports only the judge and the types.
4
- import { participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
4
+ import { participantHarnessFailed, participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
5
5
  /** A participant outcome's facts for the judge. */
6
6
  export function participantFactsOf(outcome) {
7
7
  return {
@@ -20,12 +20,47 @@ export function participantOutcomeOk(outcome, dryRun) {
20
20
  return true;
21
21
  return outcome !== undefined && participantPassed(participantFactsOf(outcome));
22
22
  }
23
+ /**
24
+ * The execution failures a run's participants caused, by kind: each session that failed in the
25
+ * harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
26
+ * item after its last request, then each desktop not confirmed released. Participants keep their
27
+ * order within each kind.
28
+ */
29
+ export function participantExecutionFailures(outcomes, runId) {
30
+ const all = outcomes ?? [];
31
+ const named = (outcome, text) => `${outcome.spec.planned.id}: ${text}`;
32
+ return [
33
+ ...all
34
+ .filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
35
+ .map((outcome) => ({
36
+ kind: "harness",
37
+ message: named(outcome, outcome.sessionError ?? outcome.session?.reason ?? "harness error"),
38
+ })),
39
+ ...all.flatMap((outcome) => outcome.providerCleanupError === undefined
40
+ ? []
41
+ : [
42
+ {
43
+ kind: "provider-cleanup",
44
+ message: named(outcome, outcome.providerCleanupError),
45
+ },
46
+ ]),
47
+ ...all.flatMap((outcome) => outcome.providerPolicyError === undefined
48
+ ? []
49
+ : [
50
+ {
51
+ kind: "provider-policy",
52
+ message: named(outcome, outcome.providerPolicyError),
53
+ },
54
+ ]),
55
+ ...unreleasedSandboxFailures(all, runId),
56
+ ];
57
+ }
23
58
  /**
24
59
  * One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
25
60
  * was not killed, or a desktop whose release says why (a local VM names its container). A
26
61
  * participant that never acquired a desktop (skipped, in-process) adds none.
27
62
  */
28
- export function unreleasedSandboxFailures(outcomes, runId) {
63
+ function unreleasedSandboxFailures(outcomes, runId) {
29
64
  return (outcomes ?? [])
30
65
  .filter((outcome) => outcome.sandboxRelease !== undefined ||
31
66
  (outcome.sandboxId !== undefined && !outcome.killed))
@@ -0,0 +1,33 @@
1
+ import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
2
+ import type { RunFeedbackCandidate } from "../../run/bundle.js";
3
+ /**
4
+ * Feedback candidates derived from what live participants actually reported.
5
+ *
6
+ * A live run's feedback draft comes from these candidates, never from a dry-run template. The
7
+ * candidate worth filing is the one the study produced: a participant
8
+ * who reported friction on the way (the most valuable thing a run captures), or one who stopped
9
+ * trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
10
+ * any falls back to a live summary in the draft layer instead of a template.
11
+ *
12
+ * Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
13
+ * through redactNarration in the loop) and passes redactText again here as defense-in-depth.
14
+ */
15
+ export declare function participantFeedbackCandidates(args: {
16
+ runId: string;
17
+ scenarioId: string;
18
+ adapterId: string;
19
+ substrate: RunFeedbackCandidate["substrate"];
20
+ participants: Array<{
21
+ /** The participant's plan id; it names the candidate and its idempotency key. */
22
+ participantId: string;
23
+ streamId: string;
24
+ personaId: string;
25
+ /** The participant's own already-redacted instructions, which the candidate's `expected`
26
+ * quotes. Participants of one study get different instructions, each naming its persona. */
27
+ goal: string;
28
+ session?: CuaLoopResult;
29
+ traceArtifactPath?: string;
30
+ screenshots: string[];
31
+ commsArtifactPath?: string;
32
+ }>;
33
+ }): RunFeedbackCandidate[];