humanish 0.111.2 → 0.111.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +13 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +21 -28
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +2 -2
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +1 -1
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +22 -0
- package/dist/run/secrets.js +33 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +88 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/package.json +1 -1
package/ARCHITECTURE.md
CHANGED
|
@@ -129,7 +129,7 @@ comments name an invariant by its rule.
|
|
|
129
129
|
| The run bundle is the source of truth, and the Observer derives from it | `verifyRun` grades only `run.json` and its artifacts (`src/verify/verify.ts`). It reads `status.json` and the sandbox and reclaim receipts only for the `RUN_NOT_FINISHED` warning (`runNotFinished` in `src/verify/liveness.ts`), which changes no check and no grade. `buildObserverData` takes the bundle (`src/observer/data.ts`) | `tests/observer/data-contract.test.ts`; `tests/verify/run-not-finished.test.ts` checks that the grades are the same when `status.json` contradicts the bundle or is gone; `docs/decisions/0001-run-bundle-is-the-source-of-truth.md` |
|
|
130
130
|
| Unsupported execution is refused before side effects | `parseStudy` (`src/study/config.ts`), `taskProtocolValidationReason` (`src/study/validation.ts`), `planComputerUseStudy` (`src/routes/computer-use/plan.ts`) | `tests/admission/library.test.ts` checks each library refusal for a run directory, a desktop module load, a caller executor or provider, and a subprocess; `tests/admission/cli.test.ts` checks each CLI refusal for a run directory, a started process and a scorer's host code; `tests/study/task-route-preflight.test.ts`; `pnpm cli:preflight:test` runs `scripts/task-route-preflight-proof.mjs` with `tests/fixtures/task-route-preflight/deny-side-effects.mjs` preloaded |
|
|
131
131
|
| A sandbox id is recorded before any work runs in it | `acquireE2BDesktopSandbox` and `acquireE2BShellSandbox` (`src/substrates/e2b/sandbox.ts`) tag the sandbox with its run's owner tags, register the create with `beginSandboxCreate` (`src/run/sandbox-creates.ts`) and write the receipt before they return the handle, and every route that creates a sandbox calls them; `reclaimRunSandboxes` (`src/run/reclaim.ts`) switches on the receipt's provider, kills by exact id and by the run's exact tags | `tests/substrates/e2b/sandbox.test.ts`, which also checks that `Sandbox.create` has one call site in `src/`, `tests/routes/terminal/acquisition-boundary.test.ts`, `tests/run/reclaim.test.ts`; `tests/routes/scripted/route.test.ts` checks create, receipt, then first work |
|
|
132
|
-
| Bundles carry no secrets | `redactText` and `scrubLiterals` (`src/evidence/redaction.ts`); `scanRunPublicSafetyArtifacts` (`src/verify/artifacts.ts`); `buildShareSafety` (`src/verify/verify.ts`) grades `share_ready`, `local_only` or `blocked`
|
|
132
|
+
| Bundles carry no secrets | `redactText` and `scrubLiterals` (`src/evidence/redaction.ts`); `RunSecrets` (`src/run/secrets.ts`) holds the literal values each run scrubs; `scanRunPublicSafetyArtifacts` (`src/verify/artifacts.ts`); `buildShareSafety` (`src/verify/verify.ts`) grades `share_ready`, `local_only` or `blocked` | `tests/evidence/redaction-hooks.test.ts`, `tests/routes/known-secrets.test.ts`, `tests/run/transient-comms-secrets.test.ts`, `tests/verify/evidence-refs.test.ts` |
|
|
133
133
|
| Goldens pin route output | `runDirSnapshot` (`tests/helpers/run-golden.ts`) snapshots a whole run folder | `tests/golden/routes/`, `tests/golden/observer-data/`, `tests/golden/labs/`; `tests/study/plan-lab.test.ts` checks that every `StudyRoute` has a dry-run golden |
|
|
134
134
|
| A run is closed on every exit, only a published run is analyzed, and `latest.json` moves only after its run is published | `runScope` and `FinishedRun` (`src/run/run.ts`); `completeAutomaticAnalysis` requires an issued `FinishedRun` for the result's run; `Run.finish` writes `latest.json` last through `writePreparedRunLatestPointer` (`src/run/contained-output.ts`) | `tests/run/run-lifecycle.test.ts`, which also checks that `src/run/run.ts` is the pointer's one writer, `tests/analysis/automatic-analysis.test.ts` |
|
|
135
135
|
|
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,23 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.111.3: Shared-world feedback candidates, scripted E2B key scrubbed, subject sandbox warnings (2026-10-06)
|
|
12
|
+
|
|
13
|
+
humanish 0.111.3 gives shared-world runs feedback candidates and repairs seven places where one
|
|
14
|
+
route's warnings, refusals, secret scrubbing or run directory checks differed from the other
|
|
15
|
+
routes'. A shared-world participant who reports friction or gives up becomes a candidate that
|
|
16
|
+
`humanish feedback` drafts, and each candidate from a run with several participants shows that
|
|
17
|
+
participant's own instructions as `expected`. A scripted run scrubs the value of `E2B_API_KEY` from
|
|
18
|
+
the errors and subject setup output it records, and writes `[REDACTED_SECRET]` where it wrote
|
|
19
|
+
`[redacted]`. A shared-world study with `caps.maxUsd` and no `caps.maxTotalUsd` warns that the cap
|
|
20
|
+
applies to each participant. The subject sandbox of a scripted clone study or a provisioned
|
|
21
|
+
shared-world study warns when its create was retried or E2B reports no size. A scripted clone study
|
|
22
|
+
missing a `subject.env` value, and a provisioned shared-world study missing `OPENAI_API_KEY`,
|
|
23
|
+
refuse with the wording computer use gives. Computer-use and shared-world runs check the run
|
|
24
|
+
directory again after a `prepareDesktop` hook and an Observer render.
|
|
25
|
+
|
|
26
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.111.3)
|
|
27
|
+
|
|
11
28
|
## 0.111.2: A taller Observer frame, even grid cards, a notice when the server stops (2026-10-06)
|
|
12
29
|
|
|
13
30
|
humanish 0.111.2 changes the Observer and nothing else. The focus view's heading line holds the
|
|
@@ -12,7 +12,7 @@ import { exposureRequestFromOptions, followDecision, followObserver, planObserve
|
|
|
12
12
|
import { writeRunFindings } from "../findings.js";
|
|
13
13
|
import { formatCuaStudyHuman } from "./study-format.js";
|
|
14
14
|
import { resolveRouteShouldOpen } from "./study-route-open.js";
|
|
15
|
-
import {
|
|
15
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
16
16
|
import { declaredParticipantCount } from "../../study/study-fields.js";
|
|
17
17
|
/**
|
|
18
18
|
* The computer-use route's CLI setup: its settings, the watch and exposure plan, the runStudyWith options
|
|
@@ -51,17 +51,12 @@ export function computerUseRouteRun(args) {
|
|
|
51
51
|
};
|
|
52
52
|
}
|
|
53
53
|
function refuseCua(args, dryRun, code, message) {
|
|
54
|
-
const result = {
|
|
55
|
-
...studyResultIdentity("computer-use", args.config.id),
|
|
56
|
-
ok: false,
|
|
57
|
-
cwd: args.options.cwd,
|
|
54
|
+
const result = refusedResult("computer-use", { studyId: args.config.id, cwd: args.options.cwd, error: { code, message } }, {
|
|
58
55
|
actor: args.config.actor?.type ?? "",
|
|
59
56
|
appUrl: "",
|
|
60
57
|
dryRun,
|
|
61
58
|
runId: args.options.runId ?? "not-created",
|
|
62
|
-
|
|
63
|
-
error: { code, message },
|
|
64
|
-
};
|
|
59
|
+
});
|
|
65
60
|
writeResult(args.command, args.io, result, formatCuaStudyHuman);
|
|
66
61
|
args.io.setExitCode(2);
|
|
67
62
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import {
|
|
2
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
3
3
|
import { formatRunHuman, parseStudyCount, writeResult, } from "../io.js";
|
|
4
4
|
import { planObserver, showObserver, staticObserverOpen } from "../observer-follow.js";
|
|
5
5
|
import { declaredParticipantCount } from "../../study/study-fields.js";
|
|
@@ -10,16 +10,14 @@ import { declaredParticipantCount } from "../../study/study-fields.js";
|
|
|
10
10
|
export function previewRouteRun(args) {
|
|
11
11
|
const participantCount = parseStudyCount(args.options.count, declaredParticipantCount(args.config) ?? 4);
|
|
12
12
|
if (participantCount === null) {
|
|
13
|
-
const result = {
|
|
14
|
-
|
|
15
|
-
ok: false,
|
|
13
|
+
const result = refusedResult("preview", {
|
|
14
|
+
studyId: args.config.id,
|
|
16
15
|
cwd: args.options.cwd,
|
|
17
|
-
warnings: [],
|
|
18
16
|
error: {
|
|
19
17
|
code: "HUMANISH_INVALID_PARTICIPANT_COUNT",
|
|
20
18
|
message: "--count must be a positive integer.",
|
|
21
19
|
},
|
|
22
|
-
};
|
|
20
|
+
}, {});
|
|
23
21
|
writeResult(args.command, args.io, result, formatRunHuman);
|
|
24
22
|
args.io.setExitCode(2);
|
|
25
23
|
return undefined;
|
|
@@ -9,7 +9,7 @@ import { followDecision, followObserver, showObserver, staticObserverOpen, withO
|
|
|
9
9
|
import { writeRunFindings } from "../findings.js";
|
|
10
10
|
import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
|
|
11
11
|
import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
12
|
-
import {
|
|
12
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
13
13
|
import { participantList } from "../../study/study-fields.js";
|
|
14
14
|
/**
|
|
15
15
|
* The shared-world route's CLI setup: dry run, open and follow semantics, the live Observer server
|
|
@@ -30,10 +30,11 @@ export function sharedWorldRouteRun(args) {
|
|
|
30
30
|
followDecision({ wantsMachine, detach: args.options.detach, serve: args.options.serve }).follow;
|
|
31
31
|
const port = parseObserverPort(args.options.port ?? "0");
|
|
32
32
|
const failConcurrent = (message, runId) => {
|
|
33
|
-
const result = {
|
|
34
|
-
|
|
35
|
-
ok: false,
|
|
33
|
+
const result = refusedResult("shared-world", {
|
|
34
|
+
studyId: args.config.id,
|
|
36
35
|
cwd: args.options.cwd,
|
|
36
|
+
error: { code: "HUMANISH_SHARED_WORLD_FAILED", message },
|
|
37
|
+
}, {
|
|
37
38
|
actor: args.config.actor?.type ?? "",
|
|
38
39
|
topology: "shared-world",
|
|
39
40
|
topologyMode: "concurrent",
|
|
@@ -42,9 +43,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
42
43
|
dryRun,
|
|
43
44
|
runId: runId ?? args.options.runId ?? "not-created",
|
|
44
45
|
roles: [],
|
|
45
|
-
|
|
46
|
-
error: { code: "HUMANISH_SHARED_WORLD_FAILED", message },
|
|
47
|
-
};
|
|
46
|
+
});
|
|
48
47
|
writeResult(args.command, args.io, result, formatConcurrentSharedWorldStudyHuman);
|
|
49
48
|
args.io.setExitCode(2);
|
|
50
49
|
};
|
|
@@ -112,11 +112,11 @@ export interface RedactionHooks {
|
|
|
112
112
|
/** The default hooks wired into every actor: redaction.ts is the source of truth. */
|
|
113
113
|
export declare const defaultRedactionHooks: RedactionHooks;
|
|
114
114
|
/**
|
|
115
|
-
* Replace each literal value with
|
|
115
|
+
* Replace each literal value with `marker`. For values that have no secret shape to
|
|
116
116
|
* pattern-match, such as provisioned keys and subject env values. `values` is read on each call,
|
|
117
117
|
* so a value registered later is scrubbed too.
|
|
118
118
|
*/
|
|
119
|
-
export declare function scrubLiterals(values: readonly string[]): (text: string) => string;
|
|
119
|
+
export declare function scrubLiterals(values: readonly string[], marker?: string): (text: string) => string;
|
|
120
120
|
/**
|
|
121
121
|
* Coerce an unknown thrown value to its message string. Prefer this over the
|
|
122
122
|
* inline `error instanceof Error ? error.message : String(error)` so error
|
|
@@ -522,12 +522,12 @@ function blurPass(src, width, height, radius, horizontal) {
|
|
|
522
522
|
return out;
|
|
523
523
|
}
|
|
524
524
|
/**
|
|
525
|
-
* Replace each literal value with
|
|
525
|
+
* Replace each literal value with `marker`. For values that have no secret shape to
|
|
526
526
|
* pattern-match, such as provisioned keys and subject env values. `values` is read on each call,
|
|
527
527
|
* so a value registered later is scrubbed too.
|
|
528
528
|
*/
|
|
529
|
-
export function scrubLiterals(values) {
|
|
530
|
-
return (text) => values.reduce((current, value) => current.split(value).join(
|
|
529
|
+
export function scrubLiterals(values, marker = "[REDACTED_SECRET]") {
|
|
530
|
+
return (text) => values.reduce((current, value) => current.split(value).join(marker), text);
|
|
531
531
|
}
|
|
532
532
|
/**
|
|
533
533
|
* Coerce an unknown thrown value to its message string. Prefer this over the
|
|
@@ -1,39 +1,8 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import { type RunFeedbackCandidate, type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
|
|
1
|
+
import { type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
|
|
3
2
|
import { type ParticipantIds } from "../../run/participant-records.js";
|
|
4
3
|
import { type CuaSubjectProvenanceArg, type ParticipantRunOutcome } from "./types.js";
|
|
5
4
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
6
5
|
export declare function describeSubjectState(state: RunSubjectProvenance["state"], dryRun: boolean): string;
|
|
7
|
-
/**
|
|
8
|
-
* Feedback candidates derived from what live participants actually reported.
|
|
9
|
-
*
|
|
10
|
-
* A live run's feedback draft comes from these candidates, never from a dry-run template. The
|
|
11
|
-
* candidate worth filing is the one the study produced: a participant
|
|
12
|
-
* who reported friction on the way (the most valuable thing a run captures), or one who stopped
|
|
13
|
-
* trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
|
|
14
|
-
* any falls back to a live summary in the draft layer instead of a template.
|
|
15
|
-
*
|
|
16
|
-
* Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
|
|
17
|
-
* through redactNarration in the loop) and passes redactText again here as defense-in-depth.
|
|
18
|
-
*/
|
|
19
|
-
export declare function participantFeedbackCandidates(args: {
|
|
20
|
-
runId: string;
|
|
21
|
-
scenarioId: string;
|
|
22
|
-
adapterId: string;
|
|
23
|
-
/** The already-redacted study goal (what bundle.scenario.goal carries). */
|
|
24
|
-
goal: string;
|
|
25
|
-
substrate: RunFeedbackCandidate["substrate"];
|
|
26
|
-
participants: Array<{
|
|
27
|
-
/** The participant's plan id; it names the candidate and its idempotency key. */
|
|
28
|
-
participantId: string;
|
|
29
|
-
streamId: string;
|
|
30
|
-
personaId: string;
|
|
31
|
-
session?: CuaLoopResult;
|
|
32
|
-
traceArtifactPath?: string;
|
|
33
|
-
screenshots: string[];
|
|
34
|
-
commsArtifactPath?: string;
|
|
35
|
-
}>;
|
|
36
|
-
}): RunFeedbackCandidate[];
|
|
37
6
|
/** Human-readable provenance line for the single-participant subject.provenance event:
|
|
38
7
|
* claims "cloned/packed and served" only when it actually happened. */
|
|
39
8
|
export declare function subjectProvenanceMessage(provenance: CuaSubjectProvenanceArg, publicAppUrl: string, dryRun: boolean, hasSession: boolean): string;
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// The parts both computer-use bundle shapes share: the subject-state story and provenance line,
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// single-bundle.ts and fanout-bundle.ts.
|
|
5
|
-
|
|
2
|
+
// provider-resource records, and the public-safe app URL label, the URL digest and the
|
|
3
|
+
// subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
|
|
4
|
+
// single-bundle.ts and fanout-bundle.ts. Feedback candidates from participant reports are in
|
|
5
|
+
// participant-feedback.ts, which the shared-world bundle reads too.
|
|
6
6
|
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
7
|
-
import { digestText
|
|
7
|
+
import { digestText } from "../../evidence/redaction.js";
|
|
8
8
|
import {} from "../../run/bundle.js";
|
|
9
9
|
import { participantResourceIds } from "../../run/participant-records.js";
|
|
10
|
-
import { resolveSelfReportedFriction } from "./self-report.js";
|
|
11
10
|
import {} from "./types.js";
|
|
12
11
|
import { plural } from "../../run/text.js";
|
|
13
12
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
@@ -25,92 +24,6 @@ export function describeSubjectState(state, dryRun) {
|
|
|
25
24
|
return "external-public (operator-declared, operator-owned public deployment; neither provisioned nor seeded)";
|
|
26
25
|
}
|
|
27
26
|
}
|
|
28
|
-
/**
|
|
29
|
-
* Feedback candidates derived from what live participants actually reported.
|
|
30
|
-
*
|
|
31
|
-
* A live run's feedback draft comes from these candidates, never from a dry-run template. The
|
|
32
|
-
* candidate worth filing is the one the study produced: a participant
|
|
33
|
-
* who reported friction on the way (the most valuable thing a run captures), or one who stopped
|
|
34
|
-
* trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
|
|
35
|
-
* any falls back to a live summary in the draft layer instead of a template.
|
|
36
|
-
*
|
|
37
|
-
* Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
|
|
38
|
-
* through redactNarration in the loop) and passes redactText again here as defense-in-depth.
|
|
39
|
-
*/
|
|
40
|
-
export function participantFeedbackCandidates(args) {
|
|
41
|
-
const candidates = [];
|
|
42
|
-
for (const participant of args.participants) {
|
|
43
|
-
const session = participant.session;
|
|
44
|
-
if (session === undefined)
|
|
45
|
-
continue;
|
|
46
|
-
const friction = resolveSelfReportedFriction(session);
|
|
47
|
-
const abandoned = session.status === "abandoned";
|
|
48
|
-
if (friction === undefined && !abandoned)
|
|
49
|
-
continue;
|
|
50
|
-
const summary = friction !== undefined
|
|
51
|
-
? `Participant ${participant.personaId} (${participant.participantId}) reported friction on the way through the study goal`
|
|
52
|
-
: `Participant ${participant.personaId} (${participant.participantId}) stopped before completing the study goal`;
|
|
53
|
-
const lastScreenshot = participant.screenshots[participant.screenshots.length - 1];
|
|
54
|
-
candidates.push({
|
|
55
|
-
schema: "humanish.feedback-candidate.v1",
|
|
56
|
-
id: `participant-report-${participant.participantId}`,
|
|
57
|
-
run_id: args.runId,
|
|
58
|
-
stream_id: participant.streamId,
|
|
59
|
-
adapter_id: args.adapterId,
|
|
60
|
-
scenario_id: args.scenarioId,
|
|
61
|
-
persona_id: participant.personaId,
|
|
62
|
-
actor: "computer-use",
|
|
63
|
-
substrate: args.substrate,
|
|
64
|
-
// The participant is reporting on the product: friction and abandonment are target-app
|
|
65
|
-
// findings by the three-roles rule. A harness failure never reaches this builder; it is
|
|
66
|
-
// not a participant report.
|
|
67
|
-
failure_owner: "target-app",
|
|
68
|
-
summary,
|
|
69
|
-
expected: args.goal,
|
|
70
|
-
actual: redactText(friction ?? session.reason),
|
|
71
|
-
evidence: [
|
|
72
|
-
...(participant.traceArtifactPath === undefined
|
|
73
|
-
? []
|
|
74
|
-
: [
|
|
75
|
-
{
|
|
76
|
-
path: participant.traceArtifactPath,
|
|
77
|
-
kind: "trace",
|
|
78
|
-
note: "Full actor trace: turns, actions, and the participant's own report.",
|
|
79
|
-
},
|
|
80
|
-
]),
|
|
81
|
-
...(lastScreenshot === undefined
|
|
82
|
-
? []
|
|
83
|
-
: [
|
|
84
|
-
{
|
|
85
|
-
path: lastScreenshot,
|
|
86
|
-
kind: "screenshot",
|
|
87
|
-
note: "Final screenshot at the moment the session ended.",
|
|
88
|
-
},
|
|
89
|
-
]),
|
|
90
|
-
...(participant.commsArtifactPath === undefined
|
|
91
|
-
? []
|
|
92
|
-
: [
|
|
93
|
-
{
|
|
94
|
-
path: participant.commsArtifactPath,
|
|
95
|
-
kind: "log",
|
|
96
|
-
note: "Digest-only comms thread captured in-sandbox.",
|
|
97
|
-
},
|
|
98
|
-
]),
|
|
99
|
-
],
|
|
100
|
-
redaction: {
|
|
101
|
-
status: "passed",
|
|
102
|
-
notes: "Quoted participant text passed the loop's known-value scrub and pattern redaction before persisting, and redactText again here.",
|
|
103
|
-
},
|
|
104
|
-
idempotency_key: `humanish:${args.runId}:${participant.participantId}:participant-report`,
|
|
105
|
-
proposed_next_state: "study-quality-review",
|
|
106
|
-
acceptance_proof: [
|
|
107
|
-
feedbackProofCommands(args.runId).verify,
|
|
108
|
-
feedbackProofCommands(args.runId).observe,
|
|
109
|
-
],
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
return candidates;
|
|
113
|
-
}
|
|
114
27
|
/** Human-readable provenance line for the single-participant subject.provenance event:
|
|
115
28
|
* claims "cloned/packed and served" only when it actually happened. */
|
|
116
29
|
export function subjectProvenanceMessage(provenance, publicAppUrl, dryRun, hasSession) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// The first half of an E2B desktop participant's preparation: acquire the sandbox, verify its screen, and
|
|
2
2
|
// provision the subject. Each step fills the participant's state; a step that cannot proceed throws.
|
|
3
3
|
import { redactText } from "../../../evidence/redaction.js";
|
|
4
|
+
import { validatePreparedRunArtifactPaths } from "../../../run/paths.js";
|
|
4
5
|
import { provisionCloneSubject } from "../../../subject/clone.js";
|
|
5
6
|
import { provisionDesktopCli } from "../../../subject/desktop-cli.js";
|
|
6
7
|
import { provisionLocalTreeSubject } from "../../../subject/local-tree.js";
|
|
@@ -94,10 +95,16 @@ export async function acquireParticipantDesktop(ctx, state) {
|
|
|
94
95
|
if ("reason" in desktopResources) {
|
|
95
96
|
warnings.push(`Desktop resource size unavailable (${desktopResources.reason}); compute cost remains unpriced.`);
|
|
96
97
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
if (deps.prepareDesktop) {
|
|
99
|
+
await deps.prepareDesktop(desktop, {
|
|
100
|
+
kind: "participant",
|
|
101
|
+
participant: { id: spec.planned.id, index: spec.planned.index, count: deps.participantCount },
|
|
102
|
+
});
|
|
103
|
+
// The caller's hook runs with this process's file access, so the run directory is checked
|
|
104
|
+
// again before more evidence goes into it.
|
|
105
|
+
if ("physicalRunRoot" in deps.artifactRoot)
|
|
106
|
+
await validatePreparedRunArtifactPaths(deps.artifactRoot);
|
|
107
|
+
}
|
|
101
108
|
if (deps.receiving) {
|
|
102
109
|
state.receivingInboxUrl = await attachReceivingInbox(shell, spec, { ...deps, receiving: deps.receiving }, ctx.targetUrl);
|
|
103
110
|
state.commsArtifactPath = "comms/receiving.json";
|
|
@@ -4,7 +4,8 @@ import { actorEnding } from "../../actors/stop-cause.js";
|
|
|
4
4
|
import { redactText } from "../../evidence/redaction.js";
|
|
5
5
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
6
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
|
-
import {
|
|
7
|
+
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
8
9
|
import { participantFactsOf } from "./participant-facts.js";
|
|
9
10
|
import { participantPassed, participantStatus } from "../../run/judge.js";
|
|
10
11
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
@@ -141,7 +142,6 @@ function fanoutFeedbackCandidates(args) {
|
|
|
141
142
|
runId: args.run.runId,
|
|
142
143
|
scenarioId: `cua-${plan.studyId}`,
|
|
143
144
|
adapterId: plan.studyId,
|
|
144
|
-
goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
145
145
|
substrate: plan.residual.execution?.target === "local" ? "local-desktop" : "e2b-desktop",
|
|
146
146
|
participants: specs.map((spec, index) => {
|
|
147
147
|
const outcome = outcomes?.[index];
|
|
@@ -149,6 +149,7 @@ function fanoutFeedbackCandidates(args) {
|
|
|
149
149
|
participantId: spec.planned.id,
|
|
150
150
|
streamId: spec.streamId,
|
|
151
151
|
personaId: spec.persona.id,
|
|
152
|
+
goal: redactText(spec.evidenceInstructions ?? spec.instructions),
|
|
152
153
|
...(outcome?.session === undefined ? {} : { session: outcome.session }),
|
|
153
154
|
...(outcome?.session === undefined
|
|
154
155
|
? {}
|
|
@@ -15,7 +15,7 @@ import { participantSubjectEnv } from "./types.js";
|
|
|
15
15
|
export async function runStudyParticipants(setup, participants) {
|
|
16
16
|
const { plan, input, cwd, streams, descriptor, run } = setup;
|
|
17
17
|
const { participantRuns, participantPlan, bundleBase } = setup;
|
|
18
|
-
const { env,
|
|
18
|
+
const { env, deps, liveTrace, externalComms, inProgress, fail } = participants;
|
|
19
19
|
const { dryRun } = plan;
|
|
20
20
|
const inProcess = plan.runner.desktop === "in-process";
|
|
21
21
|
const subjectEnvNames = [...participantSubjectEnv(plan.runner.subject)];
|
|
@@ -76,11 +76,7 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
76
76
|
env,
|
|
77
77
|
participants: participantRuns.map((spec) => spec.planned.id),
|
|
78
78
|
runPaths,
|
|
79
|
-
registerSecrets: (values) =>
|
|
80
|
-
for (const value of values)
|
|
81
|
-
if (value.length >= 4 && !knownSecretValues.includes(value))
|
|
82
|
-
knownSecretValues.push(value);
|
|
83
|
-
},
|
|
79
|
+
registerSecrets: (values) => run.secrets.add(values),
|
|
84
80
|
});
|
|
85
81
|
if (receiving)
|
|
86
82
|
deps.receiving = receiving;
|
|
@@ -129,7 +125,7 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
129
125
|
runPaths,
|
|
130
126
|
participantRuns: participantRuns,
|
|
131
127
|
outcomes,
|
|
132
|
-
knownSecretValues,
|
|
128
|
+
knownSecretValues: run.secrets.values(),
|
|
133
129
|
})
|
|
134
130
|
: [];
|
|
135
131
|
return {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { ComputerUsePlan, SharedWorldPlan } from "../../study/plan-types.js";
|
|
2
|
+
import type { StudyDeps } from "../../study/study-deps.js";
|
|
3
|
+
import type { LiveTraceFlush } from "./live-flush.js";
|
|
4
|
+
import type { CuaParticipantDeps } from "./types.js";
|
|
5
|
+
/** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
|
|
6
|
+
export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalProvisioned" | "appUrl" | "onObservedUrl">;
|
|
7
|
+
/** What every participant of a desktop run reads. Each route supplies all of it. */
|
|
8
|
+
interface DesktopRunFacts {
|
|
9
|
+
plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
|
|
10
|
+
/** The plan's brain. A local agent runs each participant on the operator's signed-in CLI. */
|
|
11
|
+
brain: CuaParticipantDeps["brain"];
|
|
12
|
+
/** What the participant's desktop does with the subject. */
|
|
13
|
+
subject: CuaParticipantDeps["subject"];
|
|
14
|
+
/** The run's env. The E2B request timeout is read from it. */
|
|
15
|
+
env: CuaParticipantDeps["env"];
|
|
16
|
+
openaiApiKey: string;
|
|
17
|
+
e2bApiKey: string;
|
|
18
|
+
/** Each participant's session budget. */
|
|
19
|
+
timeoutMs: number;
|
|
20
|
+
/** Each participant's sandbox lifetime. */
|
|
21
|
+
sandboxMs: number;
|
|
22
|
+
participantCount: number;
|
|
23
|
+
artifactRoot: CuaParticipantDeps["artifactRoot"];
|
|
24
|
+
studyCwd: string;
|
|
25
|
+
redactScreenshots: boolean;
|
|
26
|
+
scrubKnownValues: (text: string) => string;
|
|
27
|
+
runSession: CuaParticipantDeps["runSession"];
|
|
28
|
+
/** The run's live-trace flush, once a live run has started it. */
|
|
29
|
+
liveFlush: () => LiveTraceFlush["flush"] | undefined;
|
|
30
|
+
onStream: CuaParticipantDeps["onStream"];
|
|
31
|
+
reportSubjectPhase: CuaParticipantDeps["reportSubjectPhase"];
|
|
32
|
+
/** The run's test seams: the clock, the E2B SDK loader and the detached timers. */
|
|
33
|
+
seams: StudyDeps;
|
|
34
|
+
/** The caller's prepareDesktop, which each participant's desktop runs. */
|
|
35
|
+
prepareDesktop: CuaParticipantDeps["prepareDesktop"];
|
|
36
|
+
}
|
|
37
|
+
/** The fields one desktop route sets and the other does not. Undefined leaves a field out. */
|
|
38
|
+
type DesktopRouteOverrides = {
|
|
39
|
+
[K in "createDesktop" | "createProvider" | "inProcessExecutor" | "localTreeArchiveBuffer" | "externalComms" | "receiving" | "screenMismatchPolicy"]?: CuaParticipantDeps[K] | undefined;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* One run's participant deps. Call it once per run: the run budget it creates is the one ledger
|
|
43
|
+
* every participant notes its spend on, and a dry run, which never spends, gets none.
|
|
44
|
+
*/
|
|
45
|
+
export declare function desktopParticipantDeps(run: DesktopRunFacts, overrides: DesktopRouteOverrides): DesktopParticipantDeps;
|
|
46
|
+
export {};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// The deps every desktop participant reads, built here for computer use and shared world. A field
|
|
2
|
+
// both routes hand their participants is added once, in desktopParticipantDeps; each route passes
|
|
3
|
+
// what only it knows as overrides.
|
|
4
|
+
import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
|
|
5
|
+
import { makeCuaRunBudget } from "./participant-model.js";
|
|
6
|
+
/**
|
|
7
|
+
* One run's participant deps. Call it once per run: the run budget it creates is the one ledger
|
|
8
|
+
* every participant notes its spend on, and a dry run, which never spends, gets none.
|
|
9
|
+
*/
|
|
10
|
+
export function desktopParticipantDeps(run, overrides) {
|
|
11
|
+
const { plan, seams } = run;
|
|
12
|
+
const { maxTotalUsd } = plan.caps;
|
|
13
|
+
const optional = {
|
|
14
|
+
...overrides,
|
|
15
|
+
prepareDesktop: run.prepareDesktop,
|
|
16
|
+
desktopModule: seams.desktopModule,
|
|
17
|
+
detachedTimers: seams.detachedTimers,
|
|
18
|
+
runBudget: plan.dryRun || maxTotalUsd === undefined ? undefined : makeCuaRunBudget(maxTotalUsd),
|
|
19
|
+
};
|
|
20
|
+
return {
|
|
21
|
+
...definedFields(optional),
|
|
22
|
+
onTrace: (participantId, items, usage, metadata) => run.liveFlush()?.(participantId, items, usage, metadata),
|
|
23
|
+
residual: plan.residual,
|
|
24
|
+
studyId: plan.studyId,
|
|
25
|
+
caps: plan.caps,
|
|
26
|
+
brain: run.brain,
|
|
27
|
+
subject: run.subject,
|
|
28
|
+
env: run.env,
|
|
29
|
+
openaiApiKey: run.openaiApiKey,
|
|
30
|
+
e2bApiKey: run.e2bApiKey,
|
|
31
|
+
requestTimeoutMs: e2bRequestTimeoutMs(run.env),
|
|
32
|
+
sandboxMs: run.sandboxMs,
|
|
33
|
+
timeoutMs: run.timeoutMs,
|
|
34
|
+
participantCount: run.participantCount,
|
|
35
|
+
artifactRoot: run.artifactRoot,
|
|
36
|
+
studyCwd: run.studyCwd,
|
|
37
|
+
redactScreenshots: run.redactScreenshots,
|
|
38
|
+
scrubKnownValues: run.scrubKnownValues,
|
|
39
|
+
runSession: run.runSession,
|
|
40
|
+
now: seams.now ?? Date.now,
|
|
41
|
+
onStream: run.onStream,
|
|
42
|
+
reportSubjectPhase: run.reportSubjectPhase,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** The fields of `fields` that are set. The deps never carry a key whose value is undefined. */
|
|
46
|
+
function definedFields(fields) {
|
|
47
|
+
return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined));
|
|
48
|
+
}
|
|
@@ -5,8 +5,9 @@ export declare function participantFactsOf(outcome: ParticipantRunOutcome): Part
|
|
|
5
5
|
/** Whether a participant passed. A dry-run participant passes as a contract. */
|
|
6
6
|
export declare function participantOutcomeOk(outcome: ParticipantRunOutcome | undefined, dryRun: boolean): boolean;
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* The execution failures a run's participants caused, by kind: each session that failed in the
|
|
9
|
+
* harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
|
|
10
|
+
* item after its last request, then each desktop not confirmed released. Participants keep their
|
|
11
|
+
* order within each kind.
|
|
11
12
|
*/
|
|
12
|
-
export declare function
|
|
13
|
+
export declare function participantExecutionFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
|
|
2
2
|
// them. The bundle builders, the result, the participant runner and shared-world all read these,
|
|
3
3
|
// so they sit in a file that imports only the judge and the types.
|
|
4
|
-
import { participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
4
|
+
import { participantHarnessFailed, participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
5
5
|
/** A participant outcome's facts for the judge. */
|
|
6
6
|
export function participantFactsOf(outcome) {
|
|
7
7
|
return {
|
|
@@ -20,12 +20,47 @@ export function participantOutcomeOk(outcome, dryRun) {
|
|
|
20
20
|
return true;
|
|
21
21
|
return outcome !== undefined && participantPassed(participantFactsOf(outcome));
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* The execution failures a run's participants caused, by kind: each session that failed in the
|
|
25
|
+
* harness, each provider whose cleanup is unconfirmed, each provider that reported a disallowed
|
|
26
|
+
* item after its last request, then each desktop not confirmed released. Participants keep their
|
|
27
|
+
* order within each kind.
|
|
28
|
+
*/
|
|
29
|
+
export function participantExecutionFailures(outcomes, runId) {
|
|
30
|
+
const all = outcomes ?? [];
|
|
31
|
+
const named = (outcome, text) => `${outcome.spec.planned.id}: ${text}`;
|
|
32
|
+
return [
|
|
33
|
+
...all
|
|
34
|
+
.filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
|
|
35
|
+
.map((outcome) => ({
|
|
36
|
+
kind: "harness",
|
|
37
|
+
message: named(outcome, outcome.sessionError ?? outcome.session?.reason ?? "harness error"),
|
|
38
|
+
})),
|
|
39
|
+
...all.flatMap((outcome) => outcome.providerCleanupError === undefined
|
|
40
|
+
? []
|
|
41
|
+
: [
|
|
42
|
+
{
|
|
43
|
+
kind: "provider-cleanup",
|
|
44
|
+
message: named(outcome, outcome.providerCleanupError),
|
|
45
|
+
},
|
|
46
|
+
]),
|
|
47
|
+
...all.flatMap((outcome) => outcome.providerPolicyError === undefined
|
|
48
|
+
? []
|
|
49
|
+
: [
|
|
50
|
+
{
|
|
51
|
+
kind: "provider-policy",
|
|
52
|
+
message: named(outcome, outcome.providerPolicyError),
|
|
53
|
+
},
|
|
54
|
+
]),
|
|
55
|
+
...unreleasedSandboxFailures(all, runId),
|
|
56
|
+
];
|
|
57
|
+
}
|
|
23
58
|
/**
|
|
24
59
|
* One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
|
|
25
60
|
* was not killed, or a desktop whose release says why (a local VM names its container). A
|
|
26
61
|
* participant that never acquired a desktop (skipped, in-process) adds none.
|
|
27
62
|
*/
|
|
28
|
-
|
|
63
|
+
function unreleasedSandboxFailures(outcomes, runId) {
|
|
29
64
|
return (outcomes ?? [])
|
|
30
65
|
.filter((outcome) => outcome.sandboxRelease !== undefined ||
|
|
31
66
|
(outcome.sandboxId !== undefined && !outcome.killed))
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
|
|
2
|
+
import type { RunFeedbackCandidate } from "../../run/bundle.js";
|
|
3
|
+
/**
|
|
4
|
+
* Feedback candidates derived from what live participants actually reported.
|
|
5
|
+
*
|
|
6
|
+
* A live run's feedback draft comes from these candidates, never from a dry-run template. The
|
|
7
|
+
* candidate worth filing is the one the study produced: a participant
|
|
8
|
+
* who reported friction on the way (the most valuable thing a run captures), or one who stopped
|
|
9
|
+
* trying. A clean pass files nothing here: feedback exists to carry findings, and a run without
|
|
10
|
+
* any falls back to a live summary in the draft layer instead of a template.
|
|
11
|
+
*
|
|
12
|
+
* Everything quoted is already scrub+redacted (participant messages and `session.reason` pass
|
|
13
|
+
* through redactNarration in the loop) and passes redactText again here as defense-in-depth.
|
|
14
|
+
*/
|
|
15
|
+
export declare function participantFeedbackCandidates(args: {
|
|
16
|
+
runId: string;
|
|
17
|
+
scenarioId: string;
|
|
18
|
+
adapterId: string;
|
|
19
|
+
substrate: RunFeedbackCandidate["substrate"];
|
|
20
|
+
participants: Array<{
|
|
21
|
+
/** The participant's plan id; it names the candidate and its idempotency key. */
|
|
22
|
+
participantId: string;
|
|
23
|
+
streamId: string;
|
|
24
|
+
personaId: string;
|
|
25
|
+
/** The participant's own already-redacted instructions, which the candidate's `expected`
|
|
26
|
+
* quotes. Participants of one study get different instructions, each naming its persona. */
|
|
27
|
+
goal: string;
|
|
28
|
+
session?: CuaLoopResult;
|
|
29
|
+
traceArtifactPath?: string;
|
|
30
|
+
screenshots: string[];
|
|
31
|
+
commsArtifactPath?: string;
|
|
32
|
+
}>;
|
|
33
|
+
}): RunFeedbackCandidate[];
|