humanish 0.111.4 → 0.113.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +11 -0
- package/TELEMETRY.md +21 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +21 -1
- package/dist/actors/codex/restricted-participant-policy.js +20 -2
- package/dist/actors/codex/restricted-participant.d.ts +1 -1
- package/dist/actors/codex/restricted-participant.js +2 -1
- package/dist/actors/computer-use/loop/debrief.d.ts +12 -4
- package/dist/actors/computer-use/loop/debrief.js +57 -20
- package/dist/actors/computer-use/loop/ending.d.ts +5 -2
- package/dist/actors/computer-use/loop/impressions.d.ts +14 -0
- package/dist/actors/computer-use/loop/impressions.js +40 -0
- package/dist/actors/computer-use/loop/session.d.ts +3 -1
- package/dist/actors/computer-use/loop/session.js +2 -0
- package/dist/actors/computer-use/loop/trace.js +2 -0
- package/dist/actors/computer-use/loop/types.d.ts +8 -1
- package/dist/actors/computer-use/loop.d.ts +2 -1
- package/dist/actors/computer-use/loop.js +15 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +96 -0
- package/dist/actors/computer-use/openai-conversation.js +238 -0
- package/dist/actors/computer-use/openai-provider.js +38 -48
- package/dist/actors/contract.d.ts +45 -3
- package/dist/actors/contract.js +13 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/evidence-sources.d.ts +3 -1
- package/dist/analysis/evidence-sources.js +10 -1
- package/dist/analysis/evidence.js +2 -2
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +37 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +37 -20
- package/dist/cli/commands/setup.js +95 -6
- package/dist/cli/commands/study-format.js +7 -4
- package/dist/cli/commands/study.js +1 -1
- package/dist/cli/commands/tui.js +1 -1
- package/dist/cli/doctor.d.ts +2 -0
- package/dist/cli/doctor.js +18 -0
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +4 -1
- package/dist/cli/io.js +3 -2
- package/dist/cli/program.js +35 -1
- package/dist/cli/secret-prompt.d.ts +5 -1
- package/dist/cli/secret-prompt.js +6 -2
- package/dist/cli/update-check-worker.d.ts +1 -0
- package/dist/cli/update-check-worker.js +5 -0
- package/dist/cli/update-check.d.ts +46 -0
- package/dist/cli/update-check.js +203 -0
- package/dist/feedback/draft.js +32 -4
- package/dist/keys/key-resolution.js +2 -0
- package/dist/keys/key-status.d.ts +37 -0
- package/dist/keys/key-status.js +68 -0
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/local-vm.js +9 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +6 -0
- package/dist/routes/computer-use/participant-execution.js +13 -12
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/rerun-selection.js +23 -21
- package/dist/routes/computer-use/result.js +10 -17
- package/dist/routes/computer-use/self-report.js +7 -1
- package/dist/routes/computer-use/setup.js +14 -0
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/computer-use/types.d.ts +4 -1
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +24 -30
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +71 -79
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +13 -4
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +31 -0
- package/dist/run/participant-caption.js +60 -0
- package/dist/run/participant-records.d.ts +4 -2
- package/dist/run/participant-records.js +6 -0
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/run-study.js +17 -1
- package/dist/study/discover.js +6 -1
- package/dist/study/doctor.d.ts +2 -0
- package/dist/study/doctor.js +45 -7
- package/dist/study/init-templates.js +10 -5
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/plan.js +3 -2
- package/dist/study/preflight-result.d.ts +1 -0
- package/dist/study/preflight-result.js +1 -1
- package/dist/study/preflight.d.ts +2 -0
- package/dist/study/preflight.js +54 -14
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +7 -1
- package/dist/study/warnings.js +82 -4
- package/dist/substrates/local/capacity.d.ts +75 -0
- package/dist/substrates/local/capacity.js +143 -0
- package/dist/substrates/local/firecracker-desktop.d.ts +8 -0
- package/dist/substrates/local/firecracker-desktop.js +34 -3
- package/dist/substrates/local/runtime-host.d.ts +16 -3
- package/dist/substrates/local/runtime-host.js +49 -4
- package/dist/substrates/local/runtime.d.ts +10 -0
- package/dist/substrates/local/runtime.js +44 -5
- package/dist/tui-app.js +75 -75
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/actor-contract.md +20 -6
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/feedback.md +8 -1
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +65 -34
- package/docs/contracts/study-analysis.md +86 -22
- package/package.json +2 -1
- package/skills/humanish/SKILL.md +21 -10
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -2,10 +2,12 @@ import { participantAssignment } from "../../study/participant-assignment.js";
|
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/streams.js";
|
|
5
|
-
import { aggregateTaskFunnels,
|
|
5
|
+
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, sessionEndingInWords, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
6
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
7
|
+
import { actorEnding } from "../../actors/stop-cause.js";
|
|
6
8
|
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
7
9
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
8
|
-
import {
|
|
10
|
+
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
9
11
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
10
12
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
11
13
|
import { plural } from "../../run/text.js";
|
|
@@ -43,18 +45,25 @@ function runCost(args) {
|
|
|
43
45
|
}
|
|
44
46
|
/** The participant's status, reason, last frame, geometry and screenshot mode, shared by its records. */
|
|
45
47
|
function participantView(args, publicAppUrl) {
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
args.
|
|
57
|
-
|
|
48
|
+
const judgment = judgeParticipantRecords([
|
|
49
|
+
{
|
|
50
|
+
...(args.session === undefined
|
|
51
|
+
? {}
|
|
52
|
+
: {
|
|
53
|
+
status: args.session.status,
|
|
54
|
+
completionReason: args.session.completionReason,
|
|
55
|
+
reason: args.session.reason,
|
|
56
|
+
}),
|
|
57
|
+
...(args.sessionError === undefined ? {} : { sessionError: args.sessionError }),
|
|
58
|
+
inProgress: args.inProgress === true,
|
|
59
|
+
skipped: false,
|
|
60
|
+
noEngagement: args.credibility?.noEngagement === true,
|
|
61
|
+
selfReportedBlocker: args.credibility?.selfReportedBlocker === true,
|
|
62
|
+
reportedFriction: args.credibility?.reportedFriction === true,
|
|
63
|
+
},
|
|
64
|
+
]);
|
|
65
|
+
const participant = judgment.participants[0];
|
|
66
|
+
const { status, reason } = participant;
|
|
58
67
|
const lastScreenshot = args.screenshots[args.screenshots.length - 1];
|
|
59
68
|
const desktopGeometry = args.desktopRoute === false
|
|
60
69
|
? undefined
|
|
@@ -70,6 +79,8 @@ function participantView(args, publicAppUrl) {
|
|
|
70
79
|
: (args.captureRedaction ?? "raw");
|
|
71
80
|
return {
|
|
72
81
|
publicAppUrl,
|
|
82
|
+
participant,
|
|
83
|
+
tally: judgment.tally,
|
|
73
84
|
status,
|
|
74
85
|
reason,
|
|
75
86
|
lastScreenshot,
|
|
@@ -80,6 +91,10 @@ function participantView(args, publicAppUrl) {
|
|
|
80
91
|
}
|
|
81
92
|
function singleSimulation(args, view) {
|
|
82
93
|
const { publicAppUrl, status, reason } = view;
|
|
94
|
+
const name = participantCaption({
|
|
95
|
+
id: args.participantId ?? "lane-01",
|
|
96
|
+
personaId: args.persona.id,
|
|
97
|
+
});
|
|
83
98
|
return participantRecord(SINGLE, 1, {
|
|
84
99
|
personaId: args.persona.id,
|
|
85
100
|
scenarioId: `cua-${args.studyId}`,
|
|
@@ -89,23 +104,19 @@ function singleSimulation(args, view) {
|
|
|
89
104
|
progress: args.inProgress === true ? 20 : 100,
|
|
90
105
|
currentStep: reason,
|
|
91
106
|
summary: args.session
|
|
92
|
-
?
|
|
107
|
+
? `${name} used the app ${browserPlace(args)}, and ${sessionEndingInWords(args.session.trace)}.`
|
|
93
108
|
: args.inProgress === true
|
|
94
|
-
?
|
|
109
|
+
? `${name} is using the app ${browserPlace(args)}.`
|
|
95
110
|
: args.sessionError !== undefined
|
|
96
|
-
?
|
|
97
|
-
:
|
|
111
|
+
? `${name} did not finish a session: ${args.sessionError}`
|
|
112
|
+
: `${name} would use ${publicAppUrl}; no session ran.`,
|
|
98
113
|
startedAt: args.run.createdAt,
|
|
99
114
|
updatedAt: args.run.createdAt,
|
|
100
115
|
});
|
|
101
116
|
}
|
|
102
117
|
function singleStream(args, view) {
|
|
103
118
|
const { publicAppUrl, status, reason, lastScreenshot, desktopGeometry, screenshotMode } = view;
|
|
104
|
-
|
|
105
|
-
// the participant the way the verdict judged it.
|
|
106
|
-
const judged = args.session === undefined || args.inProgress === true
|
|
107
|
-
? undefined
|
|
108
|
-
: participantStatusFor(args.session.status, args.credibility);
|
|
119
|
+
const judged = args.session === undefined ? undefined : view.participant.judgedStatus;
|
|
109
120
|
return participantStream(SINGLE, {
|
|
110
121
|
...(args.assignment === undefined
|
|
111
122
|
? {}
|
|
@@ -114,7 +125,13 @@ function singleStream(args, view) {
|
|
|
114
125
|
...(args.surface === undefined ? {} : { surface: args.surface }),
|
|
115
126
|
...(args.caseGroup === undefined ? {} : { caseGroup: args.caseGroup }),
|
|
116
127
|
kind: "browser",
|
|
117
|
-
label:
|
|
128
|
+
label: participantCaption({
|
|
129
|
+
id: args.participantId ?? "lane-01",
|
|
130
|
+
personaId: args.persona.id,
|
|
131
|
+
...(args.isMobile === undefined
|
|
132
|
+
? {}
|
|
133
|
+
: { device: { name: args.deviceName ?? "", preset: { isMobile: args.isMobile } } }),
|
|
134
|
+
}),
|
|
118
135
|
status,
|
|
119
136
|
...(judged === undefined ? {} : { judgedStatus: judged }),
|
|
120
137
|
transport: "snapshot",
|
|
@@ -257,37 +274,64 @@ function singleEvents(args, view) {
|
|
|
257
274
|
}
|
|
258
275
|
return events;
|
|
259
276
|
}
|
|
277
|
+
/**
|
|
278
|
+
* The run's summary in plain sentences. A finished session gives the outcome with its denominator
|
|
279
|
+
* of one, any reason the harness did not count a pass, and how the session ended. The single
|
|
280
|
+
* participant has no gap line, so the summary carries the reason.
|
|
281
|
+
*/
|
|
282
|
+
function singleSummary(args, view, facts) {
|
|
283
|
+
if (args.inProgress === true) {
|
|
284
|
+
return "1 participant is using the app. The run is still going, so nothing here is final.";
|
|
285
|
+
}
|
|
286
|
+
if (args.dryRun) {
|
|
287
|
+
return `Dry run: 1 participant would use ${view.publicAppUrl}. Nothing was launched and $0 was spent.`;
|
|
288
|
+
}
|
|
289
|
+
if (args.session === undefined) {
|
|
290
|
+
return args.sessionError === undefined
|
|
291
|
+
? `1 participant was planned against ${view.publicAppUrl}, but no session ran.`
|
|
292
|
+
: `1 participant was planned, but the run failed before their session finished: ${view.reason}`;
|
|
293
|
+
}
|
|
294
|
+
const ending = actorEnding(args.session.trace);
|
|
295
|
+
const outcomes = formatParticipantOutcomes(view.tally, [
|
|
296
|
+
{ status: args.session.status, ...(ending === undefined ? {} : { label: ending.label }) },
|
|
297
|
+
]);
|
|
298
|
+
return [
|
|
299
|
+
`1 participant took part: ${outcomes}.`,
|
|
300
|
+
facts.tasks === undefined ? undefined : `Tasks: ${formatRunTaskFunnel(facts.tasks)}.`,
|
|
301
|
+
facts.credibilityNote,
|
|
302
|
+
`How it ended: ${view.reason}`,
|
|
303
|
+
]
|
|
304
|
+
.filter((sentence) => sentence !== undefined)
|
|
305
|
+
.join(" ");
|
|
306
|
+
}
|
|
260
307
|
function singleReview(args, view, stream) {
|
|
261
|
-
const { reason } = view;
|
|
262
308
|
// A funnel with a denominator of one is still the funnel, and its absence stays recorded: no
|
|
263
309
|
// declared protocol (or a dry run) means no `tasks` field, never an empty one.
|
|
264
310
|
const singleRunTasks = args.inProgress !== true && args.session?.trace.taskFunnel !== undefined
|
|
265
311
|
? aggregateTaskFunnels([args.session.trace.taskFunnel])
|
|
266
312
|
: undefined;
|
|
267
|
-
|
|
268
|
-
// applies (participantStatus in src/run/judge.ts). Reading the actor's own status instead would write up
|
|
269
|
-
// a run the route refused as "not a credible pass" as verdict pass, 1/1 reached the goal, and
|
|
270
|
-
// every projection of the bundle (Observer tally, `runs`, the status index) would repeat it.
|
|
271
|
-
const participantStatus = args.session === undefined
|
|
272
|
-
? undefined
|
|
273
|
-
: participantStatusFor(args.session.status, args.credibility);
|
|
274
|
-
const credibilityNote = args.session === undefined || participantStatus === args.session.status
|
|
313
|
+
const credibilityNote = args.inProgress === true || args.session?.status !== "passed"
|
|
275
314
|
? undefined
|
|
276
315
|
: args.credibility?.noEngagement === true
|
|
277
316
|
? "Not counted as a pass: the participant took no actions and said nothing."
|
|
278
|
-
:
|
|
317
|
+
: args.credibility?.selfReportedBlocker === true
|
|
318
|
+
? "Not counted as a pass: the participant's final message described a blocker."
|
|
319
|
+
: undefined;
|
|
279
320
|
const review = withCuaReviewProvenance({
|
|
280
321
|
schema: REVIEW_SCHEMA,
|
|
281
322
|
verdict: args.verdict,
|
|
282
323
|
// One participant is still a study with a denominator of one, and saying so keeps a
|
|
283
324
|
// single-participant result from being read as though it generalized.
|
|
284
|
-
...(
|
|
325
|
+
...(args.session !== undefined && args.inProgress !== true
|
|
285
326
|
? {
|
|
286
|
-
participants:
|
|
327
|
+
participants: view.tally,
|
|
287
328
|
}
|
|
288
329
|
: {}),
|
|
289
330
|
...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
|
|
290
|
-
summary:
|
|
331
|
+
summary: singleSummary(args, view, {
|
|
332
|
+
...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
|
|
333
|
+
...(credibilityNote === undefined ? {} : { credibilityNote }),
|
|
334
|
+
}),
|
|
291
335
|
gaps: args.session || args.sessionError !== undefined
|
|
292
336
|
? []
|
|
293
337
|
: args.inProgress === true
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Verdict } from "../../run/judge.js";
|
|
2
|
+
import type { LocalCapacity } from "../../substrates/local/capacity.js";
|
|
2
3
|
import type { RunDesktopRecording } from "../../evidence/desktop-recording-types.js";
|
|
3
4
|
import type { CuaExecutor, CuaLiveMetadata, CuaLoopResult, CuaProvider } from "../../actors/computer-use/loop.js";
|
|
4
5
|
import type { CommsReceivingRun } from "../../comms/receiving.js";
|
|
@@ -38,6 +39,8 @@ export declare const MIN_DERIVED_SESSION_TIMEOUT_MS: number;
|
|
|
38
39
|
*/
|
|
39
40
|
export interface LocalVmInput {
|
|
40
41
|
readonly desktop: ParticipantDesktopFactory;
|
|
42
|
+
/** How many desktops the local runtime holds, or undefined when it cannot be read. */
|
|
43
|
+
readonly capacity?: () => Promise<LocalCapacity | undefined>;
|
|
41
44
|
/** The skip reason automatic analysis records instead of running, or undefined to run it. */
|
|
42
45
|
readonly analysisRefusal: () => AutomaticAnalysisRefusal | undefined;
|
|
43
46
|
readonly signal?: AbortSignal;
|
|
@@ -182,7 +185,7 @@ export interface CuaParticipantSummary {
|
|
|
182
185
|
concurrency: number;
|
|
183
186
|
waves: number;
|
|
184
187
|
}
|
|
185
|
-
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
188
|
+
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_LOCAL_CAPACITY_EXCEEDED" | "HUMANISH_COMPUTER_USE_DESKTOP_OUT_OF_MEMORY" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
186
189
|
/** Subject provenance projection: what the actor actually drove. */
|
|
187
190
|
export interface CuaSubjectProjection {
|
|
188
191
|
source: "app-url" | "clone" | "local-tree";
|
|
@@ -36,5 +36,5 @@ interface ScriptedBundleArgs {
|
|
|
36
36
|
* the bundle-builder tests.
|
|
37
37
|
*/
|
|
38
38
|
export declare function buildScriptedStudyBundle(args: ScriptedBundleArgs): RunBundle;
|
|
39
|
-
export declare function renderScriptedReviewMarkdown(bundle: RunBundle): string;
|
|
39
|
+
export declare function renderScriptedReviewMarkdown(bundle: RunBundle, status?: unknown): string;
|
|
40
40
|
export {};
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
// Assemble the scripted-browser study's run bundle from its sessions, and the review summary and
|
|
2
2
|
// markdown that ship with it.
|
|
3
|
-
import {
|
|
4
|
-
import { reviewOutcome } from "../../run/display.js";
|
|
3
|
+
import { renderReviewMarkdown } from "../../run/review-markdown.js";
|
|
5
4
|
import { redactText } from "../../evidence/redaction.js";
|
|
6
5
|
import { buildRunCostSummary, desktopSpanToMinutes, spendFreeCostSummary, } from "../../run/cost-summary.js";
|
|
7
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
8
7
|
import {} from "../../run/streams.js";
|
|
9
8
|
import { participantEvent, recordIdOf } from "../../run/participant-records.js";
|
|
10
9
|
import { scriptedSurfaceIds, scriptedSurfaceRecords } from "./surface-records.js";
|
|
10
|
+
import { gapsListClause } from "../../run/outcomes.js";
|
|
11
11
|
import { plural } from "../../run/text.js";
|
|
12
12
|
/**
|
|
13
13
|
* Project the scripted run into a humanish.run-bundle.v1 (no schema change: a new
|
|
@@ -146,7 +146,7 @@ function buildScriptedReview(args) {
|
|
|
146
146
|
return {
|
|
147
147
|
schema: REVIEW_SCHEMA,
|
|
148
148
|
verdict: args.verdict,
|
|
149
|
-
summary: `The
|
|
149
|
+
summary: `The run failed before the script finished: ${args.sessionError}`,
|
|
150
150
|
gaps: [],
|
|
151
151
|
};
|
|
152
152
|
}
|
|
@@ -154,41 +154,31 @@ function buildScriptedReview(args) {
|
|
|
154
154
|
return {
|
|
155
155
|
schema: REVIEW_SCHEMA,
|
|
156
156
|
verdict: args.verdict,
|
|
157
|
-
summary: `Dry run
|
|
157
|
+
summary: `Dry run: scenario ${args.journey.scenarioId} (${args.scenarioSource}, ${args.journey.steps.length} steps) would replay against ${args.appUrl} on ${plural(args.surfaces.length, "browser surface")}. No browser ran and $0 was spent.`,
|
|
158
158
|
gaps: ["Live scripted session not yet run (dry run only)."],
|
|
159
159
|
};
|
|
160
160
|
}
|
|
161
161
|
const passed = args.sessionResults.filter((result) => result.status === "passed").length;
|
|
162
|
+
const ran = args.sessionResults.length;
|
|
162
163
|
return {
|
|
163
164
|
schema: REVIEW_SCHEMA,
|
|
164
165
|
verdict: args.verdict,
|
|
165
|
-
summary: `
|
|
166
|
+
summary: `Scenario ${args.journey.scenarioId} ran on ${plural(ran, "browser surface")} against ${args.appUrl}: ${passed}/${ran} passed it${gapsListClause(ran - passed, ran)}.`,
|
|
166
167
|
gaps: args.sessionResults
|
|
167
168
|
.filter((result) => result.status !== "passed")
|
|
168
169
|
.map((result) => `${result.capture.surface.id}: ${result.reason}`),
|
|
169
170
|
};
|
|
170
171
|
}
|
|
171
|
-
export function renderScriptedReviewMarkdown(bundle) {
|
|
172
|
+
export function renderScriptedReviewMarkdown(bundle, status) {
|
|
172
173
|
const subject = bundle.events.find((event) => event.type === "scripted-lab.subject.declared");
|
|
173
174
|
const spend = bundle.events.find((event) => event.type === "scripted-lab.spend");
|
|
174
175
|
const traces = bundle.streams
|
|
175
176
|
.map((stream) => ({ stream, trace: stream.actor }))
|
|
176
177
|
.filter((entry) => entry.trace !== undefined);
|
|
177
|
-
return [
|
|
178
|
-
`# ${bundle.scenario.title}`,
|
|
179
|
-
"",
|
|
180
|
-
`- run: ${bundle.runId}`,
|
|
181
|
-
`- mode: ${bundle.mode}`,
|
|
182
|
-
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
183
|
-
`- outcome: ${reviewOutcome(bundle)}`,
|
|
184
|
-
`- summary: ${bundle.review.summary}`,
|
|
178
|
+
return renderReviewMarkdown(bundle, [
|
|
185
179
|
`- scenario: ${bundle.scenario.id} @ ${bundle.scenario.sourceDigest} (${bundle.scenario.source})`,
|
|
186
180
|
...(subject ? [`- subject: ${subject.message}`] : []),
|
|
187
181
|
...(spend ? [`- spend: ${spend.message}`] : []),
|
|
188
182
|
...traces.map(({ stream, trace }) => `- ${recordIdOf(stream)}: ${trace.provider} (${trace.lane}/${trace.protocol}) ${trace.status} (${trace.completionReason}); ${plural(trace.counts.actions ?? 0, "step action")}, ${plural(trace.counts.screenshots ?? 0, "raw screenshot")}`),
|
|
189
|
-
|
|
190
|
-
? ["", "## Gaps", ...bundle.review.gaps.map((gap) => `- ${gap}`)]
|
|
191
|
-
: []),
|
|
192
|
-
"",
|
|
193
|
-
].join("\n");
|
|
183
|
+
], { status });
|
|
194
184
|
}
|
|
@@ -6,7 +6,6 @@ import { type ParticipantIds } from "../../run/participant-records.js";
|
|
|
6
6
|
import type { RunStream } from "../../run/streams.js";
|
|
7
7
|
/** What each surface's records read from the bundle arguments. */
|
|
8
8
|
export interface ScriptedSurfaceContext {
|
|
9
|
-
actorId: string;
|
|
10
9
|
appUrl: string;
|
|
11
10
|
createdAt: string;
|
|
12
11
|
journey: BrowserPersonaJourney;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
|
|
1
|
+
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
2
2
|
import { participantIdsOf, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
3
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
4
|
+
import { sessionEndingInWords } from "../../run/outcomes.js";
|
|
3
5
|
/** A surface's saved ids: `scripted-<surface>` and its `-stream`. */
|
|
4
6
|
export function scriptedSurfaceIds(surfaceId) {
|
|
5
7
|
return participantIdsOf(`scripted-${surfaceId}`, `scripted-${surfaceId}-stream`);
|
|
@@ -11,14 +13,23 @@ export function scriptedSurfaceIds(surfaceId) {
|
|
|
11
13
|
export function scriptedSurfaceRecords(context, surface, index, result, screenshots) {
|
|
12
14
|
const ids = scriptedSurfaceIds(surface.id);
|
|
13
15
|
const lastScreenshot = screenshots.at(-1);
|
|
14
|
-
const status =
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
16
|
+
const { status, reason } = judgeParticipantRecords([
|
|
17
|
+
{
|
|
18
|
+
...(result === undefined
|
|
19
|
+
? {}
|
|
20
|
+
: {
|
|
21
|
+
status: result.status,
|
|
22
|
+
completionReason: result.completionReason,
|
|
23
|
+
reason: result.reason,
|
|
24
|
+
}),
|
|
25
|
+
...(context.sessionError === undefined ? {} : { sessionError: context.sessionError }),
|
|
26
|
+
skipped: false,
|
|
27
|
+
noEngagement: false,
|
|
28
|
+
selfReportedBlocker: false,
|
|
29
|
+
},
|
|
30
|
+
], {
|
|
31
|
+
contractReason: "Dry run: the scenario was pinned without launching a browser or touching the subject app.",
|
|
32
|
+
}).participants[0];
|
|
22
33
|
const simulation = participantRecord(ids, index + 1, {
|
|
23
34
|
personaId: context.persona.id,
|
|
24
35
|
scenarioId: context.journey.scenarioId,
|
|
@@ -28,16 +39,19 @@ export function scriptedSurfaceRecords(context, surface, index, result, screensh
|
|
|
28
39
|
progress: 100,
|
|
29
40
|
currentStep: reason,
|
|
30
41
|
summary: result
|
|
31
|
-
? `
|
|
42
|
+
? `Scenario ${context.journey.scenarioId} ran in the ${surface.id} browser and ${sessionEndingInWords(result.trace)}.`
|
|
32
43
|
: context.sessionError !== undefined
|
|
33
|
-
? `The
|
|
34
|
-
: `
|
|
44
|
+
? `The run failed before the script finished: ${context.sessionError}`
|
|
45
|
+
: `Scenario ${context.journey.scenarioId} would replay in the ${surface.id} browser against ${context.appUrl}; no browser ran.`,
|
|
35
46
|
startedAt: context.createdAt,
|
|
36
47
|
updatedAt: result?.capture.capturedAt ?? context.createdAt,
|
|
37
48
|
});
|
|
38
49
|
const stream = participantStream(ids, {
|
|
39
50
|
kind: "browser",
|
|
40
|
-
label:
|
|
51
|
+
label: participantCaption({
|
|
52
|
+
personaId: context.persona.id,
|
|
53
|
+
device: { name: surface.id, preset: { isMobile: surface.viewport.isMobile } },
|
|
54
|
+
}),
|
|
41
55
|
status,
|
|
42
56
|
transport: "snapshot",
|
|
43
57
|
updatedAt: result?.capture.capturedAt ?? context.createdAt,
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { RunEvent, RunSimulation } from "../../run/bundle.js";
|
|
2
|
+
import type { judgeParticipantRecords } from "../../run/judge.js";
|
|
2
3
|
import type { RunStream } from "../../run/streams.js";
|
|
3
4
|
import type { DesktopParticipantRun } from "../computer-use/types.js";
|
|
4
5
|
import type { ConcurrentBundleArgs } from "./types.js";
|
|
5
6
|
/** What every participant's records share. */
|
|
6
7
|
export interface SharedWorldRecordContext {
|
|
7
8
|
args: ConcurrentBundleArgs;
|
|
9
|
+
participants: ReturnType<typeof judgeParticipantRecords>["participants"];
|
|
8
10
|
external: boolean;
|
|
9
11
|
inProgress: boolean;
|
|
10
|
-
/** The public-safe plane label: never the raw getHost URL or public origin. */
|
|
11
|
-
appUrl: string;
|
|
12
12
|
/** Numbers events in the order they are pushed. */
|
|
13
13
|
nextEventId: (suffix: string) => string;
|
|
14
14
|
}
|
|
@@ -3,14 +3,13 @@
|
|
|
3
3
|
// running).
|
|
4
4
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
5
5
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
6
|
-
import { judgedStatus } from "../../run/judge.js";
|
|
7
6
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
8
|
-
import { participantFactsOf } from "../computer-use/participant-facts.js";
|
|
9
7
|
import { publicSafeRouteLabel } from "./provenance.js";
|
|
10
|
-
import {
|
|
8
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
9
|
+
import { sessionEndingInWords } from "../../run/outcomes.js";
|
|
11
10
|
function participantView(ctx, spec, index) {
|
|
12
|
-
const { args, external
|
|
13
|
-
const
|
|
11
|
+
const { args, external } = ctx;
|
|
12
|
+
const name = participantCaption({ id: spec.planned.id, personaId: spec.persona.id });
|
|
14
13
|
const result = args.actorResults[index];
|
|
15
14
|
const outcome = result?.outcome;
|
|
16
15
|
const session = outcome?.session;
|
|
@@ -21,18 +20,8 @@ function participantView(ctx, spec, index) {
|
|
|
21
20
|
const route = external
|
|
22
21
|
? "[external-public-plane]"
|
|
23
22
|
: publicSafeRouteLabel(args.plan.plane.participants[index]?.entry);
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
: outcome?.sessionError !== undefined
|
|
27
|
-
? "failed"
|
|
28
|
-
: inProgress
|
|
29
|
-
? "running"
|
|
30
|
-
: "contract_proof_only";
|
|
31
|
-
const reason = session?.reason ??
|
|
32
|
-
outcome?.sessionError ??
|
|
33
|
-
(inProgress
|
|
34
|
-
? "Actor desktop is running; the attached Observer hydrates the runtime stream URL without persisting it."
|
|
35
|
-
: "Dry run: the evidence shape was written without launching a desktop or spending provider tokens.");
|
|
23
|
+
const judgment = ctx.participants[index];
|
|
24
|
+
const { status, reason } = judgment;
|
|
36
25
|
const traceScreenshotMode = session?.trace.redaction.screenshots;
|
|
37
26
|
// Include `declared` on the no-outcome fallback too (dry-run, skipped participant): otherwise an
|
|
38
27
|
// absent `declared` means either "the preset rendered faithfully" or "there was no live
|
|
@@ -53,7 +42,8 @@ function participantView(ctx, spec, index) {
|
|
|
53
42
|
? "blurred"
|
|
54
43
|
: "raw";
|
|
55
44
|
return {
|
|
56
|
-
|
|
45
|
+
name,
|
|
46
|
+
judgment,
|
|
57
47
|
outcome,
|
|
58
48
|
session,
|
|
59
49
|
screenshots,
|
|
@@ -67,7 +57,7 @@ function participantView(ctx, spec, index) {
|
|
|
67
57
|
}
|
|
68
58
|
function sharedWorldSimulation(ctx, spec, index, view) {
|
|
69
59
|
const { args, inProgress } = ctx;
|
|
70
|
-
const {
|
|
60
|
+
const { name, outcome, session } = view;
|
|
71
61
|
return participantRecord(spec, index + 1, {
|
|
72
62
|
personaId: spec.persona.id,
|
|
73
63
|
scenarioId: `concurrent-shared-world-${args.plan.studyId}`,
|
|
@@ -77,27 +67,31 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
77
67
|
progress: inProgress ? 35 : 100,
|
|
78
68
|
currentStep: view.reason,
|
|
79
69
|
summary: session
|
|
80
|
-
?
|
|
70
|
+
? `${name} used the shared app and ${sessionEndingInWords(session.trace)}.`
|
|
81
71
|
: outcome?.sessionError !== undefined
|
|
82
|
-
?
|
|
72
|
+
? `${name} did not finish a session: ${outcome.sessionError}`
|
|
83
73
|
: inProgress
|
|
84
|
-
?
|
|
85
|
-
:
|
|
74
|
+
? `${name} is using the shared app.`
|
|
75
|
+
: `${name} would use the shared app; no session ran.`,
|
|
86
76
|
startedAt: args.run.createdAt,
|
|
87
77
|
updatedAt: args.run.createdAt,
|
|
88
78
|
});
|
|
89
79
|
}
|
|
90
80
|
function sharedWorldStream(ctx, spec, index, view) {
|
|
91
81
|
const { args } = ctx;
|
|
92
|
-
const {
|
|
93
|
-
|
|
94
|
-
const
|
|
82
|
+
const { name, session, screenshots, lastScreenshot, desktopGeometry, screenshotMode } = view;
|
|
83
|
+
const judged = view.judgment.judgedStatus;
|
|
84
|
+
const caption = participantCaption({
|
|
85
|
+
id: spec.planned.id,
|
|
86
|
+
personaId: spec.persona.id,
|
|
87
|
+
device: spec.planned.device,
|
|
88
|
+
});
|
|
95
89
|
return participantStream(spec, {
|
|
96
90
|
...(spec.evidenceAssignment === undefined
|
|
97
91
|
? {}
|
|
98
92
|
: { assignment: participantAssignment(spec.evidenceAssignment) }),
|
|
99
93
|
kind: "browser",
|
|
100
|
-
label:
|
|
94
|
+
label: caption,
|
|
101
95
|
status: view.status,
|
|
102
96
|
...(judged === undefined ? {} : { judgedStatus: judged }),
|
|
103
97
|
transport: "snapshot",
|
|
@@ -106,9 +100,9 @@ function sharedWorldStream(ctx, spec, index, view) {
|
|
|
106
100
|
? {
|
|
107
101
|
kind: "screenshot",
|
|
108
102
|
url: lastScreenshot,
|
|
109
|
-
title:
|
|
103
|
+
title: `${caption} (${screenshotMode})`,
|
|
110
104
|
}
|
|
111
|
-
: { kind: "placeholder", title:
|
|
105
|
+
: { kind: "placeholder", title: caption },
|
|
112
106
|
...(desktopGeometry.viewport === undefined
|
|
113
107
|
? {}
|
|
114
108
|
: {
|
|
@@ -122,7 +116,7 @@ function sharedWorldStream(ctx, spec, index, view) {
|
|
|
122
116
|
desktopGeometry,
|
|
123
117
|
ui: {
|
|
124
118
|
route: view.route,
|
|
125
|
-
intent: `Watch
|
|
119
|
+
intent: `Watch ${name} use the shared app at the same time as the others.`,
|
|
126
120
|
state: view.reason,
|
|
127
121
|
...(session ? { actorStatus: session.status } : {}),
|
|
128
122
|
...(lastScreenshot ? { screenshotUrl: lastScreenshot } : {}),
|
|
@@ -17,4 +17,4 @@ export declare function judgeSharedWorldRun(args: Omit<ConcurrentBundleArgs, "ju
|
|
|
17
17
|
export declare function actorRunPassed(result: ActorRunResult | undefined): boolean;
|
|
18
18
|
/** Project the concurrent run into a humanish.run-bundle.v1 with the concurrent shared-world block. */
|
|
19
19
|
export declare function buildConcurrentSharedWorldBundle(args: ConcurrentBundleArgs): RunBundle;
|
|
20
|
-
export declare function renderConcurrentReviewMarkdown(bundle: RunBundle): string;
|
|
20
|
+
export declare function renderConcurrentReviewMarkdown(bundle: RunBundle, status?: unknown): string;
|