humanish 0.110.0 → 0.110.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +14 -11
- package/CHANGELOG.md +13 -0
- package/CONTEXT.md +8 -6
- package/README.md +96 -47
- package/dist/actors/computer-use/loop/ending.js +4 -4
- package/dist/actors/local-agent/claude-participant.d.ts +74 -0
- package/dist/actors/local-agent/claude-participant.js +243 -0
- package/dist/actors/local-agent/claude-session.d.ts +16 -3
- package/dist/actors/local-agent/claude-session.js +64 -23
- package/dist/actors/local-agent/cli.d.ts +11 -0
- package/dist/actors/local-agent/cli.js +103 -24
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +18 -11
- package/dist/cli/commands/analyze.js +21 -5
- package/dist/cli/commands/run-signals.d.ts +2 -0
- package/dist/cli/commands/run-signals.js +96 -44
- package/dist/cli/commands/runs.d.ts +5 -0
- package/dist/cli/commands/runs.js +86 -70
- package/dist/cli/commands/study-route-computer-use.js +2 -0
- package/dist/cli/commands/study-route-scripted.js +2 -0
- package/dist/cli/commands/study-route-shared-world.js +2 -0
- package/dist/cli/commands/study-route-terminal.js +2 -0
- package/dist/cli/doctor.d.ts +4 -1
- package/dist/cli/doctor.js +42 -4
- package/dist/cli/findings.d.ts +103 -0
- package/dist/cli/findings.js +316 -0
- package/dist/cli/first-run-path.d.ts +5 -4
- package/dist/cli/first-run-path.js +14 -7
- package/dist/cli/io.js +2 -1
- package/dist/cli/product-sentence.d.ts +3 -3
- package/dist/cli/product-sentence.js +3 -3
- package/dist/keys/env-file.d.ts +4 -0
- package/dist/keys/env-file.js +17 -0
- package/dist/observer/data.d.ts +14 -1
- package/dist/observer/data.js +19 -9
- package/dist/observer/library.d.ts +4 -0
- package/dist/observer/library.js +18 -15
- package/dist/observer/render.js +8 -2
- package/dist/observer/run-routes.d.ts +2 -0
- package/dist/observer/run-routes.js +43 -34
- package/dist/observer-app.html +3 -3
- package/dist/routes/computer-use/bundle.js +3 -1
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +3 -1
- package/dist/routes/computer-use/participant-model.js +5 -2
- package/dist/routes/computer-use/plan.js +4 -4
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/result.js +17 -28
- package/dist/routes/computer-use/single-bundle.js +7 -1
- package/dist/routes/scripted/bundle.js +4 -2
- package/dist/routes/scripted/plan.js +2 -2
- package/dist/routes/scripted/result.d.ts +3 -5
- package/dist/routes/scripted/result.js +16 -22
- package/dist/routes/scripted/scenario.js +4 -4
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.js +2 -0
- package/dist/routes/shared-world/plan.js +1 -1
- package/dist/routes/shared-world/result.js +22 -22
- package/dist/routes/terminal/bundle.js +4 -2
- package/dist/routes/terminal/dry-run.js +8 -12
- package/dist/routes/terminal/ledger.d.ts +3 -3
- package/dist/routes/terminal/ledger.js +6 -6
- package/dist/routes/terminal/live-finish.js +15 -13
- package/dist/routes/terminal/plan.js +4 -4
- package/dist/routes/terminal/result.d.ts +6 -7
- package/dist/routes/terminal/result.js +6 -22
- package/dist/routes/terminal/session.js +1 -1
- package/dist/run/bundle-shape.d.ts +3 -1
- package/dist/run/bundle-shape.js +25 -1
- package/dist/run/bundle.d.ts +24 -0
- package/dist/run/display.d.ts +59 -0
- package/dist/run/display.js +106 -0
- package/dist/run/dry-run.js +11 -7
- package/dist/run/judge.d.ts +3 -2
- package/dist/run/judge.js +10 -0
- package/dist/run/pricing.js +1 -1
- package/dist/run/projection.d.ts +0 -2
- package/dist/run/projection.js +0 -11
- package/dist/run/reclaim-outcomes.d.ts +89 -0
- package/dist/run/reclaim-outcomes.js +213 -0
- package/dist/run/reclaim.d.ts +18 -17
- package/dist/run/reclaim.js +180 -113
- package/dist/run/results.d.ts +10 -20
- package/dist/run/results.js +2 -2
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +11 -0
- package/dist/run/run.d.ts +34 -18
- package/dist/run/run.js +166 -67
- package/dist/run/sandbox-creates.d.ts +59 -0
- package/dist/run/sandbox-creates.js +138 -0
- package/dist/run/sandbox-receipts.d.ts +14 -0
- package/dist/run/sandbox-receipts.js +44 -0
- package/dist/run/stats.d.ts +5 -0
- package/dist/run/stats.js +8 -4
- package/dist/run/status.d.ts +6 -9
- package/dist/run/status.js +7 -23
- package/dist/run/stored-runs.d.ts +5 -14
- package/dist/run/stored-runs.js +31 -146
- package/dist/run/streams.d.ts +3 -2
- package/dist/study/doctor.d.ts +1 -1
- package/dist/study/doctor.js +5 -2
- package/dist/study/init.js +14 -3
- package/dist/study/keys.d.ts +12 -0
- package/dist/study/keys.js +32 -0
- package/dist/study/persona.d.ts +4 -0
- package/dist/study/persona.js +4 -2
- package/dist/substrates/e2b/peer-install.d.ts +51 -0
- package/dist/substrates/e2b/peer-install.js +161 -0
- package/dist/substrates/e2b/sandbox.d.ts +34 -6
- package/dist/substrates/e2b/sandbox.js +128 -35
- package/dist/substrates/e2b/sdk.d.ts +33 -0
- package/dist/substrates/e2b/sdk.js +21 -21
- package/dist/substrates/shell.d.ts +2 -0
- package/dist/substrates/shell.js +4 -0
- package/dist/tui-app.js +133 -133
- package/dist/verify/liveness.d.ts +19 -3
- package/dist/verify/liveness.js +46 -25
- package/dist/verify/verify.d.ts +3 -0
- package/dist/verify/verify.js +6 -2
- package/docs/architecture/actor-contract.md +2 -1
- package/docs/architecture/observer.md +22 -7
- package/docs/architecture/task-protocol-support.md +13 -13
- package/docs/architecture/terminal-product-route.md +1 -1
- package/docs/contracts/run-bundle.md +75 -6
- package/docs/contracts/schemas.md +39 -22
- package/docs/contracts/study-analysis.md +45 -0
- package/docs/contracts/study-costs.md +8 -0
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +3 -0
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +9 -3
- package/docs/principles/invariants-and-defaults.md +11 -8
- package/package.json +4 -3
- package/skills/humanish/SKILL.md +16 -2
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// shape's arguments. renderCuaReviewMarkdown writes review.md. shared-world/bundle.ts holds the
|
|
4
4
|
// same for that route.
|
|
5
5
|
import { judgeOneParticipant, judgeParticipants, verdictText, } from "../../run/judge.js";
|
|
6
|
+
import { reviewOutcome } from "../../run/display.js";
|
|
6
7
|
import { desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
7
8
|
import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
|
|
8
9
|
import { providerResourcesForOutcome } from "./bundle-parts.js";
|
|
@@ -158,7 +159,8 @@ export function renderCuaReviewMarkdown(bundle) {
|
|
|
158
159
|
"",
|
|
159
160
|
`- run: ${bundle.runId}`,
|
|
160
161
|
`- mode: ${bundle.mode}`,
|
|
161
|
-
`-
|
|
162
|
+
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
163
|
+
`- outcome: ${reviewOutcome(bundle)}`,
|
|
162
164
|
`- summary: ${bundle.review.summary}`,
|
|
163
165
|
...(provenance ? [`- subject: ${provenance.message}`] : []),
|
|
164
166
|
...(trace
|
|
@@ -269,7 +269,7 @@ function fanoutOutcomeEvents(ctx, spec, view) {
|
|
|
269
269
|
at: args.run.createdAt,
|
|
270
270
|
level: "info",
|
|
271
271
|
type: "cua-lab.contract.ready",
|
|
272
|
-
message: `Participant ${spec.planned.id}: dry-run participant ready; switch
|
|
272
|
+
message: `Participant ${spec.planned.id}: dry-run participant ready; switch mode to live for a real desktop session.`,
|
|
273
273
|
});
|
|
274
274
|
}
|
|
275
275
|
for (const warning of desktopGeometry.warnings ?? []) {
|
|
@@ -90,7 +90,9 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
90
90
|
if (closed.refusal !== undefined)
|
|
91
91
|
providerPolicyError = redactText(deps.scrubKnownValues(closed.refusal === "codex_tool_call"
|
|
92
92
|
? `Codex reported a disallowed item after the participant's last request (${closed.refusal}).`
|
|
93
|
-
:
|
|
93
|
+
: closed.refusal.startsWith("HUMANISH_CLAUDE_PARTICIPANT_")
|
|
94
|
+
? `Claude Code's stream showed a call the participant may not make after its last turn (${closed.refusal}); the participant was stopped.`
|
|
95
|
+
: `Codex output after the participant's last request could not be checked against the item policy (${closed.refusal}).`));
|
|
94
96
|
try {
|
|
95
97
|
if (!provisioned)
|
|
96
98
|
signal(false);
|
|
@@ -66,10 +66,12 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
66
66
|
? {}
|
|
67
67
|
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
68
68
|
...(brain.declaredModel === undefined ? {} : { model: brain.declaredModel }),
|
|
69
|
+
env,
|
|
69
70
|
}),
|
|
70
71
|
};
|
|
71
72
|
}
|
|
72
73
|
const claudeSession = await startClaudeSession({
|
|
74
|
+
env,
|
|
73
75
|
...(spec.planned.limits.reasoningEffort === undefined
|
|
74
76
|
? {}
|
|
75
77
|
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
@@ -139,7 +141,7 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
139
141
|
const estimate = estimateActorCostForExecution(usage, provider?.version ?? capModelId, provider?.executionProfile).estimatedCostUsd;
|
|
140
142
|
const totalUsd = deps.runBudget.note(spec.planned.id, estimate);
|
|
141
143
|
return totalUsd > deps.runBudget.maxTotalUsd
|
|
142
|
-
? `study budget reached: the run's estimated model spend $${round6(totalUsd)} crossed
|
|
144
|
+
? `study budget reached: the run's estimated model spend $${round6(totalUsd)} crossed caps.maxTotalUsd=$${deps.runBudget.maxTotalUsd}; this lane stops here and sibling lanes stop at their next turn`
|
|
143
145
|
: null;
|
|
144
146
|
},
|
|
145
147
|
}),
|
|
@@ -198,7 +200,8 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
198
200
|
}
|
|
199
201
|
}
|
|
200
202
|
try {
|
|
201
|
-
await model.claudeSession?.close();
|
|
203
|
+
const closed = await model.claudeSession?.close();
|
|
204
|
+
refusal ??= closed?.refusal;
|
|
202
205
|
}
|
|
203
206
|
catch {
|
|
204
207
|
warnings.push("Claude session cleanup failed; desktop cleanup will still run.");
|
|
@@ -148,8 +148,8 @@ function entryTargetReason(config, subjectRoute) {
|
|
|
148
148
|
return {
|
|
149
149
|
code: "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE",
|
|
150
150
|
message: provisionedRoute || localAppSubject || !allowPublicTargets
|
|
151
|
-
? "subject.appUrl and any
|
|
152
|
-
: "subject.appUrl and
|
|
151
|
+
? "subject.appUrl and any participants[].target entries must be loopback (127.0.0.1 or localhost) unless policies.allowPublicTargets is set for an app-url subject."
|
|
152
|
+
: "subject.appUrl and participants[].target entries must be valid http(s) URLs.",
|
|
153
153
|
};
|
|
154
154
|
}
|
|
155
155
|
/** Something to drive the subject: the caller's executor with its provider. */
|
|
@@ -218,7 +218,7 @@ export function planComputerUseStudy(config, input) {
|
|
|
218
218
|
const actorType = config.actors[0]?.type ?? "";
|
|
219
219
|
const descriptor = actorRegistry[actorType];
|
|
220
220
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
221
|
-
return refuse("in-scope", "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED", `
|
|
221
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered computer-use actor.`);
|
|
222
222
|
const actor = descriptor.id;
|
|
223
223
|
const rejection = cuaStudyRejection(config, hasRunSession, driving, declaredSubjectRoute(config, driving));
|
|
224
224
|
if (rejection)
|
|
@@ -227,7 +227,7 @@ export function planComputerUseStudy(config, input) {
|
|
|
227
227
|
// participants. It comes after the rules above, so a shared-world config that breaks one of them,
|
|
228
228
|
// which runStudyWith sends here, still gets that rule's message.
|
|
229
229
|
if (config.subject.topology === "shared-world")
|
|
230
|
-
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", "
|
|
230
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", "route: shared-world studies run every participant against one shared app on the shared-world route; this route runs independent participants. Run the study with runStudy or runConcurrentSharedWorld.", actor);
|
|
231
231
|
// The in-process route drives subject.appUrl on this machine and creates no desktop, so it would
|
|
232
232
|
// skip the subject a clone, local-tree or desktop-cli study declares.
|
|
233
233
|
const source = config.subject.source;
|
|
@@ -35,7 +35,7 @@ export async function liveCuaRejection(args) {
|
|
|
35
35
|
return ready.length === 0
|
|
36
36
|
? ""
|
|
37
37
|
: ` ${ready.map((agent) => agent.label).join(" and ")} reports authenticated on this machine` +
|
|
38
|
-
`. Set
|
|
38
|
+
`. Set actor.type: local-agent to use ${ready.length === 1 ? "it" : "one"} instead of a key.`;
|
|
39
39
|
})()
|
|
40
40
|
: "";
|
|
41
41
|
return {
|
|
@@ -64,12 +64,7 @@ function cuaStudyResult(args) {
|
|
|
64
64
|
// engaged session.
|
|
65
65
|
const participantOk = (outcome) => participantOutcomeOk(outcome, dryRun);
|
|
66
66
|
const adapterFailure = adapterScoreFailureMessage(bundle);
|
|
67
|
-
const ok =
|
|
68
|
-
judgment: args.judgment,
|
|
69
|
-
execution: args.execution,
|
|
70
|
-
scorerFailures: args.scorerFailures,
|
|
71
|
-
policy: OUTCOME_POLICIES["computer-use"],
|
|
72
|
-
});
|
|
67
|
+
const { ok } = args.outcome;
|
|
73
68
|
const participantWarnings = (outcomes ?? []).flatMap((outcome) => outcome.warnings);
|
|
74
69
|
const warnings = [
|
|
75
70
|
...receivingWarnings,
|
|
@@ -115,7 +110,7 @@ function cuaStudyResult(args) {
|
|
|
115
110
|
return {
|
|
116
111
|
code,
|
|
117
112
|
message: observer.ok
|
|
118
|
-
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${failing?.sessionError !== undefined ? `; first failure: ${failing.sessionError}` : failing === undefined && args.execution.failures[0] !== undefined ? `; ${args.execution.failures[0].message}` : ""}.`
|
|
113
|
+
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${failing?.sessionError !== undefined ? `; first failure: ${failing.sessionError}` : failing === undefined && args.outcome.execution.failures[0] !== undefined ? `; ${args.outcome.execution.failures[0].message}` : ""}.`
|
|
119
114
|
: (observer.error?.message ?? "Observer failed for the computer-use fan-out run."),
|
|
120
115
|
};
|
|
121
116
|
})();
|
|
@@ -165,11 +160,12 @@ function cuaStudyResult(args) {
|
|
|
165
160
|
};
|
|
166
161
|
}
|
|
167
162
|
/**
|
|
168
|
-
* The run's execution failures: each participant whose session failed
|
|
169
|
-
* cleanup is unconfirmed or that reported a disallowed item
|
|
170
|
-
*
|
|
163
|
+
* The run's execution failures before the Observer renders: each participant whose session failed
|
|
164
|
+
* in the harness, each provider whose cleanup is unconfirmed or that reported a disallowed item
|
|
165
|
+
* after its last request, and each sandbox whose release is unconfirmed. FinishedRun.renderObserver
|
|
166
|
+
* adds an Observer that did not render.
|
|
171
167
|
*/
|
|
172
|
-
function computerUseExecutionFailures(outcomes,
|
|
168
|
+
function computerUseExecutionFailures(outcomes, runId) {
|
|
173
169
|
return [
|
|
174
170
|
...(outcomes ?? [])
|
|
175
171
|
.filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
|
|
@@ -194,14 +190,6 @@ function computerUseExecutionFailures(outcomes, observer, runId) {
|
|
|
194
190
|
},
|
|
195
191
|
]),
|
|
196
192
|
...unreleasedSandboxFailures(outcomes, runId),
|
|
197
|
-
...(observer.ok
|
|
198
|
-
? []
|
|
199
|
-
: [
|
|
200
|
-
{
|
|
201
|
-
kind: "evidence",
|
|
202
|
-
message: observer.error?.message ?? "Observer failed for the computer-use run.",
|
|
203
|
-
},
|
|
204
|
-
]),
|
|
205
193
|
];
|
|
206
194
|
}
|
|
207
195
|
/**
|
|
@@ -215,7 +203,7 @@ function participantCapWarning(caps, participantCount) {
|
|
|
215
203
|
return undefined;
|
|
216
204
|
if (caps.maxTotalUsd !== undefined)
|
|
217
205
|
return undefined;
|
|
218
|
-
return `
|
|
206
|
+
return `caps.maxUsd ($${capUsd}) caps each participant, so ${participantCount} participants may spend up to ${participantCount} × $${capUsd} (about $${round6(capUsd * participantCount)}) before any of them stops. Set caps.maxTotalUsd for one budget across the study.`;
|
|
219
207
|
}
|
|
220
208
|
/** Builds and publishes the final bundle, runs the adapter hooks, renders the Observer and returns the result. */
|
|
221
209
|
export async function finishCuaRun(setup, finish, ran) {
|
|
@@ -272,11 +260,16 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
272
260
|
bundle.review = foldScorerFailures(bundle.review, scorerResult.failures);
|
|
273
261
|
if (receiving)
|
|
274
262
|
bundle.commsReceiving = receiving.snapshot();
|
|
275
|
-
const
|
|
263
|
+
const policy = OUTCOME_POLICIES["computer-use"];
|
|
264
|
+
const execution = judgeExecution(computerUseExecutionFailures(outcomes, runId), policy);
|
|
265
|
+
const finished = await run.finish(bundle, {
|
|
266
|
+
ok: resultOk({ judgment, execution, scorerFailures: scorerResult.failures, policy }),
|
|
267
|
+
execution,
|
|
268
|
+
policy,
|
|
269
|
+
});
|
|
276
270
|
const observer = await finished.renderObserver();
|
|
277
271
|
streams.attachFinal(observer);
|
|
278
|
-
|
|
279
|
-
const result = cuaStudyResult({
|
|
272
|
+
return cuaStudyResult({
|
|
280
273
|
studyId: plan.studyId,
|
|
281
274
|
cwd,
|
|
282
275
|
runId,
|
|
@@ -290,14 +283,10 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
290
283
|
participantPlan,
|
|
291
284
|
rerunLineage,
|
|
292
285
|
bundle,
|
|
293
|
-
|
|
294
|
-
execution,
|
|
286
|
+
outcome: finished.outcome,
|
|
295
287
|
observer,
|
|
296
|
-
scorerFailures: scorerResult.failures,
|
|
297
288
|
receivingWarnings,
|
|
298
289
|
aggregateWarnings,
|
|
299
290
|
adapterWarnings,
|
|
300
291
|
});
|
|
301
|
-
await finished.recordOutcome({ ok: result.ok, execution });
|
|
302
|
-
return result;
|
|
303
292
|
}
|
|
@@ -100,6 +100,11 @@ function singleSimulation(args, view) {
|
|
|
100
100
|
}
|
|
101
101
|
function singleStream(args, view) {
|
|
102
102
|
const { publicAppUrl, status, reason, lastScreenshot, desktopGeometry, screenshotMode } = view;
|
|
103
|
+
// The judge's status for a finished session, as the review counts it, so the Observer shows
|
|
104
|
+
// the participant the way the verdict judged it.
|
|
105
|
+
const judged = args.session === undefined || args.inProgress === true
|
|
106
|
+
? undefined
|
|
107
|
+
: participantStatusFor(args.session.status, args.credibility);
|
|
103
108
|
return participantStream(SINGLE, {
|
|
104
109
|
...(args.assignment === undefined
|
|
105
110
|
? {}
|
|
@@ -110,6 +115,7 @@ function singleStream(args, view) {
|
|
|
110
115
|
kind: "browser",
|
|
111
116
|
label: `${args.participantId ?? "lane-01"} · browser`,
|
|
112
117
|
status,
|
|
118
|
+
...(judged === undefined ? {} : { judgedStatus: judged }),
|
|
113
119
|
transport: "snapshot",
|
|
114
120
|
updatedAt: args.run.createdAt,
|
|
115
121
|
embed: lastScreenshot
|
|
@@ -221,7 +227,7 @@ function singleEvents(args, view) {
|
|
|
221
227
|
at: args.run.createdAt,
|
|
222
228
|
level: "info",
|
|
223
229
|
type: "cua-lab.contract.ready",
|
|
224
|
-
message: "Dry-run bundle ready; switch
|
|
230
|
+
message: "Dry-run bundle ready; switch mode to live for a real desktop session.",
|
|
225
231
|
}),
|
|
226
232
|
];
|
|
227
233
|
const record = (event) => events.push(participantEvent(SINGLE, event));
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Assemble the scripted-browser study's run bundle from its sessions, and the review summary and
|
|
2
2
|
// markdown that ship with it.
|
|
3
3
|
import { verdictText } from "../../run/judge.js";
|
|
4
|
+
import { reviewOutcome } from "../../run/display.js";
|
|
4
5
|
import { redactText } from "../../evidence/redaction.js";
|
|
5
6
|
import { buildRunCostSummary, desktopSpanToMinutes, spendFreeCostSummary, } from "../../run/cost-summary.js";
|
|
6
7
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
@@ -90,7 +91,7 @@ function scriptedEvents(args) {
|
|
|
90
91
|
type: "scripted-lab.spend",
|
|
91
92
|
message: args.subject
|
|
92
93
|
? "Scripted participant steps make no model requests; post-run analysis has a separate budget unless disabled. Live provisioned runs may spend E2B sandbox minutes to clone/serve the synthetic subject."
|
|
93
|
-
: "Scripted participant steps make no model requests and use no sandbox on this route; post-run analysis has a separate budget unless disabled.
|
|
94
|
+
: "Scripted participant steps make no model requests and use no sandbox on this route; post-run analysis has a separate budget unless disabled. mode: live gates real browser actuation against the declared app.",
|
|
94
95
|
},
|
|
95
96
|
];
|
|
96
97
|
if (args.sessionResults.length > 0) {
|
|
@@ -119,7 +120,7 @@ function scriptedEvents(args) {
|
|
|
119
120
|
at: args.run.createdAt,
|
|
120
121
|
level: "info",
|
|
121
122
|
type: "scripted-lab.contract.ready",
|
|
122
|
-
message: `Dry-run bundle ready: scenario ${args.journey.scenarioId} @ ${args.scenarioSourceDigest} (${args.scenarioSource}, ${args.journey.steps.length} step${args.journey.steps.length === 1 ? "" : "s"}) parsed and digest-pinned; switch
|
|
123
|
+
message: `Dry-run bundle ready: scenario ${args.journey.scenarioId} @ ${args.scenarioSourceDigest} (${args.scenarioSource}, ${args.journey.steps.length} step${args.journey.steps.length === 1 ? "" : "s"}) parsed and digest-pinned; switch mode to live to actuate a real browser.`,
|
|
123
124
|
});
|
|
124
125
|
}
|
|
125
126
|
return events;
|
|
@@ -179,6 +180,7 @@ export function renderScriptedReviewMarkdown(bundle) {
|
|
|
179
180
|
`- run: ${bundle.runId}`,
|
|
180
181
|
`- mode: ${bundle.mode}`,
|
|
181
182
|
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
183
|
+
`- outcome: ${reviewOutcome(bundle)}`,
|
|
182
184
|
`- summary: ${bundle.review.summary}`,
|
|
183
185
|
`- scenario: ${bundle.scenario.id} @ ${bundle.scenario.sourceDigest} (${bundle.scenario.source})`,
|
|
184
186
|
...(subject ? [`- subject: ${subject.message}`] : []),
|
|
@@ -45,7 +45,7 @@ export function planScriptedStudy(config, input) {
|
|
|
45
45
|
const actorType = config.actors[0]?.type ?? "";
|
|
46
46
|
const descriptor = actorRegistry[actorType];
|
|
47
47
|
if (!descriptor || !isScriptedBrowserActorDescriptor(descriptor))
|
|
48
|
-
return refuse("HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED", `
|
|
48
|
+
return refuse("HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered scripted-browser actor.`);
|
|
49
49
|
const actor = descriptor.id;
|
|
50
50
|
// The parser checks the loopback entry too; a library caller skips the parser. A clone's URL is
|
|
51
51
|
// minted by getHost during the run, and its evidence names only the placeholder.
|
|
@@ -65,7 +65,7 @@ export function planScriptedStudy(config, input) {
|
|
|
65
65
|
// The steps are the actor, so there is no built-in journey to fall back on.
|
|
66
66
|
const scenarioRef = config.scenario?.ref;
|
|
67
67
|
if (!scenarioRef?.trim())
|
|
68
|
-
return refuse("HUMANISH_SCRIPTED_SCENARIO_INVALID", "A scripted-browser study needs `scenario
|
|
68
|
+
return refuse("HUMANISH_SCRIPTED_SCENARIO_INVALID", "A scripted-browser study needs `scenario`: the actor runs the browser steps in that scenario file.", { actor, appUrl: evidenceAppUrlOf(subject) });
|
|
69
69
|
const requirements = [];
|
|
70
70
|
if (!input.dryRun && subject.kind === "clone") {
|
|
71
71
|
requirements.push({ kind: "key", name: "E2B_API_KEY" });
|
|
@@ -5,7 +5,6 @@ import type { ScriptedPlan } from "../../study/plan-types.js";
|
|
|
5
5
|
import type { RunBundle } from "../../run/bundle.js";
|
|
6
6
|
import type { RunScope } from "../../run/run.js";
|
|
7
7
|
import { type ExecutionFailure } from "../../run/judge.js";
|
|
8
|
-
import type { ObserverResult } from "../../observer/render.js";
|
|
9
8
|
import type { ScriptedSubject } from "./subject.js";
|
|
10
9
|
import { type ScriptedBrowserStudyResult } from "./types.js";
|
|
11
10
|
/** What the finish reads from the run. */
|
|
@@ -35,9 +34,9 @@ export interface ScriptedFinishInputs {
|
|
|
35
34
|
scriptedSubject: ScriptedSubject | undefined;
|
|
36
35
|
}
|
|
37
36
|
/**
|
|
38
|
-
* The scripted run's execution failures: the session's own error, a
|
|
39
|
-
* returned, a surface that ended in a harness error, a subject sandbox
|
|
40
|
-
* unconfirmed
|
|
37
|
+
* The scripted run's execution failures before the Observer renders: the session's own error, a
|
|
38
|
+
* live surface that never returned, a surface that ended in a harness error, and a subject sandbox
|
|
39
|
+
* whose release is unconfirmed. FinishedRun.renderObserver adds an Observer that did not render.
|
|
41
40
|
*/
|
|
42
41
|
export declare function scriptedExecutionFailures(args: {
|
|
43
42
|
dryRun: boolean;
|
|
@@ -46,6 +45,5 @@ export declare function scriptedExecutionFailures(args: {
|
|
|
46
45
|
expected: number;
|
|
47
46
|
sessionResults: readonly Pick<ScriptedBrowserSessionResult, "completionReason" | "reason">[];
|
|
48
47
|
subject: Pick<ScriptedSubject, "sandboxId" | "killed" | "releaseWarning"> | undefined;
|
|
49
|
-
observer: Pick<ObserverResult, "ok" | "error">;
|
|
50
48
|
}): ExecutionFailure[];
|
|
51
49
|
export declare function finishScriptedRun(inputs: ScriptedFinishInputs): Promise<ScriptedBrowserStudyResult>;
|
|
@@ -16,12 +16,12 @@ function scriptedSurfaceFacts(result) {
|
|
|
16
16
|
};
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
|
-
* The scripted run's execution failures: the session's own error, a
|
|
20
|
-
* returned, a surface that ended in a harness error, a subject sandbox
|
|
21
|
-
* unconfirmed
|
|
19
|
+
* The scripted run's execution failures before the Observer renders: the session's own error, a
|
|
20
|
+
* live surface that never returned, a surface that ended in a harness error, and a subject sandbox
|
|
21
|
+
* whose release is unconfirmed. FinishedRun.renderObserver adds an Observer that did not render.
|
|
22
22
|
*/
|
|
23
23
|
export function scriptedExecutionFailures(args) {
|
|
24
|
-
const { sessionError, sessionResults
|
|
24
|
+
const { sessionError, sessionResults } = args;
|
|
25
25
|
const harnessErrorSession = sessionResults.find((result) => result.completionReason === "harness_error");
|
|
26
26
|
return [
|
|
27
27
|
...(sessionError === undefined ? [] : [{ kind: "harness", message: sessionError }]),
|
|
@@ -44,14 +44,6 @@ export function scriptedExecutionFailures(args) {
|
|
|
44
44
|
...(args.subject?.sandboxId === undefined || args.subject.killed
|
|
45
45
|
? []
|
|
46
46
|
: [sandboxCleanupFailure("subject", args.subject.releaseWarning, args.runId)]),
|
|
47
|
-
...(observer.ok
|
|
48
|
-
? []
|
|
49
|
-
: [
|
|
50
|
-
{
|
|
51
|
-
kind: "evidence",
|
|
52
|
-
message: observer.error?.message ?? "Observer failed for the scripted run.",
|
|
53
|
-
},
|
|
54
|
-
]),
|
|
55
47
|
];
|
|
56
48
|
}
|
|
57
49
|
export async function finishScriptedRun(inputs) {
|
|
@@ -114,13 +106,6 @@ export async function finishScriptedRun(inputs) {
|
|
|
114
106
|
: { desktopTemplate: plan.residual.execution.desktop.template }),
|
|
115
107
|
...(hostDigest === undefined ? {} : { hostDigest }),
|
|
116
108
|
});
|
|
117
|
-
const finished = await run.finish(bundle);
|
|
118
|
-
// Surface the local-fidelity posture so the operator knows the bundle is not publish-safe as-is.
|
|
119
|
-
if (sessionResults.some((result) => result.trace.redaction.screenshots === "raw")) {
|
|
120
|
-
warnings.push("Screenshots are unblurred for local use: the bundle stays in the gitignored .humanish/ folder and nothing scans these pixels, so review them before sharing. policies.redactScreenshots is not supported on the scripted route yet.");
|
|
121
|
-
}
|
|
122
|
-
const observer = await finished.renderObserver();
|
|
123
|
-
await validatePreparedRunArtifactPaths(runPaths);
|
|
124
109
|
// A failing surface is captured evidence on this route: only the execution fails ok.
|
|
125
110
|
const policy = OUTCOME_POLICIES.scripted;
|
|
126
111
|
const execution = judgeExecution(scriptedExecutionFailures({
|
|
@@ -130,10 +115,19 @@ export async function finishScriptedRun(inputs) {
|
|
|
130
115
|
expected: surfaces.length,
|
|
131
116
|
sessionResults,
|
|
132
117
|
subject: scriptedSubject,
|
|
133
|
-
observer,
|
|
134
118
|
}), policy);
|
|
135
|
-
const
|
|
136
|
-
|
|
119
|
+
const finished = await run.finish(bundle, {
|
|
120
|
+
ok: resultOk({ judgment, execution, scorerFailures: [], policy }),
|
|
121
|
+
execution,
|
|
122
|
+
policy,
|
|
123
|
+
});
|
|
124
|
+
// Surface the local-fidelity posture so the operator knows the bundle is not publish-safe as-is.
|
|
125
|
+
if (sessionResults.some((result) => result.trace.redaction.screenshots === "raw")) {
|
|
126
|
+
warnings.push("Screenshots are unblurred for local use: the bundle stays in the gitignored .humanish/ folder and nothing scans these pixels, so review them before sharing. policies.redactScreenshots is not supported on the scripted route yet.");
|
|
127
|
+
}
|
|
128
|
+
const observer = await finished.renderObserver();
|
|
129
|
+
await validatePreparedRunArtifactPaths(runPaths);
|
|
130
|
+
const { ok } = finished.outcome;
|
|
137
131
|
const harnessErrorSession = sessionResults.find((result) => result.completionReason === "harness_error");
|
|
138
132
|
return {
|
|
139
133
|
...studyResultIdentity("scripted", plan.studyId),
|
|
@@ -26,7 +26,7 @@ export async function resolveScriptedScenario(projectRoot, ref) {
|
|
|
26
26
|
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
27
27
|
return {
|
|
28
28
|
ok: false,
|
|
29
|
-
message: `scenario
|
|
29
|
+
message: `scenario must stay inside the project directory, and "${trimmed}" does not: the run records it relative to the project.`,
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
32
|
source = relative.split(path.sep).join("/");
|
|
@@ -35,7 +35,7 @@ export async function resolveScriptedScenario(projectRoot, ref) {
|
|
|
35
35
|
if (!SCENARIO_REF_ID_PATTERN.test(trimmed)) {
|
|
36
36
|
return {
|
|
37
37
|
ok: false,
|
|
38
|
-
message: `scenario
|
|
38
|
+
message: `scenario must be a public-safe scenario id or a .yaml path inside the repo (got "${trimmed}").`,
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
41
|
const candidates = [
|
|
@@ -46,7 +46,7 @@ export async function resolveScriptedScenario(projectRoot, ref) {
|
|
|
46
46
|
if (!found) {
|
|
47
47
|
return {
|
|
48
48
|
ok: false,
|
|
49
|
-
message: `scenario
|
|
49
|
+
message: `scenario "${trimmed}" was not found (looked for ${candidates.join(", ")}).`,
|
|
50
50
|
};
|
|
51
51
|
}
|
|
52
52
|
source = found;
|
|
@@ -55,7 +55,7 @@ export async function resolveScriptedScenario(projectRoot, ref) {
|
|
|
55
55
|
const relativeScenarioPath = path.relative(projectRoot.physicalPath, absolutePath);
|
|
56
56
|
const scenarioBytes = await readContainedRegularFile(projectRoot, relativeScenarioPath);
|
|
57
57
|
if (!scenarioBytes) {
|
|
58
|
-
return { ok: false, message: `scenario
|
|
58
|
+
return { ok: false, message: `scenario "${trimmed}" could not be read (${source}).` };
|
|
59
59
|
}
|
|
60
60
|
const text = scenarioBytes.toString("utf8");
|
|
61
61
|
let raw;
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
// running).
|
|
4
4
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
5
5
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
6
|
+
import { judgedStatus } from "../../run/judge.js";
|
|
6
7
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
8
|
+
import { participantFactsOf } from "../computer-use/participant-facts.js";
|
|
7
9
|
import { publicSafeRouteLabel } from "./provenance.js";
|
|
8
10
|
import { participantTaxonomyLabel } from "./participant-specs.js";
|
|
9
11
|
function participantView(ctx, spec, index) {
|
|
@@ -88,6 +90,8 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
88
90
|
function sharedWorldStream(ctx, spec, index, view) {
|
|
89
91
|
const { args } = ctx;
|
|
90
92
|
const { taxonomy, session, screenshots, lastScreenshot, desktopGeometry, screenshotMode } = view;
|
|
93
|
+
// The judge's status for the participant, so the Observer shows it the way the verdict judged it.
|
|
94
|
+
const judged = view.outcome === undefined ? undefined : judgedStatus(participantFactsOf(view.outcome));
|
|
91
95
|
return participantStream(spec, {
|
|
92
96
|
...(spec.evidenceAssignment === undefined
|
|
93
97
|
? {}
|
|
@@ -95,6 +99,7 @@ function sharedWorldStream(ctx, spec, index, view) {
|
|
|
95
99
|
kind: "browser",
|
|
96
100
|
label: `Concurrent persona ${spec.planned.id}${taxonomy} · ${args.plan.studyId}`,
|
|
97
101
|
status: view.status,
|
|
102
|
+
...(judged === undefined ? {} : { judgedStatus: judged }),
|
|
98
103
|
transport: "snapshot",
|
|
99
104
|
updatedAt: args.run.createdAt,
|
|
100
105
|
embed: lastScreenshot
|
|
@@ -198,7 +203,7 @@ function sharedWorldEvents(ctx, spec, view) {
|
|
|
198
203
|
at: createdAt,
|
|
199
204
|
level: "info",
|
|
200
205
|
type: "concurrent-shared-world.contract.ready",
|
|
201
|
-
message: `Persona ${spec.planned.id}: dry-run actor ready; switch
|
|
206
|
+
message: `Persona ${spec.planned.id}: dry-run actor ready; switch mode to live for a real concurrent session.`,
|
|
202
207
|
});
|
|
203
208
|
}
|
|
204
209
|
return events;
|
|
@@ -9,6 +9,7 @@ import { commandDigestOf } from "../../subject/state.js";
|
|
|
9
9
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
10
10
|
import { participantFactsOf } from "../computer-use/participant-facts.js";
|
|
11
11
|
import { judgeSharedWorld, participantPassed, sharedWorldShortfall, verdictText, } from "../../run/judge.js";
|
|
12
|
+
import { reviewOutcome } from "../../run/display.js";
|
|
12
13
|
import { combineCheckpointDigest } from "./checkpoints.js";
|
|
13
14
|
import { hostOriginDigest } from "./provenance.js";
|
|
14
15
|
import { participantEvent, participantIds } from "../../run/participant-records.js";
|
|
@@ -433,6 +434,7 @@ export function renderConcurrentReviewMarkdown(bundle) {
|
|
|
433
434
|
`- topology: ${sw?.topology ?? "(none)"} / ${sw?.topologyMode ?? "(none)"}`,
|
|
434
435
|
`- personas: ${sw?.roleCount ?? 0}`,
|
|
435
436
|
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
437
|
+
`- outcome: ${reviewOutcome(bundle)}`,
|
|
436
438
|
`- summary: ${bundle.review.summary}`,
|
|
437
439
|
...(plane ? [`- plane: ${plane.message}`] : []),
|
|
438
440
|
...(concurrency ? [`- concurrency: ${concurrency.message}`] : []),
|
|
@@ -40,7 +40,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
40
40
|
const actorType = config.actors[0]?.type ?? "";
|
|
41
41
|
const descriptor = actorRegistry[actorType];
|
|
42
42
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
43
|
-
return refuse("HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED", `
|
|
43
|
+
return refuse("HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered computer-use actor.`);
|
|
44
44
|
const actor = descriptor.id;
|
|
45
45
|
// An app-url subject is the external-public plane (a real operator-owned deployment used
|
|
46
46
|
// directly, with no getHost, clone, subject sandbox or seed); every other subject is the
|
|
@@ -160,13 +160,13 @@ export function concurrentStudyFailure(envelope) {
|
|
|
160
160
|
});
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
163
|
-
* The run's execution failures: a run error (the handoff, the plane),
|
|
164
|
-
* session failed in the harness, each provider whose cleanup is unconfirmed
|
|
165
|
-
* disallowed item after its last request, each sandbox whose release is
|
|
166
|
-
* Observer that
|
|
163
|
+
* The run's execution failures before the Observer renders: a run error (the handoff, the plane),
|
|
164
|
+
* each participant whose session failed in the harness, each provider whose cleanup is unconfirmed
|
|
165
|
+
* or that reported a disallowed item after its last request, and each sandbox whose release is
|
|
166
|
+
* unconfirmed. FinishedRun.renderObserver adds an Observer that did not render.
|
|
167
167
|
*/
|
|
168
168
|
function sharedWorldExecutionFailures(args) {
|
|
169
|
-
const { runId, runError, actorResults, subject
|
|
169
|
+
const { runId, runError, actorResults, subject } = args;
|
|
170
170
|
return [
|
|
171
171
|
...(runError === undefined ? [] : [{ kind: "run", message: runError }]),
|
|
172
172
|
...actorResults
|
|
@@ -195,14 +195,6 @@ function sharedWorldExecutionFailures(args) {
|
|
|
195
195
|
...(subject.subjectSandboxId === undefined || subject.subjectKilled
|
|
196
196
|
? []
|
|
197
197
|
: [sandboxCleanupFailure("subject", subject.subjectReleaseWarning, runId)]),
|
|
198
|
-
...(observer.ok
|
|
199
|
-
? []
|
|
200
|
-
: [
|
|
201
|
-
{
|
|
202
|
-
kind: "evidence",
|
|
203
|
-
message: observer.error?.message ?? "Observer failed for the concurrent shared-world run.",
|
|
204
|
-
},
|
|
205
|
-
]),
|
|
206
198
|
];
|
|
207
199
|
}
|
|
208
200
|
/** Builds and publishes the bundle, renders the Observer and returns the study result. */
|
|
@@ -284,19 +276,28 @@ export async function finishConcurrentRun(ctx, live, results, plane) {
|
|
|
284
276
|
bundle.review = foldScorerFailures(bundle.review, scorerResult.failures);
|
|
285
277
|
if (receiving)
|
|
286
278
|
bundle.commsReceiving = receiving.snapshot();
|
|
287
|
-
const finished = await run.finish(bundle);
|
|
288
|
-
const observer = await finished.renderObserver();
|
|
289
|
-
if (observer.ok && live.observer) {
|
|
290
|
-
attachObserverRuntimeStreamUrls(observer, live.streamUrls);
|
|
291
|
-
}
|
|
292
279
|
// Concurrent "ok": every actor must produce a terminal, engaged, passed session, and the
|
|
293
280
|
// participants must show the concurrency verify requires of a pass (judgeSharedWorld). This is a
|
|
294
281
|
// harness/session-credibility gate, not mission-completion proof; a failed actor trace cannot
|
|
295
282
|
// make the route green just because the harness got a terminal.
|
|
296
|
-
const adapterFailure = adapterScoreFailureMessage(bundle);
|
|
297
283
|
const policy = OUTCOME_POLICIES["shared-world"];
|
|
298
|
-
const
|
|
299
|
-
const
|
|
284
|
+
const preRender = judgeExecution(sharedWorldExecutionFailures({ runId, runError, actorResults, subject: results }), policy);
|
|
285
|
+
const finished = await run.finish(bundle, {
|
|
286
|
+
ok: resultOk({
|
|
287
|
+
judgment,
|
|
288
|
+
execution: preRender,
|
|
289
|
+
scorerFailures: scorerResult.failures,
|
|
290
|
+
policy,
|
|
291
|
+
}),
|
|
292
|
+
execution: preRender,
|
|
293
|
+
policy,
|
|
294
|
+
});
|
|
295
|
+
const observer = await finished.renderObserver();
|
|
296
|
+
if (observer.ok && live.observer) {
|
|
297
|
+
attachObserverRuntimeStreamUrls(observer, live.streamUrls);
|
|
298
|
+
}
|
|
299
|
+
const adapterFailure = adapterScoreFailureMessage(bundle);
|
|
300
|
+
const { ok, execution } = finished.outcome;
|
|
300
301
|
const overlapProven = !dryRun && judgment.world.overlap;
|
|
301
302
|
const participantResults = concurrentParticipantResults(actorSpecs, actorResults, dryRun);
|
|
302
303
|
const errorResult = concurrentStudyError({
|
|
@@ -334,6 +335,5 @@ export async function finishConcurrentRun(ctx, live, results, plane) {
|
|
|
334
335
|
warnings: [...warnings, ...adapterWarnings, ...observer.warnings],
|
|
335
336
|
...(errorResult === undefined ? {} : { error: errorResult }),
|
|
336
337
|
};
|
|
337
|
-
await finished.recordOutcome({ ok, execution });
|
|
338
338
|
return result;
|
|
339
339
|
}
|
|
@@ -2,6 +2,7 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
2
2
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import { verdictText } from "../../run/judge.js";
|
|
5
|
+
import { reviewOutcome } from "../../run/display.js";
|
|
5
6
|
import {} from "../../run/streams.js";
|
|
6
7
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
7
8
|
import { TERMINAL_EVENTS_ARTIFACT, TERMINAL_LEDGERS_ARTIFACT, TERMINAL_TRANSCRIPT_ARTIFACT, } from "../../run/terminal-contract.js";
|
|
@@ -72,7 +73,7 @@ export function buildTerminalProductBundle(args) {
|
|
|
72
73
|
at: args.run.createdAt,
|
|
73
74
|
level: "info",
|
|
74
75
|
type: "terminal-lab.contract.ready",
|
|
75
|
-
message: "Dry-run bundle ready. Switch
|
|
76
|
+
message: "Dry-run bundle ready. Switch mode to live with the required runtime auth and caps to exercise the in-sandbox agent route, captured exec stream, and declared runtime-auth placement.",
|
|
76
77
|
}),
|
|
77
78
|
];
|
|
78
79
|
const review = {
|
|
@@ -172,7 +173,7 @@ export function buildLiveTerminalProductBundle(args) {
|
|
|
172
173
|
// so a green run never silently over-claims a fully-proven $0.
|
|
173
174
|
...(noSpend.unmeasuredLines.length > 0
|
|
174
175
|
? [
|
|
175
|
-
`${noSpendLineMeasured(noSpend) ? noSpendNotEstablished(noSpend.maxUsd ?? 0) : "No-spend proof is partial."} ${describeMeasuredSpend(args.ledgers.cost, args.trace.tokenUsage)}
|
|
176
|
+
`${noSpendLineMeasured(noSpend) ? noSpendNotEstablished(noSpend.maxUsd ?? 0) : "No-spend proof is partial."} ${describeMeasuredSpend(args.ledgers.cost, args.trace.tokenUsage)} humanish has no source for the missing lines, and a study cannot supply one.`,
|
|
176
177
|
]
|
|
177
178
|
: []),
|
|
178
179
|
],
|
|
@@ -283,6 +284,7 @@ export function renderTerminalReviewMarkdown(bundle) {
|
|
|
283
284
|
`- run: ${bundle.runId}`,
|
|
284
285
|
`- mode: ${bundle.mode}`,
|
|
285
286
|
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
287
|
+
`- outcome: ${reviewOutcome(bundle)}`,
|
|
286
288
|
`- summary: ${bundle.review.summary}`,
|
|
287
289
|
`- mission: ${bundle.scenario.goal}`,
|
|
288
290
|
...(subject ? [`- subject: ${subject.message}`] : []),
|
|
@@ -68,20 +68,16 @@ export async function runDryTerminalStudy(args) {
|
|
|
68
68
|
verdict: judgment.verdict,
|
|
69
69
|
});
|
|
70
70
|
bundle.events.push(runtimeDeclaredEvent(plan.runtime, createdAt));
|
|
71
|
-
const
|
|
71
|
+
const policy = OUTCOME_POLICIES.terminal;
|
|
72
|
+
const execution = judgeExecution([], policy);
|
|
73
|
+
const finished = await run.finish(bundle, {
|
|
74
|
+
ok: resultOk({ judgment, execution, scorerFailures: [], policy }),
|
|
75
|
+
execution,
|
|
76
|
+
policy,
|
|
77
|
+
});
|
|
72
78
|
const observer = await finished.renderObserver();
|
|
73
79
|
await validatePreparedRunArtifactPaths(finished.paths);
|
|
74
|
-
const
|
|
75
|
-
const execution = judgeExecution(observer.ok
|
|
76
|
-
? []
|
|
77
|
-
: [
|
|
78
|
-
{
|
|
79
|
-
kind: "evidence",
|
|
80
|
-
message: observer.error?.message ?? "Observer failed for the terminal-product run.",
|
|
81
|
-
},
|
|
82
|
-
], policy);
|
|
83
|
-
const ok = resultOk({ judgment, execution, scorerFailures: [], policy });
|
|
84
|
-
await finished.recordOutcome({ ok, execution });
|
|
80
|
+
const { ok } = finished.outcome;
|
|
85
81
|
return {
|
|
86
82
|
...studyResultIdentity("terminal", plan.studyId),
|
|
87
83
|
ok,
|