humanish 0.111.2 → 0.111.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +32 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/cli/findings.d.ts +15 -1
- package/dist/cli/findings.js +43 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +14 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +71 -35
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +8 -5
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +35 -0
- package/dist/run/secrets.js +158 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +89 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/docs/contracts/study-analysis.md +6 -2
- package/package.json +1 -1
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
import { pricedModel } from "../../study/plan-base.js";
|
|
2
|
-
import {
|
|
3
|
-
import { detectLocalAgents } from "../../actors/local-agent/cli.js";
|
|
4
|
-
import { localAgentRefusal } from "../../actors/local-agent/readiness.js";
|
|
5
|
-
import { describeMissingKeys } from "../../keys/key-resolution.js";
|
|
2
|
+
import { firstLiveRefusal, keysCheck, localAgentCheck, subjectEnvCheck, unpricedCapCheck, } from "../../study/requirements.js";
|
|
6
3
|
import { catchTokenOf, catchTokenRefusal } from "../../comms/external-evidence.js";
|
|
7
4
|
import { externalCatchHealthy } from "../../comms/sandbox-catch.js";
|
|
8
|
-
import { MODEL_RATES, unpricedCapMessage } from "../../run/pricing.js";
|
|
9
5
|
/** The computer-use code for each local-agent refusal; shared-world keeps the same kinds. */
|
|
10
6
|
const LOCAL_AGENT_REFUSAL_CODES = {
|
|
11
7
|
"agent-missing": "HUMANISH_COMPUTER_USE_AGENT_MISSING",
|
|
@@ -21,69 +17,41 @@ const LOCAL_AGENT_REFUSAL_CODES = {
|
|
|
21
17
|
*/
|
|
22
18
|
export async function liveCuaRejection(args) {
|
|
23
19
|
const { caps, brain, env, requirements, externalCommsConfig } = args;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
// right there, say so here rather than making them go and find an API key; that detour is
|
|
31
|
-
// where most people trying humanish stop.
|
|
32
|
-
const suggestion = missing.includes("OPENAI_API_KEY")
|
|
33
|
-
? await (async () => {
|
|
34
|
-
const ready = (await detectLocalAgents({ env })).filter((agent) => agent.authStatus === "authenticated");
|
|
35
|
-
return ready.length === 0
|
|
36
|
-
? ""
|
|
37
|
-
: ` ${ready.map((agent) => agent.label).join(" and ")} reports authenticated on this machine` +
|
|
38
|
-
`. Set actor.type: local-agent to use ${ready.length === 1 ? "it" : "one"} instead of a key.`;
|
|
39
|
-
})()
|
|
40
|
-
: "";
|
|
41
|
-
return {
|
|
20
|
+
return firstLiveRefusal([
|
|
21
|
+
// The plan lists OPENAI_API_KEY only for an openai brain (a signed-in local agent or the
|
|
22
|
+
// caller's provider needs none) and E2B_API_KEY only when this run creates hosted desktops.
|
|
23
|
+
() => keysCheck({
|
|
24
|
+
requirements,
|
|
25
|
+
env,
|
|
42
26
|
code: "HUMANISH_COMPUTER_USE_KEYS_MISSING",
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
const model = pricedModel(brain);
|
|
69
|
-
const capModelId = model.trim().toLowerCase();
|
|
70
|
-
if (!MODEL_RATES[capModelId]) {
|
|
71
|
-
return {
|
|
72
|
-
code: "HUMANISH_COMPUTER_USE_UNPRICED_CAP",
|
|
73
|
-
message: unpricedCapMessage(model),
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
// Adopter-hosted comms catch: fail closed before any sandbox is created, because a comms study
|
|
78
|
-
// whose catch is unreachable collects nothing while every participant still spends. The probe asserts
|
|
79
|
-
// humanish's own service marker in /health, so an adopter's proxy answering 200 for everything cannot
|
|
80
|
-
// pass for a catch.
|
|
81
|
-
const tokenRefusal = externalCommsConfig === undefined
|
|
82
|
-
? undefined
|
|
83
|
-
: catchTokenRefusal(catchTokenOf(externalCommsConfig, env));
|
|
27
|
+
need: (names) => `Live computer-use studies need ${names} in the environment (values are never persisted).`,
|
|
28
|
+
suggestLocalAgent: true,
|
|
29
|
+
}),
|
|
30
|
+
// A caller's createProvider makes the brain `caller`, so only the study's own local agent is
|
|
31
|
+
// checked.
|
|
32
|
+
() => localAgentCheck({ brain, env, caps, codes: LOCAL_AGENT_REFUSAL_CODES }),
|
|
33
|
+
() => subjectEnvCheck({ requirements, env, code: "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" }),
|
|
34
|
+
() => unpricedCapCheck({
|
|
35
|
+
caps,
|
|
36
|
+
model: pricedModel(brain),
|
|
37
|
+
code: "HUMANISH_COMPUTER_USE_UNPRICED_CAP",
|
|
38
|
+
}),
|
|
39
|
+
() => externalCatchCheck(externalCommsConfig, env),
|
|
40
|
+
]);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Adopter-hosted comms catch: refused before any sandbox is created, because a comms study whose
|
|
44
|
+
* catch is unreachable collects nothing while every participant still spends. The probe asserts
|
|
45
|
+
* humanish's own service marker in /health, so an adopter's proxy answering 200 for everything
|
|
46
|
+
* cannot pass for a catch.
|
|
47
|
+
*/
|
|
48
|
+
async function externalCatchCheck(externalCommsConfig, env) {
|
|
49
|
+
if (externalCommsConfig === undefined)
|
|
50
|
+
return undefined;
|
|
51
|
+
const tokenRefusal = catchTokenRefusal(catchTokenOf(externalCommsConfig, env));
|
|
84
52
|
if (tokenRefusal !== undefined)
|
|
85
53
|
return { code: "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID", message: tokenRefusal };
|
|
86
|
-
if (
|
|
54
|
+
if (!(await externalCatchHealthy(externalCommsConfig))) {
|
|
87
55
|
return {
|
|
88
56
|
code: "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE",
|
|
89
57
|
message: "The external comms catch or inbox is unreachable or incompatible (GET /health must identify humanish-comms-catch and advertise recipient-inbox-v1). Update humanish on the catch host and restart it with `humanish comms catch` on that host, or drop comms.email to run without the inbox funnel.",
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { foldScorerFailures, judgeExecution, OUTCOME_POLICIES, resultOk, } from "../../run/judge.js";
|
|
4
|
+
import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
5
|
+
import { participantExecutionFailures, participantOutcomeOk } from "./participant-facts.js";
|
|
6
|
+
import { participantCapWarning } from "./participant-model.js";
|
|
6
7
|
import { summarizeCuaDiagnostics } from "./diagnostics.js";
|
|
7
8
|
import { toParticipantResult } from "./participant-execution.js";
|
|
8
9
|
import { buildCuaRunBundle, judgeComputerUseRun } from "./bundle.js";
|
|
@@ -159,56 +160,10 @@ function cuaStudyResult(args) {
|
|
|
159
160
|
...(errorResult === undefined ? {} : { error: errorResult }),
|
|
160
161
|
};
|
|
161
162
|
}
|
|
162
|
-
/**
|
|
163
|
-
* The run's execution failures before the Observer renders: each participant whose session failed
|
|
164
|
-
* in the harness, each provider whose cleanup is unconfirmed or that reported a disallowed item
|
|
165
|
-
* after its last request, and each sandbox whose release is unconfirmed. FinishedRun.renderObserver
|
|
166
|
-
* adds an Observer that did not render.
|
|
167
|
-
*/
|
|
168
|
-
function computerUseExecutionFailures(outcomes, runId) {
|
|
169
|
-
return [
|
|
170
|
-
...(outcomes ?? [])
|
|
171
|
-
.filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
|
|
172
|
-
.map((outcome) => ({
|
|
173
|
-
kind: "harness",
|
|
174
|
-
message: `${outcome.spec.planned.id}: ${outcome.sessionError ?? outcome.session?.reason ?? "harness error"}`,
|
|
175
|
-
})),
|
|
176
|
-
...(outcomes ?? []).flatMap((outcome) => outcome.providerCleanupError === undefined
|
|
177
|
-
? []
|
|
178
|
-
: [
|
|
179
|
-
{
|
|
180
|
-
kind: "provider-cleanup",
|
|
181
|
-
message: `${outcome.spec.planned.id}: ${outcome.providerCleanupError}`,
|
|
182
|
-
},
|
|
183
|
-
]),
|
|
184
|
-
...(outcomes ?? []).flatMap((outcome) => outcome.providerPolicyError === undefined
|
|
185
|
-
? []
|
|
186
|
-
: [
|
|
187
|
-
{
|
|
188
|
-
kind: "provider-policy",
|
|
189
|
-
message: `${outcome.spec.planned.id}: ${outcome.providerPolicyError}`,
|
|
190
|
-
},
|
|
191
|
-
]),
|
|
192
|
-
...unreleasedSandboxFailures(outcomes, runId),
|
|
193
|
-
];
|
|
194
|
-
}
|
|
195
|
-
/**
|
|
196
|
-
* caps.maxUsd is enforced inside each participant's loop independently, so an
|
|
197
|
-
* N-participant fan-out can spend up to N × maxUsd before any participant aborts, while the run cost summary reports the larger
|
|
198
|
-
* aggregate. The warning names that ceiling, unless the study declared a shared maxTotalUsd budget.
|
|
199
|
-
*/
|
|
200
|
-
function participantCapWarning(caps, participantCount) {
|
|
201
|
-
const capUsd = caps.maxUsd;
|
|
202
|
-
if (capUsd === undefined || participantCount <= 1)
|
|
203
|
-
return undefined;
|
|
204
|
-
if (caps.maxTotalUsd !== undefined)
|
|
205
|
-
return undefined;
|
|
206
|
-
return `caps.maxUsd ($${capUsd}) caps each participant, so ${participantCount} participants may spend up to ${participantCount} × $${capUsd} (about $${round6(capUsd * participantCount)}) before any of them stops. Set caps.maxTotalUsd for one budget across the study.`;
|
|
207
|
-
}
|
|
208
163
|
/** Builds and publishes the final bundle, runs the adapter hooks, renders the Observer and returns the result. */
|
|
209
164
|
export async function finishCuaRun(setup, finish, ran) {
|
|
210
165
|
const { plan, input, cwd, streams, descriptor, run } = setup;
|
|
211
|
-
const { participantRuns, participantPlan,
|
|
166
|
+
const { participantRuns, participantPlan, bundleBase } = setup;
|
|
212
167
|
const { rerunLineage, publicRepo, subjectArgs } = finish;
|
|
213
168
|
const { dryRun } = plan;
|
|
214
169
|
const appUrl = plannedAppUrl(plan.runner.subject);
|
|
@@ -252,7 +207,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
252
207
|
dryRun,
|
|
253
208
|
participantCount,
|
|
254
209
|
},
|
|
255
|
-
sanitize: (text) => redactText(
|
|
210
|
+
sanitize: (text) => redactText(run.secrets.scrub(text)),
|
|
256
211
|
warnings: adapterWarnings,
|
|
257
212
|
...(input.scorerProvenance === undefined ? {} : { scorerProvenance: input.scorerProvenance }),
|
|
258
213
|
});
|
|
@@ -261,13 +216,19 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
261
216
|
if (receiving)
|
|
262
217
|
bundle.commsReceiving = receiving.snapshot();
|
|
263
218
|
const policy = OUTCOME_POLICIES["computer-use"];
|
|
264
|
-
|
|
219
|
+
// FinishedRun.renderObserver adds an Observer that did not render.
|
|
220
|
+
const execution = judgeExecution(participantExecutionFailures(outcomes, runId), policy);
|
|
265
221
|
const finished = await run.finish(bundle, {
|
|
266
222
|
ok: resultOk({ judgment, execution, scorerFailures: scorerResult.failures, policy }),
|
|
267
223
|
execution,
|
|
268
224
|
policy,
|
|
269
225
|
});
|
|
270
226
|
const observer = await finished.renderObserver();
|
|
227
|
+
// A caller's renderer runs with this process's file access, so after one that reports success
|
|
228
|
+
// the run directory is checked again before the result points anyone at it. A failed render is
|
|
229
|
+
// already an evidence failure, and its error code stays the run's diagnosis.
|
|
230
|
+
if (observer.ok)
|
|
231
|
+
await validatePreparedRunArtifactPaths(finished.paths);
|
|
271
232
|
streams.attachFinal(observer);
|
|
272
233
|
return cuaStudyResult({
|
|
273
234
|
studyId: plan.studyId,
|
|
@@ -31,7 +31,7 @@ import { finishCuaRun } from "./result.js";
|
|
|
31
31
|
import { runStudyParticipants } from "./live-phase.js";
|
|
32
32
|
import { admitCuaRun, refuseCuaStudy, startCuaRun } from "./setup.js";
|
|
33
33
|
import {} from "./types.js";
|
|
34
|
-
import {
|
|
34
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
35
35
|
/**
|
|
36
36
|
* A refused computer-use study's result, at the refusal's stage: a before-scope refusal has its own
|
|
37
37
|
* envelope and no analysis record; the others come after the cwd checks, and the participant cap after
|
|
@@ -40,18 +40,17 @@ import { studyResultIdentity } from "../../run/study-result.js";
|
|
|
40
40
|
export async function computerUseStudyRefusal(options, refusal) {
|
|
41
41
|
const { config, dryRun } = options;
|
|
42
42
|
if (refusal.stage === "before-scope")
|
|
43
|
-
return {
|
|
44
|
-
|
|
45
|
-
ok: false,
|
|
43
|
+
return refusedResult("computer-use", {
|
|
44
|
+
studyId: config.id,
|
|
46
45
|
cwd: path.resolve(options.cwd),
|
|
46
|
+
error: { code: refusal.code, message: refusal.message },
|
|
47
|
+
}, {
|
|
47
48
|
actor: config.actor?.type ?? "",
|
|
48
49
|
dryRun,
|
|
49
50
|
runId: options.runId ?? "not-created",
|
|
50
51
|
appUrl: config.subject.appUrl ?? config.subject.serve?.url ?? "",
|
|
51
52
|
lanes: [],
|
|
52
|
-
|
|
53
|
-
error: { code: refusal.code, message: refusal.message },
|
|
54
|
-
};
|
|
53
|
+
});
|
|
55
54
|
// The other refusals come after the cwd checks, and the participant cap after the personas are read.
|
|
56
55
|
return completeRefusalAnalysis(await refuseCuaStudy(options, refusal), config, options);
|
|
57
56
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type RunScope } from "../../run/run.js";
|
|
2
2
|
import type { RefusedStudy } from "../../run/route-shell.js";
|
|
3
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
3
4
|
import type { CuaActorDescriptor } from "../../actors/registry.js";
|
|
4
5
|
import type { StudyCommsEmail, StudyCommsExternal, StudyConfig } from "../../study/types.js";
|
|
5
6
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
@@ -32,15 +33,11 @@ export interface CuaRunSetup {
|
|
|
32
33
|
streams: ReturnType<typeof trackRuntimeStreams>;
|
|
33
34
|
participantRuns: DesktopParticipantRun[];
|
|
34
35
|
participantPlan: CuaParticipantPlan;
|
|
35
|
-
scrubKnownValues: (text: string) => string;
|
|
36
36
|
bundleBase: CuaRunBundleBase;
|
|
37
37
|
}
|
|
38
38
|
/** What only runStudyParticipants reads. */
|
|
39
39
|
export interface CuaParticipantsSetup {
|
|
40
40
|
env: Record<string, string | undefined>;
|
|
41
|
-
/** The array scrubKnownValues reads at each call. Email receiving appends the values it
|
|
42
|
-
* provisions before the participants run. */
|
|
43
|
-
knownSecretValues: string[];
|
|
44
41
|
deps: Omit<CuaParticipantDeps, "signalProvisioned">;
|
|
45
42
|
/** Filled by runStudyParticipants on a live run; deps.onTrace reads it. */
|
|
46
43
|
liveTrace: {
|
|
@@ -116,8 +113,7 @@ export declare function admitCuaRun(plan: ComputerUsePlan, input: ComputerUseRun
|
|
|
116
113
|
participantCount: number;
|
|
117
114
|
openaiApiKey: string;
|
|
118
115
|
e2bApiKey: string;
|
|
119
|
-
|
|
120
|
-
scrubKnownValues: (text: string) => string;
|
|
116
|
+
secrets: RunSecrets;
|
|
121
117
|
publicRepo: string | undefined;
|
|
122
118
|
localTreeArchive: LocalTreeArchive | undefined;
|
|
123
119
|
localTreeArchiveBuffer: ArrayBuffer | undefined;
|
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
// The computer-use run's setup: resolve the project and route, plan the participants, make the
|
|
2
2
|
// live checks that need no sandbox, pack a local tree, then start the run and build the
|
|
3
3
|
// participant deps and bundle base that the participants and the finish share.
|
|
4
|
-
import { randomBytes } from "node:crypto";
|
|
5
4
|
import { realpath } from "node:fs/promises";
|
|
6
5
|
import path from "node:path";
|
|
7
6
|
import {} from "../../run/run.js";
|
|
8
7
|
import { externalInboxUrl } from "../../comms/sandbox-catch.js";
|
|
9
|
-
import { redactText,
|
|
10
|
-
import {
|
|
8
|
+
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
9
|
+
import { RunSecrets } from "../../run/secrets.js";
|
|
10
|
+
import {} from "../../run/bundle.js";
|
|
11
11
|
import { assertPreparedSelectedOutputDirectory, prepareContainedOutputDirectory, prepareSelectedOutputDirectory, } from "../../run/contained-output.js";
|
|
12
12
|
import {} from "../../subject/local-tree-archive.js";
|
|
13
13
|
import { renderCuaReviewMarkdown } from "./bundle.js";
|
|
14
14
|
import { emitPreflightPlan, compileParticipantPersonas, loadCuaParticipants, sanitizeParticipantRuns, } from "./participant-runs.js";
|
|
15
|
-
import {
|
|
16
|
-
import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
|
|
15
|
+
import { desktopParticipantDeps } from "./participant-deps.js";
|
|
17
16
|
import { liveCuaRejection } from "./preflight.js";
|
|
18
17
|
import { cuaDescriptorOf, declaredAppUrl, plannedAppUrl } from "./plan.js";
|
|
19
18
|
import { trackRuntimeStreams } from "./live-flush.js";
|
|
@@ -24,7 +23,7 @@ import { packRunLocalTree } from "./local-tree-pack.js";
|
|
|
24
23
|
import { projectParticipantSubjects, subjectProvenanceArg } from "./subject-projection.js";
|
|
25
24
|
import { participantSubjectEnv, } from "./types.js";
|
|
26
25
|
import { studyPersonaIds } from "../../study/persona-resolve.js";
|
|
27
|
-
import {
|
|
26
|
+
import { refusedResult } from "../../run/study-result.js";
|
|
28
27
|
/** The physical project, bound before any caller hook runs. */
|
|
29
28
|
async function bindProject(cwd) {
|
|
30
29
|
// Capture the physical project before reading or invoking any caller hook. A supported
|
|
@@ -43,18 +42,17 @@ export async function refuseCuaStudy(options, refusal) {
|
|
|
43
42
|
// The committed personas are read before the refusal returns, so a persona-file error wins.
|
|
44
43
|
if (refusal.stage === "after-personas")
|
|
45
44
|
await compileParticipantPersonas(projectRoot, studyPersonaIds(config));
|
|
46
|
-
return {
|
|
47
|
-
|
|
48
|
-
ok: false,
|
|
45
|
+
return refusedResult("computer-use", {
|
|
46
|
+
studyId: config.id,
|
|
49
47
|
cwd: projectRoot.physicalPath,
|
|
48
|
+
error: { code: refusal.code, message: refusal.message },
|
|
49
|
+
}, {
|
|
50
50
|
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
51
51
|
appUrl: declaredAppUrl(config),
|
|
52
52
|
dryRun,
|
|
53
53
|
runId: options.runId ?? "not-created",
|
|
54
54
|
lanes: [],
|
|
55
|
-
|
|
56
|
-
error: { code: refusal.code, message: refusal.message },
|
|
57
|
-
};
|
|
55
|
+
});
|
|
58
56
|
}
|
|
59
57
|
/**
|
|
60
58
|
* Everything before the run starts: the physical project, the route, the participant plan and
|
|
@@ -73,17 +71,12 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
73
71
|
const inProcess = desktop === "in-process";
|
|
74
72
|
const appUrl = plannedAppUrl(subject);
|
|
75
73
|
const subjectEnvNames = [...participantSubjectEnv(subject)];
|
|
76
|
-
const fail = (code, message, actorLabel) => ({
|
|
77
|
-
...studyResultIdentity("computer-use", plan.studyId),
|
|
78
|
-
ok: false,
|
|
79
|
-
cwd,
|
|
74
|
+
const fail = (code, message, actorLabel) => refusedResult("computer-use", { studyId: plan.studyId, cwd, error: { code, message } }, {
|
|
80
75
|
actor: actorLabel ?? plan.actor,
|
|
81
76
|
appUrl,
|
|
82
77
|
dryRun,
|
|
83
78
|
runId: input.runId ?? "not-created",
|
|
84
79
|
lanes: [],
|
|
85
|
-
warnings: [],
|
|
86
|
-
error: { code, message },
|
|
87
80
|
});
|
|
88
81
|
const refuse = (...args) => ({
|
|
89
82
|
ok: false,
|
|
@@ -131,13 +124,13 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
131
124
|
const openaiApiKey = env.OPENAI_API_KEY?.trim() ?? "";
|
|
132
125
|
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
133
126
|
// Literal scrubber for every known provisioned value (no secret "shape" to pattern-match).
|
|
134
|
-
|
|
127
|
+
// Dry and in-process runs scrub both keys too: a mission or persona can still quote them.
|
|
128
|
+
const secrets = new RunSecrets([
|
|
135
129
|
openaiApiKey,
|
|
136
130
|
e2bApiKey,
|
|
137
131
|
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
138
|
-
]
|
|
139
|
-
|
|
140
|
-
sanitizeParticipantRuns(participantRuns, scrubKnownValues);
|
|
132
|
+
]);
|
|
133
|
+
sanitizeParticipantRuns(participantRuns, secrets.scrub);
|
|
141
134
|
const redactRepoLabel = plan.residual.policies?.redactRepos ?? subjectEnvNames.includes("GITHUB_TOKEN");
|
|
142
135
|
const publicRepo = subject.kind === "clone" ? (redactRepoLabel ? "repo-01" : subject.repo) : undefined;
|
|
143
136
|
// Key-gating is route-aware: the in-process route uses the caller's own model + executor, and
|
|
@@ -167,7 +160,7 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
167
160
|
localTreeArchiveBuffer = packed.buffer;
|
|
168
161
|
}
|
|
169
162
|
catch (error) {
|
|
170
|
-
return refuse("HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `local-tree packing failed: ${redactText(
|
|
163
|
+
return refuse("HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `local-tree packing failed: ${redactText(secrets.scrub(toErrorMessage(error)))}`, descriptor.id);
|
|
171
164
|
}
|
|
172
165
|
}
|
|
173
166
|
return {
|
|
@@ -192,8 +185,7 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
192
185
|
participantCount,
|
|
193
186
|
openaiApiKey,
|
|
194
187
|
e2bApiKey,
|
|
195
|
-
|
|
196
|
-
scrubKnownValues,
|
|
188
|
+
secrets,
|
|
197
189
|
publicRepo,
|
|
198
190
|
localTreeArchive,
|
|
199
191
|
localTreeArchiveBuffer,
|
|
@@ -202,33 +194,23 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
202
194
|
}
|
|
203
195
|
/** Starts the run and builds what the participants and the finish read: the deps and the bundle base. */
|
|
204
196
|
export async function startCuaRun(plan, input, admitted, scope) {
|
|
205
|
-
const {
|
|
197
|
+
const { cwd, descriptor, participantRuns, participantPlan, publicRepo } = admitted;
|
|
206
198
|
const { appUrl } = admitted;
|
|
207
199
|
// The run's status record exists from here on, so anything watching the runs directory can
|
|
208
200
|
// tell which study this is and that it is alive. The fail-closed returns below leave it finished
|
|
209
201
|
// with no outcome when the scope closes; a crash leaves it stale, which reads as interrupted.
|
|
210
|
-
const started = await scope.startRun({
|
|
202
|
+
const started = await scope.startRun(plan, input, {
|
|
211
203
|
cwd,
|
|
212
|
-
|
|
213
|
-
mintRunId: makeCuaRunId,
|
|
214
|
-
mode: dryRun ? "dry-run" : "live",
|
|
215
|
-
study: plan.study,
|
|
216
|
-
warnings: plan.warnings,
|
|
204
|
+
prefix: "cua",
|
|
217
205
|
renderReview: renderCuaReviewMarkdown,
|
|
218
|
-
|
|
206
|
+
secrets: admitted.secrets,
|
|
219
207
|
});
|
|
220
208
|
if (!started.ok)
|
|
221
209
|
return admitted.refuse(started.code, started.message, descriptor.id);
|
|
222
210
|
const { run } = started;
|
|
223
|
-
const {
|
|
211
|
+
const { paths: runPaths, source } = run;
|
|
224
212
|
const redactScreenshots = plan.residual.policies?.redactScreenshots === true;
|
|
225
213
|
await prepareContainedOutputDirectory(runPaths, "screenshots");
|
|
226
|
-
const source = await buildRunSource({
|
|
227
|
-
capturedAt: createdAt,
|
|
228
|
-
cwd,
|
|
229
|
-
humanishSource: "present",
|
|
230
|
-
packageName: "humanish",
|
|
231
|
-
});
|
|
232
214
|
// Live-trace flush seam: runStudyParticipants fills it when a live run has an in-progress bundle
|
|
233
215
|
// to grow; participants call it through deps.onTrace. It exists before deps so deps can reference it as
|
|
234
216
|
// a stable indirection.
|
|
@@ -279,12 +261,10 @@ export async function startCuaRun(plan, input, admitted, scope) {
|
|
|
279
261
|
streams: admitted.streams,
|
|
280
262
|
participantRuns,
|
|
281
263
|
participantPlan,
|
|
282
|
-
scrubKnownValues: admitted.scrubKnownValues,
|
|
283
264
|
bundleBase,
|
|
284
265
|
},
|
|
285
266
|
participants: {
|
|
286
267
|
env: admitted.env,
|
|
287
|
-
knownSecretValues: admitted.knownSecretValues,
|
|
288
268
|
deps,
|
|
289
269
|
liveTrace,
|
|
290
270
|
externalComms: externalCommsConfig === undefined || externalCommsEmail === undefined
|
|
@@ -300,60 +280,38 @@ export async function startCuaRun(plan, input, admitted, scope) {
|
|
|
300
280
|
finish: { rerunLineage: admitted.rerunLineage, publicRepo, subjectArgs },
|
|
301
281
|
};
|
|
302
282
|
}
|
|
303
|
-
/** The participant deps every participant reads
|
|
283
|
+
/** The participant deps every participant reads, with computer use's own overrides. */
|
|
304
284
|
function cuaParticipantDeps(plan, input, admitted, run) {
|
|
305
|
-
const { config,
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
const { externalCommsConfig, externalCommsEmail } = admitted;
|
|
309
|
-
const { appUrl } = admitted;
|
|
310
|
-
const { runPaths, redactScreenshots, liveTrace } = run;
|
|
311
|
-
const createDesktop = input.localVm?.desktop;
|
|
312
|
-
const timeoutMs = plan.sessionBudgetMs;
|
|
313
|
-
const requestTimeoutMs = e2bRequestTimeoutMs(env);
|
|
314
|
-
return {
|
|
315
|
-
...(createDesktop === undefined ? {} : { createDesktop }),
|
|
316
|
-
onTrace: (participantId, items, usage, metadata) => liveTrace.flush?.(participantId, items, usage, metadata),
|
|
317
|
-
...callerDriving(input, config),
|
|
318
|
-
residual: plan.residual,
|
|
319
|
-
studyId: plan.studyId,
|
|
320
|
-
caps: plan.caps,
|
|
321
|
-
appUrl,
|
|
285
|
+
const { config, seams, env, externalCommsConfig, externalCommsEmail } = admitted;
|
|
286
|
+
const deps = desktopParticipantDeps({
|
|
287
|
+
plan,
|
|
322
288
|
brain: plan.runner.brain,
|
|
323
289
|
subject: plan.runner.subject,
|
|
324
|
-
...(localTreeArchiveBuffer === undefined ? {} : { localTreeArchiveBuffer }),
|
|
325
290
|
env,
|
|
326
|
-
openaiApiKey,
|
|
327
|
-
e2bApiKey,
|
|
328
|
-
|
|
291
|
+
openaiApiKey: admitted.openaiApiKey,
|
|
292
|
+
e2bApiKey: admitted.e2bApiKey,
|
|
293
|
+
timeoutMs: plan.sessionBudgetMs,
|
|
329
294
|
sandboxMs: plan.sandboxMs,
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
artifactRoot: runPaths,
|
|
295
|
+
participantCount: admitted.participantCount,
|
|
296
|
+
artifactRoot: run.runPaths,
|
|
333
297
|
studyCwd: input.cwd,
|
|
334
|
-
redactScreenshots,
|
|
335
|
-
scrubKnownValues,
|
|
336
|
-
runSession,
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
...(dryRun || plan.caps.maxTotalUsd === undefined
|
|
340
|
-
? {}
|
|
341
|
-
: { runBudget: makeCuaRunBudget(plan.caps.maxTotalUsd) }),
|
|
342
|
-
...(externalCommsConfig === undefined || externalCommsEmail === undefined
|
|
343
|
-
? {}
|
|
344
|
-
: {
|
|
345
|
-
externalComms: {
|
|
346
|
-
email: externalCommsEmail,
|
|
347
|
-
inboxUrl: externalInboxUrl(externalCommsConfig),
|
|
348
|
-
},
|
|
349
|
-
}),
|
|
350
|
-
now: seams.now ?? Date.now,
|
|
351
|
-
...(seams.desktopModule === undefined ? {} : { desktopModule: seams.desktopModule }),
|
|
352
|
-
...(input.prepareDesktop === undefined ? {} : { prepareDesktop: input.prepareDesktop }),
|
|
353
|
-
...(seams.detachedTimers === undefined ? {} : { detachedTimers: seams.detachedTimers }),
|
|
354
|
-
onStream: streams.onStream,
|
|
298
|
+
redactScreenshots: run.redactScreenshots,
|
|
299
|
+
scrubKnownValues: admitted.secrets.scrub,
|
|
300
|
+
runSession: admitted.runSession,
|
|
301
|
+
liveFlush: () => run.liveTrace.flush,
|
|
302
|
+
onStream: admitted.streams.onStream,
|
|
355
303
|
reportSubjectPhase: subjectPhaseReporter(input),
|
|
356
|
-
|
|
304
|
+
seams,
|
|
305
|
+
prepareDesktop: input.prepareDesktop,
|
|
306
|
+
}, {
|
|
307
|
+
createDesktop: input.localVm?.desktop,
|
|
308
|
+
...callerDriving(input, config),
|
|
309
|
+
localTreeArchiveBuffer: admitted.localTreeArchiveBuffer,
|
|
310
|
+
externalComms: externalCommsConfig === undefined || externalCommsEmail === undefined
|
|
311
|
+
? undefined
|
|
312
|
+
: { email: externalCommsEmail, inboxUrl: externalInboxUrl(externalCommsConfig) },
|
|
313
|
+
});
|
|
314
|
+
return { ...deps, appUrl: admitted.appUrl };
|
|
357
315
|
}
|
|
358
316
|
/** The caller's createProvider and inProcess executor, with the run's config bound. */
|
|
359
317
|
function callerDriving(input, config) {
|
|
@@ -377,7 +335,3 @@ function subjectPhaseReporter(input) {
|
|
|
377
335
|
input.emit?.(phaseEvent(event, { kind: "participant", participant }));
|
|
378
336
|
};
|
|
379
337
|
}
|
|
380
|
-
function makeCuaRunId() {
|
|
381
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
382
|
-
return `cua-${stamp}-${randomBytes(4).toString("hex")}`;
|
|
383
|
-
}
|
|
@@ -3,7 +3,8 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/streams.js";
|
|
5
5
|
import { aggregateTaskFunnels, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
6
|
-
import { describeSubjectState,
|
|
6
|
+
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
7
|
+
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
7
8
|
import { participantStatus as participantStatusFor } from "../../run/judge.js";
|
|
8
9
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
9
10
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
@@ -359,13 +360,13 @@ export function buildSingleParticipantBundle(args) {
|
|
|
359
360
|
runId: args.run.runId,
|
|
360
361
|
scenarioId: `cua-${args.studyId}`,
|
|
361
362
|
adapterId: args.studyId,
|
|
362
|
-
goal: redactText(args.mission),
|
|
363
363
|
substrate: runnerSubstrate(args),
|
|
364
364
|
participants: [
|
|
365
365
|
{
|
|
366
366
|
participantId: args.participantId ?? "lane-01",
|
|
367
367
|
streamId: SINGLE.streamId,
|
|
368
368
|
personaId: args.persona.id,
|
|
369
|
+
goal: redactText(args.mission),
|
|
369
370
|
...(args.session === undefined ? {} : { session: args.session }),
|
|
370
371
|
...(args.traceArtifactPath === undefined
|
|
371
372
|
? {}
|
|
@@ -26,7 +26,7 @@ import { type RunDesktopGeometry } from "../../run/streams.js";
|
|
|
26
26
|
import { type PreparedOutputRoot } from "../../run/contained-output.js";
|
|
27
27
|
import type { ComputerUseParticipant, SharedWorldParticipant } from "../../study/plan-participants.js";
|
|
28
28
|
import type { ResolvedParticipant } from "../../run/participant.js";
|
|
29
|
-
import { type StudyResultIdentity } from "../../run/study-result.js";
|
|
29
|
+
import { type StudyResultErrorCode, type StudyResultIdentity } from "../../run/study-result.js";
|
|
30
30
|
export declare const CUA_FANOUT_STRATEGY: "per-lane-worlds";
|
|
31
31
|
export declare const CUA_MAX_CONCURRENCY_ENV = "HUMANISH_CUA_MAX_CONCURRENCY";
|
|
32
32
|
export declare const DEFAULT_APP_URL_SESSION_TIMEOUT_MS: number;
|
|
@@ -182,7 +182,7 @@ export interface CuaParticipantSummary {
|
|
|
182
182
|
concurrency: number;
|
|
183
183
|
waves: number;
|
|
184
184
|
}
|
|
185
|
-
export type CuaActorStudyErrorCode =
|
|
185
|
+
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
186
186
|
/** Subject provenance projection: what the actor actually drove. */
|
|
187
187
|
export interface CuaSubjectProjection {
|
|
188
188
|
source: "app-url" | "clone" | "local-tree";
|
package/dist/routes/preview.js
CHANGED
|
@@ -1,17 +1,11 @@
|
|
|
1
1
|
// The preview route: a study whose subject is this repo runs the synthetic dry run. planStudy decides
|
|
2
2
|
// the sim count and refuses what a synthetic run would ignore; runPreviewPlan does the rest.
|
|
3
|
-
import { studyResultIdentity } from "../run/study-result.js";
|
|
3
|
+
import { refusedResult, studyResultIdentity, } from "../run/study-result.js";
|
|
4
4
|
import { runDryRun } from "../run/dry-run.js";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
/** A refused preview's result. */
|
|
7
7
|
export function previewStudyRefusal(cwd, studyId, refusal) {
|
|
8
|
-
return {
|
|
9
|
-
...studyResultIdentity("preview", studyId),
|
|
10
|
-
ok: false,
|
|
11
|
-
cwd: path.resolve(cwd),
|
|
12
|
-
warnings: [],
|
|
13
|
-
error: { code: refusal.code, message: refusal.message },
|
|
14
|
-
};
|
|
8
|
+
return refusedResult("preview", { studyId, cwd: path.resolve(cwd), error: { code: refusal.code, message: refusal.message } }, {});
|
|
15
9
|
}
|
|
16
10
|
/** runStudyWith's step for a preview plan: it has no local checks and takes no scorer, so it returns its run. */
|
|
17
11
|
export function admitPreviewPlan(plan, input) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ActorPersonaRef } from "../../actors/contract.js";
|
|
2
2
|
import type { ScriptedBrowserSessionResult } from "../../actors/scripted-browser/actor.js";
|
|
3
3
|
import type { BrowserPersonaJourney, BrowserSurface } from "../../actors/scripted-browser/types.js";
|
|
4
|
-
import type {
|
|
4
|
+
import type { SubjectDesktopUsage } from "../../substrates/e2b/subject-sandbox.js";
|
|
5
5
|
import { type ReviewSummary, type RunBundle, type RunSubjectProvenance, type BundleRun } from "../../run/bundle.js";
|
|
6
6
|
/** What the scripted study's bundle is built from. */
|
|
7
7
|
interface ScriptedBundleArgs {
|
|
@@ -24,11 +24,7 @@ interface ScriptedBundleArgs {
|
|
|
24
24
|
source: RunBundle["source"];
|
|
25
25
|
subject?: RunSubjectProvenance;
|
|
26
26
|
/** The provisioned clone's desktop; absent when no sandbox was created. */
|
|
27
|
-
subjectDesktop?:
|
|
28
|
-
durationMs: number | undefined;
|
|
29
|
-
observation: DesktopResourceObservation | undefined;
|
|
30
|
-
killed: boolean;
|
|
31
|
-
};
|
|
27
|
+
subjectDesktop?: SubjectDesktopUsage;
|
|
32
28
|
surfaces: readonly BrowserSurface[];
|
|
33
29
|
/** The run's verdict (judgeScripted). */
|
|
34
30
|
verdict: ReviewSummary["verdict"];
|