humanish 0.111.4 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +71 -22
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
// Assemble the scripted-browser study's run bundle from its sessions, and the review summary and
|
|
2
2
|
// markdown that ship with it.
|
|
3
|
-
import {
|
|
4
|
-
import { reviewOutcome } from "../../run/display.js";
|
|
3
|
+
import { renderReviewMarkdown } from "../../run/review-markdown.js";
|
|
5
4
|
import { redactText } from "../../evidence/redaction.js";
|
|
6
5
|
import { buildRunCostSummary, desktopSpanToMinutes, spendFreeCostSummary, } from "../../run/cost-summary.js";
|
|
7
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
8
7
|
import {} from "../../run/streams.js";
|
|
9
8
|
import { participantEvent, recordIdOf } from "../../run/participant-records.js";
|
|
10
9
|
import { scriptedSurfaceIds, scriptedSurfaceRecords } from "./surface-records.js";
|
|
10
|
+
import { gapsListClause } from "../../run/outcomes.js";
|
|
11
11
|
import { plural } from "../../run/text.js";
|
|
12
12
|
/**
|
|
13
13
|
* Project the scripted run into a humanish.run-bundle.v1 (no schema change: a new
|
|
@@ -146,7 +146,7 @@ function buildScriptedReview(args) {
|
|
|
146
146
|
return {
|
|
147
147
|
schema: REVIEW_SCHEMA,
|
|
148
148
|
verdict: args.verdict,
|
|
149
|
-
summary: `The
|
|
149
|
+
summary: `The run failed before the script finished: ${args.sessionError}`,
|
|
150
150
|
gaps: [],
|
|
151
151
|
};
|
|
152
152
|
}
|
|
@@ -154,41 +154,31 @@ function buildScriptedReview(args) {
|
|
|
154
154
|
return {
|
|
155
155
|
schema: REVIEW_SCHEMA,
|
|
156
156
|
verdict: args.verdict,
|
|
157
|
-
summary: `Dry run
|
|
157
|
+
summary: `Dry run: scenario ${args.journey.scenarioId} (${args.scenarioSource}, ${args.journey.steps.length} steps) would replay against ${args.appUrl} on ${plural(args.surfaces.length, "browser surface")}. No browser ran and $0 was spent.`,
|
|
158
158
|
gaps: ["Live scripted session not yet run (dry run only)."],
|
|
159
159
|
};
|
|
160
160
|
}
|
|
161
161
|
const passed = args.sessionResults.filter((result) => result.status === "passed").length;
|
|
162
|
+
const ran = args.sessionResults.length;
|
|
162
163
|
return {
|
|
163
164
|
schema: REVIEW_SCHEMA,
|
|
164
165
|
verdict: args.verdict,
|
|
165
|
-
summary: `
|
|
166
|
+
summary: `Scenario ${args.journey.scenarioId} ran on ${plural(ran, "browser surface")} against ${args.appUrl}: ${passed}/${ran} passed it${gapsListClause(ran - passed, ran)}.`,
|
|
166
167
|
gaps: args.sessionResults
|
|
167
168
|
.filter((result) => result.status !== "passed")
|
|
168
169
|
.map((result) => `${result.capture.surface.id}: ${result.reason}`),
|
|
169
170
|
};
|
|
170
171
|
}
|
|
171
|
-
export function renderScriptedReviewMarkdown(bundle) {
|
|
172
|
+
export function renderScriptedReviewMarkdown(bundle, status) {
|
|
172
173
|
const subject = bundle.events.find((event) => event.type === "scripted-lab.subject.declared");
|
|
173
174
|
const spend = bundle.events.find((event) => event.type === "scripted-lab.spend");
|
|
174
175
|
const traces = bundle.streams
|
|
175
176
|
.map((stream) => ({ stream, trace: stream.actor }))
|
|
176
177
|
.filter((entry) => entry.trace !== undefined);
|
|
177
|
-
return [
|
|
178
|
-
`# ${bundle.scenario.title}`,
|
|
179
|
-
"",
|
|
180
|
-
`- run: ${bundle.runId}`,
|
|
181
|
-
`- mode: ${bundle.mode}`,
|
|
182
|
-
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
183
|
-
`- outcome: ${reviewOutcome(bundle)}`,
|
|
184
|
-
`- summary: ${bundle.review.summary}`,
|
|
178
|
+
return renderReviewMarkdown(bundle, [
|
|
185
179
|
`- scenario: ${bundle.scenario.id} @ ${bundle.scenario.sourceDigest} (${bundle.scenario.source})`,
|
|
186
180
|
...(subject ? [`- subject: ${subject.message}`] : []),
|
|
187
181
|
...(spend ? [`- spend: ${spend.message}`] : []),
|
|
188
182
|
...traces.map(({ stream, trace }) => `- ${recordIdOf(stream)}: ${trace.provider} (${trace.lane}/${trace.protocol}) ${trace.status} (${trace.completionReason}); ${plural(trace.counts.actions ?? 0, "step action")}, ${plural(trace.counts.screenshots ?? 0, "raw screenshot")}`),
|
|
189
|
-
|
|
190
|
-
? ["", "## Gaps", ...bundle.review.gaps.map((gap) => `- ${gap}`)]
|
|
191
|
-
: []),
|
|
192
|
-
"",
|
|
193
|
-
].join("\n");
|
|
183
|
+
], { status });
|
|
194
184
|
}
|
|
@@ -6,7 +6,6 @@ import { type ParticipantIds } from "../../run/participant-records.js";
|
|
|
6
6
|
import type { RunStream } from "../../run/streams.js";
|
|
7
7
|
/** What each surface's records read from the bundle arguments. */
|
|
8
8
|
export interface ScriptedSurfaceContext {
|
|
9
|
-
actorId: string;
|
|
10
9
|
appUrl: string;
|
|
11
10
|
createdAt: string;
|
|
12
11
|
journey: BrowserPersonaJourney;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
|
|
1
|
+
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
2
2
|
import { participantIdsOf, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
3
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
4
|
+
import { sessionEndingInWords } from "../../run/outcomes.js";
|
|
3
5
|
/** A surface's saved ids: `scripted-<surface>` and its `-stream`. */
|
|
4
6
|
export function scriptedSurfaceIds(surfaceId) {
|
|
5
7
|
return participantIdsOf(`scripted-${surfaceId}`, `scripted-${surfaceId}-stream`);
|
|
@@ -11,14 +13,23 @@ export function scriptedSurfaceIds(surfaceId) {
|
|
|
11
13
|
export function scriptedSurfaceRecords(context, surface, index, result, screenshots) {
|
|
12
14
|
const ids = scriptedSurfaceIds(surface.id);
|
|
13
15
|
const lastScreenshot = screenshots.at(-1);
|
|
14
|
-
const status =
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
16
|
+
const { status, reason } = judgeParticipantRecords([
|
|
17
|
+
{
|
|
18
|
+
...(result === undefined
|
|
19
|
+
? {}
|
|
20
|
+
: {
|
|
21
|
+
status: result.status,
|
|
22
|
+
completionReason: result.completionReason,
|
|
23
|
+
reason: result.reason,
|
|
24
|
+
}),
|
|
25
|
+
...(context.sessionError === undefined ? {} : { sessionError: context.sessionError }),
|
|
26
|
+
skipped: false,
|
|
27
|
+
noEngagement: false,
|
|
28
|
+
selfReportedBlocker: false,
|
|
29
|
+
},
|
|
30
|
+
], {
|
|
31
|
+
contractReason: "Dry run: the scenario was pinned without launching a browser or touching the subject app.",
|
|
32
|
+
}).participants[0];
|
|
22
33
|
const simulation = participantRecord(ids, index + 1, {
|
|
23
34
|
personaId: context.persona.id,
|
|
24
35
|
scenarioId: context.journey.scenarioId,
|
|
@@ -28,16 +39,19 @@ export function scriptedSurfaceRecords(context, surface, index, result, screensh
|
|
|
28
39
|
progress: 100,
|
|
29
40
|
currentStep: reason,
|
|
30
41
|
summary: result
|
|
31
|
-
? `
|
|
42
|
+
? `Scenario ${context.journey.scenarioId} ran in the ${surface.id} browser and ${sessionEndingInWords(result.trace)}.`
|
|
32
43
|
: context.sessionError !== undefined
|
|
33
|
-
? `The
|
|
34
|
-
: `
|
|
44
|
+
? `The run failed before the script finished: ${context.sessionError}`
|
|
45
|
+
: `Scenario ${context.journey.scenarioId} would replay in the ${surface.id} browser against ${context.appUrl}; no browser ran.`,
|
|
35
46
|
startedAt: context.createdAt,
|
|
36
47
|
updatedAt: result?.capture.capturedAt ?? context.createdAt,
|
|
37
48
|
});
|
|
38
49
|
const stream = participantStream(ids, {
|
|
39
50
|
kind: "browser",
|
|
40
|
-
label:
|
|
51
|
+
label: participantCaption({
|
|
52
|
+
personaId: context.persona.id,
|
|
53
|
+
device: { name: surface.id, preset: { isMobile: surface.viewport.isMobile } },
|
|
54
|
+
}),
|
|
41
55
|
status,
|
|
42
56
|
transport: "snapshot",
|
|
43
57
|
updatedAt: result?.capture.capturedAt ?? context.createdAt,
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { RunEvent, RunSimulation } from "../../run/bundle.js";
|
|
2
|
+
import type { judgeParticipantRecords } from "../../run/judge.js";
|
|
2
3
|
import type { RunStream } from "../../run/streams.js";
|
|
3
4
|
import type { DesktopParticipantRun } from "../computer-use/types.js";
|
|
4
5
|
import type { ConcurrentBundleArgs } from "./types.js";
|
|
5
6
|
/** What every participant's records share. */
|
|
6
7
|
export interface SharedWorldRecordContext {
|
|
7
8
|
args: ConcurrentBundleArgs;
|
|
9
|
+
participants: ReturnType<typeof judgeParticipantRecords>["participants"];
|
|
8
10
|
external: boolean;
|
|
9
11
|
inProgress: boolean;
|
|
10
|
-
/** The public-safe plane label: never the raw getHost URL or public origin. */
|
|
11
|
-
appUrl: string;
|
|
12
12
|
/** Numbers events in the order they are pushed. */
|
|
13
13
|
nextEventId: (suffix: string) => string;
|
|
14
14
|
}
|
|
@@ -3,14 +3,13 @@
|
|
|
3
3
|
// running).
|
|
4
4
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
5
5
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
6
|
-
import { judgedStatus } from "../../run/judge.js";
|
|
7
6
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
8
|
-
import { participantFactsOf } from "../computer-use/participant-facts.js";
|
|
9
7
|
import { publicSafeRouteLabel } from "./provenance.js";
|
|
10
|
-
import {
|
|
8
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
9
|
+
import { sessionEndingInWords } from "../../run/outcomes.js";
|
|
11
10
|
function participantView(ctx, spec, index) {
|
|
12
|
-
const { args, external
|
|
13
|
-
const
|
|
11
|
+
const { args, external } = ctx;
|
|
12
|
+
const name = participantCaption({ id: spec.planned.id, personaId: spec.persona.id });
|
|
14
13
|
const result = args.actorResults[index];
|
|
15
14
|
const outcome = result?.outcome;
|
|
16
15
|
const session = outcome?.session;
|
|
@@ -21,18 +20,8 @@ function participantView(ctx, spec, index) {
|
|
|
21
20
|
const route = external
|
|
22
21
|
? "[external-public-plane]"
|
|
23
22
|
: publicSafeRouteLabel(args.plan.plane.participants[index]?.entry);
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
: outcome?.sessionError !== undefined
|
|
27
|
-
? "failed"
|
|
28
|
-
: inProgress
|
|
29
|
-
? "running"
|
|
30
|
-
: "contract_proof_only";
|
|
31
|
-
const reason = session?.reason ??
|
|
32
|
-
outcome?.sessionError ??
|
|
33
|
-
(inProgress
|
|
34
|
-
? "Actor desktop is running; the attached Observer hydrates the runtime stream URL without persisting it."
|
|
35
|
-
: "Dry run: the evidence shape was written without launching a desktop or spending provider tokens.");
|
|
23
|
+
const judgment = ctx.participants[index];
|
|
24
|
+
const { status, reason } = judgment;
|
|
36
25
|
const traceScreenshotMode = session?.trace.redaction.screenshots;
|
|
37
26
|
// Include `declared` on the no-outcome fallback too (dry-run, skipped participant): otherwise an
|
|
38
27
|
// absent `declared` means either "the preset rendered faithfully" or "there was no live
|
|
@@ -53,7 +42,8 @@ function participantView(ctx, spec, index) {
|
|
|
53
42
|
? "blurred"
|
|
54
43
|
: "raw";
|
|
55
44
|
return {
|
|
56
|
-
|
|
45
|
+
name,
|
|
46
|
+
judgment,
|
|
57
47
|
outcome,
|
|
58
48
|
session,
|
|
59
49
|
screenshots,
|
|
@@ -67,7 +57,7 @@ function participantView(ctx, spec, index) {
|
|
|
67
57
|
}
|
|
68
58
|
function sharedWorldSimulation(ctx, spec, index, view) {
|
|
69
59
|
const { args, inProgress } = ctx;
|
|
70
|
-
const {
|
|
60
|
+
const { name, outcome, session } = view;
|
|
71
61
|
return participantRecord(spec, index + 1, {
|
|
72
62
|
personaId: spec.persona.id,
|
|
73
63
|
scenarioId: `concurrent-shared-world-${args.plan.studyId}`,
|
|
@@ -77,27 +67,30 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
77
67
|
progress: inProgress ? 35 : 100,
|
|
78
68
|
currentStep: view.reason,
|
|
79
69
|
summary: session
|
|
80
|
-
?
|
|
70
|
+
? `${name} used the shared app and ${sessionEndingInWords(session.trace)}.`
|
|
81
71
|
: outcome?.sessionError !== undefined
|
|
82
|
-
?
|
|
72
|
+
? `${name} did not finish a session: ${outcome.sessionError}`
|
|
83
73
|
: inProgress
|
|
84
|
-
?
|
|
85
|
-
:
|
|
74
|
+
? `${name} is using the shared app.`
|
|
75
|
+
: `${name} would use the shared app; no session ran.`,
|
|
86
76
|
startedAt: args.run.createdAt,
|
|
87
77
|
updatedAt: args.run.createdAt,
|
|
88
78
|
});
|
|
89
79
|
}
|
|
90
80
|
function sharedWorldStream(ctx, spec, index, view) {
|
|
91
81
|
const { args } = ctx;
|
|
92
|
-
const {
|
|
93
|
-
|
|
94
|
-
const judged = view.outcome === undefined ? undefined : judgedStatus(participantFactsOf(view.outcome));
|
|
82
|
+
const { name, session, screenshots, lastScreenshot, desktopGeometry, screenshotMode } = view;
|
|
83
|
+
const judged = view.judgment.judgedStatus;
|
|
95
84
|
return participantStream(spec, {
|
|
96
85
|
...(spec.evidenceAssignment === undefined
|
|
97
86
|
? {}
|
|
98
87
|
: { assignment: participantAssignment(spec.evidenceAssignment) }),
|
|
99
88
|
kind: "browser",
|
|
100
|
-
label:
|
|
89
|
+
label: participantCaption({
|
|
90
|
+
id: spec.planned.id,
|
|
91
|
+
personaId: spec.persona.id,
|
|
92
|
+
device: spec.planned.device,
|
|
93
|
+
}),
|
|
101
94
|
status: view.status,
|
|
102
95
|
...(judged === undefined ? {} : { judgedStatus: judged }),
|
|
103
96
|
transport: "snapshot",
|
|
@@ -122,7 +115,7 @@ function sharedWorldStream(ctx, spec, index, view) {
|
|
|
122
115
|
desktopGeometry,
|
|
123
116
|
ui: {
|
|
124
117
|
route: view.route,
|
|
125
|
-
intent: `Watch
|
|
118
|
+
intent: `Watch ${name} use the shared app at the same time as the others.`,
|
|
126
119
|
state: view.reason,
|
|
127
120
|
...(session ? { actorStatus: session.status } : {}),
|
|
128
121
|
...(lastScreenshot ? { screenshotUrl: lastScreenshot } : {}),
|
|
@@ -17,4 +17,4 @@ export declare function judgeSharedWorldRun(args: Omit<ConcurrentBundleArgs, "ju
|
|
|
17
17
|
export declare function actorRunPassed(result: ActorRunResult | undefined): boolean;
|
|
18
18
|
/** Project the concurrent run into a humanish.run-bundle.v1 with the concurrent shared-world block. */
|
|
19
19
|
export declare function buildConcurrentSharedWorldBundle(args: ConcurrentBundleArgs): RunBundle;
|
|
20
|
-
export declare function renderConcurrentReviewMarkdown(bundle: RunBundle): string;
|
|
20
|
+
export declare function renderConcurrentReviewMarkdown(bundle: RunBundle, status?: unknown): string;
|
|
@@ -9,13 +9,14 @@ import { commandDigestOf } from "../../subject/state.js";
|
|
|
9
9
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
10
10
|
import { participantFactsOf } from "../computer-use/participant-facts.js";
|
|
11
11
|
import { participantFeedbackCandidates } from "../computer-use/participant-feedback.js";
|
|
12
|
-
import { judgeSharedWorld, participantPassed, sharedWorldShortfall,
|
|
13
|
-
import {
|
|
12
|
+
import { judgeSharedWorld, judgeParticipantRecords, participantPassed, sharedWorldShortfall, } from "../../run/judge.js";
|
|
13
|
+
import { renderReviewMarkdown } from "../../run/review-markdown.js";
|
|
14
14
|
import { combineCheckpointDigest } from "./checkpoints.js";
|
|
15
15
|
import { hostOriginDigest } from "./provenance.js";
|
|
16
16
|
import { participantEvent, participantIds } from "../../run/participant-records.js";
|
|
17
17
|
import { sharedWorldParticipantRecords } from "./bundle-records.js";
|
|
18
18
|
import { CONCURRENT_ATTRIBUTION_LIMITS, EXTERNAL_PUBLIC_ATTRIBUTION_LIMITS, } from "./types.js";
|
|
19
|
+
import { gapsListClause } from "../../run/outcomes.js";
|
|
19
20
|
import { plural } from "../../run/text.js";
|
|
20
21
|
/** Max windows live at the same instant (sweep over start/end points). The simultaneity
|
|
21
22
|
* count: the participant count says how many existed; this says how many ever ran at once. */
|
|
@@ -63,33 +64,39 @@ export function actorRunPassed(result) {
|
|
|
63
64
|
return result !== undefined && participantPassed(participantFactsOf(result.outcome));
|
|
64
65
|
}
|
|
65
66
|
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* None of those is adopter-scored proof that the mission text was completed. In particular, a
|
|
73
|
-
* productive `budget_reached` session can coexist with lobby convergence without becoming a
|
|
74
|
-
* `goal_satisfied` result.
|
|
67
|
+
* The run's summary in plain sentences: who took part, how many reached the goal, and what the
|
|
68
|
+
* world showed. A participant reached the goal when its session passed the judge's checks and
|
|
69
|
+
* ended at the goal, so a productive `budget_reached` session does not count even when the lobby
|
|
70
|
+
* converged. Each participant's ending stays in `sharedWorld.outcomes`, and the plane, topology
|
|
71
|
+
* and attribution limits stay in the bundle's events and `sharedWorld` block.
|
|
75
72
|
*/
|
|
76
|
-
function
|
|
77
|
-
const
|
|
78
|
-
const
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
const
|
|
82
|
-
|
|
73
|
+
function sharedWorldSummary(args, facts) {
|
|
74
|
+
const count = args.actorSpecs.length;
|
|
75
|
+
const people = plural(count, "participant");
|
|
76
|
+
const app = facts.external ? "one public app" : "one shared app";
|
|
77
|
+
if (args.dryRun) {
|
|
78
|
+
const served = facts.external ? ", as it is deployed" : ", served from a sandbox";
|
|
79
|
+
return `Dry run: ${people} would use ${app} at the same time${served}. No sandboxes were launched and $0 was spent.`;
|
|
83
80
|
}
|
|
84
|
-
|
|
85
|
-
|
|
81
|
+
if (facts.inProgress) {
|
|
82
|
+
return `${people} ${count === 1 ? "is" : "are"} using ${app} at the same time. The run is still going, so nothing here is final.`;
|
|
86
83
|
}
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
84
|
+
const passed = facts.outcomes.filter((outcome) => outcome.ok);
|
|
85
|
+
const reached = passed.filter((outcome) => outcome.completionReason === "goal_satisfied").length;
|
|
86
|
+
const together = facts.overlaps ? " at the same time" : ", but never at the same time";
|
|
87
|
+
// A participant who did not pass has a gap line; one who passed but stopped short has none.
|
|
88
|
+
const rest = [
|
|
89
|
+
passed.length > reached ? `; ${passed.length - reached} ended without reaching it` : "",
|
|
90
|
+
gapsListClause(count - passed.length, count),
|
|
91
|
+
].join("");
|
|
92
|
+
const world = facts.external
|
|
93
|
+
? args.lobbyConvergenceDigest
|
|
94
|
+
? `All ${count} ended up in the same lobby.`
|
|
95
|
+
: "humanish did not see them all end up in the same lobby."
|
|
96
|
+
: facts.deltas === 0
|
|
97
|
+
? "The app's shared state did not change during the run."
|
|
98
|
+
: `The app's shared state changed ${plural(facts.deltas, "time")} during the run.`;
|
|
99
|
+
return `${people} used ${app}${together}. ${reached} of ${count} reached the goal${rest}. ${world}`;
|
|
93
100
|
}
|
|
94
101
|
/** The run's first two events: its creation and the shared plane's provenance. */
|
|
95
102
|
function planeEvents(args, external) {
|
|
@@ -234,19 +241,13 @@ function sharedWorldEvidence(args, external, inProgress, planeCommit) {
|
|
|
234
241
|
* A finished run's gaps: each participant that did not pass, or, when every participant passed, the
|
|
235
242
|
* world shortfall that failed the run.
|
|
236
243
|
*/
|
|
237
|
-
function finishedGaps(
|
|
238
|
-
const participantGaps =
|
|
239
|
-
.filter((result) => result.outcome.sessionError !== undefined ||
|
|
240
|
-
result.outcome.noEngagement ||
|
|
241
|
-
result.outcome.selfReportedBlocker ||
|
|
242
|
-
result.outcome.session === undefined ||
|
|
243
|
-
result.outcome.session.status !== "passed")
|
|
244
|
-
.map((result) => `${result.spec.planned.id}: ${result.outcome.sessionError ?? result.outcome.session?.reason ?? "did not pass"}`);
|
|
244
|
+
function finishedGaps(participants, shortfall) {
|
|
245
|
+
const participantGaps = participants.flatMap((participant) => participant.gapLine === undefined ? [] : [participant.gapLine]);
|
|
245
246
|
return participantGaps.length === 0 && shortfall !== undefined ? [shortfall] : participantGaps;
|
|
246
247
|
}
|
|
247
248
|
/** Records the concurrency event and returns the review. */
|
|
248
|
-
function concurrencyReview(args, external, inProgress, evidence, events, nextEventId) {
|
|
249
|
-
const { plan,
|
|
249
|
+
function concurrencyReview(args, external, inProgress, evidence, events, nextEventId, participants) {
|
|
250
|
+
const { plan, run: { createdAt }, dryRun, } = args;
|
|
250
251
|
const { sharedWorld, windows, stateSeries, outcomes } = evidence;
|
|
251
252
|
const overlaps = args.judgment.world.overlap;
|
|
252
253
|
const deltas = (stateSeries ?? []).filter((snapshot, i) => i > 0 && snapshot.digest !== (stateSeries ?? [])[i - 1].digest).length;
|
|
@@ -268,25 +269,12 @@ function concurrencyReview(args, external, inProgress, evidence, events, nextEve
|
|
|
268
269
|
message: `Concurrency: ${plural(windows.length, "participant")}${dryRun ? " (dry run; $0)" : `, up to ${maxLive} live at once (cap ${capForReport}), overlap ${overlaps ? "shown" : "not observed"}`}; ${stateSeriesLabel}${convergenceLabel}. Attribution ceiling: ${sharedWorld.attributionLimits.join(", ")}. ${dryRun ? "This dry run proves nothing about live concurrency, scale, or adoption." : "This run reports only its own observed overlap and state changes; it does not prove scale, repeatability, or adopter-harness replacement."}`,
|
|
269
270
|
});
|
|
270
271
|
// The judge's verdict (judgeSharedWorld): every participant produced a terminal, engaged, passed
|
|
271
|
-
// session
|
|
272
|
-
// is not renamed into mission success.
|
|
273
|
-
const verdict = args.judgment.verdict;
|
|
274
|
-
const actorOutcomeSummary = formatSharedWorldActorOutcomes(outcomes, actorSpecs.length);
|
|
272
|
+
// session and the world showed the concurrency verify requires.
|
|
275
273
|
const review = {
|
|
276
274
|
schema: REVIEW_SCHEMA,
|
|
277
|
-
verdict,
|
|
278
|
-
//
|
|
279
|
-
|
|
280
|
-
// report "state delta(s) under load" (live). It reports lobby convergence instead.
|
|
281
|
-
summary: dryRun
|
|
282
|
-
? external
|
|
283
|
-
? `Concurrent shared-world dry run: ${plural(actorSpecs.length, "persona")} declared against one public deployment, used as it is (no sandbox, clone or seed); no sandboxes launched, $0 spent.`
|
|
284
|
-
: `Concurrent shared-world dry run: ${plural(actorSpecs.length, "persona")} declared against one app served at a public sandbox URL (${descriptor.id}); no sandboxes launched, $0 spent.`
|
|
285
|
-
: inProgress
|
|
286
|
-
? `In-progress concurrent shared-world Observer snapshot: ${plural(actorSpecs.length, "persona")} running against one shared app; final verification is pending.`
|
|
287
|
-
: external
|
|
288
|
-
? `Concurrent shared-world (one public deployment, ${actorSpecs.length} personas at once): swarm ${verdict === "pass" ? "ran coherently" : "did not run coherently"}; ${actorOutcomeSummary}; overlap ${overlaps ? "proven" : "not observed"}; ${args.lobbyConvergenceDigest ? `${actorSpecs.length} participants converged on one lobby` : "lobby convergence not observed"}.`
|
|
289
|
-
: `Concurrent shared-world (one shared app, ${actorSpecs.length} personas at once): swarm ${verdict === "pass" ? "ran coherently" : "did not run coherently"}; ${actorOutcomeSummary}; overlap ${overlaps ? "proven" : "not observed"}; ${plural(deltas, "state delta")} under load.`,
|
|
275
|
+
verdict: args.judgment.verdict,
|
|
276
|
+
// The external-public app has no state series, so its summary reports the lobby instead.
|
|
277
|
+
summary: sharedWorldSummary(args, { external, inProgress, overlaps, deltas, outcomes }),
|
|
290
278
|
gaps: dryRun
|
|
291
279
|
? [
|
|
292
280
|
"This dry run launched no concurrent shared-world session; it checks the evidence shape only, not live behavior, scale, or adopter-harness replacement.",
|
|
@@ -295,7 +283,7 @@ function concurrencyReview(args, external, inProgress, evidence, events, nextEve
|
|
|
295
283
|
? [
|
|
296
284
|
"Final actor traces, screenshots, state deltas, and verification are pending; this Observer is for live watch only.",
|
|
297
285
|
]
|
|
298
|
-
: finishedGaps(
|
|
286
|
+
: finishedGaps(participants, sharedWorldShortfall(args.judgment.world)),
|
|
299
287
|
};
|
|
300
288
|
return review;
|
|
301
289
|
}
|
|
@@ -306,15 +294,26 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
306
294
|
const external = (args.planeClass ?? "provisioned-getHost") === "external-public";
|
|
307
295
|
const simulations = [];
|
|
308
296
|
const streams = [];
|
|
309
|
-
// Public-safe label only: neither the raw getHost URL (provisioned) nor the raw public origin
|
|
310
|
-
// (external-public) lands in the bundle. The plane identity is a digest (plane.hostDigest on
|
|
311
|
-
// getHost; plane.publicOriginDigest on external-public).
|
|
312
|
-
const appUrl = external ? "[external-public-plane]" : "[provisioned-subject]";
|
|
313
297
|
const planeCommit = external ? undefined : dryRun ? undefined : args.subjectCommit;
|
|
314
298
|
const events = planeEvents(args, external);
|
|
315
299
|
let eventSeq = 2;
|
|
316
300
|
const nextEventId = (suffix) => `event-${String(eventSeq++).padStart(3, "0")}-${suffix}`;
|
|
317
|
-
const
|
|
301
|
+
const judgment = judgeParticipantRecords(actorSpecs.map((spec, index) => ({
|
|
302
|
+
...participantFactsOf(actorResults[index]?.outcome),
|
|
303
|
+
id: spec.planned.id,
|
|
304
|
+
inProgress: inProgress && actorResults[index] === undefined,
|
|
305
|
+
})), {
|
|
306
|
+
runningReason: "Actor desktop is running; the attached Observer hydrates the runtime stream URL without persisting it.",
|
|
307
|
+
sessionLabel: "Participant",
|
|
308
|
+
missingSessionMessage: "Actor did not produce a terminal session.",
|
|
309
|
+
});
|
|
310
|
+
const recordContext = {
|
|
311
|
+
args,
|
|
312
|
+
external,
|
|
313
|
+
inProgress,
|
|
314
|
+
nextEventId,
|
|
315
|
+
participants: judgment.participants,
|
|
316
|
+
};
|
|
318
317
|
actorSpecs.forEach((spec, index) => {
|
|
319
318
|
const records = sharedWorldParticipantRecords(recordContext, spec, index);
|
|
320
319
|
simulations.push(records.simulation);
|
|
@@ -323,7 +322,7 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
323
322
|
});
|
|
324
323
|
const evidence = sharedWorldEvidence(args, external, inProgress, planeCommit);
|
|
325
324
|
const { sharedWorld } = evidence;
|
|
326
|
-
const review = concurrencyReview(args, external, inProgress, evidence, events, nextEventId);
|
|
325
|
+
const review = concurrencyReview(args, external, inProgress, evidence, events, nextEventId, judgment.participants.filter((_, index) => actorResults[index] !== undefined));
|
|
327
326
|
const anyRaw = actorResults.some((result) => result.outcome.session?.trace.redaction.screenshots === "raw");
|
|
328
327
|
const ranLive = actorResults.some((result) => result.outcome.session !== undefined || result.outcome.sessionError !== undefined);
|
|
329
328
|
const cost = concurrentCostSummary(args, inProgress);
|
|
@@ -335,7 +334,7 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
335
334
|
}),
|
|
336
335
|
persona: {
|
|
337
336
|
id: actorSpecs[0]?.persona.id ?? "concurrent-persona",
|
|
338
|
-
name:
|
|
337
|
+
name: `${plural(actorSpecs.length, "participant")} in one ${external ? "public" : "shared"} app`,
|
|
339
338
|
source: `study:${plan.studyId}`,
|
|
340
339
|
sourceDigest: actorSpecs[0]?.persona.promptDigest ?? args.seedDigest,
|
|
341
340
|
},
|
|
@@ -456,27 +455,20 @@ function declaredStateDigest(state) {
|
|
|
456
455
|
const probes = state?.checkpoint ?? [];
|
|
457
456
|
return combineCheckpointDigest(probes.map((probe) => `${probe.name}=${commandDigestOf(probe.command)}`));
|
|
458
457
|
}
|
|
459
|
-
export function renderConcurrentReviewMarkdown(bundle) {
|
|
458
|
+
export function renderConcurrentReviewMarkdown(bundle, status) {
|
|
460
459
|
const plane = bundle.events.find((event) => event.type === "concurrent-shared-world.plane.provenance");
|
|
461
460
|
const concurrency = bundle.events.find((event) => event.type === "concurrent-shared-world.concurrency");
|
|
462
461
|
const sw = bundle.sharedWorld;
|
|
463
|
-
return [
|
|
464
|
-
`# ${bundle.scenario.title}`,
|
|
465
|
-
"",
|
|
466
|
-
`- run: ${bundle.runId}`,
|
|
467
|
-
`- mode: ${bundle.mode}`,
|
|
468
|
-
`- attribution class: ${bundle.attributionClass ?? "isolated"}`,
|
|
469
|
-
`- topology: ${sw?.topology ?? "(none)"} / ${sw?.topologyMode ?? "(none)"}`,
|
|
470
|
-
`- personas: ${sw?.roleCount ?? 0}`,
|
|
471
|
-
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
472
|
-
`- outcome: ${reviewOutcome(bundle)}`,
|
|
473
|
-
`- summary: ${bundle.review.summary}`,
|
|
462
|
+
return renderReviewMarkdown(bundle, [
|
|
474
463
|
...(plane ? [`- plane: ${plane.message}`] : []),
|
|
475
464
|
...(concurrency ? [`- concurrency: ${concurrency.message}`] : []),
|
|
476
465
|
...(sw ? [`- attribution limits: ${sw.attributionLimits.join(", ")}`] : []),
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
466
|
+
], {
|
|
467
|
+
status,
|
|
468
|
+
beforeVerdict: [
|
|
469
|
+
`- attribution class: ${bundle.attributionClass ?? "isolated"}`,
|
|
470
|
+
`- topology: ${sw?.topology ?? "(none)"} / ${sw?.topologyMode ?? "(none)"}`,
|
|
471
|
+
`- personas: ${sw?.roleCount ?? 0}`,
|
|
472
|
+
],
|
|
473
|
+
});
|
|
482
474
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SharedWorldPlan } from "../../study/plan-types.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { SharedWorldParticipant } from "../../study/plan-participants.js";
|
|
3
3
|
import type { RunBundle } from "../../run/bundle.js";
|
|
4
4
|
import type { DesktopParticipantRun, ParticipantRunOutcome } from "../computer-use/types.js";
|
|
5
5
|
import { type DesktopParticipantDeps } from "../computer-use/participant-deps.js";
|
|
@@ -8,7 +8,6 @@ export declare function defaultSessionTimeoutMs(plan: SharedWorldPlan): number;
|
|
|
8
8
|
/** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
|
|
9
9
|
* same-origin relative path, validated at parse against serve.url). */
|
|
10
10
|
export declare function resolveActorEntryUrl(baseUrl: string, entry: string | undefined): string;
|
|
11
|
-
export declare function participantTaxonomyLabel(labels: Participant["labels"]): string;
|
|
12
11
|
/** Thread the host-yielded lobby code into a follower's mission at runtime (external-public route).
|
|
13
12
|
* The code flows into the follower's join instruction; it is persisted only as the composed prompt
|
|
14
13
|
* the model reads (never a raw bundle field), and humanish scrubs the code from all narration. The
|
|
@@ -45,14 +45,6 @@ export function resolveActorEntryUrl(baseUrl, entry) {
|
|
|
45
45
|
return baseUrl;
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
|
-
export function participantTaxonomyLabel(labels) {
|
|
49
|
-
const parts = [
|
|
50
|
-
labels.actorType ? `type:${labels.actorType}` : undefined,
|
|
51
|
-
labels.surface ? `surface:${labels.surface}` : undefined,
|
|
52
|
-
labels.caseGroup ? `case:${labels.caseGroup}` : undefined,
|
|
53
|
-
].filter((part) => part !== undefined);
|
|
54
|
-
return parts.length > 0 ? ` (${parts.join(" / ")})` : "";
|
|
55
|
-
}
|
|
56
48
|
/** Build one participant's DesktopParticipantRun from its plan (each participant has its own
|
|
57
49
|
* desktop, so its own device). */
|
|
58
50
|
function buildActorSpec(participant, personas) {
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
import { redactText } from "../../evidence/redaction.js";
|
|
4
4
|
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
5
5
|
import { attachObserverRuntimeStreamUrls } from "../../observer/render.js";
|
|
6
|
-
import { foldScorerFailures, judgeExecution, sandboxCleanupFailure, OUTCOME_POLICIES, resultOk, sharedWorldShortfall, } from "../../run/judge.js";
|
|
6
|
+
import { foldScorerFailures, judgeParticipantRecords, judgeExecution, sandboxCleanupFailure, OUTCOME_POLICIES, resultOk, sharedWorldShortfall, } from "../../run/judge.js";
|
|
7
7
|
import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
8
|
-
import { participantExecutionFailures } from "../computer-use/participant-facts.js";
|
|
8
|
+
import { participantExecutionFailures, participantFactsOf, } from "../computer-use/participant-facts.js";
|
|
9
9
|
import { participantCapWarning } from "../computer-use/participant-model.js";
|
|
10
10
|
import { resolveSubjectState } from "../computer-use/subject-projection.js";
|
|
11
11
|
import { actorRunPassed, buildConcurrentSharedWorldBundle, judgeSharedWorldRun, maxSimultaneousWindows, } from "./bundle.js";
|
|
@@ -70,14 +70,10 @@ function concurrentParticipantResults(actorSpecs, actorResults, dryRun) {
|
|
|
70
70
|
: {
|
|
71
71
|
error: {
|
|
72
72
|
code: "HUMANISH_SHARED_WORLD_FAILED",
|
|
73
|
-
message: result.outcome
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
? "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass."
|
|
78
|
-
: session?.completionReason === "harness_error"
|
|
79
|
-
? `Participant ended with a harness error: ${session.reason}`
|
|
80
|
-
: "Actor did not produce a terminal session."),
|
|
73
|
+
message: judgeParticipantRecords([participantFactsOf(result.outcome)], {
|
|
74
|
+
sessionLabel: "Participant",
|
|
75
|
+
missingSessionMessage: "Actor did not produce a terminal session.",
|
|
76
|
+
}).participants[0].notPassedMessage,
|
|
81
77
|
},
|
|
82
78
|
}),
|
|
83
79
|
};
|
|
@@ -126,7 +122,7 @@ function concurrentStudyError(args) {
|
|
|
126
122
|
if (passed === participantCount && args.shortfall !== undefined) {
|
|
127
123
|
return {
|
|
128
124
|
code: "HUMANISH_SHARED_WORLD_FAILED",
|
|
129
|
-
message: `
|
|
125
|
+
message: `The shared-world run failed although every participant passed. ${args.shortfall}`,
|
|
130
126
|
};
|
|
131
127
|
}
|
|
132
128
|
if (passed === participantCount && args.executionFailure !== undefined) {
|
|
@@ -137,7 +133,7 @@ function concurrentStudyError(args) {
|
|
|
137
133
|
}
|
|
138
134
|
return {
|
|
139
135
|
code: "HUMANISH_SHARED_WORLD_FAILED",
|
|
140
|
-
message: `
|
|
136
|
+
message: `The shared-world run failed: ${passed} of ${plural(participantCount, "participant")} passed. The run's review.md lists what happened to the others.`,
|
|
141
137
|
};
|
|
142
138
|
}
|
|
143
139
|
/**
|
|
@@ -71,4 +71,4 @@ export declare function buildLiveTerminalProductBundle(args: {
|
|
|
71
71
|
* the review names, beside the agent's own verdict. */
|
|
72
72
|
capFailure?: string;
|
|
73
73
|
}): RunBundle;
|
|
74
|
-
export declare function renderTerminalReviewMarkdown(bundle: RunBundle): string;
|
|
74
|
+
export declare function renderTerminalReviewMarkdown(bundle: RunBundle, status?: unknown): string;
|