humanish 0.111.4 → 0.112.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
  3. package/dist/actors/computer-use/openai-conversation.js +209 -0
  4. package/dist/actors/computer-use/openai-provider.js +25 -41
  5. package/dist/actors/contract.d.ts +2 -0
  6. package/dist/analysis/diagnostics.js +7 -12
  7. package/dist/analysis/execute.d.ts +1 -1
  8. package/dist/analysis/execute.js +35 -5
  9. package/dist/analysis/service.js +7 -1
  10. package/dist/analysis/types.d.ts +23 -0
  11. package/dist/analysis/validation.d.ts +80 -35
  12. package/dist/analysis/validation.js +58 -11
  13. package/dist/cli/commands/runs.js +4 -10
  14. package/dist/cli/findings.d.ts +34 -3
  15. package/dist/cli/findings.js +136 -18
  16. package/dist/cli/io.d.ts +2 -1
  17. package/dist/cli/io.js +3 -2
  18. package/dist/feedback/draft.js +14 -4
  19. package/dist/observer/run-routes.js +8 -4
  20. package/dist/observer-app.html +4 -4
  21. package/dist/routes/computer-use/bundle.d.ts +1 -1
  22. package/dist/routes/computer-use/bundle.js +6 -16
  23. package/dist/routes/computer-use/fanout-bundle.js +40 -37
  24. package/dist/routes/computer-use/fanout-records.d.ts +2 -0
  25. package/dist/routes/computer-use/fanout-records.js +18 -26
  26. package/dist/routes/computer-use/participant-execution.js +4 -11
  27. package/dist/routes/computer-use/participant-facts.d.ts +2 -2
  28. package/dist/routes/computer-use/participant-facts.js +10 -1
  29. package/dist/routes/computer-use/result.js +7 -15
  30. package/dist/routes/computer-use/single-bundle.d.ts +2 -0
  31. package/dist/routes/computer-use/single-bundle.js +81 -37
  32. package/dist/routes/scripted/bundle.d.ts +1 -1
  33. package/dist/routes/scripted/bundle.js +9 -19
  34. package/dist/routes/scripted/surface-records.d.ts +0 -1
  35. package/dist/routes/scripted/surface-records.js +27 -13
  36. package/dist/routes/shared-world/bundle-records.d.ts +2 -2
  37. package/dist/routes/shared-world/bundle-records.js +21 -28
  38. package/dist/routes/shared-world/bundle.d.ts +1 -1
  39. package/dist/routes/shared-world/bundle.js +70 -78
  40. package/dist/routes/shared-world/participant-specs.d.ts +1 -2
  41. package/dist/routes/shared-world/participant-specs.js +0 -8
  42. package/dist/routes/shared-world/result.js +8 -12
  43. package/dist/routes/terminal/bundle.d.ts +1 -1
  44. package/dist/routes/terminal/bundle.js +10 -19
  45. package/dist/routes/terminal/dry-run.js +1 -0
  46. package/dist/routes/terminal/live-sandbox.js +9 -2
  47. package/dist/routes/terminal/plan.js +1 -0
  48. package/dist/routes/terminal/runtime.d.ts +1 -0
  49. package/dist/routes/terminal/runtime.js +1 -0
  50. package/dist/routes/terminal/session.js +1 -0
  51. package/dist/run/costs.js +9 -20
  52. package/dist/run/detail.d.ts +1 -1
  53. package/dist/run/detail.js +10 -1
  54. package/dist/run/display.d.ts +2 -2
  55. package/dist/run/display.js +15 -18
  56. package/dist/run/dry-run.js +10 -25
  57. package/dist/run/judge.d.ts +24 -0
  58. package/dist/run/judge.js +54 -0
  59. package/dist/run/liveness.d.ts +18 -0
  60. package/dist/run/liveness.js +24 -0
  61. package/dist/run/outcomes.d.ts +8 -1
  62. package/dist/run/outcomes.js +58 -12
  63. package/dist/run/participant-caption.d.ts +30 -0
  64. package/dist/run/participant-caption.js +50 -0
  65. package/dist/run/participant-records.d.ts +1 -1
  66. package/dist/run/review-markdown.d.ts +11 -0
  67. package/dist/run/review-markdown.js +22 -0
  68. package/dist/run/run-cost.d.ts +7 -0
  69. package/dist/run/run-cost.js +25 -5
  70. package/dist/run/run-index.d.ts +4 -50
  71. package/dist/run/run-index.js +57 -87
  72. package/dist/run/run.d.ts +2 -2
  73. package/dist/run/run.js +5 -3
  74. package/dist/run/stats.js +2 -12
  75. package/dist/run/transient-comms-secrets.d.ts +8 -3
  76. package/dist/run/transient-comms-secrets.js +21 -1
  77. package/dist/study/keys.d.ts +1 -0
  78. package/dist/study/keys.js +1 -1
  79. package/dist/study/parse/execution.js +7 -1
  80. package/dist/study/plan-types.d.ts +1 -0
  81. package/dist/study/types.d.ts +4 -1
  82. package/dist/study/warnings.d.ts +2 -1
  83. package/dist/study/warnings.js +3 -2
  84. package/dist/tui-app.js +73 -73
  85. package/dist/verify/liveness.d.ts +2 -2
  86. package/dist/verify/liveness.js +7 -4
  87. package/docs/architecture/terminal-product-route.md +14 -0
  88. package/docs/contracts/run-bundle.md +10 -0
  89. package/docs/contracts/schemas.md +9 -1
  90. package/docs/contracts/study-analysis.md +71 -22
  91. package/package.json +2 -1
  92. package/dist/actors/computer-use/openai-requests.d.ts +0 -62
  93. package/dist/actors/computer-use/openai-requests.js +0 -144
@@ -1,13 +1,13 @@
1
1
  // Assemble the scripted-browser study's run bundle from its sessions, and the review summary and
2
2
  // markdown that ship with it.
3
- import { verdictText } from "../../run/judge.js";
4
- import { reviewOutcome } from "../../run/display.js";
3
+ import { renderReviewMarkdown } from "../../run/review-markdown.js";
5
4
  import { redactText } from "../../evidence/redaction.js";
6
5
  import { buildRunCostSummary, desktopSpanToMinutes, spendFreeCostSummary, } from "../../run/cost-summary.js";
7
6
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
8
7
  import {} from "../../run/streams.js";
9
8
  import { participantEvent, recordIdOf } from "../../run/participant-records.js";
10
9
  import { scriptedSurfaceIds, scriptedSurfaceRecords } from "./surface-records.js";
10
+ import { gapsListClause } from "../../run/outcomes.js";
11
11
  import { plural } from "../../run/text.js";
12
12
  /**
13
13
  * Project the scripted run into a humanish.run-bundle.v1 (no schema change: a new
@@ -146,7 +146,7 @@ function buildScriptedReview(args) {
146
146
  return {
147
147
  schema: REVIEW_SCHEMA,
148
148
  verdict: args.verdict,
149
- summary: `The scripted run failed before a terminal session verdict: ${args.sessionError}`,
149
+ summary: `The run failed before the script finished: ${args.sessionError}`,
150
150
  gaps: [],
151
151
  };
152
152
  }
@@ -154,41 +154,31 @@ function buildScriptedReview(args) {
154
154
  return {
155
155
  schema: REVIEW_SCHEMA,
156
156
  verdict: args.verdict,
157
- summary: `Dry run of scenario ${args.journey.scenarioId} (${args.scenarioSource}, ${args.journey.steps.length} steps) against ${args.appUrl}: composition and scenario checked at $0; no browser ran.`,
157
+ summary: `Dry run: scenario ${args.journey.scenarioId} (${args.scenarioSource}, ${args.journey.steps.length} steps) would replay against ${args.appUrl} on ${plural(args.surfaces.length, "browser surface")}. No browser ran and $0 was spent.`,
158
158
  gaps: ["Live scripted session not yet run (dry run only)."],
159
159
  };
160
160
  }
161
161
  const passed = args.sessionResults.filter((result) => result.status === "passed").length;
162
+ const ran = args.sessionResults.length;
162
163
  return {
163
164
  schema: REVIEW_SCHEMA,
164
165
  verdict: args.verdict,
165
- summary: `Scripted-browser actor replayed ${args.journey.scenarioId} on ${args.sessionResults.length} surface${args.sessionResults.length === 1 ? "" : "s"} against ${args.appUrl}: ${passed}/${args.sessionResults.length} satisfied the scenario predicate.`,
166
+ summary: `Scenario ${args.journey.scenarioId} ran on ${plural(ran, "browser surface")} against ${args.appUrl}: ${passed}/${ran} passed it${gapsListClause(ran - passed, ran)}.`,
166
167
  gaps: args.sessionResults
167
168
  .filter((result) => result.status !== "passed")
168
169
  .map((result) => `${result.capture.surface.id}: ${result.reason}`),
169
170
  };
170
171
  }
171
- export function renderScriptedReviewMarkdown(bundle) {
172
+ export function renderScriptedReviewMarkdown(bundle, status) {
172
173
  const subject = bundle.events.find((event) => event.type === "scripted-lab.subject.declared");
173
174
  const spend = bundle.events.find((event) => event.type === "scripted-lab.spend");
174
175
  const traces = bundle.streams
175
176
  .map((stream) => ({ stream, trace: stream.actor }))
176
177
  .filter((entry) => entry.trace !== undefined);
177
- return [
178
- `# ${bundle.scenario.title}`,
179
- "",
180
- `- run: ${bundle.runId}`,
181
- `- mode: ${bundle.mode}`,
182
- `- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
183
- `- outcome: ${reviewOutcome(bundle)}`,
184
- `- summary: ${bundle.review.summary}`,
178
+ return renderReviewMarkdown(bundle, [
185
179
  `- scenario: ${bundle.scenario.id} @ ${bundle.scenario.sourceDigest} (${bundle.scenario.source})`,
186
180
  ...(subject ? [`- subject: ${subject.message}`] : []),
187
181
  ...(spend ? [`- spend: ${spend.message}`] : []),
188
182
  ...traces.map(({ stream, trace }) => `- ${recordIdOf(stream)}: ${trace.provider} (${trace.lane}/${trace.protocol}) ${trace.status} (${trace.completionReason}); ${plural(trace.counts.actions ?? 0, "step action")}, ${plural(trace.counts.screenshots ?? 0, "raw screenshot")}`),
189
- ...(bundle.review.gaps.length > 0
190
- ? ["", "## Gaps", ...bundle.review.gaps.map((gap) => `- ${gap}`)]
191
- : []),
192
- "",
193
- ].join("\n");
183
+ ], { status });
194
184
  }
@@ -6,7 +6,6 @@ import { type ParticipantIds } from "../../run/participant-records.js";
6
6
  import type { RunStream } from "../../run/streams.js";
7
7
  /** What each surface's records read from the bundle arguments. */
8
8
  export interface ScriptedSurfaceContext {
9
- actorId: string;
10
9
  appUrl: string;
11
10
  createdAt: string;
12
11
  journey: BrowserPersonaJourney;
@@ -1,5 +1,7 @@
1
- // One scripted-browser surface's simulation and stream in the run bundle.
1
+ import { judgeParticipantRecords } from "../../run/judge.js";
2
2
  import { participantIdsOf, participantRecord, participantStream, } from "../../run/participant-records.js";
3
+ import { participantCaption } from "../../run/participant-caption.js";
4
+ import { sessionEndingInWords } from "../../run/outcomes.js";
3
5
  /** A surface's saved ids: `scripted-<surface>` and its `-stream`. */
4
6
  export function scriptedSurfaceIds(surfaceId) {
5
7
  return participantIdsOf(`scripted-${surfaceId}`, `scripted-${surfaceId}-stream`);
@@ -11,14 +13,23 @@ export function scriptedSurfaceIds(surfaceId) {
11
13
  export function scriptedSurfaceRecords(context, surface, index, result, screenshots) {
12
14
  const ids = scriptedSurfaceIds(surface.id);
13
15
  const lastScreenshot = screenshots.at(-1);
14
- const status = result
15
- ? result.status
16
- : context.sessionError !== undefined
17
- ? "failed"
18
- : "contract_proof_only";
19
- const reason = result?.reason ??
20
- context.sessionError ??
21
- "Dry run: the scenario was pinned without launching a browser or touching the subject app.";
16
+ const { status, reason } = judgeParticipantRecords([
17
+ {
18
+ ...(result === undefined
19
+ ? {}
20
+ : {
21
+ status: result.status,
22
+ completionReason: result.completionReason,
23
+ reason: result.reason,
24
+ }),
25
+ ...(context.sessionError === undefined ? {} : { sessionError: context.sessionError }),
26
+ skipped: false,
27
+ noEngagement: false,
28
+ selfReportedBlocker: false,
29
+ },
30
+ ], {
31
+ contractReason: "Dry run: the scenario was pinned without launching a browser or touching the subject app.",
32
+ }).participants[0];
22
33
  const simulation = participantRecord(ids, index + 1, {
23
34
  personaId: context.persona.id,
24
35
  scenarioId: context.journey.scenarioId,
@@ -28,16 +39,19 @@ export function scriptedSurfaceRecords(context, surface, index, result, screensh
28
39
  progress: 100,
29
40
  currentStep: reason,
30
41
  summary: result
31
- ? `Scripted-browser actor (${context.actorId}) replayed ${context.journey.scenarioId} on the ${surface.id} surface; ${result.completionReason}.`
42
+ ? `Scenario ${context.journey.scenarioId} ran in the ${surface.id} browser and ${sessionEndingInWords(result.trace)}.`
32
43
  : context.sessionError !== undefined
33
- ? `The scripted run failed before a terminal session verdict: ${context.sessionError}`
34
- : `Scripted-browser actor (${context.actorId}) against ${context.appUrl}; no session ran.`,
44
+ ? `The run failed before the script finished: ${context.sessionError}`
45
+ : `Scenario ${context.journey.scenarioId} would replay in the ${surface.id} browser against ${context.appUrl}; no browser ran.`,
35
46
  startedAt: context.createdAt,
36
47
  updatedAt: result?.capture.capturedAt ?? context.createdAt,
37
48
  });
38
49
  const stream = participantStream(ids, {
39
50
  kind: "browser",
40
- label: `${surface.label} · ${context.studyId}`,
51
+ label: participantCaption({
52
+ personaId: context.persona.id,
53
+ device: { name: surface.id, preset: { isMobile: surface.viewport.isMobile } },
54
+ }),
41
55
  status,
42
56
  transport: "snapshot",
43
57
  updatedAt: result?.capture.capturedAt ?? context.createdAt,
@@ -1,14 +1,14 @@
1
1
  import type { RunEvent, RunSimulation } from "../../run/bundle.js";
2
+ import type { judgeParticipantRecords } from "../../run/judge.js";
2
3
  import type { RunStream } from "../../run/streams.js";
3
4
  import type { DesktopParticipantRun } from "../computer-use/types.js";
4
5
  import type { ConcurrentBundleArgs } from "./types.js";
5
6
  /** What every participant's records share. */
6
7
  export interface SharedWorldRecordContext {
7
8
  args: ConcurrentBundleArgs;
9
+ participants: ReturnType<typeof judgeParticipantRecords>["participants"];
8
10
  external: boolean;
9
11
  inProgress: boolean;
10
- /** The public-safe plane label: never the raw getHost URL or public origin. */
11
- appUrl: string;
12
12
  /** Numbers events in the order they are pushed. */
13
13
  nextEventId: (suffix: string) => string;
14
14
  }
@@ -3,14 +3,13 @@
3
3
  // running).
4
4
  import { participantAssignment } from "../../study/participant-assignment.js";
5
5
  import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
6
- import { judgedStatus } from "../../run/judge.js";
7
6
  import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
8
- import { participantFactsOf } from "../computer-use/participant-facts.js";
9
7
  import { publicSafeRouteLabel } from "./provenance.js";
10
- import { participantTaxonomyLabel } from "./participant-specs.js";
8
+ import { participantCaption } from "../../run/participant-caption.js";
9
+ import { sessionEndingInWords } from "../../run/outcomes.js";
11
10
  function participantView(ctx, spec, index) {
12
- const { args, external, inProgress } = ctx;
13
- const taxonomy = participantTaxonomyLabel(spec.planned.labels);
11
+ const { args, external } = ctx;
12
+ const name = participantCaption({ id: spec.planned.id, personaId: spec.persona.id });
14
13
  const result = args.actorResults[index];
15
14
  const outcome = result?.outcome;
16
15
  const session = outcome?.session;
@@ -21,18 +20,8 @@ function participantView(ctx, spec, index) {
21
20
  const route = external
22
21
  ? "[external-public-plane]"
23
22
  : publicSafeRouteLabel(args.plan.plane.participants[index]?.entry);
24
- const status = session
25
- ? session.status
26
- : outcome?.sessionError !== undefined
27
- ? "failed"
28
- : inProgress
29
- ? "running"
30
- : "contract_proof_only";
31
- const reason = session?.reason ??
32
- outcome?.sessionError ??
33
- (inProgress
34
- ? "Actor desktop is running; the attached Observer hydrates the runtime stream URL without persisting it."
35
- : "Dry run: the evidence shape was written without launching a desktop or spending provider tokens.");
23
+ const judgment = ctx.participants[index];
24
+ const { status, reason } = judgment;
36
25
  const traceScreenshotMode = session?.trace.redaction.screenshots;
37
26
  // Include `declared` on the no-outcome fallback too (dry-run, skipped participant): otherwise an
38
27
  // absent `declared` means either "the preset rendered faithfully" or "there was no live
@@ -53,7 +42,8 @@ function participantView(ctx, spec, index) {
53
42
  ? "blurred"
54
43
  : "raw";
55
44
  return {
56
- taxonomy,
45
+ name,
46
+ judgment,
57
47
  outcome,
58
48
  session,
59
49
  screenshots,
@@ -67,7 +57,7 @@ function participantView(ctx, spec, index) {
67
57
  }
68
58
  function sharedWorldSimulation(ctx, spec, index, view) {
69
59
  const { args, inProgress } = ctx;
70
- const { taxonomy, outcome, session } = view;
60
+ const { name, outcome, session } = view;
71
61
  return participantRecord(spec, index + 1, {
72
62
  personaId: spec.persona.id,
73
63
  scenarioId: `concurrent-shared-world-${args.plan.studyId}`,
@@ -77,27 +67,30 @@ function sharedWorldSimulation(ctx, spec, index, view) {
77
67
  progress: inProgress ? 35 : 100,
78
68
  currentStep: view.reason,
79
69
  summary: session
80
- ? `Persona ${spec.planned.id}${taxonomy} (${spec.persona.id}): drove the shared plane concurrently; ${session.completionReason}.`
70
+ ? `${name} used the shared app and ${sessionEndingInWords(session.trace)}.`
81
71
  : outcome?.sessionError !== undefined
82
- ? `Persona ${spec.planned.id}${taxonomy} failed before a terminal session verdict: ${outcome.sessionError}`
72
+ ? `${name} did not finish a session: ${outcome.sessionError}`
83
73
  : inProgress
84
- ? `Persona ${spec.planned.id}${taxonomy} (${spec.persona.id}) is running against the shared plane.`
85
- : `Persona ${spec.planned.id}${taxonomy} (${spec.persona.id}) for ${args.descriptor.id} against the shared plane at ${ctx.appUrl}; no session ran.`,
74
+ ? `${name} is using the shared app.`
75
+ : `${name} would use the shared app; no session ran.`,
86
76
  startedAt: args.run.createdAt,
87
77
  updatedAt: args.run.createdAt,
88
78
  });
89
79
  }
90
80
  function sharedWorldStream(ctx, spec, index, view) {
91
81
  const { args } = ctx;
92
- const { taxonomy, session, screenshots, lastScreenshot, desktopGeometry, screenshotMode } = view;
93
- // The judge's status for the participant, so the Observer shows it the way the verdict judged it.
94
- const judged = view.outcome === undefined ? undefined : judgedStatus(participantFactsOf(view.outcome));
82
+ const { name, session, screenshots, lastScreenshot, desktopGeometry, screenshotMode } = view;
83
+ const judged = view.judgment.judgedStatus;
95
84
  return participantStream(spec, {
96
85
  ...(spec.evidenceAssignment === undefined
97
86
  ? {}
98
87
  : { assignment: participantAssignment(spec.evidenceAssignment) }),
99
88
  kind: "browser",
100
- label: `Concurrent persona ${spec.planned.id}${taxonomy} · ${args.plan.studyId}`,
89
+ label: participantCaption({
90
+ id: spec.planned.id,
91
+ personaId: spec.persona.id,
92
+ device: spec.planned.device,
93
+ }),
101
94
  status: view.status,
102
95
  ...(judged === undefined ? {} : { judgedStatus: judged }),
103
96
  transport: "snapshot",
@@ -122,7 +115,7 @@ function sharedWorldStream(ctx, spec, index, view) {
122
115
  desktopGeometry,
123
116
  ui: {
124
117
  route: view.route,
125
- intent: `Watch persona ${spec.planned.id}${taxonomy} (${spec.persona.id}) use the shared app at the same time as the other personas.`,
118
+ intent: `Watch ${name} use the shared app at the same time as the others.`,
126
119
  state: view.reason,
127
120
  ...(session ? { actorStatus: session.status } : {}),
128
121
  ...(lastScreenshot ? { screenshotUrl: lastScreenshot } : {}),
@@ -17,4 +17,4 @@ export declare function judgeSharedWorldRun(args: Omit<ConcurrentBundleArgs, "ju
17
17
  export declare function actorRunPassed(result: ActorRunResult | undefined): boolean;
18
18
  /** Project the concurrent run into a humanish.run-bundle.v1 with the concurrent shared-world block. */
19
19
  export declare function buildConcurrentSharedWorldBundle(args: ConcurrentBundleArgs): RunBundle;
20
- export declare function renderConcurrentReviewMarkdown(bundle: RunBundle): string;
20
+ export declare function renderConcurrentReviewMarkdown(bundle: RunBundle, status?: unknown): string;
@@ -9,13 +9,14 @@ import { commandDigestOf } from "../../subject/state.js";
9
9
  import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
10
10
  import { participantFactsOf } from "../computer-use/participant-facts.js";
11
11
  import { participantFeedbackCandidates } from "../computer-use/participant-feedback.js";
12
- import { judgeSharedWorld, participantPassed, sharedWorldShortfall, verdictText, } from "../../run/judge.js";
13
- import { reviewOutcome } from "../../run/display.js";
12
+ import { judgeSharedWorld, judgeParticipantRecords, participantPassed, sharedWorldShortfall, } from "../../run/judge.js";
13
+ import { renderReviewMarkdown } from "../../run/review-markdown.js";
14
14
  import { combineCheckpointDigest } from "./checkpoints.js";
15
15
  import { hostOriginDigest } from "./provenance.js";
16
16
  import { participantEvent, participantIds } from "../../run/participant-records.js";
17
17
  import { sharedWorldParticipantRecords } from "./bundle-records.js";
18
18
  import { CONCURRENT_ATTRIBUTION_LIMITS, EXTERNAL_PUBLIC_ATTRIBUTION_LIMITS, } from "./types.js";
19
+ import { gapsListClause } from "../../run/outcomes.js";
19
20
  import { plural } from "../../run/text.js";
20
21
  /** Max windows live at the same instant (sweep over start/end points). The simultaneity
21
22
  * count: the participant count says how many existed; this says how many ever ran at once. */
@@ -63,33 +64,39 @@ export function actorRunPassed(result) {
63
64
  return result !== undefined && participantPassed(participantFactsOf(result.outcome));
64
65
  }
65
66
  /**
66
- * Keep three different claims separate in the stakeholder roll-up:
67
- *
68
- * - `outcome.ok` says the actor session passed the harness's credibility checks;
69
- * - `completionReason` says how the participant session ended;
70
- * - shared-world convergence is reported by the plane-specific summary alongside this line.
71
- *
72
- * None of those is adopter-scored proof that the mission text was completed. In particular, a
73
- * productive `budget_reached` session can coexist with lobby convergence without becoming a
74
- * `goal_satisfied` result.
67
+ * The run's summary in plain sentences: who took part, how many reached the goal, and what the
68
+ * world showed. A participant reached the goal when its session passed the judge's checks and
69
+ * ended at the goal, so a productive `budget_reached` session does not count even when the lobby
70
+ * converged. Each participant's ending stays in `sharedWorld.outcomes`, and the plane, topology
71
+ * and attribution limits stay in the bundle's events and `sharedWorld` block.
75
72
  */
76
- function formatSharedWorldActorOutcomes(outcomes, expectedCount) {
77
- const passedSessions = outcomes.filter((outcome) => outcome.ok).length;
78
- const goalSatisfiedSessions = outcomes.filter((outcome) => outcome.ok && outcome.completionReason === "goal_satisfied").length;
79
- const completionReasonCounts = new Map();
80
- for (const outcome of outcomes) {
81
- const reason = outcome.completionReason ?? "not_recorded";
82
- completionReasonCounts.set(reason, (completionReasonCounts.get(reason) ?? 0) + 1);
73
+ function sharedWorldSummary(args, facts) {
74
+ const count = args.actorSpecs.length;
75
+ const people = plural(count, "participant");
76
+ const app = facts.external ? "one public app" : "one shared app";
77
+ if (args.dryRun) {
78
+ const served = facts.external ? ", as it is deployed" : ", served from a sandbox";
79
+ return `Dry run: ${people} would use ${app} at the same time${served}. No sandboxes were launched and $0 was spent.`;
83
80
  }
84
- for (let missing = outcomes.length; missing < expectedCount; missing += 1) {
85
- completionReasonCounts.set("not_recorded", (completionReasonCounts.get("not_recorded") ?? 0) + 1);
81
+ if (facts.inProgress) {
82
+ return `${people} ${count === 1 ? "is" : "are"} using ${app} at the same time. The run is still going, so nothing here is final.`;
86
83
  }
87
- const completionReasons = [...completionReasonCounts.entries()]
88
- // ASCII contract tokens: compare directly so bundle text is byte-stable across host locales.
89
- .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
90
- .map(([reason, count]) => `${reason} ${count}/${expectedCount}`)
91
- .join(", ");
92
- return `${passedSessions}/${plural(expectedCount, "actor session")} passed credibility checks; mission endpoint: ${goalSatisfiedSessions}/${expectedCount} ended goal_satisfied; completion reasons: ${completionReasons}`;
84
+ const passed = facts.outcomes.filter((outcome) => outcome.ok);
85
+ const reached = passed.filter((outcome) => outcome.completionReason === "goal_satisfied").length;
86
+ const together = facts.overlaps ? " at the same time" : ", but never at the same time";
87
+ // A participant who did not pass has a gap line; one who passed but stopped short has none.
88
+ const rest = [
89
+ passed.length > reached ? `; ${passed.length - reached} ended without reaching it` : "",
90
+ gapsListClause(count - passed.length, count),
91
+ ].join("");
92
+ const world = facts.external
93
+ ? args.lobbyConvergenceDigest
94
+ ? `All ${count} ended up in the same lobby.`
95
+ : "humanish did not see them all end up in the same lobby."
96
+ : facts.deltas === 0
97
+ ? "The app's shared state did not change during the run."
98
+ : `The app's shared state changed ${plural(facts.deltas, "time")} during the run.`;
99
+ return `${people} used ${app}${together}. ${reached} of ${count} reached the goal${rest}. ${world}`;
93
100
  }
94
101
  /** The run's first two events: its creation and the shared plane's provenance. */
95
102
  function planeEvents(args, external) {
@@ -234,19 +241,13 @@ function sharedWorldEvidence(args, external, inProgress, planeCommit) {
234
241
  * A finished run's gaps: each participant that did not pass, or, when every participant passed, the
235
242
  * world shortfall that failed the run.
236
243
  */
237
- function finishedGaps(actorResults, shortfall) {
238
- const participantGaps = actorResults
239
- .filter((result) => result.outcome.sessionError !== undefined ||
240
- result.outcome.noEngagement ||
241
- result.outcome.selfReportedBlocker ||
242
- result.outcome.session === undefined ||
243
- result.outcome.session.status !== "passed")
244
- .map((result) => `${result.spec.planned.id}: ${result.outcome.sessionError ?? result.outcome.session?.reason ?? "did not pass"}`);
244
+ function finishedGaps(participants, shortfall) {
245
+ const participantGaps = participants.flatMap((participant) => participant.gapLine === undefined ? [] : [participant.gapLine]);
245
246
  return participantGaps.length === 0 && shortfall !== undefined ? [shortfall] : participantGaps;
246
247
  }
247
248
  /** Records the concurrency event and returns the review. */
248
- function concurrencyReview(args, external, inProgress, evidence, events, nextEventId) {
249
- const { plan, descriptor, run: { createdAt }, dryRun, actorSpecs, actorResults, } = args;
249
+ function concurrencyReview(args, external, inProgress, evidence, events, nextEventId, participants) {
250
+ const { plan, run: { createdAt }, dryRun, } = args;
250
251
  const { sharedWorld, windows, stateSeries, outcomes } = evidence;
251
252
  const overlaps = args.judgment.world.overlap;
252
253
  const deltas = (stateSeries ?? []).filter((snapshot, i) => i > 0 && snapshot.digest !== (stateSeries ?? [])[i - 1].digest).length;
@@ -268,25 +269,12 @@ function concurrencyReview(args, external, inProgress, evidence, events, nextEve
268
269
  message: `Concurrency: ${plural(windows.length, "participant")}${dryRun ? " (dry run; $0)" : `, up to ${maxLive} live at once (cap ${capForReport}), overlap ${overlaps ? "shown" : "not observed"}`}; ${stateSeriesLabel}${convergenceLabel}. Attribution ceiling: ${sharedWorld.attributionLimits.join(", ")}. ${dryRun ? "This dry run proves nothing about live concurrency, scale, or adoption." : "This run reports only its own observed overlap and state changes; it does not prove scale, repeatability, or adopter-harness replacement."}`,
269
270
  });
270
271
  // The judge's verdict (judgeSharedWorld): every participant produced a terminal, engaged, passed
271
- // session. Mission endpoint and completion reasons are reported separately below; `outcomes[].ok`
272
- // is not renamed into mission success.
273
- const verdict = args.judgment.verdict;
274
- const actorOutcomeSummary = formatSharedWorldActorOutcomes(outcomes, actorSpecs.length);
272
+ // session and the world showed the concurrency verify requires.
275
273
  const review = {
276
274
  schema: REVIEW_SCHEMA,
277
- verdict,
278
- // Plane-class-aware: the external-public plane has no getHost/clone/seed and carries no
279
- // authoritative state series, so its summary must not claim a getHost-exposed plane (dry-run) nor
280
- // report "state delta(s) under load" (live). It reports lobby convergence instead.
281
- summary: dryRun
282
- ? external
283
- ? `Concurrent shared-world dry run: ${plural(actorSpecs.length, "persona")} declared against one public deployment, used as it is (no sandbox, clone or seed); no sandboxes launched, $0 spent.`
284
- : `Concurrent shared-world dry run: ${plural(actorSpecs.length, "persona")} declared against one app served at a public sandbox URL (${descriptor.id}); no sandboxes launched, $0 spent.`
285
- : inProgress
286
- ? `In-progress concurrent shared-world Observer snapshot: ${plural(actorSpecs.length, "persona")} running against one shared app; final verification is pending.`
287
- : external
288
- ? `Concurrent shared-world (one public deployment, ${actorSpecs.length} personas at once): swarm ${verdict === "pass" ? "ran coherently" : "did not run coherently"}; ${actorOutcomeSummary}; overlap ${overlaps ? "proven" : "not observed"}; ${args.lobbyConvergenceDigest ? `${actorSpecs.length} participants converged on one lobby` : "lobby convergence not observed"}.`
289
- : `Concurrent shared-world (one shared app, ${actorSpecs.length} personas at once): swarm ${verdict === "pass" ? "ran coherently" : "did not run coherently"}; ${actorOutcomeSummary}; overlap ${overlaps ? "proven" : "not observed"}; ${plural(deltas, "state delta")} under load.`,
275
+ verdict: args.judgment.verdict,
276
+ // The external-public app has no state series, so its summary reports the lobby instead.
277
+ summary: sharedWorldSummary(args, { external, inProgress, overlaps, deltas, outcomes }),
290
278
  gaps: dryRun
291
279
  ? [
292
280
  "This dry run launched no concurrent shared-world session; it checks the evidence shape only, not live behavior, scale, or adopter-harness replacement.",
@@ -295,7 +283,7 @@ function concurrencyReview(args, external, inProgress, evidence, events, nextEve
295
283
  ? [
296
284
  "Final actor traces, screenshots, state deltas, and verification are pending; this Observer is for live watch only.",
297
285
  ]
298
- : finishedGaps(actorResults, sharedWorldShortfall(args.judgment.world)),
286
+ : finishedGaps(participants, sharedWorldShortfall(args.judgment.world)),
299
287
  };
300
288
  return review;
301
289
  }
@@ -306,15 +294,26 @@ export function buildConcurrentSharedWorldBundle(args) {
306
294
  const external = (args.planeClass ?? "provisioned-getHost") === "external-public";
307
295
  const simulations = [];
308
296
  const streams = [];
309
- // Public-safe label only: neither the raw getHost URL (provisioned) nor the raw public origin
310
- // (external-public) lands in the bundle. The plane identity is a digest (plane.hostDigest on
311
- // getHost; plane.publicOriginDigest on external-public).
312
- const appUrl = external ? "[external-public-plane]" : "[provisioned-subject]";
313
297
  const planeCommit = external ? undefined : dryRun ? undefined : args.subjectCommit;
314
298
  const events = planeEvents(args, external);
315
299
  let eventSeq = 2;
316
300
  const nextEventId = (suffix) => `event-${String(eventSeq++).padStart(3, "0")}-${suffix}`;
317
- const recordContext = { args, external, inProgress, appUrl, nextEventId };
301
+ const judgment = judgeParticipantRecords(actorSpecs.map((spec, index) => ({
302
+ ...participantFactsOf(actorResults[index]?.outcome),
303
+ id: spec.planned.id,
304
+ inProgress: inProgress && actorResults[index] === undefined,
305
+ })), {
306
+ runningReason: "Actor desktop is running; the attached Observer hydrates the runtime stream URL without persisting it.",
307
+ sessionLabel: "Participant",
308
+ missingSessionMessage: "Actor did not produce a terminal session.",
309
+ });
310
+ const recordContext = {
311
+ args,
312
+ external,
313
+ inProgress,
314
+ nextEventId,
315
+ participants: judgment.participants,
316
+ };
318
317
  actorSpecs.forEach((spec, index) => {
319
318
  const records = sharedWorldParticipantRecords(recordContext, spec, index);
320
319
  simulations.push(records.simulation);
@@ -323,7 +322,7 @@ export function buildConcurrentSharedWorldBundle(args) {
323
322
  });
324
323
  const evidence = sharedWorldEvidence(args, external, inProgress, planeCommit);
325
324
  const { sharedWorld } = evidence;
326
- const review = concurrencyReview(args, external, inProgress, evidence, events, nextEventId);
325
+ const review = concurrencyReview(args, external, inProgress, evidence, events, nextEventId, judgment.participants.filter((_, index) => actorResults[index] !== undefined));
327
326
  const anyRaw = actorResults.some((result) => result.outcome.session?.trace.redaction.screenshots === "raw");
328
327
  const ranLive = actorResults.some((result) => result.outcome.session !== undefined || result.outcome.sessionError !== undefined);
329
328
  const cost = concurrentCostSummary(args, inProgress);
@@ -335,7 +334,7 @@ export function buildConcurrentSharedWorldBundle(args) {
335
334
  }),
336
335
  persona: {
337
336
  id: actorSpecs[0]?.persona.id ?? "concurrent-persona",
338
- name: `Concurrent shared-world swarm (${actorSpecs.length} personas)`,
337
+ name: `${plural(actorSpecs.length, "participant")} in one ${external ? "public" : "shared"} app`,
339
338
  source: `study:${plan.studyId}`,
340
339
  sourceDigest: actorSpecs[0]?.persona.promptDigest ?? args.seedDigest,
341
340
  },
@@ -456,27 +455,20 @@ function declaredStateDigest(state) {
456
455
  const probes = state?.checkpoint ?? [];
457
456
  return combineCheckpointDigest(probes.map((probe) => `${probe.name}=${commandDigestOf(probe.command)}`));
458
457
  }
459
- export function renderConcurrentReviewMarkdown(bundle) {
458
+ export function renderConcurrentReviewMarkdown(bundle, status) {
460
459
  const plane = bundle.events.find((event) => event.type === "concurrent-shared-world.plane.provenance");
461
460
  const concurrency = bundle.events.find((event) => event.type === "concurrent-shared-world.concurrency");
462
461
  const sw = bundle.sharedWorld;
463
- return [
464
- `# ${bundle.scenario.title}`,
465
- "",
466
- `- run: ${bundle.runId}`,
467
- `- mode: ${bundle.mode}`,
468
- `- attribution class: ${bundle.attributionClass ?? "isolated"}`,
469
- `- topology: ${sw?.topology ?? "(none)"} / ${sw?.topologyMode ?? "(none)"}`,
470
- `- personas: ${sw?.roleCount ?? 0}`,
471
- `- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
472
- `- outcome: ${reviewOutcome(bundle)}`,
473
- `- summary: ${bundle.review.summary}`,
462
+ return renderReviewMarkdown(bundle, [
474
463
  ...(plane ? [`- plane: ${plane.message}`] : []),
475
464
  ...(concurrency ? [`- concurrency: ${concurrency.message}`] : []),
476
465
  ...(sw ? [`- attribution limits: ${sw.attributionLimits.join(", ")}`] : []),
477
- ...(bundle.review.gaps.length > 0
478
- ? ["", "## Gaps", ...bundle.review.gaps.map((gap) => `- ${gap}`)]
479
- : []),
480
- "",
481
- ].join("\n");
466
+ ], {
467
+ status,
468
+ beforeVerdict: [
469
+ `- attribution class: ${bundle.attributionClass ?? "isolated"}`,
470
+ `- topology: ${sw?.topology ?? "(none)"} / ${sw?.topologyMode ?? "(none)"}`,
471
+ `- personas: ${sw?.roleCount ?? 0}`,
472
+ ],
473
+ });
482
474
  }
@@ -1,5 +1,5 @@
1
1
  import type { SharedWorldPlan } from "../../study/plan-types.js";
2
- import type { Participant, SharedWorldParticipant } from "../../study/plan-participants.js";
2
+ import type { SharedWorldParticipant } from "../../study/plan-participants.js";
3
3
  import type { RunBundle } from "../../run/bundle.js";
4
4
  import type { DesktopParticipantRun, ParticipantRunOutcome } from "../computer-use/types.js";
5
5
  import { type DesktopParticipantDeps } from "../computer-use/participant-deps.js";
@@ -8,7 +8,6 @@ export declare function defaultSessionTimeoutMs(plan: SharedWorldPlan): number;
8
8
  /** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
9
9
  * same-origin relative path, validated at parse against serve.url). */
10
10
  export declare function resolveActorEntryUrl(baseUrl: string, entry: string | undefined): string;
11
- export declare function participantTaxonomyLabel(labels: Participant["labels"]): string;
12
11
  /** Thread the host-yielded lobby code into a follower's mission at runtime (external-public route).
13
12
  * The code flows into the follower's join instruction; it is persisted only as the composed prompt
14
13
  * the model reads (never a raw bundle field), and humanish scrubs the code from all narration. The
@@ -45,14 +45,6 @@ export function resolveActorEntryUrl(baseUrl, entry) {
45
45
  return baseUrl;
46
46
  }
47
47
  }
48
- export function participantTaxonomyLabel(labels) {
49
- const parts = [
50
- labels.actorType ? `type:${labels.actorType}` : undefined,
51
- labels.surface ? `surface:${labels.surface}` : undefined,
52
- labels.caseGroup ? `case:${labels.caseGroup}` : undefined,
53
- ].filter((part) => part !== undefined);
54
- return parts.length > 0 ? ` (${parts.join(" / ")})` : "";
55
- }
56
48
  /** Build one participant's DesktopParticipantRun from its plan (each participant has its own
57
49
  * desktop, so its own device). */
58
50
  function buildActorSpec(participant, personas) {
@@ -3,9 +3,9 @@
3
3
  import { redactText } from "../../evidence/redaction.js";
4
4
  import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
5
5
  import { attachObserverRuntimeStreamUrls } from "../../observer/render.js";
6
- import { foldScorerFailures, judgeExecution, sandboxCleanupFailure, OUTCOME_POLICIES, resultOk, sharedWorldShortfall, } from "../../run/judge.js";
6
+ import { foldScorerFailures, judgeParticipantRecords, judgeExecution, sandboxCleanupFailure, OUTCOME_POLICIES, resultOk, sharedWorldShortfall, } from "../../run/judge.js";
7
7
  import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
8
- import { participantExecutionFailures } from "../computer-use/participant-facts.js";
8
+ import { participantExecutionFailures, participantFactsOf, } from "../computer-use/participant-facts.js";
9
9
  import { participantCapWarning } from "../computer-use/participant-model.js";
10
10
  import { resolveSubjectState } from "../computer-use/subject-projection.js";
11
11
  import { actorRunPassed, buildConcurrentSharedWorldBundle, judgeSharedWorldRun, maxSimultaneousWindows, } from "./bundle.js";
@@ -70,14 +70,10 @@ function concurrentParticipantResults(actorSpecs, actorResults, dryRun) {
70
70
  : {
71
71
  error: {
72
72
  code: "HUMANISH_SHARED_WORLD_FAILED",
73
- message: result.outcome.sessionError ??
74
- (result.outcome.noEngagement
75
- ? "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied."
76
- : result.outcome.selfReportedBlocker
77
- ? "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass."
78
- : session?.completionReason === "harness_error"
79
- ? `Participant ended with a harness error: ${session.reason}`
80
- : "Actor did not produce a terminal session."),
73
+ message: judgeParticipantRecords([participantFactsOf(result.outcome)], {
74
+ sessionLabel: "Participant",
75
+ missingSessionMessage: "Actor did not produce a terminal session.",
76
+ }).participants[0].notPassedMessage,
81
77
  },
82
78
  }),
83
79
  };
@@ -126,7 +122,7 @@ function concurrentStudyError(args) {
126
122
  if (passed === participantCount && args.shortfall !== undefined) {
127
123
  return {
128
124
  code: "HUMANISH_SHARED_WORLD_FAILED",
129
- message: `Concurrent shared-world run did not run coherently. ${args.shortfall}`,
125
+ message: `The shared-world run failed although every participant passed. ${args.shortfall}`,
130
126
  };
131
127
  }
132
128
  if (passed === participantCount && args.executionFailure !== undefined) {
@@ -137,7 +133,7 @@ function concurrentStudyError(args) {
137
133
  }
138
134
  return {
139
135
  code: "HUMANISH_SHARED_WORLD_FAILED",
140
- message: `Concurrent shared-world run did not run coherently: ${passed}/${plural(participantCount, "actor")} reached a terminal, engaged passed session.`,
136
+ message: `The shared-world run failed: ${passed} of ${plural(participantCount, "participant")} passed. The run's review.md lists what happened to the others.`,
141
137
  };
142
138
  }
143
139
  /**
@@ -71,4 +71,4 @@ export declare function buildLiveTerminalProductBundle(args: {
71
71
  * the review names, beside the agent's own verdict. */
72
72
  capFailure?: string;
73
73
  }): RunBundle;
74
- export declare function renderTerminalReviewMarkdown(bundle: RunBundle): string;
74
+ export declare function renderTerminalReviewMarkdown(bundle: RunBundle, status?: unknown): string;