humanish 0.127.1 → 0.129.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/CONTEXT.md +2 -2
- package/README.md +1 -1
- package/dist/actors/computer-use/loop/output-limit.js +3 -0
- package/dist/actors/computer-use/loop/types.d.ts +16 -10
- package/dist/actors/computer-use/loop.js +6 -1
- package/dist/analysis/evidence.js +1 -4
- package/dist/analysis/validation.js +1 -1
- package/dist/cli/commands/study-format.js +2 -1
- package/dist/feedback/export.js +3 -2
- package/dist/observer/data.d.ts +13 -6
- package/dist/observer/data.js +40 -10
- package/dist/observer/feed-cursors.d.ts +3 -1
- package/dist/observer/feed-cursors.js +5 -2
- package/dist/observer/run-routes.js +40 -7
- package/dist/observer-app.html +8 -8
- package/dist/routes/computer-use/fanout-bundle.js +4 -7
- package/dist/routes/computer-use/participant-model.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.d.ts +8 -0
- package/dist/routes/shared-world/bundle.js +33 -13
- package/dist/routes/shared-world/participant-specs.d.ts +2 -1
- package/dist/routes/shared-world/participant-specs.js +5 -2
- package/dist/routes/shared-world/result.js +13 -7
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/run/bundle.d.ts +7 -4
- package/dist/run/bundle.js +10 -4
- package/dist/run/detail.js +8 -2
- package/dist/run/locate.d.ts +2 -2
- package/dist/run/locate.js +2 -2
- package/dist/run/outcomes.d.ts +7 -0
- package/dist/run/outcomes.js +14 -0
- package/dist/run/run.js +11 -3
- package/dist/run/stream-trace.d.ts +41 -14
- package/dist/run/stream-trace.js +195 -42
- package/dist/run/streams.d.ts +4 -1
- package/dist/study/bundle-estimate.js +17 -17
- package/dist/verify/costs.d.ts +7 -3
- package/dist/verify/costs.js +27 -12
- package/dist/verify/verify.js +14 -8
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/observer.md +32 -10
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/run-bundle.md +102 -18
- package/docs/contracts/schemas.md +55 -45
- package/docs/contracts/study-analysis.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +1 -1
- package/package.json +1 -1
|
@@ -4,7 +4,7 @@ import { actorEnding } from "../../actors/stop-cause.js";
|
|
|
4
4
|
import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
5
5
|
import { redactText } from "../../evidence/redaction.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
|
-
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
|
+
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
8
8
|
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
9
9
|
import { subjectName } from "../../run/subject-name.js";
|
|
10
10
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
@@ -63,12 +63,9 @@ function fanoutSummary(args, facts) {
|
|
|
63
63
|
// A run where every participant was skipped has no one to describe in this sentence.
|
|
64
64
|
if (tookPart > 0)
|
|
65
65
|
sentences.push(`${plural(tookPart, "participant")} took part, ${tookPart === 1 ? "in its own copy" : "each in their own copy"} of the app: ${outcomes}${gaps}.`);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
sentences.push(`1 participant was skipped before its session started${why}.`);
|
|
70
|
-
if (facts.skipped > 1)
|
|
71
|
-
sentences.push(`${facts.skipped} participants were skipped before their sessions started${why}.`);
|
|
66
|
+
const skipped = skippedSentence(facts.skipped, gaps === "");
|
|
67
|
+
if (skipped !== "")
|
|
68
|
+
sentences.push(skipped);
|
|
72
69
|
return `${host}${rerun}${sentences.join(" ")}${tasks}`;
|
|
73
70
|
}
|
|
74
71
|
/**
|
|
@@ -153,7 +153,7 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
153
153
|
const crossed = deps.runBudget.crossed();
|
|
154
154
|
return crossed === undefined
|
|
155
155
|
? null
|
|
156
|
-
: `study budget reached: ${crossed};
|
|
156
|
+
: `study budget reached: ${crossed}; the study's participants stop before their next model request`;
|
|
157
157
|
},
|
|
158
158
|
}),
|
|
159
159
|
...(deps.onObservedUrl === undefined ? {} : { onObservedUrl: deps.onObservedUrl }),
|
|
@@ -67,7 +67,12 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
67
67
|
mode: "browser-sim",
|
|
68
68
|
progress: inProgress ? 35 : 100,
|
|
69
69
|
currentStep: view.reason,
|
|
70
|
-
summary: participantSummary(name, { name: "the shared app" }, {
|
|
70
|
+
summary: participantSummary(name, { name: "the shared app" }, {
|
|
71
|
+
trace: session?.trace,
|
|
72
|
+
skippedReason: outcome?.skippedReason,
|
|
73
|
+
sessionError: outcome?.sessionError,
|
|
74
|
+
inProgress,
|
|
75
|
+
}),
|
|
71
76
|
startedAt: args.run.createdAt,
|
|
72
77
|
updatedAt: args.run.createdAt,
|
|
73
78
|
...(args.plan.arrivals.declared
|
|
@@ -7,6 +7,14 @@ export declare function maxSimultaneousWindows(windows: Array<{
|
|
|
7
7
|
startedAt: number;
|
|
8
8
|
endedAt: number;
|
|
9
9
|
}>): number;
|
|
10
|
+
/**
|
|
11
|
+
* The windows of the participants that used the app, for overlap and the live-at-once count. A
|
|
12
|
+
* participant the run skipped before its session started never used it, though its window sits
|
|
13
|
+
* inside another participant's session. run.json's laneWindows keep that window: verify expects
|
|
14
|
+
* one per declared participant, and reads overlap from them only on a pass, which a run with a
|
|
15
|
+
* skipped participant cannot reach.
|
|
16
|
+
*/
|
|
17
|
+
export declare function usedAppWindows(results: readonly ActorRunResult[]): ActorRunResult[];
|
|
10
18
|
/**
|
|
11
19
|
* The judgment for a shared-world bundle's inputs: how each participant ended, plus what the run
|
|
12
20
|
* observed about its world: overlap, a state change under overlap on the provisioned plane, and
|
|
@@ -16,7 +16,7 @@ import { hostOriginDigest } from "./provenance.js";
|
|
|
16
16
|
import { participantEvent, participantIds } from "../../run/participant-records.js";
|
|
17
17
|
import { sharedWorldParticipantRecords } from "./bundle-records.js";
|
|
18
18
|
import { CONCURRENT_ATTRIBUTION_LIMITS, EXTERNAL_PUBLIC_ATTRIBUTION_LIMITS, } from "./types.js";
|
|
19
|
-
import { gapsListClause } from "../../run/outcomes.js";
|
|
19
|
+
import { gapsListClause, skippedSentence } from "../../run/outcomes.js";
|
|
20
20
|
import { plural } from "../../run/text.js";
|
|
21
21
|
import { describeArrivals } from "../../study/arrivals.js";
|
|
22
22
|
/** Max windows live at the same instant (sweep over start/end points). The simultaneity
|
|
@@ -38,6 +38,16 @@ export function maxSimultaneousWindows(windows) {
|
|
|
38
38
|
}
|
|
39
39
|
return max;
|
|
40
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* The windows of the participants that used the app, for overlap and the live-at-once count. A
|
|
43
|
+
* participant the run skipped before its session started never used it, though its window sits
|
|
44
|
+
* inside another participant's session. run.json's laneWindows keep that window: verify expects
|
|
45
|
+
* one per declared participant, and reads overlap from them only on a pass, which a run with a
|
|
46
|
+
* skipped participant cannot reach.
|
|
47
|
+
*/
|
|
48
|
+
export function usedAppWindows(results) {
|
|
49
|
+
return results.filter((result) => result.outcome.skippedReason === undefined);
|
|
50
|
+
}
|
|
41
51
|
/**
|
|
42
52
|
* The judgment for a shared-world bundle's inputs: how each participant ended, plus what the run
|
|
43
53
|
* observed about its world: overlap, a state change under overlap on the provisioned plane, and
|
|
@@ -52,11 +62,7 @@ export function judgeSharedWorldRun(args) {
|
|
|
52
62
|
expected: args.actorSpecs.length,
|
|
53
63
|
participants: args.actorResults.map((result) => participantFactsOf(result.outcome)),
|
|
54
64
|
world: {
|
|
55
|
-
|
|
56
|
-
...concurrencyFacts(args.actorSpecs.map((_spec, index) => ({
|
|
57
|
-
startedAt: args.actorResults[index]?.startedAt ?? 0,
|
|
58
|
-
endedAt: args.actorResults[index]?.endedAt ?? 0,
|
|
59
|
-
})), external ? undefined : args.stateSnapshots),
|
|
65
|
+
...concurrencyFacts(usedAppWindows(args.actorResults), external ? undefined : args.stateSnapshots),
|
|
60
66
|
...(external ? { lobbyConvergence: args.lobbyConvergenceDigest !== undefined } : {}),
|
|
61
67
|
},
|
|
62
68
|
});
|
|
@@ -68,8 +74,10 @@ export function actorRunPassed(result) {
|
|
|
68
74
|
* The run's summary in plain sentences: who took part, how many reached the goal, and what the
|
|
69
75
|
* world showed. A participant reached the goal when its session passed the judge's checks and
|
|
70
76
|
* ended at the goal, so a productive `budget_reached` session does not count even when the lobby
|
|
71
|
-
* converged.
|
|
72
|
-
*
|
|
77
|
+
* converged. A participant the run skipped, such as a follower stopped at the handoff, never used
|
|
78
|
+
* the app, so it is counted in a sentence of its own. Each participant's ending stays in
|
|
79
|
+
* `sharedWorld.outcomes`, and the plane, topology and attribution limits stay in the bundle's
|
|
80
|
+
* events and `sharedWorld` block.
|
|
73
81
|
*/
|
|
74
82
|
function sharedWorldSummary(args, facts) {
|
|
75
83
|
const count = args.actorSpecs.length;
|
|
@@ -79,16 +87,20 @@ function sharedWorldSummary(args, facts) {
|
|
|
79
87
|
const served = facts.external ? ", as it is deployed" : ", served from a sandbox";
|
|
80
88
|
return `Dry run: ${people} would use ${app} at the same time${served}. No sandboxes were launched and $0 was spent.`;
|
|
81
89
|
}
|
|
90
|
+
// The live summary is written once, before any participant starts, so it names the plan.
|
|
82
91
|
if (facts.inProgress) {
|
|
83
|
-
return `${people} ${count === 1 ? "is" : "are"}
|
|
92
|
+
return `${people} ${count === 1 ? "is planned, in" : "are planned, all in"} ${app}. The run is still going, so nothing here is final.`;
|
|
84
93
|
}
|
|
85
94
|
const passed = facts.outcomes.filter((outcome) => outcome.ok);
|
|
86
95
|
const reached = passed.filter((outcome) => outcome.completionReason === "goal_satisfied").length;
|
|
87
|
-
const
|
|
96
|
+
const skipped = args.actorResults.filter((result) => result.outcome.skippedReason !== undefined).length;
|
|
97
|
+
const tookPart = count - skipped;
|
|
98
|
+
const together = tookPart === 1 ? "" : facts.overlaps ? " at the same time" : ", but never at the same time";
|
|
88
99
|
// A participant who did not pass has a gap line; one who passed but stopped short has none.
|
|
100
|
+
const gaps = gapsListClause(tookPart - passed.length, tookPart);
|
|
89
101
|
const rest = [
|
|
90
102
|
passed.length > reached ? `; ${passed.length - reached} ended without reaching it` : "",
|
|
91
|
-
|
|
103
|
+
gaps,
|
|
92
104
|
].join("");
|
|
93
105
|
const world = facts.external
|
|
94
106
|
? args.lobbyConvergenceDigest
|
|
@@ -97,7 +109,15 @@ function sharedWorldSummary(args, facts) {
|
|
|
97
109
|
: facts.deltas === 0
|
|
98
110
|
? "The app's shared state did not change during the run."
|
|
99
111
|
: `The app's shared state changed ${plural(facts.deltas, "time")} during the run.`;
|
|
100
|
-
return
|
|
112
|
+
return [
|
|
113
|
+
tookPart > 0
|
|
114
|
+
? `${plural(tookPart, "participant")} used ${app}${together}. ${reached} of ${tookPart} reached the goal${rest}.`
|
|
115
|
+
: "",
|
|
116
|
+
skippedSentence(skipped, gaps === ""),
|
|
117
|
+
world,
|
|
118
|
+
]
|
|
119
|
+
.filter((sentence) => sentence !== "")
|
|
120
|
+
.join(" ");
|
|
101
121
|
}
|
|
102
122
|
/** The run's first two events: its creation and the shared plane's provenance. */
|
|
103
123
|
function planeEvents(args, external) {
|
|
@@ -261,7 +281,7 @@ function concurrencyReview(args, external, inProgress, evidence, events, nextEve
|
|
|
261
281
|
// The count that matters is how many participants were live at once, which can be fewer than exist:
|
|
262
282
|
// a 6-participant run capped at 3 must never read as 6-wide concurrency.
|
|
263
283
|
const capForReport = plan.concurrency;
|
|
264
|
-
const maxLive = maxSimultaneousWindows(
|
|
284
|
+
const maxLive = maxSimultaneousWindows(usedAppWindows(args.actorResults));
|
|
265
285
|
events.push({
|
|
266
286
|
id: nextEventId("concurrency"),
|
|
267
287
|
at: createdAt,
|
|
@@ -12,7 +12,8 @@ export declare function resolveActorEntryUrl(baseUrl: string, entry: string | un
|
|
|
12
12
|
* follower joins through the real UI (a direct /lobby/CODE visit does not auto-join a non-member). */
|
|
13
13
|
export declare function withLobbyCodeMission(spec: DesktopParticipantRun, code: string): DesktopParticipantRun;
|
|
14
14
|
/** A follower blocked by an expired deadline or an ended host. It never opened a browser;
|
|
15
|
-
* keep the actual reason rather than turning every upstream failure into a timeout.
|
|
15
|
+
* keep the actual reason rather than turning every upstream failure into a timeout. The skip
|
|
16
|
+
* reason is a sentence in the form the computer-use skips use, since run lines and gaps print it. */
|
|
16
17
|
export declare function makeBlockedFollowerOutcome(spec: DesktopParticipantRun, reason: string, timedOut: boolean): ParticipantRunOutcome;
|
|
17
18
|
/**
|
|
18
19
|
* A live run publishes an in-progress bundle before its participants start, whether or not an
|
|
@@ -63,7 +63,8 @@ export function withLobbyCodeMission(spec, code) {
|
|
|
63
63
|
};
|
|
64
64
|
}
|
|
65
65
|
/** A follower blocked by an expired deadline or an ended host. It never opened a browser;
|
|
66
|
-
* keep the actual reason rather than turning every upstream failure into a timeout.
|
|
66
|
+
* keep the actual reason rather than turning every upstream failure into a timeout. The skip
|
|
67
|
+
* reason is a sentence in the form the computer-use skips use, since run lines and gaps print it. */
|
|
67
68
|
export function makeBlockedFollowerOutcome(spec, reason, timedOut) {
|
|
68
69
|
return {
|
|
69
70
|
spec,
|
|
@@ -77,7 +78,9 @@ export function makeBlockedFollowerOutcome(spec, reason, timedOut) {
|
|
|
77
78
|
noEngagement: true,
|
|
78
79
|
selfReportedBlocker: false,
|
|
79
80
|
harnessError: false,
|
|
80
|
-
skippedReason: timedOut
|
|
81
|
+
skippedReason: timedOut
|
|
82
|
+
? "skipped: the host did not reach a lobby within the handoff deadline; no desktop was created"
|
|
83
|
+
: "skipped: the host ended before it reached a lobby; no desktop was created",
|
|
81
84
|
};
|
|
82
85
|
}
|
|
83
86
|
/**
|
|
@@ -8,7 +8,7 @@ import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
|
8
8
|
import { participantExecutionFailures, participantFactsOf, } from "../computer-use/participant-facts.js";
|
|
9
9
|
import { participantCapWarning } from "../computer-use/participant-model.js";
|
|
10
10
|
import { resolveSubjectState } from "../computer-use/subject-projection.js";
|
|
11
|
-
import { actorRunPassed, buildConcurrentSharedWorldBundle, judgeSharedWorldRun, maxSimultaneousWindows, } from "./bundle.js";
|
|
11
|
+
import { actorRunPassed, buildConcurrentSharedWorldBundle, judgeSharedWorldRun, maxSimultaneousWindows, usedAppWindows, } from "./bundle.js";
|
|
12
12
|
import { planeStateOf } from "./plan.js";
|
|
13
13
|
import { buildSubjectProvenance, hostOriginDigest } from "./provenance.js";
|
|
14
14
|
import {} from "./types.js";
|
|
@@ -47,9 +47,16 @@ function concurrentParticipantResults(actorSpecs, actorResults, dryRun) {
|
|
|
47
47
|
}
|
|
48
48
|
const session = result.outcome.session;
|
|
49
49
|
const thisOk = participantOk(result);
|
|
50
|
+
const { skippedReason } = result.outcome;
|
|
51
|
+
// The same judged record run.json writes, so the run line and the bundle name one status.
|
|
52
|
+
const record = judgeParticipantRecords([participantFactsOf(result.outcome)], {
|
|
53
|
+
sessionLabel: "Participant",
|
|
54
|
+
missingSessionMessage: "Actor did not produce a terminal session.",
|
|
55
|
+
}).participants[0];
|
|
50
56
|
return {
|
|
51
57
|
...base,
|
|
52
|
-
status:
|
|
58
|
+
status: record.status,
|
|
59
|
+
...(skippedReason === undefined ? {} : { skippedReason }),
|
|
53
60
|
ok: thisOk,
|
|
54
61
|
window: { startedAt: result.startedAt, endedAt: result.endedAt },
|
|
55
62
|
...(session
|
|
@@ -70,10 +77,7 @@ function concurrentParticipantResults(actorSpecs, actorResults, dryRun) {
|
|
|
70
77
|
: {
|
|
71
78
|
error: {
|
|
72
79
|
code: "HUMANISH_SHARED_WORLD_FAILED",
|
|
73
|
-
message:
|
|
74
|
-
sessionLabel: "Participant",
|
|
75
|
-
missingSessionMessage: "Actor did not produce a terminal session.",
|
|
76
|
-
}).participants[0].notPassedMessage,
|
|
80
|
+
message: record.notPassedMessage,
|
|
77
81
|
},
|
|
78
82
|
}),
|
|
79
83
|
};
|
|
@@ -306,7 +310,9 @@ export async function finishConcurrentRun(ctx, live, results, plane) {
|
|
|
306
310
|
? {}
|
|
307
311
|
: { subjectSandbox: { sandboxId: subjectSandboxId, killed: subjectKilled } }),
|
|
308
312
|
...(dryRun ? {} : { overlapProven }),
|
|
309
|
-
...(dryRun
|
|
313
|
+
...(dryRun
|
|
314
|
+
? {}
|
|
315
|
+
: { maxSimultaneousLanes: maxSimultaneousWindows(usedAppWindows(actorResults)) }),
|
|
310
316
|
subject,
|
|
311
317
|
roles: participantResults,
|
|
312
318
|
observer,
|
|
@@ -94,6 +94,8 @@ export interface ConcurrentSharedWorldParticipantResult {
|
|
|
94
94
|
sandboxIdDigest?: string;
|
|
95
95
|
killed: boolean;
|
|
96
96
|
};
|
|
97
|
+
/** Set when the run skipped the participant before its session started (status `blocked`). */
|
|
98
|
+
skippedReason?: string;
|
|
97
99
|
error?: {
|
|
98
100
|
code: ConcurrentSharedWorldStudyErrorCode;
|
|
99
101
|
message: string;
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -7,14 +7,17 @@ import type { RunInterruptSignal } from "./status.js";
|
|
|
7
7
|
import type { RunSimulationStatus, RunStream, RunStreamKind } from "./streams.js";
|
|
8
8
|
/**
|
|
9
9
|
* The run bundle version this release writes: each stream's trace, and each running participant's
|
|
10
|
-
* items so far, are files run.json indexes
|
|
10
|
+
* items so far, are files run.json indexes, and each participant's per-request lists are only in
|
|
11
|
+
* its trace file (REQUEST_LISTS in src/run/stream-trace.ts).
|
|
11
12
|
*/
|
|
12
|
-
export declare const RUN_BUNDLE_SCHEMA = "humanish.run-bundle.
|
|
13
|
+
export declare const RUN_BUNDLE_SCHEMA = "humanish.run-bundle.v4";
|
|
14
|
+
/** The version 0.125.0 and 0.126.0 wrote, with the per-request lists in run.json too. It is still read. */
|
|
15
|
+
export declare const RUN_BUNDLE_V3_SCHEMA = "humanish.run-bundle.v3";
|
|
13
16
|
/** The version 0.123.0 and 0.124.0 wrote, with a running participant's items inline. It is still read. */
|
|
14
17
|
export declare const RUN_BUNDLE_V2_SCHEMA = "humanish.run-bundle.v2";
|
|
15
18
|
/** The version 0.122.0 and earlier wrote, with every trace inline. It is still read. */
|
|
16
19
|
export declare const RUN_BUNDLE_V1_SCHEMA = "humanish.run-bundle.v1";
|
|
17
|
-
/** Whether `value` names a run bundle version this release reads: v3, v2 or v1. */
|
|
20
|
+
/** Whether `value` names a run bundle version this release reads: v4, v3, v2 or v1. */
|
|
18
21
|
export declare function isReadRunBundleSchema(value: unknown): value is RunBundle["schema"];
|
|
19
22
|
/** The run bundle file, relative to the run directory. */
|
|
20
23
|
export declare const RUN_BUNDLE_FILE = "run.json";
|
|
@@ -238,7 +241,7 @@ export interface RunBundle {
|
|
|
238
241
|
restrictions: ["real-communications"];
|
|
239
242
|
};
|
|
240
243
|
commsReceiving?: CommsReceivingEvidence;
|
|
241
|
-
schema: typeof RUN_BUNDLE_SCHEMA | typeof RUN_BUNDLE_V2_SCHEMA | typeof RUN_BUNDLE_V1_SCHEMA;
|
|
244
|
+
schema: typeof RUN_BUNDLE_SCHEMA | typeof RUN_BUNDLE_V3_SCHEMA | typeof RUN_BUNDLE_V2_SCHEMA | typeof RUN_BUNDLE_V1_SCHEMA;
|
|
242
245
|
runId: string;
|
|
243
246
|
mode: "dry-run" | "live";
|
|
244
247
|
simCount: number;
|
package/dist/run/bundle.js
CHANGED
|
@@ -3,16 +3,22 @@ import { captureGitState } from "./git-state.js";
|
|
|
3
3
|
import { studyFields } from "./study-provenance.js";
|
|
4
4
|
/**
|
|
5
5
|
* The run bundle version this release writes: each stream's trace, and each running participant's
|
|
6
|
-
* items so far, are files run.json indexes
|
|
6
|
+
* items so far, are files run.json indexes, and each participant's per-request lists are only in
|
|
7
|
+
* its trace file (REQUEST_LISTS in src/run/stream-trace.ts).
|
|
7
8
|
*/
|
|
8
|
-
export const RUN_BUNDLE_SCHEMA = "humanish.run-bundle.
|
|
9
|
+
export const RUN_BUNDLE_SCHEMA = "humanish.run-bundle.v4";
|
|
10
|
+
/** The version 0.125.0 and 0.126.0 wrote, with the per-request lists in run.json too. It is still read. */
|
|
11
|
+
export const RUN_BUNDLE_V3_SCHEMA = "humanish.run-bundle.v3";
|
|
9
12
|
/** The version 0.123.0 and 0.124.0 wrote, with a running participant's items inline. It is still read. */
|
|
10
13
|
export const RUN_BUNDLE_V2_SCHEMA = "humanish.run-bundle.v2";
|
|
11
14
|
/** The version 0.122.0 and earlier wrote, with every trace inline. It is still read. */
|
|
12
15
|
export const RUN_BUNDLE_V1_SCHEMA = "humanish.run-bundle.v1";
|
|
13
|
-
/** Whether `value` names a run bundle version this release reads: v3, v2 or v1. */
|
|
16
|
+
/** Whether `value` names a run bundle version this release reads: v4, v3, v2 or v1. */
|
|
14
17
|
export function isReadRunBundleSchema(value) {
|
|
15
|
-
return (value === RUN_BUNDLE_SCHEMA ||
|
|
18
|
+
return (value === RUN_BUNDLE_SCHEMA ||
|
|
19
|
+
value === RUN_BUNDLE_V3_SCHEMA ||
|
|
20
|
+
value === RUN_BUNDLE_V2_SCHEMA ||
|
|
21
|
+
value === RUN_BUNDLE_V1_SCHEMA);
|
|
16
22
|
}
|
|
17
23
|
/** The run bundle file, relative to the run directory. */
|
|
18
24
|
export const RUN_BUNDLE_FILE = "run.json";
|
package/dist/run/detail.js
CHANGED
|
@@ -72,7 +72,7 @@ function participantFrom(stream, index, savedPersonaId) {
|
|
|
72
72
|
...(typeof trace.counts?.turns === "number" ? { turns: trace.counts.turns } : {}),
|
|
73
73
|
...(typeof trace.counts?.actions === "number" ? { actions: trace.counts.actions } : {}),
|
|
74
74
|
...(thoughts > 0 ? { thoughts } : {}),
|
|
75
|
-
...costOf(trace),
|
|
75
|
+
...costOf(trace, stream.traceUnavailable === undefined),
|
|
76
76
|
...(stream.traceUnavailable === undefined
|
|
77
77
|
? {}
|
|
78
78
|
: { evidenceUnavailable: stream.traceUnavailable }),
|
|
@@ -85,13 +85,19 @@ function participantFrom(stream, index, savedPersonaId) {
|
|
|
85
85
|
* model it prices at, so the same estimator gives a figure mid-run instead of the screen reporting
|
|
86
86
|
* the cost as unknown until the moment the run ends. That is the half of a run where knowing what
|
|
87
87
|
* it is costing actually changes what you do.
|
|
88
|
+
*
|
|
89
|
+
* Without `rowsLoaded` (the loader refused the participant's trace) only a recorded figure is
|
|
90
|
+
* shown: long-context pricing needs each request's input size, and the totals alone can price
|
|
91
|
+
* under half of what the rows give.
|
|
88
92
|
*/
|
|
89
|
-
function costOf(trace) {
|
|
93
|
+
function costOf(trace, rowsLoaded) {
|
|
90
94
|
if (trace.executionProfile?.billing === "account-unknown")
|
|
91
95
|
return { estimatedCostUsd: null };
|
|
92
96
|
const recorded = trace.estimatedCost?.estimatedCostUsd;
|
|
93
97
|
if (recorded !== undefined)
|
|
94
98
|
return { estimatedCostUsd: recorded };
|
|
99
|
+
if (!rowsLoaded)
|
|
100
|
+
return {};
|
|
95
101
|
const usage = trace.tokenUsage;
|
|
96
102
|
const model = trace.ids?.model;
|
|
97
103
|
if (usage === undefined || typeof model !== "string")
|
package/dist/run/locate.d.ts
CHANGED
|
@@ -44,8 +44,8 @@ export declare function readSafeRunArtifactJson(runPaths: PreparedRunArtifactPat
|
|
|
44
44
|
* A run's bundle as `humanish verify` reads it, for a reader outside humanish: run.json of run
|
|
45
45
|
* `runInput` (a run id, or `latest`) under `cwdInput`'s `.humanish/runs/`, with every participant's
|
|
46
46
|
* trace file and, while the run is in progress, live file joined, so each stream has its
|
|
47
|
-
* `actor.items`, its screenshot entries and its `liveActor.items` as a v1
|
|
48
|
-
* reads run bundles v1, v2 and
|
|
47
|
+
* `actor.items`, its per-request lists, its screenshot entries and its `liveActor.items` as a v1
|
|
48
|
+
* run.json held them. It reads run bundles v1, v2, v3 and v4. Each file must match the index run.json records for it (path,
|
|
49
49
|
* size, sha256, item count); one that does not throws an Error named `StreamTraceRefusedError`
|
|
50
50
|
* that names the participant. Resolves to null when there is no such run, or its run.json is not
|
|
51
51
|
* a run bundle. `bundlePath` is run.json relative to `cwdInput`, `runDir` the run folder.
|
package/dist/run/locate.js
CHANGED
|
@@ -105,8 +105,8 @@ export async function readSafeRunArtifactJson(runPaths, relativePath) {
|
|
|
105
105
|
* A run's bundle as `humanish verify` reads it, for a reader outside humanish: run.json of run
|
|
106
106
|
* `runInput` (a run id, or `latest`) under `cwdInput`'s `.humanish/runs/`, with every participant's
|
|
107
107
|
* trace file and, while the run is in progress, live file joined, so each stream has its
|
|
108
|
-
* `actor.items`, its screenshot entries and its `liveActor.items` as a v1
|
|
109
|
-
* reads run bundles v1, v2 and
|
|
108
|
+
* `actor.items`, its per-request lists, its screenshot entries and its `liveActor.items` as a v1
|
|
109
|
+
* run.json held them. It reads run bundles v1, v2, v3 and v4. Each file must match the index run.json records for it (path,
|
|
110
110
|
* size, sha256, item count); one that does not throws an Error named `StreamTraceRefusedError`
|
|
111
111
|
* that names the participant. Resolves to null when there is no such run, or its run.json is not
|
|
112
112
|
* a run bundle. `bundlePath` is run.json relative to `cwdInput`, `runDir` the run folder.
|
package/dist/run/outcomes.d.ts
CHANGED
|
@@ -53,6 +53,13 @@ interface SummarySession {
|
|
|
53
53
|
export declare function participantSummary(name: string, app: SummaryApp, session: SummarySession): string;
|
|
54
54
|
/** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
|
|
55
55
|
export declare function gapsListClause(listed: number, total: number): string;
|
|
56
|
+
/**
|
|
57
|
+
* The sentence that counts the participants a run skipped before their sessions started, or ""
|
|
58
|
+
* when it skipped none. A skipped participant never used the app, so a summary counts it here and
|
|
59
|
+
* leaves it out of the sentence about those that took part. `pointAtGaps` is false when that
|
|
60
|
+
* sentence already points at the gaps.
|
|
61
|
+
*/
|
|
62
|
+
export declare function skippedSentence(skipped: number, pointAtGaps: boolean): string;
|
|
56
63
|
/** One line a stakeholder can read, with the denominator attached to every number. */
|
|
57
64
|
export declare function formatParticipantOutcomes(outcomes: ParticipantOutcomes, terminalCauses?: readonly ParticipantOutcomeDetail[]): string;
|
|
58
65
|
/** Presentation details tolerate optional legacy actor payloads without changing their tallies. */
|
package/dist/run/outcomes.js
CHANGED
|
@@ -206,6 +206,20 @@ export function gapsListClause(listed, total) {
|
|
|
206
206
|
return `; the gaps list the other ${listed}`;
|
|
207
207
|
return total === 1 ? "; the gaps say why" : `; the gaps list all ${total}`;
|
|
208
208
|
}
|
|
209
|
+
/**
|
|
210
|
+
* The sentence that counts the participants a run skipped before their sessions started, or ""
|
|
211
|
+
* when it skipped none. A skipped participant never used the app, so a summary counts it here and
|
|
212
|
+
* leaves it out of the sentence about those that took part. `pointAtGaps` is false when that
|
|
213
|
+
* sentence already points at the gaps.
|
|
214
|
+
*/
|
|
215
|
+
export function skippedSentence(skipped, pointAtGaps) {
|
|
216
|
+
const why = pointAtGaps ? "; the gaps say why" : "";
|
|
217
|
+
if (skipped === 1)
|
|
218
|
+
return `1 participant was skipped before its session started${why}.`;
|
|
219
|
+
if (skipped > 1)
|
|
220
|
+
return `${skipped} participants were skipped before their sessions started${why}.`;
|
|
221
|
+
return "";
|
|
222
|
+
}
|
|
209
223
|
/** One line a stakeholder can read, with the denominator attached to every number. */
|
|
210
224
|
export function formatParticipantOutcomes(outcomes, terminalCauses = []) {
|
|
211
225
|
if (outcomes.total === 0)
|
package/dist/run/run.js
CHANGED
|
@@ -122,8 +122,9 @@ function observerTarget(options) {
|
|
|
122
122
|
* run.json for `bundle`, written after every live file and trace file it indexes, then the removal
|
|
123
123
|
* of the live files it no longer names. Returns the bundle the review and the projections read:
|
|
124
124
|
* raw sandbox ids scrubbed, each finished trace inline, each running participant's items as the
|
|
125
|
-
* index of its live file
|
|
126
|
-
* as they are appended, so the scrub of the bundle no
|
|
125
|
+
* index of its live file with the estimate run.json records for it, and the live files' warning
|
|
126
|
+
* among the events. Live items are scrubbed as they are appended, so the scrub of the bundle no
|
|
127
|
+
* longer walks them.
|
|
127
128
|
*/
|
|
128
129
|
async function writeRunJson(paths, liveItems, bundle) {
|
|
129
130
|
const sandboxIds = await readRunSandboxIds(paths);
|
|
@@ -138,7 +139,14 @@ async function writeRunJson(paths, liveItems, bundle) {
|
|
|
138
139
|
await writeContainedOutputFile(paths, file.path, file.bytes);
|
|
139
140
|
await writeContainedOutputFile(paths, RUN_BUNDLE_FILE, json(split.bundle), "utf8");
|
|
140
141
|
await live.published();
|
|
141
|
-
|
|
142
|
+
// The split prices each running participant from the rows it leaves out of run.json.
|
|
143
|
+
const streams = publicBundle.streams.map((stream, index) => {
|
|
144
|
+
const estimate = split.bundle.streams[index]?.liveActor?.estimatedCost;
|
|
145
|
+
return stream.liveActor === undefined || estimate === undefined
|
|
146
|
+
? stream
|
|
147
|
+
: { ...stream, liveActor: { ...stream.liveActor, estimatedCost: estimate } };
|
|
148
|
+
});
|
|
149
|
+
return { ...publicBundle, streams };
|
|
142
150
|
}
|
|
143
151
|
function runPublisher(args) {
|
|
144
152
|
const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
|
|
@@ -2,17 +2,23 @@ import { type RunBundle } from "./bundle.js";
|
|
|
2
2
|
import { type PreparedOutputRoot } from "./contained-output.js";
|
|
3
3
|
import { type LiveItemsCursor } from "./live-items.js";
|
|
4
4
|
import type { LiveItemsIndex, RunStream } from "./streams.js";
|
|
5
|
-
/**
|
|
5
|
+
/**
|
|
6
|
+
* A trace that does not match its index, or cannot be read. The message names the stream, and ends
|
|
7
|
+
* with `advice`: by default, restore the run folder from a copy.
|
|
8
|
+
*/
|
|
6
9
|
export declare class StreamTraceRefusedError extends Error {
|
|
7
10
|
readonly streamId: string;
|
|
8
|
-
constructor(streamId: string, detail: string);
|
|
11
|
+
constructor(streamId: string, detail: string, advice?: string);
|
|
9
12
|
}
|
|
13
|
+
/** `trace` without the per-request lists; every other field as it is. */
|
|
14
|
+
export declare function withoutRequestLists<T extends object>(trace: T): T;
|
|
10
15
|
/**
|
|
11
16
|
* The run.json the run writes, and the trace files it writes first. Each stream with a finished
|
|
12
|
-
* actor keeps that actor without `items
|
|
13
|
-
* trace file; the file is the actor trace with the stream's
|
|
14
|
-
* stream already split (its actor read back from
|
|
15
|
-
* since its file is on disk
|
|
17
|
+
* actor keeps that actor without `items` and the per-request lists, its artifacts other than
|
|
18
|
+
* screenshots, and an index of its trace file; the file is the whole actor trace with the stream's
|
|
19
|
+
* screenshot entries as `artifacts`. A stream already split (its actor read back from run.json
|
|
20
|
+
* without its items) keeps its index, since its file is on disk, and loses any lists an older
|
|
21
|
+
* run.json held beside it. A running participant's live actor is written by liveSummary.
|
|
16
22
|
*/
|
|
17
23
|
export declare function splitStreamTraces(bundle: RunBundle): {
|
|
18
24
|
bundle: RunBundle;
|
|
@@ -36,13 +42,22 @@ export type TraceFileReader = (relativePath: string, maxBytes: number, length?:
|
|
|
36
42
|
}>;
|
|
37
43
|
/**
|
|
38
44
|
* A live file the loader joined into stream `streamId`: the index it read the items by, and the
|
|
39
|
-
* cursor it read past when the stream holds only the items after one.
|
|
45
|
+
* cursor it read past when the stream holds only the items after one. A read with `stopped` has
|
|
46
|
+
* an index that records more than the loader reads of one file, so it read nothing and joined no
|
|
47
|
+
* items past `after`, which it always has; `stopped` is the warning a reader that shows the run
|
|
48
|
+
* gives.
|
|
40
49
|
*/
|
|
41
|
-
export
|
|
50
|
+
export type LiveItemsRead = {
|
|
42
51
|
streamId: string;
|
|
43
52
|
index: LiveItemsIndex;
|
|
44
53
|
after?: LiveItemsCursor;
|
|
45
|
-
|
|
54
|
+
stopped?: undefined;
|
|
55
|
+
} | {
|
|
56
|
+
streamId: string;
|
|
57
|
+
index: LiveItemsIndex;
|
|
58
|
+
after: LiveItemsCursor;
|
|
59
|
+
stopped: string;
|
|
60
|
+
};
|
|
46
61
|
/** For a live file's index, the prefix of that file the caller holds already, if any. */
|
|
47
62
|
type LiveAfter = (index: LiveItemsIndex) => LiveItemsCursor | undefined;
|
|
48
63
|
/**
|
|
@@ -50,10 +65,17 @@ type LiveAfter = (index: LiveItemsIndex) => LiveItemsCursor | undefined;
|
|
|
50
65
|
* stream without an index is returned as it is (a v1 stream, or one with no actor). An indexed
|
|
51
66
|
* stream's file must have a trace file name (isStreamTracePath); it is read under `maxBytes` and
|
|
52
67
|
* checked against the index's byte count, sha256, item count and screenshot count, and any
|
|
53
|
-
* difference throws StreamTraceRefusedError.
|
|
68
|
+
* difference throws StreamTraceRefusedError. An index that records more than `maxBytes` throws
|
|
69
|
+
* before the file is read. The rules that need the whole bundle (an unindexed v2
|
|
54
70
|
* actor, two participants indexing one file) are loadStreamTraces'.
|
|
55
71
|
*/
|
|
56
72
|
export declare function loadStreamTrace(runPaths: PreparedOutputRoot, stream: RunStream, maxBytes?: number): Promise<RunStream>;
|
|
73
|
+
/**
|
|
74
|
+
* Whether a run.json of `schema` holds the per-request lists (REQUEST_LISTS) the writer kept in
|
|
75
|
+
* it: v1 to v3, and a schema WRITER_RULES does not name, which is read as v1. From v4 they are
|
|
76
|
+
* only in the trace file, so a running participant's `liveActor` holds none.
|
|
77
|
+
*/
|
|
78
|
+
export declare const keepsRequestListsInRunJson: (schema: string) => boolean;
|
|
57
79
|
interface LoadedTraces {
|
|
58
80
|
bundle: RunBundle;
|
|
59
81
|
refused: StreamTraceRefusedError[];
|
|
@@ -71,7 +93,9 @@ interface LoadedTraces {
|
|
|
71
93
|
* `liveAfter` names, for a live file's index, the prefix the caller holds already (the Observer
|
|
72
94
|
* feed's cursor). The loader then reads and joins only the items past it, checked by liveItemsAfter
|
|
73
95
|
* without the indexed sha256, and `live` records the cursor; a cursor past the index or off the
|
|
74
|
-
* file's lines gets the whole prefix, checked as always.
|
|
96
|
+
* file's lines gets the whole prefix, checked as always. A live file whose index records more than
|
|
97
|
+
* `maxBytes` is not read: for a cursor that fits the index the stream gets no items past it and
|
|
98
|
+
* `live` records `stopped`, and without one the stream is refused.
|
|
75
99
|
*
|
|
76
100
|
* A live file that cannot be read may have been removed by the writer after `bundle` was read: it
|
|
77
101
|
* removes a superseded generation once run.json names the next one, and every live file once the
|
|
@@ -85,13 +109,16 @@ export declare function loadStreamTraces(source: PreparedOutputRoot | TraceFileR
|
|
|
85
109
|
}): Promise<LoadedTraces>;
|
|
86
110
|
/**
|
|
87
111
|
* A loaded bundle as a reader that shows a run presents it. A refused stream keeps run.json's
|
|
88
|
-
* summary, shows none of the items run.json or a file supplied for it (its
|
|
89
|
-
* in-progress items, its screenshot entries), and carries
|
|
90
|
-
* warn event on its timeline says the same.
|
|
112
|
+
* summary, shows none of the items or per-request lists run.json or a file supplied for it (its
|
|
113
|
+
* actor's items and lists, its in-progress items, its screenshot entries), and carries
|
|
114
|
+
* `traceUnavailable` with the reason; a warn event on its timeline says the same. A reader that
|
|
115
|
+
* shows the run then never prices or checks rows the loader refused. A live file the loader
|
|
116
|
+
* stopped reading (LiveItemsRead's `stopped`) gives its stream a `live-items.not-read` warn event.
|
|
91
117
|
*/
|
|
92
118
|
export declare function displayedTraces(loaded: {
|
|
93
119
|
bundle: RunBundle;
|
|
94
120
|
refused: readonly StreamTraceRefusedError[];
|
|
121
|
+
live?: readonly LiveItemsRead[];
|
|
95
122
|
}): RunBundle;
|
|
96
123
|
/** The bundle with every stream loaded; the first refusal throws. */
|
|
97
124
|
export declare function loadRunTraces(source: PreparedOutputRoot | TraceFileReader, bundle: RunBundle, maxBytes?: number): Promise<RunBundle>;
|