humanish 0.111.4 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +71 -22
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
package/dist/run/outcomes.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ACTOR_STATUSES } from "../actors/contract.js";
|
|
2
2
|
import { cuaGoalSource, isCuaTrace, CUA_COMPLETION_NOTE, } from "../actors/goal-source.js";
|
|
3
3
|
import { actorEnding } from "../actors/stop-cause.js";
|
|
4
|
+
import { plural } from "./text.js";
|
|
4
5
|
import { isNonNegativeSafeInteger, isRecord } from "./type-guards.js";
|
|
5
6
|
/** Tally participant outcomes from actor statuses. Statuses this does not recognise are counted in
|
|
6
7
|
* `total` but nowhere else, so the parts can never exceed the whole. */
|
|
@@ -113,6 +114,46 @@ function participantCompletionLine(outcomes, terminalCauses) {
|
|
|
113
114
|
}
|
|
114
115
|
return goalLine;
|
|
115
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* How one participant's session ended, in words a participant summary can end on. A computer-use
|
|
119
|
+
* goal says where the claim came from, as the run's participant line does.
|
|
120
|
+
*/
|
|
121
|
+
export function sessionEndingInWords(trace) {
|
|
122
|
+
switch (trace.completionReason) {
|
|
123
|
+
case "goal_satisfied": {
|
|
124
|
+
const source = cuaGoalSource(trace);
|
|
125
|
+
if (source === "participant_report")
|
|
126
|
+
return "reported reaching the goal";
|
|
127
|
+
return source === "condition_matched" ? "met its completion condition" : "reached the goal";
|
|
128
|
+
}
|
|
129
|
+
case "turn_completed":
|
|
130
|
+
return "said it was done";
|
|
131
|
+
case "gave_up":
|
|
132
|
+
return "gave up";
|
|
133
|
+
case "blocked_approval":
|
|
134
|
+
return "was blocked waiting for an approval";
|
|
135
|
+
case "timed_out":
|
|
136
|
+
return "ran out of time";
|
|
137
|
+
case "budget_reached":
|
|
138
|
+
return "was stopped by a time or spend limit";
|
|
139
|
+
case "actor_error":
|
|
140
|
+
return "stopped on an error";
|
|
141
|
+
case "step_failed":
|
|
142
|
+
return "failed a scripted step";
|
|
143
|
+
case "harness_error":
|
|
144
|
+
return "was lost to a harness failure";
|
|
145
|
+
default:
|
|
146
|
+
return "ended";
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
|
|
150
|
+
export function gapsListClause(listed, total) {
|
|
151
|
+
if (listed <= 0)
|
|
152
|
+
return "";
|
|
153
|
+
if (listed < total)
|
|
154
|
+
return `; the gaps list the other ${listed}`;
|
|
155
|
+
return total === 1 ? "; the gaps say why" : `; the gaps list all ${total}`;
|
|
156
|
+
}
|
|
116
157
|
/** One line a stakeholder can read, with the denominator attached to every number. */
|
|
117
158
|
export function formatParticipantOutcomes(outcomes, terminalCauses = []) {
|
|
118
159
|
if (outcomes.total === 0)
|
|
@@ -189,20 +230,25 @@ export function withCuaReviewProvenance(review, streams) {
|
|
|
189
230
|
: !details.some((entry) => entry.goalSource !== undefined)))
|
|
190
231
|
return review;
|
|
191
232
|
const outcomes = formatParticipantOutcomes(review.participants, details);
|
|
192
|
-
|
|
193
|
-
//
|
|
194
|
-
|
|
195
|
-
const
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
.
|
|
233
|
+
const tasks = review.tasks ? ` Tasks: ${formatRunTaskFunnel(review.tasks)}.` : "";
|
|
234
|
+
// A summary that states the bare count gets it qualified where it stands, keeping rerun context,
|
|
235
|
+
// the participant's own words and adapter findings around it.
|
|
236
|
+
const { reachedGoal, total } = review.participants;
|
|
237
|
+
const bareLine = `${reachedGoal}/${total} reached the goal`;
|
|
238
|
+
const qualifiedLine = participantCompletionLine(review.participants, details);
|
|
239
|
+
// A summary with no count gets it in a header. Earlier releases wrote a header in other words.
|
|
240
|
+
const header = `${plural(total, "participant")} took part: ${outcomes}.${tasks} `;
|
|
241
|
+
const olderHeader = `Run gate: ${review.verdict}. Participants: ${outcomes}.${tasks} Recorded summary: `;
|
|
242
|
+
const summary = review.summary.startsWith(olderHeader)
|
|
243
|
+
? `${header}${review.summary.slice(olderHeader.length)}`
|
|
244
|
+
: review.summary.includes(bareLine)
|
|
245
|
+
? review.summary.split(bareLine).join(qualifiedLine)
|
|
246
|
+
: review.summary.includes(qualifiedLine)
|
|
247
|
+
? review.summary
|
|
248
|
+
: `${header}${review.summary}`;
|
|
203
249
|
return {
|
|
204
250
|
...review,
|
|
205
|
-
summary
|
|
251
|
+
summary,
|
|
206
252
|
gaps: review.participants.reachedGoal === 0
|
|
207
253
|
? review.gaps
|
|
208
254
|
: [...review.gaps.filter((gap) => gap !== CUA_COMPLETION_NOTE), CUA_COMPLETION_NOTE],
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** A participant, as much of it as a caption reads. */
|
|
2
|
+
export interface CaptionFacts {
|
|
3
|
+
/** The roster id, on a route that has a roster. */
|
|
4
|
+
readonly id?: string;
|
|
5
|
+
/** The persona, which names the participant when there is no roster id or it is only a number. */
|
|
6
|
+
readonly personaId: string;
|
|
7
|
+
/** The participant's device, when it has one. */
|
|
8
|
+
readonly device?: {
|
|
9
|
+
readonly name: string;
|
|
10
|
+
readonly preset: {
|
|
11
|
+
readonly isMobile: boolean;
|
|
12
|
+
};
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/** "Lobby host", or "Player two, phone" for a participant on a phone. */
|
|
16
|
+
export declare function participantCaption(participant: CaptionFacts): string;
|
|
17
|
+
/** The fields of a saved stream that name its participant. */
|
|
18
|
+
export interface SavedCaptionFacts {
|
|
19
|
+
readonly label: string;
|
|
20
|
+
/** The participant id a computer-use stream records. */
|
|
21
|
+
readonly participantId?: string;
|
|
22
|
+
readonly personaId?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The caption a saved stream shows: its label, which a route writes with `participantCaption`.
|
|
26
|
+
* Earlier releases wrote ids and taxonomy into the label (`lane-01 · browser`, `CUA participant
|
|
27
|
+
* <id>: <study>`, `Concurrent persona <id> (type:<t> / ...) · <study>`), so a stream saved that way
|
|
28
|
+
* is captioned again from the id it names, without a device.
|
|
29
|
+
*/
|
|
30
|
+
export declare function savedCaption(stream: SavedCaptionFacts): string;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// The words a reader sees for one participant. Routes write the caption as the stream label, and
|
|
2
|
+
// the Observer, the TUI and findings show that label, so every surface names the participant the
|
|
3
|
+
// same way. The participant's taxonomy and ids stay in the bundle data, out of the caption.
|
|
4
|
+
/** Ids `participantIdAt` makes for an undeclared roster entry. They number a position. */
|
|
5
|
+
const NUMBERED_ID = /^(lane|role)-\d+$/;
|
|
6
|
+
/** An id in words: `lobby-host` reads "Lobby host". */
|
|
7
|
+
function inWords(id) {
|
|
8
|
+
const words = id
|
|
9
|
+
.split(/[-_\s]+/)
|
|
10
|
+
.filter(Boolean)
|
|
11
|
+
.join(" ");
|
|
12
|
+
return `${words.charAt(0).toUpperCase()}${words.slice(1)}`;
|
|
13
|
+
}
|
|
14
|
+
/** A phone or a tablet. A desktop of any size goes unnamed, since most participants use one. */
|
|
15
|
+
function deviceInWords(device) {
|
|
16
|
+
if (device === undefined)
|
|
17
|
+
return undefined;
|
|
18
|
+
if (device.preset.isMobile)
|
|
19
|
+
return "phone";
|
|
20
|
+
return device.name === "tablet" ? "tablet" : undefined;
|
|
21
|
+
}
|
|
22
|
+
/** "Lobby host", or "Player two, phone" for a participant on a phone. */
|
|
23
|
+
export function participantCaption(participant) {
|
|
24
|
+
const { id, personaId } = participant;
|
|
25
|
+
const name = inWords(id === undefined || NUMBERED_ID.test(id) ? personaId : id);
|
|
26
|
+
const device = deviceInWords(participant.device);
|
|
27
|
+
return device === undefined ? name : `${name}, ${device}`;
|
|
28
|
+
}
|
|
29
|
+
const OLDER_SHARED_WORLD_LABEL = /^Concurrent persona (\S+)/;
|
|
30
|
+
const OLDER_COMPUTER_USE_LABEL = /^CUA |^\S+ · browser$/;
|
|
31
|
+
/**
|
|
32
|
+
* The caption a saved stream shows: its label, which a route writes with `participantCaption`.
|
|
33
|
+
* Earlier releases wrote ids and taxonomy into the label (`lane-01 · browser`, `CUA participant
|
|
34
|
+
* <id>: <study>`, `Concurrent persona <id> (type:<t> / ...) · <study>`), so a stream saved that way
|
|
35
|
+
* is captioned again from the id it names, without a device.
|
|
36
|
+
*/
|
|
37
|
+
export function savedCaption(stream) {
|
|
38
|
+
const { label, participantId, personaId } = stream;
|
|
39
|
+
const sharedWorldId = OLDER_SHARED_WORLD_LABEL.exec(label)?.[1];
|
|
40
|
+
if (sharedWorldId !== undefined) {
|
|
41
|
+
return participantCaption({ id: sharedWorldId, personaId: personaId ?? sharedWorldId });
|
|
42
|
+
}
|
|
43
|
+
if (OLDER_COMPUTER_USE_LABEL.test(label) && personaId !== undefined) {
|
|
44
|
+
return participantCaption({
|
|
45
|
+
...(participantId === undefined ? {} : { id: participantId }),
|
|
46
|
+
personaId,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return label;
|
|
50
|
+
}
|
|
@@ -25,4 +25,4 @@ export declare function recordIdOf(stream: RunStream): string;
|
|
|
25
25
|
/** The record id a saved event names, when it names one. */
|
|
26
26
|
export declare function eventRecordIdOf(event: RunEvent): string | undefined;
|
|
27
27
|
/** The participant id an adapter recorded on a saved stream (a fan-out `lane-NN` id), when it has one. */
|
|
28
|
-
export declare function streamParticipantIdOf(stream: RunStream): string | undefined;
|
|
28
|
+
export declare function streamParticipantIdOf(stream: Pick<RunStream, "laneId">): string | undefined;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RunBundle } from "./bundle.js";
|
|
2
|
+
type ReviewBundle = Pick<RunBundle, "runId" | "mode" | "review"> & Partial<Pick<RunBundle, "outcome" | "simulations">> & {
|
|
3
|
+
scenario: Pick<RunBundle["scenario"], "title">;
|
|
4
|
+
};
|
|
5
|
+
/** The review's common head and gaps, around the route's evidence lines. */
|
|
6
|
+
export declare function renderReviewMarkdown(bundle: ReviewBundle, middle: readonly string[], options?: {
|
|
7
|
+
beforeVerdict?: readonly string[];
|
|
8
|
+
style?: "preview";
|
|
9
|
+
status?: unknown;
|
|
10
|
+
}): string;
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { reviewOutcome } from "./display.js";
|
|
2
|
+
import { verdictText } from "./judge.js";
|
|
3
|
+
/** The review's common head and gaps, around the route's evidence lines. */
|
|
4
|
+
export function renderReviewMarkdown(bundle, middle, options = {}) {
|
|
5
|
+
const preview = options.style === "preview";
|
|
6
|
+
const field = (name, value) => preview ? [`${name[0].toUpperCase()}${name.slice(1)}: ${value}`, ""] : [`- ${name}: ${value}`];
|
|
7
|
+
return [
|
|
8
|
+
`# ${preview ? "humanish Run Review" : bundle.scenario.title}`,
|
|
9
|
+
"",
|
|
10
|
+
...field("run", bundle.runId),
|
|
11
|
+
...field("mode", bundle.mode),
|
|
12
|
+
...(options.beforeVerdict ?? []),
|
|
13
|
+
...field("verdict", verdictText(bundle.review.verdict, bundle.mode)),
|
|
14
|
+
...field("outcome", reviewOutcome(bundle, options.status)),
|
|
15
|
+
...(preview ? [bundle.review.summary, ""] : field("summary", bundle.review.summary)),
|
|
16
|
+
...middle,
|
|
17
|
+
...(preview || bundle.review.gaps.length > 0
|
|
18
|
+
? ["", "## Gaps", ...(preview ? [""] : []), ...bundle.review.gaps.map((gap) => `- ${gap}`)]
|
|
19
|
+
: []),
|
|
20
|
+
"",
|
|
21
|
+
].join("\n");
|
|
22
|
+
}
|
package/dist/run/run-cost.d.ts
CHANGED
|
@@ -34,6 +34,13 @@ export interface RunCost {
|
|
|
34
34
|
}
|
|
35
35
|
/** The accounting's analysis spend, or null when the run sent no analysis request. */
|
|
36
36
|
export declare function analysisCostOf(accounting: Pick<AnalysisAccounting, "dispatched" | "unresolved" | "unpriced" | "estimatedUsd" | "providers" | "warnings">): RunAnalysisCost | null;
|
|
37
|
+
/** The recorded estimate, or a supported zero for a dry run with no cost record. */
|
|
38
|
+
export declare function runEstimateUsd(entry: {
|
|
39
|
+
estimatedCostUsd?: number | null | undefined;
|
|
40
|
+
mode?: string | null | undefined;
|
|
41
|
+
}): number | null;
|
|
42
|
+
/** Add known estimates, retaining unknown when neither component has a price. */
|
|
43
|
+
export declare function sumEstimatedUsd(a: number | null, b: number | null): number | null;
|
|
37
44
|
export declare function runCost(subtotal: RunCostSubtotal | null | undefined, analysis?: RunAnalysisCost | null): RunCost;
|
|
38
45
|
/**
|
|
39
46
|
* A run index entry's cost: its participants-and-desktops figure, complete or a lower bound when
|
package/dist/run/run-cost.js
CHANGED
|
@@ -11,25 +11,45 @@ export function analysisCostOf(accounting) {
|
|
|
11
11
|
providers: [...accounting.providers],
|
|
12
12
|
};
|
|
13
13
|
}
|
|
14
|
+
/** The recorded estimate, or a supported zero for a dry run with no cost record. */
|
|
15
|
+
export function runEstimateUsd(entry) {
|
|
16
|
+
const usd = entry.estimatedCostUsd;
|
|
17
|
+
if (typeof usd === "number" && Number.isFinite(usd) && usd >= 0)
|
|
18
|
+
return usd;
|
|
19
|
+
return usd === undefined && entry.mode === "dry-run" ? 0 : null;
|
|
20
|
+
}
|
|
21
|
+
/** Add known estimates, retaining unknown when neither component has a price. */
|
|
22
|
+
export function sumEstimatedUsd(a, b) {
|
|
23
|
+
const left = runEstimateUsd({ estimatedCostUsd: a });
|
|
24
|
+
const right = runEstimateUsd({ estimatedCostUsd: b });
|
|
25
|
+
return left === null && right === null ? null : round6((left ?? 0) + (right ?? 0));
|
|
26
|
+
}
|
|
14
27
|
export function runCost(subtotal, analysis) {
|
|
15
28
|
if (subtotal === null || subtotal === undefined)
|
|
16
29
|
return { run: null, analysis: null, total: null };
|
|
17
30
|
const run = {
|
|
18
|
-
usd:
|
|
31
|
+
usd: runEstimateUsd({ estimatedCostUsd: subtotal.estimatedTotalUsd }),
|
|
19
32
|
complete: subtotal.fullyEstimated === undefined ? null : subtotal.fullyEstimated,
|
|
20
33
|
ratesAsOf: subtotal.ratesAsOf ?? null,
|
|
21
34
|
placeholder: subtotal.placeholder === true,
|
|
22
35
|
};
|
|
23
|
-
const spent = analysis
|
|
24
|
-
|
|
25
|
-
|
|
36
|
+
const spent = analysis == null
|
|
37
|
+
? null
|
|
38
|
+
: {
|
|
39
|
+
...analysis,
|
|
40
|
+
estimatedUsd: runEstimateUsd({ estimatedCostUsd: analysis.estimatedUsd }),
|
|
41
|
+
complete: analysis.complete &&
|
|
42
|
+
runEstimateUsd({ estimatedCostUsd: analysis.estimatedUsd }) !== null,
|
|
43
|
+
};
|
|
44
|
+
const totalUsd = sumEstimatedUsd(run.usd, spent?.estimatedUsd ?? null);
|
|
45
|
+
if (totalUsd === null)
|
|
26
46
|
return { run, analysis: spent, total: null };
|
|
27
47
|
const lowerBound = run.usd === null || run.complete === false || spent?.complete === false;
|
|
28
48
|
return {
|
|
29
49
|
run,
|
|
30
50
|
analysis: spent,
|
|
31
51
|
total: {
|
|
32
|
-
usd:
|
|
52
|
+
usd: totalUsd,
|
|
33
53
|
complete: lowerBound ? false : run.complete,
|
|
34
54
|
},
|
|
35
55
|
};
|
package/dist/run/run-index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AnalysisCostCache } from "./run-index-analysis.js";
|
|
2
2
|
import type { RunAnalysisCost } from "./run-cost.js";
|
|
3
|
-
import { type RunLiveness
|
|
3
|
+
import { type RunLiveness } from "./status.js";
|
|
4
4
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
5
5
|
import type { ExecutionFailure } from "./judge.js";
|
|
6
6
|
declare const RUN_INDEX_SCHEMA = "humanish.run-index.v1";
|
|
@@ -51,13 +51,6 @@ export interface RunIndexResult {
|
|
|
51
51
|
/** Directories that could not be read at all, by name: surfaced, never silently dropped. */
|
|
52
52
|
unreadable: string[];
|
|
53
53
|
}
|
|
54
|
-
/** What a cached entry was derived from, so a changed file invalidates exactly that entry. */
|
|
55
|
-
interface CacheKey {
|
|
56
|
-
file: string;
|
|
57
|
-
mtimeMs: number;
|
|
58
|
-
size: number;
|
|
59
|
-
ino: number;
|
|
60
|
-
}
|
|
61
54
|
/**
|
|
62
55
|
* A process-lifetime cache. Deliberately explicit rather than module-global state: a caller that
|
|
63
56
|
* refreshes on a cadence keeps one and passes it back, and a caller that wants a cold read passes
|
|
@@ -67,51 +60,12 @@ export declare class RunIndexCache {
|
|
|
67
60
|
private readonly slots;
|
|
68
61
|
/** Analysis costs, keyed on their own fingerprint: analysis lands after the status record. */
|
|
69
62
|
readonly analysis: AnalysisCostCache;
|
|
70
|
-
get(runId: string, key:
|
|
71
|
-
set(runId: string, key:
|
|
63
|
+
get(runId: string, key: string): RunIndexEntry | undefined;
|
|
64
|
+
set(runId: string, key: string, entry: RunIndexEntry): void;
|
|
72
65
|
/** Drop entries for runs that no longer exist, so a long-lived surface cannot leak. */
|
|
73
66
|
retain(runIds: Iterable<string>): void;
|
|
74
67
|
get size(): number;
|
|
75
68
|
}
|
|
76
|
-
/** The legacy shape this reads out of a bundle. Narrow on purpose: only listing facts. */
|
|
77
|
-
interface BundleFacts {
|
|
78
|
-
runId?: string;
|
|
79
|
-
mode?: string;
|
|
80
|
-
createdAt?: string;
|
|
81
|
-
study?: unknown;
|
|
82
|
-
lab?: unknown;
|
|
83
|
-
persona?: {
|
|
84
|
-
source?: string;
|
|
85
|
-
};
|
|
86
|
-
scenario?: {
|
|
87
|
-
source?: string;
|
|
88
|
-
};
|
|
89
|
-
simulations?: {
|
|
90
|
-
status?: string;
|
|
91
|
-
}[];
|
|
92
|
-
review?: {
|
|
93
|
-
verdict?: string;
|
|
94
|
-
participants?: {
|
|
95
|
-
total: number;
|
|
96
|
-
reachedGoal: number;
|
|
97
|
-
reportedFriction?: number;
|
|
98
|
-
};
|
|
99
|
-
};
|
|
100
|
-
cost?: {
|
|
101
|
-
estimatedTotalUsd?: number | null;
|
|
102
|
-
fullyEstimated?: unknown;
|
|
103
|
-
};
|
|
104
|
-
outcome?: unknown;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* One run's liveness from files already read: its status record when usable, else its bundle.
|
|
108
|
-
* `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
|
|
109
|
-
* is the status record the liveness came from, when there was one.
|
|
110
|
-
*/
|
|
111
|
-
export declare function runLiveness(runId: string, status: unknown, bundle: Pick<BundleFacts, "simulations" | "outcome">, nowMs: number): {
|
|
112
|
-
liveness: RunLiveness;
|
|
113
|
-
record?: RunStatusRecord;
|
|
114
|
-
};
|
|
115
69
|
export interface ReadRunIndexOptions {
|
|
116
70
|
/** Reused across refreshes so unchanged runs are not re-read. */
|
|
117
71
|
cache?: RunIndexCache;
|
|
@@ -122,7 +76,7 @@ export interface ReadRunIndexOptions {
|
|
|
122
76
|
limit?: number;
|
|
123
77
|
}
|
|
124
78
|
/**
|
|
125
|
-
* Read every run in `.humanish/runs`,
|
|
79
|
+
* Read every run in `.humanish/runs`, from its cached source files. Never throws for a bad run directory;
|
|
126
80
|
* an unreadable one is named in `unreadable`.
|
|
127
81
|
*/
|
|
128
82
|
export declare function readRunIndex(cwdInput: string, options?: ReadRunIndexOptions): Promise<RunIndexResult>;
|
package/dist/run/run-index.js
CHANGED
|
@@ -1,20 +1,9 @@
|
|
|
1
|
-
// The run index
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// then parses each `run.json`: measured 197ms cold / 152ms warm at 25 runs. That is fine for a
|
|
5
|
-
// command that prints once and exits, and much too hot for a surface that refreshes on a cadence
|
|
6
|
-
// over SSH. This module reads the small `status.json` record each run now writes (586 bytes beside
|
|
7
|
-
// a 92KB bundle) and caches per-run entries keyed on the stat of the file each was derived from.
|
|
8
|
-
//
|
|
9
|
-
// Rules, unchanged from the record itself:
|
|
10
|
-
// - `run.json` is the evidence-of-record. Every field here is a projection for listing and
|
|
11
|
-
// classification; nothing here is a claim about what a participant did.
|
|
12
|
-
// - A run with no status record is not assumed finished. It is classified from what is on disk:
|
|
13
|
-
// a bundle means finished, receipts without a bundle mean interrupted. That is the
|
|
14
|
-
// reading of a run whose process died (and, before this contract existed, of every run).
|
|
15
|
-
// - One unreadable run directory degrades that run, never the listing.
|
|
1
|
+
// The run index caches listing facts by both source files. run.json's outcome wins over status,
|
|
2
|
+
// including when a process stopped between publishing its bundle and finalizing its heartbeat.
|
|
3
|
+
// A status record supplies freshness only when the bundle has no outcome.
|
|
16
4
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
17
5
|
import path from "node:path";
|
|
6
|
+
import { runLiveness } from "./liveness.js";
|
|
18
7
|
import { AnalysisCostCache, readIndexedAnalysisCost } from "./run-index-analysis.js";
|
|
19
8
|
import { RUN_STATUS_FILE, classifyRunStatus, isRunStatusRecord, } from "./status.js";
|
|
20
9
|
import { studyProvenanceOf } from "./study-provenance.js";
|
|
@@ -34,11 +23,7 @@ export class RunIndexCache {
|
|
|
34
23
|
const slot = this.slots.get(runId);
|
|
35
24
|
if (slot === undefined)
|
|
36
25
|
return undefined;
|
|
37
|
-
|
|
38
|
-
slot.key.mtimeMs === key.mtimeMs &&
|
|
39
|
-
slot.key.size === key.size &&
|
|
40
|
-
slot.key.ino === key.ino;
|
|
41
|
-
return same ? slot.entry : undefined;
|
|
26
|
+
return slot.key === key ? slot.entry : undefined;
|
|
42
27
|
}
|
|
43
28
|
set(runId, key, entry) {
|
|
44
29
|
this.slots.set(runId, { key, entry });
|
|
@@ -81,7 +66,7 @@ function entryFromStatus(record, nowMs) {
|
|
|
81
66
|
return {
|
|
82
67
|
runId: record.runId,
|
|
83
68
|
derivedFrom: "status",
|
|
84
|
-
liveness:
|
|
69
|
+
liveness: runLiveness(record.runId, record, {}, nowMs).liveness,
|
|
85
70
|
mode: record.mode,
|
|
86
71
|
...(typeof record.pid === "number" ? { pid: record.pid } : {}),
|
|
87
72
|
...studyEntry(studyProvenanceOf(record)),
|
|
@@ -109,29 +94,6 @@ function entryFromStatus(record, nowMs) {
|
|
|
109
94
|
function usableStatusRecord(raw, runId) {
|
|
110
95
|
return isRunStatusRecord(raw) && raw.runId === runId;
|
|
111
96
|
}
|
|
112
|
-
/**
|
|
113
|
-
* The bundle-only reading, for a run with no usable status record. A bundle on disk usually means
|
|
114
|
-
* the run reached its final write. But a live run now flushes an in-progress bundle as it goes (so
|
|
115
|
-
* anything asking what a participant is doing has something to read), and that bundle marks its
|
|
116
|
-
* simulations `running`. With no status record there is no freshness to judge, and the
|
|
117
|
-
* reading of "it started, and nothing here says it finished" is interrupted, not finished.
|
|
118
|
-
*/
|
|
119
|
-
function bundleLiveness(bundle) {
|
|
120
|
-
if (isRunOutcome(bundle.outcome))
|
|
121
|
-
return bundle.outcome.state;
|
|
122
|
-
const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
|
|
123
|
-
return inProgress ? "interrupted" : "finished";
|
|
124
|
-
}
|
|
125
|
-
/**
|
|
126
|
-
* One run's liveness from files already read: its status record when usable, else its bundle.
|
|
127
|
-
* `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
|
|
128
|
-
* is the status record the liveness came from, when there was one.
|
|
129
|
-
*/
|
|
130
|
-
export function runLiveness(runId, status, bundle, nowMs) {
|
|
131
|
-
return usableStatusRecord(status, runId)
|
|
132
|
-
? { liveness: classifyRunStatus(status, nowMs), record: status }
|
|
133
|
-
: { liveness: bundleLiveness(bundle) };
|
|
134
|
-
}
|
|
135
97
|
function studyEntry(study) {
|
|
136
98
|
return study === undefined ? {} : { study };
|
|
137
99
|
}
|
|
@@ -141,7 +103,7 @@ function entryFromBundle(runId, bundle) {
|
|
|
141
103
|
return {
|
|
142
104
|
runId,
|
|
143
105
|
derivedFrom: "bundle",
|
|
144
|
-
liveness:
|
|
106
|
+
liveness: runLiveness(runId, undefined, { simulations: bundle.simulations, outcome }, Date.now()).liveness,
|
|
145
107
|
...(bundle.mode === "dry-run" || bundle.mode === "live" ? { mode: bundle.mode } : {}),
|
|
146
108
|
...studyEntry(studyProvenanceOf(bundle)),
|
|
147
109
|
...(bundle.createdAt === undefined ? {} : { startedAt: bundle.createdAt }),
|
|
@@ -160,7 +122,7 @@ function entryFromBundle(runId, bundle) {
|
|
|
160
122
|
};
|
|
161
123
|
}
|
|
162
124
|
/**
|
|
163
|
-
* Read every run in `.humanish/runs`,
|
|
125
|
+
* Read every run in `.humanish/runs`, from its cached source files. Never throws for a bad run directory;
|
|
164
126
|
* an unreadable one is named in `unreadable`.
|
|
165
127
|
*/
|
|
166
128
|
export async function readRunIndex(cwdInput, options = {}) {
|
|
@@ -199,52 +161,60 @@ export async function readRunIndex(cwdInput, options = {}) {
|
|
|
199
161
|
unreadable,
|
|
200
162
|
};
|
|
201
163
|
}
|
|
202
|
-
/** One run's entry,
|
|
164
|
+
/** One run's entry, from its cached source files, or "unreadable" when its bundle cannot be parsed. */
|
|
203
165
|
async function readEntry(runId, runDir, cache, nowMs) {
|
|
204
166
|
const statusFile = path.join(runDir, RUN_STATUS_FILE);
|
|
205
167
|
const bundleFile = path.join(runDir, RUN_BUNDLE_FILE);
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
const
|
|
209
|
-
if (
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
168
|
+
const [statusKey, bundleKey] = await Promise.all([statKey(statusFile), statKey(bundleFile)]);
|
|
169
|
+
const key = JSON.stringify([statusKey, bundleKey]);
|
|
170
|
+
const cached = cache?.get(runId, key);
|
|
171
|
+
if (cached !== undefined) {
|
|
172
|
+
return cached.derivedFrom === "status" &&
|
|
173
|
+
cached.updatedAt !== undefined &&
|
|
174
|
+
cached.completedAt === undefined
|
|
175
|
+
? {
|
|
176
|
+
...cached,
|
|
177
|
+
liveness: classifyRunStatus({ state: "running", updatedAt: cached.updatedAt }, nowMs),
|
|
178
|
+
}
|
|
179
|
+
: cached;
|
|
180
|
+
}
|
|
181
|
+
const [status, raw] = await Promise.all([
|
|
182
|
+
statusKey === null ? undefined : readJson(statusFile),
|
|
183
|
+
bundleKey === null ? undefined : readJson(bundleFile),
|
|
184
|
+
]);
|
|
185
|
+
const record = usableStatusRecord(status, runId) ? status : undefined;
|
|
186
|
+
const bundle = raw !== null && typeof raw === "object" ? raw : undefined;
|
|
187
|
+
let entry;
|
|
188
|
+
if (bundle !== undefined && isRunOutcome(bundle.outcome)) {
|
|
189
|
+
const metadata = record === undefined ? undefined : entryFromStatus(record, nowMs);
|
|
190
|
+
entry = {
|
|
191
|
+
...(metadata === undefined
|
|
192
|
+
? {}
|
|
193
|
+
: {
|
|
194
|
+
pid: metadata.pid,
|
|
195
|
+
startedAt: metadata.startedAt,
|
|
196
|
+
updatedAt: metadata.updatedAt,
|
|
197
|
+
...(metadata.completedAt === undefined ? {} : { completedAt: metadata.completedAt }),
|
|
198
|
+
...(metadata.durationMs === undefined ? {} : { durationMs: metadata.durationMs }),
|
|
199
|
+
...studyEntry(metadata.study),
|
|
200
|
+
}),
|
|
201
|
+
...entryFromBundle(runId, bundle),
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
else if (record !== undefined) {
|
|
205
|
+
entry = entryFromStatus(record, nowMs);
|
|
206
|
+
}
|
|
207
|
+
else if (bundle !== undefined) {
|
|
208
|
+
entry = entryFromBundle(runId, bundle);
|
|
209
|
+
}
|
|
210
|
+
else if (bundleKey !== null) {
|
|
211
|
+
return "unreadable";
|
|
230
212
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
const bundleKey = await statKey(bundleFile);
|
|
234
|
-
if (bundleKey !== null) {
|
|
235
|
-
const cached = cache?.get(runId, bundleKey);
|
|
236
|
-
if (cached !== undefined)
|
|
237
|
-
return cached;
|
|
238
|
-
const raw = await readJson(bundleFile);
|
|
239
|
-
if (raw === null || typeof raw !== "object")
|
|
240
|
-
return "unreadable";
|
|
241
|
-
const entry = entryFromBundle(runId, raw);
|
|
242
|
-
cache?.set(runId, bundleKey, entry);
|
|
243
|
-
return entry;
|
|
213
|
+
else {
|
|
214
|
+
return { runId, derivedFrom: "directory", liveness: "interrupted" };
|
|
244
215
|
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
return { runId, derivedFrom: "directory", liveness: "interrupted" };
|
|
216
|
+
cache?.set(runId, key, entry);
|
|
217
|
+
return entry;
|
|
248
218
|
}
|
|
249
219
|
/** Newest-first ordering: the most recent thing known about a run, else its id's own timestamp. */
|
|
250
220
|
function sortKey(entry) {
|
package/dist/run/run.d.ts
CHANGED
|
@@ -15,7 +15,7 @@ interface StartRunOptions {
|
|
|
15
15
|
/** `none` when the route creates no sandbox for this run; status.json records it for reclaim. */
|
|
16
16
|
sandboxes?: "none" | undefined;
|
|
17
17
|
/** review.md for the published bundle. */
|
|
18
|
-
renderReview: (bundle: RunBundle) => string;
|
|
18
|
+
renderReview: (bundle: RunBundle, status?: unknown) => string;
|
|
19
19
|
/** Used by `FinishedRun.renderObserver`; `render` is the `StudyDeps.renderObserver` seam. */
|
|
20
20
|
observer?: {
|
|
21
21
|
open: boolean;
|
|
@@ -53,7 +53,7 @@ interface RouteRunOptions {
|
|
|
53
53
|
cwd: string;
|
|
54
54
|
prefix: RunIdPrefix;
|
|
55
55
|
/** review.md for the published bundle. */
|
|
56
|
-
renderReview: (bundle: RunBundle) => string;
|
|
56
|
+
renderReview: (bundle: RunBundle, status?: unknown) => string;
|
|
57
57
|
/** `none` when the route creates no sandbox for this run; status.json records it for reclaim. */
|
|
58
58
|
sandboxes?: "none" | undefined;
|
|
59
59
|
/** Clock for `createdAt` and the latest pointer. The minted id reads the wall clock. */
|
package/dist/run/run.js
CHANGED
|
@@ -14,7 +14,8 @@ import { registerActiveRun } from "./active-runs.js";
|
|
|
14
14
|
import { writeContainedOutputFile, writePreparedRunLatestPointer } from "./contained-output.js";
|
|
15
15
|
import { scrubRunSandboxIds, withPublicSandboxIds } from "./sandbox-ids.js";
|
|
16
16
|
import { RunSecrets } from "./secrets.js";
|
|
17
|
-
import { beginRunStatus, runStatusOutcome, } from "./status.js";
|
|
17
|
+
import { RUN_STATUS_FILE, beginRunStatus, runStatusOutcome, } from "./status.js";
|
|
18
|
+
import { readRunJsonIfExists } from "./locate.js";
|
|
18
19
|
/** An entry point called in a state that does not admit it. */
|
|
19
20
|
class RunLifecycleError extends Error {
|
|
20
21
|
name = "RunLifecycleError";
|
|
@@ -165,11 +166,12 @@ function runPublisher(args) {
|
|
|
165
166
|
});
|
|
166
167
|
await writeContainedOutputFile(paths, RUN_BUNDLE_FILE, json(publicBundle), "utf8");
|
|
167
168
|
await afterBundle(publicBundle);
|
|
169
|
+
const status = recorded === undefined ? await readRunJsonIfExists(paths, RUN_STATUS_FILE) : undefined;
|
|
168
170
|
await writeContainedOutputFile(paths, "review.json", json(publicBundle.review), "utf8");
|
|
169
|
-
await writeContainedOutputFile(paths, "review.md", options.renderReview(publicBundle), "utf8");
|
|
171
|
+
await writeContainedOutputFile(paths, "review.md", options.renderReview(publicBundle, status), "utf8");
|
|
170
172
|
const events = publicBundle.events.map((event) => JSON.stringify(event)).join("\n");
|
|
171
173
|
await writeContainedOutputFile(paths, "events.ndjson", `${events}\n`, "utf8");
|
|
172
|
-
await writeContainedOutputFile(paths, "observer/observer-data.json", json(buildObserverData(publicBundle)), "utf8");
|
|
174
|
+
await writeContainedOutputFile(paths, "observer/observer-data.json", json(buildObserverData(publicBundle, undefined, status)), "utf8");
|
|
173
175
|
};
|
|
174
176
|
const snapshot = async (bundle) => {
|
|
175
177
|
await writeBundleFiles(bundle, undefined, async () => { });
|
package/dist/run/stats.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import { runDisplay, runDisplayLabel } from "./display.js";
|
|
6
6
|
import { readRunIndex } from "./run-index.js";
|
|
7
7
|
import { addCostTotals, emptyCostTotals, readCostTotals, } from "./costs.js";
|
|
8
|
+
import { runEstimateUsd } from "./run-cost.js";
|
|
8
9
|
import { round6 } from "./pricing.js";
|
|
9
10
|
import { plural } from "./text.js";
|
|
10
11
|
const STATS_SCHEMA = "humanish.stats.v2";
|
|
@@ -27,17 +28,6 @@ function addParticipants(into, entry) {
|
|
|
27
28
|
into.reachedGoal += entry.participants.reachedGoal;
|
|
28
29
|
into.reportedFriction += entry.participants.reportedFriction ?? 0;
|
|
29
30
|
}
|
|
30
|
-
/**
|
|
31
|
-
* A run's participant and desktop estimate: its recorded figure, or $0 for a dry run that records
|
|
32
|
-
* none, since a dry run makes no model request and creates no desktop. Undefined when the cost is
|
|
33
|
-
* unknown, including a dry run that records an unknown (null) figure.
|
|
34
|
-
*/
|
|
35
|
-
function runEstimateUsd(entry) {
|
|
36
|
-
const usd = entry.estimatedCostUsd;
|
|
37
|
-
if (typeof usd === "number" && Number.isFinite(usd) && usd >= 0)
|
|
38
|
-
return usd;
|
|
39
|
-
return usd === undefined && entry.mode === "dry-run" ? 0 : undefined;
|
|
40
|
-
}
|
|
41
31
|
function entryTime(entry) {
|
|
42
32
|
return entry.startedAt ?? entry.completedAt ?? entry.updatedAt;
|
|
43
33
|
}
|
|
@@ -175,7 +165,7 @@ export async function computeStats(cwdInput, options = {}) {
|
|
|
175
165
|
totals.dryRun += 1;
|
|
176
166
|
if (entry.liveness === "running")
|
|
177
167
|
totals.running += 1;
|
|
178
|
-
const runUsd = runEstimateUsd(entry);
|
|
168
|
+
const runUsd = runEstimateUsd(entry) ?? undefined;
|
|
179
169
|
if (runUsd === undefined)
|
|
180
170
|
totals.unpricedRuns += 1;
|
|
181
171
|
else
|