humanish 0.126.0 → 0.127.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/actors/closing-report.d.ts +8 -1
- package/dist/actors/closing-report.js +4 -1
- package/dist/actors/codex/restricted-participant-policy.js +2 -5
- package/dist/actors/computer-use/loop/debrief.js +6 -5
- package/dist/actors/computer-use/loop/ending.d.ts +8 -5
- package/dist/actors/computer-use/loop/ending.js +10 -7
- package/dist/actors/computer-use/loop.js +2 -2
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/contract.d.ts +10 -6
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/cli/commands/study-format.js +13 -2
- package/dist/observer/data.js +8 -5
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +28 -6
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-runs.d.ts +9 -9
- package/dist/routes/computer-use/participant-runs.js +52 -86
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +26 -16
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.js +1 -1
- package/dist/routes/computer-use/single-bundle.js +9 -1
- package/dist/routes/computer-use/types.d.ts +2 -3
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +1 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +3 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/judge.d.ts +7 -0
- package/dist/run/judge.js +4 -4
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/study/arrivals.d.ts +84 -31
- package/dist/study/arrivals.js +165 -86
- package/dist/study/bundle-estimate.d.ts +8 -7
- package/dist/study/bundle-estimate.js +131 -70
- package/dist/study/desktop-files.js +4 -4
- package/dist/study/plan-participants.d.ts +4 -9
- package/dist/study/plan-participants.js +2 -14
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +6 -4
- package/dist/study/types.d.ts +2 -1
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/docs/architecture/actor-contract.md +7 -3
- package/docs/contracts/run-bundle.md +1 -1
- package/docs/contracts/schemas.md +24 -11
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,30 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.127.1: Clock-end closing reports declare their outcome; skipped participants named as skipped (2026-10-13)
|
|
12
|
+
|
|
13
|
+
humanish 0.127.1 fixes two things the 0.127.0 release smoke found in scheduled studies. An OpenAI
|
|
14
|
+
participant asked for its closing report at a scheduled end or session cap now says how the task
|
|
15
|
+
went in a structured `outcome` field, so one that reached its goal counts in `reachedGoal` where
|
|
16
|
+
0.127.0 counted it in `ranUntilEnd`. A participant the run skips before its session starts now
|
|
17
|
+
shows its skip reason on its run line, and the summary counts it apart from those that took
|
|
18
|
+
part.
|
|
19
|
+
|
|
20
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.127.1)
|
|
21
|
+
|
|
22
|
+
## 0.127.0: Clock-end passes counted by what participants declared, a scaled wind-down, the concurrency cap in the plan (2026-10-13)
|
|
23
|
+
|
|
24
|
+
humanish 0.127.0 makes long scheduled studies report what happened. A participant that passes at
|
|
25
|
+
its scheduled end or session cap without declaring `REACHED THE GOAL.` is now counted as running
|
|
26
|
+
until that end (`participants.ranUntilEnd`), where 0.126.0 counted it as reaching the goal. The
|
|
27
|
+
default wind-down is a quarter of the window, at most 10 minutes, so a short window is no longer
|
|
28
|
+
told its end from its first turn. `HUMANISH_CUA_MAX_CONCURRENCY` now lowers the plan itself, on
|
|
29
|
+
shared-world studies too, and a participant whose scheduled end passes while it waits for a slot
|
|
30
|
+
is skipped before its desktop is created. The Observer keeps updating past 100,000 items for a
|
|
31
|
+
participant, and the bundle estimate is re-derived for run bundle v3.
|
|
32
|
+
|
|
33
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.127.0)
|
|
34
|
+
|
|
11
35
|
## 0.126.0: Per-participant time zones, session caps and desktop files, page paths, a lighter live Observer (2026-10-13)
|
|
12
36
|
|
|
13
37
|
humanish 0.126.0 adds per-participant settings for long, mixed studies and a lighter live Observer.
|
|
@@ -23,10 +23,17 @@ export declare const participantImpressionsSchema: z.ZodArray<z.ZodObject<{
|
|
|
23
23
|
text: z.ZodString;
|
|
24
24
|
}, z.core.$strict>>;
|
|
25
25
|
/**
|
|
26
|
-
* A closing report.
|
|
26
|
+
* A closing report. `outcome` is how the participant says the task went: `reached` when it is
|
|
27
|
+
* finished, `blocked` when something in the app stopped it, `not_reached` for any other reason.
|
|
28
|
+
* A report without an outcome or impressions is still valid: third-party provider ports and
|
|
27
29
|
* stored traces do not have them.
|
|
28
30
|
*/
|
|
29
31
|
export declare const closingReportSchema: z.ZodObject<{
|
|
32
|
+
outcome: z.ZodOptional<z.ZodEnum<{
|
|
33
|
+
blocked: "blocked";
|
|
34
|
+
not_reached: "not_reached";
|
|
35
|
+
reached: "reached";
|
|
36
|
+
}>>;
|
|
30
37
|
summary: z.ZodString;
|
|
31
38
|
frictionReports: z.ZodArray<z.ZodString>;
|
|
32
39
|
impressions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -32,10 +32,13 @@ const impressionSchema = z.strictObject({
|
|
|
32
32
|
/** The impressions a participant gave; an empty list means it had none. */
|
|
33
33
|
export const participantImpressionsSchema = z.array(impressionSchema).max(LIMITS.impressions);
|
|
34
34
|
/**
|
|
35
|
-
* A closing report.
|
|
35
|
+
* A closing report. `outcome` is how the participant says the task went: `reached` when it is
|
|
36
|
+
* finished, `blocked` when something in the app stopped it, `not_reached` for any other reason.
|
|
37
|
+
* A report without an outcome or impressions is still valid: third-party provider ports and
|
|
36
38
|
* stored traces do not have them.
|
|
37
39
|
*/
|
|
38
40
|
export const closingReportSchema = z.strictObject({
|
|
41
|
+
outcome: z.enum(["reached", "not_reached", "blocked"]).optional(),
|
|
39
42
|
summary: said(LIMITS.summaryChars),
|
|
40
43
|
frictionReports: z.array(said(LIMITS.frictionReportChars)).max(LIMITS.frictionReports),
|
|
41
44
|
impressions: participantImpressionsSchema.optional(),
|
|
@@ -35,11 +35,8 @@ export function participantToolSchema(speechEnabled = false) {
|
|
|
35
35
|
return z.toJSONSchema(toolInput(speechEnabled), { io: "input" });
|
|
36
36
|
}
|
|
37
37
|
export const PARTICIPANT_TOOL_SCHEMA = participantToolSchema();
|
|
38
|
-
/** The participant's final account: how the task went
|
|
39
|
-
const participantFinal =
|
|
40
|
-
outcome: z.enum(["reached", "not_reached", "blocked"]),
|
|
41
|
-
...closingReportSchema.shape,
|
|
42
|
-
});
|
|
38
|
+
/** The participant's final account: its closing report, with how the task went required. */
|
|
39
|
+
const participantFinal = closingReportSchema.required({ outcome: true });
|
|
43
40
|
export const PARTICIPANT_FINAL_SCHEMA = strictOutputSchema(participantFinal);
|
|
44
41
|
/** humanish tool arguments, validated before the shared executor sees a batch. */
|
|
45
42
|
export function parseParticipantTool(value, speechEnabled = false) {
|
|
@@ -7,7 +7,7 @@ import { CuaDeadlineError, raceSessionDeadline, requestScope } from "./race.js";
|
|
|
7
7
|
import { notice } from "./trace.js";
|
|
8
8
|
import { isCompleteTurnUsage } from "./usage.js";
|
|
9
9
|
const DEBRIEF_REQUEST_CAP_MS = 30_000;
|
|
10
|
-
const DEBRIEF_HINT = `The interactive session has ended. Return a closing account with summary, frictionReports and impressions. In summary, briefly describe only what you actually did and observed. In frictionReports, list only specific unexpected behavior, confusion, or recovery you personally encountered during this session. Preserve uncertainty. Use an empty list if you encountered none. ${IMPRESSIONS_ASK} Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions. This is a closing account, not another attempt at the task.`;
|
|
10
|
+
const DEBRIEF_HINT = `The interactive session has ended. Return a closing account with outcome, summary, frictionReports and impressions. In outcome, say how the task went: reached if you finished it, blocked if something in the app stopped you, not_reached if you did not finish it for any other reason, such as running out of time. In summary, briefly describe only what you actually did and observed. In frictionReports, list only specific unexpected behavior, confusion, or recovery you personally encountered during this session. Preserve uncertainty. Use an empty list if you encountered none. ${IMPRESSIONS_ASK} Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions. This is a closing account, not another attempt at the task.`;
|
|
11
11
|
const CLOSING_REPORT = {
|
|
12
12
|
call: (provider) => provider.debrief,
|
|
13
13
|
contextHint: DEBRIEF_HINT,
|
|
@@ -156,9 +156,10 @@ function acceptDebriefTurn(session, turn, trigger, ask, record) {
|
|
|
156
156
|
* A closing report: its summary and friction as one participant message, then its impressions.
|
|
157
157
|
* At a scheduled end or a session cap an accepted report is the participant's account of the
|
|
158
158
|
* session: its summary becomes the session's reason (loopResult), so the outcome it declares
|
|
159
|
-
* becomes the session's declared outcome.
|
|
160
|
-
*
|
|
161
|
-
*
|
|
159
|
+
* becomes the session's declared outcome. That is its `outcome` field, then the turn's (a
|
|
160
|
+
* provider port that sets it there), then the first line of its summary. The blocker rule reads a
|
|
161
|
+
* declared `reached` or `blocked` before the summary's wording. After a matched stop condition or
|
|
162
|
+
* dwell window, the condition stays the completion evidence.
|
|
162
163
|
*/
|
|
163
164
|
function readClosingReport(session, turn, trigger) {
|
|
164
165
|
const closing = closingReportSchema.safeParse(turn.closingReport);
|
|
@@ -182,7 +183,7 @@ function readClosingReport(session, turn, trigger) {
|
|
|
182
183
|
session.trace.bump("messages");
|
|
183
184
|
session.impressions = recordImpressions(session, closing.data.impressions);
|
|
184
185
|
if (trigger.kind === "scheduled_end" || trigger.kind === "session_cap")
|
|
185
|
-
session.declaredOutcome = declaredOutcomeOf(turn, closing.data.summary);
|
|
186
|
+
session.declaredOutcome = declaredOutcomeOf(turn, closing.data, closing.data.summary);
|
|
186
187
|
return {
|
|
187
188
|
status: "completed",
|
|
188
189
|
detail: "one read-only report; no additional desktop actions; original stop and task outcomes preserved",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../../contract.js";
|
|
1
|
+
import type { ActorCompletionReason, ActorStopCause, ParticipantClosingReport, ParticipantDeclaredOutcome } from "../../contract.js";
|
|
2
2
|
import type { StopConditionMatch } from "../../stop-conditions.js";
|
|
3
3
|
import type { LoopSession } from "./session.js";
|
|
4
4
|
import { type Evidence } from "./trace.js";
|
|
@@ -23,14 +23,17 @@ export interface Stop {
|
|
|
23
23
|
/**
|
|
24
24
|
* Read the fixed closing line the prompt asks for: the first non-empty line of the
|
|
25
25
|
* participant's last message or closing report summary, exactly one of three phrases,
|
|
26
|
-
* punctuation and case forgiven. Anything else is absence, never a guess.
|
|
26
|
+
* punctuation and case forgiven. Anything else is absence, never a guess. A `>` blockquote line
|
|
27
|
+
* is skipped: it quotes something else, such as the page, and the blocker scan
|
|
28
|
+
* (routes/computer-use/self-report.ts) drops it too. Exported for tests.
|
|
27
29
|
*/
|
|
28
30
|
export declare function declaredOutcomeFromClosingLine(message: string | undefined): ParticipantDeclaredOutcome | undefined;
|
|
29
31
|
/**
|
|
30
|
-
* The participant's own word for how it ended:
|
|
31
|
-
* line of `text`, its last message
|
|
32
|
+
* The participant's own word for how it ended: its closing report's `outcome`, then the turn's
|
|
33
|
+
* (Codex and provider ports that set it there), then the closing line of `text`, its last message
|
|
34
|
+
* or its closing report's summary. A self-ending turn and a closing request read it in this order.
|
|
32
35
|
*/
|
|
33
|
-
export declare function declaredOutcomeOf(turn: CuaTurn, text: string | undefined): ParticipantDeclaredOutcome | undefined;
|
|
36
|
+
export declare function declaredOutcomeOf(turn: CuaTurn, report: ParticipantClosingReport | undefined, text: string | undefined): ParticipantDeclaredOutcome | undefined;
|
|
34
37
|
/**
|
|
35
38
|
* The participant reported a natural endpoint. "not_reached" is a participant who stopped without
|
|
36
39
|
* finishing: gave_up, which tallies as abandoned. "blocked" keeps goal_satisfied here (the actor
|
|
@@ -7,7 +7,9 @@ import { plural } from "../../../run/text.js";
|
|
|
7
7
|
/**
|
|
8
8
|
* Read the fixed closing line the prompt asks for: the first non-empty line of the
|
|
9
9
|
* participant's last message or closing report summary, exactly one of three phrases,
|
|
10
|
-
* punctuation and case forgiven. Anything else is absence, never a guess.
|
|
10
|
+
* punctuation and case forgiven. Anything else is absence, never a guess. A `>` blockquote line
|
|
11
|
+
* is skipped: it quotes something else, such as the page, and the blocker scan
|
|
12
|
+
* (routes/computer-use/self-report.ts) drops it too. Exported for tests.
|
|
11
13
|
*/
|
|
12
14
|
export function declaredOutcomeFromClosingLine(message) {
|
|
13
15
|
if (message === undefined)
|
|
@@ -15,10 +17,10 @@ export function declaredOutcomeFromClosingLine(message) {
|
|
|
15
17
|
const first = message
|
|
16
18
|
.split(/\r?\n/)
|
|
17
19
|
.map((line) => line.trim())
|
|
18
|
-
.find((line) => line.length > 0);
|
|
20
|
+
.find((line) => line.length > 0 && !line.startsWith(">"));
|
|
19
21
|
if (first === undefined)
|
|
20
22
|
return undefined;
|
|
21
|
-
const normalized = first.replace(/^[*_
|
|
23
|
+
const normalized = first.replace(/^[*_#\s-]+|[*_.!\s]+$/g, "").toLowerCase();
|
|
22
24
|
if (normalized === "reached the goal")
|
|
23
25
|
return "reached";
|
|
24
26
|
if (normalized === "did not reach the goal")
|
|
@@ -28,11 +30,12 @@ export function declaredOutcomeFromClosingLine(message) {
|
|
|
28
30
|
return undefined;
|
|
29
31
|
}
|
|
30
32
|
/**
|
|
31
|
-
* The participant's own word for how it ended:
|
|
32
|
-
* line of `text`, its last message
|
|
33
|
+
* The participant's own word for how it ended: its closing report's `outcome`, then the turn's
|
|
34
|
+
* (Codex and provider ports that set it there), then the closing line of `text`, its last message
|
|
35
|
+
* or its closing report's summary. A self-ending turn and a closing request read it in this order.
|
|
33
36
|
*/
|
|
34
|
-
export function declaredOutcomeOf(turn, text) {
|
|
35
|
-
return turn.outcome ?? declaredOutcomeFromClosingLine(text);
|
|
37
|
+
export function declaredOutcomeOf(turn, report, text) {
|
|
38
|
+
return report?.outcome ?? turn.outcome ?? declaredOutcomeFromClosingLine(text);
|
|
36
39
|
}
|
|
37
40
|
/**
|
|
38
41
|
* The participant reported a natural endpoint. "not_reached" is a participant who stopped without
|
|
@@ -126,10 +126,10 @@ async function runTurns(session, conversation) {
|
|
|
126
126
|
continue;
|
|
127
127
|
}
|
|
128
128
|
if (turn.done || turn.actions.length === 0) {
|
|
129
|
+
const report = closingReportSchema.safeParse(turn.closingReport).data;
|
|
129
130
|
// The declared outcome is kept even if redacting the participant's summary fails.
|
|
130
|
-
session.declaredOutcome = declaredOutcomeOf(turn, turn.message);
|
|
131
|
+
session.declaredOutcome = declaredOutcomeOf(turn, report, turn.message);
|
|
131
132
|
const ended = stops.participantEnded(turn, session.declaredOutcome, (text) => session.redactNarration(text));
|
|
132
|
-
const report = closingReportSchema.safeParse(turn.closingReport).data;
|
|
133
133
|
// A provider that can ask for impressions alone does so after the session, like a debrief.
|
|
134
134
|
if (report?.impressions !== undefined || session.provider.requestImpressions === undefined) {
|
|
135
135
|
session.impressions =
|
|
@@ -512,7 +512,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
512
512
|
// Both are offered in both modes. A threaded request reads the session the server holds; an
|
|
513
513
|
// explicit_context one carries its newest screens whole and its oldest turns summarized past
|
|
514
514
|
// the budget, which the trace's conversation record counts. A participant that worked a long
|
|
515
|
-
//
|
|
515
|
+
// session writes its closing account from what it carries, as a person writes one from memory.
|
|
516
516
|
debrief: (req, signal) => requestTurn(req, signal, "report"),
|
|
517
517
|
requestImpressions: (req, signal) => requestTurn(req, signal, "impressions"),
|
|
518
518
|
};
|
|
@@ -25,10 +25,11 @@ export declare const ACTOR_STATUSES: readonly ["passed", "abandoned", "incomplet
|
|
|
25
25
|
export type ActorStatus = (typeof ACTOR_STATUSES)[number];
|
|
26
26
|
/**
|
|
27
27
|
* What the participant said happened, in a field rather than a paragraph. Providers whose
|
|
28
|
-
* reply is schema-constrained (the local-agent routes) fill it on their final turn
|
|
29
|
-
*
|
|
30
|
-
* the task is finished. `blocked`: something in the app
|
|
31
|
-
* the participant stopped for another reason (gave up,
|
|
28
|
+
* reply is schema-constrained (the local-agent routes) fill it on their final turn, and a closing
|
|
29
|
+
* report carries it as `outcome`; a free-text turn leaves it absent and the route falls back to
|
|
30
|
+
* reading the closing message. `reached`: the task is finished. `blocked`: something in the app
|
|
31
|
+
* stopped the participant. `not_reached`: the participant stopped for another reason (gave up,
|
|
32
|
+
* ran out of ideas).
|
|
32
33
|
*/
|
|
33
34
|
export type ParticipantDeclaredOutcome = "reached" | "not_reached" | "blocked";
|
|
34
35
|
/** Statuses that describe what happened to a participant rather than a harness malfunction. Verify
|
|
@@ -372,8 +373,11 @@ export interface ActorTrace {
|
|
|
372
373
|
reason: string;
|
|
373
374
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
|
374
375
|
usageReported?: boolean;
|
|
375
|
-
/**
|
|
376
|
-
|
|
376
|
+
/**
|
|
377
|
+
* The report's impressions are kept in `impressions`, with the messages that quote them, and
|
|
378
|
+
* the outcome it declared at a clock end in `declaredOutcome`.
|
|
379
|
+
*/
|
|
380
|
+
report?: Omit<ParticipantClosingReport, "impressions" | "outcome">;
|
|
377
381
|
/** Links the readable projection so it is not heuristically classified a second time. */
|
|
378
382
|
messageId?: string;
|
|
379
383
|
};
|
|
@@ -1,5 +1,35 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* The endpoint a computer-use completion came from, as analysis provenance records it
|
|
3
|
+
* (cuaEndpointSource). Saved analyses hash it, so a new value would make them invalid on read.
|
|
4
|
+
*/
|
|
5
|
+
export declare const CUA_ENDPOINT_SOURCES: readonly ["participant_report", "condition_matched", "unavailable"];
|
|
6
|
+
export type CuaEndpointSource = (typeof CUA_ENDPOINT_SOURCES)[number];
|
|
7
|
+
/**
|
|
8
|
+
* Presentation provenance for computer-use completion; never reclassifies the stored outcome.
|
|
9
|
+
* `scheduled_end` and `session_cap` name a session that passed at that clock end without the
|
|
10
|
+
* participant declaring it reached the goal (ranUntilClockEnd).
|
|
11
|
+
*/
|
|
12
|
+
export type CuaGoalSource = CuaEndpointSource | ClockEnd;
|
|
13
|
+
/** The two clock ends a computer-use session passes at: the stop causes of the same names. */
|
|
14
|
+
export type ClockEnd = "scheduled_end" | "session_cap";
|
|
15
|
+
export declare function isClockEnd(source: CuaGoalSource | undefined): source is ClockEnd;
|
|
16
|
+
/**
|
|
17
|
+
* The clock end a session ran until, when the participant did not declare that it reached the
|
|
18
|
+
* goal: its scheduled end or its session cap. Such a session ends as passed, because ending at
|
|
19
|
+
* the clock is the plan, and its closing report says `DID NOT REACH THE GOAL.` or declares
|
|
20
|
+
* nothing. The tally and every completion label read this, so none counts it as reaching the
|
|
21
|
+
* goal. Undefined for every other ending, including a declared `reached` at a clock end.
|
|
22
|
+
*/
|
|
23
|
+
export declare function ranUntilClockEnd(trace: {
|
|
24
|
+
readonly stopCause?: unknown;
|
|
25
|
+
readonly declaredOutcome?: unknown;
|
|
26
|
+
}): ClockEnd | undefined;
|
|
3
27
|
export declare function isCuaTrace(value: unknown): boolean;
|
|
4
28
|
export declare function cuaGoalSource(value: unknown, recordedStatus?: unknown): CuaGoalSource | undefined;
|
|
29
|
+
/**
|
|
30
|
+
* cuaGoalSource for analysis provenance. A clock end reads as it did before it had a source of
|
|
31
|
+
* its own, so an analysis saved then still matches its run when re-read. The provenance's
|
|
32
|
+
* `stopCause` and `declaredOutcome` name the clock end.
|
|
33
|
+
*/
|
|
34
|
+
export declare function cuaEndpointSource(value: unknown, recordedStatus?: unknown): CuaEndpointSource | undefined;
|
|
5
35
|
export declare const CUA_COMPLETION_NOTE = "Participant reports alone do not establish task success. A matched stop condition establishes only its declared condition. Run gate and share-safety results are separate.";
|
|
@@ -1,3 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The endpoint a computer-use completion came from, as analysis provenance records it
|
|
3
|
+
* (cuaEndpointSource). Saved analyses hash it, so a new value would make them invalid on read.
|
|
4
|
+
*/
|
|
5
|
+
export const CUA_ENDPOINT_SOURCES = [
|
|
6
|
+
"participant_report",
|
|
7
|
+
"condition_matched",
|
|
8
|
+
"unavailable",
|
|
9
|
+
];
|
|
10
|
+
export function isClockEnd(source) {
|
|
11
|
+
return source === "scheduled_end" || source === "session_cap";
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The clock end a session ran until, when the participant did not declare that it reached the
|
|
15
|
+
* goal: its scheduled end or its session cap. Such a session ends as passed, because ending at
|
|
16
|
+
* the clock is the plan, and its closing report says `DID NOT REACH THE GOAL.` or declares
|
|
17
|
+
* nothing. The tally and every completion label read this, so none counts it as reaching the
|
|
18
|
+
* goal. Undefined for every other ending, including a declared `reached` at a clock end.
|
|
19
|
+
*/
|
|
20
|
+
export function ranUntilClockEnd(trace) {
|
|
21
|
+
if (trace.stopCause !== "scheduled_end" && trace.stopCause !== "session_cap")
|
|
22
|
+
return undefined;
|
|
23
|
+
return trace.declaredOutcome === "reached" ? undefined : trace.stopCause;
|
|
24
|
+
}
|
|
1
25
|
export function isCuaTrace(value) {
|
|
2
26
|
if (typeof value !== "object" || value === null)
|
|
3
27
|
return false;
|
|
@@ -32,7 +56,22 @@ export function cuaGoalSource(value, recordedStatus) {
|
|
|
32
56
|
(item.title.startsWith("stopWhen matched:") || item.title === "dwell window complete"))) {
|
|
33
57
|
return "condition_matched";
|
|
34
58
|
}
|
|
35
|
-
|
|
36
|
-
|
|
59
|
+
return ranUntilClockEnd(trace) ?? endpointOfItems(trace.items);
|
|
60
|
+
}
|
|
61
|
+
// A legacy trace without enough detail cannot establish which endpoint ended the loop.
|
|
62
|
+
function endpointOfItems(items) {
|
|
63
|
+
return items.length > 0 ? "participant_report" : "unavailable";
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* cuaGoalSource for analysis provenance. A clock end reads as it did before it had a source of
|
|
67
|
+
* its own, so an analysis saved then still matches its run when re-read. The provenance's
|
|
68
|
+
* `stopCause` and `declaredOutcome` name the clock end.
|
|
69
|
+
*/
|
|
70
|
+
export function cuaEndpointSource(value, recordedStatus) {
|
|
71
|
+
const source = cuaGoalSource(value, recordedStatus);
|
|
72
|
+
// cuaGoalSource returns a clock end only after it has checked that `items` is an array.
|
|
73
|
+
return isClockEnd(source)
|
|
74
|
+
? endpointOfItems(value.items)
|
|
75
|
+
: source;
|
|
37
76
|
}
|
|
38
77
|
export const CUA_COMPLETION_NOTE = "Participant reports alone do not establish task success. A matched stop condition establishes only its declared condition. Run gate and share-safety results are separate.";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { receivingAnalysisContext } from "../comms/receiving-evidence.js";
|
|
2
2
|
import { eventRecordIdOf, recordIdOf, streamParticipantIdOf } from "../run/participant-records.js";
|
|
3
|
-
import {
|
|
3
|
+
import { cuaEndpointSource } from "../actors/goal-source.js";
|
|
4
4
|
import { isRecord } from "../run/type-guards.js";
|
|
5
5
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
6
6
|
import { decodesToPlainRelativePath, isEvidencePath } from "../run/evidence-files.js";
|
|
@@ -82,7 +82,7 @@ export function participantSource(stream, captureVersion) {
|
|
|
82
82
|
actorStatus: actor?.status ?? null,
|
|
83
83
|
completionReason: actor?.completionReason ?? null,
|
|
84
84
|
stopCause: actor?.stopCause ?? null,
|
|
85
|
-
goalSource:
|
|
85
|
+
goalSource: cuaEndpointSource(actor, stream.status) ?? null,
|
|
86
86
|
declaredOutcome: actor?.declaredOutcome ?? null,
|
|
87
87
|
taskOutcomes: actor?.taskFunnel === undefined
|
|
88
88
|
? null
|
package/dist/analysis/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ActorStatus, ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../actors/contract.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { CuaEndpointSource } from "../actors/goal-source.js";
|
|
3
3
|
import type { AutomaticAnalysisView } from "./job.js";
|
|
4
4
|
import type { RunAnalysisCost } from "../run/run-cost.js";
|
|
5
5
|
/** Independent interpretation of retained evidence; never a participant or harness verdict. */
|
|
@@ -55,7 +55,7 @@ export interface AnalysisParticipantInput {
|
|
|
55
55
|
actorStatus: ActorStatus | null;
|
|
56
56
|
completionReason: ActorCompletionReason | null;
|
|
57
57
|
stopCause: ActorStopCause | null;
|
|
58
|
-
goalSource:
|
|
58
|
+
goalSource: CuaEndpointSource | null;
|
|
59
59
|
declaredOutcome: ParticipantDeclaredOutcome | null;
|
|
60
60
|
taskOutcomes: Array<{
|
|
61
61
|
taskId: string;
|
|
@@ -2,6 +2,7 @@ import { validStoredCodexAnalysisConfig } from "./codex-config.js";
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
|
+
import { CUA_ENDPOINT_SOURCES } from "../actors/goal-source.js";
|
|
5
6
|
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
6
7
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
7
8
|
const text = (max) => z
|
|
@@ -178,7 +179,7 @@ const analysisParticipantProvenanceSchema = z
|
|
|
178
179
|
])
|
|
179
180
|
.nullable(),
|
|
180
181
|
stopCause: z.enum(ACTOR_STOP_CAUSES).nullable(),
|
|
181
|
-
goalSource: z.enum(
|
|
182
|
+
goalSource: z.enum(CUA_ENDPOINT_SOURCES).nullable(),
|
|
182
183
|
declaredOutcome: z.enum(["reached", "not_reached", "blocked"]).nullable(),
|
|
183
184
|
taskOutcomes: z
|
|
184
185
|
.array(z
|
|
@@ -81,10 +81,21 @@ function oneLine(text) {
|
|
|
81
81
|
const line = text.replace(/\s+/g, " ").trim();
|
|
82
82
|
return line.length > REASON_CHARS ? `${line.slice(0, REASON_CHARS - 1).trimEnd()}…` : line;
|
|
83
83
|
}
|
|
84
|
-
|
|
84
|
+
/**
|
|
85
|
+
* A participant's status, how it ended and its closing message. A participant the run skipped
|
|
86
|
+
* before its session started has no ending to classify, so its line ends on the reason the run
|
|
87
|
+
* recorded. That reason is the harness's own sentence, so it prints whole.
|
|
88
|
+
*/
|
|
89
|
+
function participantLine(participant) {
|
|
90
|
+
const { id, status, session, diagnostics, skippedReason } = participant;
|
|
91
|
+
const ending = skippedReason !== undefined
|
|
92
|
+
? [skippedReason]
|
|
93
|
+
: diagnostics
|
|
94
|
+
? [formatCuaDiagnostics(diagnostics)]
|
|
95
|
+
: [];
|
|
85
96
|
return [
|
|
86
97
|
`participant ${id}: ${participantStatus(status)}${session ? ` (${session.completionReason})` : ""}`,
|
|
87
|
-
...
|
|
98
|
+
...ending,
|
|
88
99
|
...(session ? [oneLine(session.reason)] : []),
|
|
89
100
|
].join(" · ");
|
|
90
101
|
}
|
package/dist/observer/data.js
CHANGED
|
@@ -156,6 +156,13 @@ export function observerDataFromSaved(value) {
|
|
|
156
156
|
? null
|
|
157
157
|
: joinObserverData(saved, null);
|
|
158
158
|
}
|
|
159
|
+
const GOAL_SOURCE_LABELS = {
|
|
160
|
+
participant_report: "Reported complete",
|
|
161
|
+
condition_matched: "Condition matched",
|
|
162
|
+
scheduled_end: "Ran until scheduled end",
|
|
163
|
+
session_cap: "Ran until session cap",
|
|
164
|
+
unavailable: "Completion source unavailable",
|
|
165
|
+
};
|
|
159
166
|
/** Refresh presentation from recorded actor evidence, including older exported snapshots. */
|
|
160
167
|
export function withObserverEndings(data) {
|
|
161
168
|
const streams = (data.streams ?? []).map(({ ending: _previousEnding, ...stream }) => {
|
|
@@ -166,11 +173,7 @@ export function withObserverEndings(data) {
|
|
|
166
173
|
...(ending === undefined ? {} : { ending }),
|
|
167
174
|
...((stream.status === "passed" || stream.status === "complete") && source !== undefined
|
|
168
175
|
? {
|
|
169
|
-
statusLabel: source
|
|
170
|
-
? "Reported complete"
|
|
171
|
-
: source === "condition_matched"
|
|
172
|
-
? "Condition matched"
|
|
173
|
-
: "Completion source unavailable",
|
|
176
|
+
statusLabel: GOAL_SOURCE_LABELS[source],
|
|
174
177
|
}
|
|
175
178
|
: {}),
|
|
176
179
|
...(stream.status === "incomplete" || (ending !== undefined && stream.status === "abandoned")
|