humanish 0.126.0 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/actors/computer-use/loop/ending.d.ts +3 -1
- package/dist/actors/computer-use/loop/ending.js +5 -3
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/observer/data.js +8 -5
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +3 -0
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-runs.d.ts +9 -9
- package/dist/routes/computer-use/participant-runs.js +52 -86
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +26 -16
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.js +1 -1
- package/dist/routes/computer-use/single-bundle.js +9 -1
- package/dist/routes/computer-use/types.d.ts +2 -3
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +1 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +3 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/judge.d.ts +7 -0
- package/dist/run/judge.js +4 -4
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/study/arrivals.d.ts +84 -31
- package/dist/study/arrivals.js +165 -86
- package/dist/study/bundle-estimate.d.ts +8 -7
- package/dist/study/bundle-estimate.js +131 -70
- package/dist/study/desktop-files.js +4 -4
- package/dist/study/plan-participants.d.ts +4 -9
- package/dist/study/plan-participants.js +2 -14
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +6 -4
- package/dist/study/types.d.ts +2 -1
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/docs/contracts/run-bundle.md +1 -1
- package/docs/contracts/schemas.md +24 -11
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,19 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.127.0: Clock-end passes counted by what participants declared, a scaled wind-down, the concurrency cap in the plan (2026-10-13)
|
|
12
|
+
|
|
13
|
+
humanish 0.127.0 makes long scheduled studies report what happened. A participant that passes at
|
|
14
|
+
its scheduled end or session cap without declaring `REACHED THE GOAL.` is now counted as running
|
|
15
|
+
until that end (`participants.ranUntilEnd`), where 0.126.0 counted it as reaching the goal. The
|
|
16
|
+
default wind-down is a quarter of the window, at most 10 minutes, so a short window is no longer
|
|
17
|
+
told its end from its first turn. `HUMANISH_CUA_MAX_CONCURRENCY` now lowers the plan itself, on
|
|
18
|
+
shared-world studies too, and a participant whose scheduled end passes while it waits for a slot
|
|
19
|
+
is skipped before its desktop is created. The Observer keeps updating past 100,000 items for a
|
|
20
|
+
participant, and the bundle estimate is re-derived for run bundle v3.
|
|
21
|
+
|
|
22
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.127.0)
|
|
23
|
+
|
|
11
24
|
## 0.126.0: Per-participant time zones, session caps and desktop files, page paths, a lighter live Observer (2026-10-13)
|
|
12
25
|
|
|
13
26
|
humanish 0.126.0 adds per-participant settings for long, mixed studies and a lighter live Observer.
|
|
@@ -23,7 +23,9 @@ export interface Stop {
|
|
|
23
23
|
/**
|
|
24
24
|
* Read the fixed closing line the prompt asks for: the first non-empty line of the
|
|
25
25
|
* participant's last message or closing report summary, exactly one of three phrases,
|
|
26
|
-
* punctuation and case forgiven. Anything else is absence, never a guess.
|
|
26
|
+
* punctuation and case forgiven. Anything else is absence, never a guess. A `>` blockquote line
|
|
27
|
+
* is skipped: it quotes something else, such as the page, and the blocker scan
|
|
28
|
+
* (routes/computer-use/self-report.ts) drops it too. Exported for tests.
|
|
27
29
|
*/
|
|
28
30
|
export declare function declaredOutcomeFromClosingLine(message: string | undefined): ParticipantDeclaredOutcome | undefined;
|
|
29
31
|
/**
|
|
@@ -7,7 +7,9 @@ import { plural } from "../../../run/text.js";
|
|
|
7
7
|
/**
|
|
8
8
|
* Read the fixed closing line the prompt asks for: the first non-empty line of the
|
|
9
9
|
* participant's last message or closing report summary, exactly one of three phrases,
|
|
10
|
-
* punctuation and case forgiven. Anything else is absence, never a guess.
|
|
10
|
+
* punctuation and case forgiven. Anything else is absence, never a guess. A `>` blockquote line
|
|
11
|
+
* is skipped: it quotes something else, such as the page, and the blocker scan
|
|
12
|
+
* (routes/computer-use/self-report.ts) drops it too. Exported for tests.
|
|
11
13
|
*/
|
|
12
14
|
export function declaredOutcomeFromClosingLine(message) {
|
|
13
15
|
if (message === undefined)
|
|
@@ -15,10 +17,10 @@ export function declaredOutcomeFromClosingLine(message) {
|
|
|
15
17
|
const first = message
|
|
16
18
|
.split(/\r?\n/)
|
|
17
19
|
.map((line) => line.trim())
|
|
18
|
-
.find((line) => line.length > 0);
|
|
20
|
+
.find((line) => line.length > 0 && !line.startsWith(">"));
|
|
19
21
|
if (first === undefined)
|
|
20
22
|
return undefined;
|
|
21
|
-
const normalized = first.replace(/^[*_
|
|
23
|
+
const normalized = first.replace(/^[*_#\s-]+|[*_.!\s]+$/g, "").toLowerCase();
|
|
22
24
|
if (normalized === "reached the goal")
|
|
23
25
|
return "reached";
|
|
24
26
|
if (normalized === "did not reach the goal")
|
|
@@ -512,7 +512,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
512
512
|
// Both are offered in both modes. A threaded request reads the session the server holds; an
|
|
513
513
|
// explicit_context one carries its newest screens whole and its oldest turns summarized past
|
|
514
514
|
// the budget, which the trace's conversation record counts. A participant that worked a long
|
|
515
|
-
//
|
|
515
|
+
// session writes its closing account from what it carries, as a person writes one from memory.
|
|
516
516
|
debrief: (req, signal) => requestTurn(req, signal, "report"),
|
|
517
517
|
requestImpressions: (req, signal) => requestTurn(req, signal, "impressions"),
|
|
518
518
|
};
|
|
@@ -1,5 +1,35 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* The endpoint a computer-use completion came from, as analysis provenance records it
|
|
3
|
+
* (cuaEndpointSource). Saved analyses hash it, so a new value would make them invalid on read.
|
|
4
|
+
*/
|
|
5
|
+
export declare const CUA_ENDPOINT_SOURCES: readonly ["participant_report", "condition_matched", "unavailable"];
|
|
6
|
+
export type CuaEndpointSource = (typeof CUA_ENDPOINT_SOURCES)[number];
|
|
7
|
+
/**
|
|
8
|
+
* Presentation provenance for computer-use completion; never reclassifies the stored outcome.
|
|
9
|
+
* `scheduled_end` and `session_cap` name a session that passed at that clock end without the
|
|
10
|
+
* participant declaring it reached the goal (ranUntilClockEnd).
|
|
11
|
+
*/
|
|
12
|
+
export type CuaGoalSource = CuaEndpointSource | ClockEnd;
|
|
13
|
+
/** The two clock ends a computer-use session passes at: the stop causes of the same names. */
|
|
14
|
+
export type ClockEnd = "scheduled_end" | "session_cap";
|
|
15
|
+
export declare function isClockEnd(source: CuaGoalSource | undefined): source is ClockEnd;
|
|
16
|
+
/**
|
|
17
|
+
* The clock end a session ran until, when the participant did not declare that it reached the
|
|
18
|
+
* goal: its scheduled end or its session cap. Such a session ends as passed, because ending at
|
|
19
|
+
* the clock is the plan, and its closing report says `DID NOT REACH THE GOAL.` or declares
|
|
20
|
+
* nothing. The tally and every completion label read this, so none counts it as reaching the
|
|
21
|
+
* goal. Undefined for every other ending, including a declared `reached` at a clock end.
|
|
22
|
+
*/
|
|
23
|
+
export declare function ranUntilClockEnd(trace: {
|
|
24
|
+
readonly stopCause?: unknown;
|
|
25
|
+
readonly declaredOutcome?: unknown;
|
|
26
|
+
}): ClockEnd | undefined;
|
|
3
27
|
export declare function isCuaTrace(value: unknown): boolean;
|
|
4
28
|
export declare function cuaGoalSource(value: unknown, recordedStatus?: unknown): CuaGoalSource | undefined;
|
|
29
|
+
/**
|
|
30
|
+
* cuaGoalSource for analysis provenance. A clock end reads as it did before it had a source of
|
|
31
|
+
* its own, so an analysis saved then still matches its run when re-read. The provenance's
|
|
32
|
+
* `stopCause` and `declaredOutcome` name the clock end.
|
|
33
|
+
*/
|
|
34
|
+
export declare function cuaEndpointSource(value: unknown, recordedStatus?: unknown): CuaEndpointSource | undefined;
|
|
5
35
|
export declare const CUA_COMPLETION_NOTE = "Participant reports alone do not establish task success. A matched stop condition establishes only its declared condition. Run gate and share-safety results are separate.";
|
|
@@ -1,3 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The endpoint a computer-use completion came from, as analysis provenance records it
|
|
3
|
+
* (cuaEndpointSource). Saved analyses hash it, so a new value would make them invalid on read.
|
|
4
|
+
*/
|
|
5
|
+
export const CUA_ENDPOINT_SOURCES = [
|
|
6
|
+
"participant_report",
|
|
7
|
+
"condition_matched",
|
|
8
|
+
"unavailable",
|
|
9
|
+
];
|
|
10
|
+
export function isClockEnd(source) {
|
|
11
|
+
return source === "scheduled_end" || source === "session_cap";
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The clock end a session ran until, when the participant did not declare that it reached the
|
|
15
|
+
* goal: its scheduled end or its session cap. Such a session ends as passed, because ending at
|
|
16
|
+
* the clock is the plan, and its closing report says `DID NOT REACH THE GOAL.` or declares
|
|
17
|
+
* nothing. The tally and every completion label read this, so none counts it as reaching the
|
|
18
|
+
* goal. Undefined for every other ending, including a declared `reached` at a clock end.
|
|
19
|
+
*/
|
|
20
|
+
export function ranUntilClockEnd(trace) {
|
|
21
|
+
if (trace.stopCause !== "scheduled_end" && trace.stopCause !== "session_cap")
|
|
22
|
+
return undefined;
|
|
23
|
+
return trace.declaredOutcome === "reached" ? undefined : trace.stopCause;
|
|
24
|
+
}
|
|
1
25
|
export function isCuaTrace(value) {
|
|
2
26
|
if (typeof value !== "object" || value === null)
|
|
3
27
|
return false;
|
|
@@ -32,7 +56,22 @@ export function cuaGoalSource(value, recordedStatus) {
|
|
|
32
56
|
(item.title.startsWith("stopWhen matched:") || item.title === "dwell window complete"))) {
|
|
33
57
|
return "condition_matched";
|
|
34
58
|
}
|
|
35
|
-
|
|
36
|
-
|
|
59
|
+
return ranUntilClockEnd(trace) ?? endpointOfItems(trace.items);
|
|
60
|
+
}
|
|
61
|
+
// A legacy trace without enough detail cannot establish which endpoint ended the loop.
|
|
62
|
+
function endpointOfItems(items) {
|
|
63
|
+
return items.length > 0 ? "participant_report" : "unavailable";
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* cuaGoalSource for analysis provenance. A clock end reads as it did before it had a source of
|
|
67
|
+
* its own, so an analysis saved then still matches its run when re-read. The provenance's
|
|
68
|
+
* `stopCause` and `declaredOutcome` name the clock end.
|
|
69
|
+
*/
|
|
70
|
+
export function cuaEndpointSource(value, recordedStatus) {
|
|
71
|
+
const source = cuaGoalSource(value, recordedStatus);
|
|
72
|
+
// cuaGoalSource returns a clock end only after it has checked that `items` is an array.
|
|
73
|
+
return isClockEnd(source)
|
|
74
|
+
? endpointOfItems(value.items)
|
|
75
|
+
: source;
|
|
37
76
|
}
|
|
38
77
|
export const CUA_COMPLETION_NOTE = "Participant reports alone do not establish task success. A matched stop condition establishes only its declared condition. Run gate and share-safety results are separate.";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { receivingAnalysisContext } from "../comms/receiving-evidence.js";
|
|
2
2
|
import { eventRecordIdOf, recordIdOf, streamParticipantIdOf } from "../run/participant-records.js";
|
|
3
|
-
import {
|
|
3
|
+
import { cuaEndpointSource } from "../actors/goal-source.js";
|
|
4
4
|
import { isRecord } from "../run/type-guards.js";
|
|
5
5
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
6
6
|
import { decodesToPlainRelativePath, isEvidencePath } from "../run/evidence-files.js";
|
|
@@ -82,7 +82,7 @@ export function participantSource(stream, captureVersion) {
|
|
|
82
82
|
actorStatus: actor?.status ?? null,
|
|
83
83
|
completionReason: actor?.completionReason ?? null,
|
|
84
84
|
stopCause: actor?.stopCause ?? null,
|
|
85
|
-
goalSource:
|
|
85
|
+
goalSource: cuaEndpointSource(actor, stream.status) ?? null,
|
|
86
86
|
declaredOutcome: actor?.declaredOutcome ?? null,
|
|
87
87
|
taskOutcomes: actor?.taskFunnel === undefined
|
|
88
88
|
? null
|
package/dist/analysis/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ActorStatus, ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../actors/contract.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { CuaEndpointSource } from "../actors/goal-source.js";
|
|
3
3
|
import type { AutomaticAnalysisView } from "./job.js";
|
|
4
4
|
import type { RunAnalysisCost } from "../run/run-cost.js";
|
|
5
5
|
/** Independent interpretation of retained evidence; never a participant or harness verdict. */
|
|
@@ -55,7 +55,7 @@ export interface AnalysisParticipantInput {
|
|
|
55
55
|
actorStatus: ActorStatus | null;
|
|
56
56
|
completionReason: ActorCompletionReason | null;
|
|
57
57
|
stopCause: ActorStopCause | null;
|
|
58
|
-
goalSource:
|
|
58
|
+
goalSource: CuaEndpointSource | null;
|
|
59
59
|
declaredOutcome: ParticipantDeclaredOutcome | null;
|
|
60
60
|
taskOutcomes: Array<{
|
|
61
61
|
taskId: string;
|
|
@@ -2,6 +2,7 @@ import { validStoredCodexAnalysisConfig } from "./codex-config.js";
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
|
+
import { CUA_ENDPOINT_SOURCES } from "../actors/goal-source.js";
|
|
5
6
|
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
6
7
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
7
8
|
const text = (max) => z
|
|
@@ -178,7 +179,7 @@ const analysisParticipantProvenanceSchema = z
|
|
|
178
179
|
])
|
|
179
180
|
.nullable(),
|
|
180
181
|
stopCause: z.enum(ACTOR_STOP_CAUSES).nullable(),
|
|
181
|
-
goalSource: z.enum(
|
|
182
|
+
goalSource: z.enum(CUA_ENDPOINT_SOURCES).nullable(),
|
|
182
183
|
declaredOutcome: z.enum(["reached", "not_reached", "blocked"]).nullable(),
|
|
183
184
|
taskOutcomes: z
|
|
184
185
|
.array(z
|
package/dist/observer/data.js
CHANGED
|
@@ -156,6 +156,13 @@ export function observerDataFromSaved(value) {
|
|
|
156
156
|
? null
|
|
157
157
|
: joinObserverData(saved, null);
|
|
158
158
|
}
|
|
159
|
+
const GOAL_SOURCE_LABELS = {
|
|
160
|
+
participant_report: "Reported complete",
|
|
161
|
+
condition_matched: "Condition matched",
|
|
162
|
+
scheduled_end: "Ran until scheduled end",
|
|
163
|
+
session_cap: "Ran until session cap",
|
|
164
|
+
unavailable: "Completion source unavailable",
|
|
165
|
+
};
|
|
159
166
|
/** Refresh presentation from recorded actor evidence, including older exported snapshots. */
|
|
160
167
|
export function withObserverEndings(data) {
|
|
161
168
|
const streams = (data.streams ?? []).map(({ ending: _previousEnding, ...stream }) => {
|
|
@@ -166,11 +173,7 @@ export function withObserverEndings(data) {
|
|
|
166
173
|
...(ending === undefined ? {} : { ending }),
|
|
167
174
|
...((stream.status === "passed" || stream.status === "complete") && source !== undefined
|
|
168
175
|
? {
|
|
169
|
-
statusLabel: source
|
|
170
|
-
? "Reported complete"
|
|
171
|
-
: source === "condition_matched"
|
|
172
|
-
? "Condition matched"
|
|
173
|
-
: "Completion source unavailable",
|
|
176
|
+
statusLabel: GOAL_SOURCE_LABELS[source],
|
|
174
177
|
}
|
|
175
178
|
: {}),
|
|
176
179
|
...(stream.status === "incomplete" || (ending !== undefined && stream.status === "abandoned")
|