humanish 0.111.3 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +49 -4
- package/dist/cli/findings.js +179 -23
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/route.js +1 -0
- package/dist/routes/scripted/setup.js +53 -10
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +7 -4
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/secrets.d.ts +15 -2
- package/dist/run/secrets.js +134 -9
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/requirements.js +2 -1
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +76 -23
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,39 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.112.0: Plain finding headlines, design findings, plain captions, DO_NOT_TRACK for terminal participants (2026-10-07)
|
|
12
|
+
|
|
13
|
+
humanish 0.112.0 writes findings and captions for someone who did not watch the session, and
|
|
14
|
+
terminal participants send `DO_NOT_TRACK=1` by default. Each analysis finding leads with a plain
|
|
15
|
+
headline and an account of what the person tried, what got in their way and how it seemed to feel;
|
|
16
|
+
its title, summary and observations stay underneath as the evidence. Analyses also report design
|
|
17
|
+
findings: problems a product designer would notice in the captures, each with a severity and at
|
|
18
|
+
least one cited capture. `review --json` carries them as `analysis.designFindings`, and the
|
|
19
|
+
analysis prompt is `study-evidence-7`. Participant captions name the person, such as "Lobby host,
|
|
20
|
+
phone", and every route's run summary says how many participants took part and how many reached
|
|
21
|
+
the goal. Terminal participant and product setup commands run with `DO_NOT_TRACK=1`; set
|
|
22
|
+
`execution.terminal.doNotTrack: false` to study a product's telemetry. `humanish runs` and the
|
|
23
|
+
Observer now read a run with no outcome and a running simulation as interrupted, as the TUI,
|
|
24
|
+
`stats` and `verify` already did. The `--env-file` and `cleanup` aliases are removed in the first
|
|
25
|
+
minor release on or after 2026-11-03.
|
|
26
|
+
|
|
27
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.112.0)
|
|
28
|
+
|
|
29
|
+
## 0.111.4: Encoded known values scrubbed, scripted traces scrubbed, a live command after a dry run (2026-10-06)
|
|
30
|
+
|
|
31
|
+
humanish 0.111.4 scrubs a run's known values from more of its evidence and fixes three messages.
|
|
32
|
+
Every route now also finds a known value, such as a provider key or a `subject.env` value, in its
|
|
33
|
+
encoded forms: percent-encoded, JSON-escaped, base64, base64url and hex, and where a terminal color
|
|
34
|
+
code or an output chunk boundary splits it. A scripted run scrubs its known values from the step
|
|
35
|
+
trace in `traces/<surface>.json` and `actor-<surface>.json`, and from the app URL, scenario goal,
|
|
36
|
+
title and step labels it records. Two places keep a known value: a scenario file name, and the
|
|
37
|
+
path of a computer-use `subject.appUrl`. `humanish review` on a dry run names the command that
|
|
38
|
+
starts a live run, so `next` in `review --json` holds a string where it held null. A terminal
|
|
39
|
+
refusal for a missing runtime key no longer reads `runtimeAuth "undefined"`, and a refusal that
|
|
40
|
+
names two signed-in coding agents reads "report".
|
|
41
|
+
|
|
42
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.111.4)
|
|
43
|
+
|
|
11
44
|
## 0.111.3: Shared-world feedback candidates, scripted E2B key scrubbed, subject sandbox warnings (2026-10-06)
|
|
12
45
|
|
|
13
46
|
humanish 0.111.3 gives shared-world runs feedback candidates and repairs seven places where one
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { CuaTurn, CuaTurnRequest } from "./loop.js";
|
|
2
|
+
import type { ReasoningEffort } from "../reasoning-effort.js";
|
|
3
|
+
import type { ActorConversation } from "../contract.js";
|
|
4
|
+
import { type OpenAiReasoningSummary } from "./openai-wire.js";
|
|
5
|
+
/** How a conversation starts. The model settings stay fixed for the whole session. */
|
|
6
|
+
export interface OpenAiConversationSettings {
|
|
7
|
+
readonly model: string;
|
|
8
|
+
readonly reasoningEffort: ReasoningEffort;
|
|
9
|
+
readonly maxOutputTokens?: number;
|
|
10
|
+
readonly safetyIdentifier?: string;
|
|
11
|
+
/** The summaries to ask for until the account or model rejects them; undefined asks for none. */
|
|
12
|
+
readonly reasoningSummary: OpenAiReasoningSummary | undefined;
|
|
13
|
+
/** Carry the conversation on the client from the first request (explicit_context). */
|
|
14
|
+
readonly zeroDataRetention: boolean;
|
|
15
|
+
/** The clock the trace's record reads. */
|
|
16
|
+
readonly now: () => number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* One request of the conversation. Its new items are built once, so every attempt at it, and the
|
|
20
|
+
* reply it accepts, see the same ones.
|
|
21
|
+
*/
|
|
22
|
+
export interface ConversationRequest {
|
|
23
|
+
/** The body as the conversation stands now, so an attempt after a rejection shows its effect. */
|
|
24
|
+
body(): Record<string, unknown>;
|
|
25
|
+
/**
|
|
26
|
+
* The request was answered: records a participant request with what it carried, moves the
|
|
27
|
+
* conversation on to the reply and returns the reply's turn.
|
|
28
|
+
*/
|
|
29
|
+
accept(reply: unknown): CuaTurn;
|
|
30
|
+
}
|
|
31
|
+
/** Which answer told the provider that the organization keeps no server-side conversation. */
|
|
32
|
+
export type ZdrRejection = NonNullable<ActorConversation["rejection"]>;
|
|
33
|
+
export declare class OpenAiConversation {
|
|
34
|
+
private readonly settings;
|
|
35
|
+
private lastResponseId;
|
|
36
|
+
/** computer_call ids the next request owes an output for. */
|
|
37
|
+
private pendingCallIds;
|
|
38
|
+
/** Accepted replies so far; the next request is number `replies + 1`. */
|
|
39
|
+
private replies;
|
|
40
|
+
/** The whole conversation, which explicit-context requests carry within its token budget. */
|
|
41
|
+
private readonly carried;
|
|
42
|
+
private mode;
|
|
43
|
+
/** Undefined once the account or model rejected reasoning summaries. */
|
|
44
|
+
private reasoningSummary;
|
|
45
|
+
private readonly evidence;
|
|
46
|
+
constructor(settings: OpenAiConversationSettings);
|
|
47
|
+
/**
|
|
48
|
+
* The next request, carrying `req`'s screen: the opening request, or the outputs the latest
|
|
49
|
+
* reply's computer calls are owed. A closing report asks for a strict JSON report and no tools.
|
|
50
|
+
*/
|
|
51
|
+
request(req: CuaTurnRequest, closing?: boolean): ConversationRequest;
|
|
52
|
+
/**
|
|
53
|
+
* The organization rejected server-side state: carry the conversation on the client from the
|
|
54
|
+
* request being sent, and record which answer made the switch. False when the conversation is
|
|
55
|
+
* already carried, so the rejection stands.
|
|
56
|
+
*/
|
|
57
|
+
switchToExplicitContext(rejection: ZdrRejection): boolean;
|
|
58
|
+
/**
|
|
59
|
+
* The account or model rejected reasoning summaries: stop asking for the rest of the session,
|
|
60
|
+
* from the request being sent. False when none are asked for, so the rejection stands.
|
|
61
|
+
*/
|
|
62
|
+
dropReasoningSummaries(): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Whether the server holds the whole session, so a closing report threaded on it reads every
|
|
65
|
+
* turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
|
|
66
|
+
* or switched to at runtime.
|
|
67
|
+
*/
|
|
68
|
+
get serverHoldsSession(): boolean;
|
|
69
|
+
/** The trace's account of how the conversation was carried (ActorConversation). */
|
|
70
|
+
record(): ActorConversation;
|
|
71
|
+
/**
|
|
72
|
+
* The items a request adds to the conversation: the opening message with the first screen, or
|
|
73
|
+
* an output for each pending computer_call with the latest screenshot and any hint.
|
|
74
|
+
*/
|
|
75
|
+
private newItems;
|
|
76
|
+
private context;
|
|
77
|
+
/**
|
|
78
|
+
* A participant request: the opening request, or a continuation carrying `sent`. A threaded
|
|
79
|
+
* continuation names the previous response; an explicit-context one carries the conversation
|
|
80
|
+
* ahead of `sent`.
|
|
81
|
+
*/
|
|
82
|
+
private turnBody;
|
|
83
|
+
/** The read-only closing report: no tools, a small output limit, a strict schema. */
|
|
84
|
+
private closingBody;
|
|
85
|
+
/**
|
|
86
|
+
* Move the conversation on to a reply to a request that added `sent`. A participant reply cut
|
|
87
|
+
* off by the output limit is set aside, so the next request is the same one again
|
|
88
|
+
* (outputLimitRetry); its actions never run, so no call output is owed for them.
|
|
89
|
+
*/
|
|
90
|
+
private accept;
|
|
91
|
+
}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { CarriedConversation, estimateTokens } from "./openai-context.js";
|
|
2
|
+
import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintItems, openingMessage, parseOpenAiResponse, } from "./openai-wire.js";
|
|
3
|
+
/** The closing report's output limit: its own cap, or the declared limit when that is lower. */
|
|
4
|
+
const CLOSING_OUTPUT_LIMIT = 1024;
|
|
5
|
+
export class OpenAiConversation {
|
|
6
|
+
settings;
|
|
7
|
+
lastResponseId;
|
|
8
|
+
/** computer_call ids the next request owes an output for. */
|
|
9
|
+
pendingCallIds = [];
|
|
10
|
+
/** Accepted replies so far; the next request is number `replies + 1`. */
|
|
11
|
+
replies = 0;
|
|
12
|
+
/** The whole conversation, which explicit-context requests carry within its token budget. */
|
|
13
|
+
carried = new CarriedConversation();
|
|
14
|
+
mode;
|
|
15
|
+
/** Undefined once the account or model rejected reasoning summaries. */
|
|
16
|
+
reasoningSummary;
|
|
17
|
+
evidence;
|
|
18
|
+
constructor(settings) {
|
|
19
|
+
this.settings = settings;
|
|
20
|
+
this.mode = settings.zeroDataRetention ? "explicit_context" : "threaded";
|
|
21
|
+
this.reasoningSummary = settings.reasoningSummary;
|
|
22
|
+
this.evidence = new ConversationRecord(settings.zeroDataRetention, settings.now);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The next request, carrying `req`'s screen: the opening request, or the outputs the latest
|
|
26
|
+
* reply's computer calls are owed. A closing report asks for a strict JSON report and no tools.
|
|
27
|
+
*/
|
|
28
|
+
request(req, closing = false) {
|
|
29
|
+
const sent = this.newItems(req);
|
|
30
|
+
return {
|
|
31
|
+
body: () => (closing ? this.closingBody(req, sent) : this.turnBody(req, sent)),
|
|
32
|
+
accept: (reply) => this.accept(reply, closing, sent),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The organization rejected server-side state: carry the conversation on the client from the
|
|
37
|
+
* request being sent, and record which answer made the switch. False when the conversation is
|
|
38
|
+
* already carried, so the rejection stands.
|
|
39
|
+
*/
|
|
40
|
+
switchToExplicitContext(rejection) {
|
|
41
|
+
if (this.mode === "explicit_context")
|
|
42
|
+
return false;
|
|
43
|
+
this.evidence.switched(rejection);
|
|
44
|
+
this.mode = "explicit_context";
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The account or model rejected reasoning summaries: stop asking for the rest of the session,
|
|
49
|
+
* from the request being sent. False when none are asked for, so the rejection stands.
|
|
50
|
+
*/
|
|
51
|
+
dropReasoningSummaries() {
|
|
52
|
+
if (this.reasoningSummary === undefined)
|
|
53
|
+
return false;
|
|
54
|
+
this.reasoningSummary = undefined;
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Whether the server holds the whole session, so a closing report threaded on it reads every
|
|
59
|
+
* turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
|
|
60
|
+
* or switched to at runtime.
|
|
61
|
+
*/
|
|
62
|
+
get serverHoldsSession() {
|
|
63
|
+
return this.mode === "threaded" && this.lastResponseId !== undefined;
|
|
64
|
+
}
|
|
65
|
+
/** The trace's account of how the conversation was carried (ActorConversation). */
|
|
66
|
+
record() {
|
|
67
|
+
return this.evidence.snapshot(this.mode);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The items a request adds to the conversation: the opening message with the first screen, or
|
|
71
|
+
* an output for each pending computer_call with the latest screenshot and any hint.
|
|
72
|
+
*/
|
|
73
|
+
newItems(req) {
|
|
74
|
+
if (this.replies === 0)
|
|
75
|
+
return [openingMessage(req.instructions, req.observation.screenshot)];
|
|
76
|
+
return [
|
|
77
|
+
...this.pendingCallIds.map((id) => buildCallOutput(id, req.observation.screenshot, req.acknowledgedSafetyChecks)),
|
|
78
|
+
...hintItems(req.contextHint),
|
|
79
|
+
];
|
|
80
|
+
}
|
|
81
|
+
context(instructions) {
|
|
82
|
+
const { settings } = this;
|
|
83
|
+
return {
|
|
84
|
+
stateless: this.mode === "explicit_context",
|
|
85
|
+
model: settings.model,
|
|
86
|
+
instructions,
|
|
87
|
+
reasoningEffort: settings.reasoningEffort,
|
|
88
|
+
...(settings.maxOutputTokens === undefined
|
|
89
|
+
? {}
|
|
90
|
+
: { maxOutputTokens: settings.maxOutputTokens }),
|
|
91
|
+
...(this.reasoningSummary === undefined ? {} : { reasoningSummary: this.reasoningSummary }),
|
|
92
|
+
...(settings.safetyIdentifier === undefined
|
|
93
|
+
? {}
|
|
94
|
+
: { safetyIdentifier: settings.safetyIdentifier }),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* A participant request: the opening request, or a continuation carrying `sent`. A threaded
|
|
99
|
+
* continuation names the previous response; an explicit-context one carries the conversation
|
|
100
|
+
* ahead of `sent`.
|
|
101
|
+
*/
|
|
102
|
+
turnBody(req, sent) {
|
|
103
|
+
const ctx = this.context(req.instructions);
|
|
104
|
+
if (this.replies === 0)
|
|
105
|
+
return buildInitialRequest(ctx, req.observation.screenshot);
|
|
106
|
+
return buildContinuationRequest({
|
|
107
|
+
ctx,
|
|
108
|
+
previousResponseId: this.lastResponseId,
|
|
109
|
+
callOutputs: [...sent],
|
|
110
|
+
...(this.mode === "explicit_context" ? { explicitContextItems: this.carried.carried() } : {}),
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
/** The read-only closing report: no tools, a small output limit, a strict schema. */
|
|
114
|
+
closingBody(req, sent) {
|
|
115
|
+
return {
|
|
116
|
+
...this.turnBody(req, sent),
|
|
117
|
+
tool_choice: "none",
|
|
118
|
+
max_output_tokens: Math.min(this.settings.maxOutputTokens ?? CLOSING_OUTPUT_LIMIT, CLOSING_OUTPUT_LIMIT),
|
|
119
|
+
text: {
|
|
120
|
+
format: {
|
|
121
|
+
type: "json_schema",
|
|
122
|
+
name: "participant_closing_report",
|
|
123
|
+
strict: true,
|
|
124
|
+
schema: {
|
|
125
|
+
type: "object",
|
|
126
|
+
additionalProperties: false,
|
|
127
|
+
required: ["summary", "frictionReports"],
|
|
128
|
+
properties: {
|
|
129
|
+
summary: { type: "string" },
|
|
130
|
+
frictionReports: { type: "array", items: { type: "string" } },
|
|
131
|
+
},
|
|
132
|
+
},
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Move the conversation on to a reply to a request that added `sent`. A participant reply cut
|
|
139
|
+
* off by the output limit is set aside, so the next request is the same one again
|
|
140
|
+
* (outputLimitRetry); its actions never run, so no call output is owed for them.
|
|
141
|
+
*/
|
|
142
|
+
accept(reply, closing, sent) {
|
|
143
|
+
// Recorded before the reply is accepted: accepting can cut the carried history, and the record
|
|
144
|
+
// counts what this request carried.
|
|
145
|
+
if (!closing)
|
|
146
|
+
this.evidence.requested(this.mode, this.carried, sent);
|
|
147
|
+
const parsed = parseOpenAiResponse(reply);
|
|
148
|
+
if (!closing && parsed.turn.interruption === "output_limit")
|
|
149
|
+
return parsed.turn;
|
|
150
|
+
if (parsed.turn.responseId !== undefined)
|
|
151
|
+
this.lastResponseId = parsed.turn.responseId;
|
|
152
|
+
this.pendingCallIds = parsed.callIds;
|
|
153
|
+
this.replies += 1;
|
|
154
|
+
this.carried.accept(this.replies, sent, parsed.outputItems);
|
|
155
|
+
return parsed.turn;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/** The trace's account of how a session carried its conversation (ActorConversation). */
|
|
159
|
+
class ConversationRecord {
|
|
160
|
+
now;
|
|
161
|
+
explicitReason;
|
|
162
|
+
rejection;
|
|
163
|
+
switchedAt;
|
|
164
|
+
switchedAtRequest;
|
|
165
|
+
summarizedTurns = 0;
|
|
166
|
+
requests = [];
|
|
167
|
+
constructor(configured, now) {
|
|
168
|
+
this.now = now;
|
|
169
|
+
if (configured)
|
|
170
|
+
this.explicitReason = "configured";
|
|
171
|
+
}
|
|
172
|
+
/** The organization rejected server-side state, so the request being sent is explicit_context. */
|
|
173
|
+
switched(rejection) {
|
|
174
|
+
this.explicitReason = "zdr_rejection";
|
|
175
|
+
this.rejection = rejection;
|
|
176
|
+
this.switchedAt = new Date(this.now()).toISOString();
|
|
177
|
+
// Counted like requests[], which includes replies set aside at their output limit.
|
|
178
|
+
this.switchedAtRequest = this.requests.length + 1;
|
|
179
|
+
}
|
|
180
|
+
/** A participant request went out with `sent` after `conversation`. */
|
|
181
|
+
requested(mode, conversation, sent) {
|
|
182
|
+
if (mode !== "explicit_context") {
|
|
183
|
+
this.requests.push({ mode: "threaded" });
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
const carried = conversation.size();
|
|
187
|
+
// Only summaries a request carried count; the conversation also trims after the last reply.
|
|
188
|
+
this.summarizedTurns = carried.notedTurns;
|
|
189
|
+
this.requests.push({
|
|
190
|
+
mode: "explicit_context",
|
|
191
|
+
carriedExchanges: carried.exchanges,
|
|
192
|
+
carriedScreenshots: carried.screenshots,
|
|
193
|
+
estimatedInputTokens: carried.estimatedTokens + estimateTokens(sent),
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
snapshot(mode) {
|
|
197
|
+
return {
|
|
198
|
+
mode,
|
|
199
|
+
...(this.explicitReason === undefined ? {} : { explicitReason: this.explicitReason }),
|
|
200
|
+
...(this.rejection === undefined ? {} : { rejection: this.rejection }),
|
|
201
|
+
...(this.switchedAt === undefined ? {} : { switchedAt: this.switchedAt }),
|
|
202
|
+
...(this.switchedAtRequest === undefined
|
|
203
|
+
? {}
|
|
204
|
+
: { switchedAtRequest: this.switchedAtRequest }),
|
|
205
|
+
summarizedTurns: this.summarizedTurns,
|
|
206
|
+
requests: this.requests.map((request) => ({ ...request })),
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { validClosingReport } from "./loop.js";
|
|
2
2
|
import { ComputerUseAdmissionLimitError, isComputerUseAdmissionLimitError, } from "./admission-limit.js";
|
|
3
3
|
import { ComputerUsePromptRefusedError } from "./provider-error.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { asRecord, optionalString, parseOpenAiResponse, } from "./openai-wire.js";
|
|
4
|
+
import { OpenAiConversation, } from "./openai-conversation.js";
|
|
5
|
+
import { asRecord, optionalString } from "./openai-wire.js";
|
|
7
6
|
import { redactText } from "../../evidence/redaction.js";
|
|
8
7
|
import { isMaxOutputTokens } from "../output-token-limit.js";
|
|
9
8
|
import { prepareContainedOutputFile, prepareSelectedOutputDirectory, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
10
9
|
import { OPENAI_RESPONSES_URL } from "../openai-endpoint.js";
|
|
11
10
|
// A public-safe re-derivation of the OpenAI Responses API computer-use provider,
|
|
12
11
|
// behind the CuaProvider port from src/actors/computer-use/loop.ts. The pure wire mapping
|
|
13
|
-
// (openAiActionToCua, parseOpenAiResponse and the request builders) is in openai-wire.ts
|
|
12
|
+
// (openAiActionToCua, parseOpenAiResponse and the request builders) is in openai-wire.ts, and
|
|
13
|
+
// what each request carries, with the trace's record of it, is in openai-conversation.ts.
|
|
14
14
|
// This module is the live shim: createOpenAiResponsesProvider does a raw POST to the
|
|
15
15
|
// Responses endpoint (no SDK dependency) through an injectable FetchLike seam, so the retry,
|
|
16
16
|
// ZDR fallback, and state threading are testable with a fake.
|
|
@@ -412,24 +412,17 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
412
412
|
// An unset or empty capture variable leaves capture off.
|
|
413
413
|
const capture = createWireCapture(optionalString((options.env ?? process.env)[WIRE_CAPTURE_ENV]?.trim()));
|
|
414
414
|
let interactionUsageIncomplete = false;
|
|
415
|
-
const
|
|
415
|
+
const conversation = new OpenAiConversation({
|
|
416
416
|
model,
|
|
417
417
|
reasoningEffort,
|
|
418
418
|
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
419
419
|
...(options.safetyIdentifier === undefined
|
|
420
420
|
? {}
|
|
421
421
|
: { safetyIdentifier: options.safetyIdentifier }),
|
|
422
|
-
};
|
|
423
|
-
const state = {
|
|
424
|
-
lastResponseId: undefined,
|
|
425
|
-
pendingCallIds: [],
|
|
426
|
-
replies: 0,
|
|
427
|
-
conversation: new CarriedConversation(),
|
|
428
|
-
mode: options.zeroDataRetention ? "explicit_context" : "previous_response_id",
|
|
429
|
-
// Latches to undefined (stop asking) for the rest of the session when the
|
|
430
|
-
// account/model rejects the summary request; see OpenAiResponsesProviderOptions.
|
|
431
422
|
reasoningSummary: options.reasoningSummary === "off" ? undefined : (options.reasoningSummary ?? "auto"),
|
|
432
|
-
|
|
423
|
+
zeroDataRetention: options.zeroDataRetention === true,
|
|
424
|
+
now: options.now ?? Date.now,
|
|
425
|
+
});
|
|
433
426
|
const transport = {
|
|
434
427
|
endpoint: options.endpoint ?? OPENAI_RESPONSES_URL,
|
|
435
428
|
apiKey: options.apiKey,
|
|
@@ -441,25 +434,21 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
441
434
|
},
|
|
442
435
|
};
|
|
443
436
|
const post = (body, signal, retries = maxRetries, interaction = true, spend) => postResponse(transport, body, signal, retries, interaction, spend);
|
|
444
|
-
const record = new ConversationRecord(options.zeroDataRetention === true, options.now ?? Date.now);
|
|
445
437
|
// POST with the recoverable-policy latches: a ZDR rejection switches to explicit-context mode; a
|
|
446
438
|
// reasoning-summary rejection latches summaries off. Each latch can flip only once, so the loop
|
|
447
439
|
// is bounded; anything else rethrows. The body is rebuilt per attempt so a flipped latch shows.
|
|
448
|
-
const postTurn = async (
|
|
440
|
+
const postTurn = async (request, signal, spend) => {
|
|
449
441
|
if (options.singleDispatch === true)
|
|
450
|
-
return post(
|
|
442
|
+
return post(request.body(), signal, 0);
|
|
451
443
|
for (;;) {
|
|
452
444
|
try {
|
|
453
|
-
return await post(
|
|
445
|
+
return await post(request.body(), signal, maxRetries, true, spend);
|
|
454
446
|
}
|
|
455
447
|
catch (error) {
|
|
456
|
-
if (error instanceof SummaryRejectionError &&
|
|
457
|
-
state.reasoningSummary = undefined;
|
|
448
|
+
if (error instanceof SummaryRejectionError && conversation.dropReasoningSummaries()) {
|
|
458
449
|
continue;
|
|
459
450
|
}
|
|
460
|
-
if (error instanceof ZdrError &&
|
|
461
|
-
record.switched(error.rejection);
|
|
462
|
-
state.mode = "explicit_context";
|
|
451
|
+
if (error instanceof ZdrError && conversation.switchToExplicitContext(error.rejection)) {
|
|
463
452
|
continue;
|
|
464
453
|
}
|
|
465
454
|
throw error;
|
|
@@ -468,29 +457,26 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
468
457
|
};
|
|
469
458
|
const requestTurn = async (req, signal, closing = false, spend) => {
|
|
470
459
|
await capture.prepareNext();
|
|
471
|
-
const
|
|
460
|
+
const request = conversation.request(req, closing);
|
|
472
461
|
// A closing report makes exactly one request: no HTTP or policy-latch retries.
|
|
473
462
|
const raw = closing
|
|
474
|
-
? await post(
|
|
475
|
-
: await postTurn(
|
|
476
|
-
|
|
477
|
-
record.requested(state, sent);
|
|
478
|
-
const parsed = parseOpenAiResponse(raw);
|
|
479
|
-
acceptReply(state, parsed, closing, sent);
|
|
463
|
+
? await post(request.body(), signal, 0, false)
|
|
464
|
+
: await postTurn(request, signal, spend);
|
|
465
|
+
const turn = request.accept(raw);
|
|
480
466
|
if (closing) {
|
|
481
467
|
// Refusals, incomplete output, malformed JSON, and invalid shapes remain no-report results.
|
|
482
468
|
// Never promote raw JSON or a fallback paragraph into a structured finding.
|
|
483
469
|
try {
|
|
484
|
-
const report = JSON.parse(
|
|
470
|
+
const report = JSON.parse(turn.message ?? "");
|
|
485
471
|
if (asRecord(raw).status === "completed" && validClosingReport(report)) {
|
|
486
|
-
return { ...
|
|
472
|
+
return { ...turn, closingReport: report };
|
|
487
473
|
}
|
|
488
474
|
}
|
|
489
475
|
catch {
|
|
490
476
|
/* A failed optional closing account keeps its usage and no report. */
|
|
491
477
|
}
|
|
492
478
|
}
|
|
493
|
-
return
|
|
479
|
+
return turn;
|
|
494
480
|
};
|
|
495
481
|
return {
|
|
496
482
|
id: "openai-responses-cu",
|
|
@@ -512,15 +498,13 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
512
498
|
},
|
|
513
499
|
nextTurn: (req, signal, spend) => requestTurn(req, signal, false, spend),
|
|
514
500
|
get conversation() {
|
|
515
|
-
return record
|
|
501
|
+
return conversation.record();
|
|
516
502
|
},
|
|
517
|
-
//
|
|
518
|
-
// hold the whole session a retrospective report needs. This getter follows both configured
|
|
519
|
-
// ZDR and a runtime policy latch.
|
|
503
|
+
// A retrospective report reads the session the server holds; see serverHoldsSession.
|
|
520
504
|
get debrief() {
|
|
521
|
-
return
|
|
522
|
-
?
|
|
523
|
-
:
|
|
505
|
+
return conversation.serverHoldsSession
|
|
506
|
+
? (req, signal) => requestTurn(req, signal, true)
|
|
507
|
+
: undefined;
|
|
524
508
|
},
|
|
525
509
|
};
|
|
526
510
|
}
|
|
@@ -191,6 +191,8 @@ export interface ParticipantClosingReport {
|
|
|
191
191
|
export interface ActorRuntimeProvenance {
|
|
192
192
|
schema: "humanish.actor-runtime.v1";
|
|
193
193
|
package: string;
|
|
194
|
+
/** Terminal command telemetry opt-out; absent in older bundles. */
|
|
195
|
+
doNotTrack?: boolean;
|
|
194
196
|
requestedVersion: string;
|
|
195
197
|
observedVersion?: string;
|
|
196
198
|
versionStatus: "unobserved" | "verified" | "failed";
|
|
@@ -24,6 +24,11 @@ export interface ScriptedBrowserSessionOptions {
|
|
|
24
24
|
/** DI seam; production default is launchPlaywrightChromium. */
|
|
25
25
|
launchBrowser?: (args: ScriptedBrowserLaunchArgs) => Promise<ScriptedBrowserLike>;
|
|
26
26
|
now?: () => number;
|
|
27
|
+
/**
|
|
28
|
+
* The run's scrub of its known secret values. Each URL and text the session records passes
|
|
29
|
+
* through it before the trace is written or the capture returned.
|
|
30
|
+
*/
|
|
31
|
+
scrubKnownValues?: (text: string) => string;
|
|
27
32
|
}
|
|
28
33
|
export interface ScriptedBrowserSessionResult extends ActorSessionResult {
|
|
29
34
|
/** Native evidence incl. tracePath (humanish.browser-persona-trace.v1, written to disk). */
|
|
@@ -21,7 +21,7 @@ import { CHROMIUM_EVIDENCE_HYGIENE_FLAGS } from "../../evidence/browser-hygiene.
|
|
|
21
21
|
import { redactText } from "../../evidence/redaction.js";
|
|
22
22
|
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputDirectory, prepareContainedOutputFile, prepareSelectedOutputDirectory, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
23
23
|
import { ACTOR_TRACE_SCHEMA, SCRIPTED_BROWSER_CAPABILITIES, } from "../contract.js";
|
|
24
|
-
import { buildBlockedBrowserPersonaSteps, buildBrowserTrace, captureBlockedStepScreenshot, compactBrowserError, executeBrowserPersonaStep, probeAppUrl, sanitizeBrowserEvidenceUrl, screenshotPathForBrowserStep, surfaceScreenshotPath, tracePathForBrowserSurface, } from "./steps.js";
|
|
24
|
+
import { buildBlockedBrowserPersonaSteps, buildBrowserTrace, captureBlockedStepScreenshot, compactBrowserError, executeBrowserPersonaStep, probeAppUrl, sanitizeBrowserEvidenceUrl, screenshotPathForBrowserStep, scrubRecordedStep, surfaceScreenshotPath, tracePathForBrowserSurface, } from "./steps.js";
|
|
25
25
|
import { LOOPBACK_EVIDENCE_URL_POLICY, } from "./types.js";
|
|
26
26
|
import { chromeLaunchEnv, chromeLaunchError } from "./launch-env.js";
|
|
27
27
|
/** Production default: lazy playwright-core import + chromium.launch, exactly as the driver
|
|
@@ -93,8 +93,10 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
93
93
|
const browserCommand = options.browserCommand ?? (options.launchBrowser ? "injected-browser" : "");
|
|
94
94
|
const evidenceAppUrl = options.evidenceAppUrl ?? options.appUrl;
|
|
95
95
|
const urlPolicy = options.urlPolicy ?? LOOPBACK_EVIDENCE_URL_POLICY;
|
|
96
|
+
const scrub = options.scrubKnownValues ?? ((text) => text);
|
|
96
97
|
const finish = async (args) => {
|
|
97
98
|
const completedAtMs = now();
|
|
99
|
+
const reason = scrub(args.reason);
|
|
98
100
|
const trace = await projectScriptedActorTrace({
|
|
99
101
|
artifactRoot: preparedArtifactRoot,
|
|
100
102
|
capture: args.capture,
|
|
@@ -104,14 +106,14 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
104
106
|
executedSteps: args.executedSteps,
|
|
105
107
|
journey: options.journey,
|
|
106
108
|
persona: options.persona,
|
|
107
|
-
reason
|
|
109
|
+
reason,
|
|
108
110
|
startedAt,
|
|
109
111
|
status: args.status,
|
|
110
112
|
});
|
|
111
113
|
return {
|
|
112
114
|
status: args.status,
|
|
113
115
|
completionReason: args.completionReason,
|
|
114
|
-
reason
|
|
116
|
+
reason,
|
|
115
117
|
capture: args.capture,
|
|
116
118
|
trace,
|
|
117
119
|
};
|
|
@@ -130,6 +132,7 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
130
132
|
evidenceAppUrl,
|
|
131
133
|
journey: options.journey,
|
|
132
134
|
reason,
|
|
135
|
+
scrub,
|
|
133
136
|
surface: options.surface,
|
|
134
137
|
urlPolicy,
|
|
135
138
|
});
|
|
@@ -155,6 +158,7 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
155
158
|
browserCommand,
|
|
156
159
|
evidenceAppUrl,
|
|
157
160
|
journey: options.journey,
|
|
161
|
+
scrub,
|
|
158
162
|
surface: options.surface,
|
|
159
163
|
timeoutMs: options.timeoutMs,
|
|
160
164
|
urlPolicy,
|
|
@@ -258,17 +262,18 @@ async function runScriptedJourney(args) {
|
|
|
258
262
|
finally {
|
|
259
263
|
await args.browser.close().catch(() => undefined);
|
|
260
264
|
}
|
|
265
|
+
const recordedSteps = steps.map((step) => scrubRecordedStep(step, args.scrub));
|
|
261
266
|
const completedAt = new Date().toISOString();
|
|
262
267
|
const durationMs = Date.now() - started;
|
|
263
268
|
const ok = !timedOut &&
|
|
264
269
|
httpProbe.ok &&
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const reason = ok
|
|
268
|
-
? `${args.surface.label} completed ${
|
|
269
|
-
: `${args.surface.label} scripted browser journey blocked: ${
|
|
270
|
-
const scriptedScreenshotPath = surfaceScreenshotPath(
|
|
271
|
-
await
|
|
270
|
+
recordedSteps.length === args.journey.steps.length &&
|
|
271
|
+
recordedSteps.every((step) => step.status === "passed");
|
|
272
|
+
const reason = args.scrub(ok
|
|
273
|
+
? `${args.surface.label} completed ${recordedSteps.length}/${recordedSteps.length} scripted browser steps from ${args.evidenceAppUrl}${httpProbe.status === undefined ? "" : ` with HTTP ${httpProbe.status}`}.`
|
|
274
|
+
: `${args.surface.label} scripted browser journey blocked: ${recordedSteps.find((step) => step.status !== "passed")?.reason ?? httpProbe.reason}`);
|
|
275
|
+
const scriptedScreenshotPath = surfaceScreenshotPath(recordedSteps);
|
|
276
|
+
await writeBrowserTrace(args.artifactRoot, tracePath, {
|
|
272
277
|
appUrl: args.evidenceAppUrl,
|
|
273
278
|
browserCommand: path.basename(args.browserCommand || "injected-browser"),
|
|
274
279
|
browserJourney: args.journey,
|
|
@@ -278,9 +283,9 @@ async function runScriptedJourney(args) {
|
|
|
278
283
|
ok,
|
|
279
284
|
reason,
|
|
280
285
|
...(scriptedScreenshotPath === undefined ? {} : { screenshotPath: scriptedScreenshotPath }),
|
|
281
|
-
steps,
|
|
286
|
+
steps: recordedSteps,
|
|
282
287
|
surface: args.surface,
|
|
283
|
-
}
|
|
288
|
+
}, args.scrub);
|
|
284
289
|
return {
|
|
285
290
|
capture: {
|
|
286
291
|
capturedAt: completedAt,
|
|
@@ -289,7 +294,7 @@ async function runScriptedJourney(args) {
|
|
|
289
294
|
ok,
|
|
290
295
|
reason,
|
|
291
296
|
...(scriptedScreenshotPath === undefined ? {} : { screenshotPath: scriptedScreenshotPath }),
|
|
292
|
-
steps,
|
|
297
|
+
steps: recordedSteps,
|
|
293
298
|
surface: args.surface,
|
|
294
299
|
tracePath,
|
|
295
300
|
},
|
|
@@ -327,40 +332,56 @@ async function withJourneyDeadline(promise, deadline, timeoutMs) {
|
|
|
327
332
|
async function persistScriptedFailureCapture(args) {
|
|
328
333
|
const capturedAt = new Date().toISOString();
|
|
329
334
|
const tracePath = tracePathForBrowserSurface(args.surface);
|
|
335
|
+
const reason = args.scrub(args.reason);
|
|
330
336
|
const blockedSteps = buildBlockedBrowserPersonaSteps({
|
|
331
337
|
browserJourney: args.journey,
|
|
332
338
|
currentUrl: args.appUrl,
|
|
333
|
-
reason
|
|
339
|
+
reason,
|
|
334
340
|
surface: args.surface,
|
|
335
341
|
timestamp: capturedAt,
|
|
336
342
|
urlPolicy: args.urlPolicy,
|
|
337
|
-
});
|
|
343
|
+
}).map((step) => scrubRecordedStep(step, args.scrub));
|
|
338
344
|
// Pre-actuation failure: no screenshots were written, so the surface omits the
|
|
339
345
|
// screenshot reference and the failure itself stands as the evidence.
|
|
340
346
|
const screenshotPath = surfaceScreenshotPath(blockedSteps);
|
|
341
|
-
await
|
|
347
|
+
await writeBrowserTrace(args.artifactRoot, tracePath, {
|
|
342
348
|
appUrl: args.evidenceAppUrl,
|
|
343
349
|
browserCommand: path.basename(args.browserCommand || "injected-browser"),
|
|
344
350
|
browserJourney: args.journey,
|
|
345
351
|
capturedAt,
|
|
346
352
|
durationMs: 0,
|
|
347
353
|
ok: false,
|
|
348
|
-
reason
|
|
354
|
+
reason,
|
|
349
355
|
...(screenshotPath === undefined ? {} : { screenshotPath }),
|
|
350
356
|
steps: blockedSteps,
|
|
351
357
|
surface: args.surface,
|
|
352
|
-
}
|
|
358
|
+
}, args.scrub);
|
|
353
359
|
return {
|
|
354
360
|
capturedAt,
|
|
355
361
|
durationMs: 0,
|
|
356
362
|
ok: false,
|
|
357
|
-
reason
|
|
363
|
+
reason,
|
|
358
364
|
...(screenshotPath === undefined ? {} : { screenshotPath }),
|
|
359
365
|
steps: blockedSteps,
|
|
360
366
|
surface: args.surface,
|
|
361
367
|
tracePath,
|
|
362
368
|
};
|
|
363
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* Writes one surface's native trace. Its steps and reason arrive scrubbed; the app URL and the
|
|
372
|
+
* scenario title pass through the same scrub here.
|
|
373
|
+
*/
|
|
374
|
+
async function writeBrowserTrace(artifactRoot, tracePath, trace, scrub) {
|
|
375
|
+
const recorded = buildBrowserTrace({
|
|
376
|
+
...trace,
|
|
377
|
+
appUrl: scrub(trace.appUrl),
|
|
378
|
+
browserJourney: {
|
|
379
|
+
...trace.browserJourney,
|
|
380
|
+
scenarioTitle: scrub(trace.browserJourney.scenarioTitle),
|
|
381
|
+
},
|
|
382
|
+
});
|
|
383
|
+
await writeContainedOutputFile(artifactRoot, tracePath, `${JSON.stringify(recorded, null, 2)}\n`, "utf8");
|
|
384
|
+
}
|
|
364
385
|
/** Project one surface capture into humanish.actor-trace.v1. screenshotRefs are attached only
|
|
365
386
|
* for frames that actually exist on disk (so counts match disk; blocked-not-executed steps name a
|
|
366
387
|
* path that was never written). */
|