humanish 0.111.4 → 0.113.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +11 -0
- package/TELEMETRY.md +21 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +21 -1
- package/dist/actors/codex/restricted-participant-policy.js +20 -2
- package/dist/actors/codex/restricted-participant.d.ts +1 -1
- package/dist/actors/codex/restricted-participant.js +2 -1
- package/dist/actors/computer-use/loop/debrief.d.ts +12 -4
- package/dist/actors/computer-use/loop/debrief.js +57 -20
- package/dist/actors/computer-use/loop/ending.d.ts +5 -2
- package/dist/actors/computer-use/loop/impressions.d.ts +14 -0
- package/dist/actors/computer-use/loop/impressions.js +40 -0
- package/dist/actors/computer-use/loop/session.d.ts +3 -1
- package/dist/actors/computer-use/loop/session.js +2 -0
- package/dist/actors/computer-use/loop/trace.js +2 -0
- package/dist/actors/computer-use/loop/types.d.ts +8 -1
- package/dist/actors/computer-use/loop.d.ts +2 -1
- package/dist/actors/computer-use/loop.js +15 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +96 -0
- package/dist/actors/computer-use/openai-conversation.js +238 -0
- package/dist/actors/computer-use/openai-provider.js +38 -48
- package/dist/actors/contract.d.ts +45 -3
- package/dist/actors/contract.js +13 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/evidence-sources.d.ts +3 -1
- package/dist/analysis/evidence-sources.js +10 -1
- package/dist/analysis/evidence.js +2 -2
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +37 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +37 -20
- package/dist/cli/commands/setup.js +95 -6
- package/dist/cli/commands/study-format.js +7 -4
- package/dist/cli/commands/study.js +1 -1
- package/dist/cli/commands/tui.js +1 -1
- package/dist/cli/doctor.d.ts +2 -0
- package/dist/cli/doctor.js +18 -0
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +4 -1
- package/dist/cli/io.js +3 -2
- package/dist/cli/program.js +35 -1
- package/dist/cli/secret-prompt.d.ts +5 -1
- package/dist/cli/secret-prompt.js +6 -2
- package/dist/cli/update-check-worker.d.ts +1 -0
- package/dist/cli/update-check-worker.js +5 -0
- package/dist/cli/update-check.d.ts +46 -0
- package/dist/cli/update-check.js +203 -0
- package/dist/feedback/draft.js +32 -4
- package/dist/keys/key-resolution.js +2 -0
- package/dist/keys/key-status.d.ts +37 -0
- package/dist/keys/key-status.js +68 -0
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/local-vm.js +9 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +6 -0
- package/dist/routes/computer-use/participant-execution.js +13 -12
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/rerun-selection.js +23 -21
- package/dist/routes/computer-use/result.js +10 -17
- package/dist/routes/computer-use/self-report.js +7 -1
- package/dist/routes/computer-use/setup.js +14 -0
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/computer-use/types.d.ts +4 -1
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +24 -30
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +71 -79
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +13 -4
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +31 -0
- package/dist/run/participant-caption.js +60 -0
- package/dist/run/participant-records.d.ts +4 -2
- package/dist/run/participant-records.js +6 -0
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/run-study.js +17 -1
- package/dist/study/discover.js +6 -1
- package/dist/study/doctor.d.ts +2 -0
- package/dist/study/doctor.js +45 -7
- package/dist/study/init-templates.js +10 -5
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/plan.js +3 -2
- package/dist/study/preflight-result.d.ts +1 -0
- package/dist/study/preflight-result.js +1 -1
- package/dist/study/preflight.d.ts +2 -0
- package/dist/study/preflight.js +54 -14
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +7 -1
- package/dist/study/warnings.js +82 -4
- package/dist/substrates/local/capacity.d.ts +75 -0
- package/dist/substrates/local/capacity.js +143 -0
- package/dist/substrates/local/firecracker-desktop.d.ts +8 -0
- package/dist/substrates/local/firecracker-desktop.js +34 -3
- package/dist/substrates/local/runtime-host.d.ts +16 -3
- package/dist/substrates/local/runtime-host.js +49 -4
- package/dist/substrates/local/runtime.d.ts +10 -0
- package/dist/substrates/local/runtime.js +44 -5
- package/dist/tui-app.js +75 -75
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/actor-contract.md +20 -6
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/feedback.md +8 -1
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +65 -34
- package/docs/contracts/study-analysis.md +86 -22
- package/package.json +2 -1
- package/skills/humanish/SKILL.md +21 -10
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { CuaTurn, CuaTurnRequest } from "./loop.js";
|
|
2
|
+
import type { ReasoningEffort } from "../reasoning-effort.js";
|
|
3
|
+
import { type ActorConversation } from "../contract.js";
|
|
4
|
+
import { type OpenAiReasoningSummary } from "./openai-wire.js";
|
|
5
|
+
/** How a conversation starts. The model settings stay fixed for the whole session. */
|
|
6
|
+
export interface OpenAiConversationSettings {
|
|
7
|
+
readonly model: string;
|
|
8
|
+
readonly reasoningEffort: ReasoningEffort;
|
|
9
|
+
readonly maxOutputTokens?: number;
|
|
10
|
+
readonly safetyIdentifier?: string;
|
|
11
|
+
/** The summaries to ask for until the account or model rejects them; undefined asks for none. */
|
|
12
|
+
readonly reasoningSummary: OpenAiReasoningSummary | undefined;
|
|
13
|
+
/** Carry the conversation on the client from the first request (explicit_context). */
|
|
14
|
+
readonly zeroDataRetention: boolean;
|
|
15
|
+
/** The clock the trace's record reads. */
|
|
16
|
+
readonly now: () => number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* One request of the conversation. Its new items are built once, so every attempt at it, and the
|
|
20
|
+
* reply it accepts, see the same ones.
|
|
21
|
+
*/
|
|
22
|
+
export interface ConversationRequest {
|
|
23
|
+
/** The body as the conversation stands now, so an attempt after a rejection shows its effect. */
|
|
24
|
+
body(): Record<string, unknown>;
|
|
25
|
+
/**
|
|
26
|
+
* The request was answered: records a participant request with what it carried, moves the
|
|
27
|
+
* conversation on to the reply and returns the reply's turn.
|
|
28
|
+
*/
|
|
29
|
+
accept(reply: unknown): CuaTurn;
|
|
30
|
+
}
|
|
31
|
+
/** Which answer told the provider that the organization keeps no server-side conversation. */
|
|
32
|
+
export type ZdrRejection = NonNullable<ActorConversation["rejection"]>;
|
|
33
|
+
/**
|
|
34
|
+
* What a closing request asks for: the closing report after a harness-owned stop, or only
|
|
35
|
+
* impressions after the participant ended the session itself.
|
|
36
|
+
*/
|
|
37
|
+
export type ClosingRequest = "report" | "impressions";
|
|
38
|
+
export declare class OpenAiConversation {
|
|
39
|
+
private readonly settings;
|
|
40
|
+
private lastResponseId;
|
|
41
|
+
/** computer_call ids the next request owes an output for. */
|
|
42
|
+
private pendingCallIds;
|
|
43
|
+
/** Accepted replies so far; the next request is number `replies + 1`. */
|
|
44
|
+
private replies;
|
|
45
|
+
/** The whole conversation, which explicit-context requests carry within its token budget. */
|
|
46
|
+
private readonly carried;
|
|
47
|
+
private mode;
|
|
48
|
+
/** Undefined once the account or model rejected reasoning summaries. */
|
|
49
|
+
private reasoningSummary;
|
|
50
|
+
private readonly evidence;
|
|
51
|
+
constructor(settings: OpenAiConversationSettings);
|
|
52
|
+
/**
|
|
53
|
+
* The next request, carrying `req`'s screen: the opening request, or the outputs the latest
|
|
54
|
+
* reply's computer calls are owed. A closing request asks for strict JSON and no tools.
|
|
55
|
+
*/
|
|
56
|
+
request(req: CuaTurnRequest, closing?: ClosingRequest): ConversationRequest;
|
|
57
|
+
/**
|
|
58
|
+
* The organization rejected server-side state: carry the conversation on the client from the
|
|
59
|
+
* request being sent, and record which answer made the switch. False when the conversation is
|
|
60
|
+
* already carried, so the rejection stands.
|
|
61
|
+
*/
|
|
62
|
+
switchToExplicitContext(rejection: ZdrRejection): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* The account or model rejected reasoning summaries: stop asking for the rest of the session,
|
|
65
|
+
* from the request being sent. False when none are asked for, so the rejection stands.
|
|
66
|
+
*/
|
|
67
|
+
dropReasoningSummaries(): boolean;
|
|
68
|
+
/**
|
|
69
|
+
* Whether the server holds the whole session, so a closing report threaded on it reads every
|
|
70
|
+
* turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
|
|
71
|
+
* or switched to at runtime.
|
|
72
|
+
*/
|
|
73
|
+
get serverHoldsSession(): boolean;
|
|
74
|
+
/** The trace's account of how the conversation was carried (ActorConversation). */
|
|
75
|
+
record(): ActorConversation;
|
|
76
|
+
/**
|
|
77
|
+
* The items a request adds to the conversation: the opening message with the first screen, or
|
|
78
|
+
* an output for each pending computer_call with the latest screenshot and any hint.
|
|
79
|
+
*/
|
|
80
|
+
private newItems;
|
|
81
|
+
private context;
|
|
82
|
+
/**
|
|
83
|
+
* A participant request: the opening request, or a continuation carrying `sent`. A threaded
|
|
84
|
+
* continuation names the previous response; an explicit-context one carries the conversation
|
|
85
|
+
* ahead of `sent`.
|
|
86
|
+
*/
|
|
87
|
+
private turnBody;
|
|
88
|
+
/** A read-only closing request: no tools, its own output limit, a strict schema. */
|
|
89
|
+
private closingBody;
|
|
90
|
+
/**
|
|
91
|
+
* Move the conversation on to a reply to a request that added `sent`. A participant reply cut
|
|
92
|
+
* off by the output limit is set aside, so the next request is the same one again
|
|
93
|
+
* (outputLimitRetry); its actions never run, so no call output is owed for them.
|
|
94
|
+
*/
|
|
95
|
+
private accept;
|
|
96
|
+
}
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import { PARTICIPANT_IMPRESSION_KINDS } from "../contract.js";
|
|
2
|
+
import { CarriedConversation, estimateTokens } from "./openai-context.js";
|
|
3
|
+
import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintItems, openingMessage, parseOpenAiResponse, } from "./openai-wire.js";
|
|
4
|
+
/**
|
|
5
|
+
* The closing request's output limit: its own cap, or the declared limit when that is lower. It
|
|
6
|
+
* counts reasoning too, and leaves room for six impressions after the summary and friction reports.
|
|
7
|
+
*/
|
|
8
|
+
const CLOSING_OUTPUT_LIMIT = 3072;
|
|
9
|
+
const IMPRESSIONS_SCHEMA = {
|
|
10
|
+
type: "array",
|
|
11
|
+
items: {
|
|
12
|
+
type: "object",
|
|
13
|
+
additionalProperties: false,
|
|
14
|
+
required: ["kind", "text"],
|
|
15
|
+
properties: {
|
|
16
|
+
kind: { type: "string", enum: [...PARTICIPANT_IMPRESSION_KINDS] },
|
|
17
|
+
text: { type: "string" },
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
const CLOSING_FORMATS = {
|
|
22
|
+
report: {
|
|
23
|
+
type: "json_schema",
|
|
24
|
+
name: "participant_closing_report",
|
|
25
|
+
strict: true,
|
|
26
|
+
schema: {
|
|
27
|
+
type: "object",
|
|
28
|
+
additionalProperties: false,
|
|
29
|
+
required: ["summary", "frictionReports", "impressions"],
|
|
30
|
+
properties: {
|
|
31
|
+
summary: { type: "string" },
|
|
32
|
+
frictionReports: { type: "array", items: { type: "string" } },
|
|
33
|
+
impressions: IMPRESSIONS_SCHEMA,
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
impressions: {
|
|
38
|
+
type: "json_schema",
|
|
39
|
+
name: "participant_impressions",
|
|
40
|
+
strict: true,
|
|
41
|
+
schema: {
|
|
42
|
+
type: "object",
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
required: ["impressions"],
|
|
45
|
+
properties: { impressions: IMPRESSIONS_SCHEMA },
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
export class OpenAiConversation {
|
|
50
|
+
settings;
|
|
51
|
+
lastResponseId;
|
|
52
|
+
/** computer_call ids the next request owes an output for. */
|
|
53
|
+
pendingCallIds = [];
|
|
54
|
+
/** Accepted replies so far; the next request is number `replies + 1`. */
|
|
55
|
+
replies = 0;
|
|
56
|
+
/** The whole conversation, which explicit-context requests carry within its token budget. */
|
|
57
|
+
carried = new CarriedConversation();
|
|
58
|
+
mode;
|
|
59
|
+
/** Undefined once the account or model rejected reasoning summaries. */
|
|
60
|
+
reasoningSummary;
|
|
61
|
+
evidence;
|
|
62
|
+
constructor(settings) {
|
|
63
|
+
this.settings = settings;
|
|
64
|
+
this.mode = settings.zeroDataRetention ? "explicit_context" : "threaded";
|
|
65
|
+
this.reasoningSummary = settings.reasoningSummary;
|
|
66
|
+
this.evidence = new ConversationRecord(settings.zeroDataRetention, settings.now);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The next request, carrying `req`'s screen: the opening request, or the outputs the latest
|
|
70
|
+
* reply's computer calls are owed. A closing request asks for strict JSON and no tools.
|
|
71
|
+
*/
|
|
72
|
+
request(req, closing) {
|
|
73
|
+
const sent = this.newItems(req);
|
|
74
|
+
return {
|
|
75
|
+
body: () => closing === undefined ? this.turnBody(req, sent) : this.closingBody(req, sent, closing),
|
|
76
|
+
accept: (reply) => this.accept(reply, closing !== undefined, sent),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The organization rejected server-side state: carry the conversation on the client from the
|
|
81
|
+
* request being sent, and record which answer made the switch. False when the conversation is
|
|
82
|
+
* already carried, so the rejection stands.
|
|
83
|
+
*/
|
|
84
|
+
switchToExplicitContext(rejection) {
|
|
85
|
+
if (this.mode === "explicit_context")
|
|
86
|
+
return false;
|
|
87
|
+
this.evidence.switched(rejection);
|
|
88
|
+
this.mode = "explicit_context";
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The account or model rejected reasoning summaries: stop asking for the rest of the session,
|
|
93
|
+
* from the request being sent. False when none are asked for, so the rejection stands.
|
|
94
|
+
*/
|
|
95
|
+
dropReasoningSummaries() {
|
|
96
|
+
if (this.reasoningSummary === undefined)
|
|
97
|
+
return false;
|
|
98
|
+
this.reasoningSummary = undefined;
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Whether the server holds the whole session, so a closing report threaded on it reads every
|
|
103
|
+
* turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
|
|
104
|
+
* or switched to at runtime.
|
|
105
|
+
*/
|
|
106
|
+
get serverHoldsSession() {
|
|
107
|
+
return this.mode === "threaded" && this.lastResponseId !== undefined;
|
|
108
|
+
}
|
|
109
|
+
/** The trace's account of how the conversation was carried (ActorConversation). */
|
|
110
|
+
record() {
|
|
111
|
+
return this.evidence.snapshot(this.mode);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The items a request adds to the conversation: the opening message with the first screen, or
|
|
115
|
+
* an output for each pending computer_call with the latest screenshot and any hint.
|
|
116
|
+
*/
|
|
117
|
+
newItems(req) {
|
|
118
|
+
if (this.replies === 0)
|
|
119
|
+
return [openingMessage(req.instructions, req.observation.screenshot)];
|
|
120
|
+
return [
|
|
121
|
+
...this.pendingCallIds.map((id) => buildCallOutput(id, req.observation.screenshot, req.acknowledgedSafetyChecks)),
|
|
122
|
+
...hintItems(req.contextHint),
|
|
123
|
+
];
|
|
124
|
+
}
|
|
125
|
+
context(instructions) {
|
|
126
|
+
const { settings } = this;
|
|
127
|
+
return {
|
|
128
|
+
stateless: this.mode === "explicit_context",
|
|
129
|
+
model: settings.model,
|
|
130
|
+
instructions,
|
|
131
|
+
reasoningEffort: settings.reasoningEffort,
|
|
132
|
+
...(settings.maxOutputTokens === undefined
|
|
133
|
+
? {}
|
|
134
|
+
: { maxOutputTokens: settings.maxOutputTokens }),
|
|
135
|
+
...(this.reasoningSummary === undefined ? {} : { reasoningSummary: this.reasoningSummary }),
|
|
136
|
+
...(settings.safetyIdentifier === undefined
|
|
137
|
+
? {}
|
|
138
|
+
: { safetyIdentifier: settings.safetyIdentifier }),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* A participant request: the opening request, or a continuation carrying `sent`. A threaded
|
|
143
|
+
* continuation names the previous response; an explicit-context one carries the conversation
|
|
144
|
+
* ahead of `sent`.
|
|
145
|
+
*/
|
|
146
|
+
turnBody(req, sent) {
|
|
147
|
+
const ctx = this.context(req.instructions);
|
|
148
|
+
if (this.replies === 0)
|
|
149
|
+
return buildInitialRequest(ctx, req.observation.screenshot);
|
|
150
|
+
return buildContinuationRequest({
|
|
151
|
+
ctx,
|
|
152
|
+
previousResponseId: this.lastResponseId,
|
|
153
|
+
callOutputs: [...sent],
|
|
154
|
+
...(this.mode === "explicit_context" ? { explicitContextItems: this.carried.carried() } : {}),
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
/** A read-only closing request: no tools, its own output limit, a strict schema. */
|
|
158
|
+
closingBody(req, sent, closing) {
|
|
159
|
+
return {
|
|
160
|
+
...this.turnBody(req, sent),
|
|
161
|
+
tool_choice: "none",
|
|
162
|
+
max_output_tokens: Math.min(this.settings.maxOutputTokens ?? CLOSING_OUTPUT_LIMIT, CLOSING_OUTPUT_LIMIT),
|
|
163
|
+
text: { format: CLOSING_FORMATS[closing] },
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Move the conversation on to a reply to a request that added `sent`. A participant reply cut
|
|
168
|
+
* off by the output limit is set aside, so the next request is the same one again
|
|
169
|
+
* (outputLimitRetry); its actions never run, so no call output is owed for them.
|
|
170
|
+
*/
|
|
171
|
+
accept(reply, closing, sent) {
|
|
172
|
+
// Recorded before the reply is accepted: accepting can cut the carried history, and the record
|
|
173
|
+
// counts what this request carried.
|
|
174
|
+
if (!closing)
|
|
175
|
+
this.evidence.requested(this.mode, this.carried, sent);
|
|
176
|
+
const parsed = parseOpenAiResponse(reply);
|
|
177
|
+
if (!closing && parsed.turn.interruption === "output_limit")
|
|
178
|
+
return parsed.turn;
|
|
179
|
+
if (parsed.turn.responseId !== undefined)
|
|
180
|
+
this.lastResponseId = parsed.turn.responseId;
|
|
181
|
+
this.pendingCallIds = parsed.callIds;
|
|
182
|
+
this.replies += 1;
|
|
183
|
+
this.carried.accept(this.replies, sent, parsed.outputItems);
|
|
184
|
+
return parsed.turn;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/** The trace's account of how a session carried its conversation (ActorConversation). */
|
|
188
|
+
class ConversationRecord {
|
|
189
|
+
now;
|
|
190
|
+
explicitReason;
|
|
191
|
+
rejection;
|
|
192
|
+
switchedAt;
|
|
193
|
+
switchedAtRequest;
|
|
194
|
+
summarizedTurns = 0;
|
|
195
|
+
requests = [];
|
|
196
|
+
constructor(configured, now) {
|
|
197
|
+
this.now = now;
|
|
198
|
+
if (configured)
|
|
199
|
+
this.explicitReason = "configured";
|
|
200
|
+
}
|
|
201
|
+
/** The organization rejected server-side state, so the request being sent is explicit_context. */
|
|
202
|
+
switched(rejection) {
|
|
203
|
+
this.explicitReason = "zdr_rejection";
|
|
204
|
+
this.rejection = rejection;
|
|
205
|
+
this.switchedAt = new Date(this.now()).toISOString();
|
|
206
|
+
// Counted like requests[], which includes replies set aside at their output limit.
|
|
207
|
+
this.switchedAtRequest = this.requests.length + 1;
|
|
208
|
+
}
|
|
209
|
+
/** A participant request went out with `sent` after `conversation`. */
|
|
210
|
+
requested(mode, conversation, sent) {
|
|
211
|
+
if (mode !== "explicit_context") {
|
|
212
|
+
this.requests.push({ mode: "threaded" });
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
const carried = conversation.size();
|
|
216
|
+
// Only summaries a request carried count; the conversation also trims after the last reply.
|
|
217
|
+
this.summarizedTurns = carried.notedTurns;
|
|
218
|
+
this.requests.push({
|
|
219
|
+
mode: "explicit_context",
|
|
220
|
+
carriedExchanges: carried.exchanges,
|
|
221
|
+
carriedScreenshots: carried.screenshots,
|
|
222
|
+
estimatedInputTokens: carried.estimatedTokens + estimateTokens(sent),
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
snapshot(mode) {
|
|
226
|
+
return {
|
|
227
|
+
mode,
|
|
228
|
+
...(this.explicitReason === undefined ? {} : { explicitReason: this.explicitReason }),
|
|
229
|
+
...(this.rejection === undefined ? {} : { rejection: this.rejection }),
|
|
230
|
+
...(this.switchedAt === undefined ? {} : { switchedAt: this.switchedAt }),
|
|
231
|
+
...(this.switchedAtRequest === undefined
|
|
232
|
+
? {}
|
|
233
|
+
: { switchedAtRequest: this.switchedAtRequest }),
|
|
234
|
+
summarizedTurns: this.summarizedTurns,
|
|
235
|
+
requests: this.requests.map((request) => ({ ...request })),
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { validClosingReport } from "./loop.js";
|
|
1
|
+
import { validClosingReport, validImpressionsReply } from "./loop.js";
|
|
2
2
|
import { ComputerUseAdmissionLimitError, isComputerUseAdmissionLimitError, } from "./admission-limit.js";
|
|
3
3
|
import { ComputerUsePromptRefusedError } from "./provider-error.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { asRecord, optionalString, parseOpenAiResponse, } from "./openai-wire.js";
|
|
4
|
+
import { OpenAiConversation, } from "./openai-conversation.js";
|
|
5
|
+
import { asRecord, optionalString } from "./openai-wire.js";
|
|
7
6
|
import { redactText } from "../../evidence/redaction.js";
|
|
8
7
|
import { isMaxOutputTokens } from "../output-token-limit.js";
|
|
9
8
|
import { prepareContainedOutputFile, prepareSelectedOutputDirectory, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
10
9
|
import { OPENAI_RESPONSES_URL } from "../openai-endpoint.js";
|
|
11
10
|
// A public-safe re-derivation of the OpenAI Responses API computer-use provider,
|
|
12
11
|
// behind the CuaProvider port from src/actors/computer-use/loop.ts. The pure wire mapping
|
|
13
|
-
// (openAiActionToCua, parseOpenAiResponse and the request builders) is in openai-wire.ts
|
|
12
|
+
// (openAiActionToCua, parseOpenAiResponse and the request builders) is in openai-wire.ts, and
|
|
13
|
+
// what each request carries, with the trace's record of it, is in openai-conversation.ts.
|
|
14
14
|
// This module is the live shim: createOpenAiResponsesProvider does a raw POST to the
|
|
15
15
|
// Responses endpoint (no SDK dependency) through an injectable FetchLike seam, so the retry,
|
|
16
16
|
// ZDR fallback, and state threading are testable with a fake.
|
|
@@ -412,24 +412,17 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
412
412
|
// An unset or empty capture variable leaves capture off.
|
|
413
413
|
const capture = createWireCapture(optionalString((options.env ?? process.env)[WIRE_CAPTURE_ENV]?.trim()));
|
|
414
414
|
let interactionUsageIncomplete = false;
|
|
415
|
-
const
|
|
415
|
+
const conversation = new OpenAiConversation({
|
|
416
416
|
model,
|
|
417
417
|
reasoningEffort,
|
|
418
418
|
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
419
419
|
...(options.safetyIdentifier === undefined
|
|
420
420
|
? {}
|
|
421
421
|
: { safetyIdentifier: options.safetyIdentifier }),
|
|
422
|
-
};
|
|
423
|
-
const state = {
|
|
424
|
-
lastResponseId: undefined,
|
|
425
|
-
pendingCallIds: [],
|
|
426
|
-
replies: 0,
|
|
427
|
-
conversation: new CarriedConversation(),
|
|
428
|
-
mode: options.zeroDataRetention ? "explicit_context" : "previous_response_id",
|
|
429
|
-
// Latches to undefined (stop asking) for the rest of the session when the
|
|
430
|
-
// account/model rejects the summary request; see OpenAiResponsesProviderOptions.
|
|
431
422
|
reasoningSummary: options.reasoningSummary === "off" ? undefined : (options.reasoningSummary ?? "auto"),
|
|
432
|
-
|
|
423
|
+
zeroDataRetention: options.zeroDataRetention === true,
|
|
424
|
+
now: options.now ?? Date.now,
|
|
425
|
+
});
|
|
433
426
|
const transport = {
|
|
434
427
|
endpoint: options.endpoint ?? OPENAI_RESPONSES_URL,
|
|
435
428
|
apiKey: options.apiKey,
|
|
@@ -441,56 +434,52 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
441
434
|
},
|
|
442
435
|
};
|
|
443
436
|
const post = (body, signal, retries = maxRetries, interaction = true, spend) => postResponse(transport, body, signal, retries, interaction, spend);
|
|
444
|
-
const record = new ConversationRecord(options.zeroDataRetention === true, options.now ?? Date.now);
|
|
445
437
|
// POST with the recoverable-policy latches: a ZDR rejection switches to explicit-context mode; a
|
|
446
438
|
// reasoning-summary rejection latches summaries off. Each latch can flip only once, so the loop
|
|
447
439
|
// is bounded; anything else rethrows. The body is rebuilt per attempt so a flipped latch shows.
|
|
448
|
-
const postTurn = async (
|
|
440
|
+
const postTurn = async (request, signal, spend) => {
|
|
449
441
|
if (options.singleDispatch === true)
|
|
450
|
-
return post(
|
|
442
|
+
return post(request.body(), signal, 0);
|
|
451
443
|
for (;;) {
|
|
452
444
|
try {
|
|
453
|
-
return await post(
|
|
445
|
+
return await post(request.body(), signal, maxRetries, true, spend);
|
|
454
446
|
}
|
|
455
447
|
catch (error) {
|
|
456
|
-
if (error instanceof SummaryRejectionError &&
|
|
457
|
-
state.reasoningSummary = undefined;
|
|
448
|
+
if (error instanceof SummaryRejectionError && conversation.dropReasoningSummaries()) {
|
|
458
449
|
continue;
|
|
459
450
|
}
|
|
460
|
-
if (error instanceof ZdrError &&
|
|
461
|
-
record.switched(error.rejection);
|
|
462
|
-
state.mode = "explicit_context";
|
|
451
|
+
if (error instanceof ZdrError && conversation.switchToExplicitContext(error.rejection)) {
|
|
463
452
|
continue;
|
|
464
453
|
}
|
|
465
454
|
throw error;
|
|
466
455
|
}
|
|
467
456
|
}
|
|
468
457
|
};
|
|
469
|
-
const requestTurn = async (req, signal, closing
|
|
458
|
+
const requestTurn = async (req, signal, closing, spend) => {
|
|
470
459
|
await capture.prepareNext();
|
|
471
|
-
const
|
|
472
|
-
// A closing
|
|
473
|
-
const raw = closing
|
|
474
|
-
? await
|
|
475
|
-
: await
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
const parsed = parseOpenAiResponse(raw);
|
|
479
|
-
acceptReply(state, parsed, closing, sent);
|
|
480
|
-
if (closing) {
|
|
460
|
+
const request = conversation.request(req, closing);
|
|
461
|
+
// A closing request makes exactly one request: no HTTP or policy-latch retries.
|
|
462
|
+
const raw = closing === undefined
|
|
463
|
+
? await postTurn(request, signal, spend)
|
|
464
|
+
: await post(request.body(), signal, 0, false);
|
|
465
|
+
const turn = request.accept(raw);
|
|
466
|
+
if (closing !== undefined) {
|
|
481
467
|
// Refusals, incomplete output, malformed JSON, and invalid shapes remain no-report results.
|
|
482
468
|
// Never promote raw JSON or a fallback paragraph into a structured finding.
|
|
483
469
|
try {
|
|
484
|
-
const
|
|
485
|
-
if (asRecord(raw).status === "completed"
|
|
486
|
-
|
|
470
|
+
const reply = JSON.parse(turn.message ?? "");
|
|
471
|
+
if (asRecord(raw).status === "completed") {
|
|
472
|
+
if (closing === "report" && validClosingReport(reply))
|
|
473
|
+
return { ...turn, closingReport: reply };
|
|
474
|
+
if (closing === "impressions" && validImpressionsReply(reply))
|
|
475
|
+
return { ...turn, impressions: reply.impressions };
|
|
487
476
|
}
|
|
488
477
|
}
|
|
489
478
|
catch {
|
|
490
479
|
/* A failed optional closing account keeps its usage and no report. */
|
|
491
480
|
}
|
|
492
481
|
}
|
|
493
|
-
return
|
|
482
|
+
return turn;
|
|
494
483
|
};
|
|
495
484
|
return {
|
|
496
485
|
id: "openai-responses-cu",
|
|
@@ -510,17 +499,18 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
510
499
|
get interactionUsageIncomplete() {
|
|
511
500
|
return interactionUsageIncomplete;
|
|
512
501
|
},
|
|
513
|
-
nextTurn: (req, signal, spend) => requestTurn(req, signal,
|
|
502
|
+
nextTurn: (req, signal, spend) => requestTurn(req, signal, undefined, spend),
|
|
514
503
|
get conversation() {
|
|
515
|
-
return record
|
|
504
|
+
return conversation.record();
|
|
516
505
|
},
|
|
517
|
-
//
|
|
518
|
-
// hold the whole session a retrospective report needs. This getter follows both configured
|
|
519
|
-
// ZDR and a runtime policy latch.
|
|
506
|
+
// A retrospective report reads the session the server holds; see serverHoldsSession.
|
|
520
507
|
get debrief() {
|
|
521
|
-
return
|
|
522
|
-
?
|
|
523
|
-
:
|
|
508
|
+
return conversation.serverHoldsSession
|
|
509
|
+
? (req, signal) => requestTurn(req, signal, "report")
|
|
510
|
+
: undefined;
|
|
524
511
|
},
|
|
512
|
+
// Offered in both modes. An explicit_context conversation carries its newest screens whole and
|
|
513
|
+
// earlier turns as text, which serves impressions; a closing report needs the whole session.
|
|
514
|
+
requestImpressions: (req, signal) => requestTurn(req, signal, "impressions"),
|
|
525
515
|
};
|
|
526
516
|
}
|
|
@@ -186,11 +186,43 @@ export declare function validActorProviderRequests(value: unknown): value is Act
|
|
|
186
186
|
export interface ParticipantClosingReport {
|
|
187
187
|
summary: string;
|
|
188
188
|
frictionReports: string[];
|
|
189
|
+
/** Absent when the provider did not ask for impressions; empty when the participant had none. */
|
|
190
|
+
impressions?: ParticipantImpression[];
|
|
189
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* What a participant can say about the product at the end of a session, beyond friction.
|
|
194
|
+
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
195
|
+
* life.
|
|
196
|
+
*/
|
|
197
|
+
export declare const PARTICIPANT_IMPRESSION_KINDS: readonly ["unclear", "unfinished", "untrustworthy", "liked", "missing", "unlike_my_work"];
|
|
198
|
+
type ParticipantImpressionKind = (typeof PARTICIPANT_IMPRESSION_KINDS)[number];
|
|
199
|
+
/** One first-person impression about something the participant saw in the session. */
|
|
200
|
+
export interface ParticipantImpression {
|
|
201
|
+
kind: ParticipantImpressionKind;
|
|
202
|
+
text: string;
|
|
203
|
+
}
|
|
204
|
+
/** An impression as the trace keeps it, with the message that quotes it. */
|
|
205
|
+
interface RecordedImpression extends ParticipantImpression {
|
|
206
|
+
/** The trace message carrying the impression, which the analysis quotes and cites. */
|
|
207
|
+
messageId: string;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* The impressions a session collected, or why it collected none. An empty `items` list means the
|
|
211
|
+
* participant had none to give.
|
|
212
|
+
*/
|
|
213
|
+
export type ParticipantImpressions = {
|
|
214
|
+
status: "collected";
|
|
215
|
+
items: RecordedImpression[];
|
|
216
|
+
} | {
|
|
217
|
+
status: "not_collected";
|
|
218
|
+
reason: string;
|
|
219
|
+
};
|
|
190
220
|
/** Runtime declarations and executable-version observations; not provider request attestation. */
|
|
191
221
|
export interface ActorRuntimeProvenance {
|
|
192
222
|
schema: "humanish.actor-runtime.v1";
|
|
193
223
|
package: string;
|
|
224
|
+
/** Terminal command telemetry opt-out; absent in older bundles. */
|
|
225
|
+
doNotTrack?: boolean;
|
|
194
226
|
requestedVersion: string;
|
|
195
227
|
observedVersion?: string;
|
|
196
228
|
versionStatus: "unobserved" | "verified" | "failed";
|
|
@@ -313,17 +345,27 @@ export interface ActorTrace {
|
|
|
313
345
|
* each fixed a false refusal and each left the next shape unhandled.
|
|
314
346
|
*/
|
|
315
347
|
declaredOutcome?: ParticipantDeclaredOutcome;
|
|
316
|
-
/**
|
|
348
|
+
/**
|
|
349
|
+
* The read-only closing request: a closing report after a harness-owned stop (`stop_when`,
|
|
350
|
+
* `dwell`), or impressions only after the participant ended the session itself
|
|
351
|
+
* (`participant_end`, never with `report`). Does not change task outcomes or permit actions.
|
|
352
|
+
*/
|
|
317
353
|
debrief?: {
|
|
318
|
-
trigger: "stop_when" | "dwell";
|
|
354
|
+
trigger: "stop_when" | "dwell" | "participant_end";
|
|
319
355
|
status: "completed" | "skipped" | "failed";
|
|
320
356
|
reason: string;
|
|
321
357
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
|
322
358
|
usageReported?: boolean;
|
|
323
|
-
report
|
|
359
|
+
/** The report's impressions are kept in `impressions`, with the messages that quote them. */
|
|
360
|
+
report?: Omit<ParticipantClosingReport, "impressions">;
|
|
324
361
|
/** Links the readable projection so it is not heuristically classified a second time. */
|
|
325
362
|
messageId?: string;
|
|
326
363
|
};
|
|
364
|
+
/**
|
|
365
|
+
* What the participant said about the product at the end of a computer-use session, or why
|
|
366
|
+
* nothing was collected. Absent on other routes and on older traces.
|
|
367
|
+
*/
|
|
368
|
+
impressions?: ParticipantImpressions;
|
|
327
369
|
items: ActorTraceItem[];
|
|
328
370
|
tokenUsage?: ActorTokenUsage;
|
|
329
371
|
/** A stalled or adapter-reported ambiguous interaction may have additional unreported usage.
|
package/dist/actors/contract.js
CHANGED
|
@@ -168,6 +168,19 @@ export function validActorProviderRequests(value) {
|
|
|
168
168
|
usage.input));
|
|
169
169
|
});
|
|
170
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* What a participant can say about the product at the end of a session, beyond friction.
|
|
173
|
+
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
174
|
+
* life.
|
|
175
|
+
*/
|
|
176
|
+
export const PARTICIPANT_IMPRESSION_KINDS = [
|
|
177
|
+
"unclear",
|
|
178
|
+
"unfinished",
|
|
179
|
+
"untrustworthy",
|
|
180
|
+
"liked",
|
|
181
|
+
"missing",
|
|
182
|
+
"unlike_my_work",
|
|
183
|
+
];
|
|
171
184
|
export const CODEX_APP_SERVER_CAPABILITIES = {
|
|
172
185
|
headless: true,
|
|
173
186
|
structuredTrace: true,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { lstat, readdir, rmdir, unlink } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { decodeEscapes } from "../evidence/encoded-text.js";
|
|
4
3
|
import { redactText } from "../evidence/redaction.js";
|
|
5
4
|
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, bindExistingManagedHumanishOutputDirectory, prepareManagedHumanishOutputDirectory, writeContainedOutputFile, } from "../run/contained-output.js";
|
|
6
|
-
import {
|
|
5
|
+
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
7
6
|
import { ANALYSIS_ID_PATTERN } from "./types.js";
|
|
8
7
|
// A rejected analyst response is kept for diagnosis under .humanish/analysis-diagnostics/<run>/,
|
|
9
8
|
// outside every run directory. Export, verify and the Observer read run directories only, so the
|
|
@@ -14,19 +13,15 @@ const ANALYSIS_DIAGNOSTIC_SCHEMA = "humanish.analysis-rejected-output.v1";
|
|
|
14
13
|
/** How many rejected outputs are kept across all runs; older ones are removed on each write. */
|
|
15
14
|
export const MAX_ANALYSIS_DIAGNOSTICS = 20;
|
|
16
15
|
const REDACTED = "[REDACTED_SECRET]";
|
|
17
|
-
const markers = (text) => text.split(REDACTED).length - 1;
|
|
18
16
|
/**
|
|
19
|
-
* Scrub every string, key and scalar in the output. The
|
|
20
|
-
* percent-encoded, escaped or base64-encoded
|
|
21
|
-
*
|
|
22
|
-
*
|
|
17
|
+
* Scrub every string, key and scalar in the output. The known-value scrub finds a value written
|
|
18
|
+
* percent-encoded, escaped or base64-encoded; a string where it finds nothing keeps its original
|
|
19
|
+
* spelling, so an escape in a rejected quote stays visible. A number or boolean equal to a known
|
|
20
|
+
* value becomes the marker.
|
|
23
21
|
*/
|
|
24
22
|
function scrubber() {
|
|
25
|
-
const
|
|
26
|
-
const text = (value) =>
|
|
27
|
-
const found = encoded(value);
|
|
28
|
-
return redactText(markers(found) > markers(decodeEscapes(value)) ? found : scrubTransientCommsText(value));
|
|
29
|
-
};
|
|
23
|
+
const known = transientCommsKnownValueScrub();
|
|
24
|
+
const text = (value) => redactText(known(value));
|
|
30
25
|
const scrub = (value) => {
|
|
31
26
|
if (typeof value === "string")
|
|
32
27
|
return text(value);
|
|
@@ -20,4 +20,6 @@ export interface SourceEntry {
|
|
|
20
20
|
export declare function sourceEntries(bundle: RunBundle, stream: RunStream, captureVersion?: CaptureVersion): SourceEntry[];
|
|
21
21
|
/** Max-min allocation; stable ID order breaks a remainder tie by at most one slot. */
|
|
22
22
|
export declare function fairShares(budget: number, capacities: number[]): number[];
|
|
23
|
-
|
|
23
|
+
/** The trace messages that carry a participant's closing impressions. */
|
|
24
|
+
export declare function impressionEventIds(stream: RunStream): ReadonlySet<string>;
|
|
25
|
+
export declare function sourceOrder(entries: SourceEntry[], capturesOnly: boolean, impressions?: ReadonlySet<string>): SourceEntry[];
|
|
@@ -204,7 +204,12 @@ function spreadOrder(entries) {
|
|
|
204
204
|
}
|
|
205
205
|
return result;
|
|
206
206
|
}
|
|
207
|
-
|
|
207
|
+
/** The trace messages that carry a participant's closing impressions. */
|
|
208
|
+
export function impressionEventIds(stream) {
|
|
209
|
+
const impressions = stream.actor?.impressions;
|
|
210
|
+
return new Set(impressions?.status === "collected" ? impressions.items.map((item) => item.messageId) : []);
|
|
211
|
+
}
|
|
212
|
+
export function sourceOrder(entries, capturesOnly, impressions = new Set()) {
|
|
208
213
|
const candidates = capturesOnly ? entries.filter((entry) => entry.capturePath !== null) : entries;
|
|
209
214
|
const ordered = new Set();
|
|
210
215
|
const admit = (entry) => {
|
|
@@ -214,6 +219,10 @@ export function sourceOrder(entries, capturesOnly) {
|
|
|
214
219
|
// Actor endings precede appended run bookkeeping when text slots are scarce.
|
|
215
220
|
if (!capturesOnly)
|
|
216
221
|
admit(entries.findLast((entry) => !entry.kind.startsWith("run_event:")));
|
|
222
|
+
// The participant's closing impressions are its only opinions of the product; keep them all.
|
|
223
|
+
for (const entry of entries)
|
|
224
|
+
if (impressions.has(entry.eventId))
|
|
225
|
+
admit(entry);
|
|
217
226
|
admit(candidates.at(-1));
|
|
218
227
|
admit(candidates[0]);
|
|
219
228
|
const afterCapture = [];
|