humanish 0.114.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/CONTEXT.md +1 -1
- package/README.md +7 -6
- package/TELEMETRY.md +6 -0
- package/dist/actors/closing-report-limits.d.ts +11 -0
- package/dist/actors/closing-report-limits.js +13 -0
- package/dist/actors/closing-report.d.ts +68 -0
- package/dist/actors/closing-report.js +55 -0
- package/dist/actors/codex/app-server-ui.js +11 -4
- package/dist/actors/codex/restricted-participant-policy.d.ts +4 -47
- package/dist/actors/codex/restricted-participant-policy.js +12 -40
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -10
- package/dist/actors/computer-use/loop/debrief.js +68 -70
- package/dist/actors/computer-use/loop/impressions.d.ts +2 -2
- package/dist/actors/computer-use/loop/impressions.js +6 -1
- package/dist/actors/computer-use/loop.d.ts +0 -1
- package/dist/actors/computer-use/loop.js +3 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +1 -1
- package/dist/actors/computer-use/openai-conversation.js +3 -29
- package/dist/actors/computer-use/openai-provider.js +11 -5
- package/dist/actors/contract.d.ts +2 -20
- package/dist/actors/contract.js +0 -13
- package/dist/actors/scripted-browser/steps.js +6 -5
- package/dist/analysis/admission.d.ts +14 -4
- package/dist/analysis/admission.js +14 -9
- package/dist/analysis/analysis-limits.d.ts +35 -0
- package/dist/analysis/analysis-limits.js +41 -0
- package/dist/analysis/execute.d.ts +5 -0
- package/dist/analysis/execute.js +3 -0
- package/dist/analysis/finding-lead.d.ts +27 -0
- package/dist/analysis/finding-lead.js +23 -0
- package/dist/analysis/service.js +5 -3
- package/dist/analysis/validation.js +14 -21
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/notes.js +5 -3
- package/dist/cli/commands/run-signals.js +5 -1
- package/dist/cli/commands/runs.js +2 -1
- package/dist/cli/commands/setup.js +3 -3
- package/dist/cli/commands/study-format.d.ts +5 -2
- package/dist/cli/commands/study-format.js +29 -9
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +1 -1
- package/dist/cli/commands/tui.js +59 -17
- package/dist/cli/doctor.js +9 -2
- package/dist/cli/findings-text.d.ts +5 -0
- package/dist/cli/findings-text.js +169 -0
- package/dist/cli/findings.d.ts +0 -4
- package/dist/cli/findings.js +11 -195
- package/dist/cli/io.js +10 -8
- package/dist/cli/telemetry.js +10 -22
- package/dist/cli/update-check.js +10 -19
- package/dist/cli/user-config.d.ts +11 -0
- package/dist/cli/user-config.js +25 -0
- package/dist/comms/connections.js +4 -3
- package/dist/comms/email-catch.js +7 -31
- package/dist/comms/setup.js +18 -12
- package/dist/evidence/encoded-text.d.ts +12 -11
- package/dist/evidence/encoded-text.js +81 -70
- package/dist/evidence/redaction.d.ts +11 -0
- package/dist/evidence/redaction.js +30 -11
- package/dist/evidence/secret-scrub.d.ts +26 -3
- package/dist/evidence/secret-scrub.js +172 -38
- package/dist/feedback/draft.d.ts +1 -1
- package/dist/feedback/draft.js +13 -16
- package/dist/feedback/export-bundle.js +21 -17
- package/dist/feedback/export.js +32 -46
- package/dist/feedback/feedback.js +31 -26
- package/dist/guest/desktop-keys.js +59 -5
- package/dist/keys/key-resolution.d.ts +9 -0
- package/dist/keys/key-resolution.js +62 -55
- package/dist/keys/key-status.d.ts +2 -0
- package/dist/keys/key-status.js +9 -16
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/http.d.ts +7 -0
- package/dist/observer/http.js +19 -1
- package/dist/observer/notes-route.d.ts +0 -1
- package/dist/observer/notes-route.js +24 -44
- package/dist/observer/pinned-files.js +4 -19
- package/dist/observer/render.js +6 -4
- package/dist/observer/run-routes.js +13 -28
- package/dist/observer/serve.js +2 -1
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -4
- package/dist/routes/computer-use/bundle-parts.js +2 -2
- package/dist/routes/computer-use/bundle.d.ts +4 -5
- package/dist/routes/computer-use/bundle.js +1 -3
- package/dist/routes/computer-use/fanout-records.js +10 -11
- package/dist/routes/computer-use/live-phase.d.ts +1 -1
- package/dist/routes/computer-use/live-phase.js +0 -1
- package/dist/routes/computer-use/participant-execution.d.ts +3 -2
- package/dist/routes/computer-use/participant-runs.d.ts +6 -2
- package/dist/routes/computer-use/participant-runs.js +9 -5
- package/dist/routes/computer-use/plan.d.ts +3 -0
- package/dist/routes/computer-use/plan.js +14 -11
- package/dist/routes/computer-use/rerun-selection.js +2 -2
- package/dist/routes/computer-use/result.js +3 -6
- package/dist/routes/computer-use/setup.d.ts +4 -6
- package/dist/routes/computer-use/setup.js +2 -6
- package/dist/routes/computer-use/single-bundle.d.ts +4 -5
- package/dist/routes/computer-use/single-bundle.js +22 -25
- package/dist/routes/computer-use/subject-projection.d.ts +4 -8
- package/dist/routes/computer-use/subject-projection.js +9 -31
- package/dist/routes/computer-use/types.d.ts +4 -41
- package/dist/routes/scripted/scenario.js +28 -12
- package/dist/routes/scripted/session-result.js +6 -3
- package/dist/routes/shared-world/bundle-records.js +2 -8
- package/dist/routes/shared-world/participant-specs.d.ts +0 -2
- package/dist/routes/shared-world/participant-specs.js +1 -20
- package/dist/routes/shared-world/plan.d.ts +3 -0
- package/dist/routes/shared-world/plan.js +23 -3
- package/dist/routes/shared-world/setup.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +9 -9
- package/dist/routes/terminal/ledger.js +63 -40
- package/dist/routes/terminal/live-finish.js +7 -10
- package/dist/routes/terminal/plan.d.ts +3 -0
- package/dist/routes/terminal/plan.js +8 -5
- package/dist/routes/terminal/token-usage.d.ts +1 -2
- package/dist/routes/terminal/token-usage.js +4 -6
- package/dist/routes/terminal/types.d.ts +9 -1
- package/dist/run/bundle-shape.js +6 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/contained-output.d.ts +59 -10
- package/dist/run/contained-output.js +160 -68
- package/dist/run/detail.d.ts +2 -1
- package/dist/run/detail.js +12 -7
- package/dist/run/escape-sequences.d.ts +11 -0
- package/dist/run/escape-sequences.js +93 -0
- package/dist/run/evidence-files.d.ts +8 -1
- package/dist/run/evidence-files.js +24 -18
- package/dist/run/locate.d.ts +32 -3
- package/dist/run/locate.js +45 -40
- package/dist/run/note-files.d.ts +26 -45
- package/dist/run/note-files.js +86 -58
- package/dist/run/note-shape.d.ts +35 -0
- package/dist/run/note-shape.js +20 -0
- package/dist/run/notes.d.ts +3 -14
- package/dist/run/notes.js +23 -75
- package/dist/run/outcomes.d.ts +24 -0
- package/dist/run/outcomes.js +19 -0
- package/dist/run/participant-records.d.ts +2 -0
- package/dist/run/participant-records.js +17 -2
- package/dist/run/paths.d.ts +2 -0
- package/dist/run/paths.js +5 -1
- package/dist/run/preflight-receipts.js +13 -8
- package/dist/run/project.js +7 -4
- package/dist/run/reclaim-outcomes.d.ts +8 -6
- package/dist/run/reclaim-outcomes.js +10 -6
- package/dist/run/reclaim.d.ts +5 -0
- package/dist/run/reclaim.js +43 -16
- package/dist/run/run-clock.d.ts +50 -0
- package/dist/run/run-clock.js +66 -0
- package/dist/run/run.js +5 -2
- package/dist/run/sandbox-ids.d.ts +7 -2
- package/dist/run/sandbox-ids.js +27 -11
- package/dist/run/secrets.d.ts +2 -1
- package/dist/run/secrets.js +18 -22
- package/dist/run/stored-runs.js +14 -7
- package/dist/run/transient-comms-secrets.d.ts +6 -5
- package/dist/run/transient-comms-secrets.js +23 -39
- package/dist/run-study.js +22 -13
- package/dist/study/adapter-scorer-loader.js +6 -4
- package/dist/study/discover.js +9 -6
- package/dist/study/init-paths.js +7 -5
- package/dist/study/persona-resolve.d.ts +2 -2
- package/dist/study/persona-resolve.js +9 -5
- package/dist/study/plan-types.d.ts +2 -2
- package/dist/study/plan.d.ts +5 -3
- package/dist/study/plan.js +18 -9
- package/dist/study/preflight-probes.js +11 -3
- package/dist/study/preflight.js +9 -5
- package/dist/study/summary.d.ts +12 -7
- package/dist/study/summary.js +24 -24
- package/dist/study/warnings.d.ts +19 -12
- package/dist/study/warnings.js +39 -17
- package/dist/substrates/e2b/connection.d.ts +12 -0
- package/dist/substrates/e2b/connection.js +35 -0
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -3
- package/dist/substrates/e2b/desktop-executor.js +42 -9
- package/dist/substrates/e2b/lifetime.d.ts +24 -4
- package/dist/substrates/e2b/lifetime.js +41 -4
- package/dist/substrates/e2b/sandbox.d.ts +4 -3
- package/dist/substrates/e2b/sandbox.js +13 -18
- package/dist/substrates/e2b/sdk.d.ts +20 -9
- package/dist/substrates/local/capacity.d.ts +18 -2
- package/dist/substrates/local/capacity.js +21 -2
- package/dist/substrates/local/runtime-host.js +21 -14
- package/dist/substrates/local/runtime.js +3 -11
- package/dist/tui/contract.d.ts +30 -5
- package/dist/tui/contract.js +4 -1
- package/dist/tui-app.js +149 -147
- package/dist/verify/artifacts.d.ts +9 -2
- package/dist/verify/artifacts.js +37 -13
- package/dist/verify/liveness.js +19 -5
- package/dist/verify/verify.js +54 -27
- package/docs/architecture/actor-contract.md +3 -3
- package/docs/architecture/guest-desktop.md +7 -2
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/architecture/state-driven-executor.md +5 -3
- package/docs/contracts/run-bundle.md +25 -13
- package/docs/contracts/schemas.md +38 -29
- package/docs/contracts/study-analysis.md +22 -5
- package/docs/product/automatic-analysis.md +7 -1
- package/package.json +8 -7
- package/skills/humanish/SKILL.md +1 -2
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { closingReportSchema, participantImpressionsSchema } from "../../closing-report.js";
|
|
2
2
|
import { isComputerUseAdmissionLimitError } from "../admission-limit.js";
|
|
3
3
|
import { IMPRESSIONS_ASK, IMPRESSIONS_HINT, recordImpressions } from "./impressions.js";
|
|
4
4
|
import { singleDispatch } from "./provider-call.js";
|
|
@@ -7,51 +7,27 @@ import { notice } from "./trace.js";
|
|
|
7
7
|
import { isCompleteTurnUsage } from "./usage.js";
|
|
8
8
|
const DEBRIEF_REQUEST_CAP_MS = 30_000;
|
|
9
9
|
const DEBRIEF_HINT = `The interactive session has ended. Return a closing account with summary, frictionReports and impressions. In summary, briefly describe only what you actually did and observed. In frictionReports, list only specific unexpected behavior, confusion, or recovery you personally encountered during this session. Preserve uncertainty. Use an empty list if you encountered none. ${IMPRESSIONS_ASK} Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions. This is a closing account, not another attempt at the task.`;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
report.frictionReports.every((item) => typeof item === "string" && item.trim().length > 0 && item.length <= 2_000) &&
|
|
25
|
-
(report.impressions === undefined || validImpressions(report.impressions)));
|
|
26
|
-
}
|
|
27
|
-
const REPORT_KEYS = ["summary", "frictionReports", "impressions"];
|
|
28
|
-
/** The reply to an impressions-only request: an object with the impressions and nothing else. */
|
|
29
|
-
export function validImpressionsReply(value) {
|
|
30
|
-
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
31
|
-
return false;
|
|
32
|
-
const reply = value;
|
|
33
|
-
return Object.keys(reply).length === 1 && validImpressions(reply.impressions);
|
|
34
|
-
}
|
|
35
|
-
function validImpressions(value) {
|
|
36
|
-
return (Array.isArray(value) &&
|
|
37
|
-
value.length <= 6 &&
|
|
38
|
-
value.every((item) => {
|
|
39
|
-
if (item === null || typeof item !== "object" || Array.isArray(item))
|
|
40
|
-
return false;
|
|
41
|
-
const { kind, text, ...rest } = item;
|
|
42
|
-
return (Object.keys(rest).length === 0 &&
|
|
43
|
-
PARTICIPANT_IMPRESSION_KINDS.some((known) => known === kind) &&
|
|
44
|
-
typeof text === "string" &&
|
|
45
|
-
text.trim().length > 0 &&
|
|
46
|
-
text.length <= 500);
|
|
47
|
-
}));
|
|
48
|
-
}
|
|
10
|
+
const CLOSING_REPORT = {
|
|
11
|
+
call: (provider) => provider.debrief,
|
|
12
|
+
contextHint: DEBRIEF_HINT,
|
|
13
|
+
noticeLabel: "debrief",
|
|
14
|
+
deadlineLabel: "closing report",
|
|
15
|
+
read: readClosingReport,
|
|
16
|
+
};
|
|
17
|
+
const IMPRESSIONS_ONLY = {
|
|
18
|
+
call: (provider) => provider.requestImpressions,
|
|
19
|
+
contextHint: IMPRESSIONS_HINT,
|
|
20
|
+
noticeLabel: "impressions request",
|
|
21
|
+
deadlineLabel: "impressions request",
|
|
22
|
+
read: readImpressions,
|
|
23
|
+
};
|
|
49
24
|
/**
|
|
50
25
|
* Ask the participant for its closing report after a structured stop, or for its impressions only
|
|
51
26
|
* after it ended the session itself, or record why the request was skipped. Either way the result
|
|
52
27
|
* is the trace's debrief, and the watcher gets a final flush.
|
|
53
28
|
*/
|
|
54
29
|
export async function requestDebrief(session, context, trigger) {
|
|
30
|
+
const ask = trigger.kind === "participant_end" ? IMPRESSIONS_ONLY : CLOSING_REPORT;
|
|
55
31
|
const record = (status, detail, usageReported) => {
|
|
56
32
|
const debrief = {
|
|
57
33
|
trigger: trigger.kind,
|
|
@@ -59,23 +35,23 @@ export async function requestDebrief(session, context, trigger) {
|
|
|
59
35
|
reason: session.redactNarration(detail),
|
|
60
36
|
...(usageReported === undefined ? {} : { usageReported }),
|
|
61
37
|
};
|
|
62
|
-
session.trace.record("notice", () => notice(status === "completed" ? "ok" : "warn", `participant ${
|
|
38
|
+
session.trace.record("notice", () => notice(status === "completed" ? "ok" : "warn", `participant ${ask.noticeLabel} ${status}`, session.redactNarration(detail)));
|
|
63
39
|
return debrief;
|
|
64
40
|
};
|
|
65
|
-
const plan = planDebrief(session, trigger);
|
|
41
|
+
const plan = planDebrief(session, ask, trigger);
|
|
66
42
|
const debrief = "skip" in plan
|
|
67
43
|
? record("skipped", plan.skip)
|
|
68
|
-
: await debriefExchange(session,
|
|
44
|
+
: await debriefExchange(session, context, trigger, ask, plan, record);
|
|
69
45
|
session.flush();
|
|
70
46
|
return debrief;
|
|
71
47
|
}
|
|
72
|
-
function planDebrief(session, trigger) {
|
|
48
|
+
function planDebrief(session, ask, trigger) {
|
|
73
49
|
const { provider, signal, usage } = session;
|
|
74
50
|
const { maxUsd, overRunBudget, estimateTurnCostUsd } = session.settings;
|
|
75
51
|
if (session.trace.counts.turns === 0)
|
|
76
52
|
return { skip: "the study stopped before any participant turn" };
|
|
77
|
-
const
|
|
78
|
-
if (
|
|
53
|
+
const send = ask.call(provider)?.bind(provider);
|
|
54
|
+
if (send === undefined)
|
|
79
55
|
return { skip: "this provider does not support read-only closing reports" };
|
|
80
56
|
if (usage.cleanupUnconfirmed)
|
|
81
57
|
return { skip: "participant request cleanup is unconfirmed" };
|
|
@@ -107,9 +83,9 @@ function planDebrief(session, trigger) {
|
|
|
107
83
|
skip: signal?.aborted ? "the study was cancelled" : "the session deadline was reached",
|
|
108
84
|
};
|
|
109
85
|
}
|
|
110
|
-
return {
|
|
86
|
+
return { send, boundMs };
|
|
111
87
|
}
|
|
112
|
-
async function debriefExchange(session,
|
|
88
|
+
async function debriefExchange(session, context, trigger, ask, { send, boundMs }, record) {
|
|
113
89
|
const { signal, provider } = session;
|
|
114
90
|
const scope = requestScope(signal);
|
|
115
91
|
const timer = setTimeout(() => scope.end(), boundMs);
|
|
@@ -128,12 +104,12 @@ async function debriefExchange(session, ask, context, trigger, boundMs, record)
|
|
|
128
104
|
...(context.acknowledgedSafetyChecks === undefined
|
|
129
105
|
? {}
|
|
130
106
|
: { acknowledgedSafetyChecks: context.acknowledgedSafetyChecks }),
|
|
131
|
-
contextHint:
|
|
107
|
+
contextHint: ask.contextHint,
|
|
132
108
|
};
|
|
133
109
|
const turn = provider.requestPolicy === "fail_closed"
|
|
134
|
-
? await singleDispatch(session, "debrief", (dispatchSignal) =>
|
|
135
|
-
: await raceSessionDeadline(
|
|
136
|
-
return acceptDebriefTurn(session, turn,
|
|
110
|
+
? await singleDispatch(session, "debrief", (dispatchSignal) => send(request, dispatchSignal), boundMs)
|
|
111
|
+
: await raceSessionDeadline(send(request, scope.signal), boundMs, signal);
|
|
112
|
+
return acceptDebriefTurn(session, turn, ask, record);
|
|
137
113
|
}
|
|
138
114
|
catch (error) {
|
|
139
115
|
// A failed optional report cannot rewrite the already observed structured completion.
|
|
@@ -143,7 +119,7 @@ async function debriefExchange(session, ask, context, trigger, boundMs, record)
|
|
|
143
119
|
const detail = signal?.aborted
|
|
144
120
|
? "cancelled"
|
|
145
121
|
: scope.signal.aborted || error instanceof CuaDeadlineError
|
|
146
|
-
? `${
|
|
122
|
+
? `${ask.deadlineLabel} deadline reached`
|
|
147
123
|
: error instanceof Error
|
|
148
124
|
? error.message
|
|
149
125
|
: String(error);
|
|
@@ -156,7 +132,7 @@ async function debriefExchange(session, ask, context, trigger, boundMs, record)
|
|
|
156
132
|
scope.end();
|
|
157
133
|
}
|
|
158
134
|
}
|
|
159
|
-
function acceptDebriefTurn(session, turn,
|
|
135
|
+
function acceptDebriefTurn(session, turn, ask, record) {
|
|
160
136
|
const { maxUsd, overRunBudget, estimateTurnCostUsd } = session.settings;
|
|
161
137
|
session.usage.record(turn, "debrief");
|
|
162
138
|
session.lastResponseId = turn.responseId ?? session.lastResponseId;
|
|
@@ -171,22 +147,23 @@ function acceptDebriefTurn(session, turn, trigger, record) {
|
|
|
171
147
|
if (turn.actions.length > 0 || turn.pendingSafetyChecks.length > 0) {
|
|
172
148
|
return record("failed", "the closing response requested actions or safety checks; none were executed and its report was not accepted", usageReported);
|
|
173
149
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
150
|
+
// First-party providers parse their replies already; these parses cover third-party ports.
|
|
151
|
+
const { status, detail, ...added } = ask.read(session, turn);
|
|
152
|
+
return { ...record(status, detail, usageReported), ...added };
|
|
153
|
+
}
|
|
154
|
+
/** A closing report: its summary and friction as one participant message, then its impressions. */
|
|
155
|
+
function readClosingReport(session, turn) {
|
|
156
|
+
const closing = closingReportSchema.safeParse(turn.closingReport);
|
|
157
|
+
if (!closing.success) {
|
|
158
|
+
return {
|
|
159
|
+
status: "failed",
|
|
160
|
+
detail: "the closing response did not contain a valid structured participant report",
|
|
161
|
+
};
|
|
185
162
|
}
|
|
186
163
|
const report = {
|
|
187
|
-
summary: session.redactNarration(
|
|
164
|
+
summary: session.redactNarration(closing.data.summary.trim()),
|
|
188
165
|
frictionReports: [
|
|
189
|
-
...new Set(
|
|
166
|
+
...new Set(closing.data.frictionReports.map((text) => session.redactNarration(text.trim()))),
|
|
190
167
|
],
|
|
191
168
|
};
|
|
192
169
|
const messageId = session.trace.record("message", () => ({
|
|
@@ -195,7 +172,28 @@ function acceptDebriefTurn(session, turn, trigger, record) {
|
|
|
195
172
|
text: [report.summary, ...report.frictionReports].join("\n\n"),
|
|
196
173
|
}));
|
|
197
174
|
session.trace.bump("messages");
|
|
198
|
-
session.impressions = recordImpressions(session,
|
|
199
|
-
|
|
200
|
-
|
|
175
|
+
session.impressions = recordImpressions(session, closing.data.impressions);
|
|
176
|
+
return {
|
|
177
|
+
status: "completed",
|
|
178
|
+
detail: "one read-only report; no additional desktop actions; original stop and task outcomes preserved",
|
|
179
|
+
report,
|
|
180
|
+
messageId,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
/** Impressions alone, after the participant ended the session itself. */
|
|
184
|
+
function readImpressions(session, turn) {
|
|
185
|
+
const impressions = participantImpressionsSchema.safeParse(turn.impressions);
|
|
186
|
+
if (!impressions.success) {
|
|
187
|
+
return {
|
|
188
|
+
status: "failed",
|
|
189
|
+
detail: turn.interruption === "output_limit"
|
|
190
|
+
? "the reply was cut off by the output limit"
|
|
191
|
+
: "the reply did not contain valid impressions",
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
session.impressions = recordImpressions(session, impressions.data);
|
|
195
|
+
return {
|
|
196
|
+
status: "completed",
|
|
197
|
+
detail: "one read-only impressions request; no additional desktop actions; the participant's own ending is unchanged",
|
|
198
|
+
};
|
|
201
199
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { ActorTrace, ParticipantImpression, ParticipantImpressions } from "../../contract.js";
|
|
2
2
|
import type { LoopSession } from "./session.js";
|
|
3
3
|
/** The closing-account request for impressions, shared by every provider that writes one. */
|
|
4
|
-
export declare const IMPRESSIONS_ASK
|
|
4
|
+
export declare const IMPRESSIONS_ASK: string;
|
|
5
5
|
/** The request after the participant ended the session itself without giving impressions. */
|
|
6
|
-
export declare const IMPRESSIONS_HINT
|
|
6
|
+
export declare const IMPRESSIONS_HINT: string;
|
|
7
7
|
/**
|
|
8
8
|
* Record a closing account's impressions as quotable participant messages. They answer the
|
|
9
9
|
* harness's question, so they leave counts.messages, which the hollow-completion rule reads, alone.
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
+
import { CLOSING_REPORT_LIMITS } from "../../closing-report-limits.js";
|
|
1
2
|
// A participant's impressions: what it says about the product at the end of a session, beyond
|
|
2
3
|
// friction. Each one is recorded as its own message so the analysis can quote and cite it as a
|
|
3
4
|
// participant statement, and the trace says why none were collected when a session has none.
|
|
5
|
+
// The limits are spelled out in the ask, so the participant is told the numbers its reply is
|
|
6
|
+
// checked against. Counts up to ten are written as words.
|
|
7
|
+
const COUNT_WORDS = "zero one two three four five six seven eight nine ten".split(" ");
|
|
8
|
+
const inWords = (count) => COUNT_WORDS[count] ?? String(count);
|
|
4
9
|
/** The closing-account request for impressions, shared by every provider that writes one. */
|
|
5
|
-
export const IMPRESSIONS_ASK =
|
|
10
|
+
export const IMPRESSIONS_ASK = `In impressions, list up to ${inWords(CLOSING_REPORT_LIMITS.impressions)} things you noticed about the product itself, each in the first person and about something you saw in this session. Give each one a kind: unclear when something looked confusing or hard to read; unfinished when something looked broken, unpolished or out of place; untrustworthy when something made you hesitate to trust the product; liked when something worked well or felt good; missing when you expected something and did not find it; unlike_my_work when the screen differs from how you do the same task in your own work or life, only if your persona does this task outside this product. Keep each under ${CLOSING_REPORT_LIMITS.impressionChars} characters. Use an empty list if you have none.`;
|
|
6
11
|
/** The request after the participant ended the session itself without giving impressions. */
|
|
7
12
|
export const IMPRESSIONS_HINT = `The interactive session has ended. Return only impressions. ${IMPRESSIONS_ASK} Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions.`;
|
|
8
13
|
/**
|
|
@@ -2,7 +2,6 @@ import type { CuaLoopOptions, CuaLoopResult, LoopRunOptions } from "./loop/types
|
|
|
2
2
|
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest, ShortenedWait, } from "./loop/types.js";
|
|
3
3
|
export { describeCuaAction } from "./loop/actions.js";
|
|
4
4
|
export { stableProgressKey } from "./loop/backstop.js";
|
|
5
|
-
export { validClosingReport, validImpressionsReply } from "./loop/debrief.js";
|
|
6
5
|
export { IMPRESSIONS_ASK } from "./loop/impressions.js";
|
|
7
6
|
/**
|
|
8
7
|
* Drive the loop to one explicit completion and return its CuaLoopResult: status, completion
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { closingReportSchema } from "../closing-report.js";
|
|
1
2
|
import { ComputerUseProviderError } from "./provider-error.js";
|
|
2
3
|
import { runActionBatch } from "./loop/actions.js";
|
|
3
4
|
import { advanceBackstop, startBackstop } from "./loop/backstop.js";
|
|
4
|
-
import { requestDebrief
|
|
5
|
+
import { requestDebrief } from "./loop/debrief.js";
|
|
5
6
|
import * as stops from "./loop/ending.js";
|
|
6
7
|
import { declaredOutcomeOf } from "./loop/ending.js";
|
|
7
8
|
import { notCollected, recordImpressions } from "./loop/impressions.js";
|
|
@@ -14,7 +15,6 @@ import { accountBillingConflicts } from "./loop/usage.js";
|
|
|
14
15
|
import { loopResult, notice } from "./loop/trace.js";
|
|
15
16
|
export { describeCuaAction } from "./loop/actions.js";
|
|
16
17
|
export { stableProgressKey } from "./loop/backstop.js";
|
|
17
|
-
export { validClosingReport, validImpressionsReply } from "./loop/debrief.js";
|
|
18
18
|
export { IMPRESSIONS_ASK } from "./loop/impressions.js";
|
|
19
19
|
/**
|
|
20
20
|
* Drive the loop to one explicit completion and return its CuaLoopResult: status, completion
|
|
@@ -120,7 +120,7 @@ async function runTurns(session, conversation) {
|
|
|
120
120
|
// The declared outcome is kept even if redacting the participant's summary fails.
|
|
121
121
|
session.declaredOutcome = declaredOutcomeOf(turn);
|
|
122
122
|
const ended = stops.participantEnded(turn, session.declaredOutcome, (text) => session.redactNarration(text));
|
|
123
|
-
const report =
|
|
123
|
+
const report = closingReportSchema.safeParse(turn.closingReport).data;
|
|
124
124
|
// A provider that can ask for impressions alone does so after the session, like a debrief.
|
|
125
125
|
if (report?.impressions !== undefined || session.provider.requestImpressions === undefined) {
|
|
126
126
|
session.impressions =
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CuaTurn, CuaTurnRequest } from "./loop.js";
|
|
2
2
|
import type { ReasoningEffort } from "../reasoning-effort.js";
|
|
3
|
-
import {
|
|
3
|
+
import type { ActorConversation } from "../contract.js";
|
|
4
4
|
import { type OpenAiReasoningSummary } from "./openai-wire.js";
|
|
5
5
|
/** How a conversation starts. The model settings stay fixed for the whole session. */
|
|
6
6
|
export interface OpenAiConversationSettings {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { closingReportSchema, impressionsReplySchema, strictOutputSchema, } from "../closing-report.js";
|
|
2
2
|
import { CarriedConversation, estimateTokens } from "./openai-context.js";
|
|
3
3
|
import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintItems, openingMessage, parseOpenAiResponse, } from "./openai-wire.js";
|
|
4
4
|
/**
|
|
@@ -6,44 +6,18 @@ import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintIte
|
|
|
6
6
|
* counts reasoning too, and leaves room for six impressions after the summary and friction reports.
|
|
7
7
|
*/
|
|
8
8
|
const CLOSING_OUTPUT_LIMIT = 3072;
|
|
9
|
-
const IMPRESSIONS_SCHEMA = {
|
|
10
|
-
type: "array",
|
|
11
|
-
items: {
|
|
12
|
-
type: "object",
|
|
13
|
-
additionalProperties: false,
|
|
14
|
-
required: ["kind", "text"],
|
|
15
|
-
properties: {
|
|
16
|
-
kind: { type: "string", enum: [...PARTICIPANT_IMPRESSION_KINDS] },
|
|
17
|
-
text: { type: "string" },
|
|
18
|
-
},
|
|
19
|
-
},
|
|
20
|
-
};
|
|
21
9
|
const CLOSING_FORMATS = {
|
|
22
10
|
report: {
|
|
23
11
|
type: "json_schema",
|
|
24
12
|
name: "participant_closing_report",
|
|
25
13
|
strict: true,
|
|
26
|
-
schema:
|
|
27
|
-
type: "object",
|
|
28
|
-
additionalProperties: false,
|
|
29
|
-
required: ["summary", "frictionReports", "impressions"],
|
|
30
|
-
properties: {
|
|
31
|
-
summary: { type: "string" },
|
|
32
|
-
frictionReports: { type: "array", items: { type: "string" } },
|
|
33
|
-
impressions: IMPRESSIONS_SCHEMA,
|
|
34
|
-
},
|
|
35
|
-
},
|
|
14
|
+
schema: strictOutputSchema(closingReportSchema),
|
|
36
15
|
},
|
|
37
16
|
impressions: {
|
|
38
17
|
type: "json_schema",
|
|
39
18
|
name: "participant_impressions",
|
|
40
19
|
strict: true,
|
|
41
|
-
schema:
|
|
42
|
-
type: "object",
|
|
43
|
-
additionalProperties: false,
|
|
44
|
-
required: ["impressions"],
|
|
45
|
-
properties: { impressions: IMPRESSIONS_SCHEMA },
|
|
46
|
-
},
|
|
20
|
+
schema: strictOutputSchema(impressionsReplySchema),
|
|
47
21
|
},
|
|
48
22
|
};
|
|
49
23
|
export class OpenAiConversation {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { closingReportSchema, impressionsReplySchema } from "../closing-report.js";
|
|
2
2
|
import { ComputerUseAdmissionLimitError, isComputerUseAdmissionLimitError, } from "./admission-limit.js";
|
|
3
3
|
import { ComputerUsePromptRefusedError } from "./provider-error.js";
|
|
4
4
|
import { OpenAiConversation, } from "./openai-conversation.js";
|
|
@@ -469,10 +469,16 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
469
469
|
try {
|
|
470
470
|
const reply = JSON.parse(turn.message ?? "");
|
|
471
471
|
if (asRecord(raw).status === "completed") {
|
|
472
|
-
if (closing === "report"
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
472
|
+
if (closing === "report") {
|
|
473
|
+
const report = closingReportSchema.safeParse(reply);
|
|
474
|
+
if (report.success)
|
|
475
|
+
return { ...turn, closingReport: report.data };
|
|
476
|
+
}
|
|
477
|
+
else {
|
|
478
|
+
const impressions = impressionsReplySchema.safeParse(reply);
|
|
479
|
+
if (impressions.success)
|
|
480
|
+
return { ...turn, impressions: impressions.data.impressions };
|
|
481
|
+
}
|
|
476
482
|
}
|
|
477
483
|
}
|
|
478
484
|
catch {
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { AffordanceUse } from "./affordance.js";
|
|
2
2
|
import type { ActorEstimatedCost } from "../run/pricing.js";
|
|
3
3
|
import type { TaskFunnel } from "../study/tasks.js";
|
|
4
|
+
import type { ParticipantClosingReport, ParticipantImpression } from "./closing-report.js";
|
|
4
5
|
import { type CuaProviderFailurePhase } from "./computer-use/provider-error.js";
|
|
5
6
|
export declare const ACTOR_TRACE_SCHEMA = "humanish.actor-trace.v1";
|
|
7
|
+
export type { ParticipantClosingReport, ParticipantImpression } from "./closing-report.js";
|
|
6
8
|
/**
|
|
7
9
|
* How a session ended, from the point of view of the study rather than the harness.
|
|
8
10
|
*
|
|
@@ -182,25 +184,6 @@ export interface ActorProviderRequest extends ProviderRequestReceipt {
|
|
|
182
184
|
export declare function validActorExecutionProfile(value: unknown): value is ActorExecutionProfile;
|
|
183
185
|
/** Closed per-attempt evidence; dollar amounts are never part of account usage. */
|
|
184
186
|
export declare function validActorProviderRequests(value: unknown): value is ActorProviderRequest[];
|
|
185
|
-
/** The participant's account, not an independently confirmed product diagnosis. */
|
|
186
|
-
export interface ParticipantClosingReport {
|
|
187
|
-
summary: string;
|
|
188
|
-
frictionReports: string[];
|
|
189
|
-
/** Absent when the provider did not ask for impressions; empty when the participant had none. */
|
|
190
|
-
impressions?: ParticipantImpression[];
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* What a participant can say about the product at the end of a session, beyond friction.
|
|
194
|
-
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
195
|
-
* life.
|
|
196
|
-
*/
|
|
197
|
-
export declare const PARTICIPANT_IMPRESSION_KINDS: readonly ["unclear", "unfinished", "untrustworthy", "liked", "missing", "unlike_my_work"];
|
|
198
|
-
type ParticipantImpressionKind = (typeof PARTICIPANT_IMPRESSION_KINDS)[number];
|
|
199
|
-
/** One first-person impression about something the participant saw in the session. */
|
|
200
|
-
export interface ParticipantImpression {
|
|
201
|
-
kind: ParticipantImpressionKind;
|
|
202
|
-
text: string;
|
|
203
|
-
}
|
|
204
187
|
/** An impression as the trace keeps it, with the message that quotes it. */
|
|
205
188
|
interface RecordedImpression extends ParticipantImpression {
|
|
206
189
|
/** The trace message carrying the impression, which the analysis quotes and cites. */
|
|
@@ -399,4 +382,3 @@ export interface ActorSessionResult {
|
|
|
399
382
|
export declare const CODEX_APP_SERVER_CAPABILITIES: ActorCapabilities;
|
|
400
383
|
export declare const SCRIPTED_BROWSER_CAPABILITIES: ActorCapabilities;
|
|
401
384
|
export declare const TERMINAL_AGENT_CAPABILITIES: ActorCapabilities;
|
|
402
|
-
export {};
|
package/dist/actors/contract.js
CHANGED
|
@@ -168,19 +168,6 @@ export function validActorProviderRequests(value) {
|
|
|
168
168
|
usage.input));
|
|
169
169
|
});
|
|
170
170
|
}
|
|
171
|
-
/**
|
|
172
|
-
* What a participant can say about the product at the end of a session, beyond friction.
|
|
173
|
-
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
174
|
-
* life.
|
|
175
|
-
*/
|
|
176
|
-
export const PARTICIPANT_IMPRESSION_KINDS = [
|
|
177
|
-
"unclear",
|
|
178
|
-
"unfinished",
|
|
179
|
-
"untrustworthy",
|
|
180
|
-
"liked",
|
|
181
|
-
"missing",
|
|
182
|
-
"unlike_my_work",
|
|
183
|
-
];
|
|
184
171
|
export const CODEX_APP_SERVER_CAPABILITIES = {
|
|
185
172
|
headless: true,
|
|
186
173
|
structuredTrace: true,
|
|
@@ -6,7 +6,7 @@ import os from "node:os";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
8
8
|
import { digestText, redactToSecretLabel } from "../../evidence/redaction.js";
|
|
9
|
-
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputFile, prepareSelectedOutputDirectory, readContainedRegularFile, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
9
|
+
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputFile, prepareSelectedOutputDirectory, readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
10
10
|
import { LOOPBACK_EVIDENCE_URL_POLICY, } from "./types.js";
|
|
11
11
|
export async function executeBrowserPersonaStep(args) {
|
|
12
12
|
const started = Date.now();
|
|
@@ -402,11 +402,12 @@ async function captureScriptedPageScreenshot(page) {
|
|
|
402
402
|
if (Buffer.isBuffer(returned) || returned instanceof Uint8Array) {
|
|
403
403
|
return stripPngMetadataChunks(browserScreenshotBytes(returned));
|
|
404
404
|
}
|
|
405
|
-
const
|
|
406
|
-
if (
|
|
405
|
+
const staged = await readContainedRegularFile(stagingRoot, "capture.png", RUN_ARTIFACT_MAX_BYTES);
|
|
406
|
+
if (staged.status === "missing")
|
|
407
407
|
throw new Error("Browser screenshot did not return bytes or write a single-link staging file.");
|
|
408
|
-
|
|
409
|
-
|
|
408
|
+
if (staged.status === "refused")
|
|
409
|
+
throw new Error(`Browser screenshot did not return bytes, and ${refusalText("its staging file capture.png", staged)}.`);
|
|
410
|
+
return stripPngMetadataChunks(staged.bytes);
|
|
410
411
|
}
|
|
411
412
|
finally {
|
|
412
413
|
await assertPreparedSelectedOutputDirectory(stagingRoot)
|
|
@@ -24,10 +24,20 @@ export interface RefusedAnalysisCost {
|
|
|
24
24
|
worstCaseCostUsd: number;
|
|
25
25
|
maxCostUsd: number;
|
|
26
26
|
}
|
|
27
|
-
/**
|
|
28
|
-
export
|
|
29
|
-
/**
|
|
30
|
-
|
|
27
|
+
/** A cost refusal in plain words, and the command that runs the analysis under a cap that admits it. */
|
|
28
|
+
export interface CostRefusal {
|
|
29
|
+
/** The costs, the cap and that no request was sent, ending in the lead-in to `command`. */
|
|
30
|
+
text: string;
|
|
31
|
+
/** `analyze` with `--max-cost` at the worst case rounded up to a whole dollar, at least 1. */
|
|
32
|
+
command: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Words a cost refusal. `runFlags` are the `analyze` flags that select the run, such as
|
|
36
|
+
* `--run <id>`. `cli` prefixes a humanish command: the CLI passes `cli` from
|
|
37
|
+
* src/cli/invocation.ts, and the Observer, which cannot read the install, a bare `humanish`.
|
|
38
|
+
* This module keeps only type imports, so the Observer can bundle it.
|
|
39
|
+
*/
|
|
40
|
+
export declare function costRefusal(cost: RefusedAnalysisCost, runFlags: string, cli: (rest: string) => string): CostRefusal;
|
|
31
41
|
/** The admission's refused cost, when it refused the analysis for its cost. */
|
|
32
42
|
export declare function refusedCost(admission: {
|
|
33
43
|
error: string | null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// The admission cost model for one study analysis request: the expected cost, the worst case,
|
|
2
|
-
//
|
|
1
|
+
// The admission cost model for one study analysis request: the expected cost, the worst case, the
|
|
2
|
+
// figure admission compares with the cap, and the words and command a cost refusal gives.
|
|
3
3
|
/**
|
|
4
4
|
* UTF-8 bytes per input token for the instructions, evidence packet and schema. On 148 billed
|
|
5
5
|
* gpt-6-astra analyses the evidence packet ran 3.1 to 3.3 bytes per token and the instructions
|
|
@@ -40,13 +40,18 @@ export function estimateAnalysisCost(rate, size) {
|
|
|
40
40
|
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Words a cost refusal. `runFlags` are the `analyze` flags that select the run, such as
|
|
45
|
+
* `--run <id>`. `cli` prefixes a humanish command: the CLI passes `cli` from
|
|
46
|
+
* src/cli/invocation.ts, and the Observer, which cannot read the install, a bare `humanish`.
|
|
47
|
+
* This module keeps only type imports, so the Observer can bundle it.
|
|
48
|
+
*/
|
|
49
|
+
export function costRefusal(cost, runFlags, cli) {
|
|
50
|
+
const margin = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
51
|
+
return {
|
|
52
|
+
text: `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${margin}% margin the expected cost is over the $${cost.maxCostUsd} cap, so no request was sent. To run it, raise the cap:`,
|
|
53
|
+
command: cli(`analyze ${runFlags} --max-cost ${Math.max(1, Math.ceil(cost.worstCaseCostUsd))}`),
|
|
54
|
+
};
|
|
50
55
|
}
|
|
51
56
|
/** The admission's refused cost, when it refused the analysis for its cost. */
|
|
52
57
|
export function refusedCost(admission) {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How much an analysis result may hold, for the fields the Observer checks as well. Characters are
|
|
3
|
+
* counted as `String.length` counts them.
|
|
4
|
+
*/
|
|
5
|
+
export declare const ANALYSIS_LIMITS: Readonly<{
|
|
6
|
+
/** A label: a finding's title and headline, a design finding's headline and screen, or a name. */
|
|
7
|
+
labelChars: 240;
|
|
8
|
+
experienceChars: 1200;
|
|
9
|
+
noticeChars: 1500;
|
|
10
|
+
whyItMattersChars: 1000;
|
|
11
|
+
suggestionChars: 1000;
|
|
12
|
+
/** Evidence one observation or design finding may cite. */
|
|
13
|
+
evidenceRefs: 100;
|
|
14
|
+
findings: 100;
|
|
15
|
+
designFindings: 40;
|
|
16
|
+
concernReviews: 60;
|
|
17
|
+
}>;
|
|
18
|
+
/** Whether text holds a control character other than tab, line feed and carriage return. */
|
|
19
|
+
export declare function hasControlCharacter(text: string): boolean;
|
|
20
|
+
/** The parts of an analysis result the revision boundaries read. */
|
|
21
|
+
interface RevisionFields {
|
|
22
|
+
readonly concernReviews?: unknown;
|
|
23
|
+
readonly designFindings?: unknown;
|
|
24
|
+
readonly findings: ReadonlyArray<{
|
|
25
|
+
readonly headline?: unknown;
|
|
26
|
+
readonly experience?: unknown;
|
|
27
|
+
}>;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Whether a result carries every field its prompt revision requires: concern reviews from
|
|
31
|
+
* study-evidence-5, and headlines, experiences and design findings from study-evidence-7. A prompt
|
|
32
|
+
* version without a study-evidence revision number requires none of them.
|
|
33
|
+
*/
|
|
34
|
+
export declare function hasRevisionFields(promptVersion: string, result: RevisionFields): boolean;
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// The analysis result's limits and prompt revision boundaries, kept apart from the zod schemas in
|
|
2
|
+
// validation.ts so the Observer can read them: it bundles this module, and zod stays out of its
|
|
3
|
+
// page.
|
|
4
|
+
/**
|
|
5
|
+
* How much an analysis result may hold, for the fields the Observer checks as well. Characters are
|
|
6
|
+
* counted as `String.length` counts them.
|
|
7
|
+
*/
|
|
8
|
+
export const ANALYSIS_LIMITS = Object.freeze({
|
|
9
|
+
/** A label: a finding's title and headline, a design finding's headline and screen, or a name. */
|
|
10
|
+
labelChars: 240,
|
|
11
|
+
experienceChars: 1200,
|
|
12
|
+
noticeChars: 1500,
|
|
13
|
+
whyItMattersChars: 1000,
|
|
14
|
+
suggestionChars: 1000,
|
|
15
|
+
/** Evidence one observation or design finding may cite. */
|
|
16
|
+
evidenceRefs: 100,
|
|
17
|
+
findings: 100,
|
|
18
|
+
designFindings: 40,
|
|
19
|
+
concernReviews: 60,
|
|
20
|
+
});
|
|
21
|
+
/** Whether text holds a control character other than tab, line feed and carriage return. */
|
|
22
|
+
export function hasControlCharacter(text) {
|
|
23
|
+
return /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(text);
|
|
24
|
+
}
|
|
25
|
+
// The prompt revision from which a result must carry each field. A boundary stays where it is when
|
|
26
|
+
// the prompt version advances, so a later prompt cannot bring back a legacy omission.
|
|
27
|
+
const CONCERN_REVIEWS_FROM = 5;
|
|
28
|
+
const HEADLINES_AND_DESIGN_FINDINGS_FROM = 7;
|
|
29
|
+
/**
|
|
30
|
+
* Whether a result carries every field its prompt revision requires: concern reviews from
|
|
31
|
+
* study-evidence-5, and headlines, experiences and design findings from study-evidence-7. A prompt
|
|
32
|
+
* version without a study-evidence revision number requires none of them.
|
|
33
|
+
*/
|
|
34
|
+
export function hasRevisionFields(promptVersion, result) {
|
|
35
|
+
const revision = Number(/^study-evidence-(\d+)$/.exec(promptVersion)?.[1] ?? 0);
|
|
36
|
+
if (revision >= CONCERN_REVIEWS_FROM && result.concernReviews === undefined)
|
|
37
|
+
return false;
|
|
38
|
+
return (revision < HEADLINES_AND_DESIGN_FINDINGS_FROM ||
|
|
39
|
+
(result.designFindings !== undefined &&
|
|
40
|
+
result.findings.every((finding) => finding.headline !== undefined && finding.experience !== undefined)));
|
|
41
|
+
}
|
|
@@ -16,6 +16,11 @@ export interface AnalysisAdmission {
|
|
|
16
16
|
estimatedCostUsd: number | null;
|
|
17
17
|
/** The cost if the analyst spends its whole output allowance. */
|
|
18
18
|
worstCaseCostUsd: number | null;
|
|
19
|
+
/**
|
|
20
|
+
* The expected cost plus the margin, or the worst case when that is lower: the figure admission
|
|
21
|
+
* compares with maxCostUsd, so the smallest cap that admits this analysis.
|
|
22
|
+
*/
|
|
23
|
+
admittedCostUsd: number | null;
|
|
19
24
|
maxCostUsd: number | null;
|
|
20
25
|
ratesAsOf: string | null;
|
|
21
26
|
}
|