humanish 0.112.0 → 0.113.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +11 -0
- package/TELEMETRY.md +21 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +21 -1
- package/dist/actors/codex/restricted-participant-policy.js +20 -2
- package/dist/actors/codex/restricted-participant.d.ts +1 -1
- package/dist/actors/codex/restricted-participant.js +2 -1
- package/dist/actors/computer-use/loop/debrief.d.ts +12 -4
- package/dist/actors/computer-use/loop/debrief.js +57 -20
- package/dist/actors/computer-use/loop/ending.d.ts +5 -2
- package/dist/actors/computer-use/loop/impressions.d.ts +14 -0
- package/dist/actors/computer-use/loop/impressions.js +40 -0
- package/dist/actors/computer-use/loop/session.d.ts +3 -1
- package/dist/actors/computer-use/loop/session.js +2 -0
- package/dist/actors/computer-use/loop/trace.js +2 -0
- package/dist/actors/computer-use/loop/types.d.ts +8 -1
- package/dist/actors/computer-use/loop.d.ts +2 -1
- package/dist/actors/computer-use/loop.js +15 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +9 -4
- package/dist/actors/computer-use/openai-conversation.js +53 -24
- package/dist/actors/computer-use/openai-provider.js +18 -12
- package/dist/actors/contract.d.ts +43 -3
- package/dist/actors/contract.js +13 -0
- package/dist/analysis/evidence-sources.d.ts +3 -1
- package/dist/analysis/evidence-sources.js +10 -1
- package/dist/analysis/evidence.js +2 -2
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +3 -1
- package/dist/cli/commands/runs.js +33 -10
- package/dist/cli/commands/setup.js +95 -6
- package/dist/cli/commands/study-format.js +7 -4
- package/dist/cli/commands/study.js +1 -1
- package/dist/cli/commands/tui.js +1 -1
- package/dist/cli/doctor.d.ts +2 -0
- package/dist/cli/doctor.js +18 -0
- package/dist/cli/io.d.ts +2 -0
- package/dist/cli/program.js +35 -1
- package/dist/cli/secret-prompt.d.ts +5 -1
- package/dist/cli/secret-prompt.js +6 -2
- package/dist/cli/update-check-worker.d.ts +1 -0
- package/dist/cli/update-check-worker.js +5 -0
- package/dist/cli/update-check.d.ts +46 -0
- package/dist/cli/update-check.js +203 -0
- package/dist/feedback/draft.js +18 -0
- package/dist/keys/key-resolution.js +2 -0
- package/dist/keys/key-status.d.ts +37 -0
- package/dist/keys/key-status.js +68 -0
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/local-vm.js +9 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +6 -0
- package/dist/routes/computer-use/participant-execution.js +9 -1
- package/dist/routes/computer-use/rerun-selection.js +23 -21
- package/dist/routes/computer-use/result.js +3 -2
- package/dist/routes/computer-use/self-report.js +7 -1
- package/dist/routes/computer-use/setup.js +14 -0
- package/dist/routes/computer-use/types.d.ts +4 -1
- package/dist/routes/shared-world/bundle-records.js +8 -7
- package/dist/routes/shared-world/bundle.js +1 -1
- package/dist/run/detail.js +4 -4
- package/dist/run/participant-caption.d.ts +2 -1
- package/dist/run/participant-caption.js +11 -1
- package/dist/run/participant-records.d.ts +3 -1
- package/dist/run/participant-records.js +6 -0
- package/dist/run-study.js +17 -1
- package/dist/study/discover.js +6 -1
- package/dist/study/doctor.d.ts +2 -0
- package/dist/study/doctor.js +45 -7
- package/dist/study/init-templates.js +10 -5
- package/dist/study/plan.js +3 -2
- package/dist/study/preflight-result.d.ts +1 -0
- package/dist/study/preflight-result.js +1 -1
- package/dist/study/preflight.d.ts +2 -0
- package/dist/study/preflight.js +54 -14
- package/dist/study/warnings.d.ts +5 -0
- package/dist/study/warnings.js +79 -2
- package/dist/substrates/local/capacity.d.ts +75 -0
- package/dist/substrates/local/capacity.js +143 -0
- package/dist/substrates/local/firecracker-desktop.d.ts +8 -0
- package/dist/substrates/local/firecracker-desktop.js +34 -3
- package/dist/substrates/local/runtime-host.d.ts +16 -3
- package/dist/substrates/local/runtime-host.js +49 -4
- package/dist/substrates/local/runtime.d.ts +10 -0
- package/dist/substrates/local/runtime.js +44 -5
- package/dist/tui-app.js +58 -58
- package/docs/architecture/actor-contract.md +20 -6
- package/docs/contracts/feedback.md +8 -1
- package/docs/contracts/schemas.md +57 -34
- package/docs/contracts/study-analysis.md +15 -0
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +21 -10
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CuaTurn, CuaTurnRequest } from "./loop.js";
|
|
2
2
|
import type { ReasoningEffort } from "../reasoning-effort.js";
|
|
3
|
-
import type
|
|
3
|
+
import { type ActorConversation } from "../contract.js";
|
|
4
4
|
import { type OpenAiReasoningSummary } from "./openai-wire.js";
|
|
5
5
|
/** How a conversation starts. The model settings stay fixed for the whole session. */
|
|
6
6
|
export interface OpenAiConversationSettings {
|
|
@@ -30,6 +30,11 @@ export interface ConversationRequest {
|
|
|
30
30
|
}
|
|
31
31
|
/** Which answer told the provider that the organization keeps no server-side conversation. */
|
|
32
32
|
export type ZdrRejection = NonNullable<ActorConversation["rejection"]>;
|
|
33
|
+
/**
|
|
34
|
+
* What a closing request asks for: the closing report after a harness-owned stop, or only
|
|
35
|
+
* impressions after the participant ended the session itself.
|
|
36
|
+
*/
|
|
37
|
+
export type ClosingRequest = "report" | "impressions";
|
|
33
38
|
export declare class OpenAiConversation {
|
|
34
39
|
private readonly settings;
|
|
35
40
|
private lastResponseId;
|
|
@@ -46,9 +51,9 @@ export declare class OpenAiConversation {
|
|
|
46
51
|
constructor(settings: OpenAiConversationSettings);
|
|
47
52
|
/**
|
|
48
53
|
* The next request, carrying `req`'s screen: the opening request, or the outputs the latest
|
|
49
|
-
* reply's computer calls are owed. A closing
|
|
54
|
+
* reply's computer calls are owed. A closing request asks for strict JSON and no tools.
|
|
50
55
|
*/
|
|
51
|
-
request(req: CuaTurnRequest, closing?:
|
|
56
|
+
request(req: CuaTurnRequest, closing?: ClosingRequest): ConversationRequest;
|
|
52
57
|
/**
|
|
53
58
|
* The organization rejected server-side state: carry the conversation on the client from the
|
|
54
59
|
* request being sent, and record which answer made the switch. False when the conversation is
|
|
@@ -80,7 +85,7 @@ export declare class OpenAiConversation {
|
|
|
80
85
|
* ahead of `sent`.
|
|
81
86
|
*/
|
|
82
87
|
private turnBody;
|
|
83
|
-
/**
|
|
88
|
+
/** A read-only closing request: no tools, its own output limit, a strict schema. */
|
|
84
89
|
private closingBody;
|
|
85
90
|
/**
|
|
86
91
|
* Move the conversation on to a reply to a request that added `sent`. A participant reply cut
|
|
@@ -1,7 +1,51 @@
|
|
|
1
|
+
import { PARTICIPANT_IMPRESSION_KINDS } from "../contract.js";
|
|
1
2
|
import { CarriedConversation, estimateTokens } from "./openai-context.js";
|
|
2
3
|
import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintItems, openingMessage, parseOpenAiResponse, } from "./openai-wire.js";
|
|
3
|
-
/**
|
|
4
|
-
|
|
4
|
+
/**
|
|
5
|
+
* The closing request's output limit: its own cap, or the declared limit when that is lower. It
|
|
6
|
+
* counts reasoning too, and leaves room for six impressions after the summary and friction reports.
|
|
7
|
+
*/
|
|
8
|
+
const CLOSING_OUTPUT_LIMIT = 3072;
|
|
9
|
+
const IMPRESSIONS_SCHEMA = {
|
|
10
|
+
type: "array",
|
|
11
|
+
items: {
|
|
12
|
+
type: "object",
|
|
13
|
+
additionalProperties: false,
|
|
14
|
+
required: ["kind", "text"],
|
|
15
|
+
properties: {
|
|
16
|
+
kind: { type: "string", enum: [...PARTICIPANT_IMPRESSION_KINDS] },
|
|
17
|
+
text: { type: "string" },
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
const CLOSING_FORMATS = {
|
|
22
|
+
report: {
|
|
23
|
+
type: "json_schema",
|
|
24
|
+
name: "participant_closing_report",
|
|
25
|
+
strict: true,
|
|
26
|
+
schema: {
|
|
27
|
+
type: "object",
|
|
28
|
+
additionalProperties: false,
|
|
29
|
+
required: ["summary", "frictionReports", "impressions"],
|
|
30
|
+
properties: {
|
|
31
|
+
summary: { type: "string" },
|
|
32
|
+
frictionReports: { type: "array", items: { type: "string" } },
|
|
33
|
+
impressions: IMPRESSIONS_SCHEMA,
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
impressions: {
|
|
38
|
+
type: "json_schema",
|
|
39
|
+
name: "participant_impressions",
|
|
40
|
+
strict: true,
|
|
41
|
+
schema: {
|
|
42
|
+
type: "object",
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
required: ["impressions"],
|
|
45
|
+
properties: { impressions: IMPRESSIONS_SCHEMA },
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
};
|
|
5
49
|
export class OpenAiConversation {
|
|
6
50
|
settings;
|
|
7
51
|
lastResponseId;
|
|
@@ -23,13 +67,13 @@ export class OpenAiConversation {
|
|
|
23
67
|
}
|
|
24
68
|
/**
|
|
25
69
|
* The next request, carrying `req`'s screen: the opening request, or the outputs the latest
|
|
26
|
-
* reply's computer calls are owed. A closing
|
|
70
|
+
* reply's computer calls are owed. A closing request asks for strict JSON and no tools.
|
|
27
71
|
*/
|
|
28
|
-
request(req, closing
|
|
72
|
+
request(req, closing) {
|
|
29
73
|
const sent = this.newItems(req);
|
|
30
74
|
return {
|
|
31
|
-
body: () =>
|
|
32
|
-
accept: (reply) => this.accept(reply, closing, sent),
|
|
75
|
+
body: () => closing === undefined ? this.turnBody(req, sent) : this.closingBody(req, sent, closing),
|
|
76
|
+
accept: (reply) => this.accept(reply, closing !== undefined, sent),
|
|
33
77
|
};
|
|
34
78
|
}
|
|
35
79
|
/**
|
|
@@ -110,28 +154,13 @@ export class OpenAiConversation {
|
|
|
110
154
|
...(this.mode === "explicit_context" ? { explicitContextItems: this.carried.carried() } : {}),
|
|
111
155
|
});
|
|
112
156
|
}
|
|
113
|
-
/**
|
|
114
|
-
closingBody(req, sent) {
|
|
157
|
+
/** A read-only closing request: no tools, its own output limit, a strict schema. */
|
|
158
|
+
closingBody(req, sent, closing) {
|
|
115
159
|
return {
|
|
116
160
|
...this.turnBody(req, sent),
|
|
117
161
|
tool_choice: "none",
|
|
118
162
|
max_output_tokens: Math.min(this.settings.maxOutputTokens ?? CLOSING_OUTPUT_LIMIT, CLOSING_OUTPUT_LIMIT),
|
|
119
|
-
text: {
|
|
120
|
-
format: {
|
|
121
|
-
type: "json_schema",
|
|
122
|
-
name: "participant_closing_report",
|
|
123
|
-
strict: true,
|
|
124
|
-
schema: {
|
|
125
|
-
type: "object",
|
|
126
|
-
additionalProperties: false,
|
|
127
|
-
required: ["summary", "frictionReports"],
|
|
128
|
-
properties: {
|
|
129
|
-
summary: { type: "string" },
|
|
130
|
-
frictionReports: { type: "array", items: { type: "string" } },
|
|
131
|
-
},
|
|
132
|
-
},
|
|
133
|
-
},
|
|
134
|
-
},
|
|
163
|
+
text: { format: CLOSING_FORMATS[closing] },
|
|
135
164
|
};
|
|
136
165
|
}
|
|
137
166
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { validClosingReport } from "./loop.js";
|
|
1
|
+
import { validClosingReport, validImpressionsReply } from "./loop.js";
|
|
2
2
|
import { ComputerUseAdmissionLimitError, isComputerUseAdmissionLimitError, } from "./admission-limit.js";
|
|
3
3
|
import { ComputerUsePromptRefusedError } from "./provider-error.js";
|
|
4
4
|
import { OpenAiConversation, } from "./openai-conversation.js";
|
|
@@ -455,21 +455,24 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
455
455
|
}
|
|
456
456
|
}
|
|
457
457
|
};
|
|
458
|
-
const requestTurn = async (req, signal, closing
|
|
458
|
+
const requestTurn = async (req, signal, closing, spend) => {
|
|
459
459
|
await capture.prepareNext();
|
|
460
460
|
const request = conversation.request(req, closing);
|
|
461
|
-
// A closing
|
|
462
|
-
const raw = closing
|
|
463
|
-
? await
|
|
464
|
-
: await
|
|
461
|
+
// A closing request makes exactly one request: no HTTP or policy-latch retries.
|
|
462
|
+
const raw = closing === undefined
|
|
463
|
+
? await postTurn(request, signal, spend)
|
|
464
|
+
: await post(request.body(), signal, 0, false);
|
|
465
465
|
const turn = request.accept(raw);
|
|
466
|
-
if (closing) {
|
|
466
|
+
if (closing !== undefined) {
|
|
467
467
|
// Refusals, incomplete output, malformed JSON, and invalid shapes remain no-report results.
|
|
468
468
|
// Never promote raw JSON or a fallback paragraph into a structured finding.
|
|
469
469
|
try {
|
|
470
|
-
const
|
|
471
|
-
if (asRecord(raw).status === "completed"
|
|
472
|
-
|
|
470
|
+
const reply = JSON.parse(turn.message ?? "");
|
|
471
|
+
if (asRecord(raw).status === "completed") {
|
|
472
|
+
if (closing === "report" && validClosingReport(reply))
|
|
473
|
+
return { ...turn, closingReport: reply };
|
|
474
|
+
if (closing === "impressions" && validImpressionsReply(reply))
|
|
475
|
+
return { ...turn, impressions: reply.impressions };
|
|
473
476
|
}
|
|
474
477
|
}
|
|
475
478
|
catch {
|
|
@@ -496,15 +499,18 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
496
499
|
get interactionUsageIncomplete() {
|
|
497
500
|
return interactionUsageIncomplete;
|
|
498
501
|
},
|
|
499
|
-
nextTurn: (req, signal, spend) => requestTurn(req, signal,
|
|
502
|
+
nextTurn: (req, signal, spend) => requestTurn(req, signal, undefined, spend),
|
|
500
503
|
get conversation() {
|
|
501
504
|
return conversation.record();
|
|
502
505
|
},
|
|
503
506
|
// A retrospective report reads the session the server holds; see serverHoldsSession.
|
|
504
507
|
get debrief() {
|
|
505
508
|
return conversation.serverHoldsSession
|
|
506
|
-
? (req, signal) => requestTurn(req, signal,
|
|
509
|
+
? (req, signal) => requestTurn(req, signal, "report")
|
|
507
510
|
: undefined;
|
|
508
511
|
},
|
|
512
|
+
// Offered in both modes. An explicit_context conversation carries its newest screens whole and
|
|
513
|
+
// earlier turns as text, which serves impressions; a closing report needs the whole session.
|
|
514
|
+
requestImpressions: (req, signal) => requestTurn(req, signal, "impressions"),
|
|
509
515
|
};
|
|
510
516
|
}
|
|
@@ -186,7 +186,37 @@ export declare function validActorProviderRequests(value: unknown): value is Act
|
|
|
186
186
|
export interface ParticipantClosingReport {
|
|
187
187
|
summary: string;
|
|
188
188
|
frictionReports: string[];
|
|
189
|
+
/** Absent when the provider did not ask for impressions; empty when the participant had none. */
|
|
190
|
+
impressions?: ParticipantImpression[];
|
|
189
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* What a participant can say about the product at the end of a session, beyond friction.
|
|
194
|
+
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
195
|
+
* life.
|
|
196
|
+
*/
|
|
197
|
+
export declare const PARTICIPANT_IMPRESSION_KINDS: readonly ["unclear", "unfinished", "untrustworthy", "liked", "missing", "unlike_my_work"];
|
|
198
|
+
type ParticipantImpressionKind = (typeof PARTICIPANT_IMPRESSION_KINDS)[number];
|
|
199
|
+
/** One first-person impression about something the participant saw in the session. */
|
|
200
|
+
export interface ParticipantImpression {
|
|
201
|
+
kind: ParticipantImpressionKind;
|
|
202
|
+
text: string;
|
|
203
|
+
}
|
|
204
|
+
/** An impression as the trace keeps it, with the message that quotes it. */
|
|
205
|
+
interface RecordedImpression extends ParticipantImpression {
|
|
206
|
+
/** The trace message carrying the impression, which the analysis quotes and cites. */
|
|
207
|
+
messageId: string;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* The impressions a session collected, or why it collected none. An empty `items` list means the
|
|
211
|
+
* participant had none to give.
|
|
212
|
+
*/
|
|
213
|
+
export type ParticipantImpressions = {
|
|
214
|
+
status: "collected";
|
|
215
|
+
items: RecordedImpression[];
|
|
216
|
+
} | {
|
|
217
|
+
status: "not_collected";
|
|
218
|
+
reason: string;
|
|
219
|
+
};
|
|
190
220
|
/** Runtime declarations and executable-version observations; not provider request attestation. */
|
|
191
221
|
export interface ActorRuntimeProvenance {
|
|
192
222
|
schema: "humanish.actor-runtime.v1";
|
|
@@ -315,17 +345,27 @@ export interface ActorTrace {
|
|
|
315
345
|
* each fixed a false refusal and each left the next shape unhandled.
|
|
316
346
|
*/
|
|
317
347
|
declaredOutcome?: ParticipantDeclaredOutcome;
|
|
318
|
-
/**
|
|
348
|
+
/**
|
|
349
|
+
* The read-only closing request: a closing report after a harness-owned stop (`stop_when`,
|
|
350
|
+
* `dwell`), or impressions only after the participant ended the session itself
|
|
351
|
+
* (`participant_end`, never with `report`). Does not change task outcomes or permit actions.
|
|
352
|
+
*/
|
|
319
353
|
debrief?: {
|
|
320
|
-
trigger: "stop_when" | "dwell";
|
|
354
|
+
trigger: "stop_when" | "dwell" | "participant_end";
|
|
321
355
|
status: "completed" | "skipped" | "failed";
|
|
322
356
|
reason: string;
|
|
323
357
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
|
324
358
|
usageReported?: boolean;
|
|
325
|
-
report
|
|
359
|
+
/** The report's impressions are kept in `impressions`, with the messages that quote them. */
|
|
360
|
+
report?: Omit<ParticipantClosingReport, "impressions">;
|
|
326
361
|
/** Links the readable projection so it is not heuristically classified a second time. */
|
|
327
362
|
messageId?: string;
|
|
328
363
|
};
|
|
364
|
+
/**
|
|
365
|
+
* What the participant said about the product at the end of a computer-use session, or why
|
|
366
|
+
* nothing was collected. Absent on other routes and on older traces.
|
|
367
|
+
*/
|
|
368
|
+
impressions?: ParticipantImpressions;
|
|
329
369
|
items: ActorTraceItem[];
|
|
330
370
|
tokenUsage?: ActorTokenUsage;
|
|
331
371
|
/** A stalled or adapter-reported ambiguous interaction may have additional unreported usage.
|
package/dist/actors/contract.js
CHANGED
|
@@ -168,6 +168,19 @@ export function validActorProviderRequests(value) {
|
|
|
168
168
|
usage.input));
|
|
169
169
|
});
|
|
170
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* What a participant can say about the product at the end of a session, beyond friction.
|
|
173
|
+
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
174
|
+
* life.
|
|
175
|
+
*/
|
|
176
|
+
export const PARTICIPANT_IMPRESSION_KINDS = [
|
|
177
|
+
"unclear",
|
|
178
|
+
"unfinished",
|
|
179
|
+
"untrustworthy",
|
|
180
|
+
"liked",
|
|
181
|
+
"missing",
|
|
182
|
+
"unlike_my_work",
|
|
183
|
+
];
|
|
171
184
|
export const CODEX_APP_SERVER_CAPABILITIES = {
|
|
172
185
|
headless: true,
|
|
173
186
|
structuredTrace: true,
|
|
@@ -20,4 +20,6 @@ export interface SourceEntry {
|
|
|
20
20
|
export declare function sourceEntries(bundle: RunBundle, stream: RunStream, captureVersion?: CaptureVersion): SourceEntry[];
|
|
21
21
|
/** Max-min allocation; stable ID order breaks a remainder tie by at most one slot. */
|
|
22
22
|
export declare function fairShares(budget: number, capacities: number[]): number[];
|
|
23
|
-
|
|
23
|
+
/** The trace messages that carry a participant's closing impressions. */
|
|
24
|
+
export declare function impressionEventIds(stream: RunStream): ReadonlySet<string>;
|
|
25
|
+
export declare function sourceOrder(entries: SourceEntry[], capturesOnly: boolean, impressions?: ReadonlySet<string>): SourceEntry[];
|
|
@@ -204,7 +204,12 @@ function spreadOrder(entries) {
|
|
|
204
204
|
}
|
|
205
205
|
return result;
|
|
206
206
|
}
|
|
207
|
-
|
|
207
|
+
/** The trace messages that carry a participant's closing impressions. */
|
|
208
|
+
export function impressionEventIds(stream) {
|
|
209
|
+
const impressions = stream.actor?.impressions;
|
|
210
|
+
return new Set(impressions?.status === "collected" ? impressions.items.map((item) => item.messageId) : []);
|
|
211
|
+
}
|
|
212
|
+
export function sourceOrder(entries, capturesOnly, impressions = new Set()) {
|
|
208
213
|
const candidates = capturesOnly ? entries.filter((entry) => entry.capturePath !== null) : entries;
|
|
209
214
|
const ordered = new Set();
|
|
210
215
|
const admit = (entry) => {
|
|
@@ -214,6 +219,10 @@ export function sourceOrder(entries, capturesOnly) {
|
|
|
214
219
|
// Actor endings precede appended run bookkeeping when text slots are scarce.
|
|
215
220
|
if (!capturesOnly)
|
|
216
221
|
admit(entries.findLast((entry) => !entry.kind.startsWith("run_event:")));
|
|
222
|
+
// The participant's closing impressions are its only opinions of the product; keep them all.
|
|
223
|
+
for (const entry of entries)
|
|
224
|
+
if (impressions.has(entry.eventId))
|
|
225
|
+
admit(entry);
|
|
217
226
|
admit(candidates.at(-1));
|
|
218
227
|
admit(candidates[0]);
|
|
219
228
|
const afterCapture = [];
|
|
@@ -6,7 +6,7 @@ import { isRecord } from "../run/type-guards.js";
|
|
|
6
6
|
import { RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
7
7
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
8
8
|
import { digestAnalysisInput, hashAnalysisValue, validateAnalysisInputMetadata, } from "./validation.js";
|
|
9
|
-
import { boundedText, fairShares, hasUnmappedCaptures, participantAssignment, participantSource, sourceEntries, sourceOrder, } from "./evidence-sources.js";
|
|
9
|
+
import { boundedText, fairShares, hasUnmappedCaptures, participantAssignment, participantSource, sourceEntries, sourceOrder, impressionEventIds, } from "./evidence-sources.js";
|
|
10
10
|
import { readBoundedFile, readBoundedFileResult } from "../run/evidence-files.js";
|
|
11
11
|
export const EVIDENCE_LIMITS = Object.freeze({
|
|
12
12
|
participants: 16,
|
|
@@ -312,7 +312,7 @@ function recordCaptureOmissions(packings, sel, omissions) {
|
|
|
312
312
|
omissions.add("Some declared captures have no normalized trace reference.");
|
|
313
313
|
for (const source of packing.captures.keys())
|
|
314
314
|
packing.admitted.add(source);
|
|
315
|
-
for (const source of sourceOrder(packing.entries, false)) {
|
|
315
|
+
for (const source of sourceOrder(packing.entries, false, impressionEventIds(packing.stream))) {
|
|
316
316
|
if (packing.admitted.size >= sel.evidenceShares[index])
|
|
317
317
|
break;
|
|
318
318
|
packing.admitted.add(source);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "./types.js";
|
|
2
2
|
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
|
-
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-
|
|
4
|
+
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
5
5
|
export declare const isSupportedAnalysisModel: (model: string) => boolean;
|
|
6
6
|
/** The OpenAI output limit when a study omits maxOutputTokens, and the most it may ask for. */
|
|
7
7
|
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
package/dist/analysis/execute.js
CHANGED
|
@@ -10,7 +10,7 @@ import { checkAnalysisResult, hashAnalysisValue, analysisResponseSchema, analysi
|
|
|
10
10
|
import { EVIDENCE_LIMITS } from "./evidence.js";
|
|
11
11
|
import { truncatedFrameWarning, unknownNotificationsWarning, } from "../actors/codex/restricted-notifications.js";
|
|
12
12
|
import { protocolAdditionsWarning, protocolIncompatibilityMessage, } from "../actors/codex/protocol-compat.js";
|
|
13
|
-
export const ANALYSIS_PROMPT_VERSION = "study-evidence-
|
|
13
|
+
export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
14
14
|
const SUPPORTED_ANALYSIS_MODELS = Object.freeze([
|
|
15
15
|
"gpt-6-astra",
|
|
16
16
|
"gpt-5.5",
|
|
@@ -61,6 +61,8 @@ Write for the person who will act on the report: a designer, product manager or
|
|
|
61
61
|
|
|
62
62
|
Separately, review the supplied captures as an experienced product designer would and report designFindings: problems a designer would notice in what the screens show, whether or not any participant mentioned them. Look at the size of text and controls (too small to read or to hit comfortably at this window size), layout and density (spread too wide or packed too tight for the task), visual hierarchy, legibility and contrast, wording and labels, consistency between screens, feedback after actions, and anything that makes the product look unfinished or untrustworthy. Judge the screens as the participants' personas would meet them: who they are and what they came to do. Each design finding names the screen in plain words, says what a designer notices, why it matters to a person using the product, and one concrete suggestion. Each must cite at least one supplied capture that shows the problem, and seenByStreamIds lists only participants whose cited captures show it. Rate severity by the effect on a person using the product: major when it misleads or blocks, moderate when it slows or confuses, minor when it is polish. Report only what the captures show; no capture, no design finding. Use D1, D2, and so on as IDs. An empty designFindings array is correct when the captures show no design problem worth a designer's time. Write design findings in the same plain register as headlines.
|
|
63
63
|
|
|
64
|
+
Messages that begin "Impression (kind):" are a participant's closing impressions of the product, where kind is unclear, unfinished, untrustworthy, liked, missing or unlike my work. They are that participant's own opinions: use them as participant statements and quote the words after the label. An unlike my work impression says where the screen differs from how the persona does the same task in its own work or life; with a capture of that screen it is a strong design finding, which cites both. Without such a capture, an impression supports only a finding about what the participant said.
|
|
65
|
+
|
|
64
66
|
Order findings by observed task impact, replication among exposed participants, and recovery. Preserve severity and confidence as separate fields. Do not compute a numeric frustration score or universal priority score. Use F1, F2, and so on as local finding IDs. Explain ordering with concrete evidence. Provide a short, testable next check for each finding. Say when recovery was not observed rather than claiming it was impossible. Keep titles and summaries concise and specific. The overall summary must be grounded in participant reviews and observations, including successful outcomes and evidence limits.`;
|
|
65
67
|
function instructions(config) {
|
|
66
68
|
return `${INSTRUCTIONS}\n\nResearcher question (null means use the standard review): ${JSON.stringify(config.question)}`;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
|
-
import { Command, Option } from "commander";
|
|
2
|
+
import { Command, InvalidArgumentError, Option } from "commander";
|
|
3
3
|
import { shellArg } from "../../substrates/shell.js";
|
|
4
4
|
import { computeStats, formatStatsHuman } from "../../run/stats.js";
|
|
5
5
|
import { DEFAULT_EXPORT_MAX_BYTES, exportRun, formatExportHuman } from "../../feedback/export.js";
|
|
@@ -8,7 +8,7 @@ import { verifyRun } from "../../verify/verify.js";
|
|
|
8
8
|
import { reclaimPreflightSandboxes, reclaimRunSandboxes, } from "../../run/reclaim.js";
|
|
9
9
|
import { runDisplay } from "../../run/display.js";
|
|
10
10
|
import { addRunOptions, handleRun } from "./run-command.js";
|
|
11
|
-
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, RUN_OPTION_DESCRIPTION, writeResult, humanError, } from "../io.js";
|
|
11
|
+
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, parsePositiveInteger, RUN_OPTION_DESCRIPTION, writeResult, humanError, } from "../io.js";
|
|
12
12
|
import { plural } from "../../run/text.js";
|
|
13
13
|
import { warnAndQueue } from "../deprecations.js";
|
|
14
14
|
import { formatFindings, readRunFindings } from "../findings.js";
|
|
@@ -136,29 +136,52 @@ export function registerRunsCommand(parent, io) {
|
|
|
136
136
|
io.setExitCode(result.ok ? 0 : 2);
|
|
137
137
|
});
|
|
138
138
|
}
|
|
139
|
+
/** A whole number of GiB or CPUs above zero, for `runtime setup --memory` and `--cpus`. */
|
|
140
|
+
function wholeNumber(flag) {
|
|
141
|
+
return (value) => {
|
|
142
|
+
const parsed = parsePositiveInteger(value);
|
|
143
|
+
if (parsed === null)
|
|
144
|
+
throw new InvalidArgumentError(`${flag} takes a whole number above zero, such as 8.`);
|
|
145
|
+
return parsed;
|
|
146
|
+
};
|
|
147
|
+
}
|
|
139
148
|
export function registerRuntimeCommands(parent, io) {
|
|
140
149
|
const runtime = parent
|
|
141
150
|
.command("runtime")
|
|
142
151
|
.description("Prepare or inspect the local browser runtime.");
|
|
143
152
|
for (const action of ["status", "setup"]) {
|
|
144
|
-
runtime
|
|
153
|
+
const command = runtime
|
|
145
154
|
.command(action)
|
|
146
155
|
.description(action === "status"
|
|
147
|
-
? "Check local Docker, virtualization and the cached browser image without downloads."
|
|
156
|
+
? "Check local Docker, virtualization and the cached browser image without downloads, and how many participant desktops fit."
|
|
148
157
|
: "Download and install the local browser image. Does not start a study or use model quota.")
|
|
149
158
|
.option("--json", JSON_OPTION_DESCRIPTION)
|
|
150
|
-
.option("--media", "Prepare or inspect the optional camera and speech runtime.")
|
|
151
|
-
|
|
159
|
+
.option("--media", "Prepare or inspect the optional camera and speech runtime.");
|
|
160
|
+
if (action === "setup")
|
|
161
|
+
command
|
|
162
|
+
.option("--memory <GiB>", "Mac only: the humanish Lima VM's memory. Resizes an existing VM, which stops it first.", wholeNumber("--memory"))
|
|
163
|
+
.option("--cpus <n>", "Mac only: the humanish Lima VM's CPUs. Resizes an existing VM, which stops it first.", wholeNumber("--cpus"));
|
|
164
|
+
command.action(async (options, command) => {
|
|
152
165
|
const { localRuntimeStatus, prepareLocalRuntime } = await import("../../substrates/local/runtime.js");
|
|
166
|
+
const { describeCapacity } = await import("../../substrates/local/capacity.js");
|
|
167
|
+
const media = options.media === true;
|
|
153
168
|
try {
|
|
154
|
-
if (action === "setup")
|
|
169
|
+
if (action === "setup") {
|
|
170
|
+
const size = {
|
|
171
|
+
...(options.memory === undefined ? {} : { memoryGiB: options.memory }),
|
|
172
|
+
...(options.cpus === undefined ? {} : { cpus: options.cpus }),
|
|
173
|
+
};
|
|
155
174
|
await prepareLocalRuntime({
|
|
156
|
-
media
|
|
175
|
+
media,
|
|
176
|
+
...(Object.keys(size).length === 0 ? {} : { size }),
|
|
157
177
|
progress: (message) => io.writeErr(`${message}\n`),
|
|
158
178
|
});
|
|
159
|
-
|
|
179
|
+
}
|
|
180
|
+
const status = await localRuntimeStatus({ media });
|
|
160
181
|
const result = { schema: "humanish.runtime-result.v1", ...status };
|
|
161
|
-
writeResult(command, io, result, () =>
|
|
182
|
+
writeResult(command, io, result, () => [status.message, ...(status.capacity ? [describeCapacity(status.capacity)] : [])]
|
|
183
|
+
.map((line) => `${line}\n`)
|
|
184
|
+
.join(""));
|
|
162
185
|
io.setExitCode(status.ok ? 0 : 2);
|
|
163
186
|
}
|
|
164
187
|
catch (error) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
2
|
import { listUserKeys, resolveKeyName, setUserKey, unsetUserKey, userKeyStorePath, } from "../../keys/key-resolution.js";
|
|
3
|
+
import { askForMissingKeys, formatKeyStatus, keyStatus, otherStoredKeys, } from "../../keys/key-status.js";
|
|
3
4
|
import { promptSecret } from "../secret-prompt.js";
|
|
4
5
|
import { runInit } from "../../study/init.js";
|
|
5
6
|
import { buildPayload, disabledByEnvironment, readTelemetryState, telemetryStatePath, writeTelemetryState, } from "../telemetry.js";
|
|
@@ -177,20 +178,107 @@ async function readSecretValue(useStdin, promptLabel) {
|
|
|
177
178
|
const text = Buffer.concat(chunks).toString("utf8").trim();
|
|
178
179
|
return text.length > 0 ? text : null;
|
|
179
180
|
}
|
|
180
|
-
|
|
181
|
+
// An empty line is no value here: only the walk over missing keys reads it as "skip".
|
|
182
|
+
return (await promptSecret(promptLabel, process.stdin, process.stderr)) || null;
|
|
183
|
+
}
|
|
184
|
+
/** `humanish keys` and `humanish keys status`: each provider key's source, never a value. */
|
|
185
|
+
async function writeKeyStatus(command, io) {
|
|
186
|
+
const rows = await keyStatus({ cwd: process.cwd(), env: process.env });
|
|
187
|
+
const others = otherStoredKeys(process.env);
|
|
188
|
+
const missing = rows.filter((row) => row.source === null).length;
|
|
189
|
+
const result = {
|
|
190
|
+
schema: KEYS_RESULT_SCHEMA,
|
|
191
|
+
ok: true,
|
|
192
|
+
action: "status",
|
|
193
|
+
store: userKeyStorePath(process.env),
|
|
194
|
+
names: rows.filter((row) => row.source !== null).map((row) => row.name),
|
|
195
|
+
keys: rows.map(({ name, use, source, hint }) => ({
|
|
196
|
+
name,
|
|
197
|
+
use,
|
|
198
|
+
source,
|
|
199
|
+
...(source === null ? { hint } : {}),
|
|
200
|
+
})),
|
|
201
|
+
...(others.length > 0 ? { otherStored: others } : {}),
|
|
202
|
+
message: missing === 0
|
|
203
|
+
? "Every provider key is set."
|
|
204
|
+
: `${plural(missing, "provider key")} ${missing === 1 ? "is" : "are"} missing.`,
|
|
205
|
+
};
|
|
206
|
+
writeResult(command, io, result, () => formatKeyStatus(rows, others));
|
|
207
|
+
io.setExitCode(0);
|
|
208
|
+
}
|
|
209
|
+
/** "A", "A and B", "A, B and C". */
|
|
210
|
+
function listNames(names) {
|
|
211
|
+
return names.length < 2 ? names.join("") : `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* `humanish keys set` with no key named. On a terminal it asks for each missing provider key in
|
|
215
|
+
* turn; without one there is no one to ask, so it refuses and names the --stdin form.
|
|
216
|
+
*/
|
|
217
|
+
async function setMissingKeys(command, io, useStdin) {
|
|
218
|
+
const store = userKeyStorePath(process.env);
|
|
219
|
+
const finish = (ok, names, message) => {
|
|
220
|
+
const result = {
|
|
221
|
+
schema: KEYS_RESULT_SCHEMA,
|
|
222
|
+
ok,
|
|
223
|
+
action: "set",
|
|
224
|
+
store,
|
|
225
|
+
names,
|
|
226
|
+
message,
|
|
227
|
+
};
|
|
228
|
+
writeResult(command, io, result, formatKeysHuman);
|
|
229
|
+
io.setExitCode(ok ? 0 : 2);
|
|
230
|
+
};
|
|
231
|
+
if (useStdin || process.stdin.isTTY !== true || process.stderr.isTTY !== true) {
|
|
232
|
+
finish(false, [], `missing required argument 'vendor-or-name'. Without a terminal humanish cannot ask for each key, so name the key and pipe its value, for example: printf '%s' "$E2B_API_KEY" | ${cli("keys set e2b --stdin")}`);
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
const rows = await keyStatus({ cwd: process.cwd(), env: process.env });
|
|
236
|
+
const missing = rows.filter((row) => row.source === null).length;
|
|
237
|
+
if (missing === 0) {
|
|
238
|
+
finish(true, [], `Every provider key is already set, so there is nothing to ask for. Replace one with \`${cli("keys set <vendor>")}\`.`);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
io.writeErr(`${plural(missing, "provider key")} ${missing === 1 ? "is" : "are"} missing. Paste each one when asked, or press Enter to skip it.\n`);
|
|
242
|
+
const outcome = await askForMissingKeys({
|
|
243
|
+
rows,
|
|
244
|
+
env: process.env,
|
|
245
|
+
prompt: (label) => promptSecret(label, process.stdin, process.stderr),
|
|
246
|
+
});
|
|
247
|
+
const sentences = [
|
|
248
|
+
outcome.stored.length > 0
|
|
249
|
+
? `Stored ${listNames(outcome.stored)} (0600).`
|
|
250
|
+
: "Nothing was stored.",
|
|
251
|
+
...(outcome.skipped.length > 0 ? [`Skipped ${listNames(outcome.skipped)}.`] : []),
|
|
252
|
+
...(outcome.rejected ?? []).map((entry) => `${entry.name} was not stored: ${entry.message}`),
|
|
253
|
+
...(outcome.stoppedAt === undefined
|
|
254
|
+
? []
|
|
255
|
+
: [`Stopped at ${outcome.stoppedAt}; the keys after it were not asked for.`]),
|
|
256
|
+
`\`${cli("keys")}\` shows where each key comes from.`,
|
|
257
|
+
];
|
|
258
|
+
// A cancel is the person's choice, so only a value the store refused fails the command.
|
|
259
|
+
finish(outcome.rejected === undefined, outcome.stored, sentences.join(" "));
|
|
181
260
|
}
|
|
182
261
|
export function registerKeysCommand(parent, io) {
|
|
183
262
|
const keys = parent
|
|
184
263
|
.command("keys")
|
|
185
|
-
.description("
|
|
186
|
-
.summary("
|
|
264
|
+
.description("Show which provider keys are set and where each comes from, and manage the humanish user key store that provider-key discovery reads.")
|
|
265
|
+
.summary("Check and store your provider keys.");
|
|
266
|
+
keys
|
|
267
|
+
.command("status", { isDefault: true })
|
|
268
|
+
.description("Show each provider key humanish uses: where it comes from, or the command that adds it. Values are never printed.")
|
|
269
|
+
.option("--json", JSON_OPTION_DESCRIPTION)
|
|
270
|
+
.action(async (_options, command) => writeKeyStatus(command, io));
|
|
187
271
|
keys
|
|
188
272
|
.command("set")
|
|
189
|
-
.argument("
|
|
190
|
-
.description("Store one provider key in the user store (0600), prompted with hidden input.")
|
|
273
|
+
.argument("[vendor-or-name]", "A vendor alias (openai, e2b, anthropic, github, agentmail) or a raw ENV_NAME. Without one, a terminal asks for each missing key in turn.")
|
|
274
|
+
.description("Store one provider key in the user store (0600), prompted with hidden input. With no key named, ask for each missing key in turn.")
|
|
191
275
|
.option("--stdin", "Read the value from stdin instead of prompting (for agents/pipes).")
|
|
192
276
|
.option("--json", JSON_OPTION_DESCRIPTION)
|
|
193
277
|
.action(async (vendorOrName, options, command) => {
|
|
278
|
+
if (vendorOrName === undefined) {
|
|
279
|
+
await setMissingKeys(command, io, options.stdin === true);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
194
282
|
const name = resolveKeyName(vendorOrName);
|
|
195
283
|
const storePath = userKeyStorePath(process.env);
|
|
196
284
|
if (name === null) {
|
|
@@ -294,11 +382,12 @@ export function registerKeysCommand(parent, io) {
|
|
|
294
382
|
function formatDoctorHuman(result) {
|
|
295
383
|
return ([
|
|
296
384
|
`humanish doctor ${result.ok ? "ok" : "needs setup"}`,
|
|
385
|
+
...(result.next === undefined ? [] : [`next: ${result.next}`]),
|
|
297
386
|
`cwd: ${result.cwd}`,
|
|
298
387
|
// "missing" is a verdict, and a row that never ran has none. A participant reading doctor on a
|
|
299
388
|
// fresh desktop got `- missing package.json: package.json is present and safe to read`, which
|
|
300
389
|
// contradicts itself in eleven words (labs/tui-self-study.yaml).
|
|
301
|
-
...result.checks.map((check) => `- ${check.status.replace("_", " ")} ${check.name}: ${check.message}`),
|
|
390
|
+
...result.checks.map((check) => `- ${check.status.replace("_", " ")} ${check.name}: ${check.message.replaceAll("\n", "\n ")}`),
|
|
302
391
|
].join("\n") + "\n");
|
|
303
392
|
}
|
|
304
393
|
function formatInitHuman(result) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
1
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
2
3
|
/** A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route. */
|
|
3
4
|
function runHeader(result, route) {
|
|
@@ -39,17 +40,19 @@ export function formatConcurrentSharedWorldStudyHuman(result) {
|
|
|
39
40
|
...runHeader(result, "shared-world"),
|
|
40
41
|
`run: ${result.runId}`,
|
|
41
42
|
`actor: ${result.actor}`,
|
|
42
|
-
`
|
|
43
|
+
`participants: ${result.roleCount}, up to ${result.concurrency} at once`,
|
|
43
44
|
...(result.host ? [`host: ${result.host}`] : []),
|
|
44
45
|
...(result.subject?.commit
|
|
45
|
-
? [`
|
|
46
|
+
? [`app: ${result.subject.repo}@${result.subject.commit.slice(0, 12)}`]
|
|
46
47
|
: []),
|
|
47
48
|
...(result.overlapProven === undefined
|
|
48
49
|
? []
|
|
49
50
|
: [
|
|
50
|
-
|
|
51
|
+
result.overlapProven
|
|
52
|
+
? "Participants used the app at the same time (observed in this run only)."
|
|
53
|
+
: "Participants were not observed using the app at the same time.",
|
|
51
54
|
]),
|
|
52
|
-
...result.roles.map((participant) =>
|
|
55
|
+
...result.roles.map((participant) => `${participantCaption({ id: participant.id, personaId: participant.persona })}: ${participantStatus(participant.status)}${participant.ok ? "" : ", not ok"}`),
|
|
53
56
|
...(result.subjectSandbox
|
|
54
57
|
? [
|
|
55
58
|
`subject sandbox: ${sandboxName(result.subjectSandbox)} killed=${result.subjectSandbox.killed ? "yes" : "no"}`,
|
|
@@ -140,7 +140,7 @@ function formatStudyCheckHuman(result) {
|
|
|
140
140
|
...(result.sandbox.created
|
|
141
141
|
? [`sandbox: created=yes killed=${result.sandbox.killed === true ? "yes" : "no"}`]
|
|
142
142
|
: []),
|
|
143
|
-
...result.checks.map((check) => `- ${check.ok ? "ok" : "fail"} ${check.name}: ${check.message}`),
|
|
143
|
+
...result.checks.map((check) => `- ${check.checked === false ? "not checked" : check.ok ? "ok" : "fail"} ${check.name}: ${check.message}`),
|
|
144
144
|
...result.warnings.map((warning) => `warning: ${warning}`),
|
|
145
145
|
].join("\n") + "\n";
|
|
146
146
|
return result.error === undefined ? stdout : { stdout, error: result.error };
|
package/dist/cli/commands/tui.js
CHANGED
|
@@ -278,7 +278,7 @@ async function storeAgentmailKey(session) {
|
|
|
278
278
|
const { cwd, runtime, sessionEnv } = session;
|
|
279
279
|
// startTui has unmounted: only the host reads the credential, then remounts the view.
|
|
280
280
|
const value = await runtime.promptSecret("AgentMail API key", runtime.stdin, runtime.stdout);
|
|
281
|
-
if (value === null)
|
|
281
|
+
if (value === null || value === "")
|
|
282
282
|
return "Key entry cancelled. Nothing was changed.";
|
|
283
283
|
try {
|
|
284
284
|
setUserKey("AGENTMAIL_API_KEY", value, sessionEnv);
|