humanish 0.111.4 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +71 -22
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -14,8 +14,8 @@ export interface UnfinishedRun {
|
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
16
|
* One warning, and the unfinished-run facts, when the run is not finished, by the run index's
|
|
17
|
-
* rule: its
|
|
18
|
-
*
|
|
17
|
+
* rule: its bundle outcome first, then usable status, then the bundle's simulations. Nothing
|
|
18
|
+
* for a finished run.
|
|
19
19
|
*/
|
|
20
20
|
export declare function runNotFinished(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, nowMs?: number): Promise<{
|
|
21
21
|
warnings: string[];
|
package/dist/verify/liveness.js
CHANGED
|
@@ -7,7 +7,7 @@ import path from "node:path";
|
|
|
7
7
|
import { readRunJsonIfExists, readRunTextIfExists } from "../run/locate.js";
|
|
8
8
|
import { isRecord } from "../run/type-guards.js";
|
|
9
9
|
import { RECLAIM_RECEIPT_ARTIFACT } from "../run/reclaim.js";
|
|
10
|
-
import { runLiveness } from "../run/
|
|
10
|
+
import { runLiveness } from "../run/liveness.js";
|
|
11
11
|
import { parseSandboxReceipts, SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
12
12
|
import { RUN_STATUS_FILE } from "../run/status.js";
|
|
13
13
|
import { plural } from "../run/text.js";
|
|
@@ -17,8 +17,8 @@ const RUN_NOT_FINISHED = "RUN_NOT_FINISHED";
|
|
|
17
17
|
const GONE_STATES = new Set(["killed", "already-gone"]);
|
|
18
18
|
/**
|
|
19
19
|
* One warning, and the unfinished-run facts, when the run is not finished, by the run index's
|
|
20
|
-
* rule: its
|
|
21
|
-
*
|
|
20
|
+
* rule: its bundle outcome first, then usable status, then the bundle's simulations. Nothing
|
|
21
|
+
* for a finished run.
|
|
22
22
|
*/
|
|
23
23
|
export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
|
|
24
24
|
const runId = path.basename(runPaths.absoluteRunRoot);
|
|
@@ -27,7 +27,10 @@ export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
|
|
|
27
27
|
if (liveness === "finished")
|
|
28
28
|
return { warnings: [] };
|
|
29
29
|
const seen = [];
|
|
30
|
-
if (
|
|
30
|
+
if (bundle.outcome?.state === "interrupted") {
|
|
31
|
+
seen.push(`run.json records interruption by ${bundle.outcome.signal} at ${bundle.outcome.at}`);
|
|
32
|
+
}
|
|
33
|
+
else if (record === undefined) {
|
|
31
34
|
const running = bundle.simulations.filter((simulation) => simulation.status === "running");
|
|
32
35
|
seen.push(`the run has no usable ${RUN_STATUS_FILE} and ${running.length} of ${bundle.simulations.length} simulations are still running`);
|
|
33
36
|
}
|
|
@@ -84,6 +84,20 @@ at the base tier, with cached input at the cached rate, and says so with
|
|
|
84
84
|
checked against `caps.maxUsd` keeps the provider line unmeasured, and the studied
|
|
85
85
|
product's no-spend boundary still needs separate interpretation.
|
|
86
86
|
|
|
87
|
+
## Participant telemetry
|
|
88
|
+
|
|
89
|
+
Terminal participant commands and product setup receive `DO_NOT_TRACK=1` by default.
|
|
90
|
+
Their child processes inherit it, so studied CLIs that honor the convention can exclude
|
|
91
|
+
synthetic usage. To study the product's telemetry behavior, set
|
|
92
|
+
`execution.terminal.doNotTrack: false`; humanish then leaves the variable unset.
|
|
93
|
+
Only booleans are accepted, and `true` or an omitted field enables the default.
|
|
94
|
+
|
|
95
|
+
The resolved `doNotTrack` boolean is recorded with `humanish.actor-runtime.v1` in
|
|
96
|
+
`actor.json`, `streams[].actor.runtime` in `run.json`, and `runtime` in
|
|
97
|
+
`terminal-ledgers.json`. A dry run records it in the JSON message of its
|
|
98
|
+
`terminal-lab.runtime.declared` event. Older bundles omit it. This records the
|
|
99
|
+
command environment policy; it does not prove that the studied CLI honors the variable.
|
|
100
|
+
|
|
87
101
|
## Runtime auth: raw-key placement and remaining provider access
|
|
88
102
|
|
|
89
103
|
`execution.runtimeAuth: openai-egress` is the default: a study that declares no
|
|
@@ -6,6 +6,13 @@
|
|
|
6
6
|
source of truth. This page explains the stable public fields and the extension rules; it does not
|
|
7
7
|
version the runtime separately.
|
|
8
8
|
|
|
9
|
+
Terminal runs record the resolved telemetry setting as `doNotTrack` in
|
|
10
|
+
`streams[].actor.runtime` (`humanish.actor-runtime.v1`), also retained in `actor.json`
|
|
11
|
+
and `terminal-ledgers.json`. `true` means participant and product setup commands receive
|
|
12
|
+
`DO_NOT_TRACK=1`; `false` means humanish leaves it unset. Dry runs record the same boolean
|
|
13
|
+
in the JSON message of the `terminal-lab.runtime.declared` event. Older bundles omit it.
|
|
14
|
+
The setting does not attest that the studied CLI honors the variable.
|
|
15
|
+
|
|
9
16
|
## Purpose
|
|
10
17
|
|
|
11
18
|
A run bundle is the durable evidence packet for one harness run. It should be
|
|
@@ -154,6 +161,9 @@ writing it, and on runs recorded before the field existed. For those, readers
|
|
|
154
161
|
take liveness from `status.json` and, for an older finished run, its `ok` and
|
|
155
162
|
`execution` from `status.json`'s `outcome`. That record is the only copy of an
|
|
156
163
|
older run's `ok`; for a run with `outcome` in `run.json`, `status.json` copies it.
|
|
164
|
+
The bundle's outcome wins even if the process stopped before updating `status.json`.
|
|
165
|
+
Without an outcome or a usable status record naming the run, a bundle with a simulation still
|
|
166
|
+
`running` reads as interrupted. A fresh status record keeps an in-progress bundle running.
|
|
157
167
|
|
|
158
168
|
Every surface that shows whether a run passed calls `runDisplay`
|
|
159
169
|
(`src/run/display.ts`) with the run's liveness, verdict and `ok`: the Observer
|
|
@@ -35,7 +35,7 @@ workflow without leaking private upstream truth into core.
|
|
|
35
35
|
| Substrate | reserved (never shipped) | none |
|
|
36
36
|
| Evidence stream | reserved (streams live inside the run bundle) | see [`run-bundle.md`](run-bundle.md) |
|
|
37
37
|
| Review | `humanish.review.v1` | `contract-proof-review` |
|
|
38
|
-
| Study analysis | `humanish.study-analysis.v1`
|
|
38
|
+
| Study analysis | `humanish.study-analysis.v1` (prompt `study-evidence-7` adds `findings[].headline`, `findings[].experience` and `designFindings[]`; older artifacts omit them) | see [study analysis](study-analysis.md) and synthetic analysis fixtures in `tests/` |
|
|
39
39
|
| Study analysis correction | `humanish.study-analysis-correction.v1` | see [study analysis](study-analysis.md#human-review-and-sharing) |
|
|
40
40
|
| Analysis findings view | `humanish.analysis-findings.v1` (`review --json` field `analysis`; read-only projection of the selected analysis) | see [study analysis](study-analysis.md#reading-findings) |
|
|
41
41
|
| Analysis execution receipt | `humanish.analysis-execution.v1` | see [study analysis](study-analysis.md#durable-records) |
|
|
@@ -296,6 +296,14 @@ instruction?, target?, entry?, host?, reasoningEffort?, stopWhen?, dwell? }` bec
|
|
|
296
296
|
metadata; with `openai-egress` it stays in the host-side E2B header transform
|
|
297
297
|
set at sandbox creation, and the agent gets a placeholder. A dry-run neither
|
|
298
298
|
reads nor injects it;
|
|
299
|
+
- `execution.terminal.doNotTrack`: boolean, default `true`. Terminal participant and product
|
|
300
|
+
setup commands receive `DO_NOT_TRACK=1`. Set `false` to leave that variable unset when
|
|
301
|
+
studying the product's telemetry behavior. Non-boolean values are refused at parse.
|
|
302
|
+
The resolved boolean is `doNotTrack` in `humanish.actor-runtime.v1`: `actor.json`,
|
|
303
|
+
`streams[].actor.runtime` in `run.json`, and `runtime` in `terminal-ledgers.json` for live
|
|
304
|
+
runs; the JSON message of the `terminal-lab.runtime.declared` event for dry runs.
|
|
305
|
+
Older bundles omit this field. It records the command environment policy, not whether
|
|
306
|
+
the studied CLI honored it;
|
|
299
307
|
- `mode`: `dry-run` (a synthetic bundle, no spend; the default) or `live`. On the
|
|
300
308
|
scripted route `live` gates real browser actuation against the declared app.
|
|
301
309
|
Provider spend stays $0 there by mechanism (no model runs);
|
|
@@ -161,6 +161,42 @@ thought. Older reports may omit this field and remain readable. The disclosure
|
|
|
161
161
|
is model-generated assessment, not an independent completeness audit or a human
|
|
162
162
|
reviewer annotation.
|
|
163
163
|
|
|
164
|
+
## Headlines and design findings
|
|
165
|
+
|
|
166
|
+
The `study-evidence-7` prompt writes each finding twice. `headline` is one plain sentence about what
|
|
167
|
+
happened, for a designer, product manager or developer who did not watch the session.
|
|
168
|
+
`experience` is one to three plain sentences: what the person tried, what got in their way, and
|
|
169
|
+
how it seemed to feel. Neither names participant or evidence IDs. A concern known only from what
|
|
170
|
+
participants said still reads as something they said. `title`, `summary` and `observations` keep
|
|
171
|
+
their evidence-level wording, and validation applies to them as before.
|
|
172
|
+
|
|
173
|
+
The same prompt adds `designFindings`: problems a product designer would notice in the supplied
|
|
174
|
+
captures, whether or not a participant mentioned them. Each has an `id` (`D1`, `D2`, ...), a
|
|
175
|
+
`headline`, the `screen` in plain words, what a designer would `notice`, `whyItMatters` to a
|
|
176
|
+
person using the product, one `suggestion`, a `severity` (`major` when it misleads or blocks,
|
|
177
|
+
`moderate` when it slows or confuses, `minor` when it is polish), a `confidence`,
|
|
178
|
+
`seenByStreamIds` and `evidenceIds`. Validation rejects a design finding that cites no retained
|
|
179
|
+
capture, cites evidence outside the packet, repeats an ID, or lists a participant in
|
|
180
|
+
`seenByStreamIds` without a cited capture of that participant. An empty list means the review found no design
|
|
181
|
+
problem in the captures. The model judges legibility and size from images alone; it does not
|
|
182
|
+
measure text or control sizes.
|
|
183
|
+
|
|
184
|
+
New provider responses must include `headline` and `experience` on every finding and a
|
|
185
|
+
`designFindings` list. A stored artifact with prompt `study-evidence-7` or later must carry them
|
|
186
|
+
too. Artifacts from earlier prompts load without them and render by title and summary. The
|
|
187
|
+
Observer applies the same prompt-version rules and the same field limits when it reads a saved
|
|
188
|
+
analysis, and treats an analysis that breaks them as unavailable.
|
|
189
|
+
|
|
190
|
+
The narrative scrub removes a run's known transient values, such as a received one-time code or
|
|
191
|
+
link, from every generated text field. It finds each value as written and in its encoded forms:
|
|
192
|
+
percent-encoded, JSON-escaped, base64, base64url and hex, and where escapes split it. A field
|
|
193
|
+
with a value is stored decoded with the value replaced; a field without one keeps its spelling. A
|
|
194
|
+
known value in any of these forms in an ID, enum or reference refuses the response.
|
|
195
|
+
|
|
196
|
+
When a reviewer amends a finding, its headline and experience describe the replaced claim.
|
|
197
|
+
Text output, the Observer and feedback drafts show the reviewer's claim in their place and say the
|
|
198
|
+
finding was corrected; the stored analysis keeps the original.
|
|
199
|
+
|
|
164
200
|
Every observation cites packet-local evidence IDs. The model cannot choose a
|
|
165
201
|
filesystem path or fetch another resource. Validation checks participant
|
|
166
202
|
membership, unique counts, quote fidelity, evidence type and reference
|
|
@@ -203,7 +239,7 @@ feedback quotes cite evidence IDs that the run's own `analysis.json` resolves.
|
|
|
203
239
|
Participant reviews cite distinct packet-local evidence IDs belonging only to
|
|
204
240
|
that participant's stream; source event IDs are not citation IDs. Shared
|
|
205
241
|
interactions can cite multiple included participants in findings and concern
|
|
206
|
-
reviews under the existing exposure rules. The `study-evidence-6` prompt
|
|
242
|
+
reviews under the existing exposure rules. The `study-evidence-6` prompt made
|
|
207
243
|
this distinction explicit without changing the validator or historical reports.
|
|
208
244
|
|
|
209
245
|
Elapsed replay time starts at the first retained capture. It is not a video
|
|
@@ -219,6 +255,12 @@ guarantee that the selected capture is the most relevant. Context-only and
|
|
|
219
255
|
nonvisual evidence retain their basis and original event. All cited moments stay
|
|
220
256
|
available, with exact recording links.
|
|
221
257
|
|
|
258
|
+
A finding with a headline leads its Observer row with the headline and its open panel with the
|
|
259
|
+
experience. The title, summary, assessment and observations sit in a closed **Evidence**
|
|
260
|
+
disclosure beneath. A **Design findings** section follows the ranked findings, grouped by
|
|
261
|
+
severity, with each cited capture as a thumbnail that opens that frame. Findings without a
|
|
262
|
+
headline render as described below.
|
|
263
|
+
|
|
222
264
|
Confidence, recovery and the first full evidence limitation remain visible when
|
|
223
265
|
a finding opens. Exposure and remaining unique limits are one disclosure away;
|
|
224
266
|
observation details retain each original claim and its specific limitation.
|
|
@@ -291,31 +333,35 @@ is disabled. Existing adapter captures and logs retain their contained routes.
|
|
|
291
333
|
## Reading findings
|
|
292
334
|
|
|
293
335
|
`humanish review` prints a run's analysis findings after its participant review, and
|
|
294
|
-
`humanish analyze show` prints them alone. Each finding
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
336
|
+
`humanish analyze show` prints them alone. Each finding leads with its headline and experience,
|
|
337
|
+
then shows its title as `evidence:`, its impact, confidence and recovery, the participants it
|
|
338
|
+
affected out of those exposed, the frames its evidence cites with their time since the first
|
|
339
|
+
retained capture, the capture files, its next step and the latest human review note. A finding
|
|
340
|
+
without a headline starts at its title. Design findings follow, most severe first, each with its
|
|
341
|
+
screen, notice, why it matters, suggestion, who saw it and its capture files. A live run's human
|
|
342
|
+
output ends with up to three findings and up to three design findings, one line each, and the
|
|
343
|
+
`review` command that prints all of them. `analyze show --json` prints the validated
|
|
299
344
|
analysis record and its corrections, as before. Reading findings never starts an analysis.
|
|
300
345
|
|
|
301
346
|
`review --json` carries the same view as its `analysis` field, schema
|
|
302
347
|
`humanish.analysis-findings.v1`, built in `src/cli/findings.ts`:
|
|
303
348
|
|
|
304
|
-
| Field | Meaning
|
|
305
|
-
| ------------------------------------------------------------------------------ |
|
|
306
|
-
| `schema` | `humanish.analysis-findings.v1`.
|
|
307
|
-
| `runId` | The run the view reads.
|
|
308
|
-
| `state` | `ready`, `stale`, `running`, `none`, `skipped`, `failed`, `dry_run` or `unavailable`. Only `ready` and `stale` carry findings.
|
|
309
|
-
| `reason` | The stable code behind a state other than `ready`, or null: an automatic analysis reason such as `AUTOMATIC_ANALYSIS_KEY_MISSING`, a failed attempt's error code, or a load warning such as `ANALYSIS_SOURCE_CHANGED`.
|
|
310
|
-
| `message` | What happened, in words.
|
|
311
|
-
| `next` | The command that gets findings for this run, or for a dry run the command that starts a live one. Null when no command can, as for a run whose participants left no evidence.
|
|
312
|
-
| `analysisId`, `status`, `provider`, `model`, `completedAt`, `estimatedCostUsd` | The analysis version `analyze show` selects: the newest ready one, else the newest stale one. Null without one.
|
|
313
|
-
| `runPath`, `path` | The run directory and the version's `analysis.json`, relative to the project directory.
|
|
314
|
-
| `summary`, `limitations` | The analysis's own summary and limitations.
|
|
315
|
-
| `findings[]` | Highest priority first: `id`, `title`, `summary`, `impact` (`blocked_task`, `friction`, `recovery`, `uncertain`), `confidence` (`low`, `medium`, `high`), `recovery` (`recovered`, `not_observed`, `unknown`), `affected[]` (`streamId`, `label`), `exposedCount`, `evidence[]`, `nextStep` and `correction`.
|
|
316
|
-
| `findings[].evidence[]` | Each cited item once: `id`, `streamId`, `kind`, `bases` (how the observations used it), `frame` (the latest retained capture's index from 0, null before any capture), `elapsedMs` (time since the first retained capture, not a video offset), `at` and `capture` (the file relative to the project directory, null for nonvisual evidence).
|
|
317
|
-
| `findings[].correction` | The latest `analyze correct` note on that finding version (`status`, `reason`, `replacementClaim`), or null.
|
|
318
|
-
| `
|
|
349
|
+
| Field | Meaning |
|
|
350
|
+
| ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
351
|
+
| `schema` | `humanish.analysis-findings.v1`. |
|
|
352
|
+
| `runId` | The run the view reads. |
|
|
353
|
+
| `state` | `ready`, `stale`, `running`, `none`, `skipped`, `failed`, `dry_run` or `unavailable`. Only `ready` and `stale` carry findings. |
|
|
354
|
+
| `reason` | The stable code behind a state other than `ready`, or null: an automatic analysis reason such as `AUTOMATIC_ANALYSIS_KEY_MISSING`, a failed attempt's error code, or a load warning such as `ANALYSIS_SOURCE_CHANGED`. |
|
|
355
|
+
| `message` | What happened, in words. |
|
|
356
|
+
| `next` | The command that gets findings for this run, or for a dry run the command that starts a live one. Null when no command can, as for a run whose participants left no evidence. |
|
|
357
|
+
| `analysisId`, `status`, `provider`, `model`, `completedAt`, `estimatedCostUsd` | The analysis version `analyze show` selects: the newest ready one, else the newest stale one. Null without one. |
|
|
358
|
+
| `runPath`, `path` | The run directory and the version's `analysis.json`, relative to the project directory. |
|
|
359
|
+
| `summary`, `limitations` | The analysis's own summary and limitations. |
|
|
360
|
+
| `findings[]` | Highest priority first: `id`, `headline` and `experience` (null in analyses before `study-evidence-7`), `title`, `summary`, `impact` (`blocked_task`, `friction`, `recovery`, `uncertain`), `confidence` (`low`, `medium`, `high`), `recovery` (`recovered`, `not_observed`, `unknown`), `affected[]` (`streamId`, `label`), `exposedCount`, `evidence[]`, `nextStep` and `correction`. |
|
|
361
|
+
| `findings[].evidence[]` | Each cited item once: `id`, `streamId`, `kind`, `bases` (how the observations used it), `frame` (the latest retained capture's index from 0, null before any capture), `elapsedMs` (time since the first retained capture, not a video offset), `at` and `capture` (the file relative to the project directory, null for nonvisual evidence). |
|
|
362
|
+
| `findings[].correction` | The latest `analyze correct` note on that finding version (`status`, `reason`, `replacementClaim`), or null. When `status` is `amended`, `replacementClaim` replaces the finding's headline and experience. |
|
|
363
|
+
| `designFindings` | Most severe first: `id`, `headline`, `screen`, `notice`, `whyItMatters`, `suggestion`, `severity` (`major`, `moderate`, `minor`), `confidence`, `seenBy[]` (`streamId`, `label`) and `evidence[]` (as `findings[].evidence[]`, without `bases`). Null without an analysis result, or for an analysis written before design findings. |
|
|
364
|
+
| `warnings` | Load warnings for the selected version. |
|
|
319
365
|
|
|
320
366
|
The states without findings:
|
|
321
367
|
|
|
@@ -352,7 +398,10 @@ humanish feedback issue --run latest --analysis <id> --finding F1 --repo owner/r
|
|
|
352
398
|
|
|
353
399
|
Corrections append against exact analysis and finding hashes. Confirming one
|
|
354
400
|
version does not approve a later claim. Dismissed findings cannot become feedback
|
|
355
|
-
drafts; amendments preserve the original and record the replacement.
|
|
401
|
+
drafts; amendments preserve the original and record the replacement. A reason or replacement
|
|
402
|
+
claim that matches a sensitive pattern is refused. When the correction is made inside the run's
|
|
403
|
+
own invocation, one that holds a known transient value, as written or encoded, is refused too; a
|
|
404
|
+
separate `analyze correct` command has no transient values to check against. Feedback
|
|
356
405
|
drafts include source/version/evidence references and remain explicitly
|
|
357
406
|
independent of participant-authored candidates. No command above posts to GitHub.
|
|
358
407
|
Each analysis has room for 256 correction inventory entries, including interrupted
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "humanish",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.112.0",
|
|
4
4
|
"description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cli",
|
|
@@ -76,6 +76,7 @@
|
|
|
76
76
|
"tui:test": "pnpm --filter humanish-tui test",
|
|
77
77
|
"release:dogfood": "node scripts/release-dogfood.mjs",
|
|
78
78
|
"bench": "tsx bench/run.ts",
|
|
79
|
+
"arch:bench": "tsx scripts/arch-bench.ts",
|
|
79
80
|
"browser-control:proof": "node scripts/browser-control-proof.mjs",
|
|
80
81
|
"docs:generate": "tsx scripts/generate-cli-docs.ts",
|
|
81
82
|
"docs:check": "tsx scripts/generate-cli-docs.ts --check && tsx scripts/check-doc-paths.ts && tsx scripts/check-doc-symbols.ts && tsx scripts/check-doc-comments.ts && tsx scripts/check-doc-study-fields.ts",
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
import type { CuaTurnRequest } from "./loop.js";
|
|
2
|
-
import type { ReasoningEffort } from "../reasoning-effort.js";
|
|
3
|
-
import type { ActorConversation } from "../contract.js";
|
|
4
|
-
import { CarriedConversation } from "./openai-context.js";
|
|
5
|
-
import { type OpenAiReasoningSummary, type ParsedOpenAiResponse } from "./openai-wire.js";
|
|
6
|
-
/** Request settings fixed for a provider's whole session. */
|
|
7
|
-
export interface OpenAiRequestSettings {
|
|
8
|
-
readonly model: string;
|
|
9
|
-
readonly reasoningEffort: ReasoningEffort;
|
|
10
|
-
readonly maxOutputTokens?: number;
|
|
11
|
-
readonly safetyIdentifier?: string;
|
|
12
|
-
}
|
|
13
|
-
/** The conversation as the provider has it after its last accepted reply. */
|
|
14
|
-
export interface OpenAiConversationState {
|
|
15
|
-
lastResponseId: string | undefined;
|
|
16
|
-
/** computer_call ids the next request owes an output for. */
|
|
17
|
-
pendingCallIds: string[];
|
|
18
|
-
/** Accepted replies so far; the next request is number `replies + 1`. */
|
|
19
|
-
replies: number;
|
|
20
|
-
/** The whole conversation, which explicit-context requests carry within its token budget. */
|
|
21
|
-
conversation: CarriedConversation;
|
|
22
|
-
/** explicit_context after a zero-data-retention rejection or when configured. */
|
|
23
|
-
mode: "previous_response_id" | "explicit_context";
|
|
24
|
-
/** Undefined once the account or model rejected reasoning summaries. */
|
|
25
|
-
reasoningSummary: OpenAiReasoningSummary | undefined;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* The items the next request adds to the conversation: the opening message with the first
|
|
29
|
-
* screen, or an output for each pending computer_call with the latest screenshot and any hint.
|
|
30
|
-
* Built once per turn, so every attempt at the request and the accepted reply see the same items.
|
|
31
|
-
*/
|
|
32
|
-
export declare function turnInputItems(state: OpenAiConversationState, req: CuaTurnRequest): readonly unknown[];
|
|
33
|
-
/**
|
|
34
|
-
* The body of the next participant request: the opening request, or a continuation carrying
|
|
35
|
-
* `sent` (turnInputItems). A threaded continuation names the previous response; an
|
|
36
|
-
* explicit-context one carries the conversation ahead of `sent`.
|
|
37
|
-
*/
|
|
38
|
-
export declare function turnRequestBody(settings: OpenAiRequestSettings, state: OpenAiConversationState, req: CuaTurnRequest, sent: readonly unknown[]): Record<string, unknown>;
|
|
39
|
-
/** The body of the read-only closing report: no tools, a small output limit, a strict schema. */
|
|
40
|
-
export declare function debriefRequestBody(settings: OpenAiRequestSettings, state: OpenAiConversationState, req: CuaTurnRequest, sent: readonly unknown[]): Record<string, unknown>;
|
|
41
|
-
/**
|
|
42
|
-
* Move the conversation on to an accepted reply to a request that added `sent`. A participant
|
|
43
|
-
* reply cut off by the output limit is set aside, so the next request is the same one again
|
|
44
|
-
* (outputLimitRetry); its actions never run, so no call output is owed for them.
|
|
45
|
-
*/
|
|
46
|
-
export declare function acceptReply(state: OpenAiConversationState, parsed: ParsedOpenAiResponse, closing: boolean, sent: readonly unknown[]): void;
|
|
47
|
-
/** The trace's account of how a session carried its conversation (ActorConversation). */
|
|
48
|
-
export declare class ConversationRecord {
|
|
49
|
-
private readonly now;
|
|
50
|
-
private explicitReason;
|
|
51
|
-
private rejection;
|
|
52
|
-
private switchedAt;
|
|
53
|
-
private switchedAtRequest;
|
|
54
|
-
private summarizedTurns;
|
|
55
|
-
private readonly requests;
|
|
56
|
-
constructor(configured: boolean, now: () => number);
|
|
57
|
-
/** The organization rejected server-side state, so the request being sent is explicit_context. */
|
|
58
|
-
switched(rejection: NonNullable<ActorConversation["rejection"]>): void;
|
|
59
|
-
/** A participant request just went out with `sent` after the carried conversation. */
|
|
60
|
-
requested(state: OpenAiConversationState, sent: readonly unknown[]): void;
|
|
61
|
-
snapshot(state: OpenAiConversationState): ActorConversation;
|
|
62
|
-
}
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
import { CarriedConversation, estimateTokens } from "./openai-context.js";
|
|
2
|
-
import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintItems, openingMessage, } from "./openai-wire.js";
|
|
3
|
-
/** The closing report's output limit: its own cap, or the declared limit when that is lower. */
|
|
4
|
-
const CLOSING_OUTPUT_LIMIT = 1024;
|
|
5
|
-
function requestContext(settings, state, instructions) {
|
|
6
|
-
return {
|
|
7
|
-
stateless: state.mode === "explicit_context",
|
|
8
|
-
model: settings.model,
|
|
9
|
-
instructions,
|
|
10
|
-
reasoningEffort: settings.reasoningEffort,
|
|
11
|
-
...(settings.maxOutputTokens === undefined
|
|
12
|
-
? {}
|
|
13
|
-
: { maxOutputTokens: settings.maxOutputTokens }),
|
|
14
|
-
...(state.reasoningSummary === undefined ? {} : { reasoningSummary: state.reasoningSummary }),
|
|
15
|
-
...(settings.safetyIdentifier === undefined
|
|
16
|
-
? {}
|
|
17
|
-
: { safetyIdentifier: settings.safetyIdentifier }),
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
/** No reply has been accepted yet, so the next request opens the conversation. */
|
|
21
|
-
function isFirstTurn(state) {
|
|
22
|
-
return state.replies === 0;
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* The items the next request adds to the conversation: the opening message with the first
|
|
26
|
-
* screen, or an output for each pending computer_call with the latest screenshot and any hint.
|
|
27
|
-
* Built once per turn, so every attempt at the request and the accepted reply see the same items.
|
|
28
|
-
*/
|
|
29
|
-
export function turnInputItems(state, req) {
|
|
30
|
-
if (isFirstTurn(state))
|
|
31
|
-
return [openingMessage(req.instructions, req.observation.screenshot)];
|
|
32
|
-
return [
|
|
33
|
-
...state.pendingCallIds.map((id) => buildCallOutput(id, req.observation.screenshot, req.acknowledgedSafetyChecks)),
|
|
34
|
-
...hintItems(req.contextHint),
|
|
35
|
-
];
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* The body of the next participant request: the opening request, or a continuation carrying
|
|
39
|
-
* `sent` (turnInputItems). A threaded continuation names the previous response; an
|
|
40
|
-
* explicit-context one carries the conversation ahead of `sent`.
|
|
41
|
-
*/
|
|
42
|
-
export function turnRequestBody(settings, state, req, sent) {
|
|
43
|
-
const ctx = requestContext(settings, state, req.instructions);
|
|
44
|
-
if (isFirstTurn(state))
|
|
45
|
-
return buildInitialRequest(ctx, req.observation.screenshot);
|
|
46
|
-
return buildContinuationRequest({
|
|
47
|
-
ctx,
|
|
48
|
-
previousResponseId: state.lastResponseId,
|
|
49
|
-
callOutputs: [...sent],
|
|
50
|
-
...(state.mode === "explicit_context"
|
|
51
|
-
? { explicitContextItems: state.conversation.carried() }
|
|
52
|
-
: {}),
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
|
-
/** The body of the read-only closing report: no tools, a small output limit, a strict schema. */
|
|
56
|
-
export function debriefRequestBody(settings, state, req, sent) {
|
|
57
|
-
return {
|
|
58
|
-
...turnRequestBody(settings, state, req, sent),
|
|
59
|
-
tool_choice: "none",
|
|
60
|
-
max_output_tokens: Math.min(settings.maxOutputTokens ?? CLOSING_OUTPUT_LIMIT, CLOSING_OUTPUT_LIMIT),
|
|
61
|
-
text: {
|
|
62
|
-
format: {
|
|
63
|
-
type: "json_schema",
|
|
64
|
-
name: "participant_closing_report",
|
|
65
|
-
strict: true,
|
|
66
|
-
schema: {
|
|
67
|
-
type: "object",
|
|
68
|
-
additionalProperties: false,
|
|
69
|
-
required: ["summary", "frictionReports"],
|
|
70
|
-
properties: {
|
|
71
|
-
summary: { type: "string" },
|
|
72
|
-
frictionReports: { type: "array", items: { type: "string" } },
|
|
73
|
-
},
|
|
74
|
-
},
|
|
75
|
-
},
|
|
76
|
-
},
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* Move the conversation on to an accepted reply to a request that added `sent`. A participant
|
|
81
|
-
* reply cut off by the output limit is set aside, so the next request is the same one again
|
|
82
|
-
* (outputLimitRetry); its actions never run, so no call output is owed for them.
|
|
83
|
-
*/
|
|
84
|
-
export function acceptReply(state, parsed, closing, sent) {
|
|
85
|
-
if (!closing && parsed.turn.interruption === "output_limit")
|
|
86
|
-
return;
|
|
87
|
-
if (parsed.turn.responseId !== undefined)
|
|
88
|
-
state.lastResponseId = parsed.turn.responseId;
|
|
89
|
-
state.pendingCallIds = parsed.callIds;
|
|
90
|
-
state.replies += 1;
|
|
91
|
-
state.conversation.accept(state.replies, sent, parsed.outputItems);
|
|
92
|
-
}
|
|
93
|
-
/** The trace's account of how a session carried its conversation (ActorConversation). */
|
|
94
|
-
export class ConversationRecord {
|
|
95
|
-
now;
|
|
96
|
-
explicitReason;
|
|
97
|
-
rejection;
|
|
98
|
-
switchedAt;
|
|
99
|
-
switchedAtRequest;
|
|
100
|
-
summarizedTurns = 0;
|
|
101
|
-
requests = [];
|
|
102
|
-
constructor(configured, now) {
|
|
103
|
-
this.now = now;
|
|
104
|
-
if (configured)
|
|
105
|
-
this.explicitReason = "configured";
|
|
106
|
-
}
|
|
107
|
-
/** The organization rejected server-side state, so the request being sent is explicit_context. */
|
|
108
|
-
switched(rejection) {
|
|
109
|
-
this.explicitReason = "zdr_rejection";
|
|
110
|
-
this.rejection = rejection;
|
|
111
|
-
this.switchedAt = new Date(this.now()).toISOString();
|
|
112
|
-
// Counted like requests[], which includes replies set aside at their output limit.
|
|
113
|
-
this.switchedAtRequest = this.requests.length + 1;
|
|
114
|
-
}
|
|
115
|
-
/** A participant request just went out with `sent` after the carried conversation. */
|
|
116
|
-
requested(state, sent) {
|
|
117
|
-
if (state.mode !== "explicit_context") {
|
|
118
|
-
this.requests.push({ mode: "threaded" });
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
121
|
-
const carried = state.conversation.size();
|
|
122
|
-
// Only summaries a request carried count; the conversation also trims after the last reply.
|
|
123
|
-
this.summarizedTurns = carried.notedTurns;
|
|
124
|
-
this.requests.push({
|
|
125
|
-
mode: "explicit_context",
|
|
126
|
-
carriedExchanges: carried.exchanges,
|
|
127
|
-
carriedScreenshots: carried.screenshots,
|
|
128
|
-
estimatedInputTokens: carried.estimatedTokens + estimateTokens(sent),
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
snapshot(state) {
|
|
132
|
-
return {
|
|
133
|
-
mode: state.mode === "explicit_context" ? "explicit_context" : "threaded",
|
|
134
|
-
...(this.explicitReason === undefined ? {} : { explicitReason: this.explicitReason }),
|
|
135
|
-
...(this.rejection === undefined ? {} : { rejection: this.rejection }),
|
|
136
|
-
...(this.switchedAt === undefined ? {} : { switchedAt: this.switchedAt }),
|
|
137
|
-
...(this.switchedAtRequest === undefined
|
|
138
|
-
? {}
|
|
139
|
-
: { switchedAtRequest: this.switchedAtRequest }),
|
|
140
|
-
summarizedTurns: this.summarizedTurns,
|
|
141
|
-
requests: this.requests.map((request) => ({ ...request })),
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
}
|