humanish 0.113.0 → 0.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/CONTEXT.md +1 -0
- package/README.md +1 -1
- package/dist/actors/closing-report-limits.d.ts +11 -0
- package/dist/actors/closing-report-limits.js +13 -0
- package/dist/actors/closing-report.d.ts +68 -0
- package/dist/actors/closing-report.js +55 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +4 -47
- package/dist/actors/codex/restricted-participant-policy.js +12 -40
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -10
- package/dist/actors/computer-use/loop/debrief.js +10 -46
- package/dist/actors/computer-use/loop/impressions.d.ts +2 -2
- package/dist/actors/computer-use/loop/impressions.js +6 -1
- package/dist/actors/computer-use/loop.d.ts +0 -1
- package/dist/actors/computer-use/loop.js +3 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +1 -1
- package/dist/actors/computer-use/openai-conversation.js +12 -36
- package/dist/actors/computer-use/openai-provider.js +11 -5
- package/dist/actors/contract.d.ts +6 -22
- package/dist/actors/contract.js +0 -13
- package/dist/analysis/admission.d.ts +47 -0
- package/dist/analysis/admission.js +65 -0
- package/dist/analysis/automatic-completion.d.ts +0 -6
- package/dist/analysis/automatic-completion.js +3 -3
- package/dist/analysis/automatic-config.d.ts +10 -2
- package/dist/analysis/automatic-config.js +29 -14
- package/dist/analysis/automatic.js +7 -0
- package/dist/analysis/execute.d.ts +24 -5
- package/dist/analysis/execute.js +61 -27
- package/dist/analysis/finding-lead.d.ts +27 -0
- package/dist/analysis/finding-lead.js +23 -0
- package/dist/analysis/image-tokens.d.ts +2 -0
- package/dist/analysis/image-tokens.js +1 -1
- package/dist/analysis/job.d.ts +9 -1
- package/dist/analysis/job.js +11 -2
- package/dist/analysis/service.js +8 -3
- package/dist/cli/commands/analyze.js +7 -2
- package/dist/cli/commands/notes.d.ts +3 -0
- package/dist/cli/commands/notes.js +145 -0
- package/dist/cli/commands/observe.js +1 -1
- package/dist/cli/commands/study-run.js +2 -1
- package/dist/cli/findings.js +11 -30
- package/dist/cli/io.d.ts +1 -0
- package/dist/cli/io.js +8 -12
- package/dist/cli/program.js +2 -0
- package/dist/cli/telemetry.js +10 -22
- package/dist/cli/update-check.js +6 -18
- package/dist/cli/user-config.d.ts +11 -0
- package/dist/cli/user-config.js +25 -0
- package/dist/comms/email-catch.js +7 -31
- package/dist/evidence/encoded-text.d.ts +17 -3
- package/dist/evidence/encoded-text.js +121 -46
- package/dist/evidence/redaction.d.ts +11 -0
- package/dist/evidence/redaction.js +21 -10
- package/dist/evidence/secret-scrub.d.ts +14 -1
- package/dist/evidence/secret-scrub.js +125 -28
- package/dist/feedback/draft.d.ts +19 -0
- package/dist/feedback/draft.js +62 -15
- package/dist/feedback/export-bundle.js +3 -3
- package/dist/feedback/export.js +50 -20
- package/dist/feedback/feedback.js +29 -2
- package/dist/keys/key-resolution.js +12 -17
- package/dist/observer/artifact.d.ts +5 -0
- package/dist/observer/artifact.js +18 -2
- package/dist/observer/http.d.ts +7 -0
- package/dist/observer/http.js +19 -1
- package/dist/observer/notes-route.d.ts +20 -0
- package/dist/observer/notes-route.js +120 -0
- package/dist/observer/pinned-files.d.ts +8 -1
- package/dist/observer/pinned-files.js +17 -3
- package/dist/observer/render.js +39 -14
- package/dist/observer/run-routes.d.ts +8 -2
- package/dist/observer/run-routes.js +32 -6
- package/dist/observer/serve.d.ts +3 -0
- package/dist/observer/serve.js +29 -3
- package/dist/observer-app.html +7 -6
- package/dist/run/contained-output.d.ts +8 -0
- package/dist/run/contained-output.js +27 -0
- package/dist/run/evidence-files.d.ts +7 -0
- package/dist/run/evidence-files.js +24 -14
- package/dist/run/note-files.d.ts +54 -0
- package/dist/run/note-files.js +212 -0
- package/dist/run/note-shape.d.ts +35 -0
- package/dist/run/note-shape.js +20 -0
- package/dist/run/notes.d.ts +34 -0
- package/dist/run/notes.js +192 -0
- package/dist/run/participant-caption.d.ts +9 -0
- package/dist/run/participant-caption.js +19 -0
- package/dist/run/participant-records.d.ts +2 -0
- package/dist/run/participant-records.js +17 -2
- package/dist/run/transient-comms-secrets.d.ts +6 -5
- package/dist/run/transient-comms-secrets.js +19 -24
- package/dist/study/doctor.js +1 -1
- package/dist/study/preflight-result.js +2 -1
- package/dist/study/summary.d.ts +4 -5
- package/dist/study/summary.js +6 -4
- package/dist/study/warnings.js +2 -1
- package/dist/tui-app.js +1 -1
- package/docs/architecture/actor-contract.md +3 -3
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/architecture/state-driven-executor.md +1 -1
- package/docs/contracts/errors.md +2 -2
- package/docs/contracts/run-bundle.md +61 -0
- package/docs/contracts/schemas.md +17 -14
- package/docs/contracts/study-analysis.md +35 -9
- package/docs/principles/invariants-and-defaults.md +2 -2
- package/docs/product/automatic-analysis.md +22 -16
- package/package.json +8 -7
- package/skills/humanish/SKILL.md +6 -2
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
// run's warnings. A value is found as written and in the encoded forms an HTTP client, a log line or
|
|
3
3
|
// a JSON body gives it, so a token that reaches the text percent-encoded or base64-encoded is still
|
|
4
4
|
// removed. Pattern redaction (redactText) runs after this and finds secrets by shape.
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const
|
|
5
|
+
import { escapeRegExp } from "../run/text.js";
|
|
6
|
+
import { decodeEscapes, latin1RunsAsUtf8, readingsOf } from "./encoded-text.js";
|
|
7
|
+
import { REDACTION_MARKERS } from "./redaction.js";
|
|
8
|
+
const REDACTED = REDACTION_MARKERS.secret;
|
|
9
|
+
// A form wholly inside a marker humanish writes is the marker's own text, as when a value is part
|
|
10
|
+
// of the marker's name. Other bracketed text, such as `[REDACTED_373433393231]`, is searched like
|
|
11
|
+
// any text.
|
|
12
|
+
const MARKER = new RegExp(Object.values(REDACTION_MARKERS).map(escapeRegExp).join("|"), "g");
|
|
9
13
|
// The shortest encoded form searched for. Shorter base64 or hex runs are ordinary text often enough
|
|
10
14
|
// that matching them would redact words. A value itself is searched for at any length.
|
|
11
15
|
const MIN_ENCODED_FORM = 8;
|
|
@@ -27,7 +31,9 @@ function base64Middles(bytes, encoding) {
|
|
|
27
31
|
}
|
|
28
32
|
/**
|
|
29
33
|
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
30
|
-
* `\u` escapes) and twice,
|
|
34
|
+
* `\u` escapes) and twice, its UTF-8 bytes read one per character and the reverse, base64 at each
|
|
35
|
+
* byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as
|
|
36
|
+
* a string. An
|
|
31
37
|
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
32
38
|
* is not ordinary text.
|
|
33
39
|
*/
|
|
@@ -52,33 +58,122 @@ export function encodedForms(value) {
|
|
|
52
58
|
...base64Middles(bytes, "base64"),
|
|
53
59
|
...base64Middles(bytes, "base64url"),
|
|
54
60
|
];
|
|
61
|
+
// decodeEscapes reads each percent escape as one character, so a percent-encoded non-ASCII value
|
|
62
|
+
// reads as its UTF-8 bytes one per character, é as é.
|
|
63
|
+
const bytewise = bytes.toString("latin1");
|
|
64
|
+
if (bytewise !== value)
|
|
65
|
+
escaped.push(bytewise);
|
|
66
|
+
// The mirror: a value such as xéz€ whose characters include bytes that read as UTF-8, written
|
|
67
|
+
// with them read (xéz€). No reading of the text turns xéz€ back into the value.
|
|
68
|
+
const mirrored = latin1RunsAsUtf8(value);
|
|
69
|
+
if (mirrored !== value)
|
|
70
|
+
escaped.push(mirrored);
|
|
55
71
|
return [value, ...escaped, ...binary.filter((form) => form.length >= MIN_ENCODED_FORM)];
|
|
56
72
|
}
|
|
57
|
-
/**
|
|
58
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Whether [at, end) lies wholly inside one marker. The markers are one regex's matches, so they are
|
|
75
|
+
* sorted and disjoint: only the first marker that ends after `at` can hold it.
|
|
76
|
+
*/
|
|
77
|
+
function insideMarker(markers, at, end) {
|
|
78
|
+
let low = 0;
|
|
79
|
+
let high = markers.length;
|
|
80
|
+
while (low < high) {
|
|
81
|
+
const middle = (low + high) >> 1;
|
|
82
|
+
if (markers[middle][1] > at)
|
|
83
|
+
high = middle;
|
|
84
|
+
else
|
|
85
|
+
low = middle + 1;
|
|
86
|
+
}
|
|
87
|
+
return low < markers.length && markers[low][0] <= at && end <= markers[low][1];
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The Knuth-Morris-Pratt failure table: entry `k` is the length of the longest proper prefix of
|
|
91
|
+
* the form's first `k` characters that is also their suffix.
|
|
92
|
+
*/
|
|
93
|
+
function failureTable(form) {
|
|
94
|
+
const table = new Int32Array(form.length + 1);
|
|
95
|
+
for (let at = 1, length = 0; at < form.length; at += 1) {
|
|
96
|
+
while (length > 0 && form[at] !== form[length])
|
|
97
|
+
length = table[length];
|
|
98
|
+
if (form[at] === form[length])
|
|
99
|
+
length += 1;
|
|
100
|
+
table[at + 1] = length;
|
|
101
|
+
}
|
|
102
|
+
return table;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Calls `found` with the start of every occurrence of the form, overlapping ones included, in time
|
|
106
|
+
* linear in the text. indexOf skips to the next occurrence. After one, the failure table carries
|
|
107
|
+
* the match on through the occurrences that overlap it, so no character is compared again, until
|
|
108
|
+
* no prefix of the form is left matched and indexOf takes over.
|
|
109
|
+
*/
|
|
110
|
+
function eachOccurrence(text, form, found) {
|
|
111
|
+
let table;
|
|
112
|
+
for (let at = text.indexOf(form); at !== -1;) {
|
|
113
|
+
found(at);
|
|
114
|
+
table ??= failureTable(form);
|
|
115
|
+
let matched = table[form.length];
|
|
116
|
+
let next = at + form.length;
|
|
117
|
+
for (; matched > 0 && next < text.length; next += 1) {
|
|
118
|
+
while (matched > 0 && text[next] !== form[matched])
|
|
119
|
+
matched = table[matched];
|
|
120
|
+
if (text[next] === form[matched])
|
|
121
|
+
matched += 1;
|
|
122
|
+
if (matched === form.length) {
|
|
123
|
+
found(next + 1 - form.length);
|
|
124
|
+
matched = table[form.length];
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
at = matched > 0 ? -1 : text.indexOf(form, next);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/** Every occurrence of every form not wholly inside a marker, in no order. */
|
|
131
|
+
function occurrences(text, forms) {
|
|
59
132
|
const markers = [];
|
|
60
133
|
for (const match of text.matchAll(MARKER)) {
|
|
61
134
|
const start = match.index ?? 0;
|
|
62
135
|
markers.push([start, start + match[0].length]);
|
|
63
136
|
}
|
|
137
|
+
const found = [];
|
|
138
|
+
for (const form of forms)
|
|
139
|
+
eachOccurrence(text, form, (at) => {
|
|
140
|
+
if (!insideMarker(markers, at, at + form.length))
|
|
141
|
+
found.push([at, at + form.length]);
|
|
142
|
+
});
|
|
143
|
+
return found;
|
|
144
|
+
}
|
|
145
|
+
/** The occurrences sorted and merged where they overlap or touch. */
|
|
146
|
+
function merged(found) {
|
|
147
|
+
found.sort((left, right) => left[0] - right[0]);
|
|
64
148
|
const spans = [];
|
|
65
|
-
for (const
|
|
66
|
-
|
|
67
|
-
const end = at + form.length;
|
|
68
|
-
if (!markers.some(([start, stop]) => at < stop && end > start))
|
|
69
|
-
spans.push([at, end]);
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
spans.sort((left, right) => left[0] - right[0]);
|
|
73
|
-
const merged = [];
|
|
74
|
-
for (const span of spans) {
|
|
75
|
-
const last = merged.at(-1);
|
|
149
|
+
for (const span of found) {
|
|
150
|
+
const last = spans.at(-1);
|
|
76
151
|
if (last !== undefined && span[0] <= last[1])
|
|
77
152
|
last[1] = Math.max(last[1], span[1]);
|
|
78
153
|
else
|
|
79
|
-
|
|
154
|
+
spans.push([span[0], span[1]]);
|
|
80
155
|
}
|
|
81
|
-
return
|
|
156
|
+
return spans;
|
|
157
|
+
}
|
|
158
|
+
function replaceSpans(text, spans) {
|
|
159
|
+
let result = "";
|
|
160
|
+
let cursor = 0;
|
|
161
|
+
for (const [start, end] of spans) {
|
|
162
|
+
result += text.slice(cursor, start) + REDACTED;
|
|
163
|
+
cursor = end;
|
|
164
|
+
}
|
|
165
|
+
return result + text.slice(cursor);
|
|
166
|
+
}
|
|
167
|
+
const formsOf = (values) => [
|
|
168
|
+
...new Set(values.filter((value) => value.length > 0).flatMap(encodedForms)),
|
|
169
|
+
];
|
|
170
|
+
/**
|
|
171
|
+
* A check that the text holds one of the values' forms outside the markers in any of its
|
|
172
|
+
* readingsOf. A scrub's output must pass it.
|
|
173
|
+
*/
|
|
174
|
+
export function holdsSecretValue(values) {
|
|
175
|
+
const forms = formsOf(values);
|
|
176
|
+
return (text) => readingsOf(text).some((reading) => occurrences(reading, forms).length > 0);
|
|
82
177
|
}
|
|
83
178
|
/**
|
|
84
179
|
* A scrub that replaces each non-empty value and its encoded forms with `[REDACTED_SECRET]`. Every
|
|
@@ -87,19 +182,21 @@ function secretSpans(text, forms) {
|
|
|
87
182
|
* through decodeEscapes, the decoder verify uses, and returns the decoded text: a warning with
|
|
88
183
|
* percent-encoding, JSON escapes or HTML references shows them decoded, with the values removed.
|
|
89
184
|
* That finds a value written partly encoded, which no single encoded form matches.
|
|
185
|
+
*
|
|
186
|
+
* The result's other readingsOf only detect. If one still holds a form, the scrub returns
|
|
187
|
+
* `[REDACTED_SECRET]` in place of the whole text. That happens for a value encoded twice, which
|
|
188
|
+
* the decoded text shows encoded once; for a value the UTF-8 or transfer reading holds, such as
|
|
189
|
+
* one with one of two non-ASCII characters percent-encoded; and for a value that holds part of a
|
|
190
|
+
* marker, such as `ET]x`, which the marker that replaced it can spell again.
|
|
90
191
|
*/
|
|
91
192
|
export function scrubSecretValues(values) {
|
|
92
|
-
const forms =
|
|
193
|
+
const forms = formsOf(values);
|
|
93
194
|
if (forms.length === 0)
|
|
94
195
|
return (text) => text;
|
|
196
|
+
const holds = holdsSecretValue(values);
|
|
95
197
|
return (text) => {
|
|
96
198
|
const decoded = decodeEscapes(text);
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
for (const [start, end] of secretSpans(decoded, forms)) {
|
|
100
|
-
result += decoded.slice(cursor, start) + REDACTED;
|
|
101
|
-
cursor = end;
|
|
102
|
-
}
|
|
103
|
-
return result + decoded.slice(cursor);
|
|
199
|
+
const result = replaceSpans(decoded, merged(occurrences(decoded, forms)));
|
|
200
|
+
return holds(result) ? REDACTED : result;
|
|
104
201
|
};
|
|
105
202
|
}
|
package/dist/feedback/draft.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { RunBundle, RunFeedbackCandidate } from "../run/bundle.js";
|
|
2
2
|
import type { PreparedRunArtifactPaths } from "../run/paths.js";
|
|
3
3
|
import { loadRunBundlePrepared } from "../run/locate.js";
|
|
4
|
+
import { type RunNotes } from "../run/note-files.js";
|
|
4
5
|
export declare const FEEDBACK_SCHEMA = "humanish.feedback.v1";
|
|
5
6
|
export interface FeedbackDraft {
|
|
6
7
|
schema: typeof FEEDBACK_SCHEMA;
|
|
@@ -35,6 +36,19 @@ export interface FeedbackDraft {
|
|
|
35
36
|
idempotency_key: string;
|
|
36
37
|
proposed_next_state: RunFeedbackCandidate["proposed_next_state"];
|
|
37
38
|
acceptance_proof: string[];
|
|
39
|
+
/** Notes a person added while reviewing the recording, in run clock order. Absent with none. */
|
|
40
|
+
reviewer_notes?: Array<{
|
|
41
|
+
id: string;
|
|
42
|
+
/** The moment on the run clock, as the Observer shows it: `02:31`. */
|
|
43
|
+
at: string;
|
|
44
|
+
at_ms: number;
|
|
45
|
+
/** The stream id the note file records, or null for a note on the whole run. */
|
|
46
|
+
participant: string | null;
|
|
47
|
+
/** The participant's caption, as the Observer names it. */
|
|
48
|
+
participant_caption: string | null;
|
|
49
|
+
author: string;
|
|
50
|
+
text: string;
|
|
51
|
+
}>;
|
|
38
52
|
}
|
|
39
53
|
export interface FeedbackDraftOptions {
|
|
40
54
|
/** A candidate id from `feedback list`. Absent: the first usable candidate, as before. */
|
|
@@ -53,6 +67,11 @@ export interface FeedbackRunContext {
|
|
|
53
67
|
}
|
|
54
68
|
export declare function buildDraft(bundle: RunBundle, bundlePath: string, candidateId?: string): FeedbackDraft;
|
|
55
69
|
export declare function buildAnalysisDraft(context: FeedbackRunContext, options: FeedbackDraftOptions): Promise<FeedbackDraft | null>;
|
|
70
|
+
/**
|
|
71
|
+
* The draft with the run's reviewer notes and each note's file cited. The caller passes the set it
|
|
72
|
+
* read and checked for sharing (readNotesForSharing).
|
|
73
|
+
*/
|
|
74
|
+
export declare function withReviewerNotes(draft: FeedbackDraft, notes: RunNotes | null, bundle: RunBundle, runRoot: string): FeedbackDraft;
|
|
56
75
|
export declare function isUsableFeedbackCandidate(candidate: unknown): candidate is RunFeedbackCandidate;
|
|
57
76
|
export declare function renderMarkdown(draft: FeedbackDraft, repo: string): string;
|
|
58
77
|
export {};
|
package/dist/feedback/draft.js
CHANGED
|
@@ -2,12 +2,17 @@
|
|
|
2
2
|
// participant's feedback candidate or from a reviewed analysis finding, and its markdown rendering.
|
|
3
3
|
// The commands that resolve the run and read or write drafts are in feedback.ts.
|
|
4
4
|
import path from "node:path";
|
|
5
|
+
import { findingLead } from "../analysis/finding-lead.js";
|
|
5
6
|
import { analysisSharingProblems } from "../analysis/sharing.js";
|
|
6
7
|
import { loadAnalysis } from "../analysis/load.js";
|
|
7
8
|
import { hashAnalysisValue } from "../analysis/validation.js";
|
|
8
9
|
import { formatParticipantOutcomes, formatRunTaskFunnel, participantOutcomeDetails, withCuaReviewProvenance, } from "../run/outcomes.js";
|
|
9
10
|
import { loadRunBundlePrepared } from "../run/locate.js";
|
|
10
11
|
import { isFeedbackIdempotencyKey } from "../run/feedback-shape.js";
|
|
12
|
+
import { runNoteFile } from "../run/note-files.js";
|
|
13
|
+
import { byRunTime } from "../run/note-shape.js";
|
|
14
|
+
import { formatRunTime } from "../run/notes.js";
|
|
15
|
+
import { runParticipantCaptions } from "../run/participant-records.js";
|
|
11
16
|
import { isRecord } from "../run/type-guards.js";
|
|
12
17
|
import { feedbackProofCommands, projectFeedbackAcceptanceProof } from "./proof.js";
|
|
13
18
|
export const FEEDBACK_SCHEMA = "humanish.feedback.v1";
|
|
@@ -120,8 +125,10 @@ export async function buildAnalysisDraft(context, options) {
|
|
|
120
125
|
const root = path.dirname(context.loaded.bundlePath);
|
|
121
126
|
const evidenceIds = new Set(finding.observations.flatMap((item) => item.evidenceIds));
|
|
122
127
|
const evidence = analysis.evidence.filter((item) => evidenceIds.has(item.id));
|
|
123
|
-
const
|
|
124
|
-
|
|
128
|
+
const lead = findingLead(finding, correction);
|
|
129
|
+
// An analysis written before headlines has no lead, so the amended claim or the title names it.
|
|
130
|
+
const amendedClaim = correction?.status === "amended" ? correction.replacementClaim : null;
|
|
131
|
+
const claim = lead?.headline ?? amendedClaim ?? finding.title;
|
|
125
132
|
const firstLine = claim.trim().split(/\r?\n/)[0] || `Reviewed finding ${finding.id}`;
|
|
126
133
|
const summary = Array.from(firstLine).length > 160
|
|
127
134
|
? Array.from(firstLine).slice(0, 159).join("") + "…"
|
|
@@ -145,19 +152,11 @@ export async function buildAnalysisDraft(context, options) {
|
|
|
145
152
|
summary,
|
|
146
153
|
expected: "Review the cited behavior against the participant assignment and confirm the expected product behavior.",
|
|
147
154
|
actual: [
|
|
148
|
-
|
|
149
|
-
...(finding.headline === undefined
|
|
150
|
-
? []
|
|
151
|
-
: amended
|
|
152
|
-
? [
|
|
153
|
-
"This finding was corrected in human review. The reviewer's claim above replaces its original headline and account.",
|
|
154
|
-
"",
|
|
155
|
-
]
|
|
156
|
-
: finding.experience === undefined
|
|
157
|
-
? []
|
|
158
|
-
: [finding.experience, ""]),
|
|
155
|
+
...(lead?.account ? [lead.account, ""] : []),
|
|
159
156
|
"Independent study analysis; does not replace participant feedback or recorded completion outcomes.",
|
|
160
|
-
|
|
157
|
+
amendedClaim === null
|
|
158
|
+
? finding.summary
|
|
159
|
+
: `Amended claim: ${amendedClaim}. Original analysis: ${finding.summary}`,
|
|
161
160
|
`Impact: ${finding.impact}. ${finding.affectedStreamIds.length} affected / ${finding.exposedStreamIds.length} observed exposed participants. ${finding.exposureReason}`,
|
|
162
161
|
`Recovery: ${finding.recovery}. Confidence: ${finding.confidence}.`,
|
|
163
162
|
...finding.observations.map((item) => `${item.basis}: ${item.claim}${item.limitation ? ` Limitation: ${item.limitation}` : ""}`),
|
|
@@ -224,6 +223,54 @@ function citedImpressions(bundle, evidence) {
|
|
|
224
223
|
: [{ streamId: item.streamId, kind: impression.kind, text: impression.text }];
|
|
225
224
|
});
|
|
226
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* The draft with the run's reviewer notes and each note's file cited. The caller passes the set it
|
|
228
|
+
* read and checked for sharing (readNotesForSharing).
|
|
229
|
+
*/
|
|
230
|
+
export function withReviewerNotes(draft, notes, bundle, runRoot) {
|
|
231
|
+
if (notes === null || notes.notes.length === 0)
|
|
232
|
+
return draft;
|
|
233
|
+
const captions = runParticipantCaptions(bundle);
|
|
234
|
+
const ordered = byRunTime(notes.notes);
|
|
235
|
+
return {
|
|
236
|
+
...draft,
|
|
237
|
+
evidence: [
|
|
238
|
+
...draft.evidence,
|
|
239
|
+
...ordered.map((note) => ({
|
|
240
|
+
path: path.join(runRoot, runNoteFile(note.id)),
|
|
241
|
+
kind: "review",
|
|
242
|
+
note: `Reviewer note at ${formatRunTime(note.atMs)}, added while reviewing the recording.`,
|
|
243
|
+
})),
|
|
244
|
+
],
|
|
245
|
+
reviewer_notes: ordered.map((note) => ({
|
|
246
|
+
id: note.id,
|
|
247
|
+
at: formatRunTime(note.atMs),
|
|
248
|
+
at_ms: note.atMs,
|
|
249
|
+
participant: note.participant,
|
|
250
|
+
participant_caption: note.participant === null ? null : (captions.get(note.participant) ?? note.participant),
|
|
251
|
+
author: note.author,
|
|
252
|
+
text: note.text,
|
|
253
|
+
})),
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
function reviewerNotesSection(draft) {
|
|
257
|
+
if (!draft.reviewer_notes?.length)
|
|
258
|
+
return "";
|
|
259
|
+
const items = draft.reviewer_notes.map((note) => {
|
|
260
|
+
const [first = "", ...rest] = note.text.split("\n");
|
|
261
|
+
return [
|
|
262
|
+
`- ${note.at}, ${note.participant_caption ?? "whole run"}, ${note.author}: ${first}`,
|
|
263
|
+
...rest.map((line) => ` ${line}`),
|
|
264
|
+
].join("\n");
|
|
265
|
+
});
|
|
266
|
+
return `## Reviewer notes
|
|
267
|
+
|
|
268
|
+
Notes a person added while reviewing the recording. They are not participant feedback.
|
|
269
|
+
|
|
270
|
+
${items.join("\n")}
|
|
271
|
+
|
|
272
|
+
`;
|
|
273
|
+
}
|
|
227
274
|
export function isUsableFeedbackCandidate(candidate) {
|
|
228
275
|
if (!isRecord(candidate) ||
|
|
229
276
|
candidate.schema !== "humanish.feedback-candidate.v1" ||
|
|
@@ -262,7 +309,7 @@ ${draft.expected}
|
|
|
262
309
|
|
|
263
310
|
${draft.actual}
|
|
264
311
|
|
|
265
|
-
## Evidence
|
|
312
|
+
${reviewerNotesSection(draft)}## Evidence
|
|
266
313
|
|
|
267
314
|
${draft.evidence.map((item) => `- ${item.kind} ${runRelative(draft, item.path)}: ${item.note}`).join("\n")}
|
|
268
315
|
|
|
@@ -10,7 +10,7 @@ import { renderObserver } from "../observer/render.js";
|
|
|
10
10
|
import { buildObserverData } from "../observer/data.js";
|
|
11
11
|
import { inventorySandboxIds, scrubSandboxIdBytes } from "../run/sandbox-ids.js";
|
|
12
12
|
import { containsSensitive, redactSandboxIds, redactScreenshot, redactText, } from "../evidence/redaction.js";
|
|
13
|
-
import { decodeEscapes } from "../evidence/encoded-text.js";
|
|
13
|
+
import { decodeEscapes, readingsOf } from "../evidence/encoded-text.js";
|
|
14
14
|
import { readPlainText } from "../evidence/plain-text.js";
|
|
15
15
|
import { streamScreenshotPaths } from "../verify/artifacts.js";
|
|
16
16
|
import { verifyRunPrepared } from "../verify/verify.js";
|
|
@@ -148,9 +148,9 @@ async function prospectivePhysicalPath(requested) {
|
|
|
148
148
|
function assertNoInlineRaster(text) {
|
|
149
149
|
// Decode the escapes verify decodes (decodeEscapes) before checking. Unknown embedded bytes are
|
|
150
150
|
// refused: copying a text extension must not smuggle untransformed pixels.
|
|
151
|
-
|
|
152
|
-
if (containsSensitive(text) || containsSensitive(decoded))
|
|
151
|
+
if (readingsOf(text).some(containsSensitive))
|
|
153
152
|
throw new Error("Decoded text contains a secret-shaped value or private path; export refused.");
|
|
153
|
+
const decoded = decodeEscapes(text);
|
|
154
154
|
if (/data\s*:\s*(?:image\/|application\/|text\/|[^,\s]*;base64,|,)/i.test(decoded) ||
|
|
155
155
|
/iVBORw0KGgo|\/9j\/[A-Za-z0-9+/]|R0lGOD|UklGR[A-Za-z0-9+/]{4}|<svg\b/i.test(decoded)) {
|
|
156
156
|
throw new Error("Text contains an inline image/data payload; embedded raster redaction is unsupported.");
|
package/dist/feedback/export.js
CHANGED
|
@@ -19,6 +19,8 @@ import { verifyRun } from "../verify/verify.js";
|
|
|
19
19
|
import {} from "../run/bundle.js";
|
|
20
20
|
import { exportRedactedBundle } from "./export-bundle.js";
|
|
21
21
|
import { loadAnalysis } from "../analysis/load.js";
|
|
22
|
+
import { readNotesForSharing } from "../run/note-files.js";
|
|
23
|
+
import { cli } from "../cli/invocation.js";
|
|
22
24
|
import { analysisSharingProblems } from "../analysis/sharing.js";
|
|
23
25
|
import { EVIDENCE_LIMITS, validateAnalysisEvidence } from "../analysis/evidence.js";
|
|
24
26
|
import { readBoundedFile } from "../run/evidence-files.js";
|
|
@@ -96,8 +98,13 @@ export async function exportRun(cwdInput, runInput, options = {}, deps = {}) {
|
|
|
96
98
|
if (analysisCheck === "downgraded")
|
|
97
99
|
shareReady = false;
|
|
98
100
|
warnings.push(...analysis.warnings);
|
|
101
|
+
const notes = await checkedNotes(runPaths, verified, inlined, options, warnings);
|
|
102
|
+
if (notes.check === "blocked")
|
|
103
|
+
return exportFailure(cwd, runInput, "HUMANISH_EXPORT_SHARE_SAFETY_BLOCKED", `The run's reviewer notes hold text that looks like a secret, a token or a local path, which verify may not have seen if a note was added or changed after it ran. Run \`${cli(`verify --run ${runId}`)}\`, fix the note files in notes/, and export again.`, verified.shareSafety);
|
|
104
|
+
if (notes.check === "downgraded")
|
|
105
|
+
shareReady = false;
|
|
99
106
|
const watermarked = !shareReady || omittedRecording;
|
|
100
|
-
const output = renderExportHtml(inlined, analysis, assets, verified, watermarked, omittedRecording);
|
|
107
|
+
const output = renderExportHtml(inlined, { analysis, notes: notes.notes }, assets, verified, watermarked, omittedRecording);
|
|
101
108
|
const bytes = Buffer.byteLength(output, "utf8");
|
|
102
109
|
const maxBytes = options.maxBytes ?? DEFAULT_EXPORT_MAX_BYTES;
|
|
103
110
|
if (bytes > maxBytes) {
|
|
@@ -281,7 +288,10 @@ function makePortable(inlined, verified) {
|
|
|
281
288
|
}
|
|
282
289
|
}
|
|
283
290
|
}
|
|
284
|
-
|
|
291
|
+
recordShareSafety(inlined, verified);
|
|
292
|
+
}
|
|
293
|
+
/** What verify said, in the file, so the chrome can agree with the result envelope. */
|
|
294
|
+
function recordShareSafety(inlined, verified) {
|
|
285
295
|
const publicSafety = (inlined.publicSafety ?? {});
|
|
286
296
|
inlined.publicSafety = {
|
|
287
297
|
...publicSafety,
|
|
@@ -292,6 +302,23 @@ function makePortable(inlined, verified) {
|
|
|
292
302
|
},
|
|
293
303
|
};
|
|
294
304
|
}
|
|
305
|
+
/**
|
|
306
|
+
* Adds a failed sharing check to verify's result. The reason joins verify's reasons and the status
|
|
307
|
+
* is the worse of the two, as verify grades an analysis. The export is refused unless --local-only,
|
|
308
|
+
* which writes the new status into the file.
|
|
309
|
+
*/
|
|
310
|
+
function applySharingProblem(verified, inlined, options, problem) {
|
|
311
|
+
verified.shareSafety = {
|
|
312
|
+
status: problem.status === "blocked" || verified.shareSafety.status === "blocked"
|
|
313
|
+
? "blocked"
|
|
314
|
+
: "local_only",
|
|
315
|
+
reasons: [...verified.shareSafety.reasons, problem.reason],
|
|
316
|
+
};
|
|
317
|
+
if (options.localOnly !== true)
|
|
318
|
+
return "blocked";
|
|
319
|
+
recordShareSafety(inlined, verified);
|
|
320
|
+
return "downgraded";
|
|
321
|
+
}
|
|
295
322
|
/**
|
|
296
323
|
* Revalidates the analysis against source evidence and its sharing checks. A failed check
|
|
297
324
|
* downgrades share safety, which only --local-only may export.
|
|
@@ -315,30 +342,33 @@ async function recheckAnalysis(runPaths, analysis, verified, inlined, options) {
|
|
|
315
342
|
const analysisSharing = analysisSharingProblems(analysis);
|
|
316
343
|
if (!analysisSharing.sensitive && !analysisSharing.unverified)
|
|
317
344
|
return "ok";
|
|
318
|
-
verified
|
|
345
|
+
return applySharingProblem(verified, inlined, options, {
|
|
319
346
|
status: analysisSharing.sensitive ? "blocked" : "local_only",
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
347
|
+
reason: {
|
|
348
|
+
code: "ANALYSIS_UNVERIFIED",
|
|
349
|
+
message: "The exact analysis snapshot being exported did not pass sharing checks.",
|
|
350
|
+
},
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* The run's notes for the export, read once within the notes size limit. That snapshot is scanned
|
|
355
|
+
* as verify scans run text and is the one rendered, so a note added or changed after verify ran cannot
|
|
356
|
+
* reach the file unchecked. A finding blocks the export, or downgrades it under --local-only.
|
|
357
|
+
*/
|
|
358
|
+
async function checkedNotes(runPaths, verified, inlined, options, warnings) {
|
|
359
|
+
const { notes, problem } = await readNotesForSharing(runPaths);
|
|
360
|
+
for (const skipped of notes.skipped)
|
|
361
|
+
warnings.push(`Reviewer notes: ${skipped}`);
|
|
362
|
+
return {
|
|
363
|
+
notes,
|
|
364
|
+
check: problem === null ? "ok" : applySharingProblem(verified, inlined, options, problem),
|
|
334
365
|
};
|
|
335
|
-
return "downgraded";
|
|
336
366
|
}
|
|
337
367
|
/** The portable Observer HTML, with the local-only banner when the export is watermarked. */
|
|
338
|
-
function renderExportHtml(inlined,
|
|
368
|
+
function renderExportHtml(inlined, review, assets, verified, watermarked, omittedRecording) {
|
|
339
369
|
const output = renderObserverHtml(inlined, {
|
|
340
370
|
snapshot: true,
|
|
341
|
-
|
|
371
|
+
...review,
|
|
342
372
|
assets,
|
|
343
373
|
});
|
|
344
374
|
if (!watermarked)
|
|
@@ -6,8 +6,10 @@ import path from "node:path";
|
|
|
6
6
|
import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveLatestRunDirectory, validatePreparedRunArtifactPaths, } from "../run/paths.js";
|
|
7
7
|
import { bindExistingManagedHumanishOutputDirectory, readContainedRegularFile, writeContainedOutputFile, } from "../run/contained-output.js";
|
|
8
8
|
import { loadRunBundlePrepared } from "../run/locate.js";
|
|
9
|
+
import { readNotesForSharing } from "../run/note-files.js";
|
|
10
|
+
import { cli } from "../cli/invocation.js";
|
|
9
11
|
import { verifyRunPrepared } from "../verify/verify.js";
|
|
10
|
-
import { buildAnalysisDraft, buildDraft, isUsableFeedbackCandidate, renderMarkdown, } from "./draft.js";
|
|
12
|
+
import { buildAnalysisDraft, buildDraft, isUsableFeedbackCandidate, renderMarkdown, withReviewerNotes, } from "./draft.js";
|
|
11
13
|
const FEEDBACK_RESULT_SCHEMA = "humanish.feedback-result.v1";
|
|
12
14
|
function summarizeCandidates(bundle) {
|
|
13
15
|
return bundle.feedbackCandidates
|
|
@@ -90,6 +92,14 @@ async function draftFeedbackBound(cwdInput, runInput, options = {}) {
|
|
|
90
92
|
},
|
|
91
93
|
};
|
|
92
94
|
}
|
|
95
|
+
// The notes the draft includes are this set, checked here: a note can be added or changed after
|
|
96
|
+
// verify read the run.
|
|
97
|
+
const shared = await readNotesForSharing(context.preparedRunPaths);
|
|
98
|
+
if (shared.problem !== null)
|
|
99
|
+
return {
|
|
100
|
+
context,
|
|
101
|
+
result: notesRefusal(cwd, runInput, context.storedRunId, verified, shared.problem),
|
|
102
|
+
};
|
|
93
103
|
const candidates = summarizeCandidates(context.loaded.bundle);
|
|
94
104
|
if (options.candidate !== undefined &&
|
|
95
105
|
!candidates.some((item) => item.id === options.candidate)) {
|
|
@@ -111,9 +121,11 @@ async function draftFeedbackBound(cwdInput, runInput, options = {}) {
|
|
|
111
121
|
};
|
|
112
122
|
}
|
|
113
123
|
const independent = options.analysis !== undefined || options.finding !== undefined;
|
|
114
|
-
const
|
|
124
|
+
const built = independent
|
|
115
125
|
? await buildAnalysisDraft(context, options)
|
|
116
126
|
: buildDraft(context.loaded.bundle, context.loaded.bundlePath, options.candidate);
|
|
127
|
+
const draft = built &&
|
|
128
|
+
withReviewerNotes(built, shared.notes, context.loaded.bundle, path.dirname(context.loaded.bundlePath));
|
|
117
129
|
if (!draft)
|
|
118
130
|
return {
|
|
119
131
|
context,
|
|
@@ -143,6 +155,21 @@ async function draftFeedbackBound(cwdInput, runInput, options = {}) {
|
|
|
143
155
|
},
|
|
144
156
|
};
|
|
145
157
|
}
|
|
158
|
+
/** The refusal of a draft whose reviewer notes, as read for it, hold what verify's scan flags. */
|
|
159
|
+
function notesRefusal(cwd, runInput, runId, verified, problem) {
|
|
160
|
+
return {
|
|
161
|
+
schema: FEEDBACK_RESULT_SCHEMA,
|
|
162
|
+
ok: false,
|
|
163
|
+
cwd,
|
|
164
|
+
run: runInput,
|
|
165
|
+
// A draft is written only from share_ready evidence, so any problem in its notes refuses it.
|
|
166
|
+
shareSafety: { status: "blocked", reasons: [...verified.shareSafety.reasons, problem.reason] },
|
|
167
|
+
error: {
|
|
168
|
+
code: "HUMANISH_FEEDBACK_SHARE_SAFETY_BLOCKED",
|
|
169
|
+
message: `The run's reviewer notes hold text that looks like a secret, a token or a local path, so no draft was written. Run \`${cli(`verify --run ${runId}`)}\` and fix the note files in notes/.`,
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
}
|
|
146
173
|
export async function verifyFeedback(cwdInput, runInput, options = {}) {
|
|
147
174
|
return (await verifyFeedbackBound(cwdInput, runInput, options)).result;
|
|
148
175
|
}
|
|
@@ -22,6 +22,7 @@ import { homedir } from "node:os";
|
|
|
22
22
|
import path from "node:path";
|
|
23
23
|
import { loadEnvFile } from "./env-file.js";
|
|
24
24
|
import { cli } from "../cli/invocation.js";
|
|
25
|
+
import { humanishConfigFile } from "../cli/user-config.js";
|
|
25
26
|
import { OPENAI_EGRESS_PLACEHOLDER } from "../routes/terminal/runtime-auth.js";
|
|
26
27
|
/** The only names implicit discovery may fill (and `humanish keys set` may store). Everything
|
|
27
28
|
* else in an overlay/store file is ignored-and-named: a repo-planted NODE_OPTIONS/LD_PRELOAD
|
|
@@ -41,24 +42,18 @@ const PROVIDER_KEY_SET = new Set(KNOWN_PROVIDER_KEYS);
|
|
|
41
42
|
function withoutProviderKeys(env) {
|
|
42
43
|
return Object.fromEntries(Object.entries(env).filter(([name]) => !PROVIDER_KEY_SET.has(name)));
|
|
43
44
|
}
|
|
44
|
-
/** `humanish keys set <vendor>` aliases; a raw ENV_NAME is also accepted.
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
45
|
+
/** `humanish keys set <vendor>` aliases; a raw ENV_NAME is also accepted. A Map, so a name such as
|
|
46
|
+
* `constructor` is no alias. */
|
|
47
|
+
const KEY_VENDOR_ALIASES = new Map([
|
|
48
|
+
["openai", "OPENAI_API_KEY"],
|
|
49
|
+
["e2b", "E2B_API_KEY"],
|
|
50
|
+
["anthropic", "ANTHROPIC_API_KEY"],
|
|
51
|
+
["github", "GH_TOKEN"],
|
|
52
|
+
["agentmail", "AGENTMAIL_API_KEY"],
|
|
53
|
+
]);
|
|
52
54
|
const PROJECT_OVERLAY_RELATIVE = path.join(".humanish", "local", "provider.env");
|
|
53
55
|
export function userKeyStorePath(env, deps = {}) {
|
|
54
|
-
|
|
55
|
-
const declared = env.XDG_CONFIG_HOME?.trim();
|
|
56
|
-
// The XDG spec: a relative XDG_CONFIG_HOME must be ignored. Honoring one would make the
|
|
57
|
-
// key store cwd-relative: `humanish keys set` would write a secret into the current repo.
|
|
58
|
-
const configHome = declared !== undefined && declared !== "" && path.isAbsolute(declared)
|
|
59
|
-
? declared
|
|
60
|
-
: path.join(home, ".config");
|
|
61
|
-
return path.join(configHome, "humanish", "keys.env");
|
|
56
|
+
return humanishConfigFile(env, "keys.env", deps.homeDir);
|
|
62
57
|
}
|
|
63
58
|
function e2bConfigPath(deps) {
|
|
64
59
|
return path.join(deps.homeDir ?? homedir(), ".e2b", "config.json");
|
|
@@ -306,7 +301,7 @@ export async function probeKeySources(names, args) {
|
|
|
306
301
|
const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
307
302
|
/** Resolve a `humanish keys set` target: a vendor alias or a raw env name. Null = invalid. */
|
|
308
303
|
export function resolveKeyName(vendorOrName) {
|
|
309
|
-
const alias = KEY_VENDOR_ALIASES
|
|
304
|
+
const alias = KEY_VENDOR_ALIASES.get(vendorOrName.toLowerCase());
|
|
310
305
|
if (alias !== undefined)
|
|
311
306
|
return alias;
|
|
312
307
|
return ENV_NAME.test(vendorOrName) ? vendorOrName : null;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { LoadedAnalysis } from "../analysis/types.js";
|
|
2
|
+
import type { RunNotes } from "../run/note-files.js";
|
|
2
3
|
import { type ObserverData } from "./data.js";
|
|
3
4
|
/**
|
|
4
5
|
* Resolve (and in a repo checkout, build) the Observer artifact before any run or
|
|
@@ -19,5 +20,9 @@ export type ObserverExportAssets = Record<string, {
|
|
|
19
20
|
export declare function renderObserverHtml(data: ObserverData, options?: {
|
|
20
21
|
snapshot?: boolean;
|
|
21
22
|
analysis?: LoadedAnalysis;
|
|
23
|
+
/** The run's reviewer notes, shown read-only unless `notesToken` is set. */
|
|
24
|
+
notes?: RunNotes | null;
|
|
25
|
+
/** The serving process's token for adding a note. Only a loopback server's page carries it. */
|
|
26
|
+
notesToken?: string;
|
|
22
27
|
assets?: ObserverExportAssets;
|
|
23
28
|
}): string;
|