humanish 0.111.3 → 0.111.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/cli/findings.d.ts +15 -1
- package/dist/cli/findings.js +43 -5
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/routes/scripted/route.js +1 -0
- package/dist/routes/scripted/setup.js +53 -10
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/live-finish.js +7 -4
- package/dist/routes/terminal/session.js +1 -1
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/run/secrets.d.ts +15 -2
- package/dist/run/secrets.js +134 -9
- package/dist/study/requirements.js +2 -1
- package/docs/contracts/study-analysis.md +6 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,21 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.111.4: Encoded known values scrubbed, scripted traces scrubbed, a live command after a dry run (2026-10-06)
|
|
12
|
+
|
|
13
|
+
humanish 0.111.4 scrubs a run's known values from more of its evidence and fixes three messages.
|
|
14
|
+
Every route now also finds a known value, such as a provider key or a `subject.env` value, in its
|
|
15
|
+
encoded forms: percent-encoded, JSON-escaped, base64, base64url and hex, and where a terminal color
|
|
16
|
+
code or an output chunk boundary splits it. A scripted run scrubs its known values from the step
|
|
17
|
+
trace in `traces/<surface>.json` and `actor-<surface>.json`, and from the app URL, scenario goal,
|
|
18
|
+
title and step labels it records. Two places keep a known value: a scenario file name, and the
|
|
19
|
+
path of a computer-use `subject.appUrl`. `humanish review` on a dry run names the command that
|
|
20
|
+
starts a live run, so `next` in `review --json` holds a string where it held null. A terminal
|
|
21
|
+
refusal for a missing runtime key no longer reads `runtimeAuth "undefined"`, and a refusal that
|
|
22
|
+
names two signed-in coding agents reads "report".
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.111.4)
|
|
25
|
+
|
|
11
26
|
## 0.111.3: Shared-world feedback candidates, scripted E2B key scrubbed, subject sandbox warnings (2026-10-06)
|
|
12
27
|
|
|
13
28
|
humanish 0.111.3 gives shared-world runs feedback candidates and repairs seven places where one
|
|
@@ -24,6 +24,11 @@ export interface ScriptedBrowserSessionOptions {
|
|
|
24
24
|
/** DI seam; production default is launchPlaywrightChromium. */
|
|
25
25
|
launchBrowser?: (args: ScriptedBrowserLaunchArgs) => Promise<ScriptedBrowserLike>;
|
|
26
26
|
now?: () => number;
|
|
27
|
+
/**
|
|
28
|
+
* The run's scrub of its known secret values. Each URL and text the session records passes
|
|
29
|
+
* through it before the trace is written or the capture returned.
|
|
30
|
+
*/
|
|
31
|
+
scrubKnownValues?: (text: string) => string;
|
|
27
32
|
}
|
|
28
33
|
export interface ScriptedBrowserSessionResult extends ActorSessionResult {
|
|
29
34
|
/** Native evidence incl. tracePath (humanish.browser-persona-trace.v1, written to disk). */
|
|
@@ -21,7 +21,7 @@ import { CHROMIUM_EVIDENCE_HYGIENE_FLAGS } from "../../evidence/browser-hygiene.
|
|
|
21
21
|
import { redactText } from "../../evidence/redaction.js";
|
|
22
22
|
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputDirectory, prepareContainedOutputFile, prepareSelectedOutputDirectory, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
23
23
|
import { ACTOR_TRACE_SCHEMA, SCRIPTED_BROWSER_CAPABILITIES, } from "../contract.js";
|
|
24
|
-
import { buildBlockedBrowserPersonaSteps, buildBrowserTrace, captureBlockedStepScreenshot, compactBrowserError, executeBrowserPersonaStep, probeAppUrl, sanitizeBrowserEvidenceUrl, screenshotPathForBrowserStep, surfaceScreenshotPath, tracePathForBrowserSurface, } from "./steps.js";
|
|
24
|
+
import { buildBlockedBrowserPersonaSteps, buildBrowserTrace, captureBlockedStepScreenshot, compactBrowserError, executeBrowserPersonaStep, probeAppUrl, sanitizeBrowserEvidenceUrl, screenshotPathForBrowserStep, scrubRecordedStep, surfaceScreenshotPath, tracePathForBrowserSurface, } from "./steps.js";
|
|
25
25
|
import { LOOPBACK_EVIDENCE_URL_POLICY, } from "./types.js";
|
|
26
26
|
import { chromeLaunchEnv, chromeLaunchError } from "./launch-env.js";
|
|
27
27
|
/** Production default: lazy playwright-core import + chromium.launch, exactly as the driver
|
|
@@ -93,8 +93,10 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
93
93
|
const browserCommand = options.browserCommand ?? (options.launchBrowser ? "injected-browser" : "");
|
|
94
94
|
const evidenceAppUrl = options.evidenceAppUrl ?? options.appUrl;
|
|
95
95
|
const urlPolicy = options.urlPolicy ?? LOOPBACK_EVIDENCE_URL_POLICY;
|
|
96
|
+
const scrub = options.scrubKnownValues ?? ((text) => text);
|
|
96
97
|
const finish = async (args) => {
|
|
97
98
|
const completedAtMs = now();
|
|
99
|
+
const reason = scrub(args.reason);
|
|
98
100
|
const trace = await projectScriptedActorTrace({
|
|
99
101
|
artifactRoot: preparedArtifactRoot,
|
|
100
102
|
capture: args.capture,
|
|
@@ -104,14 +106,14 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
104
106
|
executedSteps: args.executedSteps,
|
|
105
107
|
journey: options.journey,
|
|
106
108
|
persona: options.persona,
|
|
107
|
-
reason
|
|
109
|
+
reason,
|
|
108
110
|
startedAt,
|
|
109
111
|
status: args.status,
|
|
110
112
|
});
|
|
111
113
|
return {
|
|
112
114
|
status: args.status,
|
|
113
115
|
completionReason: args.completionReason,
|
|
114
|
-
reason
|
|
116
|
+
reason,
|
|
115
117
|
capture: args.capture,
|
|
116
118
|
trace,
|
|
117
119
|
};
|
|
@@ -130,6 +132,7 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
130
132
|
evidenceAppUrl,
|
|
131
133
|
journey: options.journey,
|
|
132
134
|
reason,
|
|
135
|
+
scrub,
|
|
133
136
|
surface: options.surface,
|
|
134
137
|
urlPolicy,
|
|
135
138
|
});
|
|
@@ -155,6 +158,7 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
155
158
|
browserCommand,
|
|
156
159
|
evidenceAppUrl,
|
|
157
160
|
journey: options.journey,
|
|
161
|
+
scrub,
|
|
158
162
|
surface: options.surface,
|
|
159
163
|
timeoutMs: options.timeoutMs,
|
|
160
164
|
urlPolicy,
|
|
@@ -258,17 +262,18 @@ async function runScriptedJourney(args) {
|
|
|
258
262
|
finally {
|
|
259
263
|
await args.browser.close().catch(() => undefined);
|
|
260
264
|
}
|
|
265
|
+
const recordedSteps = steps.map((step) => scrubRecordedStep(step, args.scrub));
|
|
261
266
|
const completedAt = new Date().toISOString();
|
|
262
267
|
const durationMs = Date.now() - started;
|
|
263
268
|
const ok = !timedOut &&
|
|
264
269
|
httpProbe.ok &&
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const reason = ok
|
|
268
|
-
? `${args.surface.label} completed ${
|
|
269
|
-
: `${args.surface.label} scripted browser journey blocked: ${
|
|
270
|
-
const scriptedScreenshotPath = surfaceScreenshotPath(
|
|
271
|
-
await
|
|
270
|
+
recordedSteps.length === args.journey.steps.length &&
|
|
271
|
+
recordedSteps.every((step) => step.status === "passed");
|
|
272
|
+
const reason = args.scrub(ok
|
|
273
|
+
? `${args.surface.label} completed ${recordedSteps.length}/${recordedSteps.length} scripted browser steps from ${args.evidenceAppUrl}${httpProbe.status === undefined ? "" : ` with HTTP ${httpProbe.status}`}.`
|
|
274
|
+
: `${args.surface.label} scripted browser journey blocked: ${recordedSteps.find((step) => step.status !== "passed")?.reason ?? httpProbe.reason}`);
|
|
275
|
+
const scriptedScreenshotPath = surfaceScreenshotPath(recordedSteps);
|
|
276
|
+
await writeBrowserTrace(args.artifactRoot, tracePath, {
|
|
272
277
|
appUrl: args.evidenceAppUrl,
|
|
273
278
|
browserCommand: path.basename(args.browserCommand || "injected-browser"),
|
|
274
279
|
browserJourney: args.journey,
|
|
@@ -278,9 +283,9 @@ async function runScriptedJourney(args) {
|
|
|
278
283
|
ok,
|
|
279
284
|
reason,
|
|
280
285
|
...(scriptedScreenshotPath === undefined ? {} : { screenshotPath: scriptedScreenshotPath }),
|
|
281
|
-
steps,
|
|
286
|
+
steps: recordedSteps,
|
|
282
287
|
surface: args.surface,
|
|
283
|
-
}
|
|
288
|
+
}, args.scrub);
|
|
284
289
|
return {
|
|
285
290
|
capture: {
|
|
286
291
|
capturedAt: completedAt,
|
|
@@ -289,7 +294,7 @@ async function runScriptedJourney(args) {
|
|
|
289
294
|
ok,
|
|
290
295
|
reason,
|
|
291
296
|
...(scriptedScreenshotPath === undefined ? {} : { screenshotPath: scriptedScreenshotPath }),
|
|
292
|
-
steps,
|
|
297
|
+
steps: recordedSteps,
|
|
293
298
|
surface: args.surface,
|
|
294
299
|
tracePath,
|
|
295
300
|
},
|
|
@@ -327,40 +332,56 @@ async function withJourneyDeadline(promise, deadline, timeoutMs) {
|
|
|
327
332
|
async function persistScriptedFailureCapture(args) {
|
|
328
333
|
const capturedAt = new Date().toISOString();
|
|
329
334
|
const tracePath = tracePathForBrowserSurface(args.surface);
|
|
335
|
+
const reason = args.scrub(args.reason);
|
|
330
336
|
const blockedSteps = buildBlockedBrowserPersonaSteps({
|
|
331
337
|
browserJourney: args.journey,
|
|
332
338
|
currentUrl: args.appUrl,
|
|
333
|
-
reason
|
|
339
|
+
reason,
|
|
334
340
|
surface: args.surface,
|
|
335
341
|
timestamp: capturedAt,
|
|
336
342
|
urlPolicy: args.urlPolicy,
|
|
337
|
-
});
|
|
343
|
+
}).map((step) => scrubRecordedStep(step, args.scrub));
|
|
338
344
|
// Pre-actuation failure: no screenshots were written, so the surface omits the
|
|
339
345
|
// screenshot reference and the failure itself stands as the evidence.
|
|
340
346
|
const screenshotPath = surfaceScreenshotPath(blockedSteps);
|
|
341
|
-
await
|
|
347
|
+
await writeBrowserTrace(args.artifactRoot, tracePath, {
|
|
342
348
|
appUrl: args.evidenceAppUrl,
|
|
343
349
|
browserCommand: path.basename(args.browserCommand || "injected-browser"),
|
|
344
350
|
browserJourney: args.journey,
|
|
345
351
|
capturedAt,
|
|
346
352
|
durationMs: 0,
|
|
347
353
|
ok: false,
|
|
348
|
-
reason
|
|
354
|
+
reason,
|
|
349
355
|
...(screenshotPath === undefined ? {} : { screenshotPath }),
|
|
350
356
|
steps: blockedSteps,
|
|
351
357
|
surface: args.surface,
|
|
352
|
-
}
|
|
358
|
+
}, args.scrub);
|
|
353
359
|
return {
|
|
354
360
|
capturedAt,
|
|
355
361
|
durationMs: 0,
|
|
356
362
|
ok: false,
|
|
357
|
-
reason
|
|
363
|
+
reason,
|
|
358
364
|
...(screenshotPath === undefined ? {} : { screenshotPath }),
|
|
359
365
|
steps: blockedSteps,
|
|
360
366
|
surface: args.surface,
|
|
361
367
|
tracePath,
|
|
362
368
|
};
|
|
363
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* Writes one surface's native trace. Its steps and reason arrive scrubbed; the app URL and the
|
|
372
|
+
* scenario title pass through the same scrub here.
|
|
373
|
+
*/
|
|
374
|
+
async function writeBrowserTrace(artifactRoot, tracePath, trace, scrub) {
|
|
375
|
+
const recorded = buildBrowserTrace({
|
|
376
|
+
...trace,
|
|
377
|
+
appUrl: scrub(trace.appUrl),
|
|
378
|
+
browserJourney: {
|
|
379
|
+
...trace.browserJourney,
|
|
380
|
+
scenarioTitle: scrub(trace.browserJourney.scenarioTitle),
|
|
381
|
+
},
|
|
382
|
+
});
|
|
383
|
+
await writeContainedOutputFile(artifactRoot, tracePath, `${JSON.stringify(recorded, null, 2)}\n`, "utf8");
|
|
384
|
+
}
|
|
364
385
|
/** Project one surface capture into humanish.actor-trace.v1. screenshotRefs are attached only
|
|
365
386
|
* for frames that actually exist on disk (so counts match disk; blocked-not-executed steps name a
|
|
366
387
|
* path that was never written). */
|
|
@@ -7,3 +7,5 @@ export declare function parseBrowserPersonaJourneyFromScenario(args: {
|
|
|
7
7
|
failure?: string;
|
|
8
8
|
journey?: BrowserPersonaJourney;
|
|
9
9
|
};
|
|
10
|
+
/** `value` lower-cased, with each run of other characters a `-`, cut to 80; else `fallback`. */
|
|
11
|
+
export declare function publicSafeToken(value: string | undefined, fallback: string): string;
|
|
@@ -172,7 +172,8 @@ function stringValue(value) {
|
|
|
172
172
|
function booleanValue(value) {
|
|
173
173
|
return typeof value === "boolean" ? value : undefined;
|
|
174
174
|
}
|
|
175
|
-
|
|
175
|
+
/** `value` lower-cased, with each run of other characters a `-`, cut to 80; else `fallback`. */
|
|
176
|
+
export function publicSafeToken(value, fallback) {
|
|
176
177
|
const candidate = (value ?? fallback)
|
|
177
178
|
.toLowerCase()
|
|
178
179
|
.replace(/[^a-z0-9_.-]+/g, "-")
|
|
@@ -18,6 +18,8 @@ export declare function buildBlockedBrowserPersonaSteps(args: {
|
|
|
18
18
|
timestamp: string;
|
|
19
19
|
urlPolicy?: ScriptedBrowserEvidenceUrlPolicy;
|
|
20
20
|
}): BrowserPersonaStepCapture[];
|
|
21
|
+
/** A step as the trace records it: its label, reason, URL and assertion reasons scrubbed. */
|
|
22
|
+
export declare function scrubRecordedStep(step: BrowserPersonaStepCapture, scrub: (text: string) => string): BrowserPersonaStepCapture;
|
|
21
23
|
export declare function screenshotPathForBrowserStep(surface: BrowserSurface, step: BrowserPersonaStepManifest | undefined): string;
|
|
22
24
|
export declare function tracePathForBrowserSurface(surface: BrowserSurface): string;
|
|
23
25
|
/**
|
|
@@ -191,6 +191,23 @@ export function buildBlockedBrowserPersonaSteps(args) {
|
|
|
191
191
|
url: sanitizeBrowserEvidenceUrl(args.currentUrl, args.urlPolicy),
|
|
192
192
|
}));
|
|
193
193
|
}
|
|
194
|
+
/** A step as the trace records it: its label, reason, URL and assertion reasons scrubbed. */
|
|
195
|
+
export function scrubRecordedStep(step, scrub) {
|
|
196
|
+
return {
|
|
197
|
+
...step,
|
|
198
|
+
...(step.assertions === undefined
|
|
199
|
+
? {}
|
|
200
|
+
: {
|
|
201
|
+
assertions: step.assertions.map((assertion) => ({
|
|
202
|
+
...assertion,
|
|
203
|
+
reason: scrub(assertion.reason),
|
|
204
|
+
})),
|
|
205
|
+
}),
|
|
206
|
+
label: scrub(step.label),
|
|
207
|
+
reason: scrub(step.reason),
|
|
208
|
+
url: scrub(step.url),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
194
211
|
export function screenshotPathForBrowserStep(surface, step) {
|
|
195
212
|
assertSafeOutputPathSegment(surface.id, "Browser surface id");
|
|
196
213
|
if (step) {
|
package/dist/cli/findings.d.ts
CHANGED
|
@@ -48,7 +48,10 @@ export interface AnalysisFindings {
|
|
|
48
48
|
/** The stable code behind a state other than ready, when one is recorded. */
|
|
49
49
|
reason: string | null;
|
|
50
50
|
message: string;
|
|
51
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* The command that gets findings for this run, or for a dry run the command that starts a live
|
|
53
|
+
* one. Null when none can.
|
|
54
|
+
*/
|
|
52
55
|
next: string | null;
|
|
53
56
|
analysisId: string | null;
|
|
54
57
|
status: AnalysisArtifact["status"] | null;
|
|
@@ -80,6 +83,17 @@ interface FindingsSource {
|
|
|
80
83
|
* false`, read when the view is built. Undefined when it does not, or cannot be read.
|
|
81
84
|
*/
|
|
82
85
|
analysisOffIn?: string;
|
|
86
|
+
/** For a dry run, the study a live run can start from, read when the view is built. */
|
|
87
|
+
liveStudy?: LiveStudy;
|
|
88
|
+
}
|
|
89
|
+
/** A study `run` can start live after a dry run. */
|
|
90
|
+
interface LiveStudy {
|
|
91
|
+
/** The study argument for `run`: its id, or its path when the run named it by path. */
|
|
92
|
+
run: string;
|
|
93
|
+
/** Its id, for the message. */
|
|
94
|
+
id: string;
|
|
95
|
+
/** The study file to set `mode: live` in first, when the file still starts as a dry run. */
|
|
96
|
+
setLiveIn?: string;
|
|
83
97
|
}
|
|
84
98
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
85
99
|
export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
|
package/dist/cli/findings.js
CHANGED
|
@@ -9,6 +9,7 @@ import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
|
9
9
|
import { resolvePhysicalCwd, runIdOf } from "../run/paths.js";
|
|
10
10
|
import { studyProvenanceOf } from "../run/study-provenance.js";
|
|
11
11
|
import { resolveStudyManifest } from "../study/discover.js";
|
|
12
|
+
import { routeOf } from "../study/routing.js";
|
|
12
13
|
import { plural } from "../run/text.js";
|
|
13
14
|
import { shellQuote } from "../substrates/shell.js";
|
|
14
15
|
import { analysisOutcomeText, wantsJson } from "./io.js";
|
|
@@ -102,10 +103,21 @@ function withoutFindings(source, state, reason, message, next) {
|
|
|
102
103
|
warnings: source.loaded.warnings,
|
|
103
104
|
};
|
|
104
105
|
}
|
|
106
|
+
/** A dry run's view: nothing to analyze, and the command that starts a live run. */
|
|
107
|
+
function dryRunFindings(source) {
|
|
108
|
+
const opening = "This is a dry run: no participant used the product, so there is nothing to analyze.";
|
|
109
|
+
const live = source.liveStudy;
|
|
110
|
+
if (live === undefined)
|
|
111
|
+
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs. The next command lists this project's studies; a study with mode: live runs participants.`, cli(`study list${source.cwdFlag}`));
|
|
112
|
+
const howLive = live.setLiveIn === undefined
|
|
113
|
+
? `the next command runs ${live.id} live.`
|
|
114
|
+
: `set mode: live in ${live.setLiveIn}, then run the next command.`;
|
|
115
|
+
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs: ${howLive}`, cli(`run ${live.run}${source.cwdFlag}`));
|
|
116
|
+
}
|
|
105
117
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
106
118
|
export function analysisFindings(source) {
|
|
107
119
|
if (source.mode !== undefined && source.mode !== "live")
|
|
108
|
-
return
|
|
120
|
+
return dryRunFindings(source);
|
|
109
121
|
const loaded = projectShareCheckedAnalysis(source.loaded);
|
|
110
122
|
const analysis = loaded.analysis;
|
|
111
123
|
if (analysis?.result && (loaded.state === "ready" || loaded.state === "stale")) {
|
|
@@ -170,6 +182,30 @@ async function studyAnalysisOff(cwd, study) {
|
|
|
170
182
|
return undefined;
|
|
171
183
|
return resolved.config.review?.analysis === false ? study.path : undefined;
|
|
172
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* The study a live run can start from after a dry run: the run's own study when its file runs
|
|
187
|
+
* live, else the starter study try-live when this project has one that runs live, else the run's
|
|
188
|
+
* own study once its file is set live. A preview study only dry-runs. Undefined when none applies.
|
|
189
|
+
*/
|
|
190
|
+
async function liveStudyAfterDryRun(cwd, study) {
|
|
191
|
+
const resolve = async (ref) => {
|
|
192
|
+
const resolved = await resolveStudyManifest(cwd, ref).catch(() => null);
|
|
193
|
+
return resolved?.ok ? resolved.config : undefined;
|
|
194
|
+
};
|
|
195
|
+
const ownConfig = study === undefined
|
|
196
|
+
? undefined
|
|
197
|
+
: await resolve(study.path ?? study.id).then((config) => config?.id === study.id ? config : undefined);
|
|
198
|
+
const own = study === undefined
|
|
199
|
+
? undefined
|
|
200
|
+
: { run: study.origin === "explicit" ? (study.path ?? study.id) : study.id, id: study.id };
|
|
201
|
+
if (own !== undefined && ownConfig?.mode === "live")
|
|
202
|
+
return own;
|
|
203
|
+
if ((await resolve("try-live"))?.mode === "live")
|
|
204
|
+
return { run: "try-live", id: "try-live" };
|
|
205
|
+
if (own !== undefined && ownConfig !== undefined && study?.path !== undefined)
|
|
206
|
+
return routeOf(ownConfig) === "preview" ? undefined : { ...own, setLiveIn: study.path };
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
173
209
|
/**
|
|
174
210
|
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
175
211
|
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
@@ -180,16 +216,18 @@ export async function readRunAnalysis(cwd, run, analysisId) {
|
|
|
180
216
|
if (!prepared)
|
|
181
217
|
return null;
|
|
182
218
|
const bundle = await loadRunBundlePrepared(physical, prepared).catch(() => null);
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
219
|
+
const mode = bundle?.bundle.mode;
|
|
220
|
+
const study = bundle === null ? undefined : studyProvenanceOf(bundle.bundle);
|
|
221
|
+
const analysisOffIn = mode === "live" ? await studyAnalysisOff(physical, study) : undefined;
|
|
222
|
+
const liveStudy = mode !== undefined && mode !== "live" ? await liveStudyAfterDryRun(physical, study) : undefined;
|
|
186
223
|
return {
|
|
187
224
|
runId: runIdOf(prepared),
|
|
188
|
-
mode
|
|
225
|
+
mode,
|
|
189
226
|
runRoot: prepared.relativeRunRoot,
|
|
190
227
|
loaded: await loadAnalysis(prepared, analysisId),
|
|
191
228
|
cwdFlag: cwdFlag(cwd),
|
|
192
229
|
...(analysisOffIn === undefined ? {} : { analysisOffIn }),
|
|
230
|
+
...(liveStudy === undefined ? {} : { liveStudy }),
|
|
193
231
|
};
|
|
194
232
|
}
|
|
195
233
|
/** The view of a run that could not be found. */
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
3
|
+
* `\u` escapes) and twice, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
4
|
+
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
5
|
+
* is not ordinary text.
|
|
6
|
+
*/
|
|
7
|
+
export declare function encodedForms(value: string): string[];
|
|
1
8
|
/**
|
|
2
9
|
* A scrub that replaces each non-empty value and its encoded forms with `[REDACTED_SECRET]`. Every
|
|
3
10
|
* occurrence of every form is found on its own, and overlapping finds are merged into one span, so
|
|
@@ -25,25 +25,34 @@ function base64Middles(bytes, encoding) {
|
|
|
25
25
|
return encoded.slice(start, end);
|
|
26
26
|
});
|
|
27
27
|
}
|
|
28
|
-
/**
|
|
29
|
-
|
|
28
|
+
/**
|
|
29
|
+
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
30
|
+
* `\u` escapes) and twice, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
31
|
+
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
32
|
+
* is not ordinary text.
|
|
33
|
+
*/
|
|
34
|
+
export function encodedForms(value) {
|
|
30
35
|
const bytes = Buffer.from(value, "utf8");
|
|
31
|
-
const
|
|
32
|
-
|
|
36
|
+
const json = JSON.stringify(value).slice(1, -1);
|
|
37
|
+
// A serializer that writes ASCII only, such as Python's json.dumps, spells é as \u00e9.
|
|
38
|
+
const asciiJson = json.replace(/[^\x00-\x7f]/g, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`);
|
|
39
|
+
const escaped = [json, asciiJson, JSON.stringify(json).slice(1, -1)];
|
|
40
|
+
// The encoders throw on a lone surrogate. Such a value is still found as written. encodeURI
|
|
41
|
+
// keeps a `/` or a `:` as written, as a URL path does.
|
|
42
|
+
try {
|
|
43
|
+
escaped.push(encodeURIComponent(value), encodeURI(value));
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// no percent-encoded form
|
|
47
|
+
}
|
|
48
|
+
const binary = [
|
|
33
49
|
bytes.toString("base64"),
|
|
34
50
|
bytes.toString("base64url"),
|
|
35
51
|
bytes.toString("hex"),
|
|
36
52
|
...base64Middles(bytes, "base64"),
|
|
37
53
|
...base64Middles(bytes, "base64url"),
|
|
38
54
|
];
|
|
39
|
-
|
|
40
|
-
try {
|
|
41
|
-
encoded.push(encodeURIComponent(value));
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
// no percent-encoded form
|
|
45
|
-
}
|
|
46
|
-
return [value, ...encoded.filter((form) => form.length >= MIN_ENCODED_FORM)];
|
|
55
|
+
return [value, ...escaped, ...binary.filter((form) => form.length >= MIN_ENCODED_FORM)];
|
|
47
56
|
}
|
|
48
57
|
/** Every occurrence of every form, outside the markers, merged where they overlap or touch. */
|
|
49
58
|
function secretSpans(text, forms) {
|
|
@@ -5,6 +5,7 @@ import { realpath } from "node:fs/promises";
|
|
|
5
5
|
import { firstLiveRefusal, keysCheck, subjectEnvCheck } from "../../study/requirements.js";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-command.js";
|
|
8
|
+
import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
|
|
8
9
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
9
10
|
import { RunSecrets } from "../../run/secrets.js";
|
|
10
11
|
import { evidenceAppUrlOf } from "./plan.js";
|
|
@@ -26,7 +27,16 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
26
27
|
const deps = input.deps ?? {};
|
|
27
28
|
const warnings = [];
|
|
28
29
|
const clone = plan.subject.kind === "clone" ? plan.subject : undefined;
|
|
29
|
-
const
|
|
30
|
+
const subjectEnvNames = [...(clone?.env ?? [])];
|
|
31
|
+
const env = input.env ?? process.env;
|
|
32
|
+
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
33
|
+
const secrets = new RunSecrets([
|
|
34
|
+
e2bApiKey,
|
|
35
|
+
...(clone ? [clone.repo] : []),
|
|
36
|
+
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
37
|
+
]);
|
|
38
|
+
// An app-url run records the URL it was given, in the bundle and every summary.
|
|
39
|
+
const evidenceAppUrl = secrets.scrub(evidenceAppUrlOf(plan.subject));
|
|
30
40
|
const failed = (code, message) => refusedResult("scripted", { studyId: plan.studyId, cwd, warnings, error: { code, message } }, {
|
|
31
41
|
actor: plan.actor,
|
|
32
42
|
appUrl: evidenceAppUrl,
|
|
@@ -37,16 +47,8 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
37
47
|
const urlPolicy = clone
|
|
38
48
|
? { kind: "provisioned-subject", evidenceOrigin: evidenceAppUrl }
|
|
39
49
|
: { kind: "loopback" };
|
|
40
|
-
const subjectEnvNames = [...(clone?.env ?? [])];
|
|
41
|
-
const env = input.env ?? process.env;
|
|
42
|
-
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
43
50
|
const hasGithubToken = subjectEnvNames.includes("GITHUB_TOKEN");
|
|
44
51
|
const redactRepoLabel = plan.residual.policies?.redactRepos ?? hasGithubToken;
|
|
45
|
-
const secrets = new RunSecrets([
|
|
46
|
-
e2bApiKey,
|
|
47
|
-
...(clone ? [clone.repo] : []),
|
|
48
|
-
...subjectEnvNames.map((name) => env[name] ?? ""),
|
|
49
|
-
]);
|
|
50
52
|
// A clone's URL is replaced by its getHost URL once it is served.
|
|
51
53
|
let appUrl = plan.subject.kind === "clone" ? plan.subject.serve.url : plan.subject.appUrl;
|
|
52
54
|
const scenario = await resolveScriptedScenario(projectRoot, plan.scenarioRef);
|
|
@@ -56,7 +58,17 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
56
58
|
result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
|
|
57
59
|
};
|
|
58
60
|
}
|
|
59
|
-
|
|
61
|
+
// The goal, title and step labels are recorded; a step's path, selector, value and expected text
|
|
62
|
+
// drive the browser and stay as written.
|
|
63
|
+
const journey = {
|
|
64
|
+
...scenario.journey,
|
|
65
|
+
scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
|
|
66
|
+
? "scenario"
|
|
67
|
+
: scenario.journey.scenarioId,
|
|
68
|
+
goal: secrets.scrub(scenario.journey.goal),
|
|
69
|
+
scenarioTitle: secrets.scrub(scenario.journey.scenarioTitle),
|
|
70
|
+
steps: recordedStepIds(scenario.journey.steps, secrets),
|
|
71
|
+
};
|
|
60
72
|
// The plan lists E2B_API_KEY and the subject env only for a live clone.
|
|
61
73
|
const { requirements } = plan;
|
|
62
74
|
const refusal = await firstLiveRefusal([
|
|
@@ -118,3 +130,34 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
118
130
|
},
|
|
119
131
|
};
|
|
120
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
|
|
135
|
+
* other characters as `-`, which the scrub would not find.
|
|
136
|
+
*/
|
|
137
|
+
function idHoldsKnownValue(id, secrets) {
|
|
138
|
+
return (secrets.scrub(id) !== id ||
|
|
139
|
+
secrets.values().some((value) => {
|
|
140
|
+
const token = publicSafeToken(value, "");
|
|
141
|
+
return token.length > 0 && id.includes(token);
|
|
142
|
+
}));
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The steps with their labels scrubbed and their ids safe to record. A step id names the step's
|
|
146
|
+
* screenshot file and is recorded beside it, and the parser derives an omitted id from the label,
|
|
147
|
+
* lower-cased and cut to 80 characters, so the scrub may not find a value in it. A step whose
|
|
148
|
+
* label or id holds a known value is renamed by its position instead.
|
|
149
|
+
*/
|
|
150
|
+
function recordedStepIds(steps, secrets) {
|
|
151
|
+
const taken = new Set(steps.map((step) => step.id));
|
|
152
|
+
return steps.map((step, index) => {
|
|
153
|
+
const label = secrets.scrub(step.label);
|
|
154
|
+
if (label === step.label && !idHoldsKnownValue(step.id, secrets))
|
|
155
|
+
return { ...step, label };
|
|
156
|
+
const position = `step-${String(index + 1).padStart(2, "0")}`;
|
|
157
|
+
let id = position;
|
|
158
|
+
for (let suffix = 2; taken.has(id); suffix += 1)
|
|
159
|
+
id = `${position}-${suffix}`;
|
|
160
|
+
taken.add(id);
|
|
161
|
+
return { ...step, id, label };
|
|
162
|
+
});
|
|
163
|
+
}
|
|
@@ -3,7 +3,7 @@ import type { ScriptedPlan } from "../../study/plan-types.js";
|
|
|
3
3
|
import { type PreparedRunArtifactPaths } from "../../run/paths.js";
|
|
4
4
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
5
5
|
/** One session per surface, in parallel. */
|
|
6
|
-
export declare function runScriptedSessions(session: Required<Pick<ScriptedBrowserSessionOptions, "appUrl" | "evidenceAppUrl" | "urlPolicy" | "journey" | "persona" | "timeoutMs" | "artifactRoot">>, run: {
|
|
6
|
+
export declare function runScriptedSessions(session: Required<Pick<ScriptedBrowserSessionOptions, "appUrl" | "evidenceAppUrl" | "urlPolicy" | "journey" | "persona" | "timeoutMs" | "artifactRoot" | "scrubKnownValues">>, run: {
|
|
7
7
|
surfaces: ScriptedPlan["surfaces"];
|
|
8
8
|
deps: StudyDeps;
|
|
9
9
|
browserCommand: string | undefined;
|
|
@@ -7,6 +7,7 @@ import { validateScriptedSessionResult } from "./session-result.js";
|
|
|
7
7
|
/** One session per surface, in parallel. */
|
|
8
8
|
export function runScriptedSessions(session, run) {
|
|
9
9
|
const { appUrl, evidenceAppUrl, urlPolicy, journey, persona, timeoutMs, artifactRoot } = session;
|
|
10
|
+
const { scrubKnownValues } = session;
|
|
10
11
|
const { surfaces, deps, browserCommand, runPaths, onSessionStart } = run;
|
|
11
12
|
const runSession = deps.runScriptedSession;
|
|
12
13
|
return Promise.all(surfaces.map((surface) => {
|
|
@@ -20,6 +21,7 @@ export function runScriptedSessions(session, run) {
|
|
|
20
21
|
persona,
|
|
21
22
|
timeoutMs,
|
|
22
23
|
artifactRoot,
|
|
24
|
+
scrubKnownValues,
|
|
23
25
|
...(browserCommand === undefined ? {} : { browserCommand }),
|
|
24
26
|
...(deps.launchBrowser === undefined ? {} : { launchBrowser: deps.launchBrowser }),
|
|
25
27
|
...(deps.now === undefined ? {} : { now: deps.now }),
|
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
import { DEFAULT_RUNTIME_AUTH, E2B_SYSTEM_CA_BUNDLE, OPENAI_EGRESS_PLACEHOLDER, } from "./runtime-auth.js";
|
|
2
2
|
import { TERMINAL_PRODUCT_STUDY_PROVIDER_METADATA } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* Where the study's runtime auth puts the key, as a sentence for the missing-key refusal. Typed
|
|
5
|
+
* `unknown` because a config that skipped the planner may hold a mode the type does not list.
|
|
6
|
+
*/
|
|
7
|
+
function keyPlacement(runtimeAuth) {
|
|
8
|
+
if (runtimeAuth === undefined)
|
|
9
|
+
return ` \`execution.runtimeAuth\` is unset, so the default ${DEFAULT_RUNTIME_AUTH} keeps the key outside the sandbox in an E2B header transform.`;
|
|
10
|
+
if (runtimeAuth === "openai-egress")
|
|
11
|
+
return " `execution.runtimeAuth: openai-egress` keeps the key outside the sandbox in an E2B header transform.";
|
|
12
|
+
if (runtimeAuth === "openai-env")
|
|
13
|
+
return " `execution.runtimeAuth: openai-env` passes the key to the codex command only.";
|
|
14
|
+
return "";
|
|
15
|
+
}
|
|
3
16
|
/**
|
|
4
17
|
* Resolve the runtime key on the host. openai-egress, the default, returns an inert command env
|
|
5
18
|
* while retaining the actual value for the external transform and literal redaction; openai-env
|
|
@@ -38,7 +51,7 @@ export function buildRuntimeAuth(args) {
|
|
|
38
51
|
return {
|
|
39
52
|
ok: false,
|
|
40
53
|
code: "HUMANISH_TERMINAL_RUNTIME_AUTH_MISSING",
|
|
41
|
-
message: `Live terminal-product studies
|
|
54
|
+
message: `Live terminal-product studies need ${ALLOWED_RUNTIME_KEY_NAMES.join(" or ")} in the environment, and neither is set.${keyPlacement(args.runtimeAuth)} Pass one with --dotenv; the value is never persisted.`,
|
|
42
55
|
};
|
|
43
56
|
}
|
|
44
57
|
const keyValue = args.env[keyName];
|
|
@@ -19,14 +19,17 @@ function buildLiveTrace(inputs) {
|
|
|
19
19
|
const { session } = inputs;
|
|
20
20
|
const { createdAt } = inputs.run;
|
|
21
21
|
const { terminalEvents, commandLog, discardedPrefixes } = inputs.recorder;
|
|
22
|
-
// Prefix reconciliation may cut through a known key. Scrub
|
|
23
|
-
// chunks before any transcript/trace/event artifact is
|
|
22
|
+
// Prefix reconciliation may cut through a known key. Scrub the known values, as written and in
|
|
23
|
+
// each encoded form, across the retained chunks before any transcript/trace/event artifact is
|
|
24
|
+
// persisted. Check both each stream and
|
|
24
25
|
// the combined event order that the transcript uses; either view can assemble a split value.
|
|
25
|
-
scrubSplitKnownValues(terminalEvents, inputs.run.secrets.
|
|
26
|
+
scrubSplitKnownValues(terminalEvents, (text) => inputs.run.secrets.spans(text), discardedPrefixes);
|
|
26
27
|
// Build the actor trace first (the cost ledger reads its tokenUsage). A session past the cap
|
|
27
28
|
// ends its transcript with what was not stored, and its trace says the same.
|
|
28
29
|
const cut = inputs.recorder.transcriptCut();
|
|
29
|
-
|
|
30
|
+
// Normalizing removes terminal escape sequences, which rejoins a value one of them split, so the
|
|
31
|
+
// transcript is sanitized again after it.
|
|
32
|
+
const normalizedTranscript = `${sanitize(normalizeLocalActorTranscript(terminalEvents.map((e) => e.chunk).join("")))}${cut ? `\n[humanish] ${cut.notice}` : ""}`;
|
|
30
33
|
// Parsed from the full stream: usage records arrive once per turn, and the tail
|
|
31
34
|
// would drop all but the last. Records past the cap were kept as they arrived.
|
|
32
35
|
const terminalTokenUsage = parseTerminalTokenUsage([normalizedTranscript, ...(cut?.usage ?? [])].join("\n"));
|
|
@@ -70,7 +70,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
70
70
|
const recorder = createTerminalRecorder({
|
|
71
71
|
nowIso,
|
|
72
72
|
sanitize,
|
|
73
|
-
knownSecretValues: run.secrets.
|
|
73
|
+
knownSecretValues: run.secrets.forms(),
|
|
74
74
|
verdictNonce,
|
|
75
75
|
});
|
|
76
76
|
const { version, model, modelSource, reasoningEffort } = plan.runtime;
|
|
@@ -5,9 +5,11 @@ import { type CommandLogRecord, type TerminalEventRecord } from "./types.js";
|
|
|
5
5
|
/**
|
|
6
6
|
* Per-chunk sanitization cannot recognize a value split across deliveries. Redact those complete
|
|
7
7
|
* known values before persistence without collapsing events or changing stdout/stderr ordering.
|
|
8
|
-
*
|
|
8
|
+
* `spansOf` finds them in the joined text: RunSecrets.spans, which also finds a value in an
|
|
9
|
+
* encoded form or split by a terminal escape sequence. Work backwards through the spans so edits
|
|
10
|
+
* to later text leave earlier offsets valid.
|
|
9
11
|
*/
|
|
10
|
-
export declare function scrubSplitKnownValues(events: TerminalEventRecord[],
|
|
12
|
+
export declare function scrubSplitKnownValues(events: TerminalEventRecord[], spansOf: (text: string) => ReadonlyArray<readonly [number, number]>, discardedPrefixes: Record<"stdout" | "stderr" | "combined", string>): void;
|
|
11
13
|
/** Redacted, ellipsis-prefixed tail of a captured stream/log for a message field. */
|
|
12
14
|
export declare function tailOf(text: string): string;
|
|
13
15
|
/**
|
|
@@ -4,50 +4,45 @@ import { TAIL_CHARS } from "./types.js";
|
|
|
4
4
|
/**
|
|
5
5
|
* Per-chunk sanitization cannot recognize a value split across deliveries. Redact those complete
|
|
6
6
|
* known values before persistence without collapsing events or changing stdout/stderr ordering.
|
|
7
|
-
*
|
|
7
|
+
* `spansOf` finds them in the joined text: RunSecrets.spans, which also finds a value in an
|
|
8
|
+
* encoded form or split by a terminal escape sequence. Work backwards through the spans so edits
|
|
9
|
+
* to later text leave earlier offsets valid.
|
|
8
10
|
*/
|
|
9
|
-
export function scrubSplitKnownValues(events,
|
|
11
|
+
export function scrubSplitKnownValues(events, spansOf, discardedPrefixes) {
|
|
10
12
|
for (const order of ["stdout", "stderr", "combined"]) {
|
|
11
13
|
const chunks = order === "combined" ? [...events] : events.filter((event) => event.stream === order);
|
|
12
14
|
// A virtual final chunk makes a key crossing the capture cap recognizable. Edits to retained
|
|
13
15
|
// events redact evidence; the raw overlap and this virtual chunk are never persisted.
|
|
14
16
|
if (discardedPrefixes[order])
|
|
15
17
|
chunks.push({ chunk: discardedPrefixes[order] });
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
let offset = 0;
|
|
19
|
+
const starts = chunks.map((event) => {
|
|
20
|
+
const start = offset;
|
|
21
|
+
offset += event.chunk.length;
|
|
22
|
+
return start;
|
|
23
|
+
});
|
|
24
|
+
const text = chunks.map((event) => event.chunk).join("");
|
|
25
|
+
for (const [at, end] of [...spansOf(text)].reverse()) {
|
|
26
|
+
let first = 0;
|
|
27
|
+
while (first + 1 < starts.length && (starts[first + 1] ?? Infinity) <= at)
|
|
28
|
+
first += 1;
|
|
29
|
+
let last = first;
|
|
30
|
+
while (last + 1 < starts.length && (starts[last + 1] ?? Infinity) < end)
|
|
31
|
+
last += 1;
|
|
32
|
+
const firstChunk = chunks[first];
|
|
33
|
+
const lastChunk = chunks[last];
|
|
34
|
+
if (!firstChunk || !lastChunk)
|
|
18
35
|
continue;
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const matches = [];
|
|
27
|
-
for (let at = text.indexOf(value); at !== -1; at = text.indexOf(value, at + value.length))
|
|
28
|
-
matches.push(at);
|
|
29
|
-
for (const at of matches.reverse()) {
|
|
30
|
-
let first = 0;
|
|
31
|
-
while (first + 1 < starts.length && (starts[first + 1] ?? Infinity) <= at)
|
|
32
|
-
first += 1;
|
|
33
|
-
let last = first;
|
|
34
|
-
while (last + 1 < starts.length && (starts[last + 1] ?? Infinity) < at + value.length)
|
|
35
|
-
last += 1;
|
|
36
|
-
const firstChunk = chunks[first];
|
|
37
|
-
const lastChunk = chunks[last];
|
|
38
|
-
if (!firstChunk || !lastChunk)
|
|
39
|
-
continue;
|
|
40
|
-
const before = firstChunk.chunk.slice(0, at - (starts[first] ?? 0));
|
|
41
|
-
const after = lastChunk.chunk.slice(at + value.length - (starts[last] ?? 0));
|
|
42
|
-
firstChunk.chunk = `${before}[REDACTED_SECRET]${first === last ? after : ""}`;
|
|
43
|
-
for (let index = first + 1; index < last; index += 1) {
|
|
44
|
-
const middle = chunks[index];
|
|
45
|
-
if (middle)
|
|
46
|
-
middle.chunk = "";
|
|
47
|
-
}
|
|
48
|
-
if (first !== last)
|
|
49
|
-
lastChunk.chunk = after;
|
|
36
|
+
const before = firstChunk.chunk.slice(0, at - (starts[first] ?? 0));
|
|
37
|
+
const after = lastChunk.chunk.slice(end - (starts[last] ?? 0));
|
|
38
|
+
firstChunk.chunk = `${before}[REDACTED_SECRET]${first === last ? after : ""}`;
|
|
39
|
+
for (let index = first + 1; index < last; index += 1) {
|
|
40
|
+
const middle = chunks[index];
|
|
41
|
+
if (middle)
|
|
42
|
+
middle.chunk = "";
|
|
50
43
|
}
|
|
44
|
+
if (first !== last)
|
|
45
|
+
lastChunk.chunk = after;
|
|
51
46
|
}
|
|
52
47
|
}
|
|
53
48
|
}
|
package/dist/run/secrets.d.ts
CHANGED
|
@@ -8,8 +8,10 @@ interface RunSecretsOptions {
|
|
|
8
8
|
export declare class RunSecrets {
|
|
9
9
|
#private;
|
|
10
10
|
/**
|
|
11
|
-
* Replaces every held value with the marker
|
|
12
|
-
*
|
|
11
|
+
* Replaces every held value with the marker, as written, in each encoded form, and where a URL
|
|
12
|
+
* carries it partly percent-encoded or a terminal escape sequence splits it. It reads the values
|
|
13
|
+
* on each call, so a value added after the scrub was handed to a participant or a subject is
|
|
14
|
+
* scrubbed from then on.
|
|
13
15
|
*/
|
|
14
16
|
readonly scrub: (text: string) => string;
|
|
15
17
|
/** `seed` keeps its order; values shorter than the floor are dropped. */
|
|
@@ -18,5 +20,16 @@ export declare class RunSecrets {
|
|
|
18
20
|
add(values: readonly string[]): void;
|
|
19
21
|
/** The held values, seed first, in one array that later `add` calls extend. */
|
|
20
22
|
values(): readonly string[];
|
|
23
|
+
/**
|
|
24
|
+
* Every held value as written and in each encoded form, longest first, in one array that later
|
|
25
|
+
* `add` calls extend. The terminal recorder sizes the output it keeps past its cap from these.
|
|
26
|
+
*/
|
|
27
|
+
forms(): readonly string[];
|
|
28
|
+
/**
|
|
29
|
+
* Where `text` holds a held value: each encoded form as written, and each form in the text's
|
|
30
|
+
* view without terminal escape sequences and with percent escapes decoded. Sorted, with
|
|
31
|
+
* overlapping spans merged. A scrub that replaces across chunk boundaries reads these.
|
|
32
|
+
*/
|
|
33
|
+
spans(text: string): Array<[number, number]>;
|
|
21
34
|
}
|
|
22
35
|
export {};
|
package/dist/run/secrets.js
CHANGED
|
@@ -1,33 +1,158 @@
|
|
|
1
1
|
// The literal values one run scrubs from what it writes: the provider keys and subject env values
|
|
2
2
|
// its route reads, and the values the run provisions while it runs, such as the addresses email
|
|
3
3
|
// receiving registers. These have no secret shape for pattern redaction to find, so each is
|
|
4
|
-
// replaced as written
|
|
4
|
+
// replaced as written and in the encoded forms a URL, a JSON body or a log line gives it;
|
|
5
|
+
// redactText runs after the scrub and finds secrets by shape.
|
|
5
6
|
import { scrubLiterals } from "../evidence/redaction.js";
|
|
7
|
+
import { encodedForms } from "../evidence/secret-scrub.js";
|
|
6
8
|
export class RunSecrets {
|
|
7
9
|
// Private fields stay out of JSON.stringify and object spread, so a Run that reaches a writer
|
|
8
10
|
// whole cannot carry the values with it.
|
|
9
|
-
#held;
|
|
11
|
+
#held = [];
|
|
12
|
+
/** Every held value as written and encoded, longest first. */
|
|
13
|
+
#forms = [];
|
|
10
14
|
#minLength;
|
|
11
15
|
/**
|
|
12
|
-
* Replaces every held value with the marker
|
|
13
|
-
*
|
|
16
|
+
* Replaces every held value with the marker, as written, in each encoded form, and where a URL
|
|
17
|
+
* carries it partly percent-encoded or a terminal escape sequence splits it. It reads the values
|
|
18
|
+
* on each call, so a value added after the scrub was handed to a participant or a subject is
|
|
19
|
+
* scrubbed from then on.
|
|
14
20
|
*/
|
|
15
21
|
scrub;
|
|
16
22
|
/** `seed` keeps its order; values shorter than the floor are dropped. */
|
|
17
23
|
constructor(seed, options = {}) {
|
|
18
24
|
// An empty value would match between every character.
|
|
19
25
|
this.#minLength = Math.max(1, options.minLength ?? 4);
|
|
20
|
-
this
|
|
21
|
-
|
|
26
|
+
this.add(seed);
|
|
27
|
+
const marker = options.marker ?? "[REDACTED_SECRET]";
|
|
28
|
+
// Longest first, so a form that holds a shorter one, such as a JSON-escaped value ending in a
|
|
29
|
+
// backslash, is replaced whole and leaves no stray escape behind.
|
|
30
|
+
const literal = scrubLiterals(this.#forms, marker);
|
|
31
|
+
this.scrub = (text) => {
|
|
32
|
+
const scrubbed = literal(text);
|
|
33
|
+
return replaceSpans(scrubbed, viewSpans(scrubbed, this.#forms), marker);
|
|
34
|
+
};
|
|
22
35
|
}
|
|
23
36
|
/** Holds each value the run learns while it runs, unless it is under the floor or already held. */
|
|
24
37
|
add(values) {
|
|
25
|
-
for (const value of values)
|
|
26
|
-
if (value.length
|
|
27
|
-
|
|
38
|
+
for (const value of values) {
|
|
39
|
+
if (value.length < this.#minLength || this.#held.includes(value))
|
|
40
|
+
continue;
|
|
41
|
+
this.#held.push(value);
|
|
42
|
+
for (const form of encodedForms(value))
|
|
43
|
+
if (!this.#forms.includes(form))
|
|
44
|
+
this.#forms.push(form);
|
|
45
|
+
}
|
|
46
|
+
this.#forms.sort((left, right) => right.length - left.length);
|
|
28
47
|
}
|
|
29
48
|
/** The held values, seed first, in one array that later `add` calls extend. */
|
|
30
49
|
values() {
|
|
31
50
|
return this.#held;
|
|
32
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Every held value as written and in each encoded form, longest first, in one array that later
|
|
54
|
+
* `add` calls extend. The terminal recorder sizes the output it keeps past its cap from these.
|
|
55
|
+
*/
|
|
56
|
+
forms() {
|
|
57
|
+
return this.#forms;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Where `text` holds a held value: each encoded form as written, and each form in the text's
|
|
61
|
+
* view without terminal escape sequences and with percent escapes decoded. Sorted, with
|
|
62
|
+
* overlapping spans merged. A scrub that replaces across chunk boundaries reads these.
|
|
63
|
+
*/
|
|
64
|
+
spans(text) {
|
|
65
|
+
const found = viewSpans(text, this.#forms);
|
|
66
|
+
for (const form of this.#forms)
|
|
67
|
+
for (let at = text.indexOf(form); at !== -1; at = text.indexOf(form, at + form.length))
|
|
68
|
+
found.push([at, at + form.length]);
|
|
69
|
+
return mergeSpans(found);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// What a terminal draws with (operating-system commands, control sequences and two-byte escapes,
|
|
73
|
+
// as written or JSON-escaped, as a JSON event carries a command's colored output) and what a URL
|
|
74
|
+
// encodes with (runs of percent escapes). The view a value is looked for in drops the first and
|
|
75
|
+
// decodes the second.
|
|
76
|
+
const VIEW_SEQUENCE = new RegExp([
|
|
77
|
+
"\\x1b\\][^\\x07]*(?:\\x07|\\x1b\\\\)",
|
|
78
|
+
"\\x1b\\[[0-?]*[ -/]*[@-~]",
|
|
79
|
+
"\\x1b[78=>]",
|
|
80
|
+
"\\\\u001b\\][^\\\\]*(?:\\\\u0007|\\\\u001b\\\\\\\\)",
|
|
81
|
+
"\\\\u001b\\[[0-?]*[ -/]*[@-~]",
|
|
82
|
+
"(?:%[0-9A-Fa-f]{2})+",
|
|
83
|
+
].join("|"), "g");
|
|
84
|
+
/**
|
|
85
|
+
* Where a value is found in the text's view. A browser encodes a space or a quote in a URL path
|
|
86
|
+
* and leaves a `/` or a `:` as written, so a value with both matches no single encoded form, and a
|
|
87
|
+
* color code inside a value splits it. Each found value maps back to the span of `text` it came
|
|
88
|
+
* from, so the text keeps every other character as written.
|
|
89
|
+
*/
|
|
90
|
+
function viewSpans(text, values) {
|
|
91
|
+
if (!text.includes("%") && !text.includes("\x1b") && !text.includes("\\u001b"))
|
|
92
|
+
return [];
|
|
93
|
+
// The view, and for each of its characters the span of `text` it came from. A character decoded
|
|
94
|
+
// from an escape run maps to the whole run.
|
|
95
|
+
let view = "";
|
|
96
|
+
const starts = [];
|
|
97
|
+
const ends = [];
|
|
98
|
+
const copy = (from, to) => {
|
|
99
|
+
for (let at = from; at < to; at += 1) {
|
|
100
|
+
starts.push(at);
|
|
101
|
+
ends.push(at + 1);
|
|
102
|
+
}
|
|
103
|
+
view += text.slice(from, to);
|
|
104
|
+
};
|
|
105
|
+
let cursor = 0;
|
|
106
|
+
let changed = false;
|
|
107
|
+
for (const sequence of text.matchAll(VIEW_SEQUENCE)) {
|
|
108
|
+
let plain = "";
|
|
109
|
+
if (sequence[0].startsWith("%")) {
|
|
110
|
+
try {
|
|
111
|
+
plain = decodeURIComponent(sequence[0]);
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
// Not UTF-8: the run stays as written and is copied with the text after it.
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
copy(cursor, sequence.index);
|
|
119
|
+
cursor = sequence.index + sequence[0].length;
|
|
120
|
+
for (let unit = 0; unit < plain.length; unit += 1) {
|
|
121
|
+
starts.push(sequence.index);
|
|
122
|
+
ends.push(cursor);
|
|
123
|
+
}
|
|
124
|
+
view += plain;
|
|
125
|
+
changed = true;
|
|
126
|
+
}
|
|
127
|
+
if (!changed)
|
|
128
|
+
return [];
|
|
129
|
+
copy(cursor, text.length);
|
|
130
|
+
const spans = [];
|
|
131
|
+
for (const value of values)
|
|
132
|
+
for (let at = view.indexOf(value); at !== -1; at = view.indexOf(value, at + value.length))
|
|
133
|
+
spans.push([starts[at] ?? 0, ends[at + value.length - 1] ?? text.length]);
|
|
134
|
+
return spans;
|
|
135
|
+
}
|
|
136
|
+
/** Sorted, with overlapping spans merged; spans that only touch stay apart. */
|
|
137
|
+
function mergeSpans(spans) {
|
|
138
|
+
const merged = [];
|
|
139
|
+
for (const [from, to] of [...spans].sort((left, right) => left[0] - right[0])) {
|
|
140
|
+
const last = merged.at(-1);
|
|
141
|
+
if (last !== undefined && from < last[1])
|
|
142
|
+
last[1] = Math.max(last[1], to);
|
|
143
|
+
else
|
|
144
|
+
merged.push([from, to]);
|
|
145
|
+
}
|
|
146
|
+
return merged;
|
|
147
|
+
}
|
|
148
|
+
function replaceSpans(text, spans, marker) {
|
|
149
|
+
if (spans.length === 0)
|
|
150
|
+
return text;
|
|
151
|
+
let replaced = "";
|
|
152
|
+
let written = 0;
|
|
153
|
+
for (const [from, to] of mergeSpans(spans)) {
|
|
154
|
+
replaced += `${text.slice(written, from)}${marker}`;
|
|
155
|
+
written = to;
|
|
156
|
+
}
|
|
157
|
+
return `${replaced}${text.slice(written)}`;
|
|
33
158
|
}
|
|
@@ -100,7 +100,8 @@ async function signedInAgentSuggestion(env) {
|
|
|
100
100
|
if (ready.length === 0)
|
|
101
101
|
return "";
|
|
102
102
|
const labels = ready.map((agent) => agent.label).join(" and ");
|
|
103
|
-
|
|
103
|
+
const one = ready.length === 1;
|
|
104
|
+
return ` ${labels} ${one ? "reports" : "report"} authenticated on this machine. Set actor.type: local-agent to use ${one ? "it" : "one"} instead of a key.`;
|
|
104
105
|
}
|
|
105
106
|
/**
|
|
106
107
|
* Refuses a local-agent brain whose CLI is missing, signed out, too old, or cannot honor the
|
|
@@ -308,7 +308,7 @@ analysis record and its corrections, as before. Reading findings never starts an
|
|
|
308
308
|
| `state` | `ready`, `stale`, `running`, `none`, `skipped`, `failed`, `dry_run` or `unavailable`. Only `ready` and `stale` carry findings. |
|
|
309
309
|
| `reason` | The stable code behind a state other than `ready`, or null: an automatic analysis reason such as `AUTOMATIC_ANALYSIS_KEY_MISSING`, a failed attempt's error code, or a load warning such as `ANALYSIS_SOURCE_CHANGED`. |
|
|
310
310
|
| `message` | What happened, in words. |
|
|
311
|
-
| `next` | The command that gets findings for this run, or
|
|
311
|
+
| `next` | The command that gets findings for this run, or for a dry run the command that starts a live one. Null when no command can, as for a run whose participants left no evidence. |
|
|
312
312
|
| `analysisId`, `status`, `provider`, `model`, `completedAt`, `estimatedCostUsd` | The analysis version `analyze show` selects: the newest ready one, else the newest stale one. Null without one. |
|
|
313
313
|
| `runPath`, `path` | The run directory and the version's `analysis.json`, relative to the project directory. |
|
|
314
314
|
| `summary`, `limitations` | The analysis's own summary and limitations. |
|
|
@@ -328,7 +328,11 @@ The states without findings:
|
|
|
328
328
|
reruns it with the same analyst, `--provider codex` for the Codex account analyst.
|
|
329
329
|
- `none`: no analysis ran for this live run and nothing records why, for example a run that
|
|
330
330
|
started from a library call, or one whose study file has moved or changed id.
|
|
331
|
-
- `dry_run`: a dry run has no participant evidence to analyze.
|
|
331
|
+
- `dry_run`: a dry run has no participant evidence to analyze. `next` is `run` with the run's own
|
|
332
|
+
study when its file sets `mode: live`, else `run try-live` when the project has that starter study
|
|
333
|
+
with `mode: live`, else `run` with the run's own study, and the message says to set `mode: live`
|
|
334
|
+
in its file first. A preview study only dry-runs, and a run with none of these names
|
|
335
|
+
`study list`.
|
|
332
336
|
- `unavailable`: the run or its analysis records could not be read or checked, or the analysis
|
|
333
337
|
text matched a sensitive-text pattern (`ANALYSIS_SENSITIVE_TEXT_QUARANTINED`), which the
|
|
334
338
|
Observer withholds too.
|