humanish 0.111.2 → 0.111.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +32 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/cli/findings.d.ts +15 -1
- package/dist/cli/findings.js +43 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +14 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +71 -35
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +8 -5
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +35 -0
- package/dist/run/secrets.js +158 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +89 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/docs/contracts/study-analysis.md +6 -2
- package/package.json +1 -1
|
@@ -4,50 +4,45 @@ import { TAIL_CHARS } from "./types.js";
|
|
|
4
4
|
/**
|
|
5
5
|
* Per-chunk sanitization cannot recognize a value split across deliveries. Redact those complete
|
|
6
6
|
* known values before persistence without collapsing events or changing stdout/stderr ordering.
|
|
7
|
-
*
|
|
7
|
+
* `spansOf` finds them in the joined text: RunSecrets.spans, which also finds a value in an
|
|
8
|
+
* encoded form or split by a terminal escape sequence. Work backwards through the spans so edits
|
|
9
|
+
* to later text leave earlier offsets valid.
|
|
8
10
|
*/
|
|
9
|
-
export function scrubSplitKnownValues(events,
|
|
11
|
+
export function scrubSplitKnownValues(events, spansOf, discardedPrefixes) {
|
|
10
12
|
for (const order of ["stdout", "stderr", "combined"]) {
|
|
11
13
|
const chunks = order === "combined" ? [...events] : events.filter((event) => event.stream === order);
|
|
12
14
|
// A virtual final chunk makes a key crossing the capture cap recognizable. Edits to retained
|
|
13
15
|
// events redact evidence; the raw overlap and this virtual chunk are never persisted.
|
|
14
16
|
if (discardedPrefixes[order])
|
|
15
17
|
chunks.push({ chunk: discardedPrefixes[order] });
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
let offset = 0;
|
|
19
|
+
const starts = chunks.map((event) => {
|
|
20
|
+
const start = offset;
|
|
21
|
+
offset += event.chunk.length;
|
|
22
|
+
return start;
|
|
23
|
+
});
|
|
24
|
+
const text = chunks.map((event) => event.chunk).join("");
|
|
25
|
+
for (const [at, end] of [...spansOf(text)].reverse()) {
|
|
26
|
+
let first = 0;
|
|
27
|
+
while (first + 1 < starts.length && (starts[first + 1] ?? Infinity) <= at)
|
|
28
|
+
first += 1;
|
|
29
|
+
let last = first;
|
|
30
|
+
while (last + 1 < starts.length && (starts[last + 1] ?? Infinity) < end)
|
|
31
|
+
last += 1;
|
|
32
|
+
const firstChunk = chunks[first];
|
|
33
|
+
const lastChunk = chunks[last];
|
|
34
|
+
if (!firstChunk || !lastChunk)
|
|
18
35
|
continue;
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const matches = [];
|
|
27
|
-
for (let at = text.indexOf(value); at !== -1; at = text.indexOf(value, at + value.length))
|
|
28
|
-
matches.push(at);
|
|
29
|
-
for (const at of matches.reverse()) {
|
|
30
|
-
let first = 0;
|
|
31
|
-
while (first + 1 < starts.length && (starts[first + 1] ?? Infinity) <= at)
|
|
32
|
-
first += 1;
|
|
33
|
-
let last = first;
|
|
34
|
-
while (last + 1 < starts.length && (starts[last + 1] ?? Infinity) < at + value.length)
|
|
35
|
-
last += 1;
|
|
36
|
-
const firstChunk = chunks[first];
|
|
37
|
-
const lastChunk = chunks[last];
|
|
38
|
-
if (!firstChunk || !lastChunk)
|
|
39
|
-
continue;
|
|
40
|
-
const before = firstChunk.chunk.slice(0, at - (starts[first] ?? 0));
|
|
41
|
-
const after = lastChunk.chunk.slice(at + value.length - (starts[last] ?? 0));
|
|
42
|
-
firstChunk.chunk = `${before}[REDACTED_SECRET]${first === last ? after : ""}`;
|
|
43
|
-
for (let index = first + 1; index < last; index += 1) {
|
|
44
|
-
const middle = chunks[index];
|
|
45
|
-
if (middle)
|
|
46
|
-
middle.chunk = "";
|
|
47
|
-
}
|
|
48
|
-
if (first !== last)
|
|
49
|
-
lastChunk.chunk = after;
|
|
36
|
+
const before = firstChunk.chunk.slice(0, at - (starts[first] ?? 0));
|
|
37
|
+
const after = lastChunk.chunk.slice(end - (starts[last] ?? 0));
|
|
38
|
+
firstChunk.chunk = `${before}[REDACTED_SECRET]${first === last ? after : ""}`;
|
|
39
|
+
for (let index = first + 1; index < last; index += 1) {
|
|
40
|
+
const middle = chunks[index];
|
|
41
|
+
if (middle)
|
|
42
|
+
middle.chunk = "";
|
|
50
43
|
}
|
|
44
|
+
if (first !== last)
|
|
45
|
+
lastChunk.chunk = after;
|
|
51
46
|
}
|
|
52
47
|
}
|
|
53
48
|
}
|
|
@@ -8,7 +8,7 @@ import type { TerminalPlan } from "../../study/plan-types.js";
|
|
|
8
8
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
9
9
|
import type { ObserverResult } from "../../observer/render.js";
|
|
10
10
|
import { type RunAdapterScore, type RunBundle, type RunFeedbackCandidate, type RunScorerProvenance } from "../../run/bundle.js";
|
|
11
|
-
import { type StudyResultIdentity } from "../../run/study-result.js";
|
|
11
|
+
import { type StudyResultErrorCode, type StudyResultIdentity } from "../../run/study-result.js";
|
|
12
12
|
/** Provider-neutral metadata constant: the route's non-secret tag (mirrors CUA_ACTOR_STUDY_PROVIDER_METADATA). */
|
|
13
13
|
export declare const TERMINAL_PRODUCT_STUDY_PROVIDER_METADATA: {
|
|
14
14
|
readonly mode: "terminal-product-lab";
|
|
@@ -172,7 +172,7 @@ export interface TerminalProductStudyResult extends AutomaticAnalysisResult, Stu
|
|
|
172
172
|
observer?: ObserverResult;
|
|
173
173
|
warnings: string[];
|
|
174
174
|
error?: {
|
|
175
|
-
code:
|
|
175
|
+
code: StudyResultErrorCode | "HUMANISH_TERMINAL_FAILED" | "HUMANISH_TERMINAL_ACTOR_UNSUPPORTED" | "HUMANISH_TERMINAL_SUBJECT_INVALID" | "HUMANISH_TERMINAL_KEYPLACEMENT_INVALID" | "HUMANISH_TERMINAL_RUNTIME_AUTH_MISSING" | "HUMANISH_TERMINAL_KEYS_MISSING" | "HUMANISH_TERMINAL_CAPS_MISSING" | "HUMANISH_TERMINAL_CAPS_INVALID" | "HUMANISH_TERMINAL_UNPRICED_CAP" | "HUMANISH_TERMINAL_CAPS_EXCEEDED" | "HUMANISH_TERMINAL_CREDENTIAL_DENIED" | "HUMANISH_TERMINAL_CLEANUP_UNPROVEN";
|
|
176
176
|
message: string;
|
|
177
177
|
};
|
|
178
178
|
}
|
package/dist/run/run.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { renderObserver, type ObserverResult } from "../observer/render.js";
|
|
|
2
2
|
import { type RunBundle } from "./bundle.js";
|
|
3
3
|
import { type ExecutionFailure, type ExecutionOutcome, type OutcomePolicy } from "./judge.js";
|
|
4
4
|
import { type PreparedRunArtifactPaths, type RunIdInUse } from "./paths.js";
|
|
5
|
+
import { RunSecrets } from "./secrets.js";
|
|
5
6
|
import type { RunStudyProvenance } from "./study-provenance.js";
|
|
6
7
|
interface StartRunOptions {
|
|
7
8
|
/** Project directory, in the form the route resolved it. */
|
|
@@ -24,6 +25,41 @@ interface StartRunOptions {
|
|
|
24
25
|
now?: (() => number) | undefined;
|
|
25
26
|
/** Warnings about the study's own fields. Every bundle write records each as a warn event. */
|
|
26
27
|
warnings?: readonly string[] | undefined;
|
|
28
|
+
/** The route's known values, created before the run so a refusal before it is scrubbed too. */
|
|
29
|
+
secrets?: RunSecrets | undefined;
|
|
30
|
+
}
|
|
31
|
+
/** The prefix of the run ids each recorded route mints. Preview mints its own `dryrun-` ids. */
|
|
32
|
+
type RunIdPrefix = "cua" | "concurrent-shared-world" | "scripted" | "terminal";
|
|
33
|
+
/** What a recorded route's plan holds for its run's start. */
|
|
34
|
+
interface StartRunPlan {
|
|
35
|
+
readonly dryRun: boolean;
|
|
36
|
+
readonly study?: RunStudyProvenance | undefined;
|
|
37
|
+
readonly warnings?: readonly string[] | undefined;
|
|
38
|
+
}
|
|
39
|
+
/** What a recorded route's input holds for its run's start. */
|
|
40
|
+
interface StartRunInput {
|
|
41
|
+
/** Caller-supplied id (`--run-id`, library `runId`); one is minted from the prefix when absent. */
|
|
42
|
+
readonly runId?: string | undefined;
|
|
43
|
+
/** Open the Observer page once it renders. */
|
|
44
|
+
readonly open?: boolean | undefined;
|
|
45
|
+
/** `renderObserver` is the seam FinishedRun.renderObserver calls. */
|
|
46
|
+
readonly deps?: {
|
|
47
|
+
readonly renderObserver?: typeof renderObserver | undefined;
|
|
48
|
+
} | undefined;
|
|
49
|
+
}
|
|
50
|
+
/** What a recorded route passes to startRun besides its plan and input. */
|
|
51
|
+
interface RouteRunOptions {
|
|
52
|
+
/** Project directory, in the form the route resolved it. */
|
|
53
|
+
cwd: string;
|
|
54
|
+
prefix: RunIdPrefix;
|
|
55
|
+
/** review.md for the published bundle. */
|
|
56
|
+
renderReview: (bundle: RunBundle) => string;
|
|
57
|
+
/** `none` when the route creates no sandbox for this run; status.json records it for reclaim. */
|
|
58
|
+
sandboxes?: "none" | undefined;
|
|
59
|
+
/** Clock for `createdAt` and the latest pointer. The minted id reads the wall clock. */
|
|
60
|
+
now?: (() => number) | undefined;
|
|
61
|
+
/** The route's known values, created before the run so a refusal before it is scrubbed too. */
|
|
62
|
+
secrets?: RunSecrets | undefined;
|
|
27
63
|
}
|
|
28
64
|
/** How a run ended as an execution, as run.json records it: the result's ok and execution outcome. */
|
|
29
65
|
interface RecordedOutcome {
|
|
@@ -46,6 +82,8 @@ interface Run {
|
|
|
46
82
|
readonly study?: RunStudyProvenance;
|
|
47
83
|
/** Routes write their evidence files through these and hand them to participants. */
|
|
48
84
|
readonly paths: PreparedRunArtifactPaths;
|
|
85
|
+
/** The literal values the route scrubs from this run's evidence; a value found later is added here. */
|
|
86
|
+
readonly secrets: RunSecrets;
|
|
49
87
|
/**
|
|
50
88
|
* Publish an in-progress bundle: run.json, review.json, review.md, events.ndjson and
|
|
51
89
|
* observer/observer-data.json, plus the latest pointer until one pointer write has succeeded,
|
|
@@ -74,11 +112,27 @@ interface Run {
|
|
|
74
112
|
*/
|
|
75
113
|
finish(bundle: RunBundle, outcome: FinishOutcome): Promise<FinishedRun>;
|
|
76
114
|
}
|
|
115
|
+
/** A recorded route's run, with the source its bundle records. */
|
|
116
|
+
interface RecordedRun extends Run {
|
|
117
|
+
/** The package and the project's git state, captured once the run exists, at its createdAt. */
|
|
118
|
+
readonly source: RunBundle["source"];
|
|
119
|
+
}
|
|
77
120
|
export interface RunScope {
|
|
121
|
+
/**
|
|
122
|
+
* Start a recorded route's run from its plan and input: the input's run id or one minted from
|
|
123
|
+
* `prefix`, the plan's mode, study and warnings, and an Observer that opens when the input says
|
|
124
|
+
* `open` and renders through the input's `deps.renderObserver`. Then capture the run's source.
|
|
125
|
+
* The run starts as the form below does.
|
|
126
|
+
*/
|
|
127
|
+
startRun(plan: StartRunPlan, input: StartRunInput, options: RouteRunOptions): Promise<{
|
|
128
|
+
ok: true;
|
|
129
|
+
run: RecordedRun;
|
|
130
|
+
} | RunIdInUse>;
|
|
78
131
|
/**
|
|
79
132
|
* Create `.humanish/runs/<id>` exclusively, begin status.json and wait for its first write, so
|
|
80
133
|
* a run that got past this point has a status record before any sandbox exists. An id whose
|
|
81
|
-
* directory exists is refused before any write. A scope starts at most one run.
|
|
134
|
+
* directory exists is refused before any write. A scope starts at most one run. Preview calls
|
|
135
|
+
* this form, since it builds its source before the run and mints its own id.
|
|
82
136
|
*/
|
|
83
137
|
startRun(options: StartRunOptions): Promise<{
|
|
84
138
|
ok: true;
|
package/dist/run/run.js
CHANGED
|
@@ -2,16 +2,18 @@
|
|
|
2
2
|
// token that lets the Observer and automatic analysis read it. Routes build bundles and write
|
|
3
3
|
// their own evidence files; this module owns when the run starts, how it is published, and how it
|
|
4
4
|
// is closed on every exit.
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
5
6
|
import { redactText } from "../evidence/redaction.js";
|
|
6
7
|
import { buildObserverData } from "../observer/data.js";
|
|
7
8
|
import { renderObserver } from "../observer/render.js";
|
|
8
|
-
import { RUN_BUNDLE_FILE, PUBLIC_TARGET_CWD, } from "./bundle.js";
|
|
9
|
+
import { RUN_BUNDLE_FILE, PUBLIC_TARGET_CWD, buildRunSource, } from "./bundle.js";
|
|
9
10
|
import { judgeExecution, } from "./judge.js";
|
|
10
11
|
import {} from "./results.js";
|
|
11
12
|
import { createRunArtifactPaths, validatePreparedRunArtifactPaths, } from "./paths.js";
|
|
12
13
|
import { registerActiveRun } from "./active-runs.js";
|
|
13
14
|
import { writeContainedOutputFile, writePreparedRunLatestPointer } from "./contained-output.js";
|
|
14
15
|
import { scrubRunSandboxIds, withPublicSandboxIds } from "./sandbox-ids.js";
|
|
16
|
+
import { RunSecrets } from "./secrets.js";
|
|
15
17
|
import { beginRunStatus, runStatusOutcome, } from "./status.js";
|
|
16
18
|
/** An entry point called in a state that does not admit it. */
|
|
17
19
|
class RunLifecycleError extends Error {
|
|
@@ -239,7 +241,7 @@ export async function runScope(fn) {
|
|
|
239
241
|
return operation;
|
|
240
242
|
};
|
|
241
243
|
const refuse = (message) => Promise.reject(new RunLifecycleError(message));
|
|
242
|
-
const
|
|
244
|
+
const startRunWith = (options) => {
|
|
243
245
|
if (closed)
|
|
244
246
|
return refuse("The run scope has closed.");
|
|
245
247
|
if (started)
|
|
@@ -277,6 +279,9 @@ export async function runScope(fn) {
|
|
|
277
279
|
return { ok: true, run: run.run };
|
|
278
280
|
})());
|
|
279
281
|
};
|
|
282
|
+
function startRun(...args) {
|
|
283
|
+
return args.length === 1 ? startRunWith(args[0]) : startRecordedRun(startRunWith, ...args);
|
|
284
|
+
}
|
|
280
285
|
const openRun = (options, runId, paths, runStatus) => {
|
|
281
286
|
const now = options.now ?? Date.now;
|
|
282
287
|
const createdAt = new Date(now()).toISOString();
|
|
@@ -301,6 +306,7 @@ export async function runScope(fn) {
|
|
|
301
306
|
mode: options.mode,
|
|
302
307
|
...(options.study === undefined ? {} : { study: options.study }),
|
|
303
308
|
paths,
|
|
309
|
+
secrets: options.secrets ?? new RunSecrets([]),
|
|
304
310
|
participantStarted() {
|
|
305
311
|
participantsRan = true;
|
|
306
312
|
},
|
|
@@ -348,3 +354,33 @@ export async function runScope(fn) {
|
|
|
348
354
|
}
|
|
349
355
|
return { result: outcome.result, finished };
|
|
350
356
|
}
|
|
357
|
+
/** Start a recorded route's run through `start`, the scope's own form, then capture its source. */
|
|
358
|
+
async function startRecordedRun(start, plan, input, route) {
|
|
359
|
+
const started = await start({
|
|
360
|
+
cwd: route.cwd,
|
|
361
|
+
runId: input.runId,
|
|
362
|
+
mintRunId: () => mintRunId(route.prefix),
|
|
363
|
+
mode: plan.dryRun ? "dry-run" : "live",
|
|
364
|
+
study: plan.study,
|
|
365
|
+
warnings: plan.warnings,
|
|
366
|
+
sandboxes: route.sandboxes,
|
|
367
|
+
renderReview: route.renderReview,
|
|
368
|
+
observer: { open: input.open === true, render: input.deps?.renderObserver },
|
|
369
|
+
now: route.now,
|
|
370
|
+
secrets: route.secrets,
|
|
371
|
+
});
|
|
372
|
+
if (!started.ok)
|
|
373
|
+
return started;
|
|
374
|
+
const source = await buildRunSource({
|
|
375
|
+
cwd: route.cwd,
|
|
376
|
+
capturedAt: started.run.createdAt,
|
|
377
|
+
humanishSource: "present",
|
|
378
|
+
packageName: "humanish",
|
|
379
|
+
});
|
|
380
|
+
return { ok: true, run: { ...started.run, source } };
|
|
381
|
+
}
|
|
382
|
+
/** `<prefix>-<wall-clock time, with : and . as ->-<8 hex>`. */
|
|
383
|
+
function mintRunId(prefix) {
|
|
384
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
385
|
+
return `${prefix}-${stamp}-${randomBytes(4).toString("hex")}`;
|
|
386
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/** Each route's seed list, marker and floor are its own; the defaults are what most routes use. */
|
|
2
|
+
interface RunSecretsOptions {
|
|
3
|
+
/** The text that replaces each value. */
|
|
4
|
+
readonly marker?: string;
|
|
5
|
+
/** The shortest value scrubbed. A shorter value is ordinary text often enough to leave alone. */
|
|
6
|
+
readonly minLength?: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class RunSecrets {
|
|
9
|
+
#private;
|
|
10
|
+
/**
|
|
11
|
+
* Replaces every held value with the marker, as written, in each encoded form, and where a URL
|
|
12
|
+
* carries it partly percent-encoded or a terminal escape sequence splits it. It reads the values
|
|
13
|
+
* on each call, so a value added after the scrub was handed to a participant or a subject is
|
|
14
|
+
* scrubbed from then on.
|
|
15
|
+
*/
|
|
16
|
+
readonly scrub: (text: string) => string;
|
|
17
|
+
/** `seed` keeps its order; values shorter than the floor are dropped. */
|
|
18
|
+
constructor(seed: readonly string[], options?: RunSecretsOptions);
|
|
19
|
+
/** Holds each value the run learns while it runs, unless it is under the floor or already held. */
|
|
20
|
+
add(values: readonly string[]): void;
|
|
21
|
+
/** The held values, seed first, in one array that later `add` calls extend. */
|
|
22
|
+
values(): readonly string[];
|
|
23
|
+
/**
|
|
24
|
+
* Every held value as written and in each encoded form, longest first, in one array that later
|
|
25
|
+
* `add` calls extend. The terminal recorder sizes the output it keeps past its cap from these.
|
|
26
|
+
*/
|
|
27
|
+
forms(): readonly string[];
|
|
28
|
+
/**
|
|
29
|
+
* Where `text` holds a held value: each encoded form as written, and each form in the text's
|
|
30
|
+
* view without terminal escape sequences and with percent escapes decoded. Sorted, with
|
|
31
|
+
* overlapping spans merged. A scrub that replaces across chunk boundaries reads these.
|
|
32
|
+
*/
|
|
33
|
+
spans(text: string): Array<[number, number]>;
|
|
34
|
+
}
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// The literal values one run scrubs from what it writes: the provider keys and subject env values
|
|
2
|
+
// its route reads, and the values the run provisions while it runs, such as the addresses email
|
|
3
|
+
// receiving registers. These have no secret shape for pattern redaction to find, so each is
|
|
4
|
+
// replaced as written and in the encoded forms a URL, a JSON body or a log line gives it;
|
|
5
|
+
// redactText runs after the scrub and finds secrets by shape.
|
|
6
|
+
import { scrubLiterals } from "../evidence/redaction.js";
|
|
7
|
+
import { encodedForms } from "../evidence/secret-scrub.js";
|
|
8
|
+
export class RunSecrets {
|
|
9
|
+
// Private fields stay out of JSON.stringify and object spread, so a Run that reaches a writer
|
|
10
|
+
// whole cannot carry the values with it.
|
|
11
|
+
#held = [];
|
|
12
|
+
/** Every held value as written and encoded, longest first. */
|
|
13
|
+
#forms = [];
|
|
14
|
+
#minLength;
|
|
15
|
+
/**
|
|
16
|
+
* Replaces every held value with the marker, as written, in each encoded form, and where a URL
|
|
17
|
+
* carries it partly percent-encoded or a terminal escape sequence splits it. It reads the values
|
|
18
|
+
* on each call, so a value added after the scrub was handed to a participant or a subject is
|
|
19
|
+
* scrubbed from then on.
|
|
20
|
+
*/
|
|
21
|
+
scrub;
|
|
22
|
+
/** `seed` keeps its order; values shorter than the floor are dropped. */
|
|
23
|
+
constructor(seed, options = {}) {
|
|
24
|
+
// An empty value would match between every character.
|
|
25
|
+
this.#minLength = Math.max(1, options.minLength ?? 4);
|
|
26
|
+
this.add(seed);
|
|
27
|
+
const marker = options.marker ?? "[REDACTED_SECRET]";
|
|
28
|
+
// Longest first, so a form that holds a shorter one, such as a JSON-escaped value ending in a
|
|
29
|
+
// backslash, is replaced whole and leaves no stray escape behind.
|
|
30
|
+
const literal = scrubLiterals(this.#forms, marker);
|
|
31
|
+
this.scrub = (text) => {
|
|
32
|
+
const scrubbed = literal(text);
|
|
33
|
+
return replaceSpans(scrubbed, viewSpans(scrubbed, this.#forms), marker);
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/** Holds each value the run learns while it runs, unless it is under the floor or already held. */
|
|
37
|
+
add(values) {
|
|
38
|
+
for (const value of values) {
|
|
39
|
+
if (value.length < this.#minLength || this.#held.includes(value))
|
|
40
|
+
continue;
|
|
41
|
+
this.#held.push(value);
|
|
42
|
+
for (const form of encodedForms(value))
|
|
43
|
+
if (!this.#forms.includes(form))
|
|
44
|
+
this.#forms.push(form);
|
|
45
|
+
}
|
|
46
|
+
this.#forms.sort((left, right) => right.length - left.length);
|
|
47
|
+
}
|
|
48
|
+
/** The held values, seed first, in one array that later `add` calls extend. */
|
|
49
|
+
values() {
|
|
50
|
+
return this.#held;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Every held value as written and in each encoded form, longest first, in one array that later
|
|
54
|
+
* `add` calls extend. The terminal recorder sizes the output it keeps past its cap from these.
|
|
55
|
+
*/
|
|
56
|
+
forms() {
|
|
57
|
+
return this.#forms;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Where `text` holds a held value: each encoded form as written, and each form in the text's
|
|
61
|
+
* view without terminal escape sequences and with percent escapes decoded. Sorted, with
|
|
62
|
+
* overlapping spans merged. A scrub that replaces across chunk boundaries reads these.
|
|
63
|
+
*/
|
|
64
|
+
spans(text) {
|
|
65
|
+
const found = viewSpans(text, this.#forms);
|
|
66
|
+
for (const form of this.#forms)
|
|
67
|
+
for (let at = text.indexOf(form); at !== -1; at = text.indexOf(form, at + form.length))
|
|
68
|
+
found.push([at, at + form.length]);
|
|
69
|
+
return mergeSpans(found);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// What a terminal draws with (operating-system commands, control sequences and two-byte escapes,
|
|
73
|
+
// as written or JSON-escaped, as a JSON event carries a command's colored output) and what a URL
|
|
74
|
+
// encodes with (runs of percent escapes). The view a value is looked for in drops the first and
|
|
75
|
+
// decodes the second.
|
|
76
|
+
const VIEW_SEQUENCE = new RegExp([
|
|
77
|
+
"\\x1b\\][^\\x07]*(?:\\x07|\\x1b\\\\)",
|
|
78
|
+
"\\x1b\\[[0-?]*[ -/]*[@-~]",
|
|
79
|
+
"\\x1b[78=>]",
|
|
80
|
+
"\\\\u001b\\][^\\\\]*(?:\\\\u0007|\\\\u001b\\\\\\\\)",
|
|
81
|
+
"\\\\u001b\\[[0-?]*[ -/]*[@-~]",
|
|
82
|
+
"(?:%[0-9A-Fa-f]{2})+",
|
|
83
|
+
].join("|"), "g");
|
|
84
|
+
/**
|
|
85
|
+
* Where a value is found in the text's view. A browser encodes a space or a quote in a URL path
|
|
86
|
+
* and leaves a `/` or a `:` as written, so a value with both matches no single encoded form, and a
|
|
87
|
+
* color code inside a value splits it. Each found value maps back to the span of `text` it came
|
|
88
|
+
* from, so the text keeps every other character as written.
|
|
89
|
+
*/
|
|
90
|
+
function viewSpans(text, values) {
|
|
91
|
+
if (!text.includes("%") && !text.includes("\x1b") && !text.includes("\\u001b"))
|
|
92
|
+
return [];
|
|
93
|
+
// The view, and for each of its characters the span of `text` it came from. A character decoded
|
|
94
|
+
// from an escape run maps to the whole run.
|
|
95
|
+
let view = "";
|
|
96
|
+
const starts = [];
|
|
97
|
+
const ends = [];
|
|
98
|
+
const copy = (from, to) => {
|
|
99
|
+
for (let at = from; at < to; at += 1) {
|
|
100
|
+
starts.push(at);
|
|
101
|
+
ends.push(at + 1);
|
|
102
|
+
}
|
|
103
|
+
view += text.slice(from, to);
|
|
104
|
+
};
|
|
105
|
+
let cursor = 0;
|
|
106
|
+
let changed = false;
|
|
107
|
+
for (const sequence of text.matchAll(VIEW_SEQUENCE)) {
|
|
108
|
+
let plain = "";
|
|
109
|
+
if (sequence[0].startsWith("%")) {
|
|
110
|
+
try {
|
|
111
|
+
plain = decodeURIComponent(sequence[0]);
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
// Not UTF-8: the run stays as written and is copied with the text after it.
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
copy(cursor, sequence.index);
|
|
119
|
+
cursor = sequence.index + sequence[0].length;
|
|
120
|
+
for (let unit = 0; unit < plain.length; unit += 1) {
|
|
121
|
+
starts.push(sequence.index);
|
|
122
|
+
ends.push(cursor);
|
|
123
|
+
}
|
|
124
|
+
view += plain;
|
|
125
|
+
changed = true;
|
|
126
|
+
}
|
|
127
|
+
if (!changed)
|
|
128
|
+
return [];
|
|
129
|
+
copy(cursor, text.length);
|
|
130
|
+
const spans = [];
|
|
131
|
+
for (const value of values)
|
|
132
|
+
for (let at = view.indexOf(value); at !== -1; at = view.indexOf(value, at + value.length))
|
|
133
|
+
spans.push([starts[at] ?? 0, ends[at + value.length - 1] ?? text.length]);
|
|
134
|
+
return spans;
|
|
135
|
+
}
|
|
136
|
+
/** Sorted, with overlapping spans merged; spans that only touch stay apart. */
|
|
137
|
+
function mergeSpans(spans) {
|
|
138
|
+
const merged = [];
|
|
139
|
+
for (const [from, to] of [...spans].sort((left, right) => left[0] - right[0])) {
|
|
140
|
+
const last = merged.at(-1);
|
|
141
|
+
if (last !== undefined && from < last[1])
|
|
142
|
+
last[1] = Math.max(last[1], to);
|
|
143
|
+
else
|
|
144
|
+
merged.push([from, to]);
|
|
145
|
+
}
|
|
146
|
+
return merged;
|
|
147
|
+
}
|
|
148
|
+
function replaceSpans(text, spans, marker) {
|
|
149
|
+
if (spans.length === 0)
|
|
150
|
+
return text;
|
|
151
|
+
let replaced = "";
|
|
152
|
+
let written = 0;
|
|
153
|
+
for (const [from, to] of mergeSpans(spans)) {
|
|
154
|
+
replaced += `${text.slice(written, from)}${marker}`;
|
|
155
|
+
written = to;
|
|
156
|
+
}
|
|
157
|
+
return `${replaced}${text.slice(written)}`;
|
|
158
|
+
}
|
|
@@ -7,5 +7,39 @@ export interface StudyResultIdentity<R extends string> {
|
|
|
7
7
|
/** The study's `id`. */
|
|
8
8
|
studyId: string;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* The error codes every recorded route's result can carry besides its own: a study, option or
|
|
12
|
+
* analysis refusal from before the route's checks, and a run id that is already taken. Each route's
|
|
13
|
+
* error-code union is this set plus the codes named for that route.
|
|
14
|
+
*/
|
|
15
|
+
export type StudyResultErrorCode = "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_RUN_ID_IN_USE";
|
|
10
16
|
/** The identity fields for a result of route `route` from the study `studyId`. */
|
|
11
17
|
export declare function studyResultIdentity<R extends string>(route: R, studyId: string): StudyResultIdentity<R>;
|
|
18
|
+
/** What every refused study's result carries besides its identity and its route's own fields. */
|
|
19
|
+
export interface RefusalBase<C extends string> {
|
|
20
|
+
/** The study's `id`. */
|
|
21
|
+
studyId: string;
|
|
22
|
+
cwd: string;
|
|
23
|
+
/** The caller's warnings array, kept by reference so warnings added later still show. Empty when omitted. */
|
|
24
|
+
warnings?: string[];
|
|
25
|
+
error: {
|
|
26
|
+
code: C;
|
|
27
|
+
message: string;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* A refused study's result on route `route`: its identity, `ok: false` and `cwd`, then `fields`
|
|
32
|
+
* in the caller's order, then `warnings` and `error`. A result is written as JSON, so the key order
|
|
33
|
+
* is part of the output; a site whose order differs spreads its remaining fields after this. The
|
|
34
|
+
* caller keeps runId, actor, dryRun and the route's fields, whose values and order differ by site.
|
|
35
|
+
*/
|
|
36
|
+
export declare function refusedResult<R extends string, C extends string, F extends object>(route: R, base: RefusalBase<C>, fields: F): StudyResultIdentity<R> & {
|
|
37
|
+
ok: false;
|
|
38
|
+
cwd: string;
|
|
39
|
+
} & F & {
|
|
40
|
+
warnings: string[];
|
|
41
|
+
error: {
|
|
42
|
+
code: C;
|
|
43
|
+
message: string;
|
|
44
|
+
};
|
|
45
|
+
};
|
package/dist/run/study-result.js
CHANGED
|
@@ -7,3 +7,19 @@ export const STUDY_RESULT_SCHEMA = "humanish.study-result.v1";
|
|
|
7
7
|
export function studyResultIdentity(route, studyId) {
|
|
8
8
|
return { schema: STUDY_RESULT_SCHEMA, route, studyId };
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* A refused study's result on route `route`: its identity, `ok: false` and `cwd`, then `fields`
|
|
12
|
+
* in the caller's order, then `warnings` and `error`. A result is written as JSON, so the key order
|
|
13
|
+
* is part of the output; a site whose order differs spreads its remaining fields after this. The
|
|
14
|
+
* caller keeps runId, actor, dryRun and the route's fields, whose values and order differ by site.
|
|
15
|
+
*/
|
|
16
|
+
export function refusedResult(route, base, fields) {
|
|
17
|
+
return {
|
|
18
|
+
...studyResultIdentity(route, base.studyId),
|
|
19
|
+
ok: false,
|
|
20
|
+
cwd: base.cwd,
|
|
21
|
+
...fields,
|
|
22
|
+
warnings: base.warnings ?? [],
|
|
23
|
+
error: base.error,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
export declare function missingKeys(requirements: readonly Requirement[], env: Record<string, string | undefined>): string[];
|
|
4
|
-
/** The subject env names the requirements list and `env` lacks, in declaration order. */
|
|
5
|
-
export declare function missingSubjectEnv(requirements: readonly Requirement[], env: Record<string, string | undefined>): string[];
|
|
1
|
+
import { type LocalAgentRefusal } from "../actors/local-agent/readiness.js";
|
|
2
|
+
import type { Brain, StudyPlan, Requirement } from "./plan-types.js";
|
|
6
3
|
/**
|
|
7
4
|
* The provider keys the requirements name, in plan order. A `key-one-of` names the first of its
|
|
8
5
|
* keys that `isPresent` reports, else OPENAI_API_KEY.
|
|
@@ -18,3 +15,61 @@ export declare function requiredSubjectEnv(requirements: readonly Requirement[])
|
|
|
18
15
|
* a declared scorer's host code, are not in the plan.
|
|
19
16
|
*/
|
|
20
17
|
export declare function keyNamesOf(plan: StudyPlan): ReadonlySet<string>;
|
|
18
|
+
/** A live check's refusal, in its route's error code. */
|
|
19
|
+
export interface LiveRefusal<C extends string> {
|
|
20
|
+
readonly code: C;
|
|
21
|
+
readonly message: string;
|
|
22
|
+
}
|
|
23
|
+
/** One live check: its refusal, or undefined when this machine passes it. */
|
|
24
|
+
export type LiveCheck<C extends string> = () => LiveRefusal<C> | undefined | Promise<LiveRefusal<C> | undefined>;
|
|
25
|
+
/**
|
|
26
|
+
* The first refusal of `checks`, run one at a time in the order given. Which check comes first
|
|
27
|
+
* decides which error a run with two problems reports, so each route keeps its own order.
|
|
28
|
+
*/
|
|
29
|
+
export declare function firstLiveRefusal<C extends string>(checks: readonly LiveCheck<C>[]): Promise<LiveRefusal<C> | undefined>;
|
|
30
|
+
/**
|
|
31
|
+
* Refuses a run whose environment lacks a provider key the requirements list. `need` writes the
|
|
32
|
+
* route's sentence for the missing names ("OPENAI_API_KEY and E2B_API_KEY"); the message goes on
|
|
33
|
+
* to say where discovery looked and what fills each key. With `suggestLocalAgent`, a missing
|
|
34
|
+
* OPENAI_API_KEY also names each coding agent signed in on this machine: someone new meets this
|
|
35
|
+
* refusal first, and an agent they already use runs the participants without a key.
|
|
36
|
+
*/
|
|
37
|
+
export declare function keysCheck<C extends string>(args: {
|
|
38
|
+
readonly requirements: readonly Requirement[];
|
|
39
|
+
readonly env: Record<string, string | undefined>;
|
|
40
|
+
readonly code: C;
|
|
41
|
+
readonly need: (names: string) => string;
|
|
42
|
+
readonly suggestLocalAgent?: boolean;
|
|
43
|
+
}): Promise<LiveRefusal<C> | undefined>;
|
|
44
|
+
/**
|
|
45
|
+
* Refuses a local-agent brain whose CLI is missing, signed out, too old, or cannot honor the
|
|
46
|
+
* caps, before any sandbox is paid for. `codes` names the route's code for each kind of refusal.
|
|
47
|
+
*/
|
|
48
|
+
export declare function localAgentCheck<C extends string>(args: {
|
|
49
|
+
readonly brain: Brain;
|
|
50
|
+
readonly env: Record<string, string | undefined>;
|
|
51
|
+
readonly caps: {
|
|
52
|
+
readonly maxUsd?: number;
|
|
53
|
+
readonly maxTotalUsd?: number;
|
|
54
|
+
};
|
|
55
|
+
readonly codes: Readonly<Record<LocalAgentRefusal["kind"], C>>;
|
|
56
|
+
}): Promise<LiveRefusal<C> | undefined>;
|
|
57
|
+
/** Refuses a run whose environment lacks a subject env name the requirements list. */
|
|
58
|
+
export declare function subjectEnvCheck<C extends string>(args: {
|
|
59
|
+
readonly requirements: readonly Requirement[];
|
|
60
|
+
readonly env: Record<string, string | undefined>;
|
|
61
|
+
readonly code: C;
|
|
62
|
+
}): LiveRefusal<C> | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Refuses a maxUsd or maxTotalUsd cap on a model with no rate in src/run/pricing.ts. The loop
|
|
65
|
+
* could not measure spend against the cap, and running uncapped would drop the protection the
|
|
66
|
+
* cap promised; the operator picks a priced model or removes the cap.
|
|
67
|
+
*/
|
|
68
|
+
export declare function unpricedCapCheck<C extends string>(args: {
|
|
69
|
+
readonly caps: {
|
|
70
|
+
readonly maxUsd?: number;
|
|
71
|
+
readonly maxTotalUsd?: number;
|
|
72
|
+
};
|
|
73
|
+
readonly model: string;
|
|
74
|
+
readonly code: C;
|
|
75
|
+
}): LiveRefusal<C> | undefined;
|