@namzu/sdk 62.13.0 → 62.15.0

package/src/eval/experiment.ts

@@ -1,6 +1,7 @@
 import { EvalUsageError, validateSeed } from './bootstrap.js'
 import { type SuiteFacts, buildDisclosure, describeDisclosure } from './disclosure.js'
 import {
+	clarifyDeclared,
 	describeIntervals,
 	describeSplits,
 	duplicateMessage,
@@ -9,6 +10,15 @@ import {
 	statsOver,
 } from './report-stats.js'
 import { currentEvalRunOptions } from './run-options.js'
+import { defaultSettingsSnapshot } from './settings.js'
+import {
+	HELD_OUT_FRACTION,
+	TASK_SPLIT_SEED,
+	assignSplit,
+	taskId,
+	validateSplitFilter,
+} from './split.js'
+import type { ReportTaskSplit } from './split.js'
 import type {
 	CaseResult,
 	CaseStatus,
@@ -106,8 +116,28 @@ export async function runExperiment<TInput>(
 	const alike = duplicateName(config.cases.map((c) => c.name))
 	if (alike !== undefined) throw new Error(duplicateMessage(config.name, alike))
 
+	// Which tasks run: every one, or the half assigned to one split. The
+	// assignment is a function of the task's identity, so it does not move.
+	const requested = validateSplitFilter(options.split)
+	const assigned = config.cases.map((c) => ({
+		evalCase: c,
+		...assignSplit(config.name, c.name, c.split),
+	}))
+	const chosen = assigned.filter((t) => requested === 'all' || t.split === requested)
+	const cases = chosen.map((t) => t.evalCase)
+	const taskSplit: ReportTaskSplit = {
+		requested,
+		seed: TASK_SPLIT_SEED,
+		heldOutFraction: HELD_OUT_FRACTION,
+		tasks: chosen.map((t) => ({
+			id: taskId(config.name, t.evalCase.name),
+			split: t.split,
+			declared: t.declared,
+		})),
+	}
+
 	// Case-major, so a case's repeats sit together in the report.
-	const total = config.cases.length * repeats
+	const total = cases.length * repeats
 	const results: CaseResult[] = new Array(total)
 	let cursor = 0
 
@@ -115,7 +145,7 @@ export async function runExperiment<TInput>(
 		for (;;) {
 			const index = cursor++
 			if (index >= total) return
-			const evalCase = config.cases[Math.floor(index / repeats)] as EvalCase<TInput>
+			const evalCase = cases[Math.floor(index / repeats)] as EvalCase<TInput>
 
 			const result = await evaluateCase(config, evalCase, threshold, timeoutMs)
 			result.repeat = (index % repeats) + 1
@@ -159,10 +189,16 @@ export async function runExperiment<TInput>(
 		splits: stats.splits,
 		disclosure: buildDisclosure(
 			config.disclosure,
-			{ cases: config.cases.length, repeats, seed },
+			{ cases: cases.length, repeats, seed },
 			options,
 			new Date().toISOString(),
 		),
+		settings: {
+			...(options.settings ?? defaultSettingsSnapshot()),
+			read: options.settingsUse?.read ?? false,
+			readKeys: [...(options.settingsUse?.keys ?? [])].sort(),
+		},
+		taskSplit,
 	}
 }
 
@@ -468,9 +504,27 @@ export function formatReport(report: ExperimentReport): string {
 		}
 	}
 	if (report.splits) {
-		for (const line of describeSplits(report.splits)) lines.push(`  ${line}`)
+		for (const line of describeSplits(report.splits)) {
+			// The task split is the held-out one: the line about cases a suite
+			// declared as developed against would contradict the run's own header.
+			if (report.taskSplit?.requested === 'held-out' && line.startsWith('no held-out cases'))
+				continue
+			lines.push(`  ${report.taskSplit ? clarifyDeclared(line) : line}`)
+		}
 	}
 	if (report.disclosure) lines.push(`  ${describeDisclosure(report.disclosure)}`)
+	if (report.settings) {
+		const r = report.settings
+		lines.push(
+			`  settings: "${r.name}", sha256 ${r.digest.slice(0, 12)}${r.read || Object.keys(r.values).length === 0 ? '' : ' (this suite does not read them)'}`,
+		)
+	}
+	if (report.taskSplit) {
+		const t = report.taskSplit
+		lines.push(
+			`  split: ${t.requested}, ${t.tasks.length} task${t.tasks.length === 1 ? '' : 's'} ran`,
+		)
+	}
 	lines.push('')
 
 	// Surfaced above the failures, because an inconclusive case means the

package/src/eval/index.ts

@@ -52,7 +52,33 @@ export type {
 	SuiteFacts,
 } from './disclosure.js'
 export { sdkVersion } from './disclosure.js'
-export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js'
+export {
+	currentEvalRunOptions,
+	currentEvalSettings,
+	evalSetting,
+	withEvalRunOptions,
+} from './run-options.js'
+export {
+	DEFAULT_SETTINGS_NAME,
+	SETTINGS_DIGEST_ALGORITHM,
+	buildSettingsSnapshot,
+	canonicalJson,
+	defaultSettingsSnapshot,
+	diffSettings,
+	findSecretIn,
+	validateSettingsName,
+} from './settings.js'
+export type { ReportSettings, SettingDifference, SettingsSnapshot } from './settings.js'
+export {
+	HELD_OUT_FRACTION,
+	SPLIT_FILTERS,
+	TASK_SPLIT_SEED,
+	assignSplit,
+	hashedSplit,
+	taskId,
+	validateSplitFilter,
+} from './split.js'
+export type { ReportTaskSplit, SplitFilter, TaskSplit } from './split.js'
 export type { EvalRunOptions } from './run-options.js'
 export { formatRunSummary, summarizeRun } from './run-summary.js'
 export type { RunDisclosure, RunSummary } from './run-summary.js'
@@ -67,7 +93,12 @@ export type {
 	EvalComparison,
 	Flips,
 	LoadedReportFile,
+	LoadedSettings,
 	LoadedSuite,
+	SettingsComparison,
+	SideSplit,
+	SnapshotRef,
+	WithheldReason,
 	SideNumbers,
 } from './compare.js'
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js'

package/src/eval/report-stats.ts

@@ -255,3 +255,14 @@ export function describeSplits(splits: RunStats['splits']): string[] {
 	}
 	return lines
 }
+
+/**
+ * The line about cases a suite declared as held-out is the suite's own
+ * statement. Beside the run's `split:` line it reads as a contradiction, since
+ * the hashed split puts some of the same tasks in held-out; say which is which.
+ */
+export function clarifyDeclared(line: string): string {
+	return line.startsWith('no held-out cases')
+		? `${line} (declared by the suite; the split above is the hashed one)`
+		: line
+}

package/src/eval/run-options.ts

@@ -1,4 +1,6 @@
 import { AsyncLocalStorage } from 'node:async_hooks'
+import type { SettingsSnapshot } from './settings.js'
+import type { SplitFilter } from './split.js'
 
 /**
  * Settings an operator gives a whole eval run, handed to every
@@ -24,6 +26,25 @@ export interface EvalRunOptions {
 	readonly commit?: string
 	/** Version of the suite being run, when known. */
 	readonly suiteVersion?: string
+	/**
+	 * The settings the run was given. Written into every report. A suite shapes
+	 * itself by them through {@link currentEvalSettings}; one that does not is
+	 * reported as not having read them.
+	 */
+	readonly settings?: SettingsSnapshot
+	/** Which tasks run. Absent means all of them. */
+	readonly split?: SplitFilter
+	/** Set by {@link currentEvalSettings} when a suite asks for the settings. Owned by the runner. */
+	readonly settingsUse?: { read: boolean; keys?: Set<string> }
+}
+
+/** Stands in the list of read keys for a suite that asked for the whole settings object. */
+export const READ_ALL = '*'
+
+function markRead(use: { read: boolean; keys?: Set<string> }, key: string): void {
+	const keys = use.keys ?? new Set<string>()
+	keys.add(key)
+	use.keys = keys
 }
 
 const STORE_KEY = Symbol.for('namzu.eval.runOptions')
@@ -47,3 +68,33 @@ export function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T
 export function currentEvalRunOptions(): EvalRunOptions {
 	return store().getStore() ?? {}
 }
+
+/**
+ * The values of the settings the run was given, or an empty object when it was
+ * given none. Asking marks the suite as one that read them, which its report
+ * says: a setting nobody reads cannot have changed what a suite measured.
+ */
+export function currentEvalSettings(): Readonly<Record<string, unknown>> {
+	const options = currentEvalRunOptions()
+	if (options.settingsUse) {
+		options.settingsUse.read = true
+		// The whole object was asked for: every key counts as read.
+		markRead(options.settingsUse, READ_ALL)
+	}
+	return options.settings?.values ?? {}
+}
+
+/** One setting by its dotted path, for example `limits.maxIterations`; undefined when not stated. */
+export function evalSetting(path: string): unknown {
+	const options = currentEvalRunOptions()
+	let at: unknown = options.settings?.values ?? {}
+	if (options.settingsUse) {
+		options.settingsUse.read = true
+		markRead(options.settingsUse, path)
+	}
+	for (const part of path.split('.')) {
+		if (typeof at !== 'object' || at === null || Array.isArray(at)) return undefined
+		at = (at as Record<string, unknown>)[part]
+	}
+	return at
+}

package/src/eval/run-summary.ts

@@ -2,11 +2,14 @@ import { UNKNOWN } from './disclosure.js'
 import type { RunMode } from './disclosure.js'
 import {
 	type RunStats,
+	clarifyDeclared,
 	describeIntervals,
 	describeSplits,
 	groupRuns,
 	statsOver,
 } from './report-stats.js'
+import type { SettingsSnapshot } from './settings.js'
+import type { SplitFilter } from './split.js'
 import type { ExperimentReport } from './types.js'
 
 /** What produced a whole run, across every suite in it. */
@@ -23,6 +26,10 @@ export interface RunDisclosure {
 	readonly runs: number
 	/** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
 	readonly mode: RunMode | 'mixed'
+	/** The settings the run was given. Absent from a run written before reports named them. */
+	readonly settings?: SettingsSnapshot
+	/** Which tasks ran. Absent from a run written before there were splits. */
+	readonly split?: SplitFilter
 }
 
 /** The numbers for every case of every suite taken together. */
@@ -40,6 +47,8 @@ export function summarizeRun(
 		commit: string
 		generatedAt: string
 		sdkVersion: string
+		settings?: SettingsSnapshot
+		split?: SplitFilter
 	},
 ): RunSummary {
 	// A case is named by its suite and its id: two suites may reuse a name.
@@ -74,6 +83,8 @@ export function summarizeRun(
 			cases: stats.cases,
 			runs: stats.runs,
 			mode,
+			...(context.settings ? { settings: context.settings } : {}),
+			...(context.split ? { split: context.split } : {}),
 		},
 	}
 }
@@ -96,7 +107,14 @@ export function formatRunSummary(summary: RunSummary): string {
 			`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`,
 		)
 	}
-	for (const line of describeSplits(summary.splits)) lines.push(`  ${line}`)
+	for (const line of describeSplits(summary.splits)) {
+		if (d.split === 'held-out' && line.startsWith('no held-out cases')) continue
+		lines.push(`  ${d.split ? clarifyDeclared(line) : line}`)
+	}
+	if (d.settings) {
+		lines.push(`  settings: "${d.settings.name}", sha256 ${d.settings.digest.slice(0, 12)}`)
+	}
+	if (d.split) lines.push(`  split: ${d.split}`)
 	lines.push(
 		`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`,
 	)

package/src/eval/settings.ts

@@ -0,0 +1,225 @@
+import { createHash } from 'node:crypto'
+import { EvalUsageError } from './bootstrap.js'
+
+/**
+ * The settings an eval run ran under, as a named, canonical, hashed record.
+ *
+ * Two reports of one suite say little about a harness setting unless each says
+ * exactly which settings it ran under. The snapshot is that statement: a name a
+ * person chose (or `default`), a digest of the values, and the values. The
+ * digest is taken over the values alone, in a canonical form, so it is the same
+ * whatever order the keys were written in and whatever the snapshot is called.
+ *
+ * Secrets never enter one. {@link buildSettingsSnapshot} refuses a value that
+ * names a credential or looks like one, so a report written from a snapshot
+ * cannot carry a key even if a caller passed it by mistake.
+ */
+export interface SettingsSnapshot {
+	/** What the person called it; the file's name when it came from a file. */
+	readonly name: string
+	/** SHA-256 of the canonical JSON of {@link values}, in hex. */
+	readonly digest: string
+	/** How the digest was made, so a later version can change it without a silent mismatch. */
+	readonly algorithm: typeof SETTINGS_DIGEST_ALGORITHM
+	/** The settings that differ from the defaults: whatever was stated. Empty for `default`. */
+	readonly values: Readonly<Record<string, unknown>>
+}
+
+export const SETTINGS_DIGEST_ALGORITHM = 'sha256-canonical-json-1'
+
+/** The name of the snapshot of a run that was given no settings. */
+export const DEFAULT_SETTINGS_NAME = 'default'
+
+/** What a suite that read the settings says about it, written into its report. */
+export interface ReportSettings extends SettingsSnapshot {
+	/**
+	 * Whether the suite asked for the settings while it ran. A suite that never
+	 * does cannot have been shaped by them, so a difference between two snapshots
+	 * says nothing about its numbers.
+	 */
+	readonly read: boolean
+	/**
+	 * The dotted paths the suite asked for, or `*` when it asked for the whole
+	 * object. Absent from a report that predates it, which is read as every key.
+	 */
+	readonly readKeys?: readonly string[]
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+/**
+ * Canonical JSON: keys sorted at every level, no whitespace, array order kept.
+ * A value JSON cannot carry (`undefined` inside an array, a non-finite number,
+ * a function) is refused, because a digest over a value that cannot be written
+ * back would name something nobody can reproduce. An `undefined` object field
+ * is the same as an absent one and is dropped.
+ */
+export function canonicalJson(value: unknown): string {
+	if (value === null || typeof value === 'string' || typeof value === 'boolean') {
+		return JSON.stringify(value)
+	}
+	if (typeof value === 'number') {
+		if (!Number.isFinite(value)) {
+			throw new EvalUsageError(`a setting must be a finite number; received ${String(value)}`)
+		}
+		return JSON.stringify(value)
+	}
+	if (Array.isArray(value)) {
+		return `[${value
+			.map((item) => {
+				if (item === undefined) throw new EvalUsageError('a setting list cannot hold undefined')
+				return canonicalJson(item)
+			})
+			.join(',')}]`
+	}
+	if (isRecord(value)) {
+		const parts: string[] = []
+		for (const key of Object.keys(value).sort()) {
+			const item = value[key]
+			if (item === undefined) continue
+			parts.push(`${JSON.stringify(key)}:${canonicalJson(item)}`)
+		}
+		return `{${parts.join(',')}}`
+	}
+	throw new EvalUsageError(`a setting must be plain JSON; received ${typeof value}`)
+}
+
+/** Names that hold a credential. `maxResponseTokens` and `tokenBudget` are not among them. */
+const SECRET_KEY =
+	/^(?:token|secret|password|passwd|credentials?|authorization|cookie|bearer)$|api[-_]?key|private[-_]?key|secret|passw(?:or)?d|(?:access|auth|refresh|bearer|session|id)[-_]?token|client[-_]?secret/i
+
+/** Shapes of the keys and tokens providers hand out. */
+const SECRET_VALUE = [
+	// A password written into a URL: scheme://user:password@host
+	/[A-Za-z][A-Za-z0-9+.-]*:\/\/[^/\s:@]+:[^/\s@]+@/,
+	/\bsk-[A-Za-z0-9_-]{16,}/,
+	/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9_]{20,}/,
+	/\bxox[abprs]-[A-Za-z0-9-]{10,}/,
+	/\bAKIA[0-9A-Z]{16}\b/,
+	/\bAIza[0-9A-Za-z_-]{30,}/,
+	/\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i,
+	/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/,
+	/-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+]
+
+/**
+ * The path of the first thing in `values` that is, or names, a credential, or
+ * undefined when there is none. The path never includes the secret itself.
+ */
+export function findSecretIn(values: unknown, path = ''): string | undefined {
+	if (typeof values === 'string') {
+		return SECRET_VALUE.some((pattern) => pattern.test(values)) ? path || '(value)' : undefined
+	}
+	if (Array.isArray(values)) {
+		for (const [index, item] of values.entries()) {
+			const hit = findSecretIn(item, `${path}[${index}]`)
+			if (hit !== undefined) return hit
+		}
+		return undefined
+	}
+	if (isRecord(values)) {
+		for (const key of Object.keys(values).sort()) {
+			const here = path === '' ? key : `${path}.${key}`
+			// A setting that names the environment variable holding a credential
+			// (`apiKeyEnv`) holds a name, not a secret, but is still refused: the
+			// snapshot has no business carrying where a credential lives.
+			if (SECRET_KEY.test(key)) return here
+			const hit = findSecretIn(values[key], here)
+			if (hit !== undefined) return hit
+		}
+	}
+	return undefined
+}
+
+/** A snapshot name is plain: it is printed in a comparison and used as a file label. */
+export function validateSettingsName(name: string): string {
+	if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(name)) {
+		throw new EvalUsageError(
+			`a settings name is 1 to 64 letters, digits, dots, dashes or underscores, starting with a letter or digit; received ${JSON.stringify(name)}`,
+		)
+	}
+	return name
+}
+
+/**
+ * Build a snapshot. Throws an {@link EvalUsageError} naming the path of a
+ * secret, a value JSON cannot carry, or a name that is not plain. A snapshot
+ * called `default` must have no values: `default` is the name of "no settings
+ * stated", and a snapshot with values under it would say the opposite.
+ */
+export function buildSettingsSnapshot(
+	name: string,
+	values: Readonly<Record<string, unknown>> = {},
+): SettingsSnapshot {
+	validateSettingsName(name)
+	if (!isRecord(values)) throw new EvalUsageError('settings must be an object of settings')
+	const secret = findSecretIn(values)
+	if (secret !== undefined) {
+		throw new EvalUsageError(
+			`settings "${secret}" is, or names, a credential. A credential never goes into an eval settings snapshot, because the snapshot is written into the report.`,
+		)
+	}
+	const canonical = canonicalJson(values)
+	if (name === DEFAULT_SETTINGS_NAME && canonical !== '{}') {
+		throw new EvalUsageError(
+			`the name "${DEFAULT_SETTINGS_NAME}" is for a run with no settings stated; give these settings another name`,
+		)
+	}
+	return {
+		name,
+		digest: createHash('sha256').update(canonical).digest('hex'),
+		algorithm: SETTINGS_DIGEST_ALGORITHM,
+		// A copy through the canonical form, so what is stored is what was hashed.
+		values: JSON.parse(canonical) as Record<string, unknown>,
+	}
+}
+
+/** The snapshot of a run given no settings. */
+export function defaultSettingsSnapshot(): SettingsSnapshot {
+	return buildSettingsSnapshot(DEFAULT_SETTINGS_NAME, {})
+}
+
+export interface SettingDifference {
+	/** Dotted path of the setting; a list is one value. */
+	readonly key: string
+	/** Absent on the side that does not state it. */
+	readonly a?: unknown
+	readonly b?: unknown
+}
+
+function leaves(value: unknown, path: string, into: Map<string, unknown>): void {
+	if (isRecord(value) && Object.keys(value).length > 0) {
+		for (const key of Object.keys(value))
+			leaves(value[key], path === '' ? key : `${path}.${key}`, into)
+		return
+	}
+	into.set(path, value)
+}
+
+/** What differs between two snapshots' values, key by key, sorted by key. */
+export function diffSettings(
+	a: Readonly<Record<string, unknown>>,
+	b: Readonly<Record<string, unknown>>,
+): SettingDifference[] {
+	const left = new Map<string, unknown>()
+	const right = new Map<string, unknown>()
+	leaves(a, '', left)
+	leaves(b, '', right)
+	left.delete('')
+	right.delete('')
+	const keys = [...new Set([...left.keys(), ...right.keys()])].sort()
+	const out: SettingDifference[] = []
+	for (const key of keys) {
+		const inA = left.has(key)
+		const inB = right.has(key)
+		if (inA && inB && canonicalJson(left.get(key)) === canonicalJson(right.get(key))) continue
+		out.push({
+			key,
+			...(inA ? { a: left.get(key) } : {}),
+			...(inB ? { b: right.get(key) } : {}),
+		})
+	}
+	return out
+}

package/src/eval/split.ts

@@ -0,0 +1,83 @@
+import { createHash } from 'node:crypto'
+import { EvalUsageError } from './bootstrap.js'
+import type { EvalSplit } from './types.js'
+
+/**
+ * Which tasks a harness setting may be chosen on, and which it is judged on.
+ *
+ * A setting tuned against some tasks and then judged on the same tasks has been
+ * shown to fit them, and nothing more. Every task belongs to one of two splits:
+ * `tune`, the tasks to choose settings on, and `held-out`, the tasks to judge
+ * the choice on. The split of a task is a function of its identity and a fixed
+ * seed, so the same task is in the same split on every machine, in every run,
+ * whoever runs it. A suite may state a task's split itself, and that wins.
+ */
+export type TaskSplit = 'tune' | 'held-out'
+
+/** What `namzu eval --split` selects. */
+export type SplitFilter = TaskSplit | 'all'
+
+export const SPLIT_FILTERS: readonly SplitFilter[] = ['tune', 'held-out', 'all']
+
+/** The seed of the assignment. Fixed in the kernel and written into every report. */
+export const TASK_SPLIT_SEED = 'namzu-eval-split-1'
+
+/** The share of tasks assigned to `held-out`, written into every report. */
+export const HELD_OUT_FRACTION = 0.5
+
+export const TASK_SPLIT_ALGORITHM =
+	'sha256(seed, suite, task) as a fraction; held-out when below the share'
+
+/** The id of a task: its suite and its case name, which is unique within the suite. */
+export function taskId(suite: string, name: string): string {
+	return `${suite}/${name}`
+}
+
+/** The split a seeded hash assigns to a task, whatever a suite declares. */
+export function hashedSplit(suite: string, name: string): TaskSplit {
+	const digest = createHash('sha256')
+		.update(`${TASK_SPLIT_SEED}\u0000${suite}\u0000${name}`)
+		.digest()
+	// Six bytes: 48 bits, exact in a double.
+	const fraction = digest.readUIntBE(0, 6) / 2 ** 48
+	return fraction < HELD_OUT_FRACTION ? 'held-out' : 'tune'
+}
+
+/**
+ * The split of a task. A declared split wins; `development`, the name a case
+ * declares for the tasks a harness was shaped against, is `tune`.
+ */
+export function assignSplit(
+	suite: string,
+	name: string,
+	declared?: EvalSplit,
+): { split: TaskSplit; declared: boolean } {
+	if (declared === 'held-out') return { split: 'held-out', declared: true }
+	if (declared === 'development') return { split: 'tune', declared: true }
+	return { split: hashedSplit(suite, name), declared: false }
+}
+
+export function validateSplitFilter(value: string | undefined): SplitFilter {
+	if (value === undefined) return 'all'
+	if (!SPLIT_FILTERS.includes(value as SplitFilter)) {
+		throw new EvalUsageError(
+			`split must be "tune", "held-out" or "all"; received ${JSON.stringify(value)}`,
+		)
+	}
+	return value as SplitFilter
+}
+
+/** The tasks a suite ran, and how they were assigned. Written into the suite's report. */
+export interface ReportTaskSplit {
+	/** What the run asked for. */
+	readonly requested: SplitFilter
+	readonly seed: typeof TASK_SPLIT_SEED
+	/** The share of tasks assigned to `held-out` when a suite does not say. */
+	readonly heldOutFraction: number
+	/** The tasks that ran, in the suite's order, each with its split. */
+	readonly tasks: readonly {
+		readonly id: string
+		readonly split: TaskSplit
+		readonly declared: boolean
+	}[]
+}

package/src/eval/types.ts

@@ -3,6 +3,8 @@ import type { StepResult } from '../types/session/step.js'
 import type { CaseIntervals } from './bootstrap.js'
 import type { ReportDisclosure } from './disclosure.js'
 import type { RepeatReliability, SplitSummary } from './report-stats.js'
+import type { ReportSettings } from './settings.js'
+import type { ReportTaskSplit } from './split.js'
 import type { ScoreUncertainty } from './uncertainty.js'
 
 /**
@@ -239,6 +241,16 @@ export interface ExperimentReport {
 	splits?: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>
 	/** What produced this report. */
 	disclosure?: ReportDisclosure
+	/**
+	 * The settings snapshot the suite ran under: name, digest, values, and
+	 * whether the suite read them. Absent from a report that predates it.
+	 */
+	settings?: ReportSettings
+	/**
+	 * Which split of the tasks ran, how each task was assigned, and the ids that
+	 * ran. Absent from a report that predates it.
+	 */
+	taskSplit?: ReportTaskSplit
 }
 
 /** How the repeats of one case went. */

package/src/execution/local.ts

@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process'
 import { existsSync } from 'node:fs'
 import { resolve } from 'node:path'
 import { killTree } from '../process/kill-tree.js'
+import { withSessionChildMarker } from '../process/session-child.js'
 import type { LocalExecutionContextConfig } from '../types/connector/index.js'
 import type {
 	CommandExecutor,
@@ -161,7 +162,7 @@ export class LocalExecutionContext extends BaseExecutionContext implements Comma
 		}
 
 		const cwd = options?.cwd ? resolve(this.cwd, options.cwd) : this.cwd
-		const env = { ...process.env, ...this.envVars, ...options?.env }
+		const env = withSessionChildMarker({ ...process.env, ...this.envVars, ...options?.env })
 		const timeoutMs = options?.timeoutMs ?? LOCAL_COMMAND_DEFAULT_TIMEOUT_MS
 		if (!Number.isInteger(timeoutMs) || timeoutMs < 0) {
 			throw new RangeError(