@namzu/sdk 62.14.0 → 62.15.0

package/src/eval/compare.ts

@@ -10,6 +10,8 @@ import {
 } from './bootstrap.js'
 import { UNKNOWN } from './disclosure.js'
 import { type CaseRuns, duplicateRun, groupRuns, passedEveryRun, tallyOf } from './report-stats.js'
+import { type SettingDifference, diffSettings } from './settings.js'
+import { SPLIT_FILTERS, type SplitFilter } from './split.js'
 import type { CaseResult } from './types.js'
 
 /**
@@ -31,6 +33,41 @@ export interface LoadedSuite {
 	readonly repeats: number
 	/** Whatever the file said about how the suite ran; absent in an older file. */
 	readonly facts: Readonly<Record<string, unknown>> | undefined
+	/** The settings snapshot the suite ran under; absent in a report written before there were any. */
+	readonly settings: LoadedSettings | undefined
+	/** Which split the suite ran; absent in a report written before there were any. */
+	readonly split: SplitFilter | undefined
+}
+
+export interface LoadedSettings {
+	readonly name: string
+	readonly digest: string
+	readonly values: Readonly<Record<string, unknown>>
+	/** False when the suite never asked for the settings. Absent is read as unknown. */
+	readonly read: boolean | undefined
+	/** Paths the suite asked for, `*` for the whole object; undefined when the report does not say. */
+	readonly readKeys: readonly string[] | undefined
+}
+
+function loadSettings(value: unknown): LoadedSettings | undefined {
+	if (!isRecord(value)) return undefined
+	if (typeof value.name !== 'string' || typeof value.digest !== 'string') return undefined
+	return {
+		name: value.name,
+		digest: value.digest,
+		values: isRecord(value.values) ? value.values : {},
+		read: typeof value.read === 'boolean' ? value.read : undefined,
+		readKeys: Array.isArray(value.readKeys)
+			? value.readKeys.filter((k): k is string => typeof k === 'string')
+			: undefined,
+	}
+}
+
+function loadSplit(value: unknown): SplitFilter | undefined {
+	if (!isRecord(value)) return undefined
+	return SPLIT_FILTERS.includes(value.requested as SplitFilter)
+		? (value.requested as SplitFilter)
+		: undefined
 }
 
 export interface LoadedReportFile {
@@ -81,6 +118,8 @@ export function loadEvalReportFile(json: unknown, label: string): LoadedReportFi
 			cases: groupRuns(runs),
 			repeats: typeof report.repeats === 'number' ? report.repeats : 1,
 			facts: isRecord(report.disclosure) ? report.disclosure : undefined,
+			settings: loadSettings(report.settings),
+			split: loadSplit(report.taskSplit),
 		})
 	}
 	return { label, suites }
@@ -277,10 +316,53 @@ export interface EvalComparison {
 		readonly sentence: string
 		/** Why, in plain words. */
 		readonly detail: string
+		/**
+		 * Why no winner is named although the numbers might say one. Empty when
+		 * the comparison is on the held-out split, between different settings, over
+		 * the same tasks: the one case in which it speaks.
+		 */
+		readonly withheld: readonly WithheldReason[]
 	}
 	readonly disclosure: DisclosureComparison
+	/** The settings each report ran under, and what differs between the two. */
+	readonly settings: SettingsComparison
+	/** The split each report ran. Null when a report does not say (an older one). */
+	readonly split: { readonly a: SideSplit; readonly b: SideSplit }
+}
+
+/** `mixed` when the suites of one report ran different splits. */
+export type SideSplit = SplitFilter | 'mixed' | null
+
+export interface SnapshotRef {
+	readonly name: string
+	readonly digest: string
+}
+
+export interface SettingsComparison {
+	/** Null when the report has no snapshot (it was written before reports carried one). */
+	readonly a: SnapshotRef | null
+	readonly b: SnapshotRef | null
+	/** Null when either side has no snapshot. */
+	readonly identical: boolean | null
+	/** Key by key; empty when the snapshots are identical or either is missing. */
+	readonly differences: readonly SettingDifference[]
+	/** Suites that ran under settings but never read them, so the settings cannot have moved them. */
+	readonly notRead: { readonly a: readonly string[]; readonly b: readonly string[] }
+	/** Settings that differ between the two snapshots and that no shared suite on either side read. */
+	readonly unreadKeys: readonly string[]
+	/** Whether at least one differing setting was read by a shared suite. Null when there is no difference to read. */
+	readonly anyDifferenceRead: boolean | null
 }
 
+/** Why a comparison does not name a winner, besides the numbers not supporting one. */
+export type WithheldReason =
+	| 'tune-split'
+	| 'identical-settings'
+	| 'no-snapshot'
+	| 'tasks-differ'
+	| 'not-held-out'
+	| 'settings-not-read'
+
 function sorted(refs: CaseRef[]): CaseRef[] {
 	return refs.sort((x, y) =>
 		x.suite === y.suite ? (x.case < y.case ? -1 : 1) : x.suite < y.suite ? -1 : 1,
@@ -358,7 +440,13 @@ export function compareEvalReports(
 		pValue: exactSignTestP(onlyBPasses, onlyAPasses),
 		test: 'exact two-sided sign test on the cases that flipped',
 	}
-	const verdict = decide(difference.mean, pairs.length, flips)
+	const settings = compareSettings(a, b, sharedSuiteIds(a, b))
+	const split = { a: splitOf(a), b: splitOf(b) }
+	const withheld = withheldReasons(settings, split, onlyInA.length + onlyInB.length)
+	const verdict =
+		withheld.length > 0
+			? withhold(withheld, settings, onlyInA.length, onlyInB.length, difference.mean, pairs.length)
+			: { ...decide(difference.mean, pairs.length, flips), withheld: [] as WithheldReason[] }
 	const sharedSuites = a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id))
 
 	return {
@@ -375,6 +463,130 @@ export function compareEvalReports(
 		flips,
 		verdict,
 		disclosure: compareDisclosure(a, b, sharedSuites),
+		settings,
+		split,
+	}
+}
+
+function sharedSuiteIds(a: LoadedReportFile, b: LoadedReportFile): string[] {
+	return a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id))
+}
+
+function snapshotOf(file: LoadedReportFile): LoadedSettings | undefined {
+	return file.suites.find((s) => s.settings !== undefined)?.settings
+}
+
+function splitOf(file: LoadedReportFile): SideSplit {
+	const requested = new Set(file.suites.map((s) => s.split))
+	if (requested.size === 0 || requested.has(undefined)) return null
+	return requested.size === 1 ? ([...requested][0] as SplitFilter) : 'mixed'
+}
+
+function compareSettings(
+	a: LoadedReportFile,
+	b: LoadedReportFile,
+	shared: readonly string[],
+): SettingsComparison {
+	const left = snapshotOf(a)
+	const right = snapshotOf(b)
+	const ref = (x: LoadedSettings | undefined): SnapshotRef | null =>
+		x === undefined ? null : { name: x.name, digest: x.digest }
+	const ignored = (file: LoadedReportFile) =>
+		file.suites
+			.filter(
+				(s) =>
+					shared.includes(s.id) &&
+					s.settings?.read === false &&
+					Object.keys(s.settings.values).length > 0,
+			)
+			.map((s) => s.id)
+	const identical = left && right ? left.digest === right.digest : null
+	const differences = left && right && !identical ? diffSettings(left.values, right.values) : []
+	const readers = [a, b].flatMap((file) =>
+		file.suites.filter((s) => shared.includes(s.id) && s.settings !== undefined),
+	)
+	const wasRead = (key: string): boolean =>
+		readers.some((s) => {
+			const st = s.settings as LoadedSettings
+			if (st.read === false) return false
+			// A report that does not say which keys it read is taken to have read all.
+			if (st.readKeys === undefined) return true
+			return st.readKeys.some(
+				(k) => k === '*' || k === key || key.startsWith(`${k}.`) || k.startsWith(`${key}.`),
+			)
+		})
+	const unreadKeys = differences.map((d) => d.key).filter((k) => !wasRead(k))
+	return {
+		a: ref(left),
+		b: ref(right),
+		identical,
+		differences,
+		notRead: { a: ignored(a), b: ignored(b) },
+		unreadKeys,
+		anyDifferenceRead: differences.length === 0 ? null : unreadKeys.length < differences.length,
+	}
+}
+
+function withheldReasons(
+	settings: SettingsComparison,
+	split: { a: SideSplit; b: SideSplit },
+	taskMismatches: number,
+): WithheldReason[] {
+	const out: WithheldReason[] = []
+	if (settings.a === null || settings.b === null) out.push('no-snapshot')
+	if (taskMismatches > 0) out.push('tasks-differ')
+	// The most informative reason first: identical settings say the whole
+	// difference is noise, whatever split they ran on.
+	if (settings.identical === true) out.push('identical-settings')
+	// Settings no shared suite read cannot have moved a number: whatever the
+	// difference between the runs, it is not the settings.
+	if (settings.identical === false && settings.anyDifferenceRead === false) {
+		out.push('settings-not-read')
+	}
+	const tune = split.a === 'tune' || split.b === 'tune'
+	if (tune) out.push('tune-split')
+	else if (
+		settings.a !== null &&
+		settings.b !== null &&
+		(split.a !== 'held-out' || split.b !== 'held-out')
+	) {
+		out.push('not-held-out')
+	}
+	return out
+}
+
+function short(digest: string): string {
+	return digest.slice(0, 12)
+}
+
+function withhold(
+	reasons: readonly WithheldReason[],
+	settings: SettingsComparison,
+	onlyA: number,
+	onlyB: number,
+	mean: Interval,
+	paired: number,
+): EvalComparison['verdict'] {
+	const text: Record<WithheldReason, string> = {
+		'tune-split':
+			'these are the tasks the settings were chosen on; run the held-out split to judge',
+		'identical-settings': `the two reports ran under identical settings (${settings.a?.name ?? 'unnamed'}, ${short(settings.a?.digest ?? '')}), so any difference between them is run-to-run noise`,
+		'no-snapshot':
+			'a report has no settings snapshot (it was written before reports named their settings), so what the two runs changed is not known',
+		'tasks-differ': `the two reports ran different tasks (${onlyA} only in A, ${onlyB} only in B), so their numbers are not about the same tasks`,
+		'settings-not-read':
+			'no suite the two reports share read any of the settings that differ between them, so the settings cannot have changed these numbers',
+		'not-held-out':
+			'a report ran on all tasks or a mix of splits, which include the tasks the settings were chosen on; run the held-out split to judge',
+	}
+	const numbers = mean.computed
+		? `the paired difference in mean score over ${paired} paired cases is ${signed(mean.estimate)}, 95% interval [${mean.low.toFixed(3)}, ${mean.high.toFixed(3)}]`
+		: `no interval: ${mean.reason}`
+	return {
+		claim: 'none',
+		sentence: `no winner is named: ${text[reasons[0] as WithheldReason]}`,
+		detail: `${[...reasons.slice(1).map((r) => text[r]), numbers].join('; ')}.`,
+		withheld: reasons,
 	}
 }
 
@@ -386,7 +598,11 @@ function atPrintedPrecision(value: number): number {
 	return Math.round(value * 1000) / 1000 || 0
 }
 
-function decide(mean: Interval, paired: number, flips: Flips): EvalComparison['verdict'] {
+function decide(
+	mean: Interval,
+	paired: number,
+	flips: Flips,
+): Omit<EvalComparison['verdict'], 'withheld'> {
 	const none = 'no difference can be claimed at this size'
 	if (!mean.computed) {
 		return { claim: 'none', sentence: none, detail: `no interval: ${mean.reason}.` }
@@ -453,6 +669,33 @@ export function formatEvalComparison(c: EvalComparison): string {
 		`  (${c.verdict.detail})`,
 		'',
 	]
+	const ref = (x: SnapshotRef | null) =>
+		x === null ? 'none recorded' : `"${x.name}" (sha256 ${short(x.digest)})`
+	lines.push(
+		`Settings: A ${ref(c.settings.a)}, B ${ref(c.settings.b)}. Split: A ${c.split.a ?? 'not recorded'}, B ${c.split.b ?? 'not recorded'}.`,
+	)
+	if (c.settings.identical === true) {
+		lines.push('The two runs ran under identical settings.')
+	} else if (c.settings.differences.length > 0) {
+		lines.push('What differs in the settings:')
+		const show = (v: unknown, has: boolean) => (has ? JSON.stringify(v) : 'not stated')
+		for (const x of c.settings.differences) {
+			lines.push(`  ${x.key}: A ${show(x.a, 'a' in x)}, B ${show(x.b, 'b' in x)}`)
+		}
+	}
+	for (const side of ['a', 'b'] as const) {
+		if (c.settings.notRead[side].length > 0) {
+			lines.push(
+				`Suites that ran under settings but did not read them (${side.toUpperCase()}): ${c.settings.notRead[side].join(', ')}`,
+			)
+		}
+	}
+	if (c.settings.unreadKeys.length > 0) {
+		lines.push(
+			`Settings that differ and that no shared suite read: ${c.settings.unreadKeys.join(', ')}`,
+		)
+	}
+	lines.push('')
 	const d = c.disclosure
 	if (d.differences.length === 0) {
 		lines.push('What the two reports say about how they were produced: no stated difference.')

package/src/eval/experiment.ts

@@ -1,6 +1,7 @@
 import { EvalUsageError, validateSeed } from './bootstrap.js'
 import { type SuiteFacts, buildDisclosure, describeDisclosure } from './disclosure.js'
 import {
+	clarifyDeclared,
 	describeIntervals,
 	describeSplits,
 	duplicateMessage,
@@ -9,6 +10,15 @@ import {
 	statsOver,
 } from './report-stats.js'
 import { currentEvalRunOptions } from './run-options.js'
+import { defaultSettingsSnapshot } from './settings.js'
+import {
+	HELD_OUT_FRACTION,
+	TASK_SPLIT_SEED,
+	assignSplit,
+	taskId,
+	validateSplitFilter,
+} from './split.js'
+import type { ReportTaskSplit } from './split.js'
 import type {
 	CaseResult,
 	CaseStatus,
@@ -106,8 +116,28 @@ export async function runExperiment<TInput>(
 	const alike = duplicateName(config.cases.map((c) => c.name))
 	if (alike !== undefined) throw new Error(duplicateMessage(config.name, alike))
 
+	// Which tasks run: every one, or the half assigned to one split. The
+	// assignment is a function of the task's identity, so it does not move.
+	const requested = validateSplitFilter(options.split)
+	const assigned = config.cases.map((c) => ({
+		evalCase: c,
+		...assignSplit(config.name, c.name, c.split),
+	}))
+	const chosen = assigned.filter((t) => requested === 'all' || t.split === requested)
+	const cases = chosen.map((t) => t.evalCase)
+	const taskSplit: ReportTaskSplit = {
+		requested,
+		seed: TASK_SPLIT_SEED,
+		heldOutFraction: HELD_OUT_FRACTION,
+		tasks: chosen.map((t) => ({
+			id: taskId(config.name, t.evalCase.name),
+			split: t.split,
+			declared: t.declared,
+		})),
+	}
+
 	// Case-major, so a case's repeats sit together in the report.
-	const total = config.cases.length * repeats
+	const total = cases.length * repeats
 	const results: CaseResult[] = new Array(total)
 	let cursor = 0
 
@@ -115,7 +145,7 @@ export async function runExperiment<TInput>(
 		for (;;) {
 			const index = cursor++
 			if (index >= total) return
-			const evalCase = config.cases[Math.floor(index / repeats)] as EvalCase<TInput>
+			const evalCase = cases[Math.floor(index / repeats)] as EvalCase<TInput>
 
 			const result = await evaluateCase(config, evalCase, threshold, timeoutMs)
 			result.repeat = (index % repeats) + 1
@@ -159,10 +189,16 @@ export async function runExperiment<TInput>(
 		splits: stats.splits,
 		disclosure: buildDisclosure(
 			config.disclosure,
-			{ cases: config.cases.length, repeats, seed },
+			{ cases: cases.length, repeats, seed },
 			options,
 			new Date().toISOString(),
 		),
+		settings: {
+			...(options.settings ?? defaultSettingsSnapshot()),
+			read: options.settingsUse?.read ?? false,
+			readKeys: [...(options.settingsUse?.keys ?? [])].sort(),
+		},
+		taskSplit,
 	}
 }
 
@@ -468,9 +504,27 @@ export function formatReport(report: ExperimentReport): string {
 		}
 	}
 	if (report.splits) {
-		for (const line of describeSplits(report.splits)) lines.push(`  ${line}`)
+		for (const line of describeSplits(report.splits)) {
+			// The task split is the held-out one: the line about cases a suite
+			// declared as developed against would contradict the run's own header.
+			if (report.taskSplit?.requested === 'held-out' && line.startsWith('no held-out cases'))
+				continue
+			lines.push(`  ${report.taskSplit ? clarifyDeclared(line) : line}`)
+		}
 	}
 	if (report.disclosure) lines.push(`  ${describeDisclosure(report.disclosure)}`)
+	if (report.settings) {
+		const r = report.settings
+		lines.push(
+			`  settings: "${r.name}", sha256 ${r.digest.slice(0, 12)}${r.read || Object.keys(r.values).length === 0 ? '' : ' (this suite does not read them)'}`,
+		)
+	}
+	if (report.taskSplit) {
+		const t = report.taskSplit
+		lines.push(
+			`  split: ${t.requested}, ${t.tasks.length} task${t.tasks.length === 1 ? '' : 's'} ran`,
+		)
+	}
 	lines.push('')
 
 	// Surfaced above the failures, because an inconclusive case means the

package/src/eval/index.ts

@@ -52,7 +52,33 @@ export type {
 	SuiteFacts,
 } from './disclosure.js'
 export { sdkVersion } from './disclosure.js'
-export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js'
+export {
+	currentEvalRunOptions,
+	currentEvalSettings,
+	evalSetting,
+	withEvalRunOptions,
+} from './run-options.js'
+export {
+	DEFAULT_SETTINGS_NAME,
+	SETTINGS_DIGEST_ALGORITHM,
+	buildSettingsSnapshot,
+	canonicalJson,
+	defaultSettingsSnapshot,
+	diffSettings,
+	findSecretIn,
+	validateSettingsName,
+} from './settings.js'
+export type { ReportSettings, SettingDifference, SettingsSnapshot } from './settings.js'
+export {
+	HELD_OUT_FRACTION,
+	SPLIT_FILTERS,
+	TASK_SPLIT_SEED,
+	assignSplit,
+	hashedSplit,
+	taskId,
+	validateSplitFilter,
+} from './split.js'
+export type { ReportTaskSplit, SplitFilter, TaskSplit } from './split.js'
 export type { EvalRunOptions } from './run-options.js'
 export { formatRunSummary, summarizeRun } from './run-summary.js'
 export type { RunDisclosure, RunSummary } from './run-summary.js'
@@ -67,7 +93,12 @@ export type {
 	EvalComparison,
 	Flips,
 	LoadedReportFile,
+	LoadedSettings,
 	LoadedSuite,
+	SettingsComparison,
+	SideSplit,
+	SnapshotRef,
+	WithheldReason,
 	SideNumbers,
 } from './compare.js'
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js'

package/src/eval/report-stats.ts

@@ -255,3 +255,14 @@ export function describeSplits(splits: RunStats['splits']): string[] {
 	}
 	return lines
 }
+
+/**
+ * The line about cases a suite declared as held-out is the suite's own
+ * statement. Beside the run's `split:` line it reads as a contradiction, since
+ * the hashed split puts some of the same tasks in held-out; say which is which.
+ */
+export function clarifyDeclared(line: string): string {
+	return line.startsWith('no held-out cases')
+		? `${line} (declared by the suite; the split above is the hashed one)`
+		: line
+}

package/src/eval/run-options.ts

@@ -1,4 +1,6 @@
 import { AsyncLocalStorage } from 'node:async_hooks'
+import type { SettingsSnapshot } from './settings.js'
+import type { SplitFilter } from './split.js'
 
 /**
  * Settings an operator gives a whole eval run, handed to every
@@ -24,6 +26,25 @@ export interface EvalRunOptions {
 	readonly commit?: string
 	/** Version of the suite being run, when known. */
 	readonly suiteVersion?: string
+	/**
+	 * The settings the run was given. Written into every report. A suite shapes
+	 * itself by them through {@link currentEvalSettings}; one that does not is
+	 * reported as not having read them.
+	 */
+	readonly settings?: SettingsSnapshot
+	/** Which tasks run. Absent means all of them. */
+	readonly split?: SplitFilter
+	/** Set by {@link currentEvalSettings} when a suite asks for the settings. Owned by the runner. */
+	readonly settingsUse?: { read: boolean; keys?: Set<string> }
+}
+
+/** Stands in the list of read keys for a suite that asked for the whole settings object. */
+export const READ_ALL = '*'
+
+function markRead(use: { read: boolean; keys?: Set<string> }, key: string): void {
+	const keys = use.keys ?? new Set<string>()
+	keys.add(key)
+	use.keys = keys
 }
 
 const STORE_KEY = Symbol.for('namzu.eval.runOptions')
@@ -47,3 +68,33 @@ export function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T
 export function currentEvalRunOptions(): EvalRunOptions {
 	return store().getStore() ?? {}
 }
+
+/**
+ * The values of the settings the run was given, or an empty object when it was
+ * given none. Asking marks the suite as one that read them, which its report
+ * says: a setting nobody reads cannot have changed what a suite measured.
+ */
+export function currentEvalSettings(): Readonly<Record<string, unknown>> {
+	const options = currentEvalRunOptions()
+	if (options.settingsUse) {
+		options.settingsUse.read = true
+		// The whole object was asked for: every key counts as read.
+		markRead(options.settingsUse, READ_ALL)
+	}
+	return options.settings?.values ?? {}
+}
+
+/** One setting by its dotted path, for example `limits.maxIterations`; undefined when not stated. */
+export function evalSetting(path: string): unknown {
+	const options = currentEvalRunOptions()
+	let at: unknown = options.settings?.values ?? {}
+	if (options.settingsUse) {
+		options.settingsUse.read = true
+		markRead(options.settingsUse, path)
+	}
+	for (const part of path.split('.')) {
+		if (typeof at !== 'object' || at === null || Array.isArray(at)) return undefined
+		at = (at as Record<string, unknown>)[part]
+	}
+	return at
+}

package/src/eval/run-summary.ts

@@ -2,11 +2,14 @@ import { UNKNOWN } from './disclosure.js'
 import type { RunMode } from './disclosure.js'
 import {
 	type RunStats,
+	clarifyDeclared,
 	describeIntervals,
 	describeSplits,
 	groupRuns,
 	statsOver,
 } from './report-stats.js'
+import type { SettingsSnapshot } from './settings.js'
+import type { SplitFilter } from './split.js'
 import type { ExperimentReport } from './types.js'
 
 /** What produced a whole run, across every suite in it. */
@@ -23,6 +26,10 @@ export interface RunDisclosure {
 	readonly runs: number
 	/** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
 	readonly mode: RunMode | 'mixed'
+	/** The settings the run was given. Absent from a run written before reports named them. */
+	readonly settings?: SettingsSnapshot
+	/** Which tasks ran. Absent from a run written before there were splits. */
+	readonly split?: SplitFilter
 }
 
 /** The numbers for every case of every suite taken together. */
@@ -40,6 +47,8 @@ export function summarizeRun(
 		commit: string
 		generatedAt: string
 		sdkVersion: string
+		settings?: SettingsSnapshot
+		split?: SplitFilter
 	},
 ): RunSummary {
 	// A case is named by its suite and its id: two suites may reuse a name.
@@ -74,6 +83,8 @@ export function summarizeRun(
 			cases: stats.cases,
 			runs: stats.runs,
 			mode,
+			...(context.settings ? { settings: context.settings } : {}),
+			...(context.split ? { split: context.split } : {}),
 		},
 	}
 }
@@ -96,7 +107,14 @@ export function formatRunSummary(summary: RunSummary): string {
 			`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`,
 		)
 	}
-	for (const line of describeSplits(summary.splits)) lines.push(`  ${line}`)
+	for (const line of describeSplits(summary.splits)) {
+		if (d.split === 'held-out' && line.startsWith('no held-out cases')) continue
+		lines.push(`  ${d.split ? clarifyDeclared(line) : line}`)
+	}
+	if (d.settings) {
+		lines.push(`  settings: "${d.settings.name}", sha256 ${d.settings.digest.slice(0, 12)}`)
+	}
+	if (d.split) lines.push(`  split: ${d.split}`)
 	lines.push(
 		`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`,
 	)