@namzu/sdk 36.0.0 → 37.0.0

package/src/eval/harness-verification.ts

@@ -0,0 +1,270 @@
+import type { CaseResult } from './types.js'
+
+/** A host-owned outcome, with enough identity to check pairing and cite its trace. */
+export interface HarnessTrial {
+	taskId: string
+	trial: number
+	/** Environment/seed/model/limits fingerprint. Equal within a pair, fresh in confirmation. */
+	conditions: string
+	trajectoryId: string
+	result: CaseResult
+}
+
+export interface HarnessAttribution {
+	taskId: string
+	effect: 'improvement' | 'regression' | 'unresolved'
+	reason: string
+	baselineTrajectories: readonly string[]
+	candidateTrajectories: readonly string[]
+}
+
+export interface HarnessVerificationBatch {
+	baselineRevision: string
+	candidateRevision: string
+	baseline: readonly HarnessTrial[]
+	candidate: readonly HarnessTrial[]
+	/** Supplied by an independent trace reviewer, never inferred from scores alone. */
+	attributions: readonly HarnessAttribution[]
+}
+
+export type HarnessBehaviorStatus =
+	| 'recovered'
+	| 'stable-success'
+	| 'regressed'
+	| 'still-failing'
+	| 'mixed'
+	| 'inconclusive'
+
+export interface HarnessTaskComparison {
+	taskId: string
+	status: HarnessBehaviorStatus
+	baselinePasses: number
+	candidatePasses: number
+	trials: number
+}
+
+export interface HarnessComparison {
+	tasks: readonly HarnessTaskComparison[]
+	/** Paired trial pass-rate difference, not held-out pass@1 or causal evidence. */
+	passRateDelta: number | null
+	positiveEvidence: readonly string[]
+	regressions: readonly string[]
+	unresolved: readonly string[]
+	/** Actual recorded consumption from BOTH sides, including unsuccessful runs. */
+	usage: { rollouts: number; tokens: number; costUsd: number; durationMs: number }
+}
+
+function nonempty(value: string): boolean {
+	return typeof value === 'string' && value.trim().length > 0
+}
+
+function key(trial: HarnessTrial): string {
+	return JSON.stringify([trial.taskId, trial.trial])
+}
+
+function indexTrials(trials: readonly HarnessTrial[]): Map<string, HarnessTrial> {
+	const result = new Map<string, HarnessTrial>()
+	const traces = new Set<string>()
+	for (const trial of trials) {
+		if (
+			!nonempty(trial.taskId) ||
+			!nonempty(trial.conditions) ||
+			!nonempty(trial.trajectoryId) ||
+			!Number.isSafeInteger(trial.trial) ||
+			trial.trial < 0
+		)
+			throw new Error(
+				'A harness trial needs a task, nonnegative trial index, conditions and trajectory ID.',
+			)
+		if (result.has(key(trial)) || traces.has(trial.trajectoryId))
+			throw new Error('Duplicate harness trial or trajectory ID.')
+		result.set(key(trial), trial)
+		traces.add(trial.trajectoryId)
+	}
+	return result
+}
+
+function measured(trial: HarnessTrial): boolean {
+	const { result } = trial
+	const scores = Object.values(result.scores)
+	return (
+		!result.run.error &&
+		(result.status === 'passed' || result.status === 'failed') &&
+		scores.length > 0 &&
+		scores.every((s) => !s.unavailable && Number.isFinite(s.score) && s.score >= 0 && s.score <= 1)
+	)
+}
+
+/** HarnessLens Table 6 comparison plus explicit, trace-linked attribution. No model calls. */
+export function compareHarnessTrials(batch: HarnessVerificationBatch): HarnessComparison {
+	if (
+		!nonempty(batch.baselineRevision) ||
+		!nonempty(batch.candidateRevision) ||
+		batch.baselineRevision === batch.candidateRevision
+	)
+		throw new Error('Distinct baseline and candidate revisions are required.')
+	const baseline = indexTrials(batch.baseline)
+	const candidate = indexTrials(batch.candidate)
+	if (!baseline.size || baseline.size !== candidate.size)
+		throw new Error('Complete paired trials are required.')
+	const baselineTraces = new Set(batch.baseline.map((t) => t.trajectoryId))
+	if (batch.candidate.some((t) => baselineTraces.has(t.trajectoryId)))
+		throw new Error('Baseline and candidate must have distinct trajectories.')
+	const groups = new Map<string, HarnessTrial[]>()
+	for (const [id, before] of baseline) {
+		const after = candidate.get(id)
+		if (!after || before.conditions !== after.conditions)
+			throw new Error('Harness trial conditions do not match.')
+		const group = groups.get(before.taskId) ?? []
+		group.push(before)
+		groups.set(before.taskId, group)
+	}
+	const tasks: HarnessTaskComparison[] = []
+	for (const [taskId, before] of groups) {
+		if (new Set(before.map((t) => t.conditions)).size !== before.length)
+			throw new Error('Repeated trials need distinct controlled conditions.')
+		const after = before.map((t) => candidate.get(key(t)) as HarnessTrial)
+		const b = before.filter((t) => t.result.status === 'passed').length
+		const c = after.filter((t) => t.result.status === 'passed').length
+		const status: HarnessBehaviorStatus = ![...before, ...after].every(measured)
+			? 'inconclusive'
+			: b === 0 && c > 0
+				? 'recovered'
+				: b > 0 && c === after.length
+					? 'stable-success'
+					: b > 0 && c === 0
+						? 'regressed'
+						: b === 0 && c === 0
+							? 'still-failing'
+							: 'mixed'
+		tasks.push({ taskId, status, baselinePasses: b, candidatePasses: c, trials: before.length })
+	}
+	const positive = new Set<string>()
+	const regressions = new Set<string>()
+	const unresolved = new Set(
+		tasks.filter((t) => t.status === 'inconclusive' || t.status === 'mixed').map((t) => t.taskId),
+	)
+	// Conservative Namzu policy: even an unattributed observed regression blocks promotion.
+	for (const task of tasks) if (task.status === 'regressed') regressions.add(task.taskId)
+	for (const evidence of batch.attributions) {
+		const task = tasks.find((t) => t.taskId === evidence.taskId)
+		const cites = (ids: readonly string[], trials: readonly HarnessTrial[]) =>
+			ids.length > 0 &&
+			ids.every((id) => trials.some((t) => t.taskId === evidence.taskId && t.trajectoryId === id))
+		if (
+			!task ||
+			!nonempty(evidence.reason) ||
+			!cites(evidence.baselineTrajectories, batch.baseline) ||
+			!cites(evidence.candidateTrajectories, batch.candidate) ||
+			!['improvement', 'regression', 'unresolved'].includes(evidence.effect)
+		)
+			throw new Error('Attribution must explain and cite both sides of an existing task.')
+		if (evidence.effect === 'regression') regressions.add(task.taskId)
+		else if (evidence.effect === 'unresolved') unresolved.add(task.taskId)
+		else if (
+			task.status === 'recovered' ||
+			(task.status === 'stable-success' && task.candidatePasses > task.baselinePasses)
+		)
+			positive.add(task.taskId)
+	}
+	const usage = { rollouts: baseline.size * 2, tokens: 0, costUsd: 0, durationMs: 0 }
+	for (const trial of [...batch.baseline, ...batch.candidate]) {
+		const values = [
+			trial.result.run.totalTokens,
+			trial.result.run.totalCostUsd,
+			trial.result.run.durationMs,
+		]
+		if (values.some((n) => !Number.isFinite(n) || n < 0))
+			throw new Error('Invalid recorded harness usage.')
+		usage.tokens += trial.result.run.totalTokens
+		usage.costUsd += trial.result.run.totalCostUsd
+		usage.durationMs += trial.result.run.durationMs
+	}
+	return {
+		tasks,
+		positiveEvidence: [...positive],
+		regressions: [...regressions],
+		unresolved: [...unresolved],
+		usage,
+		passRateDelta: tasks.some((t) => t.status === 'inconclusive')
+			? null
+			: tasks.reduce((sum, t) => sum + t.candidatePasses - t.baselinePasses, 0) / baseline.size,
+	}
+}
+
+export interface HarnessReview {
+	decision: 'accept' | 'reject' | 'inconclusive'
+	reason: string
+	verification: HarnessComparison
+	confirmation?: HarnessComparison
+}
+
+/**
+ * Evaluate recorded evidence; never applies a patch. Requires at least five tasks,
+ * exactly two trials per task, and a same-size confirmation with at most two reused tasks.
+ * Hosts own isolated execution, task selection, scoring and independent trace review.
+ */
+export function reviewHarnessCandidate(
+	verification: HarnessVerificationBatch,
+	confirmation?: HarnessVerificationBatch,
+): HarnessReview {
+	const first = compareHarnessTrials(verification)
+	const second = confirmation ? compareHarnessTrials(confirmation) : undefined
+	const result = (decision: HarnessReview['decision'], reason: string): HarnessReview => ({
+		decision,
+		reason,
+		verification: first,
+		...(second ? { confirmation: second } : {}),
+	})
+	const sufficient = (report: HarnessComparison) =>
+		report.tasks.length >= 5 && report.tasks.every((t) => t.trials === 2)
+	if (!sufficient(first))
+		return result(
+			'inconclusive',
+			'Verification requires at least five distinct tasks and two paired trials per task.',
+		)
+	if (first.regressions.length || second?.regressions.length)
+		return result('reject', 'Observed or attributed regression blocks promotion.')
+	if (first.unresolved.length || second?.unresolved.length)
+		return result('inconclusive', 'Incomplete or ambiguous behavioral evidence.')
+	if (!first.positiveEvidence.length)
+		return result(
+			'reject',
+			'A metric increase or preservation alone is not attributable improvement.',
+		)
+	if (!confirmation || !second)
+		return result('inconclusive', 'A complete fresh confirmation round is required.')
+	if (
+		verification.baselineRevision !== confirmation.baselineRevision ||
+		verification.candidateRevision !== confirmation.candidateRevision
+	)
+		return result('inconclusive', 'Revisions changed between verification and confirmation.')
+	const firstTasks = new Set(first.tasks.map((t) => t.taskId))
+	if (
+		!sufficient(second) ||
+		second.tasks.length !== first.tasks.length ||
+		second.tasks.filter((t) => firstTasks.has(t.taskId)).length > 2
+	)
+		return result(
+			'inconclusive',
+			'Confirmation must use the same batch size with at most two reused tasks.',
+		)
+	const conditions = new Set(verification.baseline.map((t) => t.conditions))
+	const traces = new Set(
+		[...verification.baseline, ...verification.candidate].map((t) => t.trajectoryId),
+	)
+	if (
+		confirmation.baseline.some((t) => conditions.has(t.conditions)) ||
+		[...confirmation.baseline, ...confirmation.candidate].some((t) => traces.has(t.trajectoryId))
+	)
+		return result('inconclusive', 'Confirmation requires fresh conditions and trajectories.')
+	if (!second.positiveEvidence.length || second.passRateDelta === null || second.passRateDelta <= 0)
+		return result(
+			'reject',
+			'Confirmation needs attributable improvement and a positive primary-metric difference.',
+		)
+	return result(
+		'accept',
+		'Both rounds support improvement without observed regression; confirmation improves the paired pass rate.',
+	)
+}

package/src/eval/index.ts

@@ -1,4 +1,14 @@
 export { evalRunFromQuery, evalRunFromRun } from './from-run.js'
+export { compareHarnessTrials, reviewHarnessCandidate } from './harness-verification.js'
+export type {
+	HarnessTrial,
+	HarnessAttribution,
+	HarnessVerificationBatch,
+	HarnessBehaviorStatus,
+	HarnessTaskComparison,
+	HarnessComparison,
+	HarnessReview,
+} from './harness-verification.js'
 export { formatReport, runExperiment } from './experiment.js'
 export type { ExperimentConfig } from './experiment.js'
 export { judgeScorer } from './judge.js'

package/src/execution/code-runtime/json-value.ts

@@ -0,0 +1,82 @@
+/**
+ * Serialize application data before sending it to the interpreter.
+ *
+ * JSON.stringify alone can allocate an arbitrarily large intermediate string.
+ * This writer accounts for each piece, refuses cycles and non-JSON values, and
+ * limits nesting. It never installs application objects in the guest realm.
+ * Undefined is supported only as a top-level result, represented out of band.
+ */
+export function encodeCodeValue(value: unknown, maxBytes: number): string | undefined {
+	if (value === undefined) return undefined
+	const pieces: string[] = []
+	const ancestors = new Set<object>()
+	let bytes = 0
+
+	const append = (text: string): void => {
+		bytes += Buffer.byteLength(text)
+		if (bytes > maxBytes) throw new Error(`Code runtime value exceeds ${maxBytes} bytes.`)
+		pieces.push(text)
+	}
+	const quote = (text: string): void => {
+		if (text.length > maxBytes - bytes) {
+			throw new Error(`Code runtime value exceeds ${maxBytes} bytes.`)
+		}
+		append(JSON.stringify(text))
+	}
+	const visit = (item: unknown, depth: number): void => {
+		if (depth > 64) throw new Error('Code runtime value exceeds 64 levels of nesting.')
+		if (item === null) {
+			append('null')
+			return
+		}
+		if (typeof item === 'string') {
+			quote(item)
+			return
+		}
+		if (typeof item === 'boolean') {
+			append(String(item))
+			return
+		}
+		if (typeof item === 'number' && Number.isFinite(item)) {
+			append(String(item))
+			return
+		}
+		if (typeof item !== 'object') throw new Error('Code runtime values must be JSON-safe.')
+		if (ancestors.has(item)) throw new Error('Code runtime values must not contain cycles.')
+		const array = Array.isArray(item)
+		if (
+			!array &&
+			Object.getPrototypeOf(item) !== Object.prototype &&
+			Object.getPrototypeOf(item) !== null
+		) {
+			throw new Error('Code runtime values must contain only plain objects and arrays.')
+		}
+		ancestors.add(item)
+		if (array) {
+			if (item.length > maxBytes) throw new Error(`Code runtime value exceeds ${maxBytes} bytes.`)
+			append('[')
+			for (let index = 0; index < item.length; index++) {
+				if (index > 0) append(',')
+				visit(item[index], depth + 1)
+			}
+			append(']')
+		} else {
+			append('{')
+			const keys = Object.keys(item)
+			for (const [index, key] of keys.entries()) {
+				if (index > 0) append(',')
+				quote(key)
+				append(':')
+				const descriptor = Object.getOwnPropertyDescriptor(item, key)
+				if (!descriptor || !('value' in descriptor)) {
+					throw new Error('Code runtime values must not contain accessors.')
+				}
+				visit(descriptor.value, depth + 1)
+			}
+			append('}')
+		}
+		ancestors.delete(item)
+	}
+	visit(value, 0)
+	return pieces.join('')
+}

package/src/execution/code-runtime/worker-program.ts

@@ -0,0 +1,239 @@
+/**
+ * Trusted worker bootstrap. Only vm.evalCode executes the untrusted program.
+ * Kept as source so installed/bundled SDKs need no separate worker entry file.
+ * No Node object or callback is injected into QuickJS: its native functions
+ * exchange bounded strings and interpreter-owned promise handles.
+ */
+export const QUICKJS_WORKER_SOURCE = `
+const { parentPort, workerData } = require('node:worker_threads')
+const { newQuickJSWASMModuleFromVariant, newVariant } = require(workerData.corePath)
+const variantModule = require(workerData.variantPath)
+const variant = variantModule.default || variantModule
+
+async function main() {
+	const limits = workerData.limits
+	const pages = Math.floor(limits.memoryLimitBytes / 65536)
+	const wasmMemory = new WebAssembly.Memory({ initial: Math.min(256, pages), maximum: pages })
+	const module = await newQuickJSWASMModuleFromVariant(newVariant(variant, { wasmMemory }))
+	const runtime = module.newRuntime()
+	runtime.setMemoryLimit(limits.memoryLimitBytes)
+	runtime.setMaxStackSize(512 * 1024)
+	runtime.setInterruptHandler(() => Date.now() >= workerData.deadline)
+	const vm = runtime.newContext()
+	const pending = new Map()
+	let nextId = 0
+	let printBytes = 0
+	let printCount = 0
+	let truncated = false
+	let ended = false
+	let pumping = false
+	let scheduled = false
+	let program
+	const owned = []
+
+	function keep(result) {
+		const handle = vm.unwrapResult(result)
+		owned.push(handle)
+		return handle
+	}
+	// Capture intrinsics before the program may replace its own globals.
+	const encode = keep(vm.evalCode('(' + function () {
+		const stringify = JSON.stringify
+		const finite = Number.isFinite
+		const prototype = Object.getPrototypeOf
+		const plain = Object.prototype
+		const isArray = Array.isArray
+		const keys = Object.keys
+		const descriptor = Object.getOwnPropertyDescriptor
+		return function (value) {
+			if (value === undefined) return undefined
+			const parents = new Set()
+			function check(item, depth) {
+				const kind = typeof item
+				if (item === null || kind === 'string' || kind === 'boolean' || (kind === 'number' && finite(item))) return
+				if (kind !== 'object' || depth > 64 || parents.has(item)) throw new Error('Code runtime values must be JSON-safe.')
+				if (!isArray(item) && prototype(item) !== plain && prototype(item) !== null) throw new Error('Code runtime values must use plain objects and arrays.')
+				parents.add(item)
+				for (const key of keys(item)) {
+					const property = descriptor(item, key)
+					if (!property || !('value' in property)) throw new Error('Code runtime values must not contain accessors.')
+					check(property.value, depth + 1)
+				}
+				parents.delete(item)
+			}
+			check(value, 0)
+			return stringify(value)
+		}
+	}.toString() + ')()'))
+	const decode = keep(vm.evalCode('(function () { const parse = JSON.parse; return function(value) { return parse(value); }; })()'))
+	const errorText = keep(vm.evalCode('(function () { const string = String; return function(error) { try { return typeof error.message === "string" ? error.message : string(error); } catch { return "The program failed."; } }; })()'))
+
+	function textValue(handle, maximum) {
+		const length = vm.getProp(handle, 'length')
+		try {
+			if (vm.getNumber(length) > maximum) throw new Error('Code runtime value exceeds ' + maximum + ' bytes.')
+		} finally { length.dispose() }
+		const text = vm.getString(handle)
+		if (Buffer.byteLength(text) > maximum) throw new Error('Code runtime value exceeds ' + maximum + ' bytes.')
+		return text
+	}
+	function serialize(handle, maximum) {
+		if (vm.typeof(handle) === 'undefined') return undefined
+		const encoded = vm.callFunction(encode, vm.undefined, handle)
+		if (encoded.error) {
+			encoded.error.dispose()
+			throw new Error('Code runtime values must be JSON-safe and serializable.')
+		}
+		try { return textValue(encoded.value, maximum) }
+		finally { encoded.value.dispose() }
+	}
+	function failureText(handle) {
+		const result = vm.callFunction(errorText, vm.undefined, handle)
+		if (result.error) { result.error.dispose(); return 'The program failed.' }
+		// Diagnostics have their own fixed bound. A deliberately tiny data
+		// allowance must not erase the reason or the no-replay guidance for an
+		// effect that completed but whose return value could not be delivered.
+		try { return textValue(result.value, 4096) }
+		catch { return 'The program failed with an oversized error.' }
+		finally { result.value.dispose() }
+	}
+	function dispose() {
+		for (const deferred of pending.values()) deferred.dispose()
+		pending.clear()
+		if (program && program.alive) program.dispose()
+		for (const handle of owned) if (handle.alive) handle.dispose()
+		vm.dispose()
+		runtime.dispose()
+	}
+	function finish(message) {
+		if (ended) return
+		ended = true
+		// A guest OOM can make QuickJS teardown throw. The worker still owns
+		// the whole WASM module and is terminated by its parent on settlement.
+		try { dispose() } catch {}
+		parentPort.postMessage(message)
+	}
+	function fail(error) {
+		finish(Date.now() >= workerData.deadline
+			? { kind: 'timed-out' }
+			: { kind: 'error', error: String(error && error.message || error).slice(0, 4096) })
+	}
+	function schedule() {
+		if (ended || scheduled) return
+		scheduled = true
+		setImmediate(() => { scheduled = false; pump() })
+	}
+	function pump() {
+		if (ended || pumping) return
+		pumping = true
+		try {
+			const jobs = runtime.executePendingJobs(32)
+			if (jobs.error) {
+				const error = failureText(jobs.error)
+				jobs.error.dispose()
+				throw new Error(error)
+			}
+			const state = vm.getPromiseState(program)
+			if (state.type === 'fulfilled') {
+				let json
+				try { json = serialize(state.value, limits.maxValueBytes) }
+				finally { state.value.dispose() }
+				finish({ kind: 'done', json })
+			} else if (state.type === 'rejected') {
+				let error
+				try { error = failureText(state.error) }
+				finally { state.error.dispose() }
+				fail(new Error(error))
+			} else if (runtime.hasPendingJob()) schedule()
+		} catch (error) { fail(error) }
+		finally { pumping = false }
+	}
+	function rejected(message) {
+		const deferred = vm.newPromise()
+		const error = vm.newError(message)
+		deferred.reject(error)
+		error.dispose()
+		return deferred.handle
+	}
+	const call = vm.newFunction('call', (name, input = vm.undefined) => {
+		try {
+			if (nextId >= limits.maxHostCalls) return rejected('Code runtime host-call limit reached.')
+			if (pending.size >= limits.maxPendingHostCalls) return rejected('Code runtime pending host-call limit reached.')
+			if (vm.typeof(name) !== 'string') return rejected('A host call requires a tool name string.')
+			const tool = textValue(name, Math.min(limits.maxValueBytes, 1024))
+			const json = serialize(input, limits.maxValueBytes)
+			const id = ++nextId
+			const deferred = vm.newPromise()
+			pending.set(id, deferred)
+			parentPort.postMessage({ kind: 'call', id, name: tool, json })
+			return deferred.handle
+		} catch (error) { return rejected(String(error.message || error)) }
+	})
+	vm.setProp(vm.global, 'call', call)
+	call.dispose()
+	const print = vm.newFunction('print', (...values) => {
+		if (truncated) return vm.undefined
+		try {
+			const parts = []
+			let remaining = workerData.maxOutputBytes - printBytes - (printCount > 0 ? 1 : 0)
+			for (const value of values) {
+				if (parts.length) remaining--
+				if (remaining < 0) throw new Error('output limit')
+				const text = vm.typeof(value) === 'string' ? textValue(value, remaining) : serialize(value, remaining)
+				const part = text === undefined ? '' : text
+				remaining -= Buffer.byteLength(part)
+				parts.push(part)
+			}
+			const line = parts.join(' ')
+			const size = Buffer.byteLength(line) + (printCount > 0 ? 1 : 0)
+			if (printBytes + size > workerData.maxOutputBytes) throw new Error('output limit')
+			// Empty prints consume a newline after the first, so the byte
+			// budget also bounds the number of worker messages.
+			printBytes += size
+			printCount++
+			parentPort.postMessage({ kind: 'print', line })
+		} catch {
+			truncated = true
+			parentPort.postMessage({ kind: 'truncated' })
+		}
+		return vm.undefined
+	})
+	vm.setProp(vm.global, 'print', print)
+	print.dispose()
+
+	parentPort.on('message', (message) => {
+		if (ended || message.kind !== 'call-result') return
+		const deferred = pending.get(message.id)
+		if (!deferred) return
+		pending.delete(message.id)
+		try {
+			if (message.ok) {
+				let value = vm.undefined
+				if (message.json !== undefined) {
+					if (typeof message.json !== 'string' || Buffer.byteLength(message.json) > limits.maxValueBytes) throw new Error('Host result exceeds the value limit.')
+					const input = vm.newString(message.json)
+					try { value = vm.unwrapResult(vm.callFunction(decode, vm.undefined, input)) }
+					finally { input.dispose() }
+				}
+				deferred.resolve(value)
+				if (value !== vm.undefined) value.dispose()
+			} else {
+				const error = vm.newError(String(message.error).slice(0, 4096))
+				deferred.reject(error)
+				error.dispose()
+			}
+			deferred.dispose()
+			schedule()
+		} catch (error) { deferred.dispose(); fail(error) }
+	})
+	try {
+		const result = vm.evalCode('(async () => {\\n' + workerData.source + '\\n})()', 'model-program.js')
+		if (result.error) {
+			const error = failureText(result.error)
+			result.error.dispose()
+			fail(new Error(error))
+		} else { program = result.value; pump() }
+	} catch (error) { fail(error) }
+}
+main().catch((error) => parentPort.postMessage({ kind: 'error', error: String(error && error.message || error).slice(0, 4096) }))
+`