@namzu/sdk 62.12.0 → 62.14.0

package/src/eval/run-summary.ts

@@ -0,0 +1,104 @@
+import { UNKNOWN } from './disclosure.js'
+import type { RunMode } from './disclosure.js'
+import {
+	type RunStats,
+	describeIntervals,
+	describeSplits,
+	groupRuns,
+	statsOver,
+} from './report-stats.js'
+import type { ExperimentReport } from './types.js'
+
+/** What produced a whole run, across every suite in it. */
+export interface RunDisclosure {
+	readonly harnessVersion: string
+	readonly sdkVersion: string
+	readonly generatedAt: string
+	readonly commit: string
+	readonly seed: number
+	/** Repeats per case; the largest across suites when they differ. */
+	readonly repeats: number
+	readonly suites: number
+	readonly cases: number
+	readonly runs: number
+	/** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
+	readonly mode: RunMode | 'mixed'
+}
+
+/** The numbers for every case of every suite taken together. */
+export interface RunSummary extends Omit<RunStats, 'caseSummaries' | 'reliability'> {
+	readonly disclosure: RunDisclosure
+	/** Present only when cases were repeated. */
+	readonly reliability?: RunStats['reliability']
+}
+
+export function summarizeRun(
+	entries: ReadonlyArray<{ suite: string; report: ExperimentReport }>,
+	context: {
+		seed: number
+		harnessVersion: string
+		commit: string
+		generatedAt: string
+		sdkVersion: string
+	},
+): RunSummary {
+	// A case is named by its suite and its id: two suites may reuse a name.
+	const cases = entries.flatMap(({ suite, report }) =>
+		groupRuns(report.cases ?? [], (name) => `${suite}\u0000${name}`),
+	)
+	const { caseSummaries: _perCase, reliability, ...stats } = statsOver(cases, context.seed)
+	const repeats = Math.max(1, ...entries.map((e) => e.report.repeats ?? 1))
+	// A suite that used no model at all says nothing about whether a model
+	// ran, so it does not make a run "mixed" with the suites that did.
+	const all = entries.map((e) => e.report.disclosure?.mode ?? UNKNOWN)
+	const modes = new Set(all.filter((m) => m !== 'no-model'))
+	const mode: RunMode | 'mixed' =
+		modes.size === 0
+			? all.length === 0
+				? UNKNOWN
+				: 'no-model'
+			: modes.size === 1
+				? ([...modes][0] as RunMode)
+				: 'mixed'
+	return {
+		...stats,
+		...(repeats > 1 ? { reliability } : {}),
+		disclosure: {
+			harnessVersion: context.harnessVersion,
+			sdkVersion: context.sdkVersion,
+			generatedAt: context.generatedAt,
+			commit: context.commit,
+			seed: context.seed,
+			repeats,
+			suites: entries.length,
+			cases: stats.cases,
+			runs: stats.runs,
+			mode,
+		},
+	}
+}
+
+export function formatRunSummary(summary: RunSummary): string {
+	const d = summary.disclosure
+	const lines = [
+		`Whole run: ${summary.passed}/${summary.runs} runs passed across ${d.suites} suite${d.suites === 1 ? '' : 's'}, ${summary.cases} case${summary.cases === 1 ? '' : 's'}${d.repeats > 1 ? ` x ${d.repeats} repeats` : ''}`,
+	]
+	for (const line of describeIntervals({
+		mean: summary.mean,
+		passRate: summary.passRate,
+		intervals: summary.intervals,
+	})) {
+		lines.push(`  ${line}`)
+	}
+	if (summary.reliability) {
+		const r = summary.reliability
+		lines.push(
+			`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`,
+		)
+	}
+	for (const line of describeSplits(summary.splits)) lines.push(`  ${line}`)
+	lines.push(
+		`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`,
+	)
+	return lines.join('\n')
+}

package/src/eval/scorers.ts

@@ -1,3 +1,5 @@
+import type { HandBackKind } from '../types/session/hand-back.js'
+import type { TurnOutcomeVerdict } from '../types/session/outcome.js'
 import type { EvalCase, EvalTurn, Score, Scorer } from './types.js'
 
 /**
@@ -152,6 +154,87 @@ export function containsScorer(...required: string[]): Scorer {
 	}
 }
 
+/**
+ * The turn's outcome record has one of the given verdicts.
+ *
+ * Reads the record the kernel settled from what actually ran, not the answer's
+ * words. A turn with no record scores unavailable: there is nothing to read,
+ * which is not the same as a wrong verdict.
+ */
+export function outcomeScorer(verdicts: readonly TurnOutcomeVerdict[]): Scorer {
+	return {
+		name: 'outcome',
+		// The record either says one of the verdicts or it does not.
+		severity: 'gate',
+		score(turn: EvalTurn): Score {
+			const outcome = turn.outcome
+			if (!outcome) {
+				return { score: 0, unavailable: true, reason: 'the turn has no outcome record' }
+			}
+			const ok = verdicts.includes(outcome.verdict)
+			return {
+				score: ok ? 1 : 0,
+				reason: ok
+					? `verdict ${outcome.verdict}`
+					: `verdict ${outcome.verdict}, expected one of ${verdicts.join(', ')}`,
+				details: { verdict: outcome.verdict, expected: verdicts },
+			}
+		},
+	}
+}
+
+/**
+ * Whether the answer's plain check claims agree with the turn's outcome
+ * record: 1 when `compareAnswerToOutcome` found no mismatch, 0 when it found
+ * one, with the claim and what the record holds in the detail.
+ *
+ * Pass `{ expectMismatch: true }` for a case that is built to contradict
+ * itself, so the suite proves the comparison fires as well as that it stays
+ * silent.
+ */
+export function claimMismatchScorer(opts: { expectMismatch?: boolean } = {}): Scorer {
+	const expectMismatch = opts.expectMismatch === true
+	return {
+		name: 'claim-mismatch',
+		severity: 'gate',
+		score(turn: EvalTurn): Score {
+			const found = turn.claimMismatch
+			const ok = expectMismatch ? found !== undefined : found === undefined
+			return {
+				score: ok ? 1 : 0,
+				reason: found
+					? `${found.said}, but the record: ${found.recordSays}`
+					: 'no claim in the answer contradicts the record',
+				details: { claimMismatch: found ?? null, expectMismatch },
+			}
+		},
+	}
+}
+
+/**
+ * Score whether the turn was handed back to a person.
+ *
+ * Pass `{ expectKind }` for a case built so the model hands the task back, and
+ * the scorer checks it did, and for that kind; with nothing it checks the turn
+ * was not handed back. A gate: a scripted suite pins the kernel's side of it.
+ */
+export function handedBackScorer(opts: { expectKind?: HandBackKind } = {}): Scorer {
+	return {
+		name: 'handed-back',
+		severity: 'gate',
+		score(turn: EvalTurn): Score {
+			const found = turn.handedBack
+			const ok =
+				opts.expectKind === undefined ? found === undefined : found?.kind === opts.expectKind
+			return {
+				score: ok ? 1 : 0,
+				reason: found ? `handed back (${found.kind}): ${found.blocker}` : 'not handed back',
+				details: { handedBack: found ?? null, expectKind: opts.expectKind ?? null },
+			}
+		},
+	}
+}
+
 /**
  * Judge the turn with a caller-supplied predicate.
  *

package/src/eval/types.ts

@@ -1,6 +1,20 @@
+import type { AnswerClaimMismatch, TurnOutcome } from '../types/session/outcome.js'
 import type { StepResult } from '../types/session/step.js'
+import type { CaseIntervals } from './bootstrap.js'
+import type { ReportDisclosure } from './disclosure.js'
+import type { RepeatReliability, SplitSummary } from './report-stats.js'
 import type { ScoreUncertainty } from './uncertainty.js'
 
+/**
+ * Which part of a suite a case belongs to.
+ *
+ * `development` cases are the ones the harness was shaped against; a number
+ * measured on them says how well it fits them. `held-out` cases were not
+ * looked at while it was shaped, and only they say how it does on cases it
+ * has not seen.
+ */
+export type EvalSplit = 'development' | 'held-out'
+
 /**
  * One case in a dataset: an input, and what a good run looks like.
  *
@@ -19,6 +33,11 @@ export interface EvalCase<TInput = unknown> {
 	expectedTools?: readonly string[]
 	/** Per-case scorer overrides; falls back to the dataset's scorers. */
 	scorers?: readonly Scorer[]
+	/**
+	 * `development` or `held-out`. Left out, the case is reported as
+	 * unassigned. See {@link EvalSplit}.
+	 */
+	split?: EvalSplit
 }
 
 /** What actually happened, handed to every scorer. */
@@ -36,6 +55,18 @@ export interface EvalTurn {
 	durationMs: number
 	/** Set when the turn threw rather than settling. */
 	error?: string
+	/**
+	 * What the turn changed and what checked it, from the kernel's own record of
+	 * its tool calls. Absent for a turn that never ran the loop.
+	 */
+	outcome?: TurnOutcome
+	/**
+	 * A plain claim in the answer that the outcome record contradicts, found by
+	 * `compareAnswerToOutcome`. Absent when there is none.
+	 */
+	claimMismatch?: AnswerClaimMismatch
+	/** The model handed the task back to a person and the turn ended there. Absent otherwise. */
+	handedBack?: import('../types/session/hand-back.js').HandBack
 }
 
 /**
@@ -143,6 +174,10 @@ export interface CaseResult {
 	 * mattered. Absent when no gate missed.
 	 */
 	failedGates?: readonly string[]
+	/** Which run of this case this is, from 1. Set on every run of a report that has `repeats`. */
+	repeat?: number
+	/** The case's declared split, copied here so a report can be cut by it. */
+	split?: EvalSplit
 }
 
 export interface ExperimentReport {
@@ -177,4 +212,45 @@ export interface ExperimentReport {
 	 */
 	byScorer: Record<string, number>
 	durationMs: number
+	/**
+	 * Times each case ran. When it is above 1, `cases` holds every run of
+	 * every case, in case order, each marked with its `repeat`, and `passed`,
+	 * `failed` and `inconclusive` count runs. Absent in a report written before
+	 * repeats existed, which is read as 1.
+	 */
+	repeats?: number
+	/** Seed of the bootstrap behind {@link intervals}. */
+	seed?: number
+	/**
+	 * Passed runs over judged runs. An inconclusive run is in neither number,
+	 * so it cannot pull the rate either way.
+	 */
+	passRate?: number
+	/**
+	 * 95% intervals for {@link mean} and {@link passRate}, by a bootstrap over
+	 * cases. Absent from a report that predates it.
+	 */
+	intervals?: CaseIntervals
+	/** Per case: how many of its repeats passed. Present when `repeats` is above 1. */
+	caseSummaries?: readonly CaseRepeatSummary[]
+	/** Reliability across repeats. Present when `repeats` is above 1. */
+	reliability?: RepeatReliability
+	/** Numbers per split. Keys appear only for splits that have cases. */
+	splits?: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>
+	/** What produced this report. */
+	disclosure?: ReportDisclosure
+}
+
+/** How the repeats of one case went. */
+export interface CaseRepeatSummary {
+	case: string
+	split?: EvalSplit
+	repeats: number
+	passed: number
+	failed: number
+	inconclusive: number
+	/** Mean score over the judged repeats; null when none was judged. */
+	mean: number | null
+	/** Every repeat passed. An inconclusive repeat means this is false. */
+	passedEveryRepeat: boolean
 }

package/src/manager/session/turn-recorder.ts

@@ -6,6 +6,7 @@ import {
 import { EMPTY_TOKEN_USAGE } from '../../constants/limits.js'
 import { NAMZU } from '../../constants/telemetry/index.js'
 import { resolveModelPricing } from '../../pricing/index.js'
+import { compareAnswerToOutcome } from '../../runtime/query/turn/answer-claims.js'
 import type { SessionTokenBudget } from '../../store/budget/index.js'
 import type { SessionCheckpointStore } from '../../store/checkpoint/index.js'
 import { foldSessionLog } from '../../store/session-log/incremental.js'
@@ -40,6 +41,7 @@ import {
 	type SessionEventType,
 	isEphemeralEvent,
 } from '../../types/session/events.js'
+import type { HandBack } from '../../types/session/hand-back.js'
 import type { ProjectId, TopicId } from '../../types/session/ids.js'
 import type { TurnOutcome } from '../../types/session/outcome.js'
 import type { SessionRecord } from '../../types/session/records.js'
@@ -516,6 +518,12 @@ export class TurnRecorder {
 		else this.#turn.outcome = outcome
 	}
 
+	/** The model handed the task back; set once, as the turn ends. */
+	setHandedBack(handBack: HandBack | undefined): void {
+		if (handBack === undefined) delete this.#turn.handedBack
+		else this.#turn.handedBack = handBack
+	}
+
 	setSteps(steps: readonly StepResult[]): void {
 		this.#turn.steps = steps
 	}
@@ -612,6 +620,11 @@ export class TurnRecorder {
 	settlement(status: TurnSettlement['status']): LiveTurnSettlement {
 		const turn = this.getTurn()
 		const resultMessageId = this.#lastAssistantEntry()?.id
+		// A task handed back says what blocks it in the model's words; those are not
+		// an answer to read claims out of.
+		const claimMismatch = turn.handedBack
+			? undefined
+			: compareAnswerToOutcome(turn.result, turn.outcome)
 		return {
 			status,
 			iterations: turn.currentIteration,
@@ -625,6 +638,8 @@ export class TurnRecorder {
 			abandonedTaskIds: [...(turn.abandonedTaskIds ?? [])],
 			abandonedJobIds: [...(turn.abandonedJobIds ?? [])],
 			...(turn.outcome ? { outcome: turn.outcome } : {}),
+			...(claimMismatch ? { claimMismatch } : {}),
+			...(turn.handedBack ? { handedBack: turn.handedBack } : {}),
 		}
 	}
 
@@ -1560,9 +1575,15 @@ export function eventDraft(
 		...payload
 	} = event as SessionEvent & Record<string, unknown>
 	const draft = JSON.parse(JSON.stringify(payload)) as Record<string, unknown>
-	// The outcome rides the live settlement only; the log keeps it in its own record.
+	// The outcome, the claim mismatch and the hand-back ride the live settlement
+	// only; the log keeps the outcome in its own record, the other two nowhere.
 	if (draft.settlement !== undefined && typeof draft.settlement === 'object') {
-		const { outcome: _outcome, ...settlement } = draft.settlement as Record<string, unknown>
+		const {
+			outcome: _outcome,
+			claimMismatch: _claimMismatch,
+			handedBack: _handedBack,
+			...settlement
+		} = draft.settlement as Record<string, unknown>
 		draft.settlement = settlement
 	}
 	return { ...draft, ...(turnId ? { turnId } : {}) } as Parameters<SessionLog['append']>[1]

package/src/plugin/index.ts

@@ -1,4 +1,10 @@
-export { discoverPlugins, loadPluginManifest, discoverAllPluginDirs } from './loader.js'
+export {
+	assertEnableable,
+	discoverPlugins,
+	loadPluginManifest,
+	discoverAllPluginDirs,
+	parsePluginManifestText,
+} from './loader.js'
 export { definePlugin } from './define.js'
 export type { DefinedPlugin, DefinePluginOptions } from './define.js'
 export type { PluginDiscoveryOptions, PluginEnablementCapabilities } from './loader.js'

package/src/plugin/loader.ts

@@ -97,12 +97,26 @@ export async function loadPluginManifest(
 		throw new Error(`Plugin manifest must be a regular file, not a symlink: ${manifestPath}`)
 	}
 	const raw = await readFile(manifestPath, 'utf-8')
-	const parsed: unknown = JSON.parse(raw)
-	const manifest = PluginManifestSchema.parse(parsed)
+	const manifest = parsePluginManifestText(raw)
 	assertEnableable(manifest, capabilities)
 	return manifest
 }
 
+/**
+ * A manifest's text to a validated manifest, or a thrown reason: a
+ * `SyntaxError` for text that is not JSON, a zod error (with its `issues`)
+ * for JSON that is not a manifest.
+ *
+ * The first two of the loader's three steps on the manifest text (parse,
+ * schema; {@link assertEnableable} is the third). `loadPluginManifest` calls
+ * it, and so does a host that checks a manifest without loading the plugin,
+ * so the rules live once.
+ */
+export function parsePluginManifestText(raw: string): PluginManifest {
+	const parsed: unknown = JSON.parse(raw)
+	return PluginManifestSchema.parse(parsed)
+}
+
 async function discoverScopedPlugins(
 	authorityRoot: string,
 	relativePluginDir: string,

package/src/public-runtime.ts

@@ -283,6 +283,14 @@ export {
 	SESSION_GOAL_TOOL_NAMES,
 	buildSessionGoalTools,
 } from './tools/goal/index.js'
+export {
+	DEFAULT_GOAL_REFUSALS,
+	GoalConfirmation,
+	NO_CHECK_REASON_CODE,
+	decideGoalConfirmation,
+	describeGoalEnding,
+	recordSentence,
+} from './tools/goal/confirmation.js'
 export { SessionGoalActivation } from './manager/goal/activation.js'
 export { buildToolResultHashes, hashToolResult } from './utils/hash.js'
 export {
@@ -508,6 +516,7 @@ export {
 export {
 	discoverSkills,
 	loadSkill,
+	parseSkillMarkdown,
 	resolveSkillChain,
 	SKILL_FRONTMATTER_KEYS,
 	SKILL_FRONTMATTER_TOLERATED_KEYS,
@@ -568,6 +577,7 @@ export {
 	EXPLORE_AGENT_DESCRIPTION,
 	EXPLORE_AGENT_ID,
 	EXPLORE_AGENT_PROMPT,
+	AGENT_FILE_KEYS,
 	MAX_AGENT_FILE_CHARS,
 	PLAN_AGENT_DESCRIPTION,
 	PLAN_AGENT_ID,
@@ -692,7 +702,9 @@ export {
 	definePlugin,
 	discoverAllPluginDirs,
 	discoverPlugins,
+	assertEnableable,
 	loadPluginManifest,
+	parsePluginManifestText,
 	MAX_SHELL_HOOK_TIMEOUT_MS,
 	PluginLifecycleManager,
 	PluginResolver,
@@ -902,6 +914,7 @@ export {
 	isRecognizedModernError,
 	isResourceNotFoundError,
 	isUnsupportedProtocolVersionError,
+	MCP_TASKS_EXTENSION,
 	LocalExecutionContext,
 	mcpEraCacheKey,
 	MCPHttpStatusError,
@@ -1022,6 +1035,8 @@ export type {
 	AcpClientCapabilities,
 	AcpInitializeParams,
 	AcpInitializeResult,
+	AcpKernelFacts,
+	AcpHostKernelFacts,
 	AcpSessionCancelParams,
 	AcpSessionNewParams,
 	AcpSessionNewResult,
@@ -1399,11 +1414,28 @@ export { anyOf, hasToolCall, stepCountIs } from './types/session/step.js'
 // ─── evaluation harness ──────────────────────────────────────────────────
 
 export {
+	BOOTSTRAP_RESAMPLES,
+	DEFAULT_EVAL_SEED,
+	EvalUsageError,
+	MIN_CASES_FOR_INTERVAL,
+	bootstrapCases,
+	bootstrapPairedDifference,
+	compareEvalReports,
+	exactSignTestP,
+	formatEvalComparison,
+	formatRunSummary,
+	isEvalUsageError,
+	loadEvalReportFile,
+	summarizeRun,
+	withEvalRunOptions,
 	compareHarnessTrials,
 	reviewHarnessCandidate,
+	claimMismatchScorer,
+	handedBackScorer,
 	completionScorer,
 	containsScorer,
 	customScorer,
+	outcomeScorer,
 	evalTurnFromQuery,
 	evalTurnFromTurn,
 	formatReport,
@@ -1413,6 +1445,33 @@ export {
 	trajectoryScorer,
 } from './eval/index.js'
 
+// ─── answer against record ───────────────────────────────────────────────
+
+export {
+	ANSWER_CLAIM_MAX_CHARS,
+	ANSWER_CLAIM_MAX_QUOTE_CHARS,
+	compareAnswerToOutcome,
+	describeAnswerClaimMismatch,
+} from './runtime/query/turn/answer-claims.js'
+
+// ─── handing the task back ───────────────────────────────────────────────
+
+export {
+	HAND_BACK_GUIDANCE,
+	HAND_BACK_GUIDANCE_CONTRIBUTION_ID,
+	HAND_BACK_MAX_CHARS,
+	HAND_BACK_RECEIPT,
+	HAND_BACK_SIBLING_REFUSAL,
+	HAND_BACK_TOOL_NAME,
+	HandBackInputSchema,
+	buildHandBackTool,
+	describeHandBack,
+	handBackKindSentence,
+	handBackModelLines,
+	handBackReason,
+	parseHandBack,
+} from './tools/hand-back/index.js'
+
 // ─── metrics ─────────────────────────────────────────────────────────────
 //
 // Exported so a host can record its own measurements onto the same series
@@ -1428,6 +1487,17 @@ export {
 } from './telemetry/metrics.js'
 export type { TokenUsageSample } from './telemetry/metrics.js'
 
+// The GenAI convention names on spans: the pinned version, the switch, the
+// list a test holds emitted names against. See docs/sdk/telemetry-genai.md.
+export {
+	GENAI_NON_CONVENTION_NAMES,
+	GENAI_SEMCONV_ATTRIBUTES,
+	GENAI_SEMCONV_SCHEMA_URL,
+	GENAI_SEMCONV_VERSION,
+	genAiConventionsEnabled,
+	setGenAiConventions,
+} from './telemetry/genai.js'
+
 // ─── guardrails ──────────────────────────────────────────────────────────
 
 export {

package/src/public-types.ts

@@ -406,7 +406,16 @@ export type {
 	SessionGoalStore,
 } from './store/goal/index.js'
 export type { ResolveGoalRoundAuthority } from './tools/goal/index.js'
+export type {
+	GoalConfirmationOptions,
+	GoalDecision,
+	GoalDecisionInput,
+	GoalEnding,
+	GoalEndingReport,
+	GoalStatement,
+} from './tools/goal/confirmation.js'
 export type { ActiveSessionGoal } from './manager/goal/activation.js'
+export type { HandBack, HandBackKind } from './types/session/hand-back.js'
 
 export type {
 	AgentHandle,