@namzu/sdk 62.12.0 → 62.13.0

package/src/connector/mcp/index.ts

@@ -34,6 +34,7 @@ export type {
 } from './era.js'
 export { buildEnvelope, decodeResult, encodeMcpHeaderValue } from './envelope.js'
 export type { McpEnvelope, McpEnvelopeInput, MCPDecodedResult } from './envelope.js'
+export { MCP_TASKS_EXTENSION } from './tasks.js'
 
 // The repo's first refusal path: a tool definition namzu declines to expose.
 // Exported so a host can ask the same question of a schema it holds — and

package/src/connector/mcp/tasks.ts

@@ -0,0 +1,81 @@
+import type { MCPInputRequest } from '../../types/connector/index.js'
+
+/** The extension's identifier, in both the client's and the server's `extensions`. */
+export const MCP_TASKS_EXTENSION = 'io.modelcontextprotocol/tasks'
+
+/** Poll no faster than this whatever the server suggests, and no slower than the cap. */
+export const MCP_TASK_MIN_POLL_MS = 100
+export const MCP_TASK_MAX_POLL_MS = 30_000
+/** The interval used when the server names none. */
+export const MCP_TASK_DEFAULT_POLL_MS = 1_000
+
+export type MCPTaskStatus = 'working' | 'input_required' | 'completed' | 'failed' | 'cancelled'
+
+/** One reading of a task: what `CreateTaskResult` and a `tasks/get` result both carry. */
+export interface MCPTaskState {
+	readonly taskId: string
+	readonly status: MCPTaskStatus
+	readonly statusMessage?: string
+	readonly lastUpdatedAt?: string
+	readonly pollIntervalMs?: number
+	/** `input_required`: what the server wants, as `{ method }` entries. */
+	readonly inputRequests?: readonly MCPInputRequest[]
+	/** `completed`: the result the original request would have returned. */
+	readonly result?: unknown
+	/** `failed`: the JSON-RPC error. */
+	readonly error?: { readonly code?: unknown; readonly message?: unknown; readonly data?: unknown }
+}
+
+const STATUSES = new Set<string>(['working', 'input_required', 'completed', 'failed', 'cancelled'])
+
+/** Is this result a task handle rather than a result? */
+export function isCreateTaskResult(raw: unknown): boolean {
+	return (
+		typeof raw === 'object' &&
+		raw !== null &&
+		(raw as { resultType?: unknown }).resultType === 'task'
+	)
+}
+
+/**
+ * Read a `CreateTaskResult` or a `tasks/get` result, or refuse it.
+ *
+ * Returns `undefined` for anything that is not a task with a non-empty
+ * `taskId` and a status the extension defines, so a malformed answer ends the
+ * wait with a clear error instead of polling a handle nobody can name.
+ */
+export function decodeTaskState(raw: unknown): MCPTaskState | undefined {
+	if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return undefined
+	const r = raw as Record<string, unknown>
+	if (typeof r.taskId !== 'string' || r.taskId.length === 0) return undefined
+	if (typeof r.status !== 'string' || !STATUSES.has(r.status)) return undefined
+	const inputRequests =
+		typeof r.inputRequests === 'object' && r.inputRequests !== null
+			? Object.values(r.inputRequests as Record<string, unknown>).filter(
+					(item): item is MCPInputRequest =>
+						typeof item === 'object' &&
+						item !== null &&
+						typeof (item as { method?: unknown }).method === 'string',
+				)
+			: undefined
+	return {
+		taskId: r.taskId,
+		status: r.status as MCPTaskStatus,
+		...(typeof r.statusMessage === 'string' ? { statusMessage: r.statusMessage } : {}),
+		...(typeof r.lastUpdatedAt === 'string' ? { lastUpdatedAt: r.lastUpdatedAt } : {}),
+		...(typeof r.pollIntervalMs === 'number' && Number.isFinite(r.pollIntervalMs)
+			? { pollIntervalMs: r.pollIntervalMs }
+			: {}),
+		...(inputRequests ? { inputRequests } : {}),
+		...('result' in r ? { result: r.result } : {}),
+		...(typeof r.error === 'object' && r.error !== null
+			? { error: r.error as MCPTaskState['error'] }
+			: {}),
+	}
+}
+
+/** How long to wait before the next `tasks/get`, from what the server last suggested. */
+export function taskPollDelayMs(suggested: number | undefined): number {
+	const wanted = suggested ?? MCP_TASK_DEFAULT_POLL_MS
+	return Math.min(Math.max(wanted, MCP_TASK_MIN_POLL_MS), MCP_TASK_MAX_POLL_MS)
+}

package/src/constants/mcp/index.ts

@@ -177,8 +177,13 @@ export const MCP_PARAM_HEADER_PREFIX = 'Mcp-Param-'
  * WHICH tool is being called without reading the body is the whole point of
  * mirroring it into a header.
  */
-export const MCP_NAME_HEADER_METHODS: Readonly<Record<string, 'name' | 'uri'>> = {
+export const MCP_NAME_HEADER_METHODS: Readonly<Record<string, 'name' | 'uri' | 'taskId'>> = {
 	'tools/call': 'name',
 	'prompts/get': 'name',
 	'resources/read': 'uri',
+	// The tasks extension: an intermediary routes a task's requests to the
+	// instance that holds it, by this header.
+	'tasks/get': 'taskId',
+	'tasks/update': 'taskId',
+	'tasks/cancel': 'taskId',
 }

package/src/eval/bootstrap.ts

@@ -0,0 +1,314 @@
+/**
+ * Intervals for an eval, by resampling cases.
+ *
+ * A pass rate printed alone invites conclusions the data cannot carry: on 80
+ * tasks a difference of -1.25 points came with a 95% interval from -10.0 to
+ * +7.5, which says almost nothing. The interval here is a percentile
+ * bootstrap over CASES. A case is the unit that was sampled from the world, so
+ * a case is what gets resampled, and all of a case's repeats travel with it:
+ * repeats of one case are not independent evidence about the suite, and
+ * treating them as such would narrow the interval for free.
+ *
+ * Nothing in here reads the clock or an unseeded random source. The generator
+ * is a small deterministic one fed by a seed that the report records, so the
+ * same cases and the same seed give the same interval.
+ */
+
+/** Fewest cases an interval is computed for. Below it the report says why. */
+export const MIN_CASES_FOR_INTERVAL = 5
+
+/** Resamples drawn per interval. */
+export const BOOTSTRAP_RESAMPLES = 2000
+
+/** The seed used when none is given. A fixed number, so a report is reproducible. */
+export const DEFAULT_EVAL_SEED = 1
+
+/** Coverage of every interval in this module. */
+export const INTERVAL_LEVEL = 0.95
+
+/**
+ * What one case contributes: the sum of its judged runs' scores, how many of
+ * those runs passed, and how many were judged at all. `judged` is at least 1;
+ * a case with no judged run has nothing to say and is left out by the caller.
+ */
+export interface CaseTally {
+	readonly scoreSum: number
+	readonly passes: number
+	readonly judged: number
+}
+
+/**
+ * An interval, or the reason there is none.
+ *
+ * `estimate` is the number the interval is around, and is present even when
+ * no interval is computed, so a reader always has the point value and the
+ * explanation for why it carries no bounds.
+ */
+export type Interval =
+	| {
+			readonly computed: true
+			readonly estimate: number
+			readonly low: number
+			readonly high: number
+	  }
+	| {
+			readonly computed: false
+			readonly estimate: number | null
+			readonly reason: string
+	  }
+
+/** The method and its settings, written next to the numbers so they can be reproduced. */
+export interface BootstrapMethod {
+	readonly kind: 'percentile bootstrap over cases, repeats kept together'
+	readonly level: typeof INTERVAL_LEVEL
+	readonly resamples: number
+	readonly seed: number
+	/** Cases the interval was drawn over. */
+	readonly cases: number
+	readonly minimumCases: number
+}
+
+export interface CaseIntervals {
+	readonly method: BootstrapMethod
+	/** Mean score over judged runs. */
+	readonly mean: Interval
+	/** Passed runs over judged runs. */
+	readonly passRate: Interval
+}
+
+/** The same for a paired difference, second side minus first. */
+export interface DifferenceIntervals {
+	readonly method: BootstrapMethod
+	readonly mean: Interval
+	readonly passRate: Interval
+}
+
+/**
+ * A bad `--repeat` or `--seed`. A RangeError, so existing handlers still see
+ * one, and marked with a code so a CLI can tell it from a RangeError a suite's
+ * own code threw (and from one thrown by another copy of this class).
+ */
+export class EvalUsageError extends RangeError {
+	readonly code = 'EVAL_USAGE'
+}
+
+/** True for an {@link EvalUsageError}, even one built by another copy of the SDK. */
+export function isEvalUsageError(err: unknown): err is EvalUsageError {
+	return err instanceof RangeError && (err as { code?: unknown }).code === 'EVAL_USAGE'
+}
+
+/**
+ * Validate a seed: an integer that fits in 32 bits, unsigned.
+ * Thrown as an EvalUsageError so a CLI can turn it into a usage error.
+ */
+export function validateSeed(value: number | undefined): number {
+	if (value === undefined) return DEFAULT_EVAL_SEED
+	if (!Number.isInteger(value) || value < 0 || value > 0xffffffff) {
+		throw new EvalUsageError(
+			`seed must be an integer from 0 to 4294967295, or omitted; received ${String(value)}`,
+		)
+	}
+	return value
+}
+
+/**
+ * mulberry32: a 32-bit generator with one word of state. Not for secrets;
+ * its job is to be the same on every machine for a given seed.
+ */
+export function createSeededRandom(seed: number): () => number {
+	let state = validateSeed(seed) >>> 0
+	return () => {
+		state = (state + 0x6d2b79f5) >>> 0
+		let t = state
+		t = Math.imul(t ^ (t >>> 15), t | 1)
+		t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+	}
+}
+
+function quantile(sorted: Float64Array, p: number): number {
+	const position = p * (sorted.length - 1)
+	const lower = Math.floor(position)
+	const upper = Math.ceil(position)
+	const low = sorted[lower] as number
+	const high = sorted[upper] as number
+	return low + (high - low) * (position - lower)
+}
+
+/**
+ * Draw `resamples` samples of `n` indices with replacement, hand each to
+ * `statistics`, and return the 2.5th and 97.5th percentiles of the two numbers
+ * it returns. Both statistics see the same draw, so the two intervals come
+ * from one set of resampled suites.
+ */
+function bootstrap(
+	n: number,
+	seed: number,
+	statistics: (draw: Int32Array) => readonly [number, number],
+): { mean: [number, number]; passRate: [number, number] } {
+	const random = createSeededRandom(seed)
+	const draw = new Int32Array(n)
+	const means = new Float64Array(BOOTSTRAP_RESAMPLES)
+	const rates = new Float64Array(BOOTSTRAP_RESAMPLES)
+	for (let b = 0; b < BOOTSTRAP_RESAMPLES; b++) {
+		for (let i = 0; i < n; i++) draw[i] = Math.floor(random() * n)
+		const [mean, rate] = statistics(draw)
+		means[b] = mean
+		rates[b] = rate
+	}
+	means.sort()
+	rates.sort()
+	const lowP = (1 - INTERVAL_LEVEL) / 2
+	const highP = 1 - lowP
+	return {
+		mean: [quantile(means, lowP), quantile(means, highP)],
+		passRate: [quantile(rates, lowP), quantile(rates, highP)],
+	}
+}
+
+function methodOf(seed: number, cases: number): BootstrapMethod {
+	return {
+		kind: 'percentile bootstrap over cases, repeats kept together',
+		level: INTERVAL_LEVEL,
+		resamples: BOOTSTRAP_RESAMPLES,
+		seed,
+		cases,
+		minimumCases: MIN_CASES_FOR_INTERVAL,
+	}
+}
+
+function tooFew(one: string, many: string, n: number): string {
+	return `only ${n} ${n === 1 ? one : many}; an interval needs at least ${MIN_CASES_FOR_INTERVAL}`
+}
+
+export function pooledMeanAndRate(
+	tallies: readonly CaseTally[],
+	draw?: Int32Array,
+): [number, number] {
+	let score = 0
+	let passes = 0
+	let judged = 0
+	const count = draw ? draw.length : tallies.length
+	for (let i = 0; i < count; i++) {
+		const t = tallies[draw ? (draw[i] as number) : i] as CaseTally
+		score += t.scoreSum
+		passes += t.passes
+		judged += t.judged
+	}
+	return judged === 0 ? [Number.NaN, Number.NaN] : [score / judged, passes / judged]
+}
+
+/**
+ * 95% intervals for the mean score and the pass rate over a set of cases.
+ *
+ * Each resample draws as many cases as there are, with replacement, and pools
+ * their runs. Cases without a judged run must not be passed in.
+ */
+export function bootstrapCases(tallies: readonly CaseTally[], seed: number): CaseIntervals {
+	const checked = validateSeed(seed)
+	const n = tallies.length
+	const method = methodOf(checked, n)
+	const [meanEstimate, rateEstimate] = n === 0 ? [null, null] : pooledMeanAndRate(tallies)
+	if (n < MIN_CASES_FOR_INTERVAL) {
+		const reason = tooFew('case with a judged run', 'cases with a judged run', n)
+		return {
+			method,
+			mean: { computed: false, estimate: meanEstimate, reason },
+			passRate: { computed: false, estimate: rateEstimate, reason },
+		}
+	}
+	const result = bootstrap(n, checked, (draw) => pooledMeanAndRate(tallies, draw))
+	return {
+		method,
+		mean: {
+			computed: true,
+			estimate: meanEstimate as number,
+			low: result.mean[0],
+			high: result.mean[1],
+		},
+		passRate: {
+			computed: true,
+			estimate: rateEstimate as number,
+			low: result.passRate[0],
+			high: result.passRate[1],
+		},
+	}
+}
+
+/** One case seen from both sides of a comparison. */
+export interface PairedTally {
+	readonly a: CaseTally
+	readonly b: CaseTally
+}
+
+/**
+ * 95% intervals for the difference of two runs over the same cases, `b` minus
+ * `a`. A resample draws PAIRS, so the two sides of a case always travel
+ * together; that is what makes the interval a paired one, and it is much
+ * narrower than two separate intervals when the cases are the thing that
+ * varies.
+ */
+export function bootstrapPairedDifference(
+	pairs: readonly PairedTally[],
+	seed: number,
+): DifferenceIntervals {
+	const checked = validateSeed(seed)
+	const n = pairs.length
+	const method = methodOf(checked, n)
+	const aSide = pairs.map((p) => p.a)
+	const bSide = pairs.map((p) => p.b)
+	const difference = (draw?: Int32Array): [number, number] => {
+		const [bMean, bRate] = pooledMeanAndRate(bSide, draw)
+		const [aMean, aRate] = pooledMeanAndRate(aSide, draw)
+		return [bMean - aMean, bRate - aRate]
+	}
+	const [meanEstimate, rateEstimate] = n === 0 ? [null, null] : difference()
+	if (n < MIN_CASES_FOR_INTERVAL) {
+		const reason = tooFew('paired case', 'paired cases', n)
+		return {
+			method,
+			mean: { computed: false, estimate: meanEstimate, reason },
+			passRate: { computed: false, estimate: rateEstimate, reason },
+		}
+	}
+	const result = bootstrap(n, checked, (draw) => difference(draw))
+	return {
+		method,
+		mean: {
+			computed: true,
+			estimate: meanEstimate as number,
+			low: result.mean[0],
+			high: result.mean[1],
+		},
+		passRate: {
+			computed: true,
+			estimate: rateEstimate as number,
+			low: result.passRate[0],
+			high: result.passRate[1],
+		},
+	}
+}
+
+/**
+ * Exact two-sided sign test on discordant pairs: under no difference each flip
+ * is equally likely to go either way, so the count going one way is
+ * Binomial(n, 1/2). Returns twice the smaller tail, capped at 1; with no flips
+ * at all the answer is 1.
+ */
+export function exactSignTestP(oneWay: number, otherWay: number): number {
+	const n = oneWay + otherWay
+	if (n === 0) return 1
+	const k = Math.min(oneWay, otherWay)
+	const logFactorial = new Float64Array(n + 1)
+	for (let i = 2; i <= n; i++) logFactorial[i] = (logFactorial[i - 1] as number) + Math.log(i)
+	let tail = 0
+	for (let i = 0; i <= k; i++) {
+		const logTerm =
+			(logFactorial[n] as number) -
+			(logFactorial[i] as number) -
+			(logFactorial[n - i] as number) -
+			n * Math.LN2
+		tail += Math.exp(logTerm)
+	}
+	return Math.min(1, 2 * tail)
+}