@namzu/sdk 30.2.0 → 31.1.0

package/src/eval/experiment.ts

@@ -10,6 +10,8 @@ import type {
 import type { ScoreUncertainty } from './uncertainty.js'
 import { describeUncertainty, uncertaintyOf } from './uncertainty.js'
 
+const MAX_CASE_TIMEOUT_MS = 2_147_483_647
+
 export interface ExperimentConfig<TInput = unknown> {
 	name: string
 	cases: readonly EvalCase<TInput>[]
@@ -22,18 +24,18 @@ export interface ExperimentConfig<TInput = unknown> {
 	 */
 	run: (input: TInput, evalCase: EvalCase<TInput>, signal: AbortSignal) => Promise<EvalRun>
 	/**
-	 * Deadline for a single case. Unset means no deadline.
+	 * Wall-clock deadline for one case, including execution and every
+	 * scorer. Unset means no deadline.
 	 *
-	 * `executeCase` used to be a bare await, so a `run` closure that never
-	 * settled blocked its worker and `runExperiment` never returned — no
-	 * report, no partial results, nothing to read. The documented path
-	 * inherits deadlines from the runtime it drives, so the residual is a
-	 * closure that does not go through `query()` and a mid-iteration
-	 * provider stall the between-iterations guard cannot see. Both are
-	 * reachable, and neither is the suite's fault to absorb silently.
+	 * A non-cooperative `run` closure or scorer is detached by an independent
+	 * race, so it cannot hold the suite forever. The same signal is passed to
+	 * both phases; I/O-owning scorers should forward it to their transport,
+	 * as `judgeScorer` does.
 	 *
-	 * A timed-out case is REPORTED and the suite continues, exactly like a
-	 * case that threw: forty cases should not be lost to one that hung.
+	 * A run that exhausts the budget is reported as failed. A scorer that
+	 * exhausts the remainder is unavailable, making an otherwise unjudged
+	 * case inconclusive. Either way the suite continues: forty cases should
+	 * not be lost to one operation that hung.
 	 */
 	timeoutMs?: number
 	/** Mean score a case must reach to count as passed. Default 1. */
@@ -61,6 +63,7 @@ export async function runExperiment<TInput>(
 	const startedAt = Date.now()
 	const threshold = config.passThreshold ?? 1
 	const concurrency = Math.max(1, config.concurrency ?? 1)
+	const timeoutMs = validateCaseTimeoutMs(config.timeoutMs)
 
 	const results: CaseResult[] = new Array(config.cases.length)
 	let cursor = 0
@@ -71,71 +74,7 @@ export async function runExperiment<TInput>(
 			const evalCase = config.cases[index]
 			if (!evalCase) return
 
-			const run = await executeCase(config, evalCase)
-			const scorers = evalCase.scorers ?? config.scorers
-			const scores: Record<string, Score> = {}
-
-			// Scores are keyed by name, so two scorers sharing one collapse:
-			// the mean's denominator becomes the count of distinct NAMES and
-			// the surviving score is whichever ran last. Two
-			// `containsScorer(...)` instances are both called 'contains', so
-			// this is easy to hit by accident and silently halves the
-			// evidence. Ambiguous results are worse than a loud failure.
-			const seen = new Set<string>()
-			for (const scorer of scorers) {
-				if (seen.has(scorer.name)) {
-					throw new Error(
-						`Duplicate scorer name "${scorer.name}" for case "${evalCase.name}". Scores are keyed by name, so the second would overwrite the first and the case mean would be computed over the wrong denominator. Give each scorer a distinct name.`,
-					)
-				}
-				seen.add(scorer.name)
-				scores[scorer.name] = await safeScore(scorer, run, evalCase)
-			}
-
-			// Only scores that were actually produced count — an unavailable
-			// scorer leaves the denominator alone rather than dragging the
-			// mean toward zero with a measurement that never happened.
-			const values = Object.values(scores)
-				.filter((s) => s.unavailable !== true)
-				.map((s) => s.score)
-			const mean = values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length
-
-			// A gate miss fails the case whatever the mean says. Averaging a
-			// hard check together with a fuzzy one lets three good scores
-			// carry a zero: trajectory 0 + completion 1 + contains 1 +
-			// judge 1 averages to 0.75 and reports passed at a threshold of
-			// 0.75 — the exact regression the harness exists to catch,
-			// reported green. An UNAVAILABLE gate does not fail the case;
-			// it did not judge the run at all, which is the inconclusive
-			// path, not a failure.
-			const failedGates = scorers
-				.filter((s) => s.severity === 'gate')
-				.filter((s) => {
-					const score = scores[s.name]
-					if (!score || score.unavailable === true) return false
-					return score.score < (s.threshold ?? threshold)
-				})
-				.map((s) => s.name)
-
-			const status: CaseStatus =
-				Object.keys(scores).length > 0 && values.length === 0
-					? 'inconclusive'
-					: failedGates.length > 0
-						? 'failed'
-						: mean >= threshold
-							? 'passed'
-							: 'failed'
-			const result: CaseResult = {
-				case: evalCase.name,
-				run,
-				scores,
-				mean,
-				status,
-				passed: status === 'passed',
-				// Named, not just counted: "failed" with a mean of 0.75 sends
-				// somebody to read four scores and guess which one mattered.
-				...(failedGates.length > 0 ? { failedGates } : {}),
-			}
+			const result = await evaluateCase(config, evalCase, threshold, timeoutMs)
 			results[index] = result
 			config.onCaseFinish?.(result)
 		}
@@ -165,6 +104,138 @@ export async function runExperiment<TInput>(
 	}
 }
 
+function validateCaseTimeoutMs(value: number | undefined): number | undefined {
+	if (value === undefined) return undefined
+	if (!Number.isInteger(value) || value < 1 || value > MAX_CASE_TIMEOUT_MS) {
+		throw new RangeError(
+			`timeoutMs must be an integer from 1 to ${MAX_CASE_TIMEOUT_MS}, or omitted; received ${String(value)}`,
+		)
+	}
+	return value
+}
+
+interface CaseDeadline {
+	readonly signal: AbortSignal
+	race<T>(work: Promise<T>): Promise<T>
+	dispose(): void
+}
+
+/** One wall-clock budget shared by execution and every scorer for a case. */
+function openCaseDeadline(timeoutMs: number | undefined): CaseDeadline {
+	const controller = new AbortController()
+	let timer: ReturnType<typeof setTimeout> | undefined
+	let expiry: Promise<never> | undefined
+	let onAbort: (() => void) | undefined
+
+	if (timeoutMs !== undefined && timeoutMs > 0) {
+		expiry = new Promise<never>((_resolve, reject) => {
+			onAbort = () => reject(controller.signal.reason)
+			controller.signal.addEventListener('abort', onAbort, { once: true })
+		})
+		// A case can settle just before its timer. Keep the deadline promise
+		// observed independently so a same-tick expiry cannot become an
+		// unhandled rejection while the scope is being disposed.
+		void expiry.catch(() => {})
+		timer = setTimeout(
+			() => controller.abort(new Error(`case timed out after ${timeoutMs}ms`)),
+			timeoutMs,
+		)
+	}
+
+	return {
+		signal: controller.signal,
+		async race<T>(work: Promise<T>): Promise<T> {
+			if (!expiry) return await work
+			// A non-cooperative run or scorer may reject after the deadline won.
+			// Observe that loser without making suite settlement depend on it.
+			void work.catch(() => {})
+			return await Promise.race([work, expiry])
+		},
+		dispose() {
+			clearTimeout(timer)
+			if (onAbort) controller.signal.removeEventListener('abort', onAbort)
+		},
+	}
+}
+
+async function evaluateCase<TInput>(
+	config: ExperimentConfig<TInput>,
+	evalCase: EvalCase<TInput>,
+	threshold: number,
+	timeoutMs: number | undefined,
+): Promise<CaseResult> {
+	const deadline = openCaseDeadline(timeoutMs)
+	try {
+		const run = await executeCase(config, evalCase, deadline)
+		const scorers = evalCase.scorers ?? config.scorers
+		const scores: Record<string, Score> = {}
+
+		// Scores are keyed by name, so two scorers sharing one collapse:
+		// the mean's denominator becomes the count of distinct NAMES and
+		// the surviving score is whichever ran last. Two
+		// `containsScorer(...)` instances are both called 'contains', so
+		// this is easy to hit by accident and silently halves the
+		// evidence. Ambiguous results are worse than a loud failure.
+		const seen = new Set<string>()
+		for (const scorer of scorers) {
+			if (seen.has(scorer.name)) {
+				throw new Error(
+					`Duplicate scorer name "${scorer.name}" for case "${evalCase.name}". Scores are keyed by name, so the second would overwrite the first and the case mean would be computed over the wrong denominator. Give each scorer a distinct name.`,
+				)
+			}
+			seen.add(scorer.name)
+			scores[scorer.name] = await safeScore(scorer, run, evalCase, deadline)
+		}
+
+		// Only scores that were actually produced count — an unavailable
+		// scorer leaves the denominator alone rather than dragging the
+		// mean toward zero with a measurement that never happened.
+		const values = Object.values(scores)
+			.filter((s) => s.unavailable !== true)
+			.map((s) => s.score)
+		const mean = values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length
+
+		// A gate miss fails the case whatever the mean says. Averaging a
+		// hard check together with a fuzzy one lets three good scores
+		// carry a zero: trajectory 0 + completion 1 + contains 1 +
+		// judge 1 averages to 0.75 and reports passed at a threshold of
+		// 0.75 — the exact regression the harness exists to catch,
+		// reported green. An UNAVAILABLE gate does not fail the case;
+		// it did not judge the run at all, which is the inconclusive
+		// path, not a failure.
+		const failedGates = scorers
+			.filter((s) => s.severity === 'gate')
+			.filter((s) => {
+				const score = scores[s.name]
+				if (!score || score.unavailable === true) return false
+				return score.score < (s.threshold ?? threshold)
+			})
+			.map((s) => s.name)
+
+		const status: CaseStatus =
+			Object.keys(scores).length > 0 && values.length === 0
+				? 'inconclusive'
+				: failedGates.length > 0
+					? 'failed'
+					: mean >= threshold
+						? 'passed'
+						: 'failed'
+		return {
+			case: evalCase.name,
+			run,
+			scores,
+			mean,
+			status,
+			passed: status === 'passed',
+			// Named, not just counted: "failed" with a mean of 0.75 sends
+			// somebody to read four scores and guess which one mattered.
+			...(failedGates.length > 0 ? { failedGates } : {}),
+		}
+	} finally {
+		deadline.dispose()
+	}
+}
+
 /**
  * A case that throws is a RESULT, not a crash. An eval suite whose first
  * broken case aborts the run tells you nothing about the other forty.
@@ -172,38 +243,12 @@ export async function runExperiment<TInput>(
 async function executeCase<TInput>(
 	config: ExperimentConfig<TInput>,
 	evalCase: EvalCase<TInput>,
+	deadline: CaseDeadline,
 ): Promise<EvalRun> {
 	const startedAt = Date.now()
-	const controller = new AbortController()
-	const timeoutMs = config.timeoutMs
-	// The signal is handed to `run` so a closure that drives `query()` can
-	// pass it through and actually stop working. A closure that ignores it
-	// is merely detached rather than stopped — the same bargain every other
-	// deadline in the SDK makes — but the SUITE is unblocked either way,
-	// which is the part that was missing.
-	const timer =
-		timeoutMs !== undefined && timeoutMs > 0
-			? setTimeout(
-					() => controller.abort(new Error(`case timed out after ${timeoutMs}ms`)),
-					timeoutMs,
-				)
-			: undefined
-
-	const deadline =
-		timer === undefined
-			? undefined
-			: new Promise<never>((_resolve, reject) => {
-					controller.signal.addEventListener('abort', () => reject(controller.signal.reason), {
-						once: true,
-					})
-				})
-
 	try {
-		const work = config.run(evalCase.input, evalCase, controller.signal)
-		// A rejection of the loser must never surface as an unhandled
-		// rejection after the race has already been decided.
-		if (deadline) work.catch(() => {})
-		return await (deadline ? Promise.race([work, deadline]) : work)
+		const work = config.run(evalCase.input, evalCase, deadline.signal)
+		return await deadline.race(work)
 	} catch (err) {
 		return {
 			output: null,
@@ -217,16 +262,16 @@ async function executeCase<TInput>(
 			durationMs: Date.now() - startedAt,
 			error: err instanceof Error ? err.message : String(err),
 		}
-	} finally {
-		// Only the timer needs clearing. Aborting here would fire a spurious
-		// signal at a `run` that has already settled — the deadline branch
-		// has aborted already, and every other path is done.
-		clearTimeout(timer)
 	}
 }
 
 /** A throwing scorer scores zero with the throw as its reason. */
-async function safeScore(scorer: Scorer, run: EvalRun, evalCase: EvalCase): Promise<Score> {
+async function safeScore(
+	scorer: Scorer,
+	run: EvalRun,
+	evalCase: EvalCase,
+	deadline: CaseDeadline,
+): Promise<Score> {
 	// A run that THREW scores zero, whatever the scorer would have said.
 	// `executeCase` catches the failure and returns an empty run, and an
 	// empty run walks straight into every scorer's happy path:
@@ -239,7 +284,11 @@ async function safeScore(scorer: Scorer, run: EvalRun, evalCase: EvalCase): Prom
 	}
 
 	try {
-		return await scorer.score(run, evalCase)
+		deadline.signal.throwIfAborted()
+		// Defer invocation into the promise so a synchronous scorer throw is
+		// raced and observed by the same path as an asynchronous rejection.
+		const work = Promise.resolve().then(() => scorer.score(run, evalCase, deadline.signal))
+		return await deadline.race(work)
 	} catch (err) {
 		// UNAVAILABLE, not zero. A scorer that threw did not judge the run
 		// badly — it failed to judge it at all, and the two call for

package/src/eval/judge.ts

@@ -1,4 +1,5 @@
 import { collectChatCompletion } from '../provider/collect-chat-completion.js'
+import { resolveStreamIdleTimeoutMs, withStreamIdleTimeout } from '../provider/idle-timeout.js'
 import type { LLMProvider } from '../types/provider/interface.js'
 import type { EvalCase, EvalRun, Score, Scorer } from './types.js'
 
@@ -60,6 +61,12 @@ export interface JudgeScorerConfig {
 	 * truncation rather than the run.
 	 */
 	maxOutputChars?: number
+	/**
+	 * Milliseconds without a judge chunk before its provider stream is
+	 * treated as stalled. Defaults to the SDK's finite provider-stream idle
+	 * bound. Set `0` only for explicit unbounded compatibility.
+	 */
+	streamIdleTimeoutMs?: number
 }
 
 const DEFAULT_SCALE = 4
@@ -198,6 +205,9 @@ function buildPrompt(
 
 export function judgeScorer(config: JudgeScorerConfig): Scorer {
 	const scale = config.scale ?? DEFAULT_SCALE
+	const provider = withStreamIdleTimeout(config.provider, {
+		idleTimeoutMs: resolveStreamIdleTimeoutMs(config.streamIdleTimeoutMs),
+	})
 	if (!Number.isInteger(scale) || scale < 1) {
 		throw new Error(`judgeScorer: scale must be a positive integer, got ${scale}`)
 	}
@@ -209,15 +219,16 @@ export function judgeScorer(config: JudgeScorerConfig): Scorer {
 
 	return {
 		name: config.name ?? 'judge',
-		async score(run: EvalRun, evalCase: EvalCase): Promise<Score> {
+		async score(run: EvalRun, evalCase: EvalCase, signal?: AbortSignal): Promise<Score> {
 			const response = await collectChatCompletion(
-				config.provider.chatStream({
+				provider.chatStream({
 					model: config.model,
 					messages: [{ role: 'user', content: buildPrompt(config, run, evalCase, scale) }],
 					// The same run must grade the same way twice, or a
 					// regression cannot be told from sampling noise.
 					temperature: 0,
 					maxTokens: 512,
+					...(signal ? { signal } : {}),
 				}),
 			)
 

package/src/eval/types.ts

@@ -98,7 +98,13 @@ export type ScorerSeverity = 'gate' | 'soft'
 
 export interface Scorer {
 	name: string
-	score(run: EvalRun, evalCase: EvalCase): Score | Promise<Score>
+	/**
+	 * Produce one measurement. `runExperiment` passes the case's shared
+	 * deadline signal, which covers both execution and every scorer. A pure
+	 * scorer may ignore it; a scorer that owns I/O should carry it to that
+	 * transport so timed-out work is stopped rather than merely abandoned.
+	 */
+	score(run: EvalRun, evalCase: EvalCase, signal?: AbortSignal): Score | Promise<Score>
 	/** See {@link ScorerSeverity}. Default `soft`. */
 	severity?: ScorerSeverity
 	/**

package/src/manager/agent/lifecycle.ts

@@ -40,6 +40,7 @@ import { generateTaskId } from '../../utils/id.js'
 import { SCOPE_ATTRIBUTE } from '../../utils/log/types.js'
 import { type Logger, resolveLogger } from '../../utils/logger.js'
 import { requireOpenProject } from '../project/lifecycle.js'
+import { type TopicManagerDependency, resolveTopicManager } from '../topic/dependency.js'
 import type { TopicManager } from '../topic/lifecycle.js'
 
 /**
@@ -61,19 +62,11 @@ import type { TopicManager } from '../topic/lifecycle.js'
  * registry deny-by-default while matching pattern doc §7.1 (lazy workspace
  * provisioning).
  */
-export interface AgentManagerDeps {
+interface AgentManagerBaseDeps {
 	readonly sessionStore: SessionStore
 	readonly workspaceRegistry: WorkspaceBackendRegistry
 	readonly summaryMaterializer: SessionSummaryMaterializer
 	readonly capacity: CapacityValidator
-	/**
-	 * Gate session creation on the parent Topic being `'open'` via
-	 * {@link TopicManager.requireOpen}. Added in Phase 2.6 to close the
-	 * archive-gate gap flagged by the Phase 2.5 commit: without this,
-	 * `TopicManager.archive` was best-effort because spawn could still
-	 * attach a live session under an archived Thread.
-	 */
-	readonly threadManager: TopicManager
 
 	/**
 	 * A pre-built logger. No in-package caller threads a real one today —
@@ -89,6 +82,16 @@ export interface AgentManagerDeps {
 	readonly log?: Logger
 }
 
+/**
+ * Dependencies for {@link AgentManager}.
+ *
+ * `topicManager` gates child-session creation on the parent Topic being open.
+ * The deprecated `threadManager` spelling remains accepted for one migration
+ * window through {@link TopicManagerDependency}; new callers use
+ * `topicManager`.
+ */
+export type AgentManagerDeps = AgentManagerBaseDeps & TopicManagerDependency
+
 interface ChildSpawnRecord {
 	subSessionId: SubSessionId
 	childSessionId: SessionId
@@ -138,6 +141,7 @@ export class AgentManager {
 	/** One provisioning at a time per parent — see {@link provisionSpawn}. */
 	private spawnLocks: Map<SessionId, Promise<void>> = new Map()
 	private deps: AgentManagerDeps
+	private topicManager: TopicManager
 
 	constructor(
 		registry: AgentRegistry,
@@ -146,8 +150,11 @@ export class AgentManager {
 	) {
 		this.registry = registry
 		this.config = { ...AGENT_MANAGER_DEFAULTS, ...config }
-		this.log = resolveLogger(deps.log).child({ [SCOPE_ATTRIBUTE]: 'manager/agent/lifecycle' })
+		this.log = resolveLogger(deps.log).child({
+			[SCOPE_ATTRIBUTE]: 'manager/agent/lifecycle',
+		})
 		this.deps = deps
+		this.topicManager = resolveTopicManager(deps)
 	}
 
 	async sendMessage(
@@ -387,6 +394,8 @@ export class AgentManager {
 				model: options.configOverrides?.model ?? 'default',
 				tokenBudget: allocatedTokens,
 				timeoutMs: options.budgetAllocation?.timeoutMs ?? this.config.childTimeoutMs,
+				streamIdleTimeoutMs: options.configOverrides?.streamIdleTimeoutMs,
+				maxRequestRichContentBytes: options.configOverrides?.maxRequestRichContentBytes,
 				temperature: options.configOverrides?.temperature,
 				parentSpan: options.configOverrides?.parentSpan,
 				maxIterations: options.configOverrides?.maxIterations,
@@ -408,6 +417,13 @@ export class AgentManager {
 			}
 		}
 
+		// Lineage is assigned by the spawning manager, not proposed by the
+		// child definition. A fixed configBuilder can ignore its inputs and
+		// configOverrides is caller-authored; neither may turn a child back
+		// into depth zero or attach it to a different parent run.
+		childConfig.parentRunId = context.parentRunId
+		childConfig.depth = context.depth + 1
+
 		// Stamped AFTER the builder, for the fourth time and the same reason:
 		// a `configBuilder` is written by whoever registered the agent and
 		// cannot forward a field it was never told about. A scope applied
@@ -668,7 +684,7 @@ export class AgentManager {
 		// partial/legacy path).
 		const store = this.deps.sessionStore
 
-		// Thread archive gate — runs FIRST so an archived thread fails fastest
+		// Topic archive gate — runs FIRST so an archived Topic fails fastest
 		// with the correct error (not DelegationCapacityExceeded or a project
 		// lookup error). Phase 2.6 closes the gap the Phase 2.5 commit
 		// flagged: without it, `TopicManager.archive` could be undermined by
@@ -678,7 +694,7 @@ export class AgentManager {
 		// callers of `SessionStore.createSession` bypass it — the store layer
 		// is intentionally unaware of topic status to preserve its
 		// single-responsibility boundary.
-		await this.deps.threadManager.requireOpen(context.topicId, context.tenantId)
+		await this.topicManager.requireOpen(context.topicId, context.tenantId)
 
 		// Parent session cross-check: validate that `options.parentSessionId`
 		// exists for this tenant AND lives under the same topic as the

package/src/manager/connector/environment.ts

@@ -10,6 +10,7 @@ import type {
 	ConnectorExecuteResult,
 	ConnectorInstance,
 	ConnectorLifecycleEvent,
+	ConnectorOperationOptions,
 	CredentialVault,
 	EnvironmentDescriptor,
 	ExecutionContextConfig,
@@ -205,9 +206,10 @@ export class EnvironmentConnectorManager {
 	async healthCheck(
 		envId: EnvironmentId,
 		instanceId: import('../../types/ids/index.js').ConnectorInstanceId,
+		options?: ConnectorOperationOptions,
 	): Promise<boolean> {
 		const state = this.getEnvironmentOrThrow(envId)
-		return state.manager.healthCheck(instanceId)
+		return state.manager.healthCheck(instanceId, options)
 	}
 
 	listInstances(envId: EnvironmentId): ConnectorInstance[] {