@namzu/sdk 2.0.0 → 3.0.0

package/src/telemetry/__tests__/metrics.test.ts

@@ -0,0 +1,160 @@
+import { type Attributes, type Meter, metrics } from '@opentelemetry/api'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+
+import {
+	recordModelDuration,
+	recordRunDuration,
+	recordTokenUsage,
+	recordToolCall,
+	resetRuntimeMetrics,
+} from '../metrics.js'
+
+/**
+ * Metrics lived in a bag a host was expected to construct, and nothing in
+ * the workspace ever constructed one — so the runtime emitted spans and not
+ * a single measurement. Worse, the bag bound its instruments eagerly: one
+ * built before the provider was installed captured the no-op meter and
+ * discarded every write for the rest of its life, silently, from one line
+ * of call order.
+ */
+
+interface Written {
+	instrument: string
+	value: number
+	attributes: Attributes
+}
+
+function fakeMeterProvider(written: Written[]) {
+	const instrument = (name: string) => ({
+		add: (value: number, attributes: Attributes = {}) =>
+			written.push({ instrument: name, value, attributes }),
+		record: (value: number, attributes: Attributes = {}) =>
+			written.push({ instrument: name, value, attributes }),
+	})
+	const meter = {
+		createCounter: (name: string) => instrument(name),
+		createHistogram: (name: string) => instrument(name),
+		createUpDownCounter: (name: string) => instrument(name),
+		createObservableGauge: (name: string) => instrument(name),
+		createObservableCounter: (name: string) => instrument(name),
+		createObservableUpDownCounter: (name: string) => instrument(name),
+		addBatchObservableCallback: () => {},
+		removeBatchObservableCallback: () => {},
+	} as unknown as Meter
+	return { getMeter: () => meter }
+}
+
+let written: Written[]
+
+beforeEach(() => {
+	written = []
+	resetRuntimeMetrics()
+	metrics.setGlobalMeterProvider(fakeMeterProvider(written))
+})
+
+afterEach(() => {
+	metrics.disable()
+	resetRuntimeMetrics()
+})
+
+const attributesOf = (name: string) => written.filter((w) => w.instrument === name)
+
+describe('token usage', () => {
+	it('records every kind under ONE metric, split by type', () => {
+		recordTokenUsage('m', {
+			promptTokens: 100,
+			completionTokens: 20,
+			cachedTokens: 500,
+			cacheWriteTokens: 40,
+		})
+
+		const writes = attributesOf('gen_ai.client.token.usage')
+		// It was two metrics under two names, the second invented — so a
+		// dashboard aggregating the conventional name saw input only.
+		expect(writes).toHaveLength(4)
+		expect(new Set(writes.map((w) => w.attributes['gen_ai.token.type']))).toEqual(
+			new Set(['input', 'output', 'cache_read', 'cache_write']),
+		)
+		expect(writes.reduce((sum, w) => sum + w.value, 0)).toBe(660)
+	})
+
+	it('records cache traffic as its own types rather than folding it into input', () => {
+		recordTokenUsage('m', { promptTokens: 10, completionTokens: 0, cachedTokens: 900 })
+
+		const input = attributesOf('gen_ai.client.token.usage').find(
+			(w) => w.attributes['gen_ai.token.type'] === 'input',
+		)
+		// A read bills at a fraction of the input rate; summing them would
+		// make the total unable to explain a bill.
+		expect(input?.value).toBe(10)
+	})
+
+	it('writes nothing for a kind that was zero', () => {
+		recordTokenUsage('m', { promptTokens: 5, completionTokens: 0 })
+		expect(attributesOf('gen_ai.client.token.usage')).toHaveLength(1)
+	})
+
+	it('carries the model on every write', () => {
+		recordTokenUsage('some-model', { promptTokens: 1, completionTokens: 1 })
+		for (const write of attributesOf('gen_ai.client.token.usage')) {
+			expect(write.attributes['gen_ai.request.model']).toBe('some-model')
+		}
+	})
+})
+
+describe('tool calls', () => {
+	it('records the outcome and why it failed', () => {
+		recordToolCall('read', false, 'ENOENT')
+		const write = attributesOf('gen_ai.tool.call.count')[0]
+		expect(write?.attributes['gen_ai.tool.name']).toBe('read')
+		expect(write?.attributes['namzu.tool.success']).toBe(false)
+		// A flat success rate cannot separate a broken tool from one whose
+		// input the model keeps getting wrong.
+		expect(write?.attributes['namzu.tool.error']).toBe('ENOENT')
+	})
+
+	it('leaves the error attribute off a success', () => {
+		recordToolCall('read', true)
+		expect(attributesOf('gen_ai.tool.call.count')[0]?.attributes).not.toHaveProperty(
+			'namzu.tool.error',
+		)
+	})
+})
+
+describe('durations', () => {
+	it('records a run in seconds, keyed by how it settled', () => {
+		recordRunDuration('completed', 2500)
+		const write = attributesOf('namzu.run.duration')[0]
+		expect(write?.value).toBe(2.5)
+		expect(write?.attributes['namzu.run.status']).toBe('completed')
+	})
+
+	it('records a model call in seconds', () => {
+		recordModelDuration('m', 1500)
+		expect(attributesOf('gen_ai.client.operation.duration')[0]?.value).toBe(1.5)
+	})
+})
+
+describe('binding', () => {
+	it('records through a provider installed AFTER the first write', () => {
+		// The bug this replaces: an instrument bound before registration
+		// captured the no-op meter and threw every write away forever.
+		metrics.disable()
+		resetRuntimeMetrics()
+		recordToolCall('early', true)
+
+		const late: Written[] = []
+		metrics.setGlobalMeterProvider(fakeMeterProvider(late))
+		recordToolCall('late', true)
+
+		expect(late.map((w) => w.attributes['gen_ai.tool.name'])).toEqual(['late'])
+	})
+
+	it('reuses instruments while the meter is unchanged', () => {
+		recordToolCall('a', true)
+		recordToolCall('b', true)
+		// Rebuilding per call would allocate on a hot path; both writes must
+		// still land.
+		expect(attributesOf('gen_ai.tool.call.count')).toHaveLength(2)
+	})
+})

package/src/telemetry/__tests__/span-hierarchy.test.ts

@@ -0,0 +1,94 @@
+import { type Span, SpanStatusCode, context as otelContext, trace } from '@opentelemetry/api'
+import { describe, expect, it } from 'vitest'
+
+import { parentContext } from '../attributes.js'
+
+/**
+ * OTel's GenAI conventions define a strict hierarchy —
+ * `invoke_agent` → `chat {model}` → `execute_tool` — and vendor dashboards
+ * (Langfuse, Braintrust, Phoenix, Datadog) rely on it. namzu emitted every
+ * span as a ROOT: a repo-wide grep for `context.with` / `trace.setSpan`
+ * returned zero hits, so a single 20-iteration run produced 21 disconnected
+ * traces with no waterfall and no LLM latency at all.
+ *
+ * The subtlety this file exists to pin: `startActiveSpan` does NOT hold
+ * context across `yield`. Every span-owning body in the run loop is an async
+ * generator, and a generator resumes on its CONSUMER's async context — so
+ * the naive conversion silently fails to parent anything, and the only fix
+ * that works is passing the parent explicitly.
+ */
+
+/** Minimal in-memory span capture; avoids pulling in the SDK exporter. */
+function fakeSpan(name: string): Span {
+	return {
+		spanContext: () => ({ traceId: name, spanId: name, traceFlags: 1 }),
+		setAttribute: () => fakeSpan(name),
+		setAttributes: () => fakeSpan(name),
+		addEvent: () => fakeSpan(name),
+		setStatus: () => fakeSpan(name),
+		updateName: () => fakeSpan(name),
+		end: () => {},
+		isRecording: () => true,
+		recordException: () => {},
+		addLink: () => fakeSpan(name),
+		addLinks: () => fakeSpan(name),
+	} as unknown as Span
+}
+
+describe('parentContext', () => {
+	it('returns the ambient context when there is no parent', () => {
+		expect(parentContext(undefined)).toBe(otelContext.active())
+	})
+
+	it('puts the given span into the returned context', () => {
+		const parent = fakeSpan('run')
+		const ctx = parentContext(parent)
+		expect(trace.getSpan(ctx)).toBe(parent)
+	})
+
+	it('does not mutate the ambient context', () => {
+		const before = trace.getSpan(otelContext.active())
+		parentContext(fakeSpan('run'))
+		expect(trace.getSpan(otelContext.active())).toBe(before)
+	})
+
+	it('survives an await — which is exactly what the ambient context does not', async () => {
+		// This is the whole reason the helper exists. An explicit context is
+		// a value; the ambient one is per-async-scope and is gone by the time
+		// a generator resumes on its consumer's stack.
+		const parent = fakeSpan('run')
+		const ctx = parentContext(parent)
+		await Promise.resolve()
+		await new Promise((r) => setTimeout(r, 0))
+		expect(trace.getSpan(ctx)).toBe(parent)
+	})
+
+	it('demonstrates the failure mode it replaces', async () => {
+		// `context.with` establishes an ambient parent only for the
+		// synchronous body plus awaited continuations it owns. Reading it
+		// from a *separately scheduled* task sees nothing — which is the
+		// position every async generator in the run loop is in.
+		const parent = fakeSpan('run')
+		let observedInsideDetachedTask: Span | undefined
+
+		await otelContext.with(trace.setSpan(otelContext.active(), parent), async () => {
+			await new Promise<void>((resolve) => {
+				setTimeout(() => {
+					observedInsideDetachedTask = trace.getSpan(otelContext.active())
+					resolve()
+				}, 0)
+			})
+		})
+
+		expect(observedInsideDetachedTask).toBeUndefined()
+		// …while the explicit context still carries the parent.
+		expect(trace.getSpan(parentContext(parent))).toBe(parent)
+	})
+})
+
+describe('span status constants are the ones we stamp', () => {
+	it('uses OK / ERROR rather than ad-hoc strings', () => {
+		expect(SpanStatusCode.OK).toBeDefined()
+		expect(SpanStatusCode.ERROR).toBeDefined()
+	})
+})

package/src/telemetry/attributes.ts

@@ -1,3 +1,4 @@
+import { context as otelContext, trace } from '@opentelemetry/api'
 export { GENAI, NAMZU } from '../constants/telemetry/index.js'
 
 export function agentRunSpanName(agentName: string): string {
@@ -11,3 +12,111 @@ export function agentIterationSpanName(iteration: number): string {
 export function toolSpanName(toolName: string): string {
 	return `namzu.tool.execute ${toolName}`
 }
+
+/**
+ * OTel GenAI semconv names the model-call span `chat {model}`.
+ *
+ * There was no span around the model call at all — `chatSpanName` existed
+ * in `@namzu/telemetry` with zero call sites — so a run's traces carried
+ * no LLM latency whatsoever, and the token attributes landed on the
+ * iteration span instead of the operation that produced them.
+ */
+export function chatSpanName(model: string): string {
+	return `chat ${model}`
+}
+
+/**
+ * Build a context that parents a new span to `parent`.
+ *
+ * Async generators cannot use `startActiveSpan` for parenting: a generator
+ * body resumes on its CONSUMER's async context, so whatever was active when
+ * the parent span was created is gone by the time the child is made. Every
+ * span-owning body in the run loop is a generator, which is why a 20-turn
+ * run emitted 21 disconnected root spans. Passing the parent explicitly is
+ * the only thing that works here.
+ */
+export function parentContext(parent?: import('@opentelemetry/api').Span | SerializedSpanContext) {
+	const active = otelContext.active()
+	if (!parent) return active
+	// A live span goes in as itself. Round-tripping it through
+	// `wrapSpanContext` would replace a recording span with a non-recording
+	// stand-in — same ids, but the parent stops being the object callers
+	// hold, and anything reading it back gets a hollow one.
+	if (!isSerialized(parent)) return trace.setSpan(active, parent)
+
+	const context = toSpanContext(parent)
+	if (!context) return active
+	return trace.setSpan(active, trace.wrapSpanContext(context))
+}
+
+/**
+ * A span context flattened for storage.
+ *
+ * `parentContext` used to accept only a live in-memory span, so a parent
+ * that had to survive a process boundary could not be expressed at all — a
+ * run that crashed at iteration 12 and resumed produced two traces with
+ * different trace ids and no link between them, and the crash and its
+ * recovery could not be put on one timeline. Every span carries
+ * `namzu.run.id`, which is enough to FIND both traces by query and not
+ * enough to see one waterfall; and that much disappears for a replay fork,
+ * which mints a new run id.
+ *
+ * Flat strings rather than the OTel type so it survives `JSON.stringify`
+ * into a checkpoint unchanged.
+ */
+export interface SerializedSpanContext {
+	readonly traceId: string
+	readonly spanId: string
+	readonly traceFlags: number
+	/** Whether the parent lives in a different process. Always true on read. */
+	readonly isRemote?: boolean
+}
+
+function isSerialized(
+	value: import('@opentelemetry/api').Span | SerializedSpanContext,
+): value is SerializedSpanContext {
+	return typeof (value as SerializedSpanContext).traceId === 'string'
+}
+
+const HEX_TRACE_ID = /^[0-9a-f]{32}$/
+const HEX_SPAN_ID = /^[0-9a-f]{16}$/
+
+/**
+ * Rebuild an OTel span context, or return `undefined` for one that is not
+ * usable.
+ *
+ * Both ids are validated: an exporter given a malformed or all-zero id
+ * silently drops the span, so an unchecked value turns "the resume is
+ * linked to the crash" into "there is no resume trace at all" — a strictly
+ * worse outcome than the disconnected traces this replaces.
+ */
+function toSpanContext(
+	serialized: SerializedSpanContext,
+): import('@opentelemetry/api').SpanContext | undefined {
+	const { traceId, spanId } = serialized
+	if (!HEX_TRACE_ID.test(traceId) || !HEX_SPAN_ID.test(spanId)) return undefined
+	if (/^0+$/.test(traceId) || /^0+$/.test(spanId)) return undefined
+	return {
+		traceId,
+		spanId,
+		traceFlags: serialized.traceFlags,
+		// The parent was recorded in another process, whatever that process
+		// was; saying otherwise makes a sampler treat it as locally sampled.
+		isRemote: true,
+	}
+}
+
+/** Flatten a live span for storage, or `undefined` when there is nothing real to store. */
+export function serializeSpan(
+	span?: import('@opentelemetry/api').Span,
+): SerializedSpanContext | undefined {
+	const context = span?.spanContext()
+	if (!context) return undefined
+	if (!HEX_TRACE_ID.test(context.traceId) || /^0+$/.test(context.traceId)) return undefined
+	return {
+		traceId: context.traceId,
+		spanId: context.spanId,
+		traceFlags: context.traceFlags,
+		isRemote: false,
+	}
+}

package/src/telemetry/metrics.ts

@@ -0,0 +1,190 @@
+import { type Counter, type Histogram, type Meter, metrics } from '@opentelemetry/api'
+import { GENAI, NAMZU } from '../constants/telemetry/index.js'
+
+/**
+ * The runtime's metric instruments.
+ *
+ * Metrics lived in the telemetry package as a bag a host constructed for
+ * itself — and nothing in the workspace ever constructed one, so the
+ * runtime emitted spans and not a single measurement. Defining them beside
+ * the code that records them is what stops that recurring: an instrument
+ * with no call site is visible here, where a bag with no owner was not.
+ *
+ * Every instrument is resolved LAZILY. Binding eagerly meant that a bag
+ * built before `registerTelemetry()` captured the no-op meter and threw
+ * every write away for the rest of its life — silently, and forever, from
+ * one line of call-order. Ordering is not something a caller should have
+ * to get right to be measured at all.
+ */
+
+let cached: { meter: Meter; instruments: Instruments } | undefined
+
+interface Instruments {
+	tokens: Counter
+	toolCalls: Counter
+	runDuration: Histogram
+	llmDuration: Histogram
+	timeToFirstToken: Histogram
+	toolDuration: Histogram
+}
+
+function build(meter: Meter): Instruments {
+	return {
+		// ONE counter for every token, split by `gen_ai.token.type`. It was
+		// two counters under two names, the second of them invented — so a
+		// dashboard aggregating the conventional name got input tokens only
+		// and under-reported usage by roughly half, with nothing to suggest
+		// the other half existed.
+		tokens: meter.createCounter('gen_ai.client.token.usage', {
+			description: 'Tokens used, by type',
+			unit: '{token}',
+		}),
+		toolCalls: meter.createCounter('gen_ai.tool.call.count', {
+			description: 'Tool calls executed',
+			unit: '{call}',
+		}),
+		runDuration: meter.createHistogram('namzu.run.duration', {
+			description: 'Agent run duration',
+			unit: 's',
+		}),
+		llmDuration: meter.createHistogram('gen_ai.client.operation.duration', {
+			description: 'Model request duration',
+			unit: 's',
+		}),
+		// namzu streams, so perceived latency is dominated by how long the
+		// user waits for the FIRST token, not by how long the whole request
+		// takes. The request histogram above cannot tell a fast-first-token
+		// long generation from a stalled one, and no host could recover the
+		// distinction from namzu's data in any form.
+		timeToFirstToken: meter.createHistogram('gen_ai.client.time_to_first_token', {
+			description: 'Time from request start to the first content delta',
+			unit: 's',
+		}),
+		// The wall clock was measured since the first version of the executor
+		// and emitted per call on `tool_completed`, so a p95 was computable
+		// from events — but there was no instrument, with the value already
+		// in scope one frame above the call site.
+		toolDuration: meter.createHistogram('gen_ai.tool.call.duration', {
+			description: 'Tool execution duration',
+			unit: 's',
+		}),
+	}
+}
+
+/**
+ * Resolve instruments against the meter that is live *now*.
+ *
+ * Cached against the meter identity rather than unconditionally: rebuilding
+ * on every call would allocate on a hot path, and never rebuilding is the
+ * bug this replaces. When `registerTelemetry()` swaps the global provider
+ * the meter changes, the cache misses once, and everything after that is
+ * recorded through the real one.
+ */
+function instruments(): Instruments {
+	const meter = metrics.getMeter('namzu')
+	if (cached?.meter === meter) return cached.instruments
+	const built = build(meter)
+	cached = { meter, instruments: built }
+	return built
+}
+
+/** Drop the cache. Tests that swap the global provider need this. */
+export function resetRuntimeMetrics(): void {
+	cached = undefined
+}
+
+export interface TokenUsageSample {
+	readonly promptTokens: number
+	readonly completionTokens: number
+	/** Prompt tokens served from the provider's cache. */
+	readonly cachedTokens?: number
+	/** Prompt tokens written into the provider's cache. */
+	readonly cacheWriteTokens?: number
+}
+
+/**
+ * Record what a turn cost, split by token type.
+ *
+ * Cache reads and writes are recorded as their own types rather than
+ * folded into `input`. They bill differently — a read is a fraction of the
+ * input rate and a write is a premium on it — so a total that hides them
+ * cannot explain a bill, which is the main thing anyone asks a token
+ * metric.
+ */
+export function recordTokenUsage(model: string, usage: TokenUsageSample): void {
+	const { tokens } = instruments()
+	if (usage.promptTokens > 0) {
+		tokens.add(usage.promptTokens, {
+			[GENAI.REQUEST_MODEL]: model,
+			[GENAI.TOKEN_TYPE]: 'input',
+		})
+	}
+	if (usage.completionTokens > 0) {
+		tokens.add(usage.completionTokens, {
+			[GENAI.REQUEST_MODEL]: model,
+			[GENAI.TOKEN_TYPE]: 'output',
+		})
+	}
+	if (usage.cachedTokens !== undefined && usage.cachedTokens > 0) {
+		tokens.add(usage.cachedTokens, {
+			[GENAI.REQUEST_MODEL]: model,
+			[GENAI.TOKEN_TYPE]: 'cache_read',
+		})
+	}
+	if (usage.cacheWriteTokens !== undefined && usage.cacheWriteTokens > 0) {
+		tokens.add(usage.cacheWriteTokens, {
+			[GENAI.REQUEST_MODEL]: model,
+			[GENAI.TOKEN_TYPE]: 'cache_write',
+		})
+	}
+}
+
+/**
+ * Record a tool call.
+ *
+ * `errorType` carries WHY a call failed rather than only that it did. A
+ * flat success rate cannot separate a tool that is broken from one whose
+ * input the model keeps getting wrong, and those need different fixes.
+ */
+export function recordToolCall(
+	toolName: string,
+	success: boolean,
+	errorType?: string,
+	durationMs?: number,
+): void {
+	const attributes = {
+		[GENAI.TOOL_NAME]: toolName,
+		[NAMZU.TOOL_SUCCESS]: success,
+		...(errorType !== undefined ? { [NAMZU.TOOL_ERROR]: errorType } : {}),
+	}
+	const { toolCalls, toolDuration } = instruments()
+	toolCalls.add(1, attributes)
+	// Same attributes as the count, so "which tool is slow" and "which tool
+	// fails" are answerable from one query rather than two that cannot be
+	// joined.
+	if (durationMs !== undefined) toolDuration.record(durationMs / 1000, attributes)
+}
+
+/**
+ * Record how long the first content delta took to arrive.
+ *
+ * Recorded at the first delta rather than at the end of the request,
+ * because the two are not the same measurement and only this one tracks
+ * what a person waiting actually experiences.
+ */
+export function recordTimeToFirstToken(model: string, durationMs: number): void {
+	instruments().timeToFirstToken.record(durationMs / 1000, { [GENAI.REQUEST_MODEL]: model })
+}
+
+/** Record how long a whole run took, keyed by how it settled. */
+export function recordRunDuration(status: string, durationMs: number): void {
+	instruments().runDuration.record(durationMs / 1000, { [NAMZU.RUN_STATUS]: status })
+}
+
+/** Record how long one model request took. */
+export function recordModelDuration(model: string, durationMs: number, errorType?: string): void {
+	instruments().llmDuration.record(durationMs / 1000, {
+		[GENAI.REQUEST_MODEL]: model,
+		...(errorType !== undefined ? { [NAMZU.TOOL_ERROR]: errorType } : {}),
+	})
+}

package/src/test-setup.ts

@@ -11,9 +11,9 @@
  * `ToolRegistry.execute`'s zod-validation and thrown-error branches,
  * `ConnectorToolRouter.getTools`' per-instance error catch, the
  * AgentBus listener-throw handler, and every connector's `connect()`
- * info log. GitHub Actions annotates any `[ERROR]` stderr line as a
- * workflow error; silencing the root logger during tests keeps the CI
- * log clean.
+ * info log. CI annotates any `[ERROR]` stderr line as a workflow
+ * error; silencing the root logger during tests keeps the CI log
+ * clean.
  *
  * This is test-only configuration. It runs before every test file and
  * does not affect consumers.

package/src/test-support/paths.ts

@@ -0,0 +1,17 @@
+/**
+ * Test-only path helpers.
+ *
+ * `node:path.join` produces platform-native separators, which is correct —
+ * the SDK should hand the OS the paths the OS expects. Assertions written
+ * as POSIX string literals therefore fail on Windows even though the code
+ * is right, which used to leave three suites permanently red for every
+ * Windows contributor. Compare path *structure* instead of separator style.
+ */
+
+/** Normalize a path's separators so an assertion can be written once. */
+export function posix(p: string | undefined): string {
+	return (p ?? '').replaceAll('\\', '/')
+}
+
+/** True when running on Windows, for the few cases that cannot be normalized away. */
+export const IS_WINDOWS = process.platform === 'win32'