@namzu/sdk 0.4.3 → 0.4.5

package/src/probe/registry.ts

@@ -0,0 +1,276 @@
+import type {
+	ProbeContext,
+	ProbeEvent,
+	ProbeEventKind,
+	ProbeEventOf,
+	ProbeHandler,
+	ProbeOptions,
+	Unsubscribe,
+	VetoDecision,
+	VetoHandler,
+	VetoOutcome,
+	VetoableEventKind,
+} from '../types/probe/index.js'
+import type { Logger } from '../utils/logger.js'
+
+import { ProbeNameCollisionError } from './errors.js'
+
+interface ProbeEntry {
+	readonly id: number
+	readonly name?: string
+	readonly priority: number
+	readonly handler: (event: ProbeEvent, ctx: ProbeContext) => void
+	readonly where?: (event: ProbeEvent) => boolean
+}
+
+interface VetoEntry {
+	readonly id: number
+	readonly name?: string
+	readonly priority: number
+	readonly handler: (event: ProbeEvent, ctx: ProbeContext) => VetoDecision
+	readonly where?: (event: ProbeEvent) => boolean
+}
+
+function compareVeto(a: VetoEntry, b: VetoEntry): number {
+	if (a.priority !== b.priority) return a.priority - b.priority
+	return a.id - b.id
+}
+
+function insertSortedVeto(list: VetoEntry[], entry: VetoEntry): void {
+	let lo = 0
+	let hi = list.length
+	while (lo < hi) {
+		const mid = (lo + hi) >>> 1
+		if (compareVeto(list[mid] as VetoEntry, entry) <= 0) lo = mid + 1
+		else hi = mid
+	}
+	list.splice(lo, 0, entry)
+}
+
+function removeVeto(list: VetoEntry[], entry: VetoEntry): void {
+	const idx = list.indexOf(entry)
+	if (idx >= 0) list.splice(idx, 1)
+}
+
+function normalizeDecision(decision: VetoDecision): { allow: boolean; reason?: string } {
+	if (decision === 'allow') return { allow: true }
+	if (decision === 'deny') return { allow: false }
+	return { allow: false, reason: decision.reason }
+}
+
+function compareEntries(a: ProbeEntry, b: ProbeEntry): number {
+	if (a.priority !== b.priority) return a.priority - b.priority
+	return a.id - b.id
+}
+
+function insertSorted(list: ProbeEntry[], entry: ProbeEntry): void {
+	let lo = 0
+	let hi = list.length
+	while (lo < hi) {
+		const mid = (lo + hi) >>> 1
+		if (compareEntries(list[mid] as ProbeEntry, entry) <= 0) lo = mid + 1
+		else hi = mid
+	}
+	list.splice(lo, 0, entry)
+}
+
+function removeEntry(list: ProbeEntry[], entry: ProbeEntry): void {
+	const idx = list.indexOf(entry)
+	if (idx >= 0) list.splice(idx, 1)
+}
+
+export class ProbeRegistry {
+	private readonly typedByKind: Map<ProbeEventKind, ProbeEntry[]> = new Map()
+	private readonly vetoByKind: Map<VetoableEventKind, VetoEntry[]> = new Map()
+	private readonly catchAll: ProbeEntry[] = []
+	private readonly byName: Map<string, ProbeEntry | VetoEntry> = new Map()
+	private nextId = 1
+	private log?: Logger
+
+	setLogger(log: Logger): void {
+		this.log = log.child({ component: 'ProbeRegistry' })
+	}
+
+	on<K extends ProbeEventKind>(
+		kind: K | readonly K[],
+		handler: ProbeHandler<K>,
+		opts: ProbeOptions<K> = {},
+	): Unsubscribe {
+		const kinds: readonly K[] = Array.isArray(kind) ? kind : [kind as K]
+		const entry = this.makeEntry(
+			handler as (e: ProbeEvent, c: ProbeContext) => void,
+			opts as unknown as ProbeOptions,
+		)
+
+		for (const k of kinds) {
+			let bucket = this.typedByKind.get(k)
+			if (!bucket) {
+				bucket = []
+				this.typedByKind.set(k, bucket)
+			}
+			insertSorted(bucket, entry)
+		}
+
+		return () => {
+			for (const k of kinds) {
+				const bucket = this.typedByKind.get(k)
+				if (bucket) removeEntry(bucket, entry)
+			}
+			if (entry.name) this.byName.delete(entry.name)
+		}
+	}
+
+	onAny(
+		handler: (event: ProbeEvent, ctx: ProbeContext) => void,
+		opts: ProbeOptions = {},
+	): Unsubscribe {
+		const entry = this.makeEntry(handler, opts)
+		insertSorted(this.catchAll, entry)
+		return () => {
+			removeEntry(this.catchAll, entry)
+			if (entry.name) this.byName.delete(entry.name)
+		}
+	}
+
+	dispatch(event: ProbeEvent, ctx: ProbeContext, betweenTier?: () => void): void {
+		const frozen = Object.isFrozen(event) ? event : Object.freeze(event)
+		this.runTier(this.typedByKind.get(frozen.type) ?? [], frozen, ctx)
+		if (betweenTier) {
+			try {
+				betweenTier()
+			} catch (error) {
+				this.logThrow('between-tier', frozen.type, error)
+			}
+		}
+		this.runTier(this.catchAll, frozen, ctx)
+	}
+
+	veto<K extends VetoableEventKind>(
+		kind: K,
+		handler: VetoHandler<K>,
+		opts: ProbeOptions<K> = {},
+	): Unsubscribe {
+		if (opts.name !== undefined) {
+			const existing = this.byName.get(opts.name)
+			if (existing && !opts.override) {
+				throw new ProbeNameCollisionError(opts.name)
+			}
+			if (existing && opts.override) this.removeAnywhere(existing)
+		}
+
+		const entry: VetoEntry = {
+			id: this.nextId++,
+			name: opts.name,
+			priority: opts.priority ?? 0,
+			handler: handler as (e: ProbeEvent, c: ProbeContext) => VetoDecision,
+			where: opts.where as ((event: ProbeEvent) => boolean) | undefined,
+		}
+
+		let bucket = this.vetoByKind.get(kind)
+		if (!bucket) {
+			bucket = []
+			this.vetoByKind.set(kind, bucket)
+		}
+		insertSortedVeto(bucket, entry)
+		if (entry.name) this.byName.set(entry.name, entry)
+
+		return () => {
+			const list = this.vetoByKind.get(kind)
+			if (list) removeVeto(list, entry)
+			if (entry.name) this.byName.delete(entry.name)
+		}
+	}
+
+	queryVeto<K extends VetoableEventKind>(event: ProbeEventOf<K>, ctx: ProbeContext): VetoOutcome {
+		const wide = event as unknown as ProbeEvent
+		const frozen = Object.isFrozen(wide) ? wide : Object.freeze(wide)
+		const bucket = this.vetoByKind.get(frozen.type as VetoableEventKind)
+		if (!bucket || bucket.length === 0) return { action: 'allow' }
+
+		let firstDeny: VetoOutcome | undefined
+		for (const entry of bucket) {
+			if (entry.where && !entry.where(frozen)) continue
+			let decision: VetoDecision
+			try {
+				decision = entry.handler(frozen, ctx)
+			} catch (error) {
+				this.logThrow(entry.name ?? 'unnamed', frozen.type, error)
+				continue
+			}
+			const normalized = normalizeDecision(decision)
+			if (!normalized.allow && firstDeny === undefined) {
+				firstDeny = {
+					action: 'deny',
+					probeName: entry.name,
+					reason: normalized.reason,
+				}
+			}
+		}
+
+		return firstDeny ?? { action: 'allow' }
+	}
+
+	clear(): void {
+		this.typedByKind.clear()
+		this.vetoByKind.clear()
+		this.catchAll.length = 0
+		this.byName.clear()
+	}
+
+	private runTier(entries: readonly ProbeEntry[], event: ProbeEvent, ctx: ProbeContext): void {
+		for (const entry of entries) {
+			if (entry.where && !entry.where(event)) continue
+			try {
+				entry.handler(event, ctx)
+			} catch (error) {
+				this.logThrow(entry.name ?? 'unnamed', event.type, error)
+			}
+		}
+	}
+
+	private makeEntry(
+		handler: (event: ProbeEvent, ctx: ProbeContext) => void,
+		opts: ProbeOptions,
+	): ProbeEntry {
+		if (opts.name !== undefined) {
+			const existing = this.byName.get(opts.name)
+			if (existing && !opts.override) {
+				throw new ProbeNameCollisionError(opts.name)
+			}
+			if (existing && opts.override) this.removeAnywhere(existing)
+		}
+
+		const entry: ProbeEntry = {
+			id: this.nextId++,
+			name: opts.name,
+			priority: opts.priority ?? 0,
+			handler,
+			where: opts.where as ((event: ProbeEvent) => boolean) | undefined,
+		}
+
+		if (entry.name) this.byName.set(entry.name, entry)
+		return entry
+	}
+
+	private removeAnywhere(entry: ProbeEntry | VetoEntry): void {
+		for (const bucket of this.typedByKind.values()) removeEntry(bucket, entry as ProbeEntry)
+		removeEntry(this.catchAll, entry as ProbeEntry)
+		for (const bucket of this.vetoByKind.values()) removeVeto(bucket, entry as VetoEntry)
+		if (entry.name) this.byName.delete(entry.name)
+	}
+
+	private logThrow(probeName: string, eventType: string, error: unknown): void {
+		if (!this.log) return
+		this.log.error('probe handler threw', {
+			probeName,
+			eventType,
+			error: error instanceof Error ? error.message : String(error),
+		})
+	}
+}
+
+export const probe: ProbeRegistry = new ProbeRegistry()
+
+export function createProbeRegistry(): ProbeRegistry {
+	return new ProbeRegistry()
+}

package/src/provider/instrumentation.test.ts

@@ -0,0 +1,192 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { buildProbeContext } from '../probe/context.js'
+import { createProbeRegistry } from '../probe/registry.js'
+import type { AgentBusEvent } from '../types/bus/index.js'
+import type { ChatCompletionParams, ChatCompletionResponse } from '../types/provider/chat.js'
+import type { LLMProvider } from '../types/provider/interface.js'
+import type { StreamChunk } from '../types/provider/stream.js'
+
+import { wrapProviderWithProbes } from './instrumentation.js'
+
+function makeFakeProvider(
+	overrides: Partial<{
+		chat: LLMProvider['chat']
+		chatStream: LLMProvider['chatStream']
+	}> = {},
+): LLMProvider {
+	const defaultChat: LLMProvider['chat'] = async (
+		_params: ChatCompletionParams,
+	): Promise<ChatCompletionResponse> => {
+		return {
+			content: 'ok',
+			usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+		} as unknown as ChatCompletionResponse
+	}
+	const defaultStream: LLMProvider['chatStream'] = async function* (
+		_params: ChatCompletionParams,
+	): AsyncIterable<StreamChunk> {
+		yield { delta: 'hi' } as unknown as StreamChunk
+	}
+	return {
+		id: 'p1',
+		name: 'Provider 1',
+		chat: overrides.chat ?? defaultChat,
+		chatStream: overrides.chatStream ?? defaultStream,
+	}
+}
+
+const params: ChatCompletionParams = { model: 'm1', messages: [] } as ChatCompletionParams
+
+describe('wrapProviderWithProbes — chat', () => {
+	it('emits provider_call_start before the chat call and provider_call_completed after', async () => {
+		const reg = createProbeRegistry()
+		const seen: AgentBusEvent[] = []
+		reg.onAny((event) => seen.push(event as AgentBusEvent))
+
+		const wrapped = wrapProviderWithProbes(makeFakeProvider(), { probes: reg })
+		await wrapped.chat(params)
+
+		expect(seen.map((e) => e.type)).toEqual(['provider_call_start', 'provider_call_completed'])
+		const start = seen[0] as AgentBusEvent & { type: 'provider_call_start' }
+		const completed = seen[1] as AgentBusEvent & { type: 'provider_call_completed' }
+		expect(start.providerId).toBe('p1')
+		expect(start.model).toBe('m1')
+		expect(completed.callId).toBe(start.callId)
+		expect(completed.usage).toEqual({ inputTokens: 10, outputTokens: 5, totalTokens: 15 })
+		expect(completed.durationMs).toBeGreaterThanOrEqual(0)
+	})
+
+	it('emits provider_call_failed and re-throws when chat throws', async () => {
+		const reg = createProbeRegistry()
+		const seen: AgentBusEvent[] = []
+		reg.onAny((event) => seen.push(event as AgentBusEvent))
+
+		const failing = makeFakeProvider({
+			chat: async () => {
+				throw new Error('boom')
+			},
+		})
+		const wrapped = wrapProviderWithProbes(failing, { probes: reg })
+
+		await expect(wrapped.chat(params)).rejects.toThrow('boom')
+		expect(seen.map((e) => e.type)).toEqual(['provider_call_start', 'provider_call_failed'])
+		const failed = seen[1] as AgentBusEvent & { type: 'provider_call_failed' }
+		expect(failed.error).toBe('boom')
+	})
+
+	it('correlates start and completed by callId', async () => {
+		const reg = createProbeRegistry()
+		const ids: string[] = []
+		reg.on('provider_call_start', (event) => ids.push(`s:${event.callId}`))
+		reg.on('provider_call_completed', (event) => ids.push(`c:${event.callId}`))
+
+		const wrapped = wrapProviderWithProbes(makeFakeProvider(), { probes: reg })
+		await wrapped.chat(params)
+		await wrapped.chat(params)
+
+		expect(ids.length).toBe(4)
+		expect(ids[0]?.split(':')[1]).toBe(ids[1]?.split(':')[1])
+		expect(ids[2]?.split(':')[1]).toBe(ids[3]?.split(':')[1])
+		expect(ids[0]).not.toBe(ids[2])
+	})
+
+	it('forwards optional methods (listModels, healthCheck) to the inner provider', () => {
+		const listModels = vi.fn()
+		const healthCheck = vi.fn()
+		const inner = { ...makeFakeProvider(), listModels, healthCheck }
+		const wrapped = wrapProviderWithProbes(inner)
+
+		wrapped.listModels?.()
+		wrapped.healthCheck?.()
+		expect(listModels).toHaveBeenCalledTimes(1)
+		expect(healthCheck).toHaveBeenCalledTimes(1)
+	})
+})
+
+describe('wrapProviderWithProbes — chatStream', () => {
+	it('emits provider_call_start before iteration and provider_call_completed after', async () => {
+		const reg = createProbeRegistry()
+		const seen: AgentBusEvent[] = []
+		reg.onAny((event) => seen.push(event as AgentBusEvent))
+
+		const wrapped = wrapProviderWithProbes(makeFakeProvider(), { probes: reg })
+		const chunks: StreamChunk[] = []
+		for await (const chunk of wrapped.chatStream(params)) {
+			chunks.push(chunk)
+		}
+
+		expect(chunks.length).toBe(1)
+		expect(seen.map((e) => e.type)).toEqual(['provider_call_start', 'provider_call_completed'])
+	})
+
+	it('emits provider_call_failed when the underlying stream throws mid-iteration', async () => {
+		const reg = createProbeRegistry()
+		const seen: AgentBusEvent[] = []
+		reg.onAny((event) => seen.push(event as AgentBusEvent))
+
+		const failing = makeFakeProvider({
+			chatStream: async function* (_params: ChatCompletionParams): AsyncIterable<StreamChunk> {
+				yield { delta: 'a' } as unknown as StreamChunk
+				throw new Error('stream-boom')
+			},
+		})
+		const wrapped = wrapProviderWithProbes(failing, { probes: reg })
+
+		await expect(async () => {
+			for await (const _chunk of wrapped.chatStream(params)) {
+				// noop
+			}
+		}).rejects.toThrow('stream-boom')
+
+		expect(seen.map((e) => e.type)).toEqual(['provider_call_start', 'provider_call_failed'])
+	})
+})
+
+describe('wrapProviderWithProbes — runId propagation', () => {
+	it('attaches runId to each emitted event when supplied', async () => {
+		const reg = createProbeRegistry()
+		let observedRunId: string | undefined
+		reg.on('provider_call_start', (event, ctx) => {
+			observedRunId = event.runId ?? ctx.runId
+		})
+
+		const wrapped = wrapProviderWithProbes(makeFakeProvider(), {
+			probes: reg,
+			runId: 'run_42' as never,
+		})
+		await wrapped.chat(params)
+		expect(observedRunId).toBe('run_42')
+	})
+})
+
+describe('wrapProviderWithProbes — uses singleton when no probes opt provided', () => {
+	it('still wraps successfully without throwing (smoke)', async () => {
+		// Use a fresh inner provider; we just want to verify the default path
+		// instantiates and runs. Singleton dispatch is exercised in registry tests.
+		const wrapped = wrapProviderWithProbes(makeFakeProvider())
+		await expect(wrapped.chat(params)).resolves.toBeDefined()
+	})
+})
+
+describe('wrapProviderWithProbes — context still flows through buildProbeContext', () => {
+	it('handler receives a frozen ctx', async () => {
+		const reg = createProbeRegistry()
+		let captured: Readonly<{ isReplay: boolean }> | undefined
+		reg.on('provider_call_start', (_event, ctx) => {
+			captured = ctx
+		})
+
+		const wrapped = wrapProviderWithProbes(makeFakeProvider(), { probes: reg })
+		await wrapped.chat(params)
+		expect(captured).toBeDefined()
+		expect(Object.isFrozen(captured)).toBe(true)
+		expect(captured?.isReplay).toBe(false)
+	})
+
+	it('buildProbeContext used internally returns a frozen ProbeContext (sanity check)', () => {
+		const ctx = buildProbeContext({ isReplay: true })
+		expect(ctx.isReplay).toBe(true)
+		expect(Object.isFrozen(ctx)).toBe(true)
+	})
+})

package/src/provider/instrumentation.ts

@@ -0,0 +1,139 @@
+import { buildProbeContext } from '../probe/context.js'
+import { type ProbeRegistry, probe as defaultProbeRegistry } from '../probe/registry.js'
+import type { ProviderCallId, ProviderCallUsage } from '../types/bus/index.js'
+import type { RunId } from '../types/ids/index.js'
+import type { ChatCompletionParams, ChatCompletionResponse } from '../types/provider/chat.js'
+import type { LLMProvider } from '../types/provider/interface.js'
+import type { StreamChunk } from '../types/provider/stream.js'
+
+export interface ProviderInstrumentationOptions {
+	readonly probes?: ProbeRegistry
+	readonly runId?: RunId
+}
+
+let providerCallCounter = 0
+
+function nextCallId(): ProviderCallId {
+	providerCallCounter += 1
+	return `pcall_${Date.now().toString(36)}${providerCallCounter.toString(36)}` as ProviderCallId
+}
+
+function extractUsage(response: ChatCompletionResponse): ProviderCallUsage | undefined {
+	const usage = (response as { usage?: ProviderCallUsage }).usage
+	if (!usage) return undefined
+	return {
+		inputTokens: usage.inputTokens,
+		outputTokens: usage.outputTokens,
+		totalTokens: usage.totalTokens,
+		costUsd: usage.costUsd,
+	}
+}
+
+export function wrapProviderWithProbes(
+	provider: LLMProvider,
+	opts: ProviderInstrumentationOptions = {},
+): LLMProvider {
+	const probes = opts.probes ?? defaultProbeRegistry
+	const runId = opts.runId
+
+	const wrapped: LLMProvider = {
+		id: provider.id,
+		name: provider.name,
+		listModels: provider.listModels?.bind(provider),
+		healthCheck: provider.healthCheck?.bind(provider),
+
+		async chat(params: ChatCompletionParams): Promise<ChatCompletionResponse> {
+			const callId = nextCallId()
+			const ctx = buildProbeContext({ runId })
+			const startedAt = Date.now()
+			probes.dispatch(
+				{
+					type: 'provider_call_start',
+					providerId: provider.id,
+					model: params.model,
+					callId,
+					runId,
+				},
+				ctx,
+			)
+			try {
+				const response = await provider.chat(params)
+				probes.dispatch(
+					{
+						type: 'provider_call_completed',
+						providerId: provider.id,
+						model: params.model,
+						callId,
+						runId,
+						durationMs: Date.now() - startedAt,
+						usage: extractUsage(response),
+					},
+					ctx,
+				)
+				return response
+			} catch (error) {
+				probes.dispatch(
+					{
+						type: 'provider_call_failed',
+						providerId: provider.id,
+						model: params.model,
+						callId,
+						runId,
+						durationMs: Date.now() - startedAt,
+						error: error instanceof Error ? error.message : String(error),
+					},
+					ctx,
+				)
+				throw error
+			}
+		},
+
+		async *chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
+			const callId = nextCallId()
+			const ctx = buildProbeContext({ runId })
+			const startedAt = Date.now()
+			probes.dispatch(
+				{
+					type: 'provider_call_start',
+					providerId: provider.id,
+					model: params.model,
+					callId,
+					runId,
+				},
+				ctx,
+			)
+			try {
+				for await (const chunk of provider.chatStream(params)) {
+					yield chunk
+				}
+				probes.dispatch(
+					{
+						type: 'provider_call_completed',
+						providerId: provider.id,
+						model: params.model,
+						callId,
+						runId,
+						durationMs: Date.now() - startedAt,
+					},
+					ctx,
+				)
+			} catch (error) {
+				probes.dispatch(
+					{
+						type: 'provider_call_failed',
+						providerId: provider.id,
+						model: params.model,
+						callId,
+						runId,
+						durationMs: Date.now() - startedAt,
+						error: error instanceof Error ? error.message : String(error),
+					},
+					ctx,
+				)
+				throw error
+			}
+		},
+	}
+
+	return wrapped
+}

package/src/public-runtime.ts

@@ -222,6 +222,28 @@ export {
 
 export { evaluateRule, VerificationGate } from './verification/index.js'
 
+// ─── probe (typed observation over AgentBus + RunEvent stream) ───────────
+
+export {
+	buildProbeContext,
+	createProbeRegistry,
+	probe,
+	ProbeNameCollisionError,
+	ProbeRegistry,
+	ProbeVetoError,
+} from './probe/index.js'
+
+export { wrapProviderWithProbes } from './provider/instrumentation.js'
+export type { ProviderInstrumentationOptions } from './provider/instrumentation.js'
+
+export { wrapVaultWithProbes } from './vault/instrumentation.js'
+export type { VaultInstrumentationOptions } from './vault/instrumentation.js'
+
+// Doctor runtime moved to @namzu/cli in 0.5.0. SDK keeps only the
+// protocol types under `types/doctor/` (re-exported via public-types.ts)
+// + `LLMProvider.doctorCheck?()` hook on the provider interface.
+// Operators run `npx @namzu/cli doctor`; embedded usage lives there too.
+
 // ─── session runtime — explicit named lists, no `export *` ───────────────
 // See §1.5 + §4.2 of design.md. Types flow through public-types.ts.
 

package/src/public-types.ts

@@ -41,6 +41,9 @@ export type * from './types/structured-output/index.js'
 export type * from './types/invocation/index.js'
 export type * from './types/computer-use/index.js'
 export type * from './types/verification/index.js'
+export type * from './types/bus/index.js'
+export type * from './types/probe/index.js'
+export type * from './types/doctor/index.js'
 
 // Session-hierarchy type surface (ses_010 moved entities here).
 export type * from './types/session/index.js'

package/src/runtime/query/events.ts

@@ -1,5 +1,7 @@
 import type { PlanEvent, PlanManager } from '../../manager/plan/lifecycle.js'
 import type { RunPersistence } from '../../manager/run/persistence.js'
+import { buildProbeContext } from '../../probe/context.js'
+import { type ProbeRegistry, probe as defaultProbeRegistry } from '../../probe/registry.js'
 import type { ActivityEvent, ActivityStore } from '../../store/activity/memory.js'
 import type { RunId } from '../../types/ids/index.js'
 import type { RunEvent } from '../../types/run/index.js'
@@ -10,12 +12,15 @@ export type EmitEvent = (event: RunEvent) => Promise<void>
 export class EventTranslator {
 	private pendingEvents: RunEvent[] = []
 	private runMgr: RunPersistence
+	private probes: ProbeRegistry
 
-	constructor(runMgr: RunPersistence) {
+	constructor(runMgr: RunPersistence, probeRegistry: ProbeRegistry = defaultProbeRegistry) {
 		this.runMgr = runMgr
+		this.probes = probeRegistry
 	}
 
 	readonly emitEvent: EmitEvent = async (event: RunEvent): Promise<void> => {
+		this.probes.dispatch(event, buildProbeContext({ runId: event.runId }))
 		this.pendingEvents.push(event)
 		await this.runMgr.getRunStore().appendEvent(event)
 	};