nexus-ai-pro 2.0.0 → 2.2.0

package/dist/core/nexus.js

@@ -154,13 +154,26 @@ export class NexusAI {
             }
             await this.applyBudgets(context, ticket);
             const responseFormat = context.request.responseFormat || this.config.responseFormat;
-            const cacheKey = this.isCacheEnabled() ? createCacheKey({ request: context.request, responseFormat }) : undefined;
-            if (cacheKey) {
-                const cached = await this.pipeline.trace(context, 'cacheLookup', () => this.getCachedResponse(cacheKey, context.request));
+            const cachePlan = this.isCacheEnabled() ? this.cachePlan(context.request, responseFormat) : undefined;
+            let cacheOutcome;
+            if (cachePlan && !cachePlan.read) {
+                cacheOutcome = {
+                    outcome: 'bypass',
+                    reason: context.request.responseCache === 'refresh'
+                        ? 'the request asked for a fresh answer'
+                        : 'the request skipped the cache',
+                    ...(cachePlan.namespace ? { namespace: cachePlan.namespace } : {}),
+                };
+            }
+            if (cachePlan?.read) {
+                const plan = cachePlan;
+                const lookup = await this.pipeline.trace(context, 'cacheLookup', () => this.getCachedResponse(plan, context.request));
+                cacheOutcome = lookup.outcome;
+                const cached = lookup.response;
                 if (cached) {
                     let cachedResponse = this.attachTrace({
                         ...cached,
-                        meta: { ...cached.meta, requestId: ticket.descriptor.requestId, cacheHit: true },
+                        meta: { ...cached.meta, requestId: ticket.descriptor.requestId, cacheHit: true, cache: lookup.outcome },
                     }, this.pipeline.finish(context));
                     if (this.isSecurityEnabled()) {
                         const outputResult = this.security.protectOutput(cachedResponse);
@@ -222,12 +235,24 @@ export class NexusAI {
                 });
                 context.response = formattedResponse;
             }
-            if (cacheKey) {
+            if (cachePlan?.write) {
                 const responseToCache = context.response;
                 if (!responseToCache) {
                     throw new Error('Cannot cache an empty response.');
                 }
-                await this.pipeline.trace(context, 'cacheWrite', () => this.setCachedResponse(cacheKey, context.request, responseToCache));
+                const plan = cachePlan;
+                try {
+                    await this.pipeline.trace(context, 'cacheWrite', () => this.setCachedResponse(plan, context.request, responseToCache));
+                }
+                catch (error) {
+                    cacheOutcome = {
+                        ...(cacheOutcome ?? { outcome: 'miss', reason: 'no entry for this request' }),
+                        writeError: error instanceof Error ? error.message : String(error),
+                    };
+                }
+            }
+            if (cacheOutcome && context.response) {
+                context.response = { ...context.response, meta: { ...context.response.meta, cache: cacheOutcome } };
             }
             context = await this.pipeline.runHook('beforeReturn', context);
             let responseToReturn = context.response;
@@ -547,9 +572,11 @@ export class NexusAI {
     getProviderHealth() {
         return this.health.snapshot();
     }
-    async checkProviders() {
+    async checkProviders(options = {}) {
         const results = [];
         for (const [providerName, provider] of this.providers) {
+            if (options.staleOnly && !this.health.isUnknown(providerName))
+                continue;
             const started = Date.now();
             try {
                 const ok = await provider.healthCheck();
@@ -792,35 +819,86 @@ export class NexusAI {
         });
         return this.semanticCacheLoading;
     }
-    async getCachedResponse(cacheKey, request) {
-        if (!this.config.cache?.enabled)
-            return undefined;
-        if (this.usesSemanticCache()) {
-            const semantic = await (await this.loadSemanticCache()).get(request);
-            if (semantic)
-                return semantic;
+    cachePlan(request, responseFormat) {
+        const configured = this.config.cache?.namespace;
+        const chosen = typeof configured === 'function' ? configured(request) : configured;
+        const namespace = chosen ? namespaceKey({ ...chosen, tenant: chosen.tenant ?? request.tenantId }) : undefined;
+        const base = createCacheKey({ request: { ...request, responseCache: undefined }, responseFormat });
+        return {
+            key: namespace ? `${namespace}\u0000${base}` : base,
+            ...(namespace ? { namespace } : {}),
+            scope: namespace ?? namespaceKey({ tenant: request.tenantId }),
+            read: request.responseCache === undefined,
+            write: request.responseCache !== 'bypass',
+        };
+    }
+    async getCachedResponse(plan, request) {
+        const namespace = plan.namespace ? { namespace: plan.namespace } : {};
+        const cache = this.config.cache ?? {};
+        let layer = this.usesSemanticCache() ? 'semantic' : 'exact';
+        try {
+            if (this.usesSemanticCache()) {
+                const semantic = await (await this.loadSemanticCache()).get(request, plan.scope);
+                if (semantic) {
+                    return {
+                        response: semantic,
+                        outcome: { outcome: 'hit', layer: 'semantic', reason: 'a similar request was answered', ...namespace },
+                    };
+                }
+                if (cache.strategy === 'semantic') {
+                    return {
+                        outcome: {
+                            outcome: 'miss',
+                            layer: 'semantic',
+                            reason: 'no similar request above the threshold',
+                            ...namespace,
+                        },
+                    };
+                }
+            }
+            layer = 'exact';
+            if (cache.adapter) {
+                const value = (await cache.adapter.get(plan.key));
+                return value
+                    ? { response: value, outcome: { outcome: 'hit', layer, reason: 'this request was answered', ...namespace } }
+                    : { outcome: { outcome: 'miss', layer, reason: 'no entry for this request', ...namespace } };
+            }
+            const { value, stale } = this.cache.lookup(plan.key);
+            if (value) {
+                return {
+                    response: value,
+                    outcome: { outcome: 'hit', layer, reason: 'this request was answered', ...namespace },
+                };
+            }
+            return stale
+                ? { outcome: { outcome: 'stale', layer, reason: 'the entry for this request had expired', ...namespace } }
+                : { outcome: { outcome: 'miss', layer, reason: 'no entry for this request', ...namespace } };
         }
-        if (this.config.cache.strategy === 'semantic')
-            return undefined;
-        if (this.config.cache.adapter) {
-            return (await this.config.cache.adapter.get(cacheKey));
+        catch (error) {
+            return {
+                outcome: {
+                    outcome: 'error',
+                    layer,
+                    reason: error instanceof Error ? error.message : String(error),
+                    ...namespace,
+                },
+            };
         }
-        return this.cache.get(cacheKey);
     }
-    async setCachedResponse(cacheKey, request, response) {
+    async setCachedResponse(plan, request, response) {
         if (!this.config.cache?.enabled)
             return;
         const ttl = this.config.cache.ttlSeconds || 300;
         if (this.usesSemanticCache()) {
-            await (await this.loadSemanticCache()).set(cacheKey, request, response);
+            await (await this.loadSemanticCache()).set(plan.key, request, response, plan.scope);
         }
         if (this.config.cache.strategy === 'semantic')
             return;
         if (this.config.cache.adapter) {
-            await this.config.cache.adapter.set(cacheKey, response, ttl);
+            await this.config.cache.adapter.set(plan.key, response, ttl);
             return;
         }
-        this.cache.set(cacheKey, response, ttl);
+        this.cache.set(plan.key, response, ttl);
     }
     estimateRequestCost(request) {
         return estimateCost({
@@ -854,3 +932,8 @@ function streamStopped() {
     error.name = 'AbortError';
     return error;
 }
+function namespaceKey(namespace) {
+    return [namespace.tenant, namespace.environment, namespace.modelRevision, namespace.contextVersion]
+        .map((part) => encodeURIComponent(part ?? ''))
+        .join('/');
+}

package/dist/doctor/index.d.ts

@@ -0,0 +1,90 @@
+import type { SchemaMigration } from '../utils/schema-migrations.js';
+/** How one check came out: `fail` breaks the deployment, `warn` needs a look, `skip` had nothing to check. */
+export type DoctorStatus = 'ok' | 'warn' | 'fail' | 'skip';
+/** One check's result. */
+export interface DoctorCheck {
+    /** A stable id, such as `node` or `migrations`. */
+    id: string;
+    /** What was checked, in a few words. */
+    title: string;
+    /** How it came out. */
+    status: DoctorStatus;
+    /** What was found. Never a secret's value. */
+    detail: string;
+    /** What to do about a warning or a failure. */
+    hint?: string;
+}
+/** Every check's result, and whether the deployment can run. */
+export interface DoctorReport {
+    /** The checks, in the order they ran. */
+    checks: DoctorCheck[];
+    /** True when nothing failed. */
+    ok: boolean;
+    /** Checks that failed. */
+    failures: number;
+    /** Checks that warned. */
+    warnings: number;
+}
+/** A database for the doctor to reach, as `nexus db` reaches one. */
+export interface DoctorDatabase {
+    /** A Postgres client (`query`) or a SQLite handle or client. */
+    client: unknown;
+    /** Which kind it is. Detected from the client when omitted. */
+    dialect?: 'postgres' | 'sqlite';
+    /** The migrations it should have. Defaults to every bundled adapter's on its default tables. */
+    migrations?: readonly SchemaMigration[];
+    /** The table that records applied migrations. */
+    table?: string;
+}
+/** What the doctor knows about how the application is deployed. */
+export interface DoctorDeployment {
+    /** How many replicas run it. Settings that keep state in one process are checked only above 1. */
+    replicas: number;
+    /**
+     * The stores and shared state it uses, by any name: a checkpointer, the operation store, the
+     * server's state, tenant usage, a rate-limit store, a circuit store, a cache. Each is reported when
+     * it keeps its state in process memory, where each replica would see only its own.
+     */
+    stores?: Record<string, unknown>;
+}
+/** What `diagnose()` checks. Everything is optional; a check without its input is skipped. */
+export interface DoctorOptions {
+    /** The Node.js version to check. Defaults to the running one. */
+    nodeVersion?: string;
+    /** Where credentials are read from. Defaults to `process.env`. Only names are reported. */
+    env?: Record<string, string | undefined>;
+    /** Whether a package is installed. Defaults to resolving it from the working directory. */
+    resolvePackage?: (name: string) => boolean;
+    /** A database: connectivity, and migrations pending or changed. */
+    database?: DoctorDatabase;
+    /** A Redis client to ping. */
+    redis?: {
+        ping(): unknown;
+    };
+    /** An operation store, read for leases that lapsed and a queue nobody drains. */
+    operations?: {
+        stats?(now: string): unknown;
+        list?(): unknown;
+    };
+    /** How long the oldest queued operation may wait before the queue counts as stalled. Defaults to 15 minutes. */
+    maxQueueAgeMs?: number;
+    /** A client, read for provider health. */
+    ai?: {
+        getProviderHealth(): ReadonlyArray<{
+            providerName: string;
+            healthy: boolean;
+            stale?: boolean;
+        }>;
+    };
+    /** How old the bundled model registry may be, in days. Defaults to the registry's own window. */
+    registryMaxAgeDays?: number;
+    /** How the application is deployed. */
+    deployment?: DoctorDeployment;
+    /** Replaces the clock, for tests. */
+    now?: () => Date;
+}
+/**
+ * Runs every check its options allow and reports each, never throwing: a check that cannot run is a
+ * failure with the reason.
+ */
+export declare function diagnose(options?: DoctorOptions): Promise<DoctorReport>;

package/dist/doctor/index.js

@@ -0,0 +1,237 @@
+import { createRequire } from 'node:module';
+import path from 'node:path';
+const PEERS = {
+    openai: 'the OpenAI SDK, used only where an application passes one in',
+    '@anthropic-ai/sdk': 'the Anthropic SDK, likewise',
+    ollama: 'the Ollama client',
+    zod: 'zod schemas for tools and structured output',
+    ajv: 'JSON Schema validation of structured output',
+    'ajv-formats': 'formats for ajv',
+};
+const CREDENTIALS = {
+    openai: ['OPENAI_API_KEY'],
+    anthropic: ['ANTHROPIC_API_KEY'],
+    google: ['GOOGLE_API_KEY', 'GEMINI_API_KEY'],
+    'azure-openai': ['AZURE_OPENAI_API_KEY'],
+    groq: ['GROQ_API_KEY'],
+    mistral: ['MISTRAL_API_KEY'],
+    cohere: ['COHERE_API_KEY', 'CO_API_KEY'],
+    deepseek: ['DEEPSEEK_API_KEY'],
+    openrouter: ['OPENROUTER_API_KEY'],
+};
+const PROCESS_LOCAL = {
+    MemoryGraphCheckpointer: { status: 'fail', why: 'a thread started on one replica cannot be resumed on another' },
+    MemoryOperationStore: { status: 'fail', why: 'operations and server runs are invisible to other replicas' },
+    MemoryServerStore: { status: 'fail', why: 'threads and runs exist only on the replica that created them' },
+    MemoryRunEventLog: { status: 'fail', why: 'a client reconnecting to another replica cannot follow its run' },
+    MemoryTenantUsage: { status: 'fail', why: 'each replica counts its own tenant limits, multiplying them' },
+    MemoryRateLimitStore: { status: 'warn', why: 'each replica enforces its own limit, multiplying it' },
+    MemoryCircuitStateStore: { status: 'warn', why: 'a circuit opened on one replica stays closed on the others' },
+    MemoryStore: { status: 'warn', why: 'long-term memory written on one replica is missing on the others' },
+    MemoryTraceStore: { status: 'warn', why: 'each replica keeps only its own traces' },
+    MemoryPromptStore: { status: 'warn', why: 'a prompt promoted on one replica is not served by the others' },
+    MemoryCacheAdapter: { status: 'warn', why: 'each replica caches separately' },
+    MemoryRollupStore: { status: 'warn', why: 'each replica sums only its own runs' },
+};
+export async function diagnose(options = {}) {
+    const now = options.now ?? (() => new Date());
+    const checks = [];
+    const run = async (id, title, check) => {
+        try {
+            checks.push({ id, title, ...(await check()) });
+        }
+        catch (error) {
+            checks.push({ id, title, status: 'fail', detail: error instanceof Error ? error.message : String(error) });
+        }
+    };
+    await run('node', 'Node.js version', () => nodeCheck(options.nodeVersion ?? process.versions.node));
+    await run('peers', 'Optional packages', () => peersCheck(options.resolvePackage ?? defaultResolver()));
+    await run('credentials', 'Provider credentials', () => credentialsCheck(options.env ?? process.env));
+    await run('registry', 'Model registry age', () => registryCheck(now(), options.registryMaxAgeDays));
+    await run('database', 'Database connectivity', () => databaseCheck(options.database));
+    await run('migrations', 'Schema migrations', () => migrationsCheck(options.database));
+    await run('redis', 'Redis', () => redisCheck(options.redis));
+    await run('queue', 'Operation queue and leases', () => queueCheck(options.operations, now(), options.maxQueueAgeMs));
+    await run('health', 'Provider health', () => healthCheck(options.ai));
+    await run('topology', 'Settings that only work in one process', () => topologyCheck(options.deployment));
+    const failures = checks.filter((check) => check.status === 'fail').length;
+    const warnings = checks.filter((check) => check.status === 'warn').length;
+    return { checks, ok: failures === 0, failures, warnings };
+}
+function nodeCheck(version) {
+    const major = Number(version.replace(/^v/, '').split('.')[0]);
+    return major >= 22
+        ? { status: 'ok', detail: `Node.js ${version}` }
+        : { status: 'fail', detail: `Node.js ${version}; this package needs 22 or newer`, hint: 'Upgrade Node.js.' };
+}
+function defaultResolver() {
+    const resolve = createRequire(path.join(process.cwd(), 'package.json')).resolve;
+    return (name) => {
+        try {
+            resolve(name);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    };
+}
+function peersCheck(resolve) {
+    const installed = Object.keys(PEERS).filter((name) => resolve(name));
+    const missing = Object.keys(PEERS).filter((name) => !installed.includes(name));
+    return {
+        status: 'ok',
+        detail: `Installed: ${installed.length ? installed.join(', ') : 'none'}. Not installed: ${missing.length ? missing.join(', ') : 'none'}. Every one is optional; install only what you use.`,
+    };
+}
+function credentialsCheck(env) {
+    const found = Object.entries(CREDENTIALS)
+        .filter(([, names]) => names.some((name) => Boolean(env[name]?.trim())))
+        .map(([provider]) => provider);
+    return found.length
+        ? { status: 'ok', detail: `Credentials found for ${found.join(', ')}.` }
+        : {
+            status: 'warn',
+            detail: 'No provider credential found in the environment.',
+            hint: 'Set the API key of each hosted provider you use, such as OPENAI_API_KEY. Local providers need none.',
+        };
+}
+async function registryCheck(now, maxAgeDays) {
+    const { checkRegistryFreshness } = await import('../models/registry.js');
+    const freshness = checkRegistryFreshness({ now, ...(maxAgeDays === undefined ? {} : { maxAgeDays }) });
+    return freshness.stale
+        ? {
+            status: 'warn',
+            detail: `The bundled registry was verified ${freshness.ageDays} days ago (${freshness.verifiedAt}), past its ${freshness.maxAgeDays}-day window.`,
+            hint: 'Upgrade nexus-ai-pro, or override stale entries with models.registry.',
+        }
+        : { status: 'ok', detail: `Verified ${freshness.ageDays} days ago (${freshness.verifiedAt}).` };
+}
+function dialectOf(database) {
+    if (database.dialect)
+        return database.dialect;
+    const client = database.client;
+    if (client && typeof client.query === 'function')
+        return 'postgres';
+    return 'sqlite';
+}
+async function databaseCheck(database) {
+    if (!database)
+        return { status: 'skip', detail: 'No database given.' };
+    const dialect = dialectOf(database);
+    if (dialect === 'postgres') {
+        await database.client.query('SELECT 1');
+    }
+    else {
+        const { toSqliteClient } = await import('../sqlite/client.js');
+        await toSqliteClient(database.client).all('SELECT 1', []);
+    }
+    return { status: 'ok', detail: `Reached the ${dialect === 'postgres' ? 'Postgres' : 'SQLite'} database.` };
+}
+async function migrationsCheck(database) {
+    if (!database)
+        return { status: 'skip', detail: 'No database given.' };
+    const dialect = dialectOf(database);
+    const options = database.table ? { table: database.table } : {};
+    let status;
+    if (dialect === 'postgres') {
+        const [{ postgresMigrations }, { postgresMigrationStatus }] = await Promise.all([
+            import('../postgres/index.js'),
+            import('../postgres/migrations.js'),
+        ]);
+        status = await postgresMigrationStatus(database.client, database.migrations ?? postgresMigrations(), options);
+    }
+    else {
+        const [{ sqliteMigrations }, { sqliteMigrationStatus }] = await Promise.all([
+            import('../sqlite/index.js'),
+            import('../sqlite/migrations.js'),
+        ]);
+        status = await sqliteMigrationStatus(database.client, database.migrations ?? sqliteMigrations(), options);
+    }
+    if (status.changed.length) {
+        return {
+            status: 'fail',
+            detail: `${status.changed.length} migration${status.changed.length === 1 ? '' : 's'} changed after running: ${status.changed.map(({ migration }) => `${migration.component} v${migration.version}`).join(', ')}.`,
+            hint: 'A migration never changes once applied. Check the adapter options match those it ran with.',
+        };
+    }
+    if (status.pending.length) {
+        return {
+            status: 'fail',
+            detail: `${status.pending.length} pending: ${status.pending.map((migration) => `${migration.component} v${migration.version}`).join(', ')}.`,
+            hint: 'Run: nexus db migrate --client db.mjs',
+        };
+    }
+    const newer = status.unknown.length ? ` ${status.unknown.length} recorded by a newer release.` : '';
+    return { status: 'ok', detail: `${status.applied.length} applied, none pending.${newer}` };
+}
+async function redisCheck(redis) {
+    if (!redis)
+        return { status: 'skip', detail: 'No Redis client given.' };
+    const reply = await redis.ping();
+    return { status: 'ok', detail: `PING answered ${String(reply)}.` };
+}
+async function queueCheck(operations, now, maxQueueAgeMs = 15 * 60_000) {
+    if (!operations)
+        return { status: 'skip', detail: 'No operation store given.' };
+    const { operationStats } = await import('../operations/stats.js');
+    const stats = await operationStats(operations, { now });
+    const problems = [];
+    if (stats.lapsedLeases > 0) {
+        problems.push(`${stats.lapsedLeases} running operation${stats.lapsedLeases === 1 ? '' : 's'} hold a lease that lapsed`);
+    }
+    const age = stats.oldestQueuedAt ? now.getTime() - Date.parse(stats.oldestQueuedAt) : 0;
+    if (age > maxQueueAgeMs)
+        problems.push(`the oldest queued operation has waited ${Math.round(age / 60_000)} minutes`);
+    const counts = Object.entries(stats.byStatus)
+        .map(([status, count]) => `${count} ${status}`)
+        .join(', ');
+    return problems.length
+        ? {
+            status: 'warn',
+            detail: `${problems.join('; ')}. (${counts || 'nothing unfinished'})`,
+            hint: 'Run recovery — OperationRunner.recover(), or the agent server with recoverEveryMs — and a worker that claims the queue.',
+        }
+        : { status: 'ok', detail: counts ? `Unfinished: ${counts}. No lapsed lease.` : 'Nothing unfinished.' };
+}
+function healthCheck(ai) {
+    if (!ai)
+        return { status: 'skip', detail: 'No client given.' };
+    const snapshots = ai.getProviderHealth();
+    if (snapshots.length === 0)
+        return { status: 'skip', detail: 'Health tracking is off, or no provider has been called.' };
+    const stale = snapshots.filter((item) => item.stale).map((item) => item.providerName);
+    const unhealthy = snapshots.filter((item) => !item.healthy && !item.stale).map((item) => item.providerName);
+    const issues = [
+        ...(stale.length ? [`stale: ${stale.join(', ')}`] : []),
+        ...(unhealthy.length ? [`unhealthy: ${unhealthy.join(', ')}`] : []),
+    ];
+    return issues.length
+        ? {
+            status: 'warn',
+            detail: `${issues.join('; ')}.`,
+            hint: 'Refresh stale entries with ai.checkProviders({ staleOnly: true }).',
+        }
+        : { status: 'ok', detail: `${snapshots.length} provider${snapshots.length === 1 ? '' : 's'} healthy.` };
+}
+function topologyCheck(deployment) {
+    if (!deployment)
+        return { status: 'skip', detail: 'No deployment described.' };
+    if (deployment.replicas <= 1)
+        return { status: 'ok', detail: 'One replica: in-process state is shared by everything.' };
+    const findings = [];
+    for (const [name, value] of Object.entries(deployment.stores ?? {})) {
+        const kind = value?.constructor?.name;
+        const known = kind ? PROCESS_LOCAL[kind] : undefined;
+        if (known)
+            findings.push({ status: known.status, text: `${name} is a ${kind}: ${known.why}` });
+    }
+    if (findings.length === 0) {
+        return { status: 'ok', detail: `${deployment.replicas} replicas, and nothing keeps its state in one process.` };
+    }
+    return {
+        status: findings.some((finding) => finding.status === 'fail') ? 'fail' : 'warn',
+        detail: `${deployment.replicas} replicas: ${findings.map((finding) => finding.text).join('; ')}.`,
+        hint: 'Use the Redis, Postgres, or SQLite adapter for each, or run one replica.',
+    };
+}

package/dist/evals/judge.d.ts

@@ -1,3 +1,4 @@
+import type { EvaluationContext, Evaluator } from '../types/evaluate.js';
 import type { CompletionRequest } from '../types/messages.js';
 import type { NexusResponse } from '../types/response.js';
 import type { EvalCase, EvalJudge } from './runner.js';
@@ -75,6 +76,16 @@ export declare class LLMJudge<Response = NexusResponse> {
     constructor(options: LLMJudgeOptions<Response>);
     /** Judges one answer. */
     evaluate(input: LLMJudgeInput): Promise<LLMJudgeResult>;
+    /**
+     * The judge as an evaluator for `evaluate()`, scoring each output under `key` (`judge` by
+     * default). `input` turns an evaluation context into what the judge sees; by default the output is
+     * the answer, the example's expected output the reference, and its inputs the question. The
+     * experiment records the judge's model, temperature, rubric, and a hash of its instructions.
+     */
+    asEvaluator(options?: {
+        key?: string;
+        input?: (context: EvaluationContext) => LLMJudgeInput;
+    }): Evaluator;
     /**
      * Adapts the judge to an `EvalRunner` case, optionally with a mapper that picks what the judge
      * sees.

package/dist/evals/judge.js

@@ -1,3 +1,5 @@
+import { createHash } from 'node:crypto';
+import { withProvenance } from '../evaluate/provenance.js';
 const DEFAULT_SYSTEM_PROMPT = [
     'You are a careful evaluation judge.',
     'Return only JSON with these fields: score, passed, rationale, labels.',
@@ -43,6 +45,34 @@ export class LLMJudge {
             modelUsed: modelOf(response),
         });
     }
+    asEvaluator(options = {}) {
+        const evaluator = async (context) => {
+            const input = options.input?.(context) ?? {
+                actual: text(context.output),
+                ...(context.example.expected === undefined ? {} : { expected: text(context.example.expected) }),
+                query: text(context.example.inputs),
+            };
+            const verdict = await this.evaluate(input);
+            return {
+                key: options.key ?? 'judge',
+                score: verdict.score,
+                passed: verdict.passed,
+                ...(verdict.rationale ? { comment: verdict.rationale } : {}),
+            };
+        };
+        return withProvenance(evaluator, {
+            name: options.key ?? 'judge',
+            judge: {
+                model: this.options.model,
+                promptVersion: createHash('sha256')
+                    .update(this.options.systemPrompt ?? 'default')
+                    .digest('hex')
+                    .slice(0, 12),
+                temperature: this.options.temperature ?? 0,
+            },
+            ...(this.options.rubric ? { rubric: this.options.rubric } : {}),
+        });
+    }
     asEvalJudge(mapper) {
         return async (response, testCase) => {
             const input = mapper
@@ -150,3 +180,6 @@ function clamp(value, [min, max]) {
 function midpoint([min, max]) {
     return min + (max - min) / 2;
 }
+function text(value) {
+    return typeof value === 'string' ? value : JSON.stringify(value ?? '');
+}

package/dist/evaluate/index.d.ts

@@ -5,3 +5,4 @@ export { AnnotationQueue, type AnnotationQueueOptions, evaluateOnline, type Onli
 export { evaluate, type EvaluateOptions, type EvaluationTarget, stats, summarize } from './run.js';
 export { type CachedOutput, type EvaluationCache, evaluationCacheKey, FileEvaluationCache, fingerprintOf, MemoryEvaluationCache, type MemoryEvaluationCacheOptions, } from './cache.js';
 export type { Dataset, DatasetExample, DatasetStore, EvaluationContext, EvaluationScore, Evaluator, ExampleResult, Experiment, ExperimentStore, MetricSummary, SummaryEvaluator, } from '../types/evaluate.js';
+export { withProvenance } from './provenance.js';

package/dist/evaluate/index.js

@@ -4,3 +4,4 @@ export { completed, contains, embeddingSimilarity, exactMatch, mustNotMatch, pai
 export { AnnotationQueue, evaluateOnline, } from './review.js';
 export { evaluate, stats, summarize } from './run.js';
 export { evaluationCacheKey, FileEvaluationCache, fingerprintOf, MemoryEvaluationCache, } from './cache.js';
+export { withProvenance } from './provenance.js';

package/dist/evaluate/provenance.d.ts

@@ -0,0 +1,22 @@
+import type { Evaluator, EvaluatorProvenance } from '../types/evaluate.js';
+/**
+ * Says where an evaluator's scores come from: its name and version, the judge model and prompt
+ * version behind it, and its rubric. `evaluate()` records it on the experiment, so a score of 0.87
+ * can be traced to exactly what produced it, and two experiments scored by different judges are not
+ * compared as if they were alike.
+ *
+ * Returns the same evaluator, marked; it runs exactly as before.
+ *
+ * @example
+ * ```ts
+ * const grounded = withProvenance(groundedness, {
+ *   name: 'groundedness',
+ *   version: '3',
+ *   judge: { model: 'claude-sonnet-5-5', promptVersion: 'grounded-v3', temperature: 0 },
+ *   rubric: 'Every claim is supported by a cited passage.',
+ * });
+ * ```
+ */
+export declare function withProvenance<I = unknown, O = unknown>(evaluator: Evaluator<I, O>, provenance: Omit<EvaluatorProvenance, 'keys' | 'position'>): Evaluator<I, O>;
+/** What an evaluator declared through `withProvenance()`, if anything. */
+export declare function declaredProvenance(evaluator: unknown): Omit<EvaluatorProvenance, 'keys' | 'position'> | undefined;

package/dist/evaluate/provenance.js

@@ -0,0 +1,10 @@
+const PROVENANCE = Symbol('nexus.evaluate.provenance');
+export function withProvenance(evaluator, provenance) {
+    Object.defineProperty(evaluator, PROVENANCE, { value: provenance, configurable: true });
+    return evaluator;
+}
+export function declaredProvenance(evaluator) {
+    if (typeof evaluator !== 'function')
+        return undefined;
+    return evaluator[PROVENANCE];
+}