nexus-ai-pro 2.0.0 → 2.2.0

package/dist-cjs/doctor/index.d.ts

@@ -0,0 +1,90 @@
+import type { SchemaMigration } from '../utils/schema-migrations.js';
+/** How one check came out: `fail` breaks the deployment, `warn` needs a look, `skip` had nothing to check. */
+export type DoctorStatus = 'ok' | 'warn' | 'fail' | 'skip';
+/** One check's result. */
+export interface DoctorCheck {
+    /** A stable id, such as `node` or `migrations`. */
+    id: string;
+    /** What was checked, in a few words. */
+    title: string;
+    /** How it came out. */
+    status: DoctorStatus;
+    /** What was found. Never a secret's value. */
+    detail: string;
+    /** What to do about a warning or a failure. */
+    hint?: string;
+}
+/** Every check's result, and whether the deployment can run. */
+export interface DoctorReport {
+    /** The checks, in the order they ran. */
+    checks: DoctorCheck[];
+    /** True when nothing failed. */
+    ok: boolean;
+    /** Checks that failed. */
+    failures: number;
+    /** Checks that warned. */
+    warnings: number;
+}
+/** A database for the doctor to reach, as `nexus db` reaches one. */
+export interface DoctorDatabase {
+    /** A Postgres client (`query`) or a SQLite handle or client. */
+    client: unknown;
+    /** Which kind it is. Detected from the client when omitted. */
+    dialect?: 'postgres' | 'sqlite';
+    /** The migrations it should have. Defaults to every bundled adapter's on its default tables. */
+    migrations?: readonly SchemaMigration[];
+    /** The table that records applied migrations. */
+    table?: string;
+}
+/** What the doctor knows about how the application is deployed. */
+export interface DoctorDeployment {
+    /** How many replicas run it. Settings that keep state in one process are checked only above 1. */
+    replicas: number;
+    /**
+     * The stores and shared state it uses, by any name: a checkpointer, the operation store, the
+     * server's state, tenant usage, a rate-limit store, a circuit store, a cache. Each is reported when
+     * it keeps its state in process memory, where each replica would see only its own.
+     */
+    stores?: Record<string, unknown>;
+}
+/** What `diagnose()` checks. Everything is optional; a check without its input is skipped. */
+export interface DoctorOptions {
+    /** The Node.js version to check. Defaults to the running one. */
+    nodeVersion?: string;
+    /** Where credentials are read from. Defaults to `process.env`. Only names are reported. */
+    env?: Record<string, string | undefined>;
+    /** Whether a package is installed. Defaults to resolving it from the working directory. */
+    resolvePackage?: (name: string) => boolean;
+    /** A database: connectivity, and migrations pending or changed. */
+    database?: DoctorDatabase;
+    /** A Redis client to ping. */
+    redis?: {
+        ping(): unknown;
+    };
+    /** An operation store, read for leases that lapsed and a queue nobody drains. */
+    operations?: {
+        stats?(now: string): unknown;
+        list?(): unknown;
+    };
+    /** How long the oldest queued operation may wait before the queue counts as stalled. Defaults to 15 minutes. */
+    maxQueueAgeMs?: number;
+    /** A client, read for provider health. */
+    ai?: {
+        getProviderHealth(): ReadonlyArray<{
+            providerName: string;
+            healthy: boolean;
+            stale?: boolean;
+        }>;
+    };
+    /** How old the bundled model registry may be, in days. Defaults to the registry's own window. */
+    registryMaxAgeDays?: number;
+    /** How the application is deployed. */
+    deployment?: DoctorDeployment;
+    /** Replaces the clock, for tests. */
+    now?: () => Date;
+}
+/**
+ * Runs every check its options allow and reports each, never throwing: a check that cannot run is a
+ * failure with the reason.
+ */
+export declare function diagnose(options?: DoctorOptions): Promise<DoctorReport>;

package/dist-cjs/doctor/index.js

@@ -0,0 +1,276 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diagnose = diagnose;
+const node_module_1 = require("node:module");
+const node_path_1 = __importDefault(require("node:path"));
+const PEERS = {
+    openai: 'the OpenAI SDK, used only where an application passes one in',
+    '@anthropic-ai/sdk': 'the Anthropic SDK, likewise',
+    ollama: 'the Ollama client',
+    zod: 'zod schemas for tools and structured output',
+    ajv: 'JSON Schema validation of structured output',
+    'ajv-formats': 'formats for ajv',
+};
+const CREDENTIALS = {
+    openai: ['OPENAI_API_KEY'],
+    anthropic: ['ANTHROPIC_API_KEY'],
+    google: ['GOOGLE_API_KEY', 'GEMINI_API_KEY'],
+    'azure-openai': ['AZURE_OPENAI_API_KEY'],
+    groq: ['GROQ_API_KEY'],
+    mistral: ['MISTRAL_API_KEY'],
+    cohere: ['COHERE_API_KEY', 'CO_API_KEY'],
+    deepseek: ['DEEPSEEK_API_KEY'],
+    openrouter: ['OPENROUTER_API_KEY'],
+};
+const PROCESS_LOCAL = {
+    MemoryGraphCheckpointer: { status: 'fail', why: 'a thread started on one replica cannot be resumed on another' },
+    MemoryOperationStore: { status: 'fail', why: 'operations and server runs are invisible to other replicas' },
+    MemoryServerStore: { status: 'fail', why: 'threads and runs exist only on the replica that created them' },
+    MemoryRunEventLog: { status: 'fail', why: 'a client reconnecting to another replica cannot follow its run' },
+    MemoryTenantUsage: { status: 'fail', why: 'each replica counts its own tenant limits, multiplying them' },
+    MemoryRateLimitStore: { status: 'warn', why: 'each replica enforces its own limit, multiplying it' },
+    MemoryCircuitStateStore: { status: 'warn', why: 'a circuit opened on one replica stays closed on the others' },
+    MemoryStore: { status: 'warn', why: 'long-term memory written on one replica is missing on the others' },
+    MemoryTraceStore: { status: 'warn', why: 'each replica keeps only its own traces' },
+    MemoryPromptStore: { status: 'warn', why: 'a prompt promoted on one replica is not served by the others' },
+    MemoryCacheAdapter: { status: 'warn', why: 'each replica caches separately' },
+    MemoryRollupStore: { status: 'warn', why: 'each replica sums only its own runs' },
+};
+async function diagnose(options = {}) {
+    const now = options.now ?? (() => new Date());
+    const checks = [];
+    const run = async (id, title, check) => {
+        try {
+            checks.push({ id, title, ...(await check()) });
+        }
+        catch (error) {
+            checks.push({ id, title, status: 'fail', detail: error instanceof Error ? error.message : String(error) });
+        }
+    };
+    await run('node', 'Node.js version', () => nodeCheck(options.nodeVersion ?? process.versions.node));
+    await run('peers', 'Optional packages', () => peersCheck(options.resolvePackage ?? defaultResolver()));
+    await run('credentials', 'Provider credentials', () => credentialsCheck(options.env ?? process.env));
+    await run('registry', 'Model registry age', () => registryCheck(now(), options.registryMaxAgeDays));
+    await run('database', 'Database connectivity', () => databaseCheck(options.database));
+    await run('migrations', 'Schema migrations', () => migrationsCheck(options.database));
+    await run('redis', 'Redis', () => redisCheck(options.redis));
+    await run('queue', 'Operation queue and leases', () => queueCheck(options.operations, now(), options.maxQueueAgeMs));
+    await run('health', 'Provider health', () => healthCheck(options.ai));
+    await run('topology', 'Settings that only work in one process', () => topologyCheck(options.deployment));
+    const failures = checks.filter((check) => check.status === 'fail').length;
+    const warnings = checks.filter((check) => check.status === 'warn').length;
+    return { checks, ok: failures === 0, failures, warnings };
+}
+function nodeCheck(version) {
+    const major = Number(version.replace(/^v/, '').split('.')[0]);
+    return major >= 22
+        ? { status: 'ok', detail: `Node.js ${version}` }
+        : { status: 'fail', detail: `Node.js ${version}; this package needs 22 or newer`, hint: 'Upgrade Node.js.' };
+}
+function defaultResolver() {
+    const resolve = (0, node_module_1.createRequire)(node_path_1.default.join(process.cwd(), 'package.json')).resolve;
+    return (name) => {
+        try {
+            resolve(name);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    };
+}
+function peersCheck(resolve) {
+    const installed = Object.keys(PEERS).filter((name) => resolve(name));
+    const missing = Object.keys(PEERS).filter((name) => !installed.includes(name));
+    return {
+        status: 'ok',
+        detail: `Installed: ${installed.length ? installed.join(', ') : 'none'}. Not installed: ${missing.length ? missing.join(', ') : 'none'}. Every one is optional; install only what you use.`,
+    };
+}
+function credentialsCheck(env) {
+    const found = Object.entries(CREDENTIALS)
+        .filter(([, names]) => names.some((name) => Boolean(env[name]?.trim())))
+        .map(([provider]) => provider);
+    return found.length
+        ? { status: 'ok', detail: `Credentials found for ${found.join(', ')}.` }
+        : {
+            status: 'warn',
+            detail: 'No provider credential found in the environment.',
+            hint: 'Set the API key of each hosted provider you use, such as OPENAI_API_KEY. Local providers need none.',
+        };
+}
+async function registryCheck(now, maxAgeDays) {
+    const { checkRegistryFreshness } = await Promise.resolve().then(() => __importStar(require('../models/registry.js')));
+    const freshness = checkRegistryFreshness({ now, ...(maxAgeDays === undefined ? {} : { maxAgeDays }) });
+    return freshness.stale
+        ? {
+            status: 'warn',
+            detail: `The bundled registry was verified ${freshness.ageDays} days ago (${freshness.verifiedAt}), past its ${freshness.maxAgeDays}-day window.`,
+            hint: 'Upgrade nexus-ai-pro, or override stale entries with models.registry.',
+        }
+        : { status: 'ok', detail: `Verified ${freshness.ageDays} days ago (${freshness.verifiedAt}).` };
+}
+function dialectOf(database) {
+    if (database.dialect)
+        return database.dialect;
+    const client = database.client;
+    if (client && typeof client.query === 'function')
+        return 'postgres';
+    return 'sqlite';
+}
+async function databaseCheck(database) {
+    if (!database)
+        return { status: 'skip', detail: 'No database given.' };
+    const dialect = dialectOf(database);
+    if (dialect === 'postgres') {
+        await database.client.query('SELECT 1');
+    }
+    else {
+        const { toSqliteClient } = await Promise.resolve().then(() => __importStar(require('../sqlite/client.js')));
+        await toSqliteClient(database.client).all('SELECT 1', []);
+    }
+    return { status: 'ok', detail: `Reached the ${dialect === 'postgres' ? 'Postgres' : 'SQLite'} database.` };
+}
+async function migrationsCheck(database) {
+    if (!database)
+        return { status: 'skip', detail: 'No database given.' };
+    const dialect = dialectOf(database);
+    const options = database.table ? { table: database.table } : {};
+    let status;
+    if (dialect === 'postgres') {
+        const [{ postgresMigrations }, { postgresMigrationStatus }] = await Promise.all([
+            Promise.resolve().then(() => __importStar(require('../postgres/index.js'))),
+            Promise.resolve().then(() => __importStar(require('../postgres/migrations.js'))),
+        ]);
+        status = await postgresMigrationStatus(database.client, database.migrations ?? postgresMigrations(), options);
+    }
+    else {
+        const [{ sqliteMigrations }, { sqliteMigrationStatus }] = await Promise.all([
+            Promise.resolve().then(() => __importStar(require('../sqlite/index.js'))),
+            Promise.resolve().then(() => __importStar(require('../sqlite/migrations.js'))),
+        ]);
+        status = await sqliteMigrationStatus(database.client, database.migrations ?? sqliteMigrations(), options);
+    }
+    if (status.changed.length) {
+        return {
+            status: 'fail',
+            detail: `${status.changed.length} migration${status.changed.length === 1 ? '' : 's'} changed after running: ${status.changed.map(({ migration }) => `${migration.component} v${migration.version}`).join(', ')}.`,
+            hint: 'A migration never changes once applied. Check the adapter options match those it ran with.',
+        };
+    }
+    if (status.pending.length) {
+        return {
+            status: 'fail',
+            detail: `${status.pending.length} pending: ${status.pending.map((migration) => `${migration.component} v${migration.version}`).join(', ')}.`,
+            hint: 'Run: nexus db migrate --client db.mjs',
+        };
+    }
+    const newer = status.unknown.length ? ` ${status.unknown.length} recorded by a newer release.` : '';
+    return { status: 'ok', detail: `${status.applied.length} applied, none pending.${newer}` };
+}
+async function redisCheck(redis) {
+    if (!redis)
+        return { status: 'skip', detail: 'No Redis client given.' };
+    const reply = await redis.ping();
+    return { status: 'ok', detail: `PING answered ${String(reply)}.` };
+}
+async function queueCheck(operations, now, maxQueueAgeMs = 15 * 60_000) {
+    if (!operations)
+        return { status: 'skip', detail: 'No operation store given.' };
+    const { operationStats } = await Promise.resolve().then(() => __importStar(require('../operations/stats.js')));
+    const stats = await operationStats(operations, { now });
+    const problems = [];
+    if (stats.lapsedLeases > 0) {
+        problems.push(`${stats.lapsedLeases} running operation${stats.lapsedLeases === 1 ? '' : 's'} hold a lease that lapsed`);
+    }
+    const age = stats.oldestQueuedAt ? now.getTime() - Date.parse(stats.oldestQueuedAt) : 0;
+    if (age > maxQueueAgeMs)
+        problems.push(`the oldest queued operation has waited ${Math.round(age / 60_000)} minutes`);
+    const counts = Object.entries(stats.byStatus)
+        .map(([status, count]) => `${count} ${status}`)
+        .join(', ');
+    return problems.length
+        ? {
+            status: 'warn',
+            detail: `${problems.join('; ')}. (${counts || 'nothing unfinished'})`,
+            hint: 'Run recovery — OperationRunner.recover(), or the agent server with recoverEveryMs — and a worker that claims the queue.',
+        }
+        : { status: 'ok', detail: counts ? `Unfinished: ${counts}. No lapsed lease.` : 'Nothing unfinished.' };
+}
+function healthCheck(ai) {
+    if (!ai)
+        return { status: 'skip', detail: 'No client given.' };
+    const snapshots = ai.getProviderHealth();
+    if (snapshots.length === 0)
+        return { status: 'skip', detail: 'Health tracking is off, or no provider has been called.' };
+    const stale = snapshots.filter((item) => item.stale).map((item) => item.providerName);
+    const unhealthy = snapshots.filter((item) => !item.healthy && !item.stale).map((item) => item.providerName);
+    const issues = [
+        ...(stale.length ? [`stale: ${stale.join(', ')}`] : []),
+        ...(unhealthy.length ? [`unhealthy: ${unhealthy.join(', ')}`] : []),
+    ];
+    return issues.length
+        ? {
+            status: 'warn',
+            detail: `${issues.join('; ')}.`,
+            hint: 'Refresh stale entries with ai.checkProviders({ staleOnly: true }).',
+        }
+        : { status: 'ok', detail: `${snapshots.length} provider${snapshots.length === 1 ? '' : 's'} healthy.` };
+}
+function topologyCheck(deployment) {
+    if (!deployment)
+        return { status: 'skip', detail: 'No deployment described.' };
+    if (deployment.replicas <= 1)
+        return { status: 'ok', detail: 'One replica: in-process state is shared by everything.' };
+    const findings = [];
+    for (const [name, value] of Object.entries(deployment.stores ?? {})) {
+        const kind = value?.constructor?.name;
+        const known = kind ? PROCESS_LOCAL[kind] : undefined;
+        if (known)
+            findings.push({ status: known.status, text: `${name} is a ${kind}: ${known.why}` });
+    }
+    if (findings.length === 0) {
+        return { status: 'ok', detail: `${deployment.replicas} replicas, and nothing keeps its state in one process.` };
+    }
+    return {
+        status: findings.some((finding) => finding.status === 'fail') ? 'fail' : 'warn',
+        detail: `${deployment.replicas} replicas: ${findings.map((finding) => finding.text).join('; ')}.`,
+        hint: 'Use the Redis, Postgres, or SQLite adapter for each, or run one replica.',
+    };
+}

package/dist-cjs/evals/judge.d.ts

@@ -1,3 +1,4 @@
+import type { EvaluationContext, Evaluator } from '../types/evaluate.js';
 import type { CompletionRequest } from '../types/messages.js';
 import type { NexusResponse } from '../types/response.js';
 import type { EvalCase, EvalJudge } from './runner.js';
@@ -75,6 +76,16 @@ export declare class LLMJudge<Response = NexusResponse> {
     constructor(options: LLMJudgeOptions<Response>);
     /** Judges one answer. */
     evaluate(input: LLMJudgeInput): Promise<LLMJudgeResult>;
+    /**
+     * The judge as an evaluator for `evaluate()`, scoring each output under `key` (`judge` by
+     * default). `input` turns an evaluation context into what the judge sees; by default the output is
+     * the answer, the example's expected output the reference, and its inputs the question. The
+     * experiment records the judge's model, temperature, rubric, and a hash of its instructions.
+     */
+    asEvaluator(options?: {
+        key?: string;
+        input?: (context: EvaluationContext) => LLMJudgeInput;
+    }): Evaluator;
     /**
      * Adapts the judge to an `EvalRunner` case, optionally with a mapper that picks what the judge
      * sees.

package/dist-cjs/evals/judge.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LLMJudge = void 0;
 exports.createLLMJudgeEval = createLLMJudgeEval;
 exports.parseJudgeResponse = parseJudgeResponse;
+const node_crypto_1 = require("node:crypto");
+const provenance_js_1 = require("../evaluate/provenance.js");
 const DEFAULT_SYSTEM_PROMPT = [
     'You are a careful evaluation judge.',
     'Return only JSON with these fields: score, passed, rationale, labels.',
@@ -48,6 +50,34 @@ class LLMJudge {
             modelUsed: modelOf(response),
         });
     }
+    asEvaluator(options = {}) {
+        const evaluator = async (context) => {
+            const input = options.input?.(context) ?? {
+                actual: text(context.output),
+                ...(context.example.expected === undefined ? {} : { expected: text(context.example.expected) }),
+                query: text(context.example.inputs),
+            };
+            const verdict = await this.evaluate(input);
+            return {
+                key: options.key ?? 'judge',
+                score: verdict.score,
+                passed: verdict.passed,
+                ...(verdict.rationale ? { comment: verdict.rationale } : {}),
+            };
+        };
+        return (0, provenance_js_1.withProvenance)(evaluator, {
+            name: options.key ?? 'judge',
+            judge: {
+                model: this.options.model,
+                promptVersion: (0, node_crypto_1.createHash)('sha256')
+                    .update(this.options.systemPrompt ?? 'default')
+                    .digest('hex')
+                    .slice(0, 12),
+                temperature: this.options.temperature ?? 0,
+            },
+            ...(this.options.rubric ? { rubric: this.options.rubric } : {}),
+        });
+    }
     asEvalJudge(mapper) {
         return async (response, testCase) => {
             const input = mapper
@@ -156,3 +186,6 @@ function clamp(value, [min, max]) {
 function midpoint([min, max]) {
     return min + (max - min) / 2;
 }
+function text(value) {
+    return typeof value === 'string' ? value : JSON.stringify(value ?? '');
+}

package/dist-cjs/evaluate/index.d.ts

@@ -5,3 +5,4 @@ export { AnnotationQueue, type AnnotationQueueOptions, evaluateOnline, type Onli
 export { evaluate, type EvaluateOptions, type EvaluationTarget, stats, summarize } from './run.js';
 export { type CachedOutput, type EvaluationCache, evaluationCacheKey, FileEvaluationCache, fingerprintOf, MemoryEvaluationCache, type MemoryEvaluationCacheOptions, } from './cache.js';
 export type { Dataset, DatasetExample, DatasetStore, EvaluationContext, EvaluationScore, Evaluator, ExampleResult, Experiment, ExperimentStore, MetricSummary, SummaryEvaluator, } from '../types/evaluate.js';
+export { withProvenance } from './provenance.js';

package/dist-cjs/evaluate/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MemoryEvaluationCache = exports.fingerprintOf = exports.FileEvaluationCache = exports.evaluationCacheKey = exports.summarize = exports.stats = exports.evaluate = exports.evaluateOnline = exports.AnnotationQueue = exports.underLatency = exports.underCost = exports.trajectory = exports.totalCost = exports.reciprocalRank = exports.recallAtK = exports.passRate = exports.pairwise = exports.mustNotMatch = exports.exactMatch = exports.embeddingSimilarity = exports.contains = exports.completed = exports.splitOf = exports.readExperiment = exports.MemoryExperimentStore = exports.MemoryDatasetStore = exports.FileExperimentStore = exports.FileDatasetStore = exports.datasetFromTraces = exports.createDataset = exports.contentVersion = exports.scoreMap = exports.formatComparison = exports.compareExperiments = void 0;
+exports.withProvenance = exports.MemoryEvaluationCache = exports.fingerprintOf = exports.FileEvaluationCache = exports.evaluationCacheKey = exports.summarize = exports.stats = exports.evaluate = exports.evaluateOnline = exports.AnnotationQueue = exports.underLatency = exports.underCost = exports.trajectory = exports.totalCost = exports.reciprocalRank = exports.recallAtK = exports.passRate = exports.pairwise = exports.mustNotMatch = exports.exactMatch = exports.embeddingSimilarity = exports.contains = exports.completed = exports.splitOf = exports.readExperiment = exports.MemoryExperimentStore = exports.MemoryDatasetStore = exports.FileExperimentStore = exports.FileDatasetStore = exports.datasetFromTraces = exports.createDataset = exports.contentVersion = exports.scoreMap = exports.formatComparison = exports.compareExperiments = void 0;
 var compare_js_1 = require("./compare.js");
 Object.defineProperty(exports, "compareExperiments", { enumerable: true, get: function () { return compare_js_1.compareExperiments; } });
 Object.defineProperty(exports, "formatComparison", { enumerable: true, get: function () { return compare_js_1.formatComparison; } });
@@ -41,3 +41,5 @@ Object.defineProperty(exports, "evaluationCacheKey", { enumerable: true, get: fu
 Object.defineProperty(exports, "FileEvaluationCache", { enumerable: true, get: function () { return cache_js_1.FileEvaluationCache; } });
 Object.defineProperty(exports, "fingerprintOf", { enumerable: true, get: function () { return cache_js_1.fingerprintOf; } });
 Object.defineProperty(exports, "MemoryEvaluationCache", { enumerable: true, get: function () { return cache_js_1.MemoryEvaluationCache; } });
+var provenance_js_1 = require("./provenance.js");
+Object.defineProperty(exports, "withProvenance", { enumerable: true, get: function () { return provenance_js_1.withProvenance; } });

package/dist-cjs/evaluate/provenance.d.ts

@@ -0,0 +1,22 @@
+import type { Evaluator, EvaluatorProvenance } from '../types/evaluate.js';
+/**
+ * Says where an evaluator's scores come from: its name and version, the judge model and prompt
+ * version behind it, and its rubric. `evaluate()` records it on the experiment, so a score of 0.87
+ * can be traced to exactly what produced it, and two experiments scored by different judges are not
+ * compared as if they were alike.
+ *
+ * Returns the same evaluator, marked; it runs exactly as before.
+ *
+ * @example
+ * ```ts
+ * const grounded = withProvenance(groundedness, {
+ *   name: 'groundedness',
+ *   version: '3',
+ *   judge: { model: 'claude-sonnet-5-5', promptVersion: 'grounded-v3', temperature: 0 },
+ *   rubric: 'Every claim is supported by a cited passage.',
+ * });
+ * ```
+ */
+export declare function withProvenance<I = unknown, O = unknown>(evaluator: Evaluator<I, O>, provenance: Omit<EvaluatorProvenance, 'keys' | 'position'>): Evaluator<I, O>;
+/** What an evaluator declared through `withProvenance()`, if anything. */
+export declare function declaredProvenance(evaluator: unknown): Omit<EvaluatorProvenance, 'keys' | 'position'> | undefined;

package/dist-cjs/evaluate/provenance.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withProvenance = withProvenance;
+exports.declaredProvenance = declaredProvenance;
+const PROVENANCE = Symbol('nexus.evaluate.provenance');
+function withProvenance(evaluator, provenance) {
+    Object.defineProperty(evaluator, PROVENANCE, { value: provenance, configurable: true });
+    return evaluator;
+}
+function declaredProvenance(evaluator) {
+    if (typeof evaluator !== 'function')
+        return undefined;
+    return evaluator[PROVENANCE];
+}

package/dist-cjs/evaluate/run.js

@@ -4,7 +4,9 @@ exports.evaluate = evaluate;
 exports.summarize = summarize;
 exports.stats = stats;
 const node_crypto_1 = require("node:crypto");
+const version_js_1 = require("../version.js");
 const cache_js_1 = require("./cache.js");
+const provenance_js_1 = require("./provenance.js");
 async function evaluate(target, dataset, evaluators, options = {}) {
     if (dataset.examples.length === 0)
         throw new RangeError(`Dataset "${dataset.name}" has no examples`);
@@ -19,8 +21,9 @@ async function evaluate(target, dataset, evaluators, options = {}) {
         example,
         run,
     })));
+    const produced = evaluators.map(() => new Set());
     await runConcurrently(jobs, options.concurrency ?? 4, async ({ example, run }) => {
-        const result = await evaluateOne(target, example, run, evaluators, options);
+        const result = await evaluateOne(target, example, run, evaluators, options, produced);
         results.push(result);
         options.onResult?.(result);
     }, options.signal);
@@ -40,6 +43,16 @@ async function evaluate(target, dataset, evaluators, options = {}) {
         results,
         metrics: summarize(results),
         summary,
+        evaluators: evaluators.map((evaluator, position) => {
+            const declared = (0, provenance_js_1.declaredProvenance)(evaluator);
+            return {
+                ...declared,
+                position,
+                name: declared?.name ?? (evaluator.name || `evaluator-${position}`),
+                keys: [...(produced[position] ?? [])],
+            };
+        }),
+        framework: { name: 'nexus-ai-pro', version: version_js_1.NEXUS_VERSION },
         ...(options.metadata ? { metadata: options.metadata } : {}),
         ...(options.cache
             ? {
@@ -62,7 +75,7 @@ function repetitionsOf(example, setting) {
     }
     return count;
 }
-async function evaluateOne(target, example, run, evaluators, options) {
+async function evaluateOne(target, example, run, evaluators, options, produced) {
     const started = Date.now();
     let output;
     let error;
@@ -102,9 +115,12 @@ async function evaluateOne(target, example, run, evaluators, options) {
         });
     }
     const scores = [];
-    for (const evaluator of evaluators) {
+    for (const [position, evaluator] of evaluators.entries()) {
         try {
-            scores.push(...normalizeScores(await evaluator(context), evaluator));
+            const made = normalizeScores(await evaluator(context), evaluator);
+            for (const score of made)
+                produced[position]?.add(score.key);
+            scores.push(...made);
         }
         catch (caught) {
             scores.push({

package/dist-cjs/graph/durability.d.ts

@@ -0,0 +1,27 @@
+import type { DurabilityMode } from '../types/graph.js';
+/**
+ * Writes one run's checkpoints under a durability mode.
+ *
+ * `sync` writes each one before returning. `async` chains writes behind each other, so they land in
+ * order, and makes the run wait only when `maxPending` are already in flight. `exit` keeps only the
+ * latest and writes it when the run stops. A write that fails is kept and thrown from the next call,
+ * so a background failure still fails the run.
+ */
+export declare class CheckpointWriter<T> {
+    private readonly write;
+    private readonly mode;
+    private readonly maxPending;
+    private chain;
+    private readonly inFlight;
+    private failure;
+    private unwritten;
+    constructor(write: (checkpoint: T) => Promise<void>, mode: DurabilityMode, maxPending?: number);
+    /**
+     * Hands over a checkpoint. A `durable` one is persisted, with everything before it, before this
+     * resolves, whatever the mode: a pause, a failure, the end of a run, or a recovery decision.
+     */
+    persist(checkpoint: T, durable?: boolean): Promise<void>;
+    /** Waits for every write handed over so far, and writes what `exit` mode was holding. */
+    flush(): Promise<void>;
+    private throwFailure;
+}

package/dist-cjs/graph/durability.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CheckpointWriter = void 0;
+const DEFAULT_MAX_PENDING = 8;
+class CheckpointWriter {
+    write;
+    mode;
+    maxPending;
+    chain = Promise.resolve();
+    inFlight = [];
+    failure;
+    unwritten;
+    constructor(write, mode, maxPending = DEFAULT_MAX_PENDING) {
+        this.write = write;
+        this.mode = mode;
+        this.maxPending = maxPending;
+    }
+    async persist(checkpoint, durable = false) {
+        this.throwFailure();
+        if (this.mode === 'sync') {
+            await this.write(checkpoint);
+            return;
+        }
+        if (this.mode === 'exit' && !durable) {
+            this.unwritten = checkpoint;
+            return;
+        }
+        if (this.mode === 'exit') {
+            this.unwritten = undefined;
+            await this.write(checkpoint);
+            return;
+        }
+        const link = this.chain
+            .then(() => this.write(checkpoint))
+            .catch((error) => {
+            this.failure ??= { error };
+        })
+            .finally(() => {
+            const index = this.inFlight.indexOf(link);
+            if (index >= 0)
+                this.inFlight.splice(index, 1);
+        });
+        this.chain = link;
+        this.inFlight.push(link);
+        if (durable) {
+            await this.flush();
+            return;
+        }
+        while (this.inFlight.length > Math.max(1, this.maxPending))
+            await this.inFlight[0];
+        this.throwFailure();
+    }
+    async flush() {
+        await this.chain;
+        if (this.unwritten !== undefined) {
+            const latest = this.unwritten;
+            this.unwritten = undefined;
+            await this.write(latest);
+        }
+        this.throwFailure();
+    }
+    throwFailure() {
+        if (!this.failure)
+            return;
+        const { error } = this.failure;
+        this.failure = undefined;
+        throw error;
+    }
+}
+exports.CheckpointWriter = CheckpointWriter;