nexus-ai-pro 2.2.0 → 2.4.0

package/dist-cjs/server/auth.js

@@ -0,0 +1,402 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JwtError = void 0;
+exports.createJwtVerifier = createJwtVerifier;
+exports.jwtAuth = jwtAuth;
+exports.hashApiKey = hashApiKey;
+exports.apiKeyAuth = apiKeyAuth;
+exports.trustedProxyAuth = trustedProxyAuth;
+exports.anyAuth = anyAuth;
+const tenant_js_1 = require("../utils/tenant.js");
+class JwtError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = 'JwtError';
+    }
+}
+exports.JwtError = JwtError;
+const HMAC = ['HS256', 'HS384', 'HS512'];
+const ASYMMETRIC = [
+    'RS256',
+    'RS384',
+    'RS512',
+    'PS256',
+    'PS384',
+    'PS512',
+    'ES256',
+    'ES384',
+    'ES512',
+    'EdDSA',
+];
+function cryptoParams(alg) {
+    const bits = alg.slice(2);
+    const hash = `SHA-${bits === 'DSA' ? '512' : bits}`;
+    switch (alg.slice(0, 2)) {
+        case 'HS':
+            return { importAs: { name: 'HMAC', hash }, verifyAs: { name: 'HMAC' } };
+        case 'RS':
+            return { importAs: { name: 'RSASSA-PKCS1-v1_5', hash }, verifyAs: { name: 'RSASSA-PKCS1-v1_5' } };
+        case 'PS':
+            return { importAs: { name: 'RSA-PSS', hash }, verifyAs: { name: 'RSA-PSS', saltLength: Number(bits) / 8 } };
+        case 'ES': {
+            const namedCurve = bits === '512' ? 'P-521' : `P-${bits}`;
+            return { importAs: { name: 'ECDSA', namedCurve }, verifyAs: { name: 'ECDSA', hash } };
+        }
+        default:
+            return { importAs: { name: 'Ed25519' }, verifyAs: { name: 'Ed25519' } };
+    }
+}
+function jwkFits(jwk, alg) {
+    if (jwk.alg && jwk.alg !== alg && !(alg === 'EdDSA' && jwk.alg === 'Ed25519'))
+        return false;
+    if (jwk.use && jwk.use !== 'sig')
+        return false;
+    if (jwk.key_ops && !jwk.key_ops.includes('verify'))
+        return false;
+    if (alg.startsWith('RS') || alg.startsWith('PS'))
+        return jwk.kty === 'RSA';
+    if (alg.startsWith('ES')) {
+        const curve = alg === 'ES512' ? 'P-521' : `P-${alg.slice(2)}`;
+        return jwk.kty === 'EC' && jwk.crv === curve;
+    }
+    if (alg === 'EdDSA')
+        return jwk.kty === 'OKP' && jwk.crv === 'Ed25519';
+    return false;
+}
+function createJwtVerifier(options) {
+    const sources = [options.secret, options.publicKey, options.jwks].filter((source) => source !== undefined);
+    if (sources.length !== 1) {
+        throw new TypeError('createJwtVerifier() needs exactly one of secret, publicKey, or jwks');
+    }
+    if (options.jwks !== undefined && options.audience === undefined) {
+        throw new TypeError('A key set signs tokens for every application its provider serves: set audience, or audience: false to accept any');
+    }
+    const allowed = new Set((options.algorithms ?? (options.secret !== undefined ? HMAC : ASYMMETRIC)).filter((alg) => options.secret !== undefined ? HMAC.includes(alg) : ASYMMETRIC.includes(alg)));
+    if (allowed.size === 0)
+        throw new TypeError('None of the algorithms given can be used with this kind of key');
+    const now = options.now ?? Date.now;
+    const tolerance = options.clockToleranceSec ?? 60;
+    const subtle = globalThis.crypto.subtle;
+    const imported = new Map();
+    const importOnce = (cacheKey, load) => {
+        let key = imported.get(cacheKey);
+        if (!key) {
+            key = load();
+            key.catch(() => imported.delete(cacheKey));
+            imported.set(cacheKey, key);
+        }
+        return key;
+    };
+    let keySet = options.jwks !== undefined && typeof options.jwks === 'object' && !(options.jwks instanceof URL)
+        ? { keys: options.jwks.keys, fetchedAt: Number.POSITIVE_INFINITY }
+        : undefined;
+    let lastUnknownFetch = Number.NEGATIVE_INFINITY;
+    let inFlight;
+    const refresh = () => {
+        if (inFlight)
+            return inFlight;
+        const fetcher = options.fetch ?? globalThis.fetch;
+        inFlight = (async () => {
+            const response = await fetcher(String(options.jwks), { headers: { accept: 'application/json' } });
+            if (!response.ok)
+                throw new JwtError('JWT_KEY', `The key set answered ${response.status}`);
+            const body = (await response.json());
+            if (!Array.isArray(body.keys))
+                throw new JwtError('JWT_KEY', 'The key set has no keys');
+            keySet = { keys: body.keys, fetchedAt: now() };
+            imported.clear();
+        })().finally(() => {
+            inFlight = undefined;
+        });
+        return inFlight;
+    };
+    const keyFor = async (header, alg) => {
+        const { importAs } = cryptoParams(alg);
+        if (options.secret !== undefined) {
+            const raw = typeof options.secret === 'string' ? new TextEncoder().encode(options.secret) : options.secret;
+            return importOnce(`secret:${alg}`, () => subtle.importKey('raw', raw, importAs, false, ['verify']));
+        }
+        if (options.publicKey !== undefined) {
+            const key = options.publicKey;
+            if (typeof key === 'object' && 'type' in key && 'algorithm' in key)
+                return key;
+            if (typeof key === 'string') {
+                return importOnce(`pem:${alg}`, () => subtle.importKey('spki', pemToDer(key), importAs, false, ['verify']));
+            }
+            if (!jwkFits(key, alg))
+                throw new JwtError('JWT_KEY', `The key cannot verify ${alg}`);
+            return importOnce(`jwk:${alg}`, () => subtle.importKey('jwk', key, importAs, false, ['verify']));
+        }
+        const remote = typeof options.jwks === 'string' || options.jwks instanceof URL;
+        const cacheMs = options.jwksCacheMs ?? 10 * 60_000;
+        if (remote && (!keySet || now() - keySet.fetchedAt > cacheMs))
+            await refresh();
+        const find = () => keySet?.keys.filter((jwk) => (header.kid === undefined || jwk.kid === header.kid) && jwkFits(jwk, alg)) ?? [];
+        let matches = find();
+        if (matches.length === 0 && remote && now() - lastUnknownFetch >= (options.jwksCooldownMs ?? 30_000)) {
+            lastUnknownFetch = now();
+            await refresh();
+            matches = find();
+        }
+        const [jwk] = matches;
+        if (!jwk || (header.kid === undefined && matches.length > 1)) {
+            throw new JwtError('JWT_KEY', header.kid ? `No key "${header.kid}" for ${alg}` : `No single key for ${alg}`);
+        }
+        return importOnce(`jwks:${jwk.kid ?? ''}:${alg}`, () => subtle.importKey('jwk', jwk, importAs, false, ['verify']));
+    };
+    return {
+        async verify(token) {
+            const parts = token.split('.');
+            if (parts.length !== 3 || parts.some((part) => !/^[A-Za-z0-9_-]*$/.test(part)) || !parts[2]) {
+                throw new JwtError('JWT_MALFORMED', 'The token is not a signed JWT');
+            }
+            const [encodedHeader, encodedPayload, encodedSignature] = parts;
+            const header = decodeJson(encodedHeader, 'header');
+            const payload = decodeJson(encodedPayload, 'payload');
+            if (typeof header.alg !== 'string' || !allowed.has(header.alg)) {
+                throw new JwtError('JWT_ALGORITHM', `The algorithm "${String(header.alg)}" is not accepted`);
+            }
+            if (header.crit !== undefined)
+                throw new JwtError('JWT_MALFORMED', 'The token has critical headers this verifier does not understand');
+            const alg = header.alg;
+            const key = await keyFor(header, alg).catch((error) => {
+                throw error instanceof JwtError ? error : new JwtError('JWT_KEY', `No usable key for ${alg}`);
+            });
+            const valid = await subtle
+                .verify(cryptoParams(alg).verifyAs, key, base64UrlBytes(encodedSignature), new TextEncoder().encode(`${encodedHeader}.${encodedPayload}`))
+                .catch(() => false);
+            if (!valid)
+                throw new JwtError('JWT_SIGNATURE', 'The signature does not match');
+            checkClaims(payload, options, now() / 1000, tolerance);
+            return { header, payload };
+        },
+    };
+}
+function checkClaims(payload, options, seconds, tolerance) {
+    const numeric = (name) => {
+        const value = payload[name];
+        if (value !== undefined && typeof value !== 'number')
+            throw new JwtError('JWT_CLAIM', `"${name}" is not a number`);
+        return value;
+    };
+    const exp = numeric('exp');
+    const nbf = numeric('nbf');
+    const iat = numeric('iat');
+    if (exp === undefined && options.requireExpiry !== false)
+        throw new JwtError('JWT_CLAIM', 'The token has no expiry');
+    if (exp !== undefined && seconds >= exp + tolerance)
+        throw new JwtError('JWT_EXPIRED', 'The token has expired');
+    if (nbf !== undefined && seconds + tolerance < nbf)
+        throw new JwtError('JWT_NOT_YET_VALID', 'The token is not valid yet');
+    if (options.maxAgeSec !== undefined) {
+        if (iat === undefined)
+            throw new JwtError('JWT_CLAIM', 'The token has no issue time');
+        if (seconds - iat > options.maxAgeSec + tolerance)
+            throw new JwtError('JWT_TOO_OLD', 'The token is too old');
+    }
+    if (options.issuer !== undefined) {
+        const issuers = typeof options.issuer === 'string' ? [options.issuer] : options.issuer;
+        if (typeof payload.iss !== 'string' || !issuers.includes(payload.iss)) {
+            throw new JwtError('JWT_ISSUER', 'The token is from another issuer');
+        }
+    }
+    if (options.audience !== undefined && options.audience !== false) {
+        const wanted = typeof options.audience === 'string' ? [options.audience] : options.audience;
+        const audiences = typeof payload.aud === 'string' ? [payload.aud] : Array.isArray(payload.aud) ? payload.aud : [];
+        if (!audiences.some((audience) => wanted.includes(audience))) {
+            throw new JwtError('JWT_AUDIENCE', 'The token is for another audience');
+        }
+    }
+    for (const claim of options.requiredClaims ?? []) {
+        if (payload[claim] === undefined)
+            throw new JwtError('JWT_CLAIM', `The token has no "${claim}" claim`);
+    }
+}
+function jwtAuth(options) {
+    const verifier = createJwtVerifier(options);
+    const read = options.token ?? bearerToken;
+    return async (request) => {
+        const token = read(request);
+        if (token?.split('.').length !== 3)
+            return undefined;
+        let payload;
+        try {
+            payload = (await verifier.verify(token)).payload;
+        }
+        catch (error) {
+            if (!(error instanceof JwtError))
+                throw error;
+            const description = error.message.replace(/["\\]/g, "'");
+            return refuse(401, error.message, `Bearer error="invalid_token", error_description="${description}"`);
+        }
+        try {
+            const principal = options.principal
+                ? await options.principal(payload, request)
+                : principalFromClaims(payload, options.claims);
+            if (!principal)
+                return refuse(403, 'The token is valid, but not for this server');
+            return { method: 'jwt', claims: payload, ...principal };
+        }
+        catch (error) {
+            return refuse(403, error instanceof Error ? error.message : String(error));
+        }
+    };
+}
+function principalFromClaims(payload, names = {}) {
+    const subject = claimAt(payload, names.subject ?? 'sub');
+    const tenant = claimAt(payload, names.tenant ?? 'tenant_id');
+    const roles = listOf(claimAt(payload, names.roles ?? 'roles'));
+    const scopes = listOf(names.scopes ? claimAt(payload, names.scopes) : (payload.scope ?? payload.scp));
+    return {
+        ...(typeof subject === 'string' ? { userId: subject } : {}),
+        ...(typeof tenant === 'string' ? { tenantId: (0, tenant_js_1.assertTenantId)(tenant) } : {}),
+        ...(roles ? { roles } : {}),
+        ...(scopes ? { scopes } : {}),
+    };
+}
+async function hashApiKey(key) {
+    const digest = await globalThis.crypto.subtle.digest('SHA-256', new TextEncoder().encode(key));
+    return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('');
+}
+function apiKeyAuth(options) {
+    if (!options.keys && !options.lookup)
+        throw new TypeError('apiKeyAuth() needs keys, a lookup, or both');
+    const header = options.header?.toLowerCase();
+    return async (request) => {
+        const key = header
+            ? (request.headers.get(header) ?? undefined)
+            : (request.headers.get('x-api-key') ?? bearerToken(request));
+        if (!key)
+            return undefined;
+        if (options.prefix ? !key.startsWith(options.prefix) : /^[\w-]+\.[\w-]+\.[\w-]+$/.test(key))
+            return undefined;
+        const hash = await hashApiKey(key);
+        const known = Object.hasOwn(options.keys ?? {}, hash) ? options.keys?.[hash] : undefined;
+        const principal = known ?? (await options.lookup?.(hash, request));
+        if (!principal)
+            return refuse(401, 'The API key is not valid', 'Bearer error="invalid_token"');
+        if (principal.tenantId !== undefined)
+            (0, tenant_js_1.assertTenantId)(principal.tenantId);
+        return { method: 'api-key', ...principal };
+    };
+}
+function trustedProxyAuth(options) {
+    if (!options.secret && !options.trust) {
+        throw new TypeError('trustedProxyAuth() needs a secret or a trust check: otherwise anyone who sets the headers is trusted');
+    }
+    const names = {
+        user: 'x-forwarded-user',
+        tenant: 'x-forwarded-tenant',
+        roles: 'x-forwarded-roles',
+        scopes: 'x-forwarded-scopes',
+        ...options.headers,
+    };
+    const separator = options.separator ?? ',';
+    const list = (value) => value
+        ?.split(separator)
+        .map((item) => item.trim())
+        .filter(Boolean);
+    return async (request) => {
+        if (options.secret && !sameText(request.headers.get(options.secret.header) ?? '', options.secret.value)) {
+            return undefined;
+        }
+        if (options.trust && !(await options.trust(request)))
+            return undefined;
+        const user = request.headers.get(names.user);
+        if (!user)
+            return undefined;
+        const tenant = request.headers.get(names.tenant);
+        if (tenant && !/^[A-Za-z0-9_-]{1,128}$/.test(tenant))
+            return refuse(403, `"${tenant}" is not a valid tenant id`);
+        const roles = list(request.headers.get(names.roles));
+        const scopes = list(request.headers.get(names.scopes));
+        return {
+            method: 'proxy',
+            userId: user,
+            ...(tenant ? { tenantId: tenant } : {}),
+            ...(roles?.length ? { roles } : {}),
+            ...(scopes?.length ? { scopes } : {}),
+        };
+    };
+}
+function anyAuth(...hooks) {
+    return async (request) => {
+        let refusal;
+        for (const hook of hooks) {
+            const result = await hook(request);
+            if (result instanceof Response)
+                refusal ??= result;
+            else if (result)
+                return result;
+        }
+        return refusal;
+    };
+}
+function bearerToken(request) {
+    const match = /^Bearer\s+(\S+)\s*$/i.exec(request.headers.get('authorization') ?? '');
+    return match?.[1];
+}
+function refuse(status, message, challenge) {
+    return new Response(JSON.stringify({ error: { code: status === 401 ? 'UNAUTHORIZED' : 'FORBIDDEN', message } }), {
+        status,
+        headers: {
+            'content-type': 'application/json; charset=utf-8',
+            ...(challenge ? { 'www-authenticate': challenge } : {}),
+        },
+    });
+}
+function sameText(left, right) {
+    let difference = left.length ^ right.length;
+    const length = Math.max(left.length, right.length);
+    for (let index = 0; index < length; index += 1) {
+        difference |= (left.charCodeAt(index) || 0) ^ (right.charCodeAt(index) || 0);
+    }
+    return difference === 0;
+}
+function base64UrlBytes(text) {
+    const base64 = text
+        .replace(/-/g, '+')
+        .replace(/_/g, '/')
+        .padEnd(Math.ceil(text.length / 4) * 4, '=');
+    const binary = atob(base64);
+    const bytes = new Uint8Array(binary.length);
+    for (let index = 0; index < binary.length; index += 1)
+        bytes[index] = binary.charCodeAt(index);
+    return bytes;
+}
+function decodeJson(part, what) {
+    try {
+        const value = JSON.parse(new TextDecoder().decode(base64UrlBytes(part)));
+        if (value && typeof value === 'object' && !Array.isArray(value))
+            return value;
+    }
+    catch {
+    }
+    throw new JwtError('JWT_MALFORMED', `The token's ${what} is not a JSON object`);
+}
+function pemToDer(pem) {
+    const body = pem.replace(/-----(?:BEGIN|END) PUBLIC KEY-----/g, '').replace(/\s+/g, '');
+    if (!body || pem.includes('PRIVATE KEY'))
+        throw new JwtError('JWT_KEY', 'The public key is not a PEM public key');
+    return base64UrlBytes(body.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')).buffer;
+}
+function claimAt(payload, path) {
+    if (path in payload)
+        return payload[path];
+    let current = payload;
+    for (const segment of path.split('.')) {
+        if (!current || typeof current !== 'object')
+            return undefined;
+        current = current[segment];
+    }
+    return current;
+}
+function listOf(value) {
+    if (typeof value === 'string')
+        return value.split(/\s+/).filter(Boolean);
+    if (Array.isArray(value))
+        return value.filter((item) => typeof item === 'string');
+    return undefined;
+}

package/dist-cjs/server/deployments.d.ts

@@ -1,4 +1,5 @@
 import { type Regression, type RegressionMetric, type RunSample } from '../insights/regressions.js';
+import type { EvaluationScore, Evaluator } from '../types/evaluate.js';
 import type { DeploymentChange, DeploymentChangeRecord, DeploymentRecord, ReplicaReport, RevisionChoice, RevisionRequest, ServerAssistant, ServerDeployments, ServerStateStore } from '../types/server.js';
 export type { DeploymentChange, DeploymentChangeRecord, DeploymentRecord, ReplicaReport, RevisionChoice, RevisionReason, RevisionRequest, RunRevision, } from '../types/server.js';
 /** Options for `Deployments`. */
@@ -58,6 +59,30 @@ export interface RevisionStats {
     /** Mean recorded cost of finished runs, in US dollars. */
     meanCost?: number;
 }
+/** Options for `Deployments.evaluate()`. */
+export interface RunEvaluationOptions {
+    /** What each run is scored by: evaluators from `nexus-ai-pro/evaluate`, such as an LLM judge. */
+    evaluators: ReadonlyArray<Evaluator>;
+    /** Only runs created from this time, such as the start of a canary. */
+    since?: string;
+    /** Only this revision's runs. Defaults to every revision's. */
+    revision?: string;
+    /** The share of runs scored, from 0 to 1, chosen by run id so every replica picks the same ones. Defaults to 1. */
+    sampleRate?: number;
+    /** How many recent runs are read. Defaults to 2,000. */
+    limit?: number;
+    /** Recorded as each score's source; a run already scored by this source is skipped. Defaults to `online-evaluation`. */
+    source?: string;
+}
+/** What `Deployments.evaluate()` did. */
+export interface RunEvaluationReport {
+    /** Runs scored. */
+    evaluated: number;
+    /** Runs passed over because this source had already scored them. */
+    skipped: number;
+    /** Every score given. */
+    scores: EvaluationScore[];
+}
 /**
  * Revisions, traffic splits, and replica health for the agent server, recorded in a shared store.
  *
@@ -75,6 +100,8 @@ export interface RevisionStats {
 export declare class Deployments implements ServerDeployments {
     private readonly options;
     private readonly state;
+    /** The change in progress per deployment, which the next one waits for. */
+    private readonly changing;
     private readonly now;
     private readonly registrations;
     private readonly cache;
@@ -108,10 +135,16 @@ export declare class Deployments implements ServerDeployments {
      * `expectedVersion` applies a change only to the version it was decided on, and throws
      * `DEPLOYMENT_CONFLICT` otherwise, so two people — or a person and a canary guard — cannot undo each
      * other without noticing.
+     *
+     * Changes to one deployment through this object apply one at a time, so two made at once on one
+     * replica both land, in order. The state store has no atomic compare-and-set, so two replicas
+     * changing one deployment in the same instant can still lose one change: make changes from one
+     * place, or pass `expectedVersion` and retry a conflict.
      */
     change(assistant: string, change: DeploymentChange & {
         expectedVersion?: number;
     }, by?: string): Promise<DeploymentRecord>;
+    private applyChange;
     /** Sets each revision's share of new runs. */
     split(assistant: string, traffic: Record<string, number>, options?: {
         by?: string;
@@ -159,6 +192,17 @@ export declare class Deployments implements ServerDeployments {
      * expired run is an error, a cancelled one is left out.
      */
     samples(assistant: string, revision: string, since?: string, limit?: number): Promise<RunSample[]>;
+    /**
+     * Online evaluation of the server's own runs: scores each finished run's output with evaluators
+     * from `nexus-ai-pro/evaluate`, and records the scores as the run's feedback. A canary guard given
+     * those keys in `feedback` then compares quality between revisions, beside errors, latency, and
+     * cost.
+     *
+     * A run is scored once per `source`, so calling this on a schedule never counts a run twice.
+     * Sampling is stable: the same run is always in or out of a sample, on every replica. Evaluators
+     * judge a run by its output, since a run record does not keep its input.
+     */
+    evaluate(assistant: string, options: RunEvaluationOptions): Promise<RunEvaluationReport>;
     /** The deployment the router uses, reusing a recent read. */
     private current;
     /** The deployment of a registered assistant before anything is recorded: its live revision, all traffic. */
@@ -201,8 +245,26 @@ export interface CanaryGuardOptions {
         revision: string;
         since: string;
     }) => Promise<RunSample[]>;
-    /** Feedback keys whose mean score is compared, when `samples` supplies feedback. */
+    /**
+     * Feedback keys whose mean score is compared: the scores `Deployments.evaluate()` records from
+     * online evaluation, or any a person left through the feedback route. A canary that is worse on one
+     * is rolled back like one that fails more.
+     */
     feedback?: readonly string[];
+    /**
+     * Judges each metric as an evaluation comparison does, at this confidence, such as 0.95. A bootstrap
+     * interval must not reach zero, and the change must be at least the metric's minimum effect:
+     * `minErrorRateIncrease`, `latencyIncrease` with `minLatencyChangeMs`, `costIncrease`, and
+     * `minFeedbackDrop`. A real regression rolls back, and noise or a trivial change does not. Without
+     * it, the error rate uses a z-test at 95%, and latency and cost use their margins alone.
+     */
+    confidence?: number;
+    /** The smallest rise in the error rate, absolute, that counts under `confidence`. Defaults to 0.02. */
+    minErrorRateIncrease?: number;
+    /** The smallest fall in a feedback score's mean, absolute, that counts under `confidence`. Defaults to 0.05. */
+    minFeedbackDrop?: number;
+    /** Bootstrap resamples under `confidence`. Defaults to 1,000. */
+    resamples?: number;
     /** Recorded as who made a change. Defaults to `guard`. */
     by?: string;
     /** Hears every decision, including holds. */

package/dist-cjs/server/deployments.js

@@ -12,6 +12,7 @@ const EPSILON = 1e-9;
 class Deployments {
     options;
     state;
+    changing = new Map();
     now;
     registrations = new Map();
     cache = new Map();
@@ -86,6 +87,18 @@ class Deployments {
         return (await this.get(assistant))?.history ?? [];
     }
     async change(assistant, change, by) {
+        const previous = this.changing.get(assistant) ?? Promise.resolve();
+        const next = previous.catch(() => undefined).then(() => this.applyChange(assistant, change, by));
+        this.changing.set(assistant, next);
+        try {
+            return await next;
+        }
+        finally {
+            if (this.changing.get(assistant) === next)
+                this.changing.delete(assistant);
+        }
+    }
+    async applyChange(assistant, change, by) {
         const current = await this.get(assistant);
         if (!current) {
             throw new errors_js_1.BadRequestError(`Assistant "${assistant}" has no deployment yet: serve it through Deployments.assistant() first`, 'UNKNOWN_DEPLOYMENT');
@@ -233,15 +246,67 @@ class Deployments {
         return (await this.runs(assistant, { since, limit }))
             .filter((run) => (run.revision?.id ?? '') === revision)
             .flatMap((run) => {
+            const feedback = run.feedback?.length ? { feedback: run.feedback } : {};
             if (run.status === 'failed' || run.status === 'expired') {
-                return [{ status: 'error', latencyMs: run.durationMs, cost: run.cost }];
+                return [{ status: 'error', latencyMs: run.durationMs, cost: run.cost, ...feedback }];
             }
             if (run.status === 'succeeded' || run.status === 'awaiting_input') {
-                return [{ status: 'success', latencyMs: run.durationMs, cost: run.cost }];
+                return [{ status: 'success', latencyMs: run.durationMs, cost: run.cost, ...feedback }];
             }
             return [];
         });
     }
+    async evaluate(assistant, options) {
+        const source = options.source ?? 'online-evaluation';
+        const rate = options.sampleRate ?? 1;
+        const report = { evaluated: 0, skipped: 0, scores: [] };
+        for (const run of await this.runs(assistant, options)) {
+            if (run.status !== 'succeeded')
+                continue;
+            if (options.revision !== undefined && (run.revision?.id ?? '') !== options.revision)
+                continue;
+            if (rate < 1 && bucket(`evaluate:${run.id}`) >= rate)
+                continue;
+            if (run.feedback?.some((item) => item.source === source)) {
+                report.skipped += 1;
+                continue;
+            }
+            const scores = [];
+            for (const evaluator of options.evaluators) {
+                const value = await evaluator({
+                    example: { id: run.id, inputs: undefined, ...(run.metadata ? { metadata: run.metadata } : {}) },
+                    output: run.output,
+                    latencyMs: run.durationMs ?? 0,
+                    ...(run.cost === undefined ? {} : { cost: run.cost }),
+                    run: 0,
+                });
+                if (typeof value === 'number')
+                    scores.push({ key: 'score', score: value });
+                else if (typeof value === 'boolean')
+                    scores.push({ key: 'score', score: value ? 1 : 0, passed: value });
+                else
+                    scores.push(...(Array.isArray(value) ? value : [value]));
+            }
+            const at = new Date().toISOString();
+            const latest = (await this.state.get(state_js_1.RUNS_NAMESPACE, run.id)) ?? run;
+            await this.state.put(state_js_1.RUNS_NAMESPACE, run.id, {
+                ...latest,
+                feedback: [
+                    ...(latest.feedback ?? []),
+                    ...scores.map((score) => ({
+                        key: score.key,
+                        score: score.score,
+                        ...(score.comment ? { comment: score.comment } : {}),
+                        source,
+                        createdAt: at,
+                    })),
+                ].slice(-100),
+            });
+            report.evaluated += 1;
+            report.scores.push(...scores);
+        }
+        return report;
+    }
     async current(assistant) {
         const cached = this.cache.get(assistant);
         if (cached && Date.now() - cached.at < (this.options.cacheMs ?? 2_000))
@@ -311,6 +376,14 @@ function watchCanaries(options) {
             latencyIncrease: options.latencyIncrease,
             minLatencyChangeMs: options.minLatencyChangeMs ?? 50,
             costIncrease: options.costIncrease,
+            ...(options.confidence === undefined
+                ? {}
+                : {
+                    confidence: options.confidence,
+                    minErrorRateIncrease: options.minErrorRateIncrease,
+                    minFeedbackDrop: options.minFeedbackDrop,
+                    resamples: options.resamples,
+                }),
         });
         if (regressions.length > 0) {
             await deployments.change(deployment.assistant, {

package/dist-cjs/server/runs.d.ts

@@ -1,4 +1,5 @@
 import type { OperationRunnerConfig } from '../types/operations.js';
+import type { RunFeedback } from '../types/tracing.js';
 import type { Principal, ReplicaReport, RunEventLog, RunRecord, ScalingSnapshot, ServerAssistant, ServerStateStore, TenantGate, ThreadBusyPolicy, ThreadRecord } from '../types/server.js';
 import { ServerMetrics } from './metrics.js';
 /** What a run needs to start. */
@@ -161,6 +162,14 @@ export declare class RunManager {
     start(options: StartRunOptions): Promise<RunRecord>;
     /** The rest of `start()`, once the thread, if any, is this run's. */
     private startClaimed;
+    /**
+     * Leaves feedback on a run: a person's rating, or an online evaluator's score. A canary guard
+     * compares it between revisions as their quality. Another tenant's run is not found, and a run
+     * keeps its latest 100 pieces of feedback.
+     */
+    addFeedback(runId: string, feedback: Omit<RunFeedback, 'createdAt'> & {
+        createdAt?: string;
+    }, principal?: Principal): Promise<RunRecord>;
     /** Cancels a run, including one another replica is executing. */
     cancel(runId: string, principal?: Principal, reason?: string): Promise<RunRecord>;
     /**

package/dist-cjs/server/runs.js

@@ -180,6 +180,26 @@ class RunManager {
             throw error;
         }
     }
+    async addFeedback(runId, feedback, principal) {
+        const run = await this.run(runId, principal);
+        if (typeof feedback.key !== 'string' || !feedback.key.trim())
+            throw new errors_js_2.BadRequestError('Feedback needs a key');
+        if (feedback.score !== undefined && !(typeof feedback.score === 'number' && Number.isFinite(feedback.score))) {
+            throw new errors_js_2.BadRequestError('A feedback score is a finite number');
+        }
+        const at = this.now().toISOString();
+        const entry = {
+            key: feedback.key,
+            ...(feedback.score === undefined ? {} : { score: feedback.score }),
+            ...(feedback.value === undefined ? {} : { value: feedback.value }),
+            ...(typeof feedback.comment === 'string' ? { comment: feedback.comment } : {}),
+            ...(typeof feedback.source === 'string' ? { source: feedback.source } : {}),
+            createdAt: feedback.createdAt ?? at,
+        };
+        const next = { ...run, feedback: [...(run.feedback ?? []), entry].slice(-100), updatedAt: at };
+        await this.state.put(RUNS, runId, next);
+        return next;
+    }
     async cancel(runId, principal, reason) {
         const run = await this.run(runId, principal);
         this.local.get(runId)?.cancel(reason);