nexus-ai-pro 2.4.0 → 2.5.0

package/dist/context-hub/hub.js

@@ -1,6 +1,7 @@
 import { MemoryPromptStore } from '../prompts/memory.js';
 import { canonicalJson } from '../prompts/version.js';
 import { diffContexts } from './diff.js';
+import { contextDigest, signDigest, verifiedBy, } from './signing.js';
 export class ContextHubError extends Error {
     code;
     constructor(message, code) {
@@ -36,6 +37,16 @@ export class ContextPromotionError extends ContextHubError {
         this.name = 'ContextPromotionError';
     }
 }
+export class ContextSignatureError extends ContextHubError {
+    bundle;
+    version;
+    constructor(bundle, version, reason) {
+        super(`Context bundle "${bundle}" ${version}: ${reason}`, 'CONTEXT_SIGNATURE');
+        this.bundle = bundle;
+        this.version = version;
+        this.name = 'ContextSignatureError';
+    }
+}
 export class ContextDefinitionError extends ContextHubError {
     constructor(message) {
         super(message, 'CONTEXT_DEFINITION_ERROR');
@@ -63,12 +74,19 @@ export class ContextHub {
     options;
     store;
     now;
+    trusted = new Map();
     constructor(options = {}) {
         this.options = options;
+        if (options.requireSignature && !options.keyring) {
+            throw new TypeError('requireSignature needs a keyring of the keys the hub trusts');
+        }
         this.store = options.store ?? new MemoryPromptStore();
         this.now = options.now ?? (() => new Date());
     }
     async commit(definition, options = {}) {
+        return this.commitSigned(definition, options, []);
+    }
+    async commitSigned(definition, options, inherited) {
         validate(definition);
         if (this.options.prompts) {
             for (const [role, pin] of Object.entries(definition.prompts ?? {})) {
@@ -92,6 +110,13 @@ export class ContextHub {
                 ...(options.author === undefined ? {} : { author: options.author }),
                 ...(newest ? { parent: newest.version } : {}),
             };
+            const signatures = [...inherited];
+            if (this.options.signer) {
+                const digest = await contextDigest(await this.signedContent(definition));
+                signatures.push(await signDigest(this.options.signer, digest, this.now()));
+            }
+            if (signatures.length > 0)
+                stored.signatures = signatures;
             await this.store.saveVersion(toStored(stored));
             await this.record({ name: definition.name, action: 'commit', version: id, by: options.author });
         }
@@ -118,8 +143,19 @@ export class ContextHub {
             throw new ContextNotFoundError(name, ref);
         return found;
     }
+    async verify(bundle) {
+        const key = `${bundle.name}@${bundle.version}`;
+        let pending = this.trusted.get(key);
+        if (!pending) {
+            pending = this.check(bundle, await this.signedContent(bundle));
+            this.trusted.set(key, pending);
+            pending.catch(() => this.trusted.delete(key));
+        }
+        return pending;
+    }
     async resolve(name, ref = 'latest') {
         const bundle = await this.get(name, ref);
+        await this.assertTrusted(bundle);
         return {
             bundle,
             reference: {
@@ -133,6 +169,7 @@ export class ContextHub {
         const registry = this.options.prompts;
         if (!registry)
             throw new ContextDefinitionError('Rendering a pinned prompt needs the hub to have a prompt registry');
+        await this.assertTrusted(bundle);
         const pin = bundle.prompts?.[role];
         if (!pin)
             throw new ContextNotFoundError(bundle.name, `prompt role "${role}"`);
@@ -259,15 +296,52 @@ export class ContextHub {
                 throw new ContextDefinitionError(`Prompt ${prompt.name} was exported as ${version} but its content is ${committed.version}`);
             }
         }
-        const { version, createdAt: _createdAt, parent: _parent, message, author, ...definition } = exported.bundle;
+        const { version, createdAt: _createdAt, parent: _parent, message, author, signatures = [], ...definition } = exported.bundle;
         if ((await contextVersion(definition)) !== version) {
             throw new ContextDefinitionError(`Bundle ${definition.name} was exported as ${version} but its content has changed`);
         }
-        return this.commit(definition, {
+        const digest = await contextDigest(signedContentOf(definition, exportedPrompts(exported)));
+        const claims = signatures.filter((signature) => signature?.digest === digest);
+        if (this.options.requireSignature && this.options.keyring) {
+            const signed = await verifiedBy(this.options.keyring, digest, claims);
+            if (!signed) {
+                throw new ContextSignatureError(definition.name, version, signatures.length > 0
+                    ? 'no signature verifies against its content with a trusted key'
+                    : 'it is not signed, and this hub requires a signature');
+            }
+        }
+        return this.commitSigned(definition, {
             message: message ?? 'imported',
             ...((options.by ?? author) === undefined ? {} : { author: options.by ?? author }),
             ...(options.label === undefined ? {} : { label: options.label }),
-        });
+        }, claims);
+    }
+    async assertTrusted(bundle) {
+        if (!this.options.requireSignature)
+            return;
+        const verification = await this.verify(bundle);
+        if (!verification.trusted) {
+            throw new ContextSignatureError(bundle.name, bundle.version, bundle.signatures?.length
+                ? 'no signature verifies against its content with a trusted key'
+                : 'it is not signed, and this hub requires a signature');
+        }
+    }
+    async check(bundle, content) {
+        const digest = await contextDigest(content);
+        const signature = this.options.keyring
+            ? await verifiedBy(this.options.keyring, digest, bundle.signatures)
+            : undefined;
+        return { trusted: signature !== undefined, digest, ...(signature ? { signature } : {}) };
+    }
+    async signedContent(definition) {
+        const pins = Object.entries(definition.prompts ?? {});
+        if (pins.length > 0 && !this.options.prompts) {
+            throw new ContextDefinitionError(`Bundle "${definition.name}" pins prompts; signing or verifying it needs the hub to have a prompt registry`);
+        }
+        const prompts = [];
+        for (const [, pin] of pins)
+            prompts.push(await this.options.prompts.get(pin.name, pin.version));
+        return signedContentOf(definition, prompts);
     }
     async read(name, version) {
         const stored = await this.store.getVersion(name, version);
@@ -304,6 +378,37 @@ function validate(definition) {
         }
     }
 }
+function signedContentOf(definition, prompts) {
+    const byPin = new Map(prompts.map((prompt) => [`${prompt.name}@${prompt.version}`, prompt]));
+    const pinned = {};
+    for (const [role, pin] of Object.entries(definition.prompts ?? {})) {
+        const prompt = byPin.get(`${pin.name}@${pin.version}`);
+        if (!prompt)
+            throw new ContextDefinitionError(`Bundle "${definition.name}" pins ${role} to ${pin.name}@${pin.version}, which is not at hand`);
+        pinned[role] = {
+            name: prompt.name,
+            messages: prompt.messages,
+            partials: prompt.partials,
+            config: prompt.config,
+            defaults: prompt.defaults,
+        };
+    }
+    return {
+        bundle: {
+            name: definition.name,
+            description: definition.description,
+            prompts: definition.prompts,
+            instructions: definition.instructions,
+            tools: definition.tools,
+            skills: definition.skills,
+            config: definition.config,
+        },
+        prompts: pinned,
+    };
+}
+function exportedPrompts(exported) {
+    return exported.prompts;
+}
 function toStored(bundle) {
     return { ...bundle, messages: [], variables: [] };
 }

package/dist/context-hub/index.d.ts

@@ -4,8 +4,9 @@ import type { Dataset, DatasetExample, Evaluator, Experiment } from '../types/ev
 import type { ToolDefinition } from '../types/messages.js';
 import type { ContextBundle, ContextPromotionGate } from './hub.js';
 export { type ContextChange, type ContextDiff, diffContexts, formatContextDiff } from './diff.js';
-export { type ContextBundle, type ContextBundleDefinition, type ContextBundleExport, ContextConflictError, ContextDefinitionError, type ContextHistoryEntry, ContextHub, ContextHubError, type ContextHubOptions, type ContextLabel, ContextNotFoundError, type ContextPromotionContext, ContextPromotionError, type ContextPromotionGate, type ContextPromotionResult, type ContextPromptPin, type ContextReference, type ContextSkill, type ContextTool, contextVersion, } from './hub.js';
+export { type ContextBundle, type ContextBundleDefinition, type ContextBundleExport, ContextConflictError, ContextDefinitionError, type ContextHistoryEntry, ContextHub, ContextHubError, type ContextHubOptions, type ContextLabel, ContextNotFoundError, type ContextPromotionContext, ContextPromotionError, type ContextPromotionGate, ContextSignatureError, type ContextVerification, type ContextPromotionResult, type ContextPromptPin, type ContextReference, type ContextSkill, type ContextTool, contextVersion, } from './hub.js';
 export type { ExperimentGateOptions } from '../prompts/gates.js';
+export { type ContextCryptoKey, type ContextJsonWebKey, type ContextKeyring, type ContextSignature, type ContextSignedContent, type ContextSigner, type ContextSigningKey, contextDigest, ed25519Keyring, ed25519Signer, generateContextKey, } from './signing.js';
 /**
  * A bundle's instructions joined into one system text, in the order the bundle lists them, or only
  * the ones named in `include`, in that order.

package/dist/context-hub/index.js

@@ -1,7 +1,8 @@
 import { fingerprintOf } from '../evaluate/cache.js';
 import { experimentVerdict } from '../prompts/gates.js';
 export { diffContexts, formatContextDiff } from './diff.js';
-export { ContextConflictError, ContextDefinitionError, ContextHub, ContextHubError, ContextNotFoundError, ContextPromotionError, contextVersion, } from './hub.js';
+export { ContextConflictError, ContextDefinitionError, ContextHub, ContextHubError, ContextNotFoundError, ContextPromotionError, ContextSignatureError, contextVersion, } from './hub.js';
+export { contextDigest, ed25519Keyring, ed25519Signer, generateContextKey, } from './signing.js';
 export function contextInstructions(bundle, options = {}) {
     const names = options.include ?? Object.keys(bundle.instructions ?? {});
     return names

package/dist/context-hub/signing.d.ts

@@ -0,0 +1,95 @@
+import type { PromptDefinition } from '../types/prompts.js';
+/** A JSON Web Key, as Web Crypto exports one. */
+export interface ContextJsonWebKey {
+    /** The key type: `OKP` for Ed25519. */
+    kty?: string;
+    /** The curve: `Ed25519`. */
+    crv?: string;
+    /** The public key, in base64url. */
+    x?: string;
+    /** The private key, in base64url, on a private key only. */
+    d?: string;
+    /** Anything else the exporter wrote. */
+    [field: string]: unknown;
+}
+/** A Web Crypto key, as `crypto.subtle` returns one. Any runtime's `CryptoKey` is one. */
+export interface ContextCryptoKey {
+    /** Its algorithm. */
+    readonly algorithm: unknown;
+    /** What it may do. */
+    readonly usages: readonly string[];
+}
+/** One signature on a bundle version. Stored with it, and carried by its export. */
+export interface ContextSignature {
+    /** The id of the key that signed, which a keyring looks up. */
+    keyId: string;
+    /** Always `Ed25519`. */
+    algorithm: 'Ed25519';
+    /**
+     * What was signed: `sha256:` and the full SHA-256, in hex, of the bundle's content and the content
+     * of every prompt it pins. A bundle's version is only the first 12 hex digits, enough to name a
+     * version but not to stand behind a signature.
+     */
+    digest: string;
+    /** The signature over the digest, in base64url. */
+    signature: string;
+    /** ISO-8601 time it was signed. */
+    signedAt: string;
+}
+/** Signs bundle digests with a private key, under a key id. */
+export interface ContextSigner {
+    /** The key's id, recorded on each signature so a keyring finds the public key. */
+    keyId: string;
+    /** Signs bytes. */
+    sign(data: Uint8Array): Promise<Uint8Array> | Uint8Array;
+}
+/** Verifies signatures by key id. A signature by a key the keyring does not hold never verifies. */
+export interface ContextKeyring {
+    /** Whether `signature` is `keyId`'s signature over `data`. */
+    verify(keyId: string, data: Uint8Array, signature: Uint8Array): Promise<boolean> | boolean;
+}
+/** A new signing key, with both halves as JSON Web Keys to keep in a secret store. */
+export interface ContextSigningKey {
+    /** The key's id. */
+    keyId: string;
+    /** Signs with the new key. */
+    signer: ContextSigner;
+    /** The public half, for every keyring that should trust it. */
+    publicKey: ContextJsonWebKey;
+    /** The private half. Keep it where the signer runs, and nowhere else. */
+    privateKey: ContextJsonWebKey;
+}
+/**
+ * Creates an Ed25519 signing key through Web Crypto, so on any runtime. The key id defaults to the
+ * first 16 hex digits of the public key's SHA-256, which is stable for the key and names nothing
+ * else.
+ */
+export declare function generateContextKey(keyId?: string): Promise<ContextSigningKey>;
+/** A signer from an Ed25519 private key, as a JSON Web Key or a Web Crypto key. */
+export declare function ed25519Signer(keyId: string, privateKey: ContextJsonWebKey | ContextCryptoKey): Promise<ContextSigner>;
+/** A keyring of trusted Ed25519 public keys, by key id. */
+export declare function ed25519Keyring(keys: Record<string, ContextJsonWebKey | ContextCryptoKey>): Promise<ContextKeyring>;
+/** What a bundle signature covers: the bundle's versioned content, and its pinned prompts' content. */
+export interface ContextSignedContent {
+    /** The bundle's name and the content its version covers. */
+    bundle: {
+        name: string;
+        description?: string;
+        prompts?: Record<string, {
+            name: string;
+            version: string;
+        }>;
+        instructions?: Record<string, string>;
+        tools?: unknown;
+        skills?: unknown;
+        config?: Record<string, unknown>;
+    };
+    /** Each pinned prompt's content, by role: what its own version covers. */
+    prompts: Record<string, Pick<PromptDefinition, 'name' | 'messages' | 'partials' | 'config' | 'defaults'>>;
+}
+/** The digest a signature covers, as `sha256:` and 64 hex digits. */
+export declare function contextDigest(content: ContextSignedContent): Promise<string>;
+/** Signs a digest. */
+export declare function signDigest(signer: ContextSigner, digest: string, at: Date): Promise<ContextSignature>;
+/** The first signature that is a trusted key's signature over exactly this digest, if any. */
+export declare function verifiedBy(keyring: ContextKeyring, digest: string, signatures?: readonly ContextSignature[]): Promise<ContextSignature | undefined>;

package/dist/context-hub/signing.js

@@ -0,0 +1,88 @@
+import { canonicalJson } from '../prompts/version.js';
+import { decodeBase64, encodeBase64 } from '../utils/base64.js';
+const subtle = () => globalThis.crypto.subtle;
+const ED25519 = { name: 'Ed25519' };
+export async function generateContextKey(keyId) {
+    const pair = await subtle().generateKey(ED25519, true, ['sign', 'verify']);
+    const publicKey = await subtle().exportKey('jwk', pair.publicKey);
+    const privateKey = await subtle().exportKey('jwk', pair.privateKey);
+    const id = keyId ?? (await keyIdOf(publicKey));
+    return { keyId: id, signer: await ed25519Signer(id, pair.privateKey), publicKey, privateKey };
+}
+export async function ed25519Signer(keyId, privateKey) {
+    const key = isCryptoKey(privateKey)
+        ? privateKey
+        : await subtle().importKey('jwk', privateKey, ED25519, false, ['sign']);
+    return { keyId, sign: async (data) => new Uint8Array(await subtle().sign(ED25519, key, data)) };
+}
+export async function ed25519Keyring(keys) {
+    const imported = new Map();
+    for (const [keyId, key] of Object.entries(keys)) {
+        imported.set(keyId, isCryptoKey(key) ? key : await subtle().importKey('jwk', key, ED25519, false, ['verify']));
+    }
+    return {
+        async verify(keyId, data, signature) {
+            const key = imported.get(keyId);
+            if (!key)
+                return false;
+            try {
+                return await subtle().verify(ED25519, key, signature, data);
+            }
+            catch {
+                return false;
+            }
+        },
+    };
+}
+export async function contextDigest(content) {
+    const text = canonicalJson({ format: 'nexus-context-signature', formatVersion: 1, ...content });
+    const digest = await subtle().digest('SHA-256', new TextEncoder().encode(text));
+    return `sha256:${hexOf(new Uint8Array(digest))}`;
+}
+export async function signDigest(signer, digest, at) {
+    const signature = await signer.sign(new TextEncoder().encode(digest));
+    return {
+        keyId: signer.keyId,
+        algorithm: 'Ed25519',
+        digest,
+        signature: toBase64Url(signature),
+        signedAt: at.toISOString(),
+    };
+}
+export async function verifiedBy(keyring, digest, signatures = []) {
+    const data = new TextEncoder().encode(digest);
+    for (const signature of signatures) {
+        if (signature?.algorithm !== 'Ed25519' || signature.digest !== digest)
+            continue;
+        let bytes;
+        try {
+            bytes = fromBase64Url(signature.signature);
+        }
+        catch {
+            continue;
+        }
+        if (await keyring.verify(signature.keyId, data, bytes))
+            return signature;
+    }
+    return undefined;
+}
+async function keyIdOf(publicKey) {
+    const digest = await subtle().digest('SHA-256', new TextEncoder().encode(String(publicKey.x)));
+    return hexOf(new Uint8Array(digest).subarray(0, 8));
+}
+function hexOf(bytes) {
+    let hex = '';
+    for (const byte of bytes)
+        hex += byte.toString(16).padStart(2, '0');
+    return hex;
+}
+function isCryptoKey(value) {
+    return typeof value.algorithm === 'object' && Array.isArray(value.usages);
+}
+function toBase64Url(bytes) {
+    return encodeBase64(bytes).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+function fromBase64Url(text) {
+    const padded = text.replace(/-/g, '+').replace(/_/g, '/');
+    return decodeBase64(padded + '='.repeat((4 - (padded.length % 4)) % 4));
+}

package/dist/doctor/index.d.ts

@@ -42,8 +42,10 @@ export interface DoctorDeployment {
     replicas: number;
     /**
      * The stores and shared state it uses, by any name: a checkpointer, the operation store, the
-     * server's state, tenant usage, a rate-limit store, a circuit store, a cache. Each is reported when
-     * it keeps its state in process memory, where each replica would see only its own.
+     * server's state, tenant usage, a rate-limit store, a circuit store, a cache, a tracer. Each is
+     * reported when it keeps its state in process memory, where each replica would see only its own,
+     * and when it lacks an opt-in setting a deployment needs: a Redis operation store without its
+     * dispatch index, or a tracer writing to a persistent store without incremental tracing.
      */
     stores?: Record<string, unknown>;
 }

package/dist/doctor/index.js

@@ -54,6 +54,7 @@ export async function diagnose(options = {}) {
     await run('queue', 'Operation queue and leases', () => queueCheck(options.operations, now(), options.maxQueueAgeMs));
     await run('health', 'Provider health', () => healthCheck(options.ai));
     await run('topology', 'Settings that only work in one process', () => topologyCheck(options.deployment));
+    await run('settings', 'Opt-in settings a deployment needs', () => settingsCheck(options.deployment));
     const failures = checks.filter((check) => check.status === 'fail').length;
     const warnings = checks.filter((check) => check.status === 'warn').length;
     return { checks, ok: failures === 0, failures, warnings };
@@ -235,3 +236,31 @@ function topologyCheck(deployment) {
         hint: 'Use the Redis, Postgres, or SQLite adapter for each, or run one replica.',
     };
 }
+function settingsCheck(deployment) {
+    const stores = Object.entries(deployment?.stores ?? {});
+    if (stores.length === 0)
+        return { status: 'skip', detail: 'No stores described.' };
+    const findings = [];
+    const hints = new Set();
+    for (const [name, value] of stores) {
+        const kind = kindOf(value);
+        if (kind === 'RedisOperationStore' && value.indexed === false) {
+            findings.push(`${name} is a RedisOperationStore without index: true, so finding queued work reads every record and a long queue slows every claim`);
+            hints.add('Give the Redis operation store index: true once every worker sharing its prefix runs 2.2 or later, then call reindex() once.');
+        }
+        if (kind === 'Tracer') {
+            const tracer = value;
+            const store = kindOf(tracer.store);
+            if (tracer.incremental === false && store && store !== 'MemoryTraceStore') {
+                findings.push(`${name} writes to a ${store} without incremental, so a trace is written when its root finishes and a replica that dies mid-run loses it`);
+                hints.add('Create the Tracer with incremental: true, and close what a dead process left with closeAbandonedRuns().');
+            }
+        }
+    }
+    if (findings.length === 0)
+        return { status: 'ok', detail: 'No store lacks a setting a deployment needs.' };
+    return { status: 'warn', detail: `${findings.join('; ')}.`, hint: [...hints].join(' ') };
+}
+function kindOf(value) {
+    return value?.constructor?.name;
+}

package/dist/evaluate/threads.d.ts

@@ -0,0 +1,187 @@
+/**
+ * Evaluation of whole conversations. A simulated user holds a conversation with an agent for each
+ * goal in a dataset, and thread evaluators score the conversation: whether the goal was met, each
+ * turn, and the turn where it went wrong. A multi-turn agent is then compared between versions as a
+ * single-turn one is, with `evaluate()` and `compareExperiments()`.
+ */
+import type { LLMJudge } from '../evals/judge.js';
+import type { Evaluator } from '../types/evaluate.js';
+import type { CompletionRequest, Message } from '../types/messages.js';
+import type { EvaluationTarget } from './run.js';
+/** One turn of a conversation. */
+export interface ThreadTurn {
+    /** Who spoke. */
+    role: 'user' | 'assistant';
+    /** What they said. */
+    content: string;
+    /** For an assistant turn, how long the agent took, in milliseconds. */
+    latencyMs?: number;
+    /**
+     * For an assistant turn, every message the agent added: its tool calls, their results, and the
+     * answer. The agent is given them back on later turns, and an evaluator can read its trajectory.
+     */
+    messages?: Message[];
+}
+/** Why a conversation ended. */
+export type ThreadEnding = 'goal' | 'gave-up' | 'max-turns' | 'error';
+/** A finished conversation: what a conversation target returns, and what thread evaluators read. */
+export interface Thread {
+    /** Every turn, in order. */
+    turns: ThreadTurn[];
+    /**
+     * `goal` when the user said its goal was met, `gave-up` when it said it never would be, `max-turns`
+     * when the turn limit ended it, and `error` when the agent failed.
+     */
+    ended: ThreadEnding;
+    /** The agent's error, when it failed. */
+    error?: string;
+}
+/** What a conversation is about: the user's goal, and optionally who they are and how they open. */
+export interface ThreadInputs {
+    /** What the user wants, in their words or the dataset's. */
+    goal: string;
+    /** Who the user is and how they talk, such as "an impatient customer who writes in short lines". */
+    persona?: string;
+    /** The user's first message, instead of letting the simulated user write it. */
+    opening?: string;
+}
+/** What a simulated user sees when it writes its next message. */
+export interface SimulatedUserContext {
+    /** The goal. */
+    goal: string;
+    /** The persona, when the example gives one. */
+    persona?: string;
+    /** The conversation so far. Empty for the first message. */
+    turns: readonly ThreadTurn[];
+    /** Aborted when the evaluation is cancelled or times out. */
+    signal?: AbortSignal;
+}
+/** A simulated user's move: its next message, or the end of the conversation and why. */
+export type SimulatedUserReply = {
+    message: string;
+} | {
+    done: 'goal' | 'gave-up';
+};
+/** Plays the user in a conversation. */
+export interface SimulatedUser {
+    /** The next move, given the conversation so far. */
+    next(context: SimulatedUserContext): Promise<SimulatedUserReply> | SimulatedUserReply;
+}
+/** A model client, as the judges take one. */
+export interface SimulatedUserClient {
+    /** Runs one completion. */
+    complete(request: CompletionRequest): Promise<{
+        content: string;
+    }>;
+}
+/** Options for `simulatedUser()`. */
+export interface SimulatedUserOptions {
+    /** The client the user's model is called through. */
+    client: SimulatedUserClient;
+    /** The model that plays the user. */
+    model: string;
+    /** A persona for every conversation, when the examples give none. */
+    persona?: string;
+    /** Sampling temperature. Defaults to 0, so a conversation repeats. */
+    temperature?: number;
+    /** Output token limit for each message. Defaults to 300. */
+    maxTokens?: number;
+    /** Replaces the instructions the model is given. `{goal}` and `{persona}` are filled in. */
+    instructions?: string;
+}
+/**
+ * A simulated user driven by a model: it is told the goal and the persona, sees the conversation
+ * with the roles turned around, and writes the next message, or says it is done and why.
+ */
+export declare function simulatedUser(options: SimulatedUserOptions): SimulatedUser;
+/**
+ * A user that follows a script, for deterministic tests: its messages in order, then `done`. A
+ * function decides each move itself.
+ */
+export declare function scriptedUser(script: {
+    messages: readonly string[];
+    done?: 'goal' | 'gave-up';
+} | SimulatedUser['next']): SimulatedUser;
+/** What a thread agent is told besides the conversation. */
+export interface ThreadAgentContext<I extends ThreadInputs = ThreadInputs> {
+    /** The example's inputs. */
+    inputs: I;
+    /** The turn, counted from 1. */
+    turn: number;
+    /** Names this conversation, the same on every turn and new for every conversation: a thread id. */
+    conversationId: string;
+    /** Aborted when the evaluation is cancelled or times out. */
+    signal?: AbortSignal;
+}
+/**
+ * What a thread agent answers: the reply, or the reply and every message the agent added on the way,
+ * such as its tool calls and their results. Those are given back to it on later turns.
+ */
+export type ThreadAgentReply = string | {
+    content: string;
+    messages?: Message[];
+};
+/**
+ * The agent a conversation target talks to. It is given the conversation so far, with its own earlier
+ * turns as it recorded them, and answers the user's newest message.
+ */
+export type ThreadAgent<I extends ThreadInputs = ThreadInputs> = (messages: readonly Message[], context: ThreadAgentContext<I>) => Promise<ThreadAgentReply> | ThreadAgentReply;
+/** Options for `conversationTarget()`. */
+export interface ConversationTargetOptions<I extends ThreadInputs = ThreadInputs> {
+    /** The user, or a user per example, such as one with the example's own persona. */
+    user: SimulatedUser | ((inputs: I) => SimulatedUser);
+    /** The most user messages a conversation may have. Defaults to 10. */
+    maxTurns?: number;
+}
+/**
+ * An `evaluate()` target that holds a conversation per example: the simulated user speaks, the agent
+ * answers, until the user says it is done or the turn limit is reached. It returns the `Thread`.
+ */
+export declare function conversationTarget<I extends ThreadInputs = ThreadInputs>(agent: ThreadAgent<I>, options: ConversationTargetOptions<I>): EvaluationTarget<I>;
+/** A graph a thread agent can drive: one that takes `{ messages }`, as `createAgent()` builds. */
+export interface ThreadGraph {
+    /** Runs one turn. */
+    invoke(input: {
+        messages: Message[];
+    }, options?: {
+        threadId?: string;
+        signal?: AbortSignal;
+    }): Promise<{
+        state: unknown;
+    }>;
+}
+/**
+ * An agent from `createAgent()`, or any graph that takes `{ messages }` and ends with an `answer`, as
+ * a thread agent. Each turn runs on the conversation's thread with the transcript so far, and the
+ * messages the run added to its `messages` are recorded with the turn. The agent therefore sees its
+ * own tool calls on later turns, as it does with a real user, without needing a checkpointer.
+ */
+export declare function graphThreadAgent(graph: ThreadGraph): ThreadAgent;
+/** The conversation as text, for a judge or a person. */
+export declare function formatThread(thread: Thread): string;
+/**
+ * Whether the user said its goal was met: 1 when the conversation ended on `goal`, otherwise 0, with
+ * how it ended as the comment. Key `goal_reached`.
+ */
+export declare function goalReached(): Evaluator<ThreadInputs>;
+/**
+ * Whether the goal was met, in a judge's opinion of the whole conversation, not the user's. Key
+ * `goal_completion`, with the judge's rationale as the comment.
+ */
+export declare function goalCompletion(judge: Pick<LLMJudge, 'evaluate'>): Evaluator<ThreadInputs>;
+/** How many times the assistant spoke: fewer is better. Key `turns`; compare with `lowerIsBetter: ['turns']`. */
+export declare function turnCount(): Evaluator<ThreadInputs>;
+/** Options for `turnScores()`. */
+export interface TurnScoresOptions {
+    /** The score's key. Defaults to `turn_score`. */
+    key?: string;
+    /** A turn scoring below this went wrong. Defaults to 0.5. */
+    threshold?: number;
+}
+/**
+ * Scores every assistant turn with an evaluator of single answers, and the conversation by their
+ * mean. The evaluator sees each turn as an example of its own: the user's message as the input, the
+ * conversation before it in `metadata.history`, and the turn as the output. The score's metadata holds
+ * each turn's score, and `wentWrongAt`, the first turn below the threshold, counted from 1.
+ */
+export declare function turnScores<I = unknown>(evaluator: Evaluator<I>, options?: TurnScoresOptions): Evaluator<ThreadInputs>;