nexus-ai-pro 2.19.0 → 2.20.2

package/dist/agent/openapi.js

@@ -0,0 +1,455 @@
+import { linkSignals } from '../utils/signals.js';
+export class OpenApiError extends Error {
+    code;
+    status;
+    constructor(message, code, status) {
+        super(message);
+        this.code = code;
+        this.status = status;
+        this.name = 'OpenApiError';
+    }
+}
+const METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
+const IGNORED_HEADERS = new Set(['accept', 'content-type', 'authorization']);
+const OPENAPI_ONLY = new Set(['nullable', 'example', 'xml', 'externalDocs', 'discriminator']);
+export function openApiTools(document, options = {}) {
+    const doc = parseDocument(document);
+    const apiName = options.name ?? (slug(doc.info?.title ?? '') || 'api');
+    const baseUrl = baseOf(doc, options);
+    const tools = [];
+    const operations = [];
+    const results = {};
+    const skipped = [];
+    const used = new Set();
+    for (const [path, item] of Object.entries(doc.paths ?? {})) {
+        const pathItem = deref(doc, item);
+        for (const method of METHODS) {
+            const raw = pathItem[method];
+            if (!isRecord(raw))
+                continue;
+            const operationId = typeof raw.operationId === 'string' ? raw.operationId : undefined;
+            const name = uniqueName([options.prefix, sanitize(operationId ?? `${method}_${path}`) || method].filter(Boolean).join('_'), used);
+            const operation = {
+                name,
+                method: method.toUpperCase(),
+                path,
+                ...(operationId ? { operationId } : {}),
+                ...(typeof raw.summary === 'string' ? { summary: raw.summary } : {}),
+                tags: Array.isArray(raw.tags) ? raw.tags.map(String) : [],
+                access: method === 'delete' ? 'destructive' : ['get', 'head', 'options', 'trace'].includes(method) ? 'read' : 'write',
+            };
+            if (options.include && !options.include(operation)) {
+                used.delete(name);
+                continue;
+            }
+            const built = buildOperation(doc, pathItem, raw);
+            if ('skip' in built) {
+                used.delete(name);
+                skipped.push({ method: operation.method, path, reason: built.skip });
+                continue;
+            }
+            operations.push(operation);
+            if (built.result)
+                results[name] = built.result;
+            tools.push({
+                name,
+                description: describe(raw, operation),
+                parameters: built.parameters,
+                capabilities: [
+                    `api.${operation.access}:${apiName}/${name}`,
+                    ...(baseUrl ? [`network:${new URL(baseUrl).host}`] : []),
+                ],
+                ...(baseUrl ? { execute: caller(operation, built.placements, built.bodyType, baseUrl, options) } : {}),
+            });
+        }
+    }
+    return { name: apiName, ...(baseUrl ? { baseUrl } : {}), tools, operations, results, skipped };
+}
+export async function loadOpenApiTools(url, options = {}) {
+    const response = await (options.fetch ?? globalThis.fetch)(url, {
+        headers: { accept: 'application/json' },
+        signal: AbortSignal.timeout(options.timeoutMs ?? 30_000),
+    });
+    if (!response.ok)
+        throw new OpenApiError(`The OpenAPI document at ${url} answered ${response.status}`, 'OPENAPI_REQUEST', response.status);
+    return openApiTools(await response.text(), { documentUrl: url, ...options });
+}
+function parseDocument(document) {
+    let doc = document;
+    if (typeof document === 'string') {
+        try {
+            doc = JSON.parse(document);
+        }
+        catch {
+            throw new OpenApiError('The OpenAPI document is not JSON; parse a YAML document first, and pass the object', 'OPENAPI_INVALID');
+        }
+    }
+    if (!isRecord(doc))
+        throw new OpenApiError('An OpenAPI document is an object', 'OPENAPI_INVALID');
+    if (typeof doc.swagger === 'string') {
+        throw new OpenApiError(`Swagger ${doc.swagger} is not read; convert the document to OpenAPI 3`, 'OPENAPI_UNSUPPORTED');
+    }
+    if (typeof doc.openapi !== 'string' || !doc.openapi.startsWith('3.')) {
+        throw new OpenApiError(`An OpenAPI 3 document names its version in "openapi", not ${JSON.stringify(doc.openapi)}`, 'OPENAPI_INVALID');
+    }
+    return doc;
+}
+function baseOf(doc, options) {
+    if (options.baseUrl) {
+        try {
+            return new URL(options.baseUrl).href.replace(/\/+$/, '');
+        }
+        catch {
+            throw new OpenApiError(`baseUrl must be an absolute URL, not ${JSON.stringify(options.baseUrl)}`, 'OPENAPI_INVALID');
+        }
+    }
+    const server = doc.servers?.[0];
+    if (!server || typeof server.url !== 'string')
+        return undefined;
+    const url = server.url.replace(/\{([^}]+)\}/g, (_match, name) => server.variables?.[name]?.default ?? '');
+    try {
+        return new URL(url, options.documentUrl).href.replace(/\/+$/, '');
+    }
+    catch {
+        return undefined;
+    }
+}
+function deref(doc, value, seen = []) {
+    if (!isRecord(value) || typeof value.$ref !== 'string')
+        return value;
+    const ref = value.$ref;
+    if (seen.includes(ref))
+        throw new OpenApiError(`${ref} refers to itself`, 'OPENAPI_INVALID');
+    return deref(doc, resolve(doc, ref), [...seen, ref]);
+}
+function resolve(doc, ref) {
+    if (!ref.startsWith('#')) {
+        throw new OpenApiError(`${ref} is outside the document; only references within it are followed`, 'OPENAPI_UNSUPPORTED');
+    }
+    let node = doc;
+    for (const part of ref.slice(1).split('/').slice(1)) {
+        const key = decodeURIComponent(part).replace(/~1/g, '/').replace(/~0/g, '~');
+        node = isRecord(node) || Array.isArray(node) ? node[key] : undefined;
+        if (node === undefined)
+            throw new OpenApiError(`${ref} names nothing in the document`, 'OPENAPI_INVALID');
+    }
+    return node;
+}
+function schemaOf(doc, schema, omit, seen = []) {
+    if (!isRecord(schema))
+        return {};
+    if (typeof schema.$ref === 'string') {
+        const ref = schema.$ref;
+        if (seen.includes(ref))
+            return { description: `The same shape as ${ref.split('/').at(-1)}, nested` };
+        const { $ref: _ref, ...siblings } = schema;
+        return { ...schemaOf(doc, resolve(doc, ref), omit, [...seen, ref]), ...schemaOf(doc, siblings, omit, seen) };
+    }
+    const out = {};
+    const one = (value) => (isRecord(value) ? schemaOf(doc, value, omit, seen) : value);
+    const many = (value) => (Array.isArray(value) ? value.map(one) : value);
+    const each = (value) => isRecord(value) ? Object.fromEntries(Object.entries(value).map(([key, item]) => [key, one(item)])) : value;
+    for (const [key, value] of Object.entries(schema)) {
+        if (OPENAPI_ONLY.has(key))
+            continue;
+        if (key === 'properties' && isRecord(value)) {
+            const kept = {};
+            for (const [name, property] of Object.entries(value)) {
+                const read = schemaOf(doc, property, omit, seen);
+                if (read[omit] === true)
+                    continue;
+                kept[name] = read;
+            }
+            out.properties = kept;
+        }
+        else if (['allOf', 'anyOf', 'oneOf', 'prefixItems'].includes(key))
+            out[key] = many(value);
+        else if (['$defs', 'definitions', 'patternProperties', 'dependentSchemas'].includes(key))
+            out[key] = each(value);
+        else if (['items', 'additionalProperties', 'not', 'contains', 'propertyNames', 'if', 'then', 'else'].includes(key)) {
+            out[key] = one(value);
+        }
+        else
+            out[key] = value;
+    }
+    if (Array.isArray(out.required) && isRecord(out.properties)) {
+        const properties = out.properties;
+        out.required = out.required.filter((name) => typeof name === 'string' && name in properties);
+        if (out.required.length === 0)
+            delete out.required;
+    }
+    if (schema.nullable === true) {
+        if (typeof out.type === 'string')
+            out.type = [out.type, 'null'];
+        else if (Array.isArray(out.type) && !out.type.includes('null'))
+            out.type = [...out.type, 'null'];
+    }
+    if ('example' in schema && !('examples' in schema))
+        out.examples = [schema.example];
+    return out;
+}
+function buildOperation(doc, pathItem, raw) {
+    const declared = new Map();
+    for (const list of [pathItem.parameters, raw.parameters]) {
+        if (!Array.isArray(list))
+            continue;
+        for (const item of list) {
+            const parameter = deref(doc, item);
+            if (!isRecord(parameter) || typeof parameter.name !== 'string' || typeof parameter.in !== 'string')
+                continue;
+            declared.set(`${parameter.in}\u0000${parameter.name}`, parameter);
+        }
+    }
+    const properties = {};
+    const required = [];
+    const placements = [];
+    for (const parameter of declared.values()) {
+        const place = parameter.in;
+        const name = parameter.name;
+        if (place === 'cookie' || (place === 'header' && IGNORED_HEADERS.has(name.toLowerCase())))
+            continue;
+        if (place !== 'path' && place !== 'query' && place !== 'header')
+            continue;
+        const arg = name in properties ? `${place}_${name}` : name;
+        const content = isRecord(parameter.content) ? Object.values(parameter.content)[0] : undefined;
+        const schema = schemaOf(doc, parameter.schema ?? (isRecord(content) ? content.schema : undefined), 'readOnly');
+        properties[arg] = {
+            ...schema,
+            ...(typeof parameter.description === 'string' && !schema.description
+                ? { description: parameter.description }
+                : {}),
+        };
+        if (place === 'path' || parameter.required === true)
+            required.push(arg);
+        placements.push({ arg, name, in: place });
+    }
+    let bodyType;
+    const body = deref(doc, raw.requestBody);
+    if (isRecord(body) && isRecord(body.content)) {
+        const types = Object.keys(body.content);
+        const json = types.find((type) => /^application\/(?:[\w.+-]+\+)?json\b/i.test(type));
+        const form = types.find((type) => /^application\/x-www-form-urlencoded\b/i.test(type));
+        const chosen = json ?? form;
+        if (!chosen)
+            return { skip: `its request body is ${types.join(' or ')}; only JSON and form bodies are sent` };
+        bodyType = json ? 'json' : 'form';
+        const media = body.content[chosen];
+        const schema = schemaOf(doc, isRecord(media) ? media.schema : undefined, 'readOnly');
+        const fields = isRecord(schema.properties) ? schema.properties : undefined;
+        const flat = fields &&
+            (schema.type === 'object' || schema.type === undefined) &&
+            !schema.allOf &&
+            !schema.anyOf &&
+            !schema.oneOf &&
+            Object.keys(fields).every((field) => !(field in properties));
+        if (flat) {
+            const fieldsRequired = new Set(Array.isArray(schema.required) ? schema.required.map(String) : []);
+            for (const [field, fieldSchema] of Object.entries(fields)) {
+                properties[field] = fieldSchema;
+                if (body.required === true && fieldsRequired.has(field))
+                    required.push(field);
+                placements.push({ arg: field, name: field, in: 'body-field' });
+            }
+        }
+        else {
+            const arg = 'body' in properties ? 'requestBody' : 'body';
+            properties[arg] = {
+                ...schema,
+                ...(typeof body.description === 'string' && !schema.description ? { description: body.description } : {}),
+            };
+            if (body.required === true)
+                required.push(arg);
+            placements.push({ arg, name: arg, in: 'body' });
+        }
+    }
+    let result;
+    const responses = isRecord(raw.responses) ? raw.responses : {};
+    const success = Object.keys(responses)
+        .filter((code) => /^2(?:\d\d|XX)$/i.test(code))
+        .sort()[0];
+    if (success) {
+        const response = deref(doc, responses[success]);
+        const content = isRecord(response) && isRecord(response.content) ? response.content : undefined;
+        const json = content && Object.keys(content).find((type) => /json/i.test(type));
+        const media = json && content ? content[json] : undefined;
+        if (isRecord(media) && media.schema !== undefined)
+            result = schemaOf(doc, media.schema, 'writeOnly');
+        else if (!content)
+            result = { type: 'null' };
+    }
+    return {
+        parameters: {
+            type: 'object',
+            properties,
+            ...(required.length ? { required } : {}),
+            additionalProperties: false,
+        },
+        placements,
+        ...(bodyType ? { bodyType } : {}),
+        ...(result ? { result } : {}),
+    };
+}
+function describe(raw, operation) {
+    const text = [raw.summary, raw.description].filter((part) => typeof part === 'string' && part.trim() !== '');
+    const said = text.length ? text.join('\n\n') : `${operation.method} ${operation.path}`;
+    return said.length > 1_024 ? `${said.slice(0, 1_021)}...` : said;
+}
+function caller(operation, placements, bodyType, baseUrl, options) {
+    const base = new URL(`${baseUrl}/`);
+    return async (args, context) => {
+        context?.signal?.throwIfAborted();
+        let path = operation.path;
+        for (const placement of placements.filter((item) => item.in === 'path')) {
+            const value = args[placement.arg];
+            if (value === undefined || value === null || String(value) === '') {
+                throw new OpenApiError(`${operation.name} needs "${placement.arg}"`, 'OPENAPI_INVALID');
+            }
+            const text = String(value);
+            if (text === '.' || text === '..') {
+                throw new OpenApiError(`"${placement.arg}" cannot be "${text}"`, 'OPENAPI_INVALID');
+            }
+            path = path.split(`{${placement.name}}`).join(encodeURIComponent(text));
+        }
+        const url = new URL(path.replace(/^\/+/, ''), base);
+        const headers = { accept: 'application/json' };
+        let body;
+        for (const placement of placements) {
+            const value = args[placement.arg];
+            if (value === undefined || value === null)
+                continue;
+            if (placement.in === 'query') {
+                for (const item of Array.isArray(value) ? value : [value])
+                    url.searchParams.append(placement.name, text(item));
+            }
+            else if (placement.in === 'header')
+                headers[placement.name.toLowerCase()] = text(value);
+            else if (placement.in === 'body-field') {
+                body ??= {};
+                body[placement.name] = value;
+            }
+            else if (placement.in === 'body')
+                body = value;
+        }
+        const own = typeof options.headers === 'function' ? await options.headers({ operation, context }) : options.headers;
+        for (const [name, value] of Object.entries(own ?? {}))
+            headers[name.toLowerCase()] = value;
+        let payload;
+        if (body !== undefined && bodyType === 'form') {
+            headers['content-type'] = 'application/x-www-form-urlencoded';
+            payload = new URLSearchParams(Object.entries(body).map(([key, value]) => [key, text(value)])).toString();
+        }
+        else if (body !== undefined) {
+            headers['content-type'] = 'application/json';
+            payload = JSON.stringify(body);
+        }
+        const linked = linkSignals([context?.signal, AbortSignal.timeout(options.timeoutMs ?? 30_000)]);
+        try {
+            const response = await (options.fetch ?? globalThis.fetch)(url.href, {
+                method: operation.method,
+                headers,
+                ...(payload !== undefined ? { body: payload } : {}),
+                redirect: 'manual',
+                signal: linked.signal,
+            });
+            const answer = await readCapped(response, options.maxResponseBytes ?? 1_000_000, operation);
+            if (response.status >= 300 && response.status < 400) {
+                throw new OpenApiError(`${operation.name} was redirected to ${response.headers.get('location') ?? 'another place'}, which is not followed`, 'OPENAPI_REQUEST', response.status);
+            }
+            if (!response.ok) {
+                throw new OpenApiError(`${operation.name} answered ${response.status}: ${detailOf(answer)}`, 'OPENAPI_REQUEST', response.status);
+            }
+            if (answer.trim() === '')
+                return null;
+            if (/json/i.test(response.headers.get('content-type') ?? '')) {
+                try {
+                    return JSON.parse(answer);
+                }
+                catch {
+                    throw new OpenApiError(`${operation.name} answered with JSON that does not parse`, 'OPENAPI_REQUEST', response.status);
+                }
+            }
+            return answer;
+        }
+        catch (error) {
+            if (error instanceof OpenApiError || context?.signal?.aborted)
+                throw error;
+            throw new OpenApiError(`${operation.name} could not be reached: ${linked.signal.aborted ? `no answer within ${options.timeoutMs ?? 30_000} ms` : messageOf(error)}`, 'OPENAPI_REQUEST');
+        }
+        finally {
+            linked.dispose();
+        }
+    };
+}
+async function readCapped(response, limit, operation) {
+    if (!response.body)
+        return '';
+    const reader = response.body.getReader();
+    const chunks = [];
+    let size = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        size += value.byteLength;
+        if (size > limit) {
+            await reader.cancel().catch(() => undefined);
+            throw new OpenApiError(`${operation.name} answered more than ${limit} bytes`, 'OPENAPI_REQUEST', response.status);
+        }
+        chunks.push(value);
+    }
+    const bytes = new Uint8Array(size);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return new TextDecoder().decode(bytes);
+}
+function detailOf(answer) {
+    try {
+        const parsed = JSON.parse(answer);
+        if (isRecord(parsed)) {
+            for (const key of ['error', 'message', 'detail', 'title']) {
+                const value = parsed[key];
+                if (typeof value === 'string' && value.trim())
+                    return value;
+                if (isRecord(value) && typeof value.message === 'string')
+                    return value.message;
+            }
+        }
+    }
+    catch {
+    }
+    return answer.trim().slice(0, 300) || 'no detail';
+}
+function sanitize(name) {
+    return name
+        .replace(/[^A-Za-z0-9_-]+/g, '_')
+        .replace(/_+/g, '_')
+        .replace(/^_+|_+$/g, '');
+}
+function uniqueName(name, used) {
+    const base = name.slice(0, 64);
+    let candidate = base;
+    for (let index = 2; used.has(candidate); index += 1) {
+        const suffix = `_${index}`;
+        candidate = `${base.slice(0, 64 - suffix.length)}${suffix}`;
+    }
+    used.add(candidate);
+    return candidate;
+}
+function slug(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '');
+}
+function text(value) {
+    return value !== null && typeof value === 'object' ? JSON.stringify(value) : String(value);
+}
+function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function messageOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/evaluate/emulate.d.ts

@@ -1,8 +1,3 @@
-/**
- * Tools emulated by a model, for evaluations that must not touch the world: an agent that would
- * charge a card or write to a production database runs against a model playing each tool instead,
- * with nothing sent, and the emulator's answers recorded so a run replays them exactly.
- */
 import type { CompletionRequest, ToolDefinition } from '../types/messages.js';
 import type { NexusResponse } from '../types/response.js';
 /** The model that plays the tools: the client, or anything with its `complete()`. */
@@ -89,6 +84,76 @@ export interface EmulatedToolsOptions {
     maxTokens?: number;
     /** Called after each call, with who answered it. */
     onCall?: (call: EmulatedToolCall) => void;
+    /**
+     * The world the tools act in, held across every run that shares it: each call made in it, and the
+     * changes they made, so an order one example refunds is refunded for the next. The emulator is
+     * told every change, and the latest `history` calls; its scenario is the tools' when they name
+     * none. Share one world among runs made one at a time, in order; or start each run from a
+     * `fork()`, or from a `snapshot()` read back.
+     */
+    world?: EmulatedWorld;
+    /**
+     * Tools that change nothing, by name, beside those whose every capability reads: `filesystem:read`,
+     * `mcp.read`, or `api.read`, or none at all. A world counts what any other tool did, when it did
+     * not fail, as a change.
+     */
+    reads?: readonly string[];
+}
+/** One call made in an emulated world. */
+export interface EmulatedWorldCall extends EmulatedResult {
+    /** Whether it changed the world: it succeeded, and its tool does more than read. */
+    changed: boolean;
+}
+/** An emulated world as plain JSON, to write to a file and start runs from. */
+export interface EmulatedWorldSnapshot {
+    /** The world the tools act on, before any call. */
+    scenario?: string;
+    /** Every call made in it, oldest first. */
+    calls: EmulatedWorldCall[];
+}
+/**
+ * A world emulated tools act in, kept across the runs of an evaluation: the scenario it started
+ * from, and every call made in it, so what one run changes is there for the next, as on a staging
+ * copy of a service. `snapshot()` writes it as JSON, `new EmulatedWorld(snapshot)` reads it back,
+ * and `fork()` starts a world from this one's present whose changes stay its own.
+ *
+ * ```ts
+ * const world = new EmulatedWorld({ scenario: 'Order 1042 is delivered and cost $59.' });
+ * await evaluate(
+ *   (inputs) => agentOver(emulatedTools(shop.tools, { client: ai, results: shop.results, world })).invoke(agentInput(inputs.request)),
+ *   dataset,
+ *   evaluators,
+ *   { concurrency: 1 },
+ * );
+ * ```
+ */
+export declare class EmulatedWorld {
+    /** The world the tools act on, before any call. */
+    readonly scenario?: string;
+    private readonly log;
+    constructor(start?: {
+        scenario?: string;
+        calls?: readonly EmulatedWorldCall[];
+    });
+    /** Every call made in it, oldest first. */
+    get calls(): readonly EmulatedWorldCall[];
+    /** The calls that changed it, oldest first. */
+    get changes(): readonly EmulatedWorldCall[];
+    /** Records a call made in it, and whether it changed it. */
+    record(call: EmulatedResult, changed: boolean): void;
+    /**
+     * A digest of its scenario and every change made in it, which the keys of its calls' answers carry:
+     * two worlds that started alike and changed alike have the same digest, however much was read.
+     */
+    digest(): string;
+    /** How many calls with these arguments it has seen since its last change. */
+    ordinal(tool: string, args: Record<string, unknown>): number;
+    /** It as plain JSON, to write to a file and start runs from. */
+    snapshot(): EmulatedWorldSnapshot;
+    /** A world starting from this one's present, whose changes stay its own. */
+    fork(): EmulatedWorld;
+    /** The same as `snapshot()`, so `JSON.stringify(world)` writes it. */
+    toJSON(): EmulatedWorldSnapshot;
 }
 /** Raised when a call cannot be emulated: no kept answer in `replay`, or no valid answer from the model. */
 export declare class EmulationError extends Error {
@@ -121,13 +186,15 @@ export declare class MemoryEmulationStore implements EmulationStore {
 }
 /**
  * The key of a call: its tool, its arguments, its scenario, and which call with those arguments it
- * is, so an order looked up before and after a refund keeps both answers.
+ * is, so an order looked up before and after a refund keeps both answers. In a world, also the
+ * world's `digest()`, so the same call answers again only after the same changes.
  */
 export declare function emulationKey(call: {
     tool: string;
     args: Record<string, unknown>;
     scenario?: string;
     ordinal?: number;
+    world?: string;
 }): string;
 /**
  * Tools that look the same to the agent, by name, description, parameters, and capabilities, but are